From 9d9d9a97b3050aa0dc7261be5edf196a1668cc03 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Fri, 28 Aug 2026 10:47:04 +0200 Subject: [PATCH 01/53] Add antsibull-changelog configuration --- .gitignore | 3 ++ changelogs/config.yaml | 40 ++++++++++++++++++++++++ changelogs/fragments/.keep | 0 changelogs/fragments/release_summary.yml | 8 +++++ 4 files changed, 51 insertions(+) create mode 100644 changelogs/config.yaml create mode 100644 changelogs/fragments/.keep create mode 100644 changelogs/fragments/release_summary.yml diff --git a/.gitignore b/.gitignore index 39a8b804..0318debd 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ __pycache__* .vscode .venv .fuse_hidden* + +# antsibull-changelog +changelogs/.plugin-cache.yaml diff --git a/changelogs/config.yaml b/changelogs/config.yaml new file mode 100644 index 00000000..c0c7efdc --- /dev/null +++ b/changelogs/config.yaml @@ -0,0 +1,40 @@ +--- +# Configuration for antsibull-changelog. +# See https://docs.ansible.com/projects/antsibull-changelog/changelog-configuration/ +# +# Contributors add one fragment per pull request under changelogs/fragments/. +# `antsibull-changelog release --version ` folds all fragments into +# changelogs/changelog.yaml, renders CHANGELOG.md and removes the fragments. + +changes_file: changelog.yaml +changes_format: combined +ignore_other_fragment_extensions: true +keep_fragments: false +mention_ancestor: true +notes_dir: fragments +output: + - file: CHANGELOG.md + format: md +prelude_name: release_summary +prelude_title: Release Summary +sanitize_changelog: true +sections: + - - major_changes + - Major Changes + - - minor_changes + - Minor Changes + - - breaking_changes + - Breaking Changes / Porting Guide + - - deprecated_features + - Deprecated Features + - - removed_features + - Removed Features (previously deprecated) + - - security_fixes + - Security Fixes + - - bugfixes + - Bugfixes + - - known_issues + - Known Issues +title: Netways.Elasticstack +trivial_section_name: trivial +use_fqcn: true diff --git a/changelogs/fragments/.keep b/changelogs/fragments/.keep new file mode 100644 index 00000000..e69de29b diff --git a/changelogs/fragments/release_summary.yml b/changelogs/fragments/release_summary.yml new file mode 100644 index 00000000..2b0f7948 --- /dev/null +++ b/changelogs/fragments/release_summary.yml @@ -0,0 +1,8 @@ +release_summary: | + First tagged release. This is the current state of the collection, made + referenceable so it can be pinned by version instead of by commit hash. + + It is not a cleaned up release. Variable names are not consistent yet and + some of the behaviour in the elasticsearch role is still being reworked, + in particular around upgrades. Breaking changes are collected for 1.0.0, + so pinning 0.1.0 keeps you unaffected until you move on deliberately. From d9def25a315e6b51b072e2a4e0ec134ba6473b57 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Fri, 28 Aug 2026 10:48:16 +0200 Subject: [PATCH 02/53] Add workflow to publish the collection to Ansible Galaxy --- .github/workflows/publish.yml | 51 +++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..05ce30b4 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,51 @@ +--- +# Publishes a tagged version of this collection to Ansible Galaxy. +# +# This workflow is triggered manually and never automatically, so that a tag can +# be created and inspected before an immutable version is uploaded to Galaxy. +# The tag has to exist before the workflow is started, because the build happens +# from the checked out tag and galaxy.yml at that tag determines the version. +# +# Requires the repository or organization secret GALAXY_API_KEY, holding an API +# token of an account that owns the "netways" namespace on Ansible Galaxy. + +name: publish + +on: + workflow_dispatch: + inputs: + tag: + description: 'Existing tag to build and publish, for example 0.1.0' + required: true + type: string + +permissions: + contents: read + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - name: Check out the tag + uses: actions/checkout@v7 + with: + ref: ${{ inputs.tag }} + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.11" + + - name: Install ansible-core + run: pip install ansible-core + + - name: Show the version that will be published + run: grep '^version:' galaxy.yml + + - name: Build the collection + run: ansible-galaxy collection build -vvv + + - name: Publish to Ansible Galaxy + env: + GALAXY_API_KEY: ${{ secrets.GALAXY_API_KEY }} + run: ansible-galaxy collection publish --token "$GALAXY_API_KEY" ./netways-elasticstack-*.tar.gz From ab130655ab508f41160f4bace6db8b0683ce44a7 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Fri, 28 Aug 2026 10:48:51 +0200 Subject: [PATCH 03/53] Rewrite the release documentation --- RELEASE.md | 114 ++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 100 insertions(+), 14 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 048f040c..7ff48766 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,35 +1,121 @@ # Release Workflow -*NOTE*: This is a work in progress. While we are approaching our first release, we're collecting valuable information about tasks and put them here. +How to build, tag and publish a new version of this collection. -## Preparations +## Prerequisites -### WIP note +* `antsibull-changelog` installed locally: -Remove the note above saying that this is a work and progress before the first release. And remove this section. + ```bash + pip install antsibull-changelog + ``` -### Lint +* The repository or organization secret `GALAXY_API_KEY`, holding an API token of an + account that owns the `netways` namespace on Ansible Galaxy. The token is created in + the account settings on . The same token is used for + `netways.icinga`, so whoever releases that collection can tell you where it lives. +* Permission to run workflows in this repository, or the token itself if you intend to + publish from your own machine. See step 6. -Make sure to have one last pull request to remove as much lint as possible. Same goes for deprecation warnings, linter exceptions etc. +Galaxy versions are **immutable**. Once a version is uploaded it cannot be replaced or +removed, only superseded by a higher version. Check the built archive before publishing. -### Authors +## 1. Prepare the code + +Make sure the last pull requests before the release removed as much lint as possible. +The same goes for deprecation warnings and linter exceptions. Update the [.mailmap](.mailmap) and [AUTHORS](AUTHORS) files: -``` +```bash git log --use-mailmap | grep '^Author:' | cut -f2- -d' ' | sort | uniq > AUTHORS ``` -## Tag +## 2. Set the version + +The version of this collection as seen by Ansible Galaxy comes from **galaxy.yml**, not +from the git tag. Set it to the version you are about to release. Use semantic +versioning and do not use a `v` prefix. + +## 3. Write the release summary + +This becomes the introductory paragraph of the release in the changelog. + +`changelogs/fragments/release_summary.yml`: + +```yaml +release_summary: | + Summary text for this release, for example "Bugfix release". +``` + +## 4. Generate the changelog + +Every pull request adds its own fragment under `changelogs/fragments/`. This step folds +all of them into `changelogs/changelog.yaml`, renders `CHANGELOG.md` and deletes the +fragments. + +```bash +antsibull-changelog lint +antsibull-changelog release --version +``` + +Steps 1 to 4 change files in the repository. `main` requires an approving review, so open +a pull request with the version bump, the updated `AUTHORS` and the generated changelog. +Do not skip this and push directly, even though repository admins are technically able +to. -Tag the version. Don't use `v` in the name. Use semantic versioning. +## 5. Tag the version -e.g. +Tag the merge commit on `main`. No `v` prefix, semantic versioning. +```bash +git switch main +git pull +git tag -a -m "" +git push origin ``` -git tag -as 7.6.4 + +If you cannot push tags, create the GitHub release first and let GitHub create the tag +for you. In that case do step 7 before step 6, because the workflow in step 6 needs the +tag to exist. + +## 6. Publish to Ansible Galaxy + +There is a workflow for this. It is triggered **manually only**, so that a tag can be +inspected before an immutable version reaches Galaxy. + +**Actions** -> **publish** -> **Run workflow**, then enter the tag from step 5. + +The workflow checks out that tag, builds the collection and publishes it. The version it +publishes comes from `galaxy.yml` at that tag. + +If you are not allowed to run workflows in this repository, do the same from your own +machine with the token: + +```bash +ansible-galaxy collection build -vvv +ansible-galaxy collection publish --token netways-elasticstack-.tar.gz ``` -## Release +`-vvv` lists the files that were skipped, so you can see whether `build_ignore` in +`galaxy.yml` did its job. Publishing may print errors and still have worked. Check + to confirm. + +## 7. Create the release on GitHub + +[Draft a new release](https://github.com/NETWAYS/ansible-collection-elasticstack/releases/new): + +* choose the tag from step 5 +* use the version as the title +* paste this version's section from `CHANGELOG.md` as the release notes +* credit contributors by name where it fits, for example behind the issue they reported + +Attaching the built archive to the release is optional. It lets people install the +collection without Galaxy and without git: + +```bash +ansible-galaxy collection install netways-elasticstack-.tar.gz +``` -Make the tag a release on GitHub. +If you published through the workflow in step 6, no archive exists on your machine. Build +one locally if you want to attach it. From 47716443d6ba171755aaac7c3540e03c349c5f40 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Fri, 28 Aug 2026 12:16:42 +0200 Subject: [PATCH 04/53] Keep local tooling and CI files out of the built collection --- .gitignore | 2 ++ galaxy.yml | 22 +++++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 0318debd..a5e15080 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ __pycache__* # antsibull-changelog changelogs/.plugin-cache.yaml +.claude +results-dir diff --git a/galaxy.yml b/galaxy.yml index 2b5a0a72..5c5b4622 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -29,13 +29,29 @@ documentation: https://github.com/netways/ansible-collection-elasticstack/README homepage: https://www.netways.de issues: https://github.com/netways/ansible-collection-elasticstack/issues build_ignore: + # CI, tests and local tooling - .github - .cache - molecule + - results-dir + - .config + - .yamllint + - makefile + - 'requirements*.txt' + - .docsmith + # repository housekeeping, not part of the collection - .gitignore - - daemon.json - .mailmap - - '*.tar.gz' + - daemon.json + - RELEASE.md + # changelog working directory - the rendered CHANGELOG.md is shipped + - changelogs + # local editor and tool directories, never part of a release + - .claude + - .vscode + - .idea + - __pycache__ + - .ansible - 'venv*' - - .docsmith + - '*.tar.gz' # Minimum Ansible version required: 2.14, maximum supported: 2.20 From 7ab88298333d228a11747ad56382ed0919f675f3 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Fri, 28 Aug 2026 12:16:58 +0200 Subject: [PATCH 05/53] Attach the built archive to the release when publishing --- .github/workflows/publish.yml | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 05ce30b4..00a74388 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,13 +1,15 @@ --- -# Publishes a tagged version of this collection to Ansible Galaxy. +# Publishes a tagged version of this collection to Ansible Galaxy and attaches the +# built archive to the matching GitHub release. # -# This workflow is triggered manually and never automatically, so that a tag can -# be created and inspected before an immutable version is uploaded to Galaxy. -# The tag has to exist before the workflow is started, because the build happens -# from the checked out tag and galaxy.yml at that tag determines the version. +# Triggered manually and never automatically, so that a tag can be inspected before an +# immutable version reaches Galaxy. Both the tag and the release have to exist before +# the workflow is started: the build happens from the checked out tag, galaxy.yml at +# that tag determines the published version, and the archive is uploaded to the release +# carrying the same name as the tag. # -# Requires the repository or organization secret GALAXY_API_KEY, holding an API -# token of an account that owns the "netways" namespace on Ansible Galaxy. +# Requires the repository or organization secret GALAXY_API_KEY, holding an API token of +# an account that owns the "netways" namespace on Ansible Galaxy. name: publish @@ -20,7 +22,7 @@ on: type: string permissions: - contents: read + contents: write jobs: publish: @@ -31,7 +33,7 @@ jobs: with: ref: ${{ inputs.tag }} - - name: Set up Python + - name: Set up Python 3.11 uses: actions/setup-python@v7 with: python-version: "3.11" @@ -49,3 +51,8 @@ jobs: env: GALAXY_API_KEY: ${{ secrets.GALAXY_API_KEY }} run: ansible-galaxy collection publish --token "$GALAXY_API_KEY" ./netways-elasticstack-*.tar.gz + + - name: Attach the archive to the release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh release upload "${{ inputs.tag }}" ./netways-elasticstack-*.tar.gz --clobber From 2cff45e41f9628cd901ecc6b867b6fa2f827fa91 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Fri, 28 Aug 2026 12:17:30 +0200 Subject: [PATCH 06/53] Require a changelog fragment in every pull request --- .github/workflows/test_changelog.yml | 67 ++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .github/workflows/test_changelog.yml diff --git a/.github/workflows/test_changelog.yml b/.github/workflows/test_changelog.yml new file mode 100644 index 00000000..48022d80 --- /dev/null +++ b/.github/workflows/test_changelog.yml @@ -0,0 +1,67 @@ +--- +# Every pull request has to bring a changelog fragment, so that the changelog can be +# generated at release time instead of being reconstructed from the commit history. +# +# A change that is not worth mentioning to users still needs a fragment, but its entry +# goes into the "trivial" section, which antsibull-changelog does not render. See +# CONTRIBUTING.md. + +name: Test Changelog + +on: + workflow_dispatch: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + fragment: + # Dependabot only bumps action and dependency versions, which needs no fragment. + if: >- + github.event.pull_request.draft == false + && github.actor != 'dependabot[bot]' + runs-on: ubuntu-latest + steps: + - name: Check out the codebase + uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Set up Python 3.11 + uses: actions/setup-python@v7 + with: + python-version: "3.11" + + - name: Install antsibull-changelog + run: pip install antsibull-changelog + + - name: A changelog fragment is present + env: + BASE_REF: ${{ github.event.pull_request.base.ref }} + run: | + if [ -z "$BASE_REF" ]; then + echo "Not a pull request, nothing to compare against." + exit 0 + fi + git fetch --no-tags origin "$BASE_REF" + changed=$(git diff --name-only --diff-filter=AM \ + "origin/$BASE_REF...HEAD" -- changelogs/fragments/ \ + | grep -v '^changelogs/fragments/\.keep$' || true) + if [ -z "$changed" ]; then + echo "::error::This pull request adds no changelog fragment." + echo "Add a file under changelogs/fragments/ describing the change." + echo "If the change is not relevant for users, put the entry in the" + echo "'trivial' section. See CONTRIBUTING.md." + exit 1 + fi + echo "Found:" + echo "$changed" + + - name: Fragments are valid + run: antsibull-changelog lint From f001ca388be679bd52af6a1802b1882ae71e2aa5 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Fri, 28 Aug 2026 12:18:16 +0200 Subject: [PATCH 07/53] Add contributing guide with the changelog fragment rules --- CONTRIBUTING.md | 96 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..898fb299 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,96 @@ +# Contributing + +Every kind of contribution is welcome. This page describes what we need from a pull +request so that reviewing it does not take longer than writing it. + +## Before you build something + +There is no written scope for this collection. It gets what is useful and what helps from +an automation point of view, and that is easier to answer per case than in the abstract. +Please open an issue for anything bigger than a bug fix and we will say yes or no before +you invest the work. + +For now all pull requests go against `main`. + +## What a pull request needs + +* **One topic per pull request.** Touching several roles is fine when it is the same + change in each of them, for example the same fix applied to beats, logstash, kibana and + elasticsearch. What is hard to review is a pull request that bundles unrelated changes, + because then a problem in one part holds up all the others. +* **A description that says precisely what changes and why.** If the diff is the only + place where the purpose can be found, the review is already too slow. +* **Evidence that it was tested**, against the new feature or fix and against what sits + next to it. Please say which Elastic Stack release and which ansible-core version you + tested with. +* **New or changed variables** belong in the role's `meta/argument_specs.yml` and in its + README table, otherwise the documentation drifts apart from the code. See the + Contributing section of the main README for how the table is generated. +* **Comments on anything that is not obvious.** Six months later nobody remembers what a + clever condition was for, and then it gets analysed again from scratch. +* **A changelog fragment.** See below. +* If a change **claims something about how Elastic behaves**, please link the Elastic + documentation for it. + +Contributions written with the help of AI tools are welcome under the same rules. One +addition: be able to explain any line if we ask. That is the whole difference between a +vetted change and something nobody understands. + +## Changelog fragments + +The changelog is not generated from commit messages. It is assembled from small files +that each pull request brings along, so that the description is written by the person who +knows what changed rather than reconstructed months later at release time. + +Add one file per pull request under `changelogs/fragments/`. Name it after the issue or +the change, for example `546_rolling_upgrade.yml`. Never edit an existing fragment, so +that every entry can be traced back to the pull request that introduced it. + +```yaml +# changelogs/fragments/546_rolling_upgrade.yml +bugfixes: + - Raising ``elasticstack_version`` no longer upgrades all Elasticsearch nodes at once + (https://github.com/NETWAYS/ansible-collection-elasticstack/issues/546). +``` + +Available sections, one or more per file: + +| Section | Use it for | +|---|---| +| `major_changes` | Changes that affect how the collection is used as a whole | +| `minor_changes` | New variables, new options, improved behaviour | +| `breaking_changes` | Anything users have to adapt to, for example a renamed variable | +| `deprecated_features` | Something that still works but will be removed | +| `removed_features` | Something previously deprecated that is now gone | +| `security_fixes` | Fixes with a security impact | +| `bugfixes` | Wrong or broken behaviour that now works | +| `known_issues` | Problems shipped with the release on purpose | +| `trivial` | Not rendered into the changelog. See below | + +The pull request pipeline fails when no fragment was added. + +### Changes that do not belong in the changelog + +Some changes are not worth telling users about. They still need a fragment, but the entry +goes into the `trivial` section, which is not rendered into the changelog: + +```yaml +# changelogs/fragments/fix_comment_typo.yml +trivial: + - Fixed a typo in a comment in the elasticsearch role. +``` + +Typical cases for `trivial`: + +* documentation wording, typos and formatting +* CI and workflow changes +* test and Molecule scenario changes +* refactoring with no visible effect on behaviour +* dependency bumps of test tooling + +If you are unsure whether something is trivial, ask in the pull request. It is easier to +move an entry than to notice a missing one after the release. + +## Releasing + +Maintainers only. See [RELEASE.md](RELEASE.md). From dd2afb9dc793c44586e177e8298a1815ba9a483d Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Fri, 28 Aug 2026 12:18:58 +0200 Subject: [PATCH 08/53] Document the release procedure end to end --- RELEASE.md | 108 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 63 insertions(+), 45 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 7ff48766..c40e18cf 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,6 +1,23 @@ # Release Workflow -How to build, tag and publish a new version of this collection. +How to build, tag and publish a new version of this collection. Maintainers only. + +## What a release produces + +Two different things, and it helps to keep them apart: + +**The git tag** is only a name pointing at a commit. It contains nothing itself, it makes +a state of the source tree findable again. It is what `version:` resolves to when someone +installs the collection straight from git, and it is what the publish workflow checks out. + +**The archive** `netways-elasticstack-.tar.gz` is the installable product built +from that state. `ansible-galaxy collection build` packs the collection and adds +`MANIFEST.json` and `FILES.json`, which hold the metadata from `galaxy.yml` and a checksum +for every file. This archive is what Galaxy stores and serves, and what +`ansible-galaxy collection install` unpacks. Whatever `build_ignore` in `galaxy.yml` lists +is left out of it. + +The version comes from `galaxy.yml`, not from the tag. Both have to say the same thing. ## Prerequisites @@ -11,21 +28,23 @@ How to build, tag and publish a new version of this collection. ``` * The repository or organization secret `GALAXY_API_KEY`, holding an API token of an - account that owns the `netways` namespace on Ansible Galaxy. The token is created in - the account settings on . The same token is used for - `netways.icinga`, so whoever releases that collection can tell you where it lives. + account that owns the `netways` namespace on Ansible Galaxy. Tokens are created in the + account settings on . * Permission to run workflows in this repository, or the token itself if you intend to publish from your own machine. See step 6. Galaxy versions are **immutable**. Once a version is uploaded it cannot be replaced or -removed, only superseded by a higher version. Check the built archive before publishing. +removed, only superseded by a higher version. ## 1. Prepare the code -Make sure the last pull requests before the release removed as much lint as possible. -The same goes for deprecation warnings and linter exceptions. +Make sure the last pull requests before the release removed as much lint as possible. The +same goes for deprecation warnings and linter exceptions. -Update the [.mailmap](.mailmap) and [AUTHORS](AUTHORS) files: +Update the [AUTHORS](AUTHORS) file. It is generated from the git history and lists +everyone who contributed. `--use-mailmap` folds the alternative addresses in +[.mailmap](.mailmap) into one entry per person, so add a line there first if someone shows +up twice: ```bash git log --use-mailmap | grep '^Author:' | cut -f2- -d' ' | sort | uniq > AUTHORS @@ -33,9 +52,8 @@ git log --use-mailmap | grep '^Author:' | cut -f2- -d' ' | sort | uniq > AUTHORS ## 2. Set the version -The version of this collection as seen by Ansible Galaxy comes from **galaxy.yml**, not -from the git tag. Set it to the version you are about to release. Use semantic -versioning and do not use a `v` prefix. +Set `version:` in **galaxy.yml** to the version you are about to release. Semantic +versioning, no `v` prefix. ## 3. Write the release summary @@ -50,8 +68,8 @@ release_summary: | ## 4. Generate the changelog -Every pull request adds its own fragment under `changelogs/fragments/`. This step folds -all of them into `changelogs/changelog.yaml`, renders `CHANGELOG.md` and deletes the +Every pull request brings its own fragment under `changelogs/fragments/`. This folds all +of them into `changelogs/changelog.yaml`, renders `CHANGELOG.md` and deletes the fragments. ```bash @@ -61,8 +79,7 @@ antsibull-changelog release --version Steps 1 to 4 change files in the repository. `main` requires an approving review, so open a pull request with the version bump, the updated `AUTHORS` and the generated changelog. -Do not skip this and push directly, even though repository admins are technically able -to. +Do not push this directly, even though repository admins are technically able to. ## 5. Tag the version @@ -75,47 +92,48 @@ git tag -a -m "" git push origin ``` -If you cannot push tags, create the GitHub release first and let GitHub create the tag -for you. In that case do step 7 before step 6, because the workflow in step 6 needs the -tag to exist. +If you cannot push tags, skip this and let GitHub create the tag in step 6 instead. -## 6. Publish to Ansible Galaxy +## 6. Create the release on GitHub -There is a workflow for this. It is triggered **manually only**, so that a tag can be -inspected before an immutable version reaches Galaxy. +[Draft a new release](https://github.com/NETWAYS/ansible-collection-elasticstack/releases/new): -**Actions** -> **publish** -> **Run workflow**, then enter the tag from step 5. +* choose the tag from step 5, or enter the version and let GitHub create the tag now +* use the version as the title +* paste this version's section from `CHANGELOG.md` as the release notes +* credit people by name where a change came from outside: the author of the pull request, + and the reporter of an issue where the report was the contribution -The workflow checks out that tag, builds the collection and publishes it. The version it -publishes comes from `galaxy.yml` at that tag. +The release has to exist before step 7, because the archive is uploaded to it. -If you are not allowed to run workflows in this repository, do the same from your own -machine with the token: +## 7. Publish to Ansible Galaxy -```bash -ansible-galaxy collection build -vvv -ansible-galaxy collection publish --token netways-elasticstack-.tar.gz -``` - -`-vvv` lists the files that were skipped, so you can see whether `build_ignore` in -`galaxy.yml` did its job. Publishing may print errors and still have worked. Check - to confirm. +**Actions** -> **publish** -> **Run workflow**, then enter the tag. -## 7. Create the release on GitHub +The workflow checks out that tag, builds the collection, publishes it to Galaxy and +attaches the archive to the release from step 6. It is triggered manually and never +automatically, so that a tag can be inspected before an immutable version reaches Galaxy. -[Draft a new release](https://github.com/NETWAYS/ansible-collection-elasticstack/releases/new): +Afterwards confirm on + that the version +arrived. Publishing can print errors and still have worked. -* choose the tag from step 5 -* use the version as the title -* paste this version's section from `CHANGELOG.md` as the release notes -* credit contributors by name where it fits, for example behind the issue they reported +### If you cannot run workflows -Attaching the built archive to the release is optional. It lets people install the -collection without Galaxy and without git: +Do the same from your own machine. **Clone the tag into a fresh directory first.** +`ansible-galaxy collection build` packs the working directory and not the git tree, so +untracked local files end up in the archive unless `build_ignore` happens to cover them. ```bash -ansible-galaxy collection install netways-elasticstack-.tar.gz +git clone --branch git@github.com:NETWAYS/ansible-collection-elasticstack.git release_ +cd release_ +ansible-galaxy collection build -vvv +ansible-galaxy collection publish --token netways-elasticstack-.tar.gz ``` -If you published through the workflow in step 6, no archive exists on your machine. Build -one locally if you want to attach it. +`-vvv` lists the files that were skipped, so you can see whether `build_ignore` did its +job. Then attach the archive to the release by hand, because nothing uploaded it for you: + +```bash +gh release upload netways-elasticstack-.tar.gz +``` From 0d4e9f693bedd5b1642973f5afd38397ee590d8c Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Fri, 28 Aug 2026 12:19:15 +0200 Subject: [PATCH 09/53] Document pinning a released version and point to CONTRIBUTING --- README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.md b/README.md index c296052a..875a0e64 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,19 @@ You can easily install the collection with the `ansible-galaxy` command. ansible-galaxy collection install git+https://github.com/netways/ansible-collection-elasticstack.git ``` +To pin a released version instead of the current state of `main`, use a `requirements.yml`: + +```yaml +collections: + - name: https://github.com/NETWAYS/ansible-collection-elasticstack.git + type: git + version: 0.1.0 +``` + +```bash +ansible-galaxy collection install -r requirements.yml +``` + Or if you are using Tower or AWX add the collection to your requirements file. ``` @@ -230,6 +243,10 @@ The execution order of the roles is important! (see below) Every kind of contribution is very welcome. Open [issues](https://github.com/NETWAYS/ansible-collection-elasticstack/issues) or provide [pull requests](https://github.com/NETWAYS/ansible-collection-elasticstack/pulls). +Please read [CONTRIBUTING.md](CONTRIBUTING.md) before opening a pull request. It describes +what a pull request needs, including the changelog fragment that every pull request has to +bring along. + For now we open pull requests against `main`. We are planning to introduce dedicated branches to support older versions without breaking changes. Since we don't need them for now, please check back with this section because when we decided on how to proceed, you will find the information here. For now `main` always has the newest changes and if you want a stable version, please use the newest release. ### Documentation for role variables From 9f00ec049e589da514c68ffda3b3d37e957ee2e7 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Fri, 28 Aug 2026 12:51:09 +0200 Subject: [PATCH 10/53] Do not ignore .claude so its changes stay in the history --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index a5e15080..3bb36d13 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,4 @@ __pycache__* # antsibull-changelog changelogs/.plugin-cache.yaml -.claude results-dir From 52069b44c18bda772106cb53ae1652ad912cc069 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Fri, 28 Aug 2026 12:58:28 +0200 Subject: [PATCH 11/53] Do not exclude .claude from the built collection --- galaxy.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/galaxy.yml b/galaxy.yml index 5c5b4622..bae92baa 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -47,7 +47,6 @@ build_ignore: # changelog working directory - the rendered CHANGELOG.md is shipped - changelogs # local editor and tool directories, never part of a release - - .claude - .vscode - .idea - __pycache__ From 184aee942ea7624ce698907438c6587466fa9435 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Fri, 28 Aug 2026 13:07:39 +0200 Subject: [PATCH 12/53] Name the organization as the collection author --- galaxy.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/galaxy.yml b/galaxy.yml index bae92baa..9624ad7b 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -3,11 +3,7 @@ name: elasticstack version: 0.1.0 readme: README.md authors: -- Afeef Ghannam -- Alexander Stoll -- David Okon - - Thilo Wening - - Thomas Widhalm + - NETWAYS GmbH (https://github.com/NETWAYS) description: This collection installs and configures components from the Elastic Stack. Every role can be used standalone or in combination with other roles. license: - GPL-3.0 From 50bb2f14035348d8f95bff19055ee4709a664787 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Fri, 28 Aug 2026 13:15:03 +0200 Subject: [PATCH 13/53] Drop the AUTHORS file and read contributors from the history instead --- AUTHORS | 20 -------------------- RELEASE.md | 9 ++++----- 2 files changed, 4 insertions(+), 25 deletions(-) delete mode 100644 AUTHORS diff --git a/AUTHORS b/AUTHORS deleted file mode 100644 index 4679c3c8..00000000 --- a/AUTHORS +++ /dev/null @@ -1,20 +0,0 @@ -Afeef Ghannam -Alexander Stoll -Daniel Neuberger -Daniel Patrick -dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> -Ivar Eriksen -Klaus Zerwes -Lorenz Kästle -Lucinda Siemer -Marco Hald -Markus Frosch -Markus Opolka -Matthias Döhler -Nathaniel Donahue -Patrick Dolinic -Patrick Steinmetz <132431614+psteinmetzcid@users.noreply.github.com> -Shittu Ayomide <86523250+lazyturd@users.noreply.github.com> -Thilo Wening -Thomas Widhalm -Tobias Bauriedel diff --git a/RELEASE.md b/RELEASE.md index c40e18cf..3ef5f087 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -41,13 +41,12 @@ removed, only superseded by a higher version. Make sure the last pull requests before the release removed as much lint as possible. The same goes for deprecation warnings and linter exceptions. -Update the [AUTHORS](AUTHORS) file. It is generated from the git history and lists -everyone who contributed. `--use-mailmap` folds the alternative addresses in -[.mailmap](.mailmap) into one entry per person, so add a line there first if someone shows -up twice: +If you want to credit contributors in the release notes, this lists everyone with their +number of commits. [.mailmap](.mailmap) folds alternative addresses into one entry per +person, so add a line there first if somebody shows up twice: ```bash -git log --use-mailmap | grep '^Author:' | cut -f2- -d' ' | sort | uniq > AUTHORS +git shortlog -sn --use-mailmap ``` ## 2. Set the version From 4c18bf52cb0a6e711fa773d25e10970f53beea26 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Fri, 28 Aug 2026 13:15:37 +0200 Subject: [PATCH 14/53] Fix the command for listing contributors --- RELEASE.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index 3ef5f087..febbdf00 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -46,9 +46,11 @@ number of commits. [.mailmap](.mailmap) folds alternative addresses into one ent person, so add a line there first if somebody shows up twice: ```bash -git shortlog -sn --use-mailmap +git shortlog -sn HEAD ``` +`shortlog` applies the mailmap on its own, no extra option needed. + ## 2. Set the version Set `version:` in **galaxy.yml** to the version you are about to release. Semantic From 318810e11180f3400e70e7b6af4a5ebfa391abef Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Fri, 28 Aug 2026 13:46:51 +0200 Subject: [PATCH 15/53] Require a new fragment and reject changes to existing ones --- .github/workflows/test_changelog.yml | 25 +++++++++++++++++++------ CONTRIBUTING.md | 11 +++++++---- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test_changelog.yml b/.github/workflows/test_changelog.yml index 48022d80..5bf71517 100644 --- a/.github/workflows/test_changelog.yml +++ b/.github/workflows/test_changelog.yml @@ -50,18 +50,31 @@ jobs: exit 0 fi git fetch --no-tags origin "$BASE_REF" - changed=$(git diff --name-only --diff-filter=AM \ + added=$(git diff --name-only --diff-filter=A \ "origin/$BASE_REF...HEAD" -- changelogs/fragments/ \ | grep -v '^changelogs/fragments/\.keep$' || true) - if [ -z "$changed" ]; then - echo "::error::This pull request adds no changelog fragment." - echo "Add a file under changelogs/fragments/ describing the change." + if [ -z "$added" ]; then + echo "::error::This pull request adds no new changelog fragment." + echo "Add a NEW file under changelogs/fragments/ describing the change." + echo "Do not put the entry into an existing fragment, so that every entry" + echo "stays traceable to the pull request that added it." echo "If the change is not relevant for users, put the entry in the" echo "'trivial' section. See CONTRIBUTING.md." exit 1 fi - echo "Found:" - echo "$changed" + echo "New fragments in this pull request:" + echo "$added" + + touched=$(git diff --name-only --diff-filter=MRD \ + "origin/$BASE_REF...HEAD" -- changelogs/fragments/ \ + | grep -v '^changelogs/fragments/\.keep$' || true) + if [ -n "$touched" ]; then + echo "::error::This pull request changes fragments it did not add:" + echo "$touched" + echo "Fragments belong to the pull request that created them. Add your own" + echo "instead of editing, renaming or deleting someone else's." + exit 1 + fi - name: Fragments are valid run: antsibull-changelog lint diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 898fb299..8320dcfa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,9 +42,10 @@ The changelog is not generated from commit messages. It is assembled from small that each pull request brings along, so that the description is written by the person who knows what changed rather than reconstructed months later at release time. -Add one file per pull request under `changelogs/fragments/`. Name it after the issue or -the change, for example `546_rolling_upgrade.yml`. Never edit an existing fragment, so -that every entry can be traced back to the pull request that introduced it. +Add one **new** file per pull request under `changelogs/fragments/`. Name it after the +issue or the change, for example `546_rolling_upgrade.yml`. Do not put your entry into an +existing fragment and do not rename or delete one, so that every entry stays traceable to +the pull request that introduced it. The pipeline rejects both. ```yaml # changelogs/fragments/546_rolling_upgrade.yml @@ -67,7 +68,9 @@ Available sections, one or more per file: | `known_issues` | Problems shipped with the release on purpose | | `trivial` | Not rendered into the changelog. See below | -The pull request pipeline fails when no fragment was added. +Fragments are consumed at release time: `antsibull-changelog release` folds them into the +changelog and deletes them, so the directory is empty again after every release. That is +why a new file is always the right answer, never an edit to an existing one. ### Changes that do not belong in the changelog From 281970db77efa367b34f4a67152275ef3198c273 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Tue, 1 Sep 2026 15:08:52 +0200 Subject: [PATCH 16/53] Keep dot files, contributing guide and release notes out of the archive --- .github/workflows/test_changelog.yml | 2 +- .mailmap | 11 +++++ RELEASE.md | 60 ++++++------------------ changelogs/fragments/release_summary.yml | 8 ---- galaxy.yml | 14 ++---- 5 files changed, 30 insertions(+), 65 deletions(-) delete mode 100644 changelogs/fragments/release_summary.yml diff --git a/.github/workflows/test_changelog.yml b/.github/workflows/test_changelog.yml index 5bf71517..8966ab43 100644 --- a/.github/workflows/test_changelog.yml +++ b/.github/workflows/test_changelog.yml @@ -39,7 +39,7 @@ jobs: python-version: "3.11" - name: Install antsibull-changelog - run: pip install antsibull-changelog + run: pip install "antsibull-changelog==0.35.1" - name: A changelog fragment is present env: diff --git a/.mailmap b/.mailmap index eead176c..e64fcc0b 100644 --- a/.mailmap +++ b/.mailmap @@ -1,3 +1,14 @@ +# Maps the different name and email combinations people committed with onto one +# canonical identity, so that git shortlog and git blame show one entry per person +# instead of one per address. Git applies this file automatically. +# +# Add a line when somebody shows up twice, for example after a change of employer or +# after a commit made through the GitHub web interface. Check the result with: +# git check-mailmap "Name
" +# +# Syntax: Proper Name +# See https://git-scm.com/docs/gitmailmap + Afeef Ghannam <39904920+afeefghannam89@users.noreply.github.com> Alexander Stoll <45196568+Crited@users.noreply.github.com> Daniel Neuberger diff --git a/RELEASE.md b/RELEASE.md index febbdf00..325beec4 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -2,29 +2,13 @@ How to build, tag and publish a new version of this collection. Maintainers only. -## What a release produces - -Two different things, and it helps to keep them apart: - -**The git tag** is only a name pointing at a commit. It contains nothing itself, it makes -a state of the source tree findable again. It is what `version:` resolves to when someone -installs the collection straight from git, and it is what the publish workflow checks out. - -**The archive** `netways-elasticstack-.tar.gz` is the installable product built -from that state. `ansible-galaxy collection build` packs the collection and adds -`MANIFEST.json` and `FILES.json`, which hold the metadata from `galaxy.yml` and a checksum -for every file. This archive is what Galaxy stores and serves, and what -`ansible-galaxy collection install` unpacks. Whatever `build_ignore` in `galaxy.yml` lists -is left out of it. - -The version comes from `galaxy.yml`, not from the tag. Both have to say the same thing. ## Prerequisites * `antsibull-changelog` installed locally: ```bash - pip install antsibull-changelog + pip install "antsibull-changelog==0.35.1" ``` * The repository or organization secret `GALAXY_API_KEY`, holding an API token of an @@ -36,27 +20,12 @@ The version comes from `galaxy.yml`, not from the tag. Both have to say the same Galaxy versions are **immutable**. Once a version is uploaded it cannot be replaced or removed, only superseded by a higher version. -## 1. Prepare the code - -Make sure the last pull requests before the release removed as much lint as possible. The -same goes for deprecation warnings and linter exceptions. - -If you want to credit contributors in the release notes, this lists everyone with their -number of commits. [.mailmap](.mailmap) folds alternative addresses into one entry per -person, so add a line there first if somebody shows up twice: - -```bash -git shortlog -sn HEAD -``` - -`shortlog` applies the mailmap on its own, no extra option needed. - -## 2. Set the version +## 1. Set the version Set `version:` in **galaxy.yml** to the version you are about to release. Semantic versioning, no `v` prefix. -## 3. Write the release summary +## 2. Write the release summary This becomes the introductory paragraph of the release in the changelog. @@ -67,7 +36,7 @@ release_summary: | Summary text for this release, for example "Bugfix release". ``` -## 4. Generate the changelog +## 3. Generate the changelog Every pull request brings its own fragment under `changelogs/fragments/`. This folds all of them into `changelogs/changelog.yaml`, renders `CHANGELOG.md` and deletes the @@ -78,11 +47,11 @@ antsibull-changelog lint antsibull-changelog release --version ``` -Steps 1 to 4 change files in the repository. `main` requires an approving review, so open -a pull request with the version bump, the updated `AUTHORS` and the generated changelog. +Steps 1 to 3 change files in the repository. `main` requires an approving review, so open +a pull request with the version bump and the generated changelog. Do not push this directly, even though repository admins are technically able to. -## 5. Tag the version +## 4. Tag the version Tag the merge commit on `main`. No `v` prefix, semantic versioning. @@ -93,26 +62,27 @@ git tag -a -m "" git push origin ``` -If you cannot push tags, skip this and let GitHub create the tag in step 6 instead. +If you cannot push tags, skip this and let GitHub create the tag in step 5 instead. -## 6. Create the release on GitHub +## 5. Create the release on GitHub [Draft a new release](https://github.com/NETWAYS/ansible-collection-elasticstack/releases/new): -* choose the tag from step 5, or enter the version and let GitHub create the tag now +* choose the tag from step 4, or enter the version and let GitHub create the tag now * use the version as the title * paste this version's section from `CHANGELOG.md` as the release notes * credit people by name where a change came from outside: the author of the pull request, - and the reporter of an issue where the report was the contribution + and the reporter of an issue where the report was the contribution. `git shortlog -sn HEAD` + lists everyone with their number of commits, folded per person through [.mailmap](.mailmap) -The release has to exist before step 7, because the archive is uploaded to it. +The release has to exist before step 6, because the archive is uploaded to it. -## 7. Publish to Ansible Galaxy +## 6. Publish to Ansible Galaxy **Actions** -> **publish** -> **Run workflow**, then enter the tag. The workflow checks out that tag, builds the collection, publishes it to Galaxy and -attaches the archive to the release from step 6. It is triggered manually and never +attaches the archive to the release from step 5. It is triggered manually and never automatically, so that a tag can be inspected before an immutable version reaches Galaxy. Afterwards confirm on diff --git a/changelogs/fragments/release_summary.yml b/changelogs/fragments/release_summary.yml deleted file mode 100644 index 2b0f7948..00000000 --- a/changelogs/fragments/release_summary.yml +++ /dev/null @@ -1,8 +0,0 @@ -release_summary: | - First tagged release. This is the current state of the collection, made - referenceable so it can be pinned by version instead of by commit hash. - - It is not a cleaned up release. Variable names are not consistent yet and - some of the behaviour in the elasticsearch role is still being reworked, - in particular around upgrades. Breaking changes are collected for 1.0.0, - so pinning 0.1.0 keeps you unaffected until you move on deliberately. diff --git a/galaxy.yml b/galaxy.yml index 9624ad7b..4ba0865e 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -26,27 +26,19 @@ homepage: https://www.netways.de issues: https://github.com/netways/ansible-collection-elasticstack/issues build_ignore: # CI, tests and local tooling - - .github - - .cache - molecule - results-dir - - .config - - .yamllint - makefile - 'requirements*.txt' - - .docsmith # repository housekeeping, not part of the collection - - .gitignore - - .mailmap - daemon.json - RELEASE.md + - CONTRIBUTING.md # changelog working directory - the rendered CHANGELOG.md is shipped - changelogs - # local editor and tool directories, never part of a release - - .vscode - - .idea + # no dot files or directories belong in a published collection + - '.*' - __pycache__ - - .ansible - 'venv*' - '*.tar.gz' # Minimum Ansible version required: 2.14, maximum supported: 2.20 From 3bd3488ebaa0d85858b6e29545030959016899a0 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 7 Sep 2026 12:20:53 +0200 Subject: [PATCH 17/53] Correct wrong version defaults, tested releases and the security note --- README.md | 17 ++++++++++------- docs/getting-started.md | 2 +- galaxy.yml | 4 ++-- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 875a0e64..08e27c65 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ You may want the following Ansible roles installed. There other ways to achieve ### Supported systems -We test the collection on the following Linux distributions. Each one with Elastic Stack 7 and 8. +We test the collection on the following Linux distributions, each one with Elastic Stack 8. * Rocky Linux 9 * Ubuntu 22.04 LTS @@ -113,7 +113,7 @@ The variable `elasticstack_no_log` can be set to `false` if you want to see the All packages are installed with `state: present`. When `elasticstack_version` is set to a version number (e.g. `7.17.2`), that exact version is installed and pinned. When it is left unset, the package is installed without a version, so a new setup gets the newest available version and existing installations are not upgraded automatically on later runs. -*elasticstack_release*: Major release version of Elastic stack to configure. (default: `7`) Make sure it corresponds to `elasticstack_version` if you set both. +*elasticstack_release*: Major release version of Elastic stack to configure. (default: `8`) Make sure it corresponds to `elasticstack_version` if you set both. For OSS version see `elasticstack_variant` below. @@ -139,11 +139,14 @@ The collection will make sure to upgrade Elasticsearch nodes one by one. ### Default Passwords -Default passwords can be seen during generation, or found later in `/usr/share/elasticsearch/initial_passwords` +Default passwords are generated on the CA host and can be seen during generation, or found +later in the file that `elasticstack_initial_passwords` points to (default +`/usr/share/elasticsearch/initial_passwords`). -To turn off security: - -`elasticstack_override_beats_tls: true` +Security is on by default and controlled by `elasticstack_security`. A related but much +narrower switch is `elasticstack_override_beats_tls`: it only stops TLS between Beats and +Logstash from being enabled automatically, and skips the Beats certificate generation. It does +not turn off security anywhere else. ### Requirements @@ -230,7 +233,7 @@ The execution order of the roles is important! (see below) ansible.builtin.package: name: rsyslog - name: Start rsyslog - ansible.builtin.service: + ansible.builtin.service: name: rsyslog state: started enabled: true diff --git a/docs/getting-started.md b/docs/getting-started.md index 320e757f..2d322e91 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -59,5 +59,5 @@ Variables Every role got its own set of variables, in addition a few variables are useable on any role. The collection-wide `elasticstack_*` variables are documented centrally with the [elasticstack role](../roles/elasticstack/README.md). Two common ones: -* *elasticstack_release*: Major release version of Elastic stack to configure. (default: `7`) +* *elasticstack_release*: Major release version of Elastic stack to configure. (default: `8`) * *elasticstack_variant*: Variant of the stack to install. Valid values: `elastic` or `oss`. (default: `elastic`) diff --git a/galaxy.yml b/galaxy.yml index 4ba0865e..481eb693 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -34,11 +34,11 @@ build_ignore: - daemon.json - RELEASE.md - CONTRIBUTING.md - # changelog working directory - the rendered CHANGELOG.md is shipped + # The changelog working directory, used to build the CHANGELOG.md in the repository. - changelogs # no dot files or directories belong in a published collection - '.*' - __pycache__ - 'venv*' - '*.tar.gz' -# Minimum Ansible version required: 2.14, maximum supported: 2.20 +# Minimum Ansible version is declared in meta/runtime.yml (requires_ansible). Maximum tested: 2.20 From 326f492cf69ea0104d216bf15062bbd9d1e2ad26 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 7 Sep 2026 12:21:03 +0200 Subject: [PATCH 18/53] Remove the outdated breaking change disclaimer --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 08e27c65..492cc656 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,5 @@ # Ansible Collection - netways.elasticstack -> [!CAUTION] -> The current main branch has some breaking changes. Be careful using it! Everything after commit 3d6673f6d526afeb0fd7ba382d067d76bd10bbd6 is affected! - This collection installs and manages the Elastic Stack. It provides roles for every component of the Stack. Furthermore, it is possible to differentiate between Enterprise or OSS releases. Every role is documented with all variables, please refer to the documentation found in **[Getting-Started](./docs/getting-started.md)** From cecb291f26c1d3d1e19e785fa47dcc421a250f3a Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 7 Sep 2026 12:21:08 +0200 Subject: [PATCH 19/53] Delete the role documentation stubs that only redirected --- docs/role-beats.md | 5 ----- docs/role-elasticsearch.md | 5 ----- docs/role-kibana.md | 5 ----- docs/role-logstash.md | 5 ----- docs/role-repos.md | 5 ----- 5 files changed, 25 deletions(-) delete mode 100644 docs/role-beats.md delete mode 100644 docs/role-elasticsearch.md delete mode 100644 docs/role-kibana.md delete mode 100644 docs/role-logstash.md delete mode 100644 docs/role-repos.md diff --git a/docs/role-beats.md b/docs/role-beats.md deleted file mode 100644 index 14df7b01..00000000 --- a/docs/role-beats.md +++ /dev/null @@ -1,5 +0,0 @@ -# Beats role - -The Beats role documentation now lives with the role itself: - -**➜ [roles/beats/README.md](../roles/beats/README.md)** diff --git a/docs/role-elasticsearch.md b/docs/role-elasticsearch.md deleted file mode 100644 index 0cf04638..00000000 --- a/docs/role-elasticsearch.md +++ /dev/null @@ -1,5 +0,0 @@ -# Elasticsearch role - -The Elasticsearch role documentation now lives with the role itself: - -**➜ [roles/elasticsearch/README.md](../roles/elasticsearch/README.md)** diff --git a/docs/role-kibana.md b/docs/role-kibana.md deleted file mode 100644 index bddfb880..00000000 --- a/docs/role-kibana.md +++ /dev/null @@ -1,5 +0,0 @@ -# Kibana role - -The Kibana role documentation now lives with the role itself: - -**➜ [roles/kibana/README.md](../roles/kibana/README.md)** diff --git a/docs/role-logstash.md b/docs/role-logstash.md deleted file mode 100644 index 15d1fb7f..00000000 --- a/docs/role-logstash.md +++ /dev/null @@ -1,5 +0,0 @@ -# Logstash role - -The Logstash role documentation now lives with the role itself: - -**➜ [roles/logstash/README.md](../roles/logstash/README.md)** diff --git a/docs/role-repos.md b/docs/role-repos.md deleted file mode 100644 index bce0c25a..00000000 --- a/docs/role-repos.md +++ /dev/null @@ -1,5 +0,0 @@ -# Repos role - -The Repos role documentation now lives with the role itself: - -**➜ [roles/repos/README.md](../roles/repos/README.md)** From 9c0a8d4303c87a9e48e1d0aa20a68cf5cc31591b Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 7 Sep 2026 12:21:22 +0200 Subject: [PATCH 20/53] State the pull request rules as requirements and trim the collection description --- CONTRIBUTING.md | 11 ++++------- galaxy.yml | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8320dcfa..5a15ee59 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,10 +16,10 @@ For now all pull requests go against `main`. * **One topic per pull request.** Touching several roles is fine when it is the same change in each of them, for example the same fix applied to beats, logstash, kibana and - elasticsearch. What is hard to review is a pull request that bundles unrelated changes, - because then a problem in one part holds up all the others. -* **A description that says precisely what changes and why.** If the diff is the only - place where the purpose can be found, the review is already too slow. + elasticsearch. A pull request that bundles unrelated changes gets sent back. +* **A description that says precisely what changes and why.** This is required, not + optional. A pull request whose purpose can only be reconstructed from the diff gets sent + back. * **Evidence that it was tested**, against the new feature or fix and against what sits next to it. Please say which Elastic Stack release and which ansible-core version you tested with. @@ -32,9 +32,6 @@ For now all pull requests go against `main`. * If a change **claims something about how Elastic behaves**, please link the Elastic documentation for it. -Contributions written with the help of AI tools are welcome under the same rules. One -addition: be able to explain any line if we ask. That is the whole difference between a -vetted change and something nobody understands. ## Changelog fragments diff --git a/galaxy.yml b/galaxy.yml index 481eb693..a5d3dc96 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -4,7 +4,7 @@ version: 0.1.0 readme: README.md authors: - NETWAYS GmbH (https://github.com/NETWAYS) -description: This collection installs and configures components from the Elastic Stack. Every role can be used standalone or in combination with other roles. +description: This collection installs and configures components from the Elastic Stack. license: - GPL-3.0 # min_ansible_version: "2.18" # max supported: 2.20 (commented out for schema compliance) From 5c2b6e68774c327eca23a5a7aad39e17ae40a9e6 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 7 Sep 2026 12:24:42 +0200 Subject: [PATCH 21/53] Restructure the documentation along the collection template --- CONTRIBUTING.md | 33 ++++- README.md | 316 +++++++++++++--------------------------- docs/01-requirements.md | 15 -- docs/getting-started.md | 166 +++++++++++++++------ docs/requirements.md | 27 ++++ docs/upgrades.md | 51 +++++++ 6 files changed, 329 insertions(+), 279 deletions(-) delete mode 100644 docs/01-requirements.md create mode 100644 docs/requirements.md create mode 100644 docs/upgrades.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5a15ee59..714b0285 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,8 +24,8 @@ For now all pull requests go against `main`. next to it. Please say which Elastic Stack release and which ansible-core version you tested with. * **New or changed variables** belong in the role's `meta/argument_specs.yml` and in its - README table, otherwise the documentation drifts apart from the code. See the - Contributing section of the main README for how the table is generated. + README table, otherwise the documentation drifts apart from the code. See "Documenting role + variables" below. * **Comments on anything that is not obvious.** Six months later nobody remembers what a clever condition was for, and then it gets analysed again from scratch. * **A changelog fragment.** See below. @@ -91,6 +91,35 @@ Typical cases for `trivial`: If you are unsure whether something is trivial, ask in the pull request. It is easier to move an entry than to notice a missing one after the release. +## Documenting role variables + +Role variables are documented from each role's `meta/argument_specs.yml`, which is the single +source of truth. When your pull request changes a role's variables: + +1. Update that role's `meta/argument_specs.yml` with type, default and description. +2. Regenerate the README variable table. Please do **not** edit it by hand. The table is + produced by [ansible-docsmith](https://github.com/foundata/ansible-docsmith), install it + with `pip install ansible-docsmith` if you do not have it: + + ```bash + ansible-docsmith generate roles/ --no-defaults --template-readme .docsmith/readme.md.j2 + ``` + +3. Commit the regenerated `README.md` together with your change. + +The `Test Documentation` workflow checks that each README matches its `argument_specs.yml` and +fails the pull request if they drift apart. + +## Testing + +Besides testing your change against a real setup, the repository has Molecule scenarios that +exercise the whole stack, and it checks for ansible-lint and yamllint errors. To run the linters +locally there is a `makefile`: + +```bash +make +``` + ## Releasing Maintainers only. See [RELEASE.md](RELEASE.md). diff --git a/README.md b/README.md index 492cc656..9227bd89 100644 --- a/README.md +++ b/README.md @@ -1,274 +1,158 @@ # Ansible Collection - netways.elasticstack -This collection installs and manages the Elastic Stack. It provides roles for every component of the Stack. Furthermore, it is possible to differentiate between Enterprise or OSS releases. +This collection installs and manages the Elastic Stack. It provides roles for every component +of the Stack. Furthermore, it is possible to differentiate between Enterprise or OSS releases. -Every role is documented with all variables, please refer to the documentation found in **[Getting-Started](./docs/getting-started.md)** +Every role can be used on its own or together with the others. Running a single role against an +existing or external cluster works, but a few of those paths are still incomplete. See the open +issues for details. -## Roles documentation +## Tested with Ansible -> [!NOTE] -> Some roles have fixed requirements that you must observe. Please have a look at the [requirements](docs/01-requirements.md) before using the collection. (There is a high probability that some of them will be refactored soon) +The minimum required version is declared in [meta/runtime.yml](meta/runtime.yml) and enforced by +`ansible-galaxy` on install. -* [Beats](roles/beats/README.md) -* [Elasticsearch](roles/elasticsearch/README.md) -* [Kibana](roles/kibana/README.md) -* [Logstash](roles/logstash/README.md) -* [Repos](roles/repos/README.md) +| | | +|---|---| +| ansible-core | `>= 2.18`, tested with 2.19 and 2.20 | +| Python on the control node | `>= 3.11` | +| Python on the target hosts | `>= 3.9` | +| Elastic Stack | release 8 | -## Modules documentation - -* [elasticsearch_role](docs/module-elasticsearch_role.md) -* [elasticsearch_user](docs/module-elasticsearch_user.md) - -## Global variables - -The collection-wide `elasticstack_*` variables (release, variant, inventory host -groups, ports, package repositories, and the shared CA) are documented centrally -with the [elasticstack role](roles/elasticstack/README.md). - -* `elasticstack_force_pip`: Will force installation of required Python modules via `pip`. This is useful if your package manager doesn't provide current versions of modules. (Default: `false`) See [PEP668](https://peps.python.org/pep-0668/) for more details. -* `elasticstack_manage_pip`: Will install `pip` on your system. (Default: `false`) - -## Installation - -You can easily install the collection with the `ansible-galaxy` command. - -``` -ansible-galaxy collection install git+https://github.com/netways/ansible-collection-elasticstack.git -``` - -To pin a released version instead of the current state of `main`, use a `requirements.yml`: - -```yaml -collections: - - name: https://github.com/NETWAYS/ansible-collection-elasticstack.git - type: git - version: 0.1.0 -``` - -```bash -ansible-galaxy collection install -r requirements.yml -``` - -Or if you are using Tower or AWX add the collection to your requirements file. - -``` -collections: - - name: netways.elasticstack -``` - -### Requirements - -On the Ansible control node: -* `Ansible-Core >=2.18.0` -* `Python >=3.11.0` - -You will need the following Ansible collections installed - -* `community.general` (probably already present) - -You will need these packages / libraries installed. Some very basic packages like `openssl` get handled by the collection if needed. The following list contains packages and libraries which only apply to special cases or need for you to decide on the installation method. - -* `passlib` Python library if you do not disable password hashing for logstash user and you want to use logstash role from this collection. It should be installed with pip on the Ansible controller. -* `elasticsearch` Python module. Current versions are either compatible to Elasticsearch 9 or versions lower than 9. There seems to be no version that can serve both. So for now we install a version of the client lower than 9 but need to take care to install the right one when we make this collection compatible to Elastic Stack 9 - -You may want the following Ansible roles installed. There other ways to achieve what they are doing but using them is easy and convenient. - -* `geerlingguy.redis` if you want to use logstash role - -### Supported systems - -We test the collection on the following Linux distributions, each one with Elastic Stack 8. +We test on the following Linux distributions, each one with Elastic Stack 8: * Rocky Linux 9 * Ubuntu 22.04 LTS * Debian 13 -## Caveats and information for long time users - -### Variable renaming +## External requirements -**Please note**: If you are already using this collection before version `1.0.0`, please note that we had to rename a significant amount of variables due to naming schema changes made by Ansible. Please review the variables you have set in your playbooks and variable files. +Ansible collections: -### Ulimit management for Elasticsearch - -**Warning**: The role no longer configures `ulimit`. Ensure the system's open file limit is set correctly (e.g., `ulimit -n`). Packages usually handle this, but verify with `ulimit -n` or checking `/proc//limits`. If your version does not, open an issue at the collection's issue tracker: https://github.com/netways/ansible-collection-elasticstack/issues - -## Usage - -Every role is documented with all variables, please refer to the documentation found in **[Getting-Started](./docs/getting-started.md)** - -Make sure all hosts that should be configured are part of your playbook. (See below for details on groups etc.). The collection is built to first collect all facts from all hosts (including those only running beats) and then use facts like hostnames or ip addresses to connect the tools to each other. +* `community.general` (probably already present) -You will want to have reliable DNS resolution or enter all hosts of the stack into your systems hosts files. +Python libraries on the control node: -The variable `elasticstack_no_log` can be set to `false` if you want to see the output of all tasks. It defaults to `true` because some tasks could reveal passwords in production. +* `passlib`, unless you disable password hashing for the Logstash user and you want to use the + logstash role. Install it with `pip`. +* `elasticsearch`. Current versions are compatible either with Elasticsearch 9 or with versions + lower than 9, there seems to be no version that serves both. For now we install a client + older than 9, and this needs revisiting when the collection becomes compatible with Elastic + Stack 9. -### Versions and upgrades +Ansible roles: -*elasticstack_version*: Version number of tools to install. Only set if you don't want the latest on new setups. (default: none). If you already have an installation of Elastic Stack, this collection will query the version of Elasticsearch on the CA host and use it for all further installations in the same setup. (Only if you run the `elasticsearch` role before all others) Example: `7.17.2` +* `geerlingguy.redis` if you want to use the logstash role. There are other ways to get Redis + running, but this one is easy and convenient. -All packages are installed with `state: present`. When `elasticstack_version` is set to a version number (e.g. `7.17.2`), that exact version is installed and pinned. When it is left unset, the package is installed without a version, so a new setup gets the newest available version and existing installations are not upgraded automatically on later runs. + ```bash + ansible-galaxy install geerlingguy.redis + ``` -*elasticstack_release*: Major release version of Elastic stack to configure. (default: `8`) Make sure it corresponds to `elasticstack_version` if you set both. +Some very basic packages such as `openssl` are handled by the collection itself. The list above +only contains what applies to special cases or what you need to decide about yourself. -For OSS version see `elasticstack_variant` below. +Beyond software, the roles expect a few things from your inventory. See +[Requirements](docs/requirements.md). -*elasticstack_variant*: Variant of the stack to install. Valid values: `elastic` or `oss`. (default: `elastic`) +## Included content -```yaml -roles: - - role: netways.elasticstack.kibana - vars: - elasticstack_version: 8.7.1 - - role: netways.elasticstack.elasticsearch - vars: - elasticstack_version: 8.8.1 -``` +Roles: -#### Upgrades #### +* [beats](roles/beats/README.md) +* [elasticsearch](roles/elasticsearch/README.md) +* [kibana](roles/kibana/README.md) +* [logstash](roles/logstash/README.md) +* [repos](roles/repos/README.md) +* [elasticstack](roles/elasticstack/README.md), the meta role that holds the collection-wide + `elasticstack_*` variables -Set `elasticstack_version` to the version you want to upgrade to. Positively do read and understand Elastics changelog and "breaking changes" of your target version and all between your current and the target version. Do not use unless you have a valid backup. +Modules: -If an upgrade fails, you can try re-running the collection with the same settings. There are several tasks that can provide "self-healing". Please do not rely on these mechanisms, they are more of a "convenience recovery" for easier steps. - -The collection will make sure to upgrade Elasticsearch nodes one by one. +* [elasticsearch_role](docs/module-elasticsearch_role.md) +* [elasticsearch_user](docs/module-elasticsearch_user.md) -### Default Passwords +## Collection documentation -Default passwords are generated on the CA host and can be seen during generation, or found -later in the file that `elasticstack_initial_passwords` points to (default -`/usr/share/elasticsearch/initial_passwords`). +* [Getting started](docs/getting-started.md), example playbook, inventory groups, first run +* [Requirements](docs/requirements.md), what the roles expect from your inventory +* [Versions and upgrades](docs/upgrades.md), pinning versions and upgrading the stack -Security is on by default and controlled by `elasticstack_security`. A related but much -narrower switch is `elasticstack_override_beats_tls`: it only stops TLS between Beats and -Logstash from being enabled automatically, and skips the Beats certificate generation. It does -not turn off security anywhere else. +Every role documents all of its variables in its own README, linked above. Those tables are +generated from each role's `meta/argument_specs.yml`. -### Requirements +## Using this collection -0) You need to install the redis role which is maintained by geerlingguy. +Install the collection with `ansible-galaxy`: -``` -ansible-galaxy install geerlingguy.redis +```bash +ansible-galaxy collection install netways.elasticstack ``` -1) Default: For general Elastic Stack installations using all features use the following. You will need Redis installed and running for the default setup to run. A viable way is using the `geerlingguy.redis` role. +To pin a version, use a `requirements.yml`: -2) Specific: For OSS Installation without X-Pack features you can use the following. _Note_: this is only available for version `7.x`. - -Our default configuration will collect filesystem logs placed by `rsyslog`. Therefor our example playbook makes sure, `rsyslog` is installed. If you don't want that, please change the configuration of the `beats` module. Without syslog you won't receive any messages with the default configuration. +```yaml +collections: + - name: netways.elasticstack + version: 0.1.0 +``` -There are some comments in the Playbook. Either fill them with the correct values (`remote_user`) or consider them as a hint to commonly used options. +```bash +ansible-galaxy collection install -r requirements.yml +``` -### Inventory +The same file works for Tower and AWX. You can also install straight from git, which gives you +the current state of `main` instead of a release: -_Note_: The roles rely on group names for placing services on hosts. Please make sure you have group names defined: `elasticstack_elasticsearch_group_name` (default: `elasticsearch`), `elasticstack_logstash_group_name` (default: `logstash`) and `elasticstack_kibana_group_name` (default: `kibana`) that will match your desired setup in your Ansible inventory. Hosts in these groups will get the respective services. Just restricting your plays to the appropriate hosts will not work because the roles interact with hosts from other groups e.g. for certificate generation. +```bash +ansible-galaxy collection install git+https://github.com/NETWAYS/ansible-collection-elasticstack.git +``` -The execution order of the roles is important! (see below) +Then refer to the roles by their fully qualified name: ```yaml ---- -- hosts: all - # remote_user: my_username - become: true - collections: - - netways.elasticstack - vars: - elasticstack_variant: elastic #oss - # elasticstack_release: 8 #7 - roles: - - repos - - hosts: elasticsearch - # remote_user: my_username - become: true - collections: - - netways.elasticstack - vars: - elasticstack_variant: elastic #oss - elasticsearch_jna_workaround: true - # elasticstack_release: 8 #7 - roles: - - elasticsearch - -- hosts: logstash - # remote_user: my_username become: true - collections: - - netways.elasticstack - vars: - elasticstack_variant: elastic #oss - elasticstack_override_beats_tls: true - # elasticstack_release: 8 #7 roles: - - geerlingguy.redis - - logstash - -- hosts: kibana - # remote_user: my_username - become: true - collections: - - netways.elasticstack - vars: - elasticstack_variant: elastic #oss - # elasticstack_release: 8 #7 - roles: - - kibana + - netways.elasticstack.elasticsearch +``` -- hosts: all - # remote_user: my_username - become: true - collections: - - netways.elasticstack - vars: - elasticstack_variant: elastic #oss - elasticstack_override_beats_tls: true - # elasticstack_release: 8 #7 - pre_tasks: - - name: Install Rsyslog - ansible.builtin.package: - name: rsyslog - - name: Start rsyslog - ansible.builtin.service: - name: rsyslog - state: started - enabled: true - roles: - - beats +The execution order of the roles matters, and the roles interact with hosts from other inventory +groups. Read [Getting started](docs/getting-started.md) before your first run. -``` +## Caveats and information for long time users -## Contributing +### Variable renaming -Every kind of contribution is very welcome. Open [issues](https://github.com/NETWAYS/ansible-collection-elasticstack/issues) or provide [pull requests](https://github.com/NETWAYS/ansible-collection-elasticstack/pulls). +If you have been using this collection before version `1.0.0`, note that a significant number of +variables had to be renamed because of naming schema changes made by Ansible. Please review the +variables you set in your playbooks and variable files. -Please read [CONTRIBUTING.md](CONTRIBUTING.md) before opening a pull request. It describes -what a pull request needs, including the changelog fragment that every pull request has to -bring along. +### Ulimit management for Elasticsearch -For now we open pull requests against `main`. We are planning to introduce dedicated branches to support older versions without breaking changes. Since we don't need them for now, please check back with this section because when we decided on how to proceed, you will find the information here. For now `main` always has the newest changes and if you want a stable version, please use the newest release. +The role no longer configures `ulimit`. Make sure the system's open file limit is set correctly, +packages usually handle this. Verify with `ulimit -n` or by checking `/proc//limits`. If +your version does not, please +[open an issue](https://github.com/NETWAYS/ansible-collection-elasticstack/issues). -### Documentation for role variables +## Contributing to this collection -Role variables are documented from each role's `meta/argument_specs.yml`, which is -the single source of truth. When your pull request changes a role's variables: +Every kind of contribution is welcome. Open +[issues](https://github.com/NETWAYS/ansible-collection-elasticstack/issues) or provide +[pull requests](https://github.com/NETWAYS/ansible-collection-elasticstack/pulls). -1. Update that role's `meta/argument_specs.yml` (type, default, description). -2. Regenerate the README variable table — please do **not** edit it by hand. The table is - produced by [ansible-docsmith](https://github.com/foundata/ansible-docsmith) - (install with `pip install ansible-docsmith` if you don't have it): +Pull requests go against `main`. If you need a stable state, pin a release tag. - ``` - ansible-docsmith generate roles/ --no-defaults --template-readme .docsmith/readme.md.j2 - ``` +Please read [CONTRIBUTING.md](CONTRIBUTING.md) first. It describes what a pull request needs, +including the changelog fragment that every pull request has to bring along, how role variables +are documented and how to run the tests. -3. Commit the regenerated `README.md` together with your change. +## Release notes -The `Test Documentation` workflow checks that each README matches its -`argument_specs.yml` and fails the pull request if they drift apart. +See the [releases page](https://github.com/NETWAYS/ansible-collection-elasticstack/releases). +The same notes are collected in `CHANGELOG.md`, which is generated from the changelog fragments +when a version is released. -## Testing +## Licensing -Besides real tests that the developer should do before creating a PR, we built molecule scenarios to test the complete stack. -In parallel, we check for ansible and yaml lint errors. To do this pro-actively, a `makefile` is included. You can use this by calling `make`. +GPL-3.0-or-later, see [LICENSE](LICENSE). diff --git a/docs/01-requirements.md b/docs/01-requirements.md deleted file mode 100644 index 6b27087b..00000000 --- a/docs/01-requirements.md +++ /dev/null @@ -1,15 +0,0 @@ -# Requirements - - -* Minimum Python version on Ansible controller: 3.11 -* Minimum Python version on target hosts: 3.9 - -There are some requirements that the user have to fulfill while using the collection. Some of them will be refactored and disappear from the list soon. - -**Inventory names** - -The collection provides roles for several tools (elasticsearch, kibana, ...). In some tasks hosts from the specific inventory group are fetched. The group names are customizable via variables. But using different inventory names than provided inside the variables will result in errors. - -**elasticstack_ca_host** - -This is a mandatory variable. It is used to define the host that will be used as "CA host". Per default it is already defined (first host inside the group `elasticstack_elasticsearch_group_name`). In case you are using different group names, this variable wont be set with a defualt value. diff --git a/docs/getting-started.md b/docs/getting-started.md index 2d322e91..da829aa0 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -1,63 +1,137 @@ -Collection Elastic Stack -------------------------- +# Getting started -Installation ------------ +This page shows how to run the collection. For installation see the +[README](../README.md#using-this-collection), for what the roles expect from your inventory see +[Requirements](requirements.md). -You can easily install the collection with the ansible-galaxy command. +## Inventory groups -``` -ansible-galaxy collection install netways.elasticstack -``` +The roles place services on hosts by inventory group name. Make sure the groups exist and match +your desired setup: -Or if you are using Tower or AWX add the collection to your requirements file. +| Variable | Default group name | +|---|---| +| `elasticstack_elasticsearch_group_name` | `elasticsearch` | +| `elasticstack_logstash_group_name` | `logstash` | +| `elasticstack_kibana_group_name` | `kibana` | -``` -collections: - - name: netways.elasticstack -``` +Hosts in these groups get the respective services. Restricting your plays to only the +appropriate hosts does **not** work, because the roles read facts from hosts in other groups, +for example to generate certificates. -Usage ---------- +Make sure every host that should be configured is part of your playbook, including hosts that +only run Beats. The collection first gathers facts from all hosts and then uses facts such as +host names and IP addresses to connect the tools to each other. -To use the collection in your Ansible playbook add the following key to your playbook. +You will want reliable DNS resolution, or all hosts of the stack in your systems' hosts files. -``` -- name: Playbook - hosts: some_host_pattern - collections: - - netways.elasticstack - tasks: - - name: import role logstash - import_role: - name: logstash -``` +## Before the first run -Or refer to the role with the FQCN of the role. +Redis is needed for the default setup, because the default Logstash pipeline uses it. The +example below uses the `geerlingguy.redis` role for that: -``` -- name: Playbook - hosts: some_host_pattern - tasks: - - name: import role by FQCN from a collection - import_role: - name: netways.elasticstack.logstash +```bash +ansible-galaxy install geerlingguy.redis ``` -Roles -------- +The default Beats configuration collects file system logs written by `rsyslog`, which is why the +example playbook installs it. Without syslog you will not receive any messages with the default +configuration. If you do not want that, change the configuration of the beats role. + +The OSS variant without X-Pack features is only available for Elastic Stack `7`. + +## Variables + +Every role has its own set of variables, documented in the role's README. In addition the +collection-wide `elasticstack_*` variables apply to any role and are documented centrally with +the [elasticstack role](../roles/elasticstack/README.md). Two common ones: + +* `elasticstack_release`: major release version of the Elastic Stack to configure + (default: `8`) +* `elasticstack_variant`: variant of the stack to install, `elastic` or `oss` + (default: `elastic`) + +For pinning a specific version and for upgrading, see [Versions and upgrades](upgrades.md). + +## Example playbook + +The execution order of the plays matters. There are comments in the playbook below: either fill +them with the correct values, such as `remote_user`, or take them as a hint about commonly used +options. + +```yaml +--- +- hosts: all + # remote_user: my_username + become: true + vars: + elasticstack_variant: elastic # oss + # elasticstack_release: 8 + roles: + - netways.elasticstack.repos + +- hosts: elasticsearch + # remote_user: my_username + become: true + vars: + elasticstack_variant: elastic # oss + elasticsearch_jna_workaround: true + # elasticstack_release: 8 + roles: + - netways.elasticstack.elasticsearch + +- hosts: logstash + # remote_user: my_username + become: true + vars: + elasticstack_variant: elastic # oss + elasticstack_override_beats_tls: true + # elasticstack_release: 8 + roles: + - geerlingguy.redis + - netways.elasticstack.logstash + +- hosts: kibana + # remote_user: my_username + become: true + vars: + elasticstack_variant: elastic # oss + # elasticstack_release: 8 + roles: + - netways.elasticstack.kibana + +- hosts: all + # remote_user: my_username + become: true + vars: + elasticstack_variant: elastic # oss + elasticstack_override_beats_tls: true + # elasticstack_release: 8 + pre_tasks: + - name: Install Rsyslog + ansible.builtin.package: + name: rsyslog + - name: Start rsyslog + ansible.builtin.service: + name: rsyslog + state: started + enabled: true + roles: + - netways.elasticstack.beats +``` -* [Beats](../roles/beats/README.md) -* [Elasticsearch](../roles/elasticsearch/README.md) -* [Kibana](../roles/kibana/README.md) -* [Logstash](../roles/logstash/README.md) -* [Repos](../roles/repos/README.md) +## Generated passwords +Passwords for the built-in users are generated on the CA host during the first run. They are +printed while they are generated and stored in the file that `elasticstack_initial_passwords` +points to, by default `/usr/share/elasticsearch/initial_passwords`. -Variables ------------ +Security is on by default and controlled by `elasticstack_security`. A related but much +narrower switch is `elasticstack_override_beats_tls`: it only stops TLS between Beats and +Logstash from being enabled automatically, and skips the Beats certificate generation. It does +not turn off security anywhere else. -Every role got its own set of variables, in addition a few variables are useable on any role. The collection-wide `elasticstack_*` variables are documented centrally with the [elasticstack role](../roles/elasticstack/README.md). Two common ones: +## Seeing what the tasks do -* *elasticstack_release*: Major release version of Elastic stack to configure. (default: `8`) -* *elasticstack_variant*: Variant of the stack to install. Valid values: `elastic` or `oss`. (default: `elastic`) +`elasticstack_no_log` defaults to `true`, because some tasks could reveal passwords in +production. Set it to `false` if you want to see the output of all tasks. diff --git a/docs/requirements.md b/docs/requirements.md new file mode 100644 index 00000000..8886c8c9 --- /dev/null +++ b/docs/requirements.md @@ -0,0 +1,27 @@ +# Requirements + +Software requirements, supported Ansible and Python versions and external libraries are listed +in the [README](../README.md#tested-with-ansible). This page describes what the roles expect +from your setup. + +Some of these will be refactored and disappear from this list. + +## Inventory group names + +The collection provides roles for several tools, and some tasks fetch hosts from a specific +inventory group. The group names are customizable through variables, but using group names that +differ from those variables results in errors. + +| Variable | Default group name | +|---|---| +| `elasticstack_elasticsearch_group_name` | `elasticsearch` | +| `elasticstack_logstash_group_name` | `logstash` | +| `elasticstack_kibana_group_name` | `kibana` | + +## elasticstack_ca_host + +This variable defines the host that acts as the certificate authority. It has a default: the +first host in the group named by `elasticstack_elasticsearch_group_name`. + +That default only resolves when that group exists. If you use different group names without +setting them in the variables above, the variable ends up empty and the run fails. diff --git a/docs/upgrades.md b/docs/upgrades.md new file mode 100644 index 00000000..b1736414 --- /dev/null +++ b/docs/upgrades.md @@ -0,0 +1,51 @@ +# Versions and upgrades + +## Choosing a version + +`elasticstack_version` holds the version number of the tools to install, for example `8.19.4`. +Only set it if you do not want the newest available version on a new setup. There is no default. + +If you already have an installation of the Elastic Stack, the collection queries the installed +version of Elasticsearch on the CA host and uses it for all further installations in the same +setup. This only happens when you run the `elasticsearch` role before all others. + +All packages are installed with `state: present`: + +* When `elasticstack_version` is set to a version number, that exact version is installed and + pinned. +* When it is left unset, the package is installed without a version. A new setup gets the newest + available version, and existing installations are not upgraded automatically on later runs. + +`elasticstack_release` holds the major release, for example `8` (default). If you set both, make +sure they match. + +For the OSS variant see `elasticstack_variant`, described with the +[elasticstack role](../roles/elasticstack/README.md). + +Both variables can be set per role, which is how you keep components on different versions: + +```yaml +roles: + - role: netways.elasticstack.kibana + vars: + elasticstack_version: 8.7.1 + - role: netways.elasticstack.elasticsearch + vars: + elasticstack_version: 8.8.1 +``` + +## Upgrading + +Set `elasticstack_version` to the version you want to upgrade to. + +Before you do, read and understand Elastic's changelog and the breaking changes of your target +version **and of every version between your current one and the target**. Do not upgrade without +a valid backup. + +Elasticsearch nodes are upgraded one at a time. The role disables shard allocation, stops the +node, installs the new package, starts it again and waits for the node to rejoin before it +touches the next one. + +If an upgrade fails, you can try re-running the collection with the same settings. Several tasks +provide a degree of self-healing. Do not rely on these mechanisms, they are a convenience +recovery for the easier cases and not a substitute for a backup. From cdf0e105e879e3d866ba2f8c71883bfcacf13b76 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 7 Sep 2026 13:57:26 +0200 Subject: [PATCH 22/53] Keep inventory groups and the Redis dependency documented once --- docs/getting-started.md | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index da829aa0..dbf91685 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -6,14 +6,8 @@ This page shows how to run the collection. For installation see the ## Inventory groups -The roles place services on hosts by inventory group name. Make sure the groups exist and match -your desired setup: - -| Variable | Default group name | -|---|---| -| `elasticstack_elasticsearch_group_name` | `elasticsearch` | -| `elasticstack_logstash_group_name` | `logstash` | -| `elasticstack_kibana_group_name` | `kibana` | +The roles place services on hosts by inventory group name. The group names and their defaults +are listed in [Requirements](requirements.md#inventory-group-names). Hosts in these groups get the respective services. Restricting your plays to only the appropriate hosts does **not** work, because the roles read facts from hosts in other groups, @@ -27,12 +21,9 @@ You will want reliable DNS resolution, or all hosts of the stack in your systems ## Before the first run -Redis is needed for the default setup, because the default Logstash pipeline uses it. The -example below uses the `geerlingguy.redis` role for that: - -```bash -ansible-galaxy install geerlingguy.redis -``` +Redis has to be running for the default setup, because the default Logstash pipeline uses it. +The example playbook below gets it there with the `geerlingguy.redis` role, listed under +external requirements in the [README](../README.md#external-requirements). The default Beats configuration collects file system logs written by `rsyslog`, which is why the example playbook installs it. Without syslog you will not receive any messages with the default From c773d5243e78e51bbefd88e6e318dca798f01066 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 7 Sep 2026 13:57:38 +0200 Subject: [PATCH 23/53] Read module documentation from the modules instead of copies --- README.md | 15 +++++-- docs/module-elasticsearch_role.md | 68 ------------------------------- docs/module-elasticsearch_user.md | 65 ----------------------------- 3 files changed, 12 insertions(+), 136 deletions(-) delete mode 100644 docs/module-elasticsearch_role.md delete mode 100644 docs/module-elasticsearch_user.md diff --git a/README.md b/README.md index 9227bd89..e0944445 100644 --- a/README.md +++ b/README.md @@ -69,8 +69,16 @@ Roles: Modules: -* [elasticsearch_role](docs/module-elasticsearch_role.md) -* [elasticsearch_user](docs/module-elasticsearch_user.md) +* `cert_info`, reads information from a PKCS12 certificate +* `elasticsearch_role`, creates, updates and deletes roles in Elasticsearch +* `elasticsearch_user`, creates, updates and deletes users in Elasticsearch + +Each module documents its own parameters, return values and examples. Read them with +`ansible-doc`, for example: + +```bash +ansible-doc netways.elasticstack.elasticsearch_role +``` ## Collection documentation @@ -79,7 +87,8 @@ Modules: * [Versions and upgrades](docs/upgrades.md), pinning versions and upgrading the stack Every role documents all of its variables in its own README, linked above. Those tables are -generated from each role's `meta/argument_specs.yml`. +generated from each role's `meta/argument_specs.yml`. Module documentation comes from the +modules themselves and is read with `ansible-doc`. ## Using this collection diff --git a/docs/module-elasticsearch_role.md b/docs/module-elasticsearch_role.md deleted file mode 100644 index 1127a344..00000000 --- a/docs/module-elasticsearch_role.md +++ /dev/null @@ -1,68 +0,0 @@ -Ansible module: elasticsearch_role -=== - -This module creates, updates and deletes roles from your Elasticsearch. - -Requirements ---- - -As this module uses the Elasticsearch API you will need to install the `elasticsearch` Python3 library. -``` -pip3 install elasticsearch -``` - -Module arguments ---- - -* *name*: Name of your role (**Required**) -* *cluster*: List of clusters -* *indicies*: List of indicies - * *names*: List of names (**Required**) - * *privileges*: List of privileges (**Required**) -* *state*: State of the role (Default: `present`) -* *host*: API endpoint (**Required**) -* *auth_user*: User to authenticate on the Elasticsearch API (**Required**) -* *auth_pass*: Password for the given user (**Required**) -* *verify_certs*: Verify certificates (Default: `true`) -* *ca_certs*: Verify HTTPS connection by using ca certificate. Path to ca needs to be given - -Example usage ---- -``` - - name: Create elasticsearch role 'new-role1' - netways.elasticstack.elasticsearch_role: - name: new-role1 - cluster: - - manage_own_api_key - - delegate_pki - indicies: - - names: - - default01 - privileges: - - read - - write - state: present - host: https://localhost:9200 - auth_user: elastic - auth_pass: changeMe123! - verify_certs: true - ca_certs: /etc/elasticsearch/certs/http_ca.crt - - - name: Create elasticsearch role 'new-role2' - netways.elasticstack.elasticsearch_role: - name: new-role2 - cluster: - - manage_own_api_key - - delegate_pki - indicies: - - names: - - default01 - privileges: - - read - - write - state: present - host: https://localhost:9200 - auth_user: elastic - auth_pass: changeMe123! - verify_certs: false -``` \ No newline at end of file diff --git a/docs/module-elasticsearch_user.md b/docs/module-elasticsearch_user.md deleted file mode 100644 index 9ab1ea07..00000000 --- a/docs/module-elasticsearch_user.md +++ /dev/null @@ -1,65 +0,0 @@ -Ansible module: elasticsearch_user -=== - -This module creates, updates and deletes users from your Elasticsearch. - -Requirements ---- - -As this module uses the Elasticsearch API you will need to install the `elasticsearch` Python3 library. -``` -pip3 install elasticsearch -``` - -Module arguments ---- - -* *name*: Name of your user (**Required**) -* *fullname*: Fullname of your user -* *password*: Password for your user (**Required**) -* *email*: Email for your user -* *roles*: List of roles (**Required**) -* *enabled*: Define wheter this user should be enabled (Default: `true`) -* *state*: State of the role. `absent` to delete the user (Default: `present`) -* *host*: API endpoint (**Required**) -* *auth_user*: User to authenticate on the Elasticsearch API (**Required**) -* *auth_pass*: Password for the given user (**Required**) -* *verify_certs*: Verify certificates (Default: `true`) -* *ca_certs*: Verify HTTPS connection by using ca certificate. Path to ca needs to be given - -Example usage ---- -``` - - name: Create elasticsearch user 'new-user1' - netways.elasticstack.elasticsearch_user: - name: new-user1 - fullname: New User 1 - password: changeMe321! - email: new1@user.de - roles: - - new-role - - logstash-writer - enabled: true - state: present - host: https://localhost:9200 - auth_user: elastic - auth_pass: changeMe123! - verify_certs: true - ca_certs: /etc/elasticsearch/certs/http_ca.crt - - - name: Create elasticsearch user 'new-user2' - netways.elasticstack.elasticsearch_user: - name: new-user2 - fullname: New User 2 - password: changeMe321! - email: new2@user.de - roles: - - new-role - - logstash-writer - enabled: true - state: present - host: https://localhost:9200 - auth_user: elastic - auth_pass: changeMe123! - verify_certs: false -``` \ No newline at end of file From 5bea3c433f9642ac5504b9351dda7c9e0e079192 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 7 Sep 2026 13:57:48 +0200 Subject: [PATCH 24/53] Delete the plugin directory boilerplate from the collection skeleton --- plugins/README.md | 31 ------------------------------- 1 file changed, 31 deletions(-) delete mode 100644 plugins/README.md diff --git a/plugins/README.md b/plugins/README.md deleted file mode 100644 index 29aa3195..00000000 --- a/plugins/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# Collections Plugins Directory - -This directory can be used to ship various plugins inside an Ansible collection. Each plugin is placed in a folder that -is named after the type of plugin it is in. It can also include the `module_utils` and `modules` directory that -would contain module utils and modules respectively. - -Here is an example directory of the majority of plugins currently supported by Ansible: - -``` -└── plugins - ├── action - ├── become - ├── cache - ├── callback - ├── cliconf - ├── connection - ├── filter - ├── httpapi - ├── inventory - ├── lookup - ├── module_utils - ├── modules - ├── netconf - ├── shell - ├── strategy - ├── terminal - ├── test - └── vars -``` - -A full list of plugin types can be found at [Working With Plugins](https://docs.ansible.com/ansible-core/2.13/plugins/plugins.html). From b8bbdf545c3c58682358e7f59da7106e52104f9a Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 7 Sep 2026 13:57:48 +0200 Subject: [PATCH 25/53] Fix a dead anchor and two typos in the documentation --- NOTICE.md | 2 +- plugins/module_utils/README.md | 2 +- plugins/modules/README.md | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/NOTICE.md b/NOTICE.md index e8b18e4c..8652a9e5 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -4,4 +4,4 @@ Here's a list of sponsors who contributed by having the collection improved via outsourcing to NETWAYS. -* CID GmbH : Thank you so much for sponsoring. Especially the feature to have different types of Elasticsearch nodes in the cluster and the ingetration of rolling upgrades. +* CID GmbH : Thank you so much for sponsoring. Especially the feature to have different types of Elasticsearch nodes in the cluster and the integration of rolling upgrades. diff --git a/plugins/module_utils/README.md b/plugins/module_utils/README.md index 405b9c68..a92071fb 100644 --- a/plugins/module_utils/README.md +++ b/plugins/module_utils/README.md @@ -1,7 +1,7 @@ # Documentation: netways.elasticstack module_utils ## Overview -- [`certs` module_util](#cert_info-module) +- [`certs` module_util](#netwayselasticstackcerts-function) ## `netways.elasticstack.certs` function diff --git a/plugins/modules/README.md b/plugins/modules/README.md index 18f7344d..48e7b549 100644 --- a/plugins/modules/README.md +++ b/plugins/modules/README.md @@ -8,7 +8,7 @@ - [Tested ansible-core versions](#tested-ansible-core-versions) - [Security measures](#security-measures) - [Supported extensions and values](#supported-extensions-and-values) - - [Parameters](#paramters) + - [Parameters](#parameters) - [Returns](#returns) - [Example](#example) @@ -66,7 +66,7 @@ Currently, the information of the following extensions and values will be return **SubjectKeyIdentifier**: - _digest -### Paramters +### Parameters `path`: Absolute path to certificate. (**Default:** undefined, required) From e164131c8253a17c1aac5f5d83428695f83bd511 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 7 Sep 2026 14:09:13 +0200 Subject: [PATCH 26/53] Document the cert_info extensions and safeguards in the module itself --- plugins/modules/README.md | 158 ----------------------------------- plugins/modules/cert_info.py | 15 ++++ 2 files changed, 15 insertions(+), 158 deletions(-) delete mode 100644 plugins/modules/README.md diff --git a/plugins/modules/README.md b/plugins/modules/README.md deleted file mode 100644 index 48e7b549..00000000 --- a/plugins/modules/README.md +++ /dev/null @@ -1,158 +0,0 @@ -# Documentation: netways.elasticstack modules - -## Overview -- [`cert_info` module](#cert_info-module) - - [Dependencies](#dependencies) - - [Tested python-cryptography versions >= 2.5](#tested-python-cryptography-versions--25) - - [Tested python versions](#tested-python-versions) - - [Tested ansible-core versions](#tested-ansible-core-versions) - - [Security measures](#security-measures) - - [Supported extensions and values](#supported-extensions-and-values) - - [Parameters](#parameters) - - [Returns](#returns) - - [Example](#example) - -## `cert_info` module - -The netways.elasticstack.cert_info module gathers information about pkcs12 certificates generated by the Elastic stack cert util. - -### Dependencies -- python-cryptography >= 2.5.0 on the remote node - -### Tested python-cryptography versions >= 2.5 -- 2.5 -- 3.0 -- 3.1 -- 3.2 -- 3.3 -- 3.4 -- 35.0.0 -- 36.0.0 -- 38.0.0 -- 40.0.1 - -### Tested python versions -- 2.7 -- 3.5 -- 3.6 -- 3.7 -- 3.8 -- 3.10 - -### Tested ansible-core versions -- 2.11 -- 2.12 -- 2.13 -- 2.14 - -### Security measures -- Only supported extensions with its available values will be returned. The available keys and values are applied in the code with the `SUPPORTED_EXTENSIONS` dictionary. The module will loop through it, and only if found, it will save it to the `results` variable. -- The paramters `path` and `passphrase` are set to no_log in the Ansible Module object. -- The objects `__private_key`, `__cert`, and `__additional_certs` are private and cannot be accessed globally. -- The object variables `__path` and `__passphrase` is private and cannot be accesed globally. - -### Supported extensions and values -Currently, the information of the following extensions and values will be returned (other extensions/values will be skipped): - -**BasicConstraints**: -- _ca -- _path_length - -**AuthorityKeyIdentifier**: -- _key_identifier -- _authority_cert_issuer -- _authority_cert_serial_number - -**SubjectKeyIdentifier**: -- _digest - -### Parameters - -`path`: -Absolute path to certificate. (**Default:** undefined, required) - -`password`: -The password of the pkcs12 certificate. (**Default:** No default, optional) - - -### Returns -All keys and values that will be returned with the results variable of the module: - -`issuer`: -The issuer of the CA certificate as **str**. - -`subject`: -The subject of the CA certificate as **str**. - -`version`: -The certificate version as **str** which represents an enumeration. - -`not_valid_after`: -A datetime.datetime object represented as a UTC formmated **str**. This is the beginning of the validity period. - -`not_valid_before`: -A datetime.datetime object represented as a UTC formmated **str**. This is the ending of the validity period. - -`serial_number`: -The serial number of the certificate as **str** which represents an integer. - -`extensions`: -- `oid`: The object identifier of the extension as **str** which represents a dotted string. -- `critical`: The value of critical as **str** which represents a bool. -- `values`: The keys and their values of the extension as **str**. (See: Supported extensions) - -### Example -``` -- name: Test - cert_info: - path: /opt/es-ca/elasticsearch-ca.pkcs12 - password: PleaseChangeMe - register: test - -- name: Debug - debug: - msg: "{{ test }}" -``` - -**Output**: -``` -TASK [Debug] ******************************************************************* -ok: [localhost] => { - "msg": { - "changed": false, - "extensions": { - "authorityKeyIdentifier": { - "_critical": "False", - "_dotted_string": "2.5.29.35", - "_values": { - "_authority_cert_issuer": "None", - "_authority_cert_serial_number": "None", - "_key_identifier": "82:53:20:11:C7:73:A7:5E:2A:77:C1:DF:22:E4:23:B4:C4:50:BA:CF" - } - }, - "basicConstraints": { - "_critical": "True", - "_dotted_string": "2.5.29.19", - "_values": { - "_ca": "True", - "_path_length": "None" - } - }, - "subjectKeyIdentifier": { - "_critical": "False", - "_dotted_string": "2.5.29.14", - "_values": { - "_digest": "82:53:20:11:C7:73:A7:5E:2A:77:C1:DF:22:E4:23:B4:C4:50:BA:CF" - } - } - }, - "failed": false, - "issuer": "Elastic Certificate Tool Autogenerated CA", - "not_valid_after": "2026-03-28 01:58:02", - "not_valid_before": "2023-03-29 01:58:02", - "serial_number": "719770426243590812378787092632593850366518596520", - "subject": "Elastic Certificate Tool Autogenerated CA", - "version": "Version.v3" - } -} -``` diff --git a/plugins/modules/cert_info.py b/plugins/modules/cert_info.py index 41cb73a4..ad182811 100644 --- a/plugins/modules/cert_info.py +++ b/plugins/modules/cert_info.py @@ -17,11 +17,26 @@ - Reads a PKCS12 certificate file and returns details such as issuer, subject, validity dates, serial number, and supported X.509 extensions. - Requires the C(cryptography) Python library (>= 2.5) on the target host. + - Only a fixed set of X.509 extensions is returned, everything else is skipped. + Currently supported are C(basicConstraints) with C(_ca) and C(_path_length), + C(subjectKeyIdentifier) with C(_digest), and C(authorityKeyIdentifier) with + C(_key_identifier), C(_authority_cert_issuer) and C(_authority_cert_serial_number). version_added: "1.0.0" author: - Daniel Patrick (@dpatrick) requirements: - cryptography >= 2.5 +notes: + - The supported extensions and their values are defined in the C(SUPPORTED_EXTENSIONS) + dictionary in the module code. The module iterates over it and only stores a value in + the result when it is found there, so an unexpected extension can never leak into the + output. + - Both O(path) and O(passphrase) are marked C(no_log), so they do not appear in task + output or logs. + - The path, the passphrase and the certificate are held in private attributes of the + analyzing object. The private key and any additional certificates from the PKCS12 file + are only kept in a local variable while the file is read and are never stored on the + object at all. options: path: description: Absolute path to the PKCS12 certificate file on the target host. From d15bef204a2ab6776530d6c3ac4726afd2f5d120 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 7 Sep 2026 14:18:40 +0200 Subject: [PATCH 27/53] Explain where module and module_utils documentation lives --- CONTRIBUTING.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 714b0285..a6d10c23 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -110,6 +110,17 @@ source of truth. When your pull request changes a role's variables: The `Test Documentation` workflow checks that each README matches its `argument_specs.yml` and fails the pull request if they drift apart. +## Documenting modules + +Modules document themselves through the `DOCUMENTATION`, `EXAMPLES` and `RETURN` blocks in +their own file. That is the only place, so there is nothing to keep in sync. Check your +changes with `ansible-doc netways.elasticstack.`. + +Shared code under `plugins/module_utils/` has no such block, because `ansible-doc` does not +read it. Those functions are described in +[plugins/module_utils/README.md](plugins/module_utils/README.md), please keep it current when +you change them. + ## Testing Besides testing your change against a real setup, the repository has Molecule scenarios that From 7e30abd5b7e4272aab88e35a200875e356954113 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 7 Sep 2026 15:31:15 +0200 Subject: [PATCH 28/53] Declare one current SPDX license identifier in galaxy.yml --- galaxy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/galaxy.yml b/galaxy.yml index a5d3dc96..bd7783e8 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -5,10 +5,10 @@ readme: README.md authors: - NETWAYS GmbH (https://github.com/NETWAYS) description: This collection installs and configures components from the Elastic Stack. +# Mutually exclusive with license_file. SPDX identifier, GPL-3.0 is deprecated. license: - - GPL-3.0 + - GPL-3.0-or-later # min_ansible_version: "2.18" # max supported: 2.20 (commented out for schema compliance) -license_file: 'LICENSE' tags: - elastic - elasticsearch From 1b87e543c996bb241b47a0b16a4c8f7b4c5f33ff Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 7 Sep 2026 15:31:27 +0200 Subject: [PATCH 29/53] Declare community.crypto, the beats role needs it --- README.md | 6 ++++-- galaxy.yml | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e0944445..3ca9f6f2 100644 --- a/README.md +++ b/README.md @@ -27,9 +27,11 @@ We test on the following Linux distributions, each one with Elastic Stack 8: ## External requirements -Ansible collections: +Ansible collections, both are declared as dependencies and installed for you when you install +this collection with `ansible-galaxy`: -* `community.general` (probably already present) +* `community.general`, for the `zypper` modules on SUSE hosts and the Logstash plugin module +* `community.crypto`, used by the beats role to check when a certificate expires Python libraries on the control node: diff --git a/galaxy.yml b/galaxy.yml index bd7783e8..6d74cd58 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -20,6 +20,7 @@ tags: - monitoring dependencies: "community.general": "*" + "community.crypto": "*" repository: https://github.com/netways/ansible-collection-elasticstack documentation: https://github.com/netways/ansible-collection-elasticstack/README.md homepage: https://www.netways.de From 460e84513494a01c7ad287886daabd01f412303c Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 7 Sep 2026 15:31:34 +0200 Subject: [PATCH 30/53] Fix the documentation URL in galaxy.yml --- galaxy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy.yml b/galaxy.yml index 6d74cd58..57d5fbd8 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -22,7 +22,7 @@ dependencies: "community.general": "*" "community.crypto": "*" repository: https://github.com/netways/ansible-collection-elasticstack -documentation: https://github.com/netways/ansible-collection-elasticstack/README.md +documentation: https://github.com/netways/ansible-collection-elasticstack/blob/main/README.md homepage: https://www.netways.de issues: https://github.com/netways/ansible-collection-elasticstack/issues build_ignore: From 5d9388e1119540ed79f483d768fbee0cff0da24b Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 7 Sep 2026 15:31:42 +0200 Subject: [PATCH 31/53] Set version_added of the modules to the first released version --- plugins/modules/cert_info.py | 2 +- plugins/modules/elasticsearch_role.py | 2 +- plugins/modules/elasticsearch_user.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/modules/cert_info.py b/plugins/modules/cert_info.py index ad182811..8d33971e 100644 --- a/plugins/modules/cert_info.py +++ b/plugins/modules/cert_info.py @@ -21,7 +21,7 @@ Currently supported are C(basicConstraints) with C(_ca) and C(_path_length), C(subjectKeyIdentifier) with C(_digest), and C(authorityKeyIdentifier) with C(_key_identifier), C(_authority_cert_issuer) and C(_authority_cert_serial_number). -version_added: "1.0.0" +version_added: "0.1.0" author: - Daniel Patrick (@dpatrick) requirements: diff --git a/plugins/modules/elasticsearch_role.py b/plugins/modules/elasticsearch_role.py index 05db0f33..df3749a7 100644 --- a/plugins/modules/elasticsearch_role.py +++ b/plugins/modules/elasticsearch_role.py @@ -14,7 +14,7 @@ description: - Creates, updates, or deletes Elasticsearch roles using the Security API. - Requires the C(elasticsearch) Python library on the target host. -version_added: "1.0.0" +version_added: "0.1.0" author: - Tobias Bauriedel (@tbauriedel) requirements: diff --git a/plugins/modules/elasticsearch_user.py b/plugins/modules/elasticsearch_user.py index d5b1dd4b..f70ff6a0 100644 --- a/plugins/modules/elasticsearch_user.py +++ b/plugins/modules/elasticsearch_user.py @@ -14,7 +14,7 @@ description: - Creates, updates, or deletes Elasticsearch native users using the Security API. - Requires the C(elasticsearch) Python library on the target host. -version_added: "1.0.0" +version_added: "0.1.0" author: - Tobias Bauriedel (@tbauriedel) requirements: From 2ea38b4185e16fc2f1a6bd5eefd813c60e9ec991 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 7 Sep 2026 15:31:56 +0200 Subject: [PATCH 32/53] Document both parameters of check_supported_keys --- plugins/module_utils/README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/plugins/module_utils/README.md b/plugins/module_utils/README.md index a92071fb..cf2d08a8 100644 --- a/plugins/module_utils/README.md +++ b/plugins/module_utils/README.md @@ -23,9 +23,13 @@ A function to check if the extension is supported. Returns true if extension nam ### `check_supported_keys` function -A function to check if the extensions key is supported. Returns true if extensions key is found in `SUPPORTED_EXTENSIONS` dict. +A function to check if the extensions key is supported. Returns true if the key is listed under +the given extension in the `SUPPORTED_EXTENSIONS` dict. Unlike `check_supported_extensions()` +this compares exactly, not by substring. -**Parameter:** The key name as __string__. +**Parameter:** Two, in this order: the key name as __string__, and the extension name as +__string__. The extension name has to be a key of `SUPPORTED_EXTENSIONS`, otherwise the lookup +raises a `KeyError`. **Return:** Returns a __bool__. From 93192b2e74e22cde8dfcd8da856dc114d19ee6eb Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 7 Sep 2026 15:32:07 +0200 Subject: [PATCH 33/53] Correct the constructor parameters of AnalyzeCertificate --- plugins/module_utils/README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/plugins/module_utils/README.md b/plugins/module_utils/README.md index cf2d08a8..130142d2 100644 --- a/plugins/module_utils/README.md +++ b/plugins/module_utils/README.md @@ -35,8 +35,13 @@ raises a `KeyError`. ### `AnalyzeCertificate()` object -An object to load the certificate and to gather information about it. +An object to load the certificate and to gather information about it. Constructing it already +does the work: it reads the file and fills the result dict, so there is no separate call to +start the analysis. -**Parameter:** The path (required) to the certificate and the passphrase (optional), both as __string__. +**Parameter:** Two, in this order: the `AnsibleModule` __object__ and the result __dict__ to +fill. The certificate path and the passphrase are not passed in, the object reads them from +`module.params`. It also uses the module to log, to warn and to fail. -**Return:** Returns the result dict to the Ansible module. +**Return:** The constructor returns nothing, it writes into the result dict it was given. +`return_result()` hands that dict back to the module, which passes it to `exit_json()`. From 04a22dccf2d15d3930456b5b36aa88791acbedb5 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 7 Sep 2026 15:33:06 +0200 Subject: [PATCH 34/53] Document the remaining module_utils files --- plugins/module_utils/README.md | 84 ++++++++++++++++++++++++++++++++-- 1 file changed, 80 insertions(+), 4 deletions(-) diff --git a/plugins/module_utils/README.md b/plugins/module_utils/README.md index 130142d2..48073ffb 100644 --- a/plugins/module_utils/README.md +++ b/plugins/module_utils/README.md @@ -1,17 +1,30 @@ # Documentation: netways.elasticstack module_utils +Shared code used by the modules in `plugins/modules/`. `ansible-doc` does not read this +directory, which is why it is documented here by hand instead of in a `DOCUMENTATION` block. + ## Overview -- [`certs` module_util](#netwayselasticstackcerts-function) +- [`certs`](#certs), reading PKCS12 certificates. Used by `cert_info`. +- [`api`](#api), the Elasticsearch client. Used by `elasticsearch_role` and `elasticsearch_user`. +- [`elasticsearch_role`](#elasticsearch_role), the role logic behind the `elasticsearch_role` module. +- [`elasticsearch_user`](#elasticsearch_user), the user logic behind the `elasticsearch_user` module. + +## `certs` -## `netways.elasticstack.certs` function +Requires the `cryptography` library (>= 2.5). The module that uses it fails with +`missing_required_lib()` when it is absent. + +`SUPPORTED_EXTENSIONS` is a dict at the top of the file. It maps an X.509 extension name to the +list of its attribute names that may be reported. Nothing outside it ever reaches the result, so +extending the module means extending that dict. ### `bytes_to_hex()` function -Since binascii.hexlify doesn't support a second parameter, which would define a seperator (e.g. ":") for hex strings in older Python versions like 2.6 and 2.7, we implemeted a small function to get similar results. +Since binascii.hexlify doesn't support a second parameter, which would define a separator (e.g. ":") for hex strings in older Python versions like 2.6 and 2.7, we implemented a small function to get similar results. **Parameter:** A __bytes__ object that represent a hexadecimal value (e.g. b'\\x82S \\x11\\xc7s\\xa7^*w\\xc1\\xdf\"\\xe4#\\xb4\\xc4P\\xba\\xcf') -**Return:** A hexadecimal __string__ seperated by colons (e.g. "82:53:20:11:C7:73:A7:5E:2A:77:C1:DF:22:E4:23:B4:C4:50:BA:CF") +**Return:** A hexadecimal __string__ separated by colons (e.g. "82:53:20:11:C7:73:A7:5E:2A:77:C1:DF:22:E4:23:B4:C4:50:BA:CF") ### `check_supported_extensions()` function @@ -45,3 +58,66 @@ fill. The certificate path and the passphrase are not passed in, the object read **Return:** The constructor returns nothing, it writes into the result dict it was given. `return_result()` hands that dict back to the module, which passes it to `exit_json()`. + +A broken or unreadable extension does not fail the module. It is skipped with `module.warn()`, +so a task can succeed with fewer extensions in the result than the certificate contains. + +## `api` + +Builds the Elasticsearch client for the two Elasticsearch modules. Requires the `elasticsearch` +Python library, currently a version below 9. + +### `HAS_ELASTICSEARCH` and `ELASTICSEARCH_IMPORT_ERROR` + +Set while the file is imported. `HAS_ELASTICSEARCH` is a __bool__, +`ELASTICSEARCH_IMPORT_ERROR` holds the formatted traceback as a __string__ or `None`. A module +checks the flag and reports the traceback, so a missing library produces a readable message +instead of an import error. + +### `Api.new_client_basic_auth()` static method + +Returns a connected `Elasticsearch` client authenticated with username and password. + +**Parameter:** Five keyword arguments: `host` (URL including protocol and port), `auth_user`, +`auth_pass`, `ca_certs` (path to a CA file, or `None`) and `verify_certs`. + +**Return:** An `Elasticsearch` __object__. Raises `ImportError` when the library is missing. + +The SSL context it builds sets `check_hostname` and `verify_mode` to false before `verify_certs` +is handed to the client, so the hostname in the certificate is not checked. + +## `elasticsearch_role` + +### `Role()` object + +The whole logic of the `elasticsearch_role` module. Constructing it already does the work: it +builds the client, decides between create, update and delete, and fills the result dict. + +**Parameter:** Ten keyword arguments: the result __dict__ to fill, `role_name`, `cluster`, +`indicies`, `state`, and the connection arguments `host`, `auth_user`, `auth_pass`, +`verify_certs` and `ca_certs`. + +**Return:** The constructor returns nothing. `return_result()` hands the result dict back to the +module, which passes it to `exit_json()`. + +Changes are detected against the API, not against the arguments. `handle_present()` reads the +role, writes it, and compares the response before and after, so `changed` stays false when the +role already looked that way. `handle_absent()` only deletes a role that the API reports as +existing. + +## `elasticsearch_user` + +### `User()` object + +The whole logic of the `elasticsearch_user` module, built the same way as `Role()` above. + +**Parameter:** Thirteen keyword arguments: the result __dict__ to fill, `user_name`, +`full_name`, `password`, `email`, `roles`, `enabled`, `state`, and the connection arguments +`host`, `auth_user`, `auth_pass`, `verify_certs` and `ca_certs`. + +**Return:** The constructor returns nothing. `return_result()` hands the result dict back to the +module, which passes it to `exit_json()`. + +Changes are detected the same way, by comparing the API response before and after the write. +A password cannot be read back, so setting the same password again is not reported as a change, +while setting a different one is not detected either. From a02d84a6855bc1c258d3444b4d2838fe337aba58 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 7 Sep 2026 15:33:15 +0200 Subject: [PATCH 35/53] State per component which OSS packages exist --- docs/getting-started.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index dbf91685..145d3112 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -29,7 +29,9 @@ The default Beats configuration collects file system logs written by `rsyslog`, example playbook installs it. Without syslog you will not receive any messages with the default configuration. If you do not want that, change the configuration of the beats role. -The OSS variant without X-Pack features is only available for Elastic Stack `7`. +`elasticstack_variant: oss` is not available for the whole stack on every release. OSS +Elasticsearch and Kibana exist only up to Elastic Stack `7`, while OSS Logstash and Beats are +available on later releases too. ## Variables From c064519a817e82202fca1571844677eff28201db Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 7 Sep 2026 15:33:24 +0200 Subject: [PATCH 36/53] Name the two package variants correctly in the README --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3ca9f6f2..4c0d45ee 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # Ansible Collection - netways.elasticstack This collection installs and manages the Elastic Stack. It provides roles for every component -of the Stack. Furthermore, it is possible to differentiate between Enterprise or OSS releases. +of the Stack, and it can install either the default Elastic distribution or the Apache-licensed +OSS packages. Every role can be used on its own or together with the others. Running a single role against an existing or external cluster works, but a few of those paths are still incomplete. See the open From 518c3e36b09c3a74ad33929006ba0c8a98518a24 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Tue, 8 Sep 2026 12:21:38 +0200 Subject: [PATCH 37/53] Declare elasticstack_kibana_host --- roles/elasticstack/README.md | 1 + roles/elasticstack/defaults/main.yml | 1 + roles/elasticstack/meta/argument_specs.yml | 8 ++++++++ roles/kibana/templates/kibana.yml.j2 | 2 +- 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/roles/elasticstack/README.md b/roles/elasticstack/README.md index 132a2cb5..ca7eedb4 100644 --- a/roles/elasticstack/README.md +++ b/roles/elasticstack/README.md @@ -44,6 +44,7 @@ The role runs automatically the first time any component role is applied to a ho | `elasticstack_logstash_group_name` | `str` | `"logstash"` | — | Inventory group that holds the Logstash hosts. | | `elasticstack_kibana_group_name` | `str` | `"kibana"` | — | Inventory group that holds the Kibana hosts. | | `elasticstack_elasticsearch_http_port` | `int` | `9200` | — | Elasticsearch HTTP API port. | +| `elasticstack_kibana_host` | `str` | `"{{ ansible_fqdn }}"` | — | Host name Kibana publishes itself under (server.publicBaseUrl). Defaults to the fully qualified domain name of the Kibana host. Set it when users reach Kibana under a different name, for example behind a reverse proxy or a load balancer. | | `elasticstack_kibana_port` | `int` | `5601` | — | Kibana HTTP port (also used to build the public base URL). | | `elasticstack_beats_port` | `int` | `5044` | — | Port of the Logstash Beats input that the Beats ship to. | | `elasticstack_enable_repos` | `bool` | `true` | — | Configure the Elastic package repositories (see the repos role). | diff --git a/roles/elasticstack/defaults/main.yml b/roles/elasticstack/defaults/main.yml index 4aaf3291..21219ca1 100644 --- a/roles/elasticstack/defaults/main.yml +++ b/roles/elasticstack/defaults/main.yml @@ -16,6 +16,7 @@ elasticstack_elasticsearch_http_port: 9200 elasticstack_enable_repos: true elasticstack_full_stack: true elasticstack_initial_passwords: /usr/share/elasticsearch/initial_passwords +elasticstack_kibana_host: "{{ ansible_fqdn }}" elasticstack_kibana_port: 5601 elasticstack_override_beats_tls: false elasticstack_release: 8 diff --git a/roles/elasticstack/meta/argument_specs.yml b/roles/elasticstack/meta/argument_specs.yml index 52e912f9..e7d26978 100644 --- a/roles/elasticstack/meta/argument_specs.yml +++ b/roles/elasticstack/meta/argument_specs.yml @@ -87,6 +87,14 @@ argument_specs: default: 9200 description: Elasticsearch HTTP API port. + elasticstack_kibana_host: + type: str + default: "{{ ansible_fqdn }}" + description: + Host name Kibana publishes itself under (server.publicBaseUrl). Defaults to the + fully qualified domain name of the Kibana host. Set it when users reach Kibana + under a different name, for example behind a reverse proxy or a load balancer. + elasticstack_kibana_port: type: int default: 5601 diff --git a/roles/kibana/templates/kibana.yml.j2 b/roles/kibana/templates/kibana.yml.j2 index 3ec1e288..154de717 100644 --- a/roles/kibana/templates/kibana.yml.j2 +++ b/roles/kibana/templates/kibana.yml.j2 @@ -1,5 +1,5 @@ server.host: "0.0.0.0" -server.publicBaseUrl: "http{% if kibana_tls | bool %}s{% endif %}://{{ elasticstack_kibana_host | default( ansible_fqdn ) }}:{{ elasticstack_kibana_port }}" +server.publicBaseUrl: "http{% if kibana_tls | bool %}s{% endif %}://{{ elasticstack_kibana_host }}:{{ elasticstack_kibana_port }}" {% if elasticstack_full_stack is defined and elasticstack_full_stack | bool and elasticstack_variant == "elastic" %} {%- if kibana_sniff_on_start | bool %} From 5f091e5e7cd4407e7041d9ee3bc1a1fcbf2ba6de Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Tue, 8 Sep 2026 13:04:36 +0200 Subject: [PATCH 38/53] Mark up the option reference in the elasticsearch_user documentation --- plugins/modules/elasticsearch_user.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/modules/elasticsearch_user.py b/plugins/modules/elasticsearch_user.py index f70ff6a0..32c96a8e 100644 --- a/plugins/modules/elasticsearch_user.py +++ b/plugins/modules/elasticsearch_user.py @@ -29,7 +29,7 @@ type: str required: false password: - description: Password for the user. Required when (state=present). + description: Password for the user. Required when O(state=present). type: str required: false default: null @@ -38,7 +38,7 @@ type: str required: false roles: - description: List of roles assigned to the user. Required when (state=present). + description: List of roles assigned to the user. Required when O(state=present). type: list elements: str required: false From d06d8d0c9d5b919bb83747c40f3eee19112fb392 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Tue, 8 Sep 2026 13:05:01 +0200 Subject: [PATCH 39/53] Fix a typo in the user update message --- plugins/module_utils/elasticsearch_user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/module_utils/elasticsearch_user.py b/plugins/module_utils/elasticsearch_user.py index 455724b5..b3caac58 100644 --- a/plugins/module_utils/elasticsearch_user.py +++ b/plugins/module_utils/elasticsearch_user.py @@ -61,7 +61,7 @@ def handle_present(self): if pre_user.raw != self.get().raw: self.result['changed'] = True - self.result['msg'] = self.user_name + " has beed updated" + self.result['msg'] = self.user_name + " has been updated" return From 1942944015b116e127008aa7a1d5c3ada9e18327 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Tue, 8 Sep 2026 13:05:34 +0200 Subject: [PATCH 40/53] Point the license header at the file that exists --- plugins/module_utils/api.py | 2 +- plugins/module_utils/certs.py | 2 +- plugins/module_utils/elasticsearch_role.py | 2 +- plugins/module_utils/elasticsearch_user.py | 2 +- plugins/modules/cert_info.py | 2 +- plugins/modules/elasticsearch_role.py | 2 +- plugins/modules/elasticsearch_user.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/plugins/module_utils/api.py b/plugins/module_utils/api.py index 847e0c52..0871985d 100644 --- a/plugins/module_utils/api.py +++ b/plugins/module_utils/api.py @@ -1,7 +1,7 @@ # !/usr/bin/python3 # Copyright (c) 2024, Tobias Bauriedel -# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# GNU General Public License v3.0+ (see LICENSE or # https://www.gnu.org/licenses/gpl-3.0.txt) import traceback diff --git a/plugins/module_utils/certs.py b/plugins/module_utils/certs.py index 7efbbee3..78a6c970 100644 --- a/plugins/module_utils/certs.py +++ b/plugins/module_utils/certs.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Copyright (c) 2023, Daniel Patrick -# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# GNU General Public License v3.0+ (see LICENSE or # https://www.gnu.org/licenses/gpl-3.0.txt) # SPDX-License-Identifier: GPL-3.0-or-later diff --git a/plugins/module_utils/elasticsearch_role.py b/plugins/module_utils/elasticsearch_role.py index 772eaa71..2d35ea35 100644 --- a/plugins/module_utils/elasticsearch_role.py +++ b/plugins/module_utils/elasticsearch_role.py @@ -1,5 +1,5 @@ # Copyright (c) 2024, Tobias Bauriedel -# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# GNU General Public License v3.0+ (see LICENSE or # https://www.gnu.org/licenses/gpl-3.0.txt) from ansible_collections.netways.elasticstack.plugins.module_utils.api import ( diff --git a/plugins/module_utils/elasticsearch_user.py b/plugins/module_utils/elasticsearch_user.py index b3caac58..04197cd2 100644 --- a/plugins/module_utils/elasticsearch_user.py +++ b/plugins/module_utils/elasticsearch_user.py @@ -1,5 +1,5 @@ # Copyright (c) 2024, Tobias Bauriedel -# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# GNU General Public License v3.0+ (see LICENSE or # https://www.gnu.org/licenses/gpl-3.0.txt) from ansible_collections.netways.elasticstack.plugins.module_utils.api import ( diff --git a/plugins/modules/cert_info.py b/plugins/modules/cert_info.py index 8d33971e..2632db78 100644 --- a/plugins/modules/cert_info.py +++ b/plugins/modules/cert_info.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # Copyright (c) 2023, Daniel Patrick -# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# GNU General Public License v3.0+ (see LICENSE or # https://www.gnu.org/licenses/gpl-3.0.txt) # SPDX-License-Identifier: GPL-3.0-or-later diff --git a/plugins/modules/elasticsearch_role.py b/plugins/modules/elasticsearch_role.py index df3749a7..d8916215 100644 --- a/plugins/modules/elasticsearch_role.py +++ b/plugins/modules/elasticsearch_role.py @@ -1,7 +1,7 @@ #!/usr/bin/python # Copyright (c) 2024, Tobias Bauriedel -# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# GNU General Public License v3.0+ (see LICENSE or # https://www.gnu.org/licenses/gpl-3.0.txt) from __future__ import (absolute_import, division, print_function) diff --git a/plugins/modules/elasticsearch_user.py b/plugins/modules/elasticsearch_user.py index 32c96a8e..50e4bdbb 100644 --- a/plugins/modules/elasticsearch_user.py +++ b/plugins/modules/elasticsearch_user.py @@ -1,7 +1,7 @@ #!/usr/bin/python # Copyright (c) 2024, Tobias Bauriedel -# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# GNU General Public License v3.0+ (see LICENSE or # https://www.gnu.org/licenses/gpl-3.0.txt) from __future__ import (absolute_import, division, print_function) From 4160f86eddf3e104a65afea6a2d5a2df9de422c6 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Tue, 8 Sep 2026 13:07:58 +0200 Subject: [PATCH 41/53] Use the collections keyword consistently in the examples --- README.md | 9 +++++++-- docs/getting-started.md | 24 +++++++++++++++++++----- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 4c0d45ee..7ccc98b0 100644 --- a/README.md +++ b/README.md @@ -120,15 +120,20 @@ the current state of `main` instead of a release: ansible-galaxy collection install git+https://github.com/NETWAYS/ansible-collection-elasticstack.git ``` -Then refer to the roles by their fully qualified name: +Then name the collection in the play and refer to the roles by their short name: ```yaml - hosts: elasticsearch become: true + collections: + - netways.elasticstack roles: - - netways.elasticstack.elasticsearch + - elasticsearch ``` +Fully qualified role names such as `netways.elasticstack.elasticsearch` work as well and need no +`collections` entry. + The execution order of the roles matters, and the roles interact with hosts from other inventory groups. Read [Getting started](docs/getting-started.md) before your first run. diff --git a/docs/getting-started.md b/docs/getting-started.md index 145d3112..71618201 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -57,45 +57,55 @@ options. - hosts: all # remote_user: my_username become: true + collections: + - netways.elasticstack vars: elasticstack_variant: elastic # oss # elasticstack_release: 8 roles: - - netways.elasticstack.repos + - repos - hosts: elasticsearch # remote_user: my_username become: true + collections: + - netways.elasticstack vars: elasticstack_variant: elastic # oss elasticsearch_jna_workaround: true # elasticstack_release: 8 roles: - - netways.elasticstack.elasticsearch + - elasticsearch - hosts: logstash # remote_user: my_username become: true + collections: + - netways.elasticstack vars: elasticstack_variant: elastic # oss elasticstack_override_beats_tls: true # elasticstack_release: 8 roles: - geerlingguy.redis - - netways.elasticstack.logstash + - logstash - hosts: kibana # remote_user: my_username become: true + collections: + - netways.elasticstack vars: elasticstack_variant: elastic # oss # elasticstack_release: 8 roles: - - netways.elasticstack.kibana + - kibana - hosts: all # remote_user: my_username become: true + collections: + - netways.elasticstack vars: elasticstack_variant: elastic # oss elasticstack_override_beats_tls: true @@ -110,9 +120,13 @@ options. state: started enabled: true roles: - - netways.elasticstack.beats + - beats ``` +The `collections` entry only makes the short role names resolvable. It is per play, which is why +it appears in every one of them. Inside a role it has no effect, so keep using fully qualified +module names such as `ansible.builtin.package` in your own tasks. + ## Generated passwords Passwords for the built-in users are generated on the CA host during the first run. They are From 2bd90e5bf3ea1c585ec6367db51cd4e82aaf3ead Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Tue, 8 Sep 2026 13:08:31 +0200 Subject: [PATCH 42/53] Use the collections keyword in the per-role version example --- docs/upgrades.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/upgrades.md b/docs/upgrades.md index b1736414..3ccff87a 100644 --- a/docs/upgrades.md +++ b/docs/upgrades.md @@ -25,11 +25,13 @@ For the OSS variant see `elasticstack_variant`, described with the Both variables can be set per role, which is how you keep components on different versions: ```yaml +collections: + - netways.elasticstack roles: - - role: netways.elasticstack.kibana + - role: kibana vars: elasticstack_version: 8.7.1 - - role: netways.elasticstack.elasticsearch + - role: elasticsearch vars: elasticstack_version: 8.8.1 ``` From d6549fb402a5e403c7425e9ff188fd40070e95c6 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Tue, 8 Sep 2026 13:08:53 +0200 Subject: [PATCH 43/53] Drop the closing hashes from headings --- NOTICE.md | 4 ++-- roles/beats/docs/filebeat-inputs.md | 16 ++++++++-------- roles/logstash/docs/pipelines.md | 24 ++++++++++++------------ 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/NOTICE.md b/NOTICE.md index 8652a9e5..83d51761 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -1,6 +1,6 @@ -# Notice and Acknowledgements # +# Notice and Acknowledgements -## Sponsoring ## +## Sponsoring Here's a list of sponsors who contributed by having the collection improved via outsourcing to NETWAYS. diff --git a/roles/beats/docs/filebeat-inputs.md b/roles/beats/docs/filebeat-inputs.md index cede2166..e5a5203e 100644 --- a/roles/beats/docs/filebeat-inputs.md +++ b/roles/beats/docs/filebeat-inputs.md @@ -1,11 +1,11 @@ -# Filebeat inputs # +# Filebeat inputs Filebeat can read from several sources. Each source is turned into a Filebeat input by the role. This document describes the variables that configure them. For the meaning of the individual Filebeat options, follow the links to the official Filebeat documentation. -## Log files ## +## Log files Set with `beats_filebeat_log_input` (default `true`) and `beats_filebeat_log_inputs`. @@ -45,7 +45,7 @@ beats_filebeat_log_inputs: `negate`, `match`). See [Manage multiline messages](https://www.elastic.co/docs/reference/beats/filebeat/multiline-examples). -## Fields ## +## Fields There are two ways to add fields, and they use **different shapes**: @@ -71,7 +71,7 @@ There are two ways to add fields, and they use **different shapes**: - "team: platform" ``` -## Syslog over TCP/UDP ## +## Syslog over TCP/UDP Enable a listening syslog input with `beats_filebeat_syslog_tcp` / `beats_filebeat_syslog_udp` and set the port with `beats_filebeat_syslog_tcp_port` @@ -85,7 +85,7 @@ beats_filebeat_syslog_tcp: true beats_filebeat_syslog_tcp_port: 514 ``` -## Journald ## +## Journald Enable with `beats_filebeat_journald` (default `false`, available since Filebeat 7.16) and configure inputs with `beats_filebeat_journald_inputs`. @@ -109,7 +109,7 @@ See the [`journald`](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-journald) input. -## Docker ## +## Docker Enable with `beats_filebeat_docker` (default `false`) and select containers with `beats_filebeat_docker_ids` (default `*`). **Only works on Elastic Stack release @@ -120,13 +120,13 @@ beats_filebeat_docker: true beats_filebeat_docker_ids: "*" ``` -## MySQL/MariaDB slow log ## +## MySQL/MariaDB slow log Set `beats_filebeat_mysql_slowlog_input` to `true` to collect `/var/log/mysql/*-slow.log` with the matching multiline pattern already configured. The events are tagged with `mysql.logtype: slowquery`. -## Modules ## +## Modules `beats_filebeat_modules` is a list of Filebeat module names to enable (**experimental**, unset by default). The role runs `filebeat modules enable` diff --git a/roles/logstash/docs/pipelines.md b/roles/logstash/docs/pipelines.md index 30ad89ba..a28f04eb 100644 --- a/roles/logstash/docs/pipelines.md +++ b/roles/logstash/docs/pipelines.md @@ -1,6 +1,6 @@ -# Pipelines # +# Pipelines -## Keeping an overview ## +## Keeping an overview It can be quite difficult to stay on top of your pipeline configuration because they tend to become very complex. @@ -8,7 +8,7 @@ This collection will leave some comments about how pipelines are interconnected If you set `logstash_mermaid` to `true` (which is the default), then you will also get a new file in `/etc/logstash/pipelines.mermaid`. You can paste it into a Mermaid editor in your documentation tool or in an [online Mermaid editor](https://mermaid.live/). The same content will be available on your control node in a temporary file. You can even add arbitrary code to reflect manually managed pipelines by using `logstash_mermaid_extra`. -## Git managed ## +## Git managed `logstash_pipelines` is a **list**. Each entry needs a `name`; everything else is optional. @@ -24,9 +24,9 @@ You can add a `version` attribute to your pipeline. It defaults to `main`. You c You can also determine where Logstash stores the data of each pipeline while processing it by using the `queue_type` attribute. It defaults to `memory`; use `persisted` to write the data temporarily to disk. You can limit the maximum amount of data per pipeline with `queue_max_bytes`, which defaults to `1gb`. -## Input and Output ## +## Input and Output -### Basic configuration ### +### Basic configuration To have a single Redis input and output to your files, use this. @@ -64,11 +64,11 @@ output { } ``` -### Multiple inputs ### +### Multiple inputs Just give more inputs with `name` and `key`. Every key will be read. -### More complex configuration ### +### More complex configuration If you want a bit more control over which outputs are used, the role offers more sophisticated configuration. @@ -207,24 +207,24 @@ else { Here the `default` output only receives the events that haven't already been sent to one of the others. -## Extra configuration ## +## Extra configuration -### Congestion threshold ### +### Congestion threshold Every output can have a `congestion` option with a numerical value. If the Redis key already holds more items than the value says, the output will stop. -### Unsafe shutdown ### +### Unsafe shutdown If you need unsafe Logstash shutdowns, e.g. for testing, you can set `logstash_pipeline_unsafe_shutdown` to `true`. The variable doesn't have a default so Logstash falls back to its internal default of `false`. -## Caveats ## +## Caveats There are still some minor issues you need to keep in mind: * The default output in an `exclusive: true` setup must be the last in the YAML configuration. There's no sorting, the role simply expects the default to be the last one. * The configuration *should* work but will make no sense if you have `exclusive: true` but two or more outputs without `condition`. -## Custom pipelines ## +## Custom pipelines If you have other ways of putting pipeline code into the correct directories, you can just skip the `source` option. From 9edf011ebfa05cc5cab6cd780fbb6a6d3b251716 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Tue, 8 Sep 2026 13:09:12 +0200 Subject: [PATCH 44/53] Remove a sentence that says nothing --- docs/requirements.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/requirements.md b/docs/requirements.md index 8886c8c9..494f0e6d 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -4,8 +4,6 @@ Software requirements, supported Ansible and Python versions and external librar in the [README](../README.md#tested-with-ansible). This page describes what the roles expect from your setup. -Some of these will be refactored and disappear from this list. - ## Inventory group names The collection provides roles for several tools, and some tasks fetch hosts from a specific From cef8381eefbe801ba8f1c928c11192a4a4f2f530 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Tue, 8 Sep 2026 13:12:10 +0200 Subject: [PATCH 45/53] Name the Beats correctly in the Galaxy tags --- galaxy.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/galaxy.yml b/galaxy.yml index 57d5fbd8..569e133d 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -14,9 +14,10 @@ tags: - elasticsearch - kibana - logstash - - beat - - metricbeat + - beats - filebeat + - metricbeat + - auditbeat - monitoring dependencies: "community.general": "*" From 1cc6462cf71b0d7ed9a001b4bdbf82eb680dad03 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Fri, 11 Sep 2026 10:41:32 +0200 Subject: [PATCH 46/53] Add the changelog fragment for the release preparation --- .../fragments/prepare_galaxy_release.yml | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 changelogs/fragments/prepare_galaxy_release.yml diff --git a/changelogs/fragments/prepare_galaxy_release.yml b/changelogs/fragments/prepare_galaxy_release.yml new file mode 100644 index 00000000..93b79feb --- /dev/null +++ b/changelogs/fragments/prepare_galaxy_release.yml @@ -0,0 +1,23 @@ +minor_changes: + - The collection declares ``community.crypto`` as a dependency now, so installing it with + ``ansible-galaxy`` pulls in what the beats role needs to check certificate expiration. + - "``elasticstack_kibana_host`` is a declared and documented variable now, defaulting to the + fully qualified domain name of the Kibana host. The Kibana template referenced it ad hoc + before, without a default of its own." + - The documentation was restructured along the collection template. ``docs/01-requirements.md`` + became ``docs/requirements.md``, version pinning and upgrading moved to ``docs/upgrades.md``, + and the role pages under ``docs/`` that only linked to the role READMEs were removed. Module + documentation is read with ``ansible-doc`` instead of from copies under ``docs/``. + +trivial: + - Added the release tooling, namely an antsibull-changelog configuration, a manually triggered + workflow that publishes to Ansible Galaxy and attaches the archive to the GitHub release, a + check that every pull request brings a changelog fragment, and a contributing guide. + - Corrected the collection metadata in galaxy.yml, namely one current SPDX license identifier, + a documentation URL that resolves, the organization as the author, tags that name the Beats + correctly, and build_ignore keeping local tooling and CI files out of the built archive. + - Documented the three files under ``plugins/module_utils/`` that had no description, corrected + the parameters of ``check_supported_keys`` and ``AnalyzeCertificate``, and set + ``version_added`` of the modules to the first released version. + - Corrected factual errors in the README, removed documentation that was duplicated or only + redirected, and made the role examples use one way of addressing the roles. From 77f91f0b7236051bbea4ddc911742a416a46d2cc Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 14 Sep 2026 11:42:06 +0200 Subject: [PATCH 47/53] Remove the KICS security scan --- .github/workflows/kics.yml | 47 ------------------- .gitignore | 1 - .../fragments/prepare_galaxy_release.yml | 4 ++ galaxy.yml | 1 - molecule/elasticstack_default/prepare.yml | 5 -- roles/elasticsearch/tasks/main.yml | 5 +- 6 files changed, 5 insertions(+), 58 deletions(-) delete mode 100644 .github/workflows/kics.yml diff --git a/.github/workflows/kics.yml b/.github/workflows/kics.yml deleted file mode 100644 index 1b27515e..00000000 --- a/.github/workflows/kics.yml +++ /dev/null @@ -1,47 +0,0 @@ ---- -name: KICS Security Scan -on: - workflow_dispatch: - inputs: - logLevel: - description: 'Log level' - required: true - default: 'warning' - type: choice - options: - - info - - warning - - debug - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - -# Cancel outdated pipeline runs when a new push occurs in the pull request and a new pipeline starts. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - security_scan: - # For non-PR events (workflow_dispatch, schedule, etc.), github.event.pull_request.draft is null. - # GitHub Actions coerces null == false to true (null → 0, false → 0), so the job runs for all non-draft events. - if: github.event.pull_request.draft == false - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v7 - - name: Mkdir results-dir - # make sure results dir is created - run: mkdir -p results-dir - - name: run kics Scan - uses: Checkmarx/kics-github-action@bcd2aff52160bfe5a72608452d00326e90da7a9a # v2.1.19 - with: - # path: 'roles,plugins' - path: '.' - # fail_on: high - ignore_on_exit: results - output_formats: 'json,sarif' - output_path: results-dir - - name: Upload SARIF file - uses: github/codeql-action/upload-sarif@af56b044b5d41c317aef5d19920b3183cb4fbbec # v3 - with: - sarif_file: results-dir/results.sarif diff --git a/.gitignore b/.gitignore index 3bb36d13..0318debd 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,3 @@ __pycache__* # antsibull-changelog changelogs/.plugin-cache.yaml -results-dir diff --git a/changelogs/fragments/prepare_galaxy_release.yml b/changelogs/fragments/prepare_galaxy_release.yml index 93b79feb..9cef7dc8 100644 --- a/changelogs/fragments/prepare_galaxy_release.yml +++ b/changelogs/fragments/prepare_galaxy_release.yml @@ -21,3 +21,7 @@ trivial: ``version_added`` of the modules to the first released version. - Corrected factual errors in the README, removed documentation that was duplicated or only redirected, and made the role examples use one way of addressing the roles. + - Removed the KICS security scan. Of its 226 Ansible queries, 215 cover AWS, Azure and GCP + resources this collection does not touch, and of the remaining ones only two ever fired, + one of them a false positive caused by the query not recognising fully qualified module + names. diff --git a/galaxy.yml b/galaxy.yml index 569e133d..698cdaa3 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -29,7 +29,6 @@ issues: https://github.com/netways/ansible-collection-elasticstack/issues build_ignore: # CI, tests and local tooling - molecule - - results-dir - makefile - 'requirements*.txt' # repository housekeeping, not part of the collection diff --git a/molecule/elasticstack_default/prepare.yml b/molecule/elasticstack_default/prepare.yml index 8d8c3819..37ea6841 100644 --- a/molecule/elasticstack_default/prepare.yml +++ b/molecule/elasticstack_default/prepare.yml @@ -45,13 +45,9 @@ - unzip - systemd -# KICS complains about packages being updated. -# In this case, a mere test scenario it is OK, though. - - name: Update all installed packages RHEL ansible.builtin.yum: name: '*' -# kics-scan ignore-line state: latest update_cache: yes update_only: yes @@ -60,7 +56,6 @@ - name: Update all installed packages Debian ansible.builtin.apt: name: '*' -# kics-scan ignore-line state: latest update_cache: yes when: ansible_os_family == "Debian" diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index b5d5620c..00eb880b 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -264,16 +264,13 @@ enabled: yes register: elasticsearch_freshstart -# The comment in the following task will disable KICS security checks for this -# very line. In this state of the system we can only communicate without https -# so the finding is a false positive +# Security is off in this branch, so the API is only reachable over http. - name: Handle cluster setup without security when: not elasticsearch_security | bool block: - name: Check for cluster status without security ansible.builtin.uri: -# kics-scan ignore-line url: "http://{{ elasticsearch_api_host }}:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty" register: elasticsearch_cluster_status ignore_errors: "{{ ansible_check_mode }}" From 9b56e2ca35be53088f8c9168c5f048fba70e36fe Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 14 Sep 2026 11:47:26 +0200 Subject: [PATCH 48/53] Ignore the ansible-lint working directory --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 0318debd..20d85979 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,8 @@ __pycache__* .venv .fuse_hidden* +# ansible-lint working directory, also removed by "make cleanup" +.ansible + # antsibull-changelog changelogs/.plugin-cache.yaml From 78a4ce9939969d159a13b5f3777e24443556d2ca Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 14 Sep 2026 12:07:27 +0200 Subject: [PATCH 49/53] Change the formating of ealsticstack docs --- roles/elasticstack/README.md | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/roles/elasticstack/README.md b/roles/elasticstack/README.md index ca7eedb4..4244596c 100644 --- a/roles/elasticstack/README.md +++ b/roles/elasticstack/README.md @@ -1,20 +1,12 @@ # Ansible Role: elasticstack -The shared base role of the collection. Every component role (elasticsearch, -logstash, kibana, beats, repos) imports it automatically on first use, so you -normally do not call it directly. It installs the common dependencies and, above -all, defines the collection-wide `elasticstack_*` variables — the settings shared -across all roles: release and variant, the inventory host groups, ports, the -package repositories, and the stack CA. +The shared base role of the collection. Every component role (elasticsearch,logstash, kibana, beats, repos) imports it automatically on first use, so you normally do not call it directly. It installs the common dependencies and, above all, defines the collection-wide `elasticstack_*` variables — the settings shared across all roles: release and variant, the inventory host groups, ports, the package repositories, and the stack CA. -Set these variables once (in `group_vars`, the play, or via `-e`) and every role -picks them up. This page is the central reference for them. +Set these variables once (in `group_vars`, the play, or via `-e`) and every role picks them up. This page is the central reference for them. ## Requirements -You do not run this role directly — the component roles import it automatically. -It installs the shared Python libraries (`cryptography` and the `elasticsearch` -client) and packages (`openssl`, `unzip`) that the collection needs. +You do not run this role directly — the component roles import it automatically. It installs the shared Python libraries (`cryptography` and the `elasticsearch` client) and packages (`openssl`, `unzip`) that the collection needs. ## Example From 1821428d2e9e87554e410f07a6ec4faf26a3b5a2 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 14 Sep 2026 15:43:03 +0200 Subject: [PATCH 50/53] Stack the variable metadata under the name in the README tables --- .docsmith/readme.md.j2 | 13 ++- roles/beats/README.md | 86 ++++++++-------- roles/elasticsearch/README.md | 88 ++++++++--------- roles/elasticstack/README.md | 64 ++++++------ roles/kibana/README.md | 58 +++++------ roles/logstash/README.md | 178 +++++++++++++++++----------------- 6 files changed, 245 insertions(+), 242 deletions(-) diff --git a/.docsmith/readme.md.j2 b/.docsmith/readme.md.j2 index ff54a137..12be8b55 100644 --- a/.docsmith/readme.md.j2 +++ b/.docsmith/readme.md.j2 @@ -1,11 +1,14 @@ {# Compact README variable reference for ansible-docsmith. Shared by all roles. - Renders ONE table; nested sub-options (e.g. logstash_pipelines fields) appear as - indented rows. No per-variable detail sections, no table of contents. + Renders ONE two-column table. Type, required, default and choices are stacked + under the variable name so that only two columns compete for the page width and + the description never needs horizontal scrolling. Nested sub-options (e.g. + logstash_pipelines fields) appear as indented rows. Use with: --template-readme .docsmith/readme.md.j2 #} {% macro row(name, spec, depth) -%} {% set pad = '    ' * depth %} {% set mark = '↳ ' if depth > 0 else '' %} -| {{ pad }}{{ mark }}`{{ name | ansible_escape }}` | `{{ spec.type }}`{% if spec.elements %} of `{{ spec.elements }}`{% endif %} | {% if spec.options %}…{% else %}{{ spec.default | format_default | replace('|', '\\|') }}{% endif %} | {% if spec.choices %}{{ spec.choices | map('string') | map('code_escape') | join(', ') }}{% else %}—{% endif %} | {{ spec.description | format_table_description(name, 0) }} | +{% set default = spec.default | format_default | replace('|', '\\|') -%} +| {{ pad }}{{ mark }}`{{ name | ansible_escape }}`
**Type**: `{{ spec.type }}`{% if spec.elements %} of `{{ spec.elements }}`{% endif %}{% if spec.required %}
**Required**: Yes{% endif %}{% if not spec.options and default != 'N/A' %}
**Default**: {{ default }}{% endif %}{% if spec.choices %}
**Choices**: {{ spec.choices | map('string') | map('code_escape') | join(', ') }}{% endif %} | {{ spec.description | format_table_description(name, 0) }} | {% for sn, ss in (spec.options or {}).items() %} {{ row(sn, ss, depth + 1) }} {%- endfor %} @@ -13,8 +16,8 @@ ## Role variables {% if has_options %} -| Variable | Type | Default | Choices | Description | -|----------|------|---------|---------|-------------| +| Variable | Description | +|----------|-------------| {% for name, spec in options.items() %} {{ row(name, spec, 0) }} {%- endfor %} diff --git a/roles/beats/README.md b/roles/beats/README.md index 7816e301..0e961017 100644 --- a/roles/beats/README.md +++ b/roles/beats/README.md @@ -41,49 +41,49 @@ Run only parts of the role with `--tags`: ## Role variables -| Variable | Type | Default | Choices | Description | -|----------|------|---------|---------|-------------| -| `beats_filebeat` | `bool` | `true` | — | Install and manage Filebeat. | -| `beats_auditbeat` | `bool` | `false` | — | Install and manage Auditbeat. | -| `beats_metricbeat` | `bool` | `false` | — | Install and manage Metricbeat. | -| `beats_target_hosts` | `list` of `str` | `['localhost']` | — | Hosts the Beats ship to. Only used when the role runs standalone; with the other Elastic Stack roles the targets are determined automatically. | -| `beats_fields` | `list` of `str` | N/A | — | Global fields added to the log and syslog (tcp/udp) inputs (not to the mysql, journald or docker inputs), given as a list of "key: value" strings — the global counterpart to the per-input fields. Unset by default. See the Filebeat inputs documentation. | -| `beats_logging` | `str` | `"file"` | — | Where the Beats log. Set to "file" to log into beats_logpath; any other value leaves the Beats built-in logging. | -| `beats_loglevel` | `str` | `"info"` | — | Log level for all Beats. | -| `beats_logpath` | `str` | `"/var/log/beats"` | — | Directory for the log files when beats_logging is "file". | -| `beats_filebeat_enable` | `bool` | `true` | — | Start and enable the Filebeat service. | -| `beats_filebeat_output` | `str` | `"logstash"` | `logstash`, `elasticsearch` | Where Filebeat sends its events. | -| `beats_filebeat_elastic_monitoring` | `bool` | `false` | — | Report Filebeat monitoring data through the Elastic Stack monitoring features. | -| `beats_filebeat_loadbalance` | `bool` | `true` | — | Enable load balancing for the Filebeat Logstash output. | -| `beats_filebeat_log_input` | `bool` | `true` | — | Read log files with the inputs from beats_filebeat_log_inputs. | -| `beats_filebeat_log_inputs` | `dict` | `{'messages': {'paths': ['/var/log/messages', '/var/log/syslog']}}` | — | Log files to read, keyed by a free name that becomes the filestream id. Each entry has paths and optional fields and multiline settings. See the Filebeat inputs documentation. | -| `beats_filebeat_syslog_tcp` | `bool` | `false` | — | Enable a TCP syslog input. | -| `beats_filebeat_syslog_tcp_port` | `int` | `514` | — | Port of the TCP syslog input. | -| `beats_filebeat_syslog_udp` | `bool` | `false` | — | Enable a UDP syslog input. | -| `beats_filebeat_syslog_udp_port` | `int` | `514` | — | Port of the UDP syslog input. | -| `beats_filebeat_journald` | `bool` | `false` | — | Collect logs from journald. Available since Filebeat 7.16. | -| `beats_filebeat_journald_inputs` | `dict` | `{'everything': {'id': 'everything'}}` | — | Journald inputs, keyed by a free name. Each entry has an id and optional include_matches filters. See the Filebeat inputs documentation. | -| `beats_filebeat_docker` | `bool` | `false` | — | Collect Docker container logs. Only works on Elastic Stack release 7. | -| `beats_filebeat_docker_ids` | `str` | `"*"` | — | IDs of the containers to collect logs from. | -| `beats_filebeat_mysql_slowlog_input` | `bool` | `false` | — | Collect the MySQL/MariaDB slow query log, including multiline handling. | -| `beats_filebeat_modules` | `list` of `str` | N/A | — | List of Filebeat modules to enable (experimental). Unset by default. | -| `beats_auditbeat_enable` | `bool` | `true` | — | Start and enable the Auditbeat service. | -| `beats_auditbeat_output` | `str` | `"elasticsearch"` | `logstash`, `elasticsearch` | Where Auditbeat sends its events. | -| `beats_auditbeat_setup` | `bool` | `true` | — | Run the Auditbeat setup (index management, ingest pipelines and Kibana dashboards). Only effective with the elasticsearch output; loading the dashboards additionally requires Kibana to be reachable. | -| `beats_auditbeat_loadbalance` | `bool` | `true` | — | Enable load balancing for the Auditbeat Logstash output. | -| `beats_metricbeat_enable` | `bool` | `true` | — | Start and enable the Metricbeat service. | -| `beats_metricbeat_output` | `str` | `"elasticsearch"` | `logstash`, `elasticsearch` | Where Metricbeat sends its events. | -| `beats_metricbeat_modules` | `list` of `str` | `['system']` | — | Metricbeat modules to enable. | -| `beats_metricbeat_loadbalance` | `bool` | `true` | — | Enable load balancing for the Metricbeat Logstash output. | -| `beats_security` | `bool` | `false` | — | Activate TLS for the connections to the targets. Works with the other roles and elasticstack_full_stack to create certificates automatically, or with the beats_tls_* variables for custom certificates. | -| `beats_ca_dir` | `str` | N/A | — | Base directory for the Beats certificate, key and CA file. The role fills it per variant (/opt/ca standalone, /etc/beats/certs in a full stack) if you did not set it. | -| `beats_tls_key` | `str` | `"{{ beats_ca_dir \| default('') }}/{{ inventory_hostname }}-beats.key"` | — | Path to the Beats private key. Defaults under beats_ca_dir; set it to bring your own. | -| `beats_tls_cert` | `str` | `"{{ beats_ca_dir \| default('') }}/{{ inventory_hostname }}-beats.crt"` | — | Path to the Beats certificate. Defaults under beats_ca_dir; set it to bring your own. | -| `beats_tls_cacert` | `str` | `"{{ beats_ca_dir \| default('') }}/ca.crt"` | — | Path to the CA certificate Beats trusts. Defaults under beats_ca_dir; set it to bring your own. | -| `beats_tls_key_passphrase` | `str` | `"BeatsChangeMe"` | — | Passphrase of the Beats private key. | -| `beats_cert_validity_period` | `int` | `1095` | — | Number of days the generated certificates are valid. | -| `beats_cert_expiration_buffer` | `str` | `"+30d"` | — | Renew the certificate when it would expire within this period. Uses the community.crypto check_period format (e.g. "+30d"), not a plain number. | -| `beats_cert_will_expire_soon` | `bool` | `false` | — | Set to true to force renewal of the Beats certificate. Alternatively run the playbook with the renew_beats_cert tag. | +| Variable | Description | +|----------|-------------| +| `beats_filebeat`
**Type**: `bool`
**Default**: `true` | Install and manage Filebeat. | +| `beats_auditbeat`
**Type**: `bool`
**Default**: `false` | Install and manage Auditbeat. | +| `beats_metricbeat`
**Type**: `bool`
**Default**: `false` | Install and manage Metricbeat. | +| `beats_target_hosts`
**Type**: `list` of `str`
**Default**: `['localhost']` | Hosts the Beats ship to. Only used when the role runs standalone; with the other Elastic Stack roles the targets are determined automatically. | +| `beats_fields`
**Type**: `list` of `str` | Global fields added to the log and syslog (tcp/udp) inputs (not to the mysql, journald or docker inputs), given as a list of "key: value" strings — the global counterpart to the per-input fields. Unset by default. See the Filebeat inputs documentation. | +| `beats_logging`
**Type**: `str`
**Default**: `"file"` | Where the Beats log. Set to "file" to log into beats_logpath; any other value leaves the Beats built-in logging. | +| `beats_loglevel`
**Type**: `str`
**Default**: `"info"` | Log level for all Beats. | +| `beats_logpath`
**Type**: `str`
**Default**: `"/var/log/beats"` | Directory for the log files when beats_logging is "file". | +| `beats_filebeat_enable`
**Type**: `bool`
**Default**: `true` | Start and enable the Filebeat service. | +| `beats_filebeat_output`
**Type**: `str`
**Default**: `"logstash"`
**Choices**: `logstash`, `elasticsearch` | Where Filebeat sends its events. | +| `beats_filebeat_elastic_monitoring`
**Type**: `bool`
**Default**: `false` | Report Filebeat monitoring data through the Elastic Stack monitoring features. | +| `beats_filebeat_loadbalance`
**Type**: `bool`
**Default**: `true` | Enable load balancing for the Filebeat Logstash output. | +| `beats_filebeat_log_input`
**Type**: `bool`
**Default**: `true` | Read log files with the inputs from beats_filebeat_log_inputs. | +| `beats_filebeat_log_inputs`
**Type**: `dict`
**Default**: `{'messages': {'paths': ['/var/log/messages', '/var/log/syslog']}}` | Log files to read, keyed by a free name that becomes the filestream id. Each entry has paths and optional fields and multiline settings. See the Filebeat inputs documentation. | +| `beats_filebeat_syslog_tcp`
**Type**: `bool`
**Default**: `false` | Enable a TCP syslog input. | +| `beats_filebeat_syslog_tcp_port`
**Type**: `int`
**Default**: `514` | Port of the TCP syslog input. | +| `beats_filebeat_syslog_udp`
**Type**: `bool`
**Default**: `false` | Enable a UDP syslog input. | +| `beats_filebeat_syslog_udp_port`
**Type**: `int`
**Default**: `514` | Port of the UDP syslog input. | +| `beats_filebeat_journald`
**Type**: `bool`
**Default**: `false` | Collect logs from journald. Available since Filebeat 7.16. | +| `beats_filebeat_journald_inputs`
**Type**: `dict`
**Default**: `{'everything': {'id': 'everything'}}` | Journald inputs, keyed by a free name. Each entry has an id and optional include_matches filters. See the Filebeat inputs documentation. | +| `beats_filebeat_docker`
**Type**: `bool`
**Default**: `false` | Collect Docker container logs. Only works on Elastic Stack release 7. | +| `beats_filebeat_docker_ids`
**Type**: `str`
**Default**: `"*"` | IDs of the containers to collect logs from. | +| `beats_filebeat_mysql_slowlog_input`
**Type**: `bool`
**Default**: `false` | Collect the MySQL/MariaDB slow query log, including multiline handling. | +| `beats_filebeat_modules`
**Type**: `list` of `str` | List of Filebeat modules to enable (experimental). Unset by default. | +| `beats_auditbeat_enable`
**Type**: `bool`
**Default**: `true` | Start and enable the Auditbeat service. | +| `beats_auditbeat_output`
**Type**: `str`
**Default**: `"elasticsearch"`
**Choices**: `logstash`, `elasticsearch` | Where Auditbeat sends its events. | +| `beats_auditbeat_setup`
**Type**: `bool`
**Default**: `true` | Run the Auditbeat setup (index management, ingest pipelines and Kibana dashboards). Only effective with the elasticsearch output; loading the dashboards additionally requires Kibana to be reachable. | +| `beats_auditbeat_loadbalance`
**Type**: `bool`
**Default**: `true` | Enable load balancing for the Auditbeat Logstash output. | +| `beats_metricbeat_enable`
**Type**: `bool`
**Default**: `true` | Start and enable the Metricbeat service. | +| `beats_metricbeat_output`
**Type**: `str`
**Default**: `"elasticsearch"`
**Choices**: `logstash`, `elasticsearch` | Where Metricbeat sends its events. | +| `beats_metricbeat_modules`
**Type**: `list` of `str`
**Default**: `['system']` | Metricbeat modules to enable. | +| `beats_metricbeat_loadbalance`
**Type**: `bool`
**Default**: `true` | Enable load balancing for the Metricbeat Logstash output. | +| `beats_security`
**Type**: `bool`
**Default**: `false` | Activate TLS for the connections to the targets. Works with the other roles and elasticstack_full_stack to create certificates automatically, or with the beats_tls_* variables for custom certificates. | +| `beats_ca_dir`
**Type**: `str` | Base directory for the Beats certificate, key and CA file. The role fills it per variant (/opt/ca standalone, /etc/beats/certs in a full stack) if you did not set it. | +| `beats_tls_key`
**Type**: `str`
**Default**: `"{{ beats_ca_dir \| default('') }}/{{ inventory_hostname }}-beats.key"` | Path to the Beats private key. Defaults under beats_ca_dir; set it to bring your own. | +| `beats_tls_cert`
**Type**: `str`
**Default**: `"{{ beats_ca_dir \| default('') }}/{{ inventory_hostname }}-beats.crt"` | Path to the Beats certificate. Defaults under beats_ca_dir; set it to bring your own. | +| `beats_tls_cacert`
**Type**: `str`
**Default**: `"{{ beats_ca_dir \| default('') }}/ca.crt"` | Path to the CA certificate Beats trusts. Defaults under beats_ca_dir; set it to bring your own. | +| `beats_tls_key_passphrase`
**Type**: `str`
**Default**: `"BeatsChangeMe"` | Passphrase of the Beats private key. | +| `beats_cert_validity_period`
**Type**: `int`
**Default**: `1095` | Number of days the generated certificates are valid. | +| `beats_cert_expiration_buffer`
**Type**: `str`
**Default**: `"+30d"` | Renew the certificate when it would expire within this period. Uses the community.crypto check_period format (e.g. "+30d"), not a plain number. | +| `beats_cert_will_expire_soon`
**Type**: `bool`
**Default**: `false` | Set to true to force renewal of the Beats certificate. Alternatively run the playbook with the renew_beats_cert tag. | diff --git a/roles/elasticsearch/README.md b/roles/elasticsearch/README.md index d039f979..bac7afc0 100644 --- a/roles/elasticsearch/README.md +++ b/roles/elasticsearch/README.md @@ -43,50 +43,50 @@ Run only parts of the role with `--tags`: ## Role variables -| Variable | Type | Default | Choices | Description | -|----------|------|---------|---------|-------------| -| `elasticsearch_enable` | `bool` | `true` | — | Start and enable the Elasticsearch service. | -| `elasticsearch_manage_yaml` | `bool` | `true` | — | Manage and overwrite elasticsearch.yml. | -| `elasticsearch_config_backup` | `bool` | `false` | — | Keep a backup of elasticsearch.yml whenever the role changes it. | -| `elasticsearch_clustername` | `str` | `"elasticsearch"` | — | Name of the Elasticsearch cluster (cluster.name). | -| `elasticsearch_nodename` | `str` | N/A | — | Name of this Elasticsearch node (node.name). Defaults to the host's ansible_hostname. | -| `elasticsearch_node_types` | `list` of `str` | N/A | — | Node roles for this node (node.roles), e.g. master, data, ingest. When unset, Elasticsearch uses its own defaults. See the Elasticsearch node-roles documentation for valid values. | -| `elasticsearch_datapath` | `str` | `"/var/lib/elasticsearch"` | — | Directory where Elasticsearch stores its data (path.data). | -| `elasticsearch_create_datapath` | `bool` | `false` | — | Create the data directory if it does not exist. Useful when you change elasticsearch_datapath. | -| `elasticsearch_logpath` | `str` | `"/var/log/elasticsearch"` | — | Directory where Elasticsearch stores its logs (path.logs). | -| `elasticsearch_create_logpath` | `bool` | `false` | — | Create the log directory if it does not exist. Useful when you change elasticsearch_logpath. | -| `elasticsearch_conf_dir` | `str` | `"/etc/elasticsearch/"` | — | Elasticsearch configuration directory. Used to place the jvm.options.d drop-ins. | -| `elasticsearch_group` | `str` | `"elasticsearch"` | — | OS group that owns the jvm.options.d drop-ins. The service reads them through this group, so it must match the group the package runs the service as (elasticsearch). Changing it is not recommended. | -| `elasticsearch_fs_repo` | `list` of `str` | N/A | — | Filesystem paths registered as snapshot repositories (path.repo). Every node must reach the same share under the same path. Unset by default. | -| `elasticsearch_heap` | `str` | `"{{ [[(ansible_memtotal_mb // 1024) // 2, 30] \| min, 1] \| max }}"` | — | JVM heap size in GB. Sets both -Xms and -Xmx via a jvm.options.d drop-in. The default is half of the host memory, capped between 1 and 30 GB. Set to false to leave heap sizing to Elasticsearch/the JVM. | -| `elasticsearch_heap_dump_path` | `str` | `"/var/lib/elasticsearch"` | — | Directory for JVM heap dumps (-XX:HeapDumpPath). | -| `elasticsearch_jvm_custom_parameters` | `raw` | `""` | — | Extra JVM options written to a jvm.options.d drop-in, provided as a list of option lines (e.g. ["-XX:+UseG1GC"]). Empty by default. | -| `elasticsearch_api_host` | `str` | `"localhost"` | — | Host or IP the role uses for its own Elasticsearch API connection checks. | -| `elasticsearch_network_host` | `str` | N/A | — | Network addresses Elasticsearch binds to (network.host). Accepts IPs, interface names, or Elasticsearch special values, e.g. "_ens190_,_local_". Defaults to "_local_", "_site_" when unset. | -| `elasticsearch_http_publish_host` | `str` | N/A | — | Address HTTP clients use to reach this node when sniffing (http.publish_host). Unset by default. | -| `elasticsearch_http_publish_port` | `int` | N/A | — | HTTP publish port, only needed when it must differ from http.port (http.publish_port). Unset by default. | -| `elasticsearch_transport_port` | `int` | N/A | — | Port for inter-node transport communication (transport.port). Unset by default. | -| `elasticsearch_seed_hosts` | `list` of `str` | N/A | — | Seed hosts for cluster discovery (discovery.seed_hosts). Overrides the list the role builds from the elasticsearch group. Unset by default. | -| `elasticsearch_check_calculation` | `bool` | `false` | — | When true, Elasticsearch is not installed, configured, or started: the role runs only the node-role validation and then stops each host (meta: end_host). The validation derives each node's role (master/data/other) from elasticsearch_node_types, groups the nodes, and asserts an odd number of master-eligible nodes (required for a valid quorum). Requires elasticsearch_node_types. Useful as a pre-flight inventory/topology check (before scaling or in CI) without a full deployment. | -| `elasticsearch_disable_systemcallfilterchecks` | `bool` | `false` | — | Disable the system call filter check (bootstrap.system_call_filter). Has a security impact and is only applied on Elastic Stack release 7. | -| `elasticsearch_jna_workaround` | `bool` | `false` | — | Work around systems where JNA cannot load its native library. WARNING this rewrites the sysconfig/default file and overwrites ES_JAVA_OPTS. | -| `elasticsearch_unsafe_upgrade_restart` | `bool` | `false` | — | During upgrades, update the package first and then restart the service, instead of stopping it, upgrading and starting again. Only for non-production clusters, use at your own risk. | -| `elasticsearch_security` | `bool` | `true` | — | Enable X-Pack security (transport TLS, keystore, CA and certificates). Required on Elastic Stack release 8 and later with the elastic variant. | -| `elasticsearch_http_security` | `bool` | `true` | — | Enable TLS on the HTTP layer (xpack.security.http.ssl). Only effective when elasticsearch_security is enabled. | -| `elasticsearch_security_enrollment` | `bool` | N/A | — | Control node/Kibana enrollment on a node autoconfigured for security (xpack.security.enrollment.enabled). Unset by default. | -| `elasticsearch_ml_enabled` | `bool` | `true` | — | Enable Machine Learning (xpack.ml.enabled). Only applied with the elastic variant. | -| `elasticsearch_monitoring_enabled` | `bool` | `true` | — | Enable X-Pack monitoring collection (xpack.monitoring.collection.enabled). Only applied on Elastic Stack release 7. | -| `elasticsearch_bootstrap_pw` | `str` | `"PleaseChangeMe"` | — | Bootstrap password stored in the keystore. Only takes effect when Elasticsearch is initialised for the first time; later changes do not update it. | -| `elasticsearch_http_protocol` | `str` | `"http"` | `http`, `https` | Protocol the role uses to talk to the Elasticsearch API. Set automatically to https once HTTP security is active. | -| `elasticsearch_ssl_verification_mode` | `str` | `"full"` | `full`, `certificate`, `none` | How certificates presented by the other party are verified (xpack.security.transport.ssl.verification_mode). | -| `elasticsearch_tls_key_passphrase` | `str` | `"PleaseChangeMeIndividually"` | — | Passphrase for the generated Elasticsearch certificates. Overridden by elasticstack_cert_pass when that is set. | -| `elasticsearch_cert_validity_period` | `int` | `1095` | — | Number of days the generated certificates are valid. | -| `elasticsearch_cert_expiration_buffer` | `int` | `30` | — | Renew the certificate when its remaining validity (in days) drops below this value. | -| `elasticsearch_cert_will_expire_soon` | `bool` | `false` | — | Set to true to force renewal of the Elasticsearch certificate. Alternatively run the playbook with the renew_es_cert tag. | -| `elasticsearch_extra_config` | `dict` | N/A | — | Additional elasticsearch.yml settings, given as a mapping and rendered as YAML into the configuration (via to_nice_yaml). Unset by default. | -| `elasticsearch_initialized_file` | `str` | `"{{ elasticstack_initial_passwords \| default('') \| dirname }}/cluster_initialized"` | — | Marker file the role writes once the cluster is initialised, used to detect an existing setup. Advanced; usually left at its default. | -| `elasticsearch_freshstart` | `dict` | `{'changed': False}` | — | Internal state used by the role to detect a fresh install. Do not set manually. | -| `elasticsearch_freshstart_security` | `dict` | `{'changed': False}` | — | Internal state used by the role to detect a fresh security setup. Do not set manually. | +| Variable | Description | +|----------|-------------| +| `elasticsearch_enable`
**Type**: `bool`
**Default**: `true` | Start and enable the Elasticsearch service. | +| `elasticsearch_manage_yaml`
**Type**: `bool`
**Default**: `true` | Manage and overwrite elasticsearch.yml. | +| `elasticsearch_config_backup`
**Type**: `bool`
**Default**: `false` | Keep a backup of elasticsearch.yml whenever the role changes it. | +| `elasticsearch_clustername`
**Type**: `str`
**Default**: `"elasticsearch"` | Name of the Elasticsearch cluster (cluster.name). | +| `elasticsearch_nodename`
**Type**: `str` | Name of this Elasticsearch node (node.name). Defaults to the host's ansible_hostname. | +| `elasticsearch_node_types`
**Type**: `list` of `str` | Node roles for this node (node.roles), e.g. master, data, ingest. When unset, Elasticsearch uses its own defaults. See the Elasticsearch node-roles documentation for valid values. | +| `elasticsearch_datapath`
**Type**: `str`
**Default**: `"/var/lib/elasticsearch"` | Directory where Elasticsearch stores its data (path.data). | +| `elasticsearch_create_datapath`
**Type**: `bool`
**Default**: `false` | Create the data directory if it does not exist. Useful when you change elasticsearch_datapath. | +| `elasticsearch_logpath`
**Type**: `str`
**Default**: `"/var/log/elasticsearch"` | Directory where Elasticsearch stores its logs (path.logs). | +| `elasticsearch_create_logpath`
**Type**: `bool`
**Default**: `false` | Create the log directory if it does not exist. Useful when you change elasticsearch_logpath. | +| `elasticsearch_conf_dir`
**Type**: `str`
**Default**: `"/etc/elasticsearch/"` | Elasticsearch configuration directory. Used to place the jvm.options.d drop-ins. | +| `elasticsearch_group`
**Type**: `str`
**Default**: `"elasticsearch"` | OS group that owns the jvm.options.d drop-ins. The service reads them through this group, so it must match the group the package runs the service as (elasticsearch). Changing it is not recommended. | +| `elasticsearch_fs_repo`
**Type**: `list` of `str` | Filesystem paths registered as snapshot repositories (path.repo). Every node must reach the same share under the same path. Unset by default. | +| `elasticsearch_heap`
**Type**: `str`
**Default**: `"{{ [[(ansible_memtotal_mb // 1024) // 2, 30] \| min, 1] \| max }}"` | JVM heap size in GB. Sets both -Xms and -Xmx via a jvm.options.d drop-in. The default is half of the host memory, capped between 1 and 30 GB. Set to false to leave heap sizing to Elasticsearch/the JVM. | +| `elasticsearch_heap_dump_path`
**Type**: `str`
**Default**: `"/var/lib/elasticsearch"` | Directory for JVM heap dumps (-XX:HeapDumpPath). | +| `elasticsearch_jvm_custom_parameters`
**Type**: `raw`
**Default**: `""` | Extra JVM options written to a jvm.options.d drop-in, provided as a list of option lines (e.g. ["-XX:+UseG1GC"]). Empty by default. | +| `elasticsearch_api_host`
**Type**: `str`
**Default**: `"localhost"` | Host or IP the role uses for its own Elasticsearch API connection checks. | +| `elasticsearch_network_host`
**Type**: `str` | Network addresses Elasticsearch binds to (network.host). Accepts IPs, interface names, or Elasticsearch special values, e.g. "_ens190_,_local_". Defaults to "_local_", "_site_" when unset. | +| `elasticsearch_http_publish_host`
**Type**: `str` | Address HTTP clients use to reach this node when sniffing (http.publish_host). Unset by default. | +| `elasticsearch_http_publish_port`
**Type**: `int` | HTTP publish port, only needed when it must differ from http.port (http.publish_port). Unset by default. | +| `elasticsearch_transport_port`
**Type**: `int` | Port for inter-node transport communication (transport.port). Unset by default. | +| `elasticsearch_seed_hosts`
**Type**: `list` of `str` | Seed hosts for cluster discovery (discovery.seed_hosts). Overrides the list the role builds from the elasticsearch group. Unset by default. | +| `elasticsearch_check_calculation`
**Type**: `bool`
**Default**: `false` | When true, Elasticsearch is not installed, configured, or started: the role runs only the node-role validation and then stops each host (meta: end_host). The validation derives each node's role (master/data/other) from elasticsearch_node_types, groups the nodes, and asserts an odd number of master-eligible nodes (required for a valid quorum). Requires elasticsearch_node_types. Useful as a pre-flight inventory/topology check (before scaling or in CI) without a full deployment. | +| `elasticsearch_disable_systemcallfilterchecks`
**Type**: `bool`
**Default**: `false` | Disable the system call filter check (bootstrap.system_call_filter). Has a security impact and is only applied on Elastic Stack release 7. | +| `elasticsearch_jna_workaround`
**Type**: `bool`
**Default**: `false` | Work around systems where JNA cannot load its native library. WARNING this rewrites the sysconfig/default file and overwrites ES_JAVA_OPTS. | +| `elasticsearch_unsafe_upgrade_restart`
**Type**: `bool`
**Default**: `false` | During upgrades, update the package first and then restart the service, instead of stopping it, upgrading and starting again. Only for non-production clusters, use at your own risk. | +| `elasticsearch_security`
**Type**: `bool`
**Default**: `true` | Enable X-Pack security (transport TLS, keystore, CA and certificates). Required on Elastic Stack release 8 and later with the elastic variant. | +| `elasticsearch_http_security`
**Type**: `bool`
**Default**: `true` | Enable TLS on the HTTP layer (xpack.security.http.ssl). Only effective when elasticsearch_security is enabled. | +| `elasticsearch_security_enrollment`
**Type**: `bool` | Control node/Kibana enrollment on a node autoconfigured for security (xpack.security.enrollment.enabled). Unset by default. | +| `elasticsearch_ml_enabled`
**Type**: `bool`
**Default**: `true` | Enable Machine Learning (xpack.ml.enabled). Only applied with the elastic variant. | +| `elasticsearch_monitoring_enabled`
**Type**: `bool`
**Default**: `true` | Enable X-Pack monitoring collection (xpack.monitoring.collection.enabled). Only applied on Elastic Stack release 7. | +| `elasticsearch_bootstrap_pw`
**Type**: `str`
**Default**: `"PleaseChangeMe"` | Bootstrap password stored in the keystore. Only takes effect when Elasticsearch is initialised for the first time; later changes do not update it. | +| `elasticsearch_http_protocol`
**Type**: `str`
**Default**: `"http"`
**Choices**: `http`, `https` | Protocol the role uses to talk to the Elasticsearch API. Set automatically to https once HTTP security is active. | +| `elasticsearch_ssl_verification_mode`
**Type**: `str`
**Default**: `"full"`
**Choices**: `full`, `certificate`, `none` | How certificates presented by the other party are verified (xpack.security.transport.ssl.verification_mode). | +| `elasticsearch_tls_key_passphrase`
**Type**: `str`
**Default**: `"PleaseChangeMeIndividually"` | Passphrase for the generated Elasticsearch certificates. Overridden by elasticstack_cert_pass when that is set. | +| `elasticsearch_cert_validity_period`
**Type**: `int`
**Default**: `1095` | Number of days the generated certificates are valid. | +| `elasticsearch_cert_expiration_buffer`
**Type**: `int`
**Default**: `30` | Renew the certificate when its remaining validity (in days) drops below this value. | +| `elasticsearch_cert_will_expire_soon`
**Type**: `bool`
**Default**: `false` | Set to true to force renewal of the Elasticsearch certificate. Alternatively run the playbook with the renew_es_cert tag. | +| `elasticsearch_extra_config`
**Type**: `dict` | Additional elasticsearch.yml settings, given as a mapping and rendered as YAML into the configuration (via to_nice_yaml). Unset by default. | +| `elasticsearch_initialized_file`
**Type**: `str`
**Default**: `"{{ elasticstack_initial_passwords \| default('') \| dirname }}/cluster_initialized"` | Marker file the role writes once the cluster is initialised, used to detect an existing setup. Advanced; usually left at its default. | +| `elasticsearch_freshstart`
**Type**: `dict`
**Default**: `{'changed': False}` | Internal state used by the role to detect a fresh install. Do not set manually. | +| `elasticsearch_freshstart_security`
**Type**: `dict`
**Default**: `{'changed': False}` | Internal state used by the role to detect a fresh security setup. Do not set manually. | diff --git a/roles/elasticstack/README.md b/roles/elasticstack/README.md index 4244596c..2d20d381 100644 --- a/roles/elasticstack/README.md +++ b/roles/elasticstack/README.md @@ -24,37 +24,37 @@ The role runs automatically the first time any component role is applied to a ho ## Role variables -| Variable | Type | Default | Choices | Description | -|----------|------|---------|---------|-------------| -| `elasticstack_release` | `int` | `8` | — | Major Elastic Stack release to install and configure (e.g. 7 or 8). Drives the package repository and release-dependent behaviour. | -| `elasticstack_variant` | `str` | `"elastic"` | `elastic`, `oss` | Distribution to install. "elastic" (the default) includes X-Pack; "oss" is the Apache-licensed build without X-Pack. OSS Elasticsearch and Kibana exist only up to release 7, while OSS Logstash and Beats are available on later releases too. | -| `elasticstack_version` | `str` | N/A | — | Exact package version to install (e.g. "8.8.1"). Unset by default: the collection then installs the latest, or reuses the version already present on the CA host. | -| `elasticstack_full_stack` | `bool` | `true` | — | Treat the hosts as one coordinated stack (roles read each other's hosts, share the CA and wire the connections). Set to false to run a role standalone. | -| `elasticstack_security` | `bool` | `true` | — | Enable X-Pack security across the stack. Only effective with the elastic variant. | -| `elasticstack_override_beats_tls` | `bool` | `false` | — | When true, do not auto-enable TLS between Beats and Logstash in a full stack (also skips Beats certificate generation). | -| `elasticstack_elasticsearch_group_name` | `str` | `"elasticsearch"` | — | Inventory group that holds the Elasticsearch hosts. | -| `elasticstack_logstash_group_name` | `str` | `"logstash"` | — | Inventory group that holds the Logstash hosts. | -| `elasticstack_kibana_group_name` | `str` | `"kibana"` | — | Inventory group that holds the Kibana hosts. | -| `elasticstack_elasticsearch_http_port` | `int` | `9200` | — | Elasticsearch HTTP API port. | -| `elasticstack_kibana_host` | `str` | `"{{ ansible_fqdn }}"` | — | Host name Kibana publishes itself under (server.publicBaseUrl). Defaults to the fully qualified domain name of the Kibana host. Set it when users reach Kibana under a different name, for example behind a reverse proxy or a load balancer. | -| `elasticstack_kibana_port` | `int` | `5601` | — | Kibana HTTP port (also used to build the public base URL). | -| `elasticstack_beats_port` | `int` | `5044` | — | Port of the Logstash Beats input that the Beats ship to. | -| `elasticstack_enable_repos` | `bool` | `true` | — | Configure the Elastic package repositories (see the repos role). | -| `elasticstack_repo_url` | `str` | `"https://artifacts.elastic.co/packages"` | — | Base URL of the Elastic package repositories. | -| `elasticstack_repo_key` | `str` | `"https://artifacts.elastic.co/GPG-KEY-elasticsearch"` | — | URL of the Elastic repository signing (GPG) key. | -| `elasticstack_rpm_workaround` | `bool` | `false` | — | Enable a workaround for importing the repository GPG key on RPM-based systems. | -| `elasticstack_ca_host` | `str` | `"{{ groups[elasticstack_elasticsearch_group_name][0] \| default('') }}"` | — | Inventory host that holds the stack CA and signs all component certificates. Must be an Elasticsearch host: signing uses elasticsearch-certutil, which ships with the elasticsearch package. Defaults to the first host in the elasticsearch group; override it only to pin the CA and the initial_passwords file to a specific Elasticsearch node. Only used when elasticstack_full_stack is true. | -| `elasticstack_ca_dir` | `str` | `"/opt/es-ca"` | — | Directory on the CA host where the CA and generated certificates are stored. | -| `elasticstack_ca_name` | `str` | `"CN=Elastic Certificate Tool Autogenerated CA"` | — | Distinguished name (DN) of the generated CA. | -| `elasticstack_ca_pass` | `str` | `"PleaseChangeMe"` | — | Password protecting the CA private key. | -| `elasticstack_ca_validity_period` | `int` | `1095` | — | Number of days the generated CA is valid. | -| `elasticstack_ca_expiration_buffer` | `int` | `30` | — | Renew the CA when its remaining validity (in days) drops below this value. | -| `elasticstack_ca_will_expire_soon` | `bool` | `false` | — | Set to true to force renewal of the CA and all component certificates. Alternatively run the playbook with the renew_ca tag. | -| `elasticstack_cert_pass` | `str` | N/A | — | Common passphrase for all component certificates. Unset by default; when set, it overrides each role's individual certificate passphrase. | -| `elasticstack_initial_passwords` | `str` | `"/usr/share/elasticsearch/initial_passwords"` | — | Path on the CA host where the generated initial passwords are stored. | -| `elasticstack_no_log` | `bool` | `true` | — | Hide the output of tasks that could reveal passwords. Set to false for debugging. | -| `elasticstack_manage_pip` | `bool` | `false` | — | Install pip on the target host. | -| `elasticstack_force_pip` | `bool` | `false` | — | Force installation of the required Python modules via pip (useful when the distribution packages are too old). See PEP 668. | -| `elasticstack_ci` | `bool` | `false` | — | Internal test flag. Set to true only by the collection's molecule scenarios to enable resource workarounds needed on constrained test runners (cache cleanup, relaxed Elasticsearch disk watermarks). Never set this on a real installation. | +| Variable | Description | +|----------|-------------| +| `elasticstack_release`
**Type**: `int`
**Default**: `8` | Major Elastic Stack release to install and configure (e.g. 7 or 8). Drives the package repository and release-dependent behaviour. | +| `elasticstack_variant`
**Type**: `str`
**Default**: `"elastic"`
**Choices**: `elastic`, `oss` | Distribution to install. "elastic" (the default) includes X-Pack; "oss" is the Apache-licensed build without X-Pack. OSS Elasticsearch and Kibana exist only up to release 7, while OSS Logstash and Beats are available on later releases too. | +| `elasticstack_version`
**Type**: `str` | Exact package version to install (e.g. "8.8.1"). Unset by default: the collection then installs the latest, or reuses the version already present on the CA host. | +| `elasticstack_full_stack`
**Type**: `bool`
**Default**: `true` | Treat the hosts as one coordinated stack (roles read each other's hosts, share the CA and wire the connections). Set to false to run a role standalone. | +| `elasticstack_security`
**Type**: `bool`
**Default**: `true` | Enable X-Pack security across the stack. Only effective with the elastic variant. | +| `elasticstack_override_beats_tls`
**Type**: `bool`
**Default**: `false` | When true, do not auto-enable TLS between Beats and Logstash in a full stack (also skips Beats certificate generation). | +| `elasticstack_elasticsearch_group_name`
**Type**: `str`
**Default**: `"elasticsearch"` | Inventory group that holds the Elasticsearch hosts. | +| `elasticstack_logstash_group_name`
**Type**: `str`
**Default**: `"logstash"` | Inventory group that holds the Logstash hosts. | +| `elasticstack_kibana_group_name`
**Type**: `str`
**Default**: `"kibana"` | Inventory group that holds the Kibana hosts. | +| `elasticstack_elasticsearch_http_port`
**Type**: `int`
**Default**: `9200` | Elasticsearch HTTP API port. | +| `elasticstack_kibana_host`
**Type**: `str`
**Default**: `"{{ ansible_fqdn }}"` | Host name Kibana publishes itself under (server.publicBaseUrl). Defaults to the fully qualified domain name of the Kibana host. Set it when users reach Kibana under a different name, for example behind a reverse proxy or a load balancer. | +| `elasticstack_kibana_port`
**Type**: `int`
**Default**: `5601` | Kibana HTTP port (also used to build the public base URL). | +| `elasticstack_beats_port`
**Type**: `int`
**Default**: `5044` | Port of the Logstash Beats input that the Beats ship to. | +| `elasticstack_enable_repos`
**Type**: `bool`
**Default**: `true` | Configure the Elastic package repositories (see the repos role). | +| `elasticstack_repo_url`
**Type**: `str`
**Default**: `"https://artifacts.elastic.co/packages"` | Base URL of the Elastic package repositories. | +| `elasticstack_repo_key`
**Type**: `str`
**Default**: `"https://artifacts.elastic.co/GPG-KEY-elasticsearch"` | URL of the Elastic repository signing (GPG) key. | +| `elasticstack_rpm_workaround`
**Type**: `bool`
**Default**: `false` | Enable a workaround for importing the repository GPG key on RPM-based systems. | +| `elasticstack_ca_host`
**Type**: `str`
**Default**: `"{{ groups[elasticstack_elasticsearch_group_name][0] \| default('') }}"` | Inventory host that holds the stack CA and signs all component certificates. Must be an Elasticsearch host: signing uses elasticsearch-certutil, which ships with the elasticsearch package. Defaults to the first host in the elasticsearch group; override it only to pin the CA and the initial_passwords file to a specific Elasticsearch node. Only used when elasticstack_full_stack is true. | +| `elasticstack_ca_dir`
**Type**: `str`
**Default**: `"/opt/es-ca"` | Directory on the CA host where the CA and generated certificates are stored. | +| `elasticstack_ca_name`
**Type**: `str`
**Default**: `"CN=Elastic Certificate Tool Autogenerated CA"` | Distinguished name (DN) of the generated CA. | +| `elasticstack_ca_pass`
**Type**: `str`
**Default**: `"PleaseChangeMe"` | Password protecting the CA private key. | +| `elasticstack_ca_validity_period`
**Type**: `int`
**Default**: `1095` | Number of days the generated CA is valid. | +| `elasticstack_ca_expiration_buffer`
**Type**: `int`
**Default**: `30` | Renew the CA when its remaining validity (in days) drops below this value. | +| `elasticstack_ca_will_expire_soon`
**Type**: `bool`
**Default**: `false` | Set to true to force renewal of the CA and all component certificates. Alternatively run the playbook with the renew_ca tag. | +| `elasticstack_cert_pass`
**Type**: `str` | Common passphrase for all component certificates. Unset by default; when set, it overrides each role's individual certificate passphrase. | +| `elasticstack_initial_passwords`
**Type**: `str`
**Default**: `"/usr/share/elasticsearch/initial_passwords"` | Path on the CA host where the generated initial passwords are stored. | +| `elasticstack_no_log`
**Type**: `bool`
**Default**: `true` | Hide the output of tasks that could reveal passwords. Set to false for debugging. | +| `elasticstack_manage_pip`
**Type**: `bool`
**Default**: `false` | Install pip on the target host. | +| `elasticstack_force_pip`
**Type**: `bool`
**Default**: `false` | Force installation of the required Python modules via pip (useful when the distribution packages are too old). See PEP 668. | +| `elasticstack_ci`
**Type**: `bool`
**Default**: `false` | Internal test flag. Set to true only by the collection's molecule scenarios to enable resource workarounds needed on constrained test runners (cache cleanup, relaxed Elasticsearch disk watermarks). Never set this on a real installation. | diff --git a/roles/kibana/README.md b/roles/kibana/README.md index ab0e349c..728b7f57 100644 --- a/roles/kibana/README.md +++ b/roles/kibana/README.md @@ -61,35 +61,35 @@ Run only parts of the role with `--tags`: ## Role variables -| Variable | Type | Default | Choices | Description | -|----------|------|---------|---------|-------------| -| `kibana_enable` | `bool` | `true` | — | Start and enable the Kibana service. | -| `kibana_config_backup` | `bool` | `true` | — | Keep a backup of kibana.yml whenever the role changes it. | -| `kibana_manage_yaml` | `bool` | `true` | — | Manage and overwrite kibana.yml. | -| `kibana_elasticsearch_hosts` | `list` of `str` | N/A | — | Elasticsearch hosts Kibana connects to. Defaults to the nodes from the elasticsearch group, or localhost when used standalone. | -| `kibana_security` | `bool` | `true` | — | Enable X-Pack security: connect to Elasticsearch over HTTPS with authentication and set up the encryption keys. Only effective in a full stack setup with the elastic variant. | -| `kibana_sniff_on_start` | `bool` | `false` | — | Ask Elasticsearch for the list of nodes at startup (elasticsearch.sniffOnStart). | -| `kibana_sniff_on_connection_fault` | `bool` | `false` | — | Update the list of Elasticsearch nodes on connection fault (elasticsearch.sniffOnConnectionFault). | -| `kibana_sniff_interval` | `str` | N/A | — | Interval between sniffing attempts (elasticsearch.sniffInterval, in milliseconds). Unset by default. | -| `kibana_tls` | `bool` | `false` | — | Enable TLS on the Kibana server itself (server.ssl). | -| `kibana_tls_cert` | `str` | `"/etc/kibana/certs/cert.pem"` | — | Path to the certificate for the Kibana server TLS. | -| `kibana_tls_key` | `str` | `"/etc/kibana/certs/key.pem"` | — | Path to the private key for the Kibana server TLS. | -| `kibana_tls_key_passphrase` | `str` | `"PleaseChangeMe"` | — | Passphrase for the generated Kibana security certificate (X-Pack, full stack). Overridden by elasticstack_cert_pass when that is set. | -| `kibana_cert_validity_period` | `int` | `1095` | — | Number of days the generated certificates are valid. | -| `kibana_cert_expiration_buffer` | `int` | `30` | — | Renew the certificate when its remaining validity (in days) drops below this value. | -| `kibana_cert_will_expire_soon` | `bool` | `false` | — | Set to true to force renewal of the Kibana certificate. Alternatively run the playbook with the renew_kibana_cert tag. | -| `kibana_manage_logging` | `bool` | `false` | — | Manage Kibana's own logging configuration and write log files to disk. When disabled, Kibana keeps its default behaviour and only logs to the journal. | -| `kibana_logpath` | `str` | `"/var/log/kibana"` | — | Directory for the Kibana log files. The role creates it for the kibana user when kibana_manage_logging is enabled. | -| `kibana_logfile` | `str` | `"kibana.log"` | — | Name of the log file inside kibana_logpath. | -| `kibana_loglevel` | `str` | `"info"` | `all`, `trace`, `debug`, `info`, `warn`, `error`, `fatal`, `off` | Log level of the root logger (logging.root.level). | -| `kibana_logging_layout` | `str` | `"pattern"` | `pattern`, `json` | Layout of the log file. Use "pattern" for human readable logs and "json" when the file is picked up by a log shipper. | -| `kibana_logging_console` | `bool` | `true` | — | Keep logging to the console (and therefore the journal) in addition to the log file. Set to false to log to the file only. | -| `kibana_logging_rotate_policy` | `str` | `"size-limit"` | `size-limit`, `time-interval` | Rotation policy of the log file. Kibana supports only one policy per appender, either by size (kibana_logging_rotate_size) or by time (kibana_logging_rotate_interval). | -| `kibana_logging_rotate_size` | `str` | `"100mb"` | — | Size at which the log file is rotated. Only used when kibana_logging_rotate_policy is "size-limit". | -| `kibana_logging_rotate_interval` | `str` | `"24h"` | — | Interval at which the log file is rotated. Only used when kibana_logging_rotate_policy is "time-interval". | -| `kibana_logging_rotate_keep` | `int` | `10` | — | Number of rotated log files to keep (strategy max). | -| `kibana_extra_config` | `str` | N/A | — | Extra configuration appended verbatim to kibana.yml (YAML). Unset by default. | -| `kibana_freshstart` | `dict` | `{'changed': False}` | — | Internal state used by the role to detect a fresh install. Do not set manually. | +| Variable | Description | +|----------|-------------| +| `kibana_enable`
**Type**: `bool`
**Default**: `true` | Start and enable the Kibana service. | +| `kibana_config_backup`
**Type**: `bool`
**Default**: `true` | Keep a backup of kibana.yml whenever the role changes it. | +| `kibana_manage_yaml`
**Type**: `bool`
**Default**: `true` | Manage and overwrite kibana.yml. | +| `kibana_elasticsearch_hosts`
**Type**: `list` of `str` | Elasticsearch hosts Kibana connects to. Defaults to the nodes from the elasticsearch group, or localhost when used standalone. | +| `kibana_security`
**Type**: `bool`
**Default**: `true` | Enable X-Pack security: connect to Elasticsearch over HTTPS with authentication and set up the encryption keys. Only effective in a full stack setup with the elastic variant. | +| `kibana_sniff_on_start`
**Type**: `bool`
**Default**: `false` | Ask Elasticsearch for the list of nodes at startup (elasticsearch.sniffOnStart). | +| `kibana_sniff_on_connection_fault`
**Type**: `bool`
**Default**: `false` | Update the list of Elasticsearch nodes on connection fault (elasticsearch.sniffOnConnectionFault). | +| `kibana_sniff_interval`
**Type**: `str` | Interval between sniffing attempts (elasticsearch.sniffInterval, in milliseconds). Unset by default. | +| `kibana_tls`
**Type**: `bool`
**Default**: `false` | Enable TLS on the Kibana server itself (server.ssl). | +| `kibana_tls_cert`
**Type**: `str`
**Default**: `"/etc/kibana/certs/cert.pem"` | Path to the certificate for the Kibana server TLS. | +| `kibana_tls_key`
**Type**: `str`
**Default**: `"/etc/kibana/certs/key.pem"` | Path to the private key for the Kibana server TLS. | +| `kibana_tls_key_passphrase`
**Type**: `str`
**Default**: `"PleaseChangeMe"` | Passphrase for the generated Kibana security certificate (X-Pack, full stack). Overridden by elasticstack_cert_pass when that is set. | +| `kibana_cert_validity_period`
**Type**: `int`
**Default**: `1095` | Number of days the generated certificates are valid. | +| `kibana_cert_expiration_buffer`
**Type**: `int`
**Default**: `30` | Renew the certificate when its remaining validity (in days) drops below this value. | +| `kibana_cert_will_expire_soon`
**Type**: `bool`
**Default**: `false` | Set to true to force renewal of the Kibana certificate. Alternatively run the playbook with the renew_kibana_cert tag. | +| `kibana_manage_logging`
**Type**: `bool`
**Default**: `false` | Manage Kibana's own logging configuration and write log files to disk. When disabled, Kibana keeps its default behaviour and only logs to the journal. | +| `kibana_logpath`
**Type**: `str`
**Default**: `"/var/log/kibana"` | Directory for the Kibana log files. The role creates it for the kibana user when kibana_manage_logging is enabled. | +| `kibana_logfile`
**Type**: `str`
**Default**: `"kibana.log"` | Name of the log file inside kibana_logpath. | +| `kibana_loglevel`
**Type**: `str`
**Default**: `"info"`
**Choices**: `all`, `trace`, `debug`, `info`, `warn`, `error`, `fatal`, `off` | Log level of the root logger (logging.root.level). | +| `kibana_logging_layout`
**Type**: `str`
**Default**: `"pattern"`
**Choices**: `pattern`, `json` | Layout of the log file. Use "pattern" for human readable logs and "json" when the file is picked up by a log shipper. | +| `kibana_logging_console`
**Type**: `bool`
**Default**: `true` | Keep logging to the console (and therefore the journal) in addition to the log file. Set to false to log to the file only. | +| `kibana_logging_rotate_policy`
**Type**: `str`
**Default**: `"size-limit"`
**Choices**: `size-limit`, `time-interval` | Rotation policy of the log file. Kibana supports only one policy per appender, either by size (kibana_logging_rotate_size) or by time (kibana_logging_rotate_interval). | +| `kibana_logging_rotate_size`
**Type**: `str`
**Default**: `"100mb"` | Size at which the log file is rotated. Only used when kibana_logging_rotate_policy is "size-limit". | +| `kibana_logging_rotate_interval`
**Type**: `str`
**Default**: `"24h"` | Interval at which the log file is rotated. Only used when kibana_logging_rotate_policy is "time-interval". | +| `kibana_logging_rotate_keep`
**Type**: `int`
**Default**: `10` | Number of rotated log files to keep (strategy max). | +| `kibana_extra_config`
**Type**: `str` | Extra configuration appended verbatim to kibana.yml (YAML). Unset by default. | +| `kibana_freshstart`
**Type**: `dict`
**Default**: `{'changed': False}` | Internal state used by the role to detect a fresh install. Do not set manually. | diff --git a/roles/logstash/README.md b/roles/logstash/README.md index 2cb77ed5..10496b6a 100644 --- a/roles/logstash/README.md +++ b/roles/logstash/README.md @@ -49,95 +49,95 @@ Run only parts of the role with `--tags`: ## Role variables -| Variable | Type | Default | Choices | Description | -|----------|------|---------|---------|-------------| -| `logstash_enable` | `bool` | `true` | — | Start and enable the Logstash service. | -| `logstash_config_backup` | `bool` | `false` | — | Keep backups of every configuration file the role changes. | -| `logstash_manage_yaml` | `bool` | `true` | — | Manage and overwrite logstash.yml. | -| `logstash_manage_logging` | `bool` | `false` | — | Manage the log4j2 logging configuration. | -| `logstash_heap` | `str` | `"1"` | — | JVM heap size in GB. Sets both -Xms and -Xmx via an LS_JAVA_OPTS systemd drop-in.

Elastic recommends 4-8 GB for typical ingestion and staying below 50-75% of physical memory. Keep it low when Logstash shares a host with Elasticsearch so they do not compete for memory. | -| `logstash_plugins` | `list` of `str` | N/A | — | List of Logstash plugins to install. Unset by default. | -| `logstash_config_autoreload` | `bool` | `true` | — | Enable automatic reload of the Logstash configuration. | -| `logstash_config_autoreload_interval` | `str` | N/A | — | Interval between configuration reload checks (e.g. "3s"). Only used when logstash_config_autoreload is enabled. Unset by default. | -| `logstash_config_path_data` | `str` | `"/var/lib/logstash"` | — | Logstash data directory (path.data). | -| `logstash_config_path_logs` | `str` | `"/var/log/logstash"` | — | Logstash log directory (path.logs). | -| `logstash_http_host` | `str` | N/A | — | Bind address of the Logstash monitoring API (http.host). Unset by default. | -| `logstash_http_port` | `str` | N/A | — | Port (or port range, e.g. "9600-9700") of the monitoring API (http.port). Unset by default. | -| `logstash_global_ecs` | `str` | N/A | `disabled`, `v1` | Set the global ECS compatibility mode (pipeline.ecs_compatibility). Unset by default. | -| `logstash_pipeline_unsafe_shutdown` | `bool` | N/A | — | Force Logstash to exit during shutdown even if there are in-flight events. Unset by default. | -| `logstash_legacy_monitoring` | `bool` | `true` | — | Enable legacy X-Pack monitoring. Ignored unless elasticstack_full_stack is set and only effective on Elastic Stack releases lower than 8. | -| `logstash_manage_pipelines` | `bool` | `true` | — | Manage pipelines.yml. | -| `logstash_no_pipelines` | `bool` | `false` | — | Disable all pipeline management entirely. | -| `logstash_pipelines` | `list` of `dict` | … | — | List of pipelines to configure.

Each entry needs a name. A pipeline either points to an external git repository (source/version) or defines simple input/output keys that connect to Redis. See the pipelines documentation for details. | -|     ↳ `name` | `str` | N/A | — | Unique name of the pipeline. Becomes the pipeline directory and pipeline.id. | -|     ↳ `exclusive` | `bool` | N/A | — | Mark this pipeline as the only one allowed to handle its events. | -|     ↳ `source` | `str` | N/A | — | URL of a git repository holding the pipeline configuration. | -|     ↳ `version` | `str` | N/A | — | Git branch/tag/commit to check out from source (default "main"). | -|     ↳ `queue_type` | `str` | N/A | `memory`, `persisted` | Queue type for this pipeline (default "memory"). Use "persisted" for an on-disk persistent queue. | -|     ↳ `queue_max_bytes` | `str` | N/A | — | Maximum queue size for this pipeline (default "1gb"). | -|     ↳ `input` | `list` of `dict` | … | — | Simple Redis inputs for this pipeline. | -|         ↳ `name` | `str` | N/A | — | Name of the input. | -|         ↳ `key` | `str` | N/A | — | Redis key to read from. | -|     ↳ `output` | `list` of `dict` | … | — | Simple Redis outputs for this pipeline. | -|         ↳ `name` | `str` | N/A | — | Name of the output. | -|         ↳ `key` | `str` | N/A | — | Redis key to write to. | -|         ↳ `condition` | `str` | N/A | — | Optional Logstash conditional. The output only receives events matching it. With exclusive set, the conditions are chained with else if. | -|         ↳ `congestion` | `int` | N/A | — | Optional congestion threshold. The output stops once the target Redis key holds more items than this value. | -| `logstash_elasticsearch_output` | `bool` | `true` | — | Create the default pipeline that forwards events to Elasticsearch. | -| `logstash_beats_input` | `bool` | `true` | — | Create the default pipeline with a Beats input. | -| `logstash_beats_input_congestion` | `int` | N/A | — | Congestion threshold for the default Beats input pipeline: it stops writing to Redis once the buffer key holds more items than this value. Unset by default. | -| `logstash_beats_timeout` | `str` | N/A | — | Timeout for idle client connections on the Beats input (e.g. "60s"). Unset by default. | -| `logstash_beats_tls` | `bool` | N/A | — | Activate TLS on the Beats input pipeline. Unset by default, but enabled automatically in a full stack setup unless overridden. | -| `logstash_input_queue_type` | `str` | `"memory"` | `memory`, `persisted` | Queue type for the default Beats input pipeline. Use "persisted" for an on-disk persistent queue. | -| `logstash_input_queue_max_bytes` | `str` | `"1gb"` | — | Maximum queue size for the default Beats input pipeline. | -| `logstash_forwarder_queue_type` | `str` | `"memory"` | `memory`, `persisted` | Queue type for the default Elasticsearch forwarder pipeline. Use "persisted" for an on-disk persistent queue. | -| `logstash_forwarder_queue_max_bytes` | `str` | `"1gb"` | — | Maximum queue size for the default Elasticsearch forwarder pipeline. | -| `logstash_redis_password` | `str` | N/A | — | Password used when the simple inputs/outputs connect to Redis. Unset by default. | -| `logstash_redis_input_host` | `str` | `"localhost"` | — | Redis host that simple pipeline inputs (and the default forwarder input) read from. | -| `logstash_redis_output_host` | `str` | `"localhost"` | — | Redis host that simple pipeline outputs (and the default input pipeline) write to. | -| `logstash_redis_tls` | `bool` | `false` | — | Enable TLS on the simple Redis inputs/outputs (`ssl`/`ssl_enabled`). | -| `logstash_redis_ssl_certificate` | `str` | N/A | — | Path to the SSL certificate for the Redis outputs (the redis output plugin only; the input plugin has no such option). Rendered only when logstash_redis_tls is true. | -| `logstash_redis_ssl_key` | `str` | N/A | — | Path to the SSL key for the Redis outputs. Rendered only when logstash_redis_tls is true. | -| `logstash_redis_ssl_key_passphrase` | `str` | N/A | — | Passphrase for logstash_redis_ssl_key. Rendered only when logstash_redis_tls is true. | -| `logstash_redis_ssl_certificate_authorities` | `list` of `str` | N/A | — | List of CA certificate paths used to verify the Redis server on the Redis outputs. Rendered only when logstash_redis_tls is true. | -| `logstash_redis_ssl_supported_protocols` | `str` | N/A | — | TLS protocol version accepted on the Redis outputs (for example "TLSv1.3"). Rendered only when logstash_redis_tls is true. | -| `logstash_redis_ssl_verification_mode` | `str` | N/A | — | Certificate verification mode for the Redis outputs ("full" or "none"). Rendered only when logstash_redis_tls is true. | -| `logstash_elasticsearch` | `list` of `str` | N/A | — | Elasticsearch hosts for the default output. Defaults to the nodes from the elasticsearch group, or localhost when used standalone. | -| `logstash_validate_after_inactivity` | `str` | `"300"` | — | Seconds Logstash waits before validating a previously idle connection to Elasticsearch. | -| `logstash_sniffing` | `bool` | `false` | — | Enable sniffing for additional Elasticsearch nodes. | -| `logstash_sniffing_delay` | `str` | N/A | — | Seconds to wait between sniffing attempts. Unset by default. | -| `logstash_sniffing_path` | `str` | N/A | — | HTTP path used for the sniffing requests. Unset by default. | -| `logstash_security` | `bool` | N/A | — | Enable X-Pack security (TLS) for Logstash. No default; enabled automatically in full stack mode with the elastic variant. | -| `logstash_tls_key_passphrase` | `str` | `"LogstashChangeMe"` | — | Passphrase for the generated Logstash certificates. | -| `logstash_certs_dir` | `str` | `"/etc/logstash/certs"` | — | Directory holding the Logstash certificates. Used to build several file paths. | -| `logstash_cert_validity_period` | `int` | `1095` | — | Number of days the generated certificates are valid. | -| `logstash_cert_expiration_buffer` | `int` | `30` | — | Renew the certificate when its remaining validity (in days) drops below this value. | -| `logstash_cert_will_expire_soon` | `bool` | `false` | — | Set to true to force renewal of the Logstash certificate. Alternatively run the playbook with the renew_logstash_cert tag. | -| `logstash_create_role` | `bool` | `true` | — | Create the Logstash writer role in Elasticsearch. | -| `logstash_role_name` | `str` | `"logstash_writer"` | — | Name of the Logstash writer role. | -| `logstash_role_cluster_privileges` | `list` of `str` | `['manage_index_templates', 'monitor', 'manage_ilm']` | — | Cluster privileges granted to the Logstash writer role. | -| `logstash_role_indicies_names` | `list` of `str` | `['ecs-logstash*', 'logstash*', 'logs*']` | — | Index patterns the Logstash writer role may access. | -| `logstash_role_indicies_privileges` | `list` of `str` | `['write', 'create', 'delete', 'create_index', 'manage', 'manage_ilm']` | — | Index privileges the Logstash writer role holds on its index patterns. | -| `logstash_create_user` | `bool` | `true` | — | Create the Logstash writer user in Elasticsearch. | -| `logstash_user_name` | `str` | `"logstash_writer"` | — | Name of the Logstash user connecting to Elasticsearch. | -| `logstash_user_password` | `str` | `"password"` | — | Password of the Logstash user. Must be at least 6 characters long. | -| `logstash_user_email` | `str` | `""` | — | Email address linked to the Logstash user. | -| `logstash_user_fullname` | `str` | `"Internal Logstash User"` | — | Full name linked to the Logstash user. | -| `logstash_logging_console` | `bool` | `true` | — | Log to the console (syslog when run via systemd). Only effective when logstash_manage_logging is enabled. | -| `logstash_logging_file` | `bool` | `true` | — | Log to the log file. Only effective when logstash_manage_logging is enabled. | -| `logstash_logging_slow_console` | `bool` | `true` | — | Log the slowlog to the console (syslog when run via systemd). Only effective when logstash_manage_logging is enabled. | -| `logstash_logging_slow_file` | `bool` | `true` | — | Log the slowlog to the log file. Only effective when logstash_manage_logging is enabled. | -| `logstash_logging_retention` | `str` | `"7D"` | — | Age at which rotated log files (plain, json, slowlog, deprecation and per-pipeline routing logs) are deleted by log4j2 at rollover time. Uses log4j2 IfLastModified duration syntax such as "7D", "24H" or "P30D". Only effective when logstash_manage_logging is enabled. | -| `logstash_ident` | `bool` | `true` | — | Add a field identifying the node that processed an event. | -| `logstash_ident_field_name` | `str` | `"[netways][instance]"` | — | Name of the field that identifies the instance. | -| `logstash_pipeline_identifier` | `bool` | `true` | — | Add a field identifying which pipeline processed an event. | -| `logstash_pipeline_identifier_field_name` | `str` | `"[netways][pipeline]"` | — | Name of the pipeline identifier field. | -| `logstash_pipeline_identifier_defaults` | `bool` | `false` | — | Also add pipeline identifiers in the default pipelines. This can let the defaults dominate statistics with little value. | -| `logstash_mermaid` | `bool` | `true` | — | Produce an overview of the Logstash pipelines in Mermaid syntax. | -| `logstash_mermaid_logstash` | `bool` | `true` | — | Place the Mermaid syntax into /etc/logstash/pipelines.mermaid on the Logstash hosts. | -| `logstash_mermaid_local` | `bool` | `false` | — | Place the Mermaid syntax into a temporary file on the control node. | -| `logstash_mermaid_extra` | `str` | N/A | — | Extra Mermaid syntax to append to the output (YAML multiline supported). Unset by default. | -| `logstash_freshstart` | `dict` | `{'changed': False}` | — | Internal state used by the role to detect a fresh install. Do not set manually. | +| Variable | Description | +|----------|-------------| +| `logstash_enable`
**Type**: `bool`
**Default**: `true` | Start and enable the Logstash service. | +| `logstash_config_backup`
**Type**: `bool`
**Default**: `false` | Keep backups of every configuration file the role changes. | +| `logstash_manage_yaml`
**Type**: `bool`
**Default**: `true` | Manage and overwrite logstash.yml. | +| `logstash_manage_logging`
**Type**: `bool`
**Default**: `false` | Manage the log4j2 logging configuration. | +| `logstash_heap`
**Type**: `str`
**Default**: `"1"` | JVM heap size in GB. Sets both -Xms and -Xmx via an LS_JAVA_OPTS systemd drop-in.

Elastic recommends 4-8 GB for typical ingestion and staying below 50-75% of physical memory. Keep it low when Logstash shares a host with Elasticsearch so they do not compete for memory. | +| `logstash_plugins`
**Type**: `list` of `str` | List of Logstash plugins to install. Unset by default. | +| `logstash_config_autoreload`
**Type**: `bool`
**Default**: `true` | Enable automatic reload of the Logstash configuration. | +| `logstash_config_autoreload_interval`
**Type**: `str` | Interval between configuration reload checks (e.g. "3s"). Only used when logstash_config_autoreload is enabled. Unset by default. | +| `logstash_config_path_data`
**Type**: `str`
**Default**: `"/var/lib/logstash"` | Logstash data directory (path.data). | +| `logstash_config_path_logs`
**Type**: `str`
**Default**: `"/var/log/logstash"` | Logstash log directory (path.logs). | +| `logstash_http_host`
**Type**: `str` | Bind address of the Logstash monitoring API (http.host). Unset by default. | +| `logstash_http_port`
**Type**: `str` | Port (or port range, e.g. "9600-9700") of the monitoring API (http.port). Unset by default. | +| `logstash_global_ecs`
**Type**: `str`
**Choices**: `disabled`, `v1` | Set the global ECS compatibility mode (pipeline.ecs_compatibility). Unset by default. | +| `logstash_pipeline_unsafe_shutdown`
**Type**: `bool` | Force Logstash to exit during shutdown even if there are in-flight events. Unset by default. | +| `logstash_legacy_monitoring`
**Type**: `bool`
**Default**: `true` | Enable legacy X-Pack monitoring. Ignored unless elasticstack_full_stack is set and only effective on Elastic Stack releases lower than 8. | +| `logstash_manage_pipelines`
**Type**: `bool`
**Default**: `true` | Manage pipelines.yml. | +| `logstash_no_pipelines`
**Type**: `bool`
**Default**: `false` | Disable all pipeline management entirely. | +| `logstash_pipelines`
**Type**: `list` of `dict` | List of pipelines to configure.

Each entry needs a name. A pipeline either points to an external git repository (source/version) or defines simple input/output keys that connect to Redis. See the pipelines documentation for details. | +|     ↳ `name`
**Type**: `str`
**Required**: Yes | Unique name of the pipeline. Becomes the pipeline directory and pipeline.id. | +|     ↳ `exclusive`
**Type**: `bool` | Mark this pipeline as the only one allowed to handle its events. | +|     ↳ `source`
**Type**: `str` | URL of a git repository holding the pipeline configuration. | +|     ↳ `version`
**Type**: `str` | Git branch/tag/commit to check out from source (default "main"). | +|     ↳ `queue_type`
**Type**: `str`
**Choices**: `memory`, `persisted` | Queue type for this pipeline (default "memory"). Use "persisted" for an on-disk persistent queue. | +|     ↳ `queue_max_bytes`
**Type**: `str` | Maximum queue size for this pipeline (default "1gb"). | +|     ↳ `input`
**Type**: `list` of `dict` | Simple Redis inputs for this pipeline. | +|         ↳ `name`
**Type**: `str` | Name of the input. | +|         ↳ `key`
**Type**: `str` | Redis key to read from. | +|     ↳ `output`
**Type**: `list` of `dict` | Simple Redis outputs for this pipeline. | +|         ↳ `name`
**Type**: `str` | Name of the output. | +|         ↳ `key`
**Type**: `str` | Redis key to write to. | +|         ↳ `condition`
**Type**: `str` | Optional Logstash conditional. The output only receives events matching it. With exclusive set, the conditions are chained with else if. | +|         ↳ `congestion`
**Type**: `int` | Optional congestion threshold. The output stops once the target Redis key holds more items than this value. | +| `logstash_elasticsearch_output`
**Type**: `bool`
**Default**: `true` | Create the default pipeline that forwards events to Elasticsearch. | +| `logstash_beats_input`
**Type**: `bool`
**Default**: `true` | Create the default pipeline with a Beats input. | +| `logstash_beats_input_congestion`
**Type**: `int` | Congestion threshold for the default Beats input pipeline: it stops writing to Redis once the buffer key holds more items than this value. Unset by default. | +| `logstash_beats_timeout`
**Type**: `str` | Timeout for idle client connections on the Beats input (e.g. "60s"). Unset by default. | +| `logstash_beats_tls`
**Type**: `bool` | Activate TLS on the Beats input pipeline. Unset by default, but enabled automatically in a full stack setup unless overridden. | +| `logstash_input_queue_type`
**Type**: `str`
**Default**: `"memory"`
**Choices**: `memory`, `persisted` | Queue type for the default Beats input pipeline. Use "persisted" for an on-disk persistent queue. | +| `logstash_input_queue_max_bytes`
**Type**: `str`
**Default**: `"1gb"` | Maximum queue size for the default Beats input pipeline. | +| `logstash_forwarder_queue_type`
**Type**: `str`
**Default**: `"memory"`
**Choices**: `memory`, `persisted` | Queue type for the default Elasticsearch forwarder pipeline. Use "persisted" for an on-disk persistent queue. | +| `logstash_forwarder_queue_max_bytes`
**Type**: `str`
**Default**: `"1gb"` | Maximum queue size for the default Elasticsearch forwarder pipeline. | +| `logstash_redis_password`
**Type**: `str` | Password used when the simple inputs/outputs connect to Redis. Unset by default. | +| `logstash_redis_input_host`
**Type**: `str`
**Default**: `"localhost"` | Redis host that simple pipeline inputs (and the default forwarder input) read from. | +| `logstash_redis_output_host`
**Type**: `str`
**Default**: `"localhost"` | Redis host that simple pipeline outputs (and the default input pipeline) write to. | +| `logstash_redis_tls`
**Type**: `bool`
**Default**: `false` | Enable TLS on the simple Redis inputs/outputs (`ssl`/`ssl_enabled`). | +| `logstash_redis_ssl_certificate`
**Type**: `str` | Path to the SSL certificate for the Redis outputs (the redis output plugin only; the input plugin has no such option). Rendered only when logstash_redis_tls is true. | +| `logstash_redis_ssl_key`
**Type**: `str` | Path to the SSL key for the Redis outputs. Rendered only when logstash_redis_tls is true. | +| `logstash_redis_ssl_key_passphrase`
**Type**: `str` | Passphrase for logstash_redis_ssl_key. Rendered only when logstash_redis_tls is true. | +| `logstash_redis_ssl_certificate_authorities`
**Type**: `list` of `str` | List of CA certificate paths used to verify the Redis server on the Redis outputs. Rendered only when logstash_redis_tls is true. | +| `logstash_redis_ssl_supported_protocols`
**Type**: `str` | TLS protocol version accepted on the Redis outputs (for example "TLSv1.3"). Rendered only when logstash_redis_tls is true. | +| `logstash_redis_ssl_verification_mode`
**Type**: `str` | Certificate verification mode for the Redis outputs ("full" or "none"). Rendered only when logstash_redis_tls is true. | +| `logstash_elasticsearch`
**Type**: `list` of `str` | Elasticsearch hosts for the default output. Defaults to the nodes from the elasticsearch group, or localhost when used standalone. | +| `logstash_validate_after_inactivity`
**Type**: `str`
**Default**: `"300"` | Seconds Logstash waits before validating a previously idle connection to Elasticsearch. | +| `logstash_sniffing`
**Type**: `bool`
**Default**: `false` | Enable sniffing for additional Elasticsearch nodes. | +| `logstash_sniffing_delay`
**Type**: `str` | Seconds to wait between sniffing attempts. Unset by default. | +| `logstash_sniffing_path`
**Type**: `str` | HTTP path used for the sniffing requests. Unset by default. | +| `logstash_security`
**Type**: `bool` | Enable X-Pack security (TLS) for Logstash. No default; enabled automatically in full stack mode with the elastic variant. | +| `logstash_tls_key_passphrase`
**Type**: `str`
**Default**: `"LogstashChangeMe"` | Passphrase for the generated Logstash certificates. | +| `logstash_certs_dir`
**Type**: `str`
**Default**: `"/etc/logstash/certs"` | Directory holding the Logstash certificates. Used to build several file paths. | +| `logstash_cert_validity_period`
**Type**: `int`
**Default**: `1095` | Number of days the generated certificates are valid. | +| `logstash_cert_expiration_buffer`
**Type**: `int`
**Default**: `30` | Renew the certificate when its remaining validity (in days) drops below this value. | +| `logstash_cert_will_expire_soon`
**Type**: `bool`
**Default**: `false` | Set to true to force renewal of the Logstash certificate. Alternatively run the playbook with the renew_logstash_cert tag. | +| `logstash_create_role`
**Type**: `bool`
**Default**: `true` | Create the Logstash writer role in Elasticsearch. | +| `logstash_role_name`
**Type**: `str`
**Default**: `"logstash_writer"` | Name of the Logstash writer role. | +| `logstash_role_cluster_privileges`
**Type**: `list` of `str`
**Default**: `['manage_index_templates', 'monitor', 'manage_ilm']` | Cluster privileges granted to the Logstash writer role. | +| `logstash_role_indicies_names`
**Type**: `list` of `str`
**Default**: `['ecs-logstash*', 'logstash*', 'logs*']` | Index patterns the Logstash writer role may access. | +| `logstash_role_indicies_privileges`
**Type**: `list` of `str`
**Default**: `['write', 'create', 'delete', 'create_index', 'manage', 'manage_ilm']` | Index privileges the Logstash writer role holds on its index patterns. | +| `logstash_create_user`
**Type**: `bool`
**Default**: `true` | Create the Logstash writer user in Elasticsearch. | +| `logstash_user_name`
**Type**: `str`
**Default**: `"logstash_writer"` | Name of the Logstash user connecting to Elasticsearch. | +| `logstash_user_password`
**Type**: `str`
**Default**: `"password"` | Password of the Logstash user. Must be at least 6 characters long. | +| `logstash_user_email`
**Type**: `str`
**Default**: `""` | Email address linked to the Logstash user. | +| `logstash_user_fullname`
**Type**: `str`
**Default**: `"Internal Logstash User"` | Full name linked to the Logstash user. | +| `logstash_logging_console`
**Type**: `bool`
**Default**: `true` | Log to the console (syslog when run via systemd). Only effective when logstash_manage_logging is enabled. | +| `logstash_logging_file`
**Type**: `bool`
**Default**: `true` | Log to the log file. Only effective when logstash_manage_logging is enabled. | +| `logstash_logging_slow_console`
**Type**: `bool`
**Default**: `true` | Log the slowlog to the console (syslog when run via systemd). Only effective when logstash_manage_logging is enabled. | +| `logstash_logging_slow_file`
**Type**: `bool`
**Default**: `true` | Log the slowlog to the log file. Only effective when logstash_manage_logging is enabled. | +| `logstash_logging_retention`
**Type**: `str`
**Default**: `"7D"` | Age at which rotated log files (plain, json, slowlog, deprecation and per-pipeline routing logs) are deleted by log4j2 at rollover time. Uses log4j2 IfLastModified duration syntax such as "7D", "24H" or "P30D". Only effective when logstash_manage_logging is enabled. | +| `logstash_ident`
**Type**: `bool`
**Default**: `true` | Add a field identifying the node that processed an event. | +| `logstash_ident_field_name`
**Type**: `str`
**Default**: `"[netways][instance]"` | Name of the field that identifies the instance. | +| `logstash_pipeline_identifier`
**Type**: `bool`
**Default**: `true` | Add a field identifying which pipeline processed an event. | +| `logstash_pipeline_identifier_field_name`
**Type**: `str`
**Default**: `"[netways][pipeline]"` | Name of the pipeline identifier field. | +| `logstash_pipeline_identifier_defaults`
**Type**: `bool`
**Default**: `false` | Also add pipeline identifiers in the default pipelines. This can let the defaults dominate statistics with little value. | +| `logstash_mermaid`
**Type**: `bool`
**Default**: `true` | Produce an overview of the Logstash pipelines in Mermaid syntax. | +| `logstash_mermaid_logstash`
**Type**: `bool`
**Default**: `true` | Place the Mermaid syntax into /etc/logstash/pipelines.mermaid on the Logstash hosts. | +| `logstash_mermaid_local`
**Type**: `bool`
**Default**: `false` | Place the Mermaid syntax into a temporary file on the control node. | +| `logstash_mermaid_extra`
**Type**: `str` | Extra Mermaid syntax to append to the output (YAML multiline supported). Unset by default. | +| `logstash_freshstart`
**Type**: `dict`
**Default**: `{'changed': False}` | Internal state used by the role to detect a fresh install. Do not set manually. | From 334edfc94655ab1ca6c8d8e66508fd4a4aa0dab7 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 14 Sep 2026 15:52:48 +0200 Subject: [PATCH 51/53] Write one paragraph per line in the hand-written documentation --- CONTRIBUTING.md | 72 ++++++++-------------------- README.md | 71 ++++++++------------------- RELEASE.md | 40 +++++----------- docs/getting-started.md | 57 ++++++---------------- docs/requirements.md | 14 ++---- docs/upgrades.md | 31 ++++-------- plugins/module_utils/README.md | 74 ++++++++--------------------- roles/beats/README.md | 19 ++------ roles/beats/docs/filebeat-inputs.md | 62 ++++++------------------ roles/elasticsearch/README.md | 24 ++-------- roles/kibana/README.md | 25 ++-------- roles/logstash/README.md | 18 ++----- roles/repos/README.md | 16 ++----- 13 files changed, 134 insertions(+), 389 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a6d10c23..1dcd9b59 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,48 +1,29 @@ # Contributing -Every kind of contribution is welcome. This page describes what we need from a pull -request so that reviewing it does not take longer than writing it. +Every kind of contribution is welcome. This page describes what we need from a pull request so that reviewing it does not take longer than writing it. ## Before you build something -There is no written scope for this collection. It gets what is useful and what helps from -an automation point of view, and that is easier to answer per case than in the abstract. -Please open an issue for anything bigger than a bug fix and we will say yes or no before -you invest the work. +There is no written scope for this collection. It gets what is useful and what helps from an automation point of view, and that is easier to answer per case than in the abstract. Please open an issue for anything bigger than a bug fix and we will say yes or no before you invest the work. For now all pull requests go against `main`. ## What a pull request needs -* **One topic per pull request.** Touching several roles is fine when it is the same - change in each of them, for example the same fix applied to beats, logstash, kibana and - elasticsearch. A pull request that bundles unrelated changes gets sent back. -* **A description that says precisely what changes and why.** This is required, not - optional. A pull request whose purpose can only be reconstructed from the diff gets sent - back. -* **Evidence that it was tested**, against the new feature or fix and against what sits - next to it. Please say which Elastic Stack release and which ansible-core version you - tested with. -* **New or changed variables** belong in the role's `meta/argument_specs.yml` and in its - README table, otherwise the documentation drifts apart from the code. See "Documenting role - variables" below. -* **Comments on anything that is not obvious.** Six months later nobody remembers what a - clever condition was for, and then it gets analysed again from scratch. +* **One topic per pull request.** Touching several roles is fine when it is the same change in each of them, for example the same fix applied to beats, logstash, kibana and elasticsearch. A pull request that bundles unrelated changes gets sent back. +* **A description that says precisely what changes and why.** This is required, not optional. A pull request whose purpose can only be reconstructed from the diff gets sent back. +* **Evidence that it was tested**, against the new feature or fix and against what sits next to it. Please say which Elastic Stack release and which ansible-core version you tested with. +* **New or changed variables** belong in the role's `meta/argument_specs.yml` and in its README table, otherwise the documentation drifts apart from the code. See "Documenting role variables" below. +* **Comments on anything that is not obvious.** Six months later nobody remembers what a clever condition was for, and then it gets analysed again from scratch. * **A changelog fragment.** See below. -* If a change **claims something about how Elastic behaves**, please link the Elastic - documentation for it. +* If a change **claims something about how Elastic behaves**, please link the Elastic documentation for it. ## Changelog fragments -The changelog is not generated from commit messages. It is assembled from small files -that each pull request brings along, so that the description is written by the person who -knows what changed rather than reconstructed months later at release time. +The changelog is not generated from commit messages. It is assembled from small files that each pull request brings along, so that the description is written by the person who knows what changed rather than reconstructed months later at release time. -Add one **new** file per pull request under `changelogs/fragments/`. Name it after the -issue or the change, for example `546_rolling_upgrade.yml`. Do not put your entry into an -existing fragment and do not rename or delete one, so that every entry stays traceable to -the pull request that introduced it. The pipeline rejects both. +Add one **new** file per pull request under `changelogs/fragments/`. Name it after the issue or the change, for example `546_rolling_upgrade.yml`. Do not put your entry into an existing fragment and do not rename or delete one, so that every entry stays traceable to the pull request that introduced it. The pipeline rejects both. ```yaml # changelogs/fragments/546_rolling_upgrade.yml @@ -65,14 +46,11 @@ Available sections, one or more per file: | `known_issues` | Problems shipped with the release on purpose | | `trivial` | Not rendered into the changelog. See below | -Fragments are consumed at release time: `antsibull-changelog release` folds them into the -changelog and deletes them, so the directory is empty again after every release. That is -why a new file is always the right answer, never an edit to an existing one. +Fragments are consumed at release time: `antsibull-changelog release` folds them into the changelog and deletes them, so the directory is empty again after every release. That is why a new file is always the right answer, never an edit to an existing one. ### Changes that do not belong in the changelog -Some changes are not worth telling users about. They still need a fragment, but the entry -goes into the `trivial` section, which is not rendered into the changelog: +Some changes are not worth telling users about. They still need a fragment, but the entry goes into the `trivial` section, which is not rendered into the changelog: ```yaml # changelogs/fragments/fix_comment_typo.yml @@ -88,18 +66,14 @@ Typical cases for `trivial`: * refactoring with no visible effect on behaviour * dependency bumps of test tooling -If you are unsure whether something is trivial, ask in the pull request. It is easier to -move an entry than to notice a missing one after the release. +If you are unsure whether something is trivial, ask in the pull request. It is easier to move an entry than to notice a missing one after the release. ## Documenting role variables -Role variables are documented from each role's `meta/argument_specs.yml`, which is the single -source of truth. When your pull request changes a role's variables: +Role variables are documented from each role's `meta/argument_specs.yml`, which is the single source of truth. When your pull request changes a role's variables: 1. Update that role's `meta/argument_specs.yml` with type, default and description. -2. Regenerate the README variable table. Please do **not** edit it by hand. The table is - produced by [ansible-docsmith](https://github.com/foundata/ansible-docsmith), install it - with `pip install ansible-docsmith` if you do not have it: +2. Regenerate the README variable table. Please do **not** edit it by hand. The table is produced by [ansible-docsmith](https://github.com/foundata/ansible-docsmith), install it with `pip install ansible-docsmith` if you do not have it: ```bash ansible-docsmith generate roles/ --no-defaults --template-readme .docsmith/readme.md.j2 @@ -107,25 +81,17 @@ source of truth. When your pull request changes a role's variables: 3. Commit the regenerated `README.md` together with your change. -The `Test Documentation` workflow checks that each README matches its `argument_specs.yml` and -fails the pull request if they drift apart. +The `Test Documentation` workflow checks that each README matches its `argument_specs.yml` and fails the pull request if they drift apart. ## Documenting modules -Modules document themselves through the `DOCUMENTATION`, `EXAMPLES` and `RETURN` blocks in -their own file. That is the only place, so there is nothing to keep in sync. Check your -changes with `ansible-doc netways.elasticstack.`. +Modules document themselves through the `DOCUMENTATION`, `EXAMPLES` and `RETURN` blocks in their own file. That is the only place, so there is nothing to keep in sync. Check your changes with `ansible-doc netways.elasticstack.`. -Shared code under `plugins/module_utils/` has no such block, because `ansible-doc` does not -read it. Those functions are described in -[plugins/module_utils/README.md](plugins/module_utils/README.md), please keep it current when -you change them. +Shared code under `plugins/module_utils/` has no such block, because `ansible-doc` does not read it. Those functions are described in [plugins/module_utils/README.md](plugins/module_utils/README.md), please keep it current when you change them. ## Testing -Besides testing your change against a real setup, the repository has Molecule scenarios that -exercise the whole stack, and it checks for ansible-lint and yamllint errors. To run the linters -locally there is a `makefile`: +Besides testing your change against a real setup, the repository has Molecule scenarios that exercise the whole stack, and it checks for ansible-lint and yamllint errors. To run the linters locally there is a `makefile`: ```bash make diff --git a/README.md b/README.md index 7ccc98b0..1184c085 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,12 @@ # Ansible Collection - netways.elasticstack -This collection installs and manages the Elastic Stack. It provides roles for every component -of the Stack, and it can install either the default Elastic distribution or the Apache-licensed -OSS packages. +This collection installs and manages the Elastic Stack. It provides roles for every component of the Stack, and it can install either the default Elastic distribution or the Apache-licensed OSS packages. -Every role can be used on its own or together with the others. Running a single role against an -existing or external cluster works, but a few of those paths are still incomplete. See the open -issues for details. +Every role can be used on its own or together with the others. Running a single role against an existing or external cluster works, but a few of those paths are still incomplete. See the open issues for details. ## Tested with Ansible -The minimum required version is declared in [meta/runtime.yml](meta/runtime.yml) and enforced by -`ansible-galaxy` on install. +The minimum required version is declared in [meta/runtime.yml](meta/runtime.yml) and enforced by `ansible-galaxy` on install. | | | |---|---| @@ -28,35 +23,27 @@ We test on the following Linux distributions, each one with Elastic Stack 8: ## External requirements -Ansible collections, both are declared as dependencies and installed for you when you install -this collection with `ansible-galaxy`: +Ansible collections, both are declared as dependencies and installed for you when you install this collection with `ansible-galaxy`: * `community.general`, for the `zypper` modules on SUSE hosts and the Logstash plugin module * `community.crypto`, used by the beats role to check when a certificate expires Python libraries on the control node: -* `passlib`, unless you disable password hashing for the Logstash user and you want to use the - logstash role. Install it with `pip`. -* `elasticsearch`. Current versions are compatible either with Elasticsearch 9 or with versions - lower than 9, there seems to be no version that serves both. For now we install a client - older than 9, and this needs revisiting when the collection becomes compatible with Elastic - Stack 9. +* `passlib`, unless you disable password hashing for the Logstash user and you want to use the logstash role. Install it with `pip`. +* `elasticsearch`. Current versions are compatible either with Elasticsearch 9 or with versions lower than 9, there seems to be no version that serves both. For now we install a client older than 9, and this needs revisiting when the collection becomes compatible with Elastic Stack 9. Ansible roles: -* `geerlingguy.redis` if you want to use the logstash role. There are other ways to get Redis - running, but this one is easy and convenient. +* `geerlingguy.redis` if you want to use the logstash role. There are other ways to get Redis running, but this one is easy and convenient. ```bash ansible-galaxy install geerlingguy.redis ``` -Some very basic packages such as `openssl` are handled by the collection itself. The list above -only contains what applies to special cases or what you need to decide about yourself. +Some very basic packages such as `openssl` are handled by the collection itself. The list above only contains what applies to special cases or what you need to decide about yourself. -Beyond software, the roles expect a few things from your inventory. See -[Requirements](docs/requirements.md). +Beyond software, the roles expect a few things from your inventory. See [Requirements](docs/requirements.md). ## Included content @@ -67,8 +54,7 @@ Roles: * [kibana](roles/kibana/README.md) * [logstash](roles/logstash/README.md) * [repos](roles/repos/README.md) -* [elasticstack](roles/elasticstack/README.md), the meta role that holds the collection-wide - `elasticstack_*` variables +* [elasticstack](roles/elasticstack/README.md), the meta role that holds the collection-wide `elasticstack_*` variables Modules: @@ -76,8 +62,7 @@ Modules: * `elasticsearch_role`, creates, updates and deletes roles in Elasticsearch * `elasticsearch_user`, creates, updates and deletes users in Elasticsearch -Each module documents its own parameters, return values and examples. Read them with -`ansible-doc`, for example: +Each module documents its own parameters, return values and examples. Read them with `ansible-doc`, for example: ```bash ansible-doc netways.elasticstack.elasticsearch_role @@ -89,9 +74,7 @@ ansible-doc netways.elasticstack.elasticsearch_role * [Requirements](docs/requirements.md), what the roles expect from your inventory * [Versions and upgrades](docs/upgrades.md), pinning versions and upgrading the stack -Every role documents all of its variables in its own README, linked above. Those tables are -generated from each role's `meta/argument_specs.yml`. Module documentation comes from the -modules themselves and is read with `ansible-doc`. +Every role documents all of its variables in its own README, linked above. Those tables are generated from each role's `meta/argument_specs.yml`. Module documentation comes from the modules themselves and is read with `ansible-doc`. ## Using this collection @@ -113,8 +96,7 @@ collections: ansible-galaxy collection install -r requirements.yml ``` -The same file works for Tower and AWX. You can also install straight from git, which gives you -the current state of `main` instead of a release: +The same file works for Tower and AWX. You can also install straight from git, which gives you the current state of `main` instead of a release: ```bash ansible-galaxy collection install git+https://github.com/NETWAYS/ansible-collection-elasticstack.git @@ -131,44 +113,31 @@ Then name the collection in the play and refer to the roles by their short name: - elasticsearch ``` -Fully qualified role names such as `netways.elasticstack.elasticsearch` work as well and need no -`collections` entry. +Fully qualified role names such as `netways.elasticstack.elasticsearch` work as well and need no `collections` entry. -The execution order of the roles matters, and the roles interact with hosts from other inventory -groups. Read [Getting started](docs/getting-started.md) before your first run. +The execution order of the roles matters, and the roles interact with hosts from other inventory groups. Read [Getting started](docs/getting-started.md) before your first run. ## Caveats and information for long time users ### Variable renaming -If you have been using this collection before version `1.0.0`, note that a significant number of -variables had to be renamed because of naming schema changes made by Ansible. Please review the -variables you set in your playbooks and variable files. +If you have been using this collection before version `1.0.0`, note that a significant number of variables had to be renamed because of naming schema changes made by Ansible. Please review the variables you set in your playbooks and variable files. ### Ulimit management for Elasticsearch -The role no longer configures `ulimit`. Make sure the system's open file limit is set correctly, -packages usually handle this. Verify with `ulimit -n` or by checking `/proc//limits`. If -your version does not, please -[open an issue](https://github.com/NETWAYS/ansible-collection-elasticstack/issues). +The role no longer configures `ulimit`. Make sure the system's open file limit is set correctly, packages usually handle this. Verify with `ulimit -n` or by checking `/proc//limits`. If your version does not, please [open an issue](https://github.com/NETWAYS/ansible-collection-elasticstack/issues). ## Contributing to this collection -Every kind of contribution is welcome. Open -[issues](https://github.com/NETWAYS/ansible-collection-elasticstack/issues) or provide -[pull requests](https://github.com/NETWAYS/ansible-collection-elasticstack/pulls). +Every kind of contribution is welcome. Open [issues](https://github.com/NETWAYS/ansible-collection-elasticstack/issues) or provide [pull requests](https://github.com/NETWAYS/ansible-collection-elasticstack/pulls). Pull requests go against `main`. If you need a stable state, pin a release tag. -Please read [CONTRIBUTING.md](CONTRIBUTING.md) first. It describes what a pull request needs, -including the changelog fragment that every pull request has to bring along, how role variables -are documented and how to run the tests. +Please read [CONTRIBUTING.md](CONTRIBUTING.md) first. It describes what a pull request needs, including the changelog fragment that every pull request has to bring along, how role variables are documented and how to run the tests. ## Release notes -See the [releases page](https://github.com/NETWAYS/ansible-collection-elasticstack/releases). -The same notes are collected in `CHANGELOG.md`, which is generated from the changelog fragments -when a version is released. +See the [releases page](https://github.com/NETWAYS/ansible-collection-elasticstack/releases). The same notes are collected in `CHANGELOG.md`, which is generated from the changelog fragments when a version is released. ## Licensing diff --git a/RELEASE.md b/RELEASE.md index 325beec4..e1f368ee 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -11,19 +11,14 @@ How to build, tag and publish a new version of this collection. Maintainers only pip install "antsibull-changelog==0.35.1" ``` -* The repository or organization secret `GALAXY_API_KEY`, holding an API token of an - account that owns the `netways` namespace on Ansible Galaxy. Tokens are created in the - account settings on . -* Permission to run workflows in this repository, or the token itself if you intend to - publish from your own machine. See step 6. +* The repository or organization secret `GALAXY_API_KEY`, holding an API token of an account that owns the `netways` namespace on Ansible Galaxy. Tokens are created in the account settings on . +* Permission to run workflows in this repository, or the token itself if you intend to publish from your own machine. See step 6. -Galaxy versions are **immutable**. Once a version is uploaded it cannot be replaced or -removed, only superseded by a higher version. +Galaxy versions are **immutable**. Once a version is uploaded it cannot be replaced or removed, only superseded by a higher version. ## 1. Set the version -Set `version:` in **galaxy.yml** to the version you are about to release. Semantic -versioning, no `v` prefix. +Set `version:` in **galaxy.yml** to the version you are about to release. Semantic versioning, no `v` prefix. ## 2. Write the release summary @@ -38,18 +33,14 @@ release_summary: | ## 3. Generate the changelog -Every pull request brings its own fragment under `changelogs/fragments/`. This folds all -of them into `changelogs/changelog.yaml`, renders `CHANGELOG.md` and deletes the -fragments. +Every pull request brings its own fragment under `changelogs/fragments/`. This folds all of them into `changelogs/changelog.yaml`, renders `CHANGELOG.md` and deletes the fragments. ```bash antsibull-changelog lint antsibull-changelog release --version ``` -Steps 1 to 3 change files in the repository. `main` requires an approving review, so open -a pull request with the version bump and the generated changelog. -Do not push this directly, even though repository admins are technically able to. +Steps 1 to 3 change files in the repository. `main` requires an approving review, so open a pull request with the version bump and the generated changelog. Do not push this directly, even though repository admins are technically able to. ## 4. Tag the version @@ -71,9 +62,7 @@ If you cannot push tags, skip this and let GitHub create the tag in step 5 inste * choose the tag from step 4, or enter the version and let GitHub create the tag now * use the version as the title * paste this version's section from `CHANGELOG.md` as the release notes -* credit people by name where a change came from outside: the author of the pull request, - and the reporter of an issue where the report was the contribution. `git shortlog -sn HEAD` - lists everyone with their number of commits, folded per person through [.mailmap](.mailmap) +* credit people by name where a change came from outside: the author of the pull request, and the reporter of an issue where the report was the contribution. `git shortlog -sn HEAD` lists everyone with their number of commits, folded per person through [.mailmap](.mailmap) The release has to exist before step 6, because the archive is uploaded to it. @@ -81,19 +70,13 @@ The release has to exist before step 6, because the archive is uploaded to it. **Actions** -> **publish** -> **Run workflow**, then enter the tag. -The workflow checks out that tag, builds the collection, publishes it to Galaxy and -attaches the archive to the release from step 5. It is triggered manually and never -automatically, so that a tag can be inspected before an immutable version reaches Galaxy. +The workflow checks out that tag, builds the collection, publishes it to Galaxy and attaches the archive to the release from step 5. It is triggered manually and never automatically, so that a tag can be inspected before an immutable version reaches Galaxy. -Afterwards confirm on - that the version -arrived. Publishing can print errors and still have worked. +Afterwards confirm on that the version arrived. Publishing can print errors and still have worked. ### If you cannot run workflows -Do the same from your own machine. **Clone the tag into a fresh directory first.** -`ansible-galaxy collection build` packs the working directory and not the git tree, so -untracked local files end up in the archive unless `build_ignore` happens to cover them. +Do the same from your own machine. **Clone the tag into a fresh directory first.** `ansible-galaxy collection build` packs the working directory and not the git tree, so untracked local files end up in the archive unless `build_ignore` happens to cover them. ```bash git clone --branch git@github.com:NETWAYS/ansible-collection-elasticstack.git release_ @@ -102,8 +85,7 @@ ansible-galaxy collection build -vvv ansible-galaxy collection publish --token netways-elasticstack-.tar.gz ``` -`-vvv` lists the files that were skipped, so you can see whether `build_ignore` did its -job. Then attach the archive to the release by hand, because nothing uploaded it for you: +`-vvv` lists the files that were skipped, so you can see whether `build_ignore` did its job. Then attach the archive to the release by hand, because nothing uploaded it for you: ```bash gh release upload netways-elasticstack-.tar.gz diff --git a/docs/getting-started.md b/docs/getting-started.md index 71618201..324ce3f8 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -1,56 +1,37 @@ # Getting started -This page shows how to run the collection. For installation see the -[README](../README.md#using-this-collection), for what the roles expect from your inventory see -[Requirements](requirements.md). +This page shows how to run the collection. For installation see the [README](../README.md#using-this-collection), for what the roles expect from your inventory see [Requirements](requirements.md). ## Inventory groups -The roles place services on hosts by inventory group name. The group names and their defaults -are listed in [Requirements](requirements.md#inventory-group-names). +The roles place services on hosts by inventory group name. The group names and their defaults are listed in [Requirements](requirements.md#inventory-group-names). -Hosts in these groups get the respective services. Restricting your plays to only the -appropriate hosts does **not** work, because the roles read facts from hosts in other groups, -for example to generate certificates. +Hosts in these groups get the respective services. Restricting your plays to only the appropriate hosts does **not** work, because the roles read facts from hosts in other groups, for example to generate certificates. -Make sure every host that should be configured is part of your playbook, including hosts that -only run Beats. The collection first gathers facts from all hosts and then uses facts such as -host names and IP addresses to connect the tools to each other. +Make sure every host that should be configured is part of your playbook, including hosts that only run Beats. The collection first gathers facts from all hosts and then uses facts such as host names and IP addresses to connect the tools to each other. You will want reliable DNS resolution, or all hosts of the stack in your systems' hosts files. ## Before the first run -Redis has to be running for the default setup, because the default Logstash pipeline uses it. -The example playbook below gets it there with the `geerlingguy.redis` role, listed under -external requirements in the [README](../README.md#external-requirements). +Redis has to be running for the default setup, because the default Logstash pipeline uses it. The example playbook below gets it there with the `geerlingguy.redis` role, listed under external requirements in the [README](../README.md#external-requirements). -The default Beats configuration collects file system logs written by `rsyslog`, which is why the -example playbook installs it. Without syslog you will not receive any messages with the default -configuration. If you do not want that, change the configuration of the beats role. +The default Beats configuration collects file system logs written by `rsyslog`, which is why the example playbook installs it. Without syslog you will not receive any messages with the default configuration. If you do not want that, change the configuration of the beats role. -`elasticstack_variant: oss` is not available for the whole stack on every release. OSS -Elasticsearch and Kibana exist only up to Elastic Stack `7`, while OSS Logstash and Beats are -available on later releases too. +`elasticstack_variant: oss` is not available for the whole stack on every release. OSS Elasticsearch and Kibana exist only up to Elastic Stack `7`, while OSS Logstash and Beats are available on later releases too. ## Variables -Every role has its own set of variables, documented in the role's README. In addition the -collection-wide `elasticstack_*` variables apply to any role and are documented centrally with -the [elasticstack role](../roles/elasticstack/README.md). Two common ones: +Every role has its own set of variables, documented in the role's README. In addition the collection-wide `elasticstack_*` variables apply to any role and are documented centrally with the [elasticstack role](../roles/elasticstack/README.md). Two common ones: -* `elasticstack_release`: major release version of the Elastic Stack to configure - (default: `8`) -* `elasticstack_variant`: variant of the stack to install, `elastic` or `oss` - (default: `elastic`) +* `elasticstack_release`: major release version of the Elastic Stack to configure (default: `8`) +* `elasticstack_variant`: variant of the stack to install, `elastic` or `oss` (default: `elastic`) For pinning a specific version and for upgrading, see [Versions and upgrades](upgrades.md). ## Example playbook -The execution order of the plays matters. There are comments in the playbook below: either fill -them with the correct values, such as `remote_user`, or take them as a hint about commonly used -options. +The execution order of the plays matters. There are comments in the playbook below: either fill them with the correct values, such as `remote_user`, or take them as a hint about commonly used options. ```yaml --- @@ -123,22 +104,14 @@ options. - beats ``` -The `collections` entry only makes the short role names resolvable. It is per play, which is why -it appears in every one of them. Inside a role it has no effect, so keep using fully qualified -module names such as `ansible.builtin.package` in your own tasks. +The `collections` entry only makes the short role names resolvable. It is per play, which is why it appears in every one of them. Inside a role it has no effect, so keep using fully qualified module names such as `ansible.builtin.package` in your own tasks. ## Generated passwords -Passwords for the built-in users are generated on the CA host during the first run. They are -printed while they are generated and stored in the file that `elasticstack_initial_passwords` -points to, by default `/usr/share/elasticsearch/initial_passwords`. +Passwords for the built-in users are generated on the CA host during the first run. They are printed while they are generated and stored in the file that `elasticstack_initial_passwords` points to, by default `/usr/share/elasticsearch/initial_passwords`. -Security is on by default and controlled by `elasticstack_security`. A related but much -narrower switch is `elasticstack_override_beats_tls`: it only stops TLS between Beats and -Logstash from being enabled automatically, and skips the Beats certificate generation. It does -not turn off security anywhere else. +Security is on by default and controlled by `elasticstack_security`. A related but much narrower switch is `elasticstack_override_beats_tls`: it only stops TLS between Beats and Logstash from being enabled automatically, and skips the Beats certificate generation. It does not turn off security anywhere else. ## Seeing what the tasks do -`elasticstack_no_log` defaults to `true`, because some tasks could reveal passwords in -production. Set it to `false` if you want to see the output of all tasks. +`elasticstack_no_log` defaults to `true`, because some tasks could reveal passwords in production. Set it to `false` if you want to see the output of all tasks. diff --git a/docs/requirements.md b/docs/requirements.md index 494f0e6d..92159aed 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -1,14 +1,10 @@ # Requirements -Software requirements, supported Ansible and Python versions and external libraries are listed -in the [README](../README.md#tested-with-ansible). This page describes what the roles expect -from your setup. +Software requirements, supported Ansible and Python versions and external libraries are listed in the [README](../README.md#tested-with-ansible). This page describes what the roles expect from your setup. ## Inventory group names -The collection provides roles for several tools, and some tasks fetch hosts from a specific -inventory group. The group names are customizable through variables, but using group names that -differ from those variables results in errors. +The collection provides roles for several tools, and some tasks fetch hosts from a specific inventory group. The group names are customizable through variables, but using group names that differ from those variables results in errors. | Variable | Default group name | |---|---| @@ -18,8 +14,6 @@ differ from those variables results in errors. ## elasticstack_ca_host -This variable defines the host that acts as the certificate authority. It has a default: the -first host in the group named by `elasticstack_elasticsearch_group_name`. +This variable defines the host that acts as the certificate authority. It has a default: the first host in the group named by `elasticstack_elasticsearch_group_name`. -That default only resolves when that group exists. If you use different group names without -setting them in the variables above, the variable ends up empty and the run fails. +That default only resolves when that group exists. If you use different group names without setting them in the variables above, the variable ends up empty and the run fails. diff --git a/docs/upgrades.md b/docs/upgrades.md index 3ccff87a..ee08868a 100644 --- a/docs/upgrades.md +++ b/docs/upgrades.md @@ -2,25 +2,18 @@ ## Choosing a version -`elasticstack_version` holds the version number of the tools to install, for example `8.19.4`. -Only set it if you do not want the newest available version on a new setup. There is no default. +`elasticstack_version` holds the version number of the tools to install, for example `8.19.4`. Only set it if you do not want the newest available version on a new setup. There is no default. -If you already have an installation of the Elastic Stack, the collection queries the installed -version of Elasticsearch on the CA host and uses it for all further installations in the same -setup. This only happens when you run the `elasticsearch` role before all others. +If you already have an installation of the Elastic Stack, the collection queries the installed version of Elasticsearch on the CA host and uses it for all further installations in the same setup. This only happens when you run the `elasticsearch` role before all others. All packages are installed with `state: present`: -* When `elasticstack_version` is set to a version number, that exact version is installed and - pinned. -* When it is left unset, the package is installed without a version. A new setup gets the newest - available version, and existing installations are not upgraded automatically on later runs. +* When `elasticstack_version` is set to a version number, that exact version is installed and pinned. +* When it is left unset, the package is installed without a version. A new setup gets the newest available version, and existing installations are not upgraded automatically on later runs. -`elasticstack_release` holds the major release, for example `8` (default). If you set both, make -sure they match. +`elasticstack_release` holds the major release, for example `8` (default). If you set both, make sure they match. -For the OSS variant see `elasticstack_variant`, described with the -[elasticstack role](../roles/elasticstack/README.md). +For the OSS variant see `elasticstack_variant`, described with the [elasticstack role](../roles/elasticstack/README.md). Both variables can be set per role, which is how you keep components on different versions: @@ -40,14 +33,8 @@ roles: Set `elasticstack_version` to the version you want to upgrade to. -Before you do, read and understand Elastic's changelog and the breaking changes of your target -version **and of every version between your current one and the target**. Do not upgrade without -a valid backup. +Before you do, read and understand Elastic's changelog and the breaking changes of your target version **and of every version between your current one and the target**. Do not upgrade without a valid backup. -Elasticsearch nodes are upgraded one at a time. The role disables shard allocation, stops the -node, installs the new package, starts it again and waits for the node to rejoin before it -touches the next one. +Elasticsearch nodes are upgraded one at a time. The role disables shard allocation, stops the node, installs the new package, starts it again and waits for the node to rejoin before it touches the next one. -If an upgrade fails, you can try re-running the collection with the same settings. Several tasks -provide a degree of self-healing. Do not rely on these mechanisms, they are a convenience -recovery for the easier cases and not a substitute for a backup. +If an upgrade fails, you can try re-running the collection with the same settings. Several tasks provide a degree of self-healing. Do not rely on these mechanisms, they are a convenience recovery for the easier cases and not a substitute for a backup. diff --git a/plugins/module_utils/README.md b/plugins/module_utils/README.md index 48073ffb..f9c88ddd 100644 --- a/plugins/module_utils/README.md +++ b/plugins/module_utils/README.md @@ -1,7 +1,6 @@ # Documentation: netways.elasticstack module_utils -Shared code used by the modules in `plugins/modules/`. `ansible-doc` does not read this -directory, which is why it is documented here by hand instead of in a `DOCUMENTATION` block. +Shared code used by the modules in `plugins/modules/`. `ansible-doc` does not read this directory, which is why it is documented here by hand instead of in a `DOCUMENTATION` block. ## Overview - [`certs`](#certs), reading PKCS12 certificates. Used by `cert_info`. @@ -11,12 +10,9 @@ directory, which is why it is documented here by hand instead of in a `DOCUMENTA ## `certs` -Requires the `cryptography` library (>= 2.5). The module that uses it fails with -`missing_required_lib()` when it is absent. +Requires the `cryptography` library (>= 2.5). The module that uses it fails with `missing_required_lib()` when it is absent. -`SUPPORTED_EXTENSIONS` is a dict at the top of the file. It maps an X.509 extension name to the -list of its attribute names that may be reported. Nothing outside it ever reaches the result, so -extending the module means extending that dict. +`SUPPORTED_EXTENSIONS` is a dict at the top of the file. It maps an X.509 extension name to the list of its attribute names that may be reported. Nothing outside it ever reaches the result, so extending the module means extending that dict. ### `bytes_to_hex()` function @@ -24,7 +20,7 @@ Since binascii.hexlify doesn't support a second parameter, which would define a **Parameter:** A __bytes__ object that represent a hexadecimal value (e.g. b'\\x82S \\x11\\xc7s\\xa7^*w\\xc1\\xdf\"\\xe4#\\xb4\\xc4P\\xba\\xcf') -**Return:** A hexadecimal __string__ separated by colons (e.g. "82:53:20:11:C7:73:A7:5E:2A:77:C1:DF:22:E4:23:B4:C4:50:BA:CF") +**Return:** A hexadecimal __string__ separated by colons (e.g. "82:53:20:11:C7:73:A7:5E:2A:77:C1:DF:22:E4:23:B4:C4:50:BA:CF") ### `check_supported_extensions()` function @@ -36,74 +32,51 @@ A function to check if the extension is supported. Returns true if extension nam ### `check_supported_keys` function -A function to check if the extensions key is supported. Returns true if the key is listed under -the given extension in the `SUPPORTED_EXTENSIONS` dict. Unlike `check_supported_extensions()` -this compares exactly, not by substring. +A function to check if the extensions key is supported. Returns true if the key is listed under the given extension in the `SUPPORTED_EXTENSIONS` dict. Unlike `check_supported_extensions()` this compares exactly, not by substring. -**Parameter:** Two, in this order: the key name as __string__, and the extension name as -__string__. The extension name has to be a key of `SUPPORTED_EXTENSIONS`, otherwise the lookup -raises a `KeyError`. +**Parameter:** Two, in this order: the key name as __string__, and the extension name as __string__. The extension name has to be a key of `SUPPORTED_EXTENSIONS`, otherwise the lookup raises a `KeyError`. **Return:** Returns a __bool__. ### `AnalyzeCertificate()` object -An object to load the certificate and to gather information about it. Constructing it already -does the work: it reads the file and fills the result dict, so there is no separate call to -start the analysis. +An object to load the certificate and to gather information about it. Constructing it already does the work: it reads the file and fills the result dict, so there is no separate call to start the analysis. -**Parameter:** Two, in this order: the `AnsibleModule` __object__ and the result __dict__ to -fill. The certificate path and the passphrase are not passed in, the object reads them from -`module.params`. It also uses the module to log, to warn and to fail. +**Parameter:** Two, in this order: the `AnsibleModule` __object__ and the result __dict__ to fill. The certificate path and the passphrase are not passed in, the object reads them from `module.params`. It also uses the module to log, to warn and to fail. -**Return:** The constructor returns nothing, it writes into the result dict it was given. -`return_result()` hands that dict back to the module, which passes it to `exit_json()`. +**Return:** The constructor returns nothing, it writes into the result dict it was given. `return_result()` hands that dict back to the module, which passes it to `exit_json()`. -A broken or unreadable extension does not fail the module. It is skipped with `module.warn()`, -so a task can succeed with fewer extensions in the result than the certificate contains. +A broken or unreadable extension does not fail the module. It is skipped with `module.warn()`, so a task can succeed with fewer extensions in the result than the certificate contains. ## `api` -Builds the Elasticsearch client for the two Elasticsearch modules. Requires the `elasticsearch` -Python library, currently a version below 9. +Builds the Elasticsearch client for the two Elasticsearch modules. Requires the `elasticsearch` Python library, currently a version below 9. ### `HAS_ELASTICSEARCH` and `ELASTICSEARCH_IMPORT_ERROR` -Set while the file is imported. `HAS_ELASTICSEARCH` is a __bool__, -`ELASTICSEARCH_IMPORT_ERROR` holds the formatted traceback as a __string__ or `None`. A module -checks the flag and reports the traceback, so a missing library produces a readable message -instead of an import error. +Set while the file is imported. `HAS_ELASTICSEARCH` is a __bool__, `ELASTICSEARCH_IMPORT_ERROR` holds the formatted traceback as a __string__ or `None`. A module checks the flag and reports the traceback, so a missing library produces a readable message instead of an import error. ### `Api.new_client_basic_auth()` static method Returns a connected `Elasticsearch` client authenticated with username and password. -**Parameter:** Five keyword arguments: `host` (URL including protocol and port), `auth_user`, -`auth_pass`, `ca_certs` (path to a CA file, or `None`) and `verify_certs`. +**Parameter:** Five keyword arguments: `host` (URL including protocol and port), `auth_user`, `auth_pass`, `ca_certs` (path to a CA file, or `None`) and `verify_certs`. **Return:** An `Elasticsearch` __object__. Raises `ImportError` when the library is missing. -The SSL context it builds sets `check_hostname` and `verify_mode` to false before `verify_certs` -is handed to the client, so the hostname in the certificate is not checked. +The SSL context it builds sets `check_hostname` and `verify_mode` to false before `verify_certs` is handed to the client, so the hostname in the certificate is not checked. ## `elasticsearch_role` ### `Role()` object -The whole logic of the `elasticsearch_role` module. Constructing it already does the work: it -builds the client, decides between create, update and delete, and fills the result dict. +The whole logic of the `elasticsearch_role` module. Constructing it already does the work: it builds the client, decides between create, update and delete, and fills the result dict. -**Parameter:** Ten keyword arguments: the result __dict__ to fill, `role_name`, `cluster`, -`indicies`, `state`, and the connection arguments `host`, `auth_user`, `auth_pass`, -`verify_certs` and `ca_certs`. +**Parameter:** Ten keyword arguments: the result __dict__ to fill, `role_name`, `cluster`, `indicies`, `state`, and the connection arguments `host`, `auth_user`, `auth_pass`, `verify_certs` and `ca_certs`. -**Return:** The constructor returns nothing. `return_result()` hands the result dict back to the -module, which passes it to `exit_json()`. +**Return:** The constructor returns nothing. `return_result()` hands the result dict back to the module, which passes it to `exit_json()`. -Changes are detected against the API, not against the arguments. `handle_present()` reads the -role, writes it, and compares the response before and after, so `changed` stays false when the -role already looked that way. `handle_absent()` only deletes a role that the API reports as -existing. +Changes are detected against the API, not against the arguments. `handle_present()` reads the role, writes it, and compares the response before and after, so `changed` stays false when the role already looked that way. `handle_absent()` only deletes a role that the API reports as existing. ## `elasticsearch_user` @@ -111,13 +84,8 @@ existing. The whole logic of the `elasticsearch_user` module, built the same way as `Role()` above. -**Parameter:** Thirteen keyword arguments: the result __dict__ to fill, `user_name`, -`full_name`, `password`, `email`, `roles`, `enabled`, `state`, and the connection arguments -`host`, `auth_user`, `auth_pass`, `verify_certs` and `ca_certs`. +**Parameter:** Thirteen keyword arguments: the result __dict__ to fill, `user_name`, `full_name`, `password`, `email`, `roles`, `enabled`, `state`, and the connection arguments `host`, `auth_user`, `auth_pass`, `verify_certs` and `ca_certs`. -**Return:** The constructor returns nothing. `return_result()` hands the result dict back to the -module, which passes it to `exit_json()`. +**Return:** The constructor returns nothing. `return_result()` hands the result dict back to the module, which passes it to `exit_json()`. -Changes are detected the same way, by comparing the API response before and after the write. -A password cannot be read back, so setting the same password again is not reported as a change, -while setting a different one is not detected either. +Changes are detected the same way, by comparing the API response before and after the write. A password cannot be read back, so setting the same password again is not reported as a change, while setting a different one is not detected either. diff --git a/roles/beats/README.md b/roles/beats/README.md index 0e961017..41d70ece 100644 --- a/roles/beats/README.md +++ b/roles/beats/README.md @@ -1,15 +1,11 @@ # Ansible Role: Beats -Installs and configures the Elastic [Beats](https://www.elastic.co/beats) — -Filebeat, Auditbeat and Metricbeat — on Linux systems. The role can run -standalone or together with the other Elastic Stack roles. Beats can ship to -Logstash or directly to Elasticsearch. +Installs and configures the Elastic [Beats](https://www.elastic.co/beats) — Filebeat, Auditbeat and Metricbeat — on Linux systems. The role can run standalone or together with the other Elastic Stack roles. Beats can ship to Logstash or directly to Elasticsearch. ## Requirements * The `community.crypto` collection (used to check certificate expiration). -* The Beats you want to install must be available in your software repositories — - use the [`repos`](../repos) role or provide them yourself. +* The Beats you want to install must be available in your software repositories — use the [`repos`](../repos) role or provide them yourself. ## Example @@ -25,10 +21,7 @@ Logstash or directly to Elasticsearch. ## Filebeat inputs -Filebeat can read from several sources (log files, syslog, journald, Docker, -modules). For the structure of `beats_filebeat_log_inputs`, -`beats_filebeat_journald_inputs` and the other input variables, see the -[Filebeat inputs documentation](docs/filebeat-inputs.md). +Filebeat can read from several sources (log files, syslog, journald, Docker, modules). For the structure of `beats_filebeat_log_inputs`, `beats_filebeat_journald_inputs` and the other input variables, see the [Filebeat inputs documentation](docs/filebeat-inputs.md). ## Tags @@ -89,8 +82,4 @@ Run only parts of the role with `--tags`: ## Shared variables -This role also uses the collection-wide `elasticstack_*` variables (e.g. -`elasticstack_full_stack`, `elasticstack_variant`, `elasticstack_ca_host`, -`elasticstack_ca_pass`, `elasticstack_release`, `elasticstack_version`, -`elasticstack_elasticsearch_http_port`, `elasticstack_beats_port`). They are -documented centrally with the [elasticstack role](../elasticstack/README.md). +This role also uses the collection-wide `elasticstack_*` variables (e.g. `elasticstack_full_stack`, `elasticstack_variant`, `elasticstack_ca_host`, `elasticstack_ca_pass`, `elasticstack_release`, `elasticstack_version`, `elasticstack_elasticsearch_http_port`, `elasticstack_beats_port`). They are documented centrally with the [elasticstack role](../elasticstack/README.md). diff --git a/roles/beats/docs/filebeat-inputs.md b/roles/beats/docs/filebeat-inputs.md index e5a5203e..fc34a002 100644 --- a/roles/beats/docs/filebeat-inputs.md +++ b/roles/beats/docs/filebeat-inputs.md @@ -1,19 +1,12 @@ # Filebeat inputs -Filebeat can read from several sources. Each source is turned into a Filebeat -input by the role. This document describes the variables that configure them. -For the meaning of the individual Filebeat options, follow the links to the -official Filebeat documentation. +Filebeat can read from several sources. Each source is turned into a Filebeat input by the role. This document describes the variables that configure them. For the meaning of the individual Filebeat options, follow the links to the official Filebeat documentation. ## Log files Set with `beats_filebeat_log_input` (default `true`) and `beats_filebeat_log_inputs`. -`beats_filebeat_log_inputs` is a **dictionary keyed by a free name**. That key -becomes the id of the generated -[`filestream`](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-filestream) -input (`-filestream`) — there is no separate `name` field. Each entry needs -`paths`; `fields` and `multiline` are optional. +`beats_filebeat_log_inputs` is a **dictionary keyed by a free name**. That key becomes the id of the generated [`filestream`](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-filestream) input (`-filestream`) — there is no separate `name` field. Each entry needs `paths`; `fields` and `multiline` are optional. The default reads the system log: @@ -41,16 +34,13 @@ beats_filebeat_log_inputs: match: after ``` -`multiline` maps directly to the Filebeat multiline parser (`type`, `pattern`, -`negate`, `match`). See -[Manage multiline messages](https://www.elastic.co/docs/reference/beats/filebeat/multiline-examples). +`multiline` maps directly to the Filebeat multiline parser (`type`, `pattern`, `negate`, `match`). See [Manage multiline messages](https://www.elastic.co/docs/reference/beats/filebeat/multiline-examples). ## Fields There are two ways to add fields, and they use **different shapes**: -* **Per input** — the optional `fields` inside a log input is a **dictionary** - (`key: value`), added only to that input: +* **Per input** — the optional `fields` inside a log input is a **dictionary** (`key: value`), added only to that input: ```yaml beats_filebeat_log_inputs: @@ -61,9 +51,7 @@ There are two ways to add fields, and they use **different shapes**: environment: production ``` -* **Globally** — `beats_fields` is a **list of `"key: value"` strings**. In the - current templates it is added to the log, TCP and UDP inputs; the journald, - Docker and MySQL slow-log inputs do not receive it: +* **Globally** — `beats_fields` is a **list of `"key: value"` strings**. In the current templates it is added to the log, TCP and UDP inputs; the journald, Docker and MySQL slow-log inputs do not receive it: ```yaml beats_fields: @@ -73,12 +61,7 @@ There are two ways to add fields, and they use **different shapes**: ## Syslog over TCP/UDP -Enable a listening syslog input with `beats_filebeat_syslog_tcp` / -`beats_filebeat_syslog_udp` and set the port with `beats_filebeat_syslog_tcp_port` -/ `beats_filebeat_syslog_udp_port` (both default `514`). They become a -[`tcp`](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-tcp) -or [`udp`](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-udp) -input listening on `0.0.0.0:`. The global `beats_fields` are applied here too. +Enable a listening syslog input with `beats_filebeat_syslog_tcp` / `beats_filebeat_syslog_udp` and set the port with `beats_filebeat_syslog_tcp_port` / `beats_filebeat_syslog_udp_port` (both default `514`). They become a [`tcp`](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-tcp) or [`udp`](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-udp) input listening on `0.0.0.0:`. The global `beats_fields` are applied here too. ```yaml beats_filebeat_syslog_tcp: true @@ -87,13 +70,9 @@ beats_filebeat_syslog_tcp_port: 514 ## Journald -Enable with `beats_filebeat_journald` (default `false`, available since Filebeat -7.16) and configure inputs with `beats_filebeat_journald_inputs`. +Enable with `beats_filebeat_journald` (default `false`, available since Filebeat 7.16) and configure inputs with `beats_filebeat_journald_inputs`. -This is a **dictionary keyed by a free name**. Unlike the log inputs, here the -key is only a label — the input's id comes from the `id` field, which each entry -needs. `include_matches` is optional and is itself a dictionary whose values are -the match expressions. +This is a **dictionary keyed by a free name**. Unlike the log inputs, here the key is only a label — the input's id comes from the `id` field, which each entry needs. `include_matches` is optional and is itself a dictionary whose values are the match expressions. ```yaml beats_filebeat_journald_inputs: @@ -105,15 +84,11 @@ beats_filebeat_journald_inputs: unit: '_SYSTEMD_UNIT=sshd.service' ``` -See the -[`journald`](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-journald) -input. +See the [`journald`](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-journald) input. ## Docker -Enable with `beats_filebeat_docker` (default `false`) and select containers with -`beats_filebeat_docker_ids` (default `*`). **Only works on Elastic Stack release -7.** Docker metadata is added automatically. +Enable with `beats_filebeat_docker` (default `false`) and select containers with `beats_filebeat_docker_ids` (default `*`). **Only works on Elastic Stack release 7.** Docker metadata is added automatically. ```yaml beats_filebeat_docker: true @@ -122,20 +97,13 @@ beats_filebeat_docker_ids: "*" ## MySQL/MariaDB slow log -Set `beats_filebeat_mysql_slowlog_input` to `true` to collect -`/var/log/mysql/*-slow.log` with the matching multiline pattern already -configured. The events are tagged with `mysql.logtype: slowquery`. +Set `beats_filebeat_mysql_slowlog_input` to `true` to collect `/var/log/mysql/*-slow.log` with the matching multiline pattern already configured. The events are tagged with `mysql.logtype: slowquery`. ## Modules -`beats_filebeat_modules` is a list of Filebeat module names to enable -(**experimental**, unset by default). The role runs `filebeat modules enable` -and sets up their ingest pipelines. +`beats_filebeat_modules` is a list of Filebeat module names to enable (**experimental**, unset by default). The role runs `filebeat modules enable` and sets up their ingest pipelines. -On Elastic Stack release 8 and newer (`elasticstack_release > 7`), enabling any -modules also deploys a predefined **System module** configuration to -`modules.d/system.yml`, which collects the syslog files (`/var/log/syslog`, -`/var/log/messages`) through the system module's `syslog` fileset. +On Elastic Stack release 8 and newer (`elasticstack_release > 7`), enabling any modules also deploys a predefined **System module** configuration to `modules.d/system.yml`, which collects the syslog files (`/var/log/syslog`, `/var/log/messages`) through the system module's `syslog` fileset. ```yaml beats_filebeat_modules: @@ -143,6 +111,4 @@ beats_filebeat_modules: - nginx ``` -See the -[Filebeat modules](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-modules) -reference for the available modules. +See the [Filebeat modules](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-modules) reference for the available modules. diff --git a/roles/elasticsearch/README.md b/roles/elasticsearch/README.md index bac7afc0..9594c953 100644 --- a/roles/elasticsearch/README.md +++ b/roles/elasticsearch/README.md @@ -1,24 +1,13 @@ # Ansible Role: Elasticsearch -Installs and configures [Elasticsearch](https://www.elastic.co/elasticsearch) on -Linux systems. The role manages `elasticsearch.yml` and the JVM configuration -(heap and options). With security enabled (the default) it creates a self-signed -CA and a certificate for each node, enables transport and HTTP TLS, manages the -keystore, and generates the initial passwords; the same CA is reused by the -Logstash, Kibana and Beats roles for their own certificates. It configures -cluster discovery for single- or multi-node setups and upgrades the nodes one at -a time when you raise `elasticstack_version`. +Installs and configures [Elasticsearch](https://www.elastic.co/elasticsearch) on Linux systems. The role manages `elasticsearch.yml` and the JVM configuration (heap and options). With security enabled (the default) it creates a self-signed CA and a certificate for each node, enables transport and HTTP TLS, manages the keystore, and generates the initial passwords; the same CA is reused by the Logstash, Kibana and Beats roles for their own certificates. It configures cluster discovery for single- or multi-node setups and upgrades the nodes one at a time when you raise `elasticstack_version`. -The OSS variant is only available up to Elastic Stack 7; from release 8 on only -the default `elastic` variant exists, and security is required. +The OSS variant is only available up to Elastic Stack 7; from release 8 on only the default `elastic` variant exists, and security is required. ## Requirements * The Elastic repositories configured — use the [`repos`](../repos) role. -* For the default security setup the role generates certificates with the - bundled `elasticsearch-certutil` and `openssl`. The required Python libraries - (`cryptography`, `elasticsearch`) are installed for you by the shared - `elasticstack` role. +* For the default security setup the role generates certificates with the bundled `elasticsearch-certutil` and `openssl`. The required Python libraries (`cryptography`, `elasticsearch`) are installed for you by the shared `elasticstack` role. ## Example @@ -92,9 +81,4 @@ Run only parts of the role with `--tags`: ## Shared variables -This role also uses the collection-wide `elasticstack_*` variables (e.g. -`elasticstack_full_stack`, `elasticstack_variant`, `elasticstack_release`, -`elasticstack_ca_host`, `elasticstack_ca_pass`, `elasticstack_ca_dir`, -`elasticstack_elasticsearch_http_port`, `elasticstack_initial_passwords`). They -are documented centrally with the -[elasticstack role](../elasticstack/README.md). +This role also uses the collection-wide `elasticstack_*` variables (e.g. `elasticstack_full_stack`, `elasticstack_variant`, `elasticstack_release`, `elasticstack_ca_host`, `elasticstack_ca_pass`, `elasticstack_ca_dir`, `elasticstack_elasticsearch_http_port`, `elasticstack_initial_passwords`). They are documented centrally with the [elasticstack role](../elasticstack/README.md). diff --git a/roles/kibana/README.md b/roles/kibana/README.md index 728b7f57..68cf3a21 100644 --- a/roles/kibana/README.md +++ b/roles/kibana/README.md @@ -1,18 +1,12 @@ # Ansible Role: Kibana -Installs and configures [Kibana](https://www.elastic.co/kibana) on Linux systems. -The role manages `kibana.yml` and can enable TLS for the Kibana web interface, -using a certificate you provide. In a full stack setup together with the -elasticsearch role, it also configures the connection to Elasticsearch — the TLS -trust and the `kibana_system` credentials — using the Elasticsearch CA. +Installs and configures [Kibana](https://www.elastic.co/kibana) on Linux systems. The role manages `kibana.yml` and can enable TLS for the Kibana web interface, using a certificate you provide. In a full stack setup together with the elasticsearch role, it also configures the connection to Elasticsearch — the TLS trust and the `kibana_system` credentials — using the Elasticsearch CA. ## Requirements * The Elastic repositories configured — use the [`repos`](../repos) role. * A reachable Elasticsearch instance for Kibana to connect to. -* For browser-facing TLS (`kibana_tls: true`): a certificate and key from a CA your - users trust (corporate PKI or a public CA), provided via `kibana_tls_cert` and - `kibana_tls_key`. The role does **not** generate a browser certificate. +* For browser-facing TLS (`kibana_tls: true`): a certificate and key from a CA your users trust (corporate PKI or a public CA), provided via `kibana_tls_cert` and `kibana_tls_key`. The role does **not** generate a browser certificate. ## Example @@ -26,9 +20,7 @@ trust and the `kibana_system` credentials — using the Elasticsearch CA. - kibana ``` -By default Kibana only logs to the journal, readable with -`journalctl -u kibana`. Set `kibana_manage_logging` to write a log file that -Kibana rotates itself, no logrotate needed: +By default Kibana only logs to the journal, readable with `journalctl -u kibana`. Set `kibana_manage_logging` to write a log file that Kibana rotates itself, no logrotate needed: ```yaml - name: Install Kibana with file logging @@ -46,10 +38,7 @@ Kibana rotates itself, no logrotate needed: - kibana ``` -This writes `/var/log/kibana/kibana.log` in a human readable layout and keeps -ten rotated files. Set `kibana_logging_layout` to `json` when a log shipper -reads the file, and `kibana_logging_console` to `false` to stop logging to the -journal as well. +This writes `/var/log/kibana/kibana.log` in a human readable layout and keeps ten rotated files. Set `kibana_logging_layout` to `json` when a log shipper reads the file, and `kibana_logging_console` to `false` to stop logging to the journal as well. ## Tags @@ -95,8 +84,4 @@ Run only parts of the role with `--tags`: ## Shared variables -This role also uses the collection-wide `elasticstack_*` variables (e.g. -`elasticstack_full_stack`, `elasticstack_variant`, `elasticstack_release`, -`elasticstack_ca_host`, `elasticstack_ca_pass`, `elasticstack_kibana_host`, -`elasticstack_kibana_port`, `elasticstack_elasticsearch_http_port`). They are -documented centrally with the [elasticstack role](../elasticstack/README.md). +This role also uses the collection-wide `elasticstack_*` variables (e.g. `elasticstack_full_stack`, `elasticstack_variant`, `elasticstack_release`, `elasticstack_ca_host`, `elasticstack_ca_pass`, `elasticstack_kibana_host`, `elasticstack_kibana_port`, `elasticstack_elasticsearch_http_port`). They are documented centrally with the [elasticstack role](../elasticstack/README.md). diff --git a/roles/logstash/README.md b/roles/logstash/README.md index 10496b6a..eaa1ee4d 100644 --- a/roles/logstash/README.md +++ b/roles/logstash/README.md @@ -1,11 +1,6 @@ # Ansible Role: Logstash -Installs and configures [Logstash](https://www.elastic.co/products/logstash) on -Linux systems. The role can manage `logstash.yml`, the log4j2 logging, the JVM -heap, TLS, and the Elasticsearch writer role/user. For pipelines it can create -two default pipelines — a Beats input and an Elasticsearch forwarder, both using -Redis — and manage your own pipelines, whose configuration can be checked out -from external git repositories. +Installs and configures [Logstash](https://www.elastic.co/products/logstash) on Linux systems. The role can manage `logstash.yml`, the log4j2 logging, the JVM heap, TLS, and the Elasticsearch writer role/user. For pipelines it can create two default pipelines — a Beats input and an Elasticsearch forwarder, both using Redis — and manage your own pipelines, whose configuration can be checked out from external git repositories. It works with the standard Elastic Stack packages and with Elastic's OSS variant. @@ -15,8 +10,7 @@ It works with the standard Elastic Stack packages and with Elastic's OSS variant * `passlib` (Python, on the controller) unless you disable password hashing for the Logstash user. * `curl` on the target host. * `git` on the target host if you use git-based pipelines. -* Redis if you use the default pipeline or other pipelines that talk to Redis - (e.g. via the [`geerlingguy.redis`](https://galaxy.ansible.com/geerlingguy/redis) role). +* Redis if you use the default pipeline or other pipelines that talk to Redis (e.g. via the [`geerlingguy.redis`](https://galaxy.ansible.com/geerlingguy/redis) role). You also need the Elastic repositories configured — use the [`repos`](../repos) role. @@ -34,8 +28,7 @@ You also need the Elastic repositories configured — use the [`repos`](../repos ## Pipelines -For how to configure pipelines (built-in Redis/Beats and external git -repositories), see the [pipelines documentation](docs/pipelines.md). +For how to configure pipelines (built-in Redis/Beats and external git repositories), see the [pipelines documentation](docs/pipelines.md). ## Tags @@ -143,7 +136,4 @@ Run only parts of the role with `--tags`: ## Shared variables -This role also uses the collection-wide `elasticstack_*` variables (e.g. -`elasticstack_full_stack`, `elasticstack_ca_host`, `elasticstack_ca_pass`, -`elasticstack_release`, `elasticstack_variant`). They are documented centrally -with the [elasticstack role](../elasticstack/README.md). +This role also uses the collection-wide `elasticstack_*` variables (e.g. `elasticstack_full_stack`, `elasticstack_ca_host`, `elasticstack_ca_pass`, `elasticstack_release`, `elasticstack_variant`). They are documented centrally with the [elasticstack role](../elasticstack/README.md). diff --git a/roles/repos/README.md b/roles/repos/README.md index 7080bf68..c0ca4562 100644 --- a/roles/repos/README.md +++ b/roles/repos/README.md @@ -1,16 +1,12 @@ # Ansible Role: Repos -Adds the Elastic package repositories to the target host so the other roles can -install Elastic Stack packages. It supports apt (Debian/Ubuntu), yum (RedHat) and -zypper (SUSE), for both the standard Elastic and the OSS variant. +Adds the Elastic package repositories to the target host so the other roles can install Elastic Stack packages. It supports apt (Debian/Ubuntu), yum (RedHat) and zypper (SUSE), for both the standard Elastic and the OSS variant. -Its main use is together with the other roles of this collection, which expect -the Elastic repositories to be present. +Its main use is together with the other roles of this collection, which expect the Elastic repositories to be present. ## Requirements -* The `community.general` collection — only needed on SUSE hosts, where the role - uses the `zypper` modules. +* The `community.general` collection — only needed on SUSE hosts, where the role uses the `zypper` modules. * Network access from the target host to the Elastic repository URL. The role installs the required signing tools (`gpg`/`gnupg`) itself. @@ -35,8 +31,4 @@ No variables are defined for this role. ## Shared variables -This role is configured through the collection-wide `elasticstack_*` variables — -mainly `elasticstack_release`, `elasticstack_variant`, `elasticstack_enable_repos`, -`elasticstack_repo_url`, `elasticstack_repo_key` and `elasticstack_rpm_workaround`. -They are documented centrally with the -[elasticstack role](../elasticstack/README.md). +This role is configured through the collection-wide `elasticstack_*` variables — mainly `elasticstack_release`, `elasticstack_variant`, `elasticstack_enable_repos`, `elasticstack_repo_url`, `elasticstack_repo_key` and `elasticstack_rpm_workaround`. They are documented centrally with the [elasticstack role](../elasticstack/README.md). From 6b42993f494e6c227fee5babc83f7ebb65b39051 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 14 Sep 2026 15:53:28 +0200 Subject: [PATCH 52/53] Note the README table layout in the changelog fragment --- changelogs/fragments/prepare_galaxy_release.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/changelogs/fragments/prepare_galaxy_release.yml b/changelogs/fragments/prepare_galaxy_release.yml index 9cef7dc8..54131551 100644 --- a/changelogs/fragments/prepare_galaxy_release.yml +++ b/changelogs/fragments/prepare_galaxy_release.yml @@ -25,3 +25,7 @@ trivial: resources this collection does not touch, and of the remaining ones only two ever fired, one of them a false positive caused by the query not recognising fully qualified module names. + - The role variable tables now have two columns, with type, required, default and choices + stacked under the variable name. Five columns left the description too narrow to read + without scrolling the table sideways. Required is only rendered where a variable actually + is required. From 8702045accef98f853d6eeacce6da514d9e02d1e Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Mon, 14 Sep 2026 15:55:40 +0200 Subject: [PATCH 53/53] Modify the name in docs --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1184c085..60838568 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Ansible Collection - netways.elasticstack +# Ansible Collection Elasticstack This collection installs and manages the Elastic Stack. It provides roles for every component of the Stack, and it can install either the default Elastic distribution or the Apache-licensed OSS packages.