diff --git a/.docsmith/readme.md.j2 b/.docsmith/readme.md.j2 index ff54a137..12be8b55 100644 --- a/.docsmith/readme.md.j2 +++ b/.docsmith/readme.md.j2 @@ -1,11 +1,14 @@ {# Compact README variable reference for ansible-docsmith. Shared by all roles. - Renders ONE table; nested sub-options (e.g. logstash_pipelines fields) appear as - indented rows. No per-variable detail sections, no table of contents. + Renders ONE two-column table. Type, required, default and choices are stacked + under the variable name so that only two columns compete for the page width and + the description never needs horizontal scrolling. Nested sub-options (e.g. + logstash_pipelines fields) appear as indented rows. Use with: --template-readme .docsmith/readme.md.j2 #} {% macro row(name, spec, depth) -%} {% set pad = '    ' * depth %} {% set mark = '↳ ' if depth > 0 else '' %} -| {{ pad }}{{ mark }}`{{ name | ansible_escape }}` | `{{ spec.type }}`{% if spec.elements %} of `{{ spec.elements }}`{% endif %} | {% if spec.options %}…{% else %}{{ spec.default | format_default | replace('|', '\\|') }}{% endif %} | {% if spec.choices %}{{ spec.choices | map('string') | map('code_escape') | join(', ') }}{% else %}—{% endif %} | {{ spec.description | format_table_description(name, 0) }} | +{% set default = spec.default | format_default | replace('|', '\\|') -%} +| {{ pad }}{{ mark }}`{{ name | ansible_escape }}`
**Type**: `{{ spec.type }}`{% if spec.elements %} of `{{ spec.elements }}`{% endif %}{% if spec.required %}
**Required**: Yes{% endif %}{% if not spec.options and default != 'N/A' %}
**Default**: {{ default }}{% endif %}{% if spec.choices %}
**Choices**: {{ spec.choices | map('string') | map('code_escape') | join(', ') }}{% endif %} | {{ spec.description | format_table_description(name, 0) }} | {% for sn, ss in (spec.options or {}).items() %} {{ row(sn, ss, depth + 1) }} {%- endfor %} @@ -13,8 +16,8 @@ ## Role variables {% if has_options %} -| Variable | Type | Default | Choices | Description | -|----------|------|---------|---------|-------------| +| Variable | Description | +|----------|-------------| {% for name, spec in options.items() %} {{ row(name, spec, 0) }} {%- endfor %} diff --git a/.github/workflows/kics.yml b/.github/workflows/kics.yml deleted file mode 100644 index 1b27515e..00000000 --- a/.github/workflows/kics.yml +++ /dev/null @@ -1,47 +0,0 @@ ---- -name: KICS Security Scan -on: - workflow_dispatch: - inputs: - logLevel: - description: 'Log level' - required: true - default: 'warning' - type: choice - options: - - info - - warning - - debug - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - -# Cancel outdated pipeline runs when a new push occurs in the pull request and a new pipeline starts. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - security_scan: - # For non-PR events (workflow_dispatch, schedule, etc.), github.event.pull_request.draft is null. - # GitHub Actions coerces null == false to true (null → 0, false → 0), so the job runs for all non-draft events. - if: github.event.pull_request.draft == false - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v7 - - name: Mkdir results-dir - # make sure results dir is created - run: mkdir -p results-dir - - name: run kics Scan - uses: Checkmarx/kics-github-action@bcd2aff52160bfe5a72608452d00326e90da7a9a # v2.1.19 - with: - # path: 'roles,plugins' - path: '.' - # fail_on: high - ignore_on_exit: results - output_formats: 'json,sarif' - output_path: results-dir - - name: Upload SARIF file - uses: github/codeql-action/upload-sarif@af56b044b5d41c317aef5d19920b3183cb4fbbec # v3 - with: - sarif_file: results-dir/results.sarif diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..00a74388 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,58 @@ +--- +# Publishes a tagged version of this collection to Ansible Galaxy and attaches the +# built archive to the matching GitHub release. +# +# Triggered manually and never automatically, so that a tag can be inspected before an +# immutable version reaches Galaxy. Both the tag and the release have to exist before +# the workflow is started: the build happens from the checked out tag, galaxy.yml at +# that tag determines the published version, and the archive is uploaded to the release +# carrying the same name as the tag. +# +# Requires the repository or organization secret GALAXY_API_KEY, holding an API token of +# an account that owns the "netways" namespace on Ansible Galaxy. + +name: publish + +on: + workflow_dispatch: + inputs: + tag: + description: 'Existing tag to build and publish, for example 0.1.0' + required: true + type: string + +permissions: + contents: write + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - name: Check out the tag + uses: actions/checkout@v7 + with: + ref: ${{ inputs.tag }} + + - name: Set up Python 3.11 + uses: actions/setup-python@v7 + with: + python-version: "3.11" + + - name: Install ansible-core + run: pip install ansible-core + + - name: Show the version that will be published + run: grep '^version:' galaxy.yml + + - name: Build the collection + run: ansible-galaxy collection build -vvv + + - name: Publish to Ansible Galaxy + env: + GALAXY_API_KEY: ${{ secrets.GALAXY_API_KEY }} + run: ansible-galaxy collection publish --token "$GALAXY_API_KEY" ./netways-elasticstack-*.tar.gz + + - name: Attach the archive to the release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh release upload "${{ inputs.tag }}" ./netways-elasticstack-*.tar.gz --clobber diff --git a/.github/workflows/test_changelog.yml b/.github/workflows/test_changelog.yml new file mode 100644 index 00000000..8966ab43 --- /dev/null +++ b/.github/workflows/test_changelog.yml @@ -0,0 +1,80 @@ +--- +# Every pull request has to bring a changelog fragment, so that the changelog can be +# generated at release time instead of being reconstructed from the commit history. +# +# A change that is not worth mentioning to users still needs a fragment, but its entry +# goes into the "trivial" section, which antsibull-changelog does not render. See +# CONTRIBUTING.md. + +name: Test Changelog + +on: + workflow_dispatch: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + fragment: + # Dependabot only bumps action and dependency versions, which needs no fragment. + if: >- + github.event.pull_request.draft == false + && github.actor != 'dependabot[bot]' + runs-on: ubuntu-latest + steps: + - name: Check out the codebase + uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Set up Python 3.11 + uses: actions/setup-python@v7 + with: + python-version: "3.11" + + - name: Install antsibull-changelog + run: pip install "antsibull-changelog==0.35.1" + + - name: A changelog fragment is present + env: + BASE_REF: ${{ github.event.pull_request.base.ref }} + run: | + if [ -z "$BASE_REF" ]; then + echo "Not a pull request, nothing to compare against." + exit 0 + fi + git fetch --no-tags origin "$BASE_REF" + added=$(git diff --name-only --diff-filter=A \ + "origin/$BASE_REF...HEAD" -- changelogs/fragments/ \ + | grep -v '^changelogs/fragments/\.keep$' || true) + if [ -z "$added" ]; then + echo "::error::This pull request adds no new changelog fragment." + echo "Add a NEW file under changelogs/fragments/ describing the change." + echo "Do not put the entry into an existing fragment, so that every entry" + echo "stays traceable to the pull request that added it." + echo "If the change is not relevant for users, put the entry in the" + echo "'trivial' section. See CONTRIBUTING.md." + exit 1 + fi + echo "New fragments in this pull request:" + echo "$added" + + touched=$(git diff --name-only --diff-filter=MRD \ + "origin/$BASE_REF...HEAD" -- changelogs/fragments/ \ + | grep -v '^changelogs/fragments/\.keep$' || true) + if [ -n "$touched" ]; then + echo "::error::This pull request changes fragments it did not add:" + echo "$touched" + echo "Fragments belong to the pull request that created them. Add your own" + echo "instead of editing, renaming or deleting someone else's." + exit 1 + fi + + - name: Fragments are valid + run: antsibull-changelog lint diff --git a/.gitignore b/.gitignore index 39a8b804..20d85979 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,9 @@ __pycache__* .vscode .venv .fuse_hidden* + +# ansible-lint working directory, also removed by "make cleanup" +.ansible + +# antsibull-changelog +changelogs/.plugin-cache.yaml diff --git a/.mailmap b/.mailmap index eead176c..e64fcc0b 100644 --- a/.mailmap +++ b/.mailmap @@ -1,3 +1,14 @@ +# Maps the different name and email combinations people committed with onto one +# canonical identity, so that git shortlog and git blame show one entry per person +# instead of one per address. Git applies this file automatically. +# +# Add a line when somebody shows up twice, for example after a change of employer or +# after a commit made through the GitHub web interface. Check the result with: +# git check-mailmap "Name
" +# +# Syntax: Proper Name +# See https://git-scm.com/docs/gitmailmap + Afeef Ghannam <39904920+afeefghannam89@users.noreply.github.com> Alexander Stoll <45196568+Crited@users.noreply.github.com> Daniel Neuberger diff --git a/AUTHORS b/AUTHORS deleted file mode 100644 index 4679c3c8..00000000 --- a/AUTHORS +++ /dev/null @@ -1,20 +0,0 @@ -Afeef Ghannam -Alexander Stoll -Daniel Neuberger -Daniel Patrick -dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> -Ivar Eriksen -Klaus Zerwes -Lorenz Kästle -Lucinda Siemer -Marco Hald -Markus Frosch -Markus Opolka -Matthias Döhler -Nathaniel Donahue -Patrick Dolinic -Patrick Steinmetz <132431614+psteinmetzcid@users.noreply.github.com> -Shittu Ayomide <86523250+lazyturd@users.noreply.github.com> -Thilo Wening -Thomas Widhalm -Tobias Bauriedel diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..1dcd9b59 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,102 @@ +# Contributing + +Every kind of contribution is welcome. This page describes what we need from a pull request so that reviewing it does not take longer than writing it. + +## Before you build something + +There is no written scope for this collection. It gets what is useful and what helps from an automation point of view, and that is easier to answer per case than in the abstract. Please open an issue for anything bigger than a bug fix and we will say yes or no before you invest the work. + +For now all pull requests go against `main`. + +## What a pull request needs + +* **One topic per pull request.** Touching several roles is fine when it is the same change in each of them, for example the same fix applied to beats, logstash, kibana and elasticsearch. A pull request that bundles unrelated changes gets sent back. +* **A description that says precisely what changes and why.** This is required, not optional. A pull request whose purpose can only be reconstructed from the diff gets sent back. +* **Evidence that it was tested**, against the new feature or fix and against what sits next to it. Please say which Elastic Stack release and which ansible-core version you tested with. +* **New or changed variables** belong in the role's `meta/argument_specs.yml` and in its README table, otherwise the documentation drifts apart from the code. See "Documenting role variables" below. +* **Comments on anything that is not obvious.** Six months later nobody remembers what a clever condition was for, and then it gets analysed again from scratch. +* **A changelog fragment.** See below. +* If a change **claims something about how Elastic behaves**, please link the Elastic documentation for it. + + +## Changelog fragments + +The changelog is not generated from commit messages. It is assembled from small files that each pull request brings along, so that the description is written by the person who knows what changed rather than reconstructed months later at release time. + +Add one **new** file per pull request under `changelogs/fragments/`. Name it after the issue or the change, for example `546_rolling_upgrade.yml`. Do not put your entry into an existing fragment and do not rename or delete one, so that every entry stays traceable to the pull request that introduced it. The pipeline rejects both. + +```yaml +# changelogs/fragments/546_rolling_upgrade.yml +bugfixes: + - Raising ``elasticstack_version`` no longer upgrades all Elasticsearch nodes at once + (https://github.com/NETWAYS/ansible-collection-elasticstack/issues/546). +``` + +Available sections, one or more per file: + +| Section | Use it for | +|---|---| +| `major_changes` | Changes that affect how the collection is used as a whole | +| `minor_changes` | New variables, new options, improved behaviour | +| `breaking_changes` | Anything users have to adapt to, for example a renamed variable | +| `deprecated_features` | Something that still works but will be removed | +| `removed_features` | Something previously deprecated that is now gone | +| `security_fixes` | Fixes with a security impact | +| `bugfixes` | Wrong or broken behaviour that now works | +| `known_issues` | Problems shipped with the release on purpose | +| `trivial` | Not rendered into the changelog. See below | + +Fragments are consumed at release time: `antsibull-changelog release` folds them into the changelog and deletes them, so the directory is empty again after every release. That is why a new file is always the right answer, never an edit to an existing one. + +### Changes that do not belong in the changelog + +Some changes are not worth telling users about. They still need a fragment, but the entry goes into the `trivial` section, which is not rendered into the changelog: + +```yaml +# changelogs/fragments/fix_comment_typo.yml +trivial: + - Fixed a typo in a comment in the elasticsearch role. +``` + +Typical cases for `trivial`: + +* documentation wording, typos and formatting +* CI and workflow changes +* test and Molecule scenario changes +* refactoring with no visible effect on behaviour +* dependency bumps of test tooling + +If you are unsure whether something is trivial, ask in the pull request. It is easier to move an entry than to notice a missing one after the release. + +## Documenting role variables + +Role variables are documented from each role's `meta/argument_specs.yml`, which is the single source of truth. When your pull request changes a role's variables: + +1. Update that role's `meta/argument_specs.yml` with type, default and description. +2. Regenerate the README variable table. Please do **not** edit it by hand. The table is produced by [ansible-docsmith](https://github.com/foundata/ansible-docsmith), install it with `pip install ansible-docsmith` if you do not have it: + + ```bash + ansible-docsmith generate roles/ --no-defaults --template-readme .docsmith/readme.md.j2 + ``` + +3. Commit the regenerated `README.md` together with your change. + +The `Test Documentation` workflow checks that each README matches its `argument_specs.yml` and fails the pull request if they drift apart. + +## Documenting modules + +Modules document themselves through the `DOCUMENTATION`, `EXAMPLES` and `RETURN` blocks in their own file. That is the only place, so there is nothing to keep in sync. Check your changes with `ansible-doc netways.elasticstack.`. + +Shared code under `plugins/module_utils/` has no such block, because `ansible-doc` does not read it. Those functions are described in [plugins/module_utils/README.md](plugins/module_utils/README.md), please keep it current when you change them. + +## Testing + +Besides testing your change against a real setup, the repository has Molecule scenarios that exercise the whole stack, and it checks for ansible-lint and yamllint errors. To run the linters locally there is a `makefile`: + +```bash +make +``` + +## Releasing + +Maintainers only. See [RELEASE.md](RELEASE.md). diff --git a/NOTICE.md b/NOTICE.md index e8b18e4c..83d51761 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -1,7 +1,7 @@ -# Notice and Acknowledgements # +# Notice and Acknowledgements -## Sponsoring ## +## Sponsoring Here's a list of sponsors who contributed by having the collection improved via outsourcing to NETWAYS. -* CID GmbH : Thank you so much for sponsoring. Especially the feature to have different types of Elasticsearch nodes in the cluster and the ingetration of rolling upgrades. +* CID GmbH : Thank you so much for sponsoring. Especially the feature to have different types of Elasticsearch nodes in the cluster and the integration of rolling upgrades. diff --git a/README.md b/README.md index c296052a..60838568 100644 --- a/README.md +++ b/README.md @@ -1,257 +1,144 @@ -# Ansible Collection - netways.elasticstack +# Ansible Collection Elasticstack -> [!CAUTION] -> The current main branch has some breaking changes. Be careful using it! Everything after commit 3d6673f6d526afeb0fd7ba382d067d76bd10bbd6 is affected! +This collection installs and manages the Elastic Stack. It provides roles for every component of the Stack, and it can install either the default Elastic distribution or the Apache-licensed OSS packages. -This collection installs and manages the Elastic Stack. It provides roles for every component of the Stack. Furthermore, it is possible to differentiate between Enterprise or OSS releases. +Every role can be used on its own or together with the others. Running a single role against an existing or external cluster works, but a few of those paths are still incomplete. See the open issues for details. -Every role is documented with all variables, please refer to the documentation found in **[Getting-Started](./docs/getting-started.md)** +## Tested with Ansible -## Roles documentation +The minimum required version is declared in [meta/runtime.yml](meta/runtime.yml) and enforced by `ansible-galaxy` on install. -> [!NOTE] -> Some roles have fixed requirements that you must observe. Please have a look at the [requirements](docs/01-requirements.md) before using the collection. (There is a high probability that some of them will be refactored soon) +| | | +|---|---| +| ansible-core | `>= 2.18`, tested with 2.19 and 2.20 | +| Python on the control node | `>= 3.11` | +| Python on the target hosts | `>= 3.9` | +| Elastic Stack | release 8 | -* [Beats](roles/beats/README.md) -* [Elasticsearch](roles/elasticsearch/README.md) -* [Kibana](roles/kibana/README.md) -* [Logstash](roles/logstash/README.md) -* [Repos](roles/repos/README.md) - -## Modules documentation - -* [elasticsearch_role](docs/module-elasticsearch_role.md) -* [elasticsearch_user](docs/module-elasticsearch_user.md) - -## Global variables - -The collection-wide `elasticstack_*` variables (release, variant, inventory host -groups, ports, package repositories, and the shared CA) are documented centrally -with the [elasticstack role](roles/elasticstack/README.md). - -* `elasticstack_force_pip`: Will force installation of required Python modules via `pip`. This is useful if your package manager doesn't provide current versions of modules. (Default: `false`) See [PEP668](https://peps.python.org/pep-0668/) for more details. -* `elasticstack_manage_pip`: Will install `pip` on your system. (Default: `false`) - -## Installation - -You can easily install the collection with the `ansible-galaxy` command. - -``` -ansible-galaxy collection install git+https://github.com/netways/ansible-collection-elasticstack.git -``` - -Or if you are using Tower or AWX add the collection to your requirements file. - -``` -collections: - - name: netways.elasticstack -``` - -### Requirements - -On the Ansible control node: -* `Ansible-Core >=2.18.0` -* `Python >=3.11.0` - -You will need the following Ansible collections installed - -* `community.general` (probably already present) - -You will need these packages / libraries installed. Some very basic packages like `openssl` get handled by the collection if needed. The following list contains packages and libraries which only apply to special cases or need for you to decide on the installation method. - -* `passlib` Python library if you do not disable password hashing for logstash user and you want to use logstash role from this collection. It should be installed with pip on the Ansible controller. -* `elasticsearch` Python module. Current versions are either compatible to Elasticsearch 9 or versions lower than 9. There seems to be no version that can serve both. So for now we install a version of the client lower than 9 but need to take care to install the right one when we make this collection compatible to Elastic Stack 9 - -You may want the following Ansible roles installed. There other ways to achieve what they are doing but using them is easy and convenient. - -* `geerlingguy.redis` if you want to use logstash role - -### Supported systems - -We test the collection on the following Linux distributions. Each one with Elastic Stack 7 and 8. +We test on the following Linux distributions, each one with Elastic Stack 8: * Rocky Linux 9 * Ubuntu 22.04 LTS * Debian 13 -## Caveats and information for long time users +## External requirements -### Variable renaming +Ansible collections, both are declared as dependencies and installed for you when you install this collection with `ansible-galaxy`: -**Please note**: If you are already using this collection before version `1.0.0`, please note that we had to rename a significant amount of variables due to naming schema changes made by Ansible. Please review the variables you have set in your playbooks and variable files. +* `community.general`, for the `zypper` modules on SUSE hosts and the Logstash plugin module +* `community.crypto`, used by the beats role to check when a certificate expires -### Ulimit management for Elasticsearch +Python libraries on the control node: -**Warning**: The role no longer configures `ulimit`. Ensure the system's open file limit is set correctly (e.g., `ulimit -n`). Packages usually handle this, but verify with `ulimit -n` or checking `/proc//limits`. If your version does not, open an issue at the collection's issue tracker: https://github.com/netways/ansible-collection-elasticstack/issues +* `passlib`, unless you disable password hashing for the Logstash user and you want to use the logstash role. Install it with `pip`. +* `elasticsearch`. Current versions are compatible either with Elasticsearch 9 or with versions lower than 9, there seems to be no version that serves both. For now we install a client older than 9, and this needs revisiting when the collection becomes compatible with Elastic Stack 9. -## Usage +Ansible roles: -Every role is documented with all variables, please refer to the documentation found in **[Getting-Started](./docs/getting-started.md)** +* `geerlingguy.redis` if you want to use the logstash role. There are other ways to get Redis running, but this one is easy and convenient. -Make sure all hosts that should be configured are part of your playbook. (See below for details on groups etc.). The collection is built to first collect all facts from all hosts (including those only running beats) and then use facts like hostnames or ip addresses to connect the tools to each other. + ```bash + ansible-galaxy install geerlingguy.redis + ``` -You will want to have reliable DNS resolution or enter all hosts of the stack into your systems hosts files. +Some very basic packages such as `openssl` are handled by the collection itself. The list above only contains what applies to special cases or what you need to decide about yourself. -The variable `elasticstack_no_log` can be set to `false` if you want to see the output of all tasks. It defaults to `true` because some tasks could reveal passwords in production. +Beyond software, the roles expect a few things from your inventory. See [Requirements](docs/requirements.md). -### Versions and upgrades +## Included content -*elasticstack_version*: Version number of tools to install. Only set if you don't want the latest on new setups. (default: none). If you already have an installation of Elastic Stack, this collection will query the version of Elasticsearch on the CA host and use it for all further installations in the same setup. (Only if you run the `elasticsearch` role before all others) Example: `7.17.2` +Roles: -All packages are installed with `state: present`. When `elasticstack_version` is set to a version number (e.g. `7.17.2`), that exact version is installed and pinned. When it is left unset, the package is installed without a version, so a new setup gets the newest available version and existing installations are not upgraded automatically on later runs. +* [beats](roles/beats/README.md) +* [elasticsearch](roles/elasticsearch/README.md) +* [kibana](roles/kibana/README.md) +* [logstash](roles/logstash/README.md) +* [repos](roles/repos/README.md) +* [elasticstack](roles/elasticstack/README.md), the meta role that holds the collection-wide `elasticstack_*` variables -*elasticstack_release*: Major release version of Elastic stack to configure. (default: `7`) Make sure it corresponds to `elasticstack_version` if you set both. +Modules: -For OSS version see `elasticstack_variant` below. +* `cert_info`, reads information from a PKCS12 certificate +* `elasticsearch_role`, creates, updates and deletes roles in Elasticsearch +* `elasticsearch_user`, creates, updates and deletes users in Elasticsearch -*elasticstack_variant*: Variant of the stack to install. Valid values: `elastic` or `oss`. (default: `elastic`) +Each module documents its own parameters, return values and examples. Read them with `ansible-doc`, for example: -```yaml -roles: - - role: netways.elasticstack.kibana - vars: - elasticstack_version: 8.7.1 - - role: netways.elasticstack.elasticsearch - vars: - elasticstack_version: 8.8.1 +```bash +ansible-doc netways.elasticstack.elasticsearch_role ``` -#### Upgrades #### - -Set `elasticstack_version` to the version you want to upgrade to. Positively do read and understand Elastics changelog and "breaking changes" of your target version and all between your current and the target version. Do not use unless you have a valid backup. - -If an upgrade fails, you can try re-running the collection with the same settings. There are several tasks that can provide "self-healing". Please do not rely on these mechanisms, they are more of a "convenience recovery" for easier steps. - -The collection will make sure to upgrade Elasticsearch nodes one by one. +## Collection documentation -### Default Passwords +* [Getting started](docs/getting-started.md), example playbook, inventory groups, first run +* [Requirements](docs/requirements.md), what the roles expect from your inventory +* [Versions and upgrades](docs/upgrades.md), pinning versions and upgrading the stack -Default passwords can be seen during generation, or found later in `/usr/share/elasticsearch/initial_passwords` +Every role documents all of its variables in its own README, linked above. Those tables are generated from each role's `meta/argument_specs.yml`. Module documentation comes from the modules themselves and is read with `ansible-doc`. -To turn off security: +## Using this collection -`elasticstack_override_beats_tls: true` - -### Requirements - -0) You need to install the redis role which is maintained by geerlingguy. +Install the collection with `ansible-galaxy`: +```bash +ansible-galaxy collection install netways.elasticstack ``` -ansible-galaxy install geerlingguy.redis -``` - -1) Default: For general Elastic Stack installations using all features use the following. You will need Redis installed and running for the default setup to run. A viable way is using the `geerlingguy.redis` role. -2) Specific: For OSS Installation without X-Pack features you can use the following. _Note_: this is only available for version `7.x`. +To pin a version, use a `requirements.yml`: -Our default configuration will collect filesystem logs placed by `rsyslog`. Therefor our example playbook makes sure, `rsyslog` is installed. If you don't want that, please change the configuration of the `beats` module. Without syslog you won't receive any messages with the default configuration. +```yaml +collections: + - name: netways.elasticstack + version: 0.1.0 +``` -There are some comments in the Playbook. Either fill them with the correct values (`remote_user`) or consider them as a hint to commonly used options. +```bash +ansible-galaxy collection install -r requirements.yml +``` -### Inventory +The same file works for Tower and AWX. You can also install straight from git, which gives you the current state of `main` instead of a release: -_Note_: The roles rely on group names for placing services on hosts. Please make sure you have group names defined: `elasticstack_elasticsearch_group_name` (default: `elasticsearch`), `elasticstack_logstash_group_name` (default: `logstash`) and `elasticstack_kibana_group_name` (default: `kibana`) that will match your desired setup in your Ansible inventory. Hosts in these groups will get the respective services. Just restricting your plays to the appropriate hosts will not work because the roles interact with hosts from other groups e.g. for certificate generation. +```bash +ansible-galaxy collection install git+https://github.com/NETWAYS/ansible-collection-elasticstack.git +``` -The execution order of the roles is important! (see below) +Then name the collection in the play and refer to the roles by their short name: ```yaml ---- -- hosts: all - # remote_user: my_username - become: true - collections: - - netways.elasticstack - vars: - elasticstack_variant: elastic #oss - # elasticstack_release: 8 #7 - roles: - - repos - - hosts: elasticsearch - # remote_user: my_username become: true collections: - netways.elasticstack - vars: - elasticstack_variant: elastic #oss - elasticsearch_jna_workaround: true - # elasticstack_release: 8 #7 roles: - elasticsearch +``` -- hosts: logstash - # remote_user: my_username - become: true - collections: - - netways.elasticstack - vars: - elasticstack_variant: elastic #oss - elasticstack_override_beats_tls: true - # elasticstack_release: 8 #7 - roles: - - geerlingguy.redis - - logstash +Fully qualified role names such as `netways.elasticstack.elasticsearch` work as well and need no `collections` entry. -- hosts: kibana - # remote_user: my_username - become: true - collections: - - netways.elasticstack - vars: - elasticstack_variant: elastic #oss - # elasticstack_release: 8 #7 - roles: - - kibana +The execution order of the roles matters, and the roles interact with hosts from other inventory groups. Read [Getting started](docs/getting-started.md) before your first run. -- hosts: all - # remote_user: my_username - become: true - collections: - - netways.elasticstack - vars: - elasticstack_variant: elastic #oss - elasticstack_override_beats_tls: true - # elasticstack_release: 8 #7 - pre_tasks: - - name: Install Rsyslog - ansible.builtin.package: - name: rsyslog - - name: Start rsyslog - ansible.builtin.service: - name: rsyslog - state: started - enabled: true - roles: - - beats +## Caveats and information for long time users -``` +### Variable renaming -## Contributing +If you have been using this collection before version `1.0.0`, note that a significant number of variables had to be renamed because of naming schema changes made by Ansible. Please review the variables you set in your playbooks and variable files. -Every kind of contribution is very welcome. Open [issues](https://github.com/NETWAYS/ansible-collection-elasticstack/issues) or provide [pull requests](https://github.com/NETWAYS/ansible-collection-elasticstack/pulls). +### Ulimit management for Elasticsearch -For now we open pull requests against `main`. We are planning to introduce dedicated branches to support older versions without breaking changes. Since we don't need them for now, please check back with this section because when we decided on how to proceed, you will find the information here. For now `main` always has the newest changes and if you want a stable version, please use the newest release. +The role no longer configures `ulimit`. Make sure the system's open file limit is set correctly, packages usually handle this. Verify with `ulimit -n` or by checking `/proc//limits`. If your version does not, please [open an issue](https://github.com/NETWAYS/ansible-collection-elasticstack/issues). -### Documentation for role variables +## Contributing to this collection -Role variables are documented from each role's `meta/argument_specs.yml`, which is -the single source of truth. When your pull request changes a role's variables: +Every kind of contribution is welcome. Open [issues](https://github.com/NETWAYS/ansible-collection-elasticstack/issues) or provide [pull requests](https://github.com/NETWAYS/ansible-collection-elasticstack/pulls). -1. Update that role's `meta/argument_specs.yml` (type, default, description). -2. Regenerate the README variable table — please do **not** edit it by hand. The table is - produced by [ansible-docsmith](https://github.com/foundata/ansible-docsmith) - (install with `pip install ansible-docsmith` if you don't have it): +Pull requests go against `main`. If you need a stable state, pin a release tag. - ``` - ansible-docsmith generate roles/ --no-defaults --template-readme .docsmith/readme.md.j2 - ``` +Please read [CONTRIBUTING.md](CONTRIBUTING.md) first. It describes what a pull request needs, including the changelog fragment that every pull request has to bring along, how role variables are documented and how to run the tests. -3. Commit the regenerated `README.md` together with your change. +## Release notes -The `Test Documentation` workflow checks that each README matches its -`argument_specs.yml` and fails the pull request if they drift apart. +See the [releases page](https://github.com/NETWAYS/ansible-collection-elasticstack/releases). The same notes are collected in `CHANGELOG.md`, which is generated from the changelog fragments when a version is released. -## Testing +## Licensing -Besides real tests that the developer should do before creating a PR, we built molecule scenarios to test the complete stack. -In parallel, we check for ansible and yaml lint errors. To do this pro-actively, a `makefile` is included. You can use this by calling `make`. +GPL-3.0-or-later, see [LICENSE](LICENSE). diff --git a/RELEASE.md b/RELEASE.md index 048f040c..e1f368ee 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,35 +1,92 @@ # Release Workflow -*NOTE*: This is a work in progress. While we are approaching our first release, we're collecting valuable information about tasks and put them here. +How to build, tag and publish a new version of this collection. Maintainers only. -## Preparations -### WIP note +## Prerequisites -Remove the note above saying that this is a work and progress before the first release. And remove this section. +* `antsibull-changelog` installed locally: -### Lint + ```bash + pip install "antsibull-changelog==0.35.1" + ``` -Make sure to have one last pull request to remove as much lint as possible. Same goes for deprecation warnings, linter exceptions etc. +* The repository or organization secret `GALAXY_API_KEY`, holding an API token of an account that owns the `netways` namespace on Ansible Galaxy. Tokens are created in the account settings on . +* Permission to run workflows in this repository, or the token itself if you intend to publish from your own machine. See step 6. -### Authors +Galaxy versions are **immutable**. Once a version is uploaded it cannot be replaced or removed, only superseded by a higher version. -Update the [.mailmap](.mailmap) and [AUTHORS](AUTHORS) files: +## 1. Set the version +Set `version:` in **galaxy.yml** to the version you are about to release. Semantic versioning, no `v` prefix. + +## 2. Write the release summary + +This becomes the introductory paragraph of the release in the changelog. + +`changelogs/fragments/release_summary.yml`: + +```yaml +release_summary: | + Summary text for this release, for example "Bugfix release". ``` -git log --use-mailmap | grep '^Author:' | cut -f2- -d' ' | sort | uniq > AUTHORS + +## 3. Generate the changelog + +Every pull request brings its own fragment under `changelogs/fragments/`. This folds all of them into `changelogs/changelog.yaml`, renders `CHANGELOG.md` and deletes the fragments. + +```bash +antsibull-changelog lint +antsibull-changelog release --version ``` -## Tag +Steps 1 to 3 change files in the repository. `main` requires an approving review, so open a pull request with the version bump and the generated changelog. Do not push this directly, even though repository admins are technically able to. -Tag the version. Don't use `v` in the name. Use semantic versioning. +## 4. Tag the version -e.g. +Tag the merge commit on `main`. No `v` prefix, semantic versioning. +```bash +git switch main +git pull +git tag -a -m "" +git push origin ``` -git tag -as 7.6.4 + +If you cannot push tags, skip this and let GitHub create the tag in step 5 instead. + +## 5. Create the release on GitHub + +[Draft a new release](https://github.com/NETWAYS/ansible-collection-elasticstack/releases/new): + +* choose the tag from step 4, or enter the version and let GitHub create the tag now +* use the version as the title +* paste this version's section from `CHANGELOG.md` as the release notes +* credit people by name where a change came from outside: the author of the pull request, and the reporter of an issue where the report was the contribution. `git shortlog -sn HEAD` lists everyone with their number of commits, folded per person through [.mailmap](.mailmap) + +The release has to exist before step 6, because the archive is uploaded to it. + +## 6. Publish to Ansible Galaxy + +**Actions** -> **publish** -> **Run workflow**, then enter the tag. + +The workflow checks out that tag, builds the collection, publishes it to Galaxy and attaches the archive to the release from step 5. It is triggered manually and never automatically, so that a tag can be inspected before an immutable version reaches Galaxy. + +Afterwards confirm on that the version arrived. Publishing can print errors and still have worked. + +### If you cannot run workflows + +Do the same from your own machine. **Clone the tag into a fresh directory first.** `ansible-galaxy collection build` packs the working directory and not the git tree, so untracked local files end up in the archive unless `build_ignore` happens to cover them. + +```bash +git clone --branch git@github.com:NETWAYS/ansible-collection-elasticstack.git release_ +cd release_ +ansible-galaxy collection build -vvv +ansible-galaxy collection publish --token netways-elasticstack-.tar.gz ``` -## Release +`-vvv` lists the files that were skipped, so you can see whether `build_ignore` did its job. Then attach the archive to the release by hand, because nothing uploaded it for you: -Make the tag a release on GitHub. +```bash +gh release upload netways-elasticstack-.tar.gz +``` diff --git a/changelogs/config.yaml b/changelogs/config.yaml new file mode 100644 index 00000000..c0c7efdc --- /dev/null +++ b/changelogs/config.yaml @@ -0,0 +1,40 @@ +--- +# Configuration for antsibull-changelog. +# See https://docs.ansible.com/projects/antsibull-changelog/changelog-configuration/ +# +# Contributors add one fragment per pull request under changelogs/fragments/. +# `antsibull-changelog release --version ` folds all fragments into +# changelogs/changelog.yaml, renders CHANGELOG.md and removes the fragments. + +changes_file: changelog.yaml +changes_format: combined +ignore_other_fragment_extensions: true +keep_fragments: false +mention_ancestor: true +notes_dir: fragments +output: + - file: CHANGELOG.md + format: md +prelude_name: release_summary +prelude_title: Release Summary +sanitize_changelog: true +sections: + - - major_changes + - Major Changes + - - minor_changes + - Minor Changes + - - breaking_changes + - Breaking Changes / Porting Guide + - - deprecated_features + - Deprecated Features + - - removed_features + - Removed Features (previously deprecated) + - - security_fixes + - Security Fixes + - - bugfixes + - Bugfixes + - - known_issues + - Known Issues +title: Netways.Elasticstack +trivial_section_name: trivial +use_fqcn: true diff --git a/changelogs/fragments/.keep b/changelogs/fragments/.keep new file mode 100644 index 00000000..e69de29b diff --git a/changelogs/fragments/prepare_galaxy_release.yml b/changelogs/fragments/prepare_galaxy_release.yml new file mode 100644 index 00000000..54131551 --- /dev/null +++ b/changelogs/fragments/prepare_galaxy_release.yml @@ -0,0 +1,31 @@ +minor_changes: + - The collection declares ``community.crypto`` as a dependency now, so installing it with + ``ansible-galaxy`` pulls in what the beats role needs to check certificate expiration. + - "``elasticstack_kibana_host`` is a declared and documented variable now, defaulting to the + fully qualified domain name of the Kibana host. The Kibana template referenced it ad hoc + before, without a default of its own." + - The documentation was restructured along the collection template. ``docs/01-requirements.md`` + became ``docs/requirements.md``, version pinning and upgrading moved to ``docs/upgrades.md``, + and the role pages under ``docs/`` that only linked to the role READMEs were removed. Module + documentation is read with ``ansible-doc`` instead of from copies under ``docs/``. + +trivial: + - Added the release tooling, namely an antsibull-changelog configuration, a manually triggered + workflow that publishes to Ansible Galaxy and attaches the archive to the GitHub release, a + check that every pull request brings a changelog fragment, and a contributing guide. + - Corrected the collection metadata in galaxy.yml, namely one current SPDX license identifier, + a documentation URL that resolves, the organization as the author, tags that name the Beats + correctly, and build_ignore keeping local tooling and CI files out of the built archive. + - Documented the three files under ``plugins/module_utils/`` that had no description, corrected + the parameters of ``check_supported_keys`` and ``AnalyzeCertificate``, and set + ``version_added`` of the modules to the first released version. + - Corrected factual errors in the README, removed documentation that was duplicated or only + redirected, and made the role examples use one way of addressing the roles. + - Removed the KICS security scan. Of its 226 Ansible queries, 215 cover AWS, Azure and GCP + resources this collection does not touch, and of the remaining ones only two ever fired, + one of them a false positive caused by the query not recognising fully qualified module + names. + - The role variable tables now have two columns, with type, required, default and choices + stacked under the variable name. Five columns left the description too narrow to read + without scrolling the table sideways. Required is only rendered where a variable actually + is required. diff --git a/docs/01-requirements.md b/docs/01-requirements.md deleted file mode 100644 index 6b27087b..00000000 --- a/docs/01-requirements.md +++ /dev/null @@ -1,15 +0,0 @@ -# Requirements - - -* Minimum Python version on Ansible controller: 3.11 -* Minimum Python version on target hosts: 3.9 - -There are some requirements that the user have to fulfill while using the collection. Some of them will be refactored and disappear from the list soon. - -**Inventory names** - -The collection provides roles for several tools (elasticsearch, kibana, ...). In some tasks hosts from the specific inventory group are fetched. The group names are customizable via variables. But using different inventory names than provided inside the variables will result in errors. - -**elasticstack_ca_host** - -This is a mandatory variable. It is used to define the host that will be used as "CA host". Per default it is already defined (first host inside the group `elasticstack_elasticsearch_group_name`). In case you are using different group names, this variable wont be set with a defualt value. diff --git a/docs/getting-started.md b/docs/getting-started.md index 320e757f..324ce3f8 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -1,63 +1,117 @@ -Collection Elastic Stack -------------------------- +# Getting started -Installation ------------ +This page shows how to run the collection. For installation see the [README](../README.md#using-this-collection), for what the roles expect from your inventory see [Requirements](requirements.md). -You can easily install the collection with the ansible-galaxy command. +## Inventory groups -``` -ansible-galaxy collection install netways.elasticstack -``` +The roles place services on hosts by inventory group name. The group names and their defaults are listed in [Requirements](requirements.md#inventory-group-names). -Or if you are using Tower or AWX add the collection to your requirements file. +Hosts in these groups get the respective services. Restricting your plays to only the appropriate hosts does **not** work, because the roles read facts from hosts in other groups, for example to generate certificates. -``` -collections: - - name: netways.elasticstack -``` +Make sure every host that should be configured is part of your playbook, including hosts that only run Beats. The collection first gathers facts from all hosts and then uses facts such as host names and IP addresses to connect the tools to each other. -Usage ---------- +You will want reliable DNS resolution, or all hosts of the stack in your systems' hosts files. -To use the collection in your Ansible playbook add the following key to your playbook. +## Before the first run -``` -- name: Playbook - hosts: some_host_pattern - collections: - - netways.elasticstack - tasks: - - name: import role logstash - import_role: - name: logstash -``` +Redis has to be running for the default setup, because the default Logstash pipeline uses it. The example playbook below gets it there with the `geerlingguy.redis` role, listed under external requirements in the [README](../README.md#external-requirements). -Or refer to the role with the FQCN of the role. +The default Beats configuration collects file system logs written by `rsyslog`, which is why the example playbook installs it. Without syslog you will not receive any messages with the default configuration. If you do not want that, change the configuration of the beats role. -``` -- name: Playbook - hosts: some_host_pattern - tasks: - - name: import role by FQCN from a collection - import_role: - name: netways.elasticstack.logstash +`elasticstack_variant: oss` is not available for the whole stack on every release. OSS Elasticsearch and Kibana exist only up to Elastic Stack `7`, while OSS Logstash and Beats are available on later releases too. + +## Variables + +Every role has its own set of variables, documented in the role's README. In addition the collection-wide `elasticstack_*` variables apply to any role and are documented centrally with the [elasticstack role](../roles/elasticstack/README.md). Two common ones: + +* `elasticstack_release`: major release version of the Elastic Stack to configure (default: `8`) +* `elasticstack_variant`: variant of the stack to install, `elastic` or `oss` (default: `elastic`) + +For pinning a specific version and for upgrading, see [Versions and upgrades](upgrades.md). + +## Example playbook + +The execution order of the plays matters. There are comments in the playbook below: either fill them with the correct values, such as `remote_user`, or take them as a hint about commonly used options. + +```yaml +--- +- hosts: all + # remote_user: my_username + become: true + collections: + - netways.elasticstack + vars: + elasticstack_variant: elastic # oss + # elasticstack_release: 8 + roles: + - repos + +- hosts: elasticsearch + # remote_user: my_username + become: true + collections: + - netways.elasticstack + vars: + elasticstack_variant: elastic # oss + elasticsearch_jna_workaround: true + # elasticstack_release: 8 + roles: + - elasticsearch + +- hosts: logstash + # remote_user: my_username + become: true + collections: + - netways.elasticstack + vars: + elasticstack_variant: elastic # oss + elasticstack_override_beats_tls: true + # elasticstack_release: 8 + roles: + - geerlingguy.redis + - logstash + +- hosts: kibana + # remote_user: my_username + become: true + collections: + - netways.elasticstack + vars: + elasticstack_variant: elastic # oss + # elasticstack_release: 8 + roles: + - kibana + +- hosts: all + # remote_user: my_username + become: true + collections: + - netways.elasticstack + vars: + elasticstack_variant: elastic # oss + elasticstack_override_beats_tls: true + # elasticstack_release: 8 + pre_tasks: + - name: Install Rsyslog + ansible.builtin.package: + name: rsyslog + - name: Start rsyslog + ansible.builtin.service: + name: rsyslog + state: started + enabled: true + roles: + - beats ``` -Roles -------- +The `collections` entry only makes the short role names resolvable. It is per play, which is why it appears in every one of them. Inside a role it has no effect, so keep using fully qualified module names such as `ansible.builtin.package` in your own tasks. -* [Beats](../roles/beats/README.md) -* [Elasticsearch](../roles/elasticsearch/README.md) -* [Kibana](../roles/kibana/README.md) -* [Logstash](../roles/logstash/README.md) -* [Repos](../roles/repos/README.md) +## Generated passwords +Passwords for the built-in users are generated on the CA host during the first run. They are printed while they are generated and stored in the file that `elasticstack_initial_passwords` points to, by default `/usr/share/elasticsearch/initial_passwords`. -Variables ------------ +Security is on by default and controlled by `elasticstack_security`. A related but much narrower switch is `elasticstack_override_beats_tls`: it only stops TLS between Beats and Logstash from being enabled automatically, and skips the Beats certificate generation. It does not turn off security anywhere else. -Every role got its own set of variables, in addition a few variables are useable on any role. The collection-wide `elasticstack_*` variables are documented centrally with the [elasticstack role](../roles/elasticstack/README.md). Two common ones: +## Seeing what the tasks do -* *elasticstack_release*: Major release version of Elastic stack to configure. (default: `7`) -* *elasticstack_variant*: Variant of the stack to install. Valid values: `elastic` or `oss`. (default: `elastic`) +`elasticstack_no_log` defaults to `true`, because some tasks could reveal passwords in production. Set it to `false` if you want to see the output of all tasks. diff --git a/docs/module-elasticsearch_role.md b/docs/module-elasticsearch_role.md deleted file mode 100644 index 1127a344..00000000 --- a/docs/module-elasticsearch_role.md +++ /dev/null @@ -1,68 +0,0 @@ -Ansible module: elasticsearch_role -=== - -This module creates, updates and deletes roles from your Elasticsearch. - -Requirements ---- - -As this module uses the Elasticsearch API you will need to install the `elasticsearch` Python3 library. -``` -pip3 install elasticsearch -``` - -Module arguments ---- - -* *name*: Name of your role (**Required**) -* *cluster*: List of clusters -* *indicies*: List of indicies - * *names*: List of names (**Required**) - * *privileges*: List of privileges (**Required**) -* *state*: State of the role (Default: `present`) -* *host*: API endpoint (**Required**) -* *auth_user*: User to authenticate on the Elasticsearch API (**Required**) -* *auth_pass*: Password for the given user (**Required**) -* *verify_certs*: Verify certificates (Default: `true`) -* *ca_certs*: Verify HTTPS connection by using ca certificate. Path to ca needs to be given - -Example usage ---- -``` - - name: Create elasticsearch role 'new-role1' - netways.elasticstack.elasticsearch_role: - name: new-role1 - cluster: - - manage_own_api_key - - delegate_pki - indicies: - - names: - - default01 - privileges: - - read - - write - state: present - host: https://localhost:9200 - auth_user: elastic - auth_pass: changeMe123! - verify_certs: true - ca_certs: /etc/elasticsearch/certs/http_ca.crt - - - name: Create elasticsearch role 'new-role2' - netways.elasticstack.elasticsearch_role: - name: new-role2 - cluster: - - manage_own_api_key - - delegate_pki - indicies: - - names: - - default01 - privileges: - - read - - write - state: present - host: https://localhost:9200 - auth_user: elastic - auth_pass: changeMe123! - verify_certs: false -``` \ No newline at end of file diff --git a/docs/module-elasticsearch_user.md b/docs/module-elasticsearch_user.md deleted file mode 100644 index 9ab1ea07..00000000 --- a/docs/module-elasticsearch_user.md +++ /dev/null @@ -1,65 +0,0 @@ -Ansible module: elasticsearch_user -=== - -This module creates, updates and deletes users from your Elasticsearch. - -Requirements ---- - -As this module uses the Elasticsearch API you will need to install the `elasticsearch` Python3 library. -``` -pip3 install elasticsearch -``` - -Module arguments ---- - -* *name*: Name of your user (**Required**) -* *fullname*: Fullname of your user -* *password*: Password for your user (**Required**) -* *email*: Email for your user -* *roles*: List of roles (**Required**) -* *enabled*: Define wheter this user should be enabled (Default: `true`) -* *state*: State of the role. `absent` to delete the user (Default: `present`) -* *host*: API endpoint (**Required**) -* *auth_user*: User to authenticate on the Elasticsearch API (**Required**) -* *auth_pass*: Password for the given user (**Required**) -* *verify_certs*: Verify certificates (Default: `true`) -* *ca_certs*: Verify HTTPS connection by using ca certificate. Path to ca needs to be given - -Example usage ---- -``` - - name: Create elasticsearch user 'new-user1' - netways.elasticstack.elasticsearch_user: - name: new-user1 - fullname: New User 1 - password: changeMe321! - email: new1@user.de - roles: - - new-role - - logstash-writer - enabled: true - state: present - host: https://localhost:9200 - auth_user: elastic - auth_pass: changeMe123! - verify_certs: true - ca_certs: /etc/elasticsearch/certs/http_ca.crt - - - name: Create elasticsearch user 'new-user2' - netways.elasticstack.elasticsearch_user: - name: new-user2 - fullname: New User 2 - password: changeMe321! - email: new2@user.de - roles: - - new-role - - logstash-writer - enabled: true - state: present - host: https://localhost:9200 - auth_user: elastic - auth_pass: changeMe123! - verify_certs: false -``` \ No newline at end of file diff --git a/docs/requirements.md b/docs/requirements.md new file mode 100644 index 00000000..92159aed --- /dev/null +++ b/docs/requirements.md @@ -0,0 +1,19 @@ +# Requirements + +Software requirements, supported Ansible and Python versions and external libraries are listed in the [README](../README.md#tested-with-ansible). This page describes what the roles expect from your setup. + +## Inventory group names + +The collection provides roles for several tools, and some tasks fetch hosts from a specific inventory group. The group names are customizable through variables, but using group names that differ from those variables results in errors. + +| Variable | Default group name | +|---|---| +| `elasticstack_elasticsearch_group_name` | `elasticsearch` | +| `elasticstack_logstash_group_name` | `logstash` | +| `elasticstack_kibana_group_name` | `kibana` | + +## elasticstack_ca_host + +This variable defines the host that acts as the certificate authority. It has a default: the first host in the group named by `elasticstack_elasticsearch_group_name`. + +That default only resolves when that group exists. If you use different group names without setting them in the variables above, the variable ends up empty and the run fails. diff --git a/docs/role-beats.md b/docs/role-beats.md deleted file mode 100644 index 14df7b01..00000000 --- a/docs/role-beats.md +++ /dev/null @@ -1,5 +0,0 @@ -# Beats role - -The Beats role documentation now lives with the role itself: - -**➜ [roles/beats/README.md](../roles/beats/README.md)** diff --git a/docs/role-elasticsearch.md b/docs/role-elasticsearch.md deleted file mode 100644 index 0cf04638..00000000 --- a/docs/role-elasticsearch.md +++ /dev/null @@ -1,5 +0,0 @@ -# Elasticsearch role - -The Elasticsearch role documentation now lives with the role itself: - -**➜ [roles/elasticsearch/README.md](../roles/elasticsearch/README.md)** diff --git a/docs/role-kibana.md b/docs/role-kibana.md deleted file mode 100644 index bddfb880..00000000 --- a/docs/role-kibana.md +++ /dev/null @@ -1,5 +0,0 @@ -# Kibana role - -The Kibana role documentation now lives with the role itself: - -**➜ [roles/kibana/README.md](../roles/kibana/README.md)** diff --git a/docs/role-logstash.md b/docs/role-logstash.md deleted file mode 100644 index 15d1fb7f..00000000 --- a/docs/role-logstash.md +++ /dev/null @@ -1,5 +0,0 @@ -# Logstash role - -The Logstash role documentation now lives with the role itself: - -**➜ [roles/logstash/README.md](../roles/logstash/README.md)** diff --git a/docs/role-repos.md b/docs/role-repos.md deleted file mode 100644 index bce0c25a..00000000 --- a/docs/role-repos.md +++ /dev/null @@ -1,5 +0,0 @@ -# Repos role - -The Repos role documentation now lives with the role itself: - -**➜ [roles/repos/README.md](../roles/repos/README.md)** diff --git a/docs/upgrades.md b/docs/upgrades.md new file mode 100644 index 00000000..ee08868a --- /dev/null +++ b/docs/upgrades.md @@ -0,0 +1,40 @@ +# Versions and upgrades + +## Choosing a version + +`elasticstack_version` holds the version number of the tools to install, for example `8.19.4`. Only set it if you do not want the newest available version on a new setup. There is no default. + +If you already have an installation of the Elastic Stack, the collection queries the installed version of Elasticsearch on the CA host and uses it for all further installations in the same setup. This only happens when you run the `elasticsearch` role before all others. + +All packages are installed with `state: present`: + +* When `elasticstack_version` is set to a version number, that exact version is installed and pinned. +* When it is left unset, the package is installed without a version. A new setup gets the newest available version, and existing installations are not upgraded automatically on later runs. + +`elasticstack_release` holds the major release, for example `8` (default). If you set both, make sure they match. + +For the OSS variant see `elasticstack_variant`, described with the [elasticstack role](../roles/elasticstack/README.md). + +Both variables can be set per role, which is how you keep components on different versions: + +```yaml +collections: + - netways.elasticstack +roles: + - role: kibana + vars: + elasticstack_version: 8.7.1 + - role: elasticsearch + vars: + elasticstack_version: 8.8.1 +``` + +## Upgrading + +Set `elasticstack_version` to the version you want to upgrade to. + +Before you do, read and understand Elastic's changelog and the breaking changes of your target version **and of every version between your current one and the target**. Do not upgrade without a valid backup. + +Elasticsearch nodes are upgraded one at a time. The role disables shard allocation, stops the node, installs the new package, starts it again and waits for the node to rejoin before it touches the next one. + +If an upgrade fails, you can try re-running the collection with the same settings. Several tasks provide a degree of self-healing. Do not rely on these mechanisms, they are a convenience recovery for the easier cases and not a substitute for a backup. diff --git a/galaxy.yml b/galaxy.yml index 2b5a0a72..698cdaa3 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -3,39 +3,43 @@ name: elasticstack version: 0.1.0 readme: README.md authors: -- Afeef Ghannam -- Alexander Stoll -- David Okon - - Thilo Wening - - Thomas Widhalm -description: This collection installs and configures components from the Elastic Stack. Every role can be used standalone or in combination with other roles. + - NETWAYS GmbH (https://github.com/NETWAYS) +description: This collection installs and configures components from the Elastic Stack. +# Mutually exclusive with license_file. SPDX identifier, GPL-3.0 is deprecated. license: - - GPL-3.0 + - GPL-3.0-or-later # min_ansible_version: "2.18" # max supported: 2.20 (commented out for schema compliance) -license_file: 'LICENSE' tags: - elastic - elasticsearch - kibana - logstash - - beat - - metricbeat + - beats - filebeat + - metricbeat + - auditbeat - monitoring dependencies: "community.general": "*" + "community.crypto": "*" repository: https://github.com/netways/ansible-collection-elasticstack -documentation: https://github.com/netways/ansible-collection-elasticstack/README.md +documentation: https://github.com/netways/ansible-collection-elasticstack/blob/main/README.md homepage: https://www.netways.de issues: https://github.com/netways/ansible-collection-elasticstack/issues build_ignore: - - .github - - .cache + # CI, tests and local tooling - molecule - - .gitignore + - makefile + - 'requirements*.txt' + # repository housekeeping, not part of the collection - daemon.json - - .mailmap - - '*.tar.gz' + - RELEASE.md + - CONTRIBUTING.md + # The changelog working directory, used to build the CHANGELOG.md in the repository. + - changelogs + # no dot files or directories belong in a published collection + - '.*' + - __pycache__ - 'venv*' - - .docsmith -# Minimum Ansible version required: 2.14, maximum supported: 2.20 + - '*.tar.gz' +# Minimum Ansible version is declared in meta/runtime.yml (requires_ansible). Maximum tested: 2.20 diff --git a/molecule/elasticstack_default/prepare.yml b/molecule/elasticstack_default/prepare.yml index 8d8c3819..37ea6841 100644 --- a/molecule/elasticstack_default/prepare.yml +++ b/molecule/elasticstack_default/prepare.yml @@ -45,13 +45,9 @@ - unzip - systemd -# KICS complains about packages being updated. -# In this case, a mere test scenario it is OK, though. - - name: Update all installed packages RHEL ansible.builtin.yum: name: '*' -# kics-scan ignore-line state: latest update_cache: yes update_only: yes @@ -60,7 +56,6 @@ - name: Update all installed packages Debian ansible.builtin.apt: name: '*' -# kics-scan ignore-line state: latest update_cache: yes when: ansible_os_family == "Debian" diff --git a/plugins/README.md b/plugins/README.md deleted file mode 100644 index 29aa3195..00000000 --- a/plugins/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# Collections Plugins Directory - -This directory can be used to ship various plugins inside an Ansible collection. Each plugin is placed in a folder that -is named after the type of plugin it is in. It can also include the `module_utils` and `modules` directory that -would contain module utils and modules respectively. - -Here is an example directory of the majority of plugins currently supported by Ansible: - -``` -└── plugins - ├── action - ├── become - ├── cache - ├── callback - ├── cliconf - ├── connection - ├── filter - ├── httpapi - ├── inventory - ├── lookup - ├── module_utils - ├── modules - ├── netconf - ├── shell - ├── strategy - ├── terminal - ├── test - └── vars -``` - -A full list of plugin types can be found at [Working With Plugins](https://docs.ansible.com/ansible-core/2.13/plugins/plugins.html). diff --git a/plugins/module_utils/README.md b/plugins/module_utils/README.md index 405b9c68..f9c88ddd 100644 --- a/plugins/module_utils/README.md +++ b/plugins/module_utils/README.md @@ -1,17 +1,26 @@ # Documentation: netways.elasticstack module_utils +Shared code used by the modules in `plugins/modules/`. `ansible-doc` does not read this directory, which is why it is documented here by hand instead of in a `DOCUMENTATION` block. + ## Overview -- [`certs` module_util](#cert_info-module) +- [`certs`](#certs), reading PKCS12 certificates. Used by `cert_info`. +- [`api`](#api), the Elasticsearch client. Used by `elasticsearch_role` and `elasticsearch_user`. +- [`elasticsearch_role`](#elasticsearch_role), the role logic behind the `elasticsearch_role` module. +- [`elasticsearch_user`](#elasticsearch_user), the user logic behind the `elasticsearch_user` module. + +## `certs` -## `netways.elasticstack.certs` function +Requires the `cryptography` library (>= 2.5). The module that uses it fails with `missing_required_lib()` when it is absent. + +`SUPPORTED_EXTENSIONS` is a dict at the top of the file. It maps an X.509 extension name to the list of its attribute names that may be reported. Nothing outside it ever reaches the result, so extending the module means extending that dict. ### `bytes_to_hex()` function -Since binascii.hexlify doesn't support a second parameter, which would define a seperator (e.g. ":") for hex strings in older Python versions like 2.6 and 2.7, we implemeted a small function to get similar results. +Since binascii.hexlify doesn't support a second parameter, which would define a separator (e.g. ":") for hex strings in older Python versions like 2.6 and 2.7, we implemented a small function to get similar results. **Parameter:** A __bytes__ object that represent a hexadecimal value (e.g. b'\\x82S \\x11\\xc7s\\xa7^*w\\xc1\\xdf\"\\xe4#\\xb4\\xc4P\\xba\\xcf') -**Return:** A hexadecimal __string__ seperated by colons (e.g. "82:53:20:11:C7:73:A7:5E:2A:77:C1:DF:22:E4:23:B4:C4:50:BA:CF") +**Return:** A hexadecimal __string__ separated by colons (e.g. "82:53:20:11:C7:73:A7:5E:2A:77:C1:DF:22:E4:23:B4:C4:50:BA:CF") ### `check_supported_extensions()` function @@ -23,16 +32,60 @@ A function to check if the extension is supported. Returns true if extension nam ### `check_supported_keys` function -A function to check if the extensions key is supported. Returns true if extensions key is found in `SUPPORTED_EXTENSIONS` dict. +A function to check if the extensions key is supported. Returns true if the key is listed under the given extension in the `SUPPORTED_EXTENSIONS` dict. Unlike `check_supported_extensions()` this compares exactly, not by substring. -**Parameter:** The key name as __string__. +**Parameter:** Two, in this order: the key name as __string__, and the extension name as __string__. The extension name has to be a key of `SUPPORTED_EXTENSIONS`, otherwise the lookup raises a `KeyError`. **Return:** Returns a __bool__. ### `AnalyzeCertificate()` object -An object to load the certificate and to gather information about it. +An object to load the certificate and to gather information about it. Constructing it already does the work: it reads the file and fills the result dict, so there is no separate call to start the analysis. + +**Parameter:** Two, in this order: the `AnsibleModule` __object__ and the result __dict__ to fill. The certificate path and the passphrase are not passed in, the object reads them from `module.params`. It also uses the module to log, to warn and to fail. + +**Return:** The constructor returns nothing, it writes into the result dict it was given. `return_result()` hands that dict back to the module, which passes it to `exit_json()`. + +A broken or unreadable extension does not fail the module. It is skipped with `module.warn()`, so a task can succeed with fewer extensions in the result than the certificate contains. + +## `api` + +Builds the Elasticsearch client for the two Elasticsearch modules. Requires the `elasticsearch` Python library, currently a version below 9. + +### `HAS_ELASTICSEARCH` and `ELASTICSEARCH_IMPORT_ERROR` + +Set while the file is imported. `HAS_ELASTICSEARCH` is a __bool__, `ELASTICSEARCH_IMPORT_ERROR` holds the formatted traceback as a __string__ or `None`. A module checks the flag and reports the traceback, so a missing library produces a readable message instead of an import error. + +### `Api.new_client_basic_auth()` static method + +Returns a connected `Elasticsearch` client authenticated with username and password. + +**Parameter:** Five keyword arguments: `host` (URL including protocol and port), `auth_user`, `auth_pass`, `ca_certs` (path to a CA file, or `None`) and `verify_certs`. + +**Return:** An `Elasticsearch` __object__. Raises `ImportError` when the library is missing. + +The SSL context it builds sets `check_hostname` and `verify_mode` to false before `verify_certs` is handed to the client, so the hostname in the certificate is not checked. + +## `elasticsearch_role` + +### `Role()` object + +The whole logic of the `elasticsearch_role` module. Constructing it already does the work: it builds the client, decides between create, update and delete, and fills the result dict. + +**Parameter:** Ten keyword arguments: the result __dict__ to fill, `role_name`, `cluster`, `indicies`, `state`, and the connection arguments `host`, `auth_user`, `auth_pass`, `verify_certs` and `ca_certs`. + +**Return:** The constructor returns nothing. `return_result()` hands the result dict back to the module, which passes it to `exit_json()`. + +Changes are detected against the API, not against the arguments. `handle_present()` reads the role, writes it, and compares the response before and after, so `changed` stays false when the role already looked that way. `handle_absent()` only deletes a role that the API reports as existing. + +## `elasticsearch_user` + +### `User()` object + +The whole logic of the `elasticsearch_user` module, built the same way as `Role()` above. + +**Parameter:** Thirteen keyword arguments: the result __dict__ to fill, `user_name`, `full_name`, `password`, `email`, `roles`, `enabled`, `state`, and the connection arguments `host`, `auth_user`, `auth_pass`, `verify_certs` and `ca_certs`. -**Parameter:** The path (required) to the certificate and the passphrase (optional), both as __string__. +**Return:** The constructor returns nothing. `return_result()` hands the result dict back to the module, which passes it to `exit_json()`. -**Return:** Returns the result dict to the Ansible module. +Changes are detected the same way, by comparing the API response before and after the write. A password cannot be read back, so setting the same password again is not reported as a change, while setting a different one is not detected either. diff --git a/plugins/module_utils/api.py b/plugins/module_utils/api.py index 847e0c52..0871985d 100644 --- a/plugins/module_utils/api.py +++ b/plugins/module_utils/api.py @@ -1,7 +1,7 @@ # !/usr/bin/python3 # Copyright (c) 2024, Tobias Bauriedel -# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# GNU General Public License v3.0+ (see LICENSE or # https://www.gnu.org/licenses/gpl-3.0.txt) import traceback diff --git a/plugins/module_utils/certs.py b/plugins/module_utils/certs.py index 7efbbee3..78a6c970 100644 --- a/plugins/module_utils/certs.py +++ b/plugins/module_utils/certs.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Copyright (c) 2023, Daniel Patrick -# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# GNU General Public License v3.0+ (see LICENSE or # https://www.gnu.org/licenses/gpl-3.0.txt) # SPDX-License-Identifier: GPL-3.0-or-later diff --git a/plugins/module_utils/elasticsearch_role.py b/plugins/module_utils/elasticsearch_role.py index 772eaa71..2d35ea35 100644 --- a/plugins/module_utils/elasticsearch_role.py +++ b/plugins/module_utils/elasticsearch_role.py @@ -1,5 +1,5 @@ # Copyright (c) 2024, Tobias Bauriedel -# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# GNU General Public License v3.0+ (see LICENSE or # https://www.gnu.org/licenses/gpl-3.0.txt) from ansible_collections.netways.elasticstack.plugins.module_utils.api import ( diff --git a/plugins/module_utils/elasticsearch_user.py b/plugins/module_utils/elasticsearch_user.py index 455724b5..04197cd2 100644 --- a/plugins/module_utils/elasticsearch_user.py +++ b/plugins/module_utils/elasticsearch_user.py @@ -1,5 +1,5 @@ # Copyright (c) 2024, Tobias Bauriedel -# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# GNU General Public License v3.0+ (see LICENSE or # https://www.gnu.org/licenses/gpl-3.0.txt) from ansible_collections.netways.elasticstack.plugins.module_utils.api import ( @@ -61,7 +61,7 @@ def handle_present(self): if pre_user.raw != self.get().raw: self.result['changed'] = True - self.result['msg'] = self.user_name + " has beed updated" + self.result['msg'] = self.user_name + " has been updated" return diff --git a/plugins/modules/README.md b/plugins/modules/README.md deleted file mode 100644 index 18f7344d..00000000 --- a/plugins/modules/README.md +++ /dev/null @@ -1,158 +0,0 @@ -# Documentation: netways.elasticstack modules - -## Overview -- [`cert_info` module](#cert_info-module) - - [Dependencies](#dependencies) - - [Tested python-cryptography versions >= 2.5](#tested-python-cryptography-versions--25) - - [Tested python versions](#tested-python-versions) - - [Tested ansible-core versions](#tested-ansible-core-versions) - - [Security measures](#security-measures) - - [Supported extensions and values](#supported-extensions-and-values) - - [Parameters](#paramters) - - [Returns](#returns) - - [Example](#example) - -## `cert_info` module - -The netways.elasticstack.cert_info module gathers information about pkcs12 certificates generated by the Elastic stack cert util. - -### Dependencies -- python-cryptography >= 2.5.0 on the remote node - -### Tested python-cryptography versions >= 2.5 -- 2.5 -- 3.0 -- 3.1 -- 3.2 -- 3.3 -- 3.4 -- 35.0.0 -- 36.0.0 -- 38.0.0 -- 40.0.1 - -### Tested python versions -- 2.7 -- 3.5 -- 3.6 -- 3.7 -- 3.8 -- 3.10 - -### Tested ansible-core versions -- 2.11 -- 2.12 -- 2.13 -- 2.14 - -### Security measures -- Only supported extensions with its available values will be returned. The available keys and values are applied in the code with the `SUPPORTED_EXTENSIONS` dictionary. The module will loop through it, and only if found, it will save it to the `results` variable. -- The paramters `path` and `passphrase` are set to no_log in the Ansible Module object. -- The objects `__private_key`, `__cert`, and `__additional_certs` are private and cannot be accessed globally. -- The object variables `__path` and `__passphrase` is private and cannot be accesed globally. - -### Supported extensions and values -Currently, the information of the following extensions and values will be returned (other extensions/values will be skipped): - -**BasicConstraints**: -- _ca -- _path_length - -**AuthorityKeyIdentifier**: -- _key_identifier -- _authority_cert_issuer -- _authority_cert_serial_number - -**SubjectKeyIdentifier**: -- _digest - -### Paramters - -`path`: -Absolute path to certificate. (**Default:** undefined, required) - -`password`: -The password of the pkcs12 certificate. (**Default:** No default, optional) - - -### Returns -All keys and values that will be returned with the results variable of the module: - -`issuer`: -The issuer of the CA certificate as **str**. - -`subject`: -The subject of the CA certificate as **str**. - -`version`: -The certificate version as **str** which represents an enumeration. - -`not_valid_after`: -A datetime.datetime object represented as a UTC formmated **str**. This is the beginning of the validity period. - -`not_valid_before`: -A datetime.datetime object represented as a UTC formmated **str**. This is the ending of the validity period. - -`serial_number`: -The serial number of the certificate as **str** which represents an integer. - -`extensions`: -- `oid`: The object identifier of the extension as **str** which represents a dotted string. -- `critical`: The value of critical as **str** which represents a bool. -- `values`: The keys and their values of the extension as **str**. (See: Supported extensions) - -### Example -``` -- name: Test - cert_info: - path: /opt/es-ca/elasticsearch-ca.pkcs12 - password: PleaseChangeMe - register: test - -- name: Debug - debug: - msg: "{{ test }}" -``` - -**Output**: -``` -TASK [Debug] ******************************************************************* -ok: [localhost] => { - "msg": { - "changed": false, - "extensions": { - "authorityKeyIdentifier": { - "_critical": "False", - "_dotted_string": "2.5.29.35", - "_values": { - "_authority_cert_issuer": "None", - "_authority_cert_serial_number": "None", - "_key_identifier": "82:53:20:11:C7:73:A7:5E:2A:77:C1:DF:22:E4:23:B4:C4:50:BA:CF" - } - }, - "basicConstraints": { - "_critical": "True", - "_dotted_string": "2.5.29.19", - "_values": { - "_ca": "True", - "_path_length": "None" - } - }, - "subjectKeyIdentifier": { - "_critical": "False", - "_dotted_string": "2.5.29.14", - "_values": { - "_digest": "82:53:20:11:C7:73:A7:5E:2A:77:C1:DF:22:E4:23:B4:C4:50:BA:CF" - } - } - }, - "failed": false, - "issuer": "Elastic Certificate Tool Autogenerated CA", - "not_valid_after": "2026-03-28 01:58:02", - "not_valid_before": "2023-03-29 01:58:02", - "serial_number": "719770426243590812378787092632593850366518596520", - "subject": "Elastic Certificate Tool Autogenerated CA", - "version": "Version.v3" - } -} -``` diff --git a/plugins/modules/cert_info.py b/plugins/modules/cert_info.py index 41cb73a4..2632db78 100644 --- a/plugins/modules/cert_info.py +++ b/plugins/modules/cert_info.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # Copyright (c) 2023, Daniel Patrick -# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# GNU General Public License v3.0+ (see LICENSE or # https://www.gnu.org/licenses/gpl-3.0.txt) # SPDX-License-Identifier: GPL-3.0-or-later @@ -17,11 +17,26 @@ - Reads a PKCS12 certificate file and returns details such as issuer, subject, validity dates, serial number, and supported X.509 extensions. - Requires the C(cryptography) Python library (>= 2.5) on the target host. -version_added: "1.0.0" + - Only a fixed set of X.509 extensions is returned, everything else is skipped. + Currently supported are C(basicConstraints) with C(_ca) and C(_path_length), + C(subjectKeyIdentifier) with C(_digest), and C(authorityKeyIdentifier) with + C(_key_identifier), C(_authority_cert_issuer) and C(_authority_cert_serial_number). +version_added: "0.1.0" author: - Daniel Patrick (@dpatrick) requirements: - cryptography >= 2.5 +notes: + - The supported extensions and their values are defined in the C(SUPPORTED_EXTENSIONS) + dictionary in the module code. The module iterates over it and only stores a value in + the result when it is found there, so an unexpected extension can never leak into the + output. + - Both O(path) and O(passphrase) are marked C(no_log), so they do not appear in task + output or logs. + - The path, the passphrase and the certificate are held in private attributes of the + analyzing object. The private key and any additional certificates from the PKCS12 file + are only kept in a local variable while the file is read and are never stored on the + object at all. options: path: description: Absolute path to the PKCS12 certificate file on the target host. diff --git a/plugins/modules/elasticsearch_role.py b/plugins/modules/elasticsearch_role.py index 05db0f33..d8916215 100644 --- a/plugins/modules/elasticsearch_role.py +++ b/plugins/modules/elasticsearch_role.py @@ -1,7 +1,7 @@ #!/usr/bin/python # Copyright (c) 2024, Tobias Bauriedel -# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# GNU General Public License v3.0+ (see LICENSE or # https://www.gnu.org/licenses/gpl-3.0.txt) from __future__ import (absolute_import, division, print_function) @@ -14,7 +14,7 @@ description: - Creates, updates, or deletes Elasticsearch roles using the Security API. - Requires the C(elasticsearch) Python library on the target host. -version_added: "1.0.0" +version_added: "0.1.0" author: - Tobias Bauriedel (@tbauriedel) requirements: diff --git a/plugins/modules/elasticsearch_user.py b/plugins/modules/elasticsearch_user.py index d5b1dd4b..50e4bdbb 100644 --- a/plugins/modules/elasticsearch_user.py +++ b/plugins/modules/elasticsearch_user.py @@ -1,7 +1,7 @@ #!/usr/bin/python # Copyright (c) 2024, Tobias Bauriedel -# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or +# GNU General Public License v3.0+ (see LICENSE or # https://www.gnu.org/licenses/gpl-3.0.txt) from __future__ import (absolute_import, division, print_function) @@ -14,7 +14,7 @@ description: - Creates, updates, or deletes Elasticsearch native users using the Security API. - Requires the C(elasticsearch) Python library on the target host. -version_added: "1.0.0" +version_added: "0.1.0" author: - Tobias Bauriedel (@tbauriedel) requirements: @@ -29,7 +29,7 @@ type: str required: false password: - description: Password for the user. Required when (state=present). + description: Password for the user. Required when O(state=present). type: str required: false default: null @@ -38,7 +38,7 @@ type: str required: false roles: - description: List of roles assigned to the user. Required when (state=present). + description: List of roles assigned to the user. Required when O(state=present). type: list elements: str required: false diff --git a/roles/beats/README.md b/roles/beats/README.md index 7816e301..41d70ece 100644 --- a/roles/beats/README.md +++ b/roles/beats/README.md @@ -1,15 +1,11 @@ # Ansible Role: Beats -Installs and configures the Elastic [Beats](https://www.elastic.co/beats) — -Filebeat, Auditbeat and Metricbeat — on Linux systems. The role can run -standalone or together with the other Elastic Stack roles. Beats can ship to -Logstash or directly to Elasticsearch. +Installs and configures the Elastic [Beats](https://www.elastic.co/beats) — Filebeat, Auditbeat and Metricbeat — on Linux systems. The role can run standalone or together with the other Elastic Stack roles. Beats can ship to Logstash or directly to Elasticsearch. ## Requirements * The `community.crypto` collection (used to check certificate expiration). -* The Beats you want to install must be available in your software repositories — - use the [`repos`](../repos) role or provide them yourself. +* The Beats you want to install must be available in your software repositories — use the [`repos`](../repos) role or provide them yourself. ## Example @@ -25,10 +21,7 @@ Logstash or directly to Elasticsearch. ## Filebeat inputs -Filebeat can read from several sources (log files, syslog, journald, Docker, -modules). For the structure of `beats_filebeat_log_inputs`, -`beats_filebeat_journald_inputs` and the other input variables, see the -[Filebeat inputs documentation](docs/filebeat-inputs.md). +Filebeat can read from several sources (log files, syslog, journald, Docker, modules). For the structure of `beats_filebeat_log_inputs`, `beats_filebeat_journald_inputs` and the other input variables, see the [Filebeat inputs documentation](docs/filebeat-inputs.md). ## Tags @@ -41,56 +34,52 @@ Run only parts of the role with `--tags`: ## Role variables -| Variable | Type | Default | Choices | Description | -|----------|------|---------|---------|-------------| -| `beats_filebeat` | `bool` | `true` | — | Install and manage Filebeat. | -| `beats_auditbeat` | `bool` | `false` | — | Install and manage Auditbeat. | -| `beats_metricbeat` | `bool` | `false` | — | Install and manage Metricbeat. | -| `beats_target_hosts` | `list` of `str` | `['localhost']` | — | Hosts the Beats ship to. Only used when the role runs standalone; with the other Elastic Stack roles the targets are determined automatically. | -| `beats_fields` | `list` of `str` | N/A | — | Global fields added to the log and syslog (tcp/udp) inputs (not to the mysql, journald or docker inputs), given as a list of "key: value" strings — the global counterpart to the per-input fields. Unset by default. See the Filebeat inputs documentation. | -| `beats_logging` | `str` | `"file"` | — | Where the Beats log. Set to "file" to log into beats_logpath; any other value leaves the Beats built-in logging. | -| `beats_loglevel` | `str` | `"info"` | — | Log level for all Beats. | -| `beats_logpath` | `str` | `"/var/log/beats"` | — | Directory for the log files when beats_logging is "file". | -| `beats_filebeat_enable` | `bool` | `true` | — | Start and enable the Filebeat service. | -| `beats_filebeat_output` | `str` | `"logstash"` | `logstash`, `elasticsearch` | Where Filebeat sends its events. | -| `beats_filebeat_elastic_monitoring` | `bool` | `false` | — | Report Filebeat monitoring data through the Elastic Stack monitoring features. | -| `beats_filebeat_loadbalance` | `bool` | `true` | — | Enable load balancing for the Filebeat Logstash output. | -| `beats_filebeat_log_input` | `bool` | `true` | — | Read log files with the inputs from beats_filebeat_log_inputs. | -| `beats_filebeat_log_inputs` | `dict` | `{'messages': {'paths': ['/var/log/messages', '/var/log/syslog']}}` | — | Log files to read, keyed by a free name that becomes the filestream id. Each entry has paths and optional fields and multiline settings. See the Filebeat inputs documentation. | -| `beats_filebeat_syslog_tcp` | `bool` | `false` | — | Enable a TCP syslog input. | -| `beats_filebeat_syslog_tcp_port` | `int` | `514` | — | Port of the TCP syslog input. | -| `beats_filebeat_syslog_udp` | `bool` | `false` | — | Enable a UDP syslog input. | -| `beats_filebeat_syslog_udp_port` | `int` | `514` | — | Port of the UDP syslog input. | -| `beats_filebeat_journald` | `bool` | `false` | — | Collect logs from journald. Available since Filebeat 7.16. | -| `beats_filebeat_journald_inputs` | `dict` | `{'everything': {'id': 'everything'}}` | — | Journald inputs, keyed by a free name. Each entry has an id and optional include_matches filters. See the Filebeat inputs documentation. | -| `beats_filebeat_docker` | `bool` | `false` | — | Collect Docker container logs. Only works on Elastic Stack release 7. | -| `beats_filebeat_docker_ids` | `str` | `"*"` | — | IDs of the containers to collect logs from. | -| `beats_filebeat_mysql_slowlog_input` | `bool` | `false` | — | Collect the MySQL/MariaDB slow query log, including multiline handling. | -| `beats_filebeat_modules` | `list` of `str` | N/A | — | List of Filebeat modules to enable (experimental). Unset by default. | -| `beats_auditbeat_enable` | `bool` | `true` | — | Start and enable the Auditbeat service. | -| `beats_auditbeat_output` | `str` | `"elasticsearch"` | `logstash`, `elasticsearch` | Where Auditbeat sends its events. | -| `beats_auditbeat_setup` | `bool` | `true` | — | Run the Auditbeat setup (index management, ingest pipelines and Kibana dashboards). Only effective with the elasticsearch output; loading the dashboards additionally requires Kibana to be reachable. | -| `beats_auditbeat_loadbalance` | `bool` | `true` | — | Enable load balancing for the Auditbeat Logstash output. | -| `beats_metricbeat_enable` | `bool` | `true` | — | Start and enable the Metricbeat service. | -| `beats_metricbeat_output` | `str` | `"elasticsearch"` | `logstash`, `elasticsearch` | Where Metricbeat sends its events. | -| `beats_metricbeat_modules` | `list` of `str` | `['system']` | — | Metricbeat modules to enable. | -| `beats_metricbeat_loadbalance` | `bool` | `true` | — | Enable load balancing for the Metricbeat Logstash output. | -| `beats_security` | `bool` | `false` | — | Activate TLS for the connections to the targets. Works with the other roles and elasticstack_full_stack to create certificates automatically, or with the beats_tls_* variables for custom certificates. | -| `beats_ca_dir` | `str` | N/A | — | Base directory for the Beats certificate, key and CA file. The role fills it per variant (/opt/ca standalone, /etc/beats/certs in a full stack) if you did not set it. | -| `beats_tls_key` | `str` | `"{{ beats_ca_dir \| default('') }}/{{ inventory_hostname }}-beats.key"` | — | Path to the Beats private key. Defaults under beats_ca_dir; set it to bring your own. | -| `beats_tls_cert` | `str` | `"{{ beats_ca_dir \| default('') }}/{{ inventory_hostname }}-beats.crt"` | — | Path to the Beats certificate. Defaults under beats_ca_dir; set it to bring your own. | -| `beats_tls_cacert` | `str` | `"{{ beats_ca_dir \| default('') }}/ca.crt"` | — | Path to the CA certificate Beats trusts. Defaults under beats_ca_dir; set it to bring your own. | -| `beats_tls_key_passphrase` | `str` | `"BeatsChangeMe"` | — | Passphrase of the Beats private key. | -| `beats_cert_validity_period` | `int` | `1095` | — | Number of days the generated certificates are valid. | -| `beats_cert_expiration_buffer` | `str` | `"+30d"` | — | Renew the certificate when it would expire within this period. Uses the community.crypto check_period format (e.g. "+30d"), not a plain number. | -| `beats_cert_will_expire_soon` | `bool` | `false` | — | Set to true to force renewal of the Beats certificate. Alternatively run the playbook with the renew_beats_cert tag. | +| Variable | Description | +|----------|-------------| +| `beats_filebeat`
**Type**: `bool`
**Default**: `true` | Install and manage Filebeat. | +| `beats_auditbeat`
**Type**: `bool`
**Default**: `false` | Install and manage Auditbeat. | +| `beats_metricbeat`
**Type**: `bool`
**Default**: `false` | Install and manage Metricbeat. | +| `beats_target_hosts`
**Type**: `list` of `str`
**Default**: `['localhost']` | Hosts the Beats ship to. Only used when the role runs standalone; with the other Elastic Stack roles the targets are determined automatically. | +| `beats_fields`
**Type**: `list` of `str` | Global fields added to the log and syslog (tcp/udp) inputs (not to the mysql, journald or docker inputs), given as a list of "key: value" strings — the global counterpart to the per-input fields. Unset by default. See the Filebeat inputs documentation. | +| `beats_logging`
**Type**: `str`
**Default**: `"file"` | Where the Beats log. Set to "file" to log into beats_logpath; any other value leaves the Beats built-in logging. | +| `beats_loglevel`
**Type**: `str`
**Default**: `"info"` | Log level for all Beats. | +| `beats_logpath`
**Type**: `str`
**Default**: `"/var/log/beats"` | Directory for the log files when beats_logging is "file". | +| `beats_filebeat_enable`
**Type**: `bool`
**Default**: `true` | Start and enable the Filebeat service. | +| `beats_filebeat_output`
**Type**: `str`
**Default**: `"logstash"`
**Choices**: `logstash`, `elasticsearch` | Where Filebeat sends its events. | +| `beats_filebeat_elastic_monitoring`
**Type**: `bool`
**Default**: `false` | Report Filebeat monitoring data through the Elastic Stack monitoring features. | +| `beats_filebeat_loadbalance`
**Type**: `bool`
**Default**: `true` | Enable load balancing for the Filebeat Logstash output. | +| `beats_filebeat_log_input`
**Type**: `bool`
**Default**: `true` | Read log files with the inputs from beats_filebeat_log_inputs. | +| `beats_filebeat_log_inputs`
**Type**: `dict`
**Default**: `{'messages': {'paths': ['/var/log/messages', '/var/log/syslog']}}` | Log files to read, keyed by a free name that becomes the filestream id. Each entry has paths and optional fields and multiline settings. See the Filebeat inputs documentation. | +| `beats_filebeat_syslog_tcp`
**Type**: `bool`
**Default**: `false` | Enable a TCP syslog input. | +| `beats_filebeat_syslog_tcp_port`
**Type**: `int`
**Default**: `514` | Port of the TCP syslog input. | +| `beats_filebeat_syslog_udp`
**Type**: `bool`
**Default**: `false` | Enable a UDP syslog input. | +| `beats_filebeat_syslog_udp_port`
**Type**: `int`
**Default**: `514` | Port of the UDP syslog input. | +| `beats_filebeat_journald`
**Type**: `bool`
**Default**: `false` | Collect logs from journald. Available since Filebeat 7.16. | +| `beats_filebeat_journald_inputs`
**Type**: `dict`
**Default**: `{'everything': {'id': 'everything'}}` | Journald inputs, keyed by a free name. Each entry has an id and optional include_matches filters. See the Filebeat inputs documentation. | +| `beats_filebeat_docker`
**Type**: `bool`
**Default**: `false` | Collect Docker container logs. Only works on Elastic Stack release 7. | +| `beats_filebeat_docker_ids`
**Type**: `str`
**Default**: `"*"` | IDs of the containers to collect logs from. | +| `beats_filebeat_mysql_slowlog_input`
**Type**: `bool`
**Default**: `false` | Collect the MySQL/MariaDB slow query log, including multiline handling. | +| `beats_filebeat_modules`
**Type**: `list` of `str` | List of Filebeat modules to enable (experimental). Unset by default. | +| `beats_auditbeat_enable`
**Type**: `bool`
**Default**: `true` | Start and enable the Auditbeat service. | +| `beats_auditbeat_output`
**Type**: `str`
**Default**: `"elasticsearch"`
**Choices**: `logstash`, `elasticsearch` | Where Auditbeat sends its events. | +| `beats_auditbeat_setup`
**Type**: `bool`
**Default**: `true` | Run the Auditbeat setup (index management, ingest pipelines and Kibana dashboards). Only effective with the elasticsearch output; loading the dashboards additionally requires Kibana to be reachable. | +| `beats_auditbeat_loadbalance`
**Type**: `bool`
**Default**: `true` | Enable load balancing for the Auditbeat Logstash output. | +| `beats_metricbeat_enable`
**Type**: `bool`
**Default**: `true` | Start and enable the Metricbeat service. | +| `beats_metricbeat_output`
**Type**: `str`
**Default**: `"elasticsearch"`
**Choices**: `logstash`, `elasticsearch` | Where Metricbeat sends its events. | +| `beats_metricbeat_modules`
**Type**: `list` of `str`
**Default**: `['system']` | Metricbeat modules to enable. | +| `beats_metricbeat_loadbalance`
**Type**: `bool`
**Default**: `true` | Enable load balancing for the Metricbeat Logstash output. | +| `beats_security`
**Type**: `bool`
**Default**: `false` | Activate TLS for the connections to the targets. Works with the other roles and elasticstack_full_stack to create certificates automatically, or with the beats_tls_* variables for custom certificates. | +| `beats_ca_dir`
**Type**: `str` | Base directory for the Beats certificate, key and CA file. The role fills it per variant (/opt/ca standalone, /etc/beats/certs in a full stack) if you did not set it. | +| `beats_tls_key`
**Type**: `str`
**Default**: `"{{ beats_ca_dir \| default('') }}/{{ inventory_hostname }}-beats.key"` | Path to the Beats private key. Defaults under beats_ca_dir; set it to bring your own. | +| `beats_tls_cert`
**Type**: `str`
**Default**: `"{{ beats_ca_dir \| default('') }}/{{ inventory_hostname }}-beats.crt"` | Path to the Beats certificate. Defaults under beats_ca_dir; set it to bring your own. | +| `beats_tls_cacert`
**Type**: `str`
**Default**: `"{{ beats_ca_dir \| default('') }}/ca.crt"` | Path to the CA certificate Beats trusts. Defaults under beats_ca_dir; set it to bring your own. | +| `beats_tls_key_passphrase`
**Type**: `str`
**Default**: `"BeatsChangeMe"` | Passphrase of the Beats private key. | +| `beats_cert_validity_period`
**Type**: `int`
**Default**: `1095` | Number of days the generated certificates are valid. | +| `beats_cert_expiration_buffer`
**Type**: `str`
**Default**: `"+30d"` | Renew the certificate when it would expire within this period. Uses the community.crypto check_period format (e.g. "+30d"), not a plain number. | +| `beats_cert_will_expire_soon`
**Type**: `bool`
**Default**: `false` | Set to true to force renewal of the Beats certificate. Alternatively run the playbook with the renew_beats_cert tag. | ## Shared variables -This role also uses the collection-wide `elasticstack_*` variables (e.g. -`elasticstack_full_stack`, `elasticstack_variant`, `elasticstack_ca_host`, -`elasticstack_ca_pass`, `elasticstack_release`, `elasticstack_version`, -`elasticstack_elasticsearch_http_port`, `elasticstack_beats_port`). They are -documented centrally with the [elasticstack role](../elasticstack/README.md). +This role also uses the collection-wide `elasticstack_*` variables (e.g. `elasticstack_full_stack`, `elasticstack_variant`, `elasticstack_ca_host`, `elasticstack_ca_pass`, `elasticstack_release`, `elasticstack_version`, `elasticstack_elasticsearch_http_port`, `elasticstack_beats_port`). They are documented centrally with the [elasticstack role](../elasticstack/README.md). diff --git a/roles/beats/docs/filebeat-inputs.md b/roles/beats/docs/filebeat-inputs.md index cede2166..fc34a002 100644 --- a/roles/beats/docs/filebeat-inputs.md +++ b/roles/beats/docs/filebeat-inputs.md @@ -1,19 +1,12 @@ -# Filebeat inputs # +# Filebeat inputs -Filebeat can read from several sources. Each source is turned into a Filebeat -input by the role. This document describes the variables that configure them. -For the meaning of the individual Filebeat options, follow the links to the -official Filebeat documentation. +Filebeat can read from several sources. Each source is turned into a Filebeat input by the role. This document describes the variables that configure them. For the meaning of the individual Filebeat options, follow the links to the official Filebeat documentation. -## Log files ## +## Log files Set with `beats_filebeat_log_input` (default `true`) and `beats_filebeat_log_inputs`. -`beats_filebeat_log_inputs` is a **dictionary keyed by a free name**. That key -becomes the id of the generated -[`filestream`](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-filestream) -input (`-filestream`) — there is no separate `name` field. Each entry needs -`paths`; `fields` and `multiline` are optional. +`beats_filebeat_log_inputs` is a **dictionary keyed by a free name**. That key becomes the id of the generated [`filestream`](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-filestream) input (`-filestream`) — there is no separate `name` field. Each entry needs `paths`; `fields` and `multiline` are optional. The default reads the system log: @@ -41,16 +34,13 @@ beats_filebeat_log_inputs: match: after ``` -`multiline` maps directly to the Filebeat multiline parser (`type`, `pattern`, -`negate`, `match`). See -[Manage multiline messages](https://www.elastic.co/docs/reference/beats/filebeat/multiline-examples). +`multiline` maps directly to the Filebeat multiline parser (`type`, `pattern`, `negate`, `match`). See [Manage multiline messages](https://www.elastic.co/docs/reference/beats/filebeat/multiline-examples). -## Fields ## +## Fields There are two ways to add fields, and they use **different shapes**: -* **Per input** — the optional `fields` inside a log input is a **dictionary** - (`key: value`), added only to that input: +* **Per input** — the optional `fields` inside a log input is a **dictionary** (`key: value`), added only to that input: ```yaml beats_filebeat_log_inputs: @@ -61,9 +51,7 @@ There are two ways to add fields, and they use **different shapes**: environment: production ``` -* **Globally** — `beats_fields` is a **list of `"key: value"` strings**. In the - current templates it is added to the log, TCP and UDP inputs; the journald, - Docker and MySQL slow-log inputs do not receive it: +* **Globally** — `beats_fields` is a **list of `"key: value"` strings**. In the current templates it is added to the log, TCP and UDP inputs; the journald, Docker and MySQL slow-log inputs do not receive it: ```yaml beats_fields: @@ -71,29 +59,20 @@ There are two ways to add fields, and they use **different shapes**: - "team: platform" ``` -## Syslog over TCP/UDP ## +## Syslog over TCP/UDP -Enable a listening syslog input with `beats_filebeat_syslog_tcp` / -`beats_filebeat_syslog_udp` and set the port with `beats_filebeat_syslog_tcp_port` -/ `beats_filebeat_syslog_udp_port` (both default `514`). They become a -[`tcp`](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-tcp) -or [`udp`](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-udp) -input listening on `0.0.0.0:`. The global `beats_fields` are applied here too. +Enable a listening syslog input with `beats_filebeat_syslog_tcp` / `beats_filebeat_syslog_udp` and set the port with `beats_filebeat_syslog_tcp_port` / `beats_filebeat_syslog_udp_port` (both default `514`). They become a [`tcp`](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-tcp) or [`udp`](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-udp) input listening on `0.0.0.0:`. The global `beats_fields` are applied here too. ```yaml beats_filebeat_syslog_tcp: true beats_filebeat_syslog_tcp_port: 514 ``` -## Journald ## +## Journald -Enable with `beats_filebeat_journald` (default `false`, available since Filebeat -7.16) and configure inputs with `beats_filebeat_journald_inputs`. +Enable with `beats_filebeat_journald` (default `false`, available since Filebeat 7.16) and configure inputs with `beats_filebeat_journald_inputs`. -This is a **dictionary keyed by a free name**. Unlike the log inputs, here the -key is only a label — the input's id comes from the `id` field, which each entry -needs. `include_matches` is optional and is itself a dictionary whose values are -the match expressions. +This is a **dictionary keyed by a free name**. Unlike the log inputs, here the key is only a label — the input's id comes from the `id` field, which each entry needs. `include_matches` is optional and is itself a dictionary whose values are the match expressions. ```yaml beats_filebeat_journald_inputs: @@ -105,37 +84,26 @@ beats_filebeat_journald_inputs: unit: '_SYSTEMD_UNIT=sshd.service' ``` -See the -[`journald`](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-journald) -input. +See the [`journald`](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-input-journald) input. -## Docker ## +## Docker -Enable with `beats_filebeat_docker` (default `false`) and select containers with -`beats_filebeat_docker_ids` (default `*`). **Only works on Elastic Stack release -7.** Docker metadata is added automatically. +Enable with `beats_filebeat_docker` (default `false`) and select containers with `beats_filebeat_docker_ids` (default `*`). **Only works on Elastic Stack release 7.** Docker metadata is added automatically. ```yaml beats_filebeat_docker: true beats_filebeat_docker_ids: "*" ``` -## MySQL/MariaDB slow log ## +## MySQL/MariaDB slow log -Set `beats_filebeat_mysql_slowlog_input` to `true` to collect -`/var/log/mysql/*-slow.log` with the matching multiline pattern already -configured. The events are tagged with `mysql.logtype: slowquery`. +Set `beats_filebeat_mysql_slowlog_input` to `true` to collect `/var/log/mysql/*-slow.log` with the matching multiline pattern already configured. The events are tagged with `mysql.logtype: slowquery`. -## Modules ## +## Modules -`beats_filebeat_modules` is a list of Filebeat module names to enable -(**experimental**, unset by default). The role runs `filebeat modules enable` -and sets up their ingest pipelines. +`beats_filebeat_modules` is a list of Filebeat module names to enable (**experimental**, unset by default). The role runs `filebeat modules enable` and sets up their ingest pipelines. -On Elastic Stack release 8 and newer (`elasticstack_release > 7`), enabling any -modules also deploys a predefined **System module** configuration to -`modules.d/system.yml`, which collects the syslog files (`/var/log/syslog`, -`/var/log/messages`) through the system module's `syslog` fileset. +On Elastic Stack release 8 and newer (`elasticstack_release > 7`), enabling any modules also deploys a predefined **System module** configuration to `modules.d/system.yml`, which collects the syslog files (`/var/log/syslog`, `/var/log/messages`) through the system module's `syslog` fileset. ```yaml beats_filebeat_modules: @@ -143,6 +111,4 @@ beats_filebeat_modules: - nginx ``` -See the -[Filebeat modules](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-modules) -reference for the available modules. +See the [Filebeat modules](https://www.elastic.co/docs/reference/beats/filebeat/filebeat-modules) reference for the available modules. diff --git a/roles/elasticsearch/README.md b/roles/elasticsearch/README.md index d039f979..9594c953 100644 --- a/roles/elasticsearch/README.md +++ b/roles/elasticsearch/README.md @@ -1,24 +1,13 @@ # Ansible Role: Elasticsearch -Installs and configures [Elasticsearch](https://www.elastic.co/elasticsearch) on -Linux systems. The role manages `elasticsearch.yml` and the JVM configuration -(heap and options). With security enabled (the default) it creates a self-signed -CA and a certificate for each node, enables transport and HTTP TLS, manages the -keystore, and generates the initial passwords; the same CA is reused by the -Logstash, Kibana and Beats roles for their own certificates. It configures -cluster discovery for single- or multi-node setups and upgrades the nodes one at -a time when you raise `elasticstack_version`. +Installs and configures [Elasticsearch](https://www.elastic.co/elasticsearch) on Linux systems. The role manages `elasticsearch.yml` and the JVM configuration (heap and options). With security enabled (the default) it creates a self-signed CA and a certificate for each node, enables transport and HTTP TLS, manages the keystore, and generates the initial passwords; the same CA is reused by the Logstash, Kibana and Beats roles for their own certificates. It configures cluster discovery for single- or multi-node setups and upgrades the nodes one at a time when you raise `elasticstack_version`. -The OSS variant is only available up to Elastic Stack 7; from release 8 on only -the default `elastic` variant exists, and security is required. +The OSS variant is only available up to Elastic Stack 7; from release 8 on only the default `elastic` variant exists, and security is required. ## Requirements * The Elastic repositories configured — use the [`repos`](../repos) role. -* For the default security setup the role generates certificates with the - bundled `elasticsearch-certutil` and `openssl`. The required Python libraries - (`cryptography`, `elasticsearch`) are installed for you by the shared - `elasticstack` role. +* For the default security setup the role generates certificates with the bundled `elasticsearch-certutil` and `openssl`. The required Python libraries (`cryptography`, `elasticsearch`) are installed for you by the shared `elasticstack` role. ## Example @@ -43,58 +32,53 @@ Run only parts of the role with `--tags`: ## Role variables -| Variable | Type | Default | Choices | Description | -|----------|------|---------|---------|-------------| -| `elasticsearch_enable` | `bool` | `true` | — | Start and enable the Elasticsearch service. | -| `elasticsearch_manage_yaml` | `bool` | `true` | — | Manage and overwrite elasticsearch.yml. | -| `elasticsearch_config_backup` | `bool` | `false` | — | Keep a backup of elasticsearch.yml whenever the role changes it. | -| `elasticsearch_clustername` | `str` | `"elasticsearch"` | — | Name of the Elasticsearch cluster (cluster.name). | -| `elasticsearch_nodename` | `str` | N/A | — | Name of this Elasticsearch node (node.name). Defaults to the host's ansible_hostname. | -| `elasticsearch_node_types` | `list` of `str` | N/A | — | Node roles for this node (node.roles), e.g. master, data, ingest. When unset, Elasticsearch uses its own defaults. See the Elasticsearch node-roles documentation for valid values. | -| `elasticsearch_datapath` | `str` | `"/var/lib/elasticsearch"` | — | Directory where Elasticsearch stores its data (path.data). | -| `elasticsearch_create_datapath` | `bool` | `false` | — | Create the data directory if it does not exist. Useful when you change elasticsearch_datapath. | -| `elasticsearch_logpath` | `str` | `"/var/log/elasticsearch"` | — | Directory where Elasticsearch stores its logs (path.logs). | -| `elasticsearch_create_logpath` | `bool` | `false` | — | Create the log directory if it does not exist. Useful when you change elasticsearch_logpath. | -| `elasticsearch_conf_dir` | `str` | `"/etc/elasticsearch/"` | — | Elasticsearch configuration directory. Used to place the jvm.options.d drop-ins. | -| `elasticsearch_group` | `str` | `"elasticsearch"` | — | OS group that owns the jvm.options.d drop-ins. The service reads them through this group, so it must match the group the package runs the service as (elasticsearch). Changing it is not recommended. | -| `elasticsearch_fs_repo` | `list` of `str` | N/A | — | Filesystem paths registered as snapshot repositories (path.repo). Every node must reach the same share under the same path. Unset by default. | -| `elasticsearch_heap` | `str` | `"{{ [[(ansible_memtotal_mb // 1024) // 2, 30] \| min, 1] \| max }}"` | — | JVM heap size in GB. Sets both -Xms and -Xmx via a jvm.options.d drop-in. The default is half of the host memory, capped between 1 and 30 GB. Set to false to leave heap sizing to Elasticsearch/the JVM. | -| `elasticsearch_heap_dump_path` | `str` | `"/var/lib/elasticsearch"` | — | Directory for JVM heap dumps (-XX:HeapDumpPath). | -| `elasticsearch_jvm_custom_parameters` | `raw` | `""` | — | Extra JVM options written to a jvm.options.d drop-in, provided as a list of option lines (e.g. ["-XX:+UseG1GC"]). Empty by default. | -| `elasticsearch_api_host` | `str` | `"localhost"` | — | Host or IP the role uses for its own Elasticsearch API connection checks. | -| `elasticsearch_network_host` | `str` | N/A | — | Network addresses Elasticsearch binds to (network.host). Accepts IPs, interface names, or Elasticsearch special values, e.g. "_ens190_,_local_". Defaults to "_local_", "_site_" when unset. | -| `elasticsearch_http_publish_host` | `str` | N/A | — | Address HTTP clients use to reach this node when sniffing (http.publish_host). Unset by default. | -| `elasticsearch_http_publish_port` | `int` | N/A | — | HTTP publish port, only needed when it must differ from http.port (http.publish_port). Unset by default. | -| `elasticsearch_transport_port` | `int` | N/A | — | Port for inter-node transport communication (transport.port). Unset by default. | -| `elasticsearch_seed_hosts` | `list` of `str` | N/A | — | Seed hosts for cluster discovery (discovery.seed_hosts). Overrides the list the role builds from the elasticsearch group. Unset by default. | -| `elasticsearch_check_calculation` | `bool` | `false` | — | When true, Elasticsearch is not installed, configured, or started: the role runs only the node-role validation and then stops each host (meta: end_host). The validation derives each node's role (master/data/other) from elasticsearch_node_types, groups the nodes, and asserts an odd number of master-eligible nodes (required for a valid quorum). Requires elasticsearch_node_types. Useful as a pre-flight inventory/topology check (before scaling or in CI) without a full deployment. | -| `elasticsearch_disable_systemcallfilterchecks` | `bool` | `false` | — | Disable the system call filter check (bootstrap.system_call_filter). Has a security impact and is only applied on Elastic Stack release 7. | -| `elasticsearch_jna_workaround` | `bool` | `false` | — | Work around systems where JNA cannot load its native library. WARNING this rewrites the sysconfig/default file and overwrites ES_JAVA_OPTS. | -| `elasticsearch_unsafe_upgrade_restart` | `bool` | `false` | — | During upgrades, update the package first and then restart the service, instead of stopping it, upgrading and starting again. Only for non-production clusters, use at your own risk. | -| `elasticsearch_security` | `bool` | `true` | — | Enable X-Pack security (transport TLS, keystore, CA and certificates). Required on Elastic Stack release 8 and later with the elastic variant. | -| `elasticsearch_http_security` | `bool` | `true` | — | Enable TLS on the HTTP layer (xpack.security.http.ssl). Only effective when elasticsearch_security is enabled. | -| `elasticsearch_security_enrollment` | `bool` | N/A | — | Control node/Kibana enrollment on a node autoconfigured for security (xpack.security.enrollment.enabled). Unset by default. | -| `elasticsearch_ml_enabled` | `bool` | `true` | — | Enable Machine Learning (xpack.ml.enabled). Only applied with the elastic variant. | -| `elasticsearch_monitoring_enabled` | `bool` | `true` | — | Enable X-Pack monitoring collection (xpack.monitoring.collection.enabled). Only applied on Elastic Stack release 7. | -| `elasticsearch_bootstrap_pw` | `str` | `"PleaseChangeMe"` | — | Bootstrap password stored in the keystore. Only takes effect when Elasticsearch is initialised for the first time; later changes do not update it. | -| `elasticsearch_http_protocol` | `str` | `"http"` | `http`, `https` | Protocol the role uses to talk to the Elasticsearch API. Set automatically to https once HTTP security is active. | -| `elasticsearch_ssl_verification_mode` | `str` | `"full"` | `full`, `certificate`, `none` | How certificates presented by the other party are verified (xpack.security.transport.ssl.verification_mode). | -| `elasticsearch_tls_key_passphrase` | `str` | `"PleaseChangeMeIndividually"` | — | Passphrase for the generated Elasticsearch certificates. Overridden by elasticstack_cert_pass when that is set. | -| `elasticsearch_cert_validity_period` | `int` | `1095` | — | Number of days the generated certificates are valid. | -| `elasticsearch_cert_expiration_buffer` | `int` | `30` | — | Renew the certificate when its remaining validity (in days) drops below this value. | -| `elasticsearch_cert_will_expire_soon` | `bool` | `false` | — | Set to true to force renewal of the Elasticsearch certificate. Alternatively run the playbook with the renew_es_cert tag. | -| `elasticsearch_extra_config` | `dict` | N/A | — | Additional elasticsearch.yml settings, given as a mapping and rendered as YAML into the configuration (via to_nice_yaml). Unset by default. | -| `elasticsearch_initialized_file` | `str` | `"{{ elasticstack_initial_passwords \| default('') \| dirname }}/cluster_initialized"` | — | Marker file the role writes once the cluster is initialised, used to detect an existing setup. Advanced; usually left at its default. | -| `elasticsearch_freshstart` | `dict` | `{'changed': False}` | — | Internal state used by the role to detect a fresh install. Do not set manually. | -| `elasticsearch_freshstart_security` | `dict` | `{'changed': False}` | — | Internal state used by the role to detect a fresh security setup. Do not set manually. | +| Variable | Description | +|----------|-------------| +| `elasticsearch_enable`
**Type**: `bool`
**Default**: `true` | Start and enable the Elasticsearch service. | +| `elasticsearch_manage_yaml`
**Type**: `bool`
**Default**: `true` | Manage and overwrite elasticsearch.yml. | +| `elasticsearch_config_backup`
**Type**: `bool`
**Default**: `false` | Keep a backup of elasticsearch.yml whenever the role changes it. | +| `elasticsearch_clustername`
**Type**: `str`
**Default**: `"elasticsearch"` | Name of the Elasticsearch cluster (cluster.name). | +| `elasticsearch_nodename`
**Type**: `str` | Name of this Elasticsearch node (node.name). Defaults to the host's ansible_hostname. | +| `elasticsearch_node_types`
**Type**: `list` of `str` | Node roles for this node (node.roles), e.g. master, data, ingest. When unset, Elasticsearch uses its own defaults. See the Elasticsearch node-roles documentation for valid values. | +| `elasticsearch_datapath`
**Type**: `str`
**Default**: `"/var/lib/elasticsearch"` | Directory where Elasticsearch stores its data (path.data). | +| `elasticsearch_create_datapath`
**Type**: `bool`
**Default**: `false` | Create the data directory if it does not exist. Useful when you change elasticsearch_datapath. | +| `elasticsearch_logpath`
**Type**: `str`
**Default**: `"/var/log/elasticsearch"` | Directory where Elasticsearch stores its logs (path.logs). | +| `elasticsearch_create_logpath`
**Type**: `bool`
**Default**: `false` | Create the log directory if it does not exist. Useful when you change elasticsearch_logpath. | +| `elasticsearch_conf_dir`
**Type**: `str`
**Default**: `"/etc/elasticsearch/"` | Elasticsearch configuration directory. Used to place the jvm.options.d drop-ins. | +| `elasticsearch_group`
**Type**: `str`
**Default**: `"elasticsearch"` | OS group that owns the jvm.options.d drop-ins. The service reads them through this group, so it must match the group the package runs the service as (elasticsearch). Changing it is not recommended. | +| `elasticsearch_fs_repo`
**Type**: `list` of `str` | Filesystem paths registered as snapshot repositories (path.repo). Every node must reach the same share under the same path. Unset by default. | +| `elasticsearch_heap`
**Type**: `str`
**Default**: `"{{ [[(ansible_memtotal_mb // 1024) // 2, 30] \| min, 1] \| max }}"` | JVM heap size in GB. Sets both -Xms and -Xmx via a jvm.options.d drop-in. The default is half of the host memory, capped between 1 and 30 GB. Set to false to leave heap sizing to Elasticsearch/the JVM. | +| `elasticsearch_heap_dump_path`
**Type**: `str`
**Default**: `"/var/lib/elasticsearch"` | Directory for JVM heap dumps (-XX:HeapDumpPath). | +| `elasticsearch_jvm_custom_parameters`
**Type**: `raw`
**Default**: `""` | Extra JVM options written to a jvm.options.d drop-in, provided as a list of option lines (e.g. ["-XX:+UseG1GC"]). Empty by default. | +| `elasticsearch_api_host`
**Type**: `str`
**Default**: `"localhost"` | Host or IP the role uses for its own Elasticsearch API connection checks. | +| `elasticsearch_network_host`
**Type**: `str` | Network addresses Elasticsearch binds to (network.host). Accepts IPs, interface names, or Elasticsearch special values, e.g. "_ens190_,_local_". Defaults to "_local_", "_site_" when unset. | +| `elasticsearch_http_publish_host`
**Type**: `str` | Address HTTP clients use to reach this node when sniffing (http.publish_host). Unset by default. | +| `elasticsearch_http_publish_port`
**Type**: `int` | HTTP publish port, only needed when it must differ from http.port (http.publish_port). Unset by default. | +| `elasticsearch_transport_port`
**Type**: `int` | Port for inter-node transport communication (transport.port). Unset by default. | +| `elasticsearch_seed_hosts`
**Type**: `list` of `str` | Seed hosts for cluster discovery (discovery.seed_hosts). Overrides the list the role builds from the elasticsearch group. Unset by default. | +| `elasticsearch_check_calculation`
**Type**: `bool`
**Default**: `false` | When true, Elasticsearch is not installed, configured, or started: the role runs only the node-role validation and then stops each host (meta: end_host). The validation derives each node's role (master/data/other) from elasticsearch_node_types, groups the nodes, and asserts an odd number of master-eligible nodes (required for a valid quorum). Requires elasticsearch_node_types. Useful as a pre-flight inventory/topology check (before scaling or in CI) without a full deployment. | +| `elasticsearch_disable_systemcallfilterchecks`
**Type**: `bool`
**Default**: `false` | Disable the system call filter check (bootstrap.system_call_filter). Has a security impact and is only applied on Elastic Stack release 7. | +| `elasticsearch_jna_workaround`
**Type**: `bool`
**Default**: `false` | Work around systems where JNA cannot load its native library. WARNING this rewrites the sysconfig/default file and overwrites ES_JAVA_OPTS. | +| `elasticsearch_unsafe_upgrade_restart`
**Type**: `bool`
**Default**: `false` | During upgrades, update the package first and then restart the service, instead of stopping it, upgrading and starting again. Only for non-production clusters, use at your own risk. | +| `elasticsearch_security`
**Type**: `bool`
**Default**: `true` | Enable X-Pack security (transport TLS, keystore, CA and certificates). Required on Elastic Stack release 8 and later with the elastic variant. | +| `elasticsearch_http_security`
**Type**: `bool`
**Default**: `true` | Enable TLS on the HTTP layer (xpack.security.http.ssl). Only effective when elasticsearch_security is enabled. | +| `elasticsearch_security_enrollment`
**Type**: `bool` | Control node/Kibana enrollment on a node autoconfigured for security (xpack.security.enrollment.enabled). Unset by default. | +| `elasticsearch_ml_enabled`
**Type**: `bool`
**Default**: `true` | Enable Machine Learning (xpack.ml.enabled). Only applied with the elastic variant. | +| `elasticsearch_monitoring_enabled`
**Type**: `bool`
**Default**: `true` | Enable X-Pack monitoring collection (xpack.monitoring.collection.enabled). Only applied on Elastic Stack release 7. | +| `elasticsearch_bootstrap_pw`
**Type**: `str`
**Default**: `"PleaseChangeMe"` | Bootstrap password stored in the keystore. Only takes effect when Elasticsearch is initialised for the first time; later changes do not update it. | +| `elasticsearch_http_protocol`
**Type**: `str`
**Default**: `"http"`
**Choices**: `http`, `https` | Protocol the role uses to talk to the Elasticsearch API. Set automatically to https once HTTP security is active. | +| `elasticsearch_ssl_verification_mode`
**Type**: `str`
**Default**: `"full"`
**Choices**: `full`, `certificate`, `none` | How certificates presented by the other party are verified (xpack.security.transport.ssl.verification_mode). | +| `elasticsearch_tls_key_passphrase`
**Type**: `str`
**Default**: `"PleaseChangeMeIndividually"` | Passphrase for the generated Elasticsearch certificates. Overridden by elasticstack_cert_pass when that is set. | +| `elasticsearch_cert_validity_period`
**Type**: `int`
**Default**: `1095` | Number of days the generated certificates are valid. | +| `elasticsearch_cert_expiration_buffer`
**Type**: `int`
**Default**: `30` | Renew the certificate when its remaining validity (in days) drops below this value. | +| `elasticsearch_cert_will_expire_soon`
**Type**: `bool`
**Default**: `false` | Set to true to force renewal of the Elasticsearch certificate. Alternatively run the playbook with the renew_es_cert tag. | +| `elasticsearch_extra_config`
**Type**: `dict` | Additional elasticsearch.yml settings, given as a mapping and rendered as YAML into the configuration (via to_nice_yaml). Unset by default. | +| `elasticsearch_initialized_file`
**Type**: `str`
**Default**: `"{{ elasticstack_initial_passwords \| default('') \| dirname }}/cluster_initialized"` | Marker file the role writes once the cluster is initialised, used to detect an existing setup. Advanced; usually left at its default. | +| `elasticsearch_freshstart`
**Type**: `dict`
**Default**: `{'changed': False}` | Internal state used by the role to detect a fresh install. Do not set manually. | +| `elasticsearch_freshstart_security`
**Type**: `dict`
**Default**: `{'changed': False}` | Internal state used by the role to detect a fresh security setup. Do not set manually. | ## Shared variables -This role also uses the collection-wide `elasticstack_*` variables (e.g. -`elasticstack_full_stack`, `elasticstack_variant`, `elasticstack_release`, -`elasticstack_ca_host`, `elasticstack_ca_pass`, `elasticstack_ca_dir`, -`elasticstack_elasticsearch_http_port`, `elasticstack_initial_passwords`). They -are documented centrally with the -[elasticstack role](../elasticstack/README.md). +This role also uses the collection-wide `elasticstack_*` variables (e.g. `elasticstack_full_stack`, `elasticstack_variant`, `elasticstack_release`, `elasticstack_ca_host`, `elasticstack_ca_pass`, `elasticstack_ca_dir`, `elasticstack_elasticsearch_http_port`, `elasticstack_initial_passwords`). They are documented centrally with the [elasticstack role](../elasticstack/README.md). diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml index b5d5620c..00eb880b 100644 --- a/roles/elasticsearch/tasks/main.yml +++ b/roles/elasticsearch/tasks/main.yml @@ -264,16 +264,13 @@ enabled: yes register: elasticsearch_freshstart -# The comment in the following task will disable KICS security checks for this -# very line. In this state of the system we can only communicate without https -# so the finding is a false positive +# Security is off in this branch, so the API is only reachable over http. - name: Handle cluster setup without security when: not elasticsearch_security | bool block: - name: Check for cluster status without security ansible.builtin.uri: -# kics-scan ignore-line url: "http://{{ elasticsearch_api_host }}:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty" register: elasticsearch_cluster_status ignore_errors: "{{ ansible_check_mode }}" diff --git a/roles/elasticstack/README.md b/roles/elasticstack/README.md index 132a2cb5..2d20d381 100644 --- a/roles/elasticstack/README.md +++ b/roles/elasticstack/README.md @@ -1,20 +1,12 @@ # Ansible Role: elasticstack -The shared base role of the collection. Every component role (elasticsearch, -logstash, kibana, beats, repos) imports it automatically on first use, so you -normally do not call it directly. It installs the common dependencies and, above -all, defines the collection-wide `elasticstack_*` variables — the settings shared -across all roles: release and variant, the inventory host groups, ports, the -package repositories, and the stack CA. +The shared base role of the collection. Every component role (elasticsearch,logstash, kibana, beats, repos) imports it automatically on first use, so you normally do not call it directly. It installs the common dependencies and, above all, defines the collection-wide `elasticstack_*` variables — the settings shared across all roles: release and variant, the inventory host groups, ports, the package repositories, and the stack CA. -Set these variables once (in `group_vars`, the play, or via `-e`) and every role -picks them up. This page is the central reference for them. +Set these variables once (in `group_vars`, the play, or via `-e`) and every role picks them up. This page is the central reference for them. ## Requirements -You do not run this role directly — the component roles import it automatically. -It installs the shared Python libraries (`cryptography` and the `elasticsearch` -client) and packages (`openssl`, `unzip`) that the collection needs. +You do not run this role directly — the component roles import it automatically. It installs the shared Python libraries (`cryptography` and the `elasticsearch` client) and packages (`openssl`, `unzip`) that the collection needs. ## Example @@ -32,36 +24,37 @@ The role runs automatically the first time any component role is applied to a ho ## Role variables -| Variable | Type | Default | Choices | Description | -|----------|------|---------|---------|-------------| -| `elasticstack_release` | `int` | `8` | — | Major Elastic Stack release to install and configure (e.g. 7 or 8). Drives the package repository and release-dependent behaviour. | -| `elasticstack_variant` | `str` | `"elastic"` | `elastic`, `oss` | Distribution to install. "elastic" (the default) includes X-Pack; "oss" is the Apache-licensed build without X-Pack. OSS Elasticsearch and Kibana exist only up to release 7, while OSS Logstash and Beats are available on later releases too. | -| `elasticstack_version` | `str` | N/A | — | Exact package version to install (e.g. "8.8.1"). Unset by default: the collection then installs the latest, or reuses the version already present on the CA host. | -| `elasticstack_full_stack` | `bool` | `true` | — | Treat the hosts as one coordinated stack (roles read each other's hosts, share the CA and wire the connections). Set to false to run a role standalone. | -| `elasticstack_security` | `bool` | `true` | — | Enable X-Pack security across the stack. Only effective with the elastic variant. | -| `elasticstack_override_beats_tls` | `bool` | `false` | — | When true, do not auto-enable TLS between Beats and Logstash in a full stack (also skips Beats certificate generation). | -| `elasticstack_elasticsearch_group_name` | `str` | `"elasticsearch"` | — | Inventory group that holds the Elasticsearch hosts. | -| `elasticstack_logstash_group_name` | `str` | `"logstash"` | — | Inventory group that holds the Logstash hosts. | -| `elasticstack_kibana_group_name` | `str` | `"kibana"` | — | Inventory group that holds the Kibana hosts. | -| `elasticstack_elasticsearch_http_port` | `int` | `9200` | — | Elasticsearch HTTP API port. | -| `elasticstack_kibana_port` | `int` | `5601` | — | Kibana HTTP port (also used to build the public base URL). | -| `elasticstack_beats_port` | `int` | `5044` | — | Port of the Logstash Beats input that the Beats ship to. | -| `elasticstack_enable_repos` | `bool` | `true` | — | Configure the Elastic package repositories (see the repos role). | -| `elasticstack_repo_url` | `str` | `"https://artifacts.elastic.co/packages"` | — | Base URL of the Elastic package repositories. | -| `elasticstack_repo_key` | `str` | `"https://artifacts.elastic.co/GPG-KEY-elasticsearch"` | — | URL of the Elastic repository signing (GPG) key. | -| `elasticstack_rpm_workaround` | `bool` | `false` | — | Enable a workaround for importing the repository GPG key on RPM-based systems. | -| `elasticstack_ca_host` | `str` | `"{{ groups[elasticstack_elasticsearch_group_name][0] \| default('') }}"` | — | Inventory host that holds the stack CA and signs all component certificates. Must be an Elasticsearch host: signing uses elasticsearch-certutil, which ships with the elasticsearch package. Defaults to the first host in the elasticsearch group; override it only to pin the CA and the initial_passwords file to a specific Elasticsearch node. Only used when elasticstack_full_stack is true. | -| `elasticstack_ca_dir` | `str` | `"/opt/es-ca"` | — | Directory on the CA host where the CA and generated certificates are stored. | -| `elasticstack_ca_name` | `str` | `"CN=Elastic Certificate Tool Autogenerated CA"` | — | Distinguished name (DN) of the generated CA. | -| `elasticstack_ca_pass` | `str` | `"PleaseChangeMe"` | — | Password protecting the CA private key. | -| `elasticstack_ca_validity_period` | `int` | `1095` | — | Number of days the generated CA is valid. | -| `elasticstack_ca_expiration_buffer` | `int` | `30` | — | Renew the CA when its remaining validity (in days) drops below this value. | -| `elasticstack_ca_will_expire_soon` | `bool` | `false` | — | Set to true to force renewal of the CA and all component certificates. Alternatively run the playbook with the renew_ca tag. | -| `elasticstack_cert_pass` | `str` | N/A | — | Common passphrase for all component certificates. Unset by default; when set, it overrides each role's individual certificate passphrase. | -| `elasticstack_initial_passwords` | `str` | `"/usr/share/elasticsearch/initial_passwords"` | — | Path on the CA host where the generated initial passwords are stored. | -| `elasticstack_no_log` | `bool` | `true` | — | Hide the output of tasks that could reveal passwords. Set to false for debugging. | -| `elasticstack_manage_pip` | `bool` | `false` | — | Install pip on the target host. | -| `elasticstack_force_pip` | `bool` | `false` | — | Force installation of the required Python modules via pip (useful when the distribution packages are too old). See PEP 668. | -| `elasticstack_ci` | `bool` | `false` | — | Internal test flag. Set to true only by the collection's molecule scenarios to enable resource workarounds needed on constrained test runners (cache cleanup, relaxed Elasticsearch disk watermarks). Never set this on a real installation. | +| Variable | Description | +|----------|-------------| +| `elasticstack_release`
**Type**: `int`
**Default**: `8` | Major Elastic Stack release to install and configure (e.g. 7 or 8). Drives the package repository and release-dependent behaviour. | +| `elasticstack_variant`
**Type**: `str`
**Default**: `"elastic"`
**Choices**: `elastic`, `oss` | Distribution to install. "elastic" (the default) includes X-Pack; "oss" is the Apache-licensed build without X-Pack. OSS Elasticsearch and Kibana exist only up to release 7, while OSS Logstash and Beats are available on later releases too. | +| `elasticstack_version`
**Type**: `str` | Exact package version to install (e.g. "8.8.1"). Unset by default: the collection then installs the latest, or reuses the version already present on the CA host. | +| `elasticstack_full_stack`
**Type**: `bool`
**Default**: `true` | Treat the hosts as one coordinated stack (roles read each other's hosts, share the CA and wire the connections). Set to false to run a role standalone. | +| `elasticstack_security`
**Type**: `bool`
**Default**: `true` | Enable X-Pack security across the stack. Only effective with the elastic variant. | +| `elasticstack_override_beats_tls`
**Type**: `bool`
**Default**: `false` | When true, do not auto-enable TLS between Beats and Logstash in a full stack (also skips Beats certificate generation). | +| `elasticstack_elasticsearch_group_name`
**Type**: `str`
**Default**: `"elasticsearch"` | Inventory group that holds the Elasticsearch hosts. | +| `elasticstack_logstash_group_name`
**Type**: `str`
**Default**: `"logstash"` | Inventory group that holds the Logstash hosts. | +| `elasticstack_kibana_group_name`
**Type**: `str`
**Default**: `"kibana"` | Inventory group that holds the Kibana hosts. | +| `elasticstack_elasticsearch_http_port`
**Type**: `int`
**Default**: `9200` | Elasticsearch HTTP API port. | +| `elasticstack_kibana_host`
**Type**: `str`
**Default**: `"{{ ansible_fqdn }}"` | Host name Kibana publishes itself under (server.publicBaseUrl). Defaults to the fully qualified domain name of the Kibana host. Set it when users reach Kibana under a different name, for example behind a reverse proxy or a load balancer. | +| `elasticstack_kibana_port`
**Type**: `int`
**Default**: `5601` | Kibana HTTP port (also used to build the public base URL). | +| `elasticstack_beats_port`
**Type**: `int`
**Default**: `5044` | Port of the Logstash Beats input that the Beats ship to. | +| `elasticstack_enable_repos`
**Type**: `bool`
**Default**: `true` | Configure the Elastic package repositories (see the repos role). | +| `elasticstack_repo_url`
**Type**: `str`
**Default**: `"https://artifacts.elastic.co/packages"` | Base URL of the Elastic package repositories. | +| `elasticstack_repo_key`
**Type**: `str`
**Default**: `"https://artifacts.elastic.co/GPG-KEY-elasticsearch"` | URL of the Elastic repository signing (GPG) key. | +| `elasticstack_rpm_workaround`
**Type**: `bool`
**Default**: `false` | Enable a workaround for importing the repository GPG key on RPM-based systems. | +| `elasticstack_ca_host`
**Type**: `str`
**Default**: `"{{ groups[elasticstack_elasticsearch_group_name][0] \| default('') }}"` | Inventory host that holds the stack CA and signs all component certificates. Must be an Elasticsearch host: signing uses elasticsearch-certutil, which ships with the elasticsearch package. Defaults to the first host in the elasticsearch group; override it only to pin the CA and the initial_passwords file to a specific Elasticsearch node. Only used when elasticstack_full_stack is true. | +| `elasticstack_ca_dir`
**Type**: `str`
**Default**: `"/opt/es-ca"` | Directory on the CA host where the CA and generated certificates are stored. | +| `elasticstack_ca_name`
**Type**: `str`
**Default**: `"CN=Elastic Certificate Tool Autogenerated CA"` | Distinguished name (DN) of the generated CA. | +| `elasticstack_ca_pass`
**Type**: `str`
**Default**: `"PleaseChangeMe"` | Password protecting the CA private key. | +| `elasticstack_ca_validity_period`
**Type**: `int`
**Default**: `1095` | Number of days the generated CA is valid. | +| `elasticstack_ca_expiration_buffer`
**Type**: `int`
**Default**: `30` | Renew the CA when its remaining validity (in days) drops below this value. | +| `elasticstack_ca_will_expire_soon`
**Type**: `bool`
**Default**: `false` | Set to true to force renewal of the CA and all component certificates. Alternatively run the playbook with the renew_ca tag. | +| `elasticstack_cert_pass`
**Type**: `str` | Common passphrase for all component certificates. Unset by default; when set, it overrides each role's individual certificate passphrase. | +| `elasticstack_initial_passwords`
**Type**: `str`
**Default**: `"/usr/share/elasticsearch/initial_passwords"` | Path on the CA host where the generated initial passwords are stored. | +| `elasticstack_no_log`
**Type**: `bool`
**Default**: `true` | Hide the output of tasks that could reveal passwords. Set to false for debugging. | +| `elasticstack_manage_pip`
**Type**: `bool`
**Default**: `false` | Install pip on the target host. | +| `elasticstack_force_pip`
**Type**: `bool`
**Default**: `false` | Force installation of the required Python modules via pip (useful when the distribution packages are too old). See PEP 668. | +| `elasticstack_ci`
**Type**: `bool`
**Default**: `false` | Internal test flag. Set to true only by the collection's molecule scenarios to enable resource workarounds needed on constrained test runners (cache cleanup, relaxed Elasticsearch disk watermarks). Never set this on a real installation. | diff --git a/roles/elasticstack/defaults/main.yml b/roles/elasticstack/defaults/main.yml index 4aaf3291..21219ca1 100644 --- a/roles/elasticstack/defaults/main.yml +++ b/roles/elasticstack/defaults/main.yml @@ -16,6 +16,7 @@ elasticstack_elasticsearch_http_port: 9200 elasticstack_enable_repos: true elasticstack_full_stack: true elasticstack_initial_passwords: /usr/share/elasticsearch/initial_passwords +elasticstack_kibana_host: "{{ ansible_fqdn }}" elasticstack_kibana_port: 5601 elasticstack_override_beats_tls: false elasticstack_release: 8 diff --git a/roles/elasticstack/meta/argument_specs.yml b/roles/elasticstack/meta/argument_specs.yml index 52e912f9..e7d26978 100644 --- a/roles/elasticstack/meta/argument_specs.yml +++ b/roles/elasticstack/meta/argument_specs.yml @@ -87,6 +87,14 @@ argument_specs: default: 9200 description: Elasticsearch HTTP API port. + elasticstack_kibana_host: + type: str + default: "{{ ansible_fqdn }}" + description: + Host name Kibana publishes itself under (server.publicBaseUrl). Defaults to the + fully qualified domain name of the Kibana host. Set it when users reach Kibana + under a different name, for example behind a reverse proxy or a load balancer. + elasticstack_kibana_port: type: int default: 5601 diff --git a/roles/kibana/README.md b/roles/kibana/README.md index ab0e349c..68cf3a21 100644 --- a/roles/kibana/README.md +++ b/roles/kibana/README.md @@ -1,18 +1,12 @@ # Ansible Role: Kibana -Installs and configures [Kibana](https://www.elastic.co/kibana) on Linux systems. -The role manages `kibana.yml` and can enable TLS for the Kibana web interface, -using a certificate you provide. In a full stack setup together with the -elasticsearch role, it also configures the connection to Elasticsearch — the TLS -trust and the `kibana_system` credentials — using the Elasticsearch CA. +Installs and configures [Kibana](https://www.elastic.co/kibana) on Linux systems. The role manages `kibana.yml` and can enable TLS for the Kibana web interface, using a certificate you provide. In a full stack setup together with the elasticsearch role, it also configures the connection to Elasticsearch — the TLS trust and the `kibana_system` credentials — using the Elasticsearch CA. ## Requirements * The Elastic repositories configured — use the [`repos`](../repos) role. * A reachable Elasticsearch instance for Kibana to connect to. -* For browser-facing TLS (`kibana_tls: true`): a certificate and key from a CA your - users trust (corporate PKI or a public CA), provided via `kibana_tls_cert` and - `kibana_tls_key`. The role does **not** generate a browser certificate. +* For browser-facing TLS (`kibana_tls: true`): a certificate and key from a CA your users trust (corporate PKI or a public CA), provided via `kibana_tls_cert` and `kibana_tls_key`. The role does **not** generate a browser certificate. ## Example @@ -26,9 +20,7 @@ trust and the `kibana_system` credentials — using the Elasticsearch CA. - kibana ``` -By default Kibana only logs to the journal, readable with -`journalctl -u kibana`. Set `kibana_manage_logging` to write a log file that -Kibana rotates itself, no logrotate needed: +By default Kibana only logs to the journal, readable with `journalctl -u kibana`. Set `kibana_manage_logging` to write a log file that Kibana rotates itself, no logrotate needed: ```yaml - name: Install Kibana with file logging @@ -46,10 +38,7 @@ Kibana rotates itself, no logrotate needed: - kibana ``` -This writes `/var/log/kibana/kibana.log` in a human readable layout and keeps -ten rotated files. Set `kibana_logging_layout` to `json` when a log shipper -reads the file, and `kibana_logging_console` to `false` to stop logging to the -journal as well. +This writes `/var/log/kibana/kibana.log` in a human readable layout and keeps ten rotated files. Set `kibana_logging_layout` to `json` when a log shipper reads the file, and `kibana_logging_console` to `false` to stop logging to the journal as well. ## Tags @@ -61,42 +50,38 @@ Run only parts of the role with `--tags`: ## Role variables -| Variable | Type | Default | Choices | Description | -|----------|------|---------|---------|-------------| -| `kibana_enable` | `bool` | `true` | — | Start and enable the Kibana service. | -| `kibana_config_backup` | `bool` | `true` | — | Keep a backup of kibana.yml whenever the role changes it. | -| `kibana_manage_yaml` | `bool` | `true` | — | Manage and overwrite kibana.yml. | -| `kibana_elasticsearch_hosts` | `list` of `str` | N/A | — | Elasticsearch hosts Kibana connects to. Defaults to the nodes from the elasticsearch group, or localhost when used standalone. | -| `kibana_security` | `bool` | `true` | — | Enable X-Pack security: connect to Elasticsearch over HTTPS with authentication and set up the encryption keys. Only effective in a full stack setup with the elastic variant. | -| `kibana_sniff_on_start` | `bool` | `false` | — | Ask Elasticsearch for the list of nodes at startup (elasticsearch.sniffOnStart). | -| `kibana_sniff_on_connection_fault` | `bool` | `false` | — | Update the list of Elasticsearch nodes on connection fault (elasticsearch.sniffOnConnectionFault). | -| `kibana_sniff_interval` | `str` | N/A | — | Interval between sniffing attempts (elasticsearch.sniffInterval, in milliseconds). Unset by default. | -| `kibana_tls` | `bool` | `false` | — | Enable TLS on the Kibana server itself (server.ssl). | -| `kibana_tls_cert` | `str` | `"/etc/kibana/certs/cert.pem"` | — | Path to the certificate for the Kibana server TLS. | -| `kibana_tls_key` | `str` | `"/etc/kibana/certs/key.pem"` | — | Path to the private key for the Kibana server TLS. | -| `kibana_tls_key_passphrase` | `str` | `"PleaseChangeMe"` | — | Passphrase for the generated Kibana security certificate (X-Pack, full stack). Overridden by elasticstack_cert_pass when that is set. | -| `kibana_cert_validity_period` | `int` | `1095` | — | Number of days the generated certificates are valid. | -| `kibana_cert_expiration_buffer` | `int` | `30` | — | Renew the certificate when its remaining validity (in days) drops below this value. | -| `kibana_cert_will_expire_soon` | `bool` | `false` | — | Set to true to force renewal of the Kibana certificate. Alternatively run the playbook with the renew_kibana_cert tag. | -| `kibana_manage_logging` | `bool` | `false` | — | Manage Kibana's own logging configuration and write log files to disk. When disabled, Kibana keeps its default behaviour and only logs to the journal. | -| `kibana_logpath` | `str` | `"/var/log/kibana"` | — | Directory for the Kibana log files. The role creates it for the kibana user when kibana_manage_logging is enabled. | -| `kibana_logfile` | `str` | `"kibana.log"` | — | Name of the log file inside kibana_logpath. | -| `kibana_loglevel` | `str` | `"info"` | `all`, `trace`, `debug`, `info`, `warn`, `error`, `fatal`, `off` | Log level of the root logger (logging.root.level). | -| `kibana_logging_layout` | `str` | `"pattern"` | `pattern`, `json` | Layout of the log file. Use "pattern" for human readable logs and "json" when the file is picked up by a log shipper. | -| `kibana_logging_console` | `bool` | `true` | — | Keep logging to the console (and therefore the journal) in addition to the log file. Set to false to log to the file only. | -| `kibana_logging_rotate_policy` | `str` | `"size-limit"` | `size-limit`, `time-interval` | Rotation policy of the log file. Kibana supports only one policy per appender, either by size (kibana_logging_rotate_size) or by time (kibana_logging_rotate_interval). | -| `kibana_logging_rotate_size` | `str` | `"100mb"` | — | Size at which the log file is rotated. Only used when kibana_logging_rotate_policy is "size-limit". | -| `kibana_logging_rotate_interval` | `str` | `"24h"` | — | Interval at which the log file is rotated. Only used when kibana_logging_rotate_policy is "time-interval". | -| `kibana_logging_rotate_keep` | `int` | `10` | — | Number of rotated log files to keep (strategy max). | -| `kibana_extra_config` | `str` | N/A | — | Extra configuration appended verbatim to kibana.yml (YAML). Unset by default. | -| `kibana_freshstart` | `dict` | `{'changed': False}` | — | Internal state used by the role to detect a fresh install. Do not set manually. | +| Variable | Description | +|----------|-------------| +| `kibana_enable`
**Type**: `bool`
**Default**: `true` | Start and enable the Kibana service. | +| `kibana_config_backup`
**Type**: `bool`
**Default**: `true` | Keep a backup of kibana.yml whenever the role changes it. | +| `kibana_manage_yaml`
**Type**: `bool`
**Default**: `true` | Manage and overwrite kibana.yml. | +| `kibana_elasticsearch_hosts`
**Type**: `list` of `str` | Elasticsearch hosts Kibana connects to. Defaults to the nodes from the elasticsearch group, or localhost when used standalone. | +| `kibana_security`
**Type**: `bool`
**Default**: `true` | Enable X-Pack security: connect to Elasticsearch over HTTPS with authentication and set up the encryption keys. Only effective in a full stack setup with the elastic variant. | +| `kibana_sniff_on_start`
**Type**: `bool`
**Default**: `false` | Ask Elasticsearch for the list of nodes at startup (elasticsearch.sniffOnStart). | +| `kibana_sniff_on_connection_fault`
**Type**: `bool`
**Default**: `false` | Update the list of Elasticsearch nodes on connection fault (elasticsearch.sniffOnConnectionFault). | +| `kibana_sniff_interval`
**Type**: `str` | Interval between sniffing attempts (elasticsearch.sniffInterval, in milliseconds). Unset by default. | +| `kibana_tls`
**Type**: `bool`
**Default**: `false` | Enable TLS on the Kibana server itself (server.ssl). | +| `kibana_tls_cert`
**Type**: `str`
**Default**: `"/etc/kibana/certs/cert.pem"` | Path to the certificate for the Kibana server TLS. | +| `kibana_tls_key`
**Type**: `str`
**Default**: `"/etc/kibana/certs/key.pem"` | Path to the private key for the Kibana server TLS. | +| `kibana_tls_key_passphrase`
**Type**: `str`
**Default**: `"PleaseChangeMe"` | Passphrase for the generated Kibana security certificate (X-Pack, full stack). Overridden by elasticstack_cert_pass when that is set. | +| `kibana_cert_validity_period`
**Type**: `int`
**Default**: `1095` | Number of days the generated certificates are valid. | +| `kibana_cert_expiration_buffer`
**Type**: `int`
**Default**: `30` | Renew the certificate when its remaining validity (in days) drops below this value. | +| `kibana_cert_will_expire_soon`
**Type**: `bool`
**Default**: `false` | Set to true to force renewal of the Kibana certificate. Alternatively run the playbook with the renew_kibana_cert tag. | +| `kibana_manage_logging`
**Type**: `bool`
**Default**: `false` | Manage Kibana's own logging configuration and write log files to disk. When disabled, Kibana keeps its default behaviour and only logs to the journal. | +| `kibana_logpath`
**Type**: `str`
**Default**: `"/var/log/kibana"` | Directory for the Kibana log files. The role creates it for the kibana user when kibana_manage_logging is enabled. | +| `kibana_logfile`
**Type**: `str`
**Default**: `"kibana.log"` | Name of the log file inside kibana_logpath. | +| `kibana_loglevel`
**Type**: `str`
**Default**: `"info"`
**Choices**: `all`, `trace`, `debug`, `info`, `warn`, `error`, `fatal`, `off` | Log level of the root logger (logging.root.level). | +| `kibana_logging_layout`
**Type**: `str`
**Default**: `"pattern"`
**Choices**: `pattern`, `json` | Layout of the log file. Use "pattern" for human readable logs and "json" when the file is picked up by a log shipper. | +| `kibana_logging_console`
**Type**: `bool`
**Default**: `true` | Keep logging to the console (and therefore the journal) in addition to the log file. Set to false to log to the file only. | +| `kibana_logging_rotate_policy`
**Type**: `str`
**Default**: `"size-limit"`
**Choices**: `size-limit`, `time-interval` | Rotation policy of the log file. Kibana supports only one policy per appender, either by size (kibana_logging_rotate_size) or by time (kibana_logging_rotate_interval). | +| `kibana_logging_rotate_size`
**Type**: `str`
**Default**: `"100mb"` | Size at which the log file is rotated. Only used when kibana_logging_rotate_policy is "size-limit". | +| `kibana_logging_rotate_interval`
**Type**: `str`
**Default**: `"24h"` | Interval at which the log file is rotated. Only used when kibana_logging_rotate_policy is "time-interval". | +| `kibana_logging_rotate_keep`
**Type**: `int`
**Default**: `10` | Number of rotated log files to keep (strategy max). | +| `kibana_extra_config`
**Type**: `str` | Extra configuration appended verbatim to kibana.yml (YAML). Unset by default. | +| `kibana_freshstart`
**Type**: `dict`
**Default**: `{'changed': False}` | Internal state used by the role to detect a fresh install. Do not set manually. | ## Shared variables -This role also uses the collection-wide `elasticstack_*` variables (e.g. -`elasticstack_full_stack`, `elasticstack_variant`, `elasticstack_release`, -`elasticstack_ca_host`, `elasticstack_ca_pass`, `elasticstack_kibana_host`, -`elasticstack_kibana_port`, `elasticstack_elasticsearch_http_port`). They are -documented centrally with the [elasticstack role](../elasticstack/README.md). +This role also uses the collection-wide `elasticstack_*` variables (e.g. `elasticstack_full_stack`, `elasticstack_variant`, `elasticstack_release`, `elasticstack_ca_host`, `elasticstack_ca_pass`, `elasticstack_kibana_host`, `elasticstack_kibana_port`, `elasticstack_elasticsearch_http_port`). They are documented centrally with the [elasticstack role](../elasticstack/README.md). diff --git a/roles/kibana/templates/kibana.yml.j2 b/roles/kibana/templates/kibana.yml.j2 index 3ec1e288..154de717 100644 --- a/roles/kibana/templates/kibana.yml.j2 +++ b/roles/kibana/templates/kibana.yml.j2 @@ -1,5 +1,5 @@ server.host: "0.0.0.0" -server.publicBaseUrl: "http{% if kibana_tls | bool %}s{% endif %}://{{ elasticstack_kibana_host | default( ansible_fqdn ) }}:{{ elasticstack_kibana_port }}" +server.publicBaseUrl: "http{% if kibana_tls | bool %}s{% endif %}://{{ elasticstack_kibana_host }}:{{ elasticstack_kibana_port }}" {% if elasticstack_full_stack is defined and elasticstack_full_stack | bool and elasticstack_variant == "elastic" %} {%- if kibana_sniff_on_start | bool %} diff --git a/roles/logstash/README.md b/roles/logstash/README.md index 2cb77ed5..eaa1ee4d 100644 --- a/roles/logstash/README.md +++ b/roles/logstash/README.md @@ -1,11 +1,6 @@ # Ansible Role: Logstash -Installs and configures [Logstash](https://www.elastic.co/products/logstash) on -Linux systems. The role can manage `logstash.yml`, the log4j2 logging, the JVM -heap, TLS, and the Elasticsearch writer role/user. For pipelines it can create -two default pipelines — a Beats input and an Elasticsearch forwarder, both using -Redis — and manage your own pipelines, whose configuration can be checked out -from external git repositories. +Installs and configures [Logstash](https://www.elastic.co/products/logstash) on Linux systems. The role can manage `logstash.yml`, the log4j2 logging, the JVM heap, TLS, and the Elasticsearch writer role/user. For pipelines it can create two default pipelines — a Beats input and an Elasticsearch forwarder, both using Redis — and manage your own pipelines, whose configuration can be checked out from external git repositories. It works with the standard Elastic Stack packages and with Elastic's OSS variant. @@ -15,8 +10,7 @@ It works with the standard Elastic Stack packages and with Elastic's OSS variant * `passlib` (Python, on the controller) unless you disable password hashing for the Logstash user. * `curl` on the target host. * `git` on the target host if you use git-based pipelines. -* Redis if you use the default pipeline or other pipelines that talk to Redis - (e.g. via the [`geerlingguy.redis`](https://galaxy.ansible.com/geerlingguy/redis) role). +* Redis if you use the default pipeline or other pipelines that talk to Redis (e.g. via the [`geerlingguy.redis`](https://galaxy.ansible.com/geerlingguy/redis) role). You also need the Elastic repositories configured — use the [`repos`](../repos) role. @@ -34,8 +28,7 @@ You also need the Elastic repositories configured — use the [`repos`](../repos ## Pipelines -For how to configure pipelines (built-in Redis/Beats and external git -repositories), see the [pipelines documentation](docs/pipelines.md). +For how to configure pipelines (built-in Redis/Beats and external git repositories), see the [pipelines documentation](docs/pipelines.md). ## Tags @@ -49,101 +42,98 @@ Run only parts of the role with `--tags`: ## Role variables -| Variable | Type | Default | Choices | Description | -|----------|------|---------|---------|-------------| -| `logstash_enable` | `bool` | `true` | — | Start and enable the Logstash service. | -| `logstash_config_backup` | `bool` | `false` | — | Keep backups of every configuration file the role changes. | -| `logstash_manage_yaml` | `bool` | `true` | — | Manage and overwrite logstash.yml. | -| `logstash_manage_logging` | `bool` | `false` | — | Manage the log4j2 logging configuration. | -| `logstash_heap` | `str` | `"1"` | — | JVM heap size in GB. Sets both -Xms and -Xmx via an LS_JAVA_OPTS systemd drop-in.

Elastic recommends 4-8 GB for typical ingestion and staying below 50-75% of physical memory. Keep it low when Logstash shares a host with Elasticsearch so they do not compete for memory. | -| `logstash_plugins` | `list` of `str` | N/A | — | List of Logstash plugins to install. Unset by default. | -| `logstash_config_autoreload` | `bool` | `true` | — | Enable automatic reload of the Logstash configuration. | -| `logstash_config_autoreload_interval` | `str` | N/A | — | Interval between configuration reload checks (e.g. "3s"). Only used when logstash_config_autoreload is enabled. Unset by default. | -| `logstash_config_path_data` | `str` | `"/var/lib/logstash"` | — | Logstash data directory (path.data). | -| `logstash_config_path_logs` | `str` | `"/var/log/logstash"` | — | Logstash log directory (path.logs). | -| `logstash_http_host` | `str` | N/A | — | Bind address of the Logstash monitoring API (http.host). Unset by default. | -| `logstash_http_port` | `str` | N/A | — | Port (or port range, e.g. "9600-9700") of the monitoring API (http.port). Unset by default. | -| `logstash_global_ecs` | `str` | N/A | `disabled`, `v1` | Set the global ECS compatibility mode (pipeline.ecs_compatibility). Unset by default. | -| `logstash_pipeline_unsafe_shutdown` | `bool` | N/A | — | Force Logstash to exit during shutdown even if there are in-flight events. Unset by default. | -| `logstash_legacy_monitoring` | `bool` | `true` | — | Enable legacy X-Pack monitoring. Ignored unless elasticstack_full_stack is set and only effective on Elastic Stack releases lower than 8. | -| `logstash_manage_pipelines` | `bool` | `true` | — | Manage pipelines.yml. | -| `logstash_no_pipelines` | `bool` | `false` | — | Disable all pipeline management entirely. | -| `logstash_pipelines` | `list` of `dict` | … | — | List of pipelines to configure.

Each entry needs a name. A pipeline either points to an external git repository (source/version) or defines simple input/output keys that connect to Redis. See the pipelines documentation for details. | -|     ↳ `name` | `str` | N/A | — | Unique name of the pipeline. Becomes the pipeline directory and pipeline.id. | -|     ↳ `exclusive` | `bool` | N/A | — | Mark this pipeline as the only one allowed to handle its events. | -|     ↳ `source` | `str` | N/A | — | URL of a git repository holding the pipeline configuration. | -|     ↳ `version` | `str` | N/A | — | Git branch/tag/commit to check out from source (default "main"). | -|     ↳ `queue_type` | `str` | N/A | `memory`, `persisted` | Queue type for this pipeline (default "memory"). Use "persisted" for an on-disk persistent queue. | -|     ↳ `queue_max_bytes` | `str` | N/A | — | Maximum queue size for this pipeline (default "1gb"). | -|     ↳ `input` | `list` of `dict` | … | — | Simple Redis inputs for this pipeline. | -|         ↳ `name` | `str` | N/A | — | Name of the input. | -|         ↳ `key` | `str` | N/A | — | Redis key to read from. | -|     ↳ `output` | `list` of `dict` | … | — | Simple Redis outputs for this pipeline. | -|         ↳ `name` | `str` | N/A | — | Name of the output. | -|         ↳ `key` | `str` | N/A | — | Redis key to write to. | -|         ↳ `condition` | `str` | N/A | — | Optional Logstash conditional. The output only receives events matching it. With exclusive set, the conditions are chained with else if. | -|         ↳ `congestion` | `int` | N/A | — | Optional congestion threshold. The output stops once the target Redis key holds more items than this value. | -| `logstash_elasticsearch_output` | `bool` | `true` | — | Create the default pipeline that forwards events to Elasticsearch. | -| `logstash_beats_input` | `bool` | `true` | — | Create the default pipeline with a Beats input. | -| `logstash_beats_input_congestion` | `int` | N/A | — | Congestion threshold for the default Beats input pipeline: it stops writing to Redis once the buffer key holds more items than this value. Unset by default. | -| `logstash_beats_timeout` | `str` | N/A | — | Timeout for idle client connections on the Beats input (e.g. "60s"). Unset by default. | -| `logstash_beats_tls` | `bool` | N/A | — | Activate TLS on the Beats input pipeline. Unset by default, but enabled automatically in a full stack setup unless overridden. | -| `logstash_input_queue_type` | `str` | `"memory"` | `memory`, `persisted` | Queue type for the default Beats input pipeline. Use "persisted" for an on-disk persistent queue. | -| `logstash_input_queue_max_bytes` | `str` | `"1gb"` | — | Maximum queue size for the default Beats input pipeline. | -| `logstash_forwarder_queue_type` | `str` | `"memory"` | `memory`, `persisted` | Queue type for the default Elasticsearch forwarder pipeline. Use "persisted" for an on-disk persistent queue. | -| `logstash_forwarder_queue_max_bytes` | `str` | `"1gb"` | — | Maximum queue size for the default Elasticsearch forwarder pipeline. | -| `logstash_redis_password` | `str` | N/A | — | Password used when the simple inputs/outputs connect to Redis. Unset by default. | -| `logstash_redis_input_host` | `str` | `"localhost"` | — | Redis host that simple pipeline inputs (and the default forwarder input) read from. | -| `logstash_redis_output_host` | `str` | `"localhost"` | — | Redis host that simple pipeline outputs (and the default input pipeline) write to. | -| `logstash_redis_tls` | `bool` | `false` | — | Enable TLS on the simple Redis inputs/outputs (`ssl`/`ssl_enabled`). | -| `logstash_redis_ssl_certificate` | `str` | N/A | — | Path to the SSL certificate for the Redis outputs (the redis output plugin only; the input plugin has no such option). Rendered only when logstash_redis_tls is true. | -| `logstash_redis_ssl_key` | `str` | N/A | — | Path to the SSL key for the Redis outputs. Rendered only when logstash_redis_tls is true. | -| `logstash_redis_ssl_key_passphrase` | `str` | N/A | — | Passphrase for logstash_redis_ssl_key. Rendered only when logstash_redis_tls is true. | -| `logstash_redis_ssl_certificate_authorities` | `list` of `str` | N/A | — | List of CA certificate paths used to verify the Redis server on the Redis outputs. Rendered only when logstash_redis_tls is true. | -| `logstash_redis_ssl_supported_protocols` | `str` | N/A | — | TLS protocol version accepted on the Redis outputs (for example "TLSv1.3"). Rendered only when logstash_redis_tls is true. | -| `logstash_redis_ssl_verification_mode` | `str` | N/A | — | Certificate verification mode for the Redis outputs ("full" or "none"). Rendered only when logstash_redis_tls is true. | -| `logstash_elasticsearch` | `list` of `str` | N/A | — | Elasticsearch hosts for the default output. Defaults to the nodes from the elasticsearch group, or localhost when used standalone. | -| `logstash_validate_after_inactivity` | `str` | `"300"` | — | Seconds Logstash waits before validating a previously idle connection to Elasticsearch. | -| `logstash_sniffing` | `bool` | `false` | — | Enable sniffing for additional Elasticsearch nodes. | -| `logstash_sniffing_delay` | `str` | N/A | — | Seconds to wait between sniffing attempts. Unset by default. | -| `logstash_sniffing_path` | `str` | N/A | — | HTTP path used for the sniffing requests. Unset by default. | -| `logstash_security` | `bool` | N/A | — | Enable X-Pack security (TLS) for Logstash. No default; enabled automatically in full stack mode with the elastic variant. | -| `logstash_tls_key_passphrase` | `str` | `"LogstashChangeMe"` | — | Passphrase for the generated Logstash certificates. | -| `logstash_certs_dir` | `str` | `"/etc/logstash/certs"` | — | Directory holding the Logstash certificates. Used to build several file paths. | -| `logstash_cert_validity_period` | `int` | `1095` | — | Number of days the generated certificates are valid. | -| `logstash_cert_expiration_buffer` | `int` | `30` | — | Renew the certificate when its remaining validity (in days) drops below this value. | -| `logstash_cert_will_expire_soon` | `bool` | `false` | — | Set to true to force renewal of the Logstash certificate. Alternatively run the playbook with the renew_logstash_cert tag. | -| `logstash_create_role` | `bool` | `true` | — | Create the Logstash writer role in Elasticsearch. | -| `logstash_role_name` | `str` | `"logstash_writer"` | — | Name of the Logstash writer role. | -| `logstash_role_cluster_privileges` | `list` of `str` | `['manage_index_templates', 'monitor', 'manage_ilm']` | — | Cluster privileges granted to the Logstash writer role. | -| `logstash_role_indicies_names` | `list` of `str` | `['ecs-logstash*', 'logstash*', 'logs*']` | — | Index patterns the Logstash writer role may access. | -| `logstash_role_indicies_privileges` | `list` of `str` | `['write', 'create', 'delete', 'create_index', 'manage', 'manage_ilm']` | — | Index privileges the Logstash writer role holds on its index patterns. | -| `logstash_create_user` | `bool` | `true` | — | Create the Logstash writer user in Elasticsearch. | -| `logstash_user_name` | `str` | `"logstash_writer"` | — | Name of the Logstash user connecting to Elasticsearch. | -| `logstash_user_password` | `str` | `"password"` | — | Password of the Logstash user. Must be at least 6 characters long. | -| `logstash_user_email` | `str` | `""` | — | Email address linked to the Logstash user. | -| `logstash_user_fullname` | `str` | `"Internal Logstash User"` | — | Full name linked to the Logstash user. | -| `logstash_logging_console` | `bool` | `true` | — | Log to the console (syslog when run via systemd). Only effective when logstash_manage_logging is enabled. | -| `logstash_logging_file` | `bool` | `true` | — | Log to the log file. Only effective when logstash_manage_logging is enabled. | -| `logstash_logging_slow_console` | `bool` | `true` | — | Log the slowlog to the console (syslog when run via systemd). Only effective when logstash_manage_logging is enabled. | -| `logstash_logging_slow_file` | `bool` | `true` | — | Log the slowlog to the log file. Only effective when logstash_manage_logging is enabled. | -| `logstash_logging_retention` | `str` | `"7D"` | — | Age at which rotated log files (plain, json, slowlog, deprecation and per-pipeline routing logs) are deleted by log4j2 at rollover time. Uses log4j2 IfLastModified duration syntax such as "7D", "24H" or "P30D". Only effective when logstash_manage_logging is enabled. | -| `logstash_ident` | `bool` | `true` | — | Add a field identifying the node that processed an event. | -| `logstash_ident_field_name` | `str` | `"[netways][instance]"` | — | Name of the field that identifies the instance. | -| `logstash_pipeline_identifier` | `bool` | `true` | — | Add a field identifying which pipeline processed an event. | -| `logstash_pipeline_identifier_field_name` | `str` | `"[netways][pipeline]"` | — | Name of the pipeline identifier field. | -| `logstash_pipeline_identifier_defaults` | `bool` | `false` | — | Also add pipeline identifiers in the default pipelines. This can let the defaults dominate statistics with little value. | -| `logstash_mermaid` | `bool` | `true` | — | Produce an overview of the Logstash pipelines in Mermaid syntax. | -| `logstash_mermaid_logstash` | `bool` | `true` | — | Place the Mermaid syntax into /etc/logstash/pipelines.mermaid on the Logstash hosts. | -| `logstash_mermaid_local` | `bool` | `false` | — | Place the Mermaid syntax into a temporary file on the control node. | -| `logstash_mermaid_extra` | `str` | N/A | — | Extra Mermaid syntax to append to the output (YAML multiline supported). Unset by default. | -| `logstash_freshstart` | `dict` | `{'changed': False}` | — | Internal state used by the role to detect a fresh install. Do not set manually. | +| Variable | Description | +|----------|-------------| +| `logstash_enable`
**Type**: `bool`
**Default**: `true` | Start and enable the Logstash service. | +| `logstash_config_backup`
**Type**: `bool`
**Default**: `false` | Keep backups of every configuration file the role changes. | +| `logstash_manage_yaml`
**Type**: `bool`
**Default**: `true` | Manage and overwrite logstash.yml. | +| `logstash_manage_logging`
**Type**: `bool`
**Default**: `false` | Manage the log4j2 logging configuration. | +| `logstash_heap`
**Type**: `str`
**Default**: `"1"` | JVM heap size in GB. Sets both -Xms and -Xmx via an LS_JAVA_OPTS systemd drop-in.

Elastic recommends 4-8 GB for typical ingestion and staying below 50-75% of physical memory. Keep it low when Logstash shares a host with Elasticsearch so they do not compete for memory. | +| `logstash_plugins`
**Type**: `list` of `str` | List of Logstash plugins to install. Unset by default. | +| `logstash_config_autoreload`
**Type**: `bool`
**Default**: `true` | Enable automatic reload of the Logstash configuration. | +| `logstash_config_autoreload_interval`
**Type**: `str` | Interval between configuration reload checks (e.g. "3s"). Only used when logstash_config_autoreload is enabled. Unset by default. | +| `logstash_config_path_data`
**Type**: `str`
**Default**: `"/var/lib/logstash"` | Logstash data directory (path.data). | +| `logstash_config_path_logs`
**Type**: `str`
**Default**: `"/var/log/logstash"` | Logstash log directory (path.logs). | +| `logstash_http_host`
**Type**: `str` | Bind address of the Logstash monitoring API (http.host). Unset by default. | +| `logstash_http_port`
**Type**: `str` | Port (or port range, e.g. "9600-9700") of the monitoring API (http.port). Unset by default. | +| `logstash_global_ecs`
**Type**: `str`
**Choices**: `disabled`, `v1` | Set the global ECS compatibility mode (pipeline.ecs_compatibility). Unset by default. | +| `logstash_pipeline_unsafe_shutdown`
**Type**: `bool` | Force Logstash to exit during shutdown even if there are in-flight events. Unset by default. | +| `logstash_legacy_monitoring`
**Type**: `bool`
**Default**: `true` | Enable legacy X-Pack monitoring. Ignored unless elasticstack_full_stack is set and only effective on Elastic Stack releases lower than 8. | +| `logstash_manage_pipelines`
**Type**: `bool`
**Default**: `true` | Manage pipelines.yml. | +| `logstash_no_pipelines`
**Type**: `bool`
**Default**: `false` | Disable all pipeline management entirely. | +| `logstash_pipelines`
**Type**: `list` of `dict` | List of pipelines to configure.

Each entry needs a name. A pipeline either points to an external git repository (source/version) or defines simple input/output keys that connect to Redis. See the pipelines documentation for details. | +|     ↳ `name`
**Type**: `str`
**Required**: Yes | Unique name of the pipeline. Becomes the pipeline directory and pipeline.id. | +|     ↳ `exclusive`
**Type**: `bool` | Mark this pipeline as the only one allowed to handle its events. | +|     ↳ `source`
**Type**: `str` | URL of a git repository holding the pipeline configuration. | +|     ↳ `version`
**Type**: `str` | Git branch/tag/commit to check out from source (default "main"). | +|     ↳ `queue_type`
**Type**: `str`
**Choices**: `memory`, `persisted` | Queue type for this pipeline (default "memory"). Use "persisted" for an on-disk persistent queue. | +|     ↳ `queue_max_bytes`
**Type**: `str` | Maximum queue size for this pipeline (default "1gb"). | +|     ↳ `input`
**Type**: `list` of `dict` | Simple Redis inputs for this pipeline. | +|         ↳ `name`
**Type**: `str` | Name of the input. | +|         ↳ `key`
**Type**: `str` | Redis key to read from. | +|     ↳ `output`
**Type**: `list` of `dict` | Simple Redis outputs for this pipeline. | +|         ↳ `name`
**Type**: `str` | Name of the output. | +|         ↳ `key`
**Type**: `str` | Redis key to write to. | +|         ↳ `condition`
**Type**: `str` | Optional Logstash conditional. The output only receives events matching it. With exclusive set, the conditions are chained with else if. | +|         ↳ `congestion`
**Type**: `int` | Optional congestion threshold. The output stops once the target Redis key holds more items than this value. | +| `logstash_elasticsearch_output`
**Type**: `bool`
**Default**: `true` | Create the default pipeline that forwards events to Elasticsearch. | +| `logstash_beats_input`
**Type**: `bool`
**Default**: `true` | Create the default pipeline with a Beats input. | +| `logstash_beats_input_congestion`
**Type**: `int` | Congestion threshold for the default Beats input pipeline: it stops writing to Redis once the buffer key holds more items than this value. Unset by default. | +| `logstash_beats_timeout`
**Type**: `str` | Timeout for idle client connections on the Beats input (e.g. "60s"). Unset by default. | +| `logstash_beats_tls`
**Type**: `bool` | Activate TLS on the Beats input pipeline. Unset by default, but enabled automatically in a full stack setup unless overridden. | +| `logstash_input_queue_type`
**Type**: `str`
**Default**: `"memory"`
**Choices**: `memory`, `persisted` | Queue type for the default Beats input pipeline. Use "persisted" for an on-disk persistent queue. | +| `logstash_input_queue_max_bytes`
**Type**: `str`
**Default**: `"1gb"` | Maximum queue size for the default Beats input pipeline. | +| `logstash_forwarder_queue_type`
**Type**: `str`
**Default**: `"memory"`
**Choices**: `memory`, `persisted` | Queue type for the default Elasticsearch forwarder pipeline. Use "persisted" for an on-disk persistent queue. | +| `logstash_forwarder_queue_max_bytes`
**Type**: `str`
**Default**: `"1gb"` | Maximum queue size for the default Elasticsearch forwarder pipeline. | +| `logstash_redis_password`
**Type**: `str` | Password used when the simple inputs/outputs connect to Redis. Unset by default. | +| `logstash_redis_input_host`
**Type**: `str`
**Default**: `"localhost"` | Redis host that simple pipeline inputs (and the default forwarder input) read from. | +| `logstash_redis_output_host`
**Type**: `str`
**Default**: `"localhost"` | Redis host that simple pipeline outputs (and the default input pipeline) write to. | +| `logstash_redis_tls`
**Type**: `bool`
**Default**: `false` | Enable TLS on the simple Redis inputs/outputs (`ssl`/`ssl_enabled`). | +| `logstash_redis_ssl_certificate`
**Type**: `str` | Path to the SSL certificate for the Redis outputs (the redis output plugin only; the input plugin has no such option). Rendered only when logstash_redis_tls is true. | +| `logstash_redis_ssl_key`
**Type**: `str` | Path to the SSL key for the Redis outputs. Rendered only when logstash_redis_tls is true. | +| `logstash_redis_ssl_key_passphrase`
**Type**: `str` | Passphrase for logstash_redis_ssl_key. Rendered only when logstash_redis_tls is true. | +| `logstash_redis_ssl_certificate_authorities`
**Type**: `list` of `str` | List of CA certificate paths used to verify the Redis server on the Redis outputs. Rendered only when logstash_redis_tls is true. | +| `logstash_redis_ssl_supported_protocols`
**Type**: `str` | TLS protocol version accepted on the Redis outputs (for example "TLSv1.3"). Rendered only when logstash_redis_tls is true. | +| `logstash_redis_ssl_verification_mode`
**Type**: `str` | Certificate verification mode for the Redis outputs ("full" or "none"). Rendered only when logstash_redis_tls is true. | +| `logstash_elasticsearch`
**Type**: `list` of `str` | Elasticsearch hosts for the default output. Defaults to the nodes from the elasticsearch group, or localhost when used standalone. | +| `logstash_validate_after_inactivity`
**Type**: `str`
**Default**: `"300"` | Seconds Logstash waits before validating a previously idle connection to Elasticsearch. | +| `logstash_sniffing`
**Type**: `bool`
**Default**: `false` | Enable sniffing for additional Elasticsearch nodes. | +| `logstash_sniffing_delay`
**Type**: `str` | Seconds to wait between sniffing attempts. Unset by default. | +| `logstash_sniffing_path`
**Type**: `str` | HTTP path used for the sniffing requests. Unset by default. | +| `logstash_security`
**Type**: `bool` | Enable X-Pack security (TLS) for Logstash. No default; enabled automatically in full stack mode with the elastic variant. | +| `logstash_tls_key_passphrase`
**Type**: `str`
**Default**: `"LogstashChangeMe"` | Passphrase for the generated Logstash certificates. | +| `logstash_certs_dir`
**Type**: `str`
**Default**: `"/etc/logstash/certs"` | Directory holding the Logstash certificates. Used to build several file paths. | +| `logstash_cert_validity_period`
**Type**: `int`
**Default**: `1095` | Number of days the generated certificates are valid. | +| `logstash_cert_expiration_buffer`
**Type**: `int`
**Default**: `30` | Renew the certificate when its remaining validity (in days) drops below this value. | +| `logstash_cert_will_expire_soon`
**Type**: `bool`
**Default**: `false` | Set to true to force renewal of the Logstash certificate. Alternatively run the playbook with the renew_logstash_cert tag. | +| `logstash_create_role`
**Type**: `bool`
**Default**: `true` | Create the Logstash writer role in Elasticsearch. | +| `logstash_role_name`
**Type**: `str`
**Default**: `"logstash_writer"` | Name of the Logstash writer role. | +| `logstash_role_cluster_privileges`
**Type**: `list` of `str`
**Default**: `['manage_index_templates', 'monitor', 'manage_ilm']` | Cluster privileges granted to the Logstash writer role. | +| `logstash_role_indicies_names`
**Type**: `list` of `str`
**Default**: `['ecs-logstash*', 'logstash*', 'logs*']` | Index patterns the Logstash writer role may access. | +| `logstash_role_indicies_privileges`
**Type**: `list` of `str`
**Default**: `['write', 'create', 'delete', 'create_index', 'manage', 'manage_ilm']` | Index privileges the Logstash writer role holds on its index patterns. | +| `logstash_create_user`
**Type**: `bool`
**Default**: `true` | Create the Logstash writer user in Elasticsearch. | +| `logstash_user_name`
**Type**: `str`
**Default**: `"logstash_writer"` | Name of the Logstash user connecting to Elasticsearch. | +| `logstash_user_password`
**Type**: `str`
**Default**: `"password"` | Password of the Logstash user. Must be at least 6 characters long. | +| `logstash_user_email`
**Type**: `str`
**Default**: `""` | Email address linked to the Logstash user. | +| `logstash_user_fullname`
**Type**: `str`
**Default**: `"Internal Logstash User"` | Full name linked to the Logstash user. | +| `logstash_logging_console`
**Type**: `bool`
**Default**: `true` | Log to the console (syslog when run via systemd). Only effective when logstash_manage_logging is enabled. | +| `logstash_logging_file`
**Type**: `bool`
**Default**: `true` | Log to the log file. Only effective when logstash_manage_logging is enabled. | +| `logstash_logging_slow_console`
**Type**: `bool`
**Default**: `true` | Log the slowlog to the console (syslog when run via systemd). Only effective when logstash_manage_logging is enabled. | +| `logstash_logging_slow_file`
**Type**: `bool`
**Default**: `true` | Log the slowlog to the log file. Only effective when logstash_manage_logging is enabled. | +| `logstash_logging_retention`
**Type**: `str`
**Default**: `"7D"` | Age at which rotated log files (plain, json, slowlog, deprecation and per-pipeline routing logs) are deleted by log4j2 at rollover time. Uses log4j2 IfLastModified duration syntax such as "7D", "24H" or "P30D". Only effective when logstash_manage_logging is enabled. | +| `logstash_ident`
**Type**: `bool`
**Default**: `true` | Add a field identifying the node that processed an event. | +| `logstash_ident_field_name`
**Type**: `str`
**Default**: `"[netways][instance]"` | Name of the field that identifies the instance. | +| `logstash_pipeline_identifier`
**Type**: `bool`
**Default**: `true` | Add a field identifying which pipeline processed an event. | +| `logstash_pipeline_identifier_field_name`
**Type**: `str`
**Default**: `"[netways][pipeline]"` | Name of the pipeline identifier field. | +| `logstash_pipeline_identifier_defaults`
**Type**: `bool`
**Default**: `false` | Also add pipeline identifiers in the default pipelines. This can let the defaults dominate statistics with little value. | +| `logstash_mermaid`
**Type**: `bool`
**Default**: `true` | Produce an overview of the Logstash pipelines in Mermaid syntax. | +| `logstash_mermaid_logstash`
**Type**: `bool`
**Default**: `true` | Place the Mermaid syntax into /etc/logstash/pipelines.mermaid on the Logstash hosts. | +| `logstash_mermaid_local`
**Type**: `bool`
**Default**: `false` | Place the Mermaid syntax into a temporary file on the control node. | +| `logstash_mermaid_extra`
**Type**: `str` | Extra Mermaid syntax to append to the output (YAML multiline supported). Unset by default. | +| `logstash_freshstart`
**Type**: `dict`
**Default**: `{'changed': False}` | Internal state used by the role to detect a fresh install. Do not set manually. | ## Shared variables -This role also uses the collection-wide `elasticstack_*` variables (e.g. -`elasticstack_full_stack`, `elasticstack_ca_host`, `elasticstack_ca_pass`, -`elasticstack_release`, `elasticstack_variant`). They are documented centrally -with the [elasticstack role](../elasticstack/README.md). +This role also uses the collection-wide `elasticstack_*` variables (e.g. `elasticstack_full_stack`, `elasticstack_ca_host`, `elasticstack_ca_pass`, `elasticstack_release`, `elasticstack_variant`). They are documented centrally with the [elasticstack role](../elasticstack/README.md). diff --git a/roles/logstash/docs/pipelines.md b/roles/logstash/docs/pipelines.md index 30ad89ba..a28f04eb 100644 --- a/roles/logstash/docs/pipelines.md +++ b/roles/logstash/docs/pipelines.md @@ -1,6 +1,6 @@ -# Pipelines # +# Pipelines -## Keeping an overview ## +## Keeping an overview It can be quite difficult to stay on top of your pipeline configuration because they tend to become very complex. @@ -8,7 +8,7 @@ This collection will leave some comments about how pipelines are interconnected If you set `logstash_mermaid` to `true` (which is the default), then you will also get a new file in `/etc/logstash/pipelines.mermaid`. You can paste it into a Mermaid editor in your documentation tool or in an [online Mermaid editor](https://mermaid.live/). The same content will be available on your control node in a temporary file. You can even add arbitrary code to reflect manually managed pipelines by using `logstash_mermaid_extra`. -## Git managed ## +## Git managed `logstash_pipelines` is a **list**. Each entry needs a `name`; everything else is optional. @@ -24,9 +24,9 @@ You can add a `version` attribute to your pipeline. It defaults to `main`. You c You can also determine where Logstash stores the data of each pipeline while processing it by using the `queue_type` attribute. It defaults to `memory`; use `persisted` to write the data temporarily to disk. You can limit the maximum amount of data per pipeline with `queue_max_bytes`, which defaults to `1gb`. -## Input and Output ## +## Input and Output -### Basic configuration ### +### Basic configuration To have a single Redis input and output to your files, use this. @@ -64,11 +64,11 @@ output { } ``` -### Multiple inputs ### +### Multiple inputs Just give more inputs with `name` and `key`. Every key will be read. -### More complex configuration ### +### More complex configuration If you want a bit more control over which outputs are used, the role offers more sophisticated configuration. @@ -207,24 +207,24 @@ else { Here the `default` output only receives the events that haven't already been sent to one of the others. -## Extra configuration ## +## Extra configuration -### Congestion threshold ### +### Congestion threshold Every output can have a `congestion` option with a numerical value. If the Redis key already holds more items than the value says, the output will stop. -### Unsafe shutdown ### +### Unsafe shutdown If you need unsafe Logstash shutdowns, e.g. for testing, you can set `logstash_pipeline_unsafe_shutdown` to `true`. The variable doesn't have a default so Logstash falls back to its internal default of `false`. -## Caveats ## +## Caveats There are still some minor issues you need to keep in mind: * The default output in an `exclusive: true` setup must be the last in the YAML configuration. There's no sorting, the role simply expects the default to be the last one. * The configuration *should* work but will make no sense if you have `exclusive: true` but two or more outputs without `condition`. -## Custom pipelines ## +## Custom pipelines If you have other ways of putting pipeline code into the correct directories, you can just skip the `source` option. diff --git a/roles/repos/README.md b/roles/repos/README.md index 7080bf68..c0ca4562 100644 --- a/roles/repos/README.md +++ b/roles/repos/README.md @@ -1,16 +1,12 @@ # Ansible Role: Repos -Adds the Elastic package repositories to the target host so the other roles can -install Elastic Stack packages. It supports apt (Debian/Ubuntu), yum (RedHat) and -zypper (SUSE), for both the standard Elastic and the OSS variant. +Adds the Elastic package repositories to the target host so the other roles can install Elastic Stack packages. It supports apt (Debian/Ubuntu), yum (RedHat) and zypper (SUSE), for both the standard Elastic and the OSS variant. -Its main use is together with the other roles of this collection, which expect -the Elastic repositories to be present. +Its main use is together with the other roles of this collection, which expect the Elastic repositories to be present. ## Requirements -* The `community.general` collection — only needed on SUSE hosts, where the role - uses the `zypper` modules. +* The `community.general` collection — only needed on SUSE hosts, where the role uses the `zypper` modules. * Network access from the target host to the Elastic repository URL. The role installs the required signing tools (`gpg`/`gnupg`) itself. @@ -35,8 +31,4 @@ No variables are defined for this role. ## Shared variables -This role is configured through the collection-wide `elasticstack_*` variables — -mainly `elasticstack_release`, `elasticstack_variant`, `elasticstack_enable_repos`, -`elasticstack_repo_url`, `elasticstack_repo_key` and `elasticstack_rpm_workaround`. -They are documented centrally with the -[elasticstack role](../elasticstack/README.md). +This role is configured through the collection-wide `elasticstack_*` variables — mainly `elasticstack_release`, `elasticstack_variant`, `elasticstack_enable_repos`, `elasticstack_repo_url`, `elasticstack_repo_key` and `elasticstack_rpm_workaround`. They are documented centrally with the [elasticstack role](../elasticstack/README.md).