From 1b7933fbf4fdc34e387b1fb1f87873c5547a3bed Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Thu, 22 Oct 2026 14:59:54 +0200 Subject: [PATCH 1/2] Install the elasticsearch library into its own venv on the Logstash host --- README.md | 5 +- .../fragments/elasticsearch_module_client.yml | 4 -- changelogs/fragments/python_library.yml | 16 ++++++ molecule/elasticstack_default/prepare.yml | 40 +++++++++++++++ plugins/module_utils/api.py | 4 +- requirements-test.txt | 1 - roles/elasticsearch/README.md | 2 +- roles/elasticstack/README.md | 4 +- roles/elasticstack/defaults/main.yml | 2 - roles/elasticstack/meta/argument_specs.yml | 13 ----- roles/elasticstack/tasks/packages.yml | 49 ------------------- roles/logstash/README.md | 1 + roles/logstash/tasks/logstash-security.yml | 21 ++++++++ roles/logstash/vars/main.yml | 3 ++ 14 files changed, 89 insertions(+), 76 deletions(-) create mode 100644 changelogs/fragments/python_library.yml create mode 100644 roles/logstash/vars/main.yml diff --git a/README.md b/README.md index 9f245f39..f3216680 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,10 @@ Ansible collections, both are declared as dependencies and installed for you whe Python libraries on the control node: * `passlib`, unless you disable password hashing for the Logstash user and you want to use the logstash role. Install it with `pip`. -* `elasticsearch`. Current versions are compatible either with Elasticsearch 9 or with versions lower than 9, there seems to be no version that serves both. For now we install a client older than 9, and this needs revisiting when the collection becomes compatible with Elastic Stack 9. + +Python libraries on the target hosts: + +* `elasticsearch`, version 8, for the `elasticsearch_role` and `elasticsearch_user` modules. Version 8 works with Elasticsearch 8 and 9. The logstash role installs it into its own virtual environment on the Logstash host, `/opt/netways-elasticstack/venv`, so that host needs access to PyPI or to a pip mirror, for example set in `/etc/pip.conf`. If you call the modules in your own playbooks, install version 8 on the host they run on. Ansible roles: diff --git a/changelogs/fragments/elasticsearch_module_client.yml b/changelogs/fragments/elasticsearch_module_client.yml index 8e775bcc..7baae2ac 100644 --- a/changelogs/fragments/elasticsearch_module_client.yml +++ b/changelogs/fragments/elasticsearch_module_client.yml @@ -3,10 +3,6 @@ bugfixes: when the ``elasticsearch`` Python library on the target host is version 7. That version accepts the credentials without sending them, so the modules failed with an unexplained HTTP 401 instead. - - The ``elasticstack`` role only falls back to the ``python3-elasticsearch`` distribution - package when the pip install before it actually failed. The condition it used could never - be true, so the package was installed on every host, and on EL 9 and Ubuntu 22.04 that - package is still version 7 of the library. - The ``elasticsearch_role`` and ``elasticsearch_user`` modules no longer build an SSL context with both certificate checks switched off. They hand the TLS settings to the client directly, and only for an ``https`` host, so ``ca_certs`` reaches the connection and a plain diff --git a/changelogs/fragments/python_library.yml b/changelogs/fragments/python_library.yml new file mode 100644 index 00000000..014bf384 --- /dev/null +++ b/changelogs/fragments/python_library.yml @@ -0,0 +1,16 @@ +breaking_changes: + - The ``elasticstack`` role no longer installs the ``elasticsearch`` Python library on every host, + and the variables ``elasticstack_manage_pip`` and ``elasticstack_force_pip`` are removed. Setting + them has no effect anymore, so remove them from your inventory. + +bugfixes: + - The ``logstash`` role installs version 8 of the ``elasticsearch`` Python library into its own + virtual environment, ``/opt/netways-elasticstack/venv``, on the Logstash host and runs the + ``elasticsearch_role`` and ``elasticsearch_user`` modules with it. On fresh Ubuntu 22.04 and + EL 9 hosts the library was missing or version 7 before, and creating the Logstash writer role + failed. The Logstash host needs access to PyPI or to a pip mirror. + +trivial: + - The full stack Molecule scenario removes pip, packaging and setuptools from its containers, so + it runs like a fresh host. ``requirements-test.txt`` no longer lists ``elasticsearch``, the + modules run in the containers and not on the runner. diff --git a/molecule/elasticstack_default/prepare.yml b/molecule/elasticstack_default/prepare.yml index 611cf912..3745f299 100644 --- a/molecule/elasticstack_default/prepare.yml +++ b/molecule/elasticstack_default/prepare.yml @@ -59,3 +59,43 @@ state: latest update_cache: yes when: ansible_os_family == "Debian" + + # A fresh host has no pip, packaging or setuptools, but the images bring them along. Remove + # them, so the roles have to install what they need themselves. + - name: Remove pip, packaging and setuptools that were installed with pip + ansible.builtin.command: "python3 -m pip uninstall -y {{ item }}" + register: prepare_pip_uninstall + changed_when: "'Successfully uninstalled' in prepare_pip_uninstall.stdout" + failed_when: false + # One at a time, pip stops at the first package it cannot remove. pip itself goes last. + loop: + - packaging + - setuptools + - pip + + - name: Remove pip, packaging and setuptools that came as distribution packages + ansible.builtin.package: + name: "{{ prepare_python_packages[ansible_os_family] }}" + state: absent + vars: + prepare_python_packages: + Debian: + - python3-pip + - python3-setuptools + - python3-pkg-resources + - python3-packaging + - python3-wheel + RedHat: + - python3-pip + - python3-setuptools + - python3-packaging + + - name: Check that pip, packaging and setuptools are gone + ansible.builtin.command: "python3 -c 'import {{ item }}'" + register: prepare_python_import + changed_when: false + failed_when: prepare_python_import.rc == 0 + loop: + - pip + - packaging + - pkg_resources diff --git a/plugins/module_utils/api.py b/plugins/module_utils/api.py index e896d612..625d00ff 100644 --- a/plugins/module_utils/api.py +++ b/plugins/module_utils/api.py @@ -37,8 +37,8 @@ def unsupported_elasticsearch_message(): "without sending the credentials, so every request reaches Elasticsearch " "unauthenticated and is answered with HTTP 401. The distribution package " "python3-elasticsearch is still version 7 on EL 9 and Ubuntu 22.04. Install " - "the library with pip instead, for example by setting elasticstack_force_pip " - "to true." + "version 8 with pip instead, for example into a virtual environment, and run " + "the module with its Python interpreter." % ( ".".join(str(part) for part in ELASTICSEARCH_VERSION), ".".join(str(part) for part in ELASTICSEARCH_MIN_VERSION), diff --git a/requirements-test.txt b/requirements-test.txt index b0d88da3..3b8dba26 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -3,6 +3,5 @@ molecule molecule-plugins[docker] pytest passlib -elasticsearch<9 docker>=7.1.0 requests>=2.32.2 diff --git a/roles/elasticsearch/README.md b/roles/elasticsearch/README.md index 69d666b6..5061e1c3 100644 --- a/roles/elasticsearch/README.md +++ b/roles/elasticsearch/README.md @@ -7,7 +7,7 @@ The OSS variant is only available up to Elastic Stack 7; from release 8 on only ## Requirements * The Elastic repositories configured: use the [`repos`](../repos/README.md) role or provide them yourself. -* For the default security setup the role generates certificates with the bundled `elasticsearch-certutil` and `openssl`. The required Python libraries (`cryptography`, `elasticsearch`) are installed for you by the shared `elasticstack` role. +* For the default security setup the role generates certificates with the bundled `elasticsearch-certutil` and `openssl`. The required Python library `cryptography` is installed for you by the shared `elasticstack` role. ## Example diff --git a/roles/elasticstack/README.md b/roles/elasticstack/README.md index f28afc69..33158106 100644 --- a/roles/elasticstack/README.md +++ b/roles/elasticstack/README.md @@ -6,7 +6,7 @@ Set these variables once (in `group_vars`, the play, or via `-e`) and every role ## Requirements -It installs the shared Python libraries (`cryptography` and the `elasticsearch` client) and the packages (`openssl`, `unzip`) that the collection needs. +It installs the Python library `cryptography` and the packages `openssl` and `unzip` that the collection needs. ## Example @@ -55,8 +55,6 @@ The role runs automatically the first time any component role is applied to a ho | `elasticstack_cert_pass`
**Type**: `str` | Common passphrase for all component certificates. Unset by default; when set, it overrides each role's individual certificate passphrase. | | `elasticstack_initial_passwords`
**Type**: `str`
**Default**: `"/usr/share/elasticsearch/initial_passwords"` | Path on the CA host where the generated initial passwords are stored. | | `elasticstack_no_log`
**Type**: `bool`
**Default**: `true` | Hide the output of tasks that could reveal passwords. Set to false for debugging. | -| `elasticstack_manage_pip`
**Type**: `bool`
**Default**: `false` | Install pip on the target host. | -| `elasticstack_force_pip`
**Type**: `bool`
**Default**: `false` | Force installation of the required Python modules via pip (useful when the distribution packages are too old). See PEP 668. | | `elasticstack_ci`
**Type**: `bool`
**Default**: `false` | Internal test flag. Set to true only by the collection's molecule scenarios to enable resource workarounds needed on constrained test runners (cache cleanup, relaxed Elasticsearch disk watermarks). Never set this on a real installation. | diff --git a/roles/elasticstack/defaults/main.yml b/roles/elasticstack/defaults/main.yml index 6397562c..f250e557 100644 --- a/roles/elasticstack/defaults/main.yml +++ b/roles/elasticstack/defaults/main.yml @@ -26,8 +26,6 @@ elasticstack_repo_key: https://artifacts.elastic.co/GPG-KEY-elasticsearch elasticstack_rpm_workaround: false elasticstack_security: true elasticstack_variant: elastic -elasticstack_force_pip: false -elasticstack_manage_pip: false # for debugging only elasticstack_no_log: true diff --git a/roles/elasticstack/meta/argument_specs.yml b/roles/elasticstack/meta/argument_specs.yml index e29e8390..5e69aabd 100644 --- a/roles/elasticstack/meta/argument_specs.yml +++ b/roles/elasticstack/meta/argument_specs.yml @@ -222,19 +222,6 @@ argument_specs: default: true description: Hide the output of tasks that could reveal passwords. Set to false for debugging. - # ----- Python dependencies ----- - elasticstack_manage_pip: - type: bool - default: false - description: Install pip on the target host. - - elasticstack_force_pip: - type: bool - default: false - description: >- - Force installation of the required Python modules via pip (useful when - the distribution packages are too old). See PEP 668. - # ----- Testing ----- elasticstack_ci: type: bool diff --git a/roles/elasticstack/tasks/packages.yml b/roles/elasticstack/tasks/packages.yml index 5a965f54..149a4797 100644 --- a/roles/elasticstack/tasks/packages.yml +++ b/roles/elasticstack/tasks/packages.yml @@ -19,52 +19,3 @@ - renew_beats_cert - renew_es_cert - renew_logstash_cert - -# We don't know how the user made `pip` available and we don't want -# to mess with the installation. -# But we need a way to install our dependencies - -- name: Install packages for module dependencies - ansible.builtin.package: - name: - - python3-pip - when: - - elasticstack_manage_pip | bool - -# The following block will try to install modules via pip -# If that fails it will try to install them via package manager -# Packets are sometimes too old as a dependency, so pip is -# preferred -# If that's the case the next task will allow breaking -# systems Python and just install it via pip no matter what - -- name: Install Python Modules as dependencies - when: - - not elasticstack_force_pip | bool - block: - - name: Install Python Modules via pip - ansible.builtin.pip: - name: - - 'elasticsearch<9' - - cryptography - ignore_errors: true - register: elasticstack_pip_installation - - - name: Install Python Module via package manager - ansible.builtin.package: - name: - - python3-elasticsearch - - python3-cryptography - when: - - elasticstack_pip_installation is failed - -- name: Install Python Modules via pip - forced - ansible.builtin.pip: - name: - - 'elasticsearch<9' - - cryptography - break_system_packages: true - ignore_errors: true - when: - - elasticstack_force_pip | bool - register: elasticstack_pip_installation_forced diff --git a/roles/logstash/README.md b/roles/logstash/README.md index 822a838a..b4c2fe0d 100644 --- a/roles/logstash/README.md +++ b/roles/logstash/README.md @@ -8,6 +8,7 @@ It works with the standard Elastic Stack packages and with Elastic's OSS variant * The `community.general` collection. * `passlib` (Python, on the controller) unless you disable password hashing for the Logstash user. +* Access to PyPI, or to a pip mirror set for example in `/etc/pip.conf`, if the role creates the Logstash writer role or user in Elasticsearch (`logstash_create_role`, `logstash_create_user`, full stack with security only). It installs version 8 of the `elasticsearch` Python library for that into `/opt/netways-elasticstack/venv`. * `curl` on the target host. * `git` on the target host if you use git-based pipelines. * Redis if you use the default pipeline or other pipelines that talk to Redis (e.g. via the [`geerlingguy.redis`](https://galaxy.ansible.com/geerlingguy/redis) role). diff --git a/roles/logstash/tasks/logstash-security.yml b/roles/logstash/tasks/logstash-security.yml index d3c17fe0..d0780eec 100644 --- a/roles/logstash/tasks/logstash-security.yml +++ b/roles/logstash/tasks/logstash-security.yml @@ -398,6 +398,23 @@ - configuration - logstash_configuration +- name: Install the elasticsearch Python library for the two tasks below + when: logstash_create_role | bool or logstash_create_user | bool + block: + # The pip module needs packaging in the system Python, and python3 -m venv needs + # python3-venv on Debian and Ubuntu + - name: Install the packages its virtual environment needs + ansible.builtin.package: + name: "{{ ['python3-packaging'] + (['python3-venv'] if ansible_os_family == 'Debian' else []) }}" + + # The two modules need version 8 of the library. The distribution packages are version 7 + # on Ubuntu 22.04 and EL 9 and version 9 on newer releases. + - name: Install the elasticsearch Python library into its own virtual environment + ansible.builtin.pip: + name: "elasticsearch>=8,<9" + virtualenv: "{{ _logstash_python_venv_dir }}" + virtualenv_command: "{{ ansible_python.executable }} -m venv" + - name: Create logstash role {{ logstash_role_name }} netways.elasticstack.elasticsearch_role: name: "{{ logstash_role_name }}" @@ -411,6 +428,8 @@ auth_pass: "{{ logstash_elasticstack_password.stdout }}" verify_certs: true ca_certs: "{{ logstash_certs_dir }}/ca.crt" + vars: + ansible_python_interpreter: "{{ _logstash_python_venv_dir }}/bin/python" when: logstash_create_role | bool - name: Create logstash user {{ logstash_user_name }} @@ -428,4 +447,6 @@ auth_pass: "{{ logstash_elasticstack_password.stdout }}" verify_certs: true ca_certs: "{{ logstash_certs_dir }}/ca.crt" + vars: + ansible_python_interpreter: "{{ _logstash_python_venv_dir }}/bin/python" when: logstash_create_user | bool diff --git a/roles/logstash/vars/main.yml b/roles/logstash/vars/main.yml new file mode 100644 index 00000000..f96dcf4a --- /dev/null +++ b/roles/logstash/vars/main.yml @@ -0,0 +1,3 @@ +--- +# Virtual environment with the elasticsearch Python library for elasticsearch_role and elasticsearch_user +_logstash_python_venv_dir: /opt/netways-elasticstack/venv From 882ba9d9e54630ad1dc6ac9ff12c407b0514a739 Mon Sep 17 00:00:00 2001 From: Afeef Ghannam Date: Thu, 22 Oct 2026 15:07:07 +0200 Subject: [PATCH 2/2] Leave the changelog fragment of #556 unchanged --- changelogs/fragments/elasticsearch_module_client.yml | 4 ++++ changelogs/fragments/python_library.yml | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/changelogs/fragments/elasticsearch_module_client.yml b/changelogs/fragments/elasticsearch_module_client.yml index 7baae2ac..8e775bcc 100644 --- a/changelogs/fragments/elasticsearch_module_client.yml +++ b/changelogs/fragments/elasticsearch_module_client.yml @@ -3,6 +3,10 @@ bugfixes: when the ``elasticsearch`` Python library on the target host is version 7. That version accepts the credentials without sending them, so the modules failed with an unexplained HTTP 401 instead. + - The ``elasticstack`` role only falls back to the ``python3-elasticsearch`` distribution + package when the pip install before it actually failed. The condition it used could never + be true, so the package was installed on every host, and on EL 9 and Ubuntu 22.04 that + package is still version 7 of the library. - The ``elasticsearch_role`` and ``elasticsearch_user`` modules no longer build an SSL context with both certificate checks switched off. They hand the TLS settings to the client directly, and only for an ``https`` host, so ``ca_certs`` reaches the connection and a plain diff --git a/changelogs/fragments/python_library.yml b/changelogs/fragments/python_library.yml index 014bf384..87936c77 100644 --- a/changelogs/fragments/python_library.yml +++ b/changelogs/fragments/python_library.yml @@ -1,7 +1,8 @@ breaking_changes: - The ``elasticstack`` role no longer installs the ``elasticsearch`` Python library on every host, - and the variables ``elasticstack_manage_pip`` and ``elasticstack_force_pip`` are removed. Setting - them has no effect anymore, so remove them from your inventory. + neither with pip nor from the ``python3-elasticsearch`` distribution package, and the variables + ``elasticstack_manage_pip`` and ``elasticstack_force_pip`` are removed. Setting them has no + effect anymore, so remove them from your inventory. bugfixes: - The ``logstash`` role installs version 8 of the ``elasticsearch`` Python library into its own