diff --git a/changelogs/fragments/elasticsearch_module_client.yml b/changelogs/fragments/elasticsearch_module_client.yml new file mode 100644 index 00000000..8e775bcc --- /dev/null +++ b/changelogs/fragments/elasticsearch_module_client.yml @@ -0,0 +1,16 @@ +bugfixes: + - The ``elasticsearch_role`` and ``elasticsearch_user`` modules stop with a readable message + when the ``elasticsearch`` Python library on the target host is version 7. That version + accepts the credentials without sending them, so the modules failed with an unexplained + HTTP 401 instead. + - The ``elasticstack`` role only falls back to the ``python3-elasticsearch`` distribution + package when the pip install before it actually failed. The condition it used could never + be true, so the package was installed on every host, and on EL 9 and Ubuntu 22.04 that + package is still version 7 of the library. + - The ``elasticsearch_role`` and ``elasticsearch_user`` modules no longer build an SSL context + with both certificate checks switched off. They hand the TLS settings to the client + directly, and only for an ``https`` host, so ``ca_certs`` reaches the connection and a plain + ``http`` host no longer fails while the client is being built. + - The ``logstash`` role creates the logstash user over a verified connection, like the + logstash role it creates right before it. Both talk to the same host, but the user was + created with ``verify_certs`` set to false. diff --git a/plugins/module_utils/README.md b/plugins/module_utils/README.md index f9c88ddd..b65c54c5 100644 --- a/plugins/module_utils/README.md +++ b/plugins/module_utils/README.md @@ -50,12 +50,16 @@ A broken or unreadable extension does not fail the module. It is skipped with `m ## `api` -Builds the Elasticsearch client for the two Elasticsearch modules. Requires the `elasticsearch` Python library, currently a version below 9. +Builds the Elasticsearch client for the two Elasticsearch modules. Requires the `elasticsearch` Python library, version 8. Version 9 is untested and version 7 does not work: it accepts the `basic_auth` argument without sending the credentials, so every request arrives unauthenticated, and it returns plain dictionaries instead of the response objects whose `.raw` attribute the two objects below read. ### `HAS_ELASTICSEARCH` and `ELASTICSEARCH_IMPORT_ERROR` Set while the file is imported. `HAS_ELASTICSEARCH` is a __bool__, `ELASTICSEARCH_IMPORT_ERROR` holds the formatted traceback as a __string__ or `None`. A module checks the flag and reports the traceback, so a missing library produces a readable message instead of an import error. +### `HAS_SUPPORTED_ELASTICSEARCH` and `unsupported_elasticsearch_message()` + +The same pattern for the version. `HAS_SUPPORTED_ELASTICSEARCH` is a __bool__, true when the installed library is at least `ELASTICSEARCH_MIN_VERSION`, and false when the library is missing entirely. `unsupported_elasticsearch_message()` returns the failure text as a __string__, naming the version that is installed and how to get a newer one. A module checks the flag right after `HAS_ELASTICSEARCH` and fails with that message, so an old library is reported as such instead of as an HTTP 401. + ### `Api.new_client_basic_auth()` static method Returns a connected `Elasticsearch` client authenticated with username and password. @@ -64,7 +68,7 @@ Returns a connected `Elasticsearch` client authenticated with username and passw **Return:** An `Elasticsearch` __object__. Raises `ImportError` when the library is missing. -The SSL context it builds sets `check_hostname` and `verify_mode` to false before `verify_certs` is handed to the client, so the hostname in the certificate is not checked. +The TLS arguments are only passed for an `https://` host, because the client rejects any TLS option on a plain `http://` host with a `ValueError` while it is being constructed. `ca_certs` is passed only when `verify_certs` is true, since it has no meaning without verification. With verification on, the client checks the certificate chain and the hostname. ## `elasticsearch_role` diff --git a/plugins/module_utils/api.py b/plugins/module_utils/api.py index 0871985d..e896d612 100644 --- a/plugins/module_utils/api.py +++ b/plugins/module_utils/api.py @@ -5,20 +5,47 @@ # https://www.gnu.org/licenses/gpl-3.0.txt) import traceback -import ssl from ansible.module_utils.basic import missing_required_lib +# The modules need the 8.x client. Version 7 accepts basic_auth without sending +# the credentials, and the response objects that the module_utils read through +# .raw only exist from 8.0 on. +ELASTICSEARCH_MIN_VERSION = (8, 0, 0) + try: - from elasticsearch import Elasticsearch + from elasticsearch import Elasticsearch, __version__ as ELASTICSEARCH_VERSION except ImportError: HAS_ELASTICSEARCH = False + HAS_SUPPORTED_ELASTICSEARCH = False ELASTICSEARCH_IMPORT_ERROR = traceback.format_exc() + ELASTICSEARCH_VERSION = None else: HAS_ELASTICSEARCH = True + HAS_SUPPORTED_ELASTICSEARCH = ELASTICSEARCH_VERSION >= ELASTICSEARCH_MIN_VERSION ELASTICSEARCH_IMPORT_ERROR = None +def unsupported_elasticsearch_message(): + """Message for a client that is too old, for the modules to fail with.""" + if ELASTICSEARCH_VERSION is None: + return missing_required_lib('elasticsearch') + + return ( + "The elasticsearch Python library on the target host is version %s, " + "this module needs %s or later. Version 7 accepts the basic_auth argument " + "without sending the credentials, so every request reaches Elasticsearch " + "unauthenticated and is answered with HTTP 401. The distribution package " + "python3-elasticsearch is still version 7 on EL 9 and Ubuntu 22.04. Install " + "the library with pip instead, for example by setting elasticstack_force_pip " + "to true." + % ( + ".".join(str(part) for part in ELASTICSEARCH_VERSION), + ".".join(str(part) for part in ELASTICSEARCH_MIN_VERSION), + ) + ) + + class Api(): @staticmethod @@ -26,13 +53,16 @@ def new_client_basic_auth(host, auth_user, auth_pass, ca_certs, verify_certs): if not HAS_ELASTICSEARCH: raise ImportError(missing_required_lib('elasticsearch')) - ctx = ssl.create_default_context(cafile=ca_certs) - ctx.check_hostname = False - ctx.verify_mode = False + # TLS options are only accepted for an https host, NodeConfig raises + # ValueError as soon as one of them is set on a plain http host. + kwargs = {} + if host.startswith('https://'): + kwargs['verify_certs'] = verify_certs + if verify_certs and ca_certs: + kwargs['ca_certs'] = ca_certs return Elasticsearch( hosts=[host], basic_auth=(auth_user, auth_pass), - ssl_context=ctx, - verify_certs=verify_certs + **kwargs ) diff --git a/plugins/modules/elasticsearch_role.py b/plugins/modules/elasticsearch_role.py index d8916215..8f0efcf3 100644 --- a/plugins/modules/elasticsearch_role.py +++ b/plugins/modules/elasticsearch_role.py @@ -18,7 +18,15 @@ author: - Tobias Bauriedel (@tbauriedel) requirements: - - elasticsearch < 9 + - elasticsearch >= 8, < 9 +notes: + - Version 7 of the C(elasticsearch) library accepts the credentials without sending them, which + Elasticsearch answers with HTTP 401. The module refuses to run on it and says so. The + distribution package C(python3-elasticsearch) is still version 7 on EL 9 and Ubuntu 22.04, + where the library has to come from pip instead. + - With O(verify_certs=true) the certificate chain and the hostname are both checked. O(ca_certs) + is added to the trust store the library already carries, it does not replace it, so a + certificate from a publicly trusted CA is accepted as well. options: name: description: Name of the Elasticsearch role. @@ -104,7 +112,8 @@ from ansible.module_utils.basic import AnsibleModule, missing_required_lib from ansible_collections.netways.elasticstack.plugins.module_utils.api import ( - HAS_ELASTICSEARCH, ELASTICSEARCH_IMPORT_ERROR + HAS_ELASTICSEARCH, ELASTICSEARCH_IMPORT_ERROR, + HAS_SUPPORTED_ELASTICSEARCH, unsupported_elasticsearch_message ) from ansible_collections.netways.elasticstack.plugins.module_utils.elasticsearch_role import ( Role @@ -146,6 +155,9 @@ def run_module(): exception=ELASTICSEARCH_IMPORT_ERROR ) + if not HAS_SUPPORTED_ELASTICSEARCH: + module.fail_json(msg=unsupported_elasticsearch_message()) + role = Role( result=result, role_name=module.params['name'], diff --git a/plugins/modules/elasticsearch_user.py b/plugins/modules/elasticsearch_user.py index 50e4bdbb..e682cc3d 100644 --- a/plugins/modules/elasticsearch_user.py +++ b/plugins/modules/elasticsearch_user.py @@ -18,7 +18,15 @@ author: - Tobias Bauriedel (@tbauriedel) requirements: - - elasticsearch < 9 + - elasticsearch >= 8, < 9 +notes: + - Version 7 of the C(elasticsearch) library accepts the credentials without sending them, which + Elasticsearch answers with HTTP 401. The module refuses to run on it and says so. The + distribution package C(python3-elasticsearch) is still version 7 on EL 9 and Ubuntu 22.04, + where the library has to come from pip instead. + - With O(verify_certs=true) the certificate chain and the hostname are both checked. O(ca_certs) + is added to the trust store the library already carries, it does not replace it, so a + certificate from a publicly trusted CA is accepted as well. options: name: description: Username of the Elasticsearch user. @@ -116,7 +124,8 @@ from ansible.module_utils.basic import AnsibleModule, missing_required_lib from ansible_collections.netways.elasticstack.plugins.module_utils.api import ( - HAS_ELASTICSEARCH, ELASTICSEARCH_IMPORT_ERROR + HAS_ELASTICSEARCH, ELASTICSEARCH_IMPORT_ERROR, + HAS_SUPPORTED_ELASTICSEARCH, unsupported_elasticsearch_message ) from ansible_collections.netways.elasticstack.plugins.module_utils.elasticsearch_user import ( User @@ -159,6 +168,9 @@ def run_module(): exception=ELASTICSEARCH_IMPORT_ERROR ) + if not HAS_SUPPORTED_ELASTICSEARCH: + module.fail_json(msg=unsupported_elasticsearch_message()) + user = User( result=result, user_name=module.params['name'], diff --git a/roles/elasticstack/tasks/packages.yml b/roles/elasticstack/tasks/packages.yml index 2aa122e7..5a965f54 100644 --- a/roles/elasticstack/tasks/packages.yml +++ b/roles/elasticstack/tasks/packages.yml @@ -56,7 +56,7 @@ - python3-elasticsearch - python3-cryptography when: - - not elasticstack_pip_installation | bool + - elasticstack_pip_installation is failed - name: Install Python Modules via pip - forced ansible.builtin.pip: diff --git a/roles/logstash/tasks/logstash-security.yml b/roles/logstash/tasks/logstash-security.yml index c7bdde8c..31a42f4c 100644 --- a/roles/logstash/tasks/logstash-security.yml +++ b/roles/logstash/tasks/logstash-security.yml @@ -419,6 +419,6 @@ host: "https://{{ hostvars[elasticstack_ca_host].ansible_default_ipv4.address }}:{{ elasticstack_elasticsearch_http_port }}" auth_user: elastic auth_pass: "{{ logstash_elasticstack_password.stdout }}" - verify_certs: false + verify_certs: true ca_certs: "{{ logstash_certs_dir }}/ca.crt" when: logstash_create_user | bool