From 685afcfc98c4e18ad2376791a039cde758a17e07 Mon Sep 17 00:00:00 2001 From: vladimirhasko Date: Mon, 17 Aug 2026 06:33:53 +0000 Subject: [PATCH 01/16] switching from docker.io docker-ce --- .../roles/apimon_executor/tasks/main.yml | 91 +++++++++++-- .../roles/apimon_executor/vars/Debian.yaml | 3 +- .../roles/apimon_executor/vars/Ubuntu.yaml | 17 +++ .../roles/apimon_scheduler/tasks/main.yml | 89 +++++++++++-- .../roles/apimon_scheduler/vars/Debian.yaml | 4 +- .../roles/apimon_scheduler/vars/Ubuntu.yaml | 17 +++ .../repos/files/sources.list.jammy.x86_64 | 5 + .../repos/files/sources.list.noble.x86_64 | 5 + .../roles/base/repos/tasks/Debian.yaml | 19 +++ .../project/roles/carbonapi/tasks/main.yaml | 95 +++++++++++--- .../project/roles/carbonapi/vars/Debian.yaml | 2 + .../project/roles/carbonapi/vars/Ubuntu.yaml | 17 +++ .../project/roles/epmon/tasks/main.yml | 88 +++++++++++-- .../project/roles/epmon/vars/Debian.yaml | 3 +- .../project/roles/epmon/vars/Ubuntu.yaml | 17 +++ .../project/roles/etcd/tasks/main.yaml | 83 +++++++++++- .../project/roles/etcd/vars/Debian.yaml | 3 +- .../project/roles/etcd/vars/Ubuntu.yaml | 17 +++ .../project/roles/globalmon/tasks/main.yml | 124 ++++++++++++------ .../project/roles/globalmon/vars/Debian.yaml | 3 +- .../project/roles/globalmon/vars/Ubuntu.yaml | 17 +++ .../project/roles/grafana/tasks/main.yml | 107 +++++++++++---- .../project/roles/grafana/vars/Debian.yaml | 3 +- .../project/roles/grafana/vars/Ubuntu.yaml | 17 +++ .../project/roles/graphite/tasks/main.yaml | 89 +++++++++++-- .../project/roles/graphite/vars/Debian.yaml | 4 +- .../project/roles/graphite/vars/Ubuntu.yaml | 17 +++ .../project/roles/postgresql/tasks/main.yaml | 91 +++++++++++-- .../project/roles/postgresql/vars/Debian.yaml | 5 +- .../project/roles/postgresql/vars/Ubuntu.yaml | 19 +++ .../roles/postgresql_ha/tasks/main.yaml | 83 +++++++++++- .../roles/postgresql_ha/vars/Debian.yaml | 2 + .../roles/postgresql_ha/vars/Ubuntu.yaml | 17 +++ .../project/roles/statsd/tasks/main.yaml | 81 +++++++++++- .../project/roles/statsd/vars/Debian.yaml | 7 + .../project/roles/statsd/vars/RedHat.yaml | 4 + .../project/roles/statsd/vars/Ubuntu.yaml | 17 +++ 37 files changed, 1134 insertions(+), 148 deletions(-) create mode 100644 cloudmon/ansible/project/roles/apimon_executor/vars/Ubuntu.yaml create mode 100644 cloudmon/ansible/project/roles/apimon_scheduler/vars/Ubuntu.yaml create mode 100644 cloudmon/ansible/project/roles/base/repos/files/sources.list.jammy.x86_64 create mode 100644 cloudmon/ansible/project/roles/base/repos/files/sources.list.noble.x86_64 create mode 100644 cloudmon/ansible/project/roles/base/repos/tasks/Debian.yaml create mode 100644 cloudmon/ansible/project/roles/carbonapi/vars/Ubuntu.yaml create mode 100644 cloudmon/ansible/project/roles/epmon/vars/Ubuntu.yaml create mode 100644 cloudmon/ansible/project/roles/etcd/vars/Ubuntu.yaml create mode 100644 cloudmon/ansible/project/roles/globalmon/vars/Ubuntu.yaml create mode 100644 cloudmon/ansible/project/roles/grafana/vars/Ubuntu.yaml create mode 100644 cloudmon/ansible/project/roles/graphite/vars/Ubuntu.yaml create mode 100644 cloudmon/ansible/project/roles/postgresql/vars/Ubuntu.yaml create mode 100644 cloudmon/ansible/project/roles/postgresql_ha/vars/Ubuntu.yaml create mode 100644 cloudmon/ansible/project/roles/statsd/vars/Ubuntu.yaml diff --git a/cloudmon/ansible/project/roles/apimon_executor/tasks/main.yml b/cloudmon/ansible/project/roles/apimon_executor/tasks/main.yml index 58cdd3f..be8ef36 100644 --- a/cloudmon/ansible/project/roles/apimon_executor/tasks/main.yml +++ b/cloudmon/ansible/project/roles/apimon_executor/tasks/main.yml @@ -1,26 +1,95 @@ --- -# tasks file for executor - - name: Include variables - include_vars: "{{ lookup('first_found', params) }}" + ansible.builtin.include_vars: "{{ lookup('first_found', params) }}" vars: params: files: "{{ distro_lookup_path }}" paths: - "vars" +- name: Update apt cache on Ubuntu + ansible.builtin.apt: + update_cache: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.pkg_mgr != 'atomic_container' + +- name: Remove conflicting Ubuntu Docker packages + ansible.builtin.package: + state: absent + name: "{{ remove_packages }}" + when: + - ansible_facts.distribution == 'Ubuntu' + - remove_packages is defined + - ansible_facts.pkg_mgr != 'atomic_container' + register: remove_result + until: remove_result is success + retries: 5 + - name: Install required packages - become: true ansible.builtin.package: state: present - name: "{{ item }}" - loop: - - "{{ packages }}" + name: "{{ packages }}" when: "ansible_facts.pkg_mgr != 'atomic_container'" - register: task_result - until: task_result is success + register: install_result + until: install_result is success retries: 5 +- name: Clean up stale sockets after package changes + ansible.builtin.block: + - name: Stop Docker service and socket + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + loop: + - docker.service + - docker.socket + failed_when: false + + - name: Remove stale Docker socket and PID files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/run/docker.sock + - /var/run/docker.pid + when: + - remove_result is changed or install_result is changed + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Clear systemd rate-limits and prepare containerd + ansible.builtin.block: + - name: Reset systemd failed state for Docker units + ansible.builtin.systemd: + name: "{{ item }}" + reset_failed: true + loop: + - docker.socket + - docker.service + failed_when: false + + - name: Ensure containerd service is started and enabled + ansible.builtin.systemd: + name: containerd + state: started + enabled: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Start and enable Docker socket and daemon + ansible.builtin.systemd: + name: "{{ item }}" + state: started + enabled: true + loop: + - docker.socket + - docker.service + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + - name: Create executor group become: yes ansible.builtin.group: @@ -33,7 +102,7 @@ name: "{{ executor_os_user }}" group: "{{ executor_os_group }}" groups: "{{ (container_command == 'docker') | ternary('docker', '', omit) }}" - state: "present" + state: present - name: Write Executor SystemD unit file become: true @@ -47,7 +116,7 @@ become: true ansible.builtin.file: dest: "{{ item.dest }}" - state: "directory" + state: directory owner: "{{ executor_os_user }}" group: "{{ executor_os_group }}" mode: "{{ item.mode | default(omit) }}" diff --git a/cloudmon/ansible/project/roles/apimon_executor/vars/Debian.yaml b/cloudmon/ansible/project/roles/apimon_executor/vars/Debian.yaml index eb0b8a3..98a4451 100644 --- a/cloudmon/ansible/project/roles/apimon_executor/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/apimon_executor/vars/Debian.yaml @@ -1,5 +1,6 @@ --- -ansible_python_interpreter: "/usr/bin/python3" +remove_packages: [] + packages: - docker.io container_command: docker diff --git a/cloudmon/ansible/project/roles/apimon_executor/vars/Ubuntu.yaml b/cloudmon/ansible/project/roles/apimon_executor/vars/Ubuntu.yaml new file mode 100644 index 0000000..4ca09f8 --- /dev/null +++ b/cloudmon/ansible/project/roles/apimon_executor/vars/Ubuntu.yaml @@ -0,0 +1,17 @@ +--- +remove_packages: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + +packages: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + +container_command: docker diff --git a/cloudmon/ansible/project/roles/apimon_scheduler/tasks/main.yml b/cloudmon/ansible/project/roles/apimon_scheduler/tasks/main.yml index a3d886c..d5f73ca 100644 --- a/cloudmon/ansible/project/roles/apimon_scheduler/tasks/main.yml +++ b/cloudmon/ansible/project/roles/apimon_scheduler/tasks/main.yml @@ -2,25 +2,96 @@ # tasks file for scheduler - name: Include variables - include_vars: "{{ lookup('first_found', params) }}" + ansible.builtin.include_vars: "{{ lookup('first_found', params) }}" vars: params: files: "{{ distro_lookup_path }}" paths: - "vars" +- name: Update apt cache on Ubuntu + ansible.builtin.apt: + update_cache: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.pkg_mgr != 'atomic_container' + +- name: Remove conflicting Ubuntu Docker packages + ansible.builtin.package: + state: absent + name: "{{ remove_packages }}" + when: + - ansible_facts.distribution == 'Ubuntu' + - remove_packages is defined + - ansible_facts.pkg_mgr != 'atomic_container' + register: remove_result + until: remove_result is success + retries: 5 + - name: Install required packages - become: true ansible.builtin.package: state: present - name: "{{ item }}" - loop: - - "{{ packages }}" + name: "{{ packages }}" when: "ansible_facts.pkg_mgr != 'atomic_container'" - register: task_result - until: task_result is success + register: install_result + until: install_result is success retries: 5 +- name: Clean up stale sockets after package changes + ansible.builtin.block: + - name: Stop Docker service and socket + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + loop: + - docker.service + - docker.socket + failed_when: false + + - name: Remove stale Docker socket and PID files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/run/docker.sock + - /var/run/docker.pid + when: + - remove_result is changed or install_result is changed + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Clear systemd rate-limits and prepare containerd + ansible.builtin.block: + - name: Reset systemd failed state for Docker units + ansible.builtin.systemd: + name: "{{ item }}" + reset_failed: true + loop: + - docker.socket + - docker.service + failed_when: false + + - name: Ensure containerd service is started and enabled + ansible.builtin.systemd: + name: containerd + state: started + enabled: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Start and enable Docker socket and daemon + ansible.builtin.systemd: + name: "{{ item }}" + state: started + enabled: true + loop: + - docker.socket + - docker.service + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + - include_tasks: firewall.yml - name: Create scheduler group @@ -35,7 +106,7 @@ name: "{{ scheduler_os_user }}" group: "{{ scheduler_os_group }}" groups: "{{ (container_command == 'docker') | ternary('docker', '', omit) }}" - state: "present" + state: present - name: Write Scheduler SystemD unit file become: true @@ -49,7 +120,7 @@ become: true ansible.builtin.file: dest: "{{ item.dest }}" - state: "directory" + state: directory owner: "{{ scheduler_os_user }}" group: "{{ scheduler_os_group }}" mode: "{{ item.mode | default(omit) }}" diff --git a/cloudmon/ansible/project/roles/apimon_scheduler/vars/Debian.yaml b/cloudmon/ansible/project/roles/apimon_scheduler/vars/Debian.yaml index 231a1cd..60ec67d 100644 --- a/cloudmon/ansible/project/roles/apimon_scheduler/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/apimon_scheduler/vars/Debian.yaml @@ -1,6 +1,8 @@ --- -ansible_python_interpreter: "/usr/bin/python3" +remove_packages: [] + packages: - docker.io - firewalld + - python3-firewall container_command: docker diff --git a/cloudmon/ansible/project/roles/apimon_scheduler/vars/Ubuntu.yaml b/cloudmon/ansible/project/roles/apimon_scheduler/vars/Ubuntu.yaml new file mode 100644 index 0000000..4ca09f8 --- /dev/null +++ b/cloudmon/ansible/project/roles/apimon_scheduler/vars/Ubuntu.yaml @@ -0,0 +1,17 @@ +--- +remove_packages: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + +packages: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + +container_command: docker diff --git a/cloudmon/ansible/project/roles/base/repos/files/sources.list.jammy.x86_64 b/cloudmon/ansible/project/roles/base/repos/files/sources.list.jammy.x86_64 new file mode 100644 index 0000000..bc0aff9 --- /dev/null +++ b/cloudmon/ansible/project/roles/base/repos/files/sources.list.jammy.x86_64 @@ -0,0 +1,5 @@ +deb http://de.archive.ubuntu.com/ubuntu jammy main universe +deb http://de.archive.ubuntu.com/ubuntu jammy-updates main universe +deb http://de.archive.ubuntu.com/ubuntu jammy-backports main universe +deb http://de.archive.ubuntu.com/ubuntu jammy-security main universe +deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu jammy stable diff --git a/cloudmon/ansible/project/roles/base/repos/files/sources.list.noble.x86_64 b/cloudmon/ansible/project/roles/base/repos/files/sources.list.noble.x86_64 new file mode 100644 index 0000000..9dfc2a5 --- /dev/null +++ b/cloudmon/ansible/project/roles/base/repos/files/sources.list.noble.x86_64 @@ -0,0 +1,5 @@ +deb http://de.archive.ubuntu.com/ubuntu noble main universe +deb http://de.archive.ubuntu.com/ubuntu noble-updates main universe +deb http://de.archive.ubuntu.com/ubuntu noble-backports main universe +deb http://de.archive.ubuntu.com/ubuntu noble-security main universe +deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu noble stable diff --git a/cloudmon/ansible/project/roles/base/repos/tasks/Debian.yaml b/cloudmon/ansible/project/roles/base/repos/tasks/Debian.yaml new file mode 100644 index 0000000..4f12ab8 --- /dev/null +++ b/cloudmon/ansible/project/roles/base/repos/tasks/Debian.yaml @@ -0,0 +1,19 @@ +- name: Ensure /etc/apt/keyrings directory exists + ansible.builtin.file: + path: /etc/apt/keyrings + state: directory + mode: '0755' + +- name: Download Docker official GPG key + ansible.builtin.get_url: + url: https://download.docker.com/linux/ubuntu/gpg + dest: /etc/apt/keyrings/docker.asc + mode: '0644' + failed_when: false + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Configure apt retries + copy: + mode: 0444 diff --git a/cloudmon/ansible/project/roles/carbonapi/tasks/main.yaml b/cloudmon/ansible/project/roles/carbonapi/tasks/main.yaml index 7e7f119..89e2cb6 100644 --- a/cloudmon/ansible/project/roles/carbonapi/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/carbonapi/tasks/main.yaml @@ -1,26 +1,96 @@ --- - name: Include variables - include_vars: "{{ lookup('first_found', params) }}" + ansible.builtin.include_vars: "{{ lookup('first_found', params) }}" vars: params: files: "{{ distro_lookup_path }}" paths: - "vars" +- name: Update apt cache on Ubuntu + ansible.builtin.apt: + update_cache: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.pkg_mgr != 'atomic_container' + +- name: Remove conflicting Ubuntu Docker packages + ansible.builtin.package: + state: absent + name: "{{ remove_packages }}" + when: + - ansible_facts.distribution == 'Ubuntu' + - remove_packages is defined + - ansible_facts.pkg_mgr != 'atomic_container' + register: remove_result + until: remove_result is success + retries: 5 + - name: Install required packages - become: true ansible.builtin.package: state: present - name: "{{ item }}" - loop: - - "{{ packages }}" + name: "{{ packages }}" when: "ansible_facts.pkg_mgr != 'atomic_container'" - register: task_result - until: task_result is success + register: install_result + until: install_result is success retries: 5 +- name: Clean up stale sockets after package changes + ansible.builtin.block: + - name: Stop Docker service and socket + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + loop: + - docker.service + - docker.socket + failed_when: false + + - name: Remove stale Docker socket and PID files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/run/docker.sock + - /var/run/docker.pid + when: + - remove_result is changed or install_result is changed + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Clear systemd rate-limits and prepare containerd + ansible.builtin.block: + - name: Reset systemd failed state for Docker units + ansible.builtin.systemd: + name: "{{ item }}" + reset_failed: true + loop: + - docker.socket + - docker.service + failed_when: false + + - name: Ensure containerd service is started and enabled + ansible.builtin.systemd: + name: containerd + state: started + enabled: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Start and enable Docker socket and daemon + ansible.builtin.systemd: + name: "{{ item }}" + state: started + enabled: true + loop: + - docker.socket + - docker.service + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + - name: Ensure directories exist - become: true ansible.builtin.file: state: "directory" path: "{{ item }}" @@ -32,18 +102,16 @@ - "/root/bin" - name: Write config files - become: true ansible.builtin.template: src: "{{ item }}.j2" dest: "{{ graphite_config_location }}//{{ item }}" - mode: "0644" + mode: "0664" loop: - "carbonapi.yaml" notify: - Restart carbonapi - name: Write Carbonapi Systemd unit files - become: true ansible.builtin.template: src: "{{ item.src }}" dest: "{{ item.dest }}" @@ -58,7 +126,6 @@ - Restart carbonapi - name: Write Carbonapi-healthcheck Systemd unit files - become: true ansible.builtin.template: src: "{{ item.src }}" dest: "{{ item.dest }}" @@ -73,15 +140,13 @@ - Restart carbonapi-healthcheck - name: Write carbonapi-healthcheck script - become: true ansible.builtin.template: src: "carbonapi-healthcheck.sh.j2" dest: "/root/bin/carbonapi-healthcheck.sh" mode: "0744" - name: Set proper selinux context for executable file - become: true - community.general.sefcontext: + ansible.builtin.sefcontext: target: "/root/bin/carbonapi-healthcheck.sh.j2" setype: bin_t state: present diff --git a/cloudmon/ansible/project/roles/carbonapi/vars/Debian.yaml b/cloudmon/ansible/project/roles/carbonapi/vars/Debian.yaml index 9d32282..55aedce 100644 --- a/cloudmon/ansible/project/roles/carbonapi/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/carbonapi/vars/Debian.yaml @@ -1,4 +1,6 @@ --- +remove_packages: [] + packages: - docker.io - netcat diff --git a/cloudmon/ansible/project/roles/carbonapi/vars/Ubuntu.yaml b/cloudmon/ansible/project/roles/carbonapi/vars/Ubuntu.yaml new file mode 100644 index 0000000..4ca09f8 --- /dev/null +++ b/cloudmon/ansible/project/roles/carbonapi/vars/Ubuntu.yaml @@ -0,0 +1,17 @@ +--- +remove_packages: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + +packages: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + +container_command: docker diff --git a/cloudmon/ansible/project/roles/epmon/tasks/main.yml b/cloudmon/ansible/project/roles/epmon/tasks/main.yml index 312480c..ba9b0fe 100644 --- a/cloudmon/ansible/project/roles/epmon/tasks/main.yml +++ b/cloudmon/ansible/project/roles/epmon/tasks/main.yml @@ -1,26 +1,96 @@ --- -# tasks file for epmon - - name: Include variables - include_vars: "{{ lookup('first_found', params) }}" + ansible.builtin.include_vars: "{{ lookup('first_found', params) }}" vars: params: files: "{{ distro_lookup_path }}" paths: - "vars" +- name: Update apt cache on Ubuntu + ansible.builtin.apt: + update_cache: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.pkg_mgr != 'atomic_container' + +- name: Remove conflicting Ubuntu Docker packages + ansible.builtin.package: + state: absent + name: "{{ remove_packages }}" + when: + - ansible_facts.distribution == 'Ubuntu' + - remove_packages is defined + - ansible_facts.pkg_mgr != 'atomic_container' + register: remove_result + until: remove_result is success + retries: 5 + - name: Install required packages become: true ansible.builtin.package: state: present - name: "{{ item }}" - loop: - - "{{ packages }}" + name: "{{ packages }}" when: "ansible_facts.pkg_mgr != 'atomic_container'" - register: task_result - until: task_result is success + register: install_result + until: install_result is success retries: 5 +- name: Clean up stale sockets after package changes + ansible.builtin.block: + - name: Stop Docker service and socket + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + loop: + - docker.service + - docker.socket + failed_when: false + + - name: Remove stale Docker socket and PID files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/run/docker.sock + - /var/run/docker.pid + when: + - remove_result is changed or install_result is changed + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Clear systemd rate-limits and prepare containerd + ansible.builtin.block: + - name: Reset systemd failed state for Docker units + ansible.builtin.systemd: + name: "{{ item }}" + reset_failed: true + loop: + - docker.socket + - docker.service + failed_when: false + + - name: Ensure containerd service is started and enabled + ansible.builtin.systemd: + name: containerd + state: started + enabled: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Start and enable Docker socket and daemon + ansible.builtin.systemd: + name: "{{ item }}" + state: started + enabled: true + loop: + - docker.socket + - docker.service + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + - name: Create epmon group become: yes ansible.builtin.group: @@ -47,7 +117,7 @@ become: true ansible.builtin.file: dest: "{{ item.dest }}" - state: "directory" + state: directory owner: "{{ epmon_os_user }}" group: "{{ epmon_os_group }}" mode: "{{ item.mode | default(omit) }}" diff --git a/cloudmon/ansible/project/roles/epmon/vars/Debian.yaml b/cloudmon/ansible/project/roles/epmon/vars/Debian.yaml index eb0b8a3..98a4451 100644 --- a/cloudmon/ansible/project/roles/epmon/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/epmon/vars/Debian.yaml @@ -1,5 +1,6 @@ --- -ansible_python_interpreter: "/usr/bin/python3" +remove_packages: [] + packages: - docker.io container_command: docker diff --git a/cloudmon/ansible/project/roles/epmon/vars/Ubuntu.yaml b/cloudmon/ansible/project/roles/epmon/vars/Ubuntu.yaml new file mode 100644 index 0000000..4ca09f8 --- /dev/null +++ b/cloudmon/ansible/project/roles/epmon/vars/Ubuntu.yaml @@ -0,0 +1,17 @@ +--- +remove_packages: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + +packages: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + +container_command: docker diff --git a/cloudmon/ansible/project/roles/etcd/tasks/main.yaml b/cloudmon/ansible/project/roles/etcd/tasks/main.yaml index 17ec30f..8c46752 100644 --- a/cloudmon/ansible/project/roles/etcd/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/etcd/tasks/main.yaml @@ -7,18 +7,89 @@ paths: - "vars" +- name: Update apt cache on Ubuntu + ansible.builtin.apt: + update_cache: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.pkg_mgr != 'atomic_container' + +- name: Remove conflicting Ubuntu Docker packages + ansible.builtin.package: + state: absent + name: "{{ remove_packages }}" + when: + - ansible_facts.distribution == 'Ubuntu' + - remove_packages is defined + - ansible_facts.pkg_mgr != 'atomic_container' + register: remove_result + until: remove_result is success + retries: 5 + - name: Install required packages - become: true ansible.builtin.package: state: present - name: "{{ item }}" - loop: - - "{{ packages }}" + name: "{{ packages }}" when: "ansible_facts.pkg_mgr != 'atomic_container'" - register: task_result - until: task_result is success + register: install_result + until: install_result is success retries: 5 +- name: Clean up stale sockets after package changes + ansible.builtin.block: + - name: Stop Docker service and socket + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + loop: + - docker.service + - docker.socket + failed_when: false + + - name: Remove stale Docker socket and PID files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/run/docker.sock + - /var/run/docker.pid + when: + - remove_result is changed or install_result is changed + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Clear systemd rate-limits and prepare containerd + ansible.builtin.block: + - name: Reset systemd failed state for Docker units + ansible.builtin.systemd: + name: "{{ item }}" + reset_failed: true + loop: + - docker.socket + - docker.service + failed_when: false + + - name: Ensure containerd service is started and enabled + ansible.builtin.systemd: + name: containerd + state: started + enabled: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Start and enable Docker socket and daemon + ansible.builtin.systemd: + name: "{{ item }}" + state: started + enabled: true + loop: + - docker.socket + - docker.service + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + - name: Construct cluster nodes list ansible.builtin.set_fact: etcd_cluster: "{{ (etcd_cluster|default([])) + [(item + '=https://' + hostvars[item]['internal_address'] + ':2380')] }}" diff --git a/cloudmon/ansible/project/roles/etcd/vars/Debian.yaml b/cloudmon/ansible/project/roles/etcd/vars/Debian.yaml index 3436c52..98a4451 100644 --- a/cloudmon/ansible/project/roles/etcd/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/etcd/vars/Debian.yaml @@ -1,5 +1,6 @@ --- +remove_packages: [] + packages: - docker.io - container_command: docker diff --git a/cloudmon/ansible/project/roles/etcd/vars/Ubuntu.yaml b/cloudmon/ansible/project/roles/etcd/vars/Ubuntu.yaml new file mode 100644 index 0000000..4ca09f8 --- /dev/null +++ b/cloudmon/ansible/project/roles/etcd/vars/Ubuntu.yaml @@ -0,0 +1,17 @@ +--- +remove_packages: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + +packages: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + +container_command: docker diff --git a/cloudmon/ansible/project/roles/globalmon/tasks/main.yml b/cloudmon/ansible/project/roles/globalmon/tasks/main.yml index 1b4d624..3061cc8 100644 --- a/cloudmon/ansible/project/roles/globalmon/tasks/main.yml +++ b/cloudmon/ansible/project/roles/globalmon/tasks/main.yml @@ -2,41 +2,101 @@ # tasks file for globalmon - name: Include variables - include_vars: "{{ lookup('first_found', params) }}" + ansible.builtin.include_vars: "{{ lookup('first_found', params) }}" vars: params: files: "{{ distro_lookup_path }}" paths: - "vars" -# - name: Install required packages -# become: true -# ansible.builtin.package: -# state: present -# name: "{{ item }}" -# loop: -# - "{{ packages }}" -# when: "ansible_facts.pkg_mgr != 'atomic_container'" -# register: task_result -# until: task_result is success -# retries: 5 +- name: Include base repos role for Docker CE setup + ansible.builtin.include_role: + name: base.repos -# - name: Create globalmon group -# become: true -# ansible.builtin.group: -# name: "{{ globalmon_os_group }}" -# state: "present" +- name: Update apt cache on Ubuntu + ansible.builtin.apt: + update_cache: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.pkg_mgr != 'atomic_container' + +- name: Remove conflicting Ubuntu Docker packages + ansible.builtin.package: + state: absent + name: "{{ remove_packages }}" + when: + - ansible_facts.distribution == 'Ubuntu' + - remove_packages is defined + - ansible_facts.pkg_mgr != 'atomic_container' + register: remove_result + until: remove_result is success + retries: 5 + +- name: Install required packages + ansible.builtin.package: + state: present + name: "{{ packages }}" + when: "ansible_facts.pkg_mgr != 'atomic_container'" + register: install_result + until: install_result is success + retries: 5 + +- name: Clean up stale sockets after package changes + ansible.builtin.block: + - name: Stop Docker service and socket + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + loop: + - docker.service + - docker.socket + failed_when: false + + - name: Remove stale Docker socket and PID files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/run/docker.sock + - /var/run/docker.pid + when: + - remove_result is changed or install_result is changed + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Clear systemd rate-limits and prepare containerd + ansible.builtin.block: + - name: Reset systemd failed state for Docker units + ansible.builtin.systemd: + name: "{{ item }}" + reset_failed: true + loop: + - docker.socket + - docker.service + failed_when: false + + - name: Ensure containerd service is started and enabled + ansible.builtin.systemd: + name: containerd + state: started + enabled: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' -# - name: Create globalmon user -# become: true -# ansible.builtin.user: -# name: "{{ globalmon_os_user }}" -# group: "{{ globalmon_os_group }}" -# groups: "{{ (container_command == 'docker') | ternary('docker', '', omit) }}" -# state: "present" +- name: Start and enable Docker socket and daemon + ansible.builtin.systemd: + name: "{{ item }}" + state: started + enabled: true + loop: + - docker.socket + - docker.service + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' - name: Write globalmon SystemD unit file - become: true ansible.builtin.template: src: "globalmon.service.j2" dest: "{{ globalmon_systemd_unit_path }}" @@ -44,35 +104,26 @@ - Restart globalmon - name: Create required directories - become: true ansible.builtin.file: dest: "{{ item.dest }}" - state: "directory" - # owner: "{{ globalmon_os_user }}" - # group: "{{ globalmon_os_group }}" + state: directory mode: "{{ item.mode | default(omit) }}" loop: - { dest: "{{ globalmon_config_dir }}", mode: "0755" } - { dest: "{{ globalmon_log_dir }}", mode: "0777" } - name: Write env - become: true ansible.builtin.template: src: "env_globalmon" dest: "{{ globalmon_env_path }}" - # owner: "{{ globalmon_os_user }}" - # group: "{{ globalmon_os_group }}" mode: "0644" notify: - Restart globalmon - name: Write globalmon config - become: true ansible.builtin.copy: content: "{{ globalmon_config | to_nice_yaml(indent=2, width=79) }}" dest: "{{ globalmon_config_path }}" - # group: "{{ globalmon_os_group }}" - # owner: "{{ globalmon_os_user }}" mode: "0644" when: - "globalmon_config is defined" @@ -80,12 +131,9 @@ - Restart globalmon - name: Write globalmon secure config - become: true ansible.builtin.copy: content: "{{ globalmon_secure_config | to_nice_yaml(indent=2, width=79) }}" dest: "{{ globalmon_secure_config_path }}" - # group: "{{ globalmon_os_group }}" - # owner: "{{ globalmon_os_user }}" mode: "0644" when: - "globalmon_secure_config is defined" diff --git a/cloudmon/ansible/project/roles/globalmon/vars/Debian.yaml b/cloudmon/ansible/project/roles/globalmon/vars/Debian.yaml index eb0b8a3..98a4451 100644 --- a/cloudmon/ansible/project/roles/globalmon/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/globalmon/vars/Debian.yaml @@ -1,5 +1,6 @@ --- -ansible_python_interpreter: "/usr/bin/python3" +remove_packages: [] + packages: - docker.io container_command: docker diff --git a/cloudmon/ansible/project/roles/globalmon/vars/Ubuntu.yaml b/cloudmon/ansible/project/roles/globalmon/vars/Ubuntu.yaml new file mode 100644 index 0000000..4ca09f8 --- /dev/null +++ b/cloudmon/ansible/project/roles/globalmon/vars/Ubuntu.yaml @@ -0,0 +1,17 @@ +--- +remove_packages: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + +packages: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + +container_command: docker diff --git a/cloudmon/ansible/project/roles/grafana/tasks/main.yml b/cloudmon/ansible/project/roles/grafana/tasks/main.yml index 4ca57ab..6ce8ccc 100644 --- a/cloudmon/ansible/project/roles/grafana/tasks/main.yml +++ b/cloudmon/ansible/project/roles/grafana/tasks/main.yml @@ -1,34 +1,103 @@ --- - name: Include variables - include_vars: "{{ lookup('first_found', params) }}" + ansible.builtin.include_vars: "{{ lookup('first_found', params) }}" vars: params: files: "{{ distro_lookup_path }}" paths: - "vars" +- name: Update apt cache on Ubuntu + ansible.builtin.apt: + update_cache: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.pkg_mgr != 'atomic_container' + +- name: Remove conflicting Ubuntu Docker packages + ansible.builtin.package: + state: absent + name: "{{ remove_packages }}" + when: + - ansible_facts.distribution == 'Ubuntu' + - remove_packages is defined + - ansible_facts.pkg_mgr != 'atomic_container' + register: remove_result + until: remove_result is success + retries: 5 + - name: Install required packages - become: true ansible.builtin.package: state: present - name: "{{ item }}" - loop: - - "{{ packages }}" + name: "{{ packages }}" when: "ansible_facts.pkg_mgr != 'atomic_container'" - register: task_result - until: task_result is success + register: install_result + until: install_result is success retries: 5 +- name: Clean up stale sockets after package changes + ansible.builtin.block: + - name: Stop Docker service and socket + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + loop: + - docker.service + - docker.socket + failed_when: false + + - name: Remove stale Docker socket and PID files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/run/docker.sock + - /var/run/docker.pid + when: + - remove_result is changed or install_result is changed + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Clear systemd rate-limits and prepare containerd + ansible.builtin.block: + - name: Reset systemd failed state for Docker units + ansible.builtin.systemd: + name: "{{ item }}" + reset_failed: true + loop: + - docker.socket + - docker.service + failed_when: false + + - name: Ensure containerd service is started and enabled + ansible.builtin.systemd: + name: containerd + state: started + enabled: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Start and enable Docker socket and daemon + ansible.builtin.systemd: + name: "{{ item }}" + state: started + enabled: true + loop: + - docker.socket + - docker.service + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + - include_tasks: firewall.yml - name: Create Grafana group - become: true ansible.builtin.group: name: "{{ grafana_os_user }}" state: present - name: Create Grafana user for docker/podman - become: true ansible.builtin.user: name: "{{ grafana_os_user }}" group: "{{ grafana_os_group }}" @@ -36,8 +105,7 @@ state: present - name: Create required directories - become: true - file: + ansible.builtin.file: path: "{{ item.dest }}" state: directory owner: "{{ grafana_os_user }}" @@ -48,8 +116,7 @@ - {dest: "/etc/grafana/provisioning"} - name: Write Grafana config env file - become: true - template: + ansible.builtin.template: src: "env.j2" dest: "/etc/grafana/env" owner: "{{ grafana_os_user }}" @@ -59,8 +126,7 @@ - restart grafana - name: Write Grafana Image Renderer Systemd unit file - become: true - template: + ansible.builtin.template: src: "grafana-renderer.service.j2" dest: "/etc/systemd/system/grafana-renderer.service" when: @@ -70,8 +136,7 @@ - restart grafana-renderer - name: Write Grafana Systemd unit file - become: true - template: + ansible.builtin.template: src: "grafana.service.j2" dest: "/etc/systemd/system/grafana.service" notify: @@ -81,20 +146,18 @@ meta: flush_handlers - name: Make sure the Grafana service started - become: true - systemd: + ansible.builtin.systemd: state: started name: "grafana.service" - name: Wait for grafana container to listen - become: true - wait_for: + ansible.builtin.wait_for: host: 0.0.0.0 port: 3000 timeout: 120 - name: Wait for api/health to come up - uri: + ansible.builtin.uri: url: "http://0.0.0.0:3000/api/health" status_code: 200 register: result diff --git a/cloudmon/ansible/project/roles/grafana/vars/Debian.yaml b/cloudmon/ansible/project/roles/grafana/vars/Debian.yaml index 3436c52..98a4451 100644 --- a/cloudmon/ansible/project/roles/grafana/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/grafana/vars/Debian.yaml @@ -1,5 +1,6 @@ --- +remove_packages: [] + packages: - docker.io - container_command: docker diff --git a/cloudmon/ansible/project/roles/grafana/vars/Ubuntu.yaml b/cloudmon/ansible/project/roles/grafana/vars/Ubuntu.yaml new file mode 100644 index 0000000..4ca09f8 --- /dev/null +++ b/cloudmon/ansible/project/roles/grafana/vars/Ubuntu.yaml @@ -0,0 +1,17 @@ +--- +remove_packages: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + +packages: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + +container_command: docker diff --git a/cloudmon/ansible/project/roles/graphite/tasks/main.yaml b/cloudmon/ansible/project/roles/graphite/tasks/main.yaml index ac4e03c..2011deb 100644 --- a/cloudmon/ansible/project/roles/graphite/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/graphite/tasks/main.yaml @@ -1,24 +1,95 @@ --- - name: Include variables - include_vars: "{{ lookup('first_found', params) }}" + ansible.builtin.include_vars: "{{ lookup('first_found', params) }}" vars: params: files: "{{ distro_lookup_path }}" paths: - "vars" +- name: Update apt cache on Ubuntu + ansible.builtin.apt: + update_cache: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.pkg_mgr != 'atomic_container' + +- name: Remove conflicting Ubuntu Docker packages + ansible.builtin.package: + state: absent + name: "{{ remove_packages }}" + when: + - ansible_facts.distribution == 'Ubuntu' + - remove_packages is defined + - ansible_facts.pkg_mgr != 'atomic_container' + register: remove_result + until: remove_result is success + retries: 5 + - name: Install required packages - become: true ansible.builtin.package: state: present - name: "{{ item }}" - loop: - - "{{ packages }}" + name: "{{ packages }}" when: "ansible_facts.pkg_mgr != 'atomic_container'" - register: task_result - until: task_result is success + register: install_result + until: install_result is success retries: 5 +- name: Clean up stale sockets after package changes + ansible.builtin.block: + - name: Stop Docker service and socket + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + loop: + - docker.service + - docker.socket + failed_when: false + + - name: Remove stale Docker socket and PID files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/run/docker.sock + - /var/run/docker.pid + when: + - remove_result is changed or install_result is changed + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Clear systemd rate-limits and prepare containerd + ansible.builtin.block: + - name: Reset systemd failed state for Docker units + ansible.builtin.systemd: + name: "{{ item }}" + reset_failed: true + loop: + - docker.socket + - docker.service + failed_when: false + + - name: Ensure containerd service is started and enabled + ansible.builtin.systemd: + name: containerd + state: started + enabled: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Start and enable Docker socket and daemon + ansible.builtin.systemd: + name: "{{ item }}" + state: started + enabled: true + loop: + - docker.socket + - docker.service + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + - include_tasks: firewall.yml - name: Create graphite sync group @@ -42,7 +113,7 @@ ansible.posix.authorized_key: user: "{{ graphite_sync_user }}" key: "{{ graphite_public_key }}" - state: "present" + state: present when: - "graphite_public_key is defined" - "graphite_sync_user is defined" @@ -100,7 +171,7 @@ minute: "0" hour: "2" job: > - bash -c 'echo "--- $(date) ---"; + bash -c 'echo "--- $(date) --"; find /opt/graphite/storage/whisper -type f -mtime +270 -name "*.wsp" -delete -print; find /opt/graphite/storage/whisper -depth -type d -empty -delete -print' >> /var/log/graphite-cleanup.log 2>&1 diff --git a/cloudmon/ansible/project/roles/graphite/vars/Debian.yaml b/cloudmon/ansible/project/roles/graphite/vars/Debian.yaml index da414a6..7a35884 100644 --- a/cloudmon/ansible/project/roles/graphite/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/graphite/vars/Debian.yaml @@ -1,7 +1,7 @@ --- +remove_packages: [] + packages: - docker.io - netcat - - cron - - apparmor-profiles container_command: docker diff --git a/cloudmon/ansible/project/roles/graphite/vars/Ubuntu.yaml b/cloudmon/ansible/project/roles/graphite/vars/Ubuntu.yaml new file mode 100644 index 0000000..4ca09f8 --- /dev/null +++ b/cloudmon/ansible/project/roles/graphite/vars/Ubuntu.yaml @@ -0,0 +1,17 @@ +--- +remove_packages: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + +packages: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + +container_command: docker diff --git a/cloudmon/ansible/project/roles/postgresql/tasks/main.yaml b/cloudmon/ansible/project/roles/postgresql/tasks/main.yaml index ec5c718..29eaa2c 100644 --- a/cloudmon/ansible/project/roles/postgresql/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/postgresql/tasks/main.yaml @@ -1,29 +1,100 @@ --- - name: Include variables - include_vars: "{{ lookup('first_found', params) }}" + ansible.builtin.include_vars: "{{ lookup('first_found', params) }}" vars: params: files: "{{ distro_lookup_path }}" paths: - "vars" +- name: Update apt cache on Ubuntu + ansible.builtin.apt: + update_cache: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.pkg_mgr != 'atomic_container' + +- name: Remove conflicting Ubuntu Docker packages + ansible.builtin.package: + state: absent + name: "{{ remove_packages }}" + when: + - ansible_facts.distribution == 'Ubuntu' + - remove_packages is defined + - ansible_facts.pkg_mgr != 'atomic_container' + register: remove_result + until: remove_result is success + retries: 5 + - name: Install required packages - become: true ansible.builtin.package: state: present - name: "{{ item }}" - loop: - - "{{ packages }}" + name: "{{ packages }}" when: "ansible_facts.pkg_mgr != 'atomic_container'" - register: task_result - until: task_result is success + register: install_result + until: install_result is success retries: 5 +- name: Clean up stale sockets after package changes + ansible.builtin.block: + - name: Stop Docker service and socket + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + loop: + - docker.service + - docker.socket + failed_when: false + + - name: Remove stale Docker socket and PID files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/run/docker.sock + - /var/run/docker.pid + when: + - remove_result is changed or install_result is changed + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Clear systemd rate-limits and prepare containerd + ansible.builtin.block: + - name: Reset systemd failed state for Docker units + ansible.builtin.systemd: + name: "{{ item }}" + reset_failed: true + loop: + - docker.socket + - docker.service + failed_when: false + + - name: Ensure containerd service is started and enabled + ansible.builtin.systemd: + name: containerd + state: started + enabled: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Start and enable Docker socket and daemon + ansible.builtin.systemd: + name: "{{ item }}" + state: started + enabled: true + loop: + - docker.socket + - docker.service + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + - include_tasks: firewall.yml - name: Write Postgres Systemd unit file become: true - template: + ansible.builtin.template: src: "postgres.service.j2" dest: "/etc/systemd/system/postgres.service" mode: "0644" @@ -35,13 +106,13 @@ - name: Make sure the Postgres service started become: true - systemd: + ansible.builtin.systemd: state: started name: "postgres.service" - name: Wait for postgres container to listen become: true - wait_for: + ansible.builtin.wait_for: host: 0.0.0.0 port: 5432 timeout: 120 diff --git a/cloudmon/ansible/project/roles/postgresql/vars/Debian.yaml b/cloudmon/ansible/project/roles/postgresql/vars/Debian.yaml index 4893bc6..e454211 100644 --- a/cloudmon/ansible/project/roles/postgresql/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/postgresql/vars/Debian.yaml @@ -1,7 +1,6 @@ --- +remove_packages: [] + packages: - docker.io - python3-psycopg2 - -container_command: docker -postgresql_data_dir: "/var/lib/psql" diff --git a/cloudmon/ansible/project/roles/postgresql/vars/Ubuntu.yaml b/cloudmon/ansible/project/roles/postgresql/vars/Ubuntu.yaml new file mode 100644 index 0000000..5d23161 --- /dev/null +++ b/cloudmon/ansible/project/roles/postgresql/vars/Ubuntu.yaml @@ -0,0 +1,19 @@ +--- +remove_packages: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + +packages: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + - python3-psycopg2 + +container_command: docker +postgresql_data_dir: "/var/lib/psql" diff --git a/cloudmon/ansible/project/roles/postgresql_ha/tasks/main.yaml b/cloudmon/ansible/project/roles/postgresql_ha/tasks/main.yaml index 98aa8b1..9e3f853 100644 --- a/cloudmon/ansible/project/roles/postgresql_ha/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/postgresql_ha/tasks/main.yaml @@ -7,18 +7,89 @@ paths: - "vars" +- name: Update apt cache on Ubuntu + ansible.builtin.apt: + update_cache: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.pkg_mgr != 'atomic_container' + +- name: Remove conflicting Ubuntu Docker packages + ansible.builtin.package: + state: absent + name: "{{ remove_packages }}" + when: + - ansible_facts.distribution == 'Ubuntu' + - remove_packages is defined + - ansible_facts.pkg_mgr != 'atomic_container' + register: remove_result + until: remove_result is success + retries: 5 + - name: Install required packages - become: true ansible.builtin.package: state: present - name: "{{ item }}" - loop: - - "{{ packages }}" + name: "{{ packages }}" when: "ansible_facts.pkg_mgr != 'atomic_container'" - register: task_result - until: task_result is success + register: install_result + until: install_result is success retries: 5 +- name: Clean up stale sockets after package changes + ansible.builtin.block: + - name: Stop Docker service and socket + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + loop: + - docker.service + - docker.socket + failed_when: false + + - name: Remove stale Docker socket and PID files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/run/docker.sock + - /var/run/docker.pid + when: + - remove_result is changed or install_result is changed + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Clear systemd rate-limits and prepare containerd + ansible.builtin.block: + - name: Reset systemd failed state for Docker units + ansible.builtin.systemd: + name: "{{ item }}" + reset_failed: true + loop: + - docker.socket + - docker.service + failed_when: false + + - name: Ensure containerd service is started and enabled + ansible.builtin.systemd: + name: containerd + state: started + enabled: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Start and enable Docker socket and daemon + ansible.builtin.systemd: + name: "{{ item }}" + state: started + enabled: true + loop: + - docker.socket + - docker.service + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + - name: Construct cluster etcd nodes list ansible.builtin.set_fact: etcd_cluster_nodes: "{{ (etcd_cluster_nodes|default([])) + [item] }}" diff --git a/cloudmon/ansible/project/roles/postgresql_ha/vars/Debian.yaml b/cloudmon/ansible/project/roles/postgresql_ha/vars/Debian.yaml index b7340c6..1b1b64b 100644 --- a/cloudmon/ansible/project/roles/postgresql_ha/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/postgresql_ha/vars/Debian.yaml @@ -1,4 +1,6 @@ --- +remove_packages: [] + packages: - docker.io - python3-psycopg2 diff --git a/cloudmon/ansible/project/roles/postgresql_ha/vars/Ubuntu.yaml b/cloudmon/ansible/project/roles/postgresql_ha/vars/Ubuntu.yaml new file mode 100644 index 0000000..4ca09f8 --- /dev/null +++ b/cloudmon/ansible/project/roles/postgresql_ha/vars/Ubuntu.yaml @@ -0,0 +1,17 @@ +--- +remove_packages: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + +packages: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + +container_command: docker diff --git a/cloudmon/ansible/project/roles/statsd/tasks/main.yaml b/cloudmon/ansible/project/roles/statsd/tasks/main.yaml index 3198eed..fee2ae1 100644 --- a/cloudmon/ansible/project/roles/statsd/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/statsd/tasks/main.yaml @@ -1,22 +1,99 @@ - name: Include OS-specific variables - include_vars: "{{ lookup('first_found', params) }}" + ansible.builtin.include_vars: "{{ lookup('first_found', params) }}" vars: params: files: "{{ distro_lookup_path }}" paths: - 'vars' +- name: Update apt cache on Ubuntu + ansible.builtin.apt: + update_cache: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.pkg_mgr != 'atomic_container' + +- name: Remove conflicting Ubuntu Docker packages + ansible.builtin.package: + state: absent + name: "{{ remove_packages }}" + when: + - ansible_facts.distribution == 'Ubuntu' + - remove_packages is defined + - ansible_facts.pkg_mgr != 'atomic_container' + register: remove_result + until: remove_result is success + retries: 5 + - name: Install container runtime ansible.builtin.package: name: "{{ distro_packages }}" state: present + register: install_result + until: install_result is success + retries: 5 # If we are on CoreOS we can't use the module and podman already exists when: "ansible_facts.pkg_mgr != 'atomic_container'" +- name: Clean up stale sockets after package changes + ansible.builtin.block: + - name: Stop Docker service and socket + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + loop: + - docker.service + - docker.socket + failed_when: false + + - name: Remove stale Docker socket and PID files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/run/docker.sock + - /var/run/docker.pid + when: + - remove_result is changed or install_result is changed + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Clear systemd rate-limits and prepare containerd + ansible.builtin.block: + - name: Reset systemd failed state for Docker units + ansible.builtin.systemd: + name: "{{ item }}" + reset_failed: true + loop: + - docker.socket + - docker.service + failed_when: false + + - name: Ensure containerd service is started and enabled + ansible.builtin.systemd: + name: containerd + state: started + enabled: true + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + +- name: Start and enable Docker socket and daemon + ansible.builtin.systemd: + name: "{{ item }}" + state: started + enabled: true + loop: + - docker.socket + - docker.service + when: + - ansible_facts.distribution == 'Ubuntu' + - ansible_facts.distribution_major_version == '22' + - name: Create StatsD directory ansible.builtin.file: path: "{{ statsd_config_location }}" - state: "directory" + state: directory recurse: true - name: Write StatsD config diff --git a/cloudmon/ansible/project/roles/statsd/vars/Debian.yaml b/cloudmon/ansible/project/roles/statsd/vars/Debian.yaml index 6542952..6f6f115 100644 --- a/cloudmon/ansible/project/roles/statsd/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/statsd/vars/Debian.yaml @@ -1,3 +1,10 @@ +--- +remove_packages: [] + distro_packages: - docker.io + - netcat + - cron + - apparmor-profiles + container_command: docker diff --git a/cloudmon/ansible/project/roles/statsd/vars/RedHat.yaml b/cloudmon/ansible/project/roles/statsd/vars/RedHat.yaml index 4972b68..dffe868 100644 --- a/cloudmon/ansible/project/roles/statsd/vars/RedHat.yaml +++ b/cloudmon/ansible/project/roles/statsd/vars/RedHat.yaml @@ -1,3 +1,7 @@ +--- +remove_packages: [] + distro_packages: - podman + container_command: podman diff --git a/cloudmon/ansible/project/roles/statsd/vars/Ubuntu.yaml b/cloudmon/ansible/project/roles/statsd/vars/Ubuntu.yaml new file mode 100644 index 0000000..f4d535d --- /dev/null +++ b/cloudmon/ansible/project/roles/statsd/vars/Ubuntu.yaml @@ -0,0 +1,17 @@ +--- +remove_packages: + - docker.io + - docker-doc + - docker-compose + - podman-docker + - containerd + - runc + +distro_packages: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + +container_command: docker From f334cb2212f627f781ad6efbb90f0012c581fb68 Mon Sep 17 00:00:00 2001 From: Vladimir Hasko Date: Mon, 17 Aug 2026 13:27:07 +0200 Subject: [PATCH 02/16] switching to python 3.10 for tox --- zuul.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zuul.yaml b/zuul.yaml index 2e1f966..b98d268 100644 --- a/zuul.yaml +++ b/zuul.yaml @@ -26,12 +26,12 @@ check: jobs: - otc-tox-pep8 - - otc-tox-py39 + - otc-tox-py310 - otc-tox-docs - cloudmon-deploy-ubuntu-jammy gate: jobs: - otc-tox-pep8 - - otc-tox-py39 + - otc-tox-py310 - otc-tox-docs - cloudmon-deploy-ubuntu-jammy From 178e3e07511c11d18acaa603885e6a39ac475b99 Mon Sep 17 00:00:00 2001 From: Vladimir Hasko Date: Mon, 17 Aug 2026 13:42:08 +0200 Subject: [PATCH 03/16] switching to python3.11 for tox for zuul job --- zuul.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zuul.yaml b/zuul.yaml index b98d268..be844e1 100644 --- a/zuul.yaml +++ b/zuul.yaml @@ -26,12 +26,12 @@ check: jobs: - otc-tox-pep8 - - otc-tox-py310 + - otc-tox-py311 - otc-tox-docs - cloudmon-deploy-ubuntu-jammy gate: jobs: - otc-tox-pep8 - - otc-tox-py310 + - otc-tox-py311 - otc-tox-docs - cloudmon-deploy-ubuntu-jammy From 822b922abaffdcee7ae24bed530e85e6af94f583 Mon Sep 17 00:00:00 2001 From: vladimirhasko Date: Tue, 18 Aug 2026 07:18:05 +0000 Subject: [PATCH 04/16] modifyng deploy test job to run on k8s --- playbooks_k8s/pre.yaml | 19 +++ playbooks_k8s/run.yaml | 115 ++++++++++++++++++ roles/deploy_cloudmon_k8s/defaults/main.yaml | 2 + roles/deploy_cloudmon_k8s/tasks/main.yaml | 22 ++++ .../templates/inventory.j2 | 33 +++++ zuul.yaml | 29 ++++- 6 files changed, 218 insertions(+), 2 deletions(-) create mode 100644 playbooks_k8s/pre.yaml create mode 100644 playbooks_k8s/run.yaml create mode 100644 roles/deploy_cloudmon_k8s/defaults/main.yaml create mode 100644 roles/deploy_cloudmon_k8s/tasks/main.yaml create mode 100644 roles/deploy_cloudmon_k8s/templates/inventory.j2 diff --git a/playbooks_k8s/pre.yaml b/playbooks_k8s/pre.yaml new file mode 100644 index 0000000..526856d --- /dev/null +++ b/playbooks_k8s/pre.yaml @@ -0,0 +1,19 @@ +--- +# Pre-run playbook for the Kubernetes/pod-based cloudmon-deploy job. +# +# The `zuul-debian` eco2 pod runs as root but ships without `sudo`. +# Any task that uses `become: true` will fail with "sudo: not found" +# (exit 127) unless sudo is present. We install it via `raw` (no +# interpreter dependency) since we are already root. +# +# Then we bootstrap python, pip and virtualenv via the OTC global +# roles. Docker CE is installed in the main run play (see run.yaml). +- hosts: all + pre_tasks: + - name: Ensure sudo is available + raw: command -v sudo >/dev/null 2>&1 || (apt-get update && apt-get install -y sudo) + changed_when: false + roles: + - ensure-python + - ensure-pip + - ensure-virtualenv diff --git a/playbooks_k8s/run.yaml b/playbooks_k8s/run.yaml new file mode 100644 index 0000000..c19c8ee --- /dev/null +++ b/playbooks_k8s/run.yaml @@ -0,0 +1,115 @@ +--- +# Main run playbook for the Kubernetes/pod-based cloudmon-deploy job. +# +# The whole CloudMon stack is deployed on a single `zuul-debian` pod +# (no separate controller / worker split). Docker CE is installed +# inline here because the OTC global role `ensure-docker` cannot be +# verified. +# +# Docker is needed because cloudmon's graphite/statsd/postgres +# provisioning playbooks deploy containerized components via +# ansible-runner. +- hosts: all + become: true + vars: + docker_ce_packages: + - docker-ce + - docker-ce-cli + - containerd.io + - docker-buildx-plugin + - docker-compose-plugin + tasks: + - name: Update apt cache + ansible.builtin.apt: + update_cache: true + + - name: Install prerequisites + ansible.builtin.apt: + name: + - ca-certificates + - curl + - gnupg + state: present + register: prereq_result + retries: 5 + until: prereq_result is success + + # Add Docker CE apt repository + - name: Ensure /etc/apt/keyrings directory exists + ansible.builtin.file: + path: /etc/apt/keyrings + state: directory + mode: '0755' + + - name: Download Docker official GPG key + ansible.builtin.get_url: + url: "https://download.docker.com/linux/{{ ansible_facts.distribution | lower }}/gpg" + dest: /etc/apt/keyrings/docker.asc + mode: '0644' + + - name: Add Docker CE apt repository + ansible.builtin.apt_repository: + repo: >- + deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] + https://download.docker.com/linux/{{ ansible_facts.distribution | lower }} + {{ ansible_facts.lsb_release['codename'] }} stable + state: present + + # Remove conflicting Docker packages that may be in the distro image + - name: Remove conflicting Ubuntu Docker packages + ansible.builtin.package: + state: absent + name: + - docker + - docker-engine + - docker.io + - containerd + - runc + failed_when: false + + # Install Docker CE + - name: Install Docker CE packages + ansible.builtin.apt: + name: "{{ docker_ce_packages }}" + state: present + register: docker_install + retries: 5 + until: docker_install is success + + # Clean up stale sockets after package changes + - name: Remove stale Docker socket and PID files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/run/docker.sock + - /var/run/docker.pid + failed_when: false + + # Start and enable containerd and docker + - name: Start and enable containerd + ansible.builtin.systemd: + name: containerd + state: started + enabled: true + + - name: Start and enable Docker + ansible.builtin.systemd: + name: "{{ item }}" + state: started + enabled: true + loop: + - docker.socket + - docker.service + + - name: Verify Docker daemon is running + ansible.builtin.command: + cmd: docker info + register: docker_info + changed_when: false + +- hosts: all + become: true + roles: + - install_cloudmon + - deploy_cloudmon_k8s diff --git a/roles/deploy_cloudmon_k8s/defaults/main.yaml b/roles/deploy_cloudmon_k8s/defaults/main.yaml new file mode 100644 index 0000000..129b59e --- /dev/null +++ b/roles/deploy_cloudmon_k8s/defaults/main.yaml @@ -0,0 +1,2 @@ +zuul_work_dir: "{{ zuul.project.src_dir }}" +cloudmon_venv_path: "{{ ansible_user_dir }}/.venv" diff --git a/roles/deploy_cloudmon_k8s/tasks/main.yaml b/roles/deploy_cloudmon_k8s/tasks/main.yaml new file mode 100644 index 0000000..8dcc5c0 --- /dev/null +++ b/roles/deploy_cloudmon_k8s/tasks/main.yaml @@ -0,0 +1,22 @@ +--- +- name: Make inventory + ansible.builtin.template: + dest: "{{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory" + src: "inventory.j2" + +- name: Deploy + args: + chdir: "{{ ansible_user_dir }}/{{ zuul_work_dir }}" + executable: "/bin/bash" + ansible.builtin.shell: | + source {{ cloudmon_venv_path }}/bin/activate + cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure graphite provision + cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure statsd provision + cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure postgres provision + +# cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure grafana provision +# cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure apimon provision +# cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure epmon provision +# cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure metrics processor provision +# cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure globalmon provision +# cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure status dashboard provision diff --git a/roles/deploy_cloudmon_k8s/templates/inventory.j2 b/roles/deploy_cloudmon_k8s/templates/inventory.j2 new file mode 100644 index 0000000..e7c8bac --- /dev/null +++ b/roles/deploy_cloudmon_k8s/templates/inventory.j2 @@ -0,0 +1,33 @@ +all: + hosts: + # The whole CloudMon stack is deployed on the single `zuul-debian` + # pod, so there is no separate worker node. Everything runs + # "localhost" from the pod's point of view. + localhost: + ansible_host: localhost + internal_address: localhost + children: + statsd: + hosts: + - localhost + graphite: + hosts: + - localhost + grafana: + hosts: + - localhost + schedulers: + hosts: + - localhost + executors: + hosts: + - localhost + epmons: + hosts: + - localhost + globalmons: + hosts: + - localhost + postgres: + hosts: + - localhost diff --git a/zuul.yaml b/zuul.yaml index be844e1..8606646 100644 --- a/zuul.yaml +++ b/zuul.yaml @@ -20,6 +20,31 @@ - name: worker label: ubuntu-jammy +# Kubernetes/pod-based deployment. Instead of provisioning two ubuntu-jammy +# VMs (controller + worker) this runs the whole CloudMon stack on a single +# `zuul-debian` pod. The pod runs as root but ships without sudo and has no +# multi-node SSH setup, so it uses its own playbooks (playbooks_k8s) that +# bootstrap sudo + python and deploy on one node. +- job: + name: cloudmon-deploy-k8s + abstract: true + description: | + Deploy CloudMon on Kubernetes pods + pre-run: playbooks_k8s/pre.yaml + run: playbooks_k8s/run.yaml + files: + - ^cloudmon/* + vars: + python_version: "3.11" + +- job: + name: cloudmon-deploy-zuul-debian + parent: cloudmon-deploy-k8s + nodeset: + nodes: + - name: zuul-debian + label: zuul-debian + - project: merge-mode: squash-merge default-branch: main @@ -28,10 +53,10 @@ - otc-tox-pep8 - otc-tox-py311 - otc-tox-docs - - cloudmon-deploy-ubuntu-jammy + - cloudmon-deploy-zuul-debian gate: jobs: - otc-tox-pep8 - otc-tox-py311 - otc-tox-docs - - cloudmon-deploy-ubuntu-jammy + - cloudmon-deploy-zuul-debian From e350bd14e8cfb33074e4ebf83755cf15fdc486e4 Mon Sep 17 00:00:00 2001 From: vladimirhasko Date: Tue, 18 Aug 2026 10:07:07 +0000 Subject: [PATCH 05/16] fixing debian zuul job --- .../project/roles/carbonapi/vars/Debian.yaml | 5 ++++- .../project/roles/graphite/vars/Debian.yaml | 6 ++++- .../project/roles/postgresql/vars/Debian.yaml | 7 +++++- .../project/roles/statsd/vars/Debian.yaml | 5 ++++- playbooks_k8s/run.yaml | 22 ++++++++++--------- 5 files changed, 31 insertions(+), 14 deletions(-) diff --git a/cloudmon/ansible/project/roles/carbonapi/vars/Debian.yaml b/cloudmon/ansible/project/roles/carbonapi/vars/Debian.yaml index 55aedce..1505647 100644 --- a/cloudmon/ansible/project/roles/carbonapi/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/carbonapi/vars/Debian.yaml @@ -1,8 +1,11 @@ --- +# On Debian (zuul-debian pod), the container runtime (docker-ce) is installed +# and started by the pod bootstrap in playbooks_k8s/run.yaml, which runs +# before the provisioning roles. This vars file only declares non-Docker +# application dependencies to avoid redundant/ conflicting Docker installs. remove_packages: [] packages: - - docker.io - netcat - policycoreutils-python-utils - python3-selinux diff --git a/cloudmon/ansible/project/roles/graphite/vars/Debian.yaml b/cloudmon/ansible/project/roles/graphite/vars/Debian.yaml index 7a35884..001aa59 100644 --- a/cloudmon/ansible/project/roles/graphite/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/graphite/vars/Debian.yaml @@ -1,7 +1,11 @@ --- +# On Debian (zuul-debian pod), the container runtime (docker-ce) is installed +# and started by the pod bootstrap in playbooks_k8s/run.yaml, which runs +# before the provisioning roles. This vars file only declares non-Docker +# application dependencies to avoid redundant/ conflicting Docker installs. remove_packages: [] packages: - - docker.io - netcat + container_command: docker diff --git a/cloudmon/ansible/project/roles/postgresql/vars/Debian.yaml b/cloudmon/ansible/project/roles/postgresql/vars/Debian.yaml index e454211..48863bd 100644 --- a/cloudmon/ansible/project/roles/postgresql/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/postgresql/vars/Debian.yaml @@ -1,6 +1,11 @@ --- +# On Debian (zuul-debian pod), the container runtime (docker-ce) is installed +# and started by the pod bootstrap in playbooks_k8s/run.yaml, which runs +# before the provisioning roles. This vars file only declares non-Docker +# application dependencies to avoid redundant/ conflicting Docker installs. remove_packages: [] packages: - - docker.io - python3-psycopg2 + +container_command: docker diff --git a/cloudmon/ansible/project/roles/statsd/vars/Debian.yaml b/cloudmon/ansible/project/roles/statsd/vars/Debian.yaml index 6f6f115..fdaaeea 100644 --- a/cloudmon/ansible/project/roles/statsd/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/statsd/vars/Debian.yaml @@ -1,8 +1,11 @@ --- +# On Debian (zuul-debian pod), the container runtime (docker-ce) is installed +# and started by the pod bootstrap in playbooks_k8s/run.yaml, which runs +# before the provisioning roles. This vars file only declares non-Docker +# application dependencies to avoid redundant/ conflicting Docker installs. remove_packages: [] distro_packages: - - docker.io - netcat - cron - apparmor-profiles diff --git a/playbooks_k8s/run.yaml b/playbooks_k8s/run.yaml index c19c8ee..bce1f05 100644 --- a/playbooks_k8s/run.yaml +++ b/playbooks_k8s/run.yaml @@ -3,14 +3,14 @@ # # The whole CloudMon stack is deployed on a single `zuul-debian` pod # (no separate controller / worker split). Docker CE is installed -# inline here because the OTC global role `ensure-docker` cannot be -# verified. +# inline here because there is no verified global ensure-docker role. # # Docker is needed because cloudmon's graphite/statsd/postgres # provisioning playbooks deploy containerized components via # ansible-runner. - hosts: all become: true + gather_facts: true vars: docker_ce_packages: - docker-ce @@ -43,20 +43,22 @@ - name: Download Docker official GPG key ansible.builtin.get_url: - url: "https://download.docker.com/linux/{{ ansible_facts.distribution | lower }}/gpg" + url: "https://download.docker.com/linux/{{ ansible_facts['os_release']['ID'] }}/gpg" dest: /etc/apt/keyrings/docker.asc mode: '0644' - name: Add Docker CE apt repository - ansible.builtin.apt_repository: - repo: >- - deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] - https://download.docker.com/linux/{{ ansible_facts.distribution | lower }} - {{ ansible_facts.lsb_release['codename'] }} stable - state: present + ansible.builtin.copy: + dest: /etc/apt/sources.list.d/docker-ce.list + mode: '0644' + content: >- + deb [arch={{ ansible_facts['architecture'] }} + signed-by=/etc/apt/keyrings/docker.asc] + https://download.docker.com/linux/{{ ansible_facts['os_release']['ID'] }} + {{ ansible_facts['os_release']['VERSION_CODENAME'] }} stable # Remove conflicting Docker packages that may be in the distro image - - name: Remove conflicting Ubuntu Docker packages + - name: Remove conflicting Docker packages ansible.builtin.package: state: absent name: From 68cb5f57bc83b629e5a8e4c14878a595b5236de8 Mon Sep 17 00:00:00 2001 From: vladimirhasko Date: Tue, 18 Aug 2026 10:25:10 +0000 Subject: [PATCH 06/16] run.yaml: hardcode debian bookworm repo, start daemons directly - pod has no ansible_facts: drop fact templates, hardcode debian/bookworm/amd64 for the Docker CE apt repo and GPG URL - pod has no systemd as PID 1: drop systemd/service modules, launch containerd and dockerd as nohup'd background processes and wait for the sockets before verifying with docker info - gather_facts: false (facts unavailable/unnecessary in the pod) --- playbooks_k8s/run.yaml | 49 ++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/playbooks_k8s/run.yaml b/playbooks_k8s/run.yaml index bce1f05..026f8f1 100644 --- a/playbooks_k8s/run.yaml +++ b/playbooks_k8s/run.yaml @@ -8,9 +8,13 @@ # Docker is needed because cloudmon's graphite/statsd/postgres # provisioning playbooks deploy containerized components via # ansible-runner. +# +# NOTE: the pod has no systemd as PID 1, so the docker/containerd +# daemons are started directly (background processes), not via the +# service or systemd modules. - hosts: all become: true - gather_facts: true + gather_facts: false vars: docker_ce_packages: - docker-ce @@ -41,9 +45,11 @@ state: directory mode: '0755' + # Pod environment: known zuul-debian pod (Debian bookworm, amd64). + # No ansible_facts available — hardcode values. - name: Download Docker official GPG key ansible.builtin.get_url: - url: "https://download.docker.com/linux/{{ ansible_facts['os_release']['ID'] }}/gpg" + url: "https://download.docker.com/linux/debian/gpg" dest: /etc/apt/keyrings/docker.asc mode: '0644' @@ -52,10 +58,10 @@ dest: /etc/apt/sources.list.d/docker-ce.list mode: '0644' content: >- - deb [arch={{ ansible_facts['architecture'] }} + deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] - https://download.docker.com/linux/{{ ansible_facts['os_release']['ID'] }} - {{ ansible_facts['os_release']['VERSION_CODENAME'] }} stable + https://download.docker.com/linux/debian + bookworm stable # Remove conflicting Docker packages that may be in the distro image - name: Remove conflicting Docker packages @@ -88,21 +94,26 @@ - /var/run/docker.pid failed_when: false - # Start and enable containerd and docker - - name: Start and enable containerd - ansible.builtin.systemd: - name: containerd - state: started - enabled: true + # Start daemons directly — the pod has no init system to do it. + - name: Start containerd daemon + ansible.builtin.shell: + cmd: nohup /usr/bin/containerd > /var/log/containerd.log 2>&1 & + changed_when: true - - name: Start and enable Docker - ansible.builtin.systemd: - name: "{{ item }}" - state: started - enabled: true - loop: - - docker.socket - - docker.service + - name: Wait for containerd socket (up to 30s) + ansible.builtin.wait_for: + path: /run/containerd/containerd.sock + timeout: 30 + + - name: Start dockerd daemon + ansible.builtin.shell: + cmd: nohup /usr/bin/dockerd > /var/log/dockerd.log 2>&1 & + changed_when: true + + - name: Wait for Docker socket to appear (up to 60s) + ansible.builtin.wait_for: + path: /var/run/docker.sock + timeout: 60 - name: Verify Docker daemon is running ansible.builtin.command: From 3539bc6dd97e9cb8e18ffbe17dfe006576c241ed Mon Sep 17 00:00:00 2001 From: vladimirhasko Date: Tue, 18 Aug 2026 16:07:36 +0000 Subject: [PATCH 07/16] run.yaml: apt update after adding Docker CE repo The initial apt update runs before the Docker repo file is written, so the package manager has no knowledge of docker-ce when install fires. Add a second apt update after creating the .list file. --- playbooks_k8s/run.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/playbooks_k8s/run.yaml b/playbooks_k8s/run.yaml index 026f8f1..4a7f5ff 100644 --- a/playbooks_k8s/run.yaml +++ b/playbooks_k8s/run.yaml @@ -63,6 +63,12 @@ https://download.docker.com/linux/debian bookworm stable + # Must refresh apt cache after adding the Docker CE repo, otherwise + # `apt` has no idea about the `docker-ce` packages. + - name: Update apt cache after adding Docker CE repository + ansible.builtin.apt: + update_cache: true + # Remove conflicting Docker packages that may be in the distro image - name: Remove conflicting Docker packages ansible.builtin.package: From d540d2dabeb920ab8a06bfe9cbdf9072430e8440 Mon Sep 17 00:00:00 2001 From: vladimirhasko Date: Tue, 18 Aug 2026 16:22:41 +0000 Subject: [PATCH 08/16] run.yaml: retry docker info instead of single-shot verify The socket appears early but dockerd hasn't fully initialized yet, causing 'docker info' to fail on first attempt. Replace the single verify task with a 15-attempt retry loop (5s delay, up to 75s total). Also replace 'docker version --format' with 'docker --version' to avoid Jinja2 brace escaping issues. --- playbooks_k8s/run.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/playbooks_k8s/run.yaml b/playbooks_k8s/run.yaml index 4a7f5ff..26dac97 100644 --- a/playbooks_k8s/run.yaml +++ b/playbooks_k8s/run.yaml @@ -121,11 +121,21 @@ path: /var/run/docker.sock timeout: 60 - - name: Verify Docker daemon is running + - name: Wait for Docker daemon to be ready (docker info, retries) ansible.builtin.command: cmd: docker info register: docker_info changed_when: false + retries: 15 + delay: 5 + until: docker_info.rc == 0 + + - name: Show Docker version + ansible.builtin.command: + cmd: docker --version + register: docker_ver + changed_when: false + failed_when: false - hosts: all become: true From 61884b1172ac6d37c89487de4a99c7f78c4349c2 Mon Sep 17 00:00:00 2001 From: vladimirhasko Date: Tue, 18 Aug 2026 16:28:41 +0000 Subject: [PATCH 09/16] run.yaml: add diagnostic context when dockerd fails to start Replace the silent retry loop with tasks that dump: - docker info output from last attempt - dockerd process list (ps aux | grep dockerd) - last 100 lines of /var/log/dockerd.log - last 100 lines of /var/log/containerd.log Then fail with all that context so we can see what went wrong. --- playbooks_k8s/run.yaml | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/playbooks_k8s/run.yaml b/playbooks_k8s/run.yaml index 26dac97..1876b4d 100644 --- a/playbooks_k8s/run.yaml +++ b/playbooks_k8s/run.yaml @@ -121,22 +121,44 @@ path: /var/run/docker.sock timeout: 60 - - name: Wait for Docker daemon to be ready (docker info, retries) + # After 75s of retries the daemon still hasn't started. Capture + # stderr/stdout from the last attempt, show the process list, and + # dump the daemon log so we can diagnose why it fails. + - name: Show docker info output from last attempt + ansible.builtin.debug: + var: docker_info + when: docker_info is defined + + - name: Check if dockerd process exists ansible.builtin.command: - cmd: docker info - register: docker_info + cmd: ps aux | grep -v grep | grep dockerd + register: dockerd_ps changed_when: false - retries: 15 - delay: 5 - until: docker_info.rc == 0 + failed_when: false - - name: Show Docker version + - name: Show dockerd log (last 100 lines) ansible.builtin.command: - cmd: docker --version - register: docker_ver + cmd: tail -n 100 /var/log/dockerd.log + register: dockerd_tail changed_when: false failed_when: false + - name: Show containerd log (last 100 lines) + ansible.builtin.command: + cmd: tail -n 100 /var/log/containerd.log + register: containerd_tail + changed_when: false + failed_when: false + + - name: Fail with diagnostic context + ansible.builtin.fail: + msg: >- + Docker daemon failed to start after 75s. + dockerd process: {{ dockerd_ps.stdout_lines | default(['no process found']) }} + Last docker info: {{ docker_info.msg | default('N/A') }} + dockerd log:\n{{ dockerd_tail.stdout | default('no log file') }} + containerd log:\n{{ containerd_tail.stdout | default('no log file') }} + - hosts: all become: true roles: From 7b90017ef67864cfa149a54706792ddcae490337 Mon Sep 17 00:00:00 2001 From: vladimirhasko Date: Tue, 18 Aug 2026 17:37:04 +0000 Subject: [PATCH 10/16] use privileged zuul-debian-docker label + harden dockerd flags Root cause from the Zuul log: the zuul-debian pod is not privileged (no CAP_NET_ADMIN), so dockerd cannot create the DOCKER NAT chain (iptables 'Permission denied') and exits during network-controller init. The fix is a privileged nodepool label (system-config PR, same pattern as zuul-debian-builder) plus dockerd flags suited to a systemd-less pod: --exec-opt native.cgroupdriver=cgroupfs --iptables=false --ip6tables=false --bridge=none containers will run with host networking (single-node localhost inventory does not need a bridge). --- playbooks_k8s/run.yaml | 21 ++++++++++++++++++++- zuul.yaml | 2 +- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/playbooks_k8s/run.yaml b/playbooks_k8s/run.yaml index 1876b4d..c250b68 100644 --- a/playbooks_k8s/run.yaml +++ b/playbooks_k8s/run.yaml @@ -101,6 +101,19 @@ failed_when: false # Start daemons directly — the pod has no init system to do it. + # + # dockerd flags: + # --exec-opt native.cgroupdriver=cgroupfs + # The pod has no systemd; dockerd's default cgroup driver is + # "systemd" and would fail. + # --iptables=false --ip6tables=false + # The pod may lack CAP_NET_ADMIN even when "privileged" is set + # (depending on the cluster's security policy). Disabling + # iptables lets dockerd start without NAT rules; containers + # use host networking which is all the provision playbooks need + # (single-node, localhost inventory). + # --bridge=none + # Same reason — no bridge network needed. - name: Start containerd daemon ansible.builtin.shell: cmd: nohup /usr/bin/containerd > /var/log/containerd.log 2>&1 & @@ -113,7 +126,13 @@ - name: Start dockerd daemon ansible.builtin.shell: - cmd: nohup /usr/bin/dockerd > /var/log/dockerd.log 2>&1 & + cmd: >- + nohup /usr/bin/dockerd + --exec-opt native.cgroupdriver=cgroupfs + --iptables=false + --ip6tables=false + --bridge=none + > /var/log/dockerd.log 2>&1 & changed_when: true - name: Wait for Docker socket to appear (up to 60s) diff --git a/zuul.yaml b/zuul.yaml index 8606646..112f7ce 100644 --- a/zuul.yaml +++ b/zuul.yaml @@ -43,7 +43,7 @@ nodeset: nodes: - name: zuul-debian - label: zuul-debian + label: zuul-debian-docker - project: merge-mode: squash-merge From b339c36cac6e3513564187e49d62724747a6c465 Mon Sep 17 00:00:00 2001 From: Vladimir Hasko Date: Thu, 20 Aug 2026 11:40:25 +0200 Subject: [PATCH 11/16] Update zuul.yaml --- zuul.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zuul.yaml b/zuul.yaml index 112f7ce..b185319 100644 --- a/zuul.yaml +++ b/zuul.yaml @@ -43,7 +43,7 @@ nodeset: nodes: - name: zuul-debian - label: zuul-debian-docker + label: zuul-debian-builder - project: merge-mode: squash-merge From 5573681471c9819a18a61026a6543615f067fb90 Mon Sep 17 00:00:00 2001 From: Vladimir Hasko Date: Thu, 20 Aug 2026 11:44:36 +0200 Subject: [PATCH 12/16] Update zuul.yaml --- zuul.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zuul.yaml b/zuul.yaml index b185319..112f7ce 100644 --- a/zuul.yaml +++ b/zuul.yaml @@ -43,7 +43,7 @@ nodeset: nodes: - name: zuul-debian - label: zuul-debian-builder + label: zuul-debian-docker - project: merge-mode: squash-merge From 3d91fed86115d2df704bfa1f78f32b4997aa1de0 Mon Sep 17 00:00:00 2001 From: LukasCuperDT <152291305+LukasCuperDT@users.noreply.github.com> Date: Thu, 20 Aug 2026 22:44:22 +0200 Subject: [PATCH 13/16] Fix cloudmon-deploy-zuul-debian job (#38) * Fix the docker daemon diagnostic block The 'Fail with diagnostic context' task had no when condition, so the play failed even when docker started correctly. Conversely the wait_for above it aborted the play on real failure, so the diagnostics it was meant to collect never ran in the case they were written for. Make wait_for non-fatal and put the diagnostics and the failure behind 'docker_sock is failed', so they run exactly when the socket never appeared. Also drop the reference to docker_info, which is never registered, and use pgrep instead of 'ps aux | grep' under ansible.builtin.command, which does not interpret pipes - those characters were being passed to ps as literal arguments. * Install python packaging and venv tooling before cloudmon roles install_cloudmon failed on the zuul-debian-docker node with: ModuleNotFoundError: No module named 'packaging' ansible.builtin.pip imports packaging on the target, and cloudmon_venv_path defaults to ~/.venv so the module also has to build a virtualenv. The node image provides neither, so install them before the roles run. * Surface cloudmon provision output instead of a bare rc The Deploy task ran graphite, statsd and postgres provisioning in one shell block with no 'set -e', so rc came from the last command only and an earlier failure would pass unnoticed. On failure Ansible reported just 'non-zero return code' with no stdout/stderr, and job-output.json is not uploaded, so there was nothing to debug from. Run one component per loop item, print rc/stdout/stderr for each, then fail listing the components that actually failed. * Fix inventory template: group hosts must be a mapping Every child group declared its hosts as a list: statsd: hosts: - localhost Ansible's yaml inventory plugin requires a mapping there and rejected the whole file: Invalid "hosts" entry for "statsd" group, requires a dictionary With no inventory parsed, every provisioning play matched no hosts. graphite still exited 0 while doing nothing ('skipping: no hosts matched'), so the failure surfaced only as statsd/postgres rc=1. Reproduced with ansible-inventory: the old template raises the exact error, the new one resolves all 8 groups to localhost. * Use local connection for localhost in cloudmon inventory Provisioning failed with: ssh: connect to host localhost port 22: Connection refused fatal: [localhost]: UNREACHABLE! Listing localhost explicitly in the inventory overrides Ansible's implicit local connection, so it fell back to ssh. The zuul-debian pod runs no sshd, so set ansible_connection: local. * Allow skipping host firewall management Graphite provisioning failed in the pod-based job with: Could not find the requested service firewalld: host The firewalld role is pulled in unconditionally by seven install playbooks and manages host services. A container has no service manager and no host firewall, so those tasks cannot succeed. 'Disable iptables' already had ignore_errors; 'Enable firewalld' did not, so it aborted the play. Gate the role body behind cloudmon_manage_firewall, defaulting to true so VM-based deployments are unchanged, and set it false in the k8s job inventory. * Use the block keyword instead of ansible.builtin.block Provisioning aborted with: unexpected parameter type in action: Origin: roles/graphite/tasks/main.yaml:38:3 'block' is a task keyword, not a module -- there is no ansible.builtin.block action, so ansible-core treats it as a module receiving a list. Fixes all 22 occurrences across 11 roles, introduced in 685afcf. * Use netcat-openbsd on Debian Package installation failed on the Debian bookworm node: E: Package 'netcat' has no installation candidate Package netcat is a virtual package provided by: netcat-openbsd 1.219-1 netcat-traditional 1.10-47 'netcat' is virtual on bookworm and cannot be installed directly. Use netcat-openbsd, which provides nc, in the graphite, statsd and carbonapi Debian vars. * Gate per-role firewalld tasks too Graphite provisioning still failed after gating the firewalld role: TASK [graphite : Allow graphite port] Failed to import the required Python library (firewall) Origin: roles/graphite/tasks/firewall.yml:4:3 Six roles include their own firewall.yml to open ports via ansible.posix .firewalld, which needs the firewall python library and a running firewalld. Neither exists in a container, so gate these includes behind the same cloudmon_manage_firewall variable. --- .../roles/apimon_executor/tasks/main.yml | 4 +- .../roles/apimon_scheduler/tasks/main.yml | 5 +- .../project/roles/carbonapi/tasks/main.yaml | 4 +- .../project/roles/carbonapi/vars/Debian.yaml | 2 +- .../project/roles/epmon/tasks/main.yml | 4 +- .../project/roles/etcd/tasks/main.yaml | 5 +- .../project/roles/firewalld/tasks/main.yaml | 71 ++------------ .../project/roles/firewalld/tasks/manage.yaml | 64 +++++++++++++ .../project/roles/globalmon/tasks/main.yml | 4 +- .../project/roles/grafana/tasks/main.yml | 5 +- .../project/roles/graphite/tasks/main.yaml | 5 +- .../project/roles/graphite/vars/Debian.yaml | 2 +- .../project/roles/postgresql/tasks/main.yaml | 5 +- .../roles/postgresql_ha/tasks/main.yaml | 5 +- .../project/roles/statsd/tasks/main.yaml | 4 +- .../project/roles/statsd/vars/Debian.yaml | 2 +- playbooks_k8s/run.yaml | 93 +++++++++++-------- roles/deploy_cloudmon_k8s/tasks/main.yaml | 34 ++++++- .../templates/inventory.j2 | 26 ++++-- 19 files changed, 206 insertions(+), 138 deletions(-) create mode 100644 cloudmon/ansible/project/roles/firewalld/tasks/manage.yaml diff --git a/cloudmon/ansible/project/roles/apimon_executor/tasks/main.yml b/cloudmon/ansible/project/roles/apimon_executor/tasks/main.yml index be8ef36..736c573 100644 --- a/cloudmon/ansible/project/roles/apimon_executor/tasks/main.yml +++ b/cloudmon/ansible/project/roles/apimon_executor/tasks/main.yml @@ -36,7 +36,7 @@ retries: 5 - name: Clean up stale sockets after package changes - ansible.builtin.block: + block: - name: Stop Docker service and socket ansible.builtin.systemd: name: "{{ item }}" @@ -59,7 +59,7 @@ - ansible_facts.distribution_major_version == '22' - name: Clear systemd rate-limits and prepare containerd - ansible.builtin.block: + block: - name: Reset systemd failed state for Docker units ansible.builtin.systemd: name: "{{ item }}" diff --git a/cloudmon/ansible/project/roles/apimon_scheduler/tasks/main.yml b/cloudmon/ansible/project/roles/apimon_scheduler/tasks/main.yml index d5f73ca..1c1744f 100644 --- a/cloudmon/ansible/project/roles/apimon_scheduler/tasks/main.yml +++ b/cloudmon/ansible/project/roles/apimon_scheduler/tasks/main.yml @@ -38,7 +38,7 @@ retries: 5 - name: Clean up stale sockets after package changes - ansible.builtin.block: + block: - name: Stop Docker service and socket ansible.builtin.systemd: name: "{{ item }}" @@ -61,7 +61,7 @@ - ansible_facts.distribution_major_version == '22' - name: Clear systemd rate-limits and prepare containerd - ansible.builtin.block: + block: - name: Reset systemd failed state for Docker units ansible.builtin.systemd: name: "{{ item }}" @@ -93,6 +93,7 @@ - ansible_facts.distribution_major_version == '22' - include_tasks: firewall.yml + when: cloudmon_manage_firewall | default(true) | bool - name: Create scheduler group become: yes diff --git a/cloudmon/ansible/project/roles/carbonapi/tasks/main.yaml b/cloudmon/ansible/project/roles/carbonapi/tasks/main.yaml index 89e2cb6..23e4109 100644 --- a/cloudmon/ansible/project/roles/carbonapi/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/carbonapi/tasks/main.yaml @@ -36,7 +36,7 @@ retries: 5 - name: Clean up stale sockets after package changes - ansible.builtin.block: + block: - name: Stop Docker service and socket ansible.builtin.systemd: name: "{{ item }}" @@ -59,7 +59,7 @@ - ansible_facts.distribution_major_version == '22' - name: Clear systemd rate-limits and prepare containerd - ansible.builtin.block: + block: - name: Reset systemd failed state for Docker units ansible.builtin.systemd: name: "{{ item }}" diff --git a/cloudmon/ansible/project/roles/carbonapi/vars/Debian.yaml b/cloudmon/ansible/project/roles/carbonapi/vars/Debian.yaml index 1505647..69b51f0 100644 --- a/cloudmon/ansible/project/roles/carbonapi/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/carbonapi/vars/Debian.yaml @@ -6,7 +6,7 @@ remove_packages: [] packages: - - netcat + - netcat-openbsd - policycoreutils-python-utils - python3-selinux diff --git a/cloudmon/ansible/project/roles/epmon/tasks/main.yml b/cloudmon/ansible/project/roles/epmon/tasks/main.yml index ba9b0fe..4139183 100644 --- a/cloudmon/ansible/project/roles/epmon/tasks/main.yml +++ b/cloudmon/ansible/project/roles/epmon/tasks/main.yml @@ -37,7 +37,7 @@ retries: 5 - name: Clean up stale sockets after package changes - ansible.builtin.block: + block: - name: Stop Docker service and socket ansible.builtin.systemd: name: "{{ item }}" @@ -60,7 +60,7 @@ - ansible_facts.distribution_major_version == '22' - name: Clear systemd rate-limits and prepare containerd - ansible.builtin.block: + block: - name: Reset systemd failed state for Docker units ansible.builtin.systemd: name: "{{ item }}" diff --git a/cloudmon/ansible/project/roles/etcd/tasks/main.yaml b/cloudmon/ansible/project/roles/etcd/tasks/main.yaml index 8c46752..076d958 100644 --- a/cloudmon/ansible/project/roles/etcd/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/etcd/tasks/main.yaml @@ -36,7 +36,7 @@ retries: 5 - name: Clean up stale sockets after package changes - ansible.builtin.block: + block: - name: Stop Docker service and socket ansible.builtin.systemd: name: "{{ item }}" @@ -59,7 +59,7 @@ - ansible_facts.distribution_major_version == '22' - name: Clear systemd rate-limits and prepare containerd - ansible.builtin.block: + block: - name: Reset systemd failed state for Docker units ansible.builtin.systemd: name: "{{ item }}" @@ -96,6 +96,7 @@ loop: "{{ play_hosts }}" - ansible.builtin.include_tasks: firewall.yml + when: cloudmon_manage_firewall | default(true) | bool - name: Ensure directories exist become: true diff --git a/cloudmon/ansible/project/roles/firewalld/tasks/main.yaml b/cloudmon/ansible/project/roles/firewalld/tasks/main.yaml index b6e6d0f..a5caca4 100644 --- a/cloudmon/ansible/project/roles/firewalld/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/firewalld/tasks/main.yaml @@ -1,64 +1,7 @@ -- name: Include OS-specific variables - include_vars: "{{ lookup('first_found', params) }}" - vars: - params: - files: "{{ distro_lookup_path }}" - paths: - - 'vars' - -- name: Install firewalld - ansible.builtin.package: - name: "{{ item }}" - state: "present" - loop: - - "{{ packages }}" - when: "ansible_facts.pkg_mgr != 'atomic_container'" - -- name: Enable services - ansible.posix.firewalld: - permanent: "yes" - service: "{{ item }}" - state: "enabled" - loop: "{{ firewalld_services_enable }}" - notify: - - Reload firewalld - -- name: Disable services - ansible.posix.firewalld: - permanent: "yes" - service: "{{ item }}" - state: "disabled" - loop: "{{ firewalld_services_disable }}" - notify: - - Reload firewalld - -- name: Enable ports - ansible.posix.firewalld: - permanent: "yes" - port: "{{ item }}" - state: "enabled" - loop: "{{ firewalld_ports_enable }}" - notify: - - Reload firewalld - -- name: Disable ports - ansible.posix.firewalld: - permanent: "yes" - port: "{{ item }}" - state: "disabled" - loop: "{{ firewalld_ports_disable }}" - notify: - - Reload firewalld - -- name: Disable iptables - ansible.builtin.service: - name: "iptables" - state: "stopped" - enabled: "false" - ignore_errors: "true" - -- name: Enable firewalld - ansible.builtin.service: - name: "firewalld" - state: "started" - enabled: "true" +--- +# Host firewall management is meaningless in a container (no service manager, +# no host firewall), so allow targets to opt out. Defaults to true, so +# existing VM-based deployments are unaffected. +- name: Manage host firewall + ansible.builtin.include_tasks: manage.yaml + when: cloudmon_manage_firewall | default(true) | bool diff --git a/cloudmon/ansible/project/roles/firewalld/tasks/manage.yaml b/cloudmon/ansible/project/roles/firewalld/tasks/manage.yaml new file mode 100644 index 0000000..b6e6d0f --- /dev/null +++ b/cloudmon/ansible/project/roles/firewalld/tasks/manage.yaml @@ -0,0 +1,64 @@ +- name: Include OS-specific variables + include_vars: "{{ lookup('first_found', params) }}" + vars: + params: + files: "{{ distro_lookup_path }}" + paths: + - 'vars' + +- name: Install firewalld + ansible.builtin.package: + name: "{{ item }}" + state: "present" + loop: + - "{{ packages }}" + when: "ansible_facts.pkg_mgr != 'atomic_container'" + +- name: Enable services + ansible.posix.firewalld: + permanent: "yes" + service: "{{ item }}" + state: "enabled" + loop: "{{ firewalld_services_enable }}" + notify: + - Reload firewalld + +- name: Disable services + ansible.posix.firewalld: + permanent: "yes" + service: "{{ item }}" + state: "disabled" + loop: "{{ firewalld_services_disable }}" + notify: + - Reload firewalld + +- name: Enable ports + ansible.posix.firewalld: + permanent: "yes" + port: "{{ item }}" + state: "enabled" + loop: "{{ firewalld_ports_enable }}" + notify: + - Reload firewalld + +- name: Disable ports + ansible.posix.firewalld: + permanent: "yes" + port: "{{ item }}" + state: "disabled" + loop: "{{ firewalld_ports_disable }}" + notify: + - Reload firewalld + +- name: Disable iptables + ansible.builtin.service: + name: "iptables" + state: "stopped" + enabled: "false" + ignore_errors: "true" + +- name: Enable firewalld + ansible.builtin.service: + name: "firewalld" + state: "started" + enabled: "true" diff --git a/cloudmon/ansible/project/roles/globalmon/tasks/main.yml b/cloudmon/ansible/project/roles/globalmon/tasks/main.yml index 3061cc8..0a95f0f 100644 --- a/cloudmon/ansible/project/roles/globalmon/tasks/main.yml +++ b/cloudmon/ansible/project/roles/globalmon/tasks/main.yml @@ -42,7 +42,7 @@ retries: 5 - name: Clean up stale sockets after package changes - ansible.builtin.block: + block: - name: Stop Docker service and socket ansible.builtin.systemd: name: "{{ item }}" @@ -65,7 +65,7 @@ - ansible_facts.distribution_major_version == '22' - name: Clear systemd rate-limits and prepare containerd - ansible.builtin.block: + block: - name: Reset systemd failed state for Docker units ansible.builtin.systemd: name: "{{ item }}" diff --git a/cloudmon/ansible/project/roles/grafana/tasks/main.yml b/cloudmon/ansible/project/roles/grafana/tasks/main.yml index 6ce8ccc..24cf8e8 100644 --- a/cloudmon/ansible/project/roles/grafana/tasks/main.yml +++ b/cloudmon/ansible/project/roles/grafana/tasks/main.yml @@ -36,7 +36,7 @@ retries: 5 - name: Clean up stale sockets after package changes - ansible.builtin.block: + block: - name: Stop Docker service and socket ansible.builtin.systemd: name: "{{ item }}" @@ -59,7 +59,7 @@ - ansible_facts.distribution_major_version == '22' - name: Clear systemd rate-limits and prepare containerd - ansible.builtin.block: + block: - name: Reset systemd failed state for Docker units ansible.builtin.systemd: name: "{{ item }}" @@ -91,6 +91,7 @@ - ansible_facts.distribution_major_version == '22' - include_tasks: firewall.yml + when: cloudmon_manage_firewall | default(true) | bool - name: Create Grafana group ansible.builtin.group: diff --git a/cloudmon/ansible/project/roles/graphite/tasks/main.yaml b/cloudmon/ansible/project/roles/graphite/tasks/main.yaml index 2011deb..11a44ea 100644 --- a/cloudmon/ansible/project/roles/graphite/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/graphite/tasks/main.yaml @@ -36,7 +36,7 @@ retries: 5 - name: Clean up stale sockets after package changes - ansible.builtin.block: + block: - name: Stop Docker service and socket ansible.builtin.systemd: name: "{{ item }}" @@ -59,7 +59,7 @@ - ansible_facts.distribution_major_version == '22' - name: Clear systemd rate-limits and prepare containerd - ansible.builtin.block: + block: - name: Reset systemd failed state for Docker units ansible.builtin.systemd: name: "{{ item }}" @@ -91,6 +91,7 @@ - ansible_facts.distribution_major_version == '22' - include_tasks: firewall.yml + when: cloudmon_manage_firewall | default(true) | bool - name: Create graphite sync group become: yes diff --git a/cloudmon/ansible/project/roles/graphite/vars/Debian.yaml b/cloudmon/ansible/project/roles/graphite/vars/Debian.yaml index 001aa59..3d95ca0 100644 --- a/cloudmon/ansible/project/roles/graphite/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/graphite/vars/Debian.yaml @@ -6,6 +6,6 @@ remove_packages: [] packages: - - netcat + - netcat-openbsd container_command: docker diff --git a/cloudmon/ansible/project/roles/postgresql/tasks/main.yaml b/cloudmon/ansible/project/roles/postgresql/tasks/main.yaml index 29eaa2c..ac12375 100644 --- a/cloudmon/ansible/project/roles/postgresql/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/postgresql/tasks/main.yaml @@ -36,7 +36,7 @@ retries: 5 - name: Clean up stale sockets after package changes - ansible.builtin.block: + block: - name: Stop Docker service and socket ansible.builtin.systemd: name: "{{ item }}" @@ -59,7 +59,7 @@ - ansible_facts.distribution_major_version == '22' - name: Clear systemd rate-limits and prepare containerd - ansible.builtin.block: + block: - name: Reset systemd failed state for Docker units ansible.builtin.systemd: name: "{{ item }}" @@ -91,6 +91,7 @@ - ansible_facts.distribution_major_version == '22' - include_tasks: firewall.yml + when: cloudmon_manage_firewall | default(true) | bool - name: Write Postgres Systemd unit file become: true diff --git a/cloudmon/ansible/project/roles/postgresql_ha/tasks/main.yaml b/cloudmon/ansible/project/roles/postgresql_ha/tasks/main.yaml index 9e3f853..3477d19 100644 --- a/cloudmon/ansible/project/roles/postgresql_ha/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/postgresql_ha/tasks/main.yaml @@ -36,7 +36,7 @@ retries: 5 - name: Clean up stale sockets after package changes - ansible.builtin.block: + block: - name: Stop Docker service and socket ansible.builtin.systemd: name: "{{ item }}" @@ -59,7 +59,7 @@ - ansible_facts.distribution_major_version == '22' - name: Clear systemd rate-limits and prepare containerd - ansible.builtin.block: + block: - name: Reset systemd failed state for Docker units ansible.builtin.systemd: name: "{{ item }}" @@ -102,6 +102,7 @@ when: "item != inventory_hostname" - ansible.builtin.include_tasks: firewall.yml + when: cloudmon_manage_firewall | default(true) | bool - name: Create postgres group become: yes diff --git a/cloudmon/ansible/project/roles/statsd/tasks/main.yaml b/cloudmon/ansible/project/roles/statsd/tasks/main.yaml index fee2ae1..4b3e894 100644 --- a/cloudmon/ansible/project/roles/statsd/tasks/main.yaml +++ b/cloudmon/ansible/project/roles/statsd/tasks/main.yaml @@ -36,7 +36,7 @@ when: "ansible_facts.pkg_mgr != 'atomic_container'" - name: Clean up stale sockets after package changes - ansible.builtin.block: + block: - name: Stop Docker service and socket ansible.builtin.systemd: name: "{{ item }}" @@ -59,7 +59,7 @@ - ansible_facts.distribution_major_version == '22' - name: Clear systemd rate-limits and prepare containerd - ansible.builtin.block: + block: - name: Reset systemd failed state for Docker units ansible.builtin.systemd: name: "{{ item }}" diff --git a/cloudmon/ansible/project/roles/statsd/vars/Debian.yaml b/cloudmon/ansible/project/roles/statsd/vars/Debian.yaml index fdaaeea..112be38 100644 --- a/cloudmon/ansible/project/roles/statsd/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/statsd/vars/Debian.yaml @@ -6,7 +6,7 @@ remove_packages: [] distro_packages: - - netcat + - netcat-openbsd - cron - apparmor-profiles diff --git a/playbooks_k8s/run.yaml b/playbooks_k8s/run.yaml index c250b68..07ffff6 100644 --- a/playbooks_k8s/run.yaml +++ b/playbooks_k8s/run.yaml @@ -139,47 +139,64 @@ ansible.builtin.wait_for: path: /var/run/docker.sock timeout: 60 - - # After 75s of retries the daemon still hasn't started. Capture - # stderr/stdout from the last attempt, show the process list, and - # dump the daemon log so we can diagnose why it fails. - - name: Show docker info output from last attempt - ansible.builtin.debug: - var: docker_info - when: docker_info is defined - - - name: Check if dockerd process exists - ansible.builtin.command: - cmd: ps aux | grep -v grep | grep dockerd - register: dockerd_ps - changed_when: false - failed_when: false - - - name: Show dockerd log (last 100 lines) - ansible.builtin.command: - cmd: tail -n 100 /var/log/dockerd.log - register: dockerd_tail - changed_when: false - failed_when: false - - - name: Show containerd log (last 100 lines) - ansible.builtin.command: - cmd: tail -n 100 /var/log/containerd.log - register: containerd_tail - changed_when: false - failed_when: false - - - name: Fail with diagnostic context - ansible.builtin.fail: - msg: >- - Docker daemon failed to start after 75s. - dockerd process: {{ dockerd_ps.stdout_lines | default(['no process found']) }} - Last docker info: {{ docker_info.msg | default('N/A') }} - dockerd log:\n{{ dockerd_tail.stdout | default('no log file') }} - containerd log:\n{{ containerd_tail.stdout | default('no log file') }} + register: docker_sock + ignore_errors: true + + # Only runs when the socket never appeared. wait_for above is + # non-fatal so these diagnostics can be collected before failing; + # previously it aborted the play, so they never ran on failure, and + # the fail below had no condition, so it always ran on success. + - name: Diagnose why the docker daemon did not start + when: docker_sock is failed + block: + - name: Check if dockerd process exists + ansible.builtin.command: + cmd: pgrep -a dockerd + register: dockerd_ps + changed_when: false + failed_when: false + + - name: Show dockerd log (last 100 lines) + ansible.builtin.command: + cmd: tail -n 100 /var/log/dockerd.log + register: dockerd_tail + changed_when: false + failed_when: false + + - name: Show containerd log (last 100 lines) + ansible.builtin.command: + cmd: tail -n 100 /var/log/containerd.log + register: containerd_tail + changed_when: false + failed_when: false + + - name: Fail with diagnostic context + ansible.builtin.fail: + msg: |- + Docker daemon failed to start after 60s. + dockerd process: {{ dockerd_ps.stdout_lines | default(['no process found']) }} + dockerd log: + {{ dockerd_tail.stdout | default('no log file') }} + containerd log: + {{ containerd_tail.stdout | default('no log file') }} - hosts: all become: true + pre_tasks: + # The node image ships none of these: ansible.builtin.pip imports + # `packaging` on the target, and cloudmon_venv_path makes it build a + # venv, which needs the venv/virtualenv tooling. + - name: Install Python packaging and venv tooling + ansible.builtin.apt: + name: + - python3-packaging + - python3-pip + - python3-venv + - virtualenv + state: present + register: py_prereq + retries: 5 + until: py_prereq is success roles: - install_cloudmon - deploy_cloudmon_k8s diff --git a/roles/deploy_cloudmon_k8s/tasks/main.yaml b/roles/deploy_cloudmon_k8s/tasks/main.yaml index 8dcc5c0..5e4b568 100644 --- a/roles/deploy_cloudmon_k8s/tasks/main.yaml +++ b/roles/deploy_cloudmon_k8s/tasks/main.yaml @@ -10,9 +10,37 @@ executable: "/bin/bash" ansible.builtin.shell: | source {{ cloudmon_venv_path }}/bin/activate - cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure graphite provision - cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure statsd provision - cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure postgres provision + cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure {{ item }} provision + loop: + - graphite + - statsd + - postgres + register: cloudmon_provision + failed_when: false + +# Run one component per task and report before failing. Previously all three +# ran in a single shell block with no `set -e`, so rc came from the last +# command only and the output was never surfaced -- a failure showed up as a +# bare "non-zero return code" with nothing to debug. +- name: Show cloudmon provision output + ansible.builtin.debug: + msg: |- + {{ item.item }} rc={{ item.rc }} + stdout: + {{ item.stdout | default('') }} + stderr: + {{ item.stderr | default('') }} + loop: "{{ cloudmon_provision.results }}" + loop_control: + label: "{{ item.item }}" + +- name: Fail if any cloudmon provision step failed + ansible.builtin.fail: + msg: >- + cloudmon provision failed for: + {{ cloudmon_provision.results | selectattr('rc', 'ne', 0) + | map(attribute='item') | list }} + when: cloudmon_provision.results | selectattr('rc', 'ne', 0) | list | length > 0 # cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure grafana provision # cloudmon --config etc/sample_config.yaml --inventory {{ ansible_user_dir }}/{{ zuul_work_dir }}/inventory --insecure apimon provision diff --git a/roles/deploy_cloudmon_k8s/templates/inventory.j2 b/roles/deploy_cloudmon_k8s/templates/inventory.j2 index e7c8bac..99387a7 100644 --- a/roles/deploy_cloudmon_k8s/templates/inventory.j2 +++ b/roles/deploy_cloudmon_k8s/templates/inventory.j2 @@ -5,29 +5,39 @@ all: # "localhost" from the pod's point of view. localhost: ansible_host: localhost + # Defining localhost explicitly loses Ansible's implicit local + # connection, and the pod runs no sshd, so this must be set. + ansible_connection: local internal_address: localhost + vars: + # The pod has no service manager and no host firewall, so firewalld and + # iptables service management cannot work here. + cloudmon_manage_firewall: false children: + # A group's `hosts` must be a mapping, not a list -- a list makes Ansible + # reject the whole inventory ("requires a dictionary"), after which every + # play matches no hosts. statsd: hosts: - - localhost + localhost: graphite: hosts: - - localhost + localhost: grafana: hosts: - - localhost + localhost: schedulers: hosts: - - localhost + localhost: executors: hosts: - - localhost + localhost: epmons: hosts: - - localhost + localhost: globalmons: hosts: - - localhost + localhost: postgres: hosts: - - localhost + localhost: From 658e7247cdfe02dc7417dd8e3fbf4fc38256baf1 Mon Sep 17 00:00:00 2001 From: LukasCuperDT <152291305+LukasCuperDT@users.noreply.github.com> Date: Thu, 20 Aug 2026 23:38:19 +0200 Subject: [PATCH 14/16] Install cron for the graphite cleanup crontab (#39) Graphite provisioning failed with: Failed to find required executable "crontab" in paths: ... The rewritten Debian vars dropped 'cron', which main still lists. The graphite role is the only one using ansible.builtin.cron ('Run periodic cleanup'), and that module needs the crontab binary. apparmor-profiles was dropped in the same rewrite but is never referenced by any task, so it stays out. --- cloudmon/ansible/project/roles/graphite/vars/Debian.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cloudmon/ansible/project/roles/graphite/vars/Debian.yaml b/cloudmon/ansible/project/roles/graphite/vars/Debian.yaml index 3d95ca0..cc7e97b 100644 --- a/cloudmon/ansible/project/roles/graphite/vars/Debian.yaml +++ b/cloudmon/ansible/project/roles/graphite/vars/Debian.yaml @@ -7,5 +7,7 @@ remove_packages: [] packages: - netcat-openbsd + # ansible.builtin.cron needs the crontab binary. + - cron container_command: docker From 4e9eb89d36f9244e4148789f06c1a259e1600382 Mon Sep 17 00:00:00 2001 From: LukasCuperDT <152291305+LukasCuperDT@users.noreply.github.com> Date: Fri, 21 Aug 2026 08:21:46 +0200 Subject: [PATCH 15/16] Run the deploy job on a systemd node (#40) Provisioning failed once it reached the application layer: System has not been booted with systemd as init system (PID 1). Can't operate. Failed to connect to bus: Host is down cloudmon deploys through systemd -- 15 unit templates, 68 systemd tasks and 14 daemon_reload calls -- so the node needs a real init. Switch the nodeset to the new zuul-debian-systemd label, which runs systemd as PID 1. With systemd present, let it own containerd and dockerd instead of starting them with nohup; hand-started daemons would collide with the units the cloudmon roles enable. The daemon settings now come from /etc/docker/daemon.json seeded by the pod spec. Diagnostics read the journal instead of the log files the manual start used to write. Verified on a probe pod on the cluster: PID 1 is systemd, is-system-running reports running, daemon-reload succeeds, and a unit written at runtime can be enabled and started. --- playbooks_k8s/run.yaml | 62 +++++++++++++++--------------------------- zuul.yaml | 2 +- 2 files changed, 23 insertions(+), 41 deletions(-) diff --git a/playbooks_k8s/run.yaml b/playbooks_k8s/run.yaml index 07ffff6..d39c8c5 100644 --- a/playbooks_k8s/run.yaml +++ b/playbooks_k8s/run.yaml @@ -100,40 +100,22 @@ - /var/run/docker.pid failed_when: false - # Start daemons directly — the pod has no init system to do it. - # - # dockerd flags: - # --exec-opt native.cgroupdriver=cgroupfs - # The pod has no systemd; dockerd's default cgroup driver is - # "systemd" and would fail. - # --iptables=false --ip6tables=false - # The pod may lack CAP_NET_ADMIN even when "privileged" is set - # (depending on the cluster's security policy). Disabling - # iptables lets dockerd start without NAT rules; containers - # use host networking which is all the provision playbooks need - # (single-node, localhost inventory). - # --bridge=none - # Same reason — no bridge network needed. - - name: Start containerd daemon - ansible.builtin.shell: - cmd: nohup /usr/bin/containerd > /var/log/containerd.log 2>&1 & - changed_when: true - - - name: Wait for containerd socket (up to 30s) - ansible.builtin.wait_for: - path: /run/containerd/containerd.sock - timeout: 30 - - - name: Start dockerd daemon - ansible.builtin.shell: - cmd: >- - nohup /usr/bin/dockerd - --exec-opt native.cgroupdriver=cgroupfs - --iptables=false - --ip6tables=false - --bridge=none - > /var/log/dockerd.log 2>&1 & - changed_when: true + # The node runs systemd as PID 1 (zuul-debian-systemd label), so let it own + # the daemons. Starting them by hand would collide with the units the + # cloudmon roles later enable. Daemon settings (MTU, cgroupdriver, + # iptables/bridge) are seeded into /etc/docker/daemon.json by the pod spec. + - name: Start containerd via systemd + ansible.builtin.systemd: + name: containerd + state: started + enabled: true + daemon_reload: true + + - name: Start dockerd via systemd + ansible.builtin.systemd: + name: docker + state: started + enabled: true - name: Wait for Docker socket to appear (up to 60s) ansible.builtin.wait_for: @@ -158,14 +140,14 @@ - name: Show dockerd log (last 100 lines) ansible.builtin.command: - cmd: tail -n 100 /var/log/dockerd.log + cmd: journalctl -u docker --no-pager -n 100 register: dockerd_tail changed_when: false failed_when: false - name: Show containerd log (last 100 lines) ansible.builtin.command: - cmd: tail -n 100 /var/log/containerd.log + cmd: journalctl -u containerd --no-pager -n 100 register: containerd_tail changed_when: false failed_when: false @@ -175,10 +157,10 @@ msg: |- Docker daemon failed to start after 60s. dockerd process: {{ dockerd_ps.stdout_lines | default(['no process found']) }} - dockerd log: - {{ dockerd_tail.stdout | default('no log file') }} - containerd log: - {{ containerd_tail.stdout | default('no log file') }} + docker journal: + {{ dockerd_tail.stdout | default('no journal output') }} + containerd journal: + {{ containerd_tail.stdout | default('no journal output') }} - hosts: all become: true diff --git a/zuul.yaml b/zuul.yaml index 112f7ce..a913806 100644 --- a/zuul.yaml +++ b/zuul.yaml @@ -43,7 +43,7 @@ nodeset: nodes: - name: zuul-debian - label: zuul-debian-docker + label: zuul-debian-systemd - project: merge-mode: squash-merge From 49af29cfbc4135edfaef24ab1b80bc98e8dc0dac Mon Sep 17 00:00:00 2001 From: LukasCuperDT <152291305+LukasCuperDT@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:10:22 +0200 Subject: [PATCH 16/16] Install cron for the graphite cleanup crontab (#39) Graphite provisioning failed with: Failed to find required executable "crontab" in paths: ... The rewritten Debian vars dropped 'cron', which main still lists. The graphite role is the only one using ansible.builtin.cron ('Run periodic cleanup'), and that module needs the crontab binary. apparmor-profiles was dropped in the same rewrite but is never referenced by any task, so it stays out.