Feature/logstash containerized

roles/filebeat/tasks/main.yml

@@ -38,7 +38,7 @@
 
         - name: "Copy Elasticsearch GPG key"
           ansible.builtin.copy:
-            src: "/files/GPG-KEY-elasticsearch"
+            src: "files/GPG-KEY-elasticsearch"
             dest: "/tmp/GPG-KEY-elasticsearch"
             mode: '0644'
 

roles/influxdb_docker/README.md

@@ -0,0 +1,32 @@
+influxdb_docker
+=========
+
+This role deploys a influxdb docker container
+
+Requirements
+------------
+
+Requires a logstash instance to fill the influxdb. The influxdb instance is used by the stats container. The OpenConext-deploy roles "logstash_docker" and "stats" can provide this.
+
+Role Variables
+--------------
+
+The following variables must be set:
+ - influxdb_host (defaults to localhost, is used for setting user permissions within the database)
+ - influx_stats_db (name of the database to be created)
+ - influxdb_admin_user (admin user for influxdb)
+ - influxdb_admin_password (admin password for influxdb, make sure this is securely stored for example in a vault)
+ - influxdb_stats_user (user for accessing the database)
+ - influxdb_stats_password (user password for influxdb, make sure this is securely stored for example in a vault)
+
+
+License
+--------------
+
+These files are licensed under version 2.0 of the Apache License, as described in the file [LICENSE](LICENSE).
+
+Support
+--------------
+
+* You can ask questions on the [OpenConext mailing list](https://openconext.org/get-involved/mailing-lists/) 
+* Or you can join our [Slack Workspace](https://edu.nl/ocslk)

roles/influxdb_docker/defaults/main.yml

@@ -0,0 +1,8 @@
+---
+influxdb_host: "localhost"
+# The following variables are not provided but required to run this role. Define these in your inventory or vault:
+# influx_stats_db:
+# influxdb_admin_user:
+# influxdb_admin_password:
+# influxdb_stats_user:
+# influxdb_stats_password:

roles/influxdb_docker/files/influxdb_backup

@@ -0,0 +1,2 @@
+0 3 * * * root docker exec influxdb influxd backup -portable /data/influxdb-backup/$(date +\%F)
+0 4 * * * root find /data/influxdb-backup/ -type f -mtime +1 -exec rm {} \;

roles/influxdb_docker/tasks/main.yml

@@ -0,0 +1,89 @@
+---
+- name: Create influx user
+  ansible.builtin.user:
+    name: influxdb
+    uid: "1500"
+
+- name: Ensure required host directories exists
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    owner: "influxdb"
+    group: "influxdb"
+    mode: "0755"
+  with_items:
+    - "/opt/openconext/influxdb/etc"
+    - "/opt/openconext/influxdb/varlib"
+    - "/data/influxdb-backup"
+
+- name: Create influx container
+  community.docker.docker_container:
+    name: "influxdb"
+    image: "influxdb:1.12"
+    state: started
+    restart_policy: "always"
+    recreate: false
+    networks:
+      - name: "loadbalancer"
+    ports:
+      - "8086:8086"
+    mounts:
+      - source: /opt/openconext/influxdb/varlib
+        target: /var/lib/influxdb
+        type: bind
+      - source: /opt/openconext/influxdb/etc
+        target: /etc/influxdb
+        type: bind
+      - source: /data/influxdb-backup
+        target: /data/influxdb-backup
+        type: bind
+    env:
+      INFLUXDB_DB: "{{ influx_stats_db }}"
+      INFLUXDB_ADMIN_USER: "{{ influxdb_admin_user }}"
+      INFLUXDB_ADMIN_PASSWORD: "{{ influxdb_admin_password }}"
+      INFLUXDB_READ_USER: "{{ influxdb_stats_user }}"
+      INFLUXDB_READ_USER_PASSWORD: "{{ influxdb_stats_password }}"
+      INFLUXDB_HTTP_AUTH_ENABLED: "True"
+      INFLUXDB_META_DIR: "/var/lib/influxdb/meta"
+      INFLUXDB_DATA_DIR: "/var/lib/influxdb/data"
+      INFLUXDB_COORDINATOR_WAL_DIR: "/var/lib/influxdb/wal"
+
+- name: Wait for influx to start
+  ansible.builtin.wait_for:
+    port: 8086
+    host: localhost
+    state: started
+    delay: 5
+
+- name: Grant stats user required access to InfluxDB database
+  ansible.builtin.uri:
+    url: "http://{{ influxdb_host }}:8086/query"
+    method: POST
+    url_username: "{{ influxdb_admin_user }}"
+    url_password: "{{ influxdb_admin_password }}"
+    force_basic_auth: true
+    body_format: form-urlencoded
+    body:
+      q: 'GRANT ALL PRIVILEGES TO "{{ influxdb_stats_user }}"'
+    status_code: 200
+    return_content: true
+  register: influxdb_grant
+  changed_when: false
+  no_log: true
+
+- name: Import continuous queries through the InfluxDB Docker container
+  community.docker.docker_container_exec:
+    container: "influxdb"
+    command: >
+      influx
+      -host {{ influxdb_host }}
+      -port 8086
+      -username {{ influxdb_admin_user }}
+      -password {{ influxdb_admin_password }}
+    stdin: "{{ lookup('ansible.builtin.file', 'files/continuous_queries.sql') }}"
+  no_log: true
+
+- name: Create cron file for making the backups and cleaning them up after 7 days
+  ansible.builtin.copy:
+    src: files/influxdb_backup
+    dest: /etc/cron.d/influxdb_backup

roles/logstash_docker/README.md

@@ -0,0 +1,29 @@
+logstash_docker
+=========
+
+This role deploys a logstash docker container
+
+Requirements
+------------
+
+Requires some form of input, the filebeat role from OpenConext-deploy can be used for this. Requires a influxdb instance to write to, the OpenConext-deploy role influxdb_container can be used for this. The database must be named "log_logins" for this combination to work.
+
+Role Variables
+--------------
+
+The following variables must be set:
+ - logstash_memory_gb: (defaults to 4, can be set to 1 for development/test environments)
+ - influx_stats_dbhost: (the host where influxdb runs on, if this runs in a container on the same host provide the name of the container.)
+ - influxdb_stats_user: (user that can access the influxdb database)
+ - influxdb_stats_password: (password of the user that can access the database, store this securely for example in a vault.)
+
+License
+--------------
+
+These files are licensed under version 2.0 of the Apache License, as described in the file [LICENSE](LICENSE).
+
+Support
+--------------
+
+* You can ask questions on the [OpenConext mailing list](https://openconext.org/get-involved/mailing-lists/) 
+* Or you can join our [Slack Workspace](https://edu.nl/ocslk)

roles/logstash_docker/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+logstash_memory_gb: 4
+# The following variables are not provided but required to run this role. Define these in your inventory or vault:
+# influx_stats_dbhost:
+# influxdb_stats_user:
+# influxdb_stats_password:

roles/logstash_docker/files/core/02-filebeat-input.conf

@@ -0,0 +1,6 @@
+input {
+  beats {
+    port => 5044
+    type => "log"
+      }
+}

roles/logstash_docker/files/core/13-core-ebauth.conf

@@ -0,0 +1,33 @@
+filter {
+  if [prog] == "EBAUTH" {
+    grok {
+      patterns_dir => "/etc/logstash/patterns"
+      match => { "message" => "%{SYSLOGBASE:syslogbase} %{GREEDYDATA:ebauth}" }
+    }
+}
+  json { 
+    source => "ebauth"
+    target => "ebauth"
+}
+  date {
+     match => [ "[ebauth][context][login_stamp]", "yyyy-MM-dd'T'HH:mm:ssZZ" ]
+}
+  de_dot {
+}   
+  mutate {
+      remove_field => [ "source" ]
+      remove_field => [ "syslogbase" ]
+      add_field    =>  {"[month]" => "%{+MM}"}
+      add_field    =>  {"[year]" => "%{+YYYY}"}
+}
+  fingerprint {
+      key => eighiePhie3xiu3boChaekoomai6iiYilux1zoohoh6ahgh9iaphuthaero4ahme
+      source => [ "[ebauth][context][user_id]" ]
+      method => SHA256
+      target => [ "[ebauth][context][user_id_hashed]" ]
+}
+  ruby {
+      code =>  'quarter = (event.get("month").to_i / 3.0).ceil;
+               event.set("quarter",quarter)'
+}
+}

roles/logstash_docker/files/jvm.options

@@ -0,0 +1,77 @@
+# JVM configuration
+
+# Xms represents the initial size of total heap space
+# Xmx represents the maximum size of total heap space
+
+-Xms2g
+-Xmx2g
+
+################################################################
+## Expert settings
+################################################################
+##
+## All settings below this section are considered
+## expert settings. Don't tamper with them unless
+## you understand what you are doing
+##
+################################################################
+
+## GC configuration
+# -XX:+UseParNewGC #Option does not exist
+# -XX:+UseConcMarkSweepGC
+# -XX:CMSInitiatingOccupancyFraction=75
+# -XX:+UseCMSInitiatingOccupancyOnly
+
+## Locale
+# Set the locale language
+#-Duser.language=en
+
+# Set the locale country
+#-Duser.country=US
+
+# Set the locale variant, if any
+#-Duser.variant=
+
+## basic
+
+# set the I/O temp directory
+#-Djava.io.tmpdir=$HOME
+
+# set to headless, just in case
+-Djava.awt.headless=true
+
+# ensure UTF-8 encoding by default (e.g. filenames)
+-Dfile.encoding=UTF-8
+
+# use our provided JNA always versus the system one
+#-Djna.nosys=true
+
+# Turn on JRuby invokedynamic
+-Djruby.compile.invokedynamic=true
+# Force Compilation
+-Djruby.jit.threshold=0
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails
+# heap dumps are created in the working directory of the JVM
+-XX:+HeapDumpOnOutOfMemoryError
+
+# specify an alternative path for heap dumps
+# ensure the directory exists and has sufficient space
+#-XX:HeapDumpPath=${LOGSTASH_HOME}/heapdump.hprof
+
+## GC logging
+#-XX:+PrintGCDetails
+#-XX:+PrintGCTimeStamps
+#-XX:+PrintGCDateStamps
+#-XX:+PrintClassHistogram
+#-XX:+PrintTenuringDistribution
+#-XX:+PrintGCApplicationStoppedTime
+
+# log GC status to a file with time stamps
+# ensure the directory exists
+#-Xloggc:${LS_GC_LOG_FILE}
+
+# Entropy source for randomness
+-Djava.security.egd=file:/dev/urandom