Add support for deploying Fluentd on Swarm cluster

Author: dougszumski

ansible/deploy_swarm_monitoring.yml

@@ -0,0 +1,10 @@
+#
+# Copyright StackHPC, 2018
+#
+---
+- name: Deploy Swarm cluster monitoring
+  hosts: cluster
+  become: yes
+  roles:
+  - role: monasca_fluentd
+

ansible/group_vars/all/all

@@ -15,7 +15,7 @@ alaska_cloud: alaska
 alaska_homedir: /alaska
 alaska_softiron: 10.4.99.101
 
-# OpenStack fully qualified project name
+# OpenStack fully qualified project name (used for Grafana with domain support)
 project_name: p3@default
 
 # Virtual IP address of the controller node
@@ -28,6 +28,14 @@ alaska_monitoring_server: 10.60.253.3
 monasca_agent_p3_username: p3-monasca-agent
 monasca_agent_p3_password: "{{ vault_monasca_agent_password }}"
 
+# Monasca Fluentd config
+monasca_fluentd_log_api_uri: http://{{ controller_vip }}:5607
+monasca_fluentd_keystone_uri: http://{{ controller_vip }}:5000
+monasca_fluentd_username: "{{ monasca_agent_p3_username }}"
+monasca_fluentd_password: "{{ monasca_agent_p3_password }}"
+monasca_fluentd_project_domain_id: default
+monasca_fluentd_project_name: p3
+
 # Local Grafana admin account for configuring Grafana
 grafana_admin_username: grafana-admin
 grafana_admin_password: "{{ vault_grafana_admin_password }}"

ansible/roles/monasca_fluentd/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+- name: Restart Fluentd
+  docker_container:
+    name: fluentd
+    restart: True
+  when: fluentd_config.changed

ansible/roles/monasca_fluentd/tasks/main.yml

@@ -0,0 +1,39 @@
+#
+# Copyright StackHPC, 2018
+#
+---
+- name: Create Fluentd user
+  user:
+    name: fluent
+    state: present
+
+- name: Create Fluentd config directory
+  file:
+    path: /etc/fluentd
+    state: directory
+    owner: fluent
+    group: fluent
+    mode: 0755
+
+- name: Generate Fluentd config
+  template:
+    src: fluentd.conf.j2
+    dest: /etc/fluentd/fluentd.conf
+    owner: fluent
+    group: fluent
+    mode: 0644
+  register: fluentd_config
+  notify: Restart Fluentd
+
+- name: Start Fluentd container
+  docker_container:
+    name: fluentd
+    image: stackhpc/monasca-fluentd:latest
+    state: started
+    ports:
+      - "24224:24224"
+      - "24224:24224/udp"
+    volumes:
+      - /etc/fluentd/:/fluentd/etc:ro
+    env:
+      FLUENTD_CONF: "fluentd.conf"

ansible/roles/monasca_fluentd/templates/fluentd.conf.j2

@@ -0,0 +1,45 @@
+# Accept logs from Docker Fluentd log driver
+<source>
+  @type forward
+  port 24224
+  bind 0.0.0.0
+</source>
+
+# Add a timestamp dimension to all logs to record the event time. The
+# event time is the time extracted from the log message in all cases
+# where the time_key is set, and the time the record entered fluentd
+# if no time_key is set.
+# logs.
+<filter *.**>
+  @type record_transformer
+  <record>
+    timestamp ${time}
+  </record>
+</filter>
+
+# Docker saves all logs under the 'log' field. The fluentd-monasca
+# plugin assumes that they are saved under the 'message' field. Here
+# we map the 'log' field to the 'message' field for all logs.
+<filter *.**>
+  @type record_transformer
+  enable_ruby true
+  <record>
+    message ${record["log"]}
+  </record>
+  remove_keys log
+</filter>
+
+<match *.**>
+    type copy
+    <store>
+       @type monasca
+       keystone_url {{ monasca_fluentd_keystone_uri }}
+       monasca_log_api {{ monasca_fluentd_log_api_uri }}
+       monasca_log_api_version v3.0
+       username {{ monasca_fluentd_username }}
+       password {{ monasca_fluentd_password }}
+       domain_id {{ monasca_fluentd_project_domain_id }}
+       project_name {{ monasca_fluentd_project_name }}
+    </store>
+</match>
+