Prometheus and Grafana integration for Tasks Apps

 - Add rsocker-proxy docker-compose extension.
 - Add task-batch dashboard.
 - Update the prometheus-local to include rsocket proxy as target
 - Add test task with prometheus-rsocket
 - Remove the test docker-compose metrics files
 - Modularize docker-compose
   - the docker-compose.yml contains only SCDF/Skipper configuration without any metrics
   - To enable the prometheus (rsocket) metrics configuration run docker-compose -f ./docker-compose.yml -f ./docker-compose-prometheus-rsocket.yml up
   - To enable the Influx metric configuration run docker-compose -f ./docker-compose.yml -f ./docker-compose-influxdb.yml up
   - remove the custom service-discovery definition.
 - Update prometheus image to v2.12.0
 - Add docker-compose-postgres.yml to allow overriding mysql
 - Add docker-compose-rabbbitmq.yml to allow overriding kafka
 - Add STREM_APPS_URI and TASK_APPS_URI variables to allow overriding the registered  apps and task uris

 Resolves #3452

Author: tzolov

spring-cloud-dataflow-server/docker-compose-influxdb.yml

@@ -1,91 +1,27 @@
 version: '3'
 
+# Extends the default docker-compose.yml with Influx/Grafana monitoring configuration
+# Usage: docker-compose -f ./docker-compose.yml -f ./docker-compose-influxdb.yml up
 services:
-  mysql:
-    image: mysql:5.7.26
-    container_name: dataflow-mysql
-    environment:
-      MYSQL_DATABASE: dataflow
-      MYSQL_USER: root
-      MYSQL_ROOT_PASSWORD: rootpw
-    expose:
-      - 3306
-
-  kafka:
-    image: confluentinc/cp-kafka:5.2.1
-    container_name: dataflow-kafka
-    expose:
-      - "9092"
-    environment:
-      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092
-      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
-      - KAFKA_ADVERTISED_HOST_NAME=kafka
-      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
-    depends_on:
-      - zookeeper
-
-  zookeeper:
-    image: confluentinc/cp-zookeeper:5.2.1
-    container_name: dataflow-kafka-zookeeper
-    expose:
-      - "2181"
-    environment:
-      - ZOOKEEPER_CLIENT_PORT=2181
-
   dataflow-server:
-    image: springcloud/spring-cloud-dataflow-server:${DATAFLOW_VERSION:?DATAFLOW_VERSION is not set!}
-    container_name: dataflow-server
-    ports:
-      - "9393:9393"
-    environment:
-      - spring.cloud.dataflow.applicationProperties.stream.spring.cloud.stream.kafka.binder.brokers=PLAINTEXT://kafka:9092
-      - spring.cloud.dataflow.applicationProperties.stream.spring.cloud.stream.kafka.binder.zkNodes=zookeeper:2181
-      - spring.cloud.dataflow.applicationProperties.stream.spring.cloud.stream.kafka.streams.binder.brokers=PLAINTEXT://kafka:9092
-      - spring.cloud.dataflow.applicationProperties.stream.spring.cloud.stream.kafka.streams.binder.zkNodes=zookeeper:2181
-      - spring.cloud.skipper.client.serverUri=http://skipper-server:7577/api
-      - spring.cloud.dataflow.applicationProperties.stream.management.metrics.export.influx.enabled=true
-      - spring.cloud.dataflow.applicationProperties.stream.management.metrics.export.influx.db=myinfluxdb
-      - spring.cloud.dataflow.applicationProperties.stream.management.metrics.export.influx.uri=http://influxdb:8086
-      - spring.cloud.dataflow.applicationProperties.task.management.metrics.export.influx.enabled=true
-      - spring.cloud.dataflow.applicationProperties.task.management.metrics.export.influx.db=myinfluxdb
-      - spring.cloud.dataflow.applicationProperties.task.management.metrics.export.influx.uri=http://influxdb:8086
-      - spring.cloud.dataflow.grafana-info.url=http://localhost:3000
-      - SPRING_DATASOURCE_URL=jdbc:mysql://mysql:3306/dataflow
-      - SPRING_DATASOURCE_USERNAME=root
-      - SPRING_DATASOURCE_PASSWORD=rootpw
-      - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.mariadb.jdbc.Driver
-    depends_on:
-      - kafka
-    entrypoint: "./wait-for-it.sh -t 120 mysql:3306 -- java -jar /maven/spring-cloud-dataflow-server.jar"
-
-  app-import:
-    image: springcloud/openjdk:2.0.0.RELEASE
-    container_name: dataflow-app-import
-    depends_on:
-      - dataflow-server
-    command: >
-      /bin/sh -c "
-        ./wait-for-it.sh -t 180 dataflow-server:9393;
-        wget -qO- 'http://dataflow-server:9393/apps' --post-data='uri=https://dataflow.spring.io/kafka-maven-latest&force=true';
-        echo 'Stream apps imported'
-        wget -qO- 'http://dataflow-server:9393/apps' --post-data='uri=https://dataflow.spring.io/task-maven-latest&force=true';
-        echo 'Task apps imported'"
-
-  skipper-server:
-    image: springcloud/spring-cloud-skipper-server:${SKIPPER_VERSION:?SKIPPER_VERSION is not set!}
-    container_name: skipper
-    ports:
-      - "7577:7577"
-      - "9000-9010:9000-9010"
-      - "20000-20105:20000-20105"
     environment:
-      - SPRING_CLOUD_SKIPPER_SERVER_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_PORTRANGE_LOW=20000
-      - SPRING_CLOUD_SKIPPER_SERVER_PLATFORM_LOCAL_ACCOUNTS_DEFAULT_PORTRANGE_HIGH=20100
-      - SPRING_DATASOURCE_URL=jdbc:mysql://mysql:3306/dataflow
-      - SPRING_DATASOURCE_USERNAME=root
-      - SPRING_DATASOURCE_PASSWORD=rootpw
-      - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.mariadb.jdbc.Driver
-    entrypoint: "./wait-for-it.sh -t 120 mysql:3306 -- java -Djava.security.egd=file:/dev/./urandom -jar /spring-cloud-skipper-server.jar"
+      - |
+        SPRING_APPLICATION_JSON=
+        {
+          "spring.cloud.dataflow.applicationProperties":{
+             "task.management.metrics.export.influx":{
+                "enabled":true,
+                "db":"myinfluxdb",
+                "uri":"http://influxdb:8086"
+             },
+             "stream.management.metrics.export.influx":{
+                "enabled":true,
+                "db":"myinfluxdb",
+                "uri":"http://influxdb:8086"
+             }
+          },
+          "spring.cloud.dataflow.grafana-info.url":"http://localhost:3000"
+        }
 
   influxdb:
     image: influxdb:1.7.7
@@ -98,5 +34,4 @@ services:
     container_name: grafana
     ports:
       - '3000:3000'
-volumes:
-  scdf-targets:
+

spring-cloud-dataflow-server/docker-compose-postgres.yml

@@ -0,0 +1,38 @@
+version: '3'
+
+# Reconfigures the default docker-compose.yml to replace MySQL by Postgres
+# Usage: docker-compose -f ./docker-compose.yml -f ./docker-compose-postgres.yml up
+services:
+
+  mysql:
+    entrypoint: ["echo", "MySQL service is disabled"]
+
+  postgres:
+    image: postgres:10
+    container_name: dataflow-postgres
+    restart: always
+    environment:
+      POSTGRES_DB: dataflow
+      POSTGRES_USER: root
+      POSTGRES_PASSWORD: rootpw
+    expose:
+      - 5432
+    ports:
+      - 5432:5432
+
+  dataflow-server:
+    environment:
+      - SPRING_DATASOURCE_URL=jdbc:postgresql://postgres:5432/dataflow
+      - SPRING_DATASOURCE_USERNAME=root
+      - SPRING_DATASOURCE_PASSWORD=rootpw
+      - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver
+    entrypoint: "./wait-for-it.sh -t 240 postgres:5432 -- java -jar /maven/spring-cloud-dataflow-server.jar"
+
+  skipper-server:
+    environment:
+      - SPRING_DATASOURCE_URL=jdbc:postgresql://postgres:5432/dataflow
+      - SPRING_DATASOURCE_USERNAME=root
+      - SPRING_DATASOURCE_PASSWORD=rootpw
+      - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver
+    entrypoint: "./wait-for-it.sh -t 240 postgres:5432 -- java -Djava.security.egd=file:/dev/./urandom -jar /spring-cloud-skipper-server.jar"
+

spring-cloud-dataflow-server/docker-compose-prometheus.yml

@@ -0,0 +1,53 @@
+version: '3'
+
+# Extends the default docker-compose.yml with Prometheus/Grafana monitoring configuration
+# Usage: docker-compose -f ./docker-compose.yml -f ./docker-compose-prometheus.yml up
+services:
+
+  dataflow-server:
+    environment:
+      - |
+        SPRING_APPLICATION_JSON=
+        {
+          "spring.cloud.dataflow.applicationProperties":{
+             "task.management.metrics.export.prometheus":{
+                "enabled":true,
+                "rsocket.enabled":true,
+                "rsocket.host":"prometheus-rsocket-proxy",
+                "rsocket.port":7001
+             },
+             "stream.management.metrics.export.prometheus":{
+                "enabled":true,
+                "rsocket.enabled":true,
+                "rsocket.host":"prometheus-rsocket-proxy",
+                "rsocket.port":7001
+             }
+          },
+          "spring.cloud.dataflow.grafana-info.url":"http://localhost:3000"
+        }
+
+  prometheus-rsocket-proxy:
+    image: micrometermetrics/prometheus-rsocket-proxy:0.8.0-SNAPSHOT
+    container_name: prometheus-rsocket-proxy
+    expose:
+      - '9096'
+      - '7001'
+      - '8081'
+    ports:
+      - '9096:9096'
+      - '7001:7001'
+      - '8081:8081'
+    environment:
+      - server.port=9096
+
+  grafana:
+    image: springcloud/spring-cloud-dataflow-grafana-prometheus:${DATAFLOW_VERSION:?DATAFLOW_VERSION is not set! Use 'export DATAFLOW_VERSION=local-server-image-tag'}
+    container_name: grafana
+    ports:
+      - '3000:3000'
+
+  prometheus:
+    image: springcloud/spring-cloud-dataflow-prometheus-local:${DATAFLOW_VERSION:?DATAFLOW_VERSION is not set! Use 'export DATAFLOW_VERSION=local-server-image-tag'}
+    container_name: prometheus
+    ports:
+      - '9090:9090'

spring-cloud-dataflow-server/docker-compose-rabbitmq.yml

@@ -0,0 +1,31 @@
+version: '3'
+
+# Reconfigures the default docker-compose.yml to replace Kafka/Zookeeper by RabbitMQ
+# Usage: docker-compose -f ./docker-compose.yml -f ./docker-compose-postgres.yml up
+services:
+
+  kafka:
+    entrypoint: ["echo", "kafka service is disabled"]
+
+  zookeeper:
+    entrypoint: ["echo", "zookeeper service is disabled"]
+
+  rabbitmq:
+    image: rabbitmq:3.7
+    container_name: dataflow-rabbitmq
+    expose:
+      - '5672'
+
+  dataflow-server:
+    environment:
+      - spring.cloud.dataflow.applicationProperties.stream.spring.rabbitmq.host=rabbitmq
+
+  app-import:
+    command: >
+      /bin/sh -c "
+        ./wait-for-it.sh -t 180 dataflow-server:9393;
+        wget -qO- 'http://dataflow-server:9393/apps' --post-data='uri=${STREAM_APPS_URI:-https://dataflow.spring.io/rabbitmq-maven-latest&force=true}';
+        echo 'Stream apps imported'
+        wget -qO- 'http://dataflow-server:9393/apps' --post-data='uri=${TASK_APPS_URI:-https://dataflow.spring.io/task-maven-latest&force=true}';
+        echo 'Task apps imported'"
+

spring-cloud-dataflow-server/docker-compose.yml

@@ -43,10 +43,6 @@ services:
       - spring.cloud.dataflow.applicationProperties.stream.spring.cloud.stream.kafka.binder.zkNodes=zookeeper:2181
       - spring.cloud.dataflow.applicationProperties.stream.spring.cloud.stream.kafka.streams.binder.zkNodes=zookeeper:2181
       - spring.cloud.skipper.client.serverUri=http://skipper-server:7577/api
-      - spring.cloud.dataflow.applicationProperties.stream.management.metrics.export.prometheus.enabled=true
-      - spring.cloud.dataflow.applicationProperties.stream.spring.cloud.streamapp.security.enabled=false
-      - spring.cloud.dataflow.applicationProperties.stream.management.endpoints.web.exposure.include=prometheus,info,health
-      - spring.cloud.dataflow.grafana-info.url=http://localhost:3000
       - SPRING_DATASOURCE_URL=jdbc:mysql://mysql:3306/dataflow
       - SPRING_DATASOURCE_USERNAME=root
       - SPRING_DATASOURCE_PASSWORD=rootpw
@@ -63,9 +59,9 @@ services:
     command: >
       /bin/sh -c "
         ./wait-for-it.sh -t 180 dataflow-server:9393;
-        wget -qO- 'http://dataflow-server:9393/apps' --post-data='uri=https://dataflow.spring.io/kafka-maven-latest&force=true';
+        wget -qO- 'http://dataflow-server:9393/apps' --post-data='uri=${STREAM_APPS_URI:-https://dataflow.spring.io/kafka-maven-latest&force=true}';
         echo 'Stream apps imported'
-        wget -qO- 'http://dataflow-server:9393/apps' --post-data='uri=https://dataflow.spring.io/task-maven-latest&force=true';
+        wget -qO- 'http://dataflow-server:9393/apps' --post-data='uri=${TASK_APPS_URI:-https://dataflow.spring.io/task-maven-latest&force=true}';
         echo 'Task apps imported'"
 
   skipper-server:
@@ -83,45 +79,3 @@ services:
       - SPRING_DATASOURCE_PASSWORD=rootpw
       - SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.mariadb.jdbc.Driver
     entrypoint: "./wait-for-it.sh mysql:3306 -- java -Djava.security.egd=file:/dev/./urandom -jar /spring-cloud-skipper-server.jar"
-
-  # Grafana is configured with the Prometheus datasource.
-  # Use `docker exec -it prometheus /bin/sh` to logging in the container
-  prometheus:
-    image: springcloud/spring-cloud-dataflow-prometheus-local:${DATAFLOW_VERSION:?DATAFLOW_VERSION is not set! Use 'export DATAFLOW_VERSION=local-server-image-tag'}
-    container_name: prometheus
-    volumes:
-      - 'scdf-targets:/etc/prometheus/'
-    ports:
-      - '9090:9090'
-    depends_on:
-      - service-discovery
-
-  # The service-discovery container. Required for Prometheus setup only
-  # Use `docker exec -it service-discovery /bin/sh` to logging the container
-  service-discovery:
-    image: springcloud/spring-cloud-dataflow-prometheus-service-discovery:0.0.4.RELEASE
-    container_name: service-discovery
-    volumes:
-      - 'scdf-targets:/tmp/scdf-targets/'
-    expose:
-      - '8181'
-    ports:
-      - '8181:8181'
-    environment:
-      - metrics.prometheus.target.cron=0/20 * * * * *
-      - metrics.prometheus.target.filePath=/tmp/scdf-targets/targets.json
-      - metrics.prometheus.target.discoveryUrl=http://dataflow-server:9393/runtime/apps
-      - metrics.prometheus.target.overrideIp=skipper-server
-      - server.port=8181
-    depends_on:
-      - dataflow-server
-
-  # Grafana SCDF Prometheus pre-built image:
-  grafana:
-    image: springcloud/spring-cloud-dataflow-grafana-prometheus:${DATAFLOW_VERSION:?DATAFLOW_VERSION is not set! Use 'export DATAFLOW_VERSION=local-server-image-tag'}
-    container_name: grafana
-    ports:
-      - '3000:3000'
-
-volumes:
-  scdf-targets:

src/grafana/influxdb/docker/docker-compose.yml

@@ -1,4 +1,4 @@
-FROM grafana/grafana:6.2.5
+FROM grafana/grafana:6.3.3
 ADD ./provisioning /etc/grafana/provisioning
 ADD ./config.ini /etc/grafana/config.ini
 ADD ./dashboards /var/lib/grafana/dashboards