#88 Setup Prometheus & Grafana, Spring & Minio & Postgres metrics

backend/authik/src/main/kotlin/ru/ifmo/se/dating/authik/security/AuthikSecuredPaths.kt

@@ -1,11 +1,19 @@
 package ru.ifmo.se.dating.authik.security
 
+import org.springframework.http.HttpMethod
 import org.springframework.security.web.server.util.matcher.ServerWebExchangeMatcher
 import org.springframework.stereotype.Component
-import ru.ifmo.se.dating.spring.security.auth.SpringSecuredPaths
+import ru.ifmo.se.dating.spring.security.auth.And
+import ru.ifmo.se.dating.spring.security.auth.Not
 import ru.ifmo.se.dating.spring.security.auth.Path
+import ru.ifmo.se.dating.spring.security.auth.SpringSecuredPaths
 
 @Component
 class AuthikSecuredPaths : SpringSecuredPaths {
-    override val matcher: ServerWebExchangeMatcher = Path("")
+    override val matcher: ServerWebExchangeMatcher = And(
+        Path("/api/**"),
+        Not(Path("/api/auth/telegram/web-app", HttpMethod.GET)),
+        Not(Path("/api/monitoring/healthcheck", HttpMethod.PUT)),
+        Not(Path("/actuator/**", HttpMethod.GET)),
+    )
 }

backend/config/build.gradle.kts

@@ -3,13 +3,14 @@ plugins {
 }
 
 dependencies {
-    implementation(project(":starter-service-discovery"))
-    implementation(project(":starter-tls"))
-
     implementation(libs.org.springframework.boot.spring.boot)
     implementation(libs.org.springframework.boot.spring.boot.starter.web)
     implementation(libs.org.springframework.cloud.spring.cloud.config.server)
 
+    implementation(project(":starter-monitoring"))
+    implementation(project(":starter-service-discovery"))
+    implementation(project(":starter-tls"))
+
     testImplementation(libs.org.springframework.boot.spring.boot.starter.test)
     testImplementation(libs.junit.junit)
     testImplementation(libs.org.testcontainers.vault)

backend/config/src/main/resources/application.yml

@@ -1,6 +1,6 @@
 spring:
   config:
-    import: application-service-discovery.yml,application-tls.yml
+    import: application-monitoring.yml,application-service-discovery.yml,application-tls.yml
   application:
     name: config
   profiles:

backend/foundation/build.gradle.kts

@@ -5,6 +5,7 @@ plugins {
 }
 
 dependencies {
+    api(project(":starter-monitoring"))
     api(project(":starter-service-discovery"))
     api(project(":starter-tls"))
 

backend/foundation/src/main/resources/application-foundation.yml

@@ -1,6 +1,6 @@
 spring:
   config:
-    import: application-service-discovery.yml,application-tls.yml,optional:configserver:https://config.dating.se.ifmo.ru:8080
+    import: application-monitoring.yml,application-service-discovery.yml,application-tls.yml,optional:configserver:https://config.dating.se.ifmo.ru:8080
   cloud:
     config:
       uri: https://config.dating.se.ifmo.ru:8080
@@ -27,12 +27,3 @@ springdoc:
   swagger-ui:
     url: /openapi/api.yml
     path: /swagger-ui.html
-logging:
-  level:
-    web: INFO
-    liquibase: WARN
-    r2dbc: DEBUG
-  group:
-    r2dbc: org.springframework.r2dbc,org.springframework.data.r2dbc,org.jooq.tools.LoggerListener
-  pattern:
-    console: "%clr(%d{yyyy-MM-dd'T'HH:mm:ss.SSS}){faint} %clr([%level]) %clr(%logger{36}){blue}: %msg%n"

backend/gateway/build.gradle.kts

@@ -3,6 +3,7 @@ plugins {
 }
 
 dependencies {
+    implementation(project(":starter-monitoring"))
     implementation(project(":starter-service-discovery"))
     implementation(project(":starter-tls"))
 

backend/gateway/src/main/resources/application.yml

@@ -1,6 +1,6 @@
 spring:
   config:
-    import: application-service-discovery.yml,application-tls.yml
+    import: application-monitoring.yml,application-service-discovery.yml,application-tls.yml
   application:
     name: gateway
   cloud:
@@ -146,10 +146,3 @@ springdoc:
         url: openapi/matchmaker/api.yml
       - name: people
         url: openapi/people/api.yml
-logging:
-  level:
-    gateway: WARN
-  group:
-    gateway: org.springframework.cloud.gateway.route
-  pattern:
-    console: "%clr(%d{yyyy-MM-dd'T'HH:mm:ss.SSS}){faint} %clr([%level]) %clr(%logger{36}){blue}: %msg%n"

backend/gradle/libs.versions.toml

@@ -24,6 +24,8 @@ commons-codec-commons-codec = "1.17.1"
 org-jetbrains-kotlinx-kotlinx-coroutines = "1.9.0"
 io-projectreactor-kotlin-reactor-kotlin-extensions = "1.2.3"
 
+io-micrometer = "1.14.3"
+
 io-projectreactor-reactor-test = "3.6.11"
 junit-junit = "4.13.2"
 org-testcontainers = "1.20.3"
@@ -76,6 +78,8 @@ org-liquibase-liquibase-core = { module = "org.liquibase:liquibase-core", versio
 org-postgresql-postgresql = { module = "org.postgresql:postgresql", version.ref = "org-postgresql-postgresql" }
 org-postgresql-r2dbc-postgresql = { module = "org.postgresql:r2dbc-postgresql", version.ref = "org-postgresql-r2dbc-postgresql" }
 
+io-micrometer-micrometer-registry-prometheus = { module = "io.micrometer:micrometer-registry-prometheus", version.ref = "io-micrometer" }
+
 junit-junit = { module = "junit:junit", version.ref = "junit-junit" }
 org-testcontainers-postgresql = { module = "org.testcontainers:postgresql", version.ref = "org-testcontainers" }
 org-testcontainers-r2dbc = { module = "org.testcontainers:r2dbc", version.ref = "org-testcontainers" }

backend/grafana/Dockerfile

@@ -0,0 +1,3 @@
+FROM grafana/grafana
+
+ADD ./provisioning /etc/grafana/provisioning

backend/grafana/provisioning/datasources/all.yml

@@ -0,0 +1,8 @@
+apiVersion: 1
+datasources:
+  - name: Prometheus
+    label: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://${ITMO_DATING_PROMETHEUS_HOST}:9090
+    isDefault: true

backend/haproxy/config/haproxy.cfg

@@ -18,12 +18,14 @@ defaults
 frontend internal
   bind :8445 ssl crt /usr/local/etc/haproxy/itmo-dating-backend.pem
   bind :8446 ssl crt /usr/local/etc/haproxy/itmo-dating-backend.pem
+  bind :8447
   bind :8455 ssl crt /usr/local/etc/haproxy/itmo-dating-backend.pem
   bind :8456 ssl crt /usr/local/etc/haproxy/itmo-dating-backend.pem
   bind :8457 ssl crt /usr/local/etc/haproxy/itmo-dating-backend.pem
 
   use_backend vault         if { dst_port 8445 }
   use_backend consul        if { dst_port 8446 }
+  use_backend grafana       if { dst_port 8447 }
   use_backend authik        if { dst_port 8455 }
   use_backend matchmaker    if { dst_port 8456 }
   use_backend people        if { dst_port 8457 }
@@ -36,6 +38,9 @@ backend consul
   option httpchk GET /ui
   server consul server.dc1.consul:8500 check init-addr last,libc,none
 
+backend grafana
+  server grafana grafana.dating.se.ifmo.ru:3000 check init-addr last,libc,none
+
 backend authik
   balance roundrobin
   option httpchk GET /actuator/health

backend/matchmaker/src/main/kotlin/ru/ifmo/se/dating/matchmaker/security/MatchmakerSecuredPaths.kt

@@ -15,5 +15,7 @@ class MatchmakerSecuredPaths : SpringSecuredPaths {
         Not(Path("/api/people/{person_id}", HttpMethod.PUT)),
         Not(Path("/api/monitoring/healthcheck", HttpMethod.GET)),
         Not(Path("/api/suggestions", HttpMethod.OPTIONS)),
+        Not(Path("/api/monitoring/healthcheck", HttpMethod.PUT)),
+        Not(Path("/actuator/**", HttpMethod.GET)),
     )
 }

backend/people/src/main/kotlin/ru/ifmo/se/dating/people/security/PeopleSecuredPaths.kt

@@ -12,6 +12,7 @@ import ru.ifmo.se.dating.spring.security.auth.SpringSecuredPaths
 class PeopleSecuredPaths : SpringSecuredPaths {
     override val matcher: ServerWebExchangeMatcher = And(
         Path("/api/**"),
-        Not(Path("/api/monitoring/healthcheck", HttpMethod.GET)),
+        Not(Path("/api/monitoring/healthcheck", HttpMethod.PUT)),
+        Not(Path("/actuator/**", HttpMethod.GET)),
     )
 }

backend/prometheus/Dockerfile

@@ -0,0 +1,9 @@
+FROM prom/prometheus:latest
+
+COPY ./prometheus.yml /etc/prometheus/prometheus.yml
+
+CMD [ \
+    "--config.file=/etc/prometheus/prometheus.yml", \
+    "--storage.tsdb.retention.size=500MB", \
+    "--storage.tsdb.retention.time=3d" \
+]

backend/prometheus/prometheus.yml

@@ -0,0 +1,67 @@
+global:
+  scrape_interval: 5s
+
+scrape_configs:
+
+  - job_name: authik
+    metrics_path: /actuator/prometheus
+    dns_sd_configs:
+      - names:
+          - authik-0.dating.se.ifmo.ru
+          - authik-1.dating.se.ifmo.ru
+        type: A
+        port: 8080
+    scheme: https
+    tls_config:
+      insecure_skip_verify: true
+
+  - job_name: people
+    metrics_path: /actuator/prometheus
+    dns_sd_configs:
+      - names:
+          - people-0.dating.se.ifmo.ru
+          - people-1.dating.se.ifmo.ru
+        type: A
+        port: 8080
+    scheme: https
+    tls_config:
+      insecure_skip_verify: true
+
+  - job_name: matchmaker
+    metrics_path: /actuator/prometheus
+    dns_sd_configs:
+      - names:
+          - matchmaker-0.dating.se.ifmo.ru
+          - matchmaker-1.dating.se.ifmo.ru
+        type: A
+        port: 8080
+    scheme: https
+    tls_config:
+      insecure_skip_verify: true
+
+  - job_name: gateway
+    metrics_path: /actuator/prometheus
+    dns_sd_configs:
+      - names:
+          - gateway.dating.se.ifmo.ru
+        type: A
+        port: 8080
+    scheme: https
+    tls_config:
+      insecure_skip_verify: true
+
+  - job_name: object-storage
+    metrics_path: /minio/v2/metrics/cluster
+    dns_sd_configs:
+      - names:
+          - object-storage.dating.se.ifmo.ru
+        type: A
+        port: 9000
+    scheme: http
+
+  - job_name: postgres
+    dns_sd_configs:
+      - names:
+          - database-primary-exporter.dating.se.ifmo.ru
+        type: 'A'
+        port: 9187

backend/settings.gradle.kts

@@ -12,6 +12,7 @@ include(
     ":authik",
     ":matchmaker",
     ":people",
+    ":starter-monitoring",
     ":starter-service-discovery",
     ":starter-tls",
 )

backend/starter-monitoring/build.gradle.kts

@@ -0,0 +1,11 @@
+plugins {
+    id("buildlogic.kotlin-library-conventions")
+    kotlin("plugin.spring")
+}
+
+dependencies {
+    api(project(":starter-tls"))
+
+    api(libs.org.springframework.boot.spring.boot.starter.actuator)
+    api(libs.io.micrometer.micrometer.registry.prometheus)
+}

backend/starter-monitoring/src/main/resources/application-monitoring.yml

@@ -0,0 +1,29 @@
+spring:
+  config:
+    import: application-tls.yml
+management:
+  endpoints:
+    web:
+      exposure:
+        include: health,prometheus
+  health:
+    diskspace:
+      enabled: false
+  metrics:
+    export:
+      prometheus:
+        enabled: true
+  distribution:
+    percentiles-histogram:
+      "[http.server.requests]": true
+logging:
+  level:
+    web: INFO
+    liquibase: WARN
+    r2dbc: DEBUG
+    gateway: WARN
+  group:
+    r2dbc: org.springframework.r2dbc,org.springframework.data.r2dbc,org.jooq.tools.LoggerListener
+    gateway: org.springframework.cloud.gateway.route
+  pattern:
+    console: "%clr(%d{yyyy-MM-dd'T'HH:mm:ss.SSS}){faint} %clr([%level]) %clr(%logger{36}){blue}: %msg%n"

backend/starter-service-discovery/build.gradle.kts

@@ -5,8 +5,8 @@ plugins {
 
 dependencies {
     api(project(":starter-tls"))
+    api(project(":starter-monitoring"))
 
-    api(libs.org.springframework.boot.spring.boot.starter.actuator)
     api(libs.org.springframework.cloud.spring.cloud.starter.consul.discovery)
     api(libs.org.springframework.cloud.spring.cloud.starter.loadbalancer)
 }

backend/starter-service-discovery/src/main/resources/application-service-discovery.yml

@@ -1,6 +1,6 @@
 spring:
   config:
-    import: application-tls.yml
+    import: application-tls.yml,application-monitoring.yml
   cloud:
     consul:
       scheme: https
@@ -24,9 +24,3 @@ management:
   health:
     consul:
       enabled: true
-    diskspace:
-      enabled: false
-  endpoints:
-    web:
-      exposure:
-        include: health

backend/starter-tls/build.gradle.kts

@@ -4,7 +4,7 @@ plugins {
 }
 
 dependencies {
-    api(libs.org.springframework.boot.spring.boot)
+    api(libs.org.springframework.spring.context)
     api(libs.org.springframework.spring.web)
     api(libs.org.springframework.spring.webflux)
     implementation("io.netty:netty-handler:4.1.117.Final")