Initial W3C Traceparent and otel integration.

cwms-data-api/build.gradle

@@ -147,6 +147,10 @@ dependencies {
     implementation project(":access-manager-api")
 
     implementation(libs.bundles.metrics)
+    implementation(libs.io.opentelemetry.api)
+    implementation(libs.io.opentelemetry.sdk)
+    implementation(libs.io.opentelemetry.exporter.logging)
+    implementation(libs.io.opentelemetry.instrumentation.java)
 
     implementation(libs.bundles.jackson)
 
@@ -176,6 +180,7 @@ dependencies {
     }
 
     baseLibs(libs.ch.qos.logback)
+    baseLibs(libs.io.opentelemetry.instrumentation.logback.mdc)
 
     testImplementation(libs.bundles.testcontainers)
 
@@ -327,6 +332,7 @@ task integrationTests(type: Test) {
     jvmArgs += "-Dcwms.dataapi.access.provider=MultipleAccessManager"
     jvmArgs += "-Dcwms.dataapi.access.providers=KeyAccessManager,CwmsAccessManager"
     jvmArgs += "-Dcatalina.base=$buildDir/tomcat"
+    //jvmArgs += "-Dflogger.backend_factory=com.google.common.flogger.backend.slf4j.Slf4jBackendFactory#getInstance"
 }
 
 task timeseriesReadBenchmark(type: JavaExec) {

cwms-data-api/logback.xml

@@ -14,13 +14,22 @@
         </filter>
     </appender>
 
+    <appender name="OTEL-STDERR" class="io.opentelemetry.instrumentation.logback.mdc.v1_0.OpenTelemetryAppender">
+        <appender-ref ref="STDERR"/>
+    </appender>
+
+
     <appender name="FILE" class="FileAppender">
         <encoder class="JsonEncoder"/>
         <append>false</append>
         <immediateFlush>true</immediateFlush>
         <file>build/cda.jsonl</file>
     </appender>
 
+    <appender name="OTEL-FILE" class="io.opentelemetry.instrumentation.logback.mdc.v1_0.OpenTelemetryAppender">
+    <appender-ref ref="FILE"/>
+  </appender>
+
     <appender name="FILE-PLAIN" class="FileAppender">
         <encoder>
         <!-- Format includes date, level, MDC userId, and message -->
@@ -36,8 +45,8 @@
     <logger name="org.apache" level="ERROR"/>
 
     <root level="INFO">
-        <appender-ref ref="STDERR"/>
-        <appender-ref ref="FILE"/>
+        <appender-ref ref="OTEL-STDERR"/>
+        <appender-ref ref="OTEL-FILE"/>
         <appender-ref ref="FILE-PLAIN"/>
     </root>
 </configuration>

cwms-data-api/src/docker/logback-juli.xml

@@ -21,9 +21,8 @@
     <logger name="org.apache.catalina.core.ContainerBase.[Catalina]" level="INFO" additivity="false">
         <appender-ref ref="STDERR" />
     </logger>
-
 
     <root level="INFO">
         <appender-ref ref="STDERR"/>
-    </root>   
+    </root>
 </configuration>

cwms-data-api/src/main/java/cwms/cda/ApiServlet.java

@@ -191,6 +191,7 @@
 import cwms.cda.formatters.csv.CsvExampleGenerator;
 import io.javalin.plugin.openapi.OpenApiOptions;
 import io.javalin.plugin.openapi.OpenApiPlugin;
+import io.opentelemetry.api.trace.Span;
 import io.swagger.v3.oas.models.Components;
 import io.swagger.v3.oas.models.OpenAPI;
 import io.swagger.v3.oas.models.Operation;
@@ -223,6 +224,7 @@
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import javax.sql.DataSource;
+
 import org.apache.http.entity.ContentType;
 import org.jetbrains.annotations.NotNull;
 import org.jooq.exception.DataAccessException;
@@ -301,9 +303,6 @@ public class ApiServlet extends HttpServlet {
     public static final String DEFAULT_OFFICE_KEY = "cwms.dataapi.default.office";
     public static final String DEFAULT_PROVIDER = "MultipleAccessManager";
 
-
-
-
     private MetricRegistry metrics;
     private Meter totalRequests;
 
@@ -370,6 +369,11 @@ public void init() {
                     ctx.header("X-Frame-Options", "SAMEORIGIN");
                     ctx.header("X-XSS-Protection", "1; mode=block");
                 })
+                .before(ctx -> {
+                    // now that we can get the generic route, update the name.
+                    var span = Span.current();
+                    span.updateName(ctx.method() + " " + ctx.matchedPath());
+                })
                 .exception(ApplicationException.class, (e, ctx) -> {
                     CdaError re = ExceptionTraceSupport.buildError(ctx, e.getCdaErrorMessage(),
                             e.getSource(), e.getDetails(), e);

cwms-data-api/src/main/java/cwms/cda/OpenTelemetrySetup.java

@@ -0,0 +1,34 @@
+package cwms.cda;
+
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator;
+import io.opentelemetry.context.propagation.ContextPropagators;
+import io.opentelemetry.sdk.OpenTelemetrySdk;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+
+public final class OpenTelemetrySetup {
+    private OpenTelemetrySetup() {
+        /* This utility class should not be instantiated */
+    }
+
+    /**
+   * Initializes the OpenTelemetry SDK with a logging span exporter and the W3C Trace Context
+   * propagator.
+   *
+   * @return A ready-to-use {@link OpenTelemetry} instance.
+   */
+    @SuppressWarnings("null") // nothing here can be null without other exceptions getting thrown.
+    public static void initTelemetry() {
+        SdkTracerProvider sdkTracerProvider =
+            SdkTracerProvider.builder()
+                .build();
+
+        OpenTelemetrySdk.builder()
+            .setTracerProvider(sdkTracerProvider)
+            .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance()))
+            .buildAndRegisterGlobal();
+        Runtime.getRuntime().addShutdownHook(new Thread(sdkTracerProvider::close));
+
+    }
+
+}

cwms-data-api/src/main/java/cwms/cda/servlet/InitListener.java

@@ -0,0 +1,33 @@
+package cwms.cda.servlet;
+
+import javax.servlet.ServletContextEvent;
+import javax.servlet.ServletContextListener;
+import javax.servlet.annotation.WebListener;
+
+import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator;
+import io.opentelemetry.context.propagation.ContextPropagators;
+import io.opentelemetry.sdk.OpenTelemetrySdk;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+
+@WebListener
+public class InitListener implements ServletContextListener {
+
+    private static final SdkTracerProvider sdkTracerProvider =
+            SdkTracerProvider.builder()
+                .build();
+
+    @Override
+    public void contextInitialized(ServletContextEvent sce) {
+
+
+        OpenTelemetrySdk.builder()
+            .setTracerProvider(sdkTracerProvider)
+            .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance()))
+            .buildAndRegisterGlobal();
+    }
+
+    @Override
+    public void contextDestroyed(ServletContextEvent sce) {
+        sdkTracerProvider.close();
+    }
+}

cwms-data-api/src/main/java/cwms/cda/servlet/W3CTraceFilter.java

@@ -0,0 +1,85 @@
+package cwms.cda.servlet;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.annotation.WebFilter;
+import javax.servlet.http.HttpServletRequest;
+
+import io.opentelemetry.api.GlobalOpenTelemetry;
+import io.opentelemetry.api.trace.SpanKind;
+import io.opentelemetry.api.trace.StatusCode;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.ContextKey;
+import io.opentelemetry.context.propagation.TextMapGetter;
+
+/**
+ *
+ */
+@WebFilter(urlPatterns = {"*"})
+public final class W3CTraceFilter implements Filter {
+
+    public static final ContextKey<String> TRACE_PARENT = ContextKey.named("traceparent");
+    public static final Pattern TRACE_PARENT_MATCHER =
+        Pattern.compile("[a-z0-9]{2}-[a-z0-9]{32}-[a-z0-9]{16}-[a-z0-9]{2}");
+
+    @SuppressWarnings("java:S1181") // We're catching Throwable intentionally so it can be recorded in the span.
+    @Override
+    public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain)
+            throws IOException, ServletException {
+
+        var httpRequest = (HttpServletRequest)request;
+        var spanBuilder = GlobalOpenTelemetry.getTracer("cwms-data-api")
+            // using the full URI here results in too high of cardinality so spans can't be grouped
+            // endpoints can/should add the route as attributes to the current span.
+            .spanBuilder(httpRequest.getMethod() + " " + request.getServletContext())
+            .setSpanKind(SpanKind.SERVER);
+        var provided = httpRequest.getHeader(TRACE_PARENT.toString());
+        if (provided != null && !provided.isEmpty() && TRACE_PARENT_MATCHER.matcher(provided).matches()) {
+            var propagator = GlobalOpenTelemetry.getPropagators().getTextMapPropagator();
+            var ctx = propagator.extract(Context.current(), provided, new TraceGetter());
+            spanBuilder.setParent(ctx);
+        }
+
+        var span = spanBuilder.startSpan();
+        try (var scope = span.makeCurrent()) {
+            chain.doFilter(request, response);
+        } catch (Throwable t) {
+            span.recordException(t);
+            span.setStatus(StatusCode.ERROR);
+            throw t;
+        } finally {
+            span.end();
+        }
+    }
+
+    /**
+     * A simple wrapper to just get the value in the required way.
+     */
+    private static class TraceGetter implements TextMapGetter<String>
+    {
+        @Override
+        public Iterable<String> keys(@Nonnull String carrier)
+        {
+            return List.of(TRACE_PARENT.toString());
+        }
+
+        @Override
+        @Nullable
+        public String get(@Nullable String carrier, @Nonnull String key) {
+            if (TRACE_PARENT.toString().equalsIgnoreCase(key)) {
+                return carrier;
+            } else {
+                return null;
+            }
+        }
+    }
+}

cwms-data-api/src/test/java/cwms/cda/api/rating/RatingSpecControllerTestIT.java

@@ -39,6 +39,7 @@
 import cwms.cda.data.dto.rating.RatingSpec;
 import cwms.cda.formatters.ContentType;
 import cwms.cda.formatters.Formats;
+import cwms.cda.servlet.W3CTraceFilter;
 
 import javax.servlet.http.HttpServletResponse;
 import java.util.stream.IntStream;
@@ -58,6 +59,7 @@ class RatingSpecControllerTestIT extends DataApiTestIT {
 
     @Test
     void test_empty_rating_spec() throws Exception {
+        final String TRACE_PARENT_VALUE = "00-f64a0407859e1a735c1a89c5c5b4f47f-09d07b8aaba94e49-01";
         String locationId = "RatingSpecTestEmpty";
         String officeId = "SPK";
         createLocation(locationId, true, officeId);
@@ -77,6 +79,7 @@ void test_empty_rating_spec() throws Exception {
             .contentType(Formats.XMLV2)
             .body(templateXml)
             .header("Authorization", user.toHeaderValue())
+            .header(W3CTraceFilter.TRACE_PARENT.toString(), TRACE_PARENT_VALUE)
             .queryParam(OFFICE, officeId)
         .when()
             .redirects().follow(true)
@@ -93,6 +96,7 @@ void test_empty_rating_spec() throws Exception {
             .contentType(Formats.XMLV2)
             .body(specXml)
             .header("Authorization", user.toHeaderValue())
+            .header(W3CTraceFilter.TRACE_PARENT.toString(), TRACE_PARENT_VALUE)
             .queryParam(OFFICE, officeId)
         .when()
             .redirects().follow(true)
@@ -110,6 +114,7 @@ void test_empty_rating_spec() throws Exception {
             .log().ifValidationFails(LogDetail.ALL,true)
             .accept(Formats.JSONV2)
             .queryParam(PAGE_SIZE, 500)
+            .header(W3CTraceFilter.TRACE_PARENT.toString(), TRACE_PARENT_VALUE)
         .when()
             .redirects().follow(true)
             .redirects().max(3)
@@ -129,6 +134,7 @@ void test_empty_rating_spec() throws Exception {
                 .contentType(Formats.JSONV2)
                 .queryParam(OFFICE, officeId)
                 .queryParam(RATING_ID_MASK, specContainer.specId)
+                .header(W3CTraceFilter.TRACE_PARENT.toString(), TRACE_PARENT_VALUE)
             .when()
                 .redirects().follow(true)
                 .redirects().max(3)
@@ -148,6 +154,7 @@ void test_empty_rating_spec() throws Exception {
             .header("Authorization", user.toHeaderValue())
             .queryParam(OFFICE, officeId)
             .queryParam(METHOD, JooqDao.DeleteMethod.DELETE_ALL)
+            .header(W3CTraceFilter.TRACE_PARENT.toString(), TRACE_PARENT_VALUE)
         .when()
             .redirects().follow(true)
             .redirects().max(3)

gradle/libs.versions.toml

@@ -39,6 +39,9 @@ openapi-validation = "2.44.9"
 javaparser = "3.26.2"
 togglz = "3.3.3"
 minio = "8.6.0"
+opentelemetry = "1.63.0"
+opentelemetry-java = "2.29.0"
+opentelemetry-java-logging = "2.29.0-alpha"
 
 #Overrides
 classgraph = { strictly = '4.8.176' }
@@ -63,6 +66,12 @@ google-flogger-api = { module = "com.google.flogger:flogger", version.ref = "flo
 google-flogger-system-backend = { module = "com.google.flogger:flogger-system-backend", version.ref = "flogger" }
 google-flogger-slf4j-backend = { module = "com.google.flogger:flogger-slf4j-backend", version.ref = "flogger" }
 ch-qos-logback = { module = "ch.qos.logback:logback-classic", version.ref = "logback" }
+io-opentelemetry-instrumentation-logback-mdc = { module = "io.opentelemetry.instrumentation:opentelemetry-logback-mdc-1.0", version.ref = "opentelemetry-java-logging" }
+io-opentelemetry-api = { module="io.opentelemetry:opentelemetry-api", version.ref = "opentelemetry" }
+io-opentelemetry-sdk = { module="io.opentelemetry:opentelemetry-sdk", version.ref = "opentelemetry" }
+io-opentelemetry-exporter-logging = { module="io.opentelemetry:opentelemetry-exporter-logging", version.ref="opentelemetry" }
+io-opentelemetry-instrumentation-java = { module="io.opentelemetry.instrumentation:opentelemetry-instrumentation-api", version.ref = "opentelemetry-java" }
+
 google-findbugs = { module = "com.google.code.findbugs:jsr305", version.ref = "google-findbugs" }
 google-errorProne = { module = "com.google.errorprone:error_prone_annotations", version.ref = "error_prone_annotations"}
 nucleus-data = { module = "mil.army.usace.hec:hec-nucleus-data", version.ref = "hec-nucleus" }