[HDX-4329] replace collector tailer with the supervisors' native log redirection

.changeset/supervisor-passthrough-logs.md

@@ -0,0 +1,10 @@
+---
+'@hyperdx/otel-collector': patch
+---
+
+Use the OpAMP supervisor's native `passthrough_logs` for collector log
+forwarding instead of a background `tail` process. The old approach had
+the supervisor and the tailer writing to the same stdout fd with no
+synchronization, so log lines were getting mangled by the two streams
+interleaving mid-line. The native approach has the supervisor re-emitting
+the collector's output through its own logger to avoid this.

docker/otel-collector/Dockerfile

@@ -59,9 +59,8 @@ USER ${USER_UID}:${USER_GID}
 COPY --from=supervisor --chmod=755 /usr/local/bin/opampsupervisor /opampsupervisor
 COPY --from=ocb-builder --chmod=755 /build/output/otelcol-hyperdx /otelcontribcol
 
-# Copy entrypoint and log tail wrapper scripts
+# Copy entrypoint script
 COPY --chmod=755 docker/otel-collector/entrypoint.sh /entrypoint.sh
-COPY --chmod=755 docker/otel-collector/log-tailer.sh /log-tailer.sh
 
 ## dev ##############################################################################################
 FROM base AS dev

docker/otel-collector/config.yaml

@@ -163,6 +163,14 @@ service:
                 port: 8888
     logs:
       level: ${HYPERDX_LOG_LEVEL}
+      # The collector's zap logger defaults to stderr. Under the supervisor's
+      # `passthrough_logs`, stdout is re-emitted at Info and stderr at Error,
+      # so leaving the default would surface every collector log line as an
+      # error. Writing to stdout keeps the supervisor-assigned level sane; the
+      # collector's real severity is still in the wrapped record's own `level`
+      # field. In standalone mode both streams land in the container log, so
+      # this is a no-op there.
+      output_paths: [stdout]
   extensions: [health_check]
   # Pipeline receivers and exporters are configured dynamically in
   # opampController.ts. Pipeline `processors:` lists are declared here in

docker/otel-collector/entrypoint.sh

@@ -77,20 +77,10 @@ else
   # Supervisor mode - run with OpAMP supervisor
   echo "Running in supervisor mode (OPAMP_SERVER_URL: $OPAMP_SERVER_URL)"
 
-  if [ "$OTEL_SUPERVISOR_LOGS" = "true" ]; then
-   # Start log tailer process in background for agent.log
-    # Arguments: log_file_path [check_interval_seconds]
-    /log-tailer.sh /etc/otel/supervisor-data/agent.log 1 &
-
-    # Create a agent log file for the supervisor and collector child process. Normally
-    # this file would be created as a standard file but we just want a FIFO pipe that
-    # will pass data over to the tail process in the entrypoint script. This avoids
-    # the need to the supervisor to store and forward the logs in its memory while also
-    # eliminating the need for volume based storage.
-    if [ ! -e /etc/otel/supervisor-data/agent.log ]; then
-      mkfifo /etc/otel/supervisor-data/agent.log || echo "Failed to create FIFO" >&2
-    fi
-  fi
+  # OTEL_SUPERVISOR_LOGS=true is consumed by supervisor.yaml.tmpl, which sets
+  # the supervisor's native `agent::passthrough_logs`. With this the supervisor
+  # forwards the collector's stdout/stderr through its own logger, keeping a
+  # single writer on the container's stdout.
 
   # Render the supervisor config template using gomplate
   # Write to supervisor-data directory which has proper permissions for otel user

docker/otel-collector/supervisor_docker.yaml.tmpl

@@ -18,6 +18,10 @@ capabilities:
 
 agent:
   executable: /otelcontribcol
+{{- if eq (getenv "OTEL_SUPERVISOR_LOGS") "true" }}
+  # Stream the collector's stdout/stderr through te supervisor's own logger.
+  passthrough_logs: true
+{{- end }}
   config_files:
     - /etc/otelcol-contrib/config.yaml
 {{- if getenv "CUSTOM_OTELCOL_CONFIG_FILE" }}