diff --git a/.changeset/supervisor-passthrough-logs.md b/.changeset/supervisor-passthrough-logs.md new file mode 100644 index 0000000000..cb0aa7d12b --- /dev/null +++ b/.changeset/supervisor-passthrough-logs.md @@ -0,0 +1,10 @@ +--- +'@hyperdx/otel-collector': patch +--- + +Use the OpAMP supervisor's native `passthrough_logs` for collector log +forwarding instead of a background `tail` process. The old approach had +the supervisor and the tailer writing to the same stdout fd with no +synchronization, so log lines were getting mangled by the two streams +interleaving mid-line. The native approach has the supervisor re-emitting +the collector's output through its own logger to avoid this. diff --git a/docker/otel-collector/Dockerfile b/docker/otel-collector/Dockerfile index f3aaf44853..f8323846b1 100644 --- a/docker/otel-collector/Dockerfile +++ b/docker/otel-collector/Dockerfile @@ -59,9 +59,8 @@ USER ${USER_UID}:${USER_GID} COPY --from=supervisor --chmod=755 /usr/local/bin/opampsupervisor /opampsupervisor COPY --from=ocb-builder --chmod=755 /build/output/otelcol-hyperdx /otelcontribcol -# Copy entrypoint and log tail wrapper scripts +# Copy entrypoint script COPY --chmod=755 docker/otel-collector/entrypoint.sh /entrypoint.sh -COPY --chmod=755 docker/otel-collector/log-tailer.sh /log-tailer.sh ## dev ############################################################################################## FROM base AS dev diff --git a/docker/otel-collector/config.yaml b/docker/otel-collector/config.yaml index 98cc78e800..18151391b2 100644 --- a/docker/otel-collector/config.yaml +++ b/docker/otel-collector/config.yaml @@ -163,6 +163,14 @@ service: port: 8888 logs: level: ${HYPERDX_LOG_LEVEL} + # The collector's zap logger defaults to stderr. Under the supervisor's + # `passthrough_logs`, stdout is re-emitted at Info and stderr at Error, + # so leaving the default would surface every collector log line as an + # error. Writing to stdout keeps the supervisor-assigned level sane; the + # collector's real severity is still in the wrapped record's own `level` + # field. In standalone mode both streams land in the container log, so + # this is a no-op there. + output_paths: [stdout] extensions: [health_check] # Pipeline receivers and exporters are configured dynamically in # opampController.ts. Pipeline `processors:` lists are declared here in diff --git a/docker/otel-collector/entrypoint.sh b/docker/otel-collector/entrypoint.sh index ca05993b29..30b138514f 100644 --- a/docker/otel-collector/entrypoint.sh +++ b/docker/otel-collector/entrypoint.sh @@ -77,20 +77,10 @@ else # Supervisor mode - run with OpAMP supervisor echo "Running in supervisor mode (OPAMP_SERVER_URL: $OPAMP_SERVER_URL)" - if [ "$OTEL_SUPERVISOR_LOGS" = "true" ]; then - # Start log tailer process in background for agent.log - # Arguments: log_file_path [check_interval_seconds] - /log-tailer.sh /etc/otel/supervisor-data/agent.log 1 & - - # Create a agent log file for the supervisor and collector child process. Normally - # this file would be created as a standard file but we just want a FIFO pipe that - # will pass data over to the tail process in the entrypoint script. This avoids - # the need to the supervisor to store and forward the logs in its memory while also - # eliminating the need for volume based storage. - if [ ! -e /etc/otel/supervisor-data/agent.log ]; then - mkfifo /etc/otel/supervisor-data/agent.log || echo "Failed to create FIFO" >&2 - fi - fi + # OTEL_SUPERVISOR_LOGS=true is consumed by supervisor.yaml.tmpl, which sets + # the supervisor's native `agent::passthrough_logs`. With this the supervisor + # forwards the collector's stdout/stderr through its own logger, keeping a + # single writer on the container's stdout. # Render the supervisor config template using gomplate # Write to supervisor-data directory which has proper permissions for otel user diff --git a/docker/otel-collector/log-tailer.sh b/docker/otel-collector/log-tailer.sh deleted file mode 100644 index 2dcd4adada..0000000000 --- a/docker/otel-collector/log-tailer.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh -# Generic log tailer script that follows a log file and echoes new lines to stdout -# Usage: log-tailer.sh [sleep_interval] - -# Parse arguments -LOG_FILE="${1}" -SLEEP_INTERVAL="${2:-1}" - -# Validate required argument -if [ -z "$LOG_FILE" ]; then - echo "Error: Log file path is required" >&2 - echo "Usage: $0 [sleep_interval]" >&2 - exit 1 -fi - -while true; do - # Use tail -F to follow the file by name, not by descriptor - # This handles rotation and truncation gracefully - # -n 0: Start from the end (don't output existing content) - # -F: Follow by name and retry if file is inaccessible - # -s: Sleep interval between checks - tail -n 0 -F -s "$SLEEP_INTERVAL" "$LOG_FILE" || true -done diff --git a/docker/otel-collector/supervisor_docker.yaml.tmpl b/docker/otel-collector/supervisor_docker.yaml.tmpl index f91e3cc59f..9439ae7c17 100644 --- a/docker/otel-collector/supervisor_docker.yaml.tmpl +++ b/docker/otel-collector/supervisor_docker.yaml.tmpl @@ -18,6 +18,10 @@ capabilities: agent: executable: /otelcontribcol +{{- if eq (getenv "OTEL_SUPERVISOR_LOGS") "true" }} + # Stream the collector's stdout/stderr through te supervisor's own logger. + passthrough_logs: true +{{- end }} config_files: - /etc/otelcol-contrib/config.yaml {{- if getenv "CUSTOM_OTELCOL_CONFIG_FILE" }}