chore: use --execution_log_compact_file to prevent OOM

.github/actions/bazel/action.yaml

@@ -73,46 +73,19 @@ runs:
           ${{ steps.metrics-tmpdir.outputs.dir }}/bazel-bep-*.pb.gpg
           ${{ steps.metrics-tmpdir.outputs.dir }}/profile-*.json
 
-
-      # Remove unnecessary entries from the JSON execlogs and create a CSV out of them
-    - name: Clean up execlogs
+    - name: Convert execlogs to CSV
       shell: bash
       if: (success() || failure()) && inputs.execlogs-artifact-name != ''
       run: |
-        execlogs_json_in='${{ steps.metrics-tmpdir.outputs.dir }}/execlogs.json'
-        find '${{ steps.metrics-tmpdir.outputs.dir }}' -name 'execlog-*.json' \
-          -exec cat {} \; > "$execlogs_json_in"
-
-        execlogs_csv_out='${{ steps.metrics-tmpdir.outputs.dir }}/execlogs.csv'
-        if ! [ -s "$execlogs_json_in" ]; then
-          echo "no execlogs found"
-          touch "$execlogs_csv_out" # create empty CSV
-          exit 0
-        fi
-
-        # this filters out some entries from the input JSON execlogs. Without this,
-        # jq takes 3+ minutes to parse the execlogs.
-        # Effectively removes a full line of "commandArgs", and takes care of skipping
-        # multiple lines for some multiline array values we don't care about
-        prog='
-          BEGIN { arr = 0; }
-          arr == 1 && /  }],?/ { arr = 0; next }
-          arr == 1 { next }
-          /  "commandArgs":.*/ { next }
-          /  "environmentVariables": \[\{/ { arr = 1; next; }
-          /  "inputs": \[\{/ { arr = 1; next; }
-          { print }
-          '
-
-        # Turns the JSON execlogs into a CSV with the following columns:
-        #   <label>,<output path>,<hash>
-        # only the target labels matching "whitelist_pat" are considered
-        whitelist_pat='^//'
-        time (cat "$execlogs_json_in" \
-          | awk "$prog" \
-          | jq -cMr --arg whitelist_pat "$whitelist_pat" \
-              'select(.targetLabel | test($whitelist_pat)) | .targetLabel as $targetLabel | .actualOutputs | map(. | $targetLabel+","+.path+","+.digest.hash) | .[]' \
-          >"$execlogs_csv_out")
+        execlog="$(mktemp)"
+        trap "rm -f $execlog" INT TERM EXIT
+        for execlog_zst in $(find '${{ steps.metrics-tmpdir.outputs.dir }}' -name 'execlog-*.zst'); do
+          zstd -f -d "$execlog_zst" -o "$execlog"
+          bazel run //bazel:execution_log_compact_to_csv \
+            $([[ "$(uname)" == "Linux" ]] && echo '--repository_cache=/cache/bazel') -- \
+            --input_execlog="$execlog" \
+            --whitelist_pat='^//'
+        done > '${{ steps.metrics-tmpdir.outputs.dir }}/execlogs.csv'
 
     - name: Upload execution log
       if: (success() || failure()) && inputs.execlogs-artifact-name != ''

.github/actions/bazel/bin/bazel

@@ -59,8 +59,8 @@ if [[ $bazel_command == "build" ]] || [[ $bazel_command == "test" ]]; then
     command_timestamp=$(date +%s)
     bazel_args+=(
 
-        # write execlogs, mostly for build reproducibility checks
-        --execution_log_json_file="$BAZEL_ACTION_METRICS_OUT/execlog-$command_timestamp.json"
+        # write execlog, mostly for build reproducibility checks
+        --execution_log_compact_file="$BAZEL_ACTION_METRICS_OUT/execlog-$command_timestamp.zst"
 
         --verbose_failures=true
 

WORKSPACE.bazel

@@ -265,3 +265,7 @@ load("//bazel:idl2json.bzl", "idl_to_json")
 idl_to_json(name = "idl2json")
 
 mainnet_icos_images()
+
+load("//bazel:spawn_proto_library.bzl", "spawn_proto_library")
+
+spawn_proto_library(name = "spawn_proto_library")

bazel/BUILD.bazel

@@ -1,7 +1,20 @@
 load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "string_flag")
 load("@buildifier_prebuilt//:rules.bzl", "buildifier")
+load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library")
+load("@rules_python//python:py_binary.bzl", "py_binary")
 load("//bazel:defs.bzl", "write_info_file_var")
 
+py_proto_library(
+    name = "spawn_py_pb2",
+    deps = ["@spawn_proto_library//:spawn_pb2"],
+)
+
+py_binary(
+    name = "execution_log_compact_to_csv",
+    srcs = ["execution_log_compact_to_csv.py"],
+    deps = [":spawn_py_pb2"],
+)
+
 bool_flag(
     name = "enable_malicious_code",
     build_setting_default = False,

bazel/execution_log_compact_to_csv.py

@@ -0,0 +1,106 @@
+# This Python script converts bazel's compact execution log (--execution_log_compact_file)
+# to a CSV file that contains a row per output file for every bazel target matching the given
+# --whitelist_pat regular expression. Each row contains the columns: target, path, and hash.
+#
+# Example usage:
+#
+# To invoke this script first generate a compact execution log with bazel using for example:
+#
+#   $ bazel build //rs/replica:replica --execution_log_compact_file=compact_execlog.zst
+#
+# Then decompress the file:
+#
+#   $ zstd -d compact_execlog.zst
+#
+# Finally build and run this script to convert the decompressed file to a CSV:
+#
+#   $ bazel build //bazel:execution_log_compact_to_csv && \
+#     bazel-bin/bazel/execution_log_compact_to_csv \
+#       --input_execlog=compact_execlog \
+#       --whitelist_pat='^//'
+#
+#   //rs/http_endpoints/public:build_script_,bazel-out/k8-opt-exec-ST-d57f47055a04/bin/rs/http_endpoints/public/build_script_,4463251579c194bf83a24408a184da9a7f46d5777f753e33a77afa5c2287fd1d
+#   //rs/http_endpoints/public:build_script,bazel-out/k8-opt/bin/rs/http_endpoints/public/build_script.out_dir/dashboard.rs,f0ba2e4976ac312782190b6391ce945f57708b46126b741f6336b355968a8a84
+#   ...
+#   //rs/replica:replica_lib,bazel-out/k8-opt/bin/rs/replica/libic_replica-974340861.rlib,0d45ec60aee4472020f69d0a19ac2ad4519388216f5a21e19152539fa3d0c4ce
+#   //rs/replica:replica,bazel-out/k8-opt/bin/rs/replica/replica,0ecf150fbff09dc7b92ac976827fc46179561f2e8e83487840330ce247b61b5d
+
+import argparse
+import csv
+import re
+import sys
+
+import google.protobuf.internal
+import spawn_pb2
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Convert bazel's compact execution log to a CSV file printed to stdout."
+    )
+    parser.add_argument(
+        "--input_execlog",
+        required=True,
+        help="Path to the zstd _decompressed_ input file as generated with bazel's --execution_log_compact_file option.",
+    )
+    parser.add_argument(
+        "--whitelist_pat",
+        default=None,
+        help="Regex to match target labels to include in the CSV. If omitted, all targets are included.",
+    )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Log every ExecLogEntry to stderr. This is useful for debugging.",
+    )
+    args = parser.parse_args()
+
+    # We intend to log the outputs of Spawns. Spawns refer to their output by ID
+    # so we need to keep track of the ID to entry mapping. Important note:
+    # if an entry (like a Spawn) refers to another entry (like a File or Directory)
+    # it's guaranteed the latter will precede the former.
+    id_to_entry = {}
+
+    with open(args.input_execlog, "rb") as execlog_file, sys.stdout as csv_file:
+        writer = csv.writer(csv_file)
+        while True:
+            # The execution log is a stream of protobuf messages,
+            # each preceded by a varint32 denoting its length.
+            msg_len = google.protobuf.internal.decoder._DecodeVarint32(execlog_file)
+            if msg_len is None:
+                break
+
+            # Parse the ExecLogEntry message. For the message definition see:
+            # https://github.com/bazelbuild/bazel/blob/master/src/main/protobuf/spawn.proto.
+            msg_buf = execlog_file.read(msg_len)
+            exec_log_entry = spawn_pb2.ExecLogEntry()
+            exec_log_entry.ParseFromString(msg_buf)
+
+            if args.verbose:
+                print(exec_log_entry, file=sys.stderr)
+
+            entry_type = exec_log_entry.WhichOneof("type")
+            if entry_type in ["file", "directory"]:
+                id_to_entry[exec_log_entry.id] = exec_log_entry
+            elif entry_type == "spawn":
+                label = exec_log_entry.spawn.target_label
+                if args.whitelist_pat is not None and not re.match(args.whitelist_pat, label):
+                    continue
+                for output in exec_log_entry.spawn.outputs:
+                    output_type = output.WhichOneof("type")
+                    if output_type != "output_id":
+                        continue
+                    output_entry = id_to_entry[output.output_id]
+                    output_entry_type = output_entry.WhichOneof("type")
+                    if output_entry_type == "file":
+                        writer.writerow([label, output_entry.file.path, output_entry.file.digest.hash])
+                    elif output_entry_type == "directory":
+                        dir_path = output_entry.directory.path
+                        for dir_file in output_entry.directory.files:
+                            writer.writerow([label, dir_path + "/" + dir_file.path, dir_file.digest.hash])
+                    else:
+                        raise ValueError(f"Unexpected output entry type: {output_entry_type}")
+
+
+if __name__ == "__main__":
+    main()

bazel/spawn_proto_library.bzl

@@ -0,0 +1,28 @@
+"""
+This module fetches the spawn.proto file from the bazel source code and turns it into a proto_library target.
+"""
+
+def _spawn_proto_library_impl(ctx):
+    ctx.download(
+        # We assume the ExecLogEntry proto message is forward compatible such that we can use the spawn.proto from bazel-7.6.0 even after upgrading bazel.
+        url = "https://raw.githubusercontent.com/bazelbuild/bazel/refs/tags/7.6.0/src/main/protobuf/spawn.proto",
+        output = "spawn.proto",
+        sha256 = "381bcb109e2855d4055fdc75988cc2144e0a92aa8586298123bb662cdefa6afe",
+    )
+
+    ctx.file("BUILD.bazel", """
+load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
+
+package(default_visibility = ["//visibility:public"])
+
+proto_library(
+    name = "spawn_pb2",
+    srcs = ["spawn.proto"],
+    deps = [
+        "@com_google_protobuf//:duration_proto",
+        "@com_google_protobuf//:timestamp_proto",
+    ],
+)
+    """)
+
+spawn_proto_library = repository_rule(implementation = _spawn_proto_library_impl)