From 317c41143adbccf418d86102a2c05e5e98932b48 Mon Sep 17 00:00:00 2001 From: luszczewskakasia1 Date: Tue, 9 Dec 2025 12:46:57 +0000 Subject: [PATCH] Modify benchmark scripts and CI workflow to enable gdb_mode while debugging benchmarks --- .github/workflows/sycl-linux-run-tests.yml | 8 +++++ .../workflows/sycl-ur-perf-benchmarking.yml | 11 ++++++- devops/actions/run-tests/benchmark/action.yml | 31 ++++++++++++------- devops/scripts/benchmarks/benches/compute.py | 15 +++++++++ devops/scripts/benchmarks/git_project.py | 6 +++- devops/scripts/benchmarks/utils/utils.py | 19 ++++++++++++ 6 files changed, 77 insertions(+), 13 deletions(-) diff --git a/.github/workflows/sycl-linux-run-tests.yml b/.github/workflows/sycl-linux-run-tests.yml index 09e9a91e44e32..98121e5f7aa04 100644 --- a/.github/workflows/sycl-linux-run-tests.yml +++ b/.github/workflows/sycl-linux-run-tests.yml @@ -34,6 +34,9 @@ on: Extra options to be added to LIT_OPTS. type: string default: '' + gdb_mode: + type: string + required: False repo_ref: type: string @@ -134,6 +137,10 @@ on: type: string default: 'false' required: False + benchmark_gdb_mode: + type: string + required: False + default: 'false' in_workflow_call_mode: description: | @@ -379,3 +386,4 @@ jobs: exit_on_failure: ${{ inputs.benchmark_exit_on_failure }} build_ref: ${{ inputs.repo_ref }} runner: ${{ inputs.runner }} + gdb_mode: ${{ inputs.benchmark_gdb_mode }} diff --git a/.github/workflows/sycl-ur-perf-benchmarking.yml b/.github/workflows/sycl-ur-perf-benchmarking.yml index 83ff32bfc65b9..e68e2fa8ba5b9 100644 --- a/.github/workflows/sycl-ur-perf-benchmarking.yml +++ b/.github/workflows/sycl-ur-perf-benchmarking.yml @@ -86,6 +86,12 @@ on: - 'false' - 'true' default: 'false' + gdb_mode: + type: choice + options: + - false + - true + default: false concurrency: # Cancel a currently running workflow for: @@ -183,7 +189,7 @@ jobs: uses: ./.github/workflows/sycl-linux-run-tests.yml secrets: inherit with: - name: "Benchmarks (${{ matrix.runner }}, ${{ matrix.backend }}, preset: ${{ inputs.preset }})" + name: "Benchmarks (${{ matrix.runner }}, ${{ matrix.backend }}, preset: ${{ inputs.preset }}, gdb_mode: ${{ inputs.gdb_mode }})" runner: ${{ matrix.runner }} image: ghcr.io/intel/llvm/sycl_ubuntu2404_nightly:latest image_options: -u 1001 --device=/dev/dri -v /dev/dri/by-path:/dev/dri/by-path --privileged --cap-add SYS_ADMIN @@ -193,6 +199,7 @@ jobs: benchmark_save_name: ${{ needs.sanitize_inputs_dispatch.outputs.benchmark_save_name }} benchmark_preset: ${{ inputs.preset }} benchmark_exit_on_failure: ${{ inputs.exit_on_failure }} + benchmark_gdb_mode: ${{ inputs.gdb_mode }} repo_ref: ${{ needs.sanitize_inputs_dispatch.outputs.build_ref }} toolchain_artifact: ${{ needs.build_sycl_dispatch.outputs.toolchain_artifact }} toolchain_artifact_filename: ${{ needs.build_sycl_dispatch.outputs.toolchain_artifact_filename }} @@ -241,6 +248,7 @@ jobs: benchmark_upload_results: true benchmark_save_name: ${{ matrix.save_name }} benchmark_preset: ${{ matrix.preset }} + benchmark_gdb_mode: false repo_ref: ${{ matrix.ref }} toolchain_artifact: ${{ needs.build_nightly.outputs.toolchain_artifact }} toolchain_artifact_filename: ${{ needs.build_nightly.outputs.toolchain_artifact_filename }} @@ -267,5 +275,6 @@ jobs: benchmark_preset: 'Minimal' benchmark_dry_run: true benchmark_exit_on_failure: true + benchmark_gdb_mode: ${{ inputs.gdb_mode }} repo_ref: ${{ github.sha }} # END benchmark framework builds and runs on PRs path diff --git a/devops/actions/run-tests/benchmark/action.yml b/devops/actions/run-tests/benchmark/action.yml index f4dfaef05b20a..66f9f5518d245 100644 --- a/devops/actions/run-tests/benchmark/action.yml +++ b/devops/actions/run-tests/benchmark/action.yml @@ -23,6 +23,9 @@ inputs: type: string required: False default: "" + gdb_mode: + type: string + required: False # dry-run is passed only to compare.py (to not fail on regression), not to main.py (where such flag would omit all benchmark runs) dry_run: type: string @@ -267,6 +270,7 @@ runs: env: BENCH_WORKDIR: ${{ steps.establish_outputs.outputs.BENCH_WORKDIR }} BENCHMARK_RESULTS_REPO_PATH: ${{ steps.establish_outputs.outputs.BENCHMARK_RESULTS_REPO_PATH }} + LLVM_BENCHMARKS_USE_GDB: ${{ inputs.gdb_mode }} run: | # Build and run benchmarks @@ -280,6 +284,7 @@ runs: echo "::endgroup::" echo "::group::run_benchmarks" + export LLVM_BENCHMARKS_USE_GDB=${LLVM_BENCHMARKS_USE_GDB} numactl --cpunodebind "$NUMA_NODE" --membind "$NUMA_NODE" \ ./devops/scripts/benchmarks/main.py "$BENCH_WORKDIR" \ --sycl "$(realpath $CMPLR_ROOT)" \ @@ -299,17 +304,21 @@ runs: echo "::endgroup::" echo "::group::compare_results" - python3 ./devops/scripts/benchmarks/compare.py to_hist \ - --avg-type EWMA \ - --cutoff "$(date -u -d '7 days ago' +'%Y%m%d_%H%M%S')" \ - --name "$SAVE_NAME" \ - --compare-file "${BENCHMARK_RESULTS_REPO_PATH}/results/${SAVE_NAME}_${SAVE_TIMESTAMP}.json" \ - --results-dir "${BENCHMARK_RESULTS_REPO_PATH}/results/" \ - --regression-filter '^[a-z_]+_sycl .* CPU count' \ - --regression-filter-type 'SYCL benchmark (measured using CPU cycle count)' \ - --verbose \ - --produce-github-summary \ - ${{ inputs.dry_run == 'true' && '--dry-run' || '' }} \ + if [ "$LLVM_BENCHMARKS_USE_GDB" == "false" ]; then + python3 ./devops/scripts/benchmarks/compare.py to_hist \ + --avg-type EWMA \ + --cutoff "$(date -u -d '7 days ago' +'%Y%m%d_%H%M%S')" \ + --name "$SAVE_NAME" \ + --compare-file "${BENCHMARK_RESULTS_REPO_PATH}/results/${SAVE_NAME}_${SAVE_TIMESTAMP}.json" \ + --results-dir "${BENCHMARK_RESULTS_REPO_PATH}/results/" \ + --regression-filter '^[a-z_]+_sycl .* CPU count' \ + --regression-filter-type 'SYCL benchmark (measured using CPU cycle count)' \ + --verbose \ + --produce-github-summary \ + ${{ inputs.dry_run == 'true' && '--dry-run' || '' }} + else + echo "Skipping regression comparison due to GDB mode enabled." + fi echo "::endgroup::" - name: Run benchmarks integration tests diff --git a/devops/scripts/benchmarks/benches/compute.py b/devops/scripts/benchmarks/benches/compute.py index 348fe3be79548..059d116754ef7 100644 --- a/devops/scripts/benchmarks/benches/compute.py +++ b/devops/scripts/benchmarks/benches/compute.py @@ -6,6 +6,7 @@ import csv import io import math +import os from enum import Enum from itertools import product from pathlib import Path @@ -76,6 +77,7 @@ def setup(self) -> None: "compute-benchmarks", use_installdir=False, ) + print(self._project) if not self._project.needs_rebuild(): log.info(f"Rebuilding {self._project.name} skipped") @@ -89,6 +91,13 @@ def setup(self) -> None: f"-DCMAKE_CXX_COMPILER=clang++", f"-DCMAKE_C_COMPILER=clang", ] + + is_gdb_mode = os.environ.get("LLVM_BENCHMARKS_USE_GDB", "") == "true" + if is_gdb_mode: + extra_args += [ + f"-DCMAKE_CXX_FLAGS_RELWITHDEBINFO:STRING=-O2 -g -DNDEBUG -fdebug-info-for-profiling", + ] + if options.ur_adapter == "cuda": extra_args += [ "-DBUILD_SYCL_WITH_CUDA=ON", @@ -567,6 +576,12 @@ def __enabled_runtimes(self) -> list[RUNTIMES]: return runtimes def __parse_output(self, output: str) -> list[tuple[float, float]]: + is_gdb_mode = os.environ.get("LLVM_BENCHMARKS_USE_GDB", "") == "true" + print(output) + + if is_gdb_mode: + return [(0.0, 0.0)] + csv_file = io.StringIO(output) reader = csv.reader(csv_file) next(reader, None) diff --git a/devops/scripts/benchmarks/git_project.py b/devops/scripts/benchmarks/git_project.py index 1761b0049224b..1038d66a41a20 100644 --- a/devops/scripts/benchmarks/git_project.py +++ b/devops/scripts/benchmarks/git_project.py @@ -74,11 +74,15 @@ def configure( add_sycl: bool = False, ) -> None: """Configures the project.""" + + is_gdb_mode = os.environ.get("LLVM_BENCHMARKS_USE_GDB", "") == "true" + build_type = "RelWithDebInfo" if is_gdb_mode else "Release" + cmd = [ "cmake", f"-S {self.src_dir}", f"-B {self.build_dir}", - f"-DCMAKE_BUILD_TYPE=Release", + f"-DCMAKE_BUILD_TYPE={build_type}", ] if self._use_installdir: cmd.append(f"-DCMAKE_INSTALL_PREFIX={self.install_dir}") diff --git a/devops/scripts/benchmarks/utils/utils.py b/devops/scripts/benchmarks/utils/utils.py index 6b6d0a5eb21a9..8f576271c40ad 100644 --- a/devops/scripts/benchmarks/utils/utils.py +++ b/devops/scripts/benchmarks/utils/utils.py @@ -48,6 +48,25 @@ def run( if isinstance(command, str): command = command.split() + is_gdb_mode = os.environ.get("LLVM_BENCHMARKS_USE_GDB", "") == "true" + if command[0] == "taskset" and is_gdb_mode: + gdb_cmd = [ + "gdb", + "-return-child-result", + "--batch", + "--ex", + "set confirm off", + "--ex", + "run", + "--ex", + "bt", + "--ex", + "quit", + "--args", + ] + command + command = gdb_cmd + print(f"Running in gdb mode: {command}") + env = os.environ.copy() for ldlib in ld_library: if os.path.isdir(ldlib):