diff --git a/.github/workflows/build-android.yml b/.github/workflows/build-android.yml index 5fc24d8d3492..7a41f4733660 100644 --- a/.github/workflows/build-android.yml +++ b/.github/workflows/build-android.yml @@ -16,6 +16,7 @@ on: ] pull_request: + branches-ignore: ['hrx-integration'] types: [opened, synchronize, reopened] paths: [ '.github/workflows/build-android.yml', diff --git a/.github/workflows/build-apple.yml b/.github/workflows/build-apple.yml index b99e614666e1..f443178958cb 100644 --- a/.github/workflows/build-apple.yml +++ b/.github/workflows/build-apple.yml @@ -19,6 +19,7 @@ on: ] pull_request: + branches-ignore: ['hrx-integration'] types: [opened, synchronize, reopened] paths: [ '.github/workflows/build-apple.yml', diff --git a/.github/workflows/build-cann.yml b/.github/workflows/build-cann.yml index d39b87637339..8cf3b4c356d1 100644 --- a/.github/workflows/build-cann.yml +++ b/.github/workflows/build-cann.yml @@ -16,6 +16,7 @@ on: ] pull_request: + branches-ignore: ['hrx-integration'] types: [opened, synchronize, reopened] paths: [ '.github/workflows/build-cann.yml', diff --git a/.github/workflows/build-hrx.yml b/.github/workflows/build-hrx.yml index 071f7a677ae3..d802d0978593 100644 --- a/.github/workflows/build-hrx.yml +++ b/.github/workflows/build-hrx.yml @@ -1,6 +1,9 @@ name: HRX CI on: + push: + branches: + - hrx-integration pull_request: types: [opened, synchronize, reopened] branches: @@ -16,26 +19,69 @@ concurrency: jobs: hrx-build: - name: ubuntu-latest - runs-on: ubuntu-latest + name: ${{ matrix.name }} (${{ matrix.gpu_target }}) + strategy: + # Per-arch failures are independent; keep both signals. + fail-fast: false + matrix: + include: + - name: gfx1151_strix-halo + gpu_target: gfx1151 + runs_on: linux-gfx1151-gpu-rocm + extra_rocm_artifacts: rocwmma_dev_gfx1151 + - name: gfx1201_9070 + gpu_target: gfx1201 + runs_on: linux-gfx120X-gpu-rocm + extra_rocm_artifacts: rocwmma_dev_gfx120X-all + runs-on: ${{ matrix.runs_on }} + timeout-minutes: 10 + defaults: + run: + # --noprofile skips loading ~/.bash_profile + # --norc skips loading ~/.bashrc + # -exo pipefail: + # -e exit immediately if any command fails + # -x print each command before executing + # -o pipefail ensure pipeline fails if any command in it fails + # {0} github actions placeholder for the command to run + shell: bash --noprofile --norc -exo pipefail {0} + container: + image: ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:fba5f55a122dbb15925e98c51fe65bffe88c36e11ebb25b73daf2bea04202dc3 + # --user 0:0: actions running inside the container need to write to some + # files set up outside the container by the runner agent. In + # June 2026 some runner agents set those files up with + # permissions for a "tester" user with UID/GID 1001, which + # matches the user in no_rocm_image_ubuntu24_04, and some are + # set up as root; accessing a file owned by root with user + # "tester" gives an EACCES. Running as root is the common + # denominator. + # --device kfd/dri: GPU access. + options: >- + --user 0:0 + --device /dev/kfd + --device /dev/dri env: - HRX_WORK_DIR: ${{ github.workspace }} + # Write out of tree build/install to a non-bind-mounted directory. In June + # 2026 a debug test showed some runners had assets dated from 6 months ago + # in the bind-mounted github workspace, it seems like nothing cleans it up + # currently. Writing to an non-bind-mounted directory ensures that + # rocm-artifacts/builds are cleaned up when docker container is removed. + HRX_WORK_DIR: /work # Public location for bench tooling (rsuderman/llamacpp_ci and fork # AaronStGeorge/llamacpp_ci) #TODO: switch to ROCm/llamacpp-hrx-bench once it is open sourced. BENCH_REPOSITORY: AaronStGeorge/llamacpp_ci BENCH_REF: 'main' + # actions/checkout@v6 does auto-cleanup, an in-tree build would be + # auto-cleaned as well. BENCH_DIR: ${{ github.workspace }}/bench LLAMA_SRC_DIR: ${{ github.workspace }}/llama-src - HRX_ARTIFACT_SET: 'core-with-upstream-hip' + LLAMA_BUILD_DIR: /work/llama-build + HRX_EXTRA_ROCM_ARTIFACTS: ${{ matrix.extra_rocm_artifacts }} + GGML_HRX_AMDGPU_TARGETS: ${{ matrix.gpu_target }} CCACHE_COMPILERCHECK: content steps: - - name: Checkout llama.cpp (under test) - uses: actions/checkout@v6 - with: - path: llama-src - - name: Checkout bench tooling uses: actions/checkout@v6 with: @@ -43,13 +89,25 @@ jobs: ref: ${{ env.BENCH_REF }} path: bench + - name: Runner info + env: + MATRIX_NAME: ${{ matrix.name }} + MATRIX_GPU_TARGET: ${{ matrix.gpu_target }} + MATRIX_RUNS_ON: ${{ matrix.runs_on }} + run: "${BENCH_DIR}/scripts/hrx/runner-info.sh" + + - name: Checkout llama.cpp (under test) + uses: actions/checkout@v6 + with: + path: llama-src + - name: Install ROCm build dependencies run: "${BENCH_DIR}/scripts/hrx/install-rocm-deps.sh" - name: ccache uses: ggml-org/ccache-action@v1.2.21 with: - key: ubuntu-latest-${{ env.HRX_ARTIFACT_SET }} + key: ${{ matrix.gpu_target }} evict-old-files: 1d save: ${{ github.event_name == 'pull_request' }} @@ -68,28 +126,30 @@ jobs: - name: Build llama.cpp with HRX run: "${BENCH_DIR}/scripts/hrx/build-llama-hrx.sh" - - name: Run sample MUL_MAT correctness config on CPU + - name: Run sample MUL_MAT correctness config on HRX run: | . "${BENCH_DIR}/scripts/hrx/env.sh" + . "${BENCH_DIR}/scripts/hrx/runtime-env.sh" "${BENCH_DIR}/tools/run-op-test.py" \ --test-backend-ops "${LLAMA_BUILD_DIR}/bin/test-backend-ops" \ --test-file "${BENCH_DIR}/benchmark-configs/test/mul_mat_f16.txt" \ --op MUL_MAT \ - --backend CPU \ - --output benchmark-results/sample-mul-mat-f16-cpu-test.jsonl + --backend HRX0 \ + --output benchmark-results/sample-mul-mat-f16-hrx-test.jsonl - - name: Run sample MUL_MAT benchmark config on CPU + - name: Run sample MUL_MAT benchmark config on HRX run: | . "${BENCH_DIR}/scripts/hrx/env.sh" + . "${BENCH_DIR}/scripts/hrx/runtime-env.sh" "${BENCH_DIR}/tools/run-op-perf.py" \ --test-backend-ops "${LLAMA_BUILD_DIR}/bin/test-backend-ops" \ --test-file "${BENCH_DIR}/benchmark-configs/test/mul_mat_f16.txt" \ --op MUL_MAT \ - --backend CPU \ - --output benchmark-results/sample-mul-mat-f16-cpu-perf.jsonl + --backend HRX0 \ + --output benchmark-results/sample-mul-mat-f16-hrx-perf.jsonl - - name: Upload CPU benchmark results + - name: Upload benchmark results uses: actions/upload-artifact@v5 with: - name: cpu-benchmark-results + name: benchmark-results-${{ matrix.name }} path: benchmark-results/ diff --git a/.github/workflows/build-riscv.yml b/.github/workflows/build-riscv.yml index 9733dbaa7a21..5fc7c8dbc407 100644 --- a/.github/workflows/build-riscv.yml +++ b/.github/workflows/build-riscv.yml @@ -16,6 +16,7 @@ on: ] pull_request: + branches-ignore: ['hrx-integration'] types: [opened, synchronize, reopened] paths: [ '.github/workflows/build-riscv.yml', diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml index eeea820ba169..2dc44a7c947c 100644 --- a/.github/workflows/build-self-hosted.yml +++ b/.github/workflows/build-self-hosted.yml @@ -24,6 +24,7 @@ on: ] pull_request: + branches-ignore: ['hrx-integration'] types: [opened, synchronize, reopened] paths: [ '.github/workflows/build-self-hosted.yml', diff --git a/.github/workflows/build-vulkan.yml b/.github/workflows/build-vulkan.yml index de38bb2db6d3..8615eec563b4 100644 --- a/.github/workflows/build-vulkan.yml +++ b/.github/workflows/build-vulkan.yml @@ -18,6 +18,7 @@ on: ] pull_request: + branches-ignore: ['hrx-integration'] types: [opened, synchronize, reopened] paths: [ '.github/workflows/build-vulkan.yml', diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f4ae3675602e..05a25904fef4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,6 +25,7 @@ on: ] pull_request: + branches-ignore: ['hrx-integration'] types: [opened, synchronize, reopened] paths: [ '.github/workflows/build.yml', diff --git a/.github/workflows/check-vendor.yml b/.github/workflows/check-vendor.yml index 1671ed7b8bd2..97093c7a39de 100644 --- a/.github/workflows/check-vendor.yml +++ b/.github/workflows/check-vendor.yml @@ -11,6 +11,7 @@ on: ] pull_request: + branches-ignore: ['hrx-integration'] types: [opened, synchronize, reopened] paths: [ 'vendor/**', diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index 6f648bac45b7..43e8aca91274 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -5,9 +5,11 @@ name: "Copilot Setup Steps" on: workflow_dispatch: push: + branches-ignore: ['hrx-integration'] paths: - .github/workflows/copilot-setup-steps.yml pull_request: + branches-ignore: ['hrx-integration'] paths: - .github/workflows/copilot-setup-steps.yml diff --git a/.github/workflows/hip-quality-check.yml b/.github/workflows/hip-quality-check.yml index d00d30ed65ce..cd92020c28d1 100644 --- a/.github/workflows/hip-quality-check.yml +++ b/.github/workflows/hip-quality-check.yml @@ -13,6 +13,7 @@ on: ] pull_request: + branches-ignore: ['hrx-integration'] types: [opened, synchronize, reopened] paths: [ '.github/workflows/hip-quality-check.yml', diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index eab20c68811e..ef53773bf284 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -1,6 +1,7 @@ name: "Pull Request Labeler" on: -- pull_request_target + pull_request_target: + branches-ignore: ['hrx-integration'] jobs: labeler: diff --git a/.github/workflows/pre-tokenizer-hashes.yml b/.github/workflows/pre-tokenizer-hashes.yml index 7126b62b690b..111d632d816c 100644 --- a/.github/workflows/pre-tokenizer-hashes.yml +++ b/.github/workflows/pre-tokenizer-hashes.yml @@ -2,10 +2,12 @@ name: Check Pre-Tokenizer Hashes on: push: + branches-ignore: ['hrx-integration'] paths: - 'convert_hf_to_gguf.py' - 'convert_hf_to_gguf_update.py' pull_request: + branches-ignore: ['hrx-integration'] paths: - 'convert_hf_to_gguf.py' - 'convert_hf_to_gguf_update.py' diff --git a/.github/workflows/python-check-requirements.yml b/.github/workflows/python-check-requirements.yml index 1219b8745927..926929e9250d 100644 --- a/.github/workflows/python-check-requirements.yml +++ b/.github/workflows/python-check-requirements.yml @@ -2,12 +2,14 @@ name: Python check requirements.txt on: push: + branches-ignore: ['hrx-integration'] paths: - '.github/workflows/python-check-requirements.yml' - 'scripts/check-requirements.sh' - 'convert*.py' - '**/requirements*.txt' pull_request: + branches-ignore: ['hrx-integration'] paths: - '.github/workflows/python-check-requirements.yml' - 'scripts/check-requirements.sh' diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml index 1e5d64c1aee6..4a813dbcf80e 100644 --- a/.github/workflows/python-lint.yml +++ b/.github/workflows/python-lint.yml @@ -9,6 +9,7 @@ on: '**/*.py' ] pull_request: + branches-ignore: ['hrx-integration'] types: [opened, synchronize, reopened] paths: [ '.github/workflows/python-lint.yml', diff --git a/.github/workflows/python-type-check.yml b/.github/workflows/python-type-check.yml index dc7aebe24ca2..bf7580960057 100644 --- a/.github/workflows/python-type-check.yml +++ b/.github/workflows/python-type-check.yml @@ -2,6 +2,7 @@ name: Python Type-Check on: push: + branches-ignore: ['hrx-integration'] paths: - '.github/workflows/python-type-check.yml' - 'ty.toml' @@ -9,6 +10,7 @@ on: - '**/requirements*.txt' # - 'pyrightconfig.json' pull_request: + branches-ignore: ['hrx-integration'] paths: - '.github/workflows/python-type-check.yml' - 'ty.toml' diff --git a/.github/workflows/server-webui.yml b/.github/workflows/server-webui.yml index 492107ffd851..132b7a50b822 100644 --- a/.github/workflows/server-webui.yml +++ b/.github/workflows/server-webui.yml @@ -17,6 +17,7 @@ on: 'tools/server/public/**' ] pull_request: + branches-ignore: ['hrx-integration'] types: [opened, synchronize, reopened] paths: [ '.github/workflows/server-webui.yml', diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 750c29f08e5c..4d642e541481 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -28,6 +28,7 @@ on: 'tools/server/**.*' ] pull_request: + branches-ignore: ['hrx-integration'] types: [opened, synchronize, reopened] paths: [ '.github/workflows/server.yml', diff --git a/.github/workflows/update-ops-docs.yml b/.github/workflows/update-ops-docs.yml index 2ab06eb9811d..934ea783db4a 100644 --- a/.github/workflows/update-ops-docs.yml +++ b/.github/workflows/update-ops-docs.yml @@ -2,11 +2,13 @@ name: Update Operations Documentation on: push: + branches-ignore: ['hrx-integration'] paths: - 'docs/ops.md' - 'docs/ops/**' - 'scripts/create_ops_docs.py' pull_request: + branches-ignore: ['hrx-integration'] paths: - 'docs/ops.md' - 'docs/ops/**' diff --git a/ggml/src/ggml-hrx/ggml-hrx.cpp b/ggml/src/ggml-hrx/ggml-hrx.cpp index 120a422b4d10..1f14a69a281a 100644 --- a/ggml/src/ggml-hrx/ggml-hrx.cpp +++ b/ggml/src/ggml-hrx/ggml-hrx.cpp @@ -2521,7 +2521,7 @@ static bool ggml_backend_hrx_load_catalog_provider( executable, export_ordinal, &export_info)) && export_info.binding_count == entry->binding_count && export_info.parameter_count == entry->parameter_count && - export_info.constant_count * sizeof(uint32_t) == entry->constants_size; + export_info.constant_byte_length == entry->constants_size; if (!ok) { GGML_LOG_WARN( "%s: HRX catalog kernel %s has unsupported ABI " @@ -2530,7 +2530,7 @@ static bool ggml_backend_hrx_load_catalog_provider( entry->name, export_info.binding_count, entry->binding_count, - export_info.constant_count, + export_info.constant_byte_length, entry->constants_size, export_info.parameter_count, entry->parameter_count,