From e1d9421b01c3609604ce632458178356a6504c09 Mon Sep 17 00:00:00 2001 From: Dilan Bhalla Date: Sun, 2 Nov 2025 21:16:03 -0800 Subject: [PATCH 1/7] iac windows extraction --- .github/workflows/build.yml | 31 +++++++++++++-- .github/workflows/publish.yml | 13 ++++-- scripts/create-extractor-pack.ps1 | 66 +++++++++++++++++++++++++------ scripts/install-extractor.ps1 | 26 ++++++++++++ scripts/run-tests.ps1 | 57 ++++++++++++++++++++++++++ tools/index-files.cmd | 1 + tools/qltest.cmd | 4 +- 7 files changed, 178 insertions(+), 20 deletions(-) create mode 100644 scripts/install-extractor.ps1 create mode 100644 scripts/run-tests.ps1 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1986a40..059c3b4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,9 +12,10 @@ permissions: jobs: tests: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: matrix: + os: [ubuntu-latest, windows-latest] test-folders: ["library-tests", "queries-tests"] steps: - name: "Checkout" @@ -53,8 +54,8 @@ jobs: - uses: dtolnay/rust-toolchain@4305c38b25d97ef35a8ad1f985ccf2d2242004f2 # stable if: steps.extractor-changes.outputs.src == 'true' - - name: "Build Extractor" - if: steps.extractor-changes.outputs.src == 'true' + - name: "Build Extractor (Linux/macOS)" + if: steps.extractor-changes.outputs.src == 'true' && matrix.os != 'windows-latest' env: GH_TOKEN: ${{ github.token }} run: | @@ -66,12 +67,34 @@ jobs: gh codeql resolve languages --format=json --search-path ./extractor-pack - - name: "Run Tests" + - name: "Build Extractor (Windows)" + if: steps.extractor-changes.outputs.src == 'true' && matrix.os == 'windows-latest' + env: + GH_TOKEN: ${{ github.token }} + shell: pwsh + run: | + gh extension install github/gh-codeql + gh codeql set-version latest + + .\scripts\create-extractor-pack.ps1 + + gh codeql resolve languages --format=json --search-path .\extractor-pack + + - name: "Run Tests (Linux/macOS)" + if: matrix.os != 'windows-latest' env: GH_TOKEN: ${{ github.token }} run: | ./scripts/run-tests.sh "ql/test/${{ matrix.test-folders }}" + - name: "Run Tests (Windows)" + if: matrix.os == 'windows-latest' + env: + GH_TOKEN: ${{ github.token }} + shell: pwsh + run: | + .\scripts\run-tests.ps1 "ql/test/${{ matrix.test-folders }}" + # scanning: # runs-on: ubuntu-latest # needs: [tests] diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 04522b5..c847eab 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -47,8 +47,7 @@ jobs: strategy: fail-fast: false matrix: - # TODO: Add windows-latest - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest, windows-latest] if: ${{ needs.release-check.outputs.release == 'true' }} steps: @@ -59,14 +58,20 @@ jobs: - name: "Set up Rust" uses: dtolnay/rust-toolchain@0c3131df9e5407c0c36352032d04af846dbe0fb7 # nightly - if: ${{ matrix.os != 'windows-latest' }} - - name: "Build Extractor" + - name: "Build Extractor (Linux/macOS)" if: ${{ matrix.os != 'windows-latest' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: ./scripts/create-extractor-pack.sh + - name: "Build Extractor (Windows)" + if: ${{ matrix.os == 'windows-latest' }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: pwsh + run: .\scripts\create-extractor-pack.ps1 + - name: "Upload bundle artifact" uses: actions/upload-artifact@v6 with: diff --git a/scripts/create-extractor-pack.ps1 b/scripts/create-extractor-pack.ps1 index 65b4dfc..fba204d 100644 --- a/scripts/create-extractor-pack.ps1 +++ b/scripts/create-extractor-pack.ps1 @@ -1,14 +1,58 @@ -cargo build --release +$ErrorActionPreference = "Stop" + +# Set platform +$platform = "win64" -cargo run --release -p ql-generator -- --dbscheme ql/src/ql.dbscheme --library ql/src/codeql_ql/ast/internal/TreeSitter.qll -codeql query format -i ql\src\codeql_ql\ast\internal\TreeSitter.qll +# Check for CodeQL binary +if (Get-Command "codeql" -ErrorAction SilentlyContinue) { + $CODEQL_BINARY = "codeql" +} +elseif (Get-Command "gh" -ErrorAction SilentlyContinue) { + try { + gh codeql version 2>&1 | Out-Null + $CODEQL_BINARY = "gh codeql" + } + catch { + Write-Host "Installing gh-codeql extension..." + gh extension install github/gh-codeql + $CODEQL_BINARY = "gh codeql" + } +} +else { + Write-Error "Neither 'codeql' nor 'gh' command found" + exit 1 +} +Write-Host "Creating extractor pack..." if (Test-Path -Path extractor-pack) { - rm -Recurse -Force extractor-pack -} -mkdir extractor-pack | Out-Null -cp codeql-extractor.yml, ql\src\ql.dbscheme, ql\src\ql.dbscheme.stats extractor-pack -cp -Recurse tools extractor-pack -mkdir extractor-pack\tools\win64 | Out-Null -cp target\release\ql-extractor.exe extractor-pack\tools\win64\extractor.exe -cp target\release\ql-autobuilder.exe extractor-pack\tools\win64\autobuilder.exe + Remove-Item -Recurse -Force extractor-pack +} +if (Test-Path -Path target) { + Remove-Item -Recurse -Force target +} + +Write-Host "Update submodules..." +git submodule update --init --recursive + +Write-Host "Building extractor..." +cargo build --release + +Write-Host "Generating TreeSitter library..." +cargo run --release --bin codeql-extractor-iac -- generate --dbscheme ql/lib/iac.dbscheme --library ql/lib/codeql/iac/ast/internal/TreeSitter.qll + +Write-Host "Formatting generated library..." +if ($CODEQL_BINARY -eq "gh codeql") { + gh codeql query format -i ql/lib/codeql/iac/ast/internal/TreeSitter.qll +} +else { + codeql query format -i ql/lib/codeql/iac/ast/internal/TreeSitter.qll +} + +New-Item -ItemType Directory -Path extractor-pack | Out-Null +Copy-Item -Path codeql-extractor.yml, ql/lib/iac.dbscheme, ql/lib/iac.dbscheme.stats -Destination extractor-pack/ +Copy-Item -Recurse -Path downgrades, tools -Destination extractor-pack/ + +New-Item -ItemType Directory -Path "extractor-pack/tools/$platform" -Force | Out-Null +Copy-Item -Path "target/release/codeql-extractor-iac.exe" -Destination "extractor-pack/tools/$platform/extractor.exe" + +Write-Host "Extractor pack created successfully!" diff --git a/scripts/install-extractor.ps1 b/scripts/install-extractor.ps1 new file mode 100644 index 0000000..d192284 --- /dev/null +++ b/scripts/install-extractor.ps1 @@ -0,0 +1,26 @@ +param( + [string]$ExtractorName = "iac", + [string]$ExtractorLocations = "$env:USERPROFILE\.codeql\extractors" +) + +$ErrorActionPreference = "Stop" + +Write-Host "Creating extractor directory..." +if (!(Test-Path $ExtractorLocations)) { + New-Item -ItemType Directory -Path $ExtractorLocations -Force | Out-Null +} + +Write-Host "Checking latest release..." +gh release list -L 1 -R "advanced-security/codeql-extractor-$ExtractorName" + +Write-Host "Downloading extractor pack..." +gh release download ` + -R "advanced-security/codeql-extractor-$ExtractorName" ` + -D "$ExtractorLocations" ` + --clobber ` + --pattern 'extractor-*.tar.gz' + +Write-Host "Extracting extractor pack..." +tar -zxf "$ExtractorLocations/extractor-$ExtractorName.tar.gz" --directory "$ExtractorLocations" + +Write-Host "Installation complete! Extractor installed to: $ExtractorLocations" \ No newline at end of file diff --git a/scripts/run-tests.ps1 b/scripts/run-tests.ps1 new file mode 100644 index 0000000..8bacc7f --- /dev/null +++ b/scripts/run-tests.ps1 @@ -0,0 +1,57 @@ +param( + [string]$TestsDir = "ql/test" +) + +$ErrorActionPreference = "Stop" + +# Check for CodeQL binary +if (Get-Command "codeql" -ErrorAction SilentlyContinue) { + $CODEQL_BINARY = "codeql" +} +elseif (Get-Command "gh" -ErrorAction SilentlyContinue) { + try { + gh codeql version 2>&1 | Out-Null + $CODEQL_BINARY = "gh codeql" + } + catch { + Write-Host "Installing gh-codeql extension..." + gh extension install github/gh-codeql + $CODEQL_BINARY = "gh codeql" + } +} +else { + Write-Error "Neither 'codeql' nor 'gh' command found" + exit 1 +} + +Write-Host "Installing ql/test pack dependencies..." +if ($CODEQL_BINARY -eq "gh codeql") { + gh codeql pack install ql/test +} +else { + codeql pack install ql/test +} + +Write-Host "Running tests in $TestsDir" + +if ($CODEQL_BINARY -eq "gh codeql") { + gh codeql test run ` + -j 0 ` + --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition ` + --search-path ./extractor-pack ` + "$TestsDir" +} +else { + codeql test run ` + -j 0 ` + --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition ` + --search-path ./extractor-pack ` + "$TestsDir" +} + +if ($LASTEXITCODE -ne 0) { + Write-Error "Tests failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE +} + +Write-Host "All tests passed!" \ No newline at end of file diff --git a/tools/index-files.cmd b/tools/index-files.cmd index ab05465..56a743b 100755 --- a/tools/index-files.cmd +++ b/tools/index-files.cmd @@ -1,6 +1,7 @@ @echo off type NUL && "%CODEQL_EXTRACTOR_IAC_ROOT%\tools\win64\extractor.exe" ^ + extract ^ --file-list "%1" ^ --source-archive-dir "%CODEQL_EXTRACTOR_IAC_SOURCE_ARCHIVE_DIR%" ^ --output-dir "%CODEQL_EXTRACTOR_IAC_TRAP_DIR%" diff --git a/tools/qltest.cmd b/tools/qltest.cmd index c960b58..233cb68 100755 --- a/tools/qltest.cmd +++ b/tools/qltest.cmd @@ -4,8 +4,10 @@ type NUL && "%CODEQL_DIST%\codeql.exe" database index-files ^ --prune=**/*.testproj ^ --include-extension=.hcl ^ --include-extension=.tf ^ + --include-extension=.tfvars ^ + --include-extension=.bicep ^ --size-limit=5m ^ - --language=hcl ^ + --language=iac ^ --working-dir=. ^ "%CODEQL_EXTRACTOR_IAC_WIP_DATABASE%" From e0b732f567bd7e37bee6c7dddbca6d4e68fe2ff6 Mon Sep 17 00:00:00 2001 From: Dilan Bhalla Date: Wed, 27 May 2026 10:22:41 -0700 Subject: [PATCH 2/7] Address PR review feedback and rebase on main Review fixes: - install-extractor.ps1: replace fragile backtick-continued gh release download invocation with array splatting; add LASTEXITCODE checks (fixes aegilops syntax-error report). - create-extractor-pack.ps1: normalize indentation to 4 spaces; fix catch alignment; add LASTEXITCODE checks for gh/cargo/git/tar (fixes 3 Copilot review comments). - run-tests.ps1: add LASTEXITCODE check after codeql pack install. - build.yml: fix typo 'gh extensions install' -> 'gh extension install'. Sanity-check fixes during refresh: - publish.yml: install Rust toolchain on Windows runner too (was previously skipped, leaving build to rely on preinstalled toolchain). - tools/qltest.cmd: mirror tools/qltest.sh by running a second 'database index-files' pass for .yml/.yaml/.json under --language=yaml, and restore .bicep extension. Without this, library tests for YAML/OpenAPI/CloudFormation/ARM had no source files extracted on Windows. - docs/workflows.md: document Windows PowerShell install/build/test scripts. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build.yml | 6 ++-- docs/workflows.md | 17 ++++++++++ scripts/create-extractor-pack.ps1 | 52 +++++++++++++++++++++---------- scripts/install-extractor.ps1 | 21 ++++++++++--- scripts/run-tests.ps1 | 4 +++ tools/qltest.cmd | 13 +++++++- 6 files changed, 88 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 059c3b4..ff0d2f6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -60,7 +60,7 @@ jobs: GH_TOKEN: ${{ github.token }} run: | set -e - gh extensions install github/gh-codeql + gh extension install github/gh-codeql gh codeql set-version latest ./scripts/create-extractor-pack.sh @@ -150,7 +150,7 @@ jobs: # GH_TOKEN: ${{ github.token }} # run: | # set -e - # gh extensions install github/gh-codeql + # gh extension install github/gh-codeql # gh codeql set-version latest # ./scripts/create-extractor-pack.sh @@ -163,7 +163,7 @@ jobs: # PROJECT_REPO: ${{ matrix.project }} # run: | # set -e - # gh extensions install github/gh-codeql + # gh extension install github/gh-codeql # gh codeql set-version latest # gh codeql database create --language=iac --source-root=./project --search-path ./extractor-pack iac-db diff --git a/docs/workflows.md b/docs/workflows.md index 87a87db..b8c5a74 100644 --- a/docs/workflows.md +++ b/docs/workflows.md @@ -113,3 +113,20 @@ codeql database analyze \ "$CODEQL_DATABASE" \ "advanced-security/iac-queries" ``` + +#### Install extractor on Windows + +PowerShell equivalents of the scripts above are provided for Windows. From a checkout of this repository: + +```powershell +# Install the extractor into $env:USERPROFILE\.codeql\extractors (default) +.\scripts\install-extractor.ps1 + +# Or build the extractor pack locally +.\scripts\create-extractor-pack.ps1 + +# Run the test suite +.\scripts\run-tests.ps1 ql\test\library-tests +``` + +Both `pwsh` (PowerShell 7+) and Windows PowerShell 5.1 are supported. The scripts require either `codeql` or `gh` on the `PATH`; if only `gh` is available, the `gh-codeql` extension will be installed automatically. diff --git a/scripts/create-extractor-pack.ps1 b/scripts/create-extractor-pack.ps1 index fba204d..a657c09 100644 --- a/scripts/create-extractor-pack.ps1 +++ b/scripts/create-extractor-pack.ps1 @@ -5,47 +5,67 @@ $platform = "win64" # Check for CodeQL binary if (Get-Command "codeql" -ErrorAction SilentlyContinue) { - $CODEQL_BINARY = "codeql" + $CODEQL_BINARY = "codeql" } elseif (Get-Command "gh" -ErrorAction SilentlyContinue) { - try { - gh codeql version 2>&1 | Out-Null - $CODEQL_BINARY = "gh codeql" - } - catch { - Write-Host "Installing gh-codeql extension..." - gh extension install github/gh-codeql - $CODEQL_BINARY = "gh codeql" - } + try { + gh codeql version 2>&1 | Out-Null + $CODEQL_BINARY = "gh codeql" + } + catch { + Write-Host "Installing gh-codeql extension..." + gh extension install github/gh-codeql + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to install gh-codeql extension (exit code $LASTEXITCODE)" + exit $LASTEXITCODE + } + $CODEQL_BINARY = "gh codeql" + } } else { - Write-Error "Neither 'codeql' nor 'gh' command found" - exit 1 + Write-Error "Neither 'codeql' nor 'gh' command found" + exit 1 } Write-Host "Creating extractor pack..." if (Test-Path -Path extractor-pack) { - Remove-Item -Recurse -Force extractor-pack + Remove-Item -Recurse -Force extractor-pack } if (Test-Path -Path target) { - Remove-Item -Recurse -Force target + Remove-Item -Recurse -Force target } Write-Host "Update submodules..." git submodule update --init --recursive +if ($LASTEXITCODE -ne 0) { + Write-Error "git submodule update failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE +} Write-Host "Building extractor..." cargo build --release +if ($LASTEXITCODE -ne 0) { + Write-Error "cargo build failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE +} Write-Host "Generating TreeSitter library..." cargo run --release --bin codeql-extractor-iac -- generate --dbscheme ql/lib/iac.dbscheme --library ql/lib/codeql/iac/ast/internal/TreeSitter.qll +if ($LASTEXITCODE -ne 0) { + Write-Error "TreeSitter generation failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE +} Write-Host "Formatting generated library..." if ($CODEQL_BINARY -eq "gh codeql") { - gh codeql query format -i ql/lib/codeql/iac/ast/internal/TreeSitter.qll + gh codeql query format -i ql/lib/codeql/iac/ast/internal/TreeSitter.qll } else { - codeql query format -i ql/lib/codeql/iac/ast/internal/TreeSitter.qll + codeql query format -i ql/lib/codeql/iac/ast/internal/TreeSitter.qll +} +if ($LASTEXITCODE -ne 0) { + Write-Error "codeql query format failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE } New-Item -ItemType Directory -Path extractor-pack | Out-Null diff --git a/scripts/install-extractor.ps1 b/scripts/install-extractor.ps1 index d192284..d36f9fe 100644 --- a/scripts/install-extractor.ps1 +++ b/scripts/install-extractor.ps1 @@ -14,13 +14,24 @@ Write-Host "Checking latest release..." gh release list -L 1 -R "advanced-security/codeql-extractor-$ExtractorName" Write-Host "Downloading extractor pack..." -gh release download ` - -R "advanced-security/codeql-extractor-$ExtractorName" ` - -D "$ExtractorLocations" ` - --clobber ` - --pattern 'extractor-*.tar.gz' +$ghArgs = @( + 'release', 'download', + '-R', "advanced-security/codeql-extractor-$ExtractorName", + '-D', "$ExtractorLocations", + '--clobber', + '--pattern', 'extractor-*.tar.gz' +) +& gh @ghArgs +if ($LASTEXITCODE -ne 0) { + Write-Error "gh release download failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE +} Write-Host "Extracting extractor pack..." tar -zxf "$ExtractorLocations/extractor-$ExtractorName.tar.gz" --directory "$ExtractorLocations" +if ($LASTEXITCODE -ne 0) { + Write-Error "tar extraction failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE +} Write-Host "Installation complete! Extractor installed to: $ExtractorLocations" \ No newline at end of file diff --git a/scripts/run-tests.ps1 b/scripts/run-tests.ps1 index 8bacc7f..038638e 100644 --- a/scripts/run-tests.ps1 +++ b/scripts/run-tests.ps1 @@ -31,6 +31,10 @@ if ($CODEQL_BINARY -eq "gh codeql") { else { codeql pack install ql/test } +if ($LASTEXITCODE -ne 0) { + Write-Error "codeql pack install failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE +} Write-Host "Running tests in $TestsDir" diff --git a/tools/qltest.cmd b/tools/qltest.cmd index 233cb68..88c344b 100755 --- a/tools/qltest.cmd +++ b/tools/qltest.cmd @@ -1,6 +1,6 @@ @echo off -type NUL && "%CODEQL_DIST%\codeql.exe" database index-files ^ +"%CODEQL_DIST%\codeql.exe" database index-files ^ --prune=**/*.testproj ^ --include-extension=.hcl ^ --include-extension=.tf ^ @@ -10,5 +10,16 @@ type NUL && "%CODEQL_DIST%\codeql.exe" database index-files ^ --language=iac ^ --working-dir=. ^ "%CODEQL_EXTRACTOR_IAC_WIP_DATABASE%" +if %ERRORLEVEL% neq 0 exit /b %ERRORLEVEL% + +"%CODEQL_DIST%\codeql.exe" database index-files ^ + --prune=**/*.testproj ^ + --include-extension=.yml ^ + --include-extension=.yaml ^ + --include-extension=.json ^ + --size-limit=5m ^ + --language=yaml ^ + --working-dir=. ^ + "%CODEQL_EXTRACTOR_IAC_WIP_DATABASE%" exit /b %ERRORLEVEL% From dd2875d8298dbaabdb852541feb5cdba3f7fc300 Mon Sep 17 00:00:00 2001 From: Dilan Bhalla Date: Wed, 27 May 2026 10:26:07 -0700 Subject: [PATCH 3/7] build.yml: split Download Extractor into Linux and Windows variants The previous 'Download Extracter' step used bash syntax (set -e, chmod, backslash continuations) but ran on all OSes including windows-latest, which executed it with pwsh and produced a ParserError. Also set fail-fast: false on the matrix so a failure on one OS does not cancel running tests on the other. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build.yml | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ff0d2f6..e71d20e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,6 +14,7 @@ jobs: tests: runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: os: [ubuntu-latest, windows-latest] test-folders: ["library-tests", "queries-tests"] @@ -33,8 +34,8 @@ jobs: - 'rust-toolchain.toml' - 'Cargo.*' - - name: "Download Extracter" - if: steps.extractor-changes.outputs.src == 'false' + - name: "Download Extractor (Linux/macOS)" + if: steps.extractor-changes.outputs.src == 'false' && matrix.os != 'windows-latest' env: GH_TOKEN: ${{ github.token }} run: | @@ -51,6 +52,26 @@ jobs: chmod +x extractor-pack/tools/*.sh chmod +x extractor-pack/tools/**/* + - name: "Download Extractor (Windows)" + if: steps.extractor-changes.outputs.src == 'false' && matrix.os == 'windows-latest' + env: + GH_TOKEN: ${{ github.token }} + shell: pwsh + run: | + gh release list -L 1 -R "advanced-security/codeql-extractor-iac" + + $ghArgs = @( + 'release', 'download', + '-R', 'advanced-security/codeql-extractor-iac', + '--clobber', + '--pattern', 'extractor-iac.tar.gz' + ) + & gh @ghArgs + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + tar -zxf extractor-iac.tar.gz + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + - uses: dtolnay/rust-toolchain@4305c38b25d97ef35a8ad1f985ccf2d2242004f2 # stable if: steps.extractor-changes.outputs.src == 'true' From f9427ffb6d6baa1f7a0063016192d8680da3922e Mon Sep 17 00:00:00 2001 From: Dilan Bhalla Date: Wed, 27 May 2026 10:28:45 -0700 Subject: [PATCH 4/7] scripts: detect gh-codeql extension via LASTEXITCODE, not try/catch PowerShell try/catch does not catch non-zero exit codes from native commands (only PowerShell exceptions). The previous logic in create-extractor-pack.ps1 and run-tests.ps1 assumed gh codeql would throw when the extension wasn't installed; it doesn't, so the scripts would proceed to invoke gh codeql without ever installing the extension, causing failures later (e.g. codeql pack install). Switch to checking LASTEXITCODE explicitly, which works for both PowerShell 5.1 and 7.x. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- scripts/create-extractor-pack.ps1 | 9 +++------ scripts/run-tests.ps1 | 13 +++++++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/scripts/create-extractor-pack.ps1 b/scripts/create-extractor-pack.ps1 index a657c09..86d89d1 100644 --- a/scripts/create-extractor-pack.ps1 +++ b/scripts/create-extractor-pack.ps1 @@ -8,19 +8,16 @@ if (Get-Command "codeql" -ErrorAction SilentlyContinue) { $CODEQL_BINARY = "codeql" } elseif (Get-Command "gh" -ErrorAction SilentlyContinue) { - try { - gh codeql version 2>&1 | Out-Null - $CODEQL_BINARY = "gh codeql" - } - catch { + gh codeql version 2>&1 | Out-Null + if ($LASTEXITCODE -ne 0) { Write-Host "Installing gh-codeql extension..." gh extension install github/gh-codeql if ($LASTEXITCODE -ne 0) { Write-Error "Failed to install gh-codeql extension (exit code $LASTEXITCODE)" exit $LASTEXITCODE } - $CODEQL_BINARY = "gh codeql" } + $CODEQL_BINARY = "gh codeql" } else { Write-Error "Neither 'codeql' nor 'gh' command found" diff --git a/scripts/run-tests.ps1 b/scripts/run-tests.ps1 index 038638e..2707dbf 100644 --- a/scripts/run-tests.ps1 +++ b/scripts/run-tests.ps1 @@ -9,15 +9,16 @@ if (Get-Command "codeql" -ErrorAction SilentlyContinue) { $CODEQL_BINARY = "codeql" } elseif (Get-Command "gh" -ErrorAction SilentlyContinue) { - try { - gh codeql version 2>&1 | Out-Null - $CODEQL_BINARY = "gh codeql" - } - catch { + gh codeql version 2>&1 | Out-Null + if ($LASTEXITCODE -ne 0) { Write-Host "Installing gh-codeql extension..." gh extension install github/gh-codeql - $CODEQL_BINARY = "gh codeql" + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to install gh-codeql extension (exit code $LASTEXITCODE)" + exit $LASTEXITCODE + } } + $CODEQL_BINARY = "gh codeql" } else { Write-Error "Neither 'codeql' nor 'gh' command found" From 067f76658bf09368e62f6bd4864e0a4ab0ab04cf Mon Sep 17 00:00:00 2001 From: Dilan Bhalla Date: Wed, 27 May 2026 10:41:26 -0700 Subject: [PATCH 5/7] Rebuild extractor on tools/scripts changes; remove empty .tf markers from CFN/ECS tests Two fixes for Windows CI: 1. build.yml paths-filter now triggers rebuild for tools/** and scripts/create-extractor-pack.* changes. Previously these changes caused CI to download the prebuilt extractor release containing the OLD qltest.cmd (--language=hcl), so the fixed qltest.cmd never reached test runs. 2. Removed empty .tf marker files from CloudFormation/ECS query tests. On Windows, mixing .tf and .yml in the same dir triggers a containerparent INVALID_KEY conflict (C:/ vs C: drive root normalization) between the iac and yaml extractors. The markers are 0-byte and contribute nothing to extraction. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build.yml | 4 ++++ .../CloudFormation/ECS/ContainerInsights/something.tf | 0 .../CloudFormation/ECS/LogConfiguration/something.tf | 0 .../queries-tests/CloudFormation/ECS/NetworkMode/something.tf | 0 ql/test/queries-tests/CloudFormation/ECS/NonPriv/something.tf | 0 ql/test/queries-tests/CloudFormation/ECS/PidMode/something.tf | 0 .../queries-tests/CloudFormation/ECS/PublicIP/something.tf | 0 .../CloudFormation/ECS/PublicIPTaskSet/something.tf | 0 .../CloudFormation/ECS/ReadOnlyFileSystem/something.tf | 0 ql/test/queries-tests/CloudFormation/ECS/Secrets/something.tf | 0 10 files changed, 4 insertions(+) delete mode 100644 ql/test/queries-tests/CloudFormation/ECS/ContainerInsights/something.tf delete mode 100644 ql/test/queries-tests/CloudFormation/ECS/LogConfiguration/something.tf delete mode 100644 ql/test/queries-tests/CloudFormation/ECS/NetworkMode/something.tf delete mode 100644 ql/test/queries-tests/CloudFormation/ECS/NonPriv/something.tf delete mode 100644 ql/test/queries-tests/CloudFormation/ECS/PidMode/something.tf delete mode 100644 ql/test/queries-tests/CloudFormation/ECS/PublicIP/something.tf delete mode 100644 ql/test/queries-tests/CloudFormation/ECS/PublicIPTaskSet/something.tf delete mode 100644 ql/test/queries-tests/CloudFormation/ECS/ReadOnlyFileSystem/something.tf delete mode 100644 ql/test/queries-tests/CloudFormation/ECS/Secrets/something.tf diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e71d20e..1ff2681 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,8 +31,10 @@ jobs: filters: | src: - 'extractor/**' + - 'tools/**' - 'rust-toolchain.toml' - 'Cargo.*' + - 'scripts/create-extractor-pack.*' - name: "Download Extractor (Linux/macOS)" if: steps.extractor-changes.outputs.src == 'false' && matrix.os != 'windows-latest' @@ -144,8 +146,10 @@ jobs: # filters: | # src: # - 'extractor/**' + # - 'tools/**' # - 'rust-toolchain.toml' # - 'Cargo.*' + # - 'scripts/create-extractor-pack.*' # - name: "Download Extracter" # if: steps.extractor-changes.outputs.src == 'false' diff --git a/ql/test/queries-tests/CloudFormation/ECS/ContainerInsights/something.tf b/ql/test/queries-tests/CloudFormation/ECS/ContainerInsights/something.tf deleted file mode 100644 index e69de29..0000000 diff --git a/ql/test/queries-tests/CloudFormation/ECS/LogConfiguration/something.tf b/ql/test/queries-tests/CloudFormation/ECS/LogConfiguration/something.tf deleted file mode 100644 index e69de29..0000000 diff --git a/ql/test/queries-tests/CloudFormation/ECS/NetworkMode/something.tf b/ql/test/queries-tests/CloudFormation/ECS/NetworkMode/something.tf deleted file mode 100644 index e69de29..0000000 diff --git a/ql/test/queries-tests/CloudFormation/ECS/NonPriv/something.tf b/ql/test/queries-tests/CloudFormation/ECS/NonPriv/something.tf deleted file mode 100644 index e69de29..0000000 diff --git a/ql/test/queries-tests/CloudFormation/ECS/PidMode/something.tf b/ql/test/queries-tests/CloudFormation/ECS/PidMode/something.tf deleted file mode 100644 index e69de29..0000000 diff --git a/ql/test/queries-tests/CloudFormation/ECS/PublicIP/something.tf b/ql/test/queries-tests/CloudFormation/ECS/PublicIP/something.tf deleted file mode 100644 index e69de29..0000000 diff --git a/ql/test/queries-tests/CloudFormation/ECS/PublicIPTaskSet/something.tf b/ql/test/queries-tests/CloudFormation/ECS/PublicIPTaskSet/something.tf deleted file mode 100644 index e69de29..0000000 diff --git a/ql/test/queries-tests/CloudFormation/ECS/ReadOnlyFileSystem/something.tf b/ql/test/queries-tests/CloudFormation/ECS/ReadOnlyFileSystem/something.tf deleted file mode 100644 index e69de29..0000000 diff --git a/ql/test/queries-tests/CloudFormation/ECS/Secrets/something.tf b/ql/test/queries-tests/CloudFormation/ECS/Secrets/something.tf deleted file mode 100644 index e69de29..0000000 From 6327bbaa05b23742b3f91ecd0310f45c5e9903a7 Mon Sep 17 00:00:00 2001 From: Dilan Bhalla Date: Wed, 27 May 2026 10:48:26 -0700 Subject: [PATCH 6/7] ci(windows): enable git core.longpaths before cargo build The github/codeql repo (pulled in as a Cargo git dep) contains paths >260 chars. Without core.longpaths=true, git on Windows fails with 'path too long' (class=Filesystem 30) during 'cargo fetch'. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build.yml | 2 ++ .github/workflows/publish.yml | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1ff2681..f203f18 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -96,6 +96,8 @@ jobs: GH_TOKEN: ${{ github.token }} shell: pwsh run: | + git config --global core.longpaths true + gh extension install github/gh-codeql gh codeql set-version latest diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c847eab..60a9d18 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -70,7 +70,9 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} shell: pwsh - run: .\scripts\create-extractor-pack.ps1 + run: | + git config --global core.longpaths true + .\scripts\create-extractor-pack.ps1 - name: "Upload bundle artifact" uses: actions/upload-artifact@v6 From cd36a94b382f331d5fbec3b1fe5f3425f0539890 Mon Sep 17 00:00:00 2001 From: Dilan Bhalla Date: Wed, 27 May 2026 16:57:27 -0700 Subject: [PATCH 7/7] ci: add fixed-name summary gates for required status checks Branch protection requires 'tests (library-tests)' and 'tests (queries-tests)', but the matrix emits OS-qualified names like 'tests (ubuntu-latest, library-tests)'. Two new aggregate jobs needs: tests and re-emit the bare names so required checks are satisfied without changing branch protection. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f203f18..726205d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -120,6 +120,37 @@ jobs: run: | .\scripts\run-tests.ps1 "ql/test/${{ matrix.test-folders }}" + # Fixed-name summary gates so branch protection's required checks + # ("tests (library-tests)" / "tests (queries-tests)") are satisfied + # regardless of the OS-qualified matrix check names emitted above. + library-tests-summary: + name: tests (library-tests) + runs-on: ubuntu-latest + needs: tests + if: always() + steps: + - name: Aggregate matrix result + run: | + if [ "${{ needs.tests.result }}" != "success" ]; then + echo "Matrix 'tests' did not succeed (result=${{ needs.tests.result }})" + exit 1 + fi + echo "All matrix 'tests' jobs succeeded." + + queries-tests-summary: + name: tests (queries-tests) + runs-on: ubuntu-latest + needs: tests + if: always() + steps: + - name: Aggregate matrix result + run: | + if [ "${{ needs.tests.result }}" != "success" ]; then + echo "Matrix 'tests' did not succeed (result=${{ needs.tests.result }})" + exit 1 + fi + echo "All matrix 'tests' jobs succeeded." + # scanning: # runs-on: ubuntu-latest # needs: [tests]