diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1986a40..726205d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,9 +12,11 @@ permissions: jobs: tests: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: + os: [ubuntu-latest, windows-latest] test-folders: ["library-tests", "queries-tests"] steps: - name: "Checkout" @@ -29,11 +31,13 @@ jobs: filters: | src: - 'extractor/**' + - 'tools/**' - 'rust-toolchain.toml' - 'Cargo.*' + - 'scripts/create-extractor-pack.*' - - name: "Download Extracter" - if: steps.extractor-changes.outputs.src == 'false' + - name: "Download Extractor (Linux/macOS)" + if: steps.extractor-changes.outputs.src == 'false' && matrix.os != 'windows-latest' env: GH_TOKEN: ${{ github.token }} run: | @@ -50,28 +54,103 @@ jobs: chmod +x extractor-pack/tools/*.sh chmod +x extractor-pack/tools/**/* + - name: "Download Extractor (Windows)" + if: steps.extractor-changes.outputs.src == 'false' && matrix.os == 'windows-latest' + env: + GH_TOKEN: ${{ github.token }} + shell: pwsh + run: | + gh release list -L 1 -R "advanced-security/codeql-extractor-iac" + + $ghArgs = @( + 'release', 'download', + '-R', 'advanced-security/codeql-extractor-iac', + '--clobber', + '--pattern', 'extractor-iac.tar.gz' + ) + & gh @ghArgs + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + tar -zxf extractor-iac.tar.gz + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + - uses: dtolnay/rust-toolchain@4305c38b25d97ef35a8ad1f985ccf2d2242004f2 # stable if: steps.extractor-changes.outputs.src == 'true' - - name: "Build Extractor" - if: steps.extractor-changes.outputs.src == 'true' + - name: "Build Extractor (Linux/macOS)" + if: steps.extractor-changes.outputs.src == 'true' && matrix.os != 'windows-latest' env: GH_TOKEN: ${{ github.token }} run: | set -e - gh extensions install github/gh-codeql + gh extension install github/gh-codeql gh codeql set-version latest ./scripts/create-extractor-pack.sh gh codeql resolve languages --format=json --search-path ./extractor-pack - - name: "Run Tests" + - name: "Build Extractor (Windows)" + if: steps.extractor-changes.outputs.src == 'true' && matrix.os == 'windows-latest' + env: + GH_TOKEN: ${{ github.token }} + shell: pwsh + run: | + git config --global core.longpaths true + + gh extension install github/gh-codeql + gh codeql set-version latest + + .\scripts\create-extractor-pack.ps1 + + gh codeql resolve languages --format=json --search-path .\extractor-pack + + - name: "Run Tests (Linux/macOS)" + if: matrix.os != 'windows-latest' env: GH_TOKEN: ${{ github.token }} run: | ./scripts/run-tests.sh "ql/test/${{ matrix.test-folders }}" + - name: "Run Tests (Windows)" + if: matrix.os == 'windows-latest' + env: + GH_TOKEN: ${{ github.token }} + shell: pwsh + run: | + .\scripts\run-tests.ps1 "ql/test/${{ matrix.test-folders }}" + + # Fixed-name summary gates so branch protection's required checks + # ("tests (library-tests)" / "tests (queries-tests)") are satisfied + # regardless of the OS-qualified matrix check names emitted above. + library-tests-summary: + name: tests (library-tests) + runs-on: ubuntu-latest + needs: tests + if: always() + steps: + - name: Aggregate matrix result + run: | + if [ "${{ needs.tests.result }}" != "success" ]; then + echo "Matrix 'tests' did not succeed (result=${{ needs.tests.result }})" + exit 1 + fi + echo "All matrix 'tests' jobs succeeded." + + queries-tests-summary: + name: tests (queries-tests) + runs-on: ubuntu-latest + needs: tests + if: always() + steps: + - name: Aggregate matrix result + run: | + if [ "${{ needs.tests.result }}" != "success" ]; then + echo "Matrix 'tests' did not succeed (result=${{ needs.tests.result }})" + exit 1 + fi + echo "All matrix 'tests' jobs succeeded." + # scanning: # runs-on: ubuntu-latest # needs: [tests] @@ -100,8 +179,10 @@ jobs: # filters: | # src: # - 'extractor/**' + # - 'tools/**' # - 'rust-toolchain.toml' # - 'Cargo.*' + # - 'scripts/create-extractor-pack.*' # - name: "Download Extracter" # if: steps.extractor-changes.outputs.src == 'false' @@ -127,7 +208,7 @@ jobs: # GH_TOKEN: ${{ github.token }} # run: | # set -e - # gh extensions install github/gh-codeql + # gh extension install github/gh-codeql # gh codeql set-version latest # ./scripts/create-extractor-pack.sh @@ -140,7 +221,7 @@ jobs: # PROJECT_REPO: ${{ matrix.project }} # run: | # set -e - # gh extensions install github/gh-codeql + # gh extension install github/gh-codeql # gh codeql set-version latest # gh codeql database create --language=iac --source-root=./project --search-path ./extractor-pack iac-db diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 04522b5..60a9d18 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -47,8 +47,7 @@ jobs: strategy: fail-fast: false matrix: - # TODO: Add windows-latest - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest, windows-latest] if: ${{ needs.release-check.outputs.release == 'true' }} steps: @@ -59,14 +58,22 @@ jobs: - name: "Set up Rust" uses: dtolnay/rust-toolchain@0c3131df9e5407c0c36352032d04af846dbe0fb7 # nightly - if: ${{ matrix.os != 'windows-latest' }} - - name: "Build Extractor" + - name: "Build Extractor (Linux/macOS)" if: ${{ matrix.os != 'windows-latest' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: ./scripts/create-extractor-pack.sh + - name: "Build Extractor (Windows)" + if: ${{ matrix.os == 'windows-latest' }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: pwsh + run: | + git config --global core.longpaths true + .\scripts\create-extractor-pack.ps1 + - name: "Upload bundle artifact" uses: actions/upload-artifact@v6 with: diff --git a/docs/workflows.md b/docs/workflows.md index 87a87db..b8c5a74 100644 --- a/docs/workflows.md +++ b/docs/workflows.md @@ -113,3 +113,20 @@ codeql database analyze \ "$CODEQL_DATABASE" \ "advanced-security/iac-queries" ``` + +#### Install extractor on Windows + +PowerShell equivalents of the scripts above are provided for Windows. From a checkout of this repository: + +```powershell +# Install the extractor into $env:USERPROFILE\.codeql\extractors (default) +.\scripts\install-extractor.ps1 + +# Or build the extractor pack locally +.\scripts\create-extractor-pack.ps1 + +# Run the test suite +.\scripts\run-tests.ps1 ql\test\library-tests +``` + +Both `pwsh` (PowerShell 7+) and Windows PowerShell 5.1 are supported. The scripts require either `codeql` or `gh` on the `PATH`; if only `gh` is available, the `gh-codeql` extension will be installed automatically. diff --git a/ql/test/queries-tests/CloudFormation/ECS/ContainerInsights/something.tf b/ql/test/queries-tests/CloudFormation/ECS/ContainerInsights/something.tf deleted file mode 100644 index e69de29..0000000 diff --git a/ql/test/queries-tests/CloudFormation/ECS/LogConfiguration/something.tf b/ql/test/queries-tests/CloudFormation/ECS/LogConfiguration/something.tf deleted file mode 100644 index e69de29..0000000 diff --git a/ql/test/queries-tests/CloudFormation/ECS/NetworkMode/something.tf b/ql/test/queries-tests/CloudFormation/ECS/NetworkMode/something.tf deleted file mode 100644 index e69de29..0000000 diff --git a/ql/test/queries-tests/CloudFormation/ECS/NonPriv/something.tf b/ql/test/queries-tests/CloudFormation/ECS/NonPriv/something.tf deleted file mode 100644 index e69de29..0000000 diff --git a/ql/test/queries-tests/CloudFormation/ECS/PidMode/something.tf b/ql/test/queries-tests/CloudFormation/ECS/PidMode/something.tf deleted file mode 100644 index e69de29..0000000 diff --git a/ql/test/queries-tests/CloudFormation/ECS/PublicIP/something.tf b/ql/test/queries-tests/CloudFormation/ECS/PublicIP/something.tf deleted file mode 100644 index e69de29..0000000 diff --git a/ql/test/queries-tests/CloudFormation/ECS/PublicIPTaskSet/something.tf b/ql/test/queries-tests/CloudFormation/ECS/PublicIPTaskSet/something.tf deleted file mode 100644 index e69de29..0000000 diff --git a/ql/test/queries-tests/CloudFormation/ECS/ReadOnlyFileSystem/something.tf b/ql/test/queries-tests/CloudFormation/ECS/ReadOnlyFileSystem/something.tf deleted file mode 100644 index e69de29..0000000 diff --git a/ql/test/queries-tests/CloudFormation/ECS/Secrets/something.tf b/ql/test/queries-tests/CloudFormation/ECS/Secrets/something.tf deleted file mode 100644 index e69de29..0000000 diff --git a/scripts/create-extractor-pack.ps1 b/scripts/create-extractor-pack.ps1 index 65b4dfc..86d89d1 100644 --- a/scripts/create-extractor-pack.ps1 +++ b/scripts/create-extractor-pack.ps1 @@ -1,14 +1,75 @@ -cargo build --release +$ErrorActionPreference = "Stop" + +# Set platform +$platform = "win64" -cargo run --release -p ql-generator -- --dbscheme ql/src/ql.dbscheme --library ql/src/codeql_ql/ast/internal/TreeSitter.qll -codeql query format -i ql\src\codeql_ql\ast\internal\TreeSitter.qll +# Check for CodeQL binary +if (Get-Command "codeql" -ErrorAction SilentlyContinue) { + $CODEQL_BINARY = "codeql" +} +elseif (Get-Command "gh" -ErrorAction SilentlyContinue) { + gh codeql version 2>&1 | Out-Null + if ($LASTEXITCODE -ne 0) { + Write-Host "Installing gh-codeql extension..." + gh extension install github/gh-codeql + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to install gh-codeql extension (exit code $LASTEXITCODE)" + exit $LASTEXITCODE + } + } + $CODEQL_BINARY = "gh codeql" +} +else { + Write-Error "Neither 'codeql' nor 'gh' command found" + exit 1 +} +Write-Host "Creating extractor pack..." if (Test-Path -Path extractor-pack) { - rm -Recurse -Force extractor-pack -} -mkdir extractor-pack | Out-Null -cp codeql-extractor.yml, ql\src\ql.dbscheme, ql\src\ql.dbscheme.stats extractor-pack -cp -Recurse tools extractor-pack -mkdir extractor-pack\tools\win64 | Out-Null -cp target\release\ql-extractor.exe extractor-pack\tools\win64\extractor.exe -cp target\release\ql-autobuilder.exe extractor-pack\tools\win64\autobuilder.exe + Remove-Item -Recurse -Force extractor-pack +} +if (Test-Path -Path target) { + Remove-Item -Recurse -Force target +} + +Write-Host "Update submodules..." +git submodule update --init --recursive +if ($LASTEXITCODE -ne 0) { + Write-Error "git submodule update failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE +} + +Write-Host "Building extractor..." +cargo build --release +if ($LASTEXITCODE -ne 0) { + Write-Error "cargo build failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE +} + +Write-Host "Generating TreeSitter library..." +cargo run --release --bin codeql-extractor-iac -- generate --dbscheme ql/lib/iac.dbscheme --library ql/lib/codeql/iac/ast/internal/TreeSitter.qll +if ($LASTEXITCODE -ne 0) { + Write-Error "TreeSitter generation failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE +} + +Write-Host "Formatting generated library..." +if ($CODEQL_BINARY -eq "gh codeql") { + gh codeql query format -i ql/lib/codeql/iac/ast/internal/TreeSitter.qll +} +else { + codeql query format -i ql/lib/codeql/iac/ast/internal/TreeSitter.qll +} +if ($LASTEXITCODE -ne 0) { + Write-Error "codeql query format failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE +} + +New-Item -ItemType Directory -Path extractor-pack | Out-Null +Copy-Item -Path codeql-extractor.yml, ql/lib/iac.dbscheme, ql/lib/iac.dbscheme.stats -Destination extractor-pack/ +Copy-Item -Recurse -Path downgrades, tools -Destination extractor-pack/ + +New-Item -ItemType Directory -Path "extractor-pack/tools/$platform" -Force | Out-Null +Copy-Item -Path "target/release/codeql-extractor-iac.exe" -Destination "extractor-pack/tools/$platform/extractor.exe" + +Write-Host "Extractor pack created successfully!" diff --git a/scripts/install-extractor.ps1 b/scripts/install-extractor.ps1 new file mode 100644 index 0000000..d36f9fe --- /dev/null +++ b/scripts/install-extractor.ps1 @@ -0,0 +1,37 @@ +param( + [string]$ExtractorName = "iac", + [string]$ExtractorLocations = "$env:USERPROFILE\.codeql\extractors" +) + +$ErrorActionPreference = "Stop" + +Write-Host "Creating extractor directory..." +if (!(Test-Path $ExtractorLocations)) { + New-Item -ItemType Directory -Path $ExtractorLocations -Force | Out-Null +} + +Write-Host "Checking latest release..." +gh release list -L 1 -R "advanced-security/codeql-extractor-$ExtractorName" + +Write-Host "Downloading extractor pack..." +$ghArgs = @( + 'release', 'download', + '-R', "advanced-security/codeql-extractor-$ExtractorName", + '-D', "$ExtractorLocations", + '--clobber', + '--pattern', 'extractor-*.tar.gz' +) +& gh @ghArgs +if ($LASTEXITCODE -ne 0) { + Write-Error "gh release download failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE +} + +Write-Host "Extracting extractor pack..." +tar -zxf "$ExtractorLocations/extractor-$ExtractorName.tar.gz" --directory "$ExtractorLocations" +if ($LASTEXITCODE -ne 0) { + Write-Error "tar extraction failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE +} + +Write-Host "Installation complete! Extractor installed to: $ExtractorLocations" \ No newline at end of file diff --git a/scripts/run-tests.ps1 b/scripts/run-tests.ps1 new file mode 100644 index 0000000..2707dbf --- /dev/null +++ b/scripts/run-tests.ps1 @@ -0,0 +1,62 @@ +param( + [string]$TestsDir = "ql/test" +) + +$ErrorActionPreference = "Stop" + +# Check for CodeQL binary +if (Get-Command "codeql" -ErrorAction SilentlyContinue) { + $CODEQL_BINARY = "codeql" +} +elseif (Get-Command "gh" -ErrorAction SilentlyContinue) { + gh codeql version 2>&1 | Out-Null + if ($LASTEXITCODE -ne 0) { + Write-Host "Installing gh-codeql extension..." + gh extension install github/gh-codeql + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to install gh-codeql extension (exit code $LASTEXITCODE)" + exit $LASTEXITCODE + } + } + $CODEQL_BINARY = "gh codeql" +} +else { + Write-Error "Neither 'codeql' nor 'gh' command found" + exit 1 +} + +Write-Host "Installing ql/test pack dependencies..." +if ($CODEQL_BINARY -eq "gh codeql") { + gh codeql pack install ql/test +} +else { + codeql pack install ql/test +} +if ($LASTEXITCODE -ne 0) { + Write-Error "codeql pack install failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE +} + +Write-Host "Running tests in $TestsDir" + +if ($CODEQL_BINARY -eq "gh codeql") { + gh codeql test run ` + -j 0 ` + --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition ` + --search-path ./extractor-pack ` + "$TestsDir" +} +else { + codeql test run ` + -j 0 ` + --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition ` + --search-path ./extractor-pack ` + "$TestsDir" +} + +if ($LASTEXITCODE -ne 0) { + Write-Error "Tests failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE +} + +Write-Host "All tests passed!" \ No newline at end of file diff --git a/tools/index-files.cmd b/tools/index-files.cmd index ab05465..56a743b 100755 --- a/tools/index-files.cmd +++ b/tools/index-files.cmd @@ -1,6 +1,7 @@ @echo off type NUL && "%CODEQL_EXTRACTOR_IAC_ROOT%\tools\win64\extractor.exe" ^ + extract ^ --file-list "%1" ^ --source-archive-dir "%CODEQL_EXTRACTOR_IAC_SOURCE_ARCHIVE_DIR%" ^ --output-dir "%CODEQL_EXTRACTOR_IAC_TRAP_DIR%" diff --git a/tools/qltest.cmd b/tools/qltest.cmd index c960b58..88c344b 100755 --- a/tools/qltest.cmd +++ b/tools/qltest.cmd @@ -1,11 +1,24 @@ @echo off -type NUL && "%CODEQL_DIST%\codeql.exe" database index-files ^ +"%CODEQL_DIST%\codeql.exe" database index-files ^ --prune=**/*.testproj ^ --include-extension=.hcl ^ --include-extension=.tf ^ + --include-extension=.tfvars ^ + --include-extension=.bicep ^ --size-limit=5m ^ - --language=hcl ^ + --language=iac ^ + --working-dir=. ^ + "%CODEQL_EXTRACTOR_IAC_WIP_DATABASE%" +if %ERRORLEVEL% neq 0 exit /b %ERRORLEVEL% + +"%CODEQL_DIST%\codeql.exe" database index-files ^ + --prune=**/*.testproj ^ + --include-extension=.yml ^ + --include-extension=.yaml ^ + --include-extension=.json ^ + --size-limit=5m ^ + --language=yaml ^ --working-dir=. ^ "%CODEQL_EXTRACTOR_IAC_WIP_DATABASE%"