From 5de4d9089b010f16ce69a7e82e083e9346c0d33b Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Thu, 5 Feb 2026 12:06:15 -0800 Subject: [PATCH 1/4] Abridged tests on PRs --- .github/workflows/run-tests.yml | 514 ++++++++----------------- Makefile | 9 +- tests/namespace_delete_test.go | 31 -- tests/testcore/functional_test_base.go | 36 -- tools/testrunner/testrunner.go | 103 +++++ 5 files changed, 260 insertions(+), 433 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index ce2e5b3abf..df9167bc56 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -24,24 +24,129 @@ env: DOCKER_COMPOSE_FILE: ./develop/github/docker-compose.yml TEMPORAL_VERSION_CHECK_DISABLED: 1 MAX_TEST_ATTEMPTS: 3 + TOTAL_SHARDS: 5 jobs: - set-up-functional-test: - name: Set up functional test + test-setup: + name: Test setup runs-on: ubuntu-latest outputs: - shard_indices: ${{ steps.generate_output.outputs.shard_indices }} - total_shards: ${{ steps.generate_output.outputs.shards }} - github_timeout: ${{ steps.generate_output.outputs.github_timeout }} - test_timeout: ${{ steps.generate_output.outputs.test_timeout }} - runs_on: ${{ steps.generate_output.outputs.runs_on }} - runner_x64: ${{ steps.generate_output.outputs.runner_x64 }} - runner_arm: ${{ steps.generate_output.outputs.runner_arm }} + job_matrix: ${{ steps.compute_matrix.outputs.job_matrix }} + full_test_reason: ${{ steps.compute_matrix.outputs.full_test_reason }} + runner_x64: ${{ steps.compute_matrix.outputs.runner_x64 }} + runner_arm: ${{ steps.compute_matrix.outputs.runner_arm }} steps: - - id: generate_output + - name: Checkout Code + uses: actions/checkout@v6 + with: + token: ${{ secrets.GITHUB_TOKEN }} + ref: ${{ env.COMMIT }} + fetch-depth: 0 + + - name: Fetch base branch + if: ${{ github.event_name == 'pull_request' }} + run: git fetch origin ${{ github.event.pull_request.base.ref }}:${{ github.event.pull_request.base.ref }} + + - name: Compute merge base + if: ${{ github.event_name == 'pull_request' }} run: | - shards=3 - timeout=35 # update this to TEST_TIMEOUT if you update the Makefile + MERGE_BASE="$(git merge-base "${{ env.COMMIT }}" "${{ github.event.pull_request.base.ref }}")" + echo "MERGE_BASE=${MERGE_BASE}" >> "$GITHUB_ENV" + + # Primary DBs always get full tests (5 shards), extended DBs get smoke tests (1 shard) in abridged PRs. + - name: Compute database matrix + id: compute_matrix + run: | + # DB configurations: name -> {persistence_type, persistence_driver, containers} + declare -A DB_CONFIG + DB_CONFIG[cass_es]='{"persistence_type":"nosql","persistence_driver":"cassandra","containers":["cassandra","elasticsearch"]}' + DB_CONFIG[cass_es8]='{"persistence_type":"nosql","persistence_driver":"cassandra","containers":["cassandra","elasticsearch8"]}' + DB_CONFIG[cass_os2]='{"persistence_type":"nosql","persistence_driver":"cassandra","containers":["cassandra","opensearch2"]}' + DB_CONFIG[cass_os3]='{"persistence_type":"nosql","persistence_driver":"cassandra","containers":["cassandra","opensearch3"]}' + DB_CONFIG[sqlite]='{"persistence_type":"sql","persistence_driver":"sqlite","containers":[],"arch":"arm"}' + DB_CONFIG[mysql8]='{"persistence_type":"sql","persistence_driver":"mysql8","containers":["mysql"]}' + DB_CONFIG[postgres12]='{"persistence_type":"sql","persistence_driver":"postgres12","containers":["postgresql"]}' + DB_CONFIG[postgres12_pgx]='{"persistence_type":"sql","persistence_driver":"postgres12_pgx","containers":["postgresql"]}' + + # Testing one NoSQL and one SQL database + PRIMARY_DBS=("cass_os3" "sqlite") + EXTENDED_DBS=("cass_es" "cass_es8" "cass_os2" "mysql8" "postgres12" "postgres12_pgx") + + # Smoke tests are a small subset of the full test suite + SMOKE_TESTS=( + "TestWorkflowTestSuite" + "TestSignalWorkflowTestSuite" + "TestActivityTestSuite/TestActivityHeartBeatWorkflow_Success" + "TestNDCFuncTestSuite/TestSingleBranch" + "TestFuncClustersTestSuite/TestNamespaceFailover" + ) + SMOKE_TEST_ARGS="'-run=$(IFS='|'; echo "${SMOKE_TESTS[*]}")'" + + ## Determine if this is a full or abridged (smoke) test run + + FULL_TEST_REASON="" + + # Push events (main, release branches) run all tests on all DBs + if [[ "${{ github.event_name }}" == "push" ]]; then + FULL_TEST_REASON="Running full tests on all DBs (push event)." + # Check for test-all-dbs label + elif echo '${{ toJson(github.event.pull_request.labels.*.name) }}' | jq -e 'any(. == "test-all-dbs")' > /dev/null 2>&1; then + FULL_TEST_REASON="Running full tests on all DBs (test-all-dbs label)." + # Check for persistence code changes + elif [[ -n "${MERGE_BASE:-}" ]] && git diff --name-only "$MERGE_BASE" "$COMMIT" | grep -qE "^(common/persistence/|schema/)"; then + FULL_TEST_REASON="Running full tests on all DBs (persistence code changes)." + fi + + ## Build job matrix + + MATRIX="[]" + + add_sharded_jobs() { + local db=$1 + local config=${DB_CONFIG[$db]} + for shard in $(seq 1 "$TOTAL_SHARDS"); do + MATRIX=$(jq -c \ + ". + [\$config + { + name: \"$db\", + shard_index: $shard, + total_shards: $TOTAL_SHARDS, + test_timeout: \"20m\", + github_timeout: 30 + }]" \ + --argjson config "$config" \ + <<< "$MATRIX") + done + } + + add_smoke_job() { + local db=$1 + local config=${DB_CONFIG[$db]} + MATRIX=$(jq -c \ + ". + [\$config + { + name: \"$db\", + is_smoke: true, + test_args: \$test_args, + test_timeout: \"5m\", + github_timeout: 10 + }]" \ + --argjson config "$config" \ + --arg test_args "$SMOKE_TEST_ARGS" \ + <<< "$MATRIX") + } + + # Schedule jobs for primary databases + for db in "${PRIMARY_DBS[@]}"; do + add_sharded_jobs "$db" + done + + # Schedule jobs for extended databases + for db in "${EXTENDED_DBS[@]}"; do + if [[ -n "$FULL_TEST_REASON" ]]; then + add_sharded_jobs "$db" + else + add_smoke_job "$db" + fi + done # Runner configuration: use 8-core runners for temporalio org, standard runners for forks if [[ "${{ github.repository_owner }}" == "temporalio" ]]; then @@ -51,25 +156,25 @@ jobs: runner_x64="ubuntu-latest" runner_arm="ubuntu-24.04-arm" fi - runs_on="[\"${runner_x64}\"]" { - echo "shard_indices=[ $(seq -s, 0 $((shards-1))) ]" - echo "shards=$shards" - echo "github_timeout=$((timeout+5))" - echo "test_timeout=${timeout}m" - echo "runs_on=$runs_on" + echo "job_matrix=$MATRIX" + echo "full_test_reason=$FULL_TEST_REASON" echo "runner_x64=$runner_x64" echo "runner_arm=$runner_arm" } >> "$GITHUB_OUTPUT" + echo "Generated $(jq length <<< "$MATRIX") jobs" + + - name: ${{ steps.compute_matrix.outputs.full_test_reason && 'ℹ️ Full tests' || 'ℹ️ Smoke tests' }} + run: echo "::notice::${{ steps.compute_matrix.outputs.full_test_reason || 'Running smoke tests on extended DBs. Add the test-all-dbs label to run all tests on all DBs.' }}" pre-build: name: Pre-build for cache (${{ matrix.arch }}) - needs: set-up-functional-test + needs: test-setup strategy: matrix: arch: [x64, arm] - runs-on: ${{ matrix.arch == 'arm' && needs.set-up-functional-test.outputs.runner_arm || needs.set-up-functional-test.outputs.runner_x64 }} + runs-on: ${{ matrix.arch == 'arm' && needs.test-setup.outputs.runner_arm || needs.test-setup.outputs.runner_x64 }} steps: - uses: actions/checkout@v6 with: @@ -105,8 +210,8 @@ jobs: misc-checks: name: Misc checks - needs: [pre-build, set-up-functional-test] - runs-on: ${{ needs.set-up-functional-test.outputs.runner_x64 }} + needs: [pre-build, test-setup] + runs-on: ${{ needs.test-setup.outputs.runner_x64 }} steps: - uses: actions/checkout@v6 with: @@ -144,8 +249,8 @@ jobs: unit-test: name: Unit test - needs: [pre-build, set-up-functional-test] - runs-on: ${{ needs.set-up-functional-test.outputs.runner_x64 }} + needs: [pre-build, test-setup] + runs-on: ${{ needs.test-setup.outputs.runner_x64 }} steps: - uses: actions/checkout@v6 with: @@ -217,8 +322,8 @@ jobs: integration-test: name: Integration test - needs: [pre-build, set-up-functional-test] - runs-on: ${{ needs.set-up-functional-test.outputs.runner_x64 }} + needs: [pre-build, test-setup] + runs-on: ${{ needs.test-setup.outputs.runner_x64 }} steps: - uses: actions/checkout@v6 with: @@ -303,69 +408,21 @@ jobs: run: | docker compose -f ${{ env.DOCKER_COMPOSE_FILE }} down -v - # Root job name includes matrix details so it is unique per shard. + # Root job name includes matrix details so it is unique per shard/smoke test. # This MUST stay in sync with the `job_name` passed to the job-id action below. functional-test: # Display name shown in the UI. The job-id lookup uses this exact value. - name: Functional test (${{ matrix.name }}, shard ${{ matrix.shard_index }}) - needs: [pre-build, set-up-functional-test] + name: Functional test (${{ matrix.name }}${{ matrix.is_smoke && ', smoke' || format(', shard {0}', matrix.shard_index) }}) + needs: [pre-build, test-setup] strategy: fail-fast: false matrix: - name: - - cass_es - - cass_es8 - - cass_os2 - - cass_os3 - - sqlite - - mysql8 - - postgres12 - - postgres12_pgx - shard_index: ${{ fromJson(needs.set-up-functional-test.outputs.shard_indices) }} - runs-on: ${{ fromJson(needs.set-up-functional-test.outputs.runs_on) }} - include: - - name: cass_es - persistence_type: nosql - persistence_driver: cassandra - containers: [cassandra, elasticsearch] - es_version: v7 - - name: cass_es8 - persistence_type: nosql - persistence_driver: cassandra - containers: [cassandra, elasticsearch8] - es_version: v8 - - name: cass_os2 - persistence_type: nosql - persistence_driver: cassandra - containers: [cassandra, opensearch2] - - name: cass_os3 - persistence_type: nosql - persistence_driver: cassandra - containers: [cassandra, opensearch3] - - name: sqlite - persistence_type: sql - persistence_driver: sqlite - containers: [] - arch: arm - - name: mysql8 - persistence_type: sql - persistence_driver: mysql8 - containers: [mysql] - - name: postgres12 - persistence_type: sql - persistence_driver: postgres12 - containers: [postgresql] - - name: postgres12_pgx - persistence_type: sql - persistence_driver: postgres12_pgx - containers: [postgresql] - runs-on: ${{ matrix.arch == 'arm' && needs.set-up-functional-test.outputs.runner_arm || needs.set-up-functional-test.outputs.runner_x64 }} + include: ${{ fromJson(needs.test-setup.outputs.job_matrix) }} + runs-on: ${{ matrix.arch == 'arm' && needs.test-setup.outputs.runner_arm || needs.test-setup.outputs.runner_x64 }} env: - TEST_TOTAL_SHARDS: ${{ needs.set-up-functional-test.outputs.total_shards }} - TEST_SHARD_INDEX: ${{ matrix.shard_index }} PERSISTENCE_TYPE: ${{ matrix.persistence_type }} PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }} - TEST_TIMEOUT: ${{ needs.set-up-functional-test.outputs.test_timeout }} + TEST_TIMEOUT: ${{ matrix.test_timeout }} steps: - uses: ScribeMD/docker-cache@0.3.7 with: @@ -405,12 +462,22 @@ jobs: id: get_job_id uses: ./.github/actions/get-job-id with: - job_name: Functional test (${{ matrix.name }}, shard ${{ matrix.shard_index }}) + job_name: Functional test (${{ matrix.name }}${{ matrix.is_smoke && ', smoke' || format(', shard {0}', matrix.shard_index) }}) run_id: ${{ github.run_id }} + - name: ${{ matrix.is_smoke && 'ℹ️ Smoke test' || 'ℹ️ Full test' }} + run: echo "::notice::${{ matrix.is_smoke && 'This is a smoke test. Add the test-all-dbs label to run all tests on all DBs.' || needs.test-setup.outputs.full_test_reason }}" + - name: Run functional test - timeout-minutes: ${{ fromJSON(needs.set-up-functional-test.outputs.github_timeout) }} # make sure this is larger than the test timeout in the Makefile + timeout-minutes: ${{ matrix.github_timeout }} run: ./develop/github/monitor_test.sh make functional-test-coverage + env: + # Full tests use TEMPORAL_TEST_TOTAL_SHARDS/TEMPORAL_TEST_SHARD_INDEX; + # smoke tests run specific suites via TEST_ARGS + TEMPORAL_TEST_TOTAL_SHARDS: ${{ matrix.total_shards || '' }} + TEMPORAL_TEST_SHARD_INDEX: ${{ matrix.shard_index || '' }} + TEMPORAL_TEST_SHARD_SALT: "-salt-26" + TEST_ARGS: ${{ matrix.test_args || '' }} - name: Print memory snapshot if: always() @@ -453,282 +520,7 @@ jobs: uses: actions/upload-artifact@v4.4.3 if: ${{ !cancelled() }} with: - name: junit-xml--${{github.run_id}}--${{ steps.get_job_id.outputs.job_id }}--${{github.run_attempt}}--${{matrix.name}}--${{matrix.shard_index}}--functional-test - path: ./.testoutput/junit.*.xml - include-hidden-files: true - retention-days: 28 - - # XDC matrix job. Include `${{ matrix.name }}` in the display name so each - # matrix variant has a unique name for the job-id lookup. - functional-test-xdc: - # Display name shown in the UI. The job-id lookup uses this exact value. - name: Functional test xdc (${{ matrix.name }}) - needs: [pre-build, set-up-functional-test] - strategy: - fail-fast: false - matrix: - name: - - cass_es - - cass_es8 - - cass_os2 - - cass_os3 - - mysql8 - - postgres12 - - postgres12_pgx - include: - - name: cass_es - persistence_type: nosql - persistence_driver: elasticsearch - parallel_flags: "" - containers: [cassandra, elasticsearch] - - name: cass_es8 - persistence_type: nosql - persistence_driver: elasticsearch - parallel_flags: "" - containers: [cassandra, elasticsearch8] - - name: cass_os2 - persistence_type: nosql - persistence_driver: cassandra - containers: [cassandra, opensearch2] - - name: cass_os3 - persistence_type: nosql - persistence_driver: cassandra - containers: [cassandra, opensearch3] - - name: mysql8 - persistence_type: sql - persistence_driver: mysql8 - parallel_flags: "" - containers: [mysql] - - name: postgres12 - persistence_type: sql - persistence_driver: postgres12 - parallel_flags: "-parallel=2" # reduce parallelism for postgres - containers: [postgresql] - - name: postgres12_pgx - persistence_type: sql - persistence_driver: postgres12_pgx - parallel_flags: "-parallel=2" # reduce parallelism for postgres - containers: [postgresql] - runs-on: ${{ needs.set-up-functional-test.outputs.runner_x64 }} - env: - PERSISTENCE_TYPE: ${{ matrix.persistence_type }} - PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }} - TEST_PARALLEL_FLAGS: ${{ matrix.parallel_flags }} - steps: - - uses: actions/checkout@v6 - with: - token: ${{ secrets.GITHUB_TOKEN }} - ref: ${{ env.COMMIT }} - # Resolve the numeric job ID for this job instance. - # IMPORTANT: `job_name` must exactly match the job's display name above. - - name: Get job ID - id: get_job_id - uses: ./.github/actions/get-job-id - with: - job_name: Functional test xdc (${{ matrix.name }}) - run_id: ${{ github.run_id }} - - - name: Start containerized dependencies - uses: hoverkraft-tech/compose-action@v2.0.1 - with: - compose-file: ${{ env.DOCKER_COMPOSE_FILE }} - services: "${{ join(matrix.containers, '\n') }}" - down-flags: -v - - - uses: actions/setup-go@v6 - with: - go-version-file: "go.mod" - cache: false # do our own caching - - - name: Restore dependencies - uses: actions/cache/restore@v4 - with: - path: ~/go/pkg/mod - key: go-${{ runner.os }}${{ runner.arch }}-${{ hashFiles('go.mod') }}-deps-${{ hashFiles('go.sum') }} - - - name: Restore build outputs - uses: actions/cache/restore@v4 - with: - path: ~/.cache/go-build - key: go-${{ runner.os }}${{ runner.arch }}-build-${{ env.COMMIT }} - - - name: Run functional test xdc - timeout-minutes: 35 # update this to TEST_TIMEOUT+5 if you update the Makefile - run: ./develop/github/monitor_test.sh make functional-test-xdc-coverage - - - name: Print memory snapshot - if: always() - run: if [ -f /tmp/memory_snapshot.txt ]; then cat /tmp/memory_snapshot.txt; fi - - - name: Generate crash report - if: failure() - run: | # if the tests failed, we would expect one JUnit XML report per attempt; otherwise it must have crashed - [ "$(find .testoutput -maxdepth 1 -name 'junit.*.xml' | wc -l)" -lt "$MAX_TEST_ATTEMPTS" ] && - CRASH_REPORT_NAME="$GITHUB_JOB" make report-test-crash - - - name: Generate test summary - uses: mikepenz/action-junit-report@v5.0.0-rc01 - if: failure() - with: - report_paths: ./.testoutput/junit.*.xml - detailed_summary: true - check_annotations: false - annotate_only: true - skip_annotations: true - - - name: Upload code coverage to Codecov - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - directory: ./.testoutput - flags: functional-test-xdc - - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - directory: ./.testoutput - flags: functional-test-xdc - report_type: test_results - - - name: Upload test results to GitHub - # Can't pin to major because the action linter doesn't recognize the include-hidden-files flag. - uses: actions/upload-artifact@v4.4.3 - if: ${{ !cancelled() }} - with: - name: junit-xml--${{github.run_id}}--${{ steps.get_job_id.outputs.job_id }}--${{github.run_attempt}}--${{matrix.name}}--functional-test-xdc - path: ./.testoutput - include-hidden-files: true - retention-days: 28 - - # NDC matrix job. Include `${{ matrix.name }}` in the display name so each - # matrix variant has a unique name for the job-id lookup. - functional-test-ndc: - # Display name shown in the UI. The job-id lookup uses this exact value. - name: Functional test ndc (${{ matrix.name }}) - needs: [pre-build, set-up-functional-test] - strategy: - fail-fast: false - matrix: - name: - - cass_es - - cass_es8 - - cass_os2 - - cass_os3 - - mysql8 - - postgres12 - - postgres12_pgx - include: - - name: cass_es - persistence_type: nosql - persistence_driver: elasticsearch - containers: [cassandra, elasticsearch] - es_version: v7 - - name: cass_es8 - persistence_type: nosql - persistence_driver: elasticsearch - containers: [cassandra, elasticsearch8] - es_version: v8 - - name: cass_os2 - persistence_type: nosql - persistence_driver: cassandra - containers: [cassandra, opensearch2] - - name: cass_os3 - persistence_type: nosql - persistence_driver: cassandra - containers: [cassandra, opensearch3] - - name: mysql8 - persistence_type: sql - persistence_driver: mysql8 - containers: [mysql] - - name: postgres12 - persistence_type: sql - persistence_driver: postgres12 - containers: [postgresql] - - name: postgres12_pgx - persistence_type: sql - persistence_driver: postgres12_pgx - containers: [postgresql] - runs-on: ${{ needs.set-up-functional-test.outputs.runner_x64 }} - env: - PERSISTENCE_TYPE: ${{ matrix.persistence_type }} - PERSISTENCE_DRIVER: ${{ matrix.persistence_driver }} - ES_VERSION: ${{ matrix.es_version }} - steps: - - uses: actions/checkout@v6 - with: - token: ${{ secrets.GITHUB_TOKEN }} - ref: ${{ env.COMMIT }} - # Resolve the numeric job ID for this job instance. - # IMPORTANT: `job_name` must exactly match the job's display name above. - - name: Get job ID - id: get_job_id - uses: ./.github/actions/get-job-id - with: - job_name: Functional test ndc (${{ matrix.name }}) - run_id: ${{ github.run_id }} - - - name: Start containerized dependencies - uses: hoverkraft-tech/compose-action@v2.0.1 - with: - compose-file: ${{ env.DOCKER_COMPOSE_FILE }} - services: "${{ join(matrix.containers, '\n') }}" - down-flags: -v - - - uses: actions/setup-go@v6 - with: - go-version-file: "go.mod" - cache: false # do our own caching - - - name: Restore dependencies - uses: actions/cache/restore@v4 - with: - path: ~/go/pkg/mod - key: go-${{ runner.os }}${{ runner.arch }}-${{ hashFiles('go.mod') }}-deps-${{ hashFiles('go.sum') }} - - - name: Restore build outputs - uses: actions/cache/restore@v4 - with: - path: ~/.cache/go-build - key: go-${{ runner.os }}${{ runner.arch }}-build-${{ env.COMMIT }} - - - name: Run functional test ndc - timeout-minutes: 15 - run: ./develop/github/monitor_test.sh make functional-test-ndc-coverage - - - name: Print memory snapshot - if: always() - run: if [ -f /tmp/memory_snapshot.txt ]; then cat /tmp/memory_snapshot.txt; fi - - - name: Generate crash report - if: failure() - run: | # if the tests failed, we would expect one JUnit XML report per attempt; otherwise it must have crashed - [ "$(find .testoutput -maxdepth 1 -name 'junit.*.xml' | wc -l)" -lt "$MAX_TEST_ATTEMPTS" ] && - CRASH_REPORT_NAME="$GITHUB_JOB" make report-test-crash - - - name: Upload code coverage to Codecov - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - directory: ./.testoutput - flags: functional-test-ndc - - - name: Upload test results to Codecov - if: ${{ !cancelled() }} - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - directory: ./.testoutput - flags: functional-test-ndc - report_type: test_results - - - name: Upload test results to GitHub - # Can't pin to major because the action linter doesn't recognize the include-hidden-files flag. - uses: actions/upload-artifact@v4.4.3 - if: ${{ !cancelled() }} - with: - name: junit-xml--${{github.run_id}}--${{ steps.get_job_id.outputs.job_id }}--${{github.run_attempt}}--${{matrix.name}}--functional-test-ndc + name: junit-xml--${{github.run_id}}--${{ steps.get_job_id.outputs.job_id }}--${{github.run_attempt}}--${{matrix.name}}--${{ matrix.is_smoke && 'smoke' || matrix.shard_index }}--functional-test path: ./.testoutput/junit.*.xml include-hidden-files: true retention-days: 28 @@ -741,16 +533,14 @@ jobs: - unit-test - integration-test - functional-test - - functional-test-xdc - - functional-test-ndc runs-on: ubuntu-latest env: RESULTS: ${{ toJSON(needs.*.result) }} steps: - name: Check results run: | - # all statuses must be success - if [[ -n $(echo "$RESULTS" | jq '.[] | select (. != "success")') ]]; then + # All statuses must be success or skipped + if [[ -n $(echo "$RESULTS" | jq '.[] | select (. != "success" and . != "skipped")') ]]; then exit 1 fi diff --git a/Makefile b/Makefile index 31cc4f4e53..ea874a2710 100644 --- a/Makefile +++ b/Makefile @@ -120,6 +120,7 @@ TEST_DIRS := $(sort $(dir $(filter %_test.go,$(ALL_SRC)))) FUNCTIONAL_TEST_ROOT := ./tests FUNCTIONAL_TEST_XDC_ROOT := ./tests/xdc FUNCTIONAL_TEST_NDC_ROOT := ./tests/ndc +FUNCTIONAL_TEST_ALL_ROOTS := $(FUNCTIONAL_TEST_ROOT) $(FUNCTIONAL_TEST_XDC_ROOT) $(FUNCTIONAL_TEST_NDC_ROOT) DB_INTEGRATION_TEST_ROOT := ./common/persistence/tests DB_TOOL_INTEGRATION_TEST_ROOT := ./tools/tests INTEGRATION_TEST_DIRS := $(DB_INTEGRATION_TEST_ROOT) $(DB_TOOL_INTEGRATION_TEST_ROOT) ./temporaltest @@ -484,14 +485,14 @@ integration-test-coverage: prepare-coverage-test go run ./cmd/tools/test-runner test --gotestsum-path=$(GOTESTSUM) --max-attempts=$(MAX_TEST_ATTEMPTS) --junitfile=$(NEW_REPORT) -- \ $(COMPILED_TEST_ARGS) -coverprofile=$(NEW_COVER_PROFILE) $(INTEGRATION_TEST_DIRS) -# This should use the same build flags as functional-test-coverage and functional-test-{xdc,ndc}-coverage for best build caching. +# This MUST use the same build flags as functional-test-coverage for best build caching. pre-build-functional-test-coverage: prepare-coverage-test - go test -c -cover -o /dev/null $(FUNCTIONAL_TEST_ROOT) $(TEST_ARGS) $(TEST_TAG_FLAG) $(COVERPKG_FLAG) + go test -c -cover -o /dev/null $(COMPILED_TEST_ARGS) $(COVERPKG_FLAG) $(FUNCTIONAL_TEST_ALL_ROOTS) functional-test-coverage: prepare-coverage-test - @printf $(COLOR) "Run functional tests with coverage with $(PERSISTENCE_DRIVER) driver..." + @printf $(COLOR) "Run all functional tests with coverage with $(PERSISTENCE_DRIVER) driver..." go run ./cmd/tools/test-runner test --gotestsum-path=$(GOTESTSUM) --max-attempts=$(MAX_TEST_ATTEMPTS) --junitfile=$(NEW_REPORT) -- \ - $(COMPILED_TEST_ARGS) -coverprofile=$(NEW_COVER_PROFILE) $(COVERPKG_FLAG) $(FUNCTIONAL_TEST_ROOT) \ + $(COMPILED_TEST_ARGS) -coverprofile=$(NEW_COVER_PROFILE) $(COVERPKG_FLAG) $(FUNCTIONAL_TEST_ALL_ROOTS) \ -args -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER) functional-test-xdc-coverage: prepare-coverage-test diff --git a/tests/namespace_delete_test.go b/tests/namespace_delete_test.go index d2709a4e9c..3e5c8003a6 100644 --- a/tests/namespace_delete_test.go +++ b/tests/namespace_delete_test.go @@ -4,12 +4,10 @@ import ( "context" "errors" "fmt" - "os" "strconv" "testing" "time" - "github.com/dgryski/go-farm" "github.com/google/uuid" "github.com/stretchr/testify/suite" commonpb "go.temporal.io/api/common/v1" @@ -394,35 +392,6 @@ func (s *namespaceTestSuite) Test_NamespaceDelete_CrossNamespaceChild() { // Delete second namespace and verify that parent received child termination signal. } -// checkTestShard supports test sharding based on environment variables. -func (s *namespaceTestSuite) checkTestShard() { - totalStr := os.Getenv("TEST_TOTAL_SHARDS") - indexStr := os.Getenv("TEST_SHARD_INDEX") - if totalStr == "" || indexStr == "" { - return - } - total, err := strconv.Atoi(totalStr) - if err != nil || total < 1 { - s.T().Fatal("Couldn't convert TEST_TOTAL_SHARDS") - } - index, err := strconv.Atoi(indexStr) - if err != nil || index < 0 || index >= total { - s.T().Fatal("Couldn't convert TEST_SHARD_INDEX") - } - - // This was determined empirically to distribute our existing test names - // reasonably well. This can be adjusted from time to time. - // For parallelism 4, use 11. For 3, use 26. For 2, use 20. - const salt = "-salt-26" - - nameToHash := s.T().Name() + salt - testIndex := int(farm.Fingerprint32([]byte(nameToHash))) % total - if testIndex != index { - s.T().Skipf("Skipping %s in test shard %d/%d (it runs in %d)", s.T().Name(), index+1, total, testIndex+1) - } - s.T().Logf("Running %s in test shard %d/%d", s.T().Name(), index+1, total) -} - func (s *namespaceTestSuite) Test_NamespaceDelete_Protected() { ctx := context.Background() diff --git a/tests/testcore/functional_test_base.go b/tests/testcore/functional_test_base.go index 1179cc4e54..cda025d9bb 100644 --- a/tests/testcore/functional_test_base.go +++ b/tests/testcore/functional_test_base.go @@ -9,10 +9,8 @@ import ( "maps" "os" "regexp" - "strconv" "time" - "github.com/dgryski/go-farm" "github.com/google/uuid" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -311,12 +309,7 @@ func (s *FunctionalTestBase) setupCluster(options ...TestClusterOption) { s.Require().NoError(err) } -// All test suites that inherit FunctionalTestBase and overwrite SetupTest must -// call this testcore FunctionalTestBase.SetupTest function to distribute the tests -// into partitions. Otherwise, the test suite will be executed multiple times -// in each partition. func (s *FunctionalTestBase) SetupTest() { - s.checkTestShard() s.initAssertions() s.setupSdk() s.taskPoller = taskpoller.New(s.T(), s.FrontendClient(), s.Namespace().String()) @@ -344,35 +337,6 @@ func (s *FunctionalTestBase) initAssertions() { s.UpdateUtils = updateutils.New(s.T()) } -// checkTestShard supports test sharding based on environment variables. -func (s *FunctionalTestBase) checkTestShard() { - totalStr := os.Getenv("TEST_TOTAL_SHARDS") - indexStr := os.Getenv("TEST_SHARD_INDEX") - if totalStr == "" || indexStr == "" { - return - } - total, err := strconv.Atoi(totalStr) - if err != nil || total < 1 { - s.T().Fatal("Couldn't convert TEST_TOTAL_SHARDS") - } - index, err := strconv.Atoi(indexStr) - if err != nil || index < 0 || index >= total { - s.T().Fatal("Couldn't convert TEST_SHARD_INDEX") - } - - // This was determined empirically to distribute our existing test names - // reasonably well. This can be adjusted from time to time. - // For parallelism 4, use 11. For 3, use 26. For 2, use 20. - const salt = "-salt-26" - - nameToHash := s.T().Name() + salt - testIndex := int(farm.Fingerprint32([]byte(nameToHash))) % total - if testIndex != index { - s.T().Skipf("Skipping %s in test shard %d/%d (it runs in %d)", s.T().Name(), index+1, total, testIndex+1) - } - s.T().Logf("Running %s in test shard %d/%d", s.T().Name(), index+1, total) -} - func ApplyTestClusterOptions(options []TestClusterOption) TestClusterParams { params := TestClusterParams{ ServiceOptions: make(map[primitives.ServiceName][]fx.Option), diff --git a/tools/testrunner/testrunner.go b/tools/testrunner/testrunner.go index 357f5d5dfb..673c890d5e 100644 --- a/tools/testrunner/testrunner.go +++ b/tools/testrunner/testrunner.go @@ -1,6 +1,7 @@ package testrunner import ( + "bufio" "context" "errors" "fmt" @@ -14,6 +15,7 @@ import ( "strconv" "strings" + "github.com/dgryski/go-farm" "github.com/google/uuid" ) @@ -28,6 +30,10 @@ const ( // fullRerunThreshold is the number of test failures above which we do a full // rerun instead of retrying only the failed tests. fullRerunThreshold = 20 + + shardTotalEnv = "TEMPORAL_TEST_TOTAL_SHARDS" + shardIndexEnv = "TEMPORAL_TEST_SHARD_INDEX" + shardSaltEnv = "TEMPORAL_TEST_SHARD_SALT" ) const ( @@ -70,6 +76,12 @@ type runner struct { maxAttempts int crashName string alerts []alert + + // Sharding config, parsed from environment variables and args. + shardTotal int + shardIndex int + shardSalt string + packages []string // test package paths (after "--", non-flag args) } func newRunner() *runner { @@ -82,6 +94,8 @@ func newRunner() *runner { // nolint:revive,cognitive-complexity func (r *runner) sanitizeAndParseArgs(command string, args []string) ([]string, error) { var sanitizedArgs []string + afterSep := false + hasRunFlag := false for _, arg := range args { if strings.HasPrefix(arg, maxAttemptsFlag) { var err error @@ -118,6 +132,15 @@ func (r *runner) sanitizeAndParseArgs(command string, args []string) ([]string, r.junitOutputPath = strings.Split(arg, "=")[1] } + // Track packages and -run flag after the "--" separator. + if arg == "--" { + afterSep = true + } else if afterSep && (arg == "-run" || strings.HasPrefix(arg, "-run=")) { + hasRunFlag = true + } else if afterSep && strings.HasPrefix(arg, "./") { + r.packages = append(r.packages, arg) + } + sanitizedArgs = append(sanitizedArgs, arg) } @@ -136,6 +159,12 @@ func (r *runner) sanitizeAndParseArgs(command string, args []string) ([]string, if r.gotestsumPath == "" { return nil, fmt.Errorf("missing required argument %q", gotestsumPathFlag) } + if err := r.parseEnvVars(); err != nil { + return nil, err + } + if r.shardTotal > 0 && hasRunFlag { + return nil, fmt.Errorf("-run flag cannot be combined with sharding env vars") + } case crashReportCommand: if r.crashName == "" { return nil, fmt.Errorf("missing required argument %q", crashReportNameFlag) @@ -209,6 +238,8 @@ func (r *runner) reportCrash() { } func (r *runner) runTests(ctx context.Context, args []string) { + args = r.applySharding(args) + var currentAttempt *attempt for a := 1; a <= r.maxAttempts; a++ { currentAttempt = r.newAttempt() @@ -310,6 +341,78 @@ func stripRunFromArgs(args []string) (argsNoRun []string) { return } +func (r *runner) parseEnvVars() error { + totalStr := os.Getenv(shardTotalEnv) + indexStr := os.Getenv(shardIndexEnv) + if totalStr == "" && indexStr == "" { + return nil + } + if totalStr == "" || indexStr == "" { + return fmt.Errorf("%s and %s must both be set or both be unset", shardTotalEnv, shardIndexEnv) + } + total, err := strconv.Atoi(totalStr) + if err != nil || total < 1 { + return fmt.Errorf("invalid %s: %q", shardTotalEnv, totalStr) + } + index, err := strconv.Atoi(indexStr) + if err != nil || index < 1 || index > total { + return fmt.Errorf("invalid %s: %q (must be 1..%d)", shardIndexEnv, indexStr, total) + } + salt := os.Getenv(shardSaltEnv) + if salt == "" { + return fmt.Errorf("%s must be set when sharding is enabled", shardSaltEnv) + } + r.shardTotal = total + r.shardIndex = index - 1 // convert to 0-based for hash modulo + r.shardSalt = salt + return nil +} + +// applySharding discovers top-level test names via `go test -list` and builds a +// -run filter so that only this shard's tests are executed. +// nolint:revive,deep-exit +func (r *runner) applySharding(args []string) []string { + if r.shardTotal == 0 || len(r.packages) == 0 { + return args + } + + // Discover top-level test names. + cmd := exec.Command("go", append([]string{"test", "-list", ".*"}, r.packages...)...) + cmd.Stderr = os.Stderr + out, err := cmd.Output() + if err != nil { + log.Fatalf("go test -list failed: %v", err) + } + + // Find tests for this shard. + // go test -list output contains one test name per line, plus summary lines + // like "ok \t\t