diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 8929d9b3d35c..e735c4330e01 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -4177,9 +4177,9 @@ jobs: uses: ./.github/workflows/regression.yml secrets: inherit with: - runner_type: altinity-on-demand, altinity-regression-tester + runner_type: altinity-regression-tester commit: 8d2c6d2072771d450a47faca38966da7493821e1 - arch: release + arch: x86 build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout_minutes: 300 workflow_config: ${{ needs.config_workflow.outputs.data }} @@ -4189,7 +4189,7 @@ jobs: uses: ./.github/workflows/regression.yml secrets: inherit with: - runner_type: altinity-on-demand, altinity-regression-tester-aarch64 + runner_type: altinity-regression-tester-aarch64 commit: 8d2c6d2072771d450a47faca38966da7493821e1 arch: aarch64 build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 7d62740af09b..c5c992cf93f7 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -4133,9 +4133,9 @@ jobs: uses: ./.github/workflows/regression.yml secrets: inherit with: - runner_type: altinity-on-demand, altinity-regression-tester + runner_type: altinity-regression-tester commit: 8d2c6d2072771d450a47faca38966da7493821e1 - arch: release + arch: x86 build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout_minutes: 300 workflow_config: ${{ needs.config_workflow.outputs.data }} @@ -4145,7 +4145,7 @@ jobs: uses: ./.github/workflows/regression.yml secrets: inherit with: - runner_type: altinity-on-demand, altinity-regression-tester-aarch64 + runner_type: altinity-regression-tester-aarch64 commit: 8d2c6d2072771d450a47faca38966da7493821e1 arch: aarch64 build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} diff --git a/.github/workflows/regression-reusable-suite.yml b/.github/workflows/regression-reusable-suite.yml new file mode 100644 index 000000000000..c313d69a2482 --- /dev/null +++ b/.github/workflows/regression-reusable-suite.yml @@ -0,0 +1,192 @@ +name: Regression suite +on: + workflow_call: + inputs: + ref: + description: "Commit SHA to checkout. Default: current (empty string)." + type: string + default: "" + workflow_config: + required: true + type: string + flags: + required: false + type: string + output_format: + required: true + type: string + extra_args: + required: false + type: string + suite_name: + required: true + type: string + suite_executable: + required: false + type: string + default: "regression.py" + timeout_minutes: + required: true + type: number + storage_path: + required: false + type: string + default: "" + regression_args: + required: false + type: string + default: "" + runner_type: + required: false + type: string + default: "" + runner_arch: + required: false + type: string + default: "x86" + job_name: + required: false + type: string + default: "" + part: + required: false + type: string + default: "" + build_sha: + required: false + type: string + default: "" + set_commit_status: + required: false + type: boolean + default: false +jobs: + suite: + name: ${{ format('{0}{1}', inputs.job_name != '' && inputs.job_name || inputs.suite_name, inputs.part != '' && format('_{0}', inputs.part) || '') }} + runs-on: [ + "self-hosted", + "altinity-on-demand", + "${{ inputs.runner_type }}", + ] + timeout-minutes: ${{ inputs.timeout_minutes }} + env: + SUITE: ${{ inputs.suite_name }} + SUITE_EXECUTABLE: ${{ inputs.suite_executable }} + STORAGE: ${{ inputs.storage_path }} + PART: ${{ inputs.part }} + # AWS credentials + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }} + # Docker credentials + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + # Database credentials + CHECKS_DATABASE_HOST: ${{ secrets.CHECKS_DATABASE_HOST }} + CHECKS_DATABASE_USER: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }} + CHECKS_DATABASE_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }} + # LocalStack token + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + # Python encoding + PYTHONIOENCODING: utf-8 + build_sha: ${{ inputs.build_sha }} + pr_number: ${{ github.event.number }} + artifacts: builds + # Args + args: --test-to-end + --no-colors + --local + --collect-service-logs + --output ${{ inputs.output_format }} + --attr project="${GITHUB_REPOSITORY}" project.id="${GITHUB_REPOSITORY_ID}" user.name="${GITHUB_ACTOR}" version="${{ fromJson(inputs.workflow_config).custom_data.version.string }}" package="$clickhouse_path" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" arch="$(uname -i)" + --cicd + --log raw.log + ${{ inputs.flags != 'none' && inputs.flags || ''}} + ${{ inputs.extra_args }} + artifact_paths: | + ./report.html + ./*.log.txt + ./*.log + ./*.html + ./*/_instances/*.log + ./*/_instances/*/logs/*.log + ./*/*/_instances/*/logs/*.log + ./*/*/_instances/*.log + + steps: + - name: โคต๏ธ Checkout + uses: actions/checkout@v4 + with: + repository: Altinity/clickhouse-regression + ref: ${{ inputs.ref }} + + - name: โ™ป๏ธ Cache setup + uses: ./.github/actions/cache-setup + + - name: ๐Ÿ› ๏ธ Setup + run: .github/setup.sh + + - name: ๐Ÿ“ฆ Get deb url + env: + S3_BASE_URL: https://altinity-build-artifacts.s3.amazonaws.com/ + PR_NUMBER: ${{ github.event.pull_request.number || 0 }} + run: | + REPORTS_PATH=${{ runner.temp }}/reports_dir + mkdir -p $REPORTS_PATH + cat > $REPORTS_PATH/workflow_config.json << 'EOF' + ${{ inputs.workflow_config }} + EOF + + python3 .github/get-deb-url.py --github-env $GITHUB_ENV --workflow-config $REPORTS_PATH/workflow_config.json --s3-base-url $S3_BASE_URL --pr-number $PR_NUMBER --branch-name ${{ github.ref_name }} --commit-hash ${{ inputs.build_sha || github.sha }} --binary + + - name: ๐Ÿ”„ Process regression args + run: | + REGRESSION_ARGS='${{ inputs.regression_args }}' + # AWS replacements + REGRESSION_ARGS="${REGRESSION_ARGS//'{{AWS_BUCKET}}'/${{ secrets.REGRESSION_AWS_S3_BUCKET }}}" + REGRESSION_ARGS="${REGRESSION_ARGS//'{{AWS_REGION}}'/${{ secrets.REGRESSION_AWS_S3_REGION }}}" + REGRESSION_ARGS="${REGRESSION_ARGS//'{{AWS_KEY_ID}}'/${{ secrets.REGRESSION_AWS_S3_KEY_ID }}}" + REGRESSION_ARGS="${REGRESSION_ARGS//'{{AWS_ACCESS_KEY}}'/${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }}}" + # GCS replacements + REGRESSION_ARGS="${REGRESSION_ARGS//'{{GCS_URI}}'/${{ secrets.REGRESSION_GCS_URI }}}" + REGRESSION_ARGS="${REGRESSION_ARGS//'{{GCS_KEY_ID}}'/${{ secrets.REGRESSION_GCS_KEY_ID }}}" + REGRESSION_ARGS="${REGRESSION_ARGS//'{{GCS_KEY_SECRET}}'/${{ secrets.REGRESSION_GCS_KEY_SECRET }}}" + # Azure replacements + REGRESSION_ARGS="${REGRESSION_ARGS//'{{AZURE_ACCOUNT_NAME}}'/${{ secrets.AZURE_ACCOUNT_NAME }}}" + REGRESSION_ARGS="${REGRESSION_ARGS//'{{AZURE_STORAGE_KEY}}'/${{ secrets.AZURE_STORAGE_KEY }}}" + REGRESSION_ARGS="${REGRESSION_ARGS//'{{AZURE_CONTAINER_NAME}}'/${{ secrets.AZURE_CONTAINER_NAME }}}" + echo "REGRESSION_ARGS=$REGRESSION_ARGS" >> $GITHUB_ENV + + - name: ๐Ÿงช Run ${{ env.SUITE }} suite + id: run_suite + run: python3 + -u ${{ env.SUITE }}/${{ env.SUITE_EXECUTABLE }} + --clickhouse ${{ env.clickhouse_path }} + ${{ env.REGRESSION_ARGS }} + ${{ env.args }} || EXITCODE=$?; + .github/add_link_to_logs.sh + exit $EXITCODE + + - name: ๐Ÿ“Š Set Commit Status + if: always() && inputs.set_commit_status + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + JOB_OUTCOME: ${{ steps.run_suite.outcome }} + SUITE_NAME: ${{ format('Regression {0} {1}{2}', inputs.runner_arch, inputs.job_name != '' && inputs.job_name || inputs.suite_name, inputs.part != '' && format('-{0}', inputs.part) || '') }} + run: python3 .github/set_builds_status.py + + - name: ๐Ÿ“ Create and upload logs + if: always() + run: .github/create_and_upload_logs.sh ${{ vars.UPLOAD_LOGS }} + + - name: ๐Ÿ“ค Upload logs to results database + if: always() + timeout-minutes: 20 + run: .github/upload_results_to_database.sh ${{ vars.UPLOAD_LOGS }} + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ format('{0}{1}-artifacts-{2}{3}', inputs.job_name != '' && inputs.job_name || inputs.suite_name, inputs.part != '' && format('_{0}', inputs.part) || '', inputs.runner_arch, contains(inputs.extra_args, '--use-keeper') && '_keeper' || '_zookeeper') }} + path: ${{ env.artifact_paths }} + diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 5275e2550003..df226f2c34d1 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -3,7 +3,7 @@ name: Regression test workflow - Release workflow_call: inputs: runner_type: - description: the label of runner to use, can be a simple string or a comma-separated list + description: the (meta-)label of runner to use required: true type: string commit: @@ -108,7 +108,6 @@ env: --parallel 1 --log raw.log --with-analyzer - artifacts: builds artifact_paths: | ./report.html ./*.log.txt @@ -118,35 +117,8 @@ env: ./*/_instances/*/logs/*.log ./*/*/_instances/*/logs/*.log ./*/*/_instances/*.log - build_sha: ${{ inputs.build_sha }} - pr_number: ${{ github.event.number }} - event_name: ${{ github.event_name }} - version: ${{ fromJson(inputs.workflow_config).custom_data.version.string }} - SKIP_LIST: ${{ join(fromJson(inputs.workflow_config).custom_data.ci_exclude_tags, '|') || '' }} jobs: - runner_labels_setup: - name: Compute proper runner labels for the rest of the jobs - runs-on: ubuntu-latest - outputs: - runner_labels: ${{ steps.setVariables.outputs.runner_labels }} - steps: - - id: setVariables - name: Prepare runner_labels variables for the later steps - run: | - - # Prepend self-hosted - input="self-hosted, ${input}" - - # Remove all whitespace - input="$(echo ${input} | tr -d [:space:])" - # Make something like a JSON array from comma-separated list - input="[ '${input//\,/\'\, \'}' ]" - - echo "runner_labels=$input" >> ${GITHUB_OUTPUT} - env: - input: ${{ inputs.runner_type }} - Common: if: | fromJson(inputs.workflow_config).custom_data.ci_regression_jobs[0] == null || @@ -155,64 +127,21 @@ jobs: fail-fast: false matrix: SUITE: [aes_encryption, atomic_insert, base_58, data_types, datetime64_extended_range, disk_level_encryption, dns, engines, example, extended_precision_data_types, functions, kafka, kerberos, key_value, lightweight_delete, memory, part_moves_between_shards, selects, session_timezone, swarms, version, window_functions] - needs: [runner_labels_setup] - runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} - timeout-minutes: ${{ inputs.timeout_minutes }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v4 - with: - repository: Altinity/clickhouse-regression - ref: ${{ inputs.commit }} - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{ runner.temp }}/reports_dir - SUITE=${{ matrix.SUITE }} - EOF - - name: Setup - run: .github/setup.sh - - name: Get deb url - env: - S3_BASE_URL: https://altinity-build-artifacts.s3.amazonaws.com/ - PR_NUMBER: ${{ github.event.pull_request.number || 0 }} - run: | - mkdir -p $REPORTS_PATH - cat > $REPORTS_PATH/workflow_config.json << 'EOF' - ${{ inputs.workflow_config }} - EOF - - python3 .github/get-deb-url.py --github-env $GITHUB_ENV --workflow-config $REPORTS_PATH/workflow_config.json --s3-base-url $S3_BASE_URL --pr-number $PR_NUMBER --branch-name ${{ github.ref_name }} --commit-hash ${{ inputs.build_sha }} --binary - - - name: Run ${{ env.SUITE }} suite - id: run_suite - run: EXITCODE=0; - python3 - -u ${{ env.SUITE }}/regression.py - --clickhouse-binary-path ${{ env.clickhouse_path }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name="$GITHUB_JOB (${{ matrix.SUITE }})" job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" - ${{ env.args }} || EXITCODE=$?; - .github/add_link_to_logs.sh; - exit $EXITCODE - - name: Set Commit Status - if: always() - run: python3 .github/set_builds_status.py - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - JOB_OUTCOME: ${{ steps.run_suite.outcome }} - SUITE_NAME: "Regression ${{ inputs.arch }} ${{ matrix.SUITE }}" - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - - name: Upload logs to regression results database - if: always() - timeout-minutes: 20 - run: .github/upload_results_to_database.sh 1 - - uses: actions/upload-artifact@v4 - if: always() - with: - name: ${{ env.SUITE }}-${{ inputs.arch }}-artifacts - path: ${{ env.artifact_paths}} + uses: ./.github/workflows/regression-reusable-suite.yml + with: + ref: ${{ inputs.commit }} + workflow_config: ${{ inputs.workflow_config }} + suite_name: ${{ matrix.SUITE }} + suite_executable: regression.py + output_format: new-fails + flags: --with-analyzer + timeout_minutes: ${{ inputs.timeout_minutes }} + runner_arch: ${{ inputs.arch }} + runner_type: ${{ inputs.runner_type }} + build_sha: ${{ inputs.build_sha }} + set_commit_status: true + job_name: ${{ matrix.SUITE }} + secrets: inherit AggregateFunctions: if: | @@ -222,140 +151,54 @@ jobs: fail-fast: false matrix: PART: [1, 2, 3] - needs: [runner_labels_setup] - runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} - timeout-minutes: ${{ inputs.timeout_minutes }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v4 - with: - repository: Altinity/clickhouse-regression - ref: ${{ inputs.commit }} - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{ runner.temp }}/reports_dir - SUITE=aggregate_functions - PART=${{ matrix.PART }} - EOF - - name: Setup - run: .github/setup.sh - - name: Get deb url - env: - S3_BASE_URL: https://altinity-build-artifacts.s3.amazonaws.com/ - PR_NUMBER: ${{ github.event.pull_request.number || 0 }} - run: | - mkdir -p $REPORTS_PATH - cat > $REPORTS_PATH/workflow_config.json << 'EOF' - ${{ inputs.workflow_config }} - EOF - - python3 .github/get-deb-url.py --github-env $GITHUB_ENV --workflow-config $REPORTS_PATH/workflow_config.json --s3-base-url $S3_BASE_URL --pr-number $PR_NUMBER --branch-name ${{ github.ref_name }} --commit-hash ${{ inputs.build_sha }} --binary - - - name: Run ${{ env.SUITE }} suite - id: run_suite - run: EXITCODE=0; - python3 - -u ${{ env.SUITE }}/regression.py - --clickhouse-binary-path ${{ env.clickhouse_path }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name="$GITHUB_JOB (${{ matrix.PART }})" job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" - --only "part ${{ matrix.PART }}/*" - ${{ env.args }} || EXITCODE=$?; - .github/add_link_to_logs.sh; - exit $EXITCODE - - name: Set Commit Status - if: always() - run: python3 .github/set_builds_status.py - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - JOB_OUTCOME: ${{ steps.run_suite.outcome }} - SUITE_NAME: "Regression ${{ inputs.arch }} ${{ env.SUITE }}-${{ matrix.PART }}" - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - - name: Upload logs to regression results database - if: always() - timeout-minutes: 20 - run: .github/upload_results_to_database.sh 1 - - uses: actions/upload-artifact@v4 - if: always() - with: - name: ${{ env.SUITE }}-${{ matrix.PART }}-${{ inputs.arch }}-artifacts - path: ${{ env.artifact_paths}} + uses: ./.github/workflows/regression-reusable-suite.yml + with: + ref: ${{ inputs.commit }} + workflow_config: ${{ inputs.workflow_config }} + suite_name: aggregate_functions + suite_executable: regression.py + output_format: new-fails + flags: --with-analyzer + timeout_minutes: ${{ inputs.timeout_minutes }} + runner_arch: ${{ inputs.arch }} + runner_type: ${{ inputs.runner_type }} + part: ${{ matrix.PART }} + build_sha: ${{ inputs.build_sha }} + set_commit_status: true + job_name: aggregate_functions + extra_args: --only "part ${{ matrix.PART }}/*" + secrets: inherit Alter: - if: | - fromJson(inputs.workflow_config).custom_data.ci_regression_jobs[0] == null || - contains(fromJson(inputs.workflow_config).custom_data.ci_regression_jobs, 'alter') - strategy: - fail-fast: false - matrix: - ONLY: [replace, move] - include: - - ONLY: attach - PART: 1 - - ONLY: attach - PART: 2 - needs: [runner_labels_setup] - runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} - timeout-minutes: ${{ inputs.timeout_minutes }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v4 - with: - repository: Altinity/clickhouse-regression - ref: ${{ inputs.commit }} - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{ runner.temp }}/reports_dir - SUITE=alter - STORAGE=/${{ matrix.ONLY }}_partition - PART=${{ matrix.PART }} - EOF - - name: Setup - run: .github/setup.sh - - name: Get deb url - env: - S3_BASE_URL: https://altinity-build-artifacts.s3.amazonaws.com/ - PR_NUMBER: ${{ github.event.pull_request.number || 0 }} - run: | - mkdir -p $REPORTS_PATH - cat > $REPORTS_PATH/workflow_config.json << 'EOF' - ${{ inputs.workflow_config }} - EOF - - python3 .github/get-deb-url.py --github-env $GITHUB_ENV --workflow-config $REPORTS_PATH/workflow_config.json --s3-base-url $S3_BASE_URL --pr-number $PR_NUMBER --branch-name ${{ github.ref_name }} --commit-hash ${{ inputs.build_sha }} --binary - - - name: Run ${{ env.SUITE }} suite - id: run_suite - run: EXITCODE=0; - python3 - -u alter/regression.py - --clickhouse-binary-path ${{ env.clickhouse_path }} - --only "/alter/${{ matrix.ONLY }} partition/${{ matrix.PART && format('part {0}/', matrix.PART) || '' }}*" - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name="$GITHUB_JOB (${{ matrix.ONLY }}${{ matrix.PART }})" job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" - ${{ env.args }} || EXITCODE=$?; - .github/add_link_to_logs.sh; - exit $EXITCODE - - name: Set Commit Status - if: always() - run: python3 .github/set_builds_status.py - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - JOB_OUTCOME: ${{ steps.run_suite.outcome }} - SUITE_NAME: "Regression ${{ inputs.arch }} Alter ${{ matrix.ONLY }} partition ${{ matrix.PART }}" - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - - name: Upload logs to regression results database - if: always() - timeout-minutes: 20 - run: .github/upload_results_to_database.sh 1 - - uses: actions/upload-artifact@v4 - if: always() - with: - name: alter-${{ matrix.ONLY }}${{ matrix.PART && format('-{0}', matrix.PART) || '' }}-${{ inputs.arch }}-artifacts - path: ${{ env.artifact_paths}} + if: | + fromJson(inputs.workflow_config).custom_data.ci_regression_jobs[0] == null || + contains(fromJson(inputs.workflow_config).custom_data.ci_regression_jobs, 'alter') + strategy: + fail-fast: false + matrix: + ONLY: [replace, move] + include: + - ONLY: attach + PART: 1 + - ONLY: attach + PART: 2 + uses: ./.github/workflows/regression-reusable-suite.yml + with: + ref: ${{ inputs.commit }} + workflow_config: ${{ inputs.workflow_config }} + suite_name: alter + suite_executable: regression.py + output_format: new-fails + flags: --with-analyzer + timeout_minutes: ${{ inputs.timeout_minutes }} + runner_arch: ${{ inputs.arch }} + runner_type: ${{ inputs.runner_type }} + storage_path: /${{ matrix.ONLY }}_partition + part: ${{ matrix.PART }} + build_sha: ${{ inputs.build_sha }} + set_commit_status: true + job_name: alter_${{ matrix.ONLY }} + extra_args: --only "/alter/${{ matrix.ONLY }} partition/${{ matrix.PART && format('part {0}/', matrix.PART) || '' }}*" + secrets: inherit Benchmark: if: | @@ -365,73 +208,23 @@ jobs: fail-fast: false matrix: STORAGE: [minio, aws_s3, gcs] - needs: [runner_labels_setup] - runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} - timeout-minutes: ${{ inputs.timeout_minutes }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v4 - with: - repository: Altinity/clickhouse-regression - ref: ${{ inputs.commit }} - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{ runner.temp }}/reports_dir - SUITE=ontime_benchmark - STORAGE=/${{ matrix.STORAGE }} - EOF - - name: Setup - run: .github/setup.sh - - name: Get deb url - env: - S3_BASE_URL: https://altinity-build-artifacts.s3.amazonaws.com/ - PR_NUMBER: ${{ github.event.pull_request.number || 0 }} - run: | - mkdir -p $REPORTS_PATH - cat > $REPORTS_PATH/workflow_config.json << 'EOF' - ${{ inputs.workflow_config }} - EOF - - python3 .github/get-deb-url.py --github-env $GITHUB_ENV --workflow-config $REPORTS_PATH/workflow_config.json --s3-base-url $S3_BASE_URL --pr-number $PR_NUMBER --branch-name ${{ github.ref_name }} --commit-hash ${{ inputs.build_sha }} --binary - - - name: Run ${{ env.SUITE }} suite - id: run_suite - run: EXITCODE=0; - python3 - -u ${{ env.SUITE }}/benchmark.py - --clickhouse-binary-path ${{ env.clickhouse_path }} - --storage ${{ matrix.STORAGE }} - --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} - --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} - --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} - --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} - --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} - --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} - --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name="$GITHUB_JOB (${{ matrix.STORAGE }})" job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" - ${{ env.args }} || EXITCODE=$?; - .github/add_link_to_logs.sh; - exit $EXITCODE - - name: Set Commit Status - if: always() - run: python3 .github/set_builds_status.py - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - JOB_OUTCOME: ${{ steps.run_suite.outcome }} - SUITE_NAME: "Regression ${{ inputs.arch }} Benchmark ${{ matrix.STORAGE }}" - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - - name: Upload logs to regression results database - if: always() - timeout-minutes: 20 - run: .github/upload_results_to_database.sh 1 - - uses: actions/upload-artifact@v4 - if: always() - with: - name: benchmark-${{ matrix.STORAGE }}-${{ inputs.arch }}-artifacts - path: ${{ env.artifact_paths }} + uses: ./.github/workflows/regression-reusable-suite.yml + with: + ref: ${{ inputs.commit }} + workflow_config: ${{ inputs.workflow_config }} + suite_name: ontime_benchmark + suite_executable: benchmark.py + output_format: new-fails + flags: --with-analyzer + timeout_minutes: ${{ inputs.timeout_minutes }} + runner_arch: ${{ inputs.arch }} + runner_type: ${{ inputs.runner_type }} + storage_path: /${{ matrix.STORAGE }} + build_sha: ${{ inputs.build_sha }} + set_commit_status: true + job_name: benchmark_${{ matrix.STORAGE }} + regression_args: --storage ${{ matrix.STORAGE }} --gcs-uri {{GCS_URI}} --gcs-key-id {{GCS_KEY_ID}} --gcs-key-secret {{GCS_KEY_SECRET}} --aws-s3-bucket {{AWS_BUCKET}} --aws-s3-region {{AWS_REGION}} --aws-s3-key-id {{AWS_KEY_ID}} --aws-s3-access-key {{AWS_ACCESS_KEY}} + secrets: inherit ClickHouseKeeper: if: | @@ -442,68 +235,24 @@ jobs: matrix: PART: [1, 2] SSL: [ssl, no_ssl] - needs: [runner_labels_setup] - runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} - timeout-minutes: ${{ inputs.timeout_minutes }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v4 - with: - repository: Altinity/clickhouse-regression - ref: ${{ inputs.commit }} - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{runner.temp}}/reports_dir - SUITE=clickhouse_keeper - STORAGE=/${{ matrix.SSL }} - PART=${{ matrix.PART }} - SSL=${{ matrix.SSL == 'ssl' && '--ssl' || '' }} - EOF - - name: Setup - run: .github/setup.sh - - name: Get deb url - env: - S3_BASE_URL: https://altinity-build-artifacts.s3.amazonaws.com/ - PR_NUMBER: ${{ github.event.pull_request.number || 0 }} - run: | - mkdir -p $REPORTS_PATH - cat > $REPORTS_PATH/workflow_config.json << 'EOF' - ${{ inputs.workflow_config }} - EOF - - python3 .github/get-deb-url.py --github-env $GITHUB_ENV --workflow-config $REPORTS_PATH/workflow_config.json --s3-base-url $S3_BASE_URL --pr-number $PR_NUMBER --branch-name ${{ github.ref_name }} --commit-hash ${{ inputs.build_sha }} --binary - - - name: Run ${{ env.SUITE }} suite - id: run_suite - run: EXITCODE=0; - python3 - -u ${{ env.SUITE }}/regression.py ${{ env.SSL }} - --clickhouse-binary-path ${{ env.clickhouse_path }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name="$GITHUB_JOB (${{ matrix.PART }}, ${{ matrix.SSL }})" job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" - --only "part ${{ matrix.PART }}/*" - ${{ env.args }} || EXITCODE=$?; - .github/add_link_to_logs.sh; - exit $EXITCODE - - name: Set Commit Status - if: always() - run: python3 .github/set_builds_status.py - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - JOB_OUTCOME: ${{ steps.run_suite.outcome }} - SUITE_NAME: "Regression ${{ inputs.arch }} Clickhouse Keeper ${{ matrix.SSL }} ${{ matrix.PART }}" - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - - name: Upload logs to regression results database - if: always() - timeout-minutes: 20 - run: .github/upload_results_to_database.sh 1 - - uses: actions/upload-artifact@v4 - if: always() - with: - name: ${{ env.SUITE }}-${{ matrix.PART }}-${{ inputs.arch }}-${{ matrix.SSL }}-artifacts - path: ${{ env.artifact_paths }} + uses: ./.github/workflows/regression-reusable-suite.yml + with: + ref: ${{ inputs.commit }} + workflow_config: ${{ inputs.workflow_config }} + suite_name: clickhouse_keeper + suite_executable: regression.py + output_format: new-fails + flags: --with-analyzer + timeout_minutes: ${{ inputs.timeout_minutes }} + runner_arch: ${{ inputs.arch }} + runner_type: ${{ inputs.runner_type }} + storage_path: /${{ matrix.SSL }} + part: ${{ matrix.PART }} + build_sha: ${{ inputs.build_sha }} + set_commit_status: true + job_name: clickhouse_keeper_${{ matrix.SSL }} + extra_args: ${{ matrix.SSL == 'ssl' && '--ssl' || '' }} --only "part ${{ matrix.PART }}/*" + secrets: inherit Iceberg: if: | @@ -513,72 +262,23 @@ jobs: fail-fast: false matrix: PART: [1, 2] - needs: [runner_labels_setup] - runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} - timeout-minutes: ${{ inputs.timeout_minutes }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v4 - with: - repository: Altinity/clickhouse-regression - ref: ${{ inputs.commit }} - - name: Set envs - run: | - if [ ${{ matrix.PART }} -eq 1 ]; then - echo ICEBERG_ONLY='"/iceberg/iceberg engine/rest catalog/*" "/iceberg/s3 table function/*" "/iceberg/icebergS3 table function/*" "/iceberg/iceberg cache"' >> "$GITHUB_ENV" - else - echo ICEBERG_ONLY='"/iceberg/iceberg engine/glue catalog/*" "/iceberg/iceberg table engine/*"' >> "$GITHUB_ENV" - fi - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{ runner.temp }}/reports_dir - SUITE=iceberg - PART=${{ matrix.PART }} - LOCALSTACK_AUTH_TOKEN=${{ secrets.LOCALSTACK_AUTH_TOKEN }} - EOF - - name: Setup - run: .github/setup.sh - - name: Get deb url - env: - S3_BASE_URL: https://altinity-build-artifacts.s3.amazonaws.com/ - PR_NUMBER: ${{ github.event.pull_request.number || 0 }} - run: | - mkdir -p $REPORTS_PATH - cat > $REPORTS_PATH/workflow_config.json << 'EOF' - ${{ inputs.workflow_config }} - EOF - - python3 .github/get-deb-url.py --github-env $GITHUB_ENV --workflow-config $REPORTS_PATH/workflow_config.json --s3-base-url $S3_BASE_URL --pr-number $PR_NUMBER --branch-name ${{ github.ref_name }} --commit-hash ${{ inputs.build_sha }} --binary - - - name: Run ${{ env.SUITE }} suite - id: run_suite - run: EXITCODE=0; - python3 - -u ${{ env.SUITE }}/regression.py - --clickhouse-binary-path ${{ env.clickhouse_path }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name="$GITHUB_JOB (${{ matrix.PART }})" job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" - --only ${{ env.ICEBERG_ONLY }} - ${{ env.args }} || EXITCODE=$?; - .github/add_link_to_logs.sh; - exit $EXITCODE - - name: Set Commit Status - if: always() - run: python3 .github/set_builds_status.py - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - JOB_OUTCOME: ${{ steps.run_suite.outcome }} - SUITE_NAME: "Regression ${{ inputs.arch }} ${{ env.SUITE }}-${{ matrix.PART }}" - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - - name: Upload logs to regression results database - if: always() - timeout-minutes: 20 - run: .github/upload_results_to_database.sh 1 - - uses: actions/upload-artifact@v4 - if: always() - with: - name: ${{ env.SUITE }}-${{ matrix.PART }}-${{ inputs.arch }}-artifacts - path: ${{ env.artifact_paths}} + uses: ./.github/workflows/regression-reusable-suite.yml + with: + ref: ${{ inputs.commit }} + workflow_config: ${{ inputs.workflow_config }} + suite_name: iceberg + suite_executable: regression.py + output_format: new-fails + flags: --with-analyzer + timeout_minutes: ${{ inputs.timeout_minutes }} + runner_arch: ${{ inputs.arch }} + runner_type: ${{ inputs.runner_type }} + part: ${{ matrix.PART }} + build_sha: ${{ inputs.build_sha }} + set_commit_status: true + job_name: iceberg + extra_args: --only ${{ matrix.PART == 1 && '"/iceberg/iceberg engine/rest catalog/*" "/iceberg/s3 table function/*" "/iceberg/icebergS3 table function/*" "/iceberg/iceberg cache"' || '"/iceberg/iceberg engine/glue catalog/*" "/iceberg/iceberg table engine/*"' }} + secrets: inherit LDAP: if: | fromJson(inputs.workflow_config).custom_data.ci_regression_jobs[0] == null || @@ -587,127 +287,41 @@ jobs: fail-fast: false matrix: SUITE: [authentication, external_user_directory, role_mapping] - needs: [runner_labels_setup] - runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} - timeout-minutes: ${{ inputs.timeout_minutes }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v4 - with: - repository: Altinity/clickhouse-regression - ref: ${{ inputs.commit }} - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{ runner.temp }}/reports_dir - SUITE=ldap/${{ matrix.SUITE }} - EOF - - name: Setup - run: .github/setup.sh - - name: Get deb url - env: - S3_BASE_URL: https://altinity-build-artifacts.s3.amazonaws.com/ - PR_NUMBER: ${{ github.event.pull_request.number || 0 }} - run: | - mkdir -p $REPORTS_PATH - cat > $REPORTS_PATH/workflow_config.json << 'EOF' - ${{ inputs.workflow_config }} - EOF - - python3 .github/get-deb-url.py --github-env $GITHUB_ENV --workflow-config $REPORTS_PATH/workflow_config.json --s3-base-url $S3_BASE_URL --pr-number $PR_NUMBER --branch-name ${{ github.ref_name }} --commit-hash ${{ inputs.build_sha }} --binary - - - name: Run ${{ env.SUITE }} suite - id: run_suite - run: EXITCODE=0; - python3 - -u ${{ env.SUITE }}/regression.py - --clickhouse-binary-path ${{ env.clickhouse_path }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name="$GITHUB_JOB (${{ matrix.SUITE }})" job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" - ${{ env.args }} || EXITCODE=$?; - .github/add_link_to_logs.sh; - exit $EXITCODE - - name: Set Commit Status - if: always() - run: python3 .github/set_builds_status.py - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - JOB_OUTCOME: ${{ steps.run_suite.outcome }} - SUITE_NAME: "Regression ${{ inputs.arch }} LDAP ${{ matrix.SUITE }}" - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - - name: Upload logs to regression results database - if: always() - timeout-minutes: 20 - run: .github/upload_results_to_database.sh 1 - - uses: actions/upload-artifact@v4 - if: always() - with: - name: ldap-${{ matrix.SUITE }}-${{ inputs.arch }}-artifacts - path: ${{ env.artifact_paths }} + uses: ./.github/workflows/regression-reusable-suite.yml + with: + ref: ${{ inputs.commit }} + workflow_config: ${{ inputs.workflow_config }} + suite_name: ldap/${{ matrix.SUITE }} + suite_executable: regression.py + output_format: new-fails + flags: --with-analyzer + timeout_minutes: ${{ inputs.timeout_minutes }} + runner_arch: ${{ inputs.arch }} + runner_type: ${{ inputs.runner_type }} + build_sha: ${{ inputs.build_sha }} + set_commit_status: true + job_name: ldap_${{ matrix.SUITE }} + secrets: inherit Parquet: if: | fromJson(inputs.workflow_config).custom_data.ci_regression_jobs[0] == null || contains(fromJson(inputs.workflow_config).custom_data.ci_regression_jobs, 'parquet') - needs: [runner_labels_setup] - runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} - timeout-minutes: ${{ inputs.timeout_minutes }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v4 - with: - repository: Altinity/clickhouse-regression - ref: ${{ inputs.commit }} - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{ runner.temp }}/reports_dir - SUITE=parquet - EOF - - name: Setup - run: .github/setup.sh - - name: Get deb url - env: - S3_BASE_URL: https://altinity-build-artifacts.s3.amazonaws.com/ - PR_NUMBER: ${{ github.event.pull_request.number || 0 }} - run: | - mkdir -p $REPORTS_PATH - cat > $REPORTS_PATH/workflow_config.json << 'EOF' - ${{ inputs.workflow_config }} - EOF - - python3 .github/get-deb-url.py --github-env $GITHUB_ENV --workflow-config $REPORTS_PATH/workflow_config.json --s3-base-url $S3_BASE_URL --pr-number $PR_NUMBER --branch-name ${{ github.ref_name }} --commit-hash ${{ inputs.build_sha }} --binary - - - name: Run ${{ env.SUITE }} suite - id: run_suite - run: EXITCODE=0; - python3 - -u ${{ env.SUITE }}/regression.py - --clickhouse-binary-path ${{ env.clickhouse_path }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name=$GITHUB_JOB job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" - ${{ env.args }} || EXITCODE=$?; - .github/add_link_to_logs.sh; - exit $EXITCODE - - name: Set Commit Status - if: always() - run: python3 .github/set_builds_status.py - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - JOB_OUTCOME: ${{ steps.run_suite.outcome }} - SUITE_NAME: "Regression ${{ inputs.arch }} Parquet" - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - - name: Upload logs to regression results database - if: always() - timeout-minutes: 20 - run: .github/upload_results_to_database.sh 1 - - uses: actions/upload-artifact@v4 - if: always() - with: - name: ${{ env.SUITE }}-${{ inputs.arch }}-artifacts - path: ${{ env.artifact_paths }} + uses: ./.github/workflows/regression-reusable-suite.yml + with: + ref: ${{ inputs.commit }} + workflow_config: ${{ inputs.workflow_config }} + suite_name: parquet + suite_executable: regression.py + output_format: new-fails + flags: --with-analyzer + timeout_minutes: ${{ inputs.timeout_minutes }} + runner_arch: ${{ inputs.arch }} + runner_type: ${{ inputs.runner_type }} + build_sha: ${{ inputs.build_sha }} + set_commit_status: true + job_name: parquet + secrets: inherit ParquetS3: if: | @@ -717,71 +331,23 @@ jobs: fail-fast: false matrix: STORAGE: [minio, aws_s3] - needs: [runner_labels_setup] - runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} - timeout-minutes: ${{ inputs.timeout_minutes }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v4 - with: - repository: Altinity/clickhouse-regression - ref: ${{ inputs.commit }} - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{ runner.temp }}/reports_dir - SUITE=parquet - STORAGE=${{ matrix.STORAGE}} - EOF - - name: Setup - run: .github/setup.sh - - name: Get deb url - env: - S3_BASE_URL: https://altinity-build-artifacts.s3.amazonaws.com/ - PR_NUMBER: ${{ github.event.pull_request.number || 0 }} - run: | - mkdir -p $REPORTS_PATH - cat > $REPORTS_PATH/workflow_config.json << 'EOF' - ${{ inputs.workflow_config }} - EOF - - python3 .github/get-deb-url.py --github-env $GITHUB_ENV --workflow-config $REPORTS_PATH/workflow_config.json --s3-base-url $S3_BASE_URL --pr-number $PR_NUMBER --branch-name ${{ github.ref_name }} --commit-hash ${{ inputs.build_sha }} --binary - - - name: Run ${{ env.SUITE }} suite - id: run_suite - run: EXITCODE=0; - python3 - -u ${{ env.SUITE }}/regression.py - --clickhouse-binary-path ${{ env.clickhouse_path }} - --storage ${{ matrix.STORAGE }} - --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} - --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} - --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} - --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name="$GITHUB_JOB (${{ matrix.STORAGE }})" job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" - ${{ env.args }} || EXITCODE=$?; - .github/add_link_to_logs.sh; - exit $EXITCODE - - name: Set Commit Status - if: always() - run: python3 .github/set_builds_status.py - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - JOB_OUTCOME: ${{ steps.run_suite.outcome }} - SUITE_NAME: "Regression ${{ inputs.arch }} Parquet ${{ matrix.STORAGE }}" - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - - name: Upload logs to regression results database - if: always() - timeout-minutes: 20 - run: .github/upload_results_to_database.sh 1 - - uses: actions/upload-artifact@v4 - if: always() - with: - name: ${{ env.SUITE }}-${{ env.STORAGE }}-${{ inputs.arch }}-artifacts - path: ${{ env.artifact_paths }} - + uses: ./.github/workflows/regression-reusable-suite.yml + with: + ref: ${{ inputs.commit }} + workflow_config: ${{ inputs.workflow_config }} + suite_name: parquet + suite_executable: regression.py + output_format: new-fails + flags: --with-analyzer + timeout_minutes: ${{ inputs.timeout_minutes }} + runner_arch: ${{ inputs.arch }} + runner_type: ${{ inputs.runner_type }} + storage_path: ${{ matrix.STORAGE }} + build_sha: ${{ inputs.build_sha }} + set_commit_status: true + job_name: parquet_${{ matrix.STORAGE }} + regression_args: --storage ${{ matrix.STORAGE }} --aws-s3-bucket {{AWS_BUCKET}} --aws-s3-region {{AWS_REGION}} --aws-s3-key-id {{AWS_KEY_ID}} --aws-s3-access-key {{AWS_ACCESS_KEY}} + secrets: inherit RBAC: if: | @@ -791,66 +357,23 @@ jobs: fail-fast: false matrix: PART: [1, 2, 3] - needs: [runner_labels_setup] - runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} - timeout-minutes: ${{ inputs.timeout_minutes }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v4 - with: - repository: Altinity/clickhouse-regression - ref: ${{ inputs.commit }} - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{ runner.temp }}/reports_dir - SUITE=rbac - PART=${{ matrix.PART }} - EOF - - name: Setup - run: .github/setup.sh - - name: Get deb url - env: - S3_BASE_URL: https://altinity-build-artifacts.s3.amazonaws.com/ - PR_NUMBER: ${{ github.event.pull_request.number || 0 }} - run: | - mkdir -p $REPORTS_PATH - cat > $REPORTS_PATH/workflow_config.json << 'EOF' - ${{ inputs.workflow_config }} - EOF - - python3 .github/get-deb-url.py --github-env $GITHUB_ENV --workflow-config $REPORTS_PATH/workflow_config.json --s3-base-url $S3_BASE_URL --pr-number $PR_NUMBER --branch-name ${{ github.ref_name }} --commit-hash ${{ inputs.build_sha }} --binary - - - name: Run ${{ env.SUITE }} suite - id: run_suite - run: EXITCODE=0; - python3 - -u ${{ env.SUITE }}/regression.py - --clickhouse-binary-path ${{ env.clickhouse_path }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name="$GITHUB_JOB (${{ matrix.PART }})" job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" - --only "/rbac/part ${{ matrix.PART }}/*" - ${{ env.args }} || EXITCODE=$?; - .github/add_link_to_logs.sh; - exit $EXITCODE - - name: Set Commit Status - if: always() - run: python3 .github/set_builds_status.py - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - JOB_OUTCOME: ${{ steps.run_suite.outcome }} - SUITE_NAME: "Regression ${{ inputs.arch }} RBAC ${{ matrix.PART }}" - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - - name: Upload logs to regression results database - if: always() - timeout-minutes: 20 - run: .github/upload_results_to_database.sh 1 - - uses: actions/upload-artifact@v4 - if: always() - with: - name: ${{ env.SUITE }}-${{ matrix.PART }}-${{ inputs.arch }}-artifacts - path: ${{ env.artifact_paths}} + uses: ./.github/workflows/regression-reusable-suite.yml + with: + ref: ${{ inputs.commit }} + workflow_config: ${{ inputs.workflow_config }} + suite_name: rbac + suite_executable: regression.py + output_format: new-fails + flags: --with-analyzer + timeout_minutes: ${{ inputs.timeout_minutes }} + runner_arch: ${{ inputs.arch }} + runner_type: ${{ inputs.runner_type }} + part: ${{ matrix.PART }} + build_sha: ${{ inputs.build_sha }} + set_commit_status: true + job_name: rbac + extra_args: --only "/rbac/part ${{ matrix.PART }}/*" + secrets: inherit SSLServer: if: | fromJson(inputs.workflow_config).custom_data.ci_regression_jobs[0] == null || @@ -859,66 +382,23 @@ jobs: fail-fast: false matrix: PART: [1, 2, 3] - needs: [runner_labels_setup] - runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} - timeout-minutes: ${{ inputs.timeout_minutes }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v4 - with: - repository: Altinity/clickhouse-regression - ref: ${{ inputs.commit }} - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{ runner.temp }}/reports_dir - SUITE=ssl_server - PART=${{ matrix.PART }} - EOF - - name: Setup - run: .github/setup.sh - - name: Get deb url - env: - S3_BASE_URL: https://altinity-build-artifacts.s3.amazonaws.com/ - PR_NUMBER: ${{ github.event.pull_request.number || 0 }} - run: | - mkdir -p $REPORTS_PATH - cat > $REPORTS_PATH/workflow_config.json << 'EOF' - ${{ inputs.workflow_config }} - EOF - - python3 .github/get-deb-url.py --github-env $GITHUB_ENV --workflow-config $REPORTS_PATH/workflow_config.json --s3-base-url $S3_BASE_URL --pr-number $PR_NUMBER --branch-name ${{ github.ref_name }} --commit-hash ${{ inputs.build_sha }} --binary - - - name: Run ${{ env.SUITE }} suite - id: run_suite - run: EXITCODE=0; - python3 - -u ${{ env.SUITE }}/regression.py - --clickhouse-binary-path ${{ env.clickhouse_path }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name="$GITHUB_JOB (${{ matrix.PART }})" job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" - --only "part ${{ matrix.PART }}/*" - ${{ env.args }} || EXITCODE=$?; - .github/add_link_to_logs.sh; - exit $EXITCODE - - name: Set Commit Status - if: always() - run: python3 .github/set_builds_status.py - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - JOB_OUTCOME: ${{ steps.run_suite.outcome }} - SUITE_NAME: "Regression ${{ inputs.arch }} ${{ env.SUITE }}-${{ matrix.PART }}" - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - - name: Upload logs to regression results database - if: always() - timeout-minutes: 20 - run: .github/upload_results_to_database.sh 1 - - uses: actions/upload-artifact@v4 - if: always() - with: - name: ${{ env.SUITE }}-${{ matrix.PART }}-${{ inputs.arch }}-artifacts - path: ${{ env.artifact_paths}} + uses: ./.github/workflows/regression-reusable-suite.yml + with: + ref: ${{ inputs.commit }} + workflow_config: ${{ inputs.workflow_config }} + suite_name: ssl_server + suite_executable: regression.py + output_format: new-fails + flags: --with-analyzer + timeout_minutes: ${{ inputs.timeout_minutes }} + runner_arch: ${{ inputs.arch }} + runner_type: ${{ inputs.runner_type }} + part: ${{ matrix.PART }} + build_sha: ${{ inputs.build_sha }} + set_commit_status: true + job_name: ssl_server + extra_args: --only "part ${{ matrix.PART }}/*" + secrets: inherit S3: if: | @@ -932,78 +412,25 @@ jobs: include: - STORAGE: minio PART: 3 - needs: [runner_labels_setup] - runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} - timeout-minutes: ${{ inputs.timeout_minutes }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v4 - with: - repository: Altinity/clickhouse-regression - ref: ${{ inputs.commit }} - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{ runner.temp }}/reports_dir - SUITE=s3 - PART=${{ matrix.PART }} - STORAGE=/${{ matrix.STORAGE }} - EOF - - name: Setup - run: .github/setup.sh - - name: Get deb url - env: - S3_BASE_URL: https://altinity-build-artifacts.s3.amazonaws.com/ - PR_NUMBER: ${{ github.event.pull_request.number || 0 }} - run: | - mkdir -p $REPORTS_PATH - cat > $REPORTS_PATH/workflow_config.json << 'EOF' - ${{ inputs.workflow_config }} - EOF - - python3 .github/get-deb-url.py --github-env $GITHUB_ENV --workflow-config $REPORTS_PATH/workflow_config.json --s3-base-url $S3_BASE_URL --pr-number $PR_NUMBER --branch-name ${{ github.ref_name }} --commit-hash ${{ inputs.build_sha }} --binary - - - name: Run ${{ env.SUITE }} suite - id: run_suite - run: EXITCODE=0; - python3 - -u ${{ env.SUITE }}/regression.py - --clickhouse-binary-path ${{ env.clickhouse_path }} - --storage ${{ matrix.STORAGE }} - --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} - --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} - --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} - --aws-s3-bucket ${{ secrets.REGRESSION_AWS_S3_BUCKET }} - --aws-s3-region ${{ secrets.REGRESSION_AWS_S3_REGION }} - --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} - --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} - --azure-account-name ${{ secrets.AZURE_ACCOUNT_NAME }} - --azure-storage-key ${{ secrets.AZURE_STORAGE_KEY }} - --azure-container ${{ secrets.AZURE_CONTAINER_NAME }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name="$GITHUB_JOB (${{ matrix.STORAGE }}-${{ matrix.PART }})" job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" - --only ":/try*" ":/part ${{ matrix.PART }}/*" - ${{ env.args }} || EXITCODE=$?; - .github/add_link_to_logs.sh; - exit $EXITCODE - - name: Set Commit Status - if: always() - run: python3 .github/set_builds_status.py - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - JOB_OUTCOME: ${{ steps.run_suite.outcome }} - SUITE_NAME: "Regression ${{ inputs.arch }} S3 ${{ matrix.STORAGE }}-${{ matrix.PART }}" - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - - name: Upload logs to regression results database - if: always() - timeout-minutes: 20 - run: .github/upload_results_to_database.sh 1 - - uses: actions/upload-artifact@v4 - if: always() - with: - name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-${{ matrix.PART }}-${{ inputs.arch }}-artifacts - path: ${{ env.artifact_paths}} + uses: ./.github/workflows/regression-reusable-suite.yml + with: + ref: ${{ inputs.commit }} + workflow_config: ${{ inputs.workflow_config }} + suite_name: s3 + suite_executable: regression.py + output_format: new-fails + flags: --with-analyzer + timeout_minutes: ${{ inputs.timeout_minutes }} + runner_arch: ${{ inputs.arch }} + runner_type: ${{ inputs.runner_type }} + storage_path: /${{ matrix.STORAGE }} + part: ${{ matrix.PART }} + build_sha: ${{ inputs.build_sha }} + set_commit_status: true + job_name: s3_${{ matrix.STORAGE }} + regression_args: --storage ${{ matrix.STORAGE }} --gcs-uri {{GCS_URI}} --gcs-key-id {{GCS_KEY_ID}} --gcs-key-secret {{GCS_KEY_SECRET}} --aws-s3-bucket {{AWS_BUCKET}} --aws-s3-region {{AWS_REGION}} --aws-s3-key-id {{AWS_KEY_ID}} --aws-s3-access-key {{AWS_ACCESS_KEY}} --azure-account-name {{AZURE_ACCOUNT_NAME}} --azure-storage-key {{AZURE_STORAGE_KEY}} --azure-container {{AZURE_CONTAINER_NAME}} + extra_args: --only ":/try*" ":/part ${{ matrix.PART }}/*" + secrets: inherit TieredStorage: if: | @@ -1013,69 +440,21 @@ jobs: fail-fast: false matrix: STORAGE: [local, minio, s3amazon, s3gcs] - needs: [runner_labels_setup] - runs-on: ${{ fromJson(needs.runner_labels_setup.outputs.runner_labels) }} - timeout-minutes: ${{ inputs.timeout_minutes }} - steps: - - name: Checkout regression repo - uses: actions/checkout@v4 - with: - repository: Altinity/clickhouse-regression - ref: ${{ inputs.commit }} - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - REPORTS_PATH=${{ runner.temp }}/reports_dir - SUITE=tiered_storage - STORAGE=/${{ matrix.STORAGE }} - EOF - - name: Setup - run: .github/setup.sh - - name: Get deb url - env: - S3_BASE_URL: https://altinity-build-artifacts.s3.amazonaws.com/ - PR_NUMBER: ${{ github.event.pull_request.number || 0 }} - run: | - mkdir -p $REPORTS_PATH - cat > $REPORTS_PATH/workflow_config.json << 'EOF' - ${{ inputs.workflow_config }} - EOF - - python3 .github/get-deb-url.py --github-env $GITHUB_ENV --workflow-config $REPORTS_PATH/workflow_config.json --s3-base-url $S3_BASE_URL --pr-number $PR_NUMBER --branch-name ${{ github.ref_name }} --commit-hash ${{ inputs.build_sha }} --binary - - - name: Run ${{ env.SUITE }} suite - id: run_suite - run: EXITCODE=0; - python3 - -u ${{ env.SUITE }}/regression.py - --clickhouse-binary-path ${{ env.clickhouse_path }} - --aws-s3-access-key ${{ secrets.REGRESSION_AWS_S3_SECRET_ACCESS_KEY }} - --aws-s3-key-id ${{ secrets.REGRESSION_AWS_S3_KEY_ID }} - --aws-s3-uri https://s3.${{ secrets.REGRESSION_AWS_S3_REGION}}.amazonaws.com/${{ secrets.REGRESSION_AWS_S3_BUCKET }}/data/ - --gcs-key-id ${{ secrets.REGRESSION_GCS_KEY_ID }} - --gcs-key-secret ${{ secrets.REGRESSION_GCS_KEY_SECRET }} - --gcs-uri ${{ secrets.REGRESSION_GCS_URI }} - ${{ matrix.STORAGE != 'local' && format('--with-{0}', matrix.STORAGE) || '' }} - --attr project="$GITHUB_REPOSITORY" project.id="$GITHUB_REPOSITORY_ID" package="${{ env.clickhouse_path }}" version="${{ env.version }}" user.name="$GITHUB_ACTOR" repository="https://github.com/Altinity/clickhouse-regression" commit.hash="$(git rev-parse HEAD)" job.name="$GITHUB_JOB (${{ matrix.STORAGE }})" job.retry=$GITHUB_RUN_ATTEMPT job.url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" arch="$(uname -i)" - ${{ env.args }} || EXITCODE=$?; - .github/add_link_to_logs.sh; - exit $EXITCODE - - name: Set Commit Status - if: always() - run: python3 .github/set_builds_status.py - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - JOB_OUTCOME: ${{ steps.run_suite.outcome }} - SUITE_NAME: "Regression ${{ inputs.arch }} Tiered Storage ${{ matrix.STORAGE }}" - - name: Create and upload logs - if: always() - run: .github/create_and_upload_logs.sh 1 - - name: Upload logs to regression results database - if: always() - timeout-minutes: 20 - run: .github/upload_results_to_database.sh 1 - - uses: actions/upload-artifact@v4 - if: always() - with: - name: ${{ env.SUITE }}-${{ matrix.STORAGE }}-${{ inputs.arch }}-artifacts - path: ${{ env.artifact_paths}} + uses: ./.github/workflows/regression-reusable-suite.yml + with: + ref: ${{ inputs.commit }} + workflow_config: ${{ inputs.workflow_config }} + suite_name: tiered_storage + suite_executable: regression.py + output_format: new-fails + flags: --with-analyzer + timeout_minutes: ${{ inputs.timeout_minutes }} + runner_arch: ${{ inputs.arch }} + runner_type: ${{ inputs.runner_type }} + storage_path: /${{ matrix.STORAGE }} + build_sha: ${{ inputs.build_sha }} + set_commit_status: true + job_name: tiered_storage_${{ matrix.STORAGE }} + regression_args: --aws-s3-access-key {{AWS_ACCESS_KEY}} --aws-s3-key-id {{AWS_KEY_ID}} --aws-s3-uri https://s3.{{AWS_REGION}}.amazonaws.com/{{AWS_BUCKET}}/data/ --gcs-key-id {{GCS_KEY_ID}} --gcs-key-secret {{GCS_KEY_SECRET}} --gcs-uri {{GCS_URI}} + extra_args: ${{ matrix.STORAGE != 'local' && format('--with-{0}', matrix.STORAGE) || '' }} + secrets: inherit diff --git a/ci/praktika/yaml_additional_templates.py b/ci/praktika/yaml_additional_templates.py index f92d704fd0f1..6e4ac4eb70db 100644 --- a/ci/praktika/yaml_additional_templates.py +++ b/ci/praktika/yaml_additional_templates.py @@ -67,9 +67,9 @@ class AltinityWorkflowTemplates: uses: ./.github/workflows/regression.yml secrets: inherit with: - runner_type: altinity-on-demand, altinity-regression-tester + runner_type: altinity-regression-tester commit: {REGRESSION_HASH} - arch: release + arch: x86 build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} timeout_minutes: 300 workflow_config: ${{ needs.config_workflow.outputs.data }} @@ -79,7 +79,7 @@ class AltinityWorkflowTemplates: uses: ./.github/workflows/regression.yml secrets: inherit with: - runner_type: altinity-on-demand, altinity-regression-tester-aarch64 + runner_type: altinity-regression-tester-aarch64 commit: {REGRESSION_HASH} arch: aarch64 build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}