From 035d3cf627abd94d55fa8d6d131986b914281ab0 Mon Sep 17 00:00:00 2001 From: Emmanuel Olowosulu Date: Fri, 31 Jul 2026 13:38:26 +0000 Subject: [PATCH] fix(ci): limit matrix parallelism for OIDC credential fetches With 600+ matrix jobs hitting STS AssumeRoleWithWebIdentity concurrently against the same OIDC role, contention on the endpoint may cause throttled requests. The credential action fetches a short-lived JWT (~5 min + 5 min clock-skew grace) and retries AssumeRole with that same token. If retries are delayed long enough, the token can expire before a successful call. Setting max-parallel: 100 limits concurrent STS calls per matrix. The value is not calibrated against measured throttle data; it is a conservative cap based on the reasoning above. The platform concurrency limit for the account may already cap below 100. --- .github/workflows/ec2-integration-test.yml | 4 +++ .github/workflows/test-artifacts.yml | 36 ++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/.github/workflows/ec2-integration-test.yml b/.github/workflows/ec2-integration-test.yml index 8d34cb935b..5df4f24b1c 100644 --- a/.github/workflows/ec2-integration-test.yml +++ b/.github/workflows/ec2-integration-test.yml @@ -56,6 +56,10 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false + # Cap concurrent jobs per matrix to limit simultaneous STS AssumeRoleWithWebIdentity calls. + # 100 is a conservative value, not calibrated against throttle data; + # account concurrency limits may cap lower. + max-parallel: 100 matrix: arrays: ${{ fromJson(inputs.test_props) }} permissions: diff --git a/.github/workflows/test-artifacts.yml b/.github/workflows/test-artifacts.yml index 4c20eca962..e3c50363ce 100644 --- a/.github/workflows/test-artifacts.yml +++ b/.github/workflows/test-artifacts.yml @@ -449,6 +449,9 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false + # Cap concurrent jobs per matrix to limit simultaneous STS AssumeRoleWithWebIdentity calls. + # 100 is a conservative value, not calibrated against throttle data; account concurrency limits may cap lower. + max-parallel: 100 matrix: arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_gpu_matrix) }} permissions: @@ -805,6 +808,9 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false + # Cap concurrent jobs per matrix to limit simultaneous STS AssumeRoleWithWebIdentity calls. + # 100 is a conservative value, not calibrated against throttle data; account concurrency limits may cap lower. + max-parallel: 100 matrix: arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_windows_matrix) }} permissions: @@ -915,6 +921,9 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false + # Cap concurrent jobs per matrix to limit simultaneous STS AssumeRoleWithWebIdentity calls. + # 100 is a conservative value, not calibrated against throttle data; account concurrency limits may cap lower. + max-parallel: 100 matrix: arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_mac_matrix) }} permissions: @@ -1070,6 +1079,9 @@ jobs: needs: [ GenerateTestMatrix, OutputEnvVariables ] strategy: fail-fast: false + # Cap concurrent jobs per matrix to limit simultaneous STS AssumeRoleWithWebIdentity calls. + # 100 is a conservative value, not calibrated against throttle data; account concurrency limits may cap lower. + max-parallel: 100 matrix: arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ecs_ec2_launch_daemon_matrix) }} permissions: @@ -1167,6 +1179,9 @@ jobs: needs: [ GenerateTestMatrix, OutputEnvVariables ] strategy: fail-fast: false + # Cap concurrent jobs per matrix to limit simultaneous STS AssumeRoleWithWebIdentity calls. + # 100 is a conservative value, not calibrated against throttle data; account concurrency limits may cap lower. + max-parallel: 100 matrix: arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ecs_fargate_matrix) }} permissions: @@ -1258,6 +1273,9 @@ jobs: needs: [ GenerateTestMatrix, OutputEnvVariables ] strategy: fail-fast: false + # Cap concurrent jobs per matrix to limit simultaneous STS AssumeRoleWithWebIdentity calls. + # 100 is a conservative value, not calibrated against throttle data; account concurrency limits may cap lower. + max-parallel: 100 matrix: arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.eks_daemon_matrix) }} permissions: @@ -1375,6 +1393,9 @@ jobs: needs: [ GenerateTestMatrix, OutputEnvVariables ] strategy: fail-fast: false + # Cap concurrent jobs per matrix to limit simultaneous STS AssumeRoleWithWebIdentity calls. + # 100 is a conservative value, not calibrated against throttle data; account concurrency limits may cap lower. + max-parallel: 100 matrix: arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.eks_deployment_matrix) }} permissions: @@ -1467,6 +1488,9 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false + # Cap concurrent jobs per matrix to limit simultaneous STS AssumeRoleWithWebIdentity calls. + # 100 is a conservative value, not calibrated against throttle data; account concurrency limits may cap lower. + max-parallel: 100 matrix: arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_performance_matrix) }} permissions: @@ -1556,6 +1580,9 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false + # Cap concurrent jobs per matrix to limit simultaneous STS AssumeRoleWithWebIdentity calls. + # 100 is a conservative value, not calibrated against throttle data; account concurrency limits may cap lower. + max-parallel: 100 matrix: arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_windows_performance_matrix) }} permissions: @@ -1645,6 +1672,9 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false + # Cap concurrent jobs per matrix to limit simultaneous STS AssumeRoleWithWebIdentity calls. + # 100 is a conservative value, not calibrated against throttle data; account concurrency limits may cap lower. + max-parallel: 100 matrix: arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_stress_matrix) }} permissions: @@ -1738,6 +1768,9 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false + # Cap concurrent jobs per matrix to limit simultaneous STS AssumeRoleWithWebIdentity calls. + # 100 is a conservative value, not calibrated against throttle data; account concurrency limits may cap lower. + max-parallel: 100 matrix: arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ec2_windows_stress_matrix) }} permissions: @@ -1833,6 +1866,9 @@ jobs: runs-on: ubuntu-latest strategy: fail-fast: false + # Cap concurrent jobs per matrix to limit simultaneous STS AssumeRoleWithWebIdentity calls. + # 100 is a conservative value, not calibrated against throttle data; account concurrency limits may cap lower. + max-parallel: 100 matrix: arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.eks_addon_matrix) }} permissions: