fix(ci): Fix Github action checks (#1780)

The checks are failing for 2 reasons:
1. GitHub actions dropped python3.7 support on the latest hosted runners.
   actions/setup-python#544 (comment)
2. New release of Tox was validation the python version in the environment name
   and the trailing framework version being used in the environment name was
   being treated as a python version and validated causing an issue.

Further changes:
* Added one GitHub job to check if all tests have passed. Makes it easier to configure required checks in GitHub.
* Pinning Tox to <4

Co-authored-by: Anton Pirker <anton.pirker@sentry.io>

Author: Zylphrex

.github/workflows/test-common.yml

@@ -24,7 +24,11 @@ jobs:
     continue-on-error: true
     strategy:
       matrix:
-        os: [ubuntu-latest]
+        # python3.6 reached EOL and is no longer being supported on
+        # new versions of hosted runners on Github Actions
+        # ubuntu-20.04 is the last version that supported python3.6
+        # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877
+        os: [ubuntu-20.04]
         python-version: ["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10"]
     services:
       postgres:
@@ -51,9 +55,6 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Setup Test Env
-        env:
-          PGHOST: localhost
-          PGPASSWORD: sentry
         run: |
           pip install codecov tox
 
@@ -69,4 +70,4 @@ jobs:
           ./scripts/runtox.sh "py${{ matrix.python-version }}$" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch --ignore=tests/integrations
           coverage combine .coverage*
           coverage xml -i
-          codecov --file coverage.xml
+          codecov --file coverage.xml

.github/workflows/test-integration-aiohttp.yml

@@ -27,12 +27,16 @@ jobs:
     name: aiohttp, python ${{ matrix.python-version }}, ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     timeout-minutes: 45
-    continue-on-error: true
 
     strategy:
+      fail-fast: false
       matrix:
         python-version: ["3.7","3.8","3.9","3.10"]
-        os: [ubuntu-latest]
+        # python3.6 reached EOL and is no longer being supported on
+        # new versions of hosted runners on Github Actions
+        # ubuntu-20.04 is the last version that supported python3.6
+        # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877
+        os: [ubuntu-20.04]
 
     steps:
       - uses: actions/checkout@v3
@@ -41,11 +45,8 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Setup Test Env
-        env:
-          PGHOST: localhost
-          PGPASSWORD: sentry
         run: |
-          pip install codecov tox
+          pip install codecov "tox>=3,<4"
 
       - name: Test aiohttp
         env:
@@ -60,3 +61,15 @@ jobs:
           coverage combine .coverage*
           coverage xml -i
           codecov --file coverage.xml
+
+  check_required_tests:
+    name: All aiohttp tests passed or skipped
+    needs: test
+    # Always run this, even if a dependent job failed
+    if: always()
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Check for failures
+        if: contains(needs.test.result, 'failure')
+        run: |
+          echo "One of the dependent jobs have failed. You may need to re-run it." && exit 1

.github/workflows/test-integration-asgi.yml

@@ -27,12 +27,16 @@ jobs:
     name: asgi, python ${{ matrix.python-version }}, ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     timeout-minutes: 45
-    continue-on-error: true
 
     strategy:
+      fail-fast: false
       matrix:
         python-version: ["3.7","3.8","3.9","3.10"]
-        os: [ubuntu-latest]
+        # python3.6 reached EOL and is no longer being supported on
+        # new versions of hosted runners on Github Actions
+        # ubuntu-20.04 is the last version that supported python3.6
+        # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877
+        os: [ubuntu-20.04]
 
     steps:
       - uses: actions/checkout@v3
@@ -41,11 +45,8 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Setup Test Env
-        env:
-          PGHOST: localhost
-          PGPASSWORD: sentry
         run: |
-          pip install codecov tox
+          pip install codecov "tox>=3,<4"
 
       - name: Test asgi
         env:
@@ -60,3 +61,15 @@ jobs:
           coverage combine .coverage*
           coverage xml -i
           codecov --file coverage.xml
+
+  check_required_tests:
+    name: All asgi tests passed or skipped
+    needs: test
+    # Always run this, even if a dependent job failed
+    if: always()
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Check for failures
+        if: contains(needs.test.result, 'failure')
+        run: |
+          echo "One of the dependent jobs have failed. You may need to re-run it." && exit 1

.github/workflows/test-integration-aws_lambda.yml

@@ -27,12 +27,16 @@ jobs:
     name: aws_lambda, python ${{ matrix.python-version }}, ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     timeout-minutes: 45
-    continue-on-error: true
 
     strategy:
+      fail-fast: false
       matrix:
         python-version: ["3.7"]
-        os: [ubuntu-latest]
+        # python3.6 reached EOL and is no longer being supported on
+        # new versions of hosted runners on Github Actions
+        # ubuntu-20.04 is the last version that supported python3.6
+        # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877
+        os: [ubuntu-20.04]
 
     steps:
       - uses: actions/checkout@v3
@@ -41,11 +45,8 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Setup Test Env
-        env:
-          PGHOST: localhost
-          PGPASSWORD: sentry
         run: |
-          pip install codecov tox
+          pip install codecov "tox>=3,<4"
 
       - name: Test aws_lambda
         env:
@@ -60,3 +61,15 @@ jobs:
           coverage combine .coverage*
           coverage xml -i
           codecov --file coverage.xml
+
+  check_required_tests:
+    name: All aws_lambda tests passed or skipped
+    needs: test
+    # Always run this, even if a dependent job failed
+    if: always()
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Check for failures
+        if: contains(needs.test.result, 'failure')
+        run: |
+          echo "One of the dependent jobs have failed. You may need to re-run it." && exit 1

.github/workflows/test-integration-beam.yml

@@ -27,12 +27,16 @@ jobs:
     name: beam, python ${{ matrix.python-version }}, ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     timeout-minutes: 45
-    continue-on-error: true
 
     strategy:
+      fail-fast: false
       matrix:
         python-version: ["3.7"]
-        os: [ubuntu-latest]
+        # python3.6 reached EOL and is no longer being supported on
+        # new versions of hosted runners on Github Actions
+        # ubuntu-20.04 is the last version that supported python3.6
+        # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877
+        os: [ubuntu-20.04]
 
     steps:
       - uses: actions/checkout@v3
@@ -41,11 +45,8 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Setup Test Env
-        env:
-          PGHOST: localhost
-          PGPASSWORD: sentry
         run: |
-          pip install codecov tox
+          pip install codecov "tox>=3,<4"
 
       - name: Test beam
         env:
@@ -60,3 +61,15 @@ jobs:
           coverage combine .coverage*
           coverage xml -i
           codecov --file coverage.xml
+
+  check_required_tests:
+    name: All beam tests passed or skipped
+    needs: test
+    # Always run this, even if a dependent job failed
+    if: always()
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Check for failures
+        if: contains(needs.test.result, 'failure')
+        run: |
+          echo "One of the dependent jobs have failed. You may need to re-run it." && exit 1

.github/workflows/test-integration-boto3.yml

@@ -27,12 +27,16 @@ jobs:
     name: boto3, python ${{ matrix.python-version }}, ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     timeout-minutes: 45
-    continue-on-error: true
 
     strategy:
+      fail-fast: false
       matrix:
         python-version: ["2.7","3.6","3.7","3.8"]
-        os: [ubuntu-latest]
+        # python3.6 reached EOL and is no longer being supported on
+        # new versions of hosted runners on Github Actions
+        # ubuntu-20.04 is the last version that supported python3.6
+        # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877
+        os: [ubuntu-20.04]
 
     steps:
       - uses: actions/checkout@v3
@@ -41,11 +45,8 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Setup Test Env
-        env:
-          PGHOST: localhost
-          PGPASSWORD: sentry
         run: |
-          pip install codecov tox
+          pip install codecov "tox>=3,<4"
 
       - name: Test boto3
         env:
@@ -60,3 +61,15 @@ jobs:
           coverage combine .coverage*
           coverage xml -i
           codecov --file coverage.xml
+
+  check_required_tests:
+    name: All boto3 tests passed or skipped
+    needs: test
+    # Always run this, even if a dependent job failed
+    if: always()
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Check for failures
+        if: contains(needs.test.result, 'failure')
+        run: |
+          echo "One of the dependent jobs have failed. You may need to re-run it." && exit 1

.github/workflows/test-integration-bottle.yml

@@ -27,12 +27,16 @@ jobs:
     name: bottle, python ${{ matrix.python-version }}, ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     timeout-minutes: 45
-    continue-on-error: true
 
     strategy:
+      fail-fast: false
       matrix:
         python-version: ["2.7","3.5","3.6","3.7","3.8","3.9","3.10"]
-        os: [ubuntu-latest]
+        # python3.6 reached EOL and is no longer being supported on
+        # new versions of hosted runners on Github Actions
+        # ubuntu-20.04 is the last version that supported python3.6
+        # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877
+        os: [ubuntu-20.04]
 
     steps:
       - uses: actions/checkout@v3
@@ -41,11 +45,8 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Setup Test Env
-        env:
-          PGHOST: localhost
-          PGPASSWORD: sentry
         run: |
-          pip install codecov tox
+          pip install codecov "tox>=3,<4"
 
       - name: Test bottle
         env:
@@ -60,3 +61,15 @@ jobs:
           coverage combine .coverage*
           coverage xml -i
           codecov --file coverage.xml
+
+  check_required_tests:
+    name: All bottle tests passed or skipped
+    needs: test
+    # Always run this, even if a dependent job failed
+    if: always()
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Check for failures
+        if: contains(needs.test.result, 'failure')
+        run: |
+          echo "One of the dependent jobs have failed. You may need to re-run it." && exit 1

.github/workflows/test-integration-celery.yml

@@ -27,12 +27,16 @@ jobs:
     name: celery, python ${{ matrix.python-version }}, ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     timeout-minutes: 45
-    continue-on-error: true
 
     strategy:
+      fail-fast: false
       matrix:
         python-version: ["2.7","3.5","3.6","3.7","3.8","3.9","3.10"]
-        os: [ubuntu-latest]
+        # python3.6 reached EOL and is no longer being supported on
+        # new versions of hosted runners on Github Actions
+        # ubuntu-20.04 is the last version that supported python3.6
+        # see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877
+        os: [ubuntu-20.04]
 
     steps:
       - uses: actions/checkout@v3
@@ -41,11 +45,8 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Setup Test Env
-        env:
-          PGHOST: localhost
-          PGPASSWORD: sentry
         run: |
-          pip install codecov tox
+          pip install codecov "tox>=3,<4"
 
       - name: Test celery
         env:
@@ -60,3 +61,15 @@ jobs:
           coverage combine .coverage*
           coverage xml -i
           codecov --file coverage.xml
+
+  check_required_tests:
+    name: All celery tests passed or skipped
+    needs: test
+    # Always run this, even if a dependent job failed
+    if: always()
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Check for failures
+        if: contains(needs.test.result, 'failure')
+        run: |
+          echo "One of the dependent jobs have failed. You may need to re-run it." && exit 1