Auto-retry tests on failure (#2134)

Author: sentrivana

.github/workflows/test-common.yml

@@ -26,7 +26,7 @@ jobs:
   test:
     name: common, python ${{ matrix.python-version }}, ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 45
+    timeout-minutes: 30
 
     strategy:
       fail-fast: false
@@ -49,16 +49,20 @@ jobs:
           pip install coverage "tox>=3,<4"
 
       - name: Test common
-        timeout-minutes: 45
-        shell: bash
-        run: |
-          set -x # print commands that are executed
-          coverage erase
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 15
+          max_attempts: 2
+          retry_wait_seconds: 5
+          shell: bash
+          command: |
+            set -x # print commands that are executed
+            coverage erase
 
-          # Run tests
-          ./scripts/runtox.sh "py${{ matrix.python-version }}-common" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch
-          coverage combine .coverage*
-          coverage xml -i
+            # Run tests
+            ./scripts/runtox.sh "py${{ matrix.python-version }}-common" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch &&
+            coverage combine .coverage* &&
+            coverage xml -i
 
       - uses: codecov/codecov-action@v3
         with:

.github/workflows/test-integration-aiohttp.yml

@@ -26,7 +26,7 @@ jobs:
   test:
     name: aiohttp, python ${{ matrix.python-version }}, ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 45
+    timeout-minutes: 30
 
     strategy:
       fail-fast: false
@@ -49,16 +49,20 @@ jobs:
           pip install coverage "tox>=3,<4"
 
       - name: Test aiohttp
-        timeout-minutes: 45
-        shell: bash
-        run: |
-          set -x # print commands that are executed
-          coverage erase
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 15
+          max_attempts: 2
+          retry_wait_seconds: 5
+          shell: bash
+          command: |
+            set -x # print commands that are executed
+            coverage erase
 
-          # Run tests
-          ./scripts/runtox.sh "py${{ matrix.python-version }}-aiohttp" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch
-          coverage combine .coverage*
-          coverage xml -i
+            # Run tests
+            ./scripts/runtox.sh "py${{ matrix.python-version }}-aiohttp" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch &&
+            coverage combine .coverage* &&
+            coverage xml -i
 
       - uses: codecov/codecov-action@v3
         with:

.github/workflows/test-integration-arq.yml

@@ -26,7 +26,7 @@ jobs:
   test:
     name: arq, python ${{ matrix.python-version }}, ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 45
+    timeout-minutes: 30
 
     strategy:
       fail-fast: false
@@ -49,16 +49,20 @@ jobs:
           pip install coverage "tox>=3,<4"
 
       - name: Test arq
-        timeout-minutes: 45
-        shell: bash
-        run: |
-          set -x # print commands that are executed
-          coverage erase
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 15
+          max_attempts: 2
+          retry_wait_seconds: 5
+          shell: bash
+          command: |
+            set -x # print commands that are executed
+            coverage erase
 
-          # Run tests
-          ./scripts/runtox.sh "py${{ matrix.python-version }}-arq" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch
-          coverage combine .coverage*
-          coverage xml -i
+            # Run tests
+            ./scripts/runtox.sh "py${{ matrix.python-version }}-arq" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch &&
+            coverage combine .coverage* &&
+            coverage xml -i
 
       - uses: codecov/codecov-action@v3
         with:

.github/workflows/test-integration-asgi.yml

@@ -26,7 +26,7 @@ jobs:
   test:
     name: asgi, python ${{ matrix.python-version }}, ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 45
+    timeout-minutes: 30
 
     strategy:
       fail-fast: false
@@ -49,16 +49,20 @@ jobs:
           pip install coverage "tox>=3,<4"
 
       - name: Test asgi
-        timeout-minutes: 45
-        shell: bash
-        run: |
-          set -x # print commands that are executed
-          coverage erase
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 15
+          max_attempts: 2
+          retry_wait_seconds: 5
+          shell: bash
+          command: |
+            set -x # print commands that are executed
+            coverage erase
 
-          # Run tests
-          ./scripts/runtox.sh "py${{ matrix.python-version }}-asgi" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch
-          coverage combine .coverage*
-          coverage xml -i
+            # Run tests
+            ./scripts/runtox.sh "py${{ matrix.python-version }}-asgi" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch &&
+            coverage combine .coverage* &&
+            coverage xml -i
 
       - uses: codecov/codecov-action@v3
         with:

.github/workflows/test-integration-aws_lambda.yml

@@ -26,7 +26,7 @@ jobs:
   test:
     name: aws_lambda, python ${{ matrix.python-version }}, ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 45
+    timeout-minutes: 30
 
     strategy:
       fail-fast: false
@@ -49,16 +49,20 @@ jobs:
           pip install coverage "tox>=3,<4"
 
       - name: Test aws_lambda
-        timeout-minutes: 45
-        shell: bash
-        run: |
-          set -x # print commands that are executed
-          coverage erase
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 15
+          max_attempts: 2
+          retry_wait_seconds: 5
+          shell: bash
+          command: |
+            set -x # print commands that are executed
+            coverage erase
 
-          # Run tests
-          ./scripts/runtox.sh "py${{ matrix.python-version }}-aws_lambda" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch
-          coverage combine .coverage*
-          coverage xml -i
+            # Run tests
+            ./scripts/runtox.sh "py${{ matrix.python-version }}-aws_lambda" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch &&
+            coverage combine .coverage* &&
+            coverage xml -i
 
       - uses: codecov/codecov-action@v3
         with:

.github/workflows/test-integration-beam.yml

@@ -26,7 +26,7 @@ jobs:
   test:
     name: beam, python ${{ matrix.python-version }}, ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 45
+    timeout-minutes: 30
 
     strategy:
       fail-fast: false
@@ -49,16 +49,20 @@ jobs:
           pip install coverage "tox>=3,<4"
 
       - name: Test beam
-        timeout-minutes: 45
-        shell: bash
-        run: |
-          set -x # print commands that are executed
-          coverage erase
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 15
+          max_attempts: 2
+          retry_wait_seconds: 5
+          shell: bash
+          command: |
+            set -x # print commands that are executed
+            coverage erase
 
-          # Run tests
-          ./scripts/runtox.sh "py${{ matrix.python-version }}-beam" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch
-          coverage combine .coverage*
-          coverage xml -i
+            # Run tests
+            ./scripts/runtox.sh "py${{ matrix.python-version }}-beam" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch &&
+            coverage combine .coverage* &&
+            coverage xml -i
 
       - uses: codecov/codecov-action@v3
         with:

.github/workflows/test-integration-boto3.yml

@@ -26,7 +26,7 @@ jobs:
   test:
     name: boto3, python ${{ matrix.python-version }}, ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 45
+    timeout-minutes: 30
 
     strategy:
       fail-fast: false
@@ -49,16 +49,20 @@ jobs:
           pip install coverage "tox>=3,<4"
 
       - name: Test boto3
-        timeout-minutes: 45
-        shell: bash
-        run: |
-          set -x # print commands that are executed
-          coverage erase
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 15
+          max_attempts: 2
+          retry_wait_seconds: 5
+          shell: bash
+          command: |
+            set -x # print commands that are executed
+            coverage erase
 
-          # Run tests
-          ./scripts/runtox.sh "py${{ matrix.python-version }}-boto3" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch
-          coverage combine .coverage*
-          coverage xml -i
+            # Run tests
+            ./scripts/runtox.sh "py${{ matrix.python-version }}-boto3" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch &&
+            coverage combine .coverage* &&
+            coverage xml -i
 
       - uses: codecov/codecov-action@v3
         with:

.github/workflows/test-integration-bottle.yml

@@ -26,7 +26,7 @@ jobs:
   test:
     name: bottle, python ${{ matrix.python-version }}, ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 45
+    timeout-minutes: 30
 
     strategy:
       fail-fast: false
@@ -49,16 +49,20 @@ jobs:
           pip install coverage "tox>=3,<4"
 
       - name: Test bottle
-        timeout-minutes: 45
-        shell: bash
-        run: |
-          set -x # print commands that are executed
-          coverage erase
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 15
+          max_attempts: 2
+          retry_wait_seconds: 5
+          shell: bash
+          command: |
+            set -x # print commands that are executed
+            coverage erase
 
-          # Run tests
-          ./scripts/runtox.sh "py${{ matrix.python-version }}-bottle" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch
-          coverage combine .coverage*
-          coverage xml -i
+            # Run tests
+            ./scripts/runtox.sh "py${{ matrix.python-version }}-bottle" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch &&
+            coverage combine .coverage* &&
+            coverage xml -i
 
       - uses: codecov/codecov-action@v3
         with:

.github/workflows/test-integration-celery.yml

@@ -26,7 +26,7 @@ jobs:
   test:
     name: celery, python ${{ matrix.python-version }}, ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 45
+    timeout-minutes: 30
 
     strategy:
       fail-fast: false
@@ -49,16 +49,20 @@ jobs:
           pip install coverage "tox>=3,<4"
 
       - name: Test celery
-        timeout-minutes: 45
-        shell: bash
-        run: |
-          set -x # print commands that are executed
-          coverage erase
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 15
+          max_attempts: 2
+          retry_wait_seconds: 5
+          shell: bash
+          command: |
+            set -x # print commands that are executed
+            coverage erase
 
-          # Run tests
-          ./scripts/runtox.sh "py${{ matrix.python-version }}-celery" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch
-          coverage combine .coverage*
-          coverage xml -i
+            # Run tests
+            ./scripts/runtox.sh "py${{ matrix.python-version }}-celery" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch &&
+            coverage combine .coverage* &&
+            coverage xml -i
 
       - uses: codecov/codecov-action@v3
         with:

.github/workflows/test-integration-chalice.yml

@@ -26,7 +26,7 @@ jobs:
   test:
     name: chalice, python ${{ matrix.python-version }}, ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 45
+    timeout-minutes: 30
 
     strategy:
       fail-fast: false
@@ -49,16 +49,20 @@ jobs:
           pip install coverage "tox>=3,<4"
 
       - name: Test chalice
-        timeout-minutes: 45
-        shell: bash
-        run: |
-          set -x # print commands that are executed
-          coverage erase
+        uses: nick-fields/retry@v2
+        with:
+          timeout_minutes: 15
+          max_attempts: 2
+          retry_wait_seconds: 5
+          shell: bash
+          command: |
+            set -x # print commands that are executed
+            coverage erase
 
-          # Run tests
-          ./scripts/runtox.sh "py${{ matrix.python-version }}-chalice" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch
-          coverage combine .coverage*
-          coverage xml -i
+            # Run tests
+            ./scripts/runtox.sh "py${{ matrix.python-version }}-chalice" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch &&
+            coverage combine .coverage* &&
+            coverage xml -i
 
       - uses: codecov/codecov-action@v3
         with: