ci: DH-20051: shard e2e tests and merge shard reports

.github/workflows/e2e.yml

@@ -11,33 +11,134 @@ on:
       - 'release/**'
 
 jobs:
-  e2e-test:
+  build-e2e-images:
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Build Docker images
+        run: docker compose -f ./tests/docker-compose.yml build deephaven-plugins e2e-tests
+
+      - name: Export Docker images
+        run: |
+          docker save tests-deephaven-plugins tests-e2e-tests | gzip > e2e-docker-images.tar.gz
+
+      - name: Upload Docker images
+        uses: actions/upload-artifact@v7
+        with:
+          name: e2e-docker-images
+          path: e2e-docker-images.tar.gz
+          retention-days: 1
+
+  e2e-tests:
     runs-on: ubuntu-24.04
+    needs: [build-e2e-images]
     concurrency:
-      group: ${{ github.workflow }}-${{ github.ref }}
+      group: ${{ github.workflow }}-${{ github.ref }}-e2e-tests-${{ matrix.config }}
       cancel-in-progress: true
+    strategy:
+      fail-fast: false
+      matrix:
+        config: [chromium-1-2, chromium-2-2, firefox-1-2, firefox-2-2, webkit-1-2, webkit-2-2]
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
+
+      - name: Download Docker images
+        uses: actions/download-artifact@v8
+        with:
+          name: e2e-docker-images
+
+      - name: Load Docker images
+        run: gunzip -c e2e-docker-images.tar.gz | docker load
+
+      - name: Extract browser and shard config
+        id: config
+        env:
+          MATRIX_CONFIG: ${{ matrix.config }}
+        run: |
+          IFS='-' read -r browser shard shard_total <<< "$MATRIX_CONFIG"
+          echo "browser=$browser" >> "$GITHUB_OUTPUT"
+          echo "shard=$shard" >> "$GITHUB_OUTPUT"
+          echo "shard_total=$shard_total" >> "$GITHUB_OUTPUT"
 
       - name: Run tests
-        run: './tools/run_docker.sh ./tests/docker-compose.yml e2e-tests'
+        env:
+          RUN_DOCKER_BUILD: 'false'
+        run: |
+          ./tools/run_docker.sh ./tests/docker-compose.yml e2e-tests \
+            --reporter=blob \
+            --project=${{ steps.config.outputs.browser }} \
+            --shard=${{ steps.config.outputs.shard }}/${{ steps.config.outputs.shard_total }}
 
-      - name: Upload Playwright report
-        uses: actions/upload-artifact@v4
+      - name: Upload Playwright blob report
+        uses: actions/upload-artifact@v7
         if: always()
         with:
-          name: playwright-report
-          path: playwright-report/
-          retention-days: 90
+          name: playwright-report-blob-${{ matrix.config }}
+          path: blob-report/
+          retention-days: 1
 
       - name: Dump server logs
         if: failure()
         run: docker logs deephaven-plugins > /tmp/server-log.txt
 
       - name: Upload server logs
         if: failure()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
-          name: server-logs
+          name: server-logs-${{ matrix.config }}
           path: /tmp/server-log.txt
+
+  merge-reports:
+    if: ${{ always() }}
+    runs-on: ubuntu-24.04
+    needs: [e2e-tests]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Use Node.js
+        uses: actions/setup-node@v5
+        with:
+          node-version-file: '.nvmrc'
+
+      - name: Download blob reports
+        uses: actions/download-artifact@v8
+        with:
+          pattern: playwright-report-blob-*
+
+      - name: Merge into HTML report
+        run: |
+          mkdir -p all-blob-reports
+          for dir in playwright-report-blob-*; do
+            [ -d "$dir" ] || continue
+            zip_file=$(find "$dir" -type f -name '*.zip' | head -n 1)
+            [ -n "$zip_file" ] || continue
+            shard_name=${dir#playwright-report-blob-}
+            cp "$zip_file" "all-blob-reports/${shard_name}.zip"
+          done
+
+          if ! find all-blob-reports -type f -name '*.zip' | grep -q .; then
+            echo "No blob report zip files were found after download."
+            exit 1
+          fi
+
+          npx -y playwright@1.44.1 merge-reports --reporter html,github ./all-blob-reports
+
+      - name: Upload HTML report
+        uses: actions/upload-artifact@v7
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 30
+
+  e2e-test:
+    if: ${{ always() }}
+    runs-on: ubuntu-24.04
+    needs: [e2e-tests, merge-reports]
+    steps:
+      - name: Fail if any tests failed or cancelled
+        run: exit 1
+        if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}

playwright-docker.config.ts

@@ -7,7 +7,6 @@ const config: PlaywrightTestConfig = {
     ...DefaultConfig.use,
     baseURL: 'http://deephaven-plugins:10000/ide/',
   },
-  reporter: process.env.CI ? [['github'], ['html']] : DefaultConfig.reporter,
 };
 
 export default config;

tests/Dockerfile

@@ -4,13 +4,11 @@
 FROM mcr.microsoft.com/playwright:v1.44.1-jammy AS playwright
 WORKDIR /work/
 
-# Update packages list and install some build tools.
-# Installing fonts-dejavu-core so we have some common fonts with the GH actions
-# runner that can be used to render unicode fonts. See web-client-ui README for more info.
+# Install a minimal font package so rendering is closer to GH Actions runners.
 RUN set -eux; \
     apt-get update; \
-    apt-get install build-essential --yes; \
-    apt-get install fonts-dejavu-core --yes;
+    apt-get install --no-install-recommends --yes fonts-dejavu-core; \
+    rm -rf /var/lib/apt/lists/*
 
 RUN fc-list : family;
 

tests/docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3'
-
 services:
   deephaven-plugins:
     container_name: deephaven-plugins
@@ -27,6 +25,7 @@ services:
     volumes:
       - ../tests:/work/tests
       - ../test-results:/work/test-results
+      - ../blob-report:/work/blob-report
       - ../playwright-report:/work/playwright-report
       - ../plugins:/work/plugins
     entrypoint: 'npx playwright test --config=playwright-docker.config.ts'

tests/ui_plotly.spec.ts

@@ -7,7 +7,7 @@ test.describe('plotly works in deephaven.ui', () => {
     test(name, async ({ page }) => {
       await gotoPage(page, '');
       await openPanel(page, name, SELECTORS.WIDGET_LOADER_ELEMENT_VISIBLE);
-
+      
       await expect(
         page.locator(SELECTORS.WIDGET_LOADER_ELEMENT_VISIBLE)
       ).toHaveScreenshot();

tools/run_docker.sh

@@ -3,16 +3,21 @@
 COMPOSE_FILE=$1
 shift
 
+BUILD_FLAG=(--build)
+if [[ "${RUN_DOCKER_BUILD:-true}" == "false" ]]; then
+  BUILD_FLAG=()
+fi
+
 # Start the containers
 if [[ "${CI}" == "1" || "${CI}" == "true" ]]; then
   # In CI, keep the container in case we need to dump logs in another
   # step of the GH action. It should be cleaned up automatically by the CI runner.
-  docker compose -f "${COMPOSE_FILE}" run --service-ports --build -e CI=true "$@"
+  docker compose -f "${COMPOSE_FILE}" run --service-ports "${BUILD_FLAG[@]}" -e CI=true "$@"
   exit_code=$?
   # stop instead of down to preserve container logs
   docker compose -f "${COMPOSE_FILE}" stop deephaven-plugins
 else
-  docker compose -f "${COMPOSE_FILE}" run --service-ports --rm --build "$@"
+  docker compose -f "${COMPOSE_FILE}" run --service-ports --rm "${BUILD_FLAG[@]}" "$@"
   exit_code=$?
   docker compose -f "${COMPOSE_FILE}" down
 fi