From 82c5ffb19fe6e6110bfe34088b3ce2ba516eaae7 Mon Sep 17 00:00:00 2001 From: Gildas Garcia <1122076+djhi@users.noreply.github.com> Date: Mon, 2 Mar 2026 16:43:47 +0100 Subject: [PATCH] Configure sharding for Studio e2e tests (#43211) ## Problem The e2e test suite for Studio takes around 20min. Feedback loop is too long ## Solution Try enabling [playwright sharding](https://playwright.dev/docs/test-sharding#merge-reports-cli). Only 2 shards for now ## Results Before: image image After: image ## Next steps In future dedicated PRs, improve the tests themselves. --------- Co-authored-by: Charis <26616127+charislam@users.noreply.github.com> --- .github/workflows/studio-e2e-test.yml | 83 +++++++++++++++++-- .../features/realtime-inspector.spec.ts | 4 +- e2e/studio/playwright.config.ts | 13 +-- e2e/studio/playwright.merge.config.ts | 7 ++ package.json | 11 ++- 5 files changed, 102 insertions(+), 16 deletions(-) create mode 100644 e2e/studio/playwright.merge.config.ts diff --git a/.github/workflows/studio-e2e-test.yml b/.github/workflows/studio-e2e-test.yml index e7f0cbb39b7..c2a31de86b0 100644 --- a/.github/workflows/studio-e2e-test.yml +++ b/.github/workflows/studio-e2e-test.yml @@ -15,8 +15,16 @@ permissions: jobs: test: + name: 'E2E tests' timeout-minutes: 60 runs-on: blacksmith-4vcpu-ubuntu-2404 + strategy: + fail-fast: false + matrix: + shardIndex: [1, 2] + shardTotal: [2] + outputs: + tests_ran: ${{ steps.filter.outputs.studio == 'true' }} env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} @@ -55,13 +63,65 @@ jobs: if: steps.filter.outputs.studio == 'true' run: pnpm -C e2e/studio exec playwright install chromium --with-deps --only-shell + - name: Set up NextJS/Turbo cache + if: steps.filter.outputs.studio == 'true' + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + # See here for caching with `yarn`, `bun` or other package managers https://github.com/actions/cache/blob/main/examples.md or you can leverage caching with actions/setup-node https://github.com/actions/setup-node + path: | + .turbo/cache + apps/studio/.next/build + apps/studio/.next/cache + # Generate a new cache whenever packages or source files change. + key: ${{ runner.os }}-nextjs-${{ hashFiles('pnpm-lock.yaml') }}-${{ hashFiles('apps/studio/**/*.js', 'apps/studio/**/*.jsx', 'apps/studio/**/*.ts', 'apps/studio/**/*.tsx') }} + # If source files changed but packages didn't, rebuild from a prior cache. + restore-keys: | + ${{ runner.os }}-nextjs-${{ hashFiles('pnpm-lock.yaml') }}- + - name: 🚀 Run Playwright tests against Vercel Preview if: steps.filter.outputs.studio == 'true' id: playwright - run: pnpm e2e + run: pnpm e2e --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} + + - name: Upload blob report to GitHub Actions Artifacts + if: always() && steps.filter.outputs.studio == 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: blob-report-${{ matrix.shardIndex }} + path: e2e/studio/blob-report + retention-days: 7 + + - name: Fail job if tests failed + if: steps.filter.outputs.studio == 'true' && steps.playwright.outcome != 'success' + run: | + echo "E2E tests failed" >&2 + exit 1 + + merge-reports: + name: 'E2E reports' + # Merge reports after playwright-tests, even if some shards have failed + if: ${{ !cancelled() && needs.test.outputs.tests_ran == 'true' }} + needs: [test] + runs-on: blacksmith-4vcpu-ubuntu-2404 + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + + - name: Use Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version-file: '.nvmrc' + + - name: Download blob reports from GitHub Actions Artifacts + uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v 5.0.0 + with: + path: e2e/studio/blob-report + pattern: blob-report-* + merge-multiple: true + + - name: Merge Playwright reports + run: npx playwright merge-reports --config=e2e/studio/playwright.merge.config.ts -- e2e/studio/blob-report - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - if: always() && steps.filter.outputs.studio == 'true' with: name: playwright-artifacts path: | @@ -70,14 +130,21 @@ jobs: retention-days: 7 - name: Comment Playwright test results on PR + if: always() && github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork uses: daun/playwright-report-comment@be9e270edd5ad86038604d3caa84a819a6ff6fed # v3.10.0 - if: always() && steps.filter.outputs.studio == 'true' && github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork with: report-file: e2e/studio/test-results/test-results.json comment-title: '🎭 Playwright Test Results' - - name: Fail job if tests failed - if: steps.filter.outputs.studio == 'true' && (steps.playwright.outcome != 'success' || steps.summarize.outputs.flaky_count > 0) - run: | - echo "E2E tests failed" >&2 - exit 1 + merge-results: + name: 'E2E results' + runs-on: ubuntu-latest + needs: [test] + if: ${{ !cancelled() && needs.test.outputs.tests_ran == 'true' }} + steps: + - name: All tests ok + if: ${{ !(contains(needs.*.result, 'failure')) }} + run: exit 0 + - name: Some tests failed + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 diff --git a/e2e/studio/features/realtime-inspector.spec.ts b/e2e/studio/features/realtime-inspector.spec.ts index 64737faa4a7..e14ae749e19 100644 --- a/e2e/studio/features/realtime-inspector.spec.ts +++ b/e2e/studio/features/realtime-inspector.spec.ts @@ -1,5 +1,5 @@ import { expect } from '@playwright/test' -import { test } from '../utils/test.js' + import { getMessageCount, joinChannel, @@ -10,6 +10,7 @@ import { stopListening, waitForRealtimeMessage, } from '../utils/realtime-helpers.js' +import { test } from '../utils/test.js' const testChannelName = 'pw_realtime_test_channel' @@ -116,6 +117,7 @@ test.describe('Realtime Inspector', () => { await openBroadcastModal(page) const codeEditor = page.getByRole('textbox', { name: /Editor content/i }) + await expect(codeEditor).toBeInViewport({ timeout: 5000 }) await codeEditor.click({ force: true }) await page.keyboard.press('ControlOrMeta+KeyA') await page.keyboard.type('{ invalid json }') diff --git a/e2e/studio/playwright.config.ts b/e2e/studio/playwright.config.ts index 72d0e4e8f04..2d7fe409717 100644 --- a/e2e/studio/playwright.config.ts +++ b/e2e/studio/playwright.config.ts @@ -1,4 +1,5 @@ import { defineConfig } from '@playwright/test' + import { env, STORAGE_STATE_PATH } from './env.config.js' const IS_CI = !!process.env.CI @@ -132,10 +133,12 @@ export default defineConfig({ }, }, ], - reporter: [ - ['list'], - ['html', { open: 'never' }], - ['json', { outputFile: 'test-results/test-results.json' }], - ], + reporter: IS_CI + ? [['list'], ['blob']] + : [ + ['list'], + ['html', { open: 'never' }], + ['json', { outputFile: 'test-results/test-results.json' }], + ], webServer: createWebServerConfig(), }) diff --git a/e2e/studio/playwright.merge.config.ts b/e2e/studio/playwright.merge.config.ts new file mode 100644 index 00000000000..bcfc8d05bf1 --- /dev/null +++ b/e2e/studio/playwright.merge.config.ts @@ -0,0 +1,7 @@ +export default { + testDir: './features', + reporter: [ + ['html', { open: 'never' }], + ['json', { outputFile: 'test-results/test-results.json' }], + ], +} diff --git a/package.json b/package.json index 2968550a6cf..6719ded886e 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "test:ui-patterns": "turbo run test --filter=ui-patterns", "test:studio": "turbo run test --filter=studio", "test:studio:watch": "turbo run test --filter=studio -- watch", - "e2e:setup:cli": "supabase stop --all --no-backup ; supabase start --exclude studio && supabase db reset && supabase status --output json > keys.json && node scripts/generateLocalEnv.js", + "e2e:setup:cli": "supabase stop --all --no-backup ; supabase start --exclude studio && if [ -z \"${CI}\" ]; then supabase db reset; fi && supabase status --output json > keys.json && node scripts/generateLocalEnv.js", "e2e:setup:selfhosted": "SKIP_ASSET_UPLOAD=1 pnpm e2e:setup:cli && NODE_ENV=test NODE_OPTIONS=\"--max-old-space-size=4096\" pnpm run build:studio && NODE_ENV=test pnpm --prefix ./apps/studio start", "e2e:setup:platform": "SKIP_ASSET_UPLOAD=1 NODE_OPTIONS=\"--max-old-space-size=4096\" pnpm run build:studio && pnpm --prefix ./apps/studio start", "e2e": "pnpm --prefix e2e/studio run e2e", @@ -65,6 +65,13 @@ "pnpm": "10.24", "node": ">=22" }, - "keywords": ["postgres", "firebase", "storage", "functions", "database", "auth"], + "keywords": [ + "postgres", + "firebase", + "storage", + "functions", + "database", + "auth" + ], "packageManager": "pnpm@10.24.0" }