mirror of
https://github.com/supabase/supabase.git
synced 2026-09-06 18:11:51 +08:00
Closes DOCS-1203 ## Problem The docs E2E workflow only ever tested one hardcoded page: the Next.js quickstart. All other docs content had no E2E coverage. ## Solution This PR expands the initial scaffolding to generalize the Next.js quickstart tests, page runs and checks local links, to all pages affecting Docs content: - Add `resolveDocsScope` (`e2e/docs/utils/resolve-docs-scope.ts`) to map changed guide and troubleshooting `.mdx` files to their `/docs/...` page paths, and to expand changed `_partials` to every page that includes them (including transitively, through partials nested inside other partials). Federated guide sections (`graphql`, `database/extensions/wrappers`, `ai/python`, `deployment/terraform`, `deployment/ci`) and reference docs stay out of scope, and resolution is capped at 20 pages to keep runtime bounded. - Replace the single `quickstarts.spec.ts` test with a generic `docs-pages.spec.ts` that loads whatever pages are resolved, asserting each renders with an `<h1>` and that its docs-owned links resolve. - Add `run-e2e-docs.ts` so `pnpm e2e:docs` resolves scope locally (from commits since `origin/master`, plus staged/unstaged changes) and skips Playwright entirely when nothing in scope changed. - Update `.github/workflows/docs-e2e.yml` to widen the trigger paths to all guides/troubleshooting/partials, resolve scope in a dedicated step, skip the rest of the job when scope is empty, and accept a `page_paths` input for manual `workflow_dispatch` runs. - Rewrite `e2e/docs/README.md` to document the new scoping behavior, the override envs (`DOCS_E2E_PAGE_PATHS`, `DOCS_E2E_BASE_REF`), and how CI uses the suite. - `pnpm e2e:docs:all` is also added to run tests on every page locally. Good for scoping issues but should not be included in CI. ## Manual testing Walk through the following steps to verify this works: - [x] `pnpm e2e:docs` from repo root resolves the expected pages for a local guide edit and can run against local dev **Note:** Challenges with testing on local in part because of the long lag for first page load. Recommendation to use a hosted URL is added to docs. - [x] Editing a shared `_partials` file resolves to every page that includes it (including through nested partials) - [x] `pnpm e2e:docs` exits cleanly with no Playwright run when no in-scope files changed - [x] `git diff --name-only ... | pnpm -C e2e/docs resolve-docs-scope` prints the expected page list for a sample diff - [x] Workflow run on a PR that only touches `e2e/docs`/workflow files skips the Playwright steps - [x] Manual `workflow_dispatch` run with `page_paths` set tests only those pages - [x] Run `pnpm e2e:docs:all` to run the suite on all docs content, which takes awhile ## Next steps After this PR merges, we have the scaffolding to add more fun tests like a11y 😁 <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Added scoped Docs E2E runs that target eligible doc pages based on changes, plus manual page-targeted runs and an “all eligible pages” mode. * Introduced `DOCS_E2E_PAGE_PATHS` (and updated base ref/base URL behavior) to control which pages are tested. * **Bug Fixes** * Automatically skips Playwright setup when no relevant pages are in scope; Playwright reporting now uploads only on failure. * **Documentation** * Updated the Docs E2E README with new run/CI behavior, troubleshooting notes, and commands to inspect the resolved page list. * **Tests** * Added a Docs-owned pages E2E suite; removed the Next.js quickstart E2E spec. <!-- end of auto-generated comment: release notes by coderabbit.ai --> --------- Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>
187 lines
7.2 KiB
YAML
187 lines
7.2 KiB
YAML
name: Docs E2E Tests
|
||
|
||
on:
|
||
pull_request:
|
||
types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]
|
||
branches: ['master']
|
||
paths:
|
||
- 'apps/docs/content/guides/**/*.mdx'
|
||
- 'apps/docs/content/troubleshooting/**/*.mdx'
|
||
- 'apps/docs/content/_partials/**'
|
||
- 'e2e/docs/features/**'
|
||
- 'e2e/docs/utils/**'
|
||
- 'e2e/docs/scripts/**'
|
||
- 'e2e/docs/playwright.config.ts'
|
||
- 'e2e/docs/package.json'
|
||
- 'e2e/docs/tsconfig.json'
|
||
- 'pnpm-lock.yaml'
|
||
- '.github/workflows/docs-e2e.yml'
|
||
workflow_dispatch:
|
||
inputs:
|
||
base_url:
|
||
description: 'Base URL to test against'
|
||
required: false
|
||
default: 'https://supabase.com'
|
||
type: string
|
||
page_paths:
|
||
description: 'Comma-separated /docs/... paths to test (required for manual runs)'
|
||
required: false
|
||
default: ''
|
||
type: string
|
||
|
||
concurrency:
|
||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||
cancel-in-progress: true
|
||
|
||
permissions:
|
||
contents: read
|
||
statuses: read
|
||
pull-requests: read
|
||
|
||
env:
|
||
CI: true
|
||
|
||
jobs:
|
||
e2e:
|
||
name: Docs E2E
|
||
if: github.event_name == 'workflow_dispatch' || github.event.pull_request.draft == false
|
||
timeout-minutes: 30
|
||
runs-on: blacksmith-4vcpu-ubuntu-2404
|
||
|
||
steps:
|
||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||
with:
|
||
persist-credentials: false
|
||
# Need full history on PRs so we can diff against the base branch.
|
||
# Use string '0' — numeric 0 is falsy in GitHub Actions expressions.
|
||
fetch-depth: ${{ github.event_name == 'pull_request' && '0' || '1' }}
|
||
sparse-checkout: |
|
||
e2e/docs
|
||
scripts
|
||
patches
|
||
apps/docs/content/guides
|
||
apps/docs/content/troubleshooting
|
||
apps/docs/content/_partials
|
||
apps/docs/scripts/federated-content/sources
|
||
|
||
- name: Use Node.js
|
||
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||
with:
|
||
node-version-file: '.nvmrc'
|
||
|
||
# Map changed owned content (guides, troubleshooting, partials) to page
|
||
# URLs. Harness-only PRs resolve to skip=true and exit before Playwright.
|
||
- name: Resolve docs E2E scope
|
||
id: scope
|
||
env:
|
||
EVENT_NAME: ${{ github.event_name }}
|
||
BASE_REF: ${{ github.base_ref }}
|
||
PAGE_PATHS_INPUT: ${{ inputs.page_paths }}
|
||
run: |
|
||
if [ "$EVENT_NAME" = "workflow_dispatch" ]; then
|
||
if [ -z "$PAGE_PATHS_INPUT" ]; then
|
||
echo "skip=true" >> "$GITHUB_OUTPUT"
|
||
echo "paths=" >> "$GITHUB_OUTPUT"
|
||
echo "Manual run requires the page_paths input."
|
||
exit 0
|
||
fi
|
||
echo "skip=false" >> "$GITHUB_OUTPUT"
|
||
printf 'paths=%s\n' "$PAGE_PATHS_INPUT" >> "$GITHUB_OUTPUT"
|
||
exit 0
|
||
fi
|
||
|
||
git diff --name-only --diff-filter=ACMR "origin/$BASE_REF"...HEAD \
|
||
| node --experimental-strip-types e2e/docs/scripts/resolve-docs-scope.ts
|
||
|
||
- name: Skip Playwright (no in-scope pages)
|
||
if: steps.scope.outputs.skip == 'true'
|
||
run: echo "No in-scope docs pages changed; skipping Playwright suite."
|
||
|
||
- uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
|
||
if: steps.scope.outputs.skip != 'true'
|
||
name: Install pnpm
|
||
with:
|
||
run_install: false
|
||
|
||
- name: Enable pnpm store cache
|
||
if: steps.scope.outputs.skip != 'true'
|
||
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||
with:
|
||
node-version-file: '.nvmrc'
|
||
cache: 'pnpm'
|
||
|
||
# Vercel skips the docs preview when a PR only changes the harness
|
||
# (e2e/docs, workflow). Wait for a preview only when apps/docs changed.
|
||
- name: Detect docs app changes
|
||
if: steps.scope.outputs.skip != 'true' && github.event_name == 'pull_request'
|
||
id: filter
|
||
uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
|
||
with:
|
||
filters: |
|
||
docs_app:
|
||
- 'apps/docs/**'
|
||
|
||
# Vercel's GitHub App stopped writing GitHub Deployment objects on
|
||
# 2026-02-17 (broken app auth), so vercel/wait-for-deployment-action
|
||
# times out polling that API even though the preview builds fine.
|
||
# Poll the "Vercel – docs" commit status instead — Vercel keeps posting
|
||
# those — then resolve the deployment it points to via Vercel's own API
|
||
# to get the actual preview URL. See scripts/waitForVercelDocsPreview.js.
|
||
- name: Wait for Vercel docs preview
|
||
if: steps.scope.outputs.skip != 'true' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && steps.filter.outputs.docs_app == 'true'
|
||
id: deployment
|
||
run: node scripts/waitForVercelDocsPreview.js
|
||
env:
|
||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||
VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }}
|
||
VERCEL_TEAM_ID: ${{ secrets.VERCEL_TEAM_ID }}
|
||
|
||
- name: Resolve base URL
|
||
if: steps.scope.outputs.skip != 'true'
|
||
id: base-url
|
||
env:
|
||
EVENT_NAME: ${{ github.event_name }}
|
||
BASE_URL_INPUT: ${{ inputs.base_url }}
|
||
DEPLOYMENT_URL: ${{ steps.deployment.outputs.deployment-url }}
|
||
DOCS_APP_CHANGED: ${{ steps.filter.outputs.docs_app }}
|
||
run: |
|
||
if [ "$EVENT_NAME" = "workflow_dispatch" ]; then
|
||
printf 'url=%s\n' "$BASE_URL_INPUT" >> "$GITHUB_OUTPUT"
|
||
echo "use_bypass=false" >> "$GITHUB_OUTPUT"
|
||
elif [ "$DOCS_APP_CHANGED" = "true" ] && [ -n "$DEPLOYMENT_URL" ]; then
|
||
printf 'url=%s\n' "$DEPLOYMENT_URL" >> "$GITHUB_OUTPUT"
|
||
echo "use_bypass=true" >> "$GITHUB_OUTPUT"
|
||
else
|
||
# Harness-only PRs have no docs preview; test against production.
|
||
echo "url=https://supabase.com" >> "$GITHUB_OUTPUT"
|
||
echo "use_bypass=false" >> "$GITHUB_OUTPUT"
|
||
fi
|
||
|
||
- name: Install dependencies
|
||
if: steps.scope.outputs.skip != 'true'
|
||
run: pnpm install --frozen-lockfile --filter=e2e-docs...
|
||
|
||
- name: Install Playwright Chromium
|
||
if: steps.scope.outputs.skip != 'true'
|
||
run: pnpm -C e2e/docs exec playwright install chromium --with-deps --only-shell
|
||
|
||
- name: Run docs E2E
|
||
if: steps.scope.outputs.skip != 'true'
|
||
working-directory: e2e/docs
|
||
run: pnpm run e2e:docs
|
||
env:
|
||
PLAYWRIGHT_BASE_URL: ${{ steps.base-url.outputs.url }}
|
||
DOCS_E2E_PAGE_PATHS: ${{ steps.scope.outputs.paths }}
|
||
VERCEL_AUTOMATION_BYPASS_SECRET: ${{ steps.base-url.outputs.use_bypass == 'true' && secrets.VERCEL_AUTOMATION_BYPASS_DOCS || '' }}
|
||
|
||
- name: Upload Playwright report
|
||
if: failure() && steps.scope.outputs.skip != 'true'
|
||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||
with:
|
||
name: docs-playwright-report
|
||
path: |
|
||
e2e/docs/playwright-report/
|
||
e2e/docs/test-results/
|
||
retention-days: 7
|