From e23175f00be8abd235c3f48ddae55223dd700abe Mon Sep 17 00:00:00 2001 From: Ali Waseem Date: Thu, 27 Nov 2025 08:27:50 -0700 Subject: [PATCH] Feat: E2E tests for AI assistant and log drains (#40844) * updated commands and expose ai key locally * added tests for AI assistant * added OPEN_API_KEY for e2e test suite * updated log drain options * updated README --- .github/workflows/studio-e2e-test.yml | 3 + e2e/studio/README.md | 6 ++ e2e/studio/features/assistant.spec.ts | 46 +++++++++++++++ e2e/studio/features/log-drains.spec.ts | 77 ++++++++++++++++++++++++++ package.json | 2 +- supabase/config.toml | 2 + 6 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 e2e/studio/features/assistant.spec.ts create mode 100644 e2e/studio/features/log-drains.spec.ts diff --git a/.github/workflows/studio-e2e-test.yml b/.github/workflows/studio-e2e-test.yml index 469818511b..8b68f0c0c0 100644 --- a/.github/workflows/studio-e2e-test.yml +++ b/.github/workflows/studio-e2e-test.yml @@ -20,6 +20,9 @@ jobs: # Require approval only for pull requests from forks environment: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork && 'Studio E2E Tests' || '' }} + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + steps: - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2 diff --git a/e2e/studio/README.md b/e2e/studio/README.md index c59cf163e9..6cfe357c67 100644 --- a/e2e/studio/README.md +++ b/e2e/studio/README.md @@ -16,6 +16,12 @@ cd e2e/studio pnpm exec playwright install ``` +### Environment Variables + +Some tests require specific environment variables to be set. If these are not set, the tests will be automatically skipped: + +- **`OPENAI_API_KEY`**: Required for the AI Assistant test (`assistant.spec.ts`). Without this variable, the assistant test will be skipped. + --- ## Running the tests diff --git a/e2e/studio/features/assistant.spec.ts b/e2e/studio/features/assistant.spec.ts new file mode 100644 index 0000000000..c549040e53 --- /dev/null +++ b/e2e/studio/features/assistant.spec.ts @@ -0,0 +1,46 @@ +import { expect } from '@playwright/test' +import { test } from '../utils/test.js' +import { toUrl } from '../utils/to-url.js' + +test.describe('AI Assistant', async () => { + test('Can send a message to the assistant and receive a response', async ({ page, ref }) => { + // Skip the test if the OPENAI_API_KEY is not set + test.skip(!process.env.OPENAI_API_KEY, 'OPENAI_API_KEY is not set') + + await page.goto(toUrl(`/project/${ref}`)) + + // Wait for the page to load + await expect(page.getByRole('heading', { level: 1 })).toBeVisible() + + // Click the assistant button to open the assistant panel + await page.locator('#assistant-trigger').click() + + // Wait for the assistant panel to be visible + await expect(page.getByRole('heading', { name: 'How can I assist you?' })).toBeVisible() + + // Type "hello" in the chat input + const chatInput = page.getByRole('textbox', { name: 'Chat to Postgres...' }) + await chatInput.fill('hello') + + const responsePromise = page.waitForResponse( + (response) => + response.url().includes('/api/ai/sql/generate-v4') && + response.request().method() === 'POST', + { timeout: 60000 } + ) + + // Click the send message button + const sendButton = page.getByRole('button', { name: 'Send message' }) + await sendButton.click() + + // Wait for the API request to complete + const response = await responsePromise + + // Verify the response was successful + expect(response.status()).toBe(200) + + // AI response has values + const body = await response.text() + expect(body).toContain('data') + }) +}) diff --git a/e2e/studio/features/log-drains.spec.ts b/e2e/studio/features/log-drains.spec.ts new file mode 100644 index 0000000000..6895a4f7d5 --- /dev/null +++ b/e2e/studio/features/log-drains.spec.ts @@ -0,0 +1,77 @@ +import { expect } from '@playwright/test' +import { test } from '../utils/test.js' +import { toUrl } from '../utils/to-url.js' + +const LOG_DRAIN_OPTIONS = [ + { + name: 'Custom Endpoint', + buttonText: 'Custom Endpoint Forward logs', + }, + { + name: 'Datadog', + buttonText: 'Datadog Datadog is a', + }, + { + name: 'Loki', + buttonText: 'Loki Loki is an open-source', + }, +] + +test.describe('Log Drains Settings', () => { + test.beforeEach(async ({ page, ref }) => { + // Navigate to the log drains settings page + await page.goto(toUrl(`/project/${ref}/settings/log-drains`)) + + // Wait for the page to load + await expect(page.getByRole('heading', { name: 'Log Drains', level: 1 }), { + message: 'Log Drains heading should be visible', + }).toBeVisible() + }) + + for (const option of LOG_DRAIN_OPTIONS) { + test(`Opens ${option.name} panel when clicked`, async ({ page }) => { + // Click on the log drain option button + const optionButton = page.getByRole('button', { name: option.buttonText }) + await expect(optionButton, { + message: `${option.name} button should be visible`, + }).toBeVisible() + + await optionButton.click() + + // Verify that the "Add destination" dialog opens + const dialog = page.getByRole('dialog', { name: 'Add destination' }) + await expect(dialog, { + message: `Add destination dialog should be visible for ${option.name}`, + }).toBeVisible() + + // Verify the dialog heading + await expect(dialog.getByRole('heading', { name: 'Add destination', level: 2 }), { + message: 'Dialog heading should be visible', + }).toBeVisible() + + // Verify that the Type field shows the correct option + const typeCombobox = dialog.getByRole('combobox').first() + await expect(typeCombobox, { + message: `Type combobox should contain ${option.name}`, + }).toContainText(option.name) + + // Close the dialog by pressing Escape + await page.keyboard.press('Escape') + + // Verify the dialog is closed + await expect(dialog, { + message: 'Dialog should be hidden after pressing Escape', + }).not.toBeVisible() + }) + } + + test('All log drain options are visible on the page', async ({ page }) => { + // Verify all three options are displayed + for (const option of LOG_DRAIN_OPTIONS) { + const optionButton = page.getByRole('button', { name: option.buttonText }) + await expect(optionButton, { + message: `${option.name} option should be visible`, + }).toBeVisible() + } + }) +}) diff --git a/package.json b/package.json index ba2103d98d..99f4a6d2d0 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "test:ui-patterns": "turbo run test --filter=ui-patterns", "test:studio": "turbo run test --filter=studio", "test:studio:watch": "turbo run test --filter=studio -- watch", - "e2e:setup:cli": "supabase start --exclude studio && supabase db reset && supabase status --output json > keys.json && node scripts/generateLocalEnv.js", + "e2e:setup:cli": "supabase stop --all --no-backup ; supabase start --exclude studio && supabase db reset && supabase status --output json > keys.json && node scripts/generateLocalEnv.js", "e2e:setup": "SKIP_ASSET_UPLOAD=1 pnpm e2e:setup:cli && NODE_ENV=test NODE_OPTIONS=\"--max-old-space-size=4096\" pnpm run build:studio && NODE_ENV=test pnpm --prefix ./apps/studio start --port 8082", "e2e": "pnpm --prefix e2e/studio run e2e", "e2e:ui": "pnpm --prefix e2e/studio run e2e:ui", diff --git a/supabase/config.toml b/supabase/config.toml index 177ecf2700..b3190d03dd 100644 --- a/supabase/config.toml +++ b/supabase/config.toml @@ -31,6 +31,8 @@ major_version = 15 # Port to use for Supabase Studio. port = 54323 +openai_api_key = "env(OPENAI_API_KEY)" + # Email testing server. Emails sent with the local dev setup are not actually sent - rather, they # are monitored, and you can view the emails that would have been sent from the web interface. [inbucket]