mirror of
https://github.com/supabase/supabase.git
synced 2026-09-07 02:20:52 +08:00
## Summary * Fixes [FE-4275](https://linear.app/supabase/issue/FE-4275/assistant-always-creates-notebooks-with-wrong-identifier-first-try): the assistant always created database notebook cells with a fabricated `database_identifier` (`"primary"`, later observed as `""` / `"_primary"` under different prompt wording) instead of omitting the key for the project's primary database, which tripped the tool's reject-and-retry validation on the very first attempt. * Prompt wording alone wasn't reliable — live eval runs against the real model kept substituting a new placeholder every time the prompt was tightened further. * Normalizes an empty-string `database_identifier` to absent at the schema level (`databaseIdentifierSchema` in `notebook-schema.ts`), which is inherited by every schema built from it — the AI SDK's `inputSchema` for `create_notebook`/`update_notebook`, and the write-boundary `writableNotebookSchema` used right before the PUT to the backend. * Adds an eval case (`evals/dataset.ts`) reproducing the original bug, plus unit tests covering schema-level and write-boundary normalization. ## Test plan - [X] `pnpm --filter studio exec tsc --noEmit` passes - [X] `pnpm exec prettier --check` passes on touched files - [X] Unit tests pass: `notebook-schema.test.ts`, `notebook-upsert-mutation.test.ts`, `notebook-tools.test.ts` (104 tests) - [X] Ran the new eval case against the real model 3x before the code fix (0% correctness, fabricated `""`/`"_primary"`) and 3x after (100% correctness) ## Summary by CodeRabbit * **Bug Fixes** * Improved notebook handling of empty database identifiers by treating them as absent. * Ensured notebook requests omit unused database identifier fields. * Added validation guidance for read-replica database identifiers.
175 lines
5.9 KiB
TypeScript
175 lines
5.9 KiB
TypeScript
import { safeSql } from '@supabase/pg-meta'
|
|
import { HttpResponse } from 'msw'
|
|
import { describe, expect, it } from 'vitest'
|
|
|
|
import type { WritableNotebook } from './notebook-schema'
|
|
import { createNotebook, upsertNotebook } from './notebook-upsert-mutation'
|
|
import { safeSql as safeLogSql } from '@/data/logs/safe-analytics-sql'
|
|
import { addAPIMock } from '@/tests/lib/msw'
|
|
|
|
const DATABASE_SQL = safeSql`select * from auth.users limit 100`
|
|
const LOG_SQL = "select timestamp, event_message from edge_logs where source = 'edge_logs' limit 10"
|
|
|
|
const EXISTING_CELL_ID = 'b1ffcd88-8d1a-4de7-aa5c-5aa8ac270b22'
|
|
|
|
// Create-shaped: no cell carries an `_id` — the notebook is brand new, so no cell has a
|
|
// prior version for the backend to diff against; ids are backend-generated on write.
|
|
const VALID_CONTENT: WritableNotebook = {
|
|
schema_version: 1,
|
|
cells: [
|
|
{ _tag: 'markdown_cell', text: '# Signup funnel' },
|
|
{ _tag: 'database_cell', sql: DATABASE_SQL, row_limit: 100 },
|
|
{
|
|
_tag: 'log_cell',
|
|
sql: safeLogSql`select timestamp, event_message from edge_logs where source = 'edge_logs' limit 10`,
|
|
time_range: { _tag: 'relative_time_range', unit: 'hour', amount: 1 },
|
|
},
|
|
],
|
|
}
|
|
|
|
// Update-shaped: mixes an existing cell (kept, carries its real backend-assigned _id so the
|
|
// backend can diff it against the previous version) with a newly inserted cell (no id —
|
|
// same as create, the backend assigns one on write).
|
|
const UPDATE_CONTENT: WritableNotebook = {
|
|
schema_version: 1,
|
|
cells: [
|
|
{
|
|
_tag: 'database_cell',
|
|
_id: EXISTING_CELL_ID,
|
|
sql: DATABASE_SQL,
|
|
row_limit: 100,
|
|
},
|
|
{
|
|
_tag: 'log_cell',
|
|
sql: safeLogSql`select timestamp, event_message from edge_logs where source = 'edge_logs' limit 10`,
|
|
time_range: { _tag: 'relative_time_range', unit: 'hour', amount: 1 },
|
|
},
|
|
],
|
|
}
|
|
|
|
// Missing `row_limit` on the database_cell — invalid per writableNotebookSchema. The
|
|
// point of this fixture is to prove validation runs (and rejects) before any request is
|
|
// sent, so it's cast rather than satisfying the real type.
|
|
const INVALID_CONTENT = {
|
|
schema_version: 1,
|
|
cells: [
|
|
{
|
|
_tag: 'database_cell',
|
|
sql: safeSql`select 1`,
|
|
},
|
|
],
|
|
} as unknown as WritableNotebook
|
|
|
|
describe('createNotebook', () => {
|
|
it('PUTs with type notebook, visibility project, a freshly generated id, and cells with no id', async () => {
|
|
let sentBody: Record<string, unknown> | undefined
|
|
addAPIMock({
|
|
method: 'put',
|
|
path: '/platform/projects/:ref/content',
|
|
response: async ({ request }) => {
|
|
sentBody = (await request.json()) as Record<string, unknown>
|
|
return HttpResponse.json(null)
|
|
},
|
|
})
|
|
|
|
await createNotebook({
|
|
projectRef: 'default',
|
|
name: 'Signup funnel',
|
|
content: VALID_CONTENT,
|
|
})
|
|
|
|
expect(sentBody?.type).toBe('notebook')
|
|
expect(sentBody?.visibility).toBe('project')
|
|
expect(typeof sentBody?.id).toBe('string')
|
|
|
|
const content = sentBody?.content as WritableNotebook
|
|
for (const cell of content.cells) {
|
|
expect(cell).not.toHaveProperty('id')
|
|
expect(cell).not.toHaveProperty('_id')
|
|
}
|
|
|
|
const [, databaseCell, logCell] = content.cells as Array<Record<string, unknown>>
|
|
expect(databaseCell.sql).toBe(DATABASE_SQL)
|
|
expect(logCell.sql).toBe(LOG_SQL)
|
|
})
|
|
|
|
it('rejects malformed content without making a network request', async () => {
|
|
await expect(
|
|
createNotebook({ projectRef: 'default', name: 'Bad notebook', content: INVALID_CONTENT })
|
|
).rejects.toThrow()
|
|
})
|
|
|
|
it('strips an empty-string database_identifier before sending — a caller that skipped the agent schema is not trusted to have already normalized it', async () => {
|
|
let sentBody: Record<string, unknown> | undefined
|
|
addAPIMock({
|
|
method: 'put',
|
|
path: '/platform/projects/:ref/content',
|
|
response: async ({ request }) => {
|
|
sentBody = (await request.json()) as Record<string, unknown>
|
|
return HttpResponse.json(null)
|
|
},
|
|
})
|
|
|
|
await createNotebook({
|
|
projectRef: 'default',
|
|
name: 'Signup funnel',
|
|
content: {
|
|
schema_version: 1,
|
|
cells: [
|
|
{ _tag: 'database_cell', sql: DATABASE_SQL, row_limit: 100, database_identifier: '' },
|
|
],
|
|
},
|
|
})
|
|
|
|
const content = sentBody?.content as WritableNotebook
|
|
const [databaseCell] = content.cells as Array<Record<string, unknown>>
|
|
expect(databaseCell).not.toHaveProperty('database_identifier')
|
|
})
|
|
})
|
|
|
|
describe('upsertNotebook', () => {
|
|
const NOTEBOOK_ID = 'd3aadd77-7c3c-4de7-aa5c-5aa8ac270b44'
|
|
|
|
it('PUTs with the given id, keeping the existing cell id and leaving the new cell without one', async () => {
|
|
let sentBody: Record<string, unknown> | undefined
|
|
addAPIMock({
|
|
method: 'put',
|
|
path: '/platform/projects/:ref/content',
|
|
response: async ({ request }) => {
|
|
sentBody = (await request.json()) as Record<string, unknown>
|
|
return HttpResponse.json(null)
|
|
},
|
|
})
|
|
|
|
await upsertNotebook({
|
|
projectRef: 'default',
|
|
id: NOTEBOOK_ID,
|
|
name: 'Signup funnel',
|
|
content: UPDATE_CONTENT,
|
|
})
|
|
|
|
expect(sentBody?.id).toBe(NOTEBOOK_ID)
|
|
expect(sentBody?.type).toBe('notebook')
|
|
|
|
const content = sentBody?.content as WritableNotebook
|
|
const [databaseCell, logCell] = content.cells as Array<Record<string, unknown>>
|
|
expect(databaseCell._id).toBe(EXISTING_CELL_ID)
|
|
expect(databaseCell).not.toHaveProperty('id')
|
|
expect(databaseCell.sql).toBe(DATABASE_SQL)
|
|
expect(logCell).not.toHaveProperty('id')
|
|
expect(logCell).not.toHaveProperty('_id')
|
|
expect(logCell.sql).toBe(LOG_SQL)
|
|
})
|
|
|
|
it('rejects malformed content without making a network request', async () => {
|
|
await expect(
|
|
upsertNotebook({
|
|
projectRef: 'default',
|
|
id: NOTEBOOK_ID,
|
|
name: 'Bad notebook',
|
|
content: INVALID_CONTENT,
|
|
})
|
|
).rejects.toThrow()
|
|
})
|
|
})
|