diff --git a/.github/workflows/braintrust-evals.yml b/.github/workflows/braintrust-evals.yml new file mode 100644 index 00000000000..915d2e3bb38 --- /dev/null +++ b/.github/workflows/braintrust-evals.yml @@ -0,0 +1,60 @@ +name: Run Braintrust evals + +on: + push: + branches: [master] + pull_request: + types: [opened, synchronize, labeled] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + pull-requests: write + contents: read + +jobs: + eval: + name: Run evals + if: github.event_name == 'push' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-evals')) + runs-on: ubuntu-latest + timeout-minutes: 20 + + env: + BRAINTRUST_PROJECT_ID: ${{ secrets.BRAINTRUST_PROJECT_ID }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + + steps: + - name: Checkout + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + fetch-depth: 0 + # For PR events, checkout the actual branch so Braintrust can report the correct branch name instead of detached HEAD. + # github.head_ref is the PR source branch, github.ref_name is the fallback for push events (e.g., master). + ref: ${{ github.head_ref || github.ref_name }} + + - name: Install pnpm + uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0 + with: + run_install: false + + - name: Use Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version-file: ".nvmrc" + cache: "pnpm" + + - name: Install Dependencies + run: pnpm install --frozen-lockfile + + - name: Setup Evals + run: cd apps/studio && pnpm evals:setup + + - name: Run Evals + uses: braintrustdata/eval-action@v1 + with: + api_key: ${{ secrets.BRAINTRUST_API_KEY }} + runtime: node + package_manager: pnpm + root: apps/studio diff --git a/apps/studio/evals/.gitignore b/apps/studio/evals/.gitignore new file mode 100644 index 00000000000..8085866a958 --- /dev/null +++ b/apps/studio/evals/.gitignore @@ -0,0 +1 @@ +libpg-query.wasm \ No newline at end of file diff --git a/apps/studio/evals/assistant.eval.ts b/apps/studio/evals/assistant.eval.ts new file mode 100644 index 00000000000..d2c27f40303 --- /dev/null +++ b/apps/studio/evals/assistant.eval.ts @@ -0,0 +1,123 @@ +import { openai } from '@ai-sdk/openai' +import { Eval } from 'braintrust' +import { generateAssistantResponse } from 'lib/ai/generate-assistant-response' +import { getMockTools } from 'lib/ai/tools/mock-tools' +import assert from 'node:assert' +import { dataset } from './dataset' +import { + completenessScorer, + concisenessScorer, + goalCompletionScorer, + sqlSyntaxScorer, + toolUsageScorer, +} from './scorer' +import { ToolSet, TypedToolCall, TypedToolResult } from 'ai' + +assert(process.env.BRAINTRUST_PROJECT_ID, 'BRAINTRUST_PROJECT_ID is not set') +assert(process.env.OPENAI_API_KEY, 'OPENAI_API_KEY is not set') + +Eval('Assistant', { + projectId: process.env.BRAINTRUST_PROJECT_ID, + data: () => dataset, + task: async (input) => { + const result = await generateAssistantResponse({ + model: openai('gpt-5-mini'), + messages: [{ id: '1', role: 'user', parts: [{ type: 'text', text: input }] }], + tools: await getMockTools(), + }) + + // `result.toolCalls` only shows the last step, instead aggregate tools across all steps + const steps = await result.steps + + const stepsSerialized = steps + .map((step) => { + const toolCalls = step.toolCalls + ?.map((call) => JSON.stringify({ tool: call.toolName, input: call.input })) + .join('\n') + + const text = step.text + return toolCalls ? `${text}\n${toolCalls}` : text + }) + .join('\n') + + const toolNames: string[] = [] + const sqlQueries: string[] = [] + const docs: string[] = [] + + for (const step of steps) { + for (const [i, toolCall] of step.toolCalls.entries()) { + toolNames.push(toolCall.toolName) + + const toolResult = step.toolResults.at(i) + if (!toolResult) { + continue + } + + const parsed = parseToolCall(toolCall, toolResult) + + if (parsed.sqlQuery) { + sqlQueries.push(parsed.sqlQuery) + } + if (parsed.docs) { + docs.push(...parsed.docs) + } + } + } + + return { + stepsSerialized, + toolNames, + sqlQueries, + docs, + } + }, + scores: [ + toolUsageScorer, + sqlSyntaxScorer, + goalCompletionScorer, + concisenessScorer, + completenessScorer, + ], +}) + +type ParsedToolCall = { + /** Query generated by `execute_sql` */ + sqlQuery?: string + + /** Docs text pulled in from `search_docs` */ + docs?: string[] +} + +/** + * Validate and extract relevant info from a tool call/result + */ +function parseToolCall( + toolCall: TypedToolCall, + toolResult: TypedToolResult +): ParsedToolCall { + switch (toolCall.toolName) { + case 'execute_sql': { + const sqlQuery = toolCall.input.sql + if (typeof sqlQuery !== 'string') { + return {} + } + + return { sqlQuery } + } + case 'search_docs': { + const content = toolResult.output.content + if (!content || !Array.isArray(content)) { + return {} + } + + const docs = content.map((item) => item?.text).filter((text) => typeof text === 'string') + if (docs.length === 0) { + return {} + } + + return { docs } + } + } + + return {} +} diff --git a/apps/studio/evals/dataset.ts b/apps/studio/evals/dataset.ts new file mode 100644 index 00000000000..58c31926091 --- /dev/null +++ b/apps/studio/evals/dataset.ts @@ -0,0 +1,38 @@ +import { AssistantEvalCase } from './scorer' + +export const dataset: AssistantEvalCase[] = [ + { + input: 'How do I run WASM in edge functions? Use `search_docs`.', + expected: { requiredTools: ['search_docs'] }, + metadata: { category: ['general_help'] }, + }, + { + input: 'Check if my project is having issues right now and tell me what to fix first.', + expected: { + requiredTools: ['get_advisors', 'get_logs'], + }, + metadata: { category: ['debugging', 'rls_policies'] }, + }, + { + input: 'Create a new table "foods" with columns for "name" and "color"', + expected: { + requiredTools: ['execute_sql'], + }, + metadata: { category: ['sql_generation', 'schema_design'] }, + }, + { + input: + 'Write a SQL query to select all products from the products table where the price is greater than 100', + expected: { + requiredTools: ['execute_sql'], + }, + metadata: { category: ['sql_generation'] }, + }, + { + input: 'Create an index on the products table for the name column', + expected: { + requiredTools: ['execute_sql'], + }, + metadata: { category: ['sql_generation', 'database_optimization'] }, + }, +] diff --git a/apps/studio/evals/scorer.ts b/apps/studio/evals/scorer.ts new file mode 100644 index 00000000000..62f68f41d7c --- /dev/null +++ b/apps/studio/evals/scorer.ts @@ -0,0 +1,159 @@ +import { EvalCase, EvalScorer } from 'braintrust' +import { LLMClassifierFromTemplate } from 'autoevals' +import { stripIndent } from 'common-tags' +import { parse } from 'libpg-query' + +const LLM_AS_A_JUDGE_MODEL = 'gpt-5.2-2025-12-11' + +type Input = string + +type Output = { + stepsSerialized: string + toolNames: string[] + sqlQueries: string[] + docs: string[] +} + +export type Expected = { + requiredTools?: string[] +} + +// Based on categories in the AssistantMessageRatingSubmittedEvent +export type AssistantEvalCaseCategory = + | 'sql_generation' + | 'schema_design' + | 'rls_policies' + | 'edge_functions' + | 'database_optimization' + | 'debugging' + | 'general_help' + | 'other' + +export type AssistantEvalCaseMetadata = { + category?: AssistantEvalCaseCategory[] +} + +export type AssistantEvalCase = EvalCase + +export const toolUsageScorer: EvalScorer = async ({ + output, + expected, +}) => { + if (!expected.requiredTools) return null + + const presentCount = expected.requiredTools.filter((tool) => + output.toolNames.includes(tool) + ).length + const totalCount = expected.requiredTools.length + const ratio = totalCount === 0 ? 1 : presentCount / totalCount + + return { + name: 'Tool Usage', + score: ratio, + } +} + +export const sqlSyntaxScorer: EvalScorer = async ({ output }) => { + if (output.sqlQueries === undefined || output.sqlQueries.length === 0) { + return null + } + + const errors: string[] = [] + let validQueries = 0 + + for (const sql of output.sqlQueries) { + try { + await parse(sql) + validQueries++ + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error) + errors.push(`SQL syntax error: ${errorMessage}`) + } + } + + return { + name: 'SQL Validity', + score: validQueries / output.sqlQueries.length, + metadata: errors.length > 0 ? { errors } : undefined, + } +} + +const concisenessEvaluator = LLMClassifierFromTemplate<{ input: string }>({ + name: 'Conciseness', + promptTemplate: stripIndent` + Evaluate the conciseness of this response. + + Input: {{input}} + Output: {{output}} + + Is the response concise and free of unnecessary words? + a) Very concise - no wasted words + b) Acceptable verbosity - some verbosity but acceptable + c) Too verbose - contains superfluous wording or overly verbose + `, + choiceScores: { a: 1, b: 0.5, c: 0 }, + useCoT: true, + model: LLM_AS_A_JUDGE_MODEL, +}) + +export const concisenessScorer: EvalScorer = async ({ input, output }) => { + return await concisenessEvaluator({ + input, + output: output.stepsSerialized, + }) +} + +const completenessEvaluator = LLMClassifierFromTemplate<{ input: string }>({ + name: 'Completeness', + promptTemplate: stripIndent` + Evaluate whether this response is complete and finished, or if it appears cut off or incomplete. + + Input: {{input}} + Output: {{output}} + + Does the response appear complete and finished? + a) Complete - response is complete and finished + b) Incomplete - response appears cut off, missing parts, or severely incomplete + `, + choiceScores: { a: 1, b: 0 }, + useCoT: true, + model: LLM_AS_A_JUDGE_MODEL, +}) + +export const completenessScorer: EvalScorer = async ({ + input, + output, +}) => { + return await completenessEvaluator({ + input, + output: output.stepsSerialized, + }) +} + +const goalCompletionEvaluator = LLMClassifierFromTemplate<{ input: string }>({ + name: 'Goal Completion', + promptTemplate: stripIndent` + Evaluate whether this response addresses what the user asked. + + Input: {{input}} + Output: {{output}} + + Does the response address what the user asked? + a) Fully addresses - completely answers the question or fulfills the request + b) Partially addresses - addresses some aspects but misses key parts + c) Doesn't address - off-topic or fails to address the request + `, + choiceScores: { a: 1, b: 0.5, c: 0 }, + useCoT: true, + model: LLM_AS_A_JUDGE_MODEL, +}) + +export const goalCompletionScorer: EvalScorer = async ({ + input, + output, +}) => { + return await goalCompletionEvaluator({ + input, + output: output.stepsSerialized, + }) +} diff --git a/apps/studio/lib/ai/generate-assistant-response.ts b/apps/studio/lib/ai/generate-assistant-response.ts new file mode 100644 index 00000000000..1f35a8223d1 --- /dev/null +++ b/apps/studio/lib/ai/generate-assistant-response.ts @@ -0,0 +1,130 @@ +import * as ai from 'ai' +import { + convertToModelMessages, + isToolUIPart, + type LanguageModel, + type ModelMessage, + stepCountIs, + type ToolSet, + type UIMessage, +} from 'ai' +import { wrapAISDK } from 'braintrust' +import { source } from 'common-tags' + +import type { AiOptInLevel } from 'hooks/misc/useOrgOptedIntoAi' +import { + CHAT_PROMPT, + EDGE_FUNCTION_PROMPT, + GENERAL_PROMPT, + PG_BEST_PRACTICES, + RLS_PROMPT, + REALTIME_PROMPT, + SECURITY_PROMPT, + LIMITATIONS_PROMPT, +} from 'lib/ai/prompts' +import { sanitizeMessagePart } from 'lib/ai/tools/tool-sanitizer' + +const { streamText } = wrapAISDK(ai) + +export async function generateAssistantResponse({ + messages: rawMessages, + model, + tools, + aiOptInLevel = 'schema', + getSchemas, + projectRef, + chatName, + promptProviderOptions, + providerOptions, + abortSignal, +}: { + messages: UIMessage[] + model: LanguageModel + tools: ToolSet + aiOptInLevel?: AiOptInLevel + getSchemas?: () => Promise + projectRef?: string + chatName?: string + promptProviderOptions?: Record + providerOptions?: Record + abortSignal?: AbortSignal +}) { + // Only returns last 7 messages + // Filters out tools with invalid states + // Filters out tool outputs based on opt-in level using renderingToolOutputParser + const messages = (rawMessages || []).slice(-7).map((msg) => { + if (msg && msg.role === 'assistant' && 'results' in msg) { + const cleanedMsg = { ...msg } + delete cleanedMsg.results + return cleanedMsg + } + if (msg && msg.role === 'assistant' && msg.parts) { + const cleanedParts = msg.parts + .filter((part) => { + if (isToolUIPart(part)) { + const invalidStates = ['input-streaming', 'input-available', 'output-error'] + return !invalidStates.includes(part.state) + } + return true + }) + .map((part) => { + return sanitizeMessagePart(part, aiOptInLevel) + }) + return { ...msg, parts: cleanedParts } + } + return msg + }) + + const schemasString = + aiOptInLevel !== 'disabled' && getSchemas + ? await getSchemas() + : "You don't have access to any schemas." + + // Important: do not use dynamic content in the system prompt or Bedrock will not cache it + const system = source` + ${GENERAL_PROMPT} + ${CHAT_PROMPT} + ${PG_BEST_PRACTICES} + ${RLS_PROMPT} + ${EDGE_FUNCTION_PROMPT} + ${REALTIME_PROMPT} + ${SECURITY_PROMPT} + ${LIMITATIONS_PROMPT} + ` + + // Note: these must be of type `CoreMessage` to prevent AI SDK from stripping `providerOptions` + // https://github.com/vercel/ai/blob/81ef2511311e8af34d75e37fc8204a82e775e8c3/packages/ai/core/prompt/standardize-prompt.ts#L83-L88 + const assistantContent = + projectRef || chatName || schemasString !== "You don't have access to any schemas." + ? `The user's current project is ${projectRef || 'unknown'}. Their available schemas are: ${schemasString}. The current chat name is: ${chatName || 'unnamed'}` + : undefined + + const coreMessages: ModelMessage[] = [ + { + role: 'system', + content: system, + ...(promptProviderOptions && { + providerOptions: promptProviderOptions, + }), + }, + ...(assistantContent + ? [ + { + role: 'assistant' as const, + // Add any dynamic context here + content: assistantContent, + }, + ] + : []), + ...convertToModelMessages(messages), + ] + + return streamText({ + model, + stopWhen: stepCountIs(5), + messages: coreMessages, + ...(providerOptions && { providerOptions }), + tools, + ...(abortSignal && { abortSignal }), + }) +} diff --git a/apps/studio/lib/ai/tools/mock-tools.ts b/apps/studio/lib/ai/tools/mock-tools.ts new file mode 100644 index 00000000000..7802112589f --- /dev/null +++ b/apps/studio/lib/ai/tools/mock-tools.ts @@ -0,0 +1,324 @@ +import { tool, type ToolSet } from 'ai' +import { getRenderingTools } from '../tools/rendering-tools' +import { z } from 'zod' +import { getMcpTools } from 'lib/ai/tools/mcp-tools' +import assert from 'node:assert' + +const listTablesInputSchema = z.object({ + schemas: z.array(z.string()).describe('The schema names to list.'), +}) + +const getAdvisorsInputSchema = z.object({ + type: z.enum(['security', 'performance']).optional(), +}) + +const getLogsInputSchema = z.object({ + limit: z.number().min(1).max(100).optional(), + level: z.enum(['debug', 'info', 'warning', 'error']).optional(), + source: z.enum(['postgres', 'auth', 'storage', 'edge_function']).optional(), + search: z.string().optional(), +}) + +const listPoliciesInputSchema = z.object({ + schemas: z.array(z.string()).describe('The schema names to get the policies for'), +}) + +const MOCK_TABLES_DATA = [ + { + name: 'user_documents', + rls_enabled: false, + columns: [ + { name: 'id', data_type: 'bigint' }, + { name: 'user_id', data_type: 'uuid' }, + { name: 'title', data_type: 'text' }, + ], + }, + { + name: 'customers', + rls_enabled: true, + columns: [ + { name: 'id', data_type: 'uuid' }, + { name: 'tenant_id', data_type: 'uuid' }, + { name: 'email', data_type: 'text' }, + ], + }, + { + name: 'projects', + rls_enabled: false, + columns: [ + { name: 'id', data_type: 'uuid' }, + { name: 'organization_id', data_type: 'uuid' }, + { name: 'name', data_type: 'text' }, + ], + }, + { + name: 'user_organizations', + rls_enabled: true, + columns: [ + { name: 'user_id', data_type: 'uuid' }, + { name: 'organization_id', data_type: 'uuid' }, + ], + }, +] + +const MOCK_EXTENSIONS_DATA = [ + { name: 'pgcrypto', schema: 'extensions', installed_version: '1.3' }, + { name: 'uuid-ossp', schema: 'extensions', installed_version: '1.1' }, +] + +const MOCK_EDGE_FUNCTIONS_DATA = [ + { name: 'hello-world', last_deployed_at: '2024-06-10T12:30:00Z' }, + { name: 'daily-metrics-sync', last_deployed_at: '2024-06-18T08:15:00Z' }, + { name: 'select-from-table-with-auth-rls', last_deployed_at: '2024-06-19T09:20:00Z' }, +] + +const MOCK_ADVISORIES_DATA = [ + { + id: '0016_materialized_view_in_api', + level: 'warning', + category: 'security', + message: 'Materialized views in API schema can bypass RLS. Move them to private schema.', + remediationUrl: + 'https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0016_materialized_view_in_api', + }, + { + id: '0031_functions_no_rls_guard', + level: 'notice', + category: 'security', + message: 'Function api.health_check should verify auth context before querying tables.', + remediationUrl: + 'https://supabase.com/docs/guides/database/database-advisors?queryGroups=lint&lint=0031_functions_no_rls_guard', + }, + { + id: '1012_slow_query', + level: 'info', + category: 'performance', + message: + 'Query on table edge_function_logs exceeded 3s average execution time over the last hour.', + remediationUrl: 'https://supabase.com/docs/guides/platform/performance-advisors#slow-queries', + }, +] + +const MOCK_LOGS_DATA = [ + { + id: 'log-001', + timestamp: '2024-06-20T14:12:00Z', + level: 'error', + source: 'edge_function' as const, + target: 'hello-world', + message: "TypeError: fetch failed at await supabase.functions.invoke('analytics')", + }, + { + id: 'log-002', + timestamp: '2024-06-20T14:05:30Z', + level: 'warning', + source: 'postgres' as const, + target: 'connection_pool', + message: 'Query timeout exceeded for statement SELECT * FROM public.audit_log_entries', + }, + { + id: 'log-003', + timestamp: '2024-06-20T13:59:10Z', + level: 'info', + source: 'edge_function' as const, + target: 'daily-metrics-sync', + message: 'Invocation completed in 520ms', + }, + { + id: 'log-004', + timestamp: '2024-06-20T13:50:00Z', + level: 'error', + source: 'postgres' as const, + target: 'trigger:refresh_materialized_views', + message: 'permission denied for relation user_documents', + }, + { + id: 'log-005', + timestamp: '2024-06-20T13:45:00Z', + level: 'info', + source: 'auth' as const, + target: 'email-confirmation', + message: 'Sent verification email to alex@example.com', + }, +] + +function createMockedRenderingTools() { + const renderingTools = getRenderingTools() + + return Object.fromEntries( + Object.entries(renderingTools).map(([name, baseTool]) => { + if (typeof baseTool.execute === 'function') { + return [name, baseTool] + } + + const statusMessage = + name === 'execute_sql' + ? 'SQL execution mocked successfully.' + : name === 'deploy_edge_function' + ? 'Edge Function deployment mocked successfully.' + : 'Tool call mocked successfully.' + + return [ + name, + { + ...baseTool, + execute: async () => ({ status: statusMessage }), + }, + ] + }) + ) as typeof renderingTools +} + +function createMockListTablesTool() { + return tool({ + description: 'Lists tables and columns for the provided schemas.', + inputSchema: listTablesInputSchema, + execute: async ({ schemas }: { schemas: string[] }) => { + const effectiveSchemas = schemas?.length ? schemas : ['public'] + return effectiveSchemas.map((schema) => ({ + schema, + tables: MOCK_TABLES_DATA, + })) + }, + }) +} + +function createMockListExtensionsTool() { + return tool({ + description: 'Lists installed database extensions.', + inputSchema: z.object({}), + execute: async () => { + return MOCK_EXTENSIONS_DATA + }, + }) +} + +function createMockListEdgeFunctionsTool() { + return tool({ + description: 'Lists available Supabase Edge Functions.', + inputSchema: z.object({}), + execute: async () => { + return MOCK_EDGE_FUNCTIONS_DATA + }, + }) +} + +function createMockGetAdvisorsTool() { + return tool({ + description: 'Returns advisory notices for the project (mocked).', + inputSchema: getAdvisorsInputSchema, + execute: async ({ type }: { type?: 'security' | 'performance' }) => { + if (type) { + return MOCK_ADVISORIES_DATA.filter((advisory) => advisory.category === type) + } + return MOCK_ADVISORIES_DATA + }, + }) +} + +function createMockGetLogsTool() { + return tool({ + description: 'Fetches recent project logs for debugging or health checks (mocked).', + inputSchema: getLogsInputSchema, + execute: async ({ + limit = 10, + level, + source, + search, + }: { + limit?: number + level?: 'debug' | 'info' | 'warning' | 'error' + source?: 'postgres' | 'auth' | 'storage' | 'edge_function' + search?: string + }) => { + let filtered = MOCK_LOGS_DATA + + if (level) { + filtered = filtered.filter((entry) => entry.level === level) + } + + if (source) { + filtered = filtered.filter((entry) => entry.source === source) + } + + if (search) { + const needle = search.toLowerCase() + filtered = filtered.filter((entry) => + `${entry.message} ${entry.target}`.toLowerCase().includes(needle) + ) + } + + return filtered.slice(0, limit) + }, + }) +} + +function createMockListPoliciesTool() { + return tool({ + description: 'Get existing RLS policies for provided schemas.', + inputSchema: listPoliciesInputSchema, + execute: async ({ schemas }: { schemas: string[] }) => { + const effectiveSchemas = schemas?.length ? schemas : ['public'] + const results = [] as Array<{ + schema: string + table: string + policies: Array<{ + name: string + command: 'select' | 'insert' | 'update' | 'delete' + using?: string + check?: string + }> + }> + + for (const schema of effectiveSchemas) { + if (schema !== 'public') continue + results.push( + { + schema, + table: 'customers', + policies: [ + { + name: 'customers_tenant_select', + command: 'select', + using: "(auth.jwt() ->> 'tenant_id')::uuid = tenant_id", + }, + ], + }, + { schema, table: 'user_documents', policies: [] }, + { schema, table: 'projects', policies: [] } + ) + } + return results + }, + }) +} + +/** + * Deterministic mock implementations of MCP/platform tools for evals. + * These mirror tool names used in prompts so the model can call them, + * but return stable, static data for repeatable tests. + * + * Note: search_docs uses the real implementation + */ +export async function getMockTools() { + const mockedRenderingTools = createMockedRenderingTools() + + const { search_docs } = await getMcpTools({ + accessToken: 'mock-access-token', + projectRef: 'mock-project-ref', + aiOptInLevel: 'schema_and_log_and_data', + }) + + assert(search_docs, 'search_docs tool not available from MCP server') + + return { + ...mockedRenderingTools, + search_docs, + list_tables: createMockListTablesTool(), + list_extensions: createMockListExtensionsTool(), + list_edge_functions: createMockListEdgeFunctionsTool(), + get_advisors: createMockGetAdvisorsTool(), + get_logs: createMockGetLogsTool(), + list_policies: createMockListPoliciesTool(), + } +} diff --git a/apps/studio/lib/api/generate-v4.test.ts b/apps/studio/lib/api/generate-v4.test.ts index a04be6ef01a..d584b7e5222 100644 --- a/apps/studio/lib/api/generate-v4.test.ts +++ b/apps/studio/lib/api/generate-v4.test.ts @@ -2,9 +2,10 @@ import { expect, test, vi } from 'vitest' // End of third-party imports import generateV4 from '../../pages/api/ai/sql/generate-v4' -import { sanitizeMessagePart } from '../ai/tools/tool-sanitizer' +import { sanitizeMessagePart } from 'lib/ai/tools/tool-sanitizer' +import { UIMessage } from 'ai' -vi.mock('../ai/tools/tool-sanitizer', () => ({ +vi.mock('lib/ai/tools/tool-sanitizer', () => ({ sanitizeMessagePart: vi.fn((part) => part), })) @@ -17,20 +18,24 @@ test('generateV4 calls the tool sanitizer', async () => { body: { messages: [ { + id: 'test-msg-id', role: 'assistant', parts: [ { type: 'tool-execute_sql', state: 'output-available', - output: 'test output', + toolCallId: 'test-tool-call-id', + input: { sql: 'SELECT * FROM users' }, + output: [{ id: 1, name: 'test-output' }], }, ], }, - ], + ] satisfies UIMessage[], projectRef: 'test-project', connectionString: 'test-connection', orgSlug: 'test-org', }, + on: vi.fn(), } const mockRes = { @@ -63,13 +68,15 @@ test('generateV4 calls the tool sanitizer', async () => { getTools: vi.fn().mockResolvedValue({}), })) - vi.mock('ai', () => ({ - streamText: vi.fn().mockReturnValue({ - pipeUIMessageStreamToResponse: vi.fn(), - }), - convertToModelMessages: vi.fn((msgs) => msgs), - stepCountIs: vi.fn(), - })) + vi.mock('ai', async () => { + const actual = await vi.importActual('ai') + return { + ...actual, + streamText: vi.fn().mockReturnValue({ + pipeUIMessageStreamToResponse: vi.fn(), + }), + } + }) await generateV4(mockReq as any, mockRes as any) diff --git a/apps/studio/package.json b/apps/studio/package.json index aafb78a3d81..58684683e56 100644 --- a/apps/studio/package.json +++ b/apps/studio/package.json @@ -23,7 +23,10 @@ "prettier:write": "prettier --cache --write .", "build:deno-types": "tsx scripts/deno-types.ts", "build:graphql-types": "tsx scripts/download-graphql-schema.mts && pnpm graphql-codegen --config scripts/codegen.ts", - "build:graphql-types:watch": "pnpm graphql-codegen --config scripts/codegen.ts --watch" + "build:graphql-types:watch": "pnpm graphql-codegen --config scripts/codegen.ts --watch", + "evals:setup": "cp node_modules/libpg-query/wasm/libpg-query.wasm evals/libpg-query.wasm", + "evals:run": "braintrust eval --no-send-logs evals/assistant.eval.ts", + "evals:upload": "braintrust eval evals/assistant.eval.ts" }, "dependencies": { "@ai-sdk/amazon-bedrock": "^3.0.0", @@ -183,6 +186,8 @@ "@vitest/coverage-v8": "^3.2.0", "@vitest/ui": "^3.2.0", "api-types": "workspace:*", + "autoevals": "^0.0.131", + "braintrust": "^1.0.2", "common": "workspace:*", "config": "workspace:*", "date-fns": "^2.30.0", @@ -191,6 +196,7 @@ "graphql-ws": "5.14.1", "import-in-the-middle": "^2.0.0", "jsdom-testing-mocks": "^1.13.1", + "libpg-query": "17.6.0", "msw": "^2.3.0", "next-router-mock": "^0.9.13", "node-mocks-http": "^1.17.2", diff --git a/apps/studio/pages/api/ai/sql/generate-v4.ts b/apps/studio/pages/api/ai/sql/generate-v4.ts index ecd441f5030..93aa261b14c 100644 --- a/apps/studio/pages/api/ai/sql/generate-v4.ts +++ b/apps/studio/pages/api/ai/sql/generate-v4.ts @@ -1,6 +1,5 @@ import pgMeta from '@supabase/pg-meta' -import { convertToModelMessages, type ModelMessage, stepCountIs, streamText } from 'ai' -import { source } from 'common-tags' +import { safeValidateUIMessages } from 'ai' import type { NextApiRequest, NextApiResponse } from 'next' import z from 'zod' @@ -9,18 +8,8 @@ import { executeSql } from 'data/sql/execute-sql-query' import type { AiOptInLevel } from 'hooks/misc/useOrgOptedIntoAi' import { getModel } from 'lib/ai/model' import { getOrgAIDetails } from 'lib/ai/org-ai-details' -import { - CHAT_PROMPT, - EDGE_FUNCTION_PROMPT, - GENERAL_PROMPT, - PG_BEST_PRACTICES, - RLS_PROMPT, - REALTIME_PROMPT, - SECURITY_PROMPT, - LIMITATIONS_PROMPT, -} from 'lib/ai/prompts' +import { generateAssistantResponse } from 'lib/ai/generate-assistant-response' import { getTools } from 'lib/ai/tools' -import { sanitizeMessagePart } from 'lib/ai/tools/tool-sanitizer' import apiWrapper from 'lib/api/apiWrapper' import { executeQuery } from 'lib/api/self-hosted/query' @@ -89,6 +78,14 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) { model: requestedModel, } = data + const messagesValidation = await safeValidateUIMessages({ messages: rawMessages }) + if (!messagesValidation.success) { + return res + .status(400) + .json({ error: 'Invalid request body', message: messagesValidation.error.message }) + } + const messages = messagesValidation.data + let aiOptInLevel: AiOptInLevel = 'disabled' let isLimited = false @@ -114,32 +111,6 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) { } } - // Only returns last 7 messages - // Filters out tools with invalid states - // Filters out tool outputs based on opt-in level using renderingToolOutputParser - const messages = (rawMessages || []).slice(-7).map((msg: any) => { - if (msg && msg.role === 'assistant' && 'results' in msg) { - const cleanedMsg = { ...msg } - delete cleanedMsg.results - return cleanedMsg - } - if (msg && msg.role === 'assistant' && msg.parts) { - const cleanedParts = msg.parts - .filter((part: any) => { - if (part.type.startsWith('tool-')) { - const invalidStates = ['input-streaming', 'input-available', 'output-error'] - return !invalidStates.includes(part.state) - } - return true - }) - .map((part: any) => { - return sanitizeMessagePart(part, aiOptInLevel) - }) - return { ...msg, parts: cleanedParts } - } - return msg - }) - const { model, error: modelError, @@ -157,67 +128,10 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) { } try { - // Get a list of all schemas to add to context - const pgMetaSchemasList = pgMeta.schemas.list() - type Schemas = z.infer<(typeof pgMetaSchemasList)['zod']> - - const { result: schemas } = - aiOptInLevel !== 'disabled' - ? await executeSql( - { - projectRef, - connectionString, - sql: pgMetaSchemasList.sql, - }, - undefined, - { - 'Content-Type': 'application/json', - ...(authorization && { Authorization: authorization }), - }, - IS_PLATFORM ? undefined : executeQuery - ) - : { result: [] } - - const schemasString = - schemas?.length > 0 - ? `The available database schema names are: ${JSON.stringify(schemas)}` - : "You don't have access to any schemas." - - // Important: do not use dynamic content in the system prompt or Bedrock will not cache it - const system = source` - ${GENERAL_PROMPT} - ${CHAT_PROMPT} - ${PG_BEST_PRACTICES} - ${RLS_PROMPT} - ${EDGE_FUNCTION_PROMPT} - ${REALTIME_PROMPT} - ${SECURITY_PROMPT} - ${LIMITATIONS_PROMPT} - ` - - // Note: these must be of type `CoreMessage` to prevent AI SDK from stripping `providerOptions` - // https://github.com/vercel/ai/blob/81ef2511311e8af34d75e37fc8204a82e775e8c3/packages/ai/core/prompt/standardize-prompt.ts#L83-L88 - const coreMessages: ModelMessage[] = [ - { - role: 'system', - content: system, - ...(promptProviderOptions && { - providerOptions: promptProviderOptions, - }), - }, - { - role: 'assistant', - // Add any dynamic context here - content: `The user's current project is ${projectRef}. Their available schemas are: ${schemasString}. The current chat name is: ${chatName}`, - }, - ...convertToModelMessages(messages), - ] - const abortController = new AbortController() req.on('close', () => abortController.abort()) req.on('aborted', () => abortController.abort()) - // Get tools const tools = await getTools({ projectRef, connectionString, @@ -226,12 +140,40 @@ async function handlePost(req: NextApiRequest, res: NextApiResponse) { accessToken, }) - const result = streamText({ + // Get a list of all schemas to add to context + const getSchemas = async (): Promise => { + const pgMetaSchemasList = pgMeta.schemas.list() + type Schemas = z.infer<(typeof pgMetaSchemasList)['zod']> + + const { result: schemas } = await executeSql( + { + projectRef, + connectionString, + sql: pgMetaSchemasList.sql, + }, + undefined, + { + 'Content-Type': 'application/json', + ...(authorization && { Authorization: authorization }), + }, + IS_PLATFORM ? undefined : executeQuery + ) + + return schemas?.length > 0 + ? `The available database schema names are: ${JSON.stringify(schemas)}` + : "You don't have access to any schemas." + } + + const result = await generateAssistantResponse({ + messages, model, - stopWhen: stepCountIs(5), - messages: coreMessages, - ...(providerOptions && { providerOptions }), tools, + aiOptInLevel, + getSchemas: aiOptInLevel !== 'disabled' ? getSchemas : undefined, + projectRef, + chatName, + promptProviderOptions, + providerOptions, abortSignal: abortController.signal, }) diff --git a/package.json b/package.json index ac42d0ed66b..a35f01b65ab 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "prettier-plugin-sql-cst": "^0.11.0", "rimraf": "^6.0.0", "sass": "^1.72.0", - "supabase": "^2.58.5", + "supabase": "^2.65.6", "supports-color": "^8.0.0", "tailwindcss": "catalog:", "turbo": "2.3.3", diff --git a/packages/shared-data/package.json b/packages/shared-data/package.json index 2533f1c5331..9aacccfae9e 100644 --- a/packages/shared-data/package.json +++ b/packages/shared-data/package.json @@ -8,6 +8,9 @@ "preinstall": "npx only-allow pnpm", "clean": "rimraf node_modules" }, + "devDependencies": { + "tsconfig": "workspace:" + }, "author": "", "license": "MIT" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1febf67dc73..a046f17af6c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6,12 +6,63 @@ settings: catalogs: default: + '@sentry/nextjs': + specifier: ^10.26.0 + version: 10.27.0 + '@supabase/auth-js': + specifier: 2.87.0 + version: 2.87.0 + '@supabase/postgrest-js': + specifier: 2.87.0 + version: 2.87.0 + '@supabase/realtime-js': + specifier: 2.87.0 + version: 2.87.0 '@supabase/supabase-js': specifier: 2.87.0 version: 2.87.0 + '@types/node': + specifier: ^22.0.0 + version: 22.13.14 + '@types/react': + specifier: ^18.3.0 + version: 18.3.3 + '@types/react-dom': + specifier: ^18.3.0 + version: 18.3.0 + next: + specifier: ^15.5.9 + version: 15.5.9 + react: + specifier: ^18.3.0 + version: 18.3.1 + react-dom: + specifier: ^18.3.0 + version: 18.3.1 + recharts: + specifier: ^2.15.4 + version: 2.15.4 + tailwindcss: + specifier: 3.4.1 + version: 3.4.1 tsx: specifier: 4.20.3 version: 4.20.3 + typescript: + specifier: ~5.9.0 + version: 5.9.2 + valtio: + specifier: ^1.12.0 + version: 1.12.0 + vite: + specifier: ^7.1.11 + version: 7.1.11 + vitest: + specifier: ^3.2.0 + version: 3.2.4 + zod: + specifier: ^3.25.76 + version: 3.25.76 overrides: '@eslint/eslintrc>js-yaml': ^4.1.1 @@ -52,8 +103,8 @@ importers: specifier: ^1.72.0 version: 1.72.0 supabase: - specifier: ^2.58.5 - version: 2.58.5(supports-color@8.1.1) + specifier: ^2.65.6 + version: 2.67.1(supports-color@8.1.1) supports-color: specifier: ^8.0.0 version: 8.1.1 @@ -489,7 +540,7 @@ importers: version: 0.3.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1) nuqs: specifier: ^1.19.1 - version: 1.19.1(next@15.5.9(@opentelemetry/api@1.9.0)(@playwright/test@1.56.1)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.4)) + version: 1.19.1(next@15.5.9(@babel/core@7.28.4(supports-color@8.1.1))(@opentelemetry/api@1.9.0)(@playwright/test@1.56.1)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.4)) openai: specifier: ^4.75.1 version: 4.104.0(encoding@0.1.13)(ws@8.18.3)(zod@3.25.76) @@ -1170,6 +1221,12 @@ importers: api-types: specifier: workspace:* version: link:../../packages/api-types + autoevals: + specifier: ^0.0.131 + version: 0.0.131(encoding@0.1.13)(ws@8.18.3) + braintrust: + specifier: ^1.0.2 + version: 1.0.2(@aws-sdk/credential-provider-web-identity@3.830.0)(supports-color@8.1.1)(zod@3.25.76) date-fns: specifier: ^2.30.0 version: 2.30.0 @@ -1188,6 +1245,9 @@ importers: jsdom-testing-mocks: specifier: ^1.13.1 version: 1.13.1 + libpg-query: + specifier: 17.6.0 + version: 17.6.0 msw: specifier: ^2.3.0 version: 2.11.3(@types/node@22.13.14)(typescript@5.9.2) @@ -2173,7 +2233,11 @@ importers: specifier: 'catalog:' version: 3.2.4(@types/node@22.13.14)(@vitest/ui@3.2.4)(jiti@2.5.1)(jsdom@20.0.3(supports-color@8.1.1))(msw@2.11.3(@types/node@22.13.14)(typescript@5.9.2))(sass@1.77.4)(supports-color@8.1.1)(terser@5.39.0)(tsx@4.20.3)(yaml@2.8.1) - packages/shared-data: {} + packages/shared-data: + devDependencies: + tsconfig: + specifier: 'workspace:' + version: link:../tsconfig packages/tsconfig: {} @@ -2671,6 +2735,10 @@ packages: peerDependencies: zod: ^3.25.76 || ^4 + '@ai-sdk/provider@1.1.3': + resolution: {integrity: sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg==} + engines: {node: '>=18'} + '@ai-sdk/provider@2.0.0': resolution: {integrity: sha512-6o7Y2SeO9vFKB8lArHXehNuusnpddKPk7xqL7T2/b+OvXMRIXUO1rR4wcv1hAFUAT9avGZshty3Wlua/XA7TvA==} engines: {node: '>=18'} @@ -3244,6 +3312,10 @@ packages: react: ^16.8.0 || ^17 || ^18 || ^19 react-dom: ^16.8.0 || ^17 || ^18 || ^19 + '@colors/colors@1.5.0': + resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==} + engines: {node: '>=0.1.90'} + '@contentlayer2/cli@0.4.3': resolution: {integrity: sha512-ZJ+Iiu2rVI50x60XoqnrsO/Q8eqFX5AlP1L0U/3ygaAas3tnOqTzQZ1UsxYQMpJzcLok24ddlhKfQKbCMUJPiQ==} @@ -4916,6 +4988,9 @@ packages: '@next/bundle-analyzer@16.0.4': resolution: {integrity: sha512-6IajJ23QrXW5RTJj2lRHcBM8mxcEl+vgd7XXVODQG/BcJyjgIP1k5OefdRl+P80btPvHeHoV4fIgC1so25pXcg==} + '@next/env@14.2.35': + resolution: {integrity: sha512-DuhvCtj4t9Gwrx80dmz2F4t/zKQ4ktN8WrMwOuVzkJfBilwAwGr6v16M5eI8yCuZ63H9TTuEU09Iu2HqkzFPVQ==} + '@next/env@15.5.9': resolution: {integrity: sha512-4GlTZ+EJM7WaW2HEZcyU317tIQDjkQIyENDLxYJfSWlfqguN+dHkZgyQTV/7ykvobU7yEH5gKvreNrH4B6QgIg==} @@ -6190,6 +6265,9 @@ packages: '@pgsql/types@15.0.2': resolution: {integrity: sha512-K3gtnbqbSUuUVmPm143qx5Gy2EmKuooshV95yMD48EUQ1256sgZBriEfY61OWJnlzdREdqHTIOxQqpZAb7XdZg==} + '@pgsql/types@17.6.2': + resolution: {integrity: sha512-1UtbELdbqNdyOShhrVfSz3a1gDi0s9XXiQemx+6QqtsrXe62a6zOGU+vjb2GRfG5jeEokI1zBBcfD42enRv0Rw==} + '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -9749,6 +9827,15 @@ packages: '@usercentrics/cmp-browser-sdk@4.42.0': resolution: {integrity: sha512-/cik01TdeiYUV1+EasK83Ip05TDqmowM5tmWGfRDdaOrneSic5BedBdJHKya4pP2vwiYAtzDYVJe4BwPt2M16g==} + '@vercel/functions@1.6.0': + resolution: {integrity: sha512-R6FKQrYT5MZs5IE1SqeCJWxMuBdHawFcCZboKKw8p7s+6/mcd55Gx6tWmyKnQTyrSEA04NH73Tc9CbqpEle8RA==} + engines: {node: '>= 16'} + peerDependencies: + '@aws-sdk/credential-provider-web-identity': '*' + peerDependenciesMeta: + '@aws-sdk/credential-provider-web-identity': + optional: true + '@vercel/functions@2.1.0': resolution: {integrity: sha512-1gSbK9zfrbJxk1JTBVERDhLi01mK3fz+gw4GjOjZwHnqs0zsBhQA70HGVtXQX/Z3BTRMfbpAEMVDfhecRw0lDA==} engines: {node: '>= 18'} @@ -10358,6 +10445,9 @@ packages: resolution: {integrity: sha512-Hdw8qdNiqdJ8LqT0iK0sVzkFbzg6fhnQqqfWhBDxcHZvU75+B+ayzTy8x+k5Ix0Y92XOhOUlx74ps+bA6BeYMQ==} engines: {node: '>=8'} + autoevals@0.0.131: + resolution: {integrity: sha512-F+3lraja+Ms7n1M2cpWl65N7AYx4sPocRW454H5HlSGabYMfuFOUxw8IXmEYDkQ38BxtZ0Wd5ZAQj9RF59YJWw==} + autolinker@0.28.1: resolution: {integrity: sha512-zQAFO1Dlsn69eXaO6+7YZc+v84aquQKbwpzCE3L0stj56ERn9hutFxPopViLjo9G+rWwjozRhgS5KJ25Xy19cQ==} @@ -10458,6 +10548,9 @@ packages: resolution: {integrity: sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==} engines: {node: '>=8'} + binary-search@1.3.6: + resolution: {integrity: sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==} + bindings@1.5.0: resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} @@ -10491,6 +10584,10 @@ packages: resolution: {integrity: sha512-2hCgjEmP8YLWQ130n2FerGv7rYpfBmnmp9Uy2Le1vge6X3gZIfSmEzP5QTDElFxcvVcXlEn8Aq6MU/PZygIOog==} engines: {node: '>=14.16'} + boxen@8.0.1: + resolution: {integrity: sha512-F3PH5k5juxom4xktynS7MoFY+NUWH5LC4CnH11YB8NPew+HLpmBLCybSAEyb2F+4pRXhuhWqFesoQd6DAyc2hw==} + engines: {node: '>=18'} + brace-expansion@1.1.12: resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==} @@ -10501,6 +10598,12 @@ packages: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} engines: {node: '>=8'} + braintrust@1.0.2: + resolution: {integrity: sha512-Zm8WbD18FLv5HN38TyUKw659vJdZALQ5VVVWTncMxZgPtFXeWoC7H0tInw/1EQ1pGq50v7WZtlaFBOXhUaf8DQ==} + hasBin: true + peerDependencies: + zod: ^3.25.34 + browserslist@4.26.2: resolution: {integrity: sha512-ECFzp6uFOSB+dcZ5BK/IBaGWssbSYBHvuMeMt3MMFyhI0Z8SqGgEkBLARgpRH3hutIgPVsALcMwbDrJqPxQ65A==} engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} @@ -10593,6 +10696,10 @@ packages: resolution: {integrity: sha512-xlx1yCK2Oc1APsPXDL2LdlNP6+uu8OCDdhOBSVT279M/S+y75O30C2VuD8T2ogdePBBl7PfPF4504tnLgX3zfw==} engines: {node: '>=14.16'} + camelcase@8.0.0: + resolution: {integrity: sha512-8WB3Jcas3swSvjIeA2yvCJ+Miyz5l1ZmB6HFb9R1317dt9LCQoswg/BGrmAmkWVEszSrrg4RwmO46qIm2OEnSA==} + engines: {node: '>=16'} + camelize@1.0.1: resolution: {integrity: sha512-dU+Tx2fsypxTgtLoE36npi3UqcjSSMNYfkqgmoEhtZrraP5VWq0K7FkWVTYa8eMPtnU/G2txVsfdCJTn9uzpuQ==} @@ -10682,6 +10789,9 @@ packages: resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==} engines: {node: '>= 6'} + cheminfo-types@1.8.1: + resolution: {integrity: sha512-FRcpVkox+cRovffgqNdDFQ1eUav+i/Vq/CUd1hcfEl2bevntFlzznL+jE8g4twl6ElB7gZjCko6pYpXyMn+6dA==} + chevrotain-allstar@0.3.1: resolution: {integrity: sha512-b7g+y9A0v4mxCW1qUhf3BSVPg+/NvGErk/dOkrDaHA0nQIQGAtrOjlX//9OQtRlSCy+x9rfB5N8yC71lH1nvMw==} peerDependencies: @@ -10754,10 +10864,18 @@ packages: resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==} engines: {node: '>=18'} + cli-progress@3.12.0: + resolution: {integrity: sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==} + engines: {node: '>=4'} + cli-spinners@2.9.2: resolution: {integrity: sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==} engines: {node: '>=6'} + cli-table3@0.6.5: + resolution: {integrity: sha512-+W/5efTR7y5HRD7gACw9yQjqMVvEMLBHmboM/kPWam+H+Hmyrgjh6YncVKK122YZkXrLudzTuAukUw9FnMf7IQ==} + engines: {node: 10.* || >= 12.*} + cli-truncate@2.1.0: resolution: {integrity: sha512-n8fOixwDD6b/ObinzTrp1ZKFzbgvKZvuz/TvejnLn1aQfC6r52XEx85FmuC+3HI+JM7coBRXUvNqEU2PHVrHpg==} engines: {node: '>=8'} @@ -10950,6 +11068,15 @@ packages: resolution: {integrity: sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==} engines: {node: '>= 14'} + compute-cosine-similarity@1.1.0: + resolution: {integrity: sha512-FXhNx0ILLjGi9Z9+lglLzM12+0uoTnYkHm7GiadXDAr0HGVLm25OivUS1B/LPkbzzvlcXz/1EvWg9ZYyJSdhTw==} + + compute-dot@1.1.0: + resolution: {integrity: sha512-L5Ocet4DdMrXboss13K59OK23GXjiSia7+7Ukc7q4Bl+RVpIXK2W9IHMbWDZkh+JUEvJAwOKRaJDiFUa1LTnJg==} + + compute-l2norm@1.1.0: + resolution: {integrity: sha512-6EHh1Elj90eU28SXi+h2PLnTQvZmkkHWySpoFz+WOlVNLz3DQoC4ISUHSV9n5jMxPHtKGJ01F4uu2PsXBB8sSg==} + compute-scroll-into-view@3.1.1: resolution: {integrity: sha512-VRhuHOLoKYOy4UbilLbUzbYg93XLjv2PncJC50EuTWPA3gaja1UjBsUP/D/9/juV3vQFr6XBEzn9KCAHdUvOHw==} @@ -12363,6 +12490,10 @@ packages: resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} engines: {node: '>=0.8.x'} + eventsource-parser@1.1.2: + resolution: {integrity: sha512-v0eOBUbiaFojBu2s2NPBfYUoRR9GjcDNvCXVaqEf5vVfpIAh9f8RCo4vXTP8c63QRKCFwoLpMpTdPwwhEKVgzA==} + engines: {node: '>=14.18'} + eventsource-parser@3.0.3: resolution: {integrity: sha512-nVpZkTMM9rF6AQ9gPJpFsNAMt48wIzB5TQgiTLdHiuO8XEDhUgZEhqKlZWXbIzo9VmJ/HvysHqEaVeD5v9TPvA==} engines: {node: '>=20.0.0'} @@ -12581,6 +12712,9 @@ packages: fflate@0.8.2: resolution: {integrity: sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==} + fft.js@4.0.4: + resolution: {integrity: sha512-f9c00hphOgeQTlDyavwTtu6RiK8AIFjD6+jvXkNkpeQ7rirK3uFWVpalkoS4LAwbdX7mfZ8aoBfFVQX1Re/8aw==} + figures@3.2.0: resolution: {integrity: sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==} engines: {node: '>=8'} @@ -13597,6 +13731,9 @@ packages: is-alphanumerical@2.0.1: resolution: {integrity: sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==} + is-any-array@2.0.1: + resolution: {integrity: sha512-UtilS7hLRu++wb/WBAw9bNuP1Eg04Ivn1vERJck8zJthEvXCBEBpGR/33u/xLKWEQf95803oalHrVDptcAvFdQ==} + is-array-buffer@3.0.5: resolution: {integrity: sha512-DDfANUiiG2wC1qawP66qlTugJeL5HyzMpfr8lLK+jMQirGzNod0B12cFB/9q838Ru27sBwfw78/rdoU7RERz6A==} engines: {node: '>= 0.4'} @@ -14247,6 +14384,9 @@ packages: libpg-query@15.2.0: resolution: {integrity: sha512-fBi14tsU3/ahtfU0C+/tvmtlt0EJM6kXe0Dl4/cSbiNmJ3IwLIuk3lawwiK5ByXU4sXwyAzbkvSl26jVXzYuaA==} + libpg-query@17.6.0: + resolution: {integrity: sha512-r4zOTcLTGYS5PlLQAicJ6Yi/tvZFag42YUuNEO8pi8bwt/ZZ4kj514J4QV5bOx0mZzPLF6agbfNXQVxGgmHR8g==} + light-my-request@5.14.0: resolution: {integrity: sha512-aORPWntbpH5esaYpGOOmri0OHDOe3wC5M2MQxZ9dvMLZm6DnaAn0kJlcbU9hwsQgLzmZyReKwFwwPkR+nHu5kA==} @@ -14258,6 +14398,9 @@ packages: resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==} engines: {node: '>=14'} + linear-sum-assignment@1.0.9: + resolution: {integrity: sha512-1T2Ek3sxpt2mBHeBFMRJEikiIK/yIOwf+mrxv/DkAU/5ddnCMndZL//hFH7QuHa1tbaQADzsf9t7rkGZKqoFfQ==} + linebreak@1.1.0: resolution: {integrity: sha512-MHp03UImeVhB7XZtjd0E4n6+3xr5Dq/9xI/5FptGk5FrbDR3zagPa2DS6U8ks/3HjbKWG9Q1M2ufOzxV2qLYSQ==} @@ -15119,6 +15262,24 @@ packages: engines: {node: '>=10'} hasBin: true + ml-array-max@1.2.4: + resolution: {integrity: sha512-BlEeg80jI0tW6WaPyGxf5Sa4sqvcyY6lbSn5Vcv44lp1I2GR6AWojfUvLnGTNsIXrZ8uqWmo8VcG1WpkI2ONMQ==} + + ml-array-min@1.2.3: + resolution: {integrity: sha512-VcZ5f3VZ1iihtrGvgfh/q0XlMobG6GQ8FsNyQXD3T+IlstDv85g8kfV0xUG1QPRO/t21aukaJowDzMTc7j5V6Q==} + + ml-array-rescale@1.3.7: + resolution: {integrity: sha512-48NGChTouvEo9KBctDfHC3udWnQKNKEWN0ziELvY3KG25GR5cA8K8wNVzracsqSW1QEkAXjTNx+ycgAv06/1mQ==} + + ml-matrix@6.12.1: + resolution: {integrity: sha512-TJ+8eOFdp+INvzR4zAuwBQJznDUfktMtOB6g/hUcGh3rcyjxbz4Te57Pgri8Q9bhSQ7Zys4IYOGhFdnlgeB6Lw==} + + ml-spectra-processing@14.18.2: + resolution: {integrity: sha512-n9aXXxdf+ogpYeU6FLZgClOteAE10DyUMkjeBGnvew9vmewKLfINVbMZBosW4Q1aqFDMsZjaItiJJ/UCApTIKg==} + + ml-xsadd@3.0.1: + resolution: {integrity: sha512-Fz2q6dwgzGM8wYKGArTUTZDGa4lQFA2Vi6orjGeTVRy22ZnQFKlJuwS9n8NRviqz1KHAHAzdKJwbnYhdo38uYg==} + mlly@1.8.0: resolution: {integrity: sha512-l8D9ODSRWLe2KHJSifWGwBqpTZXIXTeo8mlKjY+E2HAakaTeNpqAyBZ8GSqLzHgw4XmHmC8whvpjJNMbFZN7/g==} @@ -15199,6 +15360,10 @@ packages: muggle-string@0.4.1: resolution: {integrity: sha512-VNTrAak/KhO2i8dqqnqnAHOa3cYBwXEZe9h+D5h/1ZqFSTEFHdM65lR7RoIqq3tBBYavsOXV84NoHXZ0AkPyqQ==} + mustache@4.2.0: + resolution: {integrity: sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==} + hasBin: true + mute-stream@0.0.8: resolution: {integrity: sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==} @@ -18212,8 +18377,8 @@ packages: resolution: {integrity: sha512-TDQg6Nh7e2DaLmMjqpJt7Wwh2Au5loVPH2gwKMkD89e3pb5F/zMQGWt22asEXme031jEmhjazixFH0lwMgbRJA==} engines: {node: '>=18.0.0'} - supabase@2.58.5: - resolution: {integrity: sha512-mYZSkUIePTdmwlHd26Pff8wpmjfre8gcuWzrc5QqhZgZvCXugVzAQQhcjaQisw5kusbPQWNIjUwcHYEKmejhPw==} + supabase@2.67.1: + resolution: {integrity: sha512-d/trGytTjB/Hi625zHh2RFFttjLOkSdRTsY/N1pjxoAfG0blDiHKqPwu12VJYitg4nzgjfhjnD3pLyfU0ko5vg==} engines: {npm: '>=8'} hasBin: true @@ -18316,6 +18481,10 @@ packages: tdigest@0.1.2: resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==} + termi-link@1.1.0: + resolution: {integrity: sha512-2qSN6TnomHgVLtk+htSWbaYs4Rd2MH/RU7VpHTy6MBstyNyWbM4yKd1DCYpE3fDg8dmGWojXCngNi/MHCzGuAA==} + engines: {node: '>=12'} + terser-webpack-plugin@5.3.14: resolution: {integrity: sha512-vkZjpUjb6OMS7dhV+tILUW6BhpDR7P2L/aQSAv+Uwk+m8KATX9EccViHTJR2qDtACKPIYndLGCyl3FMo+r2LMw==} engines: {node: '>= 10.13.0'} @@ -19158,6 +19327,12 @@ packages: resolution: {integrity: sha512-OljLrQ9SQdOUqTaQxqL5dEfZWrXExyyWsozYlAWFawPVNuD83igl7uJD2RTkNMbniIYgt8l81eCJGIdQF7avLQ==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} + validate.io-array@1.0.6: + resolution: {integrity: sha512-DeOy7CnPEziggrOO5CZhVKJw6S3Yi7e9e65R1Nl/RTN1vTQKnzjfvks0/8kQ40FP/dsjRAOd4hxmJ7uLa6vxkg==} + + validate.io-function@1.0.2: + resolution: {integrity: sha512-LlFybRJEriSuBnUhQyG5bwglhh50EpTL2ul23MPIuR1odjO7XaMLFV8vHGwp7AZciFxtYOeiSCT5st+XSPONiQ==} + valtio@1.12.0: resolution: {integrity: sha512-co8NkCHeY0NsL0XsL/cSICt5VhTjwZlYT8mi50dYY5thx3r3w1D15A04Lvs9WL/y/Rf98vUKY5PAAJCTLHvkJw==} engines: {node: '>=12.20.0'} @@ -19543,6 +19718,10 @@ packages: resolution: {integrity: sha512-o0cyEG0e8GPzT4iGHphIOh0cJOV8fivsXxddQasHPHfoZf1ZexrfeA21w2NaEN1RHE+fXlfISmOE8R9N3u3Qig==} engines: {node: '>=12'} + widest-line@5.0.0: + resolution: {integrity: sha512-c9bZp7b5YtRj2wOe6dlj32MK+Bx/M/d+9VB2SHM1OtsUHR0aV0tdP6DWh/iMt0kWi1t5g1Iudu6hQRNd1A4PVA==} + engines: {node: '>=18'} + wordwrap@1.0.0: resolution: {integrity: sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==} @@ -19558,6 +19737,10 @@ packages: resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==} engines: {node: '>=12'} + wrap-ansi@9.0.2: + resolution: {integrity: sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==} + engines: {node: '>=18'} + wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} @@ -19797,6 +19980,10 @@ snapshots: eventsource-parser: 3.0.6 zod: 3.25.76 + '@ai-sdk/provider@1.1.3': + dependencies: + json-schema: 0.4.0 + '@ai-sdk/provider@2.0.0': dependencies: json-schema: 0.4.0 @@ -21204,6 +21391,9 @@ snapshots: react-dom: 18.3.1(react@18.3.1) react-is: 17.0.2 + '@colors/colors@1.5.0': + optional: true + '@contentlayer2/cli@0.4.3(esbuild@0.25.2)(markdown-wasm@1.2.0)(supports-color@8.1.1)': dependencies: '@contentlayer2/core': 0.4.3(esbuild@0.25.2)(markdown-wasm@1.2.0)(supports-color@8.1.1) @@ -23329,6 +23519,8 @@ snapshots: - bufferutil - utf-8-validate + '@next/env@14.2.35': {} + '@next/env@15.5.9': {} '@next/env@16.0.10': {} @@ -24823,6 +25015,8 @@ snapshots: '@pgsql/types@15.0.2': {} + '@pgsql/types@17.6.2': {} + '@pkgjs/parseargs@0.11.0': optional: true @@ -29322,6 +29516,10 @@ snapshots: lz-string: 1.5.0 uuid: 9.0.1 + '@vercel/functions@1.6.0(@aws-sdk/credential-provider-web-identity@3.830.0)': + optionalDependencies: + '@aws-sdk/credential-provider-web-identity': 3.830.0 + '@vercel/functions@2.1.0(@aws-sdk/credential-provider-web-identity@3.830.0)': optionalDependencies: '@aws-sdk/credential-provider-web-identity': 3.830.0 @@ -30098,6 +30296,21 @@ snapshots: auto-bind@4.0.0: {} + autoevals@0.0.131(encoding@0.1.13)(ws@8.18.3): + dependencies: + ajv: 8.17.1 + compute-cosine-similarity: 1.1.0 + js-levenshtein: 1.1.6 + js-yaml: 4.1.1 + linear-sum-assignment: 1.0.9 + mustache: 4.2.0 + openai: 4.104.0(encoding@0.1.13)(ws@8.18.3)(zod@3.25.76) + zod: 3.25.76 + zod-to-json-schema: 3.25.0(zod@3.25.76) + transitivePeerDependencies: + - encoding + - ws + autolinker@0.28.1: dependencies: gulp-header: 1.8.12 @@ -30205,6 +30418,8 @@ snapshots: binary-extensions@2.2.0: {} + binary-search@1.3.6: {} + bindings@1.5.0: dependencies: file-uri-to-path: 1.0.0 @@ -30267,6 +30482,17 @@ snapshots: widest-line: 4.0.1 wrap-ansi: 8.1.0 + boxen@8.0.1: + dependencies: + ansi-align: 3.0.1 + camelcase: 8.0.0 + chalk: 5.4.1 + cli-boxes: 3.0.0 + string-width: 7.2.0 + type-fest: 4.30.0 + widest-line: 5.0.0 + wrap-ansi: 9.0.2 + brace-expansion@1.1.12: dependencies: balanced-match: 1.0.2 @@ -30280,6 +30506,37 @@ snapshots: dependencies: fill-range: 7.1.1 + braintrust@1.0.2(@aws-sdk/credential-provider-web-identity@3.830.0)(supports-color@8.1.1)(zod@3.25.76): + dependencies: + '@ai-sdk/provider': 1.1.3 + '@next/env': 14.2.35 + '@vercel/functions': 1.6.0(@aws-sdk/credential-provider-web-identity@3.830.0) + argparse: 2.0.1 + boxen: 8.0.1 + chalk: 4.1.2 + cli-progress: 3.12.0 + cli-table3: 0.6.5 + cors: 2.8.5 + dotenv: 16.5.0 + esbuild: 0.25.2 + eventsource-parser: 1.1.2 + express: 4.22.1(supports-color@8.1.1) + graceful-fs: 4.2.11 + http-errors: 2.0.1 + minimatch: 9.0.5 + mustache: 4.2.0 + pluralize: 8.0.0 + simple-git: 3.28.0(supports-color@8.1.1) + slugify: 1.6.6 + source-map: 0.7.6 + termi-link: 1.1.0 + uuid: 9.0.1 + zod: 3.25.76 + zod-to-json-schema: 3.25.0(zod@3.25.76) + transitivePeerDependencies: + - '@aws-sdk/credential-provider-web-identity' + - supports-color + browserslist@4.26.2: dependencies: baseline-browser-mapping: 2.8.5 @@ -30416,6 +30673,8 @@ snapshots: camelcase@7.0.1: {} + camelcase@8.0.0: {} + camelize@1.0.1: {} caniuse-api@3.0.0: @@ -30534,6 +30793,8 @@ snapshots: parse5: 7.1.2 parse5-htmlparser2-tree-adapter: 7.0.0 + cheminfo-types@1.8.1: {} + chevrotain-allstar@0.3.1(chevrotain@11.0.3): dependencies: chevrotain: 11.0.3 @@ -30606,8 +30867,18 @@ snapshots: dependencies: restore-cursor: 5.1.0 + cli-progress@3.12.0: + dependencies: + string-width: 4.2.3 + cli-spinners@2.9.2: {} + cli-table3@0.6.5: + dependencies: + string-width: 4.2.3 + optionalDependencies: + '@colors/colors': 1.5.0 + cli-truncate@2.1.0: dependencies: slice-ansi: 3.0.0 @@ -30784,6 +31055,23 @@ snapshots: normalize-path: 3.0.0 readable-stream: 4.6.0 + compute-cosine-similarity@1.1.0: + dependencies: + compute-dot: 1.1.0 + compute-l2norm: 1.1.0 + validate.io-array: 1.0.6 + validate.io-function: 1.0.2 + + compute-dot@1.1.0: + dependencies: + validate.io-array: 1.0.6 + validate.io-function: 1.0.2 + + compute-l2norm@1.1.0: + dependencies: + validate.io-array: 1.0.6 + validate.io-function: 1.0.2 + compute-scroll-into-view@3.1.1: {} concat-map@0.0.1: {} @@ -32263,6 +32551,8 @@ snapshots: events@3.3.0: {} + eventsource-parser@1.1.2: {} + eventsource-parser@3.0.3: {} eventsource-parser@3.0.6: {} @@ -32582,6 +32872,8 @@ snapshots: fflate@0.8.2: {} + fft.js@4.0.4: {} + figures@3.2.0: dependencies: escape-string-regexp: 1.0.5 @@ -33836,6 +34128,8 @@ snapshots: is-alphabetical: 2.0.1 is-decimal: 2.0.1 + is-any-array@2.0.1: {} + is-array-buffer@3.0.5: dependencies: call-bind: 1.0.8 @@ -34448,6 +34742,10 @@ snapshots: - encoding - supports-color + libpg-query@17.6.0: + dependencies: + '@pgsql/types': 17.6.2 + light-my-request@5.14.0: dependencies: cookie: 0.7.2 @@ -34458,6 +34756,12 @@ snapshots: lilconfig@3.1.3: {} + linear-sum-assignment@1.0.9: + dependencies: + cheminfo-types: 1.8.1 + ml-matrix: 6.12.1 + ml-spectra-processing: 14.18.2 + linebreak@1.1.0: dependencies: base64-js: 0.0.8 @@ -35975,6 +36279,36 @@ snapshots: mkdirp@3.0.1: {} + ml-array-max@1.2.4: + dependencies: + is-any-array: 2.0.1 + + ml-array-min@1.2.3: + dependencies: + is-any-array: 2.0.1 + + ml-array-rescale@1.3.7: + dependencies: + is-any-array: 2.0.1 + ml-array-max: 1.2.4 + ml-array-min: 1.2.3 + + ml-matrix@6.12.1: + dependencies: + is-any-array: 2.0.1 + ml-array-rescale: 1.3.7 + + ml-spectra-processing@14.18.2: + dependencies: + binary-search: 1.3.6 + cheminfo-types: 1.8.1 + fft.js: 4.0.4 + is-any-array: 2.0.1 + ml-matrix: 6.12.1 + ml-xsadd: 3.0.1 + + ml-xsadd@3.0.1: {} + mlly@1.8.0: dependencies: acorn: 8.15.0 @@ -36054,6 +36388,8 @@ snapshots: muggle-string@0.4.1: {} + mustache@4.2.0: {} + mute-stream@0.0.8: {} mute-stream@2.0.0: {} @@ -36520,7 +36856,7 @@ snapshots: number-flow@0.3.7: {} - nuqs@1.19.1(next@15.5.9(@opentelemetry/api@1.9.0)(@playwright/test@1.56.1)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.4)): + nuqs@1.19.1(next@15.5.9(@babel/core@7.28.4(supports-color@8.1.1))(@opentelemetry/api@1.9.0)(@playwright/test@1.56.1)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.4)): dependencies: mitt: 3.0.1 next: 15.5.9(@babel/core@7.28.4(supports-color@8.1.1))(@opentelemetry/api@1.9.0)(@playwright/test@1.56.1)(babel-plugin-macros@3.1.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.4) @@ -39921,7 +40257,7 @@ snapshots: dependencies: openapi-fetch: 0.6.2 - supabase@2.58.5(supports-color@8.1.1): + supabase@2.67.1(supports-color@8.1.1): dependencies: bin-links: 6.0.0 https-proxy-agent: 7.0.6(supports-color@8.1.1) @@ -40065,6 +40401,8 @@ snapshots: dependencies: bintrees: 1.0.2 + termi-link@1.1.0: {} + terser-webpack-plugin@5.3.14(esbuild@0.25.2)(webpack@5.94.0(esbuild@0.25.2)): dependencies: '@jridgewell/trace-mapping': 0.3.31 @@ -40898,6 +41236,10 @@ snapshots: validate-npm-package-name@5.0.1: {} + validate.io-array@1.0.6: {} + + validate.io-function@1.0.2: {} + valtio@1.12.0(@types/react@18.3.3)(react@18.3.1): dependencies: derive-valtio: 0.1.0(valtio@1.12.0(@types/react@18.3.3)(react@18.3.1)) @@ -41471,6 +41813,10 @@ snapshots: dependencies: string-width: 5.1.2 + widest-line@5.0.0: + dependencies: + string-width: 7.2.0 + wordwrap@1.0.0: {} wrap-ansi@6.2.0: @@ -41491,6 +41837,12 @@ snapshots: string-width: 5.1.2 strip-ansi: 7.1.0 + wrap-ansi@9.0.2: + dependencies: + ansi-styles: 6.2.1 + string-width: 7.2.0 + strip-ansi: 7.1.0 + wrappy@1.0.2: {} write-file-atomic@7.0.0: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 423c8e85fb1..52f428ea86d 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -51,6 +51,7 @@ minimumReleaseAgeExclude: - iceberg-js - '@vitejs/plugin-rsc' - stripe-experiment-sync # TODO(matlin) remove, temp just to unblock launch + - braintrust onlyBuiltDependencies: - supabase diff --git a/turbo.json b/turbo.json index b2042f2d1d6..8cb394fc1ba 100644 --- a/turbo.json +++ b/turbo.json @@ -104,6 +104,8 @@ "DEFAULT_PROJECT_NAME", "DEFAULT_ORGANIZATION_NAME", "OPENAI_API_KEY", + "BRAINTRUST_API_KEY", + "BRAINTRUST_PROJECT_ID", "AUTH_JWT_SECRET", "LOGFLARE_API_KEY", "LOGFLARE_PUBLIC_ACCESS_TOKEN",