lywsvip/supabase
1
0
Fork 0
mirror of https://github.com/supabase/supabase.git synced 2026-05-06 22:18:00 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: improve docs search ux and result rankings (#19450)

Browse Source 
Improve the way docs are indexed and FTS results are ranked, so results are more relevant.

Also improve debouncing and searching UX so it feels a bit faster and you can scan the results more easily.
This commit is contained in:
Charis
2023-12-06 12:27:33 -05:00
committed by GitHub
parent 6a14570836
commit 6c4311f723
10 changed files with 364 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -82,8 +82,10 @@ typings/
.env
# excempt the apps/www .env
!apps/www/.env
.env.tmp
.env.test
.env.local
.env.staging
.env.production
# parcel-bundler cache (https://parceljs.org/)

54
apps/docs/scripts/search/generate-embeddings.ts
View File

@@ -4,6 +4,7 @@ import { parseArgs } from 'node:util'
import { OpenAI } from 'openai'
import { v4 as uuidv4 } from 'uuid'
import { fetchSources } from './sources'
import { Json, Section } from './sources/base'
dotenv.config()
@@ -63,15 +64,19 @@ async function generateEmbeddings() {
}
for (const embeddingSource of embeddingSources) {
const { type, source, path, parentPath } = embeddingSource
const { type, source, path } = embeddingSource
try {
const { checksum, meta, sections } = await embeddingSource.load()
const {
checksum,
sections,
meta = {},
}: { checksum: string; sections: Section[]; meta?: Json } = embeddingSource.process()
// Check for existing page in DB and compare checksums
const { error: fetchPageError, data: existingPage } = await supabaseClient
.from('page')
.select('id, path, checksum, parentPage:parent_page_id(id, path)')
.select('id, path, checksum')
.filter('path', 'eq', path)
.limit(1)
.maybeSingle()
@@ -82,34 +87,6 @@ async function generateEmbeddings() {
// We use checksum to determine if this page & its sections need to be regenerated
if (!shouldRefresh && existingPage?.checksum === checksum) {
const existingParentPage = Array.isArray(existingPage?.parentPage)
? existingPage?.parentPage[0]
: existingPage?.parentPage
// If parent page changed, update it
if (existingParentPage?.path !== parentPath) {
console.log(`[${path}] Parent page has changed. Updating to '${parentPath}'...`)
const { error: fetchParentPageError, data: parentPage } = await supabaseClient
.from('page')
.select()
.filter('path', 'eq', parentPath)
.limit(1)
.maybeSingle()
if (fetchParentPageError) {
throw fetchParentPageError
}
const { error: updatePageError } = await supabaseClient
.from('page')
.update({ parent_page_id: parentPage?.id })
.filter('id', 'eq', existingPage.id)
if (updatePageError) {
throw updatePageError
}
}
// No content/embedding update required on this page
// Update other meta info
const { error: updatePageError } = await supabaseClient
@@ -149,17 +126,6 @@ async function generateEmbeddings() {
}
}
const { error: fetchParentPageError, data: parentPage } = await supabaseClient
.from('page')
.select()
.filter('path', 'eq', parentPath)
.limit(1)
.maybeSingle()
if (fetchParentPageError) {
throw fetchParentPageError
}
// Create/update page record. Intentionally clear checksum until we
// have successfully generated all page sections.
const { error: upsertPageError, data: page } = await supabaseClient
@@ -171,7 +137,7 @@ async function generateEmbeddings() {
type,
source,
meta,
parent_page_id: parentPage?.id,
content: embeddingSource.extractIndexedContent(),
version: refreshVersion,
last_refresh: refreshDate,
},
@@ -200,7 +166,7 @@ async function generateEmbeddings() {
const [responseData] = embeddingResponse.data
const { error: insertPageSectionError, data: pageSection } = await supabaseClient
const { error: insertPageSectionError } = await supabaseClient
.from('page_section')
.insert({
page_id: page.id,

15
apps/docs/scripts/search/sources/base.ts
View File

@@ -9,12 +9,23 @@ export type Section = {
slug?: string
}
export abstract class BaseLoader {
type: string
constructor(public source: string, public path: string) {}
abstract load(): Promise<BaseSource[]>
}
export abstract class BaseSource {
type: string
checksum?: string
meta?: Json
sections?: Section[]
constructor(public source: string, public path: string, public parentPath?: string) {}
constructor(public source: string, public path: string) {}
abstract load(): Promise<{ checksum: string; meta?: Json; sections: Section[] }>
abstract process(): { checksum: string; meta?: Json; sections: Section[] }
abstract extractIndexedContent(): string
}

21
apps/docs/scripts/search/sources/github-discussion.ts
View File

@@ -2,7 +2,7 @@ import { createAppAuth } from '@octokit/auth-app'
import { Octokit } from '@octokit/core'
import { paginateGraphql } from '@octokit/plugin-paginate-graphql'
import { createHash } from 'crypto'
import { BaseSource } from './base'
import { BaseLoader, BaseSource } from './base'
export const ExtendedOctokit = Octokit.plugin(paginateGraphql)
export type ExtendedOctokit = InstanceType<typeof ExtendedOctokit>
@@ -74,7 +74,7 @@ export async function fetchDiscussions(owner: string, repo: string, categoryId:
return discussions
}
export class GitHubDiscussionSource extends BaseSource {
export class GitHubDiscussionLoader extends BaseLoader {
type = 'github-discussions' as const
constructor(source: string, public discussion: Discussion) {
@@ -82,6 +82,18 @@ export class GitHubDiscussionSource extends BaseSource {
}
async load() {
return [new GitHubDiscussionSource(this.source, this.path, this.discussion)]
}
}
export class GitHubDiscussionSource extends BaseSource {
type = 'github-discussions' as const
constructor(source: string, path: string, public discussion: Discussion) {
super(source, path)
}
process() {
const { id, title, updatedAt, body, databaseId } = this.discussion
const checksum = createHash('sha256').update(updatedAt).digest('base64')
@@ -116,4 +128,9 @@ export class GitHubDiscussionSource extends BaseSource {
sections,
}
}
extractIndexedContent(): string {
const sections = this.sections ?? []
return sections.map(({ heading, content }) => `${heading}\n\n${content}`).join('\n')
}
}

73
apps/docs/scripts/search/sources/index.ts
View File

@@ -1,8 +1,15 @@
import { GitHubDiscussionSource, fetchDiscussions } from './github-discussion'
import { MarkdownSource } from './markdown'
import {
GitHubDiscussionLoader,
GitHubDiscussionSource,
fetchDiscussions,
} from './github-discussion'
import { MarkdownLoader, MarkdownSource } from './markdown'
import {
CliReferenceLoader,
CliReferenceSource,
ClientLibReferenceLoader,
ClientLibReferenceSource,
OpenApiReferenceLoader,
OpenApiReferenceSource,
} from './reference-doc'
import { walk } from './util'
@@ -20,74 +27,74 @@ export type SearchSource =
* Fetches all the sources we want to index for search
*/
export async function fetchSources() {
const openApiReferenceSource = new OpenApiReferenceSource(
const openApiReferenceSource = new OpenApiReferenceLoader(
'api',
'/reference/api',
{ title: 'Management API Reference' },
'../../spec/transforms/api_v0_openapi_deparsed.json',
'../../spec/common-api-sections.json'
)
).load()
const jsLibReferenceSource = new ClientLibReferenceSource(
const jsLibReferenceSource = new ClientLibReferenceLoader(
'js-lib',
'/reference/javascript',
{ title: 'JavaScript Reference' },
'../../spec/supabase_js_v2.yml',
'../../spec/common-client-libs-sections.json'
)
).load()
const dartLibReferenceSource = new ClientLibReferenceSource(
const dartLibReferenceSource = new ClientLibReferenceLoader(
'dart-lib',
'/reference/dart',
{ title: 'Dart Reference' },
'../../spec/supabase_dart_v1.yml',
'../../spec/common-client-libs-sections.json'
)
).load()
const pythonLibReferenceSource = new ClientLibReferenceSource(
const pythonLibReferenceSource = new ClientLibReferenceLoader(
'python-lib',
'/reference/python',
{ title: 'Python Reference' },
'../../spec/supabase_py_v2.yml',
'../../spec/common-client-libs-sections.json'
)
).load()
const cSharpLibReferenceSource = new ClientLibReferenceSource(
const cSharpLibReferenceSource = new ClientLibReferenceLoader(
'csharp-lib',
'/reference/csharp',
{ title: 'C# Reference' },
'../../spec/supabase_csharp_v0.yml',
'../../spec/common-client-libs-sections.json'
)
).load()
const swiftLibReferenceSource = new ClientLibReferenceSource(
const swiftLibReferenceSource = new ClientLibReferenceLoader(
'swift-lib',
'/reference/swift',
{ title: 'Swift Reference' },
'../../spec/supabase_swift_v1.yml',
'../../spec/common-client-libs-sections.json'
)
).load()
const ktLibReferenceSource = new ClientLibReferenceSource(
const ktLibReferenceSource = new ClientLibReferenceLoader(
'kt-lib',
'/reference/kotlin',
{ title: 'Kotlin Reference' },
'../../spec/supabase_kt_v1.yml',
'../../spec/common-client-libs-sections.json'
)
).load()
const cliReferenceSource = new CliReferenceSource(
const cliReferenceSource = new CliReferenceLoader(
'cli',
'/reference/cli',
{ title: 'CLI Reference' },
'../../spec/cli_v1_commands.yaml',
'../../spec/common-cli-sections.json'
)
).load()
const guideSources = (await walk('pages'))
.filter(({ path }) => /\.mdx?$/.test(path))
.filter(({ path }) => !ignoredFiles.includes(path))
.map((entry) => new MarkdownSource('guide', entry.path))
.map((entry) => new MarkdownLoader('guide', entry.path).load())
const githubDiscussionSources = (
await fetchDiscussions(
@@ -95,20 +102,22 @@ export async function fetchSources() {
'supabase',
'DIC_kwDODMpXOc4CUvEr' // 'Troubleshooting' category
)
).map((discussion) => new GitHubDiscussionSource('supabase/supabase', discussion))
).map((discussion) => new GitHubDiscussionLoader('supabase/supabase', discussion).load())
const sources: SearchSource[] = [
openApiReferenceSource,
jsLibReferenceSource,
dartLibReferenceSource,
pythonLibReferenceSource,
cSharpLibReferenceSource,
swiftLibReferenceSource,
ktLibReferenceSource,
cliReferenceSource,
...githubDiscussionSources,
...guideSources,
]
const sources: SearchSource[] = (
await Promise.all([
openApiReferenceSource,
jsLibReferenceSource,
dartLibReferenceSource,
pythonLibReferenceSource,
cSharpLibReferenceSource,
swiftLibReferenceSource,
ktLibReferenceSource,
cliReferenceSource,
...githubDiscussionSources,
...guideSources,
])
).flat()
return sources
}

34
apps/docs/scripts/search/sources/markdown.ts
View File

@@ -10,7 +10,7 @@ import { toString } from 'mdast-util-to-string'
import { mdxjs } from 'micromark-extension-mdxjs'
import { u } from 'unist-builder'
import { filter } from 'unist-util-filter'
import { BaseSource, Json, Section } from './base'
import { BaseLoader, BaseSource, Json, Section } from './base'
/**
* Extracts ES literals from an `estree` `ObjectExpression`
@@ -189,29 +189,39 @@ export type ProcessedMdx = {
sections: Section[]
}
export class MarkdownSource extends BaseSource {
export class MarkdownLoader extends BaseLoader {
type = 'markdown' as const
constructor(source: string, public filePath: string, public parentFilePath?: string) {
constructor(source: string, public filePath: string) {
const path = filePath.replace(/^pages/, '').replace(/\.mdx?$/, '')
const parentPath = parentFilePath?.replace(/^pages/, '').replace(/\.mdx?$/, '')
super(source, path, parentPath)
super(source, path)
}
async load() {
const contents = await readFile(this.filePath, 'utf8')
return [new MarkdownSource(this.source, this.path, contents)]
}
}
const { checksum, meta, sections } = processMdxForSearch(contents)
export class MarkdownSource extends BaseSource {
type = 'markdown' as const
constructor(source: string, path: string, public contents: string) {
super(source, path)
}
process() {
const { checksum, meta, sections } = processMdxForSearch(this.contents)
this.checksum = checksum
this.meta = meta
this.sections = sections
return {
checksum,
meta,
sections,
}
return { checksum, meta, sections }
}
extractIndexedContent(): string {
const sections = this.sections ?? []
return sections.map(({ content }) => content).join('\n\n')
}
}

168
apps/docs/scripts/search/sources/reference-doc.ts
View File

@@ -4,16 +4,20 @@ import yaml from 'js-yaml'
import { OpenAPIV3 } from 'openapi-types'
import {
ICommonItem,
ICommonSection,
IFunctionDefinition,
ISpec,
} from '../../../components/reference/Reference.types'
import { CliCommand, CliSpec } from '../../../generator/types/CliSpec'
import { flattenSections } from '../../../lib/helpers'
import { enrichedOperation, gen_v3 } from '../../../lib/refGenerator/helpers'
import { BaseSource, Json } from './base'
import { BaseLoader, BaseSource, Json } from './base'
export abstract class ReferenceSource<SpecSection> extends BaseSource {
export abstract class ReferenceLoader<SpecSection> extends BaseLoader {
type = 'reference' as const
sourceConstructor: (
...args: ConstructorParameters<typeof ReferenceSource<SpecSection>>
) => ReferenceSource<SpecSection>
constructor(
source: string,
@@ -32,10 +36,6 @@ export abstract class ReferenceSource<SpecSection> extends BaseSource {
const refSections: ICommonItem[] = JSON.parse(refSectionsContents)
const flattenedRefSections = flattenSections(refSections)
const checksum = createHash('sha256')
.update(specContents + refSectionsContents)
.digest('base64')
const specSections = this.getSpecSections(specContents)
const sections = flattenedRefSections
@@ -46,16 +46,51 @@ export abstract class ReferenceSource<SpecSection> extends BaseSource {
return
}
return {
heading: refSection.title,
slug: refSection.slug,
content: `${this.meta.title} for ${refSection.title}:\n${this.formatSection(
specSection,
refSection
)}`,
}
return this.sourceConstructor(
this.source,
`${this.path}/${refSection.slug}`,
refSection,
specSection,
this.meta
)
})
.filter((section) => !!section)
.filter(Boolean)
return sections
}
abstract getSpecSections(specContents: string): SpecSection[]
abstract matchSpecSection(specSections: SpecSection[], id: string): SpecSection
}
export abstract class ReferenceSource<SpecSection> extends BaseSource {
type = 'reference' as const
constructor(
source: string,
path: string,
public refSection: ICommonSection,
public specSection: SpecSection,
public meta: Json
) {
super(source, path)
}
process() {
const checksum = createHash('sha256')
.update(JSON.stringify(this.refSection) + JSON.stringify(this.specSection))
.digest('base64')
const sections = [
{
heading: this.refSection.title,
slug: this.refSection.slug,
content: `${this.meta.title} for ${this.refSection.title}:\n${this.formatSection(
this.specSection,
this.refSection
)}`,
},
]
this.checksum = checksum
this.sections = sections
@@ -63,16 +98,31 @@ export abstract class ReferenceSource<SpecSection> extends BaseSource {
return {
checksum,
sections,
meta: this.meta,
meta: {
...this.meta,
subtitle: this.extractSubtitle(),
title: this.extractTitle(),
},
}
}
abstract getSpecSections(specContents: string): SpecSection[]
abstract matchSpecSection(specSections: SpecSection[], id: string): SpecSection
abstract formatSection(specSection: SpecSection, refSection: ICommonItem): string
abstract extractTitle(): string
abstract extractSubtitle(): string
}
export class OpenApiReferenceSource extends ReferenceSource<enrichedOperation> {
export class OpenApiReferenceLoader extends ReferenceLoader<enrichedOperation> {
constructor(
source: string,
path: string,
meta: Json,
specFilePath: string,
sectionsFilePath: string
) {
super(source, path, meta, specFilePath, sectionsFilePath)
this.sourceConstructor = (...args) => new OpenApiReferenceSource(...args)
}
getSpecSections(specContents: string): enrichedOperation[] {
const spec: OpenAPIV3.Document<{}> = JSON.parse(specContents)
@@ -85,7 +135,10 @@ export class OpenApiReferenceSource extends ReferenceSource<enrichedOperation> {
matchSpecSection(operations: enrichedOperation[], id: string): enrichedOperation {
return operations.find((operation) => operation.operationId === id)
}
formatSection(specOperation: enrichedOperation) {
}
export class OpenApiReferenceSource extends ReferenceSource<enrichedOperation> {
formatSection(specOperation: enrichedOperation, _: ICommonItem) {
const { summary, description, operation, path, tags } = specOperation
return JSON.stringify({
summary,
@@ -95,9 +148,35 @@ export class OpenApiReferenceSource extends ReferenceSource<enrichedOperation> {
tags,
})
}
extractSubtitle() {
return typeof this.specSection.description === 'string' ? this.specSection.description : ''
}
extractTitle() {
return `${this.meta.title}: ${this.specSection.operation}`
}
extractIndexedContent(): string {
const { summary, description, operation, tags } = this.specSection
return `${this.meta.title}\n\n${summary}\n\n${description}\n\n${operation}\n\n${tags.join(
', '
)}`
}
}
export class ClientLibReferenceSource extends ReferenceSource<IFunctionDefinition> {
export class ClientLibReferenceLoader extends ReferenceLoader<IFunctionDefinition> {
constructor(
source: string,
path: string,
meta: Json,
specFilePath: string,
sectionsFilePath: string
) {
super(source, path, meta, specFilePath, sectionsFilePath)
this.sourceConstructor = (...args) => new ClientLibReferenceSource(...args)
}
getSpecSections(specContents: string): IFunctionDefinition[] {
const spec = yaml.load(specContents) as ISpec
@@ -106,6 +185,9 @@ export class ClientLibReferenceSource extends ReferenceSource<IFunctionDefinitio
matchSpecSection(functionDefinitions: IFunctionDefinition[], id: string): IFunctionDefinition {
return functionDefinitions.find((functionDefinition) => functionDefinition.id === id)
}
}
export class ClientLibReferenceSource extends ReferenceSource<IFunctionDefinition> {
formatSection(functionDefinition: IFunctionDefinition, refSection: ICommonItem): string {
const { title } = refSection
const { description, title: functionName } = functionDefinition
@@ -116,9 +198,33 @@ export class ClientLibReferenceSource extends ReferenceSource<IFunctionDefinitio
functionName,
})
}
extractTitle(): string {
return this.specSection.title
}
extractSubtitle(): string {
return `${this.meta.title}: ${this.refSection.title}`
}
extractIndexedContent(): string {
const { title, description } = this.specSection
return `${this.meta.title}\n\n${title}\n\n${description}`
}
}
export class CliReferenceSource extends ReferenceSource<CliCommand> {
export class CliReferenceLoader extends ReferenceLoader<CliCommand> {
constructor(
source: string,
path: string,
meta: Json,
specFilePath: string,
sectionsFilePath: string
) {
super(source, path, meta, specFilePath, sectionsFilePath)
this.sourceConstructor = (...args) => new CliReferenceSource(...args)
}
getSpecSections(specContents: string): CliCommand[] {
const spec = yaml.load(specContents) as CliSpec
@@ -127,7 +233,10 @@ export class CliReferenceSource extends ReferenceSource<CliCommand> {
matchSpecSection(cliCommands: CliCommand[], id: string): CliCommand {
return cliCommands.find((cliCommand) => cliCommand.id === id)
}
formatSection(cliCommand: CliCommand): string {
}
export class CliReferenceSource extends ReferenceSource<CliCommand> {
formatSection(cliCommand: CliCommand, _: ICommonItem): string {
const { summary, description, usage } = cliCommand
return JSON.stringify({
summary,
@@ -135,4 +244,17 @@ export class CliReferenceSource extends ReferenceSource<CliCommand> {
usage,
})
}
extractSubtitle(): string {
return `${this.meta.title}: ${this.specSection.title}`
}
extractTitle(): string {
return this.specSection.summary
}
extractIndexedContent(): string {
const { summary, description, usage } = this.specSection
return `${this.meta.title}\n\n${summary}\n\n${description}\n\n${usage}`
}
}

11
packages/ui/src/components/Command/CommandMenu.tsx
View File

@@ -85,6 +85,9 @@ const CommandMenu = ({ projectRef }: CommandMenuProps) => {
setSearch(newValue)
}
const commandListMaxHeight =
currentPage === COMMAND_ROUTES.DOCS_SEARCH ? 'min(600px, 50vh)' : '300px'
return (
<>
<CommandDialog
@@ -109,7 +112,13 @@ const CommandMenu = ({ projectRef }: CommandMenuProps) => {
onValueChange={handleInputChange}
/>
)}
<CommandList className={['my-2', showCommandInput && 'max-h-[300px]'].join(' ')}>
<CommandList
style={{
maxHeight: commandListMaxHeight,
height: currentPage === COMMAND_ROUTES.DOCS_SEARCH ? commandListMaxHeight : 'auto',
}}
className="my-2"
>
{!currentPage && (
<>
<CommandGroup heading="Documentation">

81
packages/ui/src/components/Command/DocsSearch.tsx
View File

@@ -21,6 +21,7 @@ import {
FORCE_MOUNT_ITEM,
TextHighlighter,
} from './Command.utils'
import { useRouter } from 'next/router'
const NUMBER_SOURCES = 2
@@ -56,21 +57,6 @@ export interface Page {
sections: PageSection[]
}
function removeDoubleQuotes(inputString: string): string {
// Use the replace method with a regular expression to remove double quotes
return inputString.replace(/"/g, '')
}
const getDocsUrl = () => {
if (!process.env.NEXT_PUBLIC_SITE_URL || !process.env.NEXT_PUBLIC_LOCAL_SUPABASE) {
return 'https://supabase.com/docs'
}
const isLocal =
process.env.NEXT_PUBLIC_SITE_URL.includes('localhost') || process.env.NEXT_PUBLIC_LOCAL_SUPABASE
return isLocal ? 'http://localhost:3001/docs' : 'https://supabase.com/docs'
}
type SearchState =
| {
status: 'initial'
@@ -183,7 +169,12 @@ function reducer(state: SearchState, action: Action): SearchState {
: {
status: 'loading',
key: action.key,
staleResults: [],
staleResults:
'results' in state
? state.results
: 'staleResults' in state
? state.staleResults
: [],
}
}
return allSourcesLoaded
@@ -201,7 +192,8 @@ function reducer(state: SearchState, action: Action): SearchState {
return {
status: 'loading',
key: action.key,
staleResults: 'results' in state ? state.results : [],
staleResults:
'results' in state ? state.results : 'staleResults' in state ? state.staleResults : [],
}
case 'reset':
return {
@@ -226,9 +218,22 @@ function reducer(state: SearchState, action: Action): SearchState {
const DocsSearch = () => {
const [state, dispatch] = useReducer(reducer, { status: 'initial', key: 0 })
const supabaseClient = useSupabaseClient()
const { isLoading, setIsLoading, search, setSearch, inputRef } = useCommandMenu()
const { search, setSearch, inputRef } = useCommandMenu()
const key = useRef(0)
const initialLoad = useRef(true)
const router = useRouter()
function openLink(pageType: PageType, link: string) {
switch (pageType) {
case PageType.Markdown:
case PageType.Reference:
return router.push(link)
case PageType.GithubDiscussion:
return window.open(link, '_blank')
default:
throw new Error(`Unknown page type '${pageType}'`)
}
}
const hasResults =
state.status === 'fullResults' ||
@@ -237,8 +242,6 @@ const DocsSearch = () => {
const handleSearch = useCallback(
async (query: string) => {
setIsLoading(true)
key.current += 1
const localKey = key.current
dispatch({ type: 'newSearchDispatched', key: localKey })
@@ -273,11 +276,6 @@ const DocsSearch = () => {
message: error.message ?? '',
})
})
.finally(() => {
if (sourcesLoaded === NUMBER_SOURCES) {
setIsLoading(false)
}
})
})
},
[supabaseClient]
@@ -293,7 +291,7 @@ const DocsSearch = () => {
})
}
const debouncedSearch = useMemo(() => debounce(handleSearch, 300), [handleSearch])
const debouncedSearch = useMemo(() => debounce(handleSearch, 150), [handleSearch])
useEffect(() => {
if (initialLoad.current) {
@@ -311,7 +309,7 @@ const DocsSearch = () => {
key.current += 1
dispatch({ type: 'reset', key: key.current })
}
}, [search])
}, [search, handleSearch, debouncedSearch])
// Immediately run search if user presses enter
// and abort any debounced searches that are waiting
@@ -379,11 +377,11 @@ const DocsSearch = () => {
return (
<CommandGroup
heading=""
key={`${page.title}-group-index-${i}`}
key={`${page.path}-group`}
value={`${FORCE_MOUNT_ITEM}--${page.title}-group-index-${i}`}
>
<CommandItem
key={`${page.title}-item-index-${i}`}
key={`${page.path}-item`}
value={`${FORCE_MOUNT_ITEM}--${page.title}-item-index-${i}`}
type="block-link"
onSelect={() => {
@@ -396,9 +394,12 @@ const DocsSearch = () => {
<CommandLabel>
<TextHighlighter text={page.title} query={search} />
</CommandLabel>
{page.description && (
{(page.description || page.subtitle) && (
<div className="text-xs text-foreground-muted">
<TextHighlighter text={page.description} query={search} />
<TextHighlighter
text={page.description! || page.subtitle!}
query={search}
/>
</div>
)}
</div>
@@ -414,7 +415,7 @@ const DocsSearch = () => {
onSelect={() => {
openLink(page.type, formatSectionUrl(page, section))
}}
key={`${page.title}__${section.heading}-item-index-${i}`}
key={`${page.path}__${section.heading}-item`}
value={`${FORCE_MOUNT_ITEM}--${page.title}__${section.heading}-item-index-${i}`}
type="block-link"
>
@@ -449,7 +450,7 @@ const DocsSearch = () => {
const key = question.replace(/\s+/g, '_')
return (
<CommandItem
disabled={isLoading}
disabled={hasResults}
onSelect={() => {
if (!search) {
handleSearch(question)
@@ -499,11 +500,9 @@ const DocsSearch = () => {
export default DocsSearch
export function formatPageUrl(page: Page) {
const docsUrl = getDocsUrl()
switch (page.type) {
case PageType.Markdown:
case PageType.Reference:
return `${docsUrl}${page.path}`
case PageType.GithubDiscussion:
return page.path
default:
@@ -546,15 +545,3 @@ export function getPageSectionIcon(page: Page) {
throw new Error(`Unknown page type '${page.type}'`)
}
}
export function openLink(pageType: PageType, link: string) {
switch (pageType) {
case PageType.Markdown:
case PageType.Reference:
return window.location.assign(link)
case PageType.GithubDiscussion:
return window.open(link, '_blank')
default:
throw new Error(`Unknown page type '${pageType}'`)
}
}

68
supabase/migrations/20231127222412_search_full_text_for_fts.sql Normal file
View File

@@ -0,0 +1,68 @@
-- remove unused column
alter table page
drop column parent_page_id;
-- move indexed content for fts search from page_section to page
-- this should allow better rankings as it gives a better overview of
-- search term frequency on that page
drop index fts_search_index;
alter table page_section
drop column fts_tokens;
alter table page
add column content text;
alter table page
add column fts_tokens tsvector generated always as (to_tsvector('english', content)) stored;
create index fts_search_index_page on page using gin(fts_tokens);
-- also search against the page title if it exists, to give more
-- intuitive search rankings
alter table page
add column title_tokens tsvector generated always as (to_tsvector('english', coalesce(meta ->> 'title', ''))) stored;
create index fts_search_index_title on page using gin(title_tokens);
-- rank search by best match (title matches tend to rank better than content matches
-- due to underlying ts_rank algorithm
drop function docs_search_fts;
create or replace function docs_search_fts(query text)
returns table (
id int8,
path text,
type text,
title text,
subtitle text,
description text
)
language plpgsql
as $$
#variable_conflict use_variable
begin
return query
select
page.id,
page.path,
page.type,
page.meta ->> 'title' as title,
page.meta ->> 'subtitle' as subtitle,
page.meta ->> 'description' as description
from page
where title_tokens @@ websearch_to_tsquery(query) or fts_tokens @@ websearch_to_tsquery(query)
order by greatest(
-- Title is more important than body, so use 10 as the weighting factor
-- Cut off at max rank of 1
least(10 * ts_rank(title_tokens, websearch_to_tsquery(query)), 1),
ts_rank(fts_tokens, websearch_to_tsquery(query))
) desc
limit 10;
end;
$$;