Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
| 1 | import type { AlgoliaDocsRecord } from '@/lib/search-index'; |
| 2 | import { normalizeKnowledgeText } from './search'; |
| 3 | import type { KnowledgeSemanticRecord } from './semantic-artifact'; |
| 4 | import { |
| 5 | KNOWLEDGE_SOURCE_LABELS, |
| 6 | type KnowledgeSourceType, |
| 7 | type ProductAreaSlug, |
| 8 | } from './types'; |
| 9 | |
| 10 | export interface PublicKnowledgeCandidateRecord { |
| 11 | objectID: string; |
| 12 | sourceType: KnowledgeSourceType; |
| 13 | sourceLabel: string; |
| 14 | pageID: string; |
| 15 | title: string; |
| 16 | section: string | null; |
| 17 | description: string; |
| 18 | content: string; |
| 19 | canonicalUrl: string; |
| 20 | breadcrumbs: string[]; |
| 21 | productAreas: ProductAreaSlug[]; |
| 22 | toolkitSlugs: string[]; |
| 23 | keywords: string[]; |
| 24 | slug: string; |
| 25 | toolNames: string[]; |
| 26 | toolSlugs: string[]; |
| 27 | pageRank: number; |
| 28 | sectionRank: number; |
| 29 | lastVerifiedAt: string | null; |
| 30 | } |
| 31 | |
| 32 | /** @deprecated Use PublicKnowledgeCandidateRecord. */ |
| 33 | export type PublicKbCandidateRecord = PublicKnowledgeCandidateRecord; |
| 34 | |
| 35 | export interface FusedKnowledgeCandidate { |
| 36 | record: PublicKnowledgeCandidateRecord; |
| 37 | exactTier: number; |
| 38 | rrfScore: number; |
| 39 | keywordRank: number | null; |
| 40 | semanticRank: number | null; |
| 41 | } |
| 42 | |
| 43 | /** Converts any record classified for the public index into a keyword candidate. */ |
| 44 | export function publicKnowledgeCandidateFromSearchRecord( |
| 45 | record: AlgoliaDocsRecord, |
| 46 | ): PublicKnowledgeCandidateRecord { |
| 47 | return { |
| 48 | objectID: record.objectID, |
| 49 | sourceType: record.source_type, |
| 50 | sourceLabel: KNOWLEDGE_SOURCE_LABELS[record.source_type], |
| 51 | pageID: record.page_id, |
| 52 | title: record.title, |
| 53 | section: record.section ?? null, |
| 54 | description: record.description ?? '', |
| 55 | content: record.content, |
| 56 | canonicalUrl: record.canonical_url, |
| 57 | breadcrumbs: record.breadcrumbs ?? [], |
| 58 | productAreas: record.product_areas, |
| 59 | toolkitSlugs: record.toolkit_slugs, |
| 60 | keywords: record.keywords ?? [], |
| 61 | slug: record.slug ?? '', |
| 62 | toolNames: record.tool_names ?? [], |
| 63 | toolSlugs: record.tool_slugs ?? [], |
| 64 | pageRank: record.page_rank, |
| 65 | sectionRank: record.section_rank, |
| 66 | lastVerifiedAt: record.last_verified_at, |
| 67 | }; |
| 68 | } |
| 69 | |
| 70 | /** @deprecated Use publicKnowledgeCandidateFromSearchRecord. */ |
| 71 | export function publicKbCandidateFromAlgolia(record: AlgoliaDocsRecord): PublicKnowledgeCandidateRecord { |
| 72 | if (record.source_type !== 'kb') throw new Error(`Expected a KB record: ${record.objectID}`); |
| 73 | return publicKnowledgeCandidateFromSearchRecord(record); |
| 74 | } |
| 75 | |
| 76 | export function publicKnowledgeCandidateFromSemantic( |
| 77 | record: KnowledgeSemanticRecord, |
| 78 | ): PublicKnowledgeCandidateRecord { |
| 79 | return { |
| 80 | objectID: record.objectID, |
| 81 | sourceType: record.sourceType, |
| 82 | sourceLabel: record.sourceLabel, |
| 83 | pageID: record.pageID, |
| 84 | title: record.title, |
| 85 | section: record.section, |
| 86 | description: record.description, |
| 87 | content: record.content, |
| 88 | canonicalUrl: record.canonicalUrl, |
| 89 | breadcrumbs: record.breadcrumbs, |
| 90 | productAreas: record.productAreas, |
| 91 | toolkitSlugs: record.toolkitSlugs, |
| 92 | keywords: record.keywords, |
| 93 | slug: record.slug, |
| 94 | toolNames: record.toolNames, |
| 95 | toolSlugs: record.toolSlugs, |
| 96 | pageRank: record.pageRank, |
| 97 | sectionRank: record.sectionRank, |
| 98 | lastVerifiedAt: record.lastVerifiedAt, |
| 99 | }; |
| 100 | } |
| 101 | |
| 102 | /** @deprecated Use publicKnowledgeCandidateFromSemantic. */ |
| 103 | export const publicKbCandidateFromSemantic = publicKnowledgeCandidateFromSemantic; |
| 104 | |
| 105 | export function isStrongLexicalCandidate( |
| 106 | record: PublicKnowledgeCandidateRecord, |
| 107 | query: string, |
| 108 | ): boolean { |
| 109 | const normalizedQuery = normalizeKnowledgeText(query); |
| 110 | if (!normalizedQuery) return false; |
| 111 | const title = normalizeKnowledgeText(record.title); |
| 112 | if (title === normalizedQuery) return true; |
| 113 | const identity = [ |
| 114 | ...record.keywords, |
| 115 | record.slug, |
| 116 | ...record.toolNames, |
| 117 | ...record.toolSlugs, |
| 118 | ...record.toolkitSlugs, |
| 119 | ].map(normalizeKnowledgeText).filter(Boolean); |
| 120 | return identity.some(value => value === normalizedQuery); |
| 121 | } |
| 122 | |
| 123 | function exactTier(record: PublicKnowledgeCandidateRecord, normalizedQuery: string): number { |
| 124 | if (!normalizedQuery) return 0; |
| 125 | if (isStrongLexicalCandidate(record, normalizedQuery)) { |
| 126 | return normalizeKnowledgeText(record.title) === normalizedQuery ? 3 : 2; |
| 127 | } |
| 128 | const title = normalizeKnowledgeText(record.title); |
| 129 | const identity = [ |
| 130 | ...record.keywords, |
| 131 | record.slug, |
| 132 | ...record.toolNames, |
| 133 | ...record.toolSlugs, |
| 134 | ...record.toolkitSlugs, |
| 135 | ].map(normalizeKnowledgeText).filter(Boolean); |
| 136 | const phraseFields = [title, ...identity, normalizeKnowledgeText(record.section ?? ''), |
| 137 | normalizeKnowledgeText(record.description), normalizeKnowledgeText(record.content)]; |
| 138 | if (phraseFields.some(value => value.includes(normalizedQuery))) return 1; |
| 139 | return 0; |
| 140 | } |
| 141 | |
| 142 | function canonicalPage(url: string): string { |
| 143 | return url.split('#', 1)[0] ?? url; |
| 144 | } |
| 145 | |
| 146 | export function fusePublicKbCandidates(input: { |
| 147 | query: string; |
| 148 | keyword: PublicKnowledgeCandidateRecord[]; |
| 149 | semantic: PublicKnowledgeCandidateRecord[]; |
| 150 | limit: number; |
| 151 | rrfConstant?: number; |
| 152 | }): FusedKnowledgeCandidate[] { |
| 153 | const rrfConstant = input.rrfConstant ?? 60; |
| 154 | const byObjectID = new Map<string, FusedKnowledgeCandidate>(); |
| 155 | const add = ( |
| 156 | record: PublicKnowledgeCandidateRecord, |
| 157 | source: 'keyword' | 'semantic', |
| 158 | rank: number, |
| 159 | ) => { |
| 160 | const current = byObjectID.get(record.objectID) ?? { |
| 161 | record, |
| 162 | exactTier: exactTier(record, normalizeKnowledgeText(input.query)), |
| 163 | rrfScore: 0, |
| 164 | keywordRank: null, |
| 165 | semanticRank: null, |
| 166 | }; |
| 167 | current.rrfScore += 1 / (rrfConstant + rank); |
| 168 | if (source === 'keyword') current.keywordRank = rank; |
| 169 | else current.semanticRank = rank; |
| 170 | byObjectID.set(record.objectID, current); |
| 171 | }; |
| 172 | input.keyword.slice(0, 50).forEach((record, index) => add(record, 'keyword', index + 1)); |
| 173 | input.semantic.slice(0, 50).forEach((record, index) => add(record, 'semantic', index + 1)); |
| 174 | |
| 175 | const ranked = [...byObjectID.values()].sort((left, right) => |
| 176 | right.exactTier - left.exactTier || |
| 177 | right.rrfScore - left.rrfScore || |
| 178 | right.record.pageRank - left.record.pageRank || |
| 179 | right.record.sectionRank - left.record.sectionRank || |
| 180 | left.record.title.localeCompare(right.record.title) || |
| 181 | left.record.objectID.localeCompare(right.record.objectID), |
| 182 | ); |
| 183 | const bestByPage = new Map<string, FusedKnowledgeCandidate>(); |
| 184 | for (const candidate of ranked) { |
| 185 | const page = canonicalPage(candidate.record.canonicalUrl); |
| 186 | if (!bestByPage.has(page)) bestByPage.set(page, candidate); |
| 187 | } |
| 188 | return [...bestByPage.values()].slice(0, Math.min(Math.max(0, input.limit), 20)); |
| 189 | } |
| 190 | |
| 191 | /** @deprecated Use FusedKnowledgeCandidate. */ |
| 192 | export type FusedKbCandidate = FusedKnowledgeCandidate; |