@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
ts 192 lines 6.45 KB
Raw
1 import type { AlgoliaDocsRecord } from '@/lib/search-index';
2 import { normalizeKnowledgeText } from './search';
3 import type { KnowledgeSemanticRecord } from './semantic-artifact';
4 import {
5 KNOWLEDGE_SOURCE_LABELS,
6 type KnowledgeSourceType,
7 type ProductAreaSlug,
8 } from './types';
9
10 export interface PublicKnowledgeCandidateRecord {
11 objectID: string;
12 sourceType: KnowledgeSourceType;
13 sourceLabel: string;
14 pageID: string;
15 title: string;
16 section: string | null;
17 description: string;
18 content: string;
19 canonicalUrl: string;
20 breadcrumbs: string[];
21 productAreas: ProductAreaSlug[];
22 toolkitSlugs: string[];
23 keywords: string[];
24 slug: string;
25 toolNames: string[];
26 toolSlugs: string[];
27 pageRank: number;
28 sectionRank: number;
29 lastVerifiedAt: string | null;
30 }
31
32 /** @deprecated Use PublicKnowledgeCandidateRecord. */
33 export type PublicKbCandidateRecord = PublicKnowledgeCandidateRecord;
34
35 export interface FusedKnowledgeCandidate {
36 record: PublicKnowledgeCandidateRecord;
37 exactTier: number;
38 rrfScore: number;
39 keywordRank: number | null;
40 semanticRank: number | null;
41 }
42
43 /** Converts any record classified for the public index into a keyword candidate. */
44 export function publicKnowledgeCandidateFromSearchRecord(
45 record: AlgoliaDocsRecord,
46 ): PublicKnowledgeCandidateRecord {
47 return {
48 objectID: record.objectID,
49 sourceType: record.source_type,
50 sourceLabel: KNOWLEDGE_SOURCE_LABELS[record.source_type],
51 pageID: record.page_id,
52 title: record.title,
53 section: record.section ?? null,
54 description: record.description ?? '',
55 content: record.content,
56 canonicalUrl: record.canonical_url,
57 breadcrumbs: record.breadcrumbs ?? [],
58 productAreas: record.product_areas,
59 toolkitSlugs: record.toolkit_slugs,
60 keywords: record.keywords ?? [],
61 slug: record.slug ?? '',
62 toolNames: record.tool_names ?? [],
63 toolSlugs: record.tool_slugs ?? [],
64 pageRank: record.page_rank,
65 sectionRank: record.section_rank,
66 lastVerifiedAt: record.last_verified_at,
67 };
68 }
69
70 /** @deprecated Use publicKnowledgeCandidateFromSearchRecord. */
71 export function publicKbCandidateFromAlgolia(record: AlgoliaDocsRecord): PublicKnowledgeCandidateRecord {
72 if (record.source_type !== 'kb') throw new Error(`Expected a KB record: ${record.objectID}`);
73 return publicKnowledgeCandidateFromSearchRecord(record);
74 }
75
76 export function publicKnowledgeCandidateFromSemantic(
77 record: KnowledgeSemanticRecord,
78 ): PublicKnowledgeCandidateRecord {
79 return {
80 objectID: record.objectID,
81 sourceType: record.sourceType,
82 sourceLabel: record.sourceLabel,
83 pageID: record.pageID,
84 title: record.title,
85 section: record.section,
86 description: record.description,
87 content: record.content,
88 canonicalUrl: record.canonicalUrl,
89 breadcrumbs: record.breadcrumbs,
90 productAreas: record.productAreas,
91 toolkitSlugs: record.toolkitSlugs,
92 keywords: record.keywords,
93 slug: record.slug,
94 toolNames: record.toolNames,
95 toolSlugs: record.toolSlugs,
96 pageRank: record.pageRank,
97 sectionRank: record.sectionRank,
98 lastVerifiedAt: record.lastVerifiedAt,
99 };
100 }
101
102 /** @deprecated Use publicKnowledgeCandidateFromSemantic. */
103 export const publicKbCandidateFromSemantic = publicKnowledgeCandidateFromSemantic;
104
105 export function isStrongLexicalCandidate(
106 record: PublicKnowledgeCandidateRecord,
107 query: string,
108 ): boolean {
109 const normalizedQuery = normalizeKnowledgeText(query);
110 if (!normalizedQuery) return false;
111 const title = normalizeKnowledgeText(record.title);
112 if (title === normalizedQuery) return true;
113 const identity = [
114 ...record.keywords,
115 record.slug,
116 ...record.toolNames,
117 ...record.toolSlugs,
118 ...record.toolkitSlugs,
119 ].map(normalizeKnowledgeText).filter(Boolean);
120 return identity.some(value => value === normalizedQuery);
121 }
122
123 function exactTier(record: PublicKnowledgeCandidateRecord, normalizedQuery: string): number {
124 if (!normalizedQuery) return 0;
125 if (isStrongLexicalCandidate(record, normalizedQuery)) {
126 return normalizeKnowledgeText(record.title) === normalizedQuery ? 3 : 2;
127 }
128 const title = normalizeKnowledgeText(record.title);
129 const identity = [
130 ...record.keywords,
131 record.slug,
132 ...record.toolNames,
133 ...record.toolSlugs,
134 ...record.toolkitSlugs,
135 ].map(normalizeKnowledgeText).filter(Boolean);
136 const phraseFields = [title, ...identity, normalizeKnowledgeText(record.section ?? ''),
137 normalizeKnowledgeText(record.description), normalizeKnowledgeText(record.content)];
138 if (phraseFields.some(value => value.includes(normalizedQuery))) return 1;
139 return 0;
140 }
141
142 function canonicalPage(url: string): string {
143 return url.split('#', 1)[0] ?? url;
144 }
145
146 export function fusePublicKbCandidates(input: {
147 query: string;
148 keyword: PublicKnowledgeCandidateRecord[];
149 semantic: PublicKnowledgeCandidateRecord[];
150 limit: number;
151 rrfConstant?: number;
152 }): FusedKnowledgeCandidate[] {
153 const rrfConstant = input.rrfConstant ?? 60;
154 const byObjectID = new Map<string, FusedKnowledgeCandidate>();
155 const add = (
156 record: PublicKnowledgeCandidateRecord,
157 source: 'keyword' | 'semantic',
158 rank: number,
159 ) => {
160 const current = byObjectID.get(record.objectID) ?? {
161 record,
162 exactTier: exactTier(record, normalizeKnowledgeText(input.query)),
163 rrfScore: 0,
164 keywordRank: null,
165 semanticRank: null,
166 };
167 current.rrfScore += 1 / (rrfConstant + rank);
168 if (source === 'keyword') current.keywordRank = rank;
169 else current.semanticRank = rank;
170 byObjectID.set(record.objectID, current);
171 };
172 input.keyword.slice(0, 50).forEach((record, index) => add(record, 'keyword', index + 1));
173 input.semantic.slice(0, 50).forEach((record, index) => add(record, 'semantic', index + 1));
174
175 const ranked = [...byObjectID.values()].sort((left, right) =>
176 right.exactTier - left.exactTier ||
177 right.rrfScore - left.rrfScore ||
178 right.record.pageRank - left.record.pageRank ||
179 right.record.sectionRank - left.record.sectionRank ||
180 left.record.title.localeCompare(right.record.title) ||
181 left.record.objectID.localeCompare(right.record.objectID),
182 );
183 const bestByPage = new Map<string, FusedKnowledgeCandidate>();
184 for (const candidate of ranked) {
185 const page = canonicalPage(candidate.record.canonicalUrl);
186 if (!bestByPage.has(page)) bestByPage.set(page, candidate);
187 }
188 return [...bestByPage.values()].slice(0, Math.min(Math.max(0, input.limit), 20));
189 }
190
191 /** @deprecated Use FusedKnowledgeCandidate. */
192 export type FusedKbCandidate = FusedKnowledgeCandidate;