@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
ts 76 lines 2.9 KB
Raw
1 import { existsSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
2 import { join } from 'node:path';
3 import { getAlgoliaSearchDocuments } from '@/lib/search-index';
4 import { embedTexts } from '@/lib/knowledge/embeddings';
5 import {
6 buildSemanticArtifact,
7 docsContentHashFromRecords,
8 semanticRecordFromSearchRecord,
9 SemanticArtifactStaleError,
10 validateSemanticArtifact,
11 type KnowledgeSemanticArtifact,
12 } from '@/lib/knowledge/semantic-artifact';
13
14 const artifactPath = join(process.cwd(), 'kb', 'semantic-index.json');
15
16 const records = (await getAlgoliaSearchDocuments())
17 .filter(record => record.source_type === 'kb' || record.source_type === 'docs')
18 .sort((left, right) => left.objectID.localeCompare(right.objectID));
19 const contentHashes = new Map(
20 records.map(record => {
21 const semantic = semanticRecordFromSearchRecord(record);
22 return [semantic.objectID, semantic.contentHash];
23 }),
24 );
25 const docsContentHash = docsContentHashFromRecords(
26 records.map(semanticRecordFromSearchRecord),
27 );
28
29 if (process.argv.includes('--check')) {
30 if (!existsSync(artifactPath)) throw new Error('KB semantic artifact is missing');
31 const artifact = JSON.parse(readFileSync(artifactPath, 'utf8')) as KnowledgeSemanticArtifact;
32 try {
33 validateSemanticArtifact(artifact, {
34 docsContentHash,
35 contentHashes,
36 });
37 } catch (error) {
38 if (!(error instanceof SemanticArtifactStaleError)) throw error;
39 const message = `${error.message}. Search falls back to keyword results until the artifact is rebuilt. ` +
40 'Run bun run build:kb-semantic with OPENAI_API_KEY, or use the Docs - Rebuild KB Semantic Artifact workflow.';
41 if (!process.argv.includes('--allow-stale')) throw new Error(message);
42 console.warn(`${process.env.GITHUB_ACTIONS === 'true' ? '::warning::' : 'Warning: '}${message}`);
43 process.exit(0);
44 }
45 console.log(`KB semantic artifact is current: ${artifact.records.length} records.`);
46 process.exit(0);
47 }
48
49 const apiKey = process.env.OPENAI_API_KEY;
50 if (!apiKey) throw new Error('OPENAI_API_KEY is required to build the KB semantic artifact');
51 const previousArtifact = existsSync(artifactPath)
52 ? JSON.parse(readFileSync(artifactPath, 'utf8')) as KnowledgeSemanticArtifact
53 : undefined;
54
55 const artifact = await buildSemanticArtifact({
56 records,
57 docsContentHash,
58 builtAt: new Date().toISOString(),
59 previousArtifact,
60 embed: async texts => {
61 const vectors: number[][] = [];
62 for (let start = 0; start < texts.length; start += 64) {
63 vectors.push(...await embedTexts(texts.slice(start, start + 64), { apiKey }));
64 }
65 return vectors;
66 },
67 });
68
69 const temporaryPath = `${artifactPath}.tmp-${process.pid}`;
70 try {
71 writeFileSync(temporaryPath, `${JSON.stringify(artifact)}\n`, 'utf8');
72 renameSync(temporaryPath, artifactPath);
73 } finally {
74 rmSync(temporaryPath, { force: true });
75 }
76 console.log(`Built KB semantic artifact: ${artifact.records.length} records.`);