@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
ts 132 lines 4.28 KB
Raw
1 import { algoliasearch } from 'algoliasearch';
2 import {
3 ALGOLIA_DEFAULT_APP_ID,
4 ALGOLIA_DEFAULT_INDEX_NAME,
5 } from '@/lib/search-index';
6 import {
7 buildCompleteSearchReplacement,
8 replaceSearchDocuments,
9 } from '@/lib/knowledge/search-replacement';
10
11 function requireEnv(name: string, fallback?: string): string {
12 const value = process.env[name] ?? fallback;
13 if (!value) {
14 throw new Error(`Missing required environment variable: ${name}`);
15 }
16 return value;
17 }
18
19 const dryRun = process.argv.includes('--dry-run');
20 const showSamples = process.argv.includes('--samples');
21 const indexName =
22 process.env.ALGOLIA_INDEX_NAME ?? process.env.NEXT_PUBLIC_ALGOLIA_INDEX_NAME ?? ALGOLIA_DEFAULT_INDEX_NAME;
23
24 if (dryRun) {
25 const records = await buildCompleteSearchReplacement({ validateExternal: false });
26 if (records.length === 0) throw new Error('Refusing to sync an empty Algolia docs index.');
27 const uniquePages = new Set(records.map((record) => record.page_id)).size;
28 console.log(`Prepared ${records.length} Algolia records across ${uniquePages} docs pages for index "${indexName}".`);
29 const sourceCounts = Object.fromEntries(
30 Array.from(new Set(records.map((record) => record.source_type))).sort().map((sourceType) => [
31 sourceType,
32 records.filter((record) => record.source_type === sourceType).length,
33 ]),
34 );
35 console.log(`Source counts: ${JSON.stringify(sourceCounts)}`);
36
37 if (showSamples) {
38 console.log(JSON.stringify(records.slice(0, 5), null, 2));
39 }
40
41 process.exit(0);
42 }
43
44 const appId = requireEnv(
45 'ALGOLIA_APP_ID',
46 process.env.NEXT_PUBLIC_ALGOLIA_APP_ID ?? ALGOLIA_DEFAULT_APP_ID,
47 );
48 const adminApiKey = requireEnv('ALGOLIA_ADMIN_API_KEY');
49 const client = algoliasearch(appId, adminApiKey);
50
51 console.log('Validating all registered OAuth guides before changing Algolia...');
52 const records = await replaceSearchDocuments(
53 {
54 replaceAllObjects: ({ indexName: targetIndex, objects }) => client.replaceAllObjects({
55 indexName: targetIndex,
56 objects: objects.map((record) => ({ ...record })),
57 }),
58 },
59 indexName,
60 {
61 beforeReplace: async (preparedRecords) => {
62 if (preparedRecords.length === 0) {
63 throw new Error('Refusing to sync an empty Algolia docs index.');
64 }
65 console.log(`Configuring Algolia index "${indexName}"...`);
66 await client.setSettings({
67 indexName,
68 indexSettings: {
69 attributeForDistinct: 'page_id',
70 attributesToRetrieve: [
71 'objectID',
72 'title',
73 'description',
74 'section',
75 'content',
76 'url',
77 'section_id',
78 'breadcrumbs',
79 'page_id',
80 'type',
81 'keywords',
82 'slug',
83 'tool_names',
84 'tool_slugs',
85 'source_type',
86 'canonical_url',
87 'product_areas',
88 'toolkit_slugs',
89 'intents',
90 'last_verified_at',
91 ],
92 searchableAttributes: [
93 'unordered(title)',
94 'unordered(keywords)',
95 'unordered(slug)',
96 'unordered(section)',
97 'unordered(headings)',
98 'unordered(tool_names)',
99 'unordered(tool_slugs)',
100 'unordered(description)',
101 'unordered(content)',
102 ],
103 customRanking: [
104 'desc(page_rank)',
105 'desc(section_rank)',
106 ],
107 attributesForFaceting: [
108 'filterOnly(type)',
109 'filterOnly(source_type)',
110 'filterOnly(product_areas)',
111 'filterOnly(toolkit_slugs)',
112 'filterOnly(intents)',
113 'filterOnly(lang)',
114 'searchable(tags)',
115 ],
116 attributesToHighlight: ['title', 'section', 'content'],
117 attributesToSnippet: ['content:30'],
118 highlightPreTag: '<mark>',
119 highlightPostTag: '</mark>',
120 ignorePlurals: true,
121 minProximity: 1,
122 removeStopWords: false,
123 removeWordsIfNoResults: 'lastWords',
124 typoTolerance: true,
125 advancedSyntax: true,
126 },
127 });
128 console.log(`Replacing ${preparedRecords.length} records in Algolia index "${indexName}"...`);
129 },
130 },
131 );
132 console.log(`Synced ${records.length} records to Algolia docs search index "${indexName}".`);