Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 03:37 UTC
| 1 | import { algoliasearch } from 'algoliasearch'; |
| 2 | import { |
| 3 | ALGOLIA_DEFAULT_APP_ID, |
| 4 | ALGOLIA_DEFAULT_INDEX_NAME, |
| 5 | } from '@/lib/search-index'; |
| 6 | import { |
| 7 | buildCompleteSearchReplacement, |
| 8 | replaceSearchDocuments, |
| 9 | } from '@/lib/knowledge/search-replacement'; |
| 10 | |
| 11 | function requireEnv(name: string, fallback?: string): string { |
| 12 | const value = process.env[name] ?? fallback; |
| 13 | if (!value) { |
| 14 | throw new Error(`Missing required environment variable: ${name}`); |
| 15 | } |
| 16 | return value; |
| 17 | } |
| 18 | |
| 19 | const dryRun = process.argv.includes('--dry-run'); |
| 20 | const showSamples = process.argv.includes('--samples'); |
| 21 | const indexName = |
| 22 | process.env.ALGOLIA_INDEX_NAME ?? process.env.NEXT_PUBLIC_ALGOLIA_INDEX_NAME ?? ALGOLIA_DEFAULT_INDEX_NAME; |
| 23 | |
| 24 | if (dryRun) { |
| 25 | const records = await buildCompleteSearchReplacement({ validateExternal: false }); |
| 26 | if (records.length === 0) throw new Error('Refusing to sync an empty Algolia docs index.'); |
| 27 | const uniquePages = new Set(records.map((record) => record.page_id)).size; |
| 28 | console.log(`Prepared ${records.length} Algolia records across ${uniquePages} docs pages for index "${indexName}".`); |
| 29 | const sourceCounts = Object.fromEntries( |
| 30 | Array.from(new Set(records.map((record) => record.source_type))).sort().map((sourceType) => [ |
| 31 | sourceType, |
| 32 | records.filter((record) => record.source_type === sourceType).length, |
| 33 | ]), |
| 34 | ); |
| 35 | console.log(`Source counts: ${JSON.stringify(sourceCounts)}`); |
| 36 | |
| 37 | if (showSamples) { |
| 38 | console.log(JSON.stringify(records.slice(0, 5), null, 2)); |
| 39 | } |
| 40 | |
| 41 | process.exit(0); |
| 42 | } |
| 43 | |
| 44 | const appId = requireEnv( |
| 45 | 'ALGOLIA_APP_ID', |
| 46 | process.env.NEXT_PUBLIC_ALGOLIA_APP_ID ?? ALGOLIA_DEFAULT_APP_ID, |
| 47 | ); |
| 48 | const adminApiKey = requireEnv('ALGOLIA_ADMIN_API_KEY'); |
| 49 | const client = algoliasearch(appId, adminApiKey); |
| 50 | |
| 51 | console.log('Validating all registered OAuth guides before changing Algolia...'); |
| 52 | const records = await replaceSearchDocuments( |
| 53 | { |
| 54 | replaceAllObjects: ({ indexName: targetIndex, objects }) => client.replaceAllObjects({ |
| 55 | indexName: targetIndex, |
| 56 | objects: objects.map((record) => ({ ...record })), |
| 57 | }), |
| 58 | }, |
| 59 | indexName, |
| 60 | { |
| 61 | beforeReplace: async (preparedRecords) => { |
| 62 | if (preparedRecords.length === 0) { |
| 63 | throw new Error('Refusing to sync an empty Algolia docs index.'); |
| 64 | } |
| 65 | console.log(`Configuring Algolia index "${indexName}"...`); |
| 66 | await client.setSettings({ |
| 67 | indexName, |
| 68 | indexSettings: { |
| 69 | attributeForDistinct: 'page_id', |
| 70 | attributesToRetrieve: [ |
| 71 | 'objectID', |
| 72 | 'title', |
| 73 | 'description', |
| 74 | 'section', |
| 75 | 'content', |
| 76 | 'url', |
| 77 | 'section_id', |
| 78 | 'breadcrumbs', |
| 79 | 'page_id', |
| 80 | 'type', |
| 81 | 'keywords', |
| 82 | 'slug', |
| 83 | 'tool_names', |
| 84 | 'tool_slugs', |
| 85 | 'source_type', |
| 86 | 'canonical_url', |
| 87 | 'product_areas', |
| 88 | 'toolkit_slugs', |
| 89 | 'intents', |
| 90 | 'last_verified_at', |
| 91 | ], |
| 92 | searchableAttributes: [ |
| 93 | 'unordered(title)', |
| 94 | 'unordered(keywords)', |
| 95 | 'unordered(slug)', |
| 96 | 'unordered(section)', |
| 97 | 'unordered(headings)', |
| 98 | 'unordered(tool_names)', |
| 99 | 'unordered(tool_slugs)', |
| 100 | 'unordered(description)', |
| 101 | 'unordered(content)', |
| 102 | ], |
| 103 | customRanking: [ |
| 104 | 'desc(page_rank)', |
| 105 | 'desc(section_rank)', |
| 106 | ], |
| 107 | attributesForFaceting: [ |
| 108 | 'filterOnly(type)', |
| 109 | 'filterOnly(source_type)', |
| 110 | 'filterOnly(product_areas)', |
| 111 | 'filterOnly(toolkit_slugs)', |
| 112 | 'filterOnly(intents)', |
| 113 | 'filterOnly(lang)', |
| 114 | 'searchable(tags)', |
| 115 | ], |
| 116 | attributesToHighlight: ['title', 'section', 'content'], |
| 117 | attributesToSnippet: ['content:30'], |
| 118 | highlightPreTag: '<mark>', |
| 119 | highlightPostTag: '</mark>', |
| 120 | ignorePlurals: true, |
| 121 | minProximity: 1, |
| 122 | removeStopWords: false, |
| 123 | removeWordsIfNoResults: 'lastWords', |
| 124 | typoTolerance: true, |
| 125 | advancedSyntax: true, |
| 126 | }, |
| 127 | }); |
| 128 | console.log(`Replacing ${preparedRecords.length} records in Algolia index "${indexName}"...`); |
| 129 | }, |
| 130 | }, |
| 131 | ); |
| 132 | console.log(`Synced ${records.length} records to Algolia docs search index "${indexName}".`); |