Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 03:37 UTC
| 1 | // Use direct imports from collections to avoid top-level await in lib/source.ts. |
| 2 | import { resolveToolkitCount } from './toolkit-count'; |
| 3 | import { createHash } from 'node:crypto'; |
| 4 | import { existsSync, readdirSync, readFileSync } from 'node:fs'; |
| 5 | import { join, relative } from 'node:path'; |
| 6 | import { |
| 7 | docs, |
| 8 | reference, |
| 9 | examples, |
| 10 | toolkits, |
| 11 | knowledgeBase, |
| 12 | } from 'fumadocs-mdx:collections/server'; |
| 13 | import { loader, multiple } from 'fumadocs-core/source'; |
| 14 | import { lucideIconsPlugin } from 'fumadocs-core/source/lucide-icons'; |
| 15 | import { openapiSource, openapiPlugin } from 'fumadocs-openapi/server'; |
| 16 | import type { AdvancedIndex } from 'fumadocs-core/search/server'; |
| 17 | import type { BaseIndex } from 'fumadocs-core/search/algolia'; |
| 18 | import { openapi } from '@/lib/openapi'; |
| 19 | import { mdxToCleanMarkdown } from '@/lib/source'; |
| 20 | import { isHiddenApiTagUrl } from '@/lib/filter-api-version'; |
| 21 | import { getAllToolkitsSync } from '@/lib/toolkit-data'; |
| 22 | import { |
| 23 | classifyKnowledgeRecord, |
| 24 | normalizeKnowledgeKeywords, |
| 25 | } from '@/lib/knowledge/metadata'; |
| 26 | import { getAuthGuideSearchRecords } from '@/lib/knowledge/auth-guides'; |
| 27 | import type { |
| 28 | KnowledgeIntent, |
| 29 | KnowledgeMetadata, |
| 30 | KnowledgeSourceType, |
| 31 | ProductAreaSlug, |
| 32 | } from '@/lib/knowledge/types'; |
| 33 | |
| 34 | export const ALGOLIA_DEFAULT_APP_ID = '62HI9PQZ1L'; |
| 35 | export const ALGOLIA_DEFAULT_INDEX_NAME = 'docs_composio'; |
| 36 | |
| 37 | const MAX_CHUNK_CHARS = 3_800; |
| 38 | const MAX_CHUNK_BYTES = 9_000; |
| 39 | const MAX_TOOL_ALIAS_ITEMS = 80; |
| 40 | const MAX_TOOL_ALIAS_BYTES = 2_500; |
| 41 | |
| 42 | // Create loaders directly here to avoid the problematic lib/source.ts import in the |
| 43 | // fallback route. This route is intentionally still Fumadocs/Orama-backed for local |
| 44 | // development when Algolia env vars aren't configured. |
| 45 | const docsSource = loader({ |
| 46 | baseUrl: '/docs', |
| 47 | source: docs.toFumadocsSource(), |
| 48 | plugins: [lucideIconsPlugin()], |
| 49 | }); |
| 50 | |
| 51 | const examplesSource = loader({ |
| 52 | baseUrl: '/examples', |
| 53 | source: examples.toFumadocsSource(), |
| 54 | plugins: [lucideIconsPlugin()], |
| 55 | }); |
| 56 | |
| 57 | const toolkitsSource = loader({ |
| 58 | baseUrl: '/toolkits', |
| 59 | source: toolkits.toFumadocsSource(), |
| 60 | plugins: [lucideIconsPlugin()], |
| 61 | }); |
| 62 | |
| 63 | const knowledgeBaseSource = loader({ |
| 64 | baseUrl: '/kb', |
| 65 | source: knowledgeBase.toFumadocsSource(), |
| 66 | plugins: [lucideIconsPlugin()], |
| 67 | }); |
| 68 | |
| 69 | type SearchIndex = AdvancedIndex & { |
| 70 | keywords?: string[]; |
| 71 | }; |
| 72 | |
| 73 | export type AlgoliaDocsRecord = BaseIndex & KnowledgeMetadata & { |
| 74 | objectID: string; |
| 75 | description?: string; |
| 76 | keywords?: string[]; |
| 77 | slug?: string; |
| 78 | headings?: string[]; |
| 79 | tool_names?: string[]; |
| 80 | tool_slugs?: string[]; |
| 81 | type: string; |
| 82 | lang: string; |
| 83 | page_rank: number; |
| 84 | toolkit_popularity: number; |
| 85 | section_rank: number; |
| 86 | position: number; |
| 87 | depth: number; |
| 88 | tags?: string[]; |
| 89 | }; |
| 90 | |
| 91 | function getFrontmatter(source: string): string { |
| 92 | return source.match(/^---\n([\s\S]*?)\n---/)?.[1] ?? ''; |
| 93 | } |
| 94 | |
| 95 | function getFrontmatterValue(frontmatter: string, key: string): string | undefined { |
| 96 | // The double-quoted alternative must accept YAML escapes: a title like |
| 97 | // `"Resolve an \"App is blocked\" error"` otherwise fails the quoted branch, |
| 98 | // falls through to the catch-all, and surfaces raw quotes-and-backslashes in |
| 99 | // search results and browse pages. |
| 100 | const match = frontmatter.match( |
| 101 | new RegExp(`^${key}:\\s*(?:"((?:[^"\\\\]|\\\\.)*)"|'([^']*)'|(.+))$`, 'm') |
| 102 | ); |
| 103 | if (match?.[1] !== undefined) { |
| 104 | return match[1].replace(/\\(["\\])/g, '$1').trim(); |
| 105 | } |
| 106 | const value = match?.[2] ?? match?.[3]; |
| 107 | return value?.trim(); |
| 108 | } |
| 109 | |
| 110 | function getFrontmatterList(frontmatter: string, key: string): string[] { |
| 111 | const raw = getFrontmatterValue(frontmatter, key); |
| 112 | if (!raw) return []; |
| 113 | |
| 114 | if (raw.startsWith('[') && raw.endsWith(']')) { |
| 115 | return raw |
| 116 | .slice(1, -1) |
| 117 | .split(',') |
| 118 | .map((value) => value.trim().replace(/^['"]|['"]$/g, '')) |
| 119 | .filter(Boolean); |
| 120 | } |
| 121 | |
| 122 | return [raw]; |
| 123 | } |
| 124 | |
| 125 | function listContentFiles(dir: string): string[] { |
| 126 | if (!existsSync(dir)) return []; |
| 127 | |
| 128 | return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { |
| 129 | const path = join(dir, entry.name); |
| 130 | if (entry.isDirectory()) return listContentFiles(path); |
| 131 | if (entry.isFile() && /\.mdx?$/.test(entry.name)) return [path]; |
| 132 | return []; |
| 133 | }); |
| 134 | } |
| 135 | |
| 136 | function slugify(value: string): string { |
| 137 | return value |
| 138 | .toLowerCase() |
| 139 | .trim() |
| 140 | .replace(/[<>]/g, '') |
| 141 | .replace(/[`*_~\[\]()]/g, '') |
| 142 | .replace(/&[a-z]+;/g, '') |
| 143 | .replace(/[^\w\s-]/g, '') |
| 144 | .replace(/\s+/g, '-') |
| 145 | .replace(/-+/g, '-') |
| 146 | .replace(/^-|-$/g, ''); |
| 147 | } |
| 148 | |
| 149 | function uniqueSlug(value: string, seen: Map<string, number>): string { |
| 150 | const base = slugify(value) || 'section'; |
| 151 | const count = seen.get(base) ?? 0; |
| 152 | seen.set(base, count + 1); |
| 153 | return count === 0 ? base : `${base}-${count + 1}`; |
| 154 | } |
| 155 | |
| 156 | function splitIntoChunks(text: string): string[] { |
| 157 | const paragraphs = text |
| 158 | .split(/\n{2,}/) |
| 159 | .map((paragraph) => paragraph.trim()) |
| 160 | .filter(Boolean); |
| 161 | |
| 162 | const chunks: string[] = []; |
| 163 | let current = ''; |
| 164 | |
| 165 | const pushCurrent = () => { |
| 166 | const trimmed = current.trim(); |
| 167 | if (trimmed) chunks.push(trimmed); |
| 168 | current = ''; |
| 169 | }; |
| 170 | |
| 171 | for (const paragraph of paragraphs) { |
| 172 | const next = current ? `${current}\n\n${paragraph}` : paragraph; |
| 173 | if (next.length <= MAX_CHUNK_CHARS && Buffer.byteLength(next, 'utf8') <= MAX_CHUNK_BYTES) { |
| 174 | current = next; |
| 175 | continue; |
| 176 | } |
| 177 | |
| 178 | pushCurrent(); |
| 179 | |
| 180 | if (paragraph.length <= MAX_CHUNK_CHARS && Buffer.byteLength(paragraph, 'utf8') <= MAX_CHUNK_BYTES) { |
| 181 | current = paragraph; |
| 182 | continue; |
| 183 | } |
| 184 | |
| 185 | for (let i = 0; i < paragraph.length; i += MAX_CHUNK_CHARS) { |
| 186 | chunks.push(paragraph.slice(i, i + MAX_CHUNK_CHARS).trim()); |
| 187 | } |
| 188 | } |
| 189 | |
| 190 | pushCurrent(); |
| 191 | return chunks; |
| 192 | } |
| 193 | |
| 194 | function contentHash(value: string): string { |
| 195 | return createHash('sha1').update(value).digest('hex').slice(0, 10); |
| 196 | } |
| 197 | |
| 198 | // Sections intentionally kept out of search entirely (not just deprioritized). |
| 199 | // Direct tool execution is the legacy pre-sessions flow — we don't want it |
| 200 | // surfacing in results at all. Matches the prefix and anything beneath it. |
| 201 | const SEARCH_EXCLUDED_PREFIXES = ['/docs/tools-direct']; |
| 202 | |
| 203 | function isExcludedFromSearch(url: string): boolean { |
| 204 | const path = url.replace(/\/$/, ''); |
| 205 | return SEARCH_EXCLUDED_PREFIXES.some((prefix) => path === prefix || path.startsWith(`${prefix}/`)); |
| 206 | } |
| 207 | |
| 208 | function urlFromContentPath(path: string): { url: string; type: string } | undefined { |
| 209 | const rel = relative(join(process.cwd(), 'content'), path).replace(/\\/g, '/'); |
| 210 | const withoutExt = rel.replace(/\.mdx?$/, ''); |
| 211 | const parts = withoutExt.split('/'); |
| 212 | const collection = parts.shift(); |
| 213 | if (!collection) return undefined; |
| 214 | |
| 215 | if (collection === 'docs') { |
| 216 | return { url: `/docs/${parts.join('/')}`.replace(/\/index$/, ''), type: 'docs' }; |
| 217 | } |
| 218 | if (collection === 'examples') { |
| 219 | return { url: `/examples/${parts.join('/')}`.replace(/\/index$/, ''), type: 'examples' }; |
| 220 | } |
| 221 | if (collection === 'reference') { |
| 222 | const url = `/reference/${parts.join('/')}`.replace(/\/index$/, ''); |
| 223 | return { url, type: url.startsWith('/reference/v3') ? 'v3-reference' : 'reference' }; |
| 224 | } |
| 225 | if (collection === 'toolkits') { |
| 226 | if (parts[0] === 'faq') return undefined; |
| 227 | return { url: `/toolkits/${parts.join('/')}`.replace(/\/index$/, ''), type: 'toolkits' }; |
| 228 | } |
| 229 | if (collection === 'kb') { |
| 230 | return { url: `/kb/${parts.join('/')}`.replace(/\/index$/, ''), type: 'kb' }; |
| 231 | } |
| 232 | if (collection === 'changelog') return undefined; |
| 233 | |
| 234 | return undefined; |
| 235 | } |
| 236 | |
| 237 | function slugTokens(url: string): string { |
| 238 | return url |
| 239 | .split(/[/-]/) |
| 240 | .map((part) => part.replace(/[_#?=&.]/g, ' ')) |
| 241 | .join(' ') |
| 242 | .replace(/\s+/g, ' ') |
| 243 | .trim(); |
| 244 | } |
| 245 | |
| 246 | const typeLabels: Record<string, string> = { |
| 247 | docs: 'Docs', |
| 248 | examples: 'Example', |
| 249 | reference: 'Reference', |
| 250 | 'v3-reference': 'Legacy v3 Reference', |
| 251 | toolkits: 'Toolkit', |
| 252 | kb: 'Knowledge Base', |
| 253 | changelog: 'Changelog', |
| 254 | 'api-reference': 'API Reference', |
| 255 | }; |
| 256 | |
| 257 | function titleizeSlug(value: string): string { |
| 258 | return value |
| 259 | .split('-') |
| 260 | .filter(Boolean) |
| 261 | .map((part) => part.charAt(0).toUpperCase() + part.slice(1)) |
| 262 | .join(' ') |
| 263 | .replace(/\bApi\b/g, 'API') |
| 264 | .replace(/\bMcp\b/g, 'MCP') |
| 265 | .replace(/\bSdk\b/g, 'SDK') |
| 266 | .replace(/\bOauth\b/g, 'OAuth'); |
| 267 | } |
| 268 | |
| 269 | function breadcrumbsForUrl(url: string, type: string): string[] { |
| 270 | const label = typeLabels[type] ?? titleizeSlug(type); |
| 271 | |
| 272 | if (type === 'toolkits' || type === 'examples' || type === 'changelog') { |
| 273 | return [label]; |
| 274 | } |
| 275 | |
| 276 | const parts = url.split('/').filter(Boolean); |
| 277 | const parentParts = parts.slice(1, -1); |
| 278 | return [label, ...parentParts.map(titleizeSlug)].filter(Boolean); |
| 279 | } |
| 280 | |
| 281 | const TOOLKIT_POPULARITY_OVERRIDES: Record<string, number> = { |
| 282 | gmail: 1_000, |
| 283 | github: 980, |
| 284 | slack: 960, |
| 285 | googledrive: 940, |
| 286 | googlecalendar: 930, |
| 287 | notion: 920, |
| 288 | linear: 900, |
| 289 | jira: 890, |
| 290 | hubspot: 860, |
| 291 | salesforce: 850, |
| 292 | resend: 650, |
| 293 | zoho_mail: 520, |
| 294 | mailchimp: 500, |
| 295 | sendgrid: 480, |
| 296 | mailsoftly: 260, |
| 297 | }; |
| 298 | |
| 299 | function getToolkitSlugFromUrl(url: string): string | null { |
| 300 | return url.match(/^\/toolkits\/([^/#?]+)/)?.[1] ?? null; |
| 301 | } |
| 302 | |
| 303 | function toolkitPopularity(url: string, type: string): number { |
| 304 | if (type !== 'toolkits') return 0; |
| 305 | |
| 306 | const slug = getToolkitSlugFromUrl(url); |
| 307 | if (!slug) return 0; |
| 308 | |
| 309 | const toolkit = toolkitBySlug.get(slug); |
| 310 | const override = TOOLKIT_POPULARITY_OVERRIDES[slug] ?? 0; |
| 311 | const managedAuthBoost = toolkit?.composioManagedAuthSchemes?.length ? 120 : 0; |
| 312 | const authBoost = toolkit?.authSchemes?.includes('OAUTH2') ? 40 : 0; |
| 313 | const triggerBoost = Math.min((toolkit?.triggerCount ?? 0) * 10, 80); |
| 314 | const toolCountBoost = Math.min((toolkit?.toolCount ?? 0), 100); |
| 315 | |
| 316 | return override + managedAuthBoost + authBoost + triggerBoost + toolCountBoost; |
| 317 | } |
| 318 | |
| 319 | function pageRank(url: string, sourceType: KnowledgeSourceType): number { |
| 320 | // Prefer conceptual docs over generated/reference material when textual |
| 321 | // relevance is otherwise close. Toolkit aliases can still win earlier via |
| 322 | // searchableAttributes when the query matches a tool name/slug exactly. |
| 323 | // |
| 324 | // Hints use precise path matches against the current (nested) docs structure. |
| 325 | // When pages move, update these — a stale `.includes()` hint silently boosts |
| 326 | // nothing. See content/docs/ for the canonical layout. |
| 327 | if (sourceType === 'docs') { |
| 328 | const path = url.replace(/\/$/, ''); |
| 329 | if (path === '/docs') return 2_400; |
| 330 | if (path === '/docs/quickstart') return 2_300; |
| 331 | if (path === '/docs/how-composio-works') return 2_250; |
| 332 | if (path === '/docs/authentication') return 2_220; |
| 333 | if (path.startsWith('/docs/tools-direct/')) return 2_180; |
| 334 | if (path === '/docs/configuring-sessions') return 2_120; |
| 335 | if (path.startsWith('/docs/auth-configuration/')) return 2_080; |
| 336 | if (path === '/docs/triggers' || path.startsWith('/docs/setting-up-triggers/')) return 2_060; |
| 337 | return 2_000; |
| 338 | } |
| 339 | |
| 340 | if (sourceType === 'kb') return 1_900; |
| 341 | if (sourceType === 'oauth-guide') return 1_700; |
| 342 | if (sourceType === 'toolkit') return 1_500; |
| 343 | if (sourceType === 'example') return 1_300; |
| 344 | if (sourceType === 'reference') return 700; |
| 345 | if (sourceType === 'changelog') return 350; |
| 346 | if (sourceType === 'legacy') return 25; |
| 347 | return 400; |
| 348 | } |
| 349 | |
| 350 | function knowledgeSourceType(type: string, legacy: boolean): KnowledgeSourceType { |
| 351 | if (legacy || type === 'v3-reference') return 'legacy'; |
| 352 | if (type === 'docs') return 'docs'; |
| 353 | if (type === 'kb') return 'kb'; |
| 354 | if (type === 'toolkits') return 'toolkit'; |
| 355 | if (type === 'examples') return 'example'; |
| 356 | if (type === 'reference' || type === 'api-reference') return 'reference'; |
| 357 | if (type === 'changelog') return 'changelog'; |
| 358 | throw new Error(`Unsupported search source type: ${type}`); |
| 359 | } |
| 360 | |
| 361 | export function recordsFromMarkdownPage(input: { |
| 362 | url: string; |
| 363 | type: string; |
| 364 | title: string; |
| 365 | description?: string; |
| 366 | keywords?: string[]; |
| 367 | markdown: string; |
| 368 | breadcrumbs?: string[]; |
| 369 | tags?: string[]; |
| 370 | toolNames?: string[]; |
| 371 | toolSlugs?: string[]; |
| 372 | legacy?: boolean; |
| 373 | topics?: string[]; |
| 374 | productAreas?: ProductAreaSlug[]; |
| 375 | toolkitSlugs?: string[]; |
| 376 | intents?: KnowledgeIntent[]; |
| 377 | lastVerifiedAt?: string | null; |
| 378 | }): AlgoliaDocsRecord[] { |
| 379 | const isLegacy = input.legacy === true; |
| 380 | const sourceType = knowledgeSourceType(input.type, isLegacy); |
| 381 | const metadata = classifyKnowledgeRecord({ |
| 382 | sourceType, |
| 383 | canonicalUrl: input.url, |
| 384 | productAreas: input.productAreas, |
| 385 | topics: input.topics, |
| 386 | toolkitSlugs: input.toolkitSlugs, |
| 387 | intents: input.intents, |
| 388 | lastVerifiedAt: input.lastVerifiedAt, |
| 389 | }); |
| 390 | const resolvedPageRank = pageRank(input.url, sourceType); |
| 391 | const resolvedTags = isLegacy ? [...(input.tags ?? []), 'legacy'] : input.tags; |
| 392 | const resolvedKeywords = normalizeKnowledgeKeywords(input.keywords ?? []); |
| 393 | const clean = mdxToCleanMarkdown(input.markdown, input.url); |
| 394 | const lines = clean.split('\n'); |
| 395 | const headingSlugs = new Map<string, number>(); |
| 396 | const headings: string[] = []; |
| 397 | const sections: Array<{ heading?: string; section_id?: string; depth: number; text: string; position: number }> = []; |
| 398 | let currentHeading: string | undefined; |
| 399 | let currentSectionId: string | undefined; |
| 400 | let currentDepth = 0; |
| 401 | let currentLines: string[] = []; |
| 402 | let sectionPosition = 0; |
| 403 | let fenceMarker: string | null = null; |
| 404 | let fenceLength = 0; |
| 405 | |
| 406 | const flush = () => { |
| 407 | const text = currentLines.join('\n').trim(); |
| 408 | if (!text) return; |
| 409 | sections.push({ |
| 410 | heading: currentHeading, |
| 411 | section_id: currentSectionId, |
| 412 | depth: currentDepth, |
| 413 | text, |
| 414 | position: sectionPosition++, |
| 415 | }); |
| 416 | currentLines = []; |
| 417 | }; |
| 418 | |
| 419 | for (const line of lines) { |
| 420 | const fenceMatch = line.match(/^\s{0,3}(`{3,}|~{3,})(.*)$/); |
| 421 | if (fenceMatch) { |
| 422 | const marker = fenceMatch[1]; |
| 423 | const markerCharacter = marker[0]; |
| 424 | const suffix = fenceMatch[2]; |
| 425 | |
| 426 | if (!fenceMarker) { |
| 427 | fenceMarker = markerCharacter; |
| 428 | fenceLength = marker.length; |
| 429 | } else if ( |
| 430 | markerCharacter === fenceMarker |
| 431 | && marker.length >= fenceLength |
| 432 | && suffix.trim() === '' |
| 433 | ) { |
| 434 | fenceMarker = null; |
| 435 | fenceLength = 0; |
| 436 | } |
| 437 | |
| 438 | currentLines.push(line); |
| 439 | continue; |
| 440 | } |
| 441 | |
| 442 | const headingMatch = fenceMarker ? null : line.match(/^(#{1,6})\s+(.+)$/); |
| 443 | if (headingMatch) { |
| 444 | flush(); |
| 445 | const depth = headingMatch[1].length; |
| 446 | const heading = headingMatch[2].trim(); |
| 447 | currentHeading = heading; |
| 448 | currentSectionId = uniqueSlug(heading, headingSlugs); |
| 449 | currentDepth = depth; |
| 450 | headings.push(heading); |
| 451 | currentLines.push(line); |
| 452 | continue; |
| 453 | } |
| 454 | |
| 455 | currentLines.push(line); |
| 456 | } |
| 457 | flush(); |
| 458 | |
| 459 | const fallbackText = clean.trim() || [input.title, input.description, ...resolvedKeywords].filter(Boolean).join('\n'); |
| 460 | if (sections.length === 0 && fallbackText) { |
| 461 | sections.push({ text: fallbackText, position: 0, depth: 0 }); |
| 462 | } |
| 463 | |
| 464 | let recordPosition = 0; |
| 465 | return sections.flatMap((section) => { |
| 466 | const chunks = splitIntoChunks(section.text); |
| 467 | return chunks.map((chunk, chunkIndex) => { |
| 468 | const sectionPart = section.section_id ?? 'overview'; |
| 469 | const objectID = `${input.url}__${sectionPart}__${chunkIndex}__${contentHash(chunk)}`; |
| 470 | const sectionRank = Math.max(10, 120 - section.depth * 12 - chunkIndex * 2); |
| 471 | |
| 472 | const position = recordPosition++; |
| 473 | const includeToolkitAliases = input.type === 'toolkits' && position === 0; |
| 474 | |
| 475 | return { |
| 476 | objectID, |
| 477 | title: input.title, |
| 478 | description: input.description, |
| 479 | breadcrumbs: input.breadcrumbs, |
| 480 | url: input.url, |
| 481 | page_id: input.url, |
| 482 | section: section.heading, |
| 483 | section_id: section.section_id, |
| 484 | content: chunk, |
| 485 | keywords: resolvedKeywords, |
| 486 | slug: slugTokens(input.url), |
| 487 | headings, |
| 488 | tool_names: includeToolkitAliases ? input.toolNames : undefined, |
| 489 | tool_slugs: includeToolkitAliases ? input.toolSlugs : undefined, |
| 490 | type: input.type, |
| 491 | lang: 'en', |
| 492 | tags: resolvedTags, |
| 493 | page_rank: resolvedPageRank, |
| 494 | toolkit_popularity: toolkitPopularity( |
| 495 | input.url, |
| 496 | sourceType === 'toolkit' ? 'toolkits' : input.type, |
| 497 | ), |
| 498 | section_rank: sectionRank, |
| 499 | position, |
| 500 | depth: section.depth, |
| 501 | ...metadata, |
| 502 | } satisfies AlgoliaDocsRecord; |
| 503 | }); |
| 504 | }); |
| 505 | } |
| 506 | |
| 507 | const toolkitBySlug = new Map(getAllToolkitsSync().map((toolkit) => [toolkit.slug, toolkit])); |
| 508 | |
| 509 | function limitToolkitAliases(values: string[]): string[] { |
| 510 | const aliases: string[] = []; |
| 511 | let bytes = 0; |
| 512 | |
| 513 | for (const value of values) { |
| 514 | const trimmed = value.trim(); |
| 515 | if (!trimmed) continue; |
| 516 | |
| 517 | const nextBytes = Buffer.byteLength(JSON.stringify(trimmed), 'utf8'); |
| 518 | if (aliases.length >= MAX_TOOL_ALIAS_ITEMS || bytes + nextBytes > MAX_TOOL_ALIAS_BYTES) break; |
| 519 | |
| 520 | aliases.push(trimmed); |
| 521 | bytes += nextBytes; |
| 522 | } |
| 523 | |
| 524 | return aliases; |
| 525 | } |
| 526 | |
| 527 | function getToolkitSearchFields(slug: string): Pick<Parameters<typeof recordsFromMarkdownPage>[0], 'toolNames' | 'toolSlugs' | 'tags'> { |
| 528 | const toolkit = toolkitBySlug.get(slug); |
| 529 | if (!toolkit) return {}; |
| 530 | |
| 531 | return { |
| 532 | toolNames: limitToolkitAliases(toolkit.tools.map((tool) => tool.name).filter(Boolean)), |
| 533 | toolSlugs: limitToolkitAliases(toolkit.tools.map((tool) => tool.slug).filter(Boolean)), |
| 534 | tags: [toolkit.category].filter(Boolean) as string[], |
| 535 | }; |
| 536 | } |
| 537 | |
| 538 | function getFilesystemRecords(): AlgoliaDocsRecord[] { |
| 539 | const contentDir = join(process.cwd(), 'content'); |
| 540 | return listContentFiles(contentDir).flatMap((file) => { |
| 541 | const route = urlFromContentPath(file); |
| 542 | if (!route) return []; |
| 543 | if (isExcludedFromSearch(route.url)) return []; |
| 544 | |
| 545 | const source = resolveToolkitCount(readFileSync(file, 'utf8')); |
| 546 | const frontmatter = getFrontmatter(source); |
| 547 | const title = getFrontmatterValue(frontmatter, 'title'); |
| 548 | if (!title) return []; |
| 549 | |
| 550 | const keywords = getFrontmatterList(frontmatter, 'keywords'); |
| 551 | const topics = getFrontmatterList(frontmatter, 'topics'); |
| 552 | const explicitToolkitSlugs = getFrontmatterList(frontmatter, 'toolkitSlugs'); |
| 553 | const taggedToolkitSlugs = route.type === 'kb' && topics.includes('toolkits') |
| 554 | ? keywords.filter((keyword) => toolkitBySlug.has(keyword)) |
| 555 | : []; |
| 556 | const routeToolkitSlug = route.type === 'toolkits' |
| 557 | ? route.url.replace(/^\/toolkits\//, '').split('/')[0] |
| 558 | : null; |
| 559 | |
| 560 | const toolkitFields = route.type === 'toolkits' |
| 561 | ? getToolkitSearchFields(route.url.replace(/^\/toolkits\//, '')) |
| 562 | : {}; |
| 563 | |
| 564 | const legacy = |
| 565 | getFrontmatterValue(frontmatter, 'legacy') === 'true' || |
| 566 | getFrontmatterValue(frontmatter, 'deprecated') === 'true'; |
| 567 | |
| 568 | return recordsFromMarkdownPage({ |
| 569 | url: route.url, |
| 570 | type: route.type, |
| 571 | title, |
| 572 | description: getFrontmatterValue(frontmatter, 'description'), |
| 573 | keywords, |
| 574 | markdown: source, |
| 575 | breadcrumbs: breadcrumbsForUrl(route.url, route.type), |
| 576 | legacy, |
| 577 | topics, |
| 578 | productAreas: getFrontmatterList(frontmatter, 'productAreas').filter( |
| 579 | (area): area is ProductAreaSlug => [ |
| 580 | 'authentication-and-connected-accounts', |
| 581 | 'tools-actions-and-execution', |
| 582 | 'triggers-and-workflows', |
| 583 | 'sdk-api-and-mcp', |
| 584 | 'account-billing-and-security', |
| 585 | ].includes(area), |
| 586 | ), |
| 587 | toolkitSlugs: [ |
| 588 | ...explicitToolkitSlugs, |
| 589 | ...taggedToolkitSlugs, |
| 590 | ...(routeToolkitSlug ? [routeToolkitSlug] : []), |
| 591 | ], |
| 592 | intents: getFrontmatterList(frontmatter, 'intents') as KnowledgeIntent[], |
| 593 | lastVerifiedAt: getFrontmatterValue(frontmatter, 'lastVerifiedAt'), |
| 594 | ...toolkitFields, |
| 595 | }); |
| 596 | }); |
| 597 | } |
| 598 | |
| 599 | // Dynamic toolkit entries from toolkits.json. We include a compact sample of tool |
| 600 | // names/slugs so queries like "gmail send email" can still find the Gmail toolkit |
| 601 | // page without indexing the full tool catalog as noisy giant records. |
| 602 | const mdxToolkitSlugs = new Set( |
| 603 | toolkitsSource.getPages().map((page) => page.slugs.join('/')), |
| 604 | ); |
| 605 | |
| 606 | const dynamicToolkitIndexes = getAllToolkitsSync() |
| 607 | .filter((toolkit) => !mdxToolkitSlugs.has(toolkit.slug)) |
| 608 | .map((toolkit) => ({ |
| 609 | id: `/toolkits/${toolkit.slug}`, |
| 610 | title: toolkit.name, |
| 611 | description: toolkit.description, |
| 612 | url: `/toolkits/${toolkit.slug}`, |
| 613 | structuredData: { headings: [], contents: [] }, |
| 614 | keywords: [toolkit.slug, toolkit.category].filter(Boolean) as string[], |
| 615 | } satisfies SearchIndex)); |
| 616 | |
| 617 | function getDynamicToolkitRecords(): AlgoliaDocsRecord[] { |
| 618 | return getAllToolkitsSync() |
| 619 | .filter((toolkit) => !mdxToolkitSlugs.has(toolkit.slug)) |
| 620 | .flatMap((toolkit) => { |
| 621 | const toolsText = (toolkit.tools ?? []) |
| 622 | .slice(0, 60) |
| 623 | .map((tool) => `${tool.slug ?? ''} ${tool.name ?? ''}`) |
| 624 | .join('\n'); |
| 625 | |
| 626 | return recordsFromMarkdownPage({ |
| 627 | url: `/toolkits/${toolkit.slug}`, |
| 628 | type: 'toolkits', |
| 629 | title: toolkit.name, |
| 630 | description: toolkit.description, |
| 631 | keywords: [toolkit.slug, toolkit.category].filter(Boolean) as string[], |
| 632 | markdown: `# ${toolkit.name}\n\n${toolkit.description ?? ''}\n\n## Available tools\n\n${toolsText}`, |
| 633 | breadcrumbs: breadcrumbsForUrl(`/toolkits/${toolkit.slug}`, 'toolkits'), |
| 634 | toolkitSlugs: [toolkit.slug], |
| 635 | ...getToolkitSearchFields(toolkit.slug), |
| 636 | }); |
| 637 | }); |
| 638 | } |
| 639 | |
| 640 | function getChangelogIndexes(): SearchIndex[] { |
| 641 | const changelogDir = join(process.cwd(), 'content/changelog'); |
| 642 | if (!existsSync(changelogDir)) return []; |
| 643 | |
| 644 | return readdirSync(changelogDir) |
| 645 | .filter((file) => file.endsWith('.mdx')) |
| 646 | .flatMap((file) => { |
| 647 | const source = readFileSync(join(changelogDir, file), 'utf8'); |
| 648 | const frontmatter = getFrontmatter(source); |
| 649 | const date = getFrontmatterValue(frontmatter, 'date'); |
| 650 | const title = getFrontmatterValue(frontmatter, 'title'); |
| 651 | if (!date || !title) return []; |
| 652 | |
| 653 | const url = `/docs/changelog/${date.replace(/-/g, '/')}`; |
| 654 | |
| 655 | return [{ |
| 656 | id: `${url}#${title}`, |
| 657 | title, |
| 658 | description: getFrontmatterValue(frontmatter, 'description') ?? '', |
| 659 | url, |
| 660 | structuredData: { headings: [], contents: [] }, |
| 661 | keywords: ['changelog'], |
| 662 | } satisfies SearchIndex]; |
| 663 | }); |
| 664 | } |
| 665 | |
| 666 | function getChangelogRecords(): AlgoliaDocsRecord[] { |
| 667 | const changelogDir = join(process.cwd(), 'content/changelog'); |
| 668 | if (!existsSync(changelogDir)) return []; |
| 669 | |
| 670 | return readdirSync(changelogDir) |
| 671 | .filter((file) => file.endsWith('.mdx')) |
| 672 | .flatMap((file) => { |
| 673 | const source = readFileSync(join(changelogDir, file), 'utf8'); |
| 674 | const frontmatter = getFrontmatter(source); |
| 675 | const date = getFrontmatterValue(frontmatter, 'date'); |
| 676 | const title = getFrontmatterValue(frontmatter, 'title'); |
| 677 | if (!date || !title) return []; |
| 678 | |
| 679 | return recordsFromMarkdownPage({ |
| 680 | url: `/docs/changelog/${date.replace(/-/g, '/')}`, |
| 681 | type: 'changelog', |
| 682 | title, |
| 683 | description: getFrontmatterValue(frontmatter, 'description'), |
| 684 | keywords: ['changelog', date], |
| 685 | markdown: source, |
| 686 | breadcrumbs: breadcrumbsForUrl(`/docs/changelog/${date.replace(/-/g, '/')}`, 'changelog'), |
| 687 | }); |
| 688 | }); |
| 689 | } |
| 690 | |
| 691 | export async function getDocsSearchIndexes(): Promise<SearchIndex[]> { |
| 692 | // Load OpenAPI pages and build full reference source. |
| 693 | const openapiPages = await openapiSource(openapi, { |
| 694 | groupBy: 'tag', |
| 695 | baseDir: 'api-reference', |
| 696 | }); |
| 697 | |
| 698 | const fullReferenceSource = loader({ |
| 699 | baseUrl: '/reference', |
| 700 | source: multiple({ |
| 701 | mdx: reference.toFumadocsSource(), |
| 702 | openapi: openapiPages, |
| 703 | }), |
| 704 | plugins: [lucideIconsPlugin(), openapiPlugin()], |
| 705 | }); |
| 706 | |
| 707 | const mdxIndexes = await Promise.all([ |
| 708 | ...docsSource.getPages(), |
| 709 | ...examplesSource.getPages(), |
| 710 | ...toolkitsSource.getPages(), |
| 711 | ...knowledgeBaseSource.getPages(), |
| 712 | ...fullReferenceSource.getPages(), |
| 713 | ].filter((page) => |
| 714 | !isExcludedFromSearch(page.url) && !isHiddenApiTagUrl(page.url) |
| 715 | ).map(async (page) => { |
| 716 | // Keep section-linked prose and add code and custom-component instructions |
| 717 | // omitted by structured MDX search data. |
| 718 | const content = 'getText' in page.data |
| 719 | ? mdxToCleanMarkdown(await page.data.getText('raw'), page.url) |
| 720 | : undefined; |
| 721 | return { |
| 722 | id: page.url, |
| 723 | title: page.data.title ?? 'Untitled', |
| 724 | description: page.data.description, |
| 725 | url: page.url, |
| 726 | structuredData: content === undefined |
| 727 | ? page.data.structuredData |
| 728 | : { |
| 729 | ...page.data.structuredData, |
| 730 | contents: [ |
| 731 | ...(page.data.structuredData?.contents ?? []), |
| 732 | { heading: undefined, content }, |
| 733 | ], |
| 734 | }, |
| 735 | keywords: 'keywords' in page.data ? (page.data.keywords as string[]) : undefined, |
| 736 | } satisfies SearchIndex; |
| 737 | })); |
| 738 | const oauthIndexes = getAuthGuideSearchRecords().map((record) => ({ |
| 739 | id: record.page_id, |
| 740 | title: record.title, |
| 741 | description: record.description, |
| 742 | url: record.canonical_url, |
| 743 | structuredData: { |
| 744 | headings: [], |
| 745 | contents: [{ heading: undefined, content: record.content }], |
| 746 | }, |
| 747 | keywords: record.keywords, |
| 748 | } satisfies SearchIndex)); |
| 749 | |
| 750 | // Migrated auth guides share the toolkit page, so keep one local search hit. |
| 751 | const oauthUrls = new Set(oauthIndexes.map(index => index.url)); |
| 752 | return [ |
| 753 | ...mdxIndexes, |
| 754 | ...dynamicToolkitIndexes.filter(index => !oauthUrls.has(index.url)), |
| 755 | ...getChangelogIndexes(), |
| 756 | ...oauthIndexes, |
| 757 | ]; |
| 758 | } |
| 759 | |
| 760 | async function getOpenApiRecords(): Promise<AlgoliaDocsRecord[]> { |
| 761 | const openapiPages = await openapiSource(openapi, { |
| 762 | groupBy: 'tag', |
| 763 | baseDir: 'api-reference', |
| 764 | }); |
| 765 | |
| 766 | const openapiOnlySource = loader({ |
| 767 | baseUrl: '/reference', |
| 768 | source: openapiPages, |
| 769 | plugins: [lucideIconsPlugin(), openapiPlugin()], |
| 770 | }); |
| 771 | |
| 772 | return openapiOnlySource.getPages().filter((page) => |
| 773 | !isHiddenApiTagUrl(page.url) |
| 774 | ).flatMap((page) => { |
| 775 | const contents = [ |
| 776 | page.data.description, |
| 777 | ...(page.data.structuredData?.headings ?? []).map((heading) => heading.content), |
| 778 | ...(page.data.structuredData?.contents ?? []).map((content) => content.content), |
| 779 | ].filter(Boolean).join('\n\n'); |
| 780 | |
| 781 | return recordsFromMarkdownPage({ |
| 782 | url: page.url, |
| 783 | type: 'api-reference', |
| 784 | title: page.data.title ?? 'Untitled', |
| 785 | description: page.data.description, |
| 786 | markdown: `# ${page.data.title ?? 'Untitled'}\n\n${contents}`, |
| 787 | breadcrumbs: breadcrumbsForUrl(page.url, 'api-reference'), |
| 788 | }); |
| 789 | }); |
| 790 | } |
| 791 | |
| 792 | export async function getAlgoliaSearchDocuments(): Promise<AlgoliaDocsRecord[]> { |
| 793 | const records = [ |
| 794 | ...getFilesystemRecords(), |
| 795 | ...getDynamicToolkitRecords(), |
| 796 | ...getChangelogRecords(), |
| 797 | ...getAuthGuideSearchRecords(), |
| 798 | ...await getOpenApiRecords(), |
| 799 | ]; |
| 800 | |
| 801 | const deduped = new Map<string, AlgoliaDocsRecord>(); |
| 802 | for (const record of records) deduped.set(record.objectID, record); |
| 803 | |
| 804 | return Array.from(deduped.values()); |
| 805 | } |