Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 03:37 UTC
| 1 | #!/usr/bin/env node |
| 2 | import { readFile, writeFile } from 'node:fs/promises'; |
| 3 | import { parseArgs } from 'node:util'; |
| 4 | |
| 5 | /** |
| 6 | * @typedef {{id: string, query: string, source: string, expect: string[][]}} Query |
| 7 | * @typedef {{source?: string, text: string, generationDate?: string}} Snippet |
| 8 | * @typedef {{status: 'PASS'|'FAIL'|'ERROR', missing?: string[][], snippets?: Snippet[], error?: string}} Result |
| 9 | * @typedef {{codeId?: string, codeTitle?: string, codeDescription?: string, generationDate?: string, codeList: {code: string}[]}} CodeSnippet |
| 10 | * @typedef {{codeSnippets: CodeSnippet[], infoSnippets: {pageId?: string, content: string}[], rules?: object}} ContextResponse |
| 11 | */ |
| 12 | |
| 13 | const endpoint = 'https://context7.com/api/v2/context'; |
| 14 | /** @param {string} text */ |
| 15 | const normalize = text => text.toLowerCase().replace(/\s+/g, ' ').trim(); |
| 16 | |
| 17 | // Only retrieved content counts as evidence, never titles, URLs, or injected rules. |
| 18 | /** |
| 19 | * @param {Query} query |
| 20 | * @param {ContextResponse} payload |
| 21 | * @returns {Result} |
| 22 | */ |
| 23 | export function evaluate(query, payload) { |
| 24 | if (!Array.isArray(payload.codeSnippets) || !Array.isArray(payload.infoSnippets)) { |
| 25 | throw new Error('Invalid Context7 response: expected codeSnippets and infoSnippets arrays'); |
| 26 | } |
| 27 | const snippets = [ |
| 28 | ...payload.codeSnippets.flatMap(snippet => { |
| 29 | if (!Array.isArray(snippet.codeList)) throw new Error('Invalid Context7 codeList'); |
| 30 | return snippet.codeList.map(example => ({ |
| 31 | source: snippet.codeId, |
| 32 | text: example.code, |
| 33 | generationDate: snippet.generationDate, |
| 34 | })); |
| 35 | }), |
| 36 | ...payload.infoSnippets.map(snippet => ({ source: snippet.pageId, text: snippet.content })), |
| 37 | ]; |
| 38 | if (snippets.some(snippet => typeof snippet.text !== 'string')) { |
| 39 | throw new Error('Invalid Context7 snippet content'); |
| 40 | } |
| 41 | const content = snippets.map(snippet => normalize(snippet.text)); |
| 42 | const missing = query.expect.filter( |
| 43 | alternatives => !alternatives.some(term => content.some(text => text.includes(normalize(term)))) |
| 44 | ); |
| 45 | return { status: missing.length ? 'FAIL' : 'PASS', missing, snippets }; |
| 46 | } |
| 47 | |
| 48 | /** |
| 49 | * @param {{libraryId: string, queries: Query[]}} suite |
| 50 | * @param {{apiKey?: string, fetchImpl?: (url: URL, init: RequestInit) => Promise<Response>, log?: (line: string) => void}} options |
| 51 | */ |
| 52 | export async function runSuite(suite, { apiKey, fetchImpl = fetch, log = console.log } = {}) { |
| 53 | const results = []; |
| 54 | for (const query of suite.queries) { |
| 55 | const url = new URL(endpoint); |
| 56 | url.search = new URLSearchParams({ |
| 57 | libraryId: suite.libraryId, |
| 58 | query: query.query, |
| 59 | type: 'json', |
| 60 | }).toString(); |
| 61 | /** @type {Result} */ |
| 62 | let result; |
| 63 | try { |
| 64 | const response = await fetchImpl(url, { |
| 65 | headers: apiKey ? { Authorization: `Bearer ${apiKey}` } : {}, |
| 66 | signal: AbortSignal.timeout(60_000), |
| 67 | redirect: 'error', |
| 68 | }); |
| 69 | // 202 means indexing is pending, not a successful retrieval. |
| 70 | if (response.status !== 200) { |
| 71 | throw new Error(`Context7 HTTP ${response.status}; no retrieval evidence`); |
| 72 | } |
| 73 | result = evaluate(query, await response.json()); |
| 74 | } catch (error) { |
| 75 | // Don't print server bodies, request headers, or credentials. |
| 76 | result = { |
| 77 | status: 'ERROR', |
| 78 | error: |
| 79 | error instanceof Error |
| 80 | ? error.message.replaceAll(apiKey || '\0', '[REDACTED]') |
| 81 | : 'Retrieval failed', |
| 82 | }; |
| 83 | } |
| 84 | results.push({ ...query, ...result }); |
| 85 | log( |
| 86 | `${result.status} ${query.id}${result.missing?.length ? ` missing ${JSON.stringify(result.missing)}` : ''}${result.error ? `: ${result.error}` : ''}` |
| 87 | ); |
| 88 | } |
| 89 | const totals = { PASS: 0, FAIL: 0, ERROR: 0 }; |
| 90 | for (const result of results) totals[result.status]++; |
| 91 | log( |
| 92 | `${results.length} queries: ${totals.PASS} passed, ${totals.FAIL} failed, ${totals.ERROR} errors` |
| 93 | ); |
| 94 | return { libraryId: suite.libraryId, endpoint, totals, results }; |
| 95 | } |
| 96 | |
| 97 | if (import.meta.main) { |
| 98 | try { |
| 99 | const { values } = parseArgs({ options: { output: { type: 'string' } } }); |
| 100 | const suiteText = await readFile(new URL('./context7-queries.json', import.meta.url), 'utf8'); |
| 101 | const suite = JSON.parse(suiteText); |
| 102 | const suiteSha256 = Array.from( |
| 103 | new Uint8Array(await crypto.subtle.digest('SHA-256', new TextEncoder().encode(suiteText))), |
| 104 | byte => byte.toString(16).padStart(2, '0') |
| 105 | ).join(''); |
| 106 | const startedAt = new Date().toISOString(); |
| 107 | const report = await runSuite(suite, { apiKey: process.env.CONTEXT7_API_KEY }); |
| 108 | if (values.output) { |
| 109 | await writeFile( |
| 110 | values.output, |
| 111 | `${JSON.stringify( |
| 112 | { |
| 113 | startedAt, |
| 114 | completedAt: new Date().toISOString(), |
| 115 | suiteSha256, |
| 116 | config: JSON.parse( |
| 117 | await readFile(new URL('../context7.json', import.meta.url), 'utf8') |
| 118 | ), |
| 119 | ...report, |
| 120 | }, |
| 121 | null, |
| 122 | 2 |
| 123 | )}\n` |
| 124 | ); |
| 125 | } |
| 126 | process.exitCode = report.totals.FAIL || report.totals.ERROR ? 1 : 0; |
| 127 | } catch (error) { |
| 128 | console.error(error instanceof Error ? error.message : 'Context7 check failed'); |
| 129 | process.exitCode = 1; |
| 130 | } |
| 131 | } |