@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
mjs 131 lines 5.03 KB
Raw
1 #!/usr/bin/env node
2 import { readFile, writeFile } from 'node:fs/promises';
3 import { parseArgs } from 'node:util';
4
5 /**
6 * @typedef {{id: string, query: string, source: string, expect: string[][]}} Query
7 * @typedef {{source?: string, text: string, generationDate?: string}} Snippet
8 * @typedef {{status: 'PASS'|'FAIL'|'ERROR', missing?: string[][], snippets?: Snippet[], error?: string}} Result
9 * @typedef {{codeId?: string, codeTitle?: string, codeDescription?: string, generationDate?: string, codeList: {code: string}[]}} CodeSnippet
10 * @typedef {{codeSnippets: CodeSnippet[], infoSnippets: {pageId?: string, content: string}[], rules?: object}} ContextResponse
11 */
12
13 const endpoint = 'https://context7.com/api/v2/context';
14 /** @param {string} text */
15 const normalize = text => text.toLowerCase().replace(/\s+/g, ' ').trim();
16
17 // Only retrieved content counts as evidence, never titles, URLs, or injected rules.
18 /**
19 * @param {Query} query
20 * @param {ContextResponse} payload
21 * @returns {Result}
22 */
23 export function evaluate(query, payload) {
24 if (!Array.isArray(payload.codeSnippets) || !Array.isArray(payload.infoSnippets)) {
25 throw new Error('Invalid Context7 response: expected codeSnippets and infoSnippets arrays');
26 }
27 const snippets = [
28 ...payload.codeSnippets.flatMap(snippet => {
29 if (!Array.isArray(snippet.codeList)) throw new Error('Invalid Context7 codeList');
30 return snippet.codeList.map(example => ({
31 source: snippet.codeId,
32 text: example.code,
33 generationDate: snippet.generationDate,
34 }));
35 }),
36 ...payload.infoSnippets.map(snippet => ({ source: snippet.pageId, text: snippet.content })),
37 ];
38 if (snippets.some(snippet => typeof snippet.text !== 'string')) {
39 throw new Error('Invalid Context7 snippet content');
40 }
41 const content = snippets.map(snippet => normalize(snippet.text));
42 const missing = query.expect.filter(
43 alternatives => !alternatives.some(term => content.some(text => text.includes(normalize(term))))
44 );
45 return { status: missing.length ? 'FAIL' : 'PASS', missing, snippets };
46 }
47
48 /**
49 * @param {{libraryId: string, queries: Query[]}} suite
50 * @param {{apiKey?: string, fetchImpl?: (url: URL, init: RequestInit) => Promise<Response>, log?: (line: string) => void}} options
51 */
52 export async function runSuite(suite, { apiKey, fetchImpl = fetch, log = console.log } = {}) {
53 const results = [];
54 for (const query of suite.queries) {
55 const url = new URL(endpoint);
56 url.search = new URLSearchParams({
57 libraryId: suite.libraryId,
58 query: query.query,
59 type: 'json',
60 }).toString();
61 /** @type {Result} */
62 let result;
63 try {
64 const response = await fetchImpl(url, {
65 headers: apiKey ? { Authorization: `Bearer ${apiKey}` } : {},
66 signal: AbortSignal.timeout(60_000),
67 redirect: 'error',
68 });
69 // 202 means indexing is pending, not a successful retrieval.
70 if (response.status !== 200) {
71 throw new Error(`Context7 HTTP ${response.status}; no retrieval evidence`);
72 }
73 result = evaluate(query, await response.json());
74 } catch (error) {
75 // Don't print server bodies, request headers, or credentials.
76 result = {
77 status: 'ERROR',
78 error:
79 error instanceof Error
80 ? error.message.replaceAll(apiKey || '\0', '[REDACTED]')
81 : 'Retrieval failed',
82 };
83 }
84 results.push({ ...query, ...result });
85 log(
86 `${result.status} ${query.id}${result.missing?.length ? ` missing ${JSON.stringify(result.missing)}` : ''}${result.error ? `: ${result.error}` : ''}`
87 );
88 }
89 const totals = { PASS: 0, FAIL: 0, ERROR: 0 };
90 for (const result of results) totals[result.status]++;
91 log(
92 `${results.length} queries: ${totals.PASS} passed, ${totals.FAIL} failed, ${totals.ERROR} errors`
93 );
94 return { libraryId: suite.libraryId, endpoint, totals, results };
95 }
96
97 if (import.meta.main) {
98 try {
99 const { values } = parseArgs({ options: { output: { type: 'string' } } });
100 const suiteText = await readFile(new URL('./context7-queries.json', import.meta.url), 'utf8');
101 const suite = JSON.parse(suiteText);
102 const suiteSha256 = Array.from(
103 new Uint8Array(await crypto.subtle.digest('SHA-256', new TextEncoder().encode(suiteText))),
104 byte => byte.toString(16).padStart(2, '0')
105 ).join('');
106 const startedAt = new Date().toISOString();
107 const report = await runSuite(suite, { apiKey: process.env.CONTEXT7_API_KEY });
108 if (values.output) {
109 await writeFile(
110 values.output,
111 `${JSON.stringify(
112 {
113 startedAt,
114 completedAt: new Date().toISOString(),
115 suiteSha256,
116 config: JSON.parse(
117 await readFile(new URL('../context7.json', import.meta.url), 'utf8')
118 ),
119 ...report,
120 },
121 null,
122 2
123 )}\n`
124 );
125 }
126 process.exitCode = report.totals.FAIL || report.totals.ERROR ? 1 : 0;
127 } catch (error) {
128 console.error(error instanceof Error ? error.message : 'Context7 check failed');
129 process.exitCode = 1;
130 }
131 }