@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
mjs 118 lines 4.02 KB
Raw
1 import assert from 'node:assert/strict';
2 import { readFile } from 'node:fs/promises';
3 import { test } from 'node:test';
4 import { evaluate, runSuite } from '../scripts/context7-check.mjs';
5
6 const query = {
7 id: 'current-api',
8 query: 'Which REST API should new code use?',
9 source: 'https://example.test/current-api',
10 expect: [['https://backend.composio.dev/api/v3.1'], ['x-api-key', 'project API key']],
11 };
12 /** @param {string} content */
13 const payload = content => ({ codeSnippets: [], infoSnippets: [{ content }] });
14
15 test('stale API retrieval fails even when another required fact is present', () => {
16 const result = evaluate(
17 query,
18 payload('Use https://backend.composio.dev/api/v3 with x-api-key.')
19 );
20 assert.equal(result.status, 'FAIL');
21 assert.deepEqual(result.missing, [['https://backend.composio.dev/api/v3.1']]);
22 });
23
24 test('alternatives ignore case and whitespace, without combining unrelated snippets into one fact', () => {
25 const result = evaluate(query, {
26 codeSnippets: [
27 {
28 codeId: 'https://example.test/code',
29 codeList: [{ code: 'https://backend.composio.dev/api/v3.1' }],
30 },
31 ],
32 infoSnippets: [{ content: 'Authenticate with a PROJECT\n API KEY.' }],
33 });
34 assert.equal(result.status, 'PASS');
35 assert.equal(
36 evaluate(
37 { ...query, expect: [['project API key']] },
38 {
39 codeSnippets: [],
40 infoSnippets: [{ content: 'project API' }, { content: 'key' }],
41 }
42 ).status,
43 'FAIL'
44 );
45 });
46
47 test('URLs, titles, descriptions, and injected rules cannot mask missing documentation', () => {
48 const result = evaluate(query, {
49 codeSnippets: [
50 {
51 codeId: query.expect[0][0],
52 codeTitle: 'x-api-key',
53 codeDescription: 'x-api-key',
54 codeList: [{ code: 'obsolete example' }],
55 },
56 ],
57 infoSnippets: [{ pageId: query.expect[0][0], content: 'unrelated content' }],
58 rules: { libraryOwn: ['Use https://backend.composio.dev/api/v3.1 with x-api-key'] },
59 });
60 assert.equal(result.status, 'FAIL');
61 assert.deepEqual(result.missing, query.expect);
62 assert.equal(evaluate(query, payload('')).status, 'FAIL');
63 });
64
65 test('malformed and pending retrievals are errors, not regression passes; later queries still run', async () => {
66 const responses = [
67 new Response('{}', { status: 202 }),
68 Response.json({ codeSnippets: [], infoSnippets: [{ content: null }] }),
69 new Response('{}', { status: 429 }),
70 Response.json(payload('Use https://backend.composio.dev/api/v3.1 with x-api-key')),
71 ];
72 const result = await runSuite(
73 {
74 libraryId: '/synthetic/library',
75 queries: responses.map((_, i) => ({ ...query, id: `case-${i}` })),
76 },
77 {
78 fetchImpl: async () => {
79 const response = responses.shift();
80 assert.ok(response);
81 return response;
82 },
83 log: () => {},
84 }
85 );
86 assert.deepEqual(
87 result.results.map(item => item.status),
88 ['ERROR', 'ERROR', 'ERROR', 'PASS']
89 );
90 assert.deepEqual(result.totals, { PASS: 1, FAIL: 0, ERROR: 3 });
91 });
92
93 test('network failures are distinct from missing facts and redact credentials', async () => {
94 const result = await runSuite(
95 { libraryId: '/synthetic/library', queries: [query] },
96 {
97 apiKey: 'synthetic-secret',
98 fetchImpl: async () => {
99 throw new Error('network unavailable synthetic-secret');
100 },
101 log: () => {},
102 }
103 );
104 assert.deepEqual(result.totals, { PASS: 0, FAIL: 0, ERROR: 1 });
105 assert.equal(result.results[0].error, 'network unavailable [REDACTED]');
106 });
107
108 test('owner rules fit Context7 public schema limits instead of being rejected for length', async () => {
109 // https://context7.com/schema/context7.json: rules.maxItems and rules.items.maxLength.
110 const config = JSON.parse(await readFile(new URL('../context7.json', import.meta.url), 'utf8'));
111 assert.ok(config.rules.length <= 50, 'Context7 accepts at most 50 rules');
112 for (const rule of config.rules) {
113 assert.ok(
114 rule.length >= 1 && rule.length <= 255,
115 `Invalid Context7 rule length: ${rule.length}`
116 );
117 }
118 });