@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
ts 413 lines 15.7 KB
Raw
1 /**
2 * docs-graph.ts — content link-graph generator + connectivity checker.
3 *
4 * Builds a directed graph of the docs where nodes are pages and edges are
5 * internal `/docs/...` links in page bodies (markdown links + `href=`).
6 *
7 * It then checks the *topology* (not broken links — that's validate-links.ts):
8 * 1. Weakly-connected components. The biggest is "the main graph". Anything
9 * else is an island — a page (or a little 2-/3-loop) that doesn't mainline
10 * back into the main graph through content links.
11 * 2. Reachability from `index` — pages you can't click your way to starting
12 * from the homepage.
13 * 3. Short directed cycles (A->B->A, A->B->C->A) that are *isolated* (their
14 * own tiny component). Cycles embedded in the main graph are fine.
15 *
16 * Usage (from docs/):
17 * bun run scripts/docs-graph.ts # report for content/docs
18 * bun run scripts/docs-graph.ts --html # also write docs-graph.html
19 * bun run scripts/docs-graph.ts --dot # also write docs-graph.dot
20 * bun run scripts/docs-graph.ts --root content/docs --json
21 */
22
23 import { readFile, writeFile } from 'node:fs/promises';
24 import { glob } from 'node:fs/promises';
25 import { relative, resolve } from 'node:path';
26
27 // ---------------------------------------------------------------------------
28 // Config / args
29 // ---------------------------------------------------------------------------
30
31 const args = process.argv.slice(2);
32 const flag = (name: string) => args.includes(`--${name}`);
33 const opt = (name: string, fallback: string) => {
34 const i = args.indexOf(`--${name}`);
35 return i >= 0 && args[i + 1] ? args[i + 1] : fallback;
36 };
37
38 const ROOT = opt('root', 'content/docs'); // dir to scan, relative to cwd
39 const URL_PREFIX = opt('prefix', '/docs'); // route prefix these files serve at
40 const MAX_ISLAND = Number(opt('max-island', '4')); // components <= this are "small"
41
42 // ---------------------------------------------------------------------------
43 // Types
44 // ---------------------------------------------------------------------------
45
46 interface Node {
47 /** route relative to URL_PREFIX, e.g. "providers/openai" ("" for index) */
48 id: string;
49 /** display label */
50 label: string;
51 file: string;
52 out: Set<string>; // internal doc targets (node ids)
53 in: Set<string>;
54 /** links to other sections (/toolkits, /reference, ...) — keeps a page from
55 * looking orphaned when its only links point outside docs */
56 crossOut: string[];
57 /** links to /docs/<x> that resolve to no known page (dangling) */
58 dangling: string[];
59 }
60
61 // ---------------------------------------------------------------------------
62 // Build nodes
63 // ---------------------------------------------------------------------------
64
65 function routeFromFile(file: string): string {
66 // file is relative to ROOT, e.g. "providers/openai.mdx" or "providers/index.mdx"
67 let r = file.replace(/\.mdx$/, '');
68 r = r.replace(/\/index$/, '');
69 if (r === 'index') r = '';
70 return r;
71 }
72
73 const cwd = process.cwd();
74 const rootAbs = resolve(cwd, ROOT);
75
76 const nodes = new Map<string, Node>();
77 const filesByRoute = new Map<string, string>();
78
79 for await (const abs of glob(`${rootAbs}/**/*.mdx`)) {
80 const rel = relative(rootAbs, abs);
81 const id = routeFromFile(rel);
82 nodes.set(id, {
83 id,
84 label: id === '' ? 'index' : id,
85 file: `${ROOT}/${rel}`,
86 out: new Set(),
87 in: new Set(),
88 crossOut: [],
89 dangling: [],
90 });
91 filesByRoute.set(id, abs);
92 }
93
94 // ---------------------------------------------------------------------------
95 // Extract edges
96 // ---------------------------------------------------------------------------
97
98 // Matches markdown links `](/path...)`, href="/path", href={'/path'} etc.
99 const LINK_RE = /(?:\]\(|href\s*=\s*["'{]?)(\/[A-Za-z0-9/_\-.#?=&]+)/g;
100
101 function normalizeDocTarget(raw: string): string | null {
102 // strip anchor + query, trailing slash
103 let t = raw.split('#')[0].split('?')[0].replace(/\/+$/, '');
104 if (!t.startsWith(`${URL_PREFIX}`)) return null;
105 t = t.slice(URL_PREFIX.length).replace(/^\//, ''); // -> route id
106 return t;
107 }
108
109 for (const node of nodes.values()) {
110 const content = await readFile(filesByRoute.get(node.id)!, 'utf8');
111 // strip fenced code blocks so code samples don't create edges
112 const body = content.replace(/```[\s\S]*?```/g, '');
113 for (const m of body.matchAll(LINK_RE)) {
114 const href = m[1];
115 if (href.startsWith(`${URL_PREFIX}/`) || href === URL_PREFIX) {
116 const target = normalizeDocTarget(href);
117 if (target === null) continue;
118 if (target === node.id) continue; // self-anchor link
119 if (nodes.has(target)) {
120 node.out.add(target);
121 nodes.get(target)!.in.add(node.id);
122 } else {
123 node.dangling.push(href);
124 }
125 } else {
126 // cross-section internal link (/toolkits, /reference, /examples, ...)
127 node.crossOut.push(href.split('#')[0]);
128 }
129 }
130 }
131
132 // ---------------------------------------------------------------------------
133 // Graph algorithms
134 // ---------------------------------------------------------------------------
135
136 const ids = [...nodes.keys()];
137
138 // Weakly-connected components (treat edges as undirected)
139 function weaklyConnectedComponents(): string[][] {
140 const seen = new Set<string>();
141 const comps: string[][] = [];
142 for (const start of ids) {
143 if (seen.has(start)) continue;
144 const stack = [start];
145 const comp: string[] = [];
146 seen.add(start);
147 while (stack.length) {
148 const n = stack.pop()!;
149 comp.push(n);
150 const node = nodes.get(n)!;
151 for (const nb of [...node.out, ...node.in]) {
152 if (!seen.has(nb)) {
153 seen.add(nb);
154 stack.push(nb);
155 }
156 }
157 }
158 comps.push(comp);
159 }
160 return comps.sort((a, b) => b.length - a.length);
161 }
162
163 // Reachability from a root over directed out-edges
164 function reachableFrom(root: string): Set<string> {
165 const seen = new Set<string>([root]);
166 const stack = [root];
167 while (stack.length) {
168 const n = stack.pop()!;
169 for (const nb of nodes.get(n)!.out) {
170 if (!seen.has(nb)) {
171 seen.add(nb);
172 stack.push(nb);
173 }
174 }
175 }
176 return seen;
177 }
178
179 // Directed cycles up to length `maxLen` (deduped by canonical rotation)
180 function shortCycles(maxLen: number): string[][] {
181 const found = new Map<string, string[]>();
182 const key = (cyc: string[]) => {
183 const rots = cyc.map((_, i) => [...cyc.slice(i), ...cyc.slice(0, i)].join('>'));
184 return rots.sort()[0];
185 };
186 const dfs = (start: string, path: string[]) => {
187 const last = path[path.length - 1];
188 for (const nb of nodes.get(last)!.out) {
189 if (nb === start && path.length >= 2) {
190 const cyc = [...path];
191 found.set(key(cyc), cyc);
192 } else if (path.length < maxLen && !path.includes(nb) && nb > start) {
193 // nb > start keeps each cycle anchored at its smallest node
194 dfs(start, [...path, nb]);
195 }
196 }
197 };
198 for (const id of ids) dfs(id, [id]);
199 return [...found.values()];
200 }
201
202 const components = weaklyConnectedComponents();
203 const main = new Set(components[0] ?? []);
204 const islands = components.slice(1);
205 const reachFromIndex = nodes.has('') ? reachableFrom('') : new Set<string>();
206 const cycles = shortCycles(3);
207
208 // A short cycle is "concerning" only if its nodes are NOT part of the main
209 // component (i.e. the loop is itself an island) — embedded loops are fine.
210 const isolatedCycles = cycles.filter((c) => c.some((n) => !main.has(n)));
211
212 // Degree + hubs. A "hub" is a high-degree page (lots of things link to/from it).
213 const degree = (id: string) => nodes.get(id)!.in.size + nodes.get(id)!.out.size;
214 const degrees = ids.map(degree).sort((a, b) => a - b);
215 // top ~15% by degree, with a floor so tiny graphs don't flag everything
216 const HUB_DEGREE = Math.max(
217 Number(opt('hub-degree', '0')) || 0,
218 degrees[Math.floor(degrees.length * 0.85)] ?? 0,
219 6,
220 );
221 const isHub = (id: string) => degree(id) >= HUB_DEGREE;
222
223 // Mutual 2-cycles (A->B and B->A) — an immediate circle-back. Benign between
224 // sibling leaf pages, but a smell between two hubs: flow ping-pongs at the top
225 // of the graph instead of moving forward. We flag only the hub<->hub ones.
226 const mutual2: Array<[string, string]> = [];
227 const seenPair = new Set<string>();
228 for (const id of ids) {
229 for (const t of nodes.get(id)!.out) {
230 if (nodes.get(t)!.out.has(id)) {
231 const key = [id, t].sort().join('|');
232 if (!seenPair.has(key)) {
233 seenPair.add(key);
234 mutual2.push([id, t].sort() as [string, string]);
235 }
236 }
237 }
238 }
239 const hubLoops = mutual2
240 .filter(([a, b]) => isHub(a) && isHub(b))
241 .sort((x, y) => degree(y[0]) + degree(y[1]) - degree(x[0]) - degree(x[1]));
242
243 // ---------------------------------------------------------------------------
244 // Report
245 // ---------------------------------------------------------------------------
246
247 const totalEdges = ids.reduce((a, id) => a + nodes.get(id)!.out.size, 0);
248 const orphans = ids.filter(
249 (id) => nodes.get(id)!.in.size === 0 && nodes.get(id)!.out.size === 0,
250 );
251 const noInbound = ids.filter(
252 (id) => id !== '' && nodes.get(id)!.in.size === 0,
253 );
254 const unreachable = ids.filter((id) => !reachFromIndex.has(id));
255
256 const C = {
257 red: (s: string) => `\x1b[31m${s}\x1b[0m`,
258 yellow: (s: string) => `\x1b[33m${s}\x1b[0m`,
259 green: (s: string) => `\x1b[32m${s}\x1b[0m`,
260 dim: (s: string) => `\x1b[2m${s}\x1b[0m`,
261 bold: (s: string) => `\x1b[1m${s}\x1b[0m`,
262 };
263
264 console.log(C.bold(`\nDocs link graph — ${ROOT}`));
265 console.log(
266 `${nodes.size} pages · ${totalEdges} internal links · ${components.length} component(s)\n`,
267 );
268
269 console.log(C.bold('Main graph'));
270 console.log(` ${main.size}/${nodes.size} pages connected into the main component\n`);
271
272 if (islands.length) {
273 console.log(C.red(C.bold(`Islands (${islands.length}) — not connected to the main graph`)));
274 for (const comp of islands) {
275 const kind =
276 comp.length === 1 ? 'orphan page' : `${comp.length}-page loop/cluster`;
277 console.log(` ${C.red('●')} ${kind}`);
278 for (const n of comp) {
279 const node = nodes.get(n)!;
280 const outNote = node.crossOut.length
281 ? C.dim(` (links out to ${[...new Set(node.crossOut)].slice(0, 3).join(', ')})`)
282 : node.out.size || node.in.size
283 ? ''
284 : C.dim(' (no links at all)');
285 console.log(` ${node.label || 'index'}${outNote}`);
286 }
287 }
288 console.log();
289 } else {
290 console.log(C.green('No islands — every page connects into the main graph.\n'));
291 }
292
293 if (hubLoops.length) {
294 console.log(C.red(C.bold(`Hub ping-pong loops (${hubLoops.length})`)));
295 console.log(C.dim(' (two high-traffic pages that only circle back to each other — flow should move forward)'));
296 for (const [a, b] of hubLoops) {
297 console.log(` ${C.red('⇄')} ${a || 'index'} ⇄ ${b || 'index'} ${C.dim(`(deg ${degree(a)} / ${degree(b)})`)}`);
298 }
299 console.log();
300 } else {
301 console.log(C.green('No hub ping-pong loops.\n'));
302 }
303
304 if (isolatedCycles.length) {
305 console.log(C.yellow(C.bold(`Isolated short cycles (${isolatedCycles.length})`)));
306 console.log(C.dim(' (pages that circle straight back without joining the main graph)'));
307 for (const c of isolatedCycles) {
308 console.log(` ${C.yellow('↻')} ${c.map((n) => n || 'index').join(' → ')} → ${c[0] || 'index'}`);
309 }
310 console.log();
311 }
312
313 // Click-reachability from index is only meaningful if index actually links out.
314 // (Docs here navigate by sidebar, so an index with no body links makes every
315 // page "unreachable" — that's noise, and index already shows up as an island.)
316 if (nodes.has('') && nodes.get('')!.out.size === 0) {
317 console.log(C.dim('Click-reachability from index: N/A (index has no outbound content links)\n'));
318 } else if (unreachable.length && nodes.has('')) {
319 console.log(C.yellow(C.bold(`Not reachable from index by clicking (${unreachable.length})`)));
320 console.log(C.dim(' (in the sidebar, but no content link path from the homepage)'));
321 for (const id of unreachable.slice(0, 40)) {
322 console.log(` ${C.yellow('·')} ${id || 'index'}`);
323 }
324 if (unreachable.length > 40) console.log(C.dim(` … +${unreachable.length - 40} more`));
325 console.log();
326 }
327
328 if (noInbound.length) {
329 console.log(C.dim(`Pages with no inbound content links (${noInbound.length}): ${noInbound.slice(0, 12).join(', ')}${noInbound.length > 12 ? ', …' : ''}\n`));
330 }
331
332 // ---------------------------------------------------------------------------
333 // Optional outputs
334 // ---------------------------------------------------------------------------
335
336 if (flag('json')) {
337 const out = {
338 nodes: ids.map((id) => ({
339 id,
340 out: [...nodes.get(id)!.out],
341 in: [...nodes.get(id)!.in],
342 crossOut: nodes.get(id)!.crossOut,
343 dangling: nodes.get(id)!.dangling,
344 })),
345 components,
346 islands,
347 isolatedCycles,
348 unreachable,
349 };
350 await writeFile('docs-graph.json', JSON.stringify(out, null, 2));
351 console.log(C.dim('Wrote docs-graph.json'));
352 }
353
354 if (flag('dot')) {
355 const compIndex = new Map<string, number>();
356 components.forEach((c, i) => c.forEach((n) => compIndex.set(n, i)));
357 const palette = ['#2563eb', '#dc2626', '#d97706', '#7c3aed', '#059669', '#db2777'];
358 const lines = ['digraph docs {', ' rankdir=LR;', ' node [shape=box, style=rounded, fontname="Helvetica"];'];
359 for (const id of ids) {
360 const ci = compIndex.get(id)!;
361 const color = ci === 0 ? '#94a3b8' : palette[(ci - 1) % palette.length];
362 const isIsland = ci !== 0;
363 lines.push(` "${id || 'index'}" [color="${color}"${isIsland ? ', penwidth=2, fontcolor="' + color + '"' : ''}];`);
364 }
365 for (const id of ids) {
366 for (const t of nodes.get(id)!.out) {
367 lines.push(` "${id || 'index'}" -> "${t || 'index'}";`);
368 }
369 }
370 lines.push('}');
371 await writeFile('docs-graph.dot', lines.join('\n'));
372 console.log(C.dim('Wrote docs-graph.dot (render: dot -Tsvg docs-graph.dot -o docs-graph.svg)'));
373 }
374
375 if (flag('html')) {
376 const compIndex = new Map<string, number>();
377 components.forEach((c, i) => c.forEach((n) => compIndex.set(n, i)));
378 const data = {
379 nodes: ids.map((id) => ({
380 id: id || 'index',
381 comp: compIndex.get(id)!,
382 island: compIndex.get(id)! !== 0,
383 degree: nodes.get(id)!.in.size + nodes.get(id)!.out.size,
384 })),
385 edges: ids.flatMap((id) =>
386 [...nodes.get(id)!.out].map((t) => ({ from: id || 'index', to: t || 'index' })),
387 ),
388 };
389 const html = `<!doctype html><meta charset=utf8><title>Docs link graph</title>
390 <style>html,body{margin:0;height:100%;background:#0b0f17;color:#e5e7eb;font-family:system-ui}
391 #net{height:100vh}#legend{position:fixed;top:12px;left:12px;background:#111827cc;padding:10px 14px;border-radius:8px;font-size:13px;line-height:1.6}</style>
392 <div id=legend><b>Docs link graph</b><br><span style=color:#94a3b8>●</span> main graph &nbsp; <span style=color:#dc2626>●</span> island (disconnected)<br>node size = link count</div>
393 <div id=net></div>
394 <script src="https://unpkg.com/vis-network/standalone/umd/vis-network.min.js"></script>
395 <script>
396 const D=${JSON.stringify(data)};
397 const palette=['#94a3b8','#dc2626','#d97706','#7c3aed','#059669','#db2777','#2563eb'];
398 const nodes=new vis.DataSet(D.nodes.map(n=>({id:n.id,label:n.id,value:n.degree+1,
399 color:{background:n.island?palette[1+(n.comp-1)%6]:'#334155',border:n.island?palette[1+(n.comp-1)%6]:'#64748b'},
400 font:{color:n.island?'#fca5a5':'#cbd5e1'}})));
401 const edges=new vis.DataSet(D.edges.map(e=>({from:e.from,to:e.to,arrows:'to',color:{color:'#33415588'}})));
402 new vis.Network(document.getElementById('net'),{nodes,edges},{
403 physics:{stabilization:true,barnesHut:{gravitationalConstant:-8000,springLength:120}},
404 nodes:{shape:'dot',scaling:{min:6,max:40}},interaction:{hover:true}});
405 </script>`;
406 await writeFile('docs-graph.html', html);
407 console.log(C.dim('Wrote docs-graph.html (open in a browser)'));
408 }
409
410 // exit non-zero if there are islands, so this can gate CI
411 if (flag('check') && (islands.length || isolatedCycles.length || hubLoops.length)) {
412 process.exit(1);
413 }