Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 22, 09:30 UTC
perf: cache parsed toolkits.json in-process with O(1) slug lookup
The 6.9 MB toolkits.json was re-read and re-parsed on every detail page request. Centralise all reading into lib/toolkit-data.ts with a module-level Map cache so the file is parsed once per process lifetime. Also drops category from toolkits-list.json (unused by landing page). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
Sushmithamallesh committed
Feb 11, 2026 at 21:54 UTC
0bf5bee16580956c9b7d5d3c747abe5e17a5f0f8
4 files changed
+57
-91
docs/app/(home)/toolkits/[[...slug]]/page.tsx
+5
-34
@@ -6,12 +6,13 @@ import { ToolkitsLanding } from '@/components/toolkits/toolkits-landing';
6
import { PageActions } from '@/components/page-actions';
7
import { readFile } from 'fs/promises';
8
import { join } from 'path';
9
+import { getAllToolkits, getToolkitBySlug } from '@/lib/toolkit-data';
10
import { unified } from 'unified';
11
import remarkParse from 'remark-parse';
12
import remarkRehype from 'remark-rehype';
13
import { toHtml } from 'hast-util-to-html';
14
import type { Metadata } from 'next';
14
-import type { Toolkit, Tool, Trigger } from '@/types/toolkit';
15
+import type { Tool, Trigger } from '@/types/toolkit';
16
import type { FaqItem } from '@/components/toolkits/faq-section';
17
import { processSchema, toolFromApi } from '@/lib/toolkit-schema';
18
@@ -129,34 +130,6 @@ async function readToolkitFaq(slug: string): Promise<FaqItem[] | null> {
130
return items.length > 0 ? items : null;
131
}
132
132
-async function getToolkits(): Promise<Toolkit[]> {
133
- const filePath = join(process.cwd(), 'public/data/toolkits.json');
134
-
135
- try {
136
- const data = await readFile(filePath, 'utf-8');
137
- const toolkits = JSON.parse(data) as Toolkit[];
138
-
139
- if (!Array.isArray(toolkits)) {
140
- throw new Error('toolkits.json must contain an array');
141
- }
142
-
143
- if (toolkits.length === 0) {
144
- console.warn('[Toolkits] Warning: toolkits.json is empty');
145
- }
146
-
147
- return toolkits;
148
- } catch (error) {
149
- const err = error as NodeJS.ErrnoException;
150
- if (err.code === 'ENOENT') {
151
- throw new Error(`Toolkits data file not found: ${filePath}`);
152
- }
153
- if (error instanceof SyntaxError) {
154
- throw new Error(`Invalid JSON in toolkits.json: ${error.message}`);
155
- }
156
- throw error;
157
- }
158
-}
159
-
133
export async function generateStaticParams() {
134
// Index page
135
const indexParam = { slug: [] };
@@ -165,7 +138,7 @@ export async function generateStaticParams() {
138
const mdxParams = toolkitsSource.generateParams();
139
140
// JSON toolkit pages
168
- const toolkits = await getToolkits();
141
+ const toolkits = await getAllToolkits();
142
const jsonParams = toolkits.map((toolkit) => ({
143
slug: [toolkit.slug],
144
}));
@@ -203,8 +176,7 @@ export async function generateMetadata({ params }: { params: Promise<{ slug?: st
176
177
// Check JSON toolkit
178
if (slug.length === 1) {
206
- const toolkits = await getToolkits();
207
- const toolkit = toolkits.find((t) => t.slug === slug[0]);
179
+ const toolkit = await getToolkitBySlug(slug[0]);
180
if (toolkit) {
181
const title = `${toolkit.name?.trim() || toolkit.slug} - Composio Toolkit`;
182
const description = `Build an AI agent that connects to ${toolkit.name?.trim() || toolkit.slug} using Composio. ${toolkit.description}`;
@@ -245,8 +217,7 @@ export default async function ToolkitsPage({ params }: { params: Promise<{ slug?
217
// Check JSON toolkit
218
if (slug.length === 1) {
219
const toolkitSlug = slug[0];
248
- const toolkits = await getToolkits();
249
- const toolkit = toolkits.find((t) => t.slug === toolkitSlug);
220
+ const toolkit = await getToolkitBySlug(toolkitSlug);
221
222
if (toolkit) {
223
// Fetch detailed tool/trigger info and FAQ content in parallel
docs/app/llms.mdx/[[...slug]]/route.ts
+2
-22
@@ -13,6 +13,7 @@ import { openapi } from '@/lib/openapi';
13
import { notFound } from 'next/navigation';
14
import { readFile } from 'fs/promises';
15
import { join } from 'path';
16
+import { getAllToolkits, getToolkitBySlug } from '@/lib/toolkit-data';
17
import type { Toolkit, Tool, Trigger, ParameterSchema } from '@/types/toolkit';
18
import { processSchema, toolFromApi } from '@/lib/toolkit-schema';
19
@@ -696,27 +697,6 @@ function toolkitToMarkdown(toolkit: Toolkit, detailedTools?: Tool[], detailedTri
697
return lines.join('\n');
698
}
699
699
-async function getToolkit(slug: string): Promise<Toolkit | null> {
700
- try {
701
- const filePath = join(process.cwd(), 'public/data/toolkits.json');
702
- const data = await readFile(filePath, 'utf-8');
703
- const toolkits = JSON.parse(data) as Toolkit[];
704
- return toolkits.find((t) => t.slug === slug) || null;
705
- } catch {
706
- return null;
707
- }
708
-}
709
-
710
-async function getAllToolkits(): Promise<Toolkit[]> {
711
- try {
712
- const filePath = join(process.cwd(), 'public/data/toolkits.json');
713
- const data = await readFile(filePath, 'utf-8');
714
- return JSON.parse(data) as Toolkit[];
715
- } catch {
716
- return [];
717
- }
718
-}
719
-
700
// Generate a comprehensive toolkits index for /toolkits.md
701
async function generateToolkitsIndex(): Promise<string> {
702
const toolkits = await getAllToolkits();
@@ -878,7 +858,7 @@ export async function GET(
858
859
// Special handling for JSON toolkit pages
860
if (prefix === 'toolkits' && rest.length === 1) {
881
- const toolkit = await getToolkit(rest[0]);
861
+ const toolkit = await getToolkitBySlug(rest[0]);
862
if (toolkit) {
863
// Fetch detailed tool/trigger info and FAQ content in parallel
864
const [detailedTools, detailedTriggers, faqMarkdown] = await Promise.all([
docs/app/sitemap.ts
+2
-35
@@ -1,6 +1,4 @@
1
import type { MetadataRoute } from 'next';
2
-import { readFileSync } from 'fs';
3
-import { join } from 'path';
2
import {
3
source,
4
getReferenceSource,
@@ -9,41 +7,10 @@ import {
7
changelogEntries,
8
dateToChangelogUrl,
9
} from '@/lib/source';
10
+import { getAllToolkitsSync } from '@/lib/toolkit-data';
11
12
const baseUrl = 'https://docs.composio.dev';
13
15
-interface Toolkit {
16
- slug: string;
17
-}
18
-
19
-function getToolkitsFromJson(): Toolkit[] {
20
- const filePath = join(process.cwd(), 'public/data/toolkits.json');
21
-
22
- try {
23
- const data = readFileSync(filePath, 'utf-8');
24
- const toolkits = JSON.parse(data) as Toolkit[];
25
-
26
- if (!Array.isArray(toolkits)) {
27
- throw new Error('toolkits.json must contain an array');
28
- }
29
-
30
- if (toolkits.length === 0) {
31
- console.warn('[Sitemap] Warning: toolkits.json is empty - toolkit pages will be missing from sitemap');
32
- }
33
-
34
- return toolkits;
35
- } catch (error) {
36
- const err = error as NodeJS.ErrnoException;
37
- if (err.code === 'ENOENT') {
38
- throw new Error(`[Sitemap] Toolkits data file not found: ${filePath}`);
39
- }
40
- if (error instanceof SyntaxError) {
41
- throw new Error(`[Sitemap] Invalid JSON in toolkits.json: ${error.message}`);
42
- }
43
- throw error;
44
- }
45
-}
46
-
14
export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
15
const docsPages = source.getPages().map((page) => ({
16
url: `${baseUrl}${page.url}`,
@@ -65,7 +32,7 @@ export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
32
}));
33
34
// JSON toolkit pages (dynamically generated from toolkits.json)
68
- const toolkitsJsonPages = getToolkitsFromJson().map((toolkit) => ({
35
+ const toolkitsJsonPages = getAllToolkitsSync().map((toolkit) => ({
36
url: `${baseUrl}/toolkits/${toolkit.slug}`,
37
}));
38
docs/lib/toolkit-data.ts
new
+48
@@ -0,0 +1,48 @@
1
+import { readFile } from 'fs/promises';
2
+import { readFileSync } from 'fs';
3
+import { join } from 'path';
4
+import type { Toolkit } from '@/types/toolkit';
5
+
6
+const TOOLKITS_PATH = join(process.cwd(), 'public/data/toolkits.json');
7
+
8
+let cached: { list: Toolkit[]; bySlug: Map<string, Toolkit> } | null = null;
9
+
10
+function buildCache(toolkits: Toolkit[]) {
11
+ cached = {
12
+ list: toolkits,
13
+ bySlug: new Map(toolkits.map(t => [t.slug, t])),
14
+ };
15
+ return cached;
16
+}
17
+
18
+function parse(data: string): Toolkit[] {
19
+ const toolkits = JSON.parse(data) as Toolkit[];
20
+
21
+ if (!Array.isArray(toolkits)) {
22
+ throw new Error('toolkits.json must contain an array');
23
+ }
24
+
25
+ if (toolkits.length === 0) {
26
+ console.warn('[Toolkits] Warning: toolkits.json is empty');
27
+ }
28
+
29
+ return toolkits;
30
+}
31
+
32
+export async function getAllToolkits(): Promise<Toolkit[]> {
33
+ if (cached) return cached.list;
34
+ const data = await readFile(TOOLKITS_PATH, 'utf-8');
35
+ return buildCache(parse(data)).list;
36
+}
37
+
38
+export async function getToolkitBySlug(slug: string): Promise<Toolkit | null> {
39
+ if (cached) return cached.bySlug.get(slug) ?? null;
40
+ const data = await readFile(TOOLKITS_PATH, 'utf-8');
41
+ return buildCache(parse(data)).bySlug.get(slug) ?? null;
42
+}
43
+
44
+export function getAllToolkitsSync(): Toolkit[] {
45
+ if (cached) return cached.list;
46
+ const data = readFileSync(TOOLKITS_PATH, 'utf-8');
47
+ return buildCache(parse(data)).list;
48
+}