@devries-cameron20 / composio-1 / commits / 0bf5bee16
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

perf: cache parsed toolkits.json in-process with O(1) slug lookup

The 6.9 MB toolkits.json was re-read and re-parsed on every detail page request. Centralise all reading into lib/toolkit-data.ts with a module-level Map cache so the file is parsed once per process lifetime. Also drops category from toolkits-list.json (unused by landing page). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Sushmithamallesh committed Feb 11, 2026 at 21:54 UTC 0bf5bee16580956c9b7d5d3c747abe5e17a5f0f8
4 files changed +57 -91
docs/app/(home)/toolkits/[[...slug]]/page.tsx
+5 -34
@@ -6,12 +6,13 @@ import { ToolkitsLanding } from '@/components/toolkits/toolkits-landing';
6 import { PageActions } from '@/components/page-actions';
7 import { readFile } from 'fs/promises';
8 import { join } from 'path';
9 +import { getAllToolkits, getToolkitBySlug } from '@/lib/toolkit-data';
10 import { unified } from 'unified';
11 import remarkParse from 'remark-parse';
12 import remarkRehype from 'remark-rehype';
13 import { toHtml } from 'hast-util-to-html';
14 import type { Metadata } from 'next';
14 -import type { Toolkit, Tool, Trigger } from '@/types/toolkit';
15 +import type { Tool, Trigger } from '@/types/toolkit';
16 import type { FaqItem } from '@/components/toolkits/faq-section';
17 import { processSchema, toolFromApi } from '@/lib/toolkit-schema';
18
@@ -129,34 +130,6 @@ async function readToolkitFaq(slug: string): Promise<FaqItem[] | null> {
130 return items.length > 0 ? items : null;
131 }
132
132 -async function getToolkits(): Promise<Toolkit[]> {
133 - const filePath = join(process.cwd(), 'public/data/toolkits.json');
134 -
135 - try {
136 - const data = await readFile(filePath, 'utf-8');
137 - const toolkits = JSON.parse(data) as Toolkit[];
138 -
139 - if (!Array.isArray(toolkits)) {
140 - throw new Error('toolkits.json must contain an array');
141 - }
142 -
143 - if (toolkits.length === 0) {
144 - console.warn('[Toolkits] Warning: toolkits.json is empty');
145 - }
146 -
147 - return toolkits;
148 - } catch (error) {
149 - const err = error as NodeJS.ErrnoException;
150 - if (err.code === 'ENOENT') {
151 - throw new Error(`Toolkits data file not found: ${filePath}`);
152 - }
153 - if (error instanceof SyntaxError) {
154 - throw new Error(`Invalid JSON in toolkits.json: ${error.message}`);
155 - }
156 - throw error;
157 - }
158 -}
159 -
133 export async function generateStaticParams() {
134 // Index page
135 const indexParam = { slug: [] };
@@ -165,7 +138,7 @@ export async function generateStaticParams() {
138 const mdxParams = toolkitsSource.generateParams();
139
140 // JSON toolkit pages
168 - const toolkits = await getToolkits();
141 + const toolkits = await getAllToolkits();
142 const jsonParams = toolkits.map((toolkit) => ({
143 slug: [toolkit.slug],
144 }));
@@ -203,8 +176,7 @@ export async function generateMetadata({ params }: { params: Promise<{ slug?: st
176
177 // Check JSON toolkit
178 if (slug.length === 1) {
206 - const toolkits = await getToolkits();
207 - const toolkit = toolkits.find((t) => t.slug === slug[0]);
179 + const toolkit = await getToolkitBySlug(slug[0]);
180 if (toolkit) {
181 const title = `${toolkit.name?.trim() || toolkit.slug} - Composio Toolkit`;
182 const description = `Build an AI agent that connects to ${toolkit.name?.trim() || toolkit.slug} using Composio. ${toolkit.description}`;
@@ -245,8 +217,7 @@ export default async function ToolkitsPage({ params }: { params: Promise<{ slug?
217 // Check JSON toolkit
218 if (slug.length === 1) {
219 const toolkitSlug = slug[0];
248 - const toolkits = await getToolkits();
249 - const toolkit = toolkits.find((t) => t.slug === toolkitSlug);
220 + const toolkit = await getToolkitBySlug(toolkitSlug);
221
222 if (toolkit) {
223 // Fetch detailed tool/trigger info and FAQ content in parallel
docs/app/llms.mdx/[[...slug]]/route.ts
+2 -22
@@ -13,6 +13,7 @@ import { openapi } from '@/lib/openapi';
13 import { notFound } from 'next/navigation';
14 import { readFile } from 'fs/promises';
15 import { join } from 'path';
16 +import { getAllToolkits, getToolkitBySlug } from '@/lib/toolkit-data';
17 import type { Toolkit, Tool, Trigger, ParameterSchema } from '@/types/toolkit';
18 import { processSchema, toolFromApi } from '@/lib/toolkit-schema';
19
@@ -696,27 +697,6 @@ function toolkitToMarkdown(toolkit: Toolkit, detailedTools?: Tool[], detailedTri
697 return lines.join('\n');
698 }
699
699 -async function getToolkit(slug: string): Promise<Toolkit | null> {
700 - try {
701 - const filePath = join(process.cwd(), 'public/data/toolkits.json');
702 - const data = await readFile(filePath, 'utf-8');
703 - const toolkits = JSON.parse(data) as Toolkit[];
704 - return toolkits.find((t) => t.slug === slug) || null;
705 - } catch {
706 - return null;
707 - }
708 -}
709 -
710 -async function getAllToolkits(): Promise<Toolkit[]> {
711 - try {
712 - const filePath = join(process.cwd(), 'public/data/toolkits.json');
713 - const data = await readFile(filePath, 'utf-8');
714 - return JSON.parse(data) as Toolkit[];
715 - } catch {
716 - return [];
717 - }
718 -}
719 -
700 // Generate a comprehensive toolkits index for /toolkits.md
701 async function generateToolkitsIndex(): Promise<string> {
702 const toolkits = await getAllToolkits();
@@ -878,7 +858,7 @@ export async function GET(
858
859 // Special handling for JSON toolkit pages
860 if (prefix === 'toolkits' && rest.length === 1) {
881 - const toolkit = await getToolkit(rest[0]);
861 + const toolkit = await getToolkitBySlug(rest[0]);
862 if (toolkit) {
863 // Fetch detailed tool/trigger info and FAQ content in parallel
864 const [detailedTools, detailedTriggers, faqMarkdown] = await Promise.all([
docs/app/sitemap.ts
+2 -35
@@ -1,6 +1,4 @@
1 import type { MetadataRoute } from 'next';
2 -import { readFileSync } from 'fs';
3 -import { join } from 'path';
2 import {
3 source,
4 getReferenceSource,
@@ -9,41 +7,10 @@ import {
7 changelogEntries,
8 dateToChangelogUrl,
9 } from '@/lib/source';
10 +import { getAllToolkitsSync } from '@/lib/toolkit-data';
11
12 const baseUrl = 'https://docs.composio.dev';
13
15 -interface Toolkit {
16 - slug: string;
17 -}
18 -
19 -function getToolkitsFromJson(): Toolkit[] {
20 - const filePath = join(process.cwd(), 'public/data/toolkits.json');
21 -
22 - try {
23 - const data = readFileSync(filePath, 'utf-8');
24 - const toolkits = JSON.parse(data) as Toolkit[];
25 -
26 - if (!Array.isArray(toolkits)) {
27 - throw new Error('toolkits.json must contain an array');
28 - }
29 -
30 - if (toolkits.length === 0) {
31 - console.warn('[Sitemap] Warning: toolkits.json is empty - toolkit pages will be missing from sitemap');
32 - }
33 -
34 - return toolkits;
35 - } catch (error) {
36 - const err = error as NodeJS.ErrnoException;
37 - if (err.code === 'ENOENT') {
38 - throw new Error(`[Sitemap] Toolkits data file not found: ${filePath}`);
39 - }
40 - if (error instanceof SyntaxError) {
41 - throw new Error(`[Sitemap] Invalid JSON in toolkits.json: ${error.message}`);
42 - }
43 - throw error;
44 - }
45 -}
46 -
14 export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
15 const docsPages = source.getPages().map((page) => ({
16 url: `${baseUrl}${page.url}`,
@@ -65,7 +32,7 @@ export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
32 }));
33
34 // JSON toolkit pages (dynamically generated from toolkits.json)
68 - const toolkitsJsonPages = getToolkitsFromJson().map((toolkit) => ({
35 + const toolkitsJsonPages = getAllToolkitsSync().map((toolkit) => ({
36 url: `${baseUrl}/toolkits/${toolkit.slug}`,
37 }));
38
docs/lib/toolkit-data.ts new
+48
@@ -0,0 +1,48 @@
1 +import { readFile } from 'fs/promises';
2 +import { readFileSync } from 'fs';
3 +import { join } from 'path';
4 +import type { Toolkit } from '@/types/toolkit';
5 +
6 +const TOOLKITS_PATH = join(process.cwd(), 'public/data/toolkits.json');
7 +
8 +let cached: { list: Toolkit[]; bySlug: Map<string, Toolkit> } | null = null;
9 +
10 +function buildCache(toolkits: Toolkit[]) {
11 + cached = {
12 + list: toolkits,
13 + bySlug: new Map(toolkits.map(t => [t.slug, t])),
14 + };
15 + return cached;
16 +}
17 +
18 +function parse(data: string): Toolkit[] {
19 + const toolkits = JSON.parse(data) as Toolkit[];
20 +
21 + if (!Array.isArray(toolkits)) {
22 + throw new Error('toolkits.json must contain an array');
23 + }
24 +
25 + if (toolkits.length === 0) {
26 + console.warn('[Toolkits] Warning: toolkits.json is empty');
27 + }
28 +
29 + return toolkits;
30 +}
31 +
32 +export async function getAllToolkits(): Promise<Toolkit[]> {
33 + if (cached) return cached.list;
34 + const data = await readFile(TOOLKITS_PATH, 'utf-8');
35 + return buildCache(parse(data)).list;
36 +}
37 +
38 +export async function getToolkitBySlug(slug: string): Promise<Toolkit | null> {
39 + if (cached) return cached.bySlug.get(slug) ?? null;
40 + const data = await readFile(TOOLKITS_PATH, 'utf-8');
41 + return buildCache(parse(data)).bySlug.get(slug) ?? null;
42 +}
43 +
44 +export function getAllToolkitsSync(): Toolkit[] {
45 + if (cached) return cached.list;
46 + const data = readFileSync(TOOLKITS_PATH, 'utf-8');
47 + return buildCache(parse(data)).list;
48 +}