Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
docs: add agent-readable discovery and freshness metadata
Brendan O'Leary committed
Oct 15, 2026 at 05:26 UTC
94577129799179a57cfa1a2a237e559f37c6cdf3
30 files changed
+830
-43
.github/workflows/docs-tests.yml
+3
@@ -28,6 +28,9 @@ jobs:
28
steps:
29
- name: Checkout repository
30
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
31
+ with:
32
+ # Source freshness needs real per-file history, not checkout time.
33
+ fetch-depth: 0
34
35
- name: Setup Node.js, pnpm, Bun
36
uses: ./.github/actions/setup-node-pnpm-bun
docs/.gitignore
+3
@@ -44,3 +44,6 @@ next-env.d.ts.cache
44
/docs-graph.dot
45
/docs-graph.html
46
/docs-graph.json
47
+
48
+# Source-history manifest (generated before dev/build/typecheck; bundled for runtime).
49
+/.generated/
docs/CONTRIBUTING.md
+28
@@ -56,6 +56,34 @@ The semantic artifact covers both regular docs and KB articles, so edits to eith
56
57
The `Docs - Rebuild KB Semantic Artifact` workflow can rebuild it for eligible same-repository PRs after `Docs - Tests` runs. The scheduled support-knowledge refresh also proposes stale-artifact repairs. If you rebuild locally with `bun run build:kb-semantic`, use an authorized `OPENAI_API_KEY`: the generator sends the indexed docs content to the embeddings service. Run `bun run check:kb-semantic` without `--allow-stale` to verify a rebuild. Never edit embedding values or content hashes by hand.
58
59
+## Agent-readable representations
60
+
61
+Request a page with `Accept: text/markdown` (or `text/plain`), append `.md` or `.mdx`, or use its `/llms.mdx/` route. HTML pages with a Markdown representation advertise it with `<link rel="alternate" type="text/markdown">`. Negotiated responses include `Vary: Accept` so caches distinguish HTML from Markdown.
62
+
63
+Next.js `16.3.8` replaces middleware's `Vary` header in its Node HTML response template. The docs package carries a two-line dependency patch that appends Next's cache dimensions instead. Keep the live negotiation test when upgrading Next; remove the patch once upstream preserves middleware variance.
64
+
65
+Per-page Markdown starts with YAML front matter. For example, `/reference/authenticating-to-composio.md` identifies its HTML and Markdown URLs independently of the URL you requested:
66
+
67
+```yaml
68
+---
69
+canonical_url: "https://docs.composio.dev/reference/authenticating-to-composio"
70
+markdown_url: "https://docs.composio.dev/reference/authenticating-to-composio.md"
71
+product: "platform"
72
+version: "3.1"
73
+last_updated: null
74
+---
75
+```
76
+
77
+`product` uses the URL-based docs product, `platform` or `for-you`, without reading your product cookie. Shared pages default to `platform`. `version` is the documented REST version (`3.1` or legacy `3.0`), or `null` for pages that don't document a REST version. It isn't an SDK package version. `last_updated` is a UTC ISO 8601 timestamp, or `null` when freshness can't be established. The example uses `null`, not an invented update time.
78
+
79
+The sitemap's `lastmod` and Markdown's `last_updated` use the same build-generated source freshness data. They never use build time or filesystem modification time. Missing pages return Markdown discovery and search links with HTTP `404`; error responses don't claim a content update time.
80
+
81
+The prebuild, predev, and type-check steps generate `.generated/content-freshness.json` from each page's backing files. Deployed routes read that artifact without Git. Generated API and toolkit pages use the history of their source specifications or catalogs, so their freshness is file-level rather than operation-level.
82
+
83
+You can declare `last_updated: YYYY-MM-DD` or a timezone-qualified ISO timestamp in authored front matter. A date-only value means midnight UTC; an invalid value fails generation. Otherwise, freshness uses the source file's last Git commit. Aggregate pages use the latest known update across their backing files.
84
+
85
+Docs CI fetches full history. For other CI/CD builds, provide full history (`fetch-depth: 0` with `actions/checkout`) for complete freshness coverage. A shallow checkout retains dates for changes proven within its history but omits unknown shallow-boundary dates. Source archives without history also omit unknown dates. Neither case substitutes the checkout or build time.
86
+
87
## Request review and verify publication
88
89
Describe the reader's problem, the resulting behavior, and the checks you ran. Include screenshots for visual changes and list checks that failed or could not run. Link the Linear issue when one exists. Docs-only changes do not need a Changeset.
docs/app/(home)/docs/[[...slug]]/page.tsx
+2
-1
@@ -1,3 +1,4 @@
1
+import { markdownAlternates } from '@/lib/markdown-alternates';
2
import { getOgImageUrl, source } from '@/lib/source';
3
import { DocsBody, DocsPage, DocsTitle } from 'fumadocs-ui/layouts/docs/page';
4
import { notFound } from 'next/navigation';
@@ -76,7 +77,7 @@ export async function generateMetadata(props: PageProps<'/docs/[[...slug]]'>): P
77
? { absolute: `${page.data.title} | Composio Documentation` }
78
: page.data.title,
79
description: page.data.description,
79
- alternates: { canonical: page.url },
80
+ alternates: markdownAlternates(page.url),
81
openGraph: {
82
title: page.data.title,
83
description: page.data.description,
docs/app/(home)/kb/page.tsx
+2
-1
@@ -1,10 +1,11 @@
1
+import { markdownAlternates } from '@/lib/markdown-alternates';
2
import type { Metadata } from 'next';
3
import { KnowledgeHub } from '@/components/kb/knowledge-hub';
4
5
export const metadata: Metadata = {
6
title: 'Knowledge Base',
7
description: 'Search public Composio product knowledge across docs, support answers, OAuth guides, toolkits, examples, reference, and changelog.',
7
- alternates: { canonical: '/kb' },
8
+ alternates: markdownAlternates('/kb'),
9
};
10
11
export default function KnowledgeBasePage() {
docs/app/(home)/kb/search/page.tsx
+1
@@ -10,6 +10,7 @@ export const metadata: Metadata = {
10
title: 'Search Composio knowledge',
11
description: 'Search all public Composio product knowledge.',
12
robots: { index: false, follow: true },
13
+ alternates: { types: { 'text/markdown': '/kb/search.md' } },
14
};
15
16
interface KnowledgeSearchPageProps {
docs/app/(home)/kb/toolkit/[slug]/page.tsx
+2
-1
@@ -1,3 +1,4 @@
1
+import { markdownAlternates } from '@/lib/markdown-alternates';
2
import type { Metadata } from 'next';
3
import Link from 'next/link';
4
import { notFound, redirect } from 'next/navigation';
@@ -56,6 +57,6 @@ export async function generateMetadata({ params }: ToolkitPageProps): Promise<Me
57
return {
58
title: `${toolkit.name} knowledge`,
59
description: `Find public Composio docs, support answers, OAuth guides, examples, and reference pages for ${toolkit.name}.`,
59
- alternates: { canonical: `/kb/toolkit/${toolkit.slug}` },
60
+ alternates: markdownAlternates(`/kb/toolkit/${toolkit.slug}`),
61
};
62
}
docs/app/(home)/kb/toolkits/page.tsx
+2
-1
@@ -1,3 +1,4 @@
1
+import { markdownAlternates } from '@/lib/markdown-alternates';
2
import type { Metadata } from 'next';
3
import Link from 'next/link';
4
import { ArrowLeft } from 'lucide-react';
@@ -7,7 +8,7 @@ import { getKnowledgeToolkitSummaries } from '@/lib/knowledge/catalog';
8
export const metadata: Metadata = {
9
title: 'Browse knowledge by toolkit',
10
description: 'Find public Composio knowledge for every supported toolkit.',
10
- alternates: { canonical: '/kb/toolkits' },
11
+ alternates: markdownAlternates('/kb/toolkits'),
12
};
13
14
export default async function KnowledgeToolkitsPage() {
docs/app/(home)/kb/topic/[slug]/page.tsx
+2
-1
@@ -1,3 +1,4 @@
1
+import { markdownAlternates } from '@/lib/markdown-alternates';
2
import type { Metadata } from 'next';
3
import Link from 'next/link';
4
import { notFound, permanentRedirect } from 'next/navigation';
@@ -55,6 +56,6 @@ export async function generateMetadata({ params }: TopicPageProps): Promise<Meta
56
return {
57
title: `${area.title} knowledge`,
58
description: area.description,
58
- alternates: { canonical: `/kb/topic/${area.slug}` },
59
+ alternates: markdownAlternates(`/kb/topic/${area.slug}`),
60
};
61
}
docs/app/(home)/reference/(v31)/[[...slug]]/page.tsx
+3
-2
@@ -1,3 +1,4 @@
1
+import { markdownAlternates } from '@/lib/markdown-alternates';
2
import { getReferenceSource, getOgImageUrl } from '@/lib/source';
3
import { APIPage } from '@/components/api-page';
4
import { DocsBody, DocsPage, DocsTitle } from 'fumadocs-ui/layouts/docs/page';
@@ -93,7 +94,7 @@ export async function generateMetadata({
94
return {
95
title: 'API Reference',
96
description: 'REST API and SDK reference for Composio',
96
- alternates: { canonical: '/reference' },
97
+ alternates: markdownAlternates('/reference'),
98
openGraph: { images: [ogImage] },
99
twitter: { card: 'summary_large_image', images: [ogImage] },
100
};
@@ -112,7 +113,7 @@ export async function generateMetadata({
113
return {
114
title: page.data.title,
115
description,
115
- alternates: { canonical: page.url },
116
+ alternates: markdownAlternates(page.url),
117
openGraph: { images: [ogImage] },
118
twitter: { card: 'summary_large_image', images: [ogImage] },
119
};
docs/app/(home)/reference/v3/[[...slug]]/page.tsx
+3
-2
@@ -1,3 +1,4 @@
1
+import { markdownAlternates } from '@/lib/markdown-alternates';
2
import { getReferenceSource, getOgImageUrl } from '@/lib/source';
3
import { APIPage } from '@/components/api-page';
4
import { DocsBody, DocsPage, DocsTitle } from 'fumadocs-ui/layouts/docs/page';
@@ -102,7 +103,7 @@ export async function generateMetadata({
103
return {
104
title: 'API Reference (v3)',
105
description: 'REST API reference for Composio v3',
105
- alternates: { canonical: '/reference/v3' },
106
+ alternates: markdownAlternates('/reference/v3'),
107
openGraph: { images: [ogImage] },
108
twitter: { card: 'summary_large_image', images: [ogImage] },
109
};
@@ -121,7 +122,7 @@ export async function generateMetadata({
122
return {
123
title: page.data.title,
124
description,
124
- alternates: { canonical: page.url },
125
+ alternates: markdownAlternates(page.url),
126
openGraph: { images: [ogImage] },
127
twitter: { card: 'summary_large_image', images: [ogImage] },
128
};
docs/app/(home)/toolkits/[[...slug]]/page.tsx
+7
-4
@@ -1,3 +1,4 @@
1
+import { markdownAlternates } from '@/lib/markdown-alternates';
2
import { toolkitsSource, getOgImageUrl } from '@/lib/source';
3
import Link from 'next/link';
4
import { notFound } from 'next/navigation';
@@ -8,7 +9,7 @@ import { PageActions } from '@/components/page-actions';
9
import { EditOnGitHub } from '@/components/edit-on-github';
10
import { readFile } from 'fs/promises';
11
import { join } from 'path';
11
-import { getAllToolkits } from '@/lib/toolkit-data';
12
+import { getAllToolkits, getToolkitBySlug } from '@/lib/toolkit-data';
13
import { resolveToolkit } from '@/lib/toolkit-resolution';
14
import { unified } from 'unified';
15
import remarkParse from 'remark-parse';
@@ -159,7 +160,7 @@ export async function generateMetadata({
160
return {
161
title: 'Toolkits',
162
description: 'Browse all toolkits supported by Composio',
162
- alternates: { canonical: '/toolkits' },
163
+ alternates: markdownAlternates('/toolkits'),
164
openGraph: { images: [ogImage] },
165
twitter: { card: 'summary_large_image', images: [ogImage] },
166
};
@@ -172,7 +173,7 @@ export async function generateMetadata({
173
return {
174
title: page.data.title,
175
description: page.data.description,
175
- alternates: { canonical: page.url },
176
+ alternates: markdownAlternates(page.url),
177
openGraph: { images: [ogImage] },
178
twitter: { card: 'summary_large_image', images: [ogImage] },
179
};
@@ -190,7 +191,9 @@ export async function generateMetadata({
191
return {
192
title,
193
description,
193
- alternates: { canonical: `/toolkits/${toolkit.slug}` },
194
+ alternates: await getToolkitBySlug(toolkit.slug)
195
+ ? markdownAlternates(`/toolkits/${toolkit.slug}`)
196
+ : { canonical: `/toolkits/${toolkit.slug}` },
197
openGraph: { images: [ogImage] },
198
twitter: { card: 'summary_large_image', images: [ogImage] },
199
};
docs/app/llms.mdx/[[...slug]]/route.ts
+22
-10
@@ -20,7 +20,8 @@ import {
20
isToolVersionPath,
21
toolVersionParameterDescription,
22
} from '@/lib/api-version-guidance';
23
-import { notFound } from 'next/navigation';
23
+import { getPageLastUpdated } from '@/lib/content-freshness';
24
+import { missingMarkdownResponse, withMarkdownMetadata } from '@/lib/markdown-response';
25
import { readFile } from 'fs/promises';
26
import { join } from 'path';
27
import { getAllToolkits, getToolkitBySlug } from '@/lib/toolkit-data';
@@ -1207,7 +1208,23 @@ function metaToolsIndexToMarkdown(tools: MetaTool[]): string {
1208
return lines.join('\n') + LLM_FOOTER;
1209
}
1210
1210
-export async function GET(_req: Request, { params }: { params: Promise<{ slug?: string[] }> }) {
1211
+export async function GET(req: Request, context: { params: Promise<{ slug?: string[] }> }) {
1212
+ const { slug = [] } = await context.params;
1213
+ const pathname = `/${slug.join('/')}`;
1214
+ const response = await renderMarkdown(req, context);
1215
+ if (!response.headers.get('Content-Type')?.startsWith('text/markdown')) {
1216
+ response.headers.set('Vary', 'Accept');
1217
+ return response;
1218
+ }
1219
+ return withMarkdownMetadata(
1220
+ await response.text(),
1221
+ pathname,
1222
+ response.ok ? await getPageLastUpdated(pathname) : null,
1223
+ { status: response.status, statusText: response.statusText, headers: response.headers },
1224
+ );
1225
+}
1226
+
1227
+async function renderMarkdown(_req: Request, { params }: { params: Promise<{ slug?: string[] }> }) {
1228
try {
1229
const { slug = [] } = await params;
1230
const [prefix, ...rest] = slug;
@@ -1278,8 +1295,7 @@ export async function GET(_req: Request, { params }: { params: Promise<{ slug?:
1295
});
1296
}
1297
}
1281
- // If no entries found, fall through to notFound
1282
- notFound();
1298
+ return missingMarkdownResponse();
1299
}
1300
1301
// Special handling for meta tools - /toolkits/meta-tools and /toolkits/meta-tools/{slug}.
@@ -1320,7 +1336,7 @@ export async function GET(_req: Request, { params }: { params: Promise<{ slug?:
1336
}
1337
} else {
1338
const match = sources.find(s => s.prefix === prefix);
1323
- if (!match) notFound();
1339
+ if (!match) return missingMarkdownResponse();
1340
pageSource = match.source;
1341
}
1342
@@ -1414,12 +1430,8 @@ export async function GET(_req: Request, { params }: { params: Promise<{ slug?:
1430
}
1431
}
1432
1417
- notFound();
1433
+ return missingMarkdownResponse();
1434
} catch (e) {
1419
- // Don't catch notFound - let it propagate
1420
- if (e && typeof e === 'object' && 'digest' in e) {
1421
- throw e;
1422
- }
1435
console.error('Unexpected error in llms.mdx route:', e);
1436
return new Response(`# Error\n\nAn error occurred while generating the markdown content.`, {
1437
status: 500,
docs/app/sitemap.ts
+7
-1
@@ -10,6 +10,7 @@ import {
10
} from '@/lib/source';
11
import { getAllToolkitsSync } from '@/lib/toolkit-data';
12
import { getLocalKnowledgeDiscoveryPaths } from '@/lib/knowledge/discovery';
13
+import { getPageLastUpdated } from '@/lib/content-freshness';
14
15
const baseUrl = 'https://docs.composio.dev';
16
@@ -79,5 +80,10 @@ export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
80
...changelogPages,
81
];
82
82
- return Array.from(new Map(pages.map((page) => [page.url, page])).values());
83
+ return Promise.all(
84
+ Array.from(new Map(pages.map((page) => [page.url, page])).values()).map(async (page) => {
85
+ const lastModified = await getPageLastUpdated(new URL(page.url).pathname);
86
+ return lastModified ? { ...page, lastModified } : page;
87
+ }),
88
+ );
89
}
docs/bun.lock
+3
@@ -93,6 +93,9 @@
93
},
94
},
95
},
96
+ "patchedDependencies": {
97
+ "next@16.3.8": "patches/next@16.3.8.patch",
98
+ },
99
"overrides": {
100
"@mastra/mcp": "1.17.2",
101
},
docs/lib/content-freshness-build.ts
new
+97
@@ -0,0 +1,97 @@
1
+import { execFileSync } from 'node:child_process';
2
+import { readFileSync } from 'node:fs';
3
+import { isAbsolute, resolve } from 'node:path';
4
+import { z } from 'zod';
5
+import type { ContentFreshnessManifest } from './content-freshness';
6
+
7
+export interface FreshnessPage {
8
+ pathname: string;
9
+ /** Absolute or docs-root-relative backing source files, never generated outputs. */
10
+ sources: string[];
11
+ lastUpdated?: string;
12
+}
13
+
14
+/** Date-only declarations mean midnight UTC; timestamps must include a timezone. */
15
+export function normalizeLastUpdated(value: string): string | null {
16
+ if (z.iso.date().safeParse(value).success) return `${value}T00:00:00.000Z`;
17
+ if (!z.iso.datetime({ offset: true }).safeParse(value).success) return null;
18
+ return new Date(value).toISOString();
19
+}
20
+
21
+/** Only this explicit field is freshness metadata, not release/written/review dates. */
22
+export function getExplicitLastUpdated(raw: string): string | undefined {
23
+ const frontmatter = raw.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/)?.[1];
24
+ const value = frontmatter?.match(/^last_updated:[ \t]*(.*?)[ \t]*$/m)?.[1];
25
+ if (value === undefined) return undefined;
26
+ const unquoted = value.replace(/^(?:"(.*)"|'(.*)')$/, '$1$2');
27
+ if (!normalizeLastUpdated(unquoted)) throw new Error(`Invalid last_updated: ${value}`);
28
+ return unquoted;
29
+}
30
+
31
+function getSourceHistory(root: string): Map<string, string | null> {
32
+ const history = new Map<string, string | null>();
33
+ const git = (...args: string[]) => execFileSync('git', ['-c', 'core.quotepath=false', ...args], {
34
+ cwd: root, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024,
35
+ stdio: ['ignore', 'pipe', 'pipe'],
36
+ }).trim();
37
+ try {
38
+ const repository = git('rev-parse', '--show-toplevel');
39
+ const shallowPath = git('rev-parse', '--git-path', 'shallow');
40
+ let boundaries = new Set<string>();
41
+ try {
42
+ boundaries = new Set(readFileSync(resolve(root, shallowPath), 'utf8').trim().split('\n'));
43
+ } catch {
44
+ // A complete repository has no shallow file.
45
+ }
46
+ const log = git('log', '--format=%x1e%H%x00%cI', '--name-only', '--no-renames', '--', '.');
47
+ for (const record of log.split('\x1e')) {
48
+ const [header, ...files] = record.trim().split('\n');
49
+ if (!header) continue;
50
+ const [hash, timestamp] = header.split('\0');
51
+ const date = timestamp ? normalizeLastUpdated(timestamp) : null;
52
+ for (const file of files) {
53
+ if (!file) continue;
54
+ const absolutePath = resolve(repository, file);
55
+ if (!history.has(absolutePath)) {
56
+ // Git presents every file as added at a shallow boundary; that is not evidence.
57
+ history.set(absolutePath, hash && boundaries.has(hash) ? null : date);
58
+ }
59
+ }
60
+ }
61
+ } catch {
62
+ // Source archives and deployments without Git must not invent modification dates.
63
+ }
64
+ return history;
65
+}
66
+
67
+export function buildContentFreshness(
68
+ root: string,
69
+ pages: FreshnessPage[],
70
+ sourceLastUpdated: Record<string, string> = {},
71
+): ContentFreshnessManifest {
72
+ const history = getSourceHistory(root);
73
+ const manifest: ContentFreshnessManifest = { version: 1, pages: {} };
74
+ for (const page of pages) {
75
+ if (page.lastUpdated !== undefined) {
76
+ const explicit = normalizeLastUpdated(page.lastUpdated);
77
+ if (!explicit) throw new Error(`Invalid last_updated for ${page.pathname}: ${page.lastUpdated}`);
78
+ manifest.pages[page.pathname] = explicit;
79
+ continue;
80
+ }
81
+ const dates = [...new Set(page.sources)].map(source => {
82
+ const absolutePath = isAbsolute(source) ? source : resolve(root, source);
83
+ const explicit = sourceLastUpdated[absolutePath];
84
+ if (explicit !== undefined) {
85
+ const timestamp = normalizeLastUpdated(explicit);
86
+ if (!timestamp) throw new Error(`Invalid last_updated for ${source}: ${explicit}`);
87
+ return timestamp;
88
+ }
89
+ return history.get(absolutePath) ?? null;
90
+ });
91
+ // A partially-known aggregate is not a reliable last-modification timestamp.
92
+ if (dates.length === 0 || dates.some(date => date === null)) continue;
93
+ const known = dates.filter((date): date is string => date !== null);
94
+ manifest.pages[page.pathname] = known.reduce((latest, date) => date > latest ? date : latest);
95
+ }
96
+ return manifest;
97
+}
docs/lib/content-freshness.ts
new
+25
@@ -0,0 +1,25 @@
1
+import { readFile } from 'node:fs/promises';
2
+import { join } from 'node:path';
3
+import { z } from 'zod';
4
+
5
+export const contentFreshnessSchema = z.object({
6
+ version: z.literal(1),
7
+ pages: z.record(z.string(), z.iso.datetime({ offset: true })),
8
+});
9
+
10
+export type ContentFreshnessManifest = z.infer<typeof contentFreshnessSchema>;
11
+
12
+/** Read only the persisted build artifact: deployed functions never need Git. */
13
+export function createPageLastUpdatedReader(manifestPath: string) {
14
+ let manifest: Promise<ContentFreshnessManifest | null> | undefined;
15
+ return async (pathname: string): Promise<string | null> => {
16
+ manifest ??= readFile(manifestPath, 'utf8').then(
17
+ text => contentFreshnessSchema.parse(JSON.parse(text)),
18
+ ).catch(() => null);
19
+ const data = await manifest;
20
+ return data && Object.hasOwn(data.pages, pathname) ? data.pages[pathname] ?? null : null;
21
+ };
22
+}
23
+
24
+export const getPageLastUpdated: (pathname: string) => Promise<string | null> =
25
+ createPageLastUpdatedReader(join(process.cwd(), '.generated/content-freshness.json'));
docs/lib/create-docs-page.tsx
+2
-1
@@ -1,3 +1,4 @@
1
+import { markdownAlternates } from '@/lib/markdown-alternates';
2
import { DocsBody, DocsPage, DocsTitle } from 'fumadocs-ui/layouts/docs/page';
3
import { notFound } from 'next/navigation';
4
import { getMDXComponents } from '@/mdx-components';
@@ -64,7 +65,7 @@ export function createGenerateMetadata(source: Source, section: string = 'docs')
65
return {
66
title: page.data.title,
67
description,
67
- alternates: { canonical: page.url },
68
+ alternates: markdownAlternates(page.url),
69
openGraph: {
70
images: [ogImage],
71
},
docs/lib/markdown-alternates.ts
new
+9
@@ -0,0 +1,9 @@
1
+import type { Metadata } from 'next';
2
+
3
+/** Only use for HTML pages that have a corresponding per-page Markdown route. */
4
+export function markdownAlternates(pathname: string): Metadata['alternates'] {
5
+ return {
6
+ canonical: pathname,
7
+ types: { 'text/markdown': `${pathname}.md` },
8
+ };
9
+}
docs/lib/markdown-response.ts
new
+42
@@ -0,0 +1,42 @@
1
+import { detectReferenceApiVersion } from './api-version';
2
+import { resolveDocsProduct } from './home-navigation';
3
+
4
+const DOCS_ORIGIN = 'https://docs.composio.dev';
5
+
6
+/** JSON scalars are also YAML scalars, so URLs and strings remain safely quoted. */
7
+export function withMarkdownMetadata(
8
+ body: string,
9
+ pathname: string,
10
+ lastUpdated: string | null,
11
+ init?: ResponseInit,
12
+): Response {
13
+ const canonicalUrl = new URL(pathname, DOCS_ORIGIN).href;
14
+ const metadata = {
15
+ canonical_url: canonicalUrl,
16
+ markdown_url: `${canonicalUrl}.md`,
17
+ product: resolveDocsProduct(pathname),
18
+ version: detectReferenceApiVersion(pathname),
19
+ last_updated: lastUpdated,
20
+ };
21
+ const frontmatter = Object.entries(metadata)
22
+ .map(([key, value]) => `${key}: ${JSON.stringify(value)}`)
23
+ .join('\n');
24
+ const headers = new Headers(init?.headers);
25
+ headers.set('Content-Type', 'text/markdown; charset=utf-8');
26
+ const vary = headers.get('Vary');
27
+ if (!vary?.split(',').some(value => ['accept', '*'].includes(value.trim().toLowerCase()))) {
28
+ headers.set('Vary', vary ? `${vary}, Accept` : 'Accept');
29
+ }
30
+ return new Response(`---\n${frontmatter}\n---\n\n${body}`, { ...init, headers });
31
+}
32
+
33
+export function missingMarkdownResponse(): Response {
34
+ return new Response(
35
+ '# Page not found\n\nThis documentation page does not exist. Check the URL or use these discovery links:\n\n' +
36
+ '- [Documentation index](/llms.txt)\n' +
37
+ '- [Complete page index](/llms-index.txt)\n' +
38
+ '- [Search support knowledge](/kb/search)\n' +
39
+ '- [Quickstart](/docs/quickstart.md)\n',
40
+ { status: 404, headers: { 'Content-Type': 'text/markdown; charset=utf-8' } },
41
+ );
42
+}
docs/next.config.mjs
+2
-1
@@ -36,9 +36,10 @@ const config = {
36
// manifest data, so relying on automatic tracing would make those bundles
37
// fragile if their path construction changes.
38
outputFileTracingIncludes: {
39
+ '/sitemap.xml': ['./.generated/content-freshness.json'],
40
'/reference/**': [...OPENAPI_SPEC_FILES],
41
'/reference/v3/**': [...OPENAPI_SPEC_FILES],
41
- '/llms.mdx/**': [...OPENAPI_SPEC_FILES],
42
+ '/llms.mdx/**': [...OPENAPI_SPEC_FILES, './.generated/content-freshness.json'],
43
'/llms-full.txt/**': [...OPENAPI_SPEC_FILES],
44
'/llms.txt/**': [...OPENAPI_SPEC_FILES],
45
'/llms-index.txt/**': [...OPENAPI_SPEC_FILES, './kb/**'],
docs/package.json
+7
-3
@@ -7,18 +7,19 @@
7
},
8
"scripts": {
9
"build:agent-index": "bun scripts/build-agent-index.ts",
10
+ "build:content-freshness": "fumadocs-mdx && bun scripts/build-content-freshness.ts",
11
"build:kb-semantic": "bun run generate:kb && fumadocs-mdx && bun scripts/build-kb-semantic-index.ts",
12
"check:kb-semantic": "bun run check:kb && fumadocs-mdx && bun scripts/build-kb-semantic-index.ts --check",
13
"verify:kb": "bun scripts/verify-kb.ts",
14
"import:kb": "bun scripts/import-support-knowledge.ts",
15
"generate:kb": "bun scripts/generate-kb.ts",
16
"check:kb": "bun scripts/generate-kb.ts --check",
16
- "prebuild": "bun run generate:kb && bun scripts/build-agent-index.ts",
17
+ "prebuild": "bun run generate:kb && bun scripts/build-agent-index.ts && bun run build:content-freshness",
18
"build": "next build",
18
- "predev": "bun run generate:kb && bun scripts/build-agent-index.ts",
19
+ "predev": "bun run generate:kb && bun scripts/build-agent-index.ts && bun run build:content-freshness",
20
"dev": "next dev",
21
"start": "next start",
21
- "types:check": "bun run generate:kb && bun scripts/build-agent-index.ts && fumadocs-mdx && next typegen && bun ./node_modules/typescript-7/bin/tsc --noEmit",
22
+ "types:check": "bun run generate:kb && bun scripts/build-agent-index.ts && bun run build:content-freshness && next typegen && bun ./node_modules/typescript-7/bin/tsc --noEmit",
23
"postinstall": "bun scripts/generate-kb.ts && fumadocs-mdx",
24
"lint": "oxlint .",
25
"lint:links": "bun scripts/validate-links.ts",
@@ -124,6 +125,9 @@
125
"typescript": "^6.0.3",
126
"typescript-7": "npm:typescript@^7.0.2"
127
},
128
+ "patchedDependencies": {
129
+ "next@16.3.8": "patches/next@16.3.8.patch"
130
+ },
131
"overrides": {
132
"@mastra/mcp": "1.17.2"
133
}
docs/patches/next@16.3.8.patch
new
+26
@@ -0,0 +1,26 @@
1
+diff --git a/dist/build/templates/app-page-runtime.js b/dist/build/templates/app-page-runtime.js
2
+index dfec9df74e9095294ca8e0f1c4b9f4601421025d..085b954679b0126bda1f55cf19bac3dfde943ccf 100644
3
+--- a/dist/build/templates/app-page-runtime.js
4
++++ b/dist/build/templates/app-page-runtime.js
5
+@@ -460,7 +460,7 @@ function createAppPageEntrypoint({ tree, page, pathname, require: __next_app_req
6
+ };
7
+ try {
8
+ const varyHeader = routeModule.getVaryHeader(resolvedPathname, interceptionRoutePatterns);
9
+- res.setHeader('Vary', varyHeader);
10
++ res.appendHeader('Vary', varyHeader);
11
+ let parentSpan;
12
+ const invokeRouteModule = async (span, context)=>{
13
+ const nextReq = new _node.NodeNextRequest(req);
14
+diff --git a/dist/esm/build/templates/app-page-runtime.js b/dist/esm/build/templates/app-page-runtime.js
15
+index d991e81854e6685017936dd21fb9c4631edf9478..51ecfe3ed4b11a119b0cb6880d3076c460f49375 100644
16
+--- a/dist/esm/build/templates/app-page-runtime.js
17
++++ b/dist/esm/build/templates/app-page-runtime.js
18
+@@ -477,7 +477,7 @@ export function createAppPageEntrypoint({ tree, page, pathname, require: __next_
19
+ };
20
+ try {
21
+ const varyHeader = routeModule.getVaryHeader(resolvedPathname, interceptionRoutePatterns);
22
+- res.setHeader('Vary', varyHeader);
23
++ res.appendHeader('Vary', varyHeader);
24
+ let parentSpan;
25
+ const invokeRouteModule = async (span, context)=>{
26
+ const nextReq = new NodeNextRequest(req);
docs/proxy.ts
+7
-2
@@ -26,7 +26,9 @@ export function proxy(request: NextRequest) {
26
// Handle markdown content negotiation for AI agents
27
if (accept.includes('text/markdown') || accept.includes('text/plain')) {
28
const mdUrl = new URL(`/llms.mdx${pathname}`, request.nextUrl.origin);
29
- return NextResponse.rewrite(mdUrl);
29
+ const response = NextResponse.rewrite(mdUrl);
30
+ response.headers.append('Vary', 'Accept');
31
+ return response;
32
}
33
34
// Handle old Fern API reference URLs (kebab-case → camelCase)
@@ -51,7 +53,9 @@ export function proxy(request: NextRequest) {
53
if (newPathname !== pathname) {
54
const url = request.nextUrl.clone();
55
url.pathname = newPathname;
54
- return NextResponse.redirect(url, 301);
56
+ const response = NextResponse.redirect(url, 301);
57
+ response.headers.append('Vary', 'Accept');
58
+ return response;
59
}
60
}
61
@@ -64,6 +68,7 @@ export function proxy(request: NextRequest) {
68
69
const response = NextResponse.next({ request: { headers: requestHeaders } });
70
response.headers.set('x-pathname', pathname);
71
+ response.headers.append('Vary', 'Accept');
72
return response;
73
}
74
docs/scripts/build-content-freshness.ts
new
+108
@@ -0,0 +1,108 @@
1
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
2
+import { dirname, join, resolve } from 'node:path';
3
+import {
4
+ source, getReferenceSource, examplesSource, toolkitsSource, knowledgeBaseSource,
5
+ changelogEntries, dateToChangelogUrl,
6
+} from '../lib/source';
7
+import { getAllToolkitsSync } from '../lib/toolkit-data';
8
+import { getLocalKnowledgeDiscoveryPaths } from '../lib/knowledge/discovery';
9
+import { getPublishedKbGuides, getKbGuideUrl } from '../lib/kb/repository';
10
+import {
11
+ buildContentFreshness, getExplicitLastUpdated, type FreshnessPage,
12
+} from '../lib/content-freshness-build';
13
+
14
+const root = process.cwd();
15
+const pages = new Map<string, FreshnessPage>();
16
+const explicitSources: Record<string, string> = {};
17
+
18
+function add(pathname: string, sources: string[]) {
19
+ const current = pages.get(pathname);
20
+ pages.set(pathname, { pathname, sources: [...new Set([...(current?.sources ?? []), ...sources])] });
21
+}
22
+
23
+function authored(pathname: string, absolutePath?: string) {
24
+ if (!absolutePath || !existsSync(absolutePath)) return;
25
+ const raw = readFileSync(absolutePath, 'utf8');
26
+ const explicit = getExplicitLastUpdated(raw);
27
+ if (explicit !== undefined) explicitSources[resolve(absolutePath)] = explicit;
28
+ add(pathname, [absolutePath]);
29
+}
30
+
31
+for (const collection of [source, examplesSource, toolkitsSource, knowledgeBaseSource]) {
32
+ for (const page of collection.getPages()) authored(page.url, page.absolutePath);
33
+}
34
+
35
+const reference = await getReferenceSource();
36
+for (const page of reference.getPages()) {
37
+ const spec = page.url.startsWith('/reference/v3/') ? 'public/openapi-v3.json'
38
+ : page.url.includes('/webhook-events') ? 'public/openapi-webhooks.json'
39
+ : 'public/openapi.json';
40
+ const raw = page.absolutePath && existsSync(page.absolutePath)
41
+ ? readFileSync(page.absolutePath, 'utf8') : null;
42
+ if (raw?.includes('Auto-generated from')) {
43
+ const tag = page.url.split('/').at(-1);
44
+ const overview = `api-overviews/${tag}.mdx`;
45
+ const specs = spec === 'public/openapi-webhooks.json' ? [spec]
46
+ : ['public/openapi.json', 'public/openapi-v3.json'];
47
+ add(page.url, [...specs, 'scripts/generate-api-index.ts',
48
+ ...(existsSync(join(root, overview)) ? [overview] : [])]);
49
+ } else if (raw !== null) {
50
+ authored(page.url, page.absolutePath);
51
+ } else {
52
+ add(page.url, [spec, 'lib/openapi.ts']);
53
+ }
54
+}
55
+
56
+for (const toolkit of getAllToolkitsSync()) {
57
+ add(`/toolkits/${toolkit.slug}`, ['public/data/toolkits.json']);
58
+}
59
+
60
+// Generated MDX is a presentation artifact. Use the actual KB articles and sources.
61
+for (const guide of getPublishedKbGuides()) {
62
+ const pathname = getKbGuideUrl(guide);
63
+ pages.delete(pathname);
64
+ add(pathname, [
65
+ 'kb/manifest.json',
66
+ ...guide.sources.map(source => `kb/source/${source.sourcePath}`),
67
+ ...(guide.articlePath ? [`kb/articles/${guide.articlePath}`] : []),
68
+ ]);
69
+}
70
+
71
+const changelogSources: string[] = [];
72
+for (const entry of changelogEntries) {
73
+ const file = join(root, 'content/changelog', entry.info.path);
74
+ const filenameDate = entry.info.path.match(/^(\d{2})-(\d{2})-(\d{2})(?:-|\.)/);
75
+ const date = /^\d{4}-\d{2}-\d{2}$/.test(entry.date) ? entry.date
76
+ : filenameDate ? `20${filenameDate[3]}-${filenameDate[1]}-${filenameDate[2]}` : null;
77
+ if (date) authored(dateToChangelogUrl(date), file);
78
+ changelogSources.push(file);
79
+}
80
+add('/docs/changelog', changelogSources);
81
+add('/', ['app/(home)/page.tsx', ...(pages.get('/docs')?.sources ?? [])]);
82
+
83
+// These browse pages are aggregates over the local knowledge corpus. Require all
84
+// backing timestamps rather than claim a partially-known maximum is authoritative.
85
+pages.delete('/kb');
86
+const corpusSources = [...pages.values()].flatMap(page => page.sources);
87
+for (const pathname of await getLocalKnowledgeDiscoveryPaths()) {
88
+ if (pathname === '/kb/search' || pathname.startsWith('/kb/guide/')) continue;
89
+ add(pathname, [...corpusSources, 'public/data/toolkits-list.json', 'lib/knowledge/taxonomy.ts']);
90
+}
91
+const toolkitSources = getAllToolkitsSync().length > 0 ? ['public/data/toolkits.json'] : [];
92
+if (pages.has('/toolkits')) add('/toolkits', toolkitSources);
93
+if (pages.has('/toolkits/managed-auth')) add('/toolkits/managed-auth', toolkitSources);
94
+for (const pathname of pages.keys()) {
95
+ if (pathname === '/toolkits/meta-tools' || pathname.startsWith('/toolkits/meta-tools/')) {
96
+ add(pathname, ['public/data/meta-tools.json', 'lib/meta-tool-overrides.ts']);
97
+ }
98
+}
99
+if (pages.has('/examples')) {
100
+ add('/examples', [...pages.values()].filter(page => page.pathname.startsWith('/examples/'))
101
+ .flatMap(page => page.sources));
102
+}
103
+
104
+const manifest = buildContentFreshness(root, [...pages.values()], explicitSources);
105
+const output = join(root, '.generated/content-freshness.json');
106
+mkdirSync(dirname(output), { recursive: true });
107
+writeFileSync(output, `${JSON.stringify(manifest, null, 2)}\n`);
108
+console.log(`Persisted trustworthy freshness for ${Object.keys(manifest.pages).length} pages.`);
docs/tests/integration/agent-metadata.test.ts
new
+90
@@ -0,0 +1,90 @@
1
+import { describe, expect, test } from 'bun:test';
2
+import { z } from 'zod';
3
+import { fetchPage } from './helpers';
4
+
5
+const metadataSchema = z.object({
6
+ canonical_url: z.url(),
7
+ markdown_url: z.url(),
8
+ product: z.enum(['platform', 'for-you']),
9
+ version: z.enum(['3.1', '3.0']).nullable(),
10
+ last_updated: z.iso.datetime().nullable(),
11
+});
12
+
13
+function parseMetadata(text: string) {
14
+ const match = text.match(/^---\n([\s\S]*?)\n---\n\n/);
15
+ expect(match, 'Markdown starts with YAML metadata').not.toBeNull();
16
+ return metadataSchema.parse(Bun.YAML.parse(match![1]));
17
+}
18
+
19
+describe('agent-readable Markdown metadata', () => {
20
+ test('suffix, direct and negotiated representations identify the same resource', async () => {
21
+ const responses = await Promise.all([
22
+ fetchPage('/docs/quickstart.md'),
23
+ fetchPage('/docs/quickstart.mdx'),
24
+ fetchPage('/llms.mdx/docs/quickstart'),
25
+ fetchPage('/docs/quickstart', { headers: { Accept: 'text/markdown' } }),
26
+ fetchPage('/docs/quickstart', { headers: { Accept: 'text/plain' } }),
27
+ ]);
28
+ const metadata = await Promise.all(responses.map(async response => {
29
+ expect(response.status).toBe(200);
30
+ expect(response.headers.get('content-type')).toContain('text/markdown');
31
+ return parseMetadata(await response.text());
32
+ }));
33
+ expect(metadata[0]).toMatchObject({
34
+ canonical_url: 'https://docs.composio.dev/docs/quickstart',
35
+ markdown_url: 'https://docs.composio.dev/docs/quickstart.md',
36
+ product: 'platform',
37
+ version: null,
38
+ });
39
+ for (const item of metadata.slice(1)) expect(item).toEqual(metadata[0]);
40
+
41
+ const sitemap = await (await fetchPage('/sitemap.xml')).text();
42
+ const entry = sitemap.match(/<url>\s*<loc>https:\/\/docs\.composio\.dev\/docs\/quickstart<\/loc>\s*<lastmod>([^<]+)<\/lastmod>/);
43
+ expect(entry, 'known source freshness is published in the sitemap').not.toBeNull();
44
+ expect(metadata[0].last_updated).toBe(entry![1]);
45
+ });
46
+
47
+ test('REST versions and product routes support filtering without guessing SDK versions', async () => {
48
+ for (const [path, product, version] of [
49
+ ['/reference/authenticating-to-composio', 'platform', '3.1'],
50
+ ['/reference/v3/authentication', 'platform', '3.0'],
51
+ ['/docs/agent-plugins', 'for-you', null],
52
+ ] as const) {
53
+ const response = await fetchPage(`${path}.md`);
54
+ expect(response.status).toBe(200);
55
+ expect(parseMetadata(await response.text())).toMatchObject({ product, version });
56
+ }
57
+ });
58
+
59
+ for (const path of ['/docs/devrel-36-missing-synthetic', '/unknown-devrel-36/page']) {
60
+ for (const negotiated of [false, true]) {
61
+ test(`${path} (${negotiated ? 'negotiated' : 'suffix'}) stays useful without false success`, async () => {
62
+ const response = await fetchPage(negotiated ? path : `${path}.md`,
63
+ negotiated ? { headers: { Accept: 'text/markdown' } } : undefined);
64
+ expect(response.status).toBe(404);
65
+ expect(response.headers.get('content-type')).toContain('text/markdown');
66
+ const text = await response.text();
67
+ expect(parseMetadata(text)).toMatchObject({
68
+ canonical_url: `https://docs.composio.dev${path}`,
69
+ markdown_url: `https://docs.composio.dev${path}.md`,
70
+ last_updated: null,
71
+ });
72
+ for (const link of ['/llms.txt', '/llms-index.txt', '/kb/search', '/docs/quickstart.md']) {
73
+ expect(text).toContain(`](${link})`);
74
+ }
75
+ expect(text).not.toContain('<!DOCTYPE html>');
76
+ });
77
+ }
78
+ }
79
+
80
+ test('a missing dated changelog keeps Markdown and 404 on its direct route', async () => {
81
+ const response = await fetchPage('/llms.mdx/docs/changelog/1900/01/01');
82
+ expect(response.status).toBe(404);
83
+ expect(response.headers.get('content-type')).toContain('text/markdown');
84
+ expect(parseMetadata(await response.text())).toMatchObject({
85
+ canonical_url: 'https://docs.composio.dev/docs/changelog/1900/01/01',
86
+ markdown_url: 'https://docs.composio.dev/docs/changelog/1900/01/01.md',
87
+ last_updated: null,
88
+ });
89
+ });
90
+});
docs/tests/integration/llm-endpoints.test.ts
-12
@@ -87,18 +87,6 @@ describe("LLM endpoints - .md pages", () => {
87
expect(contentType).toContain("text/markdown");
88
});
89
90
- test(`${name} (${path}) has a title heading`, async () => {
91
- const res = await fetchPage(path);
92
- const text = await res.text();
93
- // Should start with a markdown heading
94
- expect(text).toMatch(/^#\s+.+/);
95
- });
96
-
97
- test(`${name} (${path}) has meaningful content (> 200 chars)`, async () => {
98
- const res = await fetchPage(path);
99
- const text = await res.text();
100
- expect(text.length).toBeGreaterThan(200);
101
- });
90
}
91
});
92
docs/tests/integration/markdown-discovery.test.ts
new
+107
@@ -0,0 +1,107 @@
1
+import { describe, expect, test } from 'bun:test';
2
+import { getPublishedKbGuides } from '@/lib/kb/repository';
3
+import { getKnowledgeToolkitSummaries } from '@/lib/knowledge/catalog';
4
+import { getToolkitKnowledgeRedirect } from '@/lib/knowledge/toolkit-routing';
5
+import { fetchNoRedirect, fetchPage } from './helpers';
6
+
7
+function markdownAlternates(html: string): string[] {
8
+ return (html.match(/<link\b[^>]*>/g) ?? [])
9
+ .filter(tag => /\brel="alternate"/.test(tag) && /\btype="text\/markdown"/.test(tag))
10
+ .map(tag => tag.match(/\bhref="([^"]+)"/)![1]);
11
+}
12
+
13
+async function expectDiscoverableMarkdown(pathname: string) {
14
+ const html = await fetchPage(pathname, {
15
+ headers: { Accept: 'text/html', 'User-Agent': 'facebookexternalhit/1.1' },
16
+ timeout: 30_000,
17
+ });
18
+ expect(html.status).toBe(200);
19
+ expect(html.headers.get('content-type')).toContain('text/html');
20
+ const links = markdownAlternates(await html.text());
21
+ expect(links).toHaveLength(1);
22
+ const alternate = new URL(links[0], html.url);
23
+ expect(alternate.pathname).toBe(`${pathname}.md`);
24
+ expect(alternate.search).toBe('');
25
+
26
+ // Follow the public alternate on the test server, not the metadataBase host.
27
+ const markdown = await fetchPage(alternate.pathname, { timeout: 30_000 });
28
+ expect(markdown.status).toBe(200);
29
+ expect(markdown.headers.get('content-type')).toContain('text/markdown');
30
+}
31
+
32
+describe('HTML Markdown alternate discovery', () => {
33
+ for (const pathname of [
34
+ '/docs',
35
+ '/docs/quickstart',
36
+ '/examples',
37
+ '/examples/custom-tools',
38
+ '/reference',
39
+ '/reference/v3',
40
+ '/reference/api-reference/files/postFilesUploadRequest',
41
+ '/reference/v3/api-reference/files/postFilesUploadRequest',
42
+ '/toolkits',
43
+ '/toolkits/github',
44
+ '/toolkits/managed-auth',
45
+ '/toolkits/meta-tools',
46
+ '/toolkits/meta-tools/search_tools',
47
+ '/kb',
48
+ '/kb/search',
49
+ '/kb/toolkits',
50
+ '/kb/topic/authentication-and-connected-accounts',
51
+ ]) {
52
+ test(`${pathname} advertises a working public Markdown alternate`, async () => {
53
+ await expectDiscoverableMarkdown(pathname);
54
+ });
55
+ }
56
+
57
+ test('published KB guides advertise their own Markdown URL', async () => {
58
+ const guide = getPublishedKbGuides()[0];
59
+ expect(guide).toBeDefined();
60
+ await expectDiscoverableMarkdown(`/kb/guide/${guide.slug}`);
61
+ });
62
+
63
+ test('nonredirecting toolkit knowledge pages advertise their own Markdown URL', async () => {
64
+ const toolkit = (await getKnowledgeToolkitSummaries()).find(candidate => !getToolkitKnowledgeRedirect(candidate));
65
+ expect(toolkit).toBeDefined();
66
+ await expectDiscoverableMarkdown(`/kb/toolkit/${toolkit!.slug}`);
67
+ });
68
+
69
+ test('historical changelog HTML paths retain their canonical redirects', async () => {
70
+ for (const pathname of ['/docs/changelog', '/docs/changelog/1900/01/01']) {
71
+ const response = await fetchNoRedirect(pathname, { headers: { Accept: 'text/html' } });
72
+ expect(response.status).toBe(308);
73
+ expect(new URL(response.headers.get('location')!, response.url).pathname).toBe('/reference/changelog');
74
+ }
75
+ });
76
+
77
+ test('HTML-only reference changelog does not advertise unsupported Markdown', async () => {
78
+ const response = await fetchPage('/reference/changelog');
79
+ expect(response.status).toBe(200);
80
+ expect(markdownAlternates(await response.text())).toEqual([]);
81
+ });
82
+});
83
+
84
+describe('Negotiated HTTP cache variance', () => {
85
+ for (const accept of ['text/html', 'text/markdown', 'text/plain']) {
86
+ test(`${accept} responses retain Vary: Accept`, async () => {
87
+ const response = await fetchPage('/docs/quickstart', { headers: { Accept: accept } });
88
+ expect(response.status).toBe(200);
89
+ expect(response.headers.get('content-type')).toContain(accept === 'text/html' ? 'text/html' : 'text/markdown');
90
+ const vary = (response.headers.get('vary') ?? '').toLowerCase().split(',').map(value => value.trim());
91
+ expect(vary).toContain('accept');
92
+ if (accept === 'text/html') {
93
+ expect(vary).toContain('rsc');
94
+ expect(vary).toContain('next-router-state-tree');
95
+ }
96
+ });
97
+ }
98
+
99
+ test('HTML Fern redirects vary on Accept without changing their destination', async () => {
100
+ const response = await fetchNoRedirect('/reference/api-reference/files/post-files-upload-request', {
101
+ headers: { Accept: 'text/html' },
102
+ });
103
+ expect(response.status).toBe(301);
104
+ expect(new URL(response.headers.get('location')!, response.url).pathname).toBe('/reference/api-reference/files/postFilesUploadRequest');
105
+ expect((response.headers.get('vary') ?? '').toLowerCase().split(',').map(value => value.trim())).toContain('accept');
106
+ });
107
+});
docs/tests/static/content-freshness.test.ts
new
+161
@@ -0,0 +1,161 @@
1
+import { afterEach, describe, expect, test } from 'bun:test';
2
+import { execFileSync } from 'node:child_process';
3
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
4
+import { tmpdir } from 'node:os';
5
+import { dirname, join } from 'node:path';
6
+import { pathToFileURL } from 'node:url';
7
+import {
8
+ buildContentFreshness, getExplicitLastUpdated, normalizeLastUpdated,
9
+} from '../../lib/content-freshness-build';
10
+import { createPageLastUpdatedReader } from '../../lib/content-freshness';
11
+
12
+const directories: string[] = [];
13
+const initial = '2025-01-01T12:00:00+00:00';
14
+const changed = '2025-02-02T12:00:00+00:00';
15
+const unrelated = '2025-03-03T12:00:00+00:00';
16
+
17
+function temporaryDirectory() {
18
+ const directory = mkdtempSync(join(tmpdir(), 'docs-freshness-'));
19
+ directories.push(directory);
20
+ return directory;
21
+}
22
+function git(root: string, ...args: string[]) {
23
+ return execFileSync('git', args, { cwd: root, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] });
24
+}
25
+function put(root: string, path: string, text: string) {
26
+ mkdirSync(dirname(join(root, path)), { recursive: true });
27
+ writeFileSync(join(root, path), text);
28
+}
29
+function repository() {
30
+ const root = temporaryDirectory();
31
+ git(root, 'init', '--initial-branch=main');
32
+ git(root, 'config', 'user.name', 'Freshness tests');
33
+ git(root, 'config', 'user.email', 'freshness@example.test');
34
+ return root;
35
+}
36
+function commit(root: string, timestamp: string) {
37
+ git(root, 'add', '.');
38
+ execFileSync('git', ['commit', '-m', 'fixture'], {
39
+ cwd: root, stdio: ['ignore', 'pipe', 'pipe'],
40
+ env: { ...process.env, GIT_AUTHOR_DATE: timestamp, GIT_COMMITTER_DATE: timestamp },
41
+ });
42
+}
43
+afterEach(() => {
44
+ for (const directory of directories.splice(0)) rmSync(directory, { recursive: true, force: true });
45
+});
46
+
47
+describe('source-backed page freshness', () => {
48
+ test('tracks a changed source, not unrelated commits or filesystem write time', () => {
49
+ const root = repository();
50
+ put(root, 'content/docs/guide.mdx', 'original');
51
+ put(root, 'unrelated.txt', 'original');
52
+ commit(root, initial);
53
+ put(root, 'content/docs/guide.mdx', 'updated');
54
+ commit(root, changed);
55
+ put(root, 'unrelated.txt', 'updated');
56
+ commit(root, unrelated);
57
+ // Rewriting the same bytes changes filesystem metadata, not source history.
58
+ put(root, 'content/docs/guide.mdx', 'updated');
59
+ const result = buildContentFreshness(root, [
60
+ { pathname: '/docs/guide', sources: ['content/docs/guide.mdx'] },
61
+ { pathname: '/missing', sources: ['content/docs/missing.mdx'] },
62
+ ]);
63
+ expect(result.pages['/docs/guide']).toBe('2025-02-02T12:00:00.000Z');
64
+ expect(result.pages['/missing']).toBeUndefined();
65
+ });
66
+
67
+ test('uses real spec, toolkit and KB backing sources and aggregates their latest change', () => {
68
+ const root = repository();
69
+ for (const file of ['public/openapi.json', 'public/data/toolkits.json', 'kb/source/article.md']) {
70
+ put(root, file, 'original');
71
+ }
72
+ commit(root, initial);
73
+ put(root, 'kb/source/article.md', 'updated');
74
+ commit(root, changed);
75
+ const result = buildContentFreshness(root, [
76
+ { pathname: '/reference/operation', sources: ['public/openapi.json'] },
77
+ { pathname: '/toolkits/example', sources: ['public/data/toolkits.json'] },
78
+ { pathname: '/kb/guide/article', sources: ['kb/source/article.md'] },
79
+ { pathname: '/collection', sources: ['public/openapi.json', 'kb/source/article.md'] },
80
+ { pathname: '/unknown-collection', sources: ['kb/source/article.md', 'missing.md'] },
81
+ ]);
82
+ expect(result.pages['/reference/operation']).toBe('2025-01-01T12:00:00.000Z');
83
+ expect(result.pages['/toolkits/example']).toBe('2025-01-01T12:00:00.000Z');
84
+ expect(result.pages['/kb/guide/article']).toBe('2025-02-02T12:00:00.000Z');
85
+ expect(result.pages['/collection']).toBe('2025-02-02T12:00:00.000Z');
86
+ expect(result.pages['/unknown-collection']).toBeUndefined();
87
+ });
88
+
89
+ test('honors explicit timestamps without Git and never substitutes release or review dates', () => {
90
+ const root = temporaryDirectory();
91
+ expect(getExplicitLastUpdated('---\ndate: 2025-04-01\nwritten: April 2025\nlastVerifiedAt: 2025-05-01\n---\n')).toBeUndefined();
92
+ const explicit = getExplicitLastUpdated('---\nlast_updated: "2025-04-01T14:30:00+02:00"\n---\n');
93
+ expect(explicit).toBe('2025-04-01T14:30:00+02:00');
94
+ const result = buildContentFreshness(root, [
95
+ { pathname: '/explicit', sources: [], lastUpdated: explicit },
96
+ { pathname: '/unknown', sources: ['missing.mdx'] },
97
+ { pathname: '/date-only', sources: [], lastUpdated: '2025-04-01' },
98
+ ]);
99
+ expect(result.pages['/explicit']).toBe('2025-04-01T12:30:00.000Z');
100
+ expect(result.pages['/date-only']).toBe('2025-04-01T00:00:00.000Z');
101
+ expect(result.pages['/unknown']).toBeUndefined();
102
+ expect(normalizeLastUpdated('2025-02-30')).toBeNull();
103
+ expect(normalizeLastUpdated('2025-04-01T14:30:00')).toBeNull();
104
+ expect(() => getExplicitLastUpdated('---\nlast_updated: nonsense\n---\n')).toThrow('Invalid last_updated');
105
+ expect(() => getExplicitLastUpdated('---\nlast_updated:\ntitle: Example\n---\n')).toThrow('Invalid last_updated');
106
+ });
107
+
108
+ test('honors explicit per-source declarations in changelog-style aggregates', () => {
109
+ const root = temporaryDirectory();
110
+ const result = buildContentFreshness(root, [
111
+ { pathname: '/docs/changelog/2025/04/01', sources: ['one.mdx', 'two.mdx'] },
112
+ ], {
113
+ [join(root, 'one.mdx')]: '2025-04-01',
114
+ [join(root, 'two.mdx')]: '2025-04-05',
115
+ });
116
+ expect(result.pages['/docs/changelog/2025/04/01']).toBe('2025-04-05T00:00:00.000Z');
117
+ });
118
+
119
+ test('omits shallow-boundary dates while retaining proven changes inside shallow history', () => {
120
+ const original = repository();
121
+ put(original, 'page.mdx', 'original');
122
+ put(original, 'untouched.mdx', 'original');
123
+ commit(original, initial);
124
+ put(original, 'page.mdx', 'updated');
125
+ commit(original, changed);
126
+ put(original, 'other.txt', 'unrelated');
127
+ commit(original, unrelated);
128
+ const shallow = temporaryDirectory();
129
+ git(shallow, 'clone', '--depth=2', pathToFileURL(original).href, '.');
130
+ const definitions = [
131
+ { pathname: '/page', sources: ['page.mdx'] },
132
+ { pathname: '/untouched', sources: ['untouched.mdx'] },
133
+ { pathname: '/other', sources: ['other.txt'] },
134
+ ];
135
+ const result = buildContentFreshness(shallow, definitions);
136
+ expect(result.pages['/page']).toBeUndefined();
137
+ expect(result.pages['/untouched']).toBeUndefined();
138
+ expect(result.pages['/other']).toBe('2025-03-03T12:00:00.000Z');
139
+ git(shallow, 'fetch', '--deepen=1');
140
+ const deeper = buildContentFreshness(shallow, definitions);
141
+ expect(deeper.pages['/page']).toBe('2025-02-02T12:00:00.000Z');
142
+ });
143
+
144
+ test('persists the artifact and serves it after source and Git removal', async () => {
145
+ const root = repository();
146
+ put(root, 'page.mdx', 'original');
147
+ commit(root, initial);
148
+ const manifest = buildContentFreshness(root, [{ pathname: '/page', sources: ['page.mdx'] }]);
149
+ const artifact = join(root, 'content-freshness.json');
150
+ writeFileSync(artifact, JSON.stringify(manifest));
151
+ rmSync(join(root, '.git'), { recursive: true });
152
+ rmSync(join(root, 'page.mdx'));
153
+ const read = createPageLastUpdatedReader(artifact);
154
+ expect(await read('/page')).toBe('2025-01-01T12:00:00.000Z');
155
+ expect(await read('/missing')).toBeNull();
156
+ expect(await read('toString')).toBeNull();
157
+ expect(await createPageLastUpdatedReader(join(root, 'missing.json'))('/page')).toBeNull();
158
+ put(root, 'invalid.json', '{"version":1,"pages":{"/page":"not-a-timestamp"}}');
159
+ expect(await createPageLastUpdatedReader(join(root, 'invalid.json'))('/page')).toBeNull();
160
+ });
161
+});
docs/tests/static/markdown-negotiation.test.ts
new
+57
@@ -0,0 +1,57 @@
1
+import { describe, expect, spyOn, test } from 'bun:test';
2
+import { NextRequest, NextResponse } from 'next/server';
3
+import { proxy } from '../../proxy';
4
+
5
+function request(pathname: string, accept = 'text/html') {
6
+ return new NextRequest(`https://docs.composio.dev${pathname}`, {
7
+ headers: { Accept: accept },
8
+ });
9
+}
10
+
11
+function vary(response: Response) {
12
+ return (response.headers.get('Vary') ?? '').split(',').map(value => value.trim().toLowerCase());
13
+}
14
+
15
+describe('Markdown content negotiation', () => {
16
+ test('HTML responses vary on Accept', () => {
17
+ const response = proxy(request('/docs/quickstart'));
18
+ expect(vary(response)).toContain('accept');
19
+ });
20
+
21
+ for (const accept of ['text/markdown', 'text/plain']) {
22
+ test(`${accept} rewrites to Markdown and varies on Accept`, () => {
23
+ const response = proxy(request('/docs/quickstart', accept));
24
+ expect(vary(response)).toContain('accept');
25
+ expect(response.headers.get('x-middleware-rewrite')).toBe('https://docs.composio.dev/llms.mdx/docs/quickstart');
26
+ });
27
+ }
28
+
29
+ test('HTML-only Fern normalization redirects also vary on Accept', () => {
30
+ const pathname = '/reference/api-reference/files/post-files-upload-request';
31
+ const html = proxy(request(pathname));
32
+ expect(html.status).toBe(301);
33
+ expect(html.headers.get('Location')).toBe('https://docs.composio.dev/reference/api-reference/files/postFilesUploadRequest');
34
+ expect(vary(html)).toContain('accept');
35
+
36
+ const markdown = proxy(request(pathname, 'text/markdown'));
37
+ expect(markdown.headers.get('x-middleware-rewrite')).toBe(`https://docs.composio.dev/llms.mdx${pathname}`);
38
+ expect(vary(markdown)).toContain('accept');
39
+ });
40
+
41
+ for (const method of ['next', 'rewrite', 'redirect'] as const) {
42
+ test(`${method} preserves existing Vary dimensions`, () => {
43
+ const existing = new NextResponse(null, { headers: { Vary: 'Cookie, RSC, Next-Router-State-Tree' } });
44
+ const mocked = spyOn(NextResponse, method).mockReturnValue(existing);
45
+ try {
46
+ const pathname = method === 'redirect'
47
+ ? '/reference/api-reference/files/post-files-upload-request'
48
+ : '/docs/quickstart';
49
+ const response = proxy(request(pathname, method === 'rewrite' ? 'text/markdown' : 'text/html'));
50
+ expect(vary(response)).toEqual(['cookie', 'rsc', 'next-router-state-tree', 'accept']);
51
+ } finally {
52
+ mocked.mockRestore();
53
+ }
54
+ });
55
+ }
56
+});
57
+