Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 08:30 UTC
docs: limit freshness metadata to attributable page changes
Brendan O'Leary committed
Oct 16, 2026 at 13:43 UTC
266e047b23d7f10e23e8c1b01bbd912cdca24875
3 files changed
+36
-63
docs/CONTRIBUTING.md
+3
-3
@@ -60,7 +60,7 @@ The `Docs - Rebuild KB Semantic Artifact` workflow can rebuild it for eligible s
60
61
Request a page with `Accept: text/markdown` (or `text/plain`), append `.md` or `.mdx`, or use its `/llms.mdx/` route. HTML pages with a Markdown representation advertise it with `<link rel="alternate" type="text/markdown">`. Negotiated responses include `Vary: Accept` so caches distinguish HTML from Markdown.
62
63
-Next.js `16.3.8` replaces middleware's `Vary` header in its Node HTML response template. The docs package carries a two-line dependency patch that appends Next's cache dimensions instead. Keep the live negotiation test when upgrading Next; remove the patch once upstream preserves middleware variance.
63
+Next.js `16.3.8` replaces middleware's `Vary` header in its Node HTML response template. The docs package carries a two-line dependency patch that appends Next's cache dimensions instead. The upstream regression is tracked in [Next.js issue #85852](https://github.com/vercel/next.js/issues/85852). Keep the live negotiation test when upgrading Next. Remove the patch once an unpatched production build passes that test.
64
65
Per-page Markdown starts with YAML front matter. For example, `/reference/authenticating-to-composio.md` identifies its HTML and Markdown URLs independently of the URL you requested:
66
@@ -78,9 +78,9 @@ last_updated: null
78
79
The sitemap's `lastmod` and Markdown's `last_updated` use the same build-generated source freshness data. They never use build time or filesystem modification time. Missing pages return Markdown discovery and search links with HTTP `404`; error responses don't claim a content update time.
80
81
-The prebuild, predev, and type-check steps generate `.generated/content-freshness.json` from each page's backing files. Deployed routes read that artifact without Git. Generated API and toolkit pages use the history of their source specifications or catalogs, so their freshness is file-level rather than operation-level.
81
+The prebuild, predev, and type-check steps generate `.generated/content-freshness.json` from each page's backing files. Deployed routes read that artifact without Git. Freshness is limited to authored pages and dated changelog entries. Generated API and toolkit pages, KB guides, and browse pages return `last_updated: null` and omit sitemap `lastmod`. Their shared specifications, catalogs, or manifests can change without changing the page. Add timestamps for these pages only when changes can be attributed to each page.
82
83
-You can declare `last_updated: YYYY-MM-DD` or a timezone-qualified ISO timestamp in authored front matter. A date-only value means midnight UTC; an invalid value fails generation. Otherwise, freshness uses the source file's last Git commit. Aggregate pages use the latest known update across their backing files.
83
+You can declare `last_updated: YYYY-MM-DD` or a timezone-qualified ISO timestamp in authored front matter. A date-only value means midnight UTC; an invalid value fails generation. Otherwise, freshness uses the source file's last Git commit. Dated changelog pages use the latest known update across entries for that date. A source commit records an edit, not a review or a guarantee that the guidance is current.
84
85
Docs CI fetches full history. For other CI/CD builds, provide full history (`fetch-depth: 0` with `actions/checkout`) for complete freshness coverage. A shallow checkout retains dates for changes proven within its history but omits unknown shallow-boundary dates. Source archives without history also omit unknown dates. Neither case substitutes the checkout or build time.
86
docs/scripts/build-content-freshness.ts
+10
-60
@@ -1,12 +1,9 @@
1
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
2
import { dirname, join, resolve } from 'node:path';
3
import {
4
- source, getReferenceSource, examplesSource, toolkitsSource, knowledgeBaseSource,
4
+ source, getReferenceSource, examplesSource, toolkitsSource,
5
changelogEntries, dateToChangelogUrl,
6
} from '../lib/source';
7
-import { getAllToolkitsSync } from '../lib/toolkit-data';
8
-import { getLocalKnowledgeDiscoveryPaths } from '../lib/knowledge/discovery';
9
-import { getPublishedKbGuides, getKbGuideUrl } from '../lib/kb/repository';
7
import {
8
buildContentFreshness, getExplicitLastUpdated, type FreshnessPage,
9
} from '../lib/content-freshness-build';
@@ -23,82 +20,35 @@ function add(pathname: string, sources: string[]) {
20
function authored(pathname: string, absolutePath?: string) {
21
if (!absolutePath || !existsSync(absolutePath)) return;
22
const raw = readFileSync(absolutePath, 'utf8');
23
+ // Generated files can change when an unrelated catalog entry changes.
24
+ if (raw.includes('Auto-generated')) return;
25
const explicit = getExplicitLastUpdated(raw);
26
if (explicit !== undefined) explicitSources[resolve(absolutePath)] = explicit;
27
add(pathname, [absolutePath]);
28
}
29
31
-for (const collection of [source, examplesSource, toolkitsSource, knowledgeBaseSource]) {
30
+for (const collection of [source, examplesSource, toolkitsSource]) {
31
for (const page of collection.getPages()) authored(page.url, page.absolutePath);
32
}
33
34
const reference = await getReferenceSource();
35
for (const page of reference.getPages()) {
37
- const spec = page.url.startsWith('/reference/v3/') ? 'public/openapi-v3.json'
38
- : page.url.includes('/webhook-events') ? 'public/openapi-webhooks.json'
39
- : 'public/openapi.json';
40
- const raw = page.absolutePath && existsSync(page.absolutePath)
41
- ? readFileSync(page.absolutePath, 'utf8') : null;
42
- if (raw?.includes('Auto-generated from')) {
43
- const tag = page.url.split('/').at(-1);
44
- const overview = `api-overviews/${tag}.mdx`;
45
- const specs = spec === 'public/openapi-webhooks.json' ? [spec]
46
- : ['public/openapi.json', 'public/openapi-v3.json'];
47
- add(page.url, [...specs, 'scripts/generate-api-index.ts',
48
- ...(existsSync(join(root, overview)) ? [overview] : [])]);
49
- } else if (raw !== null) {
50
- authored(page.url, page.absolutePath);
51
- } else {
52
- add(page.url, [spec, 'lib/openapi.ts']);
53
- }
36
+ authored(page.url, page.absolutePath);
37
}
38
56
-for (const toolkit of getAllToolkitsSync()) {
57
- add(`/toolkits/${toolkit.slug}`, ['public/data/toolkits.json']);
58
-}
59
-
60
-// Generated MDX is a presentation artifact. Use the actual KB articles and sources.
61
-for (const guide of getPublishedKbGuides()) {
62
- const pathname = getKbGuideUrl(guide);
63
- pages.delete(pathname);
64
- add(pathname, [
65
- 'kb/manifest.json',
66
- ...guide.sources.map(source => `kb/source/${source.sourcePath}`),
67
- ...(guide.articlePath ? [`kb/articles/${guide.articlePath}`] : []),
68
- ]);
69
-}
70
-
71
-const changelogSources: string[] = [];
39
for (const entry of changelogEntries) {
40
const file = join(root, 'content/changelog', entry.info.path);
41
const filenameDate = entry.info.path.match(/^(\d{2})-(\d{2})-(\d{2})(?:-|\.)/);
42
const date = /^\d{4}-\d{2}-\d{2}$/.test(entry.date) ? entry.date
43
: filenameDate ? `20${filenameDate[3]}-${filenameDate[1]}-${filenameDate[2]}` : null;
44
if (date) authored(dateToChangelogUrl(date), file);
78
- changelogSources.push(file);
45
}
80
-add('/docs/changelog', changelogSources);
81
-add('/', ['app/(home)/page.tsx', ...(pages.get('/docs')?.sources ?? [])]);
46
83
-// These browse pages are aggregates over the local knowledge corpus. Require all
84
-// backing timestamps rather than claim a partially-known maximum is authoritative.
85
-pages.delete('/kb');
86
-const corpusSources = [...pages.values()].flatMap(page => page.sources);
87
-for (const pathname of await getLocalKnowledgeDiscoveryPaths()) {
88
- if (pathname === '/kb/search' || pathname.startsWith('/kb/guide/')) continue;
89
- add(pathname, [...corpusSources, 'public/data/toolkits-list.json', 'lib/knowledge/taxonomy.ts']);
90
-}
91
-const toolkitSources = getAllToolkitsSync().length > 0 ? ['public/data/toolkits.json'] : [];
92
-if (pages.has('/toolkits')) add('/toolkits', toolkitSources);
93
-if (pages.has('/toolkits/managed-auth')) add('/toolkits/managed-auth', toolkitSources);
94
-for (const pathname of pages.keys()) {
95
- if (pathname === '/toolkits/meta-tools' || pathname.startsWith('/toolkits/meta-tools/')) {
96
- add(pathname, ['public/data/meta-tools.json', 'lib/meta-tool-overrides.ts']);
97
- }
98
-}
99
-if (pages.has('/examples')) {
100
- add('/examples', [...pages.values()].filter(page => page.pathname.startsWith('/examples/'))
101
- .flatMap(page => page.sources));
47
+// Browse pages include data outside their authored MDX. Omit freshness until
48
+// changes can be attributed to the page instead of an entire shared corpus.
49
+for (const pathname of ['/docs', '/examples', '/toolkits', '/toolkits/managed-auth',
50
+ '/reference', '/reference/v3']) {
51
+ pages.delete(pathname);
52
}
53
54
const manifest = buildContentFreshness(root, [...pages.values()], explicitSources);
docs/tests/integration/agent-metadata.test.ts
+23
@@ -1,5 +1,6 @@
1
import { describe, expect, test } from 'bun:test';
2
import { z } from 'zod';
3
+import { getPublishedKbGuides, getKbGuideUrl } from '@/lib/kb/repository';
4
import { fetchPage } from './helpers';
5
6
const metadataSchema = z.object({
@@ -44,6 +45,28 @@ describe('agent-readable Markdown metadata', () => {
45
expect(metadata[0].last_updated).toBe(entry![1]);
46
});
47
48
+ test('shared catalogs and manifests do not claim page freshness', async () => {
49
+ const guide = getPublishedKbGuides()[0];
50
+ expect(guide).toBeDefined();
51
+ const sitemap = await (await fetchPage('/sitemap.xml')).text();
52
+ const entries = sitemap.match(/<url>[\s\S]*?<\/url>/g) ?? [];
53
+ for (const path of [
54
+ '/reference/api-reference/files/postFilesUploadRequest',
55
+ '/reference/v3/api-reference/files/postFilesUploadRequest',
56
+ '/toolkits/github', '/toolkits/meta-tools/search_tools',
57
+ '/toolkits', '/toolkits/managed-auth', '/examples',
58
+ '/kb', '/kb/topic/authentication-and-connected-accounts',
59
+ getKbGuideUrl(guide),
60
+ ]) {
61
+ const response = await fetchPage(`${path}.md`, { timeout: 30_000 });
62
+ expect(response.status).toBe(200);
63
+ expect(parseMetadata(await response.text()).last_updated, path).toBeNull();
64
+ const entry = entries.find(item => item.includes(`<loc>https://docs.composio.dev${path}</loc>`));
65
+ expect(entry, path).toBeDefined();
66
+ expect(entry, path).not.toContain('<lastmod>');
67
+ }
68
+ });
69
+
70
test('REST versions and product routes support filtering without guessing SDK versions', async () => {
71
for (const [path, product, version] of [
72
['/reference/authenticating-to-composio', 'platform', '3.1'],