@devries-cameron20 / composio-1 / commits / 2ad6c8742
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

fix(docs): make semantic artifact freshness advisory in PR checks

Brendan O'Leary committed Sep 10, 2026 at 23:12 UTC 2ad6c8742f077b5e003059d237d641f107d7f7c4
6 files changed +88 -22
.github/workflows/docs-tests.yml
+1 -1
@@ -47,7 +47,7 @@ jobs:
47 run: bun run test
48
49 - name: Check KB semantic artifact
50 - run: bun run check:kb-semantic
50 + run: bun run check:kb-semantic --allow-stale
51
52 - name: Build
53 run: bun run build
docs/CONTRIBUTING.md
+4 -2
@@ -44,13 +44,15 @@ bun run test
44 bun run lint
45 bun run lint:links
46 bun run types:check
47 -bun run check:kb-semantic
47 +bun run check:kb-semantic --allow-stale
48 bun run build
49 ```
50
51 To check the built site's endpoints, run `bun run start` in one terminal and `bun run test:integration` in another. The integration tests use `http://localhost:3000` by default. Set `TEST_BASE_URL` if your server uses another address.
52
53 -A stale KB semantic artifact can follow a content change. The `Docs - Rebuild KB Semantic Artifact` workflow can rebuild it for eligible same-repository PRs after `Docs - Tests` runs. Check that workflow's result and rerun failed checks against the updated commit. If you rebuild locally with `bun run build:kb-semantic`, use an authorized `OPENAI_API_KEY`: the generator sends the indexed docs content to the embeddings service. Never edit embedding values or content hashes by hand.
53 +The semantic artifact covers both regular docs and KB articles, so edits to either can make it stale. PR checks use `--allow-stale` to warn about freshness while still rejecting missing or corrupt artifacts. Search falls back to keyword results until the artifact is rebuilt.
54 +
55 +The `Docs - Rebuild KB Semantic Artifact` workflow can rebuild it for eligible same-repository PRs after `Docs - Tests` runs. The scheduled support-knowledge refresh also proposes stale-artifact repairs. If you rebuild locally with `bun run build:kb-semantic`, use an authorized `OPENAI_API_KEY`: the generator sends the indexed docs content to the embeddings service. Run `bun run check:kb-semantic` without `--allow-stale` to verify a rebuild. Never edit embedding values or content hashes by hand.
56
57 ## Request review and verify publication
58
docs/lib/knowledge/semantic-artifact.ts
+21 -14
@@ -136,6 +136,9 @@ export function decodeVectors(
136 return values;
137 }
138
139 +/** A valid artifact that needs to be rebuilt for the current corpus. */
140 +export class SemanticArtifactStaleError extends Error {}
141 +
142 export function validateSemanticArtifact(
143 artifact: KnowledgeSemanticArtifact,
144 expected: {
@@ -156,13 +159,6 @@ export function validateSemanticArtifact(
159 if (artifact.source.repository !== 'ComposioHQ/support-knowledge') {
160 throw new Error('Semantic artifact source repository mismatch');
161 }
159 - const supportKnowledgeCommit = expected.supportKnowledgeCommit ?? expected.sourceCommit;
160 - if (!supportKnowledgeCommit || artifact.source.supportKnowledgeCommit !== supportKnowledgeCommit) {
161 - throw new Error('Semantic artifact source commit mismatch');
162 - }
163 - if (expected.docsContentHash && artifact.source.docsContentHash !== expected.docsContentHash) {
164 - throw new Error('Semantic artifact docs content hash mismatch');
165 - }
162 if (artifact.records.length === 0) throw new Error('Semantic artifact has no records');
163
164 const seen = new Set<string>();
@@ -175,13 +171,6 @@ export function validateSemanticArtifact(
171 throw new Error(`Semantic artifact has duplicate object ID: ${record.objectID}`);
172 }
173 seen.add(record.objectID);
178 - const expectedHash = expected.contentHashes?.get(record.objectID);
179 - if (expected.contentHashes && expectedHash !== record.contentHash) {
180 - throw new Error(`Semantic artifact content hash mismatch for ${record.objectID}`);
181 - }
182 - }
183 - if (expected.contentHashes && expected.contentHashes.size !== artifact.records.length) {
184 - throw new Error('Semantic artifact content hash record count mismatch');
174 }
175
176 const vectors = decodeVectors(artifact.vectorsBase64, artifact.records.length, artifact.dimensions);
@@ -197,6 +186,24 @@ export function validateSemanticArtifact(
186 throw new Error(`Semantic vector for ${artifact.records[row]?.objectID} is not normalized`);
187 }
188 }
189 + // Check integrity first so advisory freshness checks cannot hide corrupt artifacts.
190 + const supportKnowledgeCommit = expected.supportKnowledgeCommit ?? expected.sourceCommit;
191 + if (!supportKnowledgeCommit || artifact.source.supportKnowledgeCommit !== supportKnowledgeCommit) {
192 + throw new SemanticArtifactStaleError('Semantic artifact source commit mismatch');
193 + }
194 + if (expected.docsContentHash && artifact.source.docsContentHash !== expected.docsContentHash) {
195 + throw new SemanticArtifactStaleError('Semantic artifact docs content hash mismatch');
196 + }
197 + if (expected.contentHashes) {
198 + for (const record of artifact.records) {
199 + if (expected.contentHashes.get(record.objectID) !== record.contentHash) {
200 + throw new SemanticArtifactStaleError(`Semantic artifact content hash mismatch for ${record.objectID}`);
201 + }
202 + }
203 + if (expected.contentHashes.size !== artifact.records.length) {
204 + throw new SemanticArtifactStaleError('Semantic artifact content hash record count mismatch');
205 + }
206 + }
207 return artifact;
208 }
209
docs/scripts/build-kb-semantic-index.ts
+15 -5
@@ -6,6 +6,7 @@ import {
6 buildSemanticArtifact,
7 docsContentHashFromRecords,
8 semanticRecordFromSearchRecord,
9 + SemanticArtifactStaleError,
10 validateSemanticArtifact,
11 type KnowledgeSemanticArtifact,
12 } from '@/lib/knowledge/semantic-artifact';
@@ -37,11 +38,20 @@ const docsContentHash = docsContentHashFromRecords(
38 if (process.argv.includes('--check')) {
39 if (!existsSync(artifactPath)) throw new Error('KB semantic artifact is missing');
40 const artifact = JSON.parse(readFileSync(artifactPath, 'utf8')) as KnowledgeSemanticArtifact;
40 - validateSemanticArtifact(artifact, {
41 - supportKnowledgeCommit: manifest.source.commit,
42 - docsContentHash,
43 - contentHashes,
44 - });
41 + try {
42 + validateSemanticArtifact(artifact, {
43 + supportKnowledgeCommit: manifest.source.commit,
44 + docsContentHash,
45 + contentHashes,
46 + });
47 + } catch (error) {
48 + if (!(error instanceof SemanticArtifactStaleError)) throw error;
49 + const message = `${error.message}. Search falls back to keyword results until the artifact is rebuilt. ` +
50 + 'Run bun run build:kb-semantic with OPENAI_API_KEY, or use the Docs - Rebuild KB Semantic Artifact workflow.';
51 + if (!process.argv.includes('--allow-stale')) throw new Error(message);
52 + console.warn(`${process.env.GITHUB_ACTIONS === 'true' ? '::warning::' : 'Warning: '}${message}`);
53 + process.exit(0);
54 + }
55 console.log(`KB semantic artifact is current: ${artifact.records.length} records.`);
56 process.exit(0);
57 }
docs/tests/static/kb-semantic-artifact.test.ts
+38
@@ -11,6 +11,7 @@ import {
11 encodeVectors,
12 rankSemanticCandidates,
13 semanticRecordFromSearchRecord,
14 + SemanticArtifactStaleError,
15 validateSemanticArtifact,
16 type KnowledgeSemanticArtifact,
17 type KnowledgeSemanticRecord,
@@ -163,6 +164,43 @@ Content: Provider tokens are redacted from connected-account responses.`);
164 });
165
166 describe('KB semantic artifact', () => {
167 + test.each([
168 + { supportKnowledgeCommit: 'new-commit' },
169 + { docsContentHash: 'new-docs-hash' },
170 + { contentHashes: new Map([['exact-x', 'changed-hash']]) },
171 + { contentHashes: new Map([
172 + ['exact-x', 'hash-x'], ['diagonal', 'hash-d'], ['exact-y', 'hash-y'], ['new', 'hash-new'],
173 + ]) },
174 + ])('classifies corpus drift as stale: %j', drift => {
175 + expect(() => validateSemanticArtifact(artifact(), {
176 + dimensions: 2,
177 + supportKnowledgeCommit: 'abc1234',
178 + ...drift,
179 + })).toThrow(SemanticArtifactStaleError);
180 + });
181 +
182 + test.each([
183 + { model: 'another-model' },
184 + { vectorsBase64: encodeVectors([[2, 0], [0, 1], [0, 1]]) },
185 + { vectorsBase64: encodeVectors([[1, 0]]) },
186 + { records: [] },
187 + { records: [semanticRecord('duplicate', 'A', 'a'), semanticRecord('duplicate', 'B', 'b')] },
188 + { records: [{ ...semanticRecord('private', 'Private', 'hash'), visibility: 'private' }] },
189 + ])('rejects corruption even when the corpus is stale: %j', corruption => {
190 + let caught: unknown;
191 + try {
192 + validateSemanticArtifact({ ...artifact(), ...corruption } as KnowledgeSemanticArtifact, {
193 + dimensions: 2,
194 + supportKnowledgeCommit: 'new-commit',
195 + docsContentHash: 'new-docs-hash',
196 + });
197 + } catch (error) {
198 + caught = error;
199 + }
200 + expect(caught).toBeInstanceOf(Error);
201 + expect(caught).not.toBeInstanceOf(SemanticArtifactStaleError);
202 + });
203 +
204 test('round-trips row-major float32 vectors', () => {
205 const encoded = encodeVectors([[1, 0], [0.25, -0.5]]);
206 expect([...decodeVectors(encoded, 2, 2)]).toEqual([1, 0, 0.25, -0.5]);
docs/tests/static/kb-update-workflow.test.ts
+9
@@ -55,6 +55,13 @@ function workflowSteps(path: string, job: string): WorkflowStep[] {
55 }
56
57 describe('support knowledge refresh workflow', () => {
58 + test('allows stale embeddings in PR checks without ignoring artifact errors', () => {
59 + const check = workflowSteps(docsTestsWorkflowPath, 'test')
60 + .find(step => step.name === 'Check KB semantic artifact');
61 + expect(check?.run).toBe('bun run check:kb-semantic --allow-stale');
62 + expect(check?.['continue-on-error']).not.toBe(true);
63 + });
64 +
65 test('rebuilds stale semantic artifacts from default-branch code only for trusted current pull requests', () => {
66 const workflow = existsSync(semanticRefreshWorkflowPath)
67 ? Bun.YAML.parse(readFileSync(semanticRefreshWorkflowPath, 'utf8')) as {
@@ -190,9 +197,11 @@ echo "head_ref=$live_head_ref" >> "$GITHUB_OUTPUT"`);
197 expect(checkout?.with?.ref).toBe('${{ needs.authorize.outputs.head_sha }}');
198 expect(checkout?.with?.['persist-credentials']).toBe(false);
199 expect(freshness?.['continue-on-error']).toBe(true);
200 + expect(freshness?.run).toBe('bun run check:kb-semantic');
201 expect(build?.if).toBe("steps.freshness.outcome == 'failure'");
202 expect(build?.env).toEqual({ OPENAI_API_KEY: '${{ secrets.OPENAI_API_KEY }}' });
203 expect(verify?.if).toBe("steps.freshness.outcome == 'failure'");
204 + expect(verify?.run).toBe('bun run check:kb-semantic');
205 expect(writeToken?.if).toBe("steps.freshness.outcome == 'failure'");
206 expect(writeToken?.with).toEqual({
207 'client-id': '${{ vars.RELEASE_BOT_CLIENT_ID }}',