@devries-cameron20 / composio-1 / commits / 08cac9e91
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

test(cli): add nightly Apollo contract check against staging (#5168)

Adds a daily staging contract check for ten read-only Apollo operations, decoding responses with the CLI's generated `@composio/client/effect/schemas`. - Eight project-key checks cover toolkits, tools, trigger types, auth configs, and connected accounts. Two user-key checks cover organization/project discovery during CLI login; these need `COMPOSIO_USER_API_KEY`. - Effect `Config` rejects missing/empty keys and an origin outside staging/production before sending requests. Typed `Match`/`Effect.match` handles request failures. Requests run sequentially, without retries, with a 10-second timeout, and stop on 401, 403, or 429. - The nightly workflow runs at 05:00 UTC or on demand, installs dependencies with lifecycle scripts disabled, and notifies Slack on scheduled failures. The Step Summary encodes untrusted error text to prevent Markdown/HTML injection. Validation on the updated branch: - Nine regression tests pass against the real generated client over a stub HTTP transport: credential routing, schema paths, auth/rate-limit stopping, empty organizations, hostile report text, and configuration preflight. - Focused strict TypeScript compilation, Oxlint, Prettier, actionlint, and `git diff --check` pass. - Direct Bun invocation with a disallowed origin exits 1 before any request. - The broader CLI project typecheck has errors outside the changed files, including missing workspace package declarations. `pnpm audit --prod` still reports existing dependency vulnerabilities; this PR's lockfile matches `next`. Staging credentials were not available for live verification. The first manual run after merge must confirm that the existing staging secrets can read all ten cases.

Alberto Schiabel committed Nov 11, 2026 at 22:49 UTC 08cac9e911e22b6e0804ef711b6b812d8906c41d
3 files changed +359
.github/workflows/apollo-contract-nightly.yml new
+52
@@ -0,0 +1,52 @@
1 +name: Apollo Contract Nightly
2 +
3 +# Checks that staging Apollo still returns responses the CLI's client accepts:
4 +# a fixed set of reads the CLI makes, validated against the response schemas
5 +# the CLI ships with (`@composio/client/effect/schemas`). Read-only, no retries.
6 +# See ts/packages/cli/scripts/apollo-contract.ts for the cases.
7 +
8 +on:
9 + schedule:
10 + - cron: '0 5 * * *'
11 + workflow_dispatch:
12 +
13 +concurrency:
14 + group: ${{ github.workflow }}
15 + cancel-in-progress: false
16 +
17 +permissions:
18 + contents: read
19 +
20 +jobs:
21 + contract:
22 + name: Staging Apollo contract
23 + runs-on: ubuntu-latest
24 + timeout-minutes: 15
25 + steps:
26 + - name: Checkout Code
27 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
28 +
29 + - name: Setup Node.js, pnpm, Bun
30 + uses: ./.github/actions/setup-node-pnpm-bun
31 +
32 + - name: Install dependencies
33 + run: pnpm install --frozen-lockfile --ignore-scripts
34 +
35 + - name: Check Apollo responses against client schemas
36 + working-directory: ts/packages/cli
37 + env:
38 + COMPOSIO_BASE_URL: https://staging-backend.composio.dev
39 + COMPOSIO_API_KEY: ${{ secrets.COMPOSIO_API_KEY }}
40 + COMPOSIO_USER_API_KEY: ${{ secrets.COMPOSIO_USER_API_KEY }}
41 + run: bun scripts/apollo-contract.ts
42 +
43 + - name: Send Slack Notification (Failure)
44 + if: failure() && github.event_name == 'schedule'
45 + uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0
46 + with:
47 + webhook: ${{ secrets.SLACK_POD_DX_WEBHOOK_URL }}
48 + webhook-type: incoming-webhook
49 + payload: |
50 + {
51 + "text": "⚠️ Staging Apollo no longer matches the CLI's client schemas.\n*Run:* ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
52 + }
ts/packages/cli/scripts/apollo-contract.ts new
+161
@@ -0,0 +1,161 @@
1 +#!/usr/bin/env bun
2 +/**
3 + * Checks ten read-only Apollo responses against the CLI's generated client schemas.
4 + *
5 + * COMPOSIO_BASE_URL=https://staging-backend.composio.dev \
6 + * COMPOSIO_API_KEY=ak_... COMPOSIO_USER_API_KEY=uak_... \
7 + * bun scripts/apollo-contract.ts
8 + *
9 + * The project key covers toolkit, tool, trigger, auth-config and account reads.
10 + * The user key covers organization/project discovery used during CLI login.
11 + * Both keys are required. Requests run sequentially, without retries, and stop
12 + * at the first 401, 403 or 429.
13 + */
14 +import { appendFileSync } from 'node:fs';
15 +import { Config, Data, Effect, Match, Redacted } from 'effect';
16 +import { FetchHttpClient, HttpClient } from 'effect/http';
17 +import {
18 + // eslint-disable-next-line no-restricted-imports -- a standalone script needs its own client
19 + make as makeComposioClient,
20 + Auth,
21 + isAPIError,
22 + type ComposioError,
23 +} from '@composio/client/effect';
24 +import { schemaLoaders } from '@composio/client/effect/schemas';
25 +
26 +const ALLOWED_ORIGINS = [
27 + 'https://staging-backend.composio.dev',
28 + 'https://backend.composio.dev',
29 +] as const;
30 +const validated = { decode: true } as const;
31 +
32 +class ContractSetupError extends Data.TaggedError('ContractSetupError')<{
33 + readonly message: string;
34 +}> {}
35 +
36 +type Outcome = {
37 + readonly status: 'pass' | 'fail' | 'not run';
38 + readonly detail: string;
39 + readonly stop?: boolean;
40 +};
41 +
42 +const describeFailure = (error: ComposioError | ContractSetupError): Outcome =>
43 + Match.value(error).pipe(
44 + Match.tag('@composio/client/DecodeError', error => ({
45 + status: 'fail' as const,
46 + detail: `schema mismatch: ${error.issues
47 + .slice(0, 5)
48 + .map(issue => `${issue.path.join('.') || '(root)'}: ${issue.kind}`)
49 + .join('; ')}`,
50 + })),
51 + Match.tag('ContractSetupError', error => ({ status: 'fail' as const, detail: error.message })),
52 + Match.orElse(error => ({
53 + status: 'fail' as const,
54 + detail: isAPIError(error) ? `HTTP ${error.status}: ${error.message}` : error.message,
55 + stop: isAPIError(error) && [401, 403, 429].includes(error.status),
56 + }))
57 + );
58 +
59 +export const checkApolloContract = Effect.gen(function* () {
60 + const baseURL = yield* Config.Literals(ALLOWED_ORIGINS, 'COMPOSIO_BASE_URL');
61 + const projectKey = yield* Config.NonEmptyString('COMPOSIO_API_KEY').pipe(
62 + Config.map(Redacted.make)
63 + );
64 + const userKey = yield* Config.NonEmptyString('COMPOSIO_USER_API_KEY').pipe(
65 + Config.map(Redacted.make)
66 + );
67 + const httpClient = yield* HttpClient.HttpClient;
68 + const clientFor = (auth: Auth) =>
69 + makeComposioClient(
70 + { auth, baseURL, maxRetries: 0, timeout: 10_000, schemas: schemaLoaders },
71 + httpClient
72 + );
73 + const project = clientFor(Auth.project(projectKey));
74 + const user = clientFor(Auth.user(userKey));
75 + const cases: Record<string, Effect.Effect<unknown, ComposioError | ContractSetupError>> = {
76 + 'toolkits.list': project.toolkits.list({ limit: 5 }, validated),
77 + 'toolkits.retrieve hackernews': project.toolkits.retrieve('hackernews', null, validated),
78 + 'toolkits.retrieveCategories': project.toolkits.retrieveCategories(validated),
79 + 'tools.list hackernews': project.tools.list(
80 + { toolkit_slug: 'hackernews', limit: 5 },
81 + validated
82 + ),
83 + 'tools.retrieve HACKERNEWS_GET_USER': project.tools.retrieve(
84 + 'HACKERNEWS_GET_USER',
85 + null,
86 + validated
87 + ),
88 + 'triggersTypes.list': project.triggersTypes.list({ limit: 5 }, validated),
89 + 'authConfigs.list': project.authConfigs.list({ limit: 5 }, validated),
90 + 'connectedAccounts.list': project.connectedAccounts.list({ limit: 5 }, validated),
91 + 'org.list': user.org.list({ limit: 50 }, validated),
92 + 'org.project.list (first org)': Effect.gen(function* () {
93 + const { organizations } = yield* user.org.list({ limit: 1 }, validated);
94 + const [first] = organizations;
95 + if (first === undefined) {
96 + return yield* new ContractSetupError({
97 + message: 'the user key belongs to no organization',
98 + });
99 + }
100 + return yield* user.org.project.list(
101 + { limit: 50 },
102 + { ...validated, headers: { 'x-org-id': first.id } }
103 + );
104 + }),
105 + };
106 +
107 + let stopReason: string | undefined;
108 + const results = yield* Effect.forEach(
109 + Object.entries(cases),
110 + ([name, request]) =>
111 + Effect.gen(function* () {
112 + const outcome: Outcome = stopReason
113 + ? { status: 'not run', detail: stopReason }
114 + : yield* request.pipe(
115 + Effect.match({
116 + onSuccess: (): Outcome => ({ status: 'pass', detail: '' }),
117 + onFailure: describeFailure,
118 + })
119 + );
120 + if (outcome.stop)
121 + stopReason = `stopped after "${name}" failed with ${outcome.detail.split(':')[0]}`;
122 + return { name, outcome };
123 + }),
124 + { concurrency: 1 }
125 + );
126 + return { baseURL, results };
127 +});
128 +
129 +/** Encode punctuation so response text cannot become Markdown or HTML. */
130 +const escapeCell = (text: string) =>
131 + text
132 + .replace(/[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]/gu, ' ')
133 + .replace(/[&<>!"'`*_[\]{}()#+.\\|~:-]/g, char => `&#${char.charCodeAt(0)};`);
134 +
135 +export const renderReport = ({ baseURL, results }: Effect.Success<typeof checkApolloContract>) => {
136 + const passed = results.filter(result => result.outcome.status === 'pass').length;
137 + return [
138 + `### Apollo contract: ${baseURL}`,
139 + '',
140 + `${passed}/${results.length} cases passed against \`@composio/client\` response schemas.`,
141 + '',
142 + '| Case | Result | Detail |',
143 + '| --- | --- | --- |',
144 + ...results.map(
145 + ({ name, outcome }) => `| ${name} | ${outcome.status} | ${escapeCell(outcome.detail)} |`
146 + ),
147 + '',
148 + ].join('\n');
149 +};
150 +
151 +if (import.meta.main) {
152 + const result = await Effect.runPromise(
153 + checkApolloContract.pipe(Effect.provide(FetchHttpClient.layer))
154 + );
155 + const report = renderReport(result);
156 + // eslint-disable-next-line no-console -- the report is this script's output
157 + console.log(report);
158 + if (process.env.GITHUB_STEP_SUMMARY)
159 + appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${report}\n`);
160 + process.exitCode = result.results.every(result => result.outcome.status === 'pass') ? 0 : 1;
161 +}
ts/packages/cli/test/scripts/apollo-contract.test.ts new
+146
@@ -0,0 +1,146 @@
1 +import { describe, expect, it } from '@effect/vitest';
2 +import { ConfigProvider, Effect } from 'effect';
3 +import { HttpClient, HttpClientResponse } from 'effect/http';
4 +import { checkApolloContract, renderReport } from '../../scripts/apollo-contract';
5 +
6 +const config = {
7 + COMPOSIO_BASE_URL: 'https://staging-backend.composio.dev',
8 + COMPOSIO_API_KEY: 'test-project-key',
9 + COMPOSIO_USER_API_KEY: 'test-user-key',
10 +};
11 +
12 +describe('Apollo contract check', () => {
13 + it.effect.each([401, 403, 429])('stops after HTTP %s without retrying', status =>
14 + Effect.gen(function* () {
15 + const requests: string[] = [];
16 + const client = HttpClient.make(request => {
17 + requests.push(request.url);
18 + return Effect.succeed(
19 + HttpClientResponse.fromWeb(
20 + request,
21 + new Response(JSON.stringify({ error: { message: 'rejected' } }), { status })
22 + )
23 + );
24 + });
25 + const result = yield* checkApolloContract.pipe(
26 + Effect.provideService(HttpClient.HttpClient, client),
27 + Effect.provideService(ConfigProvider.ConfigProvider, ConfigProvider.fromEnvRecord(config))
28 + );
29 + expect(requests).toHaveLength(1);
30 + expect(result.results).toHaveLength(10);
31 + expect(result.results[0]?.outcome).toMatchObject({
32 + status: 'fail',
33 + detail: expect.stringContaining(`HTTP ${status}:`),
34 + });
35 + expect(result.results[0]?.outcome.detail).toContain('rejected');
36 + expect(result.results.slice(1).every(result => result.outcome.status === 'not run')).toBe(
37 + true
38 + );
39 + })
40 + );
41 +
42 + it.effect('reports schema paths and continues using the correct credentials', () =>
43 + Effect.gen(function* () {
44 + const requests: Array<{ url: string; headers: Readonly<Record<string, string>> }> = [];
45 + const client = HttpClient.make(request => {
46 + requests.push({ url: request.url, headers: request.headers });
47 + return Effect.succeed(HttpClientResponse.fromWeb(request, Response.json({})));
48 + });
49 + const result = yield* checkApolloContract.pipe(
50 + Effect.provideService(HttpClient.HttpClient, client),
51 + Effect.provideService(ConfigProvider.ConfigProvider, ConfigProvider.fromEnvRecord(config))
52 + );
53 + expect(requests).toHaveLength(10);
54 + expect(result.results.every(result => result.outcome.status === 'fail')).toBe(true);
55 + expect(result.results[0]?.outcome.detail).toContain('schema mismatch: items:');
56 + expect(
57 + requests
58 + .slice(0, 8)
59 + .every(request => request.headers['x-api-key'] === config.COMPOSIO_API_KEY)
60 + ).toBe(true);
61 + expect(
62 + requests
63 + .slice(8)
64 + .every(request => request.headers['x-user-api-key'] === config.COMPOSIO_USER_API_KEY)
65 + ).toBe(true);
66 + })
67 + );
68 +
69 + it.effect('reports an empty organization list as a failed project check', () =>
70 + Effect.gen(function* () {
71 + const client = HttpClient.make(request =>
72 + Effect.succeed(
73 + HttpClientResponse.fromWeb(
74 + request,
75 + Response.json(
76 + request.url.includes('/org/list')
77 + ? { organizations: [], total_pages: 1, current_page: 1, total_items: 0 }
78 + : {}
79 + )
80 + )
81 + )
82 + );
83 + const result = yield* checkApolloContract.pipe(
84 + Effect.provideService(HttpClient.HttpClient, client),
85 + Effect.provideService(ConfigProvider.ConfigProvider, ConfigProvider.fromEnvRecord(config))
86 + );
87 + expect(result.results[8]?.outcome.status).toBe('pass');
88 + expect(result.results[9]?.outcome).toMatchObject({
89 + status: 'fail',
90 + detail: 'the user key belongs to no organization',
91 + });
92 + expect(renderReport(result)).toContain('1/10 cases passed');
93 + })
94 + );
95 +
96 + it.effect('keeps hostile error text inside its report cell', () =>
97 + Effect.gen(function* () {
98 + const message =
99 + 'boom\r| forged | pass |\n<img src=x> [click](https://evil.test) `code` \\| **bold** &amp;';
100 + const client = HttpClient.make(request =>
101 + Effect.succeed(
102 + HttpClientResponse.fromWeb(
103 + request,
104 + Response.json({ error: { message } }, { status: 401 })
105 + )
106 + )
107 + );
108 + const result = yield* checkApolloContract.pipe(
109 + Effect.provideService(HttpClient.HttpClient, client),
110 + Effect.provideService(ConfigProvider.ConfigProvider, ConfigProvider.fromEnvRecord(config))
111 + );
112 + const report = renderReport(result);
113 + expect(report).not.toContain('\r');
114 + expect(report).not.toContain('<img');
115 + expect(report).not.toContain('[click]');
116 + expect(report).not.toContain('`code`');
117 + expect(report).not.toContain('**bold**');
118 + const rows = report.split('\n').filter(line => line.startsWith('|'));
119 + expect(rows).toHaveLength(12);
120 + expect(rows.every(row => row.split('|').length === 5)).toBe(true);
121 + expect(report).toContain('&#124; forged &#124; pass &#124;');
122 + expect(report).toContain('&#38;amp;');
123 + })
124 + );
125 +
126 + it.effect.each([
127 + { ...config, COMPOSIO_BASE_URL: 'https://untrusted.example' },
128 + { ...config, COMPOSIO_API_KEY: '' },
129 + { ...config, COMPOSIO_USER_API_KEY: undefined },
130 + ])('rejects invalid configuration before sending credentials', env =>
131 + Effect.gen(function* () {
132 + let requests = 0;
133 + const client = HttpClient.make(request => {
134 + requests++;
135 + return Effect.succeed(HttpClientResponse.fromWeb(request, Response.json({})));
136 + });
137 + const error = yield* checkApolloContract.pipe(
138 + Effect.provideService(HttpClient.HttpClient, client),
139 + Effect.provideService(ConfigProvider.ConfigProvider, ConfigProvider.fromEnvRecord(env)),
140 + Effect.flip
141 + );
142 + expect(error).toBeDefined();
143 + expect(requests).toBe(0);
144 + })
145 + );
146 +});