@devries-cameron20 / composio Mirror · read-only
Fork 0

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

Mirrored from https://github.com/Camo67/composio.git

Synced Nov 21, 03:37 UTC

next
yml 190 lines 7.36 KB
Raw
1 name: Examples Manifest Sweep (staging)
2
3 # Runs the example entrypoints from examples-manifest.json against the
4 # STAGING backend using a dedicated, disposable examples project — never
5 # the org's main key. Non-blocking by design: scheduled + on demand, never a
6 # required PR check. Local runs and CI share the same code path
7 # (harness/run.mjs sweep).
8 on:
9 schedule:
10 - cron: '0 7 * * *' # 07:00 UTC nightly
11 workflow_dispatch:
12 inputs:
13 lang:
14 description: 'Language to sweep'
15 type: choice
16 options: [all, ts, py]
17 default: all
18 client:
19 description: 'Client under test'
20 type: choice
21 options: [baseline, candidate]
22 default: baseline
23 ids:
24 description: 'Comma-separated entry ids (empty = full manifest)'
25 type: string
26 default: ''
27 exclude_toolkits:
28 description: 'Comma-separated toolkits to exclude, including dependent entries'
29 type: string
30 default: googledrive
31 client_tarball_url:
32 description: 'Candidate TypeScript client tarball URL (required for TS candidate entries)'
33 type: string
34 default: ''
35 client_wheel_url:
36 description: 'Candidate Python client wheel URL (required for Python candidate entries)'
37 type: string
38 default: ''
39 llm:
40 description: 'LLM traffic: mock (aimock, zero model spend) or live'
41 type: choice
42 options: [mock, live]
43 default: mock
44
45 concurrency:
46 group: ${{ github.workflow }}
47 cancel-in-progress: false
48
49 permissions:
50 contents: read
51
52 env:
53 # Staging URL is not a secret; the keys are. The key must belong to a
54 # project that holds no valuable connections or data — never the org's
55 # main project. Example runs create their own resources and only ever
56 # delete what they created.
57 COMPOSIO_BASE_URL: https://staging-backend.composio.dev
58 COMPOSIO_API_KEY: ${{ secrets.COMPOSIO_API_KEY }}
59 OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
60 ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
61 # Keep unavailable coverage explicit. The harness records excluded entry ids
62 # in its summary artifact; remove this default when staging has Drive auth.
63 # The nightly excludes googledrive; a dispatch uses the exclude_toolkits
64 # input as given (its default is googledrive), so clearing it runs the full
65 # manifest.
66 EXCLUDE_TOOLKITS: ${{ github.event_name == 'schedule' && 'googledrive' || inputs.exclude_toolkits }}
67 # mock unless explicitly dispatched live; the nightly adds a small live canary.
68 LLM_MODE: ${{ inputs.llm || 'mock' }}
69 # Entries mocks cannot cover (llmMock:false) plus one representative
70 # tool-calling example per language: the drift signal mocks can't see.
71 CANARY_IDS: ts/openai/agents-api,ts/openai/agents-api-tool-router,ts/openai/chat-completions,py/tool_router/langchain_agent
72
73 jobs:
74 examples-live:
75 name: ${{ inputs.client || 'baseline' }} sweep (${{ inputs.lang || 'all' }}, excluding ${{ (github.event_name == 'schedule' && 'googledrive' || inputs.exclude_toolkits) || 'none' }})
76 environment: staging
77 runs-on: ubuntu-latest
78 timeout-minutes: 60
79
80 steps:
81 - name: Checkout Code
82 uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
83
84 - name: Setup Node.js, pnpm, Bun
85 uses: ./.github/actions/setup-node-pnpm-bun
86 with:
87 enable-turbo-cache: 'true'
88
89 - name: Setup Python and uv
90 uses: ./.github/actions/setup-python-uv
91
92 - name: Require credentials
93 run: |
94 if [ -z "$COMPOSIO_API_KEY" ]; then
95 echo "::error::COMPOSIO_API_KEY is required."
96 exit 1
97 fi
98 # LLM keys are only needed when live model traffic will run:
99 # an explicit live dispatch, or the nightly's live canary step.
100 if [ "$LLM_MODE" = "live" ] || [ "${{ github.event_name }}" = "schedule" ]; then
101 if [ -z "$OPENAI_API_KEY" ] || [ -z "$ANTHROPIC_API_KEY" ]; then
102 echo "::error::OPENAI_API_KEY and ANTHROPIC_API_KEY are required for live LLM runs."
103 exit 1
104 fi
105 fi
106
107 - name: Install Dependencies
108 run: pnpm install --frozen-lockfile
109
110 - name: Build SDK packages
111 run: pnpm run build:packages
112
113 - name: Fetch candidate client artifacts
114 if: ${{ inputs.client == 'candidate' }}
115 env:
116 CLIENT_TARBALL_URL: ${{ inputs.client_tarball_url }}
117 CLIENT_WHEEL_URL: ${{ inputs.client_wheel_url }}
118 LANG_INPUT: ${{ inputs.lang || 'all' }}
119 run: |
120 set -euo pipefail
121 mkdir -p .artifacts/candidate
122 if [ "$LANG_INPUT" = "all" ] || [ "$LANG_INPUT" = "ts" ]; then
123 if [[ "$CLIENT_TARBALL_URL" != https://* ]]; then
124 echo "::error::TS candidate entries require an HTTPS client_tarball_url"
125 exit 1
126 fi
127 curl -fsSL -o .artifacts/candidate/composio-client.tgz "$CLIENT_TARBALL_URL"
128 echo "COMPOSIO_CLIENT_TARBALL=$PWD/.artifacts/candidate/composio-client.tgz" >> "$GITHUB_ENV"
129 fi
130 if [ "$LANG_INPUT" = "all" ] || [ "$LANG_INPUT" = "py" ]; then
131 if [[ "$CLIENT_WHEEL_URL" != https://* ]]; then
132 echo "::error::Python candidate entries require an HTTPS client_wheel_url"
133 exit 1
134 fi
135 curl -fsSL -o .artifacts/candidate/composio-client.whl "$CLIENT_WHEEL_URL"
136 echo "COMPOSIO_CLIENT_WHEEL=$PWD/.artifacts/candidate/composio-client.whl" >> "$GITHUB_ENV"
137 fi
138
139 - name: Harness selftest
140 run: node harness/run.mjs selftest
141
142 - name: Verify provisioned project state
143 env:
144 IDS_INPUT: ${{ inputs.ids }}
145 LANG_INPUT: ${{ inputs.lang || 'all' }}
146 run: |
147 set -euo pipefail
148 ARGS=(--gc --exclude-toolkits "$EXCLUDE_TOOLKITS")
149 if [ "$LANG_INPUT" != "all" ]; then
150 ARGS+=(--lang "$LANG_INPUT")
151 fi
152 if [ -n "$IDS_INPUT" ]; then
153 ARGS+=(--ids "$IDS_INPUT")
154 fi
155 node scripts/examples-provision.mjs "${ARGS[@]}" > /tmp/examples-provision.env
156
157 - name: Sweep
158 env:
159 CLIENT_INPUT: ${{ inputs.client || 'baseline' }}
160 IDS_INPUT: ${{ inputs.ids }}
161 LANG_INPUT: ${{ inputs.lang || 'all' }}
162 run: |
163 set -euo pipefail
164 # Load the provisioned COMPOSIO_EXAMPLES_* ids emitted above.
165 source /tmp/examples-provision.env
166 ARGS=(--client "$CLIENT_INPUT" --llm "$LLM_MODE")
167 if [ "$LANG_INPUT" != "all" ]; then
168 ARGS+=(--lang "$LANG_INPUT")
169 fi
170 if [ -n "$IDS_INPUT" ]; then
171 ARGS+=(--ids "$IDS_INPUT")
172 fi
173 ARGS+=(--exclude-toolkits "$EXCLUDE_TOOLKITS")
174 node harness/run.mjs sweep "${ARGS[@]}"
175
176 - name: Live LLM canary
177 if: ${{ github.event_name == 'schedule' }}
178 run: |
179 set -euo pipefail
180 source /tmp/examples-provision.env
181 node harness/run.mjs sweep --client baseline --llm live --ids "$CANARY_IDS" --exclude-toolkits "$EXCLUDE_TOOLKITS"
182
183 - name: Upload sweep results and traces
184 if: ${{ always() }}
185 uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
186 with:
187 name: examples-live-results
188 path: .artifacts/examples-parity/
189 retention-days: 14
190 if-no-files-found: ignore