Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
Mirrored from https://github.com/Camo67/composio.git
Synced Nov 21, 03:37 UTC
| 1 | name: Examples Manifest Sweep (staging) |
| 2 | |
| 3 | # Runs the example entrypoints from examples-manifest.json against the |
| 4 | # STAGING backend using a dedicated, disposable examples project — never |
| 5 | # the org's main key. Non-blocking by design: scheduled + on demand, never a |
| 6 | # required PR check. Local runs and CI share the same code path |
| 7 | # (harness/run.mjs sweep). |
| 8 | on: |
| 9 | schedule: |
| 10 | - cron: '0 7 * * *' # 07:00 UTC nightly |
| 11 | workflow_dispatch: |
| 12 | inputs: |
| 13 | lang: |
| 14 | description: 'Language to sweep' |
| 15 | type: choice |
| 16 | options: [all, ts, py] |
| 17 | default: all |
| 18 | client: |
| 19 | description: 'Client under test' |
| 20 | type: choice |
| 21 | options: [baseline, candidate] |
| 22 | default: baseline |
| 23 | ids: |
| 24 | description: 'Comma-separated entry ids (empty = full manifest)' |
| 25 | type: string |
| 26 | default: '' |
| 27 | exclude_toolkits: |
| 28 | description: 'Comma-separated toolkits to exclude, including dependent entries' |
| 29 | type: string |
| 30 | default: googledrive |
| 31 | client_tarball_url: |
| 32 | description: 'Candidate TypeScript client tarball URL (required for TS candidate entries)' |
| 33 | type: string |
| 34 | default: '' |
| 35 | client_wheel_url: |
| 36 | description: 'Candidate Python client wheel URL (required for Python candidate entries)' |
| 37 | type: string |
| 38 | default: '' |
| 39 | llm: |
| 40 | description: 'LLM traffic: mock (aimock, zero model spend) or live' |
| 41 | type: choice |
| 42 | options: [mock, live] |
| 43 | default: mock |
| 44 | |
| 45 | concurrency: |
| 46 | group: ${{ github.workflow }} |
| 47 | cancel-in-progress: false |
| 48 | |
| 49 | permissions: |
| 50 | contents: read |
| 51 | |
| 52 | env: |
| 53 | # Staging URL is not a secret; the keys are. The key must belong to a |
| 54 | # project that holds no valuable connections or data — never the org's |
| 55 | # main project. Example runs create their own resources and only ever |
| 56 | # delete what they created. |
| 57 | COMPOSIO_BASE_URL: https://staging-backend.composio.dev |
| 58 | COMPOSIO_API_KEY: ${{ secrets.COMPOSIO_API_KEY }} |
| 59 | OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} |
| 60 | ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} |
| 61 | # Keep unavailable coverage explicit. The harness records excluded entry ids |
| 62 | # in its summary artifact; remove this default when staging has Drive auth. |
| 63 | # The nightly excludes googledrive; a dispatch uses the exclude_toolkits |
| 64 | # input as given (its default is googledrive), so clearing it runs the full |
| 65 | # manifest. |
| 66 | EXCLUDE_TOOLKITS: ${{ github.event_name == 'schedule' && 'googledrive' || inputs.exclude_toolkits }} |
| 67 | # mock unless explicitly dispatched live; the nightly adds a small live canary. |
| 68 | LLM_MODE: ${{ inputs.llm || 'mock' }} |
| 69 | # Entries mocks cannot cover (llmMock:false) plus one representative |
| 70 | # tool-calling example per language: the drift signal mocks can't see. |
| 71 | CANARY_IDS: ts/openai/agents-api,ts/openai/agents-api-tool-router,ts/openai/chat-completions,py/tool_router/langchain_agent |
| 72 | |
| 73 | jobs: |
| 74 | examples-live: |
| 75 | name: ${{ inputs.client || 'baseline' }} sweep (${{ inputs.lang || 'all' }}, excluding ${{ (github.event_name == 'schedule' && 'googledrive' || inputs.exclude_toolkits) || 'none' }}) |
| 76 | environment: staging |
| 77 | runs-on: ubuntu-latest |
| 78 | timeout-minutes: 60 |
| 79 | |
| 80 | steps: |
| 81 | - name: Checkout Code |
| 82 | uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 |
| 83 | |
| 84 | - name: Setup Node.js, pnpm, Bun |
| 85 | uses: ./.github/actions/setup-node-pnpm-bun |
| 86 | with: |
| 87 | enable-turbo-cache: 'true' |
| 88 | |
| 89 | - name: Setup Python and uv |
| 90 | uses: ./.github/actions/setup-python-uv |
| 91 | |
| 92 | - name: Require credentials |
| 93 | run: | |
| 94 | if [ -z "$COMPOSIO_API_KEY" ]; then |
| 95 | echo "::error::COMPOSIO_API_KEY is required." |
| 96 | exit 1 |
| 97 | fi |
| 98 | # LLM keys are only needed when live model traffic will run: |
| 99 | # an explicit live dispatch, or the nightly's live canary step. |
| 100 | if [ "$LLM_MODE" = "live" ] || [ "${{ github.event_name }}" = "schedule" ]; then |
| 101 | if [ -z "$OPENAI_API_KEY" ] || [ -z "$ANTHROPIC_API_KEY" ]; then |
| 102 | echo "::error::OPENAI_API_KEY and ANTHROPIC_API_KEY are required for live LLM runs." |
| 103 | exit 1 |
| 104 | fi |
| 105 | fi |
| 106 | |
| 107 | - name: Install Dependencies |
| 108 | run: pnpm install --frozen-lockfile |
| 109 | |
| 110 | - name: Build SDK packages |
| 111 | run: pnpm run build:packages |
| 112 | |
| 113 | - name: Fetch candidate client artifacts |
| 114 | if: ${{ inputs.client == 'candidate' }} |
| 115 | env: |
| 116 | CLIENT_TARBALL_URL: ${{ inputs.client_tarball_url }} |
| 117 | CLIENT_WHEEL_URL: ${{ inputs.client_wheel_url }} |
| 118 | LANG_INPUT: ${{ inputs.lang || 'all' }} |
| 119 | run: | |
| 120 | set -euo pipefail |
| 121 | mkdir -p .artifacts/candidate |
| 122 | if [ "$LANG_INPUT" = "all" ] || [ "$LANG_INPUT" = "ts" ]; then |
| 123 | if [[ "$CLIENT_TARBALL_URL" != https://* ]]; then |
| 124 | echo "::error::TS candidate entries require an HTTPS client_tarball_url" |
| 125 | exit 1 |
| 126 | fi |
| 127 | curl -fsSL -o .artifacts/candidate/composio-client.tgz "$CLIENT_TARBALL_URL" |
| 128 | echo "COMPOSIO_CLIENT_TARBALL=$PWD/.artifacts/candidate/composio-client.tgz" >> "$GITHUB_ENV" |
| 129 | fi |
| 130 | if [ "$LANG_INPUT" = "all" ] || [ "$LANG_INPUT" = "py" ]; then |
| 131 | if [[ "$CLIENT_WHEEL_URL" != https://* ]]; then |
| 132 | echo "::error::Python candidate entries require an HTTPS client_wheel_url" |
| 133 | exit 1 |
| 134 | fi |
| 135 | curl -fsSL -o .artifacts/candidate/composio-client.whl "$CLIENT_WHEEL_URL" |
| 136 | echo "COMPOSIO_CLIENT_WHEEL=$PWD/.artifacts/candidate/composio-client.whl" >> "$GITHUB_ENV" |
| 137 | fi |
| 138 | |
| 139 | - name: Harness selftest |
| 140 | run: node harness/run.mjs selftest |
| 141 | |
| 142 | - name: Verify provisioned project state |
| 143 | env: |
| 144 | IDS_INPUT: ${{ inputs.ids }} |
| 145 | LANG_INPUT: ${{ inputs.lang || 'all' }} |
| 146 | run: | |
| 147 | set -euo pipefail |
| 148 | ARGS=(--gc --exclude-toolkits "$EXCLUDE_TOOLKITS") |
| 149 | if [ "$LANG_INPUT" != "all" ]; then |
| 150 | ARGS+=(--lang "$LANG_INPUT") |
| 151 | fi |
| 152 | if [ -n "$IDS_INPUT" ]; then |
| 153 | ARGS+=(--ids "$IDS_INPUT") |
| 154 | fi |
| 155 | node scripts/examples-provision.mjs "${ARGS[@]}" > /tmp/examples-provision.env |
| 156 | |
| 157 | - name: Sweep |
| 158 | env: |
| 159 | CLIENT_INPUT: ${{ inputs.client || 'baseline' }} |
| 160 | IDS_INPUT: ${{ inputs.ids }} |
| 161 | LANG_INPUT: ${{ inputs.lang || 'all' }} |
| 162 | run: | |
| 163 | set -euo pipefail |
| 164 | # Load the provisioned COMPOSIO_EXAMPLES_* ids emitted above. |
| 165 | source /tmp/examples-provision.env |
| 166 | ARGS=(--client "$CLIENT_INPUT" --llm "$LLM_MODE") |
| 167 | if [ "$LANG_INPUT" != "all" ]; then |
| 168 | ARGS+=(--lang "$LANG_INPUT") |
| 169 | fi |
| 170 | if [ -n "$IDS_INPUT" ]; then |
| 171 | ARGS+=(--ids "$IDS_INPUT") |
| 172 | fi |
| 173 | ARGS+=(--exclude-toolkits "$EXCLUDE_TOOLKITS") |
| 174 | node harness/run.mjs sweep "${ARGS[@]}" |
| 175 | |
| 176 | - name: Live LLM canary |
| 177 | if: ${{ github.event_name == 'schedule' }} |
| 178 | run: | |
| 179 | set -euo pipefail |
| 180 | source /tmp/examples-provision.env |
| 181 | node harness/run.mjs sweep --client baseline --llm live --ids "$CANARY_IDS" --exclude-toolkits "$EXCLUDE_TOOLKITS" |
| 182 | |
| 183 | - name: Upload sweep results and traces |
| 184 | if: ${{ always() }} |
| 185 | uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 |
| 186 | with: |
| 187 | name: examples-live-results |
| 188 | path: .artifacts/examples-parity/ |
| 189 | retention-days: 14 |
| 190 | if-no-files-found: ignore |