Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

next
yml 150 lines 6.8 KB
Raw
1 name: Prewarm E2E image caches
2
3 # The e2e Dockerfiles resolve their base images from public.ecr.aws, and on a
4 # cold GitHub Actions buildx cache that resolution hits anonymous ECR rate
5 # limits (HTTP 429) before a single test runs. The PR jobs that build the
6 # composio-e2e-* images only pass when their cache scope is still warm or the
7 # registry pull succeeds, so this workflow rebuilds every scope those jobs
8 # restore from (e2e-install-*, e2e-node-*, e2e-cli) and leaves the layers in
9 # the GitHub Actions cache. Scheduled runs only fire on the default branch
10 # (next); dispatch it by hand after changing a toolchain pin or an e2e
11 # Dockerfile.
12
13 on:
14 schedule:
15 - cron: '0 4 * * *'
16 workflow_dispatch:
17
18 concurrency:
19 group: prewarm-e2e-images
20 cancel-in-progress: false
21
22 permissions:
23 contents: read
24
25 jobs:
26 toolchain-versions:
27 name: Read toolchain versions and digests
28 runs-on: ubuntu-latest
29 outputs:
30 matrix: ${{ steps.versions.outputs.matrix }}
31 steps:
32 - name: Checkout
33 uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
34
35 - name: Read versions
36 id: versions
37 run: |
38 set -euo pipefail
39 toolchain_file=toolchain-versions.json
40
41 if ! jq -e '.node | type == "array" and length > 0' "$toolchain_file" >/dev/null; then
42 echo "::error::toolchain-versions.json must contain a non-empty \"node\" array."
43 exit 1
44 fi
45
46 # Prewarm pins base images by digest so a warmed cache is only reused
47 # for exactly the image the e2e jobs resolve. Without the map the
48 # node and cli entries below could drift from the digests those jobs
49 # build with.
50 if ! jq -e '.nodeDigests | type == "object"' "$toolchain_file" >/dev/null; then
51 echo "::error::toolchain-versions.json is missing the \"nodeDigests\" map ({\"<node-version>\": \"sha256:...\"}). Add it next to the \"node\" array so the e2e-node-* and e2e-cli caches can be warmed."
52 exit 1
53 fi
54
55 missing_digests=$(jq -r '. as $root | [$root.node[] | select(($root.nodeDigests[.] // "") == "")] | join(", ")' "$toolchain_file")
56 if [[ -n "$missing_digests" ]]; then
57 echo "::error::toolchain-versions.json is missing \"nodeDigests\" entries for: ${missing_digests}. Every version in \"node\" needs a \"sha256:...\" digest."
58 exit 1
59 fi
60
61 # Install image variants, mirroring the CLI Install E2E matrix. Its
62 # Dockerfile is self-contained, so these entries run the full build
63 # (cache-only would skip the only build that saves their scope).
64 install_matrix=$(jq -cn '
65 ["bash", "zsh", "fish"] | map({
66 "name": "install image (\(.))",
67 "dockerfile": "ts/e2e-tests/_utils/Dockerfile.install",
68 "tag": "composio-e2e-install:\(.)",
69 "cache-scope": "e2e-install-\(.)",
70 "cache-target": "",
71 "cache-only": "false",
72 "build-args": "INSTALL_SHELL=\(.)"
73 })')
74
75 # Node toolchain images, one per version in toolchain-versions.json.
76 # cache-only matters here and for the cli entry below: the later
77 # stages COPY host-built dist under ts/packages, which a bare checkout
78 # cannot provide, so these entries only refresh the toolchain-stage
79 # cache that the CI jobs' final builds consume.
80 node_matrix=$(jq -c '
81 . as $root
82 | [$root.node[] as $version | {
83 "name": "node toolchain (\($version))",
84 "dockerfile": "ts/e2e-tests/_utils/Dockerfile.node",
85 "tag": "composio-e2e-node:\($version)",
86 "cache-scope": "e2e-node-\($version)",
87 "cache-target": "toolchain",
88 "cache-only": "true",
89 "build-args": "NODE_VERSION=\($version)\nNODE_VERSION_DIGEST=\($root.nodeDigests[$version])"
90 }]' "$toolchain_file")
91
92 # The cli toolchain image uses the same NODE_VERSION and CLI_VERSION
93 # as the E2E Test Typescript SDK cli docker job: Node from mise.lock
94 # and the version from ts/packages/cli/package.json.
95 cli_version=$(node -p "require('./ts/packages/cli/package.json').version")
96 cli_node_version=$(bash .github/scripts/read-mise-lock-version.sh node)
97 cli_digest=$(jq -r --arg version "$cli_node_version" '.nodeDigests[$version] // ""' "$toolchain_file")
98 if [[ -z "$cli_digest" ]]; then
99 echo "::error::toolchain-versions.json has no \"nodeDigests\" entry for ${cli_node_version}, the Node.js version pinned in mise.lock that Dockerfile.cli builds on. Add a \"${cli_node_version}\" digest so the e2e-cli cache can be warmed."
100 exit 1
101 fi
102 cli_matrix=$(jq -cn \
103 --arg cli_version "$cli_version" \
104 --arg cli_node_version "$cli_node_version" \
105 --arg cli_digest "$cli_digest" \
106 '[{
107 "name": "cli toolchain",
108 "dockerfile": "ts/e2e-tests/_utils/Dockerfile.cli",
109 "tag": "composio-e2e-cli:prewarm",
110 "cache-scope": "e2e-cli",
111 "cache-target": "toolchain",
112 "cache-only": "true",
113 "build-args": "CLI_VERSION=\($cli_version)\nNODE_VERSION=\($cli_node_version)\nNODE_VERSION_DIGEST=\($cli_digest)"
114 }]')
115
116 matrix=$(jq -cn \
117 --argjson install "$install_matrix" \
118 --argjson node "$node_matrix" \
119 --argjson cli "$cli_matrix" \
120 '{include: ($install + $node + $cli)}')
121 jq . <<< "$matrix"
122 echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
123
124 prewarm:
125 name: Prewarm ${{ matrix.name }}
126 needs: toolchain-versions
127 runs-on: ubuntu-latest
128 # Cold builds are the point of this workflow, so allow more time than the
129 # warm rebuilds inside the e2e jobs.
130 timeout-minutes: 30
131 strategy:
132 fail-fast: false
133 matrix: ${{ fromJSON(needs.toolchain-versions.outputs.matrix) }}
134 steps:
135 - name: Checkout
136 uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
137
138 # cache-only refreshes the toolchain-stage cache without the final image
139 # build, whose stages COPY host-built workspace artifacts a bare checkout
140 # does not have. The install entries keep the full build: their
141 # Dockerfile is self-contained, and that build is what saves their scope.
142 - name: Prewarm the ${{ matrix.cache-scope }} cache
143 uses: ./.github/actions/build-e2e-image
144 with:
145 dockerfile: ${{ matrix.dockerfile }}
146 tag: ${{ matrix.tag }}
147 cache-scope: ${{ matrix.cache-scope }}
148 cache-target: ${{ matrix.cache-target }}
149 cache-only: ${{ matrix.cache-only }}
150 build-args: ${{ matrix.build-args }}