Add retries in preparation of test environments for build CI. (#12198)
This particular part of our CI fails relatively frequently due to external infrastructure problems. By retrying on the first few failures in a run, we should mitigate most of these issues at the cost of making the worst case runtime about 10 minutes longer.
Austin S. Hemmelgarn committed
Mar 16, 2022 at 07:33 UTC
b09acd353aa43bb19cfaba1f7d0b4af757079202
1 file changed
+42
-2
.github/workflows/build.yml
+42
-2
@@ -197,6 +197,8 @@ jobs:
197
runs-on: ubuntu-latest
198
needs:
199
- matrix
200
+ env:
201
+ RETRY_DELAY: 300
202
strategy:
203
# Unlike the actal build tests, this completes _very_ fast (average of about 3 minutes for each job), so we
204
# just run everything in parallel instead lof limiting job concurrency.
@@ -210,7 +212,43 @@ jobs:
212
id: buildx
213
uses: docker/setup-buildx-action@v1
214
- name: Build test environment
213
- id: build
215
+ id: build1
216
+ uses: docker/build-push-action@v2
217
+ continue-on-error: true # We retry 3 times at 5 minute intervals if there is a failure here.
218
+ with:
219
+ push: false
220
+ load: false
221
+ file: .github/dockerfiles/Dockerfile.build_test
222
+ build-args: |
223
+ BASE=${{ matrix.distro }}
224
+ PRE=${{ matrix.env_prep }}
225
+ RMJSONC=${{ matrix.jsonc_removal }}
226
+ outputs: type=oci,dest=/tmp/image.tar
227
+ tags: test:${{ matrix.artifact_key }}
228
+ - name: Retry delay
229
+ if: ${{ steps.build1.outcome }} == 'failure'
230
+ run: sleep "${RETRY_DELAY}"
231
+ - name: Build test environment (attempt 2)
232
+ if: ${{ steps.build1.outcome }} == 'failure'
233
+ id: build2
234
+ uses: docker/build-push-action@v2
235
+ continue-on-error: true # We retry 3 times at 5 minute intervals if there is a failure here.
236
+ with:
237
+ push: false
238
+ load: false
239
+ file: .github/dockerfiles/Dockerfile.build_test
240
+ build-args: |
241
+ BASE=${{ matrix.distro }}
242
+ PRE=${{ matrix.env_prep }}
243
+ RMJSONC=${{ matrix.jsonc_removal }}
244
+ outputs: type=oci,dest=/tmp/image.tar
245
+ tags: test:${{ matrix.artifact_key }}
246
+ - name: Retry delay
247
+ if: ${{ steps.build1.outcome }} == 'failure' && ${{ steps.build2.outcome }} == 'failure'
248
+ run: sleep "${RETRY_DELAY}"
249
+ - name: Build test environment (attempt 3)
250
+ if: ${{ steps.build1.outcome }} == 'failure' && ${{ steps.build2.outcome }} == 'failure'
251
+ id: build3
252
uses: docker/build-push-action@v2
253
with:
254
push: false
@@ -241,7 +279,9 @@ jobs:
279
${{ github.repository }}: Test environment preparation for ${{ matrix.distro }} failed.
280
Checkout: ${{ steps.checkout.outcome }}
281
Set up Buildx: ${{ steps.buildx.outcome }}
244
- Build test environment: ${{ steps.build.outcome }}
282
+ Build test environment: ${{ steps.build1.outcome }}
283
+ Build test environment (attempt 2): ${{ steps.build2.outcome }}
284
+ Build test environment (attempt 3): ${{ steps.build3.outcome }}
285
Upload: ${{ steps.upload.outcome }}
286
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
287
if: >-