ci: simplify self-hosted runner selection (#9813)
* ci: simplify self-hosted runner selection * ci: put interop jobs on self-hosted runners
Piotr Galar committed
May 9, 2023 at 12:53 UTC
c9a8d9ff92c7a578209167e38c898fc1e849de67
5 files changed
+42
-65
.github/workflows/build.yml
+25
-9
@@ -14,6 +14,10 @@ concurrency:
14
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
15
cancel-in-progress: true
16
17
+defaults:
18
+ run:
19
+ shell: bash
20
+
21
jobs:
22
interop-prep:
23
if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
@@ -44,7 +48,7 @@ jobs:
48
path: cmd/ipfs/ipfs
49
interop:
50
needs: [interop-prep]
47
- runs-on: ubuntu-latest
51
+ runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }}
52
timeout-minutes: 20
53
defaults:
54
run:
@@ -75,7 +79,7 @@ jobs:
79
npm install ipfs-interop@^10.0.1
80
working-directory: interop
81
# Run the interop tests while ignoring the js-js interop test cases
78
- - run: npx ipfs-interop -- -t node --grep '^(?!.*(js\d? -> js\d?|js-js-js))'
82
+ - run: npx ipfs-interop -- -t node --grep '^(?!.*(js\d? -> js\d?|js-js-js))' --parallel
83
env:
84
LIBP2P_TCP_REUSEPORT: false
85
LIBP2P_ALLOW_WEAK_RSA_KEYS: 1
@@ -156,7 +160,7 @@ jobs:
160
working-directory: go-ipfs-http-client
161
ipfs-webui:
162
needs: [interop-prep]
159
- runs-on: ubuntu-latest
163
+ runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }}
164
timeout-minutes: 20
165
env:
166
NO_SANDBOX: true
@@ -191,14 +195,26 @@ jobs:
195
key: ${{ runner.os }}-${{ github.job }}-${{ hashFiles('**/package-lock.json') }}
196
restore-keys: |
197
${{ runner.os }}-${{ github.job }}-
194
- - run: |
195
- npm ci --prefer-offline --no-audit --progress=false
196
- npx playwright install
198
+ - env:
199
+ NPM_CACHE_DIR: ${{ steps.npm-cache-dir.outputs.dir }}
200
+ run: |
201
+ npm ci --prefer-offline --no-audit --progress=false --cache "$NPM_CACHE_DIR"
202
+ npx playwright install --with-deps
203
+ working-directory: ipfs-webui
204
+ - id: ref
205
+ run: echo "ref=$(git rev-parse --short HEAD)" | tee -a $GITHUB_OUTPUT
206
working-directory: ipfs-webui
198
- - name: Run ipfs-webui@main build and smoke-test to confirm the upstream repo is not broken
199
- run: npm test
207
+ - id: state
208
+ env:
209
+ GITHUB_REPOSITORY: ipfs/ipfs-webui
210
+ GITHUB_REF: ${{ steps.ref.outputs.ref }}
211
+ GITHUB_TOKEN: ${{ github.token }}
212
+ run: |
213
+ echo "state=$(curl -L -H "Authorization: Bearer $GITHUB_TOKEN" "https://api.github.com/repos/$GITHUB_REPOSITORY/commits/$GITHUB_REF/status" --jq '.state')" | tee -a $GITHUB_OUTPUT
214
+ - name: Build ipfs-webui@main (state=${{ steps.state.outputs.state }})
215
+ run: npm run test:build
216
working-directory: ipfs-webui
201
- - name: Test ipfs-webui@main E2E against the locally built Kubo binary
217
+ - name: Test ipfs-webui@main (state=${{ steps.state.outputs.state }}) E2E against the locally built Kubo binary
218
run: npm run test:e2e
219
env:
220
IPFS_GO_EXEC: ${{ github.workspace }}/cmd/ipfs/ipfs
.github/workflows/gobuild.yml
+2
-5
@@ -12,12 +12,9 @@ concurrency:
12
cancel-in-progress: true
13
14
jobs:
15
- go-build-runner:
16
- if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
17
- uses: ipfs/kubo/.github/workflows/runner.yml@master
15
go-build:
19
- needs: [go-build-runner]
20
- runs-on: ${{ fromJSON(needs.go-build-runner.outputs.config).labels }}
16
+ if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
17
+ runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "4xlarge"]' || '"ubuntu-latest"') }}
18
timeout-minutes: 20
19
env:
20
TEST_DOCKER: 0
.github/workflows/gotest.yml
+6
-6
@@ -12,12 +12,9 @@ concurrency:
12
cancel-in-progress: true
13
14
jobs:
15
- go-test-runner:
16
- if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
17
- uses: ipfs/kubo/.github/workflows/runner.yml@master
15
go-test:
19
- needs: [go-test-runner]
20
- runs-on: ${{ fromJSON(needs.go-test-runner.outputs.config).labels }}
16
+ if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
17
+ runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "2xlarge"]' || '"ubuntu-latest"') }}
18
timeout-minutes: 20
19
env:
20
TEST_DOCKER: 0
@@ -41,8 +38,11 @@ jobs:
38
with:
39
name: ${{ github.job }}
40
- name: 👉️ If this step failed, go to «Summary» (top left) → inspect the «Failures/Errors» table
41
+ env:
42
+ # increasing parallelism beyond 2 doesn't speed up the tests much
43
+ PARALLEL: 2
44
run: |
45
- make -j 2 test/unit/gotest.junit.xml &&
45
+ make -j "$PARALLEL" test/unit/gotest.junit.xml &&
46
[[ ! $(jq -s -c 'map(select(.Action == "fail")) | .[]' test/unit/gotest.json) ]]
47
- name: Upload coverage to Codecov
48
uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0
.github/workflows/runner.yml
deleted
-34
@@ -1,34 +0,0 @@
1
-name: Runner
2
-
3
-on:
4
- workflow_call:
5
- outputs:
6
- config:
7
- description: "The runner's configuration"
8
- value: ${{ jobs.choose.outputs.config }}
9
-
10
-jobs:
11
- choose:
12
- runs-on: ubuntu-latest
13
- timeout-minutes: 1
14
- outputs:
15
- config: ${{ steps.config.outputs.result }}
16
- steps:
17
- - uses: actions/github-script@v6
18
- id: config
19
- with:
20
- script: |
21
- if (`${context.repo.owner}/${context.repo.repo}` === 'ipfs/kubo') {
22
- return {
23
- labels: ['self-hosted', 'linux', 'x64', 'kubo'],
24
- parallel: 10,
25
- aws: true
26
- }
27
- } else {
28
- return {
29
- labels: ['ubuntu-latest'],
30
- parallel: 3,
31
- aws: false
32
- }
33
- }
34
- - run: echo ${{ steps.config.outputs.result }}
.github/workflows/sharness.yml
+9
-11
@@ -12,12 +12,9 @@ concurrency:
12
cancel-in-progress: true
13
14
jobs:
15
- sharness-runner:
16
- if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
17
- uses: ipfs/kubo/.github/workflows/runner.yml@master
15
sharness-test:
19
- needs: [sharness-runner]
20
- runs-on: ${{ fromJSON(needs.sharness-runner.outputs.config).labels }}
16
+ if: github.repository == 'ipfs/kubo' || github.event_name == 'workflow_dispatch'
17
+ runs-on: ${{ fromJSON(github.repository == 'ipfs/kubo' && '["self-hosted", "linux", "x64", "4xlarge"]' || '"ubuntu-latest"') }}
18
timeout-minutes: 20
19
defaults:
20
run:
@@ -57,7 +54,8 @@ jobs:
54
TEST_EXPENSIVE: 1
55
IPFS_CHECK_RCMGR_DEFAULTS: 1
56
CONTINUE_ON_S_FAILURE: 1
60
- PARALLEL: ${{ fromJSON(needs.sharness-runner.outputs.config).parallel }}
57
+ # increasing parallelism beyond 10 doesn't speed up the tests much
58
+ PARALLEL: ${{ github.repository == 'ipfs/kubo' && 10 || 3 }}
59
- name: Upload coverage report
60
uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0
61
if: failure() || success()
@@ -86,12 +84,12 @@ jobs:
84
- name: Upload one-page HTML report to S3
85
id: one-page
86
uses: pl-strflt/tf-aws-gh-runner/.github/actions/upload-artifact@main
89
- if: fromJSON(needs.sharness-runner.outputs.config).aws && (failure() || success())
87
+ if: github.repository == 'ipfs/kubo' && (failure() || success())
88
with:
89
source: kubo/test/sharness/test-results/sharness.html
90
destination: sharness.html
91
- name: Upload one-page HTML report
94
- if: (! fromJSON(needs.sharness-runner.outputs.config).aws) && (failure() || success())
92
+ if: github.repository != 'ipfs/kubo' && (failure() || success())
93
uses: actions/upload-artifact@v3
94
with:
95
name: sharness.html
@@ -106,18 +104,18 @@ jobs:
104
- name: Upload full HTML report to S3
105
id: full
106
uses: pl-strflt/tf-aws-gh-runner/.github/actions/upload-artifact@main
109
- if: fromJSON(needs.sharness-runner.outputs.config).aws && (failure() || success())
107
+ if: github.repository == 'ipfs/kubo' && (failure() || success())
108
with:
109
source: kubo/test/sharness/test-results/sharness-html
110
destination: sharness-html/
111
- name: Upload full HTML report
114
- if: (! fromJSON(needs.sharness-runner.outputs.config).aws) && (failure() || success())
112
+ if: github.repository != 'ipfs/kubo' && (failure() || success())
113
uses: actions/upload-artifact@v3
114
with:
115
name: sharness-html
116
path: kubo/test/sharness/test-results/sharness-html
117
- name: Add S3 links to the summary
120
- if: fromJSON(needs.sharness-runner.outputs.config).aws && (failure() || success())
118
+ if: github.repository == 'ipfs/kubo' && (failure() || success())
119
run: echo "$MD" >> $GITHUB_STEP_SUMMARY
120
env:
121
MD: |