@cryptotaxi247 / kubo / commits / a49ff327d

ci: use dynamic config for CircleCI

When developing against ipfs/go-ipfs, we would like to be able to use the 2xlarge resource class for faster build and test cycles, but many external contributers will not have this resource class available to them. There is no direct way to change the resource class, so this uses dynamic config to generate a parameters JSON obj which is then fed into the configuration when the workflow starts, based on the Git URL of the build. For repos other than ipfs/go-ipfs, this reverts back to the "medium" resource class with a Make job parallelism of 3. (cherry picked from commit b2c3959a4f882235068fc10088844c6fddc7e7e6)

guseggert committed Sep 9, 2021 at 22:45 UTC a49ff327dca9fb33c90d37dd3120745b67755c94

2 files changed +488 -443

.circleci/config.yml

+30 -443

@@ -1,450 +1,37 @@

1 version: 2.1

2 -

3 -aliases:

4 - make_out_dirs: &make_out_dirs

5 - run: mkdir -p /tmp/circleci-artifacts /tmp/circleci-workspace /tmp/circleci-test-results/{unit,sharness}

6 - restore_gomod: &restore_gomod

7 - restore_cache:

8 - keys:

9 - - v5-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-ipfs/go.sum" }}-{{ .Environment.CIRCLE_JOB }}

10 - - v5-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-ipfs/go.sum" }}-

11 - - v5-dep-{{ .Branch }}-

12 - - v5-dep-master-

13 - store_gomod: &store_gomod

14 - save_cache:

15 - key: v5-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-ipfs/go.sum" }}-{{ .Environment.CIRCLE_JOB }}

16 - paths:

17 - - ~/go/pkg/mod

18 - - ~/.cache/go-build/

19 - only-version-tags: &only-version-tags

20 - tags:

21 - only: /^v[0-9].*/

22 - branches:

23 - ignore: /.*/

24 -

25 -default_environment: &default_environment

26 - SERVICE: circle-ci

27 - TRAVIS: 1

28 - CIRCLE: 1

29 - CIRCLE_TEST_REPORTS: /tmp/circleci-test-results

30 - CIRCLE_ARTIFACTS: /tmp/circleci-artifacts

31 - GIT_PAGER: cat

32 -

33 -executors:

34 - golang:

35 - docker:

36 - - image: cimg/go:1.16.7

37 - working_directory: ~/ipfs/go-ipfs

38 - environment:

39 - <<: *default_environment

40 - TEST_NO_DOCKER: 1

41 - TEST_NO_FUSE: 1

42 - TEST_VERBOSE: 1

43 - node:

44 - docker:

45 - - image: circleci/node:14

46 - working_directory: ~/ipfs/go-ipfs

47 - environment:

48 - <<: *default_environment

49 - node-browsers:

50 - docker:

51 - - image: circleci/node:14-browsers

52 - working_directory: ~/ipfs/go-ipfs

53 - environment:

54 - <<: *default_environment

55 - NO_SANDBOX: true

56 - IPFS_REUSEPORT: false

57 - LIBP2P_ALLOW_WEAK_RSA_KEYS: 1

58 - E2E_IPFSD_TYPE: go

59 - dockerizer:

60 - docker:

61 - - image: cimg/go:1.16.7

62 - environment:

63 - IMAGE_NAME: ipfs/go-ipfs

64 - WIP_IMAGE_TAG: wip

65 -

2 +setup: true

3 +orbs:

4 + continuation: circleci/continuation@0.2.0

5 jobs:

67 - gobuild:

68 - executor: golang

69 - steps:

70 - - checkout

71 - - *make_out_dirs

72 - - *restore_gomod

73 - - run:

74 - command: make cmd/ipfs-try-build

75 - environment:

76 - TEST_NO_FUSE: 0

77 - - run:

78 - command: make cmd/ipfs-try-build

79 - environment:

80 - TEST_NO_FUSE: 1

81 - - *store_gomod

82 - golint:

83 - executor: golang

84 - steps:

85 - - checkout

86 - - *make_out_dirs

87 - - *restore_gomod

88 - - run: |

89 - make -O test_go_lint

90 - - *store_gomod

91 - gotest:

92 - executor: golang

93 - steps:

94 - - checkout

95 - - *make_out_dirs

96 - - *restore_gomod

97 -

98 - - run: |

99 - make -j 1 test/unit/gotest.junit.xml \

100 - && [[ ! $(jq -s -c 'map(select(.Action == "fail")) | .[]' test/unit/gotest.json) ]]

101 - - run:

102 - when: always

103 - command: bash <(curl -s https://codecov.io/bash) -cF unittests -X search -f coverage/unit_tests.coverprofile

104 - - run:

105 - command: go test -v ./...

106 - working_directory: ~/ipfs/go-ipfs/docs/examples/go-ipfs-as-a-library

107 -

108 - - run:

109 - when: always

110 - command: mv "test/unit/gotest.junit.xml" /tmp/circleci-test-results/unit

111 -

112 - - *store_gomod

113 -

114 - - store_test_results:

115 - path: /tmp/circleci-test-results

116 - # Save artifacts

117 - - store_artifacts:

118 - path: /tmp/circleci-artifacts

119 - - store_artifacts:

120 - path: /tmp/circleci-test-results

121 - sharness:

122 - machine:

123 - image: ubuntu-2004:202010-01

124 - resource_class: 2xlarge

125 - working_directory: ~/ipfs/go-ipfs

126 - environment:

127 - <<: *default_environment

128 - TEST_NO_DOCKER: 1

129 - TEST_NO_FUSE: 1

130 - TEST_VERBOSE: 1

131 - steps:

132 - - run: sudo apt update

133 - - run: |

134 - mkdir ~/localgo && cd ~/localgo

135 - wget https://golang.org/dl/go1.16.7.linux-amd64.tar.gz

136 - tar xfz go1.16.7.linux-amd64.tar.gz

137 - echo "export PATH=$(pwd)/go/bin:\$PATH" >> ~/.bashrc

138 - - run: go version

139 - - run: sudo apt install socat net-tools

140 - - checkout

141 -

142 - - run:

143 - mkdir rb-pinning-service-api &&

144 - cd rb-pinning-service-api &&

145 - git init &&

146 - git remote add origin https://github.com/ipfs-shipyard/rb-pinning-service-api.git &&

147 - git fetch --depth 1 origin 773c3adbb421c551d2d89288abac3e01e1f7c3a8 &&

148 - git checkout FETCH_HEAD

149 - - run:

150 - cd rb-pinning-service-api &&

151 - docker-compose pull &&

152 - docker-compose up -d

153 -

154 - - *make_out_dirs

155 - - *restore_gomod

156 -

157 - - run:

158 - name: Setup Environment Variables

159 - # we need the docker host IP; all ports exported by child containers can be accessed there.

160 - command: echo "export TEST_DOCKER_HOST=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+')" >> $BASH_ENV

161 - - run:

162 - echo TEST_DOCKER_HOST=$TEST_DOCKER_HOST &&

163 - make -O -j 10 coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 TEST_DOCKER_HOST=$TEST_DOCKER_HOST

164 - - run:

165 - when: always

166 - command: bash <(curl -s https://codecov.io/bash) -cF sharness -X search -f coverage/sharness_tests.coverprofile

167 -

168 - - run: mv "test/sharness/test-results/sharness.xml" /tmp/circleci-test-results/sharness

169 - # make sure we fail if there are test failures

170 - - run: find test/sharness/test-results -name 't*-*.sh.*.counts' | test/sharness/lib/sharness/aggregate-results.sh | grep 'failed\s*0'

171 -

172 - - *store_gomod

173 -

174 - - store_test_results:

175 - path: /tmp/circleci-test-results

176 - # Save artifacts

177 - - store_artifacts:

178 - path: /tmp/circleci-artifacts

179 - - store_artifacts:

180 - path: /tmp/circleci-test-results

181 - build:

182 - executor: golang

6 + generate-params:

7 + executor: continuation/default

8 steps:

9 - checkout

185 - - *make_out_dirs

186 - - *restore_gomod

187 - - run:

188 - name: Building

189 - command: make build

190 - - run:

191 - name: Storing

192 - command: |

193 - mkdir -p /tmp/circleci-workspace/bin

194 - cp cmd/ipfs/ipfs /tmp/circleci-workspace/bin

195 - - persist_to_workspace:

196 - root: /tmp/circleci-workspace

197 - paths:

198 - - bin/ipfs

199 - - *store_gomod

200 - interop:

201 - executor: node

202 - parallelism: 4

203 - steps:

204 - - *make_out_dirs

205 - - attach_workspace:

206 - at: /tmp/circleci-workspace

207 - - run:

208 - name: Installing dependencies

209 - command: |

210 - npm init -y

211 - npm install ipfs@^0.52.2

212 - npm install ipfs-interop@^4.0.0

213 - npm install mocha-circleci-reporter@0.0.3

214 - working_directory: ~/ipfs/go-ipfs/interop

215 - - run:

216 - name: Running tests

217 - command: |

218 - mkdir -p /tmp/test-results/interop/

219 - export MOCHA_FILE="$(mktemp /tmp/test-results/interop/unit.XXXXXX.xml)"

220 - npx ipfs-interop -- -t node -f $(sed -n -e "s|^require('$.*$')$|test/\1|p" node_modules/ipfs-interop/test/node.js | circleci tests split) -- --reporter mocha-circleci-reporter

221 - working_directory: ~/ipfs/go-ipfs/interop

222 - environment:

223 - IPFS_REUSEPORT: false

224 - LIBP2P_ALLOW_WEAK_RSA_KEYS: 1

225 - IPFS_GO_EXEC: /tmp/circleci-workspace/bin/ipfs

226 - - store_test_results:

227 - path: /tmp/test-results

228 - go-ipfs-api:

229 - executor: golang

230 - steps:

231 - - *make_out_dirs

232 - - attach_workspace:

233 - at: /tmp/circleci-workspace

234 - - run:

235 - name: Cloning

236 - command: |

237 - git clone https://github.com/ipfs/go-ipfs-api.git

238 - git -C go-ipfs-api log -1

239 - - run:

240 - name: Starting the daemon

241 - command: /tmp/circleci-workspace/bin/ipfs daemon --init --enable-namesys-pubsub

242 - background: true

243 - - run:

244 - name: Waiting for the daemon

245 - no_output_timeout: 30s

246 - command: |

247 - while ! /tmp/circleci-workspace/bin/ipfs id --api=/ip4/127.0.0.1/tcp/5001 2>/dev/null; do

248 - sleep 1

249 - done

250 - - restore_cache:

251 - keys:

252 - - v1-go-api-{{ checksum "~/ipfs/go-ipfs/go-ipfs-api/go.sum" }}

253 - - v1-go-api-

254 - - run:

255 - command: go test -v ./...

256 - working_directory: ~/ipfs/go-ipfs/go-ipfs-api

257 - - save_cache:

258 - key: v1-go-api-{{ checksum "~/ipfs/go-ipfs/go-ipfs-api/go.sum" }}

259 - paths:

260 - - ~/go/pkg/mod

261 - - ~/.cache/go-build/

262 - - run:

263 - name: Stopping the daemon

264 - command: /tmp/circleci-workspace/bin/ipfs shutdown

265 - go-ipfs-http-client:

266 - executor: golang

267 - steps:

268 - - *make_out_dirs

269 - - attach_workspace:

270 - at: /tmp/circleci-workspace

271 - - run:

272 - name: Cloning

273 - command: |

274 - git clone https://github.com/ipfs/go-ipfs-http-client.git

275 - git -C go-ipfs-http-client log -1

276 - - restore_cache:

277 - keys:

278 - - v1-http-client-{{ checksum "~/ipfs/go-ipfs/go-ipfs-http-client/go.sum" }}

279 - - v1-http-client-

280 - - run:

281 - name: go test -v ./...

282 - command: |

283 - export PATH=/tmp/circleci-workspace/bin:$PATH

284 - go test -v ./...

285 - working_directory: ~/ipfs/go-ipfs/go-ipfs-http-client

286 - - save_cache:

287 - key: v1-http-client-{{ checksum "~/ipfs/go-ipfs/go-ipfs-http-client/go.sum" }}

288 - paths:

289 - - ~/go/pkg/mod

290 - - ~/.cache/go-build/

291 - ipfs-webui:

292 - executor: node-browsers

293 - steps:

294 - - *make_out_dirs

295 - - attach_workspace:

296 - at: /tmp/circleci-workspace

297 - - run:

298 - name: Cloning

299 - command: |

300 - git clone https://github.com/ipfs-shipyard/ipfs-webui.git

301 - git -C ipfs-webui log -1

302 - - restore_cache:

303 - keys:

304 - - v1-ipfs-webui-{{ checksum "~/ipfs/go-ipfs/ipfs-webui/package-lock.json" }}

305 - - v1-ipfs-webui-

306 - - run:

307 - name: Installing dependencies

308 - command: |

309 - npm install

310 - working_directory: ~/ipfs/go-ipfs/ipfs-webui

311 - - run:

312 - name: Running upstream tests (finish early if they fail)

313 - command: |

314 - npm test || circleci-agent step halt

315 - working_directory: ~/ipfs/go-ipfs/ipfs-webui

316 - - run:

317 - name: Running tests with go-ipfs built from current commit

318 - command: npm test

319 - working_directory: ~/ipfs/go-ipfs/ipfs-webui

320 - environment:

321 - IPFS_GO_EXEC: /tmp/circleci-workspace/bin/ipfs

322 - - save_cache:

323 - key: v1-ipfs-webui-{{ checksum "~/ipfs/go-ipfs/ipfs-webui/package-lock.json" }}

324 - paths:

325 - - ~/ipfs/go-ipfs/ipfs-webui/node_modules

326 - docker-build:

327 - executor: dockerizer

328 - steps:

329 - - checkout

330 - - setup_remote_docker:

331 - version: "19.03.13"

332 - - run:

333 - name: Build Docker image

334 - command: |

335 - docker build -t $IMAGE_NAME:$WIP_IMAGE_TAG .

336 - - run:

337 - name: Archive Docker image

338 - command: docker save -o go-ipfs-image.tar $IMAGE_NAME

339 - - persist_to_workspace:

340 - root: .

341 - paths:

342 - - ./go-ipfs-image.tar

343 - docker-build-extras:

344 - executor: dockerizer

345 - steps:

346 - - checkout

347 - - setup_remote_docker:

348 - version: "19.03.13"

349 - - run:

350 - name: Build Docker Extras image

351 - command: |

352 - docker build --build-arg IPFS_PLUGINS="peerlog" -t "$IMAGE_NAME-extras:$WIP_IMAGE_TAG" .

353 - - run:

354 - name: Archive Docker Extras image

355 - command: docker save -o go-ipfs-extras-image.tar $IMAGE_NAME

356 - - persist_to_workspace:

357 - root: .

358 - paths:

359 - - ./go-ipfs-extras-image.tar

360 - docker-push:

361 - executor: dockerizer

362 - steps:

363 - - checkout

364 - - setup_remote_docker:

365 - version: "19.03.13"

366 - - attach_workspace:

367 - at: /tmp/workspace

368 - - run:

369 - name: Load archived Docker image

370 - command: docker load -i /tmp/workspace/go-ipfs-image.tar

371 - - run:

372 - name: Publish Docker Image to Docker Hub

373 - command: |

374 - echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin

375 - ./bin/push-docker-tags.sh $(date -u +%F) "$CIRCLE_SHA1" "$CIRCLE_BRANCH" "$CIRCLE_TAG"

376 - docker-push-extras:

377 - executor: dockerizer

378 - steps:

379 - - checkout

380 - - setup_remote_docker:

381 - version: "19.03.13"

382 - - attach_workspace:

383 - at: /tmp/workspace

384 - - run:

385 - name: Load archived Docker Extras image

386 - command: docker load -i /tmp/workspace/go-ipfs-extras-image.tar

387 - - run:

388 - name: Publish Docker Extras Image to Docker Hub

389 - command: |

390 - echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin

391 - ./bin/push-docker-tags.sh $(date -u +%F) "$CIRCLE_SHA1" "$CIRCLE_BRANCH" "extras"

392 -

10 + - run:

11 + name: Generate params

12 + # for builds on the ipfs/go-ipfs repo, use 2xlarge for faster builds

13 + # but since this is not available for many contributors, we otherwise use medium

14 + command: |

15 + echo $CIRCLE_REPOSITORY_URL

16 + if [ "$CIRCLE_REPOSITORY_URL" = 'git@github.com:ipfs/go-ipfs.git' ]; then

17 + resource_class=2xlarge

18 + make_jobs=10

19 + else

20 + resource_class=medium

21 + make_jobs=3

22 + fi

23 + cat \<<- EOF > params.json

24 + {

25 + "resource_class": "$resource_class",

26 + "make_jobs": "$make_jobs"

27 + }

28 + EOF

29 + cat params.json

30 + - continuation/continue:

31 + parameters: params.json

32 + configuration_path: .circleci/main.yml

33 workflows:

34 version: 2

395 -

396 - # Runs for all branches, but not on tags

397 - # see: https://circleci.com/docs/2.0/workflows/#executing-workflows-for-a-git-tag

398 - test:

399 - jobs:

400 - - gobuild

401 - - golint

402 - - gotest

403 - - sharness

404 - - build

405 - - interop:

406 - requires:

407 - - build

408 - - go-ipfs-api:

409 - requires:

410 - - build

411 - - go-ipfs-http-client:

412 - requires:

413 - - build

414 - - ipfs-webui:

415 - requires:

416 - - build

417 - - docker-build

418 - - docker-push:

419 - # Requires dockerhub credentials, from circleci context.

420 - context: dockerhub

421 - requires:

422 - - docker-build

423 - - golint

424 - - gotest

425 - - sharness

426 - - interop

427 - - go-ipfs-api

428 - - go-ipfs-http-client

429 - - ipfs-webui

430 - filters:

431 - branches:

432 - only:

433 - - master

434 - # the bifrost-* branches are used for deploying code that hasn't hit master yet (e.g. for testing)

435 - - /^bifrost-.*/

436 -

437 -

438 - # NOTE: CircleCI only builds tags if you explicitly filter for them. That

439 - # also means tag-based jobs can only depend on other tag-based jobs, so we

440 - # use a separate workflow because every job needs to be tagged together.

441 - # see: https://circleci.com/docs/2.0/workflows/#executing-workflows-for-a-git-tag

442 - docker-on-tag:

35 + setup-workflow:

36 jobs:

444 - - docker-build:

445 - filters: *only-version-tags

446 - - docker-push:

447 - context: dockerhub

448 - filters: *only-version-tags

449 - requires:

450 - - docker-build

37 + - generate-params

.circleci/main.yml new

+458

@@ -0,0 +1,458 @@

1 +version: 2.1

2 +

3 +parameters:

4 + resource_class:

5 + type: string

6 + default: medium

7 + make_jobs:

8 + type: string

9 + default: 3

10 +

11 +aliases:

12 + make_out_dirs: &make_out_dirs

13 + run: mkdir -p /tmp/circleci-artifacts /tmp/circleci-workspace /tmp/circleci-test-results/{unit,sharness}

14 + restore_gomod: &restore_gomod

15 + restore_cache:

16 + keys:

17 + - v5-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-ipfs/go.sum" }}-{{ .Environment.CIRCLE_JOB }}

18 + - v5-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-ipfs/go.sum" }}-

19 + - v5-dep-{{ .Branch }}-

20 + - v5-dep-master-

21 + store_gomod: &store_gomod

22 + save_cache:

23 + key: v5-dep-{{ .Branch }}-{{ checksum "~/ipfs/go-ipfs/go.sum" }}-{{ .Environment.CIRCLE_JOB }}

24 + paths:

25 + - ~/go/pkg/mod

26 + - ~/.cache/go-build/

27 + only-version-tags: &only-version-tags

28 + tags:

29 + only: /^v[0-9].*/

30 + branches:

31 + ignore: /.*/

32 +

33 +default_environment: &default_environment

34 + SERVICE: circle-ci

35 + TRAVIS: 1

36 + CIRCLE: 1

37 + CIRCLE_TEST_REPORTS: /tmp/circleci-test-results

38 + CIRCLE_ARTIFACTS: /tmp/circleci-artifacts

39 + GIT_PAGER: cat

40 +

41 +executors:

42 + golang:

43 + docker:

44 + - image: cimg/go:1.16.7

45 + working_directory: ~/ipfs/go-ipfs

46 + environment:

47 + <<: *default_environment

48 + TEST_NO_DOCKER: 1

49 + TEST_NO_FUSE: 1

50 + TEST_VERBOSE: 1

51 + node:

52 + docker:

53 + - image: circleci/node:14

54 + working_directory: ~/ipfs/go-ipfs

55 + environment:

56 + <<: *default_environment

57 + node-browsers:

58 + docker:

59 + - image: circleci/node:14-browsers

60 + working_directory: ~/ipfs/go-ipfs

61 + environment:

62 + <<: *default_environment

63 + NO_SANDBOX: true

64 + IPFS_REUSEPORT: false

65 + LIBP2P_ALLOW_WEAK_RSA_KEYS: 1

66 + E2E_IPFSD_TYPE: go

67 + dockerizer:

68 + docker:

69 + - image: cimg/go:1.16.7

70 + environment:

71 + IMAGE_NAME: ipfs/go-ipfs

72 + WIP_IMAGE_TAG: wip

73 +

74 +jobs:

75 + gobuild:

76 + executor: golang

77 + steps:

78 + - checkout

79 + - *make_out_dirs

80 + - *restore_gomod

81 + - run:

82 + command: make cmd/ipfs-try-build

83 + environment:

84 + TEST_NO_FUSE: 0

85 + - run:

86 + command: make cmd/ipfs-try-build

87 + environment:

88 + TEST_NO_FUSE: 1

89 + - *store_gomod

90 + golint:

91 + executor: golang

92 + steps:

93 + - checkout

94 + - *make_out_dirs

95 + - *restore_gomod

96 + - run: |

97 + make -O test_go_lint

98 + - *store_gomod

99 + gotest:

100 + executor: golang

101 + steps:

102 + - checkout

103 + - *make_out_dirs

104 + - *restore_gomod

105 +

106 + - run: |

107 + make -j << pipeline.parameters.make_jobs >> test/unit/gotest.junit.xml \

108 + && [[ ! $(jq -s -c 'map(select(.Action == "fail")) | .[]' test/unit/gotest.json) ]]

109 + - run:

110 + when: always

111 + command: bash <(curl -s https://codecov.io/bash) -cF unittests -X search -f coverage/unit_tests.coverprofile

112 + - run:

113 + command: go test -v ./...

114 + working_directory: ~/ipfs/go-ipfs/docs/examples/go-ipfs-as-a-library

115 +

116 + - run:

117 + when: always

118 + command: mv "test/unit/gotest.junit.xml" /tmp/circleci-test-results/unit

119 +

120 + - *store_gomod

121 +

122 + - store_test_results:

123 + path: /tmp/circleci-test-results

124 + # Save artifacts

125 + - store_artifacts:

126 + path: /tmp/circleci-artifacts

127 + - store_artifacts:

128 + path: /tmp/circleci-test-results

129 + sharness:

130 + machine:

131 + image: ubuntu-2004:202010-01

132 + resource_class: << pipeline.parameters.resource_class >>

133 + working_directory: ~/ipfs/go-ipfs

134 + environment:

135 + <<: *default_environment

136 + TEST_NO_DOCKER: 1

137 + TEST_NO_FUSE: 1

138 + TEST_VERBOSE: 1

139 + steps:

140 + - run: sudo apt update

141 + - run: |

142 + mkdir ~/localgo && cd ~/localgo

143 + wget https://golang.org/dl/go1.16.7.linux-amd64.tar.gz

144 + tar xfz go1.16.7.linux-amd64.tar.gz

145 + echo "export PATH=$(pwd)/go/bin:\$PATH" >> ~/.bashrc

146 + - run: go version

147 + - run: sudo apt install socat net-tools

148 + - checkout

149 +

150 + - run:

151 + mkdir rb-pinning-service-api &&

152 + cd rb-pinning-service-api &&

153 + git init &&

154 + git remote add origin https://github.com/ipfs-shipyard/rb-pinning-service-api.git &&

155 + git fetch --depth 1 origin 773c3adbb421c551d2d89288abac3e01e1f7c3a8 &&

156 + git checkout FETCH_HEAD

157 + - run:

158 + cd rb-pinning-service-api &&

159 + docker-compose pull &&

160 + docker-compose up -d

161 +

162 + - *make_out_dirs

163 + - *restore_gomod

164 +

165 + - run:

166 + name: Setup Environment Variables

167 + # we need the docker host IP; all ports exported by child containers can be accessed there.

168 + command: echo "export TEST_DOCKER_HOST=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+')" >> $BASH_ENV

169 + - run:

170 + echo TEST_DOCKER_HOST=$TEST_DOCKER_HOST &&

171 + make -O -j << pipeline.parameters.make_jobs >> coverage/sharness_tests.coverprofile test/sharness/test-results/sharness.xml TEST_GENERATE_JUNIT=1 CONTINUE_ON_S_FAILURE=1 TEST_DOCKER_HOST=$TEST_DOCKER_HOST

172 + - run:

173 + when: always

174 + command: bash <(curl -s https://codecov.io/bash) -cF sharness -X search -f coverage/sharness_tests.coverprofile

175 +

176 + - run: mv "test/sharness/test-results/sharness.xml" /tmp/circleci-test-results/sharness

177 + # make sure we fail if there are test failures

178 + - run: find test/sharness/test-results -name 't*-*.sh.*.counts' | test/sharness/lib/sharness/aggregate-results.sh | grep 'failed\s*0'

179 +

180 + - *store_gomod

181 +

182 + - store_test_results:

183 + path: /tmp/circleci-test-results

184 + # Save artifacts

185 + - store_artifacts:

186 + path: /tmp/circleci-artifacts

187 + - store_artifacts:

188 + path: /tmp/circleci-test-results

189 + build:

190 + executor: golang

191 + steps:

192 + - checkout

193 + - *make_out_dirs

194 + - *restore_gomod

195 + - run:

196 + name: Building

197 + command: make build

198 + - run:

199 + name: Storing

200 + command: |

201 + mkdir -p /tmp/circleci-workspace/bin

202 + cp cmd/ipfs/ipfs /tmp/circleci-workspace/bin

203 + - persist_to_workspace:

204 + root: /tmp/circleci-workspace

205 + paths:

206 + - bin/ipfs

207 + - *store_gomod

208 + interop:

209 + executor: node

210 + parallelism: 4

211 + steps:

212 + - *make_out_dirs

213 + - attach_workspace:

214 + at: /tmp/circleci-workspace

215 + - run:

216 + name: Installing dependencies

217 + command: |

218 + npm init -y

219 + npm install ipfs@^0.52.2

220 + npm install ipfs-interop@^4.0.0

221 + npm install mocha-circleci-reporter@0.0.3

222 + working_directory: ~/ipfs/go-ipfs/interop

223 + - run:

224 + name: Running tests

225 + command: |

226 + mkdir -p /tmp/test-results/interop/

227 + export MOCHA_FILE="$(mktemp /tmp/test-results/interop/unit.XXXXXX.xml)"

228 + npx ipfs-interop -- -t node -f $(sed -n -e "s|^require('$.*$')$|test/\1|p" node_modules/ipfs-interop/test/node.js | circleci tests split) -- --reporter mocha-circleci-reporter

229 + working_directory: ~/ipfs/go-ipfs/interop

230 + environment:

231 + IPFS_REUSEPORT: false

232 + LIBP2P_ALLOW_WEAK_RSA_KEYS: 1

233 + IPFS_GO_EXEC: /tmp/circleci-workspace/bin/ipfs

234 + - store_test_results:

235 + path: /tmp/test-results

236 + go-ipfs-api:

237 + executor: golang

238 + steps:

239 + - *make_out_dirs

240 + - attach_workspace:

241 + at: /tmp/circleci-workspace

242 + - run:

243 + name: Cloning

244 + command: |

245 + git clone https://github.com/ipfs/go-ipfs-api.git

246 + git -C go-ipfs-api log -1

247 + - run:

248 + name: Starting the daemon

249 + command: /tmp/circleci-workspace/bin/ipfs daemon --init --enable-namesys-pubsub

250 + background: true

251 + - run:

252 + name: Waiting for the daemon

253 + no_output_timeout: 30s

254 + command: |

255 + while ! /tmp/circleci-workspace/bin/ipfs id --api=/ip4/127.0.0.1/tcp/5001 2>/dev/null; do

256 + sleep 1

257 + done

258 + - restore_cache:

259 + keys:

260 + - v1-go-api-{{ checksum "~/ipfs/go-ipfs/go-ipfs-api/go.sum" }}

261 + - v1-go-api-

262 + - run:

263 + command: go test -v ./...

264 + working_directory: ~/ipfs/go-ipfs/go-ipfs-api

265 + - save_cache:

266 + key: v1-go-api-{{ checksum "~/ipfs/go-ipfs/go-ipfs-api/go.sum" }}

267 + paths:

268 + - ~/go/pkg/mod

269 + - ~/.cache/go-build/

270 + - run:

271 + name: Stopping the daemon

272 + command: /tmp/circleci-workspace/bin/ipfs shutdown

273 + go-ipfs-http-client:

274 + executor: golang

275 + steps:

276 + - *make_out_dirs

277 + - attach_workspace:

278 + at: /tmp/circleci-workspace

279 + - run:

280 + name: Cloning

281 + command: |

282 + git clone https://github.com/ipfs/go-ipfs-http-client.git

283 + git -C go-ipfs-http-client log -1

284 + - restore_cache:

285 + keys:

286 + - v1-http-client-{{ checksum "~/ipfs/go-ipfs/go-ipfs-http-client/go.sum" }}

287 + - v1-http-client-

288 + - run:

289 + name: go test -v ./...

290 + command: |

291 + export PATH=/tmp/circleci-workspace/bin:$PATH

292 + go test -v ./...

293 + working_directory: ~/ipfs/go-ipfs/go-ipfs-http-client

294 + - save_cache:

295 + key: v1-http-client-{{ checksum "~/ipfs/go-ipfs/go-ipfs-http-client/go.sum" }}

296 + paths:

297 + - ~/go/pkg/mod

298 + - ~/.cache/go-build/

299 + ipfs-webui:

300 + executor: node-browsers

301 + steps:

302 + - *make_out_dirs

303 + - attach_workspace:

304 + at: /tmp/circleci-workspace

305 + - run:

306 + name: Cloning

307 + command: |

308 + git clone https://github.com/ipfs-shipyard/ipfs-webui.git

309 + git -C ipfs-webui log -1

310 + - restore_cache:

311 + keys:

312 + - v1-ipfs-webui-{{ checksum "~/ipfs/go-ipfs/ipfs-webui/package-lock.json" }}

313 + - v1-ipfs-webui-

314 + - run:

315 + name: Installing dependencies

316 + command: |

317 + npm install

318 + working_directory: ~/ipfs/go-ipfs/ipfs-webui

319 + - run:

320 + name: Running upstream tests (finish early if they fail)

321 + command: |

322 + npm test || circleci-agent step halt

323 + working_directory: ~/ipfs/go-ipfs/ipfs-webui

324 + - run:

325 + name: Running tests with go-ipfs built from current commit

326 + command: npm test

327 + working_directory: ~/ipfs/go-ipfs/ipfs-webui

328 + environment:

329 + IPFS_GO_EXEC: /tmp/circleci-workspace/bin/ipfs

330 + - save_cache:

331 + key: v1-ipfs-webui-{{ checksum "~/ipfs/go-ipfs/ipfs-webui/package-lock.json" }}

332 + paths:

333 + - ~/ipfs/go-ipfs/ipfs-webui/node_modules

334 + docker-build:

335 + executor: dockerizer

336 + steps:

337 + - checkout

338 + - setup_remote_docker:

339 + version: "19.03.13"

340 + - run:

341 + name: Build Docker image

342 + command: |

343 + docker build -t $IMAGE_NAME:$WIP_IMAGE_TAG .

344 + - run:

345 + name: Archive Docker image

346 + command: docker save -o go-ipfs-image.tar $IMAGE_NAME

347 + - persist_to_workspace:

348 + root: .

349 + paths:

350 + - ./go-ipfs-image.tar

351 + docker-build-extras:

352 + executor: dockerizer

353 + steps:

354 + - checkout

355 + - setup_remote_docker:

356 + version: "19.03.13"

357 + - run:

358 + name: Build Docker Extras image

359 + command: |

360 + docker build --build-arg IPFS_PLUGINS="peerlog" -t "$IMAGE_NAME-extras:$WIP_IMAGE_TAG" .

361 + - run:

362 + name: Archive Docker Extras image

363 + command: docker save -o go-ipfs-extras-image.tar $IMAGE_NAME

364 + - persist_to_workspace:

365 + root: .

366 + paths:

367 + - ./go-ipfs-extras-image.tar

368 + docker-push:

369 + executor: dockerizer

370 + steps:

371 + - checkout

372 + - setup_remote_docker:

373 + version: "19.03.13"

374 + - attach_workspace:

375 + at: /tmp/workspace

376 + - run:

377 + name: Load archived Docker image

378 + command: docker load -i /tmp/workspace/go-ipfs-image.tar

379 + - run:

380 + name: Publish Docker Image to Docker Hub

381 + command: |

382 + echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin

383 + ./bin/push-docker-tags.sh $(date -u +%F) "$CIRCLE_SHA1" "$CIRCLE_BRANCH" "$CIRCLE_TAG"

384 + docker-push-extras:

385 + executor: dockerizer

386 + steps:

387 + - checkout

388 + - setup_remote_docker:

389 + version: "19.03.13"

390 + - attach_workspace:

391 + at: /tmp/workspace

392 + - run:

393 + name: Load archived Docker Extras image

394 + command: docker load -i /tmp/workspace/go-ipfs-extras-image.tar

395 + - run:

396 + name: Publish Docker Extras Image to Docker Hub

397 + command: |

398 + echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin

399 + ./bin/push-docker-tags.sh $(date -u +%F) "$CIRCLE_SHA1" "$CIRCLE_BRANCH" "extras"

400 +

401 +workflows:

402 + version: 2

403 +

404 + # Runs for all branches, but not on tags

405 + # see: https://circleci.com/docs/2.0/workflows/#executing-workflows-for-a-git-tag

406 + test:

407 + jobs:

408 + - gobuild

409 + - golint

410 + - gotest

411 + - sharness

412 + - build

413 + - interop:

414 + requires:

415 + - build

416 + - go-ipfs-api:

417 + requires:

418 + - build

419 + - go-ipfs-http-client:

420 + requires:

421 + - build

422 + - ipfs-webui:

423 + requires:

424 + - build

425 + - docker-build

426 + - docker-push:

427 + # Requires dockerhub credentials, from circleci context.

428 + context: dockerhub

429 + requires:

430 + - docker-build

431 + - golint

432 + - gotest

433 + - sharness

434 + - interop

435 + - go-ipfs-api

436 + - go-ipfs-http-client

437 + - ipfs-webui

438 + filters:

439 + branches:

440 + only:

441 + - master

442 + # the bifrost-* branches are used for deploying code that hasn't hit master yet (e.g. for testing)

443 + - /^bifrost-.*/

444 +

445 +

446 + # NOTE: CircleCI only builds tags if you explicitly filter for them. That

447 + # also means tag-based jobs can only depend on other tag-based jobs, so we

448 + # use a separate workflow because every job needs to be tagged together.

449 + # see: https://circleci.com/docs/2.0/workflows/#executing-workflows-for-a-git-tag

450 + docker-on-tag:

451 + jobs:

452 + - docker-build:

453 + filters: *only-version-tags

454 + - docker-push:

455 + context: dockerhub

456 + filters: *only-version-tags

457 + requires:

458 + - docker-build