.travis.yml: Build docker images in parallel (#6672)
* packaging/docker/{build,publish}.sh: Ability to externally defined ARCHS * .travis.yml: Run docker build and release jobs in parallel
Konstantinos Natsakis committed
Aug 22, 2019 at 18:28 UTC
07fc4b0dcd8eef5ca7463faeb71e2146224a8df0
3 files changed
+90
-50
.travis.yml
+80
-39
@@ -60,7 +60,8 @@ stages:
60
if: branch = master AND type != pull_request AND type != cron
61
62
- name: Publish for release
63
- if: branch = master AND type != pull_request AND type != cron AND commit_message =~ /(\[netdata release candidate\]|\[netdata major release\]|\[netdata minor release\]|\[netdata patch release\])/
63
+ # We don't run on release candidates
64
+ if: branch = master AND type != pull_request AND type != cron AND commit_message =~ /(\[netdata release candidate\]|\[netdata major release\]|\[netdata minor release\]|\[netdata patch release\])/ AND tag !~ /(-rc)/
65
66
# Build DEB packages under special conditions
67
- name: Package ubuntu/* and debian/*
@@ -475,26 +476,48 @@ jobs:
476
477
# We only publish if a TAG has been set during packaging
478
- stage: Publish for release
479
+ _template: &RELEASE_TEMPLATE
480
+ git:
481
+ depth: false
482
+ before_script: post_message "TRAVIS_MESSAGE" "Publishing docker images" "${NOTIF_CHANNEL}"
483
+ script:
484
+ - echo "GIT Branch:" && git branch
485
+ - echo "Last commit:" && git log -1
486
+ - echo "GIT Describe:" && git describe
487
+ - echo "packaging/version:" && cat packaging/version
488
+ - packaging/docker/check_login.sh
489
+ - echo "Switching to latest master branch, to pick up tagging if any" && git checkout master && git pull
490
+ - packaging/docker/build.sh
491
+ - packaging/docker/publish.sh
492
+ after_failure: post_message "TRAVIS_MESSAGE" "<!here> Docker image publishing failed"
493
+
494
+ name: Build & Publish docker image for i386
495
+ <<: *RELEASE_TEMPLATE
496
+ env:
497
+ - ALLOW_SOFT_FAILURE_HERE=true
498
+ - ARCHS=i386
499
479
- name: Build & Publish docker images
480
- before_script: post_message "TRAVIS_MESSAGE" "Publishing docker images" "${NOTIF_CHANNEL}"
481
- script:
482
- - echo "GIT Branch:" && git branch
483
- - echo "Last commit:" && git log -1
484
- - echo "GIT Describe:" && git describe
485
- - echo "packaging/version:" && cat packaging/version
486
- - packaging/docker/check_login.sh
487
- - echo "Switching to latest master branch, to pick up tagging if any" && git checkout master && git pull
488
- - packaging/docker/build.sh
489
- - packaging/docker/publish.sh
490
- after_failure: post_message "TRAVIS_MESSAGE" "<!here> Docker image publishing failed"
491
- git:
492
- depth: false
493
- env: ALLOW_SOFT_FAILURE_HERE=true
494
- # We don't run on release candidates
495
- if: tag !~ /(-rc)/
500
+ - name: Build & Publish docker image for amd64
501
+ <<: *RELEASE_TEMPLATE
502
+ env:
503
+ - ALLOW_SOFT_FAILURE_HERE=true
504
+ - ARCHS=amd64
505
+
506
+ - name: Build & Publish docker image for armhf
507
+ <<: *RELEASE_TEMPLATE
508
+ env:
509
+ - ALLOW_SOFT_FAILURE_HERE=true
510
+ - ARCHS=armhf
511
+
512
+ - name: Build & Publish docker image for aarch64
513
+ <<: *RELEASE_TEMPLATE
514
+ env:
515
+ - ALLOW_SOFT_FAILURE_HERE=true
516
+ - ARCHS=aarch64
517
518
- name: Create release draft
519
+ git:
520
+ depth: false
521
before_script: post_message "TRAVIS_MESSAGE" "Drafting release on github" "${NOTIF_CHANNEL}"
522
script:
523
- echo "GIT Branch:" && git branch
@@ -503,11 +526,7 @@ jobs:
526
- echo "packaging/version:" && cat packaging/version
527
- echo "Generating release artifacts" && .travis/create_artifacts.sh # Could/should be a common storage to put this and share between jobs
528
- .travis/draft_release.sh
506
- git:
507
- depth: false
529
after_failure: post_message "TRAVIS_MESSAGE" "<!here> Draft release submission failed"
509
- # We don't run on release candidates
510
- if: tag !~ /(-rc)/
530
531
532
@@ -540,8 +559,46 @@ jobs:
559
# This is the nightly execution step
560
#
561
- stage: Nightly release
562
+ _template: &NIGHTLY_TEMPLATE
563
+ git:
564
+ depth: false
565
+ before_script: post_message "TRAVIS_MESSAGE" "Publishing docker images for nightlies" "${NOTIF_CHANNEL}"
566
+ script:
567
+ - echo "GIT Branch:" && git branch
568
+ - echo "Last commit:" && git log -1
569
+ - echo "GIT Describe:" && git describe
570
+ - echo "packaging/version:" && cat packaging/version
571
+ - docker info
572
+ - packaging/docker/check_login.sh
573
+ - packaging/docker/build.sh
574
+ - packaging/docker/publish.sh
575
+ after_failure: post_message "TRAVIS_MESSAGE" "<!here> Nightly docker image publish failed"
576
+
577
+ name: Build & Publish docker image for i386
578
+ <<: *NIGHTLY_TEMPLATE
579
+ env:
580
+ - ALLOW_SOFT_FAILURE_HERE=true
581
+ - ARCHS=i386
582
+
583
+ - name: Build & Publish docker image for amd64
584
+ <<: *NIGHTLY_TEMPLATE
585
+ env:
586
+ - ALLOW_SOFT_FAILURE_HERE=true
587
+ - ARCHS=amd64
588
+
589
+ - name: Build & Publish docker image for armhf
590
+ <<: *NIGHTLY_TEMPLATE
591
+ env:
592
+ - ALLOW_SOFT_FAILURE_HERE=true
593
+ - ARCHS=armhf
594
+
595
+ - name: Build & Publish docker image for aarch64
596
+ <<: *NIGHTLY_TEMPLATE
597
+ env:
598
+ - ALLOW_SOFT_FAILURE_HERE=true
599
+ - ARCHS=aarch64
600
544
- name: Trigger .RPM and .DEB package generation
601
+ - name: Trigger .RPM and .DEB package generation
602
before_script: post_message "TRAVIS_MESSAGE" "Starting RPM and DEB package generation for nightlies" "${NOTIF_CHANNEL}"
603
script:
604
- .travis/trigger_package_generation.sh "[Build latest]"
@@ -549,22 +606,6 @@ jobs:
606
git:
607
depth: false
608
552
- - name: Build & Publish docker images
553
- before_script: post_message "TRAVIS_MESSAGE" "Publishing docker images for nightlies" "${NOTIF_CHANNEL}"
554
- script:
555
- - echo "GIT Branch:" && git branch
556
- - echo "Last commit:" && git log -1
557
- - echo "GIT Describe:" && git describe
558
- - echo "packaging/version:" && cat packaging/version
559
- - docker info
560
- - packaging/docker/check_login.sh
561
- - packaging/docker/build.sh
562
- - packaging/docker/publish.sh
563
- after_failure: post_message "TRAVIS_MESSAGE" "<!here> Nightly docker image publish failed"
564
- git:
565
- depth: false
566
- env: ALLOW_SOFT_FAILURE_HERE=true
567
-
609
- name: Create nightly release artifacts, publish to GCS
610
before_script: post_message "TRAVIS_MESSAGE" "Starting artifacts generation for nightlies" "${NOTIF_CHANNEL}"
611
script:
packaging/docker/build.sh
+1
-1
@@ -17,7 +17,7 @@ VERSION="$1"
17
declare -A ARCH_MAP
18
ARCH_MAP=(["i386"]="386" ["amd64"]="amd64" ["armhf"]="arm" ["aarch64"]="arm64")
19
DEVEL_ARCHS=(amd64)
20
-ARCHS="${!ARCH_MAP[@]}"
20
+[ "${ARCHS}" ] || ARCHS="${!ARCH_MAP[@]}" # Use default ARCHS unless ARCHS are externally provided
21
22
if [ -z ${REPOSITORY} ]; then
23
REPOSITORY="${TRAVIS_REPO_SLUG}"
packaging/docker/publish.sh
+9
-10
@@ -19,7 +19,7 @@ VERSION="$1"
19
declare -A ARCH_MAP
20
ARCH_MAP=(["i386"]="386" ["amd64"]="amd64" ["armhf"]="arm" ["aarch64"]="arm64")
21
DEVEL_ARCHS=(amd64)
22
-ARCHS="${!ARCH_MAP[@]}"
22
+[ "${ARCHS}" ] || ARCHS="${!ARCH_MAP[@]}" # Use default ARCHS unless ARCHS are externally provided
23
DOCKER_CMD="docker --config ${WORKDIR}"
24
GIT_MAIL=${GIT_MAIL:-"bot@netdata.cloud"}
25
GIT_USER=${GIT_USER:-"netdatabot"}
@@ -87,20 +87,19 @@ echo "Waiting for images publishing to complete"
87
wait
88
89
# Recreate docker manifest list
90
+echo "Getting tag list for version '${VERSION}'.."
91
+TAGS=($(curl -s https://registry.hub.docker.com/v2/repositories/${REPOSITORY}/tags/ | jq -r '.results[]["name"]' | grep "^${VERSION}-"))
92
+
93
echo "Creating manifest list.."
91
-$DOCKER_CMD manifest create --amend "${MANIFEST_LIST}" \
92
- "${MANIFEST_LIST}-i386" \
93
- "${MANIFEST_LIST}-armhf" \
94
- "${MANIFEST_LIST}-aarch64" \
95
- "${MANIFEST_LIST}-amd64"
94
+$DOCKER_CMD manifest create --amend "${MANIFEST_LIST}" "${TAGS[@]/#/${REPOSITORY}:}"
95
96
# Annotate manifest with CPU architecture information
97
98
echo "Executing manifest annotate.."
100
-for ARCH in ${ARCHS[@]}; do
101
- TAG="${MANIFEST_LIST}-${ARCH}"
102
- echo "Annotating manifest for $ARCH, with TAG: ${TAG} (Manifest list: ${MANIFEST_LIST})"
103
- $DOCKER_CMD manifest annotate "${MANIFEST_LIST}" "${TAG}" --os linux --arch "${ARCH_MAP[$ARCH]}"
99
+for TAG in "${TAGS[@]}"; do
100
+ ARCH="${TAG#${VERSION}-}"
101
+ echo "Annotating manifest for $ARCH, with TAG: ${REPOSITORY}:${TAG} (Manifest list: ${MANIFEST_LIST})"
102
+ $DOCKER_CMD manifest annotate "${MANIFEST_LIST}" "${REPOSITORY}:${TAG}" --os linux --arch "${ARCH_MAP[$ARCH]}"
103
done
104
105
# Push manifest to docker hub