Consolidate build matrix across CI jobs. (#11807)
* Restructure CI build matrix handling for better consolidation. * Update core packaging code to use new distro.yml file. * Update repository config packaging code to use new distro.yml file.
Austin S. Hemmelgarn committed
Feb 16, 2022 at 08:53 UTC
46d73e34c0dbab3777b898fc5418f9d0a0e82489
5 files changed
+255
-157
.github/data/build-matrix.json
deleted
-98
@@ -1,98 +0,0 @@
1
-{
2
- "include": [
3
- {
4
- "distro": "alpine:edge",
5
- "artifact_key": "alpine-edge",
6
- "pre": "apk add -U bash",
7
- "rmjsonc": "apk del json-c-dev"
8
- },
9
- {
10
- "distro": "alpine:3.14",
11
- "artifact_key": "alpine-3.14",
12
- "pre": "apk add -U bash",
13
- "rmjsonc": "apk del json-c-dev"
14
- },
15
- {
16
- "distro": "alpine:3.13",
17
- "artifact_key": "alpine-3.13",
18
- "pre": "apk add -U bash",
19
- "rmjsonc": "apk del json-c-dev"
20
- },
21
- {
22
- "distro": "alpine:3.12",
23
- "artifact_key": "alpine-3.12",
24
- "pre": "apk add -U bash",
25
- "rmjsonc": "apk del json-c-dev"
26
- },
27
- {
28
- "distro": "archlinux:latest",
29
- "artifact_key": "archlinux",
30
- "pre": "pacman --noconfirm -Syu && pacman --noconfirm -Sy grep libffi"
31
- },
32
- {
33
- "distro": "centos:7",
34
- "artifact_key": "centos7"
35
- },
36
- {
37
- "distro": "rockylinux/rockylinux:8",
38
- "artifact_key": "rockylinux8",
39
- "rmjsonc": "dnf remove -y json-c-devel"
40
- },
41
- {
42
- "distro": "debian:11",
43
- "artifact_key": "debian11",
44
- "pre": "apt-get update",
45
- "rmjsonc": "apt-get remove -y libjson-c-dev"
46
- },
47
- {
48
- "distro": "debian:10",
49
- "artifact_key": "debian10",
50
- "pre": "apt-get update",
51
- "rmjsonc": "apt-get remove -y libjson-c-dev"
52
- },
53
- {
54
- "distro": "debian:9",
55
- "artifact_key": "debian9",
56
- "pre": "apt-get update",
57
- "rmjsonc": "apt-get remove -y libjson-c-dev"
58
- },
59
- {
60
- "distro": "fedora:35",
61
- "artifact_key": "fedora35",
62
- "rmjsonc": "dnf remove -y json-c-devel"
63
- },
64
- {
65
- "distro": "fedora:34",
66
- "artifact_key": "fedora34",
67
- "rmjsonc": "dnf remove -y json-c-devel"
68
- },
69
- {
70
- "distro": "opensuse/leap:15.3",
71
- "artifact_key": "opensuse15.3",
72
- "rmjsonc": "zypper rm -y libjson-c-devel"
73
- },
74
- {
75
- "distro": "oraclelinux:8",
76
- "artifact_key": "oraclelinux8",
77
- "rmjsonc": "dnf remove -y json-c-devel"
78
- },
79
- {
80
- "distro": "ubuntu:21.10",
81
- "artifact_key": "ubuntu21.10",
82
- "pre": "rm -f /etc/apt/apt.conf.d/docker && apt-get update",
83
- "rmjsonc": "apt-get remove -y libjson-c-dev"
84
- },
85
- {
86
- "distro": "ubuntu:20.04",
87
- "artifact_key": "ubuntu20.04",
88
- "pre": "apt-get update",
89
- "rmjsonc": "apt-get remove -y libjson-c-dev"
90
- },
91
- {
92
- "distro": "ubuntu:18.04",
93
- "artifact_key": "ubuntu18.04",
94
- "pre": "apt-get update",
95
- "rmjsonc": "apt-get remove -y libjson-c-dev"
96
- }
97
- ]
98
-}
.github/data/distros.yml
new
+145
@@ -0,0 +1,145 @@
1
+# This defines the full set of distros we run CI on.
2
+---
3
+platform_map: # map packaging architectures to docker platforms
4
+ i386: linux/i386
5
+ amd64: linux/amd64
6
+ armhf: linux/arm/v7
7
+ arm64: linux/arm64/v8
8
+include:
9
+ - &alpine
10
+ distro: alpine
11
+ version: edge
12
+ env_prep: |
13
+ apk add -U bash
14
+ jsonc_removal: |
15
+ apk del json-c-dev
16
+ - <<: *alpine
17
+ version: "3.15"
18
+ - <<: *alpine
19
+ version: "3.14"
20
+ - <<: *alpine
21
+ version: "3.13"
22
+ - <<: *alpine
23
+ version: "3.12"
24
+
25
+ - distro: archlinux
26
+ version: latest
27
+ env_prep: |
28
+ pacman --noconfirm -Syu && pacman --noconfirm -Sy grep libffi
29
+
30
+ - distro: rockylinux
31
+ version: "8"
32
+ base_image: rockylinux/rockylinux
33
+ jsonc_removal: |
34
+ dnf remove -y json-c-devel
35
+ packages:
36
+ type: rpm
37
+ repo_distro: el/8
38
+ arches:
39
+ - amd64
40
+ - arm64
41
+ - distro: centos
42
+ version: "7"
43
+ packages:
44
+ type: rpm
45
+ repo_distro: el/7
46
+ arches:
47
+ - amd64
48
+
49
+ - &debian
50
+ distro: debian
51
+ version: "11"
52
+ env_prep: |
53
+ apt-get update
54
+ jsonc_removal: |
55
+ apt-get purge -y libjson-c-dev
56
+ packages: &debian_packages
57
+ type: deb
58
+ repo_distro: debian/bullseye
59
+ arches:
60
+ - i386
61
+ - amd64
62
+ - armhf
63
+ - arm64
64
+ - <<: *debian
65
+ version: "10"
66
+ packages:
67
+ <<: *debian_packages
68
+ repo_distro: debian/buster
69
+ - <<: *debian
70
+ version: "9"
71
+ packages:
72
+ <<: *debian_packages
73
+ repo_distro: debian/stretch
74
+
75
+ - &fedora
76
+ distro: fedora
77
+ version: "35"
78
+ jsonc_removal: |
79
+ dnf remove -y json-c-devel
80
+ packages: &fedora_packages
81
+ type: rpm
82
+ repo_distro: fedora/35
83
+ arches:
84
+ - amd64
85
+ - armhf
86
+ - arm64
87
+ - <<: *fedora
88
+ version: "34"
89
+ packages:
90
+ <<: *fedora_packages
91
+ repo_distro: fedora/34
92
+
93
+ - &opensuse
94
+ distro: opensuse
95
+ version: "15.3"
96
+ base_image: opensuse/leap
97
+ jsonc_removal: |
98
+ zypper rm -y libjson-c-devel
99
+ packages:
100
+ type: rpm
101
+ repo_distro: opensuse/15.3
102
+ arches:
103
+ - amd64
104
+ - arm64
105
+
106
+ - distro: oraclelinux
107
+ version: "8"
108
+ jsonc_removal: |
109
+ dnf remove -y json-c-devel
110
+ packages:
111
+ type: rpm
112
+ repo_distro: ol/8
113
+ arches:
114
+ - amd64
115
+ - arm64
116
+
117
+ - &ubuntu
118
+ distro: ubuntu
119
+ version: "21.10"
120
+ env_prep: |
121
+ rm -f /etc/apt/apt.conf.d/docker && apt-get update
122
+ jsonc_removal: |
123
+ apt-get remove -y libjson-c-dev
124
+ packages: &ubuntu_packages
125
+ type: deb
126
+ repo_distro: ubuntu/impish
127
+ arches:
128
+ - amd64
129
+ - armhf
130
+ - arm64
131
+ - <<: *ubuntu
132
+ version: "20.04"
133
+ packages:
134
+ <<: *ubuntu_packages
135
+ repo_distro: ubuntu/focal
136
+ - <<: *ubuntu
137
+ version: "18.04"
138
+ packages:
139
+ <<: *ubuntu_packages
140
+ repo_distro: ubuntu/bionic
141
+ arches:
142
+ - i386
143
+ - amd64
144
+ - armhf
145
+ - arm64
.github/workflows/build.yml
+23
-7
@@ -129,13 +129,29 @@ jobs:
129
uses: actions/checkout@v2
130
- name: Prepare tools
131
run: |
132
- sudo apt-get update && sudo apt-get install -y jq
132
+ sudo apt-get update && sudo apt-get install -y python3-ruamel.yaml
133
- name: Read build matrix
134
id: set-matrix
135
+ shell: python3 {0}
136
run: |
136
- TASKS="$(jq -c . .github/data/build-matrix.json)"
137
- echo "Generated Matrix: $TASKS"
138
- echo "::set-output name=matrix::$TASKS"
137
+ from ruamel.yaml import YAML
138
+ import json
139
+ yaml = YAML(typ='safe')
140
+ with open('.github/data/distros.yml') as f:
141
+ data = yaml.load(f)
142
+ del data['platform_map']
143
+ for i, v in enumerate(data['include']):
144
+ data['include'][i]['artifact_key'] = data['include'][i]['distro'] + str(data['include'][i]['version']).replace('.', '')
145
+ if 'packages' in data['include'][i]:
146
+ del data['include'][i]['packages']
147
+ if 'base_image' in data['include'][i]:
148
+ data['include'][i]['distro'] = data['include'][i]['base_image']
149
+ del data['include'][i]['base_image']
150
+ data['include'][i]['distro'] = ':'.join([data['include'][i]['distro'], str(data['include'][i]['version'])])
151
+ del data['include'][i]['version']
152
+ matrix = json.dumps(data, sort_keys=True)
153
+ print('Generated Matrix: ' + matrix)
154
+ print('::set-output name=matrix::' + matrix)
155
156
prepare-test-images: # Prepare the test environments for our build checks. This also checks dependency handling code for each tested environment.
157
name: Prepare Test Environments
@@ -160,8 +176,8 @@ jobs:
176
file: .github/dockerfiles/Dockerfile.build_test
177
build-args: |
178
BASE=${{ matrix.distro }}
163
- PRE=${{ matrix.pre }}
164
- RMJSONC=${{ matrix.rmjsonc }}
179
+ PRE=${{ matrix.env_prep }}
180
+ RMJSONC=${{ matrix.jsonc_removal }}
181
outputs: type=oci,dest=/tmp/image.tar
182
tags: test:${{ matrix.artifact_key }}
183
- name: Upload image artifact
@@ -224,7 +240,7 @@ jobs:
240
docker run --security-opt seccomp=unconfined -w /netdata sha256:${{ steps.load.outputs.image }} \
241
/bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud'
242
- name: netdata-installer on ${{ matrix.distro }}, require cloud, no JSON-C
227
- if: matrix.rmjsonc != ''
243
+ if: matrix.jsonc_removal != ''
244
run: |
245
docker run --security-opt seccomp=unconfined -w /netdata sha256:${{ steps.load.outputs.image }} \
246
/bin/sh -c '/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud'
.github/workflows/packaging.yml
+44
-37
@@ -21,49 +21,56 @@ concurrency:
21
group: packages-${{ github.ref }}-${{ github.event_name }}
22
cancel-in-progress: true
23
jobs:
24
+ matrix:
25
+ name: Prepare Build Matrix
26
+ runs-on: ubuntu-latest
27
+ outputs:
28
+ matrix: ${{ steps.set-matrix.outputs.matrix }}
29
+ steps:
30
+ - name: Checkout
31
+ uses: actions/checkout@v2
32
+ - name: Prepare tools
33
+ run: |
34
+ sudo apt-get update && sudo apt-get install -y python3-ruamel.yaml
35
+ - name: Read build matrix
36
+ id: set-matrix
37
+ shell: python3 {0}
38
+ run: |
39
+ from ruamel.yaml import YAML
40
+ import json
41
+ yaml = YAML(typ='safe')
42
+ entries = list()
43
+
44
+ with open('.github/data/distros.yml') as f:
45
+ data = yaml.load(f)
46
+
47
+ for i, v in enumerate(data['include']):
48
+ if 'packages' in data['include'][i]:
49
+ for arch in data['include'][i]['packages']['arches']:
50
+ entries.append({
51
+ 'distro': data['include'][i]['distro'],
52
+ 'version': data['include'][i]['version'],
53
+ 'pkgclouddistro': data['include'][i]['packages']['repo_distro'],
54
+ 'format': data['include'][i]['packages']['type'],
55
+ 'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else data['include'][i]['distro'],
56
+ 'platform': data['platform_map'][arch],
57
+ 'arch': arch
58
+ })
59
+
60
+ entries.sort(key=lambda k: (k['arch'], k['distro'], k['version']))
61
+ matrix = json.dumps({'include': entries}, sort_keys=True)
62
+ print('Generated Matrix: ' + matrix)
63
+ print('::set-output name=matrix::' + matrix)
64
+
65
build:
66
name: Build
67
runs-on: ubuntu-latest
68
env:
69
DOCKER_CLI_EXPERIMENTAL: enabled
70
+ needs:
71
+ - matrix
72
strategy:
30
- matrix:
31
- include:
32
- - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
33
- - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/i386, arch: i386}
34
- - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/arm/v7, arch: armhf}
35
- - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/arm64/v8, arch: arm64}
36
- - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
37
- - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/i386, arch: i386}
38
- - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/arm/v7, arch: armhf}
39
- - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/arm64/v8, arch: arm64}
40
- - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/amd64, arch: amd64, alias: bullseye}
41
- - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/i386, arch: i386, alias: bullseye}
42
- - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/arm/v7, arch: armhf, alias: bullseye}
43
- - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/arm64/v8, arch: arm64, alias: bullseye}
44
- - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
45
- - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/i386, arch: i386}
46
- - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/arm/v7, arch: armhf}
47
- - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/arm64/v8, arch: arm64}
48
- - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
49
- - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/arm/v7, arch: armhf}
50
- - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/arm64/v8, arch: arm64}
51
- - {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
52
- - {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/arm/v7, arch: armhf}
53
- - {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/arm64/v8, arch: arm64}
54
- - {distro: centos, version: "7", pkgclouddistro: el/7, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64}
55
- - {distro: rockylinux, version: "8", pkgclouddistro: el/8, format: rpm, base_image: rockylinux/rockylinux, platform: linux/amd64, arch: amd64}
56
- - {distro: rockylinux, version: "8", pkgclouddistro: el/8, format: rpm, base_image: rockylinux/rockylinux, platform: linux/arm64/v8, arch: arm64}
57
- - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
58
- - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/arm/v7, arch: armhf}
59
- - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/arm64/v8, arch: arm64}
60
- - {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
61
- - {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/arm/v7, arch: armhf}
62
- - {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/arm64/v8, arch: arm64}
63
- - {distro: opensuse, version: "15.3", pkgclouddistro: opensuse/15.3, format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64}
64
- - {distro: opensuse, version: "15.3", pkgclouddistro: opensuse/15.3, format: rpm, base_image: opensuse/leap, platform: linux/arm64/v8, arch: arm64}
65
- - {distro: oraclelinux, version: "8", pkgclouddistro: ol/8, format: rpm, base_image: oraclelinux, platform: linux/amd64, arch: amd64}
66
- - {distro: oraclelinux, version: "8", pkgclouddistro: ol/8, format: rpm, base_image: oraclelinux, platform: linux/arm64/v8, arch: arm64}
73
+ matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
74
# We intentiaonally disable the fail-fast behavior so that a
75
# build failure for one version doesn't prevent us from publishing
76
# successfully built and tested packages for another version.
.github/workflows/repoconfig-packages.yml
+43
-15
@@ -16,28 +16,56 @@ on:
16
env:
17
DISABLE_TELEMETRY: 1
18
jobs:
19
+ matrix:
20
+ name: Prepare Build Matrix
21
+ runs-on: ubuntu-latest
22
+ outputs:
23
+ matrix: ${{ steps.set-matrix.outputs.matrix }}
24
+ steps:
25
+ - name: Checkout
26
+ uses: actions/checkout@v2
27
+ - name: Prepare tools
28
+ run: |
29
+ sudo apt-get update && sudo apt-get install -y python3-ruamel.yaml
30
+ - name: Read build matrix
31
+ id: set-matrix
32
+ shell: python3 {0}
33
+ run: |
34
+ from ruamel.yaml import YAML
35
+ import json
36
+ yaml = YAML(typ='safe')
37
+ entries = list()
38
+
39
+ with open('.github/data/distros.yml') as f:
40
+ data = yaml.load(f)
41
+
42
+ for i, v in enumerate(data['include']):
43
+ if 'packages' in data['include'][i]:
44
+ entries.append({
45
+ 'distro': data['include'][i]['distro'],
46
+ 'version': data['include'][i]['version'],
47
+ 'pkgclouddistro': data['include'][i]['packages']['repo_distro'],
48
+ 'format': data['include'][i]['packages']['type'],
49
+ 'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else data['include'][i]['distro'],
50
+ 'platform': data['platform_map']['amd64'],
51
+ 'arch': 'amd64'
52
+ })
53
+
54
+ entries.sort(key=lambda k: (k['arch'], k['distro'], k['version']))
55
+ matrix = json.dumps({'include': entries}, sort_keys=True)
56
+ print('Generated Matrix: ' + matrix)
57
+ print('::set-output name=matrix::' + matrix)
58
+
59
build:
60
name: Build
61
runs-on: ubuntu-latest
62
env:
63
DISABLE_TELEMETRY: 1
64
DOCKER_CLI_EXPERIMENTAL: enabled
65
+ needs:
66
+ - matrix
67
strategy:
26
- # This needs to be kept in sync with the matrix in packaging.yml, but should only include the AMD64 lines.
27
- matrix:
28
- include:
29
- - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
30
- - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
31
- - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
32
- - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
33
- - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
34
- - {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
35
- - {distro: centos, version: "7", pkgclouddistro: el/7, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64}
36
- - {distro: rockylinux, version: "8", pkgclouddistro: el/8, format: rpm, base_image: rockylinux/rockylinux, platform: linux/amd64, arch: amd64}
37
- - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
38
- - {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
39
- - {distro: opensuse, version: "15.3", pkgclouddistro: opensuse/15.3, format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64}
40
- - {distro: oraclelinux, version: "8", pkgclouddistro: ol/8, format: rpm, base_image: oraclelinux, platform: linux/amd64, arch: amd64}
68
+ matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
69
# We intentiaonally disable the fail-fast behavior so that a
70
# build failure for one version doesn't prevent us from publishing
71
# successfully built and tested packages for another version.