Upload packages to new self-hosted repository infrastructure. (#13240)
* Match packaging arches to distro architecture names. This will simplify handling of uploads to our new self-hosted packaging infrastructure. * Add package uploads for RPM packages on new infrastructure. * Add package upload support for DEB packages. * Saner tmpdir handling. * Upload fixes. * Differentiate different DEB distros in package names. * Workflow fixes.
Austin S. Hemmelgarn committed
Jun 28, 2022 at 09:20 UTC
e86cec2631c961b434031e2e09597701a9ec53f8
4 files changed
+92
-20
.github/data/distros.yml
+23
-12
@@ -1,10 +1,21 @@
1
# This defines the full set of distros we run CI on.
2
---
3
platform_map: # map packaging architectures to docker platforms
4
- i386: linux/i386
4
+ aarch64: linux/arm64/v8
5
amd64: linux/amd64
6
- armhf: linux/arm/v7
6
arm64: linux/arm64/v8
7
+ armhf: linux/arm/v7
8
+ armhfp: linux/arm/v7
9
+ i386: linux/i386
10
+ x86_64: linux/amd64
11
+arch_order: # sort order for per-architecture jobs in CI
12
+ - amd64
13
+ - x86_64
14
+ - i386
15
+ - armhf
16
+ - armhfp
17
+ - arm64
18
+ - aarch64
19
include:
20
- &alpine
21
distro: alpine
@@ -37,8 +48,8 @@ include:
48
type: rpm
49
repo_distro: el/9
50
arches:
40
- - amd64
41
- - arm64
51
+ - x86_64
52
+ - aarch64
53
- <<: *alma
54
version: "8"
55
packages:
@@ -51,7 +62,7 @@ include:
62
type: rpm
63
repo_distro: el/7
64
arches:
54
- - amd64
65
+ - x86_64
66
67
- &debian
68
distro: debian
@@ -88,9 +99,9 @@ include:
99
type: rpm
100
repo_distro: fedora/36
101
arches:
91
- - amd64
92
- - armhf
93
- - arm64
102
+ - x86_64
103
+ - armhfp
104
+ - aarch64
105
- <<: *fedora
106
version: "35"
107
packages:
@@ -107,8 +118,8 @@ include:
118
type: rpm
119
repo_distro: opensuse/15.4
120
arches:
110
- - amd64
111
- - arm64
121
+ - x86_64
122
+ - aarch64
123
- <<: *opensuse
124
version: "15.3"
125
packages:
@@ -123,8 +134,8 @@ include:
134
type: rpm
135
repo_distro: ol/8
136
arches:
126
- - amd64
127
- - arm64
137
+ - x86_64
138
+ - aarch64
139
140
- &ubuntu
141
distro: ubuntu
.github/scripts/package-upload.sh
new
+43
@@ -0,0 +1,43 @@
1
+#!/bin/sh
2
+
3
+set -e
4
+
5
+host="packages.netdata.cloud"
6
+user="netdatabot"
7
+
8
+distro="${1}"
9
+arch="${2}"
10
+format="${3}"
11
+repo="${4}"
12
+
13
+staging="${TMPDIR:-/tmp}/package-staging"
14
+prefix="/var/www/html/repos/${repo}/"
15
+
16
+packages="$(find artifacts -name "*.${format}")"
17
+
18
+mkdir -p "${staging}"
19
+
20
+case "${format}" in
21
+ deb)
22
+ src="${staging}/$(echo "${distro}" | cut -f 1 -d '/')/pool/"
23
+ mkdir -p "${src}"
24
+
25
+ for pkg in ${packages}; do
26
+ cp "${pkg}" "${src}"
27
+ done
28
+ ;;
29
+ rpm)
30
+ src="${staging}/${distro}/${arch}/"
31
+ mkdir -p "${src}"
32
+
33
+ for pkg in ${packages}; do
34
+ cp "${pkg}" "${src}"
35
+ done
36
+ ;;
37
+ *)
38
+ echo "Unrecognized package format ${format}."
39
+ exit 1
40
+ ;;
41
+esac
42
+
43
+rsync -vrptO "${staging}/" "${user}@${host}:${prefix}"
.github/workflows/packaging.yml
+24
-5
@@ -65,14 +65,14 @@ jobs:
65
entries.append({
66
'distro': data['include'][i]['distro'],
67
'version': data['include'][i]['version'],
68
- 'pkgclouddistro': data['include'][i]['packages']['repo_distro'],
68
+ 'repo_distro': data['include'][i]['packages']['repo_distro'],
69
'format': data['include'][i]['packages']['type'],
70
'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else data['include'][i]['distro'],
71
'platform': data['platform_map'][arch],
72
'arch': arch
73
})
74
75
- entries.sort(key=lambda k: (k['arch'], k['distro'], k['version']))
75
+ entries.sort(key=lambda k: (data['arch_order'].index(k['arch']), k['distro'], k['version']))
76
matrix = json.dumps({'include': entries}, sort_keys=True)
77
print('Generated Matrix: ' + matrix)
78
print('::set-output name=matrix::' + matrix)
@@ -223,10 +223,27 @@ jobs:
223
run: |
224
printf "Packages to upload:\n%s" "$(ls artifacts/*.${{ matrix.format }})"
225
for pkgfile in artifacts/*.${{ matrix.format }} ; do
226
- .github/scripts/package_cloud_wrapper.sh yank ${{ needs.version-check.outputs.repo }}/${{ matrix.pkgclouddistro }} \
226
+ .github/scripts/package_cloud_wrapper.sh yank ${{ needs.version-check.outputs.repo }}/${{ matrix.repo_distro }} \
227
"$(basename "${pkgfile}")" || true
228
- .github/scripts/package_cloud_wrapper.sh push ${{ needs.version-check.outputs.repo }}/${{ matrix.pkgclouddistro }} "${pkgfile}"
228
+ .github/scripts/package_cloud_wrapper.sh push ${{ needs.version-check.outputs.repo }}/${{ matrix.repo_distro }} "${pkgfile}"
229
done
230
+ - name: SSH setup
231
+ id: ssh-setup
232
+ if: github.event_name == 'workflow_dispatch'
233
+ uses: shimataro/ssh-key-action@v2
234
+ with:
235
+ key: ${{ secrets.NETDATABOT_PACKAGES_SSH_KEY }}
236
+ name: id_ecdsa
237
+ known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }}
238
+ - name: Upload to packages.netdata.cloud
239
+ id: package-upload
240
+ if: github.event_name == 'workflow_dispatch'
241
+ run: |
242
+ .github/scripts/package-upload.sh \
243
+ ${{ matrix.repo_distro }} \
244
+ ${{ matrix.arch }} \
245
+ ${{ matrix.format }} \
246
+ ${{ needs.version-check.outputs.repo }}
247
- name: Failure Notification
248
uses: rtCamp/action-slack-notify@v2
249
env:
@@ -235,7 +252,7 @@ jobs:
252
SLACK_TITLE: 'Package Build failed:'
253
SLACK_USERNAME: 'GitHub Actions'
254
SLACK_MESSAGE: |-
238
- ${{ github.repository }}: ${{ matrix.pkgclouddistro }} ${{ matrix.version }} package build for ${{ matrix.arch }} failed.
255
+ ${{ github.repository }}: ${{ matrix.repo_distro }} ${{ matrix.version }} package build for ${{ matrix.arch }} failed.
256
Checkout: ${{ steps.checkout.outcome }}
257
Setup QEMU: ${{ steps.qemu.outcome }}
258
Setup Docker: ${{ steps.docker-config.outcome }}
@@ -243,6 +260,8 @@ jobs:
260
Build: ${{ steps.build.outcome }}
261
Test: ${{ steps.test.outcome }}
262
Publish: ${{ steps.upload.outcome }}
263
+ Import SSH Key: ${{ steps.ssh-setup.outcome }}
264
+ Publish: ${{ steps.package-upload.outcome }}
265
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
266
if: >-
267
${{
.github/workflows/repoconfig-packages.yml
+2
-3
@@ -50,11 +50,10 @@ jobs:
50
'pkgclouddistro': data['include'][i]['packages']['repo_distro'],
51
'format': data['include'][i]['packages']['type'],
52
'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else data['include'][i]['distro'],
53
- 'platform': data['platform_map']['amd64'],
54
- 'arch': 'amd64'
53
+ 'platform': data['platform_map']['amd64']
54
})
55
57
- entries.sort(key=lambda k: (k['arch'], k['distro'], k['version']))
56
+ entries.sort(key=lambda k: (k['distro'], k['version']))
57
matrix = json.dumps({'include': entries}, sort_keys=True)
58
print('Generated Matrix: ' + matrix)
59
print('::set-output name=matrix::' + matrix)