Fix handling of binary package uploads. (#10860)
Austin S. Hemmelgarn committed
Mar 25, 2021 at 15:08 UTC
f3881e1cd1d2e67275ea4a66760c742b14ac430a
6 files changed
+36
-71
.github/scripts/package_cloud_wrapper.sh
+1
-1
@@ -29,7 +29,7 @@ fi
29
# Install dependency if not there
30
if ! command -v package_cloud > /dev/null 2>&1; then
31
echo "No package cloud gem found, installing"
32
- gem install -V package_cloud || (echo "Package cloud installation failed. you might want to check if required dependencies are there (ruby gcc gcc-c++ ruby-devel)" && exit 1)
32
+ sudo gem install -V package_cloud || (echo "Package cloud installation failed. you might want to check if required dependencies are there (ruby gcc gcc-c++ ruby-devel)" && exit 1)
33
else
34
echo "Found package_cloud gem, continuing"
35
fi
.github/scripts/parse_packagecloud_dist_id.py
deleted
-39
@@ -1,39 +0,0 @@
1
-#!/usr/bin/env python3
2
-'''
3
-Parse the PackageCloud distributions JSON data to get a dist ID for uploads.
4
-
5
-This takes the JSON distributions data from Packagecloud on stdin and
6
-the package format, distribution name and version as arguments, and
7
-prints either an error message or the parsed distribution ID based on
8
-the arguments.
9
-'''
10
-
11
-import json
12
-import sys
13
-
14
-fmt = sys.argv[1] # The package format ('deb' or 'rpm')
15
-distro = sys.argv[2] # The distro name
16
-version = sys.argv[3] # The distro version
17
-print(fmt)
18
-print(distro)
19
-print(version)
20
-
21
-data = json.load(sys.stdin)
22
-versions = []
23
-
24
-for entry in data[fmt]:
25
- if entry['display_name'] == distro:
26
- versions = entry['versions']
27
- break
28
-
29
-if not versions:
30
- print('Could not find version information for the requested distribution.')
31
- sys.exit(-1)
32
-
33
-for entry in versions:
34
- if entry['version_number'] == version:
35
- print(entry['id'])
36
- sys.exit(0)
37
-
38
-print('Unable to find id for requested version.')
39
-sys.exit(-1)
.github/workflows/packaging.yml
+28
-26
@@ -24,23 +24,23 @@ jobs:
24
strategy:
25
matrix:
26
include:
27
- - {distro: debian, version: "9", pkgclouddistro: Debian, pkgcloudversion: "9.0", format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
28
- - {distro: debian, version: "9", pkgclouddistro: Debian, pkgcloudversion: "9.0", format: deb, base_image: debian, platform: linux/i386, arch: i386}
29
- - {distro: debian, version: "10", pkgclouddistro: Debian, pkgcloudversion: "10.0", format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
30
- - {distro: debian, version: "10", pkgclouddistro: Debian, pkgcloudversion: "10.0", format: deb, base_image: debian, platform: linux/i386, arch: i386}
31
- - {distro: ubuntu, version: "16.04", pkgclouddistro: Ubuntu, pkgcloudversion: "16.04", format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
32
- - {distro: ubuntu, version: "16.04", pkgclouddistro: Ubuntu, pkgcloudversion: "16.04", format: deb, base_image: ubuntu, platform: linux/i386, arch: i386}
33
- - {distro: ubuntu, version: "18.04", pkgclouddistro: Ubuntu, pkgcloudversion: "18.04", format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
34
- - {distro: ubuntu, version: "18.04", pkgclouddistro: Ubuntu, pkgcloudversion: "18.04", format: deb, base_image: ubuntu, platform: linux/i386, arch: i386}
35
- - {distro: ubuntu, version: "20.04", pkgclouddistro: Ubuntu, pkgcloudversion: "20.04", format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
36
- - {distro: ubuntu, version: "20.10", pkgclouddistro: Ubuntu, pkgcloudversion: "20.10", format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
37
- - {distro: ubuntu, version: "21.04", pkgclouddistro: Ubuntu, pkgcloudversion: "20.10", format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
38
- - {distro: centos, version: "7", pkgclouddistro: Enterprise Linux, pkgcloudversion: "7.0", format: rpm, base_image: centos, platform: linux/amd64, arch: amd64}
39
- - {distro: centos, version: "8", pkgclouddistro: Enterprise Linux, pkgcloudversion: "8.0", format: rpm, base_image: centos, platform: linux/amd64, arch: amd64}
40
- - {distro: fedora, version: "32", pkgclouddistro: Fedora, pkgcloudversion: "32.0", format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
41
- - {distro: fedora, version: "33", pkgclouddistro: Fedora, pkgcloudversion: "33.0", format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
42
- - {distro: fedora, version: "34", pkgclouddistro: Fedora, pkgcloudversion: "33.0", format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
43
- - {distro: opensuse, version: "15.2", pkgclouddistro: openSUSE, pkgcloudversion: "15.2", format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64}
27
+ - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
28
+ - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/i386, arch: i386}
29
+ - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
30
+ - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/i386, arch: i386}
31
+ - {distro: ubuntu, version: "16.04", pkgclouddistro: ubuntu/xenial, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
32
+ - {distro: ubuntu, version: "16.04", pkgclouddistro: ubuntu/xenial, format: deb, base_image: ubuntu, platform: linux/i386, arch: i386}
33
+ - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
34
+ - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/i386, arch: i386}
35
+ - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
36
+ - {distro: ubuntu, version: "20.10", pkgclouddistro: ubuntu/groovy, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
37
+ - {distro: ubuntu, version: "21.04", pkgclouddistro: ubuntu/hirsute, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
38
+ - {distro: centos, version: "7", pkgclouddistro: el/7, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64}
39
+ - {distro: centos, version: "8", pkgclouddistro: el/8, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64}
40
+ - {distro: fedora, version: "32", pkgclouddistro: fedora/32, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
41
+ - {distro: fedora, version: "33", pkgclouddistro: fedora/33, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
42
+ - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
43
+ - {distro: opensuse, version: "15.2", pkgclouddistro: opensuse/15.2, format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64}
44
# We intentiaonally disable the fail-fast behavior so that a
45
# build failure for one version doesn't prevent us from publishing
46
# successfully built and tested packages for another version.
@@ -51,12 +51,14 @@ jobs:
51
uses: actions/checkout@v2
52
with:
53
fetch-depth: 0 # We need full history for versioning
54
+ submodules: true
55
- name: Checkout Tag # Otherwise check out the tag that triggered this.
55
- if: github.event_name == 'wrokflow_dispatch'
56
+ if: github.event_name == 'workflow_dispatch'
57
uses: actions/checkout@v2
58
with:
58
- refs: ${{ github.event.ref }}
59
+ ref: ${{ github.event.ref }}
60
fetch-depth: 0 # We need full history for versioning
61
+ submodules: true
62
- name: Check Base Branch
63
run: |
64
if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
@@ -74,7 +76,7 @@ jobs:
76
;;
77
*)
78
echo "repo=${{ secrets.PACKAGE_CLOUD_REPO }}-devel" >> $GITHUB_ENV
77
- echo "pkg_version=$(git describe | sed -e 's/^v//')" >> $GITHUB_ENV
79
+ echo "pkg_version=0.${GITHUB_SHA}" >> $GITHUB_ENV
80
echo "pkg_retention_days=30" >> $GITHUB_ENV
81
;;
82
esac
@@ -114,13 +116,13 @@ jobs:
116
- name: Upload
117
if: github.event_name == 'workflow_dispatch'
118
shell: bash
119
+ env:
120
+ PKG_CLOUD_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_KEY }}
121
run: |
118
- # This figures out the distribution ID for the upload.
119
- DIST_ID="$(curl https://${{ secrets.PACKAGE_CLOUD_API_KEY }}:@packagecloud.io/api/v1/distributions.json | ${GITHUB_WORKSPACE}/.github/scripts/parse_packagecloud_dist_id.py ${{ matrix.format }} ${{ matrix.pkgclouddistro }} ${{ matrix.pkgcloudversion }})"
120
- for pkgfile in artifacts/*.${FORMAT} ; do
121
- curl -F "package[distro_version_id]=${BUILD_ID}" \
122
- -F "package[package_file]=@${pkgfile}" \
123
- https://${{ secrets.PACKAGE_CLOUD_API_KEY }}:@packagecloud.io/api/v1/repos/${{ env.repo }}/packages.json || exit 1
122
+ echo "Packages to upload:\n$(ls artifacts/*.${{ matrix.format }})"
123
+ for pkgfile in artifacts/*.${{ matrix.format }} ; do
124
+ .github/scripts/package_cloud_wrapper.sh push ${{ env.repo }}/${{ matrix.pkgclouddistro }} ${pkgfile}
125
+ done
126
- name: Clean
127
if: github.event_name == 'workflow_dispatch'
128
shell: bash
packaging/Dockerfile.packager
+1
-1
@@ -36,7 +36,7 @@ ENV VERSION=$PKG_VERSION
36
COPY ./packaging/scripts/install.sh /install.sh
37
COPY ./packaging/scripts/test.sh /test.sh
38
39
-COPY --from=build /netdata/artifacts /artifacts
39
+COPY --from=build /netdata/artifacts /packages
40
41
RUN /install.sh
42
packaging/scripts/install.sh
+4
-4
@@ -7,7 +7,7 @@ install_debian_like() {
7
apt-get update
8
9
# Install NetData
10
- apt-get install -y "/artifacts/netdata_${VERSION}_${ARCH}.deb"
10
+ apt-get install -y "/packages/netdata_${VERSION}_${ARCH}.deb"
11
12
# Install testing tools
13
apt-get install -y --no-install-recommends \
@@ -23,7 +23,7 @@ install_fedora_like() {
23
pkg_version="$(echo "${VERSION}" | tr - .)"
24
25
# Install NetData
26
- "$PKGMGR" install -y /artifacts/netdata-"${pkg_version}"-*.rpm
26
+ "$PKGMGR" install -y /packages/netdata-"${pkg_version}"-*.rpm
27
28
# Install testing tools
29
"$PKGMGR" install -y curl nc jq
@@ -41,7 +41,7 @@ install_centos() {
41
"$PKGMGR" install -y epel-release
42
43
# Install NetData
44
- "$PKGMGR" install -y /artifacts/netdata-"${pkg_version}"-*.rpm
44
+ "$PKGMGR" install -y /packages/netdata-"${pkg_version}"-*.rpm
45
46
# Install testing tools
47
"$PKGMGR" install -y curl nc jq
@@ -56,7 +56,7 @@ install_suse_like() {
56
# Install NetData
57
# FIXME: Allow unsigned packages (for now) #7773
58
zypper install -y --allow-unsigned-rpm \
59
- /artifacts/netdata-"${pkg_version}"-*.rpm
59
+ /packages/netdata-"${pkg_version}"-*.rpm
60
61
# Install testing tools
62
zypper install -y --no-recommends \
packaging/scripts/test.sh
+2
@@ -39,3 +39,5 @@ cat ./response
39
jq '.version' ./response || exit 1
40
41
trap - EXIT
42
+
43
+cp -a /packages/* /artifacts