Docker-based packaging workflow in GitHub Actions. (#9964)
* Add a new GHA workflow for building and publishing binary packages. This greatly simplifies the package handling code, significantly reduces the amount of things we are running in Travis, adds better CI for package builds for PRs, finally eliminates LXC from any of our CI, and enables us to make further major improvements much easier. * Migrate repo cleanup to GHA and remove dead code. * Fix RPM package builds. * Fix DEB installation.
Austin S. Hemmelgarn committed
Mar 15, 2021 at 08:31 UTC
e54982b6c814d1c6e1dda456fcf691d9b829eb91
25 files changed
+252
-1248
.github/scripts/old_package_purging.sh
renamed
+6
-11
@@ -26,7 +26,7 @@ delete_files_for_version() {
26
pkg=${pkg/\"/}
27
pkg=${pkg/\"/}
28
echo "Attempting yank on ${pkg}.."
29
- .travis/package_management/package_cloud_wrapper.sh yank "${PACKAGING_USER}/${DEPLOY_REPO}" "${pkg}" || echo "Nothing to yank or error on ${pkg}"
29
+ .github/scripts/package_cloud_wrapper.sh yank "${REPO}" "${pkg}" || echo "Nothing to yank or error on ${pkg}"
30
done
31
}
32
@@ -34,18 +34,13 @@ delete_files_for_version() {
34
TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
35
CWD=$(git rev-parse --show-cdup)
36
if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
37
- echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository"
37
+ echo "Run as .github/scripts/$(basename "$0") from top level directory of netdata git repository"
38
echo "Old packages yanking cancelled"
39
exit 1
40
fi
41
42
-if [ -z "${PACKAGING_USER}" ]; then
43
- echo "No PACKAGING_USER variable found"
44
- exit 1
45
-fi
46
-
47
-if [ -z "${DEPLOY_REPO}" ]; then
48
- echo "No DEPLOY_REPO variable found"
42
+if [ -z "${REPO}" ]; then
43
+ echo "No REPO variable found"
44
exit 1
45
fi
46
@@ -68,8 +63,8 @@ DATE_UNTIL_TO_DELETE=$(date --date="${PACKAGE_CLOUD_RETENTION_DAYS} day ago" +%Y
63
echo "Created temp directory: ${TMP_DIR}"
64
echo "We will be purging contents up until ${DATE_UNTIL_TO_DELETE}"
65
71
-echo "Calling package could to retrieve all available packages on ${PACKAGING_USER}/${DEPLOY_REPO}"
72
-curl -sS "https://${PKG_CLOUD_TOKEN}:@packagecloud.io/api/v1/repos/${PACKAGING_USER}/${DEPLOY_REPO}/packages.json" > "${PKG_LIST_FILE}"
66
+echo "Calling package could to retrieve all available packages on ${REPO}"
67
+curl -sS "https://${PKG_CLOUD_TOKEN}:@packagecloud.io/api/v1/repos/${REPO}/packages.json" > "${PKG_LIST_FILE}"
68
69
# Get versions within the desired duration
70
#
.github/scripts/package_cloud_wrapper.sh
renamed
+1
-1
@@ -21,7 +21,7 @@ PKG_CLOUD_CONFIG="$HOME/.package_cloud_configuration.cfg"
21
TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
22
CWD=$(git rev-parse --show-cdup)
23
if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
24
- echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository"
24
+ echo "Run as .github/scripts/$(basename "$0") from top level directory of netdata git repository"
25
echo "Docker build process aborted"
26
exit 1
27
fi
.github/scripts/parse_packagecloud_dist_id.py
new
+39
@@ -0,0 +1,39 @@
1
+#!/usr/bin/env python3
2
+'''
3
+Parse the PackageCloud distributions JSON data to get a dist ID for uploads.
4
+
5
+This takes the JSON distributions data from Packagecloud on stdin and
6
+the package format, distribution name and version as arguments, and
7
+prints either an error message or the parsed distribution ID based on
8
+the arguments.
9
+'''
10
+
11
+import json
12
+import sys
13
+
14
+fmt = sys.argv[1] # The package format ('deb' or 'rpm')
15
+distro = sys.argv[2] # The distro name
16
+version = sys.argv[3] # The distro version
17
+print(fmt)
18
+print(distro)
19
+print(version)
20
+
21
+data = json.load(sys.stdin)
22
+versions = []
23
+
24
+for entry in data[fmt]:
25
+ if entry['display_name'] == distro:
26
+ versions = entry['versions']
27
+ break
28
+
29
+if not versions:
30
+ print('Could not find version information for the requested distribution.')
31
+ sys.exit(-1)
32
+
33
+for entry in versions:
34
+ if entry['version_number'] == version:
35
+ print(entry['id'])
36
+ sys.exit(0)
37
+
38
+print('Unable to find id for requested version.')
39
+sys.exit(-1)
.github/workflows/packaging.yml
new
+129
@@ -0,0 +1,129 @@
1
+---
2
+# Handles building of binary packages for the agent.
3
+name: Packages
4
+on:
5
+ pull_request:
6
+ branches:
7
+ - master
8
+ - develop
9
+ workflow_dispatch:
10
+ inputs:
11
+ type:
12
+ name: Package build type
13
+ default: devel
14
+ required: true
15
+ version:
16
+ name: Package version
17
+ required: false
18
+jobs:
19
+ build:
20
+ name: Build
21
+ runs-on: ubuntu-latest
22
+ env:
23
+ DOCKER_CLI_EXPERIMENTAL: enabled
24
+ strategy:
25
+ matrix:
26
+ include:
27
+ - {distro: debian, version: "9", pkgclouddistro: Debian, pkgcloudversion: "9.0", format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
28
+ - {distro: debian, version: "9", pkgclouddistro: Debian, pkgcloudversion: "9.0", format: deb, base_image: debian, platform: linux/i386, arch: i386}
29
+ - {distro: debian, version: "10", pkgclouddistro: Debian, pkgcloudversion: "10.0", format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
30
+ - {distro: debian, version: "10", pkgclouddistro: Debian, pkgcloudversion: "10.0", format: deb, base_image: debian, platform: linux/i386, arch: i386}
31
+ - {distro: ubuntu, version: "16.04", pkgclouddistro: Ubuntu, pkgcloudversion: "16.04", format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
32
+ - {distro: ubuntu, version: "16.04", pkgclouddistro: Ubuntu, pkgcloudversion: "16.04", format: deb, base_image: ubuntu, platform: linux/i386, arch: i386}
33
+ - {distro: ubuntu, version: "18.04", pkgclouddistro: Ubuntu, pkgcloudversion: "18.04", format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
34
+ - {distro: ubuntu, version: "18.04", pkgclouddistro: Ubuntu, pkgcloudversion: "18.04", format: deb, base_image: ubuntu, platform: linux/i386, arch: i386}
35
+ - {distro: ubuntu, version: "20.04", pkgclouddistro: Ubuntu, pkgcloudversion: "20.04", format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
36
+ - {distro: ubuntu, version: "20.10", pkgclouddistro: Ubuntu, pkgcloudversion: "20.10", format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
37
+ - {distro: centos, version: "7", pkgclouddistro: Enterprise Linux, pkgcloudversion: "7.0", format: rpm, base_image: centos, platform: linux/amd64, arch: amd64}
38
+ - {distro: centos, version: "8", pkgclouddistro: Enterprise Linux, pkgcloudversion: "8.0", format: rpm, base_image: centos, platform: linux/amd64, arch: amd64}
39
+ - {distro: fedora, version: "32", pkgclouddistro: Fedora, pkgcloudversion: "32.0", format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
40
+ - {distro: fedora, version: "33", pkgclouddistro: Fedora, pkgcloudversion: "33.0", format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
41
+ - {distro: opensuse, version: "15.2", pkgclouddistro: openSUSE, pkgcloudversion: "15.2", format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64}
42
+ # We intentiaonally disable the fail-fast behavior so that a
43
+ # build failure for one version doesn't prevent us from publishing
44
+ # successfully built and tested packages for another version.
45
+ fail-fast: false
46
+ steps:
47
+ - name: Checkout PR # Checkout the PR if it's a PR.
48
+ if: github.event_name == 'pull_request'
49
+ uses: actions/checkout@v2
50
+ with:
51
+ fetch-depth: 0 # We need full history for versioning
52
+ - name: Checkout Tag # Otherwise check out the tag that triggered this.
53
+ if: github.event_name == 'wrokflow_dispatch'
54
+ uses: actions/checkout@v2
55
+ with:
56
+ refs: ${{ github.event.ref }}
57
+ fetch-depth: 0 # We need full history for versioning
58
+ - name: Check Base Branch
59
+ run: |
60
+ if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
61
+ echo "runtype=${{ github.event.inputs.type }}" >> $GITHUB_ENV
62
+ case "${{ github.event.inputs.type }}" in
63
+ "release")
64
+ echo "repo=${{ secrets.PACKAGE_CLOUD_REPO }}" >> $GITHUB_ENV
65
+ echo "pkg_version=${{ github.event.inputs.version }}" >> $GITHUB_ENV
66
+ echo "pkg_retention_days=365" >> $GITHUB_ENV
67
+ ;;
68
+ "nightly")
69
+ echo "repo=${{ secrets.PACKAGE_CLOUD_REPO }}-edge" >> $GITHUB_ENV
70
+ echo "pkg_version=${{ github.event.inputs.version }}" >> $GITHUB_ENV
71
+ echo "pkg_retention_days=30" >> $GITHUB_ENV
72
+ ;;
73
+ *)
74
+ echo "repo=${{ secrets.PACKAGE_CLOUD_REPO }}-devel" >> $GITHUB_ENV
75
+ echo "pkg_version=$(git describe | sed -e 's/^v//')" >> $GITHUB_ENV
76
+ echo "pkg_retention_days=30" >> $GITHUB_ENV
77
+ ;;
78
+ esac
79
+ else
80
+ echo "runtype=test" >> $GITHUB_ENV
81
+ echo "pkg_version=$(cut -d'-' -f 1 packaging/version | sed -e 's/^v//')" >> $GITHUB_ENV
82
+ fi
83
+ - name: Setup QEMU
84
+ if: matrix.platform != 'linux/amd64'
85
+ uses: docker/setup-qemu-action@v1
86
+ - name: Setup Buildx
87
+ uses: docker/setup-buildx-action@v1
88
+ - name: Prepare Docker Environment
89
+ shell: bash
90
+ run: |
91
+ echo '{"cgroup-parent": "/actions_job", "experimental": true}' | sudo tee /etc/docker/daemon.json 2>/dev/null
92
+ sudo service docker restart
93
+ - name: Build Packages
94
+ uses: docker/build-push-action@v2
95
+ with:
96
+ platforms: ${{ matrix.platform }}
97
+ file: packaging/Dockerfile.packager
98
+ tags: local/package-builder:${{ matrix.distro}}${{ matrix.version }}
99
+ push: false
100
+ load: true
101
+ build-args: |
102
+ ARCH=${{ matrix.arch }}
103
+ DISTRO=${{ matrix.distro }}
104
+ TEST_BASE=${{ matrix.base_image }}
105
+ DISTRO_VERSION=${{ matrix.version }}
106
+ PKG_VERSION=${{ env.pkg_version }}
107
+ - name: Extract Packages
108
+ shell: bash
109
+ run: |
110
+ mkdir -p artifacts
111
+ docker run --platform ${{ matrix.platform }} -v $PWD/artifacts:/artifacts local/package-builder:${{ matrix.distro }}${{ matrix.version }}
112
+ - name: Upload
113
+ if: github.env.runtype == 'release' || github.env.runtype == 'nightly' || github.env.runtype == 'devel'
114
+ shell: bash
115
+ run: |
116
+ # This figures out the distribution ID for the upload.
117
+ DIST_ID="$(curl https://${{ secrets.PACKAGE_CLOUD_API_TOKEN }}:@packagecloud.io/api/v1/distributions.json | python3 .github/scripts/parse_packagecloud_dist_id.py ${{ matrix.format }} ${{ matrix.pkgclouddistro }} ${{ matrix.pkgcloudversion }})"
118
+ for pkgfile in artifacts/*.${FORMAT} ; do
119
+ curl -F "package[distro_version_id]=${BUILD_ID}" \
120
+ -F "package[package_file]=@${pkgfile}" \
121
+ https://${{ secrets.PACKAGE_CLOUD_API_TOKEN }}:@packagecloud.io/api/v1/repos/${{ env.repo }}/packages.json || exit 1
122
+ - name: Clean
123
+ if: github.env.runtype == 'release' || github.env.runtype == 'nightly' || github.env.runtype == 'devel'
124
+ shell: bash
125
+ env:
126
+ REPO: ${{ env.repo }}
127
+ PKG_CLOUD_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_TOKEN }}
128
+ PACKAGE_CLOUD_RETENTION_DAYS: ${{ env.pkg_retention_days }}
129
+ run: .github/scripts/old_package_purging.sh
.travis.yml
+7
-232
@@ -56,8 +56,6 @@ stages:
56
57
- name: Nightly release
58
if: branch = master AND type = cron AND env(RUN_NIGHTLY) = yes
59
- - name: Trigger deb and rpm package build (nightly release)
60
- if: branch = master AND type = cron AND env(RUN_NIGHTLY) = yes
59
60
# Scheduled releases
61
- name: Support activities on main branch
@@ -71,21 +69,6 @@ stages:
69
AND type != cron
70
AND tag !~ /(-rc)/
71
AND commit_message =~ /\[netdata (release candidate|(major|minor|patch) release)\]/
74
- - name: Trigger deb and rpm package build (release)
75
- if: >-
76
- branch = master
77
- AND type != pull_request
78
- AND type != cron
79
- AND tag !~ /(-rc)/
80
- AND commit_message =~ /\[netdata (release candidate|(major|minor|patch) release)\]/
81
-
82
- # Build DEB packages under special conditions
83
- - name: Package ubuntu/* and debian/*
84
- if: type != cron AND type != pull_request AND branch = master
85
-
86
- # Build RPM packages under special conditions
87
- - name: Package centos, fedora and opensuse
88
- if: type != cron AND type != pull_request AND branch = master
72
73
74
# Define stage implementation details
@@ -111,49 +94,6 @@ jobs:
94
script: docker run -it -v "${PWD}:/netdata:rw" -w /netdata "netdata/os-test:centos7" ./netdata-installer.sh --dont-wait --dont-start-it --install /tmp
95
after_failure: post_message "TRAVIS_MESSAGE" "Build/Install failed on CentOS 7"
96
114
- - name: DEB package test
115
- git:
116
- depth: false
117
- before_install:
118
- - sudo apt-get install -y wget lxc python3-lxc lxc-templates dh-make git-buildpackage build-essential libdistro-info-perl
119
- before_script:
120
- - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}"
121
- script:
122
- - echo "GIT Branch:" && git branch
123
- - echo "Last commit:" && git log -1
124
- - echo "GIT Describe:" && git describe
125
- - echo "packaging/version:" && cat packaging/version
126
- - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh
127
- - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh
128
- - sudo chown -R root:travis "/var/lib/lxc"
129
- - sudo chmod -R 750 "/var/lib/lxc"
130
- - echo "Preparing DEB packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh
131
- env:
132
- - BUILDER_NAME="builder" BUILD_DISTRO="debian" BUILD_RELEASE="buster" BUILD_STRING="debian/buster"
133
- - PACKAGE_TYPE="deb" REPO_TOOL="apt-get"
134
- after_failure: post_message "TRAVIS_MESSAGE" "Netdata DEB package build check failed."
135
-
136
- - name: RPM package test
137
- git:
138
- depth: false
139
- before_install:
140
- - sudo apt-get install -y wget lxc lxc-templates python3-lxc
141
- before_script:
142
- - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}"
143
- script:
144
- - echo "GIT Branch:" && git branch
145
- - echo "Last commit:" && git log -1
146
- - echo "GIT Describe:" && git describe
147
- - echo "packaging/version:" && cat packaging/version
148
- - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh
149
- - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh
150
- - sudo chmod -R 755 "/var/lib/lxc"
151
- - echo "Preparing RPM packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh
152
- env:
153
- - BUILDER_NAME="builder" BUILD_DISTRO="fedora" BUILD_RELEASE="32" BUILD_STRING="fedora/32"
154
- - PACKAGE_TYPE="rpm" REPO_TOOL="dnf"
155
- after_failure: post_message "TRAVIS_MESSAGE" "Netdata RPM package build check failed."
156
-
97
- stage: Support activities on main branch
98
name: Generate changelog for release (only on special and tagged commit msg)
99
before_script: post_message "TRAVIS_MESSAGE" "Support activities on main branch initiated" "${NOTIF_CHANNEL}"
@@ -169,172 +109,6 @@ jobs:
109
depth: false
110
if: commit_message =~ /\[netdata (release candidate|(major|minor|patch) release)\]/ AND tag !~ /(-rc)/ OR (env(GIT_TAG) IS present AND NOT env(GIT_TAG) IS blank)
111
172
- # ###### Packaging workflow section ######
173
- # References:
174
- # https://us.images.linuxcontainers.org
175
- # https://packagecloud.io/docs#install_repo
176
-
177
- # TODO: This section is stale, will be aligned with the RPM implementation when we get to DEB packaging
178
- - stage: Package ubuntu/* and debian/*
179
- _template: &DEB_TEMPLATE
180
- git:
181
- depth: false
182
- before_install:
183
- - sudo apt-get install -y wget lxc python3-lxc lxc-templates dh-make git-buildpackage build-essential libdistro-info-perl
184
- - source tests/installer/slack.sh
185
- before_script:
186
- - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}"
187
- script:
188
- - echo "GIT Branch:" && git branch
189
- - echo "Last commit:" && git log -1
190
- - echo "GIT Describe:" && git describe
191
- - echo "packaging/version:" && cat packaging/version
192
- - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh
193
- - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh
194
- - sudo chown -R root:travis "/var/lib/lxc"
195
- - sudo chmod -R 750 "/var/lib/lxc"
196
- - echo "Preparing DEB packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh
197
- after_failure: post_message "TRAVIS_MESSAGE" "Failed to build DEB for ${BUILD_STRING}.${BUILD_ARCH}"
198
- before_deploy:
199
- - .travis/package_management/yank_stale_pkg.sh "${PACKAGES_DIRECTORY}" "${BUILD_STRING}" || echo "No stale DEB found"
200
- deploy:
201
- - provider: packagecloud
202
- repository: "${DEPLOY_REPO}"
203
- username: "${PACKAGING_USER}"
204
- token: "${PKG_CLOUD_TOKEN}"
205
- dist: "${BUILD_STRING}"
206
- local_dir: "${PACKAGES_DIRECTORY}"
207
- skip_cleanup: true
208
- on:
209
- # Only deploy on ${USER}/netdata, master branch, when build-area directory is created
210
- repo: ${TRAVIS_REPO_SLUG}
211
- branch: "master"
212
- condition: -d "${PACKAGES_DIRECTORY}"
213
- after_deploy:
214
- - if [ -n "${BUILDER_NAME}" ]; then rm -rf /home/${BUILDER_NAME}/* && echo "Cleared /home/${BUILDER_NAME} directory" || echo "Failed to clean /home/${BUILDER_NAME} directory"; fi;
215
- - if [ -d "${PACKAGES_DIRECTORY}" ]; then rm -rf "${PACKAGES_DIRECTORY}"; fi;
216
-
217
- name: "Build & Publish DEB package for ubuntu/bionic"
218
- <<: *DEB_TEMPLATE
219
- if: commit_message =~ /\[Package (amd64|arm64|i386) DEB( Ubuntu)?\]/
220
- env:
221
- - BUILDER_NAME="builder" BUILD_DISTRO="ubuntu" BUILD_RELEASE="bionic" BUILD_STRING="ubuntu/bionic"
222
- - PACKAGE_TYPE="deb" REPO_TOOL="apt-get"
223
- - ALLOW_SOFT_FAILURE_HERE=true
224
-
225
- - name: "Build & Publish DEB package for ubuntu/xenial"
226
- <<: *DEB_TEMPLATE
227
- if: commit_message =~ /\[Package (amd64|arm64|i386) DEB( Ubuntu)?\]/
228
- env:
229
- - BUILDER_NAME="builder" BUILD_DISTRO="ubuntu" BUILD_RELEASE="xenial" BUILD_STRING="ubuntu/xenial"
230
- - PACKAGE_TYPE="deb" REPO_TOOL="apt-get"
231
- - ALLOW_SOFT_FAILURE_HERE=true
232
-
233
- - name: "Build & Publish DEB package for ubuntu/focal"
234
- <<: *DEB_TEMPLATE
235
- if: commit_message =~ /\[Package (amd64|arm64) DEB( Ubuntu)?\]/
236
- env:
237
- - BUILDER_NAME="builder" BUILD_DISTRO="ubuntu" BUILD_RELEASE="focal" BUILD_STRING="ubuntu/focal"
238
- - PACKAGE_TYPE="deb" REPO_TOOL="apt-get"
239
- - ALLOW_SOFT_FAILURE_HERE=true
240
-
241
- - name: "Build & Publish DEB package for ubuntu/groovy"
242
- <<: *DEB_TEMPLATE
243
- if: commit_message =~ /\[Package (amd64|arm64) DEB( Ubuntu)?\]/
244
- env:
245
- - BUILDER_NAME="builder" BUILD_DISTRO="ubuntu" BUILD_RELEASE="groovy" BUILD_STRING="ubuntu/groovy"
246
- - PACKAGE_TYPE="deb" REPO_TOOL="apt-get"
247
- - ALLOW_SOFT_FAILURE_HERE=true
248
-
249
- - name: "Build & Publish DEB package for debian/buster"
250
- <<: *DEB_TEMPLATE
251
- if: commit_message =~ /\[Package (amd64|arm64|i386) DEB( Debian)?\]/
252
- env:
253
- - BUILDER_NAME="builder" BUILD_DISTRO="debian" BUILD_RELEASE="buster" BUILD_STRING="debian/buster"
254
- - PACKAGE_TYPE="deb" REPO_TOOL="apt-get"
255
- - ALLOW_SOFT_FAILURE_HERE=true
256
-
257
- - name: "Build & Publish DEB package for debian/stretch"
258
- <<: *DEB_TEMPLATE
259
- if: commit_message =~ /\[Package (amd64|arm64|i386) DEB( Debian)?\]/
260
- env:
261
- - BUILDER_NAME="builder" BUILD_DISTRO="debian" BUILD_RELEASE="stretch" BUILD_STRING="debian/stretch"
262
- - PACKAGE_TYPE="deb" REPO_TOOL="apt-get"
263
- - ALLOW_SOFT_FAILURE_HERE=true
264
-
265
- - stage: Package centos, fedora and opensuse
266
- _template: &RPM_TEMPLATE
267
- git:
268
- depth: false
269
- before_install:
270
- - sudo apt-get install -y wget lxc lxc-templates python3-lxc
271
- - source tests/installer/slack.sh
272
- before_script:
273
- - export PACKAGES_DIRECTORY="$(mktemp -d -t netdata-packaging-contents-dir-XXXXXX)" && echo "Created packaging directory ${PACKAGES_DIRECTORY}"
274
- script:
275
- - echo "GIT Branch:" && git branch
276
- - echo "Last commit:" && git log -1
277
- - echo "GIT Describe:" && git describe
278
- - echo "packaging/version:" && cat packaging/version
279
- - echo "Creating LXC environment for the build" && sudo -E .travis/package_management/create_lxc_for_build.sh
280
- - echo "Building package in container" && sudo -E .travis/package_management/build_package_in_container.sh
281
- - sudo chmod -R 755 "/var/lib/lxc"
282
- - echo "Preparing RPM packaging contents for upload" && sudo -E .travis/package_management/prepare_packages.sh
283
- after_failure: post_message "TRAVIS_MESSAGE" "Failed to build RPM for ${BUILD_STRING}.${BUILD_ARCH}"
284
- before_deploy:
285
- - .travis/package_management/yank_stale_pkg.sh "${PACKAGES_DIRECTORY}" "${BUILD_STRING}" || echo "No stale RPM found"
286
- deploy:
287
- - provider: packagecloud
288
- repository: "${DEPLOY_REPO}"
289
- username: "${PACKAGING_USER}"
290
- token: "${PKG_CLOUD_TOKEN}"
291
- dist: "${BUILD_STRING}"
292
- local_dir: "${PACKAGES_DIRECTORY}"
293
- skip_cleanup: true
294
- on:
295
- # Only deploy on ${USER}/netdata, master branch, when packages directory is created
296
- repo: ${TRAVIS_REPO_SLUG}
297
- branch: "master"
298
- condition: -d "${PACKAGES_DIRECTORY}"
299
- after_deploy:
300
- - if [ -n "${BUILDER_NAME}" ]; then rm -rf /home/${BUILDER_NAME}/* && echo "Cleared /home/${BUILDER_NAME} directory" || echo "Failed to clean /home/${BUILDER_NAME} directory"; fi;
301
- - if [ -d "${PACKAGES_DIRECTORY}" ]; then rm -rf "${PACKAGES_DIRECTORY}"; fi;
302
-
303
- name: "Build & Publish RPM package for Enterprise Linux 7"
304
- <<: *RPM_TEMPLATE
305
- if: commit_message =~ /\[Package (amd64|arm64) RPM( Enterprise Linux)?\]/
306
- env:
307
- - BUILDER_NAME="builder" BUILD_DISTRO="centos" BUILD_RELEASE="7" BUILD_STRING="el/7"
308
- - PACKAGE_TYPE="rpm" REPO_TOOL="yum"
309
- - ALLOW_SOFT_FAILURE_HERE=true
310
-
311
- - name: "Build & Publish RPM package for Fedora 32"
312
- <<: *RPM_TEMPLATE
313
- if: commit_message =~ /\[Package (amd64|arm64) RPM( Fedora)?\]/
314
- env:
315
- - BUILDER_NAME="builder" BUILD_DISTRO="fedora" BUILD_RELEASE="32" BUILD_STRING="fedora/32"
316
- - PACKAGE_TYPE="rpm" REPO_TOOL="dnf"
317
- - ALLOW_SOFT_FAILURE_HERE=true
318
-
319
- - name: "Build & Publish RPM package for Fedora 33"
320
- <<: *RPM_TEMPLATE
321
- if: commit_message =~ /\[Package (amd64|arm64) RPM( Fedora)?\]/
322
- env:
323
- - BUILDER_NAME="builder" BUILD_DISTRO="fedora" BUILD_RELEASE="33" BUILD_STRING="fedora/33"
324
- - PACKAGE_TYPE="rpm" REPO_TOOL="dnf"
325
- - ALLOW_SOFT_FAILURE_HERE=true
326
-
327
- - name: "Build & Publish RPM package for openSUSE 15.2"
328
- <<: *RPM_TEMPLATE
329
- if: commit_message =~ /\[Package (amd64|arm64) RPM( openSUSE)?\]/
330
- env:
331
- - BUILDER_NAME="builder" BUILD_DISTRO="opensuse" BUILD_RELEASE="15.2" BUILD_STRING="opensuse/15.2"
332
- - PACKAGE_TYPE="rpm" REPO_TOOL="zypper"
333
- - ALLOW_SOFT_FAILURE_HERE=true
334
- # ###### End of packaging workflow section ###### #
335
- # ############################################### #
336
-
337
-
112
# We only publish if a TAG has been set during packaging
113
- stage: Publish for release
114
name: Create release draft
@@ -358,9 +132,10 @@ jobs:
132
- .travis/trigger_docker_build.sh "${GITHUB_TOKEN}" "${BUILD_VERSION}"
133
after_failure: post_message "TRAVIS_MESSAGE" "<!here> Failed to trigger docker build during release" "${NOTIF_CHANNEL}"
134
361
- - stage: Trigger deb and rpm package build (release)
362
- name: Trigger deb and rpm package build
363
- script: .travis/trigger_package_generation.sh
135
+ - name: Trigger deb and rpm package build
136
+ script:
137
+ - git checkout "${TRAVIS_BRANCH}" && export BUILD_VERSION="$(cat packaging/version | cut -d'-' -f1)"
138
+ - .travis/trigger_package_build.sh "${GITHUB_TOKEN}" "${BUILD_VERSION}" "release"
139
after_failure: post_message "TRAVIS_MESSAGE" "<!here> Failed to trigger deb and rpm package build during release" "${NOTIF_CHANNEL}"
140
141
@@ -454,7 +229,7 @@ jobs:
229
script: .travis/trigger_docker_build.sh "${GITHUB_TOKEN}" "nightly"
230
after_failure: post_message "TRAVIS_MESSAGE" "<!here> Failed to trigger docker build during nightly release" "${NOTIF_CHANNEL}"
231
457
- - stage: Trigger deb and rpm package build (nightly release)
458
- name: Trigger deb and rpm package build
459
- script: .travis/trigger_package_generation.sh "[Build latest]"
232
+ - name: Trigger DEB and RPM package build
233
+ - git checkout "${TRAVIS_BRANCH}" && export BUILD_VERSION="$(cat packaging/version)"
234
+ - .travis/trigger_package_build.sh "${GITHUB_TOKEN}" "${BUILD_VERSION}" "nightly"
235
after_failure: post_message "TRAVIS_MESSAGE" "<!here> Failed to trigger deb and rpm package build during nightly release" "${NOTIF_CHANNEL}"
.travis/package_management/build.sh
deleted
-32
@@ -1,32 +0,0 @@
1
-#!/usr/bin/env bash
2
-
3
-UNPACKAGED_NETDATA_PATH="$1"
4
-LATEST_RELEASE_VERSION="$2"
5
-
6
-if [ -z "${LATEST_RELEASE_VERSION}" ]; then
7
- echo "Parameter 'LATEST_RELEASE_VERSION' not defined"
8
- exit 1
9
-fi
10
-
11
-if [ -z "${UNPACKAGED_NETDATA_PATH}" ]; then
12
- echo "Parameter 'UNPACKAGED_NETDATA_PATH' not defined"
13
- exit 1
14
-fi
15
-
16
-echo "Running changelog generation mechanism since ${LATEST_RELEASE_VERSION}"
17
-
18
-echo "Entering ${UNPACKAGED_NETDATA_PATH}"
19
-cd "${UNPACKAGED_NETDATA_PATH}"
20
-
21
-echo "Linking debian -> contrib/debian"
22
-ln -sf contrib/debian debian
23
-
24
-echo "Executing dpkg-buildpackage"
25
-# pre/post options are after 1.18.8, is simpler to just check help for their existence than parsing version
26
-if dpkg-buildpackage --help | grep "\-\-post\-clean" 2> /dev/null > /dev/null; then
27
- dpkg-buildpackage --post-clean --pre-clean --build=binary -us -uc
28
-else
29
- dpkg-buildpackage -b -us -uc
30
-fi
31
-
32
-echo "DEB build script completed!"
.travis/package_management/build_judy.sh
deleted
-36
@@ -1,36 +0,0 @@
1
-#!/usr/bin/env bash
2
-#
3
-# Build Judy from source, you need to run this script as root.
4
-#
5
-# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
6
-#
7
-# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
8
-set -e
9
-JUDY_VER="1.0.5"
10
-JUDY_DIR="/opt/judy-${JUDY_VER}"
11
-
12
-# If we are not in netdata git repo, at the top level directory, fail
13
-TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
14
-CWD=$(git rev-parse --show-cdup)
15
-if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
16
- echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository"
17
- echo "Build Judy package from source code failed"
18
- exit 1
19
-fi
20
-
21
-echo "Fetching judy source tarball"
22
-wget -O /opt/judy.tar.gz http://downloads.sourceforge.net/project/judy/judy/Judy-${JUDY_VER}/Judy-${JUDY_VER}.tar.gz
23
-
24
-echo "Entering /opt directory and extracting tarball"
25
-cd /opt && tar -xf judy.tar.gz && rm judy.tar.gz
26
-
27
-echo "Entering ${JUDY_DIR}"
28
-cd "${JUDY_DIR}"
29
-
30
-echo "Running configure"
31
-CFLAGS="-O2 -s" CXXFLAGS="-O2 -s" ./configure
32
-
33
-echo "Compiling and installing"
34
-make && make install
35
-
36
-echo "Done, enjoy Judy!"
.travis/package_management/build_libuv.sh
deleted
-36
@@ -1,36 +0,0 @@
1
-#!/usr/bin/env bash
2
-#
3
-# Build libuv from source, you need to run this script as root.
4
-#
5
-# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
6
-#
7
-# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud>
8
-set -e
9
-LIBUV_VERSION="v1.32.0"
10
-# Their folder is libuv-1.32.0 while the tarball version is v1.32.0, so fix that until they fix it...
11
-LIBUV_DIR="/opt/libuv-${LIBUV_VERSION/v/}"
12
-
13
-# If we are not in netdata git repo, at the top level directory, fail
14
-TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
15
-CWD=$(git rev-parse --show-cdup)
16
-if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
17
- echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository"
18
- echo "Build libuv package from source code failed"
19
- exit 1
20
-fi
21
-
22
-echo "Fetching libuv from github"
23
-wget -O /opt/libuv.tar.gz "https://github.com/libuv/libuv/archive/${LIBUV_VERSION}.tar.gz"
24
-
25
-echo "Entering /opt and extracting source"
26
-cd /opt && tar -xf libuv.tar.gz && rm libuv.tar.gz
27
-
28
-echo "Entering ${LIBUV_DIR}"
29
-cd "${LIBUV_DIR}"
30
-
31
-echo "Compiling and installing"
32
-sh autogen.sh
33
-./configure
34
-make && make check && make install
35
-
36
-echo "Done, enjoy libuv!"
.travis/package_management/build_package_in_container.sh
deleted
-82
@@ -1,82 +0,0 @@
1
-#!/usr/bin/env bash
2
-#
3
-# Entry point for package build process
4
-#
5
-# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
6
-#
7
-# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
8
-#shellcheck disable=SC1091
9
-set -e
10
-
11
-# If we are not in netdata git repo, at the top level directory, fail
12
-TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
13
-CWD=$(git rev-parse --show-cdup)
14
-if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
15
- echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository"
16
- echo "Docker build process aborted"
17
- exit 1
18
-fi
19
-
20
-source .travis/package_management/functions.sh || (echo "Failed to load packaging library" && exit 1)
21
-
22
-# Check for presence of mandatory environment variables
23
-if [ -z "${BUILD_STRING}" ]; then
24
- echo "No Distribution was defined. Make sure BUILD_STRING is set on the environment before running this script"
25
- exit 1
26
-fi
27
-
28
-if [ -z "${BUILDER_NAME}" ]; then
29
- echo "No builder account and container name defined. Make sure BUILDER_NAME is set on the environment before running this script"
30
- exit 1
31
-fi
32
-
33
-if [ -z "${BUILD_DISTRO}" ]; then
34
- echo "No build distro information defined. Make sure BUILD_DISTRO is set on the environment before running this script"
35
- exit 1
36
-fi
37
-
38
-if [ -z "${BUILD_RELEASE}" ]; then
39
- echo "No build release information defined. Make sure BUILD_RELEASE is set on the environment before running this script"
40
- exit 1
41
-fi
42
-
43
-if [ -z "${PACKAGE_TYPE}" ]; then
44
- echo "No build release information defined. Make sure PACKAGE_TYPE is set on the environment before running this script"
45
- exit 1
46
-fi
47
-
48
-# Detect architecture and load extra variables needed
49
-detect_arch_from_commit
50
-
51
-case "${BUILD_ARCH}" in
52
-"all")
53
- echo "* * * Building all architectures, amd64 and i386 * * *"
54
- echo "Building for amd64.."
55
- export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-amd64"
56
- export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs"
57
- .travis/package_management/trigger_${PACKAGE_TYPE}_lxc_build.py "${CONTAINER_NAME}"
58
-
59
- echo "Building for arm64.."
60
- export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-arm64"
61
- export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs"
62
- .travis/package_management/trigger_${PACKAGE_TYPE}_lxc_build.py "${CONTAINER_NAME}"
63
-
64
- echo "Building for i386.."
65
- export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-i386"
66
- export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs"
67
- .travis/package_management/trigger_${PACKAGE_TYPE}_lxc_build.py "${CONTAINER_NAME}"
68
-
69
- ;;
70
-"amd64"|"arm64"|"i386")
71
- echo "Building for ${BUILD_ARCH}.."
72
- export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-${BUILD_ARCH}"
73
- export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs"
74
- .travis/package_management/trigger_${PACKAGE_TYPE}_lxc_build.py "${CONTAINER_NAME}"
75
- ;;
76
-*)
77
- echo "Unknown build architecture '${BUILD_ARCH}', nothing to do for build"
78
- exit 1
79
- ;;
80
-esac
81
-
82
-echo "Build process completed!"
.travis/package_management/common.py
deleted
-182
@@ -1,182 +0,0 @@
1
-#
2
-#
3
-# Python library with commonly used functions within the package management scope
4
-#
5
-# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud>
6
-
7
-import lxc
8
-import subprocess
9
-import os
10
-import sys
11
-import tempfile
12
-import shutil
13
-
14
-def fetch_version(orig_build_version):
15
- tag = None
16
- friendly_version = ""
17
-
18
- # TODO: Checksum validations
19
- if str(orig_build_version).count(".latest") == 1:
20
- version_list=str(orig_build_version).replace('v', '').split('.')
21
- minor = version_list[3] if int(version_list[2]) == 0 else (version_list[2] + version_list[3])
22
- friendly_version='.'.join(version_list[0:2]) + "." + minor
23
- else:
24
- friendly_version = orig_build_version.replace('v', '')
25
- tag = friendly_version # Go to stable tag
26
- print("Version set to %s from %s" % (friendly_version, orig_build_version))
27
-
28
- return friendly_version, tag
29
-
30
-def replace_tag(tag_name, spec, new_tag_content):
31
- print("Fixing tag %s in %s" % (tag_name, spec))
32
-
33
- ifp = open(spec, "r")
34
- config = ifp.readlines()
35
- ifp.close()
36
-
37
- source_line = -1
38
- for line in config:
39
- if str(line).count(tag_name + ":") > 0:
40
- source_line = config.index(line)
41
- print("Found line: %s in item %d" % (line, source_line))
42
- break
43
-
44
- if source_line >= 0:
45
- print("Replacing line %s with %s in spec file" %(config[source_line], new_tag_content))
46
- config[source_line] = "%s: %s\n" % (tag_name, new_tag_content)
47
- config_str = ''.join(config)
48
- ofp = open(spec, 'w')
49
- ofp.write(config_str)
50
- ofp.close()
51
-
52
-def run_command(container, command):
53
- print("Running command: %s" % command)
54
- command_result = container.attach_wait(lxc.attach_run_command, command, stdout=sys.stdout.buffer, stderr=sys.stdout.buffer)
55
-
56
- if command_result != 0:
57
- raise Exception("Command failed with exit code %d" % command_result)
58
-
59
-def run_command_in_host(cmd, cwd=None):
60
- print("Issue command in host: %s, cwd:%s" % (str(cmd), str(cwd)))
61
-
62
- proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd)
63
- o, e = proc.communicate()
64
- print('Output: ' + o.decode('ascii'))
65
- print('Error: ' + e.decode('ascii'))
66
- print('code: ' + str(proc.returncode))
67
-
68
-def prepare_repo(container):
69
- if str(os.environ["REPO_TOOL"]).count("zypper") == 1:
70
- run_command(container, [os.environ["REPO_TOOL"], "clean", "-a"])
71
- run_command(container, [os.environ["REPO_TOOL"], "--no-gpg-checks", "update", "-y"])
72
-
73
- elif str(os.environ["REPO_TOOL"]).count("yum") == 1:
74
- run_command(container, [os.environ["REPO_TOOL"], "clean", "all"])
75
- run_command(container, [os.environ["REPO_TOOL"], "update", "-y"])
76
-
77
- if os.environ["BUILD_STRING"].count("el/7") == 1 and os.environ["BUILD_ARCH"].count("i386") == 1:
78
- print ("Skipping epel-release install for %s-%s" % (os.environ["BUILD_STRING"], os.environ["BUILD_ARCH"]))
79
- else:
80
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "epel-release"])
81
-
82
- elif str(os.environ["REPO_TOOL"]).count("apt-get") == 1:
83
- if str(os.environ["BUILD_STRING"]).count("debian/jessie") == 1:
84
- run_command(container, ["bash", "-c", "echo deb http://archive.debian.org/debian/ jessie-backports main contrib non-free >> /etc/apt/sources.list.d/99-archived.list"])
85
- run_command(container, [os.environ["REPO_TOOL"], "update", "-y", '-o', 'Acquire::Check-Valid-Until=false'])
86
- else:
87
- run_command(container, [os.environ["REPO_TOOL"], "update", "-y"])
88
- else:
89
- run_command(container, [os.environ["REPO_TOOL"], "update", "-y"])
90
-
91
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "sudo"])
92
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "wget"])
93
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "bash"])
94
-
95
-def install_common_dependendencies(container):
96
- if str(os.environ["REPO_TOOL"]).count("zypper") == 1:
97
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "gcc-c++"])
98
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-glib-devel"])
99
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"])
100
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"])
101
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "snappy-devel"])
102
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-devel"])
103
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-c"])
104
-
105
- elif str(os.environ["REPO_TOOL"]).count("yum") == 1:
106
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "gcc-c++"])
107
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"])
108
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"])
109
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"])
110
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "snappy-devel"])
111
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-devel"])
112
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-c-devel"])
113
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-compiler"])
114
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libwebsockets-devel"])
115
-
116
- elif str(os.environ["REPO_TOOL"]).count("apt-get") == 1:
117
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "g++"])
118
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libipmimonitoring-dev"])
119
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libjson-c-dev"])
120
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libcups2-dev"])
121
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libsnappy-dev"])
122
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libprotobuf-dev"])
123
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libprotoc-dev"])
124
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-compiler"])
125
- if os.environ["BUILD_STRING"].count("debian/jessie") == 1:
126
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "snappy"])
127
- else:
128
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "gcc-c++"])
129
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "cups-devel"])
130
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "freeipmi-devel"])
131
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "json-c-devel"])
132
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "snappy-devel"])
133
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-devel"])
134
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-c-devel"])
135
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "protobuf-compiler"])
136
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libwebsockets-devel"])
137
-
138
- if os.environ["BUILD_STRING"].count("el/6") <= 0:
139
- run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "autogen"])
140
-
141
-def prepare_version_source(dest_archive, pkg_friendly_version, tag=None):
142
- print(".0 Preparing local implementation tarball for version %s" % pkg_friendly_version)
143
- tar_file = os.environ['LXC_CONTAINER_ROOT'] + dest_archive
144
-
145
- print(".0 Copy repo to prepare it for tarball generation")
146
- tmp_src = tempfile.mkdtemp(prefix='netdata-source-')
147
- run_command_in_host(['cp', '-r', '.', tmp_src])
148
-
149
- if tag is not None:
150
- print(".1 Checking out tag %s" % tag)
151
- run_command_in_host(['git', 'fetch', '--all'], tmp_src)
152
-
153
- # TODO: Keep in mind that tricky 'v' there, needs to be removed once we clear our versioning scheme
154
- run_command_in_host(['git', 'checkout', 'v%s' % pkg_friendly_version], tmp_src)
155
-
156
- print(".2 Tagging the code with version: %s" % pkg_friendly_version)
157
- run_command_in_host(['git', 'tag', '-a', pkg_friendly_version, '-m', 'Tagging while packaging on %s' % os.environ["CONTAINER_NAME"]], tmp_src)
158
-
159
- print(".3 Run autoreconf -ivf")
160
- run_command_in_host(['autoreconf', '-ivf'], tmp_src)
161
-
162
- print(".4 Run configure")
163
- run_command_in_host(['./configure', '--prefix=/usr', '--sysconfdir=/etc', '--localstatedir=/var', '--libdir=/usr/lib', '--libexecdir=/usr/libexec', '--with-math', '--with-zlib', '--with-user=netdata'], tmp_src)
164
-
165
- print(".5 Run make dist")
166
- run_command_in_host(['make', 'dist'], tmp_src)
167
-
168
- print(".6 Copy generated tarbal to desired path")
169
- generated_tarball = '%s/netdata-%s.tar.gz' % (tmp_src, pkg_friendly_version)
170
-
171
- if os.path.exists(generated_tarball):
172
- run_command_in_host(['sudo', 'cp', generated_tarball, tar_file])
173
-
174
- print(".7 Fixing permissions on tarball")
175
- run_command_in_host(['sudo', 'chmod', '777', tar_file])
176
-
177
- print(".8 Bring over netdata.spec, Remove temp directory");
178
- run_command_in_host(['cp', '%s/netdata.spec' % tmp_src, 'netdata.spec'])
179
- shutil.rmtree(tmp_src)
180
- else:
181
- print("I could not find (%s) on the disk, stopping the build. Kindly check the logs and try again" % generated_tarball)
182
- sys.exit(1)
.travis/package_management/configure_deb_lxc_environment.py
deleted
-90
@@ -1,90 +0,0 @@
1
-#!/usr/bin/env python3
2
-#
3
-# Prepare the build environment within the container
4
-# The script attaches to the running container and does the following:
5
-# 1) Create the container
6
-# 2) Start the container up
7
-# 3) Create the builder user
8
-# 4) Prepare the environment for DEB build
9
-#
10
-# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
11
-#
12
-# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud>
13
-
14
-import common
15
-import os
16
-import sys
17
-import lxc
18
-
19
-if len(sys.argv) != 2:
20
- print('You need to provide a container name to get things started')
21
- sys.exit(1)
22
-container_name=sys.argv[1]
23
-
24
-# Setup the container object
25
-print("Defining container %s" % container_name)
26
-container = lxc.Container(container_name)
27
-if not container.defined:
28
- raise Exception("Container %s not defined!" % container_name)
29
-
30
-# Start the container
31
-if not container.start():
32
- raise Exception("Failed to start the container")
33
-
34
-if not container.running or not container.state == "RUNNING":
35
- raise Exception('Container %s is not running, configuration process aborted ' % container_name)
36
-
37
-# Wait for connectivity
38
-print("Waiting for container connectivity to start configuration sequence")
39
-if not container.get_ips(timeout=30):
40
- raise Exception("Timeout while waiting for container")
41
-
42
-build_path = "/home/%s" % os.environ['BUILDER_NAME']
43
-
44
-# Run the required activities now
45
-# 1. Create the builder user
46
-print("1. Adding user %s" % os.environ['BUILDER_NAME'])
47
-common.run_command(container, ["useradd", "-m", os.environ['BUILDER_NAME']])
48
-
49
-# Fetch package dependencies for the build
50
-print("2. Preparing repo on LXC container")
51
-common.prepare_repo(container)
52
-
53
-print("2.1 Install .DEB build support packages")
54
-common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "dpkg-dev"])
55
-common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libdistro-info-perl"])
56
-common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "dh-make"])
57
-common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "dh-systemd"])
58
-common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "dh-autoreconf"])
59
-common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "git-buildpackage"])
60
-
61
-print("2.2 Add more dependencies")
62
-common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libnetfilter-acct-dev"])
63
-common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libcups2-dev"])
64
-
65
-print ("3.1 Run install-required-packages scriptlet")
66
-common.run_command(container, ["wget", "-T", "15", "-O", "%s/.install-required-packages.sh" % build_path, "https://raw.githubusercontent.com/netdata/netdata/master/packaging/installer/install-required-packages.sh"])
67
-common.run_command(container, ["bash", "%s/.install-required-packages.sh" % build_path, "netdata", "--dont-wait", "--non-interactive"])
68
-
69
-print("3.2 Installing package dependencies within LXC container")
70
-common.install_common_dependendencies(container)
71
-
72
-friendly_version=""
73
-dest_archive=""
74
-download_url=""
75
-tag = None
76
-friendly_version, tag = common.fetch_version(os.environ['BUILD_VERSION'])
77
-
78
-tar_file="%s/netdata-%s.tar.gz" % (os.path.dirname(dest_archive), friendly_version)
79
-
80
-print("5. I will be building version '%s' of netdata." % os.environ['BUILD_VERSION'])
81
-dest_archive="%s/netdata-%s.tar.gz" % (build_path, friendly_version)
82
-
83
-common.prepare_version_source(dest_archive, friendly_version, tag=tag)
84
-
85
-print("6. Installing build.sh script to build path")
86
-common.run_command_in_host(['sudo', 'cp', '.travis/package_management/build.sh', "%s/%s/build.sh" % (os.environ['LXC_CONTAINER_ROOT'], build_path)])
87
-common.run_command_in_host(['sudo', 'chmod', '777', "%s/%s/build.sh" % (os.environ['LXC_CONTAINER_ROOT'], build_path)])
88
-common.run_command_in_host(['sudo', 'ln', '-sf', 'contrib/debian', 'debian'])
89
-
90
-print("Done!")
.travis/package_management/configure_rpm_lxc_environment.py
deleted
-102
@@ -1,102 +0,0 @@
1
-#!/usr/bin/env python3
2
-#
3
-#
4
-# Prepare the build environment within the container
5
-# The script attaches to the running container and does the following:
6
-# 1) Create the container
7
-# 2) Start the container up
8
-# 3) Create the builder user
9
-# 4) Prepare the environment for RPM build
10
-#
11
-# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
12
-#
13
-# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud>
14
-
15
-import common
16
-import os
17
-import sys
18
-import lxc
19
-
20
-if len(sys.argv) != 2:
21
- print('You need to provide a container name to get things started')
22
- sys.exit(1)
23
-container_name=sys.argv[1]
24
-
25
-# Setup the container object
26
-print("Defining container %s" % container_name)
27
-container = lxc.Container(container_name)
28
-if not container.defined:
29
- raise Exception("Container %s not defined!" % container_name)
30
-
31
-# Start the container
32
-if not container.start():
33
- raise Exception("Failed to start the container")
34
-
35
-if not container.running or not container.state == "RUNNING":
36
- raise Exception('Container %s is not running, configuration process aborted ' % container_name)
37
-
38
-# Wait for connectivity
39
-print("Waiting for container connectivity to start configuration sequence")
40
-if not container.get_ips(timeout=30):
41
- raise Exception("Timeout while waiting for container")
42
-
43
-# Run the required activities now
44
-# Create the builder user
45
-print("1. Adding user %s" % os.environ['BUILDER_NAME'])
46
-common.run_command(container, ["useradd", "-m", os.environ['BUILDER_NAME']])
47
-
48
-# Fetch package dependencies for the build
49
-print("2.1 Preparing repo on LXC container")
50
-common.prepare_repo(container)
51
-
52
-common.run_command(container, ["wget", "-T", "15", "-O", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "https://raw.githubusercontent.com/netdata/netdata/master/packaging/installer/install-required-packages.sh"])
53
-common.run_command(container, ["bash", "/home/%s/.install-required-packages.sh" % (os.environ['BUILDER_NAME']), "netdata", "--dont-wait", "--non-interactive"])
54
-
55
-# Exceptional cases, not available everywhere
56
-#
57
-print("2.2 Running uncommon dependencies and preparing LXC environment")
58
-# Not on Centos-7
59
-if os.environ["BUILD_STRING"].count("el/7") <= 0:
60
- common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "libnetfilter_acct-devel"])
61
-
62
-# Not on Centos-6
63
-if os.environ["BUILD_STRING"].count("el/6") <= 0:
64
- common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "autoconf-archive"])
65
-
66
-print("2.3 Installing common dependencies")
67
-common.install_common_dependendencies(container)
68
-
69
-print("3. Setting up macros")
70
-common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "/bin/echo", "'%_topdir %(echo /home/" + os.environ['BUILDER_NAME'] + ")/rpmbuild' > /home/" + os.environ['BUILDER_NAME'] + "/.rpmmacros"])
71
-
72
-print("4. Create rpmbuild directory")
73
-common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/BUILD"])
74
-common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/RPMS"])
75
-common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/SOURCES"])
76
-common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/SPECS"])
77
-common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "mkdir", "-p", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild/SRPMS"])
78
-common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "ls", "-ltrR", "/home/" + os.environ['BUILDER_NAME'] + "/rpmbuild"])
79
-
80
-# Download the source
81
-rpm_friendly_version=""
82
-dest_archive=""
83
-download_url=""
84
-spec_file="/home/%s/rpmbuild/SPECS/netdata.spec" % os.environ['BUILDER_NAME']
85
-tag = None
86
-rpm_friendly_version, tag = common.fetch_version(os.environ['BUILD_VERSION'])
87
-tar_file="%s/netdata-%s.tar.gz" % (os.path.dirname(dest_archive), rpm_friendly_version)
88
-
89
-print("5. I will be building version '%s' of netdata." % os.environ['BUILD_VERSION'])
90
-dest_archive="/home/%s/rpmbuild/SOURCES/netdata-%s.tar.gz" % (os.environ['BUILDER_NAME'], rpm_friendly_version)
91
-
92
-common.prepare_version_source(dest_archive, rpm_friendly_version, tag=tag)
93
-
94
-# Extract the spec file in place
95
-print("6. Extract spec file from the source")
96
-common.run_command_in_host(['sudo', 'cp', 'netdata.spec', os.environ['LXC_CONTAINER_ROOT'] + spec_file])
97
-common.run_command_in_host(['sudo', 'chmod', '777', os.environ['LXC_CONTAINER_ROOT'] + spec_file])
98
-
99
-print("7. Temporary hack: Change Source0 to %s on spec file %s" % (dest_archive, spec_file))
100
-common.replace_tag("Source0", os.environ['LXC_CONTAINER_ROOT'] + spec_file, tar_file)
101
-
102
-print('Done!')
.travis/package_management/create_lxc_for_build.sh
deleted
-101
@@ -1,101 +0,0 @@
1
-#!/usr/bin/env bash
2
-#
3
-# This script generates an LXC container and starts it up
4
-# Once the script completes successfully, a container has become available for usage
5
-# The container image to be used and the container name to be set, are part of variables
6
-# that must be present for the script to work
7
-#
8
-# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
9
-#
10
-# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
11
-# shellcheck disable=SC1091
12
-set -e
13
-
14
-source .travis/package_management/functions.sh || (echo "Failed to load packaging library" && exit 1)
15
-
16
-# If we are not in netdata git repo, at the top level directory, fail
17
-TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
18
-CWD=$(git rev-parse --show-cdup)
19
-if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
20
- echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository"
21
- echo "LXC Container creation aborted"
22
- exit 1
23
-fi
24
-
25
-# Check for presence of mandatory environment variables
26
-if [ -z "${BUILD_STRING}" ]; then
27
- echo "No Distribution was defined. Make sure BUILD_STRING is set on the environment before running this script"
28
- exit 1
29
-fi
30
-
31
-if [ -z "${BUILDER_NAME}" ]; then
32
- echo "No builder account and container name defined. Make sure BUILDER_NAME is set on the environment before running this script"
33
- exit 1
34
-fi
35
-
36
-if [ -z "${BUILD_DISTRO}" ]; then
37
- echo "No build distro information defined. Make sure BUILD_DISTRO is set on the environment before running this script"
38
- exit 1
39
-fi
40
-
41
-if [ -z "${BUILD_RELEASE}" ]; then
42
- echo "No build release information defined. Make sure BUILD_RELEASE is set on the environment before running this script"
43
- exit 1
44
-fi
45
-
46
-if [ -z "${PACKAGE_TYPE}" ]; then
47
- echo "No build release information defined. Make sure PACKAGE_TYPE is set on the environment before running this script"
48
- exit 1
49
-fi
50
-
51
-# Detect architecture and load extra variables needed
52
-detect_arch_from_commit
53
-
54
-echo "Creating LXC container ${BUILDER_NAME}/${BUILD_STRING}/${BUILD_ARCH}...."
55
-
56
-case "${BUILD_ARCH}" in
57
-"all")
58
- # i386
59
- echo "Creating LXC Container for i386.."
60
- export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-i386"
61
- export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs"
62
- lxc-create -n "${CONTAINER_NAME}" -t "download" -- --dist "${BUILD_DISTRO}" --release "${BUILD_RELEASE}" --arch "i386" --no-validate
63
-
64
- echo "Container(s) ready. Configuring container(s).."
65
- .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}"
66
-
67
- # amd64
68
- echo "Creating LXC Container for amd64.."
69
- export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-amd64"
70
- export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs"
71
- lxc-create -n "${CONTAINER_NAME}" -t "download" -- --dist "${BUILD_DISTRO}" --release "${BUILD_RELEASE}" --arch "amd64" --no-validate
72
-
73
- echo "Container(s) ready. Configuring container(s).."
74
- .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}"
75
-
76
- # arm64
77
- echo "Creating LXC Container for arm64.."
78
- export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-arm64"
79
- export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs"
80
- lxc-create -n "${CONTAINER_NAME}" -t "download" -- --dist "${BUILD_DISTRO}" --release "${BUILD_RELEASE}" --arch "arm64" --no-validate
81
-
82
- echo "Container(s) ready. Configuring container(s).."
83
- .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}"
84
- ;;
85
-"i386"|"amd64"|"arm64")
86
- # amd64 or i386
87
- echo "Creating LXC Container for ${BUILD_ARCH}.."
88
- export CONTAINER_NAME="${BUILDER_NAME}-${BUILD_DISTRO}${BUILD_RELEASE}-${BUILD_ARCH}"
89
- export LXC_CONTAINER_ROOT="/var/lib/lxc/${CONTAINER_NAME}/rootfs"
90
- lxc-create -n "${CONTAINER_NAME}" -t "download" -- --dist "${BUILD_DISTRO}" --release "${BUILD_RELEASE}" --arch "${BUILD_ARCH}" --no-validate
91
-
92
- echo "Container(s) ready. Configuring container(s).."
93
- .travis/package_management/configure_${PACKAGE_TYPE}_lxc_environment.py "${CONTAINER_NAME}"
94
- ;;
95
-*)
96
- echo "Unknown BUILD_ARCH value '${BUILD_ARCH}' given, process failed"
97
- exit 1
98
- ;;
99
-esac
100
-
101
-echo "..LXC creation complete!"
.travis/package_management/functions.sh
deleted
-33
@@ -1,33 +0,0 @@
1
-# no-shebang-needed-its-a-library
2
-#
3
-# Utility functions for packaging in travis CI
4
-#
5
-# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
6
-#
7
-# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
8
-#shellcheck disable=SC2148
9
-set -e
10
-
11
-function detect_arch_from_commit {
12
- case "${TRAVIS_COMMIT_MESSAGE}" in
13
- "[Package amd64"*)
14
- export BUILD_ARCH="amd64"
15
- ;;
16
- "[Package i386"*)
17
- export BUILD_ARCH="i386"
18
- ;;
19
- "[Package ALL"*)
20
- export BUILD_ARCH="all"
21
- ;;
22
- "[Package arm64"*)
23
- export BUILD_ARCH="arm64"
24
- ;;
25
-
26
- *)
27
- echo "Unknown build architecture in '${TRAVIS_COMMIT_MESSAGE}'. Assuming amd64"
28
- export BUILD_ARCH="amd64"
29
- ;;
30
- esac
31
-
32
- echo "Detected build architecture ${BUILD_ARCH}"
33
-}
.travis/package_management/prepare_packages.sh
deleted
-63
@@ -1,63 +0,0 @@
1
-#!/usr/bin/env bash
2
-#
3
-# Utility that gathers generated packages,
4
-# puts them together in a local folder for deploy facility to pick up
5
-#
6
-# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
7
-#
8
-# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
9
-#shellcheck disable=SC2068
10
-set -e
11
-
12
-# If we are not in netdata git repo, at the top level directory, fail
13
-TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
14
-CWD=$(git rev-parse --show-cdup)
15
-if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
16
- echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository"
17
- echo "Package preparation aborted"
18
- exit 1
19
-fi
20
-
21
-export LXC_ROOT="/var/lib/lxc"
22
-
23
-# Go through the containers created for packaging and pick up all generated packages
24
-CREATED_CONTAINERS=$(ls -A "${LXC_ROOT}")
25
-for d in ${CREATED_CONTAINERS[@]}; do
26
- echo "Picking up packaging contents from ${d}"
27
-
28
- # Pick up any RPMS from builder
29
- RPM_BUILD_PATH="${LXC_ROOT}/${d}/rootfs/home/${BUILDER_NAME}/rpmbuild"
30
- if [ -d "${RPM_BUILD_PATH}" ]; then
31
- echo "Checking folder ${RPM_BUILD_PATH} for RPMS and SRPMS"
32
-
33
- if [ -d "${RPM_BUILD_PATH}/RPMS" ]; then
34
- echo "Copying any RPMS in '${RPM_BUILD_PATH}', copying over the following:"
35
- ls -ltrR "${RPM_BUILD_PATH}/RPMS"
36
- [[ -d "${RPM_BUILD_PATH}/RPMS/x86_64" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/x86_64/* "${PACKAGES_DIRECTORY}"
37
- [[ -d "${RPM_BUILD_PATH}/RPMS/i386" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/i386/* "${PACKAGES_DIRECTORY}"
38
- [[ -d "${RPM_BUILD_PATH}/RPMS/i686" ]] && cp -r "${RPM_BUILD_PATH}"/RPMS/i686/* "${PACKAGES_DIRECTORY}"
39
- fi
40
-
41
- if [ -d "${RPM_BUILD_PATH}/SRPMS" ]; then
42
- echo "Copying any SRPMS in '${RPM_BUILD_PATH}', copying over the following:"
43
- ls -ltrR "${RPM_BUILD_PATH}/SRPMS"
44
- [[ -d "${RPM_BUILD_PATH}/SRPMS/x86_64" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/x86_64/* "${PACKAGES_DIRECTORY}"
45
- [[ -d "${RPM_BUILD_PATH}/SRPMS/i386" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/i386/* "${PACKAGES_DIRECTORY}"
46
- [[ -d "${RPM_BUILD_PATH}/SRPMS/i686" ]] && cp -r "${RPM_BUILD_PATH}"/SRPMS/i686/* "${PACKAGES_DIRECTORY}"
47
- fi
48
- else
49
- DEB_BUILD_PATH="${LXC_ROOT}/${d}/rootfs/home/${BUILDER_NAME}"
50
- echo "Checking folder ${DEB_BUILD_PATH} for DEB packages"
51
- if [ -d "${DEB_BUILD_PATH}" ]; then
52
- cp "${DEB_BUILD_PATH}"/netdata*.ddeb "${PACKAGES_DIRECTORY}" || echo "Could not copy any .ddeb files"
53
- cp "${DEB_BUILD_PATH}"/netdata*.deb "${PACKAGES_DIRECTORY}" || echo "Could not copy any .deb files"
54
- cp "${DEB_BUILD_PATH}"/netdata*.buildinfo "${PACKAGES_DIRECTORY}" || echo "Could not copy any .buildinfo files"
55
- cp "${DEB_BUILD_PATH}"/netdata*.changes "${PACKAGES_DIRECTORY}" || echo "Could not copy any .changes files"
56
- else
57
- echo "Folder ${DEB_BUILD_PATH} does not exist or not a directory, nothing to do for package preparation"
58
- fi
59
- fi
60
-done
61
-
62
-chmod -R 777 "${PACKAGES_DIRECTORY}"
63
-echo "Packaging contents ready to ship!"
.travis/package_management/trigger_deb_lxc_build.py
deleted
-86
@@ -1,86 +0,0 @@
1
-#!/usr/bin/env python3
2
-#
3
-# This script is responsible for running the RPM build on the running container
4
-#
5
-# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
6
-#
7
-# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud>
8
-
9
-import common
10
-import os
11
-import sys
12
-import lxc
13
-
14
-if len(sys.argv) != 2:
15
- print('You need to provide a container name to get things started')
16
- sys.exit(1)
17
-container_name=sys.argv[1]
18
-
19
-# Load the container, break if its not there
20
-print("Starting up container %s" % container_name)
21
-container = lxc.Container(container_name)
22
-if not container.defined:
23
- raise Exception("Container %s does not exist!" % container_name)
24
-
25
-# Check if the container is running, attempt to start it up in case its not running
26
-if not container.running or not container.state == "RUNNING":
27
- print('Container %s is not running, attempt to start it up' % container_name)
28
-
29
- # Start the container
30
- if not container.start():
31
- raise Exception("Failed to start the container")
32
-
33
- if not container.running or not container.state == "RUNNING":
34
- raise Exception('Container %s is not running, configuration process aborted ' % container_name)
35
-
36
-# Wait for connectivity
37
-if not container.get_ips(timeout=30):
38
- raise Exception("Timeout while waiting for container")
39
-
40
-build_path = "/home/%s" % os.environ['BUILDER_NAME']
41
-
42
-print("Setting up EMAIL and DEBFULLNAME variables required by the build tools")
43
-os.environ["EMAIL"] = "bot@netdata.cloud"
44
-os.environ["DEBFULLNAME"] = "Netdata builder"
45
-
46
-# Run the build process on the container
47
-new_version, tag = common.fetch_version(os.environ['BUILD_VERSION'])
48
-print("Starting DEB build process for version %s" % new_version)
49
-
50
-netdata_tarball = "%s/netdata-%s.tar.gz" % (build_path, new_version)
51
-unpacked_netdata = netdata_tarball.replace(".tar.gz", "")
52
-
53
-print("Extracting tarball %s" % netdata_tarball)
54
-common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "tar", "xf", netdata_tarball, "-C", build_path])
55
-
56
-print("Checking version consistency")
57
-since_version = os.environ["LATEST_RELEASE_VERSION"]
58
-if str(since_version).replace('v', '') == str(new_version) and str(new_version).count('.') == 2:
59
- s = since_version.split('.')
60
- if int(s[2]) > 0:
61
- patch_prev = str(int(s[2]) - 1)
62
- since_version = s[0] + '.' + s[1] + '.' + patch_prev
63
- else:
64
- prev = str(int(s[1]) - 1)
65
- since_version = s[0] + '.' + prev + '.' + s[2]
66
-
67
- print("We seem to be building a new stable release, reduce by one since_version option. New since_version:%s" % since_version)
68
-
69
-print("Fixing changelog tags")
70
-changelog_in_host = "contrib/debian/changelog"
71
-common.run_command_in_host(['sed', '-i', 's/PREVIOUS_PACKAGE_VERSION/%s-1/g' % since_version.replace("v", ""), changelog_in_host])
72
-common.run_command_in_host(['sed', '-i', 's/PREVIOUS_PACKAGE_DATE/%s/g' % os.environ["LATEST_RELEASE_DATE"], changelog_in_host])
73
-
74
-print("Executing gbp dch command..")
75
-common.run_command_in_host(['gbp', 'dch', '--release', '--ignore-branch', '--spawn-editor=snapshot', '--since=%s' % since_version, '--new-version=%s' % new_version])
76
-
77
-print("Copying over changelog to the destination machine")
78
-common.run_command_in_host(['sudo', 'cp', 'debian/changelog', "%s/%s/netdata-%s/contrib/debian/" % (os.environ['LXC_CONTAINER_ROOT'], build_path, new_version)])
79
-
80
-print("Running debian build script since %s" % since_version)
81
-common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "%s/build.sh" % build_path, unpacked_netdata, new_version])
82
-
83
-print("Listing contents on build path")
84
-common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "ls", "-ltr", build_path])
85
-
86
-print('Done!')
.travis/package_management/trigger_rpm_lxc_build.py
deleted
-55
@@ -1,55 +0,0 @@
1
-#!/usr/bin/env python3
2
-#
3
-# This script is responsible for running the RPM build on the running container
4
-#
5
-# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
6
-#
7
-# Author : Pavlos Emm. Katsoulakis <paul@netdata.cloud>
8
-
9
-import common
10
-import os
11
-import sys
12
-import lxc
13
-
14
-if len(sys.argv) != 2:
15
- print('You need to provide a container name to get things started')
16
- sys.exit(1)
17
-container_name=sys.argv[1]
18
-
19
-# Load the container, break if its not there
20
-print("Starting up container %s" % container_name)
21
-container = lxc.Container(container_name)
22
-if not container.defined:
23
- raise Exception("Container %s does not exist!" % container_name)
24
-
25
-# Check if the container is running, attempt to start it up in case its not running
26
-if not container.running or not container.state == "RUNNING":
27
- print('Container %s is not running, attempt to start it up' % container_name)
28
-
29
- # Start the container
30
- if not container.start():
31
- raise Exception("Failed to start the container")
32
-
33
- if not container.running or not container.state == "RUNNING":
34
- raise Exception('Container %s is not running, configuration process aborted ' % container_name)
35
-
36
-# Wait for connectivity
37
-if not container.get_ips(timeout=30):
38
- raise Exception("Timeout while waiting for container")
39
-
40
-print("Adding builder specific dependencies to the LXC container")
41
-common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "rpm-build"])
42
-common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "rpm-devel"])
43
-common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "rpmlint"])
44
-common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "make"])
45
-common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "python"])
46
-common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "bash"])
47
-common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "diffutils"])
48
-common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "patch"])
49
-common.run_command(container, [os.environ["REPO_TOOL"], "install", "-y", "rpmdevtools"])
50
-
51
-# Run the build process on the container
52
-print("Starting RPM build process")
53
-common.run_command(container, ["sudo", "-u", os.environ['BUILDER_NAME'], "rpmbuild", "-ba", "--rebuild", "/home/%s/rpmbuild/SPECS/netdata.spec" % os.environ['BUILDER_NAME']])
54
-
55
-print('Done!')
.travis/package_management/yank_stale_pkg.sh
deleted
-35
@@ -1,35 +0,0 @@
1
-#!/usr/bin/env bash
2
-#
3
-# This script is responsible for the removal of stale RPM/DEB files.
4
-# It runs on the pre-deploy step and takes care of the removal of the files
5
-# prior to the upload of the freshly built ones
6
-#
7
-# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
8
-#
9
-# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
10
-#shellcheck disable=SC2010,SC2068
11
-set -e
12
-
13
-# If we are not in netdata git repo, at the top level directory, fail
14
-TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
15
-CWD=$(git rev-parse --show-cdup)
16
-if [ -n "$CWD" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
17
- echo "Run as .travis/package_management/$(basename "$0") from top level directory of netdata git repository"
18
- echo "Package yanking cancelled"
19
- exit 1
20
-fi
21
-
22
-PACKAGES_DIR="$1"
23
-DISTRO="$2"
24
-PACKAGES_LIST="$(ls -AR "${PACKAGES_DIR}" | grep -e '\.rpm' -e '\.deb' -e '\.ddeb' )"
25
-
26
-if [ ! -d "${PACKAGES_DIR}" ] || [ -z "${PACKAGES_LIST}" ]; then
27
- echo "Folder ${PACKAGES_DIR} does not seem to be a valid directory or is empty. No packages to check for yanking"
28
- exit 1
29
-fi
30
-
31
-for pkg in ${PACKAGES_LIST[@]}; do
32
- echo "Attempting yank on ${pkg}.."
33
- .travis/package_management/package_cloud_wrapper.sh yank "${PACKAGING_USER}/${DEPLOY_REPO}/${DISTRO}" "${pkg}" || echo "Nothing to yank or error on ${pkg}"
34
-done
35
-
.travis/trigger_package_build.sh
new
+20
@@ -0,0 +1,20 @@
1
+#!/bin/sh
2
+
3
+token="${1}"
4
+version="${2}"
5
+pkgtype="${3}"
6
+
7
+resp="$(curl -X POST \
8
+ -H 'Accept: application/vnd.github.v3+json' \
9
+ -H "Authorization: Bearer ${token}" \
10
+ "https://api.github.com/repos/netdata/netdata/actions/workflows/packaging.yml/dispatches" \
11
+ -d "{\"ref\": \"master\", \"inputs\": {\"version\": \"${version}\", \"type\": \"${pkgtype}\"}}")"
12
+
13
+if [ -z "${resp}" ]; then
14
+ echo "Successfully triggered binary package builds."
15
+ exit 0
16
+else
17
+ echo "Failed to trigger binary package builds. Output:"
18
+ echo "${resp}"
19
+ exit 1
20
+fi
.travis/trigger_package_generation.sh
deleted
-57
@@ -1,57 +0,0 @@
1
-#!/usr/bin/env bash
2
-#
3
-# Trigger .RPM and .DEB package generation processes
4
-#
5
-# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
6
-#
7
-# Author: Pavlos Emm. Katsoulakis <paul@netdata.cloud>
8
-set -e
9
-WAIT_TIME=15
10
-BUILD_NIGHTLY="$1"
11
-
12
-commit_change() {
13
- local ARCH="$1"
14
- local PKG="$2"
15
- local GIT_MAIL="bot@netdata.cloud"
16
- local GIT_USER="netdatabot"
17
-
18
- echo "---- Committing ${ARCH} .${PKG} package generation ----"
19
- git commit --allow-empty --author "${GIT_USER} <${GIT_MAIL}>" -m "[Package ${ARCH} ${PKG}]${BUILD_NIGHTLY} Package build process trigger"
20
-}
21
-
22
-push_change() {
23
-
24
- echo "---- Push changes to repository ----"
25
- git push "https://${GITHUB_TOKEN}:@$(git config --get remote.origin.url | sed -e 's/^https:\/\///')"
26
-}
27
-
28
-# If we are not in netdata git repo, at the top level directory, fail
29
-TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
30
-CWD=$(git rev-parse --show-cdup || echo "")
31
-if [ -n "${CWD}" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
32
- echo "Run as .travis/$(basename "$0") from top level directory of netdata git repository"
33
- echo "Changelog generation process aborted"
34
- exit 1
35
-fi
36
-
37
-echo "--- Initialize git configuration ---"
38
-git checkout master
39
-git fetch --all
40
-git pull
41
-
42
-commit_change "amd64" "DEB"
43
-push_change
44
-
45
-echo "---- Waiting for ${WAIT_TIME} seconds before triggering next process ----"
46
-sleep "${WAIT_TIME}"
47
-
48
-commit_change "i386" "DEB"
49
-push_change
50
-
51
-echo "---- Waiting for ${WAIT_TIME} seconds before triggering next process ----"
52
-sleep "${WAIT_TIME}"
53
-
54
-commit_change "amd64" "RPM"
55
-push_change
56
-
57
-echo "---- Done! ----"
contrib/debian/netdata.postinst
+1
-1
@@ -55,7 +55,7 @@ case "$1" in
55
chown -R root:netdata /var/lib/netdata/www
56
setcap cap_dac_read_search,cap_sys_ptrace+ep /usr/libexec/netdata/plugins.d/apps.plugin
57
setcap cap_dac_read_search+ep /usr/libexec/netdata/plugins.d/slabinfo.plugin
58
- setcap cap_perfmon+ep /usr/libexec/netdata/plugins.d/perf.plugin
58
+ setcap cap_perfmon+ep /usr/libexec/netdata/plugins.d/perf.plugin || setcap cap_sys_admin+ep /usr/libexec/netdata/plugins.d/perf.plugin
59
60
chmod 4750 /usr/libexec/netdata/plugins.d/cgroup-network
61
chmod 4750 /usr/libexec/netdata/plugins.d/nfacct.plugin
netdata.spec.in
+3
-1
@@ -479,7 +479,9 @@ rm -rf "${RPM_BUILD_ROOT}"
479
%endif
480
481
# perf plugin
482
-%caps(cap_perfmon=ep) %attr(0750,root,netdata) %{_libexecdir}/%{name}/plugins.d/perf.plugin
482
+# This should be CAP_PERFMON once RPM finally learns about it, but needs to be CAP_SYS_ADMIN for now.
483
+# %caps(cap_perfmon=ep) %attr(0750,root,netdata) %{_libexecdir}/%{name}/plugins.d/perf.plugin
484
+%caps(cap_sys_admin=ep) %attr(0750,root,netdata) %{_libexecdir}/%{name}/plugins.d/perf.plugin
485
486
# perf plugin
487
%caps(cap_dac_read_search=ep) %attr(0750,root,netdata) %{_libexecdir}/%{name}/plugins.d/slabinfo.plugin
packaging/Dockerfile.packager
+7
-6
@@ -1,36 +1,37 @@
1
ARG ARCH=amd64
2
ARG DISTRO=debian
3
+ARG TEST_BASE=debian
4
ARG DISTRO_VERSION=10
4
-ARG VERSION=0.1
5
+ARG PKG_VERSION=0.1
6
7
FROM netdata/package-builders:${DISTRO}${DISTRO_VERSION} AS build
8
9
ARG ARCH
10
ARG DISTRO
11
ARG DISTRO_VERSION
11
-ARG VERSION
12
+ARG PKG_VERSION
13
14
ENV ARCH=$ARCH
15
ENV DISTRO=$DISTRO
16
ENV DISTRO_VERSION=$DISTRO_VERSION
16
-ENV VERSION=$VERSION
17
+ENV VERSION=$PKG_VERSION
18
19
WORKDIR /netdata
20
COPY . .
21
22
RUN /build.sh
23
23
-FROM ${DISTRO}:${DISTRO_VERSION} AS runtime
24
+FROM ${TEST_BASE}:${DISTRO_VERSION} AS runtime
25
26
ARG ARCH
27
ARG DISTRO
28
ARG DISTRO_VERSION
28
-ARG VERSION
29
+ARG PKG_VERSION
30
31
ENV ARCH=$ARCH
32
ENV DISTRO=$DISTRO
33
ENV DISTRO_VERSION=$DISTRO_VERSION
33
-ENV VERSION=$VERSION
34
+ENV VERSION=$PKG_VERSION
35
36
COPY ./packaging/scripts/install.sh /install.sh
37
COPY ./packaging/scripts/test.sh /test.sh
packaging/scripts/install.sh
+21
-2
@@ -27,6 +27,22 @@ install_fedora_like() {
27
"$PKGMGR" install -y curl nc jq
28
}
29
30
+install_centos() {
31
+ # Using a glob pattern here because I can't reliably determine what the
32
+ # resulting package name will be (TODO: There must be a better way!)
33
+
34
+ PKGMGR="$( (command -v dnf > /dev/null && echo "dnf") || echo "yum")"
35
+
36
+ # Install EPEL (needed for `jq`
37
+ "$PKGMGR" install -y epel-release
38
+
39
+ # Install NetData
40
+ "$PKGMGR" install -y /artifacts/netdata-"${VERSION}"-*.rpm
41
+
42
+ # Install testing tools
43
+ "$PKGMGR" install -y curl nc jq
44
+}
45
+
46
install_suse_like() {
47
# Using a glob pattern here because I can't reliably determine what the
48
# resulting package name will be (TODO: There must be a better way!)
@@ -38,16 +54,19 @@ install_suse_like() {
54
55
# Install testing tools
56
zypper install -y --no-recommends \
41
- curl netcat jq
57
+ curl gnu-netcat jq
58
}
59
60
case "${DISTRO}" in
61
debian | ubuntu)
62
install_debian_like
63
;;
48
- fedora | centos)
64
+ fedora)
65
install_fedora_like
66
;;
67
+ centos)
68
+ install_centos
69
+ ;;
70
opensuse)
71
install_suse_like
72
;;
packaging/scripts/test.sh
+18
-4
@@ -1,13 +1,21 @@
1
#!/bin/sh
2
3
+dump_log() {
4
+ cat ./netdata.log
5
+}
6
+
7
+trap dump_log EXIT
8
+
9
wait_for() {
10
host="${1}"
11
port="${2}"
12
name="${3}"
7
- timeout="${4:-30}"
13
+ timeout="30"
14
15
printf "Waiting for %s on %s:%s ... " "${name}" "${host}" "${port}"
16
17
+ sleep 30
18
+
19
i=0
20
while ! nc -z "${host}" "${port}"; do
21
sleep 1
@@ -20,8 +28,14 @@ wait_for() {
28
printf "OK\n"
29
}
30
23
-netdata -D > netdata.log 2>&1 &
31
+/usr/sbin/netdata -D > ./netdata.log 2>&1 &
32
+
33
+wait_for localhost 19999 netdata || exit 1
34
+
35
+curl -sS http://127.0.0.1:19999/api/v1/info > ./response || exit 1
36
+
37
+cat ./response
38
25
-wait_for localhost 19999 netdata
39
+jq '.version' ./response || exit 1
40
27
-curl -sS http://127.0.0.1:19999/api/v1/info | jq '.version'
41
+trap - EXIT