Switch nightlies to GitHub releases. (#14020)
* Switch nightlies to GitHub releases. Instead of using GCS. * Fix CI handling. * Fix handling of download URLs for nightly builds. * Fix handling of redirects for consolidated artifact checks. * Avoid redirect issues with the test environment. * Add more info to logs for updater checks. * Ignore redirect issues for updater checks. * Fix base URL handling in updater. * Dump post-update info in CI before checking if the update worked. * Special-case a version of `latest` in updater. This is to allow CI to work correctly. * Update nightly release badge in README.md. * Fix updater check variable name. * Add a comment documenting the magic number in parse_version.
Austin S. Hemmelgarn committed
Jan 16, 2023 at 08:17 UTC
e5be62dcbcb4729d87c0a68c58b31a220b98b1c1
5 files changed
+69
-49
.github/scripts/run-updater-check.sh
+11
-2
@@ -4,11 +4,20 @@ echo ">>> Installing CI support packages..."
4
/netdata/.github/scripts/ci-support-pkgs.sh
5
echo ">>> Installing Netdata..."
6
/netdata/packaging/installer/kickstart.sh --dont-wait --build-only --disable-telemetry || exit 1
7
-echo "::group::Environment File Contents"
7
+echo "::group::>>> Pre-Update Environment File Contents"
8
cat /etc/netdata/.environment
9
echo "::endgroup::"
10
+echo "::group::>>> Pre-Update Netdata Build Info"
11
+netdata -W buildinfo
12
+echo "::endgroup::"
13
echo ">>> Updating Netdata..."
11
-export NETDATA_NIGHTLIES_BASEURL="http://localhost:8080/artifacts/" # Pull the tarball from the local web server.
14
+export NETDATA_BASE_URL="http://localhost:8080/artifacts/" # Pull the tarball from the local web server.
15
/netdata/packaging/installer/netdata-updater.sh --not-running-from-cron --no-updater-self-update || exit 1
16
+echo "::group::>>> Post-Update Environment File Contents"
17
+cat /etc/netdata/.environment
18
+echo "::endgroup::"
19
+echo "::group::>>> Post-Update Netdata Build Info"
20
+netdata -W buildinfo
21
+echo "::endgroup::"
22
echo ">>> Checking if update was successful..."
23
/netdata/.github/scripts/check-updater.sh || exit 1
.github/workflows/build.yml
+16
-6
@@ -417,10 +417,10 @@ jobs:
417
- name: Prepare artifact directory
418
id: prepare
419
run: |
420
- mkdir -p artifacts || exit 1
421
- echo "9999.0.0-0" > artifacts/latest-version.txt || exit 1
422
- cp dist-tarball/* artifacts || exit 1
423
- cd artifacts || exit 1
420
+ mkdir -p artifacts/download/latest || exit 1
421
+ echo "9999.0.0-0" > artifacts/download/latest/latest-version.txt || exit 1
422
+ cp dist-tarball/* artifacts/download/latest || exit 1
423
+ cd artifacts/download/latest || exit 1
424
ln -s ${{ needs.build-dist.outputs.distfile }} netdata-latest.tar.gz || exit 1
425
sha256sum -b ./* > "sha256sums.txt" || exit 1
426
cat sha256sums.txt
@@ -553,10 +553,15 @@ jobs:
553
with:
554
name: final-artifacts
555
path: artifacts
556
+ - name: Prepare artifacts directory
557
+ id: prepare
558
+ run: |
559
+ mkdir -p download/latest
560
+ mv artifacts/* download/latest
561
- name: Verify that artifacts work with installer
562
id: verify
563
env:
559
- NETDATA_TARBALL_BASEURL: http://localhost:8080/artifacts
564
+ NETDATA_TARBALL_BASEURL: http://localhost:8080/
565
run: packaging/installer/kickstart.sh --build-only --dont-start-it --disable-telemetry --dont-wait
566
- name: Failure Notification
567
uses: rtCamp/action-slack-notify@v2
@@ -602,10 +607,15 @@ jobs:
607
with:
608
name: final-artifacts
609
path: artifacts
610
+ - name: Prepare artifacts directory
611
+ id: prepare
612
+ run: |
613
+ mkdir -p download/latest
614
+ mv artifacts/* download/latest
615
- name: Verify that artifacts work with installer
616
id: verify
617
env:
608
- NETDATA_TARBALL_BASEURL: http://localhost:8080/artifacts
618
+ NETDATA_TARBALL_BASEURL: http://localhost:8080/
619
run: packaging/installer/kickstart.sh --static-only --dont-start-it --disable-telemetry
620
- name: Failure Notification
621
uses: rtCamp/action-slack-notify@v2
README.md
+1
-1
@@ -6,7 +6,7 @@
6
<a href="https://github.com/netdata/netdata/"><img src="https://img.shields.io/github/stars/netdata/netdata?style=social" alt="GitHub Stars"></a>
7
<br />
8
<a href="https://github.com/netdata/netdata/releases/latest"><img src="https://img.shields.io/github/release/netdata/netdata.svg" alt="Latest release"></a>
9
- <a href="https://storage.googleapis.com/netdata-nightlies/latest-version.txt"><img src="https://img.shields.io/badge/dynamic/xml?url=https://storage.googleapis.com/netdata-nightlies/latest-version.txt&label=nightly%20release&query=/text()" alt="Nightly release"></a>
9
+ <a href="https://github.com/netdata/netdata-nightlies/releases/latest"><img src="https://img.shields.io/github/release/netdata/netdata-nightlies.svg" alt="Latest nightly build"></a>
10
<br />
11
<a href="https://travis-ci.com/netdata/netdata"><img src="https://travis-ci.com/netdata/netdata.svg?branch=master" alt="Build status"></a>
12
<a href="https://bestpractices.coreinfrastructure.org/projects/2231"><img src="https://bestpractices.coreinfrastructure.org/projects/2231/badge" alt="CII Best Practices"></a>
packaging/installer/kickstart.sh
+22
-27
@@ -65,7 +65,7 @@ else
65
NETDATA_DISABLE_TELEMETRY=0
66
fi
67
68
-NETDATA_TARBALL_BASEURL="${NETDATA_TARBALL_BASEURL:-https://storage.googleapis.com/netdata-nightlies}"
68
+NETDATA_TARBALL_BASEURL="${NETDATA_TARBALL_BASEURL:-https://github.com/netdata/netdata-nightlies/releases}"
69
TELEMETRY_API_KEY="${NETDATA_POSTHOG_API_KEY:-mqkwGT0JNFqO-zX2t0mW6Tec9yooaVu7xCBlXtHnt5Y}"
70
71
if echo "${0}" | grep -q 'kickstart-static64'; then
@@ -1574,25 +1574,33 @@ set_static_archive_urls() {
1574
if [ -n "${NETDATA_OFFLINE_INSTALL_SOURCE}" ]; then
1575
path="$(cd "${NETDATA_OFFLINE_INSTALL_SOURCE}" || exit 1; pwd)"
1576
export NETDATA_STATIC_ARCHIVE_URL="file://${path}/netdata-${arch}-latest.gz.run"
1577
+ export NETDATA_STATIC_ARCHIVE_NAME="netdata-${arch}-latest.gz.run"
1578
export NETDATA_STATIC_ARCHIVE_CHECKSUM_URL="file://${path}/sha256sums.txt"
1579
elif [ "${1}" = "stable" ]; then
1580
if [ -n "${INSTALL_VERSION}" ]; then
1580
- export NETDATA_STATIC_ARCHIVE_OLD_URL="https://github.com/netdata/netdata/releases/download/v${INSTALL_VERSION}/netdata-v${INSTALL_VERSION}.gz.run"
1581
export NETDATA_STATIC_ARCHIVE_URL="https://github.com/netdata/netdata/releases/download/v${INSTALL_VERSION}/netdata-${arch}-v${INSTALL_VERSION}.gz.run"
1582
+ export NETDATA_STATIC_ARCHIVE_OLD_URL="https://github.com/netdata/netdata/releases/download/v${INSTALL_VERSION}/netdata-v${INSTALL_VERSION}.gz.run"
1583
+ export NETDATA_STATIC_ARCHIVE_NAME="netdata-${arch}-v${INSTALL_VERSION}.gz.run"
1584
+ export NETDATA_STATIC_ARCHIVE_OLD_NAME="netdata-v${INSTALL_VERSION}.gz.run"
1585
export NETDATA_STATIC_ARCHIVE_CHECKSUM_URL="https://github.com/netdata/netdata/releases/download/v${INSTALL_VERSION}/sha256sums.txt"
1586
else
1587
latest="$(get_redirect "https://github.com/netdata/netdata/releases/latest")"
1588
export NETDATA_STATIC_ARCHIVE_URL="https://github.com/netdata/netdata/releases/download/${latest}/netdata-${arch}-latest.gz.run"
1589
+ export NETDATA_STATIC_ARCHIVE_NAME="netdata-${arch}-latest.gz.run"
1590
export NETDATA_STATIC_ARCHIVE_CHECKSUM_URL="https://github.com/netdata/netdata/releases/download/${latest}/sha256sums.txt"
1591
fi
1592
else
1593
if [ -n "${INSTALL_VERSION}" ]; then
1590
- export NETDATA_STATIC_ARCHIVE_URL="${NETDATA_TARBALL_BASEURL}/netdata-${arch}-v${INSTALL_VERSION}.gz.run"
1591
- export NETDATA_STATIC_ARCHIVE_OLD_URL="${NETDATA_TARBALL_BASEURL}/netdata-v${INSTALL_VERSION}.gz.run"
1592
- export NETDATA_STATIC_ARCHIVE_CHECKSUM_URL="${NETDATA_TARBALL_BASEURL}/sha256sums.txt"
1594
+ export NETDATA_STATIC_ARCHIVE_URL="${NETDATA_TARBALL_BASEURL}/download/v${INSTALL_VERSION}/netdata-${arch}-v${INSTALL_VERSION}.gz.run"
1595
+ export NETDATA_STATIC_ARCHIVE_OLD_URL="${NETDATA_TARBALL_BASEURL}/download/v${INSTALL_VERSION}/netdata-v${INSTALL_VERSION}.gz.run"
1596
+ export NETDATA_STATIC_ARCHIVE_NAME="netdata-${arch}-v${INSTALL_VERSION}.gz.run"
1597
+ export NETDATA_STATIC_ARCHIVE_OLD_NAME="netdata-v${INSTALL_VERSION}.gz.run"
1598
+ export NETDATA_STATIC_ARCHIVE_CHECKSUM_URL="${NETDATA_TARBALL_BASEURL}/download/v${INSTALL_VERSION}/sha256sums.txt"
1599
else
1594
- export NETDATA_STATIC_ARCHIVE_URL="${NETDATA_TARBALL_BASEURL}/netdata-${arch}-latest.gz.run"
1595
- export NETDATA_STATIC_ARCHIVE_CHECKSUM_URL="${NETDATA_TARBALL_BASEURL}/sha256sums.txt"
1600
+ tag="$(get_redirect "${NETDATA_TARBALL_BASEURL}/latest")"
1601
+ export NETDATA_STATIC_ARCHIVE_URL="${NETDATA_TARBALL_BASEURL}/download/${tag}/netdata-${arch}-latest.gz.run"
1602
+ export NETDATA_STATIC_ARCHIVE_NAME="netdata-${arch}-latest.gz.run"
1603
+ export NETDATA_STATIC_ARCHIVE_CHECKSUM_URL="${NETDATA_TARBALL_BASEURL}/download/${tag}/sha256sums.txt"
1604
fi
1605
fi
1606
}
@@ -1607,23 +1615,9 @@ try_static_install() {
1615
1616
# Check status code first, so that we can provide nicer fallback for dry runs.
1617
if check_for_remote_file "${NETDATA_STATIC_ARCHIVE_URL}"; then
1610
- if [ -n "${NETDATA_OFFLINE_INSTALL_SOURCE}" ]; then
1611
- netdata_agent="$(basename "${NETDATA_STATIC_ARCHIVE_URL#"file://"}")"
1612
- elif [ -n "${INSTALL_VERSION}" ]; then
1613
- if [ "${SELECTED_RELEASE_CHANNEL}" = "stable" ]; then
1614
- netdata_agent="${NETDATA_STATIC_ARCHIVE_URL#"https://github.com/netdata/netdata/releases/download/v${INSTALL_VERSION}/"}"
1615
- else
1616
- netdata_agent="${NETDATA_STATIC_ARCHIVE_URL#"${NETDATA_TARBALL_BASEURL}/"}"
1617
- fi
1618
- else
1619
- if [ "${SELECTED_RELEASE_CHANNEL}" = "stable" ]; then
1620
- netdata_agent="${NETDATA_STATIC_ARCHIVE_URL#"https://github.com/netdata/netdata/releases/download/${latest}/"}"
1621
- else
1622
- netdata_agent="${NETDATA_STATIC_ARCHIVE_URL#"${NETDATA_TARBALL_BASEURL}/"}"
1623
- fi
1624
- fi
1618
+ netdata_agent="${NETDATA_STATIC_ARCHIVE_NAME}"
1619
elif [ "${SYSARCH}" = "x86_64" ] && check_for_remote_file "${NETDATA_STATIC_ARCHIVE_OLD_URL}"; then
1626
- netdata_agent="${NETDATA_STATIC_ARCHIVE_OLD_URL#"https://github.com/netdata/netdata/releases/download/v${INSTALL_VERSION}/"}"
1620
+ netdata_agent="${NETDATA_STATIC_ARCHIVE_OLD_NAME}"
1621
export NETDATA_STATIC_ARCHIVE_URL="${NETDATA_STATIC_ARCHIVE_OLD_URL}"
1622
else
1623
warning "There is no static build available for ${SYSARCH} CPUs. This usually means we simply do not currently provide static builds for ${SYSARCH} CPUs."
@@ -1692,11 +1686,12 @@ set_source_archive_urls() {
1686
fi
1687
else
1688
if [ -n "${INSTALL_VERSION}" ]; then
1695
- export NETDATA_SOURCE_ARCHIVE_URL="${NETDATA_TARBALL_BASEURL}/netdata-v${INSTALL_VERSION}.tar.gz"
1696
- export NETDATA_SOURCE_ARCHIVE_CHECKSUM_URL="${NETDATA_TARBALL_BASEURL}/sha256sums.txt"
1689
+ export NETDATA_SOURCE_ARCHIVE_URL="${NETDATA_TARBALL_BASEURL}/download/v${INSTALL_VERSION}/netdata-latest.tar.gz"
1690
+ export NETDATA_SOURCE_ARCHIVE_CHECKSUM_URL="${NETDATA_TARBALL_BASEURL}/download/v${INSTALL_VERSION}/sha256sums.txt"
1691
else
1698
- export NETDATA_SOURCE_ARCHIVE_URL="${NETDATA_TARBALL_BASEURL}/netdata-latest.tar.gz"
1699
- export NETDATA_SOURCE_ARCHIVE_CHECKSUM_URL="${NETDATA_TARBALL_BASEURL}/sha256sums.txt"
1692
+ tag="$(get_redirect "${NETDATA_TARBALL_BASEURL}/latest")"
1693
+ export NETDATA_SOURCE_ARCHIVE_URL="${NETDATA_TARBALL_BASEURL}/download/${tag}/netdata-latest.tar.gz"
1694
+ export NETDATA_SOURCE_ARCHIVE_CHECKSUM_URL="${NETDATA_TARBALL_BASEURL}/download/${tag}/sha256sums.txt"
1695
fi
1696
fi
1697
}
packaging/installer/netdata-updater.sh
+19
-13
@@ -34,6 +34,9 @@ set -e
34
35
PACKAGES_SCRIPT="https://raw.githubusercontent.com/netdata/netdata/master/packaging/installer/install-required-packages.sh"
36
37
+NETDATA_STABLE_BASE_URL="${NETDATA_BASE_URL:-https://github.com/netdata/netdata/releases}"
38
+NETDATA_NIGHTLY_BASE_URL="${NETDATA_BASE_URL:-https://github.com/netdata/netdata-nightlies/releases}"
39
+
40
script_dir="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)"
41
42
if [ -x "${script_dir}/netdata-updater" ]; then
@@ -369,8 +372,8 @@ download() {
372
}
373
374
get_netdata_latest_tag() {
372
- dest="${1}"
373
- url="https://github.com/netdata/netdata/releases/latest"
375
+ url="${1}/latest"
376
+ dest="${2}"
377
378
if command -v curl >/dev/null 2>&1; then
379
tag=$(curl "${url}" -s -L -I -o /dev/null -w '%{url_effective}' | grep -m 1 -o '[^/]*$')
@@ -439,7 +442,12 @@ self_update() {
442
443
parse_version() {
444
r="${1}"
442
- if echo "${r}" | grep -q '^v.*'; then
445
+ if [ "${r}" = "latest" ]; then
446
+ # If we get ‘latest’ as a version, return the largest possible
447
+ # version value.
448
+ printf "99999999999999"
449
+ return 0
450
+ elif echo "${r}" | grep -q '^v.*'; then
451
# shellcheck disable=SC2001
452
# XXX: Need a regex group substitution here.
453
r="$(echo "${r}" | sed -e 's/^v\(.*\)/\1/')"
@@ -463,9 +471,9 @@ parse_version() {
471
472
get_latest_version() {
473
if [ "${RELEASE_CHANNEL}" = "stable" ]; then
466
- get_netdata_latest_tag /dev/stdout
474
+ get_netdata_latest_tag "${NETDATA_STABLE_BASE_URL}" /dev/stdout
475
else
468
- download "$NETDATA_NIGHTLIES_BASEURL/latest-version.txt" /dev/stdout
476
+ get_netdata_latest_tag "${NETDATA_NIGHTLY_BASE_URL}" /dev/stdout
477
fi
478
}
479
@@ -529,12 +537,13 @@ set_tarball_urls() {
537
fi
538
539
if [ "$1" = "stable" ]; then
532
- latest="$(get_netdata_latest_tag /dev/stdout)"
533
- export NETDATA_TARBALL_URL="https://github.com/netdata/netdata/releases/download/$latest/${filename}"
534
- export NETDATA_TARBALL_CHECKSUM_URL="https://github.com/netdata/netdata/releases/download/$latest/sha256sums.txt"
540
+ latest="$(get_netdata_latest_tag "${NETDATA_STABLE_BASE_URL}" /dev/stdout)"
541
+ export NETDATA_TARBALL_URL="${NETDATA_STABLE_BASE_URL}/download/$latest/${filename}"
542
+ export NETDATA_TARBALL_CHECKSUM_URL="${NETDATA_STABLE_BASE_URL}/download/$latest/sha256sums.txt"
543
else
536
- export NETDATA_TARBALL_URL="$NETDATA_NIGHTLIES_BASEURL/${filename}"
537
- export NETDATA_TARBALL_CHECKSUM_URL="$NETDATA_NIGHTLIES_BASEURL/sha256sums.txt"
544
+ tag="$(get_netdata_latest_tag "${NETDATA_NIGHTLY_BASE_URL}" /dev/stdout)"
545
+ export NETDATA_TARBALL_URL="${NETDATA_NIGHTLY_BASE_URL}/download/${tag}/${filename}"
546
+ export NETDATA_TARBALL_CHECKSUM_URL="${NETDATA_NIGHTLY_BASE_URL}/download/${tag}/sha256sums.txt"
547
fi
548
}
549
@@ -907,9 +916,6 @@ export NETDATA_LIB_DIR="${NETDATA_LIB_DIR:-${NETDATA_PREFIX}/var/lib/netdata}"
916
# Source the tarball checksum, if not already available from environment (for existing installations with the old logic)
917
[ -z "${NETDATA_TARBALL_CHECKSUM}" ] && [ -f "${NETDATA_LIB_DIR}/netdata.tarball.checksum" ] && NETDATA_TARBALL_CHECKSUM="$(cat "${NETDATA_LIB_DIR}/netdata.tarball.checksum")"
918
910
-# Grab the nightlies baseurl (defaulting to our Google Storage bucket)
911
-export NETDATA_NIGHTLIES_BASEURL="${NETDATA_NIGHTLIES_BASEURL:-https://storage.googleapis.com/netdata-nightlies}"
912
-
919
if echo "$INSTALL_TYPE" | grep -qv ^binpkg && [ "${INSTALL_UID}" != "$(id -u)" ]; then
920
fatal "You are running this script as user with uid $(id -u). We recommend to run this script as root (user with uid 0)" U0011
921
fi