Fix reliability of kickstart/kickstart-static64 with checksums sometimes failing (#9165)
* Add tool to build the dist and static x864_64 artifacts * Add support for overriding the URL for installing/updating from nightlies * Refactor curl/wget usage into less places. Use curl -q and --output and -O options instead of redirecting stdout * Update md5sum(s) of kickstart/kickstart-static64
James Mills committed
Jun 3, 2020 at 00:46 UTC
57c916530c7ce9b7ca7905bfc06d676d34dd2f76
8 files changed
+72
-51
.github/scripts/build-artifacts.sh
+3
-2
@@ -16,6 +16,7 @@ prepare_build() {
16
progress "Preparing build"
17
(
18
test -d artifacts || mkdir -p artifacts
19
+ echo "${VERSION}" > packaging/version
20
) >&2
21
}
22
@@ -50,8 +51,8 @@ prepare_assets() {
51
cp packaging/version artifacts/latest-version.txt
52
53
cd artifacts || exit 1
53
- ln -s "${BASENAME}.tar.gz" netdata-latest.tar.gz
54
- ln -s "${BASENAME}.gz.run" netdata-latest.gz.run
54
+ ln -f "${BASENAME}.tar.gz" netdata-latest.tar.gz
55
+ ln -f "${BASENAME}.gz.run" netdata-latest.gz.run
56
sha256sum -b ./* > "sha256sums.txt"
57
) >&2
58
}
netdata-installer.sh
+1
-21
@@ -73,28 +73,8 @@ else
73
source "${NETDATA_SOURCE_DIR}/packaging/installer/functions.sh" || exit 1
74
fi
75
76
-download_tarball() {
77
- url="${1}"
78
- dest="${2}"
79
- name="${3}"
80
- opt="${4}"
81
-
82
- if command -v curl > /dev/null 2>&1; then
83
- run curl -sSL --connect-timeout 10 --retry 3 "${url}" > "${dest}"
84
- elif command -v wget > /dev/null 2>&1; then
85
- run wget -T 15 -O - "${url}" > "${dest}"
86
- else
87
- echo >&2
88
- echo >&2 "Downloading ${name} from '${url}' failed because of missing mandatory packages."
89
- echo >&2 "Either add packages or disable it by issuing '--disable-${opt}' in the installer"
90
- echo >&2
91
-
92
- run_failed "I need curl or wget to proceed, but neither is available on this system."
93
- fi
94
-}
95
-
76
download_go() {
97
- download_tarball "${1}" "${2}" "go.d plugin" "go"
77
+ download_file "${1}" "${2}" "go.d plugin" "go"
78
}
79
80
# make sure we save all commands we run
packaging/installer/functions.sh
+24
-2
@@ -95,7 +95,7 @@ progress() {
95
get() {
96
url="${1}"
97
if command -v curl > /dev/null 2>&1; then
98
- curl -sSL -o - --connect-timeout 10 --retry 3 "${url}"
98
+ curl -q -o - -sSL --connect-timeout 10 --retry 3 "${url}"
99
elif command -v wget > /dev/null 2>&1; then
100
wget -T 15 -O - "${url}"
101
else
@@ -103,6 +103,28 @@ get() {
103
fi
104
}
105
106
+download_file() {
107
+ url="${1}"
108
+ dest="${2}"
109
+ name="${3}"
110
+ opt="${4}"
111
+
112
+ if command -v curl > /dev/null 2>&1; then
113
+ run curl -q -sSL --connect-timeout 10 --retry 3 --output "${dest}" "${url}"
114
+ elif command -v wget > /dev/null 2>&1; then
115
+ run wget -T 15 -O "${dest}" "${url}"
116
+ else
117
+ echo >&2
118
+ echo >&2 "Downloading ${name} from '${url}' failed because of missing mandatory packages."
119
+ if [ -n "$opt" ]; then
120
+ echo >&2 "Either add packages or disable it by issuing '--disable-${opt}' in the installer"
121
+ fi
122
+ echo >&2
123
+
124
+ run_failed "I need curl or wget to proceed, but neither is available on this system."
125
+ fi
126
+}
127
+
128
# -----------------------------------------------------------------------------
129
# external component handling
130
@@ -114,7 +136,7 @@ fetch_and_verify() {
136
local override=${5}
137
138
if [ -z "${override}" ]; then
117
- get "${url}" > "${tmp}/${base_name}"
139
+ download_file "${url}" "${tmp}/${base_name}" "${component}"
140
else
141
progress "Using provided ${component} archive ${override}"
142
run cp "${override}" "${tmp}/${base_name}"
packaging/installer/kickstart-static64.sh
+18
-12
@@ -11,7 +11,11 @@
11
# --disable-telemetry Opt-out of anonymous telemetry program (DO_NOT_TRACK=1)
12
# --local-files Use a manually provided tarball for the installation
13
#
14
-# ---------------------------------------------------------------------------------------------------------------------
14
+# Environment options:
15
+#
16
+# NETDATA_TARBALL_BASEURL set the base url for downloading the dist tarball
17
+#
18
+# ----------------------------------------------------------------------------
19
# library functions copied from packaging/installer/functions.sh
20
21
setup_terminal() {
@@ -43,7 +47,7 @@ setup_terminal() {
47
}
48
setup_terminal || echo > /dev/null
49
46
-# -----------------------------------------------------------------------------
50
+# ----------------------------------------------------------------------------
51
fatal() {
52
printf >&2 "${TPUT_BGRED}${TPUT_WHITE}${TPUT_BOLD} ABORTED ${TPUT_RESET} ${*} \n\n"
53
exit 1
@@ -130,9 +134,9 @@ download() {
134
url="${1}"
135
dest="${2}"
136
if command -v curl > /dev/null 2>&1; then
133
- run curl -sSL --connect-timeout 10 --retry 3 "${url}" > "${dest}" || fatal "Cannot download ${url}"
137
+ run curl -q -sSL --connect-timeout 10 --retry 3 --output "${dest}" "${url}"
138
elif command -v wget > /dev/null 2>&1; then
135
- run wget -T 15 -O - "${url}" > "${dest}" || fatal "Cannot download ${url}"
139
+ run wget -T 15 -O "${dest}" "${url}" || fatal "Cannot download ${url}"
140
else
141
fatal "I need curl or wget to proceed, but neither is available on this system."
142
fi
@@ -147,13 +151,12 @@ set_tarball_urls() {
151
if [ "$1" = "stable" ]; then
152
local latest
153
# Simple version
150
- # latest="$(curl -sSL https://api.github.com/repos/netdata/netdata/releases/latest | grep tag_name | cut -d'"' -f4)"
154
latest="$(download "https://api.github.com/repos/netdata/netdata/releases/latest" /dev/stdout | grep tag_name | cut -d'"' -f4)"
155
export NETDATA_TARBALL_URL="https://github.com/netdata/netdata/releases/download/$latest/netdata-$latest.gz.run"
156
export NETDATA_TARBALL_CHECKSUM_URL="https://github.com/netdata/netdata/releases/download/$latest/sha256sums.txt"
157
else
155
- export NETDATA_TARBALL_URL="https://storage.googleapis.com/netdata-nightlies/netdata-latest.gz.run"
156
- export NETDATA_TARBALL_CHECKSUM_URL="https://storage.googleapis.com/netdata-nightlies/sha256sums.txt"
158
+ export NETDATA_TARBALL_URL="$NETDATA_TARBALL_BASEURL/netdata-latest.gz.run"
159
+ export NETDATA_TARBALL_CHECKSUM_URL="$NETDATA_TARBALL_BASEURL/sha256sums.txt"
160
fi
161
}
162
@@ -169,14 +172,14 @@ safe_sha256sum() {
172
fi
173
}
174
172
-# ---------------------------------------------------------------------------------------------------------------------
175
+# ----------------------------------------------------------------------------
176
umask 022
177
178
sudo=""
179
[ -z "${UID}" ] && UID="$(id -u)"
180
[ "${UID}" -ne "0" ] && sudo="sudo"
181
179
-# ---------------------------------------------------------------------------------------------------------------------
182
+# ----------------------------------------------------------------------------
183
if [ "$(uname -m)" != "x86_64" ]; then
184
fatal "Static binary versions of netdata are available only for 64bit Intel/AMD CPUs (x86_64), but yours is: $(uname -m)."
185
fi
@@ -185,7 +188,7 @@ if [ "$(uname -s)" != "Linux" ]; then
188
fatal "Static binary versions of netdata are available only for Linux, but this system is $(uname -s)"
189
fi
190
188
-# ---------------------------------------------------------------------------------------------------------------------
191
+# ----------------------------------------------------------------------------
192
opts=
193
NETDATA_INSTALLER_OPTIONS=""
194
NETDATA_UPDATES="--auto-update"
@@ -235,7 +238,10 @@ if [ ! "${DO_NOT_TRACK:-0}" -eq 0 ] || [ -n "$DO_NOT_TRACK" ]; then
238
NETDATA_INSTALLER_OPTIONS="${NETDATA_INSTALLER_OPTIONS:+${NETDATA_INSTALLER_OPTIONS} }--disable-telemtry"
239
fi
240
238
-# ---------------------------------------------------------------------------------------------------------------------
241
+# Netdata Tarball Base URL (defaults to our Google Storage Bucket)
242
+[ -z "$NETDATA_TARBALL_BASEURL" ] && NETDATA_TARBALL_BASEURL=https://storage.googleapis.com/netdata-nightlies
243
+
244
+# ----------------------------------------------------------------------------
245
TMPDIR=$(create_tmp_directory)
246
cd "${TMPDIR}" || exit 1
247
@@ -255,7 +261,7 @@ if ! grep netdata-latest.gz.run "${TMPDIR}/sha256sum.txt" | safe_sha256sum -c -
261
fatal "Static binary checksum validation failed. Stopping netdata installation and leaving binary in ${TMPDIR}"
262
fi
263
258
-# ---------------------------------------------------------------------------------------------------------------------
264
+# ----------------------------------------------------------------------------
265
progress "Installing netdata"
266
run ${sudo} sh "${TMPDIR}/netdata-latest.gz.run" ${opts} -- ${NETDATA_UPDATES} ${NETDATA_INSTALLER_OPTIONS}
267
packaging/installer/kickstart.sh
+12
-6
@@ -13,7 +13,11 @@
13
# --dont-wait do not prompt for user input
14
# --non-interactive do not prompt for user input
15
# --no-updates do not install script for daily updates
16
-# --local-files set the full path of the desired tarball to run install with
16
+# --local-files set the full path of the desired tarball to run install with
17
+#
18
+# Environment options:
19
+#
20
+# NETDATA_TARBALL_BASEURL set the base url for downloading the dist tarball
21
#
22
# This script will:
23
#
@@ -29,6 +33,9 @@
33
# External files
34
PACKAGES_SCRIPT="https://raw.githubusercontent.com/netdata/netdata/master/packaging/installer/install-required-packages.sh"
35
36
+# Netdata Tarball Base URL (defaults to our Google Storage Bucket)
37
+[ -z "$NETDATA_TARBALL_BASEURL" ] && NETDATA_TARBALL_BASEURL=https://storage.googleapis.com/netdata-nightlies
38
+
39
# ---------------------------------------------------------------------------------------------------------------------
40
# library functions copied from packaging/installer/functions.sh
41
@@ -158,9 +165,9 @@ download() {
165
url="${1}"
166
dest="${2}"
167
if command -v curl > /dev/null 2>&1; then
161
- run curl -sSL --connect-timeout 10 --retry 3 "${url}" > "${dest}" || fatal "Cannot download ${url}"
168
+ run curl -q -sSL --connect-timeout 10 --retry 3 --output "${dest}" "${url}"
169
elif command -v wget > /dev/null 2>&1; then
163
- run wget -T 15 -O - "${url}" > "${dest}" || fatal "Cannot download ${url}"
170
+ run wget -T 15 -O "${dest}" "${url}" || fatal "Cannot download ${url}"
171
else
172
fatal "I need curl or wget to proceed, but neither is available on this system."
173
fi
@@ -175,13 +182,12 @@ set_tarball_urls() {
182
if [ "$1" = "stable" ]; then
183
local latest
184
# Simple version
178
- # latest="$(curl -sSL https://api.github.com/repos/netdata/netdata/releases/latest | grep tag_name | cut -d'"' -f4)"
185
latest="$(download "https://api.github.com/repos/netdata/netdata/releases/latest" /dev/stdout | grep tag_name | cut -d'"' -f4)"
186
export NETDATA_TARBALL_URL="https://github.com/netdata/netdata/releases/download/$latest/netdata-$latest.tar.gz"
187
export NETDATA_TARBALL_CHECKSUM_URL="https://github.com/netdata/netdata/releases/download/$latest/sha256sums.txt"
188
else
183
- export NETDATA_TARBALL_URL="https://storage.googleapis.com/netdata-nightlies/netdata-latest.tar.gz"
184
- export NETDATA_TARBALL_CHECKSUM_URL="https://storage.googleapis.com/netdata-nightlies/sha256sums.txt"
189
+ export NETDATA_TARBALL_URL="$NETDATA_TARBALL_BASEURL/netdata-latest.tar.gz"
190
+ export NETDATA_TARBALL_CHECKSUM_URL="$NETDATA_TARBALL_BASEURL/sha256sums.txt"
191
fi
192
}
193
packaging/installer/methods/kickstart-64.md
+1
-1
@@ -76,7 +76,7 @@ To use `md5sum` to verify the intregity of the `kickstart-static64.sh` script yo
76
command above, run the following:
77
78
```bash
79
-[ "33ecd20452f569c1d9972bcefdf04692" = "$(curl -Ss https://my-netdata.io/kickstart-static64.sh | md5sum | cut -d ' ' -f 1)" ] && echo "OK, VALID" || echo "FAILED, INVALID"
79
+[ "ff717737ccc2212a3363dad7fa8bd20d" = "$(curl -Ss https://my-netdata.io/kickstart-static64.sh | md5sum | cut -d ' ' -f 1)" ] && echo "OK, VALID" || echo "FAILED, INVALID"
80
```
81
82
If the script is valid, this command will return `OK, VALID`.
packaging/installer/methods/kickstart.md
+1
-1
@@ -58,7 +58,7 @@ To use `md5sum` to verify the intregity of the `kickstart.sh` script you will do
58
run the following:
59
60
```bash
61
-[ "f1e7aa92ca64b797fdc14df59faad205" = "$(curl -Ss https://my-netdata.io/kickstart.sh | md5sum | cut -d ' ' -f 1)" ] && echo "OK, VALID" || echo "FAILED, INVALID"
61
+[ "2057599f8b11ce56f85aa7f26ce7b15b" = "$(curl -Ss https://my-netdata.io/kickstart.sh | md5sum | cut -d ' ' -f 1)" ] && echo "OK, VALID" || echo "FAILED, INVALID"
62
```
63
64
If the script is valid, this command will return `OK, VALID`.
packaging/installer/netdata-updater.sh
+12
-6
@@ -14,6 +14,10 @@
14
# - NETDATA_TARBALL_CHECKSUM
15
# - NETDATA_PREFIX / NETDATA_LIB_DIR (After 1.16.1 we will only depend on lib dir)
16
#
17
+# Optional environment options:
18
+#
19
+# - NETDATA_TARBALL_BASEURL (set the base url for downloading the dist tarball)
20
+#
21
# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
22
#
23
# Author: Paweł Krupa <paulfantom@gmail.com>
@@ -108,7 +112,7 @@ get_latest_version() {
112
if [ "${RELEASE_CHANNEL}" == "stable" ]; then
113
latest="$(download "https://api.github.com/repos/netdata/netdata/releases/latest" /dev/stdout | grep tag_name | cut -d'"' -f4)"
114
else
111
- latest="$(download "https://storage.googleapis.com/netdata-nightlies/latest-version.txt" /dev/stdout)"
115
+ latest="$(download "$NETDATA_TARBALL_BASEURL/latest-version.txt" /dev/stdout)"
116
fi
117
parse_version "$latest"
118
}
@@ -123,13 +127,12 @@ set_tarball_urls() {
127
if [ "$1" = "stable" ]; then
128
local latest
129
# Simple version
126
- # latest="$(curl -sSL https://api.github.com/repos/netdata/netdata/releases/latest | grep tag_name | cut -d'"' -f4)"
130
latest="$(download "https://api.github.com/repos/netdata/netdata/releases/latest" /dev/stdout | grep tag_name | cut -d'"' -f4)"
131
export NETDATA_TARBALL_URL="https://github.com/netdata/netdata/releases/download/$latest/netdata-$latest.${extension}"
132
export NETDATA_TARBALL_CHECKSUM_URL="https://github.com/netdata/netdata/releases/download/$latest/sha256sums.txt"
133
else
131
- export NETDATA_TARBALL_URL="https://storage.googleapis.com/netdata-nightlies/netdata-latest.${extension}"
132
- export NETDATA_TARBALL_CHECKSUM_URL="https://storage.googleapis.com/netdata-nightlies/sha256sums.txt"
134
+ export NETDATA_TARBALL_URL="$NETDATA_TARBALL_BASEURL/netdata-latest.${extension}"
135
+ export NETDATA_TARBALL_CHECKSUM_URL="$NETDATA_TARBALL_BASEURL/sha256sums.txt"
136
fi
137
}
138
@@ -185,8 +188,8 @@ update() {
188
do_not_start="--dont-start-it"
189
fi
190
188
- if [ -n "${NETDATA_SELECTED_DASHBOARD}" ] ; then
189
- env="NETDATA_SELECTED_DASHBOARD=${NETDATA_SELECTED_DASHBOARD}"
191
+ if [ -n "${NETDATA_SELECTED_DASHBOARD}" ]; then
192
+ env="NETDATA_SELECTED_DASHBOARD=${NETDATA_SELECTED_DASHBOARD}"
193
fi
194
195
info "Re-installing netdata..."
@@ -222,6 +225,9 @@ export NETDATA_LIB_DIR="${NETDATA_LIB_DIR:-${NETDATA_PREFIX}/var/lib/netdata}"
225
# Source the tarbal checksum, if not already available from environment (for existing installations with the old logic)
226
[[ -z "${NETDATA_TARBALL_CHECKSUM}" ]] && [[ -f ${NETDATA_LIB_DIR}/netdata.tarball.checksum ]] && NETDATA_TARBALL_CHECKSUM="$(cat "${NETDATA_LIB_DIR}/netdata.tarball.checksum")"
227
228
+# Netdata Tarball Base URL (defaults to our Google Storage Bucket)
229
+[ -z "$NETDATA_TARBALL_BASEURL" ] && NETDATA_TARBALL_BASEURL=https://storage.googleapis.com/netdata-nightlies
230
+
231
if [ "${INSTALL_UID}" != "$(id -u)" ]; then
232
fatal "You are running this script as user with uid $(id -u). We recommend to run this script as root (user with uid 0)"
233
fi