Cache invariant components in static builds to reduce build times. (#12877)
* Add basic build caching support to static builds. Cache is store din `artifacts/cache/${BUILDARCH}`. Each third-party component utilizes a separate build cache. Invalidation is only done for version changes (more rigorous invalidation is expected to be handled externally). * Integrate static build caching with CI. * Fix fping cache handling. * Test caching in CI. * Properly skip rebuilds on cache hits. * Remove static build container when done with it. * Reuse existing image automatically if it’s for the correct platform. * Test CI build caching. * Fix static build job names.
Austin S. Hemmelgarn committed
May 27, 2022 at 07:03 UTC
0542cc3d6bead0535afe5519eea459107c1c0ccd
10 files changed
+228
-108
.github/scripts/get-static-cache-key.sh
new
+15
@@ -0,0 +1,15 @@
1
+#!/bin/sh
2
+
3
+arch="${1}"
4
+platform="$(packaging/makeself/uname2platform.sh "${arch}")"
5
+
6
+docker pull --platform "${platform}" netdata/static-builder
7
+
8
+# shellcheck disable=SC2046
9
+cat $(find packaging/makeself/jobs -type f ! -regex '.*\(netdata\|-makeself\).*') > /tmp/static-cache-key-data
10
+
11
+docker run -it --rm --platform "${platform}" netdata/static-builder sh -c 'apk list -I 2>/dev/null' >> /tmp/static-cache-key-data
12
+
13
+h="$(sha256sum /tmp/static-cache-key-data | cut -f 1 -d ' ')"
14
+
15
+echo "::set-output name=key::static-${arch}-${h}"
.github/workflows/build.yml
+13
-4
@@ -88,10 +88,10 @@ jobs:
88
strategy:
89
matrix:
90
arch:
91
- - 'x86_64'
92
- - 'armv7l'
93
- - 'aarch64'
94
- - 'ppc64le'
91
+ - x86_64
92
+ - armv7l
93
+ - aarch64
94
+ - ppc64le
95
steps:
96
- name: Checkout
97
id: checkout
@@ -104,6 +104,15 @@ jobs:
104
if: github.event_name == 'workflow_dispatch' && github.event.inputs.type != 'nightly'
105
run: |
106
sed -i 's/^RELEASE_CHANNEL="nightly" *#/RELEASE_CHANNEL="stable" #/' netdata-installer.sh packaging/makeself/install-or-update.sh
107
+ - name: Get Cache Key
108
+ id: cache-key
109
+ run: .github/scripts/get-static-cache-key.sh ${{ matrix.arch }}
110
+ - name: Cache
111
+ id: cache
112
+ uses: actions/cache@v3
113
+ with:
114
+ path: artifacts/cache
115
+ key: ${{ steps.cache-key.outputs.key }}
116
- name: Build
117
if: github.event_name != 'workflow_dispatch' # Don’t use retries on PRs.
118
run: .github/scripts/build-static.sh ${{ matrix.arch }}
packaging/makeself/build-static.sh
+13
-13
@@ -9,16 +9,11 @@ BUILDARCH="${1}"
9
10
set -e
11
12
-case ${BUILDARCH} in
13
- x86_64) platform=linux/amd64 ;;
14
- armv7l) platform=linux/arm/v7 ;;
15
- aarch64) platform=linux/arm64/v8 ;;
16
- ppc64le) platform=linux/ppc64le ;;
17
- *)
18
- echo "Unknown target architecture '${BUILDARCH}'."
12
+platform="$("$(dirname "${0}")/uname2platform.sh" "${BUILDARCH}")"
13
+
14
+if [ -z "${platform}" ]; then
15
exit 1
20
- ;;
21
-esac
16
+fi
17
18
DOCKER_IMAGE_NAME="netdata/static-builder"
19
@@ -27,18 +22,23 @@ if [ "${BUILDARCH}" != "$(uname -m)" ] && [ "$(uname -m)" = 'x86_64' ] && [ -z "
22
fi
23
24
if docker inspect "${DOCKER_IMAGE_NAME}" > /dev/null 2>&1; then
30
- docker image rm "${DOCKER_IMAGE_NAME}" || exit 1
25
+ img_platform="$(docker image inspect netdata/static-builder --format '{{.Os}}/{{.Architecture}}/{{.Variant}}')"
26
+ if [ "${img_platform%'/'}" != "${platform}" ]; then
27
+ docker image rm "${DOCKER_IMAGE_NAME}" || exit 1
28
+ fi
29
fi
30
33
-docker pull --platform "${platform}" "${DOCKER_IMAGE_NAME}"
31
+if ! docker inspect "${DOCKER_IMAGE_NAME}" > /dev/null 2>&1; then
32
+ docker pull --platform "${platform}" "${DOCKER_IMAGE_NAME}"
33
+fi
34
35
# Run the build script inside the container
36
if [ -t 1 ]; then
37
- run docker run -e BUILDARCH="${BUILDARCH}" -a stdin -a stdout -a stderr -i -t -v "$(pwd)":/netdata:rw \
37
+ run docker run --rm -e BUILDARCH="${BUILDARCH}" -a stdin -a stdout -a stderr -i -t -v "$(pwd)":/netdata:rw \
38
"${DOCKER_IMAGE_NAME}" \
39
/bin/sh /netdata/packaging/makeself/build.sh "${@}"
40
else
41
- run docker run -e BUILDARCH="${BUILDARCH}" -v "$(pwd)":/netdata:rw \
41
+ run docker run --rm -e BUILDARCH="${BUILDARCH}" -v "$(pwd)":/netdata:rw \
42
-e GITHUB_ACTIONS="${GITHUB_ACTIONS}" "${DOCKER_IMAGE_NAME}" \
43
/bin/sh /netdata/packaging/makeself/build.sh "${@}"
44
fi
packaging/makeself/functions.sh
+45
-15
@@ -29,34 +29,64 @@ set -euo pipefail
29
# -----------------------------------------------------------------------------
30
31
fetch() {
32
- local dir="${1}" url="${2}" sha256="${3}"
32
+ local dir="${1}" url="${2}" sha256="${3}" key="${4}"
33
local tar="${dir}.tar.gz"
34
+ local cache="${NETDATA_SOURCE_PATH}/artifacts/cache/${BUILDARCH}/${key}"
35
35
- if [ ! -f "${NETDATA_MAKESELF_PATH}/tmp/${tar}" ]; then
36
- run wget -O "${NETDATA_MAKESELF_PATH}/tmp/${tar}" "${url}"
36
+ if [ -d "${NETDATA_MAKESELF_PATH}/tmp/${dir}" ]; then
37
+ rm -rf "${NETDATA_MAKESELF_PATH}/tmp/${dir}"
38
fi
39
39
- # Check SHA256 of gzip'd tar file (apparently alpine's sha256sum requires
40
- # two empty spaces between the checksum and the file's path)
41
- set +e
42
- echo "${sha256} ${NETDATA_MAKESELF_PATH}/tmp/${tar}" | sha256sum -c -s
43
- local rc=$?
44
- if [ ${rc} -ne 0 ]; then
45
- echo >&2 "SHA256 verification of tar file ${tar} failed (rc=${rc})"
46
- echo >&2 "expected: ${sha256}, got $(sha256sum "${NETDATA_MAKESELF_PATH}/tmp/${tar}")"
47
- exit 1
48
- fi
49
- set -e
40
+ if [ -d "${cache}/${dir}" ]; then
41
+ echo "Found cached copy of build directory for ${key}, using it."
42
+ cp -a "${cache}/${dir}" "${NETDATA_MAKESELF_PATH}/tmp/"
43
+ CACHE_HIT=1
44
+ else
45
+ echo "No cached copy of build directory for ${key} found, fetching sources instead."
46
+
47
+ if [ ! -f "${NETDATA_MAKESELF_PATH}/tmp/${tar}" ]; then
48
+ run wget -O "${NETDATA_MAKESELF_PATH}/tmp/${tar}" "${url}"
49
+ fi
50
+
51
+ # Check SHA256 of gzip'd tar file (apparently alpine's sha256sum requires
52
+ # two empty spaces between the checksum and the file's path)
53
+ set +e
54
+ echo "${sha256} ${NETDATA_MAKESELF_PATH}/tmp/${tar}" | sha256sum -c -s
55
+ local rc=$?
56
+ if [ ${rc} -ne 0 ]; then
57
+ echo >&2 "SHA256 verification of tar file ${tar} failed (rc=${rc})"
58
+ echo >&2 "expected: ${sha256}, got $(sha256sum "${NETDATA_MAKESELF_PATH}/tmp/${tar}")"
59
+ exit 1
60
+ fi
61
+ set -e
62
51
- if [ ! -d "${NETDATA_MAKESELF_PATH}/tmp/${dir}" ]; then
63
cd "${NETDATA_MAKESELF_PATH}/tmp"
64
run tar -zxpf "${tar}"
65
cd -
66
+
67
+ CACHE_HIT=0
68
fi
69
70
run cd "${NETDATA_MAKESELF_PATH}/tmp/${dir}"
71
}
72
73
+store_cache() {
74
+ key="${1}"
75
+ src="${2}"
76
+
77
+ cache="${NETDATA_SOURCE_PATH}/artifacts/cache/${BUILDARCH}/${key}"
78
+
79
+ if [ "${CACHE_HIT:-0}" -eq 0 ]; then
80
+ if [ -d "${cache}" ]; then
81
+ rm -rf "${cache}"
82
+ fi
83
+
84
+ mkdir -p "${cache}"
85
+
86
+ cp -a "${src}" "${cache}"
87
+ fi
88
+}
89
+
90
# -----------------------------------------------------------------------------
91
92
# load the functions of the netdata-installer.sh
packaging/makeself/jobs/20-openssl.install.sh
+25
-4
@@ -13,15 +13,36 @@ export CFLAGS='-fno-lto -pipe'
13
export LDFLAGS='-static'
14
export PKG_CONFIG="pkg-config --static"
15
16
-# Might be bind-mounted
17
-if [ ! -d "${NETDATA_MAKESELF_PATH}/tmp/openssl" ]; then
16
+if [ -d "${NETDATA_MAKESELF_PATH}/tmp/openssl" ]; then
17
+ rm -rf "${NETDATA_MAKESELF_PATH}/tmp/openssl"
18
+fi
19
+
20
+if [ -d "${NETDATA_MAKESELF_PATH}/tmp/openssl" ]; then
21
+ rm -rf "${NETDATA_MAKESELF_PATH}/tmp/openssl"
22
+fi
23
+
24
+cache="${NETDATA_SOURCE_PATH}/artifacts/cache/${BUILDARCH}/openssl"
25
+
26
+if [ -d "${cache}" ]; then
27
+ echo "Found cached copy of build directory for openssl, using it."
28
+ cp -a "${cache}/openssl" "${NETDATA_MAKESELF_PATH}/tmp/"
29
+ CACHE_HIT=1
30
+else
31
+ echo "No cached copy of build directory for openssl found, fetching sources instead."
32
run git clone --branch "${version}" --single-branch --depth 1 git://git.openssl.org/openssl.git "${NETDATA_MAKESELF_PATH}/tmp/openssl"
33
+ CACHE_HIT=0
34
fi
35
+
36
cd "${NETDATA_MAKESELF_PATH}/tmp/openssl" || exit 1
37
22
-run ./config -static no-tests --prefix=/openssl-static --openssldir=/opt/netdata/etc/ssl
23
-run make -j "$(nproc)"
38
+if [ "${CACHE_HIT:-0}" -eq 0 ]; then
39
+ run ./config -static no-tests --prefix=/openssl-static --openssldir=/opt/netdata/etc/ssl
40
+ run make -j "$(nproc)"
41
+fi
42
+
43
run make -j "$(nproc)" install_sw
44
45
+store_cache openssl "${NETDATA_MAKESELF_PATH}/tmp/openssl"
46
+
47
# shellcheck disable=SC2015
48
[ "${GITHUB_ACTIONS}" = "true" ] && echo "::endgroup::" || true
packaging/makeself/jobs/50-bash-5.1.16.install.sh
+27
-21
@@ -4,37 +4,43 @@
4
# shellcheck source=packaging/makeself/functions.sh
5
. "$(dirname "${0}")/../functions.sh" "${@}" || exit 1
6
7
+version="5.1.16"
8
+
9
# shellcheck disable=SC2015
10
[ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::building bash" || true
11
10
-fetch "bash-5.1.16" "http://ftp.gnu.org/gnu/bash/bash-5.1.16.tar.gz" \
11
- 5bac17218d3911834520dad13cd1f85ab944e1c09ae1aba55906be1f8192f558
12
+fetch "bash-${version}" "http://ftp.gnu.org/gnu/bash/bash-${version}.tar.gz" \
13
+ 5bac17218d3911834520dad13cd1f85ab944e1c09ae1aba55906be1f8192f558 bash
14
15
export CFLAGS="-pipe"
16
export PKG_CONFIG_PATH="/openssl-static/lib/pkgconfig"
17
16
-run ./configure \
17
- --prefix="${NETDATA_INSTALL_PATH}" \
18
- --without-bash-malloc \
19
- --enable-static-link \
20
- --enable-net-redirections \
21
- --enable-array-variables \
22
- --disable-progcomp \
23
- --disable-profiling \
24
- --disable-nls \
25
- --disable-dependency-tracking
26
-
27
-run make clean
28
-run make -j "$(nproc)"
29
-
30
-cat > examples/loadables/Makefile << EOF
31
-all:
32
-clean:
33
-install:
34
-EOF
18
+if [ "${CACHE_HIT:-0}" -eq 0 ]; then
19
+ run ./configure \
20
+ --prefix="${NETDATA_INSTALL_PATH}" \
21
+ --without-bash-malloc \
22
+ --enable-static-link \
23
+ --enable-net-redirections \
24
+ --enable-array-variables \
25
+ --disable-progcomp \
26
+ --disable-profiling \
27
+ --disable-nls \
28
+ --disable-dependency-tracking
29
+
30
+ run make clean
31
+ run make -j "$(nproc)"
32
+
33
+ cat > examples/loadables/Makefile <<-EOF
34
+ all:
35
+ clean:
36
+ install:
37
+ EOF
38
+fi
39
40
run make install
41
42
+store_cache bash "${NETDATA_MAKESELF_PATH}/tmp/bash-${version}"
43
+
44
if [ "${NETDATA_BUILD_WITH_DEBUG}" -eq 0 ]; then
45
run strip "${NETDATA_INSTALL_PATH}"/bin/bash
46
fi
packaging/makeself/jobs/50-curl-7.82.0.install.sh
+39
-32
@@ -4,49 +4,56 @@
4
# shellcheck source=packaging/makeself/functions.sh
5
. "$(dirname "${0}")/../functions.sh" "${@}" || exit 1
6
7
+version="7.82.0"
8
+
9
# shellcheck disable=SC2015
10
[ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::Building cURL" || true
11
10
-fetch "curl-7.82.0" "https://curl.haxx.se/download/curl-7.82.0.tar.gz" \
11
- 910cc5fe279dc36e2cca534172c94364cf3fcf7d6494ba56e6c61a390881ddce
12
+fetch "curl-${version}" "https://curl.haxx.se/download/curl-${version}.tar.gz" \
13
+ 910cc5fe279dc36e2cca534172c94364cf3fcf7d6494ba56e6c61a390881ddce curl
14
15
export CFLAGS="-I/openssl-static/include -pipe"
16
export LDFLAGS="-static -L/openssl-static/lib"
17
export PKG_CONFIG="pkg-config --static"
18
export PKG_CONFIG_PATH="/openssl-static/lib/pkgconfig"
19
18
-run autoreconf -fi
19
-
20
-run ./configure \
21
- --prefix="${NETDATA_INSTALL_PATH}" \
22
- --enable-optimize \
23
- --disable-shared \
24
- --enable-static \
25
- --enable-http \
26
- --disable-ldap \
27
- --disable-ldaps \
28
- --enable-proxy \
29
- --disable-dict \
30
- --disable-telnet \
31
- --disable-tftp \
32
- --disable-pop3 \
33
- --disable-imap \
34
- --disable-smb \
35
- --disable-smtp \
36
- --disable-gopher \
37
- --enable-ipv6 \
38
- --enable-cookies \
39
- --with-ca-fallback \
40
- --with-openssl \
41
- --disable-dependency-tracking
42
-
43
-# Curl autoconf does not honour the curl_LDFLAGS environment variable
44
-run sed -i -e "s/LDFLAGS =/LDFLAGS = -all-static/" src/Makefile
45
-
46
-run make clean
47
-run make -j "$(nproc)"
20
+if [ "${CACHE_HIT:-0}" -eq 0 ]; then
21
+ run autoreconf -fi
22
+
23
+ run ./configure \
24
+ --prefix="${NETDATA_INSTALL_PATH}" \
25
+ --enable-optimize \
26
+ --disable-shared \
27
+ --enable-static \
28
+ --enable-http \
29
+ --disable-ldap \
30
+ --disable-ldaps \
31
+ --enable-proxy \
32
+ --disable-dict \
33
+ --disable-telnet \
34
+ --disable-tftp \
35
+ --disable-pop3 \
36
+ --disable-imap \
37
+ --disable-smb \
38
+ --disable-smtp \
39
+ --disable-gopher \
40
+ --enable-ipv6 \
41
+ --enable-cookies \
42
+ --with-ca-fallback \
43
+ --with-openssl \
44
+ --disable-dependency-tracking
45
+
46
+ # Curl autoconf does not honour the curl_LDFLAGS environment variable
47
+ run sed -i -e "s/LDFLAGS =/LDFLAGS = -all-static/" src/Makefile
48
+
49
+ run make clean
50
+ run make -j "$(nproc)"
51
+fi
52
+
53
run make install
54
55
+store_cache curl "${NETDATA_MAKESELF_PATH}/tmp/curl-${version}"
56
+
57
if [ "${NETDATA_BUILD_WITH_DEBUG}" -eq 0 ]; then
58
run strip "${NETDATA_INSTALL_PATH}"/bin/curl
59
fi
packaging/makeself/jobs/50-fping-5.1.install.sh
+22
-15
@@ -4,32 +4,39 @@
4
# shellcheck source=packaging/makeself/functions.sh
5
. "$(dirname "${0}")/../functions.sh" "${@}" || exit 1
6
7
+version="5.1"
8
+
9
# shellcheck disable=SC2015
10
[ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::Building fping" || true
11
10
-fetch "fping-5.1" "https://fping.org/dist/fping-5.1.tar.gz" \
11
- 1ee5268c063d76646af2b4426052e7d81a42b657e6a77d8e7d3d2e60fd7409fe
12
+fetch "fping-${version}" "https://fping.org/dist/fping-${version}.tar.gz" \
13
+ 1ee5268c063d76646af2b4426052e7d81a42b657e6a77d8e7d3d2e60fd7409fe fping
14
15
export CFLAGS="-static -I/openssl-static/include -pipe"
16
export LDFLAGS="-static -L/openssl-static/lib"
17
export PKG_CONFIG_PATH="/openssl-static/lib/pkgconfig"
18
17
-run ./configure \
18
- --prefix="${NETDATA_INSTALL_PATH}" \
19
- --enable-ipv4 \
20
- --enable-ipv6 \
21
- --disable-dependency-tracking
22
-
23
-cat > doc/Makefile << EOF
24
-all:
25
-clean:
26
-install:
27
-EOF
19
+if [ "${CACHE_HIT:-0}" -eq 0 ]; then
20
+ run ./configure \
21
+ --prefix="${NETDATA_INSTALL_PATH}" \
22
+ --enable-ipv4 \
23
+ --enable-ipv6 \
24
+ --disable-dependency-tracking
25
+
26
+ cat > doc/Makefile <<-EOF
27
+ all:
28
+ clean:
29
+ install:
30
+ EOF
31
+
32
+ run make clean
33
+ run make -j "$(nproc)"
34
+fi
35
29
-run make clean
30
-run make -j "$(nproc)"
36
run make install
37
38
+store_cache fping "${NETDATA_MAKESELF_PATH}/tmp/fping-${version}"
39
+
40
if [ "${NETDATA_BUILD_WITH_DEBUG}" -eq 0 ]; then
41
run strip "${NETDATA_INSTALL_PATH}"/bin/fping
42
fi
packaging/makeself/jobs/50-ioping-1.2.install.sh
+11
-4
@@ -4,19 +4,26 @@
4
# shellcheck source=packaging/makeself/functions.sh
5
. "$(dirname "${0}")/../functions.sh" "${@}" || exit 1
6
7
+version='1.2'
8
+
9
# shellcheck disable=SC2015
10
[ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::Building ioping" || true
11
10
-fetch "ioping-1.2" "https://github.com/koct9i/ioping/archive/v1.2.tar.gz" \
11
- d3e4497c653a1e96df67c72ce2b70da18e9f5e3b93179a5bb57a6e30ceacfa75
12
+fetch "ioping-${version}" "https://github.com/koct9i/ioping/archive/v${version}.tar.gz" \
13
+ d3e4497c653a1e96df67c72ce2b70da18e9f5e3b93179a5bb57a6e30ceacfa75 ioping
14
15
export CFLAGS="-static -pipe"
16
15
-run make clean
16
-run make -j "$(nproc)"
17
+if [ "${CACHE_HIT:-0}" -eq 0 ]; then
18
+ run make clean
19
+ run make -j "$(nproc)"
20
+fi
21
+
22
run mkdir -p "${NETDATA_INSTALL_PATH}"/usr/libexec/netdata/plugins.d/
23
run install -o root -g root -m 4750 ioping "${NETDATA_INSTALL_PATH}"/usr/libexec/netdata/plugins.d/
24
25
+store_cache ioping "${NETDATA_MAKESELF_PATH}/tmp/ioping-${version}"
26
+
27
if [ "${NETDATA_BUILD_WITH_DEBUG}" -eq 0 ]; then
28
run strip "${NETDATA_INSTALL_PATH}"/usr/libexec/netdata/plugins.d/ioping
29
fi
packaging/makeself/uname2platform.sh
new
+18
@@ -0,0 +1,18 @@
1
+#!/bin/sh
2
+#
3
+# SPDX-License-Identifier: GPL-3.0-or-later
4
+
5
+set -e
6
+
7
+BUILDARCH="${1}"
8
+
9
+case "${BUILDARCH}" in
10
+ x86_64) echo "linux/amd64" ;;
11
+ armv7l) echo "linux/arm/v7" ;;
12
+ aarch64) echo "linux/arm64/v8" ;;
13
+ ppc64le) echo "linux/ppc64le" ;;
14
+ *)
15
+ echo "Unknown target architecture '${BUILDARCH}'." >&2
16
+ exit 1
17
+ ;;
18
+esac