netdata/ci: second batch of fixes for coverity scan script and others (#6804)
* netdata/ci: fine grain coverity scan toolkit 1) Deprecate coverity-install.sh 2) add set -e to raise errors more aggressively when something is wrong 3) refactor some variable definitions for temp and install paths, use mktemp and clean up temp dir when all ok * netdata/ci: reduce the scope * netdata/ci: require curl * netdata/ci: go soft on functions loading * netdata/ci: allow override of build version without touching the script * netdata/ci: handle shellcheck errors, some were silenced because we dont handle them * netdata/ci: coverity scan, parse any extra options other than --with-install and pass them to configure 1) add default configure arguments if no extras are given 2) parse all possible arguments individually, handle --with-install and pass the rest to scanit * netdata/ci: focus, argument parsing in one place a bit more consistently * netdata/ci: prepare for using * netdata/ci: coverity - add more flags, adjust deps list 1) Add with math and with zlib flags on the script 2) add xenstat dependencies 3) run in coverity with custom flags, so that we add xenstat (no need to be by default for the users of the script) * netdata/packaging: add xen-utils also * test the flow * netdata/ci: dont forget to go back to the original directory * netdata/ci: please, travis, stop it * netdata/ci: how about this then * netdata/ci: silence that, we cant call it otherwise * netdata/packaging: update documentation around xenstat on centos * netdata/packaging: dont enable xenstat at this stage * netdata/ci: add += instead of variable reusage
Paul Emm. Katsoulakis committed
Sep 12, 2019 at 12:49 UTC
5657086b1cc70cace76467b025033091ac8c3e6b
6 files changed
+83
-50
.travis.yml
+3
-2
@@ -52,7 +52,8 @@ stages:
52
53
# Nightly operations
54
- name: Nightly operations
55
- if: branch = master AND type = cron
55
+ #if: branch = master AND type = cron
56
+ if: branch =fix-coverity-toolkit
57
- name: Nightly release
58
if: branch = master AND type = cron
59
@@ -536,7 +537,7 @@ jobs:
537
- post_message "TRAVIS_MESSAGE" "Starting nightly operations" "${NOTIF_CHANNEL}"
538
- bash <(curl -sS https://raw.githubusercontent.com/netdata/netdata-demo-site/master/install-required-packages.sh) --dont-wait --non-interactive netdata
539
- sudo apt-get install -y libjson-c-dev libipmimonitoring-dev libcups2-dev libsnappy-dev libprotobuf-dev libprotoc-dev libssl-dev protobuf-compiler
539
- script: ./coverity-install.sh && ./coverity-scan.sh
540
+ script: ./coverity-scan.sh --with-install
541
after_failure: post_message "TRAVIS_MESSAGE" "<!here> Coverity nightly run has failed" "${NOTIF_CHANNEL}"
542
env:
543
- ALLOW_SOFT_FAILURE_HERE=true
Makefile.am
-1
@@ -72,7 +72,6 @@ dist_noinst_DATA= \
72
# should be proper init.d/openrc/systemd usable
73
dist_noinst_SCRIPTS= \
74
coverity-scan.sh \
75
- coverity-install.sh \
75
packaging/installer/netdata-updater.sh \
76
packaging/installer/netdata-uninstaller.sh \
77
packaging/installer/kickstart.sh \
collectors/xenstat.plugin/README.md
+1
@@ -5,6 +5,7 @@
5
## Prerequisites
6
7
1. install `xen-dom0-libs-devel` and `yajl-devel` using the package manager of your system.
8
+ Note: On Cent-OS systems you will need `centos-release-xen` repository and the required package for xen is `xen-devel`
9
10
2. re-install Netdata from source. The installer will detect that the required libraries are now available and will also build xenstat.plugin.
11
coverity-install.sh
deleted
-8
@@ -1,8 +0,0 @@
1
-#!/usr/bin/env bash
2
-# Coverity installation script
3
-#
4
-# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
5
-#
6
-# Author: Pavlos Emm. Katsoulakis (paul@netdata.cloud)
7
-
8
-exec ./coverity-scan.sh install "${@}"
coverity-scan.sh
+77
-38
@@ -1,4 +1,5 @@
1
#!/usr/bin/env bash
2
+#
3
# Coverity scan script
4
#
5
# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
@@ -6,6 +7,7 @@
7
# Author : Costa Tsaousis (costa@netdata.cloud)
8
# Author : Pawel Krupa (paulfantom)
9
# Author : Pavlos Emm. Katsoulakis (paul@netdata.cloud)
10
+# shellcheck disable=SC1091,SC2230,SC2086
11
12
# To run manually, save configuration to .coverity-scan.conf like this:
13
#
@@ -25,22 +27,30 @@
27
# this includes the token, so the default is not to print it.
28
# COVERITY_SUBMIT_DEBUG=1
29
#
30
+# Override the standard coverity build version we know is supported
31
+# COVERITY_BUILD_VERSION="cov-analysis-linux64-2019.03"
32
+#
33
# All these variables can also be exported before running this script.
34
#
35
# If the first parameter of this script is "install",
36
# coverity build tools will be downloaded and installed in /opt/coverity
37
38
+set -e
39
+
40
+INSTALL_DIR="/opt"
41
+
42
# the version of coverity to use
34
-COVERITY_BUILD_VERSION="cov-analysis-linux64-2019.03"
43
+COVERITY_BUILD_VERSION="${COVERITY_BUILD_VERSION:-cov-analysis-linux64-2019.03}"
44
36
-source packaging/installer/functions.sh || exit 1
45
+# TODO: For some reasons this does not fully load on Debian 10 (Haven't checked if it happens on other distros yet), it breaks
46
+source packaging/installer/functions.sh || echo "Failed to fully load the functions library"
47
48
cpus=$(find_processors)
49
[ -z "${cpus}" ] && cpus=1
50
51
if [ -f ".coverity-scan.conf" ]
52
then
43
- source ".coverity-scan.conf" || exit 1
53
+ source ".coverity-scan.conf"
54
fi
55
56
repo="${REPOSITORY}"
@@ -59,6 +69,10 @@ if [ -z "${token}" ]; then
69
fatal "export variable COVERITY_SCAN_TOKEN or set it in .coverity-scan.conf"
70
fi
71
72
+if ! command -v curl >/dev/null 2>&1; then
73
+ fatal "CURL is required for coverity scan to work"
74
+fi
75
+
76
# only print the output of a command
77
# when debugging is enabled
78
# used to hide the token when debugging is not enabled
@@ -74,9 +88,11 @@ debugrun() {
88
}
89
90
scanit() {
77
- export PATH="${PATH}:/opt/${COVERITY_BUILD_VERSION}/bin/"
91
+ progress "Scanning using coverity"
92
+ export PATH="${PATH}:${INSTALL_DIR}/${COVERITY_BUILD_VERSION}/bin/"
93
covbuild="${COVERITY_BUILD_PATH}"
94
[ -z "${covbuild}" ] && covbuild="$(which cov-build 2>/dev/null || command -v cov-build 2>/dev/null)"
95
+
96
if [ -z "${covbuild}" ]; then
97
fatal "Cannot find 'cov-build' binary in \$PATH. Export variable COVERITY_BUILD_PATH or set it in .coverity-scan.conf"
98
elif [ ! -x "${covbuild}" ]; then
@@ -94,54 +110,45 @@ scanit() {
110
[ -f netdata-coverity-analysis.tgz ] && run rm netdata-coverity-analysis.tgz
111
112
progress "Configuring netdata source..."
113
+
114
run autoreconf -ivf
98
- run ./configure --disable-lto \
99
- --enable-https \
100
- --enable-jsonc \
101
- --enable-plugin-nfacct \
102
- --enable-plugin-freeipmi \
103
- --enable-plugin-cups \
104
- --enable-backend-prometheus-remote-write \
105
- ${NULL}
106
-
107
- # TODO: enable these plugins too
108
- # --enable-plugin-xenstat \
109
- # --enable-backend-kinesis \
110
- # --enable-backend-mongodb \
115
+ run ./configure ${OTHER_OPTIONS}
116
117
progress "Analyzing netdata..."
113
- run "${covbuild}" --dir cov-int make -j${cpus} || exit 1
118
+ run "${covbuild}" --dir cov-int make -j${cpus}
119
120
echo >&2 "Compressing analysis..."
116
- run tar czvf netdata-coverity-analysis.tgz cov-int || exit 1
121
+ run tar czvf netdata-coverity-analysis.tgz cov-int
122
123
echo >&2 "Sending analysis to coverity for netdata version ${version} ..."
124
COVERITY_SUBMIT_RESULT=$(debugrun curl --progress-bar \
125
--form token="${token}" \
121
- --form email=${email} \
126
+ --form email="${email}" \
127
--form file=@netdata-coverity-analysis.tgz \
128
--form version="${version}" \
129
--form description="netdata, monitor everything, in real-time." \
125
- https://scan.coverity.com/builds?project=${repo})
130
+ https://scan.coverity.com/builds?project="${repo}")
131
127
- echo ${COVERITY_SUBMIT_RESULT} | grep -q -e 'Build successfully submitted' || echo >&2 "scan results were not pushed to coverity. Message was: ${COVERITY_SUBMIT_RESULT}"
132
+ echo "${COVERITY_SUBMIT_RESULT}" | grep -q -e 'Build successfully submitted' || echo >&2 "scan results were not pushed to coverity. Message was: ${COVERITY_SUBMIT_RESULT}"
133
134
progress "Coverity scan completed"
135
}
136
137
installit() {
133
- progress "Downloading coverity..."
134
- cd /tmp || exit 1
138
+ ORIGINAL_DIR="${PWD}"
139
+ TMP_DIR="$(mktemp -d /tmp/netdata-coverity-scan-XXXXX)"
140
+ progress "Downloading coverity in ${TMP_DIR}..."
141
+ cd "${TMP_DIR}"
142
136
- [ -f "${COVERITY_BUILD_VERSION}.tar.gz" ] && run rm -f "${COVERITY_BUILD_VERSION}.tar.gz"
143
debugrun curl --remote-name --remote-header-name --show-error --location --data "token=${token}&project=${repo}" https://scan.coverity.com/download/linux64
144
145
if [ -f "${COVERITY_BUILD_VERSION}.tar.gz" ]; then
146
progress "Installing coverity..."
141
- cd /opt || exit 1
142
- run sudo tar -z -x -f "/tmp/${COVERITY_BUILD_VERSION}.tar.gz" || exit 1
143
- rm "/tmp/${COVERITY_BUILD_VERSION}.tar.gz"
144
- export PATH=${PATH}:/opt/${COVERITY_BUILD_VERSION}/bin/
147
+ cd "${INSTALL_DIR}"
148
+
149
+ run sudo tar -z -x -f "${TMP_DIR}/${COVERITY_BUILD_VERSION}.tar.gz" || exit 1
150
+ rm "${TMP_DIR}/${COVERITY_BUILD_VERSION}.tar.gz"
151
+ export PATH=${PATH}:${INSTALL_DIR}/${COVERITY_BUILD_VERSION}/bin/
152
else
153
fatal "Failed to download coverity tool tarball!"
154
fi
@@ -152,16 +159,48 @@ installit() {
159
fatal "Failed to install coverity."
160
fi
161
162
+ # Clean temp directory
163
+ [ -n "${TMP_DIR}" ] && rm -rf "${TMP_DIR}"
164
+
165
progress "Coverity scan tools are installed."
166
+ cd "$ORIGINAL_DIR"
167
return 0
168
}
169
159
-if [ "${1}" = "install" ]
160
-then
161
- shift 1
162
- installit "${@}"
163
- exit $?
164
-else
165
- scanit "${@}"
166
- exit $?
167
-fi
170
+OTHER_OPTIONS="--disable-lto"
171
+OTHER_OPTIONS+=" --with-zlib"
172
+OTHER_OPTIONS+=" --with-math"
173
+OTHER_OPTIONS+=" --enable-https"
174
+OTHER_OPTIONS+=" --enable-jsonc"
175
+OTHER_OPTIONS+=" --enable-plugin-nfacct"
176
+OTHER_OPTIONS+=" --enable-plugin-freeipmi"
177
+OTHER_OPTIONS+=" --enable-plugin-cups"
178
+OTHER_OPTIONS+=" --enable-backend-prometheus-remote-write"
179
+# TODO: enable these plugins too
180
+#OTHER_OPTIONS+=" --enable-plugin-xenstat"
181
+#OTHER_OPTIONS+=" --enable-backend-kinesis"
182
+#OTHER_OPTIONS+=" --enable-backend-mongodb"
183
+
184
+FOUND_OPTS="NO"
185
+while [ -n "${1}" ]; do
186
+ if [ "${1}" = "--with-install" ]; then
187
+ progress "Running coverity install"
188
+ installit
189
+ shift 1
190
+ elif [ -n "${1}" ]; then
191
+ # Clear the default arguments, once you bump into the first argument
192
+ if [ "${FOUND_OPTS}" = "NO" ]; then
193
+ OTHER_OPTIONS="${1}"
194
+ FOUND_OPTS="YES"
195
+ else
196
+ OTHER_OPTIONS+=" ${1}"
197
+ fi
198
+
199
+ shift 1
200
+ else
201
+ break
202
+ fi
203
+done
204
+
205
+echo "Running coverity scan with extra options ${OTHER_OPTIONS}"
206
+scanit "${OTHER_OPTIONS}"
packaging/DISTRIBUTIONS.md
+2
-1
@@ -188,7 +188,8 @@ This is Netdata's TLS capability that incorporates encryption on the web server
188
189
- **Flags/instructions to enable**: None
190
- **Flags to disable from source**: --disable-plugin-xenstat
191
-- **What packages required for auto-detect?**: `xen-dom0-libs-devel`, `yajl-dev`
191
+- **What packages required for auto-detect?**: `xen-dom0-libs-devel or xen-devel`, `yajl-dev or yajl-devel`
192
+ Note: for cent-OS based systems you will need `centos-release-xen` repository to get xen-devel
193
194
#### CUPS
195