| 1 | #!/bin/sh -e |
| 2 | |
| 3 | # Upload a created tarball to Coverity Scan, as per |
| 4 | # https://scan.coverity.com/projects/qemu/builds/new |
| 5 | |
| 6 | # This work is licensed under the terms of the GNU GPL version 2, |
| 7 | # or (at your option) any later version. |
| 8 | # See the COPYING file in the top-level directory. |
| 9 | # |
| 10 | # Copyright (c) 2017-2020 Linaro Limited |
| 11 | # Written by Peter Maydell |
| 12 | |
| 13 | # Note that this script will automatically download and |
| 14 | # run the (closed-source) coverity build tools, so don't |
| 15 | # use it if you don't trust them! |
| 16 | |
| 17 | # This script assumes that you're running it from a QEMU source |
| 18 | # tree, and that tree is a fresh clean one, because we do an in-tree |
| 19 | # build. (This is necessary so that the filenames that the Coverity |
| 20 | # Scan server sees are relative paths that match up with the component |
| 21 | # regular expressions it uses; an out-of-tree build won't work for this.) |
| 22 | # The host machine should have as many of QEMU's dependencies |
| 23 | # installed as possible, for maximum coverity coverage. |
| 24 | |
| 25 | # To do an upload you need to be a maintainer in the Coverity online |
| 26 | # service, and you will need to know the "Coverity token", which is a |
| 27 | # secret 8 digit hex string. You can find that from the web UI in the |
| 28 | # project settings, if you have maintainer access there. |
| 29 | |
| 30 | # Command line options: |
| 31 | # --check-upload-only : return success if upload is possible |
| 32 | # --dry-run : run the tools, but don't actually do the upload |
| 33 | # --docker : create and work inside a container |
| 34 | # --docker-engine : specify the container engine to use (docker/podman/auto); |
| 35 | # implies --docker |
| 36 | # --update-tools-only : update the cached copy of the tools, but don't run them |
| 37 | # --no-update-tools : do not update the cached copy of the tools |
| 38 | # --tokenfile : file to read Coverity token from |
| 39 | # --version ver : specify version being analyzed (default: ask git) |
| 40 | # --description desc : specify description of this version (default: ask git) |
| 41 | # --srcdir : QEMU source tree to analyze (default: current working dir) |
| 42 | # --results-tarball : path to copy the results tarball to (default: don't |
| 43 | # copy it anywhere, just upload it) |
| 44 | # --src-tarball : tarball to untar into src dir (default: none); this |
| 45 | # is intended mainly for internal use by the Docker support |
| 46 | # |
| 47 | # User-specifiable environment variables: |
| 48 | # COVERITY_TOKEN -- Coverity token (default: looks at your |
| 49 | # coverity.token config) |
| 50 | # COVERITY_EMAIL -- the email address to use for uploads (default: |
| 51 | # looks at your git coverity.email or user.email config) |
| 52 | # COVERITY_BUILD_CMD -- make command (default: 'make -jN' where N is |
| 53 | # number of CPUs as determined by 'nproc') |
| 54 | # COVERITY_TOOL_BASE -- set to directory to put coverity tools |
| 55 | # (default: /tmp/coverity-tools) |
| 56 | # |
| 57 | # You must specify the token, either by environment variable or by |
| 58 | # putting it in a file and using --tokenfile. Everything else has |
| 59 | # a reasonable default if this is run from a git tree. |
| 60 | |
| 61 | upload_permitted() { |
| 62 | # Check whether we can do an upload to the server; will exit *the script* |
| 63 | # with status 99 if the check failed (usually a bad token); |
| 64 | # will return from the function with status 1 if the check indicated |
| 65 | # that we can't upload yet (ie we are at quota) |
| 66 | # Assumes that COVERITY_TOKEN and PROJNAME have been initialized. |
| 67 | |
| 68 | echo "Checking upload permissions..." |
| 69 | |
| 70 | if ! up_perm="$(wget https://scan.coverity.com/api/upload_permitted --post-data "token=$COVERITY_TOKEN&project=$PROJNAME" -q -O -)"; then |
| 71 | echo "Coverity Scan API access denied: bad token?" |
| 72 | exit 99 |
| 73 | fi |
| 74 | |
| 75 | # Really up_perm is a JSON response with either |
| 76 | # {upload_permitted:true} or {next_upload_permitted_at:<date>} |
| 77 | # We do some hacky string parsing instead of properly parsing it. |
| 78 | case "$up_perm" in |
| 79 | *upload_permitted*true*) |
| 80 | return 0 |
| 81 | ;; |
| 82 | *next_upload_permitted_at*) |
| 83 | return 1 |
| 84 | ;; |
| 85 | *) |
| 86 | echo "Coverity Scan upload check: unexpected result $up_perm" |
| 87 | exit 99 |
| 88 | ;; |
| 89 | esac |
| 90 | } |
| 91 | |
| 92 | |
| 93 | check_upload_permissions() { |
| 94 | # Check whether we can do an upload to the server; will exit the script |
| 95 | # with status 99 if the check failed (usually a bad token); |
| 96 | # will exit the script with status 0 if the check indicated that we |
| 97 | # can't upload yet (ie we are at quota) |
| 98 | # Assumes that COVERITY_TOKEN, PROJNAME and DRYRUN have been initialized. |
| 99 | |
| 100 | if upload_permitted; then |
| 101 | echo "Coverity Scan: upload permitted" |
| 102 | else |
| 103 | if [ "$DRYRUN" = yes ]; then |
| 104 | echo "Coverity Scan: upload quota reached, continuing dry run" |
| 105 | else |
| 106 | echo "Coverity Scan: upload quota reached; stopping here" |
| 107 | # Exit success as this isn't a build error. |
| 108 | exit 0 |
| 109 | fi |
| 110 | fi |
| 111 | } |
| 112 | |
| 113 | |
| 114 | build_docker_image() { |
| 115 | # build docker container including the coverity-scan tools |
| 116 | echo "Building docker container..." |
| 117 | # TODO: This re-unpacks the tools every time, rather than caching |
| 118 | # and reusing the image produced by the COPY of the .tgz file. |
| 119 | # Not sure why. |
| 120 | tests/docker/docker.py --engine ${DOCKER_ENGINE} build \ |
| 121 | -t coverity-scanner -f scripts/coverity-scan/coverity-scan.docker \ |
| 122 | --extra-files scripts/coverity-scan/run-coverity-scan \ |
| 123 | "$COVERITY_TOOL_BASE"/coverity_tool.tgz |
| 124 | } |
| 125 | |
| 126 | update_coverity_tools () { |
| 127 | # Check for whether we need to download the Coverity tools |
| 128 | # (either because we don't have a copy, or because it's out of date) |
| 129 | # Assumes that COVERITY_TOOL_BASE, COVERITY_TOKEN and PROJNAME are set. |
| 130 | |
| 131 | mkdir -p "$COVERITY_TOOL_BASE" |
| 132 | cd "$COVERITY_TOOL_BASE" |
| 133 | |
| 134 | echo "Checking for new version of coverity build tools..." |
| 135 | wget https://scan.coverity.com/download/cxx/linux64 --post-data "token=$COVERITY_TOKEN&project=$PROJNAME&md5=1" -O coverity_tool.md5.new |
| 136 | |
| 137 | if ! cmp -s coverity_tool.md5 coverity_tool.md5.new; then |
| 138 | # out of date md5 or no md5: download new build tool |
| 139 | # blow away the old build tool |
| 140 | echo "Downloading coverity build tools..." |
| 141 | rm -rf coverity_tool coverity_tool.tgz |
| 142 | wget https://scan.coverity.com/download/cxx/linux64 --post-data "token=$COVERITY_TOKEN&project=$PROJNAME" -O coverity_tool.tgz |
| 143 | if ! (cat coverity_tool.md5.new; echo " coverity_tool.tgz") | md5sum -c --status; then |
| 144 | echo "Downloaded tarball didn't match md5sum!" |
| 145 | exit 1 |
| 146 | fi |
| 147 | |
| 148 | if [ "$DOCKER" != yes ]; then |
| 149 | # extract the new one, keeping it corralled in a 'coverity_tool' directory |
| 150 | echo "Unpacking coverity build tools..." |
| 151 | mkdir -p coverity_tool |
| 152 | cd coverity_tool |
| 153 | tar xf ../coverity_tool.tgz |
| 154 | cd .. |
| 155 | mv coverity_tool.md5.new coverity_tool.md5 |
| 156 | fi |
| 157 | fi |
| 158 | rm -f coverity_tool.md5.new |
| 159 | cd "$SRCDIR" |
| 160 | |
| 161 | if [ "$DOCKER" = yes ]; then |
| 162 | build_docker_image |
| 163 | fi |
| 164 | } |
| 165 | |
| 166 | |
| 167 | # Check user-provided environment variables and arguments |
| 168 | DRYRUN=no |
| 169 | UPDATE=yes |
| 170 | DOCKER=no |
| 171 | PROJNAME=QEMU |
| 172 | |
| 173 | while [ "$#" -ge 1 ]; do |
| 174 | case "$1" in |
| 175 | --check-upload-only) |
| 176 | shift |
| 177 | DRYRUN=check |
| 178 | ;; |
| 179 | --dry-run) |
| 180 | shift |
| 181 | DRYRUN=yes |
| 182 | ;; |
| 183 | --no-update-tools) |
| 184 | shift |
| 185 | UPDATE=no |
| 186 | ;; |
| 187 | --update-tools-only) |
| 188 | shift |
| 189 | UPDATE=only |
| 190 | ;; |
| 191 | --version) |
| 192 | shift |
| 193 | if [ $# -eq 0 ]; then |
| 194 | echo "--version needs an argument" |
| 195 | exit 1 |
| 196 | fi |
| 197 | VERSION="$1" |
| 198 | shift |
| 199 | ;; |
| 200 | --description) |
| 201 | shift |
| 202 | if [ $# -eq 0 ]; then |
| 203 | echo "--description needs an argument" |
| 204 | exit 1 |
| 205 | fi |
| 206 | DESCRIPTION="$1" |
| 207 | shift |
| 208 | ;; |
| 209 | --tokenfile) |
| 210 | shift |
| 211 | if [ $# -eq 0 ]; then |
| 212 | echo "--tokenfile needs an argument" |
| 213 | exit 1 |
| 214 | fi |
| 215 | COVERITY_TOKEN="$(cat "$1")" |
| 216 | shift |
| 217 | ;; |
| 218 | --srcdir) |
| 219 | shift |
| 220 | if [ $# -eq 0 ]; then |
| 221 | echo "--srcdir needs an argument" |
| 222 | exit 1 |
| 223 | fi |
| 224 | SRCDIR="$1" |
| 225 | shift |
| 226 | ;; |
| 227 | --results-tarball) |
| 228 | shift |
| 229 | if [ $# -eq 0 ]; then |
| 230 | echo "--results-tarball needs an argument" |
| 231 | exit 1 |
| 232 | fi |
| 233 | RESULTSTARBALL="$1" |
| 234 | shift |
| 235 | ;; |
| 236 | --src-tarball) |
| 237 | shift |
| 238 | if [ $# -eq 0 ]; then |
| 239 | echo "--src-tarball needs an argument" |
| 240 | exit 1 |
| 241 | fi |
| 242 | SRCTARBALL="$1" |
| 243 | shift |
| 244 | ;; |
| 245 | --docker) |
| 246 | DOCKER=yes |
| 247 | DOCKER_ENGINE=auto |
| 248 | shift |
| 249 | ;; |
| 250 | --docker-engine) |
| 251 | shift |
| 252 | if [ $# -eq 0 ]; then |
| 253 | echo "--docker-engine needs an argument" |
| 254 | exit 1 |
| 255 | fi |
| 256 | DOCKER=yes |
| 257 | DOCKER_ENGINE="$1" |
| 258 | shift |
| 259 | ;; |
| 260 | *) |
| 261 | echo "Unexpected argument '$1'" |
| 262 | exit 1 |
| 263 | ;; |
| 264 | esac |
| 265 | done |
| 266 | |
| 267 | if [ -z "$COVERITY_TOKEN" ]; then |
| 268 | COVERITY_TOKEN="$(git config coverity.token)" |
| 269 | fi |
| 270 | if [ -z "$COVERITY_TOKEN" ]; then |
| 271 | echo "COVERITY_TOKEN environment variable not set" |
| 272 | exit 1 |
| 273 | fi |
| 274 | |
| 275 | if [ "$DRYRUN" = check ]; then |
| 276 | upload_permitted |
| 277 | exit $? |
| 278 | fi |
| 279 | |
| 280 | if [ -z "$COVERITY_BUILD_CMD" ]; then |
| 281 | NPROC=$(nproc) |
| 282 | COVERITY_BUILD_CMD="make -j$NPROC" |
| 283 | echo "COVERITY_BUILD_CMD: using default '$COVERITY_BUILD_CMD'" |
| 284 | fi |
| 285 | |
| 286 | if [ -z "$COVERITY_TOOL_BASE" ]; then |
| 287 | echo "COVERITY_TOOL_BASE: using default /tmp/coverity-tools" |
| 288 | COVERITY_TOOL_BASE=/tmp/coverity-tools |
| 289 | fi |
| 290 | |
| 291 | if [ -z "$SRCDIR" ]; then |
| 292 | SRCDIR="$PWD" |
| 293 | fi |
| 294 | |
| 295 | TARBALL=cov-int.tar.xz |
| 296 | |
| 297 | if [ "$UPDATE" = only ]; then |
| 298 | # Just do the tools update; we don't need to check whether |
| 299 | # we are in a source tree or have upload rights for this, |
| 300 | # so do it before some of the command line and source tree checks. |
| 301 | |
| 302 | if [ "$DOCKER" = yes ] && [ ! -z "$SRCTARBALL" ]; then |
| 303 | echo --update-tools-only --docker is incompatible with --src-tarball. |
| 304 | exit 1 |
| 305 | fi |
| 306 | |
| 307 | update_coverity_tools |
| 308 | exit 0 |
| 309 | fi |
| 310 | |
| 311 | if [ ! -e "$SRCDIR" ]; then |
| 312 | mkdir "$SRCDIR" |
| 313 | fi |
| 314 | |
| 315 | cd "$SRCDIR" |
| 316 | |
| 317 | if [ ! -z "$SRCTARBALL" ]; then |
| 318 | echo "Untarring source tarball into $SRCDIR..." |
| 319 | tar xvf "$SRCTARBALL" |
| 320 | fi |
| 321 | |
| 322 | echo "Checking this is a QEMU source tree..." |
| 323 | if ! [ -e "$SRCDIR/VERSION" ]; then |
| 324 | echo "Not in a QEMU source tree?" |
| 325 | exit 1 |
| 326 | fi |
| 327 | |
| 328 | # Fill in defaults used by the non-update-only process |
| 329 | if [ -z "$VERSION" ]; then |
| 330 | VERSION="$(git describe --always HEAD)" |
| 331 | fi |
| 332 | |
| 333 | if [ -z "$DESCRIPTION" ]; then |
| 334 | DESCRIPTION="$(git rev-parse HEAD)" |
| 335 | fi |
| 336 | |
| 337 | if [ -z "$COVERITY_EMAIL" ]; then |
| 338 | COVERITY_EMAIL="$(git config coverity.email)" |
| 339 | fi |
| 340 | if [ -z "$COVERITY_EMAIL" ]; then |
| 341 | COVERITY_EMAIL="$(git config user.email)" |
| 342 | fi |
| 343 | |
| 344 | # Otherwise, continue with the full build and upload process. |
| 345 | |
| 346 | check_upload_permissions |
| 347 | |
| 348 | if [ "$UPDATE" != no ]; then |
| 349 | update_coverity_tools |
| 350 | fi |
| 351 | |
| 352 | # Run ourselves inside docker if that's what the user wants |
| 353 | if [ "$DOCKER" = yes ]; then |
| 354 | # Put the Coverity token into a temporary file that only |
| 355 | # we have read access to, and then pass it to docker build |
| 356 | # using a volume. A volume is enough for the token not to |
| 357 | # leak into the Docker image. |
| 358 | umask 077 |
| 359 | SECRETDIR=$(mktemp -d) |
| 360 | if [ -z "$SECRETDIR" ]; then |
| 361 | echo "Failed to create temporary directory" |
| 362 | exit 1 |
| 363 | fi |
| 364 | trap 'rm -rf "$SECRETDIR"' INT TERM EXIT |
| 365 | echo "Created temporary directory $SECRETDIR" |
| 366 | SECRET="$SECRETDIR/token" |
| 367 | echo "$COVERITY_TOKEN" > "$SECRET" |
| 368 | echo "Archiving sources to be analyzed..." |
| 369 | ./scripts/archive-source.sh "$SECRETDIR/qemu-sources.tgz" |
| 370 | ARGS="--no-update-tools" |
| 371 | if [ "$DRYRUN" = yes ]; then |
| 372 | ARGS="$ARGS --dry-run" |
| 373 | fi |
| 374 | echo "Running scanner..." |
| 375 | # If we need to capture the output tarball, get the inner run to |
| 376 | # save it to the secrets directory so we can copy it out before the |
| 377 | # directory is cleaned up. |
| 378 | if [ ! -z "$RESULTSTARBALL" ]; then |
| 379 | ARGS="$ARGS --results-tarball /work/cov-int.tar.xz" |
| 380 | fi |
| 381 | # Arrange for this docker run to get access to the sources with -v. |
| 382 | # We pass through all the configuration from the outer script to the inner. |
| 383 | export COVERITY_EMAIL COVERITY_BUILD_CMD |
| 384 | tests/docker/docker.py run -it --env COVERITY_EMAIL --env COVERITY_BUILD_CMD \ |
| 385 | -v "$SECRETDIR:/work" coverity-scanner \ |
| 386 | ./run-coverity-scan --version "$VERSION" \ |
| 387 | --description "$DESCRIPTION" $ARGS --tokenfile /work/token \ |
| 388 | --srcdir /qemu --src-tarball /work/qemu-sources.tgz |
| 389 | if [ ! -z "$RESULTSTARBALL" ]; then |
| 390 | echo "Copying results tarball to $RESULTSTARBALL..." |
| 391 | cp "$SECRETDIR/cov-int.tar.xz" "$RESULTSTARBALL" |
| 392 | fi |
| 393 | echo "Docker work complete." |
| 394 | exit 0 |
| 395 | fi |
| 396 | |
| 397 | TOOLBIN="$(cd "$COVERITY_TOOL_BASE" && echo $PWD/coverity_tool/cov-analysis-*/bin)" |
| 398 | |
| 399 | if ! test -x "$TOOLBIN/cov-build"; then |
| 400 | echo "Couldn't find cov-build in the coverity build-tool directory??" |
| 401 | exit 1 |
| 402 | fi |
| 403 | |
| 404 | export PATH="$TOOLBIN:$PATH" |
| 405 | |
| 406 | cd "$SRCDIR" |
| 407 | |
| 408 | echo "Nuking build directory..." |
| 409 | rm -rf +build |
| 410 | mkdir +build |
| 411 | cd +build |
| 412 | |
| 413 | echo "Configuring..." |
| 414 | # We configure with a fixed set of enables here to ensure that we don't |
| 415 | # accidentally reduce the scope of the analysis by doing the build on |
| 416 | # the system that's missing a dependency that we need to build part of |
| 417 | # the codebase. |
| 418 | ../configure --disable-modules --enable-sdl --enable-gtk \ |
| 419 | --enable-opengl --enable-vte --enable-gnutls \ |
| 420 | --enable-nettle --enable-curses --enable-curl \ |
| 421 | --audio-drv-list=oss,alsa,sdl,pa --enable-virtfs \ |
| 422 | --enable-vnc --enable-vnc-sasl --enable-vnc-jpeg --enable-png \ |
| 423 | --enable-xen --enable-brlapi \ |
| 424 | --enable-linux-aio --enable-attr \ |
| 425 | --enable-cap-ng --enable-trace-backends=log --enable-spice --enable-rbd \ |
| 426 | --enable-libusb --enable-usb-redir \ |
| 427 | --enable-libiscsi --enable-seccomp \ |
| 428 | --enable-tpm --enable-libssh --enable-lzo --enable-snappy --enable-bzip2 \ |
| 429 | --enable-numa --enable-rdma --enable-smartcard --enable-virglrenderer \ |
| 430 | --enable-mpath \ |
| 431 | --enable-virtfs --enable-zstd |
| 432 | |
| 433 | echo "Running cov-build..." |
| 434 | rm -rf cov-int |
| 435 | mkdir cov-int |
| 436 | cov-build --dir cov-int $COVERITY_BUILD_CMD |
| 437 | |
| 438 | echo "Creating results tarball..." |
| 439 | tar cvf - cov-int | xz > "$TARBALL" |
| 440 | |
| 441 | if [ ! -z "$RESULTSTARBALL" ]; then |
| 442 | echo "Copying results tarball to $RESULTSTARBALL..." |
| 443 | cp "$TARBALL" "$RESULTSTARBALL" |
| 444 | fi |
| 445 | |
| 446 | echo "Uploading results tarball..." |
| 447 | |
| 448 | if [ "$DRYRUN" = yes ]; then |
| 449 | echo "Dry run only, not uploading $TARBALL" |
| 450 | exit 0 |
| 451 | fi |
| 452 | |
| 453 | curl --form token="$COVERITY_TOKEN" --form email="$COVERITY_EMAIL" \ |
| 454 | --form file=@"$TARBALL" --form version="$VERSION" \ |
| 455 | --form description="$DESCRIPTION" \ |
| 456 | https://scan.coverity.com/builds?project="$PROJNAME" |
| 457 | |
| 458 | echo "Done." |