@cryptotaxi247 / netdata-1 / commits / 5bcaae918

simplify CI system configuration (#4534)

[minor]

Paweł Krupa committed Nov 1, 2018 at 11:17 UTC 5bcaae918eaa3ded10000429906bba56da2b8d11

5 files changed +128 -92

.travis.yml

+48 -64

@@ -1,78 +1,62 @@

1 ----

1 dist: trusty

2 sudo: true

4 -

5 -# C includes autotools and make by default

3 language: c

4 services:

8 - - docker

5 +- docker

7 stages:

11 - - lint

12 - - test

13 - - build

14 - - name: release

15 - if: branch = master AND type != pull_request

16 - - name: package

17 - if: branch = master AND type != pull_request

8 +- test

9 +- build

10 +- name: packaging

11 + if: branch = master AND type != pull_request

13 jobs:

14 include:

21 - - stage: "lint"

22 - name: "python"

23 - install: skip

24 - script: docker run -ti --rm -v $(pwd):/code alpine/flake8:3.5.0 --max-line-length=120 --exclude=third_party,urllib3,pyyaml3,pyyaml2 /code/collectors/python.d.plugin/python.d.plugin.in /code/

25 - - name: "css"

26 - install: skip

27 - script: docker run --rm -v $(pwd):/code eeacms/csslint sh -c "csslint /code/web/gui/*.css" || echo "OK"

28 - - name: "bash"

29 - install: skip

30 - script: docker run --rm --volume "$(pwd)":/code:ro --entrypoint sh koalaman/shellcheck-alpine:v0.5.0 -c 'for file in $(find /code/ -type f -name "*.sh" ! -path "*/makeself/*" ); do if ! shellcheck --format=gcc $file; then export FAILED=true; fi; done; if [ "$FAILED" != "" ]; then exit 1; fi'

31 - - stage: "test"

32 - name: "C"

33 - install: sudo apt-get install -y libcap2-bin zlib1g-dev uuid-dev fakeroot libipmimonitoring-dev libmnl-dev libnetfilter-acct-dev

34 - script: fakeroot ./netdata-installer.sh --install $HOME --dont-wait --dont-start-it --enable-plugin-nfacct --enable-plugin-freeipmi --disable-lto && $HOME/netdata/usr/sbin/netdata -W unittest

35 - env: CFLAGS='-O1 -DNETDATA_INTERNAL_CHECKS=1 -DNETDATA_VERIFY_LOCKS=1'

36 -# env: CFLAGS='-O1 -ggdb -Wall -Wextra -Wformat-signedness -fstack-protector-all -DNETDATA_INTERNAL_CHECKS=1 -D_FORTIFY_SOURCE=2 -DNETDATA_VERIFY_LOCKS=1'

37 -# - name: "Node.js"

38 -# script: tests/run_nodejs.sh

39 - - name: "coverity"

40 - install: sudo apt-get install -y zlib1g-dev uuid-dev libipmimonitoring-dev libmnl-dev libnetfilter-acct-dev

41 - script: ./coverity-scan.sh || echo "Coverity failed :("

42 - env: INSTALL_COVERITY="true"

43 - if: type = cron

44 -

45 - - stage: "build"

46 - name: "OSX"

47 - install: brew install fakeroot ossp-uuid

48 - script: fakeroot ./netdata-installer.sh --install $HOME --dont-wait --dont-start-it

49 - os: osx

50 - - name: "ubuntu 14.04 (not containerized)"

51 - install: sudo apt-get install -y libcap2-bin zlib1g-dev uuid-dev fakeroot

52 - script: fakeroot ./netdata-installer.sh --dont-wait --dont-start-it --install $HOME

53 - - name: "ubuntu 18.04"

54 - script: ./.travis/containerized_build.sh ubuntu1804

55 - - name: "CentOS 7"

56 - script: ./.travis/containerized_build.sh centos7

57 - - name: "CentOS 6"

58 - script: ./.travis/containerized_build.sh centos6

59 - - name: "alpine"

60 - script: ./.travis/containerized_build.sh alpine

15 + - stage: test

16 + name: C

17 + install: sudo apt-get install -y libcap2-bin zlib1g-dev uuid-dev fakeroot libipmimonitoring-dev libmnl-dev libnetfilter-acct-dev

18 + script: fakeroot ./netdata-installer.sh --install $HOME --dont-wait --dont-start-it --enable-plugin-nfacct --enable-plugin-freeipmi --disable-lto && $HOME/netdata/usr/sbin/netdata -W unittest

19 + env: CFLAGS='-O1 -DNETDATA_INTERNAL_CHECKS=1 -DNETDATA_VERIFY_LOCKS=1'

20 + - name: coverity

21 + install: sudo apt-get install -y zlib1g-dev uuid-dev libipmimonitoring-dev libmnl-dev libnetfilter-acct-dev

22 + script: ./coverity-scan.sh || echo "Coverity failed :("

23 + env: INSTALL_COVERITY="true"

24 + if: type = cron

62 - - stage: "release"

63 - name: "Create release"

64 - install: sudo apt-get install -y gnupg libcap2-bin zlib1g-dev uuid-dev fakeroot

65 - script: .travis/releaser.sh

26 + - stage: build

27 + name: OSX

28 + install: brew install fakeroot ossp-uuid

29 + script: fakeroot ./netdata-installer.sh --install $HOME --dont-wait --dont-start-it

30 + os: osx

31 + - name: ubuntu 14.04 (not containerized)

32 + install: sudo apt-get install -y libcap2-bin zlib1g-dev uuid-dev fakeroot

33 + script: fakeroot ./netdata-installer.sh --dont-wait --dont-start-it --install $HOME

34 + - name: ubuntu 18.04

35 + script: ".travis/containerized_build.sh ubuntu1804"

36 + - name: CentOS 7

37 + script: ".travis/containerized_build.sh centos7"

38 + - name: CentOS 6

39 + script: ".travis/containerized_build.sh centos6"

40 + - name: alpine

41 + script: ".travis/containerized_build.sh alpine"

67 - - stage: "package"

68 - name: "Archive and self-extractor"

69 - install: sudo apt-get install -y gnupg libcap2-bin zlib1g-dev uuid-dev fakeroot

70 - script: .travis/create_artifacts.sh

71 - - name: "Docker images"

72 - install: sudo apt update -y && sudo apt install -y --only-upgrade docker-ce && docker info

73 - script: docker/build.sh

74 - env: REPOSITORY="netdata/netdata"

75 - if: type = cron OR tag =~ ^v

43 + - stage: packaging

44 + name: Update Changelog/Create release

45 + install: sudo apt-get install -y gnupg libcap2-bin zlib1g-dev uuid-dev fakeroot

46 + script: ".travis/releaser.sh"

47 + - name: Nightly tarball and self-extractor build

48 + install: sudo apt-get install -y gnupg libcap2-bin zlib1g-dev uuid-dev fakeroot

49 + script: ".travis/firehol_create_artifacts.sh"

50 + if: type = cron

51 + - name: Nightly docker images

52 + install: sudo apt update -y && sudo apt install -y --only-upgrade docker-ce && docker info

53 + script: "docker/build.sh"

54 + env: REPOSITORY="netdata/netdata"

55 + if: type = cron

57 notifications:

58 webhooks: https://app.fossa.io/hooks/travisci

59 + slack:

60 + secure: Ntl1GNjgGDdmMQsudDi/4+co6zRV8znJXOB0joLVHNV1icHvUSMKShgpY08AoWoxNYMDVHXOUzEGwiZd6MI4MleNu9gwtACwgXe+sDjgBIDvQYC6JQS7XlANJfzBlZnrGnaiVVlBmI/LajgABCsrTAYV8gVrq2zA3u56UrNwS3qRJnbaqGgH64SnHbdllHQ7l7Qcc/27voy6Op32oJhDZ5bVY5CRZUR42oPvlDx9PSD0yqSkLwFrIE0Qg+mTVDZ4Watrv8iv91ghc6GwwPvNGShcYI04P8hn08FZ4eGH47K4AZf97IDargmxPsEwkOhgcl3w4J5KHZzmVb7hsEs6+iZz4KulD2u4D/xy5BGUcRZKMuyiO99dmZO61wP0Ra0Wzfe0exG+LFxAmC0jiYS67xL0YWs2SKkO7iftthI8NX1Lus1PGleBpJ28LKQdZjaKz3w2xIRlJ45eEdwBgtnNkW68NsmJCflrNkAlaVW6+zCPMpT4rT41rpkh0Fl+YbvOhH/S3VfiTY7OqTU5B9ttgJuN/m8PWi8g5MKO2C1TWkK3cV4O2PnpugtioHuTuj2X/AKOrh7hlefedDxPZPIxFmLi3OKtgHBzwzQk79p6V9ffbbVfsO+p8kiKJuc8s0v6d++Rt2FLXwUquJGDKC0hz8cAMSUiFaP5t1TlhTk7LJ4=

61 + on_success: change

62 + on_pull_requests: false

.travis/README.md

+27 -8

@@ -27,6 +27,7 @@ installations of netdata. Jobs are run on following operating systems:

27 - ubuntu 16.04 (containerized)

28 - CentOS 6 (containerized)

29 - CentOS 7 (containerized)

30 + - alpine (containerized)

32 ### Release

@@ -41,15 +42,33 @@ Releases are generated by searching for a keyword in last commit message. Keywor

42 All keywords MUST be surrounded with square braces.

43 Alternative is to push a tag to master branch.

44 -### Packages

45 +### Packaging

46 -As a name might suggest, this stage is responsible for creating netdata packages such as:

47 +This stage is executed only on "master" branch and it is separated into 3 jobs:

48 + - Update Changelog/Create release

49 + - Nightly tarball and self-extractor build

50 + - Nightly docker images

51 +

52 +##### Update Changelog/Create release

53 +

54 +This job is running one script called `releaser.sh`, which is responsible for a couple of things. First of all it

55 +automatically updates our CHANGELOG.md file based on GitHub features (mostly labels and pull requests). Apart from

56 +that it can also create a new git tag and a github draft release connected to that tag.

57 +Releases are generated by searching for a keyword in last commit message. Keywords are:

58 + - `[patch]` or `[fix]` to bump patch number

59 + - `[minor]`, `[feature]` or `[feat]` to bump minor number

60 + - `[major]` or `[breaking change]` to bump major number

61 +All keywords MUST be surrounded with square brackets.

62 +

63 +Alternatively new release can be also created by pushing new tag to master branch.

64 +

65 +##### Nightly tarball and self-extractor build AND Nightly docker images

66 +

67 +As names might suggest those two jobs are responsible for nightly netdata package creation and are run every day (in

68 +cron). Combined they produce:

69 - docker images

48 - - tar repository (soon to be removed)

70 + - tar.gz archive (soon to be removed)

71 - self-extracting package

51 -This stage is separated into 2 parallel jobs. One creating docker images and second one creating github artifacts.

52 -Whole stage is executed only on master branch, but there are also additional parameters assigned to jobs:

53 - - docker images - execution only on cron job or when there is a tag assigned starting with `v`

54 - - artifacts - script responsible for creating artifacts doesn't have special conditions, but deployment to GitHub

55 - releases is done only when there is tag assigned

73 +Currently "Nightly tarball and self-extractor build" is using old firehol script and it is planed to be replaced with

74 +new design.

.travis/create_artifacts.sh

+18 -13

@@ -7,20 +7,25 @@ then

7 exit 1

8 fi

10 -eval "$(ssh-agent -s)"

11 -./.travis/decrypt-if-have-key decb6f6387c4

12 -export KEYSERVER=ipv4.pool.sks-keyservers.net

13 -./packaging/gpg-recv-key phil@firehol.org "0762 9FF7 89EA 6156 012F 9F50 C406 9602 1359 9237"

14 -./packaging/gpg-recv-key costa@tsaousis.gr "4DFF 624A E564 3B51 2872 1F40 29CA 3358 89B9 A863"

15 -# Run the commit hooks in case the developer didn't

16 -git diff 4b825dc642cb6eb9a060e54bf8d69288fbee4904 | ./packaging/check-files -

17 -fakeroot ./packaging/git-build

10 # Make sure stdout is in blocking mode. If we don't, then conda create will barf during downloads.

11 # See https://github.com/travis-ci/travis-ci/issues/4704#issuecomment-348435959 for details.

12 python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);'

21 -# make self-extractor

13 +echo "--- Create tarball ---"

14 +make dist

15 +echo "--- Create self-extractor ---"

16 ./makeself/build-x86_64-static.sh

23 -for i in *.tar.gz; do sha512sum -b "$i" > "$i.sha"; done

24 -for i in *.gz.run; do sha512sum -b "$i" > "$i.sha"; done

25 -sha256sum -b ./*.tar.gz ./*.gz.run > "sha256sums.txt"

26 -./.travis/deploy-if-have-key

17 +

18 +echo "--- Create checksums ---"

19 +GIT_TAG=$(git tag --points-at)

20 +if [ "${GIT_TAG}" != "" ]; then

21 + ln -s netdata-latest.gz.run "netdata-${GIT_TAG}.gz.run"

22 + ln -s netdata-*.tar.gz "netdata-${GIT_TAG}.tar.gz"

23 + sha256sum -b "netdata-${GIT_TAG}.gz.run" "netdata-${GIT_TAG}.tar.gz" > "sha256sums.txt"

24 +else

25 + sha256sum -b ./*.tar.gz ./*.gz.run > "sha256sums.txt"

26 +fi

27 +

28 +echo "checksums:"

29 +cat sha256sums.txt

30 +

31 +

.travis/firehol_create_artifacts.sh new

+31

@@ -0,0 +1,31 @@

1 +#!/bin/bash

2 +# shellcheck disable=SC2230

3 +

4 +# WARNING: This script is deprecated and placed here until @paulfantom figures out how to fully replace it

5 +

6 +if [ ! -f .gitignore ]

7 +then

8 + echo "Run as ./travis/$(basename "$0") from top level directory of git repository"

9 + exit 1

10 +fi

11 +

12 +eval "$(ssh-agent -s)"

13 +./.travis/decrypt-if-have-key decb6f6387c4

14 +export KEYSERVER=ipv4.pool.sks-keyservers.net

15 +./packaging/gpg-recv-key phil@firehol.org "0762 9FF7 89EA 6156 012F 9F50 C406 9602 1359 9237"

16 +./packaging/gpg-recv-key costa@tsaousis.gr "4DFF 624A E564 3B51 2872 1F40 29CA 3358 89B9 A863"

17 +# Run the commit hooks in case the developer didn't

18 +git diff 4b825dc642cb6eb9a060e54bf8d69288fbee4904 | ./packaging/check-files -

19 +fakeroot ./packaging/git-build

20 +# Make sure stdout is in blocking mode. If we don't, then conda create will barf during downloads.

21 +# See https://github.com/travis-ci/travis-ci/issues/4704#issuecomment-348435959 for details.

22 +python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);'

23 +echo "--- Create tarball ---"

24 +make dist

25 +echo "--- Create self-extractor ---"

26 +./makeself/build-x86_64-static.sh

27 +echo "--- Create checksums ---"

28 +for i in *.tar.gz; do sha512sum -b "$i" > "$i.sha"; done #FIXME remove?

29 +for i in *.gz.run; do sha512sum -b "$i" > "$i.sha"; done #FIXME remove?

30 +sha256sum -b ./*.tar.gz ./*.gz.run > "sha256sums.txt"

31 +./.travis/deploy-if-have-key

.travis/releaser.sh

+4 -7

@@ -94,14 +94,11 @@ if [ "${GIT_TAG}" == "HEAD" ]; then

94 exit 0

95 fi

97 -echo "---- CREATING RELEASE ARTIFACTS -----"

98 -python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);'

97 +echo "---- CREATING TAGGED DOCKER CONTAINERS ----"

98 +./docker/build.sh

100 -./makeself/build-x86_64-static.sh

101 -make dist

102 -ln -s netdata-latest.gz.run "netdata-${GIT_TAG}.gz.run"

103 -ln -s netdata-*.tar.gz "netdata-${GIT_TAG}.tar.gz"

104 -sha256sum -b "netdata-${GIT_TAG}.gz.run" "netdata-${GIT_TAG}.tar.gz" > "sha256sums.txt"

100 +echo "---- CREATING RELEASE ARTIFACTS -----"

101 +./.travis/create_artifacts.sh

102

103 echo "---- CREATING RELEASE DRAFT WITH ASSETS -----"

104 # Download hub