Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 14, 05:15 UTC
ci: Switch to Github Actions (#3152)
* WIP: Trying to make a new branch * Create fuzzing.yml * Update ci.yml * Try using reviewdog for golangci-lint * Only run lint on ubuntu * Whoops, wrong matrix variable * Let's try just ubuntu for the moment * Remove integration tests * Let's see what the tree looks like (where's the binary) * Let's plant a tree * Let's look at another tree * Burn the tree * Let's build in the right dir * Turn on publishing artifacts * Add gobin to path * Try running golangci-lint earlier * Try running golangci-lint on its own, with checkout@v1 * Try moving golangci-lint back into ci.yml as a separate job * Turn off azure-pipelines * Remove the redundant name, see how it looks * Trim down the naming some more * Turn on windows and mac * Try to fix windows build, cleanup * Try to fix strange failure on windows * Print our the coerce reason * Apparently $? is 'True' on Windows, not 1 or 0 * Try setting CGO_ENABLED as an env in yml * Try enabling/fixing the fuzzer * Print out github event to check, fix step name * Fuzzer needs the code * Add GOBIN to PATH for fuzzer * Comment out fork condition, left in-case we want it again * Remove obsolete comment * Comment out the coverage/test conversions for now * Set continue-on-error: true for fuzzer, it runs out of mem * Add some clarification to the retained commented sections
Francis Lavoie committed
Mar 20, 2020 at 10:38 UTC
76ac28a624cf3025cad9f8caa16e8c8a96b7466a
3 files changed
+203
-263
.github/workflows/ci.yml
new
+119
@@ -0,0 +1,119 @@
1
+# Used as inspiration: https://github.com/mvdan/github-actions-golang
2
+
3
+name: Cross-Platform
4
+
5
+on:
6
+ push:
7
+ branches:
8
+ - v2
9
+ pull_request:
10
+ branches:
11
+ - v2
12
+
13
+jobs:
14
+ test:
15
+ strategy:
16
+ # Default is true, cancels jobs for other platforms in the matrix if one fails
17
+ fail-fast: false
18
+ matrix:
19
+ os: [ ubuntu-latest, macos-latest, windows-latest ]
20
+ go-version: [ 1.14.x ]
21
+ runs-on: ${{ matrix.os }}
22
+
23
+ steps:
24
+ - name: Install Go
25
+ uses: actions/setup-go@v1
26
+ with:
27
+ go-version: ${{ matrix.go-version }}
28
+
29
+ - name: Checkout code
30
+ uses: actions/checkout@v2
31
+
32
+ # These tools would be useful if we later decide to reinvestigate
33
+ # publishing test/coverage reports to some tool for easier consumption
34
+ # - name: Install test and coverage analysis tools
35
+ # run: |
36
+ # go get github.com/axw/gocov/gocov
37
+ # go get github.com/AlekSi/gocov-xml
38
+ # go get -u github.com/jstemmer/go-junit-report
39
+ # echo "::add-path::$(go env GOPATH)/bin"
40
+
41
+ - name: Print Go version and environment
42
+ run: |
43
+ printf "Using go at: $(which go)\n"
44
+ printf "Go version: $(go version)\n"
45
+ printf "\n\nGo environment:\n\n"
46
+ go env
47
+ printf "\n\nSystem environment:\n\n"
48
+ env
49
+
50
+ - name: Get dependencies
51
+ run: |
52
+ go get -v -t -d ./...
53
+ # mkdir test-results
54
+
55
+ - name: Build Caddy
56
+ working-directory: ./cmd/caddy
57
+ env:
58
+ CGO_ENABLED: 0
59
+ run: |
60
+ go build -trimpath -a -ldflags="-w -s" -v
61
+
62
+ - name: Publish Build Artifact (Windows)
63
+ if: matrix.os == 'windows-latest'
64
+ uses: actions/upload-artifact@v1
65
+ with:
66
+ name: caddy_v2_${{ matrix.os }}
67
+ path: ./cmd/caddy/caddy.exe
68
+
69
+ - name: Publish Build Artifact (Linux/Mac)
70
+ if: matrix.os != 'windows-latest'
71
+ uses: actions/upload-artifact@v1
72
+ with:
73
+ name: caddy_v2_${{ matrix.os }}
74
+ path: ./cmd/caddy/caddy
75
+
76
+ # Commented bits below were useful to allow the job to continue
77
+ # even if the tests fail, so we can publish the report separately
78
+ # For info about set-output, see https://stackoverflow.com/questions/57850553/github-actions-check-steps-status
79
+ - name: Run tests
80
+ # id: step_test
81
+ # continue-on-error: true
82
+ run: |
83
+ # (go test -v -coverprofile=cover-profile.out -race ./... 2>&1) > test-results/test-result.out
84
+ go test -v -coverprofile="cover-profile.out" -race ./...
85
+ # echo "::set-output name=status::$?"
86
+
87
+ # Relevant step if we reinvestigate publishing test/coverage reports
88
+ # - name: Prepare coverage reports
89
+ # run: |
90
+ # mkdir coverage
91
+ # gocov convert cover-profile.out > coverage/coverage.json
92
+ # # Because Windows doesn't work with input redirection like *nix, but output redirection works.
93
+ # (cat ./coverage/coverage.json | gocov-xml) > coverage/coverage.xml
94
+
95
+ # To return the correct result even though we set 'continue-on-error: true'
96
+ # - name: Coerce correct build result (Windows)
97
+ # if: matrix.os == 'windows-latest' && steps.step_test.outputs.status != 'True'
98
+ # run: |
99
+ # echo "step_test ${{ steps.step_test.outputs.status }}\n"
100
+ # exit 1
101
+ # - name: Coerce correct build result (Linux/Mac)
102
+ # if: matrix.os != 'windows-latest' && steps.step_test.outputs.status != 0
103
+ # run: |
104
+ # echo "step_test ${{ steps.step_test.outputs.status }}\n"
105
+ # exit 1
106
+
107
+ # From https://github.com/reviewdog/action-golangci-lint
108
+ golangci-lint:
109
+ name: runner / golangci-lint
110
+ runs-on: ubuntu-latest
111
+ steps:
112
+ - name: Checkout code into the Go module directory
113
+ uses: actions/checkout@v2
114
+
115
+ - name: Run golangci-lint
116
+ uses: reviewdog/action-golangci-lint@v1
117
+ # uses: docker://reviewdog/action-golangci-lint:v1 # pre-build docker image
118
+ with:
119
+ github_token: ${{ secrets.github_token }}
\ No newline at end of file
.github/workflows/fuzzing.yml
new
+84
@@ -0,0 +1,84 @@
1
+name: Fuzzing
2
+
3
+on:
4
+ # Regression testing
5
+ push:
6
+ branches:
7
+ - v2
8
+ pull_request:
9
+ branches:
10
+ - v2
11
+
12
+ # Daily midnight fuzzing
13
+ schedule:
14
+ - cron: '0 0 * * *'
15
+
16
+jobs:
17
+ fuzzing:
18
+ name: Fuzzing
19
+
20
+ strategy:
21
+ matrix:
22
+ os: [ ubuntu-latest ]
23
+ go-version: [ 1.14.x ]
24
+ runs-on: ${{ matrix.os }}
25
+
26
+ steps:
27
+ - name: Install Go
28
+ uses: actions/setup-go@v1
29
+ with:
30
+ go-version: ${{ matrix.go-version }}
31
+
32
+ - name: Checkout code
33
+ uses: actions/checkout@v2
34
+
35
+ - name: Download go-fuzz tools and the Fuzzit CLI, move Fuzzit CLI to GOBIN
36
+ # If we decide we need to prevent this from running on forks, we can use this line:
37
+ # if: github.repository == 'caddyserver/caddy'
38
+ run: |
39
+ # Install Clang-7.0 because other versions seem to be missing the file libclang_rt.fuzzer-x86_64.a
40
+ sudo add-apt-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-7 main"
41
+ wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
42
+ sudo apt update && sudo apt install -y clang-7 lldb-7 lld-7
43
+
44
+ go get -v github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build
45
+ wget -q -O fuzzit https://github.com/fuzzitdev/fuzzit/releases/download/v2.4.77/fuzzit_Linux_x86_64
46
+ chmod a+x fuzzit
47
+ mv fuzzit $(go env GOPATH)/bin
48
+ echo "::add-path::$(go env GOPATH)/bin"
49
+
50
+ - name: Generate fuzzers & submit them to Fuzzit
51
+ continue-on-error: true
52
+ env:
53
+ FUZZIT_API_KEY: ${{ secrets.FUZZIT_API_KEY }}
54
+ run: |
55
+ declare -A fuzzers_funcs=(\
56
+ ["./caddyconfig/httpcaddyfile/addresses_fuzz.go"]="FuzzParseAddress" \
57
+ ["./caddyconfig/caddyfile/parse_fuzz.go"]="FuzzParseCaddyfile" \
58
+ ["./listeners_fuzz.go"]="FuzzParseNetworkAddress" \
59
+ ["./replacer_fuzz.go"]="FuzzReplacer" \
60
+ )
61
+
62
+ declare -A fuzzers_targets=(\
63
+ ["./caddyconfig/httpcaddyfile/addresses_fuzz.go"]="parse-address" \
64
+ ["./caddyconfig/caddyfile/parse_fuzz.go"]="parse-caddyfile" \
65
+ ["./listeners_fuzz.go"]="parse-network-address" \
66
+ ["./replacer_fuzz.go"]="replacer" \
67
+ )
68
+
69
+ fuzz_type="local-regression"
70
+ if [[ ${{ github.event_name }} == "schedule" ]]; then
71
+ fuzz_type="fuzzing"
72
+ fi
73
+ echo "Github event: ${{ github.event_name }}"
74
+ echo "Fuzzing type: $fuzz_type"
75
+
76
+ for f in $(find . -name \*_fuzz.go); do
77
+ FUZZER_DIRECTORY=$(dirname $f)
78
+ echo "go-fuzz-build func ${fuzzers_funcs[$f]} residing in $f"
79
+ go-fuzz-build -func "${fuzzers_funcs[$f]}" -libfuzzer -o "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}.a" $FUZZER_DIRECTORY
80
+ echo "Generating fuzzer binary of func ${fuzzers_funcs[$f]} which resides in $f"
81
+ clang-7 -fsanitize=fuzzer "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}.a" -o "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}"
82
+ fuzzit create job caddyserver/${fuzzers_targets[$f]} $FUZZER_DIRECTORY/${fuzzers_targets[$f]} --api-key ${FUZZIT_API_KEY} --type "${fuzz_type}" --branch "${SYSTEM_PULLREQUEST_SOURCEBRANCH}" --revision "${BUILD_SOURCEVERSION}"
83
+ echo "Completed $f"
84
+ done
azure-pipelines.yml
deleted
-263
@@ -1,263 +0,0 @@
1
-# Mutilated beyond recognition from the example at:
2
-# https://docs.microsoft.com/azure/devops/pipelines/languages/go
3
-
4
-trigger:
5
- - v2
6
-
7
-schedules:
8
-- cron: "0 0 * * *"
9
- displayName: Daily midnight fuzzing
10
- branches:
11
- include:
12
- - v2
13
- always: true
14
-
15
-variables:
16
- GOROOT: $(gorootDir)/go
17
- GOPATH: $(system.defaultWorkingDirectory)/gopath
18
- GOBIN: $(GOPATH)/bin
19
- modulePath: '$(GOPATH)/src/github.com/$(build.repository.name)'
20
-
21
-jobs:
22
-- job: crossPlatformTest
23
- displayName: "Cross-Platform Tests"
24
- strategy:
25
- matrix:
26
- linux:
27
- imageName: ubuntu-16.04
28
- gorootDir: /usr/local
29
- mac:
30
- imageName: macos-10.14
31
- gorootDir: /usr/local
32
- windows:
33
- imageName: windows-2019
34
- gorootDir: C:\
35
- pool:
36
- vmImage: $(imageName)
37
-
38
- steps:
39
- - bash: |
40
- latestGo=$(curl "https://golang.org/VERSION?m=text")
41
- echo "##vso[task.setvariable variable=LATEST_GO]$latestGo"
42
- echo "Latest Go version: $latestGo"
43
- displayName: "Get latest Go version"
44
-
45
- - bash: |
46
- sudo rm -f $(which go)
47
- echo '##vso[task.prependpath]$(GOBIN)'
48
- echo '##vso[task.prependpath]$(GOROOT)/bin'
49
- mkdir -p '$(modulePath)'
50
- shopt -s extglob
51
- shopt -s dotglob
52
- mv !(gopath) '$(modulePath)'
53
- displayName: Remove old Go, set GOBIN/GOROOT, and move project into GOPATH
54
-
55
- # Install Go (this varies by platform)
56
- - bash: |
57
- wget "https://dl.google.com/go/$(LATEST_GO).linux-amd64.tar.gz"
58
- sudo tar -C $(gorootDir) -xzf "$(LATEST_GO).linux-amd64.tar.gz"
59
- condition: eq( variables['Agent.OS'], 'Linux' )
60
- displayName: Install Go on Linux
61
-
62
- - bash: |
63
- wget "https://dl.google.com/go/$(LATEST_GO).darwin-amd64.tar.gz"
64
- sudo tar -C $(gorootDir) -xzf "$(LATEST_GO).darwin-amd64.tar.gz"
65
- condition: eq( variables['Agent.OS'], 'Darwin' )
66
- displayName: Install Go on macOS
67
-
68
- # The low performance is partly due to PowerShell's attempt to update the progress bar. Disabling it speeds up the process.
69
- # Reference: https://github.com/PowerShell/PowerShell/issues/2138
70
- - powershell: |
71
- $ProgressPreference = 'SilentlyContinue'
72
- Write-Host "Downloading Go..."
73
- (New-Object System.Net.WebClient).DownloadFile("https://dl.google.com/go/$(LATEST_GO).windows-amd64.zip", "$(LATEST_GO).windows-amd64.zip")
74
- Write-Host "Extracting Go... (I'm slow too)"
75
- 7z x "$(LATEST_GO).windows-amd64.zip" -o"$(gorootDir)"
76
- condition: eq( variables['Agent.OS'], 'Windows_NT' )
77
- displayName: Install Go on Windows
78
-
79
- - bash: curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.23.6
80
- displayName: Install golangci-lint
81
-
82
- - script: |
83
- go get github.com/axw/gocov/gocov
84
- go get github.com/AlekSi/gocov-xml
85
- go get -u github.com/jstemmer/go-junit-report
86
- displayName: Install test and coverage analysis tools
87
-
88
- - bash: |
89
- printf "Using go at: $(which go)\n"
90
- printf "Go version: $(go version)\n"
91
- printf "\n\nGo environment:\n\n"
92
- go env
93
- printf "\n\nSystem environment:\n\n"
94
- env
95
- displayName: Print Go version and environment
96
-
97
- - script: |
98
- go get -v -t -d ./...
99
- mkdir test-results
100
- workingDirectory: '$(modulePath)'
101
- displayName: Get dependencies
102
-
103
- - bash: CGO_ENABLED=0 go build -trimpath -a -ldflags="-w -s" -v
104
- workingDirectory: '$(modulePath)/cmd/caddy'
105
- displayName: Build Caddy
106
-
107
- - task: PublishBuildArtifacts@1
108
- condition: eq( variables['Agent.OS'], 'Windows_NT' )
109
- inputs:
110
- pathtoPublish: '$(modulePath)/cmd/caddy/caddy.exe'
111
- artifactName: caddy_v2.exe
112
-
113
- - task: PublishBuildArtifacts@1
114
- condition: ne( variables['Agent.OS'], 'Windows_NT' )
115
- inputs:
116
- pathtoPublish: '$(modulePath)/cmd/caddy/caddy'
117
- artifactName: 'caddy_v2_$(Agent.OS)'
118
-
119
- # its behavior is governed by .golangci.yml
120
- - script: |
121
- (golangci-lint run --out-format junit-xml) > test-results/lint-result.xml
122
- exit 0
123
- workingDirectory: '$(modulePath)'
124
- continueOnError: true
125
- displayName: Run lint check
126
-
127
- - script: |
128
- (go test -v -coverprofile=cover-profile.out -race ./... 2>&1) > test-results/test-result.out
129
- workingDirectory: '$(modulePath)'
130
- continueOnError: true
131
- displayName: Run tests
132
-
133
- - script: |
134
- set -e
135
- cmd/caddy/caddy start
136
- go test -v -count=1 ./caddytest/...
137
- cmd/caddy/caddy stop
138
- workingDirectory: '$(modulePath)'
139
- continueOnError: false
140
- displayName: Run Integration tests
141
-
142
- - script: |
143
- mkdir coverage
144
- gocov convert cover-profile.out > coverage/coverage.json
145
- # Because Windows doesn't work with input redirection like *nix, but output redirection works.
146
- (cat ./coverage/coverage.json | gocov-xml) > coverage/coverage.xml
147
- workingDirectory: '$(modulePath)'
148
- displayName: Prepare coverage reports
149
-
150
- - script: |
151
- (cat ./test-results/test-result.out | go-junit-report) > test-results/test-result.xml
152
- workingDirectory: '$(modulePath)'
153
- displayName: Prepare test report
154
-
155
- - task: PublishCodeCoverageResults@1
156
- displayName: Publish test coverage report
157
- inputs:
158
- codeCoverageTool: Cobertura
159
- summaryFileLocation: $(modulePath)/coverage/coverage.xml
160
-
161
- - task: PublishTestResults@2
162
- displayName: Publish unit test
163
- inputs:
164
- testResultsFormat: 'JUnit'
165
- testResultsFiles: $(modulePath)/test-results/test-result.xml
166
- testRunTitle: $(agent.OS) Unit Test
167
- mergeTestResults: false
168
-
169
- - task: PublishTestResults@2
170
- displayName: Publish lint results
171
- inputs:
172
- testResultsFormat: 'JUnit'
173
- testResultsFiles: $(modulePath)/test-results/lint-result.xml
174
- testRunTitle: $(agent.OS) Lint
175
- mergeTestResults: false
176
-
177
- - bash: |
178
- exit 1
179
- condition: eq(variables['Agent.JobStatus'], 'SucceededWithIssues')
180
- displayName: Coerce correct build result
181
-
182
-- job: fuzzing
183
- displayName: 'Fuzzing'
184
- # Only run this job on schedules or PRs for non-forks.
185
- condition: or(eq(variables['System.PullRequest.IsFork'], 'False'), eq(variables['Build.Reason'], 'Schedule') )
186
- strategy:
187
- matrix:
188
- linux:
189
- imageName: ubuntu-16.04
190
- gorootDir: /usr/local
191
- pool:
192
- vmImage: $(imageName)
193
-
194
- steps:
195
- - bash: |
196
- latestGo=$(curl "https://golang.org/VERSION?m=text")
197
- echo "##vso[task.setvariable variable=LATEST_GO]$latestGo"
198
- echo "Latest Go version: $latestGo"
199
- displayName: "Get latest Go version"
200
-
201
- - bash: |
202
- sudo rm -f $(which go)
203
- echo '##vso[task.prependpath]$(GOBIN)'
204
- echo '##vso[task.prependpath]$(GOROOT)/bin'
205
- mkdir -p '$(modulePath)'
206
- shopt -s extglob
207
- shopt -s dotglob
208
- mv !(gopath) '$(modulePath)'
209
- displayName: Remove old Go, set GOBIN/GOROOT, and move project into GOPATH
210
-
211
- - bash: |
212
- wget "https://dl.google.com/go/$(LATEST_GO).linux-amd64.tar.gz"
213
- sudo tar -C $(gorootDir) -xzf "$(LATEST_GO).linux-amd64.tar.gz"
214
- condition: eq( variables['Agent.OS'], 'Linux' )
215
- displayName: Install Go on Linux
216
-
217
- - bash: |
218
- # Install Clang-7.0 because other versions seem to be missing the file libclang_rt.fuzzer-x86_64.a
219
- sudo add-apt-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-7 main"
220
- wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
221
- sudo apt update && sudo apt install -y clang-7 lldb-7 lld-7
222
-
223
- go get -v github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build
224
- wget -q -O fuzzit https://github.com/fuzzitdev/fuzzit/releases/download/v2.4.77/fuzzit_Linux_x86_64
225
- chmod a+x fuzzit
226
- mv fuzzit $(GOBIN)
227
- displayName: Download go-fuzz tools and the Fuzzit CLI, and move Fuzzit CLI to GOBIN
228
- condition: and(eq(variables['System.PullRequest.IsFork'], 'False') , eq( variables['Agent.OS'], 'Linux' ))
229
-
230
- - bash: |
231
- declare -A fuzzers_funcs=(\
232
- ["./caddyconfig/httpcaddyfile/addresses_fuzz.go"]="FuzzParseAddress" \
233
- ["./caddyconfig/caddyfile/parse_fuzz.go"]="FuzzParseCaddyfile" \
234
- ["./listeners_fuzz.go"]="FuzzParseNetworkAddress" \
235
- ["./replacer_fuzz.go"]="FuzzReplacer" \
236
- )
237
-
238
- declare -A fuzzers_targets=(\
239
- ["./caddyconfig/httpcaddyfile/addresses_fuzz.go"]="parse-address" \
240
- ["./caddyconfig/caddyfile/parse_fuzz.go"]="parse-caddyfile" \
241
- ["./listeners_fuzz.go"]="parse-network-address" \
242
- ["./replacer_fuzz.go"]="replacer" \
243
- )
244
-
245
- fuzz_type="local-regression"
246
- if [[ $(Build.Reason) == "Schedule" ]]; then
247
- fuzz_type="fuzzing"
248
- fi
249
- echo "Fuzzing type: $fuzz_type"
250
-
251
- for f in $(find . -name \*_fuzz.go); do
252
- FUZZER_DIRECTORY=$(dirname $f)
253
- echo "go-fuzz-build func ${fuzzers_funcs[$f]} residing in $f"
254
- go-fuzz-build -func "${fuzzers_funcs[$f]}" -libfuzzer -o "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}.a" $FUZZER_DIRECTORY
255
- echo "Generating fuzzer binary of func ${fuzzers_funcs[$f]} which resides in $f"
256
- clang-7 -fsanitize=fuzzer "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}.a" -o "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}"
257
- fuzzit create job caddyserver/${fuzzers_targets[$f]} $FUZZER_DIRECTORY/${fuzzers_targets[$f]} --api-key ${FUZZIT_API_KEY} --type "${fuzz_type}" --branch "${SYSTEM_PULLREQUEST_SOURCEBRANCH}" --revision "${BUILD_SOURCEVERSION}"
258
- echo "Completed $f"
259
- done
260
- env:
261
- FUZZIT_API_KEY: $(FUZZIT_API_KEY)
262
- workingDirectory: '$(modulePath)'
263
- displayName: Generate fuzzers & submit them to Fuzzit