Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 12, 15:45 UTC
logging: Implement `append` encoder, allow flatter filters config (#6069)
* logging: Implement `add` encoder * Allow flatter config structure for `filter` & `add` * Rename to append * govulncheck was unhappy
Francis Lavoie committed
Mar 5, 2024 at 18:24 UTC
01d5568b20408a7f72fb53095e2e146f0c39672a
7 files changed
+476
-34
.github/workflows/ci.yml
+1
-1
@@ -33,7 +33,7 @@ jobs:
33
GO_SEMVER: '~1.21.0'
34
35
- go: '1.22'
36
- GO_SEMVER: '~1.22.0'
36
+ GO_SEMVER: '~1.22.1'
37
38
# Set some variables per OS, usable via ${{ matrix.VAR }}
39
# OS_LABEL: the VM label from GitHub Actions (see https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories)
.github/workflows/cross-build.yml
+1
-1
@@ -35,7 +35,7 @@ jobs:
35
# Set the minimum Go patch version for the given Go minor
36
# Usable via ${{ matrix.GO_SEMVER }}
37
- go: '1.22'
38
- GO_SEMVER: '~1.22.0'
38
+ GO_SEMVER: '~1.22.1'
39
40
runs-on: ubuntu-latest
41
continue-on-error: true
.github/workflows/lint.yml
+2
-2
@@ -43,7 +43,7 @@ jobs:
43
- uses: actions/checkout@v4
44
- uses: actions/setup-go@v5
45
with:
46
- go-version: '~1.22.0'
46
+ go-version: '~1.22.1'
47
check-latest: true
48
49
- name: golangci-lint
@@ -66,5 +66,5 @@ jobs:
66
- name: govulncheck
67
uses: golang/govulncheck-action@v1
68
with:
69
- go-version-input: '~1.22.0'
69
+ go-version-input: '~1.22.1'
70
check-latest: true
caddytest/integration/caddyfile_adapt/log_append_encoder.caddyfiletest
new
+63
@@ -0,0 +1,63 @@
1
+{
2
+ log {
3
+ format append {
4
+ wrap json
5
+ fields {
6
+ wrap "foo"
7
+ }
8
+ env {env.EXAMPLE}
9
+ int 1
10
+ float 1.1
11
+ bool true
12
+ string "string"
13
+ }
14
+ }
15
+}
16
+
17
+:80 {
18
+ respond "Hello, World!"
19
+}
20
+----------
21
+{
22
+ "logging": {
23
+ "logs": {
24
+ "default": {
25
+ "encoder": {
26
+ "fields": {
27
+ "bool": true,
28
+ "env": "{env.EXAMPLE}",
29
+ "float": 1.1,
30
+ "int": 1,
31
+ "string": "string",
32
+ "wrap": "foo"
33
+ },
34
+ "format": "append",
35
+ "wrap": {
36
+ "format": "json"
37
+ }
38
+ }
39
+ }
40
+ }
41
+ },
42
+ "apps": {
43
+ "http": {
44
+ "servers": {
45
+ "srv0": {
46
+ "listen": [
47
+ ":80"
48
+ ],
49
+ "routes": [
50
+ {
51
+ "handle": [
52
+ {
53
+ "body": "Hello, World!",
54
+ "handler": "static_response"
55
+ }
56
+ ]
57
+ }
58
+ ]
59
+ }
60
+ }
61
+ }
62
+ }
63
+}
\ No newline at end of file
caddytest/integration/caddyfile_adapt/log_filters.caddyfiletest
+18
-14
@@ -4,27 +4,31 @@ log {
4
output stdout
5
format filter {
6
wrap console
7
+
8
+ # long form, with "fields" wrapper
9
fields {
10
uri query {
11
replace foo REDACTED
12
delete bar
13
hash baz
14
}
13
- request>headers>Authorization replace REDACTED
14
- request>headers>Server delete
15
- request>headers>Cookie cookie {
16
- replace foo REDACTED
17
- delete bar
18
- hash baz
19
- }
20
- request>remote_ip ip_mask {
21
- ipv4 24
22
- ipv6 32
23
- }
24
- request>client_ip ip_mask 16 32
25
- request>headers>Regexp regexp secret REDACTED
26
- request>headers>Hash hash
15
}
16
+
17
+ # short form, flatter structure
18
+ request>headers>Authorization replace REDACTED
19
+ request>headers>Server delete
20
+ request>headers>Cookie cookie {
21
+ replace foo REDACTED
22
+ delete bar
23
+ hash baz
24
+ }
25
+ request>remote_ip ip_mask {
26
+ ipv4 24
27
+ ipv6 32
28
+ }
29
+ request>client_ip ip_mask 16 32
30
+ request>headers>Regexp regexp secret REDACTED
31
+ request>headers>Hash hash
32
}
33
}
34
----------
modules/logging/appendencoder.go
new
+357
@@ -0,0 +1,357 @@
1
+// Copyright 2015 Matthew Holt and The Caddy Authors
2
+//
3
+// Licensed under the Apache License, Version 2.0 (the "License");
4
+// you may not use this file except in compliance with the License.
5
+// You may obtain a copy of the License at
6
+//
7
+// http://www.apache.org/licenses/LICENSE-2.0
8
+//
9
+// Unless required by applicable law or agreed to in writing, software
10
+// distributed under the License is distributed on an "AS IS" BASIS,
11
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+// See the License for the specific language governing permissions and
13
+// limitations under the License.
14
+
15
+package logging
16
+
17
+import (
18
+ "encoding/json"
19
+ "fmt"
20
+ "os"
21
+ "strings"
22
+ "time"
23
+
24
+ "go.uber.org/zap"
25
+ "go.uber.org/zap/buffer"
26
+ "go.uber.org/zap/zapcore"
27
+ "golang.org/x/term"
28
+
29
+ "github.com/caddyserver/caddy/v2"
30
+ "github.com/caddyserver/caddy/v2/caddyconfig"
31
+ "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
32
+)
33
+
34
+func init() {
35
+ caddy.RegisterModule(AppendEncoder{})
36
+}
37
+
38
+// AppendEncoder can be used to add fields to all log entries
39
+// that pass through it. It is a wrapper around another
40
+// encoder, which it uses to actually encode the log entries.
41
+// It is most useful for adding information about the Caddy
42
+// instance that is producing the log entries, possibly via
43
+// an environment variable.
44
+type AppendEncoder struct {
45
+ // The underlying encoder that actually encodes the
46
+ // log entries. If not specified, defaults to "json",
47
+ // unless the output is a terminal, in which case
48
+ // it defaults to "console".
49
+ WrappedRaw json.RawMessage `json:"wrap,omitempty" caddy:"namespace=caddy.logging.encoders inline_key=format"`
50
+
51
+ // A map of field names to their values. The values
52
+ // can be global placeholders (e.g. env vars), or constants.
53
+ // Note that the encoder does not run as part of an HTTP
54
+ // request context, so request placeholders are not available.
55
+ Fields map[string]any `json:"fields,omitempty"`
56
+
57
+ wrapped zapcore.Encoder
58
+ repl *caddy.Replacer
59
+
60
+ wrappedIsDefault bool
61
+ ctx caddy.Context
62
+}
63
+
64
+// CaddyModule returns the Caddy module information.
65
+func (AppendEncoder) CaddyModule() caddy.ModuleInfo {
66
+ return caddy.ModuleInfo{
67
+ ID: "caddy.logging.encoders.append",
68
+ New: func() caddy.Module { return new(AppendEncoder) },
69
+ }
70
+}
71
+
72
+// Provision sets up the encoder.
73
+func (fe *AppendEncoder) Provision(ctx caddy.Context) error {
74
+ fe.ctx = ctx
75
+ fe.repl = caddy.NewReplacer()
76
+
77
+ if fe.WrappedRaw == nil {
78
+ // if wrap is not specified, default to JSON
79
+ fe.wrapped = &JSONEncoder{}
80
+ if p, ok := fe.wrapped.(caddy.Provisioner); ok {
81
+ if err := p.Provision(ctx); err != nil {
82
+ return fmt.Errorf("provisioning fallback encoder module: %v", err)
83
+ }
84
+ }
85
+ fe.wrappedIsDefault = true
86
+ } else {
87
+ // set up wrapped encoder
88
+ val, err := ctx.LoadModule(fe, "WrappedRaw")
89
+ if err != nil {
90
+ return fmt.Errorf("loading fallback encoder module: %v", err)
91
+ }
92
+ fe.wrapped = val.(zapcore.Encoder)
93
+ }
94
+
95
+ return nil
96
+}
97
+
98
+// ConfigureDefaultFormat will set the default format to "console"
99
+// if the writer is a terminal. If already configured, it passes
100
+// through the writer so a deeply nested encoder can configure
101
+// its own default format.
102
+func (fe *AppendEncoder) ConfigureDefaultFormat(wo caddy.WriterOpener) error {
103
+ if !fe.wrappedIsDefault {
104
+ if cfd, ok := fe.wrapped.(caddy.ConfiguresFormatterDefault); ok {
105
+ return cfd.ConfigureDefaultFormat(wo)
106
+ }
107
+ return nil
108
+ }
109
+
110
+ if caddy.IsWriterStandardStream(wo) && term.IsTerminal(int(os.Stderr.Fd())) {
111
+ fe.wrapped = &ConsoleEncoder{}
112
+ if p, ok := fe.wrapped.(caddy.Provisioner); ok {
113
+ if err := p.Provision(fe.ctx); err != nil {
114
+ return fmt.Errorf("provisioning fallback encoder module: %v", err)
115
+ }
116
+ }
117
+ }
118
+ return nil
119
+}
120
+
121
+// UnmarshalCaddyfile sets up the module from Caddyfile tokens. Syntax:
122
+//
123
+// append {
124
+// wrap <another encoder>
125
+// fields {
126
+// <field> <value>
127
+// }
128
+// <field> <value>
129
+// }
130
+func (fe *AppendEncoder) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
131
+ d.Next() // consume encoder name
132
+
133
+ // parse a field
134
+ parseField := func() error {
135
+ if fe.Fields == nil {
136
+ fe.Fields = make(map[string]any)
137
+ }
138
+ field := d.Val()
139
+ if !d.NextArg() {
140
+ return d.ArgErr()
141
+ }
142
+ fe.Fields[field] = d.ScalarVal()
143
+ if d.NextArg() {
144
+ return d.ArgErr()
145
+ }
146
+ return nil
147
+ }
148
+
149
+ for d.NextBlock(0) {
150
+ switch d.Val() {
151
+ case "wrap":
152
+ if !d.NextArg() {
153
+ return d.ArgErr()
154
+ }
155
+ moduleName := d.Val()
156
+ moduleID := "caddy.logging.encoders." + moduleName
157
+ unm, err := caddyfile.UnmarshalModule(d, moduleID)
158
+ if err != nil {
159
+ return err
160
+ }
161
+ enc, ok := unm.(zapcore.Encoder)
162
+ if !ok {
163
+ return d.Errf("module %s (%T) is not a zapcore.Encoder", moduleID, unm)
164
+ }
165
+ fe.WrappedRaw = caddyconfig.JSONModuleObject(enc, "format", moduleName, nil)
166
+
167
+ case "fields":
168
+ for nesting := d.Nesting(); d.NextBlock(nesting); {
169
+ err := parseField()
170
+ if err != nil {
171
+ return err
172
+ }
173
+ }
174
+
175
+ default:
176
+ // if unknown, assume it's a field so that
177
+ // the config can be flat
178
+ err := parseField()
179
+ if err != nil {
180
+ return err
181
+ }
182
+ }
183
+ }
184
+ return nil
185
+}
186
+
187
+// AddArray is part of the zapcore.ObjectEncoder interface.
188
+func (fe AppendEncoder) AddArray(key string, marshaler zapcore.ArrayMarshaler) error {
189
+ return fe.wrapped.AddArray(key, marshaler)
190
+}
191
+
192
+// AddObject is part of the zapcore.ObjectEncoder interface.
193
+func (fe AppendEncoder) AddObject(key string, marshaler zapcore.ObjectMarshaler) error {
194
+ return fe.wrapped.AddObject(key, marshaler)
195
+}
196
+
197
+// AddBinary is part of the zapcore.ObjectEncoder interface.
198
+func (fe AppendEncoder) AddBinary(key string, value []byte) {
199
+ fe.wrapped.AddBinary(key, value)
200
+}
201
+
202
+// AddByteString is part of the zapcore.ObjectEncoder interface.
203
+func (fe AppendEncoder) AddByteString(key string, value []byte) {
204
+ fe.wrapped.AddByteString(key, value)
205
+}
206
+
207
+// AddBool is part of the zapcore.ObjectEncoder interface.
208
+func (fe AppendEncoder) AddBool(key string, value bool) {
209
+ fe.wrapped.AddBool(key, value)
210
+}
211
+
212
+// AddComplex128 is part of the zapcore.ObjectEncoder interface.
213
+func (fe AppendEncoder) AddComplex128(key string, value complex128) {
214
+ fe.wrapped.AddComplex128(key, value)
215
+}
216
+
217
+// AddComplex64 is part of the zapcore.ObjectEncoder interface.
218
+func (fe AppendEncoder) AddComplex64(key string, value complex64) {
219
+ fe.wrapped.AddComplex64(key, value)
220
+}
221
+
222
+// AddDuration is part of the zapcore.ObjectEncoder interface.
223
+func (fe AppendEncoder) AddDuration(key string, value time.Duration) {
224
+ fe.wrapped.AddDuration(key, value)
225
+}
226
+
227
+// AddFloat64 is part of the zapcore.ObjectEncoder interface.
228
+func (fe AppendEncoder) AddFloat64(key string, value float64) {
229
+ fe.wrapped.AddFloat64(key, value)
230
+}
231
+
232
+// AddFloat32 is part of the zapcore.ObjectEncoder interface.
233
+func (fe AppendEncoder) AddFloat32(key string, value float32) {
234
+ fe.wrapped.AddFloat32(key, value)
235
+}
236
+
237
+// AddInt is part of the zapcore.ObjectEncoder interface.
238
+func (fe AppendEncoder) AddInt(key string, value int) {
239
+ fe.wrapped.AddInt(key, value)
240
+}
241
+
242
+// AddInt64 is part of the zapcore.ObjectEncoder interface.
243
+func (fe AppendEncoder) AddInt64(key string, value int64) {
244
+ fe.wrapped.AddInt64(key, value)
245
+}
246
+
247
+// AddInt32 is part of the zapcore.ObjectEncoder interface.
248
+func (fe AppendEncoder) AddInt32(key string, value int32) {
249
+ fe.wrapped.AddInt32(key, value)
250
+}
251
+
252
+// AddInt16 is part of the zapcore.ObjectEncoder interface.
253
+func (fe AppendEncoder) AddInt16(key string, value int16) {
254
+ fe.wrapped.AddInt16(key, value)
255
+}
256
+
257
+// AddInt8 is part of the zapcore.ObjectEncoder interface.
258
+func (fe AppendEncoder) AddInt8(key string, value int8) {
259
+ fe.wrapped.AddInt8(key, value)
260
+}
261
+
262
+// AddString is part of the zapcore.ObjectEncoder interface.
263
+func (fe AppendEncoder) AddString(key, value string) {
264
+ fe.wrapped.AddString(key, value)
265
+}
266
+
267
+// AddTime is part of the zapcore.ObjectEncoder interface.
268
+func (fe AppendEncoder) AddTime(key string, value time.Time) {
269
+ fe.wrapped.AddTime(key, value)
270
+}
271
+
272
+// AddUint is part of the zapcore.ObjectEncoder interface.
273
+func (fe AppendEncoder) AddUint(key string, value uint) {
274
+ fe.wrapped.AddUint(key, value)
275
+}
276
+
277
+// AddUint64 is part of the zapcore.ObjectEncoder interface.
278
+func (fe AppendEncoder) AddUint64(key string, value uint64) {
279
+ fe.wrapped.AddUint64(key, value)
280
+}
281
+
282
+// AddUint32 is part of the zapcore.ObjectEncoder interface.
283
+func (fe AppendEncoder) AddUint32(key string, value uint32) {
284
+ fe.wrapped.AddUint32(key, value)
285
+}
286
+
287
+// AddUint16 is part of the zapcore.ObjectEncoder interface.
288
+func (fe AppendEncoder) AddUint16(key string, value uint16) {
289
+ fe.wrapped.AddUint16(key, value)
290
+}
291
+
292
+// AddUint8 is part of the zapcore.ObjectEncoder interface.
293
+func (fe AppendEncoder) AddUint8(key string, value uint8) {
294
+ fe.wrapped.AddUint8(key, value)
295
+}
296
+
297
+// AddUintptr is part of the zapcore.ObjectEncoder interface.
298
+func (fe AppendEncoder) AddUintptr(key string, value uintptr) {
299
+ fe.wrapped.AddUintptr(key, value)
300
+}
301
+
302
+// AddReflected is part of the zapcore.ObjectEncoder interface.
303
+func (fe AppendEncoder) AddReflected(key string, value any) error {
304
+ return fe.wrapped.AddReflected(key, value)
305
+}
306
+
307
+// OpenNamespace is part of the zapcore.ObjectEncoder interface.
308
+func (fe AppendEncoder) OpenNamespace(key string) {
309
+ fe.wrapped.OpenNamespace(key)
310
+}
311
+
312
+// Clone is part of the zapcore.ObjectEncoder interface.
313
+func (fe AppendEncoder) Clone() zapcore.Encoder {
314
+ return AppendEncoder{
315
+ Fields: fe.Fields,
316
+ wrapped: fe.wrapped.Clone(),
317
+ repl: fe.repl,
318
+ }
319
+}
320
+
321
+// EncodeEntry partially implements the zapcore.Encoder interface.
322
+func (fe AppendEncoder) EncodeEntry(ent zapcore.Entry, fields []zapcore.Field) (*buffer.Buffer, error) {
323
+ fe.wrapped = fe.wrapped.Clone()
324
+ for _, field := range fields {
325
+ field.AddTo(fe)
326
+ }
327
+
328
+ // append fields from config
329
+ for key, value := range fe.Fields {
330
+ // if the value is a string
331
+ if str, ok := value.(string); ok {
332
+ isPlaceholder := strings.HasPrefix(str, "{") &&
333
+ strings.HasSuffix(str, "}") &&
334
+ strings.Count(str, "{") == 1
335
+ if isPlaceholder {
336
+ // and it looks like a placeholder, evaluate it
337
+ replaced, _ := fe.repl.Get(strings.Trim(str, "{}"))
338
+ zap.Any(key, replaced).AddTo(fe)
339
+ } else {
340
+ // just use the string as-is
341
+ zap.String(key, str).AddTo(fe)
342
+ }
343
+ } else {
344
+ // not a string, so use the value as any
345
+ zap.Any(key, value).AddTo(fe)
346
+ }
347
+ }
348
+
349
+ return fe.wrapped.EncodeEntry(ent, nil)
350
+}
351
+
352
+// Interface guards
353
+var (
354
+ _ zapcore.Encoder = (*AppendEncoder)(nil)
355
+ _ caddyfile.Unmarshaler = (*AppendEncoder)(nil)
356
+ _ caddy.ConfiguresFormatterDefault = (*AppendEncoder)(nil)
357
+)
modules/logging/filterencoder.go
+34
-16
@@ -145,9 +145,36 @@ func (fe *FilterEncoder) ConfigureDefaultFormat(wo caddy.WriterOpener) error {
145
// <filter options>
146
// }
147
// }
148
+// <field> <filter> {
149
+// <filter options>
150
+// }
151
// }
152
func (fe *FilterEncoder) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
153
d.Next() // consume encoder name
154
+
155
+ // parse a field
156
+ parseField := func() error {
157
+ if fe.FieldsRaw == nil {
158
+ fe.FieldsRaw = make(map[string]json.RawMessage)
159
+ }
160
+ field := d.Val()
161
+ if !d.NextArg() {
162
+ return d.ArgErr()
163
+ }
164
+ filterName := d.Val()
165
+ moduleID := "caddy.logging.encoders.filter." + filterName
166
+ unm, err := caddyfile.UnmarshalModule(d, moduleID)
167
+ if err != nil {
168
+ return err
169
+ }
170
+ filter, ok := unm.(LogFieldFilter)
171
+ if !ok {
172
+ return d.Errf("module %s (%T) is not a logging.LogFieldFilter", moduleID, unm)
173
+ }
174
+ fe.FieldsRaw[field] = caddyconfig.JSONModuleObject(filter, "filter", filterName, nil)
175
+ return nil
176
+ }
177
+
178
for d.NextBlock(0) {
179
switch d.Val() {
180
case "wrap":
@@ -168,28 +195,19 @@ func (fe *FilterEncoder) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
195
196
case "fields":
197
for nesting := d.Nesting(); d.NextBlock(nesting); {
171
- field := d.Val()
172
- if !d.NextArg() {
173
- return d.ArgErr()
174
- }
175
- filterName := d.Val()
176
- moduleID := "caddy.logging.encoders.filter." + filterName
177
- unm, err := caddyfile.UnmarshalModule(d, moduleID)
198
+ err := parseField()
199
if err != nil {
200
return err
201
}
181
- filter, ok := unm.(LogFieldFilter)
182
- if !ok {
183
- return d.Errf("module %s (%T) is not a logging.LogFieldFilter", moduleID, unm)
184
- }
185
- if fe.FieldsRaw == nil {
186
- fe.FieldsRaw = make(map[string]json.RawMessage)
187
- }
188
- fe.FieldsRaw[field] = caddyconfig.JSONModuleObject(filter, "filter", filterName, nil)
202
}
203
204
default:
192
- return d.Errf("unrecognized subdirective %s", d.Val())
205
+ // if unknown, assume it's a field so that
206
+ // the config can be flat
207
+ err := parseField()
208
+ if err != nil {
209
+ return err
210
+ }
211
}
212
}
213
return nil