go.d squidlog: improve parser init and parsing (#18368)
Ilya Mashchenko committed
Aug 19, 2024 at 11:44 UTC
6fafa32b09e45248fc7c0ac60dc722575d055224
3 files changed
+46
-19
src/go/plugin/go.d/modules/squidlog/init.go
+34
-15
@@ -3,7 +3,6 @@
3
package squidlog
4
5
import (
6
- "bytes"
6
"fmt"
7
"strings"
8
@@ -26,28 +25,48 @@ func (s *SquidLog) createLogReader() error {
25
26
func (s *SquidLog) createParser() error {
27
s.Debug("starting parser creating")
29
- lastLine, err := logs.ReadLastLine(s.file.CurrentFilename(), 0)
30
- if err != nil {
31
- return fmt.Errorf("read last line: %v", err)
32
- }
28
34
- lastLine = bytes.TrimRight(lastLine, "\n")
35
- s.Debugf("last line: '%s'", string(lastLine))
29
+ const readLastLinesNum = 100
30
37
- s.parser, err = logs.NewParser(s.ParserConfig, s.file)
31
+ lines, err := logs.ReadLastLines(s.file.CurrentFilename(), readLastLinesNum)
32
if err != nil {
39
- return fmt.Errorf("create parser: %v", err)
33
+ return fmt.Errorf("failed to read last lines: %v", err)
34
}
41
- s.Debugf("created parser: %s", s.parser.Info())
35
43
- err = s.parser.Parse(lastLine, s.line)
44
- if err != nil {
45
- return fmt.Errorf("parse last line: %v (%s)", err, string(lastLine))
36
+ var found bool
37
+ for _, line := range lines {
38
+ if line = strings.TrimSpace(line); line == "" {
39
+ continue
40
+ }
41
+
42
+ s.Debugf("last line: '%s'", line)
43
+
44
+ s.parser, err = logs.NewParser(s.ParserConfig, s.file)
45
+ if err != nil {
46
+ s.Debugf("failed to create parser from line: %v", err)
47
+ continue
48
+ }
49
+
50
+ s.line.reset()
51
+
52
+ if err = s.parser.Parse([]byte(line), s.line); err != nil {
53
+ s.Debugf("failed to parse line: %v", err)
54
+ continue
55
+ }
56
+
57
+ if err = s.line.verify(); err != nil {
58
+ s.Debugf("failed to verify line: %v", err)
59
+ continue
60
+ }
61
+
62
+ found = true
63
+ break
64
}
65
48
- if err = s.line.verify(); err != nil {
49
- return fmt.Errorf("verify last line: %v (%s)", err, string(lastLine))
66
+ if !found {
67
+ return fmt.Errorf("failed to create log parser (file '%s')", s.file.CurrentFilename())
68
}
69
+
70
return nil
71
}
72
src/go/plugin/go.d/modules/squidlog/logline.go
+8
-2
@@ -245,10 +245,16 @@ func (l *logLine) assignMimeType(mime string) error {
245
}
246
// format: type/subtype, type/subtype;parameter=value
247
i := strings.IndexByte(mime, '/')
248
- if i <= 0 || !isMimeTypeValid(mime[:i]) {
248
+ if i <= 0 {
249
return fmt.Errorf("assign '%s': %w", mime, errBadMimeType)
250
}
251
+
252
+ if !isMimeTypeValid(mime[:i]) {
253
+ return nil
254
+ }
255
+
256
l.mimeType = mime[:i] // drop subtype
257
+
258
return nil
259
}
260
@@ -345,7 +351,7 @@ func isRespTimeValid(time int) bool {
351
// isCacheCodeValid does not guarantee cache result code is valid, but it is very likely.
352
func isCacheCodeValid(code string) bool {
353
// https://wiki.squid-cache.org/SquidFaq/SquidLogs#Squid_result_codes
348
- if code == "NONE" {
354
+ if code == "NONE" || code == "NONE_NONE" {
355
return true
356
}
357
return len(code) > 5 && (code[:4] == "TCP_" || code[:4] == "UDP_")
src/go/plugin/go.d/modules/squidlog/logline_test.go
+4
-2
@@ -60,6 +60,7 @@ func TestLogLine_Assign(t *testing.T) {
60
{input: "UDP_MISS_NOFETCH", wantLine: logLine{cacheCode: "UDP_MISS_NOFETCH"}},
61
{input: "UDP_INVALID", wantLine: logLine{cacheCode: "UDP_INVALID"}},
62
{input: "NONE", wantLine: logLine{cacheCode: "NONE"}},
63
+ {input: "NONE_NONE", wantLine: logLine{cacheCode: "NONE_NONE"}},
64
{input: emptyStr, wantLine: emptyLogLine},
65
{input: hyphen, wantLine: emptyLogLine, wantErr: errBadCacheCode},
66
{input: "TCP", wantLine: emptyLogLine, wantErr: errBadCacheCode},
@@ -173,8 +174,8 @@ func TestLogLine_Assign(t *testing.T) {
174
{input: "video/3gpp", wantLine: logLine{mimeType: "video"}},
175
{input: emptyStr, wantLine: emptyLogLine},
176
{input: hyphen, wantLine: emptyLogLine},
176
- {input: "example/example", wantLine: emptyLogLine, wantErr: errBadMimeType},
177
- {input: "unknown/example", wantLine: emptyLogLine, wantErr: errBadMimeType},
177
+ {input: "example/example", wantLine: emptyLogLine},
178
+ {input: "unknown/example", wantLine: emptyLogLine},
179
{input: "audio", wantLine: emptyLogLine, wantErr: errBadMimeType},
180
{input: "/", wantLine: emptyLogLine, wantErr: errBadMimeType},
181
},
@@ -274,6 +275,7 @@ func TestLogLine_verify(t *testing.T) {
275
{input: "UDP_MISS_NOFETCH"},
276
{input: "UDP_INVALID"},
277
{input: "NONE"},
278
+ {input: "NONE_NONE"},
279
{input: emptyStr},
280
{input: "TCP", wantErr: errBadCacheCode},
281
{input: "UDP", wantErr: errBadCacheCode},