| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | package squidlog |
| 4 | |
| 5 | import ( |
| 6 | "io" |
| 7 | "runtime" |
| 8 | "strconv" |
| 9 | "strings" |
| 10 | |
| 11 | "github.com/netdata/netdata/go/plugins/pkg/stm" |
| 12 | "github.com/netdata/netdata/go/plugins/plugin/framework/collectorapi" |
| 13 | "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/logs" |
| 14 | ) |
| 15 | |
| 16 | func (c *Collector) logPanicStackIfAny() { |
| 17 | err := recover() |
| 18 | if err == nil { |
| 19 | return |
| 20 | } |
| 21 | c.Errorf("[ERROR] %s\n", err) |
| 22 | for depth := 0; ; depth++ { |
| 23 | _, file, line, ok := runtime.Caller(depth) |
| 24 | if !ok { |
| 25 | break |
| 26 | } |
| 27 | c.Errorf("======> %d: %v:%d", depth, file, line) |
| 28 | } |
| 29 | panic(err) |
| 30 | } |
| 31 | |
| 32 | func (c *Collector) collect() (map[string]int64, error) { |
| 33 | defer c.logPanicStackIfAny() |
| 34 | c.mx.reset() |
| 35 | |
| 36 | var mx map[string]int64 |
| 37 | |
| 38 | n, err := c.collectLogLines() |
| 39 | |
| 40 | if n > 0 || err == nil { |
| 41 | mx = stm.ToMap(c.mx) |
| 42 | } |
| 43 | return mx, err |
| 44 | } |
| 45 | |
| 46 | func (c *Collector) collectLogLines() (int, error) { |
| 47 | var n int |
| 48 | for { |
| 49 | c.line.reset() |
| 50 | err := c.parser.ReadLine(c.line) |
| 51 | if err != nil { |
| 52 | if err == io.EOF { |
| 53 | return n, nil |
| 54 | } |
| 55 | if !logs.IsParseError(err) { |
| 56 | return n, err |
| 57 | } |
| 58 | n++ |
| 59 | c.collectUnmatched() |
| 60 | continue |
| 61 | } |
| 62 | n++ |
| 63 | if c.line.empty() { |
| 64 | c.collectUnmatched() |
| 65 | } else { |
| 66 | c.collectLogLine() |
| 67 | } |
| 68 | } |
| 69 | } |
| 70 | |
| 71 | func (c *Collector) collectLogLine() { |
| 72 | c.mx.Requests.Inc() |
| 73 | c.collectRespTime() |
| 74 | c.collectClientAddress() |
| 75 | c.collectCacheCode() |
| 76 | c.collectHTTPCode() |
| 77 | c.collectRespSize() |
| 78 | c.collectReqMethod() |
| 79 | c.collectHierCode() |
| 80 | c.collectServerAddress() |
| 81 | c.collectMimeType() |
| 82 | } |
| 83 | |
| 84 | func (c *Collector) collectUnmatched() { |
| 85 | c.mx.Requests.Inc() |
| 86 | c.mx.Unmatched.Inc() |
| 87 | } |
| 88 | |
| 89 | func (c *Collector) collectRespTime() { |
| 90 | if !c.line.hasRespTime() { |
| 91 | return |
| 92 | } |
| 93 | c.mx.RespTime.Observe(float64(c.line.respTime)) |
| 94 | } |
| 95 | |
| 96 | func (c *Collector) collectClientAddress() { |
| 97 | if !c.line.hasClientAddress() { |
| 98 | return |
| 99 | } |
| 100 | c.mx.UniqueClients.Insert(c.line.clientAddr) |
| 101 | } |
| 102 | |
| 103 | func (c *Collector) collectCacheCode() { |
| 104 | if !c.line.hasCacheCode() { |
| 105 | return |
| 106 | } |
| 107 | |
| 108 | cntr, ok := c.mx.CacheCode.GetP(c.line.cacheCode) |
| 109 | if !ok { |
| 110 | c.addDimToCacheCodeChart(c.line.cacheCode) |
| 111 | } |
| 112 | cntr.Inc() |
| 113 | |
| 114 | tags := strings.SplitSeq(c.line.cacheCode, "_") |
| 115 | for tag := range tags { |
| 116 | c.collectCacheCodeTag(tag) |
| 117 | } |
| 118 | } |
| 119 | |
| 120 | func (c *Collector) collectHTTPCode() { |
| 121 | if !c.line.hasHTTPCode() { |
| 122 | return |
| 123 | } |
| 124 | |
| 125 | code := c.line.httpCode |
| 126 | switch { |
| 127 | case code >= 100 && code < 300, code == 0, code == 304, code == 401, code == 429: |
| 128 | c.mx.ReqSuccess.Inc() |
| 129 | case code >= 300 && code < 400: |
| 130 | c.mx.ReqRedirect.Inc() |
| 131 | case code >= 400 && code < 500: |
| 132 | c.mx.ReqBad.Inc() |
| 133 | case code >= 500 && code <= 603: |
| 134 | c.mx.ReqError.Inc() |
| 135 | } |
| 136 | |
| 137 | switch code / 100 { |
| 138 | case 0: |
| 139 | c.mx.HTTPResp0xx.Inc() |
| 140 | case 1: |
| 141 | c.mx.HTTPResp1xx.Inc() |
| 142 | case 2: |
| 143 | c.mx.HTTPResp2xx.Inc() |
| 144 | case 3: |
| 145 | c.mx.HTTPResp3xx.Inc() |
| 146 | case 4: |
| 147 | c.mx.HTTPResp4xx.Inc() |
| 148 | case 5: |
| 149 | c.mx.HTTPResp5xx.Inc() |
| 150 | case 6: |
| 151 | c.mx.HTTPResp6xx.Inc() |
| 152 | } |
| 153 | |
| 154 | codeStr := strconv.Itoa(code) |
| 155 | cntr, ok := c.mx.HTTPRespCode.GetP(codeStr) |
| 156 | if !ok { |
| 157 | c.addDimToHTTPRespCodesChart(codeStr) |
| 158 | } |
| 159 | cntr.Inc() |
| 160 | } |
| 161 | |
| 162 | func (c *Collector) collectRespSize() { |
| 163 | if !c.line.hasRespSize() { |
| 164 | return |
| 165 | } |
| 166 | c.mx.BytesSent.Add(float64(c.line.respSize)) |
| 167 | } |
| 168 | |
| 169 | func (c *Collector) collectReqMethod() { |
| 170 | if !c.line.hasReqMethod() { |
| 171 | return |
| 172 | } |
| 173 | cntr, ok := c.mx.ReqMethod.GetP(c.line.reqMethod) |
| 174 | if !ok { |
| 175 | c.addDimToReqMethodChart(c.line.reqMethod) |
| 176 | } |
| 177 | cntr.Inc() |
| 178 | } |
| 179 | |
| 180 | func (c *Collector) collectHierCode() { |
| 181 | if !c.line.hasHierCode() { |
| 182 | return |
| 183 | } |
| 184 | cntr, ok := c.mx.HierCode.GetP(c.line.hierCode) |
| 185 | if !ok { |
| 186 | c.addDimToHierCodeChart(c.line.hierCode) |
| 187 | } |
| 188 | cntr.Inc() |
| 189 | } |
| 190 | |
| 191 | func (c *Collector) collectServerAddress() { |
| 192 | if !c.line.hasServerAddress() { |
| 193 | return |
| 194 | } |
| 195 | cntr, ok := c.mx.Server.GetP(c.line.serverAddr) |
| 196 | if !ok { |
| 197 | c.addDimToServerAddressChart(c.line.serverAddr) |
| 198 | } |
| 199 | cntr.Inc() |
| 200 | } |
| 201 | |
| 202 | func (c *Collector) collectMimeType() { |
| 203 | if !c.line.hasMimeType() { |
| 204 | return |
| 205 | } |
| 206 | cntr, ok := c.mx.MimeType.GetP(c.line.mimeType) |
| 207 | if !ok { |
| 208 | c.addDimToMimeTypeChart(c.line.mimeType) |
| 209 | } |
| 210 | cntr.Inc() |
| 211 | } |
| 212 | |
| 213 | func (c *Collector) collectCacheCodeTag(tag string) { |
| 214 | // https://wiki.squid-cache.org/SquidFaq/SquidLogs#Squid_result_codes |
| 215 | switch tag { |
| 216 | default: |
| 217 | case "TCP", "UDP", "NONE": |
| 218 | cntr, ok := c.mx.CacheCodeTransportTag.GetP(tag) |
| 219 | if !ok { |
| 220 | c.addDimToCacheCodeTransportTagChart(tag) |
| 221 | } |
| 222 | cntr.Inc() |
| 223 | case "CF", "CLIENT", "IMS", "ASYNC", "SWAPFAIL", "REFRESH", "SHARED", "REPLY": |
| 224 | cntr, ok := c.mx.CacheCodeHandlingTag.GetP(tag) |
| 225 | if !ok { |
| 226 | c.addDimToCacheCodeHandlingTagChart(tag) |
| 227 | } |
| 228 | cntr.Inc() |
| 229 | case "NEGATIVE", "STALE", "OFFLINE", "INVALID", "FAIL", "MODIFIED", "UNMODIFIED", "REDIRECT": |
| 230 | cntr, ok := c.mx.CacheCodeObjectTag.GetP(tag) |
| 231 | if !ok { |
| 232 | c.addDimToCacheCodeObjectTagChart(tag) |
| 233 | } |
| 234 | cntr.Inc() |
| 235 | case "HIT", "MEM", "MISS", "DENIED", "NOFETCH", "TUNNEL": |
| 236 | cntr, ok := c.mx.CacheCodeLoadSourceTag.GetP(tag) |
| 237 | if !ok { |
| 238 | c.addDimToCacheCodeLoadSourceTagChart(tag) |
| 239 | } |
| 240 | cntr.Inc() |
| 241 | case "ABORTED", "TIMEOUT", "IGNORED": |
| 242 | cntr, ok := c.mx.CacheCodeErrorTag.GetP(tag) |
| 243 | if !ok { |
| 244 | c.addDimToCacheCodeErrorTagChart(tag) |
| 245 | } |
| 246 | cntr.Inc() |
| 247 | } |
| 248 | } |
| 249 | |
| 250 | func (c *Collector) addDimToCacheCodeChart(code string) { |
| 251 | chartID := cacheCodeChart.ID |
| 252 | dimID := pxCacheCode + code |
| 253 | c.addDimToChart(chartID, dimID, code) |
| 254 | } |
| 255 | |
| 256 | func (c *Collector) addDimToCacheCodeTransportTagChart(tag string) { |
| 257 | chartID := cacheCodeTransportTagChart.ID |
| 258 | dimID := pxTransportTag + tag |
| 259 | c.addDimToChart(chartID, dimID, tag) |
| 260 | } |
| 261 | |
| 262 | func (c *Collector) addDimToCacheCodeHandlingTagChart(tag string) { |
| 263 | chartID := cacheCodeHandlingTagChart.ID |
| 264 | dimID := pxHandlingTag + tag |
| 265 | c.addDimToChart(chartID, dimID, tag) |
| 266 | } |
| 267 | |
| 268 | func (c *Collector) addDimToCacheCodeObjectTagChart(tag string) { |
| 269 | chartID := cacheCodeObjectTagChart.ID |
| 270 | dimID := pxObjectTag + tag |
| 271 | c.addDimToChart(chartID, dimID, tag) |
| 272 | } |
| 273 | |
| 274 | func (c *Collector) addDimToCacheCodeLoadSourceTagChart(tag string) { |
| 275 | chartID := cacheCodeLoadSourceTagChart.ID |
| 276 | dimID := pxSourceTag + tag |
| 277 | c.addDimToChart(chartID, dimID, tag) |
| 278 | } |
| 279 | |
| 280 | func (c *Collector) addDimToCacheCodeErrorTagChart(tag string) { |
| 281 | chartID := cacheCodeErrorTagChart.ID |
| 282 | dimID := pxErrorTag + tag |
| 283 | c.addDimToChart(chartID, dimID, tag) |
| 284 | } |
| 285 | |
| 286 | func (c *Collector) addDimToHTTPRespCodesChart(tag string) { |
| 287 | chartID := httpRespCodesChart.ID |
| 288 | dimID := pxHTTPCode + tag |
| 289 | c.addDimToChart(chartID, dimID, tag) |
| 290 | } |
| 291 | |
| 292 | func (c *Collector) addDimToReqMethodChart(method string) { |
| 293 | chartID := reqMethodChart.ID |
| 294 | dimID := pxReqMethod + method |
| 295 | c.addDimToChart(chartID, dimID, method) |
| 296 | } |
| 297 | |
| 298 | func (c *Collector) addDimToHierCodeChart(code string) { |
| 299 | chartID := hierCodeChart.ID |
| 300 | dimID := pxHierCode + code |
| 301 | dimName := code[5:] // remove "HIER_" |
| 302 | c.addDimToChart(chartID, dimID, dimName) |
| 303 | } |
| 304 | |
| 305 | func (c *Collector) addDimToServerAddressChart(address string) { |
| 306 | chartID := serverAddrChart.ID |
| 307 | dimID := pxSrvAddr + address |
| 308 | c.addDimToChartOrCreateIfNotExist(chartID, dimID, address) |
| 309 | } |
| 310 | |
| 311 | func (c *Collector) addDimToMimeTypeChart(mimeType string) { |
| 312 | chartID := mimeTypeChart.ID |
| 313 | dimID := pxMimeType + mimeType |
| 314 | c.addDimToChartOrCreateIfNotExist(chartID, dimID, mimeType) |
| 315 | } |
| 316 | |
| 317 | func (c *Collector) addDimToChart(chartID, dimID, dimName string) { |
| 318 | chart := c.Charts().Get(chartID) |
| 319 | if chart == nil { |
| 320 | c.Warningf("add '%s' dim: couldn't find '%s' chart in charts", dimID, chartID) |
| 321 | return |
| 322 | } |
| 323 | |
| 324 | dim := &Dim{ID: dimID, Name: dimName, Algo: collectorapi.Incremental} |
| 325 | |
| 326 | if err := chart.AddDim(dim); err != nil { |
| 327 | c.Warningf("add '%s' dim: %v", dimID, err) |
| 328 | return |
| 329 | } |
| 330 | chart.MarkNotCreated() |
| 331 | } |
| 332 | |
| 333 | func (c *Collector) addDimToChartOrCreateIfNotExist(chartID, dimID, dimName string) { |
| 334 | if c.Charts().Has(chartID) { |
| 335 | c.addDimToChart(chartID, dimID, dimName) |
| 336 | return |
| 337 | } |
| 338 | |
| 339 | chart := newChartByID(chartID) |
| 340 | if chart == nil { |
| 341 | c.Warningf("add '%s' dim: couldn't create '%s' chart", dimID, chartID) |
| 342 | return |
| 343 | } |
| 344 | if err := c.Charts().Add(chart); err != nil { |
| 345 | c.Warning(err) |
| 346 | return |
| 347 | } |
| 348 | c.addDimToChart(chartID, dimID, dimName) |
| 349 | } |
| 350 | |
| 351 | func newChartByID(chartID string) *Chart { |
| 352 | switch chartID { |
| 353 | case serverAddrChart.ID: |
| 354 | return serverAddrChart.Copy() |
| 355 | case mimeTypeChart.ID: |
| 356 | return mimeTypeChart.Copy() |
| 357 | } |
| 358 | return nil |
| 359 | } |