| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | package squidlog |
| 4 | |
| 5 | import ( |
| 6 | "bytes" |
| 7 | "context" |
| 8 | "os" |
| 9 | "testing" |
| 10 | |
| 11 | "github.com/netdata/netdata/go/plugins/plugin/framework/collectorapi" |
| 12 | "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/collecttest" |
| 13 | "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/logs" |
| 14 | "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/oldmetrix" |
| 15 | "github.com/stretchr/testify/assert" |
| 16 | "github.com/stretchr/testify/require" |
| 17 | ) |
| 18 | |
| 19 | var ( |
| 20 | dataConfigJSON, _ = os.ReadFile("testdata/config.json") |
| 21 | dataConfigYAML, _ = os.ReadFile("testdata/config.yaml") |
| 22 | |
| 23 | dataNativeFormatAccessLog, _ = os.ReadFile("testdata/access.log") |
| 24 | ) |
| 25 | |
| 26 | func Test_testDataIsValid(t *testing.T) { |
| 27 | for name, data := range map[string][]byte{ |
| 28 | "dataConfigJSON": dataConfigJSON, |
| 29 | "dataConfigYAML": dataConfigYAML, |
| 30 | "dataNativeFormatAccessLog": dataNativeFormatAccessLog, |
| 31 | } { |
| 32 | require.NotNil(t, data, name) |
| 33 | } |
| 34 | } |
| 35 | |
| 36 | func TestCollector_ConfigurationSerialize(t *testing.T) { |
| 37 | collecttest.TestConfigurationSerialize(t, &Collector{}, dataConfigJSON, dataConfigYAML) |
| 38 | } |
| 39 | |
| 40 | func TestNew(t *testing.T) { |
| 41 | assert.Implements(t, (*collectorapi.CollectorV1)(nil), New()) |
| 42 | } |
| 43 | |
| 44 | func TestCollector_Init(t *testing.T) { |
| 45 | collr := New() |
| 46 | |
| 47 | assert.NoError(t, collr.Init(context.Background())) |
| 48 | } |
| 49 | |
| 50 | func TestCollector_Check(t *testing.T) { |
| 51 | } |
| 52 | |
| 53 | func TestCollector_Check_ErrorOnCreatingLogReaderNoLogFile(t *testing.T) { |
| 54 | collr := New() |
| 55 | defer collr.Cleanup(context.Background()) |
| 56 | collr.Path = "testdata/not_exists.log" |
| 57 | require.NoError(t, collr.Init(context.Background())) |
| 58 | |
| 59 | assert.Error(t, collr.Check(context.Background())) |
| 60 | } |
| 61 | |
| 62 | func TestSquid_Check_ErrorOnCreatingParserUnknownFormat(t *testing.T) { |
| 63 | collr := New() |
| 64 | defer collr.Cleanup(context.Background()) |
| 65 | collr.Path = "testdata/unknown.log" |
| 66 | require.NoError(t, collr.Init(context.Background())) |
| 67 | |
| 68 | assert.Error(t, collr.Check(context.Background())) |
| 69 | } |
| 70 | |
| 71 | func TestSquid_Check_ErrorOnCreatingParserZeroKnownFields(t *testing.T) { |
| 72 | collr := New() |
| 73 | defer collr.Cleanup(context.Background()) |
| 74 | collr.Path = "testdata/access.log" |
| 75 | collr.ParserConfig.CSV.Format = "$one $two" |
| 76 | require.NoError(t, collr.Init(context.Background())) |
| 77 | |
| 78 | assert.Error(t, collr.Check(context.Background())) |
| 79 | } |
| 80 | |
| 81 | func TestCollector_Charts(t *testing.T) { |
| 82 | assert.Nil(t, New().Charts()) |
| 83 | |
| 84 | collr := prepareSquidCollect(t) |
| 85 | assert.NotNil(t, collr.Charts()) |
| 86 | |
| 87 | } |
| 88 | |
| 89 | func TestCollector_Cleanup(t *testing.T) { |
| 90 | New().Cleanup(context.Background()) |
| 91 | } |
| 92 | |
| 93 | func TestCollector_Collect(t *testing.T) { |
| 94 | collr := prepareSquidCollect(t) |
| 95 | |
| 96 | expected := map[string]int64{ |
| 97 | "bytes_sent": 6827357, |
| 98 | "cache_error_tag_ABORTED": 326, |
| 99 | "cache_handling_tag_CF": 154, |
| 100 | "cache_handling_tag_CLIENT": 172, |
| 101 | "cache_load_source_tag_MEM": 172, |
| 102 | "cache_object_tag_NEGATIVE": 308, |
| 103 | "cache_object_tag_STALE": 172, |
| 104 | "cache_result_code_NONE": 158, |
| 105 | "cache_result_code_TCP_CF_NEGATIVE_NEGATIVE_ABORTED": 154, |
| 106 | "cache_result_code_UDP_CLIENT_STALE_MEM_ABORTED": 172, |
| 107 | "cache_transport_tag_NONE": 158, |
| 108 | "cache_transport_tag_TCP": 154, |
| 109 | "cache_transport_tag_UDP": 172, |
| 110 | "hier_code_HIER_CACHE_DIGEST_HIT": 128, |
| 111 | "hier_code_HIER_NO_CACHE_DIGEST_DIRECT": 130, |
| 112 | "hier_code_HIER_PARENT_HIT": 106, |
| 113 | "hier_code_HIER_SINGLE_PARENT": 120, |
| 114 | "http_resp_0xx": 51, |
| 115 | "http_resp_1xx": 45, |
| 116 | "http_resp_2xx": 62, |
| 117 | "http_resp_3xx": 120, |
| 118 | "http_resp_4xx": 112, |
| 119 | "http_resp_5xx": 46, |
| 120 | "http_resp_6xx": 48, |
| 121 | "http_resp_code_0": 51, |
| 122 | "http_resp_code_100": 45, |
| 123 | "http_resp_code_200": 62, |
| 124 | "http_resp_code_300": 58, |
| 125 | "http_resp_code_304": 62, |
| 126 | "http_resp_code_400": 56, |
| 127 | "http_resp_code_401": 56, |
| 128 | "http_resp_code_500": 46, |
| 129 | "http_resp_code_603": 48, |
| 130 | "mime_type_application": 52, |
| 131 | "mime_type_audio": 56, |
| 132 | "mime_type_font": 44, |
| 133 | "mime_type_image": 50, |
| 134 | "mime_type_message": 44, |
| 135 | "mime_type_model": 62, |
| 136 | "mime_type_multipart": 61, |
| 137 | "mime_type_text": 61, |
| 138 | "mime_type_video": 54, |
| 139 | "req_method_COPY": 84, |
| 140 | "req_method_GET": 70, |
| 141 | "req_method_HEAD": 59, |
| 142 | "req_method_OPTIONS": 99, |
| 143 | "req_method_POST": 74, |
| 144 | "req_method_PURGE": 98, |
| 145 | "req_type_bad": 56, |
| 146 | "req_type_error": 94, |
| 147 | "req_type_redirect": 58, |
| 148 | "req_type_success": 276, |
| 149 | "requests": 500, |
| 150 | "resp_time_avg": 3015931, |
| 151 | "resp_time_count": 484, |
| 152 | "resp_time_max": 4988000, |
| 153 | "resp_time_min": 1002000, |
| 154 | "resp_time_sum": 1459711000, |
| 155 | "server_address_2001:db8:2ce:a": 79, |
| 156 | "server_address_2001:db8:2ce:b": 89, |
| 157 | "server_address_203.0.113.100": 67, |
| 158 | "server_address_203.0.113.200": 70, |
| 159 | "server_address_content-gateway": 87, |
| 160 | "uniq_clients": 5, |
| 161 | "unmatched": 16, |
| 162 | } |
| 163 | |
| 164 | collected := collr.Collect(context.Background()) |
| 165 | |
| 166 | assert.Equal(t, expected, collected) |
| 167 | testCharts(t, collr, collected) |
| 168 | } |
| 169 | |
| 170 | func TestCollector_Collect_ReturnOldDataIfNothingRead(t *testing.T) { |
| 171 | collr := prepareSquidCollect(t) |
| 172 | |
| 173 | expected := map[string]int64{ |
| 174 | "bytes_sent": 6827357, |
| 175 | "cache_error_tag_ABORTED": 326, |
| 176 | "cache_handling_tag_CF": 154, |
| 177 | "cache_handling_tag_CLIENT": 172, |
| 178 | "cache_load_source_tag_MEM": 172, |
| 179 | "cache_object_tag_NEGATIVE": 308, |
| 180 | "cache_object_tag_STALE": 172, |
| 181 | "cache_result_code_NONE": 158, |
| 182 | "cache_result_code_TCP_CF_NEGATIVE_NEGATIVE_ABORTED": 154, |
| 183 | "cache_result_code_UDP_CLIENT_STALE_MEM_ABORTED": 172, |
| 184 | "cache_transport_tag_NONE": 158, |
| 185 | "cache_transport_tag_TCP": 154, |
| 186 | "cache_transport_tag_UDP": 172, |
| 187 | "hier_code_HIER_CACHE_DIGEST_HIT": 128, |
| 188 | "hier_code_HIER_NO_CACHE_DIGEST_DIRECT": 130, |
| 189 | "hier_code_HIER_PARENT_HIT": 106, |
| 190 | "hier_code_HIER_SINGLE_PARENT": 120, |
| 191 | "http_resp_0xx": 51, |
| 192 | "http_resp_1xx": 45, |
| 193 | "http_resp_2xx": 62, |
| 194 | "http_resp_3xx": 120, |
| 195 | "http_resp_4xx": 112, |
| 196 | "http_resp_5xx": 46, |
| 197 | "http_resp_6xx": 48, |
| 198 | "http_resp_code_0": 51, |
| 199 | "http_resp_code_100": 45, |
| 200 | "http_resp_code_200": 62, |
| 201 | "http_resp_code_300": 58, |
| 202 | "http_resp_code_304": 62, |
| 203 | "http_resp_code_400": 56, |
| 204 | "http_resp_code_401": 56, |
| 205 | "http_resp_code_500": 46, |
| 206 | "http_resp_code_603": 48, |
| 207 | "mime_type_application": 52, |
| 208 | "mime_type_audio": 56, |
| 209 | "mime_type_font": 44, |
| 210 | "mime_type_image": 50, |
| 211 | "mime_type_message": 44, |
| 212 | "mime_type_model": 62, |
| 213 | "mime_type_multipart": 61, |
| 214 | "mime_type_text": 61, |
| 215 | "mime_type_video": 54, |
| 216 | "req_method_COPY": 84, |
| 217 | "req_method_GET": 70, |
| 218 | "req_method_HEAD": 59, |
| 219 | "req_method_OPTIONS": 99, |
| 220 | "req_method_POST": 74, |
| 221 | "req_method_PURGE": 98, |
| 222 | "req_type_bad": 56, |
| 223 | "req_type_error": 94, |
| 224 | "req_type_redirect": 58, |
| 225 | "req_type_success": 276, |
| 226 | "requests": 500, |
| 227 | "resp_time_avg": 0, |
| 228 | "resp_time_count": 0, |
| 229 | "resp_time_max": 0, |
| 230 | "resp_time_min": 0, |
| 231 | "resp_time_sum": 0, |
| 232 | "server_address_2001:db8:2ce:a": 79, |
| 233 | "server_address_2001:db8:2ce:b": 89, |
| 234 | "server_address_203.0.113.100": 67, |
| 235 | "server_address_203.0.113.200": 70, |
| 236 | "server_address_content-gateway": 87, |
| 237 | "uniq_clients": 0, |
| 238 | "unmatched": 16, |
| 239 | } |
| 240 | |
| 241 | _ = collr.Collect(context.Background()) |
| 242 | |
| 243 | mx := collr.Collect(context.Background()) |
| 244 | |
| 245 | assert.Equal(t, expected, mx) |
| 246 | |
| 247 | testCharts(t, collr, mx) |
| 248 | } |
| 249 | |
| 250 | func testCharts(t *testing.T, collr *Collector, mx map[string]int64) { |
| 251 | t.Helper() |
| 252 | ensureChartsDynamicDimsCreated(t, collr) |
| 253 | collecttest.TestMetricsHasAllChartsDims(t, collr.Charts(), mx) |
| 254 | } |
| 255 | |
| 256 | func ensureChartsDynamicDimsCreated(t *testing.T, collr *Collector) { |
| 257 | ensureDynamicDimsCreated(t, collr, cacheCodeChart.ID, pxCacheCode, collr.mx.CacheCode) |
| 258 | ensureDynamicDimsCreated(t, collr, cacheCodeTransportTagChart.ID, pxTransportTag, collr.mx.CacheCodeTransportTag) |
| 259 | ensureDynamicDimsCreated(t, collr, cacheCodeHandlingTagChart.ID, pxHandlingTag, collr.mx.CacheCodeHandlingTag) |
| 260 | ensureDynamicDimsCreated(t, collr, cacheCodeObjectTagChart.ID, pxObjectTag, collr.mx.CacheCodeObjectTag) |
| 261 | ensureDynamicDimsCreated(t, collr, cacheCodeLoadSourceTagChart.ID, pxSourceTag, collr.mx.CacheCodeLoadSourceTag) |
| 262 | ensureDynamicDimsCreated(t, collr, cacheCodeErrorTagChart.ID, pxErrorTag, collr.mx.CacheCodeErrorTag) |
| 263 | ensureDynamicDimsCreated(t, collr, httpRespCodesChart.ID, pxHTTPCode, collr.mx.HTTPRespCode) |
| 264 | ensureDynamicDimsCreated(t, collr, reqMethodChart.ID, pxReqMethod, collr.mx.ReqMethod) |
| 265 | ensureDynamicDimsCreated(t, collr, hierCodeChart.ID, pxHierCode, collr.mx.HierCode) |
| 266 | ensureDynamicDimsCreated(t, collr, serverAddrChart.ID, pxSrvAddr, collr.mx.Server) |
| 267 | ensureDynamicDimsCreated(t, collr, mimeTypeChart.ID, pxMimeType, collr.mx.MimeType) |
| 268 | } |
| 269 | |
| 270 | func ensureDynamicDimsCreated(t *testing.T, collr *Collector, chartID, dimPrefix string, data oldmetrix.CounterVec) { |
| 271 | chart := collr.Charts().Get(chartID) |
| 272 | assert.NotNilf(t, chart, "chart '%s' is not created", chartID) |
| 273 | if chart == nil { |
| 274 | return |
| 275 | } |
| 276 | for v := range data { |
| 277 | id := dimPrefix + v |
| 278 | assert.Truef(t, chart.HasDim(id), "chart '%s' has no dim for '%s', expected '%s'", chart.ID, v, id) |
| 279 | } |
| 280 | } |
| 281 | |
| 282 | func prepareSquidCollect(t *testing.T) *Collector { |
| 283 | t.Helper() |
| 284 | collr := New() |
| 285 | collr.Path = "testdata/access.log" |
| 286 | require.NoError(t, collr.Init(context.Background())) |
| 287 | require.NoError(t, collr.Check(context.Background())) |
| 288 | defer collr.Cleanup(context.Background()) |
| 289 | |
| 290 | p, err := logs.NewCSVParser(collr.ParserConfig.CSV, bytes.NewReader(dataNativeFormatAccessLog)) |
| 291 | require.NoError(t, err) |
| 292 | collr.parser = p |
| 293 | return collr |
| 294 | } |
| 295 | |
| 296 | // generateLogs is used to populate 'testdata/access.log' |
| 297 | //func generateLogs(w io.Writer, num int) error { |
| 298 | // var ( |
| 299 | // client = []string{"localhost", "203.0.113.1", "203.0.113.2", "2001:db8:2ce:1", "2001:db8:2ce:2"} |
| 300 | // cacheCode = []string{"TCP_CF_NEGATIVE_NEGATIVE_ABORTED", "UDP_CLIENT_STALE_MEM_ABORTED", "NONE"} |
| 301 | // httpCode = []string{"000", "100", "200", "300", "304", "400", "401", "500", "603"} |
| 302 | // method = []string{"GET", "HEAD", "POST", "COPY", "PURGE", "OPTIONS"} |
| 303 | // hierCode = []string{"HIER_PARENT_HIT", "HIER_SINGLE_PARENT", "HIER_CACHE_DIGEST_HIT", "HIER_NO_CACHE_DIGEST_DIRECT"} |
| 304 | // server = []string{"content-gateway", "203.0.113.100", "203.0.113.200", "2001:db8:2ce:a", "2001:db8:2ce:b", "-"} |
| 305 | // mimeType = []string{"application", "audio", "font", "image", "message", "model", "multipart", "video", "text"} |
| 306 | // ) |
| 307 | // |
| 308 | // r := rand.New(rand.NewSource(time.Now().UnixNano())) |
| 309 | // randFromString := func(s []string) string { return s[r.Intn(len(s))] } |
| 310 | // randInt := func(min, max int) int { return r.Intn(max-min) + min } |
| 311 | // |
| 312 | // var line string |
| 313 | // for i := 0; i < num; i++ { |
| 314 | // unmatched := randInt(1, 100) > 95 |
| 315 | // if i > 0 && unmatched { |
| 316 | // line = "Unmatched! The rat the cat the dog chased killed ate the malt!\n" |
| 317 | // } else { |
| 318 | // // 1576177221.686 0 ::1 TCP_MISS/200 1621 GET cache_object://localhost/counters - HIER_NONE/- text/plain |
| 319 | // line = fmt.Sprintf( |
| 320 | // "1576177221.686 %d %s %s/%s %d %s cache_object://localhost/counters - %s/%s %s/plain\n", |
| 321 | // randInt(1000, 5000), |
| 322 | // randFromString(client), |
| 323 | // randFromString(cacheCode), |
| 324 | // randFromString(httpCode), |
| 325 | // randInt(9000, 19000), |
| 326 | // randFromString(method), |
| 327 | // randFromString(hierCode), |
| 328 | // randFromString(server), |
| 329 | // randFromString(mimeType), |
| 330 | // ) |
| 331 | // } |
| 332 | // _, err := fmt.Fprint(w, line) |
| 333 | // if err != nil { |
| 334 | // return err |
| 335 | // } |
| 336 | // } |
| 337 | // return nil |
| 338 | //} |