| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | package pipeline |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "fmt" |
| 8 | "os" |
| 9 | "path/filepath" |
| 10 | "strings" |
| 11 | "testing" |
| 12 | "time" |
| 13 | |
| 14 | "github.com/netdata/netdata/go/plugins/plugin/agent/discovery/sd/model" |
| 15 | "github.com/netdata/netdata/go/plugins/plugin/framework/confgroup" |
| 16 | |
| 17 | "github.com/gohugoio/hashstructure" |
| 18 | "github.com/stretchr/testify/assert" |
| 19 | "github.com/stretchr/testify/require" |
| 20 | "gopkg.in/yaml.v2" |
| 21 | ) |
| 22 | |
| 23 | func Test_defaultConfigs(t *testing.T) { |
| 24 | dir := "../../../../go.d/config/go.d/sd/" |
| 25 | entries, err := os.ReadDir(dir) |
| 26 | require.NoError(t, err) |
| 27 | |
| 28 | require.NotEmpty(t, entries) |
| 29 | factory := func(_ DiscovererPayload, _ string) ([]model.Discoverer, error) { |
| 30 | return []model.Discoverer{newMockDiscoverer("", newMockTargetGroup("test"))}, nil |
| 31 | } |
| 32 | |
| 33 | for _, e := range entries { |
| 34 | if strings.Contains(e.Name(), "prometheus") { |
| 35 | continue |
| 36 | } |
| 37 | file, err := filepath.Abs(filepath.Join(dir, e.Name())) |
| 38 | require.NoError(t, err, "abs path") |
| 39 | |
| 40 | bs, err := os.ReadFile(file) |
| 41 | require.NoErrorf(t, err, "read config file '%s'", file) |
| 42 | |
| 43 | var cfg Config |
| 44 | require.NoErrorf(t, yaml.Unmarshal(bs, &cfg), "unmarshal '%s'", e.Name()) |
| 45 | cfg.Name = strings.TrimSuffix(e.Name(), filepath.Ext(e.Name())) |
| 46 | |
| 47 | _, err = New(cfg, factory) |
| 48 | require.NoErrorf(t, err, "create pipeline '%s'", e.Name()) |
| 49 | } |
| 50 | } |
| 51 | |
| 52 | func TestNew(t *testing.T) { |
| 53 | tests := map[string]struct { |
| 54 | config string |
| 55 | wantErr bool |
| 56 | }{ |
| 57 | "fails when config unset": { |
| 58 | wantErr: true, |
| 59 | config: "", |
| 60 | }, |
| 61 | } |
| 62 | |
| 63 | for name, test := range tests { |
| 64 | t.Run(name, func(t *testing.T) { |
| 65 | |
| 66 | var cfg Config |
| 67 | err := yaml.Unmarshal([]byte(test.config), &cfg) |
| 68 | require.Nilf(t, err, "cfg unmarshal") |
| 69 | |
| 70 | _, err = New(cfg, func(_ DiscovererPayload, _ string) ([]model.Discoverer, error) { |
| 71 | return []model.Discoverer{newMockDiscoverer("", newMockTargetGroup("test"))}, nil |
| 72 | }) |
| 73 | |
| 74 | if test.wantErr { |
| 75 | assert.Error(t, err) |
| 76 | } else { |
| 77 | assert.NoError(t, err) |
| 78 | } |
| 79 | }) |
| 80 | } |
| 81 | } |
| 82 | |
| 83 | func TestPipeline_Run(t *testing.T) { |
| 84 | const config = ` |
| 85 | classify: |
| 86 | - selector: "rule1" |
| 87 | tags: "foo1" |
| 88 | match: |
| 89 | - tags: "bar1" |
| 90 | expr: '{{ glob .Name "mock*1*" }}' |
| 91 | - tags: "bar2" |
| 92 | expr: '{{ glob .Name "mock*2*" }}' |
| 93 | compose: |
| 94 | - selector: "foo1" |
| 95 | config: |
| 96 | - selector: "bar1" |
| 97 | template: | |
| 98 | name: {{ .Name }}-foobar1 |
| 99 | - selector: "bar2" |
| 100 | template: | |
| 101 | name: {{ .Name }}-foobar2 |
| 102 | ` |
| 103 | |
| 104 | const servicesConfig = ` |
| 105 | services: |
| 106 | - id: "svc-foobar1" |
| 107 | match: '{{ glob .Name "mock*1*" }}' |
| 108 | config_template: | |
| 109 | name: {{ .Name }}-foobar1 |
| 110 | - id: "svc-foobar2" |
| 111 | match: '{{ glob .Name "mock*2*" }}' |
| 112 | config_template: | |
| 113 | name: {{ .Name }}-foobar2 |
| 114 | ` |
| 115 | tests := map[string]discoverySim{ |
| 116 | "new group with no targets": { |
| 117 | config: config, |
| 118 | discoverers: []model.Discoverer{ |
| 119 | newMockDiscoverer("", |
| 120 | newMockTargetGroup("test"), |
| 121 | ), |
| 122 | }, |
| 123 | wantClassifyCalls: 0, |
| 124 | wantComposeCalls: 0, |
| 125 | wantConfGroups: nil, |
| 126 | }, |
| 127 | // Note: Legacy classify/compose config is auto-converted to services format during unmarshal. |
| 128 | // The service rule IDs become the compose selectors (bar1, bar2), so module is set to those. |
| 129 | "new group with targets": { |
| 130 | config: config, |
| 131 | discoverers: []model.Discoverer{ |
| 132 | newMockDiscoverer("rule1", |
| 133 | newMockTargetGroup("test", "mock1", "mock2"), |
| 134 | ), |
| 135 | }, |
| 136 | wantClassifyCalls: 0, // services mode - no classify |
| 137 | wantComposeCalls: 2, |
| 138 | wantConfGroups: []*confgroup.Group{ |
| 139 | prepareDiscoveredGroupWithModule("mock1-foobar1", "bar1", "mock2-foobar2", "bar2"), |
| 140 | }, |
| 141 | }, |
| 142 | "existing group with same targets": { |
| 143 | config: config, |
| 144 | discoverers: []model.Discoverer{ |
| 145 | newMockDiscoverer("rule1", |
| 146 | newMockTargetGroup("test", "mock1", "mock2"), |
| 147 | ), |
| 148 | newDelayedMockDiscoverer("rule1", 5, |
| 149 | newMockTargetGroup("test", "mock1", "mock2"), |
| 150 | ), |
| 151 | }, |
| 152 | wantClassifyCalls: 0, // services mode - no classify |
| 153 | wantComposeCalls: 2, |
| 154 | wantConfGroups: []*confgroup.Group{ |
| 155 | prepareDiscoveredGroupWithModule("mock1-foobar1", "bar1", "mock2-foobar2", "bar2"), |
| 156 | }, |
| 157 | }, |
| 158 | "existing group that previously had targets with no targets": { |
| 159 | config: config, |
| 160 | discoverers: []model.Discoverer{ |
| 161 | newMockDiscoverer("rule1", |
| 162 | newMockTargetGroup("test", "mock1", "mock2"), |
| 163 | ), |
| 164 | newDelayedMockDiscoverer("rule1", 5, |
| 165 | newMockTargetGroup("test"), |
| 166 | ), |
| 167 | }, |
| 168 | wantClassifyCalls: 0, // services mode - no classify |
| 169 | wantComposeCalls: 2, |
| 170 | wantConfGroups: []*confgroup.Group{ |
| 171 | prepareDiscoveredGroupWithModule("mock1-foobar1", "bar1", "mock2-foobar2", "bar2"), |
| 172 | prepareDiscoveredGroup(), |
| 173 | }, |
| 174 | }, |
| 175 | "existing group with old and new targets": { |
| 176 | config: config, |
| 177 | discoverers: []model.Discoverer{ |
| 178 | newMockDiscoverer("rule1", |
| 179 | newMockTargetGroup("test", "mock1", "mock2"), |
| 180 | ), |
| 181 | newDelayedMockDiscoverer("rule1", 5, |
| 182 | newMockTargetGroup("test", "mock1", "mock2", "mock11", "mock22"), |
| 183 | ), |
| 184 | }, |
| 185 | wantClassifyCalls: 0, // services mode - no classify |
| 186 | wantComposeCalls: 4, |
| 187 | wantConfGroups: []*confgroup.Group{ |
| 188 | prepareDiscoveredGroupWithModule("mock1-foobar1", "bar1", "mock2-foobar2", "bar2"), |
| 189 | prepareDiscoveredGroupWithModule("mock1-foobar1", "bar1", "mock11-foobar1", "bar1", "mock2-foobar2", "bar2", "mock22-foobar2", "bar2"), |
| 190 | }, |
| 191 | }, |
| 192 | "existing group with new targets only": { |
| 193 | config: config, |
| 194 | discoverers: []model.Discoverer{ |
| 195 | newMockDiscoverer("rule1", |
| 196 | newMockTargetGroup("test", "mock1", "mock2"), |
| 197 | ), |
| 198 | newDelayedMockDiscoverer("rule1", 5, |
| 199 | newMockTargetGroup("test", "mock11", "mock22"), |
| 200 | ), |
| 201 | }, |
| 202 | wantClassifyCalls: 0, // services mode - no classify |
| 203 | wantComposeCalls: 4, |
| 204 | wantConfGroups: []*confgroup.Group{ |
| 205 | prepareDiscoveredGroupWithModule("mock1-foobar1", "bar1", "mock2-foobar2", "bar2"), |
| 206 | prepareDiscoveredGroupWithModule("mock11-foobar1", "bar1", "mock22-foobar2", "bar2"), |
| 207 | }, |
| 208 | }, |
| 209 | "services-only: new group with targets": { |
| 210 | config: servicesConfig, |
| 211 | discoverers: []model.Discoverer{ |
| 212 | newMockDiscoverer("rule1", |
| 213 | newMockTargetGroup("test", "mock1", "mock2"), |
| 214 | ), |
| 215 | }, |
| 216 | useServices: true, // tell the simulator to wire svr-only |
| 217 | wantClassifyCalls: 0, // no classify in services mode |
| 218 | wantComposeCalls: 2, // compose called per target (2 targets) |
| 219 | wantConfGroups: []*confgroup.Group{ |
| 220 | // same expected configs as the legacy "new group with targets" |
| 221 | prepareDiscoveredGroupWithModule("mock1-foobar1", "svc-foobar1", "mock2-foobar2", "svc-foobar2"), |
| 222 | }, |
| 223 | }, |
| 224 | } |
| 225 | |
| 226 | for name, sim := range tests { |
| 227 | t.Run(name, func(t *testing.T) { |
| 228 | sim.run(t) |
| 229 | }) |
| 230 | } |
| 231 | } |
| 232 | |
| 233 | func prepareDiscoveredGroup(configNames ...string) *confgroup.Group { |
| 234 | var configs []confgroup.Config |
| 235 | |
| 236 | for _, name := range configNames { |
| 237 | configs = append(configs, confgroup.Config{}. |
| 238 | SetProvider("mock"). |
| 239 | SetSourceType(confgroup.TypeDiscovered). |
| 240 | SetSource("test"). |
| 241 | SetName(name)) |
| 242 | } |
| 243 | |
| 244 | return &confgroup.Group{ |
| 245 | Source: "test", |
| 246 | Configs: configs, |
| 247 | } |
| 248 | } |
| 249 | |
| 250 | func prepareDiscoveredGroupWithModule(values ...string) *confgroup.Group { |
| 251 | var configs []confgroup.Config |
| 252 | |
| 253 | for i := 0; i < len(values); i += 2 { |
| 254 | cfgName := values[i] |
| 255 | modName := values[i+1] |
| 256 | configs = append(configs, confgroup.Config{}. |
| 257 | SetProvider("mock"). |
| 258 | SetSourceType(confgroup.TypeDiscovered). |
| 259 | SetSource("test"). |
| 260 | SetName(cfgName). |
| 261 | SetModule(modName), |
| 262 | ) |
| 263 | } |
| 264 | |
| 265 | return &confgroup.Group{ |
| 266 | Source: "test", |
| 267 | Configs: configs, |
| 268 | } |
| 269 | } |
| 270 | |
| 271 | func newMockDiscoverer(tags string, tggs ...model.TargetGroup) *mockDiscoverer { |
| 272 | return &mockDiscoverer{ |
| 273 | tags: mustParseTags(tags), |
| 274 | tggs: tggs, |
| 275 | } |
| 276 | } |
| 277 | |
| 278 | func newDelayedMockDiscoverer(tags string, delay int, tggs ...model.TargetGroup) *mockDiscoverer { |
| 279 | return &mockDiscoverer{ |
| 280 | tags: mustParseTags(tags), |
| 281 | tggs: tggs, |
| 282 | delay: time.Duration(delay) * time.Second, |
| 283 | } |
| 284 | } |
| 285 | |
| 286 | type mockDiscoverer struct { |
| 287 | tggs []model.TargetGroup |
| 288 | tags model.Tags |
| 289 | delay time.Duration |
| 290 | } |
| 291 | |
| 292 | func (md mockDiscoverer) String() string { |
| 293 | return "mock discoverer" |
| 294 | } |
| 295 | |
| 296 | func (md mockDiscoverer) Discover(ctx context.Context, out chan<- []model.TargetGroup) { |
| 297 | for _, tgg := range md.tggs { |
| 298 | for _, tgt := range tgg.Targets() { |
| 299 | tgt.Tags().Merge(md.tags) |
| 300 | } |
| 301 | } |
| 302 | |
| 303 | select { |
| 304 | case <-ctx.Done(): |
| 305 | case <-time.After(md.delay): |
| 306 | select { |
| 307 | case <-ctx.Done(): |
| 308 | case out <- md.tggs: |
| 309 | } |
| 310 | } |
| 311 | } |
| 312 | |
| 313 | func newMockTargetGroup(source string, targets ...string) *mockTargetGroup { |
| 314 | m := &mockTargetGroup{source: source} |
| 315 | for _, name := range targets { |
| 316 | m.targets = append(m.targets, &mockTarget{Name: name}) |
| 317 | } |
| 318 | return m |
| 319 | } |
| 320 | |
| 321 | type mockTargetGroup struct { |
| 322 | targets []model.Target |
| 323 | source string |
| 324 | } |
| 325 | |
| 326 | func (mg mockTargetGroup) Targets() []model.Target { return mg.targets } |
| 327 | func (mg mockTargetGroup) Source() string { return mg.source } |
| 328 | func (mg mockTargetGroup) Provider() string { return "mock" } |
| 329 | |
| 330 | func newMockTarget(name string, tags ...string) *mockTarget { |
| 331 | m := &mockTarget{Name: name} |
| 332 | v, _ := model.ParseTags(strings.Join(tags, " ")) |
| 333 | m.Tags().Merge(v) |
| 334 | return m |
| 335 | } |
| 336 | |
| 337 | type mockTarget struct { |
| 338 | model.Base |
| 339 | Name string |
| 340 | } |
| 341 | |
| 342 | func (mt mockTarget) TUID() string { return mt.Name } |
| 343 | func (mt mockTarget) Hash() uint64 { return mustCalcHash(mt.Name) } |
| 344 | |
| 345 | func mustParseTags(line string) model.Tags { |
| 346 | v, err := model.ParseTags(line) |
| 347 | if err != nil { |
| 348 | panic(fmt.Sprintf("mustParseTags: %v", err)) |
| 349 | } |
| 350 | return v |
| 351 | } |
| 352 | |
| 353 | func mustCalcHash(obj any) uint64 { |
| 354 | hash, err := hashstructure.Hash(obj, nil) |
| 355 | if err != nil { |
| 356 | panic(fmt.Sprintf("hash calculation: %v", err)) |
| 357 | } |
| 358 | return hash |
| 359 | } |
| 360 | |
| 361 | func TestConvertOldToServices(t *testing.T) { |
| 362 | type inYAML struct { |
| 363 | Classify string |
| 364 | Compose string |
| 365 | } |
| 366 | |
| 367 | tests := map[string]struct { |
| 368 | in inYAML |
| 369 | want []ServiceRuleConfig |
| 370 | }{ |
| 371 | "basic 1:1 mapping": { |
| 372 | in: inYAML{ |
| 373 | Classify: ` |
| 374 | - name: "Applications" |
| 375 | selector: "unknown" |
| 376 | tags: "-unknown app" |
| 377 | match: |
| 378 | - tags: "activemq" |
| 379 | expr: '{{ and (eq .Port "8161") (eq .Comm "activemq") }}' |
| 380 | `, |
| 381 | Compose: ` |
| 382 | - name: "Applications" |
| 383 | selector: "app" |
| 384 | config: |
| 385 | - selector: "activemq" |
| 386 | template: | |
| 387 | module: activemq |
| 388 | name: local |
| 389 | url: http://{{.Address}} |
| 390 | webadmin: admin |
| 391 | `, |
| 392 | }, |
| 393 | want: []ServiceRuleConfig{ |
| 394 | { |
| 395 | ID: "activemq", |
| 396 | Match: `{{ and (eq .Port "8161") (eq .Comm "activemq") }}`, |
| 397 | ConfigTemplate: "module: activemq\nname: local\nurl: http://{{.Address}}\nwebadmin: admin\n", |
| 398 | }, |
| 399 | }, |
| 400 | }, |
| 401 | |
| 402 | "multiple classify exprs for same tag -> multiple service rules": { |
| 403 | in: inYAML{ |
| 404 | Classify: ` |
| 405 | - name: "Databases" |
| 406 | selector: "unknown" |
| 407 | match: |
| 408 | - tags: "redis" |
| 409 | expr: '{{ eq .Port "6379" }}' |
| 410 | - tags: "redis" |
| 411 | expr: '{{ and (eq .Comm "redis-server") (eq .Address "127.0.0.1") }}' |
| 412 | `, |
| 413 | Compose: ` |
| 414 | - name: "Databases" |
| 415 | selector: "app" |
| 416 | config: |
| 417 | - selector: "redis" |
| 418 | template: | |
| 419 | module: redis |
| 420 | name: {{ .Name }} |
| 421 | `, |
| 422 | }, |
| 423 | // NOTE: Order should follow classify expr encounter order: |
| 424 | // 1) Port-based, 2) Comm+Address-based. IDs redis, redis_2 accordingly. |
| 425 | want: []ServiceRuleConfig{ |
| 426 | { |
| 427 | ID: "redis", |
| 428 | Match: `{{ eq .Port "6379" }}`, |
| 429 | ConfigTemplate: "module: redis\nname: {{ .Name }}\n", |
| 430 | }, |
| 431 | { |
| 432 | ID: "redis_2", |
| 433 | Match: `{{ and (eq .Comm "redis-server") (eq .Address "127.0.0.1") }}`, |
| 434 | ConfigTemplate: "module: redis\nname: {{ .Name }}\n", |
| 435 | }, |
| 436 | }, |
| 437 | }, |
| 438 | |
| 439 | "ignore deletions and rule-level tags without expr": { |
| 440 | in: inYAML{ |
| 441 | Classify: ` |
| 442 | - name: "NoExpr" |
| 443 | selector: "unknown" |
| 444 | tags: "nginx" # rule-level tag: no expr -> cannot produce a service rule |
| 445 | match: [] |
| 446 | - name: "Deletions" |
| 447 | selector: "unknown" |
| 448 | match: |
| 449 | - tags: "-nginx" # deletion: ignore |
| 450 | expr: '{{ eq .Port "80" }}' # has expr but tag is a deletion, ignore |
| 451 | `, |
| 452 | Compose: ` |
| 453 | - name: "Web" |
| 454 | selector: "app" |
| 455 | config: |
| 456 | - selector: "nginx" |
| 457 | template: | |
| 458 | module: nginx |
| 459 | name: web |
| 460 | `, |
| 461 | }, |
| 462 | want: nil, // nothing to map |
| 463 | }, |
| 464 | |
| 465 | "compose selector without classify producer -> empty": { |
| 466 | in: inYAML{ |
| 467 | Classify: ` |
| 468 | - name: "App" |
| 469 | selector: "unknown" |
| 470 | match: |
| 471 | - tags: "foo" |
| 472 | expr: '{{ eq .Port "1234" }}' |
| 473 | `, |
| 474 | Compose: ` |
| 475 | - name: "App" |
| 476 | selector: "app" |
| 477 | config: |
| 478 | - selector: "bar" |
| 479 | template: | |
| 480 | module: bar |
| 481 | `, |
| 482 | }, |
| 483 | want: nil, |
| 484 | }, |
| 485 | |
| 486 | "multiple compose selectors map to different classify tags": { |
| 487 | in: inYAML{ |
| 488 | Classify: ` |
| 489 | - name: "Mixed" |
| 490 | selector: "unknown" |
| 491 | match: |
| 492 | - tags: "kafka" |
| 493 | expr: '{{ eq .Port "9092" }}' |
| 494 | - tags: "zookeeper" |
| 495 | expr: '{{ eq .Port "2181" }}' |
| 496 | `, |
| 497 | Compose: ` |
| 498 | - name: "Stream" |
| 499 | selector: "app" |
| 500 | config: |
| 501 | - selector: "zookeeper" |
| 502 | template: | |
| 503 | module: zookeeper |
| 504 | name: zk |
| 505 | - selector: "kafka" |
| 506 | template: | |
| 507 | module: kafka |
| 508 | name: broker |
| 509 | `, |
| 510 | }, |
| 511 | // Order follows compose config order; for each selector, classify exprs order is preserved. |
| 512 | want: []ServiceRuleConfig{ |
| 513 | { |
| 514 | ID: "zookeeper", |
| 515 | Match: `{{ eq .Port "2181" }}`, |
| 516 | ConfigTemplate: "module: zookeeper\nname: zk\n", |
| 517 | }, |
| 518 | { |
| 519 | ID: "kafka", |
| 520 | Match: `{{ eq .Port "9092" }}`, |
| 521 | ConfigTemplate: "module: kafka\nname: broker\n", |
| 522 | }, |
| 523 | }, |
| 524 | }, |
| 525 | } |
| 526 | |
| 527 | for name, tc := range tests { |
| 528 | t.Run(name, func(t *testing.T) { |
| 529 | var cls []ClassifyRuleConfig |
| 530 | var cmp []ComposeRuleConfig |
| 531 | |
| 532 | require.NoError(t, yaml.Unmarshal([]byte(tc.in.Classify), &cls), "classify YAML") |
| 533 | require.NoError(t, yaml.Unmarshal([]byte(tc.in.Compose), &cmp), "compose YAML") |
| 534 | |
| 535 | got, err := ConvertOldToServices(cls, cmp) |
| 536 | require.NoError(t, err) |
| 537 | |
| 538 | assert.Equal(t, tc.want, got) |
| 539 | }) |
| 540 | } |
| 541 | } |