refactor(go.d/azure_monitor): redesign discovery and profile selection (#22095)
Ilya Mashchenko committed
Mar 31, 2026 at 14:18 UTC
2364d3b09bfdee95fb17114fbe2597dbdc2a4cd0
19 files changed
+3439
-772
src/go/plugin/go.d/collector/azure_monitor/azureprofiles/catalog.go
+157
-30
@@ -24,14 +24,16 @@ const (
24
)
25
26
type Catalog struct {
27
- byID map[string]Profile
28
- stockProfileIDs map[string]struct{}
27
+ byBaseName map[string]Profile
28
+ byID map[string]Profile
29
+ stockProfileBaseNames map[string]struct{}
30
}
31
32
type catalogEntry struct {
32
- Config Profile
33
- Path string
34
- IsStock bool
33
+ Config Profile
34
+ Path string
35
+ BaseName string
36
+ IsStock bool
37
}
38
39
type DirSpec struct {
@@ -49,7 +51,7 @@ func LoadFromDefaultDirs() (Catalog, error) {
51
if err != nil {
52
return Catalog{}, err
53
}
52
- if len(catalog.stockProfileIDs) == 0 {
54
+ if len(catalog.stockProfileBaseNames) == 0 {
55
return Catalog{}, fmt.Errorf("no stock profiles found under %q", filepath.Join(pluginconfig.CollectorsStockDir(), profilesDirName, "default"))
56
}
57
return catalog, nil
@@ -57,8 +59,9 @@ func LoadFromDefaultDirs() (Catalog, error) {
59
60
func LoadFromDirs(specs []DirSpec) (Catalog, error) {
61
catalog := Catalog{
60
- byID: make(map[string]Profile),
61
- stockProfileIDs: make(map[string]struct{}),
62
+ byBaseName: make(map[string]Profile),
63
+ byID: make(map[string]Profile),
64
+ stockProfileBaseNames: make(map[string]struct{}),
65
}
66
seen := make(map[string]catalogEntry)
67
@@ -95,17 +98,17 @@ func LoadFromDirs(specs []DirSpec) (Catalog, error) {
98
return err
99
}
100
98
- id := normalizeKey(cfg.ID)
99
- if id == "" {
100
- return fmt.Errorf("profile %q: decoded empty id", path)
101
+ baseName := normalizeKey(profileBaseName(path))
102
+ if baseName == "" {
103
+ return fmt.Errorf("profile %q: decoded empty basename", path)
104
}
105
if spec.IsStock {
103
- catalog.stockProfileIDs[id] = struct{}{}
106
+ catalog.stockProfileBaseNames[baseName] = struct{}{}
107
}
108
106
- prev, exists := seen[id]
109
+ prev, exists := seen[baseName]
110
if !exists {
108
- seen[id] = catalogEntry{Config: cfg, Path: path, IsStock: spec.IsStock}
111
+ seen[baseName] = catalogEntry{Config: cfg, Path: path, BaseName: baseName, IsStock: spec.IsStock}
112
return nil
113
}
114
@@ -115,9 +118,9 @@ func LoadFromDirs(specs []DirSpec) (Catalog, error) {
118
if spec.IsStock {
119
scope = "stock"
120
}
118
- return fmt.Errorf("duplicate %s profile id %q in %q and %q", scope, id, prev.Path, path)
121
+ return fmt.Errorf("duplicate %s profile basename %q in %q and %q", scope, baseName, prev.Path, path)
122
case prev.IsStock && !spec.IsStock:
120
- seen[id] = catalogEntry{Config: cfg, Path: path, IsStock: false}
123
+ seen[baseName] = catalogEntry{Config: cfg, Path: path, BaseName: baseName, IsStock: false}
124
case !prev.IsStock && spec.IsStock:
125
// User overrides stock. Keep the existing user profile.
126
}
@@ -133,8 +136,18 @@ func LoadFromDirs(specs []DirSpec) (Catalog, error) {
136
return Catalog{}, errors.New("no Azure Monitor profiles were loaded")
137
}
138
136
- for id, entry := range seen {
139
+ idToBaseName := make(map[string]string, len(seen))
140
+ for _, entry := range seen {
141
+ id := normalizeKey(entry.Config.ID)
142
+ if id == "" {
143
+ return Catalog{}, fmt.Errorf("profile %q: decoded empty id", entry.Path)
144
+ }
145
+ if prevBaseName, ok := idToBaseName[id]; ok {
146
+ return Catalog{}, fmt.Errorf("duplicate profile id %q for basenames %q and %q", entry.Config.ID, prevBaseName, entry.BaseName)
147
+ }
148
+ idToBaseName[id] = entry.BaseName
149
catalog.byID[id] = entry.Config
150
+ catalog.byBaseName[entry.BaseName] = entry.Config
151
}
152
153
return catalog, nil
@@ -164,19 +177,6 @@ func loadProfileFile(path string) (Profile, error) {
177
return cfg, nil
178
}
179
167
-func (c Catalog) DefaultProfileIDs() []string {
168
- if len(c.stockProfileIDs) == 0 {
169
- return nil
170
- }
171
-
172
- ids := make([]string, 0, len(c.stockProfileIDs))
173
- for id := range c.stockProfileIDs {
174
- ids = append(ids, id)
175
- }
176
- sort.Strings(ids)
177
- return ids
178
-}
179
-
180
func (c Catalog) Resolve(profileIDs []string) ([]Profile, error) {
181
if len(profileIDs) == 0 {
182
return nil, errors.New("no Azure Monitor profiles selected")
@@ -194,6 +194,23 @@ func (c Catalog) Resolve(profileIDs []string) ([]Profile, error) {
194
return profiles, nil
195
}
196
197
+func (c Catalog) ResolveBaseNames(profileBaseNames []string) ([]Profile, error) {
198
+ if len(profileBaseNames) == 0 {
199
+ return nil, errors.New("no Azure Monitor profiles selected")
200
+ }
201
+
202
+ profiles := make([]Profile, 0, len(profileBaseNames))
203
+ for _, baseName := range profileBaseNames {
204
+ normalizedBaseName := normalizeKey(baseName)
205
+ prof, ok := c.byBaseName[normalizedBaseName]
206
+ if !ok {
207
+ return nil, fmt.Errorf("unknown profile basename %q", baseName)
208
+ }
209
+ profiles = append(profiles, prof)
210
+ }
211
+ return profiles, nil
212
+}
213
+
214
func (c Catalog) ProfilesForResourceTypes(types map[string]struct{}) []string {
215
if len(types) == 0 {
216
return nil
@@ -213,6 +230,116 @@ func (c Catalog) ProfilesForResourceTypes(types map[string]struct{}) []string {
230
return ids
231
}
232
233
+func (c Catalog) ResourceTypesForProfileIDs(profileIDs []string) ([]string, error) {
234
+ if len(profileIDs) == 0 {
235
+ return nil, nil
236
+ }
237
+
238
+ seen := make(map[string]struct{}, len(profileIDs))
239
+ types := make([]string, 0, len(profileIDs))
240
+ for _, id := range profileIDs {
241
+ prof, ok := c.byID[normalizeKey(id)]
242
+ if !ok {
243
+ return nil, fmt.Errorf("unknown profile %q", id)
244
+ }
245
+
246
+ rt := strings.TrimSpace(prof.ResourceType)
247
+ key := normalizeKey(rt)
248
+ if key == "" {
249
+ continue
250
+ }
251
+ if _, ok := seen[key]; ok {
252
+ continue
253
+ }
254
+ seen[key] = struct{}{}
255
+ types = append(types, rt)
256
+ }
257
+
258
+ sort.Strings(types)
259
+ return types, nil
260
+}
261
+
262
+func (c Catalog) ProfileIDsForBaseNames(profileBaseNames []string) ([]string, error) {
263
+ profiles, err := c.ResolveBaseNames(profileBaseNames)
264
+ if err != nil {
265
+ return nil, err
266
+ }
267
+
268
+ ids := make([]string, 0, len(profiles))
269
+ for _, profile := range profiles {
270
+ ids = append(ids, profile.ID)
271
+ }
272
+ return ids, nil
273
+}
274
+
275
+func (c Catalog) ResourceTypesForProfileBaseNames(profileBaseNames []string) ([]string, error) {
276
+ profiles, err := c.ResolveBaseNames(profileBaseNames)
277
+ if err != nil {
278
+ return nil, err
279
+ }
280
+
281
+ seen := make(map[string]struct{}, len(profiles))
282
+ types := make([]string, 0, len(profiles))
283
+ for _, profile := range profiles {
284
+ rt := strings.TrimSpace(profile.ResourceType)
285
+ key := normalizeKey(rt)
286
+ if key == "" {
287
+ continue
288
+ }
289
+ if _, ok := seen[key]; ok {
290
+ continue
291
+ }
292
+ seen[key] = struct{}{}
293
+ types = append(types, rt)
294
+ }
295
+
296
+ sort.Strings(types)
297
+ return types, nil
298
+}
299
+
300
+func (c Catalog) defaultProfileBaseNames() []string {
301
+ if len(c.stockProfileBaseNames) == 0 {
302
+ return nil
303
+ }
304
+
305
+ names := make([]string, 0, len(c.stockProfileBaseNames))
306
+ for name := range c.stockProfileBaseNames {
307
+ names = append(names, name)
308
+ }
309
+ sort.Strings(names)
310
+ return names
311
+}
312
+
313
+func profileBaseName(path string) string {
314
+ name := filepath.Base(path)
315
+ ext := filepath.Ext(name)
316
+ return strings.TrimSuffix(name, ext)
317
+}
318
+
319
+func (c Catalog) ResourceTypes() []string {
320
+ if len(c.byID) == 0 {
321
+ return nil
322
+ }
323
+
324
+ seen := make(map[string]struct{}, len(c.byID))
325
+ types := make([]string, 0, len(c.byID))
326
+ for _, prof := range c.byID {
327
+ rt := strings.TrimSpace(prof.ResourceType)
328
+ key := normalizeKey(rt)
329
+ if key == "" {
330
+ continue
331
+ }
332
+ if _, ok := seen[key]; ok {
333
+ continue
334
+ }
335
+ seen[key] = struct{}{}
336
+ types = append(types, rt)
337
+ }
338
+
339
+ sort.Strings(types)
340
+ return types
341
+}
342
+
343
func defaultDirSpecs() []DirSpec {
344
if executable.Name == "test" {
345
if dir := azureProfilesDirFromThisFile(); dir != "" {
src/go/plugin/go.d/collector/azure_monitor/azureprofiles/default_catalog_test.go
+12
-12
@@ -36,12 +36,8 @@ func TestLoadFromDefaultDirs_LoadsAllStockProfiles(t *testing.T) {
36
catalog, err := LoadFromDefaultDirs()
37
require.NoError(t, err)
38
39
- ids := catalog.DefaultProfileIDs()
40
- require.Len(t, ids, want)
41
-
42
- profiles, err := catalog.Resolve(ids)
43
- require.NoError(t, err)
44
- require.Len(t, profiles, want)
39
+ assert.Len(t, catalog.byBaseName, want)
40
+ assert.Len(t, catalog.byID, want)
41
}
42
43
func TestDefaultCatalog_CachesSuccessfulLoads(t *testing.T) {
@@ -59,8 +55,8 @@ func TestDefaultCatalog_CachesSuccessfulLoads(t *testing.T) {
55
require.NoError(t, err)
56
57
assert.Equal(t, 1, calls)
62
- assert.Equal(t, []string{"sql_database"}, first.DefaultProfileIDs())
63
- assert.Equal(t, []string{"sql_database"}, second.DefaultProfileIDs())
58
+ assert.Equal(t, []string{"sql_database"}, first.defaultProfileBaseNames())
59
+ assert.Equal(t, []string{"sql_database"}, second.defaultProfileBaseNames())
60
}
61
62
func TestDefaultCatalog_RetriesAfterFailure(t *testing.T) {
@@ -81,7 +77,7 @@ func TestDefaultCatalog_RetriesAfterFailure(t *testing.T) {
77
require.NoError(t, err)
78
79
assert.Equal(t, 2, calls)
84
- assert.Equal(t, []string{"postgres_flexible"}, catalog.DefaultProfileIDs())
80
+ assert.Equal(t, []string{"postgres_flexible"}, catalog.defaultProfileBaseNames())
81
}
82
83
func TestDefaultCatalog_DoesNotCacheWhenDisabled(t *testing.T) {
@@ -126,12 +122,16 @@ func stubDefaultCatalog(t *testing.T, cacheEnabled func() bool, loader func() (C
122
}
123
124
func testCatalog(id string) Catalog {
125
+ profile := Profile{ID: id, DisplayName: id}
126
return Catalog{
127
+ byBaseName: map[string]Profile{
128
+ normalizeKey(id): profile,
129
+ },
130
byID: map[string]Profile{
131
- id: {ID: id, Name: id},
131
+ id: profile,
132
},
133
- stockProfileIDs: map[string]struct{}{
134
- id: {},
133
+ stockProfileBaseNames: map[string]struct{}{
134
+ normalizeKey(id): {},
135
},
136
}
137
}
src/go/plugin/go.d/collector/azure_monitor/azureprofiles/profile.go
+2
-2
@@ -33,7 +33,7 @@ var SupportedTimeGrains = map[string]time.Duration{
33
34
type Profile struct {
35
ID string `yaml:"id" json:"id,omitempty"`
36
- Name string `yaml:"name" json:"name,omitempty"`
36
+ DisplayName string `yaml:"name" json:"name,omitempty"`
37
ResourceType string `yaml:"resource_type" json:"resource_type,omitempty"`
38
MetricNamespace string `yaml:"metric_namespace,omitempty" json:"metric_namespace,omitempty"`
39
Metrics []Metric `yaml:"metrics" json:"metrics,omitempty"`
@@ -58,7 +58,7 @@ func (p Profile) Validate(prefix string) error {
58
if !IsValidIdentityID(p.ID) {
59
errs = append(errs, fmt.Errorf("%s: 'id' must match %q", prefix, reIdentityID.String()))
60
}
61
- if strings.TrimSpace(p.Name) == "" {
61
+ if strings.TrimSpace(p.DisplayName) == "" {
62
errs = append(errs, fmt.Errorf("%s: 'name' is required", prefix))
63
}
64
if !IsValidResourceType(p.ResourceType) {
src/go/plugin/go.d/collector/azure_monitor/collect.go
+7
-11
@@ -22,7 +22,7 @@ func (c *Collector) collect(ctx context.Context) error {
22
queryBatches := c.buildQueryBatches(resources, now)
23
24
dueInstruments := dueInstrumentsForBatches(queryBatches)
25
- samples, err := c.collectQuerySamples(ctx, queryBatches, queryEndForCollect(now, c.QueryOffset))
25
+ samples, err := c.collectQuerySamples(ctx, queryBatches, now)
26
if err != nil {
27
return err
28
}
@@ -34,6 +34,10 @@ func (c *Collector) collect(ctx context.Context) error {
34
}
35
36
func (c *Collector) refreshCollectResources(ctx context.Context) ([]resourceInfo, error) {
37
+ if err := c.ensureBootstrapped(ctx); err != nil {
38
+ return nil, err
39
+ }
40
+
41
prevFetchCounter := c.discovery.FetchCounter
42
resources, err := c.refreshDiscovery(ctx, false)
43
if err != nil {
@@ -45,12 +49,12 @@ func (c *Collector) refreshCollectResources(ctx context.Context) ([]resourceInfo
49
return resources, nil
50
}
51
48
-func (c *Collector) collectQuerySamples(ctx context.Context, batches []queryBatch, queryEnd time.Time) ([]metricSample, error) {
52
+func (c *Collector) collectQuerySamples(ctx context.Context, batches []queryBatch, queryNow time.Time) ([]metricSample, error) {
53
if len(batches) == 0 {
54
return nil, nil
55
}
56
53
- results := c.queryExecutor.runQueryBatches(ctx, batches, queryEnd)
57
+ results := c.queryExecutor.runQueryBatches(ctx, batches, queryNow, c.QueryOffset)
58
59
var (
60
allSamples []metricSample
@@ -71,11 +75,3 @@ func (c *Collector) collectQuerySamples(ctx context.Context, batches []queryBatc
75
76
return allSamples, nil
77
}
74
-
75
-func queryEndForCollect(now time.Time, queryOffsetSeconds int) time.Time {
76
- queryEnd := now.Add(-secondsToDuration(queryOffsetSeconds))
77
- if queryEnd.IsZero() {
78
- return now
79
- }
80
- return queryEnd
81
-}
src/go/plugin/go.d/collector/azure_monitor/collector.go
+27
-27
@@ -5,7 +5,6 @@ package azure_monitor
5
import (
6
"context"
7
_ "embed"
8
- "errors"
8
"time"
9
10
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
@@ -35,16 +34,23 @@ func New() *Collector {
34
store := metrix.NewCollectorStore()
35
c := &Collector{
36
Config: Config{
38
- UpdateEvery: defaultUpdateEvery,
39
- AutoDetectionRetry: defaultAutoDetectRetry,
40
- Cloud: defaultCloud,
41
- DiscoveryEvery: defaultDiscoveryEvery,
42
- QueryOffset: defaultQueryOffset,
43
- Timeout: defaultTimeout,
44
- MaxConcurrency: defaultMaxConcurrency,
45
- MaxBatchResources: defaultMaxBatchResource,
46
- MaxMetricsPerQuery: defaultMaxMetricsQuery,
47
- ProfileSelectionMode: profileSelectionModeAuto,
37
+ UpdateEvery: defaultUpdateEvery,
38
+ AutoDetectionRetry: defaultAutoDetectRetry,
39
+ Cloud: defaultCloud,
40
+ Discovery: DiscoveryConfig{
41
+ RefreshEvery: defaultDiscoveryEvery,
42
+ Mode: defaultDiscoveryMode,
43
+ },
44
+ Profiles: ProfilesConfig{
45
+ Mode: defaultProfilesMode,
46
+ },
47
+ QueryOffset: defaultQueryOffset,
48
+ Timeout: defaultTimeout,
49
+ Limits: LimitsConfig{
50
+ MaxConcurrency: defaultMaxConcurrency,
51
+ MaxBatchResources: defaultMaxBatchResource,
52
+ MaxMetricsPerQuery: defaultMaxMetricsQuery,
53
+ },
54
},
55
store: store,
56
now: time.Now,
@@ -66,7 +72,9 @@ type Collector struct {
72
queryExecutor *queryExecutor
73
observations *observationState
74
69
- runtime *collectorRuntime
75
+ runtime *collectorRuntime
76
+ profileCatalog azureprofiles.Catalog
77
+ supportedResourceTypes map[string]struct{}
78
79
discovery discoveryState
80
@@ -80,20 +88,19 @@ type Collector struct {
88
}
89
90
func (c *Collector) Init(ctx context.Context) error {
83
- result, err := c.prepareInitResult(ctx)
91
+ result, err := c.prepareInitResult()
92
if err != nil {
93
return err
94
}
95
88
- if err := c.initInstruments(result.runtime); err != nil {
89
- return err
90
- }
91
-
96
c.Config = result.config
97
+ c.profileCatalog = result.profileCatalog
98
c.resourceGraph = result.resourceGraph
99
c.queryExecutor = result.queryExecutor
95
- c.observations = newObservationState(result.runtime.Instruments)
96
- c.runtime = result.runtime
100
+ c.supportedResourceTypes = result.supportedResourceTypes
101
+ c.runtime = nil
102
+ c.observations = nil
103
+ c.discovery = discoveryState{}
104
105
return nil
106
}
@@ -101,14 +108,7 @@ func (c *Collector) Init(ctx context.Context) error {
108
func (c *Collector) Configuration() any { return c.Config }
109
110
func (c *Collector) Check(ctx context.Context) error {
104
- resources, err := c.refreshDiscovery(ctx, true)
105
- if err != nil {
106
- return err
107
- }
108
- if len(resources) == 0 {
109
- return errors.New("no Azure resources discovered for the configured profiles")
110
- }
111
- return nil
111
+ return c.ensureBootstrapped(ctx)
112
}
113
114
func (c *Collector) Collect(ctx context.Context) error { return c.collect(ctx) }
src/go/plugin/go.d/collector/azure_monitor/collector_runtime.go
+8
-6
@@ -66,12 +66,13 @@ type discoveryState struct {
66
}
67
68
type resourceInfo struct {
69
- ID string
70
- UID string
71
- Name string
72
- Type string
73
- ResourceGroup string
74
- Region string
69
+ SubscriptionID string
70
+ ID string
71
+ UID string
72
+ Name string
73
+ Type string
74
+ ResourceGroup string
75
+ Region string
76
}
77
78
func (r resourceInfo) String() string {
@@ -79,6 +80,7 @@ func (r resourceInfo) String() string {
80
}
81
82
type queryBatch struct {
83
+ SubscriptionID string
84
Profile *profileRuntime
85
Metrics []*metricRuntime
86
MetricNames []string
src/go/plugin/go.d/collector/azure_monitor/collector_test.go
+981
-118
@@ -4,8 +4,10 @@ package azure_monitor
4
5
import (
6
"context"
7
+ "encoding/json"
8
"os"
9
"path/filepath"
10
+ "strings"
11
"sync"
12
"testing"
13
"time"
@@ -39,6 +41,72 @@ func Test_testDataIsValid(t *testing.T) {
41
}
42
}
43
44
+func TestConfigSchema_RuntimeContract(t *testing.T) {
45
+ raw, err := os.ReadFile("config_schema.json")
46
+ require.NoError(t, err)
47
+
48
+ var doc map[string]any
49
+ require.NoError(t, json.Unmarshal(raw, &doc))
50
+
51
+ schema := requireMapField(t, doc, "jsonSchema")
52
+ assert.ElementsMatch(t, []string{"subscription_ids", "auth"}, requireStringSliceField(t, schema, "required"))
53
+
54
+ properties := requireMapField(t, schema, "properties")
55
+
56
+ discovery := requireMapField(t, properties, "discovery")
57
+ assert.NotContains(t, discovery, "required")
58
+ discoveryProps := requireMapField(t, discovery, "properties")
59
+ assert.Contains(t, discoveryProps, "refresh_every")
60
+ assert.Contains(t, discoveryProps, "mode")
61
+ assert.NotContains(t, discoveryProps, "mode_filters")
62
+ assert.NotContains(t, discoveryProps, "mode_query")
63
+
64
+ profiles := requireMapField(t, properties, "profiles")
65
+ assert.NotContains(t, profiles, "required")
66
+ profileProps := requireMapField(t, profiles, "properties")
67
+ assert.Contains(t, profileProps, "mode")
68
+ assert.NotContains(t, profileProps, "mode_exact")
69
+ assert.NotContains(t, profileProps, "mode_combined")
70
+
71
+ uiSchema := requireMapField(t, doc, "uiSchema")
72
+ uiProfiles := requireMapField(t, uiSchema, "profiles")
73
+ _, hasIDs := uiProfiles["ids"]
74
+ assert.False(t, hasIDs)
75
+ _, hasNames := uiProfiles["names"]
76
+ assert.False(t, hasNames)
77
+ _, hasModeExact := uiProfiles["mode_exact"]
78
+ assert.True(t, hasModeExact)
79
+ _, hasModeCombined := uiProfiles["mode_combined"]
80
+ assert.True(t, hasModeCombined)
81
+}
82
+
83
+func requireMapField(t *testing.T, m map[string]any, key string) map[string]any {
84
+ t.Helper()
85
+
86
+ value, ok := m[key]
87
+ require.Truef(t, ok, "missing key %q", key)
88
+ out, ok := value.(map[string]any)
89
+ require.Truef(t, ok, "key %q is not an object", key)
90
+ return out
91
+}
92
+
93
+func requireStringSliceField(t *testing.T, m map[string]any, key string) []string {
94
+ t.Helper()
95
+
96
+ value, ok := m[key]
97
+ require.Truef(t, ok, "missing key %q", key)
98
+ items, ok := value.([]any)
99
+ require.Truef(t, ok, "key %q is not an array", key)
100
+
101
+ out := make([]string, 0, len(items))
102
+ for _, item := range items {
103
+ s, ok := item.(string)
104
+ require.Truef(t, ok, "key %q contains a non-string item", key)
105
+ out = append(out, s)
106
+ }
107
+ return out
108
+}
109
+
110
func TestCollector_ConfigurationSerialize(t *testing.T) {
111
collecttest.TestConfigurationSerialize(t, &Collector{}, dataConfigJSON, dataConfigYAML)
112
}
@@ -60,11 +128,11 @@ func TestCollector_Init(t *testing.T) {
128
},
129
"fails on negative timeout": {
130
cfg: Config{
63
- SubscriptionID: "sub-1",
64
- Timeout: confopt.Duration(-time.Second),
65
- ProfileSelectionMode: profileSelectionModeExact,
66
- ProfileSelectionModeExact: &ProfileSelectionModeExactConfig{
67
- Profiles: []string{"postgres_flexible"},
131
+ SubscriptionIDs: []string{"sub-1"},
132
+ Timeout: confopt.Duration(-time.Second),
133
+ Profiles: ProfilesConfig{
134
+ Mode: profilesModeExact,
135
+ ModeExact: &ProfilesModeConfig{Names: []string{"postgres_flexible"}},
136
},
137
Auth: cloudauth.AzureADAuthConfig{
138
Mode: cloudauth.AzureADAuthModeDefault,
@@ -153,14 +221,24 @@ func TestCollector_UsesConfiguredTimeout(t *testing.T) {
221
setup: func(c *Collector, rg *mockResourceGraph, mx *mockMetricsClient) {
222
c.Config = testConfig()
223
c.Config.Timeout = confopt.Duration(timeout)
156
- c.Config.ProfileSelectionMode = profileSelectionModeAuto
157
- c.Config.ProfileSelectionModeExact = nil
224
+ c.Config.Profiles.Mode = profilesModeAuto
225
+ c.Config.Profiles.ModeExact = nil
226
+ c.Config.Profiles.ModeCombined = nil
227
rg.resources = []map[string]any{
159
- {"type": "microsoft.dbforpostgresql/flexibleservers", "count_": int64(1)},
228
+ {
229
+ "id": "/subscriptions/sub-1/resourceGroups/rg-a/providers/Microsoft.DBforPostgreSQL/flexibleServers/pg-a",
230
+ "name": "pg-a",
231
+ "type": "Microsoft.DBforPostgreSQL/flexibleServers",
232
+ "resourceGroup": "rg-a",
233
+ "location": "eastus",
234
+ },
235
}
236
},
237
act: func(ctx context.Context, c *Collector) error {
163
- return c.Init(ctx)
238
+ if err := c.Init(ctx); err != nil {
239
+ return err
240
+ }
241
+ return c.Check(ctx)
242
},
243
deadline: func(rg *mockResourceGraph, _ *mockMetricsClient) (time.Duration, bool) {
244
return rg.lastTimeout()
@@ -184,6 +262,9 @@ func TestCollector_UsesConfiguredTimeout(t *testing.T) {
262
if err := c.Init(ctx); err != nil {
263
return err
264
}
265
+ if err := c.Check(ctx); err != nil {
266
+ return err
267
+ }
268
_, err := c.refreshDiscovery(ctx, true)
269
return err
270
},
@@ -261,6 +342,7 @@ func TestCollector_ChartTemplateYAML(t *testing.T) {
342
}
343
344
require.NoError(t, c.Init(context.Background()))
345
+ require.NoError(t, c.Check(context.Background()))
346
347
tpl := c.ChartTemplateYAML()
348
collecttest.AssertChartTemplateSchema(t, tpl)
@@ -273,9 +355,7 @@ func TestCollector_ChartTemplateYAML(t *testing.T) {
355
require.NoError(t, err)
356
}
357
276
-func TestCollector_Collect(t *testing.T) {
277
- now := time.Date(2026, 3, 7, 12, 0, 0, 0, time.UTC)
278
-
358
+func TestCollector_InitThenCheckRunsSingleDiscoveryBootstrap(t *testing.T) {
359
rg := &mockResourceGraph{
360
resources: []map[string]any{
361
{
@@ -285,53 +365,332 @@ func TestCollector_Collect(t *testing.T) {
365
"resourceGroup": "rg-a",
366
"location": "eastus",
367
},
368
+ },
369
+ }
370
+
371
+ c := New()
372
+ c.Config = testConfig()
373
+ c.newResourceGraph = func(string, azcore.TokenCredential, azcloud.Configuration) (resourceGraphClient, error) {
374
+ return rg, nil
375
+ }
376
+ c.newMetricsClient = func(string, azcore.TokenCredential, azcloud.Configuration) (metricsQueryClient, error) {
377
+ return &mockMetricsClient{}, nil
378
+ }
379
+
380
+ require.NoError(t, c.Init(context.Background()))
381
+ assert.Equal(t, 0, rg.calls())
382
+
383
+ require.NoError(t, c.Check(context.Background()))
384
+ assert.Equal(t, 1, rg.calls())
385
+}
386
+
387
+func TestCollector_RefreshDiscoveryDisabledWhenRefreshEveryZero(t *testing.T) {
388
+ now := time.Date(2026, 3, 7, 12, 0, 0, 0, time.UTC)
389
+
390
+ rg := &mockResourceGraph{
391
+ resources: []map[string]any{
392
{
289
- "id": "/subscriptions/sub-1/resourceGroups/rg-b/providers/Microsoft.DBforPostgreSQL/flexibleServers/pg-b",
290
- "name": "pg-b",
393
+ "id": "/subscriptions/sub-1/resourceGroups/rg-a/providers/Microsoft.DBforPostgreSQL/flexibleServers/pg-a",
394
+ "name": "pg-a",
395
"type": "Microsoft.DBforPostgreSQL/flexibleServers",
292
- "resourceGroup": "rg-b",
396
+ "resourceGroup": "rg-a",
397
"location": "eastus",
398
},
399
},
400
}
297
-
401
mx := &mockMetricsClient{
402
queryResponse: azmetrics.QueryResourcesResponse{MetricResults: azmetrics.MetricResults{Values: []azmetrics.MetricData{
403
{
404
ResourceID: ptrString("/subscriptions/sub-1/resourcegroups/rg-a/providers/microsoft.dbforpostgresql/flexibleservers/pg-a"),
405
Values: []azmetrics.Metric{
406
metricWithAvg("cpu_percent", now, 21.5),
304
- metricWithAvg("storage_percent", now, 61.2),
305
- },
306
- },
307
- {
308
- ResourceID: ptrString("/subscriptions/sub-1/resourcegroups/rg-b/providers/microsoft.dbforpostgresql/flexibleservers/pg-b"),
309
- Values: []azmetrics.Metric{
310
- metricWithAvg("cpu_percent", now, 33.1),
311
- metricWithAvg("storage_percent", now, 72.8),
407
},
408
},
409
}}},
410
}
411
317
- c := New()
412
+ c := newTestCollectorWithMocks(rg, mx)
413
c.Config = testConfig()
414
+ c.Config.Discovery.RefreshEvery = 0
415
c.now = func() time.Time { return now }
320
- c.newResourceGraph = func(string, azcore.TokenCredential, azcloud.Configuration) (resourceGraphClient, error) {
321
- return rg, nil
322
- }
323
- c.newMetricsClient = func(string, azcore.TokenCredential, azcloud.Configuration) (metricsQueryClient, error) {
324
- return mx, nil
325
- }
416
417
require.NoError(t, c.Init(context.Background()))
418
+ require.NoError(t, c.Check(context.Background()))
419
+ assert.Equal(t, 1, rg.calls())
420
329
- series, err := collecttest.CollectScalarSeries(c, metrix.ReadRaw())
421
+ now = now.Add(10 * time.Minute)
422
+
423
+ _, err := collecttest.CollectScalarSeries(c, metrix.ReadRaw())
424
require.NoError(t, err)
425
+ assert.Equal(t, 1, rg.calls())
426
+}
427
+
428
+func TestCollector_CollectScenarios(t *testing.T) {
429
+ now := time.Date(2026, 3, 7, 12, 0, 0, 0, time.UTC)
430
+
431
+ tests := map[string]struct {
432
+ prepare func(*Collector, *mockResourceGraph, *mockMetricsClient)
433
+ check func(*testing.T, map[string]metrix.SampleValue, error, *mockResourceGraph, *mockMetricsClient)
434
+ }{
435
+ "success on discovered resources": {
436
+ prepare: func(c *Collector, rg *mockResourceGraph, mx *mockMetricsClient) {
437
+ c.Config = testConfig()
438
+ c.now = func() time.Time { return now }
439
+ rg.resources = []map[string]any{
440
+ {
441
+ "id": "/subscriptions/sub-1/resourceGroups/rg-a/providers/Microsoft.DBforPostgreSQL/flexibleServers/pg-a",
442
+ "name": "pg-a",
443
+ "type": "Microsoft.DBforPostgreSQL/flexibleServers",
444
+ "resourceGroup": "rg-a",
445
+ "location": "eastus",
446
+ },
447
+ {
448
+ "id": "/subscriptions/sub-1/resourceGroups/rg-b/providers/Microsoft.DBforPostgreSQL/flexibleServers/pg-b",
449
+ "name": "pg-b",
450
+ "type": "Microsoft.DBforPostgreSQL/flexibleServers",
451
+ "resourceGroup": "rg-b",
452
+ "location": "eastus",
453
+ },
454
+ }
455
+ mx.queryResponse = azmetrics.QueryResourcesResponse{MetricResults: azmetrics.MetricResults{Values: []azmetrics.MetricData{
456
+ {
457
+ ResourceID: ptrString("/subscriptions/sub-1/resourcegroups/rg-a/providers/microsoft.dbforpostgresql/flexibleservers/pg-a"),
458
+ Values: []azmetrics.Metric{
459
+ metricWithAvg("cpu_percent", now, 21.5),
460
+ metricWithAvg("storage_percent", now, 61.2),
461
+ },
462
+ },
463
+ {
464
+ ResourceID: ptrString("/subscriptions/sub-1/resourcegroups/rg-b/providers/microsoft.dbforpostgresql/flexibleservers/pg-b"),
465
+ Values: []azmetrics.Metric{
466
+ metricWithAvg("cpu_percent", now, 33.1),
467
+ metricWithAvg("storage_percent", now, 72.8),
468
+ },
469
+ },
470
+ }}}
471
+ },
472
+ check: func(t *testing.T, series map[string]metrix.SampleValue, err error, rg *mockResourceGraph, mx *mockMetricsClient) {
473
+ require.NoError(t, err)
474
+ assert.GreaterOrEqual(t, len(series), 4)
475
+ assert.Equal(t, 1, rg.calls())
476
+ assert.GreaterOrEqual(t, mx.calls(), 1)
477
+ },
478
+ },
479
+ "collects across multiple subscriptions": {
480
+ prepare: func(c *Collector, rg *mockResourceGraph, mx *mockMetricsClient) {
481
+ c.Config = testConfig()
482
+ c.Config.SubscriptionIDs = []string{"sub-1", "sub-2"}
483
+ c.now = func() time.Time { return now }
484
+ rg.resources = []map[string]any{
485
+ {
486
+ "id": "/subscriptions/sub-1/resourceGroups/rg-a/providers/Microsoft.DBforPostgreSQL/flexibleServers/pg-a",
487
+ "name": "pg-a",
488
+ "type": "Microsoft.DBforPostgreSQL/flexibleServers",
489
+ "resourceGroup": "rg-a",
490
+ "location": "eastus",
491
+ },
492
+ {
493
+ "id": "/subscriptions/sub-2/resourceGroups/rg-b/providers/Microsoft.DBforPostgreSQL/flexibleServers/pg-b",
494
+ "name": "pg-b",
495
+ "type": "Microsoft.DBforPostgreSQL/flexibleServers",
496
+ "resourceGroup": "rg-b",
497
+ "location": "eastus",
498
+ },
499
+ }
500
+ mx.queryResponse = azmetrics.QueryResourcesResponse{MetricResults: azmetrics.MetricResults{Values: []azmetrics.MetricData{
501
+ {
502
+ ResourceID: ptrString("/subscriptions/sub-1/resourcegroups/rg-a/providers/microsoft.dbforpostgresql/flexibleservers/pg-a"),
503
+ Values: []azmetrics.Metric{
504
+ metricWithAvg("cpu_percent", now, 21.5),
505
+ },
506
+ },
507
+ {
508
+ ResourceID: ptrString("/subscriptions/sub-2/resourcegroups/rg-b/providers/microsoft.dbforpostgresql/flexibleservers/pg-b"),
509
+ Values: []azmetrics.Metric{
510
+ metricWithAvg("cpu_percent", now, 33.1),
511
+ },
512
+ },
513
+ }}}
514
+ },
515
+ check: func(t *testing.T, series map[string]metrix.SampleValue, err error, rg *mockResourceGraph, mx *mockMetricsClient) {
516
+ require.NoError(t, err)
517
+
518
+ var keys []string
519
+ for key := range series {
520
+ keys = append(keys, key)
521
+ }
522
+
523
+ assert.Equal(t, 1, rg.calls())
524
+ assert.ElementsMatch(t, []string{"sub-1", "sub-2"}, rg.lastSubscriptions())
525
+ assert.GreaterOrEqual(t, mx.calls(), 2)
526
+ assert.ElementsMatch(t, []string{"sub-1", "sub-2"}, uniqueStrings(mx.subscriptionCalls()))
527
+ assert.Contains(t, strings.Join(keys, "\n"), `subscription_id="sub-1"`)
528
+ assert.Contains(t, strings.Join(keys, "\n"), `subscription_id="sub-2"`)
529
+ },
530
+ },
531
+ "idle when no resources match": {
532
+ prepare: func(c *Collector, rg *mockResourceGraph, mx *mockMetricsClient) {
533
+ c.Config = testConfig()
534
+ },
535
+ check: func(t *testing.T, series map[string]metrix.SampleValue, err error, rg *mockResourceGraph, mx *mockMetricsClient) {
536
+ require.NoError(t, err)
537
+ assert.Empty(t, series)
538
+ },
539
+ },
540
+ "partial batch failure still succeeds": {
541
+ prepare: func(c *Collector, rg *mockResourceGraph, mx *mockMetricsClient) {
542
+ c.Config = testConfig()
543
+ c.Config.SubscriptionIDs = []string{"sub-1", "sub-2"}
544
+ c.now = func() time.Time { return now }
545
+ rg.resources = []map[string]any{
546
+ {
547
+ "id": "/subscriptions/sub-1/resourceGroups/rg-a/providers/Microsoft.DBforPostgreSQL/flexibleServers/pg-a",
548
+ "name": "pg-a",
549
+ "type": "Microsoft.DBforPostgreSQL/flexibleServers",
550
+ "resourceGroup": "rg-a",
551
+ "location": "eastus",
552
+ },
553
+ {
554
+ "id": "/subscriptions/sub-2/resourceGroups/rg-b/providers/Microsoft.DBforPostgreSQL/flexibleServers/pg-b",
555
+ "name": "pg-b",
556
+ "type": "Microsoft.DBforPostgreSQL/flexibleServers",
557
+ "resourceGroup": "rg-b",
558
+ "location": "eastus",
559
+ },
560
+ }
561
+ mx.queryResponses = map[string]azmetrics.QueryResourcesResponse{
562
+ "sub-1": {MetricResults: azmetrics.MetricResults{Values: []azmetrics.MetricData{
563
+ {
564
+ ResourceID: ptrString("/subscriptions/sub-1/resourcegroups/rg-a/providers/microsoft.dbforpostgresql/flexibleservers/pg-a"),
565
+ Values: []azmetrics.Metric{
566
+ metricWithAvg("cpu_percent", now, 21.5),
567
+ },
568
+ },
569
+ }}},
570
+ }
571
+ mx.queryErrors = map[string]error{
572
+ "sub-2": assert.AnError,
573
+ }
574
+ },
575
+ check: func(t *testing.T, series map[string]metrix.SampleValue, err error, rg *mockResourceGraph, mx *mockMetricsClient) {
576
+ require.NoError(t, err)
577
+ assert.NotEmpty(t, series)
578
+ assert.Contains(t, strings.Join(keysFromSeries(series), "\n"), `subscription_id="sub-1"`)
579
+ },
580
+ },
581
+ "all batches fail": {
582
+ prepare: func(c *Collector, rg *mockResourceGraph, mx *mockMetricsClient) {
583
+ c.Config = testConfig()
584
+ c.Config.SubscriptionIDs = []string{"sub-1", "sub-2"}
585
+ rg.resources = []map[string]any{
586
+ {
587
+ "id": "/subscriptions/sub-1/resourceGroups/rg-a/providers/Microsoft.DBforPostgreSQL/flexibleServers/pg-a",
588
+ "name": "pg-a",
589
+ "type": "Microsoft.DBforPostgreSQL/flexibleServers",
590
+ "resourceGroup": "rg-a",
591
+ "location": "eastus",
592
+ },
593
+ {
594
+ "id": "/subscriptions/sub-2/resourceGroups/rg-b/providers/Microsoft.DBforPostgreSQL/flexibleServers/pg-b",
595
+ "name": "pg-b",
596
+ "type": "Microsoft.DBforPostgreSQL/flexibleServers",
597
+ "resourceGroup": "rg-b",
598
+ "location": "eastus",
599
+ },
600
+ }
601
+ mx.queryErrors = map[string]error{
602
+ "sub-1": assert.AnError,
603
+ "sub-2": assert.AnError,
604
+ }
605
+ },
606
+ check: func(t *testing.T, series map[string]metrix.SampleValue, err error, rg *mockResourceGraph, mx *mockMetricsClient) {
607
+ require.Error(t, err)
608
+ assert.ErrorContains(t, err, "all Azure Monitor batch queries failed")
609
+ },
610
+ },
611
+ }
612
+
613
+ for name, tc := range tests {
614
+ t.Run(name, func(t *testing.T) {
615
+ rg := &mockResourceGraph{}
616
+ mx := &mockMetricsClient{}
617
+ c := newTestCollectorWithMocks(rg, mx)
618
+ tc.prepare(c, rg, mx)
619
+
620
+ require.NoError(t, c.Init(context.Background()))
621
+
622
+ series, err := collecttest.CollectScalarSeries(c, metrix.ReadRaw())
623
+ tc.check(t, series, err, rg, mx)
624
+ })
625
+ }
626
+}
627
+
628
+func TestCollector_RefreshDiscovery_PushesModeFiltersIntoQuery(t *testing.T) {
629
+ tests := map[string]struct {
630
+ resources []map[string]any
631
+ filters *DiscoveryFiltersConfig
632
+ wantQuery string
633
+ wantCounts int
634
+ }{
635
+ "resource groups only": {
636
+ resources: []map[string]any{
637
+ {
638
+ "id": "/subscriptions/sub-1/resourceGroups/rg-a/providers/Microsoft.DBforPostgreSQL/flexibleServers/pg-a",
639
+ "name": "pg-a",
640
+ "type": "Microsoft.DBforPostgreSQL/flexibleServers",
641
+ "resourceGroup": "rg-a",
642
+ "location": "eastus",
643
+ },
644
+ {
645
+ "id": "/subscriptions/sub-1/resourceGroups/rg-b/providers/Microsoft.DBforPostgreSQL/flexibleServers/pg-b",
646
+ "name": "pg-b",
647
+ "type": "Microsoft.DBforPostgreSQL/flexibleServers",
648
+ "resourceGroup": "rg-b",
649
+ "location": "eastus",
650
+ },
651
+ },
652
+ filters: &DiscoveryFiltersConfig{ResourceGroups: []string{"RG-B", " rg-a ", "rg-b"}},
653
+ wantCounts: 2,
654
+ wantQuery: "resources | where type in~ ('Microsoft.DBforPostgreSQL/flexibleServers') | where resourceGroup in~ ('rg-a', 'rg-b') | project id, name, type, resourceGroup, location",
655
+ },
656
+ "resource groups regions and tags": {
657
+ resources: []map[string]any{
658
+ {
659
+ "id": "/subscriptions/sub-1/resourceGroups/rg-a/providers/Microsoft.DBforPostgreSQL/flexibleServers/pg-a",
660
+ "name": "pg-a",
661
+ "type": "Microsoft.DBforPostgreSQL/flexibleServers",
662
+ "resourceGroup": "rg-a",
663
+ "location": "eastus",
664
+ },
665
+ },
666
+ filters: &DiscoveryFiltersConfig{
667
+ ResourceGroups: []string{"RG-B", " rg-a ", "rg-b"},
668
+ Regions: []string{" WestEurope ", "eastus", "EASTUS"},
669
+ Tags: map[string][]string{
670
+ "ROLE": {"worker", "api", "worker"},
671
+ " env ": {"prod"},
672
+ },
673
+ },
674
+ wantQuery: "resources | where type in~ ('Microsoft.DBforPostgreSQL/flexibleServers') | where resourceGroup in~ ('rg-a', 'rg-b') | where location in~ ('eastus', 'westeurope') | mv-expand bagexpansion=array tags | where isnotempty(tags) | extend tagKey = tostring(tags[0]), tagValue = tostring(tags[1]) | where (tagKey =~ 'env' and tagValue == 'prod') or (tagKey =~ 'role' and tagValue in ('api', 'worker')) | summarize by id, name, type, resourceGroup, location, matchedTagKey = tolower(tagKey) | summarize matchedTagKeys = count() by id, name, type, resourceGroup, location | where matchedTagKeys == 2 | project id, name, type, resourceGroup, location",
675
+ },
676
+ }
677
+
678
+ for name, tc := range tests {
679
+ t.Run(name, func(t *testing.T) {
680
+ rg := &mockResourceGraph{resources: tc.resources}
681
+ c := newTestCollectorWithMocks(rg, &mockMetricsClient{})
682
+ c.Config = testConfig()
683
+ c.Config.Discovery.ModeFilters = tc.filters
684
332
- assert.GreaterOrEqual(t, len(series), 4)
333
- assert.GreaterOrEqual(t, rg.calls(), 1)
334
- assert.GreaterOrEqual(t, mx.calls(), 1)
685
+ require.NoError(t, c.Init(context.Background()))
686
+ require.NoError(t, c.Check(context.Background()))
687
+
688
+ if tc.wantCounts > 0 {
689
+ require.Len(t, c.discovery.Resources, tc.wantCounts)
690
+ }
691
+ assert.Equal(t, tc.wantQuery, rg.lastQuery())
692
+ })
693
+ }
694
}
695
696
func TestCollector_TimeGrainScheduling(t *testing.T) {
@@ -361,7 +720,7 @@ func TestCollector_TimeGrainScheduling(t *testing.T) {
720
}
721
722
cfg := testConfig()
364
- cfg.ProfileSelectionModeExact = &ProfileSelectionModeExactConfig{Profiles: []string{"storage_slow"}}
723
+ cfg.Profiles.ModeExact = &ProfilesModeConfig{Names: []string{"storage_slow"}}
724
725
catalog := mustLoadStockCatalog(t, map[string]string{
726
"storage_slow.yaml": `
@@ -419,97 +778,484 @@ template:
778
assert.Equal(t, 1, mx.calls())
779
}
780
422
-func TestCollector_InitAutoDiscover(t *testing.T) {
781
+func TestCollector_QueryOffsetUsesEffectivePerBatchOffset(t *testing.T) {
782
+ now := time.Date(2026, 3, 7, 12, 0, 0, 0, time.UTC)
783
+
784
rg := &mockResourceGraph{
785
resources: []map[string]any{
425
- {"type": "microsoft.dbforpostgresql/flexibleservers", "count_": int64(2)},
786
+ {
787
+ "id": "/subscriptions/sub-1/resourceGroups/rg-a/providers/Microsoft.Storage/storageAccounts/st-a",
788
+ "name": "st-a",
789
+ "type": "Microsoft.Storage/storageAccounts",
790
+ "resourceGroup": "rg-a",
791
+ "location": "eastus",
792
+ },
793
},
794
}
795
429
- c := New()
430
- c.Config = testConfig()
431
- c.Config.ProfileSelectionMode = profileSelectionModeAuto
432
- c.Config.ProfileSelectionModeExact = nil
433
- c.newResourceGraph = func(string, azcore.TokenCredential, azcloud.Configuration) (resourceGraphClient, error) {
434
- return rg, nil
435
- }
436
- c.newMetricsClient = func(string, azcore.TokenCredential, azcloud.Configuration) (metricsQueryClient, error) {
437
- return &mockMetricsClient{}, nil
796
+ mx := &mockMetricsClient{
797
+ queryResponse: azmetrics.QueryResourcesResponse{MetricResults: azmetrics.MetricResults{}},
798
}
799
440
- require.NoError(t, c.Init(context.Background()))
441
- require.NotEmpty(t, c.runtime.Profiles)
442
- assert.Equal(t, "postgres_flexible", c.runtime.Profiles[0].ID)
443
-}
800
+ cfg := testConfig()
801
+ cfg.Profiles.ModeExact = &ProfilesModeConfig{Names: []string{"storage_mixed"}}
802
445
-func TestCollector_InitAutoDiscoverWithExplicit(t *testing.T) {
446
- rg := &mockResourceGraph{
447
- resources: []map[string]any{
448
- {"type": "microsoft.dbforpostgresql/flexibleservers", "count_": int64(2)},
449
- },
450
- }
803
+ catalog := mustLoadStockCatalog(t, map[string]string{
804
+ "storage_mixed.yaml": `
805
+id: storage_mixed
806
+name: Azure Storage Mixed Grains
807
+resource_type: Microsoft.Storage/storageAccounts
808
+metrics:
809
+ - id: transactions
810
+ azure_name: Transactions
811
+ time_grain: PT1M
812
+ series:
813
+ - aggregation: average
814
+ kind: gauge
815
+ - id: used_capacity
816
+ azure_name: UsedCapacity
817
+ time_grain: PT5M
818
+ series:
819
+ - aggregation: average
820
+ kind: gauge
821
+template:
822
+ family: Azure Storage Mixed
823
+ context_namespace: storage_mixed
824
+ charts:
825
+ - id: am_storage_mixed_transactions
826
+ title: Azure Storage Mixed Transactions
827
+ context: transactions
828
+ family: Throughput
829
+ type: line
830
+ units: ops/s
831
+ algorithm: absolute
832
+ dimensions:
833
+ - selector: ` + azureprofiles.ExportedSeriesName("storage_mixed", "transactions", "average") + `
834
+ name: average
835
+ - id: am_storage_mixed_used_capacity
836
+ title: Azure Storage Mixed Used Capacity
837
+ context: used_capacity
838
+ family: Capacity
839
+ type: line
840
+ units: bytes
841
+ algorithm: absolute
842
+ dimensions:
843
+ - selector: ` + azureprofiles.ExportedSeriesName("storage_mixed", "used_capacity", "average") + `
844
+ name: average
845
+`,
846
+ })
847
848
c := New()
453
- c.Config = testConfig()
454
- c.Config.ProfileSelectionMode = profileSelectionModeCombined
455
- c.Config.ProfileSelectionModeExact = nil
456
- c.Config.ProfileSelectionModeCombined = &ProfileSelectionModeCombinedConfig{
457
- Profiles: []string{"cosmos_db"},
849
+ c.Config = cfg
850
+ c.now = func() time.Time { return now }
851
+ c.loadProfileCatalog = func() (azureprofiles.Catalog, error) {
852
+ return catalog, nil
853
}
854
c.newResourceGraph = func(string, azcore.TokenCredential, azcloud.Configuration) (resourceGraphClient, error) {
855
return rg, nil
856
}
857
c.newMetricsClient = func(string, azcore.TokenCredential, azcloud.Configuration) (metricsQueryClient, error) {
463
- return &mockMetricsClient{}, nil
858
+ return mx, nil
859
}
860
861
require.NoError(t, c.Init(context.Background()))
467
- var ids []string
468
- for _, p := range c.runtime.Profiles {
469
- ids = append(ids, p.ID)
862
+
863
+ _, err := collecttest.CollectScalarSeries(c, metrix.ReadRaw())
864
+ require.NoError(t, err)
865
+
866
+ calls := mx.queryCalls()
867
+ require.Len(t, calls, 2)
868
+
869
+ byInterval := make(map[string]metricsQueryCall, len(calls))
870
+ for _, call := range calls {
871
+ byInterval[call.Interval] = call
872
}
471
- assert.Contains(t, ids, "cosmos_db")
472
- assert.Contains(t, ids, "postgres_flexible")
873
+
874
+ require.Contains(t, byInterval, "PT1M")
875
+ require.Contains(t, byInterval, "PT5M")
876
+
877
+ assert.Equal(t, "2026-03-07T11:56:00Z", byInterval["PT1M"].StartTime)
878
+ assert.Equal(t, "2026-03-07T11:57:00Z", byInterval["PT1M"].EndTime)
879
+ assert.Equal(t, []string{"Transactions"}, byInterval["PT1M"].MetricNames)
880
+
881
+ assert.Equal(t, "2026-03-07T11:50:00Z", byInterval["PT5M"].StartTime)
882
+ assert.Equal(t, "2026-03-07T11:55:00Z", byInterval["PT5M"].EndTime)
883
+ assert.Equal(t, []string{"UsedCapacity"}, byInterval["PT5M"].MetricNames)
884
}
885
475
-func TestCollector_InitDefaultModeIsAuto(t *testing.T) {
476
- c := New()
477
- c.Config = testConfig()
478
- c.Config.ProfileSelectionMode = ""
479
- c.Config.ProfileSelectionModeExact = nil
480
- c.newResourceGraph = func(string, azcore.TokenCredential, azcloud.Configuration) (resourceGraphClient, error) {
481
- return &mockResourceGraph{}, nil
886
+func TestCollector_CheckBootstrapProfileScenarios(t *testing.T) {
887
+ tests := map[string]struct {
888
+ resources []map[string]any
889
+ prepare func(*Collector)
890
+ wantErrContains string
891
+ check func(*testing.T, *Collector, *mockResourceGraph)
892
+ }{
893
+ "auto discover": {
894
+ resources: []map[string]any{
895
+ {
896
+ "id": "/subscriptions/sub-1/resourceGroups/rg-a/providers/Microsoft.DBforPostgreSQL/flexibleServers/pg-a",
897
+ "name": "pg-a",
898
+ "type": "Microsoft.DBforPostgreSQL/flexibleServers",
899
+ "resourceGroup": "rg-a",
900
+ "location": "eastus",
901
+ },
902
+ },
903
+ prepare: func(c *Collector) {
904
+ c.Config = testConfig()
905
+ c.Config.Profiles.Mode = profilesModeAuto
906
+ c.Config.Profiles.ModeExact = nil
907
+ c.Config.Profiles.ModeCombined = nil
908
+ },
909
+ check: func(t *testing.T, c *Collector, rg *mockResourceGraph) {
910
+ require.NotEmpty(t, c.runtime.Profiles)
911
+ assert.Equal(t, "postgres_flexible", c.runtime.Profiles[0].ID)
912
+ },
913
+ },
914
+ "combined with explicit profiles": {
915
+ resources: []map[string]any{
916
+ {
917
+ "id": "/subscriptions/sub-1/resourceGroups/rg-a/providers/Microsoft.DBforPostgreSQL/flexibleServers/pg-a",
918
+ "name": "pg-a",
919
+ "type": "Microsoft.DBforPostgreSQL/flexibleServers",
920
+ "resourceGroup": "rg-a",
921
+ "location": "eastus",
922
+ },
923
+ },
924
+ prepare: func(c *Collector) {
925
+ c.Config = testConfig()
926
+ c.Config.Profiles.Mode = profilesModeCombined
927
+ c.Config.Profiles.ModeExact = nil
928
+ c.Config.Profiles.ModeCombined = &ProfilesModeConfig{Names: []string{"cosmos_db"}}
929
+ },
930
+ check: func(t *testing.T, c *Collector, rg *mockResourceGraph) {
931
+ var ids []string
932
+ for _, p := range c.runtime.Profiles {
933
+ ids = append(ids, p.ID)
934
+ }
935
+ assert.Contains(t, ids, "cosmos_db")
936
+ assert.Contains(t, ids, "postgres_flexible")
937
+ },
938
+ },
939
+ "combined mode allows no auto matches": {
940
+ prepare: func(c *Collector) {
941
+ c.Config = testConfig()
942
+ c.Config.Profiles.Mode = profilesModeCombined
943
+ c.Config.Profiles.ModeExact = nil
944
+ c.Config.Profiles.ModeCombined = &ProfilesModeConfig{Names: []string{"cosmos_db"}}
945
+ },
946
+ check: func(t *testing.T, c *Collector, rg *mockResourceGraph) {
947
+ require.Len(t, c.runtime.Profiles, 1)
948
+ assert.Equal(t, "cosmos_db", c.runtime.Profiles[0].ID)
949
+ },
950
+ },
951
+ "default mode resolves to auto and fails with no matches": {
952
+ prepare: func(c *Collector) {
953
+ c.Config = testConfig()
954
+ c.Config.Profiles.Mode = ""
955
+ c.Config.Profiles.ModeExact = nil
956
+ c.Config.Profiles.ModeCombined = nil
957
+ },
958
+ wantErrContains: "auto-discovery found no Azure resources",
959
+ },
960
+ "auto discover fails when no resources match": {
961
+ prepare: func(c *Collector) {
962
+ c.Config = testConfig()
963
+ c.Config.Profiles.Mode = profilesModeAuto
964
+ c.Config.Profiles.ModeExact = nil
965
+ c.Config.Profiles.ModeCombined = nil
966
+ },
967
+ wantErrContains: "auto-discovery found no Azure resources",
968
+ },
969
}
483
- c.newMetricsClient = func(string, azcore.TokenCredential, azcloud.Configuration) (metricsQueryClient, error) {
484
- return &mockMetricsClient{}, nil
970
+
971
+ for name, tc := range tests {
972
+ t.Run(name, func(t *testing.T) {
973
+ rg := &mockResourceGraph{resources: tc.resources}
974
+ c := newTestCollectorWithMocks(rg, &mockMetricsClient{})
975
+ tc.prepare(c)
976
+
977
+ require.NoError(t, c.Init(context.Background()))
978
+ err := c.Check(context.Background())
979
+ if tc.wantErrContains != "" {
980
+ require.Error(t, err)
981
+ assert.ErrorContains(t, err, tc.wantErrContains)
982
+ return
983
+ }
984
+
985
+ require.NoError(t, err)
986
+ if tc.check != nil {
987
+ tc.check(t, c, rg)
988
+ }
989
+ })
990
+ }
991
+}
992
+
993
+func TestCollector_CheckBootstrapQueryModeScenarios(t *testing.T) {
994
+ const kql = "resources | project id, name, type, resourceGroup, location"
995
+
996
+ tests := map[string]struct {
997
+ resources []map[string]any
998
+ prepare func(*Collector)
999
+ check func(*testing.T, *Collector, *mockResourceGraph)
1000
+ }{
1001
+ "auto discover from custom query": {
1002
+ resources: []map[string]any{
1003
+ {
1004
+ "id": "/subscriptions/sub-1/resourceGroups/rg-a/providers/Microsoft.DBforPostgreSQL/flexibleServers/pg-a",
1005
+ "name": "pg-a",
1006
+ "type": "Microsoft.DBforPostgreSQL/flexibleServers",
1007
+ "resourceGroup": "rg-a",
1008
+ "location": "eastus",
1009
+ },
1010
+ },
1011
+ prepare: func(c *Collector) {
1012
+ c.Config = testConfig()
1013
+ c.Config.Profiles.Mode = profilesModeAuto
1014
+ c.Config.Profiles.ModeExact = nil
1015
+ c.Config.Profiles.ModeCombined = nil
1016
+ c.Config.Discovery.Mode = discoveryModeQuery
1017
+ c.Config.Discovery.ModeFilters = nil
1018
+ c.Config.Discovery.ModeQuery = &DiscoveryQueryConfig{KQL: kql}
1019
+ },
1020
+ check: func(t *testing.T, c *Collector, rg *mockResourceGraph) {
1021
+ require.NotEmpty(t, c.runtime.Profiles)
1022
+ assert.Equal(t, "postgres_flexible", c.runtime.Profiles[0].ID)
1023
+ assert.Equal(t, kql, rg.lastQuery())
1024
+ },
1025
+ },
1026
+ "normalizes empty location to global": {
1027
+ resources: []map[string]any{
1028
+ {
1029
+ "id": "/subscriptions/sub-1/resourceGroups/rg-a/providers/Microsoft.DBforPostgreSQL/flexibleServers/pg-a",
1030
+ "name": "pg-a",
1031
+ "type": "Microsoft.DBforPostgreSQL/flexibleServers",
1032
+ "resourceGroup": "rg-a",
1033
+ "location": "",
1034
+ },
1035
+ },
1036
+ prepare: func(c *Collector) {
1037
+ c.Config = testConfig()
1038
+ c.Config.Discovery.Mode = discoveryModeQuery
1039
+ c.Config.Discovery.ModeFilters = nil
1040
+ c.Config.Discovery.ModeQuery = &DiscoveryQueryConfig{KQL: kql}
1041
+ },
1042
+ check: func(t *testing.T, c *Collector, rg *mockResourceGraph) {
1043
+ require.Len(t, c.discovery.Resources, 1)
1044
+ assert.Equal(t, "global", c.discovery.Resources[0].Region)
1045
+ },
1046
+ },
1047
+ "exact mode ignores unsupported discovered types": {
1048
+ resources: []map[string]any{
1049
+ {
1050
+ "id": "/subscriptions/sub-1/resourceGroups/rg-a/providers/Microsoft.FakeService/fakeResources/fake-a",
1051
+ "name": "fake-a",
1052
+ "type": "Microsoft.FakeService/fakeResources",
1053
+ "resourceGroup": "rg-a",
1054
+ "location": "eastus",
1055
+ },
1056
+ },
1057
+ prepare: func(c *Collector) {
1058
+ c.Config = testConfig()
1059
+ c.Config.Discovery.Mode = discoveryModeQuery
1060
+ c.Config.Discovery.ModeFilters = nil
1061
+ c.Config.Discovery.ModeQuery = &DiscoveryQueryConfig{KQL: kql}
1062
+ },
1063
+ check: func(t *testing.T, c *Collector, rg *mockResourceGraph) {
1064
+ require.Len(t, c.runtime.Profiles, 1)
1065
+ assert.Equal(t, "postgres_flexible", c.runtime.Profiles[0].ID)
1066
+ assert.Empty(t, c.discovery.Resources)
1067
+ },
1068
+ },
1069
}
1070
487
- err := c.Init(context.Background())
488
- assert.Error(t, err)
489
- assert.Contains(t, err.Error(), "auto-discovery found no Azure resources")
1071
+ for name, tc := range tests {
1072
+ t.Run(name, func(t *testing.T) {
1073
+ rg := &mockResourceGraph{resources: tc.resources}
1074
+ c := newTestCollectorWithMocks(rg, &mockMetricsClient{})
1075
+ tc.prepare(c)
1076
+
1077
+ require.NoError(t, c.Init(context.Background()))
1078
+ require.NoError(t, c.Check(context.Background()))
1079
+ tc.check(t, c, rg)
1080
+ })
1081
+ }
1082
}
1083
492
-func TestCollector_InitAutoDiscoverNoMatchFails(t *testing.T) {
493
- rg := &mockResourceGraph{
494
- resources: []map[string]any{
495
- {"type": "microsoft.fakeservice/fakeresources", "count_": int64(3)},
1084
+func TestCollector_InitQueryModeRejectsMalformedRows(t *testing.T) {
1085
+ const kql = "resources | project id, name, type, resourceGroup, location"
1086
+
1087
+ tests := map[string]struct {
1088
+ rows []map[string]any
1089
+ wantErrContain string
1090
+ }{
1091
+ "missing required column": {
1092
+ rows: []map[string]any{
1093
+ {
1094
+ "id": "/subscriptions/sub-1/resourceGroups/rg-a/providers/Microsoft.DBforPostgreSQL/flexibleServers/pg-a",
1095
+ "name": "pg-a",
1096
+ "type": "Microsoft.DBforPostgreSQL/flexibleServers",
1097
+ "resourceGroup": "rg-a",
1098
+ },
1099
+ },
1100
+ wantErrContain: `missing required column "location"`,
1101
+ },
1102
+ "duplicate id": {
1103
+ rows: []map[string]any{
1104
+ {
1105
+ "id": "/subscriptions/sub-1/resourceGroups/rg-a/providers/Microsoft.DBforPostgreSQL/flexibleServers/pg-a",
1106
+ "name": "pg-a",
1107
+ "type": "Microsoft.DBforPostgreSQL/flexibleServers",
1108
+ "resourceGroup": "rg-a",
1109
+ "location": "eastus",
1110
+ },
1111
+ {
1112
+ "id": "/subscriptions/sub-1/resourceGroups/rg-a/providers/Microsoft.DBforPostgreSQL/flexibleServers/pg-a",
1113
+ "name": "pg-a-dup",
1114
+ "type": "Microsoft.DBforPostgreSQL/flexibleServers",
1115
+ "resourceGroup": "rg-a",
1116
+ "location": "eastus",
1117
+ },
1118
+ },
1119
+ wantErrContain: `duplicate id "/subscriptions/sub-1/resourceGroups/rg-a/providers/Microsoft.DBforPostgreSQL/flexibleServers/pg-a"`,
1120
+ },
1121
+ "invalid arm id": {
1122
+ rows: []map[string]any{
1123
+ {
1124
+ "id": "not-an-arm-id",
1125
+ "name": "pg-a",
1126
+ "type": "Microsoft.DBforPostgreSQL/flexibleServers",
1127
+ "resourceGroup": "rg-a",
1128
+ "location": "eastus",
1129
+ },
1130
+ },
1131
+ wantErrContain: `invalid ARM resource id "not-an-arm-id"`,
1132
},
1133
}
1134
499
- c := New()
500
- c.Config = testConfig()
501
- c.Config.ProfileSelectionMode = profileSelectionModeAuto
502
- c.Config.ProfileSelectionModeExact = nil
503
- c.newResourceGraph = func(string, azcore.TokenCredential, azcloud.Configuration) (resourceGraphClient, error) {
504
- return rg, nil
1135
+ for name, tc := range tests {
1136
+ t.Run(name, func(t *testing.T) {
1137
+ rg := &mockResourceGraph{resources: tc.rows}
1138
+
1139
+ c := New()
1140
+ c.Config = testConfig()
1141
+ c.Config.Profiles.Mode = profilesModeAuto
1142
+ c.Config.Profiles.ModeExact = nil
1143
+ c.Config.Profiles.ModeCombined = nil
1144
+ c.Config.Discovery.Mode = discoveryModeQuery
1145
+ c.Config.Discovery.ModeFilters = nil
1146
+ c.Config.Discovery.ModeQuery = &DiscoveryQueryConfig{KQL: kql}
1147
+ c.newResourceGraph = func(string, azcore.TokenCredential, azcloud.Configuration) (resourceGraphClient, error) {
1148
+ return rg, nil
1149
+ }
1150
+ c.newMetricsClient = func(string, azcore.TokenCredential, azcloud.Configuration) (metricsQueryClient, error) {
1151
+ return &mockMetricsClient{}, nil
1152
+ }
1153
+
1154
+ require.NoError(t, c.Init(context.Background()))
1155
+ err := c.Check(context.Background())
1156
+ require.Error(t, err)
1157
+ assert.ErrorContains(t, err, tc.wantErrContain)
1158
+ })
1159
}
506
- c.newMetricsClient = func(string, azcore.TokenCredential, azcloud.Configuration) (metricsQueryClient, error) {
507
- return &mockMetricsClient{}, nil
1160
+}
1161
+
1162
+func TestConfig_ValidateDiscoveryContracts(t *testing.T) {
1163
+ tests := map[string]struct {
1164
+ cfg Config
1165
+ wantErr bool
1166
+ wantErrContain string
1167
+ }{
1168
+ "filters mode ignores query block": {
1169
+ cfg: func() Config {
1170
+ cfg := testConfig()
1171
+ cfg.Discovery.Mode = discoveryModeFilters
1172
+ cfg.Discovery.ModeQuery = &DiscoveryQueryConfig{KQL: "resources | project id, name, type, resourceGroup, location"}
1173
+ return cfg
1174
+ }(),
1175
+ wantErr: false,
1176
+ },
1177
+ "query mode ignores filters block": {
1178
+ cfg: func() Config {
1179
+ cfg := testConfig()
1180
+ cfg.Discovery.Mode = discoveryModeQuery
1181
+ cfg.Discovery.ModeQuery = &DiscoveryQueryConfig{KQL: "resources | project id, name, type, resourceGroup, location"}
1182
+ cfg.Discovery.ModeFilters = &DiscoveryFiltersConfig{ResourceGroups: []string{"rg-a"}}
1183
+ return cfg
1184
+ }(),
1185
+ wantErr: false,
1186
+ },
1187
+ "query mode requires kql": {
1188
+ cfg: func() Config {
1189
+ cfg := testConfig()
1190
+ cfg.Discovery.Mode = discoveryModeQuery
1191
+ cfg.Discovery.ModeFilters = nil
1192
+ cfg.Discovery.ModeQuery = &DiscoveryQueryConfig{}
1193
+ return cfg
1194
+ }(),
1195
+ wantErr: true,
1196
+ wantErrContain: "'discovery.mode_query.kql' must not be empty when discovery.mode is 'query'",
1197
+ },
1198
+ "filters mode rejects empty resource group item": {
1199
+ cfg: func() Config {
1200
+ cfg := testConfig()
1201
+ cfg.Discovery.Mode = discoveryModeFilters
1202
+ cfg.Discovery.ModeFilters = &DiscoveryFiltersConfig{ResourceGroups: []string{""}}
1203
+ return cfg
1204
+ }(),
1205
+ wantErr: true,
1206
+ wantErrContain: "'discovery.mode_filters.resource_groups[0]' must not be empty",
1207
+ },
1208
+ "filters mode rejects empty region item": {
1209
+ cfg: func() Config {
1210
+ cfg := testConfig()
1211
+ cfg.Discovery.Mode = discoveryModeFilters
1212
+ cfg.Discovery.ModeFilters = &DiscoveryFiltersConfig{Regions: []string{" "}}
1213
+ return cfg
1214
+ }(),
1215
+ wantErr: true,
1216
+ wantErrContain: "'discovery.mode_filters.regions[0]' must not be empty",
1217
+ },
1218
+ "auto mode ignores explicit names": {
1219
+ cfg: func() Config {
1220
+ cfg := testConfig()
1221
+ cfg.Profiles.Mode = profilesModeAuto
1222
+ cfg.Profiles.ModeCombined = &ProfilesModeConfig{Names: []string{"cosmos_db"}}
1223
+ return cfg
1224
+ }(),
1225
+ wantErr: false,
1226
+ },
1227
+ "exact mode ignores combined block": {
1228
+ cfg: func() Config {
1229
+ cfg := testConfig()
1230
+ cfg.Profiles.Mode = profilesModeExact
1231
+ cfg.Profiles.ModeCombined = &ProfilesModeConfig{Names: []string{"cosmos_db"}}
1232
+ return cfg
1233
+ }(),
1234
+ wantErr: false,
1235
+ },
1236
+ "exact mode rejects duplicate names ignoring case": {
1237
+ cfg: func() Config {
1238
+ cfg := testConfig()
1239
+ cfg.Profiles.Mode = profilesModeExact
1240
+ cfg.Profiles.ModeExact = &ProfilesModeConfig{Names: []string{"POSTGRES_FLEXIBLE", "postgres_flexible"}}
1241
+ return cfg
1242
+ }(),
1243
+ wantErr: true,
1244
+ wantErrContain: "'profiles.mode_exact.names' contains duplicate value 'postgres_flexible'",
1245
+ },
1246
}
1247
510
- err := c.Init(context.Background())
511
- assert.Error(t, err)
512
- assert.Contains(t, err.Error(), "auto-discovery found no Azure resources")
1248
+ for name, tc := range tests {
1249
+ t.Run(name, func(t *testing.T) {
1250
+ err := tc.cfg.validate()
1251
+ if !tc.wantErr {
1252
+ require.NoError(t, err)
1253
+ return
1254
+ }
1255
+ require.Error(t, err)
1256
+ assert.ErrorContains(t, err, tc.wantErrContain)
1257
+ })
1258
+ }
1259
}
1260
1261
func TestMergeProfileIDs(t *testing.T) {
@@ -549,7 +1295,7 @@ func TestBuildCollectorRuntime_DetectsChartIDCollision(t *testing.T) {
1295
catalog := mustLoadStockCatalog(t, map[string]string{
1296
"redis_upper.yaml": `
1297
id: redis_upper
552
-name: Azure Redis Cache
1298
+name: Azure Redis Cache Upper
1299
resource_type: Microsoft.Cache/Redis
1300
metrics:
1301
- id: connectedclients
@@ -575,7 +1321,7 @@ template:
1321
`,
1322
"redis_lower.yaml": `
1323
id: redis_lower
578
-name: azure redis cache
1324
+name: Azure Redis Cache Lower
1325
resource_type: Microsoft.Cache/Redis
1326
metrics:
1327
- id: cachehits
@@ -607,19 +1353,24 @@ template:
1353
1354
func testConfig() Config {
1355
return Config{
610
- UpdateEvery: 60,
611
- AutoDetectionRetry: 0,
612
- SubscriptionID: "sub-1",
613
- Cloud: "public",
614
- DiscoveryEvery: 300,
615
- QueryOffset: 180,
616
- Timeout: defaultTimeout,
617
- MaxConcurrency: 4,
618
- MaxBatchResources: 50,
619
- MaxMetricsPerQuery: 20,
620
- ProfileSelectionMode: profileSelectionModeExact,
621
- ProfileSelectionModeExact: &ProfileSelectionModeExactConfig{
622
- Profiles: []string{"postgres_flexible"},
1356
+ UpdateEvery: 60,
1357
+ AutoDetectionRetry: 0,
1358
+ SubscriptionIDs: []string{"sub-1"},
1359
+ Cloud: "public",
1360
+ Discovery: DiscoveryConfig{
1361
+ RefreshEvery: 300,
1362
+ Mode: discoveryModeFilters,
1363
+ },
1364
+ Profiles: ProfilesConfig{
1365
+ Mode: profilesModeExact,
1366
+ ModeExact: &ProfilesModeConfig{Names: []string{"postgres_flexible"}},
1367
+ },
1368
+ QueryOffset: 180,
1369
+ Timeout: defaultTimeout,
1370
+ Limits: LimitsConfig{
1371
+ MaxConcurrency: 4,
1372
+ MaxBatchResources: 50,
1373
+ MaxMetricsPerQuery: 20,
1374
},
1375
Auth: cloudauth.AzureADAuthConfig{
1376
Mode: cloudauth.AzureADAuthModeDefault,
@@ -642,19 +1393,44 @@ func mustLoadStockCatalog(t *testing.T, files map[string]string) azureprofiles.C
1393
return catalog
1394
}
1395
1396
+func newTestCollectorWithMocks(rg *mockResourceGraph, mx *mockMetricsClient) *Collector {
1397
+ c := New()
1398
+ c.newResourceGraph = func(string, azcore.TokenCredential, azcloud.Configuration) (resourceGraphClient, error) {
1399
+ return rg, nil
1400
+ }
1401
+ c.newMetricsClient = func(string, azcore.TokenCredential, azcloud.Configuration) (metricsQueryClient, error) {
1402
+ return mx, nil
1403
+ }
1404
+ return c
1405
+}
1406
+
1407
type mockResourceGraph struct {
1408
mu sync.Mutex
1409
resources []map[string]any
1410
count int
1411
+ query string
1412
+ subs []string
1413
timeout time.Duration
1414
hasDL bool
1415
}
1416
653
-func (m *mockResourceGraph) Resources(ctx context.Context, _ armresourcegraph.QueryRequest, _ *armresourcegraph.ClientResourcesOptions) (armresourcegraph.ClientResourcesResponse, error) {
1417
+func (m *mockResourceGraph) Resources(ctx context.Context, req armresourcegraph.QueryRequest, _ *armresourcegraph.ClientResourcesOptions) (armresourcegraph.ClientResourcesResponse, error) {
1418
m.mu.Lock()
1419
defer m.mu.Unlock()
1420
1421
m.count++
1422
+ if req.Query != nil {
1423
+ m.query = *req.Query
1424
+ } else {
1425
+ m.query = ""
1426
+ }
1427
+ m.subs = m.subs[:0]
1428
+ for _, sub := range req.Subscriptions {
1429
+ if sub == nil {
1430
+ continue
1431
+ }
1432
+ m.subs = append(m.subs, *sub)
1433
+ }
1434
deadline, ok := ctx.Deadline()
1435
m.hasDL = ok
1436
if ok {
@@ -682,19 +1458,60 @@ func (m *mockResourceGraph) lastTimeout() (time.Duration, bool) {
1458
return m.timeout, m.hasDL
1459
}
1460
1461
+func (m *mockResourceGraph) lastQuery() string {
1462
+ m.mu.Lock()
1463
+ defer m.mu.Unlock()
1464
+ return m.query
1465
+}
1466
+
1467
+func (m *mockResourceGraph) lastSubscriptions() []string {
1468
+ m.mu.Lock()
1469
+ defer m.mu.Unlock()
1470
+ return append([]string(nil), m.subs...)
1471
+}
1472
+
1473
type mockMetricsClient struct {
686
- mu sync.Mutex
687
- queryResponse azmetrics.QueryResourcesResponse
688
- queryErr error
689
- count int
690
- timeout time.Duration
691
- hasDL bool
1474
+ mu sync.Mutex
1475
+ queryResponse azmetrics.QueryResourcesResponse
1476
+ queryResponses map[string]azmetrics.QueryResourcesResponse
1477
+ queryErr error
1478
+ queryErrors map[string]error
1479
+ queryCallsLog []metricsQueryCall
1480
+ count int
1481
+ subs []string
1482
+ timeout time.Duration
1483
+ hasDL bool
1484
+}
1485
+
1486
+type metricsQueryCall struct {
1487
+ SubscriptionID string
1488
+ MetricNames []string
1489
+ StartTime string
1490
+ EndTime string
1491
+ Interval string
1492
+ Aggregation string
1493
}
1494
694
-func (m *mockMetricsClient) QueryResources(ctx context.Context, _ string, _ string, _ []string, _ azmetrics.ResourceIDList, _ *azmetrics.QueryResourcesOptions) (azmetrics.QueryResourcesResponse, error) {
1495
+func (m *mockMetricsClient) QueryResources(ctx context.Context, subscriptionID string, _ string, metricNames []string, _ azmetrics.ResourceIDList, opts *azmetrics.QueryResourcesOptions) (azmetrics.QueryResourcesResponse, error) {
1496
m.mu.Lock()
1497
defer m.mu.Unlock()
1498
m.count++
1499
+ m.subs = append(m.subs, subscriptionID)
1500
+ var startTime, endTime, interval, aggregation string
1501
+ if opts != nil {
1502
+ startTime = derefString(opts.StartTime)
1503
+ endTime = derefString(opts.EndTime)
1504
+ interval = derefString(opts.Interval)
1505
+ aggregation = derefString(opts.Aggregation)
1506
+ }
1507
+ m.queryCallsLog = append(m.queryCallsLog, metricsQueryCall{
1508
+ SubscriptionID: subscriptionID,
1509
+ MetricNames: append([]string(nil), metricNames...),
1510
+ StartTime: startTime,
1511
+ EndTime: endTime,
1512
+ Interval: interval,
1513
+ Aggregation: aggregation,
1514
+ })
1515
deadline, ok := ctx.Deadline()
1516
m.hasDL = ok
1517
if ok {
@@ -702,9 +1519,15 @@ func (m *mockMetricsClient) QueryResources(ctx context.Context, _ string, _ stri
1519
} else {
1520
m.timeout = 0
1521
}
1522
+ if err, ok := m.queryErrors[subscriptionID]; ok && err != nil {
1523
+ return azmetrics.QueryResourcesResponse{}, err
1524
+ }
1525
if m.queryErr != nil {
1526
return azmetrics.QueryResourcesResponse{}, m.queryErr
1527
}
1528
+ if resp, ok := m.queryResponses[subscriptionID]; ok {
1529
+ return resp, nil
1530
+ }
1531
return m.queryResponse, nil
1532
}
1533
@@ -720,6 +1543,18 @@ func (m *mockMetricsClient) lastTimeout() (time.Duration, bool) {
1543
return m.timeout, m.hasDL
1544
}
1545
1546
+func (m *mockMetricsClient) subscriptionCalls() []string {
1547
+ m.mu.Lock()
1548
+ defer m.mu.Unlock()
1549
+ return append([]string(nil), m.subs...)
1550
+}
1551
+
1552
+func (m *mockMetricsClient) queryCalls() []metricsQueryCall {
1553
+ m.mu.Lock()
1554
+ defer m.mu.Unlock()
1555
+ return append([]metricsQueryCall(nil), m.queryCallsLog...)
1556
+}
1557
+
1558
func metricWithAvg(name string, ts time.Time, value float64) azmetrics.Metric {
1559
return azmetrics.Metric{
1560
Name: &azmetrics.LocalizableString{Value: ptrString(name)},
@@ -731,7 +1566,35 @@ func metricWithAvg(name string, ts time.Time, value float64) azmetrics.Metric {
1566
1567
func ptrString(v string) *string { return &v }
1568
1569
+func derefString(v *string) string {
1570
+ if v == nil {
1571
+ return ""
1572
+ }
1573
+ return *v
1574
+}
1575
+
1576
func assertTimeoutClose(t *testing.T, got, want time.Duration) {
1577
t.Helper()
1578
assert.InDelta(t, want.Seconds(), got.Seconds(), 1.0)
1579
}
1580
+
1581
+func uniqueStrings(values []string) []string {
1582
+ seen := make(map[string]struct{}, len(values))
1583
+ out := make([]string, 0, len(values))
1584
+ for _, value := range values {
1585
+ if _, ok := seen[value]; ok {
1586
+ continue
1587
+ }
1588
+ seen[value] = struct{}{}
1589
+ out = append(out, value)
1590
+ }
1591
+ return out
1592
+}
1593
+
1594
+func keysFromSeries(series map[string]metrix.SampleValue) []string {
1595
+ out := make([]string, 0, len(series))
1596
+ for key := range series {
1597
+ out = append(out, key)
1598
+ }
1599
+ return out
1600
+}
src/go/plugin/go.d/collector/azure_monitor/config.go
+173
-59
@@ -17,6 +17,8 @@ const (
17
defaultAutoDetectRetry = 0
18
defaultCloud = cloudPublic
19
defaultDiscoveryEvery = 300
20
+ defaultDiscoveryMode = discoveryModeFilters
21
+ defaultProfilesMode = profilesModeAuto
22
defaultQueryOffset = 180
23
defaultTimeout = confopt.Duration(30 * time.Second)
24
defaultMaxConcurrency = 4
@@ -25,9 +27,14 @@ const (
27
)
28
29
const (
28
- profileSelectionModeAuto = "auto"
29
- profileSelectionModeExact = "exact"
30
- profileSelectionModeCombined = "combined"
30
+ discoveryModeFilters = "filters"
31
+ discoveryModeQuery = "query"
32
+)
33
+
34
+const (
35
+ profilesModeAuto = "auto"
36
+ profilesModeExact = "exact"
37
+ profilesModeCombined = "combined"
38
)
39
40
const (
@@ -37,30 +44,50 @@ const (
44
)
45
46
type Config struct {
40
- Vnode string `yaml:"vnode,omitempty" json:"vnode"`
41
- UpdateEvery int `yaml:"update_every,omitempty" json:"update_every"`
42
- AutoDetectionRetry int `yaml:"autodetection_retry,omitempty" json:"autodetection_retry"`
43
- SubscriptionID string `yaml:"subscription_id" json:"subscription_id"`
44
- Cloud string `yaml:"cloud,omitempty" json:"cloud"`
45
- DiscoveryEvery int `yaml:"discovery_every,omitempty" json:"discovery_every"`
46
- QueryOffset int `yaml:"query_offset,omitempty" json:"query_offset"`
47
- Timeout confopt.Duration `yaml:"timeout,omitempty" json:"timeout"`
48
- MaxConcurrency int `yaml:"max_concurrency,omitempty" json:"max_concurrency"`
49
- MaxBatchResources int `yaml:"max_batch_resources,omitempty" json:"max_batch_resources"`
50
- MaxMetricsPerQuery int `yaml:"max_metrics_per_query,omitempty" json:"max_metrics_per_query"`
51
- ResourceGroups []string `yaml:"resource_groups,omitempty" json:"resource_groups"`
52
- ProfileSelectionMode string `yaml:"profile_selection_mode,omitempty" json:"profile_selection_mode"`
53
- ProfileSelectionModeExact *ProfileSelectionModeExactConfig `yaml:"profile_selection_mode_exact,omitempty" json:"profile_selection_mode_exact,omitempty"`
54
- ProfileSelectionModeCombined *ProfileSelectionModeCombinedConfig `yaml:"profile_selection_mode_combined,omitempty" json:"profile_selection_mode_combined,omitempty"`
55
- Auth cloudauth.AzureADAuthConfig `yaml:"auth" json:"auth"`
47
+ Vnode string `yaml:"vnode,omitempty" json:"vnode,omitempty"`
48
+ UpdateEvery int `yaml:"update_every,omitempty" json:"update_every,omitempty"`
49
+ AutoDetectionRetry int `yaml:"autodetection_retry,omitempty" json:"autodetection_retry,omitempty"`
50
+ SubscriptionIDs []string `yaml:"subscription_ids" json:"subscription_ids"`
51
+ Cloud string `yaml:"cloud,omitempty" json:"cloud"`
52
+ Discovery DiscoveryConfig `yaml:"discovery" json:"discovery"`
53
+ Profiles ProfilesConfig `yaml:"profiles" json:"profiles"`
54
+ QueryOffset int `yaml:"query_offset,omitempty" json:"query_offset"`
55
+ Timeout confopt.Duration `yaml:"timeout,omitempty" json:"timeout"`
56
+ Limits LimitsConfig `yaml:"limits" json:"limits"`
57
+ Auth cloudauth.AzureADAuthConfig `yaml:"auth" json:"auth"`
58
+}
59
+
60
+type DiscoveryConfig struct {
61
+ RefreshEvery int `yaml:"refresh_every,omitempty" json:"refresh_every"`
62
+ Mode string `yaml:"mode,omitempty" json:"mode"`
63
+ ModeFilters *DiscoveryFiltersConfig `yaml:"mode_filters,omitempty" json:"mode_filters,omitempty"`
64
+ ModeQuery *DiscoveryQueryConfig `yaml:"mode_query,omitempty" json:"mode_query,omitempty"`
65
+}
66
+
67
+type DiscoveryFiltersConfig struct {
68
+ ResourceGroups []string `yaml:"resource_groups,omitempty" json:"resource_groups,omitempty"`
69
+ Regions []string `yaml:"regions,omitempty" json:"regions,omitempty"`
70
+ Tags map[string][]string `yaml:"tags,omitempty" json:"tags,omitempty"`
71
+}
72
+
73
+type DiscoveryQueryConfig struct {
74
+ KQL string `yaml:"kql" json:"kql"`
75
}
76
58
-type ProfileSelectionModeExactConfig struct {
59
- Profiles []string `yaml:"profiles" json:"profiles"`
77
+type ProfilesModeConfig struct {
78
+ Names []string `yaml:"names,omitempty" json:"names,omitempty"`
79
}
80
62
-type ProfileSelectionModeCombinedConfig struct {
63
- Profiles []string `yaml:"profiles" json:"profiles"`
81
+type ProfilesConfig struct {
82
+ Mode string `yaml:"mode,omitempty" json:"mode"`
83
+ ModeExact *ProfilesModeConfig `yaml:"mode_exact,omitempty" json:"mode_exact,omitempty"`
84
+ ModeCombined *ProfilesModeConfig `yaml:"mode_combined,omitempty" json:"mode_combined,omitempty"`
85
+}
86
+
87
+type LimitsConfig struct {
88
+ MaxConcurrency int `yaml:"max_concurrency,omitempty" json:"max_concurrency"`
89
+ MaxBatchResources int `yaml:"max_batch_resources,omitempty" json:"max_batch_resources"`
90
+ MaxMetricsPerQuery int `yaml:"max_metrics_per_query,omitempty" json:"max_metrics_per_query"`
91
}
92
93
func (c *Config) applyDefaults() {
@@ -73,8 +100,14 @@ func (c *Config) applyDefaults() {
100
if strings.TrimSpace(c.Cloud) == "" {
101
c.Cloud = defaultCloud
102
}
76
- if c.DiscoveryEvery <= 0 {
77
- c.DiscoveryEvery = defaultDiscoveryEvery
103
+ if c.Discovery.RefreshEvery < 0 {
104
+ c.Discovery.RefreshEvery = defaultDiscoveryEvery
105
+ }
106
+ if strings.TrimSpace(c.Discovery.Mode) == "" {
107
+ c.Discovery.Mode = defaultDiscoveryMode
108
+ }
109
+ if strings.TrimSpace(c.Profiles.Mode) == "" {
110
+ c.Profiles.Mode = defaultProfilesMode
111
}
112
if c.QueryOffset <= 0 {
113
c.QueryOffset = defaultQueryOffset
@@ -82,31 +115,34 @@ func (c *Config) applyDefaults() {
115
if c.Timeout.Duration() == 0 {
116
c.Timeout = defaultTimeout
117
}
85
- if c.MaxConcurrency <= 0 {
86
- c.MaxConcurrency = defaultMaxConcurrency
87
- }
88
- if c.MaxBatchResources <= 0 {
89
- c.MaxBatchResources = defaultMaxBatchResource
118
+ if c.Limits.MaxConcurrency <= 0 {
119
+ c.Limits.MaxConcurrency = defaultMaxConcurrency
120
}
91
- if c.MaxMetricsPerQuery <= 0 {
92
- c.MaxMetricsPerQuery = defaultMaxMetricsQuery
121
+ if c.Limits.MaxBatchResources <= 0 {
122
+ c.Limits.MaxBatchResources = defaultMaxBatchResource
123
}
94
- if strings.TrimSpace(c.ProfileSelectionMode) == "" {
95
- c.ProfileSelectionMode = profileSelectionModeAuto
124
+ if c.Limits.MaxMetricsPerQuery <= 0 {
125
+ c.Limits.MaxMetricsPerQuery = defaultMaxMetricsQuery
126
}
127
}
128
129
func (c Config) validate() error {
130
var errs []error
131
102
- if strings.TrimSpace(c.SubscriptionID) == "" {
103
- errs = append(errs, errors.New("'subscription_id' is required"))
132
+ if len(c.SubscriptionIDs) == 0 {
133
+ errs = append(errs, errors.New("'subscription_ids' must contain at least one value"))
134
+ } else {
135
+ for i, v := range c.SubscriptionIDs {
136
+ if strings.TrimSpace(v) == "" {
137
+ errs = append(errs, fmt.Errorf("'subscription_ids[%d]' must not be empty", i))
138
+ }
139
+ }
140
}
141
if c.UpdateEvery < 60 {
142
errs = append(errs, errors.New("'update_every' must be >= 60 seconds"))
143
}
108
- if c.DiscoveryEvery < 60 {
109
- errs = append(errs, errors.New("'discovery_every' must be >= 60 seconds"))
144
+ if c.Discovery.RefreshEvery < 0 || (c.Discovery.RefreshEvery > 0 && c.Discovery.RefreshEvery < 60) {
145
+ errs = append(errs, errors.New("'discovery.refresh_every' must be 0 or >= 60 seconds"))
146
}
147
if c.QueryOffset < 60 {
148
errs = append(errs, errors.New("'query_offset' must be >= 60 seconds"))
@@ -114,14 +150,14 @@ func (c Config) validate() error {
150
if c.Timeout.Duration() < 0 {
151
errs = append(errs, errors.New("'timeout' cannot be negative"))
152
}
117
- if c.MaxConcurrency < 1 || c.MaxConcurrency > 64 {
118
- errs = append(errs, errors.New("'max_concurrency' must be between 1 and 64"))
153
+ if c.Limits.MaxConcurrency < 1 || c.Limits.MaxConcurrency > 64 {
154
+ errs = append(errs, errors.New("'limits.max_concurrency' must be between 1 and 64"))
155
}
120
- if c.MaxBatchResources < 1 || c.MaxBatchResources > 50 {
121
- errs = append(errs, errors.New("'max_batch_resources' must be between 1 and 50"))
156
+ if c.Limits.MaxBatchResources < 1 || c.Limits.MaxBatchResources > 50 {
157
+ errs = append(errs, errors.New("'limits.max_batch_resources' must be between 1 and 50"))
158
}
123
- if c.MaxMetricsPerQuery < 1 || c.MaxMetricsPerQuery > 20 {
124
- errs = append(errs, errors.New("'max_metrics_per_query' must be between 1 and 20"))
159
+ if c.Limits.MaxMetricsPerQuery < 1 || c.Limits.MaxMetricsPerQuery > 20 {
160
+ errs = append(errs, errors.New("'limits.max_metrics_per_query' must be between 1 and 20"))
161
}
162
163
switch strings.ToLower(strings.TrimSpace(c.Cloud)) {
@@ -134,42 +170,120 @@ func (c Config) validate() error {
170
errs = append(errs, err)
171
}
172
137
- switch strings.ToLower(strings.TrimSpace(c.ProfileSelectionMode)) {
138
- case profileSelectionModeAuto:
139
- case profileSelectionModeExact:
140
- if c.ProfileSelectionModeExact == nil || len(c.ProfileSelectionModeExact.Profiles) == 0 {
141
- errs = append(errs, errors.New("'profile_selection_mode_exact.profiles' must not be empty when mode is 'exact'"))
173
+ switch strings.ToLower(strings.TrimSpace(c.Discovery.Mode)) {
174
+ case discoveryModeFilters:
175
+ errs = append(errs, validateDiscoveryFilters(c.Discovery.ModeFilters)...)
176
+ case discoveryModeQuery:
177
+ if c.Discovery.ModeQuery == nil || strings.TrimSpace(c.Discovery.ModeQuery.KQL) == "" {
178
+ errs = append(errs, errors.New("'discovery.mode_query.kql' must not be empty when discovery.mode is 'query'"))
179
+ }
180
+ default:
181
+ errs = append(errs, fmt.Errorf("'discovery.mode' must be one of: %s, %s", discoveryModeFilters, discoveryModeQuery))
182
+ }
183
+
184
+ switch strings.ToLower(strings.TrimSpace(c.Profiles.Mode)) {
185
+ case profilesModeAuto:
186
+ case profilesModeExact:
187
+ if c.Profiles.ModeExact == nil || len(c.Profiles.ModeExact.Names) == 0 {
188
+ errs = append(errs, fmt.Errorf("'profiles.mode_exact.names' must not be empty when profiles.mode is '%s'", c.Profiles.Mode))
189
} else {
143
- errs = append(errs, validateProfilesList(c.ProfileSelectionModeExact.Profiles)...)
190
+ errs = append(errs, validateProfilesList("profiles.mode_exact.names", c.Profiles.ModeExact.Names)...)
191
}
145
- case profileSelectionModeCombined:
146
- if c.ProfileSelectionModeCombined == nil || len(c.ProfileSelectionModeCombined.Profiles) == 0 {
147
- errs = append(errs, errors.New("'profile_selection_mode_combined.profiles' must not be empty when mode is 'combined'"))
192
+ case profilesModeCombined:
193
+ if c.Profiles.ModeCombined == nil || len(c.Profiles.ModeCombined.Names) == 0 {
194
+ errs = append(errs, fmt.Errorf("'profiles.mode_combined.names' must not be empty when profiles.mode is '%s'", c.Profiles.Mode))
195
} else {
149
- errs = append(errs, validateProfilesList(c.ProfileSelectionModeCombined.Profiles)...)
196
+ errs = append(errs, validateProfilesList("profiles.mode_combined.names", c.Profiles.ModeCombined.Names)...)
197
}
198
default:
152
- errs = append(errs, fmt.Errorf("'profile_selection_mode' must be one of: %s, %s, %s",
153
- profileSelectionModeAuto, profileSelectionModeExact, profileSelectionModeCombined))
199
+ errs = append(errs, fmt.Errorf("'profiles.mode' must be one of: %s, %s, %s",
200
+ profilesModeAuto, profilesModeExact, profilesModeCombined))
201
}
202
203
return errors.Join(errs...)
204
}
205
159
-func validateProfilesList(profiles []string) []error {
206
+func validateDiscoveryFilters(filters *DiscoveryFiltersConfig) []error {
207
+ if filters == nil {
208
+ return nil
209
+ }
210
+
211
+ var errs []error
212
+ for i, v := range filters.ResourceGroups {
213
+ if strings.TrimSpace(v) == "" {
214
+ errs = append(errs, fmt.Errorf("'discovery.mode_filters.resource_groups[%d]' must not be empty", i))
215
+ }
216
+ }
217
+ for i, v := range filters.Regions {
218
+ if strings.TrimSpace(v) == "" {
219
+ errs = append(errs, fmt.Errorf("'discovery.mode_filters.regions[%d]' must not be empty", i))
220
+ }
221
+ }
222
+ for key, values := range filters.Tags {
223
+ if strings.TrimSpace(key) == "" {
224
+ errs = append(errs, errors.New("'discovery.mode_filters.tags' contains an empty key"))
225
+ continue
226
+ }
227
+ if len(values) == 0 {
228
+ errs = append(errs, fmt.Errorf("'discovery.mode_filters.tags.%s' must contain at least one value", key))
229
+ continue
230
+ }
231
+ for i, v := range values {
232
+ if strings.TrimSpace(v) == "" {
233
+ errs = append(errs, fmt.Errorf("'discovery.mode_filters.tags.%s[%d]' must not be empty", key, i))
234
+ }
235
+ }
236
+ }
237
+ return errs
238
+}
239
+
240
+func validateProfilesList(path string, profiles []string) []error {
241
var errs []error
242
seen := map[string]struct{}{}
243
for _, name := range profiles {
244
n := strings.TrimSpace(name)
245
if n == "" {
165
- errs = append(errs, errors.New("'profiles' contains an empty value"))
246
+ errs = append(errs, fmt.Errorf("'%s' contains an empty value", path))
247
continue
248
}
249
norm := stringsLowerTrim(n)
250
if _, ok := seen[norm]; ok {
170
- errs = append(errs, fmt.Errorf("'profiles' contains duplicate value '%s'", n))
251
+ errs = append(errs, fmt.Errorf("'%s' contains duplicate value '%s'", path, n))
252
}
253
seen[norm] = struct{}{}
254
}
255
return errs
256
}
257
+
258
+func (p ProfilesConfig) explicitBaseNames() []string {
259
+ switch stringsLowerTrim(p.Mode) {
260
+ case profilesModeExact:
261
+ if p.ModeExact != nil {
262
+ return p.ModeExact.Names
263
+ }
264
+ case profilesModeCombined:
265
+ if p.ModeCombined != nil {
266
+ return p.ModeCombined.Names
267
+ }
268
+ }
269
+ return nil
270
+}
271
+
272
+func (c Config) primarySubscriptionID() string {
273
+ for _, id := range c.SubscriptionIDs {
274
+ if v := strings.TrimSpace(id); v != "" {
275
+ return v
276
+ }
277
+ }
278
+ return ""
279
+}
280
+
281
+func (c Config) subscriptionIDs() []string {
282
+ out := make([]string, 0, len(c.SubscriptionIDs))
283
+ for _, id := range c.SubscriptionIDs {
284
+ if v := strings.TrimSpace(id); v != "" {
285
+ out = append(out, v)
286
+ }
287
+ }
288
+ return out
289
+}
src/go/plugin/go.d/collector/azure_monitor/config_schema.json
+340
-171
@@ -11,13 +11,6 @@
11
"minimum": 60,
12
"default": 60
13
},
14
- "timeout": {
15
- "title": "Timeout",
16
- "description": "Timeout for Azure Resource Graph and Azure Monitor API requests, in seconds.",
17
- "type": "number",
18
- "minimum": 0,
19
- "default": 30
20
- },
14
"autodetection_retry": {
15
"title": "Detection retry",
16
"description": "Recheck interval in seconds. Zero disables retries.",
@@ -25,11 +18,14 @@
18
"minimum": 0,
19
"default": 0
20
},
28
- "subscription_id": {
29
- "title": "Subscription ID",
30
- "description": "Azure subscription ID.",
31
- "type": "string",
32
- "default": ""
21
+ "subscription_ids": {
22
+ "title": "Subscription IDs",
23
+ "description": "Azure subscription IDs.",
24
+ "type": "array",
25
+ "items": {
26
+ "type": "string"
27
+ },
28
+ "minItems": 1
29
},
30
"cloud": {
31
"title": "Azure cloud",
@@ -42,66 +38,231 @@
38
],
39
"default": "public"
40
},
45
- "discovery_every": {
46
- "title": "Discovery every",
47
- "description": "Resource discovery refresh interval, in seconds.",
48
- "type": "integer",
49
- "minimum": 60,
50
- "default": 300
41
+ "discovery": {
42
+ "title": "Discovery",
43
+ "type": "object",
44
+ "properties": {
45
+ "refresh_every": {
46
+ "title": "Discovery every",
47
+ "description": "Resource discovery refresh interval, in seconds. Set to 0 to disable periodic runtime rediscovery after bootstrap.",
48
+ "type": "integer",
49
+ "minimum": 0,
50
+ "default": 300
51
+ },
52
+ "mode": {
53
+ "title": "Discovery mode",
54
+ "description": "How candidate Azure resources are selected.",
55
+ "type": "string",
56
+ "enum": [
57
+ "filters",
58
+ "query"
59
+ ],
60
+ "default": "filters"
61
+ }
62
+ },
63
+ "dependencies": {
64
+ "mode": {
65
+ "oneOf": [
66
+ {
67
+ "properties": {
68
+ "mode": {
69
+ "const": "filters"
70
+ },
71
+ "mode_filters": {
72
+ "title": "Structured filters",
73
+ "type": "object",
74
+ "properties": {
75
+ "resource_groups": {
76
+ "title": "Resource groups",
77
+ "description": "Optional list of Azure resource groups to include.",
78
+ "type": "array",
79
+ "items": {
80
+ "type": "string",
81
+ "minLength": 1
82
+ }
83
+ },
84
+ "regions": {
85
+ "title": "Regions",
86
+ "description": "Optional list of Azure regions to include.",
87
+ "type": "array",
88
+ "items": {
89
+ "type": "string",
90
+ "minLength": 1
91
+ }
92
+ },
93
+ "tags": {
94
+ "title": "Tags",
95
+ "description": "Optional tag filters. Keys map to one or more accepted values.",
96
+ "type": "object",
97
+ "additionalProperties": {
98
+ "type": "array",
99
+ "items": {
100
+ "type": "string"
101
+ },
102
+ "minItems": 1
103
+ }
104
+ }
105
+ }
106
+ }
107
+ }
108
+ },
109
+ {
110
+ "properties": {
111
+ "mode": {
112
+ "const": "query"
113
+ },
114
+ "mode_query": {
115
+ "title": "Custom query",
116
+ "type": "object",
117
+ "properties": {
118
+ "kql": {
119
+ "title": "Azure Resource Graph KQL",
120
+ "description": "Custom Azure Resource Graph query.",
121
+ "type": "string",
122
+ "minLength": 1
123
+ }
124
+ },
125
+ "required": [
126
+ "kql"
127
+ ]
128
+ }
129
+ },
130
+ "required": [
131
+ "mode_query"
132
+ ]
133
+ }
134
+ ]
135
+ }
136
+ }
137
+ },
138
+ "profiles": {
139
+ "title": "Profiles",
140
+ "type": "object",
141
+ "properties": {
142
+ "mode": {
143
+ "title": "Profiles mode",
144
+ "description": "How metric profiles are selected.",
145
+ "type": "string",
146
+ "enum": [
147
+ "auto",
148
+ "exact",
149
+ "combined"
150
+ ],
151
+ "default": "auto"
152
+ }
153
+ },
154
+ "dependencies": {
155
+ "mode": {
156
+ "oneOf": [
157
+ {
158
+ "properties": {
159
+ "mode": {
160
+ "const": "auto"
161
+ }
162
+ }
163
+ },
164
+ {
165
+ "properties": {
166
+ "mode": {
167
+ "const": "exact"
168
+ },
169
+ "mode_exact": {
170
+ "title": "Exact profiles",
171
+ "type": "object",
172
+ "properties": {
173
+ "names": {
174
+ "title": "Profile basenames",
175
+ "description": "Explicit profile file basenames used by `exact` mode. Matching is case-insensitive.",
176
+ "type": "array",
177
+ "items": {
178
+ "type": "string"
179
+ }
180
+ }
181
+ },
182
+ "required": [
183
+ "names"
184
+ ]
185
+ }
186
+ },
187
+ "required": [
188
+ "mode_exact"
189
+ ]
190
+ },
191
+ {
192
+ "properties": {
193
+ "mode": {
194
+ "const": "combined"
195
+ },
196
+ "mode_combined": {
197
+ "title": "Combined profiles",
198
+ "type": "object",
199
+ "properties": {
200
+ "names": {
201
+ "title": "Profile basenames",
202
+ "description": "Explicit profile file basenames merged with auto-discovered profiles in `combined` mode. Matching is case-insensitive.",
203
+ "type": "array",
204
+ "items": {
205
+ "type": "string"
206
+ }
207
+ }
208
+ },
209
+ "required": [
210
+ "names"
211
+ ]
212
+ }
213
+ },
214
+ "required": [
215
+ "mode_combined"
216
+ ]
217
+ }
218
+ ]
219
+ }
220
+ }
221
},
222
"query_offset": {
223
"title": "Query offset",
54
- "description": "How many seconds to shift queries into the past to avoid partial Azure Monitor windows.",
224
+ "description": "Minimum seconds to shift metric queries into the past; slower time-grain batches may use a larger effective offset automatically.",
225
"type": "integer",
226
"minimum": 60,
227
"default": 180
228
},
59
- "max_concurrency": {
60
- "title": "Max concurrency",
61
- "description": "Maximum concurrent Azure Monitor batch requests.",
62
- "type": "integer",
63
- "minimum": 1,
64
- "maximum": 64,
65
- "default": 4
66
- },
67
- "max_batch_resources": {
68
- "title": "Max resources per batch",
69
- "description": "Maximum resource IDs per batch API call.",
70
- "type": "integer",
71
- "minimum": 1,
72
- "maximum": 50,
73
- "default": 50
74
- },
75
- "max_metrics_per_query": {
76
- "title": "Max metrics per query",
77
- "description": "Maximum metric names per batch API call.",
78
- "type": "integer",
79
- "minimum": 1,
80
- "maximum": 20,
81
- "default": 20
229
+ "timeout": {
230
+ "title": "Timeout",
231
+ "description": "Timeout for Azure Resource Graph and Azure Monitor API requests, in seconds.",
232
+ "type": "number",
233
+ "minimum": 0,
234
+ "default": 30
235
},
83
- "resource_groups": {
84
- "title": "Resource groups",
85
- "description": "Optional list of Azure resource groups to include.",
86
- "type": [
87
- "array",
88
- "null"
89
- ],
90
- "items": {
91
- "type": "string"
236
+ "limits": {
237
+ "title": "Limits",
238
+ "type": "object",
239
+ "properties": {
240
+ "max_concurrency": {
241
+ "title": "Max concurrency",
242
+ "description": "Maximum concurrent Azure Monitor batch requests.",
243
+ "type": "integer",
244
+ "minimum": 1,
245
+ "maximum": 64,
246
+ "default": 4
247
+ },
248
+ "max_batch_resources": {
249
+ "title": "Max resources per batch",
250
+ "description": "Maximum resource IDs per batch API call.",
251
+ "type": "integer",
252
+ "minimum": 1,
253
+ "maximum": 50,
254
+ "default": 50
255
+ },
256
+ "max_metrics_per_query": {
257
+ "title": "Max metrics per query",
258
+ "description": "Maximum metric names per batch API call.",
259
+ "type": "integer",
260
+ "minimum": 1,
261
+ "maximum": 20,
262
+ "default": 20
263
+ }
264
}
265
},
94
- "profile_selection_mode": {
95
- "title": "Profile selection mode",
96
- "description": "How profiles are selected for monitoring.",
97
- "type": "string",
98
- "enum": [
99
- "auto",
100
- "exact",
101
- "combined"
102
- ],
103
- "default": "auto"
104
- },
266
"auth": {
267
"title": "Authentication",
268
"description": "Azure authentication mode and credentials.",
@@ -200,170 +361,178 @@
361
}
362
},
363
"required": [
203
- "subscription_id",
364
+ "subscription_ids",
365
"auth"
205
- ],
206
- "dependencies": {
207
- "profile_selection_mode": {
208
- "oneOf": [
209
- {
210
- "properties": {
211
- "profile_selection_mode": {
212
- "const": "auto"
213
- }
214
- }
215
- },
216
- {
217
- "properties": {
218
- "profile_selection_mode": {
219
- "const": "exact"
220
- },
221
- "profile_selection_mode_exact": {
222
- "title": "Exact profile selection",
223
- "description": "Settings for exact profile selection mode.",
224
- "type": "object",
225
- "properties": {
226
- "profiles": {
227
- "title": "Profiles",
228
- "description": "Profile ids to enable.",
229
- "type": "array",
230
- "items": {
231
- "type": "string"
232
- },
233
- "minItems": 1
234
- }
235
- },
236
- "required": [
237
- "profiles"
238
- ]
239
- }
240
- },
241
- "required": [
242
- "profile_selection_mode_exact"
243
- ]
244
- },
245
- {
246
- "properties": {
247
- "profile_selection_mode": {
248
- "const": "combined"
249
- },
250
- "profile_selection_mode_combined": {
251
- "title": "Combined profile selection",
252
- "description": "Settings for combined profile selection mode. Listed profiles are merged with auto-discovered profiles.",
253
- "type": "object",
254
- "properties": {
255
- "profiles": {
256
- "title": "Profiles",
257
- "description": "Profile ids to merge with auto-discovered profiles.",
258
- "type": "array",
259
- "items": {
260
- "type": "string"
261
- },
262
- "minItems": 1
263
- }
264
- },
265
- "required": [
266
- "profiles"
267
- ]
268
- }
269
- },
270
- "required": [
271
- "profile_selection_mode_combined"
272
- ]
273
- }
274
- ]
275
- }
276
- }
366
+ ]
367
},
368
"uiSchema": {
369
"uiOptions": {
370
"fullPage": true
371
},
282
- "resource_groups": {
283
- "ui:listFlavour": "list"
372
+ "subscription_ids": {
373
+ "ui:listFlavour": "list",
374
+ "ui:help": "Add one Azure subscription ID per row. This job will monitor resources across all listed subscriptions.",
375
+ "items": {
376
+ "ui:help": "Azure subscription ID.",
377
+ "ui:placeholder": "00000000-0000-0000-0000-000000000000"
378
+ }
379
},
285
- "profile_selection_mode": {
380
+ "cloud": {
381
+ "ui:help": "Select the Azure cloud environment that contains the subscriptions you want to monitor.",
382
"ui:widget": "radio",
383
"ui:options": {
384
"inline": true
289
- },
290
- "ui:help": "Choose how profiles are selected.\n\n- `auto`: Automatically discovers resource types in the subscription via Azure Resource Graph and enables matching profiles.\n- `exact`: Uses only the explicitly listed profile ids.\n- `combined`: Merges explicitly listed profile ids with auto-discovered profiles."
385
+ }
386
},
292
- "profile_selection_mode_exact": {
293
- "profiles": {
294
- "ui:listFlavour": "list"
387
+ "discovery": {
388
+ "mode": {
389
+ "ui:help": "Choose how the collector finds candidate Azure resources: `filters` uses structured subscription/resource filters, while `query` runs your custom Azure Resource Graph KQL.",
390
+ "ui:widget": "radio",
391
+ "ui:options": {
392
+ "inline": true
393
+ }
394
+ },
395
+ "mode_filters": {
396
+ "resource_groups": {
397
+ "ui:listFlavour": "list",
398
+ "ui:help": "Optional list of Azure resource groups to include. Leave empty to include matching resources from all resource groups.",
399
+ "items": {
400
+ "ui:help": "Azure resource group name.",
401
+ "ui:placeholder": "rg-production"
402
+ }
403
+ },
404
+ "regions": {
405
+ "ui:listFlavour": "list",
406
+ "ui:help": "Optional list of Azure regions to include. Leave empty to include matching resources from all regions.",
407
+ "items": {
408
+ "ui:help": "Azure region name.",
409
+ "ui:placeholder": "eastus"
410
+ }
411
+ },
412
+ "tags": {
413
+ "ui:help": "Optional tag filters. Add a tag key and then provide one or more accepted values for that key. Tag keys are matched case-insensitively and values are matched case-sensitively.",
414
+ "additionalProperties": {
415
+ "ui:listFlavour": "list",
416
+ "ui:help": "Add one or more accepted values for this tag key.",
417
+ "items": {
418
+ "ui:help": "Allowed value for the selected tag key.",
419
+ "ui:placeholder": "prod"
420
+ }
421
+ }
422
+ }
423
+ },
424
+ "mode_query": {
425
+ "kql": {
426
+ "ui:help": "Provide a custom Azure Resource Graph KQL query. The result must include resource `id`, `name`, `type`, `resourceGroup`, and `location`.",
427
+ "ui:placeholder": "resources | where tags.env == 'prod' | project id, name, type, resourceGroup, location",
428
+ "ui:widget": "textarea"
429
+ }
430
}
431
},
297
- "profile_selection_mode_combined": {
298
- "profiles": {
299
- "ui:listFlavour": "list"
432
+ "profiles": {
433
+ "mode": {
434
+ "ui:help": "Choose how metric profiles are activated: `auto` enables profiles for discovered resource types, `exact` uses only the listed profile basenames, and `combined` merges both.",
435
+ "ui:widget": "radio",
436
+ "ui:options": {
437
+ "inline": true
438
+ }
439
+ },
440
+ "mode_exact": {
441
+ "names": {
442
+ "ui:listFlavour": "list",
443
+ "ui:help": "Add Azure Monitor profile file basenames to enable explicitly in `exact` mode. Matching is case-insensitive.",
444
+ "items": {
445
+ "ui:help": "Profile file basename from the Azure Monitor profile catalog.",
446
+ "ui:placeholder": "sql_database"
447
+ }
448
+ }
449
+ },
450
+ "mode_combined": {
451
+ "names": {
452
+ "ui:listFlavour": "list",
453
+ "ui:help": "Add profile file basenames to merge with auto-discovered profiles in `combined` mode. Matching is case-insensitive.",
454
+ "items": {
455
+ "ui:help": "Profile file basename from the Azure Monitor profile catalog.",
456
+ "ui:placeholder": "sql_database"
457
+ }
458
+ }
459
}
460
},
461
"timeout": {
462
"ui:help": "Accepts decimals for sub-second granularity (for example, `0.5` for 500ms)."
463
},
305
- "cloud": {
306
- "ui:widget": "radio",
307
- "ui:options": {
308
- "inline": true
309
- }
310
- },
464
+ "limits": {},
465
"auth": {
466
"mode": {
467
+ "ui:help": "Choose how Netdata gets Azure credentials: `service_principal` uses explicit app credentials, `managed_identity` uses an attached Azure identity, and `default` uses the Azure SDK credential chain.",
468
"ui:widget": "radio",
469
"ui:options": {
470
"inline": true
316
- },
317
- "ui:help": "Choose how Netdata gets Azure credentials.\n\n- `service_principal`: Use an Azure app / service principal. Requires `tenant_id`, `client_id`, and `client_secret` in `mode_service_principal`.\n- `managed_identity`: Use the managed identity attached to the Azure resource running Netdata. Set `mode_managed_identity.client_id` only for a user-assigned identity.\n- `default`: Use the Azure SDK `DefaultAzureCredential` chain. This automatically tries available Azure credential sources, such as environment-based credentials, managed identity, and local developer credentials.\n\nUse `service_principal` for explicit app credentials. Use `managed_identity` when Netdata runs on an Azure resource with an attached identity. Use `default` when you want Azure SDK auto-discovery or local development convenience."
471
+ }
472
},
473
"mode_service_principal": {
474
+ "tenant_id": {
475
+ "ui:help": "Azure tenant ID for the service principal.",
476
+ "ui:placeholder": "00000000-0000-0000-0000-000000000000"
477
+ },
478
+ "client_id": {
479
+ "ui:help": "Azure client ID of the service principal application.",
480
+ "ui:placeholder": "00000000-0000-0000-0000-000000000000"
481
+ },
482
"client_secret": {
483
+ "ui:help": "Client secret for the Azure service principal.",
484
+ "ui:placeholder": "paste-client-secret-here",
485
"ui:widget": "password"
486
}
487
+ },
488
+ "mode_managed_identity": {
489
+ "client_id": {
490
+ "ui:help": "Optional client ID of a user-assigned managed identity. Leave empty to use the system-assigned managed identity.",
491
+ "ui:placeholder": "00000000-0000-0000-0000-000000000000"
492
+ }
493
}
494
},
495
+ "vnode": {
496
+ "ui:help": "Optional virtual node name used to group charts from this job under a specific Netdata virtual node.",
497
+ "ui:placeholder": "To use this option, first create a Virtual Node and then reference its name here."
498
+ },
499
"ui:flavour": "tabs",
500
"ui:options": {
501
"tabs": [
502
{
329
- "title": "Target",
503
+ "title": "Base",
504
"fields": [
331
- "subscription_id",
505
+ "subscription_ids",
506
"cloud",
507
+ "update_every",
508
+ "autodetection_retry",
509
+ "query_offset",
510
+ "timeout",
511
"vnode"
512
]
513
},
514
{
337
- "title": "Collection",
515
+ "title": "Auth",
516
"fields": [
339
- "update_every",
340
- "timeout",
341
- "autodetection_retry",
342
- "discovery_every",
343
- "query_offset"
517
+ "auth"
518
]
519
},
520
{
347
- "title": "Auth",
521
+ "title": "Discovery",
522
"fields": [
349
- "auth"
523
+ "discovery"
524
]
525
},
526
{
527
"title": "Profiles",
528
"fields": [
355
- "profile_selection_mode",
356
- "profile_selection_mode_exact",
357
- "profile_selection_mode_combined",
358
- "resource_groups"
529
+ "profiles"
530
]
531
},
532
{
533
"title": "Limits",
534
"fields": [
364
- "max_concurrency",
365
- "max_batch_resources",
366
- "max_metrics_per_query"
535
+ "limits"
536
]
537
}
538
]
src/go/plugin/go.d/collector/azure_monitor/discover.go
+429
-66
@@ -4,6 +4,7 @@ package azure_monitor
4
5
import (
6
"context"
7
+ "errors"
8
"fmt"
9
"slices"
10
"strings"
@@ -13,16 +14,29 @@ import (
14
"github.com/netdata/netdata/go/plugins/plugin/go.d/collector/azure_monitor/azureprofiles"
15
)
16
17
+type discoveryFetchResult struct {
18
+ Resources []resourceInfo
19
+ ByType map[string][]resourceInfo
20
+ UnsupportedTypes []string
21
+}
22
+
23
func (c *Collector) refreshDiscovery(ctx context.Context, force bool) ([]resourceInfo, error) {
24
now := c.now()
18
- if !force && !c.discovery.FetchedAt.IsZero() && now.Before(c.discovery.ExpiresAt) {
19
- return c.discovery.Resources, nil
25
+ if !force && !c.discovery.FetchedAt.IsZero() {
26
+ if c.Discovery.RefreshEvery == 0 || now.Before(c.discovery.ExpiresAt) {
27
+ return c.discovery.Resources, nil
28
+ }
29
}
30
22
- resources, byType, err := c.discoverResources(ctx)
31
+ fetched, err := c.fetchDiscovery(ctx)
32
if err != nil {
33
return nil, err
34
}
35
+ if len(fetched.UnsupportedTypes) > 0 {
36
+ c.Warningf("ignoring unsupported discovered resource types: %v", fetched.UnsupportedTypes)
37
+ }
38
+
39
+ resources, byType := filterDiscoveryResourcesByTypes(fetched.Resources, runtimeResourceTypes(c.runtime))
40
41
if !slices.Equal(resources, c.discovery.Resources) {
42
c.Infof("discovered %d resources: %v", len(resources), resources)
@@ -32,55 +46,55 @@ func (c *Collector) refreshDiscovery(ctx context.Context, force bool) ([]resourc
46
Resources: resources,
47
ByType: byType,
48
FetchedAt: now,
35
- ExpiresAt: now.Add(secondsToDuration(c.DiscoveryEvery)),
49
+ ExpiresAt: discoveryExpiresAt(now, c.Discovery.RefreshEvery),
50
FetchCounter: c.discovery.FetchCounter + 1,
51
}
52
53
return resources, nil
54
}
55
42
-// discoverResourceTypes queries Azure Resource Graph to find all resource types
43
-// present in the subscription. Used by auto-discovery to determine which profiles to activate.
44
-func discoverResourceTypes(ctx context.Context, subscriptionID string, timeout time.Duration, resourceGraph resourceGraphClient) (map[string]struct{}, error) {
45
- query := "resources | summarize count() by type"
46
- types := make(map[string]struct{})
47
-
48
- var skipToken *string
49
- for {
50
- req := armResourceGraphQuery(subscriptionID, query, skipToken)
51
- reqCtx, cancel := withOptionalTimeout(ctx, timeout)
52
- resp, err := resourceGraph.Resources(reqCtx, req, nil)
53
- cancel()
54
- if err != nil {
55
- return nil, fmt.Errorf("resource graph query: %w", err)
56
- }
56
+func discoveryExpiresAt(now time.Time, refreshEvery int) time.Time {
57
+ if refreshEvery == 0 {
58
+ return time.Time{}
59
+ }
60
+ return now.Add(secondsToDuration(refreshEvery))
61
+}
62
58
- rows, err := parseResourceGraphObjectArray(resp.Data)
63
+func (c *Collector) fetchDiscovery(ctx context.Context) (discoveryFetchResult, error) {
64
+ switch stringsLowerTrim(c.Discovery.Mode) {
65
+ case discoveryModeQuery:
66
+ return discoverResourcesFromQuery(
67
+ ctx,
68
+ c.subscriptionIDs(),
69
+ c.Timeout.Duration(),
70
+ c.resourceGraph,
71
+ c.Discovery.ModeQuery.KQL,
72
+ c.supportedResourceTypes,
73
+ )
74
+ default:
75
+ resources, byType, err := discoverResources(
76
+ ctx,
77
+ c.subscriptionIDs(),
78
+ c.Timeout.Duration(),
79
+ c.resourceGraph,
80
+ runtimeResourceTypes(c.runtime),
81
+ c.Discovery.ModeFilters,
82
+ )
83
if err != nil {
60
- return nil, err
61
- }
62
-
63
- for _, row := range rows {
64
- t := stringsLowerTrim(asString(row["type"]))
65
- if t != "" {
66
- types[t] = struct{}{}
67
- }
84
+ return discoveryFetchResult{}, err
85
}
69
-
70
- if resp.SkipToken == nil || stringsTrim(*resp.SkipToken) == "" {
71
- break
72
- }
73
- token := stringsTrim(*resp.SkipToken)
74
- skipToken = &token
86
+ return discoveryFetchResult{Resources: resources, ByType: byType}, nil
87
}
76
-
77
- return types, nil
88
}
89
80
-func (c *Collector) discoverResources(ctx context.Context) ([]resourceInfo, map[string][]resourceInfo, error) {
81
- resourceTypes := make([]string, 0, len(c.runtime.Profiles))
82
- seenTypes := map[string]struct{}{}
83
- for _, p := range c.runtime.Profiles {
90
+func runtimeResourceTypes(runtime *collectorRuntime) []string {
91
+ if runtime == nil || len(runtime.Profiles) == 0 {
92
+ return nil
93
+ }
94
+
95
+ resourceTypes := make([]string, 0, len(runtime.Profiles))
96
+ seenTypes := make(map[string]struct{}, len(runtime.Profiles))
97
+ for _, p := range runtime.Profiles {
98
t := stringsTrim(p.ResourceType)
99
if t == "" {
100
continue
@@ -92,23 +106,24 @@ func (c *Collector) discoverResources(ctx context.Context) ([]resourceInfo, map[
106
seenTypes[tLower] = struct{}{}
107
resourceTypes = append(resourceTypes, t)
108
}
109
+ return resourceTypes
110
+}
111
112
+func discoverResources(ctx context.Context, subscriptionIDs []string, timeout time.Duration, resourceGraph resourceGraphClient, resourceTypes []string, filters *DiscoveryFiltersConfig) ([]resourceInfo, map[string][]resourceInfo, error) {
113
if len(resourceTypes) == 0 {
114
return nil, map[string][]resourceInfo{}, nil
115
}
116
100
- query := buildDiscoveryQuery(resourceTypes)
117
+ query := buildDiscoveryQuery(resourceTypes, filters)
118
if query == "" {
119
return nil, nil, fmt.Errorf("failed to build resource discovery query")
120
}
121
105
- resourceGroupsFilter := make(map[string]struct{}, len(c.ResourceGroups))
106
- for _, rg := range c.ResourceGroups {
107
- n := stringsLowerTrim(rg)
108
- if n == "" {
109
- continue
110
- }
111
- resourceGroupsFilter[n] = struct{}{}
122
+ resourceGroupsFilter := normalizedDiscoveryFilterSet(nil)
123
+ regionsFilter := normalizedDiscoveryFilterSet(nil)
124
+ if filters != nil {
125
+ resourceGroupsFilter = normalizedDiscoveryFilterSet(filters.ResourceGroups)
126
+ regionsFilter = normalizedDiscoveryFilterSet(filters.Regions)
127
}
128
129
result := make([]resourceInfo, 0, 256)
@@ -116,9 +131,9 @@ func (c *Collector) discoverResources(ctx context.Context) ([]resourceInfo, map[
131
132
var skipToken *string
133
for {
119
- req := armResourceGraphQuery(c.SubscriptionID, query, skipToken)
120
- reqCtx, cancel := withOptionalTimeout(ctx, c.Timeout.Duration())
121
- resp, err := c.resourceGraph.Resources(reqCtx, req, nil)
134
+ req := armResourceGraphQuery(subscriptionIDs, query, skipToken)
135
+ reqCtx, cancel := withOptionalTimeout(ctx, timeout)
136
+ resp, err := resourceGraph.Resources(reqCtx, req, nil)
137
cancel()
138
if err != nil {
139
return nil, nil, err
@@ -148,21 +163,28 @@ func (c *Collector) discoverResources(ctx context.Context) ([]resourceInfo, map[
163
}
164
165
resourceType := stringsTrim(asString(row["type"]))
151
- if resourceType == "" {
166
+ subscriptionID, ok := parseARMResourceID(id)
167
+ if resourceType == "" || !ok {
168
continue
169
}
170
region := stringsLowerTrim(asString(row["location"]))
171
if region == "" {
172
region = "global"
173
}
174
+ if len(regionsFilter) > 0 {
175
+ if _, ok := regionsFilter[region]; !ok {
176
+ continue
177
+ }
178
+ }
179
180
result = append(result, resourceInfo{
160
- ID: id,
161
- UID: hashShort(id),
162
- Name: stringsTrim(asString(row["name"])),
163
- Type: resourceType,
164
- ResourceGroup: rg,
165
- Region: region,
181
+ SubscriptionID: subscriptionID,
182
+ ID: id,
183
+ UID: hashShort(id),
184
+ Name: stringsTrim(asString(row["name"])),
185
+ Type: resourceType,
186
+ ResourceGroup: rg,
187
+ Region: region,
188
})
189
}
190
@@ -182,7 +204,73 @@ func (c *Collector) discoverResources(ctx context.Context) ([]resourceInfo, map[
204
return result, byType, nil
205
}
206
185
-func armResourceGraphQuery(subscriptionID, query string, skipToken *string) armresourcegraph.QueryRequest {
207
+func discoverResourcesFromQuery(ctx context.Context, subscriptionIDs []string, timeout time.Duration, resourceGraph resourceGraphClient, kql string, supportedTypes map[string]struct{}) (discoveryFetchResult, error) {
208
+ query := stringsTrim(kql)
209
+ if query == "" {
210
+ return discoveryFetchResult{}, errors.New("custom discovery query is empty")
211
+ }
212
+
213
+ result := make([]resourceInfo, 0, 256)
214
+ byType := make(map[string][]resourceInfo)
215
+ unsupported := make(map[string]struct{})
216
+ seenIDs := make(map[string]struct{})
217
+
218
+ var skipToken *string
219
+ for {
220
+ req := armResourceGraphQuery(subscriptionIDs, query, skipToken)
221
+ reqCtx, cancel := withOptionalTimeout(ctx, timeout)
222
+ resp, err := resourceGraph.Resources(reqCtx, req, nil)
223
+ cancel()
224
+ if err != nil {
225
+ return discoveryFetchResult{}, err
226
+ }
227
+
228
+ rows, err := parseResourceGraphObjectArray(resp.Data)
229
+ if err != nil {
230
+ return discoveryFetchResult{}, err
231
+ }
232
+
233
+ for i, row := range rows {
234
+ resource, err := parseStrictQueryDiscoveryRow(row)
235
+ if err != nil {
236
+ return discoveryFetchResult{}, fmt.Errorf("query result row %d: %w", i, err)
237
+ }
238
+
239
+ idKey := stringsLowerTrim(resource.ID)
240
+ if _, ok := seenIDs[idKey]; ok {
241
+ return discoveryFetchResult{}, fmt.Errorf("query result contains duplicate id %q", resource.ID)
242
+ }
243
+ seenIDs[idKey] = struct{}{}
244
+
245
+ result = append(result, resource)
246
+ typeKey := stringsLowerTrim(resource.Type)
247
+ byType[typeKey] = append(byType[typeKey], resource)
248
+ if _, ok := supportedTypes[typeKey]; !ok {
249
+ unsupported[typeKey] = struct{}{}
250
+ }
251
+ }
252
+
253
+ if resp.SkipToken == nil || stringsTrim(*resp.SkipToken) == "" {
254
+ break
255
+ }
256
+ token := stringsTrim(*resp.SkipToken)
257
+ skipToken = &token
258
+ }
259
+
260
+ unsupportedTypes := make([]string, 0, len(unsupported))
261
+ for resourceType := range unsupported {
262
+ unsupportedTypes = append(unsupportedTypes, resourceType)
263
+ }
264
+ slices.Sort(unsupportedTypes)
265
+
266
+ return discoveryFetchResult{
267
+ Resources: result,
268
+ ByType: byType,
269
+ UnsupportedTypes: unsupportedTypes,
270
+ }, nil
271
+}
272
+
273
+func armResourceGraphQuery(subscriptionIDs []string, query string, skipToken *string) armresourcegraph.QueryRequest {
274
resultFormat := armresourcegraph.ResultFormatObjectArray
275
top := int32(1000)
276
options := &armresourcegraph.QueryRequestOptions{
@@ -194,19 +282,58 @@ func armResourceGraphQuery(subscriptionID, query string, skipToken *string) armr
282
options.SkipToken = &token
283
}
284
197
- subID := stringsTrim(subscriptionID)
285
+ subs := make([]*string, 0, len(subscriptionIDs))
286
+ for _, subID := range subscriptionIDs {
287
+ subID = stringsTrim(subID)
288
+ if subID == "" {
289
+ continue
290
+ }
291
+ subscription := subID
292
+ subs = append(subs, &subscription)
293
+ }
294
+
295
return armresourcegraph.QueryRequest{
296
Query: &query,
200
- Subscriptions: []*string{&subID},
297
+ Subscriptions: subs,
298
Options: options,
299
}
300
}
301
205
-func buildDiscoveryQuery(resourceTypes []string) string {
302
+func parseARMResourceID(resourceID string) (string, bool) {
303
+ resourceID = stringsTrim(resourceID)
304
+ if resourceID == "" || !strings.HasPrefix(resourceID, "/") {
305
+ return "", false
306
+ }
307
+
308
+ parts := strings.Split(strings.Trim(resourceID, "/"), "/")
309
+ if len(parts) < 6 || len(parts)%2 != 0 {
310
+ return "", false
311
+ }
312
+ if !strings.EqualFold(parts[0], "subscriptions") || stringsTrim(parts[1]) == "" {
313
+ return "", false
314
+ }
315
+
316
+ hasProviders := false
317
+ for i := 0; i+1 < len(parts); i += 2 {
318
+ if stringsTrim(parts[i]) == "" || stringsTrim(parts[i+1]) == "" {
319
+ return "", false
320
+ }
321
+ if strings.EqualFold(parts[i], "providers") {
322
+ hasProviders = true
323
+ }
324
+ }
325
+ if !hasProviders {
326
+ return "", false
327
+ }
328
+
329
+ return stringsTrim(parts[1]), true
330
+}
331
+
332
+func buildDiscoveryQuery(resourceTypes []string, filters *DiscoveryFiltersConfig) string {
333
if len(resourceTypes) == 0 {
334
return ""
335
}
209
- quoted := make([]string, 0, len(resourceTypes))
336
+ quotedTypes := make([]string, 0, len(resourceTypes))
337
for _, rt := range resourceTypes {
338
rt = stringsTrim(rt)
339
if rt == "" {
@@ -215,12 +342,248 @@ func buildDiscoveryQuery(resourceTypes []string) string {
342
if !azureprofiles.IsValidResourceType(rt) {
343
continue
344
}
218
- quoted = append(quoted, "'"+strings.ReplaceAll(rt, "'", "''")+"'")
345
+ quotedTypes = append(quotedTypes, quoteKQLString(rt))
346
}
220
- if len(quoted) == 0 {
347
+ if len(quotedTypes) == 0 {
348
return ""
349
}
223
- return "resources | where type in~ (" + strings.Join(quoted, ",") + ") | project id, name, type, resourceGroup, location"
350
+
351
+ query := "resources | where type in~ (" + strings.Join(quotedTypes, ", ") + ")"
352
+
353
+ if filters == nil {
354
+ return query + " | project id, name, type, resourceGroup, location"
355
+ }
356
+
357
+ if groups := normalizeDiscoveryFilterValues(filters.ResourceGroups); len(groups) > 0 {
358
+ quotedGroups := make([]string, 0, len(groups))
359
+ for _, rg := range groups {
360
+ quotedGroups = append(quotedGroups, quoteKQLString(rg))
361
+ }
362
+ query += " | where resourceGroup in~ (" + strings.Join(quotedGroups, ", ") + ")"
363
+ }
364
+
365
+ if regions := normalizeDiscoveryFilterValues(filters.Regions); len(regions) > 0 {
366
+ quotedRegions := make([]string, 0, len(regions))
367
+ for _, region := range regions {
368
+ quotedRegions = append(quotedRegions, quoteKQLString(region))
369
+ }
370
+ query += " | where location in~ (" + strings.Join(quotedRegions, ", ") + ")"
371
+ }
372
+
373
+ if tagFilters := normalizeDiscoveryTagFilters(filters.Tags); len(tagFilters) > 0 {
374
+ query += " | mv-expand bagexpansion=array tags"
375
+ query += " | where isnotempty(tags)"
376
+ query += " | extend tagKey = tostring(tags[0]), tagValue = tostring(tags[1])"
377
+ query += " | where " + buildDiscoveryTagPredicate(tagFilters)
378
+ query += " | summarize by id, name, type, resourceGroup, location, matchedTagKey = tolower(tagKey)"
379
+ query += " | summarize matchedTagKeys = count() by id, name, type, resourceGroup, location"
380
+ query += fmt.Sprintf(" | where matchedTagKeys == %d", len(tagFilters))
381
+ }
382
+
383
+ return query + " | project id, name, type, resourceGroup, location"
384
+}
385
+
386
+func normalizeDiscoveryFilterValues(values []string) []string {
387
+ seen := make(map[string]struct{}, len(values))
388
+ out := make([]string, 0, len(values))
389
+
390
+ for _, v := range values {
391
+ n := stringsLowerTrim(v)
392
+ if n == "" {
393
+ continue
394
+ }
395
+ if _, ok := seen[n]; ok {
396
+ continue
397
+ }
398
+ seen[n] = struct{}{}
399
+ out = append(out, n)
400
+ }
401
+
402
+ slices.Sort(out)
403
+ return out
404
+}
405
+
406
+type discoveryTagFilter struct {
407
+ Key string
408
+ Values []string
409
+}
410
+
411
+func normalizeDiscoveryTagFilters(tags map[string][]string) []discoveryTagFilter {
412
+ if len(tags) == 0 {
413
+ return nil
414
+ }
415
+
416
+ out := make([]discoveryTagFilter, 0, len(tags))
417
+ for key, values := range tags {
418
+ normalizedKey := stringsLowerTrim(key)
419
+ if normalizedKey == "" {
420
+ continue
421
+ }
422
+
423
+ seenValues := make(map[string]struct{}, len(values))
424
+ normalizedValues := make([]string, 0, len(values))
425
+ for _, value := range values {
426
+ trimmed := stringsTrim(value)
427
+ if trimmed == "" {
428
+ continue
429
+ }
430
+ if _, ok := seenValues[trimmed]; ok {
431
+ continue
432
+ }
433
+ seenValues[trimmed] = struct{}{}
434
+ normalizedValues = append(normalizedValues, trimmed)
435
+ }
436
+ if len(normalizedValues) == 0 {
437
+ continue
438
+ }
439
+
440
+ slices.Sort(normalizedValues)
441
+ out = append(out, discoveryTagFilter{Key: normalizedKey, Values: normalizedValues})
442
+ }
443
+
444
+ slices.SortFunc(out, func(a, b discoveryTagFilter) int {
445
+ switch {
446
+ case a.Key < b.Key:
447
+ return -1
448
+ case a.Key > b.Key:
449
+ return 1
450
+ default:
451
+ return 0
452
+ }
453
+ })
454
+ return out
455
+}
456
+
457
+func buildDiscoveryTagPredicate(filters []discoveryTagFilter) string {
458
+ clauses := make([]string, 0, len(filters))
459
+ for _, filter := range filters {
460
+ keyClause := "tagKey =~ " + quoteKQLString(filter.Key)
461
+ valueClause := buildDiscoveryTagValueClause(filter.Values)
462
+ clauses = append(clauses, "("+keyClause+" and "+valueClause+")")
463
+ }
464
+ return strings.Join(clauses, " or ")
465
+}
466
+
467
+func buildDiscoveryTagValueClause(values []string) string {
468
+ if len(values) == 1 {
469
+ return "tagValue == " + quoteKQLString(values[0])
470
+ }
471
+
472
+ quoted := make([]string, 0, len(values))
473
+ for _, value := range values {
474
+ quoted = append(quoted, quoteKQLString(value))
475
+ }
476
+ return "tagValue in (" + strings.Join(quoted, ", ") + ")"
477
+}
478
+
479
+func normalizedDiscoveryFilterSet(values []string) map[string]struct{} {
480
+ if len(values) == 0 {
481
+ return nil
482
+ }
483
+
484
+ set := make(map[string]struct{}, len(values))
485
+ for _, value := range normalizeDiscoveryFilterValues(values) {
486
+ set[value] = struct{}{}
487
+ }
488
+ return set
489
+}
490
+
491
+func parseStrictQueryDiscoveryRow(row map[string]any) (resourceInfo, error) {
492
+ id, err := strictQueryStringColumn(row, "id")
493
+ if err != nil {
494
+ return resourceInfo{}, err
495
+ }
496
+ subscriptionID, ok := parseARMResourceID(id)
497
+ if !ok {
498
+ return resourceInfo{}, fmt.Errorf("invalid ARM resource id %q", id)
499
+ }
500
+
501
+ name, err := strictQueryStringColumn(row, "name")
502
+ if err != nil {
503
+ return resourceInfo{}, err
504
+ }
505
+ resourceType, err := strictQueryStringColumn(row, "type")
506
+ if err != nil {
507
+ return resourceInfo{}, err
508
+ }
509
+ if resourceType == "" {
510
+ return resourceInfo{}, errors.New("column 'type' must not be empty")
511
+ }
512
+
513
+ resourceGroup, err := strictQueryStringColumn(row, "resourceGroup")
514
+ if err != nil {
515
+ return resourceInfo{}, err
516
+ }
517
+ location, err := strictQueryStringColumn(row, "location")
518
+ if err != nil {
519
+ return resourceInfo{}, err
520
+ }
521
+ region := stringsLowerTrim(location)
522
+ if region == "" {
523
+ region = "global"
524
+ }
525
+
526
+ return resourceInfo{
527
+ SubscriptionID: subscriptionID,
528
+ ID: id,
529
+ UID: hashShort(id),
530
+ Name: name,
531
+ Type: resourceType,
532
+ ResourceGroup: resourceGroup,
533
+ Region: region,
534
+ }, nil
535
+}
536
+
537
+func strictQueryStringColumn(row map[string]any, column string) (string, error) {
538
+ value, ok := row[column]
539
+ if !ok {
540
+ return "", fmt.Errorf("missing required column %q", column)
541
+ }
542
+
543
+ switch v := value.(type) {
544
+ case string:
545
+ return stringsTrim(v), nil
546
+ case fmt.Stringer:
547
+ return stringsTrim(v.String()), nil
548
+ default:
549
+ return "", fmt.Errorf("column %q must be a string", column)
550
+ }
551
+}
552
+
553
+func filterDiscoveryResourcesByTypes(resources []resourceInfo, allowedTypes []string) ([]resourceInfo, map[string][]resourceInfo) {
554
+ if len(resources) == 0 || len(allowedTypes) == 0 {
555
+ return nil, map[string][]resourceInfo{}
556
+ }
557
+
558
+ allowed := make(map[string]struct{}, len(allowedTypes))
559
+ for _, resourceType := range allowedTypes {
560
+ allowed[stringsLowerTrim(resourceType)] = struct{}{}
561
+ }
562
+
563
+ filtered := make([]resourceInfo, 0, len(resources))
564
+ byType := make(map[string][]resourceInfo)
565
+ for _, resource := range resources {
566
+ typeKey := stringsLowerTrim(resource.Type)
567
+ if _, ok := allowed[typeKey]; !ok {
568
+ continue
569
+ }
570
+ filtered = append(filtered, resource)
571
+ byType[typeKey] = append(byType[typeKey], resource)
572
+ }
573
+
574
+ return filtered, byType
575
+}
576
+
577
+func catalogResourceTypeSet(catalog azureprofiles.Catalog) map[string]struct{} {
578
+ types := make(map[string]struct{})
579
+ for _, resourceType := range catalog.ResourceTypes() {
580
+ types[stringsLowerTrim(resourceType)] = struct{}{}
581
+ }
582
+ return types
583
+}
584
+
585
+func quoteKQLString(v string) string {
586
+ return "'" + strings.ReplaceAll(v, "'", "''") + "'"
587
}
588
589
func parseResourceGraphObjectArray(v any) ([]map[string]any, error) {
src/go/plugin/go.d/collector/azure_monitor/init.go
+147
-53
@@ -6,7 +6,6 @@ import (
6
"context"
7
"errors"
8
"fmt"
9
- "time"
9
10
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
11
azcloud "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud"
@@ -17,10 +16,11 @@ import (
16
)
17
18
type initResult struct {
20
- config Config
21
- resourceGraph resourceGraphClient
22
- queryExecutor *queryExecutor
23
- runtime *collectorRuntime
19
+ config Config
20
+ profileCatalog azureprofiles.Catalog
21
+ resourceGraph resourceGraphClient
22
+ queryExecutor *queryExecutor
23
+ supportedResourceTypes map[string]struct{}
24
}
25
26
func (c *Collector) initInstruments(runtime *collectorRuntime) error {
@@ -28,7 +28,7 @@ func (c *Collector) initInstruments(runtime *collectorRuntime) error {
28
return errors.New("nil collector runtime")
29
}
30
31
- var labelKeys = []string{"resource_uid", "resource_name", "resource_group", "region", "resource_type", "profile"}
31
+ var labelKeys = []string{"resource_uid", "subscription_id", "resource_name", "resource_group", "region", "resource_type", "profile"}
32
33
vec := c.store.Write().SnapshotMeter("").Vec(labelKeys...)
34
if runtime.Instruments == nil {
@@ -56,8 +56,8 @@ func (c *Collector) initInstruments(runtime *collectorRuntime) error {
56
return nil
57
}
58
59
-func (c *Collector) prepareInitResult(ctx context.Context) (*initResult, error) {
60
- cfg, catalog, autoDiscover, explicitProfiles, err := c.prepareInitConfig()
59
+func (c *Collector) prepareInitResult() (*initResult, error) {
60
+ cfg, catalog, err := c.prepareInitConfig()
61
if err != nil {
62
return nil, err
63
}
@@ -67,62 +67,87 @@ func (c *Collector) prepareInitResult(ctx context.Context) (*initResult, error)
67
return nil, err
68
}
69
70
- var profileIDs []string
70
+ supportedResourceTypes := catalogResourceTypeSet(catalog)
71
+ return &initResult{
72
+ config: cfg,
73
+ profileCatalog: catalog,
74
+ resourceGraph: resourceGraph,
75
+ queryExecutor: queryExecutor,
76
+ supportedResourceTypes: supportedResourceTypes,
77
+ }, nil
78
+}
79
72
- if autoDiscover {
73
- profiles, err := resolveAutoProfiles(ctx, cfg.SubscriptionID, cfg.Timeout.Duration(), resourceGraph, catalog, explicitProfiles)
74
- if err != nil {
75
- return nil, fmt.Errorf("auto-discover resource types: %w", err)
76
- }
77
- if len(profiles) == 0 {
78
- return nil, errors.New("auto-discovery found no Azure resources matching any known profile")
79
- }
80
- c.Infof("auto-discovery resolved profiles: %v", profiles)
81
- profileIDs = profiles
82
- } else {
83
- profileIDs = explicitProfiles
80
+func (c *Collector) ensureBootstrapped(ctx context.Context) error {
81
+ if c.runtime != nil {
82
+ return nil
83
+ }
84
+ if c.resourceGraph == nil || c.queryExecutor == nil {
85
+ return errors.New("collector is not initialized")
86
+ }
87
+ if len(c.supportedResourceTypes) == 0 && len(c.profileCatalog.ResourceTypes()) == 0 {
88
+ return errors.New("collector profile catalog is not initialized")
89
}
90
86
- runtime, err := buildCollectorRuntimeFromConfig(profileIDs, catalog)
91
+ fetched, err := fetchInitDiscovery(ctx, c.Config, c.profileCatalog, c.resourceGraph, c.supportedResourceTypes)
92
if err != nil {
88
- return nil, fmt.Errorf("build collector runtime: %w", err)
93
+ return err
94
+ }
95
+ if len(fetched.UnsupportedTypes) > 0 {
96
+ c.Warningf("ignoring unsupported discovered resource types: %v", fetched.UnsupportedTypes)
97
}
98
91
- return &initResult{
92
- config: cfg,
93
- resourceGraph: resourceGraph,
94
- queryExecutor: queryExecutor,
95
- runtime: runtime,
96
- }, nil
99
+ profileIDs, autoProfiles, err := resolveInitProfileIDs(c.Config, c.profileCatalog, fetched.ByType)
100
+ if err != nil {
101
+ return err
102
+ }
103
+ if len(autoProfiles) > 0 {
104
+ c.Infof("auto-discovery resolved profiles: %v", autoProfiles)
105
+ }
106
+
107
+ // TODO(azure_monitor): Insert bootstrap-only metric-definition validation here.
108
+ // Resolve selected profiles, fetch one representative resource per
109
+ // (subscription_id, resource_type, metric_namespace), fail open per key on
110
+ // lookup errors, and prune unsupported metrics/aggregations/time grains
111
+ // before final runtime build. Because runtime is currently global per job,
112
+ // multi-subscription capability differences need an explicit merge rule first.
113
+ runtime, err := buildCollectorRuntimeFromConfig(profileIDs, c.profileCatalog)
114
+ if err != nil {
115
+ return fmt.Errorf("build collector runtime: %w", err)
116
+ }
117
+ if err := c.initInstruments(runtime); err != nil {
118
+ return err
119
+ }
120
+
121
+ resources, byType := filterDiscoveryResourcesByTypes(fetched.Resources, runtimeResourceTypes(runtime))
122
+ now := c.now()
123
+
124
+ c.runtime = runtime
125
+ c.observations = newObservationState(runtime.Instruments)
126
+ c.discovery = discoveryState{
127
+ Resources: resources,
128
+ ByType: byType,
129
+ FetchedAt: now,
130
+ ExpiresAt: discoveryExpiresAt(now, c.Discovery.RefreshEvery),
131
+ FetchCounter: 1,
132
+ }
133
+
134
+ return nil
135
}
136
99
-func (c *Collector) prepareInitConfig() (Config, azureprofiles.Catalog, bool, []string, error) {
137
+func (c *Collector) prepareInitConfig() (Config, azureprofiles.Catalog, error) {
138
cfg := c.Config
139
cfg.applyDefaults()
140
141
catalog, err := c.loadProfileCatalog()
142
if err != nil {
105
- return Config{}, azureprofiles.Catalog{}, false, nil, fmt.Errorf("load profiles catalog: %w", err)
143
+ return Config{}, azureprofiles.Catalog{}, fmt.Errorf("load profiles catalog: %w", err)
144
}
145
146
if err := cfg.validate(); err != nil {
109
- return Config{}, azureprofiles.Catalog{}, false, nil, fmt.Errorf("config validation: %w", err)
110
- }
111
-
112
- var autoDiscover bool
113
- var explicitProfiles []string
114
-
115
- switch stringsLowerTrim(cfg.ProfileSelectionMode) {
116
- case profileSelectionModeAuto:
117
- autoDiscover = true
118
- case profileSelectionModeExact:
119
- explicitProfiles = cfg.ProfileSelectionModeExact.Profiles
120
- case profileSelectionModeCombined:
121
- autoDiscover = true
122
- explicitProfiles = cfg.ProfileSelectionModeCombined.Profiles
147
+ return Config{}, azureprofiles.Catalog{}, fmt.Errorf("config validation: %w", err)
148
}
149
125
- return cfg, catalog, autoDiscover, explicitProfiles, nil
150
+ return cfg, catalog, nil
151
}
152
153
func (c *Collector) prepareInitClients(cfg Config) (resourceGraphClient, *queryExecutor, error) {
@@ -136,21 +161,90 @@ func (c *Collector) prepareInitClients(cfg Config) (resourceGraphClient, *queryE
161
return nil, nil, fmt.Errorf("create azure credential: %w", err)
162
}
163
139
- resourceGraph, err := c.newResourceGraph(cfg.SubscriptionID, credential, cloudCfg)
164
+ subscriptionID := cfg.primarySubscriptionID()
165
+ resourceGraph, err := c.newResourceGraph(subscriptionID, credential, cloudCfg)
166
if err != nil {
167
return nil, nil, fmt.Errorf("create resource graph client: %w", err)
168
}
169
144
- return resourceGraph, newQueryExecutor(cfg.SubscriptionID, cfg.MaxConcurrency, cfg.Timeout.Duration(), credential, cloudCfg, c.newMetricsClient), nil
170
+ return resourceGraph, newQueryExecutor(cfg.Limits.MaxConcurrency, cfg.Timeout.Duration(), credential, cloudCfg, c.newMetricsClient), nil
171
}
172
147
-func resolveAutoProfiles(ctx context.Context, subscriptionID string, timeout time.Duration, resourceGraph resourceGraphClient, catalog azureprofiles.Catalog, explicitProfiles []string) ([]string, error) {
148
- types, err := discoverResourceTypes(ctx, subscriptionID, timeout, resourceGraph)
173
+func fetchInitDiscovery(ctx context.Context, cfg Config, catalog azureprofiles.Catalog, resourceGraph resourceGraphClient, supportedResourceTypes map[string]struct{}) (discoveryFetchResult, error) {
174
+ if stringsLowerTrim(cfg.Discovery.Mode) == discoveryModeQuery {
175
+ fetched, err := discoverResourcesFromQuery(
176
+ ctx,
177
+ cfg.subscriptionIDs(),
178
+ cfg.Timeout.Duration(),
179
+ resourceGraph,
180
+ cfg.Discovery.ModeQuery.KQL,
181
+ supportedResourceTypes,
182
+ )
183
+ if err != nil {
184
+ return discoveryFetchResult{}, fmt.Errorf("discover candidate resources: %w", err)
185
+ }
186
+ return fetched, nil
187
+ }
188
+
189
+ discoveryTypes, err := initDiscoveryResourceTypes(cfg, catalog)
190
if err != nil {
150
- return nil, err
191
+ return discoveryFetchResult{}, fmt.Errorf("prepare discovery scope: %w", err)
192
+ }
193
+
194
+ resources, byType, err := discoverResources(
195
+ ctx,
196
+ cfg.subscriptionIDs(),
197
+ cfg.Timeout.Duration(),
198
+ resourceGraph,
199
+ discoveryTypes,
200
+ cfg.Discovery.ModeFilters,
201
+ )
202
+ if err != nil {
203
+ return discoveryFetchResult{}, fmt.Errorf("discover candidate resources: %w", err)
204
+ }
205
+
206
+ return discoveryFetchResult{Resources: resources, ByType: byType}, nil
207
+}
208
+
209
+func initDiscoveryResourceTypes(cfg Config, catalog azureprofiles.Catalog) ([]string, error) {
210
+ switch stringsLowerTrim(cfg.Profiles.Mode) {
211
+ case profilesModeAuto, profilesModeCombined:
212
+ return catalog.ResourceTypes(), nil
213
+ case profilesModeExact:
214
+ return catalog.ResourceTypesForProfileBaseNames(cfg.Profiles.explicitBaseNames())
215
+ default:
216
+ return nil, fmt.Errorf("unsupported profiles.mode %q", cfg.Profiles.Mode)
217
+ }
218
+}
219
+
220
+func resolveInitProfileIDs(cfg Config, catalog azureprofiles.Catalog, byType map[string][]resourceInfo) ([]string, []string, error) {
221
+ discoveredTypes := make(map[string]struct{}, len(byType))
222
+ for key := range byType {
223
+ discoveredTypes[key] = struct{}{}
224
+ }
225
+
226
+ autoProfiles := catalog.ProfilesForResourceTypes(discoveredTypes)
227
+ switch stringsLowerTrim(cfg.Profiles.Mode) {
228
+ case profilesModeAuto:
229
+ if len(autoProfiles) == 0 {
230
+ return nil, nil, errors.New("auto-discovery found no Azure resources matching any known profile")
231
+ }
232
+ return autoProfiles, autoProfiles, nil
233
+ case profilesModeExact:
234
+ explicitProfileIDs, err := catalog.ProfileIDsForBaseNames(cfg.Profiles.explicitBaseNames())
235
+ if err != nil {
236
+ return nil, nil, err
237
+ }
238
+ return explicitProfileIDs, nil, nil
239
+ case profilesModeCombined:
240
+ explicitProfileIDs, err := catalog.ProfileIDsForBaseNames(cfg.Profiles.explicitBaseNames())
241
+ if err != nil {
242
+ return nil, nil, err
243
+ }
244
+ return mergeProfileIDs(explicitProfileIDs, autoProfiles), autoProfiles, nil
245
+ default:
246
+ return nil, nil, fmt.Errorf("unsupported profiles.mode %q", cfg.Profiles.Mode)
247
}
152
- matched := catalog.ProfilesForResourceTypes(types)
153
- return mergeProfileIDs(explicitProfiles, matched), nil
248
}
249
250
func createCredential(auth cloudauth.AzureADAuthConfig, cloudCfg azcloud.Configuration) (azcore.TokenCredential, error) {
src/go/plugin/go.d/collector/azure_monitor/metadata.yaml
+915
-158
@@ -23,71 +23,116 @@ modules:
23
overview: &overview
24
data_collection:
25
metrics_description: |
26
- This collector monitors Azure resources through the Azure Monitor Metrics API. It automatically discovers
27
- resources in your subscription and collects platform metrics based on configurable profiles, providing
28
- visibility into the health and performance of over 35 Azure service types.
26
+ This collector provides real-time visibility into your Azure infrastructure by collecting platform metrics from the Azure Monitor Metrics API.
27
+
28
+ **Key capabilities:**
29
+
30
+ - **Multi-subscription** -- monitor resources across one or more Azure subscriptions in a single job
31
+ - **Automatic service detection** -- discovers resources and enables matching metric profiles without manual configuration
32
+ - **38 built-in service profiles** -- covers databases, compute, networking, storage, AI, analytics, and more
33
+ - **Flexible discovery** -- use structured filters (resource groups, regions, tags) or a custom Azure Resource Graph KQL query
34
method_description: &method_description |
30
- The collector uses Azure SDK clients for:
31
- - Authentication via Entra ID (service principal, managed identity, or default credentials)
32
- - Resource discovery via Azure Resource Graph queries
33
- - Metrics collection via Azure Monitor Metrics batch API, grouped by region and time grain
35
+ It uses the [Azure Monitor Metrics batch API](https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/migrate-to-batch-api) to collect metrics, grouping requests by subscription, region, and time grain. Resources are discovered via [Azure Resource Graph](https://learn.microsoft.com/en-us/azure/governance/resource-graph/overview) queries at startup and refreshed periodically. Authentication is handled through [Microsoft Entra ID](https://learn.microsoft.com/en-us/entra/identity/) (service principal, managed identity, or default credentials).
36
supported_platforms:
37
include: []
38
exclude: []
39
multi_instance: true
40
additional_permissions:
41
description: |
40
- The monitoring principal needs read access to Azure Resource Graph and Azure Monitor metrics for target resources.
42
+ The service principal or managed identity requires these Azure RBAC roles:
43
+
44
+ | Role | Purpose | Scope |
45
+ |:-----|:--------|:------|
46
+ | **Monitoring Reader** | Read Azure Monitor metrics | Subscription or resource group |
47
+ | **Reader** | Query Azure Resource Graph for resource discovery | Subscription or resource group |
48
default_behavior:
49
auto_detection:
50
description: |
44
- When `profile_selection_mode` is `auto` (the default), the collector queries Azure Resource Graph
45
- to discover which resource types exist in the subscription and enables matching built-in profiles automatically.
51
+ The collector has two discovery phases:
52
+
53
+ **Bootstrap (first run)**
54
+
55
+ - With the default `profiles.mode: auto`, the collector queries Azure Resource Graph within the configured `subscription_ids` to find candidate resources.
56
+ - It matches discovered resource types against built-in profiles and automatically enables the relevant ones.
57
+ - Discovery scope can be narrowed using `discovery.mode: filters` (resource groups, regions, tags) or replaced entirely with `discovery.mode: query` for a custom KQL query.
58
+ - A single job can monitor multiple subscriptions.
59
+
60
+ **Runtime (periodic refresh)**
61
+
62
+ - Periodically re-discovers resources for **already-active profile types only**.
63
+ - Controlled by `discovery.refresh_every` (default: 300 seconds, set to 0 to disable).
64
+
65
+ > **Important:** Runtime refresh does not activate new profiles. If a new resource type appears after bootstrap, restart the collector to pick it up.
66
limits:
67
description: |
48
- Azure Monitor metrics granularity is typically 1 minute.
49
- The collector enforces a minimum collection interval of 60 seconds.
68
+ - **Minimum collection interval:** 60 seconds (enforced). Azure Monitor metrics granularity is typically 1 minute.
69
+ - **Metrics reporting delay:** Azure Monitor metrics have a 1-3 minute reporting delay. The collector uses `query_offset` (default: 180s) as a minimum offset and automatically uses a larger effective offset for slower time-grain batches when needed.
70
+ - **API throttling:** Azure Monitor applies per-subscription rate limits. The collector uses bounded concurrency and batching to stay within limits, but monitoring many resources in a single subscription may require tuning `limits.*` options.
71
performance_impact:
72
description: |
52
- The collector uses bounded request concurrency and batches resources and metrics to minimize API calls.
53
- Default limits: 4 concurrent queries, 50 resources per batch, 20 metrics per query.
73
+ The collector batches resources and metrics to minimize Azure API calls and uses bounded concurrency to avoid overwhelming the API.
74
+
75
+ **Default concurrency and batching limits:**
76
+
77
+ | Setting | Default | Description |
78
+ |:--------|:--------|:------------|
79
+ | `limits.max_concurrency` | 4 | Maximum concurrent batch queries |
80
+ | `limits.max_batch_resources` | 50 | Maximum resources per batch request |
81
+ | `limits.max_metrics_per_query` | 20 | Maximum metrics per batch request |
82
+
83
+ For large deployments, consider splitting resources across multiple jobs. If you hit Azure API rate limits, reduce `max_concurrency`.
84
setup: &setup
85
prerequisites:
86
list:
87
- title: Create an Azure monitoring principal
88
description: |
59
- Create a service principal or use a managed identity with the following permissions:
89
+ The collector requires a service principal or managed identity with two Azure RBAC roles:
90
+
91
+ | Role | Purpose |
92
+ |:-----|:--------|
93
+ | **Monitoring Reader** | Access Azure Monitor metrics for target resources |
94
+ | **Reader** | Query Azure Resource Graph for resource discovery |
95
61
- 1. **Monitoring Reader** role on the target subscription or resource groups (for Azure Monitor metrics access)
62
- 2. **Reader** role for Azure Resource Graph queries (for resource discovery)
96
+ **Option A: Service principal**
97
64
- For service principal authentication:
98
```bash
66
- # Create the service principal
99
+ # Create service principal with Monitoring Reader role
100
az ad sp create-for-rbac --name "netdata-monitor" --role "Monitoring Reader" \
101
--scopes /subscriptions/<subscription-id>
102
103
+ # Add the Reader role for resource discovery
104
+ az role assignment create --assignee <appId-from-above> \
105
+ --role "Reader" --scope /subscriptions/<subscription-id>
106
+
107
# Note the appId (client_id), password (client_secret), and tenant
108
```
109
73
- For managed identity (on Azure VMs, VMSS, or AKS):
110
+ **Option B: Managed identity** (Azure VMs, VMSS, or AKS)
111
+
112
```bash
75
- # Assign Monitoring Reader role to the VM's managed identity
113
+ # Assign both roles to the VM's managed identity
114
az role assignment create --assignee <managed-identity-principal-id> \
115
--role "Monitoring Reader" --scope /subscriptions/<subscription-id>
116
+
117
+ az role assignment create --assignee <managed-identity-principal-id> \
118
+ --role "Reader" --scope /subscriptions/<subscription-id>
119
```
120
configuration:
121
file:
122
name: go.d/azure_monitor.conf
123
options:
124
description: |
84
- The following options can be defined globally: update_every, autodetection_retry.
125
+ The following options can be defined globally: `update_every`, `autodetection_retry`.
126
+
127
+ **Profile file locations:**
128
86
- Profile files are loaded from:
87
- - Stock: `/usr/lib/netdata/conf.d/go.d/azure_monitor.profiles/default/`
88
- - User: `/etc/netdata/go.d/azure_monitor.profiles/`
129
+ | Type | Path |
130
+ |:-----|:-----|
131
+ | Stock profiles | `/usr/lib/netdata/conf.d/go.d/azure_monitor.profiles/default/` |
132
+ | User overrides | `/etc/netdata/go.d/azure_monitor.profiles/` |
133
90
- User profile files with the same filename override stock profiles.
134
+ User profile files with the same `id` as a stock profile override it.
135
+ Custom profiles extend the collector's catalog -- they do not replace the discovery mechanism.
136
folding:
137
title: Config options
138
enabled: true
@@ -102,71 +147,48 @@ modules:
147
default_value: 0
148
required: false
149
group: Collection
105
- - name: subscription_id
106
- description: Azure subscription ID.
150
+ - name: subscription_ids
151
+ description: List of Azure subscription IDs to monitor. Used as the scope for resource discovery.
152
default_value: ""
153
required: true
109
- group: Target
154
+ group: Collection
155
- name: cloud
156
description: "Azure cloud environment: `public`, `government`, or `china`."
157
default_value: public
158
required: false
114
- group: Target
115
- - name: discovery_every
116
- description: Resource discovery interval in seconds.
117
- default_value: 300
118
- required: false
159
group: Collection
160
- name: query_offset
121
- description: Offset in seconds for metric query windows. Increase if metrics appear incomplete.
161
+ description: Minimum offset (seconds) subtracted from metric query windows. Increase if metrics appear incomplete.
162
default_value: 180
163
required: false
164
group: Collection
165
+ detailed_description: |
166
+ Azure Monitor metrics have a built-in reporting delay of 1-3 minutes. The collector subtracts this offset from the current time when building metric query windows to avoid fetching incomplete data points.
167
+
168
+ The configured `query_offset` acts as a minimum floor. For slower metric batches, the collector automatically uses a larger effective offset when the batch time grain is longer than the configured value.
169
+
170
+ - **Default (180s)** works for most services.
171
+ - **Longer time grains** (for example `PT5M`) automatically use at least one full time grain as the effective offset.
172
+ - **Increase to 240-300s** if you still see gaps or missing data points.
173
+ - **Do not set below 60s** -- metrics will likely be incomplete.
174
- name: timeout
175
description: Timeout for Azure Resource Graph and Azure Monitor API requests, in seconds.
176
default_value: 30
177
required: false
178
group: Collection
130
- - name: max_concurrency
131
- description: Maximum concurrent batch queries to Azure Monitor.
132
- default_value: 4
133
- required: false
134
- group: Limits
135
- - name: max_batch_resources
136
- description: Maximum resources per Azure Monitor batch request.
137
- default_value: 50
138
- required: false
139
- group: Limits
140
- - name: max_metrics_per_query
141
- description: Maximum metrics per Azure Monitor batch request.
142
- default_value: 20
143
- required: false
144
- group: Limits
145
- - name: profile_selection_mode
146
- description: "Profile selection mode: `auto` discovers matching profiles via Azure Resource Graph, `exact` uses only listed profile ids, `combined` merges listed ids with auto-discovered profiles."
147
- default_value: "auto"
148
- required: false
149
- group: Profiles
150
- - name: profile_selection_mode_exact.profiles
151
- description: Profile ids to enable (used when `profile_selection_mode` is `exact`).
152
- default_value: "[]"
153
- required: false
154
- group: Profiles
155
- - name: profile_selection_mode_combined.profiles
156
- description: Profile ids to merge with auto-discovered profiles (used when `profile_selection_mode` is `combined`).
157
- default_value: "[]"
158
- required: false
159
- group: Profiles
160
- - name: resource_groups
161
- description: Optional list of resource group names to restrict monitoring scope.
162
- default_value: "[]"
163
- required: false
164
- group: Filters
179
- name: auth.mode
166
- description: "Authentication mode: `service_principal`, `managed_identity`, or `default`."
180
+ description: "Authentication method: `service_principal`, `managed_identity`, or `default`."
181
default_value: ""
182
required: true
183
group: Authentication
184
+ detailed_description: |
185
+ Determines how the collector authenticates with Azure.
186
+
187
+ | Mode | When to use | Required options |
188
+ |:-----|:------------|:-----------------|
189
+ | `service_principal` | Running outside Azure, or when you need explicit credentials | `tenant_id`, `client_id`, `client_secret` |
190
+ | `managed_identity` | Running on Azure VMs, VMSS, or AKS with a managed identity | Optionally `client_id` for user-assigned identity |
191
+ | `default` | Uses the Azure SDK default credential chain (environment variables, managed identity, Azure CLI, etc.) | None |
192
- name: auth.mode_service_principal.tenant_id
193
description: Entra ID tenant ID (required for `service_principal` mode).
194
default_value: ""
@@ -187,6 +209,103 @@ modules:
209
default_value: ""
210
required: false
211
group: Authentication
212
+ - name: discovery.refresh_every
213
+ description: Interval (seconds) for refreshing discovered resources. Set `0` to disable runtime re-discovery after bootstrap.
214
+ default_value: 300
215
+ required: false
216
+ group: Discovery
217
+ - name: discovery.mode
218
+ description: "Resource discovery method: `filters` (structured filters) or `query` (custom KQL)."
219
+ default_value: filters
220
+ required: false
221
+ group: Discovery
222
+ detailed_description: |
223
+ Controls how the collector finds candidate Azure resources.
224
+
225
+ | Mode | Behavior |
226
+ |:-----|:---------|
227
+ | `filters` | Builds an Azure Resource Graph query from the structured `mode_filters.*` options (resource groups, regions, tags). This is the default. |
228
+ | `query` | Uses the raw KQL you provide in `discovery.mode_query.kql`. The query must project `id`, `name`, `type`, `resourceGroup`, and `location`. |
229
+ - name: discovery.mode_filters.resource_groups
230
+ description: Optional list of Azure resource groups to include in `filters` mode.
231
+ default_value: "[]"
232
+ required: false
233
+ group: Discovery
234
+ - name: discovery.mode_filters.regions
235
+ description: Optional list of Azure regions to include in `filters` mode.
236
+ default_value: "[]"
237
+ required: false
238
+ group: Discovery
239
+ - name: discovery.mode_filters.tags
240
+ description: Optional exact-match tag filters for `filters` mode. Keys are matched case-insensitively and values case-sensitively.
241
+ default_value: "{}"
242
+ required: false
243
+ group: Discovery
244
+ - name: discovery.mode_query.kql
245
+ description: Custom Azure Resource Graph KQL for `query` mode. Must project `id`, `name`, `type`, `resourceGroup`, `location`.
246
+ default_value: ""
247
+ required: false
248
+ group: Discovery
249
+ detailed_description: |
250
+ A raw Azure Resource Graph KQL query used when `discovery.mode` is `query`.
251
+
252
+ The query **must** project these five columns:
253
+
254
+ | Column | Description |
255
+ |:-------|:------------|
256
+ | `id` | Full Azure resource ID (ARM format) |
257
+ | `name` | Resource name |
258
+ | `type` | Resource type (e.g., `microsoft.sql/servers/databases`) |
259
+ | `resourceGroup` | Resource group name |
260
+ | `location` | Azure region |
261
+
262
+ Example:
263
+
264
+ ```
265
+ resources
266
+ | where tags.env =~ "prod"
267
+ | project id, name, type, resourceGroup, location
268
+ ```
269
+ - name: profiles.mode
270
+ description: "How profiles are selected: `auto` (discover from resources), `exact` (explicit list), or `combined` (both)."
271
+ default_value: auto
272
+ required: false
273
+ group: Profiles
274
+ detailed_description: |
275
+ Controls how the collector decides which metric profiles to activate.
276
+
277
+ | Mode | Behavior |
278
+ |:-----|:---------|
279
+ | `auto` | Discovers resource types in your subscriptions and enables matching built-in profiles automatically. This is the default. |
280
+ | `exact` | Uses only the profile basenames listed under `profiles.mode_exact.names`. No auto-discovery. |
281
+ | `combined` | Merges auto-discovered profiles with the basenames listed under `profiles.mode_combined.names`. |
282
+
283
+ Profile basename matching is case-insensitive. A basename is the profile filename without the `.yaml` / `.yml` suffix.
284
+ - name: profiles.mode_exact.names
285
+ description: Explicit profile file basenames used by `exact` mode. Matching is case-insensitive.
286
+ default_value: "[]"
287
+ required: false
288
+ group: Profiles
289
+ - name: profiles.mode_combined.names
290
+ description: Explicit profile file basenames merged with auto-discovered profiles in `combined` mode. Matching is case-insensitive.
291
+ default_value: "[]"
292
+ required: false
293
+ group: Profiles
294
+ - name: limits.max_concurrency
295
+ description: Maximum concurrent batch queries to Azure Monitor.
296
+ default_value: 4
297
+ required: false
298
+ group: Limits
299
+ - name: limits.max_batch_resources
300
+ description: Maximum resources per Azure Monitor batch request.
301
+ default_value: 50
302
+ required: false
303
+ group: Limits
304
+ - name: limits.max_metrics_per_query
305
+ description: Maximum metrics per Azure Monitor batch request.
306
+ default_value: 20
307
+ required: false
308
+ group: Limits
309
- name: vnode
310
description: Associates this data collection job with a [Virtual Node](https://learn.netdata.cloud/docs/netdata-agent/configuration/organize-systems-metrics-and-alerts#virtual-nodes).
311
default_value: ""
@@ -197,61 +316,74 @@ modules:
316
title: Config
317
enabled: true
318
list:
200
- - name: Service principal (auto-discover all resources)
201
- description: Authenticate with a service principal and auto-discover all supported Azure resource types in the subscription.
319
+ - name: Service principal with structured discovery
320
+ description: Authenticate with a service principal and auto-discover resources across two subscriptions, filtered to the `production-rg` resource group in `eastus` with the tag `env=prod`.
321
folding:
322
enabled: false
323
config: |
324
jobs:
325
- name: prod
207
- subscription_id: "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
326
+ subscription_ids:
327
+ - "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"
328
+ - "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"
329
+ discovery:
330
+ mode: filters
331
+ mode_filters:
332
+ resource_groups:
333
+ - production-rg
334
+ regions:
335
+ - eastus
336
+ tags:
337
+ env:
338
+ - prod
339
+ profiles:
340
+ mode: auto
341
auth:
342
mode: service_principal
343
mode_service_principal:
344
tenant_id: "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
345
client_id: "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
346
client_secret: "your-client-secret"
214
- - name: Managed identity (Azure VM/VMSS/AKS)
215
- description: Use the managed identity of the Azure VM, VMSS, or AKS node where Netdata is running.
216
- config: |
217
- jobs:
218
- - name: prod
219
- subscription_id: "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
220
- auth:
221
- mode: managed_identity
222
- - name: Specific profiles only
223
- description: Monitor only specific Azure services instead of auto-discovering all resource types.
347
+ - name: Managed identity with exact profiles
348
+ description: Use a managed identity (on an Azure VM, VMSS, or AKS) and monitor only SQL Database and PostgreSQL Flexible Server resources -- skip auto-discovery of other services.
349
config: |
350
jobs:
351
- name: databases
227
- subscription_id: "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
352
+ subscription_ids:
353
+ - "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
354
profiles:
229
- - sql_database
230
- - postgres_flexible
231
- - redis_cache
355
+ mode: exact
356
+ mode_exact:
357
+ names:
358
+ - sql_database
359
+ - postgres_flexible
360
auth:
233
- mode: service_principal
234
- mode_service_principal:
235
- tenant_id: "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
236
- client_id: "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
237
- client_secret: "your-client-secret"
238
- - name: Filter by resource group
239
- description: Only monitor resources in specific resource groups.
361
+ mode: managed_identity
362
+ - name: Custom Azure Resource Graph KQL
363
+ description: Replace the built-in discovery filters with your own KQL query. Useful when you need joins, computed columns, or filtering logic that structured filters cannot express.
364
config: |
365
jobs:
242
- - name: prod-rg
243
- subscription_id: "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
244
- resource_groups:
245
- - production-rg
246
- - staging-rg
366
+ - name: prod-query
367
+ subscription_ids:
368
+ - "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
369
+ discovery:
370
+ mode: query
371
+ mode_query:
372
+ kql: |
373
+ resources
374
+ | where tags.env =~ "prod"
375
+ | project id, name, type, resourceGroup, location
376
+ profiles:
377
+ mode: auto
378
auth:
379
mode: default
380
- name: Azure Government cloud
250
- description: Connect to Azure Government cloud environment.
381
+ description: "Connect to an Azure Government environment. Set `cloud: government` to use the correct authentication and API endpoints."
382
config: |
383
jobs:
384
- name: gov
254
- subscription_id: "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
385
+ subscription_ids:
386
+ - "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
387
cloud: government
388
auth:
389
mode: service_principal
@@ -264,25 +396,40 @@ modules:
396
list:
397
- name: No metrics are collected
398
description: |
267
- Verify the following:
268
- 1. The service principal or managed identity has **Monitoring Reader** role on the subscription or resource group.
269
- 2. The `subscription_id` in the configuration matches the subscription containing the target resources.
270
- 3. Target resources are running and producing metrics (check Azure Portal > Metrics for the resource).
271
- 4. Check the Netdata error log for authentication or API errors: `grep azure_monitor /var/log/netdata/error.log`.
399
+ Check the following:
400
+
401
+ - **Permissions** -- The principal has both **Monitoring Reader** and **Reader** roles on the target subscription.
402
+ - **Subscription IDs** -- The `subscription_ids` list includes the correct subscription(s).
403
+ - **Resources are active** -- Verify in Azure Portal > Metrics that the resources are producing metrics.
404
+ - **Collector logs** -- Check for authentication or API errors:
405
+ ```bash
406
+ # systemd
407
+ journalctl -u netdata --namespace=netdata --grep azure_monitor --since "5 minutes ago"
408
+ # non-systemd
409
+ grep azure_monitor /var/log/netdata/collector.log
410
+ ```
411
- name: Missing metrics for some resource types
412
description: |
274
- Azure Monitor profiles are matched by resource type. If a resource type exists but no metrics appear:
275
- 1. Ensure `profiles: [auto]` (default) is set, or the specific profile id is listed.
276
- 2. Verify the resource type matches a built-in profile. Run `ls /usr/lib/netdata/conf.d/go.d/azure_monitor.profiles/default/` to see available profiles.
277
- 3. Some metrics require the resource to be actively processing data (e.g., IoT Hub telemetry metrics only appear when devices send messages).
278
- - name: Metrics appear delayed
413
+ Profiles are matched by Azure resource type. If a resource type exists but metrics are missing:
414
+
415
+ - **Check profile mode** -- Ensure `profiles.mode: auto` (default), or explicitly list the profile basename under `profiles.mode_exact.names` or `profiles.mode_combined.names`.
416
+ - **Verify a built-in profile exists** -- List available profiles:
417
+ ```bash
418
+ ls /usr/lib/netdata/conf.d/go.d/azure_monitor.profiles/default/
419
+ ```
420
+ - **Check resource activity** -- Some metrics only appear when the resource is actively processing data (e.g., IoT Hub telemetry metrics require devices to be sending messages).
421
+ - **New resource types after startup** -- Runtime discovery does not activate new profiles. Restart the collector if new resource types were added after bootstrap.
422
+ - name: Charts have gaps or incomplete data
423
description: |
280
- Azure Monitor metrics have a built-in reporting delay of 1-3 minutes. The collector uses a `query_offset` (default: 180 seconds) to account for this.
281
- If metrics are missing or incomplete, try increasing `query_offset` to 240 or 300 seconds.
282
- Some metrics with longer time grains (e.g., PT5M) may take up to 5 minutes to appear.
424
+ Azure Monitor metrics have a built-in reporting delay of **1-3 minutes**.
425
+
426
+ - The collector uses `query_offset` (default: **180 seconds**) as the minimum offset for metric query windows.
427
+ - Slower time-grain batches automatically use a larger effective offset when needed.
428
+ - If metrics are still missing or incomplete, increase `query_offset` to **240** or **300** seconds.
429
- name: Authentication errors in sovereign clouds
430
description: |
431
For Azure Government or Azure China clouds, set the `cloud` parameter:
432
+
433
- Azure Government: `cloud: government`
434
- Azure China (21Vianet): `cloud: china`
435
@@ -293,11 +440,11 @@ modules:
440
title: Metrics
441
enabled: false
442
description: |
296
- Metrics depend on which Azure Monitor profiles are enabled. Each profile corresponds to an Azure
297
- service type and defines the specific metrics collected. With the default `profiles: [auto]` setting,
298
- profiles are automatically enabled for resource types found in your subscription.
443
+ The metrics collected depend on which Azure Monitor profiles are active. Each profile corresponds to an Azure service (e.g., SQL Database, Virtual Machines) and defines the specific charts and metrics for that service.
444
+
445
+ With the default `profiles.mode: auto`, profiles are activated automatically based on the resource types found in your subscriptions.
446
300
- See the service-specific integrations below for detailed metrics lists.
447
+ **See the service-specific integrations below for detailed metric lists per Azure service.**
448
availability: []
449
scopes: []
450
- <<: *module
@@ -321,7 +468,17 @@ modules:
468
<<: *overview
469
data_collection:
470
metrics_description: |
324
- Monitor SQL Managed Instance performance including virtual core CPU utilization, storage consumption, IO throughput, and average request wait times.
471
+ :::info
472
+
473
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
474
+
475
+ :::
476
+
477
+ Monitor Azure SQL Managed Instance with metrics covering:
478
+
479
+ - **Compute** -- CPU utilization (average/max), virtual core count
480
+ - **Storage** -- reserved and used storage
481
+ - **I/O** -- read/write throughput, I/O request rate
482
method_description: *method_description
483
alerts:
484
- name: am_sql_managed_instance_cpu
@@ -356,6 +513,8 @@ modules:
513
description: "The Azure resource type identifier."
514
- name: profile
515
description: "The Azure Monitor profile id."
516
+ - name: subscription_id
517
+ description: "The Azure subscription identifier."
518
- name: resource_uid
519
description: "The unique Azure resource identifier."
520
metrics:
@@ -412,7 +571,21 @@ modules:
571
<<: *overview
572
data_collection:
573
metrics_description: |
415
- Monitor SQL Database performance including CPU and DTU utilization, storage consumption, active sessions and workers, deadlocks, IO rates, tempdb usage, in-memory OLTP storage, and serverless auto-pause and billing metrics.
574
+ :::info
575
+
576
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
577
+
578
+ :::
579
+
580
+ Monitor Azure SQL Database with metrics covering:
581
+
582
+ - **CPU and DTU** -- CPU utilization (average/max), instance CPU, DTU consumption, vCore usage
583
+ - **Memory** -- instance memory utilization
584
+ - **Storage** -- data and allocated storage, storage utilization, tempdb size, in-memory OLTP storage
585
+ - **I/O** -- data read and log write utilization, tempdb log utilization
586
+ - **Connections** -- successful, failed, and firewall-blocked connections, active sessions and workers
587
+ - **Availability** -- database availability percentage
588
+ - **Advanced** -- deadlocks, replication lag, serverless CPU/memory/billing, ledger digest, free tier usage
589
method_description: *method_description
590
alerts:
591
- name: am_sql_database_availability
@@ -511,6 +684,8 @@ modules:
684
description: "The Azure resource type identifier."
685
- name: profile
686
description: "The Azure Monitor profile id."
687
+ - name: subscription_id
688
+ description: "The Azure subscription identifier."
689
- name: resource_uid
690
description: "The unique Azure resource identifier."
691
metrics:
@@ -680,7 +855,23 @@ modules:
855
<<: *overview
856
data_collection:
857
metrics_description: |
683
- Monitor PostgreSQL Flexible Server including active connections, transaction rates, replication lag, storage and backup utilization, CPU and memory usage, IO throughput, autovacuum activity, PgBouncer connection pooling, database sessions, and burstable instance CPU credits.
858
+ :::info
859
+
860
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
861
+
862
+ :::
863
+
864
+ Monitor Azure PostgreSQL Flexible Server with metrics covering:
865
+
866
+ - **Compute** -- CPU utilization, burstable CPU credits (consumed/remaining)
867
+ - **Memory** -- memory utilization
868
+ - **Storage** -- storage used/free, backup storage, WAL storage, database size, disk queue depth and saturation
869
+ - **I/O** -- IOPS (read/write), disk throughput, temp bytes/files
870
+ - **Connections** -- active connections, connection rate, max connections, PgBouncer client/server/pooled connections
871
+ - **Database** -- transaction rate, commits/rollbacks, tuple reads/writes, replication lag (time/bytes), deadlocks
872
+ - **Maintenance** -- autovacuum operations, table coverage, bloat percentage, buffer cache hit rate
873
+ - **Sessions** -- sessions by state and wait event type, backend count
874
+ - **Availability** -- database alive state
875
method_description: *method_description
876
alerts:
877
- name: am_postgres_flexible_availability
@@ -779,6 +970,8 @@ modules:
970
description: "The Azure resource type identifier."
971
- name: profile
972
description: "The Azure Monitor profile id."
973
+ - name: subscription_id
974
+ description: "The Azure subscription identifier."
975
- name: resource_uid
976
description: "The unique Azure resource identifier."
977
metrics:
@@ -1089,7 +1282,20 @@ modules:
1282
<<: *overview
1283
data_collection:
1284
metrics_description: |
1092
- Monitor Cosmos DB accounts including request unit consumption and throttling, document counts and storage, data and index sizes, replication latency, availability percentages, provisioned throughput utilization, and normalized RU consumption per partition.
1285
+ :::info
1286
+
1287
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
1288
+
1289
+ :::
1290
+
1291
+ Monitor Azure Cosmos DB with metrics covering:
1292
+
1293
+ - **Request units** -- RU consumption, provisioned throughput (provisioned/autoscale), normalized RU per partition
1294
+ - **Storage** -- data, index, and quota storage, physical partition size
1295
+ - **Latency** -- server-side latency (direct/gateway), replication latency
1296
+ - **Requests** -- total requests, API requests (Mongo/Cassandra/Gremlin), metadata requests, dedicated gateway requests
1297
+ - **Availability** -- service availability percentage
1298
+ - **Advanced** -- document count, partition count, dedicated gateway CPU/memory, integrated cache hit rate
1299
method_description: *method_description
1300
alerts:
1301
- name: am_cosmos_db_availability
@@ -1152,6 +1358,8 @@ modules:
1358
description: "The Azure resource type identifier."
1359
- name: profile
1360
description: "The Azure Monitor profile id."
1361
+ - name: subscription_id
1362
+ description: "The Azure subscription identifier."
1363
- name: resource_uid
1364
description: "The unique Azure resource identifier."
1365
metrics:
@@ -1306,7 +1514,20 @@ modules:
1514
<<: *overview
1515
data_collection:
1516
metrics_description: |
1309
- Monitor Logic Apps workflow execution including run completions and failures, action execution counts, trigger firing rates, run and action latency, billable executions, and action-level success and failure breakdowns.
1517
+ :::info
1518
+
1519
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
1520
+
1521
+ :::
1522
+
1523
+ Monitor Azure Logic Apps with metrics covering:
1524
+
1525
+ - **Runs** -- run lifecycle (started/completed/succeeded/failed/cancelled), run failure rate
1526
+ - **Actions** -- action lifecycle (started/completed/succeeded/failed/skipped)
1527
+ - **Triggers** -- trigger lifecycle (started/completed/succeeded/fired/failed/skipped)
1528
+ - **Latency** -- run, action, and trigger latency
1529
+ - **Billing** -- billable executions (total/actions/triggers), billing by type (native/connector/storage)
1530
+ - **Throttling** -- run, action, and trigger throttling events
1531
method_description: *method_description
1532
alerts:
1533
- name: am_logic_apps_run_failure_rate
@@ -1377,6 +1598,8 @@ modules:
1598
description: "The Azure resource type identifier."
1599
- name: profile
1600
description: "The Azure Monitor profile id."
1601
+ - name: subscription_id
1602
+ description: "The Azure subscription identifier."
1603
- name: resource_uid
1604
description: "The unique Azure resource identifier."
1605
metrics:
@@ -1503,7 +1726,22 @@ modules:
1726
<<: *overview
1727
data_collection:
1728
metrics_description: |
1506
- Monitor Azure Virtual Machines including CPU utilization, available memory percentage, disk IOPS and throughput for OS, data, temp, and premium cache disks, disk burst and VM-level burst credit balances, network traffic, and inbound/outbound flow creation rates.
1729
+ :::info
1730
+
1731
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
1732
+
1733
+ :::
1734
+
1735
+ Monitor Azure Virtual Machines with metrics covering:
1736
+
1737
+ - **Compute** -- CPU utilization, CPU credits (consumed/remaining)
1738
+ - **Memory** -- available memory (bytes and percentage)
1739
+ - **Disk** -- IOPS, throughput, latency, and queue depth for OS, data, and temp disks
1740
+ - **Disk burst** -- burst credits and capacity for OS and data disks, VM-level cached/uncached burst credits
1741
+ - **Disk cache** -- premium OS and data disk cache hit/miss rates
1742
+ - **Network** -- traffic in/out, network flows, flow creation rate
1743
+ - **Availability** -- VM availability state
1744
+ - **Throttling** -- cached and uncached I/O bandwidth/IOPS throttling
1745
method_description: *method_description
1746
alerts:
1747
- name: am_vm_cpu
@@ -1622,6 +1860,8 @@ modules:
1860
description: "The Azure resource type identifier."
1861
- name: profile
1862
description: "The Azure Monitor profile id."
1863
+ - name: subscription_id
1864
+ description: "The Azure subscription identifier."
1865
- name: resource_uid
1866
description: "The unique Azure resource identifier."
1867
metrics:
@@ -1888,7 +2128,18 @@ modules:
2128
<<: *overview
2129
data_collection:
2130
metrics_description: |
1891
- Monitor AKS cluster health including API server and etcd resource usage, pod scheduling status and readiness, node capacity and conditions, cluster autoscaler behavior, and per-node CPU, memory, disk, and network utilization.
2131
+ :::info
2132
+
2133
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
2134
+
2135
+ :::
2136
+
2137
+ Monitor Azure Kubernetes Service (AKS) with metrics covering:
2138
+
2139
+ - **Control plane** -- API server CPU/memory, etcd CPU/memory/database utilization, inflight requests
2140
+ - **Nodes** -- allocatable CPU/memory, per-node CPU (millicores and %), memory (RSS, working set), disk usage, network traffic
2141
+ - **Pods** -- pods by phase, pods in ready state, node conditions
2142
+ - **Autoscaler** -- autoscaler health, unneeded nodes, unschedulable pods
2143
method_description: *method_description
2144
alerts:
2145
- name: am_aks_apiserver_cpu
@@ -1959,6 +2210,8 @@ modules:
2210
description: "The Azure resource type identifier."
2211
- name: profile
2212
description: "The Azure Monitor profile id."
2213
+ - name: subscription_id
2214
+ description: "The Azure subscription identifier."
2215
- name: resource_uid
2216
description: "The unique Azure resource identifier."
2217
metrics:
@@ -2137,7 +2390,19 @@ modules:
2390
<<: *overview
2391
data_collection:
2392
metrics_description: |
2140
- Monitor Azure Storage Account operations including transaction counts, availability percentages, success and end-to-end latency, ingress and egress throughput, and used capacity.
2393
+ :::info
2394
+
2395
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
2396
+
2397
+ :::
2398
+
2399
+ Monitor Azure Storage Account with metrics covering:
2400
+
2401
+ - **Transactions** -- transaction count
2402
+ - **Latency** -- end-to-end latency and server latency (average/max)
2403
+ - **Throughput** -- ingress and egress bytes per second
2404
+ - **Availability** -- service availability percentage
2405
+ - **Capacity** -- used storage capacity
2406
method_description: *method_description
2407
alerts:
2408
- name: am_storage_accounts_availability
@@ -2180,6 +2445,8 @@ modules:
2445
description: "The Azure resource type identifier."
2446
- name: profile
2447
description: "The Azure Monitor profile id."
2448
+ - name: subscription_id
2449
+ description: "The Azure subscription identifier."
2450
- name: resource_uid
2451
description: "The unique Azure resource identifier."
2452
metrics:
@@ -2242,7 +2509,18 @@ modules:
2509
<<: *overview
2510
data_collection:
2511
metrics_description: |
2245
- Monitor Azure Load Balancer health and throughput including data path and health probe availability, SYN and SNAT connection counts, byte and packet throughput, allocated and used SNAT ports, and connection attempt rates.
2512
+ :::info
2513
+
2514
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
2515
+
2516
+ :::
2517
+
2518
+ Monitor Azure Load Balancer with metrics covering:
2519
+
2520
+ - **Availability** -- data path availability, health probe status, global backend availability
2521
+ - **Throughput** -- byte and packet throughput
2522
+ - **Connections** -- SNAT connections, SYN packet count
2523
+ - **SNAT ports** -- allocated and used SNAT ports
2524
method_description: *method_description
2525
alerts:
2526
- name: am_load_balancers_vip_availability
@@ -2281,6 +2559,8 @@ modules:
2559
description: "The Azure resource type identifier."
2560
- name: profile
2561
description: "The Azure Monitor profile id."
2562
+ - name: subscription_id
2563
+ description: "The Azure subscription identifier."
2564
- name: resource_uid
2565
description: "The unique Azure resource identifier."
2566
metrics:
@@ -2354,7 +2634,23 @@ modules:
2634
<<: *overview
2635
data_collection:
2636
metrics_description: |
2357
- Monitor App Service web applications including HTTP request rates and response status codes, response times, CPU and memory usage, network throughput, file IO operations, .NET runtime statistics (threads, GC, assemblies), Azure Functions execution counts and units, and Flex Consumption plan metrics.
2637
+ :::info
2638
+
2639
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
2640
+
2641
+ :::
2642
+
2643
+ Monitor Azure App Service with metrics covering:
2644
+
2645
+ - **Requests** -- HTTP request rate, response status codes (2xx/3xx/4xx/5xx), error detail (401/403/404/406)
2646
+ - **Performance** -- response time, request queue depth
2647
+ - **Compute** -- CPU utilization, CPU time consumed
2648
+ - **Memory** -- memory usage (average working set, working set, private bytes)
2649
+ - **Network** -- network traffic (received/sent), I/O throughput (read/write/other)
2650
+ - **I/O** -- I/O operations (read/write/other), file handles
2651
+ - **.NET runtime** -- threads, GC collections (gen0/gen1/gen2), loaded assemblies, app domains
2652
+ - **Functions** -- function executions and execution units (MB-ms), always-ready and on-demand units
2653
+ - **Health** -- health check status
2654
method_description: *method_description
2655
alerts:
2656
- name: am_app_service_health_check
@@ -2409,6 +2705,8 @@ modules:
2705
description: "The Azure resource type identifier."
2706
- name: profile
2707
description: "The Azure Monitor profile id."
2708
+ - name: subscription_id
2709
+ description: "The Azure subscription identifier."
2710
- name: resource_uid
2711
description: "The unique Azure resource identifier."
2712
metrics:
@@ -2605,7 +2903,21 @@ modules:
2903
<<: *overview
2904
data_collection:
2905
metrics_description: |
2608
- Monitor Azure Functions execution including function invocation counts, execution units (MB-milliseconds), HTTP request rates and response codes, CPU and memory consumption, and Flex Consumption plan metrics for always-ready and on-demand instances. Uses the same underlying metrics as App Service since Azure Functions runs on the App Service platform.
2906
+ :::info
2907
+
2908
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
2909
+
2910
+ :::
2911
+
2912
+ Monitor Azure Functions with metrics covering:
2913
+
2914
+ - **Executions** -- function invocation counts, execution units (MB-milliseconds)
2915
+ - **HTTP** -- request rates, response status codes
2916
+ - **Compute** -- CPU utilization, CPU time consumed
2917
+ - **Memory** -- memory usage (working set, private bytes)
2918
+ - **Flex Consumption** -- always-ready and on-demand function executions and units
2919
+
2920
+ Azure Functions runs on the App Service platform and shares the same underlying metrics.
2921
method_description: *method_description
2922
metrics:
2923
folding:
@@ -2627,6 +2939,8 @@ modules:
2939
description: "The Azure resource type identifier."
2940
- name: profile
2941
description: "The Azure Monitor profile id."
2942
+ - name: subscription_id
2943
+ description: "The Azure subscription identifier."
2944
- name: resource_uid
2945
description: "The unique Azure resource identifier."
2946
metrics:
@@ -2823,7 +3137,23 @@ modules:
3137
<<: *overview
3138
data_collection:
3139
metrics_description: |
2826
- Monitor Azure Cache for Redis including cache hit and miss rates, read and write throughput, server load and CPU utilization, memory usage, connected clients, operations per second, command processing rates, latency percentiles, key eviction and expiration, and geo-replication health and sync status. Provides per-shard breakdowns for clustered deployments.
3140
+ :::info
3141
+
3142
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
3143
+
3144
+ :::
3145
+
3146
+ Monitor Azure Cache for Redis with metrics covering:
3147
+
3148
+ - **Performance** -- operations/second, command processing rates (get/set), cache hit/miss rates
3149
+ - **Latency** -- average latency, P99 latency
3150
+ - **Compute** -- CPU utilization, server load
3151
+ - **Memory** -- memory usage (used/RSS), memory utilization
3152
+ - **Connections** -- connected clients, connection rate (created/closed)
3153
+ - **Keys** -- total keys, evicted keys, expired keys, miss rate
3154
+ - **Throughput** -- read/write bytes per second
3155
+ - **Geo-replication** -- replication health, connectivity lag, sync events, data sync offset
3156
+ - **Per-shard** -- instance-level breakdowns for hit rate, clients, commands, server load, keys, operations, throughput
3157
method_description: *method_description
3158
alerts:
3159
- name: am_redis_cache_server_load
@@ -2890,6 +3220,8 @@ modules:
3220
description: "The Azure resource type identifier."
3221
- name: profile
3222
description: "The Azure Monitor profile id."
3223
+ - name: subscription_id
3224
+ description: "The Azure subscription identifier."
3225
- name: resource_uid
3226
description: "The unique Azure resource identifier."
3227
metrics:
@@ -3120,7 +3452,21 @@ modules:
3452
<<: *overview
3453
data_collection:
3454
metrics_description: |
3123
- Monitor Event Hubs namespaces including incoming and outgoing message rates, byte throughput, captured messages and bytes, throttled and quota-exceeded request counts, active connections, and total connection counts.
3455
+ :::info
3456
+
3457
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
3458
+
3459
+ :::
3460
+
3461
+ Monitor Azure Event Hubs with metrics covering:
3462
+
3463
+ - **Messages** -- message flow (in/out), captured messages and bytes
3464
+ - **Throughput** -- data throughput (in/out bytes per second)
3465
+ - **Connections** -- active connections, connection events (opened/closed)
3466
+ - **Requests** -- incoming and successful request rates
3467
+ - **Errors** -- server errors, user errors, throttled requests, quota exceeded
3468
+ - **Replication** -- replication lag (messages and duration)
3469
+ - **Resources** -- namespace size, CPU and memory utilization
3470
method_description: *method_description
3471
alerts:
3472
- name: am_event_hubs_server_errors
@@ -3183,6 +3529,8 @@ modules:
3529
description: "The Azure resource type identifier."
3530
- name: profile
3531
description: "The Azure Monitor profile id."
3532
+ - name: subscription_id
3533
+ description: "The Azure subscription identifier."
3534
- name: resource_uid
3535
description: "The unique Azure resource identifier."
3536
metrics:
@@ -3293,7 +3641,22 @@ modules:
3641
<<: *overview
3642
data_collection:
3643
metrics_description: |
3296
- Monitor Service Bus namespaces including incoming and outgoing message rates, active connections, active and dead-lettered message counts, scheduled message counts, completed and abandoned requests, server errors, throttled requests, CPU and memory utilization, and pending checkpoint operations.
3644
+ :::info
3645
+
3646
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
3647
+
3648
+ :::
3649
+
3650
+ Monitor Azure Service Bus with metrics covering:
3651
+
3652
+ - **Messages** -- message flow (in/out), active messages, dead-lettered messages, scheduled messages, queue depth
3653
+ - **Throughput** -- data throughput (in/out bytes per second)
3654
+ - **Operations** -- completed and abandoned message operations, send latency
3655
+ - **Connections** -- active connections, connection events (opened/closed)
3656
+ - **Requests** -- incoming and successful request rates
3657
+ - **Errors** -- server errors, user errors, throttled requests
3658
+ - **Replication** -- replication lag (messages and duration)
3659
+ - **Resources** -- namespace size, CPU and memory utilization, pending checkpoint operations
3660
method_description: *method_description
3661
alerts:
3662
- name: am_service_bus_server_errors
@@ -3364,6 +3727,8 @@ modules:
3727
description: "The Azure resource type identifier."
3728
- name: profile
3729
description: "The Azure Monitor profile id."
3730
+ - name: subscription_id
3731
+ description: "The Azure subscription identifier."
3732
- name: resource_uid
3733
description: "The unique Azure resource identifier."
3734
metrics:
@@ -3493,7 +3858,21 @@ modules:
3858
<<: *overview
3859
data_collection:
3860
metrics_description: |
3496
- Monitor Application Gateway performance including throughput and traffic volume, request rates and response status codes, backend health and latency breakdown (connect, first byte, last byte), client latency, current and new connections, WebSocket sessions, capacity and compute units, CPU utilization, TLS connections, and WAF security events including rule matches, challenges, and penalty box activity.
3861
+ :::info
3862
+
3863
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
3864
+
3865
+ :::
3866
+
3867
+ Monitor Azure Application Gateway with metrics covering:
3868
+
3869
+ - **Traffic** -- throughput, traffic volume (received/sent), request rates (total/failed)
3870
+ - **Response** -- gateway and backend response status codes
3871
+ - **Backend** -- backend health (healthy/unhealthy hosts), backend latency (connect, first byte, last byte)
3872
+ - **Client** -- client latency (total time, client RTT)
3873
+ - **Connections** -- current and new connections, TLS connections, WebSocket connections
3874
+ - **Capacity** -- capacity units, compute units, billed/fixed billed, CPU utilization
3875
+ - **WAF** -- WAF requests (total/blocked/matched), rule matches (managed/custom/bot), challenges, penalty box
3876
method_description: *method_description
3877
alerts:
3878
- name: am_appgw_failed_requests
@@ -3544,6 +3923,8 @@ modules:
3923
description: "The Azure resource type identifier."
3924
- name: profile
3925
description: "The Azure Monitor profile id."
3926
+ - name: subscription_id
3927
+ description: "The Azure subscription identifier."
3928
- name: resource_uid
3929
description: "The unique Azure resource identifier."
3930
metrics:
@@ -3703,7 +4084,17 @@ modules:
4084
<<: *overview
4085
data_collection:
4086
metrics_description: |
3706
- Monitor Key Vault including overall vault availability, API saturation approaching service limits, and service API hit and latency metrics.
4087
+ :::info
4088
+
4089
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
4090
+
4091
+ :::
4092
+
4093
+ Monitor Azure Key Vault with metrics covering:
4094
+
4095
+ - **Availability** -- overall vault availability percentage
4096
+ - **API** -- API activity (hits/results), API latency
4097
+ - **Saturation** -- API saturation approaching service limits
4098
method_description: *method_description
4099
alerts:
4100
- name: am_key_vault_availability
@@ -3738,6 +4129,8 @@ modules:
4129
description: "The Azure resource type identifier."
4130
- name: profile
4131
description: "The Azure Monitor profile id."
4132
+ - name: subscription_id
4133
+ description: "The Azure subscription identifier."
4134
- name: resource_uid
4135
description: "The unique Azure resource identifier."
4136
metrics:
@@ -3786,7 +4179,21 @@ modules:
4179
<<: *overview
4180
data_collection:
4181
metrics_description: |
3789
- Monitor API Management gateway performance including request throughput, response status codes, gateway and backend response times, failed request counts, capacity utilization, event hub events, websocket message counts, and network connection status.
4182
+ :::info
4183
+
4184
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
4185
+
4186
+ :::
4187
+
4188
+ Monitor Azure API Management with metrics covering:
4189
+
4190
+ - **Requests** -- gateway request rate
4191
+ - **Latency** -- request duration (overall and backend response time)
4192
+ - **Compute** -- gateway CPU and memory utilization
4193
+ - **Capacity** -- capacity utilization percentage
4194
+ - **Events** -- EventHub events (successful/failed/dropped/rejected/throttled/timed out), EventHub bytes sent
4195
+ - **WebSockets** -- WebSocket connection attempts, WebSocket messages
4196
+ - **Network** -- network connectivity status
4197
method_description: *method_description
4198
alerts:
4199
- name: am_api_management_capacity
@@ -3853,6 +4260,8 @@ modules:
4260
description: "The Azure resource type identifier."
4261
- name: profile
4262
description: "The Azure Monitor profile id."
4263
+ - name: subscription_id
4264
+ description: "The Azure subscription identifier."
4265
- name: resource_uid
4266
description: "The unique Azure resource identifier."
4267
metrics:
@@ -3944,7 +4353,21 @@ modules:
4353
<<: *overview
4354
data_collection:
4355
metrics_description: |
3947
- Monitor Azure Front Door including request counts and rates, response sizes, total latency, origin health probe percentages, origin request counts, origin latency, WAF request counts by action and rule, and WebSocket connection metrics.
4356
+ :::info
4357
+
4358
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
4359
+
4360
+ :::
4361
+
4362
+ Monitor Azure Front Door with metrics covering:
4363
+
4364
+ - **Requests** -- client and origin request rates, origin shield requests (to shield/to origin/rate limited)
4365
+ - **Latency** -- total latency, origin latency
4366
+ - **Data transfer** -- request and response data transfer, origin shield data transfer, byte hit ratio
4367
+ - **Errors** -- 4xx and 5xx error rates
4368
+ - **Origin** -- origin health probe percentage
4369
+ - **WAF** -- WAF requests, challenges (CAPTCHA/JS challenge)
4370
+ - **WebSocket** -- WebSocket connections (requested/active), connection duration
4371
method_description: *method_description
4372
alerts:
4373
- name: am_front_door_origin_health
@@ -3995,6 +4418,8 @@ modules:
4418
description: "The Azure resource type identifier."
4419
- name: profile
4420
description: "The Azure Monitor profile id."
4421
+ - name: subscription_id
4422
+ description: "The Azure subscription identifier."
4423
- name: resource_uid
4424
description: "The unique Azure resource identifier."
4425
metrics:
@@ -4100,7 +4525,24 @@ modules:
4525
<<: *overview
4526
data_collection:
4527
metrics_description: |
4103
- Monitor Virtual Machine Scale Sets including CPU utilization, available memory percentage, disk IOPS and throughput for OS, data, temp, and premium cache disks, disk burst and VM-level burst credit balances, network traffic, and inbound/outbound flow creation rates across all instances in the scale set.
4528
+ :::info
4529
+
4530
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
4531
+
4532
+ :::
4533
+
4534
+ Monitor Azure Virtual Machine Scale Sets with metrics covering:
4535
+
4536
+ - **Compute** -- CPU utilization, CPU credits (consumed/remaining)
4537
+ - **Memory** -- available memory (bytes and percentage)
4538
+ - **Disk** -- IOPS, throughput, latency, and queue depth for OS, data, and temp disks
4539
+ - **Disk burst** -- burst credits and capacity for OS and data disks, VM-level cached/uncached burst credits
4540
+ - **Disk cache** -- premium OS and data disk cache hit/miss rates
4541
+ - **Network** -- traffic in/out, network flows, flow creation rate
4542
+ - **Availability** -- VMSS availability state
4543
+ - **Throttling** -- cached and uncached I/O bandwidth/IOPS throttling
4544
+
4545
+ Metrics are aggregated across all instances in the scale set.
4546
method_description: *method_description
4547
alerts:
4548
- name: am_vmss_cpu
@@ -4227,6 +4669,8 @@ modules:
4669
description: "The Azure resource type identifier."
4670
- name: profile
4671
description: "The Azure Monitor profile id."
4672
+ - name: subscription_id
4673
+ description: "The Azure subscription identifier."
4674
- name: resource_uid
4675
description: "The unique Azure resource identifier."
4676
metrics:
@@ -4493,7 +4937,22 @@ modules:
4937
<<: *overview
4938
data_collection:
4939
metrics_description: |
4496
- Monitor SQL Elastic Pool resource consumption including eDTU and CPU utilization, storage usage, active sessions and workers, IO rates, tempdb usage, and in-memory OLTP storage across all databases in the pool.
4940
+ :::info
4941
+
4942
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
4943
+
4944
+ :::
4945
+
4946
+ Monitor Azure SQL Elastic Pool with metrics covering:
4947
+
4948
+ - **CPU and DTU** -- CPU utilization (average/max), instance CPU, DTU consumption, eDTU and vCore usage
4949
+ - **Memory** -- instance memory utilization
4950
+ - **Storage** -- data and allocated storage, storage utilization, tempdb size, in-memory OLTP storage
4951
+ - **I/O** -- data read and log write utilization, tempdb log utilization
4952
+ - **Sessions** -- active sessions and workers count, serverless CPU/memory utilization
4953
+ - **Billing** -- serverless billing (vCore-seconds)
4954
+
4955
+ Metrics are aggregated across all databases in the pool.
4956
method_description: *method_description
4957
alerts:
4958
- name: am_sql_elastic_pool_cpu
@@ -4572,6 +5031,8 @@ modules:
5031
description: "The Azure resource type identifier."
5032
- name: profile
5033
description: "The Azure Monitor profile id."
5034
+ - name: subscription_id
5035
+ description: "The Azure subscription identifier."
5036
- name: resource_uid
5037
description: "The unique Azure resource identifier."
5038
metrics:
@@ -4700,7 +5161,23 @@ modules:
5161
<<: *overview
5162
data_collection:
5163
metrics_description: |
4703
- Monitor MySQL Flexible Server including active connections, aborted connections, query rates, replication lag, storage utilization, CPU and memory usage, IO operations, InnoDB buffer pool efficiency, network throughput, and HA replication status.
5164
+ :::info
5165
+
5166
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
5167
+
5168
+ :::
5169
+
5170
+ Monitor Azure MySQL Flexible Server with metrics covering:
5171
+
5172
+ - **Compute** -- CPU utilization, CPU credits (consumed/remaining)
5173
+ - **Memory** -- memory utilization
5174
+ - **Storage** -- storage used/limit, storage breakdown (data/ibdata1/binlog), backup storage, server log storage, I/O utilization
5175
+ - **Connections** -- active connections, aborted connections, total connections, threads running
5176
+ - **Queries** -- queries (total/slow), DML statements (select/insert/update/delete), DDL statements
5177
+ - **InnoDB** -- buffer pool I/O (read requests/disk reads), buffer pool pages, data writes, row lock time/waits
5178
+ - **Replication** -- replication lag (replica/HA), HA status (I/O/SQL), replica status
5179
+ - **Network** -- network traffic (in/out)
5180
+ - **Health** -- uptime, deadlocks, lock timeouts
5181
method_description: *method_description
5182
alerts:
5183
- name: am_mysql_flexible_cpu
@@ -4799,6 +5276,8 @@ modules:
5276
description: "The Azure resource type identifier."
5277
- name: profile
5278
description: "The Azure Monitor profile id."
5279
+ - name: subscription_id
5280
+ description: "The Azure subscription identifier."
5281
- name: resource_uid
5282
description: "The unique Azure resource identifier."
5283
metrics:
@@ -5042,7 +5521,21 @@ modules:
5521
<<: *overview
5522
data_collection:
5523
metrics_description: |
5045
- Monitor Container Apps including CPU and memory usage, network traffic, replica counts, request processing rates, response times, restart frequency, and resource reservation utilization.
5524
+ :::info
5525
+
5526
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
5527
+
5528
+ :::
5529
+
5530
+ Monitor Azure Container Apps with metrics covering:
5531
+
5532
+ - **Compute** -- CPU usage (nanocores and percentage), GPU utilization
5533
+ - **Memory** -- memory working set, memory percentage, JVM memory (total/pool/buffer)
5534
+ - **Requests** -- request rate, response time
5535
+ - **Network** -- network traffic (received/sent), resiliency pending connections and timeouts
5536
+ - **Replicas** -- replica count, restart count, reserved cores
5537
+ - **JVM** -- thread count, GC collections and duration, buffer count
5538
+ - **Resiliency** -- host ejections, request retries
5539
method_description: *method_description
5540
alerts:
5541
- name: am_container_apps_cpu_utilization
@@ -5109,6 +5602,8 @@ modules:
5602
description: "The Azure resource type identifier."
5603
- name: profile
5604
description: "The Azure Monitor profile id."
5605
+ - name: subscription_id
5606
+ description: "The Azure subscription identifier."
5607
- name: resource_uid
5608
description: "The unique Azure resource identifier."
5609
metrics:
@@ -5278,7 +5773,20 @@ modules:
5773
<<: *overview
5774
data_collection:
5775
metrics_description: |
5281
- Monitor Azure Firewall including data processed, throughput, application and network rule hit counts, SNAT port utilization, health state percentage, and latency probes.
5776
+ :::info
5777
+
5778
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
5779
+
5780
+ :::
5781
+
5782
+ Monitor Azure Firewall with metrics covering:
5783
+
5784
+ - **Traffic** -- data processed, throughput (bits/s)
5785
+ - **Rules** -- application and network rule hit counts
5786
+ - **SNAT** -- SNAT port utilization
5787
+ - **Health** -- firewall health state percentage
5788
+ - **Latency** -- latency probe
5789
+ - **Capacity** -- observed capacity units
5790
method_description: *method_description
5791
alerts:
5792
- name: am_firewall_health
@@ -5313,6 +5821,8 @@ modules:
5821
description: "The Azure resource type identifier."
5822
- name: profile
5823
description: "The Azure Monitor profile id."
5824
+ - name: subscription_id
5825
+ description: "The Azure subscription identifier."
5826
- name: resource_uid
5827
description: "The unique Azure resource identifier."
5828
metrics:
@@ -5380,7 +5890,26 @@ modules:
5890
<<: *overview
5891
data_collection:
5892
metrics_description: |
5383
- Monitor Azure AI and Cognitive Services including API call volume, success and client error rates, response latency, token processing rates for language models, content safety filtering, fine-tuning operations, provisioned throughput utilization, rate-limiting events, active inference connections, and context token cache performance.
5893
+ :::info
5894
+
5895
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
5896
+
5897
+ :::
5898
+
5899
+ Monitor Azure AI and Cognitive Services with metrics covering:
5900
+
5901
+ - **API calls** -- total calls (successful/blocked/token), model requests, OpenAI requests
5902
+ - **Errors** -- total, client, and server errors, rate limiting events
5903
+ - **Latency** -- service latency, model latency (time to response/first token/between tokens/last byte)
5904
+ - **Tokens** -- model token usage (input/output/total), OpenAI token usage (prompt/generated), cache tokens (read/write)
5905
+ - **Availability** -- service availability, model availability, OpenAI availability
5906
+ - **Content safety** -- content moderation calls (text/image), safety system events, harmful/blocked requests
5907
+ - **Speech** -- transcription, translation, synthesis, speaker recognition, voice training/hosting
5908
+ - **Vision** -- computer vision and custom vision transactions, images stored, training time
5909
+ - **Translator** -- text and document translation (standard/custom)
5910
+ - **Provisioned** -- model provisioned utilization, OpenAI provisioned-managed utilization
5911
+ - **Fine-tuning** -- training hours
5912
+ - **Personalizer** -- events, rewards, actions, feature cardinality
5913
method_description: *method_description
5914
alerts:
5915
- name: am_cognitive_services_availability
@@ -5471,6 +6000,8 @@ modules:
6000
description: "The Azure resource type identifier."
6001
- name: profile
6002
description: "The Azure Monitor profile id."
6003
+ - name: subscription_id
6004
+ description: "The Azure subscription identifier."
6005
- name: resource_uid
6006
description: "The unique Azure resource identifier."
6007
metrics:
@@ -5996,7 +6527,22 @@ modules:
6527
<<: *overview
6528
data_collection:
6529
metrics_description: |
5999
- Monitor VPN Gateway including site-to-site bandwidth and BGP peer status, point-to-site connection counts and bandwidth, per-tunnel ingress and egress traffic with packet counts and drops, IPsec security association counts, route table sizes, NAT flow counts and packet translations, and gateway-level bandwidth utilization.
6530
+ :::info
6531
+
6532
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
6533
+
6534
+ :::
6535
+
6536
+ Monitor Azure VPN Gateway with metrics covering:
6537
+
6538
+ - **Site-to-site** -- S2S bandwidth, tunnel bandwidth, tunnel bytes (ingress/egress), tunnel packets and drops
6539
+ - **Point-to-site** -- P2S connection count, P2S bandwidth
6540
+ - **BGP** -- BGP peer status, routes advertised and learned
6541
+ - **ExpressRoute** -- ExpressRoute gateway bandwidth, CPU, packets, active flows, route changes, VMs in VNet
6542
+ - **IPsec** -- MMSA and QMSA security association counts
6543
+ - **NAT** -- NAT flows, NAT allocations, NATed bytes and packets, NAT packet drops
6544
+ - **Routes** -- user VPN and VNet prefix route counts
6545
+ - **Flows** -- gateway inbound/outbound flows, tunnel total flows, peak PPS, TS mismatch drops
6546
method_description: *method_description
6547
alerts:
6548
- name: am_vpn_gateway_tunnel_packet_drops
@@ -6075,6 +6621,8 @@ modules:
6621
description: "The Azure resource type identifier."
6622
- name: profile
6623
description: "The Azure Monitor profile id."
6624
+ - name: subscription_id
6625
+ description: "The Azure subscription identifier."
6626
- name: resource_uid
6627
description: "The unique Azure resource identifier."
6628
metrics:
@@ -6343,7 +6891,18 @@ modules:
6891
<<: *overview
6892
data_collection:
6893
metrics_description: |
6346
- Monitor Event Grid topics including publish success and failure counts, publish latency, event delivery and routing rates, delivery success and failure counts, dead-lettered events, and matched event routing.
6894
+ :::info
6895
+
6896
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
6897
+
6898
+ :::
6899
+
6900
+ Monitor Azure Event Grid with metrics covering:
6901
+
6902
+ - **Publishing** -- publish rate (success/failed), publish latency
6903
+ - **Delivery** -- events delivered, failed, dropped, and dead-lettered
6904
+ - **Routing** -- matched and unmatched event routing, destination processing duration
6905
+ - **Filters** -- advanced filter evaluations
6906
method_description: *method_description
6907
alerts:
6908
- name: am_event_grid_publish_failures
@@ -6390,6 +6949,8 @@ modules:
6949
description: "The Azure resource type identifier."
6950
- name: profile
6951
description: "The Azure Monitor profile id."
6952
+ - name: subscription_id
6953
+ description: "The Azure subscription identifier."
6954
- name: resource_uid
6955
description: "The unique Azure resource identifier."
6956
metrics:
@@ -6455,7 +7016,23 @@ modules:
7016
<<: *overview
7017
data_collection:
7018
metrics_description: |
6458
- Monitor IoT Hub including device telemetry message rates and quota usage, routing delivery and latency, device twin read and write operations, direct method invocations, cloud-to-device messaging and feedback, job completion rates, device connection and authentication events, and event grid publish status.
7019
+ :::info
7020
+
7021
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
7022
+
7023
+ :::
7024
+
7025
+ Monitor Azure IoT Hub with metrics covering:
7026
+
7027
+ - **Telemetry** -- device telemetry messages (attempted/sent), throttling errors, daily message quota usage
7028
+ - **Routing** -- message deliveries by endpoint (Event Hubs, Service Bus, storage), routing latency, delivery status
7029
+ - **Device twins** -- backend and device twin reads/writes (successful/failed), query results
7030
+ - **Direct methods** -- method invocations (successful/failed), request/response sizes
7031
+ - **Cloud-to-device** -- C2D commands (completed/abandoned/rejected), expired messages
7032
+ - **Jobs** -- job completions, cancellations, list calls, twin update and method job creations
7033
+ - **Connections** -- successful connections, connected devices, total devices
7034
+ - **Event Grid** -- Event Grid deliveries, Event Grid latency
7035
+ - **Data** -- device data usage
7036
method_description: *method_description
7037
alerts:
7038
- name: am_iot_hub_d2c_telemetry_throttle
@@ -6546,6 +7123,8 @@ modules:
7123
description: "The Azure resource type identifier."
7124
- name: profile
7125
description: "The Azure Monitor profile id."
7126
+ - name: subscription_id
7127
+ description: "The Azure subscription identifier."
7128
- name: resource_uid
7129
description: "The unique Azure resource identifier."
7130
metrics:
@@ -6832,7 +7411,22 @@ modules:
7411
<<: *overview
7412
data_collection:
7413
metrics_description: |
6835
- Monitor Data Factory including pipeline, activity, and trigger run success and failure counts, integration runtime CPU and memory utilization, available capacity and queue lengths, SSIS package execution rates, copy operations throughput, data flow processing metrics, and overall factory resource utilization.
7414
+ :::info
7415
+
7416
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
7417
+
7418
+ :::
7419
+
7420
+ Monitor Azure Data Factory with metrics covering:
7421
+
7422
+ - **Pipeline runs** -- pipeline runs (succeeded/failed/cancelled), elapsed time runs
7423
+ - **Activity runs** -- activity runs (succeeded/failed/cancelled)
7424
+ - **Trigger runs** -- trigger runs (succeeded/failed/cancelled)
7425
+ - **Integration runtime** -- IR CPU and memory utilization, available nodes, queue length, task pickup delay
7426
+ - **SSIS** -- SSIS package executions (succeeded/failed/cancelled), IR start/stop operations
7427
+ - **MVNet IR** -- pipeline and copy capacity/utilization, external capacity, queue lengths
7428
+ - **Airflow IR** -- CPU and memory, DAG processing, task instances, scheduler activity, triggers, pool slots
7429
+ - **Factory** -- entity count, factory size (current/max allowed)
7430
method_description: *method_description
7431
alerts:
7432
- name: am_data_factory_pipeline_failed_runs
@@ -6959,6 +7553,8 @@ modules:
7553
description: "The Azure resource type identifier."
7554
- name: profile
7555
description: "The Azure Monitor profile id."
7556
+ - name: subscription_id
7557
+ description: "The Azure subscription identifier."
7558
- name: resource_uid
7559
description: "The unique Azure resource identifier."
7560
metrics:
@@ -7315,7 +7911,18 @@ modules:
7911
<<: *overview
7912
data_collection:
7913
metrics_description: |
7318
- Monitor NAT Gateway including byte and packet counts, connection counts, dropped packets, total SNAT connection counts, and datapath availability.
7914
+ :::info
7915
+
7916
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
7917
+
7918
+ :::
7919
+
7920
+ Monitor Azure NAT Gateway with metrics covering:
7921
+
7922
+ - **Throughput** -- byte and packet throughput
7923
+ - **Connections** -- SNAT connections, total SNAT connections
7924
+ - **Drops** -- dropped packets
7925
+ - **Availability** -- datapath availability percentage
7926
method_description: *method_description
7927
alerts:
7928
- name: am_nat_gateway_datapath_availability
@@ -7354,6 +7961,8 @@ modules:
7961
description: "The Azure resource type identifier."
7962
- name: profile
7963
description: "The Azure Monitor profile id."
7964
+ - name: subscription_id
7965
+ description: "The Azure subscription identifier."
7966
- name: resource_uid
7967
description: "The unique Azure resource identifier."
7968
metrics:
@@ -7413,7 +8022,17 @@ modules:
8022
<<: *overview
8023
data_collection:
8024
metrics_description: |
7416
- Monitor Container Instance groups including CPU and memory usage and network bytes transferred in and out.
8025
+ :::info
8026
+
8027
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
8028
+
8029
+ :::
8030
+
8031
+ Monitor Azure Container Instances with metrics covering:
8032
+
8033
+ - **Compute** -- CPU usage (average/max)
8034
+ - **Memory** -- memory usage (average/max)
8035
+ - **Network** -- network traffic (received/sent)
8036
method_description: *method_description
8037
alerts:
8038
- name: am_container_instances_cpu_usage
@@ -7452,6 +8071,8 @@ modules:
8071
description: "The Azure resource type identifier."
8072
- name: profile
8073
description: "The Azure Monitor profile id."
8074
+ - name: subscription_id
8075
+ description: "The Azure subscription identifier."
8076
- name: resource_uid
8077
description: "The unique Azure resource identifier."
8078
metrics:
@@ -7497,7 +8118,17 @@ modules:
8118
<<: *overview
8119
data_collection:
8120
metrics_description: |
7500
- Monitor Container Registry including storage usage, successful and failed pull and push operation counts, and task run duration.
8121
+ :::info
8122
+
8123
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
8124
+
8125
+ :::
8126
+
8127
+ Monitor Azure Container Registry with metrics covering:
8128
+
8129
+ - **Operations** -- image pulls (successful/total), image pushes (successful/total)
8130
+ - **Storage** -- storage used
8131
+ - **Tasks** -- task run duration, agent pool CPU time
8132
method_description: *method_description
8133
alerts:
8134
- name: am_container_registry_pull_failures
@@ -7528,6 +8159,8 @@ modules:
8159
description: "The Azure resource type identifier."
8160
- name: profile
8161
description: "The Azure Monitor profile id."
8162
+ - name: subscription_id
8163
+ description: "The Azure subscription identifier."
8164
- name: resource_uid
8165
description: "The unique Azure resource identifier."
8166
metrics:
@@ -7584,7 +8217,22 @@ modules:
8217
<<: *overview
8218
data_collection:
8219
metrics_description: |
7587
- Monitor application performance through Application Insights including availability test results and duration, server request rates and response times, dependency call tracking and failures, exception rates by source, browser page load timing breakdown, process CPU and memory usage, IO rates, HTTP request queue depth, page views, and trace volume.
8220
+ :::info
8221
+
8222
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
8223
+
8224
+ :::
8225
+
8226
+ Monitor Azure Application Insights with metrics covering:
8227
+
8228
+ - **Availability** -- availability test percentage, test duration
8229
+ - **Requests** -- server request rate, HTTP request rate, HTTP request execution time, request queue depth
8230
+ - **Responses** -- server response time, server requests (total/failed)
8231
+ - **Dependencies** -- dependency calls (total/failed), dependency duration
8232
+ - **Exceptions** -- exception rate, exceptions by source (total/browser/server)
8233
+ - **Browser** -- page load time, browser timing breakdown (network/send/receive/processing), page views
8234
+ - **Process** -- CPU utilization (process/processor), memory (available/private), I/O rate
8235
+ - **Traces** -- trace volume
8236
method_description: *method_description
8237
alerts:
8238
- name: am_appinsights_availability
@@ -7663,6 +8311,8 @@ modules:
8311
description: "The Azure resource type identifier."
8312
- name: profile
8313
description: "The Azure Monitor profile id."
8314
+ - name: subscription_id
8315
+ description: "The Azure subscription identifier."
8316
- name: resource_uid
8317
description: "The unique Azure resource identifier."
8318
metrics:
@@ -7821,7 +8471,19 @@ modules:
8471
<<: *overview
8472
data_collection:
8473
metrics_description: |
7824
- Monitor ExpressRoute circuits including bits per second in and out, ARP and BGP availability percentages, packet drops, and QoS bit rate throughput.
8474
+ :::info
8475
+
8476
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
8477
+
8478
+ :::
8479
+
8480
+ Monitor Azure ExpressRoute Circuit with metrics covering:
8481
+
8482
+ - **Throughput** -- circuit throughput (bits/s in/out), GlobalReach throughput
8483
+ - **Availability** -- ARP availability, BGP availability
8484
+ - **Bandwidth** -- bandwidth utilization (ingress/egress)
8485
+ - **QoS** -- QoS dropped bits (in/out)
8486
+ - **Routes** -- FastPath routes count
8487
method_description: *method_description
8488
alerts:
8489
- name: am_express_route_circuit_arp_availability
@@ -7868,6 +8530,8 @@ modules:
8530
description: "The Azure resource type identifier."
8531
- name: profile
8532
description: "The Azure Monitor profile id."
8533
+ - name: subscription_id
8534
+ description: "The Azure subscription identifier."
8535
- name: resource_uid
8536
description: "The unique Azure resource identifier."
8537
metrics:
@@ -7938,7 +8602,23 @@ modules:
8602
<<: *overview
8603
data_collection:
8604
metrics_description: |
7941
- Monitor Azure Machine Learning workspaces including active model deployments and registered models, pipeline run completions and failures, compute node utilization and preemptions, quota usage, managed endpoint request latency and rates, estimated GPU utilization, and storage utilization.
8605
+ :::info
8606
+
8607
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
8608
+
8609
+ :::
8610
+
8611
+ Monitor Azure Machine Learning with metrics covering:
8612
+
8613
+ - **Compute** -- CPU utilization and millicores (used/capacity), CPU memory (used/capacity)
8614
+ - **GPU** -- GPU utilization (cluster/node), GPU memory (used/capacity), GPU energy
8615
+ - **Cluster** -- total cores and nodes, cluster cores and nodes by state (active/idle/leaving/preempted/unusable)
8616
+ - **Runs** -- run completion (completed/failed/cancelled), run lifecycle, run issues (errors/warnings)
8617
+ - **Models** -- model registrations (succeeded/failed), model deployments (started/succeeded/failed)
8618
+ - **Quota** -- quota utilization
8619
+ - **Storage** -- disk I/O (read/write), disk usage (used/available), storage API calls
8620
+ - **Network** -- network traffic (in/out), InfiniBand traffic
8621
+ - **AI agents** -- agent runs, messages, tokens, tool calls, events, indexed files
8622
method_description: *method_description
8623
alerts:
8624
- name: am_ml_quota_utilization
@@ -8021,6 +8701,8 @@ modules:
8701
description: "The Azure resource type identifier."
8702
- name: profile
8703
description: "The Azure Monitor profile id."
8704
+ - name: subscription_id
8705
+ description: "The Azure subscription identifier."
8706
- name: resource_uid
8707
description: "The unique Azure resource identifier."
8708
metrics:
@@ -8260,7 +8942,24 @@ modules:
8942
<<: *overview
8943
data_collection:
8944
metrics_description: |
8263
- Monitor Azure Data Explorer (Kusto) clusters including ingestion latency, volume, and success rates, query performance and concurrency, cache utilization, CPU and memory usage, export operations, streaming ingest throughput, materialized view health, instance counts, and follower lag.
8945
+ :::info
8946
+
8947
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
8948
+
8949
+ :::
8950
+
8951
+ Monitor Azure Data Explorer (Kusto) with metrics covering:
8952
+
8953
+ - **Ingestion** -- ingestion latency, volume, result (success/failure), queue length, batch processing
8954
+ - **Queries** -- query count, query duration, concurrent queries, throttled queries/commands
8955
+ - **Streaming ingest** -- data rate, duration, result, utilization
8956
+ - **Cache** -- cache and ingestion utilization
8957
+ - **Compute** -- CPU utilization
8958
+ - **Export** -- continuous export records, result, lateness, pending jobs, export utilization
8959
+ - **Materialized views** -- view health, age, data loss, records in delta, extents rebuild
8960
+ - **Cluster** -- instance count (average/max/min), keep alive, total extents
8961
+ - **Events** -- events received/processed/dropped, blobs received/processed/dropped
8962
+ - **Advanced** -- follower latency, discovery latency, weak consistency latency, partitioning percentage
8963
method_description: *method_description
8964
alerts:
8965
- name: am_data_explorer_keep_alive
@@ -8367,6 +9066,8 @@ modules:
9066
description: "The Azure resource type identifier."
9067
- name: profile
9068
description: "The Azure Monitor profile id."
9069
+ - name: subscription_id
9070
+ description: "The Azure subscription identifier."
9071
- name: resource_uid
9072
description: "The unique Azure resource identifier."
9073
metrics:
@@ -8663,7 +9364,20 @@ modules:
9364
<<: *overview
9365
data_collection:
9366
metrics_description: |
8666
- Monitor Stream Analytics jobs including input and output event counts, streaming unit utilization, watermark delay, backlogged input events, runtime and data conversion errors, out-of-order events, and late input events.
9367
+ :::info
9368
+
9369
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
9370
+
9371
+ :::
9372
+
9373
+ Monitor Azure Stream Analytics with metrics covering:
9374
+
9375
+ - **Events** -- event flow (in/out), backlogged input events
9376
+ - **Errors** -- runtime errors, data conversion errors, deserialization errors
9377
+ - **Timing** -- late, early, and out-of-order events, watermark delay
9378
+ - **Resources** -- CPU and streaming unit memory utilization
9379
+ - **Input** -- input data throughput, input sources received
9380
+ - **Functions** -- function events and requests (total/failed)
9381
method_description: *method_description
9382
alerts:
9383
- name: am_stream_analytics_su_utilization
@@ -8726,6 +9440,8 @@ modules:
9440
description: "The Azure resource type identifier."
9441
- name: profile
9442
description: "The Azure Monitor profile id."
9443
+ - name: subscription_id
9444
+ description: "The Azure subscription identifier."
9445
- name: resource_uid
9446
description: "The unique Azure resource identifier."
9447
metrics:
@@ -8817,7 +9533,19 @@ modules:
9533
<<: *overview
9534
data_collection:
9535
metrics_description: |
8820
- Monitor Synapse Analytics workspaces including pipeline and activity run metrics, SQL request counts and data processing volumes, data flow activity execution, integration runtime CPU and memory utilization, and link table event processing.
9536
+ :::info
9537
+
9538
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
9539
+
9540
+ :::
9541
+
9542
+ Monitor Azure Synapse Analytics with metrics covering:
9543
+
9544
+ - **Pipeline** -- pipeline runs, activity runs, trigger runs
9545
+ - **SQL pool** -- built-in SQL pool requests, login attempts, data processed
9546
+ - **Streaming** -- event flow (in/out), event timing (late/early/out-of-order/backlogged), watermark delay, resource utilization, errors
9547
+ - **Streaming I/O** -- input data throughput, input sources received
9548
+ - **Link** -- connection events, processed data volume, changed rows, processing latency, table events
9549
method_description: *method_description
9550
alerts:
9551
- name: am_synapse_streaming_resource_utilization
@@ -8872,6 +9600,8 @@ modules:
9600
description: "The Azure resource type identifier."
9601
- name: profile
9602
description: "The Azure Monitor profile id."
9603
+ - name: subscription_id
9604
+ description: "The Azure subscription identifier."
9605
- name: resource_uid
9606
description: "The unique Azure resource identifier."
9607
metrics:
@@ -9010,7 +9740,18 @@ modules:
9740
<<: *overview
9741
data_collection:
9742
metrics_description: |
9013
- Monitor Log Analytics workspaces including ingestion volume and latency, query execution counts and volume, available storage capacity, and per-table breakdowns of ingestion rates and billing volume.
9743
+ :::info
9744
+
9745
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
9746
+
9747
+ :::
9748
+
9749
+ Monitor Azure Log Analytics Workspace with metrics covering:
9750
+
9751
+ - **Ingestion** -- ingestion volume (records/s), ingestion latency (average/max/min)
9752
+ - **Queries** -- query count (total/failed), query availability
9753
+ - **Export** -- exported data (bytes/s), exported records
9754
+ - **Legacy agent** -- CPU utilization (processor/privileged/user/idle), memory (available/used/free), disk (free space/utilization/I/O/queue/latency), network (traffic/packets/errors/throughput), swap, paging, system processes, uptime, events, heartbeats, users
9755
method_description: *method_description
9756
alerts:
9757
- name: am_log_analytics_query_availability
@@ -9089,6 +9830,8 @@ modules:
9830
description: "The Azure resource type identifier."
9831
- name: profile
9832
description: "The Azure Monitor profile id."
9833
+ - name: subscription_id
9834
+ description: "The Azure subscription identifier."
9835
- name: resource_uid
9836
description: "The unique Azure resource identifier."
9837
metrics:
@@ -9370,7 +10113,19 @@ modules:
10113
<<: *overview
10114
data_collection:
10115
metrics_description: |
9373
- Monitor ExpressRoute gateways including bits and packets per second for ingress and egress, connection counts, CPU utilization, active flow counts, and gateway scale unit counts.
10116
+ :::info
10117
+
10118
+ This is part of the [Azure Monitor](/src/go/plugin/go.d/collector/azure_monitor/integrations/azure_monitor.md) collector. No separate setup is needed -- a single Azure Monitor job discovers and monitors all supported resource types automatically.
10119
+
10120
+ :::
10121
+
10122
+ Monitor Azure ExpressRoute Gateway with metrics covering:
10123
+
10124
+ - **Throughput** -- gateway throughput, connection throughput (bits/s in/out), packets per second
10125
+ - **Compute** -- CPU utilization
10126
+ - **Flows** -- active flows, max flow creation rate
10127
+ - **Routes** -- routes advertised to peer, routes learned from peer, route changes
10128
+ - **Scale** -- VMs in VNet
10129
method_description: *method_description
10130
alerts:
10131
- name: am_express_route_gateway_cpu
@@ -9413,6 +10168,8 @@ modules:
10168
description: "The Azure resource type identifier."
10169
- name: profile
10170
description: "The Azure Monitor profile id."
10171
+ - name: subscription_id
10172
+ description: "The Azure subscription identifier."
10173
- name: resource_uid
10174
description: "The unique Azure resource identifier."
10175
metrics:
src/go/plugin/go.d/collector/azure_monitor/observation_state.go
+1
@@ -137,6 +137,7 @@ func accumulatorResourceUID(key string) string {
137
func labelValues(labels metrix.Labels) []string {
138
return []string{
139
labels["resource_uid"],
140
+ labels["subscription_id"],
141
labels["resource_name"],
142
labels["resource_group"],
143
labels["region"],
src/go/plugin/go.d/collector/azure_monitor/plan.go
+1
-1
@@ -63,7 +63,7 @@ func buildProfileRuntime(p azureprofiles.Profile) (*profileRuntime, error) {
63
return nil, fmt.Errorf("profile has empty id")
64
}
65
66
- name := stringsTrim(p.Name)
66
+ name := stringsTrim(p.DisplayName)
67
if name == "" {
68
return nil, fmt.Errorf("profile %q has empty name", profileID)
69
}
src/go/plugin/go.d/collector/azure_monitor/profile_catalog_test.go
+148
-8
@@ -16,7 +16,7 @@ func TestLoadProfileCatalog_LoadsStockProfiles(t *testing.T) {
16
catalog, err := azureprofiles.LoadFromDefaultDirs()
17
require.NoError(t, err)
18
19
- for _, key := range []string{
19
+ for _, baseName := range []string{
20
"sql_managed_instance",
21
"sql_database",
22
"postgres_flexible",
@@ -27,11 +27,11 @@ func TestLoadProfileCatalog_LoadsStockProfiles(t *testing.T) {
27
"storage_accounts",
28
"load_balancers",
29
} {
30
- profiles, err := catalog.Resolve([]string{key})
31
- require.NoErrorf(t, err, "expected stock profile %q", key)
30
+ profiles, err := catalog.ResolveBaseNames([]string{baseName})
31
+ require.NoErrorf(t, err, "expected stock profile %q", baseName)
32
assert.Len(t, profiles, 1)
33
}
34
- assert.GreaterOrEqual(t, len(catalog.DefaultProfileIDs()), 9)
34
+ assert.GreaterOrEqual(t, len(catalog.ResourceTypes()), 9)
35
}
36
37
func TestLoadProfileCatalogFromDirs_UserOverridesStock(t *testing.T) {
@@ -133,14 +133,154 @@ template:
133
})
134
require.NoError(t, err)
135
136
- gotProfiles, err := catalog.Resolve([]string{"sql_database"})
136
+ gotProfiles, err := catalog.ResolveBaseNames([]string{"SQL_DATABASE"})
137
require.NoError(t, err)
138
require.Len(t, gotProfiles, 1)
139
got := gotProfiles[0]
140
- assert.Equal(t, "Azure SQL Database (User Override)", got.Name)
140
+ assert.Equal(t, "Azure SQL Database (User Override)", got.DisplayName)
141
+}
142
+
143
+func TestLoadProfileCatalogFromDirs_RejectsDuplicateProfileBasenames(t *testing.T) {
144
+ dir := t.TempDir()
145
+ stockDir := filepath.Join(dir, "stock")
146
+
147
+ require.NoError(t, writeProfileFile(filepath.Join(stockDir, "sql_database.yaml"), `
148
+id: sql_database
149
+name: Azure SQL Database
150
+resource_type: Microsoft.Sql/servers/databases
151
+metrics:
152
+ - id: cpu_percent
153
+ azure_name: cpu_percent
154
+ time_grain: PT1M
155
+ series:
156
+ - aggregation: average
157
+ kind: gauge
158
+template:
159
+ family: Azure SQL Database
160
+ context_namespace: sql_database
161
+ charts:
162
+ - id: am_test_sql_database_cpu
163
+ title: Azure SQL Database CPU
164
+ context: cpu
165
+ family: Utilization
166
+ type: line
167
+ units: percentage
168
+ algorithm: absolute
169
+ label_promotion: [resource_name, resource_group, region, resource_type, profile]
170
+ instances:
171
+ by_labels: [resource_uid]
172
+ dimensions:
173
+ - selector: sql_database.cpu_percent_average
174
+ name: average
175
+`))
176
+ require.NoError(t, writeProfileFile(filepath.Join(stockDir, "nested", "sql_database.yml"), `
177
+id: sql_database_copy
178
+name: azure sql database
179
+resource_type: Microsoft.Sql/servers/databases
180
+metrics:
181
+ - id: cpu_percent
182
+ azure_name: cpu_percent
183
+ time_grain: PT1M
184
+ series:
185
+ - aggregation: average
186
+ kind: gauge
187
+template:
188
+ family: Azure SQL Database Copy
189
+ context_namespace: sql_database_copy
190
+ charts:
191
+ - id: am_test_sql_database_copy_cpu
192
+ title: Azure SQL Database Copy CPU
193
+ context: cpu
194
+ family: Utilization
195
+ type: line
196
+ units: percentage
197
+ algorithm: absolute
198
+ label_promotion: [resource_name, resource_group, region, resource_type, profile]
199
+ instances:
200
+ by_labels: [resource_uid]
201
+ dimensions:
202
+ - selector: sql_database_copy.cpu_percent_average
203
+ name: average
204
+`))
205
+
206
+ _, err := azureprofiles.LoadFromDirs([]azureprofiles.DirSpec{
207
+ {Path: stockDir, IsStock: true},
208
+ })
209
+ require.Error(t, err)
210
+ assert.Contains(t, err.Error(), `duplicate stock profile basename`)
211
+ assert.Contains(t, err.Error(), `"sql_database"`)
212
+}
213
+
214
+func TestLoadProfileCatalogFromDirs_RejectsDuplicateProfileIDsAcrossBasenames(t *testing.T) {
215
+ dir := t.TempDir()
216
+ stockDir := filepath.Join(dir, "stock")
217
+
218
+ require.NoError(t, writeProfileFile(filepath.Join(stockDir, "sql_database.yaml"), `
219
+id: sql_database
220
+name: Azure SQL Database
221
+resource_type: Microsoft.Sql/servers/databases
222
+metrics:
223
+ - id: cpu_percent
224
+ azure_name: cpu_percent
225
+ time_grain: PT1M
226
+ series:
227
+ - aggregation: average
228
+ kind: gauge
229
+template:
230
+ family: Azure SQL Database
231
+ context_namespace: sql_database
232
+ charts:
233
+ - id: am_test_sql_database_cpu
234
+ title: Azure SQL Database CPU
235
+ context: cpu
236
+ family: Utilization
237
+ type: line
238
+ units: percentage
239
+ algorithm: absolute
240
+ label_promotion: [resource_name, resource_group, region, resource_type, profile]
241
+ instances:
242
+ by_labels: [resource_uid]
243
+ dimensions:
244
+ - selector: sql_database.cpu_percent_average
245
+ name: average
246
+`))
247
+ require.NoError(t, writeProfileFile(filepath.Join(stockDir, "sql_database_copy.yaml"), `
248
+id: sql_database
249
+name: Azure SQL Database Copy
250
+resource_type: Microsoft.Sql/servers/databases
251
+metrics:
252
+ - id: cpu_percent
253
+ azure_name: cpu_percent
254
+ time_grain: PT1M
255
+ series:
256
+ - aggregation: average
257
+ kind: gauge
258
+template:
259
+ family: Azure SQL Database Copy
260
+ context_namespace: sql_database_copy
261
+ charts:
262
+ - id: am_test_sql_database_copy_cpu
263
+ title: Azure SQL Database Copy CPU
264
+ context: cpu_copy
265
+ family: Utilization
266
+ type: line
267
+ units: percentage
268
+ algorithm: absolute
269
+ label_promotion: [resource_name, resource_group, region, resource_type, profile]
270
+ instances:
271
+ by_labels: [resource_uid]
272
+ dimensions:
273
+ - selector: sql_database.cpu_percent_average
274
+ name: average
275
+`))
276
142
- defaults := catalog.DefaultProfileIDs()
143
- assert.Equal(t, []string{"postgres_flexible", "sql_database"}, defaults)
277
+ _, err := azureprofiles.LoadFromDirs([]azureprofiles.DirSpec{
278
+ {Path: stockDir, IsStock: true},
279
+ })
280
+ require.Error(t, err)
281
+ assert.Contains(t, err.Error(), `duplicate profile id`)
282
+ assert.Contains(t, err.Error(), `"sql_database"`)
283
+ assert.Contains(t, err.Error(), `"sql_database_copy"`)
284
}
285
286
func writeProfileFile(path, data string) error {
src/go/plugin/go.d/collector/azure_monitor/query_executor.go
+32
-15
@@ -17,7 +17,6 @@ import (
17
)
18
19
type queryExecutor struct {
20
- subscriptionID string
20
maxConcurrency int
21
timeout time.Duration
22
cloudCfg azcloud.Configuration
@@ -28,9 +27,8 @@ type queryExecutor struct {
27
clients map[string]metricsQueryClient
28
}
29
31
-func newQueryExecutor(subscriptionID string, maxConcurrency int, timeout time.Duration, credential azcore.TokenCredential, cloudCfg azcloud.Configuration, newClient func(endpoint string, cred azcore.TokenCredential, cloud azcloud.Configuration) (metricsQueryClient, error)) *queryExecutor {
30
+func newQueryExecutor(maxConcurrency int, timeout time.Duration, credential azcore.TokenCredential, cloudCfg azcloud.Configuration, newClient func(endpoint string, cred azcore.TokenCredential, cloud azcloud.Configuration) (metricsQueryClient, error)) *queryExecutor {
31
return &queryExecutor{
33
- subscriptionID: subscriptionID,
32
maxConcurrency: maxConcurrency,
33
timeout: timeout,
34
cloudCfg: cloudCfg,
@@ -46,7 +44,7 @@ func (e *queryExecutor) reset() {
44
e.clients = make(map[string]metricsQueryClient)
45
}
46
49
-func (e *queryExecutor) runQueryBatches(ctx context.Context, batches []queryBatch, queryEnd time.Time) []queryBatchResult {
47
+func (e *queryExecutor) runQueryBatches(ctx context.Context, batches []queryBatch, queryNow time.Time, queryOffsetSeconds int) []queryBatchResult {
48
workers := e.maxConcurrency
49
if workers < 1 {
50
workers = 1
@@ -62,7 +60,7 @@ func (e *queryExecutor) runQueryBatches(ctx context.Context, batches []queryBatc
60
for i := 0; i < workers; i++ {
61
wg.Go(func() {
62
for batch := range input {
65
- samples, err := e.executeQueryBatch(ctx, batch, queryEnd)
63
+ samples, err := e.executeQueryBatch(ctx, batch, queryNow, queryOffsetSeconds)
64
output <- queryBatchResult{Samples: samples, Err: err}
65
}
66
})
@@ -82,21 +80,21 @@ func (e *queryExecutor) runQueryBatches(ctx context.Context, batches []queryBatc
80
return results
81
}
82
85
-func (e *queryExecutor) executeQueryBatch(ctx context.Context, batch queryBatch, queryEnd time.Time) ([]metricSample, error) {
83
+func (e *queryExecutor) executeQueryBatch(ctx context.Context, batch queryBatch, queryNow time.Time, queryOffsetSeconds int) ([]metricSample, error) {
84
client, err := e.getMetricsClient(batch.Region)
85
if err != nil {
86
return nil, err
87
}
88
89
resourceIDs, resourceByID := queryBatchResourceIndex(batch.Resources)
92
- startTime, endTime, interval, aggregation := queryBatchWindow(batch, queryEnd)
90
+ startTime, endTime, interval, aggregation := queryBatchWindow(batch, queryNow, queryOffsetSeconds)
91
92
reqCtx, cancel := withOptionalTimeout(ctx, e.timeout)
93
defer cancel()
94
95
resp, err := client.QueryResources(
96
reqCtx,
99
- e.subscriptionID,
97
+ batch.SubscriptionID,
98
batch.Profile.MetricNamespace,
99
batch.MetricNames,
100
azmetrics.ResourceIDList{ResourceIDs: resourceIDs},
@@ -132,12 +130,30 @@ func queryBatchResourceIndex(resources []resourceInfo) ([]string, map[string]res
130
return resourceIDs, resourceByID
131
}
132
135
-func queryBatchWindow(batch queryBatch, queryEnd time.Time) (string, string, string, string) {
133
+func queryBatchWindow(batch queryBatch, queryNow time.Time, queryOffsetSeconds int) (string, string, string, string) {
134
+ queryEnd := queryEndForBatch(queryNow, queryOffsetSeconds, batch.TimeGrainEvery)
135
start := queryEnd.Add(-batch.TimeGrainEvery).UTC().Format(time.RFC3339)
136
end := queryEnd.UTC().Format(time.RFC3339)
137
return start, end, batch.TimeGrain, strings.Join(batch.Aggregations, ",")
138
}
139
140
+func queryEndForBatch(now time.Time, queryOffsetSeconds int, batchTimeGrainEvery time.Duration) time.Time {
141
+ offset := effectiveQueryOffset(queryOffsetSeconds, batchTimeGrainEvery)
142
+ queryEnd := now.Add(-offset)
143
+ if queryEnd.IsZero() {
144
+ return now
145
+ }
146
+ return queryEnd
147
+}
148
+
149
+func effectiveQueryOffset(queryOffsetSeconds int, batchTimeGrainEvery time.Duration) time.Duration {
150
+ offset := secondsToDuration(queryOffsetSeconds)
151
+ if batchTimeGrainEvery > offset {
152
+ return batchTimeGrainEvery
153
+ }
154
+ return offset
155
+}
156
+
157
func samplesFromQueryResponse(metricData []azmetrics.MetricData, profileID string, metricToRuntime map[string]*metricRuntime, resourceByID map[string]resourceInfo) []metricSample {
158
samples := make([]metricSample, 0, len(metricData))
159
for _, data := range metricData {
@@ -175,12 +191,13 @@ func samplesFromMetricValues(metrics []azmetrics.Metric, labels metrix.Labels, m
191
192
func resourceLabels(resource resourceInfo, profileID string) metrix.Labels {
193
return metrix.Labels{
178
- "resource_uid": resource.UID,
179
- "resource_name": resource.Name,
180
- "resource_group": resource.ResourceGroup,
181
- "region": resource.Region,
182
- "resource_type": resource.Type,
183
- "profile": profileID,
194
+ "resource_uid": resource.UID,
195
+ "subscription_id": resource.SubscriptionID,
196
+ "resource_name": resource.Name,
197
+ "resource_group": resource.ResourceGroup,
198
+ "region": resource.Region,
199
+ "resource_type": resource.Type,
200
+ "profile": profileID,
201
}
202
}
203
src/go/plugin/go.d/collector/azure_monitor/query_schedule.go
+17
-8
@@ -50,19 +50,20 @@ func (c *Collector) buildProfileQueryBatches(profile *profileRuntime, resources
50
continue
51
}
52
53
- for region, regionResources := range groupResourcesByRegion(resources) {
54
- for metricChunk := range slices.Chunk(metrics, c.MaxMetricsPerQuery) {
53
+ for key, groupedResources := range groupResourcesBySubscriptionRegion(resources) {
54
+ for metricChunk := range slices.Chunk(metrics, c.Limits.MaxMetricsPerQuery) {
55
names := batchMetricNames(metricChunk)
56
aggregations := batchAggregations(metricChunk)
57
- for resourceChunk := range slices.Chunk(regionResources, c.MaxBatchResources) {
57
+ for resourceChunk := range slices.Chunk(groupedResources, c.Limits.MaxBatchResources) {
58
batches = append(batches, queryBatch{
59
+ SubscriptionID: key.SubscriptionID,
60
Profile: profile,
61
Metrics: metricChunk,
62
MetricNames: append([]string(nil), names...),
63
Aggregations: append([]string(nil), aggregations...),
64
TimeGrain: grain,
65
TimeGrainEvery: azureprofiles.SupportedTimeGrains[grain],
65
- Region: region,
66
+ Region: key.Region,
67
Resources: append([]resourceInfo(nil), resourceChunk...),
68
})
69
}
@@ -117,11 +118,19 @@ func batchAggregations(metrics []*metricRuntime) []string {
118
return list
119
}
120
120
-func groupResourcesByRegion(resources []resourceInfo) map[string][]resourceInfo {
121
- result := make(map[string][]resourceInfo)
121
+type subscriptionRegionKey struct {
122
+ SubscriptionID string
123
+ Region string
124
+}
125
+
126
+func groupResourcesBySubscriptionRegion(resources []resourceInfo) map[subscriptionRegionKey][]resourceInfo {
127
+ result := make(map[subscriptionRegionKey][]resourceInfo)
128
for _, r := range resources {
123
- region := normalizeRegion(r.Region)
124
- result[region] = append(result[region], r)
129
+ key := subscriptionRegionKey{
130
+ SubscriptionID: stringsTrim(r.SubscriptionID),
131
+ Region: normalizeRegion(r.Region),
132
+ }
133
+ result[key] = append(result[key], r)
134
}
135
return result
136
}
src/go/plugin/go.d/collector/azure_monitor/testdata/config.json
+25
-16
@@ -1,23 +1,33 @@
1
{
2
"update_every": 60,
3
- "autodetection_retry": 0,
4
- "subscription_id": "sub-1",
3
+ "subscription_ids": [
4
+ "sub-1"
5
+ ],
6
"cloud": "public",
6
- "discovery_every": 300,
7
+ "discovery": {
8
+ "refresh_every": 300,
9
+ "mode": "filters",
10
+ "mode_filters": {
11
+ "resource_groups": [
12
+ "rg-a"
13
+ ]
14
+ }
15
+ },
16
+ "profiles": {
17
+ "mode": "exact",
18
+ "mode_exact": {
19
+ "names": [
20
+ "sql_managed_instance"
21
+ ]
22
+ }
23
+ },
24
"query_offset": 180,
25
"timeout": 30,
9
- "max_concurrency": 4,
10
- "max_batch_resources": 50,
11
- "max_metrics_per_query": 20,
12
- "profile_selection_mode": "exact",
13
- "profile_selection_mode_exact": {
14
- "profiles": [
15
- "sql_managed_instance"
16
- ]
26
+ "limits": {
27
+ "max_concurrency": 4,
28
+ "max_batch_resources": 50,
29
+ "max_metrics_per_query": 20
30
},
18
- "resource_groups": [
19
- "rg-a"
20
- ],
31
"auth": {
32
"mode": "service_principal",
33
"mode_service_principal": {
@@ -25,6 +35,5 @@
35
"client_id": "22222222-2222-2222-2222-222222222222",
36
"client_secret": "secret"
37
}
28
- },
29
- "vnode": ""
38
+ }
39
}
src/go/plugin/go.d/collector/azure_monitor/testdata/config.yaml
+17
-11
@@ -1,18 +1,24 @@
1
update_every: 60
2
-subscription_id: sub-1
2
+subscription_ids:
3
+ - sub-1
4
cloud: public
4
-discovery_every: 300
5
+discovery:
6
+ refresh_every: 300
7
+ mode: filters
8
+ mode_filters:
9
+ resource_groups:
10
+ - rg-a
11
+profiles:
12
+ mode: exact
13
+ mode_exact:
14
+ names:
15
+ - sql_managed_instance
16
query_offset: 180
17
timeout: 30
7
-max_concurrency: 4
8
-max_batch_resources: 50
9
-max_metrics_per_query: 20
10
-profile_selection_mode: exact
11
-profile_selection_mode_exact:
12
- profiles:
13
- - sql_managed_instance
14
-resource_groups:
15
- - rg-a
18
+limits:
19
+ max_concurrency: 4
20
+ max_batch_resources: 50
21
+ max_metrics_per_query: 20
22
auth:
23
mode: service_principal
24
mode_service_principal: