| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | package dcgm |
| 4 | |
| 5 | import ( |
| 6 | "fmt" |
| 7 | "hash/fnv" |
| 8 | "math" |
| 9 | "sort" |
| 10 | "strconv" |
| 11 | "strings" |
| 12 | |
| 13 | "github.com/prometheus/common/model" |
| 14 | promlabels "github.com/prometheus/prometheus/model/labels" |
| 15 | |
| 16 | "github.com/netdata/netdata/go/plugins/pkg/prometheus" |
| 17 | "github.com/netdata/netdata/go/plugins/plugin/framework/collectorapi" |
| 18 | ) |
| 19 | |
| 20 | const precision = 1000.0 |
| 21 | |
| 22 | type interconnectThroughputTotals struct { |
| 23 | instance entityInstance |
| 24 | pcie int64 |
| 25 | nvlink int64 |
| 26 | hasPcie bool |
| 27 | hasNvlink bool |
| 28 | hasExplicitNvlinkTt bool |
| 29 | } |
| 30 | |
| 31 | func (c *Collector) collect() (map[string]int64, error) { |
| 32 | mfs, err := c.prom.Scrape() |
| 33 | if err != nil { |
| 34 | return nil, err |
| 35 | } |
| 36 | |
| 37 | if mfs.Len() == 0 { |
| 38 | c.Warningf("endpoint '%s' returned 0 metric families", c.URL) |
| 39 | return nil, nil |
| 40 | } |
| 41 | |
| 42 | if c.checkMetrics && !hasDCGMMetricFamilies(mfs) { |
| 43 | return nil, fmt.Errorf("'%s' metrics have no DCGM prefix", c.URL) |
| 44 | } |
| 45 | c.checkMetrics = false |
| 46 | |
| 47 | if c.MaxTS > 0 { |
| 48 | if n := calcDCGMMetricSeries(mfs); n > c.MaxTS { |
| 49 | return nil, fmt.Errorf("'%s' num of time series (%d) > limit (%d)", c.URL, n, c.MaxTS) |
| 50 | } |
| 51 | } |
| 52 | |
| 53 | mx := make(map[string]int64) |
| 54 | totals := make(map[string]*interconnectThroughputTotals) |
| 55 | c.cache.reset() |
| 56 | |
| 57 | for _, mf := range mfs { |
| 58 | if !isDCGMMetricName(mf.Name()) { |
| 59 | continue |
| 60 | } |
| 61 | |
| 62 | if c.MaxTSPerMetric > 0 && len(mf.Metrics()) > c.MaxTSPerMetric { |
| 63 | c.Debugf( |
| 64 | "metric '%s' num of time series (%d) > limit (%d), skipping it", |
| 65 | mf.Name(), |
| 66 | len(mf.Metrics()), |
| 67 | c.MaxTSPerMetric, |
| 68 | ) |
| 69 | continue |
| 70 | } |
| 71 | |
| 72 | typ := metricFamilyKind(mf) |
| 73 | if typ == sampleUnsupported { |
| 74 | c.Debugf("metric '%s' has unsupported Prometheus type '%s', skipping it", mf.Name(), mf.Type()) |
| 75 | continue |
| 76 | } |
| 77 | for _, metric := range mf.Metrics() { |
| 78 | value, ok := metricValue(metric, typ) |
| 79 | if !ok || isInvalidMetricValue(value) { |
| 80 | continue |
| 81 | } |
| 82 | |
| 83 | instance := resolveEntityInstance(metric.Labels()) |
| 84 | spec := classifyMetric(instance.entity, mf.Name(), mf.Help(), typ) |
| 85 | skipPrimary := shouldSkipPrimarySeries(spec.Context.ID, mf.Name()) |
| 86 | scaled := int64(value * spec.Scale * precision) |
| 87 | c.accumulateInterconnectTotals(totals, instance, spec.Context.ID, mf.Name(), scaled) |
| 88 | if skipPrimary { |
| 89 | continue |
| 90 | } |
| 91 | |
| 92 | chartKey, chart := c.ensureChart(instance, spec.Context) |
| 93 | dimID := c.ensureDim(chartKey, chart, spec, metric.Labels(), typ) |
| 94 | |
| 95 | mx[dimID] += scaled |
| 96 | } |
| 97 | } |
| 98 | |
| 99 | c.emitInterconnectTotals(mx, totals) |
| 100 | c.removeStaleChartsAndDims() |
| 101 | |
| 102 | if len(mx) == 0 { |
| 103 | return nil, nil |
| 104 | } |
| 105 | |
| 106 | return mx, nil |
| 107 | } |
| 108 | |
| 109 | func (c *Collector) accumulateInterconnectTotals( |
| 110 | totals map[string]*interconnectThroughputTotals, |
| 111 | instance entityInstance, |
| 112 | contextID, metricName string, |
| 113 | scaled int64, |
| 114 | ) { |
| 115 | isPCIe := strings.HasSuffix(contextID, ".interconnect.pcie.throughput") |
| 116 | isNVLink := strings.HasSuffix(contextID, ".interconnect.nvlink.throughput") |
| 117 | if !isPCIe && !isNVLink { |
| 118 | return |
| 119 | } |
| 120 | |
| 121 | key := string(instance.entity) + "|" + instance.key |
| 122 | tot, ok := totals[key] |
| 123 | if !ok { |
| 124 | tot = &interconnectThroughputTotals{instance: instance} |
| 125 | totals[key] = tot |
| 126 | } |
| 127 | |
| 128 | if isPCIe { |
| 129 | tot.pcie += scaled |
| 130 | tot.hasPcie = true |
| 131 | return |
| 132 | } |
| 133 | |
| 134 | if isNVLinkTotalMetricName(metricName) { |
| 135 | if !tot.hasExplicitNvlinkTt { |
| 136 | tot.nvlink = 0 |
| 137 | tot.hasExplicitNvlinkTt = true |
| 138 | } |
| 139 | tot.nvlink += scaled |
| 140 | tot.hasNvlink = true |
| 141 | return |
| 142 | } |
| 143 | |
| 144 | if tot.hasExplicitNvlinkTt { |
| 145 | return |
| 146 | } |
| 147 | tot.nvlink += scaled |
| 148 | tot.hasNvlink = true |
| 149 | } |
| 150 | |
| 151 | func (c *Collector) emitInterconnectTotals(mx map[string]int64, totals map[string]*interconnectThroughputTotals) { |
| 152 | for _, tot := range totals { |
| 153 | if !tot.hasPcie && !tot.hasNvlink { |
| 154 | continue |
| 155 | } |
| 156 | |
| 157 | ctxID := fmt.Sprintf("dcgm.%s.interconnect.total.throughput", tot.instance.entity) |
| 158 | spec, ok := contextCatalog[ctxID] |
| 159 | if !ok { |
| 160 | continue |
| 161 | } |
| 162 | |
| 163 | chartKey, chart := c.ensureChart(tot.instance, spec) |
| 164 | if tot.hasPcie { |
| 165 | dimID := c.ensureDim(chartKey, chart, metricSpec{Context: spec, DimName: "pcie", Scale: 1}, nil, sampleGauge) |
| 166 | mx[dimID] += tot.pcie |
| 167 | } |
| 168 | if tot.hasNvlink { |
| 169 | dimID := c.ensureDim(chartKey, chart, metricSpec{Context: spec, DimName: "nvlink", Scale: 1}, nil, sampleGauge) |
| 170 | mx[dimID] += tot.nvlink |
| 171 | } |
| 172 | } |
| 173 | } |
| 174 | |
| 175 | func (c *Collector) ensureChart(instance entityInstance, spec contextSpec) (string, *collectorapi.Chart) { |
| 176 | chartKey := spec.ID + "|" + instance.key |
| 177 | if ch, ok := c.cache.getChart(chartKey); ok { |
| 178 | return chartKey, ch.chart |
| 179 | } |
| 180 | |
| 181 | chart := &collectorapi.Chart{ |
| 182 | ID: makeID(spec.ID, instance.key), |
| 183 | Title: spec.Title, |
| 184 | Units: spec.Units, |
| 185 | Fam: spec.Family, |
| 186 | Ctx: spec.ID, |
| 187 | Type: spec.Type, |
| 188 | Priority: spec.Priority, |
| 189 | Labels: append([]collectorapi.Label(nil), instance.chartLabels...), |
| 190 | } |
| 191 | |
| 192 | if err := c.Charts().Add(chart); err != nil { |
| 193 | c.Warning(err) |
| 194 | } |
| 195 | |
| 196 | ch := c.cache.putChart(chartKey, chart) |
| 197 | return chartKey, ch.chart |
| 198 | } |
| 199 | |
| 200 | func (c *Collector) ensureDim( |
| 201 | chartKey string, |
| 202 | chart *collectorapi.Chart, |
| 203 | spec metricSpec, |
| 204 | lbls promlabels.Labels, |
| 205 | typ sampleKind, |
| 206 | ) string { |
| 207 | extra := "" |
| 208 | if !strings.HasSuffix(spec.Context.ID, ".reliability.xid") { |
| 209 | extra = semanticDimSuffix(lbls) |
| 210 | } |
| 211 | dimName := spec.DimName |
| 212 | dimName = normalizeDimName(spec.Context.ID, dimName) |
| 213 | if extra != "" { |
| 214 | dimName = dimName + "_" + extra |
| 215 | } |
| 216 | |
| 217 | dimID := makeID(chart.ID, dimName) |
| 218 | |
| 219 | ch, ok := c.cache.charts[chartKey] |
| 220 | if !ok { |
| 221 | return dimID |
| 222 | } |
| 223 | |
| 224 | if exists := ch.touchDim(dimID); !exists { |
| 225 | dim := &collectorapi.Dim{ID: dimID, Name: dimName, Div: int(precision)} |
| 226 | switch typ { |
| 227 | case sampleCounter: |
| 228 | dim.Algo = collectorapi.Incremental |
| 229 | default: |
| 230 | dim.Algo = collectorapi.Absolute |
| 231 | } |
| 232 | if shouldHideDimensionByDefault(spec.Context.ID, dimName) { |
| 233 | dim.Hidden = true |
| 234 | } |
| 235 | |
| 236 | if err := chart.AddDim(dim); err != nil { |
| 237 | c.Warning(err) |
| 238 | } else { |
| 239 | chart.MarkNotCreated() |
| 240 | } |
| 241 | } |
| 242 | |
| 243 | return dimID |
| 244 | } |
| 245 | |
| 246 | func shouldSkipPrimarySeries(contextID, metricName string) bool { |
| 247 | // Keep NVLink total-only bandwidth in the interconnect overview context. |
| 248 | return strings.HasSuffix(contextID, ".interconnect.nvlink.throughput") && |
| 249 | isNVLinkTotalMetricName(metricName) |
| 250 | } |
| 251 | |
| 252 | func normalizeDimName(contextID, dimName string) string { |
| 253 | if isNVLinkThroughputContext(contextID) && strings.HasSuffix(dimName, "_bytes") { |
| 254 | return strings.TrimSuffix(dimName, "_bytes") |
| 255 | } |
| 256 | return dimName |
| 257 | } |
| 258 | |
| 259 | func isNVLinkThroughputContext(contextID string) bool { |
| 260 | return strings.HasSuffix(contextID, ".interconnect.nvlink.throughput") || |
| 261 | strings.HasPrefix(contextID, "dcgm.nvlink.") && strings.HasSuffix(contextID, ".interconnect.throughput") |
| 262 | } |
| 263 | |
| 264 | func metricFamilyKind(mf *prometheus.MetricFamily) sampleKind { |
| 265 | switch mf.Type() { |
| 266 | case model.MetricTypeCounter: |
| 267 | return sampleCounter |
| 268 | case model.MetricTypeGauge: |
| 269 | return sampleGauge |
| 270 | case model.MetricTypeHistogram, model.MetricTypeSummary: |
| 271 | return sampleUnsupported |
| 272 | default: |
| 273 | if strings.HasSuffix(strings.ToLower(mf.Name()), "_total") { |
| 274 | return sampleCounter |
| 275 | } |
| 276 | return sampleGauge |
| 277 | } |
| 278 | } |
| 279 | |
| 280 | func metricValue(metric prometheus.Metric, typ sampleKind) (float64, bool) { |
| 281 | if typ == sampleCounter { |
| 282 | if c := metric.Counter(); c != nil { |
| 283 | return c.Value(), true |
| 284 | } |
| 285 | if u := metric.Untyped(); u != nil { |
| 286 | return u.Value(), true |
| 287 | } |
| 288 | if g := metric.Gauge(); g != nil { |
| 289 | return g.Value(), true |
| 290 | } |
| 291 | return 0, false |
| 292 | } |
| 293 | |
| 294 | if g := metric.Gauge(); g != nil { |
| 295 | return g.Value(), true |
| 296 | } |
| 297 | if u := metric.Untyped(); u != nil { |
| 298 | return u.Value(), true |
| 299 | } |
| 300 | if c := metric.Counter(); c != nil { |
| 301 | return c.Value(), true |
| 302 | } |
| 303 | |
| 304 | return 0, false |
| 305 | } |
| 306 | |
| 307 | func hasDCGMMetricFamilies(mfs prometheus.MetricFamilies) bool { |
| 308 | for name := range mfs { |
| 309 | if isDCGMMetricName(name) { |
| 310 | return true |
| 311 | } |
| 312 | } |
| 313 | return false |
| 314 | } |
| 315 | |
| 316 | func isDCGMMetricName(name string) bool { |
| 317 | return strings.HasPrefix(name, "DCGM_") || strings.HasPrefix(strings.ToLower(name), "dcgm_") |
| 318 | } |
| 319 | |
| 320 | func calcDCGMMetricSeries(mfs prometheus.MetricFamilies) int { |
| 321 | var total int |
| 322 | for name, mf := range mfs { |
| 323 | if !isDCGMMetricName(name) { |
| 324 | continue |
| 325 | } |
| 326 | total += len(mf.Metrics()) |
| 327 | } |
| 328 | return total |
| 329 | } |
| 330 | |
| 331 | func isInvalidMetricValue(v float64) bool { |
| 332 | if math.IsNaN(v) || math.IsInf(v, 0) { |
| 333 | return true |
| 334 | } |
| 335 | // DCGM often uses large sentinel values for unsupported fields. |
| 336 | if math.Abs(v) >= 9e18 { |
| 337 | return true |
| 338 | } |
| 339 | return false |
| 340 | } |
| 341 | |
| 342 | type entityInstance struct { |
| 343 | entity metricEntity |
| 344 | key string |
| 345 | chartLabels []collectorapi.Label |
| 346 | } |
| 347 | |
| 348 | func resolveEntityInstance(lbls promlabels.Labels) entityInstance { |
| 349 | idx := make(map[string]string, len(lbls)) |
| 350 | for _, lbl := range lbls { |
| 351 | if lbl.Name == "" || lbl.Value == "" { |
| 352 | continue |
| 353 | } |
| 354 | idx[strings.ToLower(lbl.Name)] = lbl.Value |
| 355 | } |
| 356 | |
| 357 | entity := detectEntity(idx) |
| 358 | identityKeys := identityKeysForEntity(entity) |
| 359 | parts := make([]string, 0, len(identityKeys)) |
| 360 | |
| 361 | for _, key := range identityKeys { |
| 362 | v, ok := idx[key] |
| 363 | if !ok || v == "" { |
| 364 | continue |
| 365 | } |
| 366 | parts = append(parts, key+"="+v) |
| 367 | } |
| 368 | |
| 369 | if len(parts) == 0 { |
| 370 | parts = append(parts, "global") |
| 371 | } |
| 372 | |
| 373 | chartLabels := buildChartLabels(idx) |
| 374 | |
| 375 | return entityInstance{ |
| 376 | entity: entity, |
| 377 | key: strings.Join(parts, "|"), |
| 378 | chartLabels: chartLabels, |
| 379 | } |
| 380 | } |
| 381 | |
| 382 | func detectEntity(idx map[string]string) metricEntity { |
| 383 | switch { |
| 384 | case hasLabel(idx, "gpu_i_id") || hasLabel(idx, "gpu_instance_id"): |
| 385 | return entityMIG |
| 386 | case hasLabel(idx, "nvlink"): |
| 387 | return entityNVLink |
| 388 | case hasLabel(idx, "nvswitch"): |
| 389 | return entityNVSwitch |
| 390 | case hasLabel(idx, "cpucore"): |
| 391 | return entityCPUCore |
| 392 | case hasLabel(idx, "cpu"): |
| 393 | return entityCPU |
| 394 | case hasLabel(idx, "gpu") || hasLabel(idx, "uuid") || hasLabel(idx, "gpu_uuid"): |
| 395 | return entityGPU |
| 396 | default: |
| 397 | return entityExporter |
| 398 | } |
| 399 | } |
| 400 | |
| 401 | func hasLabel(idx map[string]string, key string) bool { |
| 402 | v, ok := idx[key] |
| 403 | return ok && v != "" |
| 404 | } |
| 405 | |
| 406 | func identityKeysForEntity(entity metricEntity) []string { |
| 407 | workload := []string{"namespace", "pod", "container", "job", "hpc_job", "hpc_job_id"} |
| 408 | switch entity { |
| 409 | case entityGPU: |
| 410 | return append([]string{"gpu", "uuid", "gpu_uuid"}, workload...) |
| 411 | case entityMIG: |
| 412 | return append([]string{"gpu", "uuid", "gpu_uuid", "gpu_i_id", "gpu_instance_id", "gpu_i_profile", "gpu_instance_profile"}, workload...) |
| 413 | case entityNVLink: |
| 414 | return append([]string{"nvswitch", "gpu", "gpu_uuid", "nvlink"}, workload...) |
| 415 | case entityNVSwitch: |
| 416 | return append([]string{"nvswitch"}, workload...) |
| 417 | case entityCPU: |
| 418 | return append([]string{"cpu"}, workload...) |
| 419 | case entityCPUCore: |
| 420 | return append([]string{"cpu", "cpucore"}, workload...) |
| 421 | default: |
| 422 | return append([]string{"hostname"}, workload...) |
| 423 | } |
| 424 | } |
| 425 | |
| 426 | func semanticDimSuffix(lbls promlabels.Labels) string { |
| 427 | if len(lbls) == 0 { |
| 428 | return "" |
| 429 | } |
| 430 | |
| 431 | // Only keep dynamic, semantically meaningful labels that define distinct series |
| 432 | // for a single DCGM field; avoid static identity/metadata labels in dim names. |
| 433 | allowed := map[string]bool{ |
| 434 | "err_code": true, |
| 435 | } |
| 436 | |
| 437 | tokens := make([]string, 0, len(lbls)) |
| 438 | for _, lbl := range lbls { |
| 439 | k := strings.ToLower(lbl.Name) |
| 440 | if lbl.Value == "" || !allowed[k] { |
| 441 | continue |
| 442 | } |
| 443 | tokens = append(tokens, normalizeLabelKey(k)+"_"+sanitizeID(strings.ToLower(lbl.Value))) |
| 444 | } |
| 445 | |
| 446 | if len(tokens) == 0 { |
| 447 | return "" |
| 448 | } |
| 449 | |
| 450 | sort.Strings(tokens) |
| 451 | return strings.Join(tokens, "__") |
| 452 | } |
| 453 | |
| 454 | func normalizeLabelKey(s string) string { |
| 455 | s = strings.ToLower(s) |
| 456 | return sanitizeID(s) |
| 457 | } |
| 458 | |
| 459 | func buildChartLabels(idx map[string]string) []collectorapi.Label { |
| 460 | ignore := map[string]bool{ |
| 461 | "hostname": true, // host identity is already part of Netdata host model |
| 462 | "err_code": true, // used for dynamic XID dimension split, not chart label |
| 463 | "le": true, // histogram bucket label |
| 464 | "quantile": true, // summary label |
| 465 | "__name__": true, // metric family name label, if present |
| 466 | } |
| 467 | |
| 468 | keys := make([]string, 0, len(idx)) |
| 469 | for key, value := range idx { |
| 470 | if value == "" || ignore[key] { |
| 471 | continue |
| 472 | } |
| 473 | keys = append(keys, key) |
| 474 | } |
| 475 | |
| 476 | sort.Strings(keys) |
| 477 | labels := make([]collectorapi.Label, 0, len(keys)) |
| 478 | for _, key := range keys { |
| 479 | labels = append(labels, collectorapi.Label{Key: normalizeLabelKey(key), Value: idx[key]}) |
| 480 | } |
| 481 | return labels |
| 482 | } |
| 483 | |
| 484 | func shouldHideDimensionByDefault(contextID, dimName string) bool { |
| 485 | switch { |
| 486 | case strings.HasSuffix(contextID, ".clock.frequency"): |
| 487 | return containsAny(dimName, |
| 488 | "app_mem_clock", |
| 489 | "app_sm_clock", |
| 490 | "max_mem_clock", |
| 491 | "max_sm_clock", |
| 492 | "max_video_clock", |
| 493 | ) |
| 494 | case strings.HasSuffix(contextID, ".thermal.temperature"): |
| 495 | return containsAny(dimName, |
| 496 | "gpu_max_op_temp", |
| 497 | "gpu_temp_limit", |
| 498 | "mem_max_op_temp", |
| 499 | "shutdown_temp", |
| 500 | "slowdown_temp", |
| 501 | ) |
| 502 | case strings.HasSuffix(contextID, ".power.usage"): |
| 503 | return containsAny(dimName, |
| 504 | "enforced_limit", |
| 505 | "power_mgmt_limit", |
| 506 | "power_mgmt_limit_def", |
| 507 | "power_mgmt_limit_max", |
| 508 | "power_mgmt_limit_min", |
| 509 | ) |
| 510 | case strings.HasSuffix(contextID, ".interconnect.pcie.link.generation"): |
| 511 | return containsAny(dimName, "max_link_gen") |
| 512 | case strings.HasSuffix(contextID, ".interconnect.pcie.link.width"): |
| 513 | return containsAny(dimName, "max_link_width") |
| 514 | default: |
| 515 | return false |
| 516 | } |
| 517 | } |
| 518 | |
| 519 | func isNVLinkTotalMetricName(name string) bool { |
| 520 | n := strings.ToUpper(name) |
| 521 | return strings.Contains(n, "NVLINK") && containsAny(n, "BANDWIDTH_TOTAL", "RX_BANDWIDTH_TOTAL", "TX_BANDWIDTH_TOTAL") |
| 522 | } |
| 523 | |
| 524 | func makeID(parts ...string) string { |
| 525 | raw := strings.Join(parts, "_") |
| 526 | id := sanitizeID(raw) |
| 527 | if len(id) <= 180 { |
| 528 | return id |
| 529 | } |
| 530 | |
| 531 | h := fnv.New64a() |
| 532 | _, _ = h.Write([]byte(id)) |
| 533 | checksum := strconv.FormatUint(h.Sum64(), 36) |
| 534 | return id[:140] + "_" + checksum |
| 535 | } |
| 536 | |
| 537 | func sanitizeID(s string) string { |
| 538 | if s == "" { |
| 539 | return "unknown" |
| 540 | } |
| 541 | |
| 542 | var b strings.Builder |
| 543 | b.Grow(len(s)) |
| 544 | lastUnderscore := false |
| 545 | |
| 546 | for _, r := range s { |
| 547 | isAlphaNum := (r >= 'a' && r <= 'z') || |
| 548 | (r >= 'A' && r <= 'Z') || |
| 549 | (r >= '0' && r <= '9') |
| 550 | if isAlphaNum { |
| 551 | b.WriteRune(r) |
| 552 | lastUnderscore = false |
| 553 | continue |
| 554 | } |
| 555 | |
| 556 | if !lastUnderscore { |
| 557 | b.WriteByte('_') |
| 558 | lastUnderscore = true |
| 559 | } |
| 560 | } |
| 561 | |
| 562 | id := strings.Trim(b.String(), "_") |
| 563 | if id == "" { |
| 564 | id = "unknown" |
| 565 | } |
| 566 | if id[0] >= '0' && id[0] <= '9' { |
| 567 | id = "n_" + id |
| 568 | } |
| 569 | return id |
| 570 | } |