@cryptotaxi247 / netdata-1 / commits / abeb38c40

feat(go.d): add top-queries functions for 9 additional database collectors (#21607)

Costa Tsaousis committed Jan 22, 2026 at 07:05 UTC abeb38c4032780e48b9b80acaaa427dd94dc6d61
98 files changed +7925 -313
src/go/plugin/go.d/collector/clickhouse/collector.go
+4
@@ -23,6 +23,9 @@ func init() {
23 Create: func() module.Module { return New() },
24 Config: func() any { return &Config{} },
25 JobConfigSchema: configSchema,
26 + Methods: clickhouseMethods,
27 + MethodParams: clickhouseMethodParams,
28 + HandleMethod: clickhouseHandleMethod,
29 })
30 }
31
@@ -49,6 +52,7 @@ type Config struct {
52 UpdateEvery int `yaml:"update_every,omitempty" json:"update_every"`
53 AutoDetectionRetry int `yaml:"autodetection_retry,omitempty" json:"autodetection_retry"`
54 web.HTTPConfig `yaml:",inline" json:""`
55 + TopQueriesLimit int `yaml:"top_queries_limit,omitempty" json:"top_queries_limit,omitempty"`
56 }
57
58 type (
src/go/plugin/go.d/collector/clickhouse/config_schema.json
+8
@@ -32,6 +32,14 @@
32 "minimum": 0.5,
33 "default": 1
34 },
35 + "top_queries_limit": {
36 + "title": "Top Queries Limit",
37 + "description": "Maximum number of queries to return in the top-queries function response.",
38 + "type": "integer",
39 + "minimum": 1,
40 + "maximum": 5000,
41 + "default": 500
42 + },
43 "not_follow_redirects": {
44 "title": "Not follow redirects",
45 "description": "If set, the client will not follow HTTP redirects automatically.",
src/go/plugin/go.d/collector/clickhouse/functions.go new
+515
@@ -0,0 +1,515 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package clickhouse
4 +
5 +import (
6 + "context"
7 + "encoding/json"
8 + "fmt"
9 + "strconv"
10 + "strings"
11 +
12 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
13 + "github.com/netdata/netdata/go/plugins/pkg/web"
14 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
15 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
16 +)
17 +
18 +const clickhouseMaxQueryTextLength = 4096
19 +
20 +const (
21 + paramSort = "__sort"
22 +
23 + ftString = funcapi.FieldTypeString
24 + ftInteger = funcapi.FieldTypeInteger
25 + ftFloat = funcapi.FieldTypeFloat
26 + ftDuration = funcapi.FieldTypeDuration
27 +
28 + trNone = funcapi.FieldTransformNone
29 + trNumber = funcapi.FieldTransformNumber
30 + trDuration = funcapi.FieldTransformDuration
31 +
32 + sortAsc = funcapi.FieldSortAscending
33 + sortDesc = funcapi.FieldSortDescending
34 +
35 + summaryCount = funcapi.FieldSummaryCount
36 + summarySum = funcapi.FieldSummarySum
37 + summaryMin = funcapi.FieldSummaryMin
38 + summaryMax = funcapi.FieldSummaryMax
39 + summaryMean = funcapi.FieldSummaryMean
40 +
41 + filterMulti = funcapi.FieldFilterMultiselect
42 + filterRange = funcapi.FieldFilterRange
43 +)
44 +
45 +type clickhouseColumnMeta struct {
46 + dbColumn string
47 + uiKey string
48 + displayName string
49 + dataType funcapi.FieldType
50 + units string
51 + visible bool
52 + transform funcapi.FieldTransform
53 + decimalPoints int
54 + sortDir funcapi.FieldSort
55 + summary funcapi.FieldSummary
56 + filter funcapi.FieldFilter
57 + isSortOption bool
58 + sortLabel string
59 + isDefaultSort bool
60 + isUniqueKey bool
61 + isSticky bool
62 + fullWidth bool
63 + selectExpr string
64 + isLabel bool
65 + isPrimary bool
66 + isMetric bool
67 + chartGroup string
68 + chartTitle string
69 + isDefaultChart bool
70 +}
71 +
72 +var clickhouseAllColumns = []clickhouseColumnMeta{
73 + {dbColumn: "normalized_query_hash", uiKey: "queryId", displayName: "Query ID", dataType: ftString, visible: false, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti, isUniqueKey: true, selectExpr: "toString(normalized_query_hash)"},
74 + {dbColumn: "query", uiKey: "query", displayName: "Query", dataType: ftString, visible: true, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti, isSticky: true, fullWidth: true, selectExpr: "any(query)"},
75 + {dbColumn: "current_database", uiKey: "database", displayName: "Database", dataType: ftString, visible: true, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti, selectExpr: "any(current_database)", isLabel: true, isPrimary: true},
76 + {dbColumn: "user", uiKey: "user", displayName: "User", dataType: ftString, visible: true, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti, selectExpr: "any(user)", isLabel: true},
77 +
78 + {dbColumn: "", uiKey: "calls", displayName: "Calls", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Number of Calls", selectExpr: "count()", isMetric: true, chartGroup: "Calls", chartTitle: "Number of Calls", isDefaultChart: true},
79 +
80 + {dbColumn: "query_duration_ms", uiKey: "totalTime", displayName: "Total Time", dataType: ftDuration, units: "milliseconds", visible: true, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Total Execution Time", isDefaultSort: true, selectExpr: "sum(query_duration_ms)", isMetric: true, chartGroup: "Time", chartTitle: "Execution Time", isDefaultChart: true},
81 + {dbColumn: "query_duration_ms", uiKey: "avgTime", displayName: "Avg Time", dataType: ftDuration, units: "milliseconds", visible: true, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMean, filter: filterRange, isSortOption: true, sortLabel: "Average Execution Time", selectExpr: "avg(query_duration_ms)", isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
82 + {dbColumn: "query_duration_ms", uiKey: "minTime", displayName: "Min Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMin, filter: filterRange, selectExpr: "min(query_duration_ms)", isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
83 + {dbColumn: "query_duration_ms", uiKey: "maxTime", displayName: "Max Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, filter: filterRange, selectExpr: "max(query_duration_ms)", isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
84 +
85 + {dbColumn: "read_rows", uiKey: "readRows", displayName: "Read Rows", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Rows Read", selectExpr: "sum(read_rows)", isMetric: true, chartGroup: "Rows", chartTitle: "Rows"},
86 + {dbColumn: "read_bytes", uiKey: "readBytes", displayName: "Read Bytes", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, selectExpr: "sum(read_bytes)", isMetric: true, chartGroup: "Bytes", chartTitle: "Bytes"},
87 + {dbColumn: "written_rows", uiKey: "writtenRows", displayName: "Written Rows", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, selectExpr: "sum(written_rows)", isMetric: true, chartGroup: "Rows", chartTitle: "Rows"},
88 + {dbColumn: "written_bytes", uiKey: "writtenBytes", displayName: "Written Bytes", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, selectExpr: "sum(written_bytes)", isMetric: true, chartGroup: "Bytes", chartTitle: "Bytes"},
89 + {dbColumn: "result_rows", uiKey: "resultRows", displayName: "Result Rows", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, selectExpr: "sum(result_rows)", isMetric: true, chartGroup: "Rows", chartTitle: "Rows"},
90 + {dbColumn: "result_bytes", uiKey: "resultBytes", displayName: "Result Bytes", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, selectExpr: "sum(result_bytes)", isMetric: true, chartGroup: "Bytes", chartTitle: "Bytes"},
91 + {dbColumn: "memory_usage", uiKey: "memoryUsage", displayName: "Max Memory", dataType: ftFloat, visible: false, transform: trNumber, decimalPoints: 0, sortDir: sortDesc, summary: summaryMax, filter: filterRange, selectExpr: "max(memory_usage)", isMetric: true, chartGroup: "Memory", chartTitle: "Memory"},
92 +}
93 +
94 +type clickhouseJSONResponse struct {
95 + Data []map[string]any `json:"data"`
96 +}
97 +
98 +func clickhouseMethods() []module.MethodConfig {
99 + sortOptions := buildClickHouseSortOptions(clickhouseAllColumns)
100 + return []module.MethodConfig{{
101 + ID: "top-queries",
102 + Name: "Top Queries",
103 + Help: "Top SQL queries from ClickHouse system.query_log",
104 + RequiredParams: []funcapi.ParamConfig{{
105 + ID: paramSort,
106 + Name: "Filter By",
107 + Help: "Select the primary sort column",
108 + Selection: funcapi.ParamSelect,
109 + Options: sortOptions,
110 + UniqueView: true,
111 + }},
112 + }}
113 +}
114 +
115 +func clickhouseMethodParams(ctx context.Context, job *module.Job, method string) ([]funcapi.ParamConfig, error) {
116 + collector, ok := job.Module().(*Collector)
117 + if !ok {
118 + return nil, fmt.Errorf("invalid module type")
119 + }
120 + if collector.httpClient == nil {
121 + return nil, fmt.Errorf("collector is still initializing")
122 + }
123 + switch method {
124 + case "top-queries":
125 + return collector.topQueriesParams(ctx)
126 + default:
127 + return nil, fmt.Errorf("unknown method: %s", method)
128 + }
129 +}
130 +
131 +func clickhouseHandleMethod(ctx context.Context, job *module.Job, method string, params funcapi.ResolvedParams) *module.FunctionResponse {
132 + collector, ok := job.Module().(*Collector)
133 + if !ok {
134 + return &module.FunctionResponse{Status: 500, Message: "internal error: invalid module type"}
135 + }
136 +
137 + if collector.httpClient == nil {
138 + return &module.FunctionResponse{
139 + Status: 503,
140 + Message: "collector is still initializing, please retry in a few seconds",
141 + }
142 + }
143 +
144 + switch method {
145 + case "top-queries":
146 + return collector.collectTopQueries(ctx, params.Column(paramSort))
147 + default:
148 + return &module.FunctionResponse{Status: 404, Message: fmt.Sprintf("unknown method: %s", method)}
149 + }
150 +}
151 +
152 +func buildClickHouseSortOptions(cols []clickhouseColumnMeta) []funcapi.ParamOption {
153 + var sortOptions []funcapi.ParamOption
154 + sortDir := funcapi.FieldSortDescending
155 + for _, col := range cols {
156 + if col.isSortOption {
157 + sortOptions = append(sortOptions, funcapi.ParamOption{
158 + ID: col.uiKey,
159 + Column: col.uiKey,
160 + Name: "Top queries by " + col.sortLabel,
161 + Default: col.isDefaultSort,
162 + Sort: &sortDir,
163 + })
164 + }
165 + }
166 + return sortOptions
167 +}
168 +
169 +func (c *Collector) topQueriesParams(ctx context.Context) ([]funcapi.ParamConfig, error) {
170 + available, err := c.detectQueryLogColumns(ctx)
171 + if err != nil {
172 + return nil, err
173 + }
174 + cols := c.buildAvailableClickHouseColumns(available)
175 + if len(cols) == 0 {
176 + return nil, fmt.Errorf("no columns available in system.query_log")
177 + }
178 + sortParam := funcapi.ParamConfig{
179 + ID: paramSort,
180 + Name: "Filter By",
181 + Help: "Select the primary sort column",
182 + Selection: funcapi.ParamSelect,
183 + Options: buildClickHouseSortOptions(cols),
184 + UniqueView: true,
185 + }
186 + return []funcapi.ParamConfig{sortParam}, nil
187 +}
188 +
189 +func (c *Collector) detectQueryLogColumns(ctx context.Context) (map[string]bool, error) {
190 + query := `
191 +SELECT name
192 +FROM system.columns
193 +WHERE database = 'system' AND table = 'query_log'
194 +FORMAT JSON`
195 +
196 + req, err := web.NewHTTPRequest(c.RequestConfig)
197 + if err != nil {
198 + return nil, err
199 + }
200 + req = req.WithContext(ctx)
201 + req.URL.RawQuery = makeURLQuery(query)
202 +
203 + var resp clickhouseJSONResponse
204 + if err := web.DoHTTP(c.httpClient).RequestJSON(req, &resp); err != nil {
205 + return nil, fmt.Errorf("failed to query system.columns: %w", err)
206 + }
207 +
208 + cols := make(map[string]bool, len(resp.Data))
209 + for _, row := range resp.Data {
210 + if name, ok := row["name"].(string); ok {
211 + cols[name] = true
212 + }
213 + }
214 + if len(cols) == 0 {
215 + return nil, fmt.Errorf("system.query_log not available")
216 + }
217 + return cols, nil
218 +}
219 +
220 +func (c *Collector) buildAvailableClickHouseColumns(available map[string]bool) []clickhouseColumnMeta {
221 + var cols []clickhouseColumnMeta
222 + for _, col := range clickhouseAllColumns {
223 + if col.dbColumn == "" || available[col.dbColumn] {
224 + cols = append(cols, col)
225 + }
226 + }
227 + return cols
228 +}
229 +
230 +func (c *Collector) mapAndValidateClickHouseSortColumn(input string, available []clickhouseColumnMeta) string {
231 + availableKeys := make(map[string]bool, len(available))
232 + for _, col := range available {
233 + availableKeys[col.uiKey] = true
234 + }
235 + if availableKeys[input] {
236 + return input
237 + }
238 + if availableKeys["totalTime"] {
239 + return "totalTime"
240 + }
241 + if availableKeys["calls"] {
242 + return "calls"
243 + }
244 + return available[0].uiKey
245 +}
246 +
247 +func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
248 + availableCols, err := c.detectQueryLogColumns(ctx)
249 + if err != nil {
250 + return &module.FunctionResponse{Status: 503, Message: fmt.Sprintf("system.query_log not available: %v", err)}
251 + }
252 +
253 + cols := c.buildAvailableClickHouseColumns(availableCols)
254 + if len(cols) == 0 {
255 + return &module.FunctionResponse{Status: 500, Message: "no columns available in system.query_log"}
256 + }
257 +
258 + sortColumn = c.mapAndValidateClickHouseSortColumn(sortColumn, cols)
259 +
260 + limit := c.TopQueriesLimit
261 + if limit <= 0 {
262 + limit = 500
263 + }
264 +
265 + groupKey := "normalized_query_hash"
266 + if !availableCols[groupKey] {
267 + groupKey = "query"
268 + }
269 +
270 + selectParts := make([]string, 0, len(cols))
271 + for _, col := range cols {
272 + selectParts = append(selectParts, fmt.Sprintf("%s AS `%s`", col.selectExpr, col.uiKey))
273 + }
274 +
275 + query := fmt.Sprintf(`
276 +SELECT %s
277 +FROM system.query_log
278 +WHERE type = 'QueryFinish'
279 +GROUP BY %s
280 +ORDER BY `+"`%s`"+` DESC
281 +LIMIT %d
282 +FORMAT JSON
283 +`, strings.Join(selectParts, ", "), groupKey, sortColumn, limit)
284 +
285 + req, err := web.NewHTTPRequest(c.RequestConfig)
286 + if err != nil {
287 + return &module.FunctionResponse{Status: 500, Message: err.Error()}
288 + }
289 + req = req.WithContext(ctx)
290 + req.URL.RawQuery = makeURLQuery(query)
291 +
292 + var resp clickhouseJSONResponse
293 + if err := web.DoHTTP(c.httpClient).RequestJSON(req, &resp); err != nil {
294 + if ctx.Err() == context.DeadlineExceeded {
295 + return &module.FunctionResponse{Status: 504, Message: "query timed out"}
296 + }
297 + return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("query failed: %v", err)}
298 + }
299 +
300 + data := make([][]any, 0, len(resp.Data))
301 + for _, rowMap := range resp.Data {
302 + row := make([]any, len(cols))
303 + for i, col := range cols {
304 + row[i] = normalizeClickHouseValue(col, rowMap[col.uiKey])
305 + }
306 + data = append(data, row)
307 + }
308 +
309 + sortParam := funcapi.ParamConfig{
310 + ID: paramSort,
311 + Name: "Filter By",
312 + Help: "Select the primary sort column",
313 + Selection: funcapi.ParamSelect,
314 + Options: buildClickHouseSortOptions(cols),
315 + UniqueView: true,
316 + }
317 +
318 + defaultSort := "totalTime"
319 + if !containsClickHouseColumn(cols, defaultSort) {
320 + defaultSort = "calls"
321 + }
322 +
323 + return &module.FunctionResponse{
324 + Status: 200,
325 + Help: "Top SQL queries from ClickHouse system.query_log",
326 + Columns: buildClickHouseColumns(cols),
327 + Data: data,
328 + DefaultSortColumn: defaultSort,
329 + RequiredParams: []funcapi.ParamConfig{sortParam},
330 + Charts: clickhouseTopQueriesCharts(cols),
331 + DefaultCharts: clickhouseTopQueriesDefaultCharts(cols),
332 + GroupBy: clickhouseTopQueriesGroupBy(cols),
333 + }
334 +}
335 +
336 +func normalizeClickHouseValue(col clickhouseColumnMeta, v any) any {
337 + switch col.dataType {
338 + case ftInteger:
339 + switch val := v.(type) {
340 + case float64:
341 + return int64(val)
342 + case json.Number:
343 + if i, err := val.Int64(); err == nil {
344 + return i
345 + }
346 + case string:
347 + if i, err := strconv.ParseInt(val, 10, 64); err == nil {
348 + return i
349 + }
350 + }
351 + return int64(0)
352 + case ftFloat, ftDuration:
353 + switch val := v.(type) {
354 + case float64:
355 + return val
356 + case json.Number:
357 + if f, err := val.Float64(); err == nil {
358 + return f
359 + }
360 + case string:
361 + if f, err := strconv.ParseFloat(val, 64); err == nil {
362 + return f
363 + }
364 + }
365 + return float64(0)
366 + default:
367 + if s, ok := v.(string); ok {
368 + if col.uiKey == "query" {
369 + return strmutil.TruncateText(s, clickhouseMaxQueryTextLength)
370 + }
371 + return s
372 + }
373 + if v == nil {
374 + return ""
375 + }
376 + if col.uiKey == "query" {
377 + return strmutil.TruncateText(fmt.Sprint(v), clickhouseMaxQueryTextLength)
378 + }
379 + return fmt.Sprint(v)
380 + }
381 +}
382 +
383 +func buildClickHouseColumns(cols []clickhouseColumnMeta) map[string]any {
384 + columns := make(map[string]any, len(cols))
385 + for i, col := range cols {
386 + visual := funcapi.FieldVisualValue
387 + if col.dataType == ftDuration {
388 + visual = funcapi.FieldVisualBar
389 + }
390 + colDef := funcapi.Column{
391 + Index: i,
392 + Name: col.displayName,
393 + Type: col.dataType,
394 + Units: col.units,
395 + Visualization: visual,
396 + Sort: col.sortDir,
397 + Sortable: true,
398 + Sticky: col.isSticky,
399 + Summary: col.summary,
400 + Filter: col.filter,
401 + FullWidth: col.fullWidth,
402 + Wrap: false,
403 + DefaultExpandedFilter: false,
404 + UniqueKey: col.isUniqueKey,
405 + Visible: col.visible,
406 + ValueOptions: funcapi.ValueOptions{
407 + Transform: col.transform,
408 + DecimalPoints: col.decimalPoints,
409 + DefaultValue: nil,
410 + },
411 + }
412 + columns[col.uiKey] = colDef.BuildColumn()
413 + }
414 + return columns
415 +}
416 +
417 +func containsClickHouseColumn(cols []clickhouseColumnMeta, key string) bool {
418 + for _, col := range cols {
419 + if col.uiKey == key {
420 + return true
421 + }
422 + }
423 + return false
424 +}
425 +
426 +func clickhouseTopQueriesCharts(cols []clickhouseColumnMeta) map[string]module.ChartConfig {
427 + charts := make(map[string]module.ChartConfig)
428 + for _, col := range cols {
429 + if !col.isMetric || col.chartGroup == "" {
430 + continue
431 + }
432 + cfg, ok := charts[col.chartGroup]
433 + if !ok {
434 + title := col.chartTitle
435 + if title == "" {
436 + title = col.chartGroup
437 + }
438 + cfg = module.ChartConfig{
439 + Name: title,
440 + Type: "stacked-bar",
441 + }
442 + }
443 + cfg.Columns = append(cfg.Columns, col.uiKey)
444 + charts[col.chartGroup] = cfg
445 + }
446 + return charts
447 +}
448 +
449 +func clickhouseTopQueriesDefaultCharts(cols []clickhouseColumnMeta) [][]string {
450 + label := primaryClickhouseLabel(cols)
451 + if label == "" {
452 + return nil
453 + }
454 + chartGroups := defaultClickhouseChartGroups(cols)
455 + out := make([][]string, 0, len(chartGroups))
456 + for _, group := range chartGroups {
457 + out = append(out, []string{group, label})
458 + }
459 + return out
460 +}
461 +
462 +func clickhouseTopQueriesGroupBy(cols []clickhouseColumnMeta) map[string]module.GroupByConfig {
463 + groupBy := make(map[string]module.GroupByConfig)
464 + for _, col := range cols {
465 + if !col.isLabel {
466 + continue
467 + }
468 + groupBy[col.uiKey] = module.GroupByConfig{
469 + Name: "Group by " + col.displayName,
470 + Columns: []string{col.uiKey},
471 + }
472 + }
473 + return groupBy
474 +}
475 +
476 +func primaryClickhouseLabel(cols []clickhouseColumnMeta) string {
477 + for _, col := range cols {
478 + if col.isPrimary {
479 + return col.uiKey
480 + }
481 + }
482 + for _, col := range cols {
483 + if col.isLabel {
484 + return col.uiKey
485 + }
486 + }
487 + return ""
488 +}
489 +
490 +func defaultClickhouseChartGroups(cols []clickhouseColumnMeta) []string {
491 + groups := make([]string, 0)
492 + seen := make(map[string]bool)
493 + for _, col := range cols {
494 + if !col.isMetric || col.chartGroup == "" || !col.isDefaultChart {
495 + continue
496 + }
497 + if !seen[col.chartGroup] {
498 + seen[col.chartGroup] = true
499 + groups = append(groups, col.chartGroup)
500 + }
501 + }
502 + if len(groups) > 0 {
503 + return groups
504 + }
505 + for _, col := range cols {
506 + if !col.isMetric || col.chartGroup == "" {
507 + continue
508 + }
509 + if !seen[col.chartGroup] {
510 + seen[col.chartGroup] = true
511 + groups = append(groups, col.chartGroup)
512 + }
513 + }
514 + return groups
515 +}
src/go/plugin/go.d/collector/clickhouse/functions_test.go new
+74
@@ -0,0 +1,74 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package clickhouse
4 +
5 +import (
6 + "testing"
7 +
8 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
9 + "github.com/stretchr/testify/assert"
10 +)
11 +
12 +func TestClickHouseMethods(t *testing.T) {
13 + methods := clickhouseMethods()
14 +
15 + require := assert.New(t)
16 + require.Len(methods, 1)
17 + require.Equal("top-queries", methods[0].ID)
18 + require.Equal("Top Queries", methods[0].Name)
19 + require.NotEmpty(methods[0].RequiredParams)
20 +
21 + var sortParam *funcapi.ParamConfig
22 + for i := range methods[0].RequiredParams {
23 + if methods[0].RequiredParams[i].ID == "__sort" {
24 + sortParam = &methods[0].RequiredParams[i]
25 + break
26 + }
27 + }
28 + require.NotNil(sortParam, "expected __sort required param")
29 + require.NotEmpty(sortParam.Options)
30 +}
31 +
32 +func TestClickHouseAllColumns_HasRequiredColumns(t *testing.T) {
33 + required := []string{"query", "calls", "totalTime"}
34 +
35 + uiKeys := make(map[string]bool)
36 + for _, col := range clickhouseAllColumns {
37 + uiKeys[col.uiKey] = true
38 + }
39 +
40 + for _, key := range required {
41 + assert.True(t, uiKeys[key], "column %s should be defined", key)
42 + }
43 +}
44 +
45 +func TestCollector_mapAndValidateClickHouseSortColumn(t *testing.T) {
46 + tests := map[string]struct {
47 + available []clickhouseColumnMeta
48 + input string
49 + expected string
50 + }{
51 + "valid totalTime": {
52 + available: []clickhouseColumnMeta{{uiKey: "totalTime"}, {uiKey: "calls"}},
53 + input: "totalTime",
54 + expected: "totalTime",
55 + },
56 + "invalid falls back to totalTime": {
57 + available: []clickhouseColumnMeta{{uiKey: "totalTime"}, {uiKey: "calls"}},
58 + input: "bad",
59 + expected: "totalTime",
60 + },
61 + "fallback to calls": {
62 + available: []clickhouseColumnMeta{{uiKey: "calls"}},
63 + input: "bad",
64 + expected: "calls",
65 + },
66 + }
67 +
68 + for name, tc := range tests {
69 + t.Run(name, func(t *testing.T) {
70 + c := &Collector{}
71 + assert.Equal(t, tc.expected, c.mapAndValidateClickHouseSortColumn(tc.input, tc.available))
72 + })
73 + }
74 +}
src/go/plugin/go.d/collector/cockroachdb/collector.go
+18 -5
@@ -4,6 +4,7 @@ package cockroachdb
4
5 import (
6 "context"
7 + "database/sql"
8 _ "embed"
9 "errors"
10 "fmt"
@@ -28,8 +29,11 @@ func init() {
29 Defaults: module.Defaults{
30 UpdateEvery: dbSamplingInterval,
31 },
31 - Create: func() module.Module { return New() },
32 - Config: func() any { return &Config{} },
32 + Methods: cockroachMethods,
33 + MethodParams: cockroachMethodParams,
34 + HandleMethod: cockroachHandleMethod,
35 + Create: func() module.Module { return New() },
36 + Config: func() any { return &Config{} },
37 })
38 }
39
@@ -44,15 +48,19 @@ func New() *Collector {
48 Timeout: confopt.Duration(time.Second),
49 },
50 },
51 + SQLTimeout: confopt.Duration(time.Second),
52 },
53 charts: charts.Copy(),
54 }
55 }
56
57 type Config struct {
53 - Vnode string `yaml:"vnode,omitempty" json:"vnode"`
54 - UpdateEvery int `yaml:"update_every,omitempty" json:"update_every"`
55 - AutoDetectionRetry int `yaml:"autodetection_retry,omitempty" json:"autodetection_retry"`
58 + Vnode string `yaml:"vnode,omitempty" json:"vnode"`
59 + UpdateEvery int `yaml:"update_every,omitempty" json:"update_every"`
60 + AutoDetectionRetry int `yaml:"autodetection_retry,omitempty" json:"autodetection_retry"`
61 + DSN string `yaml:"dsn,omitempty" json:"dsn,omitempty"`
62 + SQLTimeout confopt.Duration `yaml:"sql_timeout,omitempty" json:"sql_timeout,omitempty"`
63 + TopQueriesLimit int `yaml:"top_queries_limit,omitempty" json:"top_queries_limit,omitempty"`
64 web.HTTPConfig `yaml:",inline" json:""`
65 }
66
@@ -63,6 +71,8 @@ type Collector struct {
71 charts *Charts
72
73 prom prometheus.Prometheus
74 +
75 + db *sql.DB
76 }
77
78 func (c *Collector) Configuration() any {
@@ -120,4 +130,7 @@ func (c *Collector) Cleanup(context.Context) {
130 if c.prom != nil && c.prom.HTTPClient() != nil {
131 c.prom.HTTPClient().CloseIdleConnections()
132 }
133 + if c.db != nil {
134 + _ = c.db.Close()
135 + }
136 }
src/go/plugin/go.d/collector/cockroachdb/config_schema.json
+35
@@ -32,6 +32,26 @@
32 "minimum": 0.5,
33 "default": 1
34 },
35 + "dsn": {
36 + "title": "SQL DSN",
37 + "description": "CockroachDB SQL Data Source Name for query functions (top-queries, running-queries).",
38 + "type": "string"
39 + },
40 + "sql_timeout": {
41 + "title": "SQL Timeout",
42 + "description": "Timeout in seconds for SQL query functions.",
43 + "type": "number",
44 + "minimum": 0.5,
45 + "default": 1
46 + },
47 + "top_queries_limit": {
48 + "title": "Top Queries Limit",
49 + "description": "Maximum number of rows returned by the top-queries and running-queries functions.",
50 + "type": "integer",
51 + "minimum": 1,
52 + "maximum": 5000,
53 + "default": 500
54 + },
55 "not_follow_redirects": {
56 "title": "Not follow redirects",
57 "description": "If set, the client will not follow HTTP redirects automatically.",
@@ -143,6 +163,14 @@
163 "vnode"
164 ]
165 },
166 + {
167 + "title": "SQL",
168 + "fields": [
169 + "dsn",
170 + "sql_timeout",
171 + "top_queries_limit"
172 + ]
173 + },
174 {
175 "title": "Auth",
176 "fields": [
@@ -200,6 +228,13 @@
228 "timeout": {
229 "ui:help": "Accepts decimals for precise control (e.g., type 1.5 for 1.5 seconds)."
230 },
231 + "dsn": {
232 + "ui:help": "Format is `postgres://username:password@host:port/dbname?sslmode=disable`.",
233 + "ui:placeholder": "postgres://username:password@host:port/defaultdb?sslmode=disable"
234 + },
235 + "sql_timeout": {
236 + "ui:help": "Accepts decimals for precise control (e.g., type 1.5 for 1.5 seconds)."
237 + },
238 "username": {
239 "ui:widget": "password"
240 },
src/go/plugin/go.d/collector/cockroachdb/functions.go new
+553
@@ -0,0 +1,553 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package cockroachdb
4 +
5 +import (
6 + "context"
7 + "database/sql"
8 + "errors"
9 + "fmt"
10 + "strings"
11 + "time"
12 +
13 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
14 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
15 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
16 +)
17 +
18 +const crdbMaxQueryTextLength = 4096
19 +
20 +const (
21 + paramSort = "__sort"
22 +
23 + ftString = funcapi.FieldTypeString
24 + ftInteger = funcapi.FieldTypeInteger
25 + ftFloat = funcapi.FieldTypeFloat
26 + ftDuration = funcapi.FieldTypeDuration
27 +
28 + trNone = funcapi.FieldTransformNone
29 + trNumber = funcapi.FieldTransformNumber
30 + trDuration = funcapi.FieldTransformDuration
31 + trText = funcapi.FieldTransformText
32 +
33 + visValue = funcapi.FieldVisualValue
34 + visBar = funcapi.FieldVisualBar
35 +
36 + sortAsc = funcapi.FieldSortAscending
37 + sortDesc = funcapi.FieldSortDescending
38 +
39 + summaryCount = funcapi.FieldSummaryCount
40 + summarySum = funcapi.FieldSummarySum
41 + summaryMax = funcapi.FieldSummaryMax
42 + summaryMean = funcapi.FieldSummaryMean
43 +
44 + filterMulti = funcapi.FieldFilterMultiselect
45 + filterRange = funcapi.FieldFilterRange
46 +)
47 +
48 +var errSQLDSNNotSet = errors.New("SQL DSN is not set")
49 +
50 +type crdbColumnMeta struct {
51 + id string
52 + name string
53 + selectExpr string
54 + dataType funcapi.FieldType
55 + visible bool
56 + sortable bool
57 + fullWidth bool
58 + wrap bool
59 + sticky bool
60 + filter funcapi.FieldFilter
61 + visualization funcapi.FieldVisual
62 + transform funcapi.FieldTransform
63 + units string
64 + decimalPoints int
65 + uniqueKey bool
66 + sortDir funcapi.FieldSort
67 + summary funcapi.FieldSummary
68 + sortLabel string
69 + isSortOption bool
70 + isDefaultSort bool
71 + isLabel bool
72 + isPrimary bool
73 + isMetric bool
74 + chartGroup string
75 + chartTitle string
76 + isDefaultChart bool
77 +}
78 +
79 +var crdbTopColumns = []crdbColumnMeta{
80 + {id: "fingerprintId", name: "Fingerprint ID", selectExpr: "s.fingerprint_id::STRING", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, uniqueKey: true, sortDir: sortAsc, summary: summaryCount},
81 + {id: "query", name: "Query", selectExpr: "s.metadata->>'query'", dataType: ftString, visible: true, sortable: false, filter: filterMulti, transform: trText, sticky: true, fullWidth: true, wrap: true},
82 + {id: "database", name: "Database", selectExpr: "s.metadata->>'db'", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount, isLabel: true, isPrimary: true},
83 + {id: "application", name: "Application", selectExpr: "s.app_name", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount, isLabel: true},
84 + {id: "statementType", name: "Statement Type", selectExpr: "s.metadata->>'stmtTyp'", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount, isLabel: true},
85 + {id: "distributed", name: "Distributed", selectExpr: "s.metadata->>'distsql'", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
86 + {id: "fullScan", name: "Full Scan", selectExpr: "s.metadata->>'fullScan'", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
87 + {id: "implicitTxn", name: "Implicit Txn", selectExpr: "s.metadata->>'implicitTxn'", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
88 + {id: "vectorized", name: "Vectorized", selectExpr: "s.metadata->>'vec'", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
89 +
90 + {id: "executions", name: "Executions", selectExpr: "COALESCE((s.statistics->'statistics'->>'cnt')::INT8, 0)", dataType: ftInteger, visible: true, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by Executions", isMetric: true, chartGroup: "Calls", chartTitle: "Executions", isDefaultChart: true},
91 + {id: "totalTime", name: "Total Time", selectExpr: "COALESCE((s.statistics->'statistics'->'svcLat'->>'mean')::FLOAT8, 0) * COALESCE((s.statistics->'statistics'->>'cnt')::FLOAT8, 0) * 1000", dataType: ftDuration, visible: true, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summarySum, isSortOption: true, isDefaultSort: true, sortLabel: "Top queries by Total Time", isMetric: true, chartGroup: "Time", chartTitle: "Execution Time", isDefaultChart: true},
92 + {id: "meanTime", name: "Mean Time", selectExpr: "COALESCE((s.statistics->'statistics'->'svcLat'->>'mean')::FLOAT8, 0) * 1000", dataType: ftDuration, visible: true, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMean, isSortOption: true, sortLabel: "Top queries by Mean Time", isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
93 + {id: "runTime", name: "Run Time", selectExpr: "COALESCE((s.statistics->'statistics'->'runLat'->>'mean')::FLOAT8, 0) * 1000", dataType: ftDuration, visible: false, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMean, isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
94 + {id: "planTime", name: "Plan Time", selectExpr: "COALESCE((s.statistics->'statistics'->'planLat'->>'mean')::FLOAT8, 0) * 1000", dataType: ftDuration, visible: false, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMean, isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
95 + {id: "parseTime", name: "Parse Time", selectExpr: "COALESCE((s.statistics->'statistics'->'parseLat'->>'mean')::FLOAT8, 0) * 1000", dataType: ftDuration, visible: false, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMean, isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
96 +
97 + {id: "rowsRead", name: "Rows Read", selectExpr: "CAST(ROUND(COALESCE((s.statistics->'statistics'->'rowsRead'->>'mean')::FLOAT8, 0) * COALESCE((s.statistics->'statistics'->>'cnt')::FLOAT8, 0)) AS INT8)", dataType: ftInteger, visible: true, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by Rows Read", isMetric: true, chartGroup: "Rows", chartTitle: "Rows"},
98 + {id: "rowsWritten", name: "Rows Written", selectExpr: "CAST(ROUND(COALESCE((s.statistics->'statistics'->'rowsWritten'->>'mean')::FLOAT8, 0) * COALESCE((s.statistics->'statistics'->>'cnt')::FLOAT8, 0)) AS INT8)", dataType: ftInteger, visible: true, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by Rows Written", isMetric: true, chartGroup: "Rows", chartTitle: "Rows"},
99 + {id: "rowsReturned", name: "Rows Returned", selectExpr: "CAST(ROUND(COALESCE((s.statistics->'statistics'->'numRows'->>'mean')::FLOAT8, 0) * COALESCE((s.statistics->'statistics'->>'cnt')::FLOAT8, 0)) AS INT8)", dataType: ftInteger, visible: true, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by Rows Returned", isMetric: true, chartGroup: "Rows", chartTitle: "Rows"},
100 + {id: "bytesRead", name: "Bytes Read", selectExpr: "CAST(ROUND(COALESCE((s.statistics->'statistics'->'bytesRead'->>'mean')::FLOAT8, 0) * COALESCE((s.statistics->'statistics'->>'cnt')::FLOAT8, 0)) AS INT8)", dataType: ftInteger, visible: false, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by Bytes Read", isMetric: true, chartGroup: "Bytes", chartTitle: "Bytes"},
101 + {id: "maxRetries", name: "Max Retries", selectExpr: "COALESCE((s.statistics->'statistics'->>'maxRetries')::INT8, 0)", dataType: ftInteger, visible: false, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summaryMax, isMetric: true, chartGroup: "Retries", chartTitle: "Retries"},
102 +}
103 +
104 +var crdbRunningColumns = []crdbColumnMeta{
105 + {id: "queryId", name: "Query ID", selectExpr: "s.query_id::STRING", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, uniqueKey: true, sortDir: sortAsc, summary: summaryCount},
106 + {id: "query", name: "Query", selectExpr: "s.query", dataType: ftString, visible: true, sortable: false, filter: filterMulti, transform: trText, sticky: true, fullWidth: true, wrap: true},
107 + {id: "user", name: "User", selectExpr: "s.user_name", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
108 + {id: "application", name: "Application", selectExpr: "s.application_name", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
109 + {id: "clientAddress", name: "Client Address", selectExpr: "s.client_address", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
110 + {id: "nodeId", name: "Node ID", selectExpr: "s.node_id::STRING", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
111 + {id: "sessionId", name: "Session ID", selectExpr: "s.session_id::STRING", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
112 + {id: "phase", name: "Phase", selectExpr: "s.phase", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
113 + {id: "distributed", name: "Distributed", selectExpr: "s.distributed::STRING", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
114 + {id: "startTime", name: "Start Time", selectExpr: "TO_CHAR(s.start, 'YYYY-MM-DD\"T\"HH24:MI:SS.FF3')", dataType: ftString, visible: false, sortable: true, filter: filterRange, transform: trText, sortDir: sortDesc, summary: summaryMax},
115 + {id: "elapsedMs", name: "Elapsed", selectExpr: "EXTRACT(EPOCH FROM (clock_timestamp() - s.start)) * 1000", dataType: ftDuration, visible: true, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, isSortOption: true, isDefaultSort: true, sortLabel: "Running queries by Elapsed Time"},
116 +}
117 +
118 +func cockroachMethods() []module.MethodConfig {
119 + return []module.MethodConfig{
120 + {
121 + ID: "top-queries",
122 + Name: "Top Queries",
123 + Help: "Top SQL statements from crdb_internal.cluster_statement_statistics. WARNING: Query text may contain unmasked literals (potential PII).",
124 + RequiredParams: []funcapi.ParamConfig{
125 + buildCrdbSortParam(crdbTopColumns),
126 + },
127 + },
128 + {
129 + ID: "running-queries",
130 + Name: "Running Queries",
131 + Help: "Currently running SQL statements from SHOW CLUSTER STATEMENTS. WARNING: Query text may contain unmasked literals (potential PII).",
132 + RequiredParams: []funcapi.ParamConfig{
133 + buildCrdbSortParam(crdbRunningColumns),
134 + },
135 + },
136 + }
137 +}
138 +
139 +func cockroachMethodParams(_ context.Context, _ *module.Job, method string) ([]funcapi.ParamConfig, error) {
140 + switch method {
141 + case "top-queries":
142 + return []funcapi.ParamConfig{buildCrdbSortParam(crdbTopColumns)}, nil
143 + case "running-queries":
144 + return []funcapi.ParamConfig{buildCrdbSortParam(crdbRunningColumns)}, nil
145 + default:
146 + return nil, fmt.Errorf("unknown method: %s", method)
147 + }
148 +}
149 +
150 +func cockroachHandleMethod(ctx context.Context, job *module.Job, method string, params funcapi.ResolvedParams) *module.FunctionResponse {
151 + collector, ok := job.Module().(*Collector)
152 + if !ok {
153 + return &module.FunctionResponse{Status: 500, Message: "internal error: invalid module type"}
154 + }
155 +
156 + if err := collector.ensureSQL(ctx); err != nil {
157 + status := 503
158 + if errors.Is(err, errSQLDSNNotSet) {
159 + status = 400
160 + }
161 + return &module.FunctionResponse{Status: status, Message: err.Error()}
162 + }
163 +
164 + switch method {
165 + case "top-queries":
166 + return collector.collectTopQueries(ctx, params.Column(paramSort))
167 + case "running-queries":
168 + return collector.collectRunningQueries(ctx, params.Column(paramSort))
169 + default:
170 + return &module.FunctionResponse{Status: 404, Message: fmt.Sprintf("unknown method: %s", method)}
171 + }
172 +}
173 +
174 +func (c *Collector) ensureSQL(ctx context.Context) error {
175 + if c.db != nil {
176 + return nil
177 + }
178 + if c.DSN == "" {
179 + return errSQLDSNNotSet
180 + }
181 +
182 + db, err := sql.Open("pgx", c.DSN)
183 + if err != nil {
184 + return fmt.Errorf("error opening SQL connection: %w", err)
185 + }
186 + db.SetMaxOpenConns(1)
187 + db.SetMaxIdleConns(1)
188 + db.SetConnMaxLifetime(10 * time.Minute)
189 +
190 + timeout := c.sqlTimeout()
191 + pingCtx, cancel := context.WithTimeout(ctx, timeout)
192 + defer cancel()
193 + if err := db.PingContext(pingCtx); err != nil {
194 + _ = db.Close()
195 + return fmt.Errorf("error pinging SQL connection: %w", err)
196 + }
197 +
198 + setCtx, cancel := context.WithTimeout(ctx, timeout)
199 + if _, err := db.ExecContext(setCtx, "SET allow_unsafe_internals = on"); err != nil {
200 + c.Debugf("unable to set allow_unsafe_internals: %v", err)
201 + }
202 + cancel()
203 +
204 + c.db = db
205 + return nil
206 +}
207 +
208 +func (c *Collector) sqlTimeout() time.Duration {
209 + if c.SQLTimeout.Duration() > 0 {
210 + return c.SQLTimeout.Duration()
211 + }
212 + return time.Second
213 +}
214 +
215 +func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
216 + sortColumn = resolveCrdbSortColumn(crdbTopColumns, sortColumn)
217 + limit := c.TopQueriesLimit
218 + if limit <= 0 {
219 + limit = 500
220 + }
221 +
222 + query := buildCrdbTopQueriesSQL(sortColumn, limit)
223 + queryCtx, cancel := context.WithTimeout(ctx, c.sqlTimeout())
224 + defer cancel()
225 + rows, err := c.db.QueryContext(queryCtx, query, limit)
226 + if err != nil {
227 + if errors.Is(queryCtx.Err(), context.DeadlineExceeded) {
228 + return &module.FunctionResponse{Status: 504, Message: "query timed out"}
229 + }
230 + return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("query failed: %v", err)}
231 + }
232 + defer rows.Close()
233 +
234 + data, err := scanCrdbRows(rows, crdbTopColumns)
235 + if err != nil {
236 + return &module.FunctionResponse{Status: 500, Message: err.Error()}
237 + }
238 +
239 + return &module.FunctionResponse{
240 + Status: 200,
241 + Help: "Top SQL statements from crdb_internal.cluster_statement_statistics. WARNING: Query text may contain unmasked literals (potential PII).",
242 + Columns: buildCrdbColumns(crdbTopColumns),
243 + Data: data,
244 + DefaultSortColumn: sortColumn,
245 + RequiredParams: []funcapi.ParamConfig{buildCrdbSortParam(crdbTopColumns)},
246 + Charts: crdbTopQueriesCharts(crdbTopColumns),
247 + DefaultCharts: crdbTopQueriesDefaultCharts(crdbTopColumns),
248 + GroupBy: crdbTopQueriesGroupBy(crdbTopColumns),
249 + }
250 +}
251 +
252 +func (c *Collector) collectRunningQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
253 + sortColumn = resolveCrdbSortColumn(crdbRunningColumns, sortColumn)
254 + limit := c.TopQueriesLimit
255 + if limit <= 0 {
256 + limit = 500
257 + }
258 +
259 + query := buildCrdbRunningQueriesSQL(sortColumn, limit)
260 + queryCtx, cancel := context.WithTimeout(ctx, c.sqlTimeout())
261 + defer cancel()
262 + rows, err := c.db.QueryContext(queryCtx, query, limit)
263 + if err != nil {
264 + if errors.Is(queryCtx.Err(), context.DeadlineExceeded) {
265 + return &module.FunctionResponse{Status: 504, Message: "query timed out"}
266 + }
267 + return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("query failed: %v", err)}
268 + }
269 + defer rows.Close()
270 +
271 + data, err := scanCrdbRows(rows, crdbRunningColumns)
272 + if err != nil {
273 + return &module.FunctionResponse{Status: 500, Message: err.Error()}
274 + }
275 +
276 + return &module.FunctionResponse{
277 + Status: 200,
278 + Help: "Currently running SQL statements from SHOW CLUSTER STATEMENTS. WARNING: Query text may contain unmasked literals (potential PII).",
279 + Columns: buildCrdbColumns(crdbRunningColumns),
280 + Data: data,
281 + DefaultSortColumn: sortColumn,
282 + RequiredParams: []funcapi.ParamConfig{buildCrdbSortParam(crdbRunningColumns)},
283 + }
284 +}
285 +
286 +func buildCrdbSortParam(cols []crdbColumnMeta) funcapi.ParamConfig {
287 + return funcapi.ParamConfig{
288 + ID: paramSort,
289 + Name: "Filter By",
290 + Help: "Select the primary sort column",
291 + Selection: funcapi.ParamSelect,
292 + Options: buildCrdbSortOptions(cols),
293 + UniqueView: true,
294 + }
295 +}
296 +
297 +func buildCrdbSortOptions(cols []crdbColumnMeta) []funcapi.ParamOption {
298 + var sortOptions []funcapi.ParamOption
299 + sortDir := funcapi.FieldSortDescending
300 + for _, col := range cols {
301 + if !col.isSortOption {
302 + continue
303 + }
304 + opt := funcapi.ParamOption{
305 + ID: col.id,
306 + Column: col.id,
307 + Name: col.sortLabel,
308 + Sort: &sortDir,
309 + }
310 + if col.isDefaultSort {
311 + opt.Default = true
312 + }
313 + sortOptions = append(sortOptions, opt)
314 + }
315 + return sortOptions
316 +}
317 +
318 +func buildCrdbColumns(cols []crdbColumnMeta) map[string]any {
319 + result := make(map[string]any, len(cols))
320 + for i, col := range cols {
321 + visual := visValue
322 + if col.dataType == ftDuration {
323 + visual = visBar
324 + }
325 + colDef := funcapi.Column{
326 + Index: i,
327 + Name: col.name,
328 + Type: col.dataType,
329 + Units: col.units,
330 + Visualization: visual,
331 + Sort: col.sortDir,
332 + Sortable: col.sortable,
333 + Sticky: col.sticky,
334 + Summary: col.summary,
335 + Filter: col.filter,
336 + FullWidth: col.fullWidth,
337 + Wrap: col.wrap,
338 + DefaultExpandedFilter: false,
339 + UniqueKey: col.uniqueKey,
340 + Visible: col.visible,
341 + ValueOptions: funcapi.ValueOptions{
342 + Transform: col.transform,
343 + DecimalPoints: col.decimalPoints,
344 + DefaultValue: nil,
345 + },
346 + }
347 + result[col.id] = colDef.BuildColumn()
348 + }
349 + return result
350 +}
351 +
352 +func resolveCrdbSortColumn(cols []crdbColumnMeta, requested string) string {
353 + if requested != "" {
354 + for _, col := range cols {
355 + if col.id == requested && col.isSortOption {
356 + return col.id
357 + }
358 + }
359 + }
360 + for _, col := range cols {
361 + if col.isDefaultSort && col.isSortOption {
362 + return col.id
363 + }
364 + }
365 + for _, col := range cols {
366 + if col.isSortOption {
367 + return col.id
368 + }
369 + }
370 + return ""
371 +}
372 +
373 +func buildCrdbTopQueriesSQL(sortColumn string, limit int) string {
374 + selectCols := make([]string, 0, len(crdbTopColumns))
375 + for _, col := range crdbTopColumns {
376 + selectCols = append(selectCols, fmt.Sprintf("%s AS %s", col.selectExpr, col.id))
377 + }
378 + return fmt.Sprintf(`
379 +SELECT %s
380 +FROM crdb_internal.cluster_statement_statistics AS s
381 +WHERE s.metadata->>'query' IS NOT NULL
382 +ORDER BY %s DESC NULLS LAST
383 +LIMIT $1`, strings.Join(selectCols, ", "), sortColumn)
384 +}
385 +
386 +func buildCrdbRunningQueriesSQL(sortColumn string, limit int) string {
387 + selectCols := make([]string, 0, len(crdbRunningColumns))
388 + for _, col := range crdbRunningColumns {
389 + selectCols = append(selectCols, fmt.Sprintf("%s AS %s", col.selectExpr, col.id))
390 + }
391 + return fmt.Sprintf(`
392 +SELECT %s
393 +FROM [SHOW CLUSTER STATEMENTS] AS s
394 +ORDER BY %s DESC NULLS LAST
395 +LIMIT $1`, strings.Join(selectCols, ", "), sortColumn)
396 +}
397 +
398 +func scanCrdbRows(rows *sql.Rows, cols []crdbColumnMeta) ([][]any, error) {
399 + data := make([][]any, 0, 500)
400 +
401 + for rows.Next() {
402 + values := make([]any, len(cols))
403 + valuePtrs := make([]any, len(cols))
404 +
405 + for i, col := range cols {
406 + switch col.dataType {
407 + case ftString:
408 + var v sql.NullString
409 + values[i] = &v
410 + case ftInteger:
411 + var v sql.NullInt64
412 + values[i] = &v
413 + case ftFloat, ftDuration:
414 + var v sql.NullFloat64
415 + values[i] = &v
416 + default:
417 + var v any
418 + values[i] = &v
419 + }
420 + valuePtrs[i] = values[i]
421 + }
422 +
423 + if err := rows.Scan(valuePtrs...); err != nil {
424 + return nil, fmt.Errorf("row scan failed: %w", err)
425 + }
426 +
427 + row := make([]any, len(cols))
428 + for i, col := range cols {
429 + switch v := values[i].(type) {
430 + case *sql.NullString:
431 + if v.Valid {
432 + s := v.String
433 + if col.id == "query" {
434 + s = strmutil.TruncateText(s, crdbMaxQueryTextLength)
435 + }
436 + row[i] = s
437 + } else {
438 + row[i] = ""
439 + }
440 + case *sql.NullInt64:
441 + if v.Valid {
442 + row[i] = v.Int64
443 + } else {
444 + row[i] = int64(0)
445 + }
446 + case *sql.NullFloat64:
447 + if v.Valid {
448 + row[i] = v.Float64
449 + } else {
450 + row[i] = float64(0)
451 + }
452 + default:
453 + row[i] = nil
454 + }
455 + }
456 +
457 + data = append(data, row)
458 + }
459 +
460 + if err := rows.Err(); err != nil {
461 + return nil, fmt.Errorf("rows iteration error: %w", err)
462 + }
463 +
464 + return data, nil
465 +}
466 +
467 +func crdbTopQueriesCharts(cols []crdbColumnMeta) map[string]module.ChartConfig {
468 + charts := make(map[string]module.ChartConfig)
469 + for _, col := range cols {
470 + if !col.isMetric || col.chartGroup == "" {
471 + continue
472 + }
473 + cfg, ok := charts[col.chartGroup]
474 + if !ok {
475 + title := col.chartTitle
476 + if title == "" {
477 + title = col.chartGroup
478 + }
479 + cfg = module.ChartConfig{Name: title, Type: "stacked-bar"}
480 + }
481 + cfg.Columns = append(cfg.Columns, col.id)
482 + charts[col.chartGroup] = cfg
483 + }
484 + return charts
485 +}
486 +
487 +func crdbTopQueriesDefaultCharts(cols []crdbColumnMeta) [][]string {
488 + label := primaryCrdbLabel(cols)
489 + if label == "" {
490 + return nil
491 + }
492 + chartGroups := defaultCrdbChartGroups(cols)
493 + out := make([][]string, 0, len(chartGroups))
494 + for _, group := range chartGroups {
495 + out = append(out, []string{group, label})
496 + }
497 + return out
498 +}
499 +
500 +func crdbTopQueriesGroupBy(cols []crdbColumnMeta) map[string]module.GroupByConfig {
501 + groupBy := make(map[string]module.GroupByConfig)
502 + for _, col := range cols {
503 + if !col.isLabel {
504 + continue
505 + }
506 + groupBy[col.id] = module.GroupByConfig{
507 + Name: "Group by " + col.name,
508 + Columns: []string{col.id},
509 + }
510 + }
511 + return groupBy
512 +}
513 +
514 +func primaryCrdbLabel(cols []crdbColumnMeta) string {
515 + for _, col := range cols {
516 + if col.isPrimary {
517 + return col.id
518 + }
519 + }
520 + for _, col := range cols {
521 + if col.isLabel {
522 + return col.id
523 + }
524 + }
525 + return ""
526 +}
527 +
528 +func defaultCrdbChartGroups(cols []crdbColumnMeta) []string {
529 + groups := make([]string, 0)
530 + seen := make(map[string]bool)
531 + for _, col := range cols {
532 + if !col.isMetric || col.chartGroup == "" || !col.isDefaultChart {
533 + continue
534 + }
535 + if !seen[col.chartGroup] {
536 + seen[col.chartGroup] = true
537 + groups = append(groups, col.chartGroup)
538 + }
539 + }
540 + if len(groups) > 0 {
541 + return groups
542 + }
543 + for _, col := range cols {
544 + if !col.isMetric || col.chartGroup == "" {
545 + continue
546 + }
547 + if !seen[col.chartGroup] {
548 + seen[col.chartGroup] = true
549 + groups = append(groups, col.chartGroup)
550 + }
551 + }
552 + return groups
553 +}
src/go/plugin/go.d/collector/cockroachdb/functions_test.go new
+59
@@ -0,0 +1,59 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package cockroachdb
4 +
5 +import (
6 + "testing"
7 +
8 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
9 + "github.com/stretchr/testify/assert"
10 +)
11 +
12 +func TestCockroachDBMethods(t *testing.T) {
13 + methods := cockroachMethods()
14 +
15 + require := assert.New(t)
16 + require.Len(methods, 2)
17 +
18 + for _, method := range methods {
19 + require.NotEmpty(method.ID)
20 + require.NotEmpty(method.Name)
21 + require.NotEmpty(method.RequiredParams)
22 +
23 + var sortParam *funcapi.ParamConfig
24 + for i := range method.RequiredParams {
25 + if method.RequiredParams[i].ID == "__sort" {
26 + sortParam = &method.RequiredParams[i]
27 + break
28 + }
29 + }
30 + require.NotNil(sortParam, "expected __sort required param")
31 + require.NotEmpty(sortParam.Options)
32 + }
33 +}
34 +
35 +func TestCockroachDBTopColumns_HasRequiredColumns(t *testing.T) {
36 + required := []string{"fingerprintId", "query", "executions", "totalTime"}
37 +
38 + uiKeys := make(map[string]bool)
39 + for _, col := range crdbTopColumns {
40 + uiKeys[col.id] = true
41 + }
42 +
43 + for _, key := range required {
44 + assert.True(t, uiKeys[key], "column %s should be defined", key)
45 + }
46 +}
47 +
48 +func TestCockroachDBRunningColumns_HasRequiredColumns(t *testing.T) {
49 + required := []string{"queryId", "query", "elapsedMs"}
50 +
51 + uiKeys := make(map[string]bool)
52 + for _, col := range crdbRunningColumns {
53 + uiKeys[col.id] = true
54 + }
55 +
56 + for _, key := range required {
57 + assert.True(t, uiKeys[key], "column %s should be defined", key)
58 + }
59 +}
src/go/plugin/go.d/collector/cockroachdb/metadata.yaml
+31 -2
@@ -23,13 +23,20 @@ modules:
23 data_collection:
24 metrics_description: |
25 This collector monitors CockroachDB servers.
26 - method_description: ""
26 + method_description: |
27 + It scrapes Prometheus metrics from the CockroachDB `/_status/vars` endpoint.
28 +
29 + It also provides `top-queries` and `running-queries` functions using SQL statement statistics (`crdb_internal.cluster_statement_statistics`) and `SHOW CLUSTER STATEMENTS`.
30 supported_platforms:
31 include: []
32 exclude: []
33 multi_instance: true
34 additional_permissions:
32 - description: ""
35 + description: |
36 + The `top-queries` and `running-queries` functions require:
37 +
38 + - A SQL user with `VIEWACTIVITY` (or `VIEWACTIVITYREDACTED`) privileges.
39 + - Access to `crdb_internal.cluster_statement_statistics` (may require `SET allow_unsafe_internals = on` on newer versions).
40 default_behavior:
41 auto_detection:
42 description: ""
@@ -71,6 +78,21 @@ modules:
78 default_value: 1
79 required: false
80 group: Target
81 + - name: dsn
82 + description: SQL DSN used by `top-queries` and `running-queries` functions.
83 + default_value: ""
84 + required: false
85 + group: Query Functions
86 + - name: sql_timeout
87 + description: SQL query timeout (seconds) for query functions.
88 + default_value: 1
89 + required: false
90 + group: Query Functions
91 + - name: top_queries_limit
92 + description: Maximum number of rows returned by the `top-queries` and `running-queries` functions.
93 + default_value: 500
94 + required: false
95 + group: Limits
96
97 - name: username
98 description: Username for Basic HTTP authentication.
@@ -167,6 +189,13 @@ modules:
189 jobs:
190 - name: local
191 url: http://127.0.0.1:8080/_status/vars
192 + - name: Top queries
193 + description: Enable SQL query functions.
194 + config: |
195 + jobs:
196 + - name: local
197 + url: http://127.0.0.1:8080/_status/vars
198 + dsn: postgres://root@127.0.0.1:26257/defaultdb?sslmode=disable
199 - name: HTTP authentication
200 description: Local server with basic HTTP authentication.
201 config: |
src/go/plugin/go.d/collector/cockroachdb/sql.go new
+5
@@ -0,0 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package cockroachdb
4 +
5 +import _ "github.com/jackc/pgx/v5/stdlib"
src/go/plugin/go.d/collector/couchbase/collector.go
+7 -2
@@ -24,8 +24,11 @@ func init() {
24 Defaults: module.Defaults{
25 UpdateEvery: 5,
26 },
27 - Create: func() module.Module { return New() },
28 - Config: func() any { return &Config{} },
27 + Create: func() module.Module { return New() },
28 + Config: func() any { return &Config{} },
29 + Methods: couchbaseMethods,
30 + MethodParams: couchbaseMethodParams,
31 + HandleMethod: couchbaseHandleMethod,
32 })
33 }
34
@@ -50,6 +53,8 @@ type Config struct {
53 UpdateEvery int `yaml:"update_every,omitempty" json:"update_every"`
54 AutoDetectionRetry int `yaml:"autodetection_retry,omitempty" json:"autodetection_retry"`
55 web.HTTPConfig `yaml:",inline" json:""`
56 + QueryURL string `yaml:"query_url,omitempty" json:"query_url,omitempty"`
57 + TopQueriesLimit int `yaml:"top_queries_limit,omitempty" json:"top_queries_limit,omitempty"`
58 }
59
60 type Collector struct {
src/go/plugin/go.d/collector/couchbase/config_schema.json
+16
@@ -25,6 +25,12 @@
25 "default": "http://127.0.0.1:8091",
26 "format": "uri"
27 },
28 + "query_url": {
29 + "title": "Query URL",
30 + "description": "Optional. The URL of the Couchbase Query (N1QL) service. If not set, defaults to the same host as URL on port 8093.",
31 + "type": "string",
32 + "format": "uri"
33 + },
34 "timeout": {
35 "title": "Timeout",
36 "description": "The timeout in seconds for the HTTP request.",
@@ -32,6 +38,14 @@
38 "minimum": 0.5,
39 "default": 1
40 },
41 + "top_queries_limit": {
42 + "title": "Top Queries Limit",
43 + "description": "Maximum number of queries to return in the top-queries function response.",
44 + "type": "integer",
45 + "minimum": 1,
46 + "maximum": 5000,
47 + "default": 500
48 + },
49 "not_follow_redirects": {
50 "title": "Not follow redirects",
51 "description": "If set, the client will not follow HTTP redirects automatically.",
@@ -138,8 +152,10 @@
152 "update_every",
153 "autodetection_retry",
154 "url",
155 + "query_url",
156 "timeout",
157 "not_follow_redirects",
158 + "top_queries_limit",
159 "vnode"
160 ]
161 },
src/go/plugin/go.d/collector/couchbase/functions.go new
+563
@@ -0,0 +1,563 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package couchbase
4 +
5 +import (
6 + "context"
7 + "encoding/json"
8 + "fmt"
9 + "net"
10 + "net/http"
11 + "net/url"
12 + "path"
13 + "sort"
14 + "strings"
15 + "time"
16 +
17 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
18 + "github.com/netdata/netdata/go/plugins/pkg/web"
19 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
20 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
21 +)
22 +
23 +const couchbaseMaxQueryTextLength = 4096
24 +
25 +const (
26 + paramSort = "__sort"
27 +
28 + ftString = funcapi.FieldTypeString
29 + ftInteger = funcapi.FieldTypeInteger
30 + ftDuration = funcapi.FieldTypeDuration
31 + ftTimestamp = funcapi.FieldTypeTimestamp
32 +
33 + trNone = funcapi.FieldTransformNone
34 + trNumber = funcapi.FieldTransformNumber
35 + trDuration = funcapi.FieldTransformDuration
36 + trDatetime = funcapi.FieldTransformDatetime
37 + trText = funcapi.FieldTransformText
38 +
39 + visValue = funcapi.FieldVisualValue
40 + visBar = funcapi.FieldVisualBar
41 +
42 + sortAsc = funcapi.FieldSortAscending
43 + sortDesc = funcapi.FieldSortDescending
44 +
45 + summaryCount = funcapi.FieldSummaryCount
46 + summaryMax = funcapi.FieldSummaryMax
47 + summarySum = funcapi.FieldSummarySum
48 +
49 + filterMulti = funcapi.FieldFilterMultiselect
50 + filterRange = funcapi.FieldFilterRange
51 +)
52 +
53 +type couchbaseColumnMeta struct {
54 + id string
55 + name string
56 + colType funcapi.FieldType
57 + visible bool
58 + sortable bool
59 + fullWidth bool
60 + wrap bool
61 + sticky bool
62 + filter funcapi.FieldFilter
63 + visualization funcapi.FieldVisual
64 + transform funcapi.FieldTransform
65 + units string
66 + decimalPoints int
67 + uniqueKey bool
68 + sortDir funcapi.FieldSort
69 + summary funcapi.FieldSummary
70 + isLabel bool
71 + isPrimary bool
72 + isMetric bool
73 + chartGroup string
74 + chartTitle string
75 + isDefaultChart bool
76 +}
77 +
78 +var couchbaseAllColumns = []couchbaseColumnMeta{
79 + {id: "requestId", name: "Request ID", colType: ftString, visible: false, sortable: true, filter: filterMulti, visualization: visValue, transform: trText, uniqueKey: true, sortDir: sortDesc, summary: summaryCount},
80 + {id: "requestTime", name: "Request Time", colType: ftTimestamp, visible: true, sortable: true, filter: filterRange, visualization: visValue, transform: trDatetime, sortDir: sortDesc, summary: summaryMax},
81 + {id: "statement", name: "Statement", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, sticky: true, fullWidth: true, wrap: true},
82 + {id: "elapsedTime", name: "Elapsed Time", colType: ftDuration, visible: true, sortable: true, filter: filterRange, visualization: visBar, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summarySum, isMetric: true, chartGroup: "Time", chartTitle: "Elapsed & Service Time", isDefaultChart: true},
83 + {id: "serviceTime", name: "Service Time", colType: ftDuration, visible: true, sortable: true, filter: filterRange, visualization: visBar, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summarySum, isMetric: true, chartGroup: "Time", chartTitle: "Elapsed & Service Time"},
84 + {id: "resultCount", name: "Result Count", colType: ftInteger, visible: true, sortable: true, filter: filterRange, visualization: visValue, transform: trNumber, sortDir: sortDesc, summary: summarySum, isMetric: true, chartGroup: "Results", chartTitle: "Results"},
85 + {id: "resultSize", name: "Result Size", colType: ftInteger, visible: false, sortable: false, filter: filterRange, visualization: visValue, transform: trNumber, sortDir: sortDesc, summary: summarySum, isMetric: true, chartGroup: "ResultSize", chartTitle: "Result Size"},
86 + {id: "errorCount", name: "Error Count", colType: ftInteger, visible: false, sortable: false, filter: filterRange, visualization: visValue, transform: trNumber, sortDir: sortDesc, summary: summarySum, isMetric: true, chartGroup: "Errors", chartTitle: "Errors & Warnings"},
87 + {id: "warningCount", name: "Warning Count", colType: ftInteger, visible: false, sortable: false, filter: filterRange, visualization: visValue, transform: trNumber, sortDir: sortDesc, summary: summarySum, isMetric: true, chartGroup: "Errors", chartTitle: "Errors & Warnings"},
88 + {id: "user", name: "User", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, isLabel: true, isPrimary: true},
89 + {id: "clientContextID", name: "Client Context ID", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText},
90 +}
91 +
92 +type cbQueryResponse struct {
93 + Status string `json:"status"`
94 + Results []cbCompletedRequest `json:"results"`
95 + Errors []cbQueryError `json:"errors"`
96 +}
97 +
98 +type cbQueryError struct {
99 + Message string `json:"msg"`
100 +}
101 +
102 +type cbCompletedRequest struct {
103 + RequestID string `json:"requestId"`
104 + RequestTime string `json:"requestTime"`
105 + Statement string `json:"statement"`
106 + ElapsedTime string `json:"elapsedTime"`
107 + ServiceTime string `json:"serviceTime"`
108 + ResultCount json.Number `json:"resultCount"`
109 + ResultSize json.Number `json:"resultSize"`
110 + ErrorCount json.Number `json:"errorCount"`
111 + WarningCount json.Number `json:"warningCount"`
112 + User string `json:"user"`
113 + ClientContextID string `json:"clientContextID"`
114 +}
115 +
116 +type cbRow struct {
117 + RequestID string
118 + RequestTime time.Time
119 + RequestTimeRaw string
120 + Statement string
121 + ElapsedMs float64
122 + ServiceMs float64
123 + ResultCount int64
124 + ResultSize int64
125 + ErrorCount int64
126 + WarningCount int64
127 + User string
128 + ClientContextID string
129 +}
130 +
131 +func couchbaseMethods() []module.MethodConfig {
132 + sortOptions := buildCouchbaseSortOptions(couchbaseAllColumns)
133 + return []module.MethodConfig{{
134 + ID: "top-queries",
135 + Name: "Top Queries",
136 + Help: "Top N1QL requests from system:completed_requests",
137 + RequiredParams: []funcapi.ParamConfig{{
138 + ID: paramSort,
139 + Name: "Filter By",
140 + Help: "Select the primary sort column",
141 + Selection: funcapi.ParamSelect,
142 + Options: sortOptions,
143 + UniqueView: true,
144 + }},
145 + }}
146 +}
147 +
148 +func couchbaseMethodParams(_ context.Context, _ *module.Job, method string) ([]funcapi.ParamConfig, error) {
149 + switch method {
150 + case "top-queries":
151 + return []funcapi.ParamConfig{buildCouchbaseSortParam(couchbaseAllColumns)}, nil
152 + default:
153 + return nil, fmt.Errorf("unknown method: %s", method)
154 + }
155 +}
156 +
157 +func couchbaseHandleMethod(ctx context.Context, job *module.Job, method string, params funcapi.ResolvedParams) *module.FunctionResponse {
158 + collector, ok := job.Module().(*Collector)
159 + if !ok {
160 + return &module.FunctionResponse{Status: 500, Message: "internal error: invalid module type"}
161 + }
162 +
163 + if collector.httpClient == nil {
164 + return &module.FunctionResponse{
165 + Status: 503,
166 + Message: "collector is still initializing, please retry in a few seconds",
167 + }
168 + }
169 +
170 + switch method {
171 + case "top-queries":
172 + return collector.collectTopQueries(ctx, params.Column(paramSort))
173 + default:
174 + return &module.FunctionResponse{Status: 404, Message: fmt.Sprintf("unknown method: %s", method)}
175 + }
176 +}
177 +
178 +func buildCouchbaseSortOptions(cols []couchbaseColumnMeta) []funcapi.ParamOption {
179 + var sortOptions []funcapi.ParamOption
180 + sortDir := funcapi.FieldSortDescending
181 + for _, col := range cols {
182 + if !col.sortable {
183 + continue
184 + }
185 + opt := funcapi.ParamOption{
186 + ID: col.id,
187 + Column: col.id,
188 + Name: fmt.Sprintf("Top queries by %s", col.name),
189 + Sort: &sortDir,
190 + }
191 + if col.id == "elapsedTime" {
192 + opt.Default = true
193 + }
194 + sortOptions = append(sortOptions, opt)
195 + }
196 + return sortOptions
197 +}
198 +
199 +func buildCouchbaseSortParam(cols []couchbaseColumnMeta) funcapi.ParamConfig {
200 + return funcapi.ParamConfig{
201 + ID: paramSort,
202 + Name: "Filter By",
203 + Help: "Select the primary sort column",
204 + Selection: funcapi.ParamSelect,
205 + Options: buildCouchbaseSortOptions(cols),
206 + UniqueView: true,
207 + }
208 +}
209 +
210 +func buildCouchbaseColumns(cols []couchbaseColumnMeta) map[string]any {
211 + result := make(map[string]any, len(cols))
212 + for i, col := range cols {
213 + colDef := funcapi.Column{
214 + Index: i,
215 + Name: col.name,
216 + Type: col.colType,
217 + Units: col.units,
218 + Visualization: col.visualization,
219 + Sort: col.sortDir,
220 + Sortable: col.sortable,
221 + Sticky: col.sticky,
222 + Summary: col.summary,
223 + Filter: col.filter,
224 + FullWidth: col.fullWidth,
225 + Wrap: col.wrap,
226 + DefaultExpandedFilter: false,
227 + UniqueKey: col.uniqueKey,
228 + Visible: col.visible,
229 + ValueOptions: funcapi.ValueOptions{
230 + Transform: col.transform,
231 + DecimalPoints: col.decimalPoints,
232 + DefaultValue: nil,
233 + },
234 + }
235 + result[col.id] = colDef.BuildColumn()
236 + }
237 + return result
238 +}
239 +
240 +func (c *Collector) queryServiceURL() (string, error) {
241 + if c.QueryURL != "" {
242 + return c.QueryURL, nil
243 + }
244 + parsed, err := url.Parse(c.URL)
245 + if err != nil {
246 + return "", err
247 + }
248 + host := parsed.Hostname()
249 + port := parsed.Port()
250 + if port == "" || port == "8091" {
251 + port = "8093"
252 + }
253 + if port != "" {
254 + parsed.Host = net.JoinHostPort(host, port)
255 + } else {
256 + parsed.Host = host
257 + }
258 + parsed.Path = ""
259 + return parsed.String(), nil
260 +}
261 +
262 +func (c *Collector) buildQueryRequest(ctx context.Context, statement string) (*http.Request, error) {
263 + queryURL, err := c.queryServiceURL()
264 + if err != nil {
265 + return nil, err
266 + }
267 +
268 + u, err := url.Parse(queryURL)
269 + if err != nil {
270 + return nil, err
271 + }
272 + u.Path = path.Join(u.Path, "/query/service")
273 +
274 + reqCfg := c.RequestConfig
275 + reqCfg.URL = u.String()
276 + reqCfg.Method = http.MethodPost
277 + reqCfg.Body = url.Values{"statement": {statement}}.Encode()
278 + if reqCfg.Headers == nil {
279 + reqCfg.Headers = map[string]string{}
280 + }
281 + reqCfg.Headers["Content-Type"] = "application/x-www-form-urlencoded"
282 +
283 + req, err := web.NewHTTPRequest(reqCfg)
284 + if err != nil {
285 + return nil, err
286 + }
287 + return req.WithContext(ctx), nil
288 +}
289 +
290 +func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
291 + limit := c.TopQueriesLimit
292 + if limit <= 0 {
293 + limit = 500
294 + }
295 +
296 + statement := "SELECT cr.requestId, cr.requestTime, cr.statement, cr.elapsedTime, cr.serviceTime, " +
297 + "cr.resultCount, cr.resultSize, cr.errorCount, cr.warningCount, cr.users AS `user`, cr.clientContextID " +
298 + "FROM system:completed_requests AS cr"
299 +
300 + req, err := c.buildQueryRequest(ctx, statement)
301 + if err != nil {
302 + return &module.FunctionResponse{Status: 500, Message: err.Error()}
303 + }
304 +
305 + var resp cbQueryResponse
306 + if err := web.DoHTTP(c.httpClient).RequestJSON(req, &resp); err != nil {
307 + if ctx.Err() == context.DeadlineExceeded {
308 + return &module.FunctionResponse{Status: 504, Message: "query timed out"}
309 + }
310 + return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("query failed: %v", err)}
311 + }
312 +
313 + if strings.ToLower(resp.Status) != "success" {
314 + msg := "query failed"
315 + if len(resp.Errors) > 0 && resp.Errors[0].Message != "" {
316 + msg = resp.Errors[0].Message
317 + }
318 + return &module.FunctionResponse{Status: 500, Message: msg}
319 + }
320 +
321 + rows := make([]cbRow, 0, len(resp.Results))
322 + for _, r := range resp.Results {
323 + rows = append(rows, buildCouchbaseRow(r))
324 + }
325 +
326 + if len(rows) == 0 {
327 + return &module.FunctionResponse{
328 + Status: 200,
329 + Message: "No completed requests found.",
330 + Help: "Top N1QL requests from system:completed_requests",
331 + Columns: buildCouchbaseColumns(couchbaseAllColumns),
332 + Data: [][]any{},
333 + DefaultSortColumn: "elapsedTime",
334 + RequiredParams: []funcapi.ParamConfig{buildCouchbaseSortParam(couchbaseAllColumns)},
335 + Charts: couchbaseTopQueriesCharts(couchbaseAllColumns),
336 + DefaultCharts: couchbaseTopQueriesDefaultCharts(couchbaseAllColumns),
337 + GroupBy: couchbaseTopQueriesGroupBy(couchbaseAllColumns),
338 + }
339 + }
340 +
341 + sortColumn = mapCouchbaseSortColumn(sortColumn)
342 + sortCouchbaseRows(rows, sortColumn)
343 +
344 + if len(rows) > limit {
345 + rows = rows[:limit]
346 + }
347 +
348 + data := make([][]any, 0, len(rows))
349 + for _, row := range rows {
350 + out := make([]any, len(couchbaseAllColumns))
351 + for i, col := range couchbaseAllColumns {
352 + switch col.id {
353 + case "requestId":
354 + out[i] = row.RequestID
355 + case "requestTime":
356 + if row.RequestTime.IsZero() {
357 + out[i] = row.RequestTimeRaw
358 + } else {
359 + out[i] = row.RequestTime.Format(time.RFC3339Nano)
360 + }
361 + case "statement":
362 + out[i] = strmutil.TruncateText(row.Statement, couchbaseMaxQueryTextLength)
363 + case "elapsedTime":
364 + out[i] = row.ElapsedMs
365 + case "serviceTime":
366 + out[i] = row.ServiceMs
367 + case "resultCount":
368 + out[i] = row.ResultCount
369 + case "resultSize":
370 + out[i] = row.ResultSize
371 + case "errorCount":
372 + out[i] = row.ErrorCount
373 + case "warningCount":
374 + out[i] = row.WarningCount
375 + case "user":
376 + out[i] = row.User
377 + case "clientContextID":
378 + out[i] = row.ClientContextID
379 + default:
380 + out[i] = nil
381 + }
382 + }
383 + data = append(data, out)
384 + }
385 +
386 + return &module.FunctionResponse{
387 + Status: 200,
388 + Help: "Top N1QL requests from system:completed_requests",
389 + Columns: buildCouchbaseColumns(couchbaseAllColumns),
390 + Data: data,
391 + DefaultSortColumn: "elapsedTime",
392 + RequiredParams: []funcapi.ParamConfig{buildCouchbaseSortParam(couchbaseAllColumns)},
393 + Charts: couchbaseTopQueriesCharts(couchbaseAllColumns),
394 + DefaultCharts: couchbaseTopQueriesDefaultCharts(couchbaseAllColumns),
395 + GroupBy: couchbaseTopQueriesGroupBy(couchbaseAllColumns),
396 + }
397 +}
398 +
399 +func buildCouchbaseRow(r cbCompletedRequest) cbRow {
400 + row := cbRow{
401 + RequestID: r.RequestID,
402 + RequestTimeRaw: r.RequestTime,
403 + Statement: r.Statement,
404 + User: r.User,
405 + ClientContextID: r.ClientContextID,
406 + }
407 +
408 + if t, err := time.Parse(time.RFC3339Nano, r.RequestTime); err == nil {
409 + row.RequestTime = t
410 + } else if t, err := time.Parse(time.RFC3339, r.RequestTime); err == nil {
411 + row.RequestTime = t
412 + }
413 +
414 + row.ElapsedMs = parseDurationMs(r.ElapsedTime)
415 + row.ServiceMs = parseDurationMs(r.ServiceTime)
416 + row.ResultCount = parseNumber(r.ResultCount)
417 + row.ResultSize = parseNumber(r.ResultSize)
418 + row.ErrorCount = parseNumber(r.ErrorCount)
419 + row.WarningCount = parseNumber(r.WarningCount)
420 +
421 + return row
422 +}
423 +
424 +func parseDurationMs(raw string) float64 {
425 + if raw == "" {
426 + return 0
427 + }
428 + if d, err := time.ParseDuration(raw); err == nil {
429 + return float64(d) / float64(time.Millisecond)
430 + }
431 + return 0
432 +}
433 +
434 +func parseNumber(n json.Number) int64 {
435 + if n == "" {
436 + return 0
437 + }
438 + if i, err := n.Int64(); err == nil {
439 + return i
440 + }
441 + if f, err := n.Float64(); err == nil {
442 + return int64(f)
443 + }
444 + return 0
445 +}
446 +
447 +func mapCouchbaseSortColumn(col string) string {
448 + switch col {
449 + case "elapsedTime", "serviceTime", "requestTime", "resultCount":
450 + return col
451 + default:
452 + return "elapsedTime"
453 + }
454 +}
455 +
456 +func sortCouchbaseRows(rows []cbRow, sortColumn string) {
457 + switch sortColumn {
458 + case "serviceTime":
459 + sort.Slice(rows, func(i, j int) bool {
460 + return rows[i].ServiceMs > rows[j].ServiceMs
461 + })
462 + case "requestTime":
463 + sort.Slice(rows, func(i, j int) bool {
464 + return rows[i].RequestTime.After(rows[j].RequestTime)
465 + })
466 + case "resultCount":
467 + sort.Slice(rows, func(i, j int) bool {
468 + return rows[i].ResultCount > rows[j].ResultCount
469 + })
470 + default:
471 + sort.Slice(rows, func(i, j int) bool {
472 + return rows[i].ElapsedMs > rows[j].ElapsedMs
473 + })
474 + }
475 +}
476 +
477 +func couchbaseTopQueriesCharts(cols []couchbaseColumnMeta) map[string]module.ChartConfig {
478 + charts := make(map[string]module.ChartConfig)
479 + for _, col := range cols {
480 + if !col.isMetric || col.chartGroup == "" {
481 + continue
482 + }
483 + cfg, ok := charts[col.chartGroup]
484 + if !ok {
485 + title := col.chartTitle
486 + if title == "" {
487 + title = col.chartGroup
488 + }
489 + cfg = module.ChartConfig{Name: title, Type: "stacked-bar"}
490 + }
491 + cfg.Columns = append(cfg.Columns, col.id)
492 + charts[col.chartGroup] = cfg
493 + }
494 + return charts
495 +}
496 +
497 +func couchbaseTopQueriesDefaultCharts(cols []couchbaseColumnMeta) [][]string {
498 + label := primaryCouchbaseLabel(cols)
499 + if label == "" {
500 + return nil
501 + }
502 + chartGroups := defaultCouchbaseChartGroups(cols)
503 + out := make([][]string, 0, len(chartGroups))
504 + for _, group := range chartGroups {
505 + out = append(out, []string{group, label})
506 + }
507 + return out
508 +}
509 +
510 +func couchbaseTopQueriesGroupBy(cols []couchbaseColumnMeta) map[string]module.GroupByConfig {
511 + groupBy := make(map[string]module.GroupByConfig)
512 + for _, col := range cols {
513 + if !col.isLabel {
514 + continue
515 + }
516 + groupBy[col.id] = module.GroupByConfig{
517 + Name: "Group by " + col.name,
518 + Columns: []string{col.id},
519 + }
520 + }
521 + return groupBy
522 +}
523 +
524 +func primaryCouchbaseLabel(cols []couchbaseColumnMeta) string {
525 + for _, col := range cols {
526 + if col.isPrimary {
527 + return col.id
528 + }
529 + }
530 + for _, col := range cols {
531 + if col.isLabel {
532 + return col.id
533 + }
534 + }
535 + return ""
536 +}
537 +
538 +func defaultCouchbaseChartGroups(cols []couchbaseColumnMeta) []string {
539 + groups := make([]string, 0)
540 + seen := make(map[string]bool)
541 + for _, col := range cols {
542 + if !col.isMetric || col.chartGroup == "" || !col.isDefaultChart {
543 + continue
544 + }
545 + if !seen[col.chartGroup] {
546 + seen[col.chartGroup] = true
547 + groups = append(groups, col.chartGroup)
548 + }
549 + }
550 + if len(groups) > 0 {
551 + return groups
552 + }
553 + for _, col := range cols {
554 + if !col.isMetric || col.chartGroup == "" {
555 + continue
556 + }
557 + if !seen[col.chartGroup] {
558 + seen[col.chartGroup] = true
559 + groups = append(groups, col.chartGroup)
560 + }
561 + }
562 + return groups
563 +}
src/go/plugin/go.d/collector/couchbase/functions_test.go new
+62
@@ -0,0 +1,62 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package couchbase
4 +
5 +import (
6 + "testing"
7 +
8 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
9 + "github.com/stretchr/testify/assert"
10 +)
11 +
12 +func TestCouchbaseMethods(t *testing.T) {
13 + methods := couchbaseMethods()
14 +
15 + require := assert.New(t)
16 + require.Len(methods, 1)
17 + require.Equal("top-queries", methods[0].ID)
18 + require.Equal("Top Queries", methods[0].Name)
19 + require.NotEmpty(methods[0].RequiredParams)
20 +
21 + var sortParam *funcapi.ParamConfig
22 + for i := range methods[0].RequiredParams {
23 + if methods[0].RequiredParams[i].ID == "__sort" {
24 + sortParam = &methods[0].RequiredParams[i]
25 + break
26 + }
27 + }
28 + require.NotNil(sortParam, "expected __sort required param")
29 + require.NotEmpty(sortParam.Options)
30 +}
31 +
32 +func TestCouchbaseAllColumns_HasRequiredColumns(t *testing.T) {
33 + required := []string{"requestId", "statement", "elapsedTime"}
34 +
35 + uiKeys := make(map[string]bool)
36 + for _, col := range couchbaseAllColumns {
37 + uiKeys[col.id] = true
38 + }
39 +
40 + for _, key := range required {
41 + assert.True(t, uiKeys[key], "column %s should be defined", key)
42 + }
43 +}
44 +
45 +func TestMapCouchbaseSortColumn(t *testing.T) {
46 + tests := map[string]struct {
47 + input string
48 + expected string
49 + }{
50 + "elapsedTime": {input: "elapsedTime", expected: "elapsedTime"},
51 + "serviceTime": {input: "serviceTime", expected: "serviceTime"},
52 + "requestTime": {input: "requestTime", expected: "requestTime"},
53 + "resultCount": {input: "resultCount", expected: "resultCount"},
54 + "invalid": {input: "bad", expected: "elapsedTime"},
55 + }
56 +
57 + for name, tc := range tests {
58 + t.Run(name, func(t *testing.T) {
59 + assert.Equal(t, tc.expected, mapCouchbaseSortColumn(tc.input))
60 + })
61 + }
62 +}
src/go/plugin/go.d/collector/elasticsearch/collector.go
+6 -2
@@ -25,8 +25,11 @@ func init() {
25 Defaults: module.Defaults{
26 UpdateEvery: 5,
27 },
28 - Create: func() module.Module { return New() },
29 - Config: func() any { return &Config{} },
28 + Create: func() module.Module { return New() },
29 + Config: func() any { return &Config{} },
30 + Methods: elasticsearchMethods,
31 + MethodParams: elasticsearchMethodParams,
32 + HandleMethod: elasticsearchHandleMethod,
33 })
34 }
35
@@ -67,6 +70,7 @@ type Config struct {
70 DoClusterHealth bool `yaml:"collect_cluster_health" json:"collect_cluster_health"`
71 DoClusterStats bool `yaml:"collect_cluster_stats" json:"collect_cluster_stats"`
72 DoIndicesStats bool `yaml:"collect_indices_stats" json:"collect_indices_stats"`
73 + TopQueriesLimit int `yaml:"top_queries_limit,omitempty" json:"top_queries_limit,omitempty"`
74 }
75
76 type Collector struct {
src/go/plugin/go.d/collector/elasticsearch/config_schema.json
+8
@@ -32,6 +32,14 @@
32 "minimum": 0.5,
33 "default": 2
34 },
35 + "top_queries_limit": {
36 + "title": "Top Queries Limit",
37 + "description": "Maximum number of queries to return in the top-queries function response.",
38 + "type": "integer",
39 + "minimum": 1,
40 + "maximum": 5000,
41 + "default": 500
42 + },
43 "not_follow_redirects": {
44 "title": "Not follow redirects",
45 "description": "If set, the client will not follow HTTP redirects automatically.",
src/go/plugin/go.d/collector/elasticsearch/functions.go new
+465
@@ -0,0 +1,465 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package elasticsearch
4 +
5 +import (
6 + "context"
7 + "fmt"
8 + "net/url"
9 + "sort"
10 + "strconv"
11 + "strings"
12 + "time"
13 +
14 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
15 + "github.com/netdata/netdata/go/plugins/pkg/web"
16 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
17 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
18 +)
19 +
20 +const elasticMaxQueryTextLength = 4096
21 +
22 +const (
23 + paramSort = "__sort"
24 +
25 + ftString = funcapi.FieldTypeString
26 + ftDuration = funcapi.FieldTypeDuration
27 + ftTimestamp = funcapi.FieldTypeTimestamp
28 + ftBoolean = funcapi.FieldTypeBoolean
29 +
30 + trNone = funcapi.FieldTransformNone
31 + trDuration = funcapi.FieldTransformDuration
32 + trDatetime = funcapi.FieldTransformDatetime
33 + trText = funcapi.FieldTransformText
34 +
35 + visValue = funcapi.FieldVisualValue
36 + visBar = funcapi.FieldVisualBar
37 +
38 + sortAsc = funcapi.FieldSortAscending
39 + sortDesc = funcapi.FieldSortDescending
40 +
41 + summaryCount = funcapi.FieldSummaryCount
42 + summaryMax = funcapi.FieldSummaryMax
43 + summarySum = funcapi.FieldSummarySum
44 +
45 + filterMulti = funcapi.FieldFilterMultiselect
46 + filterRange = funcapi.FieldFilterRange
47 +)
48 +
49 +type esColumnMeta struct {
50 + id string
51 + name string
52 + colType funcapi.FieldType
53 + visible bool
54 + sortable bool
55 + fullWidth bool
56 + wrap bool
57 + sticky bool
58 + filter funcapi.FieldFilter
59 + visualization funcapi.FieldVisual
60 + transform funcapi.FieldTransform
61 + units string
62 + decimalPoints int
63 + uniqueKey bool
64 + sortDir funcapi.FieldSort
65 + summary funcapi.FieldSummary
66 + isLabel bool
67 + isPrimary bool
68 + isMetric bool
69 + chartGroup string
70 + chartTitle string
71 + isDefaultChart bool
72 +}
73 +
74 +var esAllColumns = []esColumnMeta{
75 + {id: "taskId", name: "Task ID", colType: ftString, visible: false, sortable: true, filter: filterMulti, visualization: visValue, transform: trText, uniqueKey: true, sortDir: sortDesc, summary: summaryCount},
76 + {id: "node", name: "Node ID", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, isLabel: true},
77 + {id: "nodeName", name: "Node Name", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, isLabel: true, isPrimary: true},
78 + {id: "action", name: "Action", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, isLabel: true},
79 + {id: "type", name: "Type", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, isLabel: true},
80 + {id: "description", name: "Description", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, sticky: true, fullWidth: true, wrap: true},
81 + {id: "startTime", name: "Start Time", colType: ftTimestamp, visible: true, sortable: true, filter: filterRange, visualization: visValue, transform: trDatetime, sortDir: sortDesc, summary: summaryMax},
82 + {id: "runningTime", name: "Running Time", colType: ftDuration, visible: true, sortable: true, filter: filterRange, visualization: visBar, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summarySum, isMetric: true, chartGroup: "RunningTime", chartTitle: "Running Time", isDefaultChart: true},
83 + {id: "cancellable", name: "Cancellable", colType: ftBoolean, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trNone},
84 + {id: "cancelled", name: "Cancelled", colType: ftBoolean, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trNone},
85 +}
86 +
87 +type esTasksResponse struct {
88 + Nodes map[string]struct {
89 + Name string `json:"name"`
90 + Tasks map[string]esTask `json:"tasks"`
91 + } `json:"nodes"`
92 +}
93 +
94 +type esTask struct {
95 + ID int64 `json:"id"`
96 + Action string `json:"action"`
97 + Type string `json:"type"`
98 + Description string `json:"description"`
99 + StartTimeInMillis int64 `json:"start_time_in_millis"`
100 + RunningTimeInNanos int64 `json:"running_time_in_nanos"`
101 + Cancellable bool `json:"cancellable"`
102 + Cancelled bool `json:"cancelled"`
103 +}
104 +
105 +type esTaskRow struct {
106 + TaskID string
107 + NodeID string
108 + NodeName string
109 + Action string
110 + Type string
111 + Description string
112 + StartTime time.Time
113 + RunningTime time.Duration
114 + Cancellable bool
115 + Cancelled bool
116 +}
117 +
118 +func elasticsearchMethods() []module.MethodConfig {
119 + sortOptions := buildElasticsearchSortOptions(esAllColumns)
120 + return []module.MethodConfig{{
121 + ID: "top-queries",
122 + Name: "Top Queries",
123 + Help: "Running queries from Elasticsearch Tasks API",
124 + RequiredParams: []funcapi.ParamConfig{{
125 + ID: paramSort,
126 + Name: "Filter By",
127 + Help: "Select the primary sort column",
128 + Selection: funcapi.ParamSelect,
129 + Options: sortOptions,
130 + UniqueView: true,
131 + }},
132 + }}
133 +}
134 +
135 +func elasticsearchMethodParams(_ context.Context, _ *module.Job, method string) ([]funcapi.ParamConfig, error) {
136 + switch method {
137 + case "top-queries":
138 + return []funcapi.ParamConfig{buildElasticsearchSortParam(esAllColumns)}, nil
139 + default:
140 + return nil, fmt.Errorf("unknown method: %s", method)
141 + }
142 +}
143 +
144 +func elasticsearchHandleMethod(ctx context.Context, job *module.Job, method string, params funcapi.ResolvedParams) *module.FunctionResponse {
145 + collector, ok := job.Module().(*Collector)
146 + if !ok {
147 + return &module.FunctionResponse{Status: 500, Message: "internal error: invalid module type"}
148 + }
149 +
150 + if collector.httpClient == nil {
151 + return &module.FunctionResponse{
152 + Status: 503,
153 + Message: "collector is still initializing, please retry in a few seconds",
154 + }
155 + }
156 +
157 + switch method {
158 + case "top-queries":
159 + return collector.collectTopQueries(ctx, params.Column(paramSort))
160 + default:
161 + return &module.FunctionResponse{Status: 404, Message: fmt.Sprintf("unknown method: %s", method)}
162 + }
163 +}
164 +
165 +func buildElasticsearchSortOptions(cols []esColumnMeta) []funcapi.ParamOption {
166 + var sortOptions []funcapi.ParamOption
167 + sortDir := funcapi.FieldSortDescending
168 + for _, col := range cols {
169 + if !col.sortable {
170 + continue
171 + }
172 + opt := funcapi.ParamOption{
173 + ID: col.id,
174 + Column: col.id,
175 + Name: fmt.Sprintf("Top queries by %s", col.name),
176 + Sort: &sortDir,
177 + }
178 + if col.id == "runningTime" {
179 + opt.Default = true
180 + }
181 + sortOptions = append(sortOptions, opt)
182 + }
183 + return sortOptions
184 +}
185 +
186 +func buildElasticsearchSortParam(cols []esColumnMeta) funcapi.ParamConfig {
187 + return funcapi.ParamConfig{
188 + ID: paramSort,
189 + Name: "Filter By",
190 + Help: "Select the primary sort column",
191 + Selection: funcapi.ParamSelect,
192 + Options: buildElasticsearchSortOptions(cols),
193 + UniqueView: true,
194 + }
195 +}
196 +
197 +func buildElasticsearchColumns(cols []esColumnMeta) map[string]any {
198 + result := make(map[string]any, len(cols))
199 + for i, col := range cols {
200 + colDef := funcapi.Column{
201 + Index: i,
202 + Name: col.name,
203 + Type: col.colType,
204 + Units: col.units,
205 + Visualization: col.visualization,
206 + Sort: col.sortDir,
207 + Sortable: col.sortable,
208 + Sticky: col.sticky,
209 + Summary: col.summary,
210 + Filter: col.filter,
211 + FullWidth: col.fullWidth,
212 + Wrap: col.wrap,
213 + DefaultExpandedFilter: false,
214 + UniqueKey: col.uniqueKey,
215 + Visible: col.visible,
216 + ValueOptions: funcapi.ValueOptions{
217 + Transform: col.transform,
218 + DecimalPoints: col.decimalPoints,
219 + DefaultValue: nil,
220 + },
221 + }
222 + result[col.id] = colDef.BuildColumn()
223 + }
224 + return result
225 +}
226 +
227 +func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
228 + limit := c.TopQueriesLimit
229 + if limit <= 0 {
230 + limit = 500
231 + }
232 +
233 + req, err := web.NewHTTPRequestWithPath(c.RequestConfig, "/_tasks")
234 + if err != nil {
235 + return &module.FunctionResponse{Status: 500, Message: err.Error()}
236 + }
237 + req = req.WithContext(ctx)
238 + q := url.Values{}
239 + q.Set("actions", "*search")
240 + q.Set("detailed", "true")
241 + req.URL.RawQuery = q.Encode()
242 +
243 + var resp esTasksResponse
244 + if err := web.DoHTTP(c.httpClient).RequestJSON(req, &resp); err != nil {
245 + if ctx.Err() == context.DeadlineExceeded {
246 + return &module.FunctionResponse{Status: 504, Message: "query timed out"}
247 + }
248 + return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("tasks query failed: %v", err)}
249 + }
250 +
251 + rows := make([]esTaskRow, 0, 100)
252 + for nodeID, node := range resp.Nodes {
253 + for taskID, task := range node.Tasks {
254 + rows = append(rows, esTaskRow{
255 + TaskID: taskID,
256 + NodeID: nodeID,
257 + NodeName: node.Name,
258 + Action: task.Action,
259 + Type: task.Type,
260 + Description: task.Description,
261 + StartTime: time.UnixMilli(task.StartTimeInMillis),
262 + RunningTime: time.Duration(task.RunningTimeInNanos),
263 + Cancellable: task.Cancellable,
264 + Cancelled: task.Cancelled,
265 + })
266 + }
267 + }
268 +
269 + if len(rows) == 0 {
270 + return &module.FunctionResponse{
271 + Status: 200,
272 + Message: "No running search tasks found.",
273 + Help: "Running queries from Elasticsearch Tasks API",
274 + Columns: buildElasticsearchColumns(esAllColumns),
275 + Data: [][]any{},
276 + DefaultSortColumn: "runningTime",
277 + RequiredParams: []funcapi.ParamConfig{buildElasticsearchSortParam(esAllColumns)},
278 + Charts: elasticsearchTopQueriesCharts(esAllColumns),
279 + DefaultCharts: elasticsearchTopQueriesDefaultCharts(esAllColumns),
280 + GroupBy: elasticsearchTopQueriesGroupBy(esAllColumns),
281 + }
282 + }
283 +
284 + sortColumn = mapElasticsearchSortColumn(sortColumn)
285 + sortElasticsearchRows(rows, sortColumn)
286 +
287 + if len(rows) > limit {
288 + rows = rows[:limit]
289 + }
290 +
291 + data := make([][]any, 0, len(rows))
292 + for _, row := range rows {
293 + out := make([]any, len(esAllColumns))
294 + for i, col := range esAllColumns {
295 + switch col.id {
296 + case "taskId":
297 + out[i] = row.TaskID
298 + case "node":
299 + out[i] = row.NodeID
300 + case "nodeName":
301 + out[i] = row.NodeName
302 + case "action":
303 + out[i] = row.Action
304 + case "type":
305 + out[i] = row.Type
306 + case "description":
307 + out[i] = strmutil.TruncateText(row.Description, elasticMaxQueryTextLength)
308 + case "startTime":
309 + out[i] = row.StartTime.Format(time.RFC3339Nano)
310 + case "runningTime":
311 + out[i] = float64(row.RunningTime) / float64(time.Millisecond)
312 + case "cancellable":
313 + out[i] = row.Cancellable
314 + case "cancelled":
315 + out[i] = row.Cancelled
316 + default:
317 + out[i] = nil
318 + }
319 + }
320 + data = append(data, out)
321 + }
322 +
323 + return &module.FunctionResponse{
324 + Status: 200,
325 + Help: "Running queries from Elasticsearch Tasks API",
326 + Columns: buildElasticsearchColumns(esAllColumns),
327 + Data: data,
328 + DefaultSortColumn: "runningTime",
329 + RequiredParams: []funcapi.ParamConfig{buildElasticsearchSortParam(esAllColumns)},
330 + Charts: elasticsearchTopQueriesCharts(esAllColumns),
331 + DefaultCharts: elasticsearchTopQueriesDefaultCharts(esAllColumns),
332 + GroupBy: elasticsearchTopQueriesGroupBy(esAllColumns),
333 + }
334 +}
335 +
336 +func mapElasticsearchSortColumn(col string) string {
337 + switch col {
338 + case "runningTime", "startTime", "taskId":
339 + return col
340 + default:
341 + return "runningTime"
342 + }
343 +}
344 +
345 +func sortElasticsearchRows(rows []esTaskRow, sortColumn string) {
346 + switch sortColumn {
347 + case "startTime":
348 + sort.Slice(rows, func(i, j int) bool {
349 + return rows[i].StartTime.After(rows[j].StartTime)
350 + })
351 + case "taskId":
352 + sort.Slice(rows, func(i, j int) bool {
353 + left, lok := parseElasticsearchTaskID(rows[i].TaskID)
354 + right, rok := parseElasticsearchTaskID(rows[j].TaskID)
355 + if lok && rok {
356 + return left > right
357 + }
358 + return rows[i].TaskID > rows[j].TaskID
359 + })
360 + default:
361 + sort.Slice(rows, func(i, j int) bool {
362 + return rows[i].RunningTime > rows[j].RunningTime
363 + })
364 + }
365 +}
366 +
367 +func parseElasticsearchTaskID(taskID string) (int64, bool) {
368 + id := taskID
369 + if idx := strings.LastIndex(id, ":"); idx != -1 {
370 + id = id[idx+1:]
371 + }
372 + val, err := strconv.ParseInt(id, 10, 64)
373 + if err != nil {
374 + return 0, false
375 + }
376 + return val, true
377 +}
378 +
379 +func elasticsearchTopQueriesCharts(cols []esColumnMeta) map[string]module.ChartConfig {
380 + charts := make(map[string]module.ChartConfig)
381 + for _, col := range cols {
382 + if !col.isMetric || col.chartGroup == "" {
383 + continue
384 + }
385 + cfg, ok := charts[col.chartGroup]
386 + if !ok {
387 + title := col.chartTitle
388 + if title == "" {
389 + title = col.chartGroup
390 + }
391 + cfg = module.ChartConfig{Name: title, Type: "stacked-bar"}
392 + }
393 + cfg.Columns = append(cfg.Columns, col.id)
394 + charts[col.chartGroup] = cfg
395 + }
396 + return charts
397 +}
398 +
399 +func elasticsearchTopQueriesDefaultCharts(cols []esColumnMeta) [][]string {
400 + label := primaryElasticsearchLabel(cols)
401 + if label == "" {
402 + return nil
403 + }
404 + chartGroups := defaultElasticsearchChartGroups(cols)
405 + out := make([][]string, 0, len(chartGroups))
406 + for _, group := range chartGroups {
407 + out = append(out, []string{group, label})
408 + }
409 + return out
410 +}
411 +
412 +func elasticsearchTopQueriesGroupBy(cols []esColumnMeta) map[string]module.GroupByConfig {
413 + groupBy := make(map[string]module.GroupByConfig)
414 + for _, col := range cols {
415 + if !col.isLabel {
416 + continue
417 + }
418 + groupBy[col.id] = module.GroupByConfig{
419 + Name: "Group by " + col.name,
420 + Columns: []string{col.id},
421 + }
422 + }
423 + return groupBy
424 +}
425 +
426 +func primaryElasticsearchLabel(cols []esColumnMeta) string {
427 + for _, col := range cols {
428 + if col.isPrimary {
429 + return col.id
430 + }
431 + }
432 + for _, col := range cols {
433 + if col.isLabel {
434 + return col.id
435 + }
436 + }
437 + return ""
438 +}
439 +
440 +func defaultElasticsearchChartGroups(cols []esColumnMeta) []string {
441 + groups := make([]string, 0)
442 + seen := make(map[string]bool)
443 + for _, col := range cols {
444 + if !col.isMetric || col.chartGroup == "" || !col.isDefaultChart {
445 + continue
446 + }
447 + if !seen[col.chartGroup] {
448 + seen[col.chartGroup] = true
449 + groups = append(groups, col.chartGroup)
450 + }
451 + }
452 + if len(groups) > 0 {
453 + return groups
454 + }
455 + for _, col := range cols {
456 + if !col.isMetric || col.chartGroup == "" {
457 + continue
458 + }
459 + if !seen[col.chartGroup] {
460 + seen[col.chartGroup] = true
461 + groups = append(groups, col.chartGroup)
462 + }
463 + }
464 + return groups
465 +}
src/go/plugin/go.d/collector/elasticsearch/functions_test.go new
+61
@@ -0,0 +1,61 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package elasticsearch
4 +
5 +import (
6 + "testing"
7 +
8 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
9 + "github.com/stretchr/testify/assert"
10 +)
11 +
12 +func TestElasticsearchMethods(t *testing.T) {
13 + methods := elasticsearchMethods()
14 +
15 + require := assert.New(t)
16 + require.Len(methods, 1)
17 + require.Equal("top-queries", methods[0].ID)
18 + require.Equal("Top Queries", methods[0].Name)
19 + require.NotEmpty(methods[0].RequiredParams)
20 +
21 + var sortParam *funcapi.ParamConfig
22 + for i := range methods[0].RequiredParams {
23 + if methods[0].RequiredParams[i].ID == "__sort" {
24 + sortParam = &methods[0].RequiredParams[i]
25 + break
26 + }
27 + }
28 + require.NotNil(sortParam, "expected __sort required param")
29 + require.NotEmpty(sortParam.Options)
30 +}
31 +
32 +func TestElasticsearchAllColumns_HasRequiredColumns(t *testing.T) {
33 + required := []string{"taskId", "description", "runningTime"}
34 +
35 + uiKeys := make(map[string]bool)
36 + for _, col := range esAllColumns {
37 + uiKeys[col.id] = true
38 + }
39 +
40 + for _, key := range required {
41 + assert.True(t, uiKeys[key], "column %s should be defined", key)
42 + }
43 +}
44 +
45 +func TestMapElasticsearchSortColumn(t *testing.T) {
46 + tests := map[string]struct {
47 + input string
48 + expected string
49 + }{
50 + "runningTime": {input: "runningTime", expected: "runningTime"},
51 + "startTime": {input: "startTime", expected: "startTime"},
52 + "taskId": {input: "taskId", expected: "taskId"},
53 + "invalid": {input: "bad", expected: "runningTime"},
54 + }
55 +
56 + for name, tc := range tests {
57 + t.Run(name, func(t *testing.T) {
58 + assert.Equal(t, tc.expected, mapElasticsearchSortColumn(tc.input))
59 + })
60 + }
61 +}
src/go/plugin/go.d/collector/mongodb/functions.go
+124 -70
@@ -51,23 +51,29 @@ const (
51
52 // mongoColumnMeta defines metadata for a column in the response
53 type mongoColumnMeta struct {
54 - id string // column ID in response (e.g., "execution_time")
55 - dbField string // MongoDB document field name (e.g., "millis")
56 - name string // display name (e.g., "Execution Time")
57 - colType funcapi.FieldType // column type: integer, duration, timestamp, string, bool
58 - visible bool // default visibility
59 - sortable bool // can be used for sorting
60 - fullWidth bool // for query text columns
61 - wrap bool // wrap text
62 - sticky bool // sticky column
63 - filter funcapi.FieldFilter // filter type: range, multiselect, text
64 - visualization funcapi.FieldVisual // visualization type: value, bar
65 - summary funcapi.FieldSummary // summary type: sum, max, or empty
66 - transform funcapi.FieldTransform // value transform: number, duration, datetime, text
67 - units string // display units (e.g., "seconds")
68 - decimalPoints int // decimal points for numeric display
69 - uniqueKey bool // unique key column
70 - expandFilter bool // default expanded filter
54 + id string // column ID in response (e.g., "execution_time")
55 + dbField string // MongoDB document field name (e.g., "millis")
56 + name string // display name (e.g., "Execution Time")
57 + colType funcapi.FieldType // column type: integer, duration, timestamp, string, bool
58 + visible bool // default visibility
59 + sortable bool // can be used for sorting
60 + fullWidth bool // for query text columns
61 + wrap bool // wrap text
62 + sticky bool // sticky column
63 + filter funcapi.FieldFilter // filter type: range, multiselect, text
64 + visualization funcapi.FieldVisual // visualization type: value, bar
65 + summary funcapi.FieldSummary // summary type: sum, max, or empty
66 + transform funcapi.FieldTransform // value transform: number, duration, datetime, text
67 + units string // display units (e.g., "seconds")
68 + decimalPoints int // decimal points for numeric display
69 + uniqueKey bool // unique key column
70 + expandFilter bool // default expanded filter
71 + isLabel bool // available for group-by
72 + isPrimary bool // primary label
73 + isMetric bool // chartable metric
74 + chartGroup string // chart group key
75 + chartTitle string // chart title
76 + isDefaultChart bool // include in default charts
77 }
78
79 // mongoAllColumns defines all available columns from system.profile
@@ -75,24 +81,24 @@ type mongoColumnMeta struct {
81 var mongoAllColumns = []mongoColumnMeta{
82 // Core fields (visible by default)
83 {id: "timestamp", dbField: "ts", name: "Timestamp", colType: ftTimestamp, visible: true, sortable: true, filter: filterRange, visualization: visValue, summary: summaryMax, transform: trDatetime, uniqueKey: true},
78 - {id: "namespace", dbField: "ns", name: "Namespace", colType: ftString, visible: true, sortable: false, sticky: true, filter: filterMulti, visualization: visValue, transform: trText, expandFilter: true},
79 - {id: "operation", dbField: "op", name: "Operation", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText},
84 + {id: "namespace", dbField: "ns", name: "Namespace", colType: ftString, visible: true, sortable: false, sticky: true, filter: filterMulti, visualization: visValue, transform: trText, expandFilter: true, isLabel: true, isPrimary: true},
85 + {id: "operation", dbField: "op", name: "Operation", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, isLabel: true},
86 {id: "query", dbField: "command", name: "Query", colType: ftString, visible: true, sortable: false, fullWidth: true, wrap: true, filter: filterMulti, visualization: visValue, transform: trText},
81 - {id: "execution_time", dbField: "millis", name: "Execution Time", colType: ftDuration, visible: true, sortable: true, filter: filterRange, visualization: visBar, summary: summarySum, transform: trDuration, units: "seconds", decimalPoints: 3},
82 - {id: "docs_examined", dbField: "docsExamined", name: "Docs Examined", colType: ftInteger, visible: true, sortable: true, filter: filterRange, visualization: visValue, summary: summarySum, transform: trNumber},
83 - {id: "keys_examined", dbField: "keysExamined", name: "Keys Examined", colType: ftInteger, visible: true, sortable: true, filter: filterRange, visualization: visValue, summary: summarySum, transform: trNumber},
84 - {id: "docs_returned", dbField: "nreturned", name: "Docs Returned", colType: ftInteger, visible: true, sortable: true, filter: filterRange, visualization: visValue, summary: summarySum, transform: trNumber},
87 + {id: "execution_time", dbField: "millis", name: "Execution Time", colType: ftDuration, visible: true, sortable: true, filter: filterRange, visualization: visBar, summary: summarySum, transform: trDuration, units: "seconds", decimalPoints: 3, isMetric: true, chartGroup: "Time", chartTitle: "Execution Time", isDefaultChart: true},
88 + {id: "docs_examined", dbField: "docsExamined", name: "Docs Examined", colType: ftInteger, visible: true, sortable: true, filter: filterRange, visualization: visValue, summary: summarySum, transform: trNumber, isMetric: true, chartGroup: "Docs", chartTitle: "Documents"},
89 + {id: "keys_examined", dbField: "keysExamined", name: "Keys Examined", colType: ftInteger, visible: true, sortable: true, filter: filterRange, visualization: visValue, summary: summarySum, transform: trNumber, isMetric: true, chartGroup: "Docs", chartTitle: "Documents"},
90 + {id: "docs_returned", dbField: "nreturned", name: "Docs Returned", colType: ftInteger, visible: true, sortable: true, filter: filterRange, visualization: visValue, summary: summarySum, transform: trNumber, isMetric: true, chartGroup: "Docs", chartTitle: "Documents"},
91 {id: "plan_summary", dbField: "planSummary", name: "Plan Summary", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText},
92
93 // Secondary fields
88 - {id: "client", dbField: "client", name: "Client", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText},
89 - {id: "user", dbField: "user", name: "User", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText},
90 - {id: "docs_deleted", dbField: "ndeleted", name: "Docs Deleted", colType: ftInteger, visible: false, sortable: true, filter: filterRange, visualization: visValue, summary: summarySum, transform: trNumber},
91 - {id: "docs_inserted", dbField: "ninserted", name: "Docs Inserted", colType: ftInteger, visible: false, sortable: true, filter: filterRange, visualization: visValue, summary: summarySum, transform: trNumber},
92 - {id: "docs_modified", dbField: "nModified", name: "Docs Modified", colType: ftInteger, visible: false, sortable: true, filter: filterRange, visualization: visValue, summary: summarySum, transform: trNumber},
93 - {id: "response_length", dbField: "responseLength", name: "Response Length", colType: ftInteger, visible: false, sortable: true, filter: filterRange, visualization: visValue, summary: summarySum, transform: trNumber},
94 - {id: "num_yield", dbField: "numYield", name: "Num Yield", colType: ftInteger, visible: false, sortable: true, filter: filterRange, visualization: visValue, summary: summarySum, transform: trNumber},
95 - {id: "app_name", dbField: "appName", name: "App Name", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText},
94 + {id: "client", dbField: "client", name: "Client", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, isLabel: true},
95 + {id: "user", dbField: "user", name: "User", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, isLabel: true},
96 + {id: "docs_deleted", dbField: "ndeleted", name: "Docs Deleted", colType: ftInteger, visible: false, sortable: true, filter: filterRange, visualization: visValue, summary: summarySum, transform: trNumber, isMetric: true, chartGroup: "Docs", chartTitle: "Documents"},
97 + {id: "docs_inserted", dbField: "ninserted", name: "Docs Inserted", colType: ftInteger, visible: false, sortable: true, filter: filterRange, visualization: visValue, summary: summarySum, transform: trNumber, isMetric: true, chartGroup: "Docs", chartTitle: "Documents"},
98 + {id: "docs_modified", dbField: "nModified", name: "Docs Modified", colType: ftInteger, visible: false, sortable: true, filter: filterRange, visualization: visValue, summary: summarySum, transform: trNumber, isMetric: true, chartGroup: "Docs", chartTitle: "Documents"},
99 + {id: "response_length", dbField: "responseLength", name: "Response Length", colType: ftInteger, visible: false, sortable: true, filter: filterRange, visualization: visValue, summary: summarySum, transform: trNumber, isMetric: true, chartGroup: "Response", chartTitle: "Response Size"},
100 + {id: "num_yield", dbField: "numYield", name: "Num Yield", colType: ftInteger, visible: false, sortable: true, filter: filterRange, visualization: visValue, summary: summarySum, transform: trNumber, isMetric: true, chartGroup: "Yield", chartTitle: "Yield"},
101 + {id: "app_name", dbField: "appName", name: "App Name", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, isLabel: true},
102 {id: "cursor_exhausted", dbField: "cursorExhausted", name: "Cursor Exhausted", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText},
103 {id: "has_sort_stage", dbField: "hasSortStage", name: "Has Sort Stage", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText},
104 {id: "uses_disk", dbField: "usedDisk", name: "Uses Disk", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText},
@@ -102,8 +108,8 @@ var mongoAllColumns = []mongoColumnMeta{
108 // Version-specific fields (hidden by default)
109 {id: "query_hash", dbField: "queryHash", name: "Query Hash", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText}, // 4.2+
110 {id: "plan_cache_key", dbField: "planCacheKey", name: "Plan Cache Key", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText}, // 4.2+
105 - {id: "planning_time", dbField: "planningTimeMicros", name: "Planning Time", colType: ftDuration, visible: false, sortable: true, filter: filterRange, visualization: visBar, summary: summarySum, transform: trDuration, units: "seconds", decimalPoints: 3},
106 - {id: "cpu_time", dbField: "cpuNanos", name: "CPU Time", colType: ftDuration, visible: false, sortable: true, filter: filterRange, visualization: visBar, summary: summarySum, transform: trDuration, units: "seconds", decimalPoints: 3},
111 + {id: "planning_time", dbField: "planningTimeMicros", name: "Planning Time", colType: ftDuration, visible: false, sortable: true, filter: filterRange, visualization: visBar, summary: summarySum, transform: trDuration, units: "seconds", decimalPoints: 3, isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
112 + {id: "cpu_time", dbField: "cpuNanos", name: "CPU Time", colType: ftDuration, visible: false, sortable: true, filter: filterRange, visualization: visBar, summary: summarySum, transform: trDuration, units: "seconds", decimalPoints: 3, isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
113 {id: "query_framework", dbField: "queryFramework", name: "Query Framework", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText}, // 7.0+
114 {id: "query_shape_hash", dbField: "queryShapeHash", name: "Query Shape Hash", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText}, // 8.0+
115 }
@@ -128,46 +134,94 @@ func optionalBool(v *bool) any {
134 }
135
136 // topQueriesCharts returns the chart configuration for top queries responses
131 -func topQueriesCharts() map[string]module.ChartConfig {
132 - return map[string]module.ChartConfig{
133 - "Time": {
134 - Name: "Execution Time",
135 - Type: "stacked-bar",
136 - Columns: []string{"execution_time"},
137 - },
138 - "DocsExamined": {
139 - Name: "Documents & Keys Examined",
140 - Type: "stacked-bar",
141 - Columns: []string{"docs_examined", "keys_examined"},
142 - },
143 - "DocsReturned": {
144 - Name: "Documents Returned",
145 - Type: "stacked-bar",
146 - Columns: []string{"docs_returned"},
147 - },
137 +func topQueriesCharts(cols []mongoColumnMeta) map[string]module.ChartConfig {
138 + charts := make(map[string]module.ChartConfig)
139 + for _, col := range cols {
140 + if !col.isMetric || col.chartGroup == "" {
141 + continue
142 + }
143 + cfg, ok := charts[col.chartGroup]
144 + if !ok {
145 + title := col.chartTitle
146 + if title == "" {
147 + title = col.chartGroup
148 + }
149 + cfg = module.ChartConfig{Name: title, Type: "stacked-bar"}
150 + }
151 + cfg.Columns = append(cfg.Columns, col.id)
152 + charts[col.chartGroup] = cfg
153 }
154 + return charts
155 }
156
157 // topQueriesDefaultCharts returns the default chart configuration
152 -func topQueriesDefaultCharts() [][]string {
153 - return [][]string{
154 - {"Time", "namespace"},
155 - {"DocsExamined", "namespace"},
158 +func topQueriesDefaultCharts(cols []mongoColumnMeta) [][]string {
159 + label := primaryMongoLabel(cols)
160 + if label == "" {
161 + return nil
162 + }
163 + chartGroups := defaultMongoChartGroups(cols)
164 + out := make([][]string, 0, len(chartGroups))
165 + for _, group := range chartGroups {
166 + out = append(out, []string{group, label})
167 }
168 + return out
169 }
170
171 // topQueriesGroupBy returns the group by configuration for top queries responses
160 -func topQueriesGroupBy() map[string]module.GroupByConfig {
161 - return map[string]module.GroupByConfig{
162 - "namespace": {
163 - Name: "Group by Namespace",
164 - Columns: []string{"namespace"},
165 - },
166 - "operation": {
167 - Name: "Group by Operation Type",
168 - Columns: []string{"operation"},
169 - },
172 +func topQueriesGroupBy(cols []mongoColumnMeta) map[string]module.GroupByConfig {
173 + groupBy := make(map[string]module.GroupByConfig)
174 + for _, col := range cols {
175 + if !col.isLabel {
176 + continue
177 + }
178 + groupBy[col.id] = module.GroupByConfig{
179 + Name: "Group by " + col.name,
180 + Columns: []string{col.id},
181 + }
182 + }
183 + return groupBy
184 +}
185 +
186 +func primaryMongoLabel(cols []mongoColumnMeta) string {
187 + for _, col := range cols {
188 + if col.isPrimary {
189 + return col.id
190 + }
191 + }
192 + for _, col := range cols {
193 + if col.isLabel {
194 + return col.id
195 + }
196 + }
197 + return ""
198 +}
199 +
200 +func defaultMongoChartGroups(cols []mongoColumnMeta) []string {
201 + groups := make([]string, 0)
202 + seen := make(map[string]bool)
203 + for _, col := range cols {
204 + if !col.isMetric || col.chartGroup == "" || !col.isDefaultChart {
205 + continue
206 + }
207 + if !seen[col.chartGroup] {
208 + seen[col.chartGroup] = true
209 + groups = append(groups, col.chartGroup)
210 + }
211 + }
212 + if len(groups) > 0 {
213 + return groups
214 + }
215 + for _, col := range cols {
216 + if !col.isMetric || col.chartGroup == "" {
217 + continue
218 + }
219 + if !seen[col.chartGroup] {
220 + seen[col.chartGroup] = true
221 + groups = append(groups, col.chartGroup)
222 + }
223 }
224 + return groups
225 }
226
227 func buildMongoSortParam(cols []mongoColumnMeta) funcapi.ParamConfig {
@@ -535,9 +589,9 @@ func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *m
589 Data: [][]any{},
590 DefaultSortColumn: "execution_time",
591 RequiredParams: []funcapi.ParamConfig{sortParam},
538 - Charts: topQueriesCharts(),
539 - DefaultCharts: topQueriesDefaultCharts(),
540 - GroupBy: topQueriesGroupBy(),
592 + Charts: topQueriesCharts(availableCols),
593 + DefaultCharts: topQueriesDefaultCharts(availableCols),
594 + GroupBy: topQueriesGroupBy(availableCols),
595 }
596
597 if len(allDocs) == 0 {
@@ -635,9 +689,9 @@ func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *m
689 Data: data,
690 DefaultSortColumn: "execution_time",
691 RequiredParams: []funcapi.ParamConfig{sortParam},
638 - Charts: topQueriesCharts(),
639 - DefaultCharts: topQueriesDefaultCharts(),
640 - GroupBy: topQueriesGroupBy(),
692 + Charts: topQueriesCharts(availableCols),
693 + DefaultCharts: topQueriesDefaultCharts(availableCols),
694 + GroupBy: topQueriesGroupBy(availableCols),
695 }
696 }
697
src/go/plugin/go.d/collector/mssql/functions.go
+161 -32
@@ -62,6 +62,12 @@ type mssqlColumnMeta struct {
62 fullWidth bool // Should column take full width
63 isIdentity bool // Is this an identity column (query_hash, query_text, etc.)
64 needsAvg bool // Needs weighted average calculation (avg_* columns)
65 + isLabel bool // Is this column a label for grouping
66 + isPrimary bool // Is this label the primary grouping
67 + isMetric bool // Is this column a chartable metric
68 + chartGroup string // Chart group key
69 + chartTitle string // Chart title
70 + isDefaultChart bool // Include this chart group in defaults
71 }
72
73 // mssqlAllColumns defines ALL possible columns from Query Store
@@ -154,6 +160,33 @@ var mssqlAllColumns = []mssqlColumnMeta{
160 {dbColumn: "stdev_tempdb_space_used", uiKey: "stdevTempdb", displayName: "StdDev TempDB", dataType: ftFloat, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
161 }
162
163 +type mssqlChartGroup struct {
164 + key string
165 + title string
166 + columns []string
167 + defaultChart bool
168 +}
169 +
170 +var mssqlChartGroups = []mssqlChartGroup{
171 + {key: "Calls", title: "Number of Calls", columns: []string{"calls"}, defaultChart: true},
172 + {key: "Time", title: "Execution Time", columns: []string{"totalTime", "avgTime", "lastTime", "minTime", "maxTime", "stdevTime"}, defaultChart: true},
173 + {key: "CPU", title: "CPU Time", columns: []string{"avgCpu", "lastCpu", "minCpu", "maxCpu", "stdevCpu"}},
174 + {key: "LogicalIO", title: "Logical I/O", columns: []string{"avgReads", "lastReads", "minReads", "maxReads", "stdevReads", "avgWrites", "lastWrites", "minWrites", "maxWrites", "stdevWrites"}},
175 + {key: "PhysicalIO", title: "Physical Reads", columns: []string{"avgPhysReads", "lastPhysReads", "minPhysReads", "maxPhysReads", "stdevPhysReads"}},
176 + {key: "CLR", title: "CLR Time", columns: []string{"avgClr", "lastClr", "minClr", "maxClr", "stdevClr"}},
177 + {key: "DOP", title: "Parallelism", columns: []string{"avgDop", "lastDop", "minDop", "maxDop", "stdevDop"}},
178 + {key: "Memory", title: "Memory Grant", columns: []string{"avgMemory", "lastMemory", "minMemory", "maxMemory", "stdevMemory"}},
179 + {key: "Rows", title: "Rows", columns: []string{"avgRows", "lastRows", "minRows", "maxRows", "stdevRows"}},
180 + {key: "LogBytes", title: "Log Bytes", columns: []string{"avgLogBytes", "lastLogBytes", "minLogBytes", "maxLogBytes", "stdevLogBytes"}},
181 + {key: "TempDB", title: "TempDB Usage", columns: []string{"avgTempdb", "lastTempdb", "minTempdb", "maxTempdb", "stdevTempdb"}},
182 +}
183 +
184 +var mssqlLabelColumns = map[string]bool{
185 + "database": true,
186 +}
187 +
188 +const mssqlPrimaryLabel = "database"
189 +
190 // mssqlMethods returns the available function methods for MSSQL
191 func mssqlMethods() []module.MethodConfig {
192 // Build sort options from column metadata
@@ -717,6 +750,8 @@ func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *m
750 defaultSort = sortOptions[0].ID
751 }
752
753 + annotatedCols := decorateMSSQLColumns(cols)
754 +
755 return &module.FunctionResponse{
756 Status: 200,
757 Help: "Top SQL queries from Query Store. WARNING: Query text may contain unmasked literals (potential PII).",
@@ -726,37 +761,131 @@ func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *m
761 RequiredParams: []funcapi.ParamConfig{sortParam},
762
763 // Charts for aggregated visualization
729 - Charts: map[string]module.ChartConfig{
730 - "Calls": {
731 - Name: "Number of Calls",
732 - Type: "stacked-bar",
733 - Columns: []string{"calls"},
734 - },
735 - "Time": {
736 - Name: "Execution Time",
737 - Type: "stacked-bar",
738 - Columns: []string{"totalTime", "avgTime"},
739 - },
740 - "CPU": {
741 - Name: "CPU Time",
742 - Type: "stacked-bar",
743 - Columns: []string{"avgCpu"},
744 - },
745 - "IO": {
746 - Name: "Logical I/O",
747 - Type: "stacked-bar",
748 - Columns: []string{"avgReads", "avgWrites"},
749 - },
750 - },
751 - DefaultCharts: [][]string{
752 - {"Time", "database"},
753 - {"Calls", "database"},
754 - },
755 - GroupBy: map[string]module.GroupByConfig{
756 - "database": {
757 - Name: "Group by Database",
758 - Columns: []string{"database"},
759 - },
760 - },
764 + Charts: mssqlTopQueriesCharts(annotatedCols),
765 + DefaultCharts: mssqlTopQueriesDefaultCharts(annotatedCols),
766 + GroupBy: mssqlTopQueriesGroupBy(annotatedCols),
767 + }
768 +}
769 +
770 +func decorateMSSQLColumns(cols []mssqlColumnMeta) []mssqlColumnMeta {
771 + out := make([]mssqlColumnMeta, len(cols))
772 + index := make(map[string]int, len(cols))
773 + for i, col := range cols {
774 + out[i] = col
775 + index[col.uiKey] = i
776 + }
777 +
778 + for i := range out {
779 + if mssqlLabelColumns[out[i].uiKey] {
780 + out[i].isLabel = true
781 + if out[i].uiKey == mssqlPrimaryLabel {
782 + out[i].isPrimary = true
783 + }
784 + }
785 + }
786 +
787 + for _, group := range mssqlChartGroups {
788 + for _, key := range group.columns {
789 + idx, ok := index[key]
790 + if !ok {
791 + continue
792 + }
793 + out[idx].isMetric = true
794 + out[idx].chartGroup = group.key
795 + out[idx].chartTitle = group.title
796 + if group.defaultChart {
797 + out[idx].isDefaultChart = true
798 + }
799 + }
800 + }
801 +
802 + return out
803 +}
804 +
805 +func mssqlTopQueriesCharts(cols []mssqlColumnMeta) map[string]module.ChartConfig {
806 + charts := make(map[string]module.ChartConfig)
807 + for _, col := range cols {
808 + if !col.isMetric || col.chartGroup == "" {
809 + continue
810 + }
811 + cfg, ok := charts[col.chartGroup]
812 + if !ok {
813 + title := col.chartTitle
814 + if title == "" {
815 + title = col.chartGroup
816 + }
817 + cfg = module.ChartConfig{Name: title, Type: "stacked-bar"}
818 + }
819 + cfg.Columns = append(cfg.Columns, col.uiKey)
820 + charts[col.chartGroup] = cfg
821 + }
822 + return charts
823 +}
824 +
825 +func mssqlTopQueriesDefaultCharts(cols []mssqlColumnMeta) [][]string {
826 + label := primaryMSSQLLabel(cols)
827 + if label == "" {
828 + return nil
829 + }
830 + chartGroups := defaultMSSQLChartGroups(cols)
831 + out := make([][]string, 0, len(chartGroups))
832 + for _, group := range chartGroups {
833 + out = append(out, []string{group, label})
834 + }
835 + return out
836 +}
837 +
838 +func mssqlTopQueriesGroupBy(cols []mssqlColumnMeta) map[string]module.GroupByConfig {
839 + groupBy := make(map[string]module.GroupByConfig)
840 + for _, col := range cols {
841 + if !col.isLabel {
842 + continue
843 + }
844 + groupBy[col.uiKey] = module.GroupByConfig{
845 + Name: "Group by " + col.displayName,
846 + Columns: []string{col.uiKey},
847 + }
848 + }
849 + return groupBy
850 +}
851 +
852 +func primaryMSSQLLabel(cols []mssqlColumnMeta) string {
853 + for _, col := range cols {
854 + if col.isPrimary {
855 + return col.uiKey
856 + }
857 + }
858 + for _, col := range cols {
859 + if col.isLabel {
860 + return col.uiKey
861 + }
862 + }
863 + return ""
864 +}
865 +
866 +func defaultMSSQLChartGroups(cols []mssqlColumnMeta) []string {
867 + groups := make([]string, 0)
868 + seen := make(map[string]bool)
869 + for _, col := range cols {
870 + if !col.isMetric || col.chartGroup == "" || !col.isDefaultChart {
871 + continue
872 + }
873 + if !seen[col.chartGroup] {
874 + seen[col.chartGroup] = true
875 + groups = append(groups, col.chartGroup)
876 + }
877 + }
878 + if len(groups) > 0 {
879 + return groups
880 + }
881 + for _, col := range cols {
882 + if !col.isMetric || col.chartGroup == "" {
883 + continue
884 + }
885 + if !seen[col.chartGroup] {
886 + seen[col.chartGroup] = true
887 + groups = append(groups, col.chartGroup)
888 + }
889 }
890 + return groups
891 }
src/go/plugin/go.d/collector/mysql/functions.go
+181 -50
@@ -41,24 +41,30 @@ const (
41
42 // mysqlColumnMeta defines metadata for a single column
43 type mysqlColumnMeta struct {
44 - dbColumn string // Column name in database (e.g., "SUM_TIMER_WAIT")
45 - uiKey string // Canonical name used everywhere: SQL alias, UI key, sort key
46 - displayName string // Display name in UI (e.g., "Total Time")
47 - dataType funcapi.FieldType // "string", "integer", "float", "duration"
48 - units string // Unit for duration/numeric types (e.g., "seconds")
49 - visible bool // Default visibility
50 - transform funcapi.FieldTransform // Transform for value_options (e.g., "duration", "number", "none")
51 - decimalPoints int // Decimal points for display
52 - sortDir funcapi.FieldSort // Sort direction: "ascending" or "descending"
53 - summary funcapi.FieldSummary // Summary function: "sum", "count", "max", "min", "mean" (UI aggregations)
54 - filter funcapi.FieldFilter // Filter type: "multiselect" or "range"
55 - isPicoseconds bool // Needs picoseconds to seconds conversion
56 - isSortOption bool // Show in sort dropdown
57 - sortLabel string // Label for sort option
58 - isDefaultSort bool // Is this the default sort option
59 - isUniqueKey bool // Is this column a unique key
60 - isSticky bool // Is this column sticky in UI
61 - fullWidth bool // Should column take full width
44 + dbColumn string // Column name in database (e.g., "SUM_TIMER_WAIT")
45 + uiKey string // Canonical name used everywhere: SQL alias, UI key, sort key
46 + displayName string // Display name in UI (e.g., "Total Time")
47 + dataType funcapi.FieldType // "string", "integer", "float", "duration"
48 + units string // Unit for duration/numeric types (e.g., "seconds")
49 + visible bool // Default visibility
50 + transform funcapi.FieldTransform // Transform for value_options (e.g., "duration", "number", "none")
51 + decimalPoints int // Decimal points for display
52 + sortDir funcapi.FieldSort // Sort direction: "ascending" or "descending"
53 + summary funcapi.FieldSummary // Summary function: "sum", "count", "max", "min", "mean" (UI aggregations)
54 + filter funcapi.FieldFilter // Filter type: "multiselect" or "range"
55 + isPicoseconds bool // Needs picoseconds to seconds conversion
56 + isSortOption bool // Show in sort dropdown
57 + sortLabel string // Label for sort option
58 + isDefaultSort bool // Is this the default sort option
59 + isUniqueKey bool // Is this column a unique key
60 + isSticky bool // Is this column sticky in UI
61 + fullWidth bool // Should column take full width
62 + isLabel bool // Is this column a label for grouping
63 + isPrimary bool // Is this label the primary grouping
64 + isMetric bool // Is this column a chartable metric
65 + chartGroup string // Chart group key
66 + chartTitle string // Chart title
67 + isDefaultChart bool // Include this chart group in defaults
68 }
69
70 // mysqlAllColumns defines ALL possible columns from events_statements_summary_by_digest
@@ -133,6 +139,35 @@ var mysqlAllColumns = []mysqlColumnMeta{
139 {dbColumn: "MAX_TOTAL_MEMORY", uiKey: "maxTotalMemory", displayName: "Max Total Memory", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange, isSortOption: true, sortLabel: "Top queries by Max Total Memory"},
140 }
141
142 +type mysqlChartGroup struct {
143 + key string
144 + title string
145 + columns []string
146 + defaultChart bool
147 +}
148 +
149 +var mysqlChartGroups = []mysqlChartGroup{
150 + {key: "Calls", title: "Number of Calls", columns: []string{"calls"}, defaultChart: true},
151 + {key: "Time", title: "Execution Time", columns: []string{"totalTime", "avgTime", "minTime", "maxTime"}, defaultChart: true},
152 + {key: "Percentiles", title: "Execution Time Percentiles", columns: []string{"p95Time", "p99Time", "p999Time"}},
153 + {key: "LockTime", title: "Lock Time", columns: []string{"lockTime"}},
154 + {key: "Errors", title: "Errors & Warnings", columns: []string{"errors", "warnings"}},
155 + {key: "Rows", title: "Rows", columns: []string{"rowsSent", "rowsExamined", "rowsAffected"}},
156 + {key: "TempTables", title: "Temp Tables", columns: []string{"tmpDiskTables", "tmpTables"}},
157 + {key: "Joins", title: "Join Operations", columns: []string{"fullJoin", "fullRangeJoin", "selectRange", "selectRangeCheck", "selectScan"}},
158 + {key: "Sort", title: "Sort Operations", columns: []string{"sortMergePasses", "sortRange", "sortRows", "sortScan"}},
159 + {key: "Index", title: "Index Usage", columns: []string{"noIndexUsed", "noGoodIndexUsed"}},
160 + {key: "CPU", title: "CPU Time", columns: []string{"cpuTime"}},
161 + {key: "Memory", title: "Memory", columns: []string{"maxControlledMemory", "maxTotalMemory"}},
162 + {key: "Sample", title: "Sample Time", columns: []string{"sampleTime"}},
163 +}
164 +
165 +var mysqlLabelColumns = map[string]bool{
166 + "schema": true,
167 +}
168 +
169 +const mysqlPrimaryLabel = "schema"
170 +
171 // mysqlMethods returns the available function methods for MySQL
172 func mysqlMethods() []module.MethodConfig {
173 // Build sort options from column metadata
@@ -558,6 +593,8 @@ func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *m
593 defaultSort = sortOptions[0].ID
594 }
595
596 + annotatedCols := decorateMySQLColumns(cols)
597 +
598 return &module.FunctionResponse{
599 Status: 200,
600 Help: "Top SQL queries from performance_schema.events_statements_summary_by_digest",
@@ -567,39 +604,133 @@ func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *m
604 RequiredParams: []funcapi.ParamConfig{sortParam},
605
606 // Charts for aggregated visualization
570 - Charts: map[string]module.ChartConfig{
571 - "Calls": {
572 - Name: "Number of Calls",
573 - Type: "stacked-bar",
574 - Columns: []string{"calls"},
575 - },
576 - "Time": {
577 - Name: "Execution Time",
578 - Type: "stacked-bar",
579 - Columns: []string{"totalTime", "avgTime"},
580 - },
581 - "Rows": {
582 - Name: "Rows",
583 - Type: "stacked-bar",
584 - Columns: []string{"rowsSent", "rowsExamined", "rowsAffected"},
585 - },
586 - "Errors": {
587 - Name: "Errors & Warnings",
588 - Type: "stacked-bar",
589 - Columns: []string{"errors", "warnings"},
590 - },
591 - },
592 - DefaultCharts: [][]string{
593 - {"Time", "schema"},
594 - {"Calls", "schema"},
595 - },
596 - GroupBy: map[string]module.GroupByConfig{
597 - "schema": {
598 - Name: "Group by Schema",
599 - Columns: []string{"schema"},
600 - },
601 - },
607 + Charts: mysqlTopQueriesCharts(annotatedCols),
608 + DefaultCharts: mysqlTopQueriesDefaultCharts(annotatedCols),
609 + GroupBy: mysqlTopQueriesGroupBy(annotatedCols),
610 + }
611 +}
612 +
613 +func decorateMySQLColumns(cols []mysqlColumnMeta) []mysqlColumnMeta {
614 + out := make([]mysqlColumnMeta, len(cols))
615 + index := make(map[string]int, len(cols))
616 + for i, col := range cols {
617 + out[i] = col
618 + index[col.uiKey] = i
619 + }
620 +
621 + for i := range out {
622 + if mysqlLabelColumns[out[i].uiKey] {
623 + out[i].isLabel = true
624 + if out[i].uiKey == mysqlPrimaryLabel {
625 + out[i].isPrimary = true
626 + }
627 + }
628 + }
629 +
630 + for _, group := range mysqlChartGroups {
631 + for _, key := range group.columns {
632 + idx, ok := index[key]
633 + if !ok {
634 + continue
635 + }
636 + out[idx].isMetric = true
637 + out[idx].chartGroup = group.key
638 + out[idx].chartTitle = group.title
639 + if group.defaultChart {
640 + out[idx].isDefaultChart = true
641 + }
642 + }
643 + }
644 +
645 + return out
646 +}
647 +
648 +func mysqlTopQueriesCharts(cols []mysqlColumnMeta) map[string]module.ChartConfig {
649 + charts := make(map[string]module.ChartConfig)
650 + for _, col := range cols {
651 + if !col.isMetric || col.chartGroup == "" {
652 + continue
653 + }
654 + cfg, ok := charts[col.chartGroup]
655 + if !ok {
656 + title := col.chartTitle
657 + if title == "" {
658 + title = col.chartGroup
659 + }
660 + cfg = module.ChartConfig{Name: title, Type: "stacked-bar"}
661 + }
662 + cfg.Columns = append(cfg.Columns, col.uiKey)
663 + charts[col.chartGroup] = cfg
664 + }
665 + return charts
666 +}
667 +
668 +func mysqlTopQueriesDefaultCharts(cols []mysqlColumnMeta) [][]string {
669 + label := primaryMySQLLabel(cols)
670 + if label == "" {
671 + return nil
672 + }
673 + chartGroups := defaultMySQLChartGroups(cols)
674 + out := make([][]string, 0, len(chartGroups))
675 + for _, group := range chartGroups {
676 + out = append(out, []string{group, label})
677 + }
678 + return out
679 +}
680 +
681 +func mysqlTopQueriesGroupBy(cols []mysqlColumnMeta) map[string]module.GroupByConfig {
682 + groupBy := make(map[string]module.GroupByConfig)
683 + for _, col := range cols {
684 + if !col.isLabel {
685 + continue
686 + }
687 + groupBy[col.uiKey] = module.GroupByConfig{
688 + Name: "Group by " + col.displayName,
689 + Columns: []string{col.uiKey},
690 + }
691 + }
692 + return groupBy
693 +}
694 +
695 +func primaryMySQLLabel(cols []mysqlColumnMeta) string {
696 + for _, col := range cols {
697 + if col.isPrimary {
698 + return col.uiKey
699 + }
700 + }
701 + for _, col := range cols {
702 + if col.isLabel {
703 + return col.uiKey
704 + }
705 + }
706 + return ""
707 +}
708 +
709 +func defaultMySQLChartGroups(cols []mysqlColumnMeta) []string {
710 + groups := make([]string, 0)
711 + seen := make(map[string]bool)
712 + for _, col := range cols {
713 + if !col.isMetric || col.chartGroup == "" || !col.isDefaultChart {
714 + continue
715 + }
716 + if !seen[col.chartGroup] {
717 + seen[col.chartGroup] = true
718 + groups = append(groups, col.chartGroup)
719 + }
720 + }
721 + if len(groups) > 0 {
722 + return groups
723 + }
724 + for _, col := range cols {
725 + if !col.isMetric || col.chartGroup == "" {
726 + continue
727 + }
728 + if !seen[col.chartGroup] {
729 + seen[col.chartGroup] = true
730 + groups = append(groups, col.chartGroup)
731 + }
732 }
733 + return groups
734 }
735
736 // checkPerformanceSchema checks if performance_schema is enabled (cached)
src/go/plugin/go.d/collector/oracledb/collector.go
+4
@@ -22,6 +22,9 @@ func init() {
22 JobConfigSchema: configSchema,
23 Create: func() module.Module { return New() },
24 Config: func() any { return &Config{} },
25 + Methods: oracleMethods,
26 + MethodParams: oracleMethodParams,
27 + HandleMethod: oracleHandleMethod,
28 })
29 }
30
@@ -42,6 +45,7 @@ type Config struct {
45 AutoDetectionRetry int `yaml:"autodetection_retry,omitempty" json:"autodetection_retry"`
46 DSN string `json:"dsn" yaml:"dsn"`
47 Timeout confopt.Duration `yaml:"timeout,omitempty" json:"timeout"`
48 + TopQueriesLimit int `yaml:"top_queries_limit,omitempty" json:"top_queries_limit,omitempty"`
49
50 charts *module.Charts
51
src/go/plugin/go.d/collector/oracledb/config_schema.json
+8
@@ -31,6 +31,14 @@
31 "minimum": 0.5,
32 "default": 1
33 },
34 + "top_queries_limit": {
35 + "title": "Top Queries Limit",
36 + "description": "Maximum number of queries to return in the top-queries and running-queries function responses.",
37 + "type": "integer",
38 + "minimum": 1,
39 + "maximum": 5000,
40 + "default": 500
41 + },
42 "vnode": {
43 "title": "Vnode",
44 "description": "Associates this data collection job with a [Virtual Node](https://learn.netdata.cloud/docs/netdata-agent/configuration/organize-systems-metrics-and-alerts#virtual-nodes).",
src/go/plugin/go.d/collector/oracledb/functions.go new
+639
@@ -0,0 +1,639 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package oracledb
4 +
5 +import (
6 + "context"
7 + "database/sql"
8 + "fmt"
9 + "strings"
10 +
11 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
12 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
13 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
14 +)
15 +
16 +const oracleMaxQueryTextLength = 4096
17 +
18 +const (
19 + paramSort = "__sort"
20 +
21 + ftString = funcapi.FieldTypeString
22 + ftInteger = funcapi.FieldTypeInteger
23 + ftDuration = funcapi.FieldTypeDuration
24 +
25 + trNone = funcapi.FieldTransformNone
26 + trNumber = funcapi.FieldTransformNumber
27 + trDuration = funcapi.FieldTransformDuration
28 + trText = funcapi.FieldTransformText
29 +
30 + visValue = funcapi.FieldVisualValue
31 + visBar = funcapi.FieldVisualBar
32 +
33 + sortAsc = funcapi.FieldSortAscending
34 + sortDesc = funcapi.FieldSortDescending
35 +
36 + summaryCount = funcapi.FieldSummaryCount
37 + summarySum = funcapi.FieldSummarySum
38 + summaryMax = funcapi.FieldSummaryMax
39 + summaryMean = funcapi.FieldSummaryMean
40 +
41 + filterMulti = funcapi.FieldFilterMultiselect
42 + filterRange = funcapi.FieldFilterRange
43 +)
44 +
45 +type oracleColumnMeta struct {
46 + id string
47 + name string
48 + selectExpr string
49 + dataType funcapi.FieldType
50 + visible bool
51 + sortable bool
52 + fullWidth bool
53 + wrap bool
54 + sticky bool
55 + filter funcapi.FieldFilter
56 + visualization funcapi.FieldVisual
57 + transform funcapi.FieldTransform
58 + units string
59 + decimalPoints int
60 + uniqueKey bool
61 + sortDir funcapi.FieldSort
62 + summary funcapi.FieldSummary
63 + sortLabel string
64 + isSortOption bool
65 + isDefaultSort bool
66 + requiresColumn string
67 + isLabel bool
68 + isPrimary bool
69 + isMetric bool
70 + chartGroup string
71 + chartTitle string
72 + isDefaultChart bool
73 +}
74 +
75 +type oracleTopLayout struct {
76 + cols []oracleColumnMeta
77 + join string
78 +}
79 +
80 +var oracleTopColumns = []oracleColumnMeta{
81 + {id: "sqlId", name: "SQL ID", selectExpr: "s.sql_id", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, uniqueKey: true, sortDir: sortAsc, summary: summaryCount},
82 + {id: "query", name: "Query", selectExpr: "s.sql_text", dataType: ftString, visible: true, sortable: false, filter: filterMulti, transform: trText, sticky: true, fullWidth: true, wrap: true},
83 + {id: "schema", name: "Schema", selectExpr: "s.parsing_schema_name", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount, isLabel: true, isPrimary: true},
84 +
85 + {id: "executions", name: "Executions", selectExpr: "NVL(s.executions, 0)", dataType: ftInteger, visible: true, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by Executions", isMetric: true, chartGroup: "Calls", chartTitle: "Executions", isDefaultChart: true},
86 + {id: "totalTime", name: "Total Time", selectExpr: "NVL(s.elapsed_time, 0) / 1000", dataType: ftDuration, visible: true, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summarySum, isSortOption: true, isDefaultSort: true, sortLabel: "Top queries by Total Time", isMetric: true, chartGroup: "Time", chartTitle: "Execution Time", isDefaultChart: true},
87 + {id: "avgTime", name: "Avg Time", selectExpr: "CASE WHEN NVL(s.executions,0) = 0 THEN 0 ELSE (s.elapsed_time / s.executions) / 1000 END", dataType: ftDuration, visible: true, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMean, isSortOption: true, sortLabel: "Top queries by Avg Time", isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
88 + {id: "cpuTime", name: "CPU Time", selectExpr: "NVL(s.cpu_time, 0) / 1000", dataType: ftDuration, visible: true, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by CPU Time", isMetric: true, chartGroup: "CPU", chartTitle: "CPU Time"},
89 +
90 + {id: "bufferGets", name: "Buffer Gets", selectExpr: "NVL(s.buffer_gets, 0)", dataType: ftInteger, visible: true, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by Buffer Gets", isMetric: true, chartGroup: "IO", chartTitle: "I/O"},
91 + {id: "diskReads", name: "Disk Reads", selectExpr: "NVL(s.disk_reads, 0)", dataType: ftInteger, visible: true, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by Disk Reads", isMetric: true, chartGroup: "IO", chartTitle: "I/O"},
92 + {id: "rowsProcessed", name: "Rows Processed", selectExpr: "NVL(s.rows_processed, 0)", dataType: ftInteger, visible: true, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by Rows Processed", isMetric: true, chartGroup: "Rows", chartTitle: "Rows"},
93 + {id: "parseCalls", name: "Parse Calls", selectExpr: "NVL(s.parse_calls, 0)", dataType: ftInteger, visible: false, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by Parse Calls", isMetric: true, chartGroup: "Parse", chartTitle: "Parse Calls"},
94 +
95 + {id: "module", name: "Module", selectExpr: "s.module", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount, requiresColumn: "MODULE", isLabel: true},
96 + {id: "action", name: "Action", selectExpr: "s.action", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount, requiresColumn: "ACTION", isLabel: true},
97 + {id: "lastActiveTime", name: "Last Active", selectExpr: "TO_CHAR(CAST(s.last_active_time AS TIMESTAMP), 'YYYY-MM-DD\"T\"HH24:MI:SS.FF3')", dataType: ftString, visible: false, sortable: false, filter: filterRange, transform: trText, requiresColumn: "LAST_ACTIVE_TIME"},
98 +}
99 +
100 +var oracleRunningColumns = []oracleColumnMeta{
101 + {id: "sessionId", name: "Session", selectExpr: "s.sid || ',' || s.serial#", dataType: ftString, visible: true, sortable: false, filter: filterMulti, transform: trText, uniqueKey: true, sortDir: sortAsc, summary: summaryCount},
102 + {id: "username", name: "User", selectExpr: "s.username", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
103 + {id: "status", name: "Status", selectExpr: "s.status", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
104 + {id: "type", name: "Type", selectExpr: "s.type", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
105 + {id: "sqlId", name: "SQL ID", selectExpr: "s.sql_id", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
106 + {id: "query", name: "Query", selectExpr: "q.sql_text", dataType: ftString, visible: true, sortable: false, filter: filterMulti, transform: trText, sticky: true, fullWidth: true, wrap: true},
107 + {id: "lastCallMs", name: "Elapsed", selectExpr: "NVL(s.last_call_et, 0) * 1000", dataType: ftDuration, visible: true, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, isSortOption: true, isDefaultSort: true, sortLabel: "Running queries by Elapsed Time"},
108 + {id: "sqlExecStart", name: "SQL Exec Start", selectExpr: "TO_CHAR(CAST(s.sql_exec_start AS TIMESTAMP), 'YYYY-MM-DD\"T\"HH24:MI:SS.FF3')", dataType: ftString, visible: false, sortable: true, filter: filterRange, transform: trText, sortDir: sortDesc, summary: summaryMax},
109 + {id: "module", name: "Module", selectExpr: "s.module", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
110 + {id: "action", name: "Action", selectExpr: "s.action", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
111 + {id: "program", name: "Program", selectExpr: "s.program", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
112 + {id: "machine", name: "Machine", selectExpr: "s.machine", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
113 +}
114 +
115 +func oracleMethods() []module.MethodConfig {
116 + return []module.MethodConfig{
117 + {
118 + ID: "top-queries",
119 + Name: "Top Queries",
120 + Help: "Top SQL statements from V$SQLSTATS. WARNING: Query text may contain unmasked literals (potential PII).",
121 + RequiredParams: []funcapi.ParamConfig{
122 + buildOracleSortParam(oracleTopColumns),
123 + },
124 + },
125 + {
126 + ID: "running-queries",
127 + Name: "Running Queries",
128 + Help: "Currently running SQL statements from V$SESSION. WARNING: Query text may contain unmasked literals (potential PII).",
129 + RequiredParams: []funcapi.ParamConfig{
130 + buildOracleSortParam(oracleRunningColumns),
131 + },
132 + },
133 + }
134 +}
135 +
136 +func oracleMethodParams(ctx context.Context, job *module.Job, method string) ([]funcapi.ParamConfig, error) {
137 + collector, ok := job.Module().(*Collector)
138 + if !ok {
139 + return nil, fmt.Errorf("invalid module type")
140 + }
141 + switch method {
142 + case "top-queries":
143 + cols := oracleTopColumns
144 + if collector.db != nil {
145 + cols = collector.oracleTopLayout(ctx).cols
146 + }
147 + return []funcapi.ParamConfig{buildOracleSortParam(cols)}, nil
148 + case "running-queries":
149 + return []funcapi.ParamConfig{buildOracleSortParam(oracleRunningColumns)}, nil
150 + default:
151 + return nil, fmt.Errorf("unknown method: %s", method)
152 + }
153 +}
154 +
155 +func oracleHandleMethod(ctx context.Context, job *module.Job, method string, params funcapi.ResolvedParams) *module.FunctionResponse {
156 + collector, ok := job.Module().(*Collector)
157 + if !ok {
158 + return &module.FunctionResponse{Status: 500, Message: "internal error: invalid module type"}
159 + }
160 +
161 + if collector.db == nil {
162 + if err := collector.openConnection(); err != nil {
163 + return &module.FunctionResponse{Status: 503, Message: "collector is still initializing, please retry in a few seconds"}
164 + }
165 + }
166 +
167 + switch method {
168 + case "top-queries":
169 + return collector.collectTopQueries(ctx, params.Column(paramSort))
170 + case "running-queries":
171 + return collector.collectRunningQueries(ctx, params.Column(paramSort))
172 + default:
173 + return &module.FunctionResponse{Status: 404, Message: fmt.Sprintf("unknown method: %s", method)}
174 + }
175 +}
176 +
177 +func buildOracleSortParam(cols []oracleColumnMeta) funcapi.ParamConfig {
178 + return funcapi.ParamConfig{
179 + ID: paramSort,
180 + Name: "Filter By",
181 + Help: "Select the primary sort column",
182 + Selection: funcapi.ParamSelect,
183 + Options: buildOracleSortOptions(cols),
184 + UniqueView: true,
185 + }
186 +}
187 +
188 +func buildOracleSortOptions(cols []oracleColumnMeta) []funcapi.ParamOption {
189 + var sortOptions []funcapi.ParamOption
190 + sortDir := funcapi.FieldSortDescending
191 + for _, col := range cols {
192 + if !col.isSortOption {
193 + continue
194 + }
195 + opt := funcapi.ParamOption{
196 + ID: col.id,
197 + Column: col.id,
198 + Name: col.sortLabel,
199 + Sort: &sortDir,
200 + }
201 + if col.isDefaultSort {
202 + opt.Default = true
203 + }
204 + sortOptions = append(sortOptions, opt)
205 + }
206 + return sortOptions
207 +}
208 +
209 +func buildOracleColumns(cols []oracleColumnMeta) map[string]any {
210 + result := make(map[string]any, len(cols))
211 + for i, col := range cols {
212 + visual := visValue
213 + if col.dataType == ftDuration {
214 + visual = visBar
215 + }
216 + colDef := funcapi.Column{
217 + Index: i,
218 + Name: col.name,
219 + Type: col.dataType,
220 + Units: col.units,
221 + Visualization: visual,
222 + Sort: col.sortDir,
223 + Sortable: col.sortable,
224 + Sticky: col.sticky,
225 + Summary: col.summary,
226 + Filter: col.filter,
227 + FullWidth: col.fullWidth,
228 + Wrap: col.wrap,
229 + DefaultExpandedFilter: false,
230 + UniqueKey: col.uniqueKey,
231 + Visible: col.visible,
232 + ValueOptions: funcapi.ValueOptions{
233 + Transform: col.transform,
234 + DecimalPoints: col.decimalPoints,
235 + DefaultValue: nil,
236 + },
237 + }
238 + result[col.id] = colDef.BuildColumn()
239 + }
240 + return result
241 +}
242 +
243 +func buildOracleSelect(cols []oracleColumnMeta) string {
244 + parts := make([]string, 0, len(cols))
245 + for _, col := range cols {
246 + expr := col.selectExpr
247 + if expr == "" {
248 + expr = col.id
249 + }
250 + parts = append(parts, fmt.Sprintf("%s AS %s", expr, col.id))
251 + }
252 + return strings.Join(parts, ", ")
253 +}
254 +
255 +func mapOracleSortColumn(input string, cols []oracleColumnMeta) string {
256 + for _, col := range cols {
257 + if col.isSortOption && col.id == input {
258 + return col.id
259 + }
260 + }
261 + for _, col := range cols {
262 + if col.isDefaultSort {
263 + return col.id
264 + }
265 + }
266 + for _, col := range cols {
267 + if col.isSortOption {
268 + return col.id
269 + }
270 + }
271 + return ""
272 +}
273 +
274 +func (c *Collector) oracleTopLayout(ctx context.Context) oracleTopLayout {
275 + available, err := c.fetchSQLStatsColumns(ctx)
276 + if err != nil {
277 + return oracleTopLayout{cols: filterOracleColumns(oracleTopColumns, nil)}
278 + }
279 + return buildOracleTopLayout(available)
280 +}
281 +
282 +func filterOracleColumns(cols []oracleColumnMeta, available map[string]bool) []oracleColumnMeta {
283 + filtered := make([]oracleColumnMeta, 0, len(cols))
284 + for _, col := range cols {
285 + if col.requiresColumn != "" {
286 + if available == nil {
287 + continue
288 + }
289 + if !available[strings.ToUpper(col.requiresColumn)] {
290 + continue
291 + }
292 + }
293 + filtered = append(filtered, col)
294 + }
295 + return filtered
296 +}
297 +
298 +func buildOracleTopLayout(available map[string]bool) oracleTopLayout {
299 + schemaExpr, schemaJoin := resolveOracleSchemaExpr(available)
300 + filtered := make([]oracleColumnMeta, 0, len(oracleTopColumns))
301 + for _, col := range oracleTopColumns {
302 + if col.id == "schema" {
303 + if schemaExpr == "" {
304 + continue
305 + }
306 + col.selectExpr = schemaExpr
307 + }
308 + if col.requiresColumn != "" && !available[strings.ToUpper(col.requiresColumn)] {
309 + continue
310 + }
311 + filtered = append(filtered, col)
312 + }
313 + return oracleTopLayout{cols: filtered, join: schemaJoin}
314 +}
315 +
316 +func resolveOracleSchemaExpr(available map[string]bool) (string, string) {
317 + switch {
318 + case available["PARSING_SCHEMA_NAME"]:
319 + return "s.parsing_schema_name", ""
320 + case available["PARSING_SCHEMA_ID"]:
321 + return "COALESCE(u.username, TO_CHAR(s.parsing_schema_id))", "LEFT JOIN all_users u ON u.user_id = s.parsing_schema_id"
322 + case available["LAST_EXEC_USER_ID"]:
323 + return "COALESCE(u.username, TO_CHAR(s.last_exec_user_id))", "LEFT JOIN all_users u ON u.user_id = s.last_exec_user_id"
324 + default:
325 + return "", ""
326 + }
327 +}
328 +
329 +func (c *Collector) fetchSQLStatsColumns(ctx context.Context) (map[string]bool, error) {
330 + rows, err := c.db.QueryContext(ctx, "SELECT * FROM v$sqlstats WHERE 1=0")
331 + if err != nil {
332 + return nil, err
333 + }
334 + defer rows.Close()
335 +
336 + names, err := rows.Columns()
337 + if err != nil {
338 + return nil, err
339 + }
340 +
341 + cols := make(map[string]bool, len(names))
342 + for _, name := range names {
343 + cols[strings.ToUpper(name)] = true
344 + }
345 +
346 + return cols, nil
347 +}
348 +
349 +func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
350 + layout := c.oracleTopLayout(ctx)
351 + topCols := layout.cols
352 + limit := c.TopQueriesLimit
353 + if limit <= 0 {
354 + limit = 500
355 + }
356 +
357 + sortColumn = mapOracleSortColumn(sortColumn, topCols)
358 + if sortColumn == "" {
359 + return &module.FunctionResponse{Status: 500, Message: "no sortable columns available"}
360 + }
361 +
362 + joinClause := ""
363 + if layout.join != "" {
364 + joinClause = "\n" + layout.join
365 + }
366 + query := fmt.Sprintf(`
367 +SELECT %s
368 +FROM v$sqlstats s
369 +%s
370 +WHERE NVL(s.executions, 0) > 0
371 +ORDER BY %s DESC NULLS LAST
372 +FETCH FIRST %d ROWS ONLY
373 +`, buildOracleSelect(topCols), joinClause, sortColumn, limit)
374 +
375 + rows, err := c.db.QueryContext(ctx, query)
376 + if err != nil {
377 + if ctx.Err() == context.DeadlineExceeded {
378 + return &module.FunctionResponse{Status: 504, Message: "query timed out"}
379 + }
380 + return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("top queries query failed: %v", err)}
381 + }
382 + defer func() { _ = rows.Close() }()
383 +
384 + data, err := scanOracleRows(rows, topCols)
385 + if err != nil {
386 + return &module.FunctionResponse{Status: 500, Message: err.Error()}
387 + }
388 + if len(data) == 0 {
389 + return &module.FunctionResponse{
390 + Status: 200,
391 + Message: "No SQL statements found.",
392 + Help: "Top SQL statements from V$SQLSTATS",
393 + Columns: buildOracleColumns(topCols),
394 + Data: [][]any{},
395 + DefaultSortColumn: sortColumn,
396 + RequiredParams: []funcapi.ParamConfig{buildOracleSortParam(topCols)},
397 + Charts: oracleTopQueriesCharts(topCols),
398 + DefaultCharts: oracleTopQueriesDefaultCharts(topCols),
399 + GroupBy: oracleTopQueriesGroupBy(topCols),
400 + }
401 + }
402 +
403 + return &module.FunctionResponse{
404 + Status: 200,
405 + Help: "Top SQL statements from V$SQLSTATS. WARNING: Query text may contain unmasked literals (potential PII).",
406 + Columns: buildOracleColumns(topCols),
407 + Data: data,
408 + DefaultSortColumn: sortColumn,
409 + RequiredParams: []funcapi.ParamConfig{buildOracleSortParam(topCols)},
410 + Charts: oracleTopQueriesCharts(topCols),
411 + DefaultCharts: oracleTopQueriesDefaultCharts(topCols),
412 + GroupBy: oracleTopQueriesGroupBy(topCols),
413 + }
414 +}
415 +
416 +func (c *Collector) collectRunningQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
417 + limit := c.TopQueriesLimit
418 + if limit <= 0 {
419 + limit = 500
420 + }
421 +
422 + sortColumn = mapOracleSortColumn(sortColumn, oracleRunningColumns)
423 + if sortColumn == "" {
424 + return &module.FunctionResponse{Status: 500, Message: "no sortable columns available"}
425 + }
426 +
427 + query := fmt.Sprintf(`
428 +SELECT %s
429 +FROM v$session s
430 +LEFT JOIN v$sql q
431 + ON q.sql_id = s.sql_id AND q.child_number = s.sql_child_number
432 +WHERE s.type = 'USER'
433 + AND s.status = 'ACTIVE'
434 + AND s.sql_id IS NOT NULL
435 +ORDER BY %s DESC NULLS LAST
436 +FETCH FIRST %d ROWS ONLY
437 +`, buildOracleSelect(oracleRunningColumns), sortColumn, limit)
438 +
439 + rows, err := c.db.QueryContext(ctx, query)
440 + if err != nil {
441 + if ctx.Err() == context.DeadlineExceeded {
442 + return &module.FunctionResponse{Status: 504, Message: "query timed out"}
443 + }
444 + return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("running queries query failed: %v", err)}
445 + }
446 + defer func() { _ = rows.Close() }()
447 +
448 + data, err := scanOracleRows(rows, oracleRunningColumns)
449 + if err != nil {
450 + return &module.FunctionResponse{Status: 500, Message: err.Error()}
451 + }
452 +
453 + if len(data) == 0 {
454 + return &module.FunctionResponse{
455 + Status: 200,
456 + Message: "No running queries found.",
457 + Help: "Currently running SQL statements from V$SESSION",
458 + Columns: buildOracleColumns(oracleRunningColumns),
459 + Data: [][]any{},
460 + DefaultSortColumn: sortColumn,
461 + RequiredParams: []funcapi.ParamConfig{buildOracleSortParam(oracleRunningColumns)},
462 + }
463 + }
464 +
465 + return &module.FunctionResponse{
466 + Status: 200,
467 + Help: "Currently running SQL statements from V$SESSION. WARNING: Query text may contain unmasked literals (potential PII).",
468 + Columns: buildOracleColumns(oracleRunningColumns),
469 + Data: data,
470 + DefaultSortColumn: sortColumn,
471 + RequiredParams: []funcapi.ParamConfig{buildOracleSortParam(oracleRunningColumns)},
472 + }
473 +}
474 +
475 +func scanOracleRows(rows *sql.Rows, cols []oracleColumnMeta) ([][]any, error) {
476 + data := make([][]any, 0, 500)
477 +
478 + for rows.Next() {
479 + values := make([]any, len(cols))
480 + valuePtrs := make([]any, len(cols))
481 +
482 + for i, col := range cols {
483 + switch col.dataType {
484 + case ftString:
485 + var v sql.NullString
486 + values[i] = &v
487 + case ftInteger:
488 + var v sql.NullInt64
489 + values[i] = &v
490 + case ftDuration:
491 + var v sql.NullFloat64
492 + values[i] = &v
493 + default:
494 + var v any
495 + values[i] = &v
496 + }
497 + valuePtrs[i] = values[i]
498 + }
499 +
500 + if err := rows.Scan(valuePtrs...); err != nil {
501 + return nil, fmt.Errorf("row scan failed: %w", err)
502 + }
503 +
504 + row := make([]any, len(cols))
505 + for i, col := range cols {
506 + switch v := values[i].(type) {
507 + case *sql.NullString:
508 + if v.Valid {
509 + s := v.String
510 + if col.id == "query" {
511 + s = strmutil.TruncateText(s, oracleMaxQueryTextLength)
512 + }
513 + row[i] = s
514 + } else {
515 + row[i] = ""
516 + }
517 + case *sql.NullInt64:
518 + if v.Valid {
519 + row[i] = v.Int64
520 + } else {
521 + row[i] = int64(0)
522 + }
523 + case *sql.NullFloat64:
524 + if v.Valid {
525 + row[i] = v.Float64
526 + } else {
527 + row[i] = float64(0)
528 + }
529 + default:
530 + row[i] = nil
531 + }
532 + }
533 +
534 + data = append(data, row)
535 + }
536 +
537 + if err := rows.Err(); err != nil {
538 + return nil, fmt.Errorf("rows iteration error: %w", err)
539 + }
540 +
541 + return data, nil
542 +}
543 +
544 +func oracleTopQueriesCharts(cols []oracleColumnMeta) map[string]module.ChartConfig {
545 + charts := make(map[string]module.ChartConfig)
546 + for _, col := range cols {
547 + if !col.isMetric || col.chartGroup == "" {
548 + continue
549 + }
550 + cfg, ok := charts[col.chartGroup]
551 + if !ok {
552 + title := col.chartTitle
553 + if title == "" {
554 + title = col.chartGroup
555 + }
556 + cfg = module.ChartConfig{Name: title, Type: "stacked-bar"}
557 + }
558 + cfg.Columns = append(cfg.Columns, col.id)
559 + charts[col.chartGroup] = cfg
560 + }
561 + return charts
562 +}
563 +
564 +func oracleTopQueriesDefaultCharts(cols []oracleColumnMeta) [][]string {
565 + label := primaryOracleLabel(cols)
566 + if label == "" {
567 + return nil
568 + }
569 + chartGroups := defaultOracleChartGroups(cols)
570 + out := make([][]string, 0, len(chartGroups))
571 + for _, group := range chartGroups {
572 + out = append(out, []string{group, label})
573 + }
574 + return out
575 +}
576 +
577 +func oracleTopQueriesGroupBy(cols []oracleColumnMeta) map[string]module.GroupByConfig {
578 + groupBy := make(map[string]module.GroupByConfig)
579 + for _, col := range cols {
580 + if !col.isLabel {
581 + continue
582 + }
583 + groupBy[col.id] = module.GroupByConfig{
584 + Name: "Group by " + col.name,
585 + Columns: []string{col.id},
586 + }
587 + }
588 + return groupBy
589 +}
590 +
591 +func hasOracleColumn(cols []oracleColumnMeta, id string) bool {
592 + for _, col := range cols {
593 + if col.id == id {
594 + return true
595 + }
596 + }
597 + return false
598 +}
599 +
600 +func primaryOracleLabel(cols []oracleColumnMeta) string {
601 + for _, col := range cols {
602 + if col.isPrimary {
603 + return col.id
604 + }
605 + }
606 + for _, col := range cols {
607 + if col.isLabel {
608 + return col.id
609 + }
610 + }
611 + return ""
612 +}
613 +
614 +func defaultOracleChartGroups(cols []oracleColumnMeta) []string {
615 + groups := make([]string, 0)
616 + seen := make(map[string]bool)
617 + for _, col := range cols {
618 + if !col.isMetric || col.chartGroup == "" || !col.isDefaultChart {
619 + continue
620 + }
621 + if !seen[col.chartGroup] {
622 + seen[col.chartGroup] = true
623 + groups = append(groups, col.chartGroup)
624 + }
625 + }
626 + if len(groups) > 0 {
627 + return groups
628 + }
629 + for _, col := range cols {
630 + if !col.isMetric || col.chartGroup == "" {
631 + continue
632 + }
633 + if !seen[col.chartGroup] {
634 + seen[col.chartGroup] = true
635 + groups = append(groups, col.chartGroup)
636 + }
637 + }
638 + return groups
639 +}
src/go/plugin/go.d/collector/oracledb/functions_test.go new
+60
@@ -0,0 +1,60 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package oracledb
4 +
5 +import (
6 + "testing"
7 +
8 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
9 + "github.com/stretchr/testify/assert"
10 +)
11 +
12 +func TestOracleMethods(t *testing.T) {
13 + methods := oracleMethods()
14 +
15 + require := assert.New(t)
16 + require.Len(methods, 2)
17 +
18 + ids := map[string]bool{}
19 + for _, m := range methods {
20 + ids[m.ID] = true
21 + var sortParam *funcapi.ParamConfig
22 + for i := range m.RequiredParams {
23 + if m.RequiredParams[i].ID == "__sort" {
24 + sortParam = &m.RequiredParams[i]
25 + break
26 + }
27 + }
28 + require.NotNil(sortParam, "expected __sort required param for %s", m.ID)
29 + require.NotEmpty(sortParam.Options)
30 + }
31 +
32 + require.True(ids["top-queries"], "top-queries method missing")
33 + require.True(ids["running-queries"], "running-queries method missing")
34 +}
35 +
36 +func TestOracleTopColumns_HasRequiredColumns(t *testing.T) {
37 + required := []string{"query", "executions", "totalTime"}
38 +
39 + uiKeys := make(map[string]bool)
40 + for _, col := range oracleTopColumns {
41 + uiKeys[col.id] = true
42 + }
43 +
44 + for _, key := range required {
45 + assert.True(t, uiKeys[key], "column %s should be defined", key)
46 + }
47 +}
48 +
49 +func TestOracleRunningColumns_HasRequiredColumns(t *testing.T) {
50 + required := []string{"sessionId", "query", "lastCallMs"}
51 +
52 + uiKeys := make(map[string]bool)
53 + for _, col := range oracleRunningColumns {
54 + uiKeys[col.id] = true
55 + }
56 +
57 + for _, key := range required {
58 + assert.True(t, uiKeys[key], "column %s should be defined", key)
59 + }
60 +}
src/go/plugin/go.d/collector/oracledb/integrations/oracle_db.md
+15 -1
@@ -24,6 +24,21 @@ Module: oracledb
24
25 This collector monitors the health and performance of Oracle DB servers and collects general statistics, replication and user metrics.
26
27 +## Functions
28 +
29 +This collector provides the following function methods (useful in the Netdata Functions UI):
30 +
31 +- `top-queries`: Top SQL statements from `V$SQLSTATS` (sorted by a selected metric).
32 +- `running-queries`: Currently running SQL statements from `V$SESSION`.
33 +
34 +**Note:** Query text may contain unmasked literals (potential PII).
35 +Ensure access controls on the Netdata dashboard are appropriate.
36 +
37 +### Required privileges for functions
38 +
39 +The database user must be able to read `V$SQLSTATS` and `V$SESSION`.
40 +Grant `SELECT_CATALOG_ROLE` or explicit `SELECT` on these views.
41 +
42
43 It establishes a connection to the Oracle DB instance via a TCP or UNIX socket and extracts metrics from the following database tables:
44
@@ -350,4 +365,3 @@ If your Netdata runs in a Docker container named "netdata" (replace if different
365 docker logs netdata 2>&1 | grep oracledb
366 ```
367
353 -
src/go/plugin/go.d/collector/oracledb/metadata.yaml
+9
@@ -38,6 +38,8 @@ modules:
38 - `dba_temp_files`
39 - `dba_tablespaces`
40 - `v$temp_space_header`
41 +
42 + It also provides `top-queries` and `running-queries` functions using `V$SQLSTATS` and `V$SESSION`.
43 default_behavior:
44 auto_detection:
45 description: |
@@ -71,6 +73,8 @@ modules:
73 GRANT CONNECT TO netdata;
74 GRANT SELECT_CATALOG_ROLE TO netdata;
75 ```
76 +
77 + The `top-queries` and `running-queries` functions require access to `V$SQLSTATS` and `V$SESSION`.
78 configuration:
79 file:
80 name: go.d/oracledb.conf
@@ -102,6 +106,11 @@ modules:
106 default_value: 1
107 required: false
108 group: Target
109 + - name: top_queries_limit
110 + description: Maximum number of rows returned by the `top-queries` and `running-queries` functions.
111 + default_value: 500
112 + required: false
113 + group: Limits
114
115 - name: vnode
116 description: Associates this data collection job with a [Virtual Node](https://learn.netdata.cloud/docs/netdata-agent/configuration/organize-systems-metrics-and-alerts#virtual-nodes).
src/go/plugin/go.d/collector/postgres/functions.go
+171 -36
@@ -77,6 +77,18 @@ type pgColumnMeta struct {
77 isSticky bool
78 // Whether this column should take full width
79 fullWidth bool
80 + // Whether this column is a label for grouping
81 + isLabel bool
82 + // Whether this label is the primary grouping
83 + isPrimary bool
84 + // Whether this column is a chartable metric
85 + isMetric bool
86 + // Chart group key
87 + chartGroup string
88 + // Chart title
89 + chartTitle string
90 + // Include this chart group in defaults
91 + isDefaultChart bool
92 }
93
94 // pgAllColumns defines ALL possible columns from pg_stat_statements
@@ -151,6 +163,37 @@ var pgAllColumns = []pgColumnMeta{
163 {dbColumn: "s.temp_blk_write_time", uiKey: "tempBlkWriteTime", displayName: "Temp Block Write Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summarySum, filter: filterRange},
164 }
165
166 +type pgChartGroup struct {
167 + key string
168 + title string
169 + columns []string
170 + defaultChart bool
171 +}
172 +
173 +var pgChartGroups = []pgChartGroup{
174 + {key: "Calls", title: "Number of Calls", columns: []string{"calls"}, defaultChart: true},
175 + {key: "Time", title: "Execution Time", columns: []string{"totalTime", "meanTime", "minTime", "maxTime", "stddevTime"}, defaultChart: true},
176 + {key: "PlanTime", title: "Planning Time", columns: []string{"totalPlanTime", "meanPlanTime", "minPlanTime", "maxPlanTime", "stddevPlanTime"}},
177 + {key: "Plans", title: "Plans", columns: []string{"plans"}},
178 + {key: "Rows", title: "Rows Returned", columns: []string{"rows"}},
179 + {key: "SharedBlocks", title: "Shared Blocks", columns: []string{"sharedBlksHit", "sharedBlksRead", "sharedBlksDirtied", "sharedBlksWritten"}},
180 + {key: "LocalBlocks", title: "Local Blocks", columns: []string{"localBlksHit", "localBlksRead", "localBlksDirtied", "localBlksWritten"}},
181 + {key: "TempBlocks", title: "Temp Blocks", columns: []string{"tempBlksRead", "tempBlksWritten"}},
182 + {key: "IOTime", title: "Block I/O Time", columns: []string{"blkReadTime", "blkWriteTime"}},
183 + {key: "WALRecords", title: "WAL Records", columns: []string{"walRecords", "walFpi"}},
184 + {key: "WALBytes", title: "WAL Bytes", columns: []string{"walBytes"}},
185 + {key: "JITCounts", title: "JIT Counts", columns: []string{"jitFunctions", "jitInliningCount", "jitOptimizationCount", "jitEmissionCount"}},
186 + {key: "JITTime", title: "JIT Time", columns: []string{"jitGenerationTime", "jitInliningTime", "jitOptimizationTime", "jitEmissionTime"}},
187 + {key: "TempIOTime", title: "Temp Block I/O Time", columns: []string{"tempBlkReadTime", "tempBlkWriteTime"}},
188 +}
189 +
190 +var pgLabelColumns = map[string]bool{
191 + "database": true,
192 + "user": true,
193 +}
194 +
195 +const pgPrimaryLabel = "database"
196 +
197 // pgMethods returns the available function methods for PostgreSQL
198 // Sort options are built dynamically based on available columns
199 func pgMethods() []module.MethodConfig {
@@ -307,6 +350,8 @@ func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *m
350 defaultSort = sortOptions[0].ID
351 }
352
353 + annotatedCols := decoratePgColumns(queryCols)
354 +
355 return &module.FunctionResponse{
356 Status: 200,
357 Help: "Top SQL queries from pg_stat_statements",
@@ -316,43 +361,133 @@ func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *m
361 RequiredParams: []funcapi.ParamConfig{sortParam},
362
363 // Charts for aggregated visualization
319 - Charts: map[string]module.ChartConfig{
320 - "Calls": {
321 - Name: "Number of Calls",
322 - Type: "stacked-bar",
323 - Columns: []string{"calls"},
324 - },
325 - "Time": {
326 - Name: "Execution Time",
327 - Type: "stacked-bar",
328 - Columns: []string{"totalTime", "meanTime"},
329 - },
330 - "Rows": {
331 - Name: "Rows Returned",
332 - Type: "stacked-bar",
333 - Columns: []string{"rows"},
334 - },
335 - "IO": {
336 - Name: "Block I/O",
337 - Type: "stacked-bar",
338 - Columns: []string{"sharedBlksHit", "sharedBlksRead"},
339 - },
340 - },
341 - DefaultCharts: [][]string{
342 - {"Time", "database"},
343 - {"Calls", "database"},
344 - },
345 - GroupBy: map[string]module.GroupByConfig{
346 - "database": {
347 - Name: "Group by Database",
348 - Columns: []string{"database"},
349 - },
350 - "user": {
351 - Name: "Group by User",
352 - Columns: []string{"user"},
353 - },
354 - },
364 + Charts: pgTopQueriesCharts(annotatedCols),
365 + DefaultCharts: pgTopQueriesDefaultCharts(annotatedCols),
366 + GroupBy: pgTopQueriesGroupBy(annotatedCols),
367 + }
368 +}
369 +
370 +func decoratePgColumns(cols []pgColumnMeta) []pgColumnMeta {
371 + out := make([]pgColumnMeta, len(cols))
372 + index := make(map[string]int, len(cols))
373 + for i, col := range cols {
374 + out[i] = col
375 + index[col.uiKey] = i
376 + }
377 +
378 + for i := range out {
379 + if pgLabelColumns[out[i].uiKey] {
380 + out[i].isLabel = true
381 + if out[i].uiKey == pgPrimaryLabel {
382 + out[i].isPrimary = true
383 + }
384 + }
385 + }
386 +
387 + for _, group := range pgChartGroups {
388 + for _, key := range group.columns {
389 + idx, ok := index[key]
390 + if !ok {
391 + continue
392 + }
393 + out[idx].isMetric = true
394 + out[idx].chartGroup = group.key
395 + out[idx].chartTitle = group.title
396 + if group.defaultChart {
397 + out[idx].isDefaultChart = true
398 + }
399 + }
400 + }
401 +
402 + return out
403 +}
404 +
405 +func pgTopQueriesCharts(cols []pgColumnMeta) map[string]module.ChartConfig {
406 + charts := make(map[string]module.ChartConfig)
407 + for _, col := range cols {
408 + if !col.isMetric || col.chartGroup == "" {
409 + continue
410 + }
411 + cfg, ok := charts[col.chartGroup]
412 + if !ok {
413 + title := col.chartTitle
414 + if title == "" {
415 + title = col.chartGroup
416 + }
417 + cfg = module.ChartConfig{Name: title, Type: "stacked-bar"}
418 + }
419 + cfg.Columns = append(cfg.Columns, col.uiKey)
420 + charts[col.chartGroup] = cfg
421 + }
422 + return charts
423 +}
424 +
425 +func pgTopQueriesDefaultCharts(cols []pgColumnMeta) [][]string {
426 + label := primaryPgLabel(cols)
427 + if label == "" {
428 + return nil
429 + }
430 + chartGroups := defaultPgChartGroups(cols)
431 + out := make([][]string, 0, len(chartGroups))
432 + for _, group := range chartGroups {
433 + out = append(out, []string{group, label})
434 + }
435 + return out
436 +}
437 +
438 +func pgTopQueriesGroupBy(cols []pgColumnMeta) map[string]module.GroupByConfig {
439 + groupBy := make(map[string]module.GroupByConfig)
440 + for _, col := range cols {
441 + if !col.isLabel {
442 + continue
443 + }
444 + groupBy[col.uiKey] = module.GroupByConfig{
445 + Name: "Group by " + col.displayName,
446 + Columns: []string{col.uiKey},
447 + }
448 + }
449 + return groupBy
450 +}
451 +
452 +func primaryPgLabel(cols []pgColumnMeta) string {
453 + for _, col := range cols {
454 + if col.isPrimary {
455 + return col.uiKey
456 + }
457 + }
458 + for _, col := range cols {
459 + if col.isLabel {
460 + return col.uiKey
461 + }
462 + }
463 + return ""
464 +}
465 +
466 +func defaultPgChartGroups(cols []pgColumnMeta) []string {
467 + groups := make([]string, 0)
468 + seen := make(map[string]bool)
469 + for _, col := range cols {
470 + if !col.isMetric || col.chartGroup == "" || !col.isDefaultChart {
471 + continue
472 + }
473 + if !seen[col.chartGroup] {
474 + seen[col.chartGroup] = true
475 + groups = append(groups, col.chartGroup)
476 + }
477 + }
478 + if len(groups) > 0 {
479 + return groups
480 + }
481 + for _, col := range cols {
482 + if !col.isMetric || col.chartGroup == "" {
483 + continue
484 + }
485 + if !seen[col.chartGroup] {
486 + seen[col.chartGroup] = true
487 + groups = append(groups, col.chartGroup)
488 + }
489 }
490 + return groups
491 }
492
493 // detectPgStatStatementsColumns queries the database to find available columns
src/go/plugin/go.d/collector/proxysql/collector.go
+7
@@ -24,6 +24,9 @@ func init() {
24 JobConfigSchema: configSchema,
25 Create: func() module.Module { return New() },
26 Config: func() any { return &Config{} },
27 + Methods: proxysqlMethods,
28 + MethodParams: proxysqlMethodParams,
29 + HandleMethod: proxysqlHandleMethod,
30 })
31 }
32
@@ -51,6 +54,7 @@ type Config struct {
54 AutoDetectionRetry int `yaml:"autodetection_retry,omitempty" json:"autodetection_retry"`
55 DSN string `yaml:"dsn" json:"dsn"`
56 Timeout confopt.Duration `yaml:"timeout,omitempty" json:"timeout"`
57 + TopQueriesLimit int `yaml:"top_queries_limit,omitempty" json:"top_queries_limit,omitempty"`
58 }
59
60 type Collector struct {
@@ -63,6 +67,9 @@ type Collector struct {
67
68 once *sync.Once
69 cache *cache
70 +
71 + queryDigestCols map[string]bool
72 + queryDigestColsMu sync.RWMutex
73 }
74
75 func (c *Collector) Configuration() any {
src/go/plugin/go.d/collector/proxysql/config_schema.json
+8
@@ -31,6 +31,14 @@
31 "minimum": 0.5,
32 "default": 1
33 },
34 + "top_queries_limit": {
35 + "title": "Top Queries Limit",
36 + "description": "Maximum number of queries to return in the top-queries function response.",
37 + "type": "integer",
38 + "minimum": 1,
39 + "maximum": 5000,
40 + "default": 500
41 + },
42 "vnode": {
43 "title": "Vnode",
44 "description": "Associates this data collection job with a [Virtual Node](https://learn.netdata.cloud/docs/netdata-agent/configuration/organize-systems-metrics-and-alerts#virtual-nodes).",
src/go/plugin/go.d/collector/proxysql/functions.go new
+523
@@ -0,0 +1,523 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package proxysql
4 +
5 +import (
6 + "context"
7 + "database/sql"
8 + "fmt"
9 + "strings"
10 +
11 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
12 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
13 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
14 +)
15 +
16 +const proxysqlMaxQueryTextLength = 4096
17 +
18 +const (
19 + paramSort = "__sort"
20 +
21 + ftString = funcapi.FieldTypeString
22 + ftInteger = funcapi.FieldTypeInteger
23 + ftDuration = funcapi.FieldTypeDuration
24 +
25 + trNone = funcapi.FieldTransformNone
26 + trNumber = funcapi.FieldTransformNumber
27 + trDuration = funcapi.FieldTransformDuration
28 +
29 + sortAsc = funcapi.FieldSortAscending
30 + sortDesc = funcapi.FieldSortDescending
31 +
32 + summaryCount = funcapi.FieldSummaryCount
33 + summarySum = funcapi.FieldSummarySum
34 + summaryMin = funcapi.FieldSummaryMin
35 + summaryMax = funcapi.FieldSummaryMax
36 + summaryMean = funcapi.FieldSummaryMean
37 +
38 + filterMulti = funcapi.FieldFilterMultiselect
39 + filterRange = funcapi.FieldFilterRange
40 +)
41 +
42 +type proxysqlColumnMeta struct {
43 + dbColumn string
44 + uiKey string
45 + displayName string
46 + dataType funcapi.FieldType
47 + units string
48 + visible bool
49 + transform funcapi.FieldTransform
50 + decimalPoints int
51 + sortDir funcapi.FieldSort
52 + summary funcapi.FieldSummary
53 + filter funcapi.FieldFilter
54 + isMicroseconds bool
55 + isSortOption bool
56 + sortLabel string
57 + isDefaultSort bool
58 + isUniqueKey bool
59 + isSticky bool
60 + fullWidth bool
61 + isLabel bool
62 + isPrimary bool
63 + isMetric bool
64 + chartGroup string
65 + chartTitle string
66 + isDefaultChart bool
67 +}
68 +
69 +var proxysqlAllColumns = []proxysqlColumnMeta{
70 + {dbColumn: "digest", uiKey: "digest", displayName: "Digest", dataType: ftString, visible: false, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti, isUniqueKey: true},
71 + {dbColumn: "digest_text", uiKey: "query", displayName: "Query", dataType: ftString, visible: true, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti, isSticky: true, fullWidth: true},
72 + {dbColumn: "schemaname", uiKey: "schema", displayName: "Schema", dataType: ftString, visible: true, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti, isLabel: true, isPrimary: true},
73 + {dbColumn: "username", uiKey: "user", displayName: "User", dataType: ftString, visible: false, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti, isLabel: true},
74 + {dbColumn: "hostgroup", uiKey: "hostgroup", displayName: "Hostgroup", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortAsc, summary: summaryCount, filter: filterRange, isLabel: true},
75 +
76 + {dbColumn: "count_star", uiKey: "calls", displayName: "Calls", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Number of Calls", isMetric: true, chartGroup: "Calls", chartTitle: "Number of Calls", isDefaultChart: true},
77 +
78 + {dbColumn: "sum_time", uiKey: "totalTime", displayName: "Total Time", dataType: ftDuration, units: "milliseconds", visible: true, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summarySum, filter: filterRange, isMicroseconds: true, isSortOption: true, sortLabel: "Total Execution Time", isDefaultSort: true, isMetric: true, chartGroup: "Time", chartTitle: "Execution Time", isDefaultChart: true},
79 + {dbColumn: "avg_time", uiKey: "avgTime", displayName: "Avg Time", dataType: ftDuration, units: "milliseconds", visible: true, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMean, filter: filterRange, isMicroseconds: true, isSortOption: true, sortLabel: "Average Execution Time", isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
80 + {dbColumn: "min_time", uiKey: "minTime", displayName: "Min Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMin, filter: filterRange, isMicroseconds: true, isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
81 + {dbColumn: "max_time", uiKey: "maxTime", displayName: "Max Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, filter: filterRange, isMicroseconds: true, isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
82 +
83 + {dbColumn: "sum_rows_affected", uiKey: "rowsAffected", displayName: "Rows Affected", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Rows Affected", isMetric: true, chartGroup: "Rows", chartTitle: "Rows"},
84 + {dbColumn: "sum_rows_sent", uiKey: "rowsSent", displayName: "Rows Sent", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Rows Sent", isMetric: true, chartGroup: "Rows", chartTitle: "Rows"},
85 + {dbColumn: "sum_errors", uiKey: "errors", displayName: "Errors", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Errors", isMetric: true, chartGroup: "Errors", chartTitle: "Errors & Warnings"},
86 + {dbColumn: "sum_warnings", uiKey: "warnings", displayName: "Warnings", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Warnings", isMetric: true, chartGroup: "Errors", chartTitle: "Errors & Warnings"},
87 +
88 + {dbColumn: "first_seen", uiKey: "firstSeen", displayName: "First Seen", dataType: ftString, visible: false, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti},
89 + {dbColumn: "last_seen", uiKey: "lastSeen", displayName: "Last Seen", dataType: ftString, visible: false, transform: trNone, sortDir: sortDesc, summary: summaryCount, filter: filterMulti},
90 +}
91 +
92 +func proxysqlMethods() []module.MethodConfig {
93 + sortOptions := buildProxySQLSortOptions(proxysqlAllColumns)
94 + return []module.MethodConfig{{
95 + ID: "top-queries",
96 + Name: "Top Queries",
97 + Help: "Top SQL queries from ProxySQL query digest stats",
98 + RequiredParams: []funcapi.ParamConfig{{
99 + ID: paramSort,
100 + Name: "Filter By",
101 + Help: "Select the primary sort column",
102 + Selection: funcapi.ParamSelect,
103 + Options: sortOptions,
104 + UniqueView: true,
105 + }},
106 + }}
107 +}
108 +
109 +func proxysqlMethodParams(ctx context.Context, job *module.Job, method string) ([]funcapi.ParamConfig, error) {
110 + collector, ok := job.Module().(*Collector)
111 + if !ok {
112 + return nil, fmt.Errorf("invalid module type")
113 + }
114 + if collector.db == nil {
115 + if err := collector.openConnection(); err != nil {
116 + return nil, err
117 + }
118 + }
119 + switch method {
120 + case "top-queries":
121 + return collector.topQueriesParams(ctx)
122 + default:
123 + return nil, fmt.Errorf("unknown method: %s", method)
124 + }
125 +}
126 +
127 +func proxysqlHandleMethod(ctx context.Context, job *module.Job, method string, params funcapi.ResolvedParams) *module.FunctionResponse {
128 + collector, ok := job.Module().(*Collector)
129 + if !ok {
130 + return &module.FunctionResponse{Status: 500, Message: "internal error: invalid module type"}
131 + }
132 +
133 + if collector.db == nil {
134 + if err := collector.openConnection(); err != nil {
135 + return &module.FunctionResponse{Status: 503, Message: fmt.Sprintf("failed to open connection: %v", err)}
136 + }
137 + }
138 +
139 + switch method {
140 + case "top-queries":
141 + return collector.collectTopQueries(ctx, params.Column(paramSort))
142 + default:
143 + return &module.FunctionResponse{Status: 404, Message: fmt.Sprintf("unknown method: %s", method)}
144 + }
145 +}
146 +
147 +func buildProxySQLSortOptions(cols []proxysqlColumnMeta) []funcapi.ParamOption {
148 + var sortOptions []funcapi.ParamOption
149 + sortDir := funcapi.FieldSortDescending
150 + for _, col := range cols {
151 + if col.isSortOption {
152 + sortOptions = append(sortOptions, funcapi.ParamOption{
153 + ID: col.uiKey,
154 + Column: col.uiKey,
155 + Name: "Top queries by " + col.sortLabel,
156 + Default: col.isDefaultSort,
157 + Sort: &sortDir,
158 + })
159 + }
160 + }
161 + return sortOptions
162 +}
163 +
164 +func buildProxySQLColumns(cols []proxysqlColumnMeta) map[string]any {
165 + columns := make(map[string]any, len(cols))
166 + for i, col := range cols {
167 + visual := funcapi.FieldVisualValue
168 + if col.dataType == ftDuration {
169 + visual = funcapi.FieldVisualBar
170 + }
171 + colDef := funcapi.Column{
172 + Index: i,
173 + Name: col.displayName,
174 + Type: col.dataType,
175 + Units: col.units,
176 + Visualization: visual,
177 + Sort: col.sortDir,
178 + Sortable: true,
179 + Sticky: col.isSticky,
180 + Summary: col.summary,
181 + Filter: col.filter,
182 + FullWidth: col.fullWidth,
183 + Wrap: false,
184 + DefaultExpandedFilter: false,
185 + UniqueKey: col.isUniqueKey,
186 + Visible: col.visible,
187 + ValueOptions: funcapi.ValueOptions{
188 + Transform: col.transform,
189 + DecimalPoints: col.decimalPoints,
190 + DefaultValue: nil,
191 + },
192 + }
193 + columns[col.uiKey] = colDef.BuildColumn()
194 + }
195 + return columns
196 +}
197 +
198 +func (c *Collector) detectProxySQLDigestColumns(ctx context.Context) (map[string]bool, error) {
199 + c.queryDigestColsMu.RLock()
200 + if c.queryDigestCols != nil {
201 + cols := c.queryDigestCols
202 + c.queryDigestColsMu.RUnlock()
203 + return cols, nil
204 + }
205 + c.queryDigestColsMu.RUnlock()
206 +
207 + c.queryDigestColsMu.Lock()
208 + defer c.queryDigestColsMu.Unlock()
209 + if c.queryDigestCols != nil {
210 + return c.queryDigestCols, nil
211 + }
212 +
213 + rows, err := c.db.QueryContext(ctx, "SELECT * FROM stats_mysql_query_digest WHERE 1=0")
214 + if err != nil {
215 + return nil, fmt.Errorf("failed to query stats_mysql_query_digest columns: %w", err)
216 + }
217 + defer rows.Close()
218 +
219 + names, err := rows.Columns()
220 + if err != nil {
221 + return nil, fmt.Errorf("failed to read columns: %w", err)
222 + }
223 +
224 + cols := make(map[string]bool, len(names))
225 + for _, name := range names {
226 + cols[strings.ToLower(name)] = true
227 + }
228 + c.queryDigestCols = cols
229 + return cols, nil
230 +}
231 +
232 +func (c *Collector) buildAvailableProxySQLColumns(available map[string]bool) []proxysqlColumnMeta {
233 + var cols []proxysqlColumnMeta
234 + for _, col := range proxysqlAllColumns {
235 + if col.dbColumn == "" || available[strings.ToLower(col.dbColumn)] {
236 + cols = append(cols, col)
237 + }
238 + }
239 + return cols
240 +}
241 +
242 +func (c *Collector) topQueriesParams(ctx context.Context) ([]funcapi.ParamConfig, error) {
243 + availableCols, err := c.detectProxySQLDigestColumns(ctx)
244 + if err != nil {
245 + return nil, err
246 + }
247 + cols := c.buildAvailableProxySQLColumns(availableCols)
248 + if len(cols) == 0 {
249 + return nil, fmt.Errorf("no columns available in stats_mysql_query_digest")
250 + }
251 + sortParam := funcapi.ParamConfig{
252 + ID: paramSort,
253 + Name: "Filter By",
254 + Help: "Select the primary sort column",
255 + Selection: funcapi.ParamSelect,
256 + Options: buildProxySQLSortOptions(cols),
257 + UniqueView: true,
258 + }
259 + return []funcapi.ParamConfig{sortParam}, nil
260 +}
261 +
262 +func (c *Collector) mapAndValidateProxySQLSortColumn(input string, available []proxysqlColumnMeta) string {
263 + availableKeys := make(map[string]bool, len(available))
264 + for _, col := range available {
265 + availableKeys[col.uiKey] = true
266 + }
267 + if availableKeys[input] {
268 + return input
269 + }
270 + if availableKeys["totalTime"] {
271 + return "totalTime"
272 + }
273 + if availableKeys["calls"] {
274 + return "calls"
275 + }
276 + return available[0].uiKey
277 +}
278 +
279 +func (c *Collector) buildProxySQLDynamicSQL(cols []proxysqlColumnMeta, sortColumn string, limit int) string {
280 + selectParts := make([]string, 0, len(cols))
281 + for _, col := range cols {
282 + expr := col.dbColumn
283 + if col.isMicroseconds {
284 + expr = fmt.Sprintf("%s/1000", col.dbColumn)
285 + }
286 + selectParts = append(selectParts, fmt.Sprintf("%s AS `%s`", expr, col.uiKey))
287 + }
288 +
289 + return fmt.Sprintf(`
290 +SELECT %s
291 +FROM stats_mysql_query_digest
292 +ORDER BY `+"`%s`"+` DESC
293 +LIMIT %d
294 +`, strings.Join(selectParts, ", "), sortColumn, limit)
295 +}
296 +
297 +func (c *Collector) scanProxySQLDynamicRows(rows *sql.Rows, cols []proxysqlColumnMeta) ([][]any, error) {
298 + data := make([][]any, 0, 500)
299 +
300 + valuePtrs := make([]any, len(cols))
301 + values := make([]any, len(cols))
302 +
303 + for rows.Next() {
304 + for i, col := range cols {
305 + switch col.dataType {
306 + case ftString:
307 + var v sql.NullString
308 + values[i] = &v
309 + case ftInteger:
310 + var v sql.NullInt64
311 + values[i] = &v
312 + case ftDuration:
313 + var v sql.NullFloat64
314 + values[i] = &v
315 + default:
316 + var v any
317 + values[i] = &v
318 + }
319 + valuePtrs[i] = values[i]
320 + }
321 +
322 + if err := rows.Scan(valuePtrs...); err != nil {
323 + return nil, fmt.Errorf("row scan failed: %w", err)
324 + }
325 +
326 + row := make([]any, len(cols))
327 + for i, col := range cols {
328 + switch v := values[i].(type) {
329 + case *sql.NullString:
330 + if v.Valid {
331 + s := v.String
332 + if col.uiKey == "query" {
333 + s = strmutil.TruncateText(s, proxysqlMaxQueryTextLength)
334 + }
335 + row[i] = s
336 + } else {
337 + row[i] = ""
338 + }
339 + case *sql.NullInt64:
340 + if v.Valid {
341 + row[i] = v.Int64
342 + } else {
343 + row[i] = int64(0)
344 + }
345 + case *sql.NullFloat64:
346 + if v.Valid {
347 + row[i] = v.Float64
348 + } else {
349 + row[i] = float64(0)
350 + }
351 + default:
352 + row[i] = nil
353 + }
354 + }
355 + data = append(data, row)
356 + }
357 +
358 + if err := rows.Err(); err != nil {
359 + return nil, fmt.Errorf("rows iteration error: %w", err)
360 + }
361 +
362 + return data, nil
363 +}
364 +
365 +func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
366 + availableCols, err := c.detectProxySQLDigestColumns(ctx)
367 + if err != nil {
368 + return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("failed to detect available columns: %v", err)}
369 + }
370 +
371 + cols := c.buildAvailableProxySQLColumns(availableCols)
372 + if len(cols) == 0 {
373 + return &module.FunctionResponse{Status: 500, Message: "no columns available in stats_mysql_query_digest"}
374 + }
375 +
376 + sortColumn = c.mapAndValidateProxySQLSortColumn(sortColumn, cols)
377 +
378 + limit := c.TopQueriesLimit
379 + if limit <= 0 {
380 + limit = 500
381 + }
382 +
383 + query := c.buildProxySQLDynamicSQL(cols, sortColumn, limit)
384 + rows, err := c.db.QueryContext(ctx, query)
385 + if err != nil {
386 + if ctx.Err() == context.DeadlineExceeded {
387 + return &module.FunctionResponse{Status: 504, Message: "query timed out"}
388 + }
389 + return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("query failed: %v", err)}
390 + }
391 + defer rows.Close()
392 +
393 + data, err := c.scanProxySQLDynamicRows(rows, cols)
394 + if err != nil {
395 + return &module.FunctionResponse{Status: 500, Message: err.Error()}
396 + }
397 +
398 + sortParam := funcapi.ParamConfig{
399 + ID: paramSort,
400 + Name: "Filter By",
401 + Help: "Select the primary sort column",
402 + Selection: funcapi.ParamSelect,
403 + Options: buildProxySQLSortOptions(cols),
404 + UniqueView: true,
405 + }
406 +
407 + defaultSort := "totalTime"
408 + if !containsProxySQLColumn(cols, defaultSort) {
409 + defaultSort = "calls"
410 + }
411 +
412 + return &module.FunctionResponse{
413 + Status: 200,
414 + Help: "Top SQL queries from ProxySQL stats_mysql_query_digest",
415 + Columns: buildProxySQLColumns(cols),
416 + Data: data,
417 + DefaultSortColumn: defaultSort,
418 + RequiredParams: []funcapi.ParamConfig{sortParam},
419 + Charts: proxysqlTopQueriesCharts(cols),
420 + DefaultCharts: proxysqlTopQueriesDefaultCharts(cols),
421 + GroupBy: proxysqlTopQueriesGroupBy(cols),
422 + }
423 +}
424 +
425 +func containsProxySQLColumn(cols []proxysqlColumnMeta, key string) bool {
426 + for _, col := range cols {
427 + if col.uiKey == key {
428 + return true
429 + }
430 + }
431 + return false
432 +}
433 +
434 +func proxysqlTopQueriesCharts(cols []proxysqlColumnMeta) map[string]module.ChartConfig {
435 + charts := make(map[string]module.ChartConfig)
436 + for _, col := range cols {
437 + if !col.isMetric || col.chartGroup == "" {
438 + continue
439 + }
440 + cfg, ok := charts[col.chartGroup]
441 + if !ok {
442 + title := col.chartTitle
443 + if title == "" {
444 + title = col.chartGroup
445 + }
446 + cfg = module.ChartConfig{
447 + Name: title,
448 + Type: "stacked-bar",
449 + }
450 + }
451 + cfg.Columns = append(cfg.Columns, col.uiKey)
452 + charts[col.chartGroup] = cfg
453 + }
454 + return charts
455 +}
456 +
457 +func proxysqlTopQueriesDefaultCharts(cols []proxysqlColumnMeta) [][]string {
458 + label := primaryProxySQLLabel(cols)
459 + if label == "" {
460 + return nil
461 + }
462 + chartGroups := defaultProxySQLChartGroups(cols)
463 + out := make([][]string, 0, len(chartGroups))
464 + for _, group := range chartGroups {
465 + out = append(out, []string{group, label})
466 + }
467 + return out
468 +}
469 +
470 +func proxysqlTopQueriesGroupBy(cols []proxysqlColumnMeta) map[string]module.GroupByConfig {
471 + groupBy := make(map[string]module.GroupByConfig)
472 + for _, col := range cols {
473 + if !col.isLabel {
474 + continue
475 + }
476 + groupBy[col.uiKey] = module.GroupByConfig{
477 + Name: "Group by " + col.displayName,
478 + Columns: []string{col.uiKey},
479 + }
480 + }
481 + return groupBy
482 +}
483 +
484 +func primaryProxySQLLabel(cols []proxysqlColumnMeta) string {
485 + for _, col := range cols {
486 + if col.isPrimary {
487 + return col.uiKey
488 + }
489 + }
490 + for _, col := range cols {
491 + if col.isLabel {
492 + return col.uiKey
493 + }
494 + }
495 + return ""
496 +}
497 +
498 +func defaultProxySQLChartGroups(cols []proxysqlColumnMeta) []string {
499 + groups := make([]string, 0)
500 + seen := make(map[string]bool)
501 + for _, col := range cols {
502 + if !col.isMetric || col.chartGroup == "" || !col.isDefaultChart {
503 + continue
504 + }
505 + if !seen[col.chartGroup] {
506 + seen[col.chartGroup] = true
507 + groups = append(groups, col.chartGroup)
508 + }
509 + }
510 + if len(groups) > 0 {
511 + return groups
512 + }
513 + for _, col := range cols {
514 + if !col.isMetric || col.chartGroup == "" {
515 + continue
516 + }
517 + if !seen[col.chartGroup] {
518 + seen[col.chartGroup] = true
519 + groups = append(groups, col.chartGroup)
520 + }
521 + }
522 + return groups
523 +}
src/go/plugin/go.d/collector/proxysql/functions_test.go new
+92
@@ -0,0 +1,92 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package proxysql
4 +
5 +import (
6 + "testing"
7 +
8 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
9 + "github.com/stretchr/testify/assert"
10 +)
11 +
12 +func TestProxySQLMethods(t *testing.T) {
13 + methods := proxysqlMethods()
14 +
15 + require := assert.New(t)
16 + require.Len(methods, 1)
17 + require.Equal("top-queries", methods[0].ID)
18 + require.Equal("Top Queries", methods[0].Name)
19 + require.NotEmpty(methods[0].RequiredParams)
20 +
21 + var sortParam *funcapi.ParamConfig
22 + for i := range methods[0].RequiredParams {
23 + if methods[0].RequiredParams[i].ID == "__sort" {
24 + sortParam = &methods[0].RequiredParams[i]
25 + break
26 + }
27 + }
28 + require.NotNil(sortParam, "expected __sort required param")
29 + require.NotEmpty(sortParam.Options)
30 +}
31 +
32 +func TestProxySQLAllColumns_HasRequiredColumns(t *testing.T) {
33 + required := []string{"digest", "query", "calls", "totalTime"}
34 +
35 + uiKeys := make(map[string]bool)
36 + for _, col := range proxysqlAllColumns {
37 + uiKeys[col.uiKey] = true
38 + }
39 +
40 + for _, key := range required {
41 + assert.True(t, uiKeys[key], "column %s should be defined", key)
42 + }
43 +}
44 +
45 +func TestCollector_mapAndValidateProxySQLSortColumn(t *testing.T) {
46 + tests := map[string]struct {
47 + available []proxysqlColumnMeta
48 + input string
49 + expected string
50 + }{
51 + "valid totalTime": {
52 + available: []proxysqlColumnMeta{{uiKey: "totalTime"}, {uiKey: "calls"}},
53 + input: "totalTime",
54 + expected: "totalTime",
55 + },
56 + "invalid falls back to totalTime": {
57 + available: []proxysqlColumnMeta{{uiKey: "totalTime"}, {uiKey: "calls"}},
58 + input: "bad",
59 + expected: "totalTime",
60 + },
61 + "fallback to calls": {
62 + available: []proxysqlColumnMeta{{uiKey: "calls"}},
63 + input: "bad",
64 + expected: "calls",
65 + },
66 + }
67 +
68 + for name, tc := range tests {
69 + t.Run(name, func(t *testing.T) {
70 + c := &Collector{}
71 + assert.Equal(t, tc.expected, c.mapAndValidateProxySQLSortColumn(tc.input, tc.available))
72 + })
73 + }
74 +}
75 +
76 +func TestCollector_buildProxySQLDynamicSQL(t *testing.T) {
77 + c := &Collector{}
78 +
79 + cols := []proxysqlColumnMeta{
80 + {dbColumn: "digest", uiKey: "digest", dataType: ftString},
81 + {dbColumn: "digest_text", uiKey: "query", dataType: ftString},
82 + {dbColumn: "count_star", uiKey: "calls", dataType: ftInteger},
83 + {dbColumn: "sum_time", uiKey: "totalTime", dataType: ftDuration, isMicroseconds: true},
84 + }
85 +
86 + sql := c.buildProxySQLDynamicSQL(cols, "totalTime", 500)
87 +
88 + assert.Contains(t, sql, "stats_mysql_query_digest")
89 + assert.Contains(t, sql, "ORDER BY `totalTime` DESC")
90 + assert.Contains(t, sql, "LIMIT 500")
91 + assert.Contains(t, sql, "sum_time/1000 AS `totalTime`")
92 +}
src/go/plugin/go.d/collector/redis/collector.go
+5
@@ -33,6 +33,9 @@ func init() {
33 JobConfigSchema: configSchema,
34 Create: func() module.Module { return New() },
35 Config: func() any { return &Config{} },
36 + Methods: redisMethods,
37 + MethodParams: redisMethodParams,
38 + HandleMethod: redisHandleMethod,
39 })
40 }
41
@@ -62,6 +65,7 @@ type Config struct {
65 Password string `yaml:"password,omitempty" json:"password"`
66 tlscfg.TLSConfig `yaml:",inline" json:""`
67 PingSamples int `yaml:"ping_samples" json:"ping_samples"`
68 + TopQueriesLimit int `yaml:"top_queries_limit,omitempty" json:"top_queries_limit,omitempty"`
69 }
70
71 type (
@@ -84,6 +88,7 @@ type (
88 redisClient interface {
89 Info(ctx context.Context, section ...string) *redis.StringCmd
90 Ping(context.Context) *redis.StatusCmd
91 + SlowLogGet(ctx context.Context, num int64) *redis.SlowLogCmd
92 Close() error
93 }
94 )
src/go/plugin/go.d/collector/redis/collector_test.go
+6
@@ -395,6 +395,12 @@ func (m *mockRedisClient) Ping(_ context.Context) (cmd *redis.StatusCmd) {
395 return redis.NewStatusResult("PONG", nil)
396 }
397
398 +func (m *mockRedisClient) SlowLogGet(ctx context.Context, num int64) *redis.SlowLogCmd {
399 + cmd := redis.NewSlowLogCmd(ctx, "slowlog", "get", num)
400 + cmd.SetVal([]redis.SlowLog{})
401 + return cmd
402 +}
403 +
404 func (m *mockRedisClient) Close() error {
405 m.calledClose = true
406 return nil
src/go/plugin/go.d/collector/redis/config_schema.json
+9
@@ -38,6 +38,14 @@
38 "minimum": 1,
39 "default": 5
40 },
41 + "top_queries_limit": {
42 + "title": "Top Queries Limit",
43 + "description": "Maximum number of queries to return in the top-queries function response.",
44 + "type": "integer",
45 + "minimum": 1,
46 + "maximum": 5000,
47 + "default": 500
48 + },
49 "vnode": {
50 "title": "Vnode",
51 "description": "Associates this data collection job with a [Virtual Node](https://learn.netdata.cloud/docs/netdata-agent/configuration/organize-systems-metrics-and-alerts#virtual-nodes).",
@@ -114,6 +122,7 @@
122 "address",
123 "timeout",
124 "ping_samples",
125 + "top_queries_limit",
126 "vnode"
127 ]
128 },
src/go/plugin/go.d/collector/redis/functions.go new
+403
@@ -0,0 +1,403 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package redis
4 +
5 +import (
6 + "context"
7 + "fmt"
8 + "sort"
9 + "strings"
10 + "time"
11 +
12 + "github.com/redis/go-redis/v9"
13 +
14 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
15 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
16 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
17 +)
18 +
19 +const redisMaxQueryTextLength = 4096
20 +
21 +const (
22 + paramSort = "__sort"
23 +
24 + ftString = funcapi.FieldTypeString
25 + ftInteger = funcapi.FieldTypeInteger
26 + ftDuration = funcapi.FieldTypeDuration
27 + ftTimestamp = funcapi.FieldTypeTimestamp
28 + trNone = funcapi.FieldTransformNone
29 + trNumber = funcapi.FieldTransformNumber
30 + trDuration = funcapi.FieldTransformDuration
31 + trDatetime = funcapi.FieldTransformDatetime
32 + trText = funcapi.FieldTransformText
33 +
34 + visValue = funcapi.FieldVisualValue
35 + visBar = funcapi.FieldVisualBar
36 +
37 + sortAsc = funcapi.FieldSortAscending
38 + sortDesc = funcapi.FieldSortDescending
39 +
40 + summaryCount = funcapi.FieldSummaryCount
41 + summaryMax = funcapi.FieldSummaryMax
42 + summarySum = funcapi.FieldSummarySum
43 +
44 + filterMulti = funcapi.FieldFilterMultiselect
45 + filterRange = funcapi.FieldFilterRange
46 +)
47 +
48 +type redisColumnMeta struct {
49 + id string
50 + name string
51 + colType funcapi.FieldType
52 + visible bool
53 + sortable bool
54 + fullWidth bool
55 + wrap bool
56 + sticky bool
57 + filter funcapi.FieldFilter
58 + visualization funcapi.FieldVisual
59 + transform funcapi.FieldTransform
60 + units string
61 + decimalPoints int
62 + uniqueKey bool
63 + sortDir funcapi.FieldSort
64 + summary funcapi.FieldSummary
65 + isLabel bool
66 + isPrimary bool
67 + isMetric bool
68 + chartGroup string
69 + chartTitle string
70 + isDefaultChart bool
71 +}
72 +
73 +var redisAllColumns = []redisColumnMeta{
74 + {id: "id", name: "ID", colType: ftInteger, visible: false, sortable: true, filter: filterRange, visualization: visValue, transform: trNumber, uniqueKey: true, sortDir: sortDesc, summary: summaryCount},
75 + {id: "timestamp", name: "Timestamp", colType: ftTimestamp, visible: true, sortable: true, filter: filterRange, visualization: visValue, transform: trDatetime, sortDir: sortDesc, summary: summaryMax},
76 + {id: "command", name: "Command", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, sticky: true, fullWidth: true, wrap: true},
77 + {id: "command_name", name: "Command Name", colType: ftString, visible: true, sortable: true, filter: filterMulti, visualization: visValue, transform: trText, sortDir: sortAsc, isLabel: true, isPrimary: true},
78 + {id: "duration", name: "Duration", colType: ftDuration, visible: true, sortable: true, filter: filterRange, visualization: visBar, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summarySum, isMetric: true, chartGroup: "Duration", chartTitle: "Execution Time", isDefaultChart: true},
79 + {id: "client_addr", name: "Client Address", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, isLabel: true},
80 + {id: "client_name", name: "Client Name", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, isLabel: true},
81 +}
82 +
83 +func redisMethods() []module.MethodConfig {
84 + sortOptions := buildRedisSortOptions(redisAllColumns)
85 + return []module.MethodConfig{{
86 + ID: "top-queries",
87 + Name: "Top Queries",
88 + Help: "Slow commands from Redis SLOWLOG. WARNING: Command arguments may contain unmasked literals (potential PII).",
89 + RequiredParams: []funcapi.ParamConfig{{
90 + ID: paramSort,
91 + Name: "Filter By",
92 + Help: "Select the primary sort column",
93 + Selection: funcapi.ParamSelect,
94 + Options: sortOptions,
95 + UniqueView: true,
96 + }},
97 + }}
98 +}
99 +
100 +func redisMethodParams(_ context.Context, _ *module.Job, method string) ([]funcapi.ParamConfig, error) {
101 + switch method {
102 + case "top-queries":
103 + return []funcapi.ParamConfig{buildRedisSortParam(redisAllColumns)}, nil
104 + default:
105 + return nil, fmt.Errorf("unknown method: %s", method)
106 + }
107 +}
108 +
109 +func redisHandleMethod(ctx context.Context, job *module.Job, method string, params funcapi.ResolvedParams) *module.FunctionResponse {
110 + collector, ok := job.Module().(*Collector)
111 + if !ok {
112 + return &module.FunctionResponse{Status: 500, Message: "internal error: invalid module type"}
113 + }
114 +
115 + if collector.rdb == nil {
116 + return &module.FunctionResponse{
117 + Status: 503,
118 + Message: "collector is still initializing, please retry in a few seconds",
119 + }
120 + }
121 +
122 + switch method {
123 + case "top-queries":
124 + return collector.collectTopQueries(ctx, params.Column(paramSort))
125 + default:
126 + return &module.FunctionResponse{Status: 404, Message: fmt.Sprintf("unknown method: %s", method)}
127 + }
128 +}
129 +
130 +func buildRedisSortOptions(cols []redisColumnMeta) []funcapi.ParamOption {
131 + var sortOptions []funcapi.ParamOption
132 + sortDir := funcapi.FieldSortDescending
133 + for _, col := range cols {
134 + if !col.sortable {
135 + continue
136 + }
137 + opt := funcapi.ParamOption{
138 + ID: col.id,
139 + Column: col.id,
140 + Name: fmt.Sprintf("Top queries by %s", col.name),
141 + Sort: &sortDir,
142 + }
143 + if col.id == "duration" {
144 + opt.Default = true
145 + }
146 + sortOptions = append(sortOptions, opt)
147 + }
148 + return sortOptions
149 +}
150 +
151 +func buildRedisSortParam(cols []redisColumnMeta) funcapi.ParamConfig {
152 + return funcapi.ParamConfig{
153 + ID: paramSort,
154 + Name: "Filter By",
155 + Help: "Select the primary sort column",
156 + Selection: funcapi.ParamSelect,
157 + Options: buildRedisSortOptions(cols),
158 + UniqueView: true,
159 + }
160 +}
161 +
162 +func buildRedisColumns(cols []redisColumnMeta) map[string]any {
163 + result := make(map[string]any, len(cols))
164 + for i, col := range cols {
165 + colDef := funcapi.Column{
166 + Index: i,
167 + Name: col.name,
168 + Type: col.colType,
169 + Units: col.units,
170 + Visualization: col.visualization,
171 + Sort: col.sortDir,
172 + Sortable: col.sortable,
173 + Sticky: col.sticky,
174 + Summary: col.summary,
175 + Filter: col.filter,
176 + FullWidth: col.fullWidth,
177 + Wrap: col.wrap,
178 + DefaultExpandedFilter: false,
179 + UniqueKey: col.uniqueKey,
180 + Visible: col.visible,
181 + ValueOptions: funcapi.ValueOptions{
182 + Transform: col.transform,
183 + DecimalPoints: col.decimalPoints,
184 + DefaultValue: nil,
185 + },
186 + }
187 + result[col.id] = colDef.BuildColumn()
188 + }
189 + return result
190 +}
191 +
192 +func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
193 + limit := c.TopQueriesLimit
194 + if limit <= 0 {
195 + limit = 500
196 + }
197 +
198 + entries, err := c.rdb.SlowLogGet(ctx, -1).Result()
199 + if err != nil {
200 + if ctx.Err() == context.DeadlineExceeded {
201 + return &module.FunctionResponse{Status: 504, Message: "query timed out"}
202 + }
203 + return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("slowlog query failed: %v", err)}
204 + }
205 +
206 + if len(entries) == 0 {
207 + return &module.FunctionResponse{
208 + Status: 200,
209 + Message: "No slow commands found. SLOWLOG may be empty or disabled.",
210 + Help: "Slow commands from Redis SLOWLOG. WARNING: Command arguments may contain unmasked literals (potential PII).",
211 + Columns: buildRedisColumns(redisAllColumns),
212 + Data: [][]any{},
213 + DefaultSortColumn: "duration",
214 + RequiredParams: []funcapi.ParamConfig{buildRedisSortParam(redisAllColumns)},
215 + Charts: redisTopQueriesCharts(redisAllColumns),
216 + DefaultCharts: redisTopQueriesDefaultCharts(redisAllColumns),
217 + GroupBy: redisTopQueriesGroupBy(redisAllColumns),
218 + }
219 + }
220 +
221 + sortColumn = mapRedisSortColumn(sortColumn)
222 + sortRedisSlowLogs(entries, sortColumn)
223 +
224 + if len(entries) > limit {
225 + entries = entries[:limit]
226 + }
227 +
228 + data := make([][]any, 0, len(entries))
229 + for _, entry := range entries {
230 + command := strings.Join(entry.Args, " ")
231 + commandName := ""
232 + if len(entry.Args) > 0 {
233 + commandName = entry.Args[0]
234 + }
235 +
236 + row := make([]any, len(redisAllColumns))
237 + for i, col := range redisAllColumns {
238 + switch col.id {
239 + case "id":
240 + row[i] = entry.ID
241 + case "timestamp":
242 + row[i] = entry.Time.Format(time.RFC3339Nano)
243 + case "command":
244 + row[i] = strmutil.TruncateText(command, redisMaxQueryTextLength)
245 + case "command_name":
246 + row[i] = commandName
247 + case "duration":
248 + row[i] = float64(entry.Duration) / float64(time.Millisecond)
249 + case "client_addr":
250 + row[i] = entry.ClientAddr
251 + case "client_name":
252 + row[i] = entry.ClientName
253 + default:
254 + row[i] = nil
255 + }
256 + }
257 + data = append(data, row)
258 + }
259 +
260 + return &module.FunctionResponse{
261 + Status: 200,
262 + Help: "Slow commands from Redis SLOWLOG. WARNING: Command arguments may contain unmasked literals (potential PII).",
263 + Columns: buildRedisColumns(redisAllColumns),
264 + Data: data,
265 + DefaultSortColumn: "duration",
266 + RequiredParams: []funcapi.ParamConfig{buildRedisSortParam(redisAllColumns)},
267 + Charts: redisTopQueriesCharts(redisAllColumns),
268 + DefaultCharts: redisTopQueriesDefaultCharts(redisAllColumns),
269 + GroupBy: redisTopQueriesGroupBy(redisAllColumns),
270 + }
271 +}
272 +
273 +func mapRedisSortColumn(col string) string {
274 + switch col {
275 + case "duration", "timestamp", "id", "command_name":
276 + return col
277 + default:
278 + return "duration"
279 + }
280 +}
281 +
282 +func sortRedisSlowLogs(entries []redis.SlowLog, sortColumn string) {
283 + switch sortColumn {
284 + case "timestamp":
285 + sort.Slice(entries, func(i, j int) bool {
286 + return entries[i].Time.After(entries[j].Time)
287 + })
288 + case "id":
289 + sort.Slice(entries, func(i, j int) bool {
290 + return entries[i].ID > entries[j].ID
291 + })
292 + case "command_name":
293 + sort.Slice(entries, func(i, j int) bool {
294 + var a, b string
295 + if len(entries[i].Args) > 0 {
296 + a = entries[i].Args[0]
297 + }
298 + if len(entries[j].Args) > 0 {
299 + b = entries[j].Args[0]
300 + }
301 + return a > b
302 + })
303 + default:
304 + sort.Slice(entries, func(i, j int) bool {
305 + return entries[i].Duration > entries[j].Duration
306 + })
307 + }
308 +}
309 +
310 +func redisTopQueriesCharts(cols []redisColumnMeta) map[string]module.ChartConfig {
311 + charts := make(map[string]module.ChartConfig)
312 + for _, col := range cols {
313 + if !col.isMetric || col.chartGroup == "" {
314 + continue
315 + }
316 + cfg, ok := charts[col.chartGroup]
317 + if !ok {
318 + title := col.chartTitle
319 + if title == "" {
320 + title = col.chartGroup
321 + }
322 + cfg = module.ChartConfig{
323 + Name: title,
324 + Type: "stacked-bar",
325 + }
326 + }
327 + cfg.Columns = append(cfg.Columns, col.id)
328 + charts[col.chartGroup] = cfg
329 + }
330 + return charts
331 +}
332 +
333 +func redisTopQueriesDefaultCharts(cols []redisColumnMeta) [][]string {
334 + label := primaryRedisLabel(cols)
335 + if label == "" {
336 + return nil
337 + }
338 +
339 + chartGroups := defaultRedisChartGroups(cols)
340 + out := make([][]string, 0, len(chartGroups))
341 + for _, group := range chartGroups {
342 + out = append(out, []string{group, label})
343 + }
344 + return out
345 +}
346 +
347 +func redisTopQueriesGroupBy(cols []redisColumnMeta) map[string]module.GroupByConfig {
348 + groupBy := make(map[string]module.GroupByConfig)
349 + for _, col := range cols {
350 + if !col.isLabel {
351 + continue
352 + }
353 + groupBy[col.id] = module.GroupByConfig{
354 + Name: "Group by " + col.name,
355 + Columns: []string{col.id},
356 + }
357 + }
358 + return groupBy
359 +}
360 +
361 +func primaryRedisLabel(cols []redisColumnMeta) string {
362 + for _, col := range cols {
363 + if col.isPrimary {
364 + return col.id
365 + }
366 + }
367 + for _, col := range cols {
368 + if col.isLabel {
369 + return col.id
370 + }
371 + }
372 + return ""
373 +}
374 +
375 +func defaultRedisChartGroups(cols []redisColumnMeta) []string {
376 + groups := make([]string, 0)
377 + seen := make(map[string]bool)
378 +
379 + for _, col := range cols {
380 + if !col.isMetric || col.chartGroup == "" || !col.isDefaultChart {
381 + continue
382 + }
383 + if !seen[col.chartGroup] {
384 + seen[col.chartGroup] = true
385 + groups = append(groups, col.chartGroup)
386 + }
387 + }
388 +
389 + if len(groups) > 0 {
390 + return groups
391 + }
392 +
393 + for _, col := range cols {
394 + if !col.isMetric || col.chartGroup == "" {
395 + continue
396 + }
397 + if !seen[col.chartGroup] {
398 + seen[col.chartGroup] = true
399 + groups = append(groups, col.chartGroup)
400 + }
401 + }
402 + return groups
403 +}
src/go/plugin/go.d/collector/redis/functions_test.go new
+74
@@ -0,0 +1,74 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package redis
4 +
5 +import (
6 + "testing"
7 +
8 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
9 + "github.com/stretchr/testify/assert"
10 +)
11 +
12 +func TestRedisMethods(t *testing.T) {
13 + methods := redisMethods()
14 +
15 + require := assert.New(t)
16 + require.Len(methods, 1)
17 + require.Equal("top-queries", methods[0].ID)
18 + require.Equal("Top Queries", methods[0].Name)
19 + require.NotEmpty(methods[0].RequiredParams)
20 +
21 + var sortParam *funcapi.ParamConfig
22 + for i := range methods[0].RequiredParams {
23 + if methods[0].RequiredParams[i].ID == "__sort" {
24 + sortParam = &methods[0].RequiredParams[i]
25 + break
26 + }
27 + }
28 + require.NotNil(sortParam, "expected __sort required param")
29 + require.NotEmpty(sortParam.Options)
30 +
31 + hasDefault := false
32 + for _, opt := range sortParam.Options {
33 + if opt.Default {
34 + hasDefault = true
35 + require.Equal("duration", opt.ID)
36 + break
37 + }
38 + }
39 + require.True(hasDefault, "should have a default sort option")
40 +}
41 +
42 +func TestRedisAllColumns_HasRequiredColumns(t *testing.T) {
43 + required := []string{"timestamp", "command", "duration"}
44 +
45 + uiKeys := make(map[string]bool)
46 + for _, col := range redisAllColumns {
47 + uiKeys[col.id] = true
48 + }
49 +
50 + for _, key := range required {
51 + assert.True(t, uiKeys[key], "column %s should be defined", key)
52 + }
53 +}
54 +
55 +func TestMapRedisSortColumn(t *testing.T) {
56 + tests := map[string]struct {
57 + input string
58 + expected string
59 + }{
60 + "duration": {input: "duration", expected: "duration"},
61 + "timestamp": {input: "timestamp", expected: "timestamp"},
62 + "id": {input: "id", expected: "id"},
63 + "unknown": {input: "unknown", expected: "duration"},
64 + "empty": {input: "", expected: "duration"},
65 + "injection": {input: "duration;drop table", expected: "duration"},
66 + "cmd_name": {input: "command_name", expected: "command_name"},
67 + }
68 +
69 + for name, tc := range tests {
70 + t.Run(name, func(t *testing.T) {
71 + assert.Equal(t, tc.expected, mapRedisSortColumn(tc.input))
72 + })
73 + }
74 +}
src/go/plugin/go.d/collector/rethinkdb/client.go
+25
@@ -12,6 +12,7 @@ import (
12
13 type rdbConn interface {
14 stats() ([][]byte, error)
15 + jobs(ctx context.Context) ([]map[string]any, error)
16 close() error
17 }
18
@@ -67,6 +68,30 @@ func (c *rethinkdbClient) stats() ([][]byte, error) {
68 return stats, nil
69 }
70
71 +func (c *rethinkdbClient) jobs(ctx context.Context) ([]map[string]any, error) {
72 + ctx, cancel := context.WithTimeout(ctx, c.timeout)
73 + defer cancel()
74 +
75 + opts := rethinkdb.RunOpts{Context: ctx}
76 +
77 + cur, err := rethinkdb.DB("rethinkdb").Table("jobs").Run(c.sess, opts)
78 + if err != nil {
79 + return nil, err
80 + }
81 +
82 + if cur.IsNil() {
83 + return nil, errors.New("no jobs found (cursor is nil)")
84 + }
85 + defer func() { _ = cur.Close() }()
86 +
87 + var rows []map[string]any
88 + if err := cur.All(&rows); err != nil {
89 + return nil, err
90 + }
91 +
92 + return rows, nil
93 +}
94 +
95 func (c *rethinkdbClient) close() (err error) {
96 return c.sess.Close()
97 }
src/go/plugin/go.d/collector/rethinkdb/collector.go
+4
@@ -20,6 +20,9 @@ func init() {
20 JobConfigSchema: configSchema,
21 Create: func() module.Module { return New() },
22 Config: func() any { return &Config{} },
23 + Methods: rethinkdbMethods,
24 + MethodParams: rethinkdbMethodParams,
25 + HandleMethod: rethinkdbHandleMethod,
26 })
27 }
28
@@ -44,6 +47,7 @@ type Config struct {
47 Timeout confopt.Duration `yaml:"timeout,omitempty" json:"timeout"`
48 Username string `yaml:"username,omitempty" json:"username"`
49 Password string `yaml:"password,omitempty" json:"password"`
50 + TopQueriesLimit int `yaml:"top_queries_limit,omitempty" json:"top_queries_limit,omitempty"`
51 }
52
53 type Collector struct {
src/go/plugin/go.d/collector/rethinkdb/collector_test.go
+4
@@ -262,6 +262,10 @@ func (m *mockRethinkdbConn) stats() ([][]byte, error) {
262 return bytes.Split(bytes.TrimSpace(m.dataStats), []byte("\n")), nil
263 }
264
265 +func (m *mockRethinkdbConn) jobs(ctx context.Context) ([]map[string]any, error) {
266 + return []map[string]any{}, nil
267 +}
268 +
269 func (m *mockRethinkdbConn) close() error {
270 m.disconnectCalled = true
271 return nil
src/go/plugin/go.d/collector/rethinkdb/config_schema.json
+8
@@ -31,6 +31,14 @@
31 "minimum": 0.5,
32 "default": 1
33 },
34 + "top_queries_limit": {
35 + "title": "Top Queries Limit",
36 + "description": "Maximum number of queries to return in the running-queries function response.",
37 + "type": "integer",
38 + "minimum": 1,
39 + "maximum": 5000,
40 + "default": 500
41 + },
42 "vnode": {
43 "title": "Vnode",
44 "description": "Associates this data collection job with a [Virtual Node](https://learn.netdata.cloud/docs/netdata-agent/configuration/organize-systems-metrics-and-alerts#virtual-nodes).",
src/go/plugin/go.d/collector/rethinkdb/functions.go new
+373
@@ -0,0 +1,373 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package rethinkdb
4 +
5 +import (
6 + "context"
7 + "fmt"
8 + "sort"
9 + "strings"
10 +
11 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
12 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
13 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
14 +)
15 +
16 +const rethinkMaxQueryTextLength = 4096
17 +
18 +const (
19 + paramSort = "__sort"
20 +
21 + ftString = funcapi.FieldTypeString
22 + ftInteger = funcapi.FieldTypeInteger
23 + ftDuration = funcapi.FieldTypeDuration
24 +
25 + trNone = funcapi.FieldTransformNone
26 + trNumber = funcapi.FieldTransformNumber
27 + trDuration = funcapi.FieldTransformDuration
28 + trText = funcapi.FieldTransformText
29 +
30 + visValue = funcapi.FieldVisualValue
31 + visBar = funcapi.FieldVisualBar
32 +
33 + sortAsc = funcapi.FieldSortAscending
34 + sortDesc = funcapi.FieldSortDescending
35 +
36 + summaryCount = funcapi.FieldSummaryCount
37 + summaryMax = funcapi.FieldSummaryMax
38 +
39 + filterMulti = funcapi.FieldFilterMultiselect
40 + filterRange = funcapi.FieldFilterRange
41 +)
42 +
43 +type rethinkColumnMeta struct {
44 + id string
45 + name string
46 + colType funcapi.FieldType
47 + visible bool
48 + sortable bool
49 + fullWidth bool
50 + wrap bool
51 + sticky bool
52 + filter funcapi.FieldFilter
53 + visualization funcapi.FieldVisual
54 + transform funcapi.FieldTransform
55 + units string
56 + decimalPoints int
57 + uniqueKey bool
58 + sortDir funcapi.FieldSort
59 + summary funcapi.FieldSummary
60 + sortLabel string
61 + isSortOption bool
62 + isDefaultSort bool
63 +}
64 +
65 +var rethinkRunningColumns = []rethinkColumnMeta{
66 + {id: "jobId", name: "Job ID", colType: ftString, visible: false, sortable: true, filter: filterMulti, visualization: visValue, transform: trText, uniqueKey: true, sortDir: sortAsc, summary: summaryCount},
67 + {id: "query", name: "Query", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, sticky: true, fullWidth: true, wrap: true},
68 + {id: "durationMs", name: "Duration", colType: ftDuration, visible: true, sortable: true, filter: filterRange, visualization: visBar, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, isSortOption: true, isDefaultSort: true, sortLabel: "Running queries by Duration"},
69 + {id: "type", name: "Type", colType: ftString, visible: true, sortable: true, filter: filterMulti, visualization: visValue, transform: trText, sortDir: sortAsc, summary: summaryCount},
70 + {id: "user", name: "User", colType: ftString, visible: true, sortable: true, filter: filterMulti, visualization: visValue, transform: trText, sortDir: sortAsc, summary: summaryCount},
71 + {id: "clientAddress", name: "Client Address", colType: ftString, visible: false, sortable: true, filter: filterMulti, visualization: visValue, transform: trText, sortDir: sortAsc, summary: summaryCount},
72 + {id: "clientPort", name: "Client Port", colType: ftInteger, visible: false, sortable: true, filter: filterRange, visualization: visValue, transform: trNumber, sortDir: sortDesc, summary: summaryMax},
73 + {id: "servers", name: "Servers", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText},
74 +}
75 +
76 +func rethinkdbMethods() []module.MethodConfig {
77 + sortOptions := buildRethinkSortOptions(rethinkRunningColumns)
78 + return []module.MethodConfig{{
79 + ID: "running-queries",
80 + Name: "Running Queries",
81 + Help: "Currently running queries from rethinkdb.jobs. WARNING: Query text may contain unmasked literals (potential PII).",
82 + RequiredParams: []funcapi.ParamConfig{{
83 + ID: paramSort,
84 + Name: "Filter By",
85 + Help: "Select the primary sort column",
86 + Selection: funcapi.ParamSelect,
87 + Options: sortOptions,
88 + UniqueView: true,
89 + }},
90 + }}
91 +}
92 +
93 +func rethinkdbMethodParams(_ context.Context, _ *module.Job, method string) ([]funcapi.ParamConfig, error) {
94 + switch method {
95 + case "running-queries":
96 + return []funcapi.ParamConfig{buildRethinkSortParam(rethinkRunningColumns)}, nil
97 + default:
98 + return nil, fmt.Errorf("unknown method: %s", method)
99 + }
100 +}
101 +
102 +func rethinkdbHandleMethod(ctx context.Context, job *module.Job, method string, params funcapi.ResolvedParams) *module.FunctionResponse {
103 + collector, ok := job.Module().(*Collector)
104 + if !ok {
105 + return &module.FunctionResponse{Status: 500, Message: "internal error: invalid module type"}
106 + }
107 +
108 + if collector.rdb == nil {
109 + conn, err := collector.newConn(collector.Config)
110 + if err != nil {
111 + return &module.FunctionResponse{Status: 503, Message: "collector is still initializing, please retry in a few seconds"}
112 + }
113 + collector.rdb = conn
114 + }
115 +
116 + switch method {
117 + case "running-queries":
118 + return collector.collectRunningQueries(ctx, params.Column(paramSort))
119 + default:
120 + return &module.FunctionResponse{Status: 404, Message: fmt.Sprintf("unknown method: %s", method)}
121 + }
122 +}
123 +
124 +func buildRethinkSortOptions(cols []rethinkColumnMeta) []funcapi.ParamOption {
125 + var sortOptions []funcapi.ParamOption
126 + sortDir := funcapi.FieldSortDescending
127 + for _, col := range cols {
128 + if !col.isSortOption {
129 + continue
130 + }
131 + opt := funcapi.ParamOption{
132 + ID: col.id,
133 + Column: col.id,
134 + Name: col.sortLabel,
135 + Sort: &sortDir,
136 + }
137 + if col.isDefaultSort {
138 + opt.Default = true
139 + }
140 + sortOptions = append(sortOptions, opt)
141 + }
142 + return sortOptions
143 +}
144 +
145 +func buildRethinkSortParam(cols []rethinkColumnMeta) funcapi.ParamConfig {
146 + return funcapi.ParamConfig{
147 + ID: paramSort,
148 + Name: "Filter By",
149 + Help: "Select the primary sort column",
150 + Selection: funcapi.ParamSelect,
151 + Options: buildRethinkSortOptions(cols),
152 + UniqueView: true,
153 + }
154 +}
155 +
156 +func buildRethinkColumns(cols []rethinkColumnMeta) map[string]any {
157 + result := make(map[string]any, len(cols))
158 + for i, col := range cols {
159 + visual := visValue
160 + if col.colType == ftDuration {
161 + visual = visBar
162 + }
163 + colDef := funcapi.Column{
164 + Index: i,
165 + Name: col.name,
166 + Type: col.colType,
167 + Units: col.units,
168 + Visualization: visual,
169 + Sort: col.sortDir,
170 + Sortable: col.sortable,
171 + Sticky: col.sticky,
172 + Summary: col.summary,
173 + Filter: col.filter,
174 + FullWidth: col.fullWidth,
175 + Wrap: col.wrap,
176 + DefaultExpandedFilter: false,
177 + UniqueKey: col.uniqueKey,
178 + Visible: col.visible,
179 + ValueOptions: funcapi.ValueOptions{
180 + Transform: col.transform,
181 + DecimalPoints: col.decimalPoints,
182 + DefaultValue: nil,
183 + },
184 + }
185 + result[col.id] = colDef.BuildColumn()
186 + }
187 + return result
188 +}
189 +
190 +func (c *Collector) collectRunningQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
191 + limit := c.TopQueriesLimit
192 + if limit <= 0 {
193 + limit = 500
194 + }
195 +
196 + if ctx.Err() == context.DeadlineExceeded {
197 + return &module.FunctionResponse{Status: 504, Message: "query timed out"}
198 + }
199 +
200 + rows, err := c.rdb.jobs(ctx)
201 + if err != nil {
202 + if ctx.Err() == context.DeadlineExceeded {
203 + return &module.FunctionResponse{Status: 504, Message: "query timed out"}
204 + }
205 + return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("jobs query failed: %v", err)}
206 + }
207 +
208 + jobRows := make([]rethinkJobRow, 0, len(rows))
209 + for _, row := range rows {
210 + jobRows = append(jobRows, parseRethinkJob(row))
211 + }
212 +
213 + if len(jobRows) == 0 {
214 + return &module.FunctionResponse{
215 + Status: 200,
216 + Message: "No running queries found.",
217 + Help: "Currently running queries from rethinkdb.jobs",
218 + Columns: buildRethinkColumns(rethinkRunningColumns),
219 + Data: [][]any{},
220 + DefaultSortColumn: mapRethinkSortColumn(sortColumn),
221 + RequiredParams: []funcapi.ParamConfig{buildRethinkSortParam(rethinkRunningColumns)},
222 + }
223 + }
224 +
225 + sortColumn = mapRethinkSortColumn(sortColumn)
226 + sortRethinkRows(jobRows, sortColumn)
227 + if len(jobRows) > limit {
228 + jobRows = jobRows[:limit]
229 + }
230 +
231 + data := make([][]any, 0, len(jobRows))
232 + for _, row := range jobRows {
233 + out := make([]any, len(rethinkRunningColumns))
234 + for i, col := range rethinkRunningColumns {
235 + switch col.id {
236 + case "jobId":
237 + out[i] = row.JobID
238 + case "query":
239 + out[i] = strmutil.TruncateText(row.Query, rethinkMaxQueryTextLength)
240 + case "durationMs":
241 + out[i] = row.DurationMs
242 + case "type":
243 + out[i] = row.Type
244 + case "user":
245 + out[i] = row.User
246 + case "clientAddress":
247 + out[i] = row.ClientAddress
248 + case "clientPort":
249 + out[i] = row.ClientPort
250 + case "servers":
251 + out[i] = row.Servers
252 + default:
253 + out[i] = nil
254 + }
255 + }
256 + data = append(data, out)
257 + }
258 +
259 + return &module.FunctionResponse{
260 + Status: 200,
261 + Help: "Currently running queries from rethinkdb.jobs. WARNING: Query text may contain unmasked literals (potential PII).",
262 + Columns: buildRethinkColumns(rethinkRunningColumns),
263 + Data: data,
264 + DefaultSortColumn: sortColumn,
265 + RequiredParams: []funcapi.ParamConfig{buildRethinkSortParam(rethinkRunningColumns)},
266 + }
267 +}
268 +
269 +type rethinkJobRow struct {
270 + JobID string
271 + Query string
272 + DurationMs float64
273 + Type string
274 + User string
275 + ClientAddress string
276 + ClientPort int64
277 + Servers string
278 +}
279 +
280 +func parseRethinkJob(row map[string]any) rethinkJobRow {
281 + info := mapStringAny(row["info"])
282 + query := fmt.Sprint(info["query"])
283 + user := fmt.Sprint(info["user"])
284 + clientAddr := fmt.Sprint(info["client_address"])
285 + clientPort := toInt64(info["client_port"])
286 +
287 + servers := ""
288 + if list, ok := row["servers"].([]any); ok {
289 + ss := make([]string, 0, len(list))
290 + for _, v := range list {
291 + ss = append(ss, fmt.Sprint(v))
292 + }
293 + servers = strings.Join(ss, ",")
294 + }
295 +
296 + return rethinkJobRow{
297 + JobID: fmt.Sprint(row["id"]),
298 + Query: query,
299 + DurationMs: toFloat64(row["duration_sec"]) * 1000,
300 + Type: fmt.Sprint(row["type"]),
301 + User: user,
302 + ClientAddress: clientAddr,
303 + ClientPort: clientPort,
304 + Servers: servers,
305 + }
306 +}
307 +
308 +func mapStringAny(v any) map[string]any {
309 + if m, ok := v.(map[string]any); ok {
310 + return m
311 + }
312 + return map[string]any{}
313 +}
314 +
315 +func toFloat64(v any) float64 {
316 + switch t := v.(type) {
317 + case float64:
318 + return t
319 + case float32:
320 + return float64(t)
321 + case int:
322 + return float64(t)
323 + case int64:
324 + return float64(t)
325 + case uint64:
326 + return float64(t)
327 + default:
328 + return 0
329 + }
330 +}
331 +
332 +func toInt64(v any) int64 {
333 + switch t := v.(type) {
334 + case int64:
335 + return t
336 + case int:
337 + return int64(t)
338 + case float64:
339 + return int64(t)
340 + case float32:
341 + return int64(t)
342 + default:
343 + return 0
344 + }
345 +}
346 +
347 +func mapRethinkSortColumn(input string) string {
348 + for _, col := range rethinkRunningColumns {
349 + if col.isSortOption && col.id == input {
350 + return col.id
351 + }
352 + }
353 + for _, col := range rethinkRunningColumns {
354 + if col.isDefaultSort {
355 + return col.id
356 + }
357 + }
358 + for _, col := range rethinkRunningColumns {
359 + if col.isSortOption {
360 + return col.id
361 + }
362 + }
363 + return ""
364 +}
365 +
366 +func sortRethinkRows(rows []rethinkJobRow, sortColumn string) {
367 + switch sortColumn {
368 + case "durationMs":
369 + sort.Slice(rows, func(i, j int) bool { return rows[i].DurationMs > rows[j].DurationMs })
370 + default:
371 + sort.Slice(rows, func(i, j int) bool { return rows[i].DurationMs > rows[j].DurationMs })
372 + }
373 +}
src/go/plugin/go.d/collector/rethinkdb/functions_test.go new
+43
@@ -0,0 +1,43 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package rethinkdb
4 +
5 +import (
6 + "testing"
7 +
8 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
9 + "github.com/stretchr/testify/assert"
10 +)
11 +
12 +func TestRethinkDBMethods(t *testing.T) {
13 + methods := rethinkdbMethods()
14 +
15 + require := assert.New(t)
16 + require.Len(methods, 1)
17 + require.Equal("running-queries", methods[0].ID)
18 + require.Equal("Running Queries", methods[0].Name)
19 + require.NotEmpty(methods[0].RequiredParams)
20 +
21 + var sortParam *funcapi.ParamConfig
22 + for i := range methods[0].RequiredParams {
23 + if methods[0].RequiredParams[i].ID == "__sort" {
24 + sortParam = &methods[0].RequiredParams[i]
25 + break
26 + }
27 + }
28 + require.NotNil(sortParam, "expected __sort required param")
29 + require.NotEmpty(sortParam.Options)
30 +}
31 +
32 +func TestRethinkDBRunningColumns_HasRequiredColumns(t *testing.T) {
33 + required := []string{"jobId", "query", "durationMs"}
34 +
35 + uiKeys := make(map[string]bool)
36 + for _, col := range rethinkRunningColumns {
37 + uiKeys[col.id] = true
38 + }
39 +
40 + for _, key := range required {
41 + assert.True(t, uiKeys[key], "column %s should be defined", key)
42 + }
43 +}
src/go/plugin/go.d/collector/rethinkdb/metadata.yaml
+6
@@ -27,6 +27,7 @@ modules:
27 For each server, it offers similar metrics.
28 method_description: |
29 The data is gathered by querying the stats table in RethinkDB, which stores real-time statistics related to the cluster and its individual servers.
30 + It also provides a `running-queries` function using the `rethinkdb.jobs` system table (admin-only).
31 supported_platforms:
32 include: []
33 exclude: []
@@ -75,6 +76,11 @@ modules:
76 default_value: 1
77 required: false
78 group: Target
79 + - name: top_queries_limit
80 + description: Maximum number of rows returned by the `running-queries` function.
81 + default_value: 500
82 + required: false
83 + group: Limits
84
85 - name: username
86 description: Username for authentication.
src/go/plugin/go.d/collector/yugabytedb/collector.go
+22 -5
@@ -4,10 +4,12 @@ package yugabytedb
4
5 import (
6 "context"
7 + "database/sql"
8 _ "embed"
9 "errors"
10 "fmt"
11 "net/http"
12 + "sync"
13 "time"
14
15 "github.com/netdata/netdata/go/plugins/pkg/confopt"
@@ -25,8 +27,11 @@ func init() {
27 Defaults: module.Defaults{
28 UpdateEvery: 5,
29 },
28 - Create: func() module.Module { return New() },
29 - Config: func() any { return &Config{} },
30 + Methods: yugabyteMethods,
31 + MethodParams: yugabyteMethodParams,
32 + HandleMethod: yugabyteHandleMethod,
33 + Create: func() module.Module { return New() },
34 + Config: func() any { return &Config{} },
35 })
36 }
37
@@ -41,6 +46,7 @@ func New() *Collector {
46 Timeout: confopt.Duration(time.Second),
47 },
48 },
49 + SQLTimeout: confopt.Duration(time.Second),
50 },
51 charts: &module.Charts{},
52
@@ -49,9 +55,12 @@ func New() *Collector {
55 }
56
57 type Config struct {
52 - Vnode string `yaml:"vnode,omitempty" json:"vnode"`
53 - UpdateEvery int `yaml:"update_every,omitempty" json:"update_every"`
54 - AutoDetectionRetry int `yaml:"autodetection_retry,omitempty" json:"autodetection_retry"`
58 + Vnode string `yaml:"vnode,omitempty" json:"vnode"`
59 + UpdateEvery int `yaml:"update_every,omitempty" json:"update_every"`
60 + AutoDetectionRetry int `yaml:"autodetection_retry,omitempty" json:"autodetection_retry"`
61 + DSN string `yaml:"dsn,omitempty" json:"dsn,omitempty"`
62 + SQLTimeout confopt.Duration `yaml:"sql_timeout,omitempty" json:"sql_timeout,omitempty"`
63 + TopQueriesLimit int `yaml:"top_queries_limit,omitempty" json:"top_queries_limit,omitempty"`
64 web.HTTPConfig `yaml:",inline" json:""`
65 }
66
@@ -67,6 +76,11 @@ type Collector struct {
76 srvType string
77
78 cache map[string]map[string]bool
79 +
80 + db *sql.DB
81 +
82 + pgStatStatementsMu sync.RWMutex
83 + pgStatStatementsColumns map[string]bool
84 }
85
86 func (c *Collector) Configuration() any {
@@ -125,4 +139,7 @@ func (c *Collector) Cleanup(context.Context) {
139 if c.httpClient != nil {
140 c.httpClient.CloseIdleConnections()
141 }
142 + if c.db != nil {
143 + _ = c.db.Close()
144 + }
145 }
src/go/plugin/go.d/collector/yugabytedb/config_schema.json
+35
@@ -32,6 +32,26 @@
32 "minimum": 0.5,
33 "default": 1
34 },
35 + "dsn": {
36 + "title": "SQL DSN",
37 + "description": "YSQL Data Source Name for query functions (top-queries, running-queries).",
38 + "type": "string"
39 + },
40 + "sql_timeout": {
41 + "title": "SQL Timeout",
42 + "description": "Timeout in seconds for SQL query functions.",
43 + "type": "number",
44 + "minimum": 0.5,
45 + "default": 1
46 + },
47 + "top_queries_limit": {
48 + "title": "Top Queries Limit",
49 + "description": "Maximum number of rows returned by the top-queries and running-queries functions.",
50 + "type": "integer",
51 + "minimum": 1,
52 + "maximum": 5000,
53 + "default": 500
54 + },
55 "not_follow_redirects": {
56 "title": "Not follow redirects",
57 "description": "If set, the client will not follow HTTP redirects automatically.",
@@ -154,6 +174,13 @@
174 "timeout": {
175 "ui:help": "Accepts decimals for precise control (e.g., type 1.5 for 1.5 seconds)."
176 },
177 + "dsn": {
178 + "ui:help": "Format is `postgres://username:password@host:port/dbname?sslmode=disable`.",
179 + "ui:placeholder": "postgres://yugabyte@127.0.0.1:5433/yugabyte?sslmode=disable"
180 + },
181 + "sql_timeout": {
182 + "ui:help": "Accepts decimals for precise control (e.g., type 1.5 for 1.5 seconds)."
183 + },
184 "username": {
185 "ui:widget": "password"
186 },
@@ -179,6 +206,14 @@
206 "vnode"
207 ]
208 },
209 + {
210 + "title": "SQL",
211 + "fields": [
212 + "dsn",
213 + "sql_timeout",
214 + "top_queries_limit"
215 + ]
216 + },
217 {
218 "title": "Auth",
219 "fields": [
src/go/plugin/go.d/collector/yugabytedb/functions.go new
+713
@@ -0,0 +1,713 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package yugabytedb
4 +
5 +import (
6 + "context"
7 + "database/sql"
8 + "errors"
9 + "fmt"
10 + "strings"
11 + "time"
12 +
13 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
14 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
15 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
16 +)
17 +
18 +const ybMaxQueryTextLength = 4096
19 +
20 +const (
21 + paramSort = "__sort"
22 +
23 + ftString = funcapi.FieldTypeString
24 + ftInteger = funcapi.FieldTypeInteger
25 + ftFloat = funcapi.FieldTypeFloat
26 + ftDuration = funcapi.FieldTypeDuration
27 +
28 + trNone = funcapi.FieldTransformNone
29 + trNumber = funcapi.FieldTransformNumber
30 + trDuration = funcapi.FieldTransformDuration
31 + trText = funcapi.FieldTransformText
32 +
33 + visValue = funcapi.FieldVisualValue
34 + visBar = funcapi.FieldVisualBar
35 +
36 + sortAsc = funcapi.FieldSortAscending
37 + sortDesc = funcapi.FieldSortDescending
38 +
39 + summaryCount = funcapi.FieldSummaryCount
40 + summarySum = funcapi.FieldSummarySum
41 + summaryMax = funcapi.FieldSummaryMax
42 + summaryMean = funcapi.FieldSummaryMean
43 +
44 + filterMulti = funcapi.FieldFilterMultiselect
45 + filterRange = funcapi.FieldFilterRange
46 +)
47 +
48 +var errYBSQLDSNNotSet = errors.New("SQL DSN is not set")
49 +
50 +type ybColumnMeta struct {
51 + id string
52 + name string
53 + selectExpr string
54 + dataType funcapi.FieldType
55 + visible bool
56 + sortable bool
57 + fullWidth bool
58 + wrap bool
59 + sticky bool
60 + filter funcapi.FieldFilter
61 + visualization funcapi.FieldVisual
62 + transform funcapi.FieldTransform
63 + units string
64 + decimalPoints int
65 + uniqueKey bool
66 + sortDir funcapi.FieldSort
67 + summary funcapi.FieldSummary
68 + sortLabel string
69 + isSortOption bool
70 + isDefaultSort bool
71 + isJoinColumn bool
72 + isLabel bool
73 + isPrimary bool
74 + isMetric bool
75 + chartGroup string
76 + chartTitle string
77 + isDefaultChart bool
78 +}
79 +
80 +var ybTopColumns = []ybColumnMeta{
81 + {id: "queryId", name: "Query ID", selectExpr: "s.queryid::text", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, uniqueKey: true, sortDir: sortAsc, summary: summaryCount},
82 + {id: "query", name: "Query", selectExpr: "s.query", dataType: ftString, visible: true, sortable: false, filter: filterMulti, transform: trText, sticky: true, fullWidth: true, wrap: true},
83 + {id: "database", name: "Database", selectExpr: "d.datname", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount, isJoinColumn: true, isLabel: true, isPrimary: true},
84 + {id: "user", name: "User", selectExpr: "u.usename", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount, isJoinColumn: true, isLabel: true},
85 +
86 + {id: "calls", name: "Calls", selectExpr: "s.calls", dataType: ftInteger, visible: true, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by Calls", isMetric: true, chartGroup: "Calls", chartTitle: "Number of Calls", isDefaultChart: true},
87 + {id: "totalTime", name: "Total Time", selectExpr: "s.total_time", dataType: ftDuration, visible: true, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summarySum, isSortOption: true, isDefaultSort: true, sortLabel: "Top queries by Total Time", isMetric: true, chartGroup: "Time", chartTitle: "Execution Time", isDefaultChart: true},
88 + {id: "meanTime", name: "Mean Time", selectExpr: "s.mean_time", dataType: ftDuration, visible: true, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMean, isSortOption: true, sortLabel: "Top queries by Mean Time", isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
89 + {id: "minTime", name: "Min Time", selectExpr: "s.min_time", dataType: ftDuration, visible: false, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
90 + {id: "maxTime", name: "Max Time", selectExpr: "s.max_time", dataType: ftDuration, visible: false, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, isSortOption: true, sortLabel: "Top queries by Max Time", isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
91 + {id: "rows", name: "Rows", selectExpr: "s.rows", dataType: ftInteger, visible: true, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by Rows Returned", isMetric: true, chartGroup: "Rows", chartTitle: "Rows"},
92 + {id: "stddevTime", name: "Stddev Time", selectExpr: "s.stddev_time", dataType: ftDuration, visible: false, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
93 +}
94 +
95 +var ybRunningColumns = []ybColumnMeta{
96 + {id: "pid", name: "PID", selectExpr: "s.pid::text", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, uniqueKey: true, sortDir: sortAsc, summary: summaryCount},
97 + {id: "query", name: "Query", selectExpr: "s.query", dataType: ftString, visible: true, sortable: false, filter: filterMulti, transform: trText, sticky: true, fullWidth: true, wrap: true},
98 + {id: "database", name: "Database", selectExpr: "s.datname", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
99 + {id: "user", name: "User", selectExpr: "s.usename", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
100 + {id: "state", name: "State", selectExpr: "s.state", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
101 + {id: "waitEventType", name: "Wait Event Type", selectExpr: "s.wait_event_type", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
102 + {id: "waitEvent", name: "Wait Event", selectExpr: "s.wait_event", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
103 + {id: "application", name: "Application", selectExpr: "s.application_name", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
104 + {id: "clientAddress", name: "Client Address", selectExpr: "s.client_addr::text", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
105 + {id: "queryStart", name: "Query Start", selectExpr: "TO_CHAR(s.query_start, 'YYYY-MM-DD\"T\"HH24:MI:SS.FF3')", dataType: ftString, visible: false, sortable: true, filter: filterRange, transform: trText, sortDir: sortDesc, summary: summaryMax},
106 + {id: "elapsedMs", name: "Elapsed", selectExpr: "CASE WHEN s.query_start IS NULL THEN 0 ELSE EXTRACT(EPOCH FROM (clock_timestamp() - s.query_start)) * 1000 END", dataType: ftDuration, visible: true, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, isSortOption: true, isDefaultSort: true, sortLabel: "Running queries by Elapsed Time"},
107 +}
108 +
109 +func yugabyteMethods() []module.MethodConfig {
110 + return []module.MethodConfig{
111 + {
112 + ID: "top-queries",
113 + Name: "Top Queries",
114 + Help: "Top SQL queries from pg_stat_statements. WARNING: Query text may contain unmasked literals (potential PII).",
115 + RequiredParams: []funcapi.ParamConfig{
116 + buildYBSortParam(ybTopColumns),
117 + },
118 + },
119 + {
120 + ID: "running-queries",
121 + Name: "Running Queries",
122 + Help: "Currently running SQL statements from pg_stat_activity. WARNING: Query text may contain unmasked literals (potential PII).",
123 + RequiredParams: []funcapi.ParamConfig{
124 + buildYBSortParam(ybRunningColumns),
125 + },
126 + },
127 + }
128 +}
129 +
130 +func yugabyteMethodParams(ctx context.Context, job *module.Job, method string) ([]funcapi.ParamConfig, error) {
131 + collector, ok := job.Module().(*Collector)
132 + if !ok {
133 + return nil, fmt.Errorf("invalid module type")
134 + }
135 +
136 + switch method {
137 + case "top-queries":
138 + cols := ybTopColumns
139 + if collector.db != nil {
140 + if available, err := collector.availableTopColumns(ctx); err == nil {
141 + cols = available
142 + }
143 + }
144 + return []funcapi.ParamConfig{buildYBSortParam(cols)}, nil
145 + case "running-queries":
146 + return []funcapi.ParamConfig{buildYBSortParam(ybRunningColumns)}, nil
147 + default:
148 + return nil, fmt.Errorf("unknown method: %s", method)
149 + }
150 +}
151 +
152 +func yugabyteHandleMethod(ctx context.Context, job *module.Job, method string, params funcapi.ResolvedParams) *module.FunctionResponse {
153 + collector, ok := job.Module().(*Collector)
154 + if !ok {
155 + return &module.FunctionResponse{Status: 500, Message: "internal error: invalid module type"}
156 + }
157 +
158 + if err := collector.ensureSQL(ctx); err != nil {
159 + status := 503
160 + if errors.Is(err, errYBSQLDSNNotSet) {
161 + status = 400
162 + }
163 + return &module.FunctionResponse{Status: status, Message: err.Error()}
164 + }
165 +
166 + switch method {
167 + case "top-queries":
168 + return collector.collectTopQueries(ctx, params.Column(paramSort))
169 + case "running-queries":
170 + return collector.collectRunningQueries(ctx, params.Column(paramSort))
171 + default:
172 + return &module.FunctionResponse{Status: 404, Message: fmt.Sprintf("unknown method: %s", method)}
173 + }
174 +}
175 +
176 +func (c *Collector) ensureSQL(ctx context.Context) error {
177 + if c.db != nil {
178 + return nil
179 + }
180 + if c.DSN == "" {
181 + return errYBSQLDSNNotSet
182 + }
183 +
184 + db, err := sql.Open("pgx", c.DSN)
185 + if err != nil {
186 + return fmt.Errorf("error opening SQL connection: %w", err)
187 + }
188 + db.SetMaxOpenConns(1)
189 + db.SetMaxIdleConns(1)
190 + db.SetConnMaxLifetime(10 * time.Minute)
191 +
192 + timeout := c.sqlTimeout()
193 + pingCtx, cancel := context.WithTimeout(ctx, timeout)
194 + defer cancel()
195 + if err := db.PingContext(pingCtx); err != nil {
196 + _ = db.Close()
197 + return fmt.Errorf("error pinging SQL connection: %w", err)
198 + }
199 +
200 + c.db = db
201 + return nil
202 +}
203 +
204 +func (c *Collector) sqlTimeout() time.Duration {
205 + if c.SQLTimeout.Duration() > 0 {
206 + return c.SQLTimeout.Duration()
207 + }
208 + return time.Second
209 +}
210 +
211 +func (c *Collector) availableTopColumns(ctx context.Context) ([]ybColumnMeta, error) {
212 + available, err := c.detectPgStatStatementsColumns(ctx)
213 + if err != nil {
214 + return nil, fmt.Errorf("failed to detect available columns: %v", err)
215 + }
216 + cols := c.buildAvailableColumns(available)
217 + if len(cols) == 0 {
218 + return nil, fmt.Errorf("no queryable columns found in pg_stat_statements")
219 + }
220 + return cols, nil
221 +}
222 +
223 +func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
224 + ok, err := c.pgStatStatementsEnabled(ctx)
225 + if err != nil {
226 + return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("failed to check pg_stat_statements: %v", err)}
227 + }
228 + if !ok {
229 + return &module.FunctionResponse{
230 + Status: 503,
231 + Message: "pg_stat_statements extension is not installed in this database. " +
232 + "Run 'CREATE EXTENSION pg_stat_statements;' in the database the collector connects to.",
233 + }
234 + }
235 +
236 + cols, err := c.availableTopColumns(ctx)
237 + if err != nil {
238 + return &module.FunctionResponse{Status: 500, Message: err.Error()}
239 + }
240 +
241 + sortColumn = resolveYBSortColumn(cols, sortColumn)
242 + limit := c.TopQueriesLimit
243 + if limit <= 0 {
244 + limit = 500
245 + }
246 +
247 + query := buildYBTopQueriesSQL(cols, sortColumn)
248 + queryCtx, cancel := context.WithTimeout(ctx, c.sqlTimeout())
249 + defer cancel()
250 + rows, err := c.db.QueryContext(queryCtx, query, limit)
251 + if err != nil {
252 + if errors.Is(queryCtx.Err(), context.DeadlineExceeded) {
253 + return &module.FunctionResponse{Status: 504, Message: "query timed out"}
254 + }
255 + return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("query failed: %v", err)}
256 + }
257 + defer rows.Close()
258 +
259 + data, err := scanYBRows(rows, cols)
260 + if err != nil {
261 + return &module.FunctionResponse{Status: 500, Message: err.Error()}
262 + }
263 +
264 + return &module.FunctionResponse{
265 + Status: 200,
266 + Help: "Top SQL queries from pg_stat_statements. WARNING: Query text may contain unmasked literals (potential PII).",
267 + Columns: buildYBColumns(cols),
268 + Data: data,
269 + DefaultSortColumn: sortColumn,
270 + RequiredParams: []funcapi.ParamConfig{buildYBSortParam(cols)},
271 + Charts: ybTopQueriesCharts(cols),
272 + DefaultCharts: ybTopQueriesDefaultCharts(cols),
273 + GroupBy: ybTopQueriesGroupBy(cols),
274 + }
275 +}
276 +
277 +func (c *Collector) collectRunningQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
278 + sortColumn = resolveYBSortColumn(ybRunningColumns, sortColumn)
279 + limit := c.TopQueriesLimit
280 + if limit <= 0 {
281 + limit = 500
282 + }
283 +
284 + query := buildYBRunningQueriesSQL(sortColumn)
285 + queryCtx, cancel := context.WithTimeout(ctx, c.sqlTimeout())
286 + defer cancel()
287 + rows, err := c.db.QueryContext(queryCtx, query, limit)
288 + if err != nil {
289 + if errors.Is(queryCtx.Err(), context.DeadlineExceeded) {
290 + return &module.FunctionResponse{Status: 504, Message: "query timed out"}
291 + }
292 + return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("query failed: %v", err)}
293 + }
294 + defer rows.Close()
295 +
296 + data, err := scanYBRows(rows, ybRunningColumns)
297 + if err != nil {
298 + return &module.FunctionResponse{Status: 500, Message: err.Error()}
299 + }
300 +
301 + return &module.FunctionResponse{
302 + Status: 200,
303 + Help: "Currently running SQL statements from pg_stat_activity. WARNING: Query text may contain unmasked literals (potential PII).",
304 + Columns: buildYBColumns(ybRunningColumns),
305 + Data: data,
306 + DefaultSortColumn: sortColumn,
307 + RequiredParams: []funcapi.ParamConfig{buildYBSortParam(ybRunningColumns)},
308 + }
309 +}
310 +
311 +func buildYBSortParam(cols []ybColumnMeta) funcapi.ParamConfig {
312 + return funcapi.ParamConfig{
313 + ID: paramSort,
314 + Name: "Filter By",
315 + Help: "Select the primary sort column",
316 + Selection: funcapi.ParamSelect,
317 + Options: buildYBSortOptions(cols),
318 + UniqueView: true,
319 + }
320 +}
321 +
322 +func buildYBSortOptions(cols []ybColumnMeta) []funcapi.ParamOption {
323 + var sortOptions []funcapi.ParamOption
324 + sortDir := funcapi.FieldSortDescending
325 + for _, col := range cols {
326 + if !col.isSortOption {
327 + continue
328 + }
329 + opt := funcapi.ParamOption{
330 + ID: col.id,
331 + Column: col.id,
332 + Name: col.sortLabel,
333 + Sort: &sortDir,
334 + }
335 + if col.isDefaultSort {
336 + opt.Default = true
337 + }
338 + sortOptions = append(sortOptions, opt)
339 + }
340 + return sortOptions
341 +}
342 +
343 +func buildYBColumns(cols []ybColumnMeta) map[string]any {
344 + result := make(map[string]any, len(cols))
345 + for i, col := range cols {
346 + visual := visValue
347 + if col.dataType == ftDuration {
348 + visual = visBar
349 + }
350 + colDef := funcapi.Column{
351 + Index: i,
352 + Name: col.name,
353 + Type: col.dataType,
354 + Units: col.units,
355 + Visualization: visual,
356 + Sort: col.sortDir,
357 + Sortable: col.sortable,
358 + Sticky: col.sticky,
359 + Summary: col.summary,
360 + Filter: col.filter,
361 + FullWidth: col.fullWidth,
362 + Wrap: col.wrap,
363 + DefaultExpandedFilter: false,
364 + UniqueKey: col.uniqueKey,
365 + Visible: col.visible,
366 + ValueOptions: funcapi.ValueOptions{
367 + Transform: col.transform,
368 + DecimalPoints: col.decimalPoints,
369 + DefaultValue: nil,
370 + },
371 + }
372 + result[col.id] = colDef.BuildColumn()
373 + }
374 + return result
375 +}
376 +
377 +func resolveYBSortColumn(cols []ybColumnMeta, requested string) string {
378 + if requested != "" {
379 + for _, col := range cols {
380 + if col.id == requested && col.isSortOption {
381 + return col.id
382 + }
383 + }
384 + }
385 + for _, col := range cols {
386 + if col.isDefaultSort && col.isSortOption {
387 + return col.id
388 + }
389 + }
390 + for _, col := range cols {
391 + if col.isSortOption {
392 + return col.id
393 + }
394 + }
395 + if len(cols) > 0 {
396 + return cols[0].id
397 + }
398 + return ""
399 +}
400 +
401 +func buildYBTopQueriesSQL(cols []ybColumnMeta, sortColumn string) string {
402 + selectCols := make([]string, 0, len(cols))
403 + for _, col := range cols {
404 + selectCols = append(selectCols, fmt.Sprintf("%s AS %s", col.selectExpr, col.id))
405 + }
406 +
407 + return fmt.Sprintf(`
408 +SELECT %s
409 +FROM pg_stat_statements s
410 +JOIN pg_database d ON d.oid = s.dbid
411 +JOIN pg_user u ON u.usesysid = s.userid
412 +ORDER BY %s DESC NULLS LAST
413 +LIMIT $1`, strings.Join(selectCols, ", "), sortColumn)
414 +}
415 +
416 +func buildYBRunningQueriesSQL(sortColumn string) string {
417 + selectCols := make([]string, 0, len(ybRunningColumns))
418 + for _, col := range ybRunningColumns {
419 + selectCols = append(selectCols, fmt.Sprintf("%s AS %s", col.selectExpr, col.id))
420 + }
421 + return fmt.Sprintf(`
422 +SELECT %s
423 +FROM pg_stat_activity s
424 +WHERE s.state IS DISTINCT FROM 'idle'
425 +ORDER BY %s DESC NULLS LAST
426 +LIMIT $1`, strings.Join(selectCols, ", "), sortColumn)
427 +}
428 +
429 +func scanYBRows(rows *sql.Rows, cols []ybColumnMeta) ([][]any, error) {
430 + data := make([][]any, 0, 500)
431 +
432 + for rows.Next() {
433 + values := make([]any, len(cols))
434 + valuePtrs := make([]any, len(cols))
435 +
436 + for i, col := range cols {
437 + switch col.dataType {
438 + case ftString:
439 + var v sql.NullString
440 + values[i] = &v
441 + case ftInteger:
442 + var v sql.NullInt64
443 + values[i] = &v
444 + case ftFloat, ftDuration:
445 + var v sql.NullFloat64
446 + values[i] = &v
447 + default:
448 + var v any
449 + values[i] = &v
450 + }
451 + valuePtrs[i] = values[i]
452 + }
453 +
454 + if err := rows.Scan(valuePtrs...); err != nil {
455 + return nil, fmt.Errorf("row scan failed: %w", err)
456 + }
457 +
458 + row := make([]any, len(cols))
459 + for i, col := range cols {
460 + switch v := values[i].(type) {
461 + case *sql.NullString:
462 + if v.Valid {
463 + s := v.String
464 + if col.id == "query" {
465 + s = strmutil.TruncateText(s, ybMaxQueryTextLength)
466 + }
467 + row[i] = s
468 + } else {
469 + row[i] = ""
470 + }
471 + case *sql.NullInt64:
472 + if v.Valid {
473 + row[i] = v.Int64
474 + } else {
475 + row[i] = int64(0)
476 + }
477 + case *sql.NullFloat64:
478 + if v.Valid {
479 + row[i] = v.Float64
480 + } else {
481 + row[i] = float64(0)
482 + }
483 + default:
484 + row[i] = nil
485 + }
486 + }
487 +
488 + data = append(data, row)
489 + }
490 +
491 + if err := rows.Err(); err != nil {
492 + return nil, fmt.Errorf("rows iteration error: %w", err)
493 + }
494 +
495 + return data, nil
496 +}
497 +
498 +func ybTopQueriesCharts(cols []ybColumnMeta) map[string]module.ChartConfig {
499 + charts := make(map[string]module.ChartConfig)
500 + for _, col := range cols {
501 + if !col.isMetric || col.chartGroup == "" {
502 + continue
503 + }
504 + cfg, ok := charts[col.chartGroup]
505 + if !ok {
506 + title := col.chartTitle
507 + if title == "" {
508 + title = col.chartGroup
509 + }
510 + cfg = module.ChartConfig{Name: title, Type: "stacked-bar"}
511 + }
512 + cfg.Columns = append(cfg.Columns, col.id)
513 + charts[col.chartGroup] = cfg
514 + }
515 + return charts
516 +}
517 +
518 +func ybTopQueriesDefaultCharts(cols []ybColumnMeta) [][]string {
519 + label := primaryYBLabel(cols)
520 + if label == "" {
521 + return nil
522 + }
523 + chartGroups := defaultYBChartGroups(cols)
524 + out := make([][]string, 0, len(chartGroups))
525 + for _, group := range chartGroups {
526 + out = append(out, []string{group, label})
527 + }
528 + return out
529 +}
530 +
531 +func ybTopQueriesGroupBy(cols []ybColumnMeta) map[string]module.GroupByConfig {
532 + groupBy := make(map[string]module.GroupByConfig)
533 + for _, col := range cols {
534 + if !col.isLabel {
535 + continue
536 + }
537 + groupBy[col.id] = module.GroupByConfig{
538 + Name: "Group by " + col.name,
539 + Columns: []string{col.id},
540 + }
541 + }
542 + return groupBy
543 +}
544 +
545 +func primaryYBLabel(cols []ybColumnMeta) string {
546 + for _, col := range cols {
547 + if col.isPrimary {
548 + return col.id
549 + }
550 + }
551 + for _, col := range cols {
552 + if col.isLabel {
553 + return col.id
554 + }
555 + }
556 + return ""
557 +}
558 +
559 +func defaultYBChartGroups(cols []ybColumnMeta) []string {
560 + groups := make([]string, 0)
561 + seen := make(map[string]bool)
562 + for _, col := range cols {
563 + if !col.isMetric || col.chartGroup == "" || !col.isDefaultChart {
564 + continue
565 + }
566 + if !seen[col.chartGroup] {
567 + seen[col.chartGroup] = true
568 + groups = append(groups, col.chartGroup)
569 + }
570 + }
571 + if len(groups) > 0 {
572 + return groups
573 + }
574 + for _, col := range cols {
575 + if !col.isMetric || col.chartGroup == "" {
576 + continue
577 + }
578 + if !seen[col.chartGroup] {
579 + seen[col.chartGroup] = true
580 + groups = append(groups, col.chartGroup)
581 + }
582 + }
583 + return groups
584 +}
585 +
586 +func (c *Collector) pgStatStatementsEnabled(ctx context.Context) (bool, error) {
587 + query := `SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements'`
588 + queryCtx, cancel := context.WithTimeout(ctx, c.sqlTimeout())
589 + defer cancel()
590 + var exists int
591 + if err := c.db.QueryRowContext(queryCtx, query).Scan(&exists); err != nil {
592 + if errors.Is(queryCtx.Err(), context.DeadlineExceeded) {
593 + return false, queryCtx.Err()
594 + }
595 + if errors.Is(err, sql.ErrNoRows) {
596 + return false, nil
597 + }
598 + return false, err
599 + }
600 + return true, nil
601 +}
602 +
603 +func (c *Collector) detectPgStatStatementsColumns(ctx context.Context) (map[string]bool, error) {
604 + c.pgStatStatementsMu.RLock()
605 + if c.pgStatStatementsColumns != nil {
606 + cols := c.pgStatStatementsColumns
607 + c.pgStatStatementsMu.RUnlock()
608 + return cols, nil
609 + }
610 + c.pgStatStatementsMu.RUnlock()
611 +
612 + c.pgStatStatementsMu.Lock()
613 + defer c.pgStatStatementsMu.Unlock()
614 +
615 + if c.pgStatStatementsColumns != nil {
616 + return c.pgStatStatementsColumns, nil
617 + }
618 +
619 + query := `
620 + SELECT column_name
621 + FROM information_schema.columns
622 + WHERE table_name = 'pg_stat_statements'
623 + AND table_schema = 'public'
624 + `
625 + queryCtx, cancel := context.WithTimeout(ctx, c.sqlTimeout())
626 + defer cancel()
627 +
628 + rows, err := c.db.QueryContext(queryCtx, query)
629 + if err != nil {
630 + return nil, fmt.Errorf("failed to query columns: %v", err)
631 + }
632 + defer rows.Close()
633 +
634 + cols := make(map[string]bool)
635 + for rows.Next() {
636 + var colName string
637 + if err := rows.Scan(&colName); err != nil {
638 + return nil, fmt.Errorf("failed to scan column name: %v", err)
639 + }
640 + cols[colName] = true
641 + }
642 +
643 + if err := rows.Err(); err != nil {
644 + return nil, fmt.Errorf("rows iteration error: %v", err)
645 + }
646 +
647 + c.pgStatStatementsColumns = cols
648 + return cols, nil
649 +}
650 +
651 +func (c *Collector) buildAvailableColumns(availableCols map[string]bool) []ybColumnMeta {
652 + result := make([]ybColumnMeta, 0, len(ybTopColumns))
653 + for _, col := range ybTopColumns {
654 + if col.isJoinColumn {
655 + result = append(result, col)
656 + continue
657 + }
658 + actual, ok := resolveYBColumn(col.selectExpr, availableCols)
659 + if !ok {
660 + continue
661 + }
662 + colCopy := col
663 + colCopy.selectExpr = actual
664 + result = append(result, colCopy)
665 + }
666 + return result
667 +}
668 +
669 +func resolveYBColumn(expr string, availableCols map[string]bool) (string, bool) {
670 + colName := expr
671 + if idx := strings.LastIndex(colName, "."); idx != -1 {
672 + colName = colName[idx+1:]
673 + }
674 +
675 + castSuffix := ""
676 + if idx := strings.Index(colName, "::"); idx != -1 {
677 + castSuffix = colName[idx:]
678 + colName = colName[:idx]
679 + }
680 +
681 + actual := colName
682 + switch colName {
683 + case "total_time":
684 + if availableCols["total_exec_time"] {
685 + actual = "total_exec_time"
686 + }
687 + case "mean_time":
688 + if availableCols["mean_exec_time"] {
689 + actual = "mean_exec_time"
690 + }
691 + case "min_time":
692 + if availableCols["min_exec_time"] {
693 + actual = "min_exec_time"
694 + }
695 + case "max_time":
696 + if availableCols["max_exec_time"] {
697 + actual = "max_exec_time"
698 + }
699 + case "stddev_time":
700 + if availableCols["stddev_exec_time"] {
701 + actual = "stddev_exec_time"
702 + }
703 + }
704 +
705 + if !availableCols[actual] {
706 + return "", false
707 + }
708 +
709 + if actual == colName {
710 + return expr, true
711 + }
712 + return strings.Replace(expr, colName+castSuffix, actual+castSuffix, 1), true
713 +}
src/go/plugin/go.d/collector/yugabytedb/functions_test.go new
+59
@@ -0,0 +1,59 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package yugabytedb
4 +
5 +import (
6 + "testing"
7 +
8 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
9 + "github.com/stretchr/testify/assert"
10 +)
11 +
12 +func TestYugabyteDBMethods(t *testing.T) {
13 + methods := yugabyteMethods()
14 +
15 + require := assert.New(t)
16 + require.Len(methods, 2)
17 +
18 + for _, method := range methods {
19 + require.NotEmpty(method.ID)
20 + require.NotEmpty(method.Name)
21 + require.NotEmpty(method.RequiredParams)
22 +
23 + var sortParam *funcapi.ParamConfig
24 + for i := range method.RequiredParams {
25 + if method.RequiredParams[i].ID == "__sort" {
26 + sortParam = &method.RequiredParams[i]
27 + break
28 + }
29 + }
30 + require.NotNil(sortParam, "expected __sort required param")
31 + require.NotEmpty(sortParam.Options)
32 + }
33 +}
34 +
35 +func TestYugabyteDBTopColumns_HasRequiredColumns(t *testing.T) {
36 + required := []string{"queryId", "query", "calls", "totalTime"}
37 +
38 + uiKeys := make(map[string]bool)
39 + for _, col := range ybTopColumns {
40 + uiKeys[col.id] = true
41 + }
42 +
43 + for _, key := range required {
44 + assert.True(t, uiKeys[key], "column %s should be defined", key)
45 + }
46 +}
47 +
48 +func TestYugabyteDBRunningColumns_HasRequiredColumns(t *testing.T) {
49 + required := []string{"pid", "query", "elapsedMs"}
50 +
51 + uiKeys := make(map[string]bool)
52 + for _, col := range ybRunningColumns {
53 + uiKeys[col.id] = true
54 + }
55 +
56 + for _, key := range required {
57 + assert.True(t, uiKeys[key], "column %s should be defined", key)
58 + }
59 +}
src/go/plugin/go.d/collector/yugabytedb/metadata.yaml
+28 -1
@@ -28,6 +28,8 @@ modules:
28 This collector monitors the activity and performance of YugabyteDB servers.
29 method_description: |
30 It sends HTTP requests to the YugabyteDB [metric endpoints](https://docs.yugabyte.com/preview/launch-and-manage/monitor-and-alert/metrics/#metric-endpoints).
31 +
32 + It also provides `top-queries` and `running-queries` functions using `pg_stat_statements` and `pg_stat_activity` from YSQL.
33 default_behavior:
34 auto_detection:
35 description: |
@@ -43,7 +45,10 @@ modules:
45 performance_impact:
46 description: ""
47 additional_permissions:
46 - description: ""
48 + description: |
49 + The `top-queries` function requires the `pg_stat_statements` extension to be installed in the target database.
50 +
51 + Viewing all running queries via `pg_stat_activity` may require elevated privileges (e.g., `pg_read_all_stats`).
52 multi_instance: true
53 supported_platforms:
54 include: []
@@ -82,6 +87,21 @@ modules:
87 default_value: 1
88 required: false
89 group: Target
90 + - name: dsn
91 + description: SQL DSN used by `top-queries` and `running-queries` functions.
92 + default_value: ""
93 + required: false
94 + group: Query Functions
95 + - name: sql_timeout
96 + description: SQL query timeout (seconds) for query functions.
97 + default_value: 1
98 + required: false
99 + group: Query Functions
100 + - name: top_queries_limit
101 + description: Maximum number of rows returned by the `top-queries` and `running-queries` functions.
102 + default_value: 500
103 + required: false
104 + group: Limits
105
106 - name: username
107 description: Username for Basic HTTP authentication.
@@ -183,6 +203,13 @@ modules:
203 # url: http://127.0.0.1:9000/prometheus-metrics # Tablet Server
204 # url: http://127.0.0.1:12000/prometheus-metrics # YCQL
205 # url: http://127.0.0.1:13000/prometheus-metrics # YSQL
206 + - name: Top queries
207 + description: Enable SQL query functions (YSQL).
208 + config: |
209 + jobs:
210 + - name: local
211 + url: http://127.0.0.1:7000/prometheus-metrics
212 + dsn: postgres://yugabyte@127.0.0.1:5433/yugabyte?sslmode=disable
213 - name: HTTP authentication
214 description: Basic HTTP authentication.
215 config: |
src/go/plugin/go.d/collector/yugabytedb/sql.go new
+5
@@ -0,0 +1,5 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package yugabytedb
4 +
5 +import _ "github.com/jackc/pgx/v5/stdlib"
src/go/tools/functions-validation/README.md
+17 -2
@@ -5,7 +5,7 @@
5 - Use `go.d.plugin --function` with the configs in `./config`.
6 - Config files live under `./config/go.d`.
7 - Validate output against the embedded schema.
8 -- Use `./e2e.sh` for automated end-to-end checks in `/tmp`.
8 +- Use `./e2e.sh` for automated end-to-end checks in `/tmp` (runs per-DB scripts).
9
10 ## Start containers
11 ```
@@ -42,9 +42,24 @@ src/go/go.d.plugin \
42 ```
43 ./e2e.sh
44 ```
45 +```
46 +./e2e.sh --jobs 4
47 +```
48 +```
49 +./e2e.sh --only postgres,mysql
50 +```
51 +```
52 +./e2e.sh --list
53 +```
54 +
55 +## Per-DB E2E (single DB)
56 +```
57 +./e2e/postgres.sh
58 +```
59
60 ### Behavior
47 -- Creates a workspace under `/tmp` and runs Docker Compose there.
61 +- Each DB script creates a workspace under `/tmp` and runs Docker Compose there.
62 +- Ports are auto-selected per run to avoid collisions.
63 - Builds `go.d.plugin` into the `/tmp` workspace.
64 - Validates schema **and** that data rows are returned for top-queries.
65 - Cleans up the `/tmp` workspace on success; keeps it on failure for debugging.
src/go/tools/functions-validation/config/go.d/clickhouse.conf new
+6
@@ -0,0 +1,6 @@
1 +jobs:
2 + - name: local
3 + url: http://127.0.0.1:8123
4 + username: default
5 + password: netdata
6 + top_queries_limit: 50
src/go/tools/functions-validation/config/go.d/cockroachdb.conf new
+6
@@ -0,0 +1,6 @@
1 +jobs:
2 + - name: local
3 + url: "http://127.0.0.1:8080/_status/vars"
4 + dsn: "postgres://root@127.0.0.1:26258/defaultdb?sslmode=disable"
5 + sql_timeout: 2
6 + top_queries_limit: 100
src/go/tools/functions-validation/config/go.d/couchbase.conf new
+7
@@ -0,0 +1,7 @@
1 +jobs:
2 + - name: local
3 + url: http://127.0.0.1:8091
4 + query_url: http://127.0.0.1:8093
5 + username: Administrator
6 + password: password
7 + top_queries_limit: 50
src/go/tools/functions-validation/config/go.d/elasticsearch.conf new
+4
@@ -0,0 +1,4 @@
1 +jobs:
2 + - name: local
3 + url: http://127.0.0.1:9200
4 + top_queries_limit: 50
src/go/tools/functions-validation/config/go.d/oracledb.conf new
+5
@@ -0,0 +1,5 @@
1 +jobs:
2 + - name: local
3 + dsn: oracle://netdata:Netdata123!@127.0.0.1:1521/FREEPDB1
4 + timeout: 5
5 + top_queries_limit: 50
src/go/tools/functions-validation/config/go.d/proxysql.conf new
+4
@@ -0,0 +1,4 @@
1 +jobs:
2 + - name: local
3 + dsn: "stats:stats@tcp(127.0.0.1:6032)/"
4 + top_queries_limit: 50
src/go/tools/functions-validation/config/go.d/redis.conf new
+4
@@ -0,0 +1,4 @@
1 +jobs:
2 + - name: local
3 + address: redis://@127.0.0.1:6379
4 + top_queries_limit: 50
src/go/tools/functions-validation/config/go.d/rethinkdb.conf new
+4
@@ -0,0 +1,4 @@
1 +jobs:
2 + - name: local
3 + address: 127.0.0.1:28015
4 + top_queries_limit: 50
src/go/tools/functions-validation/config/go.d/yugabytedb.conf new
+6
@@ -0,0 +1,6 @@
1 +jobs:
2 + - name: local
3 + url: "http://127.0.0.1:7000/prometheus-metrics"
4 + dsn: "postgres://yugabyte@127.0.0.1:5433/yugabyte?sslmode=disable"
5 + sql_timeout: 2
6 + top_queries_limit: 100
src/go/tools/functions-validation/docker-compose.yml
+259 -4
@@ -7,7 +7,7 @@ services:
7 POSTGRES_DB: netdata
8 command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements"]
9 ports:
10 - - "5432:5432"
10 + - "${POSTGRES_PORT:-5432}:5432"
11 volumes:
12 - ./seed/postgres/init.sql:/docker-entrypoint-initdb.d/init.sql:ro
13 healthcheck:
@@ -25,7 +25,7 @@ services:
25 MYSQL_PASSWORD: netdata
26 command: ["--performance_schema=ON"]
27 ports:
28 - - "3306:3306"
28 + - "${MYSQL_PORT:-3306}:3306"
29 volumes:
30 - ./seed/mysql/init.sql:/docker-entrypoint-initdb.d/init.sql:ro
31 healthcheck:
@@ -40,7 +40,7 @@ services:
40 ACCEPT_EULA: "Y"
41 MSSQL_SA_PASSWORD: "Netdata123!"
42 ports:
43 - - "1433:1433"
43 + - "${MSSQL_PORT:-1433}:1433"
44 healthcheck:
45 test: ["CMD-SHELL", "if [ -x /opt/mssql-tools18/bin/sqlcmd ]; then /opt/mssql-tools18/bin/sqlcmd -C -S localhost -U sa -P $$MSSQL_SA_PASSWORD -Q \"SELECT 1\" > /dev/null; else /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P $$MSSQL_SA_PASSWORD -Q \"SELECT 1\" > /dev/null; fi"]
46 interval: 10s
@@ -65,7 +65,7 @@ services:
65 MONGO_INITDB_ROOT_USERNAME: root
66 MONGO_INITDB_ROOT_PASSWORD: rootpw
67 ports:
68 - - "27017:27017"
68 + - "${MONGO_PORT:-27017}:27017"
69 volumes:
70 - ./seed/mongodb/init.js:/docker-entrypoint-initdb.d/init.js:ro
71 healthcheck:
@@ -86,3 +86,258 @@ services:
86 - ./seed/mongodb/init.js:/seed/init.js:ro
87 - ./seed/mongodb/init.sh:/seed/init.sh:ro
88 entrypoint: ["/bin/bash", "/seed/init.sh"]
89 +
90 + redis:
91 + image: redis:7
92 + ports:
93 + - "${REDIS_PORT:-6379}:6379"
94 + healthcheck:
95 + test: ["CMD", "redis-cli", "ping"]
96 + interval: 5s
97 + timeout: 5s
98 + retries: 10
99 +
100 + redis-init:
101 + image: redis:7
102 + depends_on:
103 + redis:
104 + condition: service_healthy
105 + volumes:
106 + - ./seed/redis/init.sh:/seed/init.sh:ro
107 + entrypoint: ["/bin/bash", "/seed/init.sh"]
108 +
109 + clickhouse:
110 + image: clickhouse/clickhouse-server:24.3
111 + environment:
112 + CLICKHOUSE_USER: default
113 + CLICKHOUSE_PASSWORD: netdata
114 + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1"
115 + ports:
116 + - "${CLICKHOUSE_HTTP_PORT:-8123}:8123"
117 + healthcheck:
118 + test: ["CMD-SHELL", "clickhouse-client --host 127.0.0.1 --user \"$${CLICKHOUSE_USER}\" --password \"$${CLICKHOUSE_PASSWORD}\" --query 'SELECT 1' > /dev/null"]
119 + interval: 5s
120 + timeout: 5s
121 + retries: 10
122 +
123 + clickhouse-init:
124 + image: clickhouse/clickhouse-server:24.3
125 + depends_on:
126 + clickhouse:
127 + condition: service_healthy
128 + environment:
129 + CH_USER: default
130 + CH_PASSWORD: netdata
131 + volumes:
132 + - ./seed/clickhouse/init.sh:/seed/init.sh:ro
133 + entrypoint: ["/bin/bash", "/seed/init.sh"]
134 +
135 + elasticsearch:
136 + image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0
137 + environment:
138 + discovery.type: single-node
139 + xpack.security.enabled: "false"
140 + ES_JAVA_OPTS: "-Xms512m -Xmx512m"
141 + ports:
142 + - "${ELASTICSEARCH_PORT:-9200}:9200"
143 + healthcheck:
144 + test: ["CMD-SHELL", "curl -fsS http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=5s > /dev/null"]
145 + interval: 10s
146 + timeout: 5s
147 + retries: 12
148 +
149 + elasticsearch-init:
150 + image: curlimages/curl:8.5.0
151 + depends_on:
152 + elasticsearch:
153 + condition: service_healthy
154 + volumes:
155 + - ./seed/elasticsearch/init.sh:/seed/init.sh:ro
156 + entrypoint: ["/bin/sh", "/seed/init.sh"]
157 +
158 + elasticsearch-searcher:
159 + image: curlimages/curl:8.5.0
160 + depends_on:
161 + elasticsearch-init:
162 + condition: service_completed_successfully
163 + volumes:
164 + - ./seed/elasticsearch/searcher.sh:/seed/searcher.sh:ro
165 + entrypoint: ["/bin/sh", "/seed/searcher.sh"]
166 +
167 + couchbase:
168 + image: couchbase/server:7.2.5
169 + ports:
170 + - "${COUCHBASE_HTTP_PORT:-8091}:8091"
171 + - "${COUCHBASE_QUERY_PORT:-8093}:8093"
172 + ulimits:
173 + nofile:
174 + soft: 200000
175 + hard: 200000
176 + healthcheck:
177 + test: ["CMD-SHELL", "code=$(curl -s -o /dev/null -w '%{http_code}' http://localhost:8091/pools); [ \"$$code\" = \"200\" ] || [ \"$$code\" = \"401\" ]"]
178 + interval: 10s
179 + timeout: 5s
180 + retries: 20
181 +
182 + couchbase-init:
183 + image: couchbase/server:7.2.5
184 + depends_on:
185 + couchbase:
186 + condition: service_healthy
187 + environment:
188 + CB_HOST: couchbase
189 + CB_ADMIN: Administrator
190 + CB_PASS: password
191 + volumes:
192 + - ./seed/couchbase/init.sh:/seed/init.sh:ro
193 + entrypoint: ["/bin/bash", "/seed/init.sh"]
194 +
195 + proxysql:
196 + image: proxysql/proxysql:2.6.6
197 + ports:
198 + - "${PROXYSQL_ADMIN_PORT:-6032}:6032"
199 + - "${PROXYSQL_MYSQL_PORT:-6033}:6033"
200 + volumes:
201 + - proxysql-data:/var/lib/proxysql
202 + - ./seed/proxysql/proxysql.cnf:/etc/proxysql.cnf:ro
203 +
204 + proxysql-init:
205 + image: mysql:8.0
206 + network_mode: "service:proxysql"
207 + depends_on:
208 + proxysql:
209 + condition: service_started
210 + mysql:
211 + condition: service_healthy
212 + environment:
213 + PROXY_HOST: 127.0.0.1
214 + ADMIN_USER: admin
215 + ADMIN_PASS: admin
216 + BACKEND_HOST: mysql
217 + BACKEND_PORT: 3306
218 + volumes:
219 + - ./seed/proxysql/init.sh:/seed/init.sh:ro
220 + entrypoint: ["/bin/bash", "/seed/init.sh"]
221 +
222 + cockroachdb:
223 + image: cockroachdb/cockroach:v24.1.0
224 + command:
225 + [
226 + "start-single-node",
227 + "--insecure",
228 + "--http-addr",
229 + "0.0.0.0:8080",
230 + "--listen-addr",
231 + "127.0.0.1:26257",
232 + "--sql-addr",
233 + "0.0.0.0:26258"
234 + ]
235 + ports:
236 + - "${COCKROACH_HTTP_PORT:-8080}:8080"
237 + - "${COCKROACH_SQL_PORT:-26258}:26258"
238 + healthcheck:
239 + test: ["CMD-SHELL", "cockroach sql --insecure --host=localhost:26258 -e \"SELECT 1\" > /dev/null"]
240 + interval: 10s
241 + timeout: 5s
242 + retries: 15
243 +
244 + cockroachdb-seed:
245 + image: cockroachdb/cockroach:v24.1.0
246 + depends_on:
247 + cockroachdb:
248 + condition: service_healthy
249 + volumes:
250 + - ./seed/cockroachdb/seed.sh:/seed/seed.sh:ro
251 + - ./seed/cockroachdb/seed.sql:/seed/seed.sql:ro
252 + entrypoint: ["/bin/sh", "/seed/seed.sh"]
253 +
254 + cockroachdb-sleep:
255 + image: cockroachdb/cockroach:v24.1.0
256 + depends_on:
257 + cockroachdb:
258 + condition: service_healthy
259 + volumes:
260 + - ./seed/cockroachdb/sleep.sh:/seed/sleep.sh:ro
261 + - ./seed/cockroachdb/sleep.sql:/seed/sleep.sql:ro
262 + entrypoint: ["/bin/sh", "/seed/sleep.sh"]
263 +
264 + yugabytedb:
265 + image: yugabytedb/yugabyte:2.21.1.0-b271
266 + command: ["/home/yugabyte/bin/yugabyted", "start", "--daemon=false"]
267 + ports:
268 + - "${YUGABYTE_MASTER_PORT:-7000}:7000"
269 + - "${YUGABYTE_YSQL_PORT:-5433}:5433"
270 + healthcheck:
271 + test: ["CMD-SHELL", "/home/yugabyte/bin/ysqlsh -h \"$(hostname -i)\" -U yugabyte -d yugabyte -c \"SELECT 1\" > /dev/null"]
272 + interval: 10s
273 + timeout: 5s
274 + retries: 20
275 +
276 + yugabytedb-seed:
277 + image: yugabytedb/yugabyte:2.21.1.0-b271
278 + depends_on:
279 + yugabytedb:
280 + condition: service_healthy
281 + volumes:
282 + - ./seed/yugabytedb/seed.sh:/seed/seed.sh:ro
283 + - ./seed/yugabytedb/seed.sql:/seed/seed.sql:ro
284 + entrypoint: ["/bin/sh", "/seed/seed.sh"]
285 +
286 + yugabytedb-sleep:
287 + image: yugabytedb/yugabyte:2.21.1.0-b271
288 + depends_on:
289 + yugabytedb:
290 + condition: service_healthy
291 + volumes:
292 + - ./seed/yugabytedb/sleep.sh:/seed/sleep.sh:ro
293 + - ./seed/yugabytedb/sleep.sql:/seed/sleep.sql:ro
294 + entrypoint: ["/bin/sh", "/seed/sleep.sh"]
295 +
296 + oracledb:
297 + image: gvenzl/oracle-free:latest
298 + environment:
299 + ORACLE_PASSWORD: "Netdata123!"
300 + APP_USER: netdata
301 + APP_USER_PASSWORD: "Netdata123!"
302 + ports:
303 + - "${ORACLE_PORT:-1521}:1521"
304 + volumes:
305 + - ./seed/oracledb/init.sql:/container-entrypoint-initdb.d/init.sql:ro
306 + healthcheck:
307 + test: ["CMD-SHELL", "echo 'SELECT 1 FROM dual;' | sqlplus -L -s \"sys/$$ORACLE_PASSWORD@//localhost:1521/FREEPDB1 as sysdba\" > /dev/null"]
308 + interval: 10s
309 + timeout: 5s
310 + retries: 20
311 +
312 + oracledb-seed:
313 + image: gvenzl/oracle-free:latest
314 + depends_on:
315 + oracledb:
316 + condition: service_healthy
317 + volumes:
318 + - ./seed/oracledb/seed.sh:/seed/seed.sh:ro
319 + - ./seed/oracledb/seed.sql:/seed/seed.sql:ro
320 + entrypoint: ["/bin/bash", "/seed/seed.sh"]
321 +
322 + oracledb-sleep:
323 + image: gvenzl/oracle-free:latest
324 + depends_on:
325 + oracledb:
326 + condition: service_healthy
327 + volumes:
328 + - ./seed/oracledb/sleep.sh:/seed/sleep.sh:ro
329 + - ./seed/oracledb/sleep.sql:/seed/sleep.sql:ro
330 + entrypoint: ["/bin/bash", "/seed/sleep.sh"]
331 +
332 + rethinkdb:
333 + image: rethinkdb:2.4
334 + ports:
335 + - "${RETHINKDB_PORT:-28015}:28015"
336 + healthcheck:
337 + test: ["CMD-SHELL", "rethinkdb --version > /dev/null"]
338 + interval: 5s
339 + timeout: 5s
340 + retries: 10
341 +
342 +volumes:
343 + proxysql-data:
src/go/tools/functions-validation/e2e.sh
+129 -99
@@ -3,7 +3,6 @@ set -euo pipefail
3
4 # Colors for output
5 RED='\033[0;31m'
6 -GREEN='\033[0;32m'
6 YELLOW='\033[1;33m'
7 GRAY='\033[0;90m'
8 NC='\033[0m' # No Color
@@ -11,10 +10,10 @@ NC='\033[0m' # No Color
10 # Execute command with visibility
11 run() {
12 # Print the command being executed
14 - printf >&2 "${GRAY}$(pwd) >${NC} "
15 - printf >&2 "${YELLOW}"
13 + printf >&2 '%s%s >%s ' "$GRAY" "$(pwd)" "$NC"
14 + printf >&2 '%s' "$YELLOW"
15 printf >&2 "%q " "$@"
17 - printf >&2 "${NC}\n"
16 + printf >&2 '%s\n' "$NC"
17
18 # Execute the command
19 set +e
@@ -31,113 +30,144 @@ run() {
30 fi
31 }
32
34 -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
35 -REPO_ROOT="$(cd "$SCRIPT_DIR/../../../.." && pwd)"
36 -WORKDIR="$(mktemp -d /tmp/netdata-functions-e2e.XXXXXX)"
37 -PROJECT_SUFFIX="$(basename "$WORKDIR")"
38 -PROJECT_SUFFIX="$(printf '%s' "$PROJECT_SUFFIX" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9_-' '-')"
39 -PROJECT_SUFFIX="${PROJECT_SUFFIX%-}"
40 -PROJECT="netdata-func-e2e-$PROJECT_SUFFIX"
41 -COMPOSE=(docker compose -f "$WORKDIR/docker-compose.yml" -p "$PROJECT")
42 -COMPOSE_STARTED=""
43 -
44 -cleanup() {
45 - local exit_code=$?
46 - set +e
47 - if [ -n "$COMPOSE_STARTED" ]; then
48 - run "${COMPOSE[@]}" down -v --remove-orphans
49 - fi
50 - if [ "$exit_code" -eq 0 ]; then
51 - run rm -rf "$WORKDIR"
52 - else
53 - echo "E2E failed. Keeping workspace: $WORKDIR" >&2
54 - fi
55 - exit $exit_code
33 +# Execute command in background with visibility
34 +LAST_BG_PID=""
35 +run_bg() {
36 + printf >&2 '%s%s >%s ' "$GRAY" "$(pwd)" "$NC"
37 + printf >&2 '%s' "$YELLOW"
38 + printf >&2 "%q " "$@"
39 + printf >&2 '%s\n' "$NC"
40 + "$@" &
41 + LAST_BG_PID=$!
42 }
57 -trap cleanup EXIT
43
59 -wait_healthy() {
60 - local service="$1"
61 - local timeout="${2:-60}"
62 - local start=$SECONDS
63 -
64 - while true; do
65 - local cid
66 - cid=$("${COMPOSE[@]}" ps -q "$service")
67 - if [ -z "$cid" ]; then
68 - if [ $((SECONDS - start)) -ge "$timeout" ]; then
69 - echo "No container found for service: $service" >&2
70 - return 1
71 - fi
72 - sleep 2
73 - continue
74 - fi
75 -
76 - local status
77 - status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$cid")"
78 - if [ "$status" = "healthy" ]; then
79 - return 0
80 - fi
81 - if [ $((SECONDS - start)) -ge "$timeout" ]; then
82 - echo "Timed out waiting for $service to be healthy" >&2
83 - return 1
84 - fi
85 - sleep 2
86 - done
44 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
45 +E2E_DIR="$SCRIPT_DIR/e2e"
46 +
47 +DBS=(postgres mysql mssql mongodb redis clickhouse elasticsearch couchbase proxysql cockroachdb yugabytedb oracledb rethinkdb)
48 +JOBS=1
49 +ONLY=""
50 +
51 +usage() {
52 + cat <<'USAGE'
53 +Usage: ./e2e.sh [--only db1,db2] [--jobs N] [--list]
54 +
55 +Options:
56 + --only Comma-separated list of DBs to run (e.g. postgres,mysql)
57 + --jobs Max number of concurrent DB runs (default: 1)
58 + --list Show available DBs
59 + --help Show this help
60 +USAGE
61 }
62
89 -run cp -a "$SCRIPT_DIR/docker-compose.yml" "$SCRIPT_DIR/seed" "$SCRIPT_DIR/config" "$WORKDIR/"
90 -
91 -run "${COMPOSE[@]}" up -d
92 -COMPOSE_STARTED="yes"
93 -
94 -wait_healthy postgres 90
95 -wait_healthy mysql 90
96 -wait_healthy mssql 120
97 -wait_healthy mongo 90
98 -
99 -run "${COMPOSE[@]}" run --rm mongo-init
63 +list_dbs() {
64 + printf '%s\n' "${DBS[@]}"
65 +}
66
101 -run bash -c "cd \"$REPO_ROOT/src/go\" && go build -o \"$WORKDIR/go.d.plugin\" ./cmd/godplugin"
67 +while [ $# -gt 0 ]; do
68 + case "$1" in
69 + --only)
70 + ONLY="${2:-}"
71 + if [ -z "$ONLY" ]; then
72 + echo "--only requires a value" >&2
73 + usage
74 + exit 1
75 + fi
76 + shift 2
77 + ;;
78 + --jobs)
79 + JOBS="${2:-}"
80 + if [ -z "$JOBS" ]; then
81 + echo "--jobs requires a value" >&2
82 + usage
83 + exit 1
84 + fi
85 + shift 2
86 + ;;
87 + --list)
88 + list_dbs
89 + exit 0
90 + ;;
91 + --help|-h)
92 + usage
93 + exit 0
94 + ;;
95 + *)
96 + echo "Unknown option: $1" >&2
97 + usage
98 + exit 1
99 + ;;
100 + esac
101 +done
102 +
103 +if ! [[ "$JOBS" =~ ^[0-9]+$ ]] || [ "$JOBS" -le 0 ]; then
104 + echo "--jobs must be a positive integer" >&2
105 + exit 1
106 +fi
107 +
108 +if [ -n "$ONLY" ]; then
109 + IFS=',' read -r -a DBS <<< "${ONLY// /}"
110 +fi
111 +
112 +for db in "${DBS[@]}"; do
113 + if [ ! -f "$E2E_DIR/${db}.sh" ]; then
114 + echo "Unknown DB script: $db" >&2
115 + exit 1
116 + fi
117 +done
118
103 -validate() {
104 - local input="$1"
105 - shift
106 - (cd "$REPO_ROOT/src/go" && run go run ./tools/functions-validation/validate --input "$input" "$@")
107 -}
119 +pids=()
120 +names=()
121 +failures=()
122
109 -run_info() {
110 - local module="$1"
111 - local output="$WORKDIR/${module}-info.json"
112 - run "$WORKDIR/go.d.plugin" \
113 - --config-dir "$WORKDIR/config" \
114 - --function "${module}:top-queries" \
115 - --function-args info \
116 - > "$output"
117 - validate "$output"
123 +wait_for_any() {
124 + local i pid status db
125 + while true; do
126 + for i in "${!pids[@]}"; do
127 + pid="${pids[$i]}"
128 + if ! kill -0 "$pid" 2>/dev/null; then
129 + set +e
130 + wait "$pid"
131 + status=$?
132 + set -e
133 + db="${names[$i]}"
134 + if [ $status -ne 0 ]; then
135 + failures+=("$db")
136 + fi
137 + unset 'pids[i]' 'names[i]'
138 + pids=("${pids[@]}")
139 + names=("${names[@]}")
140 + return 0
141 + fi
142 + done
143 + sleep 0.2
144 + done
145 }
146
120 -run_top_queries() {
121 - local module="$1"
122 - local output="$WORKDIR/${module}-top-queries.json"
123 - run "$WORKDIR/go.d.plugin" \
124 - --config-dir "$WORKDIR/config" \
125 - --function "${module}:top-queries" \
126 - --function-args __job:local \
127 - > "$output"
128 - validate "$output" --min-rows 1
147 +start_job() {
148 + local db="$1"
149 + local script="$E2E_DIR/${db}.sh"
150 + local pid
151 + run_bg bash "$script"
152 + pid="$LAST_BG_PID"
153 + pids+=("$pid")
154 + names+=("$db")
155 }
156
131 -run_info postgres
132 -run_top_queries postgres
133 -
134 -run_info mysql
135 -run_top_queries mysql
157 +for db in "${DBS[@]}"; do
158 + start_job "$db"
159 + while [ "${#pids[@]}" -ge "$JOBS" ]; do
160 + wait_for_any
161 + done
162 +done
163
137 -run_info mssql
138 -run_top_queries mssql
164 +while [ "${#pids[@]}" -gt 0 ]; do
165 + wait_for_any
166 +done
167
140 -run_info mongodb
141 -run_top_queries mongodb
168 +if [ "${#failures[@]}" -ne 0 ]; then
169 + printf >&2 "%s\n" "E2E failures: ${failures[*]}"
170 + exit 1
171 +fi
172
173 echo "E2E checks passed." >&2
src/go/tools/functions-validation/e2e/clickhouse.sh new
+23
@@ -0,0 +1,23 @@
1 +#!/usr/bin/env bash
2 +set -euo pipefail
3 +
4 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
5 +# shellcheck disable=SC1091
6 +. "$SCRIPT_DIR/lib.sh"
7 +
8 +init_workspace "clickhouse"
9 +trap cleanup EXIT
10 +
11 +CLICKHOUSE_HTTP_PORT="$(reserve_port)"
12 +write_env "CLICKHOUSE_HTTP_PORT" "$CLICKHOUSE_HTTP_PORT"
13 +replace_in_file "$WORKDIR/config/go.d/clickhouse.conf" "127.0.0.1:8123" "127.0.0.1:${CLICKHOUSE_HTTP_PORT}"
14 +
15 +compose_up clickhouse
16 +wait_healthy clickhouse 90
17 +compose_run clickhouse-init
18 +
19 +build_plugin
20 +run_info clickhouse
21 +run_top_queries clickhouse
22 +
23 +echo "E2E checks passed for clickhouse." >&2
src/go/tools/functions-validation/e2e/cockroachdb.sh new
+36
@@ -0,0 +1,36 @@
1 +#!/usr/bin/env bash
2 +set -euo pipefail
3 +
4 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
5 +# shellcheck disable=SC1091
6 +. "$SCRIPT_DIR/lib.sh"
7 +
8 +init_workspace "cockroachdb"
9 +trap cleanup EXIT
10 +
11 +COCKROACH_HTTP_PORT="$(reserve_port)"
12 +COCKROACH_SQL_PORT="$(reserve_port)"
13 +write_env "COCKROACH_HTTP_PORT" "$COCKROACH_HTTP_PORT"
14 +write_env "COCKROACH_SQL_PORT" "$COCKROACH_SQL_PORT"
15 +replace_in_file "$WORKDIR/config/go.d/cockroachdb.conf" "127.0.0.1:8080" "127.0.0.1:${COCKROACH_HTTP_PORT}"
16 +replace_in_file "$WORKDIR/config/go.d/cockroachdb.conf" "127.0.0.1:26258" "127.0.0.1:${COCKROACH_SQL_PORT}"
17 +
18 +compose_up cockroachdb
19 +wait_healthy cockroachdb 120
20 +compose_run cockroachdb-seed
21 +
22 +run_bg compose_run cockroachdb-sleep
23 +SLEEP_PID="$LAST_BG_PID"
24 +sleep 3
25 +
26 +build_plugin
27 +run_info cockroachdb
28 +run_top_queries cockroachdb
29 +run_running_queries cockroachdb 1
30 +
31 +if kill -0 "$SLEEP_PID" 2>/dev/null; then
32 + kill "$SLEEP_PID"
33 + wait "$SLEEP_PID" || true
34 +fi
35 +
36 +echo "E2E checks passed for cockroachdb." >&2
src/go/tools/functions-validation/e2e/couchbase.sh new
+44
@@ -0,0 +1,44 @@
1 +#!/usr/bin/env bash
2 +set -euo pipefail
3 +
4 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
5 +# shellcheck disable=SC1091
6 +. "$SCRIPT_DIR/lib.sh"
7 +
8 +init_workspace "couchbase"
9 +trap cleanup EXIT
10 +
11 +COUCHBASE_HTTP_PORT="$(reserve_port)"
12 +COUCHBASE_QUERY_PORT="$(reserve_port)"
13 +write_env "COUCHBASE_HTTP_PORT" "$COUCHBASE_HTTP_PORT"
14 +write_env "COUCHBASE_QUERY_PORT" "$COUCHBASE_QUERY_PORT"
15 +replace_in_file "$WORKDIR/config/go.d/couchbase.conf" "127.0.0.1:8091" "127.0.0.1:${COUCHBASE_HTTP_PORT}"
16 +replace_in_file "$WORKDIR/config/go.d/couchbase.conf" "127.0.0.1:8093" "127.0.0.1:${COUCHBASE_QUERY_PORT}"
17 +
18 +compose_up couchbase
19 +wait_healthy couchbase 180
20 +compose_run couchbase-init
21 +
22 +wait_query_service() {
23 + local attempt
24 + for attempt in $(seq 1 60); do
25 + : "$attempt" # loop counter
26 + if curl -fsS -u "Administrator:password" "http://127.0.0.1:${COUCHBASE_QUERY_PORT}/query/service" \
27 + --data-urlencode "statement=SELECT 1" > /dev/null; then
28 + return 0
29 + fi
30 + sleep 2
31 + done
32 + echo "Couchbase query service not ready on host port ${COUCHBASE_QUERY_PORT}" >&2
33 + return 1
34 +}
35 +
36 +wait_query_service
37 +curl -fsS -u "Administrator:password" "http://127.0.0.1:${COUCHBASE_QUERY_PORT}/query/service" \
38 + --data-urlencode "statement=SELECT 1" > /dev/null
39 +
40 +build_plugin
41 +run_info couchbase
42 +run_top_queries couchbase
43 +
44 +echo "E2E checks passed for couchbase." >&2
src/go/tools/functions-validation/e2e/elasticsearch.sh new
+49
@@ -0,0 +1,49 @@
1 +#!/usr/bin/env bash
2 +set -euo pipefail
3 +
4 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
5 +# shellcheck disable=SC1091
6 +. "$SCRIPT_DIR/lib.sh"
7 +
8 +run_top_queries_retry() {
9 + local module="$1"
10 + local attempts="${2:-20}"
11 + local delay="${3:-0.5}"
12 + local output="$WORKDIR/${module}-top-queries.json"
13 + local i=1
14 +
15 + while [ "$i" -le "$attempts" ]; do
16 + run "$WORKDIR/go.d.plugin" \
17 + --config-dir "$WORKDIR/config" \
18 + --function "${module}:top-queries" \
19 + --function-args __job:local \
20 + > "$output"
21 + validate "$output"
22 + if has_min_rows "$output" 1; then
23 + return 0
24 + fi
25 + sleep "$delay"
26 + i=$((i + 1))
27 + done
28 +
29 + echo "top-queries returned 0 rows after ${attempts} attempts" >&2
30 + return 1
31 +}
32 +
33 +init_workspace "elasticsearch"
34 +trap cleanup EXIT
35 +
36 +ELASTICSEARCH_PORT="$(reserve_port)"
37 +write_env "ELASTICSEARCH_PORT" "$ELASTICSEARCH_PORT"
38 +replace_in_file "$WORKDIR/config/go.d/elasticsearch.conf" "127.0.0.1:9200" "127.0.0.1:${ELASTICSEARCH_PORT}"
39 +
40 +compose_up elasticsearch
41 +wait_healthy elasticsearch 120
42 +compose_run elasticsearch-init
43 +compose_up elasticsearch-searcher
44 +
45 +build_plugin
46 +run_info elasticsearch
47 +run_top_queries_retry elasticsearch
48 +
49 +echo "E2E checks passed for elasticsearch." >&2
src/go/tools/functions-validation/e2e/lib.sh new
+260
@@ -0,0 +1,260 @@
1 +#!/usr/bin/env bash
2 +set -euo pipefail
3 +
4 +# Colors for output
5 +RED='\033[0;31m'
6 +YELLOW='\033[1;33m'
7 +GRAY='\033[0;90m'
8 +NC='\033[0m' # No Color
9 +
10 +# Execute command with visibility
11 +run() {
12 + local errexit_set=0
13 + case $- in
14 + *e*) errexit_set=1 ;;
15 + esac
16 +
17 + # Print the command being executed
18 + printf >&2 '%s%s >%s ' "$GRAY" "$(pwd)" "$NC"
19 + printf >&2 '%s' "$YELLOW"
20 + printf >&2 "%q " "$@"
21 + printf >&2 '%s\n' "$NC"
22 +
23 + # Execute the command
24 + set +e
25 + "$@"
26 + local exit_code=$?
27 + if [ $errexit_set -eq 1 ]; then
28 + set -e
29 + else
30 + set +e
31 + fi
32 + if [ $exit_code -ne 0 ]; then
33 + echo -e >&2 "${RED}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
34 + echo -e >&2 "${RED}[ERROR]${NC} Command failed with exit code ${exit_code}: ${YELLOW}$1${NC}"
35 + echo -e >&2 "${RED} Full command:${NC} $*"
36 + echo -e >&2 "${RED} Working dir:${NC} $(pwd)"
37 + echo -e >&2 "${RED}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
38 + return $exit_code
39 + fi
40 +}
41 +
42 +# Execute command in background with visibility
43 +LAST_BG_PID=""
44 +run_bg() {
45 + printf >&2 '%s%s >%s ' "$GRAY" "$(pwd)" "$NC"
46 + printf >&2 '%s' "$YELLOW"
47 + printf >&2 "%q " "$@"
48 + printf >&2 '%s\n' "$NC"
49 + "$@" &
50 + LAST_BG_PID=$!
51 +}
52 +
53 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
54 +FUNCTIONS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
55 +REPO_ROOT="$(cd "$FUNCTIONS_DIR/../../../.." && pwd)"
56 +
57 +WORKDIR=""
58 +PROJECT=""
59 +COMPOSE=()
60 +COMPOSE_STARTED=""
61 +USED_PORTS=""
62 +
63 +cleanup() {
64 + local exit_code=$?
65 + set +e
66 + if [ -n "${COMPOSE_STARTED:-}" ]; then
67 + run "${COMPOSE[@]}" down -v --remove-orphans
68 + fi
69 + if [ "$exit_code" -eq 0 ]; then
70 + run rm -rf "$WORKDIR"
71 + else
72 + echo "E2E failed. Keeping workspace: $WORKDIR" >&2
73 + fi
74 + exit $exit_code
75 +}
76 +
77 +init_workspace() {
78 + local db="$1"
79 + WORKDIR="$(mktemp -d "/tmp/netdata-functions-e2e-${db}.XXXXXX")"
80 + local project_suffix
81 + project_suffix="$(basename "$WORKDIR")"
82 + project_suffix="$(printf '%s' "$project_suffix" | tr '[:upper:]' '[:lower:]' | tr -c 'a-z0-9_-' '-')"
83 + project_suffix="${project_suffix%-}"
84 + PROJECT="netdata-func-e2e-${db}-${project_suffix}"
85 + COMPOSE=(docker compose -f "$WORKDIR/docker-compose.yml" -p "$PROJECT")
86 +
87 + run cp -a "$FUNCTIONS_DIR/docker-compose.yml" "$FUNCTIONS_DIR/seed" "$FUNCTIONS_DIR/config" "$WORKDIR/"
88 + : > "$WORKDIR/.env"
89 +}
90 +
91 +compose_up() {
92 + run "${COMPOSE[@]}" up -d "$@"
93 + COMPOSE_STARTED="yes"
94 +}
95 +
96 +compose_run() {
97 + run "${COMPOSE[@]}" run --rm "$@"
98 +}
99 +
100 +wait_healthy() {
101 + local service="$1"
102 + local timeout="${2:-60}"
103 + local start=$SECONDS
104 +
105 + while true; do
106 + local cid
107 + cid=$("${COMPOSE[@]}" ps -q "$service")
108 + if [ -z "$cid" ]; then
109 + if [ $((SECONDS - start)) -ge "$timeout" ]; then
110 + echo "No container found for service: $service" >&2
111 + return 1
112 + fi
113 + sleep 2
114 + continue
115 + fi
116 +
117 + local status
118 + status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$cid")"
119 + if [ "$status" = "healthy" ]; then
120 + return 0
121 + fi
122 + if [ $((SECONDS - start)) -ge "$timeout" ]; then
123 + echo "Timed out waiting for $service to be healthy" >&2
124 + return 1
125 + fi
126 + sleep 2
127 + done
128 +}
129 +
130 +write_env() {
131 + local key="$1"
132 + local value="$2"
133 + echo "${key}=${value}" >> "$WORKDIR/.env"
134 + export "${key}=${value}"
135 +}
136 +
137 +pick_free_port() {
138 + if command -v python3 >/dev/null 2>&1; then
139 + python3 - <<'PY'
140 +import socket
141 +s = socket.socket()
142 +s.bind(("", 0))
143 +print(s.getsockname()[1])
144 +s.close()
145 +PY
146 + elif command -v python >/dev/null 2>&1; then
147 + python - <<'PY'
148 +import socket
149 +s = socket.socket()
150 +s.bind(("", 0))
151 +print(s.getsockname()[1])
152 +s.close()
153 +PY
154 + else
155 + echo "python3 (or python) is required to select a free port" >&2
156 + return 1
157 + fi
158 +}
159 +
160 +reserve_port() {
161 + local port
162 + while true; do
163 + port="$(pick_free_port)"
164 + case " $USED_PORTS " in
165 + *" $port "*) ;;
166 + *)
167 + USED_PORTS="${USED_PORTS} ${port}"
168 + echo "$port"
169 + return 0
170 + ;;
171 + esac
172 + done
173 +}
174 +
175 +replace_in_file() {
176 + local file="$1"
177 + local search="$2"
178 + local replace="$3"
179 + run sed -i "s|$search|$replace|g" "$file"
180 +}
181 +
182 +build_plugin() {
183 + run bash -c "cd \"$REPO_ROOT/src/go\" && go build -o \"$WORKDIR/go.d.plugin\" ./cmd/godplugin"
184 +}
185 +
186 +validate() {
187 + local input="$1"
188 + shift
189 + (cd "$REPO_ROOT/src/go" && run go run ./tools/functions-validation/validate --input "$input" "$@")
190 +}
191 +
192 +run_info_method() {
193 + local module="$1"
194 + local method="$2"
195 + local output="$WORKDIR/${module}-${method}-info.json"
196 + run "$WORKDIR/go.d.plugin" \
197 + --config-dir "$WORKDIR/config" \
198 + --function "${module}:${method}" \
199 + --function-args info \
200 + > "$output"
201 + validate "$output"
202 +}
203 +
204 +run_info() {
205 + local module="$1"
206 + run_info_method "$module" "top-queries"
207 +}
208 +
209 +run_top_queries() {
210 + local module="$1"
211 + local output="$WORKDIR/${module}-top-queries.json"
212 + run "$WORKDIR/go.d.plugin" \
213 + --config-dir "$WORKDIR/config" \
214 + --function "${module}:top-queries" \
215 + --function-args __job:local \
216 + > "$output"
217 + validate "$output" --min-rows 1
218 +}
219 +
220 +run_running_queries() {
221 + local module="$1"
222 + local min_rows="${2:-1}"
223 + local output="$WORKDIR/${module}-running-queries.json"
224 + run "$WORKDIR/go.d.plugin" \
225 + --config-dir "$WORKDIR/config" \
226 + --function "${module}:running-queries" \
227 + --function-args __job:local \
228 + > "$output"
229 + validate "$output" --min-rows "$min_rows"
230 +}
231 +
232 +has_min_rows() {
233 + local input="$1"
234 + local min_rows="${2:-1}"
235 + if command -v python3 >/dev/null 2>&1; then
236 + python3 - "$input" "$min_rows" <<'PY'
237 +import json
238 +import sys
239 +
240 +path = sys.argv[1]
241 +min_rows = int(sys.argv[2])
242 +with open(path, "r", encoding="utf-8") as fh:
243 + data = json.load(fh)
244 +rows = data.get("data", [])
245 +sys.exit(0 if len(rows) >= min_rows else 1)
246 +PY
247 + else
248 + python - "$input" "$min_rows" <<'PY'
249 +import json
250 +import sys
251 +
252 +path = sys.argv[1]
253 +min_rows = int(sys.argv[2])
254 +with open(path, "r", encoding="utf-8") as fh:
255 + data = json.load(fh)
256 +rows = data.get("data", [])
257 +sys.exit(0 if len(rows) >= min_rows else 1)
258 +PY
259 + fi
260 +}
src/go/tools/functions-validation/e2e/mongodb.sh new
+23
@@ -0,0 +1,23 @@
1 +#!/usr/bin/env bash
2 +set -euo pipefail
3 +
4 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
5 +# shellcheck disable=SC1091
6 +. "$SCRIPT_DIR/lib.sh"
7 +
8 +init_workspace "mongodb"
9 +trap cleanup EXIT
10 +
11 +MONGO_PORT="$(reserve_port)"
12 +write_env "MONGO_PORT" "$MONGO_PORT"
13 +replace_in_file "$WORKDIR/config/go.d/mongodb.conf" "127.0.0.1:27017" "127.0.0.1:${MONGO_PORT}"
14 +
15 +compose_up mongo
16 +wait_healthy mongo 90
17 +compose_run mongo-init
18 +
19 +build_plugin
20 +run_info mongodb
21 +run_top_queries mongodb
22 +
23 +echo "E2E checks passed for mongodb." >&2
src/go/tools/functions-validation/e2e/mssql.sh new
+23
@@ -0,0 +1,23 @@
1 +#!/usr/bin/env bash
2 +set -euo pipefail
3 +
4 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
5 +# shellcheck disable=SC1091
6 +. "$SCRIPT_DIR/lib.sh"
7 +
8 +init_workspace "mssql"
9 +trap cleanup EXIT
10 +
11 +MSSQL_PORT="$(reserve_port)"
12 +write_env "MSSQL_PORT" "$MSSQL_PORT"
13 +replace_in_file "$WORKDIR/config/go.d/mssql.conf" "127.0.0.1:1433" "127.0.0.1:${MSSQL_PORT}"
14 +
15 +compose_up mssql
16 +wait_healthy mssql 120
17 +compose_run mssql-init
18 +
19 +build_plugin
20 +run_info mssql
21 +run_top_queries mssql
22 +
23 +echo "E2E checks passed for mssql." >&2
src/go/tools/functions-validation/e2e/mysql.sh new
+22
@@ -0,0 +1,22 @@
1 +#!/usr/bin/env bash
2 +set -euo pipefail
3 +
4 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
5 +# shellcheck disable=SC1091
6 +. "$SCRIPT_DIR/lib.sh"
7 +
8 +init_workspace "mysql"
9 +trap cleanup EXIT
10 +
11 +MYSQL_PORT="$(reserve_port)"
12 +write_env "MYSQL_PORT" "$MYSQL_PORT"
13 +replace_in_file "$WORKDIR/config/go.d/mysql.conf" "127.0.0.1:3306" "127.0.0.1:${MYSQL_PORT}"
14 +
15 +compose_up mysql
16 +wait_healthy mysql 90
17 +
18 +build_plugin
19 +run_info mysql
20 +run_top_queries mysql
21 +
22 +echo "E2E checks passed for mysql." >&2
src/go/tools/functions-validation/e2e/oracledb.sh new
+34
@@ -0,0 +1,34 @@
1 +#!/usr/bin/env bash
2 +set -euo pipefail
3 +
4 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
5 +# shellcheck disable=SC1091
6 +. "$SCRIPT_DIR/lib.sh"
7 +
8 +init_workspace "oracledb"
9 +trap cleanup EXIT
10 +
11 +ORACLE_PORT="$(reserve_port)"
12 +write_env "ORACLE_PORT" "$ORACLE_PORT"
13 +replace_in_file "$WORKDIR/config/go.d/oracledb.conf" "127.0.0.1:1521" "127.0.0.1:${ORACLE_PORT}"
14 +
15 +compose_up oracledb
16 +wait_healthy oracledb 300
17 +
18 +compose_run oracledb-seed
19 +
20 +run_bg compose_run oracledb-sleep
21 +SLEEP_PID="$LAST_BG_PID"
22 +sleep 3
23 +
24 +build_plugin
25 +run_info oracledb
26 +run_top_queries oracledb
27 +run_running_queries oracledb 1
28 +
29 +if kill -0 "$SLEEP_PID" 2>/dev/null; then
30 + kill "$SLEEP_PID"
31 + wait "$SLEEP_PID" || true
32 +fi
33 +
34 +echo "E2E checks passed for oracledb." >&2
src/go/tools/functions-validation/e2e/postgres.sh new
+22
@@ -0,0 +1,22 @@
1 +#!/usr/bin/env bash
2 +set -euo pipefail
3 +
4 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
5 +# shellcheck disable=SC1091
6 +. "$SCRIPT_DIR/lib.sh"
7 +
8 +init_workspace "postgres"
9 +trap cleanup EXIT
10 +
11 +POSTGRES_PORT="$(reserve_port)"
12 +write_env "POSTGRES_PORT" "$POSTGRES_PORT"
13 +replace_in_file "$WORKDIR/config/go.d/postgres.conf" "127.0.0.1:5432" "127.0.0.1:${POSTGRES_PORT}"
14 +
15 +compose_up postgres
16 +wait_healthy postgres 90
17 +
18 +build_plugin
19 +run_info postgres
20 +run_top_queries postgres
21 +
22 +echo "E2E checks passed for postgres." >&2
src/go/tools/functions-validation/e2e/proxysql.sh new
+63
@@ -0,0 +1,63 @@
1 +#!/usr/bin/env bash
2 +set -euo pipefail
3 +
4 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
5 +# shellcheck disable=SC1091
6 +. "$SCRIPT_DIR/lib.sh"
7 +
8 +init_workspace "proxysql"
9 +trap cleanup EXIT
10 +
11 +MYSQL_PORT="$(reserve_port)"
12 +PROXYSQL_ADMIN_PORT="$(reserve_port)"
13 +PROXYSQL_MYSQL_PORT="$(reserve_port)"
14 +write_env "MYSQL_PORT" "$MYSQL_PORT"
15 +write_env "PROXYSQL_ADMIN_PORT" "$PROXYSQL_ADMIN_PORT"
16 +write_env "PROXYSQL_MYSQL_PORT" "$PROXYSQL_MYSQL_PORT"
17 +
18 +compose_up mysql proxysql
19 +wait_healthy mysql 90
20 +compose_run proxysql-init
21 +
22 +build_plugin
23 +
24 +PROXYSQL_CID="$("${COMPOSE[@]}" ps -q proxysql)"
25 +if [ -z "$PROXYSQL_CID" ]; then
26 + echo "Unable to resolve proxysql container ID" >&2
27 + exit 1
28 +fi
29 +
30 +run_info_proxysql() {
31 + local output="$WORKDIR/proxysql-info.json"
32 + run docker run --rm \
33 + --network "container:${PROXYSQL_CID}" \
34 + -v "$WORKDIR:/work" \
35 + -w /work \
36 + debian:bookworm-slim \
37 + /work/go.d.plugin \
38 + --config-dir /work/config \
39 + --function proxysql:top-queries \
40 + --function-args info \
41 + > "$output"
42 + validate "$output"
43 +}
44 +
45 +run_top_queries_proxysql() {
46 + local output="$WORKDIR/proxysql-top-queries.json"
47 + run docker run --rm \
48 + --network "container:${PROXYSQL_CID}" \
49 + -v "$WORKDIR:/work" \
50 + -w /work \
51 + debian:bookworm-slim \
52 + /work/go.d.plugin \
53 + --config-dir /work/config \
54 + --function proxysql:top-queries \
55 + --function-args __job:local \
56 + > "$output"
57 + validate "$output" --min-rows 1
58 +}
59 +
60 +run_info_proxysql
61 +run_top_queries_proxysql
62 +
63 +echo "E2E checks passed for proxysql." >&2
src/go/tools/functions-validation/e2e/redis.sh new
+23
@@ -0,0 +1,23 @@
1 +#!/usr/bin/env bash
2 +set -euo pipefail
3 +
4 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
5 +# shellcheck disable=SC1091
6 +. "$SCRIPT_DIR/lib.sh"
7 +
8 +init_workspace "redis"
9 +trap cleanup EXIT
10 +
11 +REDIS_PORT="$(reserve_port)"
12 +write_env "REDIS_PORT" "$REDIS_PORT"
13 +replace_in_file "$WORKDIR/config/go.d/redis.conf" "127.0.0.1:6379" "127.0.0.1:${REDIS_PORT}"
14 +
15 +compose_up redis
16 +wait_healthy redis 60
17 +compose_run redis-init
18 +
19 +build_plugin
20 +run_info redis
21 +run_top_queries redis
22 +
23 +echo "E2E checks passed for redis." >&2
src/go/tools/functions-validation/e2e/rethinkdb.sh new
+57
@@ -0,0 +1,57 @@
1 +#!/usr/bin/env bash
2 +set -euo pipefail
3 +
4 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
5 +# shellcheck disable=SC1091
6 +. "$SCRIPT_DIR/lib.sh"
7 +
8 +init_workspace "rethinkdb"
9 +trap cleanup EXIT
10 +
11 +RETHINKDB_PORT="$(reserve_port)"
12 +write_env "RETHINKDB_PORT" "$RETHINKDB_PORT"
13 +replace_in_file "$WORKDIR/config/go.d/rethinkdb.conf" "127.0.0.1:28015" "127.0.0.1:${RETHINKDB_PORT}"
14 +
15 +compose_up rethinkdb
16 +wait_healthy rethinkdb 60
17 +
18 +build_plugin
19 +
20 +wait_port() {
21 + local port="$1"
22 + for _ in $(seq 1 30); do
23 + if python3 - <<PY
24 +import socket
25 +s = socket.socket()
26 +s.settimeout(1)
27 +try:
28 + s.connect(("127.0.0.1", int("${port}")))
29 + s.close()
30 + raise SystemExit(0)
31 +except Exception:
32 + raise SystemExit(1)
33 +PY
34 + then
35 + return 0
36 + fi
37 + sleep 2
38 + done
39 + echo "Timed out waiting for RethinkDB to accept connections on ${port}" >&2
40 + return 1
41 +}
42 +
43 +wait_port "$RETHINKDB_PORT"
44 +
45 +run_bg bash -c "cd \"$REPO_ROOT/src/go\" && go run ./tools/functions-validation/seed/rethinkdb/hold.go --addr 127.0.0.1:${RETHINKDB_PORT} --duration 40s"
46 +HOLD_PID="$LAST_BG_PID"
47 +sleep 3
48 +
49 +run_info_method rethinkdb running-queries
50 +run_running_queries rethinkdb 1
51 +
52 +if kill -0 "$HOLD_PID" 2>/dev/null; then
53 + kill "$HOLD_PID"
54 + wait "$HOLD_PID" || true
55 +fi
56 +
57 +echo "E2E checks passed for rethinkdb." >&2
src/go/tools/functions-validation/e2e/yugabytedb.sh new
+36
@@ -0,0 +1,36 @@
1 +#!/usr/bin/env bash
2 +set -euo pipefail
3 +
4 +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
5 +# shellcheck disable=SC1091
6 +. "$SCRIPT_DIR/lib.sh"
7 +
8 +init_workspace "yugabytedb"
9 +trap cleanup EXIT
10 +
11 +YUGABYTE_MASTER_PORT="$(reserve_port)"
12 +YUGABYTE_YSQL_PORT="$(reserve_port)"
13 +write_env "YUGABYTE_MASTER_PORT" "$YUGABYTE_MASTER_PORT"
14 +write_env "YUGABYTE_YSQL_PORT" "$YUGABYTE_YSQL_PORT"
15 +replace_in_file "$WORKDIR/config/go.d/yugabytedb.conf" "127.0.0.1:7000" "127.0.0.1:${YUGABYTE_MASTER_PORT}"
16 +replace_in_file "$WORKDIR/config/go.d/yugabytedb.conf" "127.0.0.1:5433" "127.0.0.1:${YUGABYTE_YSQL_PORT}"
17 +
18 +compose_up yugabytedb
19 +wait_healthy yugabytedb 240
20 +compose_run yugabytedb-seed
21 +
22 +run_bg compose_run yugabytedb-sleep
23 +SLEEP_PID="$LAST_BG_PID"
24 +sleep 3
25 +
26 +build_plugin
27 +run_info yugabytedb
28 +run_top_queries yugabytedb
29 +run_running_queries yugabytedb 1
30 +
31 +if kill -0 "$SLEEP_PID" 2>/dev/null; then
32 + kill "$SLEEP_PID"
33 + wait "$SLEEP_PID" || true
34 +fi
35 +
36 +echo "E2E checks passed for yugabytedb." >&2
src/go/tools/functions-validation/seed/clickhouse/init.sh new
+14
@@ -0,0 +1,14 @@
1 +#!/usr/bin/env bash
2 +set -euo pipefail
3 +
4 +CH_HOST="${CH_HOST:-clickhouse}"
5 +CH_USER="${CH_USER:-default}"
6 +CH_PASSWORD="${CH_PASSWORD:-netdata}"
7 +
8 +clickhouse-client --host "$CH_HOST" --user "$CH_USER" --password "$CH_PASSWORD" --query "CREATE DATABASE IF NOT EXISTS netdata"
9 +clickhouse-client --host "$CH_HOST" --user "$CH_USER" --password "$CH_PASSWORD" --query "CREATE TABLE IF NOT EXISTS netdata.test (id UInt64, value String) ENGINE=MergeTree() ORDER BY id"
10 +clickhouse-client --host "$CH_HOST" --user "$CH_USER" --password "$CH_PASSWORD" --query "INSERT INTO netdata.test VALUES (1,'a'), (2,'b'), (3,'c')"
11 +clickhouse-client --host "$CH_HOST" --user "$CH_USER" --password "$CH_PASSWORD" --query "SELECT count() FROM netdata.test"
12 +clickhouse-client --host "$CH_HOST" --user "$CH_USER" --password "$CH_PASSWORD" --query "SELECT * FROM netdata.test WHERE id > 0"
13 +
14 +sleep 1
src/go/tools/functions-validation/seed/cockroachdb/seed.sh new
+4
@@ -0,0 +1,4 @@
1 +#!/usr/bin/env sh
2 +set -e
3 +
4 +cockroach sql --insecure --host=cockroachdb:26258 -f /seed/seed.sql
src/go/tools/functions-validation/seed/cockroachdb/seed.sql new
+16
@@ -0,0 +1,16 @@
1 +CREATE DATABASE IF NOT EXISTS netdata;
2 +USE netdata;
3 +
4 +CREATE TABLE IF NOT EXISTS items (
5 + id INT PRIMARY KEY,
6 + name STRING
7 +);
8 +
9 +UPSERT INTO items (id, name) VALUES
10 + (1, 'alpha'),
11 + (2, 'beta'),
12 + (3, 'gamma');
13 +
14 +SELECT * FROM items WHERE id = 1;
15 +UPDATE items SET name = 'delta' WHERE id = 2;
16 +SELECT count(*) FROM items;
src/go/tools/functions-validation/seed/cockroachdb/sleep.sh new
+4
@@ -0,0 +1,4 @@
1 +#!/usr/bin/env sh
2 +set -e
3 +
4 +cockroach sql --insecure --host=cockroachdb:26258 -f /seed/sleep.sql
src/go/tools/functions-validation/seed/cockroachdb/sleep.sql new
+2
@@ -0,0 +1,2 @@
1 +USE netdata;
2 +SELECT pg_sleep(30);
src/go/tools/functions-validation/seed/couchbase/init.sh new
+71
@@ -0,0 +1,71 @@
1 +#!/usr/bin/env bash
2 +set -euo pipefail
3 +
4 +CB_HOST="${CB_HOST:-couchbase}"
5 +CB_ADMIN="${CB_ADMIN:-Administrator}"
6 +CB_PASS="${CB_PASS:-password}"
7 +
8 +cb_ready=0
9 +for i in $(seq 1 60); do
10 + code="$(curl -s -o /dev/null -w '%{http_code}' "http://${CB_HOST}:8091/pools" || true)"
11 + if [ "$code" = "200" ] || [ "$code" = "401" ]; then
12 + cb_ready=1
13 + break
14 + fi
15 + sleep 2
16 +done
17 +
18 +if [ "$cb_ready" -ne 1 ]; then
19 + echo "Couchbase API not ready after 120s" >&2
20 + exit 1
21 +fi
22 +
23 +cb_initialized_code="$(curl -s -o /dev/null -w '%{http_code}' "http://${CB_HOST}:8091/pools/default" -u "${CB_ADMIN}:${CB_PASS}" || true)"
24 +if [ "$cb_initialized_code" != "200" ]; then
25 + /opt/couchbase/bin/couchbase-cli cluster-init -c "${CB_HOST}:8091" \
26 + --cluster-username "${CB_ADMIN}" \
27 + --cluster-password "${CB_PASS}" \
28 + --services data,index,query \
29 + --cluster-ramsize 256 \
30 + --cluster-index-ramsize 256 || true
31 +
32 + /opt/couchbase/bin/couchbase-cli bucket-create -c "${CB_HOST}:8091" \
33 + -u "${CB_ADMIN}" -p "${CB_PASS}" \
34 + --bucket default \
35 + --bucket-type couchbase \
36 + --bucket-ramsize 256 || true
37 +fi
38 +
39 +query_ready=0
40 +for i in $(seq 1 60); do
41 + if curl -fsS "http://${CB_HOST}:8093/query/service" -u "${CB_ADMIN}:${CB_PASS}" -d "statement=SELECT 1" > /dev/null; then
42 + query_ready=1
43 + break
44 + fi
45 + sleep 2
46 +done
47 +
48 +if [ "$query_ready" -ne 1 ]; then
49 + echo "Couchbase query service not ready after 120s" >&2
50 + exit 1
51 +fi
52 +
53 +run_query() {
54 + local stmt="$1"
55 + local i
56 + for i in $(seq 1 30); do
57 + if curl -fsS -u "${CB_ADMIN}:${CB_PASS}" "http://${CB_HOST}:8093/query/service" \
58 + --data-urlencode "statement=${stmt}" > /dev/null; then
59 + return 0
60 + fi
61 + sleep 2
62 + done
63 + return 1
64 +}
65 +
66 +run_query "CREATE PRIMARY INDEX ON \`default\`" || true
67 +
68 +run_query "INSERT INTO \`default\` (KEY, VALUE) VALUES (\"k1\", {\"type\":\"t\",\"v\":1})"
69 +run_query "INSERT INTO \`default\` (KEY, VALUE) VALUES (\"k2\", {\"type\":\"t\",\"v\":2})"
70 +
71 +run_query "SELECT * FROM \`default\` WHERE type = \"t\""
src/go/tools/functions-validation/seed/elasticsearch/init.sh new
+23
@@ -0,0 +1,23 @@
1 +#!/bin/sh
2 +set -eu
3 +
4 +ES_URL="${ES_URL:-http://elasticsearch:9200}"
5 +
6 +until curl -fsS "$ES_URL/_cluster/health?wait_for_status=yellow&timeout=5s" > /dev/null; do
7 + sleep 2
8 +done
9 +
10 +curl -fsS -X PUT "$ES_URL/netdata" -H 'Content-Type: application/json' -d '{
11 + "settings": { "number_of_shards": 1, "number_of_replicas": 0 }
12 +}' > /dev/null || true
13 +
14 +{
15 + i=1
16 + while [ "$i" -le 1000 ]; do
17 + echo "{\"index\":{\"_index\":\"netdata\",\"_id\":\"$i\"}}"
18 + echo "{\"value\":$i,\"text\":\"value-$i\"}"
19 + i=$((i + 1))
20 + done
21 +} | curl -fsS -X POST "$ES_URL/_bulk" -H 'Content-Type: application/x-ndjson' --data-binary @- > /dev/null
22 +
23 +curl -fsS -X POST "$ES_URL/netdata/_refresh" > /dev/null
src/go/tools/functions-validation/seed/elasticsearch/searcher.sh new
+24
@@ -0,0 +1,24 @@
1 +#!/bin/sh
2 +set -eu
3 +
4 +ES_URL="${ES_URL:-http://elasticsearch:9200}"
5 +
6 +until curl -fsS "$ES_URL/_cluster/health?wait_for_status=yellow&timeout=5s" > /dev/null; do
7 + sleep 2
8 +done
9 +
10 +while true; do
11 + curl -fsS -X POST "$ES_URL/netdata/_search" -H 'Content-Type: application/json' -d '{
12 + "size": 5000,
13 + "track_total_hits": true,
14 + "query": {
15 + "script_score": {
16 + "query": { "match_all": {} },
17 + "script": {
18 + "source": "double v = 0; for (int i = 0; i < 20000; ++i) { v += Math.sqrt(_score + i); } return v;"
19 + }
20 + }
21 + }
22 + }' > /dev/null || true
23 + sleep 0.2
24 +done
src/go/tools/functions-validation/seed/oracledb/init.sql new
+35
@@ -0,0 +1,35 @@
1 +ALTER SESSION SET CONTAINER=FREEPDB1;
2 +
3 +DECLARE
4 + v_exists NUMBER := 0;
5 +BEGIN
6 + SELECT COUNT(*) INTO v_exists FROM dba_users WHERE username = 'NETDATA';
7 + IF v_exists = 0 THEN
8 + EXECUTE IMMEDIATE 'CREATE USER netdata IDENTIFIED BY "Netdata123!"';
9 + END IF;
10 +END;
11 +/
12 +
13 +GRANT CONNECT TO netdata;
14 +GRANT SELECT_CATALOG_ROLE TO netdata;
15 +GRANT SELECT ON V_$SQLSTATS TO netdata;
16 +GRANT SELECT ON V_$SESSION TO netdata;
17 +GRANT SELECT ON V_$SQL TO netdata;
18 +GRANT EXECUTE ON DBMS_LOCK TO netdata;
19 +
20 +BEGIN
21 + EXECUTE IMMEDIATE 'CREATE TABLE netdata.demo (id NUMBER PRIMARY KEY, name VARCHAR2(64))';
22 +EXCEPTION
23 + WHEN OTHERS THEN
24 + IF SQLCODE != -955 THEN
25 + RAISE;
26 + END IF;
27 +END;
28 +/
29 +
30 +MERGE INTO netdata.demo d
31 +USING (SELECT 1 AS id, 'alpha' AS name FROM dual) s
32 +ON (d.id = s.id)
33 +WHEN NOT MATCHED THEN INSERT (id, name) VALUES (s.id, s.name);
34 +
35 +COMMIT;
src/go/tools/functions-validation/seed/oracledb/seed.sh new
+10
@@ -0,0 +1,10 @@
1 +#!/usr/bin/env bash
2 +set -euo pipefail
3 +
4 +ORA_USER="${ORA_USER:-netdata}"
5 +ORA_PASS="${ORA_PASS:-Netdata123!}"
6 +ORA_HOST="${ORA_HOST:-oracledb}"
7 +ORA_PORT="${ORA_PORT:-1521}"
8 +ORA_SERVICE="${ORA_SERVICE:-FREEPDB1}"
9 +
10 +sqlplus -s "${ORA_USER}/${ORA_PASS}@//${ORA_HOST}:${ORA_PORT}/${ORA_SERVICE}" @/seed/seed.sql
src/go/tools/functions-validation/seed/oracledb/seed.sql new
+17
@@ -0,0 +1,17 @@
1 +SET PAGESIZE 0
2 +SET FEEDBACK OFF
3 +
4 +SELECT COUNT(*) FROM netdata.demo;
5 +SELECT COUNT(*) FROM netdata.demo;
6 +SELECT COUNT(*) FROM netdata.demo;
7 +SELECT COUNT(*) FROM netdata.demo;
8 +SELECT COUNT(*) FROM netdata.demo;
9 +
10 +SELECT name FROM netdata.demo WHERE id = 1;
11 +SELECT name FROM netdata.demo WHERE id = 1;
12 +SELECT name FROM netdata.demo WHERE id = 1;
13 +
14 +UPDATE netdata.demo SET name = 'beta' WHERE id = 1;
15 +UPDATE netdata.demo SET name = 'alpha' WHERE id = 1;
16 +
17 +COMMIT;
src/go/tools/functions-validation/seed/oracledb/sleep.sh new
+10
@@ -0,0 +1,10 @@
1 +#!/usr/bin/env bash
2 +set -euo pipefail
3 +
4 +ORA_USER="${ORA_USER:-netdata}"
5 +ORA_PASS="${ORA_PASS:-Netdata123!}"
6 +ORA_HOST="${ORA_HOST:-oracledb}"
7 +ORA_PORT="${ORA_PORT:-1521}"
8 +ORA_SERVICE="${ORA_SERVICE:-FREEPDB1}"
9 +
10 +sqlplus -s "${ORA_USER}/${ORA_PASS}@//${ORA_HOST}:${ORA_PORT}/${ORA_SERVICE}" @/seed/sleep.sql
src/go/tools/functions-validation/seed/oracledb/sleep.sql new
+4
@@ -0,0 +1,4 @@
1 +BEGIN
2 + DBMS_LOCK.SLEEP(30);
3 +END;
4 +/
src/go/tools/functions-validation/seed/proxysql/init.sh new
+40
@@ -0,0 +1,40 @@
1 +#!/usr/bin/env bash
2 +set -euo pipefail
3 +
4 +PROXY_HOST="${PROXY_HOST:-proxysql}"
5 +ADMIN_USER="${ADMIN_USER:-admin}"
6 +ADMIN_PASS="${ADMIN_PASS:-admin}"
7 +BACKEND_HOST="${BACKEND_HOST:-mysql}"
8 +BACKEND_PORT="${BACKEND_PORT:-3306}"
9 +
10 +for i in $(seq 1 30); do
11 + if mysql -h "$PROXY_HOST" -P 6032 -u "$ADMIN_USER" -p"$ADMIN_PASS" -e "SELECT 1" > /dev/null 2>&1; then
12 + break
13 + fi
14 + sleep 2
15 +done
16 +
17 +mysql -h "$PROXY_HOST" -P 6032 -u "$ADMIN_USER" -p"$ADMIN_PASS" <<SQL
18 +UPDATE global_variables SET variable_value='admin:admin;netdata:netdata' WHERE variable_name='admin-admin_credentials';
19 +UPDATE global_variables SET variable_value='0.0.0.0:6032' WHERE variable_name='admin-mysql_ifaces';
20 +LOAD ADMIN VARIABLES TO RUNTIME;
21 +SAVE ADMIN VARIABLES TO DISK;
22 +DELETE FROM mysql_servers;
23 +INSERT INTO mysql_servers(hostgroup_id, hostname, port) VALUES (0, '$BACKEND_HOST', $BACKEND_PORT);
24 +DELETE FROM mysql_users;
25 +INSERT INTO mysql_users(username, password, default_hostgroup) VALUES ('netdata', 'netdata', 0);
26 +LOAD MYSQL SERVERS TO RUNTIME;
27 +SAVE MYSQL SERVERS TO DISK;
28 +LOAD MYSQL USERS TO RUNTIME;
29 +SAVE MYSQL USERS TO DISK;
30 +SQL
31 +
32 +mysql -h "$PROXY_HOST" -P 6033 -u netdata -pnetdata <<SQL
33 +CREATE DATABASE IF NOT EXISTS netdata;
34 +CREATE TABLE IF NOT EXISTS netdata.t (id INT PRIMARY KEY, v VARCHAR(10));
35 +INSERT INTO netdata.t (id, v) VALUES (1, 'a') ON DUPLICATE KEY UPDATE v='a';
36 +INSERT INTO netdata.t (id, v) VALUES (2, 'b') ON DUPLICATE KEY UPDATE v='b';
37 +SELECT * FROM netdata.t WHERE id > 0;
38 +SQL
39 +
40 +sleep 2
src/go/tools/functions-validation/seed/proxysql/proxysql.cnf new
+53
@@ -0,0 +1,53 @@
1 +# proxysql config for e2e
2 +
3 +datadir="/var/lib/proxysql"
4 +errorlog="/var/lib/proxysql/proxysql.log"
5 +
6 +admin_variables=
7 +{
8 + admin_credentials="admin:admin;netdata:netdata"
9 + stats_credentials="stats:stats"
10 + mysql_ifaces="0.0.0.0:6032"
11 +}
12 +
13 +mysql_variables=
14 +{
15 + threads=4
16 + max_connections=2048
17 + default_query_delay=0
18 + default_query_timeout=36000000
19 + have_compress=true
20 + poll_timeout=2000
21 + interfaces="0.0.0.0:6033"
22 + default_schema="information_schema"
23 + stacksize=1048576
24 + server_version="5.5.30"
25 + connect_timeout_server=3000
26 + monitor_username="monitor"
27 + monitor_password="monitor"
28 + monitor_history=600000
29 + monitor_connect_interval=60000
30 + monitor_ping_interval=10000
31 + monitor_read_only_interval=1500
32 + monitor_read_only_timeout=500
33 + ping_interval_server_msec=120000
34 + ping_timeout_server=500
35 + commands_stats=true
36 + sessions_sort=true
37 + connect_retries_on_failure=10
38 +}
39 +
40 +mysql_servers = (
41 +)
42 +
43 +mysql_users: (
44 +)
45 +
46 +mysql_query_rules: (
47 +)
48 +
49 +scheduler=(
50 +)
51 +
52 +mysql_replication_hostgroups=(
53 +)
src/go/tools/functions-validation/seed/redis/init.sh new
+14
@@ -0,0 +1,14 @@
1 +#!/usr/bin/env bash
2 +set -euo pipefail
3 +
4 +REDIS_HOST="${REDIS_HOST:-redis}"
5 +REDIS_PORT="${REDIS_PORT:-6379}"
6 +
7 +redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" CONFIG SET slowlog-log-slower-than 0 > /dev/null
8 +redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" CONFIG SET slowlog-max-len 1024 > /dev/null
9 +
10 +redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" SET foo bar > /dev/null
11 +redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" GET foo > /dev/null
12 +redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" INCR counter > /dev/null
13 +redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" LPUSH list a b c > /dev/null
14 +redis-cli -h "$REDIS_HOST" -p "$REDIS_PORT" LRANGE list 0 10 > /dev/null
src/go/tools/functions-validation/seed/rethinkdb/hold.go new
+73
@@ -0,0 +1,73 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package main
4 +
5 +import (
6 + "context"
7 + "flag"
8 + "fmt"
9 + "os"
10 + "strings"
11 + "time"
12 +
13 + "gopkg.in/rethinkdb/rethinkdb-go.v6"
14 +)
15 +
16 +func main() {
17 + addr := flag.String("addr", "127.0.0.1:28015", "RethinkDB address")
18 + duration := flag.Duration("duration", 30*time.Second, "How long to keep the changefeed open")
19 + flag.Parse()
20 +
21 + sess, err := rethinkdb.Connect(rethinkdb.ConnectOpts{Address: *addr})
22 + if err != nil {
23 + fatalf("connect: %v", err)
24 + }
25 + defer func() { _ = sess.Close() }()
26 +
27 + ensureDB(sess, "netdata")
28 + ensureTable(sess, "netdata", "demo")
29 + insertSeed(sess, "netdata", "demo")
30 +
31 + ctx, cancel := context.WithTimeout(context.Background(), *duration)
32 + defer cancel()
33 +
34 + cur, err := rethinkdb.DB("netdata").Table("demo").Changes().Run(sess, rethinkdb.RunOpts{Context: ctx})
35 + if err != nil {
36 + fatalf("start changefeed: %v", err)
37 + }
38 + defer func() { _ = cur.Close() }()
39 +
40 + <-ctx.Done()
41 +}
42 +
43 +func ensureDB(sess *rethinkdb.Session, name string) {
44 + if _, err := rethinkdb.DBCreate(name).RunWrite(sess); err != nil {
45 + if !isAlreadyExists(err) {
46 + fatalf("db create: %v", err)
47 + }
48 + }
49 +}
50 +
51 +func ensureTable(sess *rethinkdb.Session, db, table string) {
52 + if _, err := rethinkdb.DB(db).TableCreate(table).RunWrite(sess); err != nil {
53 + if !isAlreadyExists(err) {
54 + fatalf("table create: %v", err)
55 + }
56 + }
57 +}
58 +
59 +func insertSeed(sess *rethinkdb.Session, db, table string) {
60 + _, _ = rethinkdb.DB(db).Table(table).Insert(map[string]any{
61 + "id": "seed",
62 + "name": "alpha",
63 + }).RunWrite(sess)
64 +}
65 +
66 +func isAlreadyExists(err error) bool {
67 + return err != nil && (strings.Contains(err.Error(), "already exists") || strings.Contains(err.Error(), "Duplicate"))
68 +}
69 +
70 +func fatalf(format string, args ...any) {
71 + _, _ = fmt.Fprintf(os.Stderr, format+"\n", args...)
72 + os.Exit(1)
73 +}
src/go/tools/functions-validation/seed/yugabytedb/seed.sh new
+4
@@ -0,0 +1,4 @@
1 +#!/usr/bin/env sh
2 +set -e
3 +
4 +/home/yugabyte/bin/ysqlsh -h yugabytedb -U yugabyte -d yugabyte -f /seed/seed.sql
src/go/tools/functions-validation/seed/yugabytedb/seed.sql new
+16
@@ -0,0 +1,16 @@
1 +CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
2 +
3 +CREATE TABLE IF NOT EXISTS items (
4 + id INT PRIMARY KEY,
5 + name TEXT
6 +);
7 +
8 +INSERT INTO items (id, name) VALUES
9 + (1, 'alpha'),
10 + (2, 'beta'),
11 + (3, 'gamma')
12 +ON CONFLICT (id) DO NOTHING;
13 +
14 +SELECT * FROM items WHERE id = 1;
15 +UPDATE items SET name = 'delta' WHERE id = 2;
16 +SELECT count(*) FROM items;
src/go/tools/functions-validation/seed/yugabytedb/sleep.sh new
+4
@@ -0,0 +1,4 @@
1 +#!/usr/bin/env sh
2 +set -e
3 +
4 +/home/yugabyte/bin/ysqlsh -h yugabytedb -U yugabyte -d yugabyte -f /seed/sleep.sql
src/go/tools/functions-validation/seed/yugabytedb/sleep.sql new
+1
@@ -0,0 +1 @@
1 +SELECT pg_sleep(30);
src/go/tools/functions-validation/validate/main.go
+6 -2
@@ -3,7 +3,6 @@
3 package main
4
5 import (
6 - "bytes"
6 "encoding/json"
7 "flag"
8 "fmt"
@@ -41,8 +40,13 @@ func main() {
40 exitErr("parse input JSON: %v", err)
41 }
42
43 + var schemaDoc any
44 + if err := json.Unmarshal(schemaBytes, &schemaDoc); err != nil {
45 + exitErr("parse schema JSON: %v", err)
46 + }
47 +
48 compiler := jsonschema.NewCompiler()
45 - if err := compiler.AddResource("schema.json", bytes.NewReader(schemaBytes)); err != nil {
49 + if err := compiler.AddResource("schema.json", schemaDoc); err != nil {
50 exitErr("add schema resource: %v", err)
51 }
52 schema, err := compiler.Compile("schema.json")