@cryptotaxi247 / netdata-1 / commits / 27984cdcd

refactor(go.d.plugin): functions restructure (#21633)

Ilya Mashchenko committed Jan 26, 2026 at 13:48 UTC 27984cdcd18e159dd79273e0349a466eef54014c
86 files changed +8367 -10022
src/go/pkg/funcapi/builders.go new
+256
@@ -0,0 +1,256 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package funcapi
4 +
5 +// DefaultMaxQueryLength is the default maximum length for query text display.
6 +const DefaultMaxQueryLength = 4096
7 +
8 +// ColumnSet wraps a typed column slice with its accessor function.
9 +// This allows all builder methods to work without repeating the accessor.
10 +//
11 +// Usage:
12 +//
13 +// cs := funcapi.Columns(cols, func(c myColumn) funcapi.ColumnMeta { return c.ColumnMeta })
14 +// response := &funcapi.FunctionResponse{
15 +// Columns: cs.BuildColumns(),
16 +// Charts: cs.BuildCharts(),
17 +// DefaultCharts: cs.BuildDefaultCharts(),
18 +// GroupBy: cs.BuildGroupBy(),
19 +// }
20 +type ColumnSet[T any] struct {
21 + cols []T
22 + getMeta func(T) ColumnMeta
23 +}
24 +
25 +// Columns creates a new ColumnSet from a typed slice and accessor.
26 +func Columns[T any](cols []T, getMeta func(T) ColumnMeta) ColumnSet[T] {
27 + return ColumnSet[T]{cols: cols, getMeta: getMeta}
28 +}
29 +
30 +// Len returns the number of columns.
31 +func (cs ColumnSet[T]) Len() int {
32 + return len(cs.cols)
33 +}
34 +
35 +// BuildColumns builds the columns map for FunctionResponse.
36 +func (cs ColumnSet[T]) BuildColumns() map[string]any {
37 + result := make(map[string]any, len(cs.cols))
38 + for i, col := range cs.cols {
39 + meta := cs.getMeta(col)
40 + vis := meta.Visualization
41 + if vis == FieldVisualValue && meta.Type == FieldTypeDuration {
42 + vis = FieldVisualBar
43 + }
44 + c := Column{
45 + Index: i,
46 + Name: meta.Tooltip,
47 + Type: meta.Type,
48 + Units: meta.Units,
49 + Visualization: vis,
50 + Sort: meta.Sort,
51 + Sortable: meta.Sortable,
52 + Sticky: meta.Sticky,
53 + Summary: meta.Summary,
54 + Filter: meta.Filter,
55 + FullWidth: meta.FullWidth,
56 + Wrap: meta.Wrap,
57 + DefaultExpandedFilter: meta.ExpandFilter,
58 + UniqueKey: meta.UniqueKey,
59 + Visible: meta.Visible,
60 + ValueOptions: ValueOptions{
61 + Transform: meta.Transform,
62 + DecimalPoints: meta.DecimalPoints,
63 + },
64 + }
65 + result[meta.Name] = c.BuildColumn()
66 + }
67 + return result
68 +}
69 +
70 +// BuildCharts builds chart configuration from column metadata.
71 +// Columns with Chart != nil are grouped by Chart.Group.
72 +func (cs ColumnSet[T]) BuildCharts() map[string]ChartConfig {
73 + charts := make(map[string]ChartConfig)
74 + for _, col := range cs.cols {
75 + meta := cs.getMeta(col)
76 + if meta.Chart == nil || meta.Chart.Group == "" {
77 + continue
78 + }
79 + cfg, ok := charts[meta.Chart.Group]
80 + if !ok {
81 + title := meta.Chart.Title
82 + if title == "" {
83 + title = meta.Chart.Group
84 + }
85 + cfg = ChartConfig{Name: title, Type: "stacked-bar"}
86 + }
87 + cfg.Columns = append(cfg.Columns, meta.Name)
88 + charts[meta.Chart.Group] = cfg
89 + }
90 + return charts
91 +}
92 +
93 +// BuildGroupBy builds group-by configuration from columns with GroupBy != nil.
94 +func (cs ColumnSet[T]) BuildGroupBy() map[string]GroupByConfig {
95 + result := make(map[string]GroupByConfig)
96 + for _, col := range cs.cols {
97 + meta := cs.getMeta(col)
98 + if meta.GroupBy == nil {
99 + continue
100 + }
101 + result[meta.Name] = GroupByConfig{
102 + Name: "Group by " + meta.Tooltip,
103 + Columns: []string{meta.Name},
104 + }
105 + }
106 + return result
107 +}
108 +
109 +// BuildDefaultCharts builds default chart configurations.
110 +// Each chart uses its own DefaultGroupBy if set, otherwise falls back to global default.
111 +func (cs ColumnSet[T]) BuildDefaultCharts() DefaultCharts {
112 + globalGroupBy := cs.FindDefaultGroupBy()
113 + groups := cs.FindDefaultChartGroups()
114 + var result DefaultCharts
115 + for _, g := range groups {
116 + groupBy := cs.findChartGroupBy(g)
117 + if groupBy == "" {
118 + groupBy = globalGroupBy
119 + }
120 + if groupBy == "" {
121 + continue
122 + }
123 + result = append(result, DefaultChart{Chart: g, GroupBy: groupBy})
124 + }
125 + return result
126 +}
127 +
128 +// findChartGroupBy returns the DefaultGroupBy for a specific chart group.
129 +func (cs ColumnSet[T]) findChartGroupBy(chartGroup string) string {
130 + for _, col := range cs.cols {
131 + meta := cs.getMeta(col)
132 + if meta.Chart != nil && meta.Chart.Group == chartGroup && meta.Chart.DefaultGroupBy != "" {
133 + return meta.Chart.DefaultGroupBy
134 + }
135 + }
136 + return ""
137 +}
138 +
139 +// BuildCharting builds the complete charting configuration.
140 +func (cs ColumnSet[T]) BuildCharting() ChartingConfig {
141 + return ChartingConfig{
142 + Charts: cs.BuildCharts(),
143 + DefaultCharts: cs.BuildDefaultCharts(),
144 + GroupBy: cs.BuildGroupBy(),
145 + }
146 +}
147 +
148 +// FindDefaultGroupBy finds the default grouping column name.
149 +// Returns the column with GroupBy.IsDefault, or falls back to any GroupBy column.
150 +func (cs ColumnSet[T]) FindDefaultGroupBy() string {
151 + for _, col := range cs.cols {
152 + meta := cs.getMeta(col)
153 + if meta.GroupBy != nil && meta.GroupBy.IsDefault {
154 + return meta.Name
155 + }
156 + }
157 + for _, col := range cs.cols {
158 + meta := cs.getMeta(col)
159 + if meta.GroupBy != nil {
160 + return meta.Name
161 + }
162 + }
163 + return ""
164 +}
165 +
166 +// FindDefaultChartGroups returns chart groups, preferring ones with IsDefault.
167 +func (cs ColumnSet[T]) FindDefaultChartGroups() []string {
168 + var groups []string
169 + seen := make(map[string]bool)
170 +
171 + // First pass: chart groups marked as default
172 + for _, col := range cs.cols {
173 + meta := cs.getMeta(col)
174 + if meta.Chart != nil && meta.Chart.Group != "" && meta.Chart.IsDefault && !seen[meta.Chart.Group] {
175 + seen[meta.Chart.Group] = true
176 + groups = append(groups, meta.Chart.Group)
177 + }
178 + }
179 + if len(groups) > 0 {
180 + return groups
181 + }
182 +
183 + // Fallback: all chart groups
184 + for _, col := range cs.cols {
185 + meta := cs.getMeta(col)
186 + if meta.Chart != nil && meta.Chart.Group != "" && !seen[meta.Chart.Group] {
187 + seen[meta.Chart.Group] = true
188 + groups = append(groups, meta.Chart.Group)
189 + }
190 + }
191 + return groups
192 +}
193 +
194 +// ContainsColumn checks if a column name exists in the set.
195 +func (cs ColumnSet[T]) ContainsColumn(name string) bool {
196 + for _, col := range cs.cols {
197 + if cs.getMeta(col).Name == name {
198 + return true
199 + }
200 + }
201 + return false
202 +}
203 +
204 +// Names returns all column names.
205 +func (cs ColumnSet[T]) Names() []string {
206 + names := make([]string, len(cs.cols))
207 + for i, col := range cs.cols {
208 + names[i] = cs.getMeta(col).Name
209 + }
210 + return names
211 +}
212 +
213 +// SortableColumn is an interface for columns that can provide sort options.
214 +// Collectors implement this on their column types to use BuildSortParam.
215 +type SortableColumn interface {
216 + IsSortOption() bool
217 + SortLabel() string
218 + IsDefaultSort() bool
219 + ColumnName() string
220 + SortColumn() string // Returns the column value for sorting; empty string uses ColumnName()
221 +}
222 +
223 +// BuildSortParam builds a sort parameter configuration from sortable columns.
224 +// Only columns where IsSortOption() returns true are included as options.
225 +// Uses SortColumn() for the Column value if non-empty, otherwise ColumnName().
226 +func BuildSortParam[T SortableColumn](cols []T) ParamConfig {
227 + var options []ParamOption
228 + sortDir := FieldSortDescending
229 + for _, col := range cols {
230 + if !col.IsSortOption() {
231 + continue
232 + }
233 + column := col.SortColumn()
234 + if column == "" {
235 + column = col.ColumnName()
236 + }
237 + opt := ParamOption{
238 + ID: col.ColumnName(),
239 + Column: column,
240 + Name: col.SortLabel(),
241 + Sort: &sortDir,
242 + }
243 + if col.IsDefaultSort() {
244 + opt.Default = true
245 + }
246 + options = append(options, opt)
247 + }
248 + return ParamConfig{
249 + ID: "__sort",
250 + Name: "Filter By",
251 + Help: "Select the primary sort column",
252 + Selection: ParamSelect,
253 + Options: options,
254 + UniqueView: true,
255 + }
256 +}
src/go/pkg/funcapi/builders_test.go new
+317
@@ -0,0 +1,317 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package funcapi
4 +
5 +import (
6 + "testing"
7 +
8 + "github.com/stretchr/testify/assert"
9 + "github.com/stretchr/testify/require"
10 +)
11 +
12 +// testColumn is a test column type that embeds ColumnMeta.
13 +type testColumn struct {
14 + ColumnMeta
15 + Extra string // Simulates collector-specific field
16 +}
17 +
18 +func testColumnSet(cols ...testColumn) ColumnSet[testColumn] {
19 + return Columns(cols, func(c testColumn) ColumnMeta { return c.ColumnMeta })
20 +}
21 +
22 +func TestColumnSet_Len(t *testing.T) {
23 + cs := testColumnSet(
24 + testColumn{ColumnMeta: ColumnMeta{Name: "a"}},
25 + testColumn{ColumnMeta: ColumnMeta{Name: "b"}},
26 + )
27 + assert.Equal(t, 2, cs.Len())
28 +}
29 +
30 +func TestColumnSet_Names(t *testing.T) {
31 + cs := testColumnSet(
32 + testColumn{ColumnMeta: ColumnMeta{Name: "col1"}},
33 + testColumn{ColumnMeta: ColumnMeta{Name: "col2"}},
34 + testColumn{ColumnMeta: ColumnMeta{Name: "col3"}},
35 + )
36 + assert.Equal(t, []string{"col1", "col2", "col3"}, cs.Names())
37 +}
38 +
39 +func TestColumnSet_ContainsColumn(t *testing.T) {
40 + cs := testColumnSet(
41 + testColumn{ColumnMeta: ColumnMeta{Name: "exists"}},
42 + )
43 + assert.True(t, cs.ContainsColumn("exists"))
44 + assert.False(t, cs.ContainsColumn("not_exists"))
45 +}
46 +
47 +func TestColumnSet_BuildColumns(t *testing.T) {
48 + cs := testColumnSet(
49 + testColumn{ColumnMeta: ColumnMeta{
50 + Name: "query",
51 + Tooltip: "Query",
52 + Type: FieldTypeString,
53 + Visible: true,
54 + Sortable: true,
55 + Filter: FieldFilterMultiselect,
56 + }},
57 + testColumn{ColumnMeta: ColumnMeta{
58 + Name: "duration",
59 + Tooltip: "Duration",
60 + Type: FieldTypeDuration,
61 + Units: "ms",
62 + Visible: true,
63 + Sortable: true,
64 + Transform: FieldTransformDuration,
65 + DecimalPoints: 2,
66 + }},
67 + )
68 +
69 + result := cs.BuildColumns()
70 +
71 + require.Len(t, result, 2)
72 + require.Contains(t, result, "query")
73 + require.Contains(t, result, "duration")
74 +
75 + // Check query column
76 + queryCol := result["query"].(map[string]any)
77 + assert.Equal(t, 0, queryCol["index"])
78 + assert.Equal(t, "Query", queryCol["name"])
79 + assert.Equal(t, "string", queryCol["type"])
80 + assert.Equal(t, true, queryCol["visible"])
81 + assert.Equal(t, true, queryCol["sortable"])
82 +
83 + // Check duration column - should auto-switch to bar visualization
84 + durationCol := result["duration"].(map[string]any)
85 + assert.Equal(t, 1, durationCol["index"])
86 + assert.Equal(t, "Duration", durationCol["name"])
87 + assert.Equal(t, "duration", durationCol["type"])
88 + assert.Equal(t, "bar", durationCol["visualization"]) // Auto-switched from value to bar
89 + assert.Equal(t, "ms", durationCol["units"])
90 +}
91 +
92 +func TestColumnSet_BuildColumns_ExplicitVisualization(t *testing.T) {
93 + // When visualization is explicitly set, it should NOT be overridden
94 + cs := testColumnSet(
95 + testColumn{ColumnMeta: ColumnMeta{
96 + Name: "duration",
97 + Tooltip: "Duration",
98 + Type: FieldTypeDuration,
99 + Visualization: FieldVisualPill, // Explicitly set to pill
100 + }},
101 + )
102 +
103 + result := cs.BuildColumns()
104 + durationCol := result["duration"].(map[string]any)
105 + assert.Equal(t, "pill", durationCol["visualization"]) // Should remain pill, not bar
106 +}
107 +
108 +func TestColumnSet_BuildCharts(t *testing.T) {
109 + cs := testColumnSet(
110 + testColumn{ColumnMeta: ColumnMeta{
111 + Name: "query",
112 + Tooltip: "Query",
113 + Chart: nil, // Not a chart metric
114 + }},
115 + testColumn{ColumnMeta: ColumnMeta{
116 + Name: "exec_time",
117 + Tooltip: "Execution Time",
118 + Chart: &ChartOptions{Group: "Time", Title: "Query Time"},
119 + }},
120 + testColumn{ColumnMeta: ColumnMeta{
121 + Name: "cpu_time",
122 + Tooltip: "CPU Time",
123 + Chart: &ChartOptions{Group: "Time", Title: "Query Time"},
124 + }},
125 + testColumn{ColumnMeta: ColumnMeta{
126 + Name: "rows",
127 + Tooltip: "Rows",
128 + Chart: &ChartOptions{Group: "Rows", Title: "Row Count"},
129 + }},
130 + )
131 +
132 + result := cs.BuildCharts()
133 +
134 + require.Len(t, result, 2)
135 +
136 + // Time chart should have 2 columns
137 + timeChart := result["Time"]
138 + assert.Equal(t, "Query Time", timeChart.Name)
139 + assert.Equal(t, "stacked-bar", timeChart.Type)
140 + assert.Equal(t, []string{"exec_time", "cpu_time"}, timeChart.Columns)
141 +
142 + // Rows chart should have 1 column
143 + rowsChart := result["Rows"]
144 + assert.Equal(t, "Row Count", rowsChart.Name)
145 + assert.Equal(t, []string{"rows"}, rowsChart.Columns)
146 +}
147 +
148 +func TestColumnSet_BuildCharts_FallbackTitle(t *testing.T) {
149 + cs := testColumnSet(
150 + testColumn{ColumnMeta: ColumnMeta{
151 + Name: "metric",
152 + Chart: &ChartOptions{Group: "MyGroup", Title: ""}, // Empty title
153 + }},
154 + )
155 +
156 + result := cs.BuildCharts()
157 + assert.Equal(t, "MyGroup", result["MyGroup"].Name) // Falls back to Group
158 +}
159 +
160 +func TestColumnSet_BuildGroupBy(t *testing.T) {
161 + cs := testColumnSet(
162 + testColumn{ColumnMeta: ColumnMeta{
163 + Name: "query",
164 + Tooltip: "Query",
165 + GroupBy: &GroupByOptions{},
166 + }},
167 + testColumn{ColumnMeta: ColumnMeta{
168 + Name: "database",
169 + Tooltip: "Database",
170 + GroupBy: &GroupByOptions{},
171 + }},
172 + testColumn{ColumnMeta: ColumnMeta{
173 + Name: "duration",
174 + Tooltip: "Duration",
175 + GroupBy: nil, // Not available for grouping
176 + }},
177 + )
178 +
179 + result := cs.BuildGroupBy()
180 +
181 + require.Len(t, result, 2)
182 + assert.Equal(t, "Group by Query", result["query"].Name)
183 + assert.Equal(t, []string{"query"}, result["query"].Columns)
184 + assert.Equal(t, "Group by Database", result["database"].Name)
185 +}
186 +
187 +func TestColumnSet_FindDefaultGroupBy(t *testing.T) {
188 + tests := []struct {
189 + name string
190 + cols []testColumn
191 + expected string
192 + }{
193 + {
194 + name: "returns default groupby if exists",
195 + cols: []testColumn{
196 + {ColumnMeta: ColumnMeta{Name: "a", GroupBy: &GroupByOptions{}}},
197 + {ColumnMeta: ColumnMeta{Name: "b", GroupBy: &GroupByOptions{IsDefault: true}}},
198 + {ColumnMeta: ColumnMeta{Name: "c", GroupBy: &GroupByOptions{}}},
199 + },
200 + expected: "b",
201 + },
202 + {
203 + name: "falls back to first groupby if no default",
204 + cols: []testColumn{
205 + {ColumnMeta: ColumnMeta{Name: "a", GroupBy: nil}},
206 + {ColumnMeta: ColumnMeta{Name: "b", GroupBy: &GroupByOptions{}}},
207 + {ColumnMeta: ColumnMeta{Name: "c", GroupBy: &GroupByOptions{}}},
208 + },
209 + expected: "b",
210 + },
211 + {
212 + name: "returns empty if no groupby columns",
213 + cols: []testColumn{
214 + {ColumnMeta: ColumnMeta{Name: "a"}},
215 + {ColumnMeta: ColumnMeta{Name: "b"}},
216 + },
217 + expected: "",
218 + },
219 + }
220 +
221 + for _, tt := range tests {
222 + t.Run(tt.name, func(t *testing.T) {
223 + cs := testColumnSet(tt.cols...)
224 + assert.Equal(t, tt.expected, cs.FindDefaultGroupBy())
225 + })
226 + }
227 +}
228 +
229 +func TestColumnSet_FindDefaultChartGroups(t *testing.T) {
230 + tests := []struct {
231 + name string
232 + cols []testColumn
233 + expected []string
234 + }{
235 + {
236 + name: "returns default chart groups",
237 + cols: []testColumn{
238 + {ColumnMeta: ColumnMeta{Name: "a", Chart: &ChartOptions{Group: "A", IsDefault: true}}},
239 + {ColumnMeta: ColumnMeta{Name: "b", Chart: &ChartOptions{Group: "B", IsDefault: false}}},
240 + {ColumnMeta: ColumnMeta{Name: "c", Chart: &ChartOptions{Group: "C", IsDefault: true}}},
241 + },
242 + expected: []string{"A", "C"},
243 + },
244 + {
245 + name: "falls back to all groups if no defaults",
246 + cols: []testColumn{
247 + {ColumnMeta: ColumnMeta{Name: "a", Chart: &ChartOptions{Group: "A", IsDefault: false}}},
248 + {ColumnMeta: ColumnMeta{Name: "b", Chart: &ChartOptions{Group: "B", IsDefault: false}}},
249 + },
250 + expected: []string{"A", "B"},
251 + },
252 + {
253 + name: "deduplicates groups",
254 + cols: []testColumn{
255 + {ColumnMeta: ColumnMeta{Name: "a", Chart: &ChartOptions{Group: "Same", IsDefault: true}}},
256 + {ColumnMeta: ColumnMeta{Name: "b", Chart: &ChartOptions{Group: "Same", IsDefault: true}}},
257 + },
258 + expected: []string{"Same"},
259 + },
260 + {
261 + name: "returns nil for no chart columns",
262 + cols: []testColumn{
263 + {ColumnMeta: ColumnMeta{Name: "a", Chart: nil}},
264 + },
265 + expected: nil,
266 + },
267 + }
268 +
269 + for _, tt := range tests {
270 + t.Run(tt.name, func(t *testing.T) {
271 + cs := testColumnSet(tt.cols...)
272 + assert.Equal(t, tt.expected, cs.FindDefaultChartGroups())
273 + })
274 + }
275 +}
276 +
277 +func TestColumnSet_BuildDefaultCharts(t *testing.T) {
278 + cs := testColumnSet(
279 + testColumn{ColumnMeta: ColumnMeta{Name: "query", GroupBy: &GroupByOptions{IsDefault: true}}},
280 + testColumn{ColumnMeta: ColumnMeta{Name: "time", Chart: &ChartOptions{Group: "Time", IsDefault: true}}},
281 + testColumn{ColumnMeta: ColumnMeta{Name: "rows", Chart: &ChartOptions{Group: "Rows", IsDefault: true}}},
282 + )
283 +
284 + result := cs.BuildDefaultCharts()
285 +
286 + expected := DefaultCharts{
287 + {Chart: "Time", GroupBy: "query"},
288 + {Chart: "Rows", GroupBy: "query"},
289 + }
290 + assert.Equal(t, expected, result)
291 +
292 + // Test Build() method for JSON output
293 + assert.Equal(t, [][]string{{"Time", "query"}, {"Rows", "query"}}, result.Build())
294 +}
295 +
296 +func TestColumnSet_BuildDefaultCharts_NoGroupBy(t *testing.T) {
297 + cs := testColumnSet(
298 + testColumn{ColumnMeta: ColumnMeta{Name: "metric", Chart: &ChartOptions{Group: "G"}}},
299 + )
300 +
301 + result := cs.BuildDefaultCharts()
302 + assert.Nil(t, result)
303 +}
304 +
305 +func TestColumnSet_Empty(t *testing.T) {
306 + cs := testColumnSet()
307 +
308 + assert.Equal(t, 0, cs.Len())
309 + assert.Empty(t, cs.Names())
310 + assert.Empty(t, cs.BuildColumns())
311 + assert.Empty(t, cs.BuildCharts())
312 + assert.Empty(t, cs.BuildGroupBy())
313 + assert.Nil(t, cs.BuildDefaultCharts())
314 + assert.Equal(t, "", cs.FindDefaultGroupBy())
315 + assert.Nil(t, cs.FindDefaultChartGroups())
316 + assert.False(t, cs.ContainsColumn("any"))
317 +}
src/go/pkg/funcapi/column_meta.go new
+60
@@ -0,0 +1,60 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package funcapi
4 +
5 +// ColumnMeta defines UI metadata for a table column.
6 +// This struct contains ONLY column display and visualization properties.
7 +//
8 +// What belongs here: How to render and display a column in the table.
9 +// What does NOT belong here: Data access (SQL, BSON), parameters (sort options).
10 +//
11 +// Collectors embed this and add their own fields for:
12 +// - Data access (SelectExpr, DBField, Value func, etc.)
13 +// - Parameter-related metadata (sort options, filter options - these are method-specific)
14 +type ColumnMeta struct {
15 + // Identity
16 + Name string // Column name/identifier in response (e.g., "execution_time")
17 + Tooltip string // Hover tooltip text shown in UI (e.g., "Execution Time")
18 +
19 + // Type and Display
20 + Type FieldType
21 + Units string
22 + Visible bool
23 + Sortable bool // Can this column be sorted in the table UI?
24 + Sticky bool
25 + FullWidth bool
26 + Wrap bool
27 +
28 + // Value Rendering
29 + Transform FieldTransform
30 + DecimalPoints int
31 + Sort FieldSort // Default sort direction for this column
32 + Summary FieldSummary
33 + Filter FieldFilter
34 + Visualization FieldVisual
35 +
36 + // Special Flags
37 + UniqueKey bool
38 + ExpandFilter bool
39 +
40 + // Chart configuration (nil = column is not a chart metric)
41 + // Used by BuildCharts() and BuildDefaultCharts()
42 + Chart *ChartOptions
43 +
44 + // GroupBy configuration (nil = column not available for grouping)
45 + // Used by BuildGroupBy() and BuildDefaultCharts()
46 + GroupBy *GroupByOptions
47 +}
48 +
49 +// ChartOptions defines how a column participates in charts.
50 +type ChartOptions struct {
51 + Group string // Which chart this column belongs to (required)
52 + Title string // Chart display title (defaults to Group if empty)
53 + IsDefault bool // Include this chart in the default charts view
54 + DefaultGroupBy string // GroupBy column name for this chart's default (optional, falls back to global)
55 +}
56 +
57 +// GroupByOptions defines how a column participates in data grouping.
58 +type GroupByOptions struct {
59 + IsDefault bool // This is the default grouping column for charts
60 +}
src/go/pkg/funcapi/handler.go new
+75
@@ -0,0 +1,75 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package funcapi
4 +
5 +import (
6 + "context"
7 + "fmt"
8 +)
9 +
10 +// MethodHandler defines the interface for handling method requests.
11 +// Methods are defined in Creator.Methods(); this interface handles the requests.
12 +//
13 +// Example implementation:
14 +//
15 +// type funcTopQueries struct {
16 +// db *sql.DB
17 +// }
18 +//
19 +// func (f *funcTopQueries) MethodParams(ctx context.Context, method string) ([]ParamConfig, error) {
20 +// return nil, nil // or return dynamic params from database
21 +// }
22 +//
23 +// func (f *funcTopQueries) Handle(ctx context.Context, method string, params ResolvedParams) *FunctionResponse {
24 +// // query database and build response
25 +// }
26 +type MethodHandler interface {
27 + // MethodParams returns dynamic params for a method.
28 + // Return nil to use static params from MethodConfig.RequiredParams.
29 + // The context should be used for timeout/cancellation of database queries.
30 + MethodParams(ctx context.Context, method string) ([]ParamConfig, error)
31 +
32 + // Handle processes a method request and returns the response.
33 + // The context should be used for timeout/cancellation of database queries.
34 + Handle(ctx context.Context, method string, params ResolvedParams) *FunctionResponse
35 +
36 + // Cleanup releases any resources held by the handler.
37 + // Called when the collector is being stopped.
38 + Cleanup(ctx context.Context)
39 +}
40 +
41 +// ErrorResponse creates an error FunctionResponse.
42 +func ErrorResponse(status int, format string, args ...any) *FunctionResponse {
43 + msg := format
44 + if len(args) > 0 {
45 + msg = fmt.Sprintf(format, args...)
46 + }
47 + return &FunctionResponse{
48 + Status: status,
49 + Message: msg,
50 + }
51 +}
52 +
53 +// NotFoundResponse returns a 404 response for unknown methods.
54 +func NotFoundResponse(method string) *FunctionResponse {
55 + return &FunctionResponse{
56 + Status: 404,
57 + Message: "unknown method: " + method,
58 + }
59 +}
60 +
61 +// UnavailableResponse returns a 503 response when data is not yet available.
62 +func UnavailableResponse(msg string) *FunctionResponse {
63 + return &FunctionResponse{
64 + Status: 503,
65 + Message: msg,
66 + }
67 +}
68 +
69 +// InternalErrorResponse returns a 500 response for internal errors.
70 +func InternalErrorResponse(format string, args ...any) *FunctionResponse {
71 + return &FunctionResponse{
72 + Status: 500,
73 + Message: fmt.Sprintf(format, args...),
74 + }
75 +}
src/go/pkg/funcapi/handler_test.go new
+86
@@ -0,0 +1,86 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package funcapi
4 +
5 +import (
6 + "context"
7 + "testing"
8 +
9 + "github.com/stretchr/testify/assert"
10 +)
11 +
12 +// mockHandler implements MethodHandler for testing.
13 +type mockHandler struct {
14 + methods []MethodConfig
15 + methodParams []ParamConfig
16 + response *FunctionResponse
17 +}
18 +
19 +func (m *mockHandler) Methods() []MethodConfig {
20 + return m.methods
21 +}
22 +
23 +func (m *mockHandler) MethodParams(ctx context.Context, method string) ([]ParamConfig, error) {
24 + return m.methodParams, nil
25 +}
26 +
27 +func (m *mockHandler) Handle(ctx context.Context, method string, params ResolvedParams) *FunctionResponse {
28 + return m.response
29 +}
30 +
31 +func (m *mockHandler) Cleanup(ctx context.Context) {}
32 +
33 +func TestMethodHandler_Interface(t *testing.T) {
34 + // Verify mockHandler implements MethodHandler
35 + var _ MethodHandler = &mockHandler{}
36 +
37 + h := &mockHandler{
38 + methods: []MethodConfig{{ID: "test", Name: "Test"}},
39 + response: &FunctionResponse{Status: 200},
40 + }
41 +
42 + assert.Len(t, h.Methods(), 1)
43 + assert.Equal(t, "test", h.Methods()[0].ID)
44 +
45 + params, err := h.MethodParams(context.Background(), "test")
46 + assert.NoError(t, err)
47 + assert.Nil(t, params)
48 +
49 + resp := h.Handle(context.Background(), "test", nil)
50 + assert.Equal(t, 200, resp.Status)
51 +}
52 +
53 +func TestErrorResponse(t *testing.T) {
54 + resp := ErrorResponse(500, "error: %s", "test")
55 +
56 + assert.Equal(t, 500, resp.Status)
57 + assert.Equal(t, "error: test", resp.Message)
58 +}
59 +
60 +func TestErrorResponse_NoArgs(t *testing.T) {
61 + resp := ErrorResponse(400, "bad request")
62 +
63 + assert.Equal(t, 400, resp.Status)
64 + assert.Equal(t, "bad request", resp.Message)
65 +}
66 +
67 +func TestNotFoundResponse(t *testing.T) {
68 + resp := NotFoundResponse("my-method")
69 +
70 + assert.Equal(t, 404, resp.Status)
71 + assert.Contains(t, resp.Message, "my-method")
72 +}
73 +
74 +func TestUnavailableResponse(t *testing.T) {
75 + resp := UnavailableResponse("data not ready")
76 +
77 + assert.Equal(t, 503, resp.Status)
78 + assert.Equal(t, "data not ready", resp.Message)
79 +}
80 +
81 +func TestInternalErrorResponse(t *testing.T) {
82 + resp := InternalErrorResponse("failed: %v", "connection refused")
83 +
84 + assert.Equal(t, 500, resp.Status)
85 + assert.Equal(t, "failed: connection refused", resp.Message)
86 +}
src/go/pkg/funcapi/response.go new
+72
@@ -0,0 +1,72 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package funcapi
4 +
5 +// MethodConfig describes a function method provided by a module.
6 +type MethodConfig struct {
7 + ID string // Method ID (e.g., "top-queries")
8 + Name string // Display name (e.g., "Top Queries")
9 + UpdateEvery int // Default UI refresh interval
10 + Help string // Description for UI
11 + RequireCloud bool // Indicates whether the method requires cloud connection
12 + RequiredParams []ParamConfig // Required parameters for this method (including __sort if used)
13 +}
14 +
15 +// FunctionResponse is the response from a module's HandleMethod.
16 +type FunctionResponse struct {
17 + Status int // HTTP-like status code (200, 400, 403, 500, 503)
18 + Message string // Error message (if Status != 200)
19 + Help string // Help text for this response
20 + Columns map[string]any // Column definitions for the table
21 + Data any // Row data: [][]any (array of arrays, ordered by column index)
22 + DefaultSortColumn string // Default sort column ID
23 +
24 + // Optional dynamic required params (override MethodConfig.RequiredParams)
25 + RequiredParams []ParamConfig
26 +
27 + // Chart configuration for visualization (embedded for JSON compatibility)
28 + ChartingConfig
29 +}
30 +
31 +// ChartingConfig groups chart visualization settings.
32 +// Embedded in FunctionResponse - JSON fields are promoted to top level.
33 +type ChartingConfig struct {
34 + Charts map[string]ChartConfig // Chart definitions (chartID -> config)
35 + DefaultCharts DefaultCharts // Default charts to display
36 + GroupBy map[string]GroupByConfig // Group-by options (groupByID -> config)
37 +}
38 +
39 +// DefaultChart represents a chart with its grouping.
40 +type DefaultChart struct {
41 + Chart string // Chart ID to display
42 + GroupBy string // Column to group by
43 +}
44 +
45 +// DefaultCharts is a list of default charts.
46 +type DefaultCharts []DefaultChart
47 +
48 +// Build converts DefaultCharts to [][]string for JSON response.
49 +// Output format: [["chartID", "groupByID"], ...]
50 +func (dc DefaultCharts) Build() [][]string {
51 + if len(dc) == 0 {
52 + return nil
53 + }
54 + result := make([][]string, len(dc))
55 + for i, c := range dc {
56 + result[i] = []string{c.Chart, c.GroupBy}
57 + }
58 + return result
59 +}
60 +
61 +// ChartConfig defines a chart for visualization.
62 +type ChartConfig struct {
63 + Name string `json:"name"`
64 + Type string `json:"type"` // "stacked-bar", "line", etc.
65 + Columns []string `json:"columns"` // Column IDs to include in chart
66 +}
67 +
68 +// GroupByConfig defines a grouping option for function responses.
69 +type GroupByConfig struct {
70 + Name string `json:"name"`
71 + Columns []string `json:"columns"` // Columns to group by
72 +}
src/go/plugin/go.d/agent/jobmgr/funcshandler.go
+18 -193
@@ -72,15 +72,22 @@ func (m *Manager) makeMethodFuncHandler(moduleName, methodID string) func(functi
72 return
73 }
74
75 - // Get the creator for this module to call HandleMethod
75 + // Get the creator for this module to call MethodHandler
76 creator, ok := m.moduleFuncs.getCreator(moduleName)
77 - if !ok || creator.HandleMethod == nil {
78 - m.respondError(fn, 500, "module '%s' does not implement HandleMethod", moduleName)
77 + if !ok || creator.MethodHandler == nil {
78 + m.respondError(fn, 500, "module '%s' does not implement MethodHandler", moduleName)
79 + return
80 + }
81 +
82 + // Get the handler for this job
83 + handler := creator.MethodHandler(job)
84 + if handler == nil {
85 + m.respondError(fn, 500, "module '%s' returned nil handler for job '%s'", moduleName, jobName)
86 return
87 }
88
89 // Resolve method-specific required params (job-aware)
83 - methodParams, paramsFromJob, err := m.resolveMethodParamsForJob(ctx, moduleName, methodID, methodCfg, job, creator)
90 + methodParams, paramsFromJob, err := m.resolveMethodParamsForJob(ctx, moduleName, methodID, methodCfg, job, handler)
91 if err != nil {
92 m.respondError(fn, 503, "job '%s' cannot provide parameters: %v", jobName, err)
93 return
@@ -102,7 +109,7 @@ func (m *Manager) makeMethodFuncHandler(moduleName, methodID string) func(functi
109 resolvedParams[paramJob] = resolvedJob
110
111 // Route to the module's handler - get DATA ONLY response
105 - dataResp := creator.HandleMethod(ctx, job, methodID, resolvedParams)
112 + dataResp := handler.Handle(ctx, methodID, resolvedParams)
113
114 // RACE CONDITION MITIGATION: Verify job was not replaced during handler execution
115 // If a config reload replaced this job while we were querying, the response
@@ -127,7 +134,8 @@ func (m *Manager) handleMethodFuncInfo(moduleName, methodID string, fn functions
134 return
135 }
136
130 - methodParams := m.unionMethodParams(moduleName, methodID, methodCfg, fn)
137 + // Use static params for info. Actual requests return job-specific params in the response.
138 + methodParams := methodCfg.RequiredParams
139 help := methodCfg.Help
140 if help == "" {
141 help = fmt.Sprintf("%s %s data function", moduleName, methodID)
@@ -153,7 +161,7 @@ func (m *Manager) handleMethodFuncInfo(moduleName, methodID string, fn functions
161 }
162
163 // respondWithParams wraps the module's data response with current required_params
156 -func (m *Manager) respondWithParams(fn functions.Function, moduleName string, dataResp *module.FunctionResponse, methodParams []funcapi.ParamConfig) {
164 +func (m *Manager) respondWithParams(fn functions.Function, moduleName string, dataResp *funcapi.FunctionResponse, methodParams []funcapi.ParamConfig) {
165 // Nil guard: if module returns nil, treat as internal error
166 if dataResp == nil {
167 m.respondError(fn, 500, "internal error: module returned nil response")
@@ -198,7 +206,7 @@ func (m *Manager) respondWithParams(fn functions.Function, moduleName string, da
206 resp["charts"] = dataResp.Charts
207 }
208 if len(dataResp.DefaultCharts) > 0 {
201 - resp["default_charts"] = dataResp.DefaultCharts
209 + resp["default_charts"] = dataResp.DefaultCharts.Build()
210 }
211 if len(dataResp.GroupBy) > 0 {
212 resp["group_by"] = dataResp.GroupBy
@@ -232,13 +240,10 @@ func (m *Manager) buildRequiredParams(moduleName string, methodParams []funcapi.
240 return required
241 }
242
235 -func (m *Manager) resolveMethodParamsForJob(ctx context.Context, moduleName, methodID string, methodCfg *module.MethodConfig, job *module.Job, creator module.Creator) ([]funcapi.ParamConfig, bool, error) {
243 +func (m *Manager) resolveMethodParamsForJob(ctx context.Context, moduleName, methodID string, methodCfg *funcapi.MethodConfig, job *module.Job, handler funcapi.MethodHandler) ([]funcapi.ParamConfig, bool, error) {
244 methodParams := methodCfg.RequiredParams
237 - if creator.MethodParams == nil {
238 - return methodParams, false, nil
239 - }
245
241 - jobParams, err := creator.MethodParams(ctx, job, methodID)
246 + jobParams, err := handler.MethodParams(ctx, methodID)
247 if err != nil {
248 return nil, false, err
249 }
@@ -249,85 +254,6 @@ func (m *Manager) resolveMethodParamsForJob(ctx context.Context, moduleName, met
254 return funcapi.MergeParamConfigs(methodParams, jobParams), true, nil
255 }
256
252 -func (m *Manager) unionMethodParams(moduleName, methodID string, methodCfg *module.MethodConfig, fn functions.Function) []funcapi.ParamConfig {
253 - baseParams := methodCfg.RequiredParams
254 -
255 - creator, ok := m.moduleFuncs.getCreator(moduleName)
256 - if !ok || creator.MethodParams == nil {
257 - return baseParams
258 - }
259 -
260 - jobs := m.moduleFuncs.getJobNames(moduleName)
261 - if len(jobs) == 0 {
262 - return baseParams
263 - }
264 -
265 - ctx, cancel := context.WithTimeout(context.Background(), fn.Timeout)
266 - defer cancel()
267 -
268 - union := []funcapi.ParamConfig{}
269 - for _, jobName := range jobs {
270 - job, ok := m.moduleFuncs.getJob(moduleName, jobName)
271 - if !ok || job == nil {
272 - continue
273 - }
274 - params, err := creator.MethodParams(ctx, job, methodID)
275 - if err != nil {
276 - m.Debugf("method params unavailable for %s:%s job '%s': %v", moduleName, methodID, jobName, err)
277 - continue
278 - }
279 - if len(params) == 0 {
280 - continue
281 - }
282 - union = mergeParamConfigsUnion(union, params)
283 - }
284 - if len(union) == 0 {
285 - return baseParams
286 - }
287 -
288 - out := make([]funcapi.ParamConfig, 0, len(baseParams)+len(union))
289 - baseIndex := make(map[string]bool, len(baseParams))
290 - unionIndex := make(map[string]int, len(union))
291 - for i, cfg := range union {
292 - if cfg.ID != "" {
293 - unionIndex[cfg.ID] = i
294 - }
295 - }
296 -
297 - for _, cfg := range baseParams {
298 - if cfg.ID != "" {
299 - baseIndex[cfg.ID] = true
300 - }
301 - if i, ok := unionIndex[cfg.ID]; ok {
302 - out = append(out, mergeParamConfigMetadata(cfg, union[i]))
303 - continue
304 - }
305 - out = append(out, cfg)
306 - }
307 -
308 - for _, cfg := range union {
309 - if cfg.ID == "" || baseIndex[cfg.ID] {
310 - continue
311 - }
312 - out = append(out, cfg)
313 - }
314 - return out
315 -}
316 -
317 -func mergeParamConfigMetadata(base, add funcapi.ParamConfig) funcapi.ParamConfig {
318 - out := add
319 - if out.Name == "" {
320 - out.Name = base.Name
321 - }
322 - if out.Help == "" {
323 - out.Help = base.Help
324 - }
325 - if base.UniqueView {
326 - out.UniqueView = true
327 - }
328 - return out
329 -}
330 -
257 func validateParamValues(methodParams []funcapi.ParamConfig, argValues map[string][]string, payload map[string]any, jobName string) error {
258 for _, cfg := range methodParams {
259 values := paramValues(argValues, payload, cfg.ID)
@@ -358,107 +284,6 @@ func allowedOptions(options []funcapi.ParamOption) map[string]bool {
284 return allowed
285 }
286
361 -func mergeParamConfigsUnion(base, add []funcapi.ParamConfig) []funcapi.ParamConfig {
362 - if len(add) == 0 {
363 - return base
364 - }
365 -
366 - out := make([]funcapi.ParamConfig, len(base))
367 - copy(out, base)
368 -
369 - index := make(map[string]int, len(out))
370 - for i, cfg := range out {
371 - if cfg.ID != "" {
372 - index[cfg.ID] = i
373 - }
374 - }
375 -
376 - for _, cfg := range add {
377 - if cfg.ID == "" {
378 - continue
379 - }
380 - if i, ok := index[cfg.ID]; ok {
381 - out[i] = mergeParamConfigOptions(out[i], cfg)
382 - continue
383 - }
384 - out = append(out, cfg)
385 - index[cfg.ID] = len(out) - 1
386 - }
387 - return out
388 -}
389 -
390 -func mergeParamConfigOptions(base, add funcapi.ParamConfig) funcapi.ParamConfig {
391 - out := base
392 - if out.Name == "" {
393 - out.Name = add.Name
394 - }
395 - if out.Help == "" {
396 - out.Help = add.Help
397 - }
398 - if out.Selection == funcapi.ParamSelect && add.Selection == funcapi.ParamMultiSelect {
399 - out.Selection = add.Selection
400 - }
401 - if add.UniqueView {
402 - out.UniqueView = true
403 - }
404 -
405 - options := make([]funcapi.ParamOption, len(out.Options))
406 - copy(options, out.Options)
407 -
408 - optIndex := make(map[string]int, len(options))
409 - for i, opt := range options {
410 - if opt.ID != "" {
411 - optIndex[opt.ID] = i
412 - }
413 - }
414 -
415 - hasDefault := false
416 - for _, opt := range options {
417 - if opt.Default {
418 - hasDefault = true
419 - break
420 - }
421 - }
422 -
423 - for _, opt := range add.Options {
424 - if opt.ID == "" {
425 - continue
426 - }
427 - if i, ok := optIndex[opt.ID]; ok {
428 - merged := options[i]
429 - if merged.Name == "" {
430 - merged.Name = opt.Name
431 - }
432 - if merged.Sort == nil && opt.Sort != nil {
433 - merged.Sort = opt.Sort
434 - }
435 - if merged.Column == "" {
436 - merged.Column = opt.Column
437 - }
438 - if opt.Default && !hasDefault {
439 - merged.Default = true
440 - hasDefault = true
441 - }
442 - // Disabled should remain false if any job supports the option.
443 - merged.Disabled = merged.Disabled && opt.Disabled
444 - options[i] = merged
445 - continue
446 - }
447 -
448 - if opt.Default && hasDefault {
449 - opt.Default = false
450 - }
451 - options = append(options, opt)
452 - optIndex[opt.ID] = len(options) - 1
453 - if opt.Default {
454 - hasDefault = true
455 - }
456 - }
457 -
458 - out.Options = options
459 - return out
460 -}
461 -
287 // respondJSON sends a JSON response to the function request
288 // The HTTP status code is extracted from the "status" field in the response
289 func (m *Manager) respondJSON(fn functions.Function, resp map[string]any) {
src/go/plugin/go.d/agent/jobmgr/funcshandler_test.go
+25 -117
@@ -4,16 +4,36 @@ package jobmgr
4
5 import (
6 "context"
7 - "errors"
7 "testing"
9 - "time"
8
9 "github.com/netdata/netdata/go/plugins/pkg/funcapi"
12 - "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/functions"
10 "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
11 "github.com/stretchr/testify/assert"
12 )
13
14 +// mockMethodHandler implements funcapi.MethodHandler for testing.
15 +type mockMethodHandler struct {
16 + job *module.Job
17 + paramsFunc func(ctx context.Context, method string) ([]funcapi.ParamConfig, error)
18 + handleFunc func(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse
19 +}
20 +
21 +func (m *mockMethodHandler) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
22 + if m.paramsFunc != nil {
23 + return m.paramsFunc(ctx, method)
24 + }
25 + return nil, nil
26 +}
27 +
28 +func (m *mockMethodHandler) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
29 + if m.handleFunc != nil {
30 + return m.handleFunc(ctx, method, params)
31 + }
32 + return nil
33 +}
34 +
35 +func (m *mockMethodHandler) Cleanup(ctx context.Context) {}
36 +
37 func TestExtractParamValues(t *testing.T) {
38 tests := map[string]struct {
39 payload map[string]any
@@ -103,8 +123,8 @@ func TestBuildRequiredParams_TypeSelect(t *testing.T) {
123 // Setup a minimal manager with test data
124 r := newModuleFuncRegistry()
125 r.registerModule("postgres", module.Creator{
106 - Methods: func() []module.MethodConfig {
107 - return []module.MethodConfig{{
126 + Methods: func() []funcapi.MethodConfig {
127 + return []funcapi.MethodConfig{{
128 ID: "top-queries",
129 Name: "Top Queries",
130 }}
@@ -153,115 +173,3 @@ func TestBuildRequiredParams_TypeSelect(t *testing.T) {
173 assert.Equal(t, "__job", params[0]["id"])
174 assert.Equal(t, "__sort", params[1]["id"])
175 }
156 -
157 -func TestUnionMethodParams_JobOptionsOverrideStatic(t *testing.T) {
158 - sortDir := funcapi.FieldSortDescending
159 - baseParams := []funcapi.ParamConfig{
160 - {
161 - ID: "__sort",
162 - Name: "Filter By",
163 - Selection: funcapi.ParamSelect,
164 - UniqueView: true,
165 - Options: []funcapi.ParamOption{
166 - {ID: "a", Name: "A", Sort: &sortDir},
167 - {ID: "b", Name: "B", Sort: &sortDir},
168 - },
169 - },
170 - {
171 - ID: "mode",
172 - Name: "Mode",
173 - Selection: funcapi.ParamSelect,
174 - Options: []funcapi.ParamOption{{ID: "x", Name: "X"}},
175 - },
176 - }
177 -
178 - r := newModuleFuncRegistry()
179 - r.registerModule("postgres", module.Creator{
180 - Methods: func() []module.MethodConfig {
181 - return []module.MethodConfig{{ID: "top-queries", RequiredParams: baseParams}}
182 - },
183 - MethodParams: func(ctx context.Context, job *module.Job, method string) ([]funcapi.ParamConfig, error) {
184 - switch job.Name() {
185 - case "job1":
186 - return []funcapi.ParamConfig{{
187 - ID: "__sort",
188 - Name: "Filter By",
189 - Selection: funcapi.ParamSelect,
190 - UniqueView: true,
191 - Options: []funcapi.ParamOption{
192 - {ID: "b", Name: "B", Sort: &sortDir},
193 - {ID: "c", Name: "C", Sort: &sortDir},
194 - },
195 - }}, nil
196 - case "job2":
197 - return []funcapi.ParamConfig{{
198 - ID: "__sort",
199 - Name: "Filter By",
200 - Selection: funcapi.ParamSelect,
201 - UniqueView: true,
202 - Options: []funcapi.ParamOption{
203 - {ID: "c", Name: "C", Sort: &sortDir},
204 - {ID: "d", Name: "D", Sort: &sortDir},
205 - },
206 - }}, nil
207 - default:
208 - return nil, nil
209 - }
210 - },
211 - })
212 - r.addJob("postgres", "job1", newTestModuleFuncsJob("job1"))
213 - r.addJob("postgres", "job2", newTestModuleFuncsJob("job2"))
214 -
215 - mgr := &Manager{moduleFuncs: r}
216 - fn := functions.Function{Timeout: time.Second}
217 -
218 - got := mgr.unionMethodParams("postgres", "top-queries", &module.MethodConfig{
219 - ID: "top-queries",
220 - RequiredParams: baseParams,
221 - }, fn)
222 -
223 - assert.Len(t, got, 2)
224 - assert.Equal(t, "__sort", got[0].ID)
225 - assert.Equal(t, "mode", got[1].ID)
226 -
227 - sortOpts := make(map[string]bool)
228 - for _, opt := range got[0].Options {
229 - sortOpts[opt.ID] = true
230 - }
231 - assert.False(t, sortOpts["a"], "static-only option should not be included when jobs provide options")
232 - assert.True(t, sortOpts["b"])
233 - assert.True(t, sortOpts["c"])
234 - assert.True(t, sortOpts["d"])
235 -}
236 -
237 -func TestUnionMethodParams_FallbackToStaticOnAllErrors(t *testing.T) {
238 - baseParams := []funcapi.ParamConfig{
239 - {
240 - ID: "__sort",
241 - Name: "Sort",
242 - Selection: funcapi.ParamSelect,
243 - Options: []funcapi.ParamOption{{ID: "a", Name: "A"}},
244 - },
245 - }
246 -
247 - r := newModuleFuncRegistry()
248 - r.registerModule("postgres", module.Creator{
249 - Methods: func() []module.MethodConfig {
250 - return []module.MethodConfig{{ID: "top-queries", RequiredParams: baseParams}}
251 - },
252 - MethodParams: func(ctx context.Context, job *module.Job, method string) ([]funcapi.ParamConfig, error) {
253 - return nil, errors.New("backend unavailable")
254 - },
255 - })
256 - r.addJob("postgres", "job1", newTestModuleFuncsJob("job1"))
257 -
258 - mgr := &Manager{moduleFuncs: r}
259 - fn := functions.Function{Timeout: time.Second}
260 -
261 - got := mgr.unionMethodParams("postgres", "top-queries", &module.MethodConfig{
262 - ID: "top-queries",
263 - RequiredParams: baseParams,
264 - }, fn)
265 -
266 - assert.Equal(t, baseParams, got)
267 -}
src/go/plugin/go.d/agent/jobmgr/modulefuncs.go
+9 -8
@@ -6,6 +6,7 @@ import (
6 "sort"
7 "sync"
8
9 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
10 "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
11 )
12
@@ -19,9 +20,9 @@ type moduleFuncRegistry struct {
20 }
21
22 type moduleFunc struct {
22 - creator module.Creator // The module creator (has Methods())
23 - methods []module.MethodConfig // Static methods from creator (ordered)
24 - methodsByID map[string]module.MethodConfig
23 + creator module.Creator // The module creator (has Methods())
24 + methods []funcapi.MethodConfig // Static methods from creator (ordered)
25 + methodsByID map[string]funcapi.MethodConfig
26 jobs map[string]*jobEntry // jobName → job entry with generation
27 lastGeneration map[string]uint64 // jobName → last known generation (persists across removals)
28 }
@@ -43,7 +44,7 @@ func (r *moduleFuncRegistry) registerModule(name string, creator module.Creator)
44 r.mu.Lock()
45 defer r.mu.Unlock()
46
46 - var methods []module.MethodConfig
47 + var methods []funcapi.MethodConfig
48 if creator.Methods != nil {
49 methods = creator.Methods()
50 }
@@ -57,11 +58,11 @@ func (r *moduleFuncRegistry) registerModule(name string, creator module.Creator)
58 }
59 }
60
60 -func indexMethods(methods []module.MethodConfig) map[string]module.MethodConfig {
61 +func indexMethods(methods []funcapi.MethodConfig) map[string]funcapi.MethodConfig {
62 if len(methods) == 0 {
63 return nil
64 }
64 - idx := make(map[string]module.MethodConfig, len(methods))
65 + idx := make(map[string]funcapi.MethodConfig, len(methods))
66 for _, m := range methods {
67 if m.ID == "" {
68 continue
@@ -147,7 +148,7 @@ func (r *moduleFuncRegistry) verifyJobGeneration(moduleName, jobName string, exp
148 }
149
150 // getMethods returns the method configurations for a module
150 -func (r *moduleFuncRegistry) getMethods(moduleName string) []module.MethodConfig {
151 +func (r *moduleFuncRegistry) getMethods(moduleName string) []funcapi.MethodConfig {
152 r.mu.RLock()
153 defer r.mu.RUnlock()
154
@@ -159,7 +160,7 @@ func (r *moduleFuncRegistry) getMethods(moduleName string) []module.MethodConfig
160 }
161
162 // getMethod returns a method config by ID for a module.
162 -func (r *moduleFuncRegistry) getMethod(moduleName, methodID string) (*module.MethodConfig, bool) {
163 +func (r *moduleFuncRegistry) getMethod(moduleName, methodID string) (*funcapi.MethodConfig, bool) {
164 r.mu.RLock()
165 defer r.mu.RUnlock()
166
src/go/plugin/go.d/agent/jobmgr/modulefuncs_test.go
+7 -6
@@ -7,6 +7,7 @@ import (
7 "io"
8 "testing"
9
10 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
11 "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
12 "github.com/stretchr/testify/assert"
13 "github.com/stretchr/testify/require"
@@ -37,8 +38,8 @@ func TestModuleFuncRegistry_RegisterModule(t *testing.T) {
38
39 for _, m := range tc.modules {
40 r.registerModule(m, module.Creator{
40 - Methods: func() []module.MethodConfig {
41 - return []module.MethodConfig{{ID: "test"}}
41 + Methods: func() []funcapi.MethodConfig {
42 + return []funcapi.MethodConfig{{ID: "test"}}
43 },
44 })
45 }
@@ -124,12 +125,12 @@ func TestModuleFuncRegistry_GenerationVerification(t *testing.T) {
125 func TestModuleFuncRegistry_GetMethods(t *testing.T) {
126 r := newModuleFuncRegistry()
127
127 - expectedMethods := []module.MethodConfig{
128 + expectedMethods := []funcapi.MethodConfig{
129 {ID: "top-queries", Name: "Top Queries"},
130 }
131
132 r.registerModule("postgres", module.Creator{
132 - Methods: func() []module.MethodConfig {
133 + Methods: func() []funcapi.MethodConfig {
134 return expectedMethods
135 },
136 })
@@ -207,8 +208,8 @@ func newTestModuleFuncsJob(name string) *module.Job {
208 func TestModuleFuncRegistry_Concurrency(t *testing.T) {
209 r := newModuleFuncRegistry()
210 r.registerModule("postgres", module.Creator{
210 - Methods: func() []module.MethodConfig {
211 - return []module.MethodConfig{{ID: "test"}}
211 + Methods: func() []funcapi.MethodConfig {
212 + return []funcapi.MethodConfig{{ID: "test"}}
213 },
214 })
215
src/go/plugin/go.d/agent/module/registry.go
+9 -57
@@ -3,7 +3,6 @@
3 package module
4
5 import (
6 - "context"
6 "fmt"
7
8 "github.com/netdata/netdata/go/plugins/pkg/funcapi"
@@ -23,50 +22,9 @@ type Defaults struct {
22 Disabled bool
23 }
24
26 -// MethodConfig describes a function method provided by a module.
27 -type MethodConfig struct {
28 - ID string // Method ID (e.g., "top-queries")
29 - Name string // Display name (e.g., "Top Queries")
30 - UpdateEvery int // Default UI refresh interval
31 - Help string // Description for UI
32 - RequireCloud bool // Indicates whether the method requires cloud connection
33 - RequiredParams []funcapi.ParamConfig // Required parameters for this method (including __sort if used)
34 -}
35 -
36 -// FunctionResponse is the response from a module's HandleMethod.
37 -type FunctionResponse struct {
38 - Status int // HTTP-like status code (200, 400, 403, 500, 503)
39 - Message string // Error message (if Status != 200)
40 - Help string // Help text for this response
41 - Columns map[string]any // Column definitions for the table
42 - Data any // Row data: [][]any (array of arrays, ordered by column index)
43 - DefaultSortColumn string // Default sort column ID
44 -
45 - // Optional dynamic required params (override MethodConfig.RequiredParams)
46 - RequiredParams []funcapi.ParamConfig
47 -
48 - // Chart configuration for visualization
49 - Charts map[string]ChartConfig // Chart definitions (chartID -> config)
50 - DefaultCharts [][]string // Default charts: [[chartID, groupByID], ...]
51 - GroupBy map[string]GroupByConfig // Group-by options (groupByID -> config)
52 -}
53 -
54 -// ChartConfig defines a chart for visualization.
55 -type ChartConfig struct {
56 - Name string `json:"name"`
57 - Type string `json:"type"` // "stacked-bar", "line", etc.
58 - Columns []string `json:"columns"` // Column IDs to include in chart
59 -}
60 -
61 -// GroupByConfig defines a grouping option for function responses.
62 -type GroupByConfig struct {
63 - Name string `json:"name"`
64 - Columns []string `json:"columns"` // Columns to group by
65 -}
66 -
25 type (
26 // Creator is a Job builder.
69 - // Optional function fields (Methods/HandleMethod) enable the FunctionProvider pattern:
27 + // Optional function fields (Methods/MethodHandler) enable the FunctionProvider pattern:
28 // modules that set these fields can expose data functions to the UI.
29 Creator struct {
30 Defaults
@@ -76,20 +34,14 @@ type (
34
35 // Optional: FunctionProvider fields for exposing data functions
36 // If Methods is non-nil, this module provides functions
79 - Methods func() []MethodConfig
80 -
81 - // Optional: MethodParams returns dynamic required params for a job+method.
82 - // Use this to provide job-specific options (e.g., based on DB capabilities).
83 - // When nil, MethodConfig.RequiredParams is used as-is.
84 - MethodParams func(ctx context.Context, job *Job, method string) ([]funcapi.ParamConfig, error)
85 -
86 - // HandleMethod handles a function request for a specific job
87 - // ctx: context with timeout from function request
88 - // job: the job instance to query
89 - // method: the method name (e.g., "top-queries")
90 - // params: resolved required params (includes __sort)
91 - // Returns: FunctionResponse with data or error
92 - HandleMethod func(ctx context.Context, job *Job, method string, params funcapi.ResolvedParams) *FunctionResponse
37 + Methods func() []funcapi.MethodConfig
38 +
39 + // Optional: MethodHandler returns a handler for method requests on a specific job.
40 + // The handler implements funcapi.MethodHandler interface with:
41 + // - MethodParams(ctx, method) for dynamic params
42 + // - Handle(ctx, method, params) for request handling
43 + // When nil, methods are disabled for this module.
44 + MethodHandler func(job *Job) funcapi.MethodHandler
45 }
46 // Registry is a collection of Creators.
47 Registry map[string]Creator
src/go/plugin/go.d/collector/clickhouse/collector.go
+10 -3
@@ -24,8 +24,7 @@ func init() {
24 Config: func() any { return &Config{} },
25 JobConfigSchema: configSchema,
26 Methods: clickhouseMethods,
27 - MethodParams: clickhouseMethodParams,
28 - HandleMethod: clickhouseHandleMethod,
27 + MethodHandler: clickhouseFunctionHandler,
28 })
29 }
30
@@ -66,6 +65,9 @@ type (
65
66 seenDisks map[string]*seenDisk
67 seenDbTables map[string]*seenTable
68 +
69 + // Function handler (singleton, initialized in Init)
70 + funcRouter *funcRouter
71 }
72 seenDisk struct{ disk string }
73 seenTable struct{ db, table string }
@@ -86,6 +88,8 @@ func (c *Collector) Init(context.Context) error {
88 }
89 c.httpClient = httpClient
90
91 + c.funcRouter = newFuncRouter(c)
92 +
93 c.Debugf("using URL %s", c.URL)
94 c.Debugf("using timeout: %s", c.Timeout)
95
@@ -122,7 +126,10 @@ func (c *Collector) Collect(context.Context) map[string]int64 {
126 return mx
127 }
128
125 -func (c *Collector) Cleanup(context.Context) {
129 +func (c *Collector) Cleanup(ctx context.Context) {
130 + if c.funcRouter != nil {
131 + c.funcRouter.Cleanup(ctx)
132 + }
133 if c.httpClient != nil {
134 c.httpClient.CloseIdleConnections()
135 }
src/go/plugin/go.d/collector/clickhouse/func_router.go new
+60
@@ -0,0 +1,60 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package clickhouse
4 +
5 +import (
6 + "context"
7 + "fmt"
8 +
9 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
10 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
11 +)
12 +
13 +func clickhouseMethods() []funcapi.MethodConfig {
14 + return []funcapi.MethodConfig{
15 + topQueriesMethodConfig(),
16 + }
17 +}
18 +
19 +func clickhouseFunctionHandler(job *module.Job) funcapi.MethodHandler {
20 + c, ok := job.Module().(*Collector)
21 + if !ok {
22 + return nil
23 + }
24 + return c.funcRouter
25 +}
26 +
27 +// funcRouter routes method calls to appropriate function handlers.
28 +type funcRouter struct {
29 + collector *Collector
30 + handlers map[string]funcapi.MethodHandler
31 +}
32 +
33 +func newFuncRouter(c *Collector) *funcRouter {
34 + r := &funcRouter{
35 + collector: c,
36 + handlers: make(map[string]funcapi.MethodHandler),
37 + }
38 + r.handlers[topQueriesMethodID] = newFuncTopQueries(r)
39 + return r
40 +}
41 +
42 +func (r *funcRouter) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
43 + if h, ok := r.handlers[method]; ok {
44 + return h.MethodParams(ctx, method)
45 + }
46 + return nil, fmt.Errorf("unknown method: %s", method)
47 +}
48 +
49 +func (r *funcRouter) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
50 + if h, ok := r.handlers[method]; ok {
51 + return h.Handle(ctx, method, params)
52 + }
53 + return funcapi.NotFoundResponse(method)
54 +}
55 +
56 +func (r *funcRouter) Cleanup(ctx context.Context) {
57 + for _, h := range r.handlers {
58 + h.Cleanup(ctx)
59 + }
60 +}
src/go/plugin/go.d/collector/clickhouse/func_top_queries.go new
+316
@@ -0,0 +1,316 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package clickhouse
4 +
5 +import (
6 + "context"
7 + "encoding/json"
8 + "fmt"
9 + "strconv"
10 + "strings"
11 +
12 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
13 + "github.com/netdata/netdata/go/plugins/pkg/web"
14 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
15 +)
16 +
17 +const (
18 + topQueriesMethodID = "top-queries"
19 + topQueriesMaxTextLength = 4096
20 +)
21 +
22 +func topQueriesMethodConfig() funcapi.MethodConfig {
23 + return funcapi.MethodConfig{
24 + ID: topQueriesMethodID,
25 + Name: "Top Queries",
26 + UpdateEvery: 10,
27 + Help: "Top SQL queries from ClickHouse system.query_log",
28 + RequireCloud: true,
29 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(topQueriesColumns)},
30 + }
31 +}
32 +
33 +// topQueriesColumn defines a column for ClickHouse top-queries function.
34 +// Embeds funcapi.ColumnMeta for UI display and adds collector-specific fields.
35 +type topQueriesColumn struct {
36 + funcapi.ColumnMeta
37 +
38 + // Data access
39 + DBColumn string // Column name in system.query_log (empty = computed)
40 + SelectExpr string // SQL expression for SELECT
41 +
42 + // Sort parameter metadata
43 + sortOpt bool // Include in __sort parameter options
44 + sortLbl string // Display label for sort option
45 + defaultSort bool // Default sort option
46 +}
47 +
48 +// funcapi.SortableColumn interface implementation for topQueriesColumn.
49 +func (c topQueriesColumn) IsSortOption() bool { return c.sortOpt }
50 +func (c topQueriesColumn) SortLabel() string { return c.sortLbl }
51 +func (c topQueriesColumn) IsDefaultSort() bool { return c.defaultSort }
52 +func (c topQueriesColumn) ColumnName() string { return c.Name }
53 +func (c topQueriesColumn) SortColumn() string { return "" }
54 +
55 +var topQueriesColumns = []topQueriesColumn{
56 + {ColumnMeta: funcapi.ColumnMeta{Name: "queryId", Tooltip: "Query ID", Type: funcapi.FieldTypeString, Visible: false, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, UniqueKey: true, Sortable: true}, DBColumn: "normalized_query_hash", SelectExpr: "toString(normalized_query_hash)"},
57 + {ColumnMeta: funcapi.ColumnMeta{Name: "query", Tooltip: "Query", Type: funcapi.FieldTypeString, Visible: true, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, Sticky: true, FullWidth: true, Sortable: true}, DBColumn: "query", SelectExpr: "any(query)"},
58 + {ColumnMeta: funcapi.ColumnMeta{Name: "database", Tooltip: "Database", Type: funcapi.FieldTypeString, Visible: true, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, GroupBy: &funcapi.GroupByOptions{IsDefault: true}, Sortable: true}, DBColumn: "current_database", SelectExpr: "any(current_database)"},
59 + {ColumnMeta: funcapi.ColumnMeta{Name: "user", Tooltip: "User", Type: funcapi.FieldTypeString, Visible: true, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, GroupBy: &funcapi.GroupByOptions{}, Sortable: true}, DBColumn: "user", SelectExpr: "any(user)"},
60 +
61 + {ColumnMeta: funcapi.ColumnMeta{Name: "calls", Tooltip: "Calls", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Chart: &funcapi.ChartOptions{Group: "Calls", Title: "Number of Calls", IsDefault: true}, Sortable: true}, SelectExpr: "count()", sortOpt: true, sortLbl: "Top queries by Number of Calls"},
62 +
63 + {ColumnMeta: funcapi.ColumnMeta{Name: "totalTime", Tooltip: "Total Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: true, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Execution Time", IsDefault: true}, Sortable: true}, DBColumn: "query_duration_ms", SelectExpr: "sum(query_duration_ms)", sortOpt: true, defaultSort: true, sortLbl: "Top queries by Total Execution Time"},
64 + {ColumnMeta: funcapi.ColumnMeta{Name: "avgTime", Tooltip: "Avg Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: true, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMean, Filter: funcapi.FieldFilterRange, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Execution Time"}, Sortable: true}, DBColumn: "query_duration_ms", SelectExpr: "avg(query_duration_ms)", sortOpt: true, sortLbl: "Top queries by Average Execution Time"},
65 + {ColumnMeta: funcapi.ColumnMeta{Name: "minTime", Tooltip: "Min Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: false, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMin, Filter: funcapi.FieldFilterRange, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Execution Time"}, Sortable: true}, DBColumn: "query_duration_ms", SelectExpr: "min(query_duration_ms)"},
66 + {ColumnMeta: funcapi.ColumnMeta{Name: "maxTime", Tooltip: "Max Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: false, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Execution Time"}, Sortable: true}, DBColumn: "query_duration_ms", SelectExpr: "max(query_duration_ms)"},
67 +
68 + {ColumnMeta: funcapi.ColumnMeta{Name: "readRows", Tooltip: "Read Rows", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Chart: &funcapi.ChartOptions{Group: "Rows", Title: "Rows"}, Sortable: true}, DBColumn: "read_rows", SelectExpr: "sum(read_rows)", sortOpt: true, sortLbl: "Top queries by Rows Read"},
69 + {ColumnMeta: funcapi.ColumnMeta{Name: "readBytes", Tooltip: "Read Bytes", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Chart: &funcapi.ChartOptions{Group: "Bytes", Title: "Bytes"}, Sortable: true}, DBColumn: "read_bytes", SelectExpr: "sum(read_bytes)"},
70 + {ColumnMeta: funcapi.ColumnMeta{Name: "writtenRows", Tooltip: "Written Rows", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Chart: &funcapi.ChartOptions{Group: "Rows", Title: "Rows"}, Sortable: true}, DBColumn: "written_rows", SelectExpr: "sum(written_rows)"},
71 + {ColumnMeta: funcapi.ColumnMeta{Name: "writtenBytes", Tooltip: "Written Bytes", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Chart: &funcapi.ChartOptions{Group: "Bytes", Title: "Bytes"}, Sortable: true}, DBColumn: "written_bytes", SelectExpr: "sum(written_bytes)"},
72 + {ColumnMeta: funcapi.ColumnMeta{Name: "resultRows", Tooltip: "Result Rows", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Chart: &funcapi.ChartOptions{Group: "Rows", Title: "Rows"}, Sortable: true}, DBColumn: "result_rows", SelectExpr: "sum(result_rows)"},
73 + {ColumnMeta: funcapi.ColumnMeta{Name: "resultBytes", Tooltip: "Result Bytes", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Chart: &funcapi.ChartOptions{Group: "Bytes", Title: "Bytes"}, Sortable: true}, DBColumn: "result_bytes", SelectExpr: "sum(result_bytes)"},
74 + {ColumnMeta: funcapi.ColumnMeta{Name: "memoryUsage", Tooltip: "Max Memory", Type: funcapi.FieldTypeFloat, Visible: false, Transform: funcapi.FieldTransformNumber, DecimalPoints: 0, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange, Chart: &funcapi.ChartOptions{Group: "Memory", Title: "Memory"}, Sortable: true}, DBColumn: "memory_usage", SelectExpr: "max(memory_usage)"},
75 +}
76 +
77 +type topQueriesJSONResponse struct {
78 + Data []map[string]any `json:"data"`
79 +}
80 +
81 +// funcTopQueries implements funcapi.MethodHandler for ClickHouse top-queries.
82 +// All function-related logic is encapsulated here, keeping Collector focused on metrics collection.
83 +type funcTopQueries struct {
84 + router *funcRouter
85 +}
86 +
87 +func newFuncTopQueries(r *funcRouter) *funcTopQueries {
88 + return &funcTopQueries{router: r}
89 +}
90 +
91 +// Compile-time interface check.
92 +var _ funcapi.MethodHandler = (*funcTopQueries)(nil)
93 +
94 +// MethodParams implements funcapi.MethodHandler.
95 +func (f *funcTopQueries) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
96 + if f.router.collector.httpClient == nil {
97 + return nil, fmt.Errorf("collector is still initializing")
98 + }
99 + switch method {
100 + case topQueriesMethodID:
101 + return f.methodParams(ctx)
102 + default:
103 + return nil, fmt.Errorf("unknown method: %s", method)
104 + }
105 +}
106 +
107 +// Handle implements funcapi.MethodHandler.
108 +func (f *funcTopQueries) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
109 + if f.router.collector.httpClient == nil {
110 + return funcapi.UnavailableResponse("collector is still initializing, please retry in a few seconds")
111 + }
112 + switch method {
113 + case topQueriesMethodID:
114 + return f.collectData(ctx, params.Column("__sort"))
115 + default:
116 + return funcapi.NotFoundResponse(method)
117 + }
118 +}
119 +
120 +// Cleanup implements funcapi.MethodHandler.
121 +func (f *funcTopQueries) Cleanup(ctx context.Context) {}
122 +
123 +func (f *funcTopQueries) methodParams(ctx context.Context) ([]funcapi.ParamConfig, error) {
124 + available, err := f.detectQueryLogColumns(ctx)
125 + if err != nil {
126 + return nil, err
127 + }
128 + cols := f.buildAvailableColumns(available)
129 + if len(cols) == 0 {
130 + return nil, fmt.Errorf("no columns available in system.query_log")
131 + }
132 + return []funcapi.ParamConfig{funcapi.BuildSortParam(cols)}, nil
133 +}
134 +
135 +func (f *funcTopQueries) detectQueryLogColumns(ctx context.Context) (map[string]bool, error) {
136 + query := `
137 +SELECT name
138 +FROM system.columns
139 +WHERE database = 'system' AND table = 'query_log'
140 +FORMAT JSON`
141 +
142 + req, err := web.NewHTTPRequest(f.router.collector.RequestConfig)
143 + if err != nil {
144 + return nil, err
145 + }
146 + req = req.WithContext(ctx)
147 + req.URL.RawQuery = makeURLQuery(query)
148 +
149 + var resp topQueriesJSONResponse
150 + if err := web.DoHTTP(f.router.collector.httpClient).RequestJSON(req, &resp); err != nil {
151 + return nil, fmt.Errorf("failed to query system.columns: %w", err)
152 + }
153 +
154 + cols := make(map[string]bool, len(resp.Data))
155 + for _, row := range resp.Data {
156 + if name, ok := row["name"].(string); ok {
157 + cols[name] = true
158 + }
159 + }
160 + if len(cols) == 0 {
161 + return nil, fmt.Errorf("system.query_log not available")
162 + }
163 + return cols, nil
164 +}
165 +
166 +func (f *funcTopQueries) collectData(ctx context.Context, sortColumn string) *funcapi.FunctionResponse {
167 + availableCols, err := f.detectQueryLogColumns(ctx)
168 + if err != nil {
169 + return funcapi.ErrorResponse(503, "system.query_log not available: %v", err)
170 + }
171 +
172 + cols := f.buildAvailableColumns(availableCols)
173 + if len(cols) == 0 {
174 + return funcapi.ErrorResponse(500, "no columns available in system.query_log")
175 + }
176 +
177 + cs := f.columnSet(cols)
178 + sortColumn = f.mapAndValidateSortColumn(sortColumn, cs)
179 +
180 + limit := f.router.collector.TopQueriesLimit
181 + if limit <= 0 {
182 + limit = 500
183 + }
184 +
185 + groupKey := "normalized_query_hash"
186 + if !availableCols[groupKey] {
187 + groupKey = "query"
188 + }
189 +
190 + selectParts := make([]string, 0, len(cols))
191 + for _, col := range cols {
192 + selectParts = append(selectParts, fmt.Sprintf("%s AS `%s`", col.SelectExpr, col.Name))
193 + }
194 +
195 + query := fmt.Sprintf(`
196 +SELECT %s
197 +FROM system.query_log
198 +WHERE type = 'QueryFinish'
199 +GROUP BY %s
200 +ORDER BY `+"`%s`"+` DESC
201 +LIMIT %d
202 +FORMAT JSON
203 +`, strings.Join(selectParts, ", "), groupKey, sortColumn, limit)
204 +
205 + req, err := web.NewHTTPRequest(f.router.collector.RequestConfig)
206 + if err != nil {
207 + return funcapi.ErrorResponse(500, "%v", err)
208 + }
209 + req = req.WithContext(ctx)
210 + req.URL.RawQuery = makeURLQuery(query)
211 +
212 + var resp topQueriesJSONResponse
213 + if err := web.DoHTTP(f.router.collector.httpClient).RequestJSON(req, &resp); err != nil {
214 + if ctx.Err() == context.DeadlineExceeded {
215 + return funcapi.ErrorResponse(504, "query timed out")
216 + }
217 + return funcapi.ErrorResponse(500, "query failed: %v", err)
218 + }
219 +
220 + data := make([][]any, 0, len(resp.Data))
221 + for _, rowMap := range resp.Data {
222 + row := make([]any, len(cols))
223 + for i, col := range cols {
224 + row[i] = f.normalizeValue(col, rowMap[col.Name])
225 + }
226 + data = append(data, row)
227 + }
228 +
229 + return &funcapi.FunctionResponse{
230 + Status: 200,
231 + Help: "Top SQL queries from ClickHouse system.query_log",
232 + Columns: cs.BuildColumns(),
233 + Data: data,
234 + DefaultSortColumn: sortColumn,
235 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(cols)},
236 + ChartingConfig: cs.BuildCharting(),
237 + }
238 +}
239 +
240 +func (f *funcTopQueries) columnSet(cols []topQueriesColumn) funcapi.ColumnSet[topQueriesColumn] {
241 + return funcapi.Columns(cols, func(c topQueriesColumn) funcapi.ColumnMeta { return c.ColumnMeta })
242 +}
243 +
244 +func (f *funcTopQueries) buildAvailableColumns(available map[string]bool) []topQueriesColumn {
245 + var cols []topQueriesColumn
246 + for _, col := range topQueriesColumns {
247 + if col.DBColumn == "" || available[col.DBColumn] {
248 + cols = append(cols, col)
249 + }
250 + }
251 + return cols
252 +}
253 +
254 +func (f *funcTopQueries) mapAndValidateSortColumn(input string, cs funcapi.ColumnSet[topQueriesColumn]) string {
255 + if cs.ContainsColumn(input) {
256 + return input
257 + }
258 + if cs.ContainsColumn("totalTime") {
259 + return "totalTime"
260 + }
261 + if cs.ContainsColumn("calls") {
262 + return "calls"
263 + }
264 + names := cs.Names()
265 + if len(names) > 0 {
266 + return names[0]
267 + }
268 + return ""
269 +}
270 +
271 +func (f *funcTopQueries) normalizeValue(col topQueriesColumn, v any) any {
272 + switch col.Type {
273 + case funcapi.FieldTypeInteger:
274 + switch val := v.(type) {
275 + case float64:
276 + return int64(val)
277 + case json.Number:
278 + if i, err := val.Int64(); err == nil {
279 + return i
280 + }
281 + case string:
282 + if i, err := strconv.ParseInt(val, 10, 64); err == nil {
283 + return i
284 + }
285 + }
286 + return int64(0)
287 + case funcapi.FieldTypeFloat, funcapi.FieldTypeDuration:
288 + switch val := v.(type) {
289 + case float64:
290 + return val
291 + case json.Number:
292 + if f, err := val.Float64(); err == nil {
293 + return f
294 + }
295 + case string:
296 + if f, err := strconv.ParseFloat(val, 64); err == nil {
297 + return f
298 + }
299 + }
300 + return float64(0)
301 + default:
302 + if s, ok := v.(string); ok {
303 + if col.Name == "query" {
304 + return strmutil.TruncateText(s, topQueriesMaxTextLength)
305 + }
306 + return s
307 + }
308 + if v == nil {
309 + return ""
310 + }
311 + if col.Name == "query" {
312 + return strmutil.TruncateText(fmt.Sprint(v), topQueriesMaxTextLength)
313 + }
314 + return fmt.Sprint(v)
315 + }
316 +}
src/go/plugin/go.d/collector/clickhouse/func_top_queries_test.go renamed
+20 -21
@@ -29,46 +29,45 @@ func TestClickHouseMethods(t *testing.T) {
29 require.NotEmpty(sortParam.Options)
30 }
31
32 -func TestClickHouseAllColumns_HasRequiredColumns(t *testing.T) {
32 +func TestTopQueriesColumns_HasRequiredColumns(t *testing.T) {
33 required := []string{"query", "calls", "totalTime"}
34
35 - uiKeys := make(map[string]bool)
36 - for _, col := range clickhouseAllColumns {
37 - uiKeys[col.uiKey] = true
38 - }
35 + f := &funcTopQueries{}
36 + cs := f.columnSet(topQueriesColumns)
37
38 for _, key := range required {
41 - assert.True(t, uiKeys[key], "column %s should be defined", key)
39 + assert.True(t, cs.ContainsColumn(key), "column %s should be defined", key)
40 }
41 }
42
45 -func TestCollector_mapAndValidateClickHouseSortColumn(t *testing.T) {
43 +func TestFuncTopQueries_MapAndValidateSortColumn(t *testing.T) {
44 tests := map[string]struct {
47 - available []clickhouseColumnMeta
48 - input string
49 - expected string
45 + columns []topQueriesColumn
46 + input string
47 + expected string
48 }{
49 "valid totalTime": {
52 - available: []clickhouseColumnMeta{{uiKey: "totalTime"}, {uiKey: "calls"}},
53 - input: "totalTime",
54 - expected: "totalTime",
50 + columns: []topQueriesColumn{{ColumnMeta: funcapi.ColumnMeta{Name: "totalTime"}}, {ColumnMeta: funcapi.ColumnMeta{Name: "calls"}}},
51 + input: "totalTime",
52 + expected: "totalTime",
53 },
54 "invalid falls back to totalTime": {
57 - available: []clickhouseColumnMeta{{uiKey: "totalTime"}, {uiKey: "calls"}},
58 - input: "bad",
59 - expected: "totalTime",
55 + columns: []topQueriesColumn{{ColumnMeta: funcapi.ColumnMeta{Name: "totalTime"}}, {ColumnMeta: funcapi.ColumnMeta{Name: "calls"}}},
56 + input: "bad",
57 + expected: "totalTime",
58 },
59 "fallback to calls": {
62 - available: []clickhouseColumnMeta{{uiKey: "calls"}},
63 - input: "bad",
64 - expected: "calls",
60 + columns: []topQueriesColumn{{ColumnMeta: funcapi.ColumnMeta{Name: "calls"}}},
61 + input: "bad",
62 + expected: "calls",
63 },
64 }
65
66 for name, tc := range tests {
67 t.Run(name, func(t *testing.T) {
70 - c := &Collector{}
71 - assert.Equal(t, tc.expected, c.mapAndValidateClickHouseSortColumn(tc.input, tc.available))
68 + f := &funcTopQueries{}
69 + cs := f.columnSet(tc.columns)
70 + assert.Equal(t, tc.expected, f.mapAndValidateSortColumn(tc.input, cs))
71 })
72 }
73 }
src/go/plugin/go.d/collector/clickhouse/functions.go deleted
-519
@@ -1,519 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -package clickhouse
4 -
5 -import (
6 - "context"
7 - "encoding/json"
8 - "fmt"
9 - "strconv"
10 - "strings"
11 -
12 - "github.com/netdata/netdata/go/plugins/pkg/funcapi"
13 - "github.com/netdata/netdata/go/plugins/pkg/web"
14 - "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
15 - "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
16 -)
17 -
18 -const clickhouseMaxQueryTextLength = 4096
19 -
20 -const (
21 - paramSort = "__sort"
22 -
23 - ftString = funcapi.FieldTypeString
24 - ftInteger = funcapi.FieldTypeInteger
25 - ftFloat = funcapi.FieldTypeFloat
26 - ftDuration = funcapi.FieldTypeDuration
27 -
28 - trNone = funcapi.FieldTransformNone
29 - trNumber = funcapi.FieldTransformNumber
30 - trDuration = funcapi.FieldTransformDuration
31 -
32 - sortAsc = funcapi.FieldSortAscending
33 - sortDesc = funcapi.FieldSortDescending
34 -
35 - summaryCount = funcapi.FieldSummaryCount
36 - summarySum = funcapi.FieldSummarySum
37 - summaryMin = funcapi.FieldSummaryMin
38 - summaryMax = funcapi.FieldSummaryMax
39 - summaryMean = funcapi.FieldSummaryMean
40 -
41 - filterMulti = funcapi.FieldFilterMultiselect
42 - filterRange = funcapi.FieldFilterRange
43 -)
44 -
45 -type clickhouseColumnMeta struct {
46 - dbColumn string
47 - uiKey string
48 - displayName string
49 - dataType funcapi.FieldType
50 - units string
51 - visible bool
52 - transform funcapi.FieldTransform
53 - decimalPoints int
54 - sortDir funcapi.FieldSort
55 - summary funcapi.FieldSummary
56 - filter funcapi.FieldFilter
57 - isSortOption bool
58 - sortLabel string
59 - isDefaultSort bool
60 - isUniqueKey bool
61 - isSticky bool
62 - fullWidth bool
63 - selectExpr string
64 - isLabel bool
65 - isPrimary bool
66 - isMetric bool
67 - chartGroup string
68 - chartTitle string
69 - isDefaultChart bool
70 -}
71 -
72 -var clickhouseAllColumns = []clickhouseColumnMeta{
73 - {dbColumn: "normalized_query_hash", uiKey: "queryId", displayName: "Query ID", dataType: ftString, visible: false, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti, isUniqueKey: true, selectExpr: "toString(normalized_query_hash)"},
74 - {dbColumn: "query", uiKey: "query", displayName: "Query", dataType: ftString, visible: true, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti, isSticky: true, fullWidth: true, selectExpr: "any(query)"},
75 - {dbColumn: "current_database", uiKey: "database", displayName: "Database", dataType: ftString, visible: true, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti, selectExpr: "any(current_database)", isLabel: true, isPrimary: true},
76 - {dbColumn: "user", uiKey: "user", displayName: "User", dataType: ftString, visible: true, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti, selectExpr: "any(user)", isLabel: true},
77 -
78 - {dbColumn: "", uiKey: "calls", displayName: "Calls", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Number of Calls", selectExpr: "count()", isMetric: true, chartGroup: "Calls", chartTitle: "Number of Calls", isDefaultChart: true},
79 -
80 - {dbColumn: "query_duration_ms", uiKey: "totalTime", displayName: "Total Time", dataType: ftDuration, units: "milliseconds", visible: true, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Total Execution Time", isDefaultSort: true, selectExpr: "sum(query_duration_ms)", isMetric: true, chartGroup: "Time", chartTitle: "Execution Time", isDefaultChart: true},
81 - {dbColumn: "query_duration_ms", uiKey: "avgTime", displayName: "Avg Time", dataType: ftDuration, units: "milliseconds", visible: true, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMean, filter: filterRange, isSortOption: true, sortLabel: "Average Execution Time", selectExpr: "avg(query_duration_ms)", isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
82 - {dbColumn: "query_duration_ms", uiKey: "minTime", displayName: "Min Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMin, filter: filterRange, selectExpr: "min(query_duration_ms)", isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
83 - {dbColumn: "query_duration_ms", uiKey: "maxTime", displayName: "Max Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, filter: filterRange, selectExpr: "max(query_duration_ms)", isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
84 -
85 - {dbColumn: "read_rows", uiKey: "readRows", displayName: "Read Rows", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Rows Read", selectExpr: "sum(read_rows)", isMetric: true, chartGroup: "Rows", chartTitle: "Rows"},
86 - {dbColumn: "read_bytes", uiKey: "readBytes", displayName: "Read Bytes", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, selectExpr: "sum(read_bytes)", isMetric: true, chartGroup: "Bytes", chartTitle: "Bytes"},
87 - {dbColumn: "written_rows", uiKey: "writtenRows", displayName: "Written Rows", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, selectExpr: "sum(written_rows)", isMetric: true, chartGroup: "Rows", chartTitle: "Rows"},
88 - {dbColumn: "written_bytes", uiKey: "writtenBytes", displayName: "Written Bytes", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, selectExpr: "sum(written_bytes)", isMetric: true, chartGroup: "Bytes", chartTitle: "Bytes"},
89 - {dbColumn: "result_rows", uiKey: "resultRows", displayName: "Result Rows", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, selectExpr: "sum(result_rows)", isMetric: true, chartGroup: "Rows", chartTitle: "Rows"},
90 - {dbColumn: "result_bytes", uiKey: "resultBytes", displayName: "Result Bytes", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, selectExpr: "sum(result_bytes)", isMetric: true, chartGroup: "Bytes", chartTitle: "Bytes"},
91 - {dbColumn: "memory_usage", uiKey: "memoryUsage", displayName: "Max Memory", dataType: ftFloat, visible: false, transform: trNumber, decimalPoints: 0, sortDir: sortDesc, summary: summaryMax, filter: filterRange, selectExpr: "max(memory_usage)", isMetric: true, chartGroup: "Memory", chartTitle: "Memory"},
92 -}
93 -
94 -type clickhouseJSONResponse struct {
95 - Data []map[string]any `json:"data"`
96 -}
97 -
98 -func clickhouseMethods() []module.MethodConfig {
99 - sortOptions := buildClickHouseSortOptions(clickhouseAllColumns)
100 - return []module.MethodConfig{
101 - {
102 - UpdateEvery: 10,
103 - ID: "top-queries",
104 - Name: "Top Queries",
105 - Help: "Top SQL queries from ClickHouse system.query_log",
106 - RequireCloud: true,
107 - RequiredParams: []funcapi.ParamConfig{{
108 - ID: paramSort,
109 - Name: "Filter By",
110 - Help: "Select the primary sort column",
111 - Selection: funcapi.ParamSelect,
112 - Options: sortOptions,
113 - UniqueView: true,
114 - }},
115 - },
116 - }
117 -}
118 -
119 -func clickhouseMethodParams(ctx context.Context, job *module.Job, method string) ([]funcapi.ParamConfig, error) {
120 - collector, ok := job.Module().(*Collector)
121 - if !ok {
122 - return nil, fmt.Errorf("invalid module type")
123 - }
124 - if collector.httpClient == nil {
125 - return nil, fmt.Errorf("collector is still initializing")
126 - }
127 - switch method {
128 - case "top-queries":
129 - return collector.topQueriesParams(ctx)
130 - default:
131 - return nil, fmt.Errorf("unknown method: %s", method)
132 - }
133 -}
134 -
135 -func clickhouseHandleMethod(ctx context.Context, job *module.Job, method string, params funcapi.ResolvedParams) *module.FunctionResponse {
136 - collector, ok := job.Module().(*Collector)
137 - if !ok {
138 - return &module.FunctionResponse{Status: 500, Message: "internal error: invalid module type"}
139 - }
140 -
141 - if collector.httpClient == nil {
142 - return &module.FunctionResponse{
143 - Status: 503,
144 - Message: "collector is still initializing, please retry in a few seconds",
145 - }
146 - }
147 -
148 - switch method {
149 - case "top-queries":
150 - return collector.collectTopQueries(ctx, params.Column(paramSort))
151 - default:
152 - return &module.FunctionResponse{Status: 404, Message: fmt.Sprintf("unknown method: %s", method)}
153 - }
154 -}
155 -
156 -func buildClickHouseSortOptions(cols []clickhouseColumnMeta) []funcapi.ParamOption {
157 - var sortOptions []funcapi.ParamOption
158 - sortDir := funcapi.FieldSortDescending
159 - for _, col := range cols {
160 - if col.isSortOption {
161 - sortOptions = append(sortOptions, funcapi.ParamOption{
162 - ID: col.uiKey,
163 - Column: col.uiKey,
164 - Name: "Top queries by " + col.sortLabel,
165 - Default: col.isDefaultSort,
166 - Sort: &sortDir,
167 - })
168 - }
169 - }
170 - return sortOptions
171 -}
172 -
173 -func (c *Collector) topQueriesParams(ctx context.Context) ([]funcapi.ParamConfig, error) {
174 - available, err := c.detectQueryLogColumns(ctx)
175 - if err != nil {
176 - return nil, err
177 - }
178 - cols := c.buildAvailableClickHouseColumns(available)
179 - if len(cols) == 0 {
180 - return nil, fmt.Errorf("no columns available in system.query_log")
181 - }
182 - sortParam := funcapi.ParamConfig{
183 - ID: paramSort,
184 - Name: "Filter By",
185 - Help: "Select the primary sort column",
186 - Selection: funcapi.ParamSelect,
187 - Options: buildClickHouseSortOptions(cols),
188 - UniqueView: true,
189 - }
190 - return []funcapi.ParamConfig{sortParam}, nil
191 -}
192 -
193 -func (c *Collector) detectQueryLogColumns(ctx context.Context) (map[string]bool, error) {
194 - query := `
195 -SELECT name
196 -FROM system.columns
197 -WHERE database = 'system' AND table = 'query_log'
198 -FORMAT JSON`
199 -
200 - req, err := web.NewHTTPRequest(c.RequestConfig)
201 - if err != nil {
202 - return nil, err
203 - }
204 - req = req.WithContext(ctx)
205 - req.URL.RawQuery = makeURLQuery(query)
206 -
207 - var resp clickhouseJSONResponse
208 - if err := web.DoHTTP(c.httpClient).RequestJSON(req, &resp); err != nil {
209 - return nil, fmt.Errorf("failed to query system.columns: %w", err)
210 - }
211 -
212 - cols := make(map[string]bool, len(resp.Data))
213 - for _, row := range resp.Data {
214 - if name, ok := row["name"].(string); ok {
215 - cols[name] = true
216 - }
217 - }
218 - if len(cols) == 0 {
219 - return nil, fmt.Errorf("system.query_log not available")
220 - }
221 - return cols, nil
222 -}
223 -
224 -func (c *Collector) buildAvailableClickHouseColumns(available map[string]bool) []clickhouseColumnMeta {
225 - var cols []clickhouseColumnMeta
226 - for _, col := range clickhouseAllColumns {
227 - if col.dbColumn == "" || available[col.dbColumn] {
228 - cols = append(cols, col)
229 - }
230 - }
231 - return cols
232 -}
233 -
234 -func (c *Collector) mapAndValidateClickHouseSortColumn(input string, available []clickhouseColumnMeta) string {
235 - availableKeys := make(map[string]bool, len(available))
236 - for _, col := range available {
237 - availableKeys[col.uiKey] = true
238 - }
239 - if availableKeys[input] {
240 - return input
241 - }
242 - if availableKeys["totalTime"] {
243 - return "totalTime"
244 - }
245 - if availableKeys["calls"] {
246 - return "calls"
247 - }
248 - return available[0].uiKey
249 -}
250 -
251 -func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
252 - availableCols, err := c.detectQueryLogColumns(ctx)
253 - if err != nil {
254 - return &module.FunctionResponse{Status: 503, Message: fmt.Sprintf("system.query_log not available: %v", err)}
255 - }
256 -
257 - cols := c.buildAvailableClickHouseColumns(availableCols)
258 - if len(cols) == 0 {
259 - return &module.FunctionResponse{Status: 500, Message: "no columns available in system.query_log"}
260 - }
261 -
262 - sortColumn = c.mapAndValidateClickHouseSortColumn(sortColumn, cols)
263 -
264 - limit := c.TopQueriesLimit
265 - if limit <= 0 {
266 - limit = 500
267 - }
268 -
269 - groupKey := "normalized_query_hash"
270 - if !availableCols[groupKey] {
271 - groupKey = "query"
272 - }
273 -
274 - selectParts := make([]string, 0, len(cols))
275 - for _, col := range cols {
276 - selectParts = append(selectParts, fmt.Sprintf("%s AS `%s`", col.selectExpr, col.uiKey))
277 - }
278 -
279 - query := fmt.Sprintf(`
280 -SELECT %s
281 -FROM system.query_log
282 -WHERE type = 'QueryFinish'
283 -GROUP BY %s
284 -ORDER BY `+"`%s`"+` DESC
285 -LIMIT %d
286 -FORMAT JSON
287 -`, strings.Join(selectParts, ", "), groupKey, sortColumn, limit)
288 -
289 - req, err := web.NewHTTPRequest(c.RequestConfig)
290 - if err != nil {
291 - return &module.FunctionResponse{Status: 500, Message: err.Error()}
292 - }
293 - req = req.WithContext(ctx)
294 - req.URL.RawQuery = makeURLQuery(query)
295 -
296 - var resp clickhouseJSONResponse
297 - if err := web.DoHTTP(c.httpClient).RequestJSON(req, &resp); err != nil {
298 - if ctx.Err() == context.DeadlineExceeded {
299 - return &module.FunctionResponse{Status: 504, Message: "query timed out"}
300 - }
301 - return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("query failed: %v", err)}
302 - }
303 -
304 - data := make([][]any, 0, len(resp.Data))
305 - for _, rowMap := range resp.Data {
306 - row := make([]any, len(cols))
307 - for i, col := range cols {
308 - row[i] = normalizeClickHouseValue(col, rowMap[col.uiKey])
309 - }
310 - data = append(data, row)
311 - }
312 -
313 - sortParam := funcapi.ParamConfig{
314 - ID: paramSort,
315 - Name: "Filter By",
316 - Help: "Select the primary sort column",
317 - Selection: funcapi.ParamSelect,
318 - Options: buildClickHouseSortOptions(cols),
319 - UniqueView: true,
320 - }
321 -
322 - defaultSort := "totalTime"
323 - if !containsClickHouseColumn(cols, defaultSort) {
324 - defaultSort = "calls"
325 - }
326 -
327 - return &module.FunctionResponse{
328 - Status: 200,
329 - Help: "Top SQL queries from ClickHouse system.query_log",
330 - Columns: buildClickHouseColumns(cols),
331 - Data: data,
332 - DefaultSortColumn: defaultSort,
333 - RequiredParams: []funcapi.ParamConfig{sortParam},
334 - Charts: clickhouseTopQueriesCharts(cols),
335 - DefaultCharts: clickhouseTopQueriesDefaultCharts(cols),
336 - GroupBy: clickhouseTopQueriesGroupBy(cols),
337 - }
338 -}
339 -
340 -func normalizeClickHouseValue(col clickhouseColumnMeta, v any) any {
341 - switch col.dataType {
342 - case ftInteger:
343 - switch val := v.(type) {
344 - case float64:
345 - return int64(val)
346 - case json.Number:
347 - if i, err := val.Int64(); err == nil {
348 - return i
349 - }
350 - case string:
351 - if i, err := strconv.ParseInt(val, 10, 64); err == nil {
352 - return i
353 - }
354 - }
355 - return int64(0)
356 - case ftFloat, ftDuration:
357 - switch val := v.(type) {
358 - case float64:
359 - return val
360 - case json.Number:
361 - if f, err := val.Float64(); err == nil {
362 - return f
363 - }
364 - case string:
365 - if f, err := strconv.ParseFloat(val, 64); err == nil {
366 - return f
367 - }
368 - }
369 - return float64(0)
370 - default:
371 - if s, ok := v.(string); ok {
372 - if col.uiKey == "query" {
373 - return strmutil.TruncateText(s, clickhouseMaxQueryTextLength)
374 - }
375 - return s
376 - }
377 - if v == nil {
378 - return ""
379 - }
380 - if col.uiKey == "query" {
381 - return strmutil.TruncateText(fmt.Sprint(v), clickhouseMaxQueryTextLength)
382 - }
383 - return fmt.Sprint(v)
384 - }
385 -}
386 -
387 -func buildClickHouseColumns(cols []clickhouseColumnMeta) map[string]any {
388 - columns := make(map[string]any, len(cols))
389 - for i, col := range cols {
390 - visual := funcapi.FieldVisualValue
391 - if col.dataType == ftDuration {
392 - visual = funcapi.FieldVisualBar
393 - }
394 - colDef := funcapi.Column{
395 - Index: i,
396 - Name: col.displayName,
397 - Type: col.dataType,
398 - Units: col.units,
399 - Visualization: visual,
400 - Sort: col.sortDir,
401 - Sortable: true,
402 - Sticky: col.isSticky,
403 - Summary: col.summary,
404 - Filter: col.filter,
405 - FullWidth: col.fullWidth,
406 - Wrap: false,
407 - DefaultExpandedFilter: false,
408 - UniqueKey: col.isUniqueKey,
409 - Visible: col.visible,
410 - ValueOptions: funcapi.ValueOptions{
411 - Transform: col.transform,
412 - DecimalPoints: col.decimalPoints,
413 - DefaultValue: nil,
414 - },
415 - }
416 - columns[col.uiKey] = colDef.BuildColumn()
417 - }
418 - return columns
419 -}
420 -
421 -func containsClickHouseColumn(cols []clickhouseColumnMeta, key string) bool {
422 - for _, col := range cols {
423 - if col.uiKey == key {
424 - return true
425 - }
426 - }
427 - return false
428 -}
429 -
430 -func clickhouseTopQueriesCharts(cols []clickhouseColumnMeta) map[string]module.ChartConfig {
431 - charts := make(map[string]module.ChartConfig)
432 - for _, col := range cols {
433 - if !col.isMetric || col.chartGroup == "" {
434 - continue
435 - }
436 - cfg, ok := charts[col.chartGroup]
437 - if !ok {
438 - title := col.chartTitle
439 - if title == "" {
440 - title = col.chartGroup
441 - }
442 - cfg = module.ChartConfig{
443 - Name: title,
444 - Type: "stacked-bar",
445 - }
446 - }
447 - cfg.Columns = append(cfg.Columns, col.uiKey)
448 - charts[col.chartGroup] = cfg
449 - }
450 - return charts
451 -}
452 -
453 -func clickhouseTopQueriesDefaultCharts(cols []clickhouseColumnMeta) [][]string {
454 - label := primaryClickhouseLabel(cols)
455 - if label == "" {
456 - return nil
457 - }
458 - chartGroups := defaultClickhouseChartGroups(cols)
459 - out := make([][]string, 0, len(chartGroups))
460 - for _, group := range chartGroups {
461 - out = append(out, []string{group, label})
462 - }
463 - return out
464 -}
465 -
466 -func clickhouseTopQueriesGroupBy(cols []clickhouseColumnMeta) map[string]module.GroupByConfig {
467 - groupBy := make(map[string]module.GroupByConfig)
468 - for _, col := range cols {
469 - if !col.isLabel {
470 - continue
471 - }
472 - groupBy[col.uiKey] = module.GroupByConfig{
473 - Name: "Group by " + col.displayName,
474 - Columns: []string{col.uiKey},
475 - }
476 - }
477 - return groupBy
478 -}
479 -
480 -func primaryClickhouseLabel(cols []clickhouseColumnMeta) string {
481 - for _, col := range cols {
482 - if col.isPrimary {
483 - return col.uiKey
484 - }
485 - }
486 - for _, col := range cols {
487 - if col.isLabel {
488 - return col.uiKey
489 - }
490 - }
491 - return ""
492 -}
493 -
494 -func defaultClickhouseChartGroups(cols []clickhouseColumnMeta) []string {
495 - groups := make([]string, 0)
496 - seen := make(map[string]bool)
497 - for _, col := range cols {
498 - if !col.isMetric || col.chartGroup == "" || !col.isDefaultChart {
499 - continue
500 - }
501 - if !seen[col.chartGroup] {
502 - seen[col.chartGroup] = true
503 - groups = append(groups, col.chartGroup)
504 - }
505 - }
506 - if len(groups) > 0 {
507 - return groups
508 - }
509 - for _, col := range cols {
510 - if !col.isMetric || col.chartGroup == "" {
511 - continue
512 - }
513 - if !seen[col.chartGroup] {
514 - seen[col.chartGroup] = true
515 - groups = append(groups, col.chartGroup)
516 - }
517 - }
518 - return groups
519 -}
src/go/plugin/go.d/collector/cockroachdb/collector.go
+10 -10
@@ -4,7 +4,6 @@ package cockroachdb
4
5 import (
6 "context"
7 - "database/sql"
7 _ "embed"
8 "errors"
9 "fmt"
@@ -29,11 +28,10 @@ func init() {
28 Defaults: module.Defaults{
29 UpdateEvery: dbSamplingInterval,
30 },
32 - Methods: cockroachMethods,
33 - MethodParams: cockroachMethodParams,
34 - HandleMethod: cockroachHandleMethod,
35 - Create: func() module.Module { return New() },
36 - Config: func() any { return &Config{} },
31 + Methods: cockroachMethods,
32 + MethodHandler: cockroachFunctionHandler,
33 + Create: func() module.Module { return New() },
34 + Config: func() any { return &Config{} },
35 })
36 }
37
@@ -72,7 +70,7 @@ type Collector struct {
70
71 prom prometheus.Prometheus
72
75 - db *sql.DB
73 + funcRouter *funcRouter
74 }
75
76 func (c *Collector) Configuration() any {
@@ -90,6 +88,8 @@ func (c *Collector) Init(context.Context) error {
88 }
89 c.prom = prom
90
91 + c.funcRouter = newFuncRouter(c)
92 +
93 if c.UpdateEvery < dbSamplingInterval {
94 c.Warningf("'update_every'(%d) is lower then CockroachDB default sampling interval (%d)",
95 c.UpdateEvery, dbSamplingInterval)
@@ -126,11 +126,11 @@ func (c *Collector) Collect(context.Context) map[string]int64 {
126 return mx
127 }
128
129 -func (c *Collector) Cleanup(context.Context) {
129 +func (c *Collector) Cleanup(ctx context.Context) {
130 if c.prom != nil && c.prom.HTTPClient() != nil {
131 c.prom.HTTPClient().CloseIdleConnections()
132 }
133 - if c.db != nil {
134 - _ = c.db.Close()
133 + if c.funcRouter != nil {
134 + c.funcRouter.Cleanup(ctx)
135 }
136 }
src/go/plugin/go.d/collector/cockroachdb/func_queries_test.go renamed
+6 -14
@@ -35,25 +35,17 @@ func TestCockroachDBMethods(t *testing.T) {
35 func TestCockroachDBTopColumns_HasRequiredColumns(t *testing.T) {
36 required := []string{"fingerprintId", "query", "executions", "totalTime"}
37
38 - uiKeys := make(map[string]bool)
39 - for _, col := range crdbTopColumns {
40 - uiKeys[col.id] = true
41 - }
42 -
43 - for _, key := range required {
44 - assert.True(t, uiKeys[key], "column %s should be defined", key)
38 + cs := funcapi.Columns(topQueriesColumns, func(c topQueriesColumn) funcapi.ColumnMeta { return c.ColumnMeta })
39 + for _, id := range required {
40 + assert.True(t, cs.ContainsColumn(id), "column %s should be defined", id)
41 }
42 }
43
44 func TestCockroachDBRunningColumns_HasRequiredColumns(t *testing.T) {
45 required := []string{"queryId", "query", "elapsedMs"}
46
51 - uiKeys := make(map[string]bool)
52 - for _, col := range crdbRunningColumns {
53 - uiKeys[col.id] = true
54 - }
55 -
56 - for _, key := range required {
57 - assert.True(t, uiKeys[key], "column %s should be defined", key)
47 + cs := funcapi.Columns(runningQueriesColumns, func(c runningQueriesColumn) funcapi.ColumnMeta { return c.ColumnMeta })
48 + for _, id := range required {
49 + assert.True(t, cs.ContainsColumn(id), "column %s should be defined", id)
50 }
51 }
src/go/plugin/go.d/collector/cockroachdb/func_router.go new
+135
@@ -0,0 +1,135 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package cockroachdb
4 +
5 +import (
6 + "context"
7 + "database/sql"
8 + "errors"
9 + "fmt"
10 + "sync"
11 + "time"
12 +
13 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
14 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
15 +)
16 +
17 +var errSQLDSNNotSet = errors.New("SQL DSN is not set")
18 +
19 +// funcRouter routes method calls to appropriate function handlers.
20 +// Owns shared SQL connection used by all function handlers.
21 +type funcRouter struct {
22 + collector *Collector // for config (DSN, SQLTimeout, TopQueriesLimit, logger)
23 +
24 + // Shared SQL connection
25 + db *sql.DB
26 + dbMu sync.Mutex
27 +
28 + handlers map[string]funcapi.MethodHandler
29 +}
30 +
31 +func newFuncRouter(c *Collector) *funcRouter {
32 + r := &funcRouter{
33 + collector: c,
34 + handlers: make(map[string]funcapi.MethodHandler),
35 + }
36 + r.handlers[topQueriesMethodID] = newFuncTopQueries(r)
37 + r.handlers[runningQueriesMethodID] = newFuncRunningQueries(r)
38 + return r
39 +}
40 +
41 +// Compile-time interface check.
42 +var _ funcapi.MethodHandler = (*funcRouter)(nil)
43 +
44 +func (r *funcRouter) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
45 + if h, ok := r.handlers[method]; ok {
46 + return h.MethodParams(ctx, method)
47 + }
48 + return nil, fmt.Errorf("unknown method: %s", method)
49 +}
50 +
51 +func (r *funcRouter) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
52 + if h, ok := r.handlers[method]; ok {
53 + return h.Handle(ctx, method, params)
54 + }
55 + return funcapi.NotFoundResponse(method)
56 +}
57 +
58 +func (r *funcRouter) Cleanup(ctx context.Context) {
59 + for _, h := range r.handlers {
60 + h.Cleanup(ctx)
61 + }
62 + r.dbMu.Lock()
63 + defer r.dbMu.Unlock()
64 + if r.db != nil {
65 + _ = r.db.Close()
66 + r.db = nil
67 + }
68 +}
69 +
70 +// ensureDB lazily initializes the SQL connection.
71 +func (r *funcRouter) ensureDB(ctx context.Context) error {
72 + r.dbMu.Lock()
73 + defer r.dbMu.Unlock()
74 +
75 + if r.db != nil {
76 + return nil
77 + }
78 + if r.collector.DSN == "" {
79 + return errSQLDSNNotSet
80 + }
81 +
82 + db, err := sql.Open("pgx", r.collector.DSN)
83 + if err != nil {
84 + return fmt.Errorf("error opening SQL connection: %w", err)
85 + }
86 + db.SetMaxOpenConns(1)
87 + db.SetMaxIdleConns(1)
88 + db.SetConnMaxLifetime(10 * time.Minute)
89 +
90 + timeout := r.sqlTimeout()
91 + pingCtx, cancel := context.WithTimeout(ctx, timeout)
92 + defer cancel()
93 + if err := db.PingContext(pingCtx); err != nil {
94 + _ = db.Close()
95 + return fmt.Errorf("error pinging SQL connection: %w", err)
96 + }
97 +
98 + setCtx, cancel := context.WithTimeout(ctx, timeout)
99 + if _, err := db.ExecContext(setCtx, "SET allow_unsafe_internals = on"); err != nil {
100 + r.collector.Debugf("unable to set allow_unsafe_internals: %v", err)
101 + }
102 + cancel()
103 +
104 + r.db = db
105 + return nil
106 +}
107 +
108 +func (r *funcRouter) sqlTimeout() time.Duration {
109 + if r.collector.SQLTimeout.Duration() > 0 {
110 + return r.collector.SQLTimeout.Duration()
111 + }
112 + return time.Second
113 +}
114 +
115 +func (r *funcRouter) topQueriesLimit() int {
116 + if r.collector.TopQueriesLimit > 0 {
117 + return r.collector.TopQueriesLimit
118 + }
119 + return 500
120 +}
121 +
122 +func cockroachMethods() []funcapi.MethodConfig {
123 + return []funcapi.MethodConfig{
124 + topQueriesMethodConfig(),
125 + runningQueriesMethodConfig(),
126 + }
127 +}
128 +
129 +func cockroachFunctionHandler(job *module.Job) funcapi.MethodHandler {
130 + c, ok := job.Module().(*Collector)
131 + if !ok {
132 + return nil
133 + }
134 + return c.funcRouter
135 +}
src/go/plugin/go.d/collector/cockroachdb/func_running_queries.go new
+229
@@ -0,0 +1,229 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package cockroachdb
4 +
5 +import (
6 + "context"
7 + "database/sql"
8 + "errors"
9 + "fmt"
10 + "strings"
11 +
12 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
13 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
14 +)
15 +
16 +const (
17 + runningQueriesMethodID = "running-queries"
18 + runningQueriesMaxTextLength = 4096
19 +)
20 +
21 +func runningQueriesMethodConfig() funcapi.MethodConfig {
22 + return funcapi.MethodConfig{
23 + ID: runningQueriesMethodID,
24 + Name: "Running Queries",
25 + UpdateEvery: 10,
26 + Help: "Currently running SQL statements from SHOW CLUSTER STATEMENTS. WARNING: Query text may contain unmasked literals (potential PII).",
27 + RequireCloud: true,
28 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(runningQueriesColumns)},
29 + }
30 +}
31 +
32 +// runningQueriesColumn embeds funcapi.ColumnMeta and adds CockroachDB-specific fields.
33 +type runningQueriesColumn struct {
34 + funcapi.ColumnMeta
35 + SelectExpr string // SQL expression for SELECT clause
36 + sortOpt bool // whether this column appears as a sort option
37 + sortLbl string // label for sort option dropdown
38 + defaultSort bool // default sort column
39 +}
40 +
41 +// funcapi.SortableColumn interface implementation for runningQueriesColumn.
42 +func (c runningQueriesColumn) IsSortOption() bool { return c.sortOpt }
43 +func (c runningQueriesColumn) SortLabel() string { return c.sortLbl }
44 +func (c runningQueriesColumn) IsDefaultSort() bool { return c.defaultSort }
45 +func (c runningQueriesColumn) ColumnName() string { return c.Name }
46 +func (c runningQueriesColumn) SortColumn() string { return "" }
47 +
48 +var runningQueriesColumns = []runningQueriesColumn{
49 + {ColumnMeta: funcapi.ColumnMeta{Name: "queryId", Tooltip: "Query ID", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, UniqueKey: true, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.query_id::STRING"},
50 + {ColumnMeta: funcapi.ColumnMeta{Name: "query", Tooltip: "Query", Type: funcapi.FieldTypeString, Visible: true, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sticky: true, FullWidth: true, Wrap: true}, SelectExpr: "s.query"},
51 + {ColumnMeta: funcapi.ColumnMeta{Name: "user", Tooltip: "User", Type: funcapi.FieldTypeString, Visible: true, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.user_name"},
52 + {ColumnMeta: funcapi.ColumnMeta{Name: "application", Tooltip: "Application", Type: funcapi.FieldTypeString, Visible: true, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.application_name"},
53 + {ColumnMeta: funcapi.ColumnMeta{Name: "clientAddress", Tooltip: "Client Address", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.client_address"},
54 + {ColumnMeta: funcapi.ColumnMeta{Name: "nodeId", Tooltip: "Node ID", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.node_id::STRING"},
55 + {ColumnMeta: funcapi.ColumnMeta{Name: "sessionId", Tooltip: "Session ID", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.session_id::STRING"},
56 + {ColumnMeta: funcapi.ColumnMeta{Name: "phase", Tooltip: "Phase", Type: funcapi.FieldTypeString, Visible: true, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.phase"},
57 + {ColumnMeta: funcapi.ColumnMeta{Name: "distributed", Tooltip: "Distributed", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.distributed::STRING"},
58 + {ColumnMeta: funcapi.ColumnMeta{Name: "startTime", Tooltip: "Start Time", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax}, SelectExpr: "TO_CHAR(s.start, 'YYYY-MM-DD\"T\"HH24:MI:SS.FF3')"},
59 + {ColumnMeta: funcapi.ColumnMeta{Name: "elapsedMs", Tooltip: "Elapsed", Type: funcapi.FieldTypeDuration, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualBar, Transform: funcapi.FieldTransformDuration, Units: "milliseconds", DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax}, SelectExpr: "EXTRACT(EPOCH FROM (clock_timestamp() - s.start)) * 1000", sortOpt: true, defaultSort: true, sortLbl: "Running queries by Elapsed Time"},
60 +}
61 +
62 +// funcRunningQueries handles the running-queries function.
63 +type funcRunningQueries struct {
64 + router *funcRouter
65 +}
66 +
67 +func newFuncRunningQueries(r *funcRouter) *funcRunningQueries {
68 + return &funcRunningQueries{router: r}
69 +}
70 +
71 +// Compile-time interface check.
72 +var _ funcapi.MethodHandler = (*funcRunningQueries)(nil)
73 +
74 +func (f *funcRunningQueries) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
75 + return []funcapi.ParamConfig{funcapi.BuildSortParam(runningQueriesColumns)}, nil
76 +}
77 +
78 +func (f *funcRunningQueries) Cleanup(ctx context.Context) {}
79 +
80 +func (f *funcRunningQueries) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
81 + if err := f.router.ensureDB(ctx); err != nil {
82 + status := 503
83 + if errors.Is(err, errSQLDSNNotSet) {
84 + status = 400
85 + }
86 + return funcapi.ErrorResponse(status, "%s", err)
87 + }
88 +
89 + sortColumn := f.resolveSortColumn(params.Column("__sort"))
90 + limit := f.router.topQueriesLimit()
91 +
92 + query := f.buildSQL(sortColumn)
93 + queryCtx, cancel := context.WithTimeout(ctx, f.router.sqlTimeout())
94 + defer cancel()
95 +
96 + rows, err := f.router.db.QueryContext(queryCtx, query, limit)
97 + if err != nil {
98 + if errors.Is(queryCtx.Err(), context.DeadlineExceeded) {
99 + return funcapi.ErrorResponse(504, "query timed out")
100 + }
101 + return funcapi.InternalErrorResponse("query failed: %v", err)
102 + }
103 + defer rows.Close()
104 +
105 + data, err := f.scanRows(rows)
106 + if err != nil {
107 + return funcapi.InternalErrorResponse("%s", err)
108 + }
109 +
110 + cs := f.columnSet()
111 + return &funcapi.FunctionResponse{
112 + Status: 200,
113 + Help: "Currently running SQL statements from SHOW CLUSTER STATEMENTS. WARNING: Query text may contain unmasked literals (potential PII).",
114 + Columns: cs.BuildColumns(),
115 + Data: data,
116 + DefaultSortColumn: sortColumn,
117 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(runningQueriesColumns)},
118 + }
119 +}
120 +
121 +func (f *funcRunningQueries) columnSet() funcapi.ColumnSet[runningQueriesColumn] {
122 + return funcapi.Columns(runningQueriesColumns, func(c runningQueriesColumn) funcapi.ColumnMeta { return c.ColumnMeta })
123 +}
124 +
125 +func (f *funcRunningQueries) resolveSortColumn(requested string) string {
126 + if requested != "" {
127 + for _, col := range runningQueriesColumns {
128 + if col.Name == requested && col.IsSortOption() {
129 + return col.Name
130 + }
131 + }
132 + }
133 + for _, col := range runningQueriesColumns {
134 + if col.IsDefaultSort() && col.IsSortOption() {
135 + return col.Name
136 + }
137 + }
138 + for _, col := range runningQueriesColumns {
139 + if col.IsSortOption() {
140 + return col.Name
141 + }
142 + }
143 + if len(runningQueriesColumns) > 0 {
144 + return runningQueriesColumns[0].Name
145 + }
146 + return ""
147 +}
148 +
149 +func (f *funcRunningQueries) buildSQL(sortColumn string) string {
150 + selectCols := make([]string, 0, len(runningQueriesColumns))
151 + for _, col := range runningQueriesColumns {
152 + selectCols = append(selectCols, fmt.Sprintf("%s AS %s", col.SelectExpr, col.Name))
153 + }
154 + return fmt.Sprintf(`
155 +SELECT %s
156 +FROM [SHOW CLUSTER STATEMENTS] AS s
157 +ORDER BY %s DESC NULLS LAST
158 +LIMIT $1`, strings.Join(selectCols, ", "), sortColumn)
159 +}
160 +
161 +func (f *funcRunningQueries) scanRows(rows *sql.Rows) ([][]any, error) {
162 + cols := runningQueriesColumns
163 + data := make([][]any, 0, 500)
164 +
165 + for rows.Next() {
166 + values := make([]any, len(cols))
167 + valuePtrs := make([]any, len(cols))
168 +
169 + for i, col := range cols {
170 + switch col.Type {
171 + case funcapi.FieldTypeString:
172 + var v sql.NullString
173 + values[i] = &v
174 + case funcapi.FieldTypeInteger:
175 + var v sql.NullInt64
176 + values[i] = &v
177 + case funcapi.FieldTypeFloat, funcapi.FieldTypeDuration:
178 + var v sql.NullFloat64
179 + values[i] = &v
180 + default:
181 + var v any
182 + values[i] = &v
183 + }
184 + valuePtrs[i] = values[i]
185 + }
186 +
187 + if err := rows.Scan(valuePtrs...); err != nil {
188 + return nil, fmt.Errorf("row scan failed: %w", err)
189 + }
190 +
191 + row := make([]any, len(cols))
192 + for i, col := range cols {
193 + switch v := values[i].(type) {
194 + case *sql.NullString:
195 + if v.Valid {
196 + s := v.String
197 + if col.Name == "query" {
198 + s = strmutil.TruncateText(s, runningQueriesMaxTextLength)
199 + }
200 + row[i] = s
201 + } else {
202 + row[i] = ""
203 + }
204 + case *sql.NullInt64:
205 + if v.Valid {
206 + row[i] = v.Int64
207 + } else {
208 + row[i] = int64(0)
209 + }
210 + case *sql.NullFloat64:
211 + if v.Valid {
212 + row[i] = v.Float64
213 + } else {
214 + row[i] = float64(0)
215 + }
216 + default:
217 + row[i] = nil
218 + }
219 + }
220 +
221 + data = append(data, row)
222 + }
223 +
224 + if err := rows.Err(); err != nil {
225 + return nil, fmt.Errorf("rows iteration error: %w", err)
226 + }
227 +
228 + return data, nil
229 +}
src/go/plugin/go.d/collector/cockroachdb/func_top_queries.go new
+242
@@ -0,0 +1,242 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package cockroachdb
4 +
5 +import (
6 + "context"
7 + "database/sql"
8 + "errors"
9 + "fmt"
10 + "strings"
11 +
12 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
13 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
14 +)
15 +
16 +const (
17 + topQueriesMethodID = "top-queries"
18 + topQueriesMaxTextLength = 4096
19 +)
20 +
21 +func topQueriesMethodConfig() funcapi.MethodConfig {
22 + return funcapi.MethodConfig{
23 + ID: topQueriesMethodID,
24 + Name: "Top Queries",
25 + UpdateEvery: 10,
26 + Help: "Top SQL statements from crdb_internal.cluster_statement_statistics. WARNING: Query text may contain unmasked literals (potential PII).",
27 + RequireCloud: true,
28 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(topQueriesColumns)},
29 + }
30 +}
31 +
32 +// topQueriesColumn embeds funcapi.ColumnMeta and adds CockroachDB-specific fields.
33 +type topQueriesColumn struct {
34 + funcapi.ColumnMeta
35 + SelectExpr string // SQL expression for SELECT clause
36 + sortOpt bool // whether this column appears as a sort option
37 + sortLbl string // label for sort option dropdown
38 + defaultSort bool // default sort column
39 +}
40 +
41 +// funcapi.SortableColumn interface implementation for topQueriesColumn.
42 +func (c topQueriesColumn) IsSortOption() bool { return c.sortOpt }
43 +func (c topQueriesColumn) SortLabel() string { return c.sortLbl }
44 +func (c topQueriesColumn) IsDefaultSort() bool { return c.defaultSort }
45 +func (c topQueriesColumn) ColumnName() string { return c.Name }
46 +func (c topQueriesColumn) SortColumn() string { return "" }
47 +
48 +var topQueriesColumns = []topQueriesColumn{
49 + {ColumnMeta: funcapi.ColumnMeta{Name: "fingerprintId", Tooltip: "Fingerprint ID", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, UniqueKey: true, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.fingerprint_id::STRING"},
50 + {ColumnMeta: funcapi.ColumnMeta{Name: "query", Tooltip: "Query", Type: funcapi.FieldTypeString, Visible: true, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sticky: true, FullWidth: true, Wrap: true}, SelectExpr: "s.metadata->>'query'"},
51 + {ColumnMeta: funcapi.ColumnMeta{Name: "database", Tooltip: "Database", Type: funcapi.FieldTypeString, Visible: true, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, GroupBy: &funcapi.GroupByOptions{IsDefault: true}}, SelectExpr: "s.metadata->>'db'"},
52 + {ColumnMeta: funcapi.ColumnMeta{Name: "application", Tooltip: "Application", Type: funcapi.FieldTypeString, Visible: true, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, GroupBy: &funcapi.GroupByOptions{}}, SelectExpr: "s.app_name"},
53 + {ColumnMeta: funcapi.ColumnMeta{Name: "statementType", Tooltip: "Statement Type", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, GroupBy: &funcapi.GroupByOptions{}}, SelectExpr: "s.metadata->>'stmtTyp'"},
54 + {ColumnMeta: funcapi.ColumnMeta{Name: "distributed", Tooltip: "Distributed", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.metadata->>'distsql'"},
55 + {ColumnMeta: funcapi.ColumnMeta{Name: "fullScan", Tooltip: "Full Scan", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.metadata->>'fullScan'"},
56 + {ColumnMeta: funcapi.ColumnMeta{Name: "implicitTxn", Tooltip: "Implicit Txn", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.metadata->>'implicitTxn'"},
57 + {ColumnMeta: funcapi.ColumnMeta{Name: "vectorized", Tooltip: "Vectorized", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.metadata->>'vec'"},
58 +
59 + {ColumnMeta: funcapi.ColumnMeta{Name: "executions", Tooltip: "Executions", Type: funcapi.FieldTypeInteger, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "Calls", Title: "Executions", IsDefault: true}}, SelectExpr: "COALESCE((s.statistics->'statistics'->>'cnt')::INT8, 0)", sortOpt: true, sortLbl: "Top queries by Executions"},
60 + {ColumnMeta: funcapi.ColumnMeta{Name: "totalTime", Tooltip: "Total Time", Type: funcapi.FieldTypeDuration, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualBar, Transform: funcapi.FieldTransformDuration, Units: "milliseconds", DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Execution Time", IsDefault: true}}, SelectExpr: "COALESCE((s.statistics->'statistics'->'svcLat'->>'mean')::FLOAT8, 0) * COALESCE((s.statistics->'statistics'->>'cnt')::FLOAT8, 0) * 1000", sortOpt: true, defaultSort: true, sortLbl: "Top queries by Total Time"},
61 + {ColumnMeta: funcapi.ColumnMeta{Name: "meanTime", Tooltip: "Mean Time", Type: funcapi.FieldTypeDuration, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualBar, Transform: funcapi.FieldTransformDuration, Units: "milliseconds", DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMean, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Execution Time"}}, SelectExpr: "COALESCE((s.statistics->'statistics'->'svcLat'->>'mean')::FLOAT8, 0) * 1000", sortOpt: true, sortLbl: "Top queries by Mean Time"},
62 + {ColumnMeta: funcapi.ColumnMeta{Name: "runTime", Tooltip: "Run Time", Type: funcapi.FieldTypeDuration, Visible: false, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualBar, Transform: funcapi.FieldTransformDuration, Units: "milliseconds", DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMean, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Execution Time"}}, SelectExpr: "COALESCE((s.statistics->'statistics'->'runLat'->>'mean')::FLOAT8, 0) * 1000"},
63 + {ColumnMeta: funcapi.ColumnMeta{Name: "planTime", Tooltip: "Plan Time", Type: funcapi.FieldTypeDuration, Visible: false, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualBar, Transform: funcapi.FieldTransformDuration, Units: "milliseconds", DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMean, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Execution Time"}}, SelectExpr: "COALESCE((s.statistics->'statistics'->'planLat'->>'mean')::FLOAT8, 0) * 1000"},
64 + {ColumnMeta: funcapi.ColumnMeta{Name: "parseTime", Tooltip: "Parse Time", Type: funcapi.FieldTypeDuration, Visible: false, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualBar, Transform: funcapi.FieldTransformDuration, Units: "milliseconds", DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMean, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Execution Time"}}, SelectExpr: "COALESCE((s.statistics->'statistics'->'parseLat'->>'mean')::FLOAT8, 0) * 1000"},
65 +
66 + {ColumnMeta: funcapi.ColumnMeta{Name: "rowsRead", Tooltip: "Rows Read", Type: funcapi.FieldTypeInteger, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "Rows", Title: "Rows"}}, SelectExpr: "CAST(ROUND(COALESCE((s.statistics->'statistics'->'rowsRead'->>'mean')::FLOAT8, 0) * COALESCE((s.statistics->'statistics'->>'cnt')::FLOAT8, 0)) AS INT8)", sortOpt: true, sortLbl: "Top queries by Rows Read"},
67 + {ColumnMeta: funcapi.ColumnMeta{Name: "rowsWritten", Tooltip: "Rows Written", Type: funcapi.FieldTypeInteger, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "Rows", Title: "Rows"}}, SelectExpr: "CAST(ROUND(COALESCE((s.statistics->'statistics'->'rowsWritten'->>'mean')::FLOAT8, 0) * COALESCE((s.statistics->'statistics'->>'cnt')::FLOAT8, 0)) AS INT8)", sortOpt: true, sortLbl: "Top queries by Rows Written"},
68 + {ColumnMeta: funcapi.ColumnMeta{Name: "rowsReturned", Tooltip: "Rows Returned", Type: funcapi.FieldTypeInteger, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "Rows", Title: "Rows"}}, SelectExpr: "CAST(ROUND(COALESCE((s.statistics->'statistics'->'numRows'->>'mean')::FLOAT8, 0) * COALESCE((s.statistics->'statistics'->>'cnt')::FLOAT8, 0)) AS INT8)", sortOpt: true, sortLbl: "Top queries by Rows Returned"},
69 + {ColumnMeta: funcapi.ColumnMeta{Name: "bytesRead", Tooltip: "Bytes Read", Type: funcapi.FieldTypeInteger, Visible: false, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "Bytes", Title: "Bytes"}}, SelectExpr: "CAST(ROUND(COALESCE((s.statistics->'statistics'->'bytesRead'->>'mean')::FLOAT8, 0) * COALESCE((s.statistics->'statistics'->>'cnt')::FLOAT8, 0)) AS INT8)", sortOpt: true, sortLbl: "Top queries by Bytes Read"},
70 + {ColumnMeta: funcapi.ColumnMeta{Name: "maxRetries", Tooltip: "Max Retries", Type: funcapi.FieldTypeInteger, Visible: false, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Chart: &funcapi.ChartOptions{Group: "Retries", Title: "Retries"}}, SelectExpr: "COALESCE((s.statistics->'statistics'->>'maxRetries')::INT8, 0)"},
71 +}
72 +
73 +// funcTopQueries handles the top-queries function.
74 +type funcTopQueries struct {
75 + router *funcRouter
76 +}
77 +
78 +func newFuncTopQueries(r *funcRouter) *funcTopQueries {
79 + return &funcTopQueries{router: r}
80 +}
81 +
82 +// Compile-time interface check.
83 +var _ funcapi.MethodHandler = (*funcTopQueries)(nil)
84 +
85 +func (f *funcTopQueries) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
86 + return []funcapi.ParamConfig{funcapi.BuildSortParam(topQueriesColumns)}, nil
87 +}
88 +
89 +func (f *funcTopQueries) Cleanup(ctx context.Context) {}
90 +
91 +func (f *funcTopQueries) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
92 + if err := f.router.ensureDB(ctx); err != nil {
93 + status := 503
94 + if errors.Is(err, errSQLDSNNotSet) {
95 + status = 400
96 + }
97 + return funcapi.ErrorResponse(status, "%s", err)
98 + }
99 +
100 + sortColumn := f.resolveSortColumn(params.Column("__sort"))
101 + limit := f.router.topQueriesLimit()
102 +
103 + query := f.buildSQL(sortColumn)
104 + queryCtx, cancel := context.WithTimeout(ctx, f.router.sqlTimeout())
105 + defer cancel()
106 +
107 + rows, err := f.router.db.QueryContext(queryCtx, query, limit)
108 + if err != nil {
109 + if errors.Is(queryCtx.Err(), context.DeadlineExceeded) {
110 + return funcapi.ErrorResponse(504, "query timed out")
111 + }
112 + return funcapi.InternalErrorResponse("query failed: %v", err)
113 + }
114 + defer rows.Close()
115 +
116 + data, err := f.scanRows(rows)
117 + if err != nil {
118 + return funcapi.InternalErrorResponse("%s", err)
119 + }
120 +
121 + cs := f.columnSet()
122 + return &funcapi.FunctionResponse{
123 + Status: 200,
124 + Help: "Top SQL statements from crdb_internal.cluster_statement_statistics. WARNING: Query text may contain unmasked literals (potential PII).",
125 + Columns: cs.BuildColumns(),
126 + Data: data,
127 + DefaultSortColumn: sortColumn,
128 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(topQueriesColumns)},
129 + ChartingConfig: cs.BuildCharting(),
130 + }
131 +}
132 +
133 +func (f *funcTopQueries) columnSet() funcapi.ColumnSet[topQueriesColumn] {
134 + return funcapi.Columns(topQueriesColumns, func(c topQueriesColumn) funcapi.ColumnMeta { return c.ColumnMeta })
135 +}
136 +
137 +func (f *funcTopQueries) resolveSortColumn(requested string) string {
138 + if requested != "" {
139 + for _, col := range topQueriesColumns {
140 + if col.Name == requested && col.IsSortOption() {
141 + return col.Name
142 + }
143 + }
144 + }
145 + for _, col := range topQueriesColumns {
146 + if col.IsDefaultSort() && col.IsSortOption() {
147 + return col.Name
148 + }
149 + }
150 + for _, col := range topQueriesColumns {
151 + if col.IsSortOption() {
152 + return col.Name
153 + }
154 + }
155 + if len(topQueriesColumns) > 0 {
156 + return topQueriesColumns[0].Name
157 + }
158 + return ""
159 +}
160 +
161 +func (f *funcTopQueries) buildSQL(sortColumn string) string {
162 + selectCols := make([]string, 0, len(topQueriesColumns))
163 + for _, col := range topQueriesColumns {
164 + selectCols = append(selectCols, fmt.Sprintf("%s AS %s", col.SelectExpr, col.Name))
165 + }
166 + return fmt.Sprintf(`
167 +SELECT %s
168 +FROM crdb_internal.cluster_statement_statistics AS s
169 +WHERE s.metadata->>'query' IS NOT NULL
170 +ORDER BY %s DESC NULLS LAST
171 +LIMIT $1`, strings.Join(selectCols, ", "), sortColumn)
172 +}
173 +
174 +func (f *funcTopQueries) scanRows(rows *sql.Rows) ([][]any, error) {
175 + cols := topQueriesColumns
176 + data := make([][]any, 0, 500)
177 +
178 + for rows.Next() {
179 + values := make([]any, len(cols))
180 + valuePtrs := make([]any, len(cols))
181 +
182 + for i, col := range cols {
183 + switch col.Type {
184 + case funcapi.FieldTypeString:
185 + var v sql.NullString
186 + values[i] = &v
187 + case funcapi.FieldTypeInteger:
188 + var v sql.NullInt64
189 + values[i] = &v
190 + case funcapi.FieldTypeFloat, funcapi.FieldTypeDuration:
191 + var v sql.NullFloat64
192 + values[i] = &v
193 + default:
194 + var v any
195 + values[i] = &v
196 + }
197 + valuePtrs[i] = values[i]
198 + }
199 +
200 + if err := rows.Scan(valuePtrs...); err != nil {
201 + return nil, fmt.Errorf("row scan failed: %w", err)
202 + }
203 +
204 + row := make([]any, len(cols))
205 + for i, col := range cols {
206 + switch v := values[i].(type) {
207 + case *sql.NullString:
208 + if v.Valid {
209 + s := v.String
210 + if col.Name == "query" {
211 + s = strmutil.TruncateText(s, topQueriesMaxTextLength)
212 + }
213 + row[i] = s
214 + } else {
215 + row[i] = ""
216 + }
217 + case *sql.NullInt64:
218 + if v.Valid {
219 + row[i] = v.Int64
220 + } else {
221 + row[i] = int64(0)
222 + }
223 + case *sql.NullFloat64:
224 + if v.Valid {
225 + row[i] = v.Float64
226 + } else {
227 + row[i] = float64(0)
228 + }
229 + default:
230 + row[i] = nil
231 + }
232 + }
233 +
234 + data = append(data, row)
235 + }
236 +
237 + if err := rows.Err(); err != nil {
238 + return nil, fmt.Errorf("rows iteration error: %w", err)
239 + }
240 +
241 + return data, nil
242 +}
src/go/plugin/go.d/collector/cockroachdb/functions.go deleted
-557
@@ -1,557 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -package cockroachdb
4 -
5 -import (
6 - "context"
7 - "database/sql"
8 - "errors"
9 - "fmt"
10 - "strings"
11 - "time"
12 -
13 - "github.com/netdata/netdata/go/plugins/pkg/funcapi"
14 - "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
15 - "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
16 -)
17 -
18 -const crdbMaxQueryTextLength = 4096
19 -
20 -const (
21 - paramSort = "__sort"
22 -
23 - ftString = funcapi.FieldTypeString
24 - ftInteger = funcapi.FieldTypeInteger
25 - ftFloat = funcapi.FieldTypeFloat
26 - ftDuration = funcapi.FieldTypeDuration
27 -
28 - trNone = funcapi.FieldTransformNone
29 - trNumber = funcapi.FieldTransformNumber
30 - trDuration = funcapi.FieldTransformDuration
31 - trText = funcapi.FieldTransformText
32 -
33 - visValue = funcapi.FieldVisualValue
34 - visBar = funcapi.FieldVisualBar
35 -
36 - sortAsc = funcapi.FieldSortAscending
37 - sortDesc = funcapi.FieldSortDescending
38 -
39 - summaryCount = funcapi.FieldSummaryCount
40 - summarySum = funcapi.FieldSummarySum
41 - summaryMax = funcapi.FieldSummaryMax
42 - summaryMean = funcapi.FieldSummaryMean
43 -
44 - filterMulti = funcapi.FieldFilterMultiselect
45 - filterRange = funcapi.FieldFilterRange
46 -)
47 -
48 -var errSQLDSNNotSet = errors.New("SQL DSN is not set")
49 -
50 -type crdbColumnMeta struct {
51 - id string
52 - name string
53 - selectExpr string
54 - dataType funcapi.FieldType
55 - visible bool
56 - sortable bool
57 - fullWidth bool
58 - wrap bool
59 - sticky bool
60 - filter funcapi.FieldFilter
61 - visualization funcapi.FieldVisual
62 - transform funcapi.FieldTransform
63 - units string
64 - decimalPoints int
65 - uniqueKey bool
66 - sortDir funcapi.FieldSort
67 - summary funcapi.FieldSummary
68 - sortLabel string
69 - isSortOption bool
70 - isDefaultSort bool
71 - isLabel bool
72 - isPrimary bool
73 - isMetric bool
74 - chartGroup string
75 - chartTitle string
76 - isDefaultChart bool
77 -}
78 -
79 -var crdbTopColumns = []crdbColumnMeta{
80 - {id: "fingerprintId", name: "Fingerprint ID", selectExpr: "s.fingerprint_id::STRING", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, uniqueKey: true, sortDir: sortAsc, summary: summaryCount},
81 - {id: "query", name: "Query", selectExpr: "s.metadata->>'query'", dataType: ftString, visible: true, sortable: false, filter: filterMulti, transform: trText, sticky: true, fullWidth: true, wrap: true},
82 - {id: "database", name: "Database", selectExpr: "s.metadata->>'db'", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount, isLabel: true, isPrimary: true},
83 - {id: "application", name: "Application", selectExpr: "s.app_name", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount, isLabel: true},
84 - {id: "statementType", name: "Statement Type", selectExpr: "s.metadata->>'stmtTyp'", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount, isLabel: true},
85 - {id: "distributed", name: "Distributed", selectExpr: "s.metadata->>'distsql'", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
86 - {id: "fullScan", name: "Full Scan", selectExpr: "s.metadata->>'fullScan'", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
87 - {id: "implicitTxn", name: "Implicit Txn", selectExpr: "s.metadata->>'implicitTxn'", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
88 - {id: "vectorized", name: "Vectorized", selectExpr: "s.metadata->>'vec'", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
89 -
90 - {id: "executions", name: "Executions", selectExpr: "COALESCE((s.statistics->'statistics'->>'cnt')::INT8, 0)", dataType: ftInteger, visible: true, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by Executions", isMetric: true, chartGroup: "Calls", chartTitle: "Executions", isDefaultChart: true},
91 - {id: "totalTime", name: "Total Time", selectExpr: "COALESCE((s.statistics->'statistics'->'svcLat'->>'mean')::FLOAT8, 0) * COALESCE((s.statistics->'statistics'->>'cnt')::FLOAT8, 0) * 1000", dataType: ftDuration, visible: true, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summarySum, isSortOption: true, isDefaultSort: true, sortLabel: "Top queries by Total Time", isMetric: true, chartGroup: "Time", chartTitle: "Execution Time", isDefaultChart: true},
92 - {id: "meanTime", name: "Mean Time", selectExpr: "COALESCE((s.statistics->'statistics'->'svcLat'->>'mean')::FLOAT8, 0) * 1000", dataType: ftDuration, visible: true, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMean, isSortOption: true, sortLabel: "Top queries by Mean Time", isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
93 - {id: "runTime", name: "Run Time", selectExpr: "COALESCE((s.statistics->'statistics'->'runLat'->>'mean')::FLOAT8, 0) * 1000", dataType: ftDuration, visible: false, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMean, isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
94 - {id: "planTime", name: "Plan Time", selectExpr: "COALESCE((s.statistics->'statistics'->'planLat'->>'mean')::FLOAT8, 0) * 1000", dataType: ftDuration, visible: false, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMean, isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
95 - {id: "parseTime", name: "Parse Time", selectExpr: "COALESCE((s.statistics->'statistics'->'parseLat'->>'mean')::FLOAT8, 0) * 1000", dataType: ftDuration, visible: false, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMean, isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
96 -
97 - {id: "rowsRead", name: "Rows Read", selectExpr: "CAST(ROUND(COALESCE((s.statistics->'statistics'->'rowsRead'->>'mean')::FLOAT8, 0) * COALESCE((s.statistics->'statistics'->>'cnt')::FLOAT8, 0)) AS INT8)", dataType: ftInteger, visible: true, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by Rows Read", isMetric: true, chartGroup: "Rows", chartTitle: "Rows"},
98 - {id: "rowsWritten", name: "Rows Written", selectExpr: "CAST(ROUND(COALESCE((s.statistics->'statistics'->'rowsWritten'->>'mean')::FLOAT8, 0) * COALESCE((s.statistics->'statistics'->>'cnt')::FLOAT8, 0)) AS INT8)", dataType: ftInteger, visible: true, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by Rows Written", isMetric: true, chartGroup: "Rows", chartTitle: "Rows"},
99 - {id: "rowsReturned", name: "Rows Returned", selectExpr: "CAST(ROUND(COALESCE((s.statistics->'statistics'->'numRows'->>'mean')::FLOAT8, 0) * COALESCE((s.statistics->'statistics'->>'cnt')::FLOAT8, 0)) AS INT8)", dataType: ftInteger, visible: true, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by Rows Returned", isMetric: true, chartGroup: "Rows", chartTitle: "Rows"},
100 - {id: "bytesRead", name: "Bytes Read", selectExpr: "CAST(ROUND(COALESCE((s.statistics->'statistics'->'bytesRead'->>'mean')::FLOAT8, 0) * COALESCE((s.statistics->'statistics'->>'cnt')::FLOAT8, 0)) AS INT8)", dataType: ftInteger, visible: false, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by Bytes Read", isMetric: true, chartGroup: "Bytes", chartTitle: "Bytes"},
101 - {id: "maxRetries", name: "Max Retries", selectExpr: "COALESCE((s.statistics->'statistics'->>'maxRetries')::INT8, 0)", dataType: ftInteger, visible: false, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summaryMax, isMetric: true, chartGroup: "Retries", chartTitle: "Retries"},
102 -}
103 -
104 -var crdbRunningColumns = []crdbColumnMeta{
105 - {id: "queryId", name: "Query ID", selectExpr: "s.query_id::STRING", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, uniqueKey: true, sortDir: sortAsc, summary: summaryCount},
106 - {id: "query", name: "Query", selectExpr: "s.query", dataType: ftString, visible: true, sortable: false, filter: filterMulti, transform: trText, sticky: true, fullWidth: true, wrap: true},
107 - {id: "user", name: "User", selectExpr: "s.user_name", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
108 - {id: "application", name: "Application", selectExpr: "s.application_name", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
109 - {id: "clientAddress", name: "Client Address", selectExpr: "s.client_address", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
110 - {id: "nodeId", name: "Node ID", selectExpr: "s.node_id::STRING", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
111 - {id: "sessionId", name: "Session ID", selectExpr: "s.session_id::STRING", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
112 - {id: "phase", name: "Phase", selectExpr: "s.phase", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
113 - {id: "distributed", name: "Distributed", selectExpr: "s.distributed::STRING", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
114 - {id: "startTime", name: "Start Time", selectExpr: "TO_CHAR(s.start, 'YYYY-MM-DD\"T\"HH24:MI:SS.FF3')", dataType: ftString, visible: false, sortable: true, filter: filterRange, transform: trText, sortDir: sortDesc, summary: summaryMax},
115 - {id: "elapsedMs", name: "Elapsed", selectExpr: "EXTRACT(EPOCH FROM (clock_timestamp() - s.start)) * 1000", dataType: ftDuration, visible: true, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, isSortOption: true, isDefaultSort: true, sortLabel: "Running queries by Elapsed Time"},
116 -}
117 -
118 -func cockroachMethods() []module.MethodConfig {
119 - return []module.MethodConfig{
120 - {
121 - UpdateEvery: 10,
122 - ID: "top-queries",
123 - Name: "Top Queries",
124 - Help: "Top SQL statements from crdb_internal.cluster_statement_statistics. WARNING: Query text may contain unmasked literals (potential PII).",
125 - RequireCloud: true,
126 - RequiredParams: []funcapi.ParamConfig{
127 - buildCrdbSortParam(crdbTopColumns),
128 - },
129 - },
130 - {
131 - UpdateEvery: 10,
132 - ID: "running-queries",
133 - Name: "Running Queries",
134 - Help: "Currently running SQL statements from SHOW CLUSTER STATEMENTS. WARNING: Query text may contain unmasked literals (potential PII).",
135 - RequireCloud: true,
136 - RequiredParams: []funcapi.ParamConfig{
137 - buildCrdbSortParam(crdbRunningColumns),
138 - },
139 - },
140 - }
141 -}
142 -
143 -func cockroachMethodParams(_ context.Context, _ *module.Job, method string) ([]funcapi.ParamConfig, error) {
144 - switch method {
145 - case "top-queries":
146 - return []funcapi.ParamConfig{buildCrdbSortParam(crdbTopColumns)}, nil
147 - case "running-queries":
148 - return []funcapi.ParamConfig{buildCrdbSortParam(crdbRunningColumns)}, nil
149 - default:
150 - return nil, fmt.Errorf("unknown method: %s", method)
151 - }
152 -}
153 -
154 -func cockroachHandleMethod(ctx context.Context, job *module.Job, method string, params funcapi.ResolvedParams) *module.FunctionResponse {
155 - collector, ok := job.Module().(*Collector)
156 - if !ok {
157 - return &module.FunctionResponse{Status: 500, Message: "internal error: invalid module type"}
158 - }
159 -
160 - if err := collector.ensureSQL(ctx); err != nil {
161 - status := 503
162 - if errors.Is(err, errSQLDSNNotSet) {
163 - status = 400
164 - }
165 - return &module.FunctionResponse{Status: status, Message: err.Error()}
166 - }
167 -
168 - switch method {
169 - case "top-queries":
170 - return collector.collectTopQueries(ctx, params.Column(paramSort))
171 - case "running-queries":
172 - return collector.collectRunningQueries(ctx, params.Column(paramSort))
173 - default:
174 - return &module.FunctionResponse{Status: 404, Message: fmt.Sprintf("unknown method: %s", method)}
175 - }
176 -}
177 -
178 -func (c *Collector) ensureSQL(ctx context.Context) error {
179 - if c.db != nil {
180 - return nil
181 - }
182 - if c.DSN == "" {
183 - return errSQLDSNNotSet
184 - }
185 -
186 - db, err := sql.Open("pgx", c.DSN)
187 - if err != nil {
188 - return fmt.Errorf("error opening SQL connection: %w", err)
189 - }
190 - db.SetMaxOpenConns(1)
191 - db.SetMaxIdleConns(1)
192 - db.SetConnMaxLifetime(10 * time.Minute)
193 -
194 - timeout := c.sqlTimeout()
195 - pingCtx, cancel := context.WithTimeout(ctx, timeout)
196 - defer cancel()
197 - if err := db.PingContext(pingCtx); err != nil {
198 - _ = db.Close()
199 - return fmt.Errorf("error pinging SQL connection: %w", err)
200 - }
201 -
202 - setCtx, cancel := context.WithTimeout(ctx, timeout)
203 - if _, err := db.ExecContext(setCtx, "SET allow_unsafe_internals = on"); err != nil {
204 - c.Debugf("unable to set allow_unsafe_internals: %v", err)
205 - }
206 - cancel()
207 -
208 - c.db = db
209 - return nil
210 -}
211 -
212 -func (c *Collector) sqlTimeout() time.Duration {
213 - if c.SQLTimeout.Duration() > 0 {
214 - return c.SQLTimeout.Duration()
215 - }
216 - return time.Second
217 -}
218 -
219 -func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
220 - sortColumn = resolveCrdbSortColumn(crdbTopColumns, sortColumn)
221 - limit := c.TopQueriesLimit
222 - if limit <= 0 {
223 - limit = 500
224 - }
225 -
226 - query := buildCrdbTopQueriesSQL(sortColumn, limit)
227 - queryCtx, cancel := context.WithTimeout(ctx, c.sqlTimeout())
228 - defer cancel()
229 - rows, err := c.db.QueryContext(queryCtx, query, limit)
230 - if err != nil {
231 - if errors.Is(queryCtx.Err(), context.DeadlineExceeded) {
232 - return &module.FunctionResponse{Status: 504, Message: "query timed out"}
233 - }
234 - return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("query failed: %v", err)}
235 - }
236 - defer rows.Close()
237 -
238 - data, err := scanCrdbRows(rows, crdbTopColumns)
239 - if err != nil {
240 - return &module.FunctionResponse{Status: 500, Message: err.Error()}
241 - }
242 -
243 - return &module.FunctionResponse{
244 - Status: 200,
245 - Help: "Top SQL statements from crdb_internal.cluster_statement_statistics. WARNING: Query text may contain unmasked literals (potential PII).",
246 - Columns: buildCrdbColumns(crdbTopColumns),
247 - Data: data,
248 - DefaultSortColumn: sortColumn,
249 - RequiredParams: []funcapi.ParamConfig{buildCrdbSortParam(crdbTopColumns)},
250 - Charts: crdbTopQueriesCharts(crdbTopColumns),
251 - DefaultCharts: crdbTopQueriesDefaultCharts(crdbTopColumns),
252 - GroupBy: crdbTopQueriesGroupBy(crdbTopColumns),
253 - }
254 -}
255 -
256 -func (c *Collector) collectRunningQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
257 - sortColumn = resolveCrdbSortColumn(crdbRunningColumns, sortColumn)
258 - limit := c.TopQueriesLimit
259 - if limit <= 0 {
260 - limit = 500
261 - }
262 -
263 - query := buildCrdbRunningQueriesSQL(sortColumn, limit)
264 - queryCtx, cancel := context.WithTimeout(ctx, c.sqlTimeout())
265 - defer cancel()
266 - rows, err := c.db.QueryContext(queryCtx, query, limit)
267 - if err != nil {
268 - if errors.Is(queryCtx.Err(), context.DeadlineExceeded) {
269 - return &module.FunctionResponse{Status: 504, Message: "query timed out"}
270 - }
271 - return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("query failed: %v", err)}
272 - }
273 - defer rows.Close()
274 -
275 - data, err := scanCrdbRows(rows, crdbRunningColumns)
276 - if err != nil {
277 - return &module.FunctionResponse{Status: 500, Message: err.Error()}
278 - }
279 -
280 - return &module.FunctionResponse{
281 - Status: 200,
282 - Help: "Currently running SQL statements from SHOW CLUSTER STATEMENTS. WARNING: Query text may contain unmasked literals (potential PII).",
283 - Columns: buildCrdbColumns(crdbRunningColumns),
284 - Data: data,
285 - DefaultSortColumn: sortColumn,
286 - RequiredParams: []funcapi.ParamConfig{buildCrdbSortParam(crdbRunningColumns)},
287 - }
288 -}
289 -
290 -func buildCrdbSortParam(cols []crdbColumnMeta) funcapi.ParamConfig {
291 - return funcapi.ParamConfig{
292 - ID: paramSort,
293 - Name: "Filter By",
294 - Help: "Select the primary sort column",
295 - Selection: funcapi.ParamSelect,
296 - Options: buildCrdbSortOptions(cols),
297 - UniqueView: true,
298 - }
299 -}
300 -
301 -func buildCrdbSortOptions(cols []crdbColumnMeta) []funcapi.ParamOption {
302 - var sortOptions []funcapi.ParamOption
303 - sortDir := funcapi.FieldSortDescending
304 - for _, col := range cols {
305 - if !col.isSortOption {
306 - continue
307 - }
308 - opt := funcapi.ParamOption{
309 - ID: col.id,
310 - Column: col.id,
311 - Name: col.sortLabel,
312 - Sort: &sortDir,
313 - }
314 - if col.isDefaultSort {
315 - opt.Default = true
316 - }
317 - sortOptions = append(sortOptions, opt)
318 - }
319 - return sortOptions
320 -}
321 -
322 -func buildCrdbColumns(cols []crdbColumnMeta) map[string]any {
323 - result := make(map[string]any, len(cols))
324 - for i, col := range cols {
325 - visual := visValue
326 - if col.dataType == ftDuration {
327 - visual = visBar
328 - }
329 - colDef := funcapi.Column{
330 - Index: i,
331 - Name: col.name,
332 - Type: col.dataType,
333 - Units: col.units,
334 - Visualization: visual,
335 - Sort: col.sortDir,
336 - Sortable: col.sortable,
337 - Sticky: col.sticky,
338 - Summary: col.summary,
339 - Filter: col.filter,
340 - FullWidth: col.fullWidth,
341 - Wrap: col.wrap,
342 - DefaultExpandedFilter: false,
343 - UniqueKey: col.uniqueKey,
344 - Visible: col.visible,
345 - ValueOptions: funcapi.ValueOptions{
346 - Transform: col.transform,
347 - DecimalPoints: col.decimalPoints,
348 - DefaultValue: nil,
349 - },
350 - }
351 - result[col.id] = colDef.BuildColumn()
352 - }
353 - return result
354 -}
355 -
356 -func resolveCrdbSortColumn(cols []crdbColumnMeta, requested string) string {
357 - if requested != "" {
358 - for _, col := range cols {
359 - if col.id == requested && col.isSortOption {
360 - return col.id
361 - }
362 - }
363 - }
364 - for _, col := range cols {
365 - if col.isDefaultSort && col.isSortOption {
366 - return col.id
367 - }
368 - }
369 - for _, col := range cols {
370 - if col.isSortOption {
371 - return col.id
372 - }
373 - }
374 - return ""
375 -}
376 -
377 -func buildCrdbTopQueriesSQL(sortColumn string, limit int) string {
378 - selectCols := make([]string, 0, len(crdbTopColumns))
379 - for _, col := range crdbTopColumns {
380 - selectCols = append(selectCols, fmt.Sprintf("%s AS %s", col.selectExpr, col.id))
381 - }
382 - return fmt.Sprintf(`
383 -SELECT %s
384 -FROM crdb_internal.cluster_statement_statistics AS s
385 -WHERE s.metadata->>'query' IS NOT NULL
386 -ORDER BY %s DESC NULLS LAST
387 -LIMIT $1`, strings.Join(selectCols, ", "), sortColumn)
388 -}
389 -
390 -func buildCrdbRunningQueriesSQL(sortColumn string, limit int) string {
391 - selectCols := make([]string, 0, len(crdbRunningColumns))
392 - for _, col := range crdbRunningColumns {
393 - selectCols = append(selectCols, fmt.Sprintf("%s AS %s", col.selectExpr, col.id))
394 - }
395 - return fmt.Sprintf(`
396 -SELECT %s
397 -FROM [SHOW CLUSTER STATEMENTS] AS s
398 -ORDER BY %s DESC NULLS LAST
399 -LIMIT $1`, strings.Join(selectCols, ", "), sortColumn)
400 -}
401 -
402 -func scanCrdbRows(rows *sql.Rows, cols []crdbColumnMeta) ([][]any, error) {
403 - data := make([][]any, 0, 500)
404 -
405 - for rows.Next() {
406 - values := make([]any, len(cols))
407 - valuePtrs := make([]any, len(cols))
408 -
409 - for i, col := range cols {
410 - switch col.dataType {
411 - case ftString:
412 - var v sql.NullString
413 - values[i] = &v
414 - case ftInteger:
415 - var v sql.NullInt64
416 - values[i] = &v
417 - case ftFloat, ftDuration:
418 - var v sql.NullFloat64
419 - values[i] = &v
420 - default:
421 - var v any
422 - values[i] = &v
423 - }
424 - valuePtrs[i] = values[i]
425 - }
426 -
427 - if err := rows.Scan(valuePtrs...); err != nil {
428 - return nil, fmt.Errorf("row scan failed: %w", err)
429 - }
430 -
431 - row := make([]any, len(cols))
432 - for i, col := range cols {
433 - switch v := values[i].(type) {
434 - case *sql.NullString:
435 - if v.Valid {
436 - s := v.String
437 - if col.id == "query" {
438 - s = strmutil.TruncateText(s, crdbMaxQueryTextLength)
439 - }
440 - row[i] = s
441 - } else {
442 - row[i] = ""
443 - }
444 - case *sql.NullInt64:
445 - if v.Valid {
446 - row[i] = v.Int64
447 - } else {
448 - row[i] = int64(0)
449 - }
450 - case *sql.NullFloat64:
451 - if v.Valid {
452 - row[i] = v.Float64
453 - } else {
454 - row[i] = float64(0)
455 - }
456 - default:
457 - row[i] = nil
458 - }
459 - }
460 -
461 - data = append(data, row)
462 - }
463 -
464 - if err := rows.Err(); err != nil {
465 - return nil, fmt.Errorf("rows iteration error: %w", err)
466 - }
467 -
468 - return data, nil
469 -}
470 -
471 -func crdbTopQueriesCharts(cols []crdbColumnMeta) map[string]module.ChartConfig {
472 - charts := make(map[string]module.ChartConfig)
473 - for _, col := range cols {
474 - if !col.isMetric || col.chartGroup == "" {
475 - continue
476 - }
477 - cfg, ok := charts[col.chartGroup]
478 - if !ok {
479 - title := col.chartTitle
480 - if title == "" {
481 - title = col.chartGroup
482 - }
483 - cfg = module.ChartConfig{Name: title, Type: "stacked-bar"}
484 - }
485 - cfg.Columns = append(cfg.Columns, col.id)
486 - charts[col.chartGroup] = cfg
487 - }
488 - return charts
489 -}
490 -
491 -func crdbTopQueriesDefaultCharts(cols []crdbColumnMeta) [][]string {
492 - label := primaryCrdbLabel(cols)
493 - if label == "" {
494 - return nil
495 - }
496 - chartGroups := defaultCrdbChartGroups(cols)
497 - out := make([][]string, 0, len(chartGroups))
498 - for _, group := range chartGroups {
499 - out = append(out, []string{group, label})
500 - }
501 - return out
502 -}
503 -
504 -func crdbTopQueriesGroupBy(cols []crdbColumnMeta) map[string]module.GroupByConfig {
505 - groupBy := make(map[string]module.GroupByConfig)
506 - for _, col := range cols {
507 - if !col.isLabel {
508 - continue
509 - }
510 - groupBy[col.id] = module.GroupByConfig{
511 - Name: "Group by " + col.name,
512 - Columns: []string{col.id},
513 - }
514 - }
515 - return groupBy
516 -}
517 -
518 -func primaryCrdbLabel(cols []crdbColumnMeta) string {
519 - for _, col := range cols {
520 - if col.isPrimary {
521 - return col.id
522 - }
523 - }
524 - for _, col := range cols {
525 - if col.isLabel {
526 - return col.id
527 - }
528 - }
529 - return ""
530 -}
531 -
532 -func defaultCrdbChartGroups(cols []crdbColumnMeta) []string {
533 - groups := make([]string, 0)
534 - seen := make(map[string]bool)
535 - for _, col := range cols {
536 - if !col.isMetric || col.chartGroup == "" || !col.isDefaultChart {
537 - continue
538 - }
539 - if !seen[col.chartGroup] {
540 - seen[col.chartGroup] = true
541 - groups = append(groups, col.chartGroup)
542 - }
543 - }
544 - if len(groups) > 0 {
545 - return groups
546 - }
547 - for _, col := range cols {
548 - if !col.isMetric || col.chartGroup == "" {
549 - continue
550 - }
551 - if !seen[col.chartGroup] {
552 - seen[col.chartGroup] = true
553 - groups = append(groups, col.chartGroup)
554 - }
555 - }
556 - return groups
557 -}
src/go/plugin/go.d/collector/couchbase/collector.go
+12 -6
@@ -24,11 +24,10 @@ func init() {
24 Defaults: module.Defaults{
25 UpdateEvery: 5,
26 },
27 - Create: func() module.Module { return New() },
28 - Config: func() any { return &Config{} },
29 - Methods: couchbaseMethods,
30 - MethodParams: couchbaseMethodParams,
31 - HandleMethod: couchbaseHandleMethod,
27 + Create: func() module.Module { return New() },
28 + Config: func() any { return &Config{} },
29 + Methods: couchbaseMethods,
30 + MethodHandler: couchbaseFunctionHandler,
31 })
32 }
33
@@ -65,6 +64,8 @@ type Collector struct {
64 charts *module.Charts
65
66 collectedBuckets map[string]bool
67 +
68 + funcRouter *funcRouter
69 }
70
71 func (c *Collector) Configuration() any {
@@ -89,6 +90,8 @@ func (c *Collector) Init(context.Context) error {
90 }
91 c.charts = charts
92
93 + c.funcRouter = newFuncRouter(c)
94 +
95 return nil
96 }
97
@@ -120,7 +123,10 @@ func (c *Collector) Collect(context.Context) map[string]int64 {
123 return mx
124 }
125
123 -func (c *Collector) Cleanup(context.Context) {
126 +func (c *Collector) Cleanup(ctx context.Context) {
127 + if c.funcRouter != nil {
128 + c.funcRouter.Cleanup(ctx)
129 + }
130 if c.httpClient == nil {
131 return
132 }
src/go/plugin/go.d/collector/couchbase/func_router.go new
+63
@@ -0,0 +1,63 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package couchbase
4 +
5 +import (
6 + "context"
7 + "fmt"
8 +
9 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
10 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
11 +)
12 +
13 +// funcRouter routes method calls to appropriate function handlers.
14 +type funcRouter struct {
15 + collector *Collector
16 + handlers map[string]funcapi.MethodHandler
17 +}
18 +
19 +func newFuncRouter(c *Collector) *funcRouter {
20 + r := &funcRouter{
21 + collector: c,
22 + handlers: make(map[string]funcapi.MethodHandler),
23 + }
24 + r.handlers[topQueriesMethodID] = newFuncTopQueries(r)
25 + return r
26 +}
27 +
28 +// Compile-time interface check.
29 +var _ funcapi.MethodHandler = (*funcRouter)(nil)
30 +
31 +func (r *funcRouter) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
32 + if h, ok := r.handlers[method]; ok {
33 + return h.MethodParams(ctx, method)
34 + }
35 + return nil, fmt.Errorf("unknown method: %s", method)
36 +}
37 +
38 +func (r *funcRouter) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
39 + if h, ok := r.handlers[method]; ok {
40 + return h.Handle(ctx, method, params)
41 + }
42 + return funcapi.NotFoundResponse(method)
43 +}
44 +
45 +func (r *funcRouter) Cleanup(ctx context.Context) {
46 + for _, h := range r.handlers {
47 + h.Cleanup(ctx)
48 + }
49 +}
50 +
51 +func couchbaseMethods() []funcapi.MethodConfig {
52 + return []funcapi.MethodConfig{
53 + topQueriesMethodConfig(),
54 + }
55 +}
56 +
57 +func couchbaseFunctionHandler(job *module.Job) funcapi.MethodHandler {
58 + c, ok := job.Module().(*Collector)
59 + if !ok {
60 + return nil
61 + }
62 + return c.funcRouter
63 +}
src/go/plugin/go.d/collector/couchbase/func_top_queries.go new
+386
@@ -0,0 +1,386 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package couchbase
4 +
5 +import (
6 + "context"
7 + "encoding/json"
8 + "fmt"
9 + "net"
10 + "net/http"
11 + "net/url"
12 + "path"
13 + "sort"
14 + "strings"
15 + "time"
16 +
17 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
18 + "github.com/netdata/netdata/go/plugins/pkg/web"
19 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
20 +)
21 +
22 +const (
23 + topQueriesMethodID = "top-queries"
24 + topQueriesMaxTextLength = 4096
25 +)
26 +
27 +func topQueriesMethodConfig() funcapi.MethodConfig {
28 + return funcapi.MethodConfig{
29 + ID: topQueriesMethodID,
30 + Name: "Top Queries",
31 + UpdateEvery: 10,
32 + Help: "Top N1QL requests from system:completed_requests",
33 + RequireCloud: true,
34 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(topQueriesColumns)},
35 + }
36 +}
37 +
38 +type topQueriesColumn struct {
39 + funcapi.ColumnMeta
40 + sortOpt bool // whether this column appears as a sort option
41 + sortLbl string // label for sort option dropdown
42 + defaultSort bool // default sort column
43 +}
44 +
45 +// funcapi.SortableColumn interface implementation for topQueriesColumn.
46 +func (c topQueriesColumn) IsSortOption() bool { return c.sortOpt }
47 +func (c topQueriesColumn) SortLabel() string { return c.sortLbl }
48 +func (c topQueriesColumn) IsDefaultSort() bool { return c.defaultSort }
49 +func (c topQueriesColumn) ColumnName() string { return c.Name }
50 +func (c topQueriesColumn) SortColumn() string { return "" }
51 +
52 +var topQueriesColumns = []topQueriesColumn{
53 + {ColumnMeta: funcapi.ColumnMeta{Name: "requestId", Tooltip: "Request ID", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, UniqueKey: true, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryCount}, sortOpt: true, sortLbl: "Top queries by Request ID"},
54 + {ColumnMeta: funcapi.ColumnMeta{Name: "requestTime", Tooltip: "Request Time", Type: funcapi.FieldTypeTimestamp, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformDatetime, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax}, sortOpt: true, sortLbl: "Top queries by Request Time"},
55 + {ColumnMeta: funcapi.ColumnMeta{Name: "statement", Tooltip: "Statement", Type: funcapi.FieldTypeString, Visible: true, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, Sticky: true, FullWidth: true, Wrap: true}},
56 + {ColumnMeta: funcapi.ColumnMeta{Name: "elapsedTime", Tooltip: "Elapsed Time", Type: funcapi.FieldTypeDuration, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualBar, Transform: funcapi.FieldTransformDuration, Units: "milliseconds", DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Elapsed & Service Time", IsDefault: true}}, sortOpt: true, defaultSort: true, sortLbl: "Top queries by Elapsed Time"},
57 + {ColumnMeta: funcapi.ColumnMeta{Name: "serviceTime", Tooltip: "Service Time", Type: funcapi.FieldTypeDuration, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualBar, Transform: funcapi.FieldTransformDuration, Units: "milliseconds", DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Elapsed & Service Time"}}, sortOpt: true, sortLbl: "Top queries by Service Time"},
58 + {ColumnMeta: funcapi.ColumnMeta{Name: "resultCount", Tooltip: "Result Count", Type: funcapi.FieldTypeInteger, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "Results", Title: "Results"}}, sortOpt: true, sortLbl: "Top queries by Result Count"},
59 + {ColumnMeta: funcapi.ColumnMeta{Name: "resultSize", Tooltip: "Result Size", Type: funcapi.FieldTypeInteger, Visible: false, Sortable: false, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "ResultSize", Title: "Result Size"}}},
60 + {ColumnMeta: funcapi.ColumnMeta{Name: "errorCount", Tooltip: "Error Count", Type: funcapi.FieldTypeInteger, Visible: false, Sortable: false, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "Errors", Title: "Errors & Warnings"}}},
61 + {ColumnMeta: funcapi.ColumnMeta{Name: "warningCount", Tooltip: "Warning Count", Type: funcapi.FieldTypeInteger, Visible: false, Sortable: false, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "Errors", Title: "Errors & Warnings"}}},
62 + {ColumnMeta: funcapi.ColumnMeta{Name: "user", Tooltip: "User", Type: funcapi.FieldTypeString, Visible: true, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, GroupBy: &funcapi.GroupByOptions{IsDefault: true}}},
63 + {ColumnMeta: funcapi.ColumnMeta{Name: "clientContextID", Tooltip: "Client Context ID", Type: funcapi.FieldTypeString, Visible: false, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText}},
64 +}
65 +
66 +type topQueriesResponse struct {
67 + Status string `json:"status"`
68 + Results []topQueriesRequestData `json:"results"`
69 + Errors []topQueriesError `json:"errors"`
70 +}
71 +
72 +type topQueriesError struct {
73 + Message string `json:"msg"`
74 +}
75 +
76 +type topQueriesRequestData struct {
77 + RequestID string `json:"requestId"`
78 + RequestTime string `json:"requestTime"`
79 + Statement string `json:"statement"`
80 + ElapsedTime string `json:"elapsedTime"`
81 + ServiceTime string `json:"serviceTime"`
82 + ResultCount json.Number `json:"resultCount"`
83 + ResultSize json.Number `json:"resultSize"`
84 + ErrorCount json.Number `json:"errorCount"`
85 + WarningCount json.Number `json:"warningCount"`
86 + User string `json:"user"`
87 + ClientContextID string `json:"clientContextID"`
88 +}
89 +
90 +type topQueriesRow struct {
91 + RequestID string
92 + RequestTime time.Time
93 + RequestTimeRaw string
94 + Statement string
95 + ElapsedMs float64
96 + ServiceMs float64
97 + ResultCount int64
98 + ResultSize int64
99 + ErrorCount int64
100 + WarningCount int64
101 + User string
102 + ClientContextID string
103 +}
104 +
105 +// funcTopQueries implements funcapi.MethodHandler for Couchbase top-queries.
106 +type funcTopQueries struct {
107 + router *funcRouter
108 +}
109 +
110 +func newFuncTopQueries(r *funcRouter) *funcTopQueries {
111 + return &funcTopQueries{router: r}
112 +}
113 +
114 +// Compile-time interface check.
115 +var _ funcapi.MethodHandler = (*funcTopQueries)(nil)
116 +
117 +// MethodParams implements funcapi.MethodHandler.
118 +func (f *funcTopQueries) MethodParams(_ context.Context, method string) ([]funcapi.ParamConfig, error) {
119 + switch method {
120 + case topQueriesMethodID:
121 + return []funcapi.ParamConfig{funcapi.BuildSortParam(topQueriesColumns)}, nil
122 + default:
123 + return nil, fmt.Errorf("unknown method: %s", method)
124 + }
125 +}
126 +
127 +// Handle implements funcapi.MethodHandler.
128 +func (f *funcTopQueries) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
129 + if f.router.collector.httpClient == nil {
130 + return funcapi.UnavailableResponse("collector is still initializing, please retry in a few seconds")
131 + }
132 +
133 + switch method {
134 + case topQueriesMethodID:
135 + return f.collectData(ctx, params.Column("__sort"))
136 + default:
137 + return funcapi.NotFoundResponse(method)
138 + }
139 +}
140 +
141 +// Cleanup implements funcapi.MethodHandler.
142 +func (f *funcTopQueries) Cleanup(ctx context.Context) {}
143 +
144 +func (f *funcTopQueries) collectData(ctx context.Context, sortColumn string) *funcapi.FunctionResponse {
145 + limit := f.router.collector.TopQueriesLimit
146 + if limit <= 0 {
147 + limit = 500
148 + }
149 +
150 + statement := "SELECT cr.requestId, cr.requestTime, cr.statement, cr.elapsedTime, cr.serviceTime, " +
151 + "cr.resultCount, cr.resultSize, cr.errorCount, cr.warningCount, cr.users AS `user`, cr.clientContextID " +
152 + "FROM system:completed_requests AS cr"
153 +
154 + req, err := f.buildQueryRequest(ctx, statement)
155 + if err != nil {
156 + return &funcapi.FunctionResponse{Status: 500, Message: err.Error()}
157 + }
158 +
159 + var resp topQueriesResponse
160 + if err := web.DoHTTP(f.router.collector.httpClient).RequestJSON(req, &resp); err != nil {
161 + if ctx.Err() == context.DeadlineExceeded {
162 + return &funcapi.FunctionResponse{Status: 504, Message: "query timed out"}
163 + }
164 + return &funcapi.FunctionResponse{Status: 500, Message: fmt.Sprintf("query failed: %v", err)}
165 + }
166 +
167 + if strings.ToLower(resp.Status) != "success" {
168 + msg := "query failed"
169 + if len(resp.Errors) > 0 && resp.Errors[0].Message != "" {
170 + msg = resp.Errors[0].Message
171 + }
172 + return &funcapi.FunctionResponse{Status: 500, Message: msg}
173 + }
174 +
175 + rows := make([]topQueriesRow, 0, len(resp.Results))
176 + for _, r := range resp.Results {
177 + rows = append(rows, f.buildRow(r))
178 + }
179 +
180 + cs := f.columnSet(topQueriesColumns)
181 + sortParam := funcapi.BuildSortParam(topQueriesColumns)
182 +
183 + if len(rows) == 0 {
184 + return &funcapi.FunctionResponse{
185 + Status: 200,
186 + Message: "No completed requests found.",
187 + Help: "Top N1QL requests from system:completed_requests",
188 + Columns: cs.BuildColumns(),
189 + Data: [][]any{},
190 + DefaultSortColumn: "elapsedTime",
191 + RequiredParams: []funcapi.ParamConfig{sortParam},
192 + ChartingConfig: cs.BuildCharting(),
193 + }
194 + }
195 +
196 + sortColumn = f.mapSortColumn(sortColumn)
197 + f.sortRows(rows, sortColumn)
198 +
199 + if len(rows) > limit {
200 + rows = rows[:limit]
201 + }
202 +
203 + data := make([][]any, 0, len(rows))
204 + for _, row := range rows {
205 + out := make([]any, len(topQueriesColumns))
206 + for i, col := range topQueriesColumns {
207 + switch col.Name {
208 + case "requestId":
209 + out[i] = row.RequestID
210 + case "requestTime":
211 + if row.RequestTime.IsZero() {
212 + out[i] = row.RequestTimeRaw
213 + } else {
214 + out[i] = row.RequestTime.Format(time.RFC3339Nano)
215 + }
216 + case "statement":
217 + out[i] = strmutil.TruncateText(row.Statement, topQueriesMaxTextLength)
218 + case "elapsedTime":
219 + out[i] = row.ElapsedMs
220 + case "serviceTime":
221 + out[i] = row.ServiceMs
222 + case "resultCount":
223 + out[i] = row.ResultCount
224 + case "resultSize":
225 + out[i] = row.ResultSize
226 + case "errorCount":
227 + out[i] = row.ErrorCount
228 + case "warningCount":
229 + out[i] = row.WarningCount
230 + case "user":
231 + out[i] = row.User
232 + case "clientContextID":
233 + out[i] = row.ClientContextID
234 + default:
235 + out[i] = nil
236 + }
237 + }
238 + data = append(data, out)
239 + }
240 +
241 + return &funcapi.FunctionResponse{
242 + Status: 200,
243 + Help: "Top N1QL requests from system:completed_requests",
244 + Columns: cs.BuildColumns(),
245 + Data: data,
246 + DefaultSortColumn: "elapsedTime",
247 + RequiredParams: []funcapi.ParamConfig{sortParam},
248 + ChartingConfig: cs.BuildCharting(),
249 + }
250 +}
251 +
252 +func (f *funcTopQueries) columnSet(cols []topQueriesColumn) funcapi.ColumnSet[topQueriesColumn] {
253 + return funcapi.Columns(cols, func(c topQueriesColumn) funcapi.ColumnMeta { return c.ColumnMeta })
254 +}
255 +
256 +func (f *funcTopQueries) queryServiceURL() (string, error) {
257 + if f.router.collector.QueryURL != "" {
258 + return f.router.collector.QueryURL, nil
259 + }
260 + parsed, err := url.Parse(f.router.collector.URL)
261 + if err != nil {
262 + return "", err
263 + }
264 + host := parsed.Hostname()
265 + port := parsed.Port()
266 + if port == "" || port == "8091" {
267 + port = "8093"
268 + }
269 + if port != "" {
270 + parsed.Host = net.JoinHostPort(host, port)
271 + } else {
272 + parsed.Host = host
273 + }
274 + parsed.Path = ""
275 + return parsed.String(), nil
276 +}
277 +
278 +func (f *funcTopQueries) buildQueryRequest(ctx context.Context, statement string) (*http.Request, error) {
279 + queryURL, err := f.queryServiceURL()
280 + if err != nil {
281 + return nil, err
282 + }
283 +
284 + u, err := url.Parse(queryURL)
285 + if err != nil {
286 + return nil, err
287 + }
288 + u.Path = path.Join(u.Path, "/query/service")
289 +
290 + reqCfg := f.router.collector.RequestConfig
291 + reqCfg.URL = u.String()
292 + reqCfg.Method = http.MethodPost
293 + reqCfg.Body = url.Values{"statement": {statement}}.Encode()
294 + if reqCfg.Headers == nil {
295 + reqCfg.Headers = map[string]string{}
296 + }
297 + reqCfg.Headers["Content-Type"] = "application/x-www-form-urlencoded"
298 +
299 + req, err := web.NewHTTPRequest(reqCfg)
300 + if err != nil {
301 + return nil, err
302 + }
303 + return req.WithContext(ctx), nil
304 +}
305 +
306 +func (f *funcTopQueries) buildRow(r topQueriesRequestData) topQueriesRow {
307 + row := topQueriesRow{
308 + RequestID: r.RequestID,
309 + RequestTimeRaw: r.RequestTime,
310 + Statement: r.Statement,
311 + User: r.User,
312 + ClientContextID: r.ClientContextID,
313 + }
314 +
315 + if t, err := time.Parse(time.RFC3339Nano, r.RequestTime); err == nil {
316 + row.RequestTime = t
317 + } else if t, err := time.Parse(time.RFC3339, r.RequestTime); err == nil {
318 + row.RequestTime = t
319 + }
320 +
321 + row.ElapsedMs = f.parseDurationMs(r.ElapsedTime)
322 + row.ServiceMs = f.parseDurationMs(r.ServiceTime)
323 + row.ResultCount = f.parseNumber(r.ResultCount)
324 + row.ResultSize = f.parseNumber(r.ResultSize)
325 + row.ErrorCount = f.parseNumber(r.ErrorCount)
326 + row.WarningCount = f.parseNumber(r.WarningCount)
327 +
328 + return row
329 +}
330 +
331 +func (f *funcTopQueries) parseDurationMs(raw string) float64 {
332 + if raw == "" {
333 + return 0
334 + }
335 + if d, err := time.ParseDuration(raw); err == nil {
336 + return float64(d) / float64(time.Millisecond)
337 + }
338 + return 0
339 +}
340 +
341 +func (f *funcTopQueries) parseNumber(n json.Number) int64 {
342 + if n == "" {
343 + return 0
344 + }
345 + if i, err := n.Int64(); err == nil {
346 + return i
347 + }
348 + if fv, err := n.Float64(); err == nil {
349 + return int64(fv)
350 + }
351 + return 0
352 +}
353 +
354 +func (f *funcTopQueries) mapSortColumn(col string) string {
355 + switch col {
356 + case "elapsedTime", "serviceTime", "requestTime", "resultCount", "requestId":
357 + return col
358 + default:
359 + return "elapsedTime"
360 + }
361 +}
362 +
363 +func (f *funcTopQueries) sortRows(rows []topQueriesRow, sortColumn string) {
364 + switch sortColumn {
365 + case "serviceTime":
366 + sort.Slice(rows, func(i, j int) bool {
367 + return rows[i].ServiceMs > rows[j].ServiceMs
368 + })
369 + case "requestTime":
370 + sort.Slice(rows, func(i, j int) bool {
371 + return rows[i].RequestTime.After(rows[j].RequestTime)
372 + })
373 + case "resultCount":
374 + sort.Slice(rows, func(i, j int) bool {
375 + return rows[i].ResultCount > rows[j].ResultCount
376 + })
377 + case "requestId":
378 + sort.Slice(rows, func(i, j int) bool {
379 + return rows[i].RequestID > rows[j].RequestID
380 + })
381 + default:
382 + sort.Slice(rows, func(i, j int) bool {
383 + return rows[i].ElapsedMs > rows[j].ElapsedMs
384 + })
385 + }
386 +}
src/go/plugin/go.d/collector/couchbase/func_top_queries_test.go renamed
+8 -10
@@ -29,20 +29,17 @@ func TestCouchbaseMethods(t *testing.T) {
29 require.NotEmpty(sortParam.Options)
30 }
31
32 -func TestCouchbaseAllColumns_HasRequiredColumns(t *testing.T) {
32 +func TestTopQueriesColumns_HasRequiredColumns(t *testing.T) {
33 required := []string{"requestId", "statement", "elapsedTime"}
34
35 - uiKeys := make(map[string]bool)
36 - for _, col := range couchbaseAllColumns {
37 - uiKeys[col.id] = true
38 - }
39 -
40 - for _, key := range required {
41 - assert.True(t, uiKeys[key], "column %s should be defined", key)
35 + f := &funcTopQueries{}
36 + cs := f.columnSet(topQueriesColumns)
37 + for _, id := range required {
38 + assert.True(t, cs.ContainsColumn(id), "column %s should be defined", id)
39 }
40 }
41
45 -func TestMapCouchbaseSortColumn(t *testing.T) {
42 +func TestFuncTopQueries_MapSortColumn(t *testing.T) {
43 tests := map[string]struct {
44 input string
45 expected string
@@ -54,9 +51,10 @@ func TestMapCouchbaseSortColumn(t *testing.T) {
51 "invalid": {input: "bad", expected: "elapsedTime"},
52 }
53
54 + f := &funcTopQueries{}
55 for name, tc := range tests {
56 t.Run(name, func(t *testing.T) {
59 - assert.Equal(t, tc.expected, mapCouchbaseSortColumn(tc.input))
57 + assert.Equal(t, tc.expected, f.mapSortColumn(tc.input))
58 })
59 }
60 }
src/go/plugin/go.d/collector/couchbase/functions.go deleted
-567
@@ -1,567 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -package couchbase
4 -
5 -import (
6 - "context"
7 - "encoding/json"
8 - "fmt"
9 - "net"
10 - "net/http"
11 - "net/url"
12 - "path"
13 - "sort"
14 - "strings"
15 - "time"
16 -
17 - "github.com/netdata/netdata/go/plugins/pkg/funcapi"
18 - "github.com/netdata/netdata/go/plugins/pkg/web"
19 - "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
20 - "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
21 -)
22 -
23 -const couchbaseMaxQueryTextLength = 4096
24 -
25 -const (
26 - paramSort = "__sort"
27 -
28 - ftString = funcapi.FieldTypeString
29 - ftInteger = funcapi.FieldTypeInteger
30 - ftDuration = funcapi.FieldTypeDuration
31 - ftTimestamp = funcapi.FieldTypeTimestamp
32 -
33 - trNone = funcapi.FieldTransformNone
34 - trNumber = funcapi.FieldTransformNumber
35 - trDuration = funcapi.FieldTransformDuration
36 - trDatetime = funcapi.FieldTransformDatetime
37 - trText = funcapi.FieldTransformText
38 -
39 - visValue = funcapi.FieldVisualValue
40 - visBar = funcapi.FieldVisualBar
41 -
42 - sortAsc = funcapi.FieldSortAscending
43 - sortDesc = funcapi.FieldSortDescending
44 -
45 - summaryCount = funcapi.FieldSummaryCount
46 - summaryMax = funcapi.FieldSummaryMax
47 - summarySum = funcapi.FieldSummarySum
48 -
49 - filterMulti = funcapi.FieldFilterMultiselect
50 - filterRange = funcapi.FieldFilterRange
51 -)
52 -
53 -type couchbaseColumnMeta struct {
54 - id string
55 - name string
56 - colType funcapi.FieldType
57 - visible bool
58 - sortable bool
59 - fullWidth bool
60 - wrap bool
61 - sticky bool
62 - filter funcapi.FieldFilter
63 - visualization funcapi.FieldVisual
64 - transform funcapi.FieldTransform
65 - units string
66 - decimalPoints int
67 - uniqueKey bool
68 - sortDir funcapi.FieldSort
69 - summary funcapi.FieldSummary
70 - isLabel bool
71 - isPrimary bool
72 - isMetric bool
73 - chartGroup string
74 - chartTitle string
75 - isDefaultChart bool
76 -}
77 -
78 -var couchbaseAllColumns = []couchbaseColumnMeta{
79 - {id: "requestId", name: "Request ID", colType: ftString, visible: false, sortable: true, filter: filterMulti, visualization: visValue, transform: trText, uniqueKey: true, sortDir: sortDesc, summary: summaryCount},
80 - {id: "requestTime", name: "Request Time", colType: ftTimestamp, visible: true, sortable: true, filter: filterRange, visualization: visValue, transform: trDatetime, sortDir: sortDesc, summary: summaryMax},
81 - {id: "statement", name: "Statement", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, sticky: true, fullWidth: true, wrap: true},
82 - {id: "elapsedTime", name: "Elapsed Time", colType: ftDuration, visible: true, sortable: true, filter: filterRange, visualization: visBar, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summarySum, isMetric: true, chartGroup: "Time", chartTitle: "Elapsed & Service Time", isDefaultChart: true},
83 - {id: "serviceTime", name: "Service Time", colType: ftDuration, visible: true, sortable: true, filter: filterRange, visualization: visBar, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summarySum, isMetric: true, chartGroup: "Time", chartTitle: "Elapsed & Service Time"},
84 - {id: "resultCount", name: "Result Count", colType: ftInteger, visible: true, sortable: true, filter: filterRange, visualization: visValue, transform: trNumber, sortDir: sortDesc, summary: summarySum, isMetric: true, chartGroup: "Results", chartTitle: "Results"},
85 - {id: "resultSize", name: "Result Size", colType: ftInteger, visible: false, sortable: false, filter: filterRange, visualization: visValue, transform: trNumber, sortDir: sortDesc, summary: summarySum, isMetric: true, chartGroup: "ResultSize", chartTitle: "Result Size"},
86 - {id: "errorCount", name: "Error Count", colType: ftInteger, visible: false, sortable: false, filter: filterRange, visualization: visValue, transform: trNumber, sortDir: sortDesc, summary: summarySum, isMetric: true, chartGroup: "Errors", chartTitle: "Errors & Warnings"},
87 - {id: "warningCount", name: "Warning Count", colType: ftInteger, visible: false, sortable: false, filter: filterRange, visualization: visValue, transform: trNumber, sortDir: sortDesc, summary: summarySum, isMetric: true, chartGroup: "Errors", chartTitle: "Errors & Warnings"},
88 - {id: "user", name: "User", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, isLabel: true, isPrimary: true},
89 - {id: "clientContextID", name: "Client Context ID", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText},
90 -}
91 -
92 -type cbQueryResponse struct {
93 - Status string `json:"status"`
94 - Results []cbCompletedRequest `json:"results"`
95 - Errors []cbQueryError `json:"errors"`
96 -}
97 -
98 -type cbQueryError struct {
99 - Message string `json:"msg"`
100 -}
101 -
102 -type cbCompletedRequest struct {
103 - RequestID string `json:"requestId"`
104 - RequestTime string `json:"requestTime"`
105 - Statement string `json:"statement"`
106 - ElapsedTime string `json:"elapsedTime"`
107 - ServiceTime string `json:"serviceTime"`
108 - ResultCount json.Number `json:"resultCount"`
109 - ResultSize json.Number `json:"resultSize"`
110 - ErrorCount json.Number `json:"errorCount"`
111 - WarningCount json.Number `json:"warningCount"`
112 - User string `json:"user"`
113 - ClientContextID string `json:"clientContextID"`
114 -}
115 -
116 -type cbRow struct {
117 - RequestID string
118 - RequestTime time.Time
119 - RequestTimeRaw string
120 - Statement string
121 - ElapsedMs float64
122 - ServiceMs float64
123 - ResultCount int64
124 - ResultSize int64
125 - ErrorCount int64
126 - WarningCount int64
127 - User string
128 - ClientContextID string
129 -}
130 -
131 -func couchbaseMethods() []module.MethodConfig {
132 - sortOptions := buildCouchbaseSortOptions(couchbaseAllColumns)
133 - return []module.MethodConfig{
134 - {
135 - UpdateEvery: 10,
136 - ID: "top-queries",
137 - Name: "Top Queries",
138 - Help: "Top N1QL requests from system:completed_requests",
139 - RequireCloud: true,
140 - RequiredParams: []funcapi.ParamConfig{{
141 - ID: paramSort,
142 - Name: "Filter By",
143 - Help: "Select the primary sort column",
144 - Selection: funcapi.ParamSelect,
145 - Options: sortOptions,
146 - UniqueView: true,
147 - }},
148 - },
149 - }
150 -}
151 -
152 -func couchbaseMethodParams(_ context.Context, _ *module.Job, method string) ([]funcapi.ParamConfig, error) {
153 - switch method {
154 - case "top-queries":
155 - return []funcapi.ParamConfig{buildCouchbaseSortParam(couchbaseAllColumns)}, nil
156 - default:
157 - return nil, fmt.Errorf("unknown method: %s", method)
158 - }
159 -}
160 -
161 -func couchbaseHandleMethod(ctx context.Context, job *module.Job, method string, params funcapi.ResolvedParams) *module.FunctionResponse {
162 - collector, ok := job.Module().(*Collector)
163 - if !ok {
164 - return &module.FunctionResponse{Status: 500, Message: "internal error: invalid module type"}
165 - }
166 -
167 - if collector.httpClient == nil {
168 - return &module.FunctionResponse{
169 - Status: 503,
170 - Message: "collector is still initializing, please retry in a few seconds",
171 - }
172 - }
173 -
174 - switch method {
175 - case "top-queries":
176 - return collector.collectTopQueries(ctx, params.Column(paramSort))
177 - default:
178 - return &module.FunctionResponse{Status: 404, Message: fmt.Sprintf("unknown method: %s", method)}
179 - }
180 -}
181 -
182 -func buildCouchbaseSortOptions(cols []couchbaseColumnMeta) []funcapi.ParamOption {
183 - var sortOptions []funcapi.ParamOption
184 - sortDir := funcapi.FieldSortDescending
185 - for _, col := range cols {
186 - if !col.sortable {
187 - continue
188 - }
189 - opt := funcapi.ParamOption{
190 - ID: col.id,
191 - Column: col.id,
192 - Name: fmt.Sprintf("Top queries by %s", col.name),
193 - Sort: &sortDir,
194 - }
195 - if col.id == "elapsedTime" {
196 - opt.Default = true
197 - }
198 - sortOptions = append(sortOptions, opt)
199 - }
200 - return sortOptions
201 -}
202 -
203 -func buildCouchbaseSortParam(cols []couchbaseColumnMeta) funcapi.ParamConfig {
204 - return funcapi.ParamConfig{
205 - ID: paramSort,
206 - Name: "Filter By",
207 - Help: "Select the primary sort column",
208 - Selection: funcapi.ParamSelect,
209 - Options: buildCouchbaseSortOptions(cols),
210 - UniqueView: true,
211 - }
212 -}
213 -
214 -func buildCouchbaseColumns(cols []couchbaseColumnMeta) map[string]any {
215 - result := make(map[string]any, len(cols))
216 - for i, col := range cols {
217 - colDef := funcapi.Column{
218 - Index: i,
219 - Name: col.name,
220 - Type: col.colType,
221 - Units: col.units,
222 - Visualization: col.visualization,
223 - Sort: col.sortDir,
224 - Sortable: col.sortable,
225 - Sticky: col.sticky,
226 - Summary: col.summary,
227 - Filter: col.filter,
228 - FullWidth: col.fullWidth,
229 - Wrap: col.wrap,
230 - DefaultExpandedFilter: false,
231 - UniqueKey: col.uniqueKey,
232 - Visible: col.visible,
233 - ValueOptions: funcapi.ValueOptions{
234 - Transform: col.transform,
235 - DecimalPoints: col.decimalPoints,
236 - DefaultValue: nil,
237 - },
238 - }
239 - result[col.id] = colDef.BuildColumn()
240 - }
241 - return result
242 -}
243 -
244 -func (c *Collector) queryServiceURL() (string, error) {
245 - if c.QueryURL != "" {
246 - return c.QueryURL, nil
247 - }
248 - parsed, err := url.Parse(c.URL)
249 - if err != nil {
250 - return "", err
251 - }
252 - host := parsed.Hostname()
253 - port := parsed.Port()
254 - if port == "" || port == "8091" {
255 - port = "8093"
256 - }
257 - if port != "" {
258 - parsed.Host = net.JoinHostPort(host, port)
259 - } else {
260 - parsed.Host = host
261 - }
262 - parsed.Path = ""
263 - return parsed.String(), nil
264 -}
265 -
266 -func (c *Collector) buildQueryRequest(ctx context.Context, statement string) (*http.Request, error) {
267 - queryURL, err := c.queryServiceURL()
268 - if err != nil {
269 - return nil, err
270 - }
271 -
272 - u, err := url.Parse(queryURL)
273 - if err != nil {
274 - return nil, err
275 - }
276 - u.Path = path.Join(u.Path, "/query/service")
277 -
278 - reqCfg := c.RequestConfig
279 - reqCfg.URL = u.String()
280 - reqCfg.Method = http.MethodPost
281 - reqCfg.Body = url.Values{"statement": {statement}}.Encode()
282 - if reqCfg.Headers == nil {
283 - reqCfg.Headers = map[string]string{}
284 - }
285 - reqCfg.Headers["Content-Type"] = "application/x-www-form-urlencoded"
286 -
287 - req, err := web.NewHTTPRequest(reqCfg)
288 - if err != nil {
289 - return nil, err
290 - }
291 - return req.WithContext(ctx), nil
292 -}
293 -
294 -func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
295 - limit := c.TopQueriesLimit
296 - if limit <= 0 {
297 - limit = 500
298 - }
299 -
300 - statement := "SELECT cr.requestId, cr.requestTime, cr.statement, cr.elapsedTime, cr.serviceTime, " +
301 - "cr.resultCount, cr.resultSize, cr.errorCount, cr.warningCount, cr.users AS `user`, cr.clientContextID " +
302 - "FROM system:completed_requests AS cr"
303 -
304 - req, err := c.buildQueryRequest(ctx, statement)
305 - if err != nil {
306 - return &module.FunctionResponse{Status: 500, Message: err.Error()}
307 - }
308 -
309 - var resp cbQueryResponse
310 - if err := web.DoHTTP(c.httpClient).RequestJSON(req, &resp); err != nil {
311 - if ctx.Err() == context.DeadlineExceeded {
312 - return &module.FunctionResponse{Status: 504, Message: "query timed out"}
313 - }
314 - return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("query failed: %v", err)}
315 - }
316 -
317 - if strings.ToLower(resp.Status) != "success" {
318 - msg := "query failed"
319 - if len(resp.Errors) > 0 && resp.Errors[0].Message != "" {
320 - msg = resp.Errors[0].Message
321 - }
322 - return &module.FunctionResponse{Status: 500, Message: msg}
323 - }
324 -
325 - rows := make([]cbRow, 0, len(resp.Results))
326 - for _, r := range resp.Results {
327 - rows = append(rows, buildCouchbaseRow(r))
328 - }
329 -
330 - if len(rows) == 0 {
331 - return &module.FunctionResponse{
332 - Status: 200,
333 - Message: "No completed requests found.",
334 - Help: "Top N1QL requests from system:completed_requests",
335 - Columns: buildCouchbaseColumns(couchbaseAllColumns),
336 - Data: [][]any{},
337 - DefaultSortColumn: "elapsedTime",
338 - RequiredParams: []funcapi.ParamConfig{buildCouchbaseSortParam(couchbaseAllColumns)},
339 - Charts: couchbaseTopQueriesCharts(couchbaseAllColumns),
340 - DefaultCharts: couchbaseTopQueriesDefaultCharts(couchbaseAllColumns),
341 - GroupBy: couchbaseTopQueriesGroupBy(couchbaseAllColumns),
342 - }
343 - }
344 -
345 - sortColumn = mapCouchbaseSortColumn(sortColumn)
346 - sortCouchbaseRows(rows, sortColumn)
347 -
348 - if len(rows) > limit {
349 - rows = rows[:limit]
350 - }
351 -
352 - data := make([][]any, 0, len(rows))
353 - for _, row := range rows {
354 - out := make([]any, len(couchbaseAllColumns))
355 - for i, col := range couchbaseAllColumns {
356 - switch col.id {
357 - case "requestId":
358 - out[i] = row.RequestID
359 - case "requestTime":
360 - if row.RequestTime.IsZero() {
361 - out[i] = row.RequestTimeRaw
362 - } else {
363 - out[i] = row.RequestTime.Format(time.RFC3339Nano)
364 - }
365 - case "statement":
366 - out[i] = strmutil.TruncateText(row.Statement, couchbaseMaxQueryTextLength)
367 - case "elapsedTime":
368 - out[i] = row.ElapsedMs
369 - case "serviceTime":
370 - out[i] = row.ServiceMs
371 - case "resultCount":
372 - out[i] = row.ResultCount
373 - case "resultSize":
374 - out[i] = row.ResultSize
375 - case "errorCount":
376 - out[i] = row.ErrorCount
377 - case "warningCount":
378 - out[i] = row.WarningCount
379 - case "user":
380 - out[i] = row.User
381 - case "clientContextID":
382 - out[i] = row.ClientContextID
383 - default:
384 - out[i] = nil
385 - }
386 - }
387 - data = append(data, out)
388 - }
389 -
390 - return &module.FunctionResponse{
391 - Status: 200,
392 - Help: "Top N1QL requests from system:completed_requests",
393 - Columns: buildCouchbaseColumns(couchbaseAllColumns),
394 - Data: data,
395 - DefaultSortColumn: "elapsedTime",
396 - RequiredParams: []funcapi.ParamConfig{buildCouchbaseSortParam(couchbaseAllColumns)},
397 - Charts: couchbaseTopQueriesCharts(couchbaseAllColumns),
398 - DefaultCharts: couchbaseTopQueriesDefaultCharts(couchbaseAllColumns),
399 - GroupBy: couchbaseTopQueriesGroupBy(couchbaseAllColumns),
400 - }
401 -}
402 -
403 -func buildCouchbaseRow(r cbCompletedRequest) cbRow {
404 - row := cbRow{
405 - RequestID: r.RequestID,
406 - RequestTimeRaw: r.RequestTime,
407 - Statement: r.Statement,
408 - User: r.User,
409 - ClientContextID: r.ClientContextID,
410 - }
411 -
412 - if t, err := time.Parse(time.RFC3339Nano, r.RequestTime); err == nil {
413 - row.RequestTime = t
414 - } else if t, err := time.Parse(time.RFC3339, r.RequestTime); err == nil {
415 - row.RequestTime = t
416 - }
417 -
418 - row.ElapsedMs = parseDurationMs(r.ElapsedTime)
419 - row.ServiceMs = parseDurationMs(r.ServiceTime)
420 - row.ResultCount = parseNumber(r.ResultCount)
421 - row.ResultSize = parseNumber(r.ResultSize)
422 - row.ErrorCount = parseNumber(r.ErrorCount)
423 - row.WarningCount = parseNumber(r.WarningCount)
424 -
425 - return row
426 -}
427 -
428 -func parseDurationMs(raw string) float64 {
429 - if raw == "" {
430 - return 0
431 - }
432 - if d, err := time.ParseDuration(raw); err == nil {
433 - return float64(d) / float64(time.Millisecond)
434 - }
435 - return 0
436 -}
437 -
438 -func parseNumber(n json.Number) int64 {
439 - if n == "" {
440 - return 0
441 - }
442 - if i, err := n.Int64(); err == nil {
443 - return i
444 - }
445 - if f, err := n.Float64(); err == nil {
446 - return int64(f)
447 - }
448 - return 0
449 -}
450 -
451 -func mapCouchbaseSortColumn(col string) string {
452 - switch col {
453 - case "elapsedTime", "serviceTime", "requestTime", "resultCount":
454 - return col
455 - default:
456 - return "elapsedTime"
457 - }
458 -}
459 -
460 -func sortCouchbaseRows(rows []cbRow, sortColumn string) {
461 - switch sortColumn {
462 - case "serviceTime":
463 - sort.Slice(rows, func(i, j int) bool {
464 - return rows[i].ServiceMs > rows[j].ServiceMs
465 - })
466 - case "requestTime":
467 - sort.Slice(rows, func(i, j int) bool {
468 - return rows[i].RequestTime.After(rows[j].RequestTime)
469 - })
470 - case "resultCount":
471 - sort.Slice(rows, func(i, j int) bool {
472 - return rows[i].ResultCount > rows[j].ResultCount
473 - })
474 - default:
475 - sort.Slice(rows, func(i, j int) bool {
476 - return rows[i].ElapsedMs > rows[j].ElapsedMs
477 - })
478 - }
479 -}
480 -
481 -func couchbaseTopQueriesCharts(cols []couchbaseColumnMeta) map[string]module.ChartConfig {
482 - charts := make(map[string]module.ChartConfig)
483 - for _, col := range cols {
484 - if !col.isMetric || col.chartGroup == "" {
485 - continue
486 - }
487 - cfg, ok := charts[col.chartGroup]
488 - if !ok {
489 - title := col.chartTitle
490 - if title == "" {
491 - title = col.chartGroup
492 - }
493 - cfg = module.ChartConfig{Name: title, Type: "stacked-bar"}
494 - }
495 - cfg.Columns = append(cfg.Columns, col.id)
496 - charts[col.chartGroup] = cfg
497 - }
498 - return charts
499 -}
500 -
501 -func couchbaseTopQueriesDefaultCharts(cols []couchbaseColumnMeta) [][]string {
502 - label := primaryCouchbaseLabel(cols)
503 - if label == "" {
504 - return nil
505 - }
506 - chartGroups := defaultCouchbaseChartGroups(cols)
507 - out := make([][]string, 0, len(chartGroups))
508 - for _, group := range chartGroups {
509 - out = append(out, []string{group, label})
510 - }
511 - return out
512 -}
513 -
514 -func couchbaseTopQueriesGroupBy(cols []couchbaseColumnMeta) map[string]module.GroupByConfig {
515 - groupBy := make(map[string]module.GroupByConfig)
516 - for _, col := range cols {
517 - if !col.isLabel {
518 - continue
519 - }
520 - groupBy[col.id] = module.GroupByConfig{
521 - Name: "Group by " + col.name,
522 - Columns: []string{col.id},
523 - }
524 - }
525 - return groupBy
526 -}
527 -
528 -func primaryCouchbaseLabel(cols []couchbaseColumnMeta) string {
529 - for _, col := range cols {
530 - if col.isPrimary {
531 - return col.id
532 - }
533 - }
534 - for _, col := range cols {
535 - if col.isLabel {
536 - return col.id
537 - }
538 - }
539 - return ""
540 -}
541 -
542 -func defaultCouchbaseChartGroups(cols []couchbaseColumnMeta) []string {
543 - groups := make([]string, 0)
544 - seen := make(map[string]bool)
545 - for _, col := range cols {
546 - if !col.isMetric || col.chartGroup == "" || !col.isDefaultChart {
547 - continue
548 - }
549 - if !seen[col.chartGroup] {
550 - seen[col.chartGroup] = true
551 - groups = append(groups, col.chartGroup)
552 - }
553 - }
554 - if len(groups) > 0 {
555 - return groups
556 - }
557 - for _, col := range cols {
558 - if !col.isMetric || col.chartGroup == "" {
559 - continue
560 - }
561 - if !seen[col.chartGroup] {
562 - seen[col.chartGroup] = true
563 - groups = append(groups, col.chartGroup)
564 - }
565 - }
566 - return groups
567 -}
src/go/plugin/go.d/collector/elasticsearch/collector.go
+12 -6
@@ -25,11 +25,10 @@ func init() {
25 Defaults: module.Defaults{
26 UpdateEvery: 5,
27 },
28 - Create: func() module.Module { return New() },
29 - Config: func() any { return &Config{} },
30 - Methods: elasticsearchMethods,
31 - MethodParams: elasticsearchMethodParams,
32 - HandleMethod: elasticsearchHandleMethod,
28 + Create: func() module.Module { return New() },
29 + Config: func() any { return &Config{} },
30 + Methods: elasticsearchMethods,
31 + MethodHandler: elasticsearchFunctionHandler,
32 })
33 }
34
@@ -86,6 +85,8 @@ type Collector struct {
85 clusterName string
86 nodes map[string]bool
87 indices map[string]bool
88 +
89 + funcRouter *funcRouter
90 }
91
92 func (c *Collector) Configuration() any {
@@ -104,6 +105,8 @@ func (c *Collector) Init(context.Context) error {
105 }
106 c.httpClient = httpClient
107
108 + c.funcRouter = newFuncRouter(c)
109 +
110 return nil
111 }
112
@@ -135,7 +138,10 @@ func (c *Collector) Collect(context.Context) map[string]int64 {
138 return mx
139 }
140
138 -func (c *Collector) Cleanup(context.Context) {
141 +func (c *Collector) Cleanup(ctx context.Context) {
142 + if c.funcRouter != nil {
143 + c.funcRouter.Cleanup(ctx)
144 + }
145 if c.httpClient != nil {
146 c.httpClient.CloseIdleConnections()
147 }
src/go/plugin/go.d/collector/elasticsearch/func_router.go new
+63
@@ -0,0 +1,63 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package elasticsearch
4 +
5 +import (
6 + "context"
7 + "fmt"
8 +
9 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
10 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
11 +)
12 +
13 +// funcRouter routes method calls to appropriate function handlers.
14 +type funcRouter struct {
15 + collector *Collector
16 + handlers map[string]funcapi.MethodHandler
17 +}
18 +
19 +func newFuncRouter(c *Collector) *funcRouter {
20 + r := &funcRouter{
21 + collector: c,
22 + handlers: make(map[string]funcapi.MethodHandler),
23 + }
24 + r.handlers[topQueriesMethodID] = newFuncTopQueries(r)
25 + return r
26 +}
27 +
28 +// Compile-time interface check.
29 +var _ funcapi.MethodHandler = (*funcRouter)(nil)
30 +
31 +func (r *funcRouter) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
32 + if h, ok := r.handlers[method]; ok {
33 + return h.MethodParams(ctx, method)
34 + }
35 + return nil, fmt.Errorf("unknown method: %s", method)
36 +}
37 +
38 +func (r *funcRouter) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
39 + if h, ok := r.handlers[method]; ok {
40 + return h.Handle(ctx, method, params)
41 + }
42 + return funcapi.NotFoundResponse(method)
43 +}
44 +
45 +func (r *funcRouter) Cleanup(ctx context.Context) {
46 + for _, h := range r.handlers {
47 + h.Cleanup(ctx)
48 + }
49 +}
50 +
51 +func elasticsearchMethods() []funcapi.MethodConfig {
52 + return []funcapi.MethodConfig{
53 + topQueriesMethodConfig(),
54 + }
55 +}
56 +
57 +func elasticsearchFunctionHandler(job *module.Job) funcapi.MethodHandler {
58 + c, ok := job.Module().(*Collector)
59 + if !ok {
60 + return nil
61 + }
62 + return c.funcRouter
63 +}
src/go/plugin/go.d/collector/elasticsearch/func_top_queries.go new
+284
@@ -0,0 +1,284 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package elasticsearch
4 +
5 +import (
6 + "context"
7 + "fmt"
8 + "net/url"
9 + "sort"
10 + "strconv"
11 + "strings"
12 + "time"
13 +
14 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
15 + "github.com/netdata/netdata/go/plugins/pkg/web"
16 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
17 +)
18 +
19 +const (
20 + topQueriesMethodID = "top-queries"
21 + topQueriesMaxTextLength = 4096
22 +)
23 +
24 +func topQueriesMethodConfig() funcapi.MethodConfig {
25 + return funcapi.MethodConfig{
26 + ID: topQueriesMethodID,
27 + Name: "Top Queries",
28 + UpdateEvery: 10,
29 + Help: "Running queries from Elasticsearch Tasks API",
30 + RequireCloud: true,
31 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(topQueriesColumns)},
32 + }
33 +}
34 +
35 +type topQueriesColumn struct {
36 + funcapi.ColumnMeta
37 + sortOpt bool // whether this column appears as a sort option
38 + sortLbl string // label for sort option dropdown
39 + defaultSort bool // default sort column
40 +}
41 +
42 +// funcapi.SortableColumn interface implementation for topQueriesColumn.
43 +func (c topQueriesColumn) IsSortOption() bool { return c.sortOpt }
44 +func (c topQueriesColumn) SortLabel() string { return c.sortLbl }
45 +func (c topQueriesColumn) IsDefaultSort() bool { return c.defaultSort }
46 +func (c topQueriesColumn) ColumnName() string { return c.Name }
47 +func (c topQueriesColumn) SortColumn() string { return "" }
48 +
49 +var topQueriesColumns = []topQueriesColumn{
50 + {ColumnMeta: funcapi.ColumnMeta{Name: "taskId", Tooltip: "Task ID", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, UniqueKey: true, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryCount}, sortOpt: true, sortLbl: "Top queries by Task ID"},
51 + {ColumnMeta: funcapi.ColumnMeta{Name: "node", Tooltip: "Node ID", Type: funcapi.FieldTypeString, Visible: true, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, GroupBy: &funcapi.GroupByOptions{}}},
52 + {ColumnMeta: funcapi.ColumnMeta{Name: "nodeName", Tooltip: "Node Name", Type: funcapi.FieldTypeString, Visible: true, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, GroupBy: &funcapi.GroupByOptions{IsDefault: true}}},
53 + {ColumnMeta: funcapi.ColumnMeta{Name: "action", Tooltip: "Action", Type: funcapi.FieldTypeString, Visible: true, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, GroupBy: &funcapi.GroupByOptions{}}},
54 + {ColumnMeta: funcapi.ColumnMeta{Name: "type", Tooltip: "Type", Type: funcapi.FieldTypeString, Visible: false, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, GroupBy: &funcapi.GroupByOptions{}}},
55 + {ColumnMeta: funcapi.ColumnMeta{Name: "description", Tooltip: "Description", Type: funcapi.FieldTypeString, Visible: true, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, Sticky: true, FullWidth: true, Wrap: true}},
56 + {ColumnMeta: funcapi.ColumnMeta{Name: "startTime", Tooltip: "Start Time", Type: funcapi.FieldTypeTimestamp, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformDatetime, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax}, sortOpt: true, sortLbl: "Top queries by Start Time"},
57 + {ColumnMeta: funcapi.ColumnMeta{Name: "runningTime", Tooltip: "Running Time", Type: funcapi.FieldTypeDuration, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualBar, Transform: funcapi.FieldTransformDuration, Units: "milliseconds", DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "RunningTime", Title: "Running Time", IsDefault: true}}, sortOpt: true, defaultSort: true, sortLbl: "Top queries by Running Time"},
58 + {ColumnMeta: funcapi.ColumnMeta{Name: "cancellable", Tooltip: "Cancellable", Type: funcapi.FieldTypeBoolean, Visible: false, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformNone}},
59 + {ColumnMeta: funcapi.ColumnMeta{Name: "cancelled", Tooltip: "Cancelled", Type: funcapi.FieldTypeBoolean, Visible: false, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformNone}},
60 +}
61 +
62 +type topQueriesResponse struct {
63 + Nodes map[string]struct {
64 + Name string `json:"name"`
65 + Tasks map[string]topQueriesTask `json:"tasks"`
66 + } `json:"nodes"`
67 +}
68 +
69 +type topQueriesTask struct {
70 + ID int64 `json:"id"`
71 + Action string `json:"action"`
72 + Type string `json:"type"`
73 + Description string `json:"description"`
74 + StartTimeInMillis int64 `json:"start_time_in_millis"`
75 + RunningTimeInNanos int64 `json:"running_time_in_nanos"`
76 + Cancellable bool `json:"cancellable"`
77 + Cancelled bool `json:"cancelled"`
78 +}
79 +
80 +type topQueriesRow struct {
81 + TaskID string
82 + NodeID string
83 + NodeName string
84 + Action string
85 + Type string
86 + Description string
87 + StartTime time.Time
88 + RunningTime time.Duration
89 + Cancellable bool
90 + Cancelled bool
91 +}
92 +
93 +// funcTopQueries implements funcapi.MethodHandler for Elasticsearch top-queries.
94 +type funcTopQueries struct {
95 + router *funcRouter
96 +}
97 +
98 +func newFuncTopQueries(r *funcRouter) *funcTopQueries {
99 + return &funcTopQueries{router: r}
100 +}
101 +
102 +// Compile-time interface check.
103 +var _ funcapi.MethodHandler = (*funcTopQueries)(nil)
104 +
105 +func (f *funcTopQueries) Cleanup(ctx context.Context) {}
106 +
107 +// MethodParams implements funcapi.MethodHandler.
108 +func (f *funcTopQueries) MethodParams(_ context.Context, method string) ([]funcapi.ParamConfig, error) {
109 + switch method {
110 + case topQueriesMethodID:
111 + return []funcapi.ParamConfig{funcapi.BuildSortParam(topQueriesColumns)}, nil
112 + default:
113 + return nil, fmt.Errorf("unknown method: %s", method)
114 + }
115 +}
116 +
117 +// Handle implements funcapi.MethodHandler.
118 +func (f *funcTopQueries) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
119 + if f.router.collector.httpClient == nil {
120 + return funcapi.UnavailableResponse("collector is still initializing, please retry in a few seconds")
121 + }
122 +
123 + switch method {
124 + case topQueriesMethodID:
125 + return f.collectData(ctx, params.Column("__sort"))
126 + default:
127 + return funcapi.NotFoundResponse(method)
128 + }
129 +}
130 +
131 +func (f *funcTopQueries) collectData(ctx context.Context, sortColumn string) *funcapi.FunctionResponse {
132 + limit := f.router.collector.TopQueriesLimit
133 + if limit <= 0 {
134 + limit = 500
135 + }
136 +
137 + req, err := web.NewHTTPRequestWithPath(f.router.collector.RequestConfig, "/_tasks")
138 + if err != nil {
139 + return &funcapi.FunctionResponse{Status: 500, Message: err.Error()}
140 + }
141 + req = req.WithContext(ctx)
142 + q := url.Values{}
143 + q.Set("actions", "*search")
144 + q.Set("detailed", "true")
145 + req.URL.RawQuery = q.Encode()
146 +
147 + var resp topQueriesResponse
148 + if err := web.DoHTTP(f.router.collector.httpClient).RequestJSON(req, &resp); err != nil {
149 + if ctx.Err() == context.DeadlineExceeded {
150 + return &funcapi.FunctionResponse{Status: 504, Message: "query timed out"}
151 + }
152 + return &funcapi.FunctionResponse{Status: 500, Message: fmt.Sprintf("tasks query failed: %v", err)}
153 + }
154 +
155 + rows := make([]topQueriesRow, 0, 100)
156 + for nodeID, node := range resp.Nodes {
157 + for taskID, task := range node.Tasks {
158 + rows = append(rows, topQueriesRow{
159 + TaskID: taskID,
160 + NodeID: nodeID,
161 + NodeName: node.Name,
162 + Action: task.Action,
163 + Type: task.Type,
164 + Description: task.Description,
165 + StartTime: time.UnixMilli(task.StartTimeInMillis),
166 + RunningTime: time.Duration(task.RunningTimeInNanos),
167 + Cancellable: task.Cancellable,
168 + Cancelled: task.Cancelled,
169 + })
170 + }
171 + }
172 +
173 + cs := f.columnSet(topQueriesColumns)
174 + sortParam := funcapi.BuildSortParam(topQueriesColumns)
175 +
176 + if len(rows) == 0 {
177 + return &funcapi.FunctionResponse{
178 + Status: 200,
179 + Message: "No running search tasks found.",
180 + Help: "Running queries from Elasticsearch Tasks API",
181 + Columns: cs.BuildColumns(),
182 + Data: [][]any{},
183 + DefaultSortColumn: "runningTime",
184 + RequiredParams: []funcapi.ParamConfig{sortParam},
185 + ChartingConfig: cs.BuildCharting(),
186 + }
187 + }
188 +
189 + sortColumn = f.mapSortColumn(sortColumn)
190 + f.sortRows(rows, sortColumn)
191 +
192 + if len(rows) > limit {
193 + rows = rows[:limit]
194 + }
195 +
196 + data := make([][]any, 0, len(rows))
197 + for _, row := range rows {
198 + out := make([]any, len(topQueriesColumns))
199 + for i, col := range topQueriesColumns {
200 + switch col.Name {
201 + case "taskId":
202 + out[i] = row.TaskID
203 + case "node":
204 + out[i] = row.NodeID
205 + case "nodeName":
206 + out[i] = row.NodeName
207 + case "action":
208 + out[i] = row.Action
209 + case "type":
210 + out[i] = row.Type
211 + case "description":
212 + out[i] = strmutil.TruncateText(row.Description, topQueriesMaxTextLength)
213 + case "startTime":
214 + out[i] = row.StartTime.Format(time.RFC3339Nano)
215 + case "runningTime":
216 + out[i] = float64(row.RunningTime) / float64(time.Millisecond)
217 + case "cancellable":
218 + out[i] = row.Cancellable
219 + case "cancelled":
220 + out[i] = row.Cancelled
221 + default:
222 + out[i] = nil
223 + }
224 + }
225 + data = append(data, out)
226 + }
227 +
228 + return &funcapi.FunctionResponse{
229 + Status: 200,
230 + Help: "Running queries from Elasticsearch Tasks API",
231 + Columns: cs.BuildColumns(),
232 + Data: data,
233 + DefaultSortColumn: "runningTime",
234 + RequiredParams: []funcapi.ParamConfig{sortParam},
235 + ChartingConfig: cs.BuildCharting(),
236 + }
237 +}
238 +
239 +func (f *funcTopQueries) columnSet(cols []topQueriesColumn) funcapi.ColumnSet[topQueriesColumn] {
240 + return funcapi.Columns(cols, func(c topQueriesColumn) funcapi.ColumnMeta { return c.ColumnMeta })
241 +}
242 +
243 +func (f *funcTopQueries) mapSortColumn(col string) string {
244 + switch col {
245 + case "runningTime", "startTime", "taskId":
246 + return col
247 + default:
248 + return "runningTime"
249 + }
250 +}
251 +
252 +func (f *funcTopQueries) sortRows(rows []topQueriesRow, sortColumn string) {
253 + switch sortColumn {
254 + case "startTime":
255 + sort.Slice(rows, func(i, j int) bool {
256 + return rows[i].StartTime.After(rows[j].StartTime)
257 + })
258 + case "taskId":
259 + sort.Slice(rows, func(i, j int) bool {
260 + left, lok := f.parseTaskID(rows[i].TaskID)
261 + right, rok := f.parseTaskID(rows[j].TaskID)
262 + if lok && rok {
263 + return left > right
264 + }
265 + return rows[i].TaskID > rows[j].TaskID
266 + })
267 + default:
268 + sort.Slice(rows, func(i, j int) bool {
269 + return rows[i].RunningTime > rows[j].RunningTime
270 + })
271 + }
272 +}
273 +
274 +func (f *funcTopQueries) parseTaskID(taskID string) (int64, bool) {
275 + id := taskID
276 + if idx := strings.LastIndex(id, ":"); idx != -1 {
277 + id = id[idx+1:]
278 + }
279 + val, err := strconv.ParseInt(id, 10, 64)
280 + if err != nil {
281 + return 0, false
282 + }
283 + return val, true
284 +}
src/go/plugin/go.d/collector/elasticsearch/func_top_queries_test.go renamed
+8 -10
@@ -29,20 +29,17 @@ func TestElasticsearchMethods(t *testing.T) {
29 require.NotEmpty(sortParam.Options)
30 }
31
32 -func TestElasticsearchAllColumns_HasRequiredColumns(t *testing.T) {
32 +func TestTopQueriesColumns_HasRequiredColumns(t *testing.T) {
33 required := []string{"taskId", "description", "runningTime"}
34
35 - uiKeys := make(map[string]bool)
36 - for _, col := range esAllColumns {
37 - uiKeys[col.id] = true
38 - }
39 -
40 - for _, key := range required {
41 - assert.True(t, uiKeys[key], "column %s should be defined", key)
35 + f := &funcTopQueries{}
36 + cs := f.columnSet(topQueriesColumns)
37 + for _, id := range required {
38 + assert.True(t, cs.ContainsColumn(id), "column %s should be defined", id)
39 }
40 }
41
45 -func TestMapElasticsearchSortColumn(t *testing.T) {
42 +func TestFuncTopQueries_MapSortColumn(t *testing.T) {
43 tests := map[string]struct {
44 input string
45 expected string
@@ -53,9 +50,10 @@ func TestMapElasticsearchSortColumn(t *testing.T) {
50 "invalid": {input: "bad", expected: "runningTime"},
51 }
52
53 + f := &funcTopQueries{}
54 for name, tc := range tests {
55 t.Run(name, func(t *testing.T) {
58 - assert.Equal(t, tc.expected, mapElasticsearchSortColumn(tc.input))
56 + assert.Equal(t, tc.expected, f.mapSortColumn(tc.input))
57 })
58 }
59 }
src/go/plugin/go.d/collector/elasticsearch/functions.go deleted
-470
@@ -1,470 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -package elasticsearch
4 -
5 -import (
6 - "context"
7 - "fmt"
8 - "net/url"
9 - "sort"
10 - "strconv"
11 - "strings"
12 - "time"
13 -
14 - "github.com/netdata/netdata/go/plugins/pkg/funcapi"
15 - "github.com/netdata/netdata/go/plugins/pkg/web"
16 - "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
17 - "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
18 -)
19 -
20 -const elasticMaxQueryTextLength = 4096
21 -
22 -const (
23 - paramSort = "__sort"
24 -
25 - ftString = funcapi.FieldTypeString
26 - ftDuration = funcapi.FieldTypeDuration
27 - ftTimestamp = funcapi.FieldTypeTimestamp
28 - ftBoolean = funcapi.FieldTypeBoolean
29 -
30 - trNone = funcapi.FieldTransformNone
31 - trDuration = funcapi.FieldTransformDuration
32 - trDatetime = funcapi.FieldTransformDatetime
33 - trText = funcapi.FieldTransformText
34 -
35 - visValue = funcapi.FieldVisualValue
36 - visBar = funcapi.FieldVisualBar
37 -
38 - sortAsc = funcapi.FieldSortAscending
39 - sortDesc = funcapi.FieldSortDescending
40 -
41 - summaryCount = funcapi.FieldSummaryCount
42 - summaryMax = funcapi.FieldSummaryMax
43 - summarySum = funcapi.FieldSummarySum
44 -
45 - filterMulti = funcapi.FieldFilterMultiselect
46 - filterRange = funcapi.FieldFilterRange
47 -)
48 -
49 -type esColumnMeta struct {
50 - id string
51 - name string
52 - colType funcapi.FieldType
53 - visible bool
54 - sortable bool
55 - fullWidth bool
56 - wrap bool
57 - sticky bool
58 - filter funcapi.FieldFilter
59 - visualization funcapi.FieldVisual
60 - transform funcapi.FieldTransform
61 - units string
62 - decimalPoints int
63 - uniqueKey bool
64 - sortDir funcapi.FieldSort
65 - summary funcapi.FieldSummary
66 - isLabel bool
67 - isPrimary bool
68 - isMetric bool
69 - chartGroup string
70 - chartTitle string
71 - isDefaultChart bool
72 -}
73 -
74 -var esAllColumns = []esColumnMeta{
75 - {id: "taskId", name: "Task ID", colType: ftString, visible: false, sortable: true, filter: filterMulti, visualization: visValue, transform: trText, uniqueKey: true, sortDir: sortDesc, summary: summaryCount},
76 - {id: "node", name: "Node ID", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, isLabel: true},
77 - {id: "nodeName", name: "Node Name", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, isLabel: true, isPrimary: true},
78 - {id: "action", name: "Action", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, isLabel: true},
79 - {id: "type", name: "Type", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, isLabel: true},
80 - {id: "description", name: "Description", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, sticky: true, fullWidth: true, wrap: true},
81 - {id: "startTime", name: "Start Time", colType: ftTimestamp, visible: true, sortable: true, filter: filterRange, visualization: visValue, transform: trDatetime, sortDir: sortDesc, summary: summaryMax},
82 - {id: "runningTime", name: "Running Time", colType: ftDuration, visible: true, sortable: true, filter: filterRange, visualization: visBar, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summarySum, isMetric: true, chartGroup: "RunningTime", chartTitle: "Running Time", isDefaultChart: true},
83 - {id: "cancellable", name: "Cancellable", colType: ftBoolean, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trNone},
84 - {id: "cancelled", name: "Cancelled", colType: ftBoolean, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trNone},
85 -}
86 -
87 -type esTasksResponse struct {
88 - Nodes map[string]struct {
89 - Name string `json:"name"`
90 - Tasks map[string]esTask `json:"tasks"`
91 - } `json:"nodes"`
92 -}
93 -
94 -type esTask struct {
95 - ID int64 `json:"id"`
96 - Action string `json:"action"`
97 - Type string `json:"type"`
98 - Description string `json:"description"`
99 - StartTimeInMillis int64 `json:"start_time_in_millis"`
100 - RunningTimeInNanos int64 `json:"running_time_in_nanos"`
101 - Cancellable bool `json:"cancellable"`
102 - Cancelled bool `json:"cancelled"`
103 -}
104 -
105 -type esTaskRow struct {
106 - TaskID string
107 - NodeID string
108 - NodeName string
109 - Action string
110 - Type string
111 - Description string
112 - StartTime time.Time
113 - RunningTime time.Duration
114 - Cancellable bool
115 - Cancelled bool
116 -}
117 -
118 -func elasticsearchMethods() []module.MethodConfig {
119 - sortOptions := buildElasticsearchSortOptions(esAllColumns)
120 - return []module.MethodConfig{
121 - {
122 - UpdateEvery: 10,
123 - ID: "top-queries",
124 - Name: "Top Queries",
125 - Help: "Running queries from Elasticsearch Tasks API",
126 - RequireCloud: true,
127 - RequiredParams: []funcapi.ParamConfig{
128 - {
129 - ID: paramSort,
130 - Name: "Filter By",
131 - Help: "Select the primary sort column",
132 - Selection: funcapi.ParamSelect,
133 - Options: sortOptions,
134 - UniqueView: true,
135 - }},
136 - },
137 - }
138 -}
139 -
140 -func elasticsearchMethodParams(_ context.Context, _ *module.Job, method string) ([]funcapi.ParamConfig, error) {
141 - switch method {
142 - case "top-queries":
143 - return []funcapi.ParamConfig{buildElasticsearchSortParam(esAllColumns)}, nil
144 - default:
145 - return nil, fmt.Errorf("unknown method: %s", method)
146 - }
147 -}
148 -
149 -func elasticsearchHandleMethod(ctx context.Context, job *module.Job, method string, params funcapi.ResolvedParams) *module.FunctionResponse {
150 - collector, ok := job.Module().(*Collector)
151 - if !ok {
152 - return &module.FunctionResponse{Status: 500, Message: "internal error: invalid module type"}
153 - }
154 -
155 - if collector.httpClient == nil {
156 - return &module.FunctionResponse{
157 - Status: 503,
158 - Message: "collector is still initializing, please retry in a few seconds",
159 - }
160 - }
161 -
162 - switch method {
163 - case "top-queries":
164 - return collector.collectTopQueries(ctx, params.Column(paramSort))
165 - default:
166 - return &module.FunctionResponse{Status: 404, Message: fmt.Sprintf("unknown method: %s", method)}
167 - }
168 -}
169 -
170 -func buildElasticsearchSortOptions(cols []esColumnMeta) []funcapi.ParamOption {
171 - var sortOptions []funcapi.ParamOption
172 - sortDir := funcapi.FieldSortDescending
173 - for _, col := range cols {
174 - if !col.sortable {
175 - continue
176 - }
177 - opt := funcapi.ParamOption{
178 - ID: col.id,
179 - Column: col.id,
180 - Name: fmt.Sprintf("Top queries by %s", col.name),
181 - Sort: &sortDir,
182 - }
183 - if col.id == "runningTime" {
184 - opt.Default = true
185 - }
186 - sortOptions = append(sortOptions, opt)
187 - }
188 - return sortOptions
189 -}
190 -
191 -func buildElasticsearchSortParam(cols []esColumnMeta) funcapi.ParamConfig {
192 - return funcapi.ParamConfig{
193 - ID: paramSort,
194 - Name: "Filter By",
195 - Help: "Select the primary sort column",
196 - Selection: funcapi.ParamSelect,
197 - Options: buildElasticsearchSortOptions(cols),
198 - UniqueView: true,
199 - }
200 -}
201 -
202 -func buildElasticsearchColumns(cols []esColumnMeta) map[string]any {
203 - result := make(map[string]any, len(cols))
204 - for i, col := range cols {
205 - colDef := funcapi.Column{
206 - Index: i,
207 - Name: col.name,
208 - Type: col.colType,
209 - Units: col.units,
210 - Visualization: col.visualization,
211 - Sort: col.sortDir,
212 - Sortable: col.sortable,
213 - Sticky: col.sticky,
214 - Summary: col.summary,
215 - Filter: col.filter,
216 - FullWidth: col.fullWidth,
217 - Wrap: col.wrap,
218 - DefaultExpandedFilter: false,
219 - UniqueKey: col.uniqueKey,
220 - Visible: col.visible,
221 - ValueOptions: funcapi.ValueOptions{
222 - Transform: col.transform,
223 - DecimalPoints: col.decimalPoints,
224 - DefaultValue: nil,
225 - },
226 - }
227 - result[col.id] = colDef.BuildColumn()
228 - }
229 - return result
230 -}
231 -
232 -func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
233 - limit := c.TopQueriesLimit
234 - if limit <= 0 {
235 - limit = 500
236 - }
237 -
238 - req, err := web.NewHTTPRequestWithPath(c.RequestConfig, "/_tasks")
239 - if err != nil {
240 - return &module.FunctionResponse{Status: 500, Message: err.Error()}
241 - }
242 - req = req.WithContext(ctx)
243 - q := url.Values{}
244 - q.Set("actions", "*search")
245 - q.Set("detailed", "true")
246 - req.URL.RawQuery = q.Encode()
247 -
248 - var resp esTasksResponse
249 - if err := web.DoHTTP(c.httpClient).RequestJSON(req, &resp); err != nil {
250 - if ctx.Err() == context.DeadlineExceeded {
251 - return &module.FunctionResponse{Status: 504, Message: "query timed out"}
252 - }
253 - return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("tasks query failed: %v", err)}
254 - }
255 -
256 - rows := make([]esTaskRow, 0, 100)
257 - for nodeID, node := range resp.Nodes {
258 - for taskID, task := range node.Tasks {
259 - rows = append(rows, esTaskRow{
260 - TaskID: taskID,
261 - NodeID: nodeID,
262 - NodeName: node.Name,
263 - Action: task.Action,
264 - Type: task.Type,
265 - Description: task.Description,
266 - StartTime: time.UnixMilli(task.StartTimeInMillis),
267 - RunningTime: time.Duration(task.RunningTimeInNanos),
268 - Cancellable: task.Cancellable,
269 - Cancelled: task.Cancelled,
270 - })
271 - }
272 - }
273 -
274 - if len(rows) == 0 {
275 - return &module.FunctionResponse{
276 - Status: 200,
277 - Message: "No running search tasks found.",
278 - Help: "Running queries from Elasticsearch Tasks API",
279 - Columns: buildElasticsearchColumns(esAllColumns),
280 - Data: [][]any{},
281 - DefaultSortColumn: "runningTime",
282 - RequiredParams: []funcapi.ParamConfig{buildElasticsearchSortParam(esAllColumns)},
283 - Charts: elasticsearchTopQueriesCharts(esAllColumns),
284 - DefaultCharts: elasticsearchTopQueriesDefaultCharts(esAllColumns),
285 - GroupBy: elasticsearchTopQueriesGroupBy(esAllColumns),
286 - }
287 - }
288 -
289 - sortColumn = mapElasticsearchSortColumn(sortColumn)
290 - sortElasticsearchRows(rows, sortColumn)
291 -
292 - if len(rows) > limit {
293 - rows = rows[:limit]
294 - }
295 -
296 - data := make([][]any, 0, len(rows))
297 - for _, row := range rows {
298 - out := make([]any, len(esAllColumns))
299 - for i, col := range esAllColumns {
300 - switch col.id {
301 - case "taskId":
302 - out[i] = row.TaskID
303 - case "node":
304 - out[i] = row.NodeID
305 - case "nodeName":
306 - out[i] = row.NodeName
307 - case "action":
308 - out[i] = row.Action
309 - case "type":
310 - out[i] = row.Type
311 - case "description":
312 - out[i] = strmutil.TruncateText(row.Description, elasticMaxQueryTextLength)
313 - case "startTime":
314 - out[i] = row.StartTime.Format(time.RFC3339Nano)
315 - case "runningTime":
316 - out[i] = float64(row.RunningTime) / float64(time.Millisecond)
317 - case "cancellable":
318 - out[i] = row.Cancellable
319 - case "cancelled":
320 - out[i] = row.Cancelled
321 - default:
322 - out[i] = nil
323 - }
324 - }
325 - data = append(data, out)
326 - }
327 -
328 - return &module.FunctionResponse{
329 - Status: 200,
330 - Help: "Running queries from Elasticsearch Tasks API",
331 - Columns: buildElasticsearchColumns(esAllColumns),
332 - Data: data,
333 - DefaultSortColumn: "runningTime",
334 - RequiredParams: []funcapi.ParamConfig{buildElasticsearchSortParam(esAllColumns)},
335 - Charts: elasticsearchTopQueriesCharts(esAllColumns),
336 - DefaultCharts: elasticsearchTopQueriesDefaultCharts(esAllColumns),
337 - GroupBy: elasticsearchTopQueriesGroupBy(esAllColumns),
338 - }
339 -}
340 -
341 -func mapElasticsearchSortColumn(col string) string {
342 - switch col {
343 - case "runningTime", "startTime", "taskId":
344 - return col
345 - default:
346 - return "runningTime"
347 - }
348 -}
349 -
350 -func sortElasticsearchRows(rows []esTaskRow, sortColumn string) {
351 - switch sortColumn {
352 - case "startTime":
353 - sort.Slice(rows, func(i, j int) bool {
354 - return rows[i].StartTime.After(rows[j].StartTime)
355 - })
356 - case "taskId":
357 - sort.Slice(rows, func(i, j int) bool {
358 - left, lok := parseElasticsearchTaskID(rows[i].TaskID)
359 - right, rok := parseElasticsearchTaskID(rows[j].TaskID)
360 - if lok && rok {
361 - return left > right
362 - }
363 - return rows[i].TaskID > rows[j].TaskID
364 - })
365 - default:
366 - sort.Slice(rows, func(i, j int) bool {
367 - return rows[i].RunningTime > rows[j].RunningTime
368 - })
369 - }
370 -}
371 -
372 -func parseElasticsearchTaskID(taskID string) (int64, bool) {
373 - id := taskID
374 - if idx := strings.LastIndex(id, ":"); idx != -1 {
375 - id = id[idx+1:]
376 - }
377 - val, err := strconv.ParseInt(id, 10, 64)
378 - if err != nil {
379 - return 0, false
380 - }
381 - return val, true
382 -}
383 -
384 -func elasticsearchTopQueriesCharts(cols []esColumnMeta) map[string]module.ChartConfig {
385 - charts := make(map[string]module.ChartConfig)
386 - for _, col := range cols {
387 - if !col.isMetric || col.chartGroup == "" {
388 - continue
389 - }
390 - cfg, ok := charts[col.chartGroup]
391 - if !ok {
392 - title := col.chartTitle
393 - if title == "" {
394 - title = col.chartGroup
395 - }
396 - cfg = module.ChartConfig{Name: title, Type: "stacked-bar"}
397 - }
398 - cfg.Columns = append(cfg.Columns, col.id)
399 - charts[col.chartGroup] = cfg
400 - }
401 - return charts
402 -}
403 -
404 -func elasticsearchTopQueriesDefaultCharts(cols []esColumnMeta) [][]string {
405 - label := primaryElasticsearchLabel(cols)
406 - if label == "" {
407 - return nil
408 - }
409 - chartGroups := defaultElasticsearchChartGroups(cols)
410 - out := make([][]string, 0, len(chartGroups))
411 - for _, group := range chartGroups {
412 - out = append(out, []string{group, label})
413 - }
414 - return out
415 -}
416 -
417 -func elasticsearchTopQueriesGroupBy(cols []esColumnMeta) map[string]module.GroupByConfig {
418 - groupBy := make(map[string]module.GroupByConfig)
419 - for _, col := range cols {
420 - if !col.isLabel {
421 - continue
422 - }
423 - groupBy[col.id] = module.GroupByConfig{
424 - Name: "Group by " + col.name,
425 - Columns: []string{col.id},
426 - }
427 - }
428 - return groupBy
429 -}
430 -
431 -func primaryElasticsearchLabel(cols []esColumnMeta) string {
432 - for _, col := range cols {
433 - if col.isPrimary {
434 - return col.id
435 - }
436 - }
437 - for _, col := range cols {
438 - if col.isLabel {
439 - return col.id
440 - }
441 - }
442 - return ""
443 -}
444 -
445 -func defaultElasticsearchChartGroups(cols []esColumnMeta) []string {
446 - groups := make([]string, 0)
447 - seen := make(map[string]bool)
448 - for _, col := range cols {
449 - if !col.isMetric || col.chartGroup == "" || !col.isDefaultChart {
450 - continue
451 - }
452 - if !seen[col.chartGroup] {
453 - seen[col.chartGroup] = true
454 - groups = append(groups, col.chartGroup)
455 - }
456 - }
457 - if len(groups) > 0 {
458 - return groups
459 - }
460 - for _, col := range cols {
461 - if !col.isMetric || col.chartGroup == "" {
462 - continue
463 - }
464 - if !seen[col.chartGroup] {
465 - seen[col.chartGroup] = true
466 - groups = append(groups, col.chartGroup)
467 - }
468 - }
469 - return groups
470 -}
src/go/plugin/go.d/collector/mongodb/collector.go
+9 -3
@@ -24,8 +24,7 @@ func init() {
24 Create: func() module.Module { return New() },
25 Config: func() any { return &Config{} },
26 Methods: mongoMethods,
27 - MethodParams: mongoMethodParams,
28 - HandleMethod: mongoHandleMethod,
27 + MethodHandler: mongoFunctionHandler,
28 })
29 }
30
@@ -90,6 +89,8 @@ type Collector struct {
89 // Top queries column cache with double-checked locking
90 topQueriesColsMu sync.RWMutex
91 topQueriesCols map[string]bool
92 +
93 + funcRouter *funcRouter
94 }
95
96 func (c *Collector) Configuration() any {
@@ -105,6 +106,8 @@ func (c *Collector) Init(context.Context) error {
106 return fmt.Errorf("init database selector: %v", err)
107 }
108
109 + c.funcRouter = newFuncRouter(c)
110 +
111 return nil
112 }
113
@@ -137,7 +140,10 @@ func (c *Collector) Collect(context.Context) map[string]int64 {
140 return mx
141 }
142
140 -func (c *Collector) Cleanup(context.Context) {
143 +func (c *Collector) Cleanup(ctx context.Context) {
144 + if c.funcRouter != nil {
145 + c.funcRouter.Cleanup(ctx)
146 + }
147 if c.conn == nil {
148 return
149 }
src/go/plugin/go.d/collector/mongodb/func_router.go new
+64
@@ -0,0 +1,64 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package mongo
4 +
5 +import (
6 + "context"
7 + "fmt"
8 +
9 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
10 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
11 +)
12 +
13 +// funcRouter routes method calls to appropriate function handlers.
14 +type funcRouter struct {
15 + collector *Collector
16 +
17 + handlers map[string]funcapi.MethodHandler
18 +}
19 +
20 +func newFuncRouter(c *Collector) *funcRouter {
21 + r := &funcRouter{
22 + collector: c,
23 + handlers: make(map[string]funcapi.MethodHandler),
24 + }
25 + r.handlers[topQueriesMethodID] = newFuncTopQueries(r)
26 + return r
27 +}
28 +
29 +// Compile-time interface check.
30 +var _ funcapi.MethodHandler = (*funcRouter)(nil)
31 +
32 +func (r *funcRouter) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
33 + if h, ok := r.handlers[method]; ok {
34 + return h.MethodParams(ctx, method)
35 + }
36 + return nil, fmt.Errorf("unknown method: %s", method)
37 +}
38 +
39 +func (r *funcRouter) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
40 + if h, ok := r.handlers[method]; ok {
41 + return h.Handle(ctx, method, params)
42 + }
43 + return funcapi.NotFoundResponse(method)
44 +}
45 +
46 +func (r *funcRouter) Cleanup(ctx context.Context) {
47 + for _, h := range r.handlers {
48 + h.Cleanup(ctx)
49 + }
50 +}
51 +
52 +func mongoMethods() []funcapi.MethodConfig {
53 + return []funcapi.MethodConfig{
54 + topQueriesMethodConfig(),
55 + }
56 +}
57 +
58 +func mongoFunctionHandler(job *module.Job) funcapi.MethodHandler {
59 + c, ok := job.Module().(*Collector)
60 + if !ok {
61 + return nil
62 + }
63 + return c.funcRouter
64 +}
src/go/plugin/go.d/collector/mongodb/func_top_queries.go new
+621
@@ -0,0 +1,621 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package mongo
4 +
5 +import (
6 + "context"
7 + "encoding/json"
8 + "errors"
9 + "fmt"
10 + "sort"
11 + "time"
12 +
13 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
14 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
15 +
16 + "go.mongodb.org/mongo-driver/bson"
17 + "go.mongodb.org/mongo-driver/mongo/options"
18 +)
19 +
20 +const (
21 + topQueriesMethodID = "top-queries"
22 + topQueriesMaxTextLength = 4096
23 + topQueriesDefaultLimit = 500
24 + topQueriesHelpText = "Top queries from MongoDB Profiler (system.profile). " +
25 + "WARNING: Query text may contain unmasked literals (potential PII). " +
26 + "Requires profiling enabled on target databases (db.setProfilingLevel)."
27 +)
28 +
29 +func topQueriesMethodConfig() funcapi.MethodConfig {
30 + return funcapi.MethodConfig{
31 + ID: topQueriesMethodID,
32 + Name: "Top Queries",
33 + UpdateEvery: 10,
34 + Help: topQueriesHelpText,
35 + RequireCloud: true,
36 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(topQueriesColumns)},
37 + }
38 +}
39 +
40 +const topQueriesParamSort = "__sort"
41 +
42 +// topQueriesColumn defines metadata for a MongoDB profile column.
43 +// Embeds funcapi.ColumnMeta for UI rendering and adds MongoDB-specific fields.
44 +type topQueriesColumn struct {
45 + funcapi.ColumnMeta
46 +
47 + // DBField is the MongoDB document field name (e.g., "millis")
48 + DBField string
49 + // sortOpt indicates whether this column can be used for sorting in params
50 + sortOpt bool
51 + // sortLbl is the label for the sort option dropdown
52 + sortLbl string
53 + // defaultSort indicates whether this is the default sort column
54 + defaultSort bool
55 +}
56 +
57 +// funcapi.SortableColumn interface implementation for topQueriesColumn.
58 +func (c topQueriesColumn) IsSortOption() bool { return c.sortOpt }
59 +func (c topQueriesColumn) SortLabel() string { return fmt.Sprintf("Top queries by %s", c.sortLbl) }
60 +func (c topQueriesColumn) IsDefaultSort() bool { return c.defaultSort }
61 +func (c topQueriesColumn) ColumnName() string { return c.Name }
62 +func (c topQueriesColumn) SortColumn() string { return c.DBField }
63 +
64 +// topQueriesColumns defines all available columns from system.profile.
65 +// Ordered by display priority (index).
66 +var topQueriesColumns = []topQueriesColumn{
67 + // Core fields (visible by default)
68 + {ColumnMeta: funcapi.ColumnMeta{Name: "timestamp", Tooltip: "Timestamp", Type: funcapi.FieldTypeTimestamp, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualValue, Summary: funcapi.FieldSummaryMax, Transform: funcapi.FieldTransformDatetime, UniqueKey: true}, DBField: "ts", sortOpt: true, sortLbl: "Timestamp"},
69 + {ColumnMeta: funcapi.ColumnMeta{Name: "namespace", Tooltip: "Namespace", Type: funcapi.FieldTypeString, Visible: true, Sortable: false, Sticky: true, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, ExpandFilter: true, GroupBy: &funcapi.GroupByOptions{IsDefault: true}}, DBField: "ns"},
70 + {ColumnMeta: funcapi.ColumnMeta{Name: "operation", Tooltip: "Operation", Type: funcapi.FieldTypeString, Visible: true, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, GroupBy: &funcapi.GroupByOptions{}}, DBField: "op"},
71 + {ColumnMeta: funcapi.ColumnMeta{Name: "query", Tooltip: "Query", Type: funcapi.FieldTypeString, Visible: true, Sortable: false, FullWidth: true, Wrap: true, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText}, DBField: "command"},
72 + {ColumnMeta: funcapi.ColumnMeta{Name: "execution_time", Tooltip: "Execution Time", Type: funcapi.FieldTypeDuration, Units: "seconds", Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualBar, Summary: funcapi.FieldSummarySum, Transform: funcapi.FieldTransformDuration, DecimalPoints: 3, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Execution Time", IsDefault: true}}, DBField: "millis", sortOpt: true, defaultSort: true, sortLbl: "Execution Time"},
73 + {ColumnMeta: funcapi.ColumnMeta{Name: "docs_examined", Tooltip: "Docs Examined", Type: funcapi.FieldTypeInteger, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualValue, Summary: funcapi.FieldSummarySum, Transform: funcapi.FieldTransformNumber, Chart: &funcapi.ChartOptions{Group: "Docs", Title: "Documents"}}, DBField: "docsExamined", sortOpt: true, sortLbl: "Docs Examined"},
74 + {ColumnMeta: funcapi.ColumnMeta{Name: "keys_examined", Tooltip: "Keys Examined", Type: funcapi.FieldTypeInteger, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualValue, Summary: funcapi.FieldSummarySum, Transform: funcapi.FieldTransformNumber, Chart: &funcapi.ChartOptions{Group: "Docs", Title: "Documents"}}, DBField: "keysExamined", sortOpt: true, sortLbl: "Keys Examined"},
75 + {ColumnMeta: funcapi.ColumnMeta{Name: "docs_returned", Tooltip: "Docs Returned", Type: funcapi.FieldTypeInteger, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualValue, Summary: funcapi.FieldSummarySum, Transform: funcapi.FieldTransformNumber, Chart: &funcapi.ChartOptions{Group: "Docs", Title: "Documents"}}, DBField: "nreturned", sortOpt: true, sortLbl: "Docs Returned"},
76 + {ColumnMeta: funcapi.ColumnMeta{Name: "plan_summary", Tooltip: "Plan Summary", Type: funcapi.FieldTypeString, Visible: true, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText}, DBField: "planSummary"},
77 +
78 + // Secondary fields
79 + {ColumnMeta: funcapi.ColumnMeta{Name: "client", Tooltip: "Client", Type: funcapi.FieldTypeString, Visible: true, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, GroupBy: &funcapi.GroupByOptions{}}, DBField: "client"},
80 + {ColumnMeta: funcapi.ColumnMeta{Name: "user", Tooltip: "User", Type: funcapi.FieldTypeString, Visible: true, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, GroupBy: &funcapi.GroupByOptions{}}, DBField: "user"},
81 + {ColumnMeta: funcapi.ColumnMeta{Name: "docs_deleted", Tooltip: "Docs Deleted", Type: funcapi.FieldTypeInteger, Visible: false, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualValue, Summary: funcapi.FieldSummarySum, Transform: funcapi.FieldTransformNumber, Chart: &funcapi.ChartOptions{Group: "Docs", Title: "Documents"}}, DBField: "ndeleted", sortOpt: true, sortLbl: "Docs Deleted"},
82 + {ColumnMeta: funcapi.ColumnMeta{Name: "docs_inserted", Tooltip: "Docs Inserted", Type: funcapi.FieldTypeInteger, Visible: false, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualValue, Summary: funcapi.FieldSummarySum, Transform: funcapi.FieldTransformNumber, Chart: &funcapi.ChartOptions{Group: "Docs", Title: "Documents"}}, DBField: "ninserted", sortOpt: true, sortLbl: "Docs Inserted"},
83 + {ColumnMeta: funcapi.ColumnMeta{Name: "docs_modified", Tooltip: "Docs Modified", Type: funcapi.FieldTypeInteger, Visible: false, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualValue, Summary: funcapi.FieldSummarySum, Transform: funcapi.FieldTransformNumber, Chart: &funcapi.ChartOptions{Group: "Docs", Title: "Documents"}}, DBField: "nModified", sortOpt: true, sortLbl: "Docs Modified"},
84 + {ColumnMeta: funcapi.ColumnMeta{Name: "response_length", Tooltip: "Response Length", Type: funcapi.FieldTypeInteger, Visible: false, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualValue, Summary: funcapi.FieldSummarySum, Transform: funcapi.FieldTransformNumber, Chart: &funcapi.ChartOptions{Group: "Response", Title: "Response Size"}}, DBField: "responseLength", sortOpt: true, sortLbl: "Response Length"},
85 + {ColumnMeta: funcapi.ColumnMeta{Name: "num_yield", Tooltip: "Num Yield", Type: funcapi.FieldTypeInteger, Visible: false, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualValue, Summary: funcapi.FieldSummarySum, Transform: funcapi.FieldTransformNumber, Chart: &funcapi.ChartOptions{Group: "Yield", Title: "Yield"}}, DBField: "numYield", sortOpt: true, sortLbl: "Num Yield"},
86 + {ColumnMeta: funcapi.ColumnMeta{Name: "app_name", Tooltip: "App Name", Type: funcapi.FieldTypeString, Visible: true, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, GroupBy: &funcapi.GroupByOptions{}}, DBField: "appName"},
87 + {ColumnMeta: funcapi.ColumnMeta{Name: "cursor_exhausted", Tooltip: "Cursor Exhausted", Type: funcapi.FieldTypeString, Visible: false, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText}, DBField: "cursorExhausted"},
88 + {ColumnMeta: funcapi.ColumnMeta{Name: "has_sort_stage", Tooltip: "Has Sort Stage", Type: funcapi.FieldTypeString, Visible: false, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText}, DBField: "hasSortStage"},
89 + {ColumnMeta: funcapi.ColumnMeta{Name: "uses_disk", Tooltip: "Uses Disk", Type: funcapi.FieldTypeString, Visible: false, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText}, DBField: "usedDisk"},
90 + {ColumnMeta: funcapi.ColumnMeta{Name: "from_multi_planner", Tooltip: "From Multi Planner", Type: funcapi.FieldTypeString, Visible: false, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText}, DBField: "fromMultiPlanner"},
91 + {ColumnMeta: funcapi.ColumnMeta{Name: "replanned", Tooltip: "Replanned", Type: funcapi.FieldTypeString, Visible: false, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText}, DBField: "replanned"},
92 +
93 + // Version-specific fields (hidden by default)
94 + {ColumnMeta: funcapi.ColumnMeta{Name: "query_hash", Tooltip: "Query Hash", Type: funcapi.FieldTypeString, Visible: false, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText}, DBField: "queryHash"}, // 4.2+
95 + {ColumnMeta: funcapi.ColumnMeta{Name: "plan_cache_key", Tooltip: "Plan Cache Key", Type: funcapi.FieldTypeString, Visible: false, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText}, DBField: "planCacheKey"}, // 4.2+
96 + {ColumnMeta: funcapi.ColumnMeta{Name: "planning_time", Tooltip: "Planning Time", Type: funcapi.FieldTypeDuration, Units: "seconds", Visible: false, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualBar, Summary: funcapi.FieldSummarySum, Transform: funcapi.FieldTransformDuration, DecimalPoints: 3, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Execution Time"}}, DBField: "planningTimeMicros", sortOpt: true, sortLbl: "Planning Time"},
97 + {ColumnMeta: funcapi.ColumnMeta{Name: "cpu_time", Tooltip: "CPU Time", Type: funcapi.FieldTypeDuration, Units: "seconds", Visible: false, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualBar, Summary: funcapi.FieldSummarySum, Transform: funcapi.FieldTransformDuration, DecimalPoints: 3, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Execution Time"}}, DBField: "cpuNanos", sortOpt: true, sortLbl: "CPU Time"},
98 + {ColumnMeta: funcapi.ColumnMeta{Name: "query_framework", Tooltip: "Query Framework", Type: funcapi.FieldTypeString, Visible: false, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText}, DBField: "queryFramework"}, // 7.0+
99 + {ColumnMeta: funcapi.ColumnMeta{Name: "query_shape_hash", Tooltip: "Query Shape Hash", Type: funcapi.FieldTypeString, Visible: false, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText}, DBField: "queryShapeHash"}, // 8.0+
100 +}
101 +
102 +// topQueriesProfileDocument represents a document from system.profile
103 +type topQueriesProfileDocument struct {
104 + // Core fields (always present)
105 + Timestamp time.Time `bson:"ts"`
106 + Op string `bson:"op"`
107 + Ns string `bson:"ns"`
108 + Command bson.M `bson:"command"`
109 + Millis int64 `bson:"millis"`
110 + PlanSummary string `bson:"planSummary"`
111 +
112 + // Common fields
113 + DocsExamined int64 `bson:"docsExamined"`
114 + KeysExamined int64 `bson:"keysExamined"`
115 + Nreturned int64 `bson:"nreturned"`
116 + Client string `bson:"client"`
117 + User string `bson:"user"`
118 + Ndeleted int64 `bson:"ndeleted"`
119 + Ninserted int64 `bson:"ninserted"`
120 + NModified int64 `bson:"nModified"`
121 + ResponseLength int64 `bson:"responseLength"`
122 + NumYield int64 `bson:"numYield"`
123 + AppName string `bson:"appName"`
124 +
125 + // Boolean fields (pointers for nil detection)
126 + CursorExhausted *bool `bson:"cursorExhausted"`
127 + HasSortStage *bool `bson:"hasSortStage"`
128 + UsedDisk *bool `bson:"usedDisk"`
129 + FromMultiPlanner *bool `bson:"fromMultiPlanner"`
130 + Replanned *bool `bson:"replanned"`
131 +
132 + // Version-specific fields
133 + QueryHash string `bson:"queryHash"` // 4.2+
134 + PlanCacheKey string `bson:"planCacheKey"` // 4.2+
135 + PlanningTimeMicros *int64 `bson:"planningTimeMicros"` // 6.2+
136 + CpuNanos *int64 `bson:"cpuNanos"` // 6.3+ Linux only
137 + QueryFramework string `bson:"queryFramework"` // 7.0+
138 + QueryShapeHash string `bson:"queryShapeHash"` // 8.0+
139 +}
140 +
141 +// funcTopQueries implements funcapi.MethodHandler for MongoDB top-queries.
142 +// All function-related logic is encapsulated here, keeping Collector focused on metrics collection.
143 +type funcTopQueries struct {
144 + router *funcRouter
145 +}
146 +
147 +func newFuncTopQueries(r *funcRouter) *funcTopQueries {
148 + return &funcTopQueries{router: r}
149 +}
150 +
151 +// Compile-time interface check.
152 +var _ funcapi.MethodHandler = (*funcTopQueries)(nil)
153 +
154 +func (f *funcTopQueries) Cleanup(ctx context.Context) {}
155 +
156 +// MethodParams implements funcapi.MethodHandler.
157 +func (f *funcTopQueries) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
158 + if f.router.collector.conn == nil {
159 + return nil, fmt.Errorf("collector is still initializing")
160 + }
161 + switch method {
162 + case topQueriesMethodID:
163 + return f.methodParams(ctx)
164 + default:
165 + return nil, fmt.Errorf("unknown method: %s", method)
166 + }
167 +}
168 +
169 +// Handle implements funcapi.MethodHandler.
170 +func (f *funcTopQueries) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
171 + if f.router.collector.conn == nil {
172 + return funcapi.UnavailableResponse("collector is still initializing, please retry in a few seconds")
173 + }
174 +
175 + switch method {
176 + case topQueriesMethodID:
177 + if !f.router.collector.Config.GetTopQueriesFunctionEnabled() {
178 + return funcapi.ErrorResponse(403, "Top Queries function has been disabled in configuration. Set 'top_queries_function_enabled: true' to enable.")
179 + }
180 + return f.collectData(ctx, params.Column(topQueriesParamSort))
181 + default:
182 + return funcapi.NotFoundResponse(method)
183 + }
184 +}
185 +
186 +func (f *funcTopQueries) methodParams(ctx context.Context) ([]funcapi.ParamConfig, error) {
187 + if !f.router.collector.Config.GetTopQueriesFunctionEnabled() {
188 + return nil, fmt.Errorf("top queries function disabled")
189 + }
190 +
191 + databases, err := f.getDatabases()
192 + if err != nil {
193 + return nil, err
194 + }
195 +
196 + availableFields, err := f.detectProfileFields(ctx, databases)
197 + if err != nil {
198 + return nil, err
199 + }
200 +
201 + availableCols := f.buildAvailableColumns(availableFields)
202 + sortParam := funcapi.BuildSortParam(availableCols)
203 + return []funcapi.ParamConfig{sortParam}, nil
204 +}
205 +
206 +func (f *funcTopQueries) collectData(ctx context.Context, sortColumn string) *funcapi.FunctionResponse {
207 + limit := f.router.collector.Config.TopQueriesLimit
208 + if limit <= 0 {
209 + limit = topQueriesDefaultLimit
210 + }
211 +
212 + // Build valid sort columns map from metadata
213 + validSortCols := make(map[string]bool)
214 + for _, col := range topQueriesColumns {
215 + if col.IsSortOption() {
216 + validSortCols[col.DBField] = true
217 + }
218 + }
219 + if !validSortCols[sortColumn] {
220 + sortColumn = "millis" // safe default
221 + }
222 +
223 + databases, err := f.getDatabases()
224 + if err != nil {
225 + return &funcapi.FunctionResponse{
226 + Status: 500,
227 + Message: fmt.Sprintf("failed to list databases: %v", err),
228 + }
229 + }
230 +
231 + // Detect available fields (with caching)
232 + availableFields, err := f.detectProfileFields(ctx, databases)
233 + if err != nil {
234 + f.router.collector.Debugf("failed to detect profile fields: %v", err)
235 + }
236 + availableCols := f.buildAvailableColumns(availableFields)
237 + cs := f.columnSet(availableCols)
238 + sortParam := funcapi.BuildSortParam(availableCols)
239 +
240 + // Query system.profile from each database
241 + var allDocs []topQueriesProfileDocument
242 + var profilingDisabledDBs []string
243 + var failedDBs []string
244 + var successfulDBs int
245 +
246 + for _, dbName := range databases {
247 + docs, enabled, err := f.querySystemProfile(ctx, dbName, sortColumn, limit)
248 + if err != nil {
249 + // Check for timeout (parent or child context)
250 + if ctx.Err() == context.DeadlineExceeded || errors.Is(err, context.DeadlineExceeded) {
251 + return &funcapi.FunctionResponse{Status: 504, Message: "query timed out"}
252 + }
253 + f.router.collector.Debugf("failed to query system.profile in %s: %v", dbName, err)
254 + failedDBs = append(failedDBs, dbName)
255 + continue
256 + }
257 +
258 + if !enabled {
259 + profilingDisabledDBs = append(profilingDisabledDBs, dbName)
260 + continue
261 + }
262 +
263 + successfulDBs++
264 + allDocs = append(allDocs, docs...)
265 + }
266 +
267 + // Check if all databases failed with errors (not just profiling disabled)
268 + if successfulDBs == 0 && len(failedDBs) > 0 && len(profilingDisabledDBs) == 0 {
269 + return &funcapi.FunctionResponse{
270 + Status: 500,
271 + Message: fmt.Sprintf("failed to query all databases: %v", failedDBs),
272 + }
273 + }
274 +
275 + // Check if profiling is disabled everywhere (no successful queries, no errors, only disabled)
276 + if len(allDocs) == 0 && len(profilingDisabledDBs) > 0 && len(failedDBs) == 0 {
277 + return &funcapi.FunctionResponse{
278 + Status: 503,
279 + Message: fmt.Sprintf(
280 + "Database profiling is disabled. Enable it with: db.setProfilingLevel(1, {slowms: 100}). "+
281 + "Disabled databases: %v", profilingDisabledDBs),
282 + }
283 + }
284 +
285 + // Check if we have a mix of failures and disabled profiling (no successful queries at all)
286 + if successfulDBs == 0 && (len(failedDBs) > 0 || len(profilingDisabledDBs) > 0) {
287 + msg := "No databases could be queried successfully."
288 + if len(failedDBs) > 0 {
289 + msg += fmt.Sprintf(" Failed: %v.", failedDBs)
290 + }
291 + if len(profilingDisabledDBs) > 0 {
292 + msg += fmt.Sprintf(" Profiling disabled: %v.", profilingDisabledDBs)
293 + }
294 + return &funcapi.FunctionResponse{
295 + Status: 503,
296 + Message: msg,
297 + }
298 + }
299 +
300 + // Build empty response structure
301 + emptyResponse := &funcapi.FunctionResponse{
302 + Status: 200,
303 + Message: "No slow queries found. Profiling may be disabled or no queries exceeded the slowms threshold.",
304 + Help: topQueriesHelpText,
305 + Columns: cs.BuildColumns(),
306 + Data: [][]any{},
307 + DefaultSortColumn: "execution_time",
308 + RequiredParams: []funcapi.ParamConfig{sortParam},
309 + ChartingConfig: cs.BuildCharting(),
310 + }
311 +
312 + if len(allDocs) == 0 {
313 + return emptyResponse
314 + }
315 +
316 + // Sort all documents by the requested column
317 + f.sortDocuments(allDocs, sortColumn)
318 +
319 + // Apply limit
320 + if len(allDocs) > limit {
321 + allDocs = allDocs[:limit]
322 + }
323 +
324 + // Convert to response format: [][]any (array of arrays, ordered by column index)
325 + data := make([][]any, 0, len(allDocs))
326 + for _, doc := range allDocs {
327 + row := make([]any, len(availableCols))
328 +
329 + // Fill each column based on available columns
330 + for i, col := range availableCols {
331 + switch col.Name {
332 + case "timestamp":
333 + row[i] = doc.Timestamp.Format(time.RFC3339Nano)
334 + case "namespace":
335 + row[i] = doc.Ns
336 + case "operation":
337 + row[i] = doc.Op
338 + case "query":
339 + cmdJSON, err := json.Marshal(doc.Command)
340 + if err != nil {
341 + cmdJSON = []byte("{}")
342 + }
343 + row[i] = strmutil.TruncateText(string(cmdJSON), topQueriesMaxTextLength)
344 + case "execution_time":
345 + row[i] = float64(doc.Millis) / 1000.0 // ms to seconds
346 + case "docs_examined":
347 + row[i] = doc.DocsExamined
348 + case "keys_examined":
349 + row[i] = doc.KeysExamined
350 + case "docs_returned":
351 + row[i] = doc.Nreturned
352 + case "plan_summary":
353 + row[i] = doc.PlanSummary
354 + case "client":
355 + row[i] = doc.Client
356 + case "user":
357 + row[i] = doc.User
358 + case "docs_deleted":
359 + row[i] = doc.Ndeleted
360 + case "docs_inserted":
361 + row[i] = doc.Ninserted
362 + case "docs_modified":
363 + row[i] = doc.NModified
364 + case "response_length":
365 + row[i] = doc.ResponseLength
366 + case "num_yield":
367 + row[i] = doc.NumYield
368 + case "app_name":
369 + row[i] = doc.AppName
370 + case "cursor_exhausted":
371 + row[i] = optionalBool(doc.CursorExhausted)
372 + case "has_sort_stage":
373 + row[i] = optionalBool(doc.HasSortStage)
374 + case "uses_disk":
375 + row[i] = optionalBool(doc.UsedDisk)
376 + case "from_multi_planner":
377 + row[i] = optionalBool(doc.FromMultiPlanner)
378 + case "replanned":
379 + row[i] = optionalBool(doc.Replanned)
380 + case "query_hash":
381 + row[i] = doc.QueryHash
382 + case "plan_cache_key":
383 + row[i] = doc.PlanCacheKey
384 + case "planning_time":
385 + row[i] = optionalDuration(doc.PlanningTimeMicros, 1000000.0) // us to seconds
386 + case "cpu_time":
387 + row[i] = optionalDuration(doc.CpuNanos, 1000000000.0) // ns to seconds
388 + case "query_framework":
389 + row[i] = doc.QueryFramework
390 + case "query_shape_hash":
391 + row[i] = doc.QueryShapeHash
392 + default:
393 + row[i] = nil
394 + }
395 + }
396 +
397 + data = append(data, row)
398 + }
399 +
400 + return &funcapi.FunctionResponse{
401 + Status: 200,
402 + Help: topQueriesHelpText,
403 + Columns: cs.BuildColumns(),
404 + Data: data,
405 + DefaultSortColumn: "execution_time",
406 + RequiredParams: []funcapi.ParamConfig{sortParam},
407 + ChartingConfig: cs.BuildCharting(),
408 + }
409 +}
410 +
411 +func (f *funcTopQueries) columnSet(cols []topQueriesColumn) funcapi.ColumnSet[topQueriesColumn] {
412 + return funcapi.Columns(cols, func(c topQueriesColumn) funcapi.ColumnMeta { return c.ColumnMeta })
413 +}
414 +
415 +func (f *funcTopQueries) getDatabases() ([]string, error) {
416 + databases, err := f.router.collector.conn.listDatabaseNames()
417 + if err != nil {
418 + return nil, err
419 + }
420 +
421 + var filteredDBs []string
422 + for _, dbName := range databases {
423 + if dbName == "admin" || dbName == "local" || dbName == "config" {
424 + continue
425 + }
426 + if f.router.collector.dbSelector != nil && !f.router.collector.dbSelector.MatchString(dbName) {
427 + continue
428 + }
429 + filteredDBs = append(filteredDBs, dbName)
430 + }
431 +
432 + return filteredDBs, nil
433 +}
434 +
435 +// detectProfileFields detects available fields in system.profile using double-checked locking
436 +func (f *funcTopQueries) detectProfileFields(ctx context.Context, databases []string) (map[string]bool, error) {
437 + // Fast path: return cached
438 + f.router.collector.topQueriesColsMu.RLock()
439 + if f.router.collector.topQueriesCols != nil {
440 + cols := f.router.collector.topQueriesCols
441 + f.router.collector.topQueriesColsMu.RUnlock()
442 + return cols, nil
443 + }
444 + f.router.collector.topQueriesColsMu.RUnlock()
445 +
446 + // Slow path: detect and cache
447 + f.router.collector.topQueriesColsMu.Lock()
448 + defer f.router.collector.topQueriesColsMu.Unlock()
449 +
450 + // Double-check after acquiring write lock
451 + if f.router.collector.topQueriesCols != nil {
452 + return f.router.collector.topQueriesCols, nil
453 + }
454 +
455 + client, ok := f.router.collector.conn.(*mongoClient)
456 + if !ok || client == nil || client.client == nil {
457 + return nil, fmt.Errorf("client not initialized")
458 + }
459 +
460 + available := make(map[string]bool)
461 +
462 + // Always include core fields that are guaranteed to exist
463 + coreFields := []string{"ts", "op", "ns", "command", "millis"}
464 + for _, fld := range coreFields {
465 + available[fld] = true
466 + }
467 +
468 + // Sample documents from system.profile to detect available fields
469 + for _, dbName := range databases {
470 + queryCtx, cancel := context.WithTimeout(ctx, client.timeout)
471 +
472 + collection := client.client.Database(dbName).Collection("system.profile")
473 + opts := options.FindOne().SetSort(bson.D{{Key: "$natural", Value: -1}})
474 +
475 + var doc bson.M
476 + err := collection.FindOne(queryCtx, bson.M{}, opts).Decode(&doc)
477 + cancel()
478 +
479 + if err != nil {
480 + continue // No documents or profiling disabled
481 + }
482 +
483 + // Add all fields found in this document
484 + for field := range doc {
485 + available[field] = true
486 + }
487 + }
488 +
489 + f.router.collector.topQueriesCols = available
490 + return available, nil
491 +}
492 +
493 +// buildAvailableColumns returns columns that are available based on detected fields.
494 +func (f *funcTopQueries) buildAvailableColumns(available map[string]bool) []topQueriesColumn {
495 + var result []topQueriesColumn
496 + for _, col := range topQueriesColumns {
497 + // command field maps to query column, always include
498 + if col.DBField == "command" || available[col.DBField] {
499 + result = append(result, col)
500 + }
501 + }
502 + return result
503 +}
504 +
505 +// querySystemProfile queries the system.profile collection for a specific database
506 +func (f *funcTopQueries) querySystemProfile(ctx context.Context, dbName, sortColumn string, limit int) ([]topQueriesProfileDocument, bool, error) {
507 + client, ok := f.router.collector.conn.(*mongoClient)
508 + if !ok || client == nil || client.client == nil {
509 + return nil, false, fmt.Errorf("client not initialized")
510 + }
511 +
512 + // Set timeout for the query
513 + queryCtx, cancel := context.WithTimeout(ctx, client.timeout)
514 + defer cancel()
515 +
516 + // Check if profiling is enabled for this database
517 + var profilingStatus struct {
518 + Was int `bson:"was"`
519 + }
520 + err := client.client.Database(dbName).RunCommand(queryCtx, bson.D{{Key: "profile", Value: -1}}).Decode(&profilingStatus)
521 + if err != nil {
522 + return nil, false, fmt.Errorf("failed to check profiling status: %w", err)
523 + }
524 +
525 + if profilingStatus.Was == 0 {
526 + return nil, false, nil // Profiling disabled
527 + }
528 +
529 + // Query system.profile
530 + collection := client.client.Database(dbName).Collection("system.profile")
531 +
532 + // Build sort order (descending for all except timestamp which can be either)
533 + sortOrder := -1 // descending by default
534 + sortField := sortColumn
535 +
536 + findOpts := options.Find().
537 + SetSort(bson.D{{Key: sortField, Value: sortOrder}}).
538 + SetLimit(int64(limit))
539 +
540 + cursor, err := collection.Find(queryCtx, bson.M{}, findOpts)
541 + if err != nil {
542 + return nil, true, fmt.Errorf("find failed: %w", err)
543 + }
544 + defer cursor.Close(queryCtx)
545 +
546 + var docs []topQueriesProfileDocument
547 + if err := cursor.All(queryCtx, &docs); err != nil {
548 + return nil, true, fmt.Errorf("cursor.All failed: %w", err)
549 + }
550 +
551 + return docs, true, nil
552 +}
553 +
554 +// sortDocuments sorts documents in place by the specified column (descending)
555 +func (f *funcTopQueries) sortDocuments(docs []topQueriesProfileDocument, sortColumn string) {
556 + sort.Slice(docs, func(i, j int) bool {
557 + switch sortColumn {
558 + case "millis":
559 + return docs[i].Millis > docs[j].Millis
560 + case "docsExamined":
561 + return docs[i].DocsExamined > docs[j].DocsExamined
562 + case "keysExamined":
563 + return docs[i].KeysExamined > docs[j].KeysExamined
564 + case "nreturned":
565 + return docs[i].Nreturned > docs[j].Nreturned
566 + case "ts":
567 + return docs[i].Timestamp.After(docs[j].Timestamp)
568 + case "ndeleted":
569 + return docs[i].Ndeleted > docs[j].Ndeleted
570 + case "ninserted":
571 + return docs[i].Ninserted > docs[j].Ninserted
572 + case "nModified":
573 + return docs[i].NModified > docs[j].NModified
574 + case "responseLength":
575 + return docs[i].ResponseLength > docs[j].ResponseLength
576 + case "numYield":
577 + return docs[i].NumYield > docs[j].NumYield
578 + case "planningTimeMicros":
579 + vi := int64(0)
580 + vj := int64(0)
581 + if docs[i].PlanningTimeMicros != nil {
582 + vi = *docs[i].PlanningTimeMicros
583 + }
584 + if docs[j].PlanningTimeMicros != nil {
585 + vj = *docs[j].PlanningTimeMicros
586 + }
587 + return vi > vj
588 + case "cpuNanos":
589 + vi := int64(0)
590 + vj := int64(0)
591 + if docs[i].CpuNanos != nil {
592 + vi = *docs[i].CpuNanos
593 + }
594 + if docs[j].CpuNanos != nil {
595 + vj = *docs[j].CpuNanos
596 + }
597 + return vi > vj
598 + default:
599 + return docs[i].Millis > docs[j].Millis
600 + }
601 + })
602 +}
603 +
604 +// optionalDuration converts an optional int64 pointer to float64 seconds, returning nil if nil.
605 +func optionalDuration(v *int64, divisor float64) any {
606 + if v == nil {
607 + return nil
608 + }
609 + return float64(*v) / divisor
610 +}
611 +
612 +// optionalBool converts a bool pointer to a display value.
613 +func optionalBool(v *bool) any {
614 + if v == nil {
615 + return nil
616 + }
617 + if *v {
618 + return "Yes"
619 + }
620 + return "No"
621 +}
src/go/plugin/go.d/collector/mongodb/func_top_queries_test.go new
+41
@@ -0,0 +1,41 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package mongo
4 +
5 +import (
6 + "testing"
7 +
8 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
9 + "github.com/stretchr/testify/assert"
10 + "github.com/stretchr/testify/require"
11 +)
12 +
13 +func TestMongoMethods(t *testing.T) {
14 + methods := mongoMethods()
15 +
16 + require := require.New(t)
17 + require.Len(methods, 1)
18 + require.Equal("top-queries", methods[0].ID)
19 + require.Equal("Top Queries", methods[0].Name)
20 + require.NotEmpty(methods[0].RequiredParams)
21 +
22 + var sortParam *funcapi.ParamConfig
23 + for i := range methods[0].RequiredParams {
24 + if methods[0].RequiredParams[i].ID == "__sort" {
25 + sortParam = &methods[0].RequiredParams[i]
26 + break
27 + }
28 + }
29 + require.NotNil(sortParam, "expected __sort required param")
30 + require.NotEmpty(sortParam.Options)
31 +}
32 +
33 +func TestTopQueriesColumns_HasRequiredColumns(t *testing.T) {
34 + required := []string{"query", "execution_time", "docs_examined"}
35 +
36 + f := &funcTopQueries{}
37 + cs := f.columnSet(topQueriesColumns)
38 + for _, id := range required {
39 + assert.True(t, cs.ContainsColumn(id), "column %s should be defined", id)
40 + }
41 +}
src/go/plugin/go.d/collector/mongodb/functions.go deleted
-837
@@ -1,837 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -package mongo
4 -
5 -import (
6 - "context"
7 - "encoding/json"
8 - "errors"
9 - "fmt"
10 - "sort"
11 - "time"
12 -
13 - "github.com/netdata/netdata/go/plugins/pkg/funcapi"
14 - "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
15 - "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
16 -
17 - "go.mongodb.org/mongo-driver/bson"
18 - "go.mongodb.org/mongo-driver/mongo/options"
19 -)
20 -
21 -const (
22 - maxQueryTextLength = 4096
23 - defaultTopQueriesLimit = 500
24 - topQueriesHelpText = "Top queries from MongoDB Profiler (system.profile). " +
25 - "WARNING: Query text may contain unmasked literals (potential PII). " +
26 - "Requires profiling enabled on target databases (db.setProfilingLevel)."
27 -)
28 -
29 -const (
30 - paramSort = "__sort"
31 -
32 - ftString = funcapi.FieldTypeString
33 - ftInteger = funcapi.FieldTypeInteger
34 - ftDuration = funcapi.FieldTypeDuration
35 - ftTimestamp = funcapi.FieldTypeTimestamp
36 -
37 - trNumber = funcapi.FieldTransformNumber
38 - trDuration = funcapi.FieldTransformDuration
39 - trDatetime = funcapi.FieldTransformDatetime
40 - trText = funcapi.FieldTransformText
41 -
42 - visValue = funcapi.FieldVisualValue
43 - visBar = funcapi.FieldVisualBar
44 -
45 - summarySum = funcapi.FieldSummarySum
46 - summaryMax = funcapi.FieldSummaryMax
47 -
48 - filterRange = funcapi.FieldFilterRange
49 - filterMulti = funcapi.FieldFilterMultiselect
50 -)
51 -
52 -// mongoColumnMeta defines metadata for a column in the response
53 -type mongoColumnMeta struct {
54 - id string // column ID in response (e.g., "execution_time")
55 - dbField string // MongoDB document field name (e.g., "millis")
56 - name string // display name (e.g., "Execution Time")
57 - colType funcapi.FieldType // column type: integer, duration, timestamp, string, bool
58 - visible bool // default visibility
59 - sortable bool // can be used for sorting
60 - fullWidth bool // for query text columns
61 - wrap bool // wrap text
62 - sticky bool // sticky column
63 - filter funcapi.FieldFilter // filter type: range, multiselect, text
64 - visualization funcapi.FieldVisual // visualization type: value, bar
65 - summary funcapi.FieldSummary // summary type: sum, max, or empty
66 - transform funcapi.FieldTransform // value transform: number, duration, datetime, text
67 - units string // display units (e.g., "seconds")
68 - decimalPoints int // decimal points for numeric display
69 - uniqueKey bool // unique key column
70 - expandFilter bool // default expanded filter
71 - isLabel bool // available for group-by
72 - isPrimary bool // primary label
73 - isMetric bool // chartable metric
74 - chartGroup string // chart group key
75 - chartTitle string // chart title
76 - isDefaultChart bool // include in default charts
77 -}
78 -
79 -// mongoAllColumns defines all available columns from system.profile
80 -// Ordered by display priority (index)
81 -var mongoAllColumns = []mongoColumnMeta{
82 - // Core fields (visible by default)
83 - {id: "timestamp", dbField: "ts", name: "Timestamp", colType: ftTimestamp, visible: true, sortable: true, filter: filterRange, visualization: visValue, summary: summaryMax, transform: trDatetime, uniqueKey: true},
84 - {id: "namespace", dbField: "ns", name: "Namespace", colType: ftString, visible: true, sortable: false, sticky: true, filter: filterMulti, visualization: visValue, transform: trText, expandFilter: true, isLabel: true, isPrimary: true},
85 - {id: "operation", dbField: "op", name: "Operation", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, isLabel: true},
86 - {id: "query", dbField: "command", name: "Query", colType: ftString, visible: true, sortable: false, fullWidth: true, wrap: true, filter: filterMulti, visualization: visValue, transform: trText},
87 - {id: "execution_time", dbField: "millis", name: "Execution Time", colType: ftDuration, visible: true, sortable: true, filter: filterRange, visualization: visBar, summary: summarySum, transform: trDuration, units: "seconds", decimalPoints: 3, isMetric: true, chartGroup: "Time", chartTitle: "Execution Time", isDefaultChart: true},
88 - {id: "docs_examined", dbField: "docsExamined", name: "Docs Examined", colType: ftInteger, visible: true, sortable: true, filter: filterRange, visualization: visValue, summary: summarySum, transform: trNumber, isMetric: true, chartGroup: "Docs", chartTitle: "Documents"},
89 - {id: "keys_examined", dbField: "keysExamined", name: "Keys Examined", colType: ftInteger, visible: true, sortable: true, filter: filterRange, visualization: visValue, summary: summarySum, transform: trNumber, isMetric: true, chartGroup: "Docs", chartTitle: "Documents"},
90 - {id: "docs_returned", dbField: "nreturned", name: "Docs Returned", colType: ftInteger, visible: true, sortable: true, filter: filterRange, visualization: visValue, summary: summarySum, transform: trNumber, isMetric: true, chartGroup: "Docs", chartTitle: "Documents"},
91 - {id: "plan_summary", dbField: "planSummary", name: "Plan Summary", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText},
92 -
93 - // Secondary fields
94 - {id: "client", dbField: "client", name: "Client", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, isLabel: true},
95 - {id: "user", dbField: "user", name: "User", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, isLabel: true},
96 - {id: "docs_deleted", dbField: "ndeleted", name: "Docs Deleted", colType: ftInteger, visible: false, sortable: true, filter: filterRange, visualization: visValue, summary: summarySum, transform: trNumber, isMetric: true, chartGroup: "Docs", chartTitle: "Documents"},
97 - {id: "docs_inserted", dbField: "ninserted", name: "Docs Inserted", colType: ftInteger, visible: false, sortable: true, filter: filterRange, visualization: visValue, summary: summarySum, transform: trNumber, isMetric: true, chartGroup: "Docs", chartTitle: "Documents"},
98 - {id: "docs_modified", dbField: "nModified", name: "Docs Modified", colType: ftInteger, visible: false, sortable: true, filter: filterRange, visualization: visValue, summary: summarySum, transform: trNumber, isMetric: true, chartGroup: "Docs", chartTitle: "Documents"},
99 - {id: "response_length", dbField: "responseLength", name: "Response Length", colType: ftInteger, visible: false, sortable: true, filter: filterRange, visualization: visValue, summary: summarySum, transform: trNumber, isMetric: true, chartGroup: "Response", chartTitle: "Response Size"},
100 - {id: "num_yield", dbField: "numYield", name: "Num Yield", colType: ftInteger, visible: false, sortable: true, filter: filterRange, visualization: visValue, summary: summarySum, transform: trNumber, isMetric: true, chartGroup: "Yield", chartTitle: "Yield"},
101 - {id: "app_name", dbField: "appName", name: "App Name", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, isLabel: true},
102 - {id: "cursor_exhausted", dbField: "cursorExhausted", name: "Cursor Exhausted", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText},
103 - {id: "has_sort_stage", dbField: "hasSortStage", name: "Has Sort Stage", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText},
104 - {id: "uses_disk", dbField: "usedDisk", name: "Uses Disk", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText},
105 - {id: "from_multi_planner", dbField: "fromMultiPlanner", name: "From Multi Planner", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText},
106 - {id: "replanned", dbField: "replanned", name: "Replanned", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText},
107 -
108 - // Version-specific fields (hidden by default)
109 - {id: "query_hash", dbField: "queryHash", name: "Query Hash", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText}, // 4.2+
110 - {id: "plan_cache_key", dbField: "planCacheKey", name: "Plan Cache Key", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText}, // 4.2+
111 - {id: "planning_time", dbField: "planningTimeMicros", name: "Planning Time", colType: ftDuration, visible: false, sortable: true, filter: filterRange, visualization: visBar, summary: summarySum, transform: trDuration, units: "seconds", decimalPoints: 3, isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
112 - {id: "cpu_time", dbField: "cpuNanos", name: "CPU Time", colType: ftDuration, visible: false, sortable: true, filter: filterRange, visualization: visBar, summary: summarySum, transform: trDuration, units: "seconds", decimalPoints: 3, isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
113 - {id: "query_framework", dbField: "queryFramework", name: "Query Framework", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText}, // 7.0+
114 - {id: "query_shape_hash", dbField: "queryShapeHash", name: "Query Shape Hash", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText}, // 8.0+
115 -}
116 -
117 -// optionalDuration converts an optional int64 pointer to float64 seconds, returning nil if nil
118 -func optionalDuration(v *int64, divisor float64) any {
119 - if v == nil {
120 - return nil
121 - }
122 - return float64(*v) / divisor
123 -}
124 -
125 -// optionalBool converts a bool pointer to a display value
126 -func optionalBool(v *bool) any {
127 - if v == nil {
128 - return nil
129 - }
130 - if *v {
131 - return "Yes"
132 - }
133 - return "No"
134 -}
135 -
136 -// topQueriesCharts returns the chart configuration for top queries responses
137 -func topQueriesCharts(cols []mongoColumnMeta) map[string]module.ChartConfig {
138 - charts := make(map[string]module.ChartConfig)
139 - for _, col := range cols {
140 - if !col.isMetric || col.chartGroup == "" {
141 - continue
142 - }
143 - cfg, ok := charts[col.chartGroup]
144 - if !ok {
145 - title := col.chartTitle
146 - if title == "" {
147 - title = col.chartGroup
148 - }
149 - cfg = module.ChartConfig{Name: title, Type: "stacked-bar"}
150 - }
151 - cfg.Columns = append(cfg.Columns, col.id)
152 - charts[col.chartGroup] = cfg
153 - }
154 - return charts
155 -}
156 -
157 -// topQueriesDefaultCharts returns the default chart configuration
158 -func topQueriesDefaultCharts(cols []mongoColumnMeta) [][]string {
159 - label := primaryMongoLabel(cols)
160 - if label == "" {
161 - return nil
162 - }
163 - chartGroups := defaultMongoChartGroups(cols)
164 - out := make([][]string, 0, len(chartGroups))
165 - for _, group := range chartGroups {
166 - out = append(out, []string{group, label})
167 - }
168 - return out
169 -}
170 -
171 -// topQueriesGroupBy returns the group by configuration for top queries responses
172 -func topQueriesGroupBy(cols []mongoColumnMeta) map[string]module.GroupByConfig {
173 - groupBy := make(map[string]module.GroupByConfig)
174 - for _, col := range cols {
175 - if !col.isLabel {
176 - continue
177 - }
178 - groupBy[col.id] = module.GroupByConfig{
179 - Name: "Group by " + col.name,
180 - Columns: []string{col.id},
181 - }
182 - }
183 - return groupBy
184 -}
185 -
186 -func primaryMongoLabel(cols []mongoColumnMeta) string {
187 - for _, col := range cols {
188 - if col.isPrimary {
189 - return col.id
190 - }
191 - }
192 - for _, col := range cols {
193 - if col.isLabel {
194 - return col.id
195 - }
196 - }
197 - return ""
198 -}
199 -
200 -func defaultMongoChartGroups(cols []mongoColumnMeta) []string {
201 - groups := make([]string, 0)
202 - seen := make(map[string]bool)
203 - for _, col := range cols {
204 - if !col.isMetric || col.chartGroup == "" || !col.isDefaultChart {
205 - continue
206 - }
207 - if !seen[col.chartGroup] {
208 - seen[col.chartGroup] = true
209 - groups = append(groups, col.chartGroup)
210 - }
211 - }
212 - if len(groups) > 0 {
213 - return groups
214 - }
215 - for _, col := range cols {
216 - if !col.isMetric || col.chartGroup == "" {
217 - continue
218 - }
219 - if !seen[col.chartGroup] {
220 - seen[col.chartGroup] = true
221 - groups = append(groups, col.chartGroup)
222 - }
223 - }
224 - return groups
225 -}
226 -
227 -func buildMongoSortParam(cols []mongoColumnMeta) funcapi.ParamConfig {
228 - var sortOptions []funcapi.ParamOption
229 - sortDir := funcapi.FieldSortDescending
230 - for _, col := range cols {
231 - if !col.sortable {
232 - continue
233 - }
234 - opt := funcapi.ParamOption{
235 - ID: col.id,
236 - Column: col.dbField,
237 - Name: fmt.Sprintf("Top queries by %s", col.name),
238 - Sort: &sortDir,
239 - }
240 - if col.id == "execution_time" {
241 - opt.Default = true
242 - }
243 - sortOptions = append(sortOptions, opt)
244 - }
245 -
246 - return funcapi.ParamConfig{
247 - ID: paramSort,
248 - Name: "Filter By",
249 - Help: "Select the primary sort column",
250 - Selection: funcapi.ParamSelect,
251 - Options: sortOptions,
252 - UniqueView: true,
253 - }
254 -}
255 -
256 -// mongoMethods returns the available function methods for MongoDB
257 -func mongoMethods() []module.MethodConfig {
258 - // Build sort options from column metadata
259 - var sortOptions []funcapi.ParamOption
260 - sortDir := funcapi.FieldSortDescending
261 - for _, col := range mongoAllColumns {
262 - if !col.sortable {
263 - continue
264 - }
265 - opt := funcapi.ParamOption{
266 - ID: col.id,
267 - Column: col.dbField,
268 - Name: fmt.Sprintf("Top queries by %s", col.name),
269 - Sort: &sortDir,
270 - }
271 - if col.id == "execution_time" {
272 - opt.Default = true
273 - }
274 - sortOptions = append(sortOptions, opt)
275 - }
276 -
277 - return []module.MethodConfig{{
278 - UpdateEvery: 10,
279 - ID: "top-queries",
280 - Name: "Top Queries",
281 - Help: topQueriesHelpText,
282 - RequireCloud: true,
283 - RequiredParams: []funcapi.ParamConfig{
284 - {
285 - ID: paramSort,
286 - Name: "Filter By",
287 - Help: "Select the primary sort column",
288 - Selection: funcapi.ParamSelect,
289 - Options: sortOptions,
290 - UniqueView: true,
291 - },
292 - },
293 - }}
294 -}
295 -
296 -func mongoMethodParams(ctx context.Context, job *module.Job, method string) ([]funcapi.ParamConfig, error) {
297 - collector, ok := job.Module().(*Collector)
298 - if !ok {
299 - return nil, fmt.Errorf("invalid module type")
300 - }
301 - if collector.conn == nil {
302 - return nil, fmt.Errorf("collector is still initializing")
303 - }
304 - switch method {
305 - case "top-queries":
306 - return collector.topQueriesParams(ctx)
307 - default:
308 - return nil, fmt.Errorf("unknown method: %s", method)
309 - }
310 -}
311 -
312 -// mongoHandleMethod handles function requests for MongoDB
313 -func mongoHandleMethod(ctx context.Context, job *module.Job, method string, params funcapi.ResolvedParams) *module.FunctionResponse {
314 - collector, ok := job.Module().(*Collector)
315 - if !ok {
316 - return &module.FunctionResponse{Status: 500, Message: "internal error: invalid module type"}
317 - }
318 -
319 - // Check if collector is initialized
320 - if collector.conn == nil {
321 - return &module.FunctionResponse{
322 - Status: 503,
323 - Message: "collector is still initializing, please retry in a few seconds",
324 - }
325 - }
326 -
327 - switch method {
328 - case "top-queries":
329 - // Check if function is enabled
330 - if !collector.Config.GetTopQueriesFunctionEnabled() {
331 - return &module.FunctionResponse{
332 - Status: 403,
333 - Message: "Top Queries function has been disabled in configuration. Set 'top_queries_function_enabled: true' to enable.",
334 - }
335 - }
336 - return collector.collectTopQueries(ctx, params.Column(paramSort))
337 - default:
338 - return &module.FunctionResponse{Status: 404, Message: fmt.Sprintf("unknown method: %s", method)}
339 - }
340 -}
341 -
342 -// profileDocument represents a document from system.profile
343 -type profileDocument struct {
344 - // Core fields (always present)
345 - Timestamp time.Time `bson:"ts"`
346 - Op string `bson:"op"`
347 - Ns string `bson:"ns"`
348 - Command bson.M `bson:"command"`
349 - Millis int64 `bson:"millis"`
350 - PlanSummary string `bson:"planSummary"`
351 -
352 - // Common fields
353 - DocsExamined int64 `bson:"docsExamined"`
354 - KeysExamined int64 `bson:"keysExamined"`
355 - Nreturned int64 `bson:"nreturned"`
356 - Client string `bson:"client"`
357 - User string `bson:"user"`
358 - Ndeleted int64 `bson:"ndeleted"`
359 - Ninserted int64 `bson:"ninserted"`
360 - NModified int64 `bson:"nModified"`
361 - ResponseLength int64 `bson:"responseLength"`
362 - NumYield int64 `bson:"numYield"`
363 - AppName string `bson:"appName"`
364 -
365 - // Boolean fields (pointers for nil detection)
366 - CursorExhausted *bool `bson:"cursorExhausted"`
367 - HasSortStage *bool `bson:"hasSortStage"`
368 - UsedDisk *bool `bson:"usedDisk"`
369 - FromMultiPlanner *bool `bson:"fromMultiPlanner"`
370 - Replanned *bool `bson:"replanned"`
371 -
372 - // Version-specific fields
373 - QueryHash string `bson:"queryHash"` // 4.2+
374 - PlanCacheKey string `bson:"planCacheKey"` // 4.2+
375 - PlanningTimeMicros *int64 `bson:"planningTimeMicros"` // 6.2+
376 - CpuNanos *int64 `bson:"cpuNanos"` // 6.3+ Linux only
377 - QueryFramework string `bson:"queryFramework"` // 7.0+
378 - QueryShapeHash string `bson:"queryShapeHash"` // 8.0+
379 -}
380 -
381 -// detectMongoProfileFields detects available fields in system.profile using double-checked locking
382 -func (c *Collector) detectMongoProfileFields(ctx context.Context, databases []string) (map[string]bool, error) {
383 - // Fast path: return cached
384 - c.topQueriesColsMu.RLock()
385 - if c.topQueriesCols != nil {
386 - cols := c.topQueriesCols
387 - c.topQueriesColsMu.RUnlock()
388 - return cols, nil
389 - }
390 - c.topQueriesColsMu.RUnlock()
391 -
392 - // Slow path: detect and cache
393 - c.topQueriesColsMu.Lock()
394 - defer c.topQueriesColsMu.Unlock()
395 -
396 - // Double-check after acquiring write lock
397 - if c.topQueriesCols != nil {
398 - return c.topQueriesCols, nil
399 - }
400 -
401 - client, ok := c.conn.(*mongoClient)
402 - if !ok || client == nil || client.client == nil {
403 - return nil, fmt.Errorf("client not initialized")
404 - }
405 -
406 - available := make(map[string]bool)
407 -
408 - // Always include core fields that are guaranteed to exist
409 - coreFields := []string{"ts", "op", "ns", "command", "millis"}
410 - for _, f := range coreFields {
411 - available[f] = true
412 - }
413 -
414 - // Sample documents from system.profile to detect available fields
415 - for _, dbName := range databases {
416 - queryCtx, cancel := context.WithTimeout(ctx, client.timeout)
417 -
418 - collection := client.client.Database(dbName).Collection("system.profile")
419 - opts := options.FindOne().SetSort(bson.D{{Key: "$natural", Value: -1}})
420 -
421 - var doc bson.M
422 - err := collection.FindOne(queryCtx, bson.M{}, opts).Decode(&doc)
423 - cancel()
424 -
425 - if err != nil {
426 - continue // No documents or profiling disabled
427 - }
428 -
429 - // Add all fields found in this document
430 - for field := range doc {
431 - available[field] = true
432 - }
433 - }
434 -
435 - c.topQueriesCols = available
436 - return available, nil
437 -}
438 -
439 -// buildAvailableMongoColumns returns columns that are available based on detected fields
440 -func buildAvailableMongoColumns(available map[string]bool) []mongoColumnMeta {
441 - var result []mongoColumnMeta
442 - for _, col := range mongoAllColumns {
443 - // command field maps to query column, always include
444 - if col.dbField == "command" || available[col.dbField] {
445 - result = append(result, col)
446 - }
447 - }
448 - return result
449 -}
450 -
451 -// buildMongoColumnsFromMeta builds the Columns map for FunctionResponse
452 -func buildMongoColumnsFromMeta(cols []mongoColumnMeta) map[string]any {
453 - result := make(map[string]any)
454 - for i, col := range cols {
455 - sortDir := funcapi.FieldSortDescending
456 - if col.colType == ftString {
457 - sortDir = funcapi.FieldSortAscending
458 - }
459 - colDef := funcapi.Column{
460 - Index: i,
461 - Name: col.name,
462 - Type: col.colType,
463 - Units: col.units,
464 - Visualization: col.visualization,
465 - Sort: sortDir,
466 - Sortable: col.sortable,
467 - Sticky: col.sticky,
468 - Summary: col.summary,
469 - Filter: col.filter,
470 - FullWidth: col.fullWidth,
471 - Wrap: col.wrap,
472 - DefaultExpandedFilter: col.expandFilter,
473 - UniqueKey: col.uniqueKey,
474 - Visible: col.visible,
475 - ValueOptions: funcapi.ValueOptions{
476 - Transform: col.transform,
477 - DecimalPoints: col.decimalPoints,
478 - DefaultValue: nil,
479 - },
480 - }
481 -
482 - result[col.id] = colDef.BuildColumn()
483 - }
484 - return result
485 -}
486 -
487 -// collectTopQueries queries system.profile for top queries across all databases
488 -func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
489 - // Get limit from config
490 - limit := c.Config.TopQueriesLimit
491 - if limit <= 0 {
492 - limit = defaultTopQueriesLimit
493 - }
494 -
495 - // Build valid sort columns map from metadata
496 - validSortCols := make(map[string]bool)
497 - for _, col := range mongoAllColumns {
498 - if col.sortable {
499 - validSortCols[col.dbField] = true
500 - }
501 - }
502 - if !validSortCols[sortColumn] {
503 - sortColumn = "millis" // safe default
504 - }
505 -
506 - // Get list of databases to query
507 - databases, err := c.topQueriesDatabases()
508 - if err != nil {
509 - return &module.FunctionResponse{
510 - Status: 500,
511 - Message: fmt.Sprintf("failed to list databases: %v", err),
512 - }
513 - }
514 - filteredDBs := databases
515 -
516 - // Detect available fields (with caching)
517 - availableFields, err := c.detectMongoProfileFields(ctx, filteredDBs)
518 - if err != nil {
519 - c.Debugf("failed to detect profile fields: %v", err)
520 - }
521 - availableCols := buildAvailableMongoColumns(availableFields)
522 - columns := buildMongoColumnsFromMeta(availableCols)
523 - sortParam := buildMongoSortParam(availableCols)
524 -
525 - // Query system.profile from each database
526 - var allDocs []profileDocument
527 - var profilingDisabledDBs []string
528 - var failedDBs []string
529 - var successfulDBs int
530 -
531 - for _, dbName := range filteredDBs {
532 - docs, enabled, err := c.querySystemProfile(ctx, dbName, sortColumn, limit)
533 - if err != nil {
534 - // Check for timeout (parent or child context)
535 - if ctx.Err() == context.DeadlineExceeded || errors.Is(err, context.DeadlineExceeded) {
536 - return &module.FunctionResponse{Status: 504, Message: "query timed out"}
537 - }
538 - c.Debugf("failed to query system.profile in %s: %v", dbName, err)
539 - failedDBs = append(failedDBs, dbName)
540 - continue
541 - }
542 -
543 - if !enabled {
544 - profilingDisabledDBs = append(profilingDisabledDBs, dbName)
545 - continue
546 - }
547 -
548 - successfulDBs++
549 - allDocs = append(allDocs, docs...)
550 - }
551 -
552 - // Check if all databases failed with errors (not just profiling disabled)
553 - if successfulDBs == 0 && len(failedDBs) > 0 && len(profilingDisabledDBs) == 0 {
554 - return &module.FunctionResponse{
555 - Status: 500,
556 - Message: fmt.Sprintf("failed to query all databases: %v", failedDBs),
557 - }
558 - }
559 -
560 - // Check if profiling is disabled everywhere (no successful queries, no errors, only disabled)
561 - if len(allDocs) == 0 && len(profilingDisabledDBs) > 0 && len(failedDBs) == 0 {
562 - return &module.FunctionResponse{
563 - Status: 503,
564 - Message: fmt.Sprintf(
565 - "Database profiling is disabled. Enable it with: db.setProfilingLevel(1, {slowms: 100}). "+
566 - "Disabled databases: %v", profilingDisabledDBs),
567 - }
568 - }
569 -
570 - // Check if we have a mix of failures and disabled profiling (no successful queries at all)
571 - if successfulDBs == 0 && (len(failedDBs) > 0 || len(profilingDisabledDBs) > 0) {
572 - msg := "No databases could be queried successfully."
573 - if len(failedDBs) > 0 {
574 - msg += fmt.Sprintf(" Failed: %v.", failedDBs)
575 - }
576 - if len(profilingDisabledDBs) > 0 {
577 - msg += fmt.Sprintf(" Profiling disabled: %v.", profilingDisabledDBs)
578 - }
579 - return &module.FunctionResponse{
580 - Status: 503,
581 - Message: msg,
582 - }
583 - }
584 -
585 - // Build empty response structure
586 - emptyResponse := &module.FunctionResponse{
587 - Status: 200,
588 - Message: "No slow queries found. Profiling may be disabled or no queries exceeded the slowms threshold.",
589 - Help: topQueriesHelpText,
590 - Columns: columns,
591 - Data: [][]any{},
592 - DefaultSortColumn: "execution_time",
593 - RequiredParams: []funcapi.ParamConfig{sortParam},
594 - Charts: topQueriesCharts(availableCols),
595 - DefaultCharts: topQueriesDefaultCharts(availableCols),
596 - GroupBy: topQueriesGroupBy(availableCols),
597 - }
598 -
599 - if len(allDocs) == 0 {
600 - return emptyResponse
601 - }
602 -
603 - // Sort all documents by the requested column
604 - sortProfileDocuments(allDocs, sortColumn)
605 -
606 - // Apply limit
607 - if len(allDocs) > limit {
608 - allDocs = allDocs[:limit]
609 - }
610 -
611 - // Convert to response format: [][]any (array of arrays, ordered by column index)
612 - data := make([][]any, 0, len(allDocs))
613 - for _, doc := range allDocs {
614 - row := make([]any, len(availableCols))
615 -
616 - // Fill each column based on available columns
617 - for i, col := range availableCols {
618 - switch col.id {
619 - case "timestamp":
620 - row[i] = doc.Timestamp.Format(time.RFC3339Nano)
621 - case "namespace":
622 - row[i] = doc.Ns
623 - case "operation":
624 - row[i] = doc.Op
625 - case "query":
626 - cmdJSON, err := json.Marshal(doc.Command)
627 - if err != nil {
628 - cmdJSON = []byte("{}")
629 - }
630 - row[i] = strmutil.TruncateText(string(cmdJSON), maxQueryTextLength)
631 - case "execution_time":
632 - row[i] = float64(doc.Millis) / 1000.0 // ms to seconds
633 - case "docs_examined":
634 - row[i] = doc.DocsExamined
635 - case "keys_examined":
636 - row[i] = doc.KeysExamined
637 - case "docs_returned":
638 - row[i] = doc.Nreturned
639 - case "plan_summary":
640 - row[i] = doc.PlanSummary
641 - case "client":
642 - row[i] = doc.Client
643 - case "user":
644 - row[i] = doc.User
645 - case "docs_deleted":
646 - row[i] = doc.Ndeleted
647 - case "docs_inserted":
648 - row[i] = doc.Ninserted
649 - case "docs_modified":
650 - row[i] = doc.NModified
651 - case "response_length":
652 - row[i] = doc.ResponseLength
653 - case "num_yield":
654 - row[i] = doc.NumYield
655 - case "app_name":
656 - row[i] = doc.AppName
657 - case "cursor_exhausted":
658 - row[i] = optionalBool(doc.CursorExhausted)
659 - case "has_sort_stage":
660 - row[i] = optionalBool(doc.HasSortStage)
661 - case "uses_disk":
662 - row[i] = optionalBool(doc.UsedDisk)
663 - case "from_multi_planner":
664 - row[i] = optionalBool(doc.FromMultiPlanner)
665 - case "replanned":
666 - row[i] = optionalBool(doc.Replanned)
667 - case "query_hash":
668 - row[i] = doc.QueryHash
669 - case "plan_cache_key":
670 - row[i] = doc.PlanCacheKey
671 - case "planning_time":
672 - row[i] = optionalDuration(doc.PlanningTimeMicros, 1000000.0) // us to seconds
673 - case "cpu_time":
674 - row[i] = optionalDuration(doc.CpuNanos, 1000000000.0) // ns to seconds
675 - case "query_framework":
676 - row[i] = doc.QueryFramework
677 - case "query_shape_hash":
678 - row[i] = doc.QueryShapeHash
679 - default:
680 - row[i] = nil
681 - }
682 - }
683 -
684 - data = append(data, row)
685 - }
686 -
687 - return &module.FunctionResponse{
688 - Status: 200,
689 - Help: topQueriesHelpText,
690 - Columns: columns,
691 - Data: data,
692 - DefaultSortColumn: "execution_time",
693 - RequiredParams: []funcapi.ParamConfig{sortParam},
694 - Charts: topQueriesCharts(availableCols),
695 - DefaultCharts: topQueriesDefaultCharts(availableCols),
696 - GroupBy: topQueriesGroupBy(availableCols),
697 - }
698 -}
699 -
700 -func (c *Collector) topQueriesDatabases() ([]string, error) {
701 - databases, err := c.conn.listDatabaseNames()
702 - if err != nil {
703 - return nil, err
704 - }
705 -
706 - var filteredDBs []string
707 - for _, dbName := range databases {
708 - if dbName == "admin" || dbName == "local" || dbName == "config" {
709 - continue
710 - }
711 - if c.dbSelector != nil && !c.dbSelector.MatchString(dbName) {
712 - continue
713 - }
714 - filteredDBs = append(filteredDBs, dbName)
715 - }
716 -
717 - return filteredDBs, nil
718 -}
719 -
720 -func (c *Collector) topQueriesParams(ctx context.Context) ([]funcapi.ParamConfig, error) {
721 - if !c.Config.GetTopQueriesFunctionEnabled() {
722 - return nil, fmt.Errorf("top queries function disabled")
723 - }
724 -
725 - databases, err := c.topQueriesDatabases()
726 - if err != nil {
727 - return nil, err
728 - }
729 -
730 - availableFields, err := c.detectMongoProfileFields(ctx, databases)
731 - if err != nil {
732 - return nil, err
733 - }
734 -
735 - availableCols := buildAvailableMongoColumns(availableFields)
736 - sortParam := buildMongoSortParam(availableCols)
737 - return []funcapi.ParamConfig{sortParam}, nil
738 -}
739 -
740 -// querySystemProfile queries the system.profile collection for a specific database
741 -func (c *Collector) querySystemProfile(ctx context.Context, dbName, sortColumn string, limit int) ([]profileDocument, bool, error) {
742 - client, ok := c.conn.(*mongoClient)
743 - if !ok || client == nil || client.client == nil {
744 - return nil, false, fmt.Errorf("client not initialized")
745 - }
746 -
747 - // Set timeout for the query
748 - queryCtx, cancel := context.WithTimeout(ctx, client.timeout)
749 - defer cancel()
750 -
751 - // Check if profiling is enabled for this database
752 - var profilingStatus struct {
753 - Was int `bson:"was"`
754 - }
755 - err := client.client.Database(dbName).RunCommand(queryCtx, bson.D{{Key: "profile", Value: -1}}).Decode(&profilingStatus)
756 - if err != nil {
757 - return nil, false, fmt.Errorf("failed to check profiling status: %w", err)
758 - }
759 -
760 - if profilingStatus.Was == 0 {
761 - return nil, false, nil // Profiling disabled
762 - }
763 -
764 - // Query system.profile
765 - collection := client.client.Database(dbName).Collection("system.profile")
766 -
767 - // Build sort order (descending for all except timestamp which can be either)
768 - sortOrder := -1 // descending by default
769 - sortField := sortColumn
770 -
771 - findOpts := options.Find().
772 - SetSort(bson.D{{Key: sortField, Value: sortOrder}}).
773 - SetLimit(int64(limit))
774 -
775 - cursor, err := collection.Find(queryCtx, bson.M{}, findOpts)
776 - if err != nil {
777 - return nil, true, fmt.Errorf("find failed: %w", err)
778 - }
779 - defer cursor.Close(queryCtx)
780 -
781 - var docs []profileDocument
782 - if err := cursor.All(queryCtx, &docs); err != nil {
783 - return nil, true, fmt.Errorf("cursor.All failed: %w", err)
784 - }
785 -
786 - return docs, true, nil
787 -}
788 -
789 -// sortProfileDocuments sorts documents in place by the specified column (descending)
790 -func sortProfileDocuments(docs []profileDocument, sortColumn string) {
791 - sort.Slice(docs, func(i, j int) bool {
792 - switch sortColumn {
793 - case "millis":
794 - return docs[i].Millis > docs[j].Millis
795 - case "docsExamined":
796 - return docs[i].DocsExamined > docs[j].DocsExamined
797 - case "keysExamined":
798 - return docs[i].KeysExamined > docs[j].KeysExamined
799 - case "nreturned":
800 - return docs[i].Nreturned > docs[j].Nreturned
801 - case "ts":
802 - return docs[i].Timestamp.After(docs[j].Timestamp)
803 - case "ndeleted":
804 - return docs[i].Ndeleted > docs[j].Ndeleted
805 - case "ninserted":
806 - return docs[i].Ninserted > docs[j].Ninserted
807 - case "nModified":
808 - return docs[i].NModified > docs[j].NModified
809 - case "responseLength":
810 - return docs[i].ResponseLength > docs[j].ResponseLength
811 - case "numYield":
812 - return docs[i].NumYield > docs[j].NumYield
813 - case "planningTimeMicros":
814 - vi := int64(0)
815 - vj := int64(0)
816 - if docs[i].PlanningTimeMicros != nil {
817 - vi = *docs[i].PlanningTimeMicros
818 - }
819 - if docs[j].PlanningTimeMicros != nil {
820 - vj = *docs[j].PlanningTimeMicros
821 - }
822 - return vi > vj
823 - case "cpuNanos":
824 - vi := int64(0)
825 - vj := int64(0)
826 - if docs[i].CpuNanos != nil {
827 - vi = *docs[i].CpuNanos
828 - }
829 - if docs[j].CpuNanos != nil {
830 - vj = *docs[j].CpuNanos
831 - }
832 - return vi > vj
833 - default:
834 - return docs[i].Millis > docs[j].Millis
835 - }
836 - })
837 -}
src/go/plugin/go.d/collector/mongodb/functions_test.go deleted
-390
@@ -1,390 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -package mongo
4 -
5 -import (
6 - "testing"
7 - "time"
8 -
9 - "github.com/netdata/netdata/go/plugins/pkg/funcapi"
10 - "github.com/stretchr/testify/assert"
11 - "github.com/stretchr/testify/require"
12 -)
13 -
14 -func TestMongoMethods(t *testing.T) {
15 - methods := mongoMethods()
16 -
17 - assert.Len(t, methods, 1)
18 - assert.Equal(t, "top-queries", methods[0].ID)
19 - assert.Equal(t, "Top Queries", methods[0].Name)
20 - assert.Contains(t, methods[0].Help, "WARNING")
21 - assert.Contains(t, methods[0].Help, "PII")
22 -
23 - var sortParam *funcapi.ParamConfig
24 - for i := range methods[0].RequiredParams {
25 - if methods[0].RequiredParams[i].ID == "__sort" {
26 - sortParam = &methods[0].RequiredParams[i]
27 - break
28 - }
29 - }
30 - require.NotNil(t, sortParam, "should have __sort param")
31 - require.NotEmpty(t, sortParam.Options, "should have sort options")
32 -
33 - // Check default sort option
34 - defaultFound := false
35 - for _, opt := range sortParam.Options {
36 - if opt.Default {
37 - defaultFound = true
38 - assert.Equal(t, "execution_time", opt.ID)
39 - assert.Equal(t, "millis", opt.Column)
40 - }
41 - }
42 - assert.True(t, defaultFound, "should have a default sort option")
43 -}
44 -
45 -func TestMongoColumnMeta(t *testing.T) {
46 - // Check that mongoAllColumns has all expected columns
47 - assert.NotEmpty(t, mongoAllColumns, "should have column definitions")
48 -
49 - // Check required core columns exist
50 - coreColumns := []string{"timestamp", "namespace", "operation", "query", "execution_time", "docs_examined", "keys_examined", "docs_returned", "plan_summary"}
51 - for _, colID := range coreColumns {
52 - found := false
53 - for _, col := range mongoAllColumns {
54 - if col.id == colID {
55 - found = true
56 - break
57 - }
58 - }
59 - assert.True(t, found, "core column %s should exist", colID)
60 - }
61 -
62 - // Check that core columns are visible by default
63 - for _, col := range mongoAllColumns {
64 - switch col.id {
65 - case "timestamp", "namespace", "operation", "query", "execution_time", "docs_examined", "keys_examined", "docs_returned", "plan_summary":
66 - assert.True(t, col.visible, "column %s should be visible by default", col.id)
67 - }
68 - }
69 -}
70 -
71 -func TestBuildAvailableMongoColumns(t *testing.T) {
72 - tests := []struct {
73 - name string
74 - available map[string]bool
75 - expectLen int
76 - }{
77 - {
78 - name: "core fields only",
79 - available: map[string]bool{
80 - "ts": true, "ns": true, "op": true, "command": true, "millis": true,
81 - },
82 - expectLen: 5, // timestamp, namespace, operation, query, execution_time
83 - },
84 - {
85 - name: "with extra fields",
86 - available: map[string]bool{
87 - "ts": true, "ns": true, "op": true, "command": true, "millis": true,
88 - "docsExamined": true, "keysExamined": true, "planSummary": true,
89 - },
90 - expectLen: 8,
91 - },
92 - {
93 - name: "all fields",
94 - available: func() map[string]bool {
95 - m := make(map[string]bool)
96 - for _, col := range mongoAllColumns {
97 - m[col.dbField] = true
98 - }
99 - return m
100 - }(),
101 - expectLen: len(mongoAllColumns),
102 - },
103 - }
104 -
105 - for _, tc := range tests {
106 - t.Run(tc.name, func(t *testing.T) {
107 - cols := buildAvailableMongoColumns(tc.available)
108 - assert.Len(t, cols, tc.expectLen)
109 - })
110 - }
111 -}
112 -
113 -func TestBuildMongoColumnsFromMeta(t *testing.T) {
114 - // Use a subset of columns for testing
115 - testCols := []mongoColumnMeta{
116 - {id: "timestamp", dbField: "ts", name: "Timestamp", colType: ftTimestamp, visible: true, sortable: true, filter: filterRange, visualization: visValue, summary: summaryMax, transform: trDatetime, uniqueKey: true},
117 - {id: "execution_time", dbField: "millis", name: "Execution Time", colType: ftDuration, visible: true, sortable: true, filter: filterRange, visualization: visBar, summary: summarySum, transform: trDuration, units: "seconds", decimalPoints: 3},
118 - {id: "query", dbField: "command", name: "Query", colType: ftString, visible: true, sortable: false, fullWidth: true, wrap: true, filter: filterMulti, visualization: visValue, transform: trText},
119 - }
120 -
121 - cols := buildMongoColumnsFromMeta(testCols)
122 -
123 - // Check required columns exist
124 - assert.Contains(t, cols, "timestamp")
125 - assert.Contains(t, cols, "execution_time")
126 - assert.Contains(t, cols, "query")
127 -
128 - // Check execution_time column properties
129 - execTimeVal, ok := cols["execution_time"]
130 - require.True(t, ok, "execution_time column must exist")
131 - execTime, ok := execTimeVal.(map[string]any)
132 - require.True(t, ok, "execution_time must be map[string]any")
133 - assert.Equal(t, "duration", execTime["type"])
134 - assert.Equal(t, true, execTime["visible"])
135 - assert.Equal(t, "seconds", execTime["units"])
136 -
137 - // Check query column properties
138 - queryVal, ok := cols["query"]
139 - require.True(t, ok, "query column must exist")
140 - query, ok := queryVal.(map[string]any)
141 - require.True(t, ok, "query must be map[string]any")
142 - assert.Equal(t, true, query["full_width"])
143 - assert.Equal(t, false, query["sortable"])
144 -}
145 -
146 -func TestSortProfileDocuments(t *testing.T) {
147 - docs := []profileDocument{
148 - {Millis: 100, DocsExamined: 50, KeysExamined: 10, Nreturned: 5},
149 - {Millis: 500, DocsExamined: 200, KeysExamined: 30, Nreturned: 20},
150 - {Millis: 200, DocsExamined: 100, KeysExamined: 20, Nreturned: 10},
151 - }
152 -
153 - tests := []struct {
154 - name string
155 - sortColumn string
156 - expected []int64 // expected order of millis values after sort
157 - }{
158 - {
159 - name: "sort by millis",
160 - sortColumn: "millis",
161 - expected: []int64{500, 200, 100},
162 - },
163 - {
164 - name: "sort by docsExamined",
165 - sortColumn: "docsExamined",
166 - expected: []int64{500, 200, 100}, // 200, 100, 50 -> millis 500, 200, 100
167 - },
168 - }
169 -
170 - for _, tc := range tests {
171 - t.Run(tc.name, func(t *testing.T) {
172 - // Make a copy to avoid modifying original
173 - docsCopy := make([]profileDocument, len(docs))
174 - copy(docsCopy, docs)
175 -
176 - sortProfileDocuments(docsCopy, tc.sortColumn)
177 -
178 - for i, expectedMillis := range tc.expected {
179 - assert.Equal(t, expectedMillis, docsCopy[i].Millis,
180 - "position %d should have millis=%d", i, expectedMillis)
181 - }
182 - })
183 - }
184 -}
185 -
186 -func TestSortProfileDocumentsByTimestamp(t *testing.T) {
187 - now := time.Now()
188 - docs := []profileDocument{
189 - {Timestamp: now.Add(-time.Hour), Millis: 100},
190 - {Timestamp: now, Millis: 200},
191 - {Timestamp: now.Add(-30 * time.Minute), Millis: 300},
192 - }
193 -
194 - sortProfileDocuments(docs, "ts")
195 -
196 - // Should be sorted by timestamp descending (most recent first)
197 - assert.Equal(t, int64(200), docs[0].Millis) // now
198 - assert.Equal(t, int64(300), docs[1].Millis) // -30min
199 - assert.Equal(t, int64(100), docs[2].Millis) // -1hr
200 -}
201 -
202 -func TestSortProfileDocumentsByNewColumns(t *testing.T) {
203 - docs := []profileDocument{
204 - {Millis: 100, Ndeleted: 5, Ninserted: 10, NModified: 15, ResponseLength: 100, NumYield: 1},
205 - {Millis: 200, Ndeleted: 15, Ninserted: 5, NModified: 10, ResponseLength: 300, NumYield: 3},
206 - {Millis: 300, Ndeleted: 10, Ninserted: 15, NModified: 5, ResponseLength: 200, NumYield: 2},
207 - }
208 -
209 - tests := []struct {
210 - name string
211 - sortColumn string
212 - expected []int64 // expected order of millis values after sort
213 - }{
214 - {
215 - name: "sort by ndeleted",
216 - sortColumn: "ndeleted",
217 - expected: []int64{200, 300, 100}, // 15, 10, 5
218 - },
219 - {
220 - name: "sort by ninserted",
221 - sortColumn: "ninserted",
222 - expected: []int64{300, 100, 200}, // 15, 10, 5
223 - },
224 - {
225 - name: "sort by nModified",
226 - sortColumn: "nModified",
227 - expected: []int64{100, 200, 300}, // 15, 10, 5
228 - },
229 - {
230 - name: "sort by responseLength",
231 - sortColumn: "responseLength",
232 - expected: []int64{200, 300, 100}, // 300, 200, 100
233 - },
234 - {
235 - name: "sort by numYield",
236 - sortColumn: "numYield",
237 - expected: []int64{200, 300, 100}, // 3, 2, 1
238 - },
239 - }
240 -
241 - for _, tc := range tests {
242 - t.Run(tc.name, func(t *testing.T) {
243 - docsCopy := make([]profileDocument, len(docs))
244 - copy(docsCopy, docs)
245 -
246 - sortProfileDocuments(docsCopy, tc.sortColumn)
247 -
248 - for i, expectedMillis := range tc.expected {
249 - assert.Equal(t, expectedMillis, docsCopy[i].Millis,
250 - "position %d should have millis=%d", i, expectedMillis)
251 - }
252 - })
253 - }
254 -}
255 -
256 -func TestSortProfileDocumentsByOptionalColumns(t *testing.T) {
257 - pt1 := int64(100)
258 - pt2 := int64(300)
259 - pt3 := int64(200)
260 - cpu1 := int64(1000)
261 - cpu2 := int64(3000)
262 - cpu3 := int64(2000)
263 -
264 - docs := []profileDocument{
265 - {Millis: 100, PlanningTimeMicros: &pt1, CpuNanos: &cpu1},
266 - {Millis: 200, PlanningTimeMicros: &pt2, CpuNanos: &cpu2},
267 - {Millis: 300, PlanningTimeMicros: &pt3, CpuNanos: &cpu3},
268 - }
269 -
270 - t.Run("sort by planningTimeMicros", func(t *testing.T) {
271 - docsCopy := make([]profileDocument, len(docs))
272 - copy(docsCopy, docs)
273 -
274 - sortProfileDocuments(docsCopy, "planningTimeMicros")
275 -
276 - // Should be sorted by planningTimeMicros descending: 300, 200, 100
277 - assert.Equal(t, int64(200), docsCopy[0].Millis) // pt2=300
278 - assert.Equal(t, int64(300), docsCopy[1].Millis) // pt3=200
279 - assert.Equal(t, int64(100), docsCopy[2].Millis) // pt1=100
280 - })
281 -
282 - t.Run("sort by cpuNanos", func(t *testing.T) {
283 - docsCopy := make([]profileDocument, len(docs))
284 - copy(docsCopy, docs)
285 -
286 - sortProfileDocuments(docsCopy, "cpuNanos")
287 -
288 - // Should be sorted by cpuNanos descending: 3000, 2000, 1000
289 - assert.Equal(t, int64(200), docsCopy[0].Millis) // cpu2=3000
290 - assert.Equal(t, int64(300), docsCopy[1].Millis) // cpu3=2000
291 - assert.Equal(t, int64(100), docsCopy[2].Millis) // cpu1=1000
292 - })
293 -
294 - t.Run("sort by planningTimeMicros with nil values", func(t *testing.T) {
295 - pt := int64(100)
296 - docsWithNil := []profileDocument{
297 - {Millis: 100, PlanningTimeMicros: nil},
298 - {Millis: 200, PlanningTimeMicros: &pt},
299 - {Millis: 300, PlanningTimeMicros: nil},
300 - }
301 -
302 - sortProfileDocuments(docsWithNil, "planningTimeMicros")
303 -
304 - // Non-nil values should come first when sorted descending
305 - assert.Equal(t, int64(200), docsWithNil[0].Millis) // has pt=100
306 - })
307 -}
308 -
309 -func TestConfigGetTopQueriesFunctionEnabled(t *testing.T) {
310 - t.Run("nil returns true (default)", func(t *testing.T) {
311 - cfg := Config{TopQueriesFunctionEnabled: nil}
312 - assert.True(t, cfg.GetTopQueriesFunctionEnabled())
313 - })
314 -
315 - t.Run("explicit true returns true", func(t *testing.T) {
316 - enabled := true
317 - cfg := Config{TopQueriesFunctionEnabled: &enabled}
318 - assert.True(t, cfg.GetTopQueriesFunctionEnabled())
319 - })
320 -
321 - t.Run("explicit false returns false", func(t *testing.T) {
322 - disabled := false
323 - cfg := Config{TopQueriesFunctionEnabled: &disabled}
324 - assert.False(t, cfg.GetTopQueriesFunctionEnabled())
325 - })
326 -}
327 -
328 -func TestTopQueriesLimitDefault(t *testing.T) {
329 - // When TopQueriesLimit is 0 or negative, should use default
330 - assert.Equal(t, 500, defaultTopQueriesLimit)
331 -
332 - cfg := Config{TopQueriesLimit: 0}
333 - // collectTopQueries would use defaultTopQueriesLimit when config is 0
334 - limit := cfg.TopQueriesLimit
335 - if limit <= 0 {
336 - limit = defaultTopQueriesLimit
337 - }
338 - assert.Equal(t, 500, limit)
339 -
340 - cfg2 := Config{TopQueriesLimit: 100}
341 - limit2 := cfg2.TopQueriesLimit
342 - if limit2 <= 0 {
343 - limit2 = defaultTopQueriesLimit
344 - }
345 - assert.Equal(t, 100, limit2)
346 -}
347 -
348 -func TestOptionalBool(t *testing.T) {
349 - t.Run("nil returns nil", func(t *testing.T) {
350 - result := optionalBool(nil)
351 - assert.Nil(t, result)
352 - })
353 -
354 - t.Run("true returns Yes", func(t *testing.T) {
355 - v := true
356 - result := optionalBool(&v)
357 - assert.Equal(t, "Yes", result)
358 - })
359 -
360 - t.Run("false returns No", func(t *testing.T) {
361 - v := false
362 - result := optionalBool(&v)
363 - assert.Equal(t, "No", result)
364 - })
365 -}
366 -
367 -func TestOptionalDuration(t *testing.T) {
368 - t.Run("nil returns nil", func(t *testing.T) {
369 - result := optionalDuration(nil, 1000.0)
370 - assert.Nil(t, result)
371 - })
372 -
373 - t.Run("value converts correctly", func(t *testing.T) {
374 - v := int64(1000000)
375 - result := optionalDuration(&v, 1000000.0)
376 - assert.Equal(t, 1.0, result)
377 - })
378 -
379 - t.Run("microseconds to seconds", func(t *testing.T) {
380 - v := int64(500000) // 500,000 microseconds = 0.5 seconds
381 - result := optionalDuration(&v, 1000000.0)
382 - assert.Equal(t, 0.5, result)
383 - })
384 -
385 - t.Run("nanoseconds to seconds", func(t *testing.T) {
386 - v := int64(1500000000) // 1.5 billion nanoseconds = 1.5 seconds
387 - result := optionalDuration(&v, 1000000000.0)
388 - assert.Equal(t, 1.5, result)
389 - })
390 -}
src/go/plugin/go.d/collector/mssql/collector.go
+13 -6
@@ -25,11 +25,10 @@ func init() {
25 Defaults: module.Defaults{
26 UpdateEvery: 10,
27 },
28 - Create: func() module.Module { return New() },
29 - Config: func() any { return &Config{} },
30 - Methods: mssqlMethods,
31 - MethodParams: mssqlMethodParams,
32 - HandleMethod: mssqlHandleMethod,
28 + Create: func() module.Module { return New() },
29 + Config: func() any { return &Config{} },
30 + Methods: mssqlMethods,
31 + MethodHandler: mssqlFunctionHandler,
32 })
33 }
34
@@ -112,6 +111,8 @@ type Collector struct {
111 // Query Store column cache (per-instance to handle different SQL Server versions)
112 queryStoreColsMu sync.RWMutex // protects queryStoreCols for concurrent access
113 queryStoreCols map[string]bool
114 +
115 + funcRouter *funcRouter
116 }
117
118 func (c *Collector) Configuration() any {
@@ -123,6 +124,9 @@ func (c *Collector) Init(context.Context) error {
124 return errors.New("config: dsn not set")
125 }
126 c.Debugf("using DSN [%s]", c.DSN)
127 +
128 + c.funcRouter = newFuncRouter(c)
129 +
130 return nil
131 }
132
@@ -150,7 +154,10 @@ func (c *Collector) Collect(context.Context) map[string]int64 {
154 return mx
155 }
156
153 -func (c *Collector) Cleanup(context.Context) {
157 +func (c *Collector) Cleanup(ctx context.Context) {
158 + if c.funcRouter != nil {
159 + c.funcRouter.Cleanup(ctx)
160 + }
161 if c.db == nil {
162 return
163 }
src/go/plugin/go.d/collector/mssql/func_router.go new
+64
@@ -0,0 +1,64 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package mssql
4 +
5 +import (
6 + "context"
7 + "fmt"
8 +
9 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
10 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
11 +)
12 +
13 +// funcRouter routes method calls to appropriate function handlers.
14 +type funcRouter struct {
15 + collector *Collector
16 +
17 + handlers map[string]funcapi.MethodHandler
18 +}
19 +
20 +func newFuncRouter(c *Collector) *funcRouter {
21 + r := &funcRouter{
22 + collector: c,
23 + handlers: make(map[string]funcapi.MethodHandler),
24 + }
25 + r.handlers[topQueriesMethodID] = newFuncTopQueries(r)
26 + return r
27 +}
28 +
29 +// Compile-time interface check.
30 +var _ funcapi.MethodHandler = (*funcRouter)(nil)
31 +
32 +func (r *funcRouter) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
33 + if h, ok := r.handlers[method]; ok {
34 + return h.MethodParams(ctx, method)
35 + }
36 + return nil, fmt.Errorf("unknown method: %s", method)
37 +}
38 +
39 +func (r *funcRouter) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
40 + if h, ok := r.handlers[method]; ok {
41 + return h.Handle(ctx, method, params)
42 + }
43 + return funcapi.NotFoundResponse(method)
44 +}
45 +
46 +func (r *funcRouter) Cleanup(ctx context.Context) {
47 + for _, h := range r.handlers {
48 + h.Cleanup(ctx)
49 + }
50 +}
51 +
52 +func mssqlMethods() []funcapi.MethodConfig {
53 + return []funcapi.MethodConfig{
54 + topQueriesMethodConfig(),
55 + }
56 +}
57 +
58 +func mssqlFunctionHandler(job *module.Job) funcapi.MethodHandler {
59 + c, ok := job.Module().(*Collector)
60 + if !ok {
61 + return nil
62 + }
63 + return c.funcRouter
64 +}
src/go/plugin/go.d/collector/mssql/func_top_queries.go new
+664
@@ -0,0 +1,664 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package mssql
4 +
5 +import (
6 + "context"
7 + "database/sql"
8 + "fmt"
9 + "strings"
10 +
11 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
12 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
13 +)
14 +
15 +const (
16 + topQueriesMethodID = "top-queries"
17 + topQueriesMaxTextLength = 4096
18 + topQueriesParamSort = "__sort"
19 +)
20 +
21 +func topQueriesMethodConfig() funcapi.MethodConfig {
22 + return funcapi.MethodConfig{
23 + ID: topQueriesMethodID,
24 + Name: "Top Queries",
25 + UpdateEvery: 10,
26 + Help: "Top SQL queries from Query Store. WARNING: Query text may contain unmasked literals (potential PII).",
27 + RequireCloud: true,
28 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(topQueriesColumns)},
29 + }
30 +}
31 +
32 +// topQueriesColumn embeds funcapi.ColumnMeta and adds MSSQL-specific fields.
33 +type topQueriesColumn struct {
34 + funcapi.ColumnMeta
35 + DBColumn string // Column name in sys.query_store_runtime_stats
36 + IsMicroseconds bool // Needs microseconds to milliseconds conversion
37 + sortOpt bool // Show in sort dropdown
38 + sortLbl string // Label for sort option
39 + defaultSort bool // Is this the default sort option
40 + IsIdentity bool // Is this an identity column (query_hash, query_text, etc.)
41 + NeedsAvg bool // Needs weighted average calculation (avg_* columns)
42 +}
43 +
44 +// topQueriesColumns defines ALL possible columns from Query Store.
45 +// Columns that don't exist in certain SQL Server versions will be filtered at runtime.
46 +var topQueriesColumns = []topQueriesColumn{
47 + // Identity columns - always available
48 + {ColumnMeta: funcapi.ColumnMeta{Name: "queryHash", Tooltip: "Query Hash", Type: funcapi.FieldTypeString, Visible: false, Transform: funcapi.FieldTransformNone, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, UniqueKey: true}, DBColumn: "query_hash", IsIdentity: true},
49 + {ColumnMeta: funcapi.ColumnMeta{Name: "query", Tooltip: "Query", Type: funcapi.FieldTypeString, Visible: true, Transform: funcapi.FieldTransformNone, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, Sticky: true, FullWidth: true}, DBColumn: "query_sql_text", IsIdentity: true},
50 + {ColumnMeta: funcapi.ColumnMeta{Name: "database", Tooltip: "Database", Type: funcapi.FieldTypeString, Visible: true, Transform: funcapi.FieldTransformNone, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect}, DBColumn: "database_name", IsIdentity: true},
51 +
52 + // Execution count - always available
53 + {ColumnMeta: funcapi.ColumnMeta{Name: "calls", Tooltip: "Calls", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange}, DBColumn: "count_executions", sortOpt: true, sortLbl: "Top queries by Number of Calls"},
54 +
55 + // Duration metrics (microseconds -> milliseconds) - SQL 2016+
56 + {ColumnMeta: funcapi.ColumnMeta{Name: "totalTime", Tooltip: "Total Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: true, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange}, DBColumn: "avg_duration", IsMicroseconds: true, sortOpt: true, sortLbl: "Top queries by Total Execution Time", defaultSort: true},
57 + {ColumnMeta: funcapi.ColumnMeta{Name: "avgTime", Tooltip: "Avg Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: true, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMean, Filter: funcapi.FieldFilterRange}, DBColumn: "avg_duration", IsMicroseconds: true, NeedsAvg: true, sortOpt: true, sortLbl: "Top queries by Average Execution Time"},
58 + {ColumnMeta: funcapi.ColumnMeta{Name: "lastTime", Tooltip: "Last Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: false, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "last_duration", IsMicroseconds: true},
59 + {ColumnMeta: funcapi.ColumnMeta{Name: "minTime", Tooltip: "Min Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: false, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMin, Filter: funcapi.FieldFilterRange}, DBColumn: "min_duration", IsMicroseconds: true},
60 + {ColumnMeta: funcapi.ColumnMeta{Name: "maxTime", Tooltip: "Max Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: false, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "max_duration", IsMicroseconds: true},
61 + {ColumnMeta: funcapi.ColumnMeta{Name: "stdevTime", Tooltip: "StdDev Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: false, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "stdev_duration", IsMicroseconds: true},
62 +
63 + // CPU time metrics (microseconds -> milliseconds)
64 + {ColumnMeta: funcapi.ColumnMeta{Name: "avgCpu", Tooltip: "Avg CPU", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: true, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMean, Filter: funcapi.FieldFilterRange}, DBColumn: "avg_cpu_time", IsMicroseconds: true, NeedsAvg: true, sortOpt: true, sortLbl: "Top queries by Average CPU Time"},
65 + {ColumnMeta: funcapi.ColumnMeta{Name: "lastCpu", Tooltip: "Last CPU", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: false, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "last_cpu_time", IsMicroseconds: true},
66 + {ColumnMeta: funcapi.ColumnMeta{Name: "minCpu", Tooltip: "Min CPU", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: false, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMin, Filter: funcapi.FieldFilterRange}, DBColumn: "min_cpu_time", IsMicroseconds: true},
67 + {ColumnMeta: funcapi.ColumnMeta{Name: "maxCpu", Tooltip: "Max CPU", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: false, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "max_cpu_time", IsMicroseconds: true},
68 + {ColumnMeta: funcapi.ColumnMeta{Name: "stdevCpu", Tooltip: "StdDev CPU", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: false, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "stdev_cpu_time", IsMicroseconds: true},
69 +
70 + // Logical I/O reads
71 + {ColumnMeta: funcapi.ColumnMeta{Name: "avgReads", Tooltip: "Avg Logical Reads", Type: funcapi.FieldTypeFloat, DecimalPoints: 0, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMean, Filter: funcapi.FieldFilterRange}, DBColumn: "avg_logical_io_reads", NeedsAvg: true, sortOpt: true, sortLbl: "Top queries by Average Logical Reads"},
72 + {ColumnMeta: funcapi.ColumnMeta{Name: "lastReads", Tooltip: "Last Logical Reads", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "last_logical_io_reads"},
73 + {ColumnMeta: funcapi.ColumnMeta{Name: "minReads", Tooltip: "Min Logical Reads", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMin, Filter: funcapi.FieldFilterRange}, DBColumn: "min_logical_io_reads"},
74 + {ColumnMeta: funcapi.ColumnMeta{Name: "maxReads", Tooltip: "Max Logical Reads", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "max_logical_io_reads"},
75 + {ColumnMeta: funcapi.ColumnMeta{Name: "stdevReads", Tooltip: "StdDev Logical Reads", Type: funcapi.FieldTypeFloat, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "stdev_logical_io_reads"},
76 +
77 + // Logical I/O writes
78 + {ColumnMeta: funcapi.ColumnMeta{Name: "avgWrites", Tooltip: "Avg Logical Writes", Type: funcapi.FieldTypeFloat, DecimalPoints: 0, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMean, Filter: funcapi.FieldFilterRange}, DBColumn: "avg_logical_io_writes", NeedsAvg: true, sortOpt: true, sortLbl: "Top queries by Average Logical Writes"},
79 + {ColumnMeta: funcapi.ColumnMeta{Name: "lastWrites", Tooltip: "Last Logical Writes", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "last_logical_io_writes"},
80 + {ColumnMeta: funcapi.ColumnMeta{Name: "minWrites", Tooltip: "Min Logical Writes", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMin, Filter: funcapi.FieldFilterRange}, DBColumn: "min_logical_io_writes"},
81 + {ColumnMeta: funcapi.ColumnMeta{Name: "maxWrites", Tooltip: "Max Logical Writes", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "max_logical_io_writes"},
82 + {ColumnMeta: funcapi.ColumnMeta{Name: "stdevWrites", Tooltip: "StdDev Logical Writes", Type: funcapi.FieldTypeFloat, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "stdev_logical_io_writes"},
83 +
84 + // Physical I/O reads
85 + {ColumnMeta: funcapi.ColumnMeta{Name: "avgPhysReads", Tooltip: "Avg Physical Reads", Type: funcapi.FieldTypeFloat, DecimalPoints: 0, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMean, Filter: funcapi.FieldFilterRange}, DBColumn: "avg_physical_io_reads", NeedsAvg: true, sortOpt: true, sortLbl: "Top queries by Average Physical Reads"},
86 + {ColumnMeta: funcapi.ColumnMeta{Name: "lastPhysReads", Tooltip: "Last Physical Reads", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "last_physical_io_reads"},
87 + {ColumnMeta: funcapi.ColumnMeta{Name: "minPhysReads", Tooltip: "Min Physical Reads", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMin, Filter: funcapi.FieldFilterRange}, DBColumn: "min_physical_io_reads"},
88 + {ColumnMeta: funcapi.ColumnMeta{Name: "maxPhysReads", Tooltip: "Max Physical Reads", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "max_physical_io_reads"},
89 + {ColumnMeta: funcapi.ColumnMeta{Name: "stdevPhysReads", Tooltip: "StdDev Physical Reads", Type: funcapi.FieldTypeFloat, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "stdev_physical_io_reads"},
90 +
91 + // CLR time (microseconds -> milliseconds)
92 + {ColumnMeta: funcapi.ColumnMeta{Name: "avgClr", Tooltip: "Avg CLR Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: false, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMean, Filter: funcapi.FieldFilterRange}, DBColumn: "avg_clr_time", IsMicroseconds: true, NeedsAvg: true},
93 + {ColumnMeta: funcapi.ColumnMeta{Name: "lastClr", Tooltip: "Last CLR Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: false, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "last_clr_time", IsMicroseconds: true},
94 + {ColumnMeta: funcapi.ColumnMeta{Name: "minClr", Tooltip: "Min CLR Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: false, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMin, Filter: funcapi.FieldFilterRange}, DBColumn: "min_clr_time", IsMicroseconds: true},
95 + {ColumnMeta: funcapi.ColumnMeta{Name: "maxClr", Tooltip: "Max CLR Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: false, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "max_clr_time", IsMicroseconds: true},
96 + {ColumnMeta: funcapi.ColumnMeta{Name: "stdevClr", Tooltip: "StdDev CLR Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: false, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "stdev_clr_time", IsMicroseconds: true},
97 +
98 + // DOP (degree of parallelism)
99 + {ColumnMeta: funcapi.ColumnMeta{Name: "avgDop", Tooltip: "Avg DOP", Type: funcapi.FieldTypeFloat, DecimalPoints: 1, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMean, Filter: funcapi.FieldFilterRange}, DBColumn: "avg_dop", NeedsAvg: true, sortOpt: true, sortLbl: "Top queries by Average Parallelism"},
100 + {ColumnMeta: funcapi.ColumnMeta{Name: "lastDop", Tooltip: "Last DOP", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "last_dop"},
101 + {ColumnMeta: funcapi.ColumnMeta{Name: "minDop", Tooltip: "Min DOP", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMin, Filter: funcapi.FieldFilterRange}, DBColumn: "min_dop"},
102 + {ColumnMeta: funcapi.ColumnMeta{Name: "maxDop", Tooltip: "Max DOP", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "max_dop"},
103 + {ColumnMeta: funcapi.ColumnMeta{Name: "stdevDop", Tooltip: "StdDev DOP", Type: funcapi.FieldTypeFloat, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "stdev_dop"},
104 +
105 + // Memory grant (8KB pages)
106 + {ColumnMeta: funcapi.ColumnMeta{Name: "avgMemory", Tooltip: "Avg Memory (8KB pages)", Type: funcapi.FieldTypeFloat, DecimalPoints: 0, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMean, Filter: funcapi.FieldFilterRange}, DBColumn: "avg_query_max_used_memory", NeedsAvg: true, sortOpt: true, sortLbl: "Top queries by Average Memory Grant"},
107 + {ColumnMeta: funcapi.ColumnMeta{Name: "lastMemory", Tooltip: "Last Memory (8KB pages)", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "last_query_max_used_memory"},
108 + {ColumnMeta: funcapi.ColumnMeta{Name: "minMemory", Tooltip: "Min Memory (8KB pages)", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMin, Filter: funcapi.FieldFilterRange}, DBColumn: "min_query_max_used_memory"},
109 + {ColumnMeta: funcapi.ColumnMeta{Name: "maxMemory", Tooltip: "Max Memory (8KB pages)", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "max_query_max_used_memory"},
110 + {ColumnMeta: funcapi.ColumnMeta{Name: "stdevMemory", Tooltip: "StdDev Memory", Type: funcapi.FieldTypeFloat, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "stdev_query_max_used_memory"},
111 +
112 + // Row count
113 + {ColumnMeta: funcapi.ColumnMeta{Name: "avgRows", Tooltip: "Avg Rows", Type: funcapi.FieldTypeFloat, DecimalPoints: 0, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMean, Filter: funcapi.FieldFilterRange}, DBColumn: "avg_rowcount", NeedsAvg: true, sortOpt: true, sortLbl: "Top queries by Average Row Count"},
114 + {ColumnMeta: funcapi.ColumnMeta{Name: "lastRows", Tooltip: "Last Rows", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "last_rowcount"},
115 + {ColumnMeta: funcapi.ColumnMeta{Name: "minRows", Tooltip: "Min Rows", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMin, Filter: funcapi.FieldFilterRange}, DBColumn: "min_rowcount"},
116 + {ColumnMeta: funcapi.ColumnMeta{Name: "maxRows", Tooltip: "Max Rows", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "max_rowcount"},
117 + {ColumnMeta: funcapi.ColumnMeta{Name: "stdevRows", Tooltip: "StdDev Rows", Type: funcapi.FieldTypeFloat, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "stdev_rowcount"},
118 +
119 + // SQL Server 2017+ log bytes
120 + {ColumnMeta: funcapi.ColumnMeta{Name: "avgLogBytes", Tooltip: "Avg Log Bytes", Type: funcapi.FieldTypeFloat, DecimalPoints: 0, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMean, Filter: funcapi.FieldFilterRange}, DBColumn: "avg_log_bytes_used", NeedsAvg: true, sortOpt: true, sortLbl: "Top queries by Average Log Bytes"},
121 + {ColumnMeta: funcapi.ColumnMeta{Name: "lastLogBytes", Tooltip: "Last Log Bytes", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "last_log_bytes_used"},
122 + {ColumnMeta: funcapi.ColumnMeta{Name: "minLogBytes", Tooltip: "Min Log Bytes", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMin, Filter: funcapi.FieldFilterRange}, DBColumn: "min_log_bytes_used"},
123 + {ColumnMeta: funcapi.ColumnMeta{Name: "maxLogBytes", Tooltip: "Max Log Bytes", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "max_log_bytes_used"},
124 + {ColumnMeta: funcapi.ColumnMeta{Name: "stdevLogBytes", Tooltip: "StdDev Log Bytes", Type: funcapi.FieldTypeFloat, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "stdev_log_bytes_used"},
125 +
126 + // SQL Server 2017+ tempdb space
127 + {ColumnMeta: funcapi.ColumnMeta{Name: "avgTempdb", Tooltip: "Avg TempDB (8KB pages)", Type: funcapi.FieldTypeFloat, DecimalPoints: 0, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMean, Filter: funcapi.FieldFilterRange}, DBColumn: "avg_tempdb_space_used", NeedsAvg: true, sortOpt: true, sortLbl: "Top queries by Average TempDB Usage"},
128 + {ColumnMeta: funcapi.ColumnMeta{Name: "lastTempdb", Tooltip: "Last TempDB (8KB pages)", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "last_tempdb_space_used"},
129 + {ColumnMeta: funcapi.ColumnMeta{Name: "minTempdb", Tooltip: "Min TempDB (8KB pages)", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMin, Filter: funcapi.FieldFilterRange}, DBColumn: "min_tempdb_space_used"},
130 + {ColumnMeta: funcapi.ColumnMeta{Name: "maxTempdb", Tooltip: "Max TempDB (8KB pages)", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "max_tempdb_space_used"},
131 + {ColumnMeta: funcapi.ColumnMeta{Name: "stdevTempdb", Tooltip: "StdDev TempDB", Type: funcapi.FieldTypeFloat, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange}, DBColumn: "stdev_tempdb_space_used"},
132 +}
133 +
134 +// funcapi.SortableColumn interface implementation for topQueriesColumn.
135 +func (c topQueriesColumn) IsSortOption() bool { return c.sortOpt }
136 +func (c topQueriesColumn) SortLabel() string { return c.sortLbl }
137 +func (c topQueriesColumn) IsDefaultSort() bool { return c.defaultSort }
138 +func (c topQueriesColumn) ColumnName() string { return c.Name }
139 +func (c topQueriesColumn) SortColumn() string { return "" }
140 +
141 +type topQueriesChartGroupDef struct {
142 + key string
143 + title string
144 + columns []string
145 + defaultChart bool
146 +}
147 +
148 +var topQueriesChartGroupDefs = []topQueriesChartGroupDef{
149 + {key: "Calls", title: "Number of Calls", columns: []string{"calls"}, defaultChart: true},
150 + {key: "Time", title: "Execution Time", columns: []string{"totalTime", "avgTime", "lastTime", "minTime", "maxTime", "stdevTime"}, defaultChart: true},
151 + {key: "CPU", title: "CPU Time", columns: []string{"avgCpu", "lastCpu", "minCpu", "maxCpu", "stdevCpu"}},
152 + {key: "LogicalIO", title: "Logical I/O", columns: []string{"avgReads", "lastReads", "minReads", "maxReads", "stdevReads", "avgWrites", "lastWrites", "minWrites", "maxWrites", "stdevWrites"}},
153 + {key: "PhysicalIO", title: "Physical Reads", columns: []string{"avgPhysReads", "lastPhysReads", "minPhysReads", "maxPhysReads", "stdevPhysReads"}},
154 + {key: "CLR", title: "CLR Time", columns: []string{"avgClr", "lastClr", "minClr", "maxClr", "stdevClr"}},
155 + {key: "DOP", title: "Parallelism", columns: []string{"avgDop", "lastDop", "minDop", "maxDop", "stdevDop"}},
156 + {key: "Memory", title: "Memory Grant", columns: []string{"avgMemory", "lastMemory", "minMemory", "maxMemory", "stdevMemory"}},
157 + {key: "Rows", title: "Rows", columns: []string{"avgRows", "lastRows", "minRows", "maxRows", "stdevRows"}},
158 + {key: "LogBytes", title: "Log Bytes", columns: []string{"avgLogBytes", "lastLogBytes", "minLogBytes", "maxLogBytes", "stdevLogBytes"}},
159 + {key: "TempDB", title: "TempDB Usage", columns: []string{"avgTempdb", "lastTempdb", "minTempdb", "maxTempdb", "stdevTempdb"}},
160 +}
161 +
162 +var topQueriesLabelColumnIDs = map[string]bool{
163 + "database": true,
164 +}
165 +
166 +const topQueriesPrimaryLabelID = "database"
167 +
168 +// topQueriesRowScanner interface for testing.
169 +type topQueriesRowScanner interface {
170 + Next() bool
171 + Scan(dest ...any) error
172 + Err() error
173 +}
174 +
175 +// funcTopQueries implements funcapi.MethodHandler for MSSQL top-queries.
176 +// All function-related logic is encapsulated here, keeping Collector focused on metrics collection.
177 +type funcTopQueries struct {
178 + router *funcRouter
179 +}
180 +
181 +func newFuncTopQueries(r *funcRouter) *funcTopQueries {
182 + return &funcTopQueries{router: r}
183 +}
184 +
185 +// Compile-time interface check.
186 +var _ funcapi.MethodHandler = (*funcTopQueries)(nil)
187 +
188 +// MethodParams implements funcapi.MethodHandler.
189 +func (f *funcTopQueries) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
190 + if f.router.collector.db == nil {
191 + return nil, fmt.Errorf("collector is still initializing")
192 + }
193 + switch method {
194 + case topQueriesMethodID:
195 + return f.methodParams(ctx)
196 + default:
197 + return nil, fmt.Errorf("unknown method: %s", method)
198 + }
199 +}
200 +
201 +// Handle implements funcapi.MethodHandler.
202 +func (f *funcTopQueries) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
203 + if f.router.collector.db == nil {
204 + return funcapi.UnavailableResponse("collector is still initializing, please retry in a few seconds")
205 + }
206 + switch method {
207 + case topQueriesMethodID:
208 + return f.collectData(ctx, params.Column(topQueriesParamSort))
209 + default:
210 + return funcapi.NotFoundResponse(method)
211 + }
212 +}
213 +
214 +// Cleanup implements funcapi.MethodHandler.
215 +func (f *funcTopQueries) Cleanup(ctx context.Context) {}
216 +
217 +func (f *funcTopQueries) methodParams(ctx context.Context) ([]funcapi.ParamConfig, error) {
218 + if !f.router.collector.Config.GetQueryStoreFunctionEnabled() {
219 + return nil, fmt.Errorf("query store function disabled")
220 + }
221 +
222 + availableCols, err := f.detectQueryStoreColumns(ctx)
223 + if err != nil {
224 + return nil, err
225 + }
226 +
227 + cols := f.buildAvailableColumns(availableCols)
228 + if len(cols) == 0 {
229 + return nil, fmt.Errorf("no columns available in Query Store")
230 + }
231 +
232 + sortParam, _ := f.buildSortParam(cols)
233 + return []funcapi.ParamConfig{sortParam}, nil
234 +}
235 +
236 +func (f *funcTopQueries) collectData(ctx context.Context, sortColumn string) *funcapi.FunctionResponse {
237 + if !f.router.collector.Config.GetQueryStoreFunctionEnabled() {
238 + return &funcapi.FunctionResponse{
239 + Status: 403,
240 + Message: "Query Store function has been disabled in configuration. " +
241 + "To enable, set query_store_function_enabled: true in the MSSQL collector config.",
242 + }
243 + }
244 +
245 + availableCols, err := f.detectQueryStoreColumns(ctx)
246 + if err != nil {
247 + return &funcapi.FunctionResponse{
248 + Status: 500,
249 + Message: fmt.Sprintf("failed to detect available columns: %v", err),
250 + }
251 + }
252 +
253 + cols := f.buildAvailableColumns(availableCols)
254 + if len(cols) == 0 {
255 + return &funcapi.FunctionResponse{
256 + Status: 500,
257 + Message: "no columns available in Query Store",
258 + }
259 + }
260 +
261 + validatedSortColumn := f.mapAndValidateSortColumn(sortColumn, cols)
262 +
263 + timeWindowDays := f.router.collector.Config.GetQueryStoreTimeWindowDays()
264 + limit := f.router.collector.TopQueriesLimit
265 + if limit <= 0 {
266 + limit = 500
267 + }
268 + query := f.buildDynamicSQL(cols, validatedSortColumn, timeWindowDays, limit)
269 +
270 + rows, err := f.router.collector.db.QueryContext(ctx, query)
271 + if err != nil {
272 + if ctx.Err() == context.DeadlineExceeded {
273 + return &funcapi.FunctionResponse{Status: 504, Message: "query timed out"}
274 + }
275 + colIDs := make([]string, len(cols))
276 + for i, col := range cols {
277 + colIDs[i] = col.Name
278 + }
279 + return &funcapi.FunctionResponse{
280 + Status: 500,
281 + Message: fmt.Sprintf("query failed: %v (sort: %s, detected cols: %v)", err, validatedSortColumn, colIDs),
282 + }
283 + }
284 + defer rows.Close()
285 +
286 + data, err := f.scanDynamicRows(rows, cols)
287 + if err != nil {
288 + return &funcapi.FunctionResponse{Status: 500, Message: err.Error()}
289 + }
290 +
291 + sortParam, sortOptions := f.buildSortParam(cols)
292 +
293 + defaultSort := ""
294 + for _, col := range cols {
295 + if col.IsDefaultSort() && col.IsSortOption() {
296 + defaultSort = col.Name
297 + break
298 + }
299 + }
300 + if defaultSort == "" && len(sortOptions) > 0 {
301 + defaultSort = sortOptions[0].ID
302 + }
303 +
304 + annotatedCols := f.decorateColumns(cols)
305 + cs := f.columnSet(annotatedCols)
306 +
307 + return &funcapi.FunctionResponse{
308 + Status: 200,
309 + Help: "Top SQL queries from Query Store. WARNING: Query text may contain unmasked literals (potential PII).",
310 + Columns: cs.BuildColumns(),
311 + Data: data,
312 + DefaultSortColumn: defaultSort,
313 + RequiredParams: []funcapi.ParamConfig{sortParam},
314 + ChartingConfig: cs.BuildCharting(),
315 + }
316 +}
317 +
318 +func (f *funcTopQueries) columnSet(cols []topQueriesColumn) funcapi.ColumnSet[topQueriesColumn] {
319 + return funcapi.Columns(cols, func(c topQueriesColumn) funcapi.ColumnMeta { return c.ColumnMeta })
320 +}
321 +
322 +func (f *funcTopQueries) detectQueryStoreColumns(ctx context.Context) (map[string]bool, error) {
323 + // Fast path: return cached result
324 + f.router.collector.queryStoreColsMu.RLock()
325 + if f.router.collector.queryStoreCols != nil {
326 + cols := f.router.collector.queryStoreCols
327 + f.router.collector.queryStoreColsMu.RUnlock()
328 + return cols, nil
329 + }
330 + f.router.collector.queryStoreColsMu.RUnlock()
331 +
332 + // Slow path: query and cache
333 + f.router.collector.queryStoreColsMu.Lock()
334 + defer f.router.collector.queryStoreColsMu.Unlock()
335 +
336 + // Double-check after acquiring write lock
337 + if f.router.collector.queryStoreCols != nil {
338 + return f.router.collector.queryStoreCols, nil
339 + }
340 +
341 + // Find any database with Query Store enabled (excluding system databases)
342 + var sampleDB string
343 + err := f.router.collector.db.QueryRowContext(ctx, `
344 + SELECT TOP 1 name
345 + FROM sys.databases
346 + WHERE is_query_store_on = 1
347 + AND name NOT IN ('master', 'tempdb', 'model', 'msdb')
348 + `).Scan(&sampleDB)
349 + if err != nil {
350 + if err == sql.ErrNoRows {
351 + return nil, fmt.Errorf("no databases have Query Store enabled")
352 + }
353 + return nil, fmt.Errorf("failed to find database with Query Store: %w", err)
354 + }
355 +
356 + // Use dynamic SQL to get column metadata from that database's Query Store view
357 + query := fmt.Sprintf(`SELECT TOP 0 * FROM [%s].sys.query_store_runtime_stats`, sampleDB)
358 + rows, err := f.router.collector.db.QueryContext(ctx, query)
359 + if err != nil {
360 + return nil, fmt.Errorf("failed to query Query Store columns from %s: %w", sampleDB, err)
361 + }
362 + defer rows.Close()
363 +
364 + columnNames, err := rows.Columns()
365 + if err != nil {
366 + return nil, fmt.Errorf("failed to get column names: %w", err)
367 + }
368 +
369 + cols := make(map[string]bool)
370 + for _, colName := range columnNames {
371 + cols[strings.ToLower(colName)] = true
372 + }
373 +
374 + if len(cols) == 0 {
375 + return nil, fmt.Errorf("no columns found in sys.query_store_runtime_stats")
376 + }
377 +
378 + // Add identity columns that are always available (from other Query Store views)
379 + cols["query_hash"] = true
380 + cols["query_sql_text"] = true
381 + cols["database_name"] = true
382 +
383 + f.router.collector.queryStoreCols = cols
384 +
385 + return cols, nil
386 +}
387 +
388 +func (f *funcTopQueries) buildAvailableColumns(availableCols map[string]bool) []topQueriesColumn {
389 + var cols []topQueriesColumn
390 + seen := make(map[string]bool)
391 +
392 + for _, col := range topQueriesColumns {
393 + if seen[col.Name] {
394 + continue
395 + }
396 + if col.IsIdentity {
397 + cols = append(cols, col)
398 + seen[col.Name] = true
399 + continue
400 + }
401 + if availableCols[col.DBColumn] {
402 + cols = append(cols, col)
403 + seen[col.Name] = true
404 + }
405 + }
406 + return cols
407 +}
408 +
409 +func (f *funcTopQueries) mapAndValidateSortColumn(sortKey string, cols []topQueriesColumn) string {
410 + for _, col := range cols {
411 + if col.Name == sortKey && col.IsSortOption() {
412 + return col.Name
413 + }
414 + }
415 +
416 + for _, col := range cols {
417 + if col.IsSortOption() {
418 + return col.Name
419 + }
420 + }
421 +
422 + for _, col := range cols {
423 + if !col.IsIdentity {
424 + return col.Name
425 + }
426 + }
427 +
428 + if len(cols) > 0 {
429 + return cols[0].Name
430 + }
431 +
432 + return ""
433 +}
434 +
435 +func (f *funcTopQueries) buildSelectExpressions(cols []topQueriesColumn, dbNameExpr string) []string {
436 + var selectParts []string
437 +
438 + for _, col := range cols {
439 + var expr string
440 + switch {
441 + case col.IsIdentity:
442 + switch col.Name {
443 + case "queryHash":
444 + expr = fmt.Sprintf("CONVERT(VARCHAR(64), q.query_hash, 1) AS [%s]", col.Name)
445 + case "query":
446 + expr = fmt.Sprintf("qt.query_sql_text AS [%s]", col.Name)
447 + case "database":
448 + expr = fmt.Sprintf("%s AS [%s]", dbNameExpr, col.Name)
449 + }
450 + case col.Name == "calls":
451 + expr = fmt.Sprintf("SUM(rs.count_executions) AS [%s]", col.Name)
452 + case col.Name == "totalTime":
453 + expr = fmt.Sprintf("SUM(rs.avg_duration * rs.count_executions) / 1000.0 AS [%s]", col.Name)
454 + case col.NeedsAvg && col.IsMicroseconds:
455 + expr = fmt.Sprintf("CASE WHEN SUM(rs.count_executions) > 0 THEN SUM(rs.%s * rs.count_executions) / SUM(rs.count_executions) / 1000.0 ELSE 0 END AS [%s]", col.DBColumn, col.Name)
456 + case col.NeedsAvg:
457 + expr = fmt.Sprintf("CASE WHEN SUM(rs.count_executions) > 0 THEN SUM(rs.%s * rs.count_executions) / SUM(rs.count_executions) ELSE 0 END AS [%s]", col.DBColumn, col.Name)
458 + case col.IsMicroseconds:
459 + aggFunc := "MAX"
460 + if strings.HasPrefix(col.DBColumn, "min_") {
461 + aggFunc = "MIN"
462 + }
463 + expr = fmt.Sprintf("%s(rs.%s) / 1000.0 AS [%s]", aggFunc, col.DBColumn, col.Name)
464 + default:
465 + aggFunc := "MAX"
466 + if strings.HasPrefix(col.DBColumn, "min_") {
467 + aggFunc = "MIN"
468 + }
469 + if strings.HasPrefix(col.DBColumn, "stdev_") {
470 + aggFunc = "MAX"
471 + }
472 + expr = fmt.Sprintf("%s(rs.%s) AS [%s]", aggFunc, col.DBColumn, col.Name)
473 + }
474 + if expr != "" {
475 + selectParts = append(selectParts, expr)
476 + }
477 + }
478 + return selectParts
479 +}
480 +
481 +func (f *funcTopQueries) buildDynamicSQL(cols []topQueriesColumn, sortColumn string, timeWindowDays int, limit int) string {
482 + selectParts := f.buildSelectExpressions(cols, "''' + name + N'''")
483 + selectExpr := strings.Join(selectParts, ",\n ")
484 +
485 + timeFilter := ""
486 + if timeWindowDays > 0 {
487 + timeFilter = fmt.Sprintf("WHERE rsi.start_time >= DATEADD(day, -%d, GETUTCDATE())", timeWindowDays)
488 + }
489 +
490 + orderByExpr := sortColumn
491 + if orderByExpr == "" {
492 + for _, col := range cols {
493 + if !col.IsIdentity {
494 + orderByExpr = col.Name
495 + break
496 + }
497 + }
498 + if orderByExpr == "" && len(cols) > 0 {
499 + orderByExpr = cols[0].Name
500 + }
501 + }
502 +
503 + return fmt.Sprintf(`
504 +DECLARE @sql NVARCHAR(MAX) = N'';
505 +
506 +SELECT @sql = @sql +
507 + CASE WHEN @sql = N'' THEN N'' ELSE N' UNION ALL ' END +
508 + N'SELECT
509 + %s
510 + FROM ' + QUOTENAME(name) + N'.sys.query_store_query q
511 + INNER JOIN ' + QUOTENAME(name) + N'.sys.query_store_query_text qt ON q.query_text_id = qt.query_text_id
512 + INNER JOIN ' + QUOTENAME(name) + N'.sys.query_store_plan p ON q.query_id = p.query_id
513 + INNER JOIN ' + QUOTENAME(name) + N'.sys.query_store_runtime_stats rs ON p.plan_id = rs.plan_id
514 + INNER JOIN ' + QUOTENAME(name) + N'.sys.query_store_runtime_stats_interval rsi ON rs.runtime_stats_interval_id = rsi.runtime_stats_interval_id
515 + %s
516 + GROUP BY q.query_hash, qt.query_sql_text'
517 +FROM sys.databases
518 +WHERE is_query_store_on = 1
519 + AND name NOT IN ('master', 'tempdb', 'model', 'msdb');
520 +
521 +IF @sql = N''
522 +BEGIN
523 + RAISERROR('No databases have Query Store enabled', 16, 1);
524 + RETURN;
525 +END
526 +
527 +SET @sql = N'SELECT TOP %d * FROM (' + @sql + N') AS combined ORDER BY [%s] DESC';
528 +EXEC sp_executesql @sql;
529 +`, selectExpr, timeFilter, limit, orderByExpr)
530 +}
531 +
532 +func (f *funcTopQueries) scanDynamicRows(rows topQueriesRowScanner, cols []topQueriesColumn) ([][]any, error) {
533 + data := make([][]any, 0, 500)
534 +
535 + for rows.Next() {
536 + values := make([]any, len(cols))
537 + valuePtrs := make([]any, len(cols))
538 +
539 + for i, col := range cols {
540 + switch col.Type {
541 + case funcapi.FieldTypeString:
542 + var v sql.NullString
543 + values[i] = &v
544 + case funcapi.FieldTypeInteger:
545 + var v sql.NullInt64
546 + values[i] = &v
547 + case funcapi.FieldTypeFloat, funcapi.FieldTypeDuration:
548 + var v sql.NullFloat64
549 + values[i] = &v
550 + default:
551 + var v any
552 + values[i] = &v
553 + }
554 + valuePtrs[i] = values[i]
555 + }
556 +
557 + if err := rows.Scan(valuePtrs...); err != nil {
558 + return nil, fmt.Errorf("row scan failed: %w", err)
559 + }
560 +
561 + row := make([]any, len(cols))
562 + for i, col := range cols {
563 + switch v := values[i].(type) {
564 + case *sql.NullString:
565 + if v.Valid {
566 + s := v.String
567 + if col.Name == "query" {
568 + s = strmutil.TruncateText(s, topQueriesMaxTextLength)
569 + }
570 + row[i] = s
571 + } else {
572 + row[i] = ""
573 + }
574 + case *sql.NullInt64:
575 + if v.Valid {
576 + row[i] = v.Int64
577 + } else {
578 + row[i] = int64(0)
579 + }
580 + case *sql.NullFloat64:
581 + if v.Valid {
582 + row[i] = v.Float64
583 + } else {
584 + row[i] = float64(0)
585 + }
586 + default:
587 + row[i] = nil
588 + }
589 + }
590 + data = append(data, row)
591 + }
592 +
593 + if err := rows.Err(); err != nil {
594 + return nil, fmt.Errorf("rows iteration error: %w", err)
595 + }
596 +
597 + return data, nil
598 +}
599 +
600 +func (f *funcTopQueries) buildSortParam(cols []topQueriesColumn) (funcapi.ParamConfig, []funcapi.ParamOption) {
601 + sortOptions := buildTopQueriesSortOptions(cols)
602 + sortParam := funcapi.ParamConfig{
603 + ID: topQueriesParamSort,
604 + Name: "Filter By",
605 + Help: "Select the primary sort column",
606 + Selection: funcapi.ParamSelect,
607 + Options: sortOptions,
608 + UniqueView: true,
609 + }
610 + return sortParam, sortOptions
611 +}
612 +
613 +func (f *funcTopQueries) decorateColumns(cols []topQueriesColumn) []topQueriesColumn {
614 + out := make([]topQueriesColumn, len(cols))
615 + index := make(map[string]int, len(cols))
616 + for i, col := range cols {
617 + out[i] = col
618 + index[col.Name] = i
619 + }
620 +
621 + for i := range out {
622 + if topQueriesLabelColumnIDs[out[i].Name] {
623 + out[i].GroupBy = &funcapi.GroupByOptions{
624 + IsDefault: out[i].Name == topQueriesPrimaryLabelID,
625 + }
626 + }
627 + }
628 +
629 + for _, group := range topQueriesChartGroupDefs {
630 + for _, key := range group.columns {
631 + idx, ok := index[key]
632 + if !ok {
633 + continue
634 + }
635 + out[idx].Chart = &funcapi.ChartOptions{
636 + Group: group.key,
637 + Title: group.title,
638 + IsDefault: group.defaultChart,
639 + }
640 + }
641 + }
642 +
643 + return out
644 +}
645 +
646 +// buildTopQueriesSortOptions builds sort options for method registration (before handler exists).
647 +func buildTopQueriesSortOptions(cols []topQueriesColumn) []funcapi.ParamOption {
648 + var sortOptions []funcapi.ParamOption
649 + sortDir := funcapi.FieldSortDescending
650 + seen := make(map[string]bool)
651 + for _, col := range cols {
652 + if col.IsSortOption() && !seen[col.Name] {
653 + seen[col.Name] = true
654 + sortOptions = append(sortOptions, funcapi.ParamOption{
655 + ID: col.Name,
656 + Column: col.Name,
657 + Name: col.SortLabel(),
658 + Default: col.IsDefaultSort(),
659 + Sort: &sortDir,
660 + })
661 + }
662 + }
663 + return sortOptions
664 +}
src/go/plugin/go.d/collector/mssql/func_top_queries_test.go new
+40
@@ -0,0 +1,40 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package mssql
4 +
5 +import (
6 + "testing"
7 +
8 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
9 + "github.com/stretchr/testify/assert"
10 +)
11 +
12 +func TestMSSQLMethods(t *testing.T) {
13 + methods := mssqlMethods()
14 +
15 + require := assert.New(t)
16 + require.Len(methods, 1)
17 + require.Equal("top-queries", methods[0].ID)
18 + require.Equal("Top Queries", methods[0].Name)
19 + require.NotEmpty(methods[0].RequiredParams)
20 +
21 + var sortParam *funcapi.ParamConfig
22 + for i := range methods[0].RequiredParams {
23 + if methods[0].RequiredParams[i].ID == "__sort" {
24 + sortParam = &methods[0].RequiredParams[i]
25 + break
26 + }
27 + }
28 + require.NotNil(sortParam, "expected __sort required param")
29 + require.NotEmpty(sortParam.Options)
30 +}
31 +
32 +func TestTopQueriesColumns_HasRequiredColumns(t *testing.T) {
33 + required := []string{"query", "totalTime", "calls"}
34 +
35 + f := &funcTopQueries{}
36 + cs := f.columnSet(topQueriesColumns)
37 + for _, id := range required {
38 + assert.True(t, cs.ContainsColumn(id), "column %s should be defined", id)
39 + }
40 +}
src/go/plugin/go.d/collector/mssql/functions.go deleted
-895
@@ -1,895 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -package mssql
4 -
5 -import (
6 - "context"
7 - "database/sql"
8 - "fmt"
9 - "strings"
10 -
11 - "github.com/netdata/netdata/go/plugins/pkg/funcapi"
12 - "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
13 - "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
14 -)
15 -
16 -const maxQueryTextLength = 4096
17 -
18 -const (
19 - paramSort = "__sort"
20 -
21 - ftString = funcapi.FieldTypeString
22 - ftInteger = funcapi.FieldTypeInteger
23 - ftFloat = funcapi.FieldTypeFloat
24 - ftDuration = funcapi.FieldTypeDuration
25 -
26 - trNone = funcapi.FieldTransformNone
27 - trNumber = funcapi.FieldTransformNumber
28 - trDuration = funcapi.FieldTransformDuration
29 -
30 - sortAsc = funcapi.FieldSortAscending
31 - sortDesc = funcapi.FieldSortDescending
32 -
33 - summaryCount = funcapi.FieldSummaryCount
34 - summarySum = funcapi.FieldSummarySum
35 - summaryMin = funcapi.FieldSummaryMin
36 - summaryMax = funcapi.FieldSummaryMax
37 - summaryMean = funcapi.FieldSummaryMean
38 -
39 - filterMulti = funcapi.FieldFilterMultiselect
40 - filterRange = funcapi.FieldFilterRange
41 -)
42 -
43 -// mssqlColumnMeta defines metadata for a single column
44 -type mssqlColumnMeta struct {
45 - dbColumn string // Column name in sys.query_store_runtime_stats
46 - uiKey string // Canonical name used everywhere: SQL alias, UI key, sort key
47 - displayName string // Display name in UI
48 - dataType funcapi.FieldType // "string", "integer", "float", "duration"
49 - units string // Unit for duration types
50 - visible bool // Default visibility
51 - transform funcapi.FieldTransform // Transform for value_options
52 - decimalPoints int // Decimal points for display
53 - sortDir funcapi.FieldSort // Sort direction: "ascending" or "descending"
54 - summary funcapi.FieldSummary // Summary function
55 - filter funcapi.FieldFilter // Filter type
56 - isMicroseconds bool // Needs μs to milliseconds conversion
57 - isSortOption bool // Show in sort dropdown
58 - sortLabel string // Label for sort option
59 - isDefaultSort bool // Is this the default sort option
60 - isUniqueKey bool // Is this column a unique key
61 - isSticky bool // Is this column sticky in UI
62 - fullWidth bool // Should column take full width
63 - isIdentity bool // Is this an identity column (query_hash, query_text, etc.)
64 - needsAvg bool // Needs weighted average calculation (avg_* columns)
65 - isLabel bool // Is this column a label for grouping
66 - isPrimary bool // Is this label the primary grouping
67 - isMetric bool // Is this column a chartable metric
68 - chartGroup string // Chart group key
69 - chartTitle string // Chart title
70 - isDefaultChart bool // Include this chart group in defaults
71 -}
72 -
73 -// mssqlAllColumns defines ALL possible columns from Query Store
74 -// Columns that don't exist in certain SQL Server versions will be filtered at runtime
75 -var mssqlAllColumns = []mssqlColumnMeta{
76 - // Identity columns - always available
77 - {dbColumn: "query_hash", uiKey: "queryHash", displayName: "Query Hash", dataType: ftString, visible: false, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti, isUniqueKey: true, isIdentity: true},
78 - {dbColumn: "query_sql_text", uiKey: "query", displayName: "Query", dataType: ftString, visible: true, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti, isSticky: true, fullWidth: true, isIdentity: true},
79 - {dbColumn: "database_name", uiKey: "database", displayName: "Database", dataType: ftString, visible: true, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti, isIdentity: true},
80 -
81 - // Execution count - always available
82 - {dbColumn: "count_executions", uiKey: "calls", displayName: "Calls", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Top queries by Number of Calls"},
83 -
84 - // Duration metrics (microseconds -> milliseconds) - SQL 2016+
85 - {dbColumn: "avg_duration", uiKey: "totalTime", displayName: "Total Time", dataType: ftDuration, units: "milliseconds", visible: true, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summarySum, filter: filterRange, isMicroseconds: true, isSortOption: true, sortLabel: "Top queries by Total Execution Time", isDefaultSort: true},
86 - {dbColumn: "avg_duration", uiKey: "avgTime", displayName: "Avg Time", dataType: ftDuration, units: "milliseconds", visible: true, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMean, filter: filterRange, isMicroseconds: true, needsAvg: true, isSortOption: true, sortLabel: "Top queries by Average Execution Time"},
87 - {dbColumn: "last_duration", uiKey: "lastTime", displayName: "Last Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, filter: filterRange, isMicroseconds: true},
88 - {dbColumn: "min_duration", uiKey: "minTime", displayName: "Min Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMin, filter: filterRange, isMicroseconds: true},
89 - {dbColumn: "max_duration", uiKey: "maxTime", displayName: "Max Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, filter: filterRange, isMicroseconds: true},
90 - {dbColumn: "stdev_duration", uiKey: "stdevTime", displayName: "StdDev Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, filter: filterRange, isMicroseconds: true},
91 -
92 - // CPU time metrics (microseconds -> milliseconds)
93 - {dbColumn: "avg_cpu_time", uiKey: "avgCpu", displayName: "Avg CPU", dataType: ftDuration, units: "milliseconds", visible: true, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMean, filter: filterRange, isMicroseconds: true, needsAvg: true, isSortOption: true, sortLabel: "Top queries by Average CPU Time"},
94 - {dbColumn: "last_cpu_time", uiKey: "lastCpu", displayName: "Last CPU", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, filter: filterRange, isMicroseconds: true},
95 - {dbColumn: "min_cpu_time", uiKey: "minCpu", displayName: "Min CPU", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMin, filter: filterRange, isMicroseconds: true},
96 - {dbColumn: "max_cpu_time", uiKey: "maxCpu", displayName: "Max CPU", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, filter: filterRange, isMicroseconds: true},
97 - {dbColumn: "stdev_cpu_time", uiKey: "stdevCpu", displayName: "StdDev CPU", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, filter: filterRange, isMicroseconds: true},
98 -
99 - // Logical I/O reads
100 - {dbColumn: "avg_logical_io_reads", uiKey: "avgReads", displayName: "Avg Logical Reads", dataType: ftFloat, visible: true, transform: trNumber, decimalPoints: 0, sortDir: sortDesc, summary: summaryMean, filter: filterRange, needsAvg: true, isSortOption: true, sortLabel: "Top queries by Average Logical Reads"},
101 - {dbColumn: "last_logical_io_reads", uiKey: "lastReads", displayName: "Last Logical Reads", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
102 - {dbColumn: "min_logical_io_reads", uiKey: "minReads", displayName: "Min Logical Reads", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMin, filter: filterRange},
103 - {dbColumn: "max_logical_io_reads", uiKey: "maxReads", displayName: "Max Logical Reads", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
104 - {dbColumn: "stdev_logical_io_reads", uiKey: "stdevReads", displayName: "StdDev Logical Reads", dataType: ftFloat, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
105 -
106 - // Logical I/O writes
107 - {dbColumn: "avg_logical_io_writes", uiKey: "avgWrites", displayName: "Avg Logical Writes", dataType: ftFloat, visible: true, transform: trNumber, decimalPoints: 0, sortDir: sortDesc, summary: summaryMean, filter: filterRange, needsAvg: true, isSortOption: true, sortLabel: "Top queries by Average Logical Writes"},
108 - {dbColumn: "last_logical_io_writes", uiKey: "lastWrites", displayName: "Last Logical Writes", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
109 - {dbColumn: "min_logical_io_writes", uiKey: "minWrites", displayName: "Min Logical Writes", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMin, filter: filterRange},
110 - {dbColumn: "max_logical_io_writes", uiKey: "maxWrites", displayName: "Max Logical Writes", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
111 - {dbColumn: "stdev_logical_io_writes", uiKey: "stdevWrites", displayName: "StdDev Logical Writes", dataType: ftFloat, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
112 -
113 - // Physical I/O reads
114 - {dbColumn: "avg_physical_io_reads", uiKey: "avgPhysReads", displayName: "Avg Physical Reads", dataType: ftFloat, visible: true, transform: trNumber, decimalPoints: 0, sortDir: sortDesc, summary: summaryMean, filter: filterRange, needsAvg: true, isSortOption: true, sortLabel: "Top queries by Average Physical Reads"},
115 - {dbColumn: "last_physical_io_reads", uiKey: "lastPhysReads", displayName: "Last Physical Reads", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
116 - {dbColumn: "min_physical_io_reads", uiKey: "minPhysReads", displayName: "Min Physical Reads", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMin, filter: filterRange},
117 - {dbColumn: "max_physical_io_reads", uiKey: "maxPhysReads", displayName: "Max Physical Reads", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
118 - {dbColumn: "stdev_physical_io_reads", uiKey: "stdevPhysReads", displayName: "StdDev Physical Reads", dataType: ftFloat, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
119 -
120 - // CLR time (microseconds -> milliseconds)
121 - {dbColumn: "avg_clr_time", uiKey: "avgClr", displayName: "Avg CLR Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMean, filter: filterRange, isMicroseconds: true, needsAvg: true},
122 - {dbColumn: "last_clr_time", uiKey: "lastClr", displayName: "Last CLR Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, filter: filterRange, isMicroseconds: true},
123 - {dbColumn: "min_clr_time", uiKey: "minClr", displayName: "Min CLR Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMin, filter: filterRange, isMicroseconds: true},
124 - {dbColumn: "max_clr_time", uiKey: "maxClr", displayName: "Max CLR Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, filter: filterRange, isMicroseconds: true},
125 - {dbColumn: "stdev_clr_time", uiKey: "stdevClr", displayName: "StdDev CLR Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, filter: filterRange, isMicroseconds: true},
126 -
127 - // DOP (degree of parallelism)
128 - {dbColumn: "avg_dop", uiKey: "avgDop", displayName: "Avg DOP", dataType: ftFloat, visible: true, transform: trNumber, decimalPoints: 1, sortDir: sortDesc, summary: summaryMean, filter: filterRange, needsAvg: true, isSortOption: true, sortLabel: "Top queries by Average Parallelism"},
129 - {dbColumn: "last_dop", uiKey: "lastDop", displayName: "Last DOP", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
130 - {dbColumn: "min_dop", uiKey: "minDop", displayName: "Min DOP", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMin, filter: filterRange},
131 - {dbColumn: "max_dop", uiKey: "maxDop", displayName: "Max DOP", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
132 - {dbColumn: "stdev_dop", uiKey: "stdevDop", displayName: "StdDev DOP", dataType: ftFloat, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
133 -
134 - // Memory grant (8KB pages)
135 - {dbColumn: "avg_query_max_used_memory", uiKey: "avgMemory", displayName: "Avg Memory (8KB pages)", dataType: ftFloat, visible: true, transform: trNumber, decimalPoints: 0, sortDir: sortDesc, summary: summaryMean, filter: filterRange, needsAvg: true, isSortOption: true, sortLabel: "Top queries by Average Memory Grant"},
136 - {dbColumn: "last_query_max_used_memory", uiKey: "lastMemory", displayName: "Last Memory (8KB pages)", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
137 - {dbColumn: "min_query_max_used_memory", uiKey: "minMemory", displayName: "Min Memory (8KB pages)", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMin, filter: filterRange},
138 - {dbColumn: "max_query_max_used_memory", uiKey: "maxMemory", displayName: "Max Memory (8KB pages)", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
139 - {dbColumn: "stdev_query_max_used_memory", uiKey: "stdevMemory", displayName: "StdDev Memory", dataType: ftFloat, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
140 -
141 - // Row count
142 - {dbColumn: "avg_rowcount", uiKey: "avgRows", displayName: "Avg Rows", dataType: ftFloat, visible: true, transform: trNumber, decimalPoints: 0, sortDir: sortDesc, summary: summaryMean, filter: filterRange, needsAvg: true, isSortOption: true, sortLabel: "Top queries by Average Row Count"},
143 - {dbColumn: "last_rowcount", uiKey: "lastRows", displayName: "Last Rows", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
144 - {dbColumn: "min_rowcount", uiKey: "minRows", displayName: "Min Rows", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMin, filter: filterRange},
145 - {dbColumn: "max_rowcount", uiKey: "maxRows", displayName: "Max Rows", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
146 - {dbColumn: "stdev_rowcount", uiKey: "stdevRows", displayName: "StdDev Rows", dataType: ftFloat, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
147 -
148 - // SQL Server 2017+ log bytes
149 - {dbColumn: "avg_log_bytes_used", uiKey: "avgLogBytes", displayName: "Avg Log Bytes", dataType: ftFloat, visible: true, transform: trNumber, decimalPoints: 0, sortDir: sortDesc, summary: summaryMean, filter: filterRange, needsAvg: true, isSortOption: true, sortLabel: "Top queries by Average Log Bytes"},
150 - {dbColumn: "last_log_bytes_used", uiKey: "lastLogBytes", displayName: "Last Log Bytes", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
151 - {dbColumn: "min_log_bytes_used", uiKey: "minLogBytes", displayName: "Min Log Bytes", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMin, filter: filterRange},
152 - {dbColumn: "max_log_bytes_used", uiKey: "maxLogBytes", displayName: "Max Log Bytes", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
153 - {dbColumn: "stdev_log_bytes_used", uiKey: "stdevLogBytes", displayName: "StdDev Log Bytes", dataType: ftFloat, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
154 -
155 - // SQL Server 2017+ tempdb space
156 - {dbColumn: "avg_tempdb_space_used", uiKey: "avgTempdb", displayName: "Avg TempDB (8KB pages)", dataType: ftFloat, visible: true, transform: trNumber, decimalPoints: 0, sortDir: sortDesc, summary: summaryMean, filter: filterRange, needsAvg: true, isSortOption: true, sortLabel: "Top queries by Average TempDB Usage"},
157 - {dbColumn: "last_tempdb_space_used", uiKey: "lastTempdb", displayName: "Last TempDB (8KB pages)", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
158 - {dbColumn: "min_tempdb_space_used", uiKey: "minTempdb", displayName: "Min TempDB (8KB pages)", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMin, filter: filterRange},
159 - {dbColumn: "max_tempdb_space_used", uiKey: "maxTempdb", displayName: "Max TempDB (8KB pages)", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
160 - {dbColumn: "stdev_tempdb_space_used", uiKey: "stdevTempdb", displayName: "StdDev TempDB", dataType: ftFloat, visible: false, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
161 -}
162 -
163 -type mssqlChartGroup struct {
164 - key string
165 - title string
166 - columns []string
167 - defaultChart bool
168 -}
169 -
170 -var mssqlChartGroups = []mssqlChartGroup{
171 - {key: "Calls", title: "Number of Calls", columns: []string{"calls"}, defaultChart: true},
172 - {key: "Time", title: "Execution Time", columns: []string{"totalTime", "avgTime", "lastTime", "minTime", "maxTime", "stdevTime"}, defaultChart: true},
173 - {key: "CPU", title: "CPU Time", columns: []string{"avgCpu", "lastCpu", "minCpu", "maxCpu", "stdevCpu"}},
174 - {key: "LogicalIO", title: "Logical I/O", columns: []string{"avgReads", "lastReads", "minReads", "maxReads", "stdevReads", "avgWrites", "lastWrites", "minWrites", "maxWrites", "stdevWrites"}},
175 - {key: "PhysicalIO", title: "Physical Reads", columns: []string{"avgPhysReads", "lastPhysReads", "minPhysReads", "maxPhysReads", "stdevPhysReads"}},
176 - {key: "CLR", title: "CLR Time", columns: []string{"avgClr", "lastClr", "minClr", "maxClr", "stdevClr"}},
177 - {key: "DOP", title: "Parallelism", columns: []string{"avgDop", "lastDop", "minDop", "maxDop", "stdevDop"}},
178 - {key: "Memory", title: "Memory Grant", columns: []string{"avgMemory", "lastMemory", "minMemory", "maxMemory", "stdevMemory"}},
179 - {key: "Rows", title: "Rows", columns: []string{"avgRows", "lastRows", "minRows", "maxRows", "stdevRows"}},
180 - {key: "LogBytes", title: "Log Bytes", columns: []string{"avgLogBytes", "lastLogBytes", "minLogBytes", "maxLogBytes", "stdevLogBytes"}},
181 - {key: "TempDB", title: "TempDB Usage", columns: []string{"avgTempdb", "lastTempdb", "minTempdb", "maxTempdb", "stdevTempdb"}},
182 -}
183 -
184 -var mssqlLabelColumns = map[string]bool{
185 - "database": true,
186 -}
187 -
188 -const mssqlPrimaryLabel = "database"
189 -
190 -// mssqlMethods returns the available function methods for MSSQL
191 -func mssqlMethods() []module.MethodConfig {
192 - // Build sort options from column metadata
193 - var sortOptions []funcapi.ParamOption
194 - sortDir := funcapi.FieldSortDescending
195 - seen := make(map[string]bool) // Avoid duplicates from totalTime/avgTime using same dbColumn
196 - for _, col := range mssqlAllColumns {
197 - if col.isSortOption && !seen[col.uiKey] {
198 - seen[col.uiKey] = true
199 - sortOptions = append(sortOptions, funcapi.ParamOption{
200 - ID: col.uiKey,
201 - Column: col.uiKey, // Use UI key for sort, we'll map internally
202 - Name: col.sortLabel,
203 - Default: col.isDefaultSort,
204 - Sort: &sortDir,
205 - })
206 - }
207 - }
208 -
209 - return []module.MethodConfig{
210 - {
211 - UpdateEvery: 10,
212 - ID: "top-queries",
213 - Name: "Top Queries",
214 - Help: "Top SQL queries from Query Store",
215 - RequireCloud: true,
216 - RequiredParams: []funcapi.ParamConfig{
217 - {
218 - ID: paramSort,
219 - Name: "Filter By",
220 - Help: "Select the primary sort column",
221 - Selection: funcapi.ParamSelect,
222 - Options: sortOptions,
223 - UniqueView: true,
224 - },
225 - },
226 - },
227 - }
228 -}
229 -
230 -func mssqlMethodParams(ctx context.Context, job *module.Job, method string) ([]funcapi.ParamConfig, error) {
231 - collector, ok := job.Module().(*Collector)
232 - if !ok {
233 - return nil, fmt.Errorf("invalid module type")
234 - }
235 - if collector.db == nil {
236 - return nil, fmt.Errorf("collector is still initializing")
237 - }
238 - switch method {
239 - case "top-queries":
240 - return collector.topQueriesParams(ctx)
241 - default:
242 - return nil, fmt.Errorf("unknown method: %s", method)
243 - }
244 -}
245 -
246 -// mssqlHandleMethod handles function requests for MSSQL
247 -func mssqlHandleMethod(ctx context.Context, job *module.Job, method string, params funcapi.ResolvedParams) *module.FunctionResponse {
248 - collector, ok := job.Module().(*Collector)
249 - if !ok {
250 - return &module.FunctionResponse{Status: 500, Message: "internal error: invalid module type"}
251 - }
252 -
253 - // Check if collector is initialized (first collect() may not have run yet)
254 - if collector.db == nil {
255 - return &module.FunctionResponse{
256 - Status: 503,
257 - Message: "collector is still initializing, please retry in a few seconds",
258 - }
259 - }
260 -
261 - switch method {
262 - case "top-queries":
263 - return collector.collectTopQueries(ctx, params.Column(paramSort))
264 - default:
265 - return &module.FunctionResponse{Status: 404, Message: fmt.Sprintf("unknown method: %s", method)}
266 - }
267 -}
268 -
269 -// detectMSSQLQueryStoreColumns queries any database with Query Store enabled to discover available columns
270 -func (c *Collector) detectMSSQLQueryStoreColumns(ctx context.Context) (map[string]bool, error) {
271 - // Fast path: return cached result
272 - c.queryStoreColsMu.RLock()
273 - if c.queryStoreCols != nil {
274 - cols := c.queryStoreCols
275 - c.queryStoreColsMu.RUnlock()
276 - return cols, nil
277 - }
278 - c.queryStoreColsMu.RUnlock()
279 -
280 - // Slow path: query and cache
281 - c.queryStoreColsMu.Lock()
282 - defer c.queryStoreColsMu.Unlock()
283 -
284 - // Double-check after acquiring write lock
285 - if c.queryStoreCols != nil {
286 - return c.queryStoreCols, nil
287 - }
288 -
289 - // Find any database with Query Store enabled (excluding system databases)
290 - var sampleDB string
291 - err := c.db.QueryRowContext(ctx, `
292 - SELECT TOP 1 name
293 - FROM sys.databases
294 - WHERE is_query_store_on = 1
295 - AND name NOT IN ('master', 'tempdb', 'model', 'msdb')
296 - `).Scan(&sampleDB)
297 - if err != nil {
298 - if err == sql.ErrNoRows {
299 - return nil, fmt.Errorf("no databases have Query Store enabled")
300 - }
301 - return nil, fmt.Errorf("failed to find database with Query Store: %w", err)
302 - }
303 -
304 - // Use dynamic SQL to get column metadata from that database's Query Store view
305 - // Three-part naming works: [DatabaseName].sys.query_store_runtime_stats
306 - query := fmt.Sprintf(`SELECT TOP 0 * FROM [%s].sys.query_store_runtime_stats`, sampleDB)
307 - rows, err := c.db.QueryContext(ctx, query)
308 - if err != nil {
309 - return nil, fmt.Errorf("failed to query Query Store columns from %s: %w", sampleDB, err)
310 - }
311 - defer rows.Close()
312 -
313 - // Get column names from result set metadata
314 - columnNames, err := rows.Columns()
315 - if err != nil {
316 - return nil, fmt.Errorf("failed to get column names: %w", err)
317 - }
318 -
319 - cols := make(map[string]bool)
320 - for _, colName := range columnNames {
321 - // Normalize to lowercase for case-insensitive comparison
322 - cols[strings.ToLower(colName)] = true
323 - }
324 -
325 - // If we found no columns, something is wrong
326 - if len(cols) == 0 {
327 - return nil, fmt.Errorf("no columns found in sys.query_store_runtime_stats")
328 - }
329 -
330 - // Add identity columns that are always available (from other Query Store views)
331 - cols["query_hash"] = true
332 - cols["query_sql_text"] = true
333 - cols["database_name"] = true
334 -
335 - // Cache the result
336 - c.queryStoreCols = cols
337 -
338 - return cols, nil
339 -}
340 -
341 -// buildAvailableMSSQLColumns filters columns based on what's available in the database
342 -func (c *Collector) buildAvailableMSSQLColumns(availableCols map[string]bool) []mssqlColumnMeta {
343 - var cols []mssqlColumnMeta
344 - seen := make(map[string]bool)
345 -
346 - for _, col := range mssqlAllColumns {
347 - // Skip duplicates (e.g., totalTime and avgTime both use avg_duration)
348 - if seen[col.uiKey] {
349 - continue
350 - }
351 - // Identity columns are always available
352 - if col.isIdentity {
353 - cols = append(cols, col)
354 - seen[col.uiKey] = true
355 - continue
356 - }
357 - // Check if the dbColumn exists
358 - if availableCols[col.dbColumn] {
359 - cols = append(cols, col)
360 - seen[col.uiKey] = true
361 - }
362 - }
363 - return cols
364 -}
365 -
366 -// mapAndValidateMSSQLSortColumn maps UI sort key to the appropriate sort expression
367 -// Uses the filtered cols list to ensure the sort column is actually in the SELECT
368 -func (c *Collector) mapAndValidateMSSQLSortColumn(sortKey string, cols []mssqlColumnMeta) string {
369 - // First, check if the requested sort key is in the available columns
370 - for _, col := range cols {
371 - if col.uiKey == sortKey && col.isSortOption {
372 - return col.uiKey
373 - }
374 - }
375 -
376 - // Fall back to the first available sort column
377 - for _, col := range cols {
378 - if col.isSortOption {
379 - return col.uiKey
380 - }
381 - }
382 -
383 - // Last resort: use first non-identity column
384 - for _, col := range cols {
385 - if !col.isIdentity {
386 - return col.uiKey
387 - }
388 - }
389 -
390 - // Absolute fallback: use first column in the list (must exist in SELECT)
391 - if len(cols) > 0 {
392 - return cols[0].uiKey
393 - }
394 -
395 - return "" // empty - will be handled by caller
396 -}
397 -
398 -// buildMSSQLSelectExpressions builds the SELECT expressions for a single database query
399 -func (c *Collector) buildMSSQLSelectExpressions(cols []mssqlColumnMeta, dbNameExpr string) []string {
400 - var selectParts []string
401 -
402 - for _, col := range cols {
403 - var expr string
404 - switch {
405 - case col.isIdentity:
406 - switch col.uiKey {
407 - case "queryHash":
408 - expr = fmt.Sprintf("CONVERT(VARCHAR(64), q.query_hash, 1) AS [%s]", col.uiKey)
409 - case "query":
410 - expr = fmt.Sprintf("qt.query_sql_text AS [%s]", col.uiKey)
411 - case "database":
412 - expr = fmt.Sprintf("%s AS [%s]", dbNameExpr, col.uiKey)
413 - }
414 - case col.uiKey == "calls":
415 - expr = fmt.Sprintf("SUM(rs.count_executions) AS [%s]", col.uiKey)
416 - case col.uiKey == "totalTime":
417 - // Total time = sum of (avg_duration * executions) converted to milliseconds
418 - expr = fmt.Sprintf("SUM(rs.avg_duration * rs.count_executions) / 1000.0 AS [%s]", col.uiKey)
419 - case col.needsAvg && col.isMicroseconds:
420 - // Weighted average with μs to milliseconds conversion
421 - expr = fmt.Sprintf("CASE WHEN SUM(rs.count_executions) > 0 THEN SUM(rs.%s * rs.count_executions) / SUM(rs.count_executions) / 1000.0 ELSE 0 END AS [%s]", col.dbColumn, col.uiKey)
422 - case col.needsAvg:
423 - // Weighted average without time conversion
424 - expr = fmt.Sprintf("CASE WHEN SUM(rs.count_executions) > 0 THEN SUM(rs.%s * rs.count_executions) / SUM(rs.count_executions) ELSE 0 END AS [%s]", col.dbColumn, col.uiKey)
425 - case col.isMicroseconds:
426 - // Aggregate with μs to milliseconds conversion
427 - aggFunc := "MAX"
428 - if strings.HasPrefix(col.dbColumn, "min_") {
429 - aggFunc = "MIN"
430 - }
431 - expr = fmt.Sprintf("%s(rs.%s) / 1000.0 AS [%s]", aggFunc, col.dbColumn, col.uiKey)
432 - default:
433 - // Simple aggregate
434 - aggFunc := "MAX"
435 - if strings.HasPrefix(col.dbColumn, "min_") {
436 - aggFunc = "MIN"
437 - }
438 - if strings.HasPrefix(col.dbColumn, "stdev_") {
439 - aggFunc = "MAX" // Use MAX for stddev aggregation
440 - }
441 - expr = fmt.Sprintf("%s(rs.%s) AS [%s]", aggFunc, col.dbColumn, col.uiKey)
442 - }
443 - if expr != "" {
444 - selectParts = append(selectParts, expr)
445 - }
446 - }
447 - return selectParts
448 -}
449 -
450 -// buildMSSQLDynamicSQL builds dynamic SQL that aggregates across all databases with Query Store enabled
451 -// Uses sp_executesql to execute the built query
452 -func (c *Collector) buildMSSQLDynamicSQL(cols []mssqlColumnMeta, sortColumn string, timeWindowDays int, limit int) string {
453 - // Build the SELECT expressions template (with placeholder for database name)
454 - // We use ''' + name + N''' to close the outer string, concatenate the db name, and reopen
455 - // This produces a properly quoted string literal like 'DatabaseName' in the final SQL
456 - selectParts := c.buildMSSQLSelectExpressions(cols, "''' + name + N'''")
457 - selectExpr := strings.Join(selectParts, ",\n ")
458 -
459 - // Time window filter
460 - timeFilter := ""
461 - if timeWindowDays > 0 {
462 - timeFilter = fmt.Sprintf("WHERE rsi.start_time >= DATEADD(day, -%d, GETUTCDATE())", timeWindowDays)
463 - }
464 -
465 - // Validate sort column
466 - orderByExpr := sortColumn
467 - if orderByExpr == "" {
468 - for _, col := range cols {
469 - if !col.isIdentity {
470 - orderByExpr = col.uiKey
471 - break
472 - }
473 - }
474 - if orderByExpr == "" && len(cols) > 0 {
475 - orderByExpr = cols[0].uiKey
476 - }
477 - }
478 -
479 - // Build the dynamic SQL that creates UNION ALL across all databases
480 - // The database names come from sys.databases, ensuring safety (no user input)
481 - return fmt.Sprintf(`
482 -DECLARE @sql NVARCHAR(MAX) = N'';
483 -
484 -SELECT @sql = @sql +
485 - CASE WHEN @sql = N'' THEN N'' ELSE N' UNION ALL ' END +
486 - N'SELECT
487 - %s
488 - FROM ' + QUOTENAME(name) + N'.sys.query_store_query q
489 - INNER JOIN ' + QUOTENAME(name) + N'.sys.query_store_query_text qt ON q.query_text_id = qt.query_text_id
490 - INNER JOIN ' + QUOTENAME(name) + N'.sys.query_store_plan p ON q.query_id = p.query_id
491 - INNER JOIN ' + QUOTENAME(name) + N'.sys.query_store_runtime_stats rs ON p.plan_id = rs.plan_id
492 - INNER JOIN ' + QUOTENAME(name) + N'.sys.query_store_runtime_stats_interval rsi ON rs.runtime_stats_interval_id = rsi.runtime_stats_interval_id
493 - %s
494 - GROUP BY q.query_hash, qt.query_sql_text'
495 -FROM sys.databases
496 -WHERE is_query_store_on = 1
497 - AND name NOT IN ('master', 'tempdb', 'model', 'msdb');
498 -
499 -IF @sql = N''
500 -BEGIN
501 - RAISERROR('No databases have Query Store enabled', 16, 1);
502 - RETURN;
503 -END
504 -
505 -SET @sql = N'SELECT TOP %d * FROM (' + @sql + N') AS combined ORDER BY [%s] DESC';
506 -EXEC sp_executesql @sql;
507 -`, selectExpr, timeFilter, limit, orderByExpr)
508 -}
509 -
510 -// mssqlRowScanner interface for testing
511 -type mssqlRowScanner interface {
512 - Next() bool
513 - Scan(dest ...any) error
514 - Err() error
515 -}
516 -
517 -// scanMSSQLDynamicRows scans rows dynamically based on column types
518 -func (c *Collector) scanMSSQLDynamicRows(rows mssqlRowScanner, cols []mssqlColumnMeta) ([][]any, error) {
519 - data := make([][]any, 0, 500)
520 -
521 - // Create value holders for scanning
522 - for rows.Next() {
523 - values := make([]any, len(cols))
524 - valuePtrs := make([]any, len(cols))
525 -
526 - for i, col := range cols {
527 - switch col.dataType {
528 - case ftString:
529 - var v sql.NullString
530 - values[i] = &v
531 - case ftInteger:
532 - var v sql.NullInt64
533 - values[i] = &v
534 - case ftFloat, ftDuration:
535 - var v sql.NullFloat64
536 - values[i] = &v
537 - default:
538 - var v any
539 - values[i] = &v
540 - }
541 - valuePtrs[i] = values[i]
542 - }
543 -
544 - if err := rows.Scan(valuePtrs...); err != nil {
545 - return nil, fmt.Errorf("row scan failed: %w", err)
546 - }
547 -
548 - // Convert scanned values to output format
549 - row := make([]any, len(cols))
550 - for i, col := range cols {
551 - switch v := values[i].(type) {
552 - case *sql.NullString:
553 - if v.Valid {
554 - s := v.String
555 - // Truncate query text
556 - if col.uiKey == "query" {
557 - s = strmutil.TruncateText(s, maxQueryTextLength)
558 - }
559 - row[i] = s
560 - } else {
561 - row[i] = ""
562 - }
563 - case *sql.NullInt64:
564 - if v.Valid {
565 - row[i] = v.Int64
566 - } else {
567 - row[i] = int64(0)
568 - }
569 - case *sql.NullFloat64:
570 - if v.Valid {
571 - row[i] = v.Float64
572 - } else {
573 - row[i] = float64(0)
574 - }
575 - default:
576 - row[i] = nil
577 - }
578 - }
579 - data = append(data, row)
580 - }
581 -
582 - if err := rows.Err(); err != nil {
583 - return nil, fmt.Errorf("rows iteration error: %w", err)
584 - }
585 -
586 - return data, nil
587 -}
588 -
589 -// buildMSSQLDynamicSortOptions builds sort options from available columns
590 -// Returns only sort options for columns that actually exist in the database
591 -func (c *Collector) buildMSSQLDynamicSortOptions(cols []mssqlColumnMeta) []funcapi.ParamOption {
592 - var sortOpts []funcapi.ParamOption
593 - seen := make(map[string]bool)
594 - sortDir := funcapi.FieldSortDescending
595 -
596 - for _, col := range cols {
597 - if col.isSortOption && !seen[col.uiKey] {
598 - seen[col.uiKey] = true
599 - sortOpts = append(sortOpts, funcapi.ParamOption{
600 - ID: col.uiKey,
601 - Column: col.uiKey,
602 - Name: col.sortLabel,
603 - Default: col.isDefaultSort,
604 - Sort: &sortDir,
605 - })
606 - }
607 - }
608 - return sortOpts
609 -}
610 -
611 -func (c *Collector) topQueriesSortParam(cols []mssqlColumnMeta) (funcapi.ParamConfig, []funcapi.ParamOption) {
612 - sortOptions := c.buildMSSQLDynamicSortOptions(cols)
613 - sortParam := funcapi.ParamConfig{
614 - ID: paramSort,
615 - Name: "Filter By",
616 - Help: "Select the primary sort column",
617 - Selection: funcapi.ParamSelect,
618 - Options: sortOptions,
619 - UniqueView: true,
620 - }
621 - return sortParam, sortOptions
622 -}
623 -
624 -func (c *Collector) topQueriesParams(ctx context.Context) ([]funcapi.ParamConfig, error) {
625 - if !c.Config.GetQueryStoreFunctionEnabled() {
626 - return nil, fmt.Errorf("query store function disabled")
627 - }
628 -
629 - availableCols, err := c.detectMSSQLQueryStoreColumns(ctx)
630 - if err != nil {
631 - return nil, err
632 - }
633 -
634 - cols := c.buildAvailableMSSQLColumns(availableCols)
635 - if len(cols) == 0 {
636 - return nil, fmt.Errorf("no columns available in Query Store")
637 - }
638 -
639 - sortParam, _ := c.topQueriesSortParam(cols)
640 - return []funcapi.ParamConfig{sortParam}, nil
641 -}
642 -
643 -// buildMSSQLDynamicColumns builds column definitions for the response
644 -func (c *Collector) buildMSSQLDynamicColumns(cols []mssqlColumnMeta) map[string]any {
645 - columns := make(map[string]any)
646 - for i, col := range cols {
647 - visual := funcapi.FieldVisualValue
648 - if col.dataType == ftDuration {
649 - visual = funcapi.FieldVisualBar
650 - }
651 - colDef := funcapi.Column{
652 - Index: i,
653 - Name: col.displayName,
654 - Type: col.dataType,
655 - Units: col.units,
656 - Visualization: visual,
657 - Sort: col.sortDir,
658 - Sortable: true,
659 - Sticky: col.isSticky,
660 - Summary: col.summary,
661 - Filter: col.filter,
662 - FullWidth: col.fullWidth,
663 - Wrap: false,
664 - DefaultExpandedFilter: false,
665 - UniqueKey: col.isUniqueKey,
666 - Visible: col.visible,
667 - ValueOptions: funcapi.ValueOptions{
668 - Transform: col.transform,
669 - DecimalPoints: col.decimalPoints,
670 - DefaultValue: nil,
671 - },
672 - }
673 - columns[col.uiKey] = colDef.BuildColumn()
674 - }
675 - return columns
676 -}
677 -
678 -// collectTopQueries queries Query Store for top queries using dynamic columns
679 -func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
680 - // Check if function is enabled
681 - if !c.Config.GetQueryStoreFunctionEnabled() {
682 - return &module.FunctionResponse{
683 - Status: 403,
684 - Message: "Query Store function has been disabled in configuration. " +
685 - "To enable, set query_store_function_enabled: true in the MSSQL collector config.",
686 - }
687 - }
688 -
689 - // Detect available columns
690 - availableCols, err := c.detectMSSQLQueryStoreColumns(ctx)
691 - if err != nil {
692 - return &module.FunctionResponse{
693 - Status: 500,
694 - Message: fmt.Sprintf("failed to detect available columns: %v", err),
695 - }
696 - }
697 -
698 - // Build list of available columns
699 - cols := c.buildAvailableMSSQLColumns(availableCols)
700 - if len(cols) == 0 {
701 - return &module.FunctionResponse{
702 - Status: 500,
703 - Message: "no columns available in Query Store",
704 - }
705 - }
706 -
707 - // Validate and map sort column (use filtered cols to ensure sort column is in SELECT)
708 - validatedSortColumn := c.mapAndValidateMSSQLSortColumn(sortColumn, cols)
709 -
710 - // Build and execute query
711 - timeWindowDays := c.Config.GetQueryStoreTimeWindowDays()
712 - limit := c.TopQueriesLimit
713 - if limit <= 0 {
714 - limit = 500
715 - }
716 - query := c.buildMSSQLDynamicSQL(cols, validatedSortColumn, timeWindowDays, limit)
717 -
718 - rows, err := c.db.QueryContext(ctx, query)
719 - if err != nil {
720 - if ctx.Err() == context.DeadlineExceeded {
721 - return &module.FunctionResponse{Status: 504, Message: "query timed out"}
722 - }
723 - // Include diagnostic info: which columns were detected and used
724 - colUIKeys := make([]string, len(cols))
725 - for i, col := range cols {
726 - colUIKeys[i] = col.uiKey
727 - }
728 - return &module.FunctionResponse{
729 - Status: 500,
730 - Message: fmt.Sprintf("query failed: %v (sort: %s, detected cols: %v)", err, validatedSortColumn, colUIKeys),
731 - }
732 - }
733 - defer rows.Close()
734 -
735 - // Scan rows dynamically
736 - data, err := c.scanMSSQLDynamicRows(rows, cols)
737 - if err != nil {
738 - return &module.FunctionResponse{Status: 500, Message: err.Error()}
739 - }
740 -
741 - // Build dynamic sort options from available columns (only those actually detected)
742 - sortParam, sortOptions := c.topQueriesSortParam(cols)
743 -
744 - // Find default sort column UI key
745 - defaultSort := ""
746 - for _, col := range cols {
747 - if col.isDefaultSort && col.isSortOption {
748 - defaultSort = col.uiKey
749 - break
750 - }
751 - }
752 - // Fallback to first sort option if no default
753 - if defaultSort == "" && len(sortOptions) > 0 {
754 - defaultSort = sortOptions[0].ID
755 - }
756 -
757 - annotatedCols := decorateMSSQLColumns(cols)
758 -
759 - return &module.FunctionResponse{
760 - Status: 200,
761 - Help: "Top SQL queries from Query Store. WARNING: Query text may contain unmasked literals (potential PII).",
762 - Columns: c.buildMSSQLDynamicColumns(cols),
763 - Data: data,
764 - DefaultSortColumn: defaultSort,
765 - RequiredParams: []funcapi.ParamConfig{sortParam},
766 -
767 - // Charts for aggregated visualization
768 - Charts: mssqlTopQueriesCharts(annotatedCols),
769 - DefaultCharts: mssqlTopQueriesDefaultCharts(annotatedCols),
770 - GroupBy: mssqlTopQueriesGroupBy(annotatedCols),
771 - }
772 -}
773 -
774 -func decorateMSSQLColumns(cols []mssqlColumnMeta) []mssqlColumnMeta {
775 - out := make([]mssqlColumnMeta, len(cols))
776 - index := make(map[string]int, len(cols))
777 - for i, col := range cols {
778 - out[i] = col
779 - index[col.uiKey] = i
780 - }
781 -
782 - for i := range out {
783 - if mssqlLabelColumns[out[i].uiKey] {
784 - out[i].isLabel = true
785 - if out[i].uiKey == mssqlPrimaryLabel {
786 - out[i].isPrimary = true
787 - }
788 - }
789 - }
790 -
791 - for _, group := range mssqlChartGroups {
792 - for _, key := range group.columns {
793 - idx, ok := index[key]
794 - if !ok {
795 - continue
796 - }
797 - out[idx].isMetric = true
798 - out[idx].chartGroup = group.key
799 - out[idx].chartTitle = group.title
800 - if group.defaultChart {
801 - out[idx].isDefaultChart = true
802 - }
803 - }
804 - }
805 -
806 - return out
807 -}
808 -
809 -func mssqlTopQueriesCharts(cols []mssqlColumnMeta) map[string]module.ChartConfig {
810 - charts := make(map[string]module.ChartConfig)
811 - for _, col := range cols {
812 - if !col.isMetric || col.chartGroup == "" {
813 - continue
814 - }
815 - cfg, ok := charts[col.chartGroup]
816 - if !ok {
817 - title := col.chartTitle
818 - if title == "" {
819 - title = col.chartGroup
820 - }
821 - cfg = module.ChartConfig{Name: title, Type: "stacked-bar"}
822 - }
823 - cfg.Columns = append(cfg.Columns, col.uiKey)
824 - charts[col.chartGroup] = cfg
825 - }
826 - return charts
827 -}
828 -
829 -func mssqlTopQueriesDefaultCharts(cols []mssqlColumnMeta) [][]string {
830 - label := primaryMSSQLLabel(cols)
831 - if label == "" {
832 - return nil
833 - }
834 - chartGroups := defaultMSSQLChartGroups(cols)
835 - out := make([][]string, 0, len(chartGroups))
836 - for _, group := range chartGroups {
837 - out = append(out, []string{group, label})
838 - }
839 - return out
840 -}
841 -
842 -func mssqlTopQueriesGroupBy(cols []mssqlColumnMeta) map[string]module.GroupByConfig {
843 - groupBy := make(map[string]module.GroupByConfig)
844 - for _, col := range cols {
845 - if !col.isLabel {
846 - continue
847 - }
848 - groupBy[col.uiKey] = module.GroupByConfig{
849 - Name: "Group by " + col.displayName,
850 - Columns: []string{col.uiKey},
851 - }
852 - }
853 - return groupBy
854 -}
855 -
856 -func primaryMSSQLLabel(cols []mssqlColumnMeta) string {
857 - for _, col := range cols {
858 - if col.isPrimary {
859 - return col.uiKey
860 - }
861 - }
862 - for _, col := range cols {
863 - if col.isLabel {
864 - return col.uiKey
865 - }
866 - }
867 - return ""
868 -}
869 -
870 -func defaultMSSQLChartGroups(cols []mssqlColumnMeta) []string {
871 - groups := make([]string, 0)
872 - seen := make(map[string]bool)
873 - for _, col := range cols {
874 - if !col.isMetric || col.chartGroup == "" || !col.isDefaultChart {
875 - continue
876 - }
877 - if !seen[col.chartGroup] {
878 - seen[col.chartGroup] = true
879 - groups = append(groups, col.chartGroup)
880 - }
881 - }
882 - if len(groups) > 0 {
883 - return groups
884 - }
885 - for _, col := range cols {
886 - if !col.isMetric || col.chartGroup == "" {
887 - continue
888 - }
889 - if !seen[col.chartGroup] {
890 - seen[col.chartGroup] = true
891 - groups = append(groups, col.chartGroup)
892 - }
893 - }
894 - return groups
895 -}
src/go/plugin/go.d/collector/mssql/functions_test.go deleted
-321
@@ -1,321 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -package mssql
4 -
5 -import (
6 - "testing"
7 -
8 - "github.com/netdata/netdata/go/plugins/pkg/funcapi"
9 - "github.com/stretchr/testify/assert"
10 -)
11 -
12 -func TestMssqlMethods(t *testing.T) {
13 - methods := mssqlMethods()
14 -
15 - require := assert.New(t)
16 - require.Len(methods, 1)
17 - require.Equal("top-queries", methods[0].ID)
18 - require.Equal("Top Queries", methods[0].Name)
19 - require.NotEmpty(methods[0].RequiredParams)
20 -
21 - // Verify at least one default sort option exists
22 - var sortParam *funcapi.ParamConfig
23 - for i := range methods[0].RequiredParams {
24 - if methods[0].RequiredParams[i].ID == "__sort" {
25 - sortParam = &methods[0].RequiredParams[i]
26 - break
27 - }
28 - }
29 - require.NotNil(sortParam, "expected __sort required param")
30 - require.NotEmpty(sortParam.Options)
31 -
32 - hasDefault := false
33 - for _, opt := range sortParam.Options {
34 - if opt.Default {
35 - hasDefault = true
36 - require.Equal("totalTime", opt.ID) // camelCase for UI
37 - break
38 - }
39 - }
40 - require.True(hasDefault, "should have a default sort option")
41 -}
42 -
43 -func TestMssqlAllColumns_HasRequiredColumns(t *testing.T) {
44 - // Verify all required base columns are defined
45 - requiredUIKeys := []string{
46 - "queryHash", "query", "database", "calls",
47 - "totalTime", "avgTime", "avgCpu",
48 - "avgReads", "avgWrites",
49 - }
50 -
51 - uiKeys := make(map[string]bool)
52 - for _, col := range mssqlAllColumns {
53 - uiKeys[col.uiKey] = true
54 - }
55 -
56 - for _, key := range requiredUIKeys {
57 - assert.True(t, uiKeys[key], "column %s should be defined in mssqlAllColumns", key)
58 - }
59 -}
60 -
61 -func TestMssqlAllColumns_HasValidMetadata(t *testing.T) {
62 - for _, col := range mssqlAllColumns {
63 - // Every column must have a UI key
64 - assert.NotEmpty(t, col.uiKey, "column %s must have uiKey", col.dbColumn)
65 -
66 - // Every column must have a display name
67 - assert.NotEmpty(t, col.displayName, "column %s must have displayName", col.uiKey)
68 -
69 - // Every column must have a data type
70 - assert.NotEqual(t, funcapi.FieldTypeNone, col.dataType, "column %s must have dataType", col.uiKey)
71 -
72 - // Duration columns must have units
73 - if col.dataType == ftDuration {
74 - assert.NotEmpty(t, col.units, "duration column %s must have units", col.uiKey)
75 - }
76 -
77 - // Sort options must have labels
78 - if col.isSortOption {
79 - assert.NotEmpty(t, col.sortLabel, "sort option column %s must have sortLabel", col.uiKey)
80 - }
81 - }
82 -}
83 -
84 -func TestCollector_mapAndValidateMSSQLSortColumn(t *testing.T) {
85 - // Build a filtered column list with common available columns
86 - availableCols := map[string]bool{"avg_duration": true, "count_executions": true}
87 - c := &Collector{}
88 - cols := c.buildAvailableMSSQLColumns(availableCols)
89 -
90 - tests := map[string]struct {
91 - cols []mssqlColumnMeta
92 - input string
93 - expected string
94 - }{
95 - "totalTime maps correctly": {
96 - cols: cols,
97 - input: "totalTime",
98 - expected: "totalTime",
99 - },
100 - "calls maps correctly": {
101 - cols: cols,
102 - input: "calls",
103 - expected: "calls",
104 - },
105 - "invalid column falls back to first sort option": {
106 - cols: cols,
107 - input: "invalid_column",
108 - expected: "calls", // first sort option in filtered cols
109 - },
110 - "SQL injection attempt falls back to first sort option": {
111 - cols: cols,
112 - input: "'; DROP TABLE users;--",
113 - expected: "calls", // first sort option in filtered cols
114 - },
115 - }
116 -
117 - for name, tc := range tests {
118 - t.Run(name, func(t *testing.T) {
119 - result := c.mapAndValidateMSSQLSortColumn(tc.input, tc.cols)
120 - assert.Equal(t, tc.expected, result)
121 - })
122 - }
123 -}
124 -
125 -func TestCollector_buildAvailableMSSQLColumns(t *testing.T) {
126 - tests := map[string]struct {
127 - availableCols map[string]bool
128 - expectCols []string // UI keys we expect to see
129 - notExpectCols []string // UI keys we don't expect
130 - }{
131 - "SQL Server 2016 columns": {
132 - availableCols: map[string]bool{
133 - "count_executions": true,
134 - "avg_duration": true, "last_duration": true, "min_duration": true, "max_duration": true,
135 - "avg_cpu_time": true, "last_cpu_time": true, "min_cpu_time": true, "max_cpu_time": true,
136 - "avg_logical_io_reads": true, "avg_logical_io_writes": true,
137 - },
138 - expectCols: []string{"queryHash", "query", "database", "calls", "totalTime", "avgTime", "avgCpu", "avgReads"},
139 - notExpectCols: []string{"avgLogBytes", "avgTempdb"}, // SQL Server 2017+ only
140 - },
141 - "SQL Server 2017 with log bytes and tempdb": {
142 - availableCols: map[string]bool{
143 - "count_executions": true,
144 - "avg_duration": true,
145 - "avg_cpu_time": true,
146 - "avg_log_bytes_used": true,
147 - "avg_tempdb_space_used": true,
148 - },
149 - expectCols: []string{"queryHash", "query", "calls", "avgLogBytes", "avgTempdb"},
150 - },
151 - }
152 -
153 - for name, tc := range tests {
154 - t.Run(name, func(t *testing.T) {
155 - c := &Collector{}
156 - cols := c.buildAvailableMSSQLColumns(tc.availableCols)
157 -
158 - // Build map of UI keys for easy lookup
159 - uiKeys := make(map[string]bool)
160 - for _, col := range cols {
161 - uiKeys[col.uiKey] = true
162 - }
163 -
164 - for _, key := range tc.expectCols {
165 - assert.True(t, uiKeys[key], "expected column %s to be present", key)
166 - }
167 - for _, key := range tc.notExpectCols {
168 - assert.False(t, uiKeys[key], "did not expect column %s to be present", key)
169 - }
170 - })
171 - }
172 -}
173 -
174 -func TestCollector_buildMSSQLDynamicSQL(t *testing.T) {
175 - c := &Collector{}
176 -
177 - cols := []mssqlColumnMeta{
178 - {dbColumn: "query_hash", uiKey: "queryHash", dataType: ftString, isIdentity: true},
179 - {dbColumn: "query_sql_text", uiKey: "query", dataType: ftString, isIdentity: true},
180 - {dbColumn: "database_name", uiKey: "database", dataType: ftString, isIdentity: true},
181 - {dbColumn: "count_executions", uiKey: "calls", dataType: ftInteger},
182 - {dbColumn: "avg_duration", uiKey: "totalTime", dataType: ftDuration, isMicroseconds: true},
183 - }
184 -
185 - // sortColumn is the uiKey which is also used as the SQL alias
186 - sql := c.buildMSSQLDynamicSQL(cols, "totalTime", 7, 500)
187 -
188 - // Basic query structure
189 - assert.Contains(t, sql, "sys.query_store_query")
190 - assert.Contains(t, sql, "AS [totalTime]")
191 - assert.Contains(t, sql, "ORDER BY [totalTime] DESC")
192 - assert.Contains(t, sql, "TOP 500")
193 - assert.Contains(t, sql, "DATEADD")
194 - assert.Contains(t, sql, "q.query_hash")
195 -
196 - // Cross-database aggregation features
197 - assert.Contains(t, sql, "QUOTENAME(name)") // Safe database name escaping
198 - assert.Contains(t, sql, "UNION ALL") // Combining results from multiple databases
199 - assert.Contains(t, sql, "sp_executesql") // Executing dynamic SQL
200 - assert.Contains(t, sql, "sys.databases") // Finding databases with Query Store
201 - assert.Contains(t, sql, "is_query_store_on = 1") // Condition for Query Store enabled
202 - assert.Contains(t, sql, "NOT IN ('master', 'tempdb', 'model', 'msdb')") // Excluding system databases
203 -}
204 -
205 -func TestCollector_buildMSSQLDynamicSQL_NoTimeFilter(t *testing.T) {
206 - c := &Collector{}
207 -
208 - cols := []mssqlColumnMeta{
209 - {dbColumn: "query_hash", uiKey: "queryHash", dataType: ftString, isIdentity: true},
210 - {dbColumn: "count_executions", uiKey: "calls", dataType: ftInteger},
211 - }
212 -
213 - sql := c.buildMSSQLDynamicSQL(cols, "calls", 0, 500)
214 -
215 - assert.Contains(t, sql, "sys.query_store_query")
216 - assert.Contains(t, sql, "TOP 500")
217 - assert.Contains(t, sql, "ORDER BY [calls] DESC")
218 - assert.NotContains(t, sql, "DATEADD")
219 -}
220 -
221 -func TestCollector_buildMSSQLDynamicColumns(t *testing.T) {
222 - c := &Collector{}
223 -
224 - cols := []mssqlColumnMeta{
225 - {uiKey: "queryHash", displayName: "Query Hash", dataType: ftString, visible: false, isUniqueKey: true, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti},
226 - {uiKey: "query", displayName: "Query", dataType: ftString, visible: true, isSticky: true, fullWidth: true, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti},
227 - {uiKey: "totalTime", displayName: "Total Time", dataType: ftDuration, units: "seconds", visible: true, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summarySum, filter: filterRange},
228 - }
229 -
230 - columns := c.buildMSSQLDynamicColumns(cols)
231 -
232 - // Verify column count
233 - assert.Len(t, columns, 3)
234 -
235 - // Verify queryHash column
236 - queryHashCol := columns["queryHash"].(map[string]any)
237 - assert.Equal(t, "Query Hash", queryHashCol["name"])
238 - assert.Equal(t, "string", queryHashCol["type"])
239 - assert.True(t, queryHashCol["unique_key"].(bool))
240 - assert.False(t, queryHashCol["visible"].(bool))
241 - assert.Equal(t, 0, queryHashCol["index"])
242 -
243 - // Verify query column
244 - queryCol := columns["query"].(map[string]any)
245 - assert.Equal(t, "Query", queryCol["name"])
246 - assert.True(t, queryCol["sticky"].(bool))
247 - assert.True(t, queryCol["full_width"].(bool))
248 - assert.Equal(t, 1, queryCol["index"])
249 -
250 - // Verify totalTime column
251 - totalTimeCol := columns["totalTime"].(map[string]any)
252 - assert.Equal(t, "Total Time", totalTimeCol["name"])
253 - assert.Equal(t, "duration", totalTimeCol["type"])
254 - assert.Equal(t, "seconds", totalTimeCol["units"])
255 - assert.Equal(t, "bar", totalTimeCol["visualization"]) // duration uses bar
256 - assert.Equal(t, 2, totalTimeCol["index"])
257 -}
258 -
259 -// Test that method config sort options have valid column references
260 -func TestMssqlMethods_SortOptionsHaveLabels(t *testing.T) {
261 - methods := mssqlMethods()
262 -
263 - for _, method := range methods {
264 - var sortParam *funcapi.ParamConfig
265 - for i := range method.RequiredParams {
266 - if method.RequiredParams[i].ID == "__sort" {
267 - sortParam = &method.RequiredParams[i]
268 - break
269 - }
270 - }
271 - assert.NotNil(t, sortParam)
272 - for _, opt := range sortParam.Options {
273 - assert.NotEmpty(t, opt.ID, "sort option must have ID")
274 - assert.NotEmpty(t, opt.Name, "sort option %s must have Name", opt.ID)
275 - assert.Contains(t, opt.Name, "Top queries by", "label should have standard prefix")
276 - }
277 - }
278 -}
279 -
280 -// TestMapAndValidateMSSQLSortColumn_NoSortOptions verifies fallback when no sort columns exist
281 -func TestMapAndValidateMSSQLSortColumn_NoSortOptions(t *testing.T) {
282 - c := &Collector{}
283 -
284 - // Only identity columns available (no sort options)
285 - identityOnlyCols := []mssqlColumnMeta{
286 - {uiKey: "queryHash", isIdentity: true},
287 - {uiKey: "query", isIdentity: true},
288 - {uiKey: "database", isIdentity: true},
289 - }
290 -
291 - result := c.mapAndValidateMSSQLSortColumn("totalTime", identityOnlyCols)
292 - // Should fall back to first column since no sort options exist
293 - assert.Equal(t, "queryHash", result, "should fall back to first column when no sort options")
294 -
295 - // Empty columns list
296 - result = c.mapAndValidateMSSQLSortColumn("totalTime", []mssqlColumnMeta{})
297 - assert.Equal(t, "", result, "should return empty string when no columns available")
298 -}
299 -
300 -// TestSortColumnValidation_SQLInjection verifies that SQL injection attempts
301 -// are handled by the validation mechanism
302 -func TestMssqlSortColumnValidation_SQLInjection(t *testing.T) {
303 - c := &Collector{}
304 - availableCols := map[string]bool{"avg_duration": true, "count_executions": true}
305 - cols := c.buildAvailableMSSQLColumns(availableCols)
306 -
307 - maliciousInputs := []string{
308 - "'; DROP TABLE sys.query_store_query; --",
309 - "total_time_ms; DELETE FROM master.dbo.sysdatabases",
310 - "1 OR 1=1",
311 - "WAITFOR DELAY '00:00:10'",
312 - "xp_cmdshell 'whoami'",
313 - }
314 -
315 - for _, input := range maliciousInputs {
316 - result := c.mapAndValidateMSSQLSortColumn(input, cols)
317 - // All malicious inputs should fall back to first available sort option
318 - assert.Equal(t, "calls", result,
319 - "malicious input should fall back to safe default: %s -> %s", input, result)
320 - }
321 -}
src/go/plugin/go.d/collector/mysql/collector.go
+9 -3
@@ -28,8 +28,7 @@ func init() {
28 Create: func() module.Module { return New() },
29 Config: func() any { return &Config{} },
30 Methods: mysqlMethods,
31 - MethodParams: mysqlMethodParams,
32 - HandleMethod: mysqlHandleMethod,
31 + MethodHandler: mysqlFunctionHandler,
32 })
33 }
34
@@ -113,6 +112,8 @@ type Collector struct {
112
113 stmtSummaryCols map[string]bool // cached column names from events_statements_summary_by_digest
114 stmtSummaryColsMu sync.RWMutex // protects stmtSummaryCols for concurrent access
115 +
116 + funcRouter *funcRouter
117 }
118
119 func (c *Collector) Configuration() any {
@@ -142,6 +143,8 @@ func (c *Collector) Init(context.Context) error {
143
144 c.Debugf("using DSN [%s]", c.DSN)
145
146 + c.funcRouter = newFuncRouter(c)
147 +
148 return nil
149 }
150
@@ -172,7 +175,10 @@ func (c *Collector) Collect(context.Context) map[string]int64 {
175 return mx
176 }
177
175 -func (c *Collector) Cleanup(context.Context) {
178 +func (c *Collector) Cleanup(ctx context.Context) {
179 + if c.funcRouter != nil {
180 + c.funcRouter.Cleanup(ctx)
181 + }
182 if c.db == nil {
183 return
184 }
src/go/plugin/go.d/collector/mysql/func_router.go new
+64
@@ -0,0 +1,64 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package mysql
4 +
5 +import (
6 + "context"
7 + "fmt"
8 +
9 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
10 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
11 +)
12 +
13 +// funcRouter routes method calls to appropriate function handlers.
14 +type funcRouter struct {
15 + collector *Collector
16 +
17 + handlers map[string]funcapi.MethodHandler
18 +}
19 +
20 +func newFuncRouter(c *Collector) *funcRouter {
21 + r := &funcRouter{
22 + collector: c,
23 + handlers: make(map[string]funcapi.MethodHandler),
24 + }
25 + r.handlers[topQueriesMethodID] = newFuncTopQueries(r)
26 + return r
27 +}
28 +
29 +// Compile-time interface check.
30 +var _ funcapi.MethodHandler = (*funcRouter)(nil)
31 +
32 +func (r *funcRouter) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
33 + if h, ok := r.handlers[method]; ok {
34 + return h.MethodParams(ctx, method)
35 + }
36 + return nil, fmt.Errorf("unknown method: %s", method)
37 +}
38 +
39 +func (r *funcRouter) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
40 + if h, ok := r.handlers[method]; ok {
41 + return h.Handle(ctx, method, params)
42 + }
43 + return funcapi.NotFoundResponse(method)
44 +}
45 +
46 +func (r *funcRouter) Cleanup(ctx context.Context) {
47 + for _, h := range r.handlers {
48 + h.Cleanup(ctx)
49 + }
50 +}
51 +
52 +func mysqlMethods() []funcapi.MethodConfig {
53 + return []funcapi.MethodConfig{
54 + topQueriesMethodConfig(),
55 + }
56 +}
57 +
58 +func mysqlFunctionHandler(job *module.Job) funcapi.MethodHandler {
59 + c, ok := job.Module().(*Collector)
60 + if !ok {
61 + return nil
62 + }
63 + return c.funcRouter
64 +}
src/go/plugin/go.d/collector/mysql/func_top_queries.go new
+567
@@ -0,0 +1,567 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package mysql
4 +
5 +import (
6 + "context"
7 + "database/sql"
8 + "fmt"
9 + "strings"
10 +
11 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
12 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
13 +)
14 +
15 +const (
16 + topQueriesMethodID = "top-queries"
17 + topQueriesMaxTextLength = 4096
18 + topQueriesParamSort = "__sort"
19 +)
20 +
21 +func topQueriesMethodConfig() funcapi.MethodConfig {
22 + return funcapi.MethodConfig{
23 + ID: topQueriesMethodID,
24 + Name: "Top Queries",
25 + UpdateEvery: 10,
26 + Help: "Top SQL queries from performance_schema",
27 + RequireCloud: true,
28 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(topQueriesColumns)},
29 + }
30 +}
31 +
32 +// topQueriesColumn defines a column for MySQL top-queries function.
33 +// Embeds funcapi.ColumnMeta for UI display and adds collector-specific fields.
34 +type topQueriesColumn struct {
35 + funcapi.ColumnMeta
36 +
37 + // Data access
38 + DBColumn string // Column name in database (e.g., "SUM_TIMER_WAIT")
39 + IsPicoseconds bool // Needs picoseconds to milliseconds conversion
40 +
41 + // Sort parameter metadata
42 + sortOpt bool // Show in sort dropdown
43 + sortLbl string // Label for sort option
44 + defaultSort bool // Is this the default sort option
45 +}
46 +
47 +// funcapi.SortableColumn interface implementation for topQueriesColumn.
48 +func (c topQueriesColumn) IsSortOption() bool { return c.sortOpt }
49 +func (c topQueriesColumn) SortLabel() string { return c.sortLbl }
50 +func (c topQueriesColumn) IsDefaultSort() bool { return c.defaultSort }
51 +func (c topQueriesColumn) ColumnName() string { return c.Name }
52 +func (c topQueriesColumn) SortColumn() string { return "" }
53 +
54 +// topQueriesColumns defines ALL possible columns from events_statements_summary_by_digest
55 +// Columns that don't exist in certain MySQL/MariaDB versions will be filtered at runtime
56 +var topQueriesColumns = []topQueriesColumn{
57 + // Identity columns - always available
58 + {ColumnMeta: funcapi.ColumnMeta{Name: "digest", Tooltip: "Digest", Type: funcapi.FieldTypeString, Visible: false, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, UniqueKey: true, Sortable: true}, DBColumn: "DIGEST"},
59 + {ColumnMeta: funcapi.ColumnMeta{Name: "query", Tooltip: "Query", Type: funcapi.FieldTypeString, Visible: true, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, Sticky: true, FullWidth: true, Sortable: true}, DBColumn: "DIGEST_TEXT"},
60 + {ColumnMeta: funcapi.ColumnMeta{Name: "schema", Tooltip: "Schema", Type: funcapi.FieldTypeString, Visible: true, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, Sortable: true}, DBColumn: "SCHEMA_NAME"},
61 +
62 + // Execution counts
63 + {ColumnMeta: funcapi.ColumnMeta{Name: "calls", Tooltip: "Calls", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "COUNT_STAR", sortOpt: true, sortLbl: "Top queries by Number of Calls"},
64 +
65 + // Timer metrics (picoseconds -> milliseconds)
66 + {ColumnMeta: funcapi.ColumnMeta{Name: "totalTime", Tooltip: "Total Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: true, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "SUM_TIMER_WAIT", IsPicoseconds: true, sortOpt: true, sortLbl: "Top queries by Total Execution Time", defaultSort: true},
67 + {ColumnMeta: funcapi.ColumnMeta{Name: "minTime", Tooltip: "Min Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: false, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMin, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "MIN_TIMER_WAIT", IsPicoseconds: true},
68 + {ColumnMeta: funcapi.ColumnMeta{Name: "avgTime", Tooltip: "Avg Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: true, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMean, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "AVG_TIMER_WAIT", IsPicoseconds: true, sortOpt: true, sortLbl: "Top queries by Average Execution Time"},
69 + {ColumnMeta: funcapi.ColumnMeta{Name: "maxTime", Tooltip: "Max Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: false, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "MAX_TIMER_WAIT", IsPicoseconds: true},
70 +
71 + // Lock time (picoseconds -> milliseconds)
72 + {ColumnMeta: funcapi.ColumnMeta{Name: "lockTime", Tooltip: "Lock Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: true, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "SUM_LOCK_TIME", IsPicoseconds: true, sortOpt: true, sortLbl: "Top queries by Lock Time"},
73 +
74 + // Error and warning counts
75 + {ColumnMeta: funcapi.ColumnMeta{Name: "errors", Tooltip: "Errors", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "SUM_ERRORS", sortOpt: true, sortLbl: "Top queries by Errors"},
76 + {ColumnMeta: funcapi.ColumnMeta{Name: "warnings", Tooltip: "Warnings", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "SUM_WARNINGS", sortOpt: true, sortLbl: "Top queries by Warnings"},
77 +
78 + // Row operations
79 + {ColumnMeta: funcapi.ColumnMeta{Name: "rowsAffected", Tooltip: "Rows Affected", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "SUM_ROWS_AFFECTED", sortOpt: true, sortLbl: "Top queries by Rows Affected"},
80 + {ColumnMeta: funcapi.ColumnMeta{Name: "rowsSent", Tooltip: "Rows Sent", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "SUM_ROWS_SENT", sortOpt: true, sortLbl: "Top queries by Rows Sent"},
81 + {ColumnMeta: funcapi.ColumnMeta{Name: "rowsExamined", Tooltip: "Rows Examined", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "SUM_ROWS_EXAMINED", sortOpt: true, sortLbl: "Top queries by Rows Examined"},
82 +
83 + // Temp table usage
84 + {ColumnMeta: funcapi.ColumnMeta{Name: "tmpDiskTables", Tooltip: "Temp Disk Tables", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "SUM_CREATED_TMP_DISK_TABLES", sortOpt: true, sortLbl: "Top queries by Temp Disk Tables"},
85 + {ColumnMeta: funcapi.ColumnMeta{Name: "tmpTables", Tooltip: "Temp Tables", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "SUM_CREATED_TMP_TABLES", sortOpt: true, sortLbl: "Top queries by Temp Tables"},
86 +
87 + // Join operations
88 + {ColumnMeta: funcapi.ColumnMeta{Name: "fullJoin", Tooltip: "Full Joins", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "SUM_SELECT_FULL_JOIN", sortOpt: true, sortLbl: "Top queries by Full Joins"},
89 + {ColumnMeta: funcapi.ColumnMeta{Name: "fullRangeJoin", Tooltip: "Full Range Joins", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "SUM_SELECT_FULL_RANGE_JOIN"},
90 + {ColumnMeta: funcapi.ColumnMeta{Name: "selectRange", Tooltip: "Select Range", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "SUM_SELECT_RANGE"},
91 + {ColumnMeta: funcapi.ColumnMeta{Name: "selectRangeCheck", Tooltip: "Select Range Check", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "SUM_SELECT_RANGE_CHECK"},
92 + {ColumnMeta: funcapi.ColumnMeta{Name: "selectScan", Tooltip: "Select Scan", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "SUM_SELECT_SCAN", sortOpt: true, sortLbl: "Top queries by Table Scans"},
93 +
94 + // Sort operations
95 + {ColumnMeta: funcapi.ColumnMeta{Name: "sortMergePasses", Tooltip: "Sort Merge Passes", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "SUM_SORT_MERGE_PASSES"},
96 + {ColumnMeta: funcapi.ColumnMeta{Name: "sortRange", Tooltip: "Sort Range", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "SUM_SORT_RANGE"},
97 + {ColumnMeta: funcapi.ColumnMeta{Name: "sortRows", Tooltip: "Sort Rows", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "SUM_SORT_ROWS", sortOpt: true, sortLbl: "Top queries by Rows Sorted"},
98 + {ColumnMeta: funcapi.ColumnMeta{Name: "sortScan", Tooltip: "Sort Scan", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "SUM_SORT_SCAN"},
99 +
100 + // Index usage
101 + {ColumnMeta: funcapi.ColumnMeta{Name: "noIndexUsed", Tooltip: "No Index Used", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "SUM_NO_INDEX_USED", sortOpt: true, sortLbl: "Top queries by No Index Used"},
102 + {ColumnMeta: funcapi.ColumnMeta{Name: "noGoodIndexUsed", Tooltip: "No Good Index Used", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "SUM_NO_GOOD_INDEX_USED"},
103 +
104 + // Timestamp columns
105 + {ColumnMeta: funcapi.ColumnMeta{Name: "firstSeen", Tooltip: "First Seen", Type: funcapi.FieldTypeString, Visible: false, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, Sortable: true}, DBColumn: "FIRST_SEEN"},
106 + {ColumnMeta: funcapi.ColumnMeta{Name: "lastSeen", Tooltip: "Last Seen", Type: funcapi.FieldTypeString, Visible: false, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, Sortable: true}, DBColumn: "LAST_SEEN"},
107 +
108 + // MySQL 8.0+ quantile columns
109 + {ColumnMeta: funcapi.ColumnMeta{Name: "p95Time", Tooltip: "P95 Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: true, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "QUANTILE_95", IsPicoseconds: true, sortOpt: true, sortLbl: "Top queries by 95th Percentile Time"},
110 + {ColumnMeta: funcapi.ColumnMeta{Name: "p99Time", Tooltip: "P99 Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: true, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "QUANTILE_99", IsPicoseconds: true, sortOpt: true, sortLbl: "Top queries by 99th Percentile Time"},
111 + {ColumnMeta: funcapi.ColumnMeta{Name: "p999Time", Tooltip: "P99.9 Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: false, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "QUANTILE_999", IsPicoseconds: true},
112 +
113 + // MySQL 8.0+ sample query
114 + {ColumnMeta: funcapi.ColumnMeta{Name: "sampleQuery", Tooltip: "Sample Query", Type: funcapi.FieldTypeString, Visible: false, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, FullWidth: true, Sortable: true}, DBColumn: "QUERY_SAMPLE_TEXT"},
115 + {ColumnMeta: funcapi.ColumnMeta{Name: "sampleSeen", Tooltip: "Sample Seen", Type: funcapi.FieldTypeString, Visible: false, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, Sortable: true}, DBColumn: "QUERY_SAMPLE_SEEN"},
116 + {ColumnMeta: funcapi.ColumnMeta{Name: "sampleTime", Tooltip: "Sample Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: false, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "QUERY_SAMPLE_TIMER_WAIT", IsPicoseconds: true},
117 +
118 + // MySQL 8.0.28+ CPU time
119 + {ColumnMeta: funcapi.ColumnMeta{Name: "cpuTime", Tooltip: "CPU Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: true, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "SUM_CPU_TIME", IsPicoseconds: true, sortOpt: true, sortLbl: "Top queries by CPU Time"},
120 +
121 + // MySQL 8.0.31+ memory columns
122 + {ColumnMeta: funcapi.ColumnMeta{Name: "maxControlledMemory", Tooltip: "Max Controlled Memory", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "MAX_CONTROLLED_MEMORY", sortOpt: true, sortLbl: "Top queries by Max Controlled Memory"},
123 + {ColumnMeta: funcapi.ColumnMeta{Name: "maxTotalMemory", Tooltip: "Max Total Memory", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "MAX_TOTAL_MEMORY", sortOpt: true, sortLbl: "Top queries by Max Total Memory"},
124 +}
125 +
126 +type topQueriesChartGroup struct {
127 + key string
128 + title string
129 + columns []string
130 + defaultChart bool
131 +}
132 +
133 +var topQueriesChartGroups = []topQueriesChartGroup{
134 + {key: "Calls", title: "Number of Calls", columns: []string{"calls"}, defaultChart: true},
135 + {key: "Time", title: "Execution Time", columns: []string{"totalTime", "avgTime", "minTime", "maxTime"}, defaultChart: true},
136 + {key: "Percentiles", title: "Execution Time Percentiles", columns: []string{"p95Time", "p99Time", "p999Time"}},
137 + {key: "LockTime", title: "Lock Time", columns: []string{"lockTime"}},
138 + {key: "Errors", title: "Errors & Warnings", columns: []string{"errors", "warnings"}},
139 + {key: "Rows", title: "Rows", columns: []string{"rowsSent", "rowsExamined", "rowsAffected"}},
140 + {key: "TempTables", title: "Temp Tables", columns: []string{"tmpDiskTables", "tmpTables"}},
141 + {key: "Joins", title: "Join Operations", columns: []string{"fullJoin", "fullRangeJoin", "selectRange", "selectRangeCheck", "selectScan"}},
142 + {key: "Sort", title: "Sort Operations", columns: []string{"sortMergePasses", "sortRange", "sortRows", "sortScan"}},
143 + {key: "Index", title: "Index Usage", columns: []string{"noIndexUsed", "noGoodIndexUsed"}},
144 + {key: "CPU", title: "CPU Time", columns: []string{"cpuTime"}},
145 + {key: "Memory", title: "Memory", columns: []string{"maxControlledMemory", "maxTotalMemory"}},
146 + {key: "Sample", title: "Sample Time", columns: []string{"sampleTime"}},
147 +}
148 +
149 +var topQueriesLabelColumns = map[string]bool{
150 + "schema": true,
151 +}
152 +
153 +const topQueriesPrimaryLabel = "schema"
154 +
155 +// topQueriesRowScanner interface for testing
156 +type topQueriesRowScanner interface {
157 + Next() bool
158 + Scan(dest ...any) error
159 + Err() error
160 +}
161 +
162 +// funcTopQueries implements funcapi.MethodHandler for MySQL top-queries.
163 +// All function-related logic is encapsulated here, keeping Collector focused on metrics collection.
164 +type funcTopQueries struct {
165 + router *funcRouter
166 +}
167 +
168 +func newFuncTopQueries(r *funcRouter) *funcTopQueries {
169 + return &funcTopQueries{router: r}
170 +}
171 +
172 +// Compile-time interface check.
173 +var _ funcapi.MethodHandler = (*funcTopQueries)(nil)
174 +
175 +// MethodParams implements funcapi.MethodHandler.
176 +func (f *funcTopQueries) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
177 + if f.router.collector.db == nil {
178 + return nil, fmt.Errorf("collector is still initializing")
179 + }
180 + switch method {
181 + case topQueriesMethodID:
182 + return f.methodParams(ctx)
183 + default:
184 + return nil, fmt.Errorf("unknown method: %s", method)
185 + }
186 +}
187 +
188 +// Handle implements funcapi.MethodHandler.
189 +func (f *funcTopQueries) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
190 + if f.router.collector.db == nil {
191 + return funcapi.UnavailableResponse("collector is still initializing, please retry in a few seconds")
192 + }
193 +
194 + switch method {
195 + case topQueriesMethodID:
196 + return f.collectData(ctx, params.Column(topQueriesParamSort))
197 + default:
198 + return funcapi.NotFoundResponse(method)
199 + }
200 +}
201 +
202 +// Cleanup implements funcapi.MethodHandler.
203 +func (f *funcTopQueries) Cleanup(ctx context.Context) {}
204 +
205 +func (f *funcTopQueries) methodParams(ctx context.Context) ([]funcapi.ParamConfig, error) {
206 + available, err := f.checkPerformanceSchema(ctx)
207 + if err != nil {
208 + return nil, err
209 + }
210 + if !available {
211 + return nil, fmt.Errorf("performance_schema is not enabled")
212 + }
213 +
214 + availableCols, err := f.detectStatementsColumns(ctx)
215 + if err != nil {
216 + return nil, err
217 + }
218 + cols := f.buildAvailableColumns(availableCols)
219 + if len(cols) == 0 {
220 + return nil, fmt.Errorf("no columns available in events_statements_summary_by_digest")
221 + }
222 +
223 + sortParam := f.buildSortParam(cols)
224 + return []funcapi.ParamConfig{sortParam}, nil
225 +}
226 +
227 +// collectData queries performance_schema for top queries using dynamic columns
228 +func (f *funcTopQueries) collectData(ctx context.Context, sortColumn string) *funcapi.FunctionResponse {
229 + // Check if performance_schema is enabled
230 + available, err := f.checkPerformanceSchema(ctx)
231 + if err != nil {
232 + return &funcapi.FunctionResponse{
233 + Status: 500,
234 + Message: fmt.Sprintf("failed to check performance_schema availability: %v", err),
235 + }
236 + }
237 + if !available {
238 + return &funcapi.FunctionResponse{
239 + Status: 503,
240 + Message: "performance_schema is not enabled",
241 + }
242 + }
243 +
244 + // Detect available columns
245 + availableCols, err := f.detectStatementsColumns(ctx)
246 + if err != nil {
247 + return &funcapi.FunctionResponse{
248 + Status: 500,
249 + Message: fmt.Sprintf("failed to detect available columns: %v", err),
250 + }
251 + }
252 +
253 + // Build list of available columns
254 + cols := f.buildAvailableColumns(availableCols)
255 + if len(cols) == 0 {
256 + return &funcapi.FunctionResponse{
257 + Status: 500,
258 + Message: "no columns available in events_statements_summary_by_digest",
259 + }
260 + }
261 +
262 + // Validate and map sort column
263 + dbSortColumn := f.mapAndValidateSortColumn(sortColumn, availableCols)
264 +
265 + // Get query limit (default 500)
266 + limit := f.router.collector.TopQueriesLimit
267 + if limit <= 0 {
268 + limit = 500
269 + }
270 +
271 + // Build and execute query
272 + query := f.buildDynamicSQL(cols, dbSortColumn, limit)
273 +
274 + rows, err := f.router.collector.db.QueryContext(ctx, query)
275 + if err != nil {
276 + if ctx.Err() == context.DeadlineExceeded {
277 + return &funcapi.FunctionResponse{Status: 504, Message: "query timed out"}
278 + }
279 + return &funcapi.FunctionResponse{Status: 500, Message: fmt.Sprintf("query failed: %v", err)}
280 + }
281 + defer rows.Close()
282 +
283 + // Scan rows dynamically
284 + data, err := f.scanDynamicRows(rows, cols)
285 + if err != nil {
286 + return &funcapi.FunctionResponse{Status: 500, Message: err.Error()}
287 + }
288 +
289 + // Build dynamic sort options from available columns (only those actually detected)
290 + sortParam := f.buildSortParam(cols)
291 + sortOptions := sortParam.Options
292 +
293 + // Find default sort column ID
294 + defaultSort := ""
295 + for _, col := range cols {
296 + if col.IsDefaultSort() && col.IsSortOption() {
297 + defaultSort = col.Name
298 + break
299 + }
300 + }
301 + // Fallback to first sort option if no default
302 + if defaultSort == "" && len(sortOptions) > 0 {
303 + defaultSort = sortOptions[0].ID
304 + }
305 +
306 + // Decorate columns with chart/label metadata and create ColumnSet
307 + annotatedCols := f.decorateColumns(cols)
308 + cs := f.columnSet(annotatedCols)
309 +
310 + return &funcapi.FunctionResponse{
311 + Status: 200,
312 + Help: "Top SQL queries from performance_schema.events_statements_summary_by_digest",
313 + Columns: cs.BuildColumns(),
314 + Data: data,
315 + DefaultSortColumn: defaultSort,
316 + RequiredParams: []funcapi.ParamConfig{sortParam},
317 + ChartingConfig: cs.BuildCharting(),
318 + }
319 +}
320 +
321 +func (f *funcTopQueries) columnSet(cols []topQueriesColumn) funcapi.ColumnSet[topQueriesColumn] {
322 + return funcapi.Columns(cols, func(c topQueriesColumn) funcapi.ColumnMeta { return c.ColumnMeta })
323 +}
324 +
325 +// checkPerformanceSchema checks if performance_schema is enabled (cached)
326 +func (f *funcTopQueries) checkPerformanceSchema(ctx context.Context) (bool, error) {
327 + // Fast path: return cached result if already checked
328 + f.router.collector.varPerfSchemaMu.RLock()
329 + cached := f.router.collector.varPerformanceSchema
330 + f.router.collector.varPerfSchemaMu.RUnlock()
331 + if cached != "" {
332 + return cached == "ON" || cached == "1", nil
333 + }
334 +
335 + // Slow path: query and cache the result
336 + // Use write lock for the entire operation to prevent duplicate queries
337 + f.router.collector.varPerfSchemaMu.Lock()
338 + defer f.router.collector.varPerfSchemaMu.Unlock()
339 +
340 + // Double-check after acquiring write lock (another goroutine may have set it)
341 + if f.router.collector.varPerformanceSchema != "" {
342 + return f.router.collector.varPerformanceSchema == "ON" || f.router.collector.varPerformanceSchema == "1", nil
343 + }
344 +
345 + var value string
346 + query := "SELECT @@performance_schema"
347 + err := f.router.collector.db.QueryRowContext(ctx, query).Scan(&value)
348 + if err != nil {
349 + return false, err
350 + }
351 +
352 + // Cache the result
353 + f.router.collector.varPerformanceSchema = value
354 + return value == "ON" || value == "1", nil
355 +}
356 +
357 +// detectStatementsColumns queries the database to discover available columns
358 +func (f *funcTopQueries) detectStatementsColumns(ctx context.Context) (map[string]bool, error) {
359 + // Fast path: return cached result
360 + f.router.collector.stmtSummaryColsMu.RLock()
361 + if f.router.collector.stmtSummaryCols != nil {
362 + cols := f.router.collector.stmtSummaryCols
363 + f.router.collector.stmtSummaryColsMu.RUnlock()
364 + return cols, nil
365 + }
366 + f.router.collector.stmtSummaryColsMu.RUnlock()
367 +
368 + // Slow path: query and cache
369 + f.router.collector.stmtSummaryColsMu.Lock()
370 + defer f.router.collector.stmtSummaryColsMu.Unlock()
371 +
372 + // Double-check after acquiring write lock
373 + if f.router.collector.stmtSummaryCols != nil {
374 + return f.router.collector.stmtSummaryCols, nil
375 + }
376 +
377 + // Query information_schema to get available columns
378 + query := `
379 + SELECT COLUMN_NAME
380 + FROM information_schema.COLUMNS
381 + WHERE TABLE_SCHEMA = 'performance_schema'
382 + AND TABLE_NAME = 'events_statements_summary_by_digest'
383 + `
384 + rows, err := f.router.collector.db.QueryContext(ctx, query)
385 + if err != nil {
386 + return nil, fmt.Errorf("failed to query column information: %w", err)
387 + }
388 + defer rows.Close()
389 +
390 + cols := make(map[string]bool)
391 + for rows.Next() {
392 + var colName string
393 + if err := rows.Scan(&colName); err != nil {
394 + return nil, fmt.Errorf("failed to scan column name: %w", err)
395 + }
396 + cols[colName] = true
397 + }
398 +
399 + if err := rows.Err(); err != nil {
400 + return nil, fmt.Errorf("error iterating columns: %w", err)
401 + }
402 +
403 + // Cache the result
404 + f.router.collector.stmtSummaryCols = cols
405 +
406 + return cols, nil
407 +}
408 +
409 +// buildAvailableColumns filters columns based on what's available in the database
410 +func (f *funcTopQueries) buildAvailableColumns(availableCols map[string]bool) []topQueriesColumn {
411 + var cols []topQueriesColumn
412 + for _, col := range topQueriesColumns {
413 + if availableCols[col.DBColumn] {
414 + cols = append(cols, col)
415 + }
416 + }
417 + return cols
418 +}
419 +
420 +// mapAndValidateSortColumn validates sort key and returns the ID to use
421 +func (f *funcTopQueries) mapAndValidateSortColumn(sortKey string, availableCols map[string]bool) string {
422 + // Find the column by ID or DBColumn
423 + for _, col := range topQueriesColumns {
424 + if (col.Name == sortKey || col.DBColumn == sortKey) && availableCols[col.DBColumn] {
425 + return col.Name
426 + }
427 + }
428 + // Default to totalTime if available
429 + if availableCols["SUM_TIMER_WAIT"] {
430 + return "totalTime"
431 + }
432 + return "calls" // Ultimate fallback
433 +}
434 +
435 +// buildDynamicSQL builds the SQL query with only available columns
436 +func (f *funcTopQueries) buildDynamicSQL(cols []topQueriesColumn, sortColumn string, limit int) string {
437 + var selectParts []string
438 + for _, col := range cols {
439 + // Use backticks to handle reserved keywords
440 + if col.IsPicoseconds {
441 + // Convert picoseconds to milliseconds (divide by 10^9)
442 + selectParts = append(selectParts, fmt.Sprintf("%s/1000000000 AS `%s`", col.DBColumn, col.Name))
443 + } else if col.DBColumn == "SCHEMA_NAME" {
444 + selectParts = append(selectParts, fmt.Sprintf("IFNULL(%s, '') AS `%s`", col.DBColumn, col.Name))
445 + } else {
446 + selectParts = append(selectParts, fmt.Sprintf("%s AS `%s`", col.DBColumn, col.Name))
447 + }
448 + }
449 +
450 + return fmt.Sprintf(`
451 +SELECT %s
452 +FROM performance_schema.events_statements_summary_by_digest
453 +WHERE DIGEST IS NOT NULL
454 +ORDER BY `+"`%s`"+` DESC
455 +LIMIT %d
456 +`, strings.Join(selectParts, ", "), sortColumn, limit)
457 +}
458 +
459 +// scanDynamicRows scans rows dynamically based on column types
460 +func (f *funcTopQueries) scanDynamicRows(rows topQueriesRowScanner, cols []topQueriesColumn) ([][]any, error) {
461 + data := make([][]any, 0, 500)
462 +
463 + // Create value holders for scanning
464 + valuePtrs := make([]any, len(cols))
465 + values := make([]any, len(cols))
466 +
467 + for rows.Next() {
468 + // Reset value holders for each row
469 + for i, col := range cols {
470 + switch col.Type {
471 + case funcapi.FieldTypeString:
472 + var v sql.NullString
473 + values[i] = &v
474 + case funcapi.FieldTypeInteger:
475 + var v sql.NullInt64
476 + values[i] = &v
477 + case funcapi.FieldTypeDuration:
478 + var v sql.NullFloat64
479 + values[i] = &v
480 + default:
481 + var v any
482 + values[i] = &v
483 + }
484 + valuePtrs[i] = values[i]
485 + }
486 +
487 + if err := rows.Scan(valuePtrs...); err != nil {
488 + return nil, fmt.Errorf("row scan failed: %w", err)
489 + }
490 +
491 + // Convert scanned values to output format
492 + row := make([]any, len(cols))
493 + for i, col := range cols {
494 + switch v := values[i].(type) {
495 + case *sql.NullString:
496 + if v.Valid {
497 + s := v.String
498 + // Truncate query text
499 + if col.Name == "query" || col.Name == "sampleQuery" {
500 + s = strmutil.TruncateText(s, topQueriesMaxTextLength)
501 + }
502 + row[i] = s
503 + } else {
504 + row[i] = ""
505 + }
506 + case *sql.NullInt64:
507 + if v.Valid {
508 + row[i] = v.Int64
509 + } else {
510 + row[i] = int64(0)
511 + }
512 + case *sql.NullFloat64:
513 + if v.Valid {
514 + row[i] = v.Float64
515 + } else {
516 + row[i] = float64(0)
517 + }
518 + default:
519 + row[i] = nil
520 + }
521 + }
522 + data = append(data, row)
523 + }
524 +
525 + if err := rows.Err(); err != nil {
526 + return nil, fmt.Errorf("rows iteration error: %w", err)
527 + }
528 +
529 + return data, nil
530 +}
531 +
532 +func (f *funcTopQueries) buildSortParam(cols []topQueriesColumn) funcapi.ParamConfig {
533 + return funcapi.BuildSortParam(cols)
534 +}
535 +
536 +func (f *funcTopQueries) decorateColumns(cols []topQueriesColumn) []topQueriesColumn {
537 + out := make([]topQueriesColumn, len(cols))
538 + index := make(map[string]int, len(cols))
539 + for i, col := range cols {
540 + out[i] = col
541 + index[col.Name] = i
542 + }
543 +
544 + for i := range out {
545 + if topQueriesLabelColumns[out[i].Name] {
546 + out[i].GroupBy = &funcapi.GroupByOptions{
547 + IsDefault: out[i].Name == topQueriesPrimaryLabel,
548 + }
549 + }
550 + }
551 +
552 + for _, group := range topQueriesChartGroups {
553 + for _, key := range group.columns {
554 + idx, ok := index[key]
555 + if !ok {
556 + continue
557 + }
558 + out[idx].Chart = &funcapi.ChartOptions{
559 + Group: group.key,
560 + Title: group.title,
561 + IsDefault: group.defaultChart,
562 + }
563 + }
564 + }
565 +
566 + return out
567 +}
src/go/plugin/go.d/collector/mysql/func_top_queries_test.go new
+40
@@ -0,0 +1,40 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package mysql
4 +
5 +import (
6 + "testing"
7 +
8 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
9 + "github.com/stretchr/testify/assert"
10 +)
11 +
12 +func TestMySQLMethods(t *testing.T) {
13 + methods := mysqlMethods()
14 +
15 + require := assert.New(t)
16 + require.Len(methods, 1)
17 + require.Equal("top-queries", methods[0].ID)
18 + require.Equal("Top Queries", methods[0].Name)
19 + require.NotEmpty(methods[0].RequiredParams)
20 +
21 + var sortParam *funcapi.ParamConfig
22 + for i := range methods[0].RequiredParams {
23 + if methods[0].RequiredParams[i].ID == "__sort" {
24 + sortParam = &methods[0].RequiredParams[i]
25 + break
26 + }
27 + }
28 + require.NotNil(sortParam, "expected __sort required param")
29 + require.NotEmpty(sortParam.Options)
30 +}
31 +
32 +func TestTopQueriesColumns_HasRequiredColumns(t *testing.T) {
33 + required := []string{"digest", "query", "totalTime", "calls"}
34 +
35 + f := &funcTopQueries{}
36 + cs := f.columnSet(topQueriesColumns)
37 + for _, id := range required {
38 + assert.True(t, cs.ContainsColumn(id), "column %s should be defined", id)
39 + }
40 +}
src/go/plugin/go.d/collector/mysql/functions.go deleted
-770
@@ -1,770 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -package mysql
4 -
5 -import (
6 - "context"
7 - "database/sql"
8 - "fmt"
9 - "strings"
10 -
11 - "github.com/netdata/netdata/go/plugins/pkg/funcapi"
12 - "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
13 - "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
14 -)
15 -
16 -const maxQueryTextLength = 4096
17 -
18 -const (
19 - paramSort = "__sort"
20 -
21 - ftString = funcapi.FieldTypeString
22 - ftInteger = funcapi.FieldTypeInteger
23 - ftDuration = funcapi.FieldTypeDuration
24 -
25 - trNone = funcapi.FieldTransformNone
26 - trNumber = funcapi.FieldTransformNumber
27 - trDuration = funcapi.FieldTransformDuration
28 -
29 - sortAsc = funcapi.FieldSortAscending
30 - sortDesc = funcapi.FieldSortDescending
31 -
32 - summaryCount = funcapi.FieldSummaryCount
33 - summarySum = funcapi.FieldSummarySum
34 - summaryMin = funcapi.FieldSummaryMin
35 - summaryMax = funcapi.FieldSummaryMax
36 - summaryMean = funcapi.FieldSummaryMean
37 -
38 - filterMulti = funcapi.FieldFilterMultiselect
39 - filterRange = funcapi.FieldFilterRange
40 -)
41 -
42 -// mysqlColumnMeta defines metadata for a single column
43 -type mysqlColumnMeta struct {
44 - dbColumn string // Column name in database (e.g., "SUM_TIMER_WAIT")
45 - uiKey string // Canonical name used everywhere: SQL alias, UI key, sort key
46 - displayName string // Display name in UI (e.g., "Total Time")
47 - dataType funcapi.FieldType // "string", "integer", "float", "duration"
48 - units string // Unit for duration/numeric types (e.g., "seconds")
49 - visible bool // Default visibility
50 - transform funcapi.FieldTransform // Transform for value_options (e.g., "duration", "number", "none")
51 - decimalPoints int // Decimal points for display
52 - sortDir funcapi.FieldSort // Sort direction: "ascending" or "descending"
53 - summary funcapi.FieldSummary // Summary function: "sum", "count", "max", "min", "mean" (UI aggregations)
54 - filter funcapi.FieldFilter // Filter type: "multiselect" or "range"
55 - isPicoseconds bool // Needs picoseconds to seconds conversion
56 - isSortOption bool // Show in sort dropdown
57 - sortLabel string // Label for sort option
58 - isDefaultSort bool // Is this the default sort option
59 - isUniqueKey bool // Is this column a unique key
60 - isSticky bool // Is this column sticky in UI
61 - fullWidth bool // Should column take full width
62 - isLabel bool // Is this column a label for grouping
63 - isPrimary bool // Is this label the primary grouping
64 - isMetric bool // Is this column a chartable metric
65 - chartGroup string // Chart group key
66 - chartTitle string // Chart title
67 - isDefaultChart bool // Include this chart group in defaults
68 -}
69 -
70 -// mysqlAllColumns defines ALL possible columns from events_statements_summary_by_digest
71 -// Columns that don't exist in certain MySQL/MariaDB versions will be filtered at runtime
72 -var mysqlAllColumns = []mysqlColumnMeta{
73 - // Identity columns - always available
74 - {dbColumn: "DIGEST", uiKey: "digest", displayName: "Digest", dataType: ftString, visible: false, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti, isUniqueKey: true},
75 - {dbColumn: "DIGEST_TEXT", uiKey: "query", displayName: "Query", dataType: ftString, visible: true, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti, isSticky: true, fullWidth: true},
76 - {dbColumn: "SCHEMA_NAME", uiKey: "schema", displayName: "Schema", dataType: ftString, visible: true, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti},
77 -
78 - // Execution counts
79 - {dbColumn: "COUNT_STAR", uiKey: "calls", displayName: "Calls", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Top queries by Number of Calls"},
80 -
81 - // Timer metrics (picoseconds -> seconds)
82 - {dbColumn: "SUM_TIMER_WAIT", uiKey: "totalTime", displayName: "Total Time", dataType: ftDuration, units: "milliseconds", visible: true, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summarySum, filter: filterRange, isPicoseconds: true, isSortOption: true, sortLabel: "Top queries by Total Execution Time", isDefaultSort: true},
83 - {dbColumn: "MIN_TIMER_WAIT", uiKey: "minTime", displayName: "Min Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMin, filter: filterRange, isPicoseconds: true},
84 - {dbColumn: "AVG_TIMER_WAIT", uiKey: "avgTime", displayName: "Avg Time", dataType: ftDuration, units: "milliseconds", visible: true, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMean, filter: filterRange, isPicoseconds: true, isSortOption: true, sortLabel: "Top queries by Average Execution Time"},
85 - {dbColumn: "MAX_TIMER_WAIT", uiKey: "maxTime", displayName: "Max Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, filter: filterRange, isPicoseconds: true},
86 -
87 - // Lock time (picoseconds -> seconds)
88 - {dbColumn: "SUM_LOCK_TIME", uiKey: "lockTime", displayName: "Lock Time", dataType: ftDuration, units: "milliseconds", visible: true, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summarySum, filter: filterRange, isPicoseconds: true, isSortOption: true, sortLabel: "Top queries by Lock Time"},
89 -
90 - // Error and warning counts
91 - {dbColumn: "SUM_ERRORS", uiKey: "errors", displayName: "Errors", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Top queries by Errors"},
92 - {dbColumn: "SUM_WARNINGS", uiKey: "warnings", displayName: "Warnings", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Top queries by Warnings"},
93 -
94 - // Row operations
95 - {dbColumn: "SUM_ROWS_AFFECTED", uiKey: "rowsAffected", displayName: "Rows Affected", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Top queries by Rows Affected"},
96 - {dbColumn: "SUM_ROWS_SENT", uiKey: "rowsSent", displayName: "Rows Sent", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Top queries by Rows Sent"},
97 - {dbColumn: "SUM_ROWS_EXAMINED", uiKey: "rowsExamined", displayName: "Rows Examined", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Top queries by Rows Examined"},
98 -
99 - // Temp table usage
100 - {dbColumn: "SUM_CREATED_TMP_DISK_TABLES", uiKey: "tmpDiskTables", displayName: "Temp Disk Tables", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Top queries by Temp Disk Tables"},
101 - {dbColumn: "SUM_CREATED_TMP_TABLES", uiKey: "tmpTables", displayName: "Temp Tables", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Top queries by Temp Tables"},
102 -
103 - // Join operations
104 - {dbColumn: "SUM_SELECT_FULL_JOIN", uiKey: "fullJoin", displayName: "Full Joins", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Top queries by Full Joins"},
105 - {dbColumn: "SUM_SELECT_FULL_RANGE_JOIN", uiKey: "fullRangeJoin", displayName: "Full Range Joins", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange},
106 - {dbColumn: "SUM_SELECT_RANGE", uiKey: "selectRange", displayName: "Select Range", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange},
107 - {dbColumn: "SUM_SELECT_RANGE_CHECK", uiKey: "selectRangeCheck", displayName: "Select Range Check", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange},
108 - {dbColumn: "SUM_SELECT_SCAN", uiKey: "selectScan", displayName: "Select Scan", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Top queries by Table Scans"},
109 -
110 - // Sort operations
111 - {dbColumn: "SUM_SORT_MERGE_PASSES", uiKey: "sortMergePasses", displayName: "Sort Merge Passes", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange},
112 - {dbColumn: "SUM_SORT_RANGE", uiKey: "sortRange", displayName: "Sort Range", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange},
113 - {dbColumn: "SUM_SORT_ROWS", uiKey: "sortRows", displayName: "Sort Rows", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Top queries by Rows Sorted"},
114 - {dbColumn: "SUM_SORT_SCAN", uiKey: "sortScan", displayName: "Sort Scan", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange},
115 -
116 - // Index usage
117 - {dbColumn: "SUM_NO_INDEX_USED", uiKey: "noIndexUsed", displayName: "No Index Used", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Top queries by No Index Used"},
118 - {dbColumn: "SUM_NO_GOOD_INDEX_USED", uiKey: "noGoodIndexUsed", displayName: "No Good Index Used", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange},
119 -
120 - // Timestamp columns
121 - {dbColumn: "FIRST_SEEN", uiKey: "firstSeen", displayName: "First Seen", dataType: ftString, visible: false, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti},
122 - {dbColumn: "LAST_SEEN", uiKey: "lastSeen", displayName: "Last Seen", dataType: ftString, visible: false, transform: trNone, sortDir: sortDesc, summary: summaryCount, filter: filterMulti},
123 -
124 - // MySQL 8.0+ quantile columns
125 - {dbColumn: "QUANTILE_95", uiKey: "p95Time", displayName: "P95 Time", dataType: ftDuration, units: "milliseconds", visible: true, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, filter: filterRange, isPicoseconds: true, isSortOption: true, sortLabel: "Top queries by 95th Percentile Time"},
126 - {dbColumn: "QUANTILE_99", uiKey: "p99Time", displayName: "P99 Time", dataType: ftDuration, units: "milliseconds", visible: true, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, filter: filterRange, isPicoseconds: true, isSortOption: true, sortLabel: "Top queries by 99th Percentile Time"},
127 - {dbColumn: "QUANTILE_999", uiKey: "p999Time", displayName: "P99.9 Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, filter: filterRange, isPicoseconds: true},
128 -
129 - // MySQL 8.0+ sample query
130 - {dbColumn: "QUERY_SAMPLE_TEXT", uiKey: "sampleQuery", displayName: "Sample Query", dataType: ftString, visible: false, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti, fullWidth: true},
131 - {dbColumn: "QUERY_SAMPLE_SEEN", uiKey: "sampleSeen", displayName: "Sample Seen", dataType: ftString, visible: false, transform: trNone, sortDir: sortDesc, summary: summaryCount, filter: filterMulti},
132 - {dbColumn: "QUERY_SAMPLE_TIMER_WAIT", uiKey: "sampleTime", displayName: "Sample Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, filter: filterRange, isPicoseconds: true},
133 -
134 - // MySQL 8.0.28+ CPU time
135 - {dbColumn: "SUM_CPU_TIME", uiKey: "cpuTime", displayName: "CPU Time", dataType: ftDuration, units: "milliseconds", visible: true, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summarySum, filter: filterRange, isPicoseconds: true, isSortOption: true, sortLabel: "Top queries by CPU Time"},
136 -
137 - // MySQL 8.0.31+ memory columns
138 - {dbColumn: "MAX_CONTROLLED_MEMORY", uiKey: "maxControlledMemory", displayName: "Max Controlled Memory", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange, isSortOption: true, sortLabel: "Top queries by Max Controlled Memory"},
139 - {dbColumn: "MAX_TOTAL_MEMORY", uiKey: "maxTotalMemory", displayName: "Max Total Memory", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summaryMax, filter: filterRange, isSortOption: true, sortLabel: "Top queries by Max Total Memory"},
140 -}
141 -
142 -type mysqlChartGroup struct {
143 - key string
144 - title string
145 - columns []string
146 - defaultChart bool
147 -}
148 -
149 -var mysqlChartGroups = []mysqlChartGroup{
150 - {key: "Calls", title: "Number of Calls", columns: []string{"calls"}, defaultChart: true},
151 - {key: "Time", title: "Execution Time", columns: []string{"totalTime", "avgTime", "minTime", "maxTime"}, defaultChart: true},
152 - {key: "Percentiles", title: "Execution Time Percentiles", columns: []string{"p95Time", "p99Time", "p999Time"}},
153 - {key: "LockTime", title: "Lock Time", columns: []string{"lockTime"}},
154 - {key: "Errors", title: "Errors & Warnings", columns: []string{"errors", "warnings"}},
155 - {key: "Rows", title: "Rows", columns: []string{"rowsSent", "rowsExamined", "rowsAffected"}},
156 - {key: "TempTables", title: "Temp Tables", columns: []string{"tmpDiskTables", "tmpTables"}},
157 - {key: "Joins", title: "Join Operations", columns: []string{"fullJoin", "fullRangeJoin", "selectRange", "selectRangeCheck", "selectScan"}},
158 - {key: "Sort", title: "Sort Operations", columns: []string{"sortMergePasses", "sortRange", "sortRows", "sortScan"}},
159 - {key: "Index", title: "Index Usage", columns: []string{"noIndexUsed", "noGoodIndexUsed"}},
160 - {key: "CPU", title: "CPU Time", columns: []string{"cpuTime"}},
161 - {key: "Memory", title: "Memory", columns: []string{"maxControlledMemory", "maxTotalMemory"}},
162 - {key: "Sample", title: "Sample Time", columns: []string{"sampleTime"}},
163 -}
164 -
165 -var mysqlLabelColumns = map[string]bool{
166 - "schema": true,
167 -}
168 -
169 -const mysqlPrimaryLabel = "schema"
170 -
171 -// mysqlMethods returns the available function methods for MySQL
172 -func mysqlMethods() []module.MethodConfig {
173 - // Build sort options from column metadata
174 - var sortOptions []funcapi.ParamOption
175 - sortDir := funcapi.FieldSortDescending
176 - for _, col := range mysqlAllColumns {
177 - if col.isSortOption {
178 - sortOptions = append(sortOptions, funcapi.ParamOption{
179 - ID: col.uiKey,
180 - Column: col.dbColumn,
181 - Name: col.sortLabel,
182 - Default: col.isDefaultSort,
183 - Sort: &sortDir,
184 - })
185 - }
186 - }
187 -
188 - return []module.MethodConfig{
189 - {
190 - UpdateEvery: 10,
191 - ID: "top-queries",
192 - Name: "Top Queries",
193 - Help: "Top SQL queries from performance_schema",
194 - RequireCloud: true,
195 - RequiredParams: []funcapi.ParamConfig{
196 - {
197 - ID: paramSort,
198 - Name: "Filter By",
199 - Help: "Select the primary sort column",
200 - Selection: funcapi.ParamSelect,
201 - Options: sortOptions,
202 - UniqueView: true,
203 - },
204 - },
205 - },
206 - }
207 -}
208 -
209 -func mysqlMethodParams(ctx context.Context, job *module.Job, method string) ([]funcapi.ParamConfig, error) {
210 - collector, ok := job.Module().(*Collector)
211 - if !ok {
212 - return nil, fmt.Errorf("invalid module type")
213 - }
214 - if collector.db == nil {
215 - return nil, fmt.Errorf("collector is still initializing")
216 - }
217 - switch method {
218 - case "top-queries":
219 - return collector.topQueriesParams(ctx)
220 - default:
221 - return nil, fmt.Errorf("unknown method: %s", method)
222 - }
223 -}
224 -
225 -// mysqlHandleMethod handles function requests for MySQL
226 -func mysqlHandleMethod(ctx context.Context, job *module.Job, method string, params funcapi.ResolvedParams) *module.FunctionResponse {
227 - collector, ok := job.Module().(*Collector)
228 - if !ok {
229 - return &module.FunctionResponse{Status: 500, Message: "internal error: invalid module type"}
230 - }
231 -
232 - // Check if collector is initialized (first collect() may not have run yet)
233 - if collector.db == nil {
234 - return &module.FunctionResponse{
235 - Status: 503,
236 - Message: "collector is still initializing, please retry in a few seconds",
237 - }
238 - }
239 -
240 - switch method {
241 - case "top-queries":
242 - return collector.collectTopQueries(ctx, params.Column(paramSort))
243 - default:
244 - return &module.FunctionResponse{Status: 404, Message: fmt.Sprintf("unknown method: %s", method)}
245 - }
246 -}
247 -
248 -// detectMySQLStatementsColumns queries the database to discover available columns
249 -func (c *Collector) detectMySQLStatementsColumns(ctx context.Context) (map[string]bool, error) {
250 - // Fast path: return cached result
251 - c.stmtSummaryColsMu.RLock()
252 - if c.stmtSummaryCols != nil {
253 - cols := c.stmtSummaryCols
254 - c.stmtSummaryColsMu.RUnlock()
255 - return cols, nil
256 - }
257 - c.stmtSummaryColsMu.RUnlock()
258 -
259 - // Slow path: query and cache
260 - c.stmtSummaryColsMu.Lock()
261 - defer c.stmtSummaryColsMu.Unlock()
262 -
263 - // Double-check after acquiring write lock
264 - if c.stmtSummaryCols != nil {
265 - return c.stmtSummaryCols, nil
266 - }
267 -
268 - // Query information_schema to get available columns
269 - query := `
270 - SELECT COLUMN_NAME
271 - FROM information_schema.COLUMNS
272 - WHERE TABLE_SCHEMA = 'performance_schema'
273 - AND TABLE_NAME = 'events_statements_summary_by_digest'
274 - `
275 - rows, err := c.db.QueryContext(ctx, query)
276 - if err != nil {
277 - return nil, fmt.Errorf("failed to query column information: %w", err)
278 - }
279 - defer rows.Close()
280 -
281 - cols := make(map[string]bool)
282 - for rows.Next() {
283 - var colName string
284 - if err := rows.Scan(&colName); err != nil {
285 - return nil, fmt.Errorf("failed to scan column name: %w", err)
286 - }
287 - cols[colName] = true
288 - }
289 -
290 - if err := rows.Err(); err != nil {
291 - return nil, fmt.Errorf("error iterating columns: %w", err)
292 - }
293 -
294 - // Cache the result
295 - c.stmtSummaryCols = cols
296 -
297 - return cols, nil
298 -}
299 -
300 -// buildAvailableColumns filters columns based on what's available in the database
301 -func (c *Collector) buildAvailableMySQLColumns(availableCols map[string]bool) []mysqlColumnMeta {
302 - var cols []mysqlColumnMeta
303 - for _, col := range mysqlAllColumns {
304 - if availableCols[col.dbColumn] {
305 - cols = append(cols, col)
306 - }
307 - }
308 - return cols
309 -}
310 -
311 -// mapAndValidateMySQLSortColumn validates sort key and returns the uiKey to use
312 -func (c *Collector) mapAndValidateMySQLSortColumn(sortKey string, availableCols map[string]bool) string {
313 - // Find the column by uiKey or dbColumn
314 - for _, col := range mysqlAllColumns {
315 - if (col.uiKey == sortKey || col.dbColumn == sortKey) && availableCols[col.dbColumn] {
316 - return col.uiKey
317 - }
318 - }
319 - // Default to totalTime if available
320 - if availableCols["SUM_TIMER_WAIT"] {
321 - return "totalTime"
322 - }
323 - return "calls" // Ultimate fallback
324 -}
325 -
326 -// buildMySQLDynamicSQL builds the SQL query with only available columns
327 -func (c *Collector) buildMySQLDynamicSQL(cols []mysqlColumnMeta, sortColumn string, limit int) string {
328 - var selectParts []string
329 - for _, col := range cols {
330 - // Use backticks to handle reserved keywords
331 - if col.isPicoseconds {
332 - // Convert picoseconds to milliseconds (divide by 10^9)
333 - selectParts = append(selectParts, fmt.Sprintf("%s/1000000000 AS `%s`", col.dbColumn, col.uiKey))
334 - } else if col.dbColumn == "SCHEMA_NAME" {
335 - selectParts = append(selectParts, fmt.Sprintf("IFNULL(%s, '') AS `%s`", col.dbColumn, col.uiKey))
336 - } else {
337 - selectParts = append(selectParts, fmt.Sprintf("%s AS `%s`", col.dbColumn, col.uiKey))
338 - }
339 - }
340 -
341 - return fmt.Sprintf(`
342 -SELECT %s
343 -FROM performance_schema.events_statements_summary_by_digest
344 -WHERE DIGEST IS NOT NULL
345 -ORDER BY `+"`%s`"+` DESC
346 -LIMIT %d
347 -`, strings.Join(selectParts, ", "), sortColumn, limit)
348 -}
349 -
350 -// scanMySQLDynamicRows scans rows dynamically based on column types
351 -func (c *Collector) scanMySQLDynamicRows(rows mysqlRowScanner, cols []mysqlColumnMeta) ([][]any, error) {
352 - data := make([][]any, 0, 500)
353 -
354 - // Create value holders for scanning
355 - valuePtrs := make([]any, len(cols))
356 - values := make([]any, len(cols))
357 -
358 - for rows.Next() {
359 - // Reset value holders for each row
360 - for i, col := range cols {
361 - switch col.dataType {
362 - case ftString:
363 - var v sql.NullString
364 - values[i] = &v
365 - case ftInteger:
366 - var v sql.NullInt64
367 - values[i] = &v
368 - case ftDuration:
369 - var v sql.NullFloat64
370 - values[i] = &v
371 - default:
372 - var v any
373 - values[i] = &v
374 - }
375 - valuePtrs[i] = values[i]
376 - }
377 -
378 - if err := rows.Scan(valuePtrs...); err != nil {
379 - return nil, fmt.Errorf("row scan failed: %w", err)
380 - }
381 -
382 - // Convert scanned values to output format
383 - row := make([]any, len(cols))
384 - for i, col := range cols {
385 - switch v := values[i].(type) {
386 - case *sql.NullString:
387 - if v.Valid {
388 - s := v.String
389 - // Truncate query text
390 - if col.uiKey == "query" || col.uiKey == "sampleQuery" {
391 - s = strmutil.TruncateText(s, maxQueryTextLength)
392 - }
393 - row[i] = s
394 - } else {
395 - row[i] = ""
396 - }
397 - case *sql.NullInt64:
398 - if v.Valid {
399 - row[i] = v.Int64
400 - } else {
401 - row[i] = int64(0)
402 - }
403 - case *sql.NullFloat64:
404 - if v.Valid {
405 - row[i] = v.Float64
406 - } else {
407 - row[i] = float64(0)
408 - }
409 - default:
410 - row[i] = nil
411 - }
412 - }
413 - data = append(data, row)
414 - }
415 -
416 - if err := rows.Err(); err != nil {
417 - return nil, fmt.Errorf("rows iteration error: %w", err)
418 - }
419 -
420 - return data, nil
421 -}
422 -
423 -// buildMySQLDynamicColumns builds column definitions for the response
424 -func (c *Collector) buildMySQLDynamicColumns(cols []mysqlColumnMeta) map[string]any {
425 - columns := make(map[string]any)
426 - for i, col := range cols {
427 - visual := funcapi.FieldVisualValue
428 - if col.dataType == ftDuration {
429 - visual = funcapi.FieldVisualBar
430 - }
431 - colDef := funcapi.Column{
432 - Index: i,
433 - Name: col.displayName,
434 - Type: col.dataType,
435 - Units: col.units,
436 - Visualization: visual,
437 - Sort: col.sortDir,
438 - Sortable: true,
439 - Sticky: col.isSticky,
440 - Summary: col.summary,
441 - Filter: col.filter,
442 - FullWidth: col.fullWidth,
443 - Wrap: false,
444 - DefaultExpandedFilter: false,
445 - UniqueKey: col.isUniqueKey,
446 - Visible: col.visible,
447 - ValueOptions: funcapi.ValueOptions{
448 - Transform: col.transform,
449 - DecimalPoints: col.decimalPoints,
450 - DefaultValue: nil,
451 - },
452 - }
453 - columns[col.uiKey] = colDef.BuildColumn()
454 - }
455 - return columns
456 -}
457 -
458 -// buildMySQLDynamicSortOptions builds sort options from available columns
459 -// Returns only sort options for columns that actually exist in the database
460 -func (c *Collector) buildMySQLDynamicSortOptions(cols []mysqlColumnMeta) []funcapi.ParamOption {
461 - var sortOpts []funcapi.ParamOption
462 - seen := make(map[string]bool)
463 - sortDir := funcapi.FieldSortDescending
464 -
465 - for _, col := range cols {
466 - if col.isSortOption && !seen[col.uiKey] {
467 - seen[col.uiKey] = true
468 - sortOpts = append(sortOpts, funcapi.ParamOption{
469 - ID: col.uiKey,
470 - Column: col.dbColumn,
471 - Name: col.sortLabel,
472 - Default: col.isDefaultSort,
473 - Sort: &sortDir,
474 - })
475 - }
476 - }
477 - return sortOpts
478 -}
479 -
480 -func (c *Collector) topQueriesSortParam(cols []mysqlColumnMeta) (funcapi.ParamConfig, []funcapi.ParamOption) {
481 - sortOptions := c.buildMySQLDynamicSortOptions(cols)
482 - sortParam := funcapi.ParamConfig{
483 - ID: paramSort,
484 - Name: "Filter By",
485 - Help: "Select the primary sort column",
486 - Selection: funcapi.ParamSelect,
487 - Options: sortOptions,
488 - UniqueView: true,
489 - }
490 - return sortParam, sortOptions
491 -}
492 -
493 -func (c *Collector) topQueriesParams(ctx context.Context) ([]funcapi.ParamConfig, error) {
494 - available, err := c.checkPerformanceSchema(ctx)
495 - if err != nil {
496 - return nil, err
497 - }
498 - if !available {
499 - return nil, fmt.Errorf("performance_schema is not enabled")
500 - }
501 -
502 - availableCols, err := c.detectMySQLStatementsColumns(ctx)
503 - if err != nil {
504 - return nil, err
505 - }
506 - cols := c.buildAvailableMySQLColumns(availableCols)
507 - if len(cols) == 0 {
508 - return nil, fmt.Errorf("no columns available in events_statements_summary_by_digest")
509 - }
510 -
511 - sortParam, _ := c.topQueriesSortParam(cols)
512 - return []funcapi.ParamConfig{sortParam}, nil
513 -}
514 -
515 -// mysqlRowScanner interface for testing
516 -type mysqlRowScanner interface {
517 - Next() bool
518 - Scan(dest ...any) error
519 - Err() error
520 -}
521 -
522 -// collectTopQueries queries performance_schema for top queries using dynamic columns
523 -func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
524 - // Check if performance_schema is enabled
525 - available, err := c.checkPerformanceSchema(ctx)
526 - if err != nil {
527 - return &module.FunctionResponse{
528 - Status: 500,
529 - Message: fmt.Sprintf("failed to check performance_schema availability: %v", err),
530 - }
531 - }
532 - if !available {
533 - return &module.FunctionResponse{
534 - Status: 503,
535 - Message: "performance_schema is not enabled",
536 - }
537 - }
538 -
539 - // Detect available columns
540 - availableCols, err := c.detectMySQLStatementsColumns(ctx)
541 - if err != nil {
542 - return &module.FunctionResponse{
543 - Status: 500,
544 - Message: fmt.Sprintf("failed to detect available columns: %v", err),
545 - }
546 - }
547 -
548 - // Build list of available columns
549 - cols := c.buildAvailableMySQLColumns(availableCols)
550 - if len(cols) == 0 {
551 - return &module.FunctionResponse{
552 - Status: 500,
553 - Message: "no columns available in events_statements_summary_by_digest",
554 - }
555 - }
556 -
557 - // Validate and map sort column
558 - dbSortColumn := c.mapAndValidateMySQLSortColumn(sortColumn, availableCols)
559 -
560 - // Get query limit (default 500)
561 - limit := c.TopQueriesLimit
562 - if limit <= 0 {
563 - limit = 500
564 - }
565 -
566 - // Build and execute query
567 - query := c.buildMySQLDynamicSQL(cols, dbSortColumn, limit)
568 -
569 - rows, err := c.db.QueryContext(ctx, query)
570 - if err != nil {
571 - if ctx.Err() == context.DeadlineExceeded {
572 - return &module.FunctionResponse{Status: 504, Message: "query timed out"}
573 - }
574 - return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("query failed: %v", err)}
575 - }
576 - defer rows.Close()
577 -
578 - // Scan rows dynamically
579 - data, err := c.scanMySQLDynamicRows(rows, cols)
580 - if err != nil {
581 - return &module.FunctionResponse{Status: 500, Message: err.Error()}
582 - }
583 -
584 - // Build dynamic sort options from available columns (only those actually detected)
585 - sortParam, sortOptions := c.topQueriesSortParam(cols)
586 -
587 - // Find default sort column UI key
588 - defaultSort := ""
589 - for _, col := range cols {
590 - if col.isDefaultSort && col.isSortOption {
591 - defaultSort = col.uiKey
592 - break
593 - }
594 - }
595 - // Fallback to first sort option if no default
596 - if defaultSort == "" && len(sortOptions) > 0 {
597 - defaultSort = sortOptions[0].ID
598 - }
599 -
600 - annotatedCols := decorateMySQLColumns(cols)
601 -
602 - return &module.FunctionResponse{
603 - Status: 200,
604 - Help: "Top SQL queries from performance_schema.events_statements_summary_by_digest",
605 - Columns: c.buildMySQLDynamicColumns(cols),
606 - Data: data,
607 - DefaultSortColumn: defaultSort,
608 - RequiredParams: []funcapi.ParamConfig{sortParam},
609 -
610 - // Charts for aggregated visualization
611 - Charts: mysqlTopQueriesCharts(annotatedCols),
612 - DefaultCharts: mysqlTopQueriesDefaultCharts(annotatedCols),
613 - GroupBy: mysqlTopQueriesGroupBy(annotatedCols),
614 - }
615 -}
616 -
617 -func decorateMySQLColumns(cols []mysqlColumnMeta) []mysqlColumnMeta {
618 - out := make([]mysqlColumnMeta, len(cols))
619 - index := make(map[string]int, len(cols))
620 - for i, col := range cols {
621 - out[i] = col
622 - index[col.uiKey] = i
623 - }
624 -
625 - for i := range out {
626 - if mysqlLabelColumns[out[i].uiKey] {
627 - out[i].isLabel = true
628 - if out[i].uiKey == mysqlPrimaryLabel {
629 - out[i].isPrimary = true
630 - }
631 - }
632 - }
633 -
634 - for _, group := range mysqlChartGroups {
635 - for _, key := range group.columns {
636 - idx, ok := index[key]
637 - if !ok {
638 - continue
639 - }
640 - out[idx].isMetric = true
641 - out[idx].chartGroup = group.key
642 - out[idx].chartTitle = group.title
643 - if group.defaultChart {
644 - out[idx].isDefaultChart = true
645 - }
646 - }
647 - }
648 -
649 - return out
650 -}
651 -
652 -func mysqlTopQueriesCharts(cols []mysqlColumnMeta) map[string]module.ChartConfig {
653 - charts := make(map[string]module.ChartConfig)
654 - for _, col := range cols {
655 - if !col.isMetric || col.chartGroup == "" {
656 - continue
657 - }
658 - cfg, ok := charts[col.chartGroup]
659 - if !ok {
660 - title := col.chartTitle
661 - if title == "" {
662 - title = col.chartGroup
663 - }
664 - cfg = module.ChartConfig{Name: title, Type: "stacked-bar"}
665 - }
666 - cfg.Columns = append(cfg.Columns, col.uiKey)
667 - charts[col.chartGroup] = cfg
668 - }
669 - return charts
670 -}
671 -
672 -func mysqlTopQueriesDefaultCharts(cols []mysqlColumnMeta) [][]string {
673 - label := primaryMySQLLabel(cols)
674 - if label == "" {
675 - return nil
676 - }
677 - chartGroups := defaultMySQLChartGroups(cols)
678 - out := make([][]string, 0, len(chartGroups))
679 - for _, group := range chartGroups {
680 - out = append(out, []string{group, label})
681 - }
682 - return out
683 -}
684 -
685 -func mysqlTopQueriesGroupBy(cols []mysqlColumnMeta) map[string]module.GroupByConfig {
686 - groupBy := make(map[string]module.GroupByConfig)
687 - for _, col := range cols {
688 - if !col.isLabel {
689 - continue
690 - }
691 - groupBy[col.uiKey] = module.GroupByConfig{
692 - Name: "Group by " + col.displayName,
693 - Columns: []string{col.uiKey},
694 - }
695 - }
696 - return groupBy
697 -}
698 -
699 -func primaryMySQLLabel(cols []mysqlColumnMeta) string {
700 - for _, col := range cols {
701 - if col.isPrimary {
702 - return col.uiKey
703 - }
704 - }
705 - for _, col := range cols {
706 - if col.isLabel {
707 - return col.uiKey
708 - }
709 - }
710 - return ""
711 -}
712 -
713 -func defaultMySQLChartGroups(cols []mysqlColumnMeta) []string {
714 - groups := make([]string, 0)
715 - seen := make(map[string]bool)
716 - for _, col := range cols {
717 - if !col.isMetric || col.chartGroup == "" || !col.isDefaultChart {
718 - continue
719 - }
720 - if !seen[col.chartGroup] {
721 - seen[col.chartGroup] = true
722 - groups = append(groups, col.chartGroup)
723 - }
724 - }
725 - if len(groups) > 0 {
726 - return groups
727 - }
728 - for _, col := range cols {
729 - if !col.isMetric || col.chartGroup == "" {
730 - continue
731 - }
732 - if !seen[col.chartGroup] {
733 - seen[col.chartGroup] = true
734 - groups = append(groups, col.chartGroup)
735 - }
736 - }
737 - return groups
738 -}
739 -
740 -// checkPerformanceSchema checks if performance_schema is enabled (cached)
741 -func (c *Collector) checkPerformanceSchema(ctx context.Context) (bool, error) {
742 - // Fast path: return cached result if already checked
743 - c.varPerfSchemaMu.RLock()
744 - cached := c.varPerformanceSchema
745 - c.varPerfSchemaMu.RUnlock()
746 - if cached != "" {
747 - return cached == "ON" || cached == "1", nil
748 - }
749 -
750 - // Slow path: query and cache the result
751 - // Use write lock for the entire operation to prevent duplicate queries
752 - c.varPerfSchemaMu.Lock()
753 - defer c.varPerfSchemaMu.Unlock()
754 -
755 - // Double-check after acquiring write lock (another goroutine may have set it)
756 - if c.varPerformanceSchema != "" {
757 - return c.varPerformanceSchema == "ON" || c.varPerformanceSchema == "1", nil
758 - }
759 -
760 - var value string
761 - query := "SELECT @@performance_schema"
762 - err := c.db.QueryRowContext(ctx, query).Scan(&value)
763 - if err != nil {
764 - return false, err
765 - }
766 -
767 - // Cache the result
768 - c.varPerformanceSchema = value
769 - return value == "ON" || value == "1", nil
770 -}
src/go/plugin/go.d/collector/mysql/functions_test.go deleted
-273
@@ -1,273 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -package mysql
4 -
5 -import (
6 - "testing"
7 -
8 - "github.com/netdata/netdata/go/plugins/pkg/funcapi"
9 - "github.com/stretchr/testify/assert"
10 -)
11 -
12 -func TestMysqlMethods(t *testing.T) {
13 - methods := mysqlMethods()
14 -
15 - require := assert.New(t)
16 - require.Len(methods, 1)
17 - require.Equal("top-queries", methods[0].ID)
18 - require.Equal("Top Queries", methods[0].Name)
19 - require.NotEmpty(methods[0].RequiredParams)
20 -
21 - // Verify at least one default sort option exists
22 - var sortParam *funcapi.ParamConfig
23 - for i := range methods[0].RequiredParams {
24 - if methods[0].RequiredParams[i].ID == "__sort" {
25 - sortParam = &methods[0].RequiredParams[i]
26 - break
27 - }
28 - }
29 - require.NotNil(sortParam, "expected __sort required param")
30 - require.NotEmpty(sortParam.Options)
31 -
32 - hasDefault := false
33 - for _, opt := range sortParam.Options {
34 - if opt.Default {
35 - hasDefault = true
36 - require.Equal("totalTime", opt.ID) // camelCase for UI
37 - break
38 - }
39 - }
40 - require.True(hasDefault, "should have a default sort option")
41 -}
42 -
43 -func TestMysqlAllColumns_HasRequiredColumns(t *testing.T) {
44 - // Verify all required base columns are defined
45 - requiredUIKeys := []string{
46 - "digest", "query", "schema", "calls",
47 - "totalTime", "avgTime", "minTime", "maxTime",
48 - "rowsSent", "rowsExamined", "noIndexUsed",
49 - }
50 -
51 - uiKeys := make(map[string]bool)
52 - for _, col := range mysqlAllColumns {
53 - uiKeys[col.uiKey] = true
54 - }
55 -
56 - for _, key := range requiredUIKeys {
57 - assert.True(t, uiKeys[key], "column %s should be defined in mysqlAllColumns", key)
58 - }
59 -}
60 -
61 -func TestMysqlAllColumns_HasValidMetadata(t *testing.T) {
62 - for _, col := range mysqlAllColumns {
63 - // Every column must have a UI key
64 - assert.NotEmpty(t, col.uiKey, "column %s must have uiKey", col.dbColumn)
65 -
66 - // Every column must have a display name
67 - assert.NotEmpty(t, col.displayName, "column %s must have displayName", col.uiKey)
68 -
69 - // Every column must have a data type
70 - assert.NotEqual(t, funcapi.FieldTypeNone, col.dataType, "column %s must have dataType", col.uiKey)
71 -
72 - // Duration columns must have units
73 - if col.dataType == ftDuration {
74 - assert.NotEmpty(t, col.units, "duration column %s must have units", col.uiKey)
75 - }
76 -
77 - // Sort options must have labels
78 - if col.isSortOption {
79 - assert.NotEmpty(t, col.sortLabel, "sort option column %s must have sortLabel", col.uiKey)
80 - }
81 - }
82 -}
83 -
84 -func TestCollector_mapAndValidateMySQLSortColumn(t *testing.T) {
85 - tests := map[string]struct {
86 - availableCols map[string]bool
87 - input string
88 - expected string
89 - }{
90 - "totalTime maps correctly": {
91 - availableCols: map[string]bool{"SUM_TIMER_WAIT": true, "COUNT_STAR": true},
92 - input: "totalTime",
93 - expected: "totalTime",
94 - },
95 - "calls maps correctly": {
96 - availableCols: map[string]bool{"SUM_TIMER_WAIT": true, "COUNT_STAR": true},
97 - input: "calls",
98 - expected: "calls",
99 - },
100 - "invalid column falls back to totalTime": {
101 - availableCols: map[string]bool{"SUM_TIMER_WAIT": true, "COUNT_STAR": true},
102 - input: "invalid_column",
103 - expected: "totalTime",
104 - },
105 - "SQL injection attempt falls back to totalTime": {
106 - availableCols: map[string]bool{"SUM_TIMER_WAIT": true, "COUNT_STAR": true},
107 - input: "'; DROP TABLE users;--",
108 - expected: "totalTime",
109 - },
110 - "falls back to calls when SUM_TIMER_WAIT unavailable": {
111 - availableCols: map[string]bool{"COUNT_STAR": true},
112 - input: "invalid_column",
113 - expected: "calls",
114 - },
115 - }
116 -
117 - for name, tc := range tests {
118 - t.Run(name, func(t *testing.T) {
119 - c := &Collector{}
120 - result := c.mapAndValidateMySQLSortColumn(tc.input, tc.availableCols)
121 - assert.Equal(t, tc.expected, result)
122 - })
123 - }
124 -}
125 -
126 -func TestCollector_buildAvailableMySQLColumns(t *testing.T) {
127 - tests := map[string]struct {
128 - availableCols map[string]bool
129 - expectCols []string // UI keys we expect to see
130 - notExpectCols []string // UI keys we don't expect
131 - }{
132 - "Basic MySQL 5.7 columns": {
133 - availableCols: map[string]bool{
134 - "DIGEST": true, "DIGEST_TEXT": true, "SCHEMA_NAME": true, "COUNT_STAR": true,
135 - "SUM_TIMER_WAIT": true, "MIN_TIMER_WAIT": true, "AVG_TIMER_WAIT": true, "MAX_TIMER_WAIT": true,
136 - "SUM_ROWS_SENT": true, "SUM_ROWS_EXAMINED": true, "SUM_NO_INDEX_USED": true,
137 - },
138 - expectCols: []string{"digest", "query", "schema", "calls", "totalTime", "rowsSent"},
139 - notExpectCols: []string{"p95Time", "cpuTime", "maxTotalMemory"}, // MySQL 8.0+ only
140 - },
141 - "MySQL 8.0 with quantiles": {
142 - availableCols: map[string]bool{
143 - "DIGEST": true, "DIGEST_TEXT": true, "SCHEMA_NAME": true, "COUNT_STAR": true,
144 - "SUM_TIMER_WAIT": true, "QUANTILE_95": true, "QUANTILE_99": true,
145 - "QUERY_SAMPLE_TEXT": true,
146 - },
147 - expectCols: []string{"digest", "query", "calls", "p95Time", "p99Time", "sampleQuery"},
148 - },
149 - }
150 -
151 - for name, tc := range tests {
152 - t.Run(name, func(t *testing.T) {
153 - c := &Collector{}
154 - cols := c.buildAvailableMySQLColumns(tc.availableCols)
155 -
156 - // Build map of UI keys for easy lookup
157 - uiKeys := make(map[string]bool)
158 - for _, col := range cols {
159 - uiKeys[col.uiKey] = true
160 - }
161 -
162 - for _, key := range tc.expectCols {
163 - assert.True(t, uiKeys[key], "expected column %s to be present", key)
164 - }
165 - for _, key := range tc.notExpectCols {
166 - assert.False(t, uiKeys[key], "did not expect column %s to be present", key)
167 - }
168 - })
169 - }
170 -}
171 -
172 -func TestCollector_buildMySQLDynamicSQL(t *testing.T) {
173 - c := &Collector{}
174 -
175 - cols := []mysqlColumnMeta{
176 - {dbColumn: "DIGEST", uiKey: "digest", dataType: ftString},
177 - {dbColumn: "DIGEST_TEXT", uiKey: "query", dataType: ftString},
178 - {dbColumn: "COUNT_STAR", uiKey: "calls", dataType: ftInteger},
179 - {dbColumn: "SUM_TIMER_WAIT", uiKey: "totalTime", dataType: ftDuration, isPicoseconds: true},
180 - }
181 -
182 - sql := c.buildMySQLDynamicSQL(cols, "totalTime", 500)
183 -
184 - assert.Contains(t, sql, "performance_schema.events_statements_summary_by_digest")
185 - assert.Contains(t, sql, "ORDER BY `totalTime` DESC")
186 - assert.Contains(t, sql, "LIMIT 500")
187 - assert.Contains(t, sql, "AS `digest`")
188 - assert.Contains(t, sql, "AS `calls`")
189 - assert.Contains(t, sql, "AS `totalTime`")
190 - // Picosecond columns should have conversion
191 - assert.Contains(t, sql, "SUM_TIMER_WAIT/1000000000 AS `totalTime`")
192 -}
193 -
194 -func TestCollector_buildMySQLDynamicColumns(t *testing.T) {
195 - c := &Collector{}
196 -
197 - cols := []mysqlColumnMeta{
198 - {uiKey: "digest", displayName: "Digest", dataType: ftString, visible: false, isUniqueKey: true, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti},
199 - {uiKey: "query", displayName: "Query", dataType: ftString, visible: true, isSticky: true, fullWidth: true, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti},
200 - {uiKey: "totalTime", displayName: "Total Time", dataType: ftDuration, units: "seconds", visible: true, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summarySum, filter: filterRange},
201 - }
202 -
203 - columns := c.buildMySQLDynamicColumns(cols)
204 -
205 - // Verify column count
206 - assert.Len(t, columns, 3)
207 -
208 - // Verify digest column
209 - digestCol := columns["digest"].(map[string]any)
210 - assert.Equal(t, "Digest", digestCol["name"])
211 - assert.Equal(t, "string", digestCol["type"])
212 - assert.True(t, digestCol["unique_key"].(bool))
213 - assert.False(t, digestCol["visible"].(bool))
214 - assert.Equal(t, 0, digestCol["index"])
215 -
216 - // Verify query column
217 - queryCol := columns["query"].(map[string]any)
218 - assert.Equal(t, "Query", queryCol["name"])
219 - assert.True(t, queryCol["sticky"].(bool))
220 - assert.True(t, queryCol["full_width"].(bool))
221 - assert.Equal(t, 1, queryCol["index"])
222 -
223 - // Verify totalTime column
224 - totalTimeCol := columns["totalTime"].(map[string]any)
225 - assert.Equal(t, "Total Time", totalTimeCol["name"])
226 - assert.Equal(t, "duration", totalTimeCol["type"])
227 - assert.Equal(t, "seconds", totalTimeCol["units"])
228 - assert.Equal(t, "bar", totalTimeCol["visualization"]) // duration uses bar
229 - assert.Equal(t, 2, totalTimeCol["index"])
230 -}
231 -
232 -// Test that method config sort options have valid column references
233 -func TestMysqlMethods_SortOptionsHaveLabels(t *testing.T) {
234 - methods := mysqlMethods()
235 -
236 - for _, method := range methods {
237 - var sortParam *funcapi.ParamConfig
238 - for i := range method.RequiredParams {
239 - if method.RequiredParams[i].ID == "__sort" {
240 - sortParam = &method.RequiredParams[i]
241 - break
242 - }
243 - }
244 - assert.NotNil(t, sortParam)
245 - for _, opt := range sortParam.Options {
246 - assert.NotEmpty(t, opt.ID, "sort option must have ID")
247 - assert.NotEmpty(t, opt.Name, "sort option %s must have Name", opt.ID)
248 - assert.Contains(t, opt.Name, "Top queries by", "label should have standard prefix")
249 - }
250 - }
251 -}
252 -
253 -// TestSortColumnValidation_SQLInjection verifies that SQL injection attempts
254 -// are handled by the validation mechanism
255 -func TestSortColumnValidation_SQLInjection(t *testing.T) {
256 - c := &Collector{}
257 - availableCols := map[string]bool{"SUM_TIMER_WAIT": true, "COUNT_STAR": true}
258 -
259 - maliciousInputs := []string{
260 - "'; DROP TABLE performance_schema; --",
261 - "COUNT_STAR; DELETE FROM mysql.user",
262 - "1 OR 1=1",
263 - "SLEEP(10)",
264 - "BENCHMARK(10000000,SHA1('test'))",
265 - }
266 -
267 - for _, input := range maliciousInputs {
268 - result := c.mapAndValidateMySQLSortColumn(input, availableCols)
269 - // All malicious inputs should fall back to safe default
270 - assert.True(t, result == "totalTime" || result == "calls",
271 - "malicious input should fall back to safe default: %s -> %s", input, result)
272 - }
273 -}
src/go/plugin/go.d/collector/oracledb/collector.go
+10 -4
@@ -22,9 +22,8 @@ func init() {
22 JobConfigSchema: configSchema,
23 Create: func() module.Module { return New() },
24 Config: func() any { return &Config{} },
25 - Methods: oracleMethods,
26 - MethodParams: oracleMethodParams,
27 - HandleMethod: oracleHandleMethod,
25 + Methods: oracledbMethods,
26 + MethodHandler: oracledbFunctionHandler,
27 })
28 }
29
@@ -60,6 +59,8 @@ type Collector struct {
59 Config `yaml:",inline" json:""`
60
61 db *sql.DB
62 +
63 + funcRouter *funcRouter
64 }
65
66 func (c *Collector) Configuration() any {
@@ -74,6 +75,8 @@ func (c *Collector) Init(context.Context) error {
75
76 c.publicDSN = dsn
77
78 + c.funcRouter = newFuncRouter(c)
79 +
80 return nil
81 }
82
@@ -106,7 +109,10 @@ func (c *Collector) Collect(context.Context) map[string]int64 {
109 return mx
110 }
111
109 -func (c *Collector) Cleanup(context.Context) {
112 +func (c *Collector) Cleanup(ctx context.Context) {
113 + if c.funcRouter != nil {
114 + c.funcRouter.Cleanup(ctx)
115 + }
116 if c.db != nil {
117 if err := c.db.Close(); err != nil {
118 c.Errorf("cleanup: error on closing connection [%s]: %v", c.publicDSN, err)
src/go/plugin/go.d/collector/oracledb/func_queries_test.go new
+63
@@ -0,0 +1,63 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package oracledb
4 +
5 +import (
6 + "testing"
7 +
8 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
9 + "github.com/stretchr/testify/assert"
10 +)
11 +
12 +func TestOracleDBMethods(t *testing.T) {
13 + methods := oracledbMethods()
14 +
15 + require := assert.New(t)
16 + require.Len(methods, 2)
17 +
18 + for _, method := range methods {
19 + require.NotEmpty(method.ID)
20 + require.NotEmpty(method.Name)
21 + require.NotEmpty(method.RequiredParams)
22 +
23 + var sortParam *funcapi.ParamConfig
24 + for i := range method.RequiredParams {
25 + if method.RequiredParams[i].ID == "__sort" {
26 + sortParam = &method.RequiredParams[i]
27 + break
28 + }
29 + }
30 + require.NotNil(sortParam, "expected __sort required param")
31 + require.NotEmpty(sortParam.Options)
32 + }
33 +}
34 +
35 +func TestQueriesTopColumns_HasRequiredColumns(t *testing.T) {
36 + required := []string{"sqlId", "query", "totalTime", "executions"}
37 +
38 + for _, id := range required {
39 + found := false
40 + for _, col := range topQueriesColumns {
41 + if col.Name == id {
42 + found = true
43 + break
44 + }
45 + }
46 + assert.True(t, found, "column %s should be defined", id)
47 + }
48 +}
49 +
50 +func TestQueriesRunningColumns_HasRequiredColumns(t *testing.T) {
51 + required := []string{"sessionId", "query", "lastCallMs"}
52 +
53 + for _, id := range required {
54 + found := false
55 + for _, col := range runningQueriesColumns {
56 + if col.Name == id {
57 + found = true
58 + break
59 + }
60 + }
61 + assert.True(t, found, "column %s should be defined", id)
62 + }
63 +}
src/go/plugin/go.d/collector/oracledb/func_router.go new
+74
@@ -0,0 +1,74 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package oracledb
4 +
5 +import (
6 + "context"
7 + "fmt"
8 +
9 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
10 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
11 +)
12 +
13 +// funcRouter routes method calls to appropriate function handlers.
14 +// Uses shared SQL connection from Collector (OracleDB metrics collection uses SQL).
15 +type funcRouter struct {
16 + collector *Collector // for shared DB access and config
17 +
18 + handlers map[string]funcapi.MethodHandler
19 +}
20 +
21 +func newFuncRouter(c *Collector) *funcRouter {
22 + r := &funcRouter{
23 + collector: c,
24 + handlers: make(map[string]funcapi.MethodHandler),
25 + }
26 + r.handlers[topQueriesMethodID] = newFuncTopQueries(r)
27 + r.handlers[runningQueriesMethodID] = newFuncRunningQueries(r)
28 + return r
29 +}
30 +
31 +// Compile-time interface check.
32 +var _ funcapi.MethodHandler = (*funcRouter)(nil)
33 +
34 +func (r *funcRouter) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
35 + if h, ok := r.handlers[method]; ok {
36 + return h.MethodParams(ctx, method)
37 + }
38 + return nil, fmt.Errorf("unknown method: %s", method)
39 +}
40 +
41 +func (r *funcRouter) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
42 + if h, ok := r.handlers[method]; ok {
43 + return h.Handle(ctx, method, params)
44 + }
45 + return funcapi.NotFoundResponse(method)
46 +}
47 +
48 +func (r *funcRouter) Cleanup(ctx context.Context) {
49 + for _, h := range r.handlers {
50 + h.Cleanup(ctx)
51 + }
52 +}
53 +
54 +func (r *funcRouter) topQueriesLimit() int {
55 + if r.collector.TopQueriesLimit > 0 {
56 + return r.collector.TopQueriesLimit
57 + }
58 + return 500
59 +}
60 +
61 +func oracledbMethods() []funcapi.MethodConfig {
62 + return []funcapi.MethodConfig{
63 + topQueriesMethodConfig(),
64 + runningQueriesMethodConfig(),
65 + }
66 +}
67 +
68 +func oracledbFunctionHandler(job *module.Job) funcapi.MethodHandler {
69 + c, ok := job.Module().(*Collector)
70 + if !ok {
71 + return nil
72 + }
73 + return c.funcRouter
74 +}
src/go/plugin/go.d/collector/oracledb/func_running_queries.go new
+246
@@ -0,0 +1,246 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package oracledb
4 +
5 +import (
6 + "context"
7 + "database/sql"
8 + "fmt"
9 + "strings"
10 +
11 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
12 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
13 +)
14 +
15 +const (
16 + runningQueriesMethodID = "running-queries"
17 + runningQueriesMaxTextLength = 4096
18 +)
19 +
20 +func runningQueriesMethodConfig() funcapi.MethodConfig {
21 + return funcapi.MethodConfig{
22 + ID: runningQueriesMethodID,
23 + Name: "Running Queries",
24 + UpdateEvery: 10,
25 + Help: "Currently running SQL statements from V$SESSION. WARNING: Query text may contain unmasked literals (potential PII).",
26 + RequireCloud: true,
27 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(runningQueriesColumns)},
28 + }
29 +}
30 +
31 +// runningQueriesColumn embeds funcapi.ColumnMeta and adds OracleDB-specific fields.
32 +type runningQueriesColumn struct {
33 + funcapi.ColumnMeta
34 + SelectExpr string // SQL expression for SELECT clause
35 + sortOpt bool // whether this column appears as a sort option
36 + sortLbl string // label for sort option dropdown
37 + defaultSort bool // default sort column
38 +}
39 +
40 +// funcapi.SortableColumn interface implementation for runningQueriesColumn.
41 +func (c runningQueriesColumn) IsSortOption() bool { return c.sortOpt }
42 +func (c runningQueriesColumn) SortLabel() string { return c.sortLbl }
43 +func (c runningQueriesColumn) IsDefaultSort() bool { return c.defaultSort }
44 +func (c runningQueriesColumn) ColumnName() string { return c.Name }
45 +func (c runningQueriesColumn) SortColumn() string { return "" }
46 +
47 +var runningQueriesColumns = []runningQueriesColumn{
48 + {ColumnMeta: funcapi.ColumnMeta{Name: "sessionId", Tooltip: "Session", Type: funcapi.FieldTypeString, Visible: true, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, UniqueKey: true, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.sid || ',' || s.serial#"},
49 + {ColumnMeta: funcapi.ColumnMeta{Name: "username", Tooltip: "User", Type: funcapi.FieldTypeString, Visible: true, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.username"},
50 + {ColumnMeta: funcapi.ColumnMeta{Name: "status", Tooltip: "Status", Type: funcapi.FieldTypeString, Visible: true, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.status"},
51 + {ColumnMeta: funcapi.ColumnMeta{Name: "type", Tooltip: "Type", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.type"},
52 + {ColumnMeta: funcapi.ColumnMeta{Name: "sqlId", Tooltip: "SQL ID", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.sql_id"},
53 + {ColumnMeta: funcapi.ColumnMeta{Name: "query", Tooltip: "Query", Type: funcapi.FieldTypeString, Visible: true, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sticky: true, FullWidth: true, Wrap: true}, SelectExpr: "q.sql_text"},
54 + {ColumnMeta: funcapi.ColumnMeta{Name: "lastCallMs", Tooltip: "Elapsed", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax}, SelectExpr: "NVL(s.last_call_et, 0) * 1000", sortOpt: true, defaultSort: true, sortLbl: "Running queries by Elapsed Time"},
55 + {ColumnMeta: funcapi.ColumnMeta{Name: "sqlExecStart", Tooltip: "SQL Exec Start", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax}, SelectExpr: "TO_CHAR(CAST(s.sql_exec_start AS TIMESTAMP), 'YYYY-MM-DD\"T\"HH24:MI:SS.FF3')"},
56 + {ColumnMeta: funcapi.ColumnMeta{Name: "module", Tooltip: "Module", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.module"},
57 + {ColumnMeta: funcapi.ColumnMeta{Name: "action", Tooltip: "Action", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.action"},
58 + {ColumnMeta: funcapi.ColumnMeta{Name: "program", Tooltip: "Program", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.program"},
59 + {ColumnMeta: funcapi.ColumnMeta{Name: "machine", Tooltip: "Machine", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.machine"},
60 +}
61 +
62 +// funcRunningQueries handles the running-queries function.
63 +type funcRunningQueries struct {
64 + router *funcRouter
65 +}
66 +
67 +func newFuncRunningQueries(r *funcRouter) *funcRunningQueries {
68 + return &funcRunningQueries{router: r}
69 +}
70 +
71 +// Compile-time interface check.
72 +var _ funcapi.MethodHandler = (*funcRunningQueries)(nil)
73 +
74 +func (f *funcRunningQueries) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
75 + return []funcapi.ParamConfig{funcapi.BuildSortParam(runningQueriesColumns)}, nil
76 +}
77 +
78 +func (f *funcRunningQueries) Cleanup(ctx context.Context) {}
79 +
80 +func (f *funcRunningQueries) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
81 + if f.router.collector.db == nil {
82 + if err := f.router.collector.openConnection(); err != nil {
83 + return funcapi.UnavailableResponse("collector is still initializing, please retry in a few seconds")
84 + }
85 + }
86 +
87 + limit := f.router.topQueriesLimit()
88 +
89 + sortColumn := f.resolveSortColumn(params.Column("__sort"))
90 + if sortColumn == "" {
91 + return funcapi.InternalErrorResponse("no sortable columns available")
92 + }
93 +
94 + query := fmt.Sprintf(`
95 +SELECT %s
96 +FROM v$session s
97 +LEFT JOIN v$sql q
98 + ON q.sql_id = s.sql_id AND q.child_number = s.sql_child_number
99 +WHERE s.type = 'USER'
100 + AND s.status = 'ACTIVE'
101 + AND s.sql_id IS NOT NULL
102 +ORDER BY %s DESC NULLS LAST
103 +FETCH FIRST %d ROWS ONLY
104 +`, f.buildSelectClause(), sortColumn, limit)
105 +
106 + rows, err := f.router.collector.db.QueryContext(ctx, query)
107 + if err != nil {
108 + if ctx.Err() == context.DeadlineExceeded {
109 + return funcapi.ErrorResponse(504, "query timed out")
110 + }
111 + return funcapi.InternalErrorResponse("running queries query failed: %v", err)
112 + }
113 + defer rows.Close()
114 +
115 + data, err := f.scanRows(rows)
116 + if err != nil {
117 + return funcapi.InternalErrorResponse("%s", err)
118 + }
119 +
120 + cs := f.columnSet()
121 + if len(data) == 0 {
122 + return &funcapi.FunctionResponse{
123 + Status: 200,
124 + Message: "No running queries found.",
125 + Help: "Currently running SQL statements from V$SESSION",
126 + Columns: cs.BuildColumns(),
127 + Data: [][]any{},
128 + DefaultSortColumn: sortColumn,
129 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(runningQueriesColumns)},
130 + }
131 + }
132 +
133 + return &funcapi.FunctionResponse{
134 + Status: 200,
135 + Help: "Currently running SQL statements from V$SESSION. WARNING: Query text may contain unmasked literals (potential PII).",
136 + Columns: cs.BuildColumns(),
137 + Data: data,
138 + DefaultSortColumn: sortColumn,
139 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(runningQueriesColumns)},
140 + }
141 +}
142 +
143 +func (f *funcRunningQueries) columnSet() funcapi.ColumnSet[runningQueriesColumn] {
144 + return funcapi.Columns(runningQueriesColumns, func(c runningQueriesColumn) funcapi.ColumnMeta { return c.ColumnMeta })
145 +}
146 +
147 +func (f *funcRunningQueries) resolveSortColumn(requested string) string {
148 + for _, col := range runningQueriesColumns {
149 + if col.IsSortOption() && col.Name == requested {
150 + return col.Name
151 + }
152 + }
153 + for _, col := range runningQueriesColumns {
154 + if col.IsDefaultSort() {
155 + return col.Name
156 + }
157 + }
158 + for _, col := range runningQueriesColumns {
159 + if col.IsSortOption() {
160 + return col.Name
161 + }
162 + }
163 + return ""
164 +}
165 +
166 +func (f *funcRunningQueries) buildSelectClause() string {
167 + parts := make([]string, 0, len(runningQueriesColumns))
168 + for _, col := range runningQueriesColumns {
169 + expr := col.SelectExpr
170 + if expr == "" {
171 + expr = col.Name
172 + }
173 + parts = append(parts, fmt.Sprintf("%s AS %s", expr, col.Name))
174 + }
175 + return strings.Join(parts, ", ")
176 +}
177 +
178 +func (f *funcRunningQueries) scanRows(rows *sql.Rows) ([][]any, error) {
179 + cols := runningQueriesColumns
180 + data := make([][]any, 0, 500)
181 +
182 + for rows.Next() {
183 + values := make([]any, len(cols))
184 + valuePtrs := make([]any, len(cols))
185 +
186 + for i, col := range cols {
187 + switch col.Type {
188 + case funcapi.FieldTypeString:
189 + var v sql.NullString
190 + values[i] = &v
191 + case funcapi.FieldTypeInteger:
192 + var v sql.NullInt64
193 + values[i] = &v
194 + case funcapi.FieldTypeDuration:
195 + var v sql.NullFloat64
196 + values[i] = &v
197 + default:
198 + var v any
199 + values[i] = &v
200 + }
201 + valuePtrs[i] = values[i]
202 + }
203 +
204 + if err := rows.Scan(valuePtrs...); err != nil {
205 + return nil, fmt.Errorf("row scan failed: %w", err)
206 + }
207 +
208 + row := make([]any, len(cols))
209 + for i, col := range cols {
210 + switch v := values[i].(type) {
211 + case *sql.NullString:
212 + if v.Valid {
213 + s := v.String
214 + if col.Name == "query" {
215 + s = strmutil.TruncateText(s, runningQueriesMaxTextLength)
216 + }
217 + row[i] = s
218 + } else {
219 + row[i] = ""
220 + }
221 + case *sql.NullInt64:
222 + if v.Valid {
223 + row[i] = v.Int64
224 + } else {
225 + row[i] = int64(0)
226 + }
227 + case *sql.NullFloat64:
228 + if v.Valid {
229 + row[i] = v.Float64
230 + } else {
231 + row[i] = float64(0)
232 + }
233 + default:
234 + row[i] = nil
235 + }
236 + }
237 +
238 + data = append(data, row)
239 + }
240 +
241 + if err := rows.Err(); err != nil {
242 + return nil, fmt.Errorf("rows iteration error: %w", err)
243 + }
244 +
245 + return data, nil
246 +}
src/go/plugin/go.d/collector/oracledb/func_top_queries.go new
+340
@@ -0,0 +1,340 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package oracledb
4 +
5 +import (
6 + "context"
7 + "database/sql"
8 + "fmt"
9 + "strings"
10 +
11 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
12 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
13 +)
14 +
15 +const (
16 + topQueriesMethodID = "top-queries"
17 + topQueriesMaxTextLength = 4096
18 +)
19 +
20 +func topQueriesMethodConfig() funcapi.MethodConfig {
21 + return funcapi.MethodConfig{
22 + ID: topQueriesMethodID,
23 + Name: "Top Queries",
24 + UpdateEvery: 10,
25 + Help: "Top SQL statements from V$SQLSTATS. WARNING: Query text may contain unmasked literals (potential PII).",
26 + RequireCloud: true,
27 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(topQueriesColumns)},
28 + }
29 +}
30 +
31 +// topQueriesColumn embeds funcapi.ColumnMeta and adds OracleDB-specific fields.
32 +type topQueriesColumn struct {
33 + funcapi.ColumnMeta
34 + SelectExpr string // SQL expression for SELECT clause
35 + sortOpt bool // whether this column appears as a sort option
36 + sortLbl string // label for sort option dropdown
37 + defaultSort bool // default sort column
38 + RequiresColumn string // only include if this column exists in the view
39 +}
40 +
41 +// funcapi.SortableColumn interface implementation for topQueriesColumn.
42 +func (c topQueriesColumn) IsSortOption() bool { return c.sortOpt }
43 +func (c topQueriesColumn) SortLabel() string { return c.sortLbl }
44 +func (c topQueriesColumn) IsDefaultSort() bool { return c.defaultSort }
45 +func (c topQueriesColumn) ColumnName() string { return c.Name }
46 +func (c topQueriesColumn) SortColumn() string { return "" }
47 +
48 +var topQueriesColumns = []topQueriesColumn{
49 + {ColumnMeta: funcapi.ColumnMeta{Name: "sqlId", Tooltip: "SQL ID", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, UniqueKey: true, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.sql_id"},
50 + {ColumnMeta: funcapi.ColumnMeta{Name: "query", Tooltip: "Query", Type: funcapi.FieldTypeString, Visible: true, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sticky: true, FullWidth: true, Wrap: true}, SelectExpr: "s.sql_text"},
51 + {ColumnMeta: funcapi.ColumnMeta{Name: "schema", Tooltip: "Schema", Type: funcapi.FieldTypeString, Visible: true, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, GroupBy: &funcapi.GroupByOptions{IsDefault: true}}, SelectExpr: "s.parsing_schema_name"},
52 +
53 + {ColumnMeta: funcapi.ColumnMeta{Name: "executions", Tooltip: "Executions", Type: funcapi.FieldTypeInteger, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "Calls", Title: "Executions", IsDefault: true}}, SelectExpr: "NVL(s.executions, 0)", sortOpt: true, sortLbl: "Top queries by Executions"},
54 + {ColumnMeta: funcapi.ColumnMeta{Name: "totalTime", Tooltip: "Total Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Execution Time", IsDefault: true}}, SelectExpr: "NVL(s.elapsed_time, 0) / 1000", sortOpt: true, defaultSort: true, sortLbl: "Top queries by Total Time"},
55 + {ColumnMeta: funcapi.ColumnMeta{Name: "avgTime", Tooltip: "Avg Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMean, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Execution Time"}}, SelectExpr: "CASE WHEN NVL(s.executions,0) = 0 THEN 0 ELSE (s.elapsed_time / s.executions) / 1000 END", sortOpt: true, sortLbl: "Top queries by Avg Time"},
56 + {ColumnMeta: funcapi.ColumnMeta{Name: "cpuTime", Tooltip: "CPU Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "CPU", Title: "CPU Time"}}, SelectExpr: "NVL(s.cpu_time, 0) / 1000", sortOpt: true, sortLbl: "Top queries by CPU Time"},
57 +
58 + {ColumnMeta: funcapi.ColumnMeta{Name: "bufferGets", Tooltip: "Buffer Gets", Type: funcapi.FieldTypeInteger, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "IO", Title: "I/O"}}, SelectExpr: "NVL(s.buffer_gets, 0)", sortOpt: true, sortLbl: "Top queries by Buffer Gets"},
59 + {ColumnMeta: funcapi.ColumnMeta{Name: "diskReads", Tooltip: "Disk Reads", Type: funcapi.FieldTypeInteger, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "IO", Title: "I/O"}}, SelectExpr: "NVL(s.disk_reads, 0)", sortOpt: true, sortLbl: "Top queries by Disk Reads"},
60 + {ColumnMeta: funcapi.ColumnMeta{Name: "rowsProcessed", Tooltip: "Rows Processed", Type: funcapi.FieldTypeInteger, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "Rows", Title: "Rows"}}, SelectExpr: "NVL(s.rows_processed, 0)", sortOpt: true, sortLbl: "Top queries by Rows Processed"},
61 + {ColumnMeta: funcapi.ColumnMeta{Name: "parseCalls", Tooltip: "Parse Calls", Type: funcapi.FieldTypeInteger, Visible: false, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "Parse", Title: "Parse Calls"}}, SelectExpr: "NVL(s.parse_calls, 0)", sortOpt: true, sortLbl: "Top queries by Parse Calls"},
62 +
63 + {ColumnMeta: funcapi.ColumnMeta{Name: "module", Tooltip: "Module", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, GroupBy: &funcapi.GroupByOptions{}}, SelectExpr: "s.module", RequiresColumn: "MODULE"},
64 + {ColumnMeta: funcapi.ColumnMeta{Name: "action", Tooltip: "Action", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, GroupBy: &funcapi.GroupByOptions{}}, SelectExpr: "s.action", RequiresColumn: "ACTION"},
65 + {ColumnMeta: funcapi.ColumnMeta{Name: "lastActiveTime", Tooltip: "Last Active", Type: funcapi.FieldTypeString, Visible: false, Sortable: false, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformText}, SelectExpr: "TO_CHAR(CAST(s.last_active_time AS TIMESTAMP), 'YYYY-MM-DD\"T\"HH24:MI:SS.FF3')", RequiresColumn: "LAST_ACTIVE_TIME"},
66 +}
67 +
68 +type topQueriesLayout struct {
69 + cols []topQueriesColumn
70 + join string
71 +}
72 +
73 +// funcTopQueries handles the top-queries function.
74 +type funcTopQueries struct {
75 + router *funcRouter
76 +}
77 +
78 +func newFuncTopQueries(r *funcRouter) *funcTopQueries {
79 + return &funcTopQueries{router: r}
80 +}
81 +
82 +// Compile-time interface check.
83 +var _ funcapi.MethodHandler = (*funcTopQueries)(nil)
84 +
85 +func (f *funcTopQueries) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
86 + cols := topQueriesColumns
87 + if f.router.collector.db != nil {
88 + cols = f.layout(ctx).cols
89 + }
90 + return []funcapi.ParamConfig{funcapi.BuildSortParam(cols)}, nil
91 +}
92 +
93 +func (f *funcTopQueries) Cleanup(ctx context.Context) {}
94 +
95 +func (f *funcTopQueries) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
96 + if f.router.collector.db == nil {
97 + if err := f.router.collector.openConnection(); err != nil {
98 + return funcapi.UnavailableResponse("collector is still initializing, please retry in a few seconds")
99 + }
100 + }
101 +
102 + layout := f.layout(ctx)
103 + cols := layout.cols
104 + limit := f.router.topQueriesLimit()
105 +
106 + sortColumn := f.resolveSortColumn(cols, params.Column("__sort"))
107 + if sortColumn == "" {
108 + return funcapi.InternalErrorResponse("no sortable columns available")
109 + }
110 +
111 + joinClause := ""
112 + if layout.join != "" {
113 + joinClause = "\n" + layout.join
114 + }
115 + query := fmt.Sprintf(`
116 +SELECT %s
117 +FROM v$sqlstats s
118 +%s
119 +WHERE NVL(s.executions, 0) > 0
120 +ORDER BY %s DESC NULLS LAST
121 +FETCH FIRST %d ROWS ONLY
122 +`, f.buildSelectClause(cols), joinClause, sortColumn, limit)
123 +
124 + rows, err := f.router.collector.db.QueryContext(ctx, query)
125 + if err != nil {
126 + if ctx.Err() == context.DeadlineExceeded {
127 + return funcapi.ErrorResponse(504, "query timed out")
128 + }
129 + return funcapi.InternalErrorResponse("top queries query failed: %v", err)
130 + }
131 + defer rows.Close()
132 +
133 + data, err := f.scanRows(rows, cols)
134 + if err != nil {
135 + return funcapi.InternalErrorResponse("%s", err)
136 + }
137 +
138 + cs := f.columnSet(cols)
139 + if len(data) == 0 {
140 + return &funcapi.FunctionResponse{
141 + Status: 200,
142 + Message: "No SQL statements found.",
143 + Help: "Top SQL statements from V$SQLSTATS",
144 + Columns: cs.BuildColumns(),
145 + Data: [][]any{},
146 + DefaultSortColumn: sortColumn,
147 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(cols)},
148 + ChartingConfig: cs.BuildCharting(),
149 + }
150 + }
151 +
152 + return &funcapi.FunctionResponse{
153 + Status: 200,
154 + Help: "Top SQL statements from V$SQLSTATS. WARNING: Query text may contain unmasked literals (potential PII).",
155 + Columns: cs.BuildColumns(),
156 + Data: data,
157 + DefaultSortColumn: sortColumn,
158 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(cols)},
159 + ChartingConfig: cs.BuildCharting(),
160 + }
161 +}
162 +
163 +func (f *funcTopQueries) columnSet(cols []topQueriesColumn) funcapi.ColumnSet[topQueriesColumn] {
164 + return funcapi.Columns(cols, func(c topQueriesColumn) funcapi.ColumnMeta { return c.ColumnMeta })
165 +}
166 +
167 +func (f *funcTopQueries) layout(ctx context.Context) topQueriesLayout {
168 + available, err := f.fetchSQLStatsColumns(ctx)
169 + if err != nil {
170 + return topQueriesLayout{cols: f.filterColumns(topQueriesColumns, nil)}
171 + }
172 + return f.buildLayout(available)
173 +}
174 +
175 +func (f *funcTopQueries) fetchSQLStatsColumns(ctx context.Context) (map[string]bool, error) {
176 + rows, err := f.router.collector.db.QueryContext(ctx, "SELECT * FROM v$sqlstats WHERE 1=0")
177 + if err != nil {
178 + return nil, err
179 + }
180 + defer rows.Close()
181 +
182 + names, err := rows.Columns()
183 + if err != nil {
184 + return nil, err
185 + }
186 +
187 + cols := make(map[string]bool, len(names))
188 + for _, name := range names {
189 + cols[strings.ToUpper(name)] = true
190 + }
191 +
192 + return cols, nil
193 +}
194 +
195 +func (f *funcTopQueries) buildLayout(available map[string]bool) topQueriesLayout {
196 + schemaExpr, schemaJoin := f.resolveSchemaExpr(available)
197 + filtered := make([]topQueriesColumn, 0, len(topQueriesColumns))
198 + for _, col := range topQueriesColumns {
199 + if col.Name == "schema" {
200 + if schemaExpr == "" {
201 + continue
202 + }
203 + col.SelectExpr = schemaExpr
204 + }
205 + if col.RequiresColumn != "" && !available[strings.ToUpper(col.RequiresColumn)] {
206 + continue
207 + }
208 + filtered = append(filtered, col)
209 + }
210 + return topQueriesLayout{cols: filtered, join: schemaJoin}
211 +}
212 +
213 +func (f *funcTopQueries) resolveSchemaExpr(available map[string]bool) (string, string) {
214 + switch {
215 + case available["PARSING_SCHEMA_NAME"]:
216 + return "s.parsing_schema_name", ""
217 + case available["PARSING_SCHEMA_ID"]:
218 + return "COALESCE(u.username, TO_CHAR(s.parsing_schema_id))", "LEFT JOIN all_users u ON u.user_id = s.parsing_schema_id"
219 + case available["LAST_EXEC_USER_ID"]:
220 + return "COALESCE(u.username, TO_CHAR(s.last_exec_user_id))", "LEFT JOIN all_users u ON u.user_id = s.last_exec_user_id"
221 + default:
222 + return "", ""
223 + }
224 +}
225 +
226 +func (f *funcTopQueries) filterColumns(cols []topQueriesColumn, available map[string]bool) []topQueriesColumn {
227 + filtered := make([]topQueriesColumn, 0, len(cols))
228 + for _, col := range cols {
229 + if col.RequiresColumn != "" {
230 + if available == nil {
231 + continue
232 + }
233 + if !available[strings.ToUpper(col.RequiresColumn)] {
234 + continue
235 + }
236 + }
237 + filtered = append(filtered, col)
238 + }
239 + return filtered
240 +}
241 +
242 +func (f *funcTopQueries) resolveSortColumn(cols []topQueriesColumn, requested string) string {
243 + for _, col := range cols {
244 + if col.IsSortOption() && col.Name == requested {
245 + return col.Name
246 + }
247 + }
248 + for _, col := range cols {
249 + if col.IsDefaultSort() {
250 + return col.Name
251 + }
252 + }
253 + for _, col := range cols {
254 + if col.IsSortOption() {
255 + return col.Name
256 + }
257 + }
258 + return ""
259 +}
260 +
261 +func (f *funcTopQueries) buildSelectClause(cols []topQueriesColumn) string {
262 + parts := make([]string, 0, len(cols))
263 + for _, col := range cols {
264 + expr := col.SelectExpr
265 + if expr == "" {
266 + expr = col.Name
267 + }
268 + parts = append(parts, fmt.Sprintf("%s AS %s", expr, col.Name))
269 + }
270 + return strings.Join(parts, ", ")
271 +}
272 +
273 +func (f *funcTopQueries) scanRows(rows *sql.Rows, cols []topQueriesColumn) ([][]any, error) {
274 + data := make([][]any, 0, 500)
275 +
276 + for rows.Next() {
277 + values := make([]any, len(cols))
278 + valuePtrs := make([]any, len(cols))
279 +
280 + for i, col := range cols {
281 + switch col.Type {
282 + case funcapi.FieldTypeString:
283 + var v sql.NullString
284 + values[i] = &v
285 + case funcapi.FieldTypeInteger:
286 + var v sql.NullInt64
287 + values[i] = &v
288 + case funcapi.FieldTypeDuration:
289 + var v sql.NullFloat64
290 + values[i] = &v
291 + default:
292 + var v any
293 + values[i] = &v
294 + }
295 + valuePtrs[i] = values[i]
296 + }
297 +
298 + if err := rows.Scan(valuePtrs...); err != nil {
299 + return nil, fmt.Errorf("row scan failed: %w", err)
300 + }
301 +
302 + row := make([]any, len(cols))
303 + for i, col := range cols {
304 + switch v := values[i].(type) {
305 + case *sql.NullString:
306 + if v.Valid {
307 + s := v.String
308 + if col.Name == "query" {
309 + s = strmutil.TruncateText(s, topQueriesMaxTextLength)
310 + }
311 + row[i] = s
312 + } else {
313 + row[i] = ""
314 + }
315 + case *sql.NullInt64:
316 + if v.Valid {
317 + row[i] = v.Int64
318 + } else {
319 + row[i] = int64(0)
320 + }
321 + case *sql.NullFloat64:
322 + if v.Valid {
323 + row[i] = v.Float64
324 + } else {
325 + row[i] = float64(0)
326 + }
327 + default:
328 + row[i] = nil
329 + }
330 + }
331 +
332 + data = append(data, row)
333 + }
334 +
335 + if err := rows.Err(); err != nil {
336 + return nil, fmt.Errorf("rows iteration error: %w", err)
337 + }
338 +
339 + return data, nil
340 +}
src/go/plugin/go.d/collector/oracledb/functions.go deleted
-643
@@ -1,643 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -package oracledb
4 -
5 -import (
6 - "context"
7 - "database/sql"
8 - "fmt"
9 - "strings"
10 -
11 - "github.com/netdata/netdata/go/plugins/pkg/funcapi"
12 - "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
13 - "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
14 -)
15 -
16 -const oracleMaxQueryTextLength = 4096
17 -
18 -const (
19 - paramSort = "__sort"
20 -
21 - ftString = funcapi.FieldTypeString
22 - ftInteger = funcapi.FieldTypeInteger
23 - ftDuration = funcapi.FieldTypeDuration
24 -
25 - trNone = funcapi.FieldTransformNone
26 - trNumber = funcapi.FieldTransformNumber
27 - trDuration = funcapi.FieldTransformDuration
28 - trText = funcapi.FieldTransformText
29 -
30 - visValue = funcapi.FieldVisualValue
31 - visBar = funcapi.FieldVisualBar
32 -
33 - sortAsc = funcapi.FieldSortAscending
34 - sortDesc = funcapi.FieldSortDescending
35 -
36 - summaryCount = funcapi.FieldSummaryCount
37 - summarySum = funcapi.FieldSummarySum
38 - summaryMax = funcapi.FieldSummaryMax
39 - summaryMean = funcapi.FieldSummaryMean
40 -
41 - filterMulti = funcapi.FieldFilterMultiselect
42 - filterRange = funcapi.FieldFilterRange
43 -)
44 -
45 -type oracleColumnMeta struct {
46 - id string
47 - name string
48 - selectExpr string
49 - dataType funcapi.FieldType
50 - visible bool
51 - sortable bool
52 - fullWidth bool
53 - wrap bool
54 - sticky bool
55 - filter funcapi.FieldFilter
56 - visualization funcapi.FieldVisual
57 - transform funcapi.FieldTransform
58 - units string
59 - decimalPoints int
60 - uniqueKey bool
61 - sortDir funcapi.FieldSort
62 - summary funcapi.FieldSummary
63 - sortLabel string
64 - isSortOption bool
65 - isDefaultSort bool
66 - requiresColumn string
67 - isLabel bool
68 - isPrimary bool
69 - isMetric bool
70 - chartGroup string
71 - chartTitle string
72 - isDefaultChart bool
73 -}
74 -
75 -type oracleTopLayout struct {
76 - cols []oracleColumnMeta
77 - join string
78 -}
79 -
80 -var oracleTopColumns = []oracleColumnMeta{
81 - {id: "sqlId", name: "SQL ID", selectExpr: "s.sql_id", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, uniqueKey: true, sortDir: sortAsc, summary: summaryCount},
82 - {id: "query", name: "Query", selectExpr: "s.sql_text", dataType: ftString, visible: true, sortable: false, filter: filterMulti, transform: trText, sticky: true, fullWidth: true, wrap: true},
83 - {id: "schema", name: "Schema", selectExpr: "s.parsing_schema_name", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount, isLabel: true, isPrimary: true},
84 -
85 - {id: "executions", name: "Executions", selectExpr: "NVL(s.executions, 0)", dataType: ftInteger, visible: true, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by Executions", isMetric: true, chartGroup: "Calls", chartTitle: "Executions", isDefaultChart: true},
86 - {id: "totalTime", name: "Total Time", selectExpr: "NVL(s.elapsed_time, 0) / 1000", dataType: ftDuration, visible: true, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summarySum, isSortOption: true, isDefaultSort: true, sortLabel: "Top queries by Total Time", isMetric: true, chartGroup: "Time", chartTitle: "Execution Time", isDefaultChart: true},
87 - {id: "avgTime", name: "Avg Time", selectExpr: "CASE WHEN NVL(s.executions,0) = 0 THEN 0 ELSE (s.elapsed_time / s.executions) / 1000 END", dataType: ftDuration, visible: true, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMean, isSortOption: true, sortLabel: "Top queries by Avg Time", isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
88 - {id: "cpuTime", name: "CPU Time", selectExpr: "NVL(s.cpu_time, 0) / 1000", dataType: ftDuration, visible: true, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by CPU Time", isMetric: true, chartGroup: "CPU", chartTitle: "CPU Time"},
89 -
90 - {id: "bufferGets", name: "Buffer Gets", selectExpr: "NVL(s.buffer_gets, 0)", dataType: ftInteger, visible: true, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by Buffer Gets", isMetric: true, chartGroup: "IO", chartTitle: "I/O"},
91 - {id: "diskReads", name: "Disk Reads", selectExpr: "NVL(s.disk_reads, 0)", dataType: ftInteger, visible: true, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by Disk Reads", isMetric: true, chartGroup: "IO", chartTitle: "I/O"},
92 - {id: "rowsProcessed", name: "Rows Processed", selectExpr: "NVL(s.rows_processed, 0)", dataType: ftInteger, visible: true, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by Rows Processed", isMetric: true, chartGroup: "Rows", chartTitle: "Rows"},
93 - {id: "parseCalls", name: "Parse Calls", selectExpr: "NVL(s.parse_calls, 0)", dataType: ftInteger, visible: false, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by Parse Calls", isMetric: true, chartGroup: "Parse", chartTitle: "Parse Calls"},
94 -
95 - {id: "module", name: "Module", selectExpr: "s.module", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount, requiresColumn: "MODULE", isLabel: true},
96 - {id: "action", name: "Action", selectExpr: "s.action", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount, requiresColumn: "ACTION", isLabel: true},
97 - {id: "lastActiveTime", name: "Last Active", selectExpr: "TO_CHAR(CAST(s.last_active_time AS TIMESTAMP), 'YYYY-MM-DD\"T\"HH24:MI:SS.FF3')", dataType: ftString, visible: false, sortable: false, filter: filterRange, transform: trText, requiresColumn: "LAST_ACTIVE_TIME"},
98 -}
99 -
100 -var oracleRunningColumns = []oracleColumnMeta{
101 - {id: "sessionId", name: "Session", selectExpr: "s.sid || ',' || s.serial#", dataType: ftString, visible: true, sortable: false, filter: filterMulti, transform: trText, uniqueKey: true, sortDir: sortAsc, summary: summaryCount},
102 - {id: "username", name: "User", selectExpr: "s.username", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
103 - {id: "status", name: "Status", selectExpr: "s.status", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
104 - {id: "type", name: "Type", selectExpr: "s.type", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
105 - {id: "sqlId", name: "SQL ID", selectExpr: "s.sql_id", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
106 - {id: "query", name: "Query", selectExpr: "q.sql_text", dataType: ftString, visible: true, sortable: false, filter: filterMulti, transform: trText, sticky: true, fullWidth: true, wrap: true},
107 - {id: "lastCallMs", name: "Elapsed", selectExpr: "NVL(s.last_call_et, 0) * 1000", dataType: ftDuration, visible: true, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, isSortOption: true, isDefaultSort: true, sortLabel: "Running queries by Elapsed Time"},
108 - {id: "sqlExecStart", name: "SQL Exec Start", selectExpr: "TO_CHAR(CAST(s.sql_exec_start AS TIMESTAMP), 'YYYY-MM-DD\"T\"HH24:MI:SS.FF3')", dataType: ftString, visible: false, sortable: true, filter: filterRange, transform: trText, sortDir: sortDesc, summary: summaryMax},
109 - {id: "module", name: "Module", selectExpr: "s.module", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
110 - {id: "action", name: "Action", selectExpr: "s.action", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
111 - {id: "program", name: "Program", selectExpr: "s.program", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
112 - {id: "machine", name: "Machine", selectExpr: "s.machine", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
113 -}
114 -
115 -func oracleMethods() []module.MethodConfig {
116 - return []module.MethodConfig{
117 - {
118 - UpdateEvery: 10,
119 - ID: "top-queries",
120 - Name: "Top Queries",
121 - Help: "Top SQL statements from V$SQLSTATS. WARNING: Query text may contain unmasked literals (potential PII).",
122 - RequireCloud: true,
123 - RequiredParams: []funcapi.ParamConfig{
124 - buildOracleSortParam(oracleTopColumns),
125 - },
126 - },
127 - {
128 - UpdateEvery: 10,
129 - ID: "running-queries",
130 - Name: "Running Queries",
131 - Help: "Currently running SQL statements from V$SESSION. WARNING: Query text may contain unmasked literals (potential PII).",
132 - RequireCloud: true,
133 - RequiredParams: []funcapi.ParamConfig{
134 - buildOracleSortParam(oracleRunningColumns),
135 - },
136 - },
137 - }
138 -}
139 -
140 -func oracleMethodParams(ctx context.Context, job *module.Job, method string) ([]funcapi.ParamConfig, error) {
141 - collector, ok := job.Module().(*Collector)
142 - if !ok {
143 - return nil, fmt.Errorf("invalid module type")
144 - }
145 - switch method {
146 - case "top-queries":
147 - cols := oracleTopColumns
148 - if collector.db != nil {
149 - cols = collector.oracleTopLayout(ctx).cols
150 - }
151 - return []funcapi.ParamConfig{buildOracleSortParam(cols)}, nil
152 - case "running-queries":
153 - return []funcapi.ParamConfig{buildOracleSortParam(oracleRunningColumns)}, nil
154 - default:
155 - return nil, fmt.Errorf("unknown method: %s", method)
156 - }
157 -}
158 -
159 -func oracleHandleMethod(ctx context.Context, job *module.Job, method string, params funcapi.ResolvedParams) *module.FunctionResponse {
160 - collector, ok := job.Module().(*Collector)
161 - if !ok {
162 - return &module.FunctionResponse{Status: 500, Message: "internal error: invalid module type"}
163 - }
164 -
165 - if collector.db == nil {
166 - if err := collector.openConnection(); err != nil {
167 - return &module.FunctionResponse{Status: 503, Message: "collector is still initializing, please retry in a few seconds"}
168 - }
169 - }
170 -
171 - switch method {
172 - case "top-queries":
173 - return collector.collectTopQueries(ctx, params.Column(paramSort))
174 - case "running-queries":
175 - return collector.collectRunningQueries(ctx, params.Column(paramSort))
176 - default:
177 - return &module.FunctionResponse{Status: 404, Message: fmt.Sprintf("unknown method: %s", method)}
178 - }
179 -}
180 -
181 -func buildOracleSortParam(cols []oracleColumnMeta) funcapi.ParamConfig {
182 - return funcapi.ParamConfig{
183 - ID: paramSort,
184 - Name: "Filter By",
185 - Help: "Select the primary sort column",
186 - Selection: funcapi.ParamSelect,
187 - Options: buildOracleSortOptions(cols),
188 - UniqueView: true,
189 - }
190 -}
191 -
192 -func buildOracleSortOptions(cols []oracleColumnMeta) []funcapi.ParamOption {
193 - var sortOptions []funcapi.ParamOption
194 - sortDir := funcapi.FieldSortDescending
195 - for _, col := range cols {
196 - if !col.isSortOption {
197 - continue
198 - }
199 - opt := funcapi.ParamOption{
200 - ID: col.id,
201 - Column: col.id,
202 - Name: col.sortLabel,
203 - Sort: &sortDir,
204 - }
205 - if col.isDefaultSort {
206 - opt.Default = true
207 - }
208 - sortOptions = append(sortOptions, opt)
209 - }
210 - return sortOptions
211 -}
212 -
213 -func buildOracleColumns(cols []oracleColumnMeta) map[string]any {
214 - result := make(map[string]any, len(cols))
215 - for i, col := range cols {
216 - visual := visValue
217 - if col.dataType == ftDuration {
218 - visual = visBar
219 - }
220 - colDef := funcapi.Column{
221 - Index: i,
222 - Name: col.name,
223 - Type: col.dataType,
224 - Units: col.units,
225 - Visualization: visual,
226 - Sort: col.sortDir,
227 - Sortable: col.sortable,
228 - Sticky: col.sticky,
229 - Summary: col.summary,
230 - Filter: col.filter,
231 - FullWidth: col.fullWidth,
232 - Wrap: col.wrap,
233 - DefaultExpandedFilter: false,
234 - UniqueKey: col.uniqueKey,
235 - Visible: col.visible,
236 - ValueOptions: funcapi.ValueOptions{
237 - Transform: col.transform,
238 - DecimalPoints: col.decimalPoints,
239 - DefaultValue: nil,
240 - },
241 - }
242 - result[col.id] = colDef.BuildColumn()
243 - }
244 - return result
245 -}
246 -
247 -func buildOracleSelect(cols []oracleColumnMeta) string {
248 - parts := make([]string, 0, len(cols))
249 - for _, col := range cols {
250 - expr := col.selectExpr
251 - if expr == "" {
252 - expr = col.id
253 - }
254 - parts = append(parts, fmt.Sprintf("%s AS %s", expr, col.id))
255 - }
256 - return strings.Join(parts, ", ")
257 -}
258 -
259 -func mapOracleSortColumn(input string, cols []oracleColumnMeta) string {
260 - for _, col := range cols {
261 - if col.isSortOption && col.id == input {
262 - return col.id
263 - }
264 - }
265 - for _, col := range cols {
266 - if col.isDefaultSort {
267 - return col.id
268 - }
269 - }
270 - for _, col := range cols {
271 - if col.isSortOption {
272 - return col.id
273 - }
274 - }
275 - return ""
276 -}
277 -
278 -func (c *Collector) oracleTopLayout(ctx context.Context) oracleTopLayout {
279 - available, err := c.fetchSQLStatsColumns(ctx)
280 - if err != nil {
281 - return oracleTopLayout{cols: filterOracleColumns(oracleTopColumns, nil)}
282 - }
283 - return buildOracleTopLayout(available)
284 -}
285 -
286 -func filterOracleColumns(cols []oracleColumnMeta, available map[string]bool) []oracleColumnMeta {
287 - filtered := make([]oracleColumnMeta, 0, len(cols))
288 - for _, col := range cols {
289 - if col.requiresColumn != "" {
290 - if available == nil {
291 - continue
292 - }
293 - if !available[strings.ToUpper(col.requiresColumn)] {
294 - continue
295 - }
296 - }
297 - filtered = append(filtered, col)
298 - }
299 - return filtered
300 -}
301 -
302 -func buildOracleTopLayout(available map[string]bool) oracleTopLayout {
303 - schemaExpr, schemaJoin := resolveOracleSchemaExpr(available)
304 - filtered := make([]oracleColumnMeta, 0, len(oracleTopColumns))
305 - for _, col := range oracleTopColumns {
306 - if col.id == "schema" {
307 - if schemaExpr == "" {
308 - continue
309 - }
310 - col.selectExpr = schemaExpr
311 - }
312 - if col.requiresColumn != "" && !available[strings.ToUpper(col.requiresColumn)] {
313 - continue
314 - }
315 - filtered = append(filtered, col)
316 - }
317 - return oracleTopLayout{cols: filtered, join: schemaJoin}
318 -}
319 -
320 -func resolveOracleSchemaExpr(available map[string]bool) (string, string) {
321 - switch {
322 - case available["PARSING_SCHEMA_NAME"]:
323 - return "s.parsing_schema_name", ""
324 - case available["PARSING_SCHEMA_ID"]:
325 - return "COALESCE(u.username, TO_CHAR(s.parsing_schema_id))", "LEFT JOIN all_users u ON u.user_id = s.parsing_schema_id"
326 - case available["LAST_EXEC_USER_ID"]:
327 - return "COALESCE(u.username, TO_CHAR(s.last_exec_user_id))", "LEFT JOIN all_users u ON u.user_id = s.last_exec_user_id"
328 - default:
329 - return "", ""
330 - }
331 -}
332 -
333 -func (c *Collector) fetchSQLStatsColumns(ctx context.Context) (map[string]bool, error) {
334 - rows, err := c.db.QueryContext(ctx, "SELECT * FROM v$sqlstats WHERE 1=0")
335 - if err != nil {
336 - return nil, err
337 - }
338 - defer rows.Close()
339 -
340 - names, err := rows.Columns()
341 - if err != nil {
342 - return nil, err
343 - }
344 -
345 - cols := make(map[string]bool, len(names))
346 - for _, name := range names {
347 - cols[strings.ToUpper(name)] = true
348 - }
349 -
350 - return cols, nil
351 -}
352 -
353 -func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
354 - layout := c.oracleTopLayout(ctx)
355 - topCols := layout.cols
356 - limit := c.TopQueriesLimit
357 - if limit <= 0 {
358 - limit = 500
359 - }
360 -
361 - sortColumn = mapOracleSortColumn(sortColumn, topCols)
362 - if sortColumn == "" {
363 - return &module.FunctionResponse{Status: 500, Message: "no sortable columns available"}
364 - }
365 -
366 - joinClause := ""
367 - if layout.join != "" {
368 - joinClause = "\n" + layout.join
369 - }
370 - query := fmt.Sprintf(`
371 -SELECT %s
372 -FROM v$sqlstats s
373 -%s
374 -WHERE NVL(s.executions, 0) > 0
375 -ORDER BY %s DESC NULLS LAST
376 -FETCH FIRST %d ROWS ONLY
377 -`, buildOracleSelect(topCols), joinClause, sortColumn, limit)
378 -
379 - rows, err := c.db.QueryContext(ctx, query)
380 - if err != nil {
381 - if ctx.Err() == context.DeadlineExceeded {
382 - return &module.FunctionResponse{Status: 504, Message: "query timed out"}
383 - }
384 - return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("top queries query failed: %v", err)}
385 - }
386 - defer func() { _ = rows.Close() }()
387 -
388 - data, err := scanOracleRows(rows, topCols)
389 - if err != nil {
390 - return &module.FunctionResponse{Status: 500, Message: err.Error()}
391 - }
392 - if len(data) == 0 {
393 - return &module.FunctionResponse{
394 - Status: 200,
395 - Message: "No SQL statements found.",
396 - Help: "Top SQL statements from V$SQLSTATS",
397 - Columns: buildOracleColumns(topCols),
398 - Data: [][]any{},
399 - DefaultSortColumn: sortColumn,
400 - RequiredParams: []funcapi.ParamConfig{buildOracleSortParam(topCols)},
401 - Charts: oracleTopQueriesCharts(topCols),
402 - DefaultCharts: oracleTopQueriesDefaultCharts(topCols),
403 - GroupBy: oracleTopQueriesGroupBy(topCols),
404 - }
405 - }
406 -
407 - return &module.FunctionResponse{
408 - Status: 200,
409 - Help: "Top SQL statements from V$SQLSTATS. WARNING: Query text may contain unmasked literals (potential PII).",
410 - Columns: buildOracleColumns(topCols),
411 - Data: data,
412 - DefaultSortColumn: sortColumn,
413 - RequiredParams: []funcapi.ParamConfig{buildOracleSortParam(topCols)},
414 - Charts: oracleTopQueriesCharts(topCols),
415 - DefaultCharts: oracleTopQueriesDefaultCharts(topCols),
416 - GroupBy: oracleTopQueriesGroupBy(topCols),
417 - }
418 -}
419 -
420 -func (c *Collector) collectRunningQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
421 - limit := c.TopQueriesLimit
422 - if limit <= 0 {
423 - limit = 500
424 - }
425 -
426 - sortColumn = mapOracleSortColumn(sortColumn, oracleRunningColumns)
427 - if sortColumn == "" {
428 - return &module.FunctionResponse{Status: 500, Message: "no sortable columns available"}
429 - }
430 -
431 - query := fmt.Sprintf(`
432 -SELECT %s
433 -FROM v$session s
434 -LEFT JOIN v$sql q
435 - ON q.sql_id = s.sql_id AND q.child_number = s.sql_child_number
436 -WHERE s.type = 'USER'
437 - AND s.status = 'ACTIVE'
438 - AND s.sql_id IS NOT NULL
439 -ORDER BY %s DESC NULLS LAST
440 -FETCH FIRST %d ROWS ONLY
441 -`, buildOracleSelect(oracleRunningColumns), sortColumn, limit)
442 -
443 - rows, err := c.db.QueryContext(ctx, query)
444 - if err != nil {
445 - if ctx.Err() == context.DeadlineExceeded {
446 - return &module.FunctionResponse{Status: 504, Message: "query timed out"}
447 - }
448 - return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("running queries query failed: %v", err)}
449 - }
450 - defer func() { _ = rows.Close() }()
451 -
452 - data, err := scanOracleRows(rows, oracleRunningColumns)
453 - if err != nil {
454 - return &module.FunctionResponse{Status: 500, Message: err.Error()}
455 - }
456 -
457 - if len(data) == 0 {
458 - return &module.FunctionResponse{
459 - Status: 200,
460 - Message: "No running queries found.",
461 - Help: "Currently running SQL statements from V$SESSION",
462 - Columns: buildOracleColumns(oracleRunningColumns),
463 - Data: [][]any{},
464 - DefaultSortColumn: sortColumn,
465 - RequiredParams: []funcapi.ParamConfig{buildOracleSortParam(oracleRunningColumns)},
466 - }
467 - }
468 -
469 - return &module.FunctionResponse{
470 - Status: 200,
471 - Help: "Currently running SQL statements from V$SESSION. WARNING: Query text may contain unmasked literals (potential PII).",
472 - Columns: buildOracleColumns(oracleRunningColumns),
473 - Data: data,
474 - DefaultSortColumn: sortColumn,
475 - RequiredParams: []funcapi.ParamConfig{buildOracleSortParam(oracleRunningColumns)},
476 - }
477 -}
478 -
479 -func scanOracleRows(rows *sql.Rows, cols []oracleColumnMeta) ([][]any, error) {
480 - data := make([][]any, 0, 500)
481 -
482 - for rows.Next() {
483 - values := make([]any, len(cols))
484 - valuePtrs := make([]any, len(cols))
485 -
486 - for i, col := range cols {
487 - switch col.dataType {
488 - case ftString:
489 - var v sql.NullString
490 - values[i] = &v
491 - case ftInteger:
492 - var v sql.NullInt64
493 - values[i] = &v
494 - case ftDuration:
495 - var v sql.NullFloat64
496 - values[i] = &v
497 - default:
498 - var v any
499 - values[i] = &v
500 - }
501 - valuePtrs[i] = values[i]
502 - }
503 -
504 - if err := rows.Scan(valuePtrs...); err != nil {
505 - return nil, fmt.Errorf("row scan failed: %w", err)
506 - }
507 -
508 - row := make([]any, len(cols))
509 - for i, col := range cols {
510 - switch v := values[i].(type) {
511 - case *sql.NullString:
512 - if v.Valid {
513 - s := v.String
514 - if col.id == "query" {
515 - s = strmutil.TruncateText(s, oracleMaxQueryTextLength)
516 - }
517 - row[i] = s
518 - } else {
519 - row[i] = ""
520 - }
521 - case *sql.NullInt64:
522 - if v.Valid {
523 - row[i] = v.Int64
524 - } else {
525 - row[i] = int64(0)
526 - }
527 - case *sql.NullFloat64:
528 - if v.Valid {
529 - row[i] = v.Float64
530 - } else {
531 - row[i] = float64(0)
532 - }
533 - default:
534 - row[i] = nil
535 - }
536 - }
537 -
538 - data = append(data, row)
539 - }
540 -
541 - if err := rows.Err(); err != nil {
542 - return nil, fmt.Errorf("rows iteration error: %w", err)
543 - }
544 -
545 - return data, nil
546 -}
547 -
548 -func oracleTopQueriesCharts(cols []oracleColumnMeta) map[string]module.ChartConfig {
549 - charts := make(map[string]module.ChartConfig)
550 - for _, col := range cols {
551 - if !col.isMetric || col.chartGroup == "" {
552 - continue
553 - }
554 - cfg, ok := charts[col.chartGroup]
555 - if !ok {
556 - title := col.chartTitle
557 - if title == "" {
558 - title = col.chartGroup
559 - }
560 - cfg = module.ChartConfig{Name: title, Type: "stacked-bar"}
561 - }
562 - cfg.Columns = append(cfg.Columns, col.id)
563 - charts[col.chartGroup] = cfg
564 - }
565 - return charts
566 -}
567 -
568 -func oracleTopQueriesDefaultCharts(cols []oracleColumnMeta) [][]string {
569 - label := primaryOracleLabel(cols)
570 - if label == "" {
571 - return nil
572 - }
573 - chartGroups := defaultOracleChartGroups(cols)
574 - out := make([][]string, 0, len(chartGroups))
575 - for _, group := range chartGroups {
576 - out = append(out, []string{group, label})
577 - }
578 - return out
579 -}
580 -
581 -func oracleTopQueriesGroupBy(cols []oracleColumnMeta) map[string]module.GroupByConfig {
582 - groupBy := make(map[string]module.GroupByConfig)
583 - for _, col := range cols {
584 - if !col.isLabel {
585 - continue
586 - }
587 - groupBy[col.id] = module.GroupByConfig{
588 - Name: "Group by " + col.name,
589 - Columns: []string{col.id},
590 - }
591 - }
592 - return groupBy
593 -}
594 -
595 -func hasOracleColumn(cols []oracleColumnMeta, id string) bool {
596 - for _, col := range cols {
597 - if col.id == id {
598 - return true
599 - }
600 - }
601 - return false
602 -}
603 -
604 -func primaryOracleLabel(cols []oracleColumnMeta) string {
605 - for _, col := range cols {
606 - if col.isPrimary {
607 - return col.id
608 - }
609 - }
610 - for _, col := range cols {
611 - if col.isLabel {
612 - return col.id
613 - }
614 - }
615 - return ""
616 -}
617 -
618 -func defaultOracleChartGroups(cols []oracleColumnMeta) []string {
619 - groups := make([]string, 0)
620 - seen := make(map[string]bool)
621 - for _, col := range cols {
622 - if !col.isMetric || col.chartGroup == "" || !col.isDefaultChart {
623 - continue
624 - }
625 - if !seen[col.chartGroup] {
626 - seen[col.chartGroup] = true
627 - groups = append(groups, col.chartGroup)
628 - }
629 - }
630 - if len(groups) > 0 {
631 - return groups
632 - }
633 - for _, col := range cols {
634 - if !col.isMetric || col.chartGroup == "" {
635 - continue
636 - }
637 - if !seen[col.chartGroup] {
638 - seen[col.chartGroup] = true
639 - groups = append(groups, col.chartGroup)
640 - }
641 - }
642 - return groups
643 -}
src/go/plugin/go.d/collector/oracledb/functions_test.go deleted
-60
@@ -1,60 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -package oracledb
4 -
5 -import (
6 - "testing"
7 -
8 - "github.com/netdata/netdata/go/plugins/pkg/funcapi"
9 - "github.com/stretchr/testify/assert"
10 -)
11 -
12 -func TestOracleMethods(t *testing.T) {
13 - methods := oracleMethods()
14 -
15 - require := assert.New(t)
16 - require.Len(methods, 2)
17 -
18 - ids := map[string]bool{}
19 - for _, m := range methods {
20 - ids[m.ID] = true
21 - var sortParam *funcapi.ParamConfig
22 - for i := range m.RequiredParams {
23 - if m.RequiredParams[i].ID == "__sort" {
24 - sortParam = &m.RequiredParams[i]
25 - break
26 - }
27 - }
28 - require.NotNil(sortParam, "expected __sort required param for %s", m.ID)
29 - require.NotEmpty(sortParam.Options)
30 - }
31 -
32 - require.True(ids["top-queries"], "top-queries method missing")
33 - require.True(ids["running-queries"], "running-queries method missing")
34 -}
35 -
36 -func TestOracleTopColumns_HasRequiredColumns(t *testing.T) {
37 - required := []string{"query", "executions", "totalTime"}
38 -
39 - uiKeys := make(map[string]bool)
40 - for _, col := range oracleTopColumns {
41 - uiKeys[col.id] = true
42 - }
43 -
44 - for _, key := range required {
45 - assert.True(t, uiKeys[key], "column %s should be defined", key)
46 - }
47 -}
48 -
49 -func TestOracleRunningColumns_HasRequiredColumns(t *testing.T) {
50 - required := []string{"sessionId", "query", "lastCallMs"}
51 -
52 - uiKeys := make(map[string]bool)
53 - for _, col := range oracleRunningColumns {
54 - uiKeys[col.id] = true
55 - }
56 -
57 - for _, key := range required {
58 - assert.True(t, uiKeys[key], "column %s should be defined", key)
59 - }
60 -}
src/go/plugin/go.d/collector/postgres/collector.go
+9 -3
@@ -28,8 +28,7 @@ func init() {
28 Create: func() module.Module { return New() },
29 Config: func() any { return &Config{} },
30 Methods: pgMethods,
31 - MethodParams: pgMethodParams,
32 - HandleMethod: pgHandleMethod,
31 + MethodHandler: pgFunctionHandler,
32 })
33 }
34
@@ -100,6 +99,8 @@ type (
99 doSlowEvery time.Duration
100
101 mx *pgMetrics
102 +
103 + funcRouter *funcRouter
104 }
105 dbConn struct {
106 db *sql.DB
@@ -127,6 +128,8 @@ func (c *Collector) Init(context.Context) error {
128 c.mx.xactTimeHist = metrix.NewHistogramWithRangeBuckets(c.XactTimeHistogram)
129 c.mx.queryTimeHist = metrix.NewHistogramWithRangeBuckets(c.QueryTimeHistogram)
130
131 + c.funcRouter = newFuncRouter(c)
132 +
133 return nil
134 }
135
@@ -157,7 +160,10 @@ func (c *Collector) Collect(context.Context) map[string]int64 {
160 return mx
161 }
162
160 -func (c *Collector) Cleanup(context.Context) {
163 +func (c *Collector) Cleanup(ctx context.Context) {
164 + if c.funcRouter != nil {
165 + c.funcRouter.Cleanup(ctx)
166 + }
167 if c.db == nil {
168 return
169 }
src/go/plugin/go.d/collector/postgres/func_router.go new
+64
@@ -0,0 +1,64 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package postgres
4 +
5 +import (
6 + "context"
7 + "fmt"
8 +
9 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
10 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
11 +)
12 +
13 +// funcRouter routes method calls to appropriate function handlers.
14 +type funcRouter struct {
15 + collector *Collector
16 +
17 + handlers map[string]funcapi.MethodHandler
18 +}
19 +
20 +func newFuncRouter(c *Collector) *funcRouter {
21 + r := &funcRouter{
22 + collector: c,
23 + handlers: make(map[string]funcapi.MethodHandler),
24 + }
25 + r.handlers[topQueriesMethodID] = newFuncTopQueries(r)
26 + return r
27 +}
28 +
29 +// Compile-time interface check.
30 +var _ funcapi.MethodHandler = (*funcRouter)(nil)
31 +
32 +func (r *funcRouter) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
33 + if h, ok := r.handlers[method]; ok {
34 + return h.MethodParams(ctx, method)
35 + }
36 + return nil, fmt.Errorf("unknown method: %s", method)
37 +}
38 +
39 +func (r *funcRouter) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
40 + if h, ok := r.handlers[method]; ok {
41 + return h.Handle(ctx, method, params)
42 + }
43 + return funcapi.NotFoundResponse(method)
44 +}
45 +
46 +func (r *funcRouter) Cleanup(ctx context.Context) {
47 + for _, h := range r.handlers {
48 + h.Cleanup(ctx)
49 + }
50 +}
51 +
52 +func pgMethods() []funcapi.MethodConfig {
53 + return []funcapi.MethodConfig{
54 + topQueriesMethodConfig(),
55 + }
56 +}
57 +
58 +func pgFunctionHandler(job *module.Job) funcapi.MethodHandler {
59 + c, ok := job.Module().(*Collector)
60 + if !ok {
61 + return nil
62 + }
63 + return c.funcRouter
64 +}
src/go/plugin/go.d/collector/postgres/func_top_queries.go new
+618
@@ -0,0 +1,618 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package postgres
4 +
5 +import (
6 + "context"
7 + "database/sql"
8 + "fmt"
9 + "strings"
10 +
11 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
12 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
13 +)
14 +
15 +const (
16 + topQueriesMethodID = "top-queries"
17 + maxQueryTextLength = 4096
18 + paramSort = "__sort"
19 +)
20 +
21 +// pgColumn defines metadata for a pg_stat_statements column.
22 +// Embeds funcapi.ColumnMeta for UI rendering and adds PG-specific fields.
23 +type pgColumn struct {
24 + funcapi.ColumnMeta
25 +
26 + // DBColumn is the database column expression (e.g., "s.queryid::text", "d.datname")
27 + DBColumn string
28 + // IsSortOption indicates whether this column appears in the sort dropdown
29 + IsSortOption bool
30 + // SortLabel is the label shown in the sort dropdown (if IsSortOption)
31 + SortLabel string
32 + // IsDefaultSort indicates whether this is the default sort column
33 + IsDefaultSort bool
34 +}
35 +
36 +// pgColumnSet creates a ColumnSet from a slice of pgColumn.
37 +func pgColumnSet(cols []pgColumn) funcapi.ColumnSet[pgColumn] {
38 + return funcapi.Columns(cols, func(c pgColumn) funcapi.ColumnMeta { return c.ColumnMeta })
39 +}
40 +
41 +// pgAllColumns defines ALL possible columns from pg_stat_statements.
42 +// Order matters - this determines column index in the response.
43 +var pgAllColumns = []pgColumn{
44 + // Core identification columns (always present)
45 + {ColumnMeta: funcapi.ColumnMeta{Name: "queryid", Tooltip: "Query ID", Type: funcapi.FieldTypeString, Visible: false, Transform: funcapi.FieldTransformNone, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, UniqueKey: true, Sortable: true}, DBColumn: "s.queryid::text"},
46 + {ColumnMeta: funcapi.ColumnMeta{Name: "query", Tooltip: "Query", Type: funcapi.FieldTypeString, Visible: true, Transform: funcapi.FieldTransformNone, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, Sticky: true, FullWidth: true, Sortable: true}, DBColumn: "s.query"},
47 + {ColumnMeta: funcapi.ColumnMeta{Name: "database", Tooltip: "Database", Type: funcapi.FieldTypeString, Visible: true, Transform: funcapi.FieldTransformNone, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, Sortable: true}, DBColumn: "d.datname"},
48 + {ColumnMeta: funcapi.ColumnMeta{Name: "user", Tooltip: "User", Type: funcapi.FieldTypeString, Visible: true, Transform: funcapi.FieldTransformNone, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, Sortable: true}, DBColumn: "u.usename"},
49 +
50 + // Execution count (always present)
51 + {ColumnMeta: funcapi.ColumnMeta{Name: "calls", Tooltip: "Calls", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.calls", IsSortOption: true, SortLabel: "Number of Calls"},
52 +
53 + // Execution time columns (names vary by version - detected dynamically)
54 + // PG <13: total_time, mean_time, min_time, max_time, stddev_time
55 + // PG 13+: total_exec_time, mean_exec_time, min_exec_time, max_exec_time, stddev_exec_time
56 + {ColumnMeta: funcapi.ColumnMeta{Name: "totalTime", Tooltip: "Total Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: true, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "total_time", IsSortOption: true, SortLabel: "Total Execution Time", IsDefaultSort: true},
57 + {ColumnMeta: funcapi.ColumnMeta{Name: "meanTime", Tooltip: "Mean Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: true, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "mean_time", IsSortOption: true, SortLabel: "Average Execution Time"},
58 + {ColumnMeta: funcapi.ColumnMeta{Name: "minTime", Tooltip: "Min Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: false, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMin, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "min_time"},
59 + {ColumnMeta: funcapi.ColumnMeta{Name: "maxTime", Tooltip: "Max Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: false, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "max_time"},
60 + {ColumnMeta: funcapi.ColumnMeta{Name: "stddevTime", Tooltip: "Stddev Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: false, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "stddev_time"},
61 +
62 + // Planning time columns (PG 13+ only)
63 + {ColumnMeta: funcapi.ColumnMeta{Name: "plans", Tooltip: "Plans", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.plans"},
64 + {ColumnMeta: funcapi.ColumnMeta{Name: "totalPlanTime", Tooltip: "Total Plan Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: false, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "total_plan_time"},
65 + {ColumnMeta: funcapi.ColumnMeta{Name: "meanPlanTime", Tooltip: "Mean Plan Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: false, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "mean_plan_time"},
66 + {ColumnMeta: funcapi.ColumnMeta{Name: "minPlanTime", Tooltip: "Min Plan Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: false, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMin, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "min_plan_time"},
67 + {ColumnMeta: funcapi.ColumnMeta{Name: "maxPlanTime", Tooltip: "Max Plan Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: false, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "max_plan_time"},
68 + {ColumnMeta: funcapi.ColumnMeta{Name: "stddevPlanTime", Tooltip: "Stddev Plan Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: false, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "stddev_plan_time"},
69 +
70 + // Row count (always present)
71 + {ColumnMeta: funcapi.ColumnMeta{Name: "rows", Tooltip: "Rows", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.rows", IsSortOption: true, SortLabel: "Rows Returned"},
72 +
73 + // Shared buffer statistics (always present)
74 + {ColumnMeta: funcapi.ColumnMeta{Name: "sharedBlksHit", Tooltip: "Shared Blocks Hit", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.shared_blks_hit", IsSortOption: true, SortLabel: "Shared Blocks Hit (Cache)"},
75 + {ColumnMeta: funcapi.ColumnMeta{Name: "sharedBlksRead", Tooltip: "Shared Blocks Read", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.shared_blks_read", IsSortOption: true, SortLabel: "Shared Blocks Read (Disk I/O)"},
76 + {ColumnMeta: funcapi.ColumnMeta{Name: "sharedBlksDirtied", Tooltip: "Shared Blocks Dirtied", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.shared_blks_dirtied"},
77 + {ColumnMeta: funcapi.ColumnMeta{Name: "sharedBlksWritten", Tooltip: "Shared Blocks Written", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.shared_blks_written"},
78 +
79 + // Local buffer statistics (always present)
80 + {ColumnMeta: funcapi.ColumnMeta{Name: "localBlksHit", Tooltip: "Local Blocks Hit", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.local_blks_hit"},
81 + {ColumnMeta: funcapi.ColumnMeta{Name: "localBlksRead", Tooltip: "Local Blocks Read", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.local_blks_read"},
82 + {ColumnMeta: funcapi.ColumnMeta{Name: "localBlksDirtied", Tooltip: "Local Blocks Dirtied", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.local_blks_dirtied"},
83 + {ColumnMeta: funcapi.ColumnMeta{Name: "localBlksWritten", Tooltip: "Local Blocks Written", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.local_blks_written"},
84 +
85 + // Temp buffer statistics (always present)
86 + {ColumnMeta: funcapi.ColumnMeta{Name: "tempBlksRead", Tooltip: "Temp Blocks Read", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.temp_blks_read"},
87 + {ColumnMeta: funcapi.ColumnMeta{Name: "tempBlksWritten", Tooltip: "Temp Blocks Written", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.temp_blks_written", IsSortOption: true, SortLabel: "Temp Blocks Written"},
88 +
89 + // I/O timing (requires track_io_timing, always present but may be 0)
90 + {ColumnMeta: funcapi.ColumnMeta{Name: "blkReadTime", Tooltip: "Block Read Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: true, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.blk_read_time"},
91 + {ColumnMeta: funcapi.ColumnMeta{Name: "blkWriteTime", Tooltip: "Block Write Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: true, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.blk_write_time"},
92 +
93 + // WAL statistics (PG 13+ only)
94 + {ColumnMeta: funcapi.ColumnMeta{Name: "walRecords", Tooltip: "WAL Records", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.wal_records"},
95 + {ColumnMeta: funcapi.ColumnMeta{Name: "walFpi", Tooltip: "WAL Full Page Images", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.wal_fpi"},
96 + {ColumnMeta: funcapi.ColumnMeta{Name: "walBytes", Tooltip: "WAL Bytes", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.wal_bytes"},
97 +
98 + // JIT statistics (PG 15+ only)
99 + {ColumnMeta: funcapi.ColumnMeta{Name: "jitFunctions", Tooltip: "JIT Functions", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.jit_functions"},
100 + {ColumnMeta: funcapi.ColumnMeta{Name: "jitGenerationTime", Tooltip: "JIT Generation Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: false, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.jit_generation_time"},
101 + {ColumnMeta: funcapi.ColumnMeta{Name: "jitInliningCount", Tooltip: "JIT Inlining Count", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.jit_inlining_count"},
102 + {ColumnMeta: funcapi.ColumnMeta{Name: "jitInliningTime", Tooltip: "JIT Inlining Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: false, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.jit_inlining_time"},
103 + {ColumnMeta: funcapi.ColumnMeta{Name: "jitOptimizationCount", Tooltip: "JIT Optimization Count", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.jit_optimization_count"},
104 + {ColumnMeta: funcapi.ColumnMeta{Name: "jitOptimizationTime", Tooltip: "JIT Optimization Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: false, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.jit_optimization_time"},
105 + {ColumnMeta: funcapi.ColumnMeta{Name: "jitEmissionCount", Tooltip: "JIT Emission Count", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.jit_emission_count"},
106 + {ColumnMeta: funcapi.ColumnMeta{Name: "jitEmissionTime", Tooltip: "JIT Emission Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: false, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.jit_emission_time"},
107 +
108 + // Temp file statistics (PG 15+ only)
109 + {ColumnMeta: funcapi.ColumnMeta{Name: "tempBlkReadTime", Tooltip: "Temp Block Read Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: false, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.temp_blk_read_time"},
110 + {ColumnMeta: funcapi.ColumnMeta{Name: "tempBlkWriteTime", Tooltip: "Temp Block Write Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: false, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "s.temp_blk_write_time"},
111 +}
112 +
113 +// pgChartGroupDefs defines chart groupings for columns. These are applied at runtime via decoratePgColumns.
114 +var pgChartGroupDefs = []struct {
115 + key string
116 + title string
117 + columns []string
118 + defaultChart bool
119 +}{
120 + {key: "Calls", title: "Number of Calls", columns: []string{"calls"}, defaultChart: true},
121 + {key: "Time", title: "Execution Time", columns: []string{"totalTime", "meanTime", "minTime", "maxTime", "stddevTime"}, defaultChart: true},
122 + {key: "PlanTime", title: "Planning Time", columns: []string{"totalPlanTime", "meanPlanTime", "minPlanTime", "maxPlanTime", "stddevPlanTime"}},
123 + {key: "Plans", title: "Plans", columns: []string{"plans"}},
124 + {key: "Rows", title: "Rows Returned", columns: []string{"rows"}},
125 + {key: "SharedBlocks", title: "Shared Blocks", columns: []string{"sharedBlksHit", "sharedBlksRead", "sharedBlksDirtied", "sharedBlksWritten"}},
126 + {key: "LocalBlocks", title: "Local Blocks", columns: []string{"localBlksHit", "localBlksRead", "localBlksDirtied", "localBlksWritten"}},
127 + {key: "TempBlocks", title: "Temp Blocks", columns: []string{"tempBlksRead", "tempBlksWritten"}},
128 + {key: "IOTime", title: "Block I/O Time", columns: []string{"blkReadTime", "blkWriteTime"}},
129 + {key: "WALRecords", title: "WAL Records", columns: []string{"walRecords", "walFpi"}},
130 + {key: "WALBytes", title: "WAL Bytes", columns: []string{"walBytes"}},
131 + {key: "JITCounts", title: "JIT Counts", columns: []string{"jitFunctions", "jitInliningCount", "jitOptimizationCount", "jitEmissionCount"}},
132 + {key: "JITTime", title: "JIT Time", columns: []string{"jitGenerationTime", "jitInliningTime", "jitOptimizationTime", "jitEmissionTime"}},
133 + {key: "TempIOTime", title: "Temp Block I/O Time", columns: []string{"tempBlkReadTime", "tempBlkWriteTime"}},
134 +}
135 +
136 +// pgLabelColumnIDs defines which columns are available for group-by.
137 +var pgLabelColumnIDs = map[string]bool{
138 + "database": true,
139 + "user": true,
140 +}
141 +
142 +const pgPrimaryLabelID = "database"
143 +
144 +func topQueriesMethodConfig() funcapi.MethodConfig {
145 + return funcapi.MethodConfig{
146 + ID: topQueriesMethodID,
147 + Name: "Top Queries",
148 + UpdateEvery: 10,
149 + Help: "Top SQL queries from pg_stat_statements",
150 + RequireCloud: true,
151 + RequiredParams: []funcapi.ParamConfig{
152 + {
153 + ID: paramSort,
154 + Name: "Filter By",
155 + Help: "Select the primary sort column",
156 + Selection: funcapi.ParamSelect,
157 + Options: buildPgSortOptions(),
158 + UniqueView: true,
159 + },
160 + },
161 + }
162 +}
163 +
164 +// Compile-time interface check.
165 +var _ funcapi.MethodHandler = (*funcTopQueries)(nil)
166 +
167 +// funcTopQueries handles the "top-queries" function for PostgreSQL.
168 +type funcTopQueries struct {
169 + router *funcRouter
170 +}
171 +
172 +func newFuncTopQueries(r *funcRouter) *funcTopQueries {
173 + return &funcTopQueries{router: r}
174 +}
175 +
176 +// MethodParams implements funcapi.MethodHandler.
177 +func (f *funcTopQueries) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
178 + if f.router.collector.db == nil {
179 + return nil, fmt.Errorf("collector is still initializing")
180 + }
181 + return f.topQueriesParams(ctx)
182 +}
183 +
184 +// Handle implements funcapi.MethodHandler.
185 +func (f *funcTopQueries) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
186 + if f.router.collector.db == nil {
187 + return funcapi.UnavailableResponse("collector is still initializing, please retry in a few seconds")
188 + }
189 + return f.collectTopQueries(ctx, params.Column(paramSort))
190 +}
191 +
192 +// buildPgSortOptions builds sort options from pgAllColumns.
193 +func buildPgSortOptions() []funcapi.ParamOption {
194 + var opts []funcapi.ParamOption
195 + sortDir := funcapi.FieldSortDescending
196 + for _, col := range pgAllColumns {
197 + if col.IsSortOption {
198 + opts = append(opts, funcapi.ParamOption{
199 + ID: col.Name,
200 + Column: col.Name,
201 + Name: "Top queries by " + col.SortLabel,
202 + Default: col.IsDefaultSort,
203 + Sort: &sortDir,
204 + })
205 + }
206 + }
207 + return opts
208 +}
209 +
210 +// collectTopQueries queries pg_stat_statements for top queries.
211 +func (f *funcTopQueries) collectTopQueries(ctx context.Context, sortColumn string) *funcapi.FunctionResponse {
212 + c := f.router.collector
213 +
214 + // Check pg_stat_statements availability (lazy check)
215 + available, err := c.checkPgStatStatements(ctx)
216 + if err != nil {
217 + return funcapi.InternalErrorResponse("failed to check pg_stat_statements availability: %v", err)
218 + }
219 + if !available {
220 + return funcapi.UnavailableResponse("pg_stat_statements extension is not installed in this database. " +
221 + "Run 'CREATE EXTENSION pg_stat_statements;' in the database the collector connects to.")
222 + }
223 +
224 + // Detect available columns (lazy detection, cached)
225 + availableCols, err := c.detectPgStatStatementsColumns(ctx)
226 + if err != nil {
227 + return funcapi.InternalErrorResponse("failed to detect available columns: %v", err)
228 + }
229 +
230 + // Build list of columns to query based on what's available
231 + queryCols := f.buildAvailableColumns(availableCols)
232 + if len(queryCols) == 0 {
233 + return funcapi.InternalErrorResponse("no queryable columns found in pg_stat_statements")
234 + }
235 +
236 + // Map and validate sort column
237 + actualSortCol := f.mapAndValidateSortColumn(sortColumn, availableCols)
238 +
239 + // Get query limit (default 500)
240 + limit := c.TopQueriesLimit
241 + if limit <= 0 {
242 + limit = 500
243 + }
244 +
245 + // Build and execute query
246 + query := f.buildDynamicSQL(queryCols, actualSortCol, limit)
247 + rows, err := c.db.QueryContext(ctx, query)
248 + if err != nil {
249 + if ctx.Err() == context.DeadlineExceeded {
250 + return funcapi.ErrorResponse(504, "query timed out")
251 + }
252 + return funcapi.InternalErrorResponse("query failed: %v", err)
253 + }
254 + defer rows.Close()
255 +
256 + // Process rows and build response
257 + data, err := f.scanDynamicRows(rows, queryCols)
258 + if err != nil {
259 + return funcapi.InternalErrorResponse("%s", err)
260 + }
261 +
262 + if err := rows.Err(); err != nil {
263 + return funcapi.InternalErrorResponse("rows iteration error: %v", err)
264 + }
265 +
266 + // Build dynamic sort options from available columns (only those actually detected)
267 + sortParam, sortOptions := f.topQueriesSortParam(queryCols)
268 +
269 + // Find default sort column from metadata
270 + defaultSort := ""
271 + for _, col := range queryCols {
272 + if col.IsDefaultSort && col.IsSortOption {
273 + defaultSort = col.Name
274 + break
275 + }
276 + }
277 + // Fallback to first sort option if no default
278 + if defaultSort == "" && len(sortOptions) > 0 {
279 + defaultSort = sortOptions[0].ID
280 + }
281 +
282 + // Decorate columns with chart/label metadata and build using ColumnSet
283 + annotatedCols := decoratePgColumns(queryCols)
284 + cs := pgColumnSet(annotatedCols)
285 +
286 + return &funcapi.FunctionResponse{
287 + Status: 200,
288 + Help: "Top SQL queries from pg_stat_statements",
289 + Columns: cs.BuildColumns(),
290 + Data: data,
291 + DefaultSortColumn: defaultSort,
292 + RequiredParams: []funcapi.ParamConfig{sortParam},
293 + ChartingConfig: cs.BuildCharting(),
294 + }
295 +}
296 +
297 +// decoratePgColumns adds label and chart metadata to columns for ColumnSet builders.
298 +func decoratePgColumns(cols []pgColumn) []pgColumn {
299 + out := make([]pgColumn, len(cols))
300 + index := make(map[string]int, len(cols))
301 + for i, col := range cols {
302 + out[i] = col
303 + index[col.Name] = i
304 + }
305 +
306 + // Mark groupby columns
307 + for i := range out {
308 + if pgLabelColumnIDs[out[i].Name] {
309 + out[i].GroupBy = &funcapi.GroupByOptions{
310 + IsDefault: out[i].Name == pgPrimaryLabelID,
311 + }
312 + }
313 + }
314 +
315 + // Mark chart columns
316 + for _, group := range pgChartGroupDefs {
317 + for _, key := range group.columns {
318 + idx, ok := index[key]
319 + if !ok {
320 + continue
321 + }
322 + out[idx].Chart = &funcapi.ChartOptions{
323 + Group: group.key,
324 + Title: group.title,
325 + IsDefault: group.defaultChart,
326 + }
327 + }
328 + }
329 +
330 + return out
331 +}
332 +
333 +// buildAvailableColumns returns column metadata for columns that exist in this PG version.
334 +func (f *funcTopQueries) buildAvailableColumns(availableCols map[string]bool) []pgColumn {
335 + c := f.router.collector
336 + var result []pgColumn
337 +
338 + for _, col := range pgAllColumns {
339 + // Extract the actual column name (remove table prefix and type cast)
340 + colName := col.DBColumn
341 + if idx := strings.LastIndex(colName, "."); idx != -1 {
342 + colName = colName[idx+1:]
343 + }
344 + // Remove PostgreSQL type cast suffix (e.g., "::text")
345 + if idx := strings.Index(colName, "::"); idx != -1 {
346 + colName = colName[:idx]
347 + }
348 +
349 + // Handle version-specific column names for time columns
350 + // PG 13+ renamed time columns: total_time -> total_exec_time, etc.
351 + actualColName := colName
352 + if c.pgVersion >= pgVersion13 {
353 + switch colName {
354 + case "total_time":
355 + actualColName = "total_exec_time"
356 + case "mean_time":
357 + actualColName = "mean_exec_time"
358 + case "min_time":
359 + actualColName = "min_exec_time"
360 + case "max_time":
361 + actualColName = "max_exec_time"
362 + case "stddev_time":
363 + actualColName = "stddev_exec_time"
364 + }
365 + }
366 +
367 + // Check if column exists (either directly or via join)
368 + // Join columns (database, user) come from other tables (d.datname, u.usename)
369 + isJoinCol := col.Name == "database" || col.Name == "user"
370 + if isJoinCol || availableCols[actualColName] {
371 + // Create a copy with the actual column name for this version
372 + colCopy := col
373 + if actualColName != colName {
374 + // Update DBColumn to use the version-specific name with alias
375 + if strings.HasPrefix(col.DBColumn, "s.") {
376 + colCopy.DBColumn = "s." + actualColName
377 + }
378 + }
379 + result = append(result, colCopy)
380 + }
381 + }
382 +
383 + return result
384 +}
385 +
386 +// mapAndValidateSortColumn maps the semantic sort column to actual SQL column.
387 +func (f *funcTopQueries) mapAndValidateSortColumn(sortColumn string, availableCols map[string]bool) string {
388 + c := f.router.collector
389 +
390 + // Map column ID back to DBColumn
391 + for _, col := range pgAllColumns {
392 + if col.Name == sortColumn || col.DBColumn == sortColumn {
393 + // Get actual column name (strip table prefix and type cast)
394 + colName := col.DBColumn
395 + if idx := strings.LastIndex(colName, "."); idx != -1 {
396 + colName = colName[idx+1:]
397 + }
398 + if idx := strings.Index(colName, "::"); idx != -1 {
399 + colName = colName[:idx]
400 + }
401 +
402 + // Handle version-specific mapping
403 + if c.pgVersion >= pgVersion13 {
404 + switch colName {
405 + case "total_time":
406 + colName = "total_exec_time"
407 + case "mean_time":
408 + colName = "mean_exec_time"
409 + case "min_time":
410 + colName = "min_exec_time"
411 + case "max_time":
412 + colName = "max_exec_time"
413 + case "stddev_time":
414 + colName = "stddev_exec_time"
415 + }
416 + }
417 +
418 + // Validate column exists
419 + if availableCols[colName] {
420 + return colName
421 + }
422 + }
423 + }
424 +
425 + // Default fallback
426 + if c.pgVersion >= pgVersion13 {
427 + return "total_exec_time"
428 + }
429 + return "total_time"
430 +}
431 +
432 +// buildDynamicSQL builds the SQL query with only available columns.
433 +func (f *funcTopQueries) buildDynamicSQL(cols []pgColumn, sortColumn string, limit int) string {
434 + c := f.router.collector
435 + var selectCols []string
436 +
437 + for _, col := range cols {
438 + colExpr := col.DBColumn
439 +
440 + // Handle version-specific column names
441 + if c.pgVersion >= pgVersion13 {
442 + switch {
443 + case strings.HasSuffix(colExpr, ".total_time"):
444 + colExpr = strings.Replace(colExpr, ".total_time", ".total_exec_time", 1)
445 + case strings.HasSuffix(colExpr, ".mean_time"):
446 + colExpr = strings.Replace(colExpr, ".mean_time", ".mean_exec_time", 1)
447 + case strings.HasSuffix(colExpr, ".min_time"):
448 + colExpr = strings.Replace(colExpr, ".min_time", ".min_exec_time", 1)
449 + case strings.HasSuffix(colExpr, ".max_time"):
450 + colExpr = strings.Replace(colExpr, ".max_time", ".max_exec_time", 1)
451 + case strings.HasSuffix(colExpr, ".stddev_time"):
452 + colExpr = strings.Replace(colExpr, ".stddev_time", ".stddev_exec_time", 1)
453 + case colExpr == "total_time":
454 + colExpr = "total_exec_time"
455 + case colExpr == "mean_time":
456 + colExpr = "mean_exec_time"
457 + case colExpr == "min_time":
458 + colExpr = "min_exec_time"
459 + case colExpr == "max_time":
460 + colExpr = "max_exec_time"
461 + case colExpr == "stddev_time":
462 + colExpr = "stddev_exec_time"
463 + }
464 + }
465 +
466 + // Use column ID as the SQL alias for consistent naming
467 + // Use double quotes to handle reserved keywords like "database", "user"
468 + selectCols = append(selectCols, fmt.Sprintf("%s AS \"%s\"", colExpr, col.Name))
469 + }
470 +
471 + return fmt.Sprintf(`
472 +SELECT %s
473 +FROM pg_stat_statements s
474 +JOIN pg_database d ON s.dbid = d.oid
475 +JOIN pg_user u ON s.userid = u.usesysid
476 +ORDER BY "%s" DESC
477 +LIMIT %d
478 +`, strings.Join(selectCols, ", "), sortColumn, limit)
479 +}
480 +
481 +// scanDynamicRows scans rows into the data array based on column types.
482 +// Uses sql.Null* types to handle NULL values safely.
483 +func (f *funcTopQueries) scanDynamicRows(rows dbRows, cols []pgColumn) ([][]any, error) {
484 + data := make([][]any, 0, 500)
485 +
486 + // Create value holders for scanning (reuse across rows for efficiency)
487 + valuePtrs := make([]any, len(cols))
488 + values := make([]any, len(cols))
489 +
490 + for rows.Next() {
491 + // Reset value holders for each row
492 + for i, col := range cols {
493 + switch col.Type {
494 + case funcapi.FieldTypeString:
495 + var v sql.NullString
496 + values[i] = &v
497 + case funcapi.FieldTypeInteger:
498 + var v sql.NullInt64
499 + values[i] = &v
500 + case funcapi.FieldTypeFloat, funcapi.FieldTypeDuration:
501 + var v sql.NullFloat64
502 + values[i] = &v
503 + default:
504 + var v sql.NullString
505 + values[i] = &v
506 + }
507 + valuePtrs[i] = values[i]
508 + }
509 +
510 + if err := rows.Scan(valuePtrs...); err != nil {
511 + return nil, fmt.Errorf("row scan failed: %v", err)
512 + }
513 +
514 + // Convert scanned values to output format
515 + row := make([]any, len(cols))
516 + for i, col := range cols {
517 + switch v := values[i].(type) {
518 + case *sql.NullString:
519 + if v.Valid {
520 + s := v.String
521 + if col.Name == "query" {
522 + row[i] = strmutil.TruncateText(s, maxQueryTextLength)
523 + } else {
524 + row[i] = s
525 + }
526 + } else {
527 + row[i] = ""
528 + }
529 + case *sql.NullInt64:
530 + if v.Valid {
531 + row[i] = v.Int64
532 + } else {
533 + row[i] = int64(0)
534 + }
535 + case *sql.NullFloat64:
536 + if v.Valid {
537 + row[i] = v.Float64
538 + } else {
539 + row[i] = float64(0)
540 + }
541 + }
542 + }
543 +
544 + data = append(data, row)
545 + }
546 +
547 + return data, nil
548 +}
549 +
550 +// buildDynamicSortOptions builds sort options from available columns.
551 +// Returns only sort options for columns that actually exist in the database.
552 +func (f *funcTopQueries) buildDynamicSortOptions(cols []pgColumn) []funcapi.ParamOption {
553 + var sortOpts []funcapi.ParamOption
554 + seen := make(map[string]bool)
555 + sortDir := funcapi.FieldSortDescending
556 +
557 + for _, col := range cols {
558 + if col.IsSortOption && !seen[col.Name] {
559 + seen[col.Name] = true
560 + sortOpts = append(sortOpts, funcapi.ParamOption{
561 + ID: col.Name,
562 + Column: col.Name,
563 + Name: col.SortLabel,
564 + Default: col.IsDefaultSort,
565 + Sort: &sortDir,
566 + })
567 + }
568 + }
569 + return sortOpts
570 +}
571 +
572 +func (f *funcTopQueries) topQueriesSortParam(queryCols []pgColumn) (funcapi.ParamConfig, []funcapi.ParamOption) {
573 + sortOptions := f.buildDynamicSortOptions(queryCols)
574 + sortParam := funcapi.ParamConfig{
575 + ID: paramSort,
576 + Name: "Filter By",
577 + Help: "Select the primary sort column",
578 + Selection: funcapi.ParamSelect,
579 + Options: sortOptions,
580 + UniqueView: true,
581 + }
582 + return sortParam, sortOptions
583 +}
584 +
585 +func (f *funcTopQueries) topQueriesParams(ctx context.Context) ([]funcapi.ParamConfig, error) {
586 + c := f.router.collector
587 +
588 + available, err := c.checkPgStatStatements(ctx)
589 + if err != nil {
590 + return nil, err
591 + }
592 + if !available {
593 + return nil, fmt.Errorf("pg_stat_statements extension is not installed")
594 + }
595 +
596 + availableCols, err := c.detectPgStatStatementsColumns(ctx)
597 + if err != nil {
598 + return nil, err
599 + }
600 +
601 + queryCols := f.buildAvailableColumns(availableCols)
602 + if len(queryCols) == 0 {
603 + return nil, fmt.Errorf("no queryable columns found in pg_stat_statements")
604 + }
605 +
606 + sortParam, _ := f.topQueriesSortParam(queryCols)
607 + return []funcapi.ParamConfig{sortParam}, nil
608 +}
609 +
610 +// Cleanup implements funcapi.MethodHandler.
611 +func (f *funcTopQueries) Cleanup(ctx context.Context) {}
612 +
613 +// dbRows interface for testing
614 +type dbRows interface {
615 + Next() bool
616 + Scan(dest ...any) error
617 + Err() error
618 +}
src/go/plugin/go.d/collector/postgres/functions.go
-805
@@ -4,496 +4,9 @@ package postgres
4
5 import (
6 "context"
7 - "database/sql"
7 "fmt"
9 - "strings"
10 -
11 - "github.com/netdata/netdata/go/plugins/pkg/funcapi"
12 - "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
13 - "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
8 )
9
16 -const maxQueryTextLength = 4096
17 -
18 -const (
19 - paramSort = "__sort"
20 -
21 - ftString = funcapi.FieldTypeString
22 - ftInteger = funcapi.FieldTypeInteger
23 - ftFloat = funcapi.FieldTypeFloat
24 - ftDuration = funcapi.FieldTypeDuration
25 -
26 - trNone = funcapi.FieldTransformNone
27 - trNumber = funcapi.FieldTransformNumber
28 - trDuration = funcapi.FieldTransformDuration
29 -
30 - sortAsc = funcapi.FieldSortAscending
31 - sortDesc = funcapi.FieldSortDescending
32 -
33 - summaryCount = funcapi.FieldSummaryCount
34 - summarySum = funcapi.FieldSummarySum
35 - summaryMin = funcapi.FieldSummaryMin
36 - summaryMax = funcapi.FieldSummaryMax
37 - summaryMean = funcapi.FieldSummaryMean
38 - summaryMedian = funcapi.FieldSummaryMedian
39 -
40 - filterMulti = funcapi.FieldFilterMultiselect
41 - filterRange = funcapi.FieldFilterRange
42 -)
43 -
44 -// pgColumnMeta defines metadata for a pg_stat_statements column
45 -type pgColumnMeta struct {
46 - // Database column name (may vary by version)
47 - dbColumn string
48 - // Canonical name used everywhere: SQL alias, UI key, sort key
49 - uiKey string
50 - // Display name in UI
51 - displayName string
52 - // Data type: "string", "integer", "float", "duration"
53 - dataType funcapi.FieldType
54 - // Unit for duration/numeric types
55 - units string
56 - // Whether visible by default
57 - visible bool
58 - // Transform for value_options
59 - transform funcapi.FieldTransform
60 - // Decimal points for display
61 - decimalPoints int
62 - // Sort direction preference
63 - sortDir funcapi.FieldSort
64 - // Summary function
65 - summary funcapi.FieldSummary
66 - // Filter type
67 - filter funcapi.FieldFilter
68 - // Whether this is a sortable option for the sort dropdown
69 - isSortOption bool
70 - // Sort option label (if isSortOption)
71 - sortLabel string
72 - // Whether this is the default sort
73 - isDefaultSort bool
74 - // Whether this is the unique key
75 - isUniqueKey bool
76 - // Whether this column is sticky (stays visible when scrolling)
77 - isSticky bool
78 - // Whether this column should take full width
79 - fullWidth bool
80 - // Whether this column is a label for grouping
81 - isLabel bool
82 - // Whether this label is the primary grouping
83 - isPrimary bool
84 - // Whether this column is a chartable metric
85 - isMetric bool
86 - // Chart group key
87 - chartGroup string
88 - // Chart title
89 - chartTitle string
90 - // Include this chart group in defaults
91 - isDefaultChart bool
92 -}
93 -
94 -// pgAllColumns defines ALL possible columns from pg_stat_statements
95 -// Order matters - this determines column index in the response
96 -var pgAllColumns = []pgColumnMeta{
97 - // Core identification columns (always present)
98 - {dbColumn: "s.queryid::text", uiKey: "queryid", displayName: "Query ID", dataType: ftString, visible: false, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti, isUniqueKey: true},
99 - {dbColumn: "s.query", uiKey: "query", displayName: "Query", dataType: ftString, visible: true, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti, isSticky: true, fullWidth: true},
100 - {dbColumn: "d.datname", uiKey: "database", displayName: "Database", dataType: ftString, visible: true, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti},
101 - {dbColumn: "u.usename", uiKey: "user", displayName: "User", dataType: ftString, visible: true, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti},
102 -
103 - // Execution count (always present)
104 - {dbColumn: "s.calls", uiKey: "calls", displayName: "Calls", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Number of Calls"},
105 -
106 - // Execution time columns (names vary by version - detected dynamically)
107 - // PG <13: total_time, mean_time, min_time, max_time, stddev_time
108 - // PG 13+: total_exec_time, mean_exec_time, min_exec_time, max_exec_time, stddev_exec_time
109 - {dbColumn: "total_time", uiKey: "totalTime", displayName: "Total Time", dataType: ftDuration, units: "milliseconds", visible: true, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Total Execution Time", isDefaultSort: true},
110 - {dbColumn: "mean_time", uiKey: "meanTime", displayName: "Mean Time", dataType: ftDuration, units: "milliseconds", visible: true, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, filter: filterRange, isSortOption: true, sortLabel: "Average Execution Time"},
111 - {dbColumn: "min_time", uiKey: "minTime", displayName: "Min Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMin, filter: filterRange},
112 - {dbColumn: "max_time", uiKey: "maxTime", displayName: "Max Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
113 - {dbColumn: "stddev_time", uiKey: "stddevTime", displayName: "Stddev Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
114 -
115 - // Planning time columns (PG 13+ only)
116 - {dbColumn: "s.plans", uiKey: "plans", displayName: "Plans", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange},
117 - {dbColumn: "total_plan_time", uiKey: "totalPlanTime", displayName: "Total Plan Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summarySum, filter: filterRange},
118 - {dbColumn: "mean_plan_time", uiKey: "meanPlanTime", displayName: "Mean Plan Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
119 - {dbColumn: "min_plan_time", uiKey: "minPlanTime", displayName: "Min Plan Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMin, filter: filterRange},
120 - {dbColumn: "max_plan_time", uiKey: "maxPlanTime", displayName: "Max Plan Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
121 - {dbColumn: "stddev_plan_time", uiKey: "stddevPlanTime", displayName: "Stddev Plan Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, filter: filterRange},
122 -
123 - // Row count (always present)
124 - {dbColumn: "s.rows", uiKey: "rows", displayName: "Rows", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Rows Returned"},
125 -
126 - // Shared buffer statistics (always present)
127 - {dbColumn: "s.shared_blks_hit", uiKey: "sharedBlksHit", displayName: "Shared Blocks Hit", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Shared Blocks Hit (Cache)"},
128 - {dbColumn: "s.shared_blks_read", uiKey: "sharedBlksRead", displayName: "Shared Blocks Read", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Shared Blocks Read (Disk I/O)"},
129 - {dbColumn: "s.shared_blks_dirtied", uiKey: "sharedBlksDirtied", displayName: "Shared Blocks Dirtied", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange},
130 - {dbColumn: "s.shared_blks_written", uiKey: "sharedBlksWritten", displayName: "Shared Blocks Written", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange},
131 -
132 - // Local buffer statistics (always present)
133 - {dbColumn: "s.local_blks_hit", uiKey: "localBlksHit", displayName: "Local Blocks Hit", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange},
134 - {dbColumn: "s.local_blks_read", uiKey: "localBlksRead", displayName: "Local Blocks Read", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange},
135 - {dbColumn: "s.local_blks_dirtied", uiKey: "localBlksDirtied", displayName: "Local Blocks Dirtied", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange},
136 - {dbColumn: "s.local_blks_written", uiKey: "localBlksWritten", displayName: "Local Blocks Written", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange},
137 -
138 - // Temp buffer statistics (always present)
139 - {dbColumn: "s.temp_blks_read", uiKey: "tempBlksRead", displayName: "Temp Blocks Read", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange},
140 - {dbColumn: "s.temp_blks_written", uiKey: "tempBlksWritten", displayName: "Temp Blocks Written", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Temp Blocks Written"},
141 -
142 - // I/O timing (requires track_io_timing, always present but may be 0)
143 - {dbColumn: "s.blk_read_time", uiKey: "blkReadTime", displayName: "Block Read Time", dataType: ftDuration, units: "milliseconds", visible: true, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summarySum, filter: filterRange},
144 - {dbColumn: "s.blk_write_time", uiKey: "blkWriteTime", displayName: "Block Write Time", dataType: ftDuration, units: "milliseconds", visible: true, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summarySum, filter: filterRange},
145 -
146 - // WAL statistics (PG 13+ only)
147 - {dbColumn: "s.wal_records", uiKey: "walRecords", displayName: "WAL Records", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange},
148 - {dbColumn: "s.wal_fpi", uiKey: "walFpi", displayName: "WAL Full Page Images", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange},
149 - {dbColumn: "s.wal_bytes", uiKey: "walBytes", displayName: "WAL Bytes", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange},
150 -
151 - // JIT statistics (PG 15+ only)
152 - {dbColumn: "s.jit_functions", uiKey: "jitFunctions", displayName: "JIT Functions", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange},
153 - {dbColumn: "s.jit_generation_time", uiKey: "jitGenerationTime", displayName: "JIT Generation Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summarySum, filter: filterRange},
154 - {dbColumn: "s.jit_inlining_count", uiKey: "jitInliningCount", displayName: "JIT Inlining Count", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange},
155 - {dbColumn: "s.jit_inlining_time", uiKey: "jitInliningTime", displayName: "JIT Inlining Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summarySum, filter: filterRange},
156 - {dbColumn: "s.jit_optimization_count", uiKey: "jitOptimizationCount", displayName: "JIT Optimization Count", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange},
157 - {dbColumn: "s.jit_optimization_time", uiKey: "jitOptimizationTime", displayName: "JIT Optimization Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summarySum, filter: filterRange},
158 - {dbColumn: "s.jit_emission_count", uiKey: "jitEmissionCount", displayName: "JIT Emission Count", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange},
159 - {dbColumn: "s.jit_emission_time", uiKey: "jitEmissionTime", displayName: "JIT Emission Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summarySum, filter: filterRange},
160 -
161 - // Temp file statistics (PG 15+ only)
162 - {dbColumn: "s.temp_blk_read_time", uiKey: "tempBlkReadTime", displayName: "Temp Block Read Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summarySum, filter: filterRange},
163 - {dbColumn: "s.temp_blk_write_time", uiKey: "tempBlkWriteTime", displayName: "Temp Block Write Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summarySum, filter: filterRange},
164 -}
165 -
166 -type pgChartGroup struct {
167 - key string
168 - title string
169 - columns []string
170 - defaultChart bool
171 -}
172 -
173 -var pgChartGroups = []pgChartGroup{
174 - {key: "Calls", title: "Number of Calls", columns: []string{"calls"}, defaultChart: true},
175 - {key: "Time", title: "Execution Time", columns: []string{"totalTime", "meanTime", "minTime", "maxTime", "stddevTime"}, defaultChart: true},
176 - {key: "PlanTime", title: "Planning Time", columns: []string{"totalPlanTime", "meanPlanTime", "minPlanTime", "maxPlanTime", "stddevPlanTime"}},
177 - {key: "Plans", title: "Plans", columns: []string{"plans"}},
178 - {key: "Rows", title: "Rows Returned", columns: []string{"rows"}},
179 - {key: "SharedBlocks", title: "Shared Blocks", columns: []string{"sharedBlksHit", "sharedBlksRead", "sharedBlksDirtied", "sharedBlksWritten"}},
180 - {key: "LocalBlocks", title: "Local Blocks", columns: []string{"localBlksHit", "localBlksRead", "localBlksDirtied", "localBlksWritten"}},
181 - {key: "TempBlocks", title: "Temp Blocks", columns: []string{"tempBlksRead", "tempBlksWritten"}},
182 - {key: "IOTime", title: "Block I/O Time", columns: []string{"blkReadTime", "blkWriteTime"}},
183 - {key: "WALRecords", title: "WAL Records", columns: []string{"walRecords", "walFpi"}},
184 - {key: "WALBytes", title: "WAL Bytes", columns: []string{"walBytes"}},
185 - {key: "JITCounts", title: "JIT Counts", columns: []string{"jitFunctions", "jitInliningCount", "jitOptimizationCount", "jitEmissionCount"}},
186 - {key: "JITTime", title: "JIT Time", columns: []string{"jitGenerationTime", "jitInliningTime", "jitOptimizationTime", "jitEmissionTime"}},
187 - {key: "TempIOTime", title: "Temp Block I/O Time", columns: []string{"tempBlkReadTime", "tempBlkWriteTime"}},
188 -}
189 -
190 -var pgLabelColumns = map[string]bool{
191 - "database": true,
192 - "user": true,
193 -}
194 -
195 -const pgPrimaryLabel = "database"
196 -
197 -// pgMethods returns the available function methods for PostgreSQL
198 -// Sort options are built dynamically based on available columns
199 -func pgMethods() []module.MethodConfig {
200 - // Build sort options from column metadata
201 - var sortOptions []funcapi.ParamOption
202 - sortDir := funcapi.FieldSortDescending
203 - for _, col := range pgAllColumns {
204 - if col.isSortOption {
205 - sortOptions = append(sortOptions, funcapi.ParamOption{
206 - ID: col.uiKey,
207 - Column: col.uiKey,
208 - Name: "Top queries by " + col.sortLabel,
209 - Default: col.isDefaultSort,
210 - Sort: &sortDir,
211 - })
212 - }
213 - }
214 -
215 - return []module.MethodConfig{
216 - {
217 - UpdateEvery: 10,
218 - ID: "top-queries",
219 - Name: "Top Queries",
220 - Help: "Top SQL queries from pg_stat_statements",
221 - RequireCloud: true,
222 - RequiredParams: []funcapi.ParamConfig{
223 - {
224 - ID: paramSort,
225 - Name: "Filter By",
226 - Help: "Select the primary sort column",
227 - Selection: funcapi.ParamSelect,
228 - Options: sortOptions,
229 - UniqueView: true,
230 - },
231 - },
232 - },
233 - }
234 -}
235 -
236 -func pgMethodParams(ctx context.Context, job *module.Job, method string) ([]funcapi.ParamConfig, error) {
237 - collector, ok := job.Module().(*Collector)
238 - if !ok {
239 - return nil, fmt.Errorf("invalid module type")
240 - }
241 - if collector.db == nil {
242 - return nil, fmt.Errorf("collector is still initializing")
243 - }
244 - switch method {
245 - case "top-queries":
246 - return collector.topQueriesParams(ctx)
247 - default:
248 - return nil, fmt.Errorf("unknown method: %s", method)
249 - }
250 -}
251 -
252 -// pgHandleMethod handles function requests for PostgreSQL
253 -func pgHandleMethod(ctx context.Context, job *module.Job, method string, params funcapi.ResolvedParams) *module.FunctionResponse {
254 - collector, ok := job.Module().(*Collector)
255 - if !ok {
256 - return &module.FunctionResponse{Status: 500, Message: "internal error: invalid module type"}
257 - }
258 -
259 - // Check if collector is initialized (first collect() may not have run yet)
260 - if collector.db == nil {
261 - return &module.FunctionResponse{
262 - Status: 503,
263 - Message: "collector is still initializing, please retry in a few seconds",
264 - }
265 - }
266 -
267 - switch method {
268 - case "top-queries":
269 - return collector.collectTopQueries(ctx, params.Column(paramSort))
270 - default:
271 - return &module.FunctionResponse{Status: 404, Message: fmt.Sprintf("unknown method: %s", method)}
272 - }
273 -}
274 -
275 -// collectTopQueries queries pg_stat_statements for top queries
276 -func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
277 - // Check pg_stat_statements availability (lazy check)
278 - available, err := c.checkPgStatStatements(ctx)
279 - if err != nil {
280 - return &module.FunctionResponse{
281 - Status: 500,
282 - Message: fmt.Sprintf("failed to check pg_stat_statements availability: %v", err),
283 - }
284 - }
285 - if !available {
286 - return &module.FunctionResponse{
287 - Status: 503,
288 - Message: "pg_stat_statements extension is not installed in this database. " +
289 - "Run 'CREATE EXTENSION pg_stat_statements;' in the database the collector connects to.",
290 - }
291 - }
292 -
293 - // Detect available columns (lazy detection, cached)
294 - availableCols, err := c.detectPgStatStatementsColumns(ctx)
295 - if err != nil {
296 - return &module.FunctionResponse{
297 - Status: 500,
298 - Message: fmt.Sprintf("failed to detect available columns: %v", err),
299 - }
300 - }
301 -
302 - // Build list of columns to query based on what's available
303 - queryCols := c.buildAvailableColumns(availableCols)
304 - if len(queryCols) == 0 {
305 - return &module.FunctionResponse{
306 - Status: 500,
307 - Message: "no queryable columns found in pg_stat_statements",
308 - }
309 - }
310 -
311 - // Map and validate sort column
312 - actualSortCol := c.mapAndValidateSortColumn(sortColumn, availableCols)
313 -
314 - // Get query limit (default 500)
315 - limit := c.TopQueriesLimit
316 - if limit <= 0 {
317 - limit = 500
318 - }
319 -
320 - // Build and execute query
321 - query := c.buildDynamicSQL(queryCols, actualSortCol, limit)
322 - rows, err := c.db.QueryContext(ctx, query)
323 - if err != nil {
324 - if ctx.Err() == context.DeadlineExceeded {
325 - return &module.FunctionResponse{Status: 504, Message: "query timed out"}
326 - }
327 - return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("query failed: %v", err)}
328 - }
329 - defer rows.Close()
330 -
331 - // Process rows and build response
332 - data, err := c.scanDynamicRows(rows, queryCols)
333 - if err != nil {
334 - return &module.FunctionResponse{Status: 500, Message: err.Error()}
335 - }
336 -
337 - if err := rows.Err(); err != nil {
338 - return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("rows iteration error: %v", err)}
339 - }
340 -
341 - // Build dynamic sort options from available columns (only those actually detected)
342 - sortParam, sortOptions := c.topQueriesSortParam(queryCols)
343 -
344 - // Find default sort column UI key from metadata
345 - defaultSort := ""
346 - for _, col := range queryCols {
347 - if col.isDefaultSort && col.isSortOption {
348 - defaultSort = col.uiKey
349 - break
350 - }
351 - }
352 - // Fallback to first sort option if no default
353 - if defaultSort == "" && len(sortOptions) > 0 {
354 - defaultSort = sortOptions[0].ID
355 - }
356 -
357 - annotatedCols := decoratePgColumns(queryCols)
358 -
359 - return &module.FunctionResponse{
360 - Status: 200,
361 - Help: "Top SQL queries from pg_stat_statements",
362 - Columns: c.buildDynamicColumns(queryCols),
363 - Data: data,
364 - DefaultSortColumn: defaultSort,
365 - RequiredParams: []funcapi.ParamConfig{sortParam},
366 -
367 - // Charts for aggregated visualization
368 - Charts: pgTopQueriesCharts(annotatedCols),
369 - DefaultCharts: pgTopQueriesDefaultCharts(annotatedCols),
370 - GroupBy: pgTopQueriesGroupBy(annotatedCols),
371 - }
372 -}
373 -
374 -func decoratePgColumns(cols []pgColumnMeta) []pgColumnMeta {
375 - out := make([]pgColumnMeta, len(cols))
376 - index := make(map[string]int, len(cols))
377 - for i, col := range cols {
378 - out[i] = col
379 - index[col.uiKey] = i
380 - }
381 -
382 - for i := range out {
383 - if pgLabelColumns[out[i].uiKey] {
384 - out[i].isLabel = true
385 - if out[i].uiKey == pgPrimaryLabel {
386 - out[i].isPrimary = true
387 - }
388 - }
389 - }
390 -
391 - for _, group := range pgChartGroups {
392 - for _, key := range group.columns {
393 - idx, ok := index[key]
394 - if !ok {
395 - continue
396 - }
397 - out[idx].isMetric = true
398 - out[idx].chartGroup = group.key
399 - out[idx].chartTitle = group.title
400 - if group.defaultChart {
401 - out[idx].isDefaultChart = true
402 - }
403 - }
404 - }
405 -
406 - return out
407 -}
408 -
409 -func pgTopQueriesCharts(cols []pgColumnMeta) map[string]module.ChartConfig {
410 - charts := make(map[string]module.ChartConfig)
411 - for _, col := range cols {
412 - if !col.isMetric || col.chartGroup == "" {
413 - continue
414 - }
415 - cfg, ok := charts[col.chartGroup]
416 - if !ok {
417 - title := col.chartTitle
418 - if title == "" {
419 - title = col.chartGroup
420 - }
421 - cfg = module.ChartConfig{Name: title, Type: "stacked-bar"}
422 - }
423 - cfg.Columns = append(cfg.Columns, col.uiKey)
424 - charts[col.chartGroup] = cfg
425 - }
426 - return charts
427 -}
428 -
429 -func pgTopQueriesDefaultCharts(cols []pgColumnMeta) [][]string {
430 - label := primaryPgLabel(cols)
431 - if label == "" {
432 - return nil
433 - }
434 - chartGroups := defaultPgChartGroups(cols)
435 - out := make([][]string, 0, len(chartGroups))
436 - for _, group := range chartGroups {
437 - out = append(out, []string{group, label})
438 - }
439 - return out
440 -}
441 -
442 -func pgTopQueriesGroupBy(cols []pgColumnMeta) map[string]module.GroupByConfig {
443 - groupBy := make(map[string]module.GroupByConfig)
444 - for _, col := range cols {
445 - if !col.isLabel {
446 - continue
447 - }
448 - groupBy[col.uiKey] = module.GroupByConfig{
449 - Name: "Group by " + col.displayName,
450 - Columns: []string{col.uiKey},
451 - }
452 - }
453 - return groupBy
454 -}
455 -
456 -func primaryPgLabel(cols []pgColumnMeta) string {
457 - for _, col := range cols {
458 - if col.isPrimary {
459 - return col.uiKey
460 - }
461 - }
462 - for _, col := range cols {
463 - if col.isLabel {
464 - return col.uiKey
465 - }
466 - }
467 - return ""
468 -}
469 -
470 -func defaultPgChartGroups(cols []pgColumnMeta) []string {
471 - groups := make([]string, 0)
472 - seen := make(map[string]bool)
473 - for _, col := range cols {
474 - if !col.isMetric || col.chartGroup == "" || !col.isDefaultChart {
475 - continue
476 - }
477 - if !seen[col.chartGroup] {
478 - seen[col.chartGroup] = true
479 - groups = append(groups, col.chartGroup)
480 - }
481 - }
482 - if len(groups) > 0 {
483 - return groups
484 - }
485 - for _, col := range cols {
486 - if !col.isMetric || col.chartGroup == "" {
487 - continue
488 - }
489 - if !seen[col.chartGroup] {
490 - seen[col.chartGroup] = true
491 - groups = append(groups, col.chartGroup)
492 - }
493 - }
494 - return groups
495 -}
496 -
10 // detectPgStatStatementsColumns queries the database to find available columns
11 func (c *Collector) detectPgStatStatementsColumns(ctx context.Context) (map[string]bool, error) {
12 // Fast path: return cached result
@@ -545,317 +58,6 @@ func (c *Collector) detectPgStatStatementsColumns(ctx context.Context) (map[stri
58 return cols, nil
59 }
60
548 -// buildAvailableColumns returns column metadata for columns that exist in this PG version
549 -func (c *Collector) buildAvailableColumns(availableCols map[string]bool) []pgColumnMeta {
550 - var result []pgColumnMeta
551 -
552 - for _, col := range pgAllColumns {
553 - // Extract the actual column name (remove table prefix and type cast)
554 - colName := col.dbColumn
555 - if idx := strings.LastIndex(colName, "."); idx != -1 {
556 - colName = colName[idx+1:]
557 - }
558 - // Remove PostgreSQL type cast suffix (e.g., "::text")
559 - if idx := strings.Index(colName, "::"); idx != -1 {
560 - colName = colName[:idx]
561 - }
562 -
563 - // Handle version-specific column names for time columns
564 - // PG 13+ renamed time columns: total_time -> total_exec_time, etc.
565 - actualColName := colName
566 - if c.pgVersion >= pgVersion13 {
567 - switch colName {
568 - case "total_time":
569 - actualColName = "total_exec_time"
570 - case "mean_time":
571 - actualColName = "mean_exec_time"
572 - case "min_time":
573 - actualColName = "min_exec_time"
574 - case "max_time":
575 - actualColName = "max_exec_time"
576 - case "stddev_time":
577 - actualColName = "stddev_exec_time"
578 - }
579 - }
580 -
581 - // Check if column exists (either directly or via join)
582 - // Join columns (database, user) come from other tables (d.datname, u.usename)
583 - isJoinCol := col.uiKey == "database" || col.uiKey == "user"
584 - if isJoinCol || availableCols[actualColName] {
585 - // Create a copy with the actual column name for this version
586 - colCopy := col
587 - if actualColName != colName {
588 - // Update dbColumn to use the version-specific name with alias
589 - prefix := "s."
590 - if strings.HasPrefix(col.dbColumn, "s.") {
591 - prefix = ""
592 - colCopy.dbColumn = "s." + actualColName
593 - }
594 - _ = prefix // suppress unused warning
595 - }
596 - result = append(result, colCopy)
597 - }
598 - }
599 -
600 - return result
601 -}
602 -
603 -// mapAndValidateSortColumn maps the semantic sort column to actual SQL column
604 -func (c *Collector) mapAndValidateSortColumn(sortColumn string, availableCols map[string]bool) string {
605 - // Map UI key back to dbColumn
606 - for _, col := range pgAllColumns {
607 - if col.uiKey == sortColumn || col.dbColumn == sortColumn {
608 - // Get actual column name (strip table prefix and type cast)
609 - colName := col.dbColumn
610 - if idx := strings.LastIndex(colName, "."); idx != -1 {
611 - colName = colName[idx+1:]
612 - }
613 - if idx := strings.Index(colName, "::"); idx != -1 {
614 - colName = colName[:idx]
615 - }
616 -
617 - // Handle version-specific mapping
618 - if c.pgVersion >= pgVersion13 {
619 - switch colName {
620 - case "total_time":
621 - colName = "total_exec_time"
622 - case "mean_time":
623 - colName = "mean_exec_time"
624 - case "min_time":
625 - colName = "min_exec_time"
626 - case "max_time":
627 - colName = "max_exec_time"
628 - case "stddev_time":
629 - colName = "stddev_exec_time"
630 - }
631 - }
632 -
633 - // Validate column exists
634 - if availableCols[colName] {
635 - return colName
636 - }
637 - }
638 - }
639 -
640 - // Default fallback
641 - if c.pgVersion >= pgVersion13 {
642 - return "total_exec_time"
643 - }
644 - return "total_time"
645 -}
646 -
647 -// buildDynamicSQL builds the SQL query with only available columns
648 -func (c *Collector) buildDynamicSQL(cols []pgColumnMeta, sortColumn string, limit int) string {
649 - var selectCols []string
650 -
651 - for _, col := range cols {
652 - colExpr := col.dbColumn
653 -
654 - // Handle version-specific column names
655 - if c.pgVersion >= pgVersion13 {
656 - switch {
657 - case strings.HasSuffix(colExpr, ".total_time"):
658 - colExpr = strings.Replace(colExpr, ".total_time", ".total_exec_time", 1)
659 - case strings.HasSuffix(colExpr, ".mean_time"):
660 - colExpr = strings.Replace(colExpr, ".mean_time", ".mean_exec_time", 1)
661 - case strings.HasSuffix(colExpr, ".min_time"):
662 - colExpr = strings.Replace(colExpr, ".min_time", ".min_exec_time", 1)
663 - case strings.HasSuffix(colExpr, ".max_time"):
664 - colExpr = strings.Replace(colExpr, ".max_time", ".max_exec_time", 1)
665 - case strings.HasSuffix(colExpr, ".stddev_time"):
666 - colExpr = strings.Replace(colExpr, ".stddev_time", ".stddev_exec_time", 1)
667 - case colExpr == "total_time":
668 - colExpr = "total_exec_time"
669 - case colExpr == "mean_time":
670 - colExpr = "mean_exec_time"
671 - case colExpr == "min_time":
672 - colExpr = "min_exec_time"
673 - case colExpr == "max_time":
674 - colExpr = "max_exec_time"
675 - case colExpr == "stddev_time":
676 - colExpr = "stddev_exec_time"
677 - }
678 - }
679 -
680 - // Always use uiKey as the SQL alias for consistent naming
681 - // Use double quotes to handle reserved keywords like "database", "user"
682 - selectCols = append(selectCols, fmt.Sprintf("%s AS \"%s\"", colExpr, col.uiKey))
683 - }
684 -
685 - return fmt.Sprintf(`
686 -SELECT %s
687 -FROM pg_stat_statements s
688 -JOIN pg_database d ON s.dbid = d.oid
689 -JOIN pg_user u ON s.userid = u.usesysid
690 -ORDER BY "%s" DESC
691 -LIMIT %d
692 -`, strings.Join(selectCols, ", "), sortColumn, limit)
693 -}
694 -
695 -// scanDynamicRows scans rows into the data array based on column types
696 -// Uses sql.Null* types to handle NULL values safely
697 -func (c *Collector) scanDynamicRows(rows dbRows, cols []pgColumnMeta) ([][]any, error) {
698 - data := make([][]any, 0, 500)
699 -
700 - // Create value holders for scanning (reuse across rows for efficiency)
701 - valuePtrs := make([]any, len(cols))
702 - values := make([]any, len(cols))
703 -
704 - for rows.Next() {
705 - // Reset value holders for each row
706 - for i, col := range cols {
707 - switch col.dataType {
708 - case ftString:
709 - var v sql.NullString
710 - values[i] = &v
711 - case ftInteger:
712 - var v sql.NullInt64
713 - values[i] = &v
714 - case ftFloat, ftDuration:
715 - var v sql.NullFloat64
716 - values[i] = &v
717 - default:
718 - var v sql.NullString
719 - values[i] = &v
720 - }
721 - valuePtrs[i] = values[i]
722 - }
723 -
724 - if err := rows.Scan(valuePtrs...); err != nil {
725 - return nil, fmt.Errorf("row scan failed: %v", err)
726 - }
727 -
728 - // Convert scanned values to output format
729 - row := make([]any, len(cols))
730 - for i, col := range cols {
731 - switch v := values[i].(type) {
732 - case *sql.NullString:
733 - if v.Valid {
734 - s := v.String
735 - if col.uiKey == "query" {
736 - row[i] = strmutil.TruncateText(s, maxQueryTextLength)
737 - } else {
738 - row[i] = s
739 - }
740 - } else {
741 - row[i] = ""
742 - }
743 - case *sql.NullInt64:
744 - if v.Valid {
745 - row[i] = v.Int64
746 - } else {
747 - row[i] = int64(0)
748 - }
749 - case *sql.NullFloat64:
750 - if v.Valid {
751 - row[i] = v.Float64
752 - } else {
753 - row[i] = float64(0)
754 - }
755 - }
756 - }
757 -
758 - data = append(data, row)
759 - }
760 -
761 - return data, nil
762 -}
763 -
764 -// buildDynamicColumns builds column definitions for the response
765 -func (c *Collector) buildDynamicColumns(cols []pgColumnMeta) map[string]any {
766 - result := make(map[string]any)
767 -
768 - for i, col := range cols {
769 - visual := funcapi.FieldVisualValue
770 - if col.dataType == ftDuration {
771 - visual = funcapi.FieldVisualBar
772 - }
773 - colDef := funcapi.Column{
774 - Index: i,
775 - Name: col.displayName,
776 - Type: col.dataType,
777 - Units: col.units,
778 - Visualization: visual,
779 - Sort: col.sortDir,
780 - Sortable: true,
781 - Sticky: col.isSticky,
782 - Summary: col.summary,
783 - Filter: col.filter,
784 - FullWidth: col.fullWidth,
785 - Wrap: false,
786 - DefaultExpandedFilter: false,
787 - UniqueKey: col.isUniqueKey,
788 - Visible: col.visible,
789 - ValueOptions: funcapi.ValueOptions{
790 - Transform: col.transform,
791 - DecimalPoints: col.decimalPoints,
792 - DefaultValue: nil,
793 - },
794 - }
795 - result[col.uiKey] = colDef.BuildColumn()
796 - }
797 -
798 - return result
799 -}
800 -
801 -// buildDynamicSortOptions builds sort options from available columns
802 -// Returns only sort options for columns that actually exist in the database
803 -func (c *Collector) buildDynamicSortOptions(cols []pgColumnMeta) []funcapi.ParamOption {
804 - var sortOpts []funcapi.ParamOption
805 - seen := make(map[string]bool)
806 - sortDir := funcapi.FieldSortDescending
807 -
808 - for _, col := range cols {
809 - if col.isSortOption && !seen[col.uiKey] {
810 - seen[col.uiKey] = true
811 - sortOpts = append(sortOpts, funcapi.ParamOption{
812 - ID: col.uiKey,
813 - Column: col.uiKey,
814 - Name: col.sortLabel,
815 - Default: col.isDefaultSort,
816 - Sort: &sortDir,
817 - })
818 - }
819 - }
820 - return sortOpts
821 -}
822 -
823 -func (c *Collector) topQueriesSortParam(queryCols []pgColumnMeta) (funcapi.ParamConfig, []funcapi.ParamOption) {
824 - sortOptions := c.buildDynamicSortOptions(queryCols)
825 - sortParam := funcapi.ParamConfig{
826 - ID: paramSort,
827 - Name: "Filter By",
828 - Help: "Select the primary sort column",
829 - Selection: funcapi.ParamSelect,
830 - Options: sortOptions,
831 - UniqueView: true,
832 - }
833 - return sortParam, sortOptions
834 -}
835 -
836 -func (c *Collector) topQueriesParams(ctx context.Context) ([]funcapi.ParamConfig, error) {
837 - available, err := c.checkPgStatStatements(ctx)
838 - if err != nil {
839 - return nil, err
840 - }
841 - if !available {
842 - return nil, fmt.Errorf("pg_stat_statements extension is not installed")
843 - }
844 -
845 - availableCols, err := c.detectPgStatStatementsColumns(ctx)
846 - if err != nil {
847 - return nil, err
848 - }
849 -
850 - queryCols := c.buildAvailableColumns(availableCols)
851 - if len(queryCols) == 0 {
852 - return nil, fmt.Errorf("no queryable columns found in pg_stat_statements")
853 - }
854 -
855 - sortParam, _ := c.topQueriesSortParam(queryCols)
856 - return []funcapi.ParamConfig{sortParam}, nil
857 -}
858 -
61 // checkPgStatStatements checks if pg_stat_statements extension is available
62 // Only positive results are cached - negative results are re-checked each time
63 // so users don't need to restart after installing the extension
@@ -885,10 +87,3 @@ func (c *Collector) checkPgStatStatements(ctx context.Context) (bool, error) {
87
88 return exists, nil
89 }
888 -
889 -// dbRows interface for testing
890 -type dbRows interface {
891 - Next() bool
892 - Scan(dest ...any) error
893 - Err() error
894 -}
src/go/plugin/go.d/collector/postgres/functions_test.go
+44 -47
@@ -45,46 +45,43 @@ func TestPgMethods(t *testing.T) {
45
46 func TestPgAllColumns_HasRequiredColumns(t *testing.T) {
47 // Verify all required base columns are defined
48 - requiredUIKeys := []string{
48 + requiredIDs := []string{
49 "queryid", "query", "database", "user", "calls",
50 "totalTime", "meanTime", "minTime", "maxTime",
51 "rows", "sharedBlksHit", "sharedBlksRead", "tempBlksWritten",
52 }
53
54 - uiKeys := make(map[string]bool)
55 - for _, col := range pgAllColumns {
56 - uiKeys[col.uiKey] = true
57 - }
54 + cs := pgColumnSet(pgAllColumns)
55
59 - for _, key := range requiredUIKeys {
60 - assert.True(t, uiKeys[key], "column %s should be defined in pgAllColumns", key)
56 + for _, id := range requiredIDs {
57 + assert.True(t, cs.ContainsColumn(id), "column %s should be defined in pgAllColumns", id)
58 }
59 }
60
61 func TestPgAllColumns_HasValidMetadata(t *testing.T) {
62 for _, col := range pgAllColumns {
66 - // Every column must have a UI key
67 - assert.NotEmpty(t, col.uiKey, "column %s must have uiKey", col.dbColumn)
63 + // Every column must have an ID
64 + assert.NotEmpty(t, col.Name, "column %s must have Name", col.DBColumn)
65
69 - // Every column must have a display name
70 - assert.NotEmpty(t, col.displayName, "column %s must have displayName", col.uiKey)
66 + // Every column must have a display name (tooltip)
67 + assert.NotEmpty(t, col.Tooltip, "column %s must have Tooltip", col.Name)
68
69 // Every column must have a data type
73 - assert.NotEqual(t, funcapi.FieldTypeNone, col.dataType, "column %s must have dataType", col.uiKey)
70 + assert.NotEqual(t, funcapi.FieldTypeNone, col.Type, "column %s must have Type", col.Name)
71
72 // Duration columns must have units
76 - if col.dataType == ftDuration {
77 - assert.NotEmpty(t, col.units, "duration column %s must have units", col.uiKey)
73 + if col.Type == funcapi.FieldTypeDuration {
74 + assert.NotEmpty(t, col.Units, "duration column %s must have Units", col.Name)
75 }
76
77 // Sort options must have labels
81 - if col.isSortOption {
82 - assert.NotEmpty(t, col.sortLabel, "sort option column %s must have sortLabel", col.uiKey)
78 + if col.IsSortOption {
79 + assert.NotEmpty(t, col.SortLabel, "sort option column %s must have SortLabel", col.Name)
80 }
81 }
82 }
83
87 -func TestCollector_mapAndValidateSortColumn(t *testing.T) {
84 +func TestFuncTopQueries_mapAndValidateSortColumn(t *testing.T) {
85 tests := map[string]struct {
86 pgVersion int
87 availableCols map[string]bool
@@ -132,18 +129,20 @@ func TestCollector_mapAndValidateSortColumn(t *testing.T) {
129 for name, tc := range tests {
130 t.Run(name, func(t *testing.T) {
131 c := &Collector{pgVersion: tc.pgVersion}
135 - result := c.mapAndValidateSortColumn(tc.input, tc.availableCols)
132 + r := &funcRouter{collector: c}
133 + f := &funcTopQueries{router: r}
134 + result := f.mapAndValidateSortColumn(tc.input, tc.availableCols)
135 assert.Equal(t, tc.expected, result)
136 })
137 }
138 }
139
141 -func TestCollector_buildAvailableColumns(t *testing.T) {
140 +func TestFuncTopQueries_buildAvailableColumns(t *testing.T) {
141 tests := map[string]struct {
142 pgVersion int
143 availableCols map[string]bool
145 - expectCols []string // UI keys we expect to see
146 - notExpectCols []string // UI keys we don't expect
144 + expectCols []string // Column IDs we expect to see
145 + notExpectCols []string // Column IDs we don't expect
146 }{
147 "PG12 with basic columns": {
148 pgVersion: pgVersionOld,
@@ -170,25 +169,22 @@ func TestCollector_buildAvailableColumns(t *testing.T) {
169 for name, tc := range tests {
170 t.Run(name, func(t *testing.T) {
171 c := &Collector{pgVersion: tc.pgVersion}
173 - cols := c.buildAvailableColumns(tc.availableCols)
172 + r := &funcRouter{collector: c}
173 + f := &funcTopQueries{router: r}
174 + cols := f.buildAvailableColumns(tc.availableCols)
175 + cs := pgColumnSet(cols)
176
175 - // Build map of UI keys for easy lookup
176 - uiKeys := make(map[string]bool)
177 - for _, col := range cols {
178 - uiKeys[col.uiKey] = true
177 + for _, id := range tc.expectCols {
178 + assert.True(t, cs.ContainsColumn(id), "expected column %s to be present", id)
179 }
180 -
181 - for _, key := range tc.expectCols {
182 - assert.True(t, uiKeys[key], "expected column %s to be present", key)
183 - }
184 - for _, key := range tc.notExpectCols {
185 - assert.False(t, uiKeys[key], "did not expect column %s to be present", key)
180 + for _, id := range tc.notExpectCols {
181 + assert.False(t, cs.ContainsColumn(id), "did not expect column %s to be present", id)
182 }
183 })
184 }
185 }
186
191 -func TestCollector_buildDynamicSQL(t *testing.T) {
187 +func TestFuncTopQueries_buildDynamicSQL(t *testing.T) {
188 tests := map[string]struct {
189 pgVersion int
190 sortColumn string
@@ -209,16 +205,18 @@ func TestCollector_buildDynamicSQL(t *testing.T) {
205 for name, tc := range tests {
206 t.Run(name, func(t *testing.T) {
207 c := &Collector{pgVersion: tc.pgVersion}
208 + r := &funcRouter{collector: c}
209 + f := &funcTopQueries{router: r}
210
211 // Build minimal column set for test
214 - cols := []pgColumnMeta{
215 - {dbColumn: "s.queryid", uiKey: "queryid", dataType: ftString},
216 - {dbColumn: "s.query", uiKey: "query", dataType: ftString},
217 - {dbColumn: "s.calls", uiKey: "calls", dataType: ftInteger},
218 - {dbColumn: "total_time", uiKey: "totalTime", dataType: ftDuration},
212 + cols := []pgColumn{
213 + {ColumnMeta: funcapi.ColumnMeta{Name: "queryid", Type: funcapi.FieldTypeString}, DBColumn: "s.queryid"},
214 + {ColumnMeta: funcapi.ColumnMeta{Name: "query", Type: funcapi.FieldTypeString}, DBColumn: "s.query"},
215 + {ColumnMeta: funcapi.ColumnMeta{Name: "calls", Type: funcapi.FieldTypeInteger}, DBColumn: "s.calls"},
216 + {ColumnMeta: funcapi.ColumnMeta{Name: "totalTime", Type: funcapi.FieldTypeDuration}, DBColumn: "total_time"},
217 }
218
221 - sql := c.buildDynamicSQL(cols, tc.sortColumn, 500)
219 + sql := f.buildDynamicSQL(cols, tc.sortColumn, 500)
220
221 assert.Contains(t, sql, "pg_stat_statements")
222 assert.Contains(t, sql, tc.sortColumn)
@@ -228,16 +226,15 @@ func TestCollector_buildDynamicSQL(t *testing.T) {
226 }
227 }
228
231 -func TestCollector_buildDynamicColumns(t *testing.T) {
232 - c := &Collector{}
233 -
234 - cols := []pgColumnMeta{
235 - {uiKey: "queryid", displayName: "Query ID", dataType: ftString, visible: false, isUniqueKey: true, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti},
236 - {uiKey: "query", displayName: "Query", dataType: ftString, visible: true, isSticky: true, fullWidth: true, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti},
237 - {uiKey: "totalTime", displayName: "Total Time", dataType: ftDuration, units: "seconds", visible: true, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summarySum, filter: filterRange},
229 +func TestPgColumnSet_BuildColumns(t *testing.T) {
230 + cols := []pgColumn{
231 + {ColumnMeta: funcapi.ColumnMeta{Name: "queryid", Tooltip: "Query ID", Type: funcapi.FieldTypeString, Visible: false, UniqueKey: true, Transform: funcapi.FieldTransformNone, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, Sortable: true}, DBColumn: "s.queryid"},
232 + {ColumnMeta: funcapi.ColumnMeta{Name: "query", Tooltip: "Query", Type: funcapi.FieldTypeString, Visible: true, Sticky: true, FullWidth: true, Transform: funcapi.FieldTransformNone, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, Sortable: true}, DBColumn: "s.query"},
233 + {ColumnMeta: funcapi.ColumnMeta{Name: "totalTime", Tooltip: "Total Time", Type: funcapi.FieldTypeDuration, Units: "seconds", Visible: true, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, DBColumn: "total_time"},
234 }
235
240 - columns := c.buildDynamicColumns(cols)
236 + cs := pgColumnSet(cols)
237 + columns := cs.BuildColumns()
238
239 // Verify column count
240 assert.Len(t, columns, 3)
src/go/plugin/go.d/collector/proxysql/collector.go
+12 -4
@@ -25,13 +25,12 @@ func init() {
25 Create: func() module.Module { return New() },
26 Config: func() any { return &Config{} },
27 Methods: proxysqlMethods,
28 - MethodParams: proxysqlMethodParams,
29 - HandleMethod: proxysqlHandleMethod,
28 + MethodHandler: proxysqlFunctionHandler,
29 })
30 }
31
32 func New() *Collector {
34 - return &Collector{
33 + c := &Collector{
34 Config: Config{
35 DSN: "stats:stats@tcp(127.0.0.1:6032)/",
36 Timeout: confopt.Duration(time.Second),
@@ -46,6 +45,10 @@ func New() *Collector {
45 hostgroups: make(map[string]*hostgroupCache),
46 },
47 }
48 +
49 + c.funcRouter = newFuncRouter(c)
50 +
51 + return c
52 }
53
54 type Config struct {
@@ -68,6 +71,8 @@ type Collector struct {
71 once *sync.Once
72 cache *cache
73
74 + funcRouter *funcRouter // function router for method handlers
75 +
76 queryDigestCols map[string]bool
77 queryDigestColsMu sync.RWMutex
78 }
@@ -113,7 +118,10 @@ func (c *Collector) Collect(context.Context) map[string]int64 {
118 return mx
119 }
120
116 -func (c *Collector) Cleanup(context.Context) {
121 +func (c *Collector) Cleanup(ctx context.Context) {
122 + if c.funcRouter != nil {
123 + c.funcRouter.Cleanup(ctx)
124 + }
125 if c.db == nil {
126 return
127 }
src/go/plugin/go.d/collector/proxysql/func_router.go new
+64
@@ -0,0 +1,64 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package proxysql
4 +
5 +import (
6 + "context"
7 + "fmt"
8 +
9 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
10 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
11 +)
12 +
13 +// funcRouter routes method calls to appropriate function handlers.
14 +type funcRouter struct {
15 + collector *Collector
16 +
17 + handlers map[string]funcapi.MethodHandler
18 +}
19 +
20 +func newFuncRouter(c *Collector) *funcRouter {
21 + r := &funcRouter{
22 + collector: c,
23 + handlers: make(map[string]funcapi.MethodHandler),
24 + }
25 + r.handlers[topQueriesMethodID] = newFuncTopQueries(r)
26 + return r
27 +}
28 +
29 +// Compile-time interface check.
30 +var _ funcapi.MethodHandler = (*funcRouter)(nil)
31 +
32 +func (r *funcRouter) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
33 + if h, ok := r.handlers[method]; ok {
34 + return h.MethodParams(ctx, method)
35 + }
36 + return nil, fmt.Errorf("unknown method: %s", method)
37 +}
38 +
39 +func (r *funcRouter) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
40 + if h, ok := r.handlers[method]; ok {
41 + return h.Handle(ctx, method, params)
42 + }
43 + return funcapi.NotFoundResponse(method)
44 +}
45 +
46 +func (r *funcRouter) Cleanup(ctx context.Context) {
47 + for _, h := range r.handlers {
48 + h.Cleanup(ctx)
49 + }
50 +}
51 +
52 +func proxysqlMethods() []funcapi.MethodConfig {
53 + return []funcapi.MethodConfig{
54 + topQueriesMethodConfig(),
55 + }
56 +}
57 +
58 +func proxysqlFunctionHandler(job *module.Job) funcapi.MethodHandler {
59 + c, ok := job.Module().(*Collector)
60 + if !ok {
61 + return nil
62 + }
63 + return c.funcRouter
64 +}
src/go/plugin/go.d/collector/proxysql/functions.go
+123 -308
@@ -9,194 +9,127 @@ import (
9 "strings"
10
11 "github.com/netdata/netdata/go/plugins/pkg/funcapi"
12 - "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
12 "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
13 )
14
15 const proxysqlMaxQueryTextLength = 4096
16
17 const (
19 - paramSort = "__sort"
18 + topQueriesMethodID = "top-queries"
19 + paramSort = "__sort"
20 +)
21
21 - ftString = funcapi.FieldTypeString
22 - ftInteger = funcapi.FieldTypeInteger
23 - ftDuration = funcapi.FieldTypeDuration
22 +func topQueriesMethodConfig() funcapi.MethodConfig {
23 + return funcapi.MethodConfig{
24 + ID: topQueriesMethodID,
25 + Name: "Top Queries",
26 + UpdateEvery: 10,
27 + Help: "Top SQL queries from ProxySQL query digest stats",
28 + RequireCloud: true,
29 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(proxysqlAllColumns)},
30 + }
31 +}
32
25 - trNone = funcapi.FieldTransformNone
26 - trNumber = funcapi.FieldTransformNumber
27 - trDuration = funcapi.FieldTransformDuration
33 +// proxysqlColumn defines metadata for a ProxySQL query digest column.
34 +// Embeds funcapi.ColumnMeta for UI rendering and adds ProxySQL-specific fields.
35 +type proxysqlColumn struct {
36 + funcapi.ColumnMeta
37 +
38 + // DBColumn is the database column name
39 + DBColumn string
40 + // IsMicroseconds indicates if the value is in microseconds (needs /1000 conversion)
41 + IsMicroseconds bool
42 + // sortOpt indicates whether this column appears in the sort dropdown
43 + sortOpt bool
44 + // sortLbl is the label shown in the sort dropdown
45 + sortLbl string
46 + // defaultSort indicates whether this is the default sort column
47 + defaultSort bool
48 +}
49
29 - sortAsc = funcapi.FieldSortAscending
30 - sortDesc = funcapi.FieldSortDescending
50 +// funcapi.SortableColumn interface implementation for proxysqlColumn.
51 +func (c proxysqlColumn) IsSortOption() bool { return c.sortOpt }
52 +func (c proxysqlColumn) SortLabel() string { return c.sortLbl }
53 +func (c proxysqlColumn) IsDefaultSort() bool { return c.defaultSort }
54 +func (c proxysqlColumn) ColumnName() string { return c.Name }
55 +func (c proxysqlColumn) SortColumn() string { return "" }
56
32 - summaryCount = funcapi.FieldSummaryCount
33 - summarySum = funcapi.FieldSummarySum
34 - summaryMin = funcapi.FieldSummaryMin
35 - summaryMax = funcapi.FieldSummaryMax
36 - summaryMean = funcapi.FieldSummaryMean
57 +// proxysqlColumnSet creates a ColumnSet from a slice of proxysqlColumn.
58 +func proxysqlColumnSet(cols []proxysqlColumn) funcapi.ColumnSet[proxysqlColumn] {
59 + return funcapi.Columns(cols, func(c proxysqlColumn) funcapi.ColumnMeta { return c.ColumnMeta })
60 +}
61
38 - filterMulti = funcapi.FieldFilterMultiselect
39 - filterRange = funcapi.FieldFilterRange
40 -)
62 +var proxysqlAllColumns = []proxysqlColumn{
63 + {ColumnMeta: funcapi.ColumnMeta{Name: "digest", Tooltip: "Digest", Type: funcapi.FieldTypeString, Visible: false, Transform: funcapi.FieldTransformNone, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, UniqueKey: true, Sortable: true}, DBColumn: "digest"},
64 + {ColumnMeta: funcapi.ColumnMeta{Name: "query", Tooltip: "Query", Type: funcapi.FieldTypeString, Visible: true, Transform: funcapi.FieldTransformNone, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, Sticky: true, FullWidth: true, Sortable: true}, DBColumn: "digest_text"},
65 + {ColumnMeta: funcapi.ColumnMeta{Name: "schema", Tooltip: "Schema", Type: funcapi.FieldTypeString, Visible: true, Transform: funcapi.FieldTransformNone, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, GroupBy: &funcapi.GroupByOptions{IsDefault: true}, Sortable: true}, DBColumn: "schemaname"},
66 + {ColumnMeta: funcapi.ColumnMeta{Name: "user", Tooltip: "User", Type: funcapi.FieldTypeString, Visible: false, Transform: funcapi.FieldTransformNone, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, GroupBy: &funcapi.GroupByOptions{}, Sortable: true}, DBColumn: "username"},
67 + {ColumnMeta: funcapi.ColumnMeta{Name: "hostgroup", Tooltip: "Hostgroup", Type: funcapi.FieldTypeInteger, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterRange, GroupBy: &funcapi.GroupByOptions{}, Sortable: true}, DBColumn: "hostgroup"},
68
42 -type proxysqlColumnMeta struct {
43 - dbColumn string
44 - uiKey string
45 - displayName string
46 - dataType funcapi.FieldType
47 - units string
48 - visible bool
49 - transform funcapi.FieldTransform
50 - decimalPoints int
51 - sortDir funcapi.FieldSort
52 - summary funcapi.FieldSummary
53 - filter funcapi.FieldFilter
54 - isMicroseconds bool
55 - isSortOption bool
56 - sortLabel string
57 - isDefaultSort bool
58 - isUniqueKey bool
59 - isSticky bool
60 - fullWidth bool
61 - isLabel bool
62 - isPrimary bool
63 - isMetric bool
64 - chartGroup string
65 - chartTitle string
66 - isDefaultChart bool
67 -}
69 + {ColumnMeta: funcapi.ColumnMeta{Name: "calls", Tooltip: "Calls", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Chart: &funcapi.ChartOptions{Group: "Calls", Title: "Number of Calls", IsDefault: true}, Sortable: true}, DBColumn: "count_star", sortOpt: true, sortLbl: "Top queries by Number of Calls"},
70
69 -var proxysqlAllColumns = []proxysqlColumnMeta{
70 - {dbColumn: "digest", uiKey: "digest", displayName: "Digest", dataType: ftString, visible: false, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti, isUniqueKey: true},
71 - {dbColumn: "digest_text", uiKey: "query", displayName: "Query", dataType: ftString, visible: true, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti, isSticky: true, fullWidth: true},
72 - {dbColumn: "schemaname", uiKey: "schema", displayName: "Schema", dataType: ftString, visible: true, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti, isLabel: true, isPrimary: true},
73 - {dbColumn: "username", uiKey: "user", displayName: "User", dataType: ftString, visible: false, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti, isLabel: true},
74 - {dbColumn: "hostgroup", uiKey: "hostgroup", displayName: "Hostgroup", dataType: ftInteger, visible: false, transform: trNumber, sortDir: sortAsc, summary: summaryCount, filter: filterRange, isLabel: true},
71 + {ColumnMeta: funcapi.ColumnMeta{Name: "totalTime", Tooltip: "Total Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: true, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Execution Time", IsDefault: true}, Sortable: true}, DBColumn: "sum_time", IsMicroseconds: true, sortOpt: true, sortLbl: "Top queries by Total Execution Time", defaultSort: true},
72 + {ColumnMeta: funcapi.ColumnMeta{Name: "avgTime", Tooltip: "Avg Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: true, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMean, Filter: funcapi.FieldFilterRange, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Execution Time"}, Sortable: true}, DBColumn: "avg_time", IsMicroseconds: true, sortOpt: true, sortLbl: "Top queries by Average Execution Time"},
73 + {ColumnMeta: funcapi.ColumnMeta{Name: "minTime", Tooltip: "Min Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: false, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMin, Filter: funcapi.FieldFilterRange, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Execution Time"}, Sortable: true}, DBColumn: "min_time", IsMicroseconds: true},
74 + {ColumnMeta: funcapi.ColumnMeta{Name: "maxTime", Tooltip: "Max Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", Visible: false, Transform: funcapi.FieldTransformDuration, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Filter: funcapi.FieldFilterRange, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Execution Time"}, Sortable: true}, DBColumn: "max_time", IsMicroseconds: true},
75
76 - {dbColumn: "count_star", uiKey: "calls", displayName: "Calls", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Number of Calls", isMetric: true, chartGroup: "Calls", chartTitle: "Number of Calls", isDefaultChart: true},
76 + {ColumnMeta: funcapi.ColumnMeta{Name: "rowsAffected", Tooltip: "Rows Affected", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Chart: &funcapi.ChartOptions{Group: "Rows", Title: "Rows"}, Sortable: true}, DBColumn: "sum_rows_affected", sortOpt: true, sortLbl: "Top queries by Rows Affected"},
77 + {ColumnMeta: funcapi.ColumnMeta{Name: "rowsSent", Tooltip: "Rows Sent", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Chart: &funcapi.ChartOptions{Group: "Rows", Title: "Rows"}, Sortable: true}, DBColumn: "sum_rows_sent", sortOpt: true, sortLbl: "Top queries by Rows Sent"},
78 + {ColumnMeta: funcapi.ColumnMeta{Name: "errors", Tooltip: "Errors", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Chart: &funcapi.ChartOptions{Group: "Errors", Title: "Errors & Warnings"}, Sortable: true}, DBColumn: "sum_errors", sortOpt: true, sortLbl: "Top queries by Errors"},
79 + {ColumnMeta: funcapi.ColumnMeta{Name: "warnings", Tooltip: "Warnings", Type: funcapi.FieldTypeInteger, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Chart: &funcapi.ChartOptions{Group: "Errors", Title: "Errors & Warnings"}, Sortable: true}, DBColumn: "sum_warnings", sortOpt: true, sortLbl: "Top queries by Warnings"},
80
78 - {dbColumn: "sum_time", uiKey: "totalTime", displayName: "Total Time", dataType: ftDuration, units: "milliseconds", visible: true, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summarySum, filter: filterRange, isMicroseconds: true, isSortOption: true, sortLabel: "Total Execution Time", isDefaultSort: true, isMetric: true, chartGroup: "Time", chartTitle: "Execution Time", isDefaultChart: true},
79 - {dbColumn: "avg_time", uiKey: "avgTime", displayName: "Avg Time", dataType: ftDuration, units: "milliseconds", visible: true, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMean, filter: filterRange, isMicroseconds: true, isSortOption: true, sortLabel: "Average Execution Time", isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
80 - {dbColumn: "min_time", uiKey: "minTime", displayName: "Min Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMin, filter: filterRange, isMicroseconds: true, isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
81 - {dbColumn: "max_time", uiKey: "maxTime", displayName: "Max Time", dataType: ftDuration, units: "milliseconds", visible: false, transform: trDuration, decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, filter: filterRange, isMicroseconds: true, isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
81 + {ColumnMeta: funcapi.ColumnMeta{Name: "firstSeen", Tooltip: "First Seen", Type: funcapi.FieldTypeString, Visible: false, Transform: funcapi.FieldTransformNone, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, Sortable: true}, DBColumn: "first_seen"},
82 + {ColumnMeta: funcapi.ColumnMeta{Name: "lastSeen", Tooltip: "Last Seen", Type: funcapi.FieldTypeString, Visible: false, Transform: funcapi.FieldTransformNone, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, Sortable: true}, DBColumn: "last_seen"},
83 +}
84
83 - {dbColumn: "sum_rows_affected", uiKey: "rowsAffected", displayName: "Rows Affected", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Rows Affected", isMetric: true, chartGroup: "Rows", chartTitle: "Rows"},
84 - {dbColumn: "sum_rows_sent", uiKey: "rowsSent", displayName: "Rows Sent", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Rows Sent", isMetric: true, chartGroup: "Rows", chartTitle: "Rows"},
85 - {dbColumn: "sum_errors", uiKey: "errors", displayName: "Errors", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Errors", isMetric: true, chartGroup: "Errors", chartTitle: "Errors & Warnings"},
86 - {dbColumn: "sum_warnings", uiKey: "warnings", displayName: "Warnings", dataType: ftInteger, visible: true, transform: trNumber, sortDir: sortDesc, summary: summarySum, filter: filterRange, isSortOption: true, sortLabel: "Warnings", isMetric: true, chartGroup: "Errors", chartTitle: "Errors & Warnings"},
85 +// Compile-time interface check.
86 +var _ funcapi.MethodHandler = (*funcTopQueries)(nil)
87
88 - {dbColumn: "first_seen", uiKey: "firstSeen", displayName: "First Seen", dataType: ftString, visible: false, transform: trNone, sortDir: sortAsc, summary: summaryCount, filter: filterMulti},
89 - {dbColumn: "last_seen", uiKey: "lastSeen", displayName: "Last Seen", dataType: ftString, visible: false, transform: trNone, sortDir: sortDesc, summary: summaryCount, filter: filterMulti},
88 +// funcTopQueries handles the "top-queries" function for ProxySQL.
89 +type funcTopQueries struct {
90 + router *funcRouter
91 }
92
92 -func proxysqlMethods() []module.MethodConfig {
93 - sortOptions := buildProxySQLSortOptions(proxysqlAllColumns)
94 - return []module.MethodConfig{
95 - {
96 - UpdateEvery: 10,
97 - ID: "top-queries",
98 - Name: "Top Queries",
99 - Help: "Top SQL queries from ProxySQL query digest stats",
100 - RequireCloud: true,
101 - RequiredParams: []funcapi.ParamConfig{{
102 - ID: paramSort,
103 - Name: "Filter By",
104 - Help: "Select the primary sort column",
105 - Selection: funcapi.ParamSelect,
106 - Options: sortOptions,
107 - UniqueView: true,
108 - }},
109 - },
110 - }
93 +func newFuncTopQueries(r *funcRouter) *funcTopQueries {
94 + return &funcTopQueries{router: r}
95 }
96
113 -func proxysqlMethodParams(ctx context.Context, job *module.Job, method string) ([]funcapi.ParamConfig, error) {
114 - collector, ok := job.Module().(*Collector)
115 - if !ok {
116 - return nil, fmt.Errorf("invalid module type")
97 +// MethodParams implements funcapi.MethodHandler.
98 +func (f *funcTopQueries) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
99 + if method != topQueriesMethodID {
100 + return nil, nil
101 }
118 - if collector.db == nil {
119 - if err := collector.openConnection(); err != nil {
102 +
103 + c := f.router.collector
104 + if c.db == nil {
105 + if err := c.openConnection(); err != nil {
106 return nil, err
107 }
108 }
123 - switch method {
124 - case "top-queries":
125 - return collector.topQueriesParams(ctx)
126 - default:
127 - return nil, fmt.Errorf("unknown method: %s", method)
128 - }
109 +
110 + return c.topQueriesParams(ctx)
111 }
112
131 -func proxysqlHandleMethod(ctx context.Context, job *module.Job, method string, params funcapi.ResolvedParams) *module.FunctionResponse {
132 - collector, ok := job.Module().(*Collector)
133 - if !ok {
134 - return &module.FunctionResponse{Status: 500, Message: "internal error: invalid module type"}
113 +// Handle implements funcapi.MethodHandler.
114 +func (f *funcTopQueries) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
115 + if method != topQueriesMethodID {
116 + return funcapi.NotFoundResponse(method)
117 }
118
137 - if collector.db == nil {
138 - if err := collector.openConnection(); err != nil {
139 - return &module.FunctionResponse{Status: 503, Message: fmt.Sprintf("failed to open connection: %v", err)}
119 + c := f.router.collector
120 + if c.db == nil {
121 + if err := c.openConnection(); err != nil {
122 + return funcapi.UnavailableResponse(fmt.Sprintf("failed to open connection: %v", err))
123 }
124 }
125
143 - switch method {
144 - case "top-queries":
145 - return collector.collectTopQueries(ctx, params.Column(paramSort))
146 - default:
147 - return &module.FunctionResponse{Status: 404, Message: fmt.Sprintf("unknown method: %s", method)}
148 - }
126 + return c.collectTopQueries(ctx, params.Column(paramSort))
127 }
128
151 -func buildProxySQLSortOptions(cols []proxysqlColumnMeta) []funcapi.ParamOption {
152 - var sortOptions []funcapi.ParamOption
153 - sortDir := funcapi.FieldSortDescending
154 - for _, col := range cols {
155 - if col.isSortOption {
156 - sortOptions = append(sortOptions, funcapi.ParamOption{
157 - ID: col.uiKey,
158 - Column: col.uiKey,
159 - Name: "Top queries by " + col.sortLabel,
160 - Default: col.isDefaultSort,
161 - Sort: &sortDir,
162 - })
163 - }
164 - }
165 - return sortOptions
166 -}
129 +func (f *funcTopQueries) Cleanup(ctx context.Context) {}
130
168 -func buildProxySQLColumns(cols []proxysqlColumnMeta) map[string]any {
169 - columns := make(map[string]any, len(cols))
170 - for i, col := range cols {
171 - visual := funcapi.FieldVisualValue
172 - if col.dataType == ftDuration {
173 - visual = funcapi.FieldVisualBar
174 - }
175 - colDef := funcapi.Column{
176 - Index: i,
177 - Name: col.displayName,
178 - Type: col.dataType,
179 - Units: col.units,
180 - Visualization: visual,
181 - Sort: col.sortDir,
182 - Sortable: true,
183 - Sticky: col.isSticky,
184 - Summary: col.summary,
185 - Filter: col.filter,
186 - FullWidth: col.fullWidth,
187 - Wrap: false,
188 - DefaultExpandedFilter: false,
189 - UniqueKey: col.isUniqueKey,
190 - Visible: col.visible,
191 - ValueOptions: funcapi.ValueOptions{
192 - Transform: col.transform,
193 - DecimalPoints: col.decimalPoints,
194 - DefaultValue: nil,
195 - },
196 - }
197 - columns[col.uiKey] = colDef.BuildColumn()
198 - }
199 - return columns
131 +func buildProxySQLSortParam(cols []proxysqlColumn) funcapi.ParamConfig {
132 + return funcapi.BuildSortParam(cols)
133 }
134
135 func (c *Collector) detectProxySQLDigestColumns(ctx context.Context) (map[string]bool, error) {
@@ -233,10 +166,10 @@ func (c *Collector) detectProxySQLDigestColumns(ctx context.Context) (map[string
166 return cols, nil
167 }
168
236 -func (c *Collector) buildAvailableProxySQLColumns(available map[string]bool) []proxysqlColumnMeta {
237 - var cols []proxysqlColumnMeta
169 +func (c *Collector) buildAvailableProxySQLColumns(available map[string]bool) []proxysqlColumn {
170 + var cols []proxysqlColumn
171 for _, col := range proxysqlAllColumns {
239 - if col.dbColumn == "" || available[strings.ToLower(col.dbColumn)] {
172 + if col.DBColumn == "" || available[strings.ToLower(col.DBColumn)] {
173 cols = append(cols, col)
174 }
175 }
@@ -252,42 +185,34 @@ func (c *Collector) topQueriesParams(ctx context.Context) ([]funcapi.ParamConfig
185 if len(cols) == 0 {
186 return nil, fmt.Errorf("no columns available in stats_mysql_query_digest")
187 }
255 - sortParam := funcapi.ParamConfig{
256 - ID: paramSort,
257 - Name: "Filter By",
258 - Help: "Select the primary sort column",
259 - Selection: funcapi.ParamSelect,
260 - Options: buildProxySQLSortOptions(cols),
261 - UniqueView: true,
262 - }
263 - return []funcapi.ParamConfig{sortParam}, nil
188 + return []funcapi.ParamConfig{buildProxySQLSortParam(cols)}, nil
189 }
190
266 -func (c *Collector) mapAndValidateProxySQLSortColumn(input string, available []proxysqlColumnMeta) string {
267 - availableKeys := make(map[string]bool, len(available))
268 - for _, col := range available {
269 - availableKeys[col.uiKey] = true
270 - }
271 - if availableKeys[input] {
191 +func (c *Collector) mapAndValidateProxySQLSortColumn(input string, cs funcapi.ColumnSet[proxysqlColumn]) string {
192 + if cs.ContainsColumn(input) {
193 return input
194 }
274 - if availableKeys["totalTime"] {
195 + if cs.ContainsColumn("totalTime") {
196 return "totalTime"
197 }
277 - if availableKeys["calls"] {
198 + if cs.ContainsColumn("calls") {
199 return "calls"
200 }
280 - return available[0].uiKey
201 + names := cs.Names()
202 + if len(names) > 0 {
203 + return names[0]
204 + }
205 + return ""
206 }
207
283 -func (c *Collector) buildProxySQLDynamicSQL(cols []proxysqlColumnMeta, sortColumn string, limit int) string {
208 +func (c *Collector) buildProxySQLDynamicSQL(cols []proxysqlColumn, sortColumn string, limit int) string {
209 selectParts := make([]string, 0, len(cols))
210 for _, col := range cols {
286 - expr := col.dbColumn
287 - if col.isMicroseconds {
288 - expr = fmt.Sprintf("%s/1000", col.dbColumn)
211 + expr := col.DBColumn
212 + if col.IsMicroseconds {
213 + expr = fmt.Sprintf("%s/1000", col.DBColumn)
214 }
290 - selectParts = append(selectParts, fmt.Sprintf("%s AS `%s`", expr, col.uiKey))
215 + selectParts = append(selectParts, fmt.Sprintf("%s AS `%s`", expr, col.Name))
216 }
217
218 return fmt.Sprintf(`
@@ -298,7 +223,7 @@ LIMIT %d
223 `, strings.Join(selectParts, ", "), sortColumn, limit)
224 }
225
301 -func (c *Collector) scanProxySQLDynamicRows(rows *sql.Rows, cols []proxysqlColumnMeta) ([][]any, error) {
226 +func (c *Collector) scanProxySQLDynamicRows(rows *sql.Rows, cols []proxysqlColumn) ([][]any, error) {
227 data := make([][]any, 0, 500)
228
229 valuePtrs := make([]any, len(cols))
@@ -306,14 +231,14 @@ func (c *Collector) scanProxySQLDynamicRows(rows *sql.Rows, cols []proxysqlColum
231
232 for rows.Next() {
233 for i, col := range cols {
309 - switch col.dataType {
310 - case ftString:
234 + switch col.Type {
235 + case funcapi.FieldTypeString:
236 var v sql.NullString
237 values[i] = &v
313 - case ftInteger:
238 + case funcapi.FieldTypeInteger:
239 var v sql.NullInt64
240 values[i] = &v
316 - case ftDuration:
241 + case funcapi.FieldTypeDuration:
242 var v sql.NullFloat64
243 values[i] = &v
244 default:
@@ -333,7 +258,7 @@ func (c *Collector) scanProxySQLDynamicRows(rows *sql.Rows, cols []proxysqlColum
258 case *sql.NullString:
259 if v.Valid {
260 s := v.String
336 - if col.uiKey == "query" {
261 + if col.Name == "query" {
262 s = strmutil.TruncateText(s, proxysqlMaxQueryTextLength)
263 }
264 row[i] = s
@@ -366,18 +291,19 @@ func (c *Collector) scanProxySQLDynamicRows(rows *sql.Rows, cols []proxysqlColum
291 return data, nil
292 }
293
369 -func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
294 +func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *funcapi.FunctionResponse {
295 availableCols, err := c.detectProxySQLDigestColumns(ctx)
296 if err != nil {
372 - return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("failed to detect available columns: %v", err)}
297 + return &funcapi.FunctionResponse{Status: 500, Message: fmt.Sprintf("failed to detect available columns: %v", err)}
298 }
299
300 cols := c.buildAvailableProxySQLColumns(availableCols)
301 if len(cols) == 0 {
377 - return &module.FunctionResponse{Status: 500, Message: "no columns available in stats_mysql_query_digest"}
302 + return &funcapi.FunctionResponse{Status: 500, Message: "no columns available in stats_mysql_query_digest"}
303 }
304
380 - sortColumn = c.mapAndValidateProxySQLSortColumn(sortColumn, cols)
305 + cs := proxysqlColumnSet(cols)
306 + sortColumn = c.mapAndValidateProxySQLSortColumn(sortColumn, cs)
307
308 limit := c.TopQueriesLimit
309 if limit <= 0 {
@@ -388,140 +314,29 @@ func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *m
314 rows, err := c.db.QueryContext(ctx, query)
315 if err != nil {
316 if ctx.Err() == context.DeadlineExceeded {
391 - return &module.FunctionResponse{Status: 504, Message: "query timed out"}
317 + return &funcapi.FunctionResponse{Status: 504, Message: "query timed out"}
318 }
393 - return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("query failed: %v", err)}
319 + return &funcapi.FunctionResponse{Status: 500, Message: fmt.Sprintf("query failed: %v", err)}
320 }
321 defer rows.Close()
322
323 data, err := c.scanProxySQLDynamicRows(rows, cols)
324 if err != nil {
399 - return &module.FunctionResponse{Status: 500, Message: err.Error()}
400 - }
401 -
402 - sortParam := funcapi.ParamConfig{
403 - ID: paramSort,
404 - Name: "Filter By",
405 - Help: "Select the primary sort column",
406 - Selection: funcapi.ParamSelect,
407 - Options: buildProxySQLSortOptions(cols),
408 - UniqueView: true,
325 + return &funcapi.FunctionResponse{Status: 500, Message: err.Error()}
326 }
327
328 defaultSort := "totalTime"
412 - if !containsProxySQLColumn(cols, defaultSort) {
329 + if !cs.ContainsColumn(defaultSort) {
330 defaultSort = "calls"
331 }
332
416 - return &module.FunctionResponse{
333 + return &funcapi.FunctionResponse{
334 Status: 200,
335 Help: "Top SQL queries from ProxySQL stats_mysql_query_digest",
419 - Columns: buildProxySQLColumns(cols),
336 + Columns: cs.BuildColumns(),
337 Data: data,
338 DefaultSortColumn: defaultSort,
422 - RequiredParams: []funcapi.ParamConfig{sortParam},
423 - Charts: proxysqlTopQueriesCharts(cols),
424 - DefaultCharts: proxysqlTopQueriesDefaultCharts(cols),
425 - GroupBy: proxysqlTopQueriesGroupBy(cols),
426 - }
427 -}
428 -
429 -func containsProxySQLColumn(cols []proxysqlColumnMeta, key string) bool {
430 - for _, col := range cols {
431 - if col.uiKey == key {
432 - return true
433 - }
434 - }
435 - return false
436 -}
437 -
438 -func proxysqlTopQueriesCharts(cols []proxysqlColumnMeta) map[string]module.ChartConfig {
439 - charts := make(map[string]module.ChartConfig)
440 - for _, col := range cols {
441 - if !col.isMetric || col.chartGroup == "" {
442 - continue
443 - }
444 - cfg, ok := charts[col.chartGroup]
445 - if !ok {
446 - title := col.chartTitle
447 - if title == "" {
448 - title = col.chartGroup
449 - }
450 - cfg = module.ChartConfig{
451 - Name: title,
452 - Type: "stacked-bar",
453 - }
454 - }
455 - cfg.Columns = append(cfg.Columns, col.uiKey)
456 - charts[col.chartGroup] = cfg
457 - }
458 - return charts
459 -}
460 -
461 -func proxysqlTopQueriesDefaultCharts(cols []proxysqlColumnMeta) [][]string {
462 - label := primaryProxySQLLabel(cols)
463 - if label == "" {
464 - return nil
465 - }
466 - chartGroups := defaultProxySQLChartGroups(cols)
467 - out := make([][]string, 0, len(chartGroups))
468 - for _, group := range chartGroups {
469 - out = append(out, []string{group, label})
470 - }
471 - return out
472 -}
473 -
474 -func proxysqlTopQueriesGroupBy(cols []proxysqlColumnMeta) map[string]module.GroupByConfig {
475 - groupBy := make(map[string]module.GroupByConfig)
476 - for _, col := range cols {
477 - if !col.isLabel {
478 - continue
479 - }
480 - groupBy[col.uiKey] = module.GroupByConfig{
481 - Name: "Group by " + col.displayName,
482 - Columns: []string{col.uiKey},
483 - }
484 - }
485 - return groupBy
486 -}
487 -
488 -func primaryProxySQLLabel(cols []proxysqlColumnMeta) string {
489 - for _, col := range cols {
490 - if col.isPrimary {
491 - return col.uiKey
492 - }
493 - }
494 - for _, col := range cols {
495 - if col.isLabel {
496 - return col.uiKey
497 - }
498 - }
499 - return ""
500 -}
501 -
502 -func defaultProxySQLChartGroups(cols []proxysqlColumnMeta) []string {
503 - groups := make([]string, 0)
504 - seen := make(map[string]bool)
505 - for _, col := range cols {
506 - if !col.isMetric || col.chartGroup == "" || !col.isDefaultChart {
507 - continue
508 - }
509 - if !seen[col.chartGroup] {
510 - seen[col.chartGroup] = true
511 - groups = append(groups, col.chartGroup)
512 - }
513 - }
514 - if len(groups) > 0 {
515 - return groups
516 - }
517 - for _, col := range cols {
518 - if !col.isMetric || col.chartGroup == "" {
519 - continue
520 - }
521 - if !seen[col.chartGroup] {
522 - seen[col.chartGroup] = true
523 - groups = append(groups, col.chartGroup)
524 - }
339 + RequiredParams: []funcapi.ParamConfig{buildProxySQLSortParam(cols)},
340 + ChartingConfig: cs.BuildCharting(),
341 }
526 - return groups
342 }
src/go/plugin/go.d/collector/proxysql/functions_test.go
+22 -24
@@ -32,43 +32,41 @@ func TestProxySQLMethods(t *testing.T) {
32 func TestProxySQLAllColumns_HasRequiredColumns(t *testing.T) {
33 required := []string{"digest", "query", "calls", "totalTime"}
34
35 - uiKeys := make(map[string]bool)
36 - for _, col := range proxysqlAllColumns {
37 - uiKeys[col.uiKey] = true
38 - }
35 + cs := proxysqlColumnSet(proxysqlAllColumns)
36
40 - for _, key := range required {
41 - assert.True(t, uiKeys[key], "column %s should be defined", key)
37 + for _, id := range required {
38 + assert.True(t, cs.ContainsColumn(id), "column %s should be defined", id)
39 }
40 }
41
42 func TestCollector_mapAndValidateProxySQLSortColumn(t *testing.T) {
43 tests := map[string]struct {
47 - available []proxysqlColumnMeta
48 - input string
49 - expected string
44 + columns []proxysqlColumn
45 + input string
46 + expected string
47 }{
48 "valid totalTime": {
52 - available: []proxysqlColumnMeta{{uiKey: "totalTime"}, {uiKey: "calls"}},
53 - input: "totalTime",
54 - expected: "totalTime",
49 + columns: []proxysqlColumn{{ColumnMeta: funcapi.ColumnMeta{Name: "totalTime"}}, {ColumnMeta: funcapi.ColumnMeta{Name: "calls"}}},
50 + input: "totalTime",
51 + expected: "totalTime",
52 },
53 "invalid falls back to totalTime": {
57 - available: []proxysqlColumnMeta{{uiKey: "totalTime"}, {uiKey: "calls"}},
58 - input: "bad",
59 - expected: "totalTime",
54 + columns: []proxysqlColumn{{ColumnMeta: funcapi.ColumnMeta{Name: "totalTime"}}, {ColumnMeta: funcapi.ColumnMeta{Name: "calls"}}},
55 + input: "bad",
56 + expected: "totalTime",
57 },
58 "fallback to calls": {
62 - available: []proxysqlColumnMeta{{uiKey: "calls"}},
63 - input: "bad",
64 - expected: "calls",
59 + columns: []proxysqlColumn{{ColumnMeta: funcapi.ColumnMeta{Name: "calls"}}},
60 + input: "bad",
61 + expected: "calls",
62 },
63 }
64
65 for name, tc := range tests {
66 t.Run(name, func(t *testing.T) {
67 c := &Collector{}
71 - assert.Equal(t, tc.expected, c.mapAndValidateProxySQLSortColumn(tc.input, tc.available))
68 + cs := proxysqlColumnSet(tc.columns)
69 + assert.Equal(t, tc.expected, c.mapAndValidateProxySQLSortColumn(tc.input, cs))
70 })
71 }
72 }
@@ -76,11 +74,11 @@ func TestCollector_mapAndValidateProxySQLSortColumn(t *testing.T) {
74 func TestCollector_buildProxySQLDynamicSQL(t *testing.T) {
75 c := &Collector{}
76
79 - cols := []proxysqlColumnMeta{
80 - {dbColumn: "digest", uiKey: "digest", dataType: ftString},
81 - {dbColumn: "digest_text", uiKey: "query", dataType: ftString},
82 - {dbColumn: "count_star", uiKey: "calls", dataType: ftInteger},
83 - {dbColumn: "sum_time", uiKey: "totalTime", dataType: ftDuration, isMicroseconds: true},
77 + cols := []proxysqlColumn{
78 + {ColumnMeta: funcapi.ColumnMeta{Name: "digest", Type: funcapi.FieldTypeString}, DBColumn: "digest"},
79 + {ColumnMeta: funcapi.ColumnMeta{Name: "query", Type: funcapi.FieldTypeString}, DBColumn: "digest_text"},
80 + {ColumnMeta: funcapi.ColumnMeta{Name: "calls", Type: funcapi.FieldTypeInteger}, DBColumn: "count_star"},
81 + {ColumnMeta: funcapi.ColumnMeta{Name: "totalTime", Type: funcapi.FieldTypeDuration}, DBColumn: "sum_time", IsMicroseconds: true},
82 }
83
84 sql := c.buildProxySQLDynamicSQL(cols, "totalTime", 500)
src/go/plugin/go.d/collector/redis/collector.go
+10 -4
@@ -34,13 +34,12 @@ func init() {
34 Create: func() module.Module { return New() },
35 Config: func() any { return &Config{} },
36 Methods: redisMethods,
37 - MethodParams: redisMethodParams,
38 - HandleMethod: redisHandleMethod,
37 + MethodHandler: redisFunctionHandler,
38 })
39 }
40
41 func New() *Collector {
43 - return &Collector{
42 + c := &Collector{
43 Config: Config{
44 Address: "redis://@localhost:6379",
45 Timeout: confopt.Duration(time.Second),
@@ -53,6 +52,8 @@ func New() *Collector {
52 collectedCommands: make(map[string]bool),
53 collectedDbs: make(map[string]bool),
54 }
55 + c.funcRouter = newFuncRouter(c)
56 + return c
57 }
58
59 type Config struct {
@@ -79,6 +80,8 @@ type (
80
81 rdb redisClient
82
83 + funcRouter *funcRouter
84 +
85 server string
86 version *semver.Version
87 pingSummary metrix.Summary
@@ -145,7 +148,10 @@ func (c *Collector) Collect(context.Context) map[string]int64 {
148 return ms
149 }
150
148 -func (c *Collector) Cleanup(context.Context) {
151 +func (c *Collector) Cleanup(ctx context.Context) {
152 + if c.funcRouter != nil {
153 + c.funcRouter.Cleanup(ctx)
154 + }
155 if c.rdb == nil {
156 return
157 }
src/go/plugin/go.d/collector/redis/func_router.go new
+64
@@ -0,0 +1,64 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package redis
4 +
5 +import (
6 + "context"
7 + "fmt"
8 +
9 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
10 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
11 +)
12 +
13 +// funcRouter routes method calls to appropriate function handlers.
14 +type funcRouter struct {
15 + collector *Collector
16 +
17 + handlers map[string]funcapi.MethodHandler
18 +}
19 +
20 +func newFuncRouter(c *Collector) *funcRouter {
21 + r := &funcRouter{
22 + collector: c,
23 + handlers: make(map[string]funcapi.MethodHandler),
24 + }
25 + r.handlers[topQueriesMethodID] = newFuncTopQueries(r)
26 + return r
27 +}
28 +
29 +// Compile-time interface check.
30 +var _ funcapi.MethodHandler = (*funcRouter)(nil)
31 +
32 +func (r *funcRouter) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
33 + if h, ok := r.handlers[method]; ok {
34 + return h.MethodParams(ctx, method)
35 + }
36 + return nil, fmt.Errorf("unknown method: %s", method)
37 +}
38 +
39 +func (r *funcRouter) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
40 + if h, ok := r.handlers[method]; ok {
41 + return h.Handle(ctx, method, params)
42 + }
43 + return funcapi.NotFoundResponse(method)
44 +}
45 +
46 +func (r *funcRouter) Cleanup(ctx context.Context) {
47 + for _, h := range r.handlers {
48 + h.Cleanup(ctx)
49 + }
50 +}
51 +
52 +func redisMethods() []funcapi.MethodConfig {
53 + return []funcapi.MethodConfig{
54 + topQueriesMethodConfig(),
55 + }
56 +}
57 +
58 +func redisFunctionHandler(job *module.Job) funcapi.MethodHandler {
59 + c, ok := job.Module().(*Collector)
60 + if !ok {
61 + return nil
62 + }
63 + return c.funcRouter
64 +}
src/go/plugin/go.d/collector/redis/func_top_queries.go new
+215
@@ -0,0 +1,215 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package redis
4 +
5 +import (
6 + "context"
7 + "fmt"
8 + "sort"
9 + "strings"
10 + "time"
11 +
12 + "github.com/redis/go-redis/v9"
13 +
14 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
15 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
16 +)
17 +
18 +const (
19 + topQueriesMethodID = "top-queries"
20 + redisMaxQueryTextLength = 4096
21 +)
22 +
23 +func topQueriesMethodConfig() funcapi.MethodConfig {
24 + return funcapi.MethodConfig{
25 + ID: topQueriesMethodID,
26 + Name: "Top Queries",
27 + UpdateEvery: 10,
28 + Help: "Slow commands from Redis SLOWLOG. WARNING: Command arguments may contain unmasked literals (potential PII).",
29 + RequireCloud: true,
30 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(redisAllColumns)},
31 + }
32 +}
33 +
34 +type redisColumn struct {
35 + funcapi.ColumnMeta
36 + sortOpt bool // whether this column appears as a sort option
37 + sortLbl string // label for sort option dropdown
38 + defaultSort bool // default sort column
39 +}
40 +
41 +// funcapi.SortableColumn interface implementation for redisColumn.
42 +func (c redisColumn) IsSortOption() bool { return c.sortOpt }
43 +func (c redisColumn) SortLabel() string { return c.sortLbl }
44 +func (c redisColumn) IsDefaultSort() bool { return c.defaultSort }
45 +func (c redisColumn) ColumnName() string { return c.Name }
46 +func (c redisColumn) SortColumn() string { return "" }
47 +
48 +func redisColumnSet(cols []redisColumn) funcapi.ColumnSet[redisColumn] {
49 + return funcapi.Columns(cols, func(c redisColumn) funcapi.ColumnMeta { return c.ColumnMeta })
50 +}
51 +
52 +var redisAllColumns = []redisColumn{
53 + {ColumnMeta: funcapi.ColumnMeta{Name: "id", Tooltip: "ID", Type: funcapi.FieldTypeInteger, Visible: false, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformNumber, UniqueKey: true, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryCount}, sortOpt: true, sortLbl: "Top queries by ID"},
54 + {ColumnMeta: funcapi.ColumnMeta{Name: "timestamp", Tooltip: "Timestamp", Type: funcapi.FieldTypeTimestamp, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformDatetime, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax}, sortOpt: true, sortLbl: "Top queries by Timestamp"},
55 + {ColumnMeta: funcapi.ColumnMeta{Name: "command", Tooltip: "Command", Type: funcapi.FieldTypeString, Visible: true, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, Sticky: true, FullWidth: true, Wrap: true}},
56 + {ColumnMeta: funcapi.ColumnMeta{Name: "command_name", Tooltip: "Command Name", Type: funcapi.FieldTypeString, Visible: true, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, GroupBy: &funcapi.GroupByOptions{IsDefault: true}}, sortOpt: true, sortLbl: "Top queries by Command Name"},
57 + {ColumnMeta: funcapi.ColumnMeta{Name: "duration", Tooltip: "Duration", Type: funcapi.FieldTypeDuration, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualBar, Transform: funcapi.FieldTransformDuration, Units: "milliseconds", DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "Duration", Title: "Execution Time", IsDefault: true}}, sortOpt: true, sortLbl: "Top queries by Duration", defaultSort: true},
58 + {ColumnMeta: funcapi.ColumnMeta{Name: "client_addr", Tooltip: "Client Address", Type: funcapi.FieldTypeString, Visible: false, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, GroupBy: &funcapi.GroupByOptions{}}},
59 + {ColumnMeta: funcapi.ColumnMeta{Name: "client_name", Tooltip: "Client Name", Type: funcapi.FieldTypeString, Visible: false, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, GroupBy: &funcapi.GroupByOptions{}}},
60 +}
61 +
62 +// Compile-time interface check.
63 +var _ funcapi.MethodHandler = (*funcTopQueries)(nil)
64 +
65 +// funcTopQueries handles the "top-queries" function for Redis.
66 +type funcTopQueries struct {
67 + router *funcRouter
68 +}
69 +
70 +func newFuncTopQueries(r *funcRouter) *funcTopQueries {
71 + return &funcTopQueries{router: r}
72 +}
73 +
74 +// MethodParams implements funcapi.MethodHandler.
75 +func (f *funcTopQueries) MethodParams(_ context.Context, method string) ([]funcapi.ParamConfig, error) {
76 + if method != topQueriesMethodID {
77 + return nil, fmt.Errorf("unknown method: %s", method)
78 + }
79 + return []funcapi.ParamConfig{funcapi.BuildSortParam(redisAllColumns)}, nil
80 +}
81 +
82 +// Handle implements funcapi.MethodHandler.
83 +func (f *funcTopQueries) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
84 + if method != topQueriesMethodID {
85 + return funcapi.NotFoundResponse(method)
86 + }
87 +
88 + if f.router.collector.rdb == nil {
89 + return funcapi.UnavailableResponse("collector is still initializing, please retry in a few seconds")
90 + }
91 +
92 + return f.collectTopQueries(ctx, params.Column("__sort"))
93 +}
94 +
95 +// Cleanup implements funcapi.MethodHandler.
96 +func (f *funcTopQueries) Cleanup(ctx context.Context) {}
97 +
98 +func (f *funcTopQueries) collectTopQueries(ctx context.Context, sortColumn string) *funcapi.FunctionResponse {
99 + c := f.router.collector
100 +
101 + limit := c.TopQueriesLimit
102 + if limit <= 0 {
103 + limit = 500
104 + }
105 +
106 + entries, err := c.rdb.SlowLogGet(ctx, -1).Result()
107 + if err != nil {
108 + if ctx.Err() == context.DeadlineExceeded {
109 + return &funcapi.FunctionResponse{Status: 504, Message: "query timed out"}
110 + }
111 + return &funcapi.FunctionResponse{Status: 500, Message: fmt.Sprintf("slowlog query failed: %v", err)}
112 + }
113 +
114 + cs := redisColumnSet(redisAllColumns)
115 + sortParam := funcapi.BuildSortParam(redisAllColumns)
116 +
117 + if len(entries) == 0 {
118 + return &funcapi.FunctionResponse{
119 + Status: 200,
120 + Message: "No slow commands found. SLOWLOG may be empty or disabled.",
121 + Help: "Slow commands from Redis SLOWLOG. WARNING: Command arguments may contain unmasked literals (potential PII).",
122 + Columns: cs.BuildColumns(),
123 + Data: [][]any{},
124 + DefaultSortColumn: "duration",
125 + RequiredParams: []funcapi.ParamConfig{sortParam},
126 + ChartingConfig: cs.BuildCharting(),
127 + }
128 + }
129 +
130 + sortColumn = mapRedisSortColumn(sortColumn)
131 + sortRedisSlowLogs(entries, sortColumn)
132 +
133 + if len(entries) > limit {
134 + entries = entries[:limit]
135 + }
136 +
137 + data := make([][]any, 0, len(entries))
138 + for _, entry := range entries {
139 + command := strings.Join(entry.Args, " ")
140 + commandName := ""
141 + if len(entry.Args) > 0 {
142 + commandName = entry.Args[0]
143 + }
144 +
145 + row := make([]any, len(redisAllColumns))
146 + for i, col := range redisAllColumns {
147 + switch col.Name {
148 + case "id":
149 + row[i] = entry.ID
150 + case "timestamp":
151 + row[i] = entry.Time.Format(time.RFC3339Nano)
152 + case "command":
153 + row[i] = strmutil.TruncateText(command, redisMaxQueryTextLength)
154 + case "command_name":
155 + row[i] = commandName
156 + case "duration":
157 + row[i] = float64(entry.Duration) / float64(time.Millisecond)
158 + case "client_addr":
159 + row[i] = entry.ClientAddr
160 + case "client_name":
161 + row[i] = entry.ClientName
162 + default:
163 + row[i] = nil
164 + }
165 + }
166 + data = append(data, row)
167 + }
168 +
169 + return &funcapi.FunctionResponse{
170 + Status: 200,
171 + Help: "Slow commands from Redis SLOWLOG. WARNING: Command arguments may contain unmasked literals (potential PII).",
172 + Columns: cs.BuildColumns(),
173 + Data: data,
174 + DefaultSortColumn: "duration",
175 + RequiredParams: []funcapi.ParamConfig{sortParam},
176 + ChartingConfig: cs.BuildCharting(),
177 + }
178 +}
179 +
180 +func mapRedisSortColumn(col string) string {
181 + switch col {
182 + case "duration", "timestamp", "id", "command_name":
183 + return col
184 + default:
185 + return "duration"
186 + }
187 +}
188 +
189 +func sortRedisSlowLogs(entries []redis.SlowLog, sortColumn string) {
190 + switch sortColumn {
191 + case "timestamp":
192 + sort.Slice(entries, func(i, j int) bool {
193 + return entries[i].Time.After(entries[j].Time)
194 + })
195 + case "id":
196 + sort.Slice(entries, func(i, j int) bool {
197 + return entries[i].ID > entries[j].ID
198 + })
199 + case "command_name":
200 + sort.Slice(entries, func(i, j int) bool {
201 + var a, b string
202 + if len(entries[i].Args) > 0 {
203 + a = entries[i].Args[0]
204 + }
205 + if len(entries[j].Args) > 0 {
206 + b = entries[j].Args[0]
207 + }
208 + return a < b
209 + })
210 + default:
211 + sort.Slice(entries, func(i, j int) bool {
212 + return entries[i].Duration > entries[j].Duration
213 + })
214 + }
215 +}
src/go/plugin/go.d/collector/redis/functions.go deleted
-407
@@ -1,407 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -package redis
4 -
5 -import (
6 - "context"
7 - "fmt"
8 - "sort"
9 - "strings"
10 - "time"
11 -
12 - "github.com/redis/go-redis/v9"
13 -
14 - "github.com/netdata/netdata/go/plugins/pkg/funcapi"
15 - "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
16 - "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
17 -)
18 -
19 -const redisMaxQueryTextLength = 4096
20 -
21 -const (
22 - paramSort = "__sort"
23 -
24 - ftString = funcapi.FieldTypeString
25 - ftInteger = funcapi.FieldTypeInteger
26 - ftDuration = funcapi.FieldTypeDuration
27 - ftTimestamp = funcapi.FieldTypeTimestamp
28 - trNone = funcapi.FieldTransformNone
29 - trNumber = funcapi.FieldTransformNumber
30 - trDuration = funcapi.FieldTransformDuration
31 - trDatetime = funcapi.FieldTransformDatetime
32 - trText = funcapi.FieldTransformText
33 -
34 - visValue = funcapi.FieldVisualValue
35 - visBar = funcapi.FieldVisualBar
36 -
37 - sortAsc = funcapi.FieldSortAscending
38 - sortDesc = funcapi.FieldSortDescending
39 -
40 - summaryCount = funcapi.FieldSummaryCount
41 - summaryMax = funcapi.FieldSummaryMax
42 - summarySum = funcapi.FieldSummarySum
43 -
44 - filterMulti = funcapi.FieldFilterMultiselect
45 - filterRange = funcapi.FieldFilterRange
46 -)
47 -
48 -type redisColumnMeta struct {
49 - id string
50 - name string
51 - colType funcapi.FieldType
52 - visible bool
53 - sortable bool
54 - fullWidth bool
55 - wrap bool
56 - sticky bool
57 - filter funcapi.FieldFilter
58 - visualization funcapi.FieldVisual
59 - transform funcapi.FieldTransform
60 - units string
61 - decimalPoints int
62 - uniqueKey bool
63 - sortDir funcapi.FieldSort
64 - summary funcapi.FieldSummary
65 - isLabel bool
66 - isPrimary bool
67 - isMetric bool
68 - chartGroup string
69 - chartTitle string
70 - isDefaultChart bool
71 -}
72 -
73 -var redisAllColumns = []redisColumnMeta{
74 - {id: "id", name: "ID", colType: ftInteger, visible: false, sortable: true, filter: filterRange, visualization: visValue, transform: trNumber, uniqueKey: true, sortDir: sortDesc, summary: summaryCount},
75 - {id: "timestamp", name: "Timestamp", colType: ftTimestamp, visible: true, sortable: true, filter: filterRange, visualization: visValue, transform: trDatetime, sortDir: sortDesc, summary: summaryMax},
76 - {id: "command", name: "Command", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, sticky: true, fullWidth: true, wrap: true},
77 - {id: "command_name", name: "Command Name", colType: ftString, visible: true, sortable: true, filter: filterMulti, visualization: visValue, transform: trText, sortDir: sortAsc, isLabel: true, isPrimary: true},
78 - {id: "duration", name: "Duration", colType: ftDuration, visible: true, sortable: true, filter: filterRange, visualization: visBar, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summarySum, isMetric: true, chartGroup: "Duration", chartTitle: "Execution Time", isDefaultChart: true},
79 - {id: "client_addr", name: "Client Address", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, isLabel: true},
80 - {id: "client_name", name: "Client Name", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, isLabel: true},
81 -}
82 -
83 -func redisMethods() []module.MethodConfig {
84 - sortOptions := buildRedisSortOptions(redisAllColumns)
85 - return []module.MethodConfig{
86 - {
87 - UpdateEvery: 10,
88 - ID: "top-queries",
89 - Name: "Top Queries",
90 - Help: "Slow commands from Redis SLOWLOG. WARNING: Command arguments may contain unmasked literals (potential PII).",
91 - RequireCloud: true,
92 - RequiredParams: []funcapi.ParamConfig{{
93 - ID: paramSort,
94 - Name: "Filter By",
95 - Help: "Select the primary sort column",
96 - Selection: funcapi.ParamSelect,
97 - Options: sortOptions,
98 - UniqueView: true,
99 - }},
100 - },
101 - }
102 -}
103 -
104 -func redisMethodParams(_ context.Context, _ *module.Job, method string) ([]funcapi.ParamConfig, error) {
105 - switch method {
106 - case "top-queries":
107 - return []funcapi.ParamConfig{buildRedisSortParam(redisAllColumns)}, nil
108 - default:
109 - return nil, fmt.Errorf("unknown method: %s", method)
110 - }
111 -}
112 -
113 -func redisHandleMethod(ctx context.Context, job *module.Job, method string, params funcapi.ResolvedParams) *module.FunctionResponse {
114 - collector, ok := job.Module().(*Collector)
115 - if !ok {
116 - return &module.FunctionResponse{Status: 500, Message: "internal error: invalid module type"}
117 - }
118 -
119 - if collector.rdb == nil {
120 - return &module.FunctionResponse{
121 - Status: 503,
122 - Message: "collector is still initializing, please retry in a few seconds",
123 - }
124 - }
125 -
126 - switch method {
127 - case "top-queries":
128 - return collector.collectTopQueries(ctx, params.Column(paramSort))
129 - default:
130 - return &module.FunctionResponse{Status: 404, Message: fmt.Sprintf("unknown method: %s", method)}
131 - }
132 -}
133 -
134 -func buildRedisSortOptions(cols []redisColumnMeta) []funcapi.ParamOption {
135 - var sortOptions []funcapi.ParamOption
136 - sortDir := funcapi.FieldSortDescending
137 - for _, col := range cols {
138 - if !col.sortable {
139 - continue
140 - }
141 - opt := funcapi.ParamOption{
142 - ID: col.id,
143 - Column: col.id,
144 - Name: fmt.Sprintf("Top queries by %s", col.name),
145 - Sort: &sortDir,
146 - }
147 - if col.id == "duration" {
148 - opt.Default = true
149 - }
150 - sortOptions = append(sortOptions, opt)
151 - }
152 - return sortOptions
153 -}
154 -
155 -func buildRedisSortParam(cols []redisColumnMeta) funcapi.ParamConfig {
156 - return funcapi.ParamConfig{
157 - ID: paramSort,
158 - Name: "Filter By",
159 - Help: "Select the primary sort column",
160 - Selection: funcapi.ParamSelect,
161 - Options: buildRedisSortOptions(cols),
162 - UniqueView: true,
163 - }
164 -}
165 -
166 -func buildRedisColumns(cols []redisColumnMeta) map[string]any {
167 - result := make(map[string]any, len(cols))
168 - for i, col := range cols {
169 - colDef := funcapi.Column{
170 - Index: i,
171 - Name: col.name,
172 - Type: col.colType,
173 - Units: col.units,
174 - Visualization: col.visualization,
175 - Sort: col.sortDir,
176 - Sortable: col.sortable,
177 - Sticky: col.sticky,
178 - Summary: col.summary,
179 - Filter: col.filter,
180 - FullWidth: col.fullWidth,
181 - Wrap: col.wrap,
182 - DefaultExpandedFilter: false,
183 - UniqueKey: col.uniqueKey,
184 - Visible: col.visible,
185 - ValueOptions: funcapi.ValueOptions{
186 - Transform: col.transform,
187 - DecimalPoints: col.decimalPoints,
188 - DefaultValue: nil,
189 - },
190 - }
191 - result[col.id] = colDef.BuildColumn()
192 - }
193 - return result
194 -}
195 -
196 -func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
197 - limit := c.TopQueriesLimit
198 - if limit <= 0 {
199 - limit = 500
200 - }
201 -
202 - entries, err := c.rdb.SlowLogGet(ctx, -1).Result()
203 - if err != nil {
204 - if ctx.Err() == context.DeadlineExceeded {
205 - return &module.FunctionResponse{Status: 504, Message: "query timed out"}
206 - }
207 - return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("slowlog query failed: %v", err)}
208 - }
209 -
210 - if len(entries) == 0 {
211 - return &module.FunctionResponse{
212 - Status: 200,
213 - Message: "No slow commands found. SLOWLOG may be empty or disabled.",
214 - Help: "Slow commands from Redis SLOWLOG. WARNING: Command arguments may contain unmasked literals (potential PII).",
215 - Columns: buildRedisColumns(redisAllColumns),
216 - Data: [][]any{},
217 - DefaultSortColumn: "duration",
218 - RequiredParams: []funcapi.ParamConfig{buildRedisSortParam(redisAllColumns)},
219 - Charts: redisTopQueriesCharts(redisAllColumns),
220 - DefaultCharts: redisTopQueriesDefaultCharts(redisAllColumns),
221 - GroupBy: redisTopQueriesGroupBy(redisAllColumns),
222 - }
223 - }
224 -
225 - sortColumn = mapRedisSortColumn(sortColumn)
226 - sortRedisSlowLogs(entries, sortColumn)
227 -
228 - if len(entries) > limit {
229 - entries = entries[:limit]
230 - }
231 -
232 - data := make([][]any, 0, len(entries))
233 - for _, entry := range entries {
234 - command := strings.Join(entry.Args, " ")
235 - commandName := ""
236 - if len(entry.Args) > 0 {
237 - commandName = entry.Args[0]
238 - }
239 -
240 - row := make([]any, len(redisAllColumns))
241 - for i, col := range redisAllColumns {
242 - switch col.id {
243 - case "id":
244 - row[i] = entry.ID
245 - case "timestamp":
246 - row[i] = entry.Time.Format(time.RFC3339Nano)
247 - case "command":
248 - row[i] = strmutil.TruncateText(command, redisMaxQueryTextLength)
249 - case "command_name":
250 - row[i] = commandName
251 - case "duration":
252 - row[i] = float64(entry.Duration) / float64(time.Millisecond)
253 - case "client_addr":
254 - row[i] = entry.ClientAddr
255 - case "client_name":
256 - row[i] = entry.ClientName
257 - default:
258 - row[i] = nil
259 - }
260 - }
261 - data = append(data, row)
262 - }
263 -
264 - return &module.FunctionResponse{
265 - Status: 200,
266 - Help: "Slow commands from Redis SLOWLOG. WARNING: Command arguments may contain unmasked literals (potential PII).",
267 - Columns: buildRedisColumns(redisAllColumns),
268 - Data: data,
269 - DefaultSortColumn: "duration",
270 - RequiredParams: []funcapi.ParamConfig{buildRedisSortParam(redisAllColumns)},
271 - Charts: redisTopQueriesCharts(redisAllColumns),
272 - DefaultCharts: redisTopQueriesDefaultCharts(redisAllColumns),
273 - GroupBy: redisTopQueriesGroupBy(redisAllColumns),
274 - }
275 -}
276 -
277 -func mapRedisSortColumn(col string) string {
278 - switch col {
279 - case "duration", "timestamp", "id", "command_name":
280 - return col
281 - default:
282 - return "duration"
283 - }
284 -}
285 -
286 -func sortRedisSlowLogs(entries []redis.SlowLog, sortColumn string) {
287 - switch sortColumn {
288 - case "timestamp":
289 - sort.Slice(entries, func(i, j int) bool {
290 - return entries[i].Time.After(entries[j].Time)
291 - })
292 - case "id":
293 - sort.Slice(entries, func(i, j int) bool {
294 - return entries[i].ID > entries[j].ID
295 - })
296 - case "command_name":
297 - sort.Slice(entries, func(i, j int) bool {
298 - var a, b string
299 - if len(entries[i].Args) > 0 {
300 - a = entries[i].Args[0]
301 - }
302 - if len(entries[j].Args) > 0 {
303 - b = entries[j].Args[0]
304 - }
305 - return a > b
306 - })
307 - default:
308 - sort.Slice(entries, func(i, j int) bool {
309 - return entries[i].Duration > entries[j].Duration
310 - })
311 - }
312 -}
313 -
314 -func redisTopQueriesCharts(cols []redisColumnMeta) map[string]module.ChartConfig {
315 - charts := make(map[string]module.ChartConfig)
316 - for _, col := range cols {
317 - if !col.isMetric || col.chartGroup == "" {
318 - continue
319 - }
320 - cfg, ok := charts[col.chartGroup]
321 - if !ok {
322 - title := col.chartTitle
323 - if title == "" {
324 - title = col.chartGroup
325 - }
326 - cfg = module.ChartConfig{
327 - Name: title,
328 - Type: "stacked-bar",
329 - }
330 - }
331 - cfg.Columns = append(cfg.Columns, col.id)
332 - charts[col.chartGroup] = cfg
333 - }
334 - return charts
335 -}
336 -
337 -func redisTopQueriesDefaultCharts(cols []redisColumnMeta) [][]string {
338 - label := primaryRedisLabel(cols)
339 - if label == "" {
340 - return nil
341 - }
342 -
343 - chartGroups := defaultRedisChartGroups(cols)
344 - out := make([][]string, 0, len(chartGroups))
345 - for _, group := range chartGroups {
346 - out = append(out, []string{group, label})
347 - }
348 - return out
349 -}
350 -
351 -func redisTopQueriesGroupBy(cols []redisColumnMeta) map[string]module.GroupByConfig {
352 - groupBy := make(map[string]module.GroupByConfig)
353 - for _, col := range cols {
354 - if !col.isLabel {
355 - continue
356 - }
357 - groupBy[col.id] = module.GroupByConfig{
358 - Name: "Group by " + col.name,
359 - Columns: []string{col.id},
360 - }
361 - }
362 - return groupBy
363 -}
364 -
365 -func primaryRedisLabel(cols []redisColumnMeta) string {
366 - for _, col := range cols {
367 - if col.isPrimary {
368 - return col.id
369 - }
370 - }
371 - for _, col := range cols {
372 - if col.isLabel {
373 - return col.id
374 - }
375 - }
376 - return ""
377 -}
378 -
379 -func defaultRedisChartGroups(cols []redisColumnMeta) []string {
380 - groups := make([]string, 0)
381 - seen := make(map[string]bool)
382 -
383 - for _, col := range cols {
384 - if !col.isMetric || col.chartGroup == "" || !col.isDefaultChart {
385 - continue
386 - }
387 - if !seen[col.chartGroup] {
388 - seen[col.chartGroup] = true
389 - groups = append(groups, col.chartGroup)
390 - }
391 - }
392 -
393 - if len(groups) > 0 {
394 - return groups
395 - }
396 -
397 - for _, col := range cols {
398 - if !col.isMetric || col.chartGroup == "" {
399 - continue
400 - }
401 - if !seen[col.chartGroup] {
402 - seen[col.chartGroup] = true
403 - groups = append(groups, col.chartGroup)
404 - }
405 - }
406 - return groups
407 -}
src/go/plugin/go.d/collector/redis/functions_test.go
+3 -7
@@ -42,13 +42,9 @@ func TestRedisMethods(t *testing.T) {
42 func TestRedisAllColumns_HasRequiredColumns(t *testing.T) {
43 required := []string{"timestamp", "command", "duration"}
44
45 - uiKeys := make(map[string]bool)
46 - for _, col := range redisAllColumns {
47 - uiKeys[col.id] = true
48 - }
49 -
50 - for _, key := range required {
51 - assert.True(t, uiKeys[key], "column %s should be defined", key)
45 + cs := redisColumnSet(redisAllColumns)
46 + for _, id := range required {
47 + assert.True(t, cs.ContainsColumn(id), "column %s should be defined", id)
48 }
49 }
50
src/go/plugin/go.d/collector/rethinkdb/collector.go
+12 -4
@@ -21,13 +21,12 @@ func init() {
21 Create: func() module.Module { return New() },
22 Config: func() any { return &Config{} },
23 Methods: rethinkdbMethods,
24 - MethodParams: rethinkdbMethodParams,
25 - HandleMethod: rethinkdbHandleMethod,
24 + MethodHandler: rethinkdbFunctionHandler,
25 })
26 }
27
28 func New() *Collector {
30 - return &Collector{
29 + c := &Collector{
30 Config: Config{
31 Address: "127.0.0.1:28015",
32 Timeout: confopt.Duration(time.Second * 1),
@@ -37,6 +36,10 @@ func New() *Collector {
36 newConn: newRethinkdbConn,
37 seenServers: make(map[string]bool),
38 }
39 +
40 + c.funcRouter = newFuncRouter(c)
41 +
42 + return c
43 }
44
45 type Config struct {
@@ -59,6 +62,8 @@ type Collector struct {
62 newConn func(cfg Config) (rdbConn, error)
63 rdb rdbConn
64
65 + funcRouter *funcRouter // function router for method handlers
66 +
67 seenServers map[string]bool
68 }
69
@@ -100,7 +105,10 @@ func (c *Collector) Collect(context.Context) map[string]int64 {
105 return ms
106 }
107
103 -func (c *Collector) Cleanup(context.Context) {
108 +func (c *Collector) Cleanup(ctx context.Context) {
109 + if c.funcRouter != nil {
110 + c.funcRouter.Cleanup(ctx)
111 + }
112 if c.rdb != nil {
113 if err := c.rdb.close(); err != nil {
114 c.Warningf("cleanup: error on closing client [%s]: %v", c.Address, err)
src/go/plugin/go.d/collector/rethinkdb/func_router.go new
+68
@@ -0,0 +1,68 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package rethinkdb
4 +
5 +import (
6 + "context"
7 + "fmt"
8 +
9 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
10 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
11 +)
12 +
13 +// funcRouter routes method calls to appropriate function handlers.
14 +type funcRouter struct {
15 + collector *Collector
16 +
17 + handlers map[string]funcapi.MethodHandler
18 +}
19 +
20 +func newFuncRouter(c *Collector) *funcRouter {
21 + r := &funcRouter{
22 + collector: c,
23 + handlers: make(map[string]funcapi.MethodHandler),
24 + }
25 + r.handlers[runningQueriesMethodID] = newFuncRunningQueries(r)
26 + return r
27 +}
28 +
29 +// Compile-time interface check.
30 +var _ funcapi.MethodHandler = (*funcRouter)(nil)
31 +
32 +func (r *funcRouter) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
33 + if h, ok := r.handlers[method]; ok {
34 + return h.MethodParams(ctx, method)
35 + }
36 + return nil, fmt.Errorf("unknown method: %s", method)
37 +}
38 +
39 +func (r *funcRouter) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
40 + if r.collector.rdb == nil {
41 + return funcapi.UnavailableResponse("collector is still initializing, please retry in a few seconds")
42 + }
43 +
44 + if h, ok := r.handlers[method]; ok {
45 + return h.Handle(ctx, method, params)
46 + }
47 + return funcapi.NotFoundResponse(method)
48 +}
49 +
50 +func (r *funcRouter) Cleanup(ctx context.Context) {
51 + for _, h := range r.handlers {
52 + h.Cleanup(ctx)
53 + }
54 +}
55 +
56 +func rethinkdbMethods() []funcapi.MethodConfig {
57 + return []funcapi.MethodConfig{
58 + runningQueriesMethodConfig(),
59 + }
60 +}
61 +
62 +func rethinkdbFunctionHandler(job *module.Job) funcapi.MethodHandler {
63 + c, ok := job.Module().(*Collector)
64 + if !ok {
65 + return nil
66 + }
67 + return c.funcRouter
68 +}
src/go/plugin/go.d/collector/rethinkdb/func_running_queries.go new
+278
@@ -0,0 +1,278 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package rethinkdb
4 +
5 +import (
6 + "context"
7 + "fmt"
8 + "sort"
9 + "strings"
10 +
11 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
12 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
13 +)
14 +
15 +const (
16 + runningQueriesMethodID = "running-queries"
17 + rethinkMaxQueryTextLength = 4096
18 +)
19 +
20 +func runningQueriesMethodConfig() funcapi.MethodConfig {
21 + return funcapi.MethodConfig{
22 + ID: runningQueriesMethodID,
23 + Name: "Running Queries",
24 + UpdateEvery: 10,
25 + Help: "Currently running queries from rethinkdb.jobs. WARNING: Query text may contain unmasked literals (potential PII).",
26 + RequireCloud: true,
27 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(rethinkRunningColumns)},
28 + }
29 +}
30 +
31 +// Compile-time interface check.
32 +var _ funcapi.MethodHandler = (*funcRunningQueries)(nil)
33 +
34 +// funcRunningQueries handles the "running-queries" function for RethinkDB.
35 +type funcRunningQueries struct {
36 + router *funcRouter
37 +}
38 +
39 +func newFuncRunningQueries(r *funcRouter) *funcRunningQueries {
40 + return &funcRunningQueries{router: r}
41 +}
42 +
43 +// MethodParams implements funcapi.MethodHandler.
44 +func (f *funcRunningQueries) Cleanup(ctx context.Context) {}
45 +
46 +func (f *funcRunningQueries) MethodParams(_ context.Context, method string) ([]funcapi.ParamConfig, error) {
47 + if method != runningQueriesMethodID {
48 + return nil, fmt.Errorf("unknown method: %s", method)
49 + }
50 + return []funcapi.ParamConfig{funcapi.BuildSortParam(rethinkRunningColumns)}, nil
51 +}
52 +
53 +// Handle implements funcapi.MethodHandler.
54 +func (f *funcRunningQueries) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
55 + if method != runningQueriesMethodID {
56 + return funcapi.NotFoundResponse(method)
57 + }
58 +
59 + return f.collectRunningQueries(ctx, params.Column("__sort"))
60 +}
61 +
62 +func (f *funcRunningQueries) collectRunningQueries(ctx context.Context, sortColumn string) *funcapi.FunctionResponse {
63 + c := f.router.collector
64 +
65 + limit := c.TopQueriesLimit
66 + if limit <= 0 {
67 + limit = 500
68 + }
69 +
70 + if ctx.Err() == context.DeadlineExceeded {
71 + return &funcapi.FunctionResponse{Status: 504, Message: "query timed out"}
72 + }
73 +
74 + rows, err := c.rdb.jobs(ctx)
75 + if err != nil {
76 + if ctx.Err() == context.DeadlineExceeded {
77 + return &funcapi.FunctionResponse{Status: 504, Message: "query timed out"}
78 + }
79 + return &funcapi.FunctionResponse{Status: 500, Message: fmt.Sprintf("jobs query failed: %v", err)}
80 + }
81 +
82 + jobRows := make([]rethinkJobRow, 0, len(rows))
83 + for _, row := range rows {
84 + jobRows = append(jobRows, parseRethinkJob(row))
85 + }
86 +
87 + cs := rethinkColumnSet(rethinkRunningColumns)
88 +
89 + if len(jobRows) == 0 {
90 + return &funcapi.FunctionResponse{
91 + Status: 200,
92 + Message: "No running queries found.",
93 + Help: "Currently running queries from rethinkdb.jobs",
94 + Columns: cs.BuildColumns(),
95 + Data: [][]any{},
96 + DefaultSortColumn: mapRethinkSortColumn(sortColumn),
97 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(rethinkRunningColumns)},
98 + }
99 + }
100 +
101 + sortColumn = mapRethinkSortColumn(sortColumn)
102 + sortRethinkRows(jobRows, sortColumn)
103 + if len(jobRows) > limit {
104 + jobRows = jobRows[:limit]
105 + }
106 +
107 + data := make([][]any, 0, len(jobRows))
108 + for _, row := range jobRows {
109 + out := make([]any, len(rethinkRunningColumns))
110 + for i, col := range rethinkRunningColumns {
111 + switch col.Name {
112 + case "jobId":
113 + out[i] = row.JobID
114 + case "query":
115 + out[i] = strmutil.TruncateText(row.Query, rethinkMaxQueryTextLength)
116 + case "durationMs":
117 + out[i] = row.DurationMs
118 + case "type":
119 + out[i] = row.Type
120 + case "user":
121 + out[i] = row.User
122 + case "clientAddress":
123 + out[i] = row.ClientAddress
124 + case "clientPort":
125 + out[i] = row.ClientPort
126 + case "servers":
127 + out[i] = row.Servers
128 + default:
129 + out[i] = nil
130 + }
131 + }
132 + data = append(data, out)
133 + }
134 +
135 + return &funcapi.FunctionResponse{
136 + Status: 200,
137 + Help: "Currently running queries from rethinkdb.jobs. WARNING: Query text may contain unmasked literals (potential PII).",
138 + Columns: cs.BuildColumns(),
139 + Data: data,
140 + DefaultSortColumn: sortColumn,
141 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(rethinkRunningColumns)},
142 + }
143 +}
144 +
145 +type rethinkColumn struct {
146 + funcapi.ColumnMeta
147 + sortOpt bool // whether this column appears as a sort option
148 + sortLbl string // label for sort option dropdown
149 + defaultSort bool // default sort column
150 +}
151 +
152 +// funcapi.SortableColumn interface implementation for rethinkColumn.
153 +func (c rethinkColumn) IsSortOption() bool { return c.sortOpt }
154 +func (c rethinkColumn) SortLabel() string { return c.sortLbl }
155 +func (c rethinkColumn) IsDefaultSort() bool { return c.defaultSort }
156 +func (c rethinkColumn) ColumnName() string { return c.Name }
157 +func (c rethinkColumn) SortColumn() string { return "" }
158 +
159 +func rethinkColumnSet(cols []rethinkColumn) funcapi.ColumnSet[rethinkColumn] {
160 + return funcapi.Columns(cols, func(c rethinkColumn) funcapi.ColumnMeta { return c.ColumnMeta })
161 +}
162 +
163 +var rethinkRunningColumns = []rethinkColumn{
164 + {ColumnMeta: funcapi.ColumnMeta{Name: "jobId", Tooltip: "Job ID", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, UniqueKey: true, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}},
165 + {ColumnMeta: funcapi.ColumnMeta{Name: "query", Tooltip: "Query", Type: funcapi.FieldTypeString, Visible: true, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, Sticky: true, FullWidth: true, Wrap: true}},
166 + {ColumnMeta: funcapi.ColumnMeta{Name: "durationMs", Tooltip: "Duration", Type: funcapi.FieldTypeDuration, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualBar, Transform: funcapi.FieldTransformDuration, Units: "milliseconds", DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax}, sortOpt: true, defaultSort: true, sortLbl: "Running queries by Duration"},
167 + {ColumnMeta: funcapi.ColumnMeta{Name: "type", Tooltip: "Type", Type: funcapi.FieldTypeString, Visible: true, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}},
168 + {ColumnMeta: funcapi.ColumnMeta{Name: "user", Tooltip: "User", Type: funcapi.FieldTypeString, Visible: true, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}},
169 + {ColumnMeta: funcapi.ColumnMeta{Name: "clientAddress", Tooltip: "Client Address", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}},
170 + {ColumnMeta: funcapi.ColumnMeta{Name: "clientPort", Tooltip: "Client Port", Type: funcapi.FieldTypeInteger, Visible: false, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax}},
171 + {ColumnMeta: funcapi.ColumnMeta{Name: "servers", Tooltip: "Servers", Type: funcapi.FieldTypeString, Visible: false, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText}},
172 +}
173 +
174 +type rethinkJobRow struct {
175 + JobID string
176 + Query string
177 + DurationMs float64
178 + Type string
179 + User string
180 + ClientAddress string
181 + ClientPort int64
182 + Servers string
183 +}
184 +
185 +func parseRethinkJob(row map[string]any) rethinkJobRow {
186 + info := mapStringAny(row["info"])
187 + query := fmt.Sprint(info["query"])
188 + user := fmt.Sprint(info["user"])
189 + clientAddr := fmt.Sprint(info["client_address"])
190 + clientPort := toInt64(info["client_port"])
191 +
192 + servers := ""
193 + if list, ok := row["servers"].([]any); ok {
194 + ss := make([]string, 0, len(list))
195 + for _, v := range list {
196 + ss = append(ss, fmt.Sprint(v))
197 + }
198 + servers = strings.Join(ss, ",")
199 + }
200 +
201 + return rethinkJobRow{
202 + JobID: fmt.Sprint(row["id"]),
203 + Query: query,
204 + DurationMs: toFloat64(row["duration_sec"]) * 1000,
205 + Type: fmt.Sprint(row["type"]),
206 + User: user,
207 + ClientAddress: clientAddr,
208 + ClientPort: clientPort,
209 + Servers: servers,
210 + }
211 +}
212 +
213 +func mapStringAny(v any) map[string]any {
214 + if m, ok := v.(map[string]any); ok {
215 + return m
216 + }
217 + return map[string]any{}
218 +}
219 +
220 +func toFloat64(v any) float64 {
221 + switch t := v.(type) {
222 + case float64:
223 + return t
224 + case float32:
225 + return float64(t)
226 + case int:
227 + return float64(t)
228 + case int64:
229 + return float64(t)
230 + case uint64:
231 + return float64(t)
232 + default:
233 + return 0
234 + }
235 +}
236 +
237 +func toInt64(v any) int64 {
238 + switch t := v.(type) {
239 + case int64:
240 + return t
241 + case int:
242 + return int64(t)
243 + case float64:
244 + return int64(t)
245 + case float32:
246 + return int64(t)
247 + default:
248 + return 0
249 + }
250 +}
251 +
252 +func mapRethinkSortColumn(input string) string {
253 + for _, col := range rethinkRunningColumns {
254 + if col.IsSortOption() && col.Name == input {
255 + return col.Name
256 + }
257 + }
258 + for _, col := range rethinkRunningColumns {
259 + if col.IsDefaultSort() {
260 + return col.Name
261 + }
262 + }
263 + for _, col := range rethinkRunningColumns {
264 + if col.IsSortOption() {
265 + return col.Name
266 + }
267 + }
268 + return ""
269 +}
270 +
271 +func sortRethinkRows(rows []rethinkJobRow, sortColumn string) {
272 + switch sortColumn {
273 + case "durationMs":
274 + sort.Slice(rows, func(i, j int) bool { return rows[i].DurationMs > rows[j].DurationMs })
275 + default:
276 + sort.Slice(rows, func(i, j int) bool { return rows[i].DurationMs > rows[j].DurationMs })
277 + }
278 +}
src/go/plugin/go.d/collector/rethinkdb/functions.go deleted
-378
@@ -1,378 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -package rethinkdb
4 -
5 -import (
6 - "context"
7 - "fmt"
8 - "sort"
9 - "strings"
10 -
11 - "github.com/netdata/netdata/go/plugins/pkg/funcapi"
12 - "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
13 - "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
14 -)
15 -
16 -const rethinkMaxQueryTextLength = 4096
17 -
18 -const (
19 - paramSort = "__sort"
20 -
21 - ftString = funcapi.FieldTypeString
22 - ftInteger = funcapi.FieldTypeInteger
23 - ftDuration = funcapi.FieldTypeDuration
24 -
25 - trNone = funcapi.FieldTransformNone
26 - trNumber = funcapi.FieldTransformNumber
27 - trDuration = funcapi.FieldTransformDuration
28 - trText = funcapi.FieldTransformText
29 -
30 - visValue = funcapi.FieldVisualValue
31 - visBar = funcapi.FieldVisualBar
32 -
33 - sortAsc = funcapi.FieldSortAscending
34 - sortDesc = funcapi.FieldSortDescending
35 -
36 - summaryCount = funcapi.FieldSummaryCount
37 - summaryMax = funcapi.FieldSummaryMax
38 -
39 - filterMulti = funcapi.FieldFilterMultiselect
40 - filterRange = funcapi.FieldFilterRange
41 -)
42 -
43 -type rethinkColumnMeta struct {
44 - id string
45 - name string
46 - colType funcapi.FieldType
47 - visible bool
48 - sortable bool
49 - fullWidth bool
50 - wrap bool
51 - sticky bool
52 - filter funcapi.FieldFilter
53 - visualization funcapi.FieldVisual
54 - transform funcapi.FieldTransform
55 - units string
56 - decimalPoints int
57 - uniqueKey bool
58 - sortDir funcapi.FieldSort
59 - summary funcapi.FieldSummary
60 - sortLabel string
61 - isSortOption bool
62 - isDefaultSort bool
63 -}
64 -
65 -var rethinkRunningColumns = []rethinkColumnMeta{
66 - {id: "jobId", name: "Job ID", colType: ftString, visible: false, sortable: true, filter: filterMulti, visualization: visValue, transform: trText, uniqueKey: true, sortDir: sortAsc, summary: summaryCount},
67 - {id: "query", name: "Query", colType: ftString, visible: true, sortable: false, filter: filterMulti, visualization: visValue, transform: trText, sticky: true, fullWidth: true, wrap: true},
68 - {id: "durationMs", name: "Duration", colType: ftDuration, visible: true, sortable: true, filter: filterRange, visualization: visBar, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, isSortOption: true, isDefaultSort: true, sortLabel: "Running queries by Duration"},
69 - {id: "type", name: "Type", colType: ftString, visible: true, sortable: true, filter: filterMulti, visualization: visValue, transform: trText, sortDir: sortAsc, summary: summaryCount},
70 - {id: "user", name: "User", colType: ftString, visible: true, sortable: true, filter: filterMulti, visualization: visValue, transform: trText, sortDir: sortAsc, summary: summaryCount},
71 - {id: "clientAddress", name: "Client Address", colType: ftString, visible: false, sortable: true, filter: filterMulti, visualization: visValue, transform: trText, sortDir: sortAsc, summary: summaryCount},
72 - {id: "clientPort", name: "Client Port", colType: ftInteger, visible: false, sortable: true, filter: filterRange, visualization: visValue, transform: trNumber, sortDir: sortDesc, summary: summaryMax},
73 - {id: "servers", name: "Servers", colType: ftString, visible: false, sortable: false, filter: filterMulti, visualization: visValue, transform: trText},
74 -}
75 -
76 -func rethinkdbMethods() []module.MethodConfig {
77 - sortOptions := buildRethinkSortOptions(rethinkRunningColumns)
78 - return []module.MethodConfig{
79 - {
80 - UpdateEvery: 10,
81 - ID: "running-queries",
82 - Name: "Running Queries",
83 - Help: "Currently running queries from rethinkdb.jobs. WARNING: Query text may contain unmasked literals (potential PII).",
84 - RequireCloud: true,
85 - RequiredParams: []funcapi.ParamConfig{
86 - {
87 - ID: paramSort,
88 - Name: "Filter By",
89 - Help: "Select the primary sort column",
90 - Selection: funcapi.ParamSelect,
91 - Options: sortOptions,
92 - UniqueView: true,
93 - }},
94 - },
95 - }
96 -}
97 -
98 -func rethinkdbMethodParams(_ context.Context, _ *module.Job, method string) ([]funcapi.ParamConfig, error) {
99 - switch method {
100 - case "running-queries":
101 - return []funcapi.ParamConfig{buildRethinkSortParam(rethinkRunningColumns)}, nil
102 - default:
103 - return nil, fmt.Errorf("unknown method: %s", method)
104 - }
105 -}
106 -
107 -func rethinkdbHandleMethod(ctx context.Context, job *module.Job, method string, params funcapi.ResolvedParams) *module.FunctionResponse {
108 - collector, ok := job.Module().(*Collector)
109 - if !ok {
110 - return &module.FunctionResponse{Status: 500, Message: "internal error: invalid module type"}
111 - }
112 -
113 - if collector.rdb == nil {
114 - conn, err := collector.newConn(collector.Config)
115 - if err != nil {
116 - return &module.FunctionResponse{Status: 503, Message: "collector is still initializing, please retry in a few seconds"}
117 - }
118 - collector.rdb = conn
119 - }
120 -
121 - switch method {
122 - case "running-queries":
123 - return collector.collectRunningQueries(ctx, params.Column(paramSort))
124 - default:
125 - return &module.FunctionResponse{Status: 404, Message: fmt.Sprintf("unknown method: %s", method)}
126 - }
127 -}
128 -
129 -func buildRethinkSortOptions(cols []rethinkColumnMeta) []funcapi.ParamOption {
130 - var sortOptions []funcapi.ParamOption
131 - sortDir := funcapi.FieldSortDescending
132 - for _, col := range cols {
133 - if !col.isSortOption {
134 - continue
135 - }
136 - opt := funcapi.ParamOption{
137 - ID: col.id,
138 - Column: col.id,
139 - Name: col.sortLabel,
140 - Sort: &sortDir,
141 - }
142 - if col.isDefaultSort {
143 - opt.Default = true
144 - }
145 - sortOptions = append(sortOptions, opt)
146 - }
147 - return sortOptions
148 -}
149 -
150 -func buildRethinkSortParam(cols []rethinkColumnMeta) funcapi.ParamConfig {
151 - return funcapi.ParamConfig{
152 - ID: paramSort,
153 - Name: "Filter By",
154 - Help: "Select the primary sort column",
155 - Selection: funcapi.ParamSelect,
156 - Options: buildRethinkSortOptions(cols),
157 - UniqueView: true,
158 - }
159 -}
160 -
161 -func buildRethinkColumns(cols []rethinkColumnMeta) map[string]any {
162 - result := make(map[string]any, len(cols))
163 - for i, col := range cols {
164 - visual := visValue
165 - if col.colType == ftDuration {
166 - visual = visBar
167 - }
168 - colDef := funcapi.Column{
169 - Index: i,
170 - Name: col.name,
171 - Type: col.colType,
172 - Units: col.units,
173 - Visualization: visual,
174 - Sort: col.sortDir,
175 - Sortable: col.sortable,
176 - Sticky: col.sticky,
177 - Summary: col.summary,
178 - Filter: col.filter,
179 - FullWidth: col.fullWidth,
180 - Wrap: col.wrap,
181 - DefaultExpandedFilter: false,
182 - UniqueKey: col.uniqueKey,
183 - Visible: col.visible,
184 - ValueOptions: funcapi.ValueOptions{
185 - Transform: col.transform,
186 - DecimalPoints: col.decimalPoints,
187 - DefaultValue: nil,
188 - },
189 - }
190 - result[col.id] = colDef.BuildColumn()
191 - }
192 - return result
193 -}
194 -
195 -func (c *Collector) collectRunningQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
196 - limit := c.TopQueriesLimit
197 - if limit <= 0 {
198 - limit = 500
199 - }
200 -
201 - if ctx.Err() == context.DeadlineExceeded {
202 - return &module.FunctionResponse{Status: 504, Message: "query timed out"}
203 - }
204 -
205 - rows, err := c.rdb.jobs(ctx)
206 - if err != nil {
207 - if ctx.Err() == context.DeadlineExceeded {
208 - return &module.FunctionResponse{Status: 504, Message: "query timed out"}
209 - }
210 - return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("jobs query failed: %v", err)}
211 - }
212 -
213 - jobRows := make([]rethinkJobRow, 0, len(rows))
214 - for _, row := range rows {
215 - jobRows = append(jobRows, parseRethinkJob(row))
216 - }
217 -
218 - if len(jobRows) == 0 {
219 - return &module.FunctionResponse{
220 - Status: 200,
221 - Message: "No running queries found.",
222 - Help: "Currently running queries from rethinkdb.jobs",
223 - Columns: buildRethinkColumns(rethinkRunningColumns),
224 - Data: [][]any{},
225 - DefaultSortColumn: mapRethinkSortColumn(sortColumn),
226 - RequiredParams: []funcapi.ParamConfig{buildRethinkSortParam(rethinkRunningColumns)},
227 - }
228 - }
229 -
230 - sortColumn = mapRethinkSortColumn(sortColumn)
231 - sortRethinkRows(jobRows, sortColumn)
232 - if len(jobRows) > limit {
233 - jobRows = jobRows[:limit]
234 - }
235 -
236 - data := make([][]any, 0, len(jobRows))
237 - for _, row := range jobRows {
238 - out := make([]any, len(rethinkRunningColumns))
239 - for i, col := range rethinkRunningColumns {
240 - switch col.id {
241 - case "jobId":
242 - out[i] = row.JobID
243 - case "query":
244 - out[i] = strmutil.TruncateText(row.Query, rethinkMaxQueryTextLength)
245 - case "durationMs":
246 - out[i] = row.DurationMs
247 - case "type":
248 - out[i] = row.Type
249 - case "user":
250 - out[i] = row.User
251 - case "clientAddress":
252 - out[i] = row.ClientAddress
253 - case "clientPort":
254 - out[i] = row.ClientPort
255 - case "servers":
256 - out[i] = row.Servers
257 - default:
258 - out[i] = nil
259 - }
260 - }
261 - data = append(data, out)
262 - }
263 -
264 - return &module.FunctionResponse{
265 - Status: 200,
266 - Help: "Currently running queries from rethinkdb.jobs. WARNING: Query text may contain unmasked literals (potential PII).",
267 - Columns: buildRethinkColumns(rethinkRunningColumns),
268 - Data: data,
269 - DefaultSortColumn: sortColumn,
270 - RequiredParams: []funcapi.ParamConfig{buildRethinkSortParam(rethinkRunningColumns)},
271 - }
272 -}
273 -
274 -type rethinkJobRow struct {
275 - JobID string
276 - Query string
277 - DurationMs float64
278 - Type string
279 - User string
280 - ClientAddress string
281 - ClientPort int64
282 - Servers string
283 -}
284 -
285 -func parseRethinkJob(row map[string]any) rethinkJobRow {
286 - info := mapStringAny(row["info"])
287 - query := fmt.Sprint(info["query"])
288 - user := fmt.Sprint(info["user"])
289 - clientAddr := fmt.Sprint(info["client_address"])
290 - clientPort := toInt64(info["client_port"])
291 -
292 - servers := ""
293 - if list, ok := row["servers"].([]any); ok {
294 - ss := make([]string, 0, len(list))
295 - for _, v := range list {
296 - ss = append(ss, fmt.Sprint(v))
297 - }
298 - servers = strings.Join(ss, ",")
299 - }
300 -
301 - return rethinkJobRow{
302 - JobID: fmt.Sprint(row["id"]),
303 - Query: query,
304 - DurationMs: toFloat64(row["duration_sec"]) * 1000,
305 - Type: fmt.Sprint(row["type"]),
306 - User: user,
307 - ClientAddress: clientAddr,
308 - ClientPort: clientPort,
309 - Servers: servers,
310 - }
311 -}
312 -
313 -func mapStringAny(v any) map[string]any {
314 - if m, ok := v.(map[string]any); ok {
315 - return m
316 - }
317 - return map[string]any{}
318 -}
319 -
320 -func toFloat64(v any) float64 {
321 - switch t := v.(type) {
322 - case float64:
323 - return t
324 - case float32:
325 - return float64(t)
326 - case int:
327 - return float64(t)
328 - case int64:
329 - return float64(t)
330 - case uint64:
331 - return float64(t)
332 - default:
333 - return 0
334 - }
335 -}
336 -
337 -func toInt64(v any) int64 {
338 - switch t := v.(type) {
339 - case int64:
340 - return t
341 - case int:
342 - return int64(t)
343 - case float64:
344 - return int64(t)
345 - case float32:
346 - return int64(t)
347 - default:
348 - return 0
349 - }
350 -}
351 -
352 -func mapRethinkSortColumn(input string) string {
353 - for _, col := range rethinkRunningColumns {
354 - if col.isSortOption && col.id == input {
355 - return col.id
356 - }
357 - }
358 - for _, col := range rethinkRunningColumns {
359 - if col.isDefaultSort {
360 - return col.id
361 - }
362 - }
363 - for _, col := range rethinkRunningColumns {
364 - if col.isSortOption {
365 - return col.id
366 - }
367 - }
368 - return ""
369 -}
370 -
371 -func sortRethinkRows(rows []rethinkJobRow, sortColumn string) {
372 - switch sortColumn {
373 - case "durationMs":
374 - sort.Slice(rows, func(i, j int) bool { return rows[i].DurationMs > rows[j].DurationMs })
375 - default:
376 - sort.Slice(rows, func(i, j int) bool { return rows[i].DurationMs > rows[j].DurationMs })
377 - }
378 -}
src/go/plugin/go.d/collector/rethinkdb/functions_test.go
+3 -7
@@ -32,12 +32,8 @@ func TestRethinkDBMethods(t *testing.T) {
32 func TestRethinkDBRunningColumns_HasRequiredColumns(t *testing.T) {
33 required := []string{"jobId", "query", "durationMs"}
34
35 - uiKeys := make(map[string]bool)
36 - for _, col := range rethinkRunningColumns {
37 - uiKeys[col.id] = true
38 - }
39 -
40 - for _, key := range required {
41 - assert.True(t, uiKeys[key], "column %s should be defined", key)
35 + cs := rethinkColumnSet(rethinkRunningColumns)
36 + for _, id := range required {
37 + assert.True(t, cs.ContainsColumn(id), "column %s should be defined", id)
38 }
39 }
src/go/plugin/go.d/collector/snmp/collector.go
+11 -9
@@ -29,11 +29,10 @@ func init() {
29 Defaults: module.Defaults{
30 UpdateEvery: 10,
31 },
32 - Create: func() module.Module { return New() },
33 - Config: func() any { return &Config{} },
34 - Methods: snmpMethods,
35 - MethodParams: snmpMethodParams,
36 - HandleMethod: snmpHandleMethod,
32 + Create: func() module.Module { return New() },
33 + Config: func() any { return &Config{} },
34 + Methods: snmpMethods,
35 + MethodHandler: snmpFunctionHandler,
36 })
37 }
38
@@ -81,7 +80,7 @@ func New() *Collector {
80 },
81 }
82
84 - c.funcIfaces = newFuncInterfaces(c.ifaceCache)
83 + c.funcRouter = newFuncRouter(c.ifaceCache)
84
85 return c
86 }
@@ -98,8 +97,8 @@ type (
97 seenTableMetrics map[string]bool
98 seenProfiles map[string]bool
99
101 - ifaceCache *ifaceCache // interface metrics cache for functions
102 - funcIfaces *funcInterfaces // interfaces function handler
100 + ifaceCache *ifaceCache // interface metrics cache for functions
101 + funcRouter *funcRouter // function router for method handlers
102
103 prober ping.Prober
104 newProber func(ping.ProberConfig, *logger.Logger) ping.Prober
@@ -180,7 +179,10 @@ func (c *Collector) Collect(ctx context.Context) map[string]int64 {
179 return mx
180 }
181
183 -func (c *Collector) Cleanup(context.Context) {
182 +func (c *Collector) Cleanup(ctx context.Context) {
183 + if c.funcRouter != nil {
184 + c.funcRouter.Cleanup(ctx)
185 + }
186 if c.snmpClient != nil {
187 _ = c.snmpClient.Close()
188 }
src/go/plugin/go.d/collector/snmp/func_interfaces.go
+114 -429
@@ -4,170 +4,121 @@ package snmp
4
5 import (
6 "context"
7 - "fmt"
7 "sort"
8
9 "github.com/netdata/netdata/go/plugins/pkg/funcapi"
11 - "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
10 )
11
12 +// Compile-time interface check.
13 +var _ funcapi.MethodHandler = (*funcInterfaces)(nil)
14 +
15 // funcInterfaces handles the "interfaces" function for SNMP devices.
16 // It provides network interface traffic and status metrics from cached SNMP data.
17 type funcInterfaces struct {
17 - cache *ifaceCache
18 + router *funcRouter
19 }
20
20 -func newFuncInterfaces(cache *ifaceCache) *funcInterfaces {
21 - return &funcInterfaces{cache: cache}
21 +func newFuncInterfaces(r *funcRouter) *funcInterfaces {
22 + return &funcInterfaces{router: r}
23 }
24
24 -// methods returns the method configurations for this function.
25 -func (f *funcInterfaces) methods() []module.MethodConfig {
26 - return []module.MethodConfig{
27 - {
28 - UpdateEvery: 10,
29 - ID: "interfaces",
30 - Name: "Network Interfaces",
31 - Help: "Network interface traffic and status metrics",
32 - RequiredParams: []funcapi.ParamConfig{{
33 - ID: funcIfacesParamTypeGroup,
34 - Name: "Type Group",
35 - Help: "Filter by interface type group",
36 - Selection: funcapi.ParamSelect,
37 - Options: []funcapi.ParamOption{
38 - {ID: "ethernet", Name: "Ethernet", Default: true},
39 - {ID: "aggregation", Name: "Aggregation"},
40 - {ID: "virtual", Name: "Virtual"},
41 - {ID: "other", Name: "Other"},
42 - },
43 - }},
44 - },
45 - }
46 -}
25 +const (
26 + ifacesMethodID = "interfaces"
27 + ifacesParamTypeGroup = "if_type_group"
28 + ifacesDefaultTypeGroup = "ethernet"
29 +)
30
48 -// methodParams returns params for the given method.
49 -func (f *funcInterfaces) methodParams(method string) ([]funcapi.ParamConfig, error) {
50 - if method != "interfaces" {
51 - return nil, fmt.Errorf("unknown method: %s", method)
31 +func ifacesMethodConfig() funcapi.MethodConfig {
32 + return funcapi.MethodConfig{
33 + ID: ifacesMethodID,
34 + Name: "Network Interfaces",
35 + UpdateEvery: 10,
36 + Help: "Network interface traffic and status metrics",
37 + RequiredParams: []funcapi.ParamConfig{{
38 + ID: ifacesParamTypeGroup,
39 + Name: "Type Group",
40 + Help: "Filter by interface type group",
41 + Selection: funcapi.ParamSelect,
42 + Options: []funcapi.ParamOption{
43 + {ID: "ethernet", Name: "Ethernet", Default: true},
44 + {ID: "aggregation", Name: "Aggregation"},
45 + {ID: "virtual", Name: "Virtual"},
46 + {ID: "other", Name: "Other"},
47 + },
48 + }},
49 }
50 +}
51
54 - methods := f.methods()
55 - if len(methods) > 0 {
56 - return methods[0].RequiredParams, nil
52 +// MethodParams implements funcapi.MethodHandler.
53 +func (f *funcInterfaces) MethodParams(_ context.Context, method string) ([]funcapi.ParamConfig, error) {
54 + if method != ifacesMethodID {
55 + return nil, nil
56 }
58 - return nil, nil
57 + return []funcapi.ParamConfig{{
58 + ID: ifacesParamTypeGroup,
59 + Name: "Type Group",
60 + Help: "Filter by interface type group",
61 + Selection: funcapi.ParamSelect,
62 + Options: []funcapi.ParamOption{
63 + {ID: "ethernet", Name: "Ethernet", Default: true},
64 + {ID: "aggregation", Name: "Aggregation"},
65 + {ID: "virtual", Name: "Virtual"},
66 + {ID: "other", Name: "Other"},
67 + },
68 + }}, nil
69 }
70
61 -// handle processes a function request and returns the response.
62 -func (f *funcInterfaces) handle(method string, params funcapi.ResolvedParams) *module.FunctionResponse {
63 - if method != "interfaces" {
64 - return &module.FunctionResponse{Status: 404, Message: fmt.Sprintf("unknown method: %s", method)}
71 +// Cleanup implements funcapi.MethodHandler.
72 +func (f *funcInterfaces) Cleanup(_ context.Context) {}
73 +
74 +// Handle implements funcapi.MethodHandler.
75 +func (f *funcInterfaces) Handle(_ context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
76 + if method != ifacesMethodID {
77 + return funcapi.NotFoundResponse(method)
78 }
79
67 - if f.cache == nil {
68 - return &module.FunctionResponse{
69 - Status: 503,
70 - Message: "interface data not available yet, please retry after data collection",
71 - }
80 + if f.router.ifaceCache == nil {
81 + return funcapi.UnavailableResponse("interface data not available yet, please retry after data collection")
82 }
83
74 - f.cache.mu.RLock()
75 - defer f.cache.mu.RUnlock()
84 + f.router.ifaceCache.mu.RLock()
85 + defer f.router.ifaceCache.mu.RUnlock()
86
77 - typeGroupFilter := params.GetOne(funcIfacesParamTypeGroup)
87 + typeGroupFilter := params.GetOne(ifacesParamTypeGroup)
88 if typeGroupFilter == "" {
79 - typeGroupFilter = "ethernet"
89 + typeGroupFilter = ifacesDefaultTypeGroup
90 }
91
92 // Build data rows from cache
83 - data := make([][]any, 0, len(f.cache.interfaces))
84 - for _, entry := range f.cache.interfaces {
93 + data := make([][]any, 0, len(f.router.ifaceCache.interfaces))
94 + for _, entry := range f.router.ifaceCache.interfaces {
95 if !matchesTypeGroup(entry.ifTypeGroup, typeGroupFilter) {
96 continue
97 }
88 - row := f.buildRow(entry)
89 - data = append(data, row)
98 + data = append(data, f.buildRow(entry))
99 }
100
92 - // Sort data based on params
101 f.sortData(data, f.defaultSortColumn())
102
95 - return &module.FunctionResponse{
103 + cs := snmpColumnSet(snmpAllColumns)
104 +
105 + return &funcapi.FunctionResponse{
106 Status: 200,
107 Help: "Network interface traffic and status metrics",
98 - Columns: f.buildColumns(),
108 + Columns: f.buildColumns(cs),
109 Data: data,
110 DefaultSortColumn: f.defaultSortColumn(),
101 -
102 - // Charts for aggregated visualization
103 - Charts: map[string]module.ChartConfig{
104 - "Traffic": {
105 - Name: "Traffic",
106 - Type: "stacked-bar",
107 - Columns: []string{"Traffic In", "Traffic Out"},
108 - },
109 - "UnicastPackets": {
110 - Name: "Unicast Packets",
111 - Type: "stacked-bar",
112 - Columns: []string{"Unicast In", "Unicast Out"},
113 - },
114 - "BroadcastPackets": {
115 - Name: "Broadcast Packets",
116 - Type: "stacked-bar",
117 - Columns: []string{"Broadcast In", "Broadcast Out"},
118 - },
119 - "MulticastPackets": {
120 - Name: "Multicast Packets",
121 - Type: "stacked-bar",
122 - Columns: []string{"Multicast In", "Multicast Out"},
123 - },
124 - "OperationalStatus": {
125 - Name: "Operational Status",
126 - Type: "stacked-bar",
127 - Columns: []string{"Oper Status"},
128 - },
129 - },
130 - DefaultCharts: [][]string{
131 - {"Traffic", "Type"},
132 - {"OperationalStatus", "Oper Status"},
133 - },
134 - GroupBy: map[string]module.GroupByConfig{
135 - "Type": {
136 - Name: "Group by Type",
137 - Columns: []string{"Type"},
138 - },
139 - },
111 + ChartingConfig: cs.BuildCharting(),
112 }
113 }
114
115 // buildColumns builds column definitions for the response.
144 -func (f *funcInterfaces) buildColumns() map[string]any {
145 - columns := make(map[string]any)
146 -
147 - for i, col := range funcIfacesColumns {
148 - colDef := funcapi.Column{
149 - Index: i,
150 - Name: col.name,
151 - Type: col.dataType,
152 - Units: col.units,
153 - Visualization: col.visual,
154 - Sort: col.sortDir,
155 - Sortable: true,
156 - Sticky: col.sticky,
157 - Summary: col.summary,
158 - Filter: col.filter,
159 - Visible: col.visible,
160 - ValueOptions: funcapi.ValueOptions{
161 - Transform: col.transform,
162 - DecimalPoints: col.decimals,
163 - DefaultValue: nil,
164 - },
165 - }
166 - columns[col.key] = colDef.BuildColumn()
167 - }
116 +func (f *funcInterfaces) buildColumns(cs funcapi.ColumnSet[snmpColumn]) map[string]any {
117 + columns := cs.BuildColumns()
118
119 + // Add rowOptions column (not part of the regular column set)
120 rowOptions := funcapi.Column{
170 - Index: len(funcIfacesColumns),
121 + Index: cs.Len(),
122 Name: "rowOptions",
123 Type: funcapi.FieldTypeNone,
124 Visualization: funcapi.FieldVisualRowOptions,
@@ -190,20 +141,19 @@ func (f *funcInterfaces) buildColumns() map[string]any {
141 }
142
143 // buildRow builds a data row from an interface entry.
193 -// Column order is determined by funcIfacesColumns - each column's value() extracts the data.
144 func (f *funcInterfaces) buildRow(entry *ifaceEntry) []any {
195 - row := make([]any, len(funcIfacesColumns)+1)
196 - for i, col := range funcIfacesColumns {
197 - row[i] = col.value(entry)
145 + row := make([]any, len(snmpAllColumns)+1)
146 + for i, col := range snmpAllColumns {
147 + row[i] = col.Value(entry)
148 }
149 if isIfaceDown(entry) {
200 - for i, col := range funcIfacesColumns {
201 - if col.dataType == funcapi.FieldTypeFloat {
150 + for i, col := range snmpAllColumns {
151 + if col.Type == funcapi.FieldTypeFloat {
152 row[i] = nil
153 }
154 }
155 }
206 - row[len(funcIfacesColumns)] = rowOptionsForIface(entry)
156 + row[len(snmpAllColumns)] = rowOptionsForIface(entry)
157 return row
158 }
159
@@ -213,14 +163,13 @@ func (f *funcInterfaces) sortData(data [][]any, sortColumn string) {
163 return
164 }
165
216 - // Find column index and sort direction
166 colIdx := 0
167 sortDir := funcapi.FieldSortAscending
168
220 - for i, col := range funcIfacesColumns {
221 - if col.key == sortColumn {
169 + for i, col := range snmpAllColumns {
170 + if col.Name == sortColumn {
171 colIdx = i
223 - sortDir = col.sortDir
172 + sortDir = col.Sort
173 break
174 }
175 }
@@ -229,7 +178,6 @@ func (f *funcInterfaces) sortData(data [][]any, sortColumn string) {
178 vi := data[i][colIdx]
179 vj := data[j][colIdx]
180
232 - // Handle nil values - put them at the end
181 if vi == nil && vj == nil {
182 return false
183 }
@@ -240,7 +188,6 @@ func (f *funcInterfaces) sortData(data [][]any, sortColumn string) {
188 return true
189 }
190
243 - // Compare based on type
191 switch a := vi.(type) {
192 case string:
193 b := vj.(string)
@@ -262,9 +209,9 @@ func (f *funcInterfaces) sortData(data [][]any, sortColumn string) {
209
210 // defaultSortColumn returns the default sort column key.
211 func (f *funcInterfaces) defaultSortColumn() string {
265 - for _, col := range funcIfacesColumns {
266 - if col.defaultSort {
267 - return col.key
212 + for _, col := range snmpAllColumns {
213 + if col.DefaultSort {
214 + return col.Name
215 }
216 }
217 return "Interface"
@@ -296,7 +243,6 @@ func matchesTypeGroup(group, filter string) bool {
243 return group == filter
244 }
245
299 -// ptrToAny converts a *float64 to any, returning nil if the pointer is nil.
246 func ptrToAny(p *float64) any {
247 if p == nil {
248 return nil
@@ -327,300 +273,39 @@ func sumRates(vals ...*float64) *float64 {
273 return &sum
274 }
275
330 -// Package-level registration functions that delegate to funcInterfaces.
331 -
332 -func snmpMethods() []module.MethodConfig {
333 - return (&funcInterfaces{}).methods()
334 -}
335 -
336 -func snmpMethodParams(_ context.Context, job *module.Job, method string) ([]funcapi.ParamConfig, error) {
337 - c, ok := job.Module().(*Collector)
338 - if !ok {
339 - return nil, fmt.Errorf("invalid module type")
340 - }
341 - return c.funcIfaces.methodParams(method)
342 -}
343 -
344 -func snmpHandleMethod(_ context.Context, job *module.Job, method string, params funcapi.ResolvedParams) *module.FunctionResponse {
345 - c, ok := job.Module().(*Collector)
346 - if !ok {
347 - return &module.FunctionResponse{Status: 500, Message: "internal error: invalid module type"}
348 - }
349 - return c.funcIfaces.handle(method, params)
276 +// snmpColumn defines a column for SNMP interfaces function.
277 +type snmpColumn struct {
278 + funcapi.ColumnMeta
279 + Value func(*ifaceEntry) any // Extracts value from entry
280 + DefaultSort bool // Is default sort column
281 }
282
352 -// funcIfacesParamTypeGroup is the parameter ID for type group filtering.
353 -const funcIfacesParamTypeGroup = "if_type_group"
354 -
355 -// funcIfacesColumn defines a column with its metadata and value extractor.
356 -// The value function extracts the column's data from an ifaceEntry.
357 -type funcIfacesColumn struct {
358 - key string // column header value (must be unique)
359 - name string // tooltip value
360 - value func(*ifaceEntry) any // extracts value from entry
361 - dataType funcapi.FieldType // string, float, etc.
362 - units string // display units (bytes/s, packets/s)
363 - visual funcapi.FieldVisual // visualization type
364 - visible bool // shown by default
365 - transform funcapi.FieldTransform // number formatting
366 - decimals int // decimal points
367 - sortDir funcapi.FieldSort // asc or desc
368 - summary funcapi.FieldSummary // count, sum, etc.
369 - filter funcapi.FieldFilter // multiselect, range, etc.
370 - sortOption string // if non-empty, appears in sort dropdown
371 - defaultSort bool // is default sort column
372 - sticky bool // sticky column
283 +func snmpColumnSet(cols []snmpColumn) funcapi.ColumnSet[snmpColumn] {
284 + return funcapi.Columns(cols, func(c snmpColumn) funcapi.ColumnMeta { return c.ColumnMeta })
285 }
286
375 -// funcIfacesColumns defines all columns for the interfaces function.
376 -// Each column includes its value extractor - single source of truth.
377 -var funcIfacesColumns = []funcIfacesColumn{
378 - {
379 - key: "Interface",
380 - name: "",
381 - value: func(e *ifaceEntry) any { return e.name },
382 - dataType: funcapi.FieldTypeString,
383 - visible: true,
384 - sortDir: funcapi.FieldSortAscending,
385 - summary: funcapi.FieldSummaryCount,
386 - filter: funcapi.FieldFilterMultiselect,
387 - defaultSort: true,
388 - sticky: true,
389 - },
390 - {
391 - key: "Type",
392 - name: "IANA ifType (IF-MIB)",
393 - value: func(e *ifaceEntry) any { return e.ifType },
394 - dataType: funcapi.FieldTypeString,
395 - visible: false,
396 - sortDir: funcapi.FieldSortAscending,
397 - summary: funcapi.FieldSummaryCount,
398 - filter: funcapi.FieldFilterMultiselect,
399 - },
400 - {
401 - key: "Type Group",
402 - name: "Custom mapping of IANA ifType into groups",
403 - value: func(e *ifaceEntry) any { return e.ifTypeGroup },
404 - dataType: funcapi.FieldTypeString,
405 - visible: true,
406 - sortDir: funcapi.FieldSortAscending,
407 - summary: funcapi.FieldSummaryCount,
408 - filter: funcapi.FieldFilterMultiselect,
409 - },
410 - {
411 - key: "Admin Status",
412 - name: "Administrative status: up, down, testing",
413 - value: func(e *ifaceEntry) any { return e.adminStatus },
414 - dataType: funcapi.FieldTypeString,
415 - visible: true,
416 - sortDir: funcapi.FieldSortAscending,
417 - summary: funcapi.FieldSummaryCount,
418 - filter: funcapi.FieldFilterMultiselect,
419 - },
420 - {
421 - key: "Oper Status",
422 - name: "Operational status: up, down, testing, unknown, dormant, notPresent, lowerLayerDown",
423 - value: func(e *ifaceEntry) any { return e.operStatus },
424 - dataType: funcapi.FieldTypeString,
425 - visible: true,
426 - sortDir: funcapi.FieldSortAscending,
427 - summary: funcapi.FieldSummaryCount,
428 - filter: funcapi.FieldFilterMultiselect,
429 - },
430 - {
431 - key: "Traffic In",
432 - name: "",
433 - value: func(e *ifaceEntry) any { return ptrToAnyScale(e.rates.trafficIn, 1_000_000) },
434 - dataType: funcapi.FieldTypeFloat,
435 - units: "Mbits",
436 - visual: funcapi.FieldVisualBar,
437 - visible: true,
438 - transform: funcapi.FieldTransformNumber,
439 - decimals: 2,
440 - sortDir: funcapi.FieldSortDescending,
441 - summary: funcapi.FieldSummarySum,
442 - filter: funcapi.FieldFilterRange,
443 - },
444 - {
445 - key: "Traffic Out",
446 - name: "",
447 - value: func(e *ifaceEntry) any { return ptrToAnyScale(e.rates.trafficOut, 1_000_000) },
448 - dataType: funcapi.FieldTypeFloat,
449 - units: "Mbits",
450 - visual: funcapi.FieldVisualBar,
451 - visible: true,
452 - transform: funcapi.FieldTransformNumber,
453 - decimals: 2,
454 - sortDir: funcapi.FieldSortDescending,
455 - summary: funcapi.FieldSummarySum,
456 - filter: funcapi.FieldFilterRange,
457 - },
458 - {
459 - key: "Unicast In",
460 - name: "",
461 - value: func(e *ifaceEntry) any { return ptrToAnyScale(e.rates.ucastPktsIn, 1_000) },
462 - dataType: funcapi.FieldTypeFloat,
463 - units: "Kpps",
464 - visual: funcapi.FieldVisualBar,
465 - visible: false,
466 - transform: funcapi.FieldTransformNumber,
467 - decimals: 2,
468 - sortDir: funcapi.FieldSortDescending,
469 - summary: funcapi.FieldSummarySum,
470 - filter: funcapi.FieldFilterRange,
471 - },
472 - {
473 - key: "Unicast Out",
474 - name: "",
475 - value: func(e *ifaceEntry) any { return ptrToAnyScale(e.rates.ucastPktsOut, 1_000) },
476 - dataType: funcapi.FieldTypeFloat,
477 - units: "Kpps",
478 - visual: funcapi.FieldVisualBar,
479 - visible: false,
480 - transform: funcapi.FieldTransformNumber,
481 - decimals: 2,
482 - sortDir: funcapi.FieldSortDescending,
483 - summary: funcapi.FieldSummarySum,
484 - filter: funcapi.FieldFilterRange,
485 - },
486 - {
487 - key: "Broadcast In",
488 - name: "",
489 - value: func(e *ifaceEntry) any { return ptrToAnyScale(e.rates.bcastPktsIn, 1_000) },
490 - dataType: funcapi.FieldTypeFloat,
491 - units: "Kpps",
492 - visual: funcapi.FieldVisualBar,
493 - visible: false,
494 - transform: funcapi.FieldTransformNumber,
495 - decimals: 2,
496 - sortDir: funcapi.FieldSortDescending,
497 - summary: funcapi.FieldSummarySum,
498 - filter: funcapi.FieldFilterRange,
499 - },
500 - {
501 - key: "Broadcast Out",
502 - name: "",
503 - value: func(e *ifaceEntry) any { return ptrToAnyScale(e.rates.bcastPktsOut, 1_000) },
504 - dataType: funcapi.FieldTypeFloat,
505 - units: "Kpps",
506 - visual: funcapi.FieldVisualBar,
507 - visible: false,
508 - transform: funcapi.FieldTransformNumber,
509 - decimals: 2,
510 - sortDir: funcapi.FieldSortDescending,
511 - summary: funcapi.FieldSummarySum,
512 - filter: funcapi.FieldFilterRange,
513 - },
514 - {
515 - key: "Packets In",
516 - name: "",
517 - value: func(e *ifaceEntry) any {
518 - return ptrToAnyScale(sumRates(e.rates.ucastPktsIn, e.rates.bcastPktsIn, e.rates.mcastPktsIn), 1_000)
519 - },
520 - dataType: funcapi.FieldTypeFloat,
521 - units: "Kpps",
522 - visual: funcapi.FieldVisualBar,
523 - visible: true,
524 - transform: funcapi.FieldTransformNumber,
525 - decimals: 2,
526 - sortDir: funcapi.FieldSortDescending,
527 - summary: funcapi.FieldSummarySum,
528 - filter: funcapi.FieldFilterRange,
529 - },
530 - {
531 - key: "Packets Out",
532 - name: "",
533 - value: func(e *ifaceEntry) any {
534 - return ptrToAnyScale(sumRates(e.rates.ucastPktsOut, e.rates.bcastPktsOut, e.rates.mcastPktsOut), 1_000)
535 - },
536 - dataType: funcapi.FieldTypeFloat,
537 - units: "Kpps",
538 - visual: funcapi.FieldVisualBar,
539 - visible: true,
540 - transform: funcapi.FieldTransformNumber,
541 - decimals: 2,
542 - sortDir: funcapi.FieldSortDescending,
543 - summary: funcapi.FieldSummarySum,
544 - filter: funcapi.FieldFilterRange,
545 - },
546 - {
547 - key: "Errors In",
548 - name: "",
549 - value: func(e *ifaceEntry) any { return ptrToAny(e.rates.errorsIn) },
550 - dataType: funcapi.FieldTypeFloat,
551 - units: "packets/s",
552 - visual: funcapi.FieldVisualBar,
553 - visible: false,
554 - transform: funcapi.FieldTransformNumber,
555 - sortDir: funcapi.FieldSortDescending,
556 - summary: funcapi.FieldSummarySum,
557 - filter: funcapi.FieldFilterRange,
558 - },
559 - {
560 - key: "Errors Out",
561 - name: "",
562 - value: func(e *ifaceEntry) any { return ptrToAny(e.rates.errorsOut) },
563 - dataType: funcapi.FieldTypeFloat,
564 - units: "packets/s",
565 - visual: funcapi.FieldVisualBar,
566 - visible: false,
567 - transform: funcapi.FieldTransformNumber,
568 - sortDir: funcapi.FieldSortDescending,
569 - summary: funcapi.FieldSummarySum,
570 - filter: funcapi.FieldFilterRange,
571 - },
572 - {
573 - key: "Discards In",
574 - name: "",
575 - value: func(e *ifaceEntry) any { return ptrToAny(e.rates.discardsIn) },
576 - dataType: funcapi.FieldTypeFloat,
577 - units: "packets/s",
578 - visual: funcapi.FieldVisualBar,
579 - visible: true,
580 - transform: funcapi.FieldTransformNumber,
581 - sortDir: funcapi.FieldSortDescending,
582 - summary: funcapi.FieldSummarySum,
583 - filter: funcapi.FieldFilterRange,
584 - },
585 - {
586 - key: "Discards Out",
587 - name: "",
588 - value: func(e *ifaceEntry) any { return ptrToAny(e.rates.discardsOut) },
589 - dataType: funcapi.FieldTypeFloat,
590 - units: "packets/s",
591 - visual: funcapi.FieldVisualBar,
592 - visible: true,
593 - transform: funcapi.FieldTransformNumber,
594 - sortDir: funcapi.FieldSortDescending,
595 - summary: funcapi.FieldSummarySum,
596 - filter: funcapi.FieldFilterRange,
597 - },
598 - {
599 - key: "Multicast In",
600 - name: "",
601 - value: func(e *ifaceEntry) any { return ptrToAnyScale(e.rates.mcastPktsIn, 1_000) },
602 - dataType: funcapi.FieldTypeFloat,
603 - units: "Kpps",
604 - visual: funcapi.FieldVisualBar,
605 - visible: false,
606 - transform: funcapi.FieldTransformNumber,
607 - decimals: 2,
608 - sortDir: funcapi.FieldSortDescending,
609 - summary: funcapi.FieldSummarySum,
610 - filter: funcapi.FieldFilterRange,
611 - },
612 - {
613 - key: "Multicast Out",
614 - name: "",
615 - value: func(e *ifaceEntry) any { return ptrToAnyScale(e.rates.mcastPktsOut, 1_000) },
616 - dataType: funcapi.FieldTypeFloat,
617 - units: "Kpps",
618 - visual: funcapi.FieldVisualBar,
619 - visible: false,
620 - transform: funcapi.FieldTransformNumber,
621 - decimals: 2,
622 - sortDir: funcapi.FieldSortDescending,
623 - summary: funcapi.FieldSummarySum,
624 - filter: funcapi.FieldFilterRange,
625 - },
287 +var snmpAllColumns = []snmpColumn{
288 + {ColumnMeta: funcapi.ColumnMeta{Name: "Interface", Tooltip: "Interface", Type: funcapi.FieldTypeString, Visible: true, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, Sticky: true, Sortable: true}, Value: func(e *ifaceEntry) any { return e.name }, DefaultSort: true},
289 + {ColumnMeta: funcapi.ColumnMeta{Name: "Type", Tooltip: "IANA ifType (IF-MIB)", Type: funcapi.FieldTypeString, Visible: false, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, Sortable: true, GroupBy: &funcapi.GroupByOptions{IsDefault: true}}, Value: func(e *ifaceEntry) any { return e.ifType }},
290 + {ColumnMeta: funcapi.ColumnMeta{Name: "Type Group", Tooltip: "Custom mapping of IANA ifType into groups", Type: funcapi.FieldTypeString, Visible: true, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, Sortable: true}, Value: func(e *ifaceEntry) any { return e.ifTypeGroup }},
291 + {ColumnMeta: funcapi.ColumnMeta{Name: "Admin Status", Tooltip: "Administrative status: up, down, testing", Type: funcapi.FieldTypeString, Visible: true, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, Sortable: true}, Value: func(e *ifaceEntry) any { return e.adminStatus }},
292 + {ColumnMeta: funcapi.ColumnMeta{Name: "Oper Status", Tooltip: "Operational status: up, down, testing, unknown, dormant, notPresent, lowerLayerDown", Type: funcapi.FieldTypeString, Visible: true, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, Filter: funcapi.FieldFilterMultiselect, Sortable: true, Chart: &funcapi.ChartOptions{Group: "OperationalStatus", Title: "Operational Status", IsDefault: true, DefaultGroupBy: "Oper Status"}, GroupBy: &funcapi.GroupByOptions{}}, Value: func(e *ifaceEntry) any { return e.operStatus }},
293 + {ColumnMeta: funcapi.ColumnMeta{Name: "Traffic In", Tooltip: "Traffic In", Type: funcapi.FieldTypeFloat, Units: "Mbits", Visualization: funcapi.FieldVisualBar, Visible: true, Transform: funcapi.FieldTransformNumber, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true, Chart: &funcapi.ChartOptions{Group: "Traffic", IsDefault: true, DefaultGroupBy: "Type"}}, Value: func(e *ifaceEntry) any { return ptrToAnyScale(e.rates.trafficIn, 1_000_000) }},
294 + {ColumnMeta: funcapi.ColumnMeta{Name: "Traffic Out", Tooltip: "Traffic Out", Type: funcapi.FieldTypeFloat, Units: "Mbits", Visualization: funcapi.FieldVisualBar, Visible: true, Transform: funcapi.FieldTransformNumber, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true, Chart: &funcapi.ChartOptions{Group: "Traffic"}}, Value: func(e *ifaceEntry) any { return ptrToAnyScale(e.rates.trafficOut, 1_000_000) }},
295 + {ColumnMeta: funcapi.ColumnMeta{Name: "Unicast In", Tooltip: "Unicast In", Type: funcapi.FieldTypeFloat, Units: "Kpps", Visualization: funcapi.FieldVisualBar, Visible: false, Transform: funcapi.FieldTransformNumber, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true, Chart: &funcapi.ChartOptions{Group: "UnicastPackets", Title: "Unicast Packets"}}, Value: func(e *ifaceEntry) any { return ptrToAnyScale(e.rates.ucastPktsIn, 1_000) }},
296 + {ColumnMeta: funcapi.ColumnMeta{Name: "Unicast Out", Tooltip: "Unicast Out", Type: funcapi.FieldTypeFloat, Units: "Kpps", Visualization: funcapi.FieldVisualBar, Visible: false, Transform: funcapi.FieldTransformNumber, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true, Chart: &funcapi.ChartOptions{Group: "UnicastPackets"}}, Value: func(e *ifaceEntry) any { return ptrToAnyScale(e.rates.ucastPktsOut, 1_000) }},
297 + {ColumnMeta: funcapi.ColumnMeta{Name: "Broadcast In", Tooltip: "Broadcast In", Type: funcapi.FieldTypeFloat, Units: "Kpps", Visualization: funcapi.FieldVisualBar, Visible: false, Transform: funcapi.FieldTransformNumber, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true, Chart: &funcapi.ChartOptions{Group: "BroadcastPackets", Title: "Broadcast Packets"}}, Value: func(e *ifaceEntry) any { return ptrToAnyScale(e.rates.bcastPktsIn, 1_000) }},
298 + {ColumnMeta: funcapi.ColumnMeta{Name: "Broadcast Out", Tooltip: "Broadcast Out", Type: funcapi.FieldTypeFloat, Units: "Kpps", Visualization: funcapi.FieldVisualBar, Visible: false, Transform: funcapi.FieldTransformNumber, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true, Chart: &funcapi.ChartOptions{Group: "BroadcastPackets"}}, Value: func(e *ifaceEntry) any { return ptrToAnyScale(e.rates.bcastPktsOut, 1_000) }},
299 + {ColumnMeta: funcapi.ColumnMeta{Name: "Packets In", Tooltip: "Packets In", Type: funcapi.FieldTypeFloat, Units: "Kpps", Visualization: funcapi.FieldVisualBar, Visible: true, Transform: funcapi.FieldTransformNumber, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, Value: func(e *ifaceEntry) any {
300 + return ptrToAnyScale(sumRates(e.rates.ucastPktsIn, e.rates.bcastPktsIn, e.rates.mcastPktsIn), 1_000)
301 + }},
302 + {ColumnMeta: funcapi.ColumnMeta{Name: "Packets Out", Tooltip: "Packets Out", Type: funcapi.FieldTypeFloat, Units: "Kpps", Visualization: funcapi.FieldVisualBar, Visible: true, Transform: funcapi.FieldTransformNumber, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, Value: func(e *ifaceEntry) any {
303 + return ptrToAnyScale(sumRates(e.rates.ucastPktsOut, e.rates.bcastPktsOut, e.rates.mcastPktsOut), 1_000)
304 + }},
305 + {ColumnMeta: funcapi.ColumnMeta{Name: "Errors In", Tooltip: "Errors In", Type: funcapi.FieldTypeFloat, Units: "packets/s", Visualization: funcapi.FieldVisualBar, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, Value: func(e *ifaceEntry) any { return ptrToAny(e.rates.errorsIn) }},
306 + {ColumnMeta: funcapi.ColumnMeta{Name: "Errors Out", Tooltip: "Errors Out", Type: funcapi.FieldTypeFloat, Units: "packets/s", Visualization: funcapi.FieldVisualBar, Visible: false, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, Value: func(e *ifaceEntry) any { return ptrToAny(e.rates.errorsOut) }},
307 + {ColumnMeta: funcapi.ColumnMeta{Name: "Discards In", Tooltip: "Discards In", Type: funcapi.FieldTypeFloat, Units: "packets/s", Visualization: funcapi.FieldVisualBar, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, Value: func(e *ifaceEntry) any { return ptrToAny(e.rates.discardsIn) }},
308 + {ColumnMeta: funcapi.ColumnMeta{Name: "Discards Out", Tooltip: "Discards Out", Type: funcapi.FieldTypeFloat, Units: "packets/s", Visualization: funcapi.FieldVisualBar, Visible: true, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true}, Value: func(e *ifaceEntry) any { return ptrToAny(e.rates.discardsOut) }},
309 + {ColumnMeta: funcapi.ColumnMeta{Name: "Multicast In", Tooltip: "Multicast In", Type: funcapi.FieldTypeFloat, Units: "Kpps", Visualization: funcapi.FieldVisualBar, Visible: false, Transform: funcapi.FieldTransformNumber, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true, Chart: &funcapi.ChartOptions{Group: "MulticastPackets", Title: "Multicast Packets"}}, Value: func(e *ifaceEntry) any { return ptrToAnyScale(e.rates.mcastPktsIn, 1_000) }},
310 + {ColumnMeta: funcapi.ColumnMeta{Name: "Multicast Out", Tooltip: "Multicast Out", Type: funcapi.FieldTypeFloat, Units: "Kpps", Visualization: funcapi.FieldVisualBar, Visible: false, Transform: funcapi.FieldTransformNumber, DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Filter: funcapi.FieldFilterRange, Sortable: true, Chart: &funcapi.ChartOptions{Group: "MulticastPackets"}}, Value: func(e *ifaceEntry) any { return ptrToAnyScale(e.rates.mcastPktsOut, 1_000) }},
311 }
src/go/plugin/go.d/collector/snmp/func_interfaces_test.go
+64 -62
@@ -3,14 +3,20 @@
3 package snmp
4
5 import (
6 + "context"
7 "testing"
8
9 "github.com/netdata/netdata/go/plugins/pkg/funcapi"
9 - "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
10 "github.com/stretchr/testify/assert"
11 "github.com/stretchr/testify/require"
12 )
13
14 +// newTestFuncInterfaces creates a funcInterfaces for testing with the given cache.
15 +func newTestFuncInterfaces(cache *ifaceCache) *funcInterfaces {
16 + r := &funcRouter{ifaceCache: cache}
17 + return newFuncInterfaces(r)
18 +}
19 +
20 func TestSnmpMethods(t *testing.T) {
21 methods := snmpMethods()
22
@@ -60,26 +66,18 @@ func TestFuncIfacesColumns(t *testing.T) {
66 "Multicast In", "Multicast Out",
67 }
68
63 - keys := make(map[string]bool)
64 - for _, col := range funcIfacesColumns {
65 - keys[col.key] = true
66 - }
67 -
69 + cs := snmpColumnSet(snmpAllColumns)
70 for _, key := range requiredKeys {
69 - assert.True(t, keys[key], "column %s should be defined", key)
71 + assert.True(t, cs.ContainsColumn(key), "column %s should be defined", key)
72 }
73 },
74 },
75 "has valid metadata": {
76 validate: func(t *testing.T) {
75 - for _, col := range funcIfacesColumns {
76 - assert.NotEmpty(t, col.key, "column must have key")
77 - assert.NotEqual(t, funcapi.FieldTypeNone, col.dataType, "column %s must have dataType", col.key)
78 - assert.NotNil(t, col.value, "column %s must have value extractor", col.key)
79 -
80 - if col.sortOption != "" {
81 - assert.NotEmpty(t, col.sortOption, "sort option column %s must have sortOption label", col.key)
82 - }
77 + for _, col := range snmpAllColumns {
78 + assert.NotEmpty(t, col.Name, "column must have ID")
79 + assert.NotEqual(t, funcapi.FieldTypeNone, col.Type, "column %s must have Type", col.Name)
80 + assert.NotNil(t, col.Value, "column %s must have Value extractor", col.Name)
81 }
82 },
83 },
@@ -101,30 +99,30 @@ func TestFuncIfacesColumns(t *testing.T) {
99 }
100
101 // Test each column's value extractor
104 - for _, col := range funcIfacesColumns {
102 + for _, col := range snmpAllColumns {
103 // Should not panic
106 - _ = col.value(entry)
104 + _ = col.Value(entry)
105 }
106
107 // Verify specific values
110 - for _, col := range funcIfacesColumns {
111 - switch col.key {
108 + for _, col := range snmpAllColumns {
109 + switch col.Name {
110 case "Interface":
113 - assert.Equal(t, "eth0", col.value(entry))
111 + assert.Equal(t, "eth0", col.Value(entry))
112 case "Type":
115 - assert.Equal(t, "ethernetCsmacd", col.value(entry))
113 + assert.Equal(t, "ethernetCsmacd", col.Value(entry))
114 case "Type Group":
117 - assert.Equal(t, "ethernet", col.value(entry))
115 + assert.Equal(t, "ethernet", col.Value(entry))
116 case "Traffic In":
119 - assert.Equal(t, rate/1_000_000, col.value(entry))
117 + assert.Equal(t, rate/1_000_000, col.Value(entry))
118 case "Packets In":
121 - assert.Equal(t, rate/1_000, col.value(entry))
119 + assert.Equal(t, rate/1_000, col.Value(entry))
120 case "Errors In":
123 - assert.Equal(t, rate, col.value(entry))
121 + assert.Equal(t, rate, col.Value(entry))
122 case "Discards In":
125 - assert.Equal(t, rate, col.value(entry))
123 + assert.Equal(t, rate, col.Value(entry))
124 case "Admin Status":
127 - assert.Equal(t, "up", col.value(entry))
125 + assert.Equal(t, "up", col.Value(entry))
126 }
127 }
128 },
@@ -140,7 +138,7 @@ func TestFuncIfacesColumns(t *testing.T) {
138 }
139 f := &funcInterfaces{}
140 row := f.buildRow(entry)
143 - assert.Len(t, row, len(funcIfacesColumns)+1, "row length must match column count")
141 + assert.Len(t, row, len(snmpAllColumns)+1, "row length must match column count")
142 },
143 },
144 }
@@ -154,21 +152,22 @@ func TestFuncIfacesColumns(t *testing.T) {
152
153 func TestFuncInterfaces_buildColumns(t *testing.T) {
154 f := &funcInterfaces{}
157 - columns := f.buildColumns()
155 + cs := snmpColumnSet(snmpAllColumns)
156 + columns := f.buildColumns(cs)
157
158 require.NotEmpty(t, columns)
160 - assert.Len(t, columns, len(funcIfacesColumns)+1)
159 + assert.Len(t, columns, len(snmpAllColumns)+1)
160
161 // Verify all columns are present
163 - for _, col := range funcIfacesColumns {
164 - colDef, ok := columns[col.key]
165 - assert.True(t, ok, "column %s should be in result", col.key)
162 + for _, col := range snmpAllColumns {
163 + colDef, ok := columns[col.Name]
164 + assert.True(t, ok, "column %s should be in result", col.Name)
165 assert.NotNil(t, colDef)
166
167 // Verify column is a map with expected fields
168 colMap, ok := colDef.(map[string]any)
170 - require.True(t, ok, "column %s should be a map", col.key)
171 - assert.Equal(t, col.name, colMap["name"])
169 + require.True(t, ok, "column %s should be a map", col.Name)
170 + assert.Equal(t, col.Tooltip, colMap["name"])
171 }
172
173 rowOptions, ok := columns["rowOptions"]
@@ -222,7 +221,7 @@ func TestFuncInterfaces_buildRow(t *testing.T) {
221 packetsOutIdx := findColIdx("Packets Out")
222 adminIdx := findColIdx("Admin Status")
223 operIdx := findColIdx("Oper Status")
225 - rowOptionsIdx := len(funcIfacesColumns)
224 + rowOptionsIdx := len(snmpAllColumns)
225
226 assert.Equal(t, "eth0", row[nameIdx])
227 assert.Equal(t, "ethernetCsmacd", row[typeIdx])
@@ -252,7 +251,7 @@ func TestFuncInterfaces_buildRow(t *testing.T) {
251 trafficOutIdx := findColIdx("Traffic Out")
252 adminIdx := findColIdx("Admin Status")
253 operIdx := findColIdx("Oper Status")
255 - rowOptionsIdx := len(funcIfacesColumns)
254 + rowOptionsIdx := len(snmpAllColumns)
255
256 assert.Equal(t, "eth1", row[nameIdx])
257 assert.Equal(t, "other", row[typeIdx])
@@ -280,7 +279,7 @@ func TestFuncInterfaces_buildRow(t *testing.T) {
279 trafficInIdx := findColIdx("Traffic In")
280 trafficOutIdx := findColIdx("Traffic Out")
281 errorsInIdx := findColIdx("Errors In")
283 - rowOptionsIdx := len(funcIfacesColumns)
282 + rowOptionsIdx := len(snmpAllColumns)
283
284 assert.Nil(t, row[trafficInIdx])
285 assert.Nil(t, row[trafficOutIdx])
@@ -306,7 +305,7 @@ func TestFuncInterfaces_buildRow(t *testing.T) {
305 trafficInIdx := findColIdx("Traffic In")
306 trafficOutIdx := findColIdx("Traffic Out")
307 ucastInIdx := findColIdx("Unicast In")
309 - rowOptionsIdx := len(funcIfacesColumns)
308 + rowOptionsIdx := len(snmpAllColumns)
309
310 assert.Equal(t, "eth2", row[nameIdx])
311 assert.Nil(t, row[trafficInIdx])
@@ -321,7 +320,7 @@ func TestFuncInterfaces_buildRow(t *testing.T) {
320 t.Run(name, func(t *testing.T) {
321 f := &funcInterfaces{}
322 row := f.buildRow(tc.entry)
324 - require.Len(t, row, len(funcIfacesColumns)+1)
323 + require.Len(t, row, len(snmpAllColumns)+1)
324 tc.validate(t, row)
325 })
326 }
@@ -334,9 +333,9 @@ func TestFuncInterfaces_sortData(t *testing.T) {
333
334 // Helper to build a test row with name and trafficIn
335 buildTestRow := func(name string, trafficIn *float64) []any {
337 - row := make([]any, len(funcIfacesColumns)+1)
338 - for i, col := range funcIfacesColumns {
339 - switch col.key {
336 + row := make([]any, len(snmpAllColumns)+1)
337 + for i, col := range snmpAllColumns {
338 + switch col.Name {
339 case "Interface":
340 row[i] = name
341 case "Traffic In":
@@ -344,14 +343,14 @@ func TestFuncInterfaces_sortData(t *testing.T) {
343 case "Traffic Out":
344 row[i] = ptrToAny(trafficIn) // reuse for simplicity
345 default:
347 - if col.dataType == funcapi.FieldTypeString {
346 + if col.Type == funcapi.FieldTypeString {
347 row[i] = "test"
348 } else {
349 row[i] = nil
350 }
351 }
352 }
354 - row[len(funcIfacesColumns)] = nil
353 + row[len(snmpAllColumns)] = nil
354 return row
355 }
356
@@ -426,40 +425,40 @@ func TestFuncInterfaces_handle(t *testing.T) {
425 setup func() *funcInterfaces
426 method string
427 params funcapi.ResolvedParams
429 - validate func(t *testing.T, resp *module.FunctionResponse)
428 + validate func(t *testing.T, resp *funcapi.FunctionResponse)
429 }{
430 "unknown method returns 404": {
431 setup: func() *funcInterfaces {
433 - return newFuncInterfaces(newIfaceCache())
432 + return newTestFuncInterfaces(newIfaceCache())
433 },
434 method: "unknown",
435 params: funcapi.ResolvedParams{},
437 - validate: func(t *testing.T, resp *module.FunctionResponse) {
436 + validate: func(t *testing.T, resp *funcapi.FunctionResponse) {
437 assert.Equal(t, 404, resp.Status)
438 assert.Contains(t, resp.Message, "unknown method")
439 },
440 },
441 "nil cache returns 503": {
442 setup: func() *funcInterfaces {
444 - return &funcInterfaces{cache: nil}
443 + return newTestFuncInterfaces(nil)
444 },
445 method: "interfaces",
446 params: funcapi.ResolvedParams{},
448 - validate: func(t *testing.T, resp *module.FunctionResponse) {
447 + validate: func(t *testing.T, resp *funcapi.FunctionResponse) {
448 assert.Equal(t, 503, resp.Status)
449 assert.Contains(t, resp.Message, "not available")
450 },
451 },
452 "empty cache returns 200 with empty data": {
453 setup: func() *funcInterfaces {
455 - return newFuncInterfaces(newIfaceCache())
454 + return newTestFuncInterfaces(newIfaceCache())
455 },
456 method: "interfaces",
457 params: resolveIfaceParams(nil),
459 - validate: func(t *testing.T, resp *module.FunctionResponse) {
458 + validate: func(t *testing.T, resp *funcapi.FunctionResponse) {
459 assert.Equal(t, 200, resp.Status)
460 assert.NotNil(t, resp.Columns)
462 - assert.Len(t, resp.Columns, len(funcIfacesColumns)+1)
461 + assert.Len(t, resp.Columns, len(snmpAllColumns)+1)
462
463 data, ok := resp.Data.([][]any)
464 require.True(t, ok)
@@ -487,11 +486,11 @@ func TestFuncInterfaces_handle(t *testing.T) {
486 adminStatus: "down",
487 operStatus: "down",
488 }
490 - return newFuncInterfaces(cache)
489 + return newTestFuncInterfaces(cache)
490 },
491 method: "interfaces",
492 params: resolveIfaceParams(nil),
494 - validate: func(t *testing.T, resp *module.FunctionResponse) {
493 + validate: func(t *testing.T, resp *funcapi.FunctionResponse) {
494 assert.Equal(t, 200, resp.Status)
495 assert.Equal(t, "Interface", resp.DefaultSortColumn)
496
@@ -507,7 +506,10 @@ func TestFuncInterfaces_handle(t *testing.T) {
506
507 // Verify DefaultCharts
508 require.NotEmpty(t, resp.DefaultCharts)
510 - assert.Equal(t, [][]string{{"Traffic", "Type"}, {"OperationalStatus", "Oper Status"}}, resp.DefaultCharts)
509 + assert.ElementsMatch(t, funcapi.DefaultCharts{
510 + {Chart: "Traffic", GroupBy: "Type"},
511 + {Chart: "OperationalStatus", GroupBy: "Oper Status"},
512 + }, resp.DefaultCharts)
513
514 // Verify GroupBy
515 require.NotNil(t, resp.GroupBy)
@@ -552,11 +554,11 @@ func TestFuncInterfaces_handle(t *testing.T) {
554 adminStatus: "up",
555 operStatus: "up",
556 }
555 - return newFuncInterfaces(cache)
557 + return newTestFuncInterfaces(cache)
558 },
559 method: "interfaces",
560 params: resolveIfaceParams(map[string][]string{"if_type_group": {"other"}}),
559 - validate: func(t *testing.T, resp *module.FunctionResponse) {
561 + validate: func(t *testing.T, resp *funcapi.FunctionResponse) {
562 assert.Equal(t, 200, resp.Status)
563
564 data, ok := resp.Data.([][]any)
@@ -573,7 +575,7 @@ func TestFuncInterfaces_handle(t *testing.T) {
575 for name, tc := range tests {
576 t.Run(name, func(t *testing.T) {
577 f := tc.setup()
576 - resp := f.handle(tc.method, tc.params)
578 + resp := f.Handle(context.Background(), tc.method, tc.params)
579 tc.validate(t, resp)
580 })
581 }
@@ -613,8 +615,8 @@ func TestPtrToAny(t *testing.T) {
615
616 // findColIdx finds the index of a column by key.
617 func findColIdx(key string) int {
616 - for i, col := range funcIfacesColumns {
617 - if col.key == key {
618 + for i, col := range snmpAllColumns {
619 + if col.Name == key {
620 return i
621 }
622 }
@@ -623,7 +625,7 @@ func findColIdx(key string) int {
625
626 func resolveIfaceParams(values map[string][]string) funcapi.ResolvedParams {
627 f := &funcInterfaces{}
626 - params, err := f.methodParams("interfaces")
628 + params, err := f.MethodParams(context.Background(), "interfaces")
629 if err != nil {
630 return nil
631 }
src/go/plugin/go.d/collector/snmp/func_router.go new
+64
@@ -0,0 +1,64 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package snmp
4 +
5 +import (
6 + "context"
7 + "fmt"
8 +
9 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
10 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
11 +)
12 +
13 +// funcRouter routes method calls to appropriate function handlers.
14 +type funcRouter struct {
15 + ifaceCache *ifaceCache
16 +
17 + handlers map[string]funcapi.MethodHandler
18 +}
19 +
20 +func newFuncRouter(cache *ifaceCache) *funcRouter {
21 + r := &funcRouter{
22 + ifaceCache: cache,
23 + handlers: make(map[string]funcapi.MethodHandler),
24 + }
25 + r.handlers[ifacesMethodID] = newFuncInterfaces(r)
26 + return r
27 +}
28 +
29 +// Compile-time interface check.
30 +var _ funcapi.MethodHandler = (*funcRouter)(nil)
31 +
32 +func (r *funcRouter) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
33 + if h, ok := r.handlers[method]; ok {
34 + return h.MethodParams(ctx, method)
35 + }
36 + return nil, fmt.Errorf("unknown method: %s", method)
37 +}
38 +
39 +func (r *funcRouter) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
40 + if h, ok := r.handlers[method]; ok {
41 + return h.Handle(ctx, method, params)
42 + }
43 + return funcapi.NotFoundResponse(method)
44 +}
45 +
46 +func (r *funcRouter) Cleanup(ctx context.Context) {
47 + for _, h := range r.handlers {
48 + h.Cleanup(ctx)
49 + }
50 +}
51 +
52 +func snmpMethods() []funcapi.MethodConfig {
53 + return []funcapi.MethodConfig{
54 + ifacesMethodConfig(),
55 + }
56 +}
57 +
58 +func snmpFunctionHandler(job *module.Job) funcapi.MethodHandler {
59 + c, ok := job.Module().(*Collector)
60 + if !ok {
61 + return nil
62 + }
63 + return c.funcRouter
64 +}
src/go/plugin/go.d/collector/yugabytedb/collector.go
+10 -14
@@ -4,12 +4,10 @@ package yugabytedb
4
5 import (
6 "context"
7 - "database/sql"
7 _ "embed"
8 "errors"
9 "fmt"
10 "net/http"
12 - "sync"
11 "time"
12
13 "github.com/netdata/netdata/go/plugins/pkg/confopt"
@@ -27,11 +25,10 @@ func init() {
25 Defaults: module.Defaults{
26 UpdateEvery: 5,
27 },
30 - Methods: yugabyteMethods,
31 - MethodParams: yugabyteMethodParams,
32 - HandleMethod: yugabyteHandleMethod,
33 - Create: func() module.Module { return New() },
34 - Config: func() any { return &Config{} },
28 + Methods: yugabyteMethods,
29 + MethodHandler: yugabyteFunctionHandler,
30 + Create: func() module.Module { return New() },
31 + Config: func() any { return &Config{} },
32 })
33 }
34
@@ -77,10 +74,7 @@ type Collector struct {
74
75 cache map[string]map[string]bool
76
80 - db *sql.DB
81 -
82 - pgStatStatementsMu sync.RWMutex
83 - pgStatStatementsColumns map[string]bool
77 + funcRouter *funcRouter
78 }
79
80 func (c *Collector) Configuration() any {
@@ -104,6 +98,8 @@ func (c *Collector) Init(context.Context) error {
98 }
99 c.prom = prom
100
101 + c.funcRouter = newFuncRouter(c)
102 +
103 return nil
104 }
105
@@ -135,11 +131,11 @@ func (c *Collector) Collect(context.Context) map[string]int64 {
131 return mx
132 }
133
138 -func (c *Collector) Cleanup(context.Context) {
134 +func (c *Collector) Cleanup(ctx context.Context) {
135 if c.httpClient != nil {
136 c.httpClient.CloseIdleConnections()
137 }
142 - if c.db != nil {
143 - _ = c.db.Close()
138 + if c.funcRouter != nil {
139 + c.funcRouter.Cleanup(ctx)
140 }
141 }
src/go/plugin/go.d/collector/yugabytedb/func_router.go new
+133
@@ -0,0 +1,133 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package yugabytedb
4 +
5 +import (
6 + "context"
7 + "database/sql"
8 + "errors"
9 + "fmt"
10 + "sync"
11 + "time"
12 +
13 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
14 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
15 +)
16 +
17 +var errSQLDSNNotSet = errors.New("SQL DSN is not set")
18 +
19 +// funcRouter routes method calls to appropriate function handlers.
20 +// Owns shared SQL connection used by all function handlers.
21 +type funcRouter struct {
22 + collector *Collector // for config (DSN, SQLTimeout, TopQueriesLimit, logger)
23 +
24 + handlers map[string]funcapi.MethodHandler
25 +
26 + // Shared SQL connection
27 + db *sql.DB
28 + dbMu sync.Mutex
29 +
30 + // Column detection cache for pg_stat_statements
31 + pgStatStatementsColumns map[string]bool
32 + pgStatStatementsColumnsMu sync.RWMutex
33 +}
34 +
35 +func newFuncRouter(c *Collector) *funcRouter {
36 + r := &funcRouter{
37 + collector: c,
38 + handlers: make(map[string]funcapi.MethodHandler),
39 + }
40 + r.handlers[topQueriesMethodID] = newFuncTopQueries(r)
41 + r.handlers[runningQueriesMethodID] = newFuncRunningQueries(r)
42 + return r
43 +}
44 +
45 +// Compile-time interface check.
46 +var _ funcapi.MethodHandler = (*funcRouter)(nil)
47 +
48 +func (r *funcRouter) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
49 + if h, ok := r.handlers[method]; ok {
50 + return h.MethodParams(ctx, method)
51 + }
52 + return nil, fmt.Errorf("unknown method: %s", method)
53 +}
54 +
55 +func (r *funcRouter) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
56 + if h, ok := r.handlers[method]; ok {
57 + return h.Handle(ctx, method, params)
58 + }
59 + return funcapi.NotFoundResponse(method)
60 +}
61 +
62 +func (r *funcRouter) Cleanup(ctx context.Context) {
63 + for _, h := range r.handlers {
64 + h.Cleanup(ctx)
65 + }
66 + r.dbMu.Lock()
67 + defer r.dbMu.Unlock()
68 + if r.db != nil {
69 + _ = r.db.Close()
70 + r.db = nil
71 + }
72 +}
73 +
74 +// ensureDB lazily initializes the SQL connection.
75 +func (r *funcRouter) ensureDB(ctx context.Context) error {
76 + r.dbMu.Lock()
77 + defer r.dbMu.Unlock()
78 +
79 + if r.db != nil {
80 + return nil
81 + }
82 + if r.collector.DSN == "" {
83 + return errSQLDSNNotSet
84 + }
85 +
86 + db, err := sql.Open("pgx", r.collector.DSN)
87 + if err != nil {
88 + return fmt.Errorf("error opening SQL connection: %w", err)
89 + }
90 + db.SetMaxOpenConns(1)
91 + db.SetMaxIdleConns(1)
92 + db.SetConnMaxLifetime(10 * time.Minute)
93 +
94 + timeout := r.sqlTimeout()
95 + pingCtx, cancel := context.WithTimeout(ctx, timeout)
96 + defer cancel()
97 + if err := db.PingContext(pingCtx); err != nil {
98 + _ = db.Close()
99 + return fmt.Errorf("error pinging SQL connection: %w", err)
100 + }
101 +
102 + r.db = db
103 + return nil
104 +}
105 +
106 +func (r *funcRouter) sqlTimeout() time.Duration {
107 + if r.collector.SQLTimeout.Duration() > 0 {
108 + return r.collector.SQLTimeout.Duration()
109 + }
110 + return time.Second
111 +}
112 +
113 +func (r *funcRouter) topQueriesLimit() int {
114 + if r.collector.TopQueriesLimit > 0 {
115 + return r.collector.TopQueriesLimit
116 + }
117 + return 500
118 +}
119 +
120 +func yugabyteMethods() []funcapi.MethodConfig {
121 + return []funcapi.MethodConfig{
122 + topQueriesMethodConfig(),
123 + runningQueriesMethodConfig(),
124 + }
125 +}
126 +
127 +func yugabyteFunctionHandler(job *module.Job) funcapi.MethodHandler {
128 + c, ok := job.Module().(*Collector)
129 + if !ok {
130 + return nil
131 + }
132 + return c.funcRouter
133 +}
src/go/plugin/go.d/collector/yugabytedb/func_running_queries.go new
+230
@@ -0,0 +1,230 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package yugabytedb
4 +
5 +import (
6 + "context"
7 + "database/sql"
8 + "errors"
9 + "fmt"
10 + "strings"
11 +
12 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
13 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
14 +)
15 +
16 +const (
17 + runningQueriesMethodID = "running-queries"
18 + runningQueriesMaxTextLength = 4096
19 +)
20 +
21 +func runningQueriesMethodConfig() funcapi.MethodConfig {
22 + return funcapi.MethodConfig{
23 + ID: runningQueriesMethodID,
24 + Name: "Running Queries",
25 + UpdateEvery: 10,
26 + Help: "Currently running SQL statements from pg_stat_activity. WARNING: Query text may contain unmasked literals (potential PII).",
27 + RequireCloud: true,
28 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(runningQueriesColumns)},
29 + }
30 +}
31 +
32 +// runningQueriesColumn embeds funcapi.ColumnMeta and adds YugabyteDB-specific fields.
33 +type runningQueriesColumn struct {
34 + funcapi.ColumnMeta
35 + SelectExpr string // SQL expression for SELECT clause
36 + sortOpt bool // whether this column appears as a sort option
37 + sortLbl string // label for sort option dropdown
38 + defaultSort bool // default sort column
39 +}
40 +
41 +var runningQueriesColumns = []runningQueriesColumn{
42 + {ColumnMeta: funcapi.ColumnMeta{Name: "pid", Tooltip: "PID", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, UniqueKey: true, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.pid::text"},
43 + {ColumnMeta: funcapi.ColumnMeta{Name: "query", Tooltip: "Query", Type: funcapi.FieldTypeString, Visible: true, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sticky: true, FullWidth: true, Wrap: true}, SelectExpr: "s.query"},
44 + {ColumnMeta: funcapi.ColumnMeta{Name: "database", Tooltip: "Database", Type: funcapi.FieldTypeString, Visible: true, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.datname"},
45 + {ColumnMeta: funcapi.ColumnMeta{Name: "user", Tooltip: "User", Type: funcapi.FieldTypeString, Visible: true, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.usename"},
46 + {ColumnMeta: funcapi.ColumnMeta{Name: "state", Tooltip: "State", Type: funcapi.FieldTypeString, Visible: true, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.state"},
47 + {ColumnMeta: funcapi.ColumnMeta{Name: "waitEventType", Tooltip: "Wait Event Type", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.wait_event_type"},
48 + {ColumnMeta: funcapi.ColumnMeta{Name: "waitEvent", Tooltip: "Wait Event", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.wait_event"},
49 + {ColumnMeta: funcapi.ColumnMeta{Name: "application", Tooltip: "Application", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.application_name"},
50 + {ColumnMeta: funcapi.ColumnMeta{Name: "clientAddress", Tooltip: "Client Address", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.client_addr::text"},
51 + {ColumnMeta: funcapi.ColumnMeta{Name: "queryStart", Tooltip: "Query Start", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax}, SelectExpr: "TO_CHAR(s.query_start, 'YYYY-MM-DD\"T\"HH24:MI:SS.FF3')"},
52 + {ColumnMeta: funcapi.ColumnMeta{Name: "elapsedMs", Tooltip: "Elapsed", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax}, SelectExpr: "CASE WHEN s.query_start IS NULL THEN 0 ELSE EXTRACT(EPOCH FROM (clock_timestamp() - s.query_start)) * 1000 END", sortOpt: true, defaultSort: true, sortLbl: "Running queries by Elapsed Time"},
53 +}
54 +
55 +// funcapi.SortableColumn interface implementation for runningQueriesColumn.
56 +func (c runningQueriesColumn) IsSortOption() bool { return c.sortOpt }
57 +func (c runningQueriesColumn) SortLabel() string { return c.sortLbl }
58 +func (c runningQueriesColumn) IsDefaultSort() bool { return c.defaultSort }
59 +func (c runningQueriesColumn) ColumnName() string { return c.Name }
60 +func (c runningQueriesColumn) SortColumn() string { return "" }
61 +
62 +// funcRunningQueries handles the running-queries function.
63 +type funcRunningQueries struct {
64 + router *funcRouter
65 +}
66 +
67 +func newFuncRunningQueries(r *funcRouter) *funcRunningQueries {
68 + return &funcRunningQueries{router: r}
69 +}
70 +
71 +// Compile-time interface check.
72 +var _ funcapi.MethodHandler = (*funcRunningQueries)(nil)
73 +
74 +func (f *funcRunningQueries) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
75 + return []funcapi.ParamConfig{funcapi.BuildSortParam(runningQueriesColumns)}, nil
76 +}
77 +
78 +func (f *funcRunningQueries) Cleanup(ctx context.Context) {}
79 +
80 +func (f *funcRunningQueries) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
81 + if err := f.router.ensureDB(ctx); err != nil {
82 + status := 503
83 + if errors.Is(err, errSQLDSNNotSet) {
84 + status = 400
85 + }
86 + return funcapi.ErrorResponse(status, "%s", err)
87 + }
88 +
89 + sortColumn := f.resolveSortColumn(params.Column("__sort"))
90 + limit := f.router.topQueriesLimit()
91 +
92 + query := f.buildSQL(sortColumn)
93 + queryCtx, cancel := context.WithTimeout(ctx, f.router.sqlTimeout())
94 + defer cancel()
95 +
96 + rows, err := f.router.db.QueryContext(queryCtx, query, limit)
97 + if err != nil {
98 + if errors.Is(queryCtx.Err(), context.DeadlineExceeded) {
99 + return funcapi.ErrorResponse(504, "query timed out")
100 + }
101 + return funcapi.InternalErrorResponse("query failed: %v", err)
102 + }
103 + defer rows.Close()
104 +
105 + data, err := f.scanRows(rows)
106 + if err != nil {
107 + return funcapi.InternalErrorResponse("%s", err)
108 + }
109 +
110 + cs := f.columnSet()
111 + return &funcapi.FunctionResponse{
112 + Status: 200,
113 + Help: "Currently running SQL statements from pg_stat_activity. WARNING: Query text may contain unmasked literals (potential PII).",
114 + Columns: cs.BuildColumns(),
115 + Data: data,
116 + DefaultSortColumn: sortColumn,
117 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(runningQueriesColumns)},
118 + }
119 +}
120 +
121 +func (f *funcRunningQueries) columnSet() funcapi.ColumnSet[runningQueriesColumn] {
122 + return funcapi.Columns(runningQueriesColumns, func(c runningQueriesColumn) funcapi.ColumnMeta { return c.ColumnMeta })
123 +}
124 +
125 +func (f *funcRunningQueries) resolveSortColumn(requested string) string {
126 + if requested != "" {
127 + for _, col := range runningQueriesColumns {
128 + if col.Name == requested && col.IsSortOption() {
129 + return col.Name
130 + }
131 + }
132 + }
133 + for _, col := range runningQueriesColumns {
134 + if col.IsDefaultSort() && col.IsSortOption() {
135 + return col.Name
136 + }
137 + }
138 + for _, col := range runningQueriesColumns {
139 + if col.IsSortOption() {
140 + return col.Name
141 + }
142 + }
143 + if len(runningQueriesColumns) > 0 {
144 + return runningQueriesColumns[0].Name
145 + }
146 + return ""
147 +}
148 +
149 +func (f *funcRunningQueries) buildSQL(sortColumn string) string {
150 + selectCols := make([]string, 0, len(runningQueriesColumns))
151 + for _, col := range runningQueriesColumns {
152 + selectCols = append(selectCols, fmt.Sprintf("%s AS %s", col.SelectExpr, col.Name))
153 + }
154 + return fmt.Sprintf(`
155 +SELECT %s
156 +FROM pg_stat_activity s
157 +WHERE s.state IS DISTINCT FROM 'idle'
158 +ORDER BY %s DESC NULLS LAST
159 +LIMIT $1`, strings.Join(selectCols, ", "), sortColumn)
160 +}
161 +
162 +func (f *funcRunningQueries) scanRows(rows *sql.Rows) ([][]any, error) {
163 + cols := runningQueriesColumns
164 + data := make([][]any, 0, 500)
165 +
166 + for rows.Next() {
167 + values := make([]any, len(cols))
168 + valuePtrs := make([]any, len(cols))
169 +
170 + for i, col := range cols {
171 + switch col.Type {
172 + case funcapi.FieldTypeString:
173 + var v sql.NullString
174 + values[i] = &v
175 + case funcapi.FieldTypeInteger:
176 + var v sql.NullInt64
177 + values[i] = &v
178 + case funcapi.FieldTypeFloat, funcapi.FieldTypeDuration:
179 + var v sql.NullFloat64
180 + values[i] = &v
181 + default:
182 + var v any
183 + values[i] = &v
184 + }
185 + valuePtrs[i] = values[i]
186 + }
187 +
188 + if err := rows.Scan(valuePtrs...); err != nil {
189 + return nil, fmt.Errorf("row scan failed: %w", err)
190 + }
191 +
192 + row := make([]any, len(cols))
193 + for i, col := range cols {
194 + switch v := values[i].(type) {
195 + case *sql.NullString:
196 + if v.Valid {
197 + s := v.String
198 + if col.Name == "query" {
199 + s = strmutil.TruncateText(s, runningQueriesMaxTextLength)
200 + }
201 + row[i] = s
202 + } else {
203 + row[i] = ""
204 + }
205 + case *sql.NullInt64:
206 + if v.Valid {
207 + row[i] = v.Int64
208 + } else {
209 + row[i] = int64(0)
210 + }
211 + case *sql.NullFloat64:
212 + if v.Valid {
213 + row[i] = v.Float64
214 + } else {
215 + row[i] = float64(0)
216 + }
217 + default:
218 + row[i] = nil
219 + }
220 + }
221 +
222 + data = append(data, row)
223 + }
224 +
225 + if err := rows.Err(); err != nil {
226 + return nil, fmt.Errorf("rows iteration error: %w", err)
227 + }
228 +
229 + return data, nil
230 +}
src/go/plugin/go.d/collector/yugabytedb/func_top_queries.go new
+399
@@ -0,0 +1,399 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package yugabytedb
4 +
5 +import (
6 + "context"
7 + "database/sql"
8 + "errors"
9 + "fmt"
10 + "strings"
11 +
12 + "github.com/netdata/netdata/go/plugins/pkg/funcapi"
13 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
14 +)
15 +
16 +const (
17 + topQueriesMethodID = "top-queries"
18 + topQueriesMaxTextLength = 4096
19 +)
20 +
21 +func topQueriesMethodConfig() funcapi.MethodConfig {
22 + return funcapi.MethodConfig{
23 + ID: topQueriesMethodID,
24 + Name: "Top Queries",
25 + UpdateEvery: 10,
26 + Help: "Top SQL queries from pg_stat_statements. WARNING: Query text may contain unmasked literals (potential PII).",
27 + RequireCloud: true,
28 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(topQueriesColumns)},
29 + }
30 +}
31 +
32 +// topQueriesColumn embeds funcapi.ColumnMeta and adds YugabyteDB-specific fields.
33 +type topQueriesColumn struct {
34 + funcapi.ColumnMeta
35 + SelectExpr string // SQL expression for SELECT clause
36 + sortOpt bool // whether this column appears as a sort option
37 + sortLbl string // label for sort option dropdown
38 + defaultSort bool // default sort column
39 + IsJoinColumn bool // column comes from a JOIN, not pg_stat_statements
40 +}
41 +
42 +var topQueriesColumns = []topQueriesColumn{
43 + {ColumnMeta: funcapi.ColumnMeta{Name: "queryId", Tooltip: "Query ID", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, UniqueKey: true, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount}, SelectExpr: "s.queryid::text"},
44 + {ColumnMeta: funcapi.ColumnMeta{Name: "query", Tooltip: "Query", Type: funcapi.FieldTypeString, Visible: true, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sticky: true, FullWidth: true, Wrap: true}, SelectExpr: "s.query"},
45 + {ColumnMeta: funcapi.ColumnMeta{Name: "database", Tooltip: "Database", Type: funcapi.FieldTypeString, Visible: true, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, GroupBy: &funcapi.GroupByOptions{IsDefault: true}}, SelectExpr: "d.datname", IsJoinColumn: true},
46 + {ColumnMeta: funcapi.ColumnMeta{Name: "user", Tooltip: "User", Type: funcapi.FieldTypeString, Visible: true, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Transform: funcapi.FieldTransformText, Sort: funcapi.FieldSortAscending, Summary: funcapi.FieldSummaryCount, GroupBy: &funcapi.GroupByOptions{}}, SelectExpr: "u.usename", IsJoinColumn: true},
47 +
48 + {ColumnMeta: funcapi.ColumnMeta{Name: "calls", Tooltip: "Calls", Type: funcapi.FieldTypeInteger, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "Calls", Title: "Number of Calls", IsDefault: true}}, SelectExpr: "s.calls", sortOpt: true, sortLbl: "Top queries by Calls"},
49 + {ColumnMeta: funcapi.ColumnMeta{Name: "totalTime", Tooltip: "Total Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Execution Time", IsDefault: true}}, SelectExpr: "s.total_time", sortOpt: true, defaultSort: true, sortLbl: "Top queries by Total Time"},
50 + {ColumnMeta: funcapi.ColumnMeta{Name: "meanTime", Tooltip: "Mean Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMean, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Execution Time"}}, SelectExpr: "s.mean_time", sortOpt: true, sortLbl: "Top queries by Mean Time"},
51 + {ColumnMeta: funcapi.ColumnMeta{Name: "minTime", Tooltip: "Min Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: false, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMin, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Execution Time"}}, SelectExpr: "s.min_time"},
52 + {ColumnMeta: funcapi.ColumnMeta{Name: "maxTime", Tooltip: "Max Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: false, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Execution Time"}}, SelectExpr: "s.max_time", sortOpt: true, sortLbl: "Top queries by Max Time"},
53 + {ColumnMeta: funcapi.ColumnMeta{Name: "rows", Tooltip: "Rows", Type: funcapi.FieldTypeInteger, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "Rows", Title: "Rows"}}, SelectExpr: "s.rows", sortOpt: true, sortLbl: "Top queries by Rows Returned"},
54 + {ColumnMeta: funcapi.ColumnMeta{Name: "stddevTime", Tooltip: "Stddev Time", Type: funcapi.FieldTypeDuration, Units: "milliseconds", DecimalPoints: 2, Visible: false, Sortable: true, Filter: funcapi.FieldFilterRange, Transform: funcapi.FieldTransformDuration, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Execution Time"}}, SelectExpr: "s.stddev_time"},
55 +}
56 +
57 +// funcapi.SortableColumn interface implementation for topQueriesColumn.
58 +func (c topQueriesColumn) IsSortOption() bool { return c.sortOpt }
59 +func (c topQueriesColumn) SortLabel() string { return c.sortLbl }
60 +func (c topQueriesColumn) IsDefaultSort() bool { return c.defaultSort }
61 +func (c topQueriesColumn) ColumnName() string { return c.Name }
62 +func (c topQueriesColumn) SortColumn() string { return "" }
63 +
64 +// funcTopQueries handles the top-queries function.
65 +type funcTopQueries struct {
66 + router *funcRouter
67 +}
68 +
69 +func newFuncTopQueries(r *funcRouter) *funcTopQueries {
70 + return &funcTopQueries{router: r}
71 +}
72 +
73 +// Compile-time interface check.
74 +var _ funcapi.MethodHandler = (*funcTopQueries)(nil)
75 +
76 +func (f *funcTopQueries) MethodParams(ctx context.Context, method string) ([]funcapi.ParamConfig, error) {
77 + if err := f.router.ensureDB(ctx); err != nil {
78 + return nil, nil // Use static RequiredParams
79 + }
80 + cols, err := f.availableColumns(ctx)
81 + if err != nil {
82 + return nil, nil // Use static RequiredParams
83 + }
84 + return []funcapi.ParamConfig{funcapi.BuildSortParam(cols)}, nil
85 +}
86 +
87 +func (f *funcTopQueries) Cleanup(ctx context.Context) {}
88 +
89 +func (f *funcTopQueries) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse {
90 + if err := f.router.ensureDB(ctx); err != nil {
91 + status := 503
92 + if errors.Is(err, errSQLDSNNotSet) {
93 + status = 400
94 + }
95 + return funcapi.ErrorResponse(status, "%s", err)
96 + }
97 +
98 + ok, err := f.pgStatStatementsEnabled(ctx)
99 + if err != nil {
100 + return funcapi.InternalErrorResponse("failed to check pg_stat_statements: %v", err)
101 + }
102 + if !ok {
103 + return funcapi.UnavailableResponse(
104 + "pg_stat_statements extension is not installed in this database. " +
105 + "Run 'CREATE EXTENSION pg_stat_statements;' in the database the collector connects to.",
106 + )
107 + }
108 +
109 + cols, err := f.availableColumns(ctx)
110 + if err != nil {
111 + return funcapi.InternalErrorResponse("%s", err)
112 + }
113 +
114 + sortColumn := f.resolveSortColumn(cols, params.Column("__sort"))
115 + limit := f.router.topQueriesLimit()
116 +
117 + query := f.buildSQL(cols, sortColumn)
118 + queryCtx, cancel := context.WithTimeout(ctx, f.router.sqlTimeout())
119 + defer cancel()
120 +
121 + rows, err := f.router.db.QueryContext(queryCtx, query, limit)
122 + if err != nil {
123 + if errors.Is(queryCtx.Err(), context.DeadlineExceeded) {
124 + return funcapi.ErrorResponse(504, "query timed out")
125 + }
126 + return funcapi.InternalErrorResponse("query failed: %v", err)
127 + }
128 + defer rows.Close()
129 +
130 + data, err := f.scanRows(rows, cols)
131 + if err != nil {
132 + return funcapi.InternalErrorResponse("%s", err)
133 + }
134 +
135 + cs := f.columnSet(cols)
136 + return &funcapi.FunctionResponse{
137 + Status: 200,
138 + Help: "Top SQL queries from pg_stat_statements. WARNING: Query text may contain unmasked literals (potential PII).",
139 + Columns: cs.BuildColumns(),
140 + Data: data,
141 + DefaultSortColumn: sortColumn,
142 + RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(cols)},
143 + ChartingConfig: cs.BuildCharting(),
144 + }
145 +}
146 +
147 +func (f *funcTopQueries) columnSet(cols []topQueriesColumn) funcapi.ColumnSet[topQueriesColumn] {
148 + return funcapi.Columns(cols, func(c topQueriesColumn) funcapi.ColumnMeta { return c.ColumnMeta })
149 +}
150 +
151 +func (f *funcTopQueries) availableColumns(ctx context.Context) ([]topQueriesColumn, error) {
152 + available, err := f.detectPgStatStatementsColumns(ctx)
153 + if err != nil {
154 + return nil, fmt.Errorf("failed to detect available columns: %v", err)
155 + }
156 + cols := f.buildAvailableColumns(available)
157 + if len(cols) == 0 {
158 + return nil, fmt.Errorf("no queryable columns found in pg_stat_statements")
159 + }
160 + return cols, nil
161 +}
162 +
163 +func (f *funcTopQueries) pgStatStatementsEnabled(ctx context.Context) (bool, error) {
164 + query := `SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements'`
165 + queryCtx, cancel := context.WithTimeout(ctx, f.router.sqlTimeout())
166 + defer cancel()
167 +
168 + var exists int
169 + if err := f.router.db.QueryRowContext(queryCtx, query).Scan(&exists); err != nil {
170 + if errors.Is(queryCtx.Err(), context.DeadlineExceeded) {
171 + return false, queryCtx.Err()
172 + }
173 + if errors.Is(err, sql.ErrNoRows) {
174 + return false, nil
175 + }
176 + return false, err
177 + }
178 + return true, nil
179 +}
180 +
181 +func (f *funcTopQueries) detectPgStatStatementsColumns(ctx context.Context) (map[string]bool, error) {
182 + f.router.pgStatStatementsColumnsMu.RLock()
183 + if f.router.pgStatStatementsColumns != nil {
184 + cols := f.router.pgStatStatementsColumns
185 + f.router.pgStatStatementsColumnsMu.RUnlock()
186 + return cols, nil
187 + }
188 + f.router.pgStatStatementsColumnsMu.RUnlock()
189 +
190 + f.router.pgStatStatementsColumnsMu.Lock()
191 + defer f.router.pgStatStatementsColumnsMu.Unlock()
192 +
193 + if f.router.pgStatStatementsColumns != nil {
194 + return f.router.pgStatStatementsColumns, nil
195 + }
196 +
197 + query := `
198 + SELECT column_name
199 + FROM information_schema.columns
200 + WHERE table_name = 'pg_stat_statements'
201 + AND table_schema = 'public'
202 + `
203 + queryCtx, cancel := context.WithTimeout(ctx, f.router.sqlTimeout())
204 + defer cancel()
205 +
206 + rows, err := f.router.db.QueryContext(queryCtx, query)
207 + if err != nil {
208 + return nil, fmt.Errorf("failed to query columns: %v", err)
209 + }
210 + defer rows.Close()
211 +
212 + cols := make(map[string]bool)
213 + for rows.Next() {
214 + var colName string
215 + if err := rows.Scan(&colName); err != nil {
216 + return nil, fmt.Errorf("failed to scan column name: %v", err)
217 + }
218 + cols[colName] = true
219 + }
220 +
221 + if err := rows.Err(); err != nil {
222 + return nil, fmt.Errorf("rows iteration error: %v", err)
223 + }
224 +
225 + f.router.pgStatStatementsColumns = cols
226 + return cols, nil
227 +}
228 +
229 +func (f *funcTopQueries) buildAvailableColumns(availableCols map[string]bool) []topQueriesColumn {
230 + result := make([]topQueriesColumn, 0, len(topQueriesColumns))
231 + for _, col := range topQueriesColumns {
232 + if col.IsJoinColumn {
233 + result = append(result, col)
234 + continue
235 + }
236 + actual, ok := f.resolveColumnExpr(col.SelectExpr, availableCols)
237 + if !ok {
238 + continue
239 + }
240 + colCopy := col
241 + colCopy.SelectExpr = actual
242 + result = append(result, colCopy)
243 + }
244 + return result
245 +}
246 +
247 +func (f *funcTopQueries) resolveColumnExpr(expr string, availableCols map[string]bool) (string, bool) {
248 + colName := expr
249 + if idx := strings.LastIndex(colName, "."); idx != -1 {
250 + colName = colName[idx+1:]
251 + }
252 +
253 + castSuffix := ""
254 + if idx := strings.Index(colName, "::"); idx != -1 {
255 + castSuffix = colName[idx:]
256 + colName = colName[:idx]
257 + }
258 +
259 + actual := colName
260 + switch colName {
261 + case "total_time":
262 + if availableCols["total_exec_time"] {
263 + actual = "total_exec_time"
264 + }
265 + case "mean_time":
266 + if availableCols["mean_exec_time"] {
267 + actual = "mean_exec_time"
268 + }
269 + case "min_time":
270 + if availableCols["min_exec_time"] {
271 + actual = "min_exec_time"
272 + }
273 + case "max_time":
274 + if availableCols["max_exec_time"] {
275 + actual = "max_exec_time"
276 + }
277 + case "stddev_time":
278 + if availableCols["stddev_exec_time"] {
279 + actual = "stddev_exec_time"
280 + }
281 + }
282 +
283 + if !availableCols[actual] {
284 + return "", false
285 + }
286 +
287 + if actual == colName {
288 + return expr, true
289 + }
290 + return strings.Replace(expr, colName+castSuffix, actual+castSuffix, 1), true
291 +}
292 +
293 +func (f *funcTopQueries) resolveSortColumn(cols []topQueriesColumn, requested string) string {
294 + if requested != "" {
295 + for _, col := range cols {
296 + if col.Name == requested && col.IsSortOption() {
297 + return col.Name
298 + }
299 + }
300 + }
301 + for _, col := range cols {
302 + if col.IsDefaultSort() && col.IsSortOption() {
303 + return col.Name
304 + }
305 + }
306 + for _, col := range cols {
307 + if col.IsSortOption() {
308 + return col.Name
309 + }
310 + }
311 + if len(cols) > 0 {
312 + return cols[0].Name
313 + }
314 + return ""
315 +}
316 +
317 +func (f *funcTopQueries) buildSQL(cols []topQueriesColumn, sortColumn string) string {
318 + selectCols := make([]string, 0, len(cols))
319 + for _, col := range cols {
320 + selectCols = append(selectCols, fmt.Sprintf("%s AS %s", col.SelectExpr, col.Name))
321 + }
322 +
323 + return fmt.Sprintf(`
324 +SELECT %s
325 +FROM pg_stat_statements s
326 +JOIN pg_database d ON d.oid = s.dbid
327 +JOIN pg_user u ON u.usesysid = s.userid
328 +ORDER BY %s DESC NULLS LAST
329 +LIMIT $1`, strings.Join(selectCols, ", "), sortColumn)
330 +}
331 +
332 +func (f *funcTopQueries) scanRows(rows *sql.Rows, cols []topQueriesColumn) ([][]any, error) {
333 + data := make([][]any, 0, 500)
334 +
335 + for rows.Next() {
336 + values := make([]any, len(cols))
337 + valuePtrs := make([]any, len(cols))
338 +
339 + for i, col := range cols {
340 + switch col.Type {
341 + case funcapi.FieldTypeString:
342 + var v sql.NullString
343 + values[i] = &v
344 + case funcapi.FieldTypeInteger:
345 + var v sql.NullInt64
346 + values[i] = &v
347 + case funcapi.FieldTypeFloat, funcapi.FieldTypeDuration:
348 + var v sql.NullFloat64
349 + values[i] = &v
350 + default:
351 + var v any
352 + values[i] = &v
353 + }
354 + valuePtrs[i] = values[i]
355 + }
356 +
357 + if err := rows.Scan(valuePtrs...); err != nil {
358 + return nil, fmt.Errorf("row scan failed: %w", err)
359 + }
360 +
361 + row := make([]any, len(cols))
362 + for i, col := range cols {
363 + switch v := values[i].(type) {
364 + case *sql.NullString:
365 + if v.Valid {
366 + s := v.String
367 + if col.Name == "query" {
368 + s = strmutil.TruncateText(s, topQueriesMaxTextLength)
369 + }
370 + row[i] = s
371 + } else {
372 + row[i] = ""
373 + }
374 + case *sql.NullInt64:
375 + if v.Valid {
376 + row[i] = v.Int64
377 + } else {
378 + row[i] = int64(0)
379 + }
380 + case *sql.NullFloat64:
381 + if v.Valid {
382 + row[i] = v.Float64
383 + } else {
384 + row[i] = float64(0)
385 + }
386 + default:
387 + row[i] = nil
388 + }
389 + }
390 +
391 + data = append(data, row)
392 + }
393 +
394 + if err := rows.Err(); err != nil {
395 + return nil, fmt.Errorf("rows iteration error: %w", err)
396 + }
397 +
398 + return data, nil
399 +}
src/go/plugin/go.d/collector/yugabytedb/functions.go deleted
-717
@@ -1,717 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -package yugabytedb
4 -
5 -import (
6 - "context"
7 - "database/sql"
8 - "errors"
9 - "fmt"
10 - "strings"
11 - "time"
12 -
13 - "github.com/netdata/netdata/go/plugins/pkg/funcapi"
14 - "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"
15 - "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil"
16 -)
17 -
18 -const ybMaxQueryTextLength = 4096
19 -
20 -const (
21 - paramSort = "__sort"
22 -
23 - ftString = funcapi.FieldTypeString
24 - ftInteger = funcapi.FieldTypeInteger
25 - ftFloat = funcapi.FieldTypeFloat
26 - ftDuration = funcapi.FieldTypeDuration
27 -
28 - trNone = funcapi.FieldTransformNone
29 - trNumber = funcapi.FieldTransformNumber
30 - trDuration = funcapi.FieldTransformDuration
31 - trText = funcapi.FieldTransformText
32 -
33 - visValue = funcapi.FieldVisualValue
34 - visBar = funcapi.FieldVisualBar
35 -
36 - sortAsc = funcapi.FieldSortAscending
37 - sortDesc = funcapi.FieldSortDescending
38 -
39 - summaryCount = funcapi.FieldSummaryCount
40 - summarySum = funcapi.FieldSummarySum
41 - summaryMax = funcapi.FieldSummaryMax
42 - summaryMean = funcapi.FieldSummaryMean
43 -
44 - filterMulti = funcapi.FieldFilterMultiselect
45 - filterRange = funcapi.FieldFilterRange
46 -)
47 -
48 -var errYBSQLDSNNotSet = errors.New("SQL DSN is not set")
49 -
50 -type ybColumnMeta struct {
51 - id string
52 - name string
53 - selectExpr string
54 - dataType funcapi.FieldType
55 - visible bool
56 - sortable bool
57 - fullWidth bool
58 - wrap bool
59 - sticky bool
60 - filter funcapi.FieldFilter
61 - visualization funcapi.FieldVisual
62 - transform funcapi.FieldTransform
63 - units string
64 - decimalPoints int
65 - uniqueKey bool
66 - sortDir funcapi.FieldSort
67 - summary funcapi.FieldSummary
68 - sortLabel string
69 - isSortOption bool
70 - isDefaultSort bool
71 - isJoinColumn bool
72 - isLabel bool
73 - isPrimary bool
74 - isMetric bool
75 - chartGroup string
76 - chartTitle string
77 - isDefaultChart bool
78 -}
79 -
80 -var ybTopColumns = []ybColumnMeta{
81 - {id: "queryId", name: "Query ID", selectExpr: "s.queryid::text", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, uniqueKey: true, sortDir: sortAsc, summary: summaryCount},
82 - {id: "query", name: "Query", selectExpr: "s.query", dataType: ftString, visible: true, sortable: false, filter: filterMulti, transform: trText, sticky: true, fullWidth: true, wrap: true},
83 - {id: "database", name: "Database", selectExpr: "d.datname", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount, isJoinColumn: true, isLabel: true, isPrimary: true},
84 - {id: "user", name: "User", selectExpr: "u.usename", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount, isJoinColumn: true, isLabel: true},
85 -
86 - {id: "calls", name: "Calls", selectExpr: "s.calls", dataType: ftInteger, visible: true, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by Calls", isMetric: true, chartGroup: "Calls", chartTitle: "Number of Calls", isDefaultChart: true},
87 - {id: "totalTime", name: "Total Time", selectExpr: "s.total_time", dataType: ftDuration, visible: true, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summarySum, isSortOption: true, isDefaultSort: true, sortLabel: "Top queries by Total Time", isMetric: true, chartGroup: "Time", chartTitle: "Execution Time", isDefaultChart: true},
88 - {id: "meanTime", name: "Mean Time", selectExpr: "s.mean_time", dataType: ftDuration, visible: true, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMean, isSortOption: true, sortLabel: "Top queries by Mean Time", isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
89 - {id: "minTime", name: "Min Time", selectExpr: "s.min_time", dataType: ftDuration, visible: false, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
90 - {id: "maxTime", name: "Max Time", selectExpr: "s.max_time", dataType: ftDuration, visible: false, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, isSortOption: true, sortLabel: "Top queries by Max Time", isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
91 - {id: "rows", name: "Rows", selectExpr: "s.rows", dataType: ftInteger, visible: true, sortable: true, filter: filterRange, transform: trNumber, sortDir: sortDesc, summary: summarySum, isSortOption: true, sortLabel: "Top queries by Rows Returned", isMetric: true, chartGroup: "Rows", chartTitle: "Rows"},
92 - {id: "stddevTime", name: "Stddev Time", selectExpr: "s.stddev_time", dataType: ftDuration, visible: false, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, isMetric: true, chartGroup: "Time", chartTitle: "Execution Time"},
93 -}
94 -
95 -var ybRunningColumns = []ybColumnMeta{
96 - {id: "pid", name: "PID", selectExpr: "s.pid::text", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, uniqueKey: true, sortDir: sortAsc, summary: summaryCount},
97 - {id: "query", name: "Query", selectExpr: "s.query", dataType: ftString, visible: true, sortable: false, filter: filterMulti, transform: trText, sticky: true, fullWidth: true, wrap: true},
98 - {id: "database", name: "Database", selectExpr: "s.datname", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
99 - {id: "user", name: "User", selectExpr: "s.usename", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
100 - {id: "state", name: "State", selectExpr: "s.state", dataType: ftString, visible: true, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
101 - {id: "waitEventType", name: "Wait Event Type", selectExpr: "s.wait_event_type", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
102 - {id: "waitEvent", name: "Wait Event", selectExpr: "s.wait_event", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
103 - {id: "application", name: "Application", selectExpr: "s.application_name", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
104 - {id: "clientAddress", name: "Client Address", selectExpr: "s.client_addr::text", dataType: ftString, visible: false, sortable: true, filter: filterMulti, transform: trText, sortDir: sortAsc, summary: summaryCount},
105 - {id: "queryStart", name: "Query Start", selectExpr: "TO_CHAR(s.query_start, 'YYYY-MM-DD\"T\"HH24:MI:SS.FF3')", dataType: ftString, visible: false, sortable: true, filter: filterRange, transform: trText, sortDir: sortDesc, summary: summaryMax},
106 - {id: "elapsedMs", name: "Elapsed", selectExpr: "CASE WHEN s.query_start IS NULL THEN 0 ELSE EXTRACT(EPOCH FROM (clock_timestamp() - s.query_start)) * 1000 END", dataType: ftDuration, visible: true, sortable: true, filter: filterRange, transform: trDuration, units: "milliseconds", decimalPoints: 2, sortDir: sortDesc, summary: summaryMax, isSortOption: true, isDefaultSort: true, sortLabel: "Running queries by Elapsed Time"},
107 -}
108 -
109 -func yugabyteMethods() []module.MethodConfig {
110 - return []module.MethodConfig{
111 - {
112 - UpdateEvery: 10,
113 - ID: "top-queries",
114 - Name: "Top Queries",
115 - Help: "Top SQL queries from pg_stat_statements. WARNING: Query text may contain unmasked literals (potential PII).",
116 - RequireCloud: true,
117 - RequiredParams: []funcapi.ParamConfig{
118 - buildYBSortParam(ybTopColumns),
119 - },
120 - },
121 - {
122 - UpdateEvery: 10,
123 - ID: "running-queries",
124 - Name: "Running Queries",
125 - Help: "Currently running SQL statements from pg_stat_activity. WARNING: Query text may contain unmasked literals (potential PII).",
126 - RequireCloud: true,
127 - RequiredParams: []funcapi.ParamConfig{
128 - buildYBSortParam(ybRunningColumns),
129 - },
130 - },
131 - }
132 -}
133 -
134 -func yugabyteMethodParams(ctx context.Context, job *module.Job, method string) ([]funcapi.ParamConfig, error) {
135 - collector, ok := job.Module().(*Collector)
136 - if !ok {
137 - return nil, fmt.Errorf("invalid module type")
138 - }
139 -
140 - switch method {
141 - case "top-queries":
142 - cols := ybTopColumns
143 - if collector.db != nil {
144 - if available, err := collector.availableTopColumns(ctx); err == nil {
145 - cols = available
146 - }
147 - }
148 - return []funcapi.ParamConfig{buildYBSortParam(cols)}, nil
149 - case "running-queries":
150 - return []funcapi.ParamConfig{buildYBSortParam(ybRunningColumns)}, nil
151 - default:
152 - return nil, fmt.Errorf("unknown method: %s", method)
153 - }
154 -}
155 -
156 -func yugabyteHandleMethod(ctx context.Context, job *module.Job, method string, params funcapi.ResolvedParams) *module.FunctionResponse {
157 - collector, ok := job.Module().(*Collector)
158 - if !ok {
159 - return &module.FunctionResponse{Status: 500, Message: "internal error: invalid module type"}
160 - }
161 -
162 - if err := collector.ensureSQL(ctx); err != nil {
163 - status := 503
164 - if errors.Is(err, errYBSQLDSNNotSet) {
165 - status = 400
166 - }
167 - return &module.FunctionResponse{Status: status, Message: err.Error()}
168 - }
169 -
170 - switch method {
171 - case "top-queries":
172 - return collector.collectTopQueries(ctx, params.Column(paramSort))
173 - case "running-queries":
174 - return collector.collectRunningQueries(ctx, params.Column(paramSort))
175 - default:
176 - return &module.FunctionResponse{Status: 404, Message: fmt.Sprintf("unknown method: %s", method)}
177 - }
178 -}
179 -
180 -func (c *Collector) ensureSQL(ctx context.Context) error {
181 - if c.db != nil {
182 - return nil
183 - }
184 - if c.DSN == "" {
185 - return errYBSQLDSNNotSet
186 - }
187 -
188 - db, err := sql.Open("pgx", c.DSN)
189 - if err != nil {
190 - return fmt.Errorf("error opening SQL connection: %w", err)
191 - }
192 - db.SetMaxOpenConns(1)
193 - db.SetMaxIdleConns(1)
194 - db.SetConnMaxLifetime(10 * time.Minute)
195 -
196 - timeout := c.sqlTimeout()
197 - pingCtx, cancel := context.WithTimeout(ctx, timeout)
198 - defer cancel()
199 - if err := db.PingContext(pingCtx); err != nil {
200 - _ = db.Close()
201 - return fmt.Errorf("error pinging SQL connection: %w", err)
202 - }
203 -
204 - c.db = db
205 - return nil
206 -}
207 -
208 -func (c *Collector) sqlTimeout() time.Duration {
209 - if c.SQLTimeout.Duration() > 0 {
210 - return c.SQLTimeout.Duration()
211 - }
212 - return time.Second
213 -}
214 -
215 -func (c *Collector) availableTopColumns(ctx context.Context) ([]ybColumnMeta, error) {
216 - available, err := c.detectPgStatStatementsColumns(ctx)
217 - if err != nil {
218 - return nil, fmt.Errorf("failed to detect available columns: %v", err)
219 - }
220 - cols := c.buildAvailableColumns(available)
221 - if len(cols) == 0 {
222 - return nil, fmt.Errorf("no queryable columns found in pg_stat_statements")
223 - }
224 - return cols, nil
225 -}
226 -
227 -func (c *Collector) collectTopQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
228 - ok, err := c.pgStatStatementsEnabled(ctx)
229 - if err != nil {
230 - return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("failed to check pg_stat_statements: %v", err)}
231 - }
232 - if !ok {
233 - return &module.FunctionResponse{
234 - Status: 503,
235 - Message: "pg_stat_statements extension is not installed in this database. " +
236 - "Run 'CREATE EXTENSION pg_stat_statements;' in the database the collector connects to.",
237 - }
238 - }
239 -
240 - cols, err := c.availableTopColumns(ctx)
241 - if err != nil {
242 - return &module.FunctionResponse{Status: 500, Message: err.Error()}
243 - }
244 -
245 - sortColumn = resolveYBSortColumn(cols, sortColumn)
246 - limit := c.TopQueriesLimit
247 - if limit <= 0 {
248 - limit = 500
249 - }
250 -
251 - query := buildYBTopQueriesSQL(cols, sortColumn)
252 - queryCtx, cancel := context.WithTimeout(ctx, c.sqlTimeout())
253 - defer cancel()
254 - rows, err := c.db.QueryContext(queryCtx, query, limit)
255 - if err != nil {
256 - if errors.Is(queryCtx.Err(), context.DeadlineExceeded) {
257 - return &module.FunctionResponse{Status: 504, Message: "query timed out"}
258 - }
259 - return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("query failed: %v", err)}
260 - }
261 - defer rows.Close()
262 -
263 - data, err := scanYBRows(rows, cols)
264 - if err != nil {
265 - return &module.FunctionResponse{Status: 500, Message: err.Error()}
266 - }
267 -
268 - return &module.FunctionResponse{
269 - Status: 200,
270 - Help: "Top SQL queries from pg_stat_statements. WARNING: Query text may contain unmasked literals (potential PII).",
271 - Columns: buildYBColumns(cols),
272 - Data: data,
273 - DefaultSortColumn: sortColumn,
274 - RequiredParams: []funcapi.ParamConfig{buildYBSortParam(cols)},
275 - Charts: ybTopQueriesCharts(cols),
276 - DefaultCharts: ybTopQueriesDefaultCharts(cols),
277 - GroupBy: ybTopQueriesGroupBy(cols),
278 - }
279 -}
280 -
281 -func (c *Collector) collectRunningQueries(ctx context.Context, sortColumn string) *module.FunctionResponse {
282 - sortColumn = resolveYBSortColumn(ybRunningColumns, sortColumn)
283 - limit := c.TopQueriesLimit
284 - if limit <= 0 {
285 - limit = 500
286 - }
287 -
288 - query := buildYBRunningQueriesSQL(sortColumn)
289 - queryCtx, cancel := context.WithTimeout(ctx, c.sqlTimeout())
290 - defer cancel()
291 - rows, err := c.db.QueryContext(queryCtx, query, limit)
292 - if err != nil {
293 - if errors.Is(queryCtx.Err(), context.DeadlineExceeded) {
294 - return &module.FunctionResponse{Status: 504, Message: "query timed out"}
295 - }
296 - return &module.FunctionResponse{Status: 500, Message: fmt.Sprintf("query failed: %v", err)}
297 - }
298 - defer rows.Close()
299 -
300 - data, err := scanYBRows(rows, ybRunningColumns)
301 - if err != nil {
302 - return &module.FunctionResponse{Status: 500, Message: err.Error()}
303 - }
304 -
305 - return &module.FunctionResponse{
306 - Status: 200,
307 - Help: "Currently running SQL statements from pg_stat_activity. WARNING: Query text may contain unmasked literals (potential PII).",
308 - Columns: buildYBColumns(ybRunningColumns),
309 - Data: data,
310 - DefaultSortColumn: sortColumn,
311 - RequiredParams: []funcapi.ParamConfig{buildYBSortParam(ybRunningColumns)},
312 - }
313 -}
314 -
315 -func buildYBSortParam(cols []ybColumnMeta) funcapi.ParamConfig {
316 - return funcapi.ParamConfig{
317 - ID: paramSort,
318 - Name: "Filter By",
319 - Help: "Select the primary sort column",
320 - Selection: funcapi.ParamSelect,
321 - Options: buildYBSortOptions(cols),
322 - UniqueView: true,
323 - }
324 -}
325 -
326 -func buildYBSortOptions(cols []ybColumnMeta) []funcapi.ParamOption {
327 - var sortOptions []funcapi.ParamOption
328 - sortDir := funcapi.FieldSortDescending
329 - for _, col := range cols {
330 - if !col.isSortOption {
331 - continue
332 - }
333 - opt := funcapi.ParamOption{
334 - ID: col.id,
335 - Column: col.id,
336 - Name: col.sortLabel,
337 - Sort: &sortDir,
338 - }
339 - if col.isDefaultSort {
340 - opt.Default = true
341 - }
342 - sortOptions = append(sortOptions, opt)
343 - }
344 - return sortOptions
345 -}
346 -
347 -func buildYBColumns(cols []ybColumnMeta) map[string]any {
348 - result := make(map[string]any, len(cols))
349 - for i, col := range cols {
350 - visual := visValue
351 - if col.dataType == ftDuration {
352 - visual = visBar
353 - }
354 - colDef := funcapi.Column{
355 - Index: i,
356 - Name: col.name,
357 - Type: col.dataType,
358 - Units: col.units,
359 - Visualization: visual,
360 - Sort: col.sortDir,
361 - Sortable: col.sortable,
362 - Sticky: col.sticky,
363 - Summary: col.summary,
364 - Filter: col.filter,
365 - FullWidth: col.fullWidth,
366 - Wrap: col.wrap,
367 - DefaultExpandedFilter: false,
368 - UniqueKey: col.uniqueKey,
369 - Visible: col.visible,
370 - ValueOptions: funcapi.ValueOptions{
371 - Transform: col.transform,
372 - DecimalPoints: col.decimalPoints,
373 - DefaultValue: nil,
374 - },
375 - }
376 - result[col.id] = colDef.BuildColumn()
377 - }
378 - return result
379 -}
380 -
381 -func resolveYBSortColumn(cols []ybColumnMeta, requested string) string {
382 - if requested != "" {
383 - for _, col := range cols {
384 - if col.id == requested && col.isSortOption {
385 - return col.id
386 - }
387 - }
388 - }
389 - for _, col := range cols {
390 - if col.isDefaultSort && col.isSortOption {
391 - return col.id
392 - }
393 - }
394 - for _, col := range cols {
395 - if col.isSortOption {
396 - return col.id
397 - }
398 - }
399 - if len(cols) > 0 {
400 - return cols[0].id
401 - }
402 - return ""
403 -}
404 -
405 -func buildYBTopQueriesSQL(cols []ybColumnMeta, sortColumn string) string {
406 - selectCols := make([]string, 0, len(cols))
407 - for _, col := range cols {
408 - selectCols = append(selectCols, fmt.Sprintf("%s AS %s", col.selectExpr, col.id))
409 - }
410 -
411 - return fmt.Sprintf(`
412 -SELECT %s
413 -FROM pg_stat_statements s
414 -JOIN pg_database d ON d.oid = s.dbid
415 -JOIN pg_user u ON u.usesysid = s.userid
416 -ORDER BY %s DESC NULLS LAST
417 -LIMIT $1`, strings.Join(selectCols, ", "), sortColumn)
418 -}
419 -
420 -func buildYBRunningQueriesSQL(sortColumn string) string {
421 - selectCols := make([]string, 0, len(ybRunningColumns))
422 - for _, col := range ybRunningColumns {
423 - selectCols = append(selectCols, fmt.Sprintf("%s AS %s", col.selectExpr, col.id))
424 - }
425 - return fmt.Sprintf(`
426 -SELECT %s
427 -FROM pg_stat_activity s
428 -WHERE s.state IS DISTINCT FROM 'idle'
429 -ORDER BY %s DESC NULLS LAST
430 -LIMIT $1`, strings.Join(selectCols, ", "), sortColumn)
431 -}
432 -
433 -func scanYBRows(rows *sql.Rows, cols []ybColumnMeta) ([][]any, error) {
434 - data := make([][]any, 0, 500)
435 -
436 - for rows.Next() {
437 - values := make([]any, len(cols))
438 - valuePtrs := make([]any, len(cols))
439 -
440 - for i, col := range cols {
441 - switch col.dataType {
442 - case ftString:
443 - var v sql.NullString
444 - values[i] = &v
445 - case ftInteger:
446 - var v sql.NullInt64
447 - values[i] = &v
448 - case ftFloat, ftDuration:
449 - var v sql.NullFloat64
450 - values[i] = &v
451 - default:
452 - var v any
453 - values[i] = &v
454 - }
455 - valuePtrs[i] = values[i]
456 - }
457 -
458 - if err := rows.Scan(valuePtrs...); err != nil {
459 - return nil, fmt.Errorf("row scan failed: %w", err)
460 - }
461 -
462 - row := make([]any, len(cols))
463 - for i, col := range cols {
464 - switch v := values[i].(type) {
465 - case *sql.NullString:
466 - if v.Valid {
467 - s := v.String
468 - if col.id == "query" {
469 - s = strmutil.TruncateText(s, ybMaxQueryTextLength)
470 - }
471 - row[i] = s
472 - } else {
473 - row[i] = ""
474 - }
475 - case *sql.NullInt64:
476 - if v.Valid {
477 - row[i] = v.Int64
478 - } else {
479 - row[i] = int64(0)
480 - }
481 - case *sql.NullFloat64:
482 - if v.Valid {
483 - row[i] = v.Float64
484 - } else {
485 - row[i] = float64(0)
486 - }
487 - default:
488 - row[i] = nil
489 - }
490 - }
491 -
492 - data = append(data, row)
493 - }
494 -
495 - if err := rows.Err(); err != nil {
496 - return nil, fmt.Errorf("rows iteration error: %w", err)
497 - }
498 -
499 - return data, nil
500 -}
501 -
502 -func ybTopQueriesCharts(cols []ybColumnMeta) map[string]module.ChartConfig {
503 - charts := make(map[string]module.ChartConfig)
504 - for _, col := range cols {
505 - if !col.isMetric || col.chartGroup == "" {
506 - continue
507 - }
508 - cfg, ok := charts[col.chartGroup]
509 - if !ok {
510 - title := col.chartTitle
511 - if title == "" {
512 - title = col.chartGroup
513 - }
514 - cfg = module.ChartConfig{Name: title, Type: "stacked-bar"}
515 - }
516 - cfg.Columns = append(cfg.Columns, col.id)
517 - charts[col.chartGroup] = cfg
518 - }
519 - return charts
520 -}
521 -
522 -func ybTopQueriesDefaultCharts(cols []ybColumnMeta) [][]string {
523 - label := primaryYBLabel(cols)
524 - if label == "" {
525 - return nil
526 - }
527 - chartGroups := defaultYBChartGroups(cols)
528 - out := make([][]string, 0, len(chartGroups))
529 - for _, group := range chartGroups {
530 - out = append(out, []string{group, label})
531 - }
532 - return out
533 -}
534 -
535 -func ybTopQueriesGroupBy(cols []ybColumnMeta) map[string]module.GroupByConfig {
536 - groupBy := make(map[string]module.GroupByConfig)
537 - for _, col := range cols {
538 - if !col.isLabel {
539 - continue
540 - }
541 - groupBy[col.id] = module.GroupByConfig{
542 - Name: "Group by " + col.name,
543 - Columns: []string{col.id},
544 - }
545 - }
546 - return groupBy
547 -}
548 -
549 -func primaryYBLabel(cols []ybColumnMeta) string {
550 - for _, col := range cols {
551 - if col.isPrimary {
552 - return col.id
553 - }
554 - }
555 - for _, col := range cols {
556 - if col.isLabel {
557 - return col.id
558 - }
559 - }
560 - return ""
561 -}
562 -
563 -func defaultYBChartGroups(cols []ybColumnMeta) []string {
564 - groups := make([]string, 0)
565 - seen := make(map[string]bool)
566 - for _, col := range cols {
567 - if !col.isMetric || col.chartGroup == "" || !col.isDefaultChart {
568 - continue
569 - }
570 - if !seen[col.chartGroup] {
571 - seen[col.chartGroup] = true
572 - groups = append(groups, col.chartGroup)
573 - }
574 - }
575 - if len(groups) > 0 {
576 - return groups
577 - }
578 - for _, col := range cols {
579 - if !col.isMetric || col.chartGroup == "" {
580 - continue
581 - }
582 - if !seen[col.chartGroup] {
583 - seen[col.chartGroup] = true
584 - groups = append(groups, col.chartGroup)
585 - }
586 - }
587 - return groups
588 -}
589 -
590 -func (c *Collector) pgStatStatementsEnabled(ctx context.Context) (bool, error) {
591 - query := `SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements'`
592 - queryCtx, cancel := context.WithTimeout(ctx, c.sqlTimeout())
593 - defer cancel()
594 - var exists int
595 - if err := c.db.QueryRowContext(queryCtx, query).Scan(&exists); err != nil {
596 - if errors.Is(queryCtx.Err(), context.DeadlineExceeded) {
597 - return false, queryCtx.Err()
598 - }
599 - if errors.Is(err, sql.ErrNoRows) {
600 - return false, nil
601 - }
602 - return false, err
603 - }
604 - return true, nil
605 -}
606 -
607 -func (c *Collector) detectPgStatStatementsColumns(ctx context.Context) (map[string]bool, error) {
608 - c.pgStatStatementsMu.RLock()
609 - if c.pgStatStatementsColumns != nil {
610 - cols := c.pgStatStatementsColumns
611 - c.pgStatStatementsMu.RUnlock()
612 - return cols, nil
613 - }
614 - c.pgStatStatementsMu.RUnlock()
615 -
616 - c.pgStatStatementsMu.Lock()
617 - defer c.pgStatStatementsMu.Unlock()
618 -
619 - if c.pgStatStatementsColumns != nil {
620 - return c.pgStatStatementsColumns, nil
621 - }
622 -
623 - query := `
624 - SELECT column_name
625 - FROM information_schema.columns
626 - WHERE table_name = 'pg_stat_statements'
627 - AND table_schema = 'public'
628 - `
629 - queryCtx, cancel := context.WithTimeout(ctx, c.sqlTimeout())
630 - defer cancel()
631 -
632 - rows, err := c.db.QueryContext(queryCtx, query)
633 - if err != nil {
634 - return nil, fmt.Errorf("failed to query columns: %v", err)
635 - }
636 - defer rows.Close()
637 -
638 - cols := make(map[string]bool)
639 - for rows.Next() {
640 - var colName string
641 - if err := rows.Scan(&colName); err != nil {
642 - return nil, fmt.Errorf("failed to scan column name: %v", err)
643 - }
644 - cols[colName] = true
645 - }
646 -
647 - if err := rows.Err(); err != nil {
648 - return nil, fmt.Errorf("rows iteration error: %v", err)
649 - }
650 -
651 - c.pgStatStatementsColumns = cols
652 - return cols, nil
653 -}
654 -
655 -func (c *Collector) buildAvailableColumns(availableCols map[string]bool) []ybColumnMeta {
656 - result := make([]ybColumnMeta, 0, len(ybTopColumns))
657 - for _, col := range ybTopColumns {
658 - if col.isJoinColumn {
659 - result = append(result, col)
660 - continue
661 - }
662 - actual, ok := resolveYBColumn(col.selectExpr, availableCols)
663 - if !ok {
664 - continue
665 - }
666 - colCopy := col
667 - colCopy.selectExpr = actual
668 - result = append(result, colCopy)
669 - }
670 - return result
671 -}
672 -
673 -func resolveYBColumn(expr string, availableCols map[string]bool) (string, bool) {
674 - colName := expr
675 - if idx := strings.LastIndex(colName, "."); idx != -1 {
676 - colName = colName[idx+1:]
677 - }
678 -
679 - castSuffix := ""
680 - if idx := strings.Index(colName, "::"); idx != -1 {
681 - castSuffix = colName[idx:]
682 - colName = colName[:idx]
683 - }
684 -
685 - actual := colName
686 - switch colName {
687 - case "total_time":
688 - if availableCols["total_exec_time"] {
689 - actual = "total_exec_time"
690 - }
691 - case "mean_time":
692 - if availableCols["mean_exec_time"] {
693 - actual = "mean_exec_time"
694 - }
695 - case "min_time":
696 - if availableCols["min_exec_time"] {
697 - actual = "min_exec_time"
698 - }
699 - case "max_time":
700 - if availableCols["max_exec_time"] {
701 - actual = "max_exec_time"
702 - }
703 - case "stddev_time":
704 - if availableCols["stddev_exec_time"] {
705 - actual = "stddev_exec_time"
706 - }
707 - }
708 -
709 - if !availableCols[actual] {
710 - return "", false
711 - }
712 -
713 - if actual == colName {
714 - return expr, true
715 - }
716 - return strings.Replace(expr, colName+castSuffix, actual+castSuffix, 1), true
717 -}
src/go/plugin/go.d/collector/yugabytedb/functions_test.go
+6 -14
@@ -35,25 +35,17 @@ func TestYugabyteDBMethods(t *testing.T) {
35 func TestYugabyteDBTopColumns_HasRequiredColumns(t *testing.T) {
36 required := []string{"queryId", "query", "calls", "totalTime"}
37
38 - uiKeys := make(map[string]bool)
39 - for _, col := range ybTopColumns {
40 - uiKeys[col.id] = true
41 - }
42 -
43 - for _, key := range required {
44 - assert.True(t, uiKeys[key], "column %s should be defined", key)
38 + cs := funcapi.Columns(topQueriesColumns, func(c topQueriesColumn) funcapi.ColumnMeta { return c.ColumnMeta })
39 + for _, id := range required {
40 + assert.True(t, cs.ContainsColumn(id), "column %s should be defined", id)
41 }
42 }
43
44 func TestYugabyteDBRunningColumns_HasRequiredColumns(t *testing.T) {
45 required := []string{"pid", "query", "elapsedMs"}
46
51 - uiKeys := make(map[string]bool)
52 - for _, col := range ybRunningColumns {
53 - uiKeys[col.id] = true
54 - }
55 -
56 - for _, key := range required {
57 - assert.True(t, uiKeys[key], "column %s should be defined", key)
47 + cs := funcapi.Columns(runningQueriesColumns, func(c runningQueriesColumn) funcapi.ColumnMeta { return c.ColumnMeta })
48 + for _, id := range required {
49 + assert.True(t, cs.ContainsColumn(id), "column %s should be defined", id)
50 }
51 }