| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | package couchbase |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "encoding/json" |
| 8 | "fmt" |
| 9 | "net" |
| 10 | "net/http" |
| 11 | "net/url" |
| 12 | "path" |
| 13 | "sort" |
| 14 | "strings" |
| 15 | "time" |
| 16 | |
| 17 | "github.com/netdata/netdata/go/plugins/pkg/funcapi" |
| 18 | "github.com/netdata/netdata/go/plugins/pkg/web" |
| 19 | "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/strmutil" |
| 20 | ) |
| 21 | |
| 22 | const ( |
| 23 | topQueriesMethodID = "top-queries" |
| 24 | topQueriesMaxTextLength = 4096 |
| 25 | ) |
| 26 | |
| 27 | func topQueriesMethodConfig() funcapi.MethodConfig { |
| 28 | return funcapi.MethodConfig{ |
| 29 | ID: topQueriesMethodID, |
| 30 | Name: "Top Queries", |
| 31 | UpdateEvery: 10, |
| 32 | Help: "Top N1QL requests from system:completed_requests", |
| 33 | RequireCloud: true, |
| 34 | RequiredParams: []funcapi.ParamConfig{funcapi.BuildSortParam(topQueriesColumns)}, |
| 35 | } |
| 36 | } |
| 37 | |
| 38 | type topQueriesColumn struct { |
| 39 | funcapi.ColumnMeta |
| 40 | sortOpt bool // whether this column appears as a sort option |
| 41 | sortLbl string // label for sort option dropdown |
| 42 | defaultSort bool // default sort column |
| 43 | } |
| 44 | |
| 45 | // funcapi.SortableColumn interface implementation for topQueriesColumn. |
| 46 | func (c topQueriesColumn) IsSortOption() bool { return c.sortOpt } |
| 47 | func (c topQueriesColumn) SortLabel() string { return c.sortLbl } |
| 48 | func (c topQueriesColumn) IsDefaultSort() bool { return c.defaultSort } |
| 49 | func (c topQueriesColumn) ColumnName() string { return c.Name } |
| 50 | func (c topQueriesColumn) SortColumn() string { return "" } |
| 51 | |
| 52 | var topQueriesColumns = []topQueriesColumn{ |
| 53 | {ColumnMeta: funcapi.ColumnMeta{Name: "requestId", Tooltip: "Request ID", Type: funcapi.FieldTypeString, Visible: false, Sortable: true, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, UniqueKey: true, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryCount}, sortOpt: true, sortLbl: "Top queries by Request ID"}, |
| 54 | {ColumnMeta: funcapi.ColumnMeta{Name: "requestTime", Tooltip: "Request Time", Type: funcapi.FieldTypeTimestamp, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformDatetime, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummaryMax}, sortOpt: true, sortLbl: "Top queries by Request Time"}, |
| 55 | {ColumnMeta: funcapi.ColumnMeta{Name: "statement", Tooltip: "Statement", Type: funcapi.FieldTypeString, Visible: true, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, Sticky: true, FullWidth: true, Wrap: true}}, |
| 56 | {ColumnMeta: funcapi.ColumnMeta{Name: "elapsedTime", Tooltip: "Elapsed Time", Type: funcapi.FieldTypeDuration, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualBar, Transform: funcapi.FieldTransformDuration, Units: "milliseconds", DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Elapsed & Service Time", IsDefault: true}}, sortOpt: true, defaultSort: true, sortLbl: "Top queries by Elapsed Time"}, |
| 57 | {ColumnMeta: funcapi.ColumnMeta{Name: "serviceTime", Tooltip: "Service Time", Type: funcapi.FieldTypeDuration, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualBar, Transform: funcapi.FieldTransformDuration, Units: "milliseconds", DecimalPoints: 2, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "Time", Title: "Elapsed & Service Time"}}, sortOpt: true, sortLbl: "Top queries by Service Time"}, |
| 58 | {ColumnMeta: funcapi.ColumnMeta{Name: "resultCount", Tooltip: "Result Count", Type: funcapi.FieldTypeInteger, Visible: true, Sortable: true, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "Results", Title: "Results"}}, sortOpt: true, sortLbl: "Top queries by Result Count"}, |
| 59 | {ColumnMeta: funcapi.ColumnMeta{Name: "resultSize", Tooltip: "Result Size", Type: funcapi.FieldTypeInteger, Visible: false, Sortable: false, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "ResultSize", Title: "Result Size"}}}, |
| 60 | {ColumnMeta: funcapi.ColumnMeta{Name: "errorCount", Tooltip: "Error Count", Type: funcapi.FieldTypeInteger, Visible: false, Sortable: false, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "Errors", Title: "Errors & Warnings"}}}, |
| 61 | {ColumnMeta: funcapi.ColumnMeta{Name: "warningCount", Tooltip: "Warning Count", Type: funcapi.FieldTypeInteger, Visible: false, Sortable: false, Filter: funcapi.FieldFilterRange, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformNumber, Sort: funcapi.FieldSortDescending, Summary: funcapi.FieldSummarySum, Chart: &funcapi.ChartOptions{Group: "Errors", Title: "Errors & Warnings"}}}, |
| 62 | {ColumnMeta: funcapi.ColumnMeta{Name: "user", Tooltip: "User", Type: funcapi.FieldTypeString, Visible: true, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText, GroupBy: &funcapi.GroupByOptions{IsDefault: true}}}, |
| 63 | {ColumnMeta: funcapi.ColumnMeta{Name: "clientContextID", Tooltip: "Client Context ID", Type: funcapi.FieldTypeString, Visible: false, Sortable: false, Filter: funcapi.FieldFilterMultiselect, Visualization: funcapi.FieldVisualValue, Transform: funcapi.FieldTransformText}}, |
| 64 | } |
| 65 | |
| 66 | type topQueriesResponse struct { |
| 67 | Status string `json:"status"` |
| 68 | Results []topQueriesRequestData `json:"results"` |
| 69 | Errors []topQueriesError `json:"errors"` |
| 70 | } |
| 71 | |
| 72 | type topQueriesError struct { |
| 73 | Message string `json:"msg"` |
| 74 | } |
| 75 | |
| 76 | type topQueriesRequestData struct { |
| 77 | RequestID string `json:"requestId"` |
| 78 | RequestTime string `json:"requestTime"` |
| 79 | Statement string `json:"statement"` |
| 80 | ElapsedTime string `json:"elapsedTime"` |
| 81 | ServiceTime string `json:"serviceTime"` |
| 82 | ResultCount json.Number `json:"resultCount"` |
| 83 | ResultSize json.Number `json:"resultSize"` |
| 84 | ErrorCount json.Number `json:"errorCount"` |
| 85 | WarningCount json.Number `json:"warningCount"` |
| 86 | User string `json:"user"` |
| 87 | ClientContextID string `json:"clientContextID"` |
| 88 | } |
| 89 | |
| 90 | type topQueriesRow struct { |
| 91 | RequestID string |
| 92 | RequestTime time.Time |
| 93 | RequestTimeRaw string |
| 94 | Statement string |
| 95 | ElapsedMs float64 |
| 96 | ServiceMs float64 |
| 97 | ResultCount int64 |
| 98 | ResultSize int64 |
| 99 | ErrorCount int64 |
| 100 | WarningCount int64 |
| 101 | User string |
| 102 | ClientContextID string |
| 103 | } |
| 104 | |
| 105 | // funcTopQueries implements funcapi.MethodHandler for Couchbase top-queries. |
| 106 | type funcTopQueries struct { |
| 107 | router *funcRouter |
| 108 | } |
| 109 | |
| 110 | func newFuncTopQueries(r *funcRouter) *funcTopQueries { |
| 111 | return &funcTopQueries{router: r} |
| 112 | } |
| 113 | |
| 114 | // Compile-time interface check. |
| 115 | var _ funcapi.MethodHandler = (*funcTopQueries)(nil) |
| 116 | |
| 117 | // MethodParams implements funcapi.MethodHandler. |
| 118 | func (f *funcTopQueries) MethodParams(_ context.Context, method string) ([]funcapi.ParamConfig, error) { |
| 119 | switch method { |
| 120 | case topQueriesMethodID: |
| 121 | if f.router.collector.Functions.TopQueries.Disabled { |
| 122 | return nil, fmt.Errorf("top-queries function disabled in configuration") |
| 123 | } |
| 124 | return []funcapi.ParamConfig{funcapi.BuildSortParam(topQueriesColumns)}, nil |
| 125 | default: |
| 126 | return nil, fmt.Errorf("unknown method: %s", method) |
| 127 | } |
| 128 | } |
| 129 | |
| 130 | // Handle implements funcapi.MethodHandler. |
| 131 | func (f *funcTopQueries) Handle(ctx context.Context, method string, params funcapi.ResolvedParams) *funcapi.FunctionResponse { |
| 132 | if f.router.collector.httpClient == nil { |
| 133 | return funcapi.UnavailableResponse("collector is still initializing, please retry in a few seconds") |
| 134 | } |
| 135 | |
| 136 | switch method { |
| 137 | case topQueriesMethodID: |
| 138 | if f.router.collector.Functions.TopQueries.Disabled { |
| 139 | return funcapi.UnavailableResponse("top-queries function has been disabled in configuration") |
| 140 | } |
| 141 | queryCtx, cancel := context.WithTimeout(ctx, f.router.collector.topQueriesTimeout()) |
| 142 | defer cancel() |
| 143 | return f.collectData(queryCtx, params.Column("__sort")) |
| 144 | default: |
| 145 | return funcapi.NotFoundResponse(method) |
| 146 | } |
| 147 | } |
| 148 | |
| 149 | // Cleanup implements funcapi.MethodHandler. |
| 150 | func (f *funcTopQueries) Cleanup(ctx context.Context) {} |
| 151 | |
| 152 | func (f *funcTopQueries) collectData(ctx context.Context, sortColumn string) *funcapi.FunctionResponse { |
| 153 | limit := f.router.collector.topQueriesLimit() |
| 154 | |
| 155 | statement := "SELECT cr.requestId, cr.requestTime, cr.statement, cr.elapsedTime, cr.serviceTime, " + |
| 156 | "cr.resultCount, cr.resultSize, cr.errorCount, cr.warningCount, cr.users AS `user`, cr.clientContextID " + |
| 157 | "FROM system:completed_requests AS cr" |
| 158 | |
| 159 | req, err := f.buildQueryRequest(ctx, statement) |
| 160 | if err != nil { |
| 161 | return &funcapi.FunctionResponse{Status: 500, Message: err.Error()} |
| 162 | } |
| 163 | |
| 164 | var resp topQueriesResponse |
| 165 | if err := web.DoHTTP(f.router.collector.httpClient).RequestJSON(req, &resp); err != nil { |
| 166 | if ctx.Err() == context.DeadlineExceeded { |
| 167 | return &funcapi.FunctionResponse{Status: 504, Message: "query timed out"} |
| 168 | } |
| 169 | return &funcapi.FunctionResponse{Status: 500, Message: fmt.Sprintf("query failed: %v", err)} |
| 170 | } |
| 171 | |
| 172 | if strings.ToLower(resp.Status) != "success" { |
| 173 | msg := "query failed" |
| 174 | if len(resp.Errors) > 0 && resp.Errors[0].Message != "" { |
| 175 | msg = resp.Errors[0].Message |
| 176 | } |
| 177 | return &funcapi.FunctionResponse{Status: 500, Message: msg} |
| 178 | } |
| 179 | |
| 180 | rows := make([]topQueriesRow, 0, len(resp.Results)) |
| 181 | for _, r := range resp.Results { |
| 182 | rows = append(rows, f.buildRow(r)) |
| 183 | } |
| 184 | |
| 185 | cs := f.columnSet(topQueriesColumns) |
| 186 | sortParam := funcapi.BuildSortParam(topQueriesColumns) |
| 187 | |
| 188 | if len(rows) == 0 { |
| 189 | return &funcapi.FunctionResponse{ |
| 190 | Status: 200, |
| 191 | Message: "No completed requests found.", |
| 192 | Help: "Top N1QL requests from system:completed_requests", |
| 193 | Columns: cs.BuildColumns(), |
| 194 | Data: [][]any{}, |
| 195 | DefaultSortColumn: "elapsedTime", |
| 196 | RequiredParams: []funcapi.ParamConfig{sortParam}, |
| 197 | ChartingConfig: cs.BuildCharting(), |
| 198 | } |
| 199 | } |
| 200 | |
| 201 | sortColumn = f.mapSortColumn(sortColumn) |
| 202 | f.sortRows(rows, sortColumn) |
| 203 | |
| 204 | if len(rows) > limit { |
| 205 | rows = rows[:limit] |
| 206 | } |
| 207 | |
| 208 | data := make([][]any, 0, len(rows)) |
| 209 | for _, row := range rows { |
| 210 | out := make([]any, len(topQueriesColumns)) |
| 211 | for i, col := range topQueriesColumns { |
| 212 | switch col.Name { |
| 213 | case "requestId": |
| 214 | out[i] = row.RequestID |
| 215 | case "requestTime": |
| 216 | if row.RequestTime.IsZero() { |
| 217 | out[i] = row.RequestTimeRaw |
| 218 | } else { |
| 219 | out[i] = row.RequestTime.Format(time.RFC3339Nano) |
| 220 | } |
| 221 | case "statement": |
| 222 | out[i] = strmutil.TruncateText(row.Statement, topQueriesMaxTextLength) |
| 223 | case "elapsedTime": |
| 224 | out[i] = row.ElapsedMs |
| 225 | case "serviceTime": |
| 226 | out[i] = row.ServiceMs |
| 227 | case "resultCount": |
| 228 | out[i] = row.ResultCount |
| 229 | case "resultSize": |
| 230 | out[i] = row.ResultSize |
| 231 | case "errorCount": |
| 232 | out[i] = row.ErrorCount |
| 233 | case "warningCount": |
| 234 | out[i] = row.WarningCount |
| 235 | case "user": |
| 236 | out[i] = row.User |
| 237 | case "clientContextID": |
| 238 | out[i] = row.ClientContextID |
| 239 | default: |
| 240 | out[i] = nil |
| 241 | } |
| 242 | } |
| 243 | data = append(data, out) |
| 244 | } |
| 245 | |
| 246 | return &funcapi.FunctionResponse{ |
| 247 | Status: 200, |
| 248 | Help: "Top N1QL requests from system:completed_requests", |
| 249 | Columns: cs.BuildColumns(), |
| 250 | Data: data, |
| 251 | DefaultSortColumn: "elapsedTime", |
| 252 | RequiredParams: []funcapi.ParamConfig{sortParam}, |
| 253 | ChartingConfig: cs.BuildCharting(), |
| 254 | } |
| 255 | } |
| 256 | |
| 257 | func (f *funcTopQueries) columnSet(cols []topQueriesColumn) funcapi.ColumnSet[topQueriesColumn] { |
| 258 | return funcapi.Columns(cols, func(c topQueriesColumn) funcapi.ColumnMeta { return c.ColumnMeta }) |
| 259 | } |
| 260 | |
| 261 | func (f *funcTopQueries) queryServiceURL() (string, error) { |
| 262 | if f.router.collector.QueryURL != "" { |
| 263 | return f.router.collector.QueryURL, nil |
| 264 | } |
| 265 | parsed, err := url.Parse(f.router.collector.URL) |
| 266 | if err != nil { |
| 267 | return "", err |
| 268 | } |
| 269 | host := parsed.Hostname() |
| 270 | port := parsed.Port() |
| 271 | if port == "" || port == "8091" { |
| 272 | port = "8093" |
| 273 | } |
| 274 | if port != "" { |
| 275 | parsed.Host = net.JoinHostPort(host, port) |
| 276 | } else { |
| 277 | parsed.Host = host |
| 278 | } |
| 279 | parsed.Path = "" |
| 280 | return parsed.String(), nil |
| 281 | } |
| 282 | |
| 283 | func (f *funcTopQueries) buildQueryRequest(ctx context.Context, statement string) (*http.Request, error) { |
| 284 | queryURL, err := f.queryServiceURL() |
| 285 | if err != nil { |
| 286 | return nil, err |
| 287 | } |
| 288 | |
| 289 | u, err := url.Parse(queryURL) |
| 290 | if err != nil { |
| 291 | return nil, err |
| 292 | } |
| 293 | u.Path = path.Join(u.Path, "/query/service") |
| 294 | |
| 295 | reqCfg := f.router.collector.RequestConfig |
| 296 | reqCfg.URL = u.String() |
| 297 | reqCfg.Method = http.MethodPost |
| 298 | reqCfg.Body = url.Values{"statement": {statement}}.Encode() |
| 299 | if reqCfg.Headers == nil { |
| 300 | reqCfg.Headers = map[string]string{} |
| 301 | } |
| 302 | reqCfg.Headers["Content-Type"] = "application/x-www-form-urlencoded" |
| 303 | |
| 304 | req, err := web.NewHTTPRequest(reqCfg) |
| 305 | if err != nil { |
| 306 | return nil, err |
| 307 | } |
| 308 | return req.WithContext(ctx), nil |
| 309 | } |
| 310 | |
| 311 | func (f *funcTopQueries) buildRow(r topQueriesRequestData) topQueriesRow { |
| 312 | row := topQueriesRow{ |
| 313 | RequestID: r.RequestID, |
| 314 | RequestTimeRaw: r.RequestTime, |
| 315 | Statement: r.Statement, |
| 316 | User: r.User, |
| 317 | ClientContextID: r.ClientContextID, |
| 318 | } |
| 319 | |
| 320 | if t, err := time.Parse(time.RFC3339Nano, r.RequestTime); err == nil { |
| 321 | row.RequestTime = t |
| 322 | } else if t, err := time.Parse(time.RFC3339, r.RequestTime); err == nil { |
| 323 | row.RequestTime = t |
| 324 | } |
| 325 | |
| 326 | row.ElapsedMs = f.parseDurationMs(r.ElapsedTime) |
| 327 | row.ServiceMs = f.parseDurationMs(r.ServiceTime) |
| 328 | row.ResultCount = f.parseNumber(r.ResultCount) |
| 329 | row.ResultSize = f.parseNumber(r.ResultSize) |
| 330 | row.ErrorCount = f.parseNumber(r.ErrorCount) |
| 331 | row.WarningCount = f.parseNumber(r.WarningCount) |
| 332 | |
| 333 | return row |
| 334 | } |
| 335 | |
| 336 | func (f *funcTopQueries) parseDurationMs(raw string) float64 { |
| 337 | if raw == "" { |
| 338 | return 0 |
| 339 | } |
| 340 | if d, err := time.ParseDuration(raw); err == nil { |
| 341 | return float64(d) / float64(time.Millisecond) |
| 342 | } |
| 343 | return 0 |
| 344 | } |
| 345 | |
| 346 | func (f *funcTopQueries) parseNumber(n json.Number) int64 { |
| 347 | if n == "" { |
| 348 | return 0 |
| 349 | } |
| 350 | if i, err := n.Int64(); err == nil { |
| 351 | return i |
| 352 | } |
| 353 | if fv, err := n.Float64(); err == nil { |
| 354 | return int64(fv) |
| 355 | } |
| 356 | return 0 |
| 357 | } |
| 358 | |
| 359 | func (f *funcTopQueries) mapSortColumn(col string) string { |
| 360 | switch col { |
| 361 | case "elapsedTime", "serviceTime", "requestTime", "resultCount", "requestId": |
| 362 | return col |
| 363 | default: |
| 364 | return "elapsedTime" |
| 365 | } |
| 366 | } |
| 367 | |
| 368 | func (f *funcTopQueries) sortRows(rows []topQueriesRow, sortColumn string) { |
| 369 | switch sortColumn { |
| 370 | case "serviceTime": |
| 371 | sort.Slice(rows, func(i, j int) bool { |
| 372 | return rows[i].ServiceMs > rows[j].ServiceMs |
| 373 | }) |
| 374 | case "requestTime": |
| 375 | sort.Slice(rows, func(i, j int) bool { |
| 376 | return rows[i].RequestTime.After(rows[j].RequestTime) |
| 377 | }) |
| 378 | case "resultCount": |
| 379 | sort.Slice(rows, func(i, j int) bool { |
| 380 | return rows[i].ResultCount > rows[j].ResultCount |
| 381 | }) |
| 382 | case "requestId": |
| 383 | sort.Slice(rows, func(i, j int) bool { |
| 384 | return rows[i].RequestID > rows[j].RequestID |
| 385 | }) |
| 386 | default: |
| 387 | sort.Slice(rows, func(i, j int) bool { |
| 388 | return rows[i].ElapsedMs > rows[j].ElapsedMs |
| 389 | }) |
| 390 | } |
| 391 | } |