agent-events: Consolidate metrics into a single labeled counter (#20067)
- Replace multiple individual metrics with a unified 'agent_events_requests_ratio_total' metric - Uses status as a label to distinguish between different types of requests: total, success, duplicate, error, etc. - Makes visualization in dashboards easier with a single status-based chart - Maintains full metric coverage with improved efficiency
Costa Tsaousis committed
Apr 6, 2025 at 16:05 UTC
6680cc80dfe68e21e91fb3645426f96e99ff3745
2 files changed
+30
-24
packaging/tools/agent-events/server.go
+27
-20
@@ -25,6 +25,7 @@ import (
25
"github.com/prometheus/client_golang/prometheus/promhttp"
26
"github.com/tidwall/gjson"
27
"go.opentelemetry.io/otel"
28
+ "go.opentelemetry.io/otel/attribute"
29
"go.opentelemetry.io/otel/exporters/prometheus"
30
"go.opentelemetry.io/otel/metric"
31
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
@@ -65,12 +66,7 @@ var (
66
meter metric.Meter
67
68
// Counters
68
- requestsTotal metric.Int64Counter
69
- duplicateRequests metric.Int64Counter
70
- methodNotAllowedRequests metric.Int64Counter
71
- badRequestsTotal metric.Int64Counter
72
- entityTooLargeRequests metric.Int64Counter
73
- internalErrorRequests metric.Int64Counter
69
+ requestsCounter metric.Int64Counter // Unified counter with status label
70
bytesReceived metric.Int64Counter
71
72
// Gauges
@@ -119,6 +115,19 @@ func initMetrics() (*prometheus.Exporter, error) {
115
}
116
return counter, err
117
}
118
+
119
+ // Create a counter with a status label for consolidated metrics
120
+ createLabeledCounter := func(name, desc string) (metric.Int64Counter, error) {
121
+ counter, err := meter.Int64Counter(
122
+ name,
123
+ metric.WithDescription(desc),
124
+ metric.WithUnit("1"),
125
+ )
126
+ if err != nil {
127
+ slog.Error("failed to create labeled counter", "name", name, "error", err)
128
+ }
129
+ return counter, err
130
+ }
131
createGauge := func(name, desc string) (metric.Int64ObservableGauge, error) {
132
gauge, err := meter.Int64ObservableGauge(name, metric.WithDescription(desc))
133
if err != nil {
@@ -137,13 +146,9 @@ func initMetrics() (*prometheus.Exporter, error) {
146
return hist, err
147
}
148
140
- requestsTotal, _ = createCounter("agent_events_requests_total", "Total number of requests received")
141
- duplicateRequests, _ = createCounter("agent_events_duplicate_requests_total", "Total number of duplicate requests detected")
142
- methodNotAllowedRequests, _ = createCounter("agent_events_method_not_allowed_requests_total", "Total number of requests with incorrect HTTP method")
143
- badRequestsTotal, _ = createCounter("agent_events_bad_request_requests_total", "Total number of requests with invalid JSON")
144
- entityTooLargeRequests, _ = createCounter("agent_events_entity_too_large_requests_total", "Total number of requests exceeding size limits")
145
- internalErrorRequests, _ = createCounter("agent_events_internal_error_requests_total", "Total number of internal server errors")
146
- bytesReceived, _ = createCounter("agent_events_received_bytes_total", "Total number of bytes received in request bodies")
149
+ // Create unified request counter with status label
150
+ requestsCounter, _ = createLabeledCounter("agent_events_requests_ratio_total", "Total number of requests by status")
151
+ bytesReceived, _ = createCounter("agent_events_bytes_received_total", "Total number of bytes received in request bodies")
152
dedupCacheSize, _ = createGauge("agent_events_dedup_cache_entries", "Current number of entries in the deduplication cache")
153
activeConnectionsGauge, _ = createGauge("agent_events_active_connections", "Number of currently active connections")
154
uptimeGauge, _ = createGauge("agent_events_uptime_seconds", "How long the server has been running in seconds")
@@ -255,7 +260,7 @@ func handler(w http.ResponseWriter, r *http.Request) {
260
ctx := r.Context()
261
262
// Record metrics using OTEL API
258
- requestsTotal.Add(ctx, 1)
263
+ requestsCounter.Add(ctx, 1, metric.WithAttributes(attribute.String("status", "total")))
264
defer func() {
265
requestDuration.Record(ctx, time.Since(requestStartTime).Seconds())
266
}()
@@ -264,7 +269,7 @@ func handler(w http.ResponseWriter, r *http.Request) {
269
if r.Method != http.MethodPost {
270
http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
271
slog.Info("request discarded", "reason", "method_not_allowed", "method", r.Method, "remote_addr", r.RemoteAddr)
267
- methodNotAllowedRequests.Add(ctx, 1)
272
+ requestsCounter.Add(ctx, 1, metric.WithAttributes(attribute.String("status", "method_not_allowed")))
273
return
274
}
275
@@ -276,11 +281,11 @@ func handler(w http.ResponseWriter, r *http.Request) {
281
if errors.As(err, &maxBytesErr) {
282
http.Error(w, fmt.Sprintf("Request body exceeds limit (%d bytes)", maxRequestBodySize), http.StatusRequestEntityTooLarge)
283
slog.Info("request discarded", "reason", "body_too_large", "limit", maxRequestBodySize, "remote_addr", r.RemoteAddr)
279
- entityTooLargeRequests.Add(ctx, 1)
284
+ requestsCounter.Add(ctx, 1, metric.WithAttributes(attribute.String("status", "entity_too_large")))
285
} else {
286
http.Error(w, "Error reading request", http.StatusInternalServerError)
287
slog.Error("request discarded", "reason", "error_reading_body", "error", err)
283
- internalErrorRequests.Add(ctx, 1)
288
+ requestsCounter.Add(ctx, 1, metric.WithAttributes(attribute.String("status", "internal_error")))
289
}
290
return
291
}
@@ -292,7 +297,7 @@ func handler(w http.ResponseWriter, r *http.Request) {
297
http.Error(w, "Invalid JSON", http.StatusBadRequest)
298
bodyDetail := ""; if slog.Default().Enabled(context.Background(), slog.LevelDebug) { bodyDetail = fmt.Sprintf(", Body: %s", string(body)) } else { bodyDetail = fmt.Sprintf(", Body snippet: %s", limitString(string(body), 100)) }
299
slog.Warn("request discarded", "reason", "invalid_json", "error", err.Error(), "body_detail", bodyDetail)
295
- badRequestsTotal.Add(ctx, 1)
300
+ requestsCounter.Add(ctx, 1, metric.WithAttributes(attribute.String("status", "bad_request")))
301
return
302
}
303
@@ -311,7 +316,7 @@ func handler(w http.ResponseWriter, r *http.Request) {
316
317
if !checkAndRecordHash(dedupHash) {
318
shouldProcess = false
314
- duplicateRequests.Add(ctx, 1)
319
+ requestsCounter.Add(ctx, 1, metric.WithAttributes(attribute.String("status", "duplicate")))
320
slog.Debug("discarded duplicate request", "hash", fmt.Sprintf("%x", dedupHash), "note", "timestamp refreshed")
321
if _, err := w.Write([]byte("OK")); err != nil { slog.Error("error writing response", "context", "after_duplicate_discard", "error", err) }
322
return
@@ -336,12 +341,14 @@ func handler(w http.ResponseWriter, r *http.Request) {
341
if err != nil {
342
http.Error(w, "Internal Server Error during output marshal", http.StatusInternalServerError)
343
slog.Error("request discarded", "reason", "json_marshal_failed", "error", err)
339
- internalErrorRequests.Add(ctx, 1)
344
+ requestsCounter.Add(ctx, 1, metric.WithAttributes(attribute.String("status", "internal_error")))
345
return
346
}
347
348
// Write Output & Response
349
fmt.Println(string(outputBytes))
350
+ // Count successful processing
351
+ requestsCounter.Add(ctx, 1, metric.WithAttributes(attribute.String("status", "success")))
352
if _, err := w.Write([]byte("OK")); err != nil {
353
slog.Error("error writing response", "context", "after_successful_processing", "error", err)
354
}
packaging/tools/agent-events/server_test.go
+3
-4
@@ -244,11 +244,10 @@ func TestHandler(t *testing.T) {
244
245
// Check for presence of key metrics (adjust names if needed)
246
expectedMetrics := []string{
247
- "agent_events_requests_total",
248
- "agent_events_duplicate_requests_total",
249
- "agent_events_method_not_allowed_requests_total",
247
+ "agent_events_requests_ratio_total", // Consolidated metric with status labels
248
+ "agent_events_bytes_received_total",
249
"agent_events_dedup_cache_entries",
251
- "agent_events_request_duration_seconds_count", // Check for count specifically
250
+ "agent_events_request_duration_seconds", // Check for histograms
251
"go_goroutines",
252
}
253
for _, metricName := range expectedMetrics {