@cryptotaxi247 / netdata-1 / commits / e032ee04e

agent-events: add deduplicating web server (#20014)

add deduplicating web server

Costa Tsaousis committed Mar 31, 2025 at 18:04 UTC e032ee04e7cc594e992fa5f3e9e3fce56a899542
5 files changed +497 -22
packaging/tools/agent-events/.gitignore
+2
@@ -1 +1,3 @@
1 server
2 +go.mod
3 +go.sum
\ No newline at end of file
packaging/tools/agent-events/build.sh
+24 -1
@@ -1,3 +1,26 @@
1 #!/bin/sh
2
3 -go build server.go
3 +# Exit immediately if a command exits with a non-zero status.
4 +set -e
5 +
6 +# Optional: Clean slate - remove existing module files
7 +test -f "go.mod" && rm -f go.mod
8 +test -f "go.sum" && rm -f go.sum
9 +
10 +# 1. Initialize the Go module
11 +echo "Initializing Go module..."
12 +go mod init server
13 +
14 +# 2. Tidy dependencies
15 +echo "Tidying dependencies..."
16 +go mod tidy
17 +
18 +# 3. Build the main server executable using '.' for current package
19 +echo "Building server executable..."
20 +go build -o ./server .
21 +
22 +# 4. Run the unit tests
23 +echo "Running unit tests..."
24 +go test -v
25 +
26 +echo "Build and test script finished."
packaging/tools/agent-events/run.sh
+1 -1
@@ -1,6 +1,6 @@
1 #!/usr/bin/env bash
2
3 -stdbuf -oL /opt/agent-events/server --port=30001 2>/dev/null \
3 +stdbuf -oL /opt/agent-events/server --port=30001 --dedup-key agent.ephemeral_id --dedup-window 1800 2>/dev/null \
4 | stdbuf -oL log2journal json \
5 --prefix 'AE_' \
6 --inject 'SYSLOG_IDENTIFIER=agent-events' \
packaging/tools/agent-events/server.go
+210 -20
@@ -1,6 +1,9 @@
1 package main
2
3 import (
4 + "crypto/sha256"
5 + "encoding/json"
6 + "errors"
7 "flag"
8 "fmt"
9 "io"
@@ -8,50 +11,237 @@ import (
11 "net/http"
12 "os"
13 "strings"
14 + "sync"
15 + "time"
16 +
17 + "github.com/tidwall/gjson"
18 +)
19 +
20 +// --- Constants ---
21 +const (
22 + maxRequestBodySize = 20 * 1024 // 20 KiB
23 +)
24 +
25 +// --- Custom Flag Type for Multi-use --dedup-key ---
26 +type dedupPaths []string
27 +
28 +func (d *dedupPaths) String() string { return fmt.Sprintf("%v", *d) }
29 +func (d *dedupPaths) Set(value string) error {
30 + if value == "" {
31 + return fmt.Errorf("dedup-key path cannot be empty")
32 + }
33 + *d = append(*d, value)
34 + return nil
35 +}
36 +
37 +// --- Global variables ---
38 +var (
39 + seenIDs map[[32]byte]seenEntry
40 + mapMutex = &sync.Mutex{}
41 + dedupWindow time.Duration
42 + debugMode bool
43 + keyPaths dedupPaths
44 + dedupSeparator string
45 )
46
47 +// --- Data Structures ---
48 +type seenEntry struct {
49 + timestamp time.Time
50 +}
51 +
52 +// --- Core Logic Functions ---
53 +
54 +// checkAndRecordHash accepts the SHA256 hash ([32]byte) for checking.
55 +// It now REFRESHES the timestamp whenever a hash is found,
56 +// effectively creating a sliding deduplication window.
57 +func checkAndRecordHash(hash [32]byte) bool {
58 + now := time.Now()
59 + mapMutex.Lock()
60 + defer mapMutex.Unlock()
61 +
62 + var zeroHash [32]byte
63 + if hash == zeroHash {
64 + log.Println("Warning: checkAndRecordHash received potentially zero hash.")
65 + // Decide if zero hash should always be discarded, e.g. return false
66 + }
67 +
68 + // Check if the hash exists in the map
69 + entry, found := seenIDs[hash]
70 +
71 + if found {
72 + // --- Hash Found ---
73 + // Check if it was a duplicate based on the *previous* timestamp
74 + isRecentDuplicate := now.Sub(entry.timestamp) < dedupWindow
75 +
76 + // *** Always update the timestamp to 'now' to refresh the window ***
77 + seenIDs[hash] = seenEntry{timestamp: now}
78 +
79 + // Return 'false' if it was a recent duplicate (suppress processing),
80 + // return 'true' if it was found but expired (allow processing).
81 + return !isRecentDuplicate
82 +
83 + } else {
84 + // --- Hash Not Found ---
85 + // Record the new hash with the current timestamp
86 + seenIDs[hash] = seenEntry{timestamp: now}
87 + // Return 'true' as this is the first time (or first time after expiry)
88 + return true
89 + }
90 +}
91 +
92 +
93 +// cleanupExpiredEntries uses the hash ([32]byte) as the key type.
94 +func cleanupExpiredEntries(interval time.Duration) {
95 + ticker := time.NewTicker(interval)
96 + defer ticker.Stop()
97 + cleanedCount := 0
98 + lastCleanupLogTime := time.Now()
99 + for range ticker.C {
100 + mapMutex.Lock()
101 + now := time.Now()
102 + for h, entry := range seenIDs {
103 + if now.Sub(entry.timestamp) >= dedupWindow {
104 + delete(seenIDs, h)
105 + cleanedCount++
106 + }
107 + }
108 + mapMutex.Unlock()
109 + // Simplified periodic logging for cleanup
110 + if cleanedCount > 0 && time.Since(lastCleanupLogTime) > time.Hour {
111 + if debugMode {
112 + log.Printf("Debug: Cleaned up %d expired entries in the past hour.", cleanedCount)
113 + }
114 + cleanedCount = 0 // Reset count after logging
115 + lastCleanupLogTime = time.Now()
116 + }
117 + }
118 +}
119 +
120 +// --- HTTP Handler ---
121 func handler(w http.ResponseWriter, r *http.Request) {
122 if r.Method != http.MethodPost {
123 http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
16 - log.Printf("Method not allowed: %s\n", r.Method)
124 + log.Printf("Discarded: Method not allowed (%s) from %s", r.Method, r.RemoteAddr)
125 return
126 }
19 -
20 - // Read the entire request body.
127 + r.Body = http.MaxBytesReader(w, r.Body, maxRequestBodySize)
128 body, err := io.ReadAll(r.Body)
129 if err != nil {
23 - http.Error(w, "Error reading request", http.StatusInternalServerError)
24 - log.Printf("Error reading request: %v\n", err)
130 + var maxBytesErr *http.MaxBytesError
131 + if errors.As(err, &maxBytesErr) {
132 + http.Error(w, fmt.Sprintf("Request body exceeds limit (%d bytes)", maxRequestBodySize), http.StatusRequestEntityTooLarge)
133 + log.Printf("Discarded: Request body too large (limit %d bytes) from %s", maxRequestBodySize, r.RemoteAddr)
134 + } else {
135 + http.Error(w, "Error reading request", http.StatusInternalServerError)
136 + log.Printf("Discarded: Error reading request body: %v", err)
137 + }
138 return
139 }
27 - defer r.Body.Close()
140
29 - // Remove all newline characters.
30 - cleaned := strings.ReplaceAll(string(body), "\n", "")
31 - cleaned = strings.ReplaceAll(cleaned, "\r", "")
141 + shouldProcess := true
142 + if len(keyPaths) > 0 {
143 + var keyBuilder strings.Builder
144 + for i, path := range keyPaths {
145 + result := gjson.GetBytes(body, path)
146 + var valueStr string
147 + if result.Exists() { valueStr = result.String() } else { valueStr = "" }
148 + keyBuilder.WriteString(valueStr)
149 + if i < len(keyPaths)-1 { keyBuilder.WriteString(dedupSeparator) }
150 + }
151 + finalKeyString := keyBuilder.String()
152 + dedupHash := sha256.Sum256([]byte(finalKeyString))
153 + if debugMode {
154 + log.Printf("Debug: Generated dedup key string: \"%s\"", finalKeyString)
155 + log.Printf("Debug: Generated dedup hash: %x", dedupHash)
156 + }
157
33 - // Write to stdout with a single newline at the end.
34 - fmt.Println(cleaned)
158 + // Call the updated checkAndRecordHash function
159 + if !checkAndRecordHash(dedupHash) {
160 + // It was determined to be a duplicate (based on previous timestamp)
161 + shouldProcess = false
162 + if debugMode { log.Printf("Debug: Discarded duplicate hash: %x (timestamp refreshed)", dedupHash) } // Updated log message
163 + // Respond OK for duplicate and stop processing
164 + if _, err := w.Write([]byte("OK")); err != nil { log.Printf("Error writing response after duplicate discard: %v", err) }
165 + return // Exit handler early for duplicates
166 + }
167 + // If we reach here, it was not a recent duplicate (new or expired)
168 + } else {
169 + if debugMode { log.Println("Debug: No --dedup-key flags provided, skipping deduplication.") }
170 + }
171
36 - // Respond with OK.
37 - if _, err := w.Write([]byte("OK")); err != nil {
38 - log.Printf("Error writing response: %v\n", err)
39 - return
172 + if shouldProcess {
173 + var fullData interface{}
174 + if err := json.Unmarshal(body, &fullData); err != nil {
175 + http.Error(w, "Invalid JSON for full parsing", http.StatusBadRequest)
176 + bodyDetail := ""
177 + if debugMode { bodyDetail = fmt.Sprintf(", Body: %s", string(body)) } else { bodyDetail = fmt.Sprintf(", Body snippet: %s", limitString(string(body), 100)) }
178 + log.Printf("Discarded: Failed to fully parse JSON (post-dedup): %v%s", err, bodyDetail)
179 + return
180 + }
181 + outputBytes, err := json.Marshal(fullData)
182 + if err != nil {
183 + http.Error(w, "Internal Server Error during output marshal", http.StatusInternalServerError)
184 + log.Printf("Discarded: Failed to marshal JSON for output: %v", err)
185 + return
186 + }
187 + fmt.Println(string(outputBytes))
188 + if _, err := w.Write([]byte("OK")); err != nil { log.Printf("Error writing OK response: %v", err) }
189 }
190 }
191
192 +// --- Main Function ---
193 func main() {
44 - // Configure logging to write to stderr.
194 log.SetOutput(os.Stderr)
195 log.SetFlags(log.LstdFlags | log.Lshortfile)
196
48 - // Parse the port from the command line.
197 + // --- Command Line Flags ---
198 port := flag.Int("port", 8080, "Port to listen on")
199 + dedupSeconds := flag.Int("dedup-window", 1800, "Deduplication window in seconds (e.g., 1800 for 30 minutes)")
200 + flag.BoolVar(&debugMode, "debug", false, "Enable debug mode for verbose logging")
201 + flag.Var(&keyPaths, "dedup-key", "JSON path (dot-notation) for deduplication key (can be used multiple times)")
202 + flag.StringVar(&dedupSeparator, "dedup-separator", "-", "Separator used between values from multiple --dedup-key paths")
203 flag.Parse()
204
205 + seenIDs = make(map[[32]byte]seenEntry)
206 + dedupWindow = time.Duration(*dedupSeconds) * time.Second
207 +
208 + if dedupWindow > 0 && len(keyPaths) > 0 {
209 + cleanupInterval := dedupWindow / 10
210 + if cleanupInterval < 1*time.Minute { cleanupInterval = 1 * time.Minute } else if cleanupInterval > 15*time.Minute { cleanupInterval = 15 * time.Minute }
211 + log.Printf("Cleanup goroutine started. Interval: %v", cleanupInterval)
212 + go cleanupExpiredEntries(cleanupInterval)
213 + } else if dedupWindow <= 0 && len(keyPaths) > 0 {
214 + log.Println("Warning: Deduplication keys provided, but window is zero or negative. Deduplication effectively disabled.")
215 + }
216 +
217 + // --- Configure HTTP Server ---
218 + readTimeout := 10 * time.Second
219 + writeTimeout := 10 * time.Second
220 + idleTimeout := 60 * time.Second
221 + server := &http.Server{
222 + Addr: fmt.Sprintf(":%d", *port),
223 + Handler: http.DefaultServeMux,
224 + ReadTimeout: readTimeout,
225 + WriteTimeout: writeTimeout,
226 + IdleTimeout: idleTimeout,
227 + }
228 http.HandleFunc("/", handler)
53 - log.Printf("Server listening on port %d\n", *port)
54 - if err := http.ListenAndServe(fmt.Sprintf(":%d", *port), nil); err != nil {
55 - log.Fatalf("Server failed: %v\n", err)
229 +
230 + // --- Start Server ---
231 + log.Printf("Server listening on port %d", *port)
232 + log.Printf("Maximum request body size: %d bytes", maxRequestBodySize)
233 + if len(keyPaths) > 0 {
234 + log.Printf("Deduplication enabled: Keys=%v, Separator='%s', Window=%v (Sliding window: timestamp refreshed on duplicate)", keyPaths, dedupSeparator, dedupWindow) // Updated log message
235 + } else {
236 + log.Println("Deduplication disabled (no --dedup-key specified).")
237 }
238 + log.Printf("Debug mode enabled: %t", debugMode)
239 + log.Printf("Server timeouts -> Read: %v, Write: %v, Idle: %v", readTimeout, writeTimeout, idleTimeout)
240 + log.Fatal(server.ListenAndServe())
241 +}
242 +
243 +// --- Helper Functions ---
244 +func limitString(s string, maxLen int) string {
245 + if len(s) <= maxLen { return s }
246 + return s[:maxLen] + "..."
247 }
packaging/tools/agent-events/server_test.go new
+260
@@ -0,0 +1,260 @@
1 +package main
2 +
3 +import (
4 + "bytes"
5 + "io"
6 + "log"
7 + "net/http"
8 + "net/http/httptest"
9 + "os"
10 + "strings"
11 + "testing"
12 + "time"
13 +)
14 +
15 +// Helper function to capture stdout/stderr during a test run
16 +func captureOutput(t *testing.T, f func()) (stdout, stderr string) {
17 + t.Helper() // Marks this as a helper function for testing framework
18 +
19 + originalStdout := os.Stdout
20 + originalStderr := os.Stderr
21 + originalLogOutput := log.Writer() // Get current log output writer
22 +
23 + // Create pipes to capture output
24 + rOut, wOut, _ := os.Pipe()
25 + rErr, wErr, _ := os.Pipe()
26 +
27 + // Redirect stdout and stderr
28 + os.Stdout = wOut
29 + os.Stderr = wErr
30 + log.SetOutput(wErr) // Redirect default logger to stderr pipe
31 +
32 + // Use t.Cleanup to ensure restoration even if the test panics
33 + t.Cleanup(func() {
34 + os.Stdout = originalStdout
35 + os.Stderr = originalStderr
36 + log.SetOutput(originalLogOutput) // Restore original log output
37 + })
38 +
39 + // Channels to signal when reading is done
40 + outCh := make(chan string)
41 + errCh := make(chan string)
42 +
43 + // Goroutine to read stdout
44 + go func() {
45 + var buf bytes.Buffer
46 + _, _ = io.Copy(&buf, rOut)
47 + outCh <- buf.String()
48 + }()
49 +
50 + // Goroutine to read stderr
51 + go func() {
52 + var buf bytes.Buffer
53 + _, _ = io.Copy(&buf, rErr)
54 + errCh <- buf.String()
55 + }()
56 +
57 + // --- Execute the function under test ---
58 + f()
59 + // --- ---
60 +
61 + // Close the writers to signal EOF to the readers
62 + _ = wOut.Close()
63 + _ = wErr.Close()
64 +
65 + // Read captured output
66 + stdout = <-outCh
67 + stderr = <-errCh
68 +
69 + // Optional: Print captured output via test logger if needed for debugging
70 + // t.Logf("Captured Stdout:\n%s", stdout)
71 + // t.Logf("Captured Stderr:\n%s", stderr)
72 +
73 + return stdout, stderr
74 +}
75 +
76 +// --- Test Suite ---
77 +
78 +func TestHandler(t *testing.T) {
79 + // --- Test Setup ---
80 + // Configure global variables for the tests
81 + keyPaths = []string{"id"} // Simple dedup key for testing
82 + dedupSeparator = "-"
83 + dedupWindow = 30 * time.Second // Use a reasonable window for tests
84 + debugMode = false // Start with debug off, can enable per test case
85 +
86 + // Ensure the map is initialized and clean before starting tests
87 + mapMutex.Lock()
88 + seenIDs = make(map[[32]byte]seenEntry)
89 + mapMutex.Unlock()
90 +
91 + // Helper to reset state between sub-tests
92 + resetState := func() {
93 + mapMutex.Lock()
94 + seenIDs = make(map[[32]byte]seenEntry) // Clear the map
95 + mapMutex.Unlock()
96 + keyPaths = []string{"id"} // Reset paths just in case
97 + debugMode = false
98 + // Reset other globals if they were modified
99 + }
100 +
101 + // --- Test Cases ---
102 +
103 + t.Run("FirstValidRequest", func(t *testing.T) {
104 + t.Cleanup(resetState) // Ensure state is reset after this sub-test
105 +
106 + // Prepare request
107 + jsonBody := `{"id": "uuid-1", "data": "value1"}`
108 + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(jsonBody))
109 + rr := httptest.NewRecorder() // Records the HTTP response
110 +
111 + // Execute handler and capture output
112 + stdout, stderr := captureOutput(t, func() {
113 + handler(rr, req)
114 + })
115 +
116 + // Assertions
117 + if status := rr.Code; status != http.StatusOK {
118 + t.Errorf("handler returned wrong status code: got %v want %v", status, http.StatusOK)
119 + }
120 + expectedResponse := `OK`
121 + if rr.Body.String() != expectedResponse {
122 + t.Errorf("handler returned unexpected body: got %v want %v", rr.Body.String(), expectedResponse)
123 + }
124 + // Check stdout contains the *exact* JSON (json.Marshal might reorder fields)
125 + // A simpler check is that it's not empty and maybe contains key parts.
126 + // For exact match, we'd need to unmarshal stdout and compare.
127 + if !strings.Contains(stdout, `"id":"uuid-1"`) || !strings.Contains(stdout, `"data":"value1"`) {
128 + t.Errorf("handler produced unexpected stdout:\ngot: %q\nwant it to contain parts of: %q", stdout, jsonBody)
129 + }
130 + if stderr != "" {
131 + t.Errorf("handler produced unexpected stderr: got %q want empty", stderr)
132 + }
133 +
134 + // Check internal state (optional, needs mutex)
135 + mapMutex.Lock()
136 + if len(seenIDs) != 1 {
137 + t.Errorf("expected 1 entry in seenIDs map, got %d", len(seenIDs))
138 + }
139 + mapMutex.Unlock()
140 + })
141 +
142 + t.Run("DuplicateRequestWithinWindow", func(t *testing.T) {
143 + t.Cleanup(resetState)
144 +
145 + // --- Setup: Simulate the first request having happened ---
146 + firstJsonBody := `{"id": "uuid-2", "data": "value2"}`
147 + firstReq := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(firstJsonBody))
148 + firstRr := httptest.NewRecorder()
149 + // Run handler once, ignore output for this setup run
150 + captureOutput(t, func() { handler(firstRr, firstReq) })
151 + if firstRr.Code != http.StatusOK {
152 + t.Fatalf("Setup failed: first request did not return OK")
153 + }
154 + // Verify setup placed item in map
155 + mapMutex.Lock()
156 + if len(seenIDs) != 1 {
157 + t.Fatalf("Setup failed: map size not 1 after first request")
158 + }
159 + mapMutex.Unlock()
160 + // --- End Setup ---
161 +
162 +
163 + // Prepare the duplicate request
164 + // Note: Using the *same* body string as the first request
165 + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(firstJsonBody))
166 + rr := httptest.NewRecorder()
167 +
168 + // Execute handler and capture output
169 + stdout, stderr := captureOutput(t, func() {
170 + handler(rr, req)
171 + })
172 +
173 + // Assertions
174 + if status := rr.Code; status != http.StatusOK {
175 + t.Errorf("handler returned wrong status code for duplicate: got %v want %v", status, http.StatusOK)
176 + }
177 + expectedResponse := `OK`
178 + if rr.Body.String() != expectedResponse {
179 + t.Errorf("handler returned unexpected body for duplicate: got %v want %v", rr.Body.String(), expectedResponse)
180 + }
181 + // Stdout should be empty for a duplicate
182 + if stdout != "" {
183 + t.Errorf("handler produced unexpected stdout for duplicate: got %q want empty", stdout)
184 + }
185 + // Stderr should be empty (unless debug mode logs duplicates)
186 + if stderr != "" {
187 + t.Errorf("handler produced unexpected stderr for duplicate: got %q want empty", stderr)
188 + }
189 + // Map size should remain 1
190 + mapMutex.Lock()
191 + if len(seenIDs) != 1 {
192 + t.Errorf("expected 1 entry in seenIDs map after duplicate, got %d", len(seenIDs))
193 + }
194 + mapMutex.Unlock()
195 + })
196 +
197 + t.Run("InvalidJSON", func(t *testing.T) {
198 + t.Cleanup(resetState)
199 +
200 + // Prepare request
201 + jsonBody := `{"id": "uuid-3", "data":` // Invalid JSON
202 + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(jsonBody))
203 + rr := httptest.NewRecorder()
204 +
205 + // Execute handler and capture output
206 + stdout, stderr := captureOutput(t, func() {
207 + handler(rr, req)
208 + })
209 +
210 + // Assertions
211 + if status := rr.Code; status != http.StatusBadRequest {
212 + t.Errorf("handler returned wrong status code for invalid JSON: got %v want %v", status, http.StatusBadRequest)
213 + }
214 + // Stdout should be empty
215 + if stdout != "" {
216 + t.Errorf("handler produced unexpected stdout for invalid JSON: got %q want empty", stdout)
217 + }
218 + // Stderr should contain the parsing error log message
219 + if !strings.Contains(stderr, "Failed to fully parse JSON") {
220 + t.Errorf("handler did not produce expected stderr log for invalid JSON: got %q", stderr)
221 + }
222 + })
223 +
224 + t.Run("MissingDedupKey", func(t *testing.T) {
225 + t.Cleanup(resetState)
226 +
227 + // Prepare request - JSON is valid but missing the 'id' field used by keyPaths
228 + jsonBody := `{"other_id": "uuid-4", "data": "value4"}`
229 + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(jsonBody))
230 + rr := httptest.NewRecorder()
231 +
232 + // Execute handler and capture output
233 + stdout, stderr := captureOutput(t, func() {
234 + handler(rr, req)
235 + })
236 +
237 + // Assertions for missing key (results in empty string "" for the key part)
238 + // This *should* be processed correctly, as "" is a valid key string before hashing
239 + // The hash of "" will be deduplicated like any other hash.
240 +
241 + if status := rr.Code; status != http.StatusOK {
242 + t.Errorf("handler returned wrong status code for missing key: got %v want %v", status, http.StatusOK)
243 + }
244 + // Stdout should contain the JSON
245 + if !strings.Contains(stdout, `"other_id":"uuid-4"`) || !strings.Contains(stdout, `"data":"value4"`) {
246 + t.Errorf("handler produced unexpected stdout for missing key:\ngot: %q\nwant it to contain parts of: %q", stdout, jsonBody)
247 + }
248 + if stderr != "" {
249 + t.Errorf("handler produced unexpected stderr for missing key: got %q want empty", stderr)
250 + }
251 + // Check map (hash of "" should be present)
252 + mapMutex.Lock()
253 + if len(seenIDs) != 1 {
254 + t.Errorf("expected 1 entry in seenIDs map for missing key, got %d", len(seenIDs))
255 + }
256 + mapMutex.Unlock()
257 + })
258 +
259 + // Add more test cases: Wrong method (GET), expired duplicate, multiple dedup keys, etc.
260 +}