@cryptotaxi247 / netdata-1 / commits / d290766df

add go.d/rspamd (#17679)

Ilya Mashchenko committed May 16, 2024 at 16:21 UTC d290766df9809b70dcfbe98ff0ff7b20393402a9
15 files changed +1101 -10
src/go/collectors/go.d.plugin/README.md
+7 -7
@@ -17,9 +17,8 @@ collection modules written in `go`.
17 1. It runs as an independent process (`ps fax` shows it).
18 2. It is started and stopped automatically by Netdata.
19 3. It communicates with Netdata via a unidirectional pipe (sending data to the Netdata daemon).
20 -4. Supports any number of data collection [modules](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules).
21 -5. Allows each [module](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules) to have any number of data
22 - collection jobs.
20 +4. Supports any number of data collection modules.
21 +5. Allows each module to have any number of data collection jobs.
22
23 ## Bug reports, feature requests, and questions
24
@@ -115,6 +114,7 @@ see the appropriate collector readme.
114 | [pulsar](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules/portcheck) | Apache Pulsar |
115 | [rabbitmq](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules/rabbitmq) | RabbitMQ |
116 | [redis](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules/redis) | Redis |
117 +| [rspamd](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules/rspamd) | Rspamd |
118 | [scaleio](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules/scaleio) | Dell EMC ScaleIO |
119 | [sensors](https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules) | Hardware Sensors |
120 | [SNMP](https://github.com/netdata/netdata/blob/master/src/go/collectors/go.d.plugin/modules/snmp) | SNMP |
@@ -141,8 +141,8 @@ see the appropriate collector readme.
141 ## Configuration
142
143 Edit the `go.d.conf` configuration file using `edit-config` from the
144 -Netdata [config directory](https://github.com/netdata/netdata/blob/master/docs/netdata-agent/configuration/README.md#the-netdata-config-directory), which is typically
145 -at `/etc/netdata`.
144 +Netdata [config directory](https://github.com/netdata/netdata/blob/master/docs/netdata-agent/configuration/README.md#the-netdata-config-directory),
145 +which is typically at `/etc/netdata`.
146
147 ```bash
148 cd /etc/netdata # Replace this path with your Netdata config directory
@@ -173,8 +173,8 @@ modules:
173 example: yes
174 ```
175
176 -Then [restart netdata](https://github.com/netdata/netdata/blob/master/packaging/installer/README.md#maintaining-a-netdata-agent-installation) for the
177 -change to take effect.
176 +Then [restart netdata](https://github.com/netdata/netdata/blob/master/packaging/installer/README.md#maintaining-a-netdata-agent-installation)
177 +for the change to take effect.
178
179 ## Contributing
180
src/go/collectors/go.d.plugin/agent/discovery/sd/discoverer/netlisteners/netlisteners.go
+2 -2
@@ -310,10 +310,10 @@ func (e *localListenersExec) discover(ctx context.Context) ([]byte, error) {
310 func extractComm(cmdLine string) string {
311 i := strings.IndexByte(cmdLine, ' ')
312 if i <= 0 {
313 - return cmdLine
313 + return strings.TrimSuffix(cmdLine, ":")
314 }
315 _, comm := filepath.Split(cmdLine[:i])
316 - return comm
316 + return strings.TrimSuffix(comm, ":")
317 }
318
319 func calcHash(obj any) (uint64, error) {
src/go/collectors/go.d.plugin/config/go.d.conf
+1
@@ -78,6 +78,7 @@ modules:
78 # pulsar: yes
79 # rabbitmq: yes
80 # redis: yes
81 +# rspamd: yes
82 # scaleio: yes
83 # sensors: yes
84 # snmp: yes
src/go/collectors/go.d.plugin/config/go.d/rspamd.conf new
+6
@@ -0,0 +1,6 @@
1 +## All available configuration options, their descriptions and default values:
2 +## https://github.com/netdata/netdata/tree/master/src/go/collectors/go.d.plugin/modules/rspamd#readme
3 +
4 +#jobs:
5 +# - name: local
6 +# url: http://127.0.0.1:11334
src/go/collectors/go.d.plugin/config/go.d/sd/net_listeners.conf
+8 -1
@@ -69,7 +69,7 @@ classify:
69 - tags: "mysql"
70 expr: '{{ or (eq .Port "3306") (eq .Comm "mysqld" "mariadbd") }}'
71 - tags: "nginx"
72 - expr: '{{ and (eq .Port "80" "8080") (eq .Comm "nginx" "nginx:") }}'
72 + expr: '{{ and (eq .Port "80" "8080") (eq .Comm "nginx") }}'
73 - tags: "ntpd"
74 expr: '{{ or (eq .Port "123") (eq .Comm "ntpd") }}'
75 - tags: "openvpn"
@@ -92,6 +92,8 @@ classify:
92 expr: '{{ or (eq .Port "15672") (glob .Cmdline "*rabbitmq*") }}'
93 - tags: "redis"
94 expr: '{{ or (eq .Port "6379") (eq .Comm "redis-server") }}'
95 + - tags: "rspamd"
96 + expr: '{{ or (eq .Port "11334") (eq .Comm "rspamd") }}'
97 - tags: "supervisord"
98 expr: '{{ and (eq .Port "9001") (eq .Comm "supervisord") }}'
99 - tags: "traefik"
@@ -321,6 +323,11 @@ compose:
323 module: pika
324 name: local
325 address: redis://@{{.IPAddress}}:{{.Port}}
326 + - selector: "rspamd"
327 + template: |
328 + module: rspamd
329 + name: local
330 + url: http://{{.Address}}
331 - selector: "postgres"
332 template: |
333 - module: postgres
src/go/collectors/go.d.plugin/modules/init.go
+1
@@ -70,6 +70,7 @@ import (
70 _ "github.com/netdata/netdata/go/go.d.plugin/modules/pulsar"
71 _ "github.com/netdata/netdata/go/go.d.plugin/modules/rabbitmq"
72 _ "github.com/netdata/netdata/go/go.d.plugin/modules/redis"
73 + _ "github.com/netdata/netdata/go/go.d.plugin/modules/rspamd"
74 _ "github.com/netdata/netdata/go/go.d.plugin/modules/scaleio"
75 _ "github.com/netdata/netdata/go/go.d.plugin/modules/sensors"
76 _ "github.com/netdata/netdata/go/go.d.plugin/modules/smartctl"
src/go/collectors/go.d.plugin/modules/rspamd/charts.go new
+110
@@ -0,0 +1,110 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package rspamd
4 +
5 +import "github.com/netdata/netdata/go/go.d.plugin/agent/module"
6 +
7 +const (
8 + prioClassifications = module.Priority + iota
9 + prioActions
10 + prioScans
11 + prioLearns
12 + prioConnections
13 + prioControlConnections
14 +)
15 +
16 +var charts = module.Charts{
17 + classificationsChartTmpl.Copy(),
18 +
19 + actionsChart.Copy(),
20 +
21 + scanChartTmpl.Copy(),
22 + learnChartTmpl.Copy(),
23 +
24 + connectionsChartTmpl.Copy(),
25 + controlConnectionsChartTmpl.Copy(),
26 +}
27 +
28 +var (
29 + classificationsChartTmpl = module.Chart{
30 + ID: "classifications",
31 + Title: "Classifications",
32 + Units: "messages/s",
33 + Fam: "classification",
34 + Ctx: "rspamd.classifications",
35 + Type: module.Stacked,
36 + Priority: prioClassifications,
37 + Dims: module.Dims{
38 + {ID: "ham_count", Name: "ham", Algo: module.Incremental},
39 + {ID: "spam_count", Name: "spam", Algo: module.Incremental},
40 + },
41 + }
42 +
43 + actionsChart = module.Chart{
44 + ID: "actions",
45 + Title: "Actions",
46 + Units: "messages/s",
47 + Fam: "actions",
48 + Ctx: "rspamd.actions",
49 + Type: module.Stacked,
50 + Priority: prioActions,
51 + Dims: module.Dims{
52 + {ID: "actions_reject", Name: "reject", Algo: module.Incremental},
53 + {ID: "actions_soft_reject", Name: "soft_reject", Algo: module.Incremental},
54 + {ID: "actions_rewrite_subject", Name: "rewrite_subject", Algo: module.Incremental},
55 + {ID: "actions_add_header", Name: "add_header", Algo: module.Incremental},
56 + {ID: "actions_greylist", Name: "greylist", Algo: module.Incremental},
57 + {ID: "actions_custom", Name: "custom", Algo: module.Incremental},
58 + {ID: "actions_discard", Name: "discard", Algo: module.Incremental},
59 + {ID: "actions_quarantine", Name: "quarantine", Algo: module.Incremental},
60 + {ID: "actions_no_action", Name: "no_action", Algo: module.Incremental},
61 + },
62 + }
63 +
64 + scanChartTmpl = module.Chart{
65 + ID: "scans",
66 + Title: "Scanned messages",
67 + Units: "messages/s",
68 + Fam: "training",
69 + Ctx: "rspamd.scans",
70 + Priority: prioScans,
71 + Dims: module.Dims{
72 + {ID: "scanned", Name: "scanned", Algo: module.Incremental},
73 + },
74 + }
75 +
76 + learnChartTmpl = module.Chart{
77 + ID: "learns",
78 + Title: "Learned messages",
79 + Units: "messages/s",
80 + Fam: "training",
81 + Ctx: "rspamd.learns",
82 + Priority: prioLearns,
83 + Dims: module.Dims{
84 + {ID: "learned", Name: "learned", Algo: module.Incremental},
85 + },
86 + }
87 +
88 + connectionsChartTmpl = module.Chart{
89 + ID: "connections",
90 + Title: "Connections",
91 + Units: "connections/s",
92 + Fam: "connections",
93 + Ctx: "rspamd.connections",
94 + Priority: prioConnections,
95 + Dims: module.Dims{
96 + {ID: "connections", Name: "connections", Algo: module.Incremental},
97 + },
98 + }
99 + controlConnectionsChartTmpl = module.Chart{
100 + ID: "control_connections",
101 + Title: "Control connections",
102 + Units: "connections/s",
103 + Fam: "connections",
104 + Ctx: "rspamd.control_connections",
105 + Priority: prioControlConnections,
106 + Dims: module.Dims{
107 + {ID: "control_connections", Name: "control_connections", Algo: module.Incremental},
108 + },
109 + }
110 +)
src/go/collectors/go.d.plugin/modules/rspamd/collect.go new
+94
@@ -0,0 +1,94 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package rspamd
4 +
5 +import (
6 + "encoding/json"
7 + "fmt"
8 + "io"
9 + "net/http"
10 +
11 + "github.com/netdata/netdata/go/go.d.plugin/pkg/stm"
12 + "github.com/netdata/netdata/go/go.d.plugin/pkg/web"
13 +)
14 +
15 +type rspamdStats struct {
16 + Version string `json:"version"`
17 + ConfigId string `json:"config_id"`
18 + Scanned int64 `json:"scanned" stm:"scanned"`
19 + Learned int64 `json:"learned" stm:"learned"`
20 + Actions struct {
21 + Reject int64 `json:"reject" stm:"reject"`
22 + SoftReject int64 `json:"soft reject" stm:"soft_reject"`
23 + RewriteSubject int64 `json:"rewrite subject" stm:"rewrite_subject"`
24 + AddHeader int64 `json:"add header" stm:"add_header"`
25 + Greylist int64 `json:"greylist" stm:"greylist"`
26 + NoAction int64 `json:"no action" stm:"no_action"`
27 + InvalidMaxAction int64 `json:"invalid max action" stm:"invalid_max_action"`
28 + Custom int64 `json:"custom" stm:"custom"`
29 + Discard int64 `json:"discard" stm:"discard"`
30 + Quarantine int64 `json:"quarantine" stm:"quarantine"`
31 + UnknownAction int64 `json:"unknown action" stm:"unknown_action"`
32 + } `json:"actions" stm:"actions"`
33 + ScanTimes []float64 `json:"scan_times"`
34 + SpamCount int64 `json:"spam_count" stm:"spam_count"`
35 + HamCount int64 `json:"ham_count" stm:"ham_count"`
36 + Connections int64 `json:"connections" stm:"connections"`
37 + ControlConnections int64 `json:"control_connections" stm:"control_connections"`
38 + FuzzyHashes map[string]int64 `json:"fuzzy_hashes"`
39 +}
40 +
41 +func (r *Rspamd) collect() (map[string]int64, error) {
42 + stats, err := r.queryRspamdStats()
43 + if err != nil {
44 + return nil, err
45 + }
46 +
47 + mx := stm.ToMap(stats)
48 +
49 + return mx, nil
50 +}
51 +
52 +func (r *Rspamd) queryRspamdStats() (*rspamdStats, error) {
53 + req, err := web.NewHTTPRequest(r.Request)
54 + if err != nil {
55 + return nil, err
56 + }
57 +
58 + req.URL.Path = "/stat"
59 +
60 + var stats rspamdStats
61 + if err := r.doOKDecode(req, &stats); err != nil {
62 + return nil, err
63 + }
64 +
65 + if stats.Version == "" || stats.ConfigId == "" || len(stats.FuzzyHashes) == 0 {
66 + return nil, fmt.Errorf("unexpected response: not rspamd data")
67 + }
68 +
69 + return &stats, nil
70 +}
71 +
72 +func (r *Rspamd) doOKDecode(req *http.Request, in interface{}) error {
73 + resp, err := r.httpClient.Do(req)
74 + if err != nil {
75 + return fmt.Errorf("error on HTTP request '%s': %v", req.URL, err)
76 + }
77 + defer closeBody(resp)
78 +
79 + if resp.StatusCode != http.StatusOK {
80 + return fmt.Errorf("'%s' returned HTTP status code: %d", req.URL, resp.StatusCode)
81 + }
82 +
83 + if err := json.NewDecoder(resp.Body).Decode(in); err != nil {
84 + return fmt.Errorf("error on decoding response from '%s': %v", req.URL, err)
85 + }
86 + return nil
87 +}
88 +
89 +func closeBody(resp *http.Response) {
90 + if resp != nil && resp.Body != nil {
91 + _, _ = io.Copy(io.Discard, resp.Body)
92 + _ = resp.Body.Close()
93 + }
94 +}
src/go/collectors/go.d.plugin/modules/rspamd/config_schema.json new
+177
@@ -0,0 +1,177 @@
1 +{
2 + "jsonSchema": {
3 + "$schema": "http://json-schema.org/draft-07/schema#",
4 + "title": "Rspamd collector configuration.",
5 + "type": "object",
6 + "properties": {
7 + "update_every": {
8 + "title": "Update every",
9 + "description": "Data collection interval, measured in seconds.",
10 + "type": "integer",
11 + "minimum": 1,
12 + "default": 1
13 + },
14 + "url": {
15 + "title": "URL",
16 + "description": "The base URL of the Rspamd [controller worker](https://rspamd.com/doc/workers/controller.html).",
17 + "type": "string",
18 + "default": "http://127.0.0.1:11334",
19 + "format": "uri"
20 + },
21 + "timeout": {
22 + "title": "Timeout",
23 + "description": "The timeout in seconds for the HTTP request.",
24 + "type": "number",
25 + "minimum": 0.5,
26 + "default": 1
27 + },
28 + "not_follow_redirects": {
29 + "title": "Not follow redirects",
30 + "description": "If set, the client will not follow HTTP redirects automatically.",
31 + "type": "boolean"
32 + },
33 + "username": {
34 + "title": "Username",
35 + "description": "The username for basic authentication.",
36 + "type": "string",
37 + "sensitive": true
38 + },
39 + "password": {
40 + "title": "Password",
41 + "description": "The password for basic authentication.",
42 + "type": "string",
43 + "sensitive": true
44 + },
45 + "proxy_url": {
46 + "title": "Proxy URL",
47 + "description": "The URL of the proxy server.",
48 + "type": "string"
49 + },
50 + "proxy_username": {
51 + "title": "Proxy username",
52 + "description": "The username for proxy authentication.",
53 + "type": "string",
54 + "sensitive": true
55 + },
56 + "proxy_password": {
57 + "title": "Proxy password",
58 + "description": "The password for proxy authentication.",
59 + "type": "string",
60 + "sensitive": true
61 + },
62 + "headers": {
63 + "title": "Headers",
64 + "description": "Additional HTTP headers to include in the request.",
65 + "type": [
66 + "object",
67 + "null"
68 + ],
69 + "additionalProperties": {
70 + "type": "string"
71 + }
72 + },
73 + "tls_skip_verify": {
74 + "title": "Skip TLS verification",
75 + "description": "If set, TLS certificate verification will be skipped.",
76 + "type": "boolean"
77 + },
78 + "tls_ca": {
79 + "title": "TLS CA",
80 + "description": "The path to the CA certificate file for TLS verification.",
81 + "type": "string",
82 + "pattern": "^$|^/"
83 + },
84 + "tls_cert": {
85 + "title": "TLS certificate",
86 + "description": "The path to the client certificate file for TLS authentication.",
87 + "type": "string",
88 + "pattern": "^$|^/"
89 + },
90 + "tls_key": {
91 + "title": "TLS key",
92 + "description": "The path to the client key file for TLS authentication.",
93 + "type": "string",
94 + "pattern": "^$|^/"
95 + },
96 + "body": {
97 + "title": "Body",
98 + "type": "string"
99 + },
100 + "method": {
101 + "title": "Method",
102 + "type": "string"
103 + }
104 + },
105 + "required": [
106 + "url"
107 + ],
108 + "additionalProperties": false,
109 + "patternProperties": {
110 + "^name$": {}
111 + }
112 + },
113 + "uiSchema": {
114 + "uiOptions": {
115 + "fullPage": true
116 + },
117 + "body": {
118 + "ui:widget": "hidden"
119 + },
120 + "method": {
121 + "ui:widget": "hidden"
122 + },
123 + "timeout": {
124 + "ui:help": "Accepts decimals for precise control (e.g., type 1.5 for 1.5 seconds)."
125 + },
126 + "password": {
127 + "ui:widget": "password"
128 + },
129 + "proxy_password": {
130 + "ui:widget": "password"
131 + },
132 + "ui:flavour": "tabs",
133 + "ui:options": {
134 + "tabs": [
135 + {
136 + "title": "Base",
137 + "fields": [
138 + "update_every",
139 + "url",
140 + "timeout",
141 + "not_follow_redirects"
142 + ]
143 + },
144 + {
145 + "title": "Auth",
146 + "fields": [
147 + "username",
148 + "password"
149 + ]
150 + },
151 + {
152 + "title": "TLS",
153 + "fields": [
154 + "tls_skip_verify",
155 + "tls_ca",
156 + "tls_cert",
157 + "tls_key"
158 + ]
159 + },
160 + {
161 + "title": "Proxy",
162 + "fields": [
163 + "proxy_url",
164 + "proxy_username",
165 + "proxy_password"
166 + ]
167 + },
168 + {
169 + "title": "Headers",
170 + "fields": [
171 + "headers"
172 + ]
173 + }
174 + ]
175 + }
176 + }
177 +}
src/go/collectors/go.d.plugin/modules/rspamd/metadata.yaml new
+221
@@ -0,0 +1,221 @@
1 +plugin_name: go.d.plugin
2 +modules:
3 + - meta:
4 + id: collector-go.d.plugin-rspamd
5 + plugin_name: go.d.plugin
6 + module_name: rspamd
7 + monitored_instance:
8 + name: Rspamd
9 + link: https://rspamd.com/
10 + categories:
11 + - data-collection.security-systems
12 + icon_filename: globe.svg
13 + related_resources:
14 + integrations:
15 + list:
16 + - plugin_name: go.d.plugin
17 + module_name: httpcheck
18 + - plugin_name: apps.plugin
19 + module_name: apps
20 + alternative_monitored_instances: []
21 + info_provided_to_referring_integrations:
22 + description: ""
23 + keywords:
24 + - spam
25 + - rspamd
26 + - email
27 + most_popular: false
28 + overview:
29 + data_collection:
30 + metrics_description: |
31 + This collector monitors the activity and performance of Rspamd servers. It gathers various metrics including scanned emails, learned messages, spam/ham counts, and actions taken on emails (reject, rewrite, etc.).
32 + method_description: |
33 + It retrieves statistics from Rspamd's [built-in web server](https://rspamd.com/doc/workers/controller.html) by making HTTP requests to the `/stat` endpoint.
34 + default_behavior:
35 + auto_detection:
36 + description: |
37 + By default, it detects Rspamd instances running on localhost that are listening on port 11334.
38 + limits:
39 + description: ""
40 + performance_impact:
41 + description: ""
42 + additional_permissions:
43 + description: ""
44 + multi_instance: true
45 + supported_platforms:
46 + include: []
47 + exclude: []
48 + setup:
49 + prerequisites:
50 + list: []
51 + configuration:
52 + file:
53 + name: go.d/rspamd.conf
54 + options:
55 + description: |
56 + The following options can be defined globally: update_every, autodetection_retry.
57 + folding:
58 + title: Config options
59 + enabled: true
60 + list:
61 + - name: update_every
62 + description: Data collection frequency.
63 + default_value: 1
64 + required: false
65 + - name: autodetection_retry
66 + description: Recheck interval in seconds. Zero means no recheck will be scheduled.
67 + default_value: 0
68 + required: false
69 + - name: url
70 + description: Server URL.
71 + default_value: http://127.0.0.1:11334
72 + required: true
73 + - name: timeout
74 + description: HTTP request timeout.
75 + default_value: 1
76 + required: false
77 + - name: username
78 + description: Username for basic HTTP authentication.
79 + default_value: ""
80 + required: false
81 + - name: password
82 + description: Password for basic HTTP authentication.
83 + default_value: ""
84 + required: false
85 + - name: proxy_url
86 + description: Proxy URL.
87 + default_value: ""
88 + required: false
89 + - name: proxy_username
90 + description: Username for proxy basic HTTP authentication.
91 + default_value: ""
92 + required: false
93 + - name: proxy_password
94 + description: Password for proxy basic HTTP authentication.
95 + default_value: ""
96 + required: false
97 + - name: method
98 + description: HTTP request method.
99 + default_value: GET
100 + required: false
101 + - name: body
102 + description: HTTP request body.
103 + default_value: ""
104 + required: false
105 + - name: headers
106 + description: HTTP request headers.
107 + default_value: ""
108 + required: false
109 + - name: not_follow_redirects
110 + description: Redirect handling policy. Controls whether the client follows redirects.
111 + default_value: false
112 + required: false
113 + - name: tls_skip_verify
114 + description: Server certificate chain and hostname validation policy. Controls whether the client performs this check.
115 + default_value: false
116 + required: false
117 + - name: tls_ca
118 + description: Certification authority that the client uses when verifying the server's certificates.
119 + default_value: ""
120 + required: false
121 + - name: tls_cert
122 + description: Client TLS certificate.
123 + default_value: ""
124 + required: false
125 + - name: tls_key
126 + description: Client TLS key.
127 + default_value: ""
128 + required: false
129 + examples:
130 + folding:
131 + title: Config
132 + enabled: true
133 + list:
134 + - name: Basic
135 + description: A basic example configuration.
136 + folding:
137 + enabled: false
138 + config: |
139 + jobs:
140 + - name: local
141 + url: http://127.0.0.1:11334
142 + - name: HTTP authentication
143 + description: Basic HTTP authentication.
144 + config: |
145 + jobs:
146 + - name: local
147 + url: http://127.0.0.1:11334
148 + username: username
149 + password: password
150 + - name: Multi-instance
151 + description: |
152 + > **Note**: When you define multiple jobs, their names must be unique.
153 +
154 + Collecting metrics from local and remote instances.
155 + config: |
156 + jobs:
157 + - name: local
158 + url: http://127.0.0.1:11334
159 +
160 + - name: remote
161 + url: http://192.0.2.1:11334
162 + troubleshooting:
163 + problems:
164 + list: []
165 + alerts: []
166 + metrics:
167 + folding:
168 + title: Metrics
169 + enabled: false
170 + description: ""
171 + availability: []
172 + scopes:
173 + - name: global
174 + description: These metrics refer to the entire monitored application.
175 + labels: []
176 + metrics:
177 + - name: rspamd.classifications
178 + description: Classifications
179 + unit: messages/s
180 + chart_type: stacked
181 + dimensions:
182 + - name: ham
183 + - name: spam
184 + - name: rspamd.actions
185 + description: Actions
186 + unit: messages/s
187 + chart_type: stacked
188 + dimensions:
189 + - name: reject
190 + - name: soft_reject
191 + - name: rewrite_subject
192 + - name: add_header
193 + - name: greylist
194 + - name: custom
195 + - name: discard
196 + - name: quarantine
197 + - name: no_action
198 + - name: rspamd.scans
199 + description: Scanned messages
200 + unit: messages/s
201 + chart_type: line
202 + dimensions:
203 + - name: scanned
204 + - name: rspamd.learns
205 + description: Learned messages
206 + unit: messages/s
207 + chart_type: line
208 + dimensions:
209 + - name: learned
210 + - name: rspamd.connections
211 + description: Connections
212 + unit: connections/s
213 + chart_type: line
214 + dimensions:
215 + - name: connections
216 + - name: rspamd.control_connections
217 + description: Control connections
218 + unit: connections/s
219 + chart_type: line
220 + dimensions:
221 + - name: control_connections
src/go/collectors/go.d.plugin/modules/rspamd/rspamd.go new
+113
@@ -0,0 +1,113 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package rspamd
4 +
5 +import (
6 + _ "embed"
7 + "errors"
8 + "net/http"
9 + "time"
10 +
11 + "github.com/netdata/netdata/go/go.d.plugin/agent/module"
12 + "github.com/netdata/netdata/go/go.d.plugin/pkg/web"
13 +)
14 +
15 +//go:embed "config_schema.json"
16 +var configSchema string
17 +
18 +func init() {
19 + module.Register("rspamd", module.Creator{
20 + JobConfigSchema: configSchema,
21 + Create: func() module.Module { return New() },
22 + })
23 +}
24 +
25 +func New() *Rspamd {
26 + return &Rspamd{
27 + Config: Config{
28 + HTTP: web.HTTP{
29 + Request: web.Request{
30 + URL: "http://127.0.0.1:11334",
31 + },
32 + Client: web.Client{
33 + Timeout: web.Duration(time.Second * 1),
34 + },
35 + },
36 + },
37 + charts: charts.Copy(),
38 + }
39 +}
40 +
41 +type Config struct {
42 + web.HTTP `yaml:",inline" json:""`
43 + UpdateEvery int `yaml:"update_every" json:"update_every"`
44 +}
45 +
46 +type Rspamd struct {
47 + module.Base
48 + Config `yaml:",inline" json:""`
49 +
50 + charts *module.Charts
51 +
52 + httpClient *http.Client
53 +}
54 +
55 +func (r *Rspamd) Configuration() any {
56 + return r.Config
57 +}
58 +
59 +func (r *Rspamd) Init() error {
60 + if r.URL == "" {
61 + r.Error("URL not set")
62 + return errors.New("url not set")
63 + }
64 +
65 + client, err := web.NewHTTPClient(r.Client)
66 + if err != nil {
67 + r.Error(err)
68 + return err
69 + }
70 + r.httpClient = client
71 +
72 + r.Debugf("using URL %s", r.URL)
73 + r.Debugf("using timeout: %s", r.Timeout)
74 +
75 + return nil
76 +}
77 +
78 +func (r *Rspamd) Check() error {
79 + mx, err := r.collect()
80 + if err != nil {
81 + r.Error(err)
82 + return err
83 + }
84 +
85 + if len(mx) == 0 {
86 + return errors.New("no metrics collected")
87 + }
88 +
89 + return nil
90 +}
91 +
92 +func (r *Rspamd) Charts() *module.Charts {
93 + return r.charts
94 +}
95 +
96 +func (r *Rspamd) Collect() map[string]int64 {
97 + mx, err := r.collect()
98 + if err != nil {
99 + r.Error(err)
100 + }
101 +
102 + if len(mx) == 0 {
103 + return nil
104 + }
105 +
106 + return mx
107 +}
108 +
109 +func (r *Rspamd) Cleanup() {
110 + if r.httpClient != nil {
111 + r.httpClient.CloseIdleConnections()
112 + }
113 +}
src/go/collectors/go.d.plugin/modules/rspamd/rspamd_test.go new
+258
@@ -0,0 +1,258 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package rspamd
4 +
5 +import (
6 + "net/http"
7 + "net/http/httptest"
8 + "os"
9 + "testing"
10 +
11 + "github.com/netdata/netdata/go/go.d.plugin/agent/module"
12 + "github.com/netdata/netdata/go/go.d.plugin/pkg/web"
13 +
14 + "github.com/stretchr/testify/assert"
15 + "github.com/stretchr/testify/require"
16 +)
17 +
18 +var (
19 + dataConfigJSON, _ = os.ReadFile("testdata/config.json")
20 + dataConfigYAML, _ = os.ReadFile("testdata/config.yaml")
21 +
22 + dataV34Stat, _ = os.ReadFile("testdata/v3.4-stat.json")
23 +)
24 +
25 +func Test_testDataIsValid(t *testing.T) {
26 + for name, data := range map[string][]byte{
27 + "dataConfigJSON": dataConfigJSON,
28 + "dataConfigYAML": dataConfigYAML,
29 + "dataV34Stat": dataV34Stat,
30 + } {
31 + require.NotNil(t, data, name)
32 + }
33 +}
34 +
35 +func TestRspamd_ConfigurationSerialize(t *testing.T) {
36 + module.TestConfigurationSerialize(t, &Rspamd{}, dataConfigJSON, dataConfigYAML)
37 +}
38 +
39 +func TestRspamd_Init(t *testing.T) {
40 + tests := map[string]struct {
41 + wantFail bool
42 + config Config
43 + }{
44 + "success with default": {
45 + wantFail: false,
46 + config: New().Config,
47 + },
48 + "fail when URL not set": {
49 + wantFail: true,
50 + config: Config{
51 + HTTP: web.HTTP{
52 + Request: web.Request{URL: ""},
53 + },
54 + },
55 + },
56 + }
57 +
58 + for name, test := range tests {
59 + t.Run(name, func(t *testing.T) {
60 + rsp := New()
61 + rsp.Config = test.config
62 +
63 + if test.wantFail {
64 + assert.Error(t, rsp.Init())
65 + } else {
66 + assert.NoError(t, rsp.Init())
67 + }
68 + })
69 + }
70 +}
71 +
72 +func TestRspamd_Charts(t *testing.T) {
73 + assert.NotNil(t, New().Charts())
74 +}
75 +
76 +func TestRspamd_Check(t *testing.T) {
77 + tests := map[string]struct {
78 + wantFail bool
79 + prepare func(t *testing.T) (*Rspamd, func())
80 + }{
81 + "success on valid response": {
82 + wantFail: false,
83 + prepare: prepareCaseOk,
84 + },
85 + "fails on unexpected json response": {
86 + wantFail: true,
87 + prepare: prepareCaseUnexpectedJsonResponse,
88 + },
89 + "fails on invalid format response": {
90 + wantFail: true,
91 + prepare: prepareCaseInvalidFormatResponse,
92 + },
93 + "fails on connection refused": {
94 + wantFail: true,
95 + prepare: prepareCaseConnectionRefused,
96 + },
97 + }
98 +
99 + for name, test := range tests {
100 + t.Run(name, func(t *testing.T) {
101 + rsp, cleanup := test.prepare(t)
102 + defer cleanup()
103 +
104 + if test.wantFail {
105 + assert.Error(t, rsp.Check())
106 + } else {
107 + assert.NoError(t, rsp.Check())
108 + }
109 + })
110 + }
111 +}
112 +
113 +func TestRspamd_Collect(t *testing.T) {
114 + tests := map[string]struct {
115 + prepare func(t *testing.T) (*Rspamd, func())
116 + wantMetrics map[string]int64
117 + }{
118 + "success on valid response": {
119 + prepare: prepareCaseOk,
120 + wantMetrics: map[string]int64{
121 + "actions_add_header": 1,
122 + "actions_custom": 0,
123 + "actions_discard": 0,
124 + "actions_greylist": 1,
125 + "actions_invalid_max_action": 0,
126 + "actions_no_action": 1,
127 + "actions_quarantine": 0,
128 + "actions_reject": 1,
129 + "actions_rewrite_subject": 1,
130 + "actions_soft_reject": 1,
131 + "actions_unknown_action": 0,
132 + "connections": 1,
133 + "control_connections": 117,
134 + "ham_count": 1,
135 + "learned": 1,
136 + "scanned": 1,
137 + "spam_count": 1,
138 + },
139 + },
140 + "fails on unexpected json response": {
141 + prepare: prepareCaseUnexpectedJsonResponse,
142 + },
143 + "fails on invalid format response": {
144 + prepare: prepareCaseInvalidFormatResponse,
145 + },
146 + "fails on connection refused": {
147 + prepare: prepareCaseConnectionRefused,
148 + },
149 + }
150 +
151 + for name, test := range tests {
152 + t.Run(name, func(t *testing.T) {
153 + rsp, cleanup := test.prepare(t)
154 + defer cleanup()
155 +
156 + mx := rsp.Collect()
157 +
158 + require.Equal(t, test.wantMetrics, mx)
159 + if len(test.wantMetrics) > 0 {
160 + testMetricsHasAllChartsDims(t, rsp, mx)
161 + }
162 + })
163 + }
164 +}
165 +
166 +func testMetricsHasAllChartsDims(t *testing.T, rsp *Rspamd, mx map[string]int64) {
167 + for _, chart := range *rsp.Charts() {
168 + if chart.Obsolete {
169 + continue
170 + }
171 + for _, dim := range chart.Dims {
172 + _, ok := mx[dim.ID]
173 + assert.Truef(t, ok, "collected metrics has no data for dim '%s' chart '%s'", dim.ID, chart.ID)
174 + }
175 + for _, v := range chart.Vars {
176 + _, ok := mx[v.ID]
177 + assert.Truef(t, ok, "collected metrics has no data for var '%s' chart '%s'", v.ID, chart.ID)
178 + }
179 + }
180 +}
181 +
182 +func prepareCaseOk(t *testing.T) (*Rspamd, func()) {
183 + t.Helper()
184 + srv := httptest.NewServer(http.HandlerFunc(
185 + func(w http.ResponseWriter, r *http.Request) {
186 + switch r.URL.Path {
187 + case "/stat":
188 + _, _ = w.Write(dataV34Stat)
189 + default:
190 + w.WriteHeader(http.StatusNotFound)
191 + }
192 + }))
193 +
194 + rsp := New()
195 + rsp.URL = srv.URL
196 + require.NoError(t, rsp.Init())
197 +
198 + return rsp, srv.Close
199 +}
200 +
201 +func prepareCaseUnexpectedJsonResponse(t *testing.T) (*Rspamd, func()) {
202 + t.Helper()
203 + resp := `
204 +{
205 + "elephant": {
206 + "burn": false,
207 + "mountain": true,
208 + "fog": false,
209 + "skin": -1561907625,
210 + "burst": "anyway",
211 + "shadow": 1558616893
212 + },
213 + "start": "ever",
214 + "base": 2093056027,
215 + "mission": -2007590351,
216 + "victory": 999053756,
217 + "die": false
218 +}
219 +`
220 + srv := httptest.NewServer(http.HandlerFunc(
221 + func(w http.ResponseWriter, r *http.Request) {
222 + switch r.URL.Path {
223 + case "/stat":
224 + _, _ = w.Write([]byte(resp))
225 + default:
226 + w.WriteHeader(http.StatusNotFound)
227 + }
228 + }))
229 +
230 + rsp := New()
231 + rsp.URL = srv.URL
232 + require.NoError(t, rsp.Init())
233 +
234 + return rsp, srv.Close
235 +}
236 +
237 +func prepareCaseInvalidFormatResponse(t *testing.T) (*Rspamd, func()) {
238 + t.Helper()
239 + srv := httptest.NewServer(http.HandlerFunc(
240 + func(w http.ResponseWriter, r *http.Request) {
241 + _, _ = w.Write([]byte("hello and\n goodbye"))
242 + }))
243 +
244 + rsp := New()
245 + rsp.URL = srv.URL
246 + require.NoError(t, rsp.Init())
247 +
248 + return rsp, srv.Close
249 +}
250 +
251 +func prepareCaseConnectionRefused(t *testing.T) (*Rspamd, func()) {
252 + t.Helper()
253 + rsp := New()
254 + rsp.URL = "http://127.0.0.1:65001/stat"
255 + require.NoError(t, rsp.Init())
256 +
257 + return rsp, func() {}
258 +}
src/go/collectors/go.d.plugin/modules/rspamd/testdata/config.json new
+20
@@ -0,0 +1,20 @@
1 +{
2 + "update_every": 123,
3 + "url": "ok",
4 + "body": "ok",
5 + "method": "ok",
6 + "headers": {
7 + "ok": "ok"
8 + },
9 + "username": "ok",
10 + "password": "ok",
11 + "proxy_url": "ok",
12 + "proxy_username": "ok",
13 + "proxy_password": "ok",
14 + "timeout": 123.123,
15 + "not_follow_redirects": true,
16 + "tls_ca": "ok",
17 + "tls_cert": "ok",
18 + "tls_key": "ok",
19 + "tls_skip_verify": true
20 +}
src/go/collectors/go.d.plugin/modules/rspamd/testdata/config.yaml new
+17
@@ -0,0 +1,17 @@
1 +update_every: 123
2 +url: "ok"
3 +body: "ok"
4 +method: "ok"
5 +headers:
6 + ok: "ok"
7 +username: "ok"
8 +password: "ok"
9 +proxy_url: "ok"
10 +proxy_username: "ok"
11 +proxy_password: "ok"
12 +timeout: 123.123
13 +not_follow_redirects: yes
14 +tls_ca: "ok"
15 +tls_cert: "ok"
16 +tls_key: "ok"
17 +tls_skip_verify: yes
src/go/collectors/go.d.plugin/modules/rspamd/testdata/v3.4-stat.json new
+66
@@ -0,0 +1,66 @@
1 +{
2 + "version": "3.4",
3 + "config_id": "gkwm3ysiqrx96kj1mwnfashx9hkypj833w1tgjaw4nysgwwxqthh7q78hyrezi9gzamke3n9ea7u8cjrzru7i5p4z7r9xhcoitjpjyy",
4 + "uptime": 1774,
5 + "read_only": false,
6 + "scanned": 1,
7 + "learned": 1,
8 + "actions": {
9 + "reject": 1,
10 + "soft reject": 1,
11 + "rewrite subject": 1,
12 + "add header": 1,
13 + "greylist": 1,
14 + "no action": 1
15 + },
16 + "scan_times": [
17 + null,
18 + null,
19 + null,
20 + null,
21 + null,
22 + null,
23 + null,
24 + null,
25 + null,
26 + null,
27 + null,
28 + null,
29 + null,
30 + null,
31 + null,
32 + null,
33 + null,
34 + null,
35 + null,
36 + null,
37 + null,
38 + null,
39 + null,
40 + null,
41 + null,
42 + null,
43 + null,
44 + null,
45 + null,
46 + null,
47 + null
48 + ],
49 + "spam_count": 1,
50 + "ham_count": 1,
51 + "connections": 1,
52 + "control_connections": 117,
53 + "pools_allocated": 184,
54 + "pools_freed": 147,
55 + "bytes_allocated": 28807460,
56 + "chunks_allocated": 282,
57 + "shared_chunks_allocated": 4,
58 + "chunks_freed": 0,
59 + "chunks_oversized": 2,
60 + "fragmented": 0,
61 + "total_learns": 0,
62 + "statfiles": [],
63 + "fuzzy_hashes": {
64 + "rspamd.com": 446607461
65 + }
66 +}