Provide runtime ml info from a new endpoint. (#11886)
* Provide runtime ml info from a new endpoint. * Add hosts & charts to skip from ML in the /info endpoint. This information belongs in /info, and not in /ml-info, because the value of these variables can not change at the agent's runtime. * Use strdupz instead of strdup.
vkalintiris committed
Dec 22, 2021 at 17:13 UTC
5049de8f7a132205b32f2c523998f93cbbfc0481
6 files changed
+54
-12
ml/Config.cc
+7
-7
@@ -47,12 +47,6 @@ void Config::readMLConfig(void) {
47
double ADWindowRateThreshold = config_get_float(ConfigSectionML, "window minimum anomaly rate", 0.25);
48
double ADDimensionRateThreshold = config_get_float(ConfigSectionML, "anomaly event min dimension rate threshold", 0.05);
49
50
- std::string HostsToSkip = config_get(ConfigSectionML, "hosts to skip from training", "!*");
51
- std::string ChartsToSkip = config_get(ConfigSectionML, "charts to skip from training",
52
- "!system.* !cpu.* !mem.* !disk.* !disk_* "
53
- "!ip.* !ipv4.* !ipv6.* !net.* !net_* !netfilter.* "
54
- "!services.* !apps.* !groups.* !user.* !ebpf.* !netdata.* *");
55
-
50
std::stringstream SS;
51
SS << netdata_configured_cache_dir << "/anomaly-detection.db";
52
Cfg.AnomalyDBPath = SS.str();
@@ -123,6 +117,12 @@ void Config::readMLConfig(void) {
117
Cfg.ADWindowRateThreshold = ADWindowRateThreshold;
118
Cfg.ADDimensionRateThreshold = ADDimensionRateThreshold;
119
126
- Cfg.SP_HostsToSkip = simple_pattern_create(HostsToSkip.c_str(), NULL, SIMPLE_PATTERN_EXACT);
120
+ Cfg.HostsToSkip = config_get(ConfigSectionML, "hosts to skip from training", "!*");
121
+ Cfg.SP_HostsToSkip = simple_pattern_create(Cfg.HostsToSkip.c_str(), NULL, SIMPLE_PATTERN_EXACT);
122
+
123
+ Cfg.ChartsToSkip = config_get(ConfigSectionML, "charts to skip from training",
124
+ "!system.* !cpu.* !mem.* !disk.* !disk_* "
125
+ "!ip.* !ipv4.* !ipv6.* !net.* !net_* !netfilter.* "
126
+ "!services.* !apps.* !groups.* !user.* !ebpf.* !netdata.* *");
127
Cfg.SP_ChartsToSkip = simple_pattern_create(ChartsToSkip.c_str(), NULL, SIMPLE_PATTERN_EXACT);
128
}
ml/Config.h
+3
@@ -30,7 +30,10 @@ public:
30
double ADWindowRateThreshold;
31
double ADDimensionRateThreshold;
32
33
+ std::string HostsToSkip;
34
SIMPLE_PATTERN *SP_HostsToSkip;
35
+
36
+ std::string ChartsToSkip;
37
SIMPLE_PATTERN *SP_ChartsToSkip;
38
39
std::string AnomalyDBPath;
ml/Host.cc
+4
@@ -282,6 +282,9 @@ void RrdHost::getConfigAsJson(nlohmann::json &Json) const {
282
Json["idle-window-size"] = Cfg.ADIdleWindowSize;
283
Json["window-rate-threshold"] = Cfg.ADWindowRateThreshold;
284
Json["dimension-rate-threshold"] = Cfg.ADDimensionRateThreshold;
285
+
286
+ Json["hosts-to-skip"] = Cfg.HostsToSkip;
287
+ Json["charts-to-skip"] = Cfg.ChartsToSkip;
288
}
289
290
std::pair<Dimension *, Duration<double>>
@@ -441,6 +444,7 @@ void DetectableHost::detect() {
444
}
445
446
void DetectableHost::getDetectionInfoAsJson(nlohmann::json &Json) const {
447
+ Json["version"] = 1;
448
Json["anomalous-dimensions"] = NumAnomalousDimensions;
449
Json["normal-dimensions"] = NumNormalDimensions;
450
Json["total-dimensions"] = NumAnomalousDimensions + NumNormalDimensions;
ml/ml.cc
+13
-1
@@ -75,7 +75,6 @@ char *ml_get_host_info(RRDHOST *RH) {
75
if (RH && RH->ml_host) {
76
Host *H = static_cast<Host *>(RH->ml_host);
77
H->getConfigAsJson(ConfigJson);
78
- H->getDetectionInfoAsJson(ConfigJson);
78
} else {
79
ConfigJson["enabled"] = false;
80
}
@@ -83,6 +82,19 @@ char *ml_get_host_info(RRDHOST *RH) {
82
return strdup(ConfigJson.dump(2, '\t').c_str());
83
}
84
85
+char *ml_get_host_runtime_info(RRDHOST *RH) {
86
+ nlohmann::json ConfigJson;
87
+
88
+ if (RH && RH->ml_host) {
89
+ Host *H = static_cast<Host *>(RH->ml_host);
90
+ H->getDetectionInfoAsJson(ConfigJson);
91
+ } else {
92
+ return nullptr;
93
+ }
94
+
95
+ return strdup(ConfigJson.dump(1, '\t').c_str());
96
+}
97
+
98
bool ml_is_anomalous(RRDDIM *RD, double Value, bool Exists) {
99
Dimension *D = static_cast<Dimension *>(RD->state->ml_dimension);
100
if (!D)
ml/ml.h
+1
@@ -18,6 +18,7 @@ void ml_new_host(RRDHOST *RH);
18
void ml_delete_host(RRDHOST *RH);
19
20
char *ml_get_host_info(RRDHOST *RH);
21
+char *ml_get_host_runtime_info(RRDHOST *RH);
22
23
void ml_new_dimension(RRDDIM *RD);
24
void ml_delete_dimension(RRDDIM *RD);
web/api/web_api_v1.c
+26
-4
@@ -1130,11 +1130,11 @@ int web_client_api_request_v1_anomaly_events(RRDHOST *host, struct web_client *w
1130
1131
char *s;
1132
if (!before || !after)
1133
- s = strdup("{\"error\": \"missing after/before parameters\" }\n");
1133
+ s = strdupz("{\"error\": \"missing after/before parameters\" }\n");
1134
else {
1135
s = ml_get_anomaly_events(host, "AD1", 1, after, before);
1136
if (!s)
1137
- s = strdup("{\"error\": \"json string is empty\" }\n");
1137
+ s = strdupz("{\"error\": \"json string is empty\" }\n");
1138
}
1139
1140
BUFFER *wb = w->response.data;
@@ -1174,11 +1174,11 @@ int web_client_api_request_v1_anomaly_event_info(RRDHOST *host, struct web_clien
1174
1175
char *s;
1176
if (!before || !after)
1177
- s = strdup("{\"error\": \"missing after/before parameters\" }\n");
1177
+ s = strdupz("{\"error\": \"missing after/before parameters\" }\n");
1178
else {
1179
s = ml_get_anomaly_event_info(host, "AD1", 1, after, before);
1180
if (!s)
1181
- s = strdup("{\"error\": \"json string is empty\" }\n");
1181
+ s = strdupz("{\"error\": \"json string is empty\" }\n");
1182
}
1183
1184
BUFFER *wb = w->response.data;
@@ -1190,6 +1190,27 @@ int web_client_api_request_v1_anomaly_event_info(RRDHOST *host, struct web_clien
1190
freez(s);
1191
return HTTP_RESP_OK;
1192
}
1193
+
1194
+int web_client_api_request_v1_ml_info(RRDHOST *host, struct web_client *w, char *url) {
1195
+ (void) url;
1196
+
1197
+ if (!netdata_ready)
1198
+ return HTTP_RESP_BACKEND_FETCH_FAILED;
1199
+
1200
+ char *s = ml_get_host_runtime_info(host);
1201
+ if (!s)
1202
+ s = strdupz("{\"error\": \"json string is empty\" }\n");
1203
+
1204
+ BUFFER *wb = w->response.data;
1205
+ buffer_flush(wb);
1206
+ wb->contenttype = CT_APPLICATION_JSON;
1207
+ buffer_strcat(wb, s);
1208
+ buffer_no_cacheable(wb);
1209
+
1210
+ freez(s);
1211
+ return HTTP_RESP_OK;
1212
+}
1213
+
1214
#endif // defined(ENABLE_ML)
1215
1216
inline int web_client_api_request_v1_info(RRDHOST *host, struct web_client *w, char *url) {
@@ -1250,6 +1271,7 @@ static struct api_command {
1271
#if defined(ENABLE_ML)
1272
{ "anomaly_events", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_anomaly_events },
1273
{ "anomaly_event_info", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_anomaly_event_info },
1274
+ { "ml_info", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_ml_info },
1275
#endif
1276
1277
{ "manage/health", 0, WEB_CLIENT_ACL_MGMT, web_client_api_request_v1_mgmt_health },