@cryptotaxi247 / netdata-1 / commits / 934f62026

facets: data-only queries (#15961)

Costa Tsaousis committed Sep 14, 2023 at 15:59 UTC 934f620265d2e7a2be160cb084ec01875ed3491e

7 files changed +571 -212

collectors/apps.plugin/apps_plugin.c

+33 -21

@@ -4575,7 +4575,7 @@ static int check_capabilities() {

4575 }

4576 #endif

4577

4578 -static netdata_mutex_t mutex = NETDATA_MUTEX_INITIALIZER;

4578 +static netdata_mutex_t stdout_mutex = NETDATA_MUTEX_INITIALIZER;

4579

4580 #define PROCESS_FILTER_CATEGORY "category:"

4581 #define PROCESS_FILTER_USER "user:"

@@ -4629,8 +4629,8 @@ static void get_MemTotal(void) {

4629 }

4630

4631 static void apps_plugin_function_processes_help(const char *transaction) {

4632 - pluginsd_function_result_begin_to_stdout(transaction, HTTP_RESP_OK, "text/plain", now_realtime_sec() + 3600);

4633 - fprintf(stdout, "%s",

4632 + BUFFER *wb = buffer_create(0, NULL);

4633 + buffer_sprintf(wb, "%s",

4634 "apps.plugin / processes\n"

4635 "\n"

4636 "Function `processes` presents all the currently running processes of the system.\n"

@@ -4660,7 +4660,12 @@ static void apps_plugin_function_processes_help(const char *transaction) {

4660 "\n"

4661 "Filters can be combined. Each filter can be given only one time.\n"

4662 );

4663 - pluginsd_function_result_end_to_stdout();

4663 +

4664 + netdata_mutex_lock(&stdout_mutex);

4665 + pluginsd_function_result_to_stdout(transaction, HTTP_RESP_OK, "text/plain", now_realtime_sec() + 3600, wb);

4666 + netdata_mutex_unlock(&stdout_mutex);

4667 +

4668 + buffer_free(wb);

4669 }

4670

4671 #define add_value_field_llu_with_max(wb, key, value) do { \

@@ -4696,24 +4701,31 @@ static void function_processes(const char *transaction, char *function __maybe_u

4701 if(!category && strncmp(keyword, PROCESS_FILTER_CATEGORY, strlen(PROCESS_FILTER_CATEGORY)) == 0) {

4702 category = find_target_by_name(apps_groups_root_target, &keyword[strlen(PROCESS_FILTER_CATEGORY)]);

4703 if(!category) {

4704 + netdata_mutex_lock(&stdout_mutex);

4705 pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_BAD_REQUEST,

4706 "No category with that name found.");

4707 + netdata_mutex_unlock(&stdout_mutex);

4708 +

4709 return;

4710 }

4711 }

4712 else if(!user && strncmp(keyword, PROCESS_FILTER_USER, strlen(PROCESS_FILTER_USER)) == 0) {

4713 user = find_target_by_name(users_root_target, &keyword[strlen(PROCESS_FILTER_USER)]);

4714 if(!user) {

4715 + netdata_mutex_lock(&stdout_mutex);

4716 pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_BAD_REQUEST,

4717 "No user with that name found.");

4718 + netdata_mutex_unlock(&stdout_mutex);

4719 return;

4720 }

4721 }

4722 else if(strncmp(keyword, PROCESS_FILTER_GROUP, strlen(PROCESS_FILTER_GROUP)) == 0) {

4723 group = find_target_by_name(groups_root_target, &keyword[strlen(PROCESS_FILTER_GROUP)]);

4724 if(!group) {

4725 + netdata_mutex_lock(&stdout_mutex);

4726 pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_BAD_REQUEST,

4727 "No group with that name found.");

4728 + netdata_mutex_unlock(&stdout_mutex);

4729 return;

4730 }

4731 }

@@ -4739,13 +4751,14 @@ static void function_processes(const char *transaction, char *function __maybe_u

4751 else {

4752 char msg[PLUGINSD_LINE_MAX];

4753 snprintfz(msg, PLUGINSD_LINE_MAX, "Invalid parameter '%s'", keyword);

4754 + netdata_mutex_lock(&stdout_mutex);

4755 pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_BAD_REQUEST, msg);

4756 + netdata_mutex_unlock(&stdout_mutex);

4757 return;

4758 }

4759 }

4760

4761 time_t expires = now_realtime_sec() + update_every;

4748 - pluginsd_function_result_begin_to_stdout(transaction, HTTP_RESP_OK, "application/json", expires);

4762

4763 unsigned int cpu_divisor = time_factor * RATES_DETAIL / 100;

4764 unsigned int memory_divisor = 1024;

@@ -5523,10 +5536,11 @@ static void function_processes(const char *transaction, char *function __maybe_u

5536 buffer_json_member_add_time_t(wb, "expires", expires);

5537 buffer_json_finalize(wb);

5538

5526 - fwrite(buffer_tostring(wb), buffer_strlen(wb), 1, stdout);

5527 - buffer_free(wb);

5539 + netdata_mutex_lock(&stdout_mutex);

5540 + pluginsd_function_result_to_stdout(transaction, HTTP_RESP_OK, "application/json", expires, wb);

5541 + netdata_mutex_unlock(&stdout_mutex);

5542

5529 - pluginsd_function_result_end_to_stdout();

5543 + buffer_free(wb);

5544 }

5545

5546 static bool apps_plugin_exit = false;

@@ -5560,16 +5574,14 @@ static void *reader_main(void *arg __maybe_unused) {

5574

5575 // internal_error(true, "Received function '%s', transaction '%s', timeout %d", function, transaction, timeout);

5576

5563 - netdata_mutex_lock(&mutex);

5564 -

5577 if(strncmp(function, "processes", strlen("processes")) == 0)

5578 function_processes(transaction, function, buffer, PLUGINSD_LINE_MAX + 1, timeout);

5567 - else

5579 + else {

5580 + netdata_mutex_lock(&stdout_mutex);

5581 pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_NOT_FOUND,

5582 "No function with this name found in apps.plugin.");

5570 -

5571 - fflush(stdout);

5572 - netdata_mutex_unlock(&mutex);

5583 + netdata_mutex_unlock(&stdout_mutex);

5584 + }

5585

5586 // internal_error(true, "Done with function '%s', transaction '%s', timeout %d", function, transaction, timeout);

5587 }

@@ -5692,7 +5704,7 @@ int main(int argc, char **argv) {

5704

5705 netdata_thread_t reader_thread;

5706 netdata_thread_create(&reader_thread, "APPS_READER", NETDATA_THREAD_OPTION_DONT_LOG, reader_main, NULL);

5695 - netdata_mutex_lock(&mutex);

5707 + netdata_mutex_lock(&stdout_mutex);

5708

5709 APPS_PLUGIN_GLOBAL_FUNCTIONS();

5710

@@ -5701,7 +5713,7 @@ int main(int argc, char **argv) {

5713 heartbeat_t hb;

5714 heartbeat_init(&hb);

5715 for(; !apps_plugin_exit ; global_iterations_counter++) {

5704 - netdata_mutex_unlock(&mutex);

5716 + netdata_mutex_unlock(&stdout_mutex);

5717

5718 #ifdef NETDATA_PROFILING

5719 #warning "compiling for profiling"

@@ -5712,16 +5724,16 @@ int main(int argc, char **argv) {

5724 #else

5725 usec_t dt = heartbeat_next(&hb, step);

5726 #endif

5715 - netdata_mutex_lock(&mutex);

5727 + netdata_mutex_lock(&stdout_mutex);

5728

5729 struct pollfd pollfd = { .fd = fileno(stdout), .events = POLLERR };

5730 if (unlikely(poll(&pollfd, 1, 0) < 0)) {

5719 - netdata_mutex_unlock(&mutex);

5731 + netdata_mutex_unlock(&stdout_mutex);

5732 netdata_thread_cancel(reader_thread);

5733 fatal("Cannot check if a pipe is available");

5734 }

5735 if (unlikely(pollfd.revents & POLLERR)) {

5724 - netdata_mutex_unlock(&mutex);

5736 + netdata_mutex_unlock(&stdout_mutex);

5737 netdata_thread_cancel(reader_thread);

5738 fatal("Received error on read pipe.");

5739 }

@@ -5732,7 +5744,7 @@ int main(int argc, char **argv) {

5744 if(!collect_data_for_all_processes()) {

5745 netdata_log_error("Cannot collect /proc data for running processes. Disabling apps.plugin...");

5746 printf("DISABLE\n");

5735 - netdata_mutex_unlock(&mutex);

5747 + netdata_mutex_unlock(&stdout_mutex);

5748 netdata_thread_cancel(reader_thread);

5749 exit(1);

5750 }

@@ -5769,5 +5781,5 @@ int main(int argc, char **argv) {

5781

5782 debug_log("done Loop No %zu", global_iterations_counter);

5783 }

5772 - netdata_mutex_unlock(&mutex);

5784 + netdata_mutex_unlock(&stdout_mutex);

5785 }

collectors/ebpf.plugin/ebpf_functions.c

+11 -17

@@ -37,9 +37,8 @@ ebpf_module_t *ebpf_functions_select_module(const char *thread_name) {

37 * @param transaction the transaction id that Netdata sent for this function execution

38 */

39 static void ebpf_function_thread_manipulation_help(const char *transaction) {

40 - pthread_mutex_lock(&lock);

41 - pluginsd_function_result_begin_to_stdout(transaction, HTTP_RESP_OK, "text/plain", now_realtime_sec() + 3600);

42 - fprintf(stdout, "%s",

40 + BUFFER *wb = buffer_create(0, NULL);

41 + buffer_sprintf(wb, "%s",

42 "ebpf.plugin / thread\n"

43 "\n"

44 "Function `thread` allows user to control eBPF threads.\n"

@@ -59,9 +58,12 @@ static void ebpf_function_thread_manipulation_help(const char *transaction) {

58 "Filters can be combined. Each filter can be given only one time.\n"

59 "Process thread is not controlled by functions until we finish the creation of functions per thread..\n"

60 );

62 - pluginsd_function_result_end_to_stdout();

63 - fflush(stdout);

61 +

62 + pthread_mutex_lock(&lock);

63 + pluginsd_function_result_to_stdout(transaction, HTTP_RESP_OK, "text/plain", now_realtime_sec() + 3600, wb);

64 pthread_mutex_unlock(&lock);

65 +

66 + buffer_free(wb);

67 }

@@ -79,12 +81,9 @@ static void ebpf_function_thread_manipulation_help(const char *transaction) {

81 * @param msg the error message

82 */

83 static void ebpf_function_error(const char *transaction, int code, const char *msg) {

82 - char buffer[PLUGINSD_LINE_MAX + 1];

83 - json_escape_string(buffer, msg, PLUGINSD_LINE_MAX);

84 -

85 - pluginsd_function_result_begin_to_stdout(transaction, code, "application/json", now_realtime_sec());

86 - fprintf(stdout, "{\"status\":%d,\"error_message\":\"%s\"}", code, buffer);

87 - pluginsd_function_result_end_to_stdout();

84 + pthread_mutex_lock(&lock);

85 + pluginsd_function_json_error_to_stdout(transaction, code, msg);

86 + pthread_mutex_unlock(&lock);

87 }

89 /*****************************************************************

@@ -350,12 +349,7 @@ static void ebpf_function_thread_manipulation(const char *transaction,

349

350 // Lock necessary to avoid race condition

351 pthread_mutex_lock(&lock);

353 - pluginsd_function_result_begin_to_stdout(transaction, HTTP_RESP_OK, "application/json", expires);

354 -

355 - fwrite(buffer_tostring(wb), buffer_strlen(wb), 1, stdout);

356 -

357 - pluginsd_function_result_end_to_stdout();

358 - fflush(stdout);

352 + pluginsd_function_result_to_stdout(transaction, HTTP_RESP_OK, "application/json", expires, wb);

353 pthread_mutex_unlock(&lock);

354

355 buffer_free(wb);

collectors/plugins.d/plugins_d.h

@@ -130,6 +130,14 @@ static inline void pluginsd_function_json_error_to_stdout(const char *transactio

130 pluginsd_function_result_begin_to_stdout(transaction, code, "application/json", now_realtime_sec());

131 fprintf(stdout, "{\"status\":%d,\"error_message\":\"%s\"}", code, buffer);

132 pluginsd_function_result_end_to_stdout();

133 + fflush(stdout);

134 +}

135 +

136 +static inline void pluginsd_function_result_to_stdout(const char *transaction, int code, const char *content_type, time_t expires, BUFFER *result) {

137 + pluginsd_function_result_begin_to_stdout(transaction, code, content_type, expires);

138 + fwrite(buffer_tostring(result), buffer_strlen(result), 1, stdout);

139 + pluginsd_function_result_end_to_stdout();

140 + fflush(stdout);

141 }

142

143 #endif /* NETDATA_PLUGINS_D_H */

collectors/systemd-journal.plugin/systemd-journal.c

+318 -70

@@ -22,6 +22,7 @@

22 #define SYSTEMD_JOURNAL_MAX_PARAMS 100

23 #define SYSTEMD_JOURNAL_DEFAULT_QUERY_DURATION (3 * 3600)

24 #define SYSTEMD_JOURNAL_DEFAULT_ITEMS_PER_QUERY 200

25 +#define SYSTEMD_JOURNAL_EXCESS_ROWS_ALLOWED 50

27 #define JOURNAL_PARAMETER_HELP "help"

28 #define JOURNAL_PARAMETER_AFTER "after"

@@ -33,6 +34,7 @@

34 #define JOURNAL_PARAMETER_HISTOGRAM "histogram"

35 #define JOURNAL_PARAMETER_DIRECTION "direction"

36 #define JOURNAL_PARAMETER_IF_MODIFIED_SINCE "if_modified_since"

37 +#define JOURNAL_PARAMETER_DATA_ONLY "data_only"

38 #define JOURNAL_PARAMETER_SOURCE "source"

39 #define JOURNAL_PARAMETER_INFO "info"

@@ -53,7 +55,7 @@

55 "|IMAGE_NAME" \

56 ""

56 -static netdata_mutex_t mutex = NETDATA_MUTEX_INITIALIZER;

58 +static netdata_mutex_t stdout_mutex = NETDATA_MUTEX_INITIALIZER;

59 static bool plugin_should_exit = false;

61 DICTIONARY *uids = NULL;

@@ -61,7 +63,7 @@ DICTIONARY *gids = NULL;

64 // ----------------------------------------------------------------------------

64 -int systemd_journal_query(BUFFER *wb, FACETS *facets, usec_t after_ut, usec_t before_ut, usec_t if_modified_since, usec_t stop_monotonic_ut) {

66 +static inline sd_journal *netdata_open_systemd_journal(void) {

67 sd_journal *j = NULL;

68 int r;

@@ -82,36 +84,78 @@ int systemd_journal_query(BUFFER *wb, FACETS *facets, usec_t after_ut, usec_t be

85 if (r < 0) {

86 netdata_log_error("SYSTEMD-JOURNAL: Failed to open SystemD Journal, with error %d", r);

85 - return HTTP_RESP_INTERNAL_SERVER_ERROR;

87 + return NULL;

88 }

88 - facets_rows_begin(facets);

90 + return j;

91 +}

92 +

93 +typedef enum {

94 + ND_SD_JOURNAL_FAILED_TO_SEEK,

95 + ND_SD_JOURNAL_TIMED_OUT,

96 + ND_SD_JOURNAL_OK,

97 + ND_SD_JOURNAL_NOT_MODIFIED,

98 +} ND_SD_JOURNAL_STATUS;

99 +

100 +static inline bool netdata_systemd_journal_seek_to(sd_journal *j, usec_t timestamp) {

101 + if(sd_journal_seek_realtime_usec(j, timestamp) < 0) {

102 + netdata_log_error("SYSTEMD-JOURNAL: Failed to seek to %" PRIu64, timestamp);

103 + if(sd_journal_seek_tail(j) < 0) {

104 + netdata_log_error("SYSTEMD-JOURNAL: Failed to seek to journal's tail");

105 + return false;

106 + }

107 + }

108 +

109 + return true;

110 +}

111 +

112 +static inline void netdata_systemd_journal_process_row(sd_journal *j, FACETS *facets) {

113 + const void *data;

114 + size_t length;

115 + SD_JOURNAL_FOREACH_DATA(j, data, length) {

116 + const char *key = data;

117 + const char *equal = strchr(key, '=');

118 + if(unlikely(!equal))

119 + continue;

120 +

121 + const char *value = ++equal;

122 + size_t key_length = value - key; // including '\0'

123 +

124 + char key_copy[key_length];

125 + memcpy(key_copy, key, key_length - 1);

126 + key_copy[key_length - 1] = '\0';

127 +

128 + size_t value_length = length - key_length; // without '\0'

129 + facets_add_key_value_length(facets, key_copy, key_length - 1, value, value_length <= FACET_MAX_VALUE_LENGTH ? value_length : FACET_MAX_VALUE_LENGTH);

130 + }

131 +}

132 +

133 +ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_full(sd_journal *j, BUFFER *wb, FACETS *facets, usec_t after_ut, usec_t before_ut, usec_t if_modified_since, usec_t stop_monotonic_ut, usec_t *last_modified) {

134 + if(!netdata_systemd_journal_seek_to(j, before_ut))

135 + return ND_SD_JOURNAL_FAILED_TO_SEEK;

136

90 - uint64_t first_msg_ut = 0;

137 + size_t errors_no_timestamp = 0;

138 + usec_t first_msg_ut = 0;

139 bool timed_out = false;

140 size_t row_counter = 0;

141

142 // the entries are not guaranteed to be sorted, so we process up to 100 entries beyond

143 // the end of the query to find possibly useful logs for our time-frame

96 - size_t excess_rows_allowed = 100;

144 + size_t excess_rows_allowed = SYSTEMD_JOURNAL_EXCESS_ROWS_ALLOWED;

145

98 - if(sd_journal_seek_realtime_usec(j, before_ut) < 0) {

99 - netdata_log_error("SYSTEMD-JOURNAL: Failed to seek to %" PRIu64, before_ut);

100 - if(sd_journal_seek_tail(j) < 0) {

101 - netdata_log_error("SYSTEMD-JOURNAL: Failed to seek to journal's tail");

102 - goto finalize;

103 - }

104 - }

146 + facets_rows_begin(facets);

147 while (sd_journal_previous(j) > 0) {

148 row_counter++;

149

108 - uint64_t msg_ut;

109 - sd_journal_get_realtime_usec(j, &msg_ut);

150 + usec_t msg_ut;

151 + if(sd_journal_get_realtime_usec(j, &msg_ut) < 0) {

152 + errors_no_timestamp++;

153 + continue;

154 + }

155

156 if(unlikely(!first_msg_ut)) {

157 if(msg_ut == if_modified_since) {

113 - sd_journal_close(j);

114 - return HTTP_RESP_NOT_MODIFIED;

158 + return ND_SD_JOURNAL_NOT_MODIFIED;

159 }

160

161 first_msg_ut = msg_ut;

@@ -127,26 +171,65 @@ int systemd_journal_query(BUFFER *wb, FACETS *facets, usec_t after_ut, usec_t be

171 continue;

172 }

173

130 - const void *data;

131 - size_t length;

132 - SD_JOURNAL_FOREACH_DATA(j, data, length) {

133 - const char *key = data;

134 - const char *equal = strchr(key, '=');

135 - if(unlikely(!equal))

136 - continue;

174 + netdata_systemd_journal_process_row(j, facets);

175 + facets_row_finished(facets, msg_ut);

176 +

177 + if((row_counter % 100) == 0 && now_monotonic_usec() > stop_monotonic_ut) {

178 + timed_out = true;

179 + break;

180 + }

181 + }

182 +

183 + if(errors_no_timestamp)

184 + netdata_log_error("SYSTEMD-JOURNAL: %zu lines did not have timestamps", errors_no_timestamp);

185 +

186 + *last_modified = first_msg_ut;

187

138 - const char *value = ++equal;

139 - size_t key_length = value - key; // including '\0'

188 + if(timed_out)

189 + return ND_SD_JOURNAL_TIMED_OUT;

190

141 - char key_copy[key_length];

142 - memcpy(key_copy, key, key_length - 1);

143 - key_copy[key_length - 1] = '\0';

191 + return ND_SD_JOURNAL_OK;

192 +}

193 +

194 +ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_data_forward(sd_journal *j, BUFFER *wb, FACETS *facets, usec_t after_ut, usec_t before_ut, usec_t anchor, size_t entries, usec_t stop_monotonic_ut) {

195 + if(!netdata_systemd_journal_seek_to(j, anchor))

196 + return ND_SD_JOURNAL_FAILED_TO_SEEK;

197 +

198 + size_t errors_no_timestamp = 0;

199 + bool timed_out = false;

200 + size_t row_counter = 0;

201 + size_t rows_added = 0;

202 +

203 + // the entries are not guaranteed to be sorted, so we process up to 100 entries beyond

204 + // the end of the query to find possibly useful logs for our time-frame

205 + size_t excess_rows_allowed = SYSTEMD_JOURNAL_EXCESS_ROWS_ALLOWED;

206 +

207 + facets_rows_begin(facets);

208 + while (sd_journal_next(j) > 0) {

209 + row_counter++;

210 +

211 + usec_t msg_ut;

212 + if(sd_journal_get_realtime_usec(j, &msg_ut) < 0) {

213 + errors_no_timestamp++;

214 + continue;

215 + }

216 +

217 + if (msg_ut > before_ut || msg_ut <= anchor)

218 + continue;

219 +

220 + if (msg_ut < after_ut) {

221 + if(--excess_rows_allowed == 0)

222 + break;

223

145 - size_t value_length = length - key_length; // without '\0'

146 - facets_add_key_value_length(facets, key_copy, key_length - 1, value, value_length <= FACET_MAX_VALUE_LENGTH ? value_length : FACET_MAX_VALUE_LENGTH);

224 + continue;

225 }

226

227 + if(rows_added > entries && --excess_rows_allowed == 0)

228 + break;

229 +

230 + netdata_systemd_journal_process_row(j, facets);

231 facets_row_finished(facets, msg_ut);

232 + rows_added++;

233

234 if((row_counter % 100) == 0 && now_monotonic_usec() > stop_monotonic_ut) {

235 timed_out = true;

@@ -154,27 +237,147 @@ int systemd_journal_query(BUFFER *wb, FACETS *facets, usec_t after_ut, usec_t be

237 }

238 }

239

157 -finalize:

240 + if(errors_no_timestamp)

241 + netdata_log_error("SYSTEMD-JOURNAL: %zu lines did not have timestamps", errors_no_timestamp);

242 +

243 + if(timed_out)

244 + return ND_SD_JOURNAL_TIMED_OUT;

245 +

246 + return ND_SD_JOURNAL_OK;

247 +}

248 +

249 +ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_data_backward(sd_journal *j, BUFFER *wb, FACETS *facets, usec_t after_ut, usec_t before_ut, usec_t anchor, size_t entries, usec_t stop_monotonic_ut) {

250 + if(!netdata_systemd_journal_seek_to(j, anchor))

251 + return ND_SD_JOURNAL_FAILED_TO_SEEK;

252 +

253 + size_t errors_no_timestamp = 0;

254 + bool timed_out = false;

255 + size_t row_counter = 0;

256 + size_t rows_added = 0;

257 +

258 + // the entries are not guaranteed to be sorted, so we process up to 100 entries beyond

259 + // the end of the query to find possibly useful logs for our time-frame

260 + size_t excess_rows_allowed = SYSTEMD_JOURNAL_EXCESS_ROWS_ALLOWED;

261 +

262 + facets_rows_begin(facets);

263 + while (sd_journal_previous(j) > 0) {

264 + row_counter++;

265 +

266 + usec_t msg_ut;

267 + if(sd_journal_get_realtime_usec(j, &msg_ut) < 0) {

268 + errors_no_timestamp++;

269 + continue;

270 + }

271 +

272 + if (msg_ut > before_ut || msg_ut >= anchor)

273 + continue;

274 +

275 + if (msg_ut < after_ut) {

276 + if(--excess_rows_allowed == 0)

277 + break;

278 +

279 + continue;

280 + }

281 +

282 + if(rows_added > entries && --excess_rows_allowed == 0)

283 + break;

284 +

285 + netdata_systemd_journal_process_row(j, facets);

286 + facets_row_finished(facets, msg_ut);

287 + rows_added++;

288 +

289 + if((row_counter % 100) == 0 && now_monotonic_usec() > stop_monotonic_ut) {

290 + timed_out = true;

291 + break;

292 + }

293 + }

294 +

295 + if(errors_no_timestamp)

296 + netdata_log_error("SYSTEMD-JOURNAL: %zu lines did not have timestamps", errors_no_timestamp);

297 +

298 + if(timed_out)

299 + return ND_SD_JOURNAL_TIMED_OUT;

300 +

301 + return ND_SD_JOURNAL_OK;

302 +}

303 +

304 +bool netdata_systemd_journal_check_if_modified_since(sd_journal *j, usec_t seek_to, usec_t last_modified) {

305 + // return true, if data have been modified since the timestamp

306 +

307 + if(!last_modified || !seek_to)

308 + return false;

309 +

310 + if(!netdata_systemd_journal_seek_to(j, seek_to))

311 + return false;

312 +

313 + usec_t first_msg_ut = 0;

314 + while (sd_journal_previous(j) > 0) {

315 + usec_t msg_ut;

316 + if(sd_journal_get_realtime_usec(j, &msg_ut) < 0)

317 + continue;

318 +

319 + first_msg_ut = msg_ut;

320 + break;

321 + }

322 +

323 + return first_msg_ut != last_modified;

324 +}

325 +

326 +static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets,

327 + usec_t after_ut, usec_t before_ut,

328 + usec_t anchor, FACETS_ANCHOR_DIRECTION direction, size_t entries,

329 + usec_t if_modified_since, bool data_only,

330 + usec_t stop_monotonic_ut) {

331 + sd_journal *j = netdata_open_systemd_journal();

332 + if(!j)

333 + return HTTP_RESP_INTERNAL_SERVER_ERROR;

334 +

335 + usec_t last_modified = 0;

336 +

337 + ND_SD_JOURNAL_STATUS status;

338 +

339 + if(data_only && anchor /* && !netdata_systemd_journal_check_if_modified_since(j, before_ut, if_modified_since) */) {

340 + facets_data_only_mode(facets);

341 +

342 + // we can do a data-only query

343 + if(direction == FACETS_ANCHOR_DIRECTION_FORWARD)

344 + status = netdata_systemd_journal_query_data_forward(j, wb, facets, after_ut, before_ut, anchor, entries, stop_monotonic_ut);

345 + else

346 + status = netdata_systemd_journal_query_data_backward(j, wb, facets, after_ut, before_ut, anchor, entries, stop_monotonic_ut);

347 + }

348 + else {

349 + // we have to do a full query

350 + status = netdata_systemd_journal_query_full(j, wb, facets,

351 + after_ut, before_ut, if_modified_since,

352 + stop_monotonic_ut, &last_modified);

353 + }

354 +

355 sd_journal_close(j);

356

160 - buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);

161 - buffer_json_member_add_boolean(wb, "partial", timed_out);

357 + if(status == ND_SD_JOURNAL_NOT_MODIFIED)

358 + return HTTP_RESP_NOT_MODIFIED;

359 +

360 + buffer_json_member_add_uint64(wb, "status", status == ND_SD_JOURNAL_FAILED_TO_SEEK ? HTTP_RESP_INTERNAL_SERVER_ERROR : HTTP_RESP_OK);

361 + buffer_json_member_add_boolean(wb, "partial", status != ND_SD_JOURNAL_OK);

362 buffer_json_member_add_string(wb, "type", "table");

163 - buffer_json_member_add_time_t(wb, "update_every", 1);

164 - buffer_json_member_add_string(wb, "help", SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION);

165 - buffer_json_member_add_uint64(wb, "last_modified", first_msg_ut);

363 +

364 + if(!data_only) {

365 + buffer_json_member_add_time_t(wb, "update_every", 1);

366 + buffer_json_member_add_string(wb, "help", SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION);

367 + buffer_json_member_add_uint64(wb, "last_modified", last_modified);

368 + }

369

370 facets_report(facets, wb);

371

169 - buffer_json_member_add_time_t(wb, "expires", now_realtime_sec());

372 + buffer_json_member_add_time_t(wb, "expires", now_realtime_sec() + data_only ? 3600 : 0);

373 buffer_json_finalize(wb);

374

172 - return HTTP_RESP_OK;

375 + return status == ND_SD_JOURNAL_FAILED_TO_SEEK ? HTTP_RESP_INTERNAL_SERVER_ERROR : HTTP_RESP_OK;

376 }

377

175 -static void systemd_journal_function_help(const char *transaction) {

176 - pluginsd_function_result_begin_to_stdout(transaction, HTTP_RESP_OK, "text/plain", now_realtime_sec() + 3600);

177 - fprintf(stdout,

378 +static void netdata_systemd_journal_function_help(const char *transaction) {

379 + BUFFER *wb = buffer_create(0, NULL);

380 + buffer_sprintf(wb,

381 "%s / %s\n"

382 "\n"

383 "%s\n"

@@ -212,7 +415,12 @@ static void systemd_journal_function_help(const char *transaction) {

415 , -SYSTEMD_JOURNAL_DEFAULT_QUERY_DURATION

416 , SYSTEMD_JOURNAL_DEFAULT_ITEMS_PER_QUERY

417 );

215 - pluginsd_function_result_end_to_stdout();

418 +

419 + netdata_mutex_lock(&stdout_mutex);

420 + pluginsd_function_result_to_stdout(transaction, HTTP_RESP_OK, "text/plain", now_realtime_sec() + 3600, wb);

421 + netdata_mutex_unlock(&stdout_mutex);

422 +

423 + buffer_free(wb);

424 }

425

426 static const char *syslog_facility_to_name(int facility) {

@@ -255,6 +463,26 @@ static const char *syslog_priority_to_name(int priority) {

463 }

464 }

465

466 +static FACET_ROW_SEVERITY syslog_priority_to_facet_severity(int priority) {

467 + // same to

468 + // https://github.com/systemd/systemd/blob/aab9e4b2b86905a15944a1ac81e471b5b7075932/src/basic/terminal-util.c#L1501

469 + // function get_log_colors()

470 +

471 + if(priority <= LOG_ERR)

472 + return FACET_ROW_SEVERITY_CRITICAL;

473 +

474 + else if (priority <= LOG_WARNING)

475 + return FACET_ROW_SEVERITY_WARNING;

476 +

477 + else if(priority <= LOG_NOTICE)

478 + return FACET_ROW_SEVERITY_NOTICE;

479 +

480 + else if(priority >= LOG_DEBUG)

481 + return FACET_ROW_SEVERITY_DEBUG;

482 +

483 + return FACET_ROW_SEVERITY_NORMAL;

484 +}

485 +

486 static char *uid_to_username(uid_t uid, char *buffer, size_t buffer_size) {

487 struct passwd pw, *result;

488 char tmp[1024 + 1];

@@ -277,7 +505,7 @@ static char *gid_to_groupname(gid_t gid, char* buffer, size_t buffer_size) {

505 return buffer;

506 }

507

280 -static void systemd_journal_transform_syslog_facility(FACETS *facets __maybe_unused, BUFFER *wb, void *data __maybe_unused) {

508 +static void netdata_systemd_journal_transform_syslog_facility(FACETS *facets __maybe_unused, BUFFER *wb, void *data __maybe_unused) {

509 const char *v = buffer_tostring(wb);

510 if(*v && isdigit(*v)) {

511 int facility = str2i(buffer_tostring(wb));

@@ -289,7 +517,7 @@ static void systemd_journal_transform_syslog_facility(FACETS *facets __maybe_unu

517 }

518 }

519

292 -static void systemd_journal_transform_priority(FACETS *facets __maybe_unused, BUFFER *wb, void *data __maybe_unused) {

520 +static void netdata_systemd_journal_transform_priority(FACETS *facets __maybe_unused, BUFFER *wb, void *data __maybe_unused) {

521 const char *v = buffer_tostring(wb);

522 if(*v && isdigit(*v)) {

523 int priority = str2i(buffer_tostring(wb));

@@ -298,10 +526,12 @@ static void systemd_journal_transform_priority(FACETS *facets __maybe_unused, BU

526 buffer_flush(wb);

527 buffer_strcat(wb, name);

528 }

529 +

530 + facets_set_current_row_severity(facets, syslog_priority_to_facet_severity(priority));

531 }

532 }

533

304 -static void systemd_journal_transform_uid(FACETS *facets __maybe_unused, BUFFER *wb, void *data) {

534 +static void netdata_systemd_journal_transform_uid(FACETS *facets __maybe_unused, BUFFER *wb, void *data) {

535 DICTIONARY *cache = data;

536 const char *v = buffer_tostring(wb);

537 if(*v && isdigit(*v)) {

@@ -321,7 +551,7 @@ static void systemd_journal_transform_uid(FACETS *facets __maybe_unused, BUFFER

551 }

552 }

553

324 -static void systemd_journal_transform_gid(FACETS *facets __maybe_unused, BUFFER *wb, void *data) {

554 +static void netdata_systemd_journal_transform_gid(FACETS *facets __maybe_unused, BUFFER *wb, void *data) {

555 DICTIONARY *cache = data;

556 const char *v = buffer_tostring(wb);

557 if(*v && isdigit(*v)) {

@@ -341,7 +571,7 @@ static void systemd_journal_transform_gid(FACETS *facets __maybe_unused, BUFFER

571 }

572 }

573

344 -static void systemd_journal_dynamic_row_id(FACETS *facets __maybe_unused, BUFFER *json_array, FACET_ROW_KEY_VALUE *rkv, FACET_ROW *row, void *data __maybe_unused) {

574 +static void netdata_systemd_journal_dynamic_row_id(FACETS *facets __maybe_unused, BUFFER *json_array, FACET_ROW_KEY_VALUE *rkv, FACET_ROW *row, void *data __maybe_unused) {

575 FACET_ROW_KEY_VALUE *pid_rkv = dictionary_get(row->dict, "_PID");

576 const char *pid = pid_rkv ? buffer_tostring(pid_rkv->wb) : FACET_VALUE_UNSET;

577

@@ -363,6 +593,12 @@ static void systemd_journal_dynamic_row_id(FACETS *facets __maybe_unused, BUFFER

593 buffer_json_add_array_item_string(json_array, buffer_tostring(rkv->wb));

594 }

595

596 +static void netdata_systemd_journal_rich_message(FACETS *facets __maybe_unused, BUFFER *json_array, FACET_ROW_KEY_VALUE *rkv, FACET_ROW *row, void *data __maybe_unused) {

597 + buffer_json_add_array_item_object(json_array);

598 + buffer_json_member_add_string(json_array, "value", buffer_tostring(rkv->wb));

599 + buffer_json_object_close(json_array);

600 +}

601 +

602 static void function_systemd_journal(const char *transaction, char *function, char *line_buffer __maybe_unused, int line_max __maybe_unused, int timeout __maybe_unused) {

603 BUFFER *wb = buffer_create(0, NULL);

604 buffer_flush(wb);

@@ -384,34 +620,41 @@ static void function_systemd_journal(const char *transaction, char *function, ch

620 facets_accepted_param(facets, JOURNAL_PARAMETER_FACETS);

621 facets_accepted_param(facets, JOURNAL_PARAMETER_HISTOGRAM);

622 facets_accepted_param(facets, JOURNAL_PARAMETER_IF_MODIFIED_SINCE);

623 + facets_accepted_param(facets, JOURNAL_PARAMETER_DATA_ONLY);

624

625 // register the fields in the order you want them on the dashboard

626

627 facets_register_dynamic_key_name(facets, "ND_JOURNAL_PROCESS",

628 FACET_KEY_OPTION_NEVER_FACET | FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_FTS,

392 - systemd_journal_dynamic_row_id, NULL);

629 + netdata_systemd_journal_dynamic_row_id, NULL);

630

631 facets_register_key_name(facets, "MESSAGE",

395 - FACET_KEY_OPTION_NEVER_FACET | FACET_KEY_OPTION_MAIN_TEXT | FACET_KEY_OPTION_VISIBLE |

396 - FACET_KEY_OPTION_FTS);

632 + FACET_KEY_OPTION_NEVER_FACET | FACET_KEY_OPTION_MAIN_TEXT |

633 + FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_FTS);

634 +

635 +// facets_register_dynamic_key_name(facets, "MESSAGE",

636 +// FACET_KEY_OPTION_NEVER_FACET | FACET_KEY_OPTION_MAIN_TEXT | FACET_KEY_OPTION_RICH_TEXT |

637 +// FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_FTS,

638 +// netdata_systemd_journal_rich_message, NULL);

639

640 facets_register_key_name_transformation(facets, "PRIORITY", FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS,

399 - systemd_journal_transform_priority, NULL);

641 + netdata_systemd_journal_transform_priority, NULL);

642

643 facets_register_key_name_transformation(facets, "SYSLOG_FACILITY", FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS,

402 - systemd_journal_transform_syslog_facility, NULL);

644 + netdata_systemd_journal_transform_syslog_facility, NULL);

645

646 facets_register_key_name(facets, "SYSLOG_IDENTIFIER", FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS);

647 facets_register_key_name(facets, "UNIT", FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS);

648 facets_register_key_name(facets, "USER_UNIT", FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS);

649

650 facets_register_key_name_transformation(facets, "_UID", FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS,

409 - systemd_journal_transform_uid, uids);

651 + netdata_systemd_journal_transform_uid, uids);

652

653 facets_register_key_name_transformation(facets, "_GID", FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS,

412 - systemd_journal_transform_gid, gids);

654 + netdata_systemd_journal_transform_gid, gids);

655

656 bool info = false;

657 + bool data_only = false;

658 time_t after_s = 0, before_s = 0;

659 usec_t anchor = 0;

660 usec_t if_modified_since = 0;

@@ -430,12 +673,15 @@ static void function_systemd_journal(const char *transaction, char *function, ch

673 if(!keyword) break;

674

675 if(strcmp(keyword, JOURNAL_PARAMETER_HELP) == 0) {

433 - systemd_journal_function_help(transaction);

676 + netdata_systemd_journal_function_help(transaction);

677 goto cleanup;

678 }

679 else if(strcmp(keyword, JOURNAL_PARAMETER_INFO) == 0) {

680 info = true;

681 }

682 + else if(strcmp(keyword, JOURNAL_PARAMETER_DATA_ONLY) == 0) {

683 + data_only = true;

684 + }

685 else if(strncmp(keyword, JOURNAL_PARAMETER_SOURCE ":", sizeof(JOURNAL_PARAMETER_SOURCE ":") - 1) == 0) {

686 source = &keyword[sizeof(JOURNAL_PARAMETER_SOURCE ":") - 1];

687 }

@@ -580,18 +826,22 @@ static void function_systemd_journal(const char *transaction, char *function, ch

826 facets_set_query(facets, query);

827 facets_set_histogram(facets, chart ? chart : "PRIORITY", after_s * USEC_PER_SEC, before_s * USEC_PER_SEC);

828

583 - response = systemd_journal_query(wb, facets, after_s * USEC_PER_SEC, before_s * USEC_PER_SEC,

584 - if_modified_since, now_monotonic_usec() + (timeout - 1) * USEC_PER_SEC);

829 + response = netdata_systemd_journal_query(wb, facets, after_s * USEC_PER_SEC, before_s * USEC_PER_SEC,

830 + anchor, direction, last,

831 + if_modified_since, data_only,

832 + now_monotonic_usec() + (timeout - 1) * USEC_PER_SEC);

833

834 if(response != HTTP_RESP_OK) {

835 + netdata_mutex_lock(&stdout_mutex);

836 pluginsd_function_json_error_to_stdout(transaction, response, "failed");

837 + netdata_mutex_unlock(&stdout_mutex);

838 goto cleanup;

839 }

840

841 output:

592 - pluginsd_function_result_begin_to_stdout(transaction, response, "application/json", expires);

593 - fwrite(buffer_tostring(wb), buffer_strlen(wb), 1, stdout);

594 - pluginsd_function_result_end_to_stdout();

842 + netdata_mutex_lock(&stdout_mutex);

843 + pluginsd_function_result_to_stdout(transaction, response, "application/json", expires, wb);

844 + netdata_mutex_unlock(&stdout_mutex);

845

846 cleanup:

847 facets_destroy(facets);

@@ -625,16 +875,14 @@ static void *reader_main(void *arg __maybe_unused) {

875 int timeout = str2i(timeout_s);

876 if(timeout <= 0) timeout = SYSTEMD_JOURNAL_DEFAULT_TIMEOUT;

877

628 - netdata_mutex_lock(&mutex);

629 -

878 if(strncmp(function, SYSTEMD_JOURNAL_FUNCTION_NAME, strlen(SYSTEMD_JOURNAL_FUNCTION_NAME)) == 0)

879 function_systemd_journal(transaction, function, buffer, PLUGINSD_LINE_MAX + 1, timeout);

632 - else

880 + else {

881 + netdata_mutex_lock(&stdout_mutex);

882 pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_NOT_FOUND,

883 "No function with this name found in systemd-journal.plugin.");

635 -

636 - fflush(stdout);

637 - netdata_mutex_unlock(&mutex);

884 + netdata_mutex_unlock(&stdout_mutex);

885 + }

886 }

887 }

888 else

@@ -691,16 +939,16 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) {

939 usec_t step = 1000 * USEC_PER_MS;

940 bool tty = isatty(fileno(stderr)) == 1;

941

694 - netdata_mutex_lock(&mutex);

942 + netdata_mutex_lock(&stdout_mutex);

943 fprintf(stdout, PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\"\n",

944 SYSTEMD_JOURNAL_FUNCTION_NAME, SYSTEMD_JOURNAL_DEFAULT_TIMEOUT, SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION);

945

946 heartbeat_t hb;

947 heartbeat_init(&hb);

948 for(iteration = 0; 1 ; iteration++) {

701 - netdata_mutex_unlock(&mutex);

949 + netdata_mutex_unlock(&stdout_mutex);

950 heartbeat_next(&hb, step);

703 - netdata_mutex_lock(&mutex);

951 + netdata_mutex_lock(&stdout_mutex);

952

953 if(!tty)

954 fprintf(stdout, "\n");

libnetdata/buffer/buffer.h

+19 -7

@@ -938,10 +938,12 @@ typedef enum __attribute__((packed)) {

938 RRDF_FIELD_OPTS_VISIBLE = (1 << 1), // the field should be visible by default

939 RRDF_FIELD_OPTS_STICKY = (1 << 2), // the field should be sticky

940 RRDF_FIELD_OPTS_FULL_WIDTH = (1 << 3), // the field should get full width

941 - RRDF_FIELD_OPTS_WRAP = (1 << 4), // the field should get full width

941 + RRDF_FIELD_OPTS_WRAP = (1 << 4), // the field should wrap

942 + RRDR_FIELD_OPTS_DUMMY = (1 << 5), // not a presentable field

943 } RRDF_FIELD_OPTIONS;

944

945 typedef enum __attribute__((packed)) {

946 + RRDF_FIELD_TYPE_NONE,

947 RRDF_FIELD_TYPE_INTEGER,

948 RRDF_FIELD_TYPE_STRING,

949 RRDF_FIELD_TYPE_DETAIL_STRING,

@@ -954,6 +956,9 @@ typedef enum __attribute__((packed)) {

956 static inline const char *rrdf_field_type_to_string(RRDF_FIELD_TYPE type) {

957 switch(type) {

958 default:

959 + case RRDF_FIELD_TYPE_NONE:

960 + return "none";

961 +

962 case RRDF_FIELD_TYPE_INTEGER:

963 return "integer";

964

@@ -978,10 +983,11 @@ static inline const char *rrdf_field_type_to_string(RRDF_FIELD_TYPE type) {

983 }

984

985 typedef enum __attribute__((packed)) {

981 - RRDF_FIELD_VISUAL_VALUE, // show the value, possibly applying a transformation

982 - RRDF_FIELD_VISUAL_BAR, // show the value and a bar, respecting the max field to fill the bar at 100%

983 - RRDF_FIELD_VISUAL_PILL, //

984 - RRDF_FIELD_VISUAL_MARKDOC, //

986 + RRDF_FIELD_VISUAL_VALUE, // show the value, possibly applying a transformation

987 + RRDF_FIELD_VISUAL_BAR, // show the value and a bar, respecting the max field to fill the bar at 100%

988 + RRDF_FIELD_VISUAL_PILL, //

989 + RRDF_FIELD_VISUAL_RICH, //

990 + RRDR_FIELD_VISUAL_ROW_OPTIONS, // this is a dummy column that is used for row options

991 } RRDF_FIELD_VISUAL;

992

993 static inline const char *rrdf_field_visual_to_string(RRDF_FIELD_VISUAL visual) {

@@ -996,8 +1002,11 @@ static inline const char *rrdf_field_visual_to_string(RRDF_FIELD_VISUAL visual)

1002 case RRDF_FIELD_VISUAL_PILL:

1003 return "pill";

1004

999 - case RRDF_FIELD_VISUAL_MARKDOC:

1000 - return "markdoc";

1005 + case RRDF_FIELD_VISUAL_RICH:

1006 + return "richValue";

1007 +

1008 + case RRDR_FIELD_VISUAL_ROW_OPTIONS:

1009 + return "rowOptions";

1010 }

1011 }

1012

@@ -1144,6 +1153,9 @@ buffer_rrdf_table_add_field(BUFFER *wb, size_t field_id, const char *key, const

1153

1154 buffer_json_member_add_boolean(wb, "full_width", options & RRDF_FIELD_OPTS_FULL_WIDTH);

1155 buffer_json_member_add_boolean(wb, "wrap", options & RRDF_FIELD_OPTS_WRAP);

1156 +

1157 + if(options & RRDR_FIELD_OPTS_DUMMY)

1158 + buffer_json_member_add_boolean(wb, "dummy", true);

1159 }

1160 buffer_json_object_close(wb);

1161 }

libnetdata/facets/facets.c

+166 -96

@@ -108,7 +108,6 @@ struct facets {

108

109 SIMPLE_PATTERN *query; // the full text search pattern

110 size_t keys_filtered_by_query; // the number of fields we do full text search (constant)

111 - size_t keys_matched_by_query; // the number of fields matched the full text search (per row)

111

112 DICTIONARY *accepted_params;

113

@@ -120,6 +119,11 @@ struct facets {

119 uint32_t max_items_to_return;

120 uint32_t order;

121

122 + struct {

123 + FACET_ROW_SEVERITY severity;

124 + size_t keys_matched_by_query; // the number of fields matched the full text search (per row)

125 + } current_row;

126 +

127 struct {

128 char *chart;

129 bool enabled;

@@ -647,7 +651,12 @@ static void facets_histogram_generate(FACETS *facets, FACET_KEY *k, BUFFER *wb)

651

652 buffer_json_member_add_object(wb, "view");

653 {

650 - buffer_json_member_add_string(wb, "title", "Events Distribution");

654 + char title[1024 + 1] = "Events Distribution";

655 + FACET_KEY *k = dictionary_get(facets->keys, facets->histogram.chart);

656 + if(k && k->name)

657 + snprintfz(title, 1024, "Events Distribution by %s", k->name);

658 +

659 + buffer_json_member_add_string(wb, "title", title);

660 buffer_json_member_add_time_t(wb, "update_every", facets->histogram.slot_width_ut / USEC_PER_SEC);

661 buffer_json_member_add_time_t(wb, "after", facets->histogram.after_ut / USEC_PER_SEC);

662 buffer_json_member_add_time_t(wb, "before", facets->histogram.before_ut / USEC_PER_SEC);

@@ -733,7 +742,7 @@ static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k) {

742 }

743 }

744

736 - if(included && !excluded) {

745 + if(included && !excluded && !(facets->options & FACETS_OPTION_DISABLE_ALL_FACETS)) {

746 k->options |= FACET_KEY_OPTION_FACET;

747 k->options &= ~FACET_KEY_OPTION_NO_FACET;

748 return true;

@@ -988,6 +997,14 @@ void facets_register_facet_id_filter(FACETS *facets, const char *key_id, char *v

997 dictionary_set(k->values, value_ids, &tv, sizeof(tv));

998 }

999

1000 +void facets_set_current_row_severity(FACETS *facets, FACET_ROW_SEVERITY severity) {

1001 + facets->current_row.severity = severity;

1002 +}

1003 +

1004 +void facets_data_only_mode(FACETS *facets) {

1005 + facets->options |= FACETS_OPTION_DISABLE_ALL_FACETS | FACETS_OPTION_DISABLE_HISTOGRAM | FACETS_OPTION_DATA_ONLY;

1006 +}

1007 +

1008 // ----------------------------------------------------------------------------

1009

1010 static inline void facets_check_value(FACETS *facets __maybe_unused, FACET_KEY *k) {

@@ -1017,7 +1034,7 @@ static inline void facets_check_value(FACETS *facets __maybe_unused, FACET_KEY *

1034 if(facets->query && !k->current_value.empty && ((k->options & FACET_KEY_OPTION_FTS) || facets->options & FACETS_OPTION_ALL_KEYS_FTS)) {

1035 facets->operations.fts.searches++;

1036 if(simple_pattern_matches(facets->query, buffer_tostring(k->current_value.b)))

1020 - facets->keys_matched_by_query++;

1037 + facets->current_row.keys_matched_by_query++;

1038 }

1039

1040 if(k->values) {

@@ -1123,6 +1140,7 @@ static FACET_ROW *facets_row_create(FACETS *facets, usec_t usec, FACET_ROW *into

1140 facets->operations.rows.created++;

1141 }

1142

1143 + row->severity = facets->current_row.severity;

1144 row->usec = usec;

1145

1146 FACET_KEY *k;

@@ -1273,11 +1291,12 @@ void facets_rows_begin(FACETS *facets) {

1291 }

1292 // dfe_done(k);

1293

1276 - facets->keys_matched_by_query = 0;

1294 + facets->current_row.severity = FACET_ROW_SEVERITY_NORMAL;

1295 + facets->current_row.keys_matched_by_query = 0;

1296 }

1297

1298 void facets_row_finished(FACETS *facets, usec_t usec) {

1280 - if(facets->query && facets->keys_filtered_by_query && !facets->keys_matched_by_query)

1299 + if(facets->query && facets->keys_filtered_by_query && !facets->current_row.keys_matched_by_query)

1300 goto cleanup;

1301

1302 facets->operations.rows.evaluated++;

@@ -1345,6 +1364,26 @@ cleanup:

1364 // ----------------------------------------------------------------------------

1365 // output

1366

1367 +static const char *facets_severity_to_string(FACET_ROW_SEVERITY severity) {

1368 + switch(severity) {

1369 + default:

1370 + case FACET_ROW_SEVERITY_NORMAL:

1371 + return "normal";

1372 +

1373 + case FACET_ROW_SEVERITY_DEBUG:

1374 + return "debug";

1375 +

1376 + case FACET_ROW_SEVERITY_NOTICE:

1377 + return "notice";

1378 +

1379 + case FACET_ROW_SEVERITY_WARNING:

1380 + return "warning";

1381 +

1382 + case FACET_ROW_SEVERITY_CRITICAL:

1383 + return "critical";

1384 + }

1385 +}

1386 +

1387 void facets_accepted_parameters_to_json_array(FACETS *facets, BUFFER *wb, bool with_keys) {

1388 buffer_json_member_add_array(wb, "accepted_params");

1389 {

@@ -1371,104 +1410,126 @@ void facets_accepted_parameters_to_json_array(FACETS *facets, BUFFER *wb, bool w

1410 }

1411

1412 void facets_report(FACETS *facets, BUFFER *wb) {

1374 - buffer_json_member_add_boolean(wb, "show_ids", false);

1375 - buffer_json_member_add_boolean(wb, "has_history", true);

1413 + if(!(facets->options & FACETS_OPTION_DATA_ONLY)) {

1414 + buffer_json_member_add_boolean(wb, "show_ids", false); // do not show the column ids to the user

1415 + buffer_json_member_add_boolean(wb, "has_history", true); // enable date-time picker with after-before

1416

1377 - buffer_json_member_add_object(wb, "pagination");

1378 - buffer_json_member_add_boolean(wb, "enabled", true);

1379 - buffer_json_member_add_string(wb, "key", "anchor");

1380 - buffer_json_member_add_string(wb, "column", "timestamp");

1381 - buffer_json_object_close(wb);

1417 + buffer_json_member_add_object(wb, "pagination");

1418 + {

1419 + buffer_json_member_add_boolean(wb, "enabled", true);

1420 + buffer_json_member_add_string(wb, "key", "anchor");

1421 + buffer_json_member_add_string(wb, "column", "timestamp");

1422 + }

1423 + buffer_json_object_close(wb); // pagination

1424

1383 - facets_accepted_parameters_to_json_array(facets, wb, true);

1425 + facets_accepted_parameters_to_json_array(facets, wb, true);

1426 + }

1427

1385 - buffer_json_member_add_array(wb, "facets");

1386 - {

1387 - FACET_KEY *k;

1388 - dfe_start_read(facets->keys, k) {

1389 - if(!k->values)

1390 - continue;

1428 + if(!(facets->options & FACETS_OPTION_DISABLE_ALL_FACETS)) {

1429 + buffer_json_member_add_array(wb, "facets");

1430 + {

1431 + FACET_KEY *k;

1432 + dfe_start_read(facets->keys, k) {

1433 + if(!k->values)

1434 + continue;

1435

1392 - buffer_json_add_array_item_object(wb); // key

1393 - {

1394 - buffer_json_member_add_string(wb, "id", k_dfe.name);

1395 - buffer_json_member_add_string(wb, "name", k->name);

1436 + buffer_json_add_array_item_object(wb); // key

1437 + {

1438 + buffer_json_member_add_string(wb, "id", k_dfe.name);

1439 + buffer_json_member_add_string(wb, "name", k->name);

1440

1397 - if(!k->order)

1398 - k->order = facets->order++;

1441 + if(!k->order)

1442 + k->order = facets->order++;

1443

1400 - buffer_json_member_add_uint64(wb, "order", k->order);

1401 - buffer_json_member_add_array(wb, "options");

1402 - {

1403 - FACET_VALUE *v;

1404 - dfe_start_read(k->values, v) {

1405 - buffer_json_add_array_item_object(wb);

1406 - {

1407 - buffer_json_member_add_string(wb, "id", v_dfe.name);

1408 - buffer_json_member_add_string(wb, "name", v->name);

1409 - buffer_json_member_add_uint64(wb, "count", v->final_facet_value_counter);

1444 + buffer_json_member_add_uint64(wb, "order", k->order);

1445 + buffer_json_member_add_array(wb, "options");

1446 + {

1447 + FACET_VALUE *v;

1448 + dfe_start_read(k->values, v) {

1449 + buffer_json_add_array_item_object(wb);

1450 + {

1451 + buffer_json_member_add_string(wb, "id", v_dfe.name);

1452 + buffer_json_member_add_string(wb, "name", v->name);

1453 + buffer_json_member_add_uint64(wb, "count", v->final_facet_value_counter);

1454 + }

1455 + buffer_json_object_close(wb);

1456 }

1411 - buffer_json_object_close(wb);

1457 + dfe_done(v);

1458 }

1413 - dfe_done(v);

1459 + buffer_json_array_close(wb); // options

1460 }

1415 - buffer_json_array_close(wb); // options

1461 + buffer_json_object_close(wb); // key

1462 }

1417 - buffer_json_object_close(wb); // key

1463 + dfe_done(k);

1464 }

1419 - dfe_done(k);

1465 + buffer_json_array_close(wb); // facets

1466 }

1421 - buffer_json_array_close(wb); // facets

1422 -

1423 - buffer_json_member_add_object(wb, "columns");

1424 - {

1425 - size_t field_id = 0;

1426 - buffer_rrdf_table_add_field(

1427 - wb, field_id++,

1428 - "timestamp", "Timestamp",

1429 - RRDF_FIELD_TYPE_TIMESTAMP,

1430 - RRDF_FIELD_VISUAL_VALUE,

1431 - RRDF_FIELD_TRANSFORM_DATETIME_USEC, 0, NULL, NAN,

1432 - RRDF_FIELD_SORT_DESCENDING,

1433 - NULL,

1434 - RRDF_FIELD_SUMMARY_COUNT,

1435 - RRDF_FIELD_FILTER_RANGE,

1436 - RRDF_FIELD_OPTS_VISIBLE | RRDF_FIELD_OPTS_UNIQUE_KEY,

1437 - NULL);

1438 -

1439 - FACET_KEY *k;

1440 - dfe_start_read(facets->keys, k) {

1441 - RRDF_FIELD_OPTIONS options = RRDF_FIELD_OPTS_NONE;

1442 - bool visible = k->options & (FACET_KEY_OPTION_VISIBLE|FACET_KEY_OPTION_STICKY);

1443 -

1444 - if((facets->options & FACETS_OPTION_ALL_FACETS_VISIBLE && k->values))

1445 - visible = true;

1446 -

1447 - if(!visible)

1448 - visible = simple_pattern_matches(facets->visible_keys, k->name);

1449 -

1450 - if(visible)

1451 - options |= RRDF_FIELD_OPTS_VISIBLE;

1452 -

1453 - if(k->options & FACET_KEY_OPTION_MAIN_TEXT)

1454 - options |= RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_WRAP;

1467

1468 + if(!(facets->options & FACETS_OPTION_DATA_ONLY)) {

1469 + buffer_json_member_add_object(wb, "columns");

1470 + {

1471 + size_t field_id = 0;

1472 buffer_rrdf_table_add_field(

1473 wb, field_id++,

1458 - k_dfe.name, k->name ? k->name : k_dfe.name,

1459 - RRDF_FIELD_TYPE_STRING,

1474 + "timestamp", "Timestamp",

1475 + RRDF_FIELD_TYPE_TIMESTAMP,

1476 RRDF_FIELD_VISUAL_VALUE,

1477 + RRDF_FIELD_TRANSFORM_DATETIME_USEC, 0, NULL, NAN,

1478 + RRDF_FIELD_SORT_DESCENDING,

1479 + NULL,

1480 + RRDF_FIELD_SUMMARY_COUNT,

1481 + RRDF_FIELD_FILTER_RANGE,

1482 + RRDF_FIELD_OPTS_VISIBLE | RRDF_FIELD_OPTS_UNIQUE_KEY,

1483 + NULL);

1484 +

1485 + buffer_rrdf_table_add_field(

1486 + wb, field_id++,

1487 + "rowOptions", "rowOptions",

1488 + RRDF_FIELD_TYPE_NONE,

1489 + RRDR_FIELD_VISUAL_ROW_OPTIONS,

1490 RRDF_FIELD_TRANSFORM_NONE, 0, NULL, NAN,

1462 - RRDF_FIELD_SORT_ASCENDING,

1491 + RRDF_FIELD_SORT_FIXED,

1492 NULL,

1493 RRDF_FIELD_SUMMARY_COUNT,

1465 - (k->options & FACET_KEY_OPTION_NEVER_FACET) ? RRDF_FIELD_FILTER_NONE : RRDF_FIELD_FILTER_FACET,

1466 - options,

1467 - FACET_VALUE_UNSET);

1494 + RRDF_FIELD_FILTER_NONE,

1495 + RRDR_FIELD_OPTS_DUMMY,

1496 + NULL);

1497 +

1498 + FACET_KEY *k;

1499 + dfe_start_read(facets->keys, k){

1500 + RRDF_FIELD_OPTIONS options = RRDF_FIELD_OPTS_NONE;

1501 + bool visible = k->options & (FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_STICKY);

1502 +

1503 + if ((facets->options & FACETS_OPTION_ALL_FACETS_VISIBLE && k->values))

1504 + visible = true;

1505 +

1506 + if (!visible)

1507 + visible = simple_pattern_matches(facets->visible_keys, k->name);

1508 +

1509 + if (visible)

1510 + options |= RRDF_FIELD_OPTS_VISIBLE;

1511 +

1512 + if (k->options & FACET_KEY_OPTION_MAIN_TEXT)

1513 + options |= RRDF_FIELD_OPTS_FULL_WIDTH | RRDF_FIELD_OPTS_WRAP;

1514 +

1515 + buffer_rrdf_table_add_field(

1516 + wb, field_id++,

1517 + k_dfe.name, k->name ? k->name : k_dfe.name,

1518 + RRDF_FIELD_TYPE_STRING,

1519 + (k->options & FACET_KEY_OPTION_RICH_TEXT) ? RRDF_FIELD_VISUAL_RICH : RRDF_FIELD_VISUAL_VALUE,

1520 + RRDF_FIELD_TRANSFORM_NONE, 0, NULL, NAN,

1521 + RRDF_FIELD_SORT_ASCENDING,

1522 + NULL,

1523 + RRDF_FIELD_SUMMARY_COUNT,

1524 + (k->options & FACET_KEY_OPTION_NEVER_FACET) ? RRDF_FIELD_FILTER_NONE

1525 + : RRDF_FIELD_FILTER_FACET,

1526 + options,

1527 + FACET_VALUE_UNSET);

1528 + }

1529 + dfe_done(k);

1530 }

1469 - dfe_done(k);

1531 + buffer_json_object_close(wb); // columns

1532 }

1471 - buffer_json_object_close(wb); // columns

1533

1534 buffer_json_member_add_array(wb, "data");

1535 {

@@ -1488,6 +1549,11 @@ void facets_report(FACETS *facets, BUFFER *wb) {

1549

1550 buffer_json_add_array_item_array(wb); // each row

1551 buffer_json_add_array_item_uint64(wb, row->usec);

1552 + buffer_json_add_array_item_object(wb);

1553 + {

1554 + buffer_json_member_add_string(wb, "severity", facets_severity_to_string(row->severity));

1555 + }

1556 + buffer_json_object_close(wb);

1557

1558 FACET_KEY *k;

1559 dfe_start_read(facets->keys, k)

@@ -1514,11 +1580,13 @@ void facets_report(FACETS *facets, BUFFER *wb) {

1580 }

1581 buffer_json_array_close(wb); // data

1582

1517 - buffer_json_member_add_string(wb, "default_sort_column", "timestamp");

1518 - buffer_json_member_add_array(wb, "default_charts");

1519 - buffer_json_array_close(wb);

1583 + if(!(facets->options & FACETS_OPTION_DATA_ONLY)) {

1584 + buffer_json_member_add_string(wb, "default_sort_column", "timestamp");

1585 + buffer_json_member_add_array(wb, "default_charts");

1586 + buffer_json_array_close(wb);

1587 + }

1588

1521 - if(facets->histogram.enabled) {

1589 + if(facets->histogram.enabled && !(facets->options & FACETS_OPTION_DISABLE_HISTOGRAM)) {

1590 const char *first_histogram = NULL;

1591 buffer_json_member_add_array(wb, "available_histograms");

1592 {

@@ -1561,16 +1629,18 @@ void facets_report(FACETS *facets, BUFFER *wb) {

1629 }

1630 }

1631

1564 - buffer_json_member_add_object(wb, "items");

1565 - {

1566 - buffer_json_member_add_uint64(wb, "evaluated", facets->operations.rows.evaluated);

1567 - buffer_json_member_add_uint64(wb, "matched", facets->operations.rows.matched);

1568 - buffer_json_member_add_uint64(wb, "returned", facets->items_to_return);

1569 - buffer_json_member_add_uint64(wb, "max_to_return", facets->max_items_to_return);

1570 - buffer_json_member_add_uint64(wb, "before", facets->operations.skips_before);

1571 - buffer_json_member_add_uint64(wb, "after", facets->operations.skips_after + facets->operations.shifts);

1632 + if(!(facets->options & FACETS_OPTION_DATA_ONLY)) {

1633 + buffer_json_member_add_object(wb, "items");

1634 + {

1635 + buffer_json_member_add_uint64(wb, "evaluated", facets->operations.rows.evaluated);

1636 + buffer_json_member_add_uint64(wb, "matched", facets->operations.rows.matched);

1637 + buffer_json_member_add_uint64(wb, "returned", facets->items_to_return);

1638 + buffer_json_member_add_uint64(wb, "max_to_return", facets->max_items_to_return);

1639 + buffer_json_member_add_uint64(wb, "before", facets->operations.skips_before);

1640 + buffer_json_member_add_uint64(wb, "after", facets->operations.skips_after + facets->operations.shifts);

1641 + }

1642 + buffer_json_object_close(wb); // items

1643 }

1573 - buffer_json_object_close(wb); // items

1644

1645 buffer_json_member_add_object(wb, "stats");

1646 {

libnetdata/facets/facets.h

+16 -1

@@ -18,9 +18,18 @@ typedef enum __attribute__((packed)) {

18 FACET_KEY_OPTION_VISIBLE = (1 << 4), // should be in the default table

19 FACET_KEY_OPTION_FTS = (1 << 5), // the key is filterable by full text search (FTS)

20 FACET_KEY_OPTION_MAIN_TEXT = (1 << 6), // full width and wrap

21 - FACET_KEY_OPTION_REORDER = (1 << 7), // give the key a new order id on first encounter

21 + FACET_KEY_OPTION_RICH_TEXT = (1 << 7),

22 + FACET_KEY_OPTION_REORDER = (1 << 8), // give the key a new order id on first encounter

23 } FACET_KEY_OPTIONS;

25 +typedef enum __attribute__((packed)) {

26 + FACET_ROW_SEVERITY_DEBUG, // lowest - not important

27 + FACET_ROW_SEVERITY_NORMAL, // the default

28 + FACET_ROW_SEVERITY_NOTICE, // bold

29 + FACET_ROW_SEVERITY_WARNING, // yellow + bold

30 + FACET_ROW_SEVERITY_CRITICAL, // red + bold

31 +} FACET_ROW_SEVERITY;

32 +

33 typedef struct facet_row_key_value {

34 const char *tmp;

35 BUFFER *wb;

@@ -30,6 +39,7 @@ typedef struct facet_row_key_value {

39 typedef struct facet_row {

40 usec_t usec;

41 DICTIONARY *dict;

42 + FACET_ROW_SEVERITY severity;

43 struct facet_row *prev, *next;

44 } FACET_ROW;

@@ -47,6 +57,9 @@ FACET_KEY *facets_register_key_name_transformation(FACETS *facets, const char *k

57 typedef enum __attribute__((packed)) {

58 FACETS_OPTION_ALL_FACETS_VISIBLE = (1 << 0), // all facets, should be visible by default in the table

59 FACETS_OPTION_ALL_KEYS_FTS = (1 << 1), // all keys are searchable by full text search

60 + FACETS_OPTION_DISABLE_ALL_FACETS = (1 << 2),

61 + FACETS_OPTION_DISABLE_HISTOGRAM = (1 << 3),

62 + FACETS_OPTION_DATA_ONLY = (1 << 4),

63 } FACETS_OPTIONS;

65 FACETS *facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys);

@@ -70,5 +83,7 @@ void facets_add_key_value_length(FACETS *facets, const char *key, size_t key_len

84 void facets_report(FACETS *facets, BUFFER *wb);

85 void facets_accepted_parameters_to_json_array(FACETS *facets, BUFFER *wb, bool with_keys);

86 +void facets_set_current_row_severity(FACETS *facets, FACET_ROW_SEVERITY severity);

87 +void facets_data_only_mode(FACETS *facets);

89 #endif