@cryptotaxi247 / netdata-1 / commits / 31924c0eb

systemd-journal and facets improvements (#15926)

Costa Tsaousis committed Sep 8, 2023 at 11:32 UTC 31924c0eb2fb245bdf047a5f4fe9f5ef10baf24e

7 files changed +224 -115

collectors/apps.plugin/apps_plugin.c

+9 -5

@@ -4696,21 +4696,24 @@ static void function_processes(const char *transaction, char *function __maybe_u

4696 if(!category && strncmp(keyword, PROCESS_FILTER_CATEGORY, strlen(PROCESS_FILTER_CATEGORY)) == 0) {

4697 category = find_target_by_name(apps_groups_root_target, &keyword[strlen(PROCESS_FILTER_CATEGORY)]);

4698 if(!category) {

4699 - pluginsd_function_json_error(transaction, HTTP_RESP_BAD_REQUEST, "No category with that name found.");

4699 + pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_BAD_REQUEST,

4700 + "No category with that name found.");

4701 return;

4702 }

4703 }

4704 else if(!user && strncmp(keyword, PROCESS_FILTER_USER, strlen(PROCESS_FILTER_USER)) == 0) {

4705 user = find_target_by_name(users_root_target, &keyword[strlen(PROCESS_FILTER_USER)]);

4706 if(!user) {

4706 - pluginsd_function_json_error(transaction, HTTP_RESP_BAD_REQUEST, "No user with that name found.");

4707 + pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_BAD_REQUEST,

4708 + "No user with that name found.");

4709 return;

4710 }

4711 }

4712 else if(strncmp(keyword, PROCESS_FILTER_GROUP, strlen(PROCESS_FILTER_GROUP)) == 0) {

4713 group = find_target_by_name(groups_root_target, &keyword[strlen(PROCESS_FILTER_GROUP)]);

4714 if(!group) {

4713 - pluginsd_function_json_error(transaction, HTTP_RESP_BAD_REQUEST, "No group with that name found.");

4715 + pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_BAD_REQUEST,

4716 + "No group with that name found.");

4717 return;

4718 }

4719 }

@@ -4736,7 +4739,7 @@ static void function_processes(const char *transaction, char *function __maybe_u

4739 else {

4740 char msg[PLUGINSD_LINE_MAX];

4741 snprintfz(msg, PLUGINSD_LINE_MAX, "Invalid parameter '%s'", keyword);

4739 - pluginsd_function_json_error(transaction, HTTP_RESP_BAD_REQUEST, msg);

4742 + pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_BAD_REQUEST, msg);

4743 return;

4744 }

4745 }

@@ -5562,7 +5565,8 @@ static void *reader_main(void *arg __maybe_unused) {

5565 if(strncmp(function, "processes", strlen("processes")) == 0)

5566 function_processes(transaction, function, buffer, PLUGINSD_LINE_MAX + 1, timeout);

5567 else

5565 - pluginsd_function_json_error(transaction, HTTP_RESP_NOT_FOUND, "No function with this name found in apps.plugin.");

5568 + pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_NOT_FOUND,

5569 + "No function with this name found in apps.plugin.");

5570

5571 fflush(stdout);

5572 netdata_mutex_unlock(&mutex);

collectors/plugins.d/plugins_d.h

+1 -1

@@ -123,7 +123,7 @@ size_t pluginsd_initialize_plugin_directories();

123 #define pluginsd_function_result_end_to_stdout() \

124 fprintf(stdout, "\n" PLUGINSD_KEYWORD_FUNCTION_RESULT_END "\n")

125

126 -static inline void pluginsd_function_json_error(const char *transaction, int code, const char *msg) {

126 +static inline void pluginsd_function_json_error_to_stdout(const char *transaction, int code, const char *msg) {

127 char buffer[PLUGINSD_LINE_MAX + 1];

128 json_escape_string(buffer, msg, PLUGINSD_LINE_MAX);

129

collectors/plugins.d/pluginsd_parser.h

+6 -3

@@ -10,6 +10,9 @@

10 // this has to be in-sync with the same at receiver.c

11 #define WORKER_RECEIVER_JOB_REPLICATION_COMPLETION (WORKER_PARSER_FIRST_JOB - 3)

13 +// this controls the max response size of a function

14 +#define PLUGINSD_MAX_DEFERRED_SIZE (20 * 1024 * 1024)

15 +

16 // PARSER return codes

17 typedef enum __attribute__ ((__packed__)) parser_rc {

18 PARSER_RC_OK, // Callback was successful, go on

@@ -157,9 +160,9 @@ static inline int parser_action(PARSER *parser, char *input) {

160 if(!has_keyword || strcmp(command, parser->defer.end_keyword) != 0) {

161 if(parser->defer.response) {

162 buffer_strcat(parser->defer.response, input);

160 - if(buffer_strlen(parser->defer.response) > 10 * 1024 * 1024) {

161 - // more than 10MB of data

162 - // a bad plugin that did not send the end_keyword

163 + if(buffer_strlen(parser->defer.response) > PLUGINSD_MAX_DEFERRED_SIZE) {

164 + // more than PLUGINSD_MAX_DEFERRED_SIZE of data,

165 + // or a bad plugin that did not send the end_keyword

166 internal_error(true, "PLUGINSD: deferred response is too big (%zu bytes). Stopping this plugin.", buffer_strlen(parser->defer.response));

167 return 1;

168 }

collectors/systemd-journal.plugin/systemd-journal.c

+37 -18

@@ -5,7 +5,7 @@

5 * GPL v3+

6 */

8 -// TODO - 1) MARKDOC

8 +// TODO - 1) MARKDOC 2) HELP TEXT

10 #include "collectors/all.h"

11 #include "libnetdata/libnetdata.h"

@@ -31,6 +31,9 @@

31 #define JOURNAL_PARAMETER_QUERY "query"

32 #define JOURNAL_PARAMETER_FACETS "facets"

33 #define JOURNAL_PARAMETER_HISTOGRAM "histogram"

34 +#define JOURNAL_PARAMETER_DIRECTION "direction"

35 +#define JOURNAL_PARAMETER_IF_MODIFIED_SINCE "if_modified_since"

36 +

38 #define SYSTEMD_ALWAYS_VISIBLE_KEYS NULL

39 #define SYSTEMD_KEYS_EXCLUDED_FROM_FACETS NULL

@@ -39,19 +42,11 @@

42 "|SYSLOG_IDENTIFIER" \

43 "|SYSLOG_FACILITY" \

44 "|PRIORITY" \

42 - "|_HOSTNAME" \

43 - "|_RUNTIME_SCOPE" \

44 - "|_PID" \

45 "|_UID" \

46 "|_GID" \

47 "|_SYSTEMD_UNIT" \

48 "|_SYSTEMD_SLICE" \

49 - "|_SYSTEMD_USER_SLICE" \

49 "|_COMM" \

51 - "|_EXE" \

52 - "|_SYSTEMD_CGROUP" \

53 - "|_SYSTEMD_USER_UNIT" \

54 - "|USER_UNIT" \

50 "|UNIT" \

51 "|CONTAINER_NAME" \

52 "|IMAGE_NAME" \

@@ -65,7 +60,7 @@ DICTIONARY *gids = NULL;

61 // ----------------------------------------------------------------------------

68 -int systemd_journal_query(BUFFER *wb, FACETS *facets, usec_t after_ut, usec_t before_ut, usec_t stop_monotonic_ut) {

63 +int systemd_journal_query(BUFFER *wb, FACETS *facets, usec_t after_ut, usec_t before_ut, usec_t if_modified_since, usec_t stop_monotonic_ut) {

64 sd_journal *j = NULL;

65 int r;

@@ -91,6 +86,7 @@ int systemd_journal_query(BUFFER *wb, FACETS *facets, usec_t after_ut, usec_t be

87 facets_rows_begin(facets);

89 + uint64_t first_msg_ut = 0;

90 bool timed_out = false;

91 size_t row_counter = 0;

92 sd_journal_seek_realtime_usec(j, before_ut);

@@ -99,6 +95,16 @@ int systemd_journal_query(BUFFER *wb, FACETS *facets, usec_t after_ut, usec_t be

96 uint64_t msg_ut;

97 sd_journal_get_realtime_usec(j, &msg_ut);

98 +

99 + if(unlikely(!first_msg_ut)) {

100 + if(msg_ut == if_modified_since) {

101 + sd_journal_close(j);

102 + return HTTP_RESP_NOT_MODIFIED;

103 + }

104 +

105 + first_msg_ut = msg_ut;

106 + }

107 +

108 if (msg_ut < after_ut)

109 break;

110

@@ -136,6 +142,7 @@ int systemd_journal_query(BUFFER *wb, FACETS *facets, usec_t after_ut, usec_t be

142 buffer_json_member_add_string(wb, "type", "table");

143 buffer_json_member_add_time_t(wb, "update_every", 1);

144 buffer_json_member_add_string(wb, "help", SYSTEMD_JOURNAL_FUNCTION_DESCRIPTION);

145 + buffer_json_member_add_uint64(wb, "last_modified", first_msg_ut);

146

147 facets_report(facets, wb);

148

@@ -341,7 +348,7 @@ static void function_systemd_journal(const char *transaction, char *function, ch

348 buffer_flush(wb);

349 buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAY_ITEMS);

350

344 - FACETS *facets = facets_create(50, 0, FACETS_OPTION_ALL_KEYS_FTS,

351 + FACETS *facets = facets_create(50, FACETS_OPTION_ALL_KEYS_FTS,

352 SYSTEMD_ALWAYS_VISIBLE_KEYS,

353 SYSTEMD_KEYS_INCLUDED_IN_FACETS,

354 SYSTEMD_KEYS_EXCLUDED_FROM_FACETS);

@@ -349,10 +356,12 @@ static void function_systemd_journal(const char *transaction, char *function, ch

356 facets_accepted_param(facets, JOURNAL_PARAMETER_AFTER);

357 facets_accepted_param(facets, JOURNAL_PARAMETER_BEFORE);

358 facets_accepted_param(facets, JOURNAL_PARAMETER_ANCHOR);

359 + facets_accepted_param(facets, JOURNAL_PARAMETER_DIRECTION);

360 facets_accepted_param(facets, JOURNAL_PARAMETER_LAST);

361 facets_accepted_param(facets, JOURNAL_PARAMETER_QUERY);

362 facets_accepted_param(facets, JOURNAL_PARAMETER_FACETS);

363 facets_accepted_param(facets, JOURNAL_PARAMETER_HISTOGRAM);

364 + facets_accepted_param(facets, JOURNAL_PARAMETER_IF_MODIFIED_SINCE);

365

366 // register the fields in the order you want them on the dashboard

367

@@ -382,7 +391,9 @@ static void function_systemd_journal(const char *transaction, char *function, ch

391

392 time_t after_s = 0, before_s = 0;

393 usec_t anchor = 0;

394 + usec_t if_modified_since = 0;

395 size_t last = 0;

396 + FACETS_ANCHOR_DIRECTION direction = FACETS_ANCHOR_DIRECTION_BACKWARD;

397 const char *query = NULL;

398 const char *chart = NULL;

399

@@ -404,9 +415,15 @@ static void function_systemd_journal(const char *transaction, char *function, ch

415 else if(strncmp(keyword, JOURNAL_PARAMETER_BEFORE ":", sizeof(JOURNAL_PARAMETER_BEFORE ":") - 1) == 0) {

416 before_s = str2l(&keyword[sizeof(JOURNAL_PARAMETER_BEFORE ":") - 1]);

417 }

418 + else if(strncmp(keyword, JOURNAL_PARAMETER_IF_MODIFIED_SINCE ":", sizeof(JOURNAL_PARAMETER_IF_MODIFIED_SINCE ":") - 1) == 0) {

419 + if_modified_since = str2ull(&keyword[sizeof(JOURNAL_PARAMETER_IF_MODIFIED_SINCE ":") - 1], NULL);

420 + }

421 else if(strncmp(keyword, JOURNAL_PARAMETER_ANCHOR ":", sizeof(JOURNAL_PARAMETER_ANCHOR ":") - 1) == 0) {

422 anchor = str2ull(&keyword[sizeof(JOURNAL_PARAMETER_ANCHOR ":") - 1], NULL);

423 }

424 + else if(strncmp(keyword, JOURNAL_PARAMETER_DIRECTION ":", sizeof(JOURNAL_PARAMETER_DIRECTION ":") - 1) == 0) {

425 + direction = strcasecmp(&keyword[sizeof(JOURNAL_PARAMETER_DIRECTION ":") - 1], "forward") ? FACETS_ANCHOR_DIRECTION_FORWARD : FACETS_ANCHOR_DIRECTION_BACKWARD;

426 + }

427 else if(strncmp(keyword, JOURNAL_PARAMETER_LAST ":", sizeof(JOURNAL_PARAMETER_LAST ":") - 1) == 0) {

428 last = str2ul(&keyword[sizeof(JOURNAL_PARAMETER_LAST ":") - 1]);

429 }

@@ -483,7 +500,9 @@ static void function_systemd_journal(const char *transaction, char *function, ch

500

501 buffer_json_member_add_time_t(wb, "after", after_s);

502 buffer_json_member_add_time_t(wb, "before", before_s);

503 + buffer_json_member_add_uint64(wb, "if_modified_since", if_modified_since);

504 buffer_json_member_add_uint64(wb, "anchor", anchor);

505 + buffer_json_member_add_string(wb, "direction", direction == FACETS_ANCHOR_DIRECTION_FORWARD ? "forward" : "backward");

506 buffer_json_member_add_uint64(wb, "last", last);

507 buffer_json_member_add_string(wb, "query", query);

508 buffer_json_member_add_string(wb, "chart", chart);

@@ -491,21 +510,20 @@ static void function_systemd_journal(const char *transaction, char *function, ch

510 buffer_json_object_close(wb); // request

511

512 facets_set_items(facets, last);

494 - facets_set_anchor(facets, anchor);

513 + facets_set_anchor(facets, anchor, direction);

514 facets_set_query(facets, query);

515 facets_set_histogram(facets, chart ? chart : "PRIORITY", after_s * USEC_PER_SEC, before_s * USEC_PER_SEC);

516

517 int response = systemd_journal_query(wb, facets, after_s * USEC_PER_SEC, before_s * USEC_PER_SEC,

499 - now_monotonic_usec() + (timeout - 1) * USEC_PER_SEC);

518 + if_modified_since, now_monotonic_usec() + (timeout - 1) * USEC_PER_SEC);

519

520 if(response != HTTP_RESP_OK) {

502 - pluginsd_function_json_error(transaction, response, "failed");

521 + pluginsd_function_json_error_to_stdout(transaction, response, "failed");

522 goto cleanup;

523 }

524

506 - pluginsd_function_result_begin_to_stdout(transaction, HTTP_RESP_OK, "application/json", expires);

525 + pluginsd_function_result_begin_to_stdout(transaction, response, "application/json", expires);

526 fwrite(buffer_tostring(wb), buffer_strlen(wb), 1, stdout);

508 -

527 pluginsd_function_result_end_to_stdout();

528

529 cleanup:

@@ -545,7 +563,8 @@ static void *reader_main(void *arg __maybe_unused) {

563 if(strncmp(function, SYSTEMD_JOURNAL_FUNCTION_NAME, strlen(SYSTEMD_JOURNAL_FUNCTION_NAME)) == 0)

564 function_systemd_journal(transaction, function, buffer, PLUGINSD_LINE_MAX + 1, timeout);

565 else

548 - pluginsd_function_json_error(transaction, HTTP_RESP_NOT_FOUND, "No function with this name found in systemd-journal.plugin.");

566 + pluginsd_function_json_error_to_stdout(transaction, HTTP_RESP_NOT_FOUND,

567 + "No function with this name found in systemd-journal.plugin.");

568

569 fflush(stdout);

570 netdata_mutex_unlock(&mutex);

@@ -586,7 +605,7 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) {

605 // debug

606

607 if(argc == 2 && strcmp(argv[1], "debug") == 0) {

589 - char buf[] = "systemd-journal after:-86400 before:0 last:500";

608 + char buf[] = "systemd-journal after:-864000 before:0 last:500";

609 function_systemd_journal("123", buf, "", 0, 30);

610 exit(1);

611 }

libnetdata/facets/facets.c

+163 -86

@@ -9,11 +9,17 @@ static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row);

9 static inline void uint64_to_char(uint64_t num, char *out) {

10 static const char id_encoding_characters[64 + 1] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ.abcdefghijklmnopqrstuvwxyz_0123456789";

12 - int i;

13 - for(i = 10; i >= 0; --i) {

14 - out[i] = id_encoding_characters[num & 63];

15 - num >>= 6;

16 - }

12 + out[10] = id_encoding_characters[num & 63]; num >>= 6;

13 + out[9] = id_encoding_characters[num & 63]; num >>= 6;

14 + out[8] = id_encoding_characters[num & 63]; num >>= 6;

15 + out[7] = id_encoding_characters[num & 63]; num >>= 6;

16 + out[6] = id_encoding_characters[num & 63]; num >>= 6;

17 + out[5] = id_encoding_characters[num & 63]; num >>= 6;

18 + out[4] = id_encoding_characters[num & 63]; num >>= 6;

19 + out[3] = id_encoding_characters[num & 63]; num >>= 6;

20 + out[2] = id_encoding_characters[num & 63]; num >>= 6;

21 + out[1] = id_encoding_characters[num & 63]; num >>= 6;

22 + out[0] = id_encoding_characters[num & 63];

23 }

25 inline void facets_string_hash(const char *src, size_t len, char *out) {

@@ -79,7 +85,11 @@ struct facets {

85 SIMPLE_PATTERN *included_keys;

87 FACETS_OPTIONS options;

82 - usec_t anchor;

88 +

89 + struct {

90 + usec_t key;

91 + FACETS_ANCHOR_DIRECTION direction;

92 + } anchor;

94 SIMPLE_PATTERN *query; // the full text search pattern

95 size_t keys_filtered_by_query; // the number of fields we do full text search (constant)

@@ -99,7 +109,7 @@ struct facets {

109 char *chart;

110 bool enabled;

111 uint32_t slots;

102 - usec_t slot_width;

112 + usec_t slot_width_ut;

113 usec_t after_ut;

114 usec_t before_ut;

115 } histogram;

@@ -125,42 +135,51 @@ struct facets {

135

136 static usec_t calculate_histogram_bar_width(usec_t after_ut, usec_t before_ut) {

137 // Array of valid durations in seconds

128 - static time_t valid_durations[] = {

129 - 1,

130 - 15,

131 - 30,

138 + static time_t valid_durations_s[] = {

139 + 1, 2, 5, 10, 15, 30, // seconds

140 1 * 60, 2 * 60, 3 * 60, 5 * 60, 10 * 60, 15 * 60, 30 * 60, // minutes

141 1 * 3600, 2 * 3600, 6 * 3600, 8 * 3600, 12 * 3600, // hours

142 1 * 86400, 2 * 86400, 3 * 86400, 5 * 86400, 7 * 86400, 14 * 86400, // days

143 1 * (30*86400) // months

144 };

137 - static int array_size = sizeof(valid_durations) / sizeof(valid_durations[0]);

145 + static int array_size = sizeof(valid_durations_s) / sizeof(valid_durations_s[0]);

146

139 - usec_t duration = before_ut - after_ut;

140 - usec_t bar_width = 1 * 60;

147 + usec_t duration_ut = before_ut - after_ut;

148 + usec_t bar_width_ut = 1 * 60 * USEC_PER_SEC;

149

150 for (int i = array_size - 1; i >= 0; --i) {

143 - if (duration / (valid_durations[i] * 60) >= HISTOGRAM_COLUMNS) {

144 - bar_width = valid_durations[i] * 60;

151 + if (duration_ut / (valid_durations_s[i] * USEC_PER_SEC) >= HISTOGRAM_COLUMNS) {

152 + bar_width_ut = valid_durations_s[i] * USEC_PER_SEC;

153 break;

154 }

155 }

156

149 - return bar_width;

157 + return bar_width_ut;

158 }

159

160 static inline usec_t facets_histogram_slot_baseline_ut(FACETS *facets, usec_t ut) {

153 - usec_t delta = ut % facets->histogram.slot_width;

154 - return ut - delta;

161 + usec_t delta_ut = ut % facets->histogram.slot_width_ut;

162 + return ut - delta_ut;

163 }

164

165 void facets_set_histogram(FACETS *facets, const char *chart, usec_t after_ut, usec_t before_ut) {

166 + if(after_ut > before_ut) {

167 + usec_t t = after_ut;

168 + after_ut = before_ut;

169 + before_ut = t;

170 + }

171 +

172 facets->histogram.enabled = true;

173 facets->histogram.chart = chart ? strdupz(chart) : NULL;

160 - facets->histogram.slot_width = calculate_histogram_bar_width(after_ut, before_ut);

174 + facets->histogram.slot_width_ut = calculate_histogram_bar_width(after_ut, before_ut);

175 facets->histogram.after_ut = facets_histogram_slot_baseline_ut(facets, after_ut);

162 - facets->histogram.before_ut = facets_histogram_slot_baseline_ut(facets, before_ut) + facets->histogram.slot_width;

163 - facets->histogram.slots = (facets->histogram.before_ut - facets->histogram.after_ut) / facets->histogram.slot_width + 1;

176 + facets->histogram.before_ut = facets_histogram_slot_baseline_ut(facets, before_ut) + facets->histogram.slot_width_ut;

177 + facets->histogram.slots = (facets->histogram.before_ut - facets->histogram.after_ut) / facets->histogram.slot_width_ut + 1;

178 +

179 + if(facets->histogram.slots > 1000) {

180 + facets->histogram.slots = 1000 + 1;

181 + facets->histogram.slot_width_ut = (facets->histogram.before_ut - facets->histogram.after_ut) / 1000;

182 + }

183 }

184

185 static inline void facets_histogram_update_value(FACETS *facets, FACET_KEY *k __maybe_unused, FACET_VALUE *v, usec_t usec) {

@@ -178,7 +197,7 @@ static inline void facets_histogram_update_value(FACETS *facets, FACET_KEY *k __

197 if(base_ut > facets->histogram.before_ut)

198 base_ut = facets->histogram.before_ut;

199

181 - uint32_t slot = (base_ut - facets->histogram.after_ut) / facets->histogram.slot_width;

200 + uint32_t slot = (base_ut - facets->histogram.after_ut) / facets->histogram.slot_width_ut;

201

202 if(unlikely(slot >= facets->histogram.slots))

203 slot = facets->histogram.slots - 1;

@@ -186,9 +205,12 @@ static inline void facets_histogram_update_value(FACETS *facets, FACET_KEY *k __

205 v->histogram[slot]++;

206 }

207

189 -static inline void facets_histogram_value_names(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) {

208 +static inline void facets_histogram_value_names(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key, const char *first_key) {

209 buffer_json_member_add_array(wb, key);

210 {

211 + if(first_key)

212 + buffer_json_add_array_item_string(wb, first_key);

213 +

214 FACET_VALUE *v;

215 dfe_start_read(k->values, v) {

216 if(unlikely(!v->histogram))

@@ -478,25 +500,31 @@ static void facets_histogram_generate(FACETS *facets, FACET_KEY *k, BUFFER *wb)

500 buffer_json_member_add_uint64(wb, "sl", 1);

501 buffer_json_member_add_uint64(wb, "qr", 1);

502 }

481 - buffer_json_object_close(wb); // nodes;

503 + buffer_json_object_close(wb); // nodes

504 buffer_json_member_add_object(wb, "contexts");

505 {

506 buffer_json_member_add_uint64(wb, "sl", 1);

507 buffer_json_member_add_uint64(wb, "qr", 1);

508 }

487 - buffer_json_object_close(wb); // contexts;

509 + buffer_json_object_close(wb); // contexts

510 + buffer_json_member_add_object(wb, "instances");

511 + {

512 + buffer_json_member_add_uint64(wb, "sl", 1);

513 + buffer_json_member_add_uint64(wb, "qr", 1);

514 + }

515 + buffer_json_object_close(wb); // instances

516 buffer_json_member_add_object(wb, "dimensions");

517 {

518 buffer_json_member_add_uint64(wb, "sl", dimensions);

519 buffer_json_member_add_uint64(wb, "qr", dimensions);

520 }

493 - buffer_json_object_close(wb); // contexts;

521 + buffer_json_object_close(wb); // dimension

522 }

523 buffer_json_object_close(wb); // totals

524

525 buffer_json_member_add_object(wb, "result");

526 {

499 - facets_histogram_value_names(wb, facets, k, "labels");

527 + facets_histogram_value_names(wb, facets, k, "labels", "time");

528

529 buffer_json_member_add_object(wb, "point");

530 {

@@ -522,8 +550,8 @@ static void facets_histogram_generate(FACETS *facets, FACET_KEY *k, BUFFER *wb)

550 buffer_json_add_array_item_array(wb); // point

551

552 buffer_json_add_array_item_uint64(wb, v->histogram[i]);

525 - buffer_json_add_array_item_uint64(wb, 0);

526 - buffer_json_add_array_item_uint64(wb, 1);

553 + buffer_json_add_array_item_uint64(wb, 0); // arp - anomaly rate

554 + buffer_json_add_array_item_uint64(wb, 0); // pa - point annotation

555

556 buffer_json_array_close(wb); // point

557 }

@@ -531,7 +559,7 @@ static void facets_histogram_generate(FACETS *facets, FACET_KEY *k, BUFFER *wb)

559 }

560 buffer_json_array_close(wb); // row

561

534 - t += facets->histogram.slot_width;

562 + t += facets->histogram.slot_width_ut;

563 }

564 }

565 buffer_json_array_close(wb); //data

@@ -541,13 +569,13 @@ static void facets_histogram_generate(FACETS *facets, FACET_KEY *k, BUFFER *wb)

569 buffer_json_member_add_object(wb, "db");

570 {

571 buffer_json_member_add_uint64(wb, "tiers", 1);

544 - buffer_json_member_add_uint64(wb, "update_every", 1);

572 + buffer_json_member_add_uint64(wb, "update_every", facets->histogram.slot_width_ut / USEC_PER_SEC);

573 buffer_json_member_add_time_t(wb, "first_entry", facets->histogram.after_ut / USEC_PER_SEC);

574 buffer_json_member_add_time_t(wb, "last_entry", facets->histogram.before_ut / USEC_PER_SEC);

575 buffer_json_member_add_string(wb, "units", "events");

576 buffer_json_member_add_object(wb, "dimensions");

577 {

550 - facets_histogram_value_names(wb, facets, k, "ids");

578 + facets_histogram_value_names(wb, facets, k, "ids", NULL);

579 facets_histogram_value_units(wb, facets, k, "units");

580

581 buffer_json_member_add_object(wb, "sts");

@@ -569,7 +597,7 @@ static void facets_histogram_generate(FACETS *facets, FACET_KEY *k, BUFFER *wb)

597 buffer_json_member_add_uint64(wb, "tier", 0);

598 buffer_json_member_add_uint64(wb, "queries", 1);

599 buffer_json_member_add_uint64(wb, "points", count);

572 - buffer_json_member_add_time_t(wb, "update_every", 1);

600 + buffer_json_member_add_time_t(wb, "update_every", facets->histogram.slot_width_ut / USEC_PER_SEC);

601 buffer_json_member_add_time_t(wb, "first_entry", facets->histogram.after_ut / USEC_PER_SEC);

602 buffer_json_member_add_time_t(wb, "last_entry", facets->histogram.before_ut / USEC_PER_SEC);

603 }

@@ -582,11 +610,11 @@ static void facets_histogram_generate(FACETS *facets, FACET_KEY *k, BUFFER *wb)

610 buffer_json_member_add_object(wb, "view");

611 {

612 buffer_json_member_add_string(wb, "title", "Events Distribution");

585 - buffer_json_member_add_time_t(wb, "update_every", 1);

613 + buffer_json_member_add_time_t(wb, "update_every", facets->histogram.slot_width_ut / USEC_PER_SEC);

614 buffer_json_member_add_time_t(wb, "after", facets->histogram.after_ut / USEC_PER_SEC);

615 buffer_json_member_add_time_t(wb, "before", facets->histogram.before_ut / USEC_PER_SEC);

616 buffer_json_member_add_string(wb, "units", "events");

589 - buffer_json_member_add_string(wb, "chart_type", "stacked");

617 + buffer_json_member_add_string(wb, "chart_type", "stackedBar");

618 buffer_json_member_add_object(wb, "dimensions");

619 {

620 buffer_json_member_add_array(wb, "grouped_by");

@@ -595,8 +623,8 @@ static void facets_histogram_generate(FACETS *facets, FACET_KEY *k, BUFFER *wb)

623 }

624 buffer_json_array_close(wb); // grouped_by

625

598 - facets_histogram_value_names(wb, facets, k, "ids");

599 - facets_histogram_value_names(wb, facets, k, "names");

626 + facets_histogram_value_names(wb, facets, k, "ids", NULL);

627 + facets_histogram_value_names(wb, facets, k, "names", NULL);

628 facets_histogram_value_units(wb, facets, k, "units");

629

630 buffer_json_member_add_object(wb, "sts");

@@ -792,7 +820,7 @@ static void facet_key_delete_callback(const DICTIONARY_ITEM *item __maybe_unused

820

821 // ----------------------------------------------------------------------------

822

795 -FACETS *facets_create(uint32_t items_to_return, usec_t anchor, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys) {

823 +FACETS *facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys) {

824 FACETS *facets = callocz(1, sizeof(FACETS));

825 facets->options = options;

826 facets->keys = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_DONT_OVERWRITE_VALUE|DICT_OPTION_FIXED_SIZE, NULL, sizeof(FACET_KEY));

@@ -810,7 +838,8 @@ FACETS *facets_create(uint32_t items_to_return, usec_t anchor, FACETS_OPTIONS op

838 facets->visible_keys = simple_pattern_create(visible_keys, "|", SIMPLE_PATTERN_EXACT, true);

839

840 facets->max_items_to_return = items_to_return;

813 - facets->anchor = anchor;

841 + facets->anchor.key = 0;

842 + facets->anchor.direction = FACETS_ANCHOR_DIRECTION_BACKWARD;

843 facets->order = 1;

844

845 return facets;

@@ -877,8 +906,9 @@ void facets_set_items(FACETS *facets, uint32_t items) {

906 facets->max_items_to_return = items;

907 }

908

880 -void facets_set_anchor(FACETS *facets, usec_t anchor) {

881 - facets->anchor = anchor;

909 +void facets_set_anchor(FACETS *facets, usec_t anchor, FACETS_ANCHOR_DIRECTION direction) {

910 + facets->anchor.key = anchor;

911 + facets->anchor.direction = direction;

912 }

913

914 inline FACET_KEY *facets_register_facet_id(FACETS *facets, const char *key_id, FACET_KEY_OPTIONS options) {

@@ -1027,24 +1057,12 @@ static FACET_ROW *facets_row_create(FACETS *facets, usec_t usec, FACET_ROW *into

1057

1058 // ----------------------------------------------------------------------------

1059

1030 -static void facets_row_keep(FACETS *facets, usec_t usec) {

1031 - facets->operations.matched++;

1060 +static inline FACET_ROW *facets_row_keep_seek_to_position(FACETS *facets, usec_t usec) {

1061 + if(usec < facets->base->prev->usec)

1062 + return facets->base->prev;

1063

1033 - if(usec < facets->anchor) {

1034 - facets->operations.skips_before++;

1035 - return;

1036 - }

1037 -

1038 - if(unlikely(!facets->base)) {

1039 - facets->operations.last_added = facets_row_create(facets, usec, NULL);

1040 - DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next);

1041 - facets->items_to_return++;

1042 - facets->operations.first++;

1043 - return;

1044 - }

1045 -

1046 - if(likely(usec > facets->base->prev->usec))

1047 - facets->operations.last_added = facets->base->prev;

1064 + if(usec > facets->base->usec)

1065 + return facets->base;

1066

1067 FACET_ROW *last = facets->operations.last_added;

1068 while(last->prev != facets->base->prev && usec > last->prev->usec) {

@@ -1057,45 +1075,104 @@ static void facets_row_keep(FACETS *facets, usec_t usec) {

1075 facets->operations.forwards++;

1076 }

1077

1060 - if(facets->items_to_return >= facets->max_items_to_return) {

1061 - if(last == facets->base->prev && usec < last->usec) {

1062 - facets->operations.skips_after++;

1063 - return;

1064 - }

1065 - }

1078 + return last;

1079 +}

1080

1081 +static void facets_row_keep_first_entry(FACETS *facets, usec_t usec) {

1082 + facets->operations.last_added = facets_row_create(facets, usec, NULL);

1083 + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next);

1084 facets->items_to_return++;

1085 + facets->operations.first++;

1086 +}

1087 +

1088 +static void facets_row_keep(FACETS *facets, usec_t usec) {

1089 + facets->operations.matched++;

1090 +

1091 + if(facets->anchor.key) {

1092 + // we have an anchor key

1093 + // we don't want to keep rows on the other side of the direction

1094

1069 - if(usec > last->usec) {

1070 - if(facets->items_to_return > facets->max_items_to_return) {

1071 - facets->items_to_return--;

1072 - facets->operations.shifts++;

1073 - facets->operations.last_added = facets->base->prev;

1074 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next);

1075 - facets->operations.last_added = facets_row_create(facets, usec, facets->operations.last_added);

1095 + switch (facets->anchor.direction) {

1096 + default:

1097 + case FACETS_ANCHOR_DIRECTION_BACKWARD:

1098 + if (usec < facets->anchor.key) {

1099 + facets->operations.skips_before++;

1100 + return;

1101 + }

1102 + break;

1103 +

1104 + case FACETS_ANCHOR_DIRECTION_FORWARD:

1105 + if (usec > facets->anchor.key) {

1106 + facets->operations.skips_after++;

1107 + return;

1108 + }

1109 + break;

1110 }

1077 - DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next);

1078 - facets->operations.prepends++;

1111 }

1080 - else {

1081 - facets->operations.last_added = facets_row_create(facets, usec, NULL);

1082 - DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->base, facets->operations.last_added, prev, next);

1083 - facets->operations.appends++;

1112 +

1113 + if(unlikely(!facets->base)) {

1114 + // the first row to keep

1115 + facets_row_keep_first_entry(facets, usec);

1116 + return;

1117 }

1118

1086 - while(facets->items_to_return > facets->max_items_to_return) {

1087 - // we have to remove something

1119 + FACET_ROW *closest = facets_row_keep_seek_to_position(facets, usec);

1120 + FACET_ROW *to_replace = NULL;

1121

1089 - FACET_ROW *tmp = facets->base->prev;

1090 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, tmp, prev, next);

1091 - facets->items_to_return--;

1122 + if(likely(facets->items_to_return >= facets->max_items_to_return)) {

1123 + // we have enough items to return already

1124 +

1125 + switch(facets->anchor.direction) {

1126 + default:

1127 + case FACETS_ANCHOR_DIRECTION_BACKWARD:

1128 + if(closest == facets->base->prev && usec < closest->usec) {

1129 + // this is to the end of the list, belonging to the next page

1130 + facets->operations.skips_after++;

1131 + return;

1132 + }

1133

1093 - if(unlikely(facets->operations.last_added == tmp))

1094 - facets->operations.last_added = facets->base->prev;

1134 + // it seems we need to remove an item - the last one

1135 + to_replace = facets->base->prev;

1136 + if(closest == to_replace)

1137 + closest = to_replace->prev;

1138 +

1139 + break;

1140 +

1141 + case FACETS_ANCHOR_DIRECTION_FORWARD:

1142 + if(closest == facets->base && usec > closest->usec) {

1143 + // this is to the beginning of the list, belonging to the next page

1144 + facets->operations.skips_before++;

1145 + return;

1146 + }

1147 +

1148 + // it seems we need to remove an item - the first one

1149 + to_replace = facets->base;

1150 + if(closest == to_replace)

1151 + closest = to_replace->next;

1152 +

1153 + break;

1154 + }

1155

1096 - facets_row_free(facets, tmp);

1156 facets->operations.shifts++;

1157 + facets->items_to_return--;

1158 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(facets->base, to_replace, prev, next);

1159 + }

1160 +

1161 + internal_fatal(!closest, "FACETS: closest cannot be NULL");

1162 + internal_fatal(closest == to_replace, "FACETS: closest cannot be the same as to_replace");

1163 +

1164 + facets->operations.last_added = facets_row_create(facets, usec, to_replace);

1165 +

1166 + if(usec < closest->usec) {

1167 + DOUBLE_LINKED_LIST_INSERT_ITEM_AFTER_UNSAFE(facets->base, closest, facets->operations.last_added, prev, next);

1168 + facets->operations.appends++;

1169 }

1170 + else {

1171 + DOUBLE_LINKED_LIST_INSERT_ITEM_BEFORE_UNSAFE(facets->base, closest, facets->operations.last_added, prev, next);

1172 + facets->operations.prepends++;

1173 + }

1174 +

1175 + facets->items_to_return++;

1176 }

1177

1178 void facets_rows_begin(FACETS *facets) {

libnetdata/facets/facets.h

+7 -2

@@ -5,6 +5,11 @@

6 #define FACET_VALUE_UNSET "-"

8 +typedef enum __attribute__((packed)) {

9 + FACETS_ANCHOR_DIRECTION_FORWARD,

10 + FACETS_ANCHOR_DIRECTION_BACKWARD,

11 +} FACETS_ANCHOR_DIRECTION;

12 +

13 typedef enum __attribute__((packed)) {

14 FACET_KEY_OPTION_FACET = (1 << 0), // filterable values

15 FACET_KEY_OPTION_NO_FACET = (1 << 1), // non-filterable value

@@ -42,7 +47,7 @@ typedef enum __attribute__((packed)) {

47 FACETS_OPTION_ALL_KEYS_FTS = (1 << 1), // all keys are searchable by full text search

48 } FACETS_OPTIONS;

45 -FACETS *facets_create(uint32_t items_to_return, usec_t anchor, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys);

50 +FACETS *facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys);

51 void facets_destroy(FACETS *facets);

53 void facets_accepted_param(FACETS *facets, const char *param);

@@ -53,7 +58,7 @@ void facets_row_finished(FACETS *facets, usec_t usec);

58 FACET_KEY *facets_register_key_name(FACETS *facets, const char *key, FACET_KEY_OPTIONS options);

59 void facets_set_query(FACETS *facets, const char *query);

60 void facets_set_items(FACETS *facets, uint32_t items);

56 -void facets_set_anchor(FACETS *facets, usec_t anchor);

61 +void facets_set_anchor(FACETS *facets, usec_t anchor, FACETS_ANCHOR_DIRECTION direction);

62 FACET_KEY *facets_register_facet_id(FACETS *facets, const char *key_id, FACET_KEY_OPTIONS options);

63 void facets_register_facet_id_filter(FACETS *facets, const char *key_id, char *value_ids, FACET_KEY_OPTIONS options);

64 void facets_set_histogram(FACETS *facets, const char *chart, usec_t after_ut, usec_t before_ut);

libnetdata/http/http_defs.h

@@ -8,6 +8,7 @@

9 // HTTP_CODES 3XX Redirections

10 #define HTTP_RESP_MOVED_PERM 301

11 +#define HTTP_RESP_NOT_MODIFIED 304

12 #define HTTP_RESP_REDIR_TEMP 307

13 #define HTTP_RESP_REDIR_PERM 308

14 #define HTTP_RESP_HTTPS_UPGRADE 399