@cryptotaxi247 / netdata-1 / commits / cfe9dc428

extend the trimming window to avoid empty points at the end of queries (#15773)

Costa Tsaousis committed Aug 8, 2023 at 21:43 UTC cfe9dc42862f08670657688d2b6cddd6e2492c68

6 files changed +28 -13

collectors/apps.plugin/apps_plugin.c

+1 -1

@@ -4746,7 +4746,7 @@ static void function_processes(const char *transaction, char *function __maybe_u

4746 unsigned int io_divisor = 1024 * RATES_DETAIL;

4747

4748 BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX, NULL);

4749 - buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAYS);

4749 + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAY_ITEMS);

4750 buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);

4751 buffer_json_member_add_string(wb, "type", "table");

4752 buffer_json_member_add_time_t(wb, "update_every", update_every);

collectors/ebpf.plugin/ebpf_functions.c

+1 -1

@@ -206,7 +206,7 @@ static void ebpf_function_thread_manipulation(const char *transaction,

206 time_t expires = now_realtime_sec() + em->update_every;

207

208 BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX, NULL);

209 - buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAYS);

209 + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAY_ITEMS);

210 buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);

211 buffer_json_member_add_string(wb, "type", "table");

212 buffer_json_member_add_time_t(wb, "update_every", em->update_every);

collectors/systemd-journal.plugin/systemd-journal.c

+1 -1

@@ -323,7 +323,7 @@ static void function_systemd_journal(const char *transaction, char *function, ch

323

324 BUFFER *wb = buffer_create(0, NULL);

325 buffer_flush(wb);

326 - buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAYS);

326 + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAY_ITEMS);

327

328 FACETS *facets = facets_create(50, 0, FACETS_OPTION_ALL_KEYS_FTS,

329 SYSTEMD_ALWAYS_VISIBLE_KEYS,

libnetdata/buffer/buffer.h

+16 -5

@@ -71,7 +71,7 @@ typedef enum __attribute__ ((__packed__)) {

71 typedef enum __attribute__ ((__packed__)) {

72 BUFFER_JSON_OPTIONS_DEFAULT = 0,

73 BUFFER_JSON_OPTIONS_MINIFY = (1 << 0),

74 - BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAYS = (1 << 1),

74 + BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAY_ITEMS = (1 << 1),

75 } BUFFER_JSON_OPTIONS;

77 typedef struct web_buffer {

@@ -670,12 +670,16 @@ static inline void buffer_print_spaces(BUFFER *wb, size_t spaces) {

670 buffer_overflow_check(wb);

671 }

672

673 -static inline void buffer_print_json_comma_newline_spacing(BUFFER *wb) {

673 +static inline void buffer_print_json_comma(BUFFER *wb) {

674 if(wb->json.stack[wb->json.depth].count)

675 buffer_fast_strcat(wb, ",", 1);

676 +}

677 +

678 +static inline void buffer_print_json_comma_newline_spacing(BUFFER *wb) {

679 + buffer_print_json_comma(wb);

680

681 if((wb->json.options & BUFFER_JSON_OPTIONS_MINIFY) ||

678 - (wb->json.stack[wb->json.depth].type == BUFFER_JSON_ARRAY && !(wb->json.options & BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAYS)))

682 + (wb->json.stack[wb->json.depth].type == BUFFER_JSON_ARRAY && !(wb->json.options & BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAY_ITEMS)))

683 return;

684

685 buffer_fast_strcat(wb, "\n", 1);

@@ -799,7 +803,14 @@ static inline void buffer_json_member_add_array(BUFFER *wb, const char *key) {

803 }

804

805 static inline void buffer_json_add_array_item_array(BUFFER *wb) {

802 - buffer_print_json_comma_newline_spacing(wb);

806 + if(!(wb->json.options & BUFFER_JSON_OPTIONS_MINIFY) && wb->json.stack[wb->json.depth].type == BUFFER_JSON_ARRAY) {

807 + // an array inside another array

808 + buffer_print_json_comma(wb);

809 + buffer_fast_strcat(wb, "\n", 1);

810 + buffer_print_spaces(wb, wb->json.depth + 1);

811 + }

812 + else

813 + buffer_print_json_comma_newline_spacing(wb);

814

815 buffer_fast_strcat(wb, "[", 1);

816 wb->json.stack[wb->json.depth].count++;

@@ -918,7 +929,7 @@ static inline void buffer_json_array_close(BUFFER *wb) {

929 assert(wb->json.depth >= 0 && "BUFFER JSON: nothing is open to close it");

930 assert(wb->json.stack[wb->json.depth].type == BUFFER_JSON_ARRAY && "BUFFER JSON: an array is not open to close it");

931 #endif

921 - if(wb->json.options & BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAYS) {

932 + if(wb->json.options & BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAY_ITEMS) {

933 buffer_fast_strcat(wb, "\n", 1);

934 buffer_print_spaces(wb, wb->json.depth);

935 }

libnetdata/facets/facets.c

-1

@@ -622,7 +622,6 @@ void facets_row_finished(FACETS *facets, usec_t usec) {

622 // dfe_start_read(facets->keys, k) {

623 for(k = facets->keys_ll ; k ; k = k->next) {

624 if(!k->key_found_in_row) {

625 - internal_fatal(buffer_strlen(k->current_value.b), "key is not found in row but it has a current value");

625 // put the FACET_VALUE_UNSET value into it

626 facets_check_value(facets, k);

627 }

web/api/queries/query.c

+9 -4

@@ -2906,11 +2906,11 @@ static RRDR *rrd2rrdr_group_by_initialize(ONEWAYALLOC *owa, QUERY_TARGET *qt) {

2906 }

2907

2908 // initialize partial trimming

2909 - r->partial_data_trimming.max_update_every = update_every_max;

2909 + r->partial_data_trimming.max_update_every = update_every_max * 2;

2910 r->partial_data_trimming.expected_after =

2911 (!query_target_aggregatable(qt) &&

2912 - qt->window.before >= qt->window.now - update_every_max) ?

2913 - qt->window.before - update_every_max :

2912 + qt->window.before >= qt->window.now - r->partial_data_trimming.max_update_every) ?

2913 + qt->window.before - r->partial_data_trimming.max_update_every :

2914 qt->window.before;

2915 r->partial_data_trimming.trimmed_after = qt->window.before;

2916

@@ -3062,6 +3062,8 @@ static void rrdr2rrdr_group_by_partial_trimming(RRDR *r) {

3062 if(unlikely(i < 0))

3063 return;

3064

3065 + // internal_error(true, "Found trimmable index %zd (from 0 to %zu)", i, r->n - 1);

3066 +

3067 size_t last_row_gbc = 0;

3068 for (; i < (ssize_t)r->n; i++) {

3069 size_t row_gbc = 0;

@@ -3072,8 +3074,11 @@ static void rrdr2rrdr_group_by_partial_trimming(RRDR *r) {

3074 row_gbc += r->gbc[ i * r->d + d ];

3075 }

3076

3075 - if (unlikely(r->t[i] >= trimmable_after && row_gbc < last_row_gbc)) {

3077 + // internal_error(true, "GBC of index %zd is %zu", i, row_gbc);

3078 +

3079 + if (unlikely(r->t[i] >= trimmable_after && (row_gbc < last_row_gbc || !row_gbc))) {

3080 // discard the rest of the points

3081 + // internal_error(true, "Discarding points %zd to %zu", i, r->n - 1);

3082 r->partial_data_trimming.trimmed_after = r->t[i];

3083 r->rows = i;

3084 break;