@cryptotaxi247 / netdata-1 / commits / 60c00474b

logs enhancements (#20027)

* allow filtering on fields, without making them facets * show all values on histogram * histogram now has both ids and names for dimensions * added colors for PRIORITY on linux and LEVEL on windows * Revert "added colors for PRIORITY on linux and LEVEL on windows" This reverts commit f93ce7265d68889e3c25c66177c442eea45553fd.

Costa Tsaousis committed Apr 1, 2025 at 22:42 UTC 60c00474b1b0b3675fa8326cdce96e8b551c428d

3 files changed +72 -49

src/libnetdata/facets/facets.c

+69 -47

@@ -1067,7 +1067,7 @@ static inline void facets_key_value_transformed(FACETS *facets, FACET_KEY *k, FA

1067 buffer_strcat(dst, facets_key_value_cached(k, v, facets->report.used_hashes_registry));

1068 }

1069

1070 -static inline void facets_histogram_value_names(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key, const char *first_key) {

1070 +static inline void facets_histogram_value_ids(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key, const char *first_key) {

1071 CLEAN_BUFFER *tb = buffer_create(0, NULL);

1072

1073 buffer_json_member_add_array(wb, key);

@@ -1078,9 +1078,25 @@ static inline void facets_histogram_value_names(BUFFER *wb, FACETS *facets __may

1078 if(k && k->values.enabled) {

1079 FACET_VALUE *v;

1080 foreach_value_in_key(k, v) {

1081 - if (unlikely(!v->histogram))

1082 - continue;

1081 + buffer_json_add_array_item_string(wb, facets_key_value_id(k ,v));

1082 + }

1083 + foreach_value_in_key_done(v);

1084 + }

1085 + }

1086 + buffer_json_array_close(wb); // key

1087 +}

1088 +

1089 +static inline void facets_histogram_value_names(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key, const char *first_key) {

1090 + CLEAN_BUFFER *tb = buffer_create(0, NULL);

1091

1092 + buffer_json_member_add_array(wb, key);

1093 + {

1094 + if(first_key)

1095 + buffer_json_add_array_item_string(wb, first_key);

1096 +

1097 + if(k && k->values.enabled) {

1098 + FACET_VALUE *v;

1099 + foreach_value_in_key(k, v) {

1100 facets_key_value_transformed(facets, k, v, tb, FACETS_TRANSFORM_HISTOGRAM);

1101 buffer_json_add_array_item_string(wb, buffer_tostring(tb));

1102 }

@@ -1096,9 +1112,6 @@ static inline void facets_histogram_value_colors(BUFFER *wb, FACETS *facets __ma

1112 if(k && k->values.enabled) {

1113 FACET_VALUE *v;

1114 foreach_value_in_key(k, v) {

1099 - if (unlikely(!v->histogram))

1100 - continue;

1101 -

1115 buffer_json_add_array_item_string(wb, v->color);

1116 }

1117 foreach_value_in_key_done(v);

@@ -1113,9 +1126,6 @@ static inline void facets_histogram_value_units(BUFFER *wb, FACETS *facets __may

1126 if(k && k->values.enabled) {

1127 FACET_VALUE *v;

1128 foreach_value_in_key(k, v) {

1116 - if (unlikely(!v->histogram))

1117 - continue;

1118 -

1129 buffer_json_add_array_item_string(wb, "events");

1130 }

1131 foreach_value_in_key_done(v);

@@ -1130,9 +1140,6 @@ static inline void facets_histogram_value_min(BUFFER *wb, FACETS *facets __maybe

1140 if(k && k->values.enabled) {

1141 FACET_VALUE *v;

1142 foreach_value_in_key(k, v) {

1133 - if (unlikely(!v->histogram))

1134 - continue;

1135 -

1143 buffer_json_add_array_item_uint64(wb, v->min);

1144 }

1145 foreach_value_in_key_done(v);

@@ -1147,9 +1154,6 @@ static inline void facets_histogram_value_max(BUFFER *wb, FACETS *facets __maybe

1154 if(k && k->values.enabled) {

1155 FACET_VALUE *v;

1156 foreach_value_in_key(k, v) {

1150 - if (unlikely(!v->histogram))

1151 - continue;

1152 -

1157 buffer_json_add_array_item_uint64(wb, v->max);

1158 }

1159 foreach_value_in_key_done(v);

@@ -1164,9 +1168,6 @@ static inline void facets_histogram_value_avg(BUFFER *wb, FACETS *facets __maybe

1168 if(k && k->values.enabled) {

1169 FACET_VALUE *v;

1170 foreach_value_in_key(k, v) {

1167 - if (unlikely(!v->histogram))

1168 - continue;

1169 -

1171 buffer_json_add_array_item_double(wb, (double) v->sum / (double) facets->histogram.slots);

1172 }

1173 foreach_value_in_key_done(v);

@@ -1181,9 +1182,6 @@ static inline void facets_histogram_value_arp(BUFFER *wb, FACETS *facets __maybe

1182 if(k && k->values.enabled) {

1183 FACET_VALUE *v;

1184 foreach_value_in_key(k, v) {

1184 - if (unlikely(!v->histogram))

1185 - continue;

1186 -

1185 buffer_json_add_array_item_uint64(wb, 0);

1186 }

1187 foreach_value_in_key_done(v);

@@ -1198,10 +1196,10 @@ static inline void facets_histogram_value_con(BUFFER *wb, FACETS *facets __maybe

1196 if(k && k->values.enabled) {

1197 FACET_VALUE *v;

1198 foreach_value_in_key(k, v) {

1201 - if (unlikely(!v->histogram))

1202 - continue;

1203 -

1204 - buffer_json_add_array_item_double(wb, (double) v->sum * 100.0 / (double) sum);

1199 + if(sum)

1200 + buffer_json_add_array_item_double(wb, (double) v->sum * 100.0 / (double) sum);

1201 + else

1202 + buffer_json_add_array_item_double(wb, 0.0);

1203 }

1204 foreach_value_in_key_done(v);

1205 }

@@ -1218,8 +1216,10 @@ static void facets_histogram_generate(FACETS *facets, FACET_KEY *k, BUFFER *wb)

1216 if(k && k->values.enabled) {

1217 FACET_VALUE *v;

1218 foreach_value_in_key(k, v) {

1221 - if (unlikely(!v->histogram))

1219 + if (unlikely(!v->histogram)) {

1220 + v->min = v->max = v->sum = 0;

1221 continue;

1222 + }

1223

1224 dimensions++;

1225

@@ -1377,25 +1377,41 @@ static void facets_histogram_generate(FACETS *facets, FACET_KEY *k, BUFFER *wb)

1377 FACET_VALUE *v;

1378

1379 foreach_value_in_key(k, v) {

1380 - if(unlikely(!v->histogram))

1381 - continue;

1380 + uint64_t d_sl, d_qr;

1381 + uint64_t d_min, d_max;

1382 + double d_avg, d_con;

1383 +

1384 + if(likely(v->histogram)) {

1385 + d_sl = d_qr = 1;

1386 + d_min = v->min;

1387 + d_max = v->max;

1388 + d_avg = (double) v->sum / (double) facets->histogram.slots;

1389 + d_con = (double) v->sum * 100.0 / (double) sum;

1390 + }

1391 + else {

1392 + d_sl = d_qr = 0;

1393 + d_min = d_max = 0;

1394 + d_avg = d_con = 0.0;

1395 + }

1396

1397 buffer_json_add_array_item_object(wb); // dimension

1398 {

1399 + buffer_json_member_add_string(wb, "id", facets_key_value_id(k, v));

1400 +

1401 facets_key_value_transformed(facets, k, v, tmp, FACETS_TRANSFORM_HISTOGRAM);

1386 - buffer_json_member_add_string(wb, "id", buffer_tostring(tmp));

1402 + buffer_json_member_add_string(wb, "nm", buffer_tostring(tmp));

1403 buffer_json_member_add_object(wb, "ds");

1404 {

1389 - buffer_json_member_add_uint64(wb, "sl", 1);

1390 - buffer_json_member_add_uint64(wb, "qr", 1);

1405 + buffer_json_member_add_uint64(wb, "sl", d_sl);

1406 + buffer_json_member_add_uint64(wb, "qr", d_qr);

1407 }

1408 buffer_json_object_close(wb); // ds

1409 buffer_json_member_add_object(wb, "sts");

1410 {

1395 - buffer_json_member_add_uint64(wb, "min", v->min);

1396 - buffer_json_member_add_uint64(wb, "max", v->max);

1397 - buffer_json_member_add_double(wb, "avg", (double)v->sum / (double)facets->histogram.slots);

1398 - buffer_json_member_add_double(wb, "con", (double)v->sum * 100.0 / (double)sum);

1411 + buffer_json_member_add_uint64(wb, "min", d_min);

1412 + buffer_json_member_add_uint64(wb, "max", d_max);

1413 + buffer_json_member_add_double(wb, "avg", d_avg);

1414 + buffer_json_member_add_double(wb, "con", d_con);

1415 }

1416 buffer_json_object_close(wb); // sts

1417 buffer_json_member_add_uint64(wb, "pri", pri++);

@@ -1469,12 +1485,13 @@ static void facets_histogram_generate(FACETS *facets, FACET_KEY *k, BUFFER *wb)

1485

1486 FACET_VALUE *v;

1487 foreach_value_in_key(k, v) {

1472 - if (unlikely(!v->histogram))

1473 - continue;

1474 -

1488 buffer_json_add_array_item_array(wb); // point

1489

1477 - buffer_json_add_array_item_uint64(wb, v->histogram[i]);

1490 + if(v->histogram)

1491 + buffer_json_add_array_item_uint64(wb, v->histogram[i]);

1492 + else

1493 + buffer_json_add_array_item_double(wb, NAN);

1494 +

1495 buffer_json_add_array_item_uint64(wb, 0); // arp - anomaly rate

1496 buffer_json_add_array_item_uint64(wb, 0); // pa - point annotation

1497

@@ -1501,7 +1518,8 @@ static void facets_histogram_generate(FACETS *facets, FACET_KEY *k, BUFFER *wb)

1518 buffer_json_member_add_string(wb, "units", "events");

1519 buffer_json_member_add_object(wb, "dimensions");

1520 {

1504 - facets_histogram_value_names(wb, facets, k, "ids", NULL);

1521 + facets_histogram_value_ids(wb, facets, k, "ids", NULL);

1522 + facets_histogram_value_names(wb, facets, k, "names", NULL);

1523 facets_histogram_value_units(wb, facets, k, "units");

1524

1525 buffer_json_member_add_object(wb, "sts");

@@ -1555,7 +1573,7 @@ static void facets_histogram_generate(FACETS *facets, FACET_KEY *k, BUFFER *wb)

1573 }

1574 buffer_json_array_close(wb); // grouped_by

1575

1558 - facets_histogram_value_names(wb, facets, k, "ids", NULL);

1576 + facets_histogram_value_ids(wb, facets, k, "ids", NULL);

1577 facets_histogram_value_names(wb, facets, k, "names", NULL);

1578 facets_histogram_value_colors(wb, facets, k, "colors");

1579 facets_histogram_value_units(wb, facets, k, "units");

@@ -1840,17 +1858,21 @@ inline FACET_KEY *facets_register_facet_id(FACETS *facets, const char *key_id, F

1858

1859 void facets_register_facet_filter_id(FACETS *facets, const char *key_id, const char *value_id, FACET_KEY_OPTIONS options) {

1860 FACET_KEY *k = facets_register_facet_id(facets, key_id, options);

1843 - if(k) {

1844 - if(is_valid_string_hash(value_id)) {

1845 - k->default_selected_for_values = false;

1846 - FACET_VALUE_ADD_OR_UPDATE_SELECTED(k, NULL, str_to_facets_hash(value_id));

1847 - }

1861 + if(k && is_valid_string_hash(value_id)) {

1862 + if(!(k->options & FACET_KEY_OPTION_FACET))

1863 + k->options |= FACET_KEY_OPTION_FILTER_ONLY;

1864 +

1865 + k->default_selected_for_values = false;

1866 + FACET_VALUE_ADD_OR_UPDATE_SELECTED(k, NULL, str_to_facets_hash(value_id));

1867 }

1868 }

1869

1870 void facets_register_facet_filter(FACETS *facets, const char *key, const char *value, FACET_KEY_OPTIONS options) {

1871 FACET_KEY *k = facets_register_facet(facets, key, options);

1872 if(k) {

1873 + if(!(k->options & FACET_KEY_OPTION_FACET))

1874 + k->options |= FACET_KEY_OPTION_FILTER_ONLY;

1875 +

1876 FACETS_HASH hash = FACETS_HASH_FUNCTION(value, strlen(value));

1877 k->default_selected_for_values = false;

1878 FACET_VALUE_ADD_OR_UPDATE_SELECTED(k, value, hash);

@@ -2591,7 +2613,7 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry)

2613 CLEAN_BUFFER *tb = buffer_create(0, NULL);

2614 FACET_KEY *k;

2615 foreach_key_in_facets(facets, k) {

2594 - if(!k->values.enabled || k->options & FACET_KEY_OPTION_HIDDEN)

2616 + if(!k->values.enabled || k->options & (FACET_KEY_OPTION_HIDDEN|FACET_KEY_OPTION_FILTER_ONLY))

2617 continue;

2618

2619 facets_sort_and_reorder_values(k);

src/libnetdata/facets/facets.h

@@ -38,6 +38,7 @@ typedef enum __attribute__((packed)) {

38 FACET_KEY_OPTION_EXPANDED_FILTER = (1 << 11), // the presentation should have this filter expanded by default

39 FACET_KEY_OPTION_PRETTY_XML = (1 << 12), // instruct the UI to parse this as an XML document

40 FACET_KEY_OPTION_HIDDEN = (1 << 13), // do not include this field in the response

41 + FACET_KEY_OPTION_FILTER_ONLY = (1 << 14), // the key is filterable, but not to be exposed as facet

42 } FACET_KEY_OPTIONS;

44 typedef enum __attribute__((packed)) {

src/libnetdata/facets/logs_query_status.h

+2 -2

@@ -442,7 +442,7 @@ static inline bool lqs_request_parse_json_payload(json_object *jobj, void *data,

442 else {

443 // Call facets_register_facet_id_filter for each value

444 facets_register_facet_filter(

445 - facets, key, value, FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_REORDER);

445 + facets, key, value, FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_REORDER);

446

447 rq->filters++;

448 }

@@ -643,7 +643,7 @@ static inline bool lqs_request_parse_GET(LOGS_QUERY_STATUS *lqs, BUFFER *wb, cha

643

644 facets_register_facet_filter_id(

645 facets, keyword, value,

646 - FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_REORDER);

646 + FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_REORDER);

647

648 buffer_json_add_array_item_string(wb, value);

649 rq->filters++;