@cryptotaxi247 / netdata-1 / commits / bfe964bfb

rrdcontext support for hidden charts (#13466)

* rrdcontext support for hidden charts * support unhidding charts

Costa Tsaousis committed Aug 1, 2022 at 12:17 UTC bfe964bfbb52bd21c23e941faa37be0ef00df95b

3 files changed +82 -12

database/rrdcontext.c

+78 -11

@@ -26,6 +26,7 @@ typedef enum {

26 RRD_FLAG_LIVE_RETENTION = (1 << 5), // we have got live retention from the database

27 RRD_FLAG_QUEUED = (1 << 6), // this context is currently queued to be dispatched to hub

28 RRD_FLAG_DONT_PROCESS = (1 << 7), // don't process updates for this object

29 + RRD_FLAG_HIDDEN = (1 << 8), // don't expose this to the hub or the API

31 RRD_FLAG_UPDATE_REASON_LOAD_SQL = (1 << 10), // this object has just been loaded from SQL

32 RRD_FLAG_UPDATE_REASON_NEW_OBJECT = (1 << 11), // this object has just been created

@@ -47,6 +48,7 @@ typedef enum {

48 RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD = (1 << 27), // this context belongs to a host that just disconnected

49 RRD_FLAG_UPDATE_REASON_DB_ROTATION = (1 << 28), // this context changed because of a db rotation

50 RRD_FLAG_UPDATE_REASON_UNUSED = (1 << 29), // this context is not used anymore

51 + RRD_FLAG_UPDATE_REASON_CHANGED_FLAGS = (1 << 30), // this context is not used anymore

52 } RRD_FLAGS;

54 #define RRD_FLAG_ALL_UPDATE_REASONS ( \

@@ -70,11 +72,13 @@ typedef enum {

72 |RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD \

73 |RRD_FLAG_UPDATE_REASON_DB_ROTATION \

74 |RRD_FLAG_UPDATE_REASON_UNUSED \

73 -)

75 + |RRD_FLAG_UPDATE_REASON_CHANGED_FLAGS \

76 + )

78 #define RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS ( \

79 RRD_FLAG_ARCHIVED \

80 |RRD_FLAG_DONT_PROCESS \

81 + |RRD_FLAG_HIDDEN \

82 |RRD_FLAG_ALL_UPDATE_REASONS \

83 )

@@ -151,6 +155,7 @@ static struct rrdcontext_reason {

155 { RRD_FLAG_UPDATE_REASON_CHANGED_NAME, "changed name", 60 * USEC_PER_SEC },

156 { RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD, "child disconnected", 30 * USEC_PER_SEC },

157 { RRD_FLAG_UPDATE_REASON_DB_ROTATION, "db rotation", 60 * USEC_PER_SEC },

158 + { RRD_FLAG_UPDATE_REASON_CHANGED_FLAGS, "changed flags", 60 * USEC_PER_SEC },

159

160 // terminator

161 { 0, NULL, 0 },

@@ -329,6 +334,9 @@ static void rrd_flags_to_buffer(RRD_FLAGS flags, BUFFER *wb) {

334

335 if(flags & RRD_FLAG_DONT_PROCESS)

336 buffer_strcat(wb, "DONT_PROCESS ");

337 +

338 + if(flags & RRD_FLAG_HIDDEN)

339 + buffer_strcat(wb, "HIDDEN ");

340 }

341

342 static void rrd_reasons_to_buffer(RRD_FLAGS flags, BUFFER *wb) {

@@ -768,6 +776,11 @@ static void rrdinstance_free(RRDINSTANCE *ri) {

776 }

777

778 static void rrdinstance_insert_callback(const char *id __maybe_unused, void *value, void *data) {

779 + static STRING *ml_anomaly_rates_id = NULL;

780 +

781 + if(unlikely(!ml_anomaly_rates_id))

782 + ml_anomaly_rates_id = string_strdupz(ML_ANOMALY_RATES_CHART_ID);

783 +

784 RRDINSTANCE *ri = value;

785

786 // link it to its parent

@@ -788,6 +801,17 @@ static void rrdinstance_insert_callback(const char *id __maybe_unused, void *val

801 ri->flags |= RRD_FLAG_OWN_LABELS;

802 }

803

804 + if(ri->rrdset) {

805 + if(unlikely((ri->rrdset->flags & RRDSET_FLAG_HIDDEN) || (ri->rrdset->state && ri->rrdset->state->is_ar_chart)))

806 + ri->flags |= RRD_FLAG_HIDDEN;

807 + else

808 + ri->flags &= ~RRD_FLAG_HIDDEN;

809 + }

810 +

811 + // we need this when loading from SQL

812 + if(unlikely(ri->id == ml_anomaly_rates_id))

813 + ri->flags |= RRD_FLAG_HIDDEN;

814 +

815 rrdmetrics_create(ri);

816 rrdinstance_log(ri, "INSERT");

817

@@ -878,12 +902,23 @@ static void rrdinstance_conflict_callback(const char *id __maybe_unused, void *o

902 if(ri->rrdset != ri_new->rrdset) {

903 ri->rrdset = ri_new->rrdset;

904

881 - if(ri->flags & RRD_FLAG_OWN_LABELS) {

905 + if(ri->rrdset && (ri->flags & RRD_FLAG_OWN_LABELS)) {

906 DICTIONARY *old = ri->rrdlabels;

907 ri->rrdlabels = ri->rrdset->state->chart_labels;

908 ri->flags &= ~RRD_FLAG_OWN_LABELS;

909 rrdlabels_destroy(old);

910 }

911 + else if(!ri->rrdset && !(ri->flags & RRD_FLAG_OWN_LABELS)) {

912 + ri->rrdlabels = rrdlabels_create();

913 + ri->flags |= RRD_FLAG_OWN_LABELS;

914 + }

915 + }

916 +

917 + if(ri->rrdset) {

918 + if(unlikely((ri->rrdset->flags & RRDSET_FLAG_HIDDEN) || (ri->rrdset->state && ri->rrdset->state->is_ar_chart)))

919 + ri->flags |= RRD_FLAG_HIDDEN;

920 + else

921 + ri->flags &= ~RRD_FLAG_HIDDEN;

922 }

923

924 ri->flags |= (ri_new->flags & RRD_FLAGS_ALLOWED_EXTERNALLY_ON_NEW_OBJECTS);

@@ -1234,11 +1269,24 @@ static inline void rrdinstance_updated_rrdset_name(RRDSET *st) {

1269 rrdinstance_trigger_updates(ri, false, true);

1270 }

1271

1272 +static inline void rrdinstance_updated_rrdset_flags_no_action(RRDINSTANCE *ri, RRDSET *st) {

1273 + if(unlikely(st->flags & (RRDSET_FLAG_ARCHIVED | RRDSET_FLAG_OBSOLETE)))

1274 + rrd_flag_set_archived(ri);

1275 +

1276 + if(unlikely((st->flags & RRDSET_FLAG_HIDDEN) && !(ri->flags & RRD_FLAG_HIDDEN))) {

1277 + ri->flags |= RRD_FLAG_HIDDEN;

1278 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_FLAGS);

1279 + }

1280 + else if(unlikely(!(st->flags & RRDSET_FLAG_HIDDEN) && (ri->flags & RRD_FLAG_HIDDEN))) {

1281 + ri->flags &= ~RRD_FLAG_HIDDEN;

1282 + rrd_flag_set_updated(ri, RRD_FLAG_UPDATE_REASON_CHANGED_FLAGS);

1283 + }

1284 +}

1285 +

1286 static inline void rrdinstance_updated_rrdset_flags(RRDSET *st) {

1287 RRDINSTANCE *ri = rrdset_get_rrdinstance(st);

1288

1240 - if(unlikely(st->flags & (RRDSET_FLAG_ARCHIVED | RRDSET_FLAG_OBSOLETE)))

1241 - rrd_flag_set_archived(ri);

1289 + rrdinstance_updated_rrdset_flags_no_action(ri, st);

1290

1291 ri->flags &= ~RRD_FLAG_DONT_PROCESS;

1292 rrdinstance_trigger_updates(ri, false, true);

@@ -1248,10 +1296,11 @@ static inline void rrdinstance_updated_rrdset_flags(RRDSET *st) {

1296 static inline void rrdinstance_collected_rrdset(RRDSET *st) {

1297 RRDINSTANCE *ri = rrdset_get_rrdinstance(st);

1298

1299 + rrdinstance_updated_rrdset_flags_no_action(ri, st);

1300 +

1301 if(unlikely(!rrd_flag_is_collected(ri)))

1302 rrd_flag_set_collected(ri);

1303

1254 - // the chart is collected, let's process it now

1304 if(unlikely(ri->flags & RRD_FLAG_DONT_PROCESS))

1305 ri->flags &= ~RRD_FLAG_DONT_PROCESS;

1306

@@ -1304,12 +1353,14 @@ static void rrdcontext_message_send_unsafe(RRDCONTEXT *rc, bool snapshot __maybe

1353 .deleted = rc->hub.deleted,

1354 };

1355

1307 - if(snapshot) {

1308 - if(!rc->hub.deleted)

1309 - contexts_snapshot_add_ctx_update(bundle, &message);

1356 + if(likely(!(rc->flags & RRD_FLAG_HIDDEN))) {

1357 + if (snapshot) {

1358 + if (!rc->hub.deleted)

1359 + contexts_snapshot_add_ctx_update(bundle, &message);

1360 + }

1361 + else

1362 + contexts_updated_add_ctx_update(bundle, &message);

1363 }

1311 - else

1312 - contexts_updated_add_ctx_update(bundle, &message);

1364 #endif

1365

1366 // store it to SQL

@@ -1584,10 +1635,13 @@ static void rrdcontext_trigger_updates(RRDCONTEXT *rc, bool force) {

1635 size_t min_priority = LONG_MAX;

1636 time_t min_first_time_t = LONG_MAX, max_last_time_t = 0;

1637 size_t instances_active = 0, instances_deleted = 0;

1587 - bool live_retention = true, currently_collected = false;

1638 + bool live_retention = true, currently_collected = false, hidden = true;

1639 {

1640 RRDINSTANCE *ri;

1641 dfe_start_read(rc->rrdinstances, ri) {

1642 + if(likely(!(ri->flags & RRD_FLAG_HIDDEN)))

1643 + hidden = false;

1644 +

1645 if(!(ri->flags & RRD_FLAG_LIVE_RETENTION))

1646 live_retention = false;

1647

@@ -1621,6 +1675,11 @@ static void rrdcontext_trigger_updates(RRDCONTEXT *rc, bool force) {

1675 dfe_done(ri);

1676 }

1677

1678 + if(hidden && !(rc->flags & RRD_FLAG_HIDDEN))

1679 + rc->flags |= RRD_FLAG_HIDDEN;

1680 + else if(!hidden && (rc->flags & RRD_FLAG_HIDDEN))

1681 + rc->flags &= ~RRD_FLAG_HIDDEN;

1682 +

1683 if(live_retention && !(rc->flags & RRD_FLAG_LIVE_RETENTION))

1684 rc->flags |= RRD_FLAG_LIVE_RETENTION;

1685 else if(!live_retention && (rc->flags & RRD_FLAG_LIVE_RETENTION))

@@ -2166,6 +2225,9 @@ static inline int rrdcontext_to_json_callback(const char *id, void *value, void

2225 time_t before = t_parent->before;

2226 bool has_filter = t_parent->chart_label_key || t_parent->chart_labels_filter || t_parent->chart_dimensions;

2227

2228 + if(unlikely((rc->flags & RRD_FLAG_HIDDEN) && !(options & RRDCONTEXT_OPTION_SHOW_HIDDEN)))

2229 + return 0;

2230 +

2231 if((rc->flags & RRD_FLAG_DELETED) && !(options & RRDCONTEXT_OPTION_SHOW_DELETED))

2232 return 0;

2233

@@ -2539,6 +2601,11 @@ static uint64_t rrdcontext_version_hash_with_callback(

2601

2602 rrdcontext_lock(rc);

2603

2604 + if(unlikely(rc->flags & RRD_FLAG_HIDDEN)) {

2605 + rrdcontext_unlock(rc);

2606 + continue;

2607 + }

2608 +

2609 if(unlikely(callback))

2610 callback(rc, snapshot, bundle);

2611

database/rrdcontext.h

+2 -1

@@ -44,10 +44,11 @@ typedef enum {

44 RRDCONTEXT_OPTION_SHOW_DELETED = (1 << 5),

45 RRDCONTEXT_OPTION_DEEPSCAN = (1 << 6),

46 RRDCONTEXT_OPTION_SHOW_UUIDS = (1 << 7),

47 + RRDCONTEXT_OPTION_SHOW_HIDDEN = (1 << 8),

48 RRDCONTEXT_OPTION_SKIP_ID = (1 << 31), // internal use

49 } RRDCONTEXT_TO_JSON_OPTIONS;

53 extern int rrdcontext_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, RRDCONTEXT_TO_JSON_OPTIONS options, const char *context, SIMPLE_PATTERN *chart_label_key, SIMPLE_PATTERN *chart_labels_filter, SIMPLE_PATTERN *chart_dimensions);

54 extern int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, RRDCONTEXT_TO_JSON_OPTIONS options, SIMPLE_PATTERN *chart_label_key, SIMPLE_PATTERN *chart_labels_filter, SIMPLE_PATTERN *chart_dimensions);

web/api/web_api_v1.c

@@ -399,6 +399,8 @@ static RRDCONTEXT_TO_JSON_OPTIONS rrdcontext_to_json_parse_options(char *o) {

399 options |= RRDCONTEXT_OPTION_SHOW_LABELS;

400 else if(!strcmp(tok, "deepscan"))

401 options |= RRDCONTEXT_OPTION_DEEPSCAN;

402 + else if(!strcmp(tok, "hidden"))

403 + options |= RRDCONTEXT_OPTION_SHOW_HIDDEN;

404 }

405

406 return options;