@cryptotaxi247 / netdata-1 / commits / 43c749b07

Obvious memory reductions (#15204)

* remove rd->update_every * reduce amount of memory for RRDDIM * reorgnize rrddim->db entries * optimize rrdset and statsd * optimize dictionaries * RW_SPINLOCK for dictionaries * fix codeql warning * rw_spinlock improvements * remove obsolete assertion * fix crash on health_alarm_log_process() * use RW_SPINLOCK for AVL trees * add RW_SPINLOCK read/write trylock * pgc and mrg now use rw_spinlocks; cache line optimizations for mrg * thread tag of dbegnine init * append created datafile, lockless * make DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE friendly for lockless use * thread cancelability in spinlocks; optimize thread cancelability management * introduce a JudyL to index datafiles and use it during queries to quickly find the relevant files * use the last timestamp of each journal file for indexing * when the previous cannot be found, start from the beginning * add more stats to PDC to trace routing easier * rename spinlock functions * fix for spinlock renames * revert statsd socket statistics to size_t * turn fatal into internal_fatal() * show candidates always * show connected status and connection attempts

Costa Tsaousis committed Jun 19, 2023 at 23:19 UTC 43c749b07d07e79dae8111dcdb7bc1a46c3dda1b

55 files changed +1183 -884

collectors/freebsd.plugin/freebsd_sysctl.c

+2 -2

@@ -459,9 +459,9 @@ int do_dev_cpu_temperature(int update_every, usec_t dt) {

459 static RRDDIM **rd_pcpu_temperature;

460

461 if (unlikely(number_of_cpus != old_number_of_cpus)) {

462 - rd_pcpu_temperature = reallocz(rd_pcpu_temperature, sizeof(RRDDIM) * number_of_cpus);

462 + rd_pcpu_temperature = reallocz(rd_pcpu_temperature, sizeof(RRDDIM *) * number_of_cpus);

463 if (unlikely(number_of_cpus > old_number_of_cpus))

464 - memset(&rd_pcpu_temperature[old_number_of_cpus], 0, sizeof(RRDDIM) * (number_of_cpus - old_number_of_cpus));

464 + memset(&rd_pcpu_temperature[old_number_of_cpus], 0, sizeof(RRDDIM *) * (number_of_cpus - old_number_of_cpus));

465 }

466

467 if (unlikely(!st)) {

collectors/plugins.d/plugins_d.c

+12 -12

@@ -22,28 +22,28 @@ inline size_t pluginsd_initialize_plugin_directories()

22 }

24 static inline void plugin_set_disabled(struct plugind *cd) {

25 - netdata_spinlock_lock(&cd->unsafe.spinlock);

25 + spinlock_lock(&cd->unsafe.spinlock);

26 cd->unsafe.enabled = false;

27 - netdata_spinlock_unlock(&cd->unsafe.spinlock);

27 + spinlock_unlock(&cd->unsafe.spinlock);

28 }

30 bool plugin_is_enabled(struct plugind *cd) {

31 - netdata_spinlock_lock(&cd->unsafe.spinlock);

31 + spinlock_lock(&cd->unsafe.spinlock);

32 bool ret = cd->unsafe.enabled;

33 - netdata_spinlock_unlock(&cd->unsafe.spinlock);

33 + spinlock_unlock(&cd->unsafe.spinlock);

34 return ret;

35 }

37 static inline void plugin_set_running(struct plugind *cd) {

38 - netdata_spinlock_lock(&cd->unsafe.spinlock);

38 + spinlock_lock(&cd->unsafe.spinlock);

39 cd->unsafe.running = true;

40 - netdata_spinlock_unlock(&cd->unsafe.spinlock);

40 + spinlock_unlock(&cd->unsafe.spinlock);

41 }

43 static inline bool plugin_is_running(struct plugind *cd) {

44 - netdata_spinlock_lock(&cd->unsafe.spinlock);

44 + spinlock_lock(&cd->unsafe.spinlock);

45 bool ret = cd->unsafe.running;

46 - netdata_spinlock_unlock(&cd->unsafe.spinlock);

46 + spinlock_unlock(&cd->unsafe.spinlock);

47 return ret;

48 }

@@ -53,7 +53,7 @@ static void pluginsd_worker_thread_cleanup(void *arg)

54 worker_unregister();

56 - netdata_spinlock_lock(&cd->unsafe.spinlock);

56 + spinlock_lock(&cd->unsafe.spinlock);

58 cd->unsafe.running = false;

59 cd->unsafe.thread = 0;

@@ -61,7 +61,7 @@ static void pluginsd_worker_thread_cleanup(void *arg)

61 pid_t pid = cd->unsafe.pid;

62 cd->unsafe.pid = 0;

64 - netdata_spinlock_unlock(&cd->unsafe.spinlock);

64 + spinlock_unlock(&cd->unsafe.spinlock);

66 if (pid) {

67 siginfo_t info;

@@ -190,14 +190,14 @@ static void pluginsd_main_cleanup(void *data) {

190

191 struct plugind *cd;

192 for (cd = pluginsd_root; cd; cd = cd->next) {

193 - netdata_spinlock_lock(&cd->unsafe.spinlock);

193 + spinlock_lock(&cd->unsafe.spinlock);

194 if (cd->unsafe.enabled && cd->unsafe.running && cd->unsafe.thread != 0) {

195 info("PLUGINSD: 'host:%s', stopping plugin thread: %s",

196 rrdhost_hostname(cd->host), cd->id);

197

198 netdata_thread_cancel(cd->unsafe.thread);

199 }

200 - netdata_spinlock_unlock(&cd->unsafe.spinlock);

200 + spinlock_unlock(&cd->unsafe.spinlock);

201 }

202

203 info("PLUGINSD: cleanup completed.");

collectors/plugins.d/pluginsd_parser.c

+16 -16

@@ -11,7 +11,7 @@ static ssize_t send_to_plugin(const char *txt, void *data) {

11 return 0;

13 errno = 0;

14 - netdata_spinlock_lock(&parser->writer.spinlock);

14 + spinlock_lock(&parser->writer.spinlock);

15 ssize_t bytes = -1;

17 #ifdef ENABLE_HTTPS

@@ -24,7 +24,7 @@ static ssize_t send_to_plugin(const char *txt, void *data) {

24 else

25 error("PLUGINSD: cannot send command (SSL)");

27 - netdata_spinlock_unlock(&parser->writer.spinlock);

27 + spinlock_unlock(&parser->writer.spinlock);

28 return bytes;

29 }

30 #endif

@@ -39,7 +39,7 @@ static ssize_t send_to_plugin(const char *txt, void *data) {

39 else

40 fflush(parser->fp_output);

42 - netdata_spinlock_unlock(&parser->writer.spinlock);

42 + spinlock_unlock(&parser->writer.spinlock);

43 return bytes;

44 }

@@ -52,18 +52,18 @@ static ssize_t send_to_plugin(const char *txt, void *data) {

52 sent = write(parser->fd, &txt[bytes], total - bytes);

53 if(sent <= 0) {

54 error("PLUGINSD: cannot send command (fd)");

55 - netdata_spinlock_unlock(&parser->writer.spinlock);

55 + spinlock_unlock(&parser->writer.spinlock);

56 return -3;

57 }

58 bytes += sent;

59 }

60 while(bytes < total);

62 - netdata_spinlock_unlock(&parser->writer.spinlock);

62 + spinlock_unlock(&parser->writer.spinlock);

63 return (int)bytes;

64 }

66 - netdata_spinlock_unlock(&parser->writer.spinlock);

66 + spinlock_unlock(&parser->writer.spinlock);

67 error("PLUGINSD: cannot send command (no output socket/pipe/file given to plugins.d parser)");

68 return -4;

69 }

@@ -93,7 +93,7 @@ static inline RRDSET *pluginsd_get_chart_from_parent(void *user) {

93 static inline void pluginsd_lock_rrdset_data_collection(void *user) {

94 PARSER_USER_OBJECT *u = (PARSER_USER_OBJECT *) user;

95 if(u->st && !u->v2.locked_data_collection) {

96 - netdata_spinlock_lock(&u->st->data_collection_lock);

96 + spinlock_lock(&u->st->data_collection_lock);

97 u->v2.locked_data_collection = true;

98 }

99 }

@@ -101,7 +101,7 @@ static inline void pluginsd_lock_rrdset_data_collection(void *user) {

101 static inline bool pluginsd_unlock_rrdset_data_collection(void *user) {

102 PARSER_USER_OBJECT *u = (PARSER_USER_OBJECT *) user;

103 if(u->st && u->v2.locked_data_collection) {

104 - netdata_spinlock_unlock(&u->st->data_collection_lock);

104 + spinlock_unlock(&u->st->data_collection_lock);

105 u->v2.locked_data_collection = false;

106 return true;

107 }

@@ -1233,9 +1233,9 @@ PARSER_RC pluginsd_replay_begin(char **words, size_t num_words, void *user) {

1233 st->counter_done++;

1234

1235 // these are only needed for db mode RAM, SAVE, MAP, ALLOC

1236 - st->current_entry++;

1237 - if(st->current_entry >= st->entries)

1238 - st->current_entry -= st->entries;

1236 + st->db.current_entry++;

1237 + if(st->db.current_entry >= st->db.entries)

1238 + st->db.current_entry -= st->db.entries;

1239

1240 ((PARSER_USER_OBJECT *) user)->replay.start_time = start_time;

1241 ((PARSER_USER_OBJECT *) user)->replay.end_time = end_time;

@@ -1660,9 +1660,9 @@ PARSER_RC pluginsd_begin_v2(char **words, size_t num_words, void *user) {

1660 st->counter_done++;

1661

1662 // these are only needed for db mode RAM, SAVE, MAP, ALLOC

1663 - st->current_entry++;

1664 - if(st->current_entry >= st->entries)

1665 - st->current_entry -= st->entries;

1663 + st->db.current_entry++;

1664 + if(st->db.current_entry >= st->db.entries)

1665 + st->db.current_entry -= st->db.entries;

1666

1667 timing_step(TIMING_STEP_BEGIN2_STORE);

1668

@@ -1774,7 +1774,7 @@ PARSER_RC pluginsd_set_v2(char **words, size_t num_words, void *user) {

1774 rd->last_stored_value = value;

1775 rd->last_calculated_value = value;

1776 rd->collections_counter++;

1777 - rd->updated = true;

1777 + rrddim_set_updated(rd);

1778

1779 timing_step(TIMING_STEP_SET2_STORE);

1780

@@ -1831,7 +1831,7 @@ PARSER_RC pluginsd_end_v2(char **words __maybe_unused, size_t num_words __maybe_

1831 rrddim_foreach_read(rd, st) {

1832 rd->calculated_value = 0;

1833 rd->collected_value = 0;

1834 - rd->updated = false;

1834 + rrddim_clear_updated(rd);

1835 }

1836 rrddim_foreach_done(rd);

1837

collectors/statsd.plugin/statsd.c

+56 -76

@@ -34,7 +34,7 @@ typedef struct statsd_metric_gauge {

34 } STATSD_METRIC_GAUGE;

36 typedef struct statsd_metric_counter { // counter and meter

37 - long long value;

37 + collected_number value;

38 } STATSD_METRIC_COUNTER;

40 typedef struct statsd_histogram_extensions {

@@ -57,8 +57,8 @@ typedef struct statsd_histogram_extensions {

57 RRDDIM *rd_stddev;

58 //RRDDIM *rd_sum;

60 - size_t size;

61 - size_t used;

60 + uint32_t size;

61 + uint32_t used;

62 NETDATA_DOUBLE *values; // dynamic array of values collected

63 } STATSD_METRIC_HISTOGRAM_EXTENSIONS;

@@ -68,24 +68,22 @@ typedef struct statsd_metric_histogram { // histogram and timer

69 typedef struct statsd_metric_set {

70 DICTIONARY *dict;

71 - size_t unique;

71 } STATSD_METRIC_SET;

73 typedef struct statsd_metric_dictionary_item {

75 - size_t count;

74 + uint32_t count;

75 RRDDIM *rd;

76 } STATSD_METRIC_DICTIONARY_ITEM;

78 typedef struct statsd_metric_dictionary {

79 DICTIONARY *dict;

81 - size_t unique;

80 } STATSD_METRIC_DICTIONARY;

83 // --------------------------------------------------------------------------------------------------------------------

84 // this is a metric - for all types of metrics

88 -typedef enum statsd_metric_options {

86 +typedef enum __attribute__((packed)) statsd_metric_options {

87 STATSD_METRIC_OPTION_NONE = 0x00000000, // no options set

88 STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED = 0x00000001, // do not update the chart dimension, when this metric is not collected

89 STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED = 0x00000002, // render a private chart for this metric

@@ -99,7 +97,7 @@ typedef enum statsd_metric_options {

97 STATSD_METRIC_OPTION_UPDATED_CHART_METADATA = 0x00000200, // set when the private chart metadata have been updated via tags

98 } STATS_METRIC_OPTIONS;

102 -typedef enum statsd_metric_type {

100 +typedef enum __attribute__((packed)) statsd_metric_type {

101 STATSD_METRIC_TYPE_GAUGE,

102 STATSD_METRIC_TYPE_COUNTER,

103 STATSD_METRIC_TYPE_METER,

@@ -118,7 +116,7 @@ typedef struct statsd_metric {

116

117 // metadata about data collection

118 collected_number events; // the number of times this metric has been collected (never resets)

121 - size_t count; // the number of times this metric has been collected since the last flush

119 + uint32_t count; // the number of times this metric has been collected since the last flush

120

121 // the actual collected data

122 union {

@@ -151,22 +149,21 @@ typedef struct statsd_metric {

149

150 typedef struct statsd_index {

151 char *name; // the name of the index of metrics

154 - size_t events; // the number of events processed for this index

155 - size_t metrics; // the number of metrics in this index

156 - size_t useful; // the number of useful metrics in this index

152 + uint32_t events; // the number of events processed for this index

153 + uint32_t metrics; // the number of metrics in this index

154 + uint32_t useful; // the number of useful metrics in this index

155

156 + STATS_METRIC_OPTIONS default_options; // default options for all metrics in this index

157 STATSD_METRIC_TYPE type; // the type of index

158 DICTIONARY *dict;

159

160 STATSD_METRIC *first_useful; // the linked list of useful metrics (new metrics are added in front)

162 -

163 - STATS_METRIC_OPTIONS default_options; // default options for all metrics in this index

161 } STATSD_INDEX;

162

163 // --------------------------------------------------------------------------------------------------------------------

164 // synthetic charts

165

169 -typedef enum statsd_app_chart_dimension_value_type {

166 +typedef enum __attribute__((packed)) statsd_app_chart_dimension_value_type {

167 STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS,

168 STATSD_APP_CHART_DIM_VALUE_TYPE_LAST,

169 STATSD_APP_CHART_DIM_VALUE_TYPE_AVERAGE,

@@ -183,18 +180,18 @@ typedef struct statsd_app_chart_dimension {

180 const char *metric; // the source metric name of this dimension

181 uint32_t metric_hash; // hash for fast string comparisons

182

186 - SIMPLE_PATTERN *metric_pattern; // set when the 'metric' is a simple pattern

187 -

188 - collected_number multiplier; // the multiplier of the dimension

189 - collected_number divisor; // the divisor of the dimension

183 + int32_t multiplier; // the multiplier of the dimension

184 + int32_t divisor; // the divisor of the dimension

185 RRDDIM_FLAGS flags; // the RRDDIM flags for this dimension

186 RRDDIM_OPTIONS options; // the RRDDIM options for this dimension

187 + RRD_ALGORITHM algorithm; // the algorithm of this dimension

188

189 STATSD_APP_CHART_DIM_VALUE_TYPE value_type; // which value to use of the source metric

190

191 + SIMPLE_PATTERN *metric_pattern; // set when the 'metric' is a simple pattern

192 +

193 RRDDIM *rd; // a pointer to the RRDDIM that has been created for this dimension

194 collected_number *value_ptr; // a pointer to the source metric value

197 - RRD_ALGORITHM algorithm; // the algorithm of this dimension

195

196 struct statsd_app_chart_dimension *next; // the next dimension for this chart

197 } STATSD_APP_CHART_DIM;

@@ -207,11 +204,11 @@ typedef struct statsd_app_chart {

204 const char *context;

205 const char *units;

206 const char *module;

210 - long priority;

207 + int32_t priority;

208 RRDSET_TYPE chart_type;

209 STATSD_APP_CHART_DIM *dimensions;

213 - size_t dimensions_count;

214 - size_t dimensions_linked_count;

210 + uint32_t dimensions_count;

211 + uint32_t dimensions_linked_count;

212

213 RRDSET *st;

214 struct statsd_app_chart *next;

@@ -222,8 +219,8 @@ typedef struct statsd_app {

219 SIMPLE_PATTERN *metrics;

220 STATS_METRIC_OPTIONS default_options;

221 RRD_MEMORY_MODE rrd_memory_mode;

222 + int32_t rrd_history_entries;

223 DICTIONARY *dict;

226 - long rrd_history_entries;

224

225 const char *source;

226 STATSD_APP_CHART *charts;

@@ -236,7 +233,7 @@ typedef struct statsd_app {

233 struct collection_thread_status {

234 SPINLOCK spinlock;

235 bool running;

239 - size_t max_sockets;

236 + uint32_t max_sockets;

237

238 netdata_thread_t thread;

239 };

@@ -262,23 +259,22 @@ static struct statsd {

259 size_t udp_packets_received;

260 size_t udp_bytes_read;

261

265 - int enabled;

266 - int update_every;

262 + int32_t update_every;

263 + bool enabled;

264 + bool private_charts_hidden;

265 SIMPLE_PATTERN *charts_for;

266

269 - size_t tcp_idle_timeout;

267 + uint32_t tcp_idle_timeout;

268 collected_number decimal_detail;

271 - size_t private_charts;

272 - size_t max_private_charts_hard;

273 - long private_charts_rrd_history_entries;

274 - unsigned int private_charts_hidden:1;

269 + uint32_t private_charts;

270 + uint32_t max_private_charts_hard;

271

272 STATSD_APP *apps;

277 - size_t recvmmsg_size;

278 - size_t histogram_increase_step;

273 + uint32_t recvmmsg_size;

274 + uint32_t histogram_increase_step;

275 + uint32_t dictionary_max_unique;

276 double histogram_percentile;

277 char *histogram_percentile_str;

281 - size_t dictionary_max_unique;

278

279 int threads;

280 struct collection_thread_status *collection_threads_status;

@@ -287,7 +283,7 @@ static struct statsd {

283 } statsd = {

284 .enabled = 1,

285 .max_private_charts_hard = 1000,

290 - .private_charts_hidden = 0,

286 + .private_charts_hidden = false,

287 .recvmmsg_size = 10,

288 .decimal_detail = STATSD_DECIMAL_DETAIL,

289

@@ -571,13 +567,6 @@ static inline void statsd_process_histogram_or_timer(STATSD_METRIC *m, const cha

567 #define statsd_process_timer(m, value, sampling) statsd_process_histogram_or_timer(m, value, sampling, "timer")

568 #define statsd_process_histogram(m, value, sampling) statsd_process_histogram_or_timer(m, value, sampling, "histogram")

569

574 -static void dictionary_metric_set_value_insert_callback(const DICTIONARY_ITEM *item, void *value, void *data) {

575 - (void)item;

576 - (void)value;

577 - STATSD_METRIC *m = (STATSD_METRIC *)data;

578 - m->set.unique++;

579 -}

580 -

570 static inline void statsd_process_set(STATSD_METRIC *m, const char *value) {

571 if(!is_metric_useful_for_collection(m)) return;

572

@@ -594,11 +583,8 @@ static inline void statsd_process_set(STATSD_METRIC *m, const char *value) {

583 statsd_reset_metric(m);

584 }

585

597 - if (unlikely(!m->set.dict)) {

598 - m->set.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);

599 - dictionary_register_insert_callback(m->set.dict, dictionary_metric_set_value_insert_callback, m);

600 - m->set.unique = 0;

601 - }

586 + if (unlikely(!m->set.dict))

587 + m->set.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);

588

589 if(unlikely(value_is_zinit(value))) {

590 // magic loading of metric, without affecting anything

@@ -616,13 +602,6 @@ static inline void statsd_process_set(STATSD_METRIC *m, const char *value) {

602 }

603 }

604

619 -static void dictionary_metric_dict_value_insert_callback(const DICTIONARY_ITEM *item, void *value, void *data) {

620 - (void)item;

621 - (void)value;

622 - STATSD_METRIC *m = (STATSD_METRIC *)data;

623 - m->dictionary.unique++;

624 -}

625 -

605 static inline void statsd_process_dictionary(STATSD_METRIC *m, const char *value) {

606 if(!is_metric_useful_for_collection(m)) return;

607

@@ -634,11 +613,8 @@ static inline void statsd_process_dictionary(STATSD_METRIC *m, const char *value

613 if(unlikely(m->reset))

614 statsd_reset_metric(m);

615

637 - if (unlikely(!m->dictionary.dict)) {

638 - m->dictionary.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);

639 - dictionary_register_insert_callback(m->dictionary.dict, dictionary_metric_dict_value_insert_callback, m);

640 - m->dictionary.unique = 0;

641 - }

616 + if (unlikely(!m->dictionary.dict))

617 + m->dictionary.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);

618

619 if(unlikely(value_is_zinit(value))) {

620 // magic loading of metric, without affecting anything

@@ -647,7 +623,7 @@ static inline void statsd_process_dictionary(STATSD_METRIC *m, const char *value

623 STATSD_METRIC_DICTIONARY_ITEM *t = (STATSD_METRIC_DICTIONARY_ITEM *)dictionary_get(m->dictionary.dict, value);

624

625 if (unlikely(!t)) {

650 - if(!t && m->dictionary.unique >= statsd.dictionary_max_unique)

626 + if(!t && dictionary_entries(m->dictionary.dict) >= statsd.dictionary_max_unique)

627 value = "other";

628

629 t = (STATSD_METRIC_DICTIONARY_ITEM *)dictionary_set(m->dictionary.dict, value, NULL, sizeof(STATSD_METRIC_DICTIONARY_ITEM));

@@ -1096,9 +1072,9 @@ static int statsd_snd_callback(POLLINFO *pi, short int *events) {

1072

1073 void statsd_collector_thread_cleanup(void *data) {

1074 struct statsd_udp *d = data;

1099 - netdata_spinlock_lock(&d->status->spinlock);

1075 + spinlock_lock(&d->status->spinlock);

1076 d->status->running = false;

1101 - netdata_spinlock_unlock(&d->status->spinlock);

1077 + spinlock_unlock(&d->status->spinlock);

1078

1079 collector_info("cleaning up...");

1080

@@ -1121,9 +1097,9 @@ static bool statsd_should_stop(void) {

1097

1098 void *statsd_collector_thread(void *ptr) {

1099 struct collection_thread_status *status = ptr;

1124 - netdata_spinlock_lock(&status->spinlock);

1100 + spinlock_lock(&status->spinlock);

1101 status->running = true;

1126 - netdata_spinlock_unlock(&status->spinlock);

1102 + spinlock_unlock(&status->spinlock);

1103

1104 worker_register("STATSD");

1105 worker_register_job_name(WORKER_JOB_TYPE_TCP_CONNECTED, "tcp connect");

@@ -1255,7 +1231,7 @@ static STATSD_APP_CHART_DIM *add_dimension_to_app_chart(

1231 }

1232 chart->dimensions_count++;

1233

1258 - debug(D_STATSD, "Added dimension '%s' to chart '%s' of app '%s', for metric '%s', with type %u, multiplier " COLLECTED_NUMBER_FORMAT ", divisor " COLLECTED_NUMBER_FORMAT,

1234 + debug(D_STATSD, "Added dimension '%s' to chart '%s' of app '%s', for metric '%s', with type %u, multiplier %d, divisor %d",

1235 dim->name, chart->id, app->name, dim->metric, dim->value_type, dim->multiplier, dim->divisor);

1236

1237 return dim;

@@ -1909,7 +1885,7 @@ static inline void statsd_flush_set(STATSD_METRIC *m) {

1885

1886 int updated = 0;

1887 if(unlikely(!m->reset && m->count)) {

1912 - m->last = (collected_number)m->set.unique;

1888 + m->last = (collected_number)dictionary_entries(m->set.dict);

1889

1890 m->reset = 1;

1891 updated = 1;

@@ -1927,7 +1903,7 @@ static inline void statsd_flush_dictionary(STATSD_METRIC *m) {

1903

1904 int updated = 0;

1905 if(unlikely(!m->reset && m->count)) {

1930 - m->last = (collected_number)m->dictionary.unique;

1906 + m->last = (collected_number)dictionary_entries(m->dictionary.dict);

1907

1908 m->reset = 1;

1909 updated = 1;

@@ -1939,13 +1915,13 @@ static inline void statsd_flush_dictionary(STATSD_METRIC *m) {

1915 if(unlikely(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))))

1916 statsd_private_chart_dictionary(m);

1917

1942 - if(m->dictionary.unique >= statsd.dictionary_max_unique) {

1918 + if(dictionary_entries(m->dictionary.dict) >= statsd.dictionary_max_unique) {

1919 if(!(m->options & STATSD_METRIC_OPTION_COLLECTION_FULL_LOGGED)) {

1920 m->options |= STATSD_METRIC_OPTION_COLLECTION_FULL_LOGGED;

1921 collector_info(

1922 "STATSD dictionary '%s' reach max of %zu items - try increasing 'dictionaries max unique dimensions' in netdata.conf",

1923 m->name,

1948 - m->dictionary.unique);

1924 + dictionary_entries(m->dictionary.dict));

1925 }

1926 }

1927 }

@@ -2314,8 +2290,13 @@ static inline void statsd_flush_index_metrics(STATSD_INDEX *index, void (*flush_

2290

2291 if(unlikely(!(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_CHECKED))) {

2292 if(unlikely(statsd.private_charts >= statsd.max_private_charts_hard)) {

2317 - debug(D_STATSD, "STATSD: metric '%s' will not be charted, because the hard limit of the maximum number of charts has been reached.", m->name);

2318 - collector_info("STATSD: metric '%s' will not be charted, because the hard limit of the maximum number of charts (%zu) has been reached. Increase the number of charts by editing netdata.conf, [statsd] section.", m->name, statsd.max_private_charts_hard);

2293 + debug(D_STATSD, "STATSD: metric '%s' will not be charted, because the hard limit of the maximum number "

2294 + "of charts has been reached.", m->name);

2295 +

2296 + collector_info("STATSD: metric '%s' will not be charted, because the hard limit of the maximum number "

2297 + "of charts (%u) has been reached. Increase the number of charts by editing netdata.conf, "

2298 + "[statsd] section.", m->name, statsd.max_private_charts_hard);

2299 +

2300 m->options &= ~STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED;

2301 }

2302 else {

@@ -2366,7 +2347,7 @@ static void statsd_main_cleanup(void *data) {

2347 if (statsd.collection_threads_status) {

2348 int i;

2349 for (i = 0; i < statsd.threads; i++) {

2369 - netdata_spinlock_lock(&statsd.collection_threads_status[i].spinlock);

2350 + spinlock_lock(&statsd.collection_threads_status[i].spinlock);

2351 if(statsd.collection_threads_status[i].running) {

2352 collector_info("STATSD: stopping data collection thread %d...", i + 1);

2353 netdata_thread_cancel(statsd.collection_threads_status[i].thread);

@@ -2374,7 +2355,7 @@ static void statsd_main_cleanup(void *data) {

2355 else {

2356 collector_info("STATSD: data collection thread %d found stopped.", i + 1);

2357 }

2377 - netdata_spinlock_unlock(&statsd.collection_threads_status[i].spinlock);

2358 + spinlock_unlock(&statsd.collection_threads_status[i].spinlock);

2359 }

2360 }

2361

@@ -2466,7 +2447,6 @@ void *statsd_main(void *ptr) {

2447 config_get(CONFIG_SECTION_STATSD, "create private charts for metrics matching", "*"), NULL,

2448 SIMPLE_PATTERN_EXACT, true);

2449 statsd.max_private_charts_hard = (size_t)config_get_number(CONFIG_SECTION_STATSD, "max private charts hard limit", (long long)statsd.max_private_charts_hard);

2469 - statsd.private_charts_rrd_history_entries = (int)config_get_number(CONFIG_SECTION_STATSD, "private charts history", default_rrd_history_entries);

2450 statsd.decimal_detail = (collected_number)config_get_number(CONFIG_SECTION_STATSD, "decimal detail", (long long int)statsd.decimal_detail);

2451 statsd.tcp_idle_timeout = (size_t) config_get_number(CONFIG_SECTION_STATSD, "disconnect idle tcp clients after seconds", (long long int)statsd.tcp_idle_timeout);

2452 statsd.private_charts_hidden = (unsigned int)config_get_boolean(CONFIG_SECTION_STATSD, "private charts hidden", statsd.private_charts_hidden);

@@ -2549,7 +2529,7 @@ void *statsd_main(void *ptr) {

2529 statsd.collection_threads_status[i].max_sockets = max_sockets / statsd.threads;

2530 char tag[NETDATA_THREAD_TAG_MAX + 1];

2531 snprintfz(tag, NETDATA_THREAD_TAG_MAX, "STATSD_IN[%d]", i + 1);

2552 - netdata_spinlock_init(&statsd.collection_threads_status[i].spinlock);

2532 + spinlock_init(&statsd.collection_threads_status[i].spinlock);

2533 netdata_thread_create(&statsd.collection_threads_status[i].thread, tag, NETDATA_THREAD_OPTION_DEFAULT, statsd_collector_thread, &statsd.collection_threads_status[i]);

2534 }

2535

daemon/global_statistics.c

+1 -1

@@ -1718,7 +1718,7 @@ static void dbengine2_statistics_charts(void) {

1718 cache_efficiency_stats = rrdeng_get_cache_efficiency_stats();

1719

1720 mrg_stats_old = mrg_stats;

1721 - mrg_stats = mrg_get_statistics(main_mrg);

1721 + mrg_get_statistics(main_mrg, &mrg_stats);

1722

1723 struct rrdeng_buffer_sizes buffers = rrdeng_get_buffer_sizes();

1724 size_t buffers_total_size = buffers.handles + buffers.xt_buf + buffers.xt_io + buffers.pdc + buffers.descriptors +

daemon/main.c

+14 -14

@@ -60,7 +60,7 @@ SERVICE_THREAD *service_register(SERVICE_THREAD_TYPE thread_type, request_quit_t

60 SERVICE_THREAD *sth = NULL;

61 pid_t tid = gettid();

63 - netdata_spinlock_lock(&service_globals.lock);

63 + spinlock_lock(&service_globals.lock);

64 Pvoid_t *PValue = JudyLIns(&service_globals.pid_judy, tid, PJE0);

65 if(!*PValue) {

66 sth = callocz(1, sizeof(SERVICE_THREAD));

@@ -87,7 +87,7 @@ SERVICE_THREAD *service_register(SERVICE_THREAD_TYPE thread_type, request_quit_t

87 else {

88 sth = *PValue;

89 }

90 - netdata_spinlock_unlock(&service_globals.lock);

90 + spinlock_unlock(&service_globals.lock);

92 return sth;

93 }

@@ -95,13 +95,13 @@ SERVICE_THREAD *service_register(SERVICE_THREAD_TYPE thread_type, request_quit_t

95 void service_exits(void) {

96 pid_t tid = gettid();

98 - netdata_spinlock_lock(&service_globals.lock);

98 + spinlock_lock(&service_globals.lock);

99 Pvoid_t *PValue = JudyLGet(service_globals.pid_judy, tid, PJE0);

100 if(PValue) {

101 freez(*PValue);

102 JudyLDel(&service_globals.pid_judy, tid, PJE0);

103 }

104 - netdata_spinlock_unlock(&service_globals.lock);

104 + spinlock_unlock(&service_globals.lock);

105 }

106

107 bool service_running(SERVICE_TYPE service) {

@@ -124,7 +124,7 @@ bool service_running(SERVICE_TYPE service) {

124 void service_signal_exit(SERVICE_TYPE service) {

125 __atomic_and_fetch(&service_globals.running, ~(service), __ATOMIC_RELAXED);

126

127 - netdata_spinlock_lock(&service_globals.lock);

127 + spinlock_lock(&service_globals.lock);

128

129 Pvoid_t *PValue;

130 Word_t tid = 0;

@@ -133,14 +133,14 @@ void service_signal_exit(SERVICE_TYPE service) {

133 SERVICE_THREAD *sth = *PValue;

134

135 if((sth->services & service) && sth->request_quit_callback) {

136 - netdata_spinlock_unlock(&service_globals.lock);

136 + spinlock_unlock(&service_globals.lock);

137 sth->request_quit_callback(sth->data);

138 - netdata_spinlock_lock(&service_globals.lock);

138 + spinlock_lock(&service_globals.lock);

139 continue;

140 }

141 }

142

143 - netdata_spinlock_unlock(&service_globals.lock);

143 + spinlock_unlock(&service_globals.lock);

144 }

145

146 static void service_to_buffer(BUFFER *wb, SERVICE_TYPE service) {

@@ -187,7 +187,7 @@ static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) {

187 {

188 buffer_flush(thread_list);

189

190 - netdata_spinlock_lock(&service_globals.lock);

190 + spinlock_lock(&service_globals.lock);

191

192 Pvoid_t *PValue;

193 Word_t tid = 0;

@@ -217,15 +217,15 @@ static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) {

217 running_services |= sth->services & service;

218

219 if(sth->force_quit_callback) {

220 - netdata_spinlock_unlock(&service_globals.lock);

220 + spinlock_unlock(&service_globals.lock);

221 sth->force_quit_callback(sth->data);

222 - netdata_spinlock_lock(&service_globals.lock);

222 + spinlock_lock(&service_globals.lock);

223 continue;

224 }

225 }

226 }

227

228 - netdata_spinlock_unlock(&service_globals.lock);

228 + spinlock_unlock(&service_globals.lock);

229 }

230

231 service_signal_exit(service);

@@ -244,7 +244,7 @@ static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) {

244 running_services = 0;

245 buffer_flush(thread_list);

246

247 - netdata_spinlock_lock(&service_globals.lock);

247 + spinlock_lock(&service_globals.lock);

248

249 Pvoid_t *PValue;

250 Word_t tid = 0;

@@ -262,7 +262,7 @@ static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) {

262 }

263 }

264

265 - netdata_spinlock_unlock(&service_globals.lock);

265 + spinlock_unlock(&service_globals.lock);

266

267 if(running) {

268 log_countdown_ut -= (log_countdown_ut >= sleep_ut) ? sleep_ut : log_countdown_ut;

daemon/unit_test.c

+6 -5

@@ -1334,13 +1334,14 @@ int run_test(struct test *test)

1334 int errors = 0;

1335

1336 if(st->counter != test->result_entries) {

1337 - fprintf(stderr, " %s stored %zu entries, but we were expecting %lu, ### E R R O R ###\n", test->name, st->counter, test->result_entries);

1337 + fprintf(stderr, " %s stored %u entries, but we were expecting %lu, ### E R R O R ###\n",

1338 + test->name, st->counter, test->result_entries);

1339 errors++;

1340 }

1341

1342 unsigned long max = (st->counter < test->result_entries)?st->counter:test->result_entries;

1343 for(c = 0 ; c < max ; c++) {

1343 - NETDATA_DOUBLE v = unpack_storage_number(rd->db[c]);

1344 + NETDATA_DOUBLE v = unpack_storage_number(rd->db.data[c]);

1345 NETDATA_DOUBLE n = unpack_storage_number(pack_storage_number(test->results[c], SN_DEFAULT_FLAGS));

1346 int same = (roundndd(v * 10000000.0) == roundndd(n * 10000000.0))?1:0;

1347 fprintf(stderr, " %s/%s: checking position %lu (at %"PRId64" secs), expecting value " NETDATA_DOUBLE_FORMAT

@@ -1352,7 +1353,7 @@ int run_test(struct test *test)

1353 if(!same) errors++;

1354

1355 if(rd2) {

1355 - v = unpack_storage_number(rd2->db[c]);

1356 + v = unpack_storage_number(rd2->db.data[c]);

1357 n = test->results2[c];

1358 same = (roundndd(v * 10000000.0) == roundndd(n * 10000000.0))?1:0;

1359 fprintf(stderr, " %s/%s: checking position %lu (at %"PRId64" secs), expecting value " NETDATA_DOUBLE_FORMAT

@@ -1602,7 +1603,7 @@ int unit_test(long delay, long shift)

1603 fprintf(stderr, "\nPOSITION: c = %lu, EXPECTED VALUE %lu\n", c, (oincrement + c * increment + increment * (1000000 - shift) / 1000000 )* 10);

1604

1605 rrddim_foreach_read(rd, st) {

1605 - sn = rd->db[c];

1606 + sn = rd->db.data[c];

1607 cn = unpack_storage_number(sn);

1608 fprintf(stderr, "\t %s " NETDATA_DOUBLE_FORMAT " (PACKED AS " STORAGE_NUMBER_FORMAT ") -> ", rrddim_id(rd), cn, sn);

1609

@@ -1821,7 +1822,7 @@ static inline void rrddim_set_by_pointer_fake_time(RRDDIM *rd, collected_number

1822 rd->last_collected_time.tv_sec = now;

1823 rd->last_collected_time.tv_usec = 0;

1824 rd->collected_value = value;

1824 - rd->updated = 1;

1825 + rrddim_set_updated(rd);

1826

1827 rd->collections_counter++;

1828

database/contexts/api_v2.c

+16 -18

@@ -299,19 +299,17 @@ static void rrdhost_sender_to_json(BUFFER *wb, RRDHOST_STATUS *s, const char *ke

299 return;

300

301 buffer_json_member_add_object(wb, key);

302 + {

303 + buffer_json_member_add_uint64(wb, "id", s->stream.id);

304 + buffer_json_member_add_uint64(wb, "hops", s->stream.hops);

305 + buffer_json_member_add_string(wb, "status", rrdhost_streaming_status_to_string(s->stream.status));

306 + buffer_json_member_add_time_t(wb, "since", s->stream.since);

307 + buffer_json_member_add_time_t(wb, "age", s->now - s->stream.since);

308

303 - buffer_json_member_add_uint64(wb, "id", s->stream.id);

304 - buffer_json_member_add_uint64(wb, "hops", s->stream.hops);

305 - buffer_json_member_add_string(wb, "status", rrdhost_streaming_status_to_string(s->stream.status));

306 - buffer_json_member_add_time_t(wb, "since", s->stream.since);

307 - buffer_json_member_add_time_t(wb, "age", s->now - s->stream.since);

308 -

309 - if(s->stream.status == RRDHOST_STREAM_STATUS_OFFLINE)

310 - buffer_json_member_add_string(wb, "reason", s->stream.reason);

311 -

312 - if(s->stream.status == RRDHOST_STREAM_STATUS_REPLICATING || s->stream.status == RRDHOST_STREAM_STATUS_ONLINE) {

309 + if (s->stream.status == RRDHOST_STREAM_STATUS_OFFLINE)

310 + buffer_json_member_add_string(wb, "reason", s->stream.reason);

311

314 - if(s->stream.status == RRDHOST_STREAM_STATUS_REPLICATING) {

312 + if (s->stream.status == RRDHOST_STREAM_STATUS_REPLICATING) {

313 buffer_json_member_add_object(wb, "replication");

314 {

315 buffer_json_member_add_boolean(wb, "in_progress", s->stream.replication.in_progress);

@@ -346,6 +344,7 @@ static void rrdhost_sender_to_json(BUFFER *wb, RRDHOST_STATUS *s, const char *ke

344 struct rrdpush_destinations *d;

345 for (d = s->host->destinations; d; d = d->next) {

346 buffer_json_add_array_item_object(wb);

347 + buffer_json_member_add_uint64(wb, "attempts", d->attempts);

348 {

349

350 if (d->ssl) {

@@ -357,13 +356,12 @@ static void rrdhost_sender_to_json(BUFFER *wb, RRDHOST_STATUS *s, const char *ke

356

357 buffer_json_member_add_time_t(wb, "last_check", d->last_attempt);

358 buffer_json_member_add_time_t(wb, "age", s->now - d->last_attempt);

360 - buffer_json_member_add_string(wb, "last_error", d->last_error);

361 - buffer_json_member_add_string(wb, "last_handshake",

362 - stream_handshake_error_to_string(d->last_handshake));

363 - buffer_json_member_add_time_t(wb, "next_check", d->postpone_reconnection_until);

364 - buffer_json_member_add_time_t(wb, "next_in",

365 - (d->postpone_reconnection_until > s->now) ?

366 - d->postpone_reconnection_until - s->now : 0);

359 + buffer_json_member_add_string_or_omit(wb, "last_error", d->last_error);

360 + buffer_json_member_add_string(wb, "last_handshake", stream_handshake_error_to_string(d->last_handshake));

361 + if(d->postpone_reconnection_until > s->now) {

362 + buffer_json_member_add_time_t(wb, "next_check", d->postpone_reconnection_until);

363 + buffer_json_member_add_time_t(wb, "next_in", d->postpone_reconnection_until - s->now);

364 + }

365 }

366 buffer_json_object_close(wb); // each candidate

367 }

database/contexts/query_target.c

+8 -8

@@ -131,10 +131,10 @@ void query_target_release(QUERY_TARGET *qt) {

131

132 qt->id[0] = '\0';

133

134 - netdata_spinlock_lock(&query_target_base.used.spinlock);

134 + spinlock_lock(&query_target_base.used.spinlock);

135 DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(query_target_base.used.base, qt, internal.prev, internal.next);

136 query_target_base.used.count--;

137 - netdata_spinlock_unlock(&query_target_base.used.spinlock);

137 + spinlock_unlock(&query_target_base.used.spinlock);

138

139 qt->internal.used = false;

140 thread_qt = NULL;

@@ -143,29 +143,29 @@ void query_target_release(QUERY_TARGET *qt) {

143 query_target_destroy(qt);

144 }

145 else {

146 - netdata_spinlock_lock(&query_target_base.available.spinlock);

146 + spinlock_lock(&query_target_base.available.spinlock);

147 DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(query_target_base.available.base, qt, internal.prev, internal.next);

148 query_target_base.available.count++;

149 - netdata_spinlock_unlock(&query_target_base.available.spinlock);

149 + spinlock_unlock(&query_target_base.available.spinlock);

150 }

151 }

152

153 static QUERY_TARGET *query_target_get(void) {

154 - netdata_spinlock_lock(&query_target_base.available.spinlock);

154 + spinlock_lock(&query_target_base.available.spinlock);

155 QUERY_TARGET *qt = query_target_base.available.base;

156 if (qt) {

157 DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(query_target_base.available.base, qt, internal.prev, internal.next);

158 query_target_base.available.count--;

159 }

160 - netdata_spinlock_unlock(&query_target_base.available.spinlock);

160 + spinlock_unlock(&query_target_base.available.spinlock);

161

162 if(unlikely(!qt))

163 qt = callocz(1, sizeof(*qt));

164

165 - netdata_spinlock_lock(&query_target_base.used.spinlock);

165 + spinlock_lock(&query_target_base.used.spinlock);

166 DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(query_target_base.used.base, qt, internal.prev, internal.next);

167 query_target_base.used.count++;

168 - netdata_spinlock_unlock(&query_target_base.used.spinlock);

168 + spinlock_unlock(&query_target_base.used.spinlock);

169

170 qt->internal.used = true;

171 qt->internal.queries++;

database/contexts/worker.c

+2 -2

@@ -171,7 +171,7 @@ static void rrdhost_update_cached_retention(RRDHOST *host, time_t first_time_s,

171 if(unlikely(!host))

172 return;

173

174 - netdata_spinlock_lock(&host->retention.spinlock);

174 + spinlock_lock(&host->retention.spinlock);

175

176 if(global) {

177 host->retention.first_time_s = first_time_s;

@@ -185,7 +185,7 @@ static void rrdhost_update_cached_retention(RRDHOST *host, time_t first_time_s,

185 host->retention.last_time_s = last_time_s;

186 }

187

188 - netdata_spinlock_unlock(&host->retention.spinlock);

188 + spinlock_unlock(&host->retention.spinlock);

189 }

190

191 void rrdcontext_recalculate_context_retention(RRDCONTEXT *rc, RRD_FLAGS reason, bool worker_jobs) {

database/engine/cache.c

+27 -29

@@ -112,8 +112,9 @@ struct pgc {

112 PGC_CACHE_LINE_PADDING(0);

113

114 struct pgc_index {

115 - netdata_rwlock_t rwlock;

115 + RW_SPINLOCK rw_spinlock;

116 Pvoid_t sections_judy;

117 + PGC_CACHE_LINE_PADDING(0);

118 } *index;

119

120 PGC_CACHE_LINE_PADDING(1);

@@ -222,43 +223,40 @@ static inline size_t pgc_indexing_partition(PGC *cache, Word_t metric_id) {

223 }

224

225 static inline void pgc_index_read_lock(PGC *cache, size_t partition) {

225 - netdata_rwlock_rdlock(&cache->index[partition].rwlock);

226 + rw_spinlock_read_lock(&cache->index[partition].rw_spinlock);

227 }

228 static inline void pgc_index_read_unlock(PGC *cache, size_t partition) {

228 - netdata_rwlock_unlock(&cache->index[partition].rwlock);

229 + rw_spinlock_read_unlock(&cache->index[partition].rw_spinlock);

230 }

230 -//static inline bool pgc_index_write_trylock(PGC *cache, size_t partition) {

231 -// return !netdata_rwlock_trywrlock(&cache->index[partition].rwlock);

232 -//}

231 static inline void pgc_index_write_lock(PGC *cache, size_t partition) {

234 - netdata_rwlock_wrlock(&cache->index[partition].rwlock);

232 + rw_spinlock_write_lock(&cache->index[partition].rw_spinlock);

233 }

234 static inline void pgc_index_write_unlock(PGC *cache, size_t partition) {

237 - netdata_rwlock_unlock(&cache->index[partition].rwlock);

235 + rw_spinlock_write_unlock(&cache->index[partition].rw_spinlock);

236 }

237

238 static inline bool pgc_ll_trylock(PGC *cache __maybe_unused, struct pgc_linked_list *ll) {

241 - return netdata_spinlock_trylock(&ll->spinlock);

239 + return spinlock_trylock(&ll->spinlock);

240 }

241

242 static inline void pgc_ll_lock(PGC *cache __maybe_unused, struct pgc_linked_list *ll) {

245 - netdata_spinlock_lock(&ll->spinlock);

243 + spinlock_lock(&ll->spinlock);

244 }

245

246 static inline void pgc_ll_unlock(PGC *cache __maybe_unused, struct pgc_linked_list *ll) {

249 - netdata_spinlock_unlock(&ll->spinlock);

247 + spinlock_unlock(&ll->spinlock);

248 }

249

250 static inline bool page_transition_trylock(PGC *cache __maybe_unused, PGC_PAGE *page) {

253 - return netdata_spinlock_trylock(&page->transition_spinlock);

251 + return spinlock_trylock(&page->transition_spinlock);

252 }

253

254 static inline void page_transition_lock(PGC *cache __maybe_unused, PGC_PAGE *page) {

257 - netdata_spinlock_lock(&page->transition_spinlock);

255 + spinlock_lock(&page->transition_spinlock);

256 }

257

258 static inline void page_transition_unlock(PGC *cache __maybe_unused, PGC_PAGE *page) {

261 - netdata_spinlock_unlock(&page->transition_spinlock);

259 + spinlock_unlock(&page->transition_spinlock);

260 }

261

262 // ----------------------------------------------------------------------------

@@ -267,9 +265,9 @@ static inline void page_transition_unlock(PGC *cache __maybe_unused, PGC_PAGE *p

265 static inline size_t cache_usage_per1000(PGC *cache, size_t *size_to_evict) {

266

267 if(size_to_evict)

270 - netdata_spinlock_lock(&cache->usage.spinlock);

268 + spinlock_lock(&cache->usage.spinlock);

269

272 - else if(!netdata_spinlock_trylock(&cache->usage.spinlock))

270 + else if(!spinlock_trylock(&cache->usage.spinlock))

271 return __atomic_load_n(&cache->usage.per1000, __ATOMIC_RELAXED);

272

273 size_t current_cache_size;

@@ -319,7 +317,7 @@ static inline size_t cache_usage_per1000(PGC *cache, size_t *size_to_evict) {

317 __atomic_store_n(&cache->stats.wanted_cache_size, wanted_cache_size, __ATOMIC_RELAXED);

318 __atomic_store_n(&cache->stats.current_cache_size, current_cache_size, __ATOMIC_RELAXED);

319

322 - netdata_spinlock_unlock(&cache->usage.spinlock);

320 + spinlock_unlock(&cache->usage.spinlock);

321

322 if(size_to_evict) {

323 size_t target = (size_t)((unsigned long long)wanted_cache_size * (unsigned long long)cache->config.evict_low_threshold_per1000 / 1000ULL);

@@ -422,7 +420,7 @@ static void pgc_section_pages_static_aral_init(void) {

420 static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER;

421

422 if(unlikely(!pgc_section_pages_aral)) {

425 - netdata_spinlock_lock(&spinlock);

423 + spinlock_lock(&spinlock);

424

425 // we have to check again

426 if(!pgc_section_pages_aral)

@@ -433,7 +431,7 @@ static void pgc_section_pages_static_aral_init(void) {

431 65536, NULL,

432 NULL, NULL, false, false);

433

436 - netdata_spinlock_unlock(&spinlock);

434 + spinlock_unlock(&spinlock);

435 }

436 }

437

@@ -1255,7 +1253,7 @@ static PGC_PAGE *page_add(PGC *cache, PGC_ENTRY *entry, bool *added) {

1253 page->update_every_s = entry->update_every_s,

1254 page->data = entry->data;

1255 page->assumed_size = page_assumed_size(cache, entry->size);

1258 - netdata_spinlock_init(&page->transition_spinlock);

1256 + spinlock_init(&page->transition_spinlock);

1257 page->link.prev = NULL;

1258 page->link.next = NULL;

1259

@@ -1378,7 +1376,7 @@ static PGC_PAGE *page_find_and_acquire(PGC *cache, Word_t section, Word_t metric

1376 Word_t time = start_time_s;

1377

1378 // find the previous page

1381 - page_ptr = JudyLLast(*pages_judy_pptr, &time, PJE0);

1379 + page_ptr = JudyLPrev(*pages_judy_pptr, &time, PJE0);

1380 if(unlikely(page_ptr == PJERR))

1381 fatal("DBENGINE CACHE: corrupted page in pages judy array #2");

1382

@@ -1779,11 +1777,11 @@ PGC *pgc_create(const char *name,

1777 cache->index = callocz(cache->config.partitions, sizeof(struct pgc_index));

1778

1779 for(size_t part = 0; part < cache->config.partitions ; part++)

1782 - netdata_rwlock_init(&cache->index[part].rwlock);

1780 + rw_spinlock_init(&cache->index[part].rw_spinlock);

1781

1784 - netdata_spinlock_init(&cache->hot.spinlock);

1785 - netdata_spinlock_init(&cache->dirty.spinlock);

1786 - netdata_spinlock_init(&cache->clean.spinlock);

1782 + spinlock_init(&cache->hot.spinlock);

1783 + spinlock_init(&cache->dirty.spinlock);

1784 + spinlock_init(&cache->clean.spinlock);

1785

1786 cache->hot.flags = PGC_PAGE_HOT;

1787 cache->hot.linked_list_in_sections_judy = true;

@@ -1853,8 +1851,8 @@ void pgc_destroy(PGC *cache) {

1851 else {

1852 pointer_destroy_index(cache);

1853

1856 - for(size_t part = 0; part < cache->config.partitions ; part++)

1857 - netdata_rwlock_destroy(&cache->index[part].rwlock);

1854 +// for(size_t part = 0; part < cache->config.partitions ; part++)

1855 +// netdata_rwlock_destroy(&cache->index[part].rw_spinlock);

1856

1857 #ifdef PGC_WITH_ARAL

1858 for(size_t part = 0; part < cache->config.partitions ; part++)

@@ -2091,7 +2089,7 @@ void pgc_open_cache_to_journal_v2(PGC *cache, Word_t section, unsigned datafile_

2089 }

2090

2091 struct section_pages *sp = *section_pages_pptr;

2094 - if(!netdata_spinlock_trylock(&sp->migration_to_v2_spinlock)) {

2092 + if(!spinlock_trylock(&sp->migration_to_v2_spinlock)) {

2093 info("DBENGINE: migration to journal v2 for datafile %u is postponed, another jv2 indexer is already running for this section", datafile_fileno);

2094 pgc_ll_unlock(cache, &cache->hot);

2095 return;

@@ -2205,7 +2203,7 @@ void pgc_open_cache_to_journal_v2(PGC *cache, Word_t section, unsigned datafile_

2203 pgc_ll_lock(cache, &cache->hot);

2204 }

2205

2208 - netdata_spinlock_unlock(&sp->migration_to_v2_spinlock);

2206 + spinlock_unlock(&sp->migration_to_v2_spinlock);

2207 pgc_ll_unlock(cache, &cache->hot);

2208

2209 // callback

database/engine/cache.h

+1 -1

@@ -31,7 +31,7 @@ typedef struct pgc_entry {

31 uint8_t *custom_data;

32 } PGC_ENTRY;

34 -#define PGC_CACHE_LINE_PADDING(x) uint8_t padding##x[128]

34 +#define PGC_CACHE_LINE_PADDING(x) uint8_t padding##x[64]

36 struct pgc_queue_statistics {

37 size_t entries;

database/engine/datafile.c

+36 -29

@@ -1,11 +1,15 @@

1 // SPDX-License-Identifier: GPL-3.0-or-later

2 #include "rrdengine.h"

4 -void datafile_list_insert(struct rrdengine_instance *ctx, struct rrdengine_datafile *datafile)

4 +void datafile_list_insert(struct rrdengine_instance *ctx, struct rrdengine_datafile *datafile, bool having_lock)

5 {

6 - uv_rwlock_wrlock(&ctx->datafiles.rwlock);

6 + if(!having_lock)

7 + uv_rwlock_wrlock(&ctx->datafiles.rwlock);

8 +

9 DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(ctx->datafiles.first, datafile, prev, next);

8 - uv_rwlock_wrunlock(&ctx->datafiles.rwlock);

10 +

11 + if(!having_lock)

12 + uv_rwlock_wrunlock(&ctx->datafiles.rwlock);

13 }

15 void datafile_list_delete_unsafe(struct rrdengine_instance *ctx, struct rrdengine_datafile *datafile)

@@ -27,9 +31,9 @@ static struct rrdengine_datafile *datafile_alloc_and_init(struct rrdengine_insta

32 datafile->users.available = true;

30 - netdata_spinlock_init(&datafile->users.spinlock);

31 - netdata_spinlock_init(&datafile->writers.spinlock);

32 - netdata_spinlock_init(&datafile->extent_queries.spinlock);

34 + spinlock_init(&datafile->users.spinlock);

35 + spinlock_init(&datafile->writers.spinlock);

36 + spinlock_init(&datafile->extent_queries.spinlock);

38 return datafile;

39 }

@@ -37,7 +41,7 @@ static struct rrdengine_datafile *datafile_alloc_and_init(struct rrdengine_insta

41 bool datafile_acquire(struct rrdengine_datafile *df, DATAFILE_ACQUIRE_REASONS reason) {

42 bool ret;

40 - netdata_spinlock_lock(&df->users.spinlock);

44 + spinlock_lock(&df->users.spinlock);

46 if(df->users.available) {

47 ret = true;

@@ -47,25 +51,25 @@ bool datafile_acquire(struct rrdengine_datafile *df, DATAFILE_ACQUIRE_REASONS re

51 else

52 ret = false;

50 - netdata_spinlock_unlock(&df->users.spinlock);

54 + spinlock_unlock(&df->users.spinlock);

56 return ret;

57 }

59 void datafile_release(struct rrdengine_datafile *df, DATAFILE_ACQUIRE_REASONS reason) {

56 - netdata_spinlock_lock(&df->users.spinlock);

60 + spinlock_lock(&df->users.spinlock);

61 if(!df->users.lockers)

62 fatal("DBENGINE DATAFILE: cannot release a datafile that is not acquired");

64 df->users.lockers--;

65 df->users.lockers_by_reason[reason]--;

62 - netdata_spinlock_unlock(&df->users.spinlock);

66 + spinlock_unlock(&df->users.spinlock);

67 }

69 bool datafile_acquire_for_deletion(struct rrdengine_datafile *df) {

70 bool can_be_deleted = false;

68 - netdata_spinlock_lock(&df->users.spinlock);

72 + spinlock_lock(&df->users.spinlock);

73 df->users.available = false;

75 if(!df->users.lockers)

@@ -75,9 +79,9 @@ bool datafile_acquire_for_deletion(struct rrdengine_datafile *df) {

79 // there are lockers

81 // evict any pages referencing this in the open cache

78 - netdata_spinlock_unlock(&df->users.spinlock);

82 + spinlock_unlock(&df->users.spinlock);

83 pgc_open_evict_clean_pages_of_datafile(open_cache, df);

80 - netdata_spinlock_lock(&df->users.spinlock);

84 + spinlock_lock(&df->users.spinlock);

86 if(!df->users.lockers)

87 can_be_deleted = true;

@@ -86,12 +90,12 @@ bool datafile_acquire_for_deletion(struct rrdengine_datafile *df) {

90 // there are lockers still

92 // count the number of pages referencing this in the open cache

89 - netdata_spinlock_unlock(&df->users.spinlock);

93 + spinlock_unlock(&df->users.spinlock);

94 usec_t time_to_scan_ut = now_monotonic_usec();

95 size_t clean_pages_in_open_cache = pgc_count_clean_pages_having_data_ptr(open_cache, (Word_t)df->ctx, df);

96 size_t hot_pages_in_open_cache = pgc_count_hot_pages_having_data_ptr(open_cache, (Word_t)df->ctx, df);

97 time_to_scan_ut = now_monotonic_usec() - time_to_scan_ut;

94 - netdata_spinlock_lock(&df->users.spinlock);

98 + spinlock_lock(&df->users.spinlock);

100 if(!df->users.lockers)

101 can_be_deleted = true;

@@ -149,7 +153,7 @@ bool datafile_acquire_for_deletion(struct rrdengine_datafile *df) {

153 time_to_scan_ut);

154 }

155 }

152 - netdata_spinlock_unlock(&df->users.spinlock);

156 + spinlock_unlock(&df->users.spinlock);

157

158 return can_be_deleted;

159 }

@@ -410,11 +414,12 @@ static int scan_data_files(struct rrdengine_instance *ctx)

414 freez(datafiles);

415 return 0;

416 }

413 - if (matched_files == MAX_DATAFILES) {

417 +

418 + if (matched_files == MAX_DATAFILES)

419 error("DBENGINE: warning: hit maximum database engine file limit of %d files", MAX_DATAFILES);

415 - }

420 +

421 qsort(datafiles, matched_files, sizeof(*datafiles), scan_data_files_cmp);

417 - /* TODO: change this when tiering is implemented */

422 +

423 ctx->atomic.last_fileno = datafiles[matched_files - 1]->fileno;

424

425 for (failed_to_load = 0, i = 0 ; i < matched_files ; ++i) {

@@ -422,9 +427,9 @@ static int scan_data_files(struct rrdengine_instance *ctx)

427

428 datafile = datafiles[i];

429 ret = load_data_file(datafile);

425 - if (0 != ret) {

430 + if (0 != ret)

431 must_delete_pair = 1;

427 - }

432 +

433 journalfile = journalfile_alloc_and_init(datafile);

434 ret = journalfile_load(ctx, journalfile, datafile);

435 if (0 != ret) {

@@ -432,6 +437,7 @@ static int scan_data_files(struct rrdengine_instance *ctx)

437 close_data_file(datafile);

438 must_delete_pair = 1;

439 }

440 +

441 if (must_delete_pair) {

442 char path[RRDENG_PATH_MAX];

443

@@ -453,8 +459,9 @@ static int scan_data_files(struct rrdengine_instance *ctx)

459 }

460

461 ctx_current_disk_space_increase(ctx, datafile->pos + journalfile->unsafe.pos);

456 - datafile_list_insert(ctx, datafile);

462 + datafile_list_insert(ctx, datafile, false);

463 }

464 +

465 matched_files -= failed_to_load;

466 freez(datafiles);

467

@@ -462,7 +469,7 @@ static int scan_data_files(struct rrdengine_instance *ctx)

469 }

470

471 /* Creates a datafile and a journalfile pair */

465 -int create_new_datafile_pair(struct rrdengine_instance *ctx)

472 +int create_new_datafile_pair(struct rrdengine_instance *ctx, bool having_lock)

473 {

474 __atomic_add_fetch(&rrdeng_cache_efficiency_stats.datafile_creation_started, 1, __ATOMIC_RELAXED);

475

@@ -490,7 +497,7 @@ int create_new_datafile_pair(struct rrdengine_instance *ctx)

497 info("DBENGINE: created journal file \"%s\".", path);

498

499 ctx_current_disk_space_increase(ctx, datafile->pos + journalfile->unsafe.pos);

493 - datafile_list_insert(ctx, datafile);

500 + datafile_list_insert(ctx, datafile, having_lock);

501 ctx_last_fileno_increment(ctx);

502

503 return 0;

@@ -519,7 +526,7 @@ int init_data_files(struct rrdengine_instance *ctx)

526 } else if (0 == ret) {

527 info("DBENGINE: data files not found, creating in path \"%s\".", ctx->config.dbfiles_path);

528 ctx->atomic.last_fileno = 0;

522 - ret = create_new_datafile_pair(ctx);

529 + ret = create_new_datafile_pair(ctx, false);

530 if (ret) {

531 error("DBENGINE: failed to create data and journal files in path \"%s\".", ctx->config.dbfiles_path);

532 return ret;

@@ -527,7 +534,7 @@ int init_data_files(struct rrdengine_instance *ctx)

534 }

535 else {

536 if (ctx->loading.create_new_datafile_pair)

530 - create_new_datafile_pair(ctx);

537 + create_new_datafile_pair(ctx, false);

538

539 while(rrdeng_ctx_exceeded_disk_quota(ctx))

540 datafile_delete(ctx, ctx->datafiles.first, false, false);

@@ -569,11 +576,11 @@ void finalize_data_files(struct rrdengine_instance *ctx)

576 bool available = false;

577 do {

578 uv_rwlock_wrlock(&ctx->datafiles.rwlock);

572 - netdata_spinlock_lock(&datafile->writers.spinlock);

579 + spinlock_lock(&datafile->writers.spinlock);

580 available = (datafile->writers.running || datafile->writers.flushed_to_open_running) ? false : true;

581

582 if(!available) {

576 - netdata_spinlock_unlock(&datafile->writers.spinlock);

583 + spinlock_unlock(&datafile->writers.spinlock);

584 uv_rwlock_wrunlock(&ctx->datafiles.rwlock);

585 if(!logged) {

586 info("Waiting for writers to data file %u of tier %d to finish...", datafile->fileno, ctx->config.tier);

@@ -586,7 +593,7 @@ void finalize_data_files(struct rrdengine_instance *ctx)

593 journalfile_close(journalfile, datafile);

594 close_data_file(datafile);

595 datafile_list_delete_unsafe(ctx, datafile);

589 - netdata_spinlock_unlock(&datafile->writers.spinlock);

596 + spinlock_unlock(&datafile->writers.spinlock);

597 uv_rwlock_wrunlock(&ctx->datafiles.rwlock);

598

599 freez(journalfile);

database/engine/datafile.h

+3 -3

@@ -21,7 +21,7 @@ struct rrdengine_instance;

21 #endif

23 #define MIN_DATAFILE_SIZE (4LU * 1024LU * 1024LU)

24 -#define MAX_DATAFILES (65536) /* Supports up to 64TiB for now */

24 +#define MAX_DATAFILES (65536 * 4) /* Supports up to 64TiB for now */

25 #define TARGET_DATAFILES (50)

27 typedef enum __attribute__ ((__packed__)) {

@@ -74,14 +74,14 @@ bool datafile_acquire(struct rrdengine_datafile *df, DATAFILE_ACQUIRE_REASONS re

74 void datafile_release(struct rrdengine_datafile *df, DATAFILE_ACQUIRE_REASONS reason);

75 bool datafile_acquire_for_deletion(struct rrdengine_datafile *df);

77 -void datafile_list_insert(struct rrdengine_instance *ctx, struct rrdengine_datafile *datafile);

77 +void datafile_list_insert(struct rrdengine_instance *ctx, struct rrdengine_datafile *datafile, bool having_lock);

78 void datafile_list_delete_unsafe(struct rrdengine_instance *ctx, struct rrdengine_datafile *datafile);

79 void generate_datafilepath(struct rrdengine_datafile *datafile, char *str, size_t maxlen);

80 int close_data_file(struct rrdengine_datafile *datafile);

81 int unlink_data_file(struct rrdengine_datafile *datafile);

82 int destroy_data_file_unsafe(struct rrdengine_datafile *datafile);

83 int create_data_file(struct rrdengine_datafile *datafile);

84 -int create_new_datafile_pair(struct rrdengine_instance *ctx);

84 +int create_new_datafile_pair(struct rrdengine_instance *ctx, bool having_lock);

85 int init_data_files(struct rrdengine_instance *ctx);

86 void finalize_data_files(struct rrdengine_instance *ctx);

database/engine/journalfile.c

+158 -36

@@ -92,10 +92,10 @@ void journalfile_v1_extent_write(struct rrdengine_instance *ctx, struct rrdengin

92 io_descr->buf = wal->buf;

93 io_descr->bytes = wal->buf_size;

95 - netdata_spinlock_lock(&journalfile->unsafe.spinlock);

95 + spinlock_lock(&journalfile->unsafe.spinlock);

96 io_descr->pos = journalfile->unsafe.pos;

97 journalfile->unsafe.pos += wal->buf_size;

98 - netdata_spinlock_unlock(&journalfile->unsafe.spinlock);

98 + spinlock_unlock(&journalfile->unsafe.spinlock);

100 io_descr->req.data = wal;

101 io_descr->data = journalfile;

@@ -122,10 +122,128 @@ void journalfile_v1_generate_path(struct rrdengine_datafile *datafile, char *str

122 datafile->ctx->config.dbfiles_path, datafile->tier, datafile->fileno);

123 }

124

125 +// ----------------------------------------------------------------------------

126 +

127 +struct rrdengine_datafile *njfv2idx_find_and_acquire_j2_header(NJFV2IDX_FIND_STATE *s) {

128 + struct rrdengine_datafile *datafile = NULL;

129 +

130 + rw_spinlock_read_lock(&s->ctx->njfv2idx.spinlock);

131 +

132 + Pvoid_t *PValue = NULL;

133 +

134 + if(unlikely(!s->init)) {

135 + s->init = true;

136 + s->last = s->wanted_start_time_s;

137 +

138 + PValue = JudyLPrev(s->ctx->njfv2idx.JudyL, &s->last, PJE0);

139 + if (unlikely(PValue == PJERR))

140 + fatal("DBENGINE: NJFV2IDX corrupted judy array");

141 +

142 + if(!PValue) {

143 + s->last = 0;

144 + PValue = JudyLFirst(s->ctx->njfv2idx.JudyL, &s->last, PJE0);

145 + if (unlikely(PValue == PJERR))

146 + fatal("DBENGINE: NJFV2IDX corrupted judy array");

147 +

148 + if(!PValue)

149 + s->last = s->wanted_start_time_s;

150 + }

151 + }

152 +

153 + while(1) {

154 + if (likely(!PValue)) {

155 + PValue = JudyLNext(s->ctx->njfv2idx.JudyL, &s->last, PJE0);

156 + if (unlikely(PValue == PJERR))

157 + fatal("DBENGINE: NJFV2IDX corrupted judy array");

158 +

159 + if(!PValue) {

160 + // cannot find anything after that point

161 + datafile = NULL;

162 + break;

163 + }

164 + }

165 +

166 + datafile = *PValue;

167 + TIME_RANGE_COMPARE rc = is_page_in_time_range(datafile->journalfile->v2.first_time_s,

168 + datafile->journalfile->v2.last_time_s,

169 + s->wanted_start_time_s,

170 + s->wanted_end_time_s);

171 +

172 + if(rc == PAGE_IS_IN_RANGE) {

173 + // this is good to return

174 + break;

175 + }

176 + else if(rc == PAGE_IS_IN_THE_PAST) {

177 + // continue to get the next

178 + datafile = NULL;

179 + PValue = NULL;

180 + continue;

181 + }

182 + else /* PAGE_IS_IN_THE_FUTURE */ {

183 + // we finished - no more datafiles

184 + datafile = NULL;

185 + PValue = NULL;

186 + break;

187 + }

188 + }

189 +

190 + if(datafile)

191 + s->j2_header_acquired = journalfile_v2_data_acquire(datafile->journalfile, NULL,

192 + s->wanted_start_time_s,

193 + s->wanted_end_time_s);

194 + else

195 + s->j2_header_acquired = NULL;

196 +

197 + rw_spinlock_read_unlock(&s->ctx->njfv2idx.spinlock);

198 +

199 + return datafile;

200 +}

201 +

202 +static void njfv2idx_add(struct rrdengine_datafile *datafile) {

203 + internal_fatal(datafile->journalfile->v2.last_time_s <= 0, "DBENGINE: NJFV2IDX trying to index a journal file with invalid first_time_s");

204 +

205 + rw_spinlock_write_lock(&datafile->ctx->njfv2idx.spinlock);

206 + datafile->journalfile->njfv2idx.indexed_as = datafile->journalfile->v2.last_time_s;

207 +

208 + do {

209 + internal_fatal(datafile->journalfile->njfv2idx.indexed_as <= 0, "DBENGINE: NJFV2IDX journalfile is already indexed");

210 +

211 + Pvoid_t *PValue = JudyLIns(&datafile->ctx->njfv2idx.JudyL, datafile->journalfile->njfv2idx.indexed_as, PJE0);

212 + if (!PValue || PValue == PJERR)

213 + fatal("DBENGINE: NJFV2IDX corrupted judy array");

214 +

215 + if (unlikely(*PValue)) {

216 + // already there

217 + datafile->journalfile->njfv2idx.indexed_as++;

218 + }

219 + else {

220 + *PValue = datafile;

221 + break;

222 + }

223 + } while(0);

224 +

225 + rw_spinlock_write_unlock(&datafile->ctx->njfv2idx.spinlock);

226 +}

227 +

228 +static void njfv2idx_remove(struct rrdengine_datafile *datafile) {

229 + internal_fatal(!datafile->journalfile->njfv2idx.indexed_as, "DBENGINE: NJFV2IDX journalfile to remove is not indexed");

230 +

231 + rw_spinlock_write_lock(&datafile->ctx->njfv2idx.spinlock);

232 +

233 + int rc = JudyLDel(&datafile->ctx->njfv2idx.JudyL, datafile->journalfile->njfv2idx.indexed_as, PJE0);

234 + internal_fatal(!rc, "DBENGINE: NJFV2IDX cannot remove entry");

235 +

236 + datafile->journalfile->njfv2idx.indexed_as = 0;

237 +

238 + rw_spinlock_write_unlock(&datafile->ctx->njfv2idx.spinlock);

239 +}

240 +

241 +// ----------------------------------------------------------------------------

242 +

243 static struct journal_v2_header *journalfile_v2_mounted_data_get(struct rrdengine_journalfile *journalfile, size_t *data_size) {

244 struct journal_v2_header *j2_header = NULL;

245

128 - netdata_spinlock_lock(&journalfile->mmap.spinlock);

246 + spinlock_lock(&journalfile->mmap.spinlock);

247

248 if(!journalfile->mmap.data) {

249 journalfile->mmap.data = mmap(NULL, journalfile->mmap.size, PROT_READ, MAP_SHARED, journalfile->mmap.fd, 0);

@@ -136,9 +254,9 @@ static struct journal_v2_header *journalfile_v2_mounted_data_get(struct rrdengin

254 journalfile->mmap.data = NULL;

255 journalfile->mmap.size = 0;

256

139 - netdata_spinlock_lock(&journalfile->v2.spinlock);

257 + spinlock_lock(&journalfile->v2.spinlock);

258 journalfile->v2.flags &= ~(JOURNALFILE_FLAG_IS_AVAILABLE | JOURNALFILE_FLAG_IS_MOUNTED);

141 - netdata_spinlock_unlock(&journalfile->v2.spinlock);

259 + spinlock_unlock(&journalfile->v2.spinlock);

260

261 ctx_fs_error(journalfile->datafile->ctx);

262 }

@@ -150,9 +268,9 @@ static struct journal_v2_header *journalfile_v2_mounted_data_get(struct rrdengin

268 madvise_random(journalfile->mmap.data, journalfile->mmap.size);

269 madvise_dontneed(journalfile->mmap.data, journalfile->mmap.size);

270

153 - netdata_spinlock_lock(&journalfile->v2.spinlock);

271 + spinlock_lock(&journalfile->v2.spinlock);

272 journalfile->v2.flags |= JOURNALFILE_FLAG_IS_AVAILABLE | JOURNALFILE_FLAG_IS_MOUNTED;

155 - netdata_spinlock_unlock(&journalfile->v2.spinlock);

273 + spinlock_unlock(&journalfile->v2.spinlock);

274 }

275 }

276

@@ -163,7 +281,7 @@ static struct journal_v2_header *journalfile_v2_mounted_data_get(struct rrdengin

281 *data_size = journalfile->mmap.size;

282 }

283

166 - netdata_spinlock_unlock(&journalfile->mmap.spinlock);

284 + spinlock_unlock(&journalfile->mmap.spinlock);

285

286 return j2_header;

287 }

@@ -173,20 +291,20 @@ static bool journalfile_v2_mounted_data_unmount(struct rrdengine_journalfile *jo

291

292 if(!have_locks) {

293 if(!wait) {

176 - if (!netdata_spinlock_trylock(&journalfile->mmap.spinlock))

294 + if (!spinlock_trylock(&journalfile->mmap.spinlock))

295 return false;

296 }

297 else

180 - netdata_spinlock_lock(&journalfile->mmap.spinlock);

298 + spinlock_lock(&journalfile->mmap.spinlock);

299

300 if(!wait) {

183 - if(!netdata_spinlock_trylock(&journalfile->v2.spinlock)) {

184 - netdata_spinlock_unlock(&journalfile->mmap.spinlock);

301 + if(!spinlock_trylock(&journalfile->v2.spinlock)) {

302 + spinlock_unlock(&journalfile->mmap.spinlock);

303 return false;

304 }

305 }

306 else

189 - netdata_spinlock_lock(&journalfile->v2.spinlock);

307 + spinlock_lock(&journalfile->v2.spinlock);

308 }

309

310 if(!journalfile->v2.refcount) {

@@ -209,8 +327,8 @@ static bool journalfile_v2_mounted_data_unmount(struct rrdengine_journalfile *jo

327 }

328

329 if(!have_locks) {

212 - netdata_spinlock_unlock(&journalfile->v2.spinlock);

213 - netdata_spinlock_unlock(&journalfile->mmap.spinlock);

330 + spinlock_unlock(&journalfile->v2.spinlock);

331 + spinlock_unlock(&journalfile->mmap.spinlock);

332 }

333

334 return unmounted;

@@ -230,7 +348,7 @@ void journalfile_v2_data_unmount_cleanup(time_t now_s) {

348 for (datafile = ctx->datafiles.first; datafile; datafile = datafile->next) {

349 struct rrdengine_journalfile *journalfile = datafile->journalfile;

350

233 - if(!netdata_spinlock_trylock(&journalfile->v2.spinlock))

351 + if(!spinlock_trylock(&journalfile->v2.spinlock))

352 continue;

353

354 bool unmount = false;

@@ -244,7 +362,7 @@ void journalfile_v2_data_unmount_cleanup(time_t now_s) {

362 // 2 minutes have passed since last use

363 unmount = true;

364 }

247 - netdata_spinlock_unlock(&journalfile->v2.spinlock);

365 + spinlock_unlock(&journalfile->v2.spinlock);

366

367 if (unmount)

368 journalfile_v2_mounted_data_unmount(journalfile, false, false);

@@ -254,7 +372,7 @@ void journalfile_v2_data_unmount_cleanup(time_t now_s) {

372 }

373

374 struct journal_v2_header *journalfile_v2_data_acquire(struct rrdengine_journalfile *journalfile, size_t *data_size, time_t wanted_first_time_s, time_t wanted_last_time_s) {

257 - netdata_spinlock_lock(&journalfile->v2.spinlock);

375 + spinlock_lock(&journalfile->v2.spinlock);

376

377 bool has_data = (journalfile->v2.flags & JOURNALFILE_FLAG_IS_AVAILABLE);

378 bool is_mounted = (journalfile->v2.flags & JOURNALFILE_FLAG_IS_MOUNTED);

@@ -276,7 +394,7 @@ struct journal_v2_header *journalfile_v2_data_acquire(struct rrdengine_journalfi

394

395 }

396 }

279 - netdata_spinlock_unlock(&journalfile->v2.spinlock);

397 + spinlock_unlock(&journalfile->v2.spinlock);

398

399 if(do_we_need_it)

400 return journalfile_v2_mounted_data_get(journalfile, data_size);

@@ -285,7 +403,7 @@ struct journal_v2_header *journalfile_v2_data_acquire(struct rrdengine_journalfi

403 }

404

405 void journalfile_v2_data_release(struct rrdengine_journalfile *journalfile) {

288 - netdata_spinlock_lock(&journalfile->v2.spinlock);

406 + spinlock_lock(&journalfile->v2.spinlock);

407

408 internal_fatal(!journalfile->mmap.data, "trying to release a journalfile without data");

409 internal_fatal(journalfile->v2.refcount < 1, "trying to release a non-acquired journalfile");

@@ -300,7 +418,7 @@ void journalfile_v2_data_release(struct rrdengine_journalfile *journalfile) {

418 if(journalfile->v2.flags & JOURNALFILE_FLAG_MOUNTED_FOR_RETENTION)

419 unmount = true;

420 }

303 - netdata_spinlock_unlock(&journalfile->v2.spinlock);

421 + spinlock_unlock(&journalfile->v2.spinlock);

422

423 if(unmount)

424 journalfile_v2_mounted_data_unmount(journalfile, false, true);

@@ -308,25 +426,25 @@ void journalfile_v2_data_release(struct rrdengine_journalfile *journalfile) {

426

427 bool journalfile_v2_data_available(struct rrdengine_journalfile *journalfile) {

428

311 - netdata_spinlock_lock(&journalfile->v2.spinlock);

429 + spinlock_lock(&journalfile->v2.spinlock);

430 bool has_data = (journalfile->v2.flags & JOURNALFILE_FLAG_IS_AVAILABLE);

313 - netdata_spinlock_unlock(&journalfile->v2.spinlock);

431 + spinlock_unlock(&journalfile->v2.spinlock);

432

433 return has_data;

434 }

435

436 size_t journalfile_v2_data_size_get(struct rrdengine_journalfile *journalfile) {

437

320 - netdata_spinlock_lock(&journalfile->mmap.spinlock);

438 + spinlock_lock(&journalfile->mmap.spinlock);

439 size_t data_size = journalfile->mmap.size;

322 - netdata_spinlock_unlock(&journalfile->mmap.spinlock);

440 + spinlock_unlock(&journalfile->mmap.spinlock);

441

442 return data_size;

443 }

444

445 void journalfile_v2_data_set(struct rrdengine_journalfile *journalfile, int fd, void *journal_data, uint32_t journal_data_size) {

328 - netdata_spinlock_lock(&journalfile->mmap.spinlock);

329 - netdata_spinlock_lock(&journalfile->v2.spinlock);

446 + spinlock_lock(&journalfile->mmap.spinlock);

447 + spinlock_lock(&journalfile->v2.spinlock);

448

449 internal_fatal(journalfile->mmap.fd != -1, "DBENGINE JOURNALFILE: trying to re-set journal fd");

450 internal_fatal(journalfile->mmap.data, "DBENGINE JOURNALFILE: trying to re-set journal_data");

@@ -344,19 +462,23 @@ void journalfile_v2_data_set(struct rrdengine_journalfile *journalfile, int fd,

462

463 journalfile_v2_mounted_data_unmount(journalfile, true, true);

464

347 - netdata_spinlock_unlock(&journalfile->v2.spinlock);

348 - netdata_spinlock_unlock(&journalfile->mmap.spinlock);

465 + spinlock_unlock(&journalfile->v2.spinlock);

466 + spinlock_unlock(&journalfile->mmap.spinlock);

467 +

468 + njfv2idx_add(journalfile->datafile);

469 }

470

471 static void journalfile_v2_data_unmap_permanently(struct rrdengine_journalfile *journalfile) {

472 + njfv2idx_remove(journalfile->datafile);

473 +

474 bool has_references = false;

475

476 do {

477 if (has_references)

478 sleep_usec(10 * USEC_PER_MS);

479

358 - netdata_spinlock_lock(&journalfile->mmap.spinlock);

359 - netdata_spinlock_lock(&journalfile->v2.spinlock);

480 + spinlock_lock(&journalfile->mmap.spinlock);

481 + spinlock_lock(&journalfile->v2.spinlock);

482

483 if(journalfile_v2_mounted_data_unmount(journalfile, true, true)) {

484 if(journalfile->mmap.fd != -1)

@@ -374,8 +496,8 @@ static void journalfile_v2_data_unmap_permanently(struct rrdengine_journalfile *

496 internal_error(true, "DBENGINE JOURNALFILE: waiting for journalfile to be available to unmap...");

497 }

498

377 - netdata_spinlock_unlock(&journalfile->v2.spinlock);

378 - netdata_spinlock_unlock(&journalfile->mmap.spinlock);

499 + spinlock_unlock(&journalfile->v2.spinlock);

500 + spinlock_unlock(&journalfile->mmap.spinlock);

501

502 } while(has_references);

503 }

@@ -384,9 +506,9 @@ struct rrdengine_journalfile *journalfile_alloc_and_init(struct rrdengine_datafi

506 {

507 struct rrdengine_journalfile *journalfile = callocz(1, sizeof(struct rrdengine_journalfile));

508 journalfile->datafile = datafile;

387 - netdata_spinlock_init(&journalfile->mmap.spinlock);

388 - netdata_spinlock_init(&journalfile->v2.spinlock);

389 - netdata_spinlock_init(&journalfile->unsafe.spinlock);

509 + spinlock_init(&journalfile->mmap.spinlock);

510 + spinlock_init(&journalfile->v2.spinlock);

511 + spinlock_init(&journalfile->unsafe.spinlock);

512 journalfile->mmap.fd = -1;

513 datafile->journalfile = journalfile;

514 return journalfile;

database/engine/journalfile.h

+17 -2

@@ -41,6 +41,10 @@ struct rrdengine_journalfile {

41 time_t not_needed_since_s;

42 } v2;

44 + struct {

45 + Word_t indexed_as;

46 + } njfv2idx;

47 +

48 struct {

49 SPINLOCK spinlock;

50 uint64_t pos;

@@ -51,9 +55,9 @@ struct rrdengine_journalfile {

55 };

57 static inline uint64_t journalfile_current_size(struct rrdengine_journalfile *journalfile) {

54 - netdata_spinlock_lock(&journalfile->unsafe.spinlock);

58 + spinlock_lock(&journalfile->unsafe.spinlock);

59 uint64_t size = journalfile->unsafe.pos;

56 - netdata_spinlock_unlock(&journalfile->unsafe.spinlock);

60 + spinlock_unlock(&journalfile->unsafe.spinlock);

61 return size;

62 }

@@ -157,4 +161,15 @@ struct journal_v2_header *journalfile_v2_data_acquire(struct rrdengine_journalfi

161 void journalfile_v2_data_release(struct rrdengine_journalfile *journalfile);

162 void journalfile_v2_data_unmount_cleanup(time_t now_s);

163

164 +typedef struct {

165 + bool init;

166 + Word_t last;

167 + time_t wanted_start_time_s;

168 + time_t wanted_end_time_s;

169 + struct rrdengine_instance *ctx;

170 + struct journal_v2_header *j2_header_acquired;

171 +} NJFV2IDX_FIND_STATE;

172 +

173 +struct rrdengine_datafile *njfv2idx_find_and_acquire_j2_header(NJFV2IDX_FIND_STATE *s);

174 +

175 #endif /* NETDATA_JOURNALFILE_H */

\ No newline at end of file

database/engine/metric.c

+133 -92

@@ -16,6 +16,7 @@ struct metric {

16 time_t latest_time_s_hot; // latest time of the currently collected page

17 uint32_t latest_update_every_s; //

18 pid_t writer;

19 + uint8_t partition;

20 METRIC_FLAGS flags;

21 REFCOUNT refcount;

22 SPINLOCK spinlock; // protects all variable members

@@ -30,73 +31,88 @@ struct mrg {

31 ARAL *aral[MRG_PARTITIONS];

33 struct pgc_index {

33 - netdata_rwlock_t rwlock;

34 + MRG_CACHE_LINE_PADDING(0);

35 +

36 + RW_SPINLOCK rw_spinlock;

37 +

38 + MRG_CACHE_LINE_PADDING(1);

39 +

40 Pvoid_t uuid_judy; // each UUID has a JudyL of sections (tiers)

35 - } index[MRG_PARTITIONS];

37 - struct mrg_statistics stats;

42 + MRG_CACHE_LINE_PADDING(2);

43 +

44 + struct mrg_statistics stats;

45 +

46 + MRG_CACHE_LINE_PADDING(3);

47 + } index[MRG_PARTITIONS];

49 +#ifdef NETDATA_INTERNAL_CHECKS

50 size_t entries_per_partition[MRG_PARTITIONS];

51 +#endif

52 };

42 -static inline void MRG_STATS_DUPLICATE_ADD(MRG *mrg) {

43 - __atomic_add_fetch(&mrg->stats.additions_duplicate, 1, __ATOMIC_RELAXED);

54 +static inline void MRG_STATS_DUPLICATE_ADD(MRG *mrg, size_t partition) {

55 + mrg->index[partition].stats.additions_duplicate++;

56 }

58 static inline void MRG_STATS_ADDED_METRIC(MRG *mrg, size_t partition) {

47 - __atomic_add_fetch(&mrg->stats.entries, 1, __ATOMIC_RELAXED);

48 - __atomic_add_fetch(&mrg->stats.additions, 1, __ATOMIC_RELAXED);

49 - __atomic_add_fetch(&mrg->stats.size, sizeof(METRIC), __ATOMIC_RELAXED);

59 + mrg->index[partition].stats.entries++;

60 + mrg->index[partition].stats.additions++;

61 + mrg->index[partition].stats.size += sizeof(METRIC);

63 +#ifdef NETDATA_INTERNAL_CHECKS

64 __atomic_add_fetch(&mrg->entries_per_partition[partition], 1, __ATOMIC_RELAXED);

65 +#endif

66 }

68 static inline void MRG_STATS_DELETED_METRIC(MRG *mrg, size_t partition) {

55 - __atomic_sub_fetch(&mrg->stats.entries, 1, __ATOMIC_RELAXED);

56 - __atomic_sub_fetch(&mrg->stats.size, sizeof(METRIC), __ATOMIC_RELAXED);

57 - __atomic_add_fetch(&mrg->stats.deletions, 1, __ATOMIC_RELAXED);

69 + mrg->index[partition].stats.entries--;

70 + mrg->index[partition].stats.size -= sizeof(METRIC);

71 + mrg->index[partition].stats.deletions++;

73 +#ifdef NETDATA_INTERNAL_CHECKS

74 __atomic_sub_fetch(&mrg->entries_per_partition[partition], 1, __ATOMIC_RELAXED);

75 +#endif

76 }

62 -static inline void MRG_STATS_SEARCH_HIT(MRG *mrg) {

63 - __atomic_add_fetch(&mrg->stats.search_hits, 1, __ATOMIC_RELAXED);

78 +static inline void MRG_STATS_SEARCH_HIT(MRG *mrg, size_t partition) {

79 + __atomic_add_fetch(&mrg->index[partition].stats.search_hits, 1, __ATOMIC_RELAXED);

80 }

66 -static inline void MRG_STATS_SEARCH_MISS(MRG *mrg) {

67 - __atomic_add_fetch(&mrg->stats.search_misses, 1, __ATOMIC_RELAXED);

82 +static inline void MRG_STATS_SEARCH_MISS(MRG *mrg, size_t partition) {

83 + __atomic_add_fetch(&mrg->index[partition].stats.search_misses, 1, __ATOMIC_RELAXED);

84 }

70 -static inline void MRG_STATS_DELETE_MISS(MRG *mrg) {

71 - __atomic_add_fetch(&mrg->stats.delete_misses, 1, __ATOMIC_RELAXED);

86 +static inline void MRG_STATS_DELETE_MISS(MRG *mrg, size_t partition) {

87 + mrg->index[partition].stats.delete_misses++;

88 }

90 static inline void mrg_index_read_lock(MRG *mrg, size_t partition) {

75 - netdata_rwlock_rdlock(&mrg->index[partition].rwlock);

91 + rw_spinlock_read_lock(&mrg->index[partition].rw_spinlock);

92 }

93 static inline void mrg_index_read_unlock(MRG *mrg, size_t partition) {

78 - netdata_rwlock_unlock(&mrg->index[partition].rwlock);

94 + rw_spinlock_read_unlock(&mrg->index[partition].rw_spinlock);

95 }

96 static inline void mrg_index_write_lock(MRG *mrg, size_t partition) {

81 - netdata_rwlock_wrlock(&mrg->index[partition].rwlock);

97 + rw_spinlock_write_lock(&mrg->index[partition].rw_spinlock);

98 }

99 static inline void mrg_index_write_unlock(MRG *mrg, size_t partition) {

84 - netdata_rwlock_unlock(&mrg->index[partition].rwlock);

100 + rw_spinlock_write_unlock(&mrg->index[partition].rw_spinlock);

101 }

102

87 -static inline void mrg_stats_size_judyl_change(MRG *mrg, size_t mem_before_judyl, size_t mem_after_judyl) {

103 +static inline void mrg_stats_size_judyl_change(MRG *mrg, size_t mem_before_judyl, size_t mem_after_judyl, size_t partition) {

104 if(mem_after_judyl > mem_before_judyl)

89 - __atomic_add_fetch(&mrg->stats.size, mem_after_judyl - mem_before_judyl, __ATOMIC_RELAXED);

105 + __atomic_add_fetch(&mrg->index[partition].stats.size, mem_after_judyl - mem_before_judyl, __ATOMIC_RELAXED);

106 else if(mem_after_judyl < mem_before_judyl)

91 - __atomic_sub_fetch(&mrg->stats.size, mem_before_judyl - mem_after_judyl, __ATOMIC_RELAXED);

107 + __atomic_sub_fetch(&mrg->index[partition].stats.size, mem_before_judyl - mem_after_judyl, __ATOMIC_RELAXED);

108 }

109

94 -static inline void mrg_stats_size_judyhs_added_uuid(MRG *mrg) {

95 - __atomic_add_fetch(&mrg->stats.size, JUDYHS_INDEX_SIZE_ESTIMATE(sizeof(uuid_t)), __ATOMIC_RELAXED);

110 +static inline void mrg_stats_size_judyhs_added_uuid(MRG *mrg, size_t partition) {

111 + __atomic_add_fetch(&mrg->index[partition].stats.size, JUDYHS_INDEX_SIZE_ESTIMATE(sizeof(uuid_t)), __ATOMIC_RELAXED);

112 }

113

98 -static inline void mrg_stats_size_judyhs_removed_uuid(MRG *mrg) {

99 - __atomic_sub_fetch(&mrg->stats.size, JUDYHS_INDEX_SIZE_ESTIMATE(sizeof(uuid_t)), __ATOMIC_RELAXED);

114 +static inline void mrg_stats_size_judyhs_removed_uuid(MRG *mrg, size_t partition) {

115 + __atomic_sub_fetch(&mrg->index[partition].stats.size, JUDYHS_INDEX_SIZE_ESTIMATE(sizeof(uuid_t)), __ATOMIC_RELAXED);

116 }

117

118 static inline size_t uuid_partition(MRG *mrg __maybe_unused, uuid_t *uuid) {

@@ -105,25 +121,28 @@ static inline size_t uuid_partition(MRG *mrg __maybe_unused, uuid_t *uuid) {

121 }

122

123 static inline bool metric_has_retention_unsafe(MRG *mrg __maybe_unused, METRIC *metric) {

124 + size_t partition = metric->partition;

125 +

126 bool has_retention = (metric->first_time_s > 0 || metric->latest_time_s_clean > 0 || metric->latest_time_s_hot > 0);

127

128 if(has_retention && !(metric->flags & METRIC_FLAG_HAS_RETENTION)) {

129 metric->flags |= METRIC_FLAG_HAS_RETENTION;

112 - __atomic_add_fetch(&mrg->stats.entries_with_retention, 1, __ATOMIC_RELAXED);

130 + __atomic_add_fetch(&mrg->index[partition].stats.entries_with_retention, 1, __ATOMIC_RELAXED);

131 }

132 else if(!has_retention && (metric->flags & METRIC_FLAG_HAS_RETENTION)) {

133 metric->flags &= ~METRIC_FLAG_HAS_RETENTION;

116 - __atomic_sub_fetch(&mrg->stats.entries_with_retention, 1, __ATOMIC_RELAXED);

134 + __atomic_sub_fetch(&mrg->index[partition].stats.entries_with_retention, 1, __ATOMIC_RELAXED);

135 }

136

137 return has_retention;

138 }

139

140 static inline REFCOUNT metric_acquire(MRG *mrg __maybe_unused, METRIC *metric, bool having_spinlock) {

141 + size_t partition = metric->partition;

142 REFCOUNT refcount;

143

144 if(!having_spinlock)

126 - netdata_spinlock_lock(&metric->spinlock);

145 + spinlock_lock(&metric->spinlock);

146

147 if(unlikely(metric->refcount < 0))

148 fatal("METRIC: refcount is %d (negative) during acquire", metric->refcount);

@@ -134,21 +153,22 @@ static inline REFCOUNT metric_acquire(MRG *mrg __maybe_unused, METRIC *metric, b

153 metric_has_retention_unsafe(mrg, metric);

154

155 if(!having_spinlock)

137 - netdata_spinlock_unlock(&metric->spinlock);

156 + spinlock_unlock(&metric->spinlock);

157

158 if(refcount == 1)

140 - __atomic_add_fetch(&mrg->stats.entries_referenced, 1, __ATOMIC_RELAXED);

159 + __atomic_add_fetch(&mrg->index[partition].stats.entries_referenced, 1, __ATOMIC_RELAXED);

160

142 - __atomic_add_fetch(&mrg->stats.current_references, 1, __ATOMIC_RELAXED);

161 + __atomic_add_fetch(&mrg->index[partition].stats.current_references, 1, __ATOMIC_RELAXED);

162

163 return refcount;

164 }

165

166 static inline bool metric_release_and_can_be_deleted(MRG *mrg __maybe_unused, METRIC *metric) {

167 bool ret = true;

168 + size_t partition = metric->partition;

169 REFCOUNT refcount;

170

151 - netdata_spinlock_lock(&metric->spinlock);

171 + spinlock_lock(&metric->spinlock);

172

173 if(unlikely(metric->refcount <= 0))

174 fatal("METRIC: refcount is %d (zero or negative) during release", metric->refcount);

@@ -158,12 +178,12 @@ static inline bool metric_release_and_can_be_deleted(MRG *mrg __maybe_unused, ME

178 if(likely(metric_has_retention_unsafe(mrg, metric) || refcount != 0))

179 ret = false;

180

161 - netdata_spinlock_unlock(&metric->spinlock);

181 + spinlock_unlock(&metric->spinlock);

182

183 if(unlikely(!refcount))

164 - __atomic_sub_fetch(&mrg->stats.entries_referenced, 1, __ATOMIC_RELAXED);

184 + __atomic_sub_fetch(&mrg->index[partition].stats.entries_referenced, 1, __ATOMIC_RELAXED);

185

166 - __atomic_sub_fetch(&mrg->stats.current_references, 1, __ATOMIC_RELAXED);

186 + __atomic_sub_fetch(&mrg->index[partition].stats.current_references, 1, __ATOMIC_RELAXED);

187

188 return ret;

189 }

@@ -182,12 +202,12 @@ static METRIC *metric_add_and_acquire(MRG *mrg, MRG_ENTRY *entry, bool *ret) {

202 fatal("DBENGINE METRIC: corrupted UUIDs JudyHS array");

203

204 if(unlikely(!*sections_judy_pptr))

185 - mrg_stats_size_judyhs_added_uuid(mrg);

205 + mrg_stats_size_judyhs_added_uuid(mrg, partition);

206

207 mem_before_judyl = JudyLMemUsed(*sections_judy_pptr);

208 Pvoid_t *PValue = JudyLIns(sections_judy_pptr, entry->section, PJE0);

209 mem_after_judyl = JudyLMemUsed(*sections_judy_pptr);

190 - mrg_stats_size_judyl_change(mrg, mem_before_judyl, mem_after_judyl);

210 + mrg_stats_size_judyl_change(mrg, mem_before_judyl, mem_after_judyl, partition);

211

212 if(unlikely(!PValue || PValue == PJERR))

213 fatal("DBENGINE METRIC: corrupted section JudyL array");

@@ -196,6 +216,9 @@ static METRIC *metric_add_and_acquire(MRG *mrg, MRG_ENTRY *entry, bool *ret) {

216 METRIC *metric = *PValue;

217

218 metric_acquire(mrg, metric, false);

219 +

220 + MRG_STATS_DUPLICATE_ADD(mrg, partition);

221 +

222 mrg_index_write_unlock(mrg, partition);

223

224 if(ret)

@@ -203,7 +226,6 @@ static METRIC *metric_add_and_acquire(MRG *mrg, MRG_ENTRY *entry, bool *ret) {

226

227 aral_freez(mrg->aral[partition], allocation);

228

206 - MRG_STATS_DUPLICATE_ADD(mrg);

229 return metric;

230 }

231

@@ -217,17 +239,18 @@ static METRIC *metric_add_and_acquire(MRG *mrg, MRG_ENTRY *entry, bool *ret) {

239 metric->writer = 0;

240 metric->refcount = 0;

241 metric->flags = 0;

220 - netdata_spinlock_init(&metric->spinlock);

242 + metric->partition = partition;

243 + spinlock_init(&metric->spinlock);

244 metric_acquire(mrg, metric, true); // no spinlock use required here

245 *PValue = metric;

246

247 + MRG_STATS_ADDED_METRIC(mrg, partition);

248 +

249 mrg_index_write_unlock(mrg, partition);

250

251 if(ret)

252 *ret = true;

253

229 - MRG_STATS_ADDED_METRIC(mrg, partition);

230 -

254 return metric;

255 }

256

@@ -239,14 +262,14 @@ static METRIC *metric_get_and_acquire(MRG *mrg, uuid_t *uuid, Word_t section) {

262 Pvoid_t *sections_judy_pptr = JudyHSGet(mrg->index[partition].uuid_judy, uuid, sizeof(uuid_t));

263 if(unlikely(!sections_judy_pptr)) {

264 mrg_index_read_unlock(mrg, partition);

242 - MRG_STATS_SEARCH_MISS(mrg);

265 + MRG_STATS_SEARCH_MISS(mrg, partition);

266 return NULL;

267 }

268

269 Pvoid_t *PValue = JudyLGet(*sections_judy_pptr, section, PJE0);

270 if(unlikely(!PValue)) {

271 mrg_index_read_unlock(mrg, partition);

249 - MRG_STATS_SEARCH_MISS(mrg);

272 + MRG_STATS_SEARCH_MISS(mrg, partition);

273 return NULL;

274 }

275

@@ -256,7 +279,7 @@ static METRIC *metric_get_and_acquire(MRG *mrg, uuid_t *uuid, Word_t section) {

279

280 mrg_index_read_unlock(mrg, partition);

281

259 - MRG_STATS_SEARCH_HIT(mrg);

282 + MRG_STATS_SEARCH_HIT(mrg, partition);

283 return metric;

284 }

285

@@ -268,26 +291,26 @@ static bool acquired_metric_del(MRG *mrg, METRIC *metric) {

291 mrg_index_write_lock(mrg, partition);

292

293 if(!metric_release_and_can_be_deleted(mrg, metric)) {

294 + mrg->index[partition].stats.delete_having_retention_or_referenced++;

295 mrg_index_write_unlock(mrg, partition);

272 - __atomic_add_fetch(&mrg->stats.delete_having_retention_or_referenced, 1, __ATOMIC_RELAXED);

296 return false;

297 }

298

299 Pvoid_t *sections_judy_pptr = JudyHSGet(mrg->index[partition].uuid_judy, &metric->uuid, sizeof(uuid_t));

300 if(unlikely(!sections_judy_pptr || !*sections_judy_pptr)) {

301 + MRG_STATS_DELETE_MISS(mrg, partition);

302 mrg_index_write_unlock(mrg, partition);

279 - MRG_STATS_DELETE_MISS(mrg);

303 return false;

304 }

305

306 mem_before_judyl = JudyLMemUsed(*sections_judy_pptr);

307 int rc = JudyLDel(sections_judy_pptr, metric->section, PJE0);

308 mem_after_judyl = JudyLMemUsed(*sections_judy_pptr);

286 - mrg_stats_size_judyl_change(mrg, mem_before_judyl, mem_after_judyl);

309 + mrg_stats_size_judyl_change(mrg, mem_before_judyl, mem_after_judyl, partition);

310

311 if(unlikely(!rc)) {

312 + MRG_STATS_DELETE_MISS(mrg, partition);

313 mrg_index_write_unlock(mrg, partition);

290 - MRG_STATS_DELETE_MISS(mrg);

314 return false;

315 }

316

@@ -295,15 +318,15 @@ static bool acquired_metric_del(MRG *mrg, METRIC *metric) {

318 rc = JudyHSDel(&mrg->index[partition].uuid_judy, &metric->uuid, sizeof(uuid_t), PJE0);

319 if(unlikely(!rc))

320 fatal("DBENGINE METRIC: cannot delete UUID from JudyHS");

298 - mrg_stats_size_judyhs_removed_uuid(mrg);

321 + mrg_stats_size_judyhs_removed_uuid(mrg, partition);

322 }

323

324 + MRG_STATS_DELETED_METRIC(mrg, partition);

325 +

326 mrg_index_write_unlock(mrg, partition);

327

328 aral_freez(mrg->aral[partition], metric);

329

305 - MRG_STATS_DELETED_METRIC(mrg, partition);

306 -

330 return true;

331 }

332

@@ -314,7 +337,7 @@ MRG *mrg_create(void) {

337 MRG *mrg = callocz(1, sizeof(MRG));

338

339 for(size_t i = 0; i < MRG_PARTITIONS ; i++) {

317 - netdata_rwlock_init(&mrg->index[i].rwlock);

340 + rw_spinlock_init(&mrg->index[i].rw_spinlock);

341

342 char buf[ARAL_MAX_NAME + 1];

343 snprintfz(buf, ARAL_MAX_NAME, "mrg[%zu]", i);

@@ -328,8 +351,6 @@ MRG *mrg_create(void) {

351 false);

352 }

353

331 - mrg->stats.size = sizeof(MRG);

332 -

354 return mrg;

355 }

356

@@ -393,10 +414,10 @@ bool mrg_metric_set_first_time_s(MRG *mrg __maybe_unused, METRIC *metric, time_t

414 if(unlikely(first_time_s < 0))

415 return false;

416

396 - netdata_spinlock_lock(&metric->spinlock);

417 + spinlock_lock(&metric->spinlock);

418 metric->first_time_s = first_time_s;

419 metric_has_retention_unsafe(mrg, metric);

399 - netdata_spinlock_unlock(&metric->spinlock);

420 + spinlock_unlock(&metric->spinlock);

421

422 return true;

423 }

@@ -421,7 +442,7 @@ void mrg_metric_expand_retention(MRG *mrg __maybe_unused, METRIC *metric, time_t

442 if(unlikely(!first_time_s && !last_time_s && !update_every_s))

443 return;

444

424 - netdata_spinlock_lock(&metric->spinlock);

445 + spinlock_lock(&metric->spinlock);

446

447 if(unlikely(first_time_s && (!metric->first_time_s || first_time_s < metric->first_time_s)))

448 metric->first_time_s = first_time_s;

@@ -436,7 +457,7 @@ void mrg_metric_expand_retention(MRG *mrg __maybe_unused, METRIC *metric, time_t

457 metric->latest_update_every_s = (uint32_t) update_every_s;

458

459 metric_has_retention_unsafe(mrg, metric);

439 - netdata_spinlock_unlock(&metric->spinlock);

460 + spinlock_unlock(&metric->spinlock);

461 }

462

463 bool mrg_metric_set_first_time_s_if_bigger(MRG *mrg __maybe_unused, METRIC *metric, time_t first_time_s) {

@@ -444,13 +465,13 @@ bool mrg_metric_set_first_time_s_if_bigger(MRG *mrg __maybe_unused, METRIC *metr

465

466 bool ret = false;

467

447 - netdata_spinlock_lock(&metric->spinlock);

468 + spinlock_lock(&metric->spinlock);

469 if(first_time_s > metric->first_time_s) {

470 metric->first_time_s = first_time_s;

471 ret = true;

472 }

473 metric_has_retention_unsafe(mrg, metric);

453 - netdata_spinlock_unlock(&metric->spinlock);

474 + spinlock_unlock(&metric->spinlock);

475

476 return ret;

477 }

@@ -458,7 +479,7 @@ bool mrg_metric_set_first_time_s_if_bigger(MRG *mrg __maybe_unused, METRIC *metr

479 time_t mrg_metric_get_first_time_s(MRG *mrg __maybe_unused, METRIC *metric) {

480 time_t first_time_s;

481

461 - netdata_spinlock_lock(&metric->spinlock);

482 + spinlock_lock(&metric->spinlock);

483

484 if(unlikely(!metric->first_time_s)) {

485 if(metric->latest_time_s_clean)

@@ -470,13 +491,13 @@ time_t mrg_metric_get_first_time_s(MRG *mrg __maybe_unused, METRIC *metric) {

491

492 first_time_s = metric->first_time_s;

493

473 - netdata_spinlock_unlock(&metric->spinlock);

494 + spinlock_unlock(&metric->spinlock);

495

496 return first_time_s;

497 }

498

499 void mrg_metric_get_retention(MRG *mrg __maybe_unused, METRIC *metric, time_t *first_time_s, time_t *last_time_s, time_t *update_every_s) {

479 - netdata_spinlock_lock(&metric->spinlock);

500 + spinlock_lock(&metric->spinlock);

501

502 if(unlikely(!metric->first_time_s)) {

503 if(metric->latest_time_s_clean)

@@ -490,7 +511,7 @@ void mrg_metric_get_retention(MRG *mrg __maybe_unused, METRIC *metric, time_t *f

511 *last_time_s = MAX(metric->latest_time_s_clean, metric->latest_time_s_hot);

512 *update_every_s = metric->latest_update_every_s;

513

493 - netdata_spinlock_unlock(&metric->spinlock);

514 + spinlock_unlock(&metric->spinlock);

515 }

516

517 bool mrg_metric_set_clean_latest_time_s(MRG *mrg __maybe_unused, METRIC *metric, time_t latest_time_s) {

@@ -499,7 +520,7 @@ bool mrg_metric_set_clean_latest_time_s(MRG *mrg __maybe_unused, METRIC *metric,

520 if(unlikely(latest_time_s < 0))

521 return false;

522

502 - netdata_spinlock_lock(&metric->spinlock);

523 + spinlock_lock(&metric->spinlock);

524

525 // internal_fatal(latest_time_s > max_acceptable_collected_time(),

526 // "DBENGINE METRIC: metric latest time is in the future");

@@ -513,7 +534,7 @@ bool mrg_metric_set_clean_latest_time_s(MRG *mrg __maybe_unused, METRIC *metric,

534 metric->first_time_s = latest_time_s;

535

536 metric_has_retention_unsafe(mrg, metric);

516 - netdata_spinlock_unlock(&metric->spinlock);

537 + spinlock_unlock(&metric->spinlock);

538 return true;

539 }

540

@@ -551,7 +572,7 @@ bool mrg_metric_zero_disk_retention(MRG *mrg __maybe_unused, METRIC *metric) {

572 if (min_first_time_s == LONG_MAX)

573 min_first_time_s = 0;

574

554 - netdata_spinlock_lock(&metric->spinlock);

575 + spinlock_lock(&metric->spinlock);

576 if (--countdown && !min_first_time_s && metric->latest_time_s_hot)

577 do_again = true;

578 else {

@@ -563,7 +584,7 @@ bool mrg_metric_zero_disk_retention(MRG *mrg __maybe_unused, METRIC *metric) {

584

585 ret = metric_has_retention_unsafe(mrg, metric);

586 }

566 - netdata_spinlock_unlock(&metric->spinlock);

587 + spinlock_unlock(&metric->spinlock);

588 } while(do_again);

589

590 return ret;

@@ -578,22 +599,22 @@ bool mrg_metric_set_hot_latest_time_s(MRG *mrg __maybe_unused, METRIC *metric, t

599 if(unlikely(latest_time_s < 0))

600 return false;

601

581 - netdata_spinlock_lock(&metric->spinlock);

602 + spinlock_lock(&metric->spinlock);

603 metric->latest_time_s_hot = latest_time_s;

604

605 if(unlikely(!metric->first_time_s))

606 metric->first_time_s = latest_time_s;

607

608 metric_has_retention_unsafe(mrg, metric);

588 - netdata_spinlock_unlock(&metric->spinlock);

609 + spinlock_unlock(&metric->spinlock);

610 return true;

611 }

612

613 time_t mrg_metric_get_latest_time_s(MRG *mrg __maybe_unused, METRIC *metric) {

614 time_t max;

594 - netdata_spinlock_lock(&metric->spinlock);

615 + spinlock_lock(&metric->spinlock);

616 max = MAX(metric->latest_time_s_clean, metric->latest_time_s_hot);

596 - netdata_spinlock_unlock(&metric->spinlock);

617 + spinlock_unlock(&metric->spinlock);

618 return max;

619 }

620

@@ -603,9 +624,9 @@ bool mrg_metric_set_update_every(MRG *mrg __maybe_unused, METRIC *metric, time_t

624 if(update_every_s <= 0)

625 return false;

626

606 - netdata_spinlock_lock(&metric->spinlock);

627 + spinlock_lock(&metric->spinlock);

628 metric->latest_update_every_s = (uint32_t) update_every_s;

608 - netdata_spinlock_unlock(&metric->spinlock);

629 + spinlock_unlock(&metric->spinlock);

630

631 return true;

632 }

@@ -616,10 +637,10 @@ bool mrg_metric_set_update_every_s_if_zero(MRG *mrg __maybe_unused, METRIC *metr

637 if(update_every_s <= 0)

638 return false;

639

619 - netdata_spinlock_lock(&metric->spinlock);

640 + spinlock_lock(&metric->spinlock);

641 if(!metric->latest_update_every_s)

642 metric->latest_update_every_s = (uint32_t) update_every_s;

622 - netdata_spinlock_unlock(&metric->spinlock);

643 + spinlock_unlock(&metric->spinlock);

644

645 return true;

646 }

@@ -627,42 +648,60 @@ bool mrg_metric_set_update_every_s_if_zero(MRG *mrg __maybe_unused, METRIC *metr

648 time_t mrg_metric_get_update_every_s(MRG *mrg __maybe_unused, METRIC *metric) {

649 time_t update_every_s;

650

630 - netdata_spinlock_lock(&metric->spinlock);

651 + spinlock_lock(&metric->spinlock);

652 update_every_s = metric->latest_update_every_s;

632 - netdata_spinlock_unlock(&metric->spinlock);

653 + spinlock_unlock(&metric->spinlock);

654

655 return update_every_s;

656 }

657

658 bool mrg_metric_set_writer(MRG *mrg, METRIC *metric) {

659 bool done = false;

639 - netdata_spinlock_lock(&metric->spinlock);

660 + spinlock_lock(&metric->spinlock);

661 if(!metric->writer) {

662 metric->writer = gettid();

642 - __atomic_add_fetch(&mrg->stats.writers, 1, __ATOMIC_RELAXED);

663 + __atomic_add_fetch(&mrg->index[metric->partition].stats.writers, 1, __ATOMIC_RELAXED);

664 done = true;

665 }

666 else

646 - __atomic_add_fetch(&mrg->stats.writers_conflicts, 1, __ATOMIC_RELAXED);

647 - netdata_spinlock_unlock(&metric->spinlock);

667 + __atomic_add_fetch(&mrg->index[metric->partition].stats.writers_conflicts, 1, __ATOMIC_RELAXED);

668 + spinlock_unlock(&metric->spinlock);

669 return done;

670 }

671

672 bool mrg_metric_clear_writer(MRG *mrg, METRIC *metric) {

673 bool done = false;

653 - netdata_spinlock_lock(&metric->spinlock);

674 + spinlock_lock(&metric->spinlock);

675 if(metric->writer) {

676 metric->writer = 0;

656 - __atomic_sub_fetch(&mrg->stats.writers, 1, __ATOMIC_RELAXED);

677 + __atomic_sub_fetch(&mrg->index[metric->partition].stats.writers, 1, __ATOMIC_RELAXED);

678 done = true;

679 }

659 - netdata_spinlock_unlock(&metric->spinlock);

680 + spinlock_unlock(&metric->spinlock);

681 return done;

682 }

683

663 -struct mrg_statistics mrg_get_statistics(MRG *mrg) {

664 - // FIXME - use atomics

665 - return mrg->stats;

684 +void mrg_get_statistics(MRG *mrg, struct mrg_statistics *s) {

685 + memset(s, 0, sizeof(struct mrg_statistics));

686 +

687 + for(int i = 0; i < MRG_PARTITIONS ;i++) {

688 + s->entries += __atomic_load_n(&mrg->index[i].stats.entries, __ATOMIC_RELAXED);

689 + s->entries_referenced += __atomic_load_n(&mrg->index[i].stats.entries_referenced, __ATOMIC_RELAXED);

690 + s->entries_with_retention += __atomic_load_n(&mrg->index[i].stats.entries_with_retention, __ATOMIC_RELAXED);

691 + s->size += __atomic_load_n(&mrg->index[i].stats.size, __ATOMIC_RELAXED);

692 + s->current_references += __atomic_load_n(&mrg->index[i].stats.current_references, __ATOMIC_RELAXED);

693 + s->additions += __atomic_load_n(&mrg->index[i].stats.additions, __ATOMIC_RELAXED);

694 + s->additions_duplicate += __atomic_load_n(&mrg->index[i].stats.additions_duplicate, __ATOMIC_RELAXED);

695 + s->deletions += __atomic_load_n(&mrg->index[i].stats.deletions, __ATOMIC_RELAXED);

696 + s->delete_having_retention_or_referenced += __atomic_load_n(&mrg->index[i].stats.delete_having_retention_or_referenced, __ATOMIC_RELAXED);

697 + s->delete_misses += __atomic_load_n(&mrg->index[i].stats.delete_misses, __ATOMIC_RELAXED);

698 + s->search_hits += __atomic_load_n(&mrg->index[i].stats.search_hits, __ATOMIC_RELAXED);

699 + s->search_misses += __atomic_load_n(&mrg->index[i].stats.search_misses, __ATOMIC_RELAXED);

700 + s->writers += __atomic_load_n(&mrg->index[i].stats.writers, __ATOMIC_RELAXED);

701 + s->writers_conflicts += __atomic_load_n(&mrg->index[i].stats.writers_conflicts, __ATOMIC_RELAXED);

702 + }

703 +

704 + s->size += sizeof(MRG);

705 }

706

707 // ----------------------------------------------------------------------------

@@ -850,7 +889,9 @@ int mrg_unittest(void) {

889 if(!mrg_metric_release_and_delete(mrg, m1_t1))

890 fatal("DBENGINE METRIC: cannot delete the second metric");

891

853 - if(mrg->stats.entries != 0)

892 + struct mrg_statistics s;

893 + mrg_get_statistics(mrg, &s);

894 + if(s.entries != 0)

895 fatal("DBENGINE METRIC: invalid entries counter");

896

897 #ifdef MRG_STRESS_TEST

database/engine/metric.h

+17 -6

@@ -5,6 +5,8 @@

6 #define MRG_PARTITIONS 10

8 +#define MRG_CACHE_LINE_PADDING(x) uint8_t padding##x[64]

9 +

10 typedef struct metric METRIC;

11 typedef struct mrg MRG;

@@ -17,14 +19,10 @@ typedef struct mrg_entry {

19 } MRG_ENTRY;

21 struct mrg_statistics {

22 + // non-atomic - under a write lock

23 size_t entries;

21 - size_t entries_referenced;

22 - size_t entries_with_retention;

23 -

24 size_t size; // total memory used, with indexing

26 - size_t current_references;

27 -

26 size_t additions;

27 size_t additions_duplicate;

@@ -32,9 +30,22 @@ struct mrg_statistics {

30 size_t delete_having_retention_or_referenced;

31 size_t delete_misses;

33 + // atomic - multiple readers / writers

34 +

35 + MRG_CACHE_LINE_PADDING(0);

36 + size_t entries_referenced;

37 +

38 + MRG_CACHE_LINE_PADDING(1);

39 + size_t entries_with_retention;

40 +

41 + MRG_CACHE_LINE_PADDING(2);

42 + size_t current_references;

43 +

44 + MRG_CACHE_LINE_PADDING(3);

45 size_t search_hits;

46 size_t search_misses;

48 + MRG_CACHE_LINE_PADDING(4);

49 size_t writers;

50 size_t writers_conflicts;

51 };

@@ -72,7 +83,7 @@ bool mrg_metric_zero_disk_retention(MRG *mrg __maybe_unused, METRIC *metric);

83 bool mrg_metric_set_writer(MRG *mrg, METRIC *metric);

84 bool mrg_metric_clear_writer(MRG *mrg, METRIC *metric);

75 -struct mrg_statistics mrg_get_statistics(MRG *mrg);

86 +void mrg_get_statistics(MRG *mrg, struct mrg_statistics *s);

87 size_t mrg_aral_structures(void);

88 size_t mrg_aral_overhead(void);

database/engine/pagecache.c

+51 -31

@@ -387,15 +387,17 @@ static size_t list_has_time_gaps(

387 time_t wanted_end_time_s,

388 size_t *pages_total,

389 size_t *pages_found_pass4,

390 - size_t *pages_pending,

390 + size_t *pages_to_load_from_disk,

391 size_t *pages_overlapping,

392 time_t *optimal_end_time_s,

393 - bool populate_gaps

393 + bool populate_gaps,

394 + PDC_PAGE_STATUS *common_status

395 ) {

396 // we will recalculate these, so zero them

396 - *pages_pending = 0;

397 + *pages_to_load_from_disk = 0;

398 *pages_overlapping = 0;

399 *optimal_end_time_s = 0;

400 + *common_status = 0;

401

402 bool first;

403 Pvoid_t *PValue;

@@ -461,6 +463,7 @@ static size_t list_has_time_gaps(

463 (*pages_overlapping)++;

464 pd->status |= PDC_PAGE_SKIP;

465 pd->status &= ~(PDC_PAGE_READY | PDC_PAGE_DISK_PENDING);

466 + *common_status |= pd->status;

467 continue;

468 }

469

@@ -480,7 +483,7 @@ static size_t list_has_time_gaps(

483

484 }

485 else if(!(pd->status & PDC_PAGE_FAILED) && (pd->status & PDC_PAGE_DATAFILE_ACQUIRED)) {

483 - (*pages_pending)++;

486 + (*pages_to_load_from_disk)++;

487

488 pd->status |= PDC_PAGE_DISK_PENDING;

489

@@ -495,6 +498,8 @@ static size_t list_has_time_gaps(

498 pd->status &= ~PDC_PAGE_DISK_PENDING;

499 pd->status |= (PDC_PAGE_READY | PDC_PAGE_PRELOADED);

500 }

501 +

502 + *common_status |= pd->status;

503 }

504

505 internal_fatal(pages_pass2 != pages_pass3,

@@ -505,6 +510,8 @@ static size_t list_has_time_gaps(

510 return gaps;

511 }

512

513 +// ----------------------------------------------------------------------------

514 +

515 typedef void (*page_found_callback_t)(PGC_PAGE *page, void *data);

516 static size_t get_page_list_from_journal_v2(struct rrdengine_instance *ctx, METRIC *metric, usec_t start_time_ut, usec_t end_time_ut, page_found_callback_t callback, void *callback_data) {

517 uuid_t *uuid = mrg_metric_uuid(main_mrg, metric);

@@ -515,12 +522,19 @@ static size_t get_page_list_from_journal_v2(struct rrdengine_instance *ctx, METR

522

523 size_t pages_found = 0;

524

518 - uv_rwlock_rdlock(&ctx->datafiles.rwlock);

525 + NJFV2IDX_FIND_STATE state = {

526 + .init = false,

527 + .last = 0,

528 + .ctx = ctx,

529 + .wanted_start_time_s = wanted_start_time_s,

530 + .wanted_end_time_s = wanted_end_time_s,

531 + .j2_header_acquired = NULL,

532 + };

533 +

534 struct rrdengine_datafile *datafile;

520 - for(datafile = ctx->datafiles.first; datafile ; datafile = datafile->next) {

521 - struct journal_v2_header *j2_header = journalfile_v2_data_acquire(datafile->journalfile, NULL,

522 - wanted_start_time_s,

523 - wanted_end_time_s);

535 + while((datafile = njfv2idx_find_and_acquire_j2_header(&state))) {

536 + struct journal_v2_header *j2_header = state.j2_header_acquired;

537 +

538 if (unlikely(!j2_header))

539 continue;

540

@@ -595,7 +609,6 @@ static size_t get_page_list_from_journal_v2(struct rrdengine_instance *ctx, METR

609

610 journalfile_v2_data_release(datafile->journalfile);

611 }

598 - uv_rwlock_rdunlock(&ctx->datafiles.rwlock);

612

613 return pages_found;

614 }

@@ -644,10 +657,13 @@ static Pvoid_t get_page_list(

657 METRIC *metric,

658 usec_t start_time_ut,

659 usec_t end_time_ut,

647 - size_t *pages_to_load,

648 - time_t *optimal_end_time_s

660 + time_t *optimal_end_time_s,

661 + size_t *pages_to_load_from_disk,

662 + PDC_PAGE_STATUS *common_status

663 ) {

664 *optimal_end_time_s = 0;

665 + *pages_to_load_from_disk = 0;

666 + *common_status = 0;

667

668 Pvoid_t JudyL_page_array = (Pvoid_t) NULL;

669

@@ -658,14 +674,13 @@ static Pvoid_t get_page_list(

674 pages_found_in_open_cache = 0,

675 pages_found_in_journals_v2 = 0,

676 pages_found_pass4 = 0,

661 - pages_pending = 0,

677 pages_overlapping = 0,

678 pages_total = 0;

679

680 size_t cache_gaps = 0, query_gaps = 0;

681 bool done_v2 = false, done_open = false;

682

668 - usec_t pass1_ut = 0, pass2_ut = 0, pass3_ut = 0, pass4_ut = 0;

683 + usec_t pass1_ut = 0, pass2_ut = 0, pass3_ut = 0, pass4_ut = 0, finish_ut = 0;

684

685 // --------------------------------------------------------------

686 // PASS 1: Check what the main page cache has available

@@ -680,8 +695,8 @@ static Pvoid_t get_page_list(

695

696 if(pages_found_in_main_cache && !cache_gaps) {

697 query_gaps = list_has_time_gaps(ctx, metric, JudyL_page_array, wanted_start_time_s, wanted_end_time_s,

683 - &pages_total, &pages_found_pass4, &pages_pending, &pages_overlapping,

684 - optimal_end_time_s, false);

698 + &pages_total, &pages_found_pass4, pages_to_load_from_disk, &pages_overlapping,

699 + optimal_end_time_s, false, common_status);

700

701 if (pages_total && !query_gaps)

702 goto we_are_done;

@@ -702,8 +717,8 @@ static Pvoid_t get_page_list(

717

718 if(pages_found_in_open_cache) {

719 query_gaps = list_has_time_gaps(ctx, metric, JudyL_page_array, wanted_start_time_s, wanted_end_time_s,

705 - &pages_total, &pages_found_pass4, &pages_pending, &pages_overlapping,

706 - optimal_end_time_s, false);

720 + &pages_total, &pages_found_pass4, pages_to_load_from_disk, &pages_overlapping,

721 + optimal_end_time_s, false, common_status);

722

723 if (pages_total && !query_gaps)

724 goto we_are_done;

@@ -726,15 +741,11 @@ static Pvoid_t get_page_list(

741

742 pass4_ut = now_monotonic_usec();

743 query_gaps = list_has_time_gaps(ctx, metric, JudyL_page_array, wanted_start_time_s, wanted_end_time_s,

729 - &pages_total, &pages_found_pass4, &pages_pending, &pages_overlapping,

730 - optimal_end_time_s, true);

744 + &pages_total, &pages_found_pass4, pages_to_load_from_disk, &pages_overlapping,

745 + optimal_end_time_s, true, common_status);

746

747 we_are_done:

733 -

734 - if(pages_to_load)

735 - *pages_to_load = pages_pending;

736 -

737 - usec_t finish_ut = now_monotonic_usec();

748 + finish_ut = now_monotonic_usec();

749 time_delta(finish_ut, pass4_ut);

750 time_delta(finish_ut, pass3_ut);

751 time_delta(finish_ut, pass2_ut);

@@ -754,7 +765,7 @@ we_are_done:

765 __atomic_add_fetch(&rrdeng_cache_efficiency_stats.pages_meta_source_journal_v2, pages_found_in_journals_v2, __ATOMIC_RELAXED);

766 __atomic_add_fetch(&rrdeng_cache_efficiency_stats.pages_data_source_main_cache, pages_found_in_main_cache, __ATOMIC_RELAXED);

767 __atomic_add_fetch(&rrdeng_cache_efficiency_stats.pages_data_source_main_cache_at_pass4, pages_found_pass4, __ATOMIC_RELAXED);

757 - __atomic_add_fetch(&rrdeng_cache_efficiency_stats.pages_to_load_from_disk, pages_pending, __ATOMIC_RELAXED);

768 + __atomic_add_fetch(&rrdeng_cache_efficiency_stats.pages_to_load_from_disk, *pages_to_load_from_disk, __ATOMIC_RELAXED);

769 __atomic_add_fetch(&rrdeng_cache_efficiency_stats.pages_overlapping_skipped, pages_overlapping, __ATOMIC_RELAXED);

770

771 return JudyL_page_array;

@@ -773,14 +784,23 @@ void rrdeng_prep_query(struct page_details_control *pdc, bool worker) {

784 if(worker)

785 worker_is_busy(UV_EVENT_DBENGINE_QUERY);

786

776 - size_t pages_to_load = 0;

787 pdc->page_list_JudyL = get_page_list(pdc->ctx, pdc->metric,

788 pdc->start_time_s * USEC_PER_SEC,

789 pdc->end_time_s * USEC_PER_SEC,

780 - &pages_to_load,

781 - &pdc->optimal_end_time_s);

790 + &pdc->optimal_end_time_s,

791 + &pdc->pages_to_load_from_disk,

792 + &pdc->common_status);

793 +

794 + internal_fatal(pdc->pages_to_load_from_disk && !(pdc->common_status & PDC_PAGE_DISK_PENDING),

795 + "DBENGINE: PDC reports there are %zu pages to load from disk, "

796 + "but none of the pages has the PDC_PAGE_DISK_PENDING flag",

797 + pdc->pages_to_load_from_disk);

798 +

799 + internal_fatal(!pdc->pages_to_load_from_disk && (pdc->common_status & PDC_PAGE_DISK_PENDING),

800 + "DBENGINE: PDC reports there are no pages to load from disk, "

801 + "but one or more pages have the PDC_PAGE_DISK_PENDING flag");

802

783 - if (pages_to_load && pdc->page_list_JudyL) {

803 + if (pdc->pages_to_load_from_disk && pdc->page_list_JudyL) {

804 pdc_acquire(pdc); // we get 1 for the 1st worker in the chain: do_read_page_list_work()

805 usec_t start_ut = now_monotonic_usec();

806 if(likely(pdc->priority == STORAGE_PRIORITY_SYNCHRONOUS))

@@ -822,7 +842,7 @@ void pg_cache_preload(struct rrdeng_query_handle *handle) {

842 handle->pdc->optimal_end_time_s = handle->end_time_s;

843 handle->pdc->ctx = handle->ctx;

844 handle->pdc->refcount = 1;

825 - netdata_spinlock_init(&handle->pdc->refcount_spinlock);

845 + spinlock_init(&handle->pdc->refcount_spinlock);

846 completion_init(&handle->pdc->prep_completion);

847 completion_init(&handle->pdc->page_completion);

848

database/engine/pdc.c

+15 -15

@@ -198,7 +198,7 @@ void extent_buffer_init(void) {

198 void extent_buffer_cleanup1(void) {

199 struct extent_buffer *item = NULL;

200

201 - if(!netdata_spinlock_trylock(&extent_buffer_globals.protected.spinlock))

201 + if(!spinlock_trylock(&extent_buffer_globals.protected.spinlock))

202 return;

203

204 if(extent_buffer_globals.protected.available_items && extent_buffer_globals.protected.available > 1) {

@@ -207,7 +207,7 @@ void extent_buffer_cleanup1(void) {

207 extent_buffer_globals.protected.available--;

208 }

209

210 - netdata_spinlock_unlock(&extent_buffer_globals.protected.spinlock);

210 + spinlock_unlock(&extent_buffer_globals.protected.spinlock);

211

212 if(item) {

213 size_t bytes = sizeof(struct extent_buffer) + item->bytes;

@@ -225,13 +225,13 @@ struct extent_buffer *extent_buffer_get(size_t size) {

225 if(size < extent_buffer_globals.max_size)

226 size = extent_buffer_globals.max_size;

227

228 - netdata_spinlock_lock(&extent_buffer_globals.protected.spinlock);

228 + spinlock_lock(&extent_buffer_globals.protected.spinlock);

229 if(likely(extent_buffer_globals.protected.available_items)) {

230 eb = extent_buffer_globals.protected.available_items;

231 DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(extent_buffer_globals.protected.available_items, eb, cache.prev, cache.next);

232 extent_buffer_globals.protected.available--;

233 }

234 - netdata_spinlock_unlock(&extent_buffer_globals.protected.spinlock);

234 + spinlock_unlock(&extent_buffer_globals.protected.spinlock);

235

236 if(unlikely(eb && eb->bytes < size)) {

237 size_t bytes = sizeof(struct extent_buffer) + eb->bytes;

@@ -255,10 +255,10 @@ struct extent_buffer *extent_buffer_get(size_t size) {

255 void extent_buffer_release(struct extent_buffer *eb) {

256 if(unlikely(!eb)) return;

257

258 - netdata_spinlock_lock(&extent_buffer_globals.protected.spinlock);

258 + spinlock_lock(&extent_buffer_globals.protected.spinlock);

259 DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(extent_buffer_globals.protected.available_items, eb, cache.prev, cache.next);

260 extent_buffer_globals.protected.available++;

261 - netdata_spinlock_unlock(&extent_buffer_globals.protected.spinlock);

261 + spinlock_unlock(&extent_buffer_globals.protected.spinlock);

262 }

263

264 size_t extent_buffer_cache_size(void) {

@@ -400,20 +400,20 @@ static void pdc_destroy(PDC *pdc) {

400 }

401

402 void pdc_acquire(PDC *pdc) {

403 - netdata_spinlock_lock(&pdc->refcount_spinlock);

403 + spinlock_lock(&pdc->refcount_spinlock);

404

405 if(pdc->refcount < 1)

406 fatal("DBENGINE: pdc is not referenced and cannot be acquired");

407

408 pdc->refcount++;

409 - netdata_spinlock_unlock(&pdc->refcount_spinlock);

409 + spinlock_unlock(&pdc->refcount_spinlock);

410 }

411

412 bool pdc_release_and_destroy_if_unreferenced(PDC *pdc, bool worker, bool router __maybe_unused) {

413 if(unlikely(!pdc))

414 return true;

415

416 - netdata_spinlock_lock(&pdc->refcount_spinlock);

416 + spinlock_lock(&pdc->refcount_spinlock);

417

418 if(pdc->refcount <= 0)

419 fatal("DBENGINE: pdc is not referenced and cannot be released");

@@ -429,12 +429,12 @@ bool pdc_release_and_destroy_if_unreferenced(PDC *pdc, bool worker, bool router

429 }

430

431 if (pdc->refcount == 0) {

432 - netdata_spinlock_unlock(&pdc->refcount_spinlock);

432 + spinlock_unlock(&pdc->refcount_spinlock);

433 pdc_destroy(pdc);

434 return true;

435 }

436

437 - netdata_spinlock_unlock(&pdc->refcount_spinlock);

437 + spinlock_unlock(&pdc->refcount_spinlock);

438 return false;

439 }

440

@@ -456,7 +456,7 @@ static struct rrdeng_cmd *epdl_get_cmd(void *epdl_ptr) {

456 static bool epdl_pending_add(EPDL *epdl) {

457 bool added_new;

458

459 - netdata_spinlock_lock(&epdl->datafile->extent_queries.spinlock);

459 + spinlock_lock(&epdl->datafile->extent_queries.spinlock);

460 Pvoid_t *PValue = JudyLIns(&epdl->datafile->extent_queries.pending_epdl_by_extent_offset_judyL, epdl->extent_offset, PJE0);

461 internal_fatal(!PValue || PValue == PJERR, "DBENGINE: corrupted pending extent judy");

462

@@ -478,20 +478,20 @@ static bool epdl_pending_add(EPDL *epdl) {

478 DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(base, epdl, query.prev, query.next);

479 *PValue = base;

480

481 - netdata_spinlock_unlock(&epdl->datafile->extent_queries.spinlock);

481 + spinlock_unlock(&epdl->datafile->extent_queries.spinlock);

482

483 return added_new;

484 }

485

486 static void epdl_pending_del(EPDL *epdl) {

487 - netdata_spinlock_lock(&epdl->datafile->extent_queries.spinlock);

487 + spinlock_lock(&epdl->datafile->extent_queries.spinlock);

488 if(epdl->head_to_datafile_extent_queries_pending_for_extent) {

489 epdl->head_to_datafile_extent_queries_pending_for_extent = false;

490 int rc = JudyLDel(&epdl->datafile->extent_queries.pending_epdl_by_extent_offset_judyL, epdl->extent_offset, PJE0);

491 (void) rc;

492 internal_fatal(!rc, "DBENGINE: epdl not found in pending list");

493 }

494 - netdata_spinlock_unlock(&epdl->datafile->extent_queries.spinlock);

494 + spinlock_unlock(&epdl->datafile->extent_queries.spinlock);

495 }

496

497 void pdc_to_epdl_router(struct rrdengine_instance *ctx, PDC *pdc, execute_extent_page_details_list_t exec_first_extent_list, execute_extent_page_details_list_t exec_rest_extent_list)

database/engine/rrdengine.c

+34 -34

@@ -351,7 +351,7 @@ static struct {

351 static void wal_cleanup1(void) {

352 WAL *wal = NULL;

353

354 - if(!netdata_spinlock_trylock(&wal_globals.protected.spinlock))

354 + if(!spinlock_trylock(&wal_globals.protected.spinlock))

355 return;

356

357 if(wal_globals.protected.available_items && wal_globals.protected.available > storage_tiers) {

@@ -360,7 +360,7 @@ static void wal_cleanup1(void) {

360 wal_globals.protected.available--;

361 }

362

363 - netdata_spinlock_unlock(&wal_globals.protected.spinlock);

363 + spinlock_unlock(&wal_globals.protected.spinlock);

364

365 if(wal) {

366 posix_memfree(wal->buf);

@@ -375,7 +375,7 @@ WAL *wal_get(struct rrdengine_instance *ctx, unsigned size) {

375

376 WAL *wal = NULL;

377

378 - netdata_spinlock_lock(&wal_globals.protected.spinlock);

378 + spinlock_lock(&wal_globals.protected.spinlock);

379

380 if(likely(wal_globals.protected.available_items)) {

381 wal = wal_globals.protected.available_items;

@@ -384,7 +384,7 @@ WAL *wal_get(struct rrdengine_instance *ctx, unsigned size) {

384 }

385

386 uint64_t transaction_id = __atomic_fetch_add(&ctx->atomic.transaction_id, 1, __ATOMIC_RELAXED);

387 - netdata_spinlock_unlock(&wal_globals.protected.spinlock);

387 + spinlock_unlock(&wal_globals.protected.spinlock);

388

389 if(unlikely(!wal)) {

390 wal = mallocz(sizeof(WAL));

@@ -416,10 +416,10 @@ WAL *wal_get(struct rrdengine_instance *ctx, unsigned size) {

416 void wal_release(WAL *wal) {

417 if(unlikely(!wal)) return;

418

419 - netdata_spinlock_lock(&wal_globals.protected.spinlock);

419 + spinlock_lock(&wal_globals.protected.spinlock);

420 DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(wal_globals.protected.available_items, wal, cache.prev, cache.next);

421 wal_globals.protected.available++;

422 - netdata_spinlock_unlock(&wal_globals.protected.spinlock);

422 + spinlock_unlock(&wal_globals.protected.spinlock);

423 }

424

425 // ----------------------------------------------------------------------------

@@ -459,7 +459,7 @@ void rrdeng_dequeue_epdl_cmd(struct rrdeng_cmd *cmd) {

459 }

460

461 void rrdeng_req_cmd(requeue_callback_t get_cmd_cb, void *data, STORAGE_PRIORITY priority) {

462 - netdata_spinlock_lock(&rrdeng_main.cmd_queue.unsafe.spinlock);

462 + spinlock_lock(&rrdeng_main.cmd_queue.unsafe.spinlock);

463

464 struct rrdeng_cmd *cmd = get_cmd_cb(data);

465 if(cmd) {

@@ -472,7 +472,7 @@ void rrdeng_req_cmd(requeue_callback_t get_cmd_cb, void *data, STORAGE_PRIORITY

472 }

473 }

474

475 - netdata_spinlock_unlock(&rrdeng_main.cmd_queue.unsafe.spinlock);

475 + spinlock_unlock(&rrdeng_main.cmd_queue.unsafe.spinlock);

476 }

477

478 void rrdeng_enq_cmd(struct rrdengine_instance *ctx, enum rrdeng_opcode opcode, void *data, struct completion *completion,

@@ -489,12 +489,12 @@ void rrdeng_enq_cmd(struct rrdengine_instance *ctx, enum rrdeng_opcode opcode, v

489 cmd->priority = priority;

490 cmd->dequeue_cb = dequeue_cb;

491

492 - netdata_spinlock_lock(&rrdeng_main.cmd_queue.unsafe.spinlock);

492 + spinlock_lock(&rrdeng_main.cmd_queue.unsafe.spinlock);

493 DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(rrdeng_main.cmd_queue.unsafe.waiting_items_by_priority[priority], cmd, queue.prev, queue.next);

494 rrdeng_main.cmd_queue.unsafe.waiting++;

495 if(enqueue_cb)

496 enqueue_cb(cmd);

497 - netdata_spinlock_unlock(&rrdeng_main.cmd_queue.unsafe.spinlock);

497 + spinlock_unlock(&rrdeng_main.cmd_queue.unsafe.spinlock);

498

499 fatal_assert(0 == uv_async_send(&rrdeng_main.async));

500 }

@@ -532,7 +532,7 @@ static inline struct rrdeng_cmd rrdeng_deq_cmd(bool from_worker) {

532 }

533

534 // find an opcode to execute from the queue

535 - netdata_spinlock_lock(&rrdeng_main.cmd_queue.unsafe.spinlock);

535 + spinlock_lock(&rrdeng_main.cmd_queue.unsafe.spinlock);

536 for(STORAGE_PRIORITY priority = min_priority; priority <= max_priority ; priority++) {

537 cmd = rrdeng_main.cmd_queue.unsafe.waiting_items_by_priority[priority];

538 if(cmd) {

@@ -559,7 +559,7 @@ static inline struct rrdeng_cmd rrdeng_deq_cmd(bool from_worker) {

559 cmd->dequeue_cb = NULL;

560 }

561

562 - netdata_spinlock_unlock(&rrdeng_main.cmd_queue.unsafe.spinlock);

562 + spinlock_unlock(&rrdeng_main.cmd_queue.unsafe.spinlock);

563

564 struct rrdeng_cmd ret;

565 if(cmd) {

@@ -712,9 +712,9 @@ static void *extent_flushed_to_open_tp_worker(struct rrdengine_instance *ctx __m

712 posix_memfree(xt_io_descr->buf);

713 extent_io_descriptor_release(xt_io_descr);

714

715 - netdata_spinlock_lock(&datafile->writers.spinlock);

715 + spinlock_lock(&datafile->writers.spinlock);

716 datafile->writers.flushed_to_open_running--;

717 - netdata_spinlock_unlock(&datafile->writers.spinlock);

717 + spinlock_unlock(&datafile->writers.spinlock);

718

719 if(datafile->fileno != ctx_last_fileno_get(ctx) && still_running)

720 // we just finished a flushing on a datafile that is not the active one

@@ -738,10 +738,10 @@ static void after_extent_write_datafile_io(uv_fs_t *uv_fs_request) {

738

739 journalfile_v1_extent_write(ctx, xt_io_descr->datafile, xt_io_descr->wal, &rrdeng_main.loop);

740

741 - netdata_spinlock_lock(&datafile->writers.spinlock);

741 + spinlock_lock(&datafile->writers.spinlock);

742 datafile->writers.running--;

743 datafile->writers.flushed_to_open_running++;

744 - netdata_spinlock_unlock(&datafile->writers.spinlock);

744 + spinlock_unlock(&datafile->writers.spinlock);

745

746 rrdeng_enq_cmd(xt_io_descr->ctx,

747 RRDENG_OPCODE_FLUSHED_TO_OPEN,

@@ -756,12 +756,12 @@ static void after_extent_write_datafile_io(uv_fs_t *uv_fs_request) {

756

757 static bool datafile_is_full(struct rrdengine_instance *ctx, struct rrdengine_datafile *datafile) {

758 bool ret = false;

759 - netdata_spinlock_lock(&datafile->writers.spinlock);

759 + spinlock_lock(&datafile->writers.spinlock);

760

761 if(ctx_is_available_for_queries(ctx) && datafile->pos > rrdeng_target_data_file_size(ctx))

762 ret = true;

763

764 - netdata_spinlock_unlock(&datafile->writers.spinlock);

764 + spinlock_unlock(&datafile->writers.spinlock);

765

766 return ret;

767 }

@@ -773,9 +773,9 @@ static struct rrdengine_datafile *get_datafile_to_write_extent(struct rrdengine_

773 uv_rwlock_rdlock(&ctx->datafiles.rwlock);

774 datafile = ctx->datafiles.first->prev;

775 // become a writer on this datafile, to prevent it from vanishing

776 - netdata_spinlock_lock(&datafile->writers.spinlock);

776 + spinlock_lock(&datafile->writers.spinlock);

777 datafile->writers.running++;

778 - netdata_spinlock_unlock(&datafile->writers.spinlock);

778 + spinlock_unlock(&datafile->writers.spinlock);

779 uv_rwlock_rdunlock(&ctx->datafiles.rwlock);

780

781 if(datafile_is_full(ctx, datafile)) {

@@ -791,7 +791,7 @@ static struct rrdengine_datafile *get_datafile_to_write_extent(struct rrdengine_

791 datafile = ctx->datafiles.first->prev;

792 uv_rwlock_rdunlock(&ctx->datafiles.rwlock);

793

794 - if(datafile_is_full(ctx, datafile) && create_new_datafile_pair(ctx) == 0)

794 + if(datafile_is_full(ctx, datafile) && create_new_datafile_pair(ctx, true) == 0)

795 rrdeng_enq_cmd(ctx, RRDENG_OPCODE_JOURNAL_INDEX, datafile, NULL, STORAGE_PRIORITY_INTERNAL_DBENGINE, NULL,

796 NULL);

797

@@ -801,15 +801,15 @@ static struct rrdengine_datafile *get_datafile_to_write_extent(struct rrdengine_

801 uv_rwlock_rdlock(&ctx->datafiles.rwlock);

802 datafile = ctx->datafiles.first->prev;

803 // become a writer on this datafile, to prevent it from vanishing

804 - netdata_spinlock_lock(&datafile->writers.spinlock);

804 + spinlock_lock(&datafile->writers.spinlock);

805 datafile->writers.running++;

806 - netdata_spinlock_unlock(&datafile->writers.spinlock);

806 + spinlock_unlock(&datafile->writers.spinlock);

807 uv_rwlock_rdunlock(&ctx->datafiles.rwlock);

808

809 // release the writers on the old datafile

810 - netdata_spinlock_lock(&old_datafile->writers.spinlock);

810 + spinlock_lock(&old_datafile->writers.spinlock);

811 old_datafile->writers.running--;

812 - netdata_spinlock_unlock(&old_datafile->writers.spinlock);

812 + spinlock_unlock(&old_datafile->writers.spinlock);

813 }

814

815 return datafile;

@@ -921,11 +921,11 @@ static struct extent_io_descriptor *datafile_extent_build(struct rrdengine_insta

921 real_io_size = ALIGN_BYTES_CEILING(size_bytes);

922

923 datafile = get_datafile_to_write_extent(ctx);

924 - netdata_spinlock_lock(&datafile->writers.spinlock);

924 + spinlock_lock(&datafile->writers.spinlock);

925 xt_io_descr->datafile = datafile;

926 xt_io_descr->pos = datafile->pos;

927 datafile->pos += real_io_size;

928 - netdata_spinlock_unlock(&datafile->writers.spinlock);

928 + spinlock_unlock(&datafile->writers.spinlock);

929

930 xt_io_descr->bytes = size_bytes;

931 xt_io_descr->uv_fs_request.data = xt_io_descr;

@@ -1334,11 +1334,11 @@ static void *populate_mrg_tp_worker(struct rrdengine_instance *ctx __maybe_unuse

1334 // find a datafile to work

1335 uv_rwlock_rdlock(&ctx->datafiles.rwlock);

1336 for(datafile = ctx->datafiles.first; datafile ; datafile = datafile->next) {

1337 - if(!netdata_spinlock_trylock(&datafile->populate_mrg.spinlock))

1337 + if(!spinlock_trylock(&datafile->populate_mrg.spinlock))

1338 continue;

1339

1340 if(datafile->populate_mrg.populated) {

1341 - netdata_spinlock_unlock(&datafile->populate_mrg.spinlock);

1341 + spinlock_unlock(&datafile->populate_mrg.spinlock);

1342 continue;

1343 }

1344

@@ -1352,7 +1352,7 @@ static void *populate_mrg_tp_worker(struct rrdengine_instance *ctx __maybe_unuse

1352

1353 journalfile_v2_populate_retention_to_mrg(ctx, datafile->journalfile);

1354 datafile->populate_mrg.populated = true;

1355 - netdata_spinlock_unlock(&datafile->populate_mrg.spinlock);

1355 + spinlock_unlock(&datafile->populate_mrg.spinlock);

1356

1357 } while(1);

1358

@@ -1496,9 +1496,9 @@ static void *journal_v2_indexing_tp_worker(struct rrdengine_instance *ctx __mayb

1496 continue;

1497 }

1498

1499 - netdata_spinlock_lock(&datafile->writers.spinlock);

1499 + spinlock_lock(&datafile->writers.spinlock);

1500 bool available = (datafile->writers.running || datafile->writers.flushed_to_open_running) ? false : true;

1501 - netdata_spinlock_unlock(&datafile->writers.spinlock);

1501 + spinlock_unlock(&datafile->writers.spinlock);

1502

1503 if(!available) {

1504 info("DBENGINE: journal file %u needs to be indexed, but it has writers working on it - skipping it for now", datafile->fileno);

@@ -1623,7 +1623,7 @@ bool rrdeng_dbengine_spawn(struct rrdengine_instance *ctx __maybe_unused) {

1623 static bool spawned = false;

1624 static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER;

1625

1626 - netdata_spinlock_lock(&spinlock);

1626 + spinlock_lock(&spinlock);

1627

1628 if(!spawned) {

1629 int ret;

@@ -1658,7 +1658,7 @@ bool rrdeng_dbengine_spawn(struct rrdengine_instance *ctx __maybe_unused) {

1658 spawned = true;

1659 }

1660

1661 - netdata_spinlock_unlock(&spinlock);

1661 + spinlock_unlock(&spinlock);

1662 return true;

1663 }

1664

database/engine/rrdengine.h

+33 -25

@@ -34,31 +34,6 @@ struct rrdeng_cmd;

34 #define RRDENG_FILE_NUMBER_SCAN_TMPL "%1u-%10u"

35 #define RRDENG_FILE_NUMBER_PRINT_TMPL "%1.1u-%10.10u"

37 -typedef struct page_details_control {

38 - struct rrdengine_instance *ctx;

39 - struct metric *metric;

40 -

41 - struct completion prep_completion;

42 - struct completion page_completion; // sync between the query thread and the workers

43 -

44 - Pvoid_t page_list_JudyL; // the list of page details

45 - unsigned completed_jobs; // the number of jobs completed last time the query thread checked

46 - bool workers_should_stop; // true when the query thread left and the workers should stop

47 - bool prep_done;

48 -

49 - SPINLOCK refcount_spinlock; // spinlock to protect refcount

50 - int32_t refcount; // the number of workers currently working on this request + 1 for the query thread

51 - size_t executed_with_gaps;

52 -

53 - time_t start_time_s;

54 - time_t end_time_s;

55 - STORAGE_PRIORITY priority;

56 -

57 - time_t optimal_end_time_s;

58 -} PDC;

59 -

60 -PDC *pdc_get(void);

61 -

37 typedef enum __attribute__ ((__packed__)) {

38 // final status for all pages

39 // if a page does not have one of these, it is considered unroutable

@@ -99,6 +74,34 @@ typedef enum __attribute__ ((__packed__)) {

75 #define PDC_PAGE_QUERY_GLOBAL_SKIP_LIST (PDC_PAGE_FAILED | PDC_PAGE_SKIP | PDC_PAGE_INVALID | PDC_PAGE_RELEASED)

77 +typedef struct page_details_control {

78 + struct rrdengine_instance *ctx;

79 + struct metric *metric;

80 +

81 + struct completion prep_completion;

82 + struct completion page_completion; // sync between the query thread and the workers

83 +

84 + Pvoid_t page_list_JudyL; // the list of page details

85 + unsigned completed_jobs; // the number of jobs completed last time the query thread checked

86 + bool workers_should_stop; // true when the query thread left and the workers should stop

87 + bool prep_done;

88 +

89 + PDC_PAGE_STATUS common_status;

90 + size_t pages_to_load_from_disk;

91 +

92 + SPINLOCK refcount_spinlock; // spinlock to protect refcount

93 + int32_t refcount; // the number of workers currently working on this request + 1 for the query thread

94 + size_t executed_with_gaps;

95 +

96 + time_t start_time_s;

97 + time_t end_time_s;

98 + STORAGE_PRIORITY priority;

99 +

100 + time_t optimal_end_time_s;

101 +} PDC;

102 +

103 +PDC *pdc_get(void);

104 +

105 struct page_details {

106 struct {

107 struct rrdengine_datafile *ptr;

@@ -361,6 +364,11 @@ struct rrdengine_instance {

364 struct rrdengine_datafile *first; // oldest - the newest with ->first->prev

365 } datafiles;

366

367 + struct {

368 + RW_SPINLOCK spinlock;

369 + Pvoid_t JudyL;

370 + } njfv2idx;

371 +

372 struct {

373 unsigned last_fileno; // newest index of datafile and journalfile

374 unsigned last_flush_fileno; // newest index of datafile received data

database/engine/rrdengineapi.c

+9 -9

@@ -703,14 +703,14 @@ static void register_query_handle(struct rrdeng_query_handle *handle) {

703 handle->query_pid = gettid();

704 handle->started_time_s = now_realtime_sec();

705

706 - netdata_spinlock_lock(&global_query_handle_spinlock);

706 + spinlock_lock(&global_query_handle_spinlock);

707 DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(global_query_handle_ll, handle, prev, next);

708 - netdata_spinlock_unlock(&global_query_handle_spinlock);

708 + spinlock_unlock(&global_query_handle_spinlock);

709 }

710 static void unregister_query_handle(struct rrdeng_query_handle *handle) {

711 - netdata_spinlock_lock(&global_query_handle_spinlock);

711 + spinlock_lock(&global_query_handle_spinlock);

712 DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(global_query_handle_ll, handle, prev, next);

713 - netdata_spinlock_unlock(&global_query_handle_spinlock);

713 + spinlock_unlock(&global_query_handle_spinlock);

714 }

715 #else

716 static void register_query_handle(struct rrdeng_query_handle *handle __maybe_unused) {

@@ -1076,14 +1076,14 @@ static void rrdeng_populate_mrg(struct rrdengine_instance *ctx) {

1076 if(cpus > datafiles)

1077 cpus = datafiles;

1078

1079 - if(cpus < 1)

1080 - cpus = 1;

1081 -

1079 if(cpus > (size_t)libuv_worker_threads)

1080 cpus = (size_t)libuv_worker_threads;

1081

1085 - if(cpus > MRG_PARTITIONS)

1086 - cpus = MRG_PARTITIONS;

1082 + if(cpus >= MRG_PARTITIONS / 2)

1083 + cpus = MRG_PARTITIONS / 2 - 1;

1084 +

1085 + if(cpus < 1)

1086 + cpus = 1;

1087

1088 info("DBENGINE: populating retention to MRG from %zu journal files of tier %d, using %zu threads...", datafiles, ctx->config.tier, cpus);

1089

database/ram/rrddim_mem.c

+7 -7

@@ -35,15 +35,15 @@ struct mem_metric_handle {

36 static void update_metric_handle_from_rrddim(struct mem_metric_handle *mh, RRDDIM *rd) {

37 mh->counter = rd->rrdset->counter;

38 - mh->entries = rd->rrdset->entries;

39 - mh->current_entry = rd->rrdset->current_entry;

38 + mh->entries = rd->rrdset->db.entries;

39 + mh->current_entry = rd->rrdset->db.current_entry;

40 mh->last_updated_s = rd->rrdset->last_updated.tv_sec;

41 mh->update_every_s = rd->rrdset->update_every;

42 }

44 static void check_metric_handle_from_rrddim(struct mem_metric_handle *mh) {

45 RRDDIM *rd = mh->rd; (void)rd;

46 - internal_fatal(mh->entries != (size_t)rd->rrdset->entries, "RRDDIM: entries do not match");

46 + internal_fatal(mh->entries != (size_t)rd->rrdset->db.entries, "RRDDIM: entries do not match");

47 internal_fatal(mh->update_every_s != rd->rrdset->update_every, "RRDDIM: update every does not match");

48 }

@@ -161,7 +161,7 @@ void rrddim_store_metric_flush(STORAGE_COLLECT_HANDLE *collection_handle) {

161 storage_number empty = pack_storage_number(NAN, SN_FLAG_NONE);

162

163 for(size_t i = 0; i < entries ;i++)

164 - rd->db[i] = empty;

164 + rd->db.data[i] = empty;

165

166 mh->counter = 0;

167 mh->last_updated_s = 0;

@@ -192,7 +192,7 @@ static inline void rrddim_fill_the_gap(STORAGE_COLLECT_HANDLE *collection_handle

192 // fill the dimension

193 size_t c;

194 for(c = 0; c < entries && now_store_s <= now_collect_s ; now_store_s += update_every_s, c++) {

195 - rd->db[current_entry++] = empty;

195 + rd->db.data[current_entry++] = empty;

196

197 if(unlikely(current_entry >= entries))

198 current_entry = 0;

@@ -227,7 +227,7 @@ void rrddim_collect_store_metric(STORAGE_COLLECT_HANDLE *collection_handle,

227 if(unlikely(mh->last_updated_s && point_in_time_s - mh->update_every_s > mh->last_updated_s))

228 rrddim_fill_the_gap(collection_handle, point_in_time_s);

229

230 - rd->db[mh->current_entry] = pack_storage_number(n, flags);

230 + rd->db.data[mh->current_entry] = pack_storage_number(n, flags);

231 mh->counter++;

232 mh->current_entry = (mh->current_entry + 1) >= mh->entries ? 0 : mh->current_entry + 1;

233 mh->last_updated_s = point_in_time_s;

@@ -390,7 +390,7 @@ STORAGE_POINT rrddim_query_next_metric(struct storage_engine_query_handle *handl

390 return sp;

391 }

392

393 - storage_number n = rd->db[slot++];

393 + storage_number n = rd->db.data[slot++];

394 if(unlikely(slot >= entries)) slot = 0;

395

396 h->slot = slot;

database/rrd.h

+47 -43

@@ -235,6 +235,8 @@ typedef enum __attribute__ ((__packed__)) rrddim_options {

235 RRDDIM_OPTION_HIDDEN = (1 << 0), // this dimension will not be offered to callers

236 RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS = (1 << 1), // do not offer RESET or OVERFLOW info to callers

237 RRDDIM_OPTION_BACKFILLED_HIGH_TIERS = (1 << 2), // when set, we have backfilled higher tiers

238 + RRDDIM_OPTION_UPDATED = (1 << 3), // single-threaded collector updated flag

239 + RRDDIM_OPTION_EXPOSED = (1 << 4), // single-threaded collector exposed flag

240

241 // this is 8-bit

242 } RRDDIM_OPTIONS;

@@ -354,15 +356,10 @@ struct rrddim {

356 RRD_MEMORY_MODE rrd_memory_mode; // the memory mode for this dimension

357 RRDDIM_FLAGS flags; // run time changing status flags

358

357 - bool updated; // 1 when the dimension has been updated since the last processing

358 - bool exposed; // 1 when set what have sent this dimension to the central netdata

359 + int32_t multiplier; // the multiplier of the collected values

360 + int32_t divisor; // the divider of the collected values

361

360 - collected_number multiplier; // the multiplier of the collected values

361 - collected_number divisor; // the divider of the collected values

362 -

363 - int update_every; // every how many seconds is this updated

364 - // TODO - remove update_every from rrddim

365 - // it is always the same in rrdset

362 + uint32_t collections_counter; // the number of times we added values to this rrddim

363

364 // ------------------------------------------------------------------------

365 // operational state members

@@ -373,19 +370,15 @@ struct rrddim {

370 // linking to siblings and parents

371

372 struct rrdset *rrdset;

376 -

373 RRDMETRIC_ACQUIRED *rrdmetric; // the rrdmetric of this dimension

374

375 // ------------------------------------------------------------------------

376 // data collection members

377

382 - struct rrddim_tier tiers[RRD_STORAGE_TIERS]; // our tiers of databases

383 -

378 struct timeval last_collected_time; // when was this dimension last updated

379 // this is actual date time we updated the last_collected_value

380 // THIS IS DIFFERENT FROM THE SAME MEMBER OF RRDSET

381

388 - size_t collections_counter; // the number of times we added values to this rrddim

382 collected_number collected_value_max; // the absolute maximum of the collected value

383

384 NETDATA_DOUBLE calculated_value; // the current calculated value, after applying the algorithm - resets to zero after being used

@@ -406,14 +399,30 @@ struct rrddim {

399 // TODO - they should be managed by storage engine

400 // (RRDDIM_DB_STATE ptr to an undefined structure, and a call to clean this up during destruction)

401

409 - size_t memsize; // the memory allocated for this dimension (without RRDDIM)

410 - void *rd_on_file; // pointer to the header written on disk

411 - storage_number *db; // the array of values

402 + struct {

403 + size_t memsize; // the memory allocated for this dimension (without RRDDIM)

404 + void *rd_on_file; // pointer to the header written on disk

405 + storage_number *data; // the array of values

406 + } db;

407 +

408 + // ------------------------------------------------------------------------

409 +

410 + struct rrddim_tier tiers[]; // our tiers of databases

411 };

412

413 +size_t rrddim_size(void);

414 +

415 #define rrddim_id(rd) string2str((rd)->id)

416 #define rrddim_name(rd) string2str((rd) ->name)

417

418 +#define rrddim_check_updated(rd) ((rd)->options & RRDDIM_OPTION_UPDATED)

419 +#define rrddim_set_updated(rd) (rd)->options |= RRDDIM_OPTION_UPDATED

420 +#define rrddim_clear_updated(rd) (rd)->options &= ~RRDDIM_OPTION_UPDATED

421 +

422 +#define rrddim_check_exposed(rd) ((rd)->options & RRDDIM_OPTION_EXPOSED)

423 +#define rrddim_set_exposed(rd) (rd)->options |= RRDDIM_OPTION_EXPOSED

424 +#define rrddim_clear_exposed(rd) (rd)->options &= ~RRDDIM_OPTION_EXPOSED

425 +

426 // returns the RRDDIM cache filename, or NULL if it does not exist

427 const char *rrddim_cache_filename(RRDDIM *rd);

428

@@ -770,18 +779,15 @@ struct rrdset {

779 STRING *plugin_name; // the name of the plugin that generated this

780 STRING *module_name; // the name of the plugin module that generated this

781

773 - RRDSET_TYPE chart_type; // line, area, stacked

774 -

775 - long priority; // the sorting priority of this chart

776 -

777 - int update_every; // data collection frequency

782 + int32_t priority; // the sorting priority of this chart

783 + int32_t update_every; // data collection frequency

784

785 DICTIONARY *rrdlabels; // chart labels

786 DICTIONARY *rrdsetvar_root_index; // chart variables

787 DICTIONARY *rrddimvar_root_index; // dimension variables

788 // we use this dictionary to manage their allocation

789

784 - rrd_ml_chart_t *ml_chart;

790 + RRDSET_TYPE chart_type; // line, area, stacked

791

792 // ------------------------------------------------------------------------

793 // operational state members

@@ -791,6 +797,8 @@ struct rrdset {

797

798 DICTIONARY *rrddim_root_index; // dimensions index

799

800 + rrd_ml_chart_t *ml_chart;

801 +

802 STORAGE_METRICS_GROUP *storage_metrics_groups[RRD_STORAGE_TIERS];

803

804 // ------------------------------------------------------------------------

@@ -806,11 +814,10 @@ struct rrdset {

814

815 SPINLOCK data_collection_lock;

816

809 - size_t counter; // the number of times we added values to this database

810 - size_t counter_done; // the number of times rrdset_done() has been called

817 + uint32_t counter; // the number of times we added values to this database

818 + uint32_t counter_done; // the number of times rrdset_done() has been called

819

820 time_t last_accessed_time_s; // the last time this RRDSET has been accessed

813 -

821 usec_t usec_since_last_update; // the time in microseconds since the last collection of data

822

823 struct timeval last_updated; // when this data set was last updated (updated every time the rrd_stats_done() function)

@@ -830,18 +837,15 @@ struct rrdset {

837 // TODO - they should be managed by storage engine

838 // (RRDSET_DB_STATE ptr to an undefined structure, and a call to clean this up during destruction)

839

833 - char *cache_dir; // the directory to store dimensions

834 - void *st_on_file; // compatibility with V019 RRDSET files

835 -

836 - // ------------------------------------------------------------------------

837 - // db mode RAM, SAVE, MAP, ALLOC, NONE specifics

838 - // TODO - they should be managed by storage engine

839 - // (RRDSET_DB_STATE ptr to an undefined structure, and a call to clean this up during destruction)

840 + struct {

841 + char *cache_dir; // the directory to store dimensions

842 + void *st_on_file; // compatibility with V019 RRDSET files

843

841 - long entries; // total number of entries in the data set

844 + int32_t entries; // total number of entries in the data set

845

843 - long current_entry; // the entry that is currently being updated

844 - // it goes around in a round-robin fashion

846 + int32_t current_entry; // the entry that is currently being updated

847 + // it goes around in a round-robin fashion

848 + } db;

849

850 // ------------------------------------------------------------------------

851 // exporting to 3rd party time-series members

@@ -867,9 +871,9 @@ struct rrdset {

871 } alerts;

872

873 struct {

870 - size_t pos;

871 - size_t size;

872 - size_t used;

874 + uint32_t pos;

875 + uint32_t size;

876 + uint32_t used;

877 RRDDIM_ACQUIRED **rda;

878 } pluginsd;

879

@@ -1158,8 +1162,8 @@ struct rrdhost {

1162 RRDHOST_FLAGS flags; // runtime flags about this RRDHOST (atomics on this)

1163 RRDHOST_FLAGS *exporting_flags; // array of flags for exporting connector instances

1164

1161 - int rrd_update_every; // the update frequency of the host

1162 - long rrd_history_entries; // the number of history entries for the host's charts

1165 + int32_t rrd_update_every; // the update frequency of the host

1166 + int32_t rrd_history_entries; // the number of history entries for the host's charts

1167

1168 RRD_MEMORY_MODE rrd_memory_mode; // the configured memory more for the charts of this host

1169 // the actual per tier is at .db[tier].mode

@@ -1505,8 +1509,8 @@ RRDDIM *rrddim_add_custom(RRDSET *st

1509

1510 int rrddim_reset_name(RRDSET *st, RRDDIM *rd, const char *name);

1511 int rrddim_set_algorithm(RRDSET *st, RRDDIM *rd, RRD_ALGORITHM algorithm);

1508 -int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, collected_number multiplier);

1509 -int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor);

1512 +int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, int32_t multiplier);

1513 +int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, int32_t divisor);

1514

1515 RRDDIM *rrddim_find(RRDSET *st, const char *id);

1516 RRDDIM_ACQUIRED *rrddim_find_and_acquire(RRDSET *st, const char *id);

@@ -1572,10 +1576,10 @@ void store_metric_collection_completed(void);

1576

1577 static inline void rrdhost_retention(RRDHOST *host, time_t now, bool online, time_t *from, time_t *to) {

1578 time_t first_time_s = 0, last_time_s = 0;

1575 - netdata_spinlock_lock(&host->retention.spinlock);

1579 + spinlock_lock(&host->retention.spinlock);

1580 first_time_s = host->retention.first_time_s;

1581 last_time_s = host->retention.last_time_s;

1578 - netdata_spinlock_unlock(&host->retention.spinlock);

1582 + spinlock_unlock(&host->retention.spinlock);

1583

1584 if(from)

1585 *from = first_time_s;

database/rrddim.c

+53 -45

@@ -46,8 +46,6 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v

46 rd->divisor = ctr->divisor;

47 if(!rd->divisor) rd->divisor = 1;

49 - rd->update_every = st->update_every;

50 -

49 rd->rrdset = st;

51 if(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST))

@@ -61,27 +59,27 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v

59 }

61 if(ctr->memory_mode == RRD_MEMORY_MODE_RAM) {

64 - size_t entries = st->entries;

62 + size_t entries = st->db.entries;

63 if(!entries) entries = 5;

67 - rd->db = netdata_mmap(NULL, entries * sizeof(storage_number), MAP_PRIVATE, 1, false, NULL);

68 - if(!rd->db) {

65 + rd->db.data = netdata_mmap(NULL, entries * sizeof(storage_number), MAP_PRIVATE, 1, false, NULL);

66 + if(!rd->db.data) {

67 info("Failed to use memory mode ram for chart '%s', dimension '%s', falling back to alloc", rrdset_name(st), rrddim_name(rd));

68 ctr->memory_mode = RRD_MEMORY_MODE_ALLOC;

69 }

70 else {

73 - rd->memsize = entries * sizeof(storage_number);

74 - __atomic_add_fetch(&rrddim_db_memory_size, rd->memsize, __ATOMIC_RELAXED);

71 + rd->db.memsize = entries * sizeof(storage_number);

72 + __atomic_add_fetch(&rrddim_db_memory_size, rd->db.memsize, __ATOMIC_RELAXED);

73 }

74 }

76 if(ctr->memory_mode == RRD_MEMORY_MODE_ALLOC || ctr->memory_mode == RRD_MEMORY_MODE_NONE) {

79 - size_t entries = st->entries;

77 + size_t entries = st->db.entries;

78 if(entries < 5) entries = 5;

82 - rd->db = rrddim_alloc_db(entries);

83 - rd->memsize = entries * sizeof(storage_number);

84 - __atomic_add_fetch(&rrddim_db_memory_size, rd->memsize, __ATOMIC_RELAXED);

80 + rd->db.data = rrddim_alloc_db(entries);

81 + rd->db.memsize = entries * sizeof(storage_number);

82 + __atomic_add_fetch(&rrddim_db_memory_size, rd->db.memsize, __ATOMIC_RELAXED);

83 }

85 rd->rrd_memory_mode = ctr->memory_mode;

@@ -135,7 +133,9 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v

133 if(td && (td->algorithm != rd->algorithm || ABS(td->multiplier) != ABS(rd->multiplier) || ABS(td->divisor) != ABS(rd->divisor))) {

134 if(!rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) {

135 #ifdef NETDATA_INTERNAL_CHECKS

138 - info("Dimension '%s' added on chart '%s' of host '%s' is not homogeneous to other dimensions already present (algorithm is '%s' vs '%s', multiplier is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ", divisor is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ").",

136 + info("Dimension '%s' added on chart '%s' of host '%s' is not homogeneous to other dimensions already "

137 + "present (algorithm is '%s' vs '%s', multiplier is %d vs %d, "

138 + "divisor is %d vs %d).",

139 rrddim_name(rd),

140 rrdset_name(st),

141 rrdhost_hostname(host),

@@ -223,13 +223,13 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v

223 rd->tiers[tier].db_metric_handle = NULL;

224 }

225

226 - if(rd->db) {

227 - __atomic_sub_fetch(&rrddim_db_memory_size, rd->memsize, __ATOMIC_RELAXED);

226 + if(rd->db.data) {

227 + __atomic_sub_fetch(&rrddim_db_memory_size, rd->db.memsize, __ATOMIC_RELAXED);

228

229 if(rd->rrd_memory_mode == RRD_MEMORY_MODE_RAM)

230 - netdata_munmap(rd->db, rd->memsize);

230 + netdata_munmap(rd->db.data, rd->db.memsize);

231 else

232 - freez(rd->db);

232 + freez(rd->db.data);

233 }

234

235 string_freez(rd->id);

@@ -289,10 +289,14 @@ static void rrddim_react_callback(const DICTIONARY_ITEM *item __maybe_unused, vo

289 rrdcontext_updated_rrddim(rd);

290 }

291

292 +size_t rrddim_size(void) {

293 + return sizeof(RRDDIM) + storage_tiers * sizeof(struct rrddim_tier);

294 +}

295 +

296 void rrddim_index_init(RRDSET *st) {

297 if(!st->rrddim_root_index) {

298 st->rrddim_root_index = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,

295 - &dictionary_stats_category_rrdset_rrddim, sizeof(RRDDIM));

299 + &dictionary_stats_category_rrdset_rrddim, rrddim_size());

300

301 dictionary_register_insert_callback(st->rrddim_root_index, rrddim_insert_callback, NULL);

302 dictionary_register_conflict_callback(st->rrddim_root_index, rrddim_conflict_callback, NULL);

@@ -365,7 +369,7 @@ inline int rrddim_reset_name(RRDSET *st, RRDDIM *rd, const char *name) {

369

370 rrddimvar_rename_all(rd);

371

368 - rd->exposed = 0;

372 + rrddim_clear_exposed(rd);

373 rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);

374

375 return 1;

@@ -377,33 +381,35 @@ inline int rrddim_set_algorithm(RRDSET *st, RRDDIM *rd, RRD_ALGORITHM algorithm)

381

382 debug(D_RRD_CALLS, "Updating algorithm of dimension '%s/%s' from %s to %s", rrdset_id(st), rrddim_name(rd), rrd_algorithm_name(rd->algorithm), rrd_algorithm_name(algorithm));

383 rd->algorithm = algorithm;

380 - rd->exposed = 0;

384 + rrddim_clear_exposed(rd);

385 rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);

386 rrdset_flag_set(st, RRDSET_FLAG_HOMOGENEOUS_CHECK);

387 rrdcontext_updated_rrddim_algorithm(rd);

388 return 1;

389 }

390

387 -inline int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, collected_number multiplier) {

391 +inline int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, int32_t multiplier) {

392 if(unlikely(rd->multiplier == multiplier))

393 return 0;

394

391 - debug(D_RRD_CALLS, "Updating multiplier of dimension '%s/%s' from " COLLECTED_NUMBER_FORMAT " to " COLLECTED_NUMBER_FORMAT, rrdset_id(st), rrddim_name(rd), rd->multiplier, multiplier);

395 + debug(D_RRD_CALLS, "Updating multiplier of dimension '%s/%s' from %d to %d",

396 + rrdset_id(st), rrddim_name(rd), rd->multiplier, multiplier);

397 rd->multiplier = multiplier;

393 - rd->exposed = 0;

398 + rrddim_clear_exposed(rd);

399 rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);

400 rrdset_flag_set(st, RRDSET_FLAG_HOMOGENEOUS_CHECK);

401 rrdcontext_updated_rrddim_multiplier(rd);

402 return 1;

403 }

404

400 -inline int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor) {

405 +inline int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, int32_t divisor) {

406 if(unlikely(rd->divisor == divisor))

407 return 0;

408

404 - debug(D_RRD_CALLS, "Updating divisor of dimension '%s/%s' from " COLLECTED_NUMBER_FORMAT " to " COLLECTED_NUMBER_FORMAT, rrdset_id(st), rrddim_name(rd), rd->divisor, divisor);

409 + debug(D_RRD_CALLS, "Updating divisor of dimension '%s/%s' from %d to %d",

410 + rrdset_id(st), rrddim_name(rd), rd->divisor, divisor);

411 rd->divisor = divisor;

406 - rd->exposed = 0;

412 + rrddim_clear_exposed(rd);

413 rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);

414 rrdset_flag_set(st, RRDSET_FLAG_HOMOGENEOUS_CHECK);

415 rrdcontext_updated_rrddim_divisor(rd);

@@ -471,7 +477,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st

477 .memory_mode = memory_mode,

478 };

479

474 - RRDDIM *rd = dictionary_set_advanced(st->rrddim_root_index, tmp.id, -1, NULL, sizeof(RRDDIM), &tmp);

480 + RRDDIM *rd = dictionary_set_advanced(st->rrddim_root_index, tmp.id, -1, NULL, rrddim_size(), &tmp);

481 return(rd);

482 }

483

@@ -561,7 +567,7 @@ collected_number rrddim_timed_set_by_pointer(RRDSET *st __maybe_unused, RRDDIM *

567

568 rd->last_collected_time = collected_time;

569 rd->collected_value = value;

564 - rd->updated = 1;

570 + rrddim_set_updated(rd);

571 rd->collections_counter++;

572

573 collected_number v = (value >= 0) ? value : -value;

@@ -635,8 +641,8 @@ size_t rrddim_memory_file_header_size(void) {

641 }

642

643 void rrddim_memory_file_update(RRDDIM *rd) {

638 - if(!rd || !rd->rd_on_file) return;

639 - struct rrddim_map_save_v019 *rd_on_file = rd->rd_on_file;

644 + if(!rd || !rd->db.rd_on_file) return;

645 + struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file;

646

647 rd_on_file->last_collected_time.tv_sec = rd->last_collected_time.tv_sec;

648 rd_on_file->last_collected_time.tv_usec = rd->last_collected_time.tv_usec;

@@ -644,33 +650,33 @@ void rrddim_memory_file_update(RRDDIM *rd) {

650 }

651

652 void rrddim_memory_file_free(RRDDIM *rd) {

647 - if(!rd || !rd->rd_on_file) return;

653 + if(!rd || !rd->db.rd_on_file) return;

654

655 // needed for memory mode map, to save the latest state

656 rrddim_memory_file_update(rd);

657

652 - struct rrddim_map_save_v019 *rd_on_file = rd->rd_on_file;

658 + struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file;

659 __atomic_sub_fetch(&rrddim_db_memory_size, rd_on_file->memsize + strlen(rd_on_file->cache_filename), __ATOMIC_RELAXED);

660 freez(rd_on_file->cache_filename);

661 netdata_munmap(rd_on_file, rd_on_file->memsize);

662

663 // remove the pointers from the RRDDIM

658 - rd->rd_on_file = NULL;

659 - rd->db = NULL;

664 + rd->db.rd_on_file = NULL;

665 + rd->db.data = NULL;

666 }

667

668 const char *rrddim_cache_filename(RRDDIM *rd) {

663 - if(!rd || !rd->rd_on_file) return NULL;

664 - struct rrddim_map_save_v019 *rd_on_file = rd->rd_on_file;

669 + if(!rd || !rd->db.rd_on_file) return NULL;

670 + struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file;

671 return rd_on_file->cache_filename;

672 }

673

674 void rrddim_memory_file_save(RRDDIM *rd) {

669 - if(!rd || !rd->rd_on_file) return;

675 + if(!rd || !rd->db.rd_on_file) return;

676

677 rrddim_memory_file_update(rd);

678

673 - struct rrddim_map_save_v019 *rd_on_file = rd->rd_on_file;

679 + struct rrddim_map_save_v019 *rd_on_file = rd->db.rd_on_file;

680 if(rd_on_file->rrd_memory_mode != RRD_MEMORY_MODE_SAVE) return;

681

682 memory_file_save(rd_on_file->cache_filename, rd_on_file, rd_on_file->memsize);

@@ -682,7 +688,7 @@ bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MO

688

689 struct rrddim_map_save_v019 *rd_on_file = NULL;

690

685 - unsigned long size = sizeof(struct rrddim_map_save_v019) + (st->entries * sizeof(storage_number));

691 + unsigned long size = sizeof(struct rrddim_map_save_v019) + (st->db.entries * sizeof(storage_number));

692

693 char filename[FILENAME_MAX + 1];

694 char fullfilename[FILENAME_MAX + 1];

@@ -728,10 +734,12 @@ bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MO

734 fullfilename, rd->algorithm, rrd_algorithm_name(rd->algorithm), rd_on_file->algorithm, rrd_algorithm_name(rd_on_file->algorithm));

735

736 if(rd_on_file->multiplier != rd->multiplier)

731 - info("File %s does not have the expected multiplier (expected " COLLECTED_NUMBER_FORMAT ", found " COLLECTED_NUMBER_FORMAT "). Previous values may be wrong.", fullfilename, rd->multiplier, rd_on_file->multiplier);

737 + info("File %s does not have the expected multiplier (expected %d, found %ld). "

738 + "Previous values may be wrong.", fullfilename, rd->multiplier, (long)rd_on_file->multiplier);

739

740 if(rd_on_file->divisor != rd->divisor)

734 - info("File %s does not have the expected divisor (expected " COLLECTED_NUMBER_FORMAT ", found " COLLECTED_NUMBER_FORMAT "). Previous values may be wrong.", fullfilename, rd->divisor, rd_on_file->divisor);

741 + info("File %s does not have the expected divisor (expected %d, found %ld). "

742 + "Previous values may be wrong.", fullfilename, rd->divisor, (long)rd_on_file->divisor);

743 }

744

745 // zero the entire header

@@ -742,17 +750,17 @@ bool rrddim_memory_load_or_create_map_save(RRDSET *st, RRDDIM *rd, RRD_MEMORY_MO

750 rd_on_file->algorithm = rd->algorithm;

751 rd_on_file->multiplier = rd->multiplier;

752 rd_on_file->divisor = rd->divisor;

745 - rd_on_file->entries = st->entries;

746 - rd_on_file->update_every = rd->update_every;

753 + rd_on_file->entries = st->db.entries;

754 + rd_on_file->update_every = rd->rrdset->update_every;

755 rd_on_file->memsize = size;

756 rd_on_file->rrd_memory_mode = memory_mode;

757 rd_on_file->cache_filename = strdupz(fullfilename);

758

759 __atomic_add_fetch(&rrddim_db_memory_size, rd_on_file->memsize + strlen(rd_on_file->cache_filename), __ATOMIC_RELAXED);

760

753 - rd->db = &rd_on_file->values[0];

754 - rd->rd_on_file = rd_on_file;

755 - rd->memsize = size;

761 + rd->db.data = &rd_on_file->values[0];

762 + rd->db.rd_on_file = rd_on_file;

763 + rd->db.memsize = size;

764 rrddim_memory_file_update(rd);

765

766 return true;

database/rrdhost.c

+17 -8

@@ -529,7 +529,7 @@ int is_legacy = 1;

529 ", program_version '%s'"

530 ", update every %d"

531 ", memory mode %s"

532 - ", history entries %ld"

532 + ", history entries %d"

533 ", streaming %s"

534 " (to '%s' with api key '%s')"

535 ", health %s"

@@ -594,7 +594,7 @@ static void rrdhost_update(RRDHOST *host

594 {

595 UNUSED(guid);

596

597 - netdata_spinlock_lock(&host->rrdhost_update_lock);

597 + spinlock_lock(&host->rrdhost_update_lock);

598

599 host->health.health_enabled = (mode == RRD_MEMORY_MODE_NONE) ? 0 : health_enabled;

600

@@ -633,13 +633,19 @@ static void rrdhost_update(RRDHOST *host

633 }

634

635 if(host->rrd_update_every != update_every)

636 - error("Host '%s' has an update frequency of %d seconds, but the wanted one is %d seconds. Restart netdata here to apply the new settings.", rrdhost_hostname(host), host->rrd_update_every, update_every);

636 + error("Host '%s' has an update frequency of %d seconds, but the wanted one is %d seconds. "

637 + "Restart netdata here to apply the new settings.",

638 + rrdhost_hostname(host), host->rrd_update_every, update_every);

639

640 if(host->rrd_memory_mode != mode)

639 - error("Host '%s' has memory mode '%s', but the wanted one is '%s'. Restart netdata here to apply the new settings.", rrdhost_hostname(host), rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode));

641 + error("Host '%s' has memory mode '%s', but the wanted one is '%s'. "

642 + "Restart netdata here to apply the new settings.",

643 + rrdhost_hostname(host), rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode));

644

645 else if(host->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && host->rrd_history_entries < history)

642 - error("Host '%s' has history of %ld entries, but the wanted one is %ld entries. Restart netdata here to apply the new settings.", rrdhost_hostname(host), host->rrd_history_entries, history);

646 + error("Host '%s' has history of %d entries, but the wanted one is %ld entries. "

647 + "Restart netdata here to apply the new settings.",

648 + rrdhost_hostname(host), host->rrd_history_entries, history);

649

650 // update host tags

651 rrdhost_init_tags(host, tags);

@@ -682,7 +688,7 @@ static void rrdhost_update(RRDHOST *host

688 info("Host %s is not in archived mode anymore", rrdhost_hostname(host));

689 }

690

685 - netdata_spinlock_unlock(&host->rrdhost_update_lock);

691 + spinlock_unlock(&host->rrdhost_update_lock);

692 }

693

694 RRDHOST *rrdhost_find_or_create(

@@ -912,9 +918,12 @@ void dbengine_init(char *hostname) {

918 strncpyz(tiers_init[tier].path, dbenginepath, FILENAME_MAX);

919 tiers_init[tier].ret = 0;

920

915 - if(parallel_initialization)

916 - netdata_thread_create(&tiers_init[tier].thread, "DBENGINE_INIT", NETDATA_THREAD_OPTION_JOINABLE,

921 + if(parallel_initialization) {

922 + char tag[NETDATA_THREAD_TAG_MAX + 1];

923 + snprintfz(tag, NETDATA_THREAD_TAG_MAX, "DBENGINIT[%zu]", tier);

924 + netdata_thread_create(&tiers_init[tier].thread, tag, NETDATA_THREAD_OPTION_JOINABLE,

925 dbengine_tier_init, &tiers_init[tier]);

926 + }

927 else

928 dbengine_tier_init(&tiers_init[tier]);

929 }

database/rrdset.c

+53 -50

@@ -128,14 +128,14 @@ static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v

128 st->module_name = rrd_string_strdupz(ctr->module);

129 st->priority = ctr->priority;

130

131 - st->entries = (ctr->memory_mode != RRD_MEMORY_MODE_DBENGINE) ? align_entries_to_pagesize(ctr->memory_mode, ctr->history_entries) : 5;

131 + st->db.entries = (ctr->memory_mode != RRD_MEMORY_MODE_DBENGINE) ? align_entries_to_pagesize(ctr->memory_mode, ctr->history_entries) : 5;

132 st->update_every = ctr->update_every;

133 st->rrd_memory_mode = ctr->memory_mode;

134

135 st->chart_type = ctr->chart_type;

136 st->rrdhost = host;

137

138 - netdata_spinlock_init(&st->data_collection_lock);

138 + spinlock_init(&st->data_collection_lock);

139

140 st->flags = RRDSET_FLAG_SYNC_CLOCK

141 | RRDSET_FLAG_INDEXED_ID

@@ -278,7 +278,7 @@ static void rrdset_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v

278 string_freez(st->module_name);

279

280 freez(st->exporting_flags);

281 - freez(st->cache_dir);

281 + freez(st->db.cache_dir);

282 }

283

284 // the item to be inserted, is already in the dictionary

@@ -702,8 +702,8 @@ inline void rrdset_update_heterogeneous_flag(RRDSET *st) {

702

703 bool init = false, is_heterogeneous = false;

704 RRD_ALGORITHM algorithm;

705 - collected_number multiplier;

706 - collected_number divisor;

705 + int32_t multiplier;

706 + int32_t divisor;

707

708 rrddim_foreach_read(rd, st) {

709 if(!init) {

@@ -717,7 +717,9 @@ inline void rrdset_update_heterogeneous_flag(RRDSET *st) {

717 if(algorithm != rd->algorithm || multiplier != ABS(rd->multiplier) || divisor != ABS(rd->divisor)) {

718 if(!rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS)) {

719 #ifdef NETDATA_INTERNAL_CHECKS

720 - info("Dimension '%s' added on chart '%s' of host '%s' is not homogeneous to other dimensions already present (algorithm is '%s' vs '%s', multiplier is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ", divisor is " COLLECTED_NUMBER_FORMAT " vs " COLLECTED_NUMBER_FORMAT ").",

720 + info("Dimension '%s' added on chart '%s' of host '%s' is not homogeneous to other dimensions already present "

721 + "(algorithm is '%s' vs '%s', multiplier is %d vs %d, "

722 + "divisor is %d vs %d).",

723 rrddim_name(rd),

724 rrdset_name(st),

725 rrdhost_hostname(host),

@@ -751,7 +753,7 @@ void rrdset_reset(RRDSET *st) {

753 st->last_collected_time.tv_usec = 0;

754 st->last_updated.tv_sec = 0;

755 st->last_updated.tv_usec = 0;

754 - st->current_entry = 0;

756 + st->db.current_entry = 0;

757 st->counter = 0;

758 st->counter_done = 0;

759

@@ -856,8 +858,8 @@ void rrdset_delete_files(RRDSET *st) {

858 }

859 rrddim_foreach_done(rd);

860

859 - if(st->cache_dir)

860 - recursively_delete_dir(st->cache_dir, "left-over chart");

861 + if(st->db.cache_dir)

862 + recursively_delete_dir(st->db.cache_dir, "left-over chart");

863 }

864

865 void rrdset_delete_obsolete_dimensions(RRDSET *st) {

@@ -1000,7 +1002,7 @@ void rrdset_timed_next(RRDSET *st, struct timeval now, usec_t duration_since_las

1002 // oops! the database is in the future

1003 #ifdef NETDATA_INTERNAL_CHECKS

1004 info("RRD database for chart '%s' on host '%s' is %0.5" NETDATA_DOUBLE_MODIFIER

1003 - " secs in the future (counter #%zu, update #%zu). Adjusting it to current time."

1005 + " secs in the future (counter #%u, update #%u). Adjusting it to current time."

1006 , rrdset_id(st)

1007 , rrdhost_hostname(st->rrdhost)

1008 , (NETDATA_DOUBLE)-since_last_usec / USEC_PER_SEC

@@ -1019,7 +1021,9 @@ void rrdset_timed_next(RRDSET *st, struct timeval now, usec_t duration_since_las

1021 // oops! the database is too far behind

1022 #ifdef NETDATA_INTERNAL_CHECKS

1023 info("RRD database for chart '%s' on host '%s' is %0.5" NETDATA_DOUBLE_MODIFIER

1022 - " secs in the past (counter #%zu, update #%zu). Adjusting it to current time.", rrdset_id(st), rrdhost_hostname(st->rrdhost), (NETDATA_DOUBLE)since_last_usec / USEC_PER_SEC, st->counter, st->counter_done);

1024 + " secs in the past (counter #%u, update #%u). Adjusting it to current time.",

1025 + rrdset_id(st), rrdhost_hostname(st->rrdhost), (NETDATA_DOUBLE)since_last_usec / USEC_PER_SEC,

1026 + st->counter, st->counter_done);

1027 #endif

1028

1029 duration_since_last_update = (usec_t)since_last_usec;

@@ -1126,7 +1130,7 @@ static inline void rrdset_init_last_updated_time(RRDSET *st) {

1130 static __thread size_t rrdset_done_statistics_points_stored_per_tier[RRD_STORAGE_TIERS];

1131

1132 static inline time_t tier_next_point_time_s(RRDDIM *rd, struct rrddim_tier *t, time_t now_s) {

1129 - time_t loop = (time_t)rd->update_every * (time_t)t->tier_grouping;

1133 + time_t loop = (time_t)rd->rrdset->update_every * (time_t)t->tier_grouping;

1134 return now_s + loop - ((now_s + loop) % loop);

1135 }

1136

@@ -1231,7 +1235,7 @@ void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n,

1235 time_t now_s = (time_t)(point_end_time_ut / USEC_PER_SEC);

1236

1237 STORAGE_POINT sp = {

1234 - .start_time_s = now_s - rd->update_every,

1238 + .start_time_s = now_s - rd->rrdset->update_every,

1239 .end_time_s = now_s,

1240 .min = n,

1241 .max = n,

@@ -1320,7 +1324,7 @@ static inline size_t rrdset_done_interpolate(

1324 if((now_collect_ut % (update_every_ut)) == 0) iterations++;

1325

1326 size_t counter = st->counter;

1323 - long current_entry = st->current_entry;

1327 + long current_entry = st->db.current_entry;

1328

1329 SN_FLAGS storage_flags = SN_DEFAULT_FLAGS;

1330

@@ -1437,7 +1441,7 @@ static inline size_t rrdset_done_interpolate(

1441 continue;

1442 }

1443

1440 - if(likely(rd->updated && rd->collections_counter > 1 && iterations < gap_when_lost_iterations_above)) {

1444 + if(likely(rrddim_check_updated(rd) && rd->collections_counter > 1 && iterations < gap_when_lost_iterations_above)) {

1445 uint32_t dim_storage_flags = storage_flags;

1446

1447 if (ml_dimension_is_anomalous(rd, current_time_s, new_value, true)) {

@@ -1472,7 +1476,7 @@ static inline size_t rrdset_done_interpolate(

1476 storage_flags = SN_DEFAULT_FLAGS;

1477

1478 st->counter = ++counter;

1475 - st->current_entry = current_entry = ((current_entry + 1) >= st->entries) ? 0 : current_entry + 1;

1479 + st->db.current_entry = current_entry = ((current_entry + 1) >= st->db.entries) ? 0 : current_entry + 1;

1480

1481 st->last_updated.tv_sec = (time_t) (last_ut / USEC_PER_SEC);

1482 st->last_updated.tv_usec = 0;

@@ -1507,7 +1511,7 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next)

1511 if(unlikely(rrdhost_has_rrdpush_sender_enabled(st->rrdhost)))

1512 stream_buffer = rrdset_push_metric_initialize(st, now.tv_sec);

1513

1510 - netdata_spinlock_lock(&st->data_collection_lock);

1514 + spinlock_lock(&st->data_collection_lock);

1515

1516 if (pending_rrdset_next)

1517 rrdset_timed_next(st, now, 0ULL);

@@ -1529,7 +1533,7 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next)

1533

1534 RRDSET_FLAGS rrdset_flags = rrdset_flag_check(st, ~0);

1535 if(unlikely(rrdset_flags & RRDSET_FLAG_COLLECTION_FINISHED)) {

1532 - netdata_spinlock_unlock(&st->data_collection_lock);

1536 + spinlock_unlock(&st->data_collection_lock);

1537 return;

1538 }

1539

@@ -1539,9 +1543,10 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next)

1543 }

1544

1545 // check if the chart has a long time to be updated

1542 - if(unlikely(st->usec_since_last_update > MAX(st->entries, 60) * update_every_ut)) {

1543 - info("host '%s', chart '%s': took too long to be updated (counter #%zu, update #%zu, %0.3" NETDATA_DOUBLE_MODIFIER

1544 - " secs). Resetting it.", rrdhost_hostname(st->rrdhost), rrdset_id(st), st->counter, st->counter_done, (NETDATA_DOUBLE)st->usec_since_last_update / USEC_PER_SEC);

1546 + if(unlikely(st->usec_since_last_update > MAX(st->db.entries, 60) * update_every_ut)) {

1547 + info("host '%s', chart '%s': took too long to be updated (counter #%u, update #%u, %0.3" NETDATA_DOUBLE_MODIFIER

1548 + " secs). Resetting it.", rrdhost_hostname(st->rrdhost), rrdset_id(st), st->counter, st->counter_done,

1549 + (NETDATA_DOUBLE)st->usec_since_last_update / USEC_PER_SEC);

1550 rrdset_reset(st);

1551 st->usec_since_last_update = update_every_ut;

1552 store_this_entry = 0;

@@ -1579,7 +1584,7 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next)

1584 }

1585

1586 // check if we will re-write the entire data set

1582 - if(unlikely(dt_usec(&st->last_collected_time, &st->last_updated) > st->entries * update_every_ut &&

1587 + if(unlikely(dt_usec(&st->last_collected_time, &st->last_updated) > st->db.entries * update_every_ut &&

1588 st->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)) {

1589 info(

1590 "'%s': too old data (last updated at %"PRId64".%"PRId64", last collected at %"PRId64".%"PRId64"). "

@@ -1659,7 +1664,7 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next)

1664 rda->rd = dictionary_acquired_item_value(rda->item);

1665

1666 // calculate totals

1662 - if(likely(rd->updated)) {

1667 + if(likely(rrddim_check_updated(rd))) {

1668 // if the new is smaller than the old (an overflow, or reset), set the old equal to the new

1669 // to reset the calculation (it will give zero as the calculation for this second)

1670 if(unlikely(rd->algorithm == RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL && rd->last_collected_value > rd->collected_value)) {

@@ -1700,7 +1705,7 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next)

1705 rd = rda->rd;

1706 if(unlikely(!rd)) continue;

1707

1703 - if(unlikely(!rd->updated)) {

1708 + if(unlikely(!rrddim_check_updated(rd))) {

1709 rd->calculated_value = 0;

1710 continue;

1711 }

@@ -1905,7 +1910,7 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next)

1910 rd = rda->rd;

1911 if(unlikely(!rd)) continue;

1912

1908 - if(unlikely(!rd->updated))

1913 + if(unlikely(!rrddim_check_updated(rd)))

1914 continue;

1915

1916 rrdset_debug(st, "%s: setting last_collected_value (old: " COLLECTED_NUMBER_FORMAT ") to last_collected_value (new: " COLLECTED_NUMBER_FORMAT ")", rrddim_name(rd), rd->last_collected_value, rd->collected_value);

@@ -1943,7 +1948,7 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next)

1948

1949 rd->calculated_value = 0;

1950 rd->collected_value = 0;

1946 - rd->updated = 0;

1951 + rrddim_clear_updated(rd);

1952

1953 rrdset_debug(st, "%s: END "

1954 " last_collected_value = " COLLECTED_NUMBER_FORMAT

@@ -1958,7 +1963,7 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next)

1963 );

1964 }

1965

1961 - netdata_spinlock_unlock(&st->data_collection_lock);

1966 + spinlock_unlock(&st->data_collection_lock);

1967 rrdset_push_metrics_finished(&stream_buffer, st);

1968

1969 // ALL DONE ABOUT THE DATA UPDATE

@@ -1991,6 +1996,8 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next)

1996 }

1997

1998 time_t rrdset_set_update_every_s(RRDSET *st, time_t update_every_s) {

1999 + if(unlikely(update_every_s == st->update_every))

2000 + return st->update_every;

2001

2002 internal_error(true, "RRDSET '%s' switching update every from %d to %d",

2003 rrdset_id(st), (int)st->update_every, (int)update_every_s);

@@ -2007,10 +2014,6 @@ time_t rrdset_set_update_every_s(RRDSET *st, time_t update_every_s) {

2014 rd->tiers[tier].db_collection_handle,

2015 (int)(st->rrdhost->db[tier].tier_grouping * st->update_every));

2016 }

2010 -

2011 - assert(rd->update_every == (int) prev_update_every_s &&

2012 - "chart's update every differs from the update every of its dimensions");

2013 - rd->update_every = st->update_every;

2017 }

2018 rrddim_foreach_done(rd);

2019

@@ -2094,10 +2097,10 @@ struct rrdset_map_save_v019 {

2097 };

2098

2099 void rrdset_memory_file_update(RRDSET *st) {

2097 - if(!st->st_on_file) return;

2098 - struct rrdset_map_save_v019 *st_on_file = st->st_on_file;

2100 + if(!st->db.st_on_file) return;

2101 + struct rrdset_map_save_v019 *st_on_file = st->db.st_on_file;

2102

2100 - st_on_file->current_entry = st->current_entry;

2103 + st_on_file->current_entry = st->db.current_entry;

2104 st_on_file->counter = st->counter;

2105 st_on_file->usec_since_last_update = st->usec_since_last_update;

2106 st_on_file->last_updated.tv_sec = st->last_updated.tv_sec;

@@ -2105,41 +2108,41 @@ void rrdset_memory_file_update(RRDSET *st) {

2108 }

2109

2110 const char *rrdset_cache_filename(RRDSET *st) {

2108 - if(!st->st_on_file) return NULL;

2109 - struct rrdset_map_save_v019 *st_on_file = st->st_on_file;

2111 + if(!st->db.st_on_file) return NULL;

2112 + struct rrdset_map_save_v019 *st_on_file = st->db.st_on_file;

2113 return st_on_file->cache_filename;

2114 }

2115

2116 const char *rrdset_cache_dir(RRDSET *st) {

2114 - if(!st->cache_dir)

2115 - st->cache_dir = rrdhost_cache_dir_for_rrdset_alloc(st->rrdhost, rrdset_id(st));

2117 + if(!st->db.cache_dir)

2118 + st->db.cache_dir = rrdhost_cache_dir_for_rrdset_alloc(st->rrdhost, rrdset_id(st));

2119

2117 - return st->cache_dir;

2120 + return st->db.cache_dir;

2121 }

2122

2123 void rrdset_memory_file_free(RRDSET *st) {

2121 - if(!st->st_on_file) return;

2124 + if(!st->db.st_on_file) return;

2125

2126 // needed for memory mode map, to save the latest state

2127 rrdset_memory_file_update(st);

2128

2126 - struct rrdset_map_save_v019 *st_on_file = st->st_on_file;

2129 + struct rrdset_map_save_v019 *st_on_file = st->db.st_on_file;

2130 __atomic_sub_fetch(&rrddim_db_memory_size, st_on_file->memsize, __ATOMIC_RELAXED);

2131 netdata_munmap(st_on_file, st_on_file->memsize);

2132

2133 // remove the pointers from the RRDDIM

2131 - st->st_on_file = NULL;

2134 + st->db.st_on_file = NULL;

2135 }

2136

2137 void rrdset_memory_file_save(RRDSET *st) {

2135 - if(!st->st_on_file) return;

2138 + if(!st->db.st_on_file) return;

2139

2140 rrdset_memory_file_update(st);

2141

2139 - struct rrdset_map_save_v019 *st_on_file = st->st_on_file;

2142 + struct rrdset_map_save_v019 *st_on_file = st->db.st_on_file;

2143 if(st_on_file->rrd_memory_mode != RRD_MEMORY_MODE_SAVE) return;

2144

2142 - memory_file_save(st_on_file->cache_filename, st->st_on_file, st_on_file->memsize);

2145 + memory_file_save(st_on_file->cache_filename, st->db.st_on_file, st_on_file->memsize);

2146 }

2147

2148 bool rrdset_memory_load_or_create_map_save(RRDSET *st, RRD_MEMORY_MODE memory_mode) {

@@ -2166,7 +2169,7 @@ bool rrdset_memory_load_or_create_map_save(RRDSET *st, RRD_MEMORY_MODE memory_mo

2169 error("File '%s' contents are not for chart '%s'. Clearing it.", fullfilename, rrdset_id(st));

2170 memset(st_on_file, 0, size);

2171 }

2169 - else if(st_on_file->memsize != size || st_on_file->entries != st->entries) {

2172 + else if(st_on_file->memsize != size || st_on_file->entries != st->db.entries) {

2173 error("File '%s' does not have the desired size. Clearing it.", fullfilename);

2174 memset(st_on_file, 0, size);

2175 }

@@ -2174,7 +2177,7 @@ bool rrdset_memory_load_or_create_map_save(RRDSET *st, RRD_MEMORY_MODE memory_mo

2177 error("File '%s' does not have the desired granularity. Clearing it.", fullfilename);

2178 memset(st_on_file, 0, size);

2179 }

2177 - else if((now_s - st_on_file->last_updated.tv_sec) > st->update_every * st->entries) {

2180 + else if((now_s - st_on_file->last_updated.tv_sec) > (long)st->update_every * (long)st->db.entries) {

2181 info("File '%s' is too old. Clearing it.", fullfilename);

2182 memset(st_on_file, 0, size);

2183 }

@@ -2194,14 +2197,14 @@ bool rrdset_memory_load_or_create_map_save(RRDSET *st, RRD_MEMORY_MODE memory_mo

2197 }

2198

2199 // copy the useful values to st

2197 - st->current_entry = st_on_file->current_entry;

2200 + st->db.current_entry = st_on_file->current_entry;

2201 st->counter = st_on_file->counter;

2202 st->usec_since_last_update = st_on_file->usec_since_last_update;

2203 st->last_updated.tv_sec = st_on_file->last_updated.tv_sec;

2204 st->last_updated.tv_usec = st_on_file->last_updated.tv_usec;

2205

2206 // link it to st

2204 - st->st_on_file = st_on_file;

2207 + st->db.st_on_file = st_on_file;

2208

2209 // clear everything

2210 memset(st_on_file, 0, size);

@@ -2211,7 +2214,7 @@ bool rrdset_memory_load_or_create_map_save(RRDSET *st, RRD_MEMORY_MODE memory_mo

2214 strcpy(st_on_file->cache_filename, fullfilename);

2215 strcpy(st_on_file->magic, RRDSET_MAGIC_V019);

2216 st_on_file->memsize = size;

2214 - st_on_file->entries = st->entries;

2217 + st_on_file->entries = st->db.entries;

2218 st_on_file->update_every = st->update_every;

2219 st_on_file->rrd_memory_mode = memory_mode;

2220

database/sqlite/sqlite_metadata.c

+1 -1

@@ -522,7 +522,7 @@ static int store_chart_metadata(RRDSET *st)

522 if (unlikely(rc != SQLITE_OK))

523 goto bind_fail;

524

525 - rc = sqlite3_bind_int(res, ++param, (int) st->entries);

525 + rc = sqlite3_bind_int(res, ++param, (int) st->db.entries);

526 if (unlikely(rc != SQLITE_OK))

527 goto bind_fail;

528

exporting/prometheus/prometheus.c

+1 -1

@@ -497,7 +497,7 @@ static void generate_as_collected_prom_help(BUFFER *wb, struct gen_parameters *p

497 if (prometheus_collector)

498 buffer_sprintf(wb, "1 / 1");

499 else

500 - buffer_sprintf(wb, COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT, p->rd->multiplier, p->rd->divisor);

500 + buffer_sprintf(wb, "%d / %d", p->rd->multiplier, p->rd->divisor);

501

502 buffer_sprintf(wb, " %s %s (%s)\n", p->relation, rrdset_units(p->st), p->type);

503 }

exporting/tests/exporting_fixtures.c

+1 -1

@@ -91,7 +91,7 @@ int setup_rrdhost()

92 st->rrddim_root_index = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE);

93 dictionary_register_insert_callback(st->rrddim_root_index, rrddim_insert_callback, NULL);

94 - st->dimensions = dictionary_set_advanced(st->rrddim_root_index, "dimension_id", -1, NULL, sizeof(RRDDIM), st);

94 + st->dimensions = dictionary_set_advanced(st->rrddim_root_index, "dimension_id", -1, NULL, rrddim_size(), st);

96 return 0;

97 }

libnetdata/aral/aral.c

+13 -13

@@ -141,39 +141,39 @@ static size_t aral_align_alloc_size(ARAL *ar, uint64_t size) {

141

142 static inline void aral_lock(ARAL *ar) {

143 if(likely(!(ar->config.options & ARAL_LOCKLESS)))

144 - netdata_spinlock_lock(&ar->aral_lock.spinlock);

144 + spinlock_lock(&ar->aral_lock.spinlock);

145 }

146

147 static inline void aral_unlock(ARAL *ar) {

148 if(likely(!(ar->config.options & ARAL_LOCKLESS)))

149 - netdata_spinlock_unlock(&ar->aral_lock.spinlock);

149 + spinlock_unlock(&ar->aral_lock.spinlock);

150 }

151

152 static inline void aral_page_free_lock(ARAL *ar, ARAL_PAGE *page) {

153 if(likely(!(ar->config.options & ARAL_LOCKLESS)))

154 - netdata_spinlock_lock(&page->free.spinlock);

154 + spinlock_lock(&page->free.spinlock);

155 }

156

157 static inline void aral_page_free_unlock(ARAL *ar, ARAL_PAGE *page) {

158 if(likely(!(ar->config.options & ARAL_LOCKLESS)))

159 - netdata_spinlock_unlock(&page->free.spinlock);

159 + spinlock_unlock(&page->free.spinlock);

160 }

161

162 static inline bool aral_adders_trylock(ARAL *ar) {

163 if(likely(!(ar->config.options & ARAL_LOCKLESS)))

164 - return netdata_spinlock_trylock(&ar->adders.spinlock);

164 + return spinlock_trylock(&ar->adders.spinlock);

165

166 return true;

167 }

168

169 static inline void aral_adders_lock(ARAL *ar) {

170 if(likely(!(ar->config.options & ARAL_LOCKLESS)))

171 - netdata_spinlock_lock(&ar->adders.spinlock);

171 + spinlock_lock(&ar->adders.spinlock);

172 }

173

174 static inline void aral_adders_unlock(ARAL *ar) {

175 if(likely(!(ar->config.options & ARAL_LOCKLESS)))

176 - netdata_spinlock_unlock(&ar->adders.spinlock);

176 + spinlock_unlock(&ar->adders.spinlock);

177 }

178

179 static void aral_delete_leftover_files(const char *name, const char *path, const char *required_prefix) {

@@ -273,7 +273,7 @@ size_t aral_next_allocation_size___adders_lock_needed(ARAL *ar) {

273

274 static ARAL_PAGE *aral_create_page___no_lock_needed(ARAL *ar, size_t size TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {

275 ARAL_PAGE *page = callocz(1, sizeof(ARAL_PAGE));

276 - netdata_spinlock_init(&page->free.spinlock);

276 + spinlock_init(&page->free.spinlock);

277 page->size = size;

278 page->max_elements = page->size / ar->config.element_size;

279 page->aral_lock.free_elements = page->max_elements;

@@ -713,7 +713,7 @@ ARAL *aral_create(const char *name, size_t element_size, size_t initial_page_ele

713 ar->config.mmap.cache_dir = cache_dir;

714 ar->config.mmap.enabled = mmap;

715 strncpyz(ar->config.name, name, ARAL_MAX_NAME);

716 - netdata_spinlock_init(&ar->aral_lock.spinlock);

716 + spinlock_init(&ar->aral_lock.spinlock);

717

718 if(stats) {

719 ar->stats = stats;

@@ -839,7 +839,7 @@ size_t aral_by_size_overhead(void) {

839 }

840

841 ARAL *aral_by_size_acquire(size_t size) {

842 - netdata_spinlock_lock(&aral_by_size_globals.spinlock);

842 + spinlock_lock(&aral_by_size_globals.spinlock);

843

844 ARAL *ar = NULL;

845

@@ -867,7 +867,7 @@ ARAL *aral_by_size_acquire(size_t size) {

867 }

868 }

869

870 - netdata_spinlock_unlock(&aral_by_size_globals.spinlock);

870 + spinlock_unlock(&aral_by_size_globals.spinlock);

871

872 return ar;

873 }

@@ -876,7 +876,7 @@ void aral_by_size_release(ARAL *ar) {

876 size_t size = aral_element_size(ar);

877

878 if(size <= ARAL_BY_SIZE_MAX_SIZE) {

879 - netdata_spinlock_lock(&aral_by_size_globals.spinlock);

879 + spinlock_lock(&aral_by_size_globals.spinlock);

880

881 internal_fatal(aral_by_size_globals.array[size].ar != ar,

882 "ARAL BY SIZE: aral pointers do not match");

@@ -890,7 +890,7 @@ void aral_by_size_release(ARAL *ar) {

890 // aral_by_size_globals.array[size].ar = NULL;

891 // }

892

893 - netdata_spinlock_unlock(&aral_by_size_globals.spinlock);

893 + spinlock_unlock(&aral_by_size_globals.spinlock);

894 }

895 else

896 aral_destroy(ar);

libnetdata/avl/avl.c

+32 -47

@@ -316,34 +316,36 @@ int avl_traverse(avl_tree_type *tree, int (*callback)(void * /*entry*/, void * /

316 // ---------------------------

317 // locks

318

319 -void avl_read_lock(avl_tree_lock *t) {

320 -#ifndef AVL_WITHOUT_PTHREADS

321 -#ifdef AVL_LOCK_WITH_MUTEX

322 - netdata_mutex_lock(&t->mutex);

323 -#else

319 +static inline void avl_read_lock(avl_tree_lock *t) {

320 +#if defined(AVL_LOCK_WITH_RWLOCK)

321 netdata_rwlock_rdlock(&t->rwlock);

322 +#else

323 + rw_spinlock_read_lock(&t->rwlock);

324 #endif

326 -#endif /* AVL_WITHOUT_PTHREADS */

325 }

326

329 -void avl_write_lock(avl_tree_lock *t) {

330 -#ifndef AVL_WITHOUT_PTHREADS

331 -#ifdef AVL_LOCK_WITH_MUTEX

332 - netdata_mutex_lock(&t->mutex);

333 -#else

327 +static inline void avl_write_lock(avl_tree_lock *t) {

328 +#if defined(AVL_LOCK_WITH_RWLOCK)

329 netdata_rwlock_wrlock(&t->rwlock);

330 +#else

331 + rw_spinlock_write_lock(&t->rwlock);

332 #endif

336 -#endif /* AVL_WITHOUT_PTHREADS */

333 }

334

339 -void avl_unlock(avl_tree_lock *t) {

340 -#ifndef AVL_WITHOUT_PTHREADS

341 -#ifdef AVL_LOCK_WITH_MUTEX

342 - netdata_mutex_unlock(&t->mutex);

335 +static inline void avl_read_unlock(avl_tree_lock *t) {

336 +#if defined(AVL_LOCK_WITH_RWLOCK)

337 + netdata_rwlock_unlock(&t->rwlock);

338 #else

339 + rw_spinlock_read_unlock(&t->rwlock);

340 +#endif

341 +}

342 +

343 +static inline void avl_write_unlock(avl_tree_lock *t) {

344 +#if defined(AVL_LOCK_WITH_RWLOCK)

345 netdata_rwlock_unlock(&t->rwlock);

346 +#else

347 + rw_spinlock_write_unlock(&t->rwlock);

348 #endif

346 -#endif /* AVL_WITHOUT_PTHREADS */

349 }

350

351 // ---------------------------

@@ -352,63 +354,46 @@ void avl_unlock(avl_tree_lock *t) {

354 void avl_init_lock(avl_tree_lock *tree, int (*compar)(void * /*a*/, void * /*b*/)) {

355 avl_init(&tree->avl_tree, compar);

356

355 -#ifndef AVL_WITHOUT_PTHREADS

356 - int lock;

357 -

358 -#ifdef AVL_LOCK_WITH_MUTEX

359 - lock = netdata_mutex_init(&tree->mutex, NULL);

357 +#if defined(AVL_LOCK_WITH_RWLOCK)

358 + if(netdata_rwlock_init(&tree->rwlock) != 0)

359 + fatal("Failed to initialize AVL rwlock");

360 #else

361 - lock = netdata_rwlock_init(&tree->rwlock);

361 + rw_spinlock_init(&tree->rwlock);

362 #endif

363 -

364 - if(lock != 0)

365 - fatal("Failed to initialize AVL mutex/rwlock, error: %d", lock);

366 -

367 -#endif /* AVL_WITHOUT_PTHREADS */

363 }

364

370 -void avl_destroy_lock(avl_tree_lock *tree) {

371 -#ifndef AVL_WITHOUT_PTHREADS

372 - int lock;

373 -

374 -#ifdef AVL_LOCK_WITH_MUTEX

375 - lock = netdata_mutex_destroy(&tree->mutex);

376 -#else

377 - lock = netdata_rwlock_destroy(&tree->rwlock);

365 +void avl_destroy_lock(avl_tree_lock *tree __maybe_unused) {

366 +#if defined(AVL_LOCK_WITH_RWLOCK)

367 + if(netdata_rwlock_destroy(&tree->rwlock) != 0)

368 + fatal("Failed to destroy AVL rwlock");

369 #endif

379 -

380 - if(lock != 0)

381 - fatal("Failed to destroy AVL mutex/rwlock, error: %d", lock);

382 -

383 -#endif /* AVL_WITHOUT_PTHREADS */

370 }

371

372 avl_t *avl_search_lock(avl_tree_lock *tree, avl_t *item) {

373 avl_read_lock(tree);

374 avl_t *ret = avl_search(&tree->avl_tree, item);

389 - avl_unlock(tree);

375 + avl_read_unlock(tree);

376 return ret;

377 }

378

379 avl_t * avl_remove_lock(avl_tree_lock *tree, avl_t *item) {

380 avl_write_lock(tree);

381 avl_t *ret = avl_remove(&tree->avl_tree, item);

396 - avl_unlock(tree);

382 + avl_write_unlock(tree);

383 return ret;

384 }

385

386 avl_t *avl_insert_lock(avl_tree_lock *tree, avl_t *item) {

387 avl_write_lock(tree);

388 avl_t * ret = avl_insert(&tree->avl_tree, item);

403 - avl_unlock(tree);

389 + avl_write_unlock(tree);

390 return ret;

391 }

392

393 int avl_traverse_lock(avl_tree_lock *tree, int (*callback)(void * /*entry*/, void * /*data*/), void *data) {

408 - int ret;

394 avl_read_lock(tree);

410 - ret = avl_traverse(&tree->avl_tree, callback, data);

411 - avl_unlock(tree);

395 + int ret = avl_traverse(&tree->avl_tree, callback, data);

396 + avl_read_unlock(tree);

397 return ret;

398 }

399

libnetdata/avl/avl.h

+8 -19

@@ -10,20 +10,11 @@

10 #define AVL_MAX_HEIGHT 92

11 #endif

13 -#ifndef AVL_WITHOUT_PTHREADS

14 -#include <pthread.h>

15 -

16 -// #define AVL_LOCK_WITH_MUTEX 1

17 -

18 -#ifdef AVL_LOCK_WITH_MUTEX

19 -#define AVL_LOCK_INITIALIZER NETDATA_MUTEX_INITIALIZER

20 -#else /* AVL_LOCK_WITH_MUTEX */

13 +#if defined(AVL_LOCK_WITH_RWLOCK)

14 #define AVL_LOCK_INITIALIZER NETDATA_RWLOCK_INITIALIZER

22 -#endif /* AVL_LOCK_WITH_MUTEX */

23 -

24 -#else /* AVL_WITHOUT_PTHREADS */

25 -#define AVL_LOCK_INITIALIZER

26 -#endif /* AVL_WITHOUT_PTHREADS */

15 +#else

16 +#define AVL_LOCK_INITIALIZER NETDATA_RW_SPINLOCK_INITIALIZER

17 +#endif

19 /* Data structures */

@@ -42,13 +33,11 @@ typedef struct avl_tree_type {

33 typedef struct avl_tree_lock {

34 avl_tree_type avl_tree;

45 -#ifndef AVL_WITHOUT_PTHREADS

46 -#ifdef AVL_LOCK_WITH_MUTEX

47 - netdata_mutex_t mutex;

48 -#else /* AVL_LOCK_WITH_MUTEX */

36 +#if defined(AVL_LOCK_WITH_RWLOCK)

37 netdata_rwlock_t rwlock;

50 -#endif /* AVL_LOCK_WITH_MUTEX */

51 -#endif /* AVL_WITHOUT_PTHREADS */

38 +#else

39 + RW_SPINLOCK rwlock;

40 +#endif

41 } avl_tree_lock;

43 /* Public methods */

libnetdata/dictionary/dictionary.c

+34 -39

@@ -147,14 +147,14 @@ struct dictionary {

147

148 struct { // support for multiple indexing engines

149 Pvoid_t JudyHSArray; // the hash table

150 - netdata_rwlock_t rwlock; // protect the index

150 + RW_SPINLOCK rw_spinlock; // protect the index

151 } index;

152

153 struct {

154 DICTIONARY_ITEM *list; // the double linked list of all items in the dictionary

155 - netdata_rwlock_t rwlock; // protect the linked-list

155 + RW_SPINLOCK rw_spinlock; // protect the linked-list

156 pid_t writer_pid; // the gettid() of the writer

157 - size_t writer_depth; // nesting of write locks

157 + uint32_t writer_depth; // nesting of write locks

158 } items;

159

160 struct dictionary_hooks *hooks; // pointer to external function callbacks to be called at certain points

@@ -163,7 +163,7 @@ struct dictionary {

163 DICTIONARY *master; // the master dictionary

164 DICTIONARY *next; // linked list for delayed destruction (garbage collection of whole dictionaries)

165

166 - size_t version; // the current version of the dictionary

166 + uint32_t version; // the current version of the dictionary

167 // it is incremented when:

168 // - item added

169 // - item removed

@@ -171,9 +171,9 @@ struct dictionary {

171 // - conflict callback returns true

172 // - function dictionary_version_increment() is called

173

174 - long int entries; // how many items are currently in the index (the linked list may have more)

175 - long int referenced_items; // how many items of the dictionary are currently being used by 3rd parties

176 - long int pending_deletion_items; // how many items of the dictionary have been deleted, but have not been removed yet

174 + int32_t entries; // how many items are currently in the index (the linked list may have more)

175 + int32_t referenced_items; // how many items of the dictionary are currently being used by 3rd parties

176 + int32_t pending_deletion_items; // how many items of the dictionary have been deleted, but have not been removed yet

177

178 #ifdef NETDATA_DICTIONARY_VALIDATE_POINTERS

179 netdata_mutex_t global_pointer_registry_mutex;

@@ -632,19 +632,14 @@ static void dictionary_execute_delete_callback(DICTIONARY *dict, DICTIONARY_ITEM

632

633 static inline size_t dictionary_locks_init(DICTIONARY *dict) {

634 if(likely(!is_dictionary_single_threaded(dict))) {

635 - netdata_rwlock_init(&dict->index.rwlock);

636 - netdata_rwlock_init(&dict->items.rwlock);

635 + rw_spinlock_init(&dict->index.rw_spinlock);

636 + rw_spinlock_init(&dict->items.rw_spinlock);

637 }

638

639 return 0;

640 }

641

642 -static inline size_t dictionary_locks_destroy(DICTIONARY *dict) {

643 - if(likely(!is_dictionary_single_threaded(dict))) {

644 - netdata_rwlock_destroy(&dict->index.rwlock);

645 - netdata_rwlock_destroy(&dict->items.rwlock);

646 - }

647 -

642 +static inline size_t dictionary_locks_destroy(DICTIONARY *dict __maybe_unused) {

643 return 0;

644 }

645

@@ -676,11 +671,11 @@ static inline void ll_recursive_lock(DICTIONARY *dict, char rw) {

671

672 if(rw == DICTIONARY_LOCK_READ || rw == DICTIONARY_LOCK_REENTRANT || rw == 'R') {

673 // read lock

679 - netdata_rwlock_rdlock(&dict->items.rwlock);

674 + rw_spinlock_read_lock(&dict->items.rw_spinlock);

675 }

676 else {

677 // write lock

683 - netdata_rwlock_wrlock(&dict->items.rwlock);

678 + rw_spinlock_write_lock(&dict->items.rw_spinlock);

679 ll_recursive_lock_set_thread_as_writer(dict);

680 }

681 }

@@ -697,14 +692,14 @@ static inline void ll_recursive_unlock(DICTIONARY *dict, char rw) {

692 if(rw == DICTIONARY_LOCK_READ || rw == DICTIONARY_LOCK_REENTRANT || rw == 'R') {

693 // read unlock

694

700 - netdata_rwlock_unlock(&dict->items.rwlock);

695 + rw_spinlock_read_unlock(&dict->items.rw_spinlock);

696 }

697 else {

698 // write unlock

699

700 ll_recursive_unlock_unset_thread_writer(dict);

701

707 - netdata_rwlock_unlock(&dict->items.rwlock);

702 + rw_spinlock_write_unlock(&dict->items.rw_spinlock);

703 }

704 }

705

@@ -719,27 +714,27 @@ static inline void dictionary_index_lock_rdlock(DICTIONARY *dict) {

714 if(unlikely(is_dictionary_single_threaded(dict)))

715 return;

716

722 - netdata_rwlock_rdlock(&dict->index.rwlock);

717 + rw_spinlock_read_lock(&dict->index.rw_spinlock);

718 }

719

720 static inline void dictionary_index_rdlock_unlock(DICTIONARY *dict) {

721 if(unlikely(is_dictionary_single_threaded(dict)))

722 return;

723

729 - netdata_rwlock_unlock(&dict->index.rwlock);

724 + rw_spinlock_read_unlock(&dict->index.rw_spinlock);

725 }

726

727 static inline void dictionary_index_lock_wrlock(DICTIONARY *dict) {

728 if(unlikely(is_dictionary_single_threaded(dict)))

729 return;

730

736 - netdata_rwlock_wrlock(&dict->index.rwlock);

731 + rw_spinlock_write_lock(&dict->index.rw_spinlock);

732 }

733 static inline void dictionary_index_wrlock_unlock(DICTIONARY *dict) {

734 if(unlikely(is_dictionary_single_threaded(dict)))

735 return;

736

742 - netdata_rwlock_unlock(&dict->index.rwlock);

737 + rw_spinlock_write_unlock(&dict->index.rw_spinlock);

738 }

739

740 // ----------------------------------------------------------------------------

@@ -1232,7 +1227,7 @@ void dictionary_static_items_aral_init(void) {

1227 static SPINLOCK spinlock;

1228

1229 if(unlikely(!dict_items_aral || !dict_shared_items_aral)) {

1235 - netdata_spinlock_lock(&spinlock);

1230 + spinlock_lock(&spinlock);

1231

1232 // we have to check again

1233 if(!dict_items_aral)

@@ -1254,7 +1249,7 @@ void dictionary_static_items_aral_init(void) {

1249 aral_by_size_statistics(),

1250 NULL, NULL, false, false);

1251

1257 - netdata_spinlock_unlock(&spinlock);

1252 + spinlock_unlock(&spinlock);

1253 }

1254 }

1255

@@ -2096,7 +2091,7 @@ size_t dictionary_destroy(DICTIONARY *dict) {

2091

2092 internal_error(

2093 true,

2099 - "DICTIONARY: delaying destruction of dictionary created from %s() %zu@%s, because it has %ld referenced items in it (%ld total).",

2094 + "DICTIONARY: delaying destruction of dictionary created from %s() %zu@%s, because it has %d referenced items in it (%d total).",

2095 dict->creation_function,

2096 dict->creation_line,

2097 dict->creation_file,

@@ -2842,7 +2837,7 @@ static usec_t dictionary_unittest_run_and_measure_time(DICTIONARY *dict, char *m

2837 }

2838 }

2839

2845 - fprintf(stderr, " %zu errors, %ld (found %ld) items in dictionary, %ld (found %ld) referenced, %ld (found %ld) deleted, %llu usec \n",

2840 + fprintf(stderr, " %zu errors, %d (found %ld) items in dictionary, %d (found %ld) referenced, %d (found %ld) deleted, %llu usec \n",

2841 errs, dict?dict->entries:0, found_ok, dict?dict->referenced_items:0, found_referenced, dict?dict->pending_deletion_items:0, found_deleted, dt);

2842 *errors += errs;

2843 return dt;

@@ -2984,7 +2979,7 @@ static size_t unittest_check_dictionary(const char *label, DICTIONARY *dict, siz

2979 referenced++;

2980 }

2981

2987 - fprintf(stderr, "DICT %-20s: dictionary active items reported %ld, counted %zu, expected %zu...\t\t\t",

2982 + fprintf(stderr, "DICT %-20s: dictionary active items reported %d, counted %zu, expected %zu...\t\t\t",

2983 label, dict->entries, active, active_items);

2984 if(active != active_items || active != (size_t)dict->entries) {

2985 fprintf(stderr, "FAILED\n");

@@ -3002,7 +2997,7 @@ static size_t unittest_check_dictionary(const char *label, DICTIONARY *dict, siz

2997 else

2998 fprintf(stderr, "OK\n");

2999

3005 - fprintf(stderr, "DICT %-20s: dictionary referenced items reported %ld, counted %zu, expected %zu...\t\t",

3000 + fprintf(stderr, "DICT %-20s: dictionary referenced items reported %d, counted %zu, expected %zu...\t\t",

3001 label, dict->referenced_items, referenced, referenced_items);

3002 if(referenced != referenced_items || dict->referenced_items != (long int)referenced) {

3003 fprintf(stderr, "FAILED\n");

@@ -3011,7 +3006,7 @@ static size_t unittest_check_dictionary(const char *label, DICTIONARY *dict, siz

3006 else

3007 fprintf(stderr, "OK\n");

3008

3014 - fprintf(stderr, "DICT %-20s: dictionary pending deletion items reported %ld, counted %zu, expected %zu...\t",

3009 + fprintf(stderr, "DICT %-20s: dictionary pending deletion items reported %d, counted %zu, expected %zu...\t",

3010 label, dict->pending_deletion_items, pending, pending_deletion);

3011 if(pending != pending_deletion || pending != (size_t)dict->pending_deletion_items) {

3012 fprintf(stderr, "FAILED\n");

@@ -3257,9 +3252,9 @@ static int dictionary_unittest_threads() {

3252 ", searches %zu"

3253 ", resets %zu"

3254 ", flushes %zu"

3260 - ", entries %ld"

3261 - ", referenced_items %ld"

3262 - ", pending deletions %ld"

3255 + ", entries %d"

3256 + ", referenced_items %d"

3257 + ", pending deletions %d"

3258 ", check spins %zu"

3259 ", insert spins %zu"

3260 ", delete spins %zu"

@@ -3418,9 +3413,9 @@ static int dictionary_unittest_view_threads() {

3413 ", deletes %zu"

3414 ", searches %zu"

3415 ", resets %zu"

3421 - ", entries %ld"

3422 - ", referenced_items %ld"

3423 - ", pending deletions %ld"

3416 + ", entries %d"

3417 + ", referenced_items %d"

3418 + ", pending deletions %d"

3419 ", check spins %zu"

3420 ", insert spins %zu"

3421 ", delete spins %zu"

@@ -3443,9 +3438,9 @@ static int dictionary_unittest_view_threads() {

3438 ", deletes %zu"

3439 ", searches %zu"

3440 ", resets %zu"

3446 - ", entries %ld"

3447 - ", referenced_items %ld"

3448 - ", pending deletions %ld"

3441 + ", entries %d"

3442 + ", referenced_items %d"

3443 + ", pending deletions %d"

3444 ", check spins %zu"

3445 ", insert spins %zu"

3446 ", delete spins %zu"

libnetdata/dictionary/dictionary.h

+1 -1

@@ -46,7 +46,7 @@

46 typedef struct dictionary DICTIONARY;

47 typedef struct dictionary_item DICTIONARY_ITEM;

49 -typedef enum dictionary_options {

49 +typedef enum __attribute__((packed)) dictionary_options {

50 DICT_OPTION_NONE = 0, // the default is the opposite of all below

51 DICT_OPTION_SINGLE_THREADED = (1 << 0), // don't use any locks (default: use locks)

52 DICT_OPTION_VALUE_LINK_DONT_CLONE = (1 << 1), // don't copy the value, just point to the one provided (default: copy)

libnetdata/july/july.c

+6 -6

@@ -59,7 +59,7 @@ static struct {

59 void julyl_cleanup1(void) {

60 struct JulyL *item = NULL;

62 - if(!netdata_spinlock_trylock(&julyl_globals.protected.spinlock))

62 + if(!spinlock_trylock(&julyl_globals.protected.spinlock))

63 return;

65 if(julyl_globals.protected.available_items && julyl_globals.protected.available > 10) {

@@ -68,7 +68,7 @@ void julyl_cleanup1(void) {

68 julyl_globals.protected.available--;

69 }

71 - netdata_spinlock_unlock(&julyl_globals.protected.spinlock);

71 + spinlock_unlock(&julyl_globals.protected.spinlock);

73 if(item) {

74 size_t bytes = item->bytes;

@@ -81,7 +81,7 @@ void julyl_cleanup1(void) {

81 struct JulyL *julyl_get(void) {

82 struct JulyL *j;

84 - netdata_spinlock_lock(&julyl_globals.protected.spinlock);

84 + spinlock_lock(&julyl_globals.protected.spinlock);

86 j = julyl_globals.protected.available_items;

87 if(likely(j)) {

@@ -89,7 +89,7 @@ struct JulyL *julyl_get(void) {

89 julyl_globals.protected.available--;

90 }

92 - netdata_spinlock_unlock(&julyl_globals.protected.spinlock);

92 + spinlock_unlock(&julyl_globals.protected.spinlock);

94 if(unlikely(!j)) {

95 size_t bytes = sizeof(struct JulyL) + JULYL_MIN_ENTRIES * sizeof(struct JulyL_item);

@@ -113,10 +113,10 @@ static void julyl_release(struct JulyL *j) {

113 __atomic_add_fetch(&julyl_globals.atomics.bytes_moved, j->bytes_moved, __ATOMIC_RELAXED);

114 __atomic_add_fetch(&julyl_globals.atomics.reallocs, j->reallocs, __ATOMIC_RELAXED);

115

116 - netdata_spinlock_lock(&julyl_globals.protected.spinlock);

116 + spinlock_lock(&julyl_globals.protected.spinlock);

117 DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(julyl_globals.protected.available_items, j, cache.prev, cache.next);

118 julyl_globals.protected.available++;

119 - netdata_spinlock_unlock(&julyl_globals.protected.spinlock);

119 + spinlock_unlock(&julyl_globals.protected.spinlock);

120 }

121

122 size_t julyl_cache_size(void) {

libnetdata/libnetdata.c

+1 -1

@@ -249,7 +249,7 @@ static avl_tree_lock malloc_trace_index = {

249 .avl_tree = {

250 .root = NULL,

251 .compar = malloc_trace_compare},

252 - .rwlock = NETDATA_RWLOCK_INITIALIZER

252 + .rwlock = AVL_LOCK_INITIALIZER

253 };

254

255 int malloc_trace_walkthrough(int (*callback)(void *item, void *data), void *data) {

libnetdata/libnetdata.h

+4 -3

@@ -256,16 +256,17 @@ size_t judy_aral_structures(void);

256

257 #define DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(head, item, prev, next) \

258 do { \

259 + \

260 + (item)->next = NULL; \

261 + \

262 if(likely(head)) { \

263 (item)->prev = (head)->prev; \

264 (head)->prev->next = (item); \

265 (head)->prev = (item); \

263 - (item)->next = NULL; \

266 } \

267 else { \

268 + (item)->prev = (item); \

269 (head) = (item); \

267 - (head)->prev = (head); \

268 - (head)->next = NULL; \

270 } \

271 \

272 } while (0)

libnetdata/locks/locks.c

+105 -21

@@ -28,37 +28,49 @@ static __thread size_t netdata_locks_acquired_rwlocks = 0;

28 static __thread size_t netdata_locks_acquired_mutexes = 0;

30 inline void netdata_thread_disable_cancelability(void) {

31 - int old;

32 - int ret = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old);

33 - if(ret != 0)

34 - error("THREAD_CANCELABILITY: pthread_setcancelstate() on thread %s returned error %d", netdata_thread_tag(), ret);

35 - else {

36 - if(!netdata_thread_nested_disables)

37 - netdata_thread_first_cancelability = old;

31 + if(!netdata_thread_nested_disables) {

32 + int old;

33 + int ret = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old);

34 +

35 + if(ret != 0)

36 + error("THREAD_CANCELABILITY: pthread_setcancelstate() on thread %s returned error %d",

37 + netdata_thread_tag(), ret);

39 - netdata_thread_nested_disables++;

39 + netdata_thread_first_cancelability = old;

40 }

41 +

42 + netdata_thread_nested_disables++;

43 }

45 inline void netdata_thread_enable_cancelability(void) {

44 - if(netdata_thread_nested_disables < 1) {

45 - error("THREAD_CANCELABILITY: netdata_thread_enable_cancelability(): invalid thread cancelability count %d on thread %s - results will be undefined - please report this!",

46 + if(unlikely(netdata_thread_nested_disables < 1)) {

47 + internal_fatal(true, "THREAD_CANCELABILITY: trying to enable cancelability, but it was not not disabled");

48 +

49 + error("THREAD_CANCELABILITY: netdata_thread_enable_cancelability(): invalid thread cancelability count %d "

50 + "on thread %s - results will be undefined - please report this!",

51 netdata_thread_nested_disables, netdata_thread_tag());

52 +

53 + netdata_thread_nested_disables = 1;

54 }

48 - else if(netdata_thread_nested_disables == 1) {

55 +

56 + if(netdata_thread_nested_disables == 1) {

57 int old = 1;

58 int ret = pthread_setcancelstate(netdata_thread_first_cancelability, &old);

59 if(ret != 0)

60 error("THREAD_CANCELABILITY: pthread_setcancelstate() on thread %s returned error %d", netdata_thread_tag(), ret);

61 else {

54 - if(old != PTHREAD_CANCEL_DISABLE)

55 - error("THREAD_CANCELABILITY: netdata_thread_enable_cancelability(): old thread cancelability on thread %s was changed, expected DISABLED (%d), found %s (%d) - please report this!", netdata_thread_tag(), PTHREAD_CANCEL_DISABLE, (old == PTHREAD_CANCEL_ENABLE)?"ENABLED":"UNKNOWN", old);

62 + if(old != PTHREAD_CANCEL_DISABLE) {

63 + internal_fatal(true, "THREAD_CANCELABILITY: invalid old state cancelability");

64 +

65 + error("THREAD_CANCELABILITY: netdata_thread_enable_cancelability(): old thread cancelability "

66 + "on thread %s was changed, expected DISABLED (%d), found %s (%d) - please report this!",

67 + netdata_thread_tag(), PTHREAD_CANCEL_DISABLE,

68 + (old == PTHREAD_CANCEL_ENABLE) ? "ENABLED" : "UNKNOWN", old);

69 + }

70 }

57 -

58 - netdata_thread_nested_disables = 0;

71 }

60 - else

61 - netdata_thread_nested_disables--;

72 +

73 + netdata_thread_nested_disables--;

74 }

76 // ----------------------------------------------------------------------------

@@ -278,11 +290,11 @@ int __netdata_rwlock_trywrlock(netdata_rwlock_t *rwlock) {

290 // spinlock implementation

291 // https://www.youtube.com/watch?v=rmGJc9PXpuE&t=41s

292

281 -void netdata_spinlock_init(SPINLOCK *spinlock) {

293 +void spinlock_init(SPINLOCK *spinlock) {

294 memset(spinlock, 0, sizeof(SPINLOCK));

295 }

296

285 -void netdata_spinlock_lock(SPINLOCK *spinlock) {

297 +void spinlock_lock(SPINLOCK *spinlock) {

298 static const struct timespec ns = { .tv_sec = 0, .tv_nsec = 1 };

299

300 #ifdef NETDATA_INTERNAL_CHECKS

@@ -314,7 +326,7 @@ void netdata_spinlock_lock(SPINLOCK *spinlock) {

326 #endif

327 }

328

317 -void netdata_spinlock_unlock(SPINLOCK *spinlock) {

329 +void spinlock_unlock(SPINLOCK *spinlock) {

330 #ifdef NETDATA_INTERNAL_CHECKS

331 spinlock->locker_pid = 0;

332 #endif

@@ -322,7 +334,7 @@ void netdata_spinlock_unlock(SPINLOCK *spinlock) {

334 netdata_thread_enable_cancelability();

335 }

336

325 -bool netdata_spinlock_trylock(SPINLOCK *spinlock) {

337 +bool spinlock_trylock(SPINLOCK *spinlock) {

338 netdata_thread_disable_cancelability();

339

340 if(!__atomic_load_n(&spinlock->locked, __ATOMIC_RELAXED) &&

@@ -331,9 +343,81 @@ bool netdata_spinlock_trylock(SPINLOCK *spinlock) {

343 return true;

344

345 // we didn't get the lock

346 + netdata_thread_enable_cancelability();

347 + return false;

348 +}

349 +

350 +// ----------------------------------------------------------------------------

351 +// rw_spinlock implementation

352 +

353 +void rw_spinlock_init(RW_SPINLOCK *rw_spinlock) {

354 + rw_spinlock->readers = 0;

355 + spinlock_init(&rw_spinlock->spinlock);

356 +}

357 +

358 +void rw_spinlock_read_lock(RW_SPINLOCK *rw_spinlock) {

359 + netdata_thread_disable_cancelability();

360 +

361 + spinlock_lock(&rw_spinlock->spinlock);

362 + __atomic_add_fetch(&rw_spinlock->readers, 1, __ATOMIC_RELAXED);

363 + spinlock_unlock(&rw_spinlock->spinlock);

364 +}

365 +

366 +void rw_spinlock_read_unlock(RW_SPINLOCK *rw_spinlock) {

367 +#ifndef NETDATA_INTERNAL_CHECKS

368 + __atomic_sub_fetch(&rw_spinlock->readers, 1, __ATOMIC_RELAXED);

369 +#else

370 + int32_t x = __atomic_sub_fetch(&rw_spinlock->readers, 1, __ATOMIC_RELAXED);

371 + if(x < 0)

372 + fatal("RW_SPINLOCK: readers is negative %d", x);

373 +#endif

374 +

375 + netdata_thread_enable_cancelability();

376 +}

377 +

378 +void rw_spinlock_write_lock(RW_SPINLOCK *rw_spinlock) {

379 + static const struct timespec ns = { .tv_sec = 0, .tv_nsec = 1 };

380 +

381 + spinlock_lock(&rw_spinlock->spinlock);

382 + size_t count = 0;

383 + while (__atomic_load_n(&rw_spinlock->readers, __ATOMIC_RELAXED) > 0) {

384 + // Busy wait until all readers have released their locks.

385 + if(++count > 1000)

386 + nanosleep(&ns, NULL);

387 + }

388 +}

389 +

390 +void rw_spinlock_write_unlock(RW_SPINLOCK *rw_spinlock) {

391 + spinlock_unlock(&rw_spinlock->spinlock);

392 +}

393 +

394 +bool rw_spinlock_tryread_lock(RW_SPINLOCK *rw_spinlock) {

395 + if(spinlock_trylock(&rw_spinlock->spinlock)) {

396 + __atomic_add_fetch(&rw_spinlock->readers, 1, __ATOMIC_RELAXED);

397 + spinlock_unlock(&rw_spinlock->spinlock);

398 + netdata_thread_disable_cancelability();

399 + return true;

400 + }

401 +

402 + return false;

403 +}

404 +

405 +bool rw_spinlock_trywrite_lock(RW_SPINLOCK *rw_spinlock) {

406 + if(spinlock_trylock(&rw_spinlock->spinlock)) {

407 + if (__atomic_load_n(&rw_spinlock->readers, __ATOMIC_RELAXED) == 0) {

408 + // No readers, we've successfully acquired the write lock

409 + return true;

410 + }

411 + else {

412 + // There are readers, unlock the spinlock and return false

413 + spinlock_unlock(&rw_spinlock->spinlock);

414 + }

415 + }

416 +

417 return false;

418 }

419

420 +

421 #ifdef NETDATA_TRACE_RWLOCKS

422

423 // ----------------------------------------------------------------------------

libnetdata/locks/locks.h

+20 -4

@@ -20,10 +20,26 @@ typedef struct netdata_spinlock {

20 #define NETDATA_SPINLOCK_INITIALIZER \

21 { .locked = false }

23 -void netdata_spinlock_init(SPINLOCK *spinlock);

24 -void netdata_spinlock_lock(SPINLOCK *spinlock);

25 -void netdata_spinlock_unlock(SPINLOCK *spinlock);

26 -bool netdata_spinlock_trylock(SPINLOCK *spinlock);

23 +void spinlock_init(SPINLOCK *spinlock);

24 +void spinlock_lock(SPINLOCK *spinlock);

25 +void spinlock_unlock(SPINLOCK *spinlock);

26 +bool spinlock_trylock(SPINLOCK *spinlock);

27 +

28 +typedef struct netdata_rw_spinlock {

29 + int32_t readers;

30 + SPINLOCK spinlock;

31 +} RW_SPINLOCK;

32 +

33 +#define NETDATA_RW_SPINLOCK_INITIALIZER \

34 + { .readers = 0, .spinlock = NETDATA_SPINLOCK_INITIALIZER }

35 +

36 +void rw_spinlock_init(RW_SPINLOCK *rw_spinlock);

37 +void rw_spinlock_read_lock(RW_SPINLOCK *rw_spinlock);

38 +void rw_spinlock_read_unlock(RW_SPINLOCK *rw_spinlock);

39 +void rw_spinlock_write_lock(RW_SPINLOCK *rw_spinlock);

40 +void rw_spinlock_write_unlock(RW_SPINLOCK *rw_spinlock);

41 +bool rw_spinlock_tryread_lock(RW_SPINLOCK *rw_spinlock);

42 +bool rw_spinlock_trywrite_lock(RW_SPINLOCK *rw_spinlock);

44 #ifdef NETDATA_TRACE_RWLOCKS

libnetdata/parser/parser.c

+1 -1

@@ -46,7 +46,7 @@ PARSER *parser_init(void *user, FILE *fp_input, FILE *fp_output, int fd,

46 parser->flags = flags;

47 parser->worker_job_next_id = WORKER_PARSER_FIRST_JOB;

49 - netdata_spinlock_init(&parser->writer.spinlock);

49 + spinlock_init(&parser->writer.spinlock);

50 return parser;

51 }

libnetdata/socket/security.c

+2 -2

@@ -571,7 +571,7 @@ static SSL_CTX * netdata_ssl_create_server_ctx(unsigned long mode) {

571 */

572 void netdata_ssl_initialize_ctx(int selector) {

573 static SPINLOCK sp = NETDATA_SPINLOCK_INITIALIZER;

574 - netdata_spinlock_lock(&sp);

574 + spinlock_lock(&sp);

575

576 switch (selector) {

577 case NETDATA_SSL_WEB_SERVER_CTX: {

@@ -621,7 +621,7 @@ void netdata_ssl_initialize_ctx(int selector) {

621 }

622 }

623

624 - netdata_spinlock_unlock(&sp);

624 + spinlock_unlock(&sp);

625 }

626

627 /**

libnetdata/worker_utilization/worker_utilization.c

+15 -15

@@ -70,9 +70,9 @@ static inline usec_t worker_now_monotonic_usec(void) {

70 }

72 size_t workers_allocated_memory(void) {

73 - netdata_spinlock_lock(&workers_globals.spinlock);

73 + spinlock_lock(&workers_globals.spinlock);

74 size_t memory = workers_globals.memory;

75 - netdata_spinlock_unlock(&workers_globals.spinlock);

75 + spinlock_unlock(&workers_globals.spinlock);

77 return memory;

78 }

@@ -91,7 +91,7 @@ void worker_register(const char *name) {

91 worker->last_action = WORKER_IDLE;

93 size_t name_size = strlen(name) + 1;

94 - netdata_spinlock_lock(&workers_globals.spinlock);

94 + spinlock_lock(&workers_globals.spinlock);

96 workers_globals.memory += sizeof(struct worker) + strlen(worker->tag) + 1 + strlen(worker->workname) + 1;

@@ -100,18 +100,18 @@ void worker_register(const char *name) {

100 struct workers_workname *workname = *PValue;

101 if(!workname) {

102 workname = mallocz(sizeof(struct workers_workname));

103 - netdata_spinlock_init(&workname->spinlock);

103 + spinlock_init(&workname->spinlock);

104 workname->base = NULL;

105 *PValue = workname;

106

107 workers_globals.memory += sizeof(struct workers_workname) + JUDYHS_INDEX_SIZE_ESTIMATE(name_size);

108 }

109

110 - netdata_spinlock_lock(&workname->spinlock);

110 + spinlock_lock(&workname->spinlock);

111 DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(workname->base, worker, prev, next);

112 - netdata_spinlock_unlock(&workname->spinlock);

112 + spinlock_unlock(&workname->spinlock);

113

114 - netdata_spinlock_unlock(&workers_globals.spinlock);

114 + spinlock_unlock(&workers_globals.spinlock);

115 }

116

117 void worker_register_job_custom_metric(size_t job_id, const char *name, const char *units, WORKER_METRIC_TYPE type) {

@@ -144,13 +144,13 @@ void worker_unregister(void) {

144 if(unlikely(!worker)) return;

145

146 size_t workname_size = strlen(worker->workname) + 1;

147 - netdata_spinlock_lock(&workers_globals.spinlock);

147 + spinlock_lock(&workers_globals.spinlock);

148 Pvoid_t *PValue = JudyHSGet(workers_globals.worknames_JudyHS, (void *)worker->workname, workname_size);

149 if(PValue) {

150 struct workers_workname *workname = *PValue;

151 - netdata_spinlock_lock(&workname->spinlock);

151 + spinlock_lock(&workname->spinlock);

152 DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(workname->base, worker, prev, next);

153 - netdata_spinlock_unlock(&workname->spinlock);

153 + spinlock_unlock(&workname->spinlock);

154

155 if(!workname->base) {

156 JudyHSDel(&workers_globals.worknames_JudyHS, (void *) worker->workname, workname_size, PJE0);

@@ -159,7 +159,7 @@ void worker_unregister(void) {

159 }

160 }

161 workers_globals.memory -= sizeof(struct worker) + strlen(worker->tag) + 1 + strlen(worker->workname) + 1;

162 - netdata_spinlock_unlock(&workers_globals.spinlock);

162 + spinlock_unlock(&workers_globals.spinlock);

163

164 for(int i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES ;i++) {

165 string_freez(worker->per_job_type[i].name);

@@ -247,7 +247,7 @@ void workers_foreach(const char *name, void (*callback)(

247 , NETDATA_DOUBLE *job_custom_values

248 )

249 , void *data) {

250 - netdata_spinlock_lock(&workers_globals.spinlock);

250 + spinlock_lock(&workers_globals.spinlock);

251 usec_t busy_time, delta;

252 size_t i, jobs_started, jobs_running;

253

@@ -256,12 +256,12 @@ void workers_foreach(const char *name, void (*callback)(

256 Pvoid_t *PValue = JudyHSGet(workers_globals.worknames_JudyHS, (void *)name, workname_size);

257 if(PValue) {

258 workname = *PValue;

259 - netdata_spinlock_lock(&workname->spinlock);

259 + spinlock_lock(&workname->spinlock);

260 }

261 else

262 workname = NULL;

263

264 - netdata_spinlock_unlock(&workers_globals.spinlock);

264 + spinlock_unlock(&workers_globals.spinlock);

265

266 if(!workname)

267 return;

@@ -379,5 +379,5 @@ void workers_foreach(const char *name, void (*callback)(

379 );

380 }

381

382 - netdata_spinlock_unlock(&workname->spinlock);

382 + spinlock_unlock(&workname->spinlock);

383 }

ml/ml.cc

+17 -17

@@ -337,7 +337,7 @@ ml_dimension_calculated_numbers(ml_training_thread_t *training_thread, ml_dimens

337 // Figure out what our time window should be.

338 training_response.query_before_t = training_response.last_entry_on_response;

339 training_response.query_after_t = std::max(

340 - training_response.query_before_t - static_cast<time_t>((max_n - 1) * dim->rd->update_every),

340 + training_response.query_before_t - static_cast<time_t>((max_n - 1) * dim->rd->rrdset->update_every),

341 training_response.first_entry_on_response

342 );

343

@@ -568,9 +568,9 @@ int ml_dimension_load_models(RRDDIM *rd) {

568 if (!dim)

569 return 0;

570

571 - netdata_spinlock_lock(&dim->slock);

571 + spinlock_lock(&dim->slock);

572 bool is_empty = dim->km_contexts.empty();

573 - netdata_spinlock_unlock(&dim->slock);

573 + spinlock_unlock(&dim->slock);

574

575 if (!is_empty)

576 return 0;

@@ -602,7 +602,7 @@ int ml_dimension_load_models(RRDDIM *rd) {

602 if (unlikely(rc != SQLITE_OK))

603 goto bind_fail;

604

605 - netdata_spinlock_lock(&dim->slock);

605 + spinlock_lock(&dim->slock);

606

607 dim->km_contexts.reserve(Cfg.num_models_to_use);

608 while ((rc = sqlite3_step_monitored(res)) == SQLITE_ROW) {

@@ -639,7 +639,7 @@ int ml_dimension_load_models(RRDDIM *rd) {

639 dim->ts = TRAINING_STATUS_TRAINED;

640 }

641

642 - netdata_spinlock_unlock(&dim->slock);

642 + spinlock_unlock(&dim->slock);

643

644 if (unlikely(rc != SQLITE_DONE))

645 error_report("Failed to load models, rc = %d", rc);

@@ -666,7 +666,7 @@ ml_dimension_train_model(ml_training_thread_t *training_thread, ml_dimension_t *

666 ml_training_response_t training_response = P.second;

667

668 if (training_response.result != TRAINING_RESULT_OK) {

669 - netdata_spinlock_lock(&dim->slock);

669 + spinlock_lock(&dim->slock);

670

671 dim->mt = METRIC_TYPE_CONSTANT;

672

@@ -688,7 +688,7 @@ ml_dimension_train_model(ml_training_thread_t *training_thread, ml_dimension_t *

688 dim->last_training_time = training_response.last_entry_on_response;

689 enum ml_training_result result = training_response.result;

690

691 - netdata_spinlock_unlock(&dim->slock);

691 + spinlock_unlock(&dim->slock);

692

693 return result;

694 }

@@ -714,7 +714,7 @@ ml_dimension_train_model(ml_training_thread_t *training_thread, ml_dimension_t *

714 // update models

715 worker_is_busy(WORKER_TRAIN_UPDATE_MODELS);

716 {

717 - netdata_spinlock_lock(&dim->slock);

717 + spinlock_lock(&dim->slock);

718

719 if (dim->km_contexts.size() < Cfg.num_models_to_use) {

720 dim->km_contexts.push_back(std::move(dim->kmeans));

@@ -753,7 +753,7 @@ ml_dimension_train_model(ml_training_thread_t *training_thread, ml_dimension_t *

753 model_info.kmeans = dim->km_contexts.back();

754 training_thread->pending_model_info.push_back(model_info);

755

756 - netdata_spinlock_unlock(&dim->slock);

756 + spinlock_unlock(&dim->slock);

757 }

758

759 return training_response.result;

@@ -782,7 +782,7 @@ ml_dimension_schedule_for_training(ml_dimension_t *dim, time_t curr_time)

782 break;

783 case TRAINING_STATUS_SILENCED:

784 case TRAINING_STATUS_TRAINED:

785 - if ((dim->last_training_time + (Cfg.train_every * dim->rd->update_every)) < curr_time) {

785 + if ((dim->last_training_time + (Cfg.train_every * dim->rd->rrdset->update_every)) < curr_time) {

786 schedule_for_training = true;

787 dim->ts = TRAINING_STATUS_PENDING_WITH_MODEL;

788 }

@@ -852,7 +852,7 @@ ml_dimension_predict(ml_dimension_t *dim, time_t curr_time, calculated_number_t

852 /*

853 * Lock to predict and possibly schedule the dimension for training

854 */

855 - if (netdata_spinlock_trylock(&dim->slock) == 0)

855 + if (spinlock_trylock(&dim->slock) == 0)

856 return false;

857

858 // Mark the metric time as variable if we received different values

@@ -867,7 +867,7 @@ ml_dimension_predict(ml_dimension_t *dim, time_t curr_time, calculated_number_t

867 case TRAINING_STATUS_UNTRAINED:

868 case TRAINING_STATUS_PENDING_WITHOUT_MODEL: {

869 case TRAINING_STATUS_SILENCED:

870 - netdata_spinlock_unlock(&dim->slock);

870 + spinlock_unlock(&dim->slock);

871 return false;

872 }

873 default:

@@ -892,7 +892,7 @@ ml_dimension_predict(ml_dimension_t *dim, time_t curr_time, calculated_number_t

892

893 if (anomaly_score < (100 * Cfg.dimension_anomaly_score_threshold)) {

894 global_statistics_ml_models_consulted(models_consulted);

895 - netdata_spinlock_unlock(&dim->slock);

895 + spinlock_unlock(&dim->slock);

896 return false;

897 }

898

@@ -906,7 +906,7 @@ ml_dimension_predict(ml_dimension_t *dim, time_t curr_time, calculated_number_t

906 dim->ts = TRAINING_STATUS_SILENCED;

907 }

908

909 - netdata_spinlock_unlock(&dim->slock);

909 + spinlock_unlock(&dim->slock);

910

911 global_statistics_ml_models_consulted(models_consulted);

912 return sum;

@@ -1277,7 +1277,7 @@ void ml_host_stop(RRDHOST *rh) {

1277 if (!dim)

1278 continue;

1279

1280 - netdata_spinlock_lock(&dim->slock);

1280 + spinlock_lock(&dim->slock);

1281

1282 // reset dim

1283 // TODO: should we drop in-mem models, or mark them as stale? Is it

@@ -1292,7 +1292,7 @@ void ml_host_stop(RRDHOST *rh) {

1292

1293 ml_kmeans_init(&dim->kmeans);

1294

1295 - netdata_spinlock_unlock(&dim->slock);

1295 + spinlock_unlock(&dim->slock);

1296 }

1297 rrddim_foreach_done(rdp);

1298 }

@@ -1459,7 +1459,7 @@ void ml_dimension_new(RRDDIM *rd)

1459 else

1460 dim->mls = MACHINE_LEARNING_STATUS_ENABLED;

1461

1462 - netdata_spinlock_init(&dim->slock);

1462 + spinlock_init(&dim->slock);

1463

1464 dim->km_contexts.reserve(Cfg.num_models_to_use);

1465

streaming/receiver.c

+1 -1

@@ -720,7 +720,7 @@ static void rrdpush_receive(struct receiver_state *rpt)

720 #ifdef NETDATA_INTERNAL_CHECKS

721 info("STREAM '%s' [receive from [%s]:%s]: "

722 "client willing to stream metrics for host '%s' with machine_guid '%s': "

723 - "update every = %d, history = %ld, memory mode = %s, health %s,%s tags '%s'"

723 + "update every = %d, history = %d, memory mode = %s, health %s,%s tags '%s'"

724 , rpt->hostname

725 , rpt->client_ip

726 , rpt->client_port

streaming/replication.c

+12 -12

@@ -40,9 +40,9 @@ static struct replication_query_statistics replication_queries = {

40 };

42 struct replication_query_statistics replication_get_query_statistics(void) {

43 - netdata_spinlock_lock(&replication_queries.spinlock);

43 + spinlock_lock(&replication_queries.spinlock);

44 struct replication_query_statistics ret = replication_queries;

45 - netdata_spinlock_unlock(&replication_queries.spinlock);

45 + spinlock_unlock(&replication_queries.spinlock);

46 return ret;

47 }

@@ -144,7 +144,7 @@ static struct replication_query *replication_query_prepare(

144 }

145

146 if(q->query.enable_streaming) {

147 - netdata_spinlock_lock(&st->data_collection_lock);

147 + spinlock_lock(&st->data_collection_lock);

148 q->query.locked_data_collection = true;

149

150 if (st->last_updated.tv_sec > q->query.before) {

@@ -168,7 +168,7 @@ static struct replication_query *replication_query_prepare(

168 size_t count = 0;

169 RRDDIM *rd;

170 rrddim_foreach_read(rd, st) {

171 - if (unlikely(!rd || !rd_dfe.item || !rd->exposed))

171 + if (unlikely(!rd || !rd_dfe.item || !rrddim_check_exposed(rd)))

172 continue;

173

174 if (unlikely(rd_dfe.counter >= q->dimensions)) {

@@ -198,7 +198,7 @@ static struct replication_query *replication_query_prepare(

198 q->query.execute = false;

199

200 if(q->query.locked_data_collection) {

201 - netdata_spinlock_unlock(&st->data_collection_lock);

201 + spinlock_unlock(&st->data_collection_lock);

202 q->query.locked_data_collection = false;

203 }

204

@@ -216,7 +216,7 @@ static void replication_send_chart_collection_state(BUFFER *wb, RRDSET *st, STRE

216 NUMBER_ENCODING encoding = (capabilities & STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_DECIMAL;

217 RRDDIM *rd;

218 rrddim_foreach_read(rd, st){

219 - if (!rd->exposed) continue;

219 + if (!rrddim_check_exposed(rd)) continue;

220

221 buffer_fast_strcat(wb, PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE " '",

222 sizeof(PLUGINSD_KEYWORD_REPLAY_RRDDIM_STATE) - 1 + 2);

@@ -248,7 +248,7 @@ static void replication_query_finalize(BUFFER *wb, struct replication_query *q,

248 replication_send_chart_collection_state(wb, q->st, q->query.capabilities);

249

250 if(q->query.locked_data_collection) {

251 - netdata_spinlock_unlock(&q->st->data_collection_lock);

251 + spinlock_unlock(&q->st->data_collection_lock);

252 q->query.locked_data_collection = false;

253 }

254

@@ -269,7 +269,7 @@ static void replication_query_finalize(BUFFER *wb, struct replication_query *q,

269 }

270

271 if(executed) {

272 - netdata_spinlock_lock(&replication_queries.spinlock);

272 + spinlock_lock(&replication_queries.spinlock);

273 replication_queries.queries_started += queries;

274 replication_queries.queries_finished += queries;

275 replication_queries.points_read += q->points_read;

@@ -280,7 +280,7 @@ static void replication_query_finalize(BUFFER *wb, struct replication_query *q,

280 s->replication.latest_completed_before_t = q->query.before;

281 }

282

283 - netdata_spinlock_unlock(&replication_queries.spinlock);

283 + spinlock_unlock(&replication_queries.spinlock);

284 }

285

286 __atomic_sub_fetch(&replication_buffers_allocated, sizeof(struct replication_query) + dimensions * sizeof(struct replication_dimension), __ATOMIC_RELAXED);

@@ -678,7 +678,7 @@ bool replication_response_execute_and_finalize(struct replication_query *q, size

678 }

679

680 if(locked_data_collection)

681 - netdata_spinlock_unlock(&st->data_collection_lock);

681 + spinlock_unlock(&st->data_collection_lock);

682

683 return enable_streaming;

684 }

@@ -1056,11 +1056,11 @@ static inline bool replication_recursive_lock_mode(char mode) {

1056

1057 if(mode == 'L') { // (L)ock

1058 if(++recursions == 1)

1059 - netdata_spinlock_lock(&replication_globals.spinlock);

1059 + spinlock_lock(&replication_globals.spinlock);

1060 }

1061 else if(mode == 'U') { // (U)nlock

1062 if(--recursions == 0)

1063 - netdata_spinlock_unlock(&replication_globals.spinlock);

1063 + spinlock_unlock(&replication_globals.spinlock);

1064 }

1065 else if(mode == 'C') { // (C)heck

1066 if(recursions > 0)

streaming/rrdpush.c

+18 -14

@@ -264,7 +264,7 @@ static inline bool rrdpush_send_chart_definition(BUFFER *wb, RRDSET *st) {

264 // send the chart

265 buffer_sprintf(

266 wb

267 - , "CHART \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" %ld %d \"%s %s %s %s\" \"%s\" \"%s\"\n"

267 + , "CHART \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" %d %d \"%s %s %s %s\" \"%s\" \"%s\"\n"

268 , rrdset_id(st)

269 , name

270 , rrdset_title(st)

@@ -291,7 +291,7 @@ static inline bool rrdpush_send_chart_definition(BUFFER *wb, RRDSET *st) {

291 rrddim_foreach_read(rd, st) {

292 buffer_sprintf(

293 wb

294 - , "DIMENSION \"%s\" \"%s\" \"%s\" " COLLECTED_NUMBER_FORMAT " " COLLECTED_NUMBER_FORMAT " \"%s %s %s\"\n"

294 + , "DIMENSION \"%s\" \"%s\" \"%s\" %d %d \"%s %s %s\"\n"

295 , rrddim_id(rd)

296 , rrddim_name(rd)

297 , rrd_algorithm_name(rd->algorithm)

@@ -301,7 +301,7 @@ static inline bool rrdpush_send_chart_definition(BUFFER *wb, RRDSET *st) {

301 , rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN)?"hidden":""

302 , rrddim_option_check(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS)?"noreset":""

303 );

304 - rd->exposed = 1;

304 + rrddim_set_exposed(rd);

305 }

306 rrddim_foreach_done(rd);

307

@@ -355,10 +355,10 @@ static void rrdpush_send_chart_metrics(BUFFER *wb, RRDSET *st, struct sender_sta

355

356 RRDDIM *rd;

357 rrddim_foreach_read(rd, st) {

358 - if(unlikely(!rd->updated))

358 + if(unlikely(!rrddim_check_updated(rd)))

359 continue;

360

361 - if(likely(rd->exposed)) {

361 + if(likely(rrddim_check_exposed(rd))) {

362 buffer_fast_strcat(wb, "SET \"", 5);

363 buffer_fast_strcat(wb, rrddim_id(rd), string_strlen(rd->id));

364 buffer_fast_strcat(wb, "\" = ", 4);

@@ -598,6 +598,7 @@ int connect_to_one_of_destinations(

598 *reconnects_counter += 1;

599

600 d->last_attempt = now;

601 + d->attempts++;

602 sock = connect_to_this(string2str(d->destination), default_port, timeout);

603

604 if (sock != -1) {

@@ -1091,13 +1092,13 @@ int rrdpush_receiver_thread_spawn(struct web_client *w, char *decoded_query_stri

1092 static time_t last_stream_accepted_t = 0;

1093

1094 time_t now = now_realtime_sec();

1094 - netdata_spinlock_lock(&spinlock);

1095 + spinlock_lock(&spinlock);

1096

1097 if(unlikely(last_stream_accepted_t == 0))

1098 last_stream_accepted_t = now;

1099

1100 if(now - last_stream_accepted_t < web_client_streaming_rate_t) {

1100 - netdata_spinlock_unlock(&spinlock);

1101 + spinlock_unlock(&spinlock);

1102

1103 char msg[100 + 1];

1104 snprintfz(msg, 100,

@@ -1114,7 +1115,7 @@ int rrdpush_receiver_thread_spawn(struct web_client *w, char *decoded_query_stri

1115 }

1116

1117 last_stream_accepted_t = now;

1117 - netdata_spinlock_unlock(&spinlock);

1118 + spinlock_unlock(&spinlock);

1119 }

1120

1121 /*

@@ -1220,11 +1221,10 @@ static struct {

1221 STREAM_HANDSHAKE err;

1222 const char *str;

1223 } handshake_errors[] = {

1223 - { STREAM_HANDSHAKE_OK_V5, "OK_V5" },

1224 - { STREAM_HANDSHAKE_OK_V4, "OK_V4" },

1225 - { STREAM_HANDSHAKE_OK_V3, "OK_V3" },

1226 - { STREAM_HANDSHAKE_OK_V2, "OK_V2" },

1227 - { STREAM_HANDSHAKE_OK_V1, "OK_V1" },

1224 + { STREAM_HANDSHAKE_OK_V3, "CONNECTED" },

1225 + { STREAM_HANDSHAKE_OK_V2, "CONNECTED" },

1226 + { STREAM_HANDSHAKE_OK_V1, "CONNECTED" },

1227 + { STREAM_HANDSHAKE_NEVER, "NOT_TRIED_YET" },

1228 { STREAM_HANDSHAKE_ERROR_BAD_HANDSHAKE, "BAD HANDSHAKE" },

1229 { STREAM_HANDSHAKE_ERROR_LOCALHOST, "LOCALHOST" },

1230 { STREAM_HANDSHAKE_ERROR_ALREADY_CONNECTED, "ALREADY CONNECTED" },

@@ -1241,12 +1241,16 @@ static struct {

1241 };

1242

1243 const char *stream_handshake_error_to_string(STREAM_HANDSHAKE handshake_error) {

1244 + if(handshake_error >= STREAM_HANDSHAKE_OK_V1)

1245 + // handshake_error is the whole version / capabilities number

1246 + return "CONNECTED";

1247 +

1248 for(size_t i = 0; handshake_errors[i].str ; i++) {

1249 if(handshake_error == handshake_errors[i].err)

1250 return handshake_errors[i].str;

1251 }

1252

1249 - return "";

1253 + return "UNKNOWN";

1254 }

1255

1256 static struct {

streaming/rrdpush.h

+5 -5

@@ -78,11 +78,10 @@ STREAM_CAPABILITIES stream_our_capabilities(RRDHOST *host, bool sender);

78 #define START_STREAMING_ERROR_INITIALIZATION "The server is initializing. Try later."

80 typedef enum {

81 - STREAM_HANDSHAKE_OK_V5 = 5, // COMPRESSION

82 - STREAM_HANDSHAKE_OK_V4 = 4, // CLABELS

83 - STREAM_HANDSHAKE_OK_V3 = 3, // CLAIM

84 - STREAM_HANDSHAKE_OK_V2 = 2, // HLABELS

85 - STREAM_HANDSHAKE_OK_V1 = 1,

81 + STREAM_HANDSHAKE_OK_V3 = 3, // v3+

82 + STREAM_HANDSHAKE_OK_V2 = 2, // v2

83 + STREAM_HANDSHAKE_OK_V1 = 1, // v1

84 + STREAM_HANDSHAKE_NEVER = 0, // never tried to connect

85 STREAM_HANDSHAKE_ERROR_BAD_HANDSHAKE = -1,

86 STREAM_HANDSHAKE_ERROR_LOCALHOST = -2,

87 STREAM_HANDSHAKE_ERROR_ALREADY_CONNECTED = -3,

@@ -334,6 +333,7 @@ struct receiver_state {

333 struct rrdpush_destinations {

334 STRING *destination;

335 bool ssl;

336 + uint32_t attempts;

337

338 const char *last_error;

339 time_t last_attempt;

streaming/sender.c

+4 -4

@@ -258,7 +258,7 @@ static void rrdpush_sender_thread_reset_all_charts(RRDHOST *host) {

258

259 RRDDIM *rd;

260 rrddim_foreach_read(rd, st)

261 - rd->exposed = 0;

261 + rrddim_clear_exposed(rd);

262 rrddim_foreach_done(rd);

263 }

264 rrdset_foreach_done(st);

@@ -1188,10 +1188,10 @@ static void rrdpush_sender_thread_cleanup_callback(void *ptr) {

1188 void rrdpush_initialize_ssl_ctx(RRDHOST *host) {

1189 #ifdef ENABLE_HTTPS

1190 static SPINLOCK sp = NETDATA_SPINLOCK_INITIALIZER;

1191 - netdata_spinlock_lock(&sp);

1191 + spinlock_lock(&sp);

1192

1193 if(netdata_ssl_streaming_sender_ctx || !host) {

1194 - netdata_spinlock_unlock(&sp);

1194 + spinlock_unlock(&sp);

1195 return;

1196 }

1197

@@ -1207,7 +1207,7 @@ void rrdpush_initialize_ssl_ctx(RRDHOST *host) {

1207 }

1208 }

1209

1210 - netdata_spinlock_unlock(&sp);

1210 + spinlock_unlock(&sp);

1211 #endif

1212 }

1213

web/api/formatters/charts2json.c

+1 -1

@@ -53,7 +53,7 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived

53 ",\n\t\"os\": \"%s\""

54 ",\n\t\"timezone\": \"%s\""

55 ",\n\t\"update_every\": %d"

56 - ",\n\t\"history\": %ld"

56 + ",\n\t\"history\": %d"

57 ",\n\t\"memory_mode\": \"%s\""

58 ",\n\t\"custom_info\": \"%s\""

59 ",\n\t\"charts\": {"

web/api/formatters/rrdset2json.c

+2 -2

@@ -37,7 +37,7 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor

37 "\t\t\t\"family\": \"%s\",\n"

38 "\t\t\t\"context\": \"%s\",\n"

39 "\t\t\t\"title\": \"%s (%s)\",\n"

40 - "\t\t\t\"priority\": %ld,\n"

40 + "\t\t\t\"priority\": %d,\n"

41 "\t\t\t\"plugin\": \"%s\",\n"

42 "\t\t\t\"module\": \"%s\",\n"

43 "\t\t\t\"units\": \"%s\",\n"

@@ -90,7 +90,7 @@ void rrdset2json(RRDSET *st, BUFFER *wb, size_t *dimensions_count, size_t *memor

90 rrddim_foreach_read(rd, st) {

91 if(rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN) || rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) continue;

93 - memory += sizeof(RRDDIM) + rd->memsize;

93 + memory += rrddim_size() + rd->db.memsize;

95 if (dimensions)

96 buffer_strcat(wb, ",\n\t\t\t\t\"");

web/api/queries/query.c

+1 -1

@@ -1952,7 +1952,7 @@ void rrdr_fill_tier_gap_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s

1952 if(unlikely(!t)) return;

1953

1954 time_t latest_time_s = storage_engine_latest_time_s(t->backend, t->db_metric_handle);

1955 - time_t granularity = (time_t)t->tier_grouping * (time_t)rd->update_every;

1955 + time_t granularity = (time_t)t->tier_grouping * (time_t)rd->rrdset->update_every;

1956 time_t time_diff = now_s - latest_time_s;

1957

1958 // if the user wants only NEW backfilling, and we don't have any data

web/rtc/webrtc.c

+18 -18

@@ -387,9 +387,9 @@ static void myClosedCallback(int id, void *user_ptr) {

387 __atomic_store_n(&chan->open, false, __ATOMIC_RELAXED);

388 internal_error(true, "WEBRTC[%d],DC[%d]: data channel closed.", chan->conn->pc, chan->dc);

389

390 - netdata_spinlock_lock(&chan->conn->channels.spinlock);

390 + spinlock_lock(&chan->conn->channels.spinlock);

391 DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(chan->conn->channels.head, chan, link.prev, link.next);

392 - netdata_spinlock_unlock(&chan->conn->channels.spinlock);

392 + spinlock_unlock(&chan->conn->channels.spinlock);

393

394 log_access("WEBRTC[%d],DC[%d]: %d DATA CHANNEL '%s' CLOSED", chan->conn->pc, chan->dc, gettid(), chan->label);

395

@@ -451,9 +451,9 @@ static void myDataChannelCallback(int pc, int dc, void *user_ptr) {

451 chan->dc = dc;

452 chan->conn = conn;

453

454 - netdata_spinlock_lock(&conn->channels.spinlock);

454 + spinlock_lock(&conn->channels.spinlock);

455 DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(conn->channels.head, chan, link.prev, link.next);

456 - netdata_spinlock_unlock(&conn->channels.spinlock);

456 + spinlock_unlock(&conn->channels.spinlock);

457

458 rtcSetUserPointer(dc, chan);

459

@@ -486,9 +486,9 @@ static void myDataChannelCallback(int pc, int dc, void *user_ptr) {

486

487 static inline void webrtc_destroy_connection_unsafe(WEBRTC_CONN *conn) {

488 if(conn->state == RTC_CLOSED) {

489 - netdata_spinlock_lock(&conn->channels.spinlock);

489 + spinlock_lock(&conn->channels.spinlock);

490 WEBRTC_DC *chan = conn->channels.head;

491 - netdata_spinlock_unlock(&conn->channels.spinlock);

491 + spinlock_unlock(&conn->channels.spinlock);

492

493 if(!chan) {

494 internal_error(true, "WEBRTC[%d]: destroying connection", conn->pc);

@@ -502,25 +502,25 @@ static inline void webrtc_destroy_connection_unsafe(WEBRTC_CONN *conn) {

502 }

503

504 static void cleanupConnections() {

505 - netdata_spinlock_lock(&webrtc_base.unsafe.spinlock);

505 + spinlock_lock(&webrtc_base.unsafe.spinlock);

506 WEBRTC_CONN *conn = webrtc_base.unsafe.head;

507 while(conn) {

508 WEBRTC_CONN *conn_next = conn->link.next;

509 webrtc_destroy_connection_unsafe(conn);

510 conn = conn_next;

511 }

512 - netdata_spinlock_unlock(&webrtc_base.unsafe.spinlock);

512 + spinlock_unlock(&webrtc_base.unsafe.spinlock);

513 }

514

515 static WEBRTC_CONN *webrtc_create_connection(void) {

516 WEBRTC_CONN *conn = callocz(1, sizeof(WEBRTC_CONN));

517

518 - netdata_spinlock_init(&conn->response.spinlock);

519 - netdata_spinlock_init(&conn->channels.spinlock);

518 + spinlock_init(&conn->response.spinlock);

519 + spinlock_init(&conn->channels.spinlock);

520

521 - netdata_spinlock_lock(&webrtc_base.unsafe.spinlock);

521 + spinlock_lock(&webrtc_base.unsafe.spinlock);

522 DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(webrtc_base.unsafe.head, conn, link.prev, link.next);

523 - netdata_spinlock_unlock(&webrtc_base.unsafe.spinlock);

523 + spinlock_unlock(&webrtc_base.unsafe.spinlock);

524 return conn;

525 }

526

@@ -531,13 +531,13 @@ static void myDescriptionCallback(int pc __maybe_unused, const char *sdp, const

531 internal_fatal(conn->pc != pc, "WEBRTC[%d]: pc mismatch, expected %d, got %d", conn->pc, conn->pc, pc);

532

533 internal_error(true, "WEBRTC[%d]: local description type '%s': %s", conn->pc, type, sdp);

534 - netdata_spinlock_lock(&conn->response.spinlock);

534 + spinlock_lock(&conn->response.spinlock);

535 if(!conn->response.candidates) {

536 buffer_json_member_add_string(conn->response.wb, "sdp", sdp);

537 buffer_json_member_add_string(conn->response.wb, "type", type);

538 conn->response.sdp = true;

539 }

540 - netdata_spinlock_unlock(&conn->response.spinlock);

540 + spinlock_unlock(&conn->response.spinlock);

541

542 conn->local_max_message_size = find_max_message_size_in_sdp(sdp);

543 }

@@ -548,7 +548,7 @@ static void myCandidateCallback(int pc __maybe_unused, const char *cand, const c

548 WEBRTC_CONN *conn = user_ptr;

549 internal_fatal(conn->pc != pc, "WEBRTC[%d]: pc mismatch, expected %d, got %d", conn->pc, conn->pc, pc);

550

551 - netdata_spinlock_lock(&conn->response.spinlock);

551 + spinlock_lock(&conn->response.spinlock);

552 if(!conn->response.candidates) {

553 buffer_json_member_add_array(conn->response.wb, "candidates");

554 conn->response.candidates = true;

@@ -556,7 +556,7 @@ static void myCandidateCallback(int pc __maybe_unused, const char *cand, const c

556

557 internal_error(true, "WEBRTC[%d]: local candidate '%s', mid '%s'", conn->pc, cand, mid);

558 buffer_json_add_array_item_string(conn->response.wb, cand);

559 - netdata_spinlock_unlock(&conn->response.spinlock);

559 + spinlock_unlock(&conn->response.spinlock);

560 }

561

562 static void myStateChangeCallback(int pc __maybe_unused, rtcState state, void *user_ptr) {

@@ -595,9 +595,9 @@ static void myStateChangeCallback(int pc __maybe_unused, rtcState state, void *u

595 case RTC_CLOSED:

596 log_access("WEBRTC[%d]: %d CONNECTION CLOSED", conn->pc, gettid());

597 internal_error(true, "WEBRTC[%d]: closed.", conn->pc);

598 - netdata_spinlock_lock(&webrtc_base.unsafe.spinlock);

598 + spinlock_lock(&webrtc_base.unsafe.spinlock);

599 webrtc_destroy_connection_unsafe(conn);

600 - netdata_spinlock_unlock(&webrtc_base.unsafe.spinlock);

600 + spinlock_unlock(&webrtc_base.unsafe.spinlock);

601 break;

602 }

603 }

web/server/web_client_cache.c

+15 -15

@@ -58,7 +58,7 @@ void web_client_cache_destroy(void) {

59 struct web_client *w, *t;

61 - netdata_spinlock_lock(&web_clients_cache.avail.spinlock);

61 + spinlock_lock(&web_clients_cache.avail.spinlock);

62 w = web_clients_cache.avail.head;

63 while(w) {

64 t = w;

@@ -67,10 +67,10 @@ void web_client_cache_destroy(void) {

67 }

68 web_clients_cache.avail.head = NULL;

69 web_clients_cache.avail.count = 0;

70 - netdata_spinlock_unlock(&web_clients_cache.avail.spinlock);

70 + spinlock_unlock(&web_clients_cache.avail.spinlock);

72 // DO NOT FREE THEM IF THEY ARE USED

73 -// netdata_spinlock_lock(&web_clients_cache.used.spinlock);

73 +// spinlock_lock(&web_clients_cache.used.spinlock);

74 // w = web_clients_cache.used.head;

75 // while(w) {

76 // t = w;

@@ -81,37 +81,37 @@ void web_client_cache_destroy(void) {

81 // web_clients_cache.used.count = 0;

82 // web_clients_cache.used.reused = 0;

83 // web_clients_cache.used.allocated = 0;

84 -// netdata_spinlock_unlock(&web_clients_cache.used.spinlock);

84 +// spinlock_unlock(&web_clients_cache.used.spinlock);

85 }

87 struct web_client *web_client_get_from_cache(void) {

88 - netdata_spinlock_lock(&web_clients_cache.avail.spinlock);

88 + spinlock_lock(&web_clients_cache.avail.spinlock);

89 struct web_client *w = web_clients_cache.avail.head;

90 if(w) {

91 // get it from avail

92 DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(web_clients_cache.avail.head, w, cache.prev, cache.next);

93 web_clients_cache.avail.count--;

94 - netdata_spinlock_unlock(&web_clients_cache.avail.spinlock);

94 + spinlock_unlock(&web_clients_cache.avail.spinlock);

96 web_client_zero(w);

98 - netdata_spinlock_lock(&web_clients_cache.used.spinlock);

98 + spinlock_lock(&web_clients_cache.used.spinlock);

99 web_clients_cache.used.reused++;

100 }

101 else {

102 - netdata_spinlock_unlock(&web_clients_cache.avail.spinlock);

102 + spinlock_unlock(&web_clients_cache.avail.spinlock);

103

104 // allocate it

105 w = web_client_create(&netdata_buffers_statistics.buffers_web);

106

107 - netdata_spinlock_lock(&web_clients_cache.used.spinlock);

107 + spinlock_lock(&web_clients_cache.used.spinlock);

108 web_clients_cache.used.allocated++;

109 }

110

111 // link it to used web clients

112 DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(web_clients_cache.used.head, w, cache.prev, cache.next);

113 web_clients_cache.used.count++;

114 - netdata_spinlock_unlock(&web_clients_cache.used.spinlock);

114 + spinlock_unlock(&web_clients_cache.used.spinlock);

115

116 // initialize it

117 w->use_count++;

@@ -128,14 +128,14 @@ void web_client_release_to_cache(struct web_client *w) {

128 #endif

129

130 // unlink it from the used

131 - netdata_spinlock_lock(&web_clients_cache.used.spinlock);

131 + spinlock_lock(&web_clients_cache.used.spinlock);

132 DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(web_clients_cache.used.head, w, cache.prev, cache.next);

133 ssize_t used_count = (ssize_t)--web_clients_cache.used.count;

134 - netdata_spinlock_unlock(&web_clients_cache.used.spinlock);

134 + spinlock_unlock(&web_clients_cache.used.spinlock);

135

136 - netdata_spinlock_lock(&web_clients_cache.avail.spinlock);

136 + spinlock_lock(&web_clients_cache.avail.spinlock);

137 if(w->use_count > 100 || (used_count > 0 && web_clients_cache.avail.count >= 2 * (size_t)used_count) || (used_count <= 10 && web_clients_cache.avail.count >= 20)) {

138 - netdata_spinlock_unlock(&web_clients_cache.avail.spinlock);

138 + spinlock_unlock(&web_clients_cache.avail.spinlock);

139

140 // we have too many of them - free it

141 web_client_free(w);

@@ -144,6 +144,6 @@ void web_client_release_to_cache(struct web_client *w) {

144 // link it to the avail

145 DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(web_clients_cache.avail.head, w, cache.prev, cache.next);

146 web_clients_cache.avail.count++;

147 - netdata_spinlock_unlock(&web_clients_cache.avail.spinlock);

147 + spinlock_unlock(&web_clients_cache.avail.spinlock);

148 }

149 }