Detect stored metric size by page type (#13334)
* Report unknown page only once Get metric storage size by the page type Verify validity of the page and skip problematic ones * Change PAGE_SIZE to PAGE_POINT_SIZE_BYTES * Add bitmap256 and unittests * Fix unit test tier_page_type array page_type_size arrays * Add another counter to not rely on uint8_t overflow to stop the test loop
Stelios Fragkakis committed
Jul 11, 2022 at 20:40 UTC
87e9700b2fb93731154eb59cbb53c69e55fbcc6b
11 files changed
+184
-27
daemon/main.c
+2
-1
@@ -973,7 +973,8 @@ int main(int argc, char **argv) {
973
return 1;
974
if (unit_test_str2ld())
975
return 1;
976
-
976
+ if (unit_test_bitmap256())
977
+ return 1;
978
// No call to load the config file on this code-path
979
post_conf_load(&user);
980
get_netdata_configured_variables();
daemon/unit_test.c
+107
@@ -1606,6 +1606,113 @@ error:
1606
return 1;
1607
}
1608
1609
+int unit_test_bitmap256(void) {
1610
+ fprintf(stderr, "%s() running...\n", __FUNCTION__ );
1611
+
1612
+ BITMAP256 test_bitmap = {0};
1613
+
1614
+ bitmap256_set_bit(&test_bitmap, 0, 1);
1615
+ bitmap256_set_bit(&test_bitmap, 64, 1);
1616
+ bitmap256_set_bit(&test_bitmap, 128, 1);
1617
+ bitmap256_set_bit(&test_bitmap, 192, 1);
1618
+ if (test_bitmap.data[0] == 1)
1619
+ fprintf(stderr, "%s() INDEX 1 is OK\n", __FUNCTION__ );
1620
+ if (test_bitmap.data[1] == 1)
1621
+ fprintf(stderr, "%s() INDEX 65 is OK\n", __FUNCTION__ );
1622
+ if (test_bitmap.data[2] == 1)
1623
+ fprintf(stderr, "%s() INDEX 129 is OK\n", __FUNCTION__ );
1624
+ if (test_bitmap.data[3] == 1)
1625
+ fprintf(stderr, "%s() INDEX 192 is OK\n", __FUNCTION__ );
1626
+
1627
+ uint8_t i=0;
1628
+ int j = 0;
1629
+ do {
1630
+ bitmap256_set_bit(&test_bitmap, i++, 1);
1631
+ j++;
1632
+ } while (j < 256);
1633
+
1634
+ if (test_bitmap.data[0] == 0xffffffff)
1635
+ fprintf(stderr, "%s() INDEX 0 is fully set OK\n", __FUNCTION__);
1636
+ if (test_bitmap.data[1] == 0xffffffff)
1637
+ fprintf(stderr, "%s() INDEX 1 is fully set OK\n", __FUNCTION__);
1638
+ if (test_bitmap.data[2] == 0xffffffff)
1639
+ fprintf(stderr, "%s() INDEX 2 is fully set OK\n", __FUNCTION__);
1640
+ if (test_bitmap.data[3] == 0xffffffff)
1641
+ fprintf(stderr, "%s() INDEX 3 is fully set OK\n", __FUNCTION__);
1642
+
1643
+ i = 0;
1644
+ j = 0;
1645
+ do {
1646
+ bitmap256_set_bit(&test_bitmap, i++, 0);
1647
+ j++;
1648
+ } while (j < 256);
1649
+
1650
+ if (test_bitmap.data[0] == 0)
1651
+ fprintf(stderr, "%s() INDEX 0 is reset OK\n", __FUNCTION__);
1652
+ else {
1653
+ fprintf(stderr, "%s() INDEX 0 is not reset FAILED\n", __FUNCTION__);
1654
+ return 1;
1655
+ }
1656
+ if (test_bitmap.data[1] == 0)
1657
+ fprintf(stderr, "%s() INDEX 1 is reset OK\n", __FUNCTION__);
1658
+ else {
1659
+ fprintf(stderr, "%s() INDEX 1 is not reset FAILED\n", __FUNCTION__);
1660
+ return 1;
1661
+ }
1662
+
1663
+ if (test_bitmap.data[2] == 0)
1664
+ fprintf(stderr, "%s() INDEX 2 is reset OK\n", __FUNCTION__);
1665
+ else {
1666
+ fprintf(stderr, "%s() INDEX 2 is not reset FAILED\n", __FUNCTION__);
1667
+ return 1;
1668
+ }
1669
+
1670
+ if (test_bitmap.data[3] == 0)
1671
+ fprintf(stderr, "%s() INDEX 3 is reset OK\n", __FUNCTION__);
1672
+ else {
1673
+ fprintf(stderr, "%s() INDEX 3 is not reset FAILED\n", __FUNCTION__);
1674
+ return 1;
1675
+ }
1676
+
1677
+ i=0;
1678
+ j = 0;
1679
+ do {
1680
+ bitmap256_set_bit(&test_bitmap, i, 1);
1681
+ i += 4;
1682
+ j += 4;
1683
+ } while (j < 256);
1684
+
1685
+ if (test_bitmap.data[0] == 0x11111111)
1686
+ fprintf(stderr, "%s() INDEX 0 is 0x11111111 set OK\n", __FUNCTION__);
1687
+ else {
1688
+ fprintf(stderr, "%s() INDEX 0 is %lx expected 0x11111111\n", __FUNCTION__, test_bitmap.data[0]);
1689
+ return 1;
1690
+ }
1691
+
1692
+ if (test_bitmap.data[1] == 0x11111111)
1693
+ fprintf(stderr, "%s() INDEX 1 is 0x11111111 set OK\n", __FUNCTION__);
1694
+ else {
1695
+ fprintf(stderr, "%s() INDEX 1 is %lx expected 0x11111111\n", __FUNCTION__, test_bitmap.data[1]);
1696
+ return 1;
1697
+ }
1698
+
1699
+ if (test_bitmap.data[2] == 0x11111111)
1700
+ fprintf(stderr, "%s() INDEX 2 is 0x11111111 set OK\n", __FUNCTION__);
1701
+ else {
1702
+ fprintf(stderr, "%s() INDEX 2 is %lx expected 0x11111111\n", __FUNCTION__, test_bitmap.data[2]);
1703
+ return 1;
1704
+ }
1705
+
1706
+ if (test_bitmap.data[3] == 0x11111111)
1707
+ fprintf(stderr, "%s() INDEX 3 is 0x11111111 set OK\n", __FUNCTION__);
1708
+ else {
1709
+ fprintf(stderr, "%s() INDEX 3 is %lx expected 0x11111111\n", __FUNCTION__, test_bitmap.data[3]);
1710
+ return 1;
1711
+ }
1712
+
1713
+ fprintf(stderr, "%s() tests passed\n", __FUNCTION__);
1714
+ return 0;
1715
+}
1716
1717
#ifdef ENABLE_DBENGINE
1718
static inline void rrddim_set_by_pointer_fake_time(RRDDIM *rd, collected_number value, time_t now)
daemon/unit_test.h
+1
@@ -10,6 +10,7 @@ extern int unit_test_str2ld(void);
10
extern int unit_test_buffer(void);
11
extern int unit_test_static_threads(void);
12
extern int test_sqlite(void);
13
+extern int unit_test_bitmap256(void);
14
#ifdef ENABLE_DBENGINE
15
extern int test_dbengine(void);
16
extern void generate_dbengine_dataset(unsigned history_seconds);
database/engine/journalfile.c
+26
-5
@@ -275,6 +275,7 @@ static int check_journal_file_superblock(uv_file file)
275
static void restore_extent_metadata(struct rrdengine_instance *ctx, struct rrdengine_journalfile *journalfile,
276
void *buf, unsigned max_size)
277
{
278
+ static BITMAP256 page_error_map;
279
struct page_cache *pg_cache = &ctx->pg_cache;
280
unsigned i, count, payload_length, descr_size, valid_pages;
281
struct rrdeng_page_descr *descr;
@@ -301,11 +302,31 @@ static void restore_extent_metadata(struct rrdengine_instance *ctx, struct rrden
302
uuid_t *temp_id;
303
Pvoid_t *PValue;
304
struct pg_cache_page_index *page_index = NULL;
305
+ uint8_t page_type = jf_metric_data->descr[i].type;
306
305
- if (jf_metric_data->descr[i].type > PAGE_TYPE_MAX) {
306
- error("Unknown page type %d encountered.", jf_metric_data->descr[i].type );
307
+ if (page_type > PAGE_TYPE_MAX) {
308
+ if (!bitmap256_get_bit(&page_error_map, page_type)) {
309
+ error("Unknown page type %d encountered.", page_type);
310
+ bitmap256_set_bit(&page_error_map, page_type, 1);
311
+ }
312
continue;
313
}
314
+ uint64_t start_time = jf_metric_data->descr[i].start_time;
315
+ uint64_t end_time = jf_metric_data->descr[i].end_time;
316
+
317
+ if (unlikely(start_time > end_time)) {
318
+ error("Invalid page encountered, start time %lu > end time %lu", start_time , end_time );
319
+ continue;
320
+ }
321
+
322
+ if (unlikely(start_time == end_time)) {
323
+ size_t entries = jf_metric_data->descr[i].page_length / page_type_size[page_type];
324
+ if (unlikely(entries > 1)) {
325
+ error("Invalid page encountered, start time %lu = end time but %zu entries were found", start_time, entries);
326
+ continue;
327
+ }
328
+ }
329
+
330
temp_id = (uuid_t *)jf_metric_data->descr[i].uuid;
331
332
uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
@@ -327,11 +348,11 @@ static void restore_extent_metadata(struct rrdengine_instance *ctx, struct rrden
348
349
descr = pg_cache_create_descr();
350
descr->page_length = jf_metric_data->descr[i].page_length;
330
- descr->start_time = jf_metric_data->descr[i].start_time;
331
- descr->end_time = jf_metric_data->descr[i].end_time;
351
+ descr->start_time = start_time;
352
+ descr->end_time = end_time;
353
descr->id = &page_index->id;
354
descr->extent = extent;
334
- descr->type = jf_metric_data->descr[i].type;
355
+ descr->type = page_type;
356
extent->pages[valid_pages++] = descr;
357
pg_cache_insert(ctx, page_index, descr);
358
}
database/engine/pagecache.c
+2
-2
@@ -1334,7 +1334,7 @@ void free_page_cache(struct rrdengine_instance *ctx)
1334
pages_size_per_type[descr->type] += descr->page_length;
1335
pages_count_per_type[descr->type]++;
1336
1337
- size_t points_in_page = (descr->page_length / ctx->storage_size);
1337
+ size_t points_in_page = (descr->page_length / PAGE_POINT_SIZE_BYTES(descr));
1338
size_t page_duration = ((descr->end_time - descr->start_time) / USEC_PER_SEC);
1339
size_t update_every = (page_duration == 0) ? 1 : page_duration / (points_in_page - 1);
1340
@@ -1351,7 +1351,7 @@ void free_page_cache(struct rrdengine_instance *ctx)
1351
page_duration = update_every * points_in_page;
1352
metric_duration += page_duration;
1353
seconds_in_db += page_duration;
1354
- points_in_db += descr->page_length / ctx->storage_size;
1354
+ points_in_db += descr->page_length / PAGE_POINT_SIZE_BYTES(descr);
1355
}
1356
else
1357
metric_single_point_pages++;
database/engine/rrddiskprotocol.h
+1
-1
@@ -36,7 +36,7 @@ struct rrdeng_df_sb {
36
*/
37
#define PAGE_METRICS (0)
38
#define PAGE_TIER (1)
39
-#define PAGE_TYPE_MAX (1)
39
+#define PAGE_TYPE_MAX 1 // Maximum page type (inclusive)
40
41
/*
42
* Data file page descriptor
database/engine/rrdengine.h
+1
-2
@@ -243,14 +243,13 @@ struct rrdengine_instance {
243
uint64_t disk_space;
244
uint64_t max_disk_space;
245
int tier;
246
- size_t storage_size;
246
unsigned last_fileno; /* newest index of datafile and journalfile */
247
unsigned long max_cache_pages;
248
unsigned long cache_pages_low_watermark;
249
unsigned long metric_API_max_producers;
250
251
uint8_t quiesce; /* set to SET_QUIESCE before shutdown of the engine */
253
- uint8_t page_type; /* set to SET_QUIESCE before shutdown of the engine */
252
+ uint8_t page_type; /* Default page type for this context */
253
254
struct rrdengine_statistics stats;
255
};
database/engine/rrdengineapi.c
+19
-16
@@ -11,6 +11,12 @@ struct rrdengine_instance multidb_ctx_storage_tier4;
11
#error RRD_STORAGE_TIERS is not 5 - you need to add allocations here
12
#endif
13
struct rrdengine_instance *multidb_ctx[RRD_STORAGE_TIERS];
14
+uint8_t tier_page_type[RRD_STORAGE_TIERS] = {PAGE_METRICS, PAGE_TIER, PAGE_TIER, PAGE_TIER, PAGE_TIER};
15
+
16
+#if PAGE_TYPE_MAX != 1
17
+#error PAGE_TYPE_MAX is not 1 - you need to add allocations here
18
+#endif
19
+size_t page_type_size[256] = {sizeof(storage_number), sizeof(storage_number_tier1_t)};
20
21
__attribute__((constructor)) void initialize_multidb_ctx(void) {
22
multidb_ctx[0] = &multidb_ctx_storage_tier0;
@@ -169,14 +175,14 @@ STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *db_metri
175
}
176
177
/* The page must be populated and referenced */
172
-static int page_has_only_empty_metrics(struct rrdeng_page_descr *descr, size_t storage_size)
178
+static int page_has_only_empty_metrics(struct rrdeng_page_descr *descr)
179
{
180
unsigned i;
181
uint8_t has_only_empty_metrics = 1;
182
storage_number *page;
183
184
page = descr->pg_cache_descr->page;
179
- for (i = 0 ; i < descr->page_length / storage_size; ++i) {
185
+ for (i = 0 ; i < descr->page_length / PAGE_POINT_SIZE_BYTES(descr); ++i) {
186
if (SN_EMPTY_SLOT != page[i]) {
187
has_only_empty_metrics = 0;
188
break;
@@ -199,7 +205,7 @@ void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *collection_h
205
206
rrd_stat_atomic_add(&ctx->stats.metric_API_producers, -1);
207
202
- page_is_empty = page_has_only_empty_metrics(descr, ctx->storage_size);
208
+ page_is_empty = page_has_only_empty_metrics(descr);
209
if (page_is_empty) {
210
debug(D_RRDENGINE, "Page has empty metrics only, deleting:");
211
if (unlikely(debug_flags & D_RRDENGINE))
@@ -232,7 +238,6 @@ void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle, usec_t
238
239
void *page;
240
uint8_t must_flush_unaligned_page = 0, perfect_page_alignment = 0;
235
- size_t storage_size = ctx->storage_size;
241
242
if (descr) {
243
/* Make alignment decisions */
@@ -242,7 +247,7 @@ void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle, usec_t
247
perfect_page_alignment = 1;
248
}
249
/* is the metric far enough out of alignment with the others? */
245
- if (unlikely(descr->page_length + storage_size < rd->rrdset->rrddim_page_alignment)) {
250
+ if (unlikely(descr->page_length + PAGE_POINT_SIZE_BYTES(descr) < rd->rrdset->rrddim_page_alignment)) {
251
handle->unaligned_page = 1;
252
debug(D_RRDENGINE, "Metric page is not aligned with chart:");
253
if (unlikely(debug_flags & D_RRDENGINE))
@@ -250,14 +255,14 @@ void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle, usec_t
255
}
256
if (unlikely(handle->unaligned_page &&
257
/* did the other metrics change page? */
253
- rd->rrdset->rrddim_page_alignment <= storage_size)) {
258
+ rd->rrdset->rrddim_page_alignment <= PAGE_POINT_SIZE_BYTES(descr))) {
259
debug(D_RRDENGINE, "Flushing unaligned metric page.");
260
must_flush_unaligned_page = 1;
261
handle->unaligned_page = 0;
262
}
263
}
264
if (unlikely(NULL == descr ||
260
- descr->page_length + storage_size > RRDENG_BLOCK_SIZE ||
265
+ descr->page_length + PAGE_POINT_SIZE_BYTES(descr) > RRDENG_BLOCK_SIZE ||
266
must_flush_unaligned_page)) {
267
rrdeng_store_metric_flush_current_page(collection_handle);
268
@@ -278,7 +283,7 @@ void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle, usec_t
283
284
switch (descr->type) {
285
case PAGE_METRICS: {
281
- ((storage_number *)page)[descr->page_length / storage_size] = pack_storage_number(n, flags);
286
+ ((storage_number *)page)[descr->page_length / PAGE_POINT_SIZE_BYTES(descr)] = pack_storage_number(n, flags);
287
}
288
break;
289
@@ -289,7 +294,7 @@ void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle, usec_t
294
number_tier1.max_value = (float)max_value;
295
number_tier1.anomaly_count = anomaly_count;
296
number_tier1.count = count;
292
- ((storage_number_tier1_t *)page)[descr->page_length / storage_size] = number_tier1;
297
+ ((storage_number_tier1_t *)page)[descr->page_length / PAGE_POINT_SIZE_BYTES(descr)] = number_tier1;
298
}
299
break;
300
@@ -303,7 +308,7 @@ void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle, usec_t
308
break;
309
}
310
306
- pg_cache_atomic_set_pg_info(descr, point_in_time, descr->page_length + storage_size);
311
+ pg_cache_atomic_set_pg_info(descr, point_in_time, descr->page_length + PAGE_POINT_SIZE_BYTES(descr));
312
313
if (perfect_page_alignment)
314
rd->rrdset->rrddim_page_alignment = descr->page_length;
@@ -441,7 +446,7 @@ static int rrdeng_load_page_next(struct rrddim_query_handle *rrdimm_handle) {
446
447
if (unlikely(descr->start_time != page_end_time && next_page_time > descr->start_time)) {
448
// we're in the middle of the page somewhere
444
- unsigned entries = page_length / ctx->storage_size;
449
+ unsigned entries = page_length / PAGE_POINT_SIZE_BYTES(descr);
450
position = ((uint64_t)(next_page_time - descr->start_time)) * (entries - 1) /
451
(page_end_time - descr->start_time);
452
}
@@ -451,7 +456,7 @@ static int rrdeng_load_page_next(struct rrddim_query_handle *rrdimm_handle) {
456
handle->page_end_time = page_end_time;
457
handle->page_length = page_length;
458
handle->page = descr->pg_cache_descr->page;
454
- usec_t entries = handle->entries = page_length / ctx->storage_size;
459
+ usec_t entries = handle->entries = page_length / PAGE_POINT_SIZE_BYTES(descr);
460
if (likely(entries > 1))
461
handle->dt = (page_end_time - descr->start_time) / (entries - 1);
462
else {
@@ -815,14 +820,12 @@ int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_p
820
if(NULL == ctxp) {
821
ctx = multidb_ctx[tier];
822
memset(ctx, 0, sizeof(*ctx));
818
- ctx->storage_size = (tier == 0) ? sizeof(storage_number) : sizeof(storage_number_tier1_t);
819
- ctx->tier = tier;
820
- ctx->page_type = !tier ? PAGE_METRICS : PAGE_TIER; // TODO: In the future it can be different page type per tier
823
}
824
else {
825
*ctxp = ctx = callocz(1, sizeof(*ctx));
824
- ctx->storage_size = sizeof(storage_number);
826
}
827
+ ctx->tier = tier;
828
+ ctx->page_type = tier_page_type[tier];
829
ctx->global_compress_alg = RRD_LZ4;
830
if (page_cache_mb < RRDENG_MIN_PAGE_CACHE_SIZE_MB)
831
page_cache_mb = RRDENG_MIN_PAGE_CACHE_SIZE_MB;
database/engine/rrdengineapi.h
+3
@@ -20,6 +20,9 @@ extern int default_rrdeng_disk_quota_mb;
20
extern int default_multidb_disk_quota_mb;
21
extern uint8_t rrdeng_drop_metrics_under_page_cache_pressure;
22
extern struct rrdengine_instance *multidb_ctx[RRD_STORAGE_TIERS];
23
+extern size_t page_type_size[];
24
+
25
+#define PAGE_POINT_SIZE_BYTES(x) page_type_size[(x)->type]
26
27
struct rrdeng_region_info {
28
time_t start_time;
libnetdata/libnetdata.c
+16
@@ -1534,3 +1534,19 @@ char *find_and_replace(const char *src, const char *find, const char *replace, c
1534
1535
return value;
1536
}
1537
+
1538
+
1539
+bool bitmap256_get_bit(BITMAP256 *ptr, uint8_t idx) {
1540
+ if (unlikely(!ptr))
1541
+ return false;
1542
+ return (ptr->data[idx / 64] & (1 << (idx % 64)));
1543
+}
1544
+
1545
+void bitmap256_set_bit(BITMAP256 *ptr, uint8_t idx, bool value) {
1546
+ if (unlikely(!ptr))
1547
+ return;
1548
+ if (likely(value))
1549
+ ptr->data[idx / 64] |= (1U << (idx % 64));
1550
+ else
1551
+ ptr->data[idx / 64] &= ~(1U << (idx % 64));
1552
+}
libnetdata/libnetdata.h
+6
@@ -311,6 +311,12 @@ extern char *find_and_replace(const char *src, const char *find, const char *rep
311
// Taken from linux kernel
312
#define BUILD_BUG_ON(condition) ((void)sizeof(char[1 - 2*!!(condition)]))
313
314
+typedef struct bitmap256 {
315
+ uint64_t data[4];
316
+} BITMAP256;
317
+
318
+extern bool bitmap256_get_bit(BITMAP256 *ptr, uint8_t idx);
319
+extern void bitmap256_set_bit(BITMAP256 *ptr, uint8_t idx, bool value);
320
321
extern void netdata_cleanup_and_exit(int ret) NORETURN;
322
extern void send_statistics(const char *action, const char *action_result, const char *action_data);