Update journal v2 (#14750)
* Add update every in the metric index (new v2 version) Switch to using memcmp instead of uuid_compare to build and search v2 index files * Remove chart label cleanup during startup
Stelios Fragkakis committed
Mar 21, 2023 at 21:28 UTC
757c01aacfbe8cae892935a262f180c52355b874
4 files changed
+19
-28
database/engine/journalfile.c
+11
-20
@@ -894,16 +894,6 @@ static int journalfile_v2_validate(void *data_start, size_t journal_v2_file_size
894
return 0;
895
}
896
897
-static inline time_t get_metric_latest_update_every(struct journal_page_header *metric_list_header)
898
-{
899
- struct journal_page_list *metric_page =
900
- (struct journal_page_list *)((uint8_t *)metric_list_header + sizeof(*metric_list_header));
901
- uint32_t entries = metric_list_header->entries;
902
- if (unlikely(!entries))
903
- return 0;
904
- return (time_t)metric_page[entries - 1].update_every_s;
905
-}
906
-
897
void journalfile_v2_populate_retention_to_mrg(struct rrdengine_instance *ctx, struct rrdengine_journalfile *journalfile) {
898
usec_t started_ut = now_monotonic_usec();
899
@@ -921,15 +911,10 @@ void journalfile_v2_populate_retention_to_mrg(struct rrdengine_instance *ctx, st
911
for (size_t i=0; i < entries; i++) {
912
time_t start_time_s = header_start_time_s + metric->delta_start_s;
913
time_t end_time_s = header_start_time_s + metric->delta_end_s;
924
- time_t update_every_s = get_metric_latest_update_every((struct journal_page_header *) (data_start + metric->page_offset));
914
+
915
update_metric_retention_and_granularity_by_uuid(
926
- ctx, &metric->uuid, start_time_s, end_time_s, update_every_s, now_s);
916
+ ctx, &metric->uuid, start_time_s, end_time_s, (time_t) metric->update_every_s, now_s);
917
928
-#ifdef NETDATA_INTERNAL_CHECKS
929
- struct journal_page_header *metric_list_header = (void *) (data_start + metric->page_offset);
930
- fatal_assert(uuid_compare(metric_list_header->uuid, metric->uuid) == 0);
931
- fatal_assert(metric->entries == metric_list_header->entries);
932
-#endif
918
metric++;
919
}
920
@@ -1048,7 +1033,7 @@ static int journalfile_metric_compare (const void *item1, const void *item2)
1033
const struct jv2_metrics_info *metric1 = ((struct journal_metric_list_to_sort *) item1)->metric_info;
1034
const struct jv2_metrics_info *metric2 = ((struct journal_metric_list_to_sort *) item2)->metric_info;
1035
1051
- return uuid_compare(*(metric1->uuid), *(metric2->uuid));
1036
+ return memcmp(metric1->uuid, metric2->uuid, sizeof(uuid_t));
1037
}
1038
1039
@@ -1094,6 +1079,7 @@ void *journalfile_v2_write_metric_page(struct journal_v2_header *j2_header, void
1079
metric->page_offset = pages_offset;
1080
metric->delta_start_s = (uint32_t)(metric_info->first_time_s - (time_t)(j2_header->start_time_ut / USEC_PER_SEC));
1081
metric->delta_end_s = (uint32_t)(metric_info->last_time_s - (time_t)(j2_header->start_time_ut / USEC_PER_SEC));
1082
+ metric->update_every_s = 0;
1083
1084
return ++metric;
1085
}
@@ -1146,7 +1132,8 @@ void *journalfile_v2_write_data_page(struct journal_v2_header *j2_header, void *
1132
}
1133
1134
// Must be recorded in metric_info->entries
1149
-void *journalfile_v2_write_descriptors(struct journal_v2_header *j2_header, void *data, struct jv2_metrics_info *metric_info)
1135
+static void *journalfile_v2_write_descriptors(struct journal_v2_header *j2_header, void *data, struct jv2_metrics_info *metric_info,
1136
+ struct journal_metric_list *current_metric)
1137
{
1138
Pvoid_t *PValue;
1139
@@ -1158,13 +1145,16 @@ void *journalfile_v2_write_descriptors(struct journal_v2_header *j2_header, void
1145
Word_t index_time = 0;
1146
bool first = true;
1147
struct jv2_page_info *page_info;
1148
+ uint32_t update_every_s = 0;
1149
while ((PValue = JudyLFirstThenNext(JudyL_array, &index_time, &first))) {
1150
page_info = *PValue;
1151
// Write one descriptor and return the next data page location
1152
data_page = journalfile_v2_write_data_page(j2_header, (void *) data_page, page_info);
1153
+ update_every_s = (uint32_t) page_info->update_every_s;
1154
if (NULL == data_page)
1155
break;
1156
}
1157
+ current_metric->update_every_s = update_every_s;
1158
return data_page;
1159
}
1160
@@ -1301,6 +1291,7 @@ void journalfile_migrate_to_v2_callback(Word_t section, unsigned datafile_fileno
1291
// Calculate current UUID offset from start of file. We will store this in the data page header
1292
uint32_t uuid_offset = data - data_start;
1293
1294
+ struct journal_metric_list *current_metric = (void *) data;
1295
// Write the UUID we are processing
1296
data = (void *) journalfile_v2_write_metric_page(&j2_header, data, metric_info, pages_offset);
1297
if (unlikely(!data))
@@ -1318,7 +1309,7 @@ void journalfile_migrate_to_v2_callback(Word_t section, unsigned datafile_fileno
1309
uuid_offset);
1310
1311
// Start writing descr @ time
1321
- void *page_trailer = journalfile_v2_write_descriptors(&j2_header, metric_page, metric_info);
1312
+ void *page_trailer = journalfile_v2_write_descriptors(&j2_header, metric_page, metric_info, current_metric);
1313
if (unlikely(!page_trailer))
1314
break;
1315
database/engine/journalfile.h
+7
-6
@@ -59,9 +59,9 @@ static inline uint64_t journalfile_current_size(struct rrdengine_journalfile *jo
59
60
// Journal v2 structures
61
62
-#define JOURVAL_V2_MAGIC (0x01221019)
63
-#define JOURVAL_V2_REBUILD_MAGIC (0x00221019)
64
-#define JOURVAL_V2_SKIP_MAGIC (0x02221019)
62
+#define JOURVAL_V2_MAGIC (0x01230317)
63
+#define JOURVAL_V2_REBUILD_MAGIC (0x00230317)
64
+#define JOURVAL_V2_SKIP_MAGIC (0x02230317)
65
66
struct journal_v2_block_trailer {
67
union {
@@ -93,13 +93,14 @@ struct journal_page_list {
93
};
94
95
// UUID_LIST
96
-// 32 bytes
96
+// 36 bytes
97
struct journal_metric_list {
98
uuid_t uuid;
99
- uint32_t entries; // Number of entries
100
- uint32_t page_offset; // OFFSET that contains entries * struct( journal_page_list )
99
+ uint32_t entries; // Number of entries
100
+ uint32_t page_offset; // OFFSET that contains entries * struct( journal_page_list )
101
uint32_t delta_start_s; // Min time of metric
102
uint32_t delta_end_s; // Max time of metric (to be used to populate page_index)
103
+ uint32_t update_every_s; // Last update every for this metric in this journal (last page collected)
104
};
105
106
// 16 bytes
database/engine/pagecache.c
+1
-1
@@ -101,7 +101,7 @@ inline TIME_RANGE_COMPARE is_page_in_time_range(time_t page_first_time_s, time_t
101
102
static int journal_metric_uuid_compare(const void *key, const void *metric)
103
{
104
- return uuid_compare(*(uuid_t *) key, ((struct journal_metric_list *) metric)->uuid);
104
+ return memcmp(key, &(((struct journal_metric_list *) metric)->uuid), sizeof(uuid_t));
105
}
106
107
static inline struct page_details *pdc_find_page_for_time(
database/sqlite/sqlite_functions.c
-1
@@ -49,7 +49,6 @@ const char *database_config[] = {
49
const char *database_cleanup[] = {
50
"DELETE FROM chart WHERE chart_id NOT IN (SELECT chart_id FROM dimension);",
51
"DELETE FROM host WHERE host_id NOT IN (SELECT host_id FROM chart);",
52
- "DELETE FROM chart_label WHERE chart_id NOT IN (SELECT chart_id FROM chart);",
52
"DELETE FROM node_instance WHERE host_id NOT IN (SELECT host_id FROM host);",
53
"DELETE FROM host_info WHERE host_id NOT IN (SELECT host_id FROM host);",
54
"DELETE FROM host_label WHERE host_id NOT IN (SELECT host_id FROM host);",