@cryptotaxi247 / netdata-1 / commits / 81171c91b

Document journal v2 index file format. (#19701)

vkalintiris committed Feb 24, 2025 at 17:30 UTC 81171c91bc992befd0f8d33861010d07dbb353c4
1 file changed +163 -49
src/database/engine/journalfile.h
+163 -49
@@ -75,67 +75,181 @@ struct journal_v2_block_trailer {
75 };
76 };
77
78 -// Journal V2
79 -// 28 bytes
78 +/*
79 + * Journal File V2 Format
80 + *
81 + * File Layout:
82 + * +------------------------------------------------+
83 + * | HEADER SECTION |
84 + * | +--------------------------------------------+
85 + * | | Header (72 bytes) |
86 + * | | - magic, times, counts, offsets |
87 + * | +--------------------------------------------+
88 + * | | Padding (to 4096 bytes) |
89 + * +------------------------------------------------+
90 + * | EXTENT SECTION |
91 + * | +--------------------------------------------+
92 + * | | Extent Items |
93 + * | | - Array of Extent Entries (16 bytes each)|
94 + * | +--------------------------------------------+
95 + * | | Extent Section Trailer (4 bytes CRC) |
96 + * +------------------------------------------------+
97 + * | METRICS SECTION |
98 + * | +--------------------------------------------+
99 + * | | Metric List |
100 + * | | - Array of Metric Entries (36 bytes each)|
101 + * | +--------------------------------------------+
102 + * | | Metric List Trailer (4 bytes CRC) |
103 + * +------------------------------------------------+
104 + * | PAGE DATA SECTION |
105 + * | +--------------------------------------------+
106 + * | | For each metric: |
107 + * | | - Page Header (28 bytes) |
108 + * | | - Page Entries (20 bytes each) |
109 + * | | - Page Trailer (4 bytes CRC) |
110 + * +------------------------------------------------+
111 + * | FILE TRAILER |
112 + * | - File CRC (4 bytes) |
113 + * +------------------------------------------------+
114 + *
115 + * Data Integrity:
116 + * - All sections have CRC32 checksums
117 + * - The file header CRC is stored in the file trailer
118 + * - Each section (extent list, metric list) has its own trailer with CRC
119 + * - Each page header includes a CRC of its content
120 + *
121 + * Time Representation:
122 + * - File-level times are stored in microseconds (start_time_ut, end_time_ut)
123 + * - Page-level times are stored as deltas in seconds relative to the journal start time
124 + */
125 +
126 +// Journal v2 header (72 bytes)
127 +struct journal_v2_header {
128 + // File type identifier
129 + // - 0x01230317: Normal journal file
130 + // - 0x00230317: File needs rebuild
131 + // - 0x02230317: File should be skipped
132 + uint32_t magic;
133 +
134 + // --- implicit padding of 4-bytes because the struct is not packed ---
135 +
136 + // Mininimum start time in microseconds
137 + usec_t start_time_ut;
138 + // Maximum end time in microseconds
139 + usec_t end_time_ut;
140 +
141 + // Number of extents
142 + uint32_t extent_count;
143 + // Offset to extents section
144 + uint32_t extent_offset;
145 +
146 + // Number of metrics
147 + uint32_t metric_count;
148 + // Offset to metrics section
149 + uint32_t metric_offset;
150 +
151 + // Total count of pages
152 + uint32_t page_count;
153 + // Offset to page data section
154 + uint32_t page_offset;
155 +
156 + // Offset to extents section CRC
157 + uint32_t extent_trailer_offset;
158 + // Offset to metrics section CRC
159 + uint32_t metric_trailer_offset;
160 +
161 + // Size of original journal file
162 + uint32_t journal_v1_file_size;
163 + // Total file size
164 + uint32_t journal_v2_file_size;
165 +
166 + // Pointer used only when writing to build up the memory-mapped file.
167 + void *data;
168 +};
169 +
170 +// Reserve the first 4 KiB for the journal v2 header
171 +#define JOURNAL_V2_HEADER_PADDING_SZ (RRDENG_BLOCK_SIZE - (sizeof(struct journal_v2_header)))
172 +
173 +// Extent section item (16 bytes)
174 +struct journal_extent_list {
175 + // Extent offset in datafile
176 + uint64_t datafile_offset;
177 +
178 + // Size of the extent
179 + uint32_t datafile_size;
180 +
181 + // Index of the data file
182 + uint16_t file_index;
183 +
184 + // Number of pages in the extent (not all are necesssarily valid)
185 + uint8_t pages;
186 +
187 + // --- implicit padding of 1-byte because the struct is not packed ---
188 +};
189 +
190 +// Metric section item (36 bytes)
191 +struct journal_metric_list {
192 + // Unique identifier of the metric
193 + nd_uuid_t uuid;
194 +
195 + // Number of pages for this metric
196 + uint32_t entries;
197 +
198 + // Offset to the page data section
199 + // Points to: journal_page_header + (entries * journal_page_list)
200 + uint32_t page_offset;
201 +
202 + // Start time relative to journal start
203 + uint32_t delta_start_s;
204 +
205 + // End time relative to journal start
206 + uint32_t delta_end_s;
207 +
208 + // Last update every for this metric in this journal (last page collected)
209 + uint32_t update_every_s;
210 +};
211 +
212 +// Page section item header (28 bytes)
213 struct journal_page_header {
214 + // CRC32 of the header
215 union {
82 - uint8_t checksum[CHECKSUM_SZ]; // CRC check
216 + uint8_t checksum[CHECKSUM_SZ];
217 uint32_t crc;
218 };
85 - uint32_t uuid_offset; // Points back to the UUID list which should point here (UUIDs should much)
86 - uint32_t entries; // Entries
87 - nd_uuid_t uuid; // Which UUID this is
219 +
220 + // Offset to corresponding metric in metric list
221 + uint32_t uuid_offset;
222 +
223 + // Number of page items that follow
224 + uint32_t entries;
225 +
226 + // UUID of the metric
227 + nd_uuid_t uuid;
228 };
229
90 -// 20 bytes
230 +// Page section item (20 bytes)
231 struct journal_page_list {
92 - uint32_t delta_start_s; // relative to the start time of journal
93 - uint32_t delta_end_s; // relative to delta_start
94 - uint32_t extent_index; // Index to the extent (extent list) (bytes from BASE)
232 + // Start time relative to journal start
233 + uint32_t delta_start_s;
234 +
235 + // End time relative to journal start
236 + uint32_t delta_end_s;
237 +
238 + // Offset into extent section
239 + uint32_t extent_index;
240 +
241 + // Update frequency
242 uint32_t update_every_s;
96 - uint16_t page_length;
97 - uint8_t type;
98 -};
243
100 -// UUID_LIST
101 -// 36 bytes
102 -struct journal_metric_list {
103 - nd_uuid_t uuid;
104 - uint32_t entries; // Number of entries
105 - uint32_t page_offset; // OFFSET that contains entries * struct( journal_page_list )
106 - uint32_t delta_start_s; // Min time of metric
107 - uint32_t delta_end_s; // Max time of metric (to be used to populate page_index)
108 - uint32_t update_every_s; // Last update every for this metric in this journal (last page collected)
109 -};
244 + // Length of the page
245 + uint16_t page_length;
246
111 -// 16 bytes
112 -struct journal_extent_list {
113 - uint64_t datafile_offset; // Datafile offset to find the extent
114 - uint32_t datafile_size; // Size of the extent
115 - uint16_t file_index; // which file index is this datafile[index]
116 - uint8_t pages; // number of pages (not all are necesssarily valid)
117 -};
247 + // Page type identifier
248 + uint8_t type;
249
119 -// 72 bytes
120 -struct journal_v2_header {
121 - uint32_t magic;
122 - usec_t start_time_ut; // Min start time of journal
123 - usec_t end_time_ut; // Maximum end time of journal
124 - uint32_t extent_count; // Count of extents
125 - uint32_t extent_offset;
126 - uint32_t metric_count; // Count of metrics (unique UUIDS)
127 - uint32_t metric_offset;
128 - uint32_t page_count; // Total count of pages (descriptors @ time)
129 - uint32_t page_offset;
130 - uint32_t extent_trailer_offset; // CRC for entent list
131 - uint32_t metric_trailer_offset; // CRC for metric list
132 - uint32_t journal_v1_file_size; // This is the original journal file
133 - uint32_t journal_v2_file_size; // This is the total file size
134 - void *data; // Used when building the index
250 + // --- implicit padding of 1-byte because the struct is not packed ---
251 };
252
137 -#define JOURNAL_V2_HEADER_PADDING_SZ (RRDENG_BLOCK_SIZE - (sizeof(struct journal_v2_header)))
138 -
253 struct wal;
254
255 void journalfile_v1_generate_path(struct rrdengine_datafile *datafile, char *str, size_t maxlen);
@@ -173,4 +287,4 @@ typedef struct {
287
288 struct rrdengine_datafile *njfv2idx_find_and_acquire_j2_header(NJFV2IDX_FIND_STATE *s);
289
176 -#endif /* NETDATA_JOURNALFILE_H */
\ No newline at end of file
290 +#endif /* NETDATA_JOURNALFILE_H */