@matheusbsbs / matheus-bagliano / commits / e48d427268

reftable: implement stack, a mutable database of reftable files.

Signed-off-by: Han-Wen Nienhuys <hanwen@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Han-Wen Nienhuys committed Oct 7, 2021 at 20:25 UTC e48d427268f965bf381a2f4fa445b5ade7913880
6 files changed +2520
Makefile
+1
@@ -2479,6 +2479,7 @@ REFTABLE_TEST_OBJS += reftable/pq_test.o
2479 REFTABLE_TEST_OBJS += reftable/record_test.o
2480 REFTABLE_TEST_OBJS += reftable/readwrite_test.o
2481 REFTABLE_TEST_OBJS += reftable/refname_test.o
2482 +REFTABLE_TEST_OBJS += reftable/stack_test.o
2483 REFTABLE_TEST_OBJS += reftable/test_framework.o
2484 REFTABLE_TEST_OBJS += reftable/tree_test.o
2485
reftable/reftable-stack.h new
+128
@@ -0,0 +1,128 @@
1 +/*
2 +Copyright 2020 Google LLC
3 +
4 +Use of this source code is governed by a BSD-style
5 +license that can be found in the LICENSE file or at
6 +https://developers.google.com/open-source/licenses/bsd
7 +*/
8 +
9 +#ifndef REFTABLE_STACK_H
10 +#define REFTABLE_STACK_H
11 +
12 +#include "reftable-writer.h"
13 +
14 +/*
15 + * The stack presents an interface to a mutable sequence of reftables.
16 +
17 + * A stack can be mutated by pushing a table to the top of the stack.
18 +
19 + * The reftable_stack automatically compacts files on disk to ensure good
20 + * amortized performance.
21 + *
22 + * For windows and other platforms that cannot have open files as rename
23 + * destinations, concurrent access from multiple processes needs the rand()
24 + * random seed to be randomized.
25 + */
26 +struct reftable_stack;
27 +
28 +/* open a new reftable stack. The tables along with the table list will be
29 + * stored in 'dir'. Typically, this should be .git/reftables.
30 + */
31 +int reftable_new_stack(struct reftable_stack **dest, const char *dir,
32 + struct reftable_write_options config);
33 +
34 +/* returns the update_index at which a next table should be written. */
35 +uint64_t reftable_stack_next_update_index(struct reftable_stack *st);
36 +
37 +/* holds a transaction to add tables at the top of a stack. */
38 +struct reftable_addition;
39 +
40 +/*
41 + * returns a new transaction to add reftables to the given stack. As a side
42 + * effect, the ref database is locked.
43 + */
44 +int reftable_stack_new_addition(struct reftable_addition **dest,
45 + struct reftable_stack *st);
46 +
47 +/* Adds a reftable to transaction. */
48 +int reftable_addition_add(struct reftable_addition *add,
49 + int (*write_table)(struct reftable_writer *wr,
50 + void *arg),
51 + void *arg);
52 +
53 +/* Commits the transaction, releasing the lock. After calling this,
54 + * reftable_addition_destroy should still be called.
55 + */
56 +int reftable_addition_commit(struct reftable_addition *add);
57 +
58 +/* Release all non-committed data from the transaction, and deallocate the
59 + * transaction. Releases the lock if held. */
60 +void reftable_addition_destroy(struct reftable_addition *add);
61 +
62 +/* add a new table to the stack. The write_table function must call
63 + * reftable_writer_set_limits, add refs and return an error value. */
64 +int reftable_stack_add(struct reftable_stack *st,
65 + int (*write_table)(struct reftable_writer *wr,
66 + void *write_arg),
67 + void *write_arg);
68 +
69 +/* returns the merged_table for seeking. This table is valid until the
70 + * next write or reload, and should not be closed or deleted.
71 + */
72 +struct reftable_merged_table *
73 +reftable_stack_merged_table(struct reftable_stack *st);
74 +
75 +/* frees all resources associated with the stack. */
76 +void reftable_stack_destroy(struct reftable_stack *st);
77 +
78 +/* Reloads the stack if necessary. This is very cheap to run if the stack was up
79 + * to date */
80 +int reftable_stack_reload(struct reftable_stack *st);
81 +
82 +/* Policy for expiring reflog entries. */
83 +struct reftable_log_expiry_config {
84 + /* Drop entries older than this timestamp */
85 + uint64_t time;
86 +
87 + /* Drop older entries */
88 + uint64_t min_update_index;
89 +};
90 +
91 +/* compacts all reftables into a giant table. Expire reflog entries if config is
92 + * non-NULL */
93 +int reftable_stack_compact_all(struct reftable_stack *st,
94 + struct reftable_log_expiry_config *config);
95 +
96 +/* heuristically compact unbalanced table stack. */
97 +int reftable_stack_auto_compact(struct reftable_stack *st);
98 +
99 +/* delete stale .ref tables. */
100 +int reftable_stack_clean(struct reftable_stack *st);
101 +
102 +/* convenience function to read a single ref. Returns < 0 for error, 0 for
103 + * success, and 1 if ref not found. */
104 +int reftable_stack_read_ref(struct reftable_stack *st, const char *refname,
105 + struct reftable_ref_record *ref);
106 +
107 +/* convenience function to read a single log. Returns < 0 for error, 0 for
108 + * success, and 1 if ref not found. */
109 +int reftable_stack_read_log(struct reftable_stack *st, const char *refname,
110 + struct reftable_log_record *log);
111 +
112 +/* statistics on past compactions. */
113 +struct reftable_compaction_stats {
114 + uint64_t bytes; /* total number of bytes written */
115 + uint64_t entries_written; /* total number of entries written, including
116 + failures. */
117 + int attempts; /* how often we tried to compact */
118 + int failures; /* failures happen on concurrent updates */
119 +};
120 +
121 +/* return statistics for compaction up till now. */
122 +struct reftable_compaction_stats *
123 +reftable_stack_compaction_stats(struct reftable_stack *st);
124 +
125 +/* print the entire stack represented by the directory */
126 +int reftable_stack_print_directory(const char *stackdir, uint32_t hash_id);
127 +
128 +#endif
reftable/stack.c new
+1396
@@ -0,0 +1,1396 @@
1 +/*
2 +Copyright 2020 Google LLC
3 +
4 +Use of this source code is governed by a BSD-style
5 +license that can be found in the LICENSE file or at
6 +https://developers.google.com/open-source/licenses/bsd
7 +*/
8 +
9 +#include "stack.h"
10 +
11 +#include "system.h"
12 +#include "merged.h"
13 +#include "reader.h"
14 +#include "refname.h"
15 +#include "reftable-error.h"
16 +#include "reftable-record.h"
17 +#include "reftable-merged.h"
18 +#include "writer.h"
19 +
20 +static int stack_try_add(struct reftable_stack *st,
21 + int (*write_table)(struct reftable_writer *wr,
22 + void *arg),
23 + void *arg);
24 +static int stack_write_compact(struct reftable_stack *st,
25 + struct reftable_writer *wr, int first, int last,
26 + struct reftable_log_expiry_config *config);
27 +static int stack_check_addition(struct reftable_stack *st,
28 + const char *new_tab_name);
29 +static void reftable_addition_close(struct reftable_addition *add);
30 +static int reftable_stack_reload_maybe_reuse(struct reftable_stack *st,
31 + int reuse_open);
32 +
33 +static void stack_filename(struct strbuf *dest, struct reftable_stack *st,
34 + const char *name)
35 +{
36 + strbuf_reset(dest);
37 + strbuf_addstr(dest, st->reftable_dir);
38 + strbuf_addstr(dest, "/");
39 + strbuf_addstr(dest, name);
40 +}
41 +
42 +static ssize_t reftable_fd_write(void *arg, const void *data, size_t sz)
43 +{
44 + int *fdp = (int *)arg;
45 + return write(*fdp, data, sz);
46 +}
47 +
48 +int reftable_new_stack(struct reftable_stack **dest, const char *dir,
49 + struct reftable_write_options config)
50 +{
51 + struct reftable_stack *p =
52 + reftable_calloc(sizeof(struct reftable_stack));
53 + struct strbuf list_file_name = STRBUF_INIT;
54 + int err = 0;
55 +
56 + if (config.hash_id == 0) {
57 + config.hash_id = GIT_SHA1_FORMAT_ID;
58 + }
59 +
60 + *dest = NULL;
61 +
62 + strbuf_reset(&list_file_name);
63 + strbuf_addstr(&list_file_name, dir);
64 + strbuf_addstr(&list_file_name, "/tables.list");
65 +
66 + p->list_file = strbuf_detach(&list_file_name, NULL);
67 + p->reftable_dir = xstrdup(dir);
68 + p->config = config;
69 +
70 + err = reftable_stack_reload_maybe_reuse(p, 1);
71 + if (err < 0) {
72 + reftable_stack_destroy(p);
73 + } else {
74 + *dest = p;
75 + }
76 + return err;
77 +}
78 +
79 +static int fd_read_lines(int fd, char ***namesp)
80 +{
81 + off_t size = lseek(fd, 0, SEEK_END);
82 + char *buf = NULL;
83 + int err = 0;
84 + if (size < 0) {
85 + err = REFTABLE_IO_ERROR;
86 + goto done;
87 + }
88 + err = lseek(fd, 0, SEEK_SET);
89 + if (err < 0) {
90 + err = REFTABLE_IO_ERROR;
91 + goto done;
92 + }
93 +
94 + buf = reftable_malloc(size + 1);
95 + if (read(fd, buf, size) != size) {
96 + err = REFTABLE_IO_ERROR;
97 + goto done;
98 + }
99 + buf[size] = 0;
100 +
101 + parse_names(buf, size, namesp);
102 +
103 +done:
104 + reftable_free(buf);
105 + return err;
106 +}
107 +
108 +int read_lines(const char *filename, char ***namesp)
109 +{
110 + int fd = open(filename, O_RDONLY);
111 + int err = 0;
112 + if (fd < 0) {
113 + if (errno == ENOENT) {
114 + *namesp = reftable_calloc(sizeof(char *));
115 + return 0;
116 + }
117 +
118 + return REFTABLE_IO_ERROR;
119 + }
120 + err = fd_read_lines(fd, namesp);
121 + close(fd);
122 + return err;
123 +}
124 +
125 +struct reftable_merged_table *
126 +reftable_stack_merged_table(struct reftable_stack *st)
127 +{
128 + return st->merged;
129 +}
130 +
131 +static int has_name(char **names, const char *name)
132 +{
133 + while (*names) {
134 + if (!strcmp(*names, name))
135 + return 1;
136 + names++;
137 + }
138 + return 0;
139 +}
140 +
141 +/* Close and free the stack */
142 +void reftable_stack_destroy(struct reftable_stack *st)
143 +{
144 + char **names = NULL;
145 + int err = 0;
146 + if (st->merged) {
147 + reftable_merged_table_free(st->merged);
148 + st->merged = NULL;
149 + }
150 +
151 + err = read_lines(st->list_file, &names);
152 + if (err < 0) {
153 + FREE_AND_NULL(names);
154 + }
155 +
156 + if (st->readers) {
157 + int i = 0;
158 + struct strbuf filename = STRBUF_INIT;
159 + for (i = 0; i < st->readers_len; i++) {
160 + const char *name = reader_name(st->readers[i]);
161 + strbuf_reset(&filename);
162 + if (names && !has_name(names, name)) {
163 + stack_filename(&filename, st, name);
164 + }
165 + reftable_reader_free(st->readers[i]);
166 +
167 + if (filename.len) {
168 + /* On Windows, can only unlink after closing. */
169 + unlink(filename.buf);
170 + }
171 + }
172 + strbuf_release(&filename);
173 + st->readers_len = 0;
174 + FREE_AND_NULL(st->readers);
175 + }
176 + FREE_AND_NULL(st->list_file);
177 + FREE_AND_NULL(st->reftable_dir);
178 + reftable_free(st);
179 + free_names(names);
180 +}
181 +
182 +static struct reftable_reader **stack_copy_readers(struct reftable_stack *st,
183 + int cur_len)
184 +{
185 + struct reftable_reader **cur =
186 + reftable_calloc(sizeof(struct reftable_reader *) * cur_len);
187 + int i = 0;
188 + for (i = 0; i < cur_len; i++) {
189 + cur[i] = st->readers[i];
190 + }
191 + return cur;
192 +}
193 +
194 +static int reftable_stack_reload_once(struct reftable_stack *st, char **names,
195 + int reuse_open)
196 +{
197 + int cur_len = !st->merged ? 0 : st->merged->stack_len;
198 + struct reftable_reader **cur = stack_copy_readers(st, cur_len);
199 + int err = 0;
200 + int names_len = names_length(names);
201 + struct reftable_reader **new_readers =
202 + reftable_calloc(sizeof(struct reftable_reader *) * names_len);
203 + struct reftable_table *new_tables =
204 + reftable_calloc(sizeof(struct reftable_table) * names_len);
205 + int new_readers_len = 0;
206 + struct reftable_merged_table *new_merged = NULL;
207 + int i;
208 +
209 + while (*names) {
210 + struct reftable_reader *rd = NULL;
211 + char *name = *names++;
212 +
213 + /* this is linear; we assume compaction keeps the number of
214 + tables under control so this is not quadratic. */
215 + int j = 0;
216 + for (j = 0; reuse_open && j < cur_len; j++) {
217 + if (cur[j] && 0 == strcmp(cur[j]->name, name)) {
218 + rd = cur[j];
219 + cur[j] = NULL;
220 + break;
221 + }
222 + }
223 +
224 + if (!rd) {
225 + struct reftable_block_source src = { NULL };
226 + struct strbuf table_path = STRBUF_INIT;
227 + stack_filename(&table_path, st, name);
228 +
229 + err = reftable_block_source_from_file(&src,
230 + table_path.buf);
231 + strbuf_release(&table_path);
232 +
233 + if (err < 0)
234 + goto done;
235 +
236 + err = reftable_new_reader(&rd, &src, name);
237 + if (err < 0)
238 + goto done;
239 + }
240 +
241 + new_readers[new_readers_len] = rd;
242 + reftable_table_from_reader(&new_tables[new_readers_len], rd);
243 + new_readers_len++;
244 + }
245 +
246 + /* success! */
247 + err = reftable_new_merged_table(&new_merged, new_tables,
248 + new_readers_len, st->config.hash_id);
249 + if (err < 0)
250 + goto done;
251 +
252 + new_tables = NULL;
253 + st->readers_len = new_readers_len;
254 + if (st->merged) {
255 + merged_table_release(st->merged);
256 + reftable_merged_table_free(st->merged);
257 + }
258 + if (st->readers) {
259 + reftable_free(st->readers);
260 + }
261 + st->readers = new_readers;
262 + new_readers = NULL;
263 + new_readers_len = 0;
264 +
265 + new_merged->suppress_deletions = 1;
266 + st->merged = new_merged;
267 + for (i = 0; i < cur_len; i++) {
268 + if (cur[i]) {
269 + const char *name = reader_name(cur[i]);
270 + struct strbuf filename = STRBUF_INIT;
271 + stack_filename(&filename, st, name);
272 +
273 + reader_close(cur[i]);
274 + reftable_reader_free(cur[i]);
275 +
276 + /* On Windows, can only unlink after closing. */
277 + unlink(filename.buf);
278 +
279 + strbuf_release(&filename);
280 + }
281 + }
282 +
283 +done:
284 + for (i = 0; i < new_readers_len; i++) {
285 + reader_close(new_readers[i]);
286 + reftable_reader_free(new_readers[i]);
287 + }
288 + reftable_free(new_readers);
289 + reftable_free(new_tables);
290 + reftable_free(cur);
291 + return err;
292 +}
293 +
294 +/* return negative if a before b. */
295 +static int tv_cmp(struct timeval *a, struct timeval *b)
296 +{
297 + time_t diff = a->tv_sec - b->tv_sec;
298 + int udiff = a->tv_usec - b->tv_usec;
299 +
300 + if (diff != 0)
301 + return diff;
302 +
303 + return udiff;
304 +}
305 +
306 +static int reftable_stack_reload_maybe_reuse(struct reftable_stack *st,
307 + int reuse_open)
308 +{
309 + struct timeval deadline = { 0 };
310 + int err = gettimeofday(&deadline, NULL);
311 + int64_t delay = 0;
312 + int tries = 0;
313 + if (err < 0)
314 + return err;
315 +
316 + deadline.tv_sec += 3;
317 + while (1) {
318 + char **names = NULL;
319 + char **names_after = NULL;
320 + struct timeval now = { 0 };
321 + int err = gettimeofday(&now, NULL);
322 + int err2 = 0;
323 + if (err < 0) {
324 + return err;
325 + }
326 +
327 + /* Only look at deadlines after the first few times. This
328 + simplifies debugging in GDB */
329 + tries++;
330 + if (tries > 3 && tv_cmp(&now, &deadline) >= 0) {
331 + break;
332 + }
333 +
334 + err = read_lines(st->list_file, &names);
335 + if (err < 0) {
336 + free_names(names);
337 + return err;
338 + }
339 + err = reftable_stack_reload_once(st, names, reuse_open);
340 + if (err == 0) {
341 + free_names(names);
342 + break;
343 + }
344 + if (err != REFTABLE_NOT_EXIST_ERROR) {
345 + free_names(names);
346 + return err;
347 + }
348 +
349 + /* err == REFTABLE_NOT_EXIST_ERROR can be caused by a concurrent
350 + writer. Check if there was one by checking if the name list
351 + changed.
352 + */
353 + err2 = read_lines(st->list_file, &names_after);
354 + if (err2 < 0) {
355 + free_names(names);
356 + return err2;
357 + }
358 +
359 + if (names_equal(names_after, names)) {
360 + free_names(names);
361 + free_names(names_after);
362 + return err;
363 + }
364 + free_names(names);
365 + free_names(names_after);
366 +
367 + delay = delay + (delay * rand()) / RAND_MAX + 1;
368 + sleep_millisec(delay);
369 + }
370 +
371 + return 0;
372 +}
373 +
374 +/* -1 = error
375 + 0 = up to date
376 + 1 = changed. */
377 +static int stack_uptodate(struct reftable_stack *st)
378 +{
379 + char **names = NULL;
380 + int err = read_lines(st->list_file, &names);
381 + int i = 0;
382 + if (err < 0)
383 + return err;
384 +
385 + for (i = 0; i < st->readers_len; i++) {
386 + if (!names[i]) {
387 + err = 1;
388 + goto done;
389 + }
390 +
391 + if (strcmp(st->readers[i]->name, names[i])) {
392 + err = 1;
393 + goto done;
394 + }
395 + }
396 +
397 + if (names[st->merged->stack_len]) {
398 + err = 1;
399 + goto done;
400 + }
401 +
402 +done:
403 + free_names(names);
404 + return err;
405 +}
406 +
407 +int reftable_stack_reload(struct reftable_stack *st)
408 +{
409 + int err = stack_uptodate(st);
410 + if (err > 0)
411 + return reftable_stack_reload_maybe_reuse(st, 1);
412 + return err;
413 +}
414 +
415 +int reftable_stack_add(struct reftable_stack *st,
416 + int (*write)(struct reftable_writer *wr, void *arg),
417 + void *arg)
418 +{
419 + int err = stack_try_add(st, write, arg);
420 + if (err < 0) {
421 + if (err == REFTABLE_LOCK_ERROR) {
422 + /* Ignore error return, we want to propagate
423 + REFTABLE_LOCK_ERROR.
424 + */
425 + reftable_stack_reload(st);
426 + }
427 + return err;
428 + }
429 +
430 + if (!st->disable_auto_compact)
431 + return reftable_stack_auto_compact(st);
432 +
433 + return 0;
434 +}
435 +
436 +static void format_name(struct strbuf *dest, uint64_t min, uint64_t max)
437 +{
438 + char buf[100];
439 + uint32_t rnd = (uint32_t)rand();
440 + snprintf(buf, sizeof(buf), "0x%012" PRIx64 "-0x%012" PRIx64 "-%08x",
441 + min, max, rnd);
442 + strbuf_reset(dest);
443 + strbuf_addstr(dest, buf);
444 +}
445 +
446 +struct reftable_addition {
447 + int lock_file_fd;
448 + struct strbuf lock_file_name;
449 + struct reftable_stack *stack;
450 +
451 + char **new_tables;
452 + int new_tables_len;
453 + uint64_t next_update_index;
454 +};
455 +
456 +#define REFTABLE_ADDITION_INIT \
457 + { \
458 + .lock_file_name = STRBUF_INIT \
459 + }
460 +
461 +static int reftable_stack_init_addition(struct reftable_addition *add,
462 + struct reftable_stack *st)
463 +{
464 + int err = 0;
465 + add->stack = st;
466 +
467 + strbuf_reset(&add->lock_file_name);
468 + strbuf_addstr(&add->lock_file_name, st->list_file);
469 + strbuf_addstr(&add->lock_file_name, ".lock");
470 +
471 + add->lock_file_fd = open(add->lock_file_name.buf,
472 + O_EXCL | O_CREAT | O_WRONLY, 0644);
473 + if (add->lock_file_fd < 0) {
474 + if (errno == EEXIST) {
475 + err = REFTABLE_LOCK_ERROR;
476 + } else {
477 + err = REFTABLE_IO_ERROR;
478 + }
479 + goto done;
480 + }
481 + err = stack_uptodate(st);
482 + if (err < 0)
483 + goto done;
484 +
485 + if (err > 1) {
486 + err = REFTABLE_LOCK_ERROR;
487 + goto done;
488 + }
489 +
490 + add->next_update_index = reftable_stack_next_update_index(st);
491 +done:
492 + if (err) {
493 + reftable_addition_close(add);
494 + }
495 + return err;
496 +}
497 +
498 +static void reftable_addition_close(struct reftable_addition *add)
499 +{
500 + int i = 0;
501 + struct strbuf nm = STRBUF_INIT;
502 + for (i = 0; i < add->new_tables_len; i++) {
503 + stack_filename(&nm, add->stack, add->new_tables[i]);
504 + unlink(nm.buf);
505 + reftable_free(add->new_tables[i]);
506 + add->new_tables[i] = NULL;
507 + }
508 + reftable_free(add->new_tables);
509 + add->new_tables = NULL;
510 + add->new_tables_len = 0;
511 +
512 + if (add->lock_file_fd > 0) {
513 + close(add->lock_file_fd);
514 + add->lock_file_fd = 0;
515 + }
516 + if (add->lock_file_name.len > 0) {
517 + unlink(add->lock_file_name.buf);
518 + strbuf_release(&add->lock_file_name);
519 + }
520 +
521 + strbuf_release(&nm);
522 +}
523 +
524 +void reftable_addition_destroy(struct reftable_addition *add)
525 +{
526 + if (!add) {
527 + return;
528 + }
529 + reftable_addition_close(add);
530 + reftable_free(add);
531 +}
532 +
533 +int reftable_addition_commit(struct reftable_addition *add)
534 +{
535 + struct strbuf table_list = STRBUF_INIT;
536 + int i = 0;
537 + int err = 0;
538 + if (add->new_tables_len == 0)
539 + goto done;
540 +
541 + for (i = 0; i < add->stack->merged->stack_len; i++) {
542 + strbuf_addstr(&table_list, add->stack->readers[i]->name);
543 + strbuf_addstr(&table_list, "\n");
544 + }
545 + for (i = 0; i < add->new_tables_len; i++) {
546 + strbuf_addstr(&table_list, add->new_tables[i]);
547 + strbuf_addstr(&table_list, "\n");
548 + }
549 +
550 + err = write(add->lock_file_fd, table_list.buf, table_list.len);
551 + strbuf_release(&table_list);
552 + if (err < 0) {
553 + err = REFTABLE_IO_ERROR;
554 + goto done;
555 + }
556 +
557 + err = close(add->lock_file_fd);
558 + add->lock_file_fd = 0;
559 + if (err < 0) {
560 + err = REFTABLE_IO_ERROR;
561 + goto done;
562 + }
563 +
564 + err = rename(add->lock_file_name.buf, add->stack->list_file);
565 + if (err < 0) {
566 + err = REFTABLE_IO_ERROR;
567 + goto done;
568 + }
569 +
570 + /* success, no more state to clean up. */
571 + strbuf_release(&add->lock_file_name);
572 + for (i = 0; i < add->new_tables_len; i++) {
573 + reftable_free(add->new_tables[i]);
574 + }
575 + reftable_free(add->new_tables);
576 + add->new_tables = NULL;
577 + add->new_tables_len = 0;
578 +
579 + err = reftable_stack_reload(add->stack);
580 +done:
581 + reftable_addition_close(add);
582 + return err;
583 +}
584 +
585 +int reftable_stack_new_addition(struct reftable_addition **dest,
586 + struct reftable_stack *st)
587 +{
588 + int err = 0;
589 + struct reftable_addition empty = REFTABLE_ADDITION_INIT;
590 + *dest = reftable_calloc(sizeof(**dest));
591 + **dest = empty;
592 + err = reftable_stack_init_addition(*dest, st);
593 + if (err) {
594 + reftable_free(*dest);
595 + *dest = NULL;
596 + }
597 + return err;
598 +}
599 +
600 +static int stack_try_add(struct reftable_stack *st,
601 + int (*write_table)(struct reftable_writer *wr,
602 + void *arg),
603 + void *arg)
604 +{
605 + struct reftable_addition add = REFTABLE_ADDITION_INIT;
606 + int err = reftable_stack_init_addition(&add, st);
607 + if (err < 0)
608 + goto done;
609 + if (err > 0) {
610 + err = REFTABLE_LOCK_ERROR;
611 + goto done;
612 + }
613 +
614 + err = reftable_addition_add(&add, write_table, arg);
615 + if (err < 0)
616 + goto done;
617 +
618 + err = reftable_addition_commit(&add);
619 +done:
620 + reftable_addition_close(&add);
621 + return err;
622 +}
623 +
624 +int reftable_addition_add(struct reftable_addition *add,
625 + int (*write_table)(struct reftable_writer *wr,
626 + void *arg),
627 + void *arg)
628 +{
629 + struct strbuf temp_tab_file_name = STRBUF_INIT;
630 + struct strbuf tab_file_name = STRBUF_INIT;
631 + struct strbuf next_name = STRBUF_INIT;
632 + struct reftable_writer *wr = NULL;
633 + int err = 0;
634 + int tab_fd = 0;
635 +
636 + strbuf_reset(&next_name);
637 + format_name(&next_name, add->next_update_index, add->next_update_index);
638 +
639 + stack_filename(&temp_tab_file_name, add->stack, next_name.buf);
640 + strbuf_addstr(&temp_tab_file_name, ".temp.XXXXXX");
641 +
642 + tab_fd = mkstemp(temp_tab_file_name.buf);
643 + if (tab_fd < 0) {
644 + err = REFTABLE_IO_ERROR;
645 + goto done;
646 + }
647 +
648 + wr = reftable_new_writer(reftable_fd_write, &tab_fd,
649 + &add->stack->config);
650 + err = write_table(wr, arg);
651 + if (err < 0)
652 + goto done;
653 +
654 + err = reftable_writer_close(wr);
655 + if (err == REFTABLE_EMPTY_TABLE_ERROR) {
656 + err = 0;
657 + goto done;
658 + }
659 + if (err < 0)
660 + goto done;
661 +
662 + err = close(tab_fd);
663 + tab_fd = 0;
664 + if (err < 0) {
665 + err = REFTABLE_IO_ERROR;
666 + goto done;
667 + }
668 +
669 + err = stack_check_addition(add->stack, temp_tab_file_name.buf);
670 + if (err < 0)
671 + goto done;
672 +
673 + if (wr->min_update_index < add->next_update_index) {
674 + err = REFTABLE_API_ERROR;
675 + goto done;
676 + }
677 +
678 + format_name(&next_name, wr->min_update_index, wr->max_update_index);
679 + strbuf_addstr(&next_name, ".ref");
680 +
681 + stack_filename(&tab_file_name, add->stack, next_name.buf);
682 +
683 + /*
684 + On windows, this relies on rand() picking a unique destination name.
685 + Maybe we should do retry loop as well?
686 + */
687 + err = rename(temp_tab_file_name.buf, tab_file_name.buf);
688 + if (err < 0) {
689 + err = REFTABLE_IO_ERROR;
690 + goto done;
691 + }
692 +
693 + add->new_tables = reftable_realloc(add->new_tables,
694 + sizeof(*add->new_tables) *
695 + (add->new_tables_len + 1));
696 + add->new_tables[add->new_tables_len] = strbuf_detach(&next_name, NULL);
697 + add->new_tables_len++;
698 +done:
699 + if (tab_fd > 0) {
700 + close(tab_fd);
701 + tab_fd = 0;
702 + }
703 + if (temp_tab_file_name.len > 0) {
704 + unlink(temp_tab_file_name.buf);
705 + }
706 +
707 + strbuf_release(&temp_tab_file_name);
708 + strbuf_release(&tab_file_name);
709 + strbuf_release(&next_name);
710 + reftable_writer_free(wr);
711 + return err;
712 +}
713 +
714 +uint64_t reftable_stack_next_update_index(struct reftable_stack *st)
715 +{
716 + int sz = st->merged->stack_len;
717 + if (sz > 0)
718 + return reftable_reader_max_update_index(st->readers[sz - 1]) +
719 + 1;
720 + return 1;
721 +}
722 +
723 +static int stack_compact_locked(struct reftable_stack *st, int first, int last,
724 + struct strbuf *temp_tab,
725 + struct reftable_log_expiry_config *config)
726 +{
727 + struct strbuf next_name = STRBUF_INIT;
728 + int tab_fd = -1;
729 + struct reftable_writer *wr = NULL;
730 + int err = 0;
731 +
732 + format_name(&next_name,
733 + reftable_reader_min_update_index(st->readers[first]),
734 + reftable_reader_max_update_index(st->readers[last]));
735 +
736 + stack_filename(temp_tab, st, next_name.buf);
737 + strbuf_addstr(temp_tab, ".temp.XXXXXX");
738 +
739 + tab_fd = mkstemp(temp_tab->buf);
740 + wr = reftable_new_writer(reftable_fd_write, &tab_fd, &st->config);
741 +
742 + err = stack_write_compact(st, wr, first, last, config);
743 + if (err < 0)
744 + goto done;
745 + err = reftable_writer_close(wr);
746 + if (err < 0)
747 + goto done;
748 +
749 + err = close(tab_fd);
750 + tab_fd = 0;
751 +
752 +done:
753 + reftable_writer_free(wr);
754 + if (tab_fd > 0) {
755 + close(tab_fd);
756 + tab_fd = 0;
757 + }
758 + if (err != 0 && temp_tab->len > 0) {
759 + unlink(temp_tab->buf);
760 + strbuf_release(temp_tab);
761 + }
762 + strbuf_release(&next_name);
763 + return err;
764 +}
765 +
766 +static int stack_write_compact(struct reftable_stack *st,
767 + struct reftable_writer *wr, int first, int last,
768 + struct reftable_log_expiry_config *config)
769 +{
770 + int subtabs_len = last - first + 1;
771 + struct reftable_table *subtabs = reftable_calloc(
772 + sizeof(struct reftable_table) * (last - first + 1));
773 + struct reftable_merged_table *mt = NULL;
774 + int err = 0;
775 + struct reftable_iterator it = { NULL };
776 + struct reftable_ref_record ref = { NULL };
777 + struct reftable_log_record log = { NULL };
778 +
779 + uint64_t entries = 0;
780 +
781 + int i = 0, j = 0;
782 + for (i = first, j = 0; i <= last; i++) {
783 + struct reftable_reader *t = st->readers[i];
784 + reftable_table_from_reader(&subtabs[j++], t);
785 + st->stats.bytes += t->size;
786 + }
787 + reftable_writer_set_limits(wr, st->readers[first]->min_update_index,
788 + st->readers[last]->max_update_index);
789 +
790 + err = reftable_new_merged_table(&mt, subtabs, subtabs_len,
791 + st->config.hash_id);
792 + if (err < 0) {
793 + reftable_free(subtabs);
794 + goto done;
795 + }
796 +
797 + err = reftable_merged_table_seek_ref(mt, &it, "");
798 + if (err < 0)
799 + goto done;
800 +
801 + while (1) {
802 + err = reftable_iterator_next_ref(&it, &ref);
803 + if (err > 0) {
804 + err = 0;
805 + break;
806 + }
807 + if (err < 0) {
808 + break;
809 + }
810 +
811 + if (first == 0 && reftable_ref_record_is_deletion(&ref)) {
812 + continue;
813 + }
814 +
815 + err = reftable_writer_add_ref(wr, &ref);
816 + if (err < 0) {
817 + break;
818 + }
819 + entries++;
820 + }
821 + reftable_iterator_destroy(&it);
822 +
823 + err = reftable_merged_table_seek_log(mt, &it, "");
824 + if (err < 0)
825 + goto done;
826 +
827 + while (1) {
828 + err = reftable_iterator_next_log(&it, &log);
829 + if (err > 0) {
830 + err = 0;
831 + break;
832 + }
833 + if (err < 0) {
834 + break;
835 + }
836 + if (first == 0 && reftable_log_record_is_deletion(&log)) {
837 + continue;
838 + }
839 +
840 + if (config && config->min_update_index > 0 &&
841 + log.update_index < config->min_update_index) {
842 + continue;
843 + }
844 +
845 + if (config && config->time > 0 &&
846 + log.value.update.time < config->time) {
847 + continue;
848 + }
849 +
850 + err = reftable_writer_add_log(wr, &log);
851 + if (err < 0) {
852 + break;
853 + }
854 + entries++;
855 + }
856 +
857 +done:
858 + reftable_iterator_destroy(&it);
859 + if (mt) {
860 + merged_table_release(mt);
861 + reftable_merged_table_free(mt);
862 + }
863 + reftable_ref_record_release(&ref);
864 + reftable_log_record_release(&log);
865 + st->stats.entries_written += entries;
866 + return err;
867 +}
868 +
869 +/* < 0: error. 0 == OK, > 0 attempt failed; could retry. */
870 +static int stack_compact_range(struct reftable_stack *st, int first, int last,
871 + struct reftable_log_expiry_config *expiry)
872 +{
873 + struct strbuf temp_tab_file_name = STRBUF_INIT;
874 + struct strbuf new_table_name = STRBUF_INIT;
875 + struct strbuf lock_file_name = STRBUF_INIT;
876 + struct strbuf ref_list_contents = STRBUF_INIT;
877 + struct strbuf new_table_path = STRBUF_INIT;
878 + int err = 0;
879 + int have_lock = 0;
880 + int lock_file_fd = 0;
881 + int compact_count = last - first + 1;
882 + char **listp = NULL;
883 + char **delete_on_success =
884 + reftable_calloc(sizeof(char *) * (compact_count + 1));
885 + char **subtable_locks =
886 + reftable_calloc(sizeof(char *) * (compact_count + 1));
887 + int i = 0;
888 + int j = 0;
889 + int is_empty_table = 0;
890 +
891 + if (first > last || (!expiry && first == last)) {
892 + err = 0;
893 + goto done;
894 + }
895 +
896 + st->stats.attempts++;
897 +
898 + strbuf_reset(&lock_file_name);
899 + strbuf_addstr(&lock_file_name, st->list_file);
900 + strbuf_addstr(&lock_file_name, ".lock");
901 +
902 + lock_file_fd =
903 + open(lock_file_name.buf, O_EXCL | O_CREAT | O_WRONLY, 0644);
904 + if (lock_file_fd < 0) {
905 + if (errno == EEXIST) {
906 + err = 1;
907 + } else {
908 + err = REFTABLE_IO_ERROR;
909 + }
910 + goto done;
911 + }
912 + /* Don't want to write to the lock for now. */
913 + close(lock_file_fd);
914 + lock_file_fd = 0;
915 +
916 + have_lock = 1;
917 + err = stack_uptodate(st);
918 + if (err != 0)
919 + goto done;
920 +
921 + for (i = first, j = 0; i <= last; i++) {
922 + struct strbuf subtab_file_name = STRBUF_INIT;
923 + struct strbuf subtab_lock = STRBUF_INIT;
924 + int sublock_file_fd = -1;
925 +
926 + stack_filename(&subtab_file_name, st,
927 + reader_name(st->readers[i]));
928 +
929 + strbuf_reset(&subtab_lock);
930 + strbuf_addbuf(&subtab_lock, &subtab_file_name);
931 + strbuf_addstr(&subtab_lock, ".lock");
932 +
933 + sublock_file_fd = open(subtab_lock.buf,
934 + O_EXCL | O_CREAT | O_WRONLY, 0644);
935 + if (sublock_file_fd > 0) {
936 + close(sublock_file_fd);
937 + } else if (sublock_file_fd < 0) {
938 + if (errno == EEXIST) {
939 + err = 1;
940 + } else {
941 + err = REFTABLE_IO_ERROR;
942 + }
943 + }
944 +
945 + subtable_locks[j] = subtab_lock.buf;
946 + delete_on_success[j] = subtab_file_name.buf;
947 + j++;
948 +
949 + if (err != 0)
950 + goto done;
951 + }
952 +
953 + err = unlink(lock_file_name.buf);
954 + if (err < 0)
955 + goto done;
956 + have_lock = 0;
957 +
958 + err = stack_compact_locked(st, first, last, &temp_tab_file_name,
959 + expiry);
960 + /* Compaction + tombstones can create an empty table out of non-empty
961 + * tables. */
962 + is_empty_table = (err == REFTABLE_EMPTY_TABLE_ERROR);
963 + if (is_empty_table) {
964 + err = 0;
965 + }
966 + if (err < 0)
967 + goto done;
968 +
969 + lock_file_fd =
970 + open(lock_file_name.buf, O_EXCL | O_CREAT | O_WRONLY, 0644);
971 + if (lock_file_fd < 0) {
972 + if (errno == EEXIST) {
973 + err = 1;
974 + } else {
975 + err = REFTABLE_IO_ERROR;
976 + }
977 + goto done;
978 + }
979 + have_lock = 1;
980 +
981 + format_name(&new_table_name, st->readers[first]->min_update_index,
982 + st->readers[last]->max_update_index);
983 + strbuf_addstr(&new_table_name, ".ref");
984 +
985 + stack_filename(&new_table_path, st, new_table_name.buf);
986 +
987 + if (!is_empty_table) {
988 + /* retry? */
989 + err = rename(temp_tab_file_name.buf, new_table_path.buf);
990 + if (err < 0) {
991 + err = REFTABLE_IO_ERROR;
992 + goto done;
993 + }
994 + }
995 +
996 + for (i = 0; i < first; i++) {
997 + strbuf_addstr(&ref_list_contents, st->readers[i]->name);
998 + strbuf_addstr(&ref_list_contents, "\n");
999 + }
1000 + if (!is_empty_table) {
1001 + strbuf_addbuf(&ref_list_contents, &new_table_name);
1002 + strbuf_addstr(&ref_list_contents, "\n");
1003 + }
1004 + for (i = last + 1; i < st->merged->stack_len; i++) {
1005 + strbuf_addstr(&ref_list_contents, st->readers[i]->name);
1006 + strbuf_addstr(&ref_list_contents, "\n");
1007 + }
1008 +
1009 + err = write(lock_file_fd, ref_list_contents.buf, ref_list_contents.len);
1010 + if (err < 0) {
1011 + err = REFTABLE_IO_ERROR;
1012 + unlink(new_table_path.buf);
1013 + goto done;
1014 + }
1015 + err = close(lock_file_fd);
1016 + lock_file_fd = 0;
1017 + if (err < 0) {
1018 + err = REFTABLE_IO_ERROR;
1019 + unlink(new_table_path.buf);
1020 + goto done;
1021 + }
1022 +
1023 + err = rename(lock_file_name.buf, st->list_file);
1024 + if (err < 0) {
1025 + err = REFTABLE_IO_ERROR;
1026 + unlink(new_table_path.buf);
1027 + goto done;
1028 + }
1029 + have_lock = 0;
1030 +
1031 + /* Reload the stack before deleting. On windows, we can only delete the
1032 + files after we closed them.
1033 + */
1034 + err = reftable_stack_reload_maybe_reuse(st, first < last);
1035 +
1036 + listp = delete_on_success;
1037 + while (*listp) {
1038 + if (strcmp(*listp, new_table_path.buf)) {
1039 + unlink(*listp);
1040 + }
1041 + listp++;
1042 + }
1043 +
1044 +done:
1045 + free_names(delete_on_success);
1046 +
1047 + listp = subtable_locks;
1048 + while (*listp) {
1049 + unlink(*listp);
1050 + listp++;
1051 + }
1052 + free_names(subtable_locks);
1053 + if (lock_file_fd > 0) {
1054 + close(lock_file_fd);
1055 + lock_file_fd = 0;
1056 + }
1057 + if (have_lock) {
1058 + unlink(lock_file_name.buf);
1059 + }
1060 + strbuf_release(&new_table_name);
1061 + strbuf_release(&new_table_path);
1062 + strbuf_release(&ref_list_contents);
1063 + strbuf_release(&temp_tab_file_name);
1064 + strbuf_release(&lock_file_name);
1065 + return err;
1066 +}
1067 +
1068 +int reftable_stack_compact_all(struct reftable_stack *st,
1069 + struct reftable_log_expiry_config *config)
1070 +{
1071 + return stack_compact_range(st, 0, st->merged->stack_len - 1, config);
1072 +}
1073 +
1074 +static int stack_compact_range_stats(struct reftable_stack *st, int first,
1075 + int last,
1076 + struct reftable_log_expiry_config *config)
1077 +{
1078 + int err = stack_compact_range(st, first, last, config);
1079 + if (err > 0) {
1080 + st->stats.failures++;
1081 + }
1082 + return err;
1083 +}
1084 +
1085 +static int segment_size(struct segment *s)
1086 +{
1087 + return s->end - s->start;
1088 +}
1089 +
1090 +int fastlog2(uint64_t sz)
1091 +{
1092 + int l = 0;
1093 + if (sz == 0)
1094 + return 0;
1095 + for (; sz; sz /= 2) {
1096 + l++;
1097 + }
1098 + return l - 1;
1099 +}
1100 +
1101 +struct segment *sizes_to_segments(int *seglen, uint64_t *sizes, int n)
1102 +{
1103 + struct segment *segs = reftable_calloc(sizeof(struct segment) * n);
1104 + int next = 0;
1105 + struct segment cur = { 0 };
1106 + int i = 0;
1107 +
1108 + if (n == 0) {
1109 + *seglen = 0;
1110 + return segs;
1111 + }
1112 + for (i = 0; i < n; i++) {
1113 + int log = fastlog2(sizes[i]);
1114 + if (cur.log != log && cur.bytes > 0) {
1115 + struct segment fresh = {
1116 + .start = i,
1117 + };
1118 +
1119 + segs[next++] = cur;
1120 + cur = fresh;
1121 + }
1122 +
1123 + cur.log = log;
1124 + cur.end = i + 1;
1125 + cur.bytes += sizes[i];
1126 + }
1127 + segs[next++] = cur;
1128 + *seglen = next;
1129 + return segs;
1130 +}
1131 +
1132 +struct segment suggest_compaction_segment(uint64_t *sizes, int n)
1133 +{
1134 + int seglen = 0;
1135 + struct segment *segs = sizes_to_segments(&seglen, sizes, n);
1136 + struct segment min_seg = {
1137 + .log = 64,
1138 + };
1139 + int i = 0;
1140 + for (i = 0; i < seglen; i++) {
1141 + if (segment_size(&segs[i]) == 1) {
1142 + continue;
1143 + }
1144 +
1145 + if (segs[i].log < min_seg.log) {
1146 + min_seg = segs[i];
1147 + }
1148 + }
1149 +
1150 + while (min_seg.start > 0) {
1151 + int prev = min_seg.start - 1;
1152 + if (fastlog2(min_seg.bytes) < fastlog2(sizes[prev])) {
1153 + break;
1154 + }
1155 +
1156 + min_seg.start = prev;
1157 + min_seg.bytes += sizes[prev];
1158 + }
1159 +
1160 + reftable_free(segs);
1161 + return min_seg;
1162 +}
1163 +
1164 +static uint64_t *stack_table_sizes_for_compaction(struct reftable_stack *st)
1165 +{
1166 + uint64_t *sizes =
1167 + reftable_calloc(sizeof(uint64_t) * st->merged->stack_len);
1168 + int version = (st->config.hash_id == GIT_SHA1_FORMAT_ID) ? 1 : 2;
1169 + int overhead = header_size(version) - 1;
1170 + int i = 0;
1171 + for (i = 0; i < st->merged->stack_len; i++) {
1172 + sizes[i] = st->readers[i]->size - overhead;
1173 + }
1174 + return sizes;
1175 +}
1176 +
1177 +int reftable_stack_auto_compact(struct reftable_stack *st)
1178 +{
1179 + uint64_t *sizes = stack_table_sizes_for_compaction(st);
1180 + struct segment seg =
1181 + suggest_compaction_segment(sizes, st->merged->stack_len);
1182 + reftable_free(sizes);
1183 + if (segment_size(&seg) > 0)
1184 + return stack_compact_range_stats(st, seg.start, seg.end - 1,
1185 + NULL);
1186 +
1187 + return 0;
1188 +}
1189 +
1190 +struct reftable_compaction_stats *
1191 +reftable_stack_compaction_stats(struct reftable_stack *st)
1192 +{
1193 + return &st->stats;
1194 +}
1195 +
1196 +int reftable_stack_read_ref(struct reftable_stack *st, const char *refname,
1197 + struct reftable_ref_record *ref)
1198 +{
1199 + struct reftable_table tab = { NULL };
1200 + reftable_table_from_merged_table(&tab, reftable_stack_merged_table(st));
1201 + return reftable_table_read_ref(&tab, refname, ref);
1202 +}
1203 +
1204 +int reftable_stack_read_log(struct reftable_stack *st, const char *refname,
1205 + struct reftable_log_record *log)
1206 +{
1207 + struct reftable_iterator it = { NULL };
1208 + struct reftable_merged_table *mt = reftable_stack_merged_table(st);
1209 + int err = reftable_merged_table_seek_log(mt, &it, refname);
1210 + if (err)
1211 + goto done;
1212 +
1213 + err = reftable_iterator_next_log(&it, log);
1214 + if (err)
1215 + goto done;
1216 +
1217 + if (strcmp(log->refname, refname) ||
1218 + reftable_log_record_is_deletion(log)) {
1219 + err = 1;
1220 + goto done;
1221 + }
1222 +
1223 +done:
1224 + if (err) {
1225 + reftable_log_record_release(log);
1226 + }
1227 + reftable_iterator_destroy(&it);
1228 + return err;
1229 +}
1230 +
1231 +static int stack_check_addition(struct reftable_stack *st,
1232 + const char *new_tab_name)
1233 +{
1234 + int err = 0;
1235 + struct reftable_block_source src = { NULL };
1236 + struct reftable_reader *rd = NULL;
1237 + struct reftable_table tab = { NULL };
1238 + struct reftable_ref_record *refs = NULL;
1239 + struct reftable_iterator it = { NULL };
1240 + int cap = 0;
1241 + int len = 0;
1242 + int i = 0;
1243 +
1244 + if (st->config.skip_name_check)
1245 + return 0;
1246 +
1247 + err = reftable_block_source_from_file(&src, new_tab_name);
1248 + if (err < 0)
1249 + goto done;
1250 +
1251 + err = reftable_new_reader(&rd, &src, new_tab_name);
1252 + if (err < 0)
1253 + goto done;
1254 +
1255 + err = reftable_reader_seek_ref(rd, &it, "");
1256 + if (err > 0) {
1257 + err = 0;
1258 + goto done;
1259 + }
1260 + if (err < 0)
1261 + goto done;
1262 +
1263 + while (1) {
1264 + struct reftable_ref_record ref = { NULL };
1265 + err = reftable_iterator_next_ref(&it, &ref);
1266 + if (err > 0) {
1267 + break;
1268 + }
1269 + if (err < 0)
1270 + goto done;
1271 +
1272 + if (len >= cap) {
1273 + cap = 2 * cap + 1;
1274 + refs = reftable_realloc(refs, cap * sizeof(refs[0]));
1275 + }
1276 +
1277 + refs[len++] = ref;
1278 + }
1279 +
1280 + reftable_table_from_merged_table(&tab, reftable_stack_merged_table(st));
1281 +
1282 + err = validate_ref_record_addition(tab, refs, len);
1283 +
1284 +done:
1285 + for (i = 0; i < len; i++) {
1286 + reftable_ref_record_release(&refs[i]);
1287 + }
1288 +
1289 + free(refs);
1290 + reftable_iterator_destroy(&it);
1291 + reftable_reader_free(rd);
1292 + return err;
1293 +}
1294 +
1295 +static int is_table_name(const char *s)
1296 +{
1297 + const char *dot = strrchr(s, '.');
1298 + return dot && !strcmp(dot, ".ref");
1299 +}
1300 +
1301 +static void remove_maybe_stale_table(struct reftable_stack *st, uint64_t max,
1302 + const char *name)
1303 +{
1304 + int err = 0;
1305 + uint64_t update_idx = 0;
1306 + struct reftable_block_source src = { NULL };
1307 + struct reftable_reader *rd = NULL;
1308 + struct strbuf table_path = STRBUF_INIT;
1309 + stack_filename(&table_path, st, name);
1310 +
1311 + err = reftable_block_source_from_file(&src, table_path.buf);
1312 + if (err < 0)
1313 + goto done;
1314 +
1315 + err = reftable_new_reader(&rd, &src, name);
1316 + if (err < 0)
1317 + goto done;
1318 +
1319 + update_idx = reftable_reader_max_update_index(rd);
1320 + reftable_reader_free(rd);
1321 +
1322 + if (update_idx <= max) {
1323 + unlink(table_path.buf);
1324 + }
1325 +done:
1326 + strbuf_release(&table_path);
1327 +}
1328 +
1329 +static int reftable_stack_clean_locked(struct reftable_stack *st)
1330 +{
1331 + uint64_t max = reftable_merged_table_max_update_index(
1332 + reftable_stack_merged_table(st));
1333 + DIR *dir = opendir(st->reftable_dir);
1334 + struct dirent *d = NULL;
1335 + if (!dir) {
1336 + return REFTABLE_IO_ERROR;
1337 + }
1338 +
1339 + while ((d = readdir(dir))) {
1340 + int i = 0;
1341 + int found = 0;
1342 + if (!is_table_name(d->d_name))
1343 + continue;
1344 +
1345 + for (i = 0; !found && i < st->readers_len; i++) {
1346 + found = !strcmp(reader_name(st->readers[i]), d->d_name);
1347 + }
1348 + if (found)
1349 + continue;
1350 +
1351 + remove_maybe_stale_table(st, max, d->d_name);
1352 + }
1353 +
1354 + closedir(dir);
1355 + return 0;
1356 +}
1357 +
1358 +int reftable_stack_clean(struct reftable_stack *st)
1359 +{
1360 + struct reftable_addition *add = NULL;
1361 + int err = reftable_stack_new_addition(&add, st);
1362 + if (err < 0) {
1363 + goto done;
1364 + }
1365 +
1366 + err = reftable_stack_reload(st);
1367 + if (err < 0) {
1368 + goto done;
1369 + }
1370 +
1371 + err = reftable_stack_clean_locked(st);
1372 +
1373 +done:
1374 + reftable_addition_destroy(add);
1375 + return err;
1376 +}
1377 +
1378 +int reftable_stack_print_directory(const char *stackdir, uint32_t hash_id)
1379 +{
1380 + struct reftable_stack *stack = NULL;
1381 + struct reftable_write_options cfg = { .hash_id = hash_id };
1382 + struct reftable_merged_table *merged = NULL;
1383 + struct reftable_table table = { NULL };
1384 +
1385 + int err = reftable_new_stack(&stack, stackdir, cfg);
1386 + if (err < 0)
1387 + goto done;
1388 +
1389 + merged = reftable_stack_merged_table(stack);
1390 + reftable_table_from_merged_table(&table, merged);
1391 + err = reftable_table_print(&table);
1392 +done:
1393 + if (stack)
1394 + reftable_stack_destroy(stack);
1395 + return err;
1396 +}
reftable/stack.h new
+41
@@ -0,0 +1,41 @@
1 +/*
2 +Copyright 2020 Google LLC
3 +
4 +Use of this source code is governed by a BSD-style
5 +license that can be found in the LICENSE file or at
6 +https://developers.google.com/open-source/licenses/bsd
7 +*/
8 +
9 +#ifndef STACK_H
10 +#define STACK_H
11 +
12 +#include "system.h"
13 +#include "reftable-writer.h"
14 +#include "reftable-stack.h"
15 +
16 +struct reftable_stack {
17 + char *list_file;
18 + char *reftable_dir;
19 + int disable_auto_compact;
20 +
21 + struct reftable_write_options config;
22 +
23 + struct reftable_reader **readers;
24 + size_t readers_len;
25 + struct reftable_merged_table *merged;
26 + struct reftable_compaction_stats stats;
27 +};
28 +
29 +int read_lines(const char *filename, char ***lines);
30 +
31 +struct segment {
32 + int start, end;
33 + int log;
34 + uint64_t bytes;
35 +};
36 +
37 +int fastlog2(uint64_t sz);
38 +struct segment *sizes_to_segments(int *seglen, uint64_t *sizes, int n);
39 +struct segment suggest_compaction_segment(uint64_t *sizes, int n);
40 +
41 +#endif
reftable/stack_test.c new
+953
@@ -0,0 +1,953 @@
1 +/*
2 +Copyright 2020 Google LLC
3 +
4 +Use of this source code is governed by a BSD-style
5 +license that can be found in the LICENSE file or at
6 +https://developers.google.com/open-source/licenses/bsd
7 +*/
8 +
9 +#include "stack.h"
10 +
11 +#include "system.h"
12 +
13 +#include "reftable-reader.h"
14 +#include "merged.h"
15 +#include "basics.h"
16 +#include "constants.h"
17 +#include "record.h"
18 +#include "test_framework.h"
19 +#include "reftable-tests.h"
20 +
21 +#include <sys/types.h>
22 +#include <dirent.h>
23 +
24 +static void clear_dir(const char *dirname)
25 +{
26 + struct strbuf path = STRBUF_INIT;
27 + strbuf_addstr(&path, dirname);
28 + remove_dir_recursively(&path, 0);
29 + strbuf_release(&path);
30 +}
31 +
32 +static int count_dir_entries(const char *dirname)
33 +{
34 + DIR *dir = opendir(dirname);
35 + int len = 0;
36 + struct dirent *d;
37 + if (dir == NULL)
38 + return 0;
39 +
40 + while ((d = readdir(dir))) {
41 + if (!strcmp(d->d_name, "..") || !strcmp(d->d_name, "."))
42 + continue;
43 + len++;
44 + }
45 + closedir(dir);
46 + return len;
47 +}
48 +
49 +/*
50 + * Work linenumber into the tempdir, so we can see which tests forget to
51 + * cleanup.
52 + */
53 +static char *get_tmp_template(int linenumber)
54 +{
55 + const char *tmp = getenv("TMPDIR");
56 + static char template[1024];
57 + snprintf(template, sizeof(template) - 1, "%s/stack_test-%d.XXXXXX",
58 + tmp ? tmp : "/tmp", linenumber);
59 + return template;
60 +}
61 +
62 +static char *get_tmp_dir(int linenumber)
63 +{
64 + char *dir = get_tmp_template(linenumber);
65 + EXPECT(mkdtemp(dir));
66 + return dir;
67 +}
68 +
69 +static void test_read_file(void)
70 +{
71 + char *fn = get_tmp_template(__LINE__);
72 + int fd = mkstemp(fn);
73 + char out[1024] = "line1\n\nline2\nline3";
74 + int n, err;
75 + char **names = NULL;
76 + char *want[] = { "line1", "line2", "line3" };
77 + int i = 0;
78 +
79 + EXPECT(fd > 0);
80 + n = write(fd, out, strlen(out));
81 + EXPECT(n == strlen(out));
82 + err = close(fd);
83 + EXPECT(err >= 0);
84 +
85 + err = read_lines(fn, &names);
86 + EXPECT_ERR(err);
87 +
88 + for (i = 0; names[i]; i++) {
89 + EXPECT(0 == strcmp(want[i], names[i]));
90 + }
91 + free_names(names);
92 + remove(fn);
93 +}
94 +
95 +static void test_parse_names(void)
96 +{
97 + char buf[] = "line\n";
98 + char **names = NULL;
99 + parse_names(buf, strlen(buf), &names);
100 +
101 + EXPECT(NULL != names[0]);
102 + EXPECT(0 == strcmp(names[0], "line"));
103 + EXPECT(NULL == names[1]);
104 + free_names(names);
105 +}
106 +
107 +static void test_names_equal(void)
108 +{
109 + char *a[] = { "a", "b", "c", NULL };
110 + char *b[] = { "a", "b", "d", NULL };
111 + char *c[] = { "a", "b", NULL };
112 +
113 + EXPECT(names_equal(a, a));
114 + EXPECT(!names_equal(a, b));
115 + EXPECT(!names_equal(a, c));
116 +}
117 +
118 +static int write_test_ref(struct reftable_writer *wr, void *arg)
119 +{
120 + struct reftable_ref_record *ref = arg;
121 + reftable_writer_set_limits(wr, ref->update_index, ref->update_index);
122 + return reftable_writer_add_ref(wr, ref);
123 +}
124 +
125 +struct write_log_arg {
126 + struct reftable_log_record *log;
127 + uint64_t update_index;
128 +};
129 +
130 +static int write_test_log(struct reftable_writer *wr, void *arg)
131 +{
132 + struct write_log_arg *wla = arg;
133 +
134 + reftable_writer_set_limits(wr, wla->update_index, wla->update_index);
135 + return reftable_writer_add_log(wr, wla->log);
136 +}
137 +
138 +static void test_reftable_stack_add_one(void)
139 +{
140 + char *dir = get_tmp_dir(__LINE__);
141 +
142 + struct reftable_write_options cfg = { 0 };
143 + struct reftable_stack *st = NULL;
144 + int err;
145 + struct reftable_ref_record ref = {
146 + .refname = "HEAD",
147 + .update_index = 1,
148 + .value_type = REFTABLE_REF_SYMREF,
149 + .value.symref = "master",
150 + };
151 + struct reftable_ref_record dest = { NULL };
152 +
153 +
154 + err = reftable_new_stack(&st, dir, cfg);
155 + EXPECT_ERR(err);
156 +
157 + err = reftable_stack_add(st, &write_test_ref, &ref);
158 + EXPECT_ERR(err);
159 +
160 + err = reftable_stack_read_ref(st, ref.refname, &dest);
161 + EXPECT_ERR(err);
162 + EXPECT(0 == strcmp("master", dest.value.symref));
163 +
164 + printf("testing print functionality:\n");
165 + err = reftable_stack_print_directory(dir, GIT_SHA1_FORMAT_ID);
166 + EXPECT_ERR(err);
167 +
168 + err = reftable_stack_print_directory(dir, GIT_SHA256_FORMAT_ID);
169 + EXPECT(err == REFTABLE_FORMAT_ERROR);
170 +
171 + reftable_ref_record_release(&dest);
172 + reftable_stack_destroy(st);
173 + clear_dir(dir);
174 +}
175 +
176 +static void test_reftable_stack_uptodate(void)
177 +{
178 + struct reftable_write_options cfg = { 0 };
179 + struct reftable_stack *st1 = NULL;
180 + struct reftable_stack *st2 = NULL;
181 + char *dir = get_tmp_dir(__LINE__);
182 +
183 + int err;
184 + struct reftable_ref_record ref1 = {
185 + .refname = "HEAD",
186 + .update_index = 1,
187 + .value_type = REFTABLE_REF_SYMREF,
188 + .value.symref = "master",
189 + };
190 + struct reftable_ref_record ref2 = {
191 + .refname = "branch2",
192 + .update_index = 2,
193 + .value_type = REFTABLE_REF_SYMREF,
194 + .value.symref = "master",
195 + };
196 +
197 +
198 + /* simulate multi-process access to the same stack
199 + by creating two stacks for the same directory.
200 + */
201 + err = reftable_new_stack(&st1, dir, cfg);
202 + EXPECT_ERR(err);
203 +
204 + err = reftable_new_stack(&st2, dir, cfg);
205 + EXPECT_ERR(err);
206 +
207 + err = reftable_stack_add(st1, &write_test_ref, &ref1);
208 + EXPECT_ERR(err);
209 +
210 + err = reftable_stack_add(st2, &write_test_ref, &ref2);
211 + EXPECT(err == REFTABLE_LOCK_ERROR);
212 +
213 + err = reftable_stack_reload(st2);
214 + EXPECT_ERR(err);
215 +
216 + err = reftable_stack_add(st2, &write_test_ref, &ref2);
217 + EXPECT_ERR(err);
218 + reftable_stack_destroy(st1);
219 + reftable_stack_destroy(st2);
220 + clear_dir(dir);
221 +}
222 +
223 +static void test_reftable_stack_transaction_api(void)
224 +{
225 + char *dir = get_tmp_dir(__LINE__);
226 +
227 + struct reftable_write_options cfg = { 0 };
228 + struct reftable_stack *st = NULL;
229 + int err;
230 + struct reftable_addition *add = NULL;
231 +
232 + struct reftable_ref_record ref = {
233 + .refname = "HEAD",
234 + .update_index = 1,
235 + .value_type = REFTABLE_REF_SYMREF,
236 + .value.symref = "master",
237 + };
238 + struct reftable_ref_record dest = { NULL };
239 +
240 +
241 + err = reftable_new_stack(&st, dir, cfg);
242 + EXPECT_ERR(err);
243 +
244 + reftable_addition_destroy(add);
245 +
246 + err = reftable_stack_new_addition(&add, st);
247 + EXPECT_ERR(err);
248 +
249 + err = reftable_addition_add(add, &write_test_ref, &ref);
250 + EXPECT_ERR(err);
251 +
252 + err = reftable_addition_commit(add);
253 + EXPECT_ERR(err);
254 +
255 + reftable_addition_destroy(add);
256 +
257 + err = reftable_stack_read_ref(st, ref.refname, &dest);
258 + EXPECT_ERR(err);
259 + EXPECT(REFTABLE_REF_SYMREF == dest.value_type);
260 + EXPECT(0 == strcmp("master", dest.value.symref));
261 +
262 + reftable_ref_record_release(&dest);
263 + reftable_stack_destroy(st);
264 + clear_dir(dir);
265 +}
266 +
267 +static void test_reftable_stack_validate_refname(void)
268 +{
269 + struct reftable_write_options cfg = { 0 };
270 + struct reftable_stack *st = NULL;
271 + int err;
272 + char *dir = get_tmp_dir(__LINE__);
273 +
274 + int i;
275 + struct reftable_ref_record ref = {
276 + .refname = "a/b",
277 + .update_index = 1,
278 + .value_type = REFTABLE_REF_SYMREF,
279 + .value.symref = "master",
280 + };
281 + char *additions[] = { "a", "a/b/c" };
282 +
283 + err = reftable_new_stack(&st, dir, cfg);
284 + EXPECT_ERR(err);
285 +
286 + err = reftable_stack_add(st, &write_test_ref, &ref);
287 + EXPECT_ERR(err);
288 +
289 + for (i = 0; i < ARRAY_SIZE(additions); i++) {
290 + struct reftable_ref_record ref = {
291 + .refname = additions[i],
292 + .update_index = 1,
293 + .value_type = REFTABLE_REF_SYMREF,
294 + .value.symref = "master",
295 + };
296 +
297 + err = reftable_stack_add(st, &write_test_ref, &ref);
298 + EXPECT(err == REFTABLE_NAME_CONFLICT);
299 + }
300 +
301 + reftable_stack_destroy(st);
302 + clear_dir(dir);
303 +}
304 +
305 +static int write_error(struct reftable_writer *wr, void *arg)
306 +{
307 + return *((int *)arg);
308 +}
309 +
310 +static void test_reftable_stack_update_index_check(void)
311 +{
312 + char *dir = get_tmp_dir(__LINE__);
313 +
314 + struct reftable_write_options cfg = { 0 };
315 + struct reftable_stack *st = NULL;
316 + int err;
317 + struct reftable_ref_record ref1 = {
318 + .refname = "name1",
319 + .update_index = 1,
320 + .value_type = REFTABLE_REF_SYMREF,
321 + .value.symref = "master",
322 + };
323 + struct reftable_ref_record ref2 = {
324 + .refname = "name2",
325 + .update_index = 1,
326 + .value_type = REFTABLE_REF_SYMREF,
327 + .value.symref = "master",
328 + };
329 +
330 + err = reftable_new_stack(&st, dir, cfg);
331 + EXPECT_ERR(err);
332 +
333 + err = reftable_stack_add(st, &write_test_ref, &ref1);
334 + EXPECT_ERR(err);
335 +
336 + err = reftable_stack_add(st, &write_test_ref, &ref2);
337 + EXPECT(err == REFTABLE_API_ERROR);
338 + reftable_stack_destroy(st);
339 + clear_dir(dir);
340 +}
341 +
342 +static void test_reftable_stack_lock_failure(void)
343 +{
344 + char *dir = get_tmp_dir(__LINE__);
345 +
346 + struct reftable_write_options cfg = { 0 };
347 + struct reftable_stack *st = NULL;
348 + int err, i;
349 +
350 + err = reftable_new_stack(&st, dir, cfg);
351 + EXPECT_ERR(err);
352 + for (i = -1; i != REFTABLE_EMPTY_TABLE_ERROR; i--) {
353 + err = reftable_stack_add(st, &write_error, &i);
354 + EXPECT(err == i);
355 + }
356 +
357 + reftable_stack_destroy(st);
358 + clear_dir(dir);
359 +}
360 +
361 +static void test_reftable_stack_add(void)
362 +{
363 + int i = 0;
364 + int err = 0;
365 + struct reftable_write_options cfg = {
366 + .exact_log_message = 1,
367 + };
368 + struct reftable_stack *st = NULL;
369 + char *dir = get_tmp_dir(__LINE__);
370 +
371 + struct reftable_ref_record refs[2] = { { NULL } };
372 + struct reftable_log_record logs[2] = { { NULL } };
373 + int N = ARRAY_SIZE(refs);
374 +
375 +
376 + err = reftable_new_stack(&st, dir, cfg);
377 + EXPECT_ERR(err);
378 + st->disable_auto_compact = 1;
379 +
380 + for (i = 0; i < N; i++) {
381 + char buf[256];
382 + snprintf(buf, sizeof(buf), "branch%02d", i);
383 + refs[i].refname = xstrdup(buf);
384 + refs[i].update_index = i + 1;
385 + refs[i].value_type = REFTABLE_REF_VAL1;
386 + refs[i].value.val1 = reftable_malloc(GIT_SHA1_RAWSZ);
387 + set_test_hash(refs[i].value.val1, i);
388 +
389 + logs[i].refname = xstrdup(buf);
390 + logs[i].update_index = N + i + 1;
391 + logs[i].value_type = REFTABLE_LOG_UPDATE;
392 +
393 + logs[i].value.update.new_hash = reftable_malloc(GIT_SHA1_RAWSZ);
394 + logs[i].value.update.email = xstrdup("identity@invalid");
395 + set_test_hash(logs[i].value.update.new_hash, i);
396 + }
397 +
398 + for (i = 0; i < N; i++) {
399 + int err = reftable_stack_add(st, &write_test_ref, &refs[i]);
400 + EXPECT_ERR(err);
401 + }
402 +
403 + for (i = 0; i < N; i++) {
404 + struct write_log_arg arg = {
405 + .log = &logs[i],
406 + .update_index = reftable_stack_next_update_index(st),
407 + };
408 + int err = reftable_stack_add(st, &write_test_log, &arg);
409 + EXPECT_ERR(err);
410 + }
411 +
412 + err = reftable_stack_compact_all(st, NULL);
413 + EXPECT_ERR(err);
414 +
415 + for (i = 0; i < N; i++) {
416 + struct reftable_ref_record dest = { NULL };
417 +
418 + int err = reftable_stack_read_ref(st, refs[i].refname, &dest);
419 + EXPECT_ERR(err);
420 + EXPECT(reftable_ref_record_equal(&dest, refs + i,
421 + GIT_SHA1_RAWSZ));
422 + reftable_ref_record_release(&dest);
423 + }
424 +
425 + for (i = 0; i < N; i++) {
426 + struct reftable_log_record dest = { NULL };
427 + int err = reftable_stack_read_log(st, refs[i].refname, &dest);
428 + EXPECT_ERR(err);
429 + EXPECT(reftable_log_record_equal(&dest, logs + i,
430 + GIT_SHA1_RAWSZ));
431 + reftable_log_record_release(&dest);
432 + }
433 +
434 + /* cleanup */
435 + reftable_stack_destroy(st);
436 + for (i = 0; i < N; i++) {
437 + reftable_ref_record_release(&refs[i]);
438 + reftable_log_record_release(&logs[i]);
439 + }
440 + clear_dir(dir);
441 +}
442 +
443 +static void test_reftable_stack_log_normalize(void)
444 +{
445 + int err = 0;
446 + struct reftable_write_options cfg = {
447 + 0,
448 + };
449 + struct reftable_stack *st = NULL;
450 + char *dir = get_tmp_dir(__LINE__);
451 +
452 + uint8_t h1[GIT_SHA1_RAWSZ] = { 0x01 }, h2[GIT_SHA1_RAWSZ] = { 0x02 };
453 +
454 + struct reftable_log_record input = { .refname = "branch",
455 + .update_index = 1,
456 + .value_type = REFTABLE_LOG_UPDATE,
457 + .value = { .update = {
458 + .new_hash = h1,
459 + .old_hash = h2,
460 + } } };
461 + struct reftable_log_record dest = {
462 + .update_index = 0,
463 + };
464 + struct write_log_arg arg = {
465 + .log = &input,
466 + .update_index = 1,
467 + };
468 +
469 + err = reftable_new_stack(&st, dir, cfg);
470 + EXPECT_ERR(err);
471 +
472 + input.value.update.message = "one\ntwo";
473 + err = reftable_stack_add(st, &write_test_log, &arg);
474 + EXPECT(err == REFTABLE_API_ERROR);
475 +
476 + input.value.update.message = "one";
477 + err = reftable_stack_add(st, &write_test_log, &arg);
478 + EXPECT_ERR(err);
479 +
480 + err = reftable_stack_read_log(st, input.refname, &dest);
481 + EXPECT_ERR(err);
482 + EXPECT(0 == strcmp(dest.value.update.message, "one\n"));
483 +
484 + input.value.update.message = "two\n";
485 + arg.update_index = 2;
486 + err = reftable_stack_add(st, &write_test_log, &arg);
487 + EXPECT_ERR(err);
488 + err = reftable_stack_read_log(st, input.refname, &dest);
489 + EXPECT_ERR(err);
490 + EXPECT(0 == strcmp(dest.value.update.message, "two\n"));
491 +
492 + /* cleanup */
493 + reftable_stack_destroy(st);
494 + reftable_log_record_release(&dest);
495 + clear_dir(dir);
496 +}
497 +
498 +static void test_reftable_stack_tombstone(void)
499 +{
500 + int i = 0;
501 + char *dir = get_tmp_dir(__LINE__);
502 +
503 + struct reftable_write_options cfg = { 0 };
504 + struct reftable_stack *st = NULL;
505 + int err;
506 + struct reftable_ref_record refs[2] = { { NULL } };
507 + struct reftable_log_record logs[2] = { { NULL } };
508 + int N = ARRAY_SIZE(refs);
509 + struct reftable_ref_record dest = { NULL };
510 + struct reftable_log_record log_dest = { NULL };
511 +
512 +
513 + err = reftable_new_stack(&st, dir, cfg);
514 + EXPECT_ERR(err);
515 +
516 + /* even entries add the refs, odd entries delete them. */
517 + for (i = 0; i < N; i++) {
518 + const char *buf = "branch";
519 + refs[i].refname = xstrdup(buf);
520 + refs[i].update_index = i + 1;
521 + if (i % 2 == 0) {
522 + refs[i].value_type = REFTABLE_REF_VAL1;
523 + refs[i].value.val1 = reftable_malloc(GIT_SHA1_RAWSZ);
524 + set_test_hash(refs[i].value.val1, i);
525 + }
526 +
527 + logs[i].refname = xstrdup(buf);
528 + /* update_index is part of the key. */
529 + logs[i].update_index = 42;
530 + if (i % 2 == 0) {
531 + logs[i].value_type = REFTABLE_LOG_UPDATE;
532 + logs[i].value.update.new_hash =
533 + reftable_malloc(GIT_SHA1_RAWSZ);
534 + set_test_hash(logs[i].value.update.new_hash, i);
535 + logs[i].value.update.email =
536 + xstrdup("identity@invalid");
537 + }
538 + }
539 + for (i = 0; i < N; i++) {
540 + int err = reftable_stack_add(st, &write_test_ref, &refs[i]);
541 + EXPECT_ERR(err);
542 + }
543 +
544 + for (i = 0; i < N; i++) {
545 + struct write_log_arg arg = {
546 + .log = &logs[i],
547 + .update_index = reftable_stack_next_update_index(st),
548 + };
549 + int err = reftable_stack_add(st, &write_test_log, &arg);
550 + EXPECT_ERR(err);
551 + }
552 +
553 + err = reftable_stack_read_ref(st, "branch", &dest);
554 + EXPECT(err == 1);
555 + reftable_ref_record_release(&dest);
556 +
557 + err = reftable_stack_read_log(st, "branch", &log_dest);
558 + EXPECT(err == 1);
559 + reftable_log_record_release(&log_dest);
560 +
561 + err = reftable_stack_compact_all(st, NULL);
562 + EXPECT_ERR(err);
563 +
564 + err = reftable_stack_read_ref(st, "branch", &dest);
565 + EXPECT(err == 1);
566 +
567 + err = reftable_stack_read_log(st, "branch", &log_dest);
568 + EXPECT(err == 1);
569 + reftable_ref_record_release(&dest);
570 + reftable_log_record_release(&log_dest);
571 +
572 + /* cleanup */
573 + reftable_stack_destroy(st);
574 + for (i = 0; i < N; i++) {
575 + reftable_ref_record_release(&refs[i]);
576 + reftable_log_record_release(&logs[i]);
577 + }
578 + clear_dir(dir);
579 +}
580 +
581 +static void test_reftable_stack_hash_id(void)
582 +{
583 + char *dir = get_tmp_dir(__LINE__);
584 +
585 + struct reftable_write_options cfg = { 0 };
586 + struct reftable_stack *st = NULL;
587 + int err;
588 +
589 + struct reftable_ref_record ref = {
590 + .refname = "master",
591 + .value_type = REFTABLE_REF_SYMREF,
592 + .value.symref = "target",
593 + .update_index = 1,
594 + };
595 + struct reftable_write_options cfg32 = { .hash_id = GIT_SHA256_FORMAT_ID };
596 + struct reftable_stack *st32 = NULL;
597 + struct reftable_write_options cfg_default = { 0 };
598 + struct reftable_stack *st_default = NULL;
599 + struct reftable_ref_record dest = { NULL };
600 +
601 + err = reftable_new_stack(&st, dir, cfg);
602 + EXPECT_ERR(err);
603 +
604 + err = reftable_stack_add(st, &write_test_ref, &ref);
605 + EXPECT_ERR(err);
606 +
607 + /* can't read it with the wrong hash ID. */
608 + err = reftable_new_stack(&st32, dir, cfg32);
609 + EXPECT(err == REFTABLE_FORMAT_ERROR);
610 +
611 + /* check that we can read it back with default config too. */
612 + err = reftable_new_stack(&st_default, dir, cfg_default);
613 + EXPECT_ERR(err);
614 +
615 + err = reftable_stack_read_ref(st_default, "master", &dest);
616 + EXPECT_ERR(err);
617 +
618 + EXPECT(reftable_ref_record_equal(&ref, &dest, GIT_SHA1_RAWSZ));
619 + reftable_ref_record_release(&dest);
620 + reftable_stack_destroy(st);
621 + reftable_stack_destroy(st_default);
622 + clear_dir(dir);
623 +}
624 +
625 +static void test_log2(void)
626 +{
627 + EXPECT(1 == fastlog2(3));
628 + EXPECT(2 == fastlog2(4));
629 + EXPECT(2 == fastlog2(5));
630 +}
631 +
632 +static void test_sizes_to_segments(void)
633 +{
634 + uint64_t sizes[] = { 2, 3, 4, 5, 7, 9 };
635 + /* .................0 1 2 3 4 5 */
636 +
637 + int seglen = 0;
638 + struct segment *segs =
639 + sizes_to_segments(&seglen, sizes, ARRAY_SIZE(sizes));
640 + EXPECT(segs[2].log == 3);
641 + EXPECT(segs[2].start == 5);
642 + EXPECT(segs[2].end == 6);
643 +
644 + EXPECT(segs[1].log == 2);
645 + EXPECT(segs[1].start == 2);
646 + EXPECT(segs[1].end == 5);
647 + reftable_free(segs);
648 +}
649 +
650 +static void test_sizes_to_segments_empty(void)
651 +{
652 + int seglen = 0;
653 + struct segment *segs = sizes_to_segments(&seglen, NULL, 0);
654 + EXPECT(seglen == 0);
655 + reftable_free(segs);
656 +}
657 +
658 +static void test_sizes_to_segments_all_equal(void)
659 +{
660 + uint64_t sizes[] = { 5, 5 };
661 +
662 + int seglen = 0;
663 + struct segment *segs =
664 + sizes_to_segments(&seglen, sizes, ARRAY_SIZE(sizes));
665 + EXPECT(seglen == 1);
666 + EXPECT(segs[0].start == 0);
667 + EXPECT(segs[0].end == 2);
668 + reftable_free(segs);
669 +}
670 +
671 +static void test_suggest_compaction_segment(void)
672 +{
673 + uint64_t sizes[] = { 128, 64, 17, 16, 9, 9, 9, 16, 16 };
674 + /* .................0 1 2 3 4 5 6 */
675 + struct segment min =
676 + suggest_compaction_segment(sizes, ARRAY_SIZE(sizes));
677 + EXPECT(min.start == 2);
678 + EXPECT(min.end == 7);
679 +}
680 +
681 +static void test_suggest_compaction_segment_nothing(void)
682 +{
683 + uint64_t sizes[] = { 64, 32, 16, 8, 4, 2 };
684 + struct segment result =
685 + suggest_compaction_segment(sizes, ARRAY_SIZE(sizes));
686 + EXPECT(result.start == result.end);
687 +}
688 +
689 +static void test_reflog_expire(void)
690 +{
691 + char *dir = get_tmp_dir(__LINE__);
692 +
693 + struct reftable_write_options cfg = { 0 };
694 + struct reftable_stack *st = NULL;
695 + struct reftable_log_record logs[20] = { { NULL } };
696 + int N = ARRAY_SIZE(logs) - 1;
697 + int i = 0;
698 + int err;
699 + struct reftable_log_expiry_config expiry = {
700 + .time = 10,
701 + };
702 + struct reftable_log_record log = { NULL };
703 +
704 +
705 + err = reftable_new_stack(&st, dir, cfg);
706 + EXPECT_ERR(err);
707 +
708 + for (i = 1; i <= N; i++) {
709 + char buf[256];
710 + snprintf(buf, sizeof(buf), "branch%02d", i);
711 +
712 + logs[i].refname = xstrdup(buf);
713 + logs[i].update_index = i;
714 + logs[i].value_type = REFTABLE_LOG_UPDATE;
715 + logs[i].value.update.time = i;
716 + logs[i].value.update.new_hash = reftable_malloc(GIT_SHA1_RAWSZ);
717 + logs[i].value.update.email = xstrdup("identity@invalid");
718 + set_test_hash(logs[i].value.update.new_hash, i);
719 + }
720 +
721 + for (i = 1; i <= N; i++) {
722 + struct write_log_arg arg = {
723 + .log = &logs[i],
724 + .update_index = reftable_stack_next_update_index(st),
725 + };
726 + int err = reftable_stack_add(st, &write_test_log, &arg);
727 + EXPECT_ERR(err);
728 + }
729 +
730 + err = reftable_stack_compact_all(st, NULL);
731 + EXPECT_ERR(err);
732 +
733 + err = reftable_stack_compact_all(st, &expiry);
734 + EXPECT_ERR(err);
735 +
736 + err = reftable_stack_read_log(st, logs[9].refname, &log);
737 + EXPECT(err == 1);
738 +
739 + err = reftable_stack_read_log(st, logs[11].refname, &log);
740 + EXPECT_ERR(err);
741 +
742 + expiry.min_update_index = 15;
743 + err = reftable_stack_compact_all(st, &expiry);
744 + EXPECT_ERR(err);
745 +
746 + err = reftable_stack_read_log(st, logs[14].refname, &log);
747 + EXPECT(err == 1);
748 +
749 + err = reftable_stack_read_log(st, logs[16].refname, &log);
750 + EXPECT_ERR(err);
751 +
752 + /* cleanup */
753 + reftable_stack_destroy(st);
754 + for (i = 0; i <= N; i++) {
755 + reftable_log_record_release(&logs[i]);
756 + }
757 + clear_dir(dir);
758 + reftable_log_record_release(&log);
759 +}
760 +
761 +static int write_nothing(struct reftable_writer *wr, void *arg)
762 +{
763 + reftable_writer_set_limits(wr, 1, 1);
764 + return 0;
765 +}
766 +
767 +static void test_empty_add(void)
768 +{
769 + struct reftable_write_options cfg = { 0 };
770 + struct reftable_stack *st = NULL;
771 + int err;
772 + char *dir = get_tmp_dir(__LINE__);
773 +
774 + struct reftable_stack *st2 = NULL;
775 +
776 +
777 + err = reftable_new_stack(&st, dir, cfg);
778 + EXPECT_ERR(err);
779 +
780 + err = reftable_stack_add(st, &write_nothing, NULL);
781 + EXPECT_ERR(err);
782 +
783 + err = reftable_new_stack(&st2, dir, cfg);
784 + EXPECT_ERR(err);
785 + clear_dir(dir);
786 + reftable_stack_destroy(st);
787 + reftable_stack_destroy(st2);
788 +}
789 +
790 +static void test_reftable_stack_auto_compaction(void)
791 +{
792 + struct reftable_write_options cfg = { 0 };
793 + struct reftable_stack *st = NULL;
794 + char *dir = get_tmp_dir(__LINE__);
795 +
796 + int err, i;
797 + int N = 100;
798 +
799 + err = reftable_new_stack(&st, dir, cfg);
800 + EXPECT_ERR(err);
801 +
802 + st->disable_auto_compact = 1; /* call manually below for coverage. */
803 + for (i = 0; i < N; i++) {
804 + char name[100];
805 + struct reftable_ref_record ref = {
806 + .refname = name,
807 + .update_index = reftable_stack_next_update_index(st),
808 + .value_type = REFTABLE_REF_SYMREF,
809 + .value.symref = "master",
810 + };
811 + snprintf(name, sizeof(name), "branch%04d", i);
812 +
813 + err = reftable_stack_add(st, &write_test_ref, &ref);
814 + EXPECT_ERR(err);
815 +
816 + err = reftable_stack_auto_compact(st);
817 + EXPECT(i < 3 || st->merged->stack_len < 2 * fastlog2(i));
818 + }
819 +
820 + EXPECT(reftable_stack_compaction_stats(st)->entries_written <
821 + (uint64_t)(N * fastlog2(N)));
822 +
823 + reftable_stack_destroy(st);
824 + clear_dir(dir);
825 +}
826 +
827 +static void test_reftable_stack_compaction_concurrent(void)
828 +{
829 + struct reftable_write_options cfg = { 0 };
830 + struct reftable_stack *st1 = NULL, *st2 = NULL;
831 + char *dir = get_tmp_dir(__LINE__);
832 +
833 + int err, i;
834 + int N = 3;
835 +
836 + err = reftable_new_stack(&st1, dir, cfg);
837 + EXPECT_ERR(err);
838 +
839 + for (i = 0; i < N; i++) {
840 + char name[100];
841 + struct reftable_ref_record ref = {
842 + .refname = name,
843 + .update_index = reftable_stack_next_update_index(st1),
844 + .value_type = REFTABLE_REF_SYMREF,
845 + .value.symref = "master",
846 + };
847 + snprintf(name, sizeof(name), "branch%04d", i);
848 +
849 + err = reftable_stack_add(st1, &write_test_ref, &ref);
850 + EXPECT_ERR(err);
851 + }
852 +
853 + err = reftable_new_stack(&st2, dir, cfg);
854 + EXPECT_ERR(err);
855 +
856 + err = reftable_stack_compact_all(st1, NULL);
857 + EXPECT_ERR(err);
858 +
859 + reftable_stack_destroy(st1);
860 + reftable_stack_destroy(st2);
861 +
862 + EXPECT(count_dir_entries(dir) == 2);
863 + clear_dir(dir);
864 +}
865 +
866 +static void unclean_stack_close(struct reftable_stack *st)
867 +{
868 + /* break abstraction boundary to simulate unclean shutdown. */
869 + int i = 0;
870 + for (; i < st->readers_len; i++) {
871 + reftable_reader_free(st->readers[i]);
872 + }
873 + st->readers_len = 0;
874 + FREE_AND_NULL(st->readers);
875 +}
876 +
877 +static void test_reftable_stack_compaction_concurrent_clean(void)
878 +{
879 + struct reftable_write_options cfg = { 0 };
880 + struct reftable_stack *st1 = NULL, *st2 = NULL, *st3 = NULL;
881 + char *dir = get_tmp_dir(__LINE__);
882 +
883 + int err, i;
884 + int N = 3;
885 +
886 + err = reftable_new_stack(&st1, dir, cfg);
887 + EXPECT_ERR(err);
888 +
889 + for (i = 0; i < N; i++) {
890 + char name[100];
891 + struct reftable_ref_record ref = {
892 + .refname = name,
893 + .update_index = reftable_stack_next_update_index(st1),
894 + .value_type = REFTABLE_REF_SYMREF,
895 + .value.symref = "master",
896 + };
897 + snprintf(name, sizeof(name), "branch%04d", i);
898 +
899 + err = reftable_stack_add(st1, &write_test_ref, &ref);
900 + EXPECT_ERR(err);
901 + }
902 +
903 + err = reftable_new_stack(&st2, dir, cfg);
904 + EXPECT_ERR(err);
905 +
906 + err = reftable_stack_compact_all(st1, NULL);
907 + EXPECT_ERR(err);
908 +
909 + unclean_stack_close(st1);
910 + unclean_stack_close(st2);
911 +
912 + err = reftable_new_stack(&st3, dir, cfg);
913 + EXPECT_ERR(err);
914 +
915 + err = reftable_stack_clean(st3);
916 + EXPECT_ERR(err);
917 + EXPECT(count_dir_entries(dir) == 2);
918 +
919 + reftable_stack_destroy(st1);
920 + reftable_stack_destroy(st2);
921 + reftable_stack_destroy(st3);
922 +
923 + clear_dir(dir);
924 +}
925 +
926 +int stack_test_main(int argc, const char *argv[])
927 +{
928 + RUN_TEST(test_empty_add);
929 + RUN_TEST(test_log2);
930 + RUN_TEST(test_names_equal);
931 + RUN_TEST(test_parse_names);
932 + RUN_TEST(test_read_file);
933 + RUN_TEST(test_reflog_expire);
934 + RUN_TEST(test_reftable_stack_add);
935 + RUN_TEST(test_reftable_stack_add_one);
936 + RUN_TEST(test_reftable_stack_auto_compaction);
937 + RUN_TEST(test_reftable_stack_compaction_concurrent);
938 + RUN_TEST(test_reftable_stack_compaction_concurrent_clean);
939 + RUN_TEST(test_reftable_stack_hash_id);
940 + RUN_TEST(test_reftable_stack_lock_failure);
941 + RUN_TEST(test_reftable_stack_log_normalize);
942 + RUN_TEST(test_reftable_stack_tombstone);
943 + RUN_TEST(test_reftable_stack_transaction_api);
944 + RUN_TEST(test_reftable_stack_update_index_check);
945 + RUN_TEST(test_reftable_stack_uptodate);
946 + RUN_TEST(test_reftable_stack_validate_refname);
947 + RUN_TEST(test_sizes_to_segments);
948 + RUN_TEST(test_sizes_to_segments_all_equal);
949 + RUN_TEST(test_sizes_to_segments_empty);
950 + RUN_TEST(test_suggest_compaction_segment);
951 + RUN_TEST(test_suggest_compaction_segment_nothing);
952 + return 0;
953 +}
t/helper/test-reftable.c
+1
@@ -10,6 +10,7 @@ int cmd__reftable(int argc, const char **argv)
10 record_test_main(argc, argv);
11 refname_test_main(argc, argv);
12 readwrite_test_main(argc, argv);
13 + stack_test_main(argc, argv);
14 tree_test_main(argc, argv);
15 return 0;
16 }