@matheusbsbs / matheus-bagliano / commits / 5c086453ff

reftable/stack: perform auto-compaction with transactional interface

Whenever updating references or reflog entries in the reftable stack, we need to add a new table to the stack, thus growing the stack's length by one. The stack can grow to become quite long rather quickly, leading to performance issues when trying to read records. But besides performance issues, this can also lead to exhaustion of file descriptors very rapidly as every single table requires a separate descriptor when opening the stack. While git-pack-refs(1) fixes this issue for us by merging the tables, it runs too irregularly to keep the length of the stack within reasonable limits. This is why the reftable stack has an auto-compaction mechanism: `reftable_stack_add()` will call `reftable_stack_auto_compact()` after its added the new table, which will auto-compact the stack as required. But while this logic works alright for `reftable_stack_add()`, we do not do the same in `reftable_addition_commit()`, which is the transactional equivalent to the former function that allows us to write multiple updates to the stack atomically. Consequentially, we will easily run into file descriptor exhaustion in code paths that use many separate transactions like e.g. non-atomic fetches. Fix this issue by calling `reftable_stack_auto_compact()`. Signed-off-by: Patrick Steinhardt <ps@pks.im> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Patrick Steinhardt committed Dec 11, 2023 at 10:07 UTC 5c086453ff32a2ca07ac455594d59a9943b5d113

2 files changed +62

reftable/stack.c

@@ -584,6 +584,12 @@ int reftable_addition_commit(struct reftable_addition *add)

584 add->new_tables_len = 0;

585

586 err = reftable_stack_reload(add->stack);

587 + if (err)

588 + goto done;

589 +

590 + if (!add->stack->disable_auto_compact)

591 + err = reftable_stack_auto_compact(add->stack);

592 +

593 done:

594 reftable_addition_close(add);

595 return err;

reftable/stack_test.c

+56

@@ -289,6 +289,61 @@ static void test_reftable_stack_transaction_api(void)

289 clear_dir(dir);

290 }

291

292 +static void test_reftable_stack_transaction_api_performs_auto_compaction(void)

293 +{

294 + char *dir = get_tmp_dir(__LINE__);

295 + struct reftable_write_options cfg = {0};

296 + struct reftable_addition *add = NULL;

297 + struct reftable_stack *st = NULL;

298 + int i, n = 20, err;

299 +

300 + err = reftable_new_stack(&st, dir, cfg);

301 + EXPECT_ERR(err);

302 +

303 + for (i = 0; i <= n; i++) {

304 + struct reftable_ref_record ref = {

305 + .update_index = reftable_stack_next_update_index(st),

306 + .value_type = REFTABLE_REF_SYMREF,

307 + .value.symref = "master",

308 + };

309 + char name[100];

310 +

311 + snprintf(name, sizeof(name), "branch%04d", i);

312 + ref.refname = name;

313 +

314 + /*

315 + * Disable auto-compaction for all but the last runs. Like this

316 + * we can ensure that we indeed honor this setting and have

317 + * better control over when exactly auto compaction runs.

318 + */

319 + st->disable_auto_compact = i != n;

320 +

321 + err = reftable_stack_new_addition(&add, st);

322 + EXPECT_ERR(err);

323 +

324 + err = reftable_addition_add(add, &write_test_ref, &ref);

325 + EXPECT_ERR(err);

326 +

327 + err = reftable_addition_commit(add);

328 + EXPECT_ERR(err);

329 +

330 + reftable_addition_destroy(add);

331 +

332 + /*

333 + * The stack length should grow continuously for all runs where

334 + * auto compaction is disabled. When enabled, we should merge

335 + * all tables in the stack.

336 + */

337 + if (i != n)

338 + EXPECT(st->merged->stack_len == i + 1);

339 + else

340 + EXPECT(st->merged->stack_len == 1);

341 + }

342 +

343 + reftable_stack_destroy(st);

344 + clear_dir(dir);

345 +}

346 +

347 static void test_reftable_stack_validate_refname(void)

348 {

349 struct reftable_write_options cfg = { 0 };

@@ -1016,6 +1071,7 @@ int stack_test_main(int argc, const char *argv[])

1071 RUN_TEST(test_reftable_stack_log_normalize);

1072 RUN_TEST(test_reftable_stack_tombstone);

1073 RUN_TEST(test_reftable_stack_transaction_api);

1074 + RUN_TEST(test_reftable_stack_transaction_api_performs_auto_compaction);

1075 RUN_TEST(test_reftable_stack_update_index_check);

1076 RUN_TEST(test_reftable_stack_uptodate);

1077 RUN_TEST(test_reftable_stack_validate_refname);