@matheusbsbs / matheus-bagliano / commits / bc39b6a796

refs/reftable: introduce "reftable.lockTimeout"

When multiple concurrent processes try to update references in a repository they may try to lock the same lockfiles. This can happen even when the updates are non-conflicting and can both be applied, so it doesn't always make sense to abort the transaction immediately. Both the "loose" and "packed" backends thus have a grace period that they wait for the lock to be released that can be controlled via the config values "core.filesRefLockTimeout" and "core.packedRefsTimeout", respectively. The reftable backend doesn't have such a setting yet and instead fails immediately when it sees such a lock. But the exact same concepts apply here as they do apply to the other backends. Introduce a new "reftable.lockTimeout" config that controls how long we may wait for a "tables.list" lock to be released. The default value of this config is 100ms, which is the same default as we have it for the "loose" backend. Note that even though we also lock individual tables, this config really only applies to the "tables.list" file. This is because individual tables are only ever locked when we already hold the "tables.list" lock during compaction. When we observe such a lock we in fact do not want to compact the table at all because it is already in the process of being compacted by a concurrent process. So applying the same timeout here would not make any sense and only delay progress. Signed-off-by: Patrick Steinhardt <ps@pks.im> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Patrick Steinhardt committed Sep 24, 2024 at 07:33 UTC bc39b6a796eb707c34ffb759d50a75f96313f26c
5 files changed +66 -6
Documentation/config/reftable.txt
+8
@@ -46,3 +46,11 @@ reftable.geometricFactor::
46 By default, the geometric sequence uses a factor of 2, meaning that for any
47 table, the next-biggest table must at least be twice as big. A maximum factor
48 of 256 is supported.
49 +
50 +reftable.lockTimeout::
51 + Whenever the reftable backend appends a new table to the stack, it has
52 + to lock the central "tables.list" file before updating it. This config
53 + controls how long the process will wait to acquire the lock in case
54 + another process has already acquired it. Value 0 means not to retry at
55 + all; -1 means to try indefinitely. Default is 100 (i.e., retry for
56 + 100ms).
refs/reftable-backend.c
+8
@@ -256,6 +256,13 @@ static int reftable_be_config(const char *var, const char *value,
256 if (factor > UINT8_MAX)
257 die("reftable geometric factor cannot exceed %u", (unsigned)UINT8_MAX);
258 opts->auto_compaction_factor = factor;
259 + } else if (!strcmp(var, "reftable.locktimeout")) {
260 + int64_t lock_timeout = git_config_int64(var, value, ctx->kvi);
261 + if (lock_timeout > LONG_MAX)
262 + die("reftable lock timeout cannot exceed %"PRIdMAX, (intmax_t)LONG_MAX);
263 + if (lock_timeout < 0 && lock_timeout != -1)
264 + die("reftable lock timeout does not support negative values other than -1");
265 + opts->lock_timeout_ms = lock_timeout;
266 }
267
268 return 0;
@@ -281,6 +288,7 @@ static struct ref_store *reftable_be_init(struct repository *repo,
288 refs->write_options.default_permissions = calc_shared_perm(0666 & ~mask);
289 refs->write_options.disable_auto_compact =
290 !git_env_bool("GIT_TEST_REFTABLE_AUTOCOMPACTION", 1);
291 + refs->write_options.lock_timeout_ms = 100;
292
293 git_config(reftable_be_config, &refs->write_options);
294
reftable/reftable-writer.h
+11
@@ -51,6 +51,17 @@ struct reftable_write_options {
51 * tables to compact. Defaults to 2 if unset.
52 */
53 uint8_t auto_compaction_factor;
54 +
55 + /*
56 + * The number of milliseconds to wait when trying to lock "tables.list".
57 + * Note that this does not apply to locking individual tables, as these
58 + * should only ever be locked when already holding the "tables.list"
59 + * lock.
60 + *
61 + * Passing 0 will fail immediately when the file is locked, passing a
62 + * negative value will cause us to block indefinitely.
63 + */
64 + long lock_timeout_ms;
65 };
66
67 /* reftable_block_stats holds statistics for a single block type */
reftable/stack.c
+12 -6
@@ -603,8 +603,10 @@ static int reftable_stack_init_addition(struct reftable_addition *add,
603
604 add->stack = st;
605
606 - err = hold_lock_file_for_update(&add->tables_list_lock, st->list_file,
607 - LOCK_NO_DEREF);
606 + err = hold_lock_file_for_update_timeout(&add->tables_list_lock,
607 + st->list_file,
608 + LOCK_NO_DEREF,
609 + st->opts.lock_timeout_ms);
610 if (err < 0) {
611 if (errno == EEXIST) {
612 err = REFTABLE_LOCK_ERROR;
@@ -1056,8 +1058,10 @@ static int stack_compact_range(struct reftable_stack *st,
1058 * Hold the lock so that we can read "tables.list" and lock all tables
1059 * which are part of the user-specified range.
1060 */
1059 - err = hold_lock_file_for_update(&tables_list_lock, st->list_file,
1060 - LOCK_NO_DEREF);
1061 + err = hold_lock_file_for_update_timeout(&tables_list_lock,
1062 + st->list_file,
1063 + LOCK_NO_DEREF,
1064 + st->opts.lock_timeout_ms);
1065 if (err < 0) {
1066 if (errno == EEXIST)
1067 err = REFTABLE_LOCK_ERROR;
@@ -1156,8 +1160,10 @@ static int stack_compact_range(struct reftable_stack *st,
1160 * "tables.list". We'll then replace the compacted range of tables with
1161 * the new table.
1162 */
1159 - err = hold_lock_file_for_update(&tables_list_lock, st->list_file,
1160 - LOCK_NO_DEREF);
1163 + err = hold_lock_file_for_update_timeout(&tables_list_lock,
1164 + st->list_file,
1165 + LOCK_NO_DEREF,
1166 + st->opts.lock_timeout_ms);
1167 if (err < 0) {
1168 if (errno == EEXIST)
1169 err = REFTABLE_LOCK_ERROR;
t/t0610-reftable-basics.sh
+27
@@ -423,6 +423,33 @@ test_expect_success 'ref transaction: fails gracefully when auto compaction fail
423 )
424 '
425
426 +test_expect_success 'ref transaction: timeout acquiring tables.list lock' '
427 + test_when_finished "rm -rf repo" &&
428 + git init repo &&
429 + (
430 + cd repo &&
431 + test_commit initial &&
432 + >.git/reftable/tables.list.lock &&
433 + test_must_fail git update-ref refs/heads/branch HEAD 2>err &&
434 + test_grep "cannot lock references" err
435 + )
436 +'
437 +
438 +test_expect_success 'ref transaction: retry acquiring tables.list lock' '
439 + test_when_finished "rm -rf repo" &&
440 + git init repo &&
441 + (
442 + cd repo &&
443 + test_commit initial &&
444 + LOCK=.git/reftable/tables.list.lock &&
445 + >$LOCK &&
446 + {
447 + ( sleep 1 && rm -f $LOCK ) &
448 + } &&
449 + git -c reftable.lockTimeout=5000 update-ref refs/heads/branch HEAD
450 + )
451 +'
452 +
453 test_expect_success 'pack-refs: compacts tables' '
454 test_when_finished "rm -rf repo" &&
455 git init repo &&