@matheusbsbs / matheus-bagliano / commits / bc67b4ab5f

reftable: write correct max_update_index to header

In 297c09eabb (refs: allow multiple reflog entries for the same refname, 2024-12-16), the reftable backend learned to handle multiple reflog entries within the same transaction. This was done modifying the `update_index` for reflogs with multiple indices. During writing the logs, the `max_update_index` of the writer was modified to ensure the limits were raised to the modified `update_index`s. However, since ref entries are written before the modification to the `max_update_index`, if there are multiple blocks to be written, the reftable backend writes the header with the old `max_update_index`. When all logs are finally written, the footer will be written with the new `min_update_index`. This causes a mismatch between the header and the footer and causes the reftable file to be corrupted. The existing tests only spawn a single block and since headers are lazily written with the first block, the tests didn't capture this bug. To fix the issue, the appropriate `max_update_index` limit must be set even before the first block is written. Add a `max_index` field to the transaction which holds the `max_index` within all its updates, then propagate this value to the reftable backend, wherein this is used to the set the `max_update_index` correctly. Add a test which creates a few thousand reference updates with multiple reflog entries, which should trigger the bug. Reported-by: brian m. carlson <sandals@crustytoothpaste.net> Signed-off-by: Karthik Nayak <karthik.188@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Karthik Nayak committed Jan 15, 2025 at 11:54 UTC bc67b4ab5f8bc268ecd2d9bb7dc1b7bf26884a8e

4 files changed +30 -10

refs.c

@@ -1297,6 +1297,13 @@ int ref_transaction_update_reflog(struct ref_transaction *transaction,

1297 update->flags &= ~REF_HAVE_OLD;

1298 update->index = index;

1299

1300 + /*

1301 + * Reference backends may need to know the max index to optimize

1302 + * their writes. So we store the max_index on the transaction level.

1303 + */

1304 + if (index > transaction->max_index)

1305 + transaction->max_index = index;

1306 +

1307 return 0;

1308 }

1309

refs/refs-internal.h

@@ -203,6 +203,7 @@ struct ref_transaction {

203 enum ref_transaction_state state;

204 void *backend_data;

205 unsigned int flags;

206 + unsigned int max_index;

207 };

208

209 /*

refs/reftable-backend.c

+10 -10

@@ -852,6 +852,7 @@ struct write_transaction_table_arg {

852 size_t updates_nr;

853 size_t updates_alloc;

854 size_t updates_expected;

855 + unsigned int max_index;

856 };

857

858 struct reftable_transaction_data {

@@ -1302,7 +1303,6 @@ static int write_transaction_table(struct reftable_writer *writer, void *cb_data

1303 struct reftable_log_record *logs = NULL;

1304 struct ident_split committer_ident = {0};

1305 size_t logs_nr = 0, logs_alloc = 0, i;

1305 - uint64_t max_update_index = ts;

1306 const char *committer_info;

1307 int ret = 0;

1308

@@ -1312,7 +1312,12 @@ static int write_transaction_table(struct reftable_writer *writer, void *cb_data

1312

1313 QSORT(arg->updates, arg->updates_nr, transaction_update_cmp);

1314

1315 - reftable_writer_set_limits(writer, ts, ts);

1315 + /*

1316 + * During reflog migration, we add indexes for a single reflog with

1317 + * multiple entries. Each entry will contain a different update_index,

1318 + * so set the limits accordingly.

1319 + */

1320 + reftable_writer_set_limits(writer, ts, ts + arg->max_index);

1321

1322 for (i = 0; i < arg->updates_nr; i++) {

1323 struct reftable_transaction_update *tx_update = &arg->updates[i];

@@ -1414,12 +1419,6 @@ static int write_transaction_table(struct reftable_writer *writer, void *cb_data

1419 */

1420 log->update_index = ts + u->index;

1421

1417 - /*

1418 - * Note the max update_index so the limit can be set later on.

1419 - */

1420 - if (log->update_index > max_update_index)

1421 - max_update_index = log->update_index;

1422 -

1422 log->refname = xstrdup(u->refname);

1423 memcpy(log->value.update.new_hash,

1424 u->new_oid.hash, GIT_MAX_RAWSZ);

@@ -1483,8 +1482,6 @@ static int write_transaction_table(struct reftable_writer *writer, void *cb_data

1482 * and log blocks.

1483 */

1484 if (logs) {

1486 - reftable_writer_set_limits(writer, ts, max_update_index);

1487 -

1485 ret = reftable_writer_add_logs(writer, logs, logs_nr);

1486 if (ret < 0)

1487 goto done;

@@ -1505,6 +1502,9 @@ static int reftable_be_transaction_finish(struct ref_store *ref_store UNUSED,

1502 struct reftable_transaction_data *tx_data = transaction->backend_data;

1503 int ret = 0;

1504

1505 + if (tx_data->args)

1506 + tx_data->args->max_index = transaction->max_index;

1507 +

1508 for (size_t i = 0; i < tx_data->args_nr; i++) {

1509 ret = reftable_addition_add(tx_data->args[i].addition,

1510 write_transaction_table, &tx_data->args[i]);

t/t1460-refs-migrate.sh

+12

@@ -227,6 +227,18 @@ do

227 done

228 done

229

230 +test_expect_success 'multiple reftable blocks with multiple entries' '

231 + test_when_finished "rm -rf repo" &&

232 + git init --ref-format=files repo &&

233 + test_commit -C repo first &&

234 + printf "create refs/heads/ref-%d HEAD\n" $(test_seq 5000) >stdin &&

235 + git -C repo update-ref --stdin <stdin &&

236 + test_commit -C repo second &&

237 + printf "update refs/heads/ref-%d HEAD\n" $(test_seq 3000) >stdin &&

238 + git -C repo update-ref --stdin <stdin &&

239 + test_migration repo reftable

240 +'

241 +

242 test_expect_success 'migrating from files format deletes backend files' '

243 test_when_finished "rm -rf repo" &&

244 git init --ref-format=files repo &&