@matheusbsbs / matheus-bagliano / commits / 23fc8e4f61

refs: implement batch reference update support

Git supports making reference updates with or without transactions. Updates with transactions are generally better optimized. But transactions are all or nothing. This means, if a user wants to batch updates to take advantage of the optimizations without the hard requirement that all updates must succeed, there is no way currently to do so. Particularly with the reftable backend where batching multiple reference updates is more efficient than performing them sequentially. Introduce batched update support with a new flag, 'REF_TRANSACTION_ALLOW_FAILURE'. Batched updates while different from transactions, use the transaction infrastructure under the hood. When enabled, this flag allows individual reference updates that would typically cause the entire transaction to fail due to non-system-related errors to be marked as rejected while permitting other updates to proceed. System errors referred by 'REF_TRANSACTION_ERROR_GENERIC' continue to result in the entire transaction failing. This approach enhances flexibility while preserving transactional integrity where necessary. The implementation introduces several key components: - Add 'rejection_err' field to struct `ref_update` to track failed updates with failure reason. - Add a new struct `ref_transaction_rejections` and a field within `ref_transaction` to this struct to allow quick iteration over rejected updates. - Modify reference backends (files, packed, reftable) to handle partial transactions by using `ref_transaction_set_rejected()` instead of failing the entire transaction when `REF_TRANSACTION_ALLOW_FAILURE` is set. - Add `ref_transaction_for_each_rejected_update()` to let callers examine which updates were rejected and why. This foundational change enables batched update support throughout the reference subsystem. A following commit will expose this capability to users by adding a `--batch-updates` flag to 'git-update-ref(1)', providing both a user-facing feature and a testable implementation. Signed-off-by: Karthik Nayak <karthik.188@gmail.com> Acked-by: Patrick Steinhardt <ps@pks.im> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Karthik Nayak committed Apr 8, 2025 at 10:51 UTC 23fc8e4f613179900ce28da959757a387543b468

6 files changed +156 -4

refs.c

+61

@@ -1176,6 +1176,10 @@ struct ref_transaction *ref_store_transaction_begin(struct ref_store *refs,

1176 tr->ref_store = refs;

1177 tr->flags = flags;

1178 string_list_init_dup(&tr->refnames);

1179 +

1180 + if (flags & REF_TRANSACTION_ALLOW_FAILURE)

1181 + CALLOC_ARRAY(tr->rejections, 1);

1182 +

1183 return tr;

1184 }

1185

@@ -1206,11 +1210,45 @@ void ref_transaction_free(struct ref_transaction *transaction)

1210 free((char *)transaction->updates[i]->old_target);

1211 free(transaction->updates[i]);

1212 }

1213 +

1214 + if (transaction->rejections)

1215 + free(transaction->rejections->update_indices);

1216 + free(transaction->rejections);

1217 +

1218 string_list_clear(&transaction->refnames, 0);

1219 free(transaction->updates);

1220 free(transaction);

1221 }

1222

1223 +int ref_transaction_maybe_set_rejected(struct ref_transaction *transaction,

1224 + size_t update_idx,

1225 + enum ref_transaction_error err)

1226 +{

1227 + if (update_idx >= transaction->nr)

1228 + BUG("trying to set rejection on invalid update index");

1229 +

1230 + if (!(transaction->flags & REF_TRANSACTION_ALLOW_FAILURE))

1231 + return 0;

1232 +

1233 + if (!transaction->rejections)

1234 + BUG("transaction not inititalized with failure support");

1235 +

1236 + /*

1237 + * Don't accept generic errors, since these errors are not user

1238 + * input related.

1239 + */

1240 + if (err == REF_TRANSACTION_ERROR_GENERIC)

1241 + return 0;

1242 +

1243 + transaction->updates[update_idx]->rejection_err = err;

1244 + ALLOC_GROW(transaction->rejections->update_indices,

1245 + transaction->rejections->nr + 1,

1246 + transaction->rejections->alloc);

1247 + transaction->rejections->update_indices[transaction->rejections->nr++] = update_idx;

1248 +

1249 + return 1;

1250 +}

1251 +

1252 struct ref_update *ref_transaction_add_update(

1253 struct ref_transaction *transaction,

1254 const char *refname, unsigned int flags,

@@ -1236,6 +1274,7 @@ struct ref_update *ref_transaction_add_update(

1274 transaction->updates[transaction->nr++] = update;

1275

1276 update->flags = flags;

1277 + update->rejection_err = 0;

1278

1279 update->new_target = xstrdup_or_null(new_target);

1280 update->old_target = xstrdup_or_null(old_target);

@@ -2728,6 +2767,28 @@ void ref_transaction_for_each_queued_update(struct ref_transaction *transaction,

2767 }

2768 }

2769

2770 +void ref_transaction_for_each_rejected_update(struct ref_transaction *transaction,

2771 + ref_transaction_for_each_rejected_update_fn cb,

2772 + void *cb_data)

2773 +{

2774 + if (!transaction->rejections)

2775 + return;

2776 +

2777 + for (size_t i = 0; i < transaction->rejections->nr; i++) {

2778 + size_t update_index = transaction->rejections->update_indices[i];

2779 + struct ref_update *update = transaction->updates[update_index];

2780 +

2781 + if (!update->rejection_err)

2782 + continue;

2783 +

2784 + cb(update->refname,

2785 + (update->flags & REF_HAVE_OLD) ? &update->old_oid : NULL,

2786 + (update->flags & REF_HAVE_NEW) ? &update->new_oid : NULL,

2787 + update->old_target, update->new_target,

2788 + update->rejection_err, cb_data);

2789 + }

2790 +}

2791 +

2792 int refs_delete_refs(struct ref_store *refs, const char *logmsg,

2793 struct string_list *refnames, unsigned int flags)

2794 {

refs.h

+22

@@ -667,6 +667,13 @@ enum ref_transaction_flag {

667 * either be absent or null_oid.

668 */

669 REF_TRANSACTION_FLAG_INITIAL = (1 << 0),

670 +

671 + /*

672 + * The transaction mechanism by default fails all updates if any conflict

673 + * is detected. This flag allows transactions to partially apply updates

674 + * while rejecting updates which do not match the expected state.

675 + */

676 + REF_TRANSACTION_ALLOW_FAILURE = (1 << 1),

677 };

678

679 /*

@@ -897,6 +904,21 @@ void ref_transaction_for_each_queued_update(struct ref_transaction *transaction,

904 ref_transaction_for_each_queued_update_fn cb,

905 void *cb_data);

906

907 +/*

908 + * Execute the given callback function for each of the reference updates which

909 + * have been rejected in the given transaction.

910 + */

911 +typedef void ref_transaction_for_each_rejected_update_fn(const char *refname,

912 + const struct object_id *old_oid,

913 + const struct object_id *new_oid,

914 + const char *old_target,

915 + const char *new_target,

916 + enum ref_transaction_error err,

917 + void *cb_data);

918 +void ref_transaction_for_each_rejected_update(struct ref_transaction *transaction,

919 + ref_transaction_for_each_rejected_update_fn cb,

920 + void *cb_data);

921 +

922 /*

923 * Free `*transaction` and all associated data.

924 */

refs/files-backend.c

+11 -1

@@ -2852,8 +2852,15 @@ static int files_transaction_prepare(struct ref_store *ref_store,

2852 ret = lock_ref_for_update(refs, update, transaction,

2853 head_ref, &refnames_to_check,

2854 err);

2855 - if (ret)

2855 + if (ret) {

2856 + if (ref_transaction_maybe_set_rejected(transaction, i, ret)) {

2857 + strbuf_reset(err);

2858 + ret = 0;

2859 +

2860 + continue;

2861 + }

2862 goto cleanup;

2863 + }

2864

2865 if (update->flags & REF_DELETING &&

2866 !(update->flags & REF_LOG_ONLY) &&

@@ -3151,6 +3158,9 @@ static int files_transaction_finish(struct ref_store *ref_store,

3158 struct ref_update *update = transaction->updates[i];

3159 struct ref_lock *lock = update->backend_data;

3160

3161 + if (update->rejection_err)

3162 + continue;

3163 +

3164 if (update->flags & REF_NEEDS_COMMIT ||

3165 update->flags & REF_LOG_ONLY) {

3166 if (parse_and_write_reflog(refs, update, lock, err)) {

refs/packed-backend.c

+25 -2

@@ -1327,10 +1327,11 @@ static int packed_ref_store_remove_on_disk(struct ref_store *ref_store,

1327 * remain locked when it is done.

1328 */

1329 static enum ref_transaction_error write_with_updates(struct packed_ref_store *refs,

1330 - struct string_list *updates,

1330 + struct ref_transaction *transaction,

1331 struct strbuf *err)

1332 {

1333 enum ref_transaction_error ret = REF_TRANSACTION_ERROR_GENERIC;

1334 + struct string_list *updates = &transaction->refnames;

1335 struct ref_iterator *iter = NULL;

1336 size_t i;

1337 int ok;

@@ -1411,6 +1412,13 @@ static enum ref_transaction_error write_with_updates(struct packed_ref_store *re

1412 "reference already exists",

1413 update->refname);

1414 ret = REF_TRANSACTION_ERROR_CREATE_EXISTS;

1415 +

1416 + if (ref_transaction_maybe_set_rejected(transaction, i, ret)) {

1417 + strbuf_reset(err);

1418 + ret = 0;

1419 + continue;

1420 + }

1421 +

1422 goto error;

1423 } else if (!oideq(&update->old_oid, iter->oid)) {

1424 strbuf_addf(err, "cannot update ref '%s': "

@@ -1419,6 +1427,13 @@ static enum ref_transaction_error write_with_updates(struct packed_ref_store *re

1427 oid_to_hex(iter->oid),

1428 oid_to_hex(&update->old_oid));

1429 ret = REF_TRANSACTION_ERROR_INCORRECT_OLD_VALUE;

1430 +

1431 + if (ref_transaction_maybe_set_rejected(transaction, i, ret)) {

1432 + strbuf_reset(err);

1433 + ret = 0;

1434 + continue;

1435 + }

1436 +

1437 goto error;

1438 }

1439 }

@@ -1456,6 +1471,13 @@ static enum ref_transaction_error write_with_updates(struct packed_ref_store *re

1471 update->refname,

1472 oid_to_hex(&update->old_oid));

1473 ret = REF_TRANSACTION_ERROR_NONEXISTENT_REF;

1474 +

1475 + if (ref_transaction_maybe_set_rejected(transaction, i, ret)) {

1476 + strbuf_reset(err);

1477 + ret = 0;

1478 + continue;

1479 + }

1480 +

1481 goto error;

1482 }

1483 }

@@ -1521,6 +1543,7 @@ static enum ref_transaction_error write_with_updates(struct packed_ref_store *re

1543 write_error:

1544 strbuf_addf(err, "error writing to %s: %s",

1545 get_tempfile_path(refs->tempfile), strerror(errno));

1546 + ret = REF_TRANSACTION_ERROR_GENERIC;

1547

1548 error:

1549 ref_iterator_free(iter);

@@ -1679,7 +1702,7 @@ static int packed_transaction_prepare(struct ref_store *ref_store,

1702 data->own_lock = 1;

1703 }

1704

1682 - ret = write_with_updates(refs, &transaction->refnames, err);

1705 + ret = write_with_updates(refs, transaction, err);

1706 if (ret)

1707 goto failure;

1708

refs/refs-internal.h

+26

@@ -123,6 +123,12 @@ struct ref_update {

123 */

124 uint64_t index;

125

126 + /*

127 + * Used in batched reference updates to mark if a given update

128 + * was rejected.

129 + */

130 + enum ref_transaction_error rejection_err;

131 +

132 /*

133 * If this ref_update was split off of a symref update via

134 * split_symref_update(), then this member points at that

@@ -142,6 +148,13 @@ int refs_read_raw_ref(struct ref_store *ref_store, const char *refname,

148 struct object_id *oid, struct strbuf *referent,

149 unsigned int *type, int *failure_errno);

150

151 +/*

152 + * Mark a given update as rejected with a given reason.

153 + */

154 +int ref_transaction_maybe_set_rejected(struct ref_transaction *transaction,

155 + size_t update_idx,

156 + enum ref_transaction_error err);

157 +

158 /*

159 * Add a ref_update with the specified properties to transaction, and

160 * return a pointer to the new object. This function does not verify

@@ -183,6 +196,18 @@ enum ref_transaction_state {

196 REF_TRANSACTION_CLOSED = 2

197 };

198

199 +/*

200 + * Data structure to hold indices of updates which were rejected, for batched

201 + * reference updates. While the updates themselves hold the rejection error,

202 + * this structure allows a transaction to iterate only over the rejected

203 + * updates.

204 + */

205 +struct ref_transaction_rejections {

206 + size_t *update_indices;

207 + size_t alloc;

208 + size_t nr;

209 +};

210 +

211 /*

212 * Data structure for holding a reference transaction, which can

213 * consist of checks and updates to multiple references, carried out

@@ -195,6 +220,7 @@ struct ref_transaction {

220 size_t alloc;

221 size_t nr;

222 enum ref_transaction_state state;

223 + struct ref_transaction_rejections *rejections;

224 void *backend_data;

225 unsigned int flags;

226 uint64_t max_index;

refs/reftable-backend.c

+11 -1

@@ -1371,8 +1371,15 @@ static int reftable_be_transaction_prepare(struct ref_store *ref_store,

1371 transaction->updates[i],

1372 &refnames_to_check, head_type,

1373 &head_referent, &referent, err);

1374 - if (ret)

1374 + if (ret) {

1375 + if (ref_transaction_maybe_set_rejected(transaction, i, ret)) {

1376 + strbuf_reset(err);

1377 + ret = 0;

1378 +

1379 + continue;

1380 + }

1381 goto done;

1382 + }

1383 }

1384

1385 ret = refs_verify_refnames_available(ref_store, &refnames_to_check,

@@ -1454,6 +1461,9 @@ static int write_transaction_table(struct reftable_writer *writer, void *cb_data

1461 struct reftable_transaction_update *tx_update = &arg->updates[i];

1462 struct ref_update *u = tx_update->update;

1463

1464 + if (u->rejection_err)

1465 + continue;

1466 +

1467 /*

1468 * Write a reflog entry when updating a ref to point to

1469 * something new in either of the following cases: