@matheusbsbs / matheus-bagliano / commits / 08944d1c22

midx: preliminary support for `--refs-snapshot`

To figure out which commits we can write a bitmap for, the multi-pack index/bitmap code does a reachability traversal, marking any commit which can be found in the MIDX as eligible to receive a bitmap. This approach will cause a problem when multi-pack bitmaps are able to be generated from `git repack`, since the reference tips can change during the repack. Even though we ignore commits that don't exist in the MIDX (when doing a scan of the ref tips), it's possible that a commit in the MIDX reaches something that isn't. This can happen when a multi-pack index contains some pack which refers to loose objects (e.g., if a pack was pushed after starting the repack but before generating the MIDX which depends on an object which is stored as loose in the repository, and by definition isn't included in the multi-pack index). By taking a snapshot of the references before we start repacking, we can close that race window. In the above scenario (where we have a packed object pointing at a loose one), we'll either (a) take a snapshot of the references before seeing the packed one, or (b) take it after, at which point we can guarantee that the loose object will be packed and included in the MIDX. This patch does just that. It writes a temporary "reference snapshot", which is a list of OIDs that are at the ref tips before writing a multi-pack bitmap. References that are "preferred" (i.e,. are a suffix of at least one value of the 'pack.preferBitmapTips' configuration) are marked with a special '+'. The format is simple: one line per commit at each tip, with an optional '+' at the beginning (for preferred references, as described above). When provided, the reference snapshot is used to drive bitmap selection instead of the MIDX code doing its own traversal. When it isn't provided, the usual traversal takes place instead. Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Taylor Blau committed Sep 28, 2021 at 21:55 UTC 08944d1c221a7f4fe42a50c0f11f129769edc9b1

6 files changed +164 -13

Documentation/git-multi-pack-index.txt

+15

@@ -49,6 +49,21 @@ write::

49 --stdin-packs::

50 Write a multi-pack index containing only the set of

51 line-delimited pack index basenames provided over stdin.

52 +

53 + --refs-snapshot=<path>::

54 + With `--bitmap`, optionally specify a file which

55 + contains a "refs snapshot" taken prior to repacking.

56 ++

57 +A reference snapshot is composed of line-delimited OIDs corresponding to

58 +the reference tips, usually taken by `git repack` prior to generating a

59 +new pack. A line may optionally start with a `+` character to indicate

60 +that the reference which corresponds to that OID is "preferred" (see

61 +linkgit:git-config[1]'s `pack.preferBitmapTips`.)

62 ++

63 +The file given at `<path>` is expected to be readable, and can contain

64 +duplicates. (If a given OID is given more than once, it is marked as

65 +preferred if at least one instance of it begins with the special `+`

66 +marker).

67 --

69 verify::

builtin/multi-pack-index.c

+8 -3

@@ -7,7 +7,8 @@

7 #include "object-store.h"

9 #define BUILTIN_MIDX_WRITE_USAGE \

10 - N_("git multi-pack-index [<options>] write [--preferred-pack=<pack>]")

10 + N_("git multi-pack-index [<options>] write [--preferred-pack=<pack>]" \

11 + "[--refs-snapshot=<path>]")

13 #define BUILTIN_MIDX_VERIFY_USAGE \

14 N_("git multi-pack-index [<options>] verify")

@@ -45,6 +46,7 @@ static char const * const builtin_multi_pack_index_usage[] = {

46 static struct opts_multi_pack_index {

47 const char *object_dir;

48 const char *preferred_pack;

49 + const char *refs_snapshot;

50 unsigned long batch_size;

51 unsigned flags;

52 int stdin_packs;

@@ -83,6 +85,8 @@ static int cmd_multi_pack_index_write(int argc, const char **argv)

85 MIDX_WRITE_BITMAP | MIDX_WRITE_REV_INDEX),

86 OPT_BOOL(0, "stdin-packs", &opts.stdin_packs,

87 N_("write multi-pack index containing only given indexes")),

88 + OPT_FILENAME(0, "refs-snapshot", &opts.refs_snapshot,

89 + N_("refs snapshot for selecting bitmap commits")),

90 OPT_END(),

91 };

@@ -106,7 +110,8 @@ static int cmd_multi_pack_index_write(int argc, const char **argv)

110 read_packs_from_stdin(&packs);

111

112 ret = write_midx_file_only(opts.object_dir, &packs,

109 - opts.preferred_pack, opts.flags);

113 + opts.preferred_pack,

114 + opts.refs_snapshot, opts.flags);

115

116 string_list_clear(&packs, 0);

117

@@ -114,7 +119,7 @@ static int cmd_multi_pack_index_write(int argc, const char **argv)

119

120 }

121 return write_midx_file(opts.object_dir, opts.preferred_pack,

117 - opts.flags);

122 + opts.refs_snapshot, opts.flags);

123 }

124

125 static int cmd_multi_pack_index_verify(int argc, const char **argv)

builtin/repack.c

+1 -1

@@ -733,7 +733,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix)

733 unsigned flags = 0;

734 if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP, 0))

735 flags |= MIDX_WRITE_BITMAP | MIDX_WRITE_REV_INDEX;

736 - write_midx_file(get_object_directory(), NULL, flags);

736 + write_midx_file(get_object_directory(), NULL, NULL, flags);

737 }

738

739 string_list_clear(&names, 0);

midx.c

+53 -8

@@ -983,7 +983,43 @@ static void bitmap_show_commit(struct commit *commit, void *_data)

983 data->commits[data->commits_nr++] = commit;

984 }

985

986 +static int read_refs_snapshot(const char *refs_snapshot,

987 + struct rev_info *revs)

988 +{

989 + struct strbuf buf = STRBUF_INIT;

990 + struct object_id oid;

991 + FILE *f = xfopen(refs_snapshot, "r");

992 +

993 + while (strbuf_getline(&buf, f) != EOF) {

994 + struct object *object;

995 + int preferred = 0;

996 + char *hex = buf.buf;

997 + const char *end = NULL;

998 +

999 + if (buf.len && *buf.buf == '+') {

1000 + preferred = 1;

1001 + hex = &buf.buf[1];

1002 + }

1003 +

1004 + if (parse_oid_hex(hex, &oid, &end) < 0)

1005 + die(_("could not parse line: %s"), buf.buf);

1006 + if (*end)

1007 + die(_("malformed line: %s"), buf.buf);

1008 +

1009 + object = parse_object_or_die(&oid, NULL);

1010 + if (preferred)

1011 + object->flags |= NEEDS_BITMAP;

1012 +

1013 + add_pending_object(revs, object, "");

1014 + }

1015 +

1016 + fclose(f);

1017 + strbuf_release(&buf);

1018 + return 0;

1019 +}

1020 +

1021 static struct commit **find_commits_for_midx_bitmap(uint32_t *indexed_commits_nr_p,

1022 + const char *refs_snapshot,

1023 struct write_midx_context *ctx)

1024 {

1025 struct rev_info revs;

@@ -992,8 +1028,12 @@ static struct commit **find_commits_for_midx_bitmap(uint32_t *indexed_commits_nr

1028 cb.ctx = ctx;

1029

1030 repo_init_revisions(the_repository, &revs, NULL);

995 - setup_revisions(0, NULL, &revs, NULL);

996 - for_each_ref(add_ref_to_pending, &revs);

1031 + if (refs_snapshot) {

1032 + read_refs_snapshot(refs_snapshot, &revs);

1033 + } else {

1034 + setup_revisions(0, NULL, &revs, NULL);

1035 + for_each_ref(add_ref_to_pending, &revs);

1036 + }

1037

1038 /*

1039 * Skipping promisor objects here is intentional, since it only excludes

@@ -1022,6 +1062,7 @@ static struct commit **find_commits_for_midx_bitmap(uint32_t *indexed_commits_nr

1062

1063 static int write_midx_bitmap(char *midx_name, unsigned char *midx_hash,

1064 struct write_midx_context *ctx,

1065 + const char *refs_snapshot,

1066 unsigned flags)

1067 {

1068 struct packing_data pdata;

@@ -1033,7 +1074,7 @@ static int write_midx_bitmap(char *midx_name, unsigned char *midx_hash,

1074

1075 prepare_midx_packing_data(&pdata, ctx);

1076

1036 - commits = find_commits_for_midx_bitmap(&commits_nr, ctx);

1077 + commits = find_commits_for_midx_bitmap(&commits_nr, refs_snapshot, ctx);

1078

1079 /*

1080 * Build the MIDX-order index based on pdata.objects (which is already

@@ -1081,6 +1122,7 @@ static int write_midx_internal(const char *object_dir,

1122 struct string_list *packs_to_include,

1123 struct string_list *packs_to_drop,

1124 const char *preferred_pack_name,

1125 + const char *refs_snapshot,

1126 unsigned flags)

1127 {

1128 char *midx_name;

@@ -1374,7 +1416,8 @@ static int write_midx_internal(const char *object_dir,

1416 if (flags & MIDX_WRITE_REV_INDEX)

1417 write_midx_reverse_index(midx_name, midx_hash, &ctx);

1418 if (flags & MIDX_WRITE_BITMAP) {

1377 - if (write_midx_bitmap(midx_name, midx_hash, &ctx, flags) < 0) {

1419 + if (write_midx_bitmap(midx_name, midx_hash, &ctx,

1420 + refs_snapshot, flags) < 0) {

1421 error(_("could not write multi-pack bitmap"));

1422 result = 1;

1423 goto cleanup;

@@ -1409,19 +1452,21 @@ cleanup:

1452

1453 int write_midx_file(const char *object_dir,

1454 const char *preferred_pack_name,

1455 + const char *refs_snapshot,

1456 unsigned flags)

1457 {

1458 return write_midx_internal(object_dir, NULL, NULL, preferred_pack_name,

1415 - flags);

1459 + refs_snapshot, flags);

1460 }

1461

1462 int write_midx_file_only(const char *object_dir,

1463 struct string_list *packs_to_include,

1464 const char *preferred_pack_name,

1465 + const char *refs_snapshot,

1466 unsigned flags)

1467 {

1468 return write_midx_internal(object_dir, packs_to_include, NULL,

1424 - preferred_pack_name, flags);

1469 + preferred_pack_name, refs_snapshot, flags);

1470 }

1471

1472 struct clear_midx_data {

@@ -1701,7 +1746,7 @@ int expire_midx_packs(struct repository *r, const char *object_dir, unsigned fla

1746 free(count);

1747

1748 if (packs_to_drop.nr) {

1704 - result = write_midx_internal(object_dir, NULL, &packs_to_drop, NULL, flags);

1749 + result = write_midx_internal(object_dir, NULL, &packs_to_drop, NULL, NULL, flags);

1750 m = NULL;

1751 }

1752

@@ -1892,7 +1937,7 @@ int midx_repack(struct repository *r, const char *object_dir, size_t batch_size,

1937 goto cleanup;

1938 }

1939

1895 - result = write_midx_internal(object_dir, NULL, NULL, NULL, flags);

1940 + result = write_midx_internal(object_dir, NULL, NULL, NULL, NULL, flags);

1941 m = NULL;

1942

1943 cleanup:

midx.h

+5 -1

@@ -62,14 +62,18 @@ int fill_midx_entry(struct repository *r, const struct object_id *oid, struct pa

62 int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name);

63 int prepare_multi_pack_index_one(struct repository *r, const char *object_dir, int local);

65 -int write_midx_file(const char *object_dir, const char *preferred_pack_name, unsigned flags);

65 /*

66 * Variant of write_midx_file which writes a MIDX containing only the packs

67 * specified in packs_to_include.

68 */

69 +int write_midx_file(const char *object_dir,

70 + const char *preferred_pack_name,

71 + const char *refs_snapshot,

72 + unsigned flags);

73 int write_midx_file_only(const char *object_dir,

74 struct string_list *packs_to_include,

75 const char *preferred_pack_name,

76 + const char *refs_snapshot,

77 unsigned flags);

78 void clear_midx_file(struct repository *r);

79 int verify_midx_file(struct repository *r, const char *object_dir, unsigned flags);

t/t5326-multi-pack-bitmaps.sh

+82

@@ -283,4 +283,86 @@ test_expect_success 'pack.preferBitmapTips' '

283 )

284 '

285

286 +test_expect_success 'writing a bitmap with --refs-snapshot' '

287 + git init repo &&

288 + test_when_finished "rm -fr repo" &&

289 + (

290 + cd repo &&

291 +

292 + test_commit one &&

293 + test_commit two &&

294 +

295 + git rev-parse one >snapshot &&

296 +

297 + git repack -ad &&

298 +

299 + # First, write a MIDX which see both refs/tags/one and

300 + # refs/tags/two (causing both of those commits to receive

301 + # bitmaps).

302 + git multi-pack-index write --bitmap &&

303 +

304 + test_path_is_file $midx &&

305 + test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&

306 +

307 + test-tool bitmap list-commits | sort >bitmaps &&

308 + grep "$(git rev-parse one)" bitmaps &&

309 + grep "$(git rev-parse two)" bitmaps &&

310 +

311 + rm -fr $midx-$(midx_checksum $objdir).bitmap &&

312 + rm -fr $midx-$(midx_checksum $objdir).rev &&

313 + rm -fr $midx &&

314 +

315 + # Then again, but with a refs snapshot which only sees

316 + # refs/tags/one.

317 + git multi-pack-index write --bitmap --refs-snapshot=snapshot &&

318 +

319 + test_path_is_file $midx &&

320 + test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&

321 +

322 + test-tool bitmap list-commits | sort >bitmaps &&

323 + grep "$(git rev-parse one)" bitmaps &&

324 + ! grep "$(git rev-parse two)" bitmaps

325 + )

326 +'

327 +

328 +test_expect_success 'write a bitmap with --refs-snapshot (preferred tips)' '

329 + git init repo &&

330 + test_when_finished "rm -fr repo" &&

331 + (

332 + cd repo &&

333 +

334 + test_commit_bulk --message="%s" 103 &&

335 +

336 + git log --format="%H" >commits.raw &&

337 + sort <commits.raw >commits &&

338 +

339 + git log --format="create refs/tags/%s %H" HEAD >refs &&

340 + git update-ref --stdin <refs &&

341 +

342 + git multi-pack-index write --bitmap &&

343 + test_path_is_file $midx &&

344 + test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&

345 +

346 + test-tool bitmap list-commits | sort >bitmaps &&

347 + comm -13 bitmaps commits >before &&

348 + test_line_count = 1 before &&

349 +

350 + (

351 + grep -vf before commits.raw &&

352 + # mark missing commits as preferred

353 + sed "s/^/+/" before

354 + ) >snapshot &&

355 +

356 + rm -fr $midx-$(midx_checksum $objdir).bitmap &&

357 + rm -fr $midx-$(midx_checksum $objdir).rev &&

358 + rm -fr $midx &&

359 +

360 + git multi-pack-index write --bitmap --refs-snapshot=snapshot &&

361 + test-tool bitmap list-commits | sort >bitmaps &&

362 + comm -13 bitmaps commits >after &&

363 +

364 + ! test_cmp before after

365 + )

366 +'

367 +

368 test_done