builtin/repack.c: pass `--refs-snapshot` when writing bitmaps
To prevent the race described in an earlier patch, generate and pass a reference snapshot to the multi-pack bitmap code, if we are writing one from `git repack`. This patch is mostly limited to creating a temporary file, and then calling for_each_ref(). Except we try to minimize duplicates, since doing so can drastically reduce the size in network-of-forks style repositories. In the kernel's fork network (the repository containing all objects from the kernel and all its forks), deduplicating the references drops the snapshot size from 934 MB to just 12 MB. But since we're handling duplicates in this way, we have to make sure that we preferred references (those listed in pack.preferBitmapTips) before non-preferred ones (to avoid recording an object which is pointed at by a preferred tip as non-preferred). We accomplish this by doing separate passes over the references: first visiting each prefix in pack.preferBitmapTips, and then over the rest of the references. Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Taylor Blau committed
Oct 1, 2021 at 18:38 UTC
324efc90d1b1a660388f8cabbb72e803160082d3
2 files changed
+124
builtin/repack.c
+82
@@ -15,6 +15,8 @@
15
#include "promisor-remote.h"
16
#include "shallow.h"
17
#include "pack.h"
18
+#include "pack-bitmap.h"
19
+#include "refs.h"
20
21
static int delta_base_offset = 1;
22
static int pack_kept_objects = -1;
@@ -453,6 +455,68 @@ static void clear_pack_geometry(struct pack_geometry *geometry)
455
geometry->split = 0;
456
}
457
458
+struct midx_snapshot_ref_data {
459
+ struct tempfile *f;
460
+ struct oidset seen;
461
+ int preferred;
462
+};
463
+
464
+static int midx_snapshot_ref_one(const char *refname,
465
+ const struct object_id *oid,
466
+ int flag, void *_data)
467
+{
468
+ struct midx_snapshot_ref_data *data = _data;
469
+ struct object_id peeled;
470
+
471
+ if (!peel_iterated_oid(oid, &peeled))
472
+ oid = &peeled;
473
+
474
+ if (oidset_insert(&data->seen, oid))
475
+ return 0; /* already seen */
476
+
477
+ if (oid_object_info(the_repository, oid, NULL) != OBJ_COMMIT)
478
+ return 0;
479
+
480
+ fprintf(data->f->fp, "%s%s\n", data->preferred ? "+" : "",
481
+ oid_to_hex(oid));
482
+
483
+ return 0;
484
+}
485
+
486
+static void midx_snapshot_refs(struct tempfile *f)
487
+{
488
+ struct midx_snapshot_ref_data data;
489
+ const struct string_list *preferred = bitmap_preferred_tips(the_repository);
490
+
491
+ data.f = f;
492
+ data.preferred = 0;
493
+ oidset_init(&data.seen, 0);
494
+
495
+ if (!fdopen_tempfile(f, "w"))
496
+ die(_("could not open tempfile %s for writing"),
497
+ get_tempfile_path(f));
498
+
499
+ if (preferred) {
500
+ struct string_list_item *item;
501
+
502
+ data.preferred = 1;
503
+ for_each_string_list_item(item, preferred)
504
+ for_each_ref_in(item->string, midx_snapshot_ref_one, &data);
505
+ data.preferred = 0;
506
+ }
507
+
508
+ for_each_ref(midx_snapshot_ref_one, &data);
509
+
510
+ if (close_tempfile_gently(f)) {
511
+ int save_errno = errno;
512
+ delete_tempfile(&f);
513
+ errno = save_errno;
514
+ die_errno(_("could not close refs snapshot tempfile"));
515
+ }
516
+
517
+ oidset_clear(&data.seen);
518
+}
519
+
520
static void midx_included_packs(struct string_list *include,
521
struct string_list *existing_nonkept_packs,
522
struct string_list *existing_kept_packs,
@@ -488,6 +552,7 @@ static void midx_included_packs(struct string_list *include,
552
553
static int write_midx_included_packs(struct string_list *include,
554
struct pack_geometry *geometry,
555
+ const char *refs_snapshot,
556
int show_progress, int write_bitmaps)
557
{
558
struct child_process cmd = CHILD_PROCESS_INIT;
@@ -517,6 +582,9 @@ static int write_midx_included_packs(struct string_list *include,
582
strvec_pushf(&cmd.args, "--preferred-pack=%s",
583
pack_basename(largest));
584
585
+ if (refs_snapshot)
586
+ strvec_pushf(&cmd.args, "--refs-snapshot=%s", refs_snapshot);
587
+
588
ret = start_command(&cmd);
589
if (ret)
590
return ret;
@@ -539,6 +607,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
607
struct string_list existing_kept_packs = STRING_LIST_INIT_DUP;
608
struct pack_geometry *geometry = NULL;
609
struct strbuf line = STRBUF_INIT;
610
+ struct tempfile *refs_snapshot = NULL;
611
int i, ext, ret;
612
FILE *out;
613
int show_progress = isatty(2);
@@ -627,6 +696,18 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
696
if (write_bitmaps && !(pack_everything & ALL_INTO_ONE) && !write_midx)
697
die(_(incremental_bitmap_conflict_error));
698
699
+ if (write_midx && write_bitmaps) {
700
+ struct strbuf path = STRBUF_INIT;
701
+
702
+ strbuf_addf(&path, "%s/%s_XXXXXX", get_object_directory(),
703
+ "bitmap-ref-tips");
704
+
705
+ refs_snapshot = xmks_tempfile(path.buf);
706
+ midx_snapshot_refs(refs_snapshot);
707
+
708
+ strbuf_release(&path);
709
+ }
710
+
711
if (geometric_factor) {
712
if (pack_everything)
713
die(_("--geometric is incompatible with -A, -a"));
@@ -809,6 +890,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
890
&existing_kept_packs, &names, geometry);
891
892
ret = write_midx_included_packs(&include, geometry,
893
+ refs_snapshot ? get_tempfile_path(refs_snapshot) : NULL,
894
show_progress, write_bitmaps > 0);
895
896
string_list_clear(&include, 0);
t/t7700-repack.sh
+42
@@ -330,4 +330,46 @@ test_expect_success 'cleans up MIDX when appropriate' '
330
)
331
'
332
333
+test_expect_success '--write-midx with preferred bitmap tips' '
334
+ git init midx-preferred-tips &&
335
+ test_when_finished "rm -fr midx-preferred-tips" &&
336
+ (
337
+ cd midx-preferred-tips &&
338
+
339
+ test_commit_bulk --message="%s" 103 &&
340
+
341
+ git log --format="%H" >commits.raw &&
342
+ sort <commits.raw >commits &&
343
+
344
+ git log --format="create refs/tags/%s/%s %H" HEAD >refs &&
345
+ git update-ref --stdin <refs &&
346
+
347
+ git repack --write-midx --write-bitmap-index &&
348
+ test_path_is_file $midx &&
349
+ test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
350
+
351
+ test-tool bitmap list-commits | sort >bitmaps &&
352
+ comm -13 bitmaps commits >before &&
353
+ test_line_count = 1 before &&
354
+
355
+ rm -fr $midx-$(midx_checksum $objdir).bitmap &&
356
+ rm -fr $midx-$(midx_checksum $objdir).rev &&
357
+ rm -fr $midx &&
358
+
359
+ # instead of constructing the snapshot ourselves (c.f., the test
360
+ # "write a bitmap with --refs-snapshot (preferred tips)" in
361
+ # t5326), mark the missing commit as preferred by adding it to
362
+ # the pack.preferBitmapTips configuration.
363
+ git for-each-ref --format="%(refname:rstrip=1)" \
364
+ --points-at="$(cat before)" >missing &&
365
+ git config pack.preferBitmapTips "$(cat missing)" &&
366
+ git repack --write-midx --write-bitmap-index &&
367
+
368
+ test-tool bitmap list-commits | sort >bitmaps &&
369
+ comm -13 bitmaps commits >after &&
370
+
371
+ ! test_cmp before after
372
+ )
373
+'
374
+
375
test_done