builtin/repack.c: support generating a cruft pack
Expose a way to split the contents of a repository into a main and cruft pack when doing an all-into-one repack with `git repack --cruft -d`, and a complementary configuration variable. Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Taylor Blau committed
May 20, 2022 at 19:18 UTC
f9825d1cf752b8d04a3e9193ff6fdb54d09e28a3
4 files changed
+319
-6
Documentation/git-repack.txt
+11
@@ -63,6 +63,17 @@ to the new separate pack will be written.
63
Also run 'git prune-packed' to remove redundant
64
loose object files.
65
66
+--cruft::
67
+ Same as `-a`, unless `-d` is used. Then any unreachable objects
68
+ are packed into a separate cruft pack. Unreachable objects can
69
+ be pruned using the normal expiry rules with the next `git gc`
70
+ invocation (see linkgit:git-gc[1]). Incompatible with `-k`.
71
+
72
+--cruft-expiration=<approxidate>::
73
+ Expire unreachable objects older than `<approxidate>`
74
+ immediately instead of waiting for the next `git gc` invocation.
75
+ Only useful with `--cruft -d`.
76
+
77
-l::
78
Pass the `--local` option to 'git pack-objects'. See
79
linkgit:git-pack-objects[1].
Documentation/technical/cruft-packs.txt
+1
-1
@@ -17,7 +17,7 @@ pruned according to normal expiry rules with the next 'git gc' invocation.
17
18
Unreachable objects aren't removed immediately, since doing so could race with
19
an incoming push which may reference an object which is about to be deleted.
20
-Instead, those unreachable objects are stored as loose object and stay that way
20
+Instead, those unreachable objects are stored as loose objects and stay that way
21
until they are older than the expiration window, at which point they are removed
22
by linkgit:git-prune[1].
23
builtin/repack.c
+100
-5
@@ -18,12 +18,18 @@
18
#include "pack-bitmap.h"
19
#include "refs.h"
20
21
+#define ALL_INTO_ONE 1
22
+#define LOOSEN_UNREACHABLE 2
23
+#define PACK_CRUFT 4
24
+
25
+static int pack_everything;
26
static int delta_base_offset = 1;
27
static int pack_kept_objects = -1;
28
static int write_bitmaps = -1;
29
static int use_delta_islands;
30
static int run_update_server_info = 1;
31
static char *packdir, *packtmp_name, *packtmp;
32
+static char *cruft_expiration;
33
34
static const char *const git_repack_usage[] = {
35
N_("git repack [<options>]"),
@@ -305,9 +311,6 @@ static void repack_promisor_objects(const struct pack_objects_args *args,
311
die(_("could not finish pack-objects to repack promisor objects"));
312
}
313
308
-#define ALL_INTO_ONE 1
309
-#define LOOSEN_UNREACHABLE 2
310
-
314
struct pack_geometry {
315
struct packed_git **pack;
316
uint32_t pack_nr, pack_alloc;
@@ -344,6 +347,8 @@ static void init_pack_geometry(struct pack_geometry **geometry_p)
347
for (p = get_all_packs(the_repository); p; p = p->next) {
348
if (!pack_kept_objects && p->pack_keep)
349
continue;
350
+ if (p->is_cruft)
351
+ continue;
352
353
ALLOC_GROW(geometry->pack,
354
geometry->pack_nr + 1,
@@ -605,6 +610,67 @@ static int write_midx_included_packs(struct string_list *include,
610
return finish_command(&cmd);
611
}
612
613
+static int write_cruft_pack(const struct pack_objects_args *args,
614
+ const char *pack_prefix,
615
+ struct string_list *names,
616
+ struct string_list *existing_packs,
617
+ struct string_list *existing_kept_packs)
618
+{
619
+ struct child_process cmd = CHILD_PROCESS_INIT;
620
+ struct strbuf line = STRBUF_INIT;
621
+ struct string_list_item *item;
622
+ FILE *in, *out;
623
+ int ret;
624
+
625
+ prepare_pack_objects(&cmd, args);
626
+
627
+ strvec_push(&cmd.args, "--cruft");
628
+ if (cruft_expiration)
629
+ strvec_pushf(&cmd.args, "--cruft-expiration=%s",
630
+ cruft_expiration);
631
+
632
+ strvec_push(&cmd.args, "--honor-pack-keep");
633
+ strvec_push(&cmd.args, "--non-empty");
634
+ strvec_push(&cmd.args, "--max-pack-size=0");
635
+
636
+ cmd.in = -1;
637
+
638
+ ret = start_command(&cmd);
639
+ if (ret)
640
+ return ret;
641
+
642
+ /*
643
+ * names has a confusing double use: it both provides the list
644
+ * of just-written new packs, and accepts the name of the cruft
645
+ * pack we are writing.
646
+ *
647
+ * By the time it is read here, it contains only the pack(s)
648
+ * that were just written, which is exactly the set of packs we
649
+ * want to consider kept.
650
+ */
651
+ in = xfdopen(cmd.in, "w");
652
+ for_each_string_list_item(item, names)
653
+ fprintf(in, "%s-%s.pack\n", pack_prefix, item->string);
654
+ for_each_string_list_item(item, existing_packs)
655
+ fprintf(in, "-%s.pack\n", item->string);
656
+ for_each_string_list_item(item, existing_kept_packs)
657
+ fprintf(in, "%s.pack\n", item->string);
658
+ fclose(in);
659
+
660
+ out = xfdopen(cmd.out, "r");
661
+ while (strbuf_getline_lf(&line, out) != EOF) {
662
+ if (line.len != the_hash_algo->hexsz)
663
+ die(_("repack: Expecting full hex object ID lines only "
664
+ "from pack-objects."));
665
+ string_list_append(names, line.buf);
666
+ }
667
+ fclose(out);
668
+
669
+ strbuf_release(&line);
670
+
671
+ return finish_command(&cmd);
672
+}
673
+
674
int cmd_repack(int argc, const char **argv, const char *prefix)
675
{
676
struct child_process cmd = CHILD_PROCESS_INIT;
@@ -621,7 +687,6 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
687
int show_progress;
688
689
/* variables to be filled by option parsing */
624
- int pack_everything = 0;
690
int delete_redundant = 0;
691
const char *unpack_unreachable = NULL;
692
int keep_unreachable = 0;
@@ -636,6 +701,11 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
701
OPT_BIT('A', NULL, &pack_everything,
702
N_("same as -a, and turn unreachable objects loose"),
703
LOOSEN_UNREACHABLE | ALL_INTO_ONE),
704
+ OPT_BIT(0, "cruft", &pack_everything,
705
+ N_("same as -a, pack unreachable cruft objects separately"),
706
+ PACK_CRUFT),
707
+ OPT_STRING(0, "cruft-expiration", &cruft_expiration, N_("approxidate"),
708
+ N_("with -C, expire objects older than this")),
709
OPT_BOOL('d', NULL, &delete_redundant,
710
N_("remove redundant packs, and run git-prune-packed")),
711
OPT_BOOL('f', NULL, &po_args.no_reuse_delta,
@@ -688,6 +758,15 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
758
(unpack_unreachable || (pack_everything & LOOSEN_UNREACHABLE)))
759
die(_("options '%s' and '%s' cannot be used together"), "--keep-unreachable", "-A");
760
761
+ if (pack_everything & PACK_CRUFT) {
762
+ pack_everything |= ALL_INTO_ONE;
763
+
764
+ if (unpack_unreachable || (pack_everything & LOOSEN_UNREACHABLE))
765
+ die(_("options '%s' and '%s' cannot be used together"), "--cruft", "-A");
766
+ if (keep_unreachable)
767
+ die(_("options '%s' and '%s' cannot be used together"), "--cruft", "-k");
768
+ }
769
+
770
if (write_bitmaps < 0) {
771
if (!write_midx &&
772
(!(pack_everything & ALL_INTO_ONE) || !is_bare_repository()))
@@ -771,7 +850,8 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
850
if (pack_everything & ALL_INTO_ONE) {
851
repack_promisor_objects(&po_args, &names);
852
774
- if (existing_nonkept_packs.nr && delete_redundant) {
853
+ if (existing_nonkept_packs.nr && delete_redundant &&
854
+ !(pack_everything & PACK_CRUFT)) {
855
for_each_string_list_item(item, &names) {
856
strvec_pushf(&cmd.args, "--keep-pack=%s-%s.pack",
857
packtmp_name, item->string);
@@ -833,6 +913,21 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
913
if (!names.nr && !po_args.quiet)
914
printf_ln(_("Nothing new to pack."));
915
916
+ if (pack_everything & PACK_CRUFT) {
917
+ const char *pack_prefix;
918
+ if (!skip_prefix(packtmp, packdir, &pack_prefix))
919
+ die(_("pack prefix %s does not begin with objdir %s"),
920
+ packtmp, packdir);
921
+ if (*pack_prefix == '/')
922
+ pack_prefix++;
923
+
924
+ ret = write_cruft_pack(&po_args, pack_prefix, &names,
925
+ &existing_nonkept_packs,
926
+ &existing_kept_packs);
927
+ if (ret)
928
+ return ret;
929
+ }
930
+
931
for_each_string_list_item(item, &names) {
932
item->util = (void *)(uintptr_t)populate_pack_exts(item->string);
933
}
t/t5329-pack-objects-cruft.sh
+207
@@ -358,4 +358,211 @@ test_expect_success 'expired objects are pruned' '
358
)
359
'
360
361
+test_expect_success 'repack --cruft generates a cruft pack' '
362
+ git init repo &&
363
+ test_when_finished "rm -fr repo" &&
364
+ (
365
+ cd repo &&
366
+
367
+ test_commit reachable &&
368
+ git branch -M main &&
369
+ git checkout --orphan other &&
370
+ test_commit unreachable &&
371
+
372
+ git checkout main &&
373
+ git branch -D other &&
374
+ git tag -d unreachable &&
375
+ # objects are not cruft if they are contained in the reflogs
376
+ git reflog expire --all --expire=all &&
377
+
378
+ git rev-list --objects --all --no-object-names >reachable.raw &&
379
+ git cat-file --batch-all-objects --batch-check="%(objectname)" >objects &&
380
+ sort <reachable.raw >reachable &&
381
+ comm -13 reachable objects >unreachable &&
382
+
383
+ git repack --cruft -d &&
384
+
385
+ cruft=$(basename $(ls $packdir/pack-*.mtimes) .mtimes) &&
386
+ pack=$(basename $(ls $packdir/pack-*.pack | grep -v $cruft) .pack) &&
387
+
388
+ git show-index <$packdir/$pack.idx >actual.raw &&
389
+ cut -f2 -d" " actual.raw | sort >actual &&
390
+ test_cmp reachable actual &&
391
+
392
+ git show-index <$packdir/$cruft.idx >actual.raw &&
393
+ cut -f2 -d" " actual.raw | sort >actual &&
394
+ test_cmp unreachable actual
395
+ )
396
+'
397
+
398
+test_expect_success 'loose objects mtimes upsert others' '
399
+ git init repo &&
400
+ test_when_finished "rm -fr repo" &&
401
+ (
402
+ cd repo &&
403
+
404
+ test_commit reachable &&
405
+ git repack -Ad &&
406
+ git branch -M main &&
407
+
408
+ git checkout --orphan other &&
409
+ test_commit cruft &&
410
+ # incremental repack, leaving existing objects loose (so
411
+ # they can be "freshened")
412
+ git repack &&
413
+
414
+ tip="$(git rev-parse cruft)" &&
415
+ path="$objdir/$(test_oid_to_path "$tip")" &&
416
+ test-tool chmtime --get +1000 "$path" >expect &&
417
+
418
+ git checkout main &&
419
+ git branch -D other &&
420
+ git tag -d cruft &&
421
+ git reflog expire --all --expire=all &&
422
+
423
+ git repack --cruft -d &&
424
+
425
+ mtimes="$(basename $(ls $packdir/pack-*.mtimes))" &&
426
+ test-tool pack-mtimes "$mtimes" >actual.raw &&
427
+ grep "$tip" actual.raw | cut -d" " -f2 >actual &&
428
+ test_cmp expect actual
429
+ )
430
+'
431
+
432
+test_expect_success 'cruft packs are not included in geometric repack' '
433
+ git init repo &&
434
+ test_when_finished "rm -fr repo" &&
435
+ (
436
+ cd repo &&
437
+
438
+ test_commit reachable &&
439
+ git repack -Ad &&
440
+ git branch -M main &&
441
+
442
+ git checkout --orphan other &&
443
+ test_commit cruft &&
444
+ git repack -d &&
445
+
446
+ git checkout main &&
447
+ git branch -D other &&
448
+ git tag -d cruft &&
449
+ git reflog expire --all --expire=all &&
450
+
451
+ git repack --cruft &&
452
+
453
+ find $packdir -type f | sort >before &&
454
+ git repack --geometric=2 -d &&
455
+ find $packdir -type f | sort >after &&
456
+
457
+ test_cmp before after
458
+ )
459
+'
460
+
461
+test_expect_success 'repack --geometric collects once-cruft objects' '
462
+ git init repo &&
463
+ test_when_finished "rm -fr repo" &&
464
+ (
465
+ cd repo &&
466
+
467
+ test_commit reachable &&
468
+ git repack -Ad &&
469
+ git branch -M main &&
470
+
471
+ git checkout --orphan other &&
472
+ git rm -rf . &&
473
+ test_commit --no-tag cruft &&
474
+ cruft="$(git rev-parse HEAD)" &&
475
+
476
+ git checkout main &&
477
+ git branch -D other &&
478
+ git reflog expire --all --expire=all &&
479
+
480
+ # Pack the objects created in the previous step into a cruft
481
+ # pack. Intentionally leave loose copies of those objects
482
+ # around so we can pick them up in a subsequent --geometric
483
+ # reapack.
484
+ git repack --cruft &&
485
+
486
+ # Now make those objects reachable, and ensure that they are
487
+ # packed into the new pack created via a --geometric repack.
488
+ git update-ref refs/heads/other $cruft &&
489
+
490
+ # Without this object, the set of unpacked objects is exactly
491
+ # the set of objects already in the cruft pack. Tweak that set
492
+ # to ensure we do not overwrite the cruft pack entirely.
493
+ test_commit reachable2 &&
494
+
495
+ find $packdir -name "pack-*.idx" | sort >before &&
496
+ git repack --geometric=2 -d &&
497
+ find $packdir -name "pack-*.idx" | sort >after &&
498
+
499
+ {
500
+ git rev-list --objects --no-object-names $cruft &&
501
+ git rev-list --objects --no-object-names reachable..reachable2
502
+ } >want.raw &&
503
+ sort want.raw >want &&
504
+
505
+ pack=$(comm -13 before after) &&
506
+ git show-index <$pack >objects.raw &&
507
+
508
+ cut -d" " -f2 objects.raw | sort >got &&
509
+
510
+ test_cmp want got
511
+ )
512
+'
513
+
514
+test_expect_success 'cruft repack with no reachable objects' '
515
+ git init repo &&
516
+ test_when_finished "rm -fr repo" &&
517
+ (
518
+ cd repo &&
519
+
520
+ test_commit base &&
521
+ git repack -ad &&
522
+
523
+ base="$(git rev-parse base)" &&
524
+
525
+ git for-each-ref --format="delete %(refname)" >in &&
526
+ git update-ref --stdin <in &&
527
+ git reflog expire --all --expire=all &&
528
+ rm -fr .git/index &&
529
+
530
+ git repack --cruft -d &&
531
+
532
+ git cat-file -t $base
533
+ )
534
+'
535
+
536
+test_expect_success 'cruft repack ignores --max-pack-size' '
537
+ git init max-pack-size &&
538
+ (
539
+ cd max-pack-size &&
540
+ test_commit base &&
541
+ # two cruft objects which exceed the maximum pack size
542
+ test-tool genrandom foo 1048576 | git hash-object --stdin -w &&
543
+ test-tool genrandom bar 1048576 | git hash-object --stdin -w &&
544
+ git repack --cruft --max-pack-size=1M &&
545
+ find $packdir -name "*.mtimes" >cruft &&
546
+ test_line_count = 1 cruft &&
547
+ test-tool pack-mtimes "$(basename "$(cat cruft)")" >objects &&
548
+ test_line_count = 2 objects
549
+ )
550
+'
551
+
552
+test_expect_success 'cruft repack ignores pack.packSizeLimit' '
553
+ (
554
+ cd max-pack-size &&
555
+ # repack everything back together to remove the existing cruft
556
+ # pack (but to keep its objects)
557
+ git repack -adk &&
558
+ git -c pack.packSizeLimit=1M repack --cruft &&
559
+ # ensure the same post condition is met when --max-pack-size
560
+ # would otherwise be inferred from the configuration
561
+ find $packdir -name "*.mtimes" >cruft &&
562
+ test_line_count = 1 cruft &&
563
+ test-tool pack-mtimes "$(basename "$(cat cruft)")" >objects &&
564
+ test_line_count = 2 objects
565
+ )
566
+'
567
+
568
test_done