multi-pack-index: implement 'expire' subcommand
The 'git multi-pack-index expire' subcommand looks at the existing mult-pack-index, counts the number of objects referenced in each pack-file, deletes the pack-fils with no referenced objects, and rewrites the multi-pack-index to no longer reference those packs. Refactor the write_midx_file() method to call write_midx_internal() which now takes an existing 'struct multi_pack_index' and a list of pack-files to drop (as specified by the names of their pack- indexes). As we write the new multi-pack-index, we drop those file names from the list of known pack-files. The expire_midx_packs() method removes the unreferenced pack-files after carefully closing the packs to avoid open handles. Test that a new pack-file that covers the contents of two other pack-files leads to those pack-files being deleted during the expire subcommand. Be sure to read the multi-pack-index to ensure it no longer references those packs. Signed-off-by: Derrick Stolee <dstolee@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Derrick Stolee committed
Jun 10, 2019 at 16:35 UTC
19575c7c8e60c95c6714e51039ee9a21721cc31d
2 files changed
+129
-10
midx.c
+109
-10
@@ -34,6 +34,8 @@
34
#define MIDX_CHUNK_LARGE_OFFSET_WIDTH (sizeof(uint64_t))
35
#define MIDX_LARGE_OFFSET_NEEDED 0x80000000
36
37
+#define PACK_EXPIRED UINT_MAX
38
+
39
static char *get_midx_filename(const char *object_dir)
40
{
41
return xstrfmt("%s/pack/multi-pack-index", object_dir);
@@ -431,6 +433,7 @@ struct pack_info {
433
uint32_t orig_pack_int_id;
434
char *pack_name;
435
struct packed_git *p;
436
+ unsigned expired : 1;
437
};
438
439
static int pack_info_compare(const void *_a, const void *_b)
@@ -478,6 +481,7 @@ static void add_pack_to_midx(const char *full_path, size_t full_path_len,
481
482
packs->info[packs->nr].pack_name = xstrdup(file_name);
483
packs->info[packs->nr].orig_pack_int_id = packs->nr;
484
+ packs->info[packs->nr].expired = 0;
485
packs->nr++;
486
}
487
}
@@ -637,13 +641,17 @@ static size_t write_midx_pack_names(struct hashfile *f,
641
size_t written = 0;
642
643
for (i = 0; i < num_packs; i++) {
640
- size_t writelen = strlen(info[i].pack_name) + 1;
644
+ size_t writelen;
645
+
646
+ if (info[i].expired)
647
+ continue;
648
649
if (i && strcmp(info[i].pack_name, info[i - 1].pack_name) <= 0)
650
BUG("incorrect pack-file order: %s before %s",
651
info[i - 1].pack_name,
652
info[i].pack_name);
653
654
+ writelen = strlen(info[i].pack_name) + 1;
655
hashwrite(f, info[i].pack_name, writelen);
656
written += writelen;
657
}
@@ -725,6 +733,11 @@ static size_t write_midx_object_offsets(struct hashfile *f, int large_offset_nee
733
for (i = 0; i < nr_objects; i++) {
734
struct pack_midx_entry *obj = list++;
735
736
+ if (perm[obj->pack_int_id] == PACK_EXPIRED)
737
+ BUG("object %s is in an expired pack with int-id %d",
738
+ oid_to_hex(&obj->oid),
739
+ obj->pack_int_id);
740
+
741
hashwrite_be32(f, perm[obj->pack_int_id]);
742
743
if (large_offset_needed && obj->offset >> 31)
@@ -771,7 +784,8 @@ static size_t write_midx_large_offsets(struct hashfile *f, uint32_t nr_large_off
784
return written;
785
}
786
774
-int write_midx_file(const char *object_dir)
787
+static int write_midx_internal(const char *object_dir, struct multi_pack_index *m,
788
+ struct string_list *packs_to_drop)
789
{
790
unsigned char cur_chunk, num_chunks = 0;
791
char *midx_name;
@@ -787,6 +801,8 @@ int write_midx_file(const char *object_dir)
801
struct pack_midx_entry *entries = NULL;
802
int large_offsets_needed = 0;
803
int pack_name_concat_len = 0;
804
+ int dropped_packs = 0;
805
+ int result = 0;
806
807
midx_name = get_midx_filename(object_dir);
808
if (safe_create_leading_directories(midx_name)) {
@@ -795,7 +811,10 @@ int write_midx_file(const char *object_dir)
811
midx_name);
812
}
813
798
- packs.m = load_multi_pack_index(object_dir, 1);
814
+ if (m)
815
+ packs.m = m;
816
+ else
817
+ packs.m = load_multi_pack_index(object_dir, 1);
818
819
packs.nr = 0;
820
packs.alloc = packs.m ? packs.m->num_packs : 16;
@@ -809,13 +828,14 @@ int write_midx_file(const char *object_dir)
828
packs.info[packs.nr].orig_pack_int_id = i;
829
packs.info[packs.nr].pack_name = xstrdup(packs.m->pack_names[i]);
830
packs.info[packs.nr].p = NULL;
831
+ packs.info[packs.nr].expired = 0;
832
packs.nr++;
833
}
834
}
835
836
for_each_file_in_pack_dir(object_dir, add_pack_to_midx, &packs);
837
818
- if (packs.m && packs.nr == packs.m->num_packs)
838
+ if (packs.m && packs.nr == packs.m->num_packs && !packs_to_drop)
839
goto cleanup;
840
841
entries = get_sorted_entries(packs.m, packs.info, packs.nr, &nr_entries);
@@ -829,6 +849,34 @@ int write_midx_file(const char *object_dir)
849
850
QSORT(packs.info, packs.nr, pack_info_compare);
851
852
+ if (packs_to_drop && packs_to_drop->nr) {
853
+ int drop_index = 0;
854
+ int missing_drops = 0;
855
+
856
+ for (i = 0; i < packs.nr && drop_index < packs_to_drop->nr; i++) {
857
+ int cmp = strcmp(packs.info[i].pack_name,
858
+ packs_to_drop->items[drop_index].string);
859
+
860
+ if (!cmp) {
861
+ drop_index++;
862
+ packs.info[i].expired = 1;
863
+ } else if (cmp > 0) {
864
+ error(_("did not see pack-file %s to drop"),
865
+ packs_to_drop->items[drop_index].string);
866
+ drop_index++;
867
+ missing_drops++;
868
+ i--;
869
+ } else {
870
+ packs.info[i].expired = 0;
871
+ }
872
+ }
873
+
874
+ if (missing_drops) {
875
+ result = 1;
876
+ goto cleanup;
877
+ }
878
+ }
879
+
880
/*
881
* pack_perm stores a permutation between pack-int-ids from the
882
* previous multi-pack-index to the new one we are writing:
@@ -837,11 +885,18 @@ int write_midx_file(const char *object_dir)
885
*/
886
ALLOC_ARRAY(pack_perm, packs.nr);
887
for (i = 0; i < packs.nr; i++) {
840
- pack_perm[packs.info[i].orig_pack_int_id] = i;
888
+ if (packs.info[i].expired) {
889
+ dropped_packs++;
890
+ pack_perm[packs.info[i].orig_pack_int_id] = PACK_EXPIRED;
891
+ } else {
892
+ pack_perm[packs.info[i].orig_pack_int_id] = i - dropped_packs;
893
+ }
894
}
895
843
- for (i = 0; i < packs.nr; i++)
844
- pack_name_concat_len += strlen(packs.info[i].pack_name) + 1;
896
+ for (i = 0; i < packs.nr; i++) {
897
+ if (!packs.info[i].expired)
898
+ pack_name_concat_len += strlen(packs.info[i].pack_name) + 1;
899
+ }
900
901
if (pack_name_concat_len % MIDX_CHUNK_ALIGNMENT)
902
pack_name_concat_len += MIDX_CHUNK_ALIGNMENT -
@@ -857,7 +912,7 @@ int write_midx_file(const char *object_dir)
912
cur_chunk = 0;
913
num_chunks = large_offsets_needed ? 5 : 4;
914
860
- written = write_midx_header(f, num_chunks, packs.nr);
915
+ written = write_midx_header(f, num_chunks, packs.nr - dropped_packs);
916
917
chunk_ids[cur_chunk] = MIDX_CHUNKID_PACKNAMES;
918
chunk_offsets[cur_chunk] = written + (num_chunks + 1) * MIDX_CHUNKLOOKUP_WIDTH;
@@ -958,7 +1013,12 @@ cleanup:
1013
free(entries);
1014
free(pack_perm);
1015
free(midx_name);
961
- return 0;
1016
+ return result;
1017
+}
1018
+
1019
+int write_midx_file(const char *object_dir)
1020
+{
1021
+ return write_midx_internal(object_dir, NULL, NULL);
1022
}
1023
1024
void clear_midx_file(struct repository *r)
@@ -1125,5 +1185,44 @@ int verify_midx_file(struct repository *r, const char *object_dir)
1185
1186
int expire_midx_packs(struct repository *r, const char *object_dir)
1187
{
1128
- return 0;
1188
+ uint32_t i, *count, result = 0;
1189
+ struct string_list packs_to_drop = STRING_LIST_INIT_DUP;
1190
+ struct multi_pack_index *m = load_multi_pack_index(object_dir, 1);
1191
+
1192
+ if (!m)
1193
+ return 0;
1194
+
1195
+ count = xcalloc(m->num_packs, sizeof(uint32_t));
1196
+ for (i = 0; i < m->num_objects; i++) {
1197
+ int pack_int_id = nth_midxed_pack_int_id(m, i);
1198
+ count[pack_int_id]++;
1199
+ }
1200
+
1201
+ for (i = 0; i < m->num_packs; i++) {
1202
+ char *pack_name;
1203
+
1204
+ if (count[i])
1205
+ continue;
1206
+
1207
+ if (prepare_midx_pack(r, m, i))
1208
+ continue;
1209
+
1210
+ if (m->packs[i]->pack_keep)
1211
+ continue;
1212
+
1213
+ pack_name = xstrdup(m->packs[i]->pack_name);
1214
+ close_pack(m->packs[i]);
1215
+
1216
+ string_list_insert(&packs_to_drop, m->pack_names[i]);
1217
+ unlink_pack_path(pack_name, 0);
1218
+ free(pack_name);
1219
+ }
1220
+
1221
+ free(count);
1222
+
1223
+ if (packs_to_drop.nr)
1224
+ result = write_midx_internal(object_dir, m, &packs_to_drop);
1225
+
1226
+ string_list_clear(&packs_to_drop, 0);
1227
+ return result;
1228
}
t/t5319-multi-pack-index.sh
+20
@@ -412,4 +412,24 @@ test_expect_success 'expire does not remove any packs' '
412
)
413
'
414
415
+test_expect_success 'expire removes unreferenced packs' '
416
+ (
417
+ cd dup &&
418
+ git pack-objects --revs .git/objects/pack/pack-combined <<-EOF &&
419
+ refs/heads/A
420
+ ^refs/heads/C
421
+ EOF
422
+ git multi-pack-index write &&
423
+ ls .git/objects/pack | grep -v -e pack-[AB] >expect &&
424
+ git multi-pack-index expire &&
425
+ ls .git/objects/pack >actual &&
426
+ test_cmp expect actual &&
427
+ ls .git/objects/pack/ | grep idx >expect-idx &&
428
+ test-tool read-midx .git/objects | grep idx >actual-midx &&
429
+ test_cmp expect-idx actual-midx &&
430
+ git multi-pack-index verify &&
431
+ git fsck
432
+ )
433
+'
434
+
435
test_done