pack-bitmap-write: learn pack.writeBitmapLookupTable and add tests
Teach Git to provide a way for users to enable/disable bitmap lookup table extension by providing a config option named 'writeBitmapLookupTable'. Default is false. Also add test to verify writting of lookup table. Mentored-by: Taylor Blau <me@ttaylorr.com> Co-Mentored-by: Kaartic Sivaraam <kaartic.sivaraam@gmail.com> Co-Authored-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Abhradeep Chakraborty <chakrabortyabhradeep79@gmail.com> Reviewed-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Abhradeep Chakraborty committed
Aug 14, 2022 at 16:55 UTC
76f14b777c13ac8d8f6a2e5812945e725bea3d84
9 files changed
+733
-583
Documentation/config/pack.txt
+7
@@ -164,6 +164,13 @@ When writing a multi-pack reachability bitmap, no new namehashes are
164
computed; instead, any namehashes stored in an existing bitmap are
165
permuted into their appropriate location when writing a new bitmap.
166
167
+pack.writeBitmapLookupTable::
168
+ When true, Git will include a "lookup table" section in the
169
+ bitmap index (if one is written). This table is used to defer
170
+ loading individual bitmaps as late as possible. This can be
171
+ beneficial in repositories that have relatively large bitmap
172
+ indexes. Defaults to false.
173
+
174
pack.writeReverseIndex::
175
When true, git will write a corresponding .rev file (see:
176
link:../technical/pack-format.html[Documentation/technical/pack-format.txt])
builtin/multi-pack-index.c
+7
@@ -87,6 +87,13 @@ static int git_multi_pack_index_write_config(const char *var, const char *value,
87
opts.flags &= ~MIDX_WRITE_BITMAP_HASH_CACHE;
88
}
89
90
+ if (!strcmp(var, "pack.writebitmaplookuptable")) {
91
+ if (git_config_bool(var, value))
92
+ opts.flags |= MIDX_WRITE_BITMAP_LOOKUP_TABLE;
93
+ else
94
+ opts.flags &= ~MIDX_WRITE_BITMAP_LOOKUP_TABLE;
95
+ }
96
+
97
/*
98
* We should never make a fall-back call to 'git_default_config', since
99
* this was already called in 'cmd_multi_pack_index()'.
builtin/pack-objects.c
+8
@@ -3148,6 +3148,14 @@ static int git_pack_config(const char *k, const char *v, void *cb)
3148
else
3149
write_bitmap_options &= ~BITMAP_OPT_HASH_CACHE;
3150
}
3151
+
3152
+ if (!strcmp(k, "pack.writebitmaplookuptable")) {
3153
+ if (git_config_bool(k, v))
3154
+ write_bitmap_options |= BITMAP_OPT_LOOKUP_TABLE;
3155
+ else
3156
+ write_bitmap_options &= ~BITMAP_OPT_LOOKUP_TABLE;
3157
+ }
3158
+
3159
if (!strcmp(k, "pack.usebitmaps")) {
3160
use_bitmap_index_default = git_config_bool(k, v);
3161
return 0;
midx.c
+3
@@ -1070,6 +1070,9 @@ static int write_midx_bitmap(const char *midx_name,
1070
if (flags & MIDX_WRITE_BITMAP_HASH_CACHE)
1071
options |= BITMAP_OPT_HASH_CACHE;
1072
1073
+ if (flags & MIDX_WRITE_BITMAP_LOOKUP_TABLE)
1074
+ options |= BITMAP_OPT_LOOKUP_TABLE;
1075
+
1076
/*
1077
* Build the MIDX-order index based on pdata.objects (which is already
1078
* in MIDX order; c.f., 'midx_pack_order_cmp()' for the definition of
midx.h
+1
@@ -47,6 +47,7 @@ struct multi_pack_index {
47
#define MIDX_WRITE_REV_INDEX (1 << 1)
48
#define MIDX_WRITE_BITMAP (1 << 2)
49
#define MIDX_WRITE_BITMAP_HASH_CACHE (1 << 3)
50
+#define MIDX_WRITE_BITMAP_LOOKUP_TABLE (1 << 4)
51
52
const unsigned char *get_midx_checksum(struct multi_pack_index *m);
53
void get_midx_filename(struct strbuf *out, const char *object_dir);
t/t5310-pack-bitmaps.sh
+410
-382
@@ -26,22 +26,413 @@ has_any () {
26
grep -Ff "$1" "$2"
27
}
28
29
-setup_bitmap_history
30
-
31
-test_expect_success 'setup writing bitmaps during repack' '
32
- git config repack.writeBitmaps true
33
-'
34
-
35
-test_expect_success 'full repack creates bitmaps' '
36
- GIT_TRACE2_EVENT="$(pwd)/trace" \
29
+test_bitmap_cases () {
30
+ writeLookupTable=false
31
+ for i in "$@"
32
+ do
33
+ case "$i" in
34
+ "pack.writeBitmapLookupTable") writeLookupTable=true;;
35
+ esac
36
+ done
37
+
38
+ test_expect_success 'setup test repository' '
39
+ rm -fr * .git &&
40
+ git init &&
41
+ git config pack.writeBitmapLookupTable '"$writeLookupTable"'
42
+ '
43
+ setup_bitmap_history
44
+
45
+ test_expect_success 'setup writing bitmaps during repack' '
46
+ git config repack.writeBitmaps true
47
+ '
48
+
49
+ test_expect_success 'full repack creates bitmaps' '
50
+ GIT_TRACE2_EVENT="$(pwd)/trace" \
51
+ git repack -ad &&
52
+ ls .git/objects/pack/ | grep bitmap >output &&
53
+ test_line_count = 1 output &&
54
+ grep "\"key\":\"num_selected_commits\",\"value\":\"106\"" trace &&
55
+ grep "\"key\":\"num_maximal_commits\",\"value\":\"107\"" trace
56
+ '
57
+
58
+ basic_bitmap_tests
59
+
60
+ test_expect_success 'pack-objects respects --local (non-local loose)' '
61
+ git init --bare alt.git &&
62
+ echo $(pwd)/alt.git/objects >.git/objects/info/alternates &&
63
+ echo content1 >file1 &&
64
+ # non-local loose object which is not present in bitmapped pack
65
+ altblob=$(GIT_DIR=alt.git git hash-object -w file1) &&
66
+ # non-local loose object which is also present in bitmapped pack
67
+ git cat-file blob $blob | GIT_DIR=alt.git git hash-object -w --stdin &&
68
+ git add file1 &&
69
+ test_tick &&
70
+ git commit -m commit_file1 &&
71
+ echo HEAD | git pack-objects --local --stdout --revs >1.pack &&
72
+ git index-pack 1.pack &&
73
+ list_packed_objects 1.idx >1.objects &&
74
+ printf "%s\n" "$altblob" "$blob" >nonlocal-loose &&
75
+ ! has_any nonlocal-loose 1.objects
76
+ '
77
+
78
+ test_expect_success 'pack-objects respects --honor-pack-keep (local non-bitmapped pack)' '
79
+ echo content2 >file2 &&
80
+ blob2=$(git hash-object -w file2) &&
81
+ git add file2 &&
82
+ test_tick &&
83
+ git commit -m commit_file2 &&
84
+ printf "%s\n" "$blob2" "$bitmaptip" >keepobjects &&
85
+ pack2=$(git pack-objects pack2 <keepobjects) &&
86
+ mv pack2-$pack2.* .git/objects/pack/ &&
87
+ >.git/objects/pack/pack2-$pack2.keep &&
88
+ rm $(objpath $blob2) &&
89
+ echo HEAD | git pack-objects --honor-pack-keep --stdout --revs >2a.pack &&
90
+ git index-pack 2a.pack &&
91
+ list_packed_objects 2a.idx >2a.objects &&
92
+ ! has_any keepobjects 2a.objects
93
+ '
94
+
95
+ test_expect_success 'pack-objects respects --local (non-local pack)' '
96
+ mv .git/objects/pack/pack2-$pack2.* alt.git/objects/pack/ &&
97
+ echo HEAD | git pack-objects --local --stdout --revs >2b.pack &&
98
+ git index-pack 2b.pack &&
99
+ list_packed_objects 2b.idx >2b.objects &&
100
+ ! has_any keepobjects 2b.objects
101
+ '
102
+
103
+ test_expect_success 'pack-objects respects --honor-pack-keep (local bitmapped pack)' '
104
+ ls .git/objects/pack/ | grep bitmap >output &&
105
+ test_line_count = 1 output &&
106
+ packbitmap=$(basename $(cat output) .bitmap) &&
107
+ list_packed_objects .git/objects/pack/$packbitmap.idx >packbitmap.objects &&
108
+ test_when_finished "rm -f .git/objects/pack/$packbitmap.keep" &&
109
+ >.git/objects/pack/$packbitmap.keep &&
110
+ echo HEAD | git pack-objects --honor-pack-keep --stdout --revs >3a.pack &&
111
+ git index-pack 3a.pack &&
112
+ list_packed_objects 3a.idx >3a.objects &&
113
+ ! has_any packbitmap.objects 3a.objects
114
+ '
115
+
116
+ test_expect_success 'pack-objects respects --local (non-local bitmapped pack)' '
117
+ mv .git/objects/pack/$packbitmap.* alt.git/objects/pack/ &&
118
+ rm -f .git/objects/pack/multi-pack-index &&
119
+ test_when_finished "mv alt.git/objects/pack/$packbitmap.* .git/objects/pack/" &&
120
+ echo HEAD | git pack-objects --local --stdout --revs >3b.pack &&
121
+ git index-pack 3b.pack &&
122
+ list_packed_objects 3b.idx >3b.objects &&
123
+ ! has_any packbitmap.objects 3b.objects
124
+ '
125
+
126
+ test_expect_success 'pack-objects to file can use bitmap' '
127
+ # make sure we still have 1 bitmap index from previous tests
128
+ ls .git/objects/pack/ | grep bitmap >output &&
129
+ test_line_count = 1 output &&
130
+ # verify equivalent packs are generated with/without using bitmap index
131
+ packasha1=$(git pack-objects --no-use-bitmap-index --all packa </dev/null) &&
132
+ packbsha1=$(git pack-objects --use-bitmap-index --all packb </dev/null) &&
133
+ list_packed_objects packa-$packasha1.idx >packa.objects &&
134
+ list_packed_objects packb-$packbsha1.idx >packb.objects &&
135
+ test_cmp packa.objects packb.objects
136
+ '
137
+
138
+ test_expect_success 'full repack, reusing previous bitmaps' '
139
git repack -ad &&
38
- ls .git/objects/pack/ | grep bitmap >output &&
39
- test_line_count = 1 output &&
40
- grep "\"key\":\"num_selected_commits\",\"value\":\"106\"" trace &&
41
- grep "\"key\":\"num_maximal_commits\",\"value\":\"107\"" trace
42
-'
140
+ ls .git/objects/pack/ | grep bitmap >output &&
141
+ test_line_count = 1 output
142
+ '
143
+
144
+ test_expect_success 'fetch (full bitmap)' '
145
+ git --git-dir=clone.git fetch origin second:second &&
146
+ git rev-parse HEAD >expect &&
147
+ git --git-dir=clone.git rev-parse HEAD >actual &&
148
+ test_cmp expect actual
149
+ '
150
+
151
+ test_expect_success 'create objects for missing-HAVE tests' '
152
+ blob=$(echo "missing have" | git hash-object -w --stdin) &&
153
+ tree=$(printf "100644 blob $blob\tfile\n" | git mktree) &&
154
+ parent=$(echo parent | git commit-tree $tree) &&
155
+ commit=$(echo commit | git commit-tree $tree -p $parent) &&
156
+ cat >revs <<-EOF
157
+ HEAD
158
+ ^HEAD^
159
+ ^$commit
160
+ EOF
161
+ '
162
+
163
+ test_expect_success 'pack-objects respects --incremental' '
164
+ cat >revs2 <<-EOF &&
165
+ HEAD
166
+ $commit
167
+ EOF
168
+ git pack-objects --incremental --stdout --revs <revs2 >4.pack &&
169
+ git index-pack 4.pack &&
170
+ list_packed_objects 4.idx >4.objects &&
171
+ test_line_count = 4 4.objects &&
172
+ git rev-list --objects $commit >revlist &&
173
+ cut -d" " -f1 revlist |sort >objects &&
174
+ test_cmp 4.objects objects
175
+ '
176
+
177
+ test_expect_success 'pack with missing blob' '
178
+ rm $(objpath $blob) &&
179
+ git pack-objects --stdout --revs <revs >/dev/null
180
+ '
181
+
182
+ test_expect_success 'pack with missing tree' '
183
+ rm $(objpath $tree) &&
184
+ git pack-objects --stdout --revs <revs >/dev/null
185
+ '
186
+
187
+ test_expect_success 'pack with missing parent' '
188
+ rm $(objpath $parent) &&
189
+ git pack-objects --stdout --revs <revs >/dev/null
190
+ '
191
+
192
+ test_expect_success JGIT,SHA1 'we can read jgit bitmaps' '
193
+ git clone --bare . compat-jgit.git &&
194
+ (
195
+ cd compat-jgit.git &&
196
+ rm -f objects/pack/*.bitmap &&
197
+ jgit gc &&
198
+ git rev-list --test-bitmap HEAD
199
+ )
200
+ '
201
+
202
+ test_expect_success JGIT,SHA1 'jgit can read our bitmaps' '
203
+ git clone --bare . compat-us.git &&
204
+ (
205
+ cd compat-us.git &&
206
+ git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
207
+ git repack -adb &&
208
+ # jgit gc will barf if it does not like our bitmaps
209
+ jgit gc
210
+ )
211
+ '
212
+
213
+ test_expect_success 'splitting packs does not generate bogus bitmaps' '
214
+ test-tool genrandom foo $((1024 * 1024)) >rand &&
215
+ git add rand &&
216
+ git commit -m "commit with big file" &&
217
+ git -c pack.packSizeLimit=500k repack -adb &&
218
+ git init --bare no-bitmaps.git &&
219
+ git -C no-bitmaps.git fetch .. HEAD
220
+ '
221
+
222
+ test_expect_success 'set up reusable pack' '
223
+ rm -f .git/objects/pack/*.keep &&
224
+ git repack -adb &&
225
+ reusable_pack () {
226
+ git for-each-ref --format="%(objectname)" |
227
+ git pack-objects --delta-base-offset --revs --stdout "$@"
228
+ }
229
+ '
230
+
231
+ test_expect_success 'pack reuse respects --honor-pack-keep' '
232
+ test_when_finished "rm -f .git/objects/pack/*.keep" &&
233
+ for i in .git/objects/pack/*.pack
234
+ do
235
+ >${i%.pack}.keep || return 1
236
+ done &&
237
+ reusable_pack --honor-pack-keep >empty.pack &&
238
+ git index-pack empty.pack &&
239
+ git show-index <empty.idx >actual &&
240
+ test_must_be_empty actual
241
+ '
242
+
243
+ test_expect_success 'pack reuse respects --local' '
244
+ mv .git/objects/pack/* alt.git/objects/pack/ &&
245
+ test_when_finished "mv alt.git/objects/pack/* .git/objects/pack/" &&
246
+ reusable_pack --local >empty.pack &&
247
+ git index-pack empty.pack &&
248
+ git show-index <empty.idx >actual &&
249
+ test_must_be_empty actual
250
+ '
251
+
252
+ test_expect_success 'pack reuse respects --incremental' '
253
+ reusable_pack --incremental >empty.pack &&
254
+ git index-pack empty.pack &&
255
+ git show-index <empty.idx >actual &&
256
+ test_must_be_empty actual
257
+ '
258
+
259
+ test_expect_success 'truncated bitmap fails gracefully (ewah)' '
260
+ test_config pack.writebitmaphashcache false &&
261
+ git repack -ad &&
262
+ git rev-list --use-bitmap-index --count --all >expect &&
263
+ bitmap=$(ls .git/objects/pack/*.bitmap) &&
264
+ test_when_finished "rm -f $bitmap" &&
265
+ test_copy_bytes 256 <$bitmap >$bitmap.tmp &&
266
+ mv -f $bitmap.tmp $bitmap &&
267
+ git rev-list --use-bitmap-index --count --all >actual 2>stderr &&
268
+ test_cmp expect actual &&
269
+ test_i18ngrep corrupt.ewah.bitmap stderr
270
+ '
271
+
272
+ test_expect_success 'truncated bitmap fails gracefully (cache)' '
273
+ git repack -ad &&
274
+ git rev-list --use-bitmap-index --count --all >expect &&
275
+ bitmap=$(ls .git/objects/pack/*.bitmap) &&
276
+ test_when_finished "rm -f $bitmap" &&
277
+ test_copy_bytes 512 <$bitmap >$bitmap.tmp &&
278
+ mv -f $bitmap.tmp $bitmap &&
279
+ git rev-list --use-bitmap-index --count --all >actual 2>stderr &&
280
+ test_cmp expect actual &&
281
+ test_i18ngrep corrupted.bitmap.index stderr
282
+ '
283
+
284
+ # Create a state of history with these properties:
285
+ #
286
+ # - refs that allow a client to fetch some new history, while sharing some old
287
+ # history with the server; we use branches delta-reuse-old and
288
+ # delta-reuse-new here
289
+ #
290
+ # - the new history contains an object that is stored on the server as a delta
291
+ # against a base that is in the old history
292
+ #
293
+ # - the base object is not immediately reachable from the tip of the old
294
+ # history; finding it would involve digging down through history we know the
295
+ # other side has
296
+ #
297
+ # This should result in a state where fetching from old->new would not
298
+ # traditionally reuse the on-disk delta (because we'd have to dig to realize
299
+ # that the client has it), but we will do so if bitmaps can tell us cheaply
300
+ # that the other side has it.
301
+ test_expect_success 'set up thin delta-reuse parent' '
302
+ # This first commit contains the buried base object.
303
+ test-tool genrandom delta 16384 >file &&
304
+ git add file &&
305
+ git commit -m "delta base" &&
306
+ base=$(git rev-parse --verify HEAD:file) &&
307
+
308
+ # These intermediate commits bury the base back in history.
309
+ # This becomes the "old" state.
310
+ for i in 1 2 3 4 5
311
+ do
312
+ echo $i >file &&
313
+ git commit -am "intermediate $i" || return 1
314
+ done &&
315
+ git branch delta-reuse-old &&
316
+
317
+ # And now our new history has a delta against the buried base. Note
318
+ # that this must be smaller than the original file, since pack-objects
319
+ # prefers to create deltas from smaller objects to larger.
320
+ test-tool genrandom delta 16300 >file &&
321
+ git commit -am "delta result" &&
322
+ delta=$(git rev-parse --verify HEAD:file) &&
323
+ git branch delta-reuse-new &&
324
+
325
+ # Repack with bitmaps and double check that we have the expected delta
326
+ # relationship.
327
+ git repack -adb &&
328
+ have_delta $delta $base
329
+ '
330
+
331
+ # Now we can sanity-check the non-bitmap behavior (that the server is not able
332
+ # to reuse the delta). This isn't strictly something we care about, so this
333
+ # test could be scrapped in the future. But it makes sure that the next test is
334
+ # actually triggering the feature we want.
335
+ #
336
+ # Note that our tools for working with on-the-wire "thin" packs are limited. So
337
+ # we actually perform the fetch, retain the resulting pack, and inspect the
338
+ # result.
339
+ test_expect_success 'fetch without bitmaps ignores delta against old base' '
340
+ test_config pack.usebitmaps false &&
341
+ test_when_finished "rm -rf client.git" &&
342
+ git init --bare client.git &&
343
+ (
344
+ cd client.git &&
345
+ git config transfer.unpackLimit 1 &&
346
+ git fetch .. delta-reuse-old:delta-reuse-old &&
347
+ git fetch .. delta-reuse-new:delta-reuse-new &&
348
+ have_delta $delta $ZERO_OID
349
+ )
350
+ '
351
+
352
+ # And do the same for the bitmap case, where we do expect to find the delta.
353
+ test_expect_success 'fetch with bitmaps can reuse old base' '
354
+ test_config pack.usebitmaps true &&
355
+ test_when_finished "rm -rf client.git" &&
356
+ git init --bare client.git &&
357
+ (
358
+ cd client.git &&
359
+ git config transfer.unpackLimit 1 &&
360
+ git fetch .. delta-reuse-old:delta-reuse-old &&
361
+ git fetch .. delta-reuse-new:delta-reuse-new &&
362
+ have_delta $delta $base
363
+ )
364
+ '
365
+
366
+ test_expect_success 'pack.preferBitmapTips' '
367
+ git init repo &&
368
+ test_when_finished "rm -fr repo" &&
369
+ (
370
+ cd repo &&
371
+ git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
372
+
373
+ # create enough commits that not all are receive bitmap
374
+ # coverage even if they are all at the tip of some reference.
375
+ test_commit_bulk --message="%s" 103 &&
376
+
377
+ git rev-list HEAD >commits.raw &&
378
+ sort <commits.raw >commits &&
379
+
380
+ git log --format="create refs/tags/%s %H" HEAD >refs &&
381
+ git update-ref --stdin <refs &&
382
+
383
+ git repack -adb &&
384
+ test-tool bitmap list-commits | sort >bitmaps &&
385
+
386
+ # remember which commits did not receive bitmaps
387
+ comm -13 bitmaps commits >before &&
388
+ test_file_not_empty before &&
389
+
390
+ # mark the commits which did not receive bitmaps as preferred,
391
+ # and generate the bitmap again
392
+ perl -pe "s{^}{create refs/tags/include/$. }" <before |
393
+ git update-ref --stdin &&
394
+ git -c pack.preferBitmapTips=refs/tags/include repack -adb &&
395
+
396
+ # finally, check that the commit(s) without bitmap coverage
397
+ # are not the same ones as before
398
+ test-tool bitmap list-commits | sort >bitmaps &&
399
+ comm -13 bitmaps commits >after &&
400
+
401
+ ! test_cmp before after
402
+ )
403
+ '
404
+
405
+ test_expect_success 'complains about multiple pack bitmaps' '
406
+ rm -fr repo &&
407
+ git init repo &&
408
+ test_when_finished "rm -fr repo" &&
409
+ (
410
+ cd repo &&
411
+ git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
412
+
413
+ test_commit base &&
414
+
415
+ git repack -adb &&
416
+ bitmap="$(ls .git/objects/pack/pack-*.bitmap)" &&
417
+ mv "$bitmap" "$bitmap.bak" &&
418
+
419
+ test_commit other &&
420
+ git repack -ab &&
421
+
422
+ mv "$bitmap.bak" "$bitmap" &&
423
+
424
+ find .git/objects/pack -type f -name "*.pack" >packs &&
425
+ find .git/objects/pack -type f -name "*.bitmap" >bitmaps &&
426
+ test_line_count = 2 packs &&
427
+ test_line_count = 2 bitmaps &&
428
+
429
+ git rev-list --use-bitmap-index HEAD 2>err &&
430
+ grep "ignoring extra bitmap file" err
431
+ )
432
+ '
433
+}
434
44
-basic_bitmap_tests
435
+test_bitmap_cases
436
437
test_expect_success 'incremental repack fails when bitmaps are requested' '
438
test_commit more-1 &&
@@ -54,375 +445,12 @@ test_expect_success 'incremental repack can disable bitmaps' '
445
git repack -d --no-write-bitmap-index
446
'
447
57
-test_expect_success 'pack-objects respects --local (non-local loose)' '
58
- git init --bare alt.git &&
59
- echo $(pwd)/alt.git/objects >.git/objects/info/alternates &&
60
- echo content1 >file1 &&
61
- # non-local loose object which is not present in bitmapped pack
62
- altblob=$(GIT_DIR=alt.git git hash-object -w file1) &&
63
- # non-local loose object which is also present in bitmapped pack
64
- git cat-file blob $blob | GIT_DIR=alt.git git hash-object -w --stdin &&
65
- git add file1 &&
66
- test_tick &&
67
- git commit -m commit_file1 &&
68
- echo HEAD | git pack-objects --local --stdout --revs >1.pack &&
69
- git index-pack 1.pack &&
70
- list_packed_objects 1.idx >1.objects &&
71
- printf "%s\n" "$altblob" "$blob" >nonlocal-loose &&
72
- ! has_any nonlocal-loose 1.objects
73
-'
74
-
75
-test_expect_success 'pack-objects respects --honor-pack-keep (local non-bitmapped pack)' '
76
- echo content2 >file2 &&
77
- blob2=$(git hash-object -w file2) &&
78
- git add file2 &&
79
- test_tick &&
80
- git commit -m commit_file2 &&
81
- printf "%s\n" "$blob2" "$bitmaptip" >keepobjects &&
82
- pack2=$(git pack-objects pack2 <keepobjects) &&
83
- mv pack2-$pack2.* .git/objects/pack/ &&
84
- >.git/objects/pack/pack2-$pack2.keep &&
85
- rm $(objpath $blob2) &&
86
- echo HEAD | git pack-objects --honor-pack-keep --stdout --revs >2a.pack &&
87
- git index-pack 2a.pack &&
88
- list_packed_objects 2a.idx >2a.objects &&
89
- ! has_any keepobjects 2a.objects
90
-'
91
-
92
-test_expect_success 'pack-objects respects --local (non-local pack)' '
93
- mv .git/objects/pack/pack2-$pack2.* alt.git/objects/pack/ &&
94
- echo HEAD | git pack-objects --local --stdout --revs >2b.pack &&
95
- git index-pack 2b.pack &&
96
- list_packed_objects 2b.idx >2b.objects &&
97
- ! has_any keepobjects 2b.objects
98
-'
99
-
100
-test_expect_success 'pack-objects respects --honor-pack-keep (local bitmapped pack)' '
101
- ls .git/objects/pack/ | grep bitmap >output &&
102
- test_line_count = 1 output &&
103
- packbitmap=$(basename $(cat output) .bitmap) &&
104
- list_packed_objects .git/objects/pack/$packbitmap.idx >packbitmap.objects &&
105
- test_when_finished "rm -f .git/objects/pack/$packbitmap.keep" &&
106
- >.git/objects/pack/$packbitmap.keep &&
107
- echo HEAD | git pack-objects --honor-pack-keep --stdout --revs >3a.pack &&
108
- git index-pack 3a.pack &&
109
- list_packed_objects 3a.idx >3a.objects &&
110
- ! has_any packbitmap.objects 3a.objects
111
-'
112
-
113
-test_expect_success 'pack-objects respects --local (non-local bitmapped pack)' '
114
- mv .git/objects/pack/$packbitmap.* alt.git/objects/pack/ &&
115
- rm -f .git/objects/pack/multi-pack-index &&
116
- test_when_finished "mv alt.git/objects/pack/$packbitmap.* .git/objects/pack/" &&
117
- echo HEAD | git pack-objects --local --stdout --revs >3b.pack &&
118
- git index-pack 3b.pack &&
119
- list_packed_objects 3b.idx >3b.objects &&
120
- ! has_any packbitmap.objects 3b.objects
121
-'
122
-
123
-test_expect_success 'pack-objects to file can use bitmap' '
124
- # make sure we still have 1 bitmap index from previous tests
125
- ls .git/objects/pack/ | grep bitmap >output &&
126
- test_line_count = 1 output &&
127
- # verify equivalent packs are generated with/without using bitmap index
128
- packasha1=$(git pack-objects --no-use-bitmap-index --all packa </dev/null) &&
129
- packbsha1=$(git pack-objects --use-bitmap-index --all packb </dev/null) &&
130
- list_packed_objects packa-$packasha1.idx >packa.objects &&
131
- list_packed_objects packb-$packbsha1.idx >packb.objects &&
132
- test_cmp packa.objects packb.objects
133
-'
134
-
135
-test_expect_success 'full repack, reusing previous bitmaps' '
136
- git repack -ad &&
137
- ls .git/objects/pack/ | grep bitmap >output &&
138
- test_line_count = 1 output
139
-'
140
-
141
-test_expect_success 'fetch (full bitmap)' '
142
- git --git-dir=clone.git fetch origin second:second &&
143
- git rev-parse HEAD >expect &&
144
- git --git-dir=clone.git rev-parse HEAD >actual &&
145
- test_cmp expect actual
146
-'
147
-
148
-test_expect_success 'create objects for missing-HAVE tests' '
149
- blob=$(echo "missing have" | git hash-object -w --stdin) &&
150
- tree=$(printf "100644 blob $blob\tfile\n" | git mktree) &&
151
- parent=$(echo parent | git commit-tree $tree) &&
152
- commit=$(echo commit | git commit-tree $tree -p $parent) &&
153
- cat >revs <<-EOF
154
- HEAD
155
- ^HEAD^
156
- ^$commit
157
- EOF
158
-'
159
-
160
-test_expect_success 'pack-objects respects --incremental' '
161
- cat >revs2 <<-EOF &&
162
- HEAD
163
- $commit
164
- EOF
165
- git pack-objects --incremental --stdout --revs <revs2 >4.pack &&
166
- git index-pack 4.pack &&
167
- list_packed_objects 4.idx >4.objects &&
168
- test_line_count = 4 4.objects &&
169
- git rev-list --objects $commit >revlist &&
170
- cut -d" " -f1 revlist |sort >objects &&
171
- test_cmp 4.objects objects
172
-'
173
-
174
-test_expect_success 'pack with missing blob' '
175
- rm $(objpath $blob) &&
176
- git pack-objects --stdout --revs <revs >/dev/null
177
-'
448
+test_bitmap_cases "pack.writeBitmapLookupTable"
449
179
-test_expect_success 'pack with missing tree' '
180
- rm $(objpath $tree) &&
181
- git pack-objects --stdout --revs <revs >/dev/null
182
-'
183
-
184
-test_expect_success 'pack with missing parent' '
185
- rm $(objpath $parent) &&
186
- git pack-objects --stdout --revs <revs >/dev/null
187
-'
188
-
189
-test_expect_success JGIT,SHA1 'we can read jgit bitmaps' '
190
- git clone --bare . compat-jgit.git &&
191
- (
192
- cd compat-jgit.git &&
193
- rm -f objects/pack/*.bitmap &&
194
- jgit gc &&
195
- git rev-list --test-bitmap HEAD
196
- )
197
-'
198
-
199
-test_expect_success JGIT,SHA1 'jgit can read our bitmaps' '
200
- git clone --bare . compat-us.git &&
201
- (
202
- cd compat-us.git &&
203
- git repack -adb &&
204
- # jgit gc will barf if it does not like our bitmaps
205
- jgit gc
206
- )
207
-'
208
-
209
-test_expect_success 'splitting packs does not generate bogus bitmaps' '
210
- test-tool genrandom foo $((1024 * 1024)) >rand &&
211
- git add rand &&
212
- git commit -m "commit with big file" &&
213
- git -c pack.packSizeLimit=500k repack -adb &&
214
- git init --bare no-bitmaps.git &&
215
- git -C no-bitmaps.git fetch .. HEAD
216
-'
217
-
218
-test_expect_success 'set up reusable pack' '
219
- rm -f .git/objects/pack/*.keep &&
220
- git repack -adb &&
221
- reusable_pack () {
222
- git for-each-ref --format="%(objectname)" |
223
- git pack-objects --delta-base-offset --revs --stdout "$@"
224
- }
225
-'
226
-
227
-test_expect_success 'pack reuse respects --honor-pack-keep' '
228
- test_when_finished "rm -f .git/objects/pack/*.keep" &&
229
- for i in .git/objects/pack/*.pack
230
- do
231
- >${i%.pack}.keep || return 1
232
- done &&
233
- reusable_pack --honor-pack-keep >empty.pack &&
234
- git index-pack empty.pack &&
235
- git show-index <empty.idx >actual &&
236
- test_must_be_empty actual
237
-'
238
-
239
-test_expect_success 'pack reuse respects --local' '
240
- mv .git/objects/pack/* alt.git/objects/pack/ &&
241
- test_when_finished "mv alt.git/objects/pack/* .git/objects/pack/" &&
242
- reusable_pack --local >empty.pack &&
243
- git index-pack empty.pack &&
244
- git show-index <empty.idx >actual &&
245
- test_must_be_empty actual
246
-'
247
-
248
-test_expect_success 'pack reuse respects --incremental' '
249
- reusable_pack --incremental >empty.pack &&
250
- git index-pack empty.pack &&
251
- git show-index <empty.idx >actual &&
252
- test_must_be_empty actual
253
-'
254
-
255
-test_expect_success 'truncated bitmap fails gracefully (ewah)' '
256
- test_config pack.writebitmaphashcache false &&
257
- git repack -ad &&
258
- git rev-list --use-bitmap-index --count --all >expect &&
259
- bitmap=$(ls .git/objects/pack/*.bitmap) &&
260
- test_when_finished "rm -f $bitmap" &&
261
- test_copy_bytes 256 <$bitmap >$bitmap.tmp &&
262
- mv -f $bitmap.tmp $bitmap &&
263
- git rev-list --use-bitmap-index --count --all >actual 2>stderr &&
264
- test_cmp expect actual &&
265
- test_i18ngrep corrupt.ewah.bitmap stderr
266
-'
267
-
268
-test_expect_success 'truncated bitmap fails gracefully (cache)' '
269
- git repack -ad &&
270
- git rev-list --use-bitmap-index --count --all >expect &&
271
- bitmap=$(ls .git/objects/pack/*.bitmap) &&
272
- test_when_finished "rm -f $bitmap" &&
273
- test_copy_bytes 512 <$bitmap >$bitmap.tmp &&
274
- mv -f $bitmap.tmp $bitmap &&
275
- git rev-list --use-bitmap-index --count --all >actual 2>stderr &&
276
- test_cmp expect actual &&
277
- test_i18ngrep corrupted.bitmap.index stderr
278
-'
279
-
280
-# Create a state of history with these properties:
281
-#
282
-# - refs that allow a client to fetch some new history, while sharing some old
283
-# history with the server; we use branches delta-reuse-old and
284
-# delta-reuse-new here
285
-#
286
-# - the new history contains an object that is stored on the server as a delta
287
-# against a base that is in the old history
288
-#
289
-# - the base object is not immediately reachable from the tip of the old
290
-# history; finding it would involve digging down through history we know the
291
-# other side has
292
-#
293
-# This should result in a state where fetching from old->new would not
294
-# traditionally reuse the on-disk delta (because we'd have to dig to realize
295
-# that the client has it), but we will do so if bitmaps can tell us cheaply
296
-# that the other side has it.
297
-test_expect_success 'set up thin delta-reuse parent' '
298
- # This first commit contains the buried base object.
299
- test-tool genrandom delta 16384 >file &&
300
- git add file &&
301
- git commit -m "delta base" &&
302
- base=$(git rev-parse --verify HEAD:file) &&
303
-
304
- # These intermediate commits bury the base back in history.
305
- # This becomes the "old" state.
306
- for i in 1 2 3 4 5
307
- do
308
- echo $i >file &&
309
- git commit -am "intermediate $i" || return 1
310
- done &&
311
- git branch delta-reuse-old &&
312
-
313
- # And now our new history has a delta against the buried base. Note
314
- # that this must be smaller than the original file, since pack-objects
315
- # prefers to create deltas from smaller objects to larger.
316
- test-tool genrandom delta 16300 >file &&
317
- git commit -am "delta result" &&
318
- delta=$(git rev-parse --verify HEAD:file) &&
319
- git branch delta-reuse-new &&
320
-
321
- # Repack with bitmaps and double check that we have the expected delta
322
- # relationship.
323
- git repack -adb &&
324
- have_delta $delta $base
325
-'
326
-
327
-# Now we can sanity-check the non-bitmap behavior (that the server is not able
328
-# to reuse the delta). This isn't strictly something we care about, so this
329
-# test could be scrapped in the future. But it makes sure that the next test is
330
-# actually triggering the feature we want.
331
-#
332
-# Note that our tools for working with on-the-wire "thin" packs are limited. So
333
-# we actually perform the fetch, retain the resulting pack, and inspect the
334
-# result.
335
-test_expect_success 'fetch without bitmaps ignores delta against old base' '
336
- test_config pack.usebitmaps false &&
337
- test_when_finished "rm -rf client.git" &&
338
- git init --bare client.git &&
339
- (
340
- cd client.git &&
341
- git config transfer.unpackLimit 1 &&
342
- git fetch .. delta-reuse-old:delta-reuse-old &&
343
- git fetch .. delta-reuse-new:delta-reuse-new &&
344
- have_delta $delta $ZERO_OID
345
- )
346
-'
347
-
348
-# And do the same for the bitmap case, where we do expect to find the delta.
349
-test_expect_success 'fetch with bitmaps can reuse old base' '
350
- test_config pack.usebitmaps true &&
351
- test_when_finished "rm -rf client.git" &&
352
- git init --bare client.git &&
353
- (
354
- cd client.git &&
355
- git config transfer.unpackLimit 1 &&
356
- git fetch .. delta-reuse-old:delta-reuse-old &&
357
- git fetch .. delta-reuse-new:delta-reuse-new &&
358
- have_delta $delta $base
359
- )
360
-'
361
-
362
-test_expect_success 'pack.preferBitmapTips' '
363
- git init repo &&
364
- test_when_finished "rm -fr repo" &&
365
- (
366
- cd repo &&
367
-
368
- # create enough commits that not all are receive bitmap
369
- # coverage even if they are all at the tip of some reference.
370
- test_commit_bulk --message="%s" 103 &&
371
-
372
- git rev-list HEAD >commits.raw &&
373
- sort <commits.raw >commits &&
374
-
375
- git log --format="create refs/tags/%s %H" HEAD >refs &&
376
- git update-ref --stdin <refs &&
377
-
378
- git repack -adb &&
379
- test-tool bitmap list-commits | sort >bitmaps &&
380
-
381
- # remember which commits did not receive bitmaps
382
- comm -13 bitmaps commits >before &&
383
- test_file_not_empty before &&
384
-
385
- # mark the commits which did not receive bitmaps as preferred,
386
- # and generate the bitmap again
387
- perl -pe "s{^}{create refs/tags/include/$. }" <before |
388
- git update-ref --stdin &&
389
- git -c pack.preferBitmapTips=refs/tags/include repack -adb &&
390
-
391
- # finally, check that the commit(s) without bitmap coverage
392
- # are not the same ones as before
393
- test-tool bitmap list-commits | sort >bitmaps &&
394
- comm -13 bitmaps commits >after &&
395
-
396
- ! test_cmp before after
397
- )
398
-'
399
-
400
-test_expect_success 'complains about multiple pack bitmaps' '
401
- rm -fr repo &&
402
- git init repo &&
403
- test_when_finished "rm -fr repo" &&
404
- (
405
- cd repo &&
406
-
407
- test_commit base &&
408
-
409
- git repack -adb &&
410
- bitmap="$(ls .git/objects/pack/pack-*.bitmap)" &&
411
- mv "$bitmap" "$bitmap.bak" &&
412
-
413
- test_commit other &&
414
- git repack -ab &&
415
-
416
- mv "$bitmap.bak" "$bitmap" &&
417
-
418
- find .git/objects/pack -type f -name "*.pack" >packs &&
419
- find .git/objects/pack -type f -name "*.bitmap" >bitmaps &&
420
- test_line_count = 2 packs &&
421
- test_line_count = 2 bitmaps &&
422
-
423
- git rev-list --use-bitmap-index HEAD 2>err &&
424
- grep "ignoring extra bitmap file" err
425
- )
450
+test_expect_success 'verify writing bitmap lookup table when enabled' '
451
+ GIT_TRACE2_EVENT="$(pwd)/trace2" \
452
+ git repack -ad &&
453
+ grep "\"label\":\"writing_lookup_table\"" trace2
454
'
455
456
test_done
t/t5311-pack-bitmaps-shallow.sh
+35
-18
@@ -17,23 +17,40 @@ test_description='check bitmap operation with shallow repositories'
17
# the tree for A. But in a shallow one, we've grafted away
18
# A, and fetching A to B requires that the other side send
19
# us the tree for file=1.
20
-test_expect_success 'setup shallow repo' '
21
- echo 1 >file &&
22
- git add file &&
23
- git commit -m orig &&
24
- echo 2 >file &&
25
- git commit -a -m update &&
26
- git clone --no-local --bare --depth=1 . shallow.git &&
27
- echo 1 >file &&
28
- git commit -a -m repeat
29
-'
30
-
31
-test_expect_success 'turn on bitmaps in the parent' '
32
- git repack -adb
33
-'
34
-
35
-test_expect_success 'shallow fetch from bitmapped repo' '
36
- (cd shallow.git && git fetch)
37
-'
20
+test_shallow_bitmaps () {
21
+ writeLookupTable=false
22
+
23
+ for i in "$@"
24
+ do
25
+ case $i in
26
+ "pack.writeBitmapLookupTable") writeLookupTable=true;;
27
+ esac
28
+ done
29
+
30
+ test_expect_success 'setup shallow repo' '
31
+ rm -rf * .git &&
32
+ git init &&
33
+ git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
34
+ echo 1 >file &&
35
+ git add file &&
36
+ git commit -m orig &&
37
+ echo 2 >file &&
38
+ git commit -a -m update &&
39
+ git clone --no-local --bare --depth=1 . shallow.git &&
40
+ echo 1 >file &&
41
+ git commit -a -m repeat
42
+ '
43
+
44
+ test_expect_success 'turn on bitmaps in the parent' '
45
+ git repack -adb
46
+ '
47
+
48
+ test_expect_success 'shallow fetch from bitmapped repo' '
49
+ (cd shallow.git && git fetch)
50
+ '
51
+}
52
+
53
+test_shallow_bitmaps
54
+test_shallow_bitmaps "pack.writeBitmapLookupTable"
55
56
test_done
t/t5326-multi-pack-bitmaps.sh
+240
-181
@@ -15,17 +15,24 @@ GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=0
15
sane_unset GIT_TEST_MIDX_WRITE_REV
16
sane_unset GIT_TEST_MIDX_READ_RIDX
17
18
-midx_bitmap_core
19
-
18
bitmap_reuse_tests() {
19
from=$1
20
to=$2
21
+ writeLookupTable=false
22
+
23
+ for i in $3-${$#}
24
+ do
25
+ case $i in
26
+ "pack.writeBitmapLookupTable") writeLookupTable=true;;
27
+ esac
28
+ done
29
30
test_expect_success "setup pack reuse tests ($from -> $to)" '
31
rm -fr repo &&
32
git init repo &&
33
(
34
cd repo &&
35
+ git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
36
test_commit_bulk 16 &&
37
git tag old-tip &&
38
@@ -43,6 +50,7 @@ bitmap_reuse_tests() {
50
test_expect_success "build bitmap from existing ($from -> $to)" '
51
(
52
cd repo &&
53
+ git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
54
test_commit_bulk --id=further 16 &&
55
git tag new-tip &&
56
@@ -59,6 +67,7 @@ bitmap_reuse_tests() {
67
test_expect_success "verify resulting bitmaps ($from -> $to)" '
68
(
69
cd repo &&
70
+ git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
71
git for-each-ref &&
72
git rev-list --test-bitmap refs/tags/old-tip &&
73
git rev-list --test-bitmap refs/tags/new-tip
@@ -66,244 +75,294 @@ bitmap_reuse_tests() {
75
'
76
}
77
69
-bitmap_reuse_tests 'pack' 'MIDX'
70
-bitmap_reuse_tests 'MIDX' 'pack'
71
-bitmap_reuse_tests 'MIDX' 'MIDX'
78
+test_midx_bitmap_cases () {
79
+ writeLookupTable=false
80
+ writeBitmapLookupTable=
81
+
82
+ for i in "$@"
83
+ do
84
+ case $i in
85
+ "pack.writeBitmapLookupTable")
86
+ writeLookupTable=true
87
+ writeBitmapLookupTable="$i"
88
+ ;;
89
+ esac
90
+ done
91
+
92
+ test_expect_success 'setup test_repository' '
93
+ rm -rf * .git &&
94
+ git init &&
95
+ git config pack.writeBitmapLookupTable '"$writeLookupTable"'
96
+ '
97
73
-test_expect_success 'missing object closure fails gracefully' '
74
- rm -fr repo &&
75
- git init repo &&
76
- test_when_finished "rm -fr repo" &&
77
- (
78
- cd repo &&
98
+ midx_bitmap_core
99
80
- test_commit loose &&
81
- test_commit packed &&
100
+ bitmap_reuse_tests 'pack' 'MIDX' "$writeBitmapLookupTable"
101
+ bitmap_reuse_tests 'MIDX' 'pack' "$writeBitmapLookupTable"
102
+ bitmap_reuse_tests 'MIDX' 'MIDX' "$writeBitmapLookupTable"
103
83
- # Do not pass "--revs"; we want a pack without the "loose"
84
- # commit.
85
- git pack-objects $objdir/pack/pack <<-EOF &&
86
- $(git rev-parse packed)
87
- EOF
104
+ test_expect_success 'missing object closure fails gracefully' '
105
+ rm -fr repo &&
106
+ git init repo &&
107
+ test_when_finished "rm -fr repo" &&
108
+ (
109
+ cd repo &&
110
+ git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
111
89
- test_must_fail git multi-pack-index write --bitmap 2>err &&
90
- grep "doesn.t have full closure" err &&
91
- test_path_is_missing $midx
92
- )
93
-'
112
+ test_commit loose &&
113
+ test_commit packed &&
114
95
-midx_bitmap_partial_tests
115
+ # Do not pass "--revs"; we want a pack without the "loose"
116
+ # commit.
117
+ git pack-objects $objdir/pack/pack <<-EOF &&
118
+ $(git rev-parse packed)
119
+ EOF
120
97
-test_expect_success 'removing a MIDX clears stale bitmaps' '
98
- rm -fr repo &&
99
- git init repo &&
100
- test_when_finished "rm -fr repo" &&
101
- (
102
- cd repo &&
103
- test_commit base &&
104
- git repack &&
105
- git multi-pack-index write --bitmap &&
121
+ test_must_fail git multi-pack-index write --bitmap 2>err &&
122
+ grep "doesn.t have full closure" err &&
123
+ test_path_is_missing $midx
124
+ )
125
+ '
126
107
- # Write a MIDX and bitmap; remove the MIDX but leave the bitmap.
108
- stale_bitmap=$midx-$(midx_checksum $objdir).bitmap &&
109
- rm $midx &&
127
+ midx_bitmap_partial_tests
128
111
- # Then write a new MIDX.
112
- test_commit new &&
113
- git repack &&
114
- git multi-pack-index write --bitmap &&
129
+ test_expect_success 'removing a MIDX clears stale bitmaps' '
130
+ rm -fr repo &&
131
+ git init repo &&
132
+ test_when_finished "rm -fr repo" &&
133
+ (
134
+ cd repo &&
135
+ git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
136
+ test_commit base &&
137
+ git repack &&
138
+ git multi-pack-index write --bitmap &&
139
+
140
+ # Write a MIDX and bitmap; remove the MIDX but leave the bitmap.
141
+ stale_bitmap=$midx-$(midx_checksum $objdir).bitmap &&
142
+ rm $midx &&
143
+
144
+ # Then write a new MIDX.
145
+ test_commit new &&
146
+ git repack &&
147
+ git multi-pack-index write --bitmap &&
148
+
149
+ test_path_is_file $midx &&
150
+ test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
151
+ test_path_is_missing $stale_bitmap
152
+ )
153
+ '
154
116
- test_path_is_file $midx &&
117
- test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
118
- test_path_is_missing $stale_bitmap
119
- )
120
-'
155
+ test_expect_success 'pack.preferBitmapTips' '
156
+ git init repo &&
157
+ test_when_finished "rm -fr repo" &&
158
+ (
159
+ cd repo &&
160
+ git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
161
122
-test_expect_success 'pack.preferBitmapTips' '
123
- git init repo &&
124
- test_when_finished "rm -fr repo" &&
125
- (
126
- cd repo &&
162
+ test_commit_bulk --message="%s" 103 &&
163
128
- test_commit_bulk --message="%s" 103 &&
164
+ git log --format="%H" >commits.raw &&
165
+ sort <commits.raw >commits &&
166
130
- git log --format="%H" >commits.raw &&
131
- sort <commits.raw >commits &&
167
+ git log --format="create refs/tags/%s %H" HEAD >refs &&
168
+ git update-ref --stdin <refs &&
169
133
- git log --format="create refs/tags/%s %H" HEAD >refs &&
134
- git update-ref --stdin <refs &&
170
+ git multi-pack-index write --bitmap &&
171
+ test_path_is_file $midx &&
172
+ test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
173
136
- git multi-pack-index write --bitmap &&
137
- test_path_is_file $midx &&
138
- test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
174
+ test-tool bitmap list-commits | sort >bitmaps &&
175
+ comm -13 bitmaps commits >before &&
176
+ test_line_count = 1 before &&
177
140
- test-tool bitmap list-commits | sort >bitmaps &&
141
- comm -13 bitmaps commits >before &&
142
- test_line_count = 1 before &&
178
+ perl -ne "printf(\"create refs/tags/include/%d \", $.); print" \
179
+ <before | git update-ref --stdin &&
180
144
- perl -ne "printf(\"create refs/tags/include/%d \", $.); print" \
145
- <before | git update-ref --stdin &&
181
+ rm -fr $midx-$(midx_checksum $objdir).bitmap &&
182
+ rm -fr $midx &&
183
147
- rm -fr $midx-$(midx_checksum $objdir).bitmap &&
148
- rm -fr $midx &&
184
+ git -c pack.preferBitmapTips=refs/tags/include \
185
+ multi-pack-index write --bitmap &&
186
+ test-tool bitmap list-commits | sort >bitmaps &&
187
+ comm -13 bitmaps commits >after &&
188
150
- git -c pack.preferBitmapTips=refs/tags/include \
151
- multi-pack-index write --bitmap &&
152
- test-tool bitmap list-commits | sort >bitmaps &&
153
- comm -13 bitmaps commits >after &&
189
+ ! test_cmp before after
190
+ )
191
+ '
192
155
- ! test_cmp before after
156
- )
157
-'
193
+ test_expect_success 'writing a bitmap with --refs-snapshot' '
194
+ git init repo &&
195
+ test_when_finished "rm -fr repo" &&
196
+ (
197
+ cd repo &&
198
+ git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
199
159
-test_expect_success 'writing a bitmap with --refs-snapshot' '
160
- git init repo &&
161
- test_when_finished "rm -fr repo" &&
162
- (
163
- cd repo &&
200
+ test_commit one &&
201
+ test_commit two &&
202
165
- test_commit one &&
166
- test_commit two &&
203
+ git rev-parse one >snapshot &&
204
168
- git rev-parse one >snapshot &&
205
+ git repack -ad &&
206
170
- git repack -ad &&
207
+ # First, write a MIDX which see both refs/tags/one and
208
+ # refs/tags/two (causing both of those commits to receive
209
+ # bitmaps).
210
+ git multi-pack-index write --bitmap &&
211
172
- # First, write a MIDX which see both refs/tags/one and
173
- # refs/tags/two (causing both of those commits to receive
174
- # bitmaps).
175
- git multi-pack-index write --bitmap &&
212
+ test_path_is_file $midx &&
213
+ test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
214
177
- test_path_is_file $midx &&
178
- test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
215
+ test-tool bitmap list-commits | sort >bitmaps &&
216
+ grep "$(git rev-parse one)" bitmaps &&
217
+ grep "$(git rev-parse two)" bitmaps &&
218
180
- test-tool bitmap list-commits | sort >bitmaps &&
181
- grep "$(git rev-parse one)" bitmaps &&
182
- grep "$(git rev-parse two)" bitmaps &&
219
+ rm -fr $midx-$(midx_checksum $objdir).bitmap &&
220
+ rm -fr $midx &&
221
184
- rm -fr $midx-$(midx_checksum $objdir).bitmap &&
185
- rm -fr $midx &&
222
+ # Then again, but with a refs snapshot which only sees
223
+ # refs/tags/one.
224
+ git multi-pack-index write --bitmap --refs-snapshot=snapshot &&
225
187
- # Then again, but with a refs snapshot which only sees
188
- # refs/tags/one.
189
- git multi-pack-index write --bitmap --refs-snapshot=snapshot &&
226
+ test_path_is_file $midx &&
227
+ test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
228
191
- test_path_is_file $midx &&
192
- test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
229
+ test-tool bitmap list-commits | sort >bitmaps &&
230
+ grep "$(git rev-parse one)" bitmaps &&
231
+ ! grep "$(git rev-parse two)" bitmaps
232
+ )
233
+ '
234
194
- test-tool bitmap list-commits | sort >bitmaps &&
195
- grep "$(git rev-parse one)" bitmaps &&
196
- ! grep "$(git rev-parse two)" bitmaps
197
- )
198
-'
235
+ test_expect_success 'write a bitmap with --refs-snapshot (preferred tips)' '
236
+ git init repo &&
237
+ test_when_finished "rm -fr repo" &&
238
+ (
239
+ cd repo &&
240
+ git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
241
200
-test_expect_success 'write a bitmap with --refs-snapshot (preferred tips)' '
201
- git init repo &&
202
- test_when_finished "rm -fr repo" &&
203
- (
204
- cd repo &&
242
+ test_commit_bulk --message="%s" 103 &&
243
206
- test_commit_bulk --message="%s" 103 &&
244
+ git log --format="%H" >commits.raw &&
245
+ sort <commits.raw >commits &&
246
208
- git log --format="%H" >commits.raw &&
209
- sort <commits.raw >commits &&
247
+ git log --format="create refs/tags/%s %H" HEAD >refs &&
248
+ git update-ref --stdin <refs &&
249
211
- git log --format="create refs/tags/%s %H" HEAD >refs &&
212
- git update-ref --stdin <refs &&
250
+ git multi-pack-index write --bitmap &&
251
+ test_path_is_file $midx &&
252
+ test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
253
214
- git multi-pack-index write --bitmap &&
215
- test_path_is_file $midx &&
216
- test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
254
+ test-tool bitmap list-commits | sort >bitmaps &&
255
+ comm -13 bitmaps commits >before &&
256
+ test_line_count = 1 before &&
257
218
- test-tool bitmap list-commits | sort >bitmaps &&
219
- comm -13 bitmaps commits >before &&
220
- test_line_count = 1 before &&
258
+ (
259
+ grep -vf before commits.raw &&
260
+ # mark missing commits as preferred
261
+ sed "s/^/+/" before
262
+ ) >snapshot &&
263
264
+ rm -fr $midx-$(midx_checksum $objdir).bitmap &&
265
+ rm -fr $midx &&
266
+
267
+ git multi-pack-index write --bitmap --refs-snapshot=snapshot &&
268
+ test-tool bitmap list-commits | sort >bitmaps &&
269
+ comm -13 bitmaps commits >after &&
270
+
271
+ ! test_cmp before after
272
+ )
273
+ '
274
+
275
+ test_expect_success 'hash-cache values are propagated from pack bitmaps' '
276
+ rm -fr repo &&
277
+ git init repo &&
278
+ test_when_finished "rm -fr repo" &&
279
(
223
- grep -vf before commits.raw &&
224
- # mark missing commits as preferred
225
- sed "s/^/+/" before
226
- ) >snapshot &&
280
+ cd repo &&
281
+ git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
282
228
- rm -fr $midx-$(midx_checksum $objdir).bitmap &&
229
- rm -fr $midx &&
283
+ test_commit base &&
284
+ test_commit base2 &&
285
+ git repack -adb &&
286
231
- git multi-pack-index write --bitmap --refs-snapshot=snapshot &&
232
- test-tool bitmap list-commits | sort >bitmaps &&
233
- comm -13 bitmaps commits >after &&
287
+ test-tool bitmap dump-hashes >pack.raw &&
288
+ test_file_not_empty pack.raw &&
289
+ sort pack.raw >pack.hashes &&
290
235
- ! test_cmp before after
236
- )
237
-'
291
+ test_commit new &&
292
+ git repack &&
293
+ git multi-pack-index write --bitmap &&
294
239
-test_expect_success 'hash-cache values are propagated from pack bitmaps' '
240
- rm -fr repo &&
241
- git init repo &&
242
- test_when_finished "rm -fr repo" &&
243
- (
244
- cd repo &&
295
+ test-tool bitmap dump-hashes >midx.raw &&
296
+ sort midx.raw >midx.hashes &&
297
246
- test_commit base &&
247
- test_commit base2 &&
248
- git repack -adb &&
298
+ # ensure that every namehash in the pack bitmap can be found in
299
+ # the midx bitmap (i.e., that there are no oid-namehash pairs
300
+ # unique to the pack bitmap).
301
+ comm -23 pack.hashes midx.hashes >dropped.hashes &&
302
+ test_must_be_empty dropped.hashes
303
+ )
304
+ '
305
250
- test-tool bitmap dump-hashes >pack.raw &&
251
- test_file_not_empty pack.raw &&
252
- sort pack.raw >pack.hashes &&
306
+ test_expect_success 'no .bitmap is written without any objects' '
307
+ rm -fr repo &&
308
+ git init repo &&
309
+ test_when_finished "rm -fr repo" &&
310
+ (
311
+ cd repo &&
312
+ git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
313
254
- test_commit new &&
255
- git repack &&
256
- git multi-pack-index write --bitmap &&
314
+ empty="$(git pack-objects $objdir/pack/pack </dev/null)" &&
315
+ cat >packs <<-EOF &&
316
+ pack-$empty.idx
317
+ EOF
318
258
- test-tool bitmap dump-hashes >midx.raw &&
259
- sort midx.raw >midx.hashes &&
319
+ git multi-pack-index write --bitmap --stdin-packs \
320
+ <packs 2>err &&
321
261
- # ensure that every namehash in the pack bitmap can be found in
262
- # the midx bitmap (i.e., that there are no oid-namehash pairs
263
- # unique to the pack bitmap).
264
- comm -23 pack.hashes midx.hashes >dropped.hashes &&
265
- test_must_be_empty dropped.hashes
266
- )
267
-'
322
+ grep "bitmap without any objects" err &&
323
269
-test_expect_success 'no .bitmap is written without any objects' '
270
- rm -fr repo &&
271
- git init repo &&
272
- test_when_finished "rm -fr repo" &&
273
- (
274
- cd repo &&
324
+ test_path_is_file $midx &&
325
+ test_path_is_missing $midx-$(midx_checksum $objdir).bitmap
326
+ )
327
+ '
328
+
329
+ test_expect_success 'graceful fallback when missing reverse index' '
330
+ rm -fr repo &&
331
+ git init repo &&
332
+ test_when_finished "rm -fr repo" &&
333
+ (
334
+ cd repo &&
335
+ git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
336
276
- empty="$(git pack-objects $objdir/pack/pack </dev/null)" &&
277
- cat >packs <<-EOF &&
278
- pack-$empty.idx
279
- EOF
337
+ test_commit base &&
338
281
- git multi-pack-index write --bitmap --stdin-packs \
282
- <packs 2>err &&
339
+ # write a pack and MIDX bitmap containing base
340
+ git repack -adb &&
341
+ git multi-pack-index write --bitmap &&
342
284
- grep "bitmap without any objects" err &&
343
+ GIT_TEST_MIDX_READ_RIDX=0 \
344
+ git rev-list --use-bitmap-index HEAD 2>err &&
345
+ ! grep "ignoring extra bitmap file" err
346
+ )
347
+ '
348
+}
349
286
- test_path_is_file $midx &&
287
- test_path_is_missing $midx-$(midx_checksum $objdir).bitmap
288
- )
289
-'
350
+test_midx_bitmap_cases
351
+
352
+test_midx_bitmap_cases "pack.writeBitmapLookupTable"
353
291
-test_expect_success 'graceful fallback when missing reverse index' '
354
+test_expect_success 'multi-pack-index write writes lookup table if enabled' '
355
rm -fr repo &&
356
git init repo &&
357
test_when_finished "rm -fr repo" &&
358
(
359
cd repo &&
297
-
360
test_commit base &&
299
-
300
- # write a pack and MIDX bitmap containing base
301
- git repack -adb &&
302
- git multi-pack-index write --bitmap &&
303
-
304
- GIT_TEST_MIDX_READ_RIDX=0 \
305
- git rev-list --use-bitmap-index HEAD 2>err &&
306
- ! grep "ignoring extra bitmap file" err
361
+ git config pack.writeBitmapLookupTable true &&
362
+ git repack -ad &&
363
+ GIT_TRACE2_EVENT="$(pwd)/trace" \
364
+ git multi-pack-index write --bitmap &&
365
+ grep "\"label\":\"writing_lookup_table\"" trace
366
)
367
'
368
t/t5327-multi-pack-bitmaps-rev.sh
+22
-2
@@ -17,7 +17,27 @@ GIT_TEST_MIDX_READ_RIDX=0
17
export GIT_TEST_MIDX_WRITE_REV
18
export GIT_TEST_MIDX_READ_RIDX
19
20
-midx_bitmap_core rev
21
-midx_bitmap_partial_tests rev
20
+test_midx_bitmap_rev () {
21
+ writeLookupTable=false
22
+
23
+ for i in "$@"
24
+ do
25
+ case $i in
26
+ "pack.writeBitmapLookupTable") writeLookupTable=true;;
27
+ esac
28
+ done
29
+
30
+ test_expect_success 'setup bitmap config' '
31
+ rm -rf * .git &&
32
+ git init &&
33
+ git config pack.writeBitmapLookupTable '"$writeLookupTable"'
34
+ '
35
+
36
+ midx_bitmap_core rev
37
+ midx_bitmap_partial_tests rev
38
+}
39
+
40
+test_midx_bitmap_rev
41
+test_midx_bitmap_rev "pack.writeBitmapLookupTable"
42
43
test_done