t5310: move some tests to lib-bitmap.sh
We'll soon be adding a test script that will cover many of the same bitmap concepts as t5310, but for MIDX bitmaps. Let's pull out as many of the applicable tests as we can so we don't have to rewrite them. There should be no functional change to t5310; we still run the same operations in the same order. Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Taylor Blau committed
Aug 31, 2021 at 16:52 UTC
aeb4657242a4f7d9e507f88b24652f30f748cb56
2 files changed
+241
-224
t/lib-bitmap.sh
+236
@@ -1,3 +1,6 @@
1
+# Helpers for scripts testing bitmap functionality; see t5310 for
2
+# example usage.
3
+
4
# Compare a file containing rev-list bitmap traversal output to its non-bitmap
5
# counterpart. You can't just use test_cmp for this, because the two produce
6
# subtly different output:
@@ -24,3 +27,236 @@ test_bitmap_traversal () {
27
test_cmp "$1.normalized" "$2.normalized" &&
28
rm -f "$1.normalized" "$2.normalized"
29
}
30
+
31
+# To ensure the logic for "maximal commits" is exercised, make
32
+# the repository a bit more complicated.
33
+#
34
+# other second
35
+# * *
36
+# (99 commits) (99 commits)
37
+# * *
38
+# |\ /|
39
+# | * octo-other octo-second * |
40
+# |/|\_________ ____________/|\|
41
+# | \ \/ __________/ |
42
+# | | ________/\ / |
43
+# * |/ * merge-right *
44
+# | _|__________/ \____________ |
45
+# |/ | \|
46
+# (l1) * * merge-left * (r1)
47
+# | / \________________________ |
48
+# |/ \|
49
+# (l2) * * (r2)
50
+# \___________________________ |
51
+# \|
52
+# * (base)
53
+#
54
+# We only push bits down the first-parent history, which
55
+# makes some of these commits unimportant!
56
+#
57
+# The important part for the maximal commit algorithm is how
58
+# the bitmasks are extended. Assuming starting bit positions
59
+# for second (bit 0) and other (bit 1), the bitmasks at the
60
+# end should be:
61
+#
62
+# second: 1 (maximal, selected)
63
+# other: 01 (maximal, selected)
64
+# (base): 11 (maximal)
65
+#
66
+# This complicated history was important for a previous
67
+# version of the walk that guarantees never walking a
68
+# commit multiple times. That goal might be important
69
+# again, so preserve this complicated case. For now, this
70
+# test will guarantee that the bitmaps are computed
71
+# correctly, even with the repeat calculations.
72
+setup_bitmap_history() {
73
+ test_expect_success 'setup repo with moderate-sized history' '
74
+ test_commit_bulk --id=file 10 &&
75
+ git branch -M second &&
76
+ git checkout -b other HEAD~5 &&
77
+ test_commit_bulk --id=side 10 &&
78
+
79
+ # add complicated history setup, including merges and
80
+ # ambiguous merge-bases
81
+
82
+ git checkout -b merge-left other~2 &&
83
+ git merge second~2 -m "merge-left" &&
84
+
85
+ git checkout -b merge-right second~1 &&
86
+ git merge other~1 -m "merge-right" &&
87
+
88
+ git checkout -b octo-second second &&
89
+ git merge merge-left merge-right -m "octopus-second" &&
90
+
91
+ git checkout -b octo-other other &&
92
+ git merge merge-left merge-right -m "octopus-other" &&
93
+
94
+ git checkout other &&
95
+ git merge octo-other -m "pull octopus" &&
96
+
97
+ git checkout second &&
98
+ git merge octo-second -m "pull octopus" &&
99
+
100
+ # Remove these branches so they are not selected
101
+ # as bitmap tips
102
+ git branch -D merge-left &&
103
+ git branch -D merge-right &&
104
+ git branch -D octo-other &&
105
+ git branch -D octo-second &&
106
+
107
+ # add padding to make these merges less interesting
108
+ # and avoid having them selected for bitmaps
109
+ test_commit_bulk --id=file 100 &&
110
+ git checkout other &&
111
+ test_commit_bulk --id=side 100 &&
112
+ git checkout second &&
113
+
114
+ bitmaptip=$(git rev-parse second) &&
115
+ blob=$(echo tagged-blob | git hash-object -w --stdin) &&
116
+ git tag tagged-blob $blob
117
+ '
118
+}
119
+
120
+rev_list_tests_head () {
121
+ test_expect_success "counting commits via bitmap ($state, $branch)" '
122
+ git rev-list --count $branch >expect &&
123
+ git rev-list --use-bitmap-index --count $branch >actual &&
124
+ test_cmp expect actual
125
+ '
126
+
127
+ test_expect_success "counting partial commits via bitmap ($state, $branch)" '
128
+ git rev-list --count $branch~5..$branch >expect &&
129
+ git rev-list --use-bitmap-index --count $branch~5..$branch >actual &&
130
+ test_cmp expect actual
131
+ '
132
+
133
+ test_expect_success "counting commits with limit ($state, $branch)" '
134
+ git rev-list --count -n 1 $branch >expect &&
135
+ git rev-list --use-bitmap-index --count -n 1 $branch >actual &&
136
+ test_cmp expect actual
137
+ '
138
+
139
+ test_expect_success "counting non-linear history ($state, $branch)" '
140
+ git rev-list --count other...second >expect &&
141
+ git rev-list --use-bitmap-index --count other...second >actual &&
142
+ test_cmp expect actual
143
+ '
144
+
145
+ test_expect_success "counting commits with limiting ($state, $branch)" '
146
+ git rev-list --count $branch -- 1.t >expect &&
147
+ git rev-list --use-bitmap-index --count $branch -- 1.t >actual &&
148
+ test_cmp expect actual
149
+ '
150
+
151
+ test_expect_success "counting objects via bitmap ($state, $branch)" '
152
+ git rev-list --count --objects $branch >expect &&
153
+ git rev-list --use-bitmap-index --count --objects $branch >actual &&
154
+ test_cmp expect actual
155
+ '
156
+
157
+ test_expect_success "enumerate commits ($state, $branch)" '
158
+ git rev-list --use-bitmap-index $branch >actual &&
159
+ git rev-list $branch >expect &&
160
+ test_bitmap_traversal --no-confirm-bitmaps expect actual
161
+ '
162
+
163
+ test_expect_success "enumerate --objects ($state, $branch)" '
164
+ git rev-list --objects --use-bitmap-index $branch >actual &&
165
+ git rev-list --objects $branch >expect &&
166
+ test_bitmap_traversal expect actual
167
+ '
168
+
169
+ test_expect_success "bitmap --objects handles non-commit objects ($state, $branch)" '
170
+ git rev-list --objects --use-bitmap-index $branch tagged-blob >actual &&
171
+ grep $blob actual
172
+ '
173
+}
174
+
175
+rev_list_tests () {
176
+ state=$1
177
+
178
+ for branch in "second" "other"
179
+ do
180
+ rev_list_tests_head
181
+ done
182
+}
183
+
184
+basic_bitmap_tests () {
185
+ tip="$1"
186
+ test_expect_success 'rev-list --test-bitmap verifies bitmaps' "
187
+ git rev-list --test-bitmap "${tip:-HEAD}"
188
+ "
189
+
190
+ rev_list_tests 'full bitmap'
191
+
192
+ test_expect_success 'clone from bitmapped repository' '
193
+ rm -fr clone.git &&
194
+ git clone --no-local --bare . clone.git &&
195
+ git rev-parse HEAD >expect &&
196
+ git --git-dir=clone.git rev-parse HEAD >actual &&
197
+ test_cmp expect actual
198
+ '
199
+
200
+ test_expect_success 'partial clone from bitmapped repository' '
201
+ test_config uploadpack.allowfilter true &&
202
+ rm -fr partial-clone.git &&
203
+ git clone --no-local --bare --filter=blob:none . partial-clone.git &&
204
+ (
205
+ cd partial-clone.git &&
206
+ pack=$(echo objects/pack/*.pack) &&
207
+ git verify-pack -v "$pack" >have &&
208
+ awk "/blob/ { print \$1 }" <have >blobs &&
209
+ # we expect this single blob because of the direct ref
210
+ git rev-parse refs/tags/tagged-blob >expect &&
211
+ test_cmp expect blobs
212
+ )
213
+ '
214
+
215
+ test_expect_success 'setup further non-bitmapped commits' '
216
+ test_commit_bulk --id=further 10
217
+ '
218
+
219
+ rev_list_tests 'partial bitmap'
220
+
221
+ test_expect_success 'fetch (partial bitmap)' '
222
+ git --git-dir=clone.git fetch origin second:second &&
223
+ git rev-parse HEAD >expect &&
224
+ git --git-dir=clone.git rev-parse HEAD >actual &&
225
+ test_cmp expect actual
226
+ '
227
+
228
+ test_expect_success 'enumerating progress counts pack-reused objects' '
229
+ count=$(git rev-list --objects --all --count) &&
230
+ git repack -adb &&
231
+
232
+ # check first with only reused objects; confirm that our
233
+ # progress showed the right number, and also that we did
234
+ # pack-reuse as expected. Check only the final "done"
235
+ # line of the meter (there may be an arbitrary number of
236
+ # intermediate lines ending with CR).
237
+ GIT_PROGRESS_DELAY=0 \
238
+ git pack-objects --all --stdout --progress \
239
+ </dev/null >/dev/null 2>stderr &&
240
+ grep "Enumerating objects: $count, done" stderr &&
241
+ grep "pack-reused $count" stderr &&
242
+
243
+ # now the same but with one non-reused object
244
+ git commit --allow-empty -m "an extra commit object" &&
245
+ GIT_PROGRESS_DELAY=0 \
246
+ git pack-objects --all --stdout --progress \
247
+ </dev/null >/dev/null 2>stderr &&
248
+ grep "Enumerating objects: $((count+1)), done" stderr &&
249
+ grep "pack-reused $count" stderr
250
+ '
251
+}
252
+
253
+# have_delta <obj> <expected_base>
254
+#
255
+# Note that because this relies on cat-file, it might find _any_ copy of an
256
+# object in the repository. The caller is responsible for making sure
257
+# there's only one (e.g., via "repack -ad", or having just fetched a copy).
258
+have_delta () {
259
+ echo $2 >expect &&
260
+ echo $1 | git cat-file --batch-check="%(deltabase)" >actual &&
261
+ test_cmp expect actual
262
+}
t/t5310-pack-bitmaps.sh
+5
-224
@@ -25,93 +25,10 @@ has_any () {
25
grep -Ff "$1" "$2"
26
}
27
28
-# To ensure the logic for "maximal commits" is exercised, make
29
-# the repository a bit more complicated.
30
-#
31
-# other second
32
-# * *
33
-# (99 commits) (99 commits)
34
-# * *
35
-# |\ /|
36
-# | * octo-other octo-second * |
37
-# |/|\_________ ____________/|\|
38
-# | \ \/ __________/ |
39
-# | | ________/\ / |
40
-# * |/ * merge-right *
41
-# | _|__________/ \____________ |
42
-# |/ | \|
43
-# (l1) * * merge-left * (r1)
44
-# | / \________________________ |
45
-# |/ \|
46
-# (l2) * * (r2)
47
-# \___________________________ |
48
-# \|
49
-# * (base)
50
-#
51
-# We only push bits down the first-parent history, which
52
-# makes some of these commits unimportant!
53
-#
54
-# The important part for the maximal commit algorithm is how
55
-# the bitmasks are extended. Assuming starting bit positions
56
-# for second (bit 0) and other (bit 1), the bitmasks at the
57
-# end should be:
58
-#
59
-# second: 1 (maximal, selected)
60
-# other: 01 (maximal, selected)
61
-# (base): 11 (maximal)
62
-#
63
-# This complicated history was important for a previous
64
-# version of the walk that guarantees never walking a
65
-# commit multiple times. That goal might be important
66
-# again, so preserve this complicated case. For now, this
67
-# test will guarantee that the bitmaps are computed
68
-# correctly, even with the repeat calculations.
69
-
70
-test_expect_success 'setup repo with moderate-sized history' '
71
- test_commit_bulk --id=file 10 &&
72
- git branch -M second &&
73
- git checkout -b other HEAD~5 &&
74
- test_commit_bulk --id=side 10 &&
75
-
76
- # add complicated history setup, including merges and
77
- # ambiguous merge-bases
78
-
79
- git checkout -b merge-left other~2 &&
80
- git merge second~2 -m "merge-left" &&
81
-
82
- git checkout -b merge-right second~1 &&
83
- git merge other~1 -m "merge-right" &&
84
-
85
- git checkout -b octo-second second &&
86
- git merge merge-left merge-right -m "octopus-second" &&
87
-
88
- git checkout -b octo-other other &&
89
- git merge merge-left merge-right -m "octopus-other" &&
90
-
91
- git checkout other &&
92
- git merge octo-other -m "pull octopus" &&
93
-
94
- git checkout second &&
95
- git merge octo-second -m "pull octopus" &&
96
-
97
- # Remove these branches so they are not selected
98
- # as bitmap tips
99
- git branch -D merge-left &&
100
- git branch -D merge-right &&
101
- git branch -D octo-other &&
102
- git branch -D octo-second &&
103
-
104
- # add padding to make these merges less interesting
105
- # and avoid having them selected for bitmaps
106
- test_commit_bulk --id=file 100 &&
107
- git checkout other &&
108
- test_commit_bulk --id=side 100 &&
109
- git checkout second &&
110
-
111
- bitmaptip=$(git rev-parse second) &&
112
- blob=$(echo tagged-blob | git hash-object -w --stdin) &&
113
- git tag tagged-blob $blob &&
114
- git config repack.writebitmaps true
28
+setup_bitmap_history
29
+
30
+test_expect_success 'setup writing bitmaps during repack' '
31
+ git config repack.writeBitmaps true
32
'
33
34
test_expect_success 'full repack creates bitmaps' '
@@ -123,109 +40,7 @@ test_expect_success 'full repack creates bitmaps' '
40
grep "\"key\":\"num_maximal_commits\",\"value\":\"107\"" trace
41
'
42
126
-test_expect_success 'rev-list --test-bitmap verifies bitmaps' '
127
- git rev-list --test-bitmap HEAD
128
-'
129
-
130
-rev_list_tests_head () {
131
- test_expect_success "counting commits via bitmap ($state, $branch)" '
132
- git rev-list --count $branch >expect &&
133
- git rev-list --use-bitmap-index --count $branch >actual &&
134
- test_cmp expect actual
135
- '
136
-
137
- test_expect_success "counting partial commits via bitmap ($state, $branch)" '
138
- git rev-list --count $branch~5..$branch >expect &&
139
- git rev-list --use-bitmap-index --count $branch~5..$branch >actual &&
140
- test_cmp expect actual
141
- '
142
-
143
- test_expect_success "counting commits with limit ($state, $branch)" '
144
- git rev-list --count -n 1 $branch >expect &&
145
- git rev-list --use-bitmap-index --count -n 1 $branch >actual &&
146
- test_cmp expect actual
147
- '
148
-
149
- test_expect_success "counting non-linear history ($state, $branch)" '
150
- git rev-list --count other...second >expect &&
151
- git rev-list --use-bitmap-index --count other...second >actual &&
152
- test_cmp expect actual
153
- '
154
-
155
- test_expect_success "counting commits with limiting ($state, $branch)" '
156
- git rev-list --count $branch -- 1.t >expect &&
157
- git rev-list --use-bitmap-index --count $branch -- 1.t >actual &&
158
- test_cmp expect actual
159
- '
160
-
161
- test_expect_success "counting objects via bitmap ($state, $branch)" '
162
- git rev-list --count --objects $branch >expect &&
163
- git rev-list --use-bitmap-index --count --objects $branch >actual &&
164
- test_cmp expect actual
165
- '
166
-
167
- test_expect_success "enumerate commits ($state, $branch)" '
168
- git rev-list --use-bitmap-index $branch >actual &&
169
- git rev-list $branch >expect &&
170
- test_bitmap_traversal --no-confirm-bitmaps expect actual
171
- '
172
-
173
- test_expect_success "enumerate --objects ($state, $branch)" '
174
- git rev-list --objects --use-bitmap-index $branch >actual &&
175
- git rev-list --objects $branch >expect &&
176
- test_bitmap_traversal expect actual
177
- '
178
-
179
- test_expect_success "bitmap --objects handles non-commit objects ($state, $branch)" '
180
- git rev-list --objects --use-bitmap-index $branch tagged-blob >actual &&
181
- grep $blob actual
182
- '
183
-}
184
-
185
-rev_list_tests () {
186
- state=$1
187
-
188
- for branch in "second" "other"
189
- do
190
- rev_list_tests_head
191
- done
192
-}
193
-
194
-rev_list_tests 'full bitmap'
195
-
196
-test_expect_success 'clone from bitmapped repository' '
197
- git clone --no-local --bare . clone.git &&
198
- git rev-parse HEAD >expect &&
199
- git --git-dir=clone.git rev-parse HEAD >actual &&
200
- test_cmp expect actual
201
-'
202
-
203
-test_expect_success 'partial clone from bitmapped repository' '
204
- test_config uploadpack.allowfilter true &&
205
- git clone --no-local --bare --filter=blob:none . partial-clone.git &&
206
- (
207
- cd partial-clone.git &&
208
- pack=$(echo objects/pack/*.pack) &&
209
- git verify-pack -v "$pack" >have &&
210
- awk "/blob/ { print \$1 }" <have >blobs &&
211
- # we expect this single blob because of the direct ref
212
- git rev-parse refs/tags/tagged-blob >expect &&
213
- test_cmp expect blobs
214
- )
215
-'
216
-
217
-test_expect_success 'setup further non-bitmapped commits' '
218
- test_commit_bulk --id=further 10
219
-'
220
-
221
-rev_list_tests 'partial bitmap'
222
-
223
-test_expect_success 'fetch (partial bitmap)' '
224
- git --git-dir=clone.git fetch origin second:second &&
225
- git rev-parse HEAD >expect &&
226
- git --git-dir=clone.git rev-parse HEAD >actual &&
227
- test_cmp expect actual
228
-'
43
+basic_bitmap_tests
44
45
test_expect_success 'incremental repack fails when bitmaps are requested' '
46
test_commit more-1 &&
@@ -461,40 +276,6 @@ test_expect_success 'truncated bitmap fails gracefully (cache)' '
276
test_i18ngrep corrupted.bitmap.index stderr
277
'
278
464
-test_expect_success 'enumerating progress counts pack-reused objects' '
465
- count=$(git rev-list --objects --all --count) &&
466
- git repack -adb &&
467
-
468
- # check first with only reused objects; confirm that our progress
469
- # showed the right number, and also that we did pack-reuse as expected.
470
- # Check only the final "done" line of the meter (there may be an
471
- # arbitrary number of intermediate lines ending with CR).
472
- GIT_PROGRESS_DELAY=0 \
473
- git pack-objects --all --stdout --progress \
474
- </dev/null >/dev/null 2>stderr &&
475
- grep "Enumerating objects: $count, done" stderr &&
476
- grep "pack-reused $count" stderr &&
477
-
478
- # now the same but with one non-reused object
479
- git commit --allow-empty -m "an extra commit object" &&
480
- GIT_PROGRESS_DELAY=0 \
481
- git pack-objects --all --stdout --progress \
482
- </dev/null >/dev/null 2>stderr &&
483
- grep "Enumerating objects: $((count+1)), done" stderr &&
484
- grep "pack-reused $count" stderr
485
-'
486
-
487
-# have_delta <obj> <expected_base>
488
-#
489
-# Note that because this relies on cat-file, it might find _any_ copy of an
490
-# object in the repository. The caller is responsible for making sure
491
-# there's only one (e.g., via "repack -ad", or having just fetched a copy).
492
-have_delta () {
493
- echo $2 >expect &&
494
- echo $1 | git cat-file --batch-check="%(deltabase)" >actual &&
495
- test_cmp expect actual
496
-}
497
-
279
# Create a state of history with these properties:
280
#
281
# - refs that allow a client to fetch some new history, while sharing some old