pack-bitmap.c: use pseudo-merges during traversal
Now that all of the groundwork has been laid to support reading and using pseudo-merges, make use of that work in this commit by teaching the pack-bitmap machinery to use pseudo-merge(s) when available during traversal. The basic operation is as follows: - When enumerating objects on either side of a reachability query, first see if any subset of the roots satisfies some pseudo-merge bitmap. If it does, apply that pseudo-merge bitmap. - If any pseudo-merge bitmap(s) were applied in the previous step, OR them into the result[^1]. Then repeat the process over all pseudo-merge bitmaps (we'll refer to this as "cascading" pseudo-merges). Once this is done, OR in the resulting bitmap. - If there is no fill-in traversal to be done, return the bitmap for that side of the reachability query. If there is fill-in traversal, then for each commit we encounter via show_commit(), check to see if any unsatisfied pseudo-merges containing that commit as one of its parents has been made satisfied by the presence of that commit. If so, OR in the object set from that pseudo-merge bitmap, and then cascade. If not, continue traversal. A similar implementation is present in the boundary-based bitmap traversal routines. [^1]: Importantly, we cannot OR in the entire set of roots along with the objects reachable from whatever pseudo-merge bitmaps were satisfied. This may leave some dangling bits corresponding to any unsatisfied root(s) getting OR'd into the resulting bitmap, tricking other parts of the traversal into thinking we already have a reachability closure over those commit(s) when we do not. Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Taylor Blau committed
May 23, 2024 at 17:27 UTC
11d45a6e6a0639705b7ccf767e897e1870ece89b
2 files changed
+439
-1
pack-bitmap.c
+111
-1
@@ -114,6 +114,9 @@ struct bitmap_index {
114
unsigned int version;
115
};
116
117
+static int pseudo_merges_satisfied_nr;
118
+static int pseudo_merges_cascades_nr;
119
+
120
static struct ewah_bitmap *lookup_stored_bitmap(struct stored_bitmap *st)
121
{
122
struct ewah_bitmap *parent;
@@ -1006,6 +1009,22 @@ static void show_commit(struct commit *commit UNUSED,
1009
{
1010
}
1011
1012
+static unsigned apply_pseudo_merges_for_commit_1(struct bitmap_index *bitmap_git,
1013
+ struct bitmap *result,
1014
+ struct commit *commit,
1015
+ uint32_t commit_pos)
1016
+{
1017
+ int ret;
1018
+
1019
+ ret = apply_pseudo_merges_for_commit(&bitmap_git->pseudo_merges,
1020
+ result, commit, commit_pos);
1021
+
1022
+ if (ret)
1023
+ pseudo_merges_satisfied_nr += ret;
1024
+
1025
+ return ret;
1026
+}
1027
+
1028
static int add_to_include_set(struct bitmap_index *bitmap_git,
1029
struct include_data *data,
1030
struct commit *commit,
@@ -1026,6 +1045,10 @@ static int add_to_include_set(struct bitmap_index *bitmap_git,
1045
}
1046
1047
bitmap_set(data->base, bitmap_pos);
1048
+ if (apply_pseudo_merges_for_commit_1(bitmap_git, data->base, commit,
1049
+ bitmap_pos))
1050
+ return 0;
1051
+
1052
return 1;
1053
}
1054
@@ -1151,6 +1174,20 @@ static void show_boundary_object(struct object *object UNUSED,
1174
BUG("should not be called");
1175
}
1176
1177
+static unsigned cascade_pseudo_merges_1(struct bitmap_index *bitmap_git,
1178
+ struct bitmap *result,
1179
+ struct bitmap *roots)
1180
+{
1181
+ int ret = cascade_pseudo_merges(&bitmap_git->pseudo_merges,
1182
+ result, roots);
1183
+ if (ret) {
1184
+ pseudo_merges_cascades_nr++;
1185
+ pseudo_merges_satisfied_nr += ret;
1186
+ }
1187
+
1188
+ return ret;
1189
+}
1190
+
1191
static struct bitmap *find_boundary_objects(struct bitmap_index *bitmap_git,
1192
struct rev_info *revs,
1193
struct object_list *roots)
@@ -1160,6 +1197,7 @@ static struct bitmap *find_boundary_objects(struct bitmap_index *bitmap_git,
1197
unsigned int i;
1198
unsigned int tmp_blobs, tmp_trees, tmp_tags;
1199
int any_missing = 0;
1200
+ int existing_bitmaps = 0;
1201
1202
cb.bitmap_git = bitmap_git;
1203
cb.base = bitmap_new();
@@ -1167,6 +1205,25 @@ static struct bitmap *find_boundary_objects(struct bitmap_index *bitmap_git,
1205
1206
revs->ignore_missing_links = 1;
1207
1208
+ if (bitmap_git->pseudo_merges.nr) {
1209
+ struct bitmap *roots_bitmap = bitmap_new();
1210
+ struct object_list *objects = NULL;
1211
+
1212
+ for (objects = roots; objects; objects = objects->next) {
1213
+ struct object *object = objects->item;
1214
+ int pos;
1215
+
1216
+ pos = bitmap_position(bitmap_git, &object->oid);
1217
+ if (pos < 0)
1218
+ continue;
1219
+
1220
+ bitmap_set(roots_bitmap, pos);
1221
+ }
1222
+
1223
+ if (!cascade_pseudo_merges_1(bitmap_git, cb.base, roots_bitmap))
1224
+ bitmap_free(roots_bitmap);
1225
+ }
1226
+
1227
/*
1228
* OR in any existing reachability bitmaps among `roots` into
1229
* `cb.base`.
@@ -1178,8 +1235,10 @@ static struct bitmap *find_boundary_objects(struct bitmap_index *bitmap_git,
1235
continue;
1236
1237
if (add_commit_to_bitmap(bitmap_git, &cb.base,
1181
- (struct commit *)object))
1238
+ (struct commit *)object)) {
1239
+ existing_bitmaps = 1;
1240
continue;
1241
+ }
1242
1243
any_missing = 1;
1244
}
@@ -1187,6 +1246,9 @@ static struct bitmap *find_boundary_objects(struct bitmap_index *bitmap_git,
1246
if (!any_missing)
1247
goto cleanup;
1248
1249
+ if (existing_bitmaps)
1250
+ cascade_pseudo_merges_1(bitmap_git, cb.base, NULL);
1251
+
1252
tmp_blobs = revs->blob_objects;
1253
tmp_trees = revs->tree_objects;
1254
tmp_tags = revs->blob_objects;
@@ -1242,6 +1304,13 @@ cleanup:
1304
return cb.base;
1305
}
1306
1307
+static void unsatisfy_all_pseudo_merges(struct bitmap_index *bitmap_git)
1308
+{
1309
+ uint32_t i;
1310
+ for (i = 0; i < bitmap_git->pseudo_merges.nr; i++)
1311
+ bitmap_git->pseudo_merges.v[i].satisfied = 0;
1312
+}
1313
+
1314
static struct bitmap *find_objects(struct bitmap_index *bitmap_git,
1315
struct rev_info *revs,
1316
struct object_list *roots,
@@ -1249,9 +1318,32 @@ static struct bitmap *find_objects(struct bitmap_index *bitmap_git,
1318
{
1319
struct bitmap *base = NULL;
1320
int needs_walk = 0;
1321
+ unsigned existing_bitmaps = 0;
1322
1323
struct object_list *not_mapped = NULL;
1324
1325
+ unsatisfy_all_pseudo_merges(bitmap_git);
1326
+
1327
+ if (bitmap_git->pseudo_merges.nr) {
1328
+ struct bitmap *roots_bitmap = bitmap_new();
1329
+ struct object_list *objects = NULL;
1330
+
1331
+ for (objects = roots; objects; objects = objects->next) {
1332
+ struct object *object = objects->item;
1333
+ int pos;
1334
+
1335
+ pos = bitmap_position(bitmap_git, &object->oid);
1336
+ if (pos < 0)
1337
+ continue;
1338
+
1339
+ bitmap_set(roots_bitmap, pos);
1340
+ }
1341
+
1342
+ base = bitmap_new();
1343
+ if (!cascade_pseudo_merges_1(bitmap_git, base, roots_bitmap))
1344
+ bitmap_free(roots_bitmap);
1345
+ }
1346
+
1347
/*
1348
* Go through all the roots for the walk. The ones that have bitmaps
1349
* on the bitmap index will be `or`ed together to form an initial
@@ -1262,11 +1354,21 @@ static struct bitmap *find_objects(struct bitmap_index *bitmap_git,
1354
*/
1355
while (roots) {
1356
struct object *object = roots->item;
1357
+
1358
roots = roots->next;
1359
1360
+ if (base) {
1361
+ int pos = bitmap_position(bitmap_git, &object->oid);
1362
+ if (pos > 0 && bitmap_get(base, pos)) {
1363
+ object->flags |= SEEN;
1364
+ continue;
1365
+ }
1366
+ }
1367
+
1368
if (object->type == OBJ_COMMIT &&
1369
add_commit_to_bitmap(bitmap_git, &base, (struct commit *)object)) {
1370
object->flags |= SEEN;
1371
+ existing_bitmaps = 1;
1372
continue;
1373
}
1374
@@ -1282,6 +1384,9 @@ static struct bitmap *find_objects(struct bitmap_index *bitmap_git,
1384
1385
roots = not_mapped;
1386
1387
+ if (existing_bitmaps)
1388
+ cascade_pseudo_merges_1(bitmap_git, base, NULL);
1389
+
1390
/*
1391
* Let's iterate through all the roots that don't have bitmaps to
1392
* check if we can determine them to be reachable from the existing
@@ -1866,6 +1971,11 @@ struct bitmap_index *prepare_bitmap_walk(struct rev_info *revs,
1971
object_list_free(&wants);
1972
object_list_free(&haves);
1973
1974
+ trace2_data_intmax("bitmap", the_repository, "pseudo_merges_satisfied",
1975
+ pseudo_merges_satisfied_nr);
1976
+ trace2_data_intmax("bitmap", the_repository, "pseudo_merges_cascades",
1977
+ pseudo_merges_cascades_nr);
1978
+
1979
return bitmap_git;
1980
1981
cleanup:
t/t5333-pseudo-merge-bitmaps.sh
new
+328
@@ -0,0 +1,328 @@
1
+#!/bin/sh
2
+
3
+test_description='pseudo-merge bitmaps'
4
+
5
+GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=0
6
+
7
+. ./test-lib.sh
8
+
9
+test_pseudo_merges () {
10
+ test-tool bitmap dump-pseudo-merges
11
+}
12
+
13
+test_pseudo_merge_commits () {
14
+ test-tool bitmap dump-pseudo-merge-commits "$1"
15
+}
16
+
17
+test_pseudo_merges_satisfied () {
18
+ test_trace2_data bitmap pseudo_merges_satisfied "$1"
19
+}
20
+
21
+test_pseudo_merges_cascades () {
22
+ test_trace2_data bitmap pseudo_merges_cascades "$1"
23
+}
24
+
25
+tag_everything () {
26
+ git rev-list --all --no-object-names >in &&
27
+ perl -lne '
28
+ print "create refs/tags/" . $. . " " . $1 if /([0-9a-f]+)/
29
+ ' <in | git update-ref --stdin
30
+}
31
+
32
+test_expect_success 'setup' '
33
+ test_commit_bulk 512 &&
34
+ tag_everything
35
+'
36
+
37
+test_expect_success 'bitmap traversal without pseudo-merges' '
38
+ git repack -adb &&
39
+
40
+ git rev-list --count --all --objects >expect &&
41
+
42
+ : >trace2.txt &&
43
+ GIT_TRACE2_EVENT=$PWD/trace2.txt \
44
+ git rev-list --count --all --objects --use-bitmap-index >actual &&
45
+
46
+ test_pseudo_merges_satisfied 0 <trace2.txt &&
47
+ test_pseudo_merges_cascades 0 <trace2.txt &&
48
+ test_pseudo_merges >merges &&
49
+ test_must_be_empty merges &&
50
+ test_cmp expect actual
51
+'
52
+
53
+test_expect_success 'pseudo-merges accurately represent their objects' '
54
+ test_config bitmapPseudoMerge.test.pattern "refs/tags/" &&
55
+ test_config bitmapPseudoMerge.test.maxMerges 8 &&
56
+ test_config bitmapPseudoMerge.test.stableThreshold never &&
57
+
58
+ git repack -adb &&
59
+
60
+ test_pseudo_merges >merges &&
61
+ test_line_count = 8 merges &&
62
+
63
+ for i in $(test_seq 0 $(($(wc -l <merges)-1)))
64
+ do
65
+ test-tool bitmap dump-pseudo-merge-commits $i >commits &&
66
+
67
+ git rev-list --objects --no-object-names --stdin <commits >expect.raw &&
68
+ test-tool bitmap dump-pseudo-merge-objects $i >actual.raw &&
69
+
70
+ sort -u <expect.raw >expect &&
71
+ sort -u <actual.raw >actual &&
72
+
73
+ test_cmp expect actual || return 1
74
+ done
75
+'
76
+
77
+test_expect_success 'bitmap traversal with pseudo-merges' '
78
+ : >trace2.txt &&
79
+ GIT_TRACE2_EVENT=$PWD/trace2.txt \
80
+ git rev-list --count --all --objects --use-bitmap-index >actual &&
81
+ git rev-list --count --all --objects >expect &&
82
+
83
+ test_pseudo_merges_satisfied 8 <trace2.txt &&
84
+ test_pseudo_merges_cascades 1 <trace2.txt &&
85
+ test_cmp expect actual
86
+'
87
+
88
+test_expect_success 'stale bitmap traversal with pseudo-merges' '
89
+ test_commit other &&
90
+
91
+ : >trace2.txt &&
92
+ GIT_TRACE2_EVENT=$PWD/trace2.txt \
93
+ git rev-list --count --all --objects --use-bitmap-index >actual &&
94
+ git rev-list --count --all --objects >expect &&
95
+
96
+ test_pseudo_merges_satisfied 8 <trace2.txt &&
97
+ test_pseudo_merges_cascades 1 <trace2.txt &&
98
+ test_cmp expect actual
99
+'
100
+
101
+test_expect_success 'bitmapPseudoMerge.sampleRate adjusts commit selection rate' '
102
+ test_config bitmapPseudoMerge.test.pattern "refs/tags/" &&
103
+ test_config bitmapPseudoMerge.test.maxMerges 1 &&
104
+ test_config bitmapPseudoMerge.test.stableThreshold never &&
105
+
106
+ commits_nr=$(git rev-list --all --count) &&
107
+
108
+ for rate in 1.0 0.5 0.25
109
+ do
110
+ git -c bitmapPseudoMerge.test.sampleRate=$rate repack -adb &&
111
+
112
+ test_pseudo_merges >merges &&
113
+ test_line_count = 1 merges &&
114
+ test_pseudo_merge_commits 0 >commits &&
115
+
116
+ test-tool bitmap list-commits >bitmaps &&
117
+ bitmaps_nr="$(wc -l <bitmaps)" &&
118
+
119
+ perl -MPOSIX -e "print ceil(\$ARGV[0]*(\$ARGV[1]-\$ARGV[2]))" \
120
+ "$rate" "$commits_nr" "$bitmaps_nr" >expect &&
121
+
122
+ test $(cat expect) -eq $(wc -l <commits) || return 1
123
+ done
124
+'
125
+
126
+test_expect_success 'bitmapPseudoMerge.threshold excludes newer commits' '
127
+ git init pseudo-merge-threshold &&
128
+ (
129
+ cd pseudo-merge-threshold &&
130
+
131
+ new="1672549200" && # 2023-01-01
132
+ old="1641013200" && # 2022-01-01
133
+
134
+ GIT_COMMITTER_DATE="$new +0000" &&
135
+ export GIT_COMMITTER_DATE &&
136
+ test_commit_bulk --message="new" --notick 128 &&
137
+
138
+ GIT_COMMITTER_DATE="$old +0000" &&
139
+ export GIT_COMMITTER_DATE &&
140
+ test_commit_bulk --message="old" --notick 128 &&
141
+
142
+ tag_everything &&
143
+
144
+ git \
145
+ -c bitmapPseudoMerge.test.pattern="refs/tags/" \
146
+ -c bitmapPseudoMerge.test.maxMerges=1 \
147
+ -c bitmapPseudoMerge.test.threshold=$(($new - 1)) \
148
+ -c bitmapPseudoMerge.test.stableThreshold=never \
149
+ repack -adb &&
150
+
151
+ test_pseudo_merges >merges &&
152
+ test_line_count = 1 merges &&
153
+
154
+ test_pseudo_merge_commits 0 >oids &&
155
+ git cat-file --batch <oids >commits &&
156
+
157
+ test $(wc -l <oids) = $(grep -c "^committer.*$old +0000$" commits)
158
+ )
159
+'
160
+
161
+test_expect_success 'bitmapPseudoMerge.stableThreshold creates stable groups' '
162
+ (
163
+ cd pseudo-merge-threshold &&
164
+
165
+ new="1672549200" && # 2023-01-01
166
+ mid="1654059600" && # 2022-06-01
167
+ old="1641013200" && # 2022-01-01
168
+
169
+ GIT_COMMITTER_DATE="$mid +0000" &&
170
+ export GIT_COMMITTER_DATE &&
171
+ test_commit_bulk --message="mid" --notick 128 &&
172
+
173
+ git for-each-ref --format="delete %(refname)" refs/tags >in &&
174
+ git update-ref --stdin <in &&
175
+
176
+ tag_everything &&
177
+
178
+ git \
179
+ -c bitmapPseudoMerge.test.pattern="refs/tags/" \
180
+ -c bitmapPseudoMerge.test.maxMerges=1 \
181
+ -c bitmapPseudoMerge.test.threshold=$(($new - 1)) \
182
+ -c bitmapPseudoMerge.test.stableThreshold=$(($mid - 1)) \
183
+ -c bitmapPseudoMerge.test.stableSize=10 \
184
+ repack -adb &&
185
+
186
+ test_pseudo_merges >merges &&
187
+ merges_nr="$(wc -l <merges)" &&
188
+
189
+ for i in $(test_seq $(($merges_nr - 1)))
190
+ do
191
+ test_pseudo_merge_commits 0 >oids &&
192
+ git cat-file --batch <oids >commits &&
193
+
194
+ expect="$(grep -c "^committer.*$old +0000$" commits)" &&
195
+ actual="$(wc -l <oids)" &&
196
+
197
+ test $expect = $actual || return 1
198
+ done &&
199
+
200
+ test_pseudo_merge_commits $(($merges_nr - 1)) >oids &&
201
+ git cat-file --batch <oids >commits &&
202
+ test $(wc -l <oids) = $(grep -c "^committer.*$mid +0000$" commits)
203
+ )
204
+'
205
+
206
+test_expect_success 'out of order thresholds are rejected' '
207
+ test_must_fail git \
208
+ -c bitmapPseudoMerge.test.pattern="refs/*" \
209
+ -c bitmapPseudoMerge.test.threshold=1.month.ago \
210
+ -c bitmapPseudoMerge.test.stableThreshold=1.week.ago \
211
+ repack -adb 2>err &&
212
+
213
+ cat >expect <<-EOF &&
214
+ fatal: pseudo-merge group ${SQ}test${SQ} has unstable threshold before stable one
215
+ EOF
216
+
217
+ test_cmp expect err
218
+'
219
+
220
+test_expect_success 'pseudo-merge pattern with capture groups' '
221
+ git init pseudo-merge-captures &&
222
+ (
223
+ cd pseudo-merge-captures &&
224
+
225
+ test_commit_bulk 128 &&
226
+ tag_everything &&
227
+
228
+ for r in $(test_seq 8)
229
+ do
230
+ test_commit_bulk 16 &&
231
+
232
+ git rev-list HEAD~16.. >in &&
233
+
234
+ perl -lne "print \"create refs/remotes/$r/tags/\$. \$_\"" <in |
235
+ git update-ref --stdin || return 1
236
+ done &&
237
+
238
+ git \
239
+ -c bitmapPseudoMerge.tags.pattern="refs/remotes/([0-9]+)/tags/" \
240
+ -c bitmapPseudoMerge.tags.maxMerges=1 \
241
+ repack -adb &&
242
+
243
+ git for-each-ref --format="%(objectname) %(refname)" >refs &&
244
+
245
+ test_pseudo_merges >merges &&
246
+ for m in $(test_seq 0 $(($(wc -l <merges) - 1)))
247
+ do
248
+ test_pseudo_merge_commits $m >oids &&
249
+ grep -f oids refs |
250
+ perl -lne "print \$1 if /refs\/remotes\/([0-9]+)/" |
251
+ sort -u || return 1
252
+ done >remotes &&
253
+
254
+ test $(wc -l <remotes) -eq $(sort -u <remotes | wc -l)
255
+ )
256
+'
257
+
258
+test_expect_success 'pseudo-merge overlap setup' '
259
+ git init pseudo-merge-overlap &&
260
+ (
261
+ cd pseudo-merge-overlap &&
262
+
263
+ test_commit_bulk 256 &&
264
+ tag_everything &&
265
+
266
+ git \
267
+ -c bitmapPseudoMerge.all.pattern="refs/" \
268
+ -c bitmapPseudoMerge.all.maxMerges=1 \
269
+ -c bitmapPseudoMerge.all.stableThreshold=never \
270
+ -c bitmapPseudoMerge.tags.pattern="refs/tags/" \
271
+ -c bitmapPseudoMerge.tags.maxMerges=1 \
272
+ -c bitmapPseudoMerge.tags.stableThreshold=never \
273
+ repack -adb
274
+ )
275
+'
276
+
277
+test_expect_success 'pseudo-merge overlap generates overlapping groups' '
278
+ (
279
+ cd pseudo-merge-overlap &&
280
+
281
+ test_pseudo_merges >merges &&
282
+ test_line_count = 2 merges &&
283
+
284
+ test_pseudo_merge_commits 0 >commits-0.raw &&
285
+ test_pseudo_merge_commits 1 >commits-1.raw &&
286
+
287
+ sort commits-0.raw >commits-0 &&
288
+ sort commits-1.raw >commits-1 &&
289
+
290
+ comm -12 commits-0 commits-1 >overlap &&
291
+
292
+ test_line_count -gt 0 overlap
293
+ )
294
+'
295
+
296
+test_expect_success 'pseudo-merge overlap traversal' '
297
+ (
298
+ cd pseudo-merge-overlap &&
299
+
300
+ : >trace2.txt &&
301
+ GIT_TRACE2_EVENT=$PWD/trace2.txt \
302
+ git rev-list --count --all --objects --use-bitmap-index >actual &&
303
+ git rev-list --count --all --objects >expect &&
304
+
305
+ test_pseudo_merges_satisfied 2 <trace2.txt &&
306
+ test_pseudo_merges_cascades 1 <trace2.txt &&
307
+ test_cmp expect actual
308
+ )
309
+'
310
+
311
+test_expect_success 'pseudo-merge overlap stale traversal' '
312
+ (
313
+ cd pseudo-merge-overlap &&
314
+
315
+ test_commit other &&
316
+
317
+ : >trace2.txt &&
318
+ GIT_TRACE2_EVENT=$PWD/trace2.txt \
319
+ git rev-list --count --all --objects --use-bitmap-index >actual &&
320
+ git rev-list --count --all --objects >expect &&
321
+
322
+ test_pseudo_merges_satisfied 2 <trace2.txt &&
323
+ test_pseudo_merges_cascades 1 <trace2.txt &&
324
+ test_cmp expect actual
325
+ )
326
+'
327
+
328
+test_done