t6421: add tests checking for excessive object downloads during merge
Signed-off-by: Elijah Newren <newren@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Elijah Newren committed
Jun 22, 2021 at 08:04 UTC
c75c42395267f3b484f47404ae746d323586a929
1 file changed
+440
t/t6421-merge-partial-clone.sh
new
+440
@@ -0,0 +1,440 @@
1
+#!/bin/sh
2
+
3
+test_description="limiting blob downloads when merging with partial clones"
4
+# Uses a methodology similar to
5
+# t6042: corner cases with renames but not criss-cross merges
6
+# t6036: corner cases with both renames and criss-cross merges
7
+# t6423: directory rename detection
8
+#
9
+# The setup for all of them, pictorially, is:
10
+#
11
+# A
12
+# o
13
+# / \
14
+# O o ?
15
+# \ /
16
+# o
17
+# B
18
+#
19
+# To help make it easier to follow the flow of tests, they have been
20
+# divided into sections and each test will start with a quick explanation
21
+# of what commits O, A, and B contain.
22
+#
23
+# Notation:
24
+# z/{b,c} means files z/b and z/c both exist
25
+# x/d_1 means file x/d exists with content d1. (Purpose of the
26
+# underscore notation is to differentiate different
27
+# files that might be renamed into each other's paths.)
28
+
29
+. ./test-lib.sh
30
+. "$TEST_DIRECTORY"/lib-merge.sh
31
+
32
+test_setup_repo () {
33
+ test -d server && return
34
+ test_create_repo server &&
35
+ (
36
+ cd server &&
37
+
38
+ git config uploadpack.allowfilter 1 &&
39
+ git config uploadpack.allowanysha1inwant 1 &&
40
+
41
+ mkdir -p general &&
42
+ test_seq 2 9 >general/leap1 &&
43
+ cp general/leap1 general/leap2 &&
44
+ echo leap2 >>general/leap2 &&
45
+
46
+ mkdir -p basename &&
47
+ cp general/leap1 basename/numbers &&
48
+ cp general/leap1 basename/sequence &&
49
+ cp general/leap1 basename/values &&
50
+ echo numbers >>basename/numbers &&
51
+ echo sequence >>basename/sequence &&
52
+ echo values >>basename/values &&
53
+
54
+ mkdir -p dir/unchanged &&
55
+ mkdir -p dir/subdir/tweaked &&
56
+ echo a >dir/subdir/a &&
57
+ echo b >dir/subdir/b &&
58
+ echo c >dir/subdir/c &&
59
+ echo d >dir/subdir/d &&
60
+ echo e >dir/subdir/e &&
61
+ cp general/leap1 dir/subdir/Makefile &&
62
+ echo toplevel makefile >>dir/subdir/Makefile &&
63
+ echo f >dir/subdir/tweaked/f &&
64
+ echo g >dir/subdir/tweaked/g &&
65
+ echo h >dir/subdir/tweaked/h &&
66
+ echo subdirectory makefile >dir/subdir/tweaked/Makefile &&
67
+ for i in $(test_seq 1 88)
68
+ do
69
+ echo content $i >dir/unchanged/file_$i
70
+ done &&
71
+ git add . &&
72
+ git commit -m "O" &&
73
+
74
+ git branch O &&
75
+ git branch A &&
76
+ git branch B-single &&
77
+ git branch B-dir &&
78
+ git branch B-many &&
79
+
80
+ git switch A &&
81
+
82
+ git rm general/leap* &&
83
+ mkdir general/ &&
84
+ test_seq 1 9 >general/jump1 &&
85
+ cp general/jump1 general/jump2 &&
86
+ echo leap2 >>general/jump2 &&
87
+
88
+ rm basename/numbers basename/sequence basename/values &&
89
+ mkdir -p basename/subdir/
90
+ cp general/jump1 basename/subdir/numbers &&
91
+ cp general/jump1 basename/subdir/sequence &&
92
+ cp general/jump1 basename/subdir/values &&
93
+ echo numbers >>basename/subdir/numbers &&
94
+ echo sequence >>basename/subdir/sequence &&
95
+ echo values >>basename/subdir/values &&
96
+
97
+ git rm dir/subdir/tweaked/f &&
98
+ echo more >>dir/subdir/e &&
99
+ echo more >>dir/subdir/Makefile &&
100
+ echo more >>dir/subdir/tweaked/Makefile &&
101
+ mkdir dir/subdir/newsubdir &&
102
+ echo rust code >dir/subdir/newsubdir/newfile.rs &&
103
+ git mv dir/subdir/e dir/subdir/newsubdir/ &&
104
+ git mv dir folder &&
105
+ git add . &&
106
+ git commit -m "A" &&
107
+
108
+ git switch B-single &&
109
+ echo new first line >dir/subdir/Makefile &&
110
+ cat general/leap1 >>dir/subdir/Makefile &&
111
+ echo toplevel makefile >>dir/subdir/Makefile &&
112
+ echo perl code >general/newfile.pl &&
113
+ git add . &&
114
+ git commit -m "B-single" &&
115
+
116
+ git switch B-dir &&
117
+ echo java code >dir/subdir/newfile.java &&
118
+ echo scala code >dir/subdir/newfile.scala &&
119
+ echo groovy code >dir/subdir/newfile.groovy &&
120
+ git add . &&
121
+ git commit -m "B-dir" &&
122
+
123
+ git switch B-many &&
124
+ test_seq 2 10 >general/leap1 &&
125
+ rm general/leap2 &&
126
+ cp general/leap1 general/leap2 &&
127
+ echo leap2 >>general/leap2 &&
128
+
129
+ rm basename/numbers basename/sequence basename/values &&
130
+ mkdir -p basename/subdir/
131
+ cp general/leap1 basename/subdir/numbers &&
132
+ cp general/leap1 basename/subdir/sequence &&
133
+ cp general/leap1 basename/subdir/values &&
134
+ echo numbers >>basename/subdir/numbers &&
135
+ echo sequence >>basename/subdir/sequence &&
136
+ echo values >>basename/subdir/values &&
137
+
138
+ mkdir dir/subdir/newsubdir/ &&
139
+ echo c code >dir/subdir/newfile.c &&
140
+ echo python code >dir/subdir/newsubdir/newfile.py &&
141
+ git add . &&
142
+ git commit -m "B-many" &&
143
+
144
+ git switch A
145
+ )
146
+}
147
+
148
+# Testcase: Objects downloaded for single relevant rename
149
+# Commit O:
150
+# general/{leap1_O, leap2_O}
151
+# basename/{numbers_O, sequence_O, values_O}
152
+# dir/subdir/{a,b,c,d,e_O,Makefile_TOP_O}
153
+# dir/subdir/tweaked/{f,g,h,Makefile_SUB_O}
154
+# dir/unchanged/<LOTS OF FILES>
155
+# Commit A:
156
+# (Rename leap->jump, rename basename/ -> basename/subdir/, rename dir/
157
+# -> folder/, move e into newsubdir, add newfile.rs, remove f, modify
158
+# both both Makefiles and jumps)
159
+# general/{jump1_A, jump2_A}
160
+# basename/subdir/{numbers_A, sequence_A, values_A}
161
+# folder/subdir/{a,b,c,d,Makefile_TOP_A}
162
+# folder/subdir/newsubdir/{e_A,newfile.rs}
163
+# folder/subdir/tweaked/{g,h,Makefile_SUB_A}
164
+# folder/unchanged/<LOTS OF FILES>
165
+# Commit B(-single):
166
+# (add newfile.pl, tweak Makefile_TOP)
167
+# general/{leap1_O, leap2_O,newfile.pl}
168
+# basename/{numbers_O, sequence_O, values_O}
169
+# dir/{a,b,c,d,e_O,Makefile_TOP_B}
170
+# dir/tweaked/{f,g,h,Makefile_SUB_O}
171
+# dir/unchanged/<LOTS OF FILES>
172
+# Expected:
173
+# general/{jump1_A, jump2_A,newfile.pl}
174
+# basename/subdir/{numbers_A, sequence_A, values_A}
175
+# folder/subdir/{a,b,c,d,Makefile_TOP_Merged}
176
+# folder/subdir/newsubdir/{e_A,newfile.rs}
177
+# folder/subdir/tweaked/{g,h,Makefile_SUB_A}
178
+# folder/unchanged/<LOTS OF FILES>
179
+#
180
+# Objects that need to be fetched:
181
+# Rename detection:
182
+# Side1 (O->A):
183
+# Basename-matches rename detection only needs to fetch these objects:
184
+# Makefile_TOP_O, Makefile_TOP_A
185
+# (Despite many renames, all others are content irrelevant. They
186
+# are also location irrelevant because newfile.rs was added on
187
+# the side doing the directory rename, and newfile.pl was added to
188
+# a directory that was not renamed on either side.)
189
+# General rename detection only needs to fetch these objects:
190
+# <None>
191
+# (Even though newfile.rs, jump[12], basename/subdir/*, and e
192
+# could all be used as destinations in rename detection, the
193
+# basename detection for Makefile matches up all relevant
194
+# sources, so these other files never end up needing to be
195
+# used)
196
+# Side2 (O->B):
197
+# Basename-matches rename detection only needs to fetch these objects:
198
+# <None>
199
+# (there are no deleted files, so no possible sources)
200
+# General rename detection only needs to fetch these objects:
201
+# <None>
202
+# (there are no deleted files, so no possible sources)
203
+# Merge:
204
+# 3-way content merge needs to grab these objects:
205
+# Makefile_TOP_B
206
+# Nothing else needs to fetch objects
207
+#
208
+# Summary: 2 fetches (1 for 2 objects, 1 for 1 object)
209
+#
210
+test_expect_merge_algorithm failure failure 'Objects downloaded for single relevant rename' '
211
+ test_setup_repo &&
212
+ git clone --sparse --filter=blob:none "file://$(pwd)/server" objects-single &&
213
+ (
214
+ cd objects-single &&
215
+
216
+ git rev-list --objects --all --missing=print |
217
+ grep "^?" | sort >missing-objects-before &&
218
+
219
+ git checkout -q origin/A &&
220
+
221
+ GIT_TRACE2_PERF="$(pwd)/trace.output" git \
222
+ -c merge.directoryRenames=true merge --no-stat \
223
+ --no-progress origin/B-single &&
224
+
225
+ # Check the number of objects we reported we would fetch
226
+ cat >expect <<-EOF &&
227
+ fetch_count:2
228
+ fetch_count:1
229
+ EOF
230
+ grep fetch_count trace.output | cut -d "|" -f 9 | tr -d " ." >actual &&
231
+ test_cmp expect actual &&
232
+
233
+ # Check the number of fetch commands exec-ed
234
+ grep d0.*fetch.negotiationAlgorithm trace.output >fetches &&
235
+ test_line_count = 2 fetches &&
236
+
237
+ git rev-list --objects --all --missing=print |
238
+ grep "^?" | sort >missing-objects-after &&
239
+ comm -2 -3 missing-objects-before missing-objects-after >old &&
240
+ comm -1 -3 missing-objects-before missing-objects-after >new &&
241
+ # No new missing objects
242
+ test_must_be_empty new &&
243
+ # Fetched 2 + 1 = 3 objects
244
+ test_line_count = 3 old
245
+ )
246
+'
247
+
248
+# Testcase: Objects downloaded for directory rename
249
+# Commit O:
250
+# general/{leap1_O, leap2_O}
251
+# basename/{numbers_O, sequence_O, values_O}
252
+# dir/subdir/{a,b,c,d,e_O,Makefile_TOP_O}
253
+# dir/subdir/tweaked/{f,g,h,Makefile_SUB_O}
254
+# dir/unchanged/<LOTS OF FILES>
255
+# Commit A:
256
+# (Rename leap->jump, rename basename/ -> basename/subdir/, rename dir/ ->
257
+# folder/, move e into newsubdir, add newfile.rs, remove f, modify
258
+# both Makefiles and jumps)
259
+# general/{jump1_A, jump2_A}
260
+# basename/subdir/{numbers_A, sequence_A, values_A}
261
+# folder/subdir/{a,b,c,d,Makefile_TOP_A}
262
+# folder/subdir/newsubdir/{e_A,newfile.rs}
263
+# folder/subdir/tweaked/{g,h,Makefile_SUB_A}
264
+# folder/unchanged/<LOTS OF FILES>
265
+# Commit B(-dir):
266
+# (add dir/subdir/newfile.{java,scala,groovy}
267
+# general/{leap1_O, leap2_O}
268
+# basename/{numbers_O, sequence_O, values_O}
269
+# dir/subdir/{a,b,c,d,e_O,Makefile_TOP_O,
270
+# newfile.java,newfile.scala,newfile.groovy}
271
+# dir/subdir/tweaked/{f,g,h,Makefile_SUB_O}
272
+# dir/unchanged/<LOTS OF FILES>
273
+# Expected:
274
+# general/{jump1_A, jump2_A}
275
+# basename/subdir/{numbers_A, sequence_A, values_A}
276
+# folder/subdir/{a,b,c,d,Makefile_TOP_A,
277
+# newfile.java,newfile.scala,newfile.groovy}
278
+# folder/subdir/newsubdir/{e_A,newfile.rs}
279
+# folder/subdir/tweaked/{g,h,Makefile_SUB_A}
280
+# folder/unchanged/<LOTS OF FILES>
281
+#
282
+# Objects that need to be fetched:
283
+# Makefile_TOP_O, Makefile_TOP_A
284
+# Makefile_SUB_O, Makefile_SUB_A
285
+# e_O, e_A
286
+# * Despite A's rename of jump->leap, those renames are irrelevant.
287
+# * Despite A's rename of basename/ -> basename/subdir/, those renames are
288
+# irrelevant.
289
+# * Because of A's rename of dir/ -> folder/ and B-dir's addition of
290
+# newfile.* into dir/subdir/, we need to determine directory renames.
291
+# (Technically, there are enough exact renames to determine directory
292
+# rename detection, but the current implementation always does
293
+# basename searching before directory rename detection. Running it
294
+# also before basename searching would mean doing directory rename
295
+# detection twice, but it's a bit expensive to do that and cases like
296
+# this are not all that common.)
297
+# Summary: 1 fetches for 6 objects
298
+#
299
+test_expect_merge_algorithm failure failure 'Objects downloaded when a directory rename triggered' '
300
+ test_setup_repo &&
301
+ git clone --sparse --filter=blob:none "file://$(pwd)/server" objects-dir &&
302
+ (
303
+ cd objects-dir &&
304
+
305
+ git rev-list --objects --all --missing=print |
306
+ grep "^?" | sort >missing-objects-before &&
307
+
308
+ git checkout -q origin/A &&
309
+
310
+ GIT_TRACE2_PERF="$(pwd)/trace.output" git \
311
+ -c merge.directoryRenames=true merge --no-stat \
312
+ --no-progress origin/B-dir &&
313
+
314
+ # Check the number of objects we reported we would fetch
315
+ cat >expect <<-EOF &&
316
+ fetch_count:6
317
+ EOF
318
+ grep fetch_count trace.output | cut -d "|" -f 9 | tr -d " ." >actual &&
319
+ test_cmp expect actual &&
320
+
321
+ # Check the number of fetch commands exec-ed
322
+ grep d0.*fetch.negotiationAlgorithm trace.output >fetches &&
323
+ test_line_count = 1 fetches &&
324
+
325
+ git rev-list --objects --all --missing=print |
326
+ grep "^?" | sort >missing-objects-after &&
327
+ comm -2 -3 missing-objects-before missing-objects-after >old &&
328
+ comm -1 -3 missing-objects-before missing-objects-after >new &&
329
+ # No new missing objects
330
+ test_must_be_empty new &&
331
+ # Fetched 6 objects
332
+ test_line_count = 6 old
333
+ )
334
+'
335
+
336
+# Testcase: Objects downloaded with lots of renames and modifications
337
+# Commit O:
338
+# general/{leap1_O, leap2_O}
339
+# basename/{numbers_O, sequence_O, values_O}
340
+# dir/subdir/{a,b,c,d,e_O,Makefile_TOP_O}
341
+# dir/subdir/tweaked/{f,g,h,Makefile_SUB_O}
342
+# dir/unchanged/<LOTS OF FILES>
343
+# Commit A:
344
+# (Rename leap->jump, rename basename/ -> basename/subdir/, rename dir/
345
+# -> folder/, move e into newsubdir, add newfile.rs, remove f, modify
346
+# both both Makefiles and jumps)
347
+# general/{jump1_A, jump2_A}
348
+# basename/subdir/{numbers_A, sequence_A, values_A}
349
+# folder/subdir/{a,b,c,d,Makefile_TOP_A}
350
+# folder/subdir/newsubdir/{e_A,newfile.rs}
351
+# folder/subdir/tweaked/{g,h,Makefile_SUB_A}
352
+# folder/unchanged/<LOTS OF FILES>
353
+# Commit B(-minimal):
354
+# (modify both leaps, rename basename/ -> basename/subdir/, add
355
+# newfile.{c,py})
356
+# general/{leap1_B, leap2_B}
357
+# basename/subdir/{numbers_B, sequence_B, values_B}
358
+# dir/{a,b,c,d,e_O,Makefile_TOP_O,newfile.c}
359
+# dir/tweaked/{f,g,h,Makefile_SUB_O,newfile.py}
360
+# dir/unchanged/<LOTS OF FILES>
361
+# Expected:
362
+# general/{jump1_Merged, jump2_Merged}
363
+# basename/subdir/{numbers_Merged, sequence_Merged, values_Merged}
364
+# folder/subdir/{a,b,c,d,Makefile_TOP_A,newfile.c}
365
+# folder/subdir/newsubdir/e_A
366
+# folder/subdir/tweaked/{g,h,Makefile_SUB_A,newfile.py}
367
+# folder/unchanged/<LOTS OF FILES>
368
+#
369
+# Objects that need to be fetched:
370
+# Rename detection:
371
+# Side1 (O->A):
372
+# Basename-matches rename detection only needs to fetch these objects:
373
+# numbers_O, numbers_A
374
+# sequence_O, sequence_A
375
+# values_O, values_A
376
+# Makefile_TOP_O, Makefile_TOP_A
377
+# Makefile_SUB_O, Makefile_SUB_A
378
+# e_O, e_A
379
+# General rename detection only needs to fetch these objects:
380
+# leap1_O, leap2_O
381
+# jump1_A, jump2_A, newfile.rs
382
+# (only need remaining relevant sources, but any relevant sources need
383
+# to be matched against all possible unpaired destinations)
384
+# Side2 (O->B):
385
+# Basename-matches rename detection only needs to fetch these objects:
386
+# numbers_B
387
+# sequence_B
388
+# values_B
389
+# (because numbers_O, sequence_O, and values_O already fetched above)
390
+# General rename detection only needs to fetch these objects:
391
+# <None>
392
+# Merge:
393
+# 3-way content merge needs to grab these objects:
394
+# leap1_B
395
+# leap2_B
396
+# Nothing else needs to fetch objects
397
+#
398
+# Summary: 4 fetches (1 for 6 objects, 1 for 8, 1 for 3, 1 for 2)
399
+#
400
+test_expect_merge_algorithm failure failure 'Objects downloaded with lots of renames and modifications' '
401
+ test_setup_repo &&
402
+ git clone --sparse --filter=blob:none "file://$(pwd)/server" objects-many &&
403
+ (
404
+ cd objects-many &&
405
+
406
+ git rev-list --objects --all --missing=print |
407
+ grep "^?" | sort >missing-objects-before &&
408
+
409
+ git checkout -q origin/A &&
410
+
411
+ GIT_TRACE2_PERF="$(pwd)/trace.output" git \
412
+ -c merge.directoryRenames=true merge --no-stat \
413
+ --no-progress origin/B-many &&
414
+
415
+ # Check the number of objects we reported we would fetch
416
+ cat >expect <<-EOF &&
417
+ fetch_count:12
418
+ fetch_count:5
419
+ fetch_count:3
420
+ fetch_count:2
421
+ EOF
422
+ grep fetch_count trace.output | cut -d "|" -f 9 | tr -d " ." >actual &&
423
+ test_cmp expect actual &&
424
+
425
+ # Check the number of fetch commands exec-ed
426
+ grep d0.*fetch.negotiationAlgorithm trace.output >fetches &&
427
+ test_line_count = 4 fetches &&
428
+
429
+ git rev-list --objects --all --missing=print |
430
+ grep "^?" | sort >missing-objects-after &&
431
+ comm -2 -3 missing-objects-before missing-objects-after >old &&
432
+ comm -1 -3 missing-objects-before missing-objects-after >new &&
433
+ # No new missing objects
434
+ test_must_be_empty new &&
435
+ # Fetched 12 + 5 + 3 + 2 = 22 objects
436
+ test_line_count = 22 old
437
+ )
438
+'
439
+
440
+test_done