| 1 | #!/bin/sh |
| 2 | |
| 3 | test_description="limiting blob downloads when merging with partial clones" |
| 4 | # Uses a methodology similar to |
| 5 | # t6042: corner cases with renames but not criss-cross merges |
| 6 | # t6036: corner cases with both renames and criss-cross merges |
| 7 | # t6423: directory rename detection |
| 8 | # |
| 9 | # The setup for all of them, pictorially, is: |
| 10 | # |
| 11 | # A |
| 12 | # o |
| 13 | # / \ |
| 14 | # O o ? |
| 15 | # \ / |
| 16 | # o |
| 17 | # B |
| 18 | # |
| 19 | # To help make it easier to follow the flow of tests, they have been |
| 20 | # divided into sections and each test will start with a quick explanation |
| 21 | # of what commits O, A, and B contain. |
| 22 | # |
| 23 | # Notation: |
| 24 | # z/{b,c} means files z/b and z/c both exist |
| 25 | # x/d_1 means file x/d exists with content d1. (Purpose of the |
| 26 | # underscore notation is to differentiate different |
| 27 | # files that might be renamed into each other's paths.) |
| 28 | |
| 29 | . ./test-lib.sh |
| 30 | |
| 31 | test_setup_repo () { |
| 32 | test -d server && return |
| 33 | git init server && |
| 34 | ( |
| 35 | cd server && |
| 36 | |
| 37 | git config uploadpack.allowfilter 1 && |
| 38 | git config uploadpack.allowanysha1inwant 1 && |
| 39 | |
| 40 | mkdir -p general && |
| 41 | test_seq 2 9 >general/leap1 && |
| 42 | cp general/leap1 general/leap2 && |
| 43 | echo leap2 >>general/leap2 && |
| 44 | |
| 45 | mkdir -p basename && |
| 46 | cp general/leap1 basename/numbers && |
| 47 | cp general/leap1 basename/sequence && |
| 48 | cp general/leap1 basename/values && |
| 49 | echo numbers >>basename/numbers && |
| 50 | echo sequence >>basename/sequence && |
| 51 | echo values >>basename/values && |
| 52 | |
| 53 | mkdir -p dir/unchanged && |
| 54 | mkdir -p dir/subdir/tweaked && |
| 55 | echo a >dir/subdir/a && |
| 56 | echo b >dir/subdir/b && |
| 57 | echo c >dir/subdir/c && |
| 58 | echo d >dir/subdir/d && |
| 59 | echo e >dir/subdir/e && |
| 60 | cp general/leap1 dir/subdir/Makefile && |
| 61 | echo toplevel makefile >>dir/subdir/Makefile && |
| 62 | echo f >dir/subdir/tweaked/f && |
| 63 | echo g >dir/subdir/tweaked/g && |
| 64 | echo h >dir/subdir/tweaked/h && |
| 65 | echo subdirectory makefile >dir/subdir/tweaked/Makefile && |
| 66 | for i in $(test_seq 1 88) |
| 67 | do |
| 68 | echo content $i >dir/unchanged/file_$i |
| 69 | done && |
| 70 | git add . && |
| 71 | git commit -m "O" && |
| 72 | |
| 73 | git branch O && |
| 74 | git branch A && |
| 75 | git branch B-single && |
| 76 | git branch B-dir && |
| 77 | git branch B-many && |
| 78 | |
| 79 | git switch A && |
| 80 | |
| 81 | git rm general/leap* && |
| 82 | mkdir general/ && |
| 83 | test_seq 1 9 >general/jump1 && |
| 84 | cp general/jump1 general/jump2 && |
| 85 | echo leap2 >>general/jump2 && |
| 86 | |
| 87 | rm basename/numbers basename/sequence basename/values && |
| 88 | mkdir -p basename/subdir/ |
| 89 | cp general/jump1 basename/subdir/numbers && |
| 90 | cp general/jump1 basename/subdir/sequence && |
| 91 | cp general/jump1 basename/subdir/values && |
| 92 | echo numbers >>basename/subdir/numbers && |
| 93 | echo sequence >>basename/subdir/sequence && |
| 94 | echo values >>basename/subdir/values && |
| 95 | |
| 96 | git rm dir/subdir/tweaked/f && |
| 97 | echo more >>dir/subdir/e && |
| 98 | echo more >>dir/subdir/Makefile && |
| 99 | echo more >>dir/subdir/tweaked/Makefile && |
| 100 | mkdir dir/subdir/newsubdir && |
| 101 | echo rust code >dir/subdir/newsubdir/newfile.rs && |
| 102 | git mv dir/subdir/e dir/subdir/newsubdir/ && |
| 103 | git mv dir folder && |
| 104 | git add . && |
| 105 | git commit -m "A" && |
| 106 | |
| 107 | git switch B-single && |
| 108 | echo new first line >dir/subdir/Makefile && |
| 109 | cat general/leap1 >>dir/subdir/Makefile && |
| 110 | echo toplevel makefile >>dir/subdir/Makefile && |
| 111 | echo perl code >general/newfile.pl && |
| 112 | git add . && |
| 113 | git commit -m "B-single" && |
| 114 | |
| 115 | git switch B-dir && |
| 116 | echo java code >dir/subdir/newfile.java && |
| 117 | echo scala code >dir/subdir/newfile.scala && |
| 118 | echo groovy code >dir/subdir/newfile.groovy && |
| 119 | git add . && |
| 120 | git commit -m "B-dir" && |
| 121 | |
| 122 | git switch B-many && |
| 123 | test_seq 2 10 >general/leap1 && |
| 124 | rm general/leap2 && |
| 125 | cp general/leap1 general/leap2 && |
| 126 | echo leap2 >>general/leap2 && |
| 127 | |
| 128 | rm basename/numbers basename/sequence basename/values && |
| 129 | mkdir -p basename/subdir/ |
| 130 | cp general/leap1 basename/subdir/numbers && |
| 131 | cp general/leap1 basename/subdir/sequence && |
| 132 | cp general/leap1 basename/subdir/values && |
| 133 | echo numbers >>basename/subdir/numbers && |
| 134 | echo sequence >>basename/subdir/sequence && |
| 135 | echo values >>basename/subdir/values && |
| 136 | |
| 137 | mkdir dir/subdir/newsubdir/ && |
| 138 | echo c code >dir/subdir/newfile.c && |
| 139 | echo python code >dir/subdir/newsubdir/newfile.py && |
| 140 | git add . && |
| 141 | git commit -m "B-many" && |
| 142 | |
| 143 | git switch A |
| 144 | ) |
| 145 | } |
| 146 | |
| 147 | # Testcase: Objects downloaded for single relevant rename |
| 148 | # Commit O: |
| 149 | # general/{leap1_O, leap2_O} |
| 150 | # basename/{numbers_O, sequence_O, values_O} |
| 151 | # dir/subdir/{a,b,c,d,e_O,Makefile_TOP_O} |
| 152 | # dir/subdir/tweaked/{f,g,h,Makefile_SUB_O} |
| 153 | # dir/unchanged/<LOTS OF FILES> |
| 154 | # Commit A: |
| 155 | # (Rename leap->jump, rename basename/ -> basename/subdir/, rename dir/ |
| 156 | # -> folder/, move e into newsubdir, add newfile.rs, remove f, modify |
| 157 | # both Makefiles and jumps) |
| 158 | # general/{jump1_A, jump2_A} |
| 159 | # basename/subdir/{numbers_A, sequence_A, values_A} |
| 160 | # folder/subdir/{a,b,c,d,Makefile_TOP_A} |
| 161 | # folder/subdir/newsubdir/{e_A,newfile.rs} |
| 162 | # folder/subdir/tweaked/{g,h,Makefile_SUB_A} |
| 163 | # folder/unchanged/<LOTS OF FILES> |
| 164 | # Commit B(-single): |
| 165 | # (add newfile.pl, tweak Makefile_TOP) |
| 166 | # general/{leap1_O, leap2_O,newfile.pl} |
| 167 | # basename/{numbers_O, sequence_O, values_O} |
| 168 | # dir/{a,b,c,d,e_O,Makefile_TOP_B} |
| 169 | # dir/tweaked/{f,g,h,Makefile_SUB_O} |
| 170 | # dir/unchanged/<LOTS OF FILES> |
| 171 | # Expected: |
| 172 | # general/{jump1_A, jump2_A,newfile.pl} |
| 173 | # basename/subdir/{numbers_A, sequence_A, values_A} |
| 174 | # folder/subdir/{a,b,c,d,Makefile_TOP_Merged} |
| 175 | # folder/subdir/newsubdir/{e_A,newfile.rs} |
| 176 | # folder/subdir/tweaked/{g,h,Makefile_SUB_A} |
| 177 | # folder/unchanged/<LOTS OF FILES> |
| 178 | # |
| 179 | # Objects that need to be fetched: |
| 180 | # Rename detection: |
| 181 | # Side1 (O->A): |
| 182 | # Basename-matches rename detection only needs to fetch these objects: |
| 183 | # Makefile_TOP_O, Makefile_TOP_A |
| 184 | # (Despite many renames, all others are content irrelevant. They |
| 185 | # are also location irrelevant because newfile.rs was added on |
| 186 | # the side doing the directory rename, and newfile.pl was added to |
| 187 | # a directory that was not renamed on either side.) |
| 188 | # General rename detection only needs to fetch these objects: |
| 189 | # <None> |
| 190 | # (Even though newfile.rs, jump[12], basename/subdir/*, and e |
| 191 | # could all be used as destinations in rename detection, the |
| 192 | # basename detection for Makefile matches up all relevant |
| 193 | # sources, so these other files never end up needing to be |
| 194 | # used) |
| 195 | # Side2 (O->B): |
| 196 | # Basename-matches rename detection only needs to fetch these objects: |
| 197 | # <None> |
| 198 | # (there are no deleted files, so no possible sources) |
| 199 | # General rename detection only needs to fetch these objects: |
| 200 | # <None> |
| 201 | # (there are no deleted files, so no possible sources) |
| 202 | # Merge: |
| 203 | # 3-way content merge needs to grab these objects: |
| 204 | # Makefile_TOP_B |
| 205 | # Nothing else needs to fetch objects |
| 206 | # |
| 207 | # Summary: 2 fetches (1 for 2 objects, 1 for 1 object) |
| 208 | # |
| 209 | test_expect_success 'Objects downloaded for single relevant rename' ' |
| 210 | test_setup_repo && |
| 211 | git clone --sparse --filter=blob:none "file://$(pwd)/server" objects-single && |
| 212 | ( |
| 213 | cd objects-single && |
| 214 | |
| 215 | git rev-list --objects --all --missing=print | |
| 216 | grep "^?" | sort >missing-objects-before && |
| 217 | |
| 218 | git checkout -q origin/A && |
| 219 | |
| 220 | GIT_TRACE2_PERF="$(pwd)/trace.output" git \ |
| 221 | -c merge.directoryRenames=true merge --no-stat \ |
| 222 | --no-progress origin/B-single && |
| 223 | |
| 224 | # Check the number of objects we reported we would fetch |
| 225 | cat >expect <<-EOF && |
| 226 | fetch_count:2 |
| 227 | fetch_count:1 |
| 228 | EOF |
| 229 | grep fetch_count trace.output | cut -d "|" -f 9 | tr -d " ." >actual && |
| 230 | test_cmp expect actual && |
| 231 | |
| 232 | # Check the number of fetch commands exec-ed by filtering trace to |
| 233 | # child_start events by the top-level program (2nd field == d0) |
| 234 | grep " d0 .* child_start .*fetch.negotiationAlgorithm" trace.output >fetches && |
| 235 | test_line_count = 2 fetches && |
| 236 | |
| 237 | git rev-list --objects --all --missing=print | |
| 238 | grep "^?" | sort >missing-objects-after && |
| 239 | comm -2 -3 missing-objects-before missing-objects-after >old && |
| 240 | comm -1 -3 missing-objects-before missing-objects-after >new && |
| 241 | # No new missing objects |
| 242 | test_must_be_empty new && |
| 243 | # Fetched 2 + 1 = 3 objects |
| 244 | test_line_count = 3 old |
| 245 | ) |
| 246 | ' |
| 247 | |
| 248 | # Testcase: Objects downloaded for directory rename |
| 249 | # Commit O: |
| 250 | # general/{leap1_O, leap2_O} |
| 251 | # basename/{numbers_O, sequence_O, values_O} |
| 252 | # dir/subdir/{a,b,c,d,e_O,Makefile_TOP_O} |
| 253 | # dir/subdir/tweaked/{f,g,h,Makefile_SUB_O} |
| 254 | # dir/unchanged/<LOTS OF FILES> |
| 255 | # Commit A: |
| 256 | # (Rename leap->jump, rename basename/ -> basename/subdir/, rename dir/ -> |
| 257 | # folder/, move e into newsubdir, add newfile.rs, remove f, modify |
| 258 | # both Makefiles and jumps) |
| 259 | # general/{jump1_A, jump2_A} |
| 260 | # basename/subdir/{numbers_A, sequence_A, values_A} |
| 261 | # folder/subdir/{a,b,c,d,Makefile_TOP_A} |
| 262 | # folder/subdir/newsubdir/{e_A,newfile.rs} |
| 263 | # folder/subdir/tweaked/{g,h,Makefile_SUB_A} |
| 264 | # folder/unchanged/<LOTS OF FILES> |
| 265 | # Commit B(-dir): |
| 266 | # (add dir/subdir/newfile.{java,scala,groovy} |
| 267 | # general/{leap1_O, leap2_O} |
| 268 | # basename/{numbers_O, sequence_O, values_O} |
| 269 | # dir/subdir/{a,b,c,d,e_O,Makefile_TOP_O, |
| 270 | # newfile.java,newfile.scala,newfile.groovy} |
| 271 | # dir/subdir/tweaked/{f,g,h,Makefile_SUB_O} |
| 272 | # dir/unchanged/<LOTS OF FILES> |
| 273 | # Expected: |
| 274 | # general/{jump1_A, jump2_A} |
| 275 | # basename/subdir/{numbers_A, sequence_A, values_A} |
| 276 | # folder/subdir/{a,b,c,d,Makefile_TOP_A, |
| 277 | # newfile.java,newfile.scala,newfile.groovy} |
| 278 | # folder/subdir/newsubdir/{e_A,newfile.rs} |
| 279 | # folder/subdir/tweaked/{g,h,Makefile_SUB_A} |
| 280 | # folder/unchanged/<LOTS OF FILES> |
| 281 | # |
| 282 | # Objects that need to be fetched: |
| 283 | # Makefile_TOP_O, Makefile_TOP_A |
| 284 | # Makefile_SUB_O, Makefile_SUB_A |
| 285 | # e_O, e_A |
| 286 | # * Despite A's rename of jump->leap, those renames are irrelevant. |
| 287 | # * Despite A's rename of basename/ -> basename/subdir/, those renames are |
| 288 | # irrelevant. |
| 289 | # * Because of A's rename of dir/ -> folder/ and B-dir's addition of |
| 290 | # newfile.* into dir/subdir/, we need to determine directory renames. |
| 291 | # (Technically, there are enough exact renames to determine directory |
| 292 | # rename detection, but the current implementation always does |
| 293 | # basename searching before directory rename detection. Running it |
| 294 | # also before basename searching would mean doing directory rename |
| 295 | # detection twice, but it's a bit expensive to do that and cases like |
| 296 | # this are not all that common.) |
| 297 | # Summary: 1 fetches for 6 objects |
| 298 | # |
| 299 | test_expect_success 'Objects downloaded when a directory rename triggered' ' |
| 300 | test_setup_repo && |
| 301 | git clone --sparse --filter=blob:none "file://$(pwd)/server" objects-dir && |
| 302 | ( |
| 303 | cd objects-dir && |
| 304 | |
| 305 | git rev-list --objects --all --missing=print | |
| 306 | grep "^?" | sort >missing-objects-before && |
| 307 | |
| 308 | git checkout -q origin/A && |
| 309 | |
| 310 | GIT_TRACE2_PERF="$(pwd)/trace.output" git \ |
| 311 | -c merge.directoryRenames=true merge --no-stat \ |
| 312 | --no-progress origin/B-dir && |
| 313 | |
| 314 | # Check the number of objects we reported we would fetch |
| 315 | cat >expect <<-EOF && |
| 316 | fetch_count:6 |
| 317 | EOF |
| 318 | grep fetch_count trace.output | cut -d "|" -f 9 | tr -d " ." >actual && |
| 319 | test_cmp expect actual && |
| 320 | |
| 321 | # Check the number of fetch commands exec-ed by filtering trace to |
| 322 | # child_start events by the top-level program (2nd field == d0) |
| 323 | grep " d0 .* child_start .*fetch.negotiationAlgorithm" trace.output >fetches && |
| 324 | test_line_count = 1 fetches && |
| 325 | |
| 326 | git rev-list --objects --all --missing=print | |
| 327 | grep "^?" | sort >missing-objects-after && |
| 328 | comm -2 -3 missing-objects-before missing-objects-after >old && |
| 329 | comm -1 -3 missing-objects-before missing-objects-after >new && |
| 330 | # No new missing objects |
| 331 | test_must_be_empty new && |
| 332 | # Fetched 6 objects |
| 333 | test_line_count = 6 old |
| 334 | ) |
| 335 | ' |
| 336 | |
| 337 | # Testcase: Objects downloaded with lots of renames and modifications |
| 338 | # Commit O: |
| 339 | # general/{leap1_O, leap2_O} |
| 340 | # basename/{numbers_O, sequence_O, values_O} |
| 341 | # dir/subdir/{a,b,c,d,e_O,Makefile_TOP_O} |
| 342 | # dir/subdir/tweaked/{f,g,h,Makefile_SUB_O} |
| 343 | # dir/unchanged/<LOTS OF FILES> |
| 344 | # Commit A: |
| 345 | # (Rename leap->jump, rename basename/ -> basename/subdir/, rename dir/ |
| 346 | # -> folder/, move e into newsubdir, add newfile.rs, remove f, modify |
| 347 | # both Makefiles and jumps) |
| 348 | # general/{jump1_A, jump2_A} |
| 349 | # basename/subdir/{numbers_A, sequence_A, values_A} |
| 350 | # folder/subdir/{a,b,c,d,Makefile_TOP_A} |
| 351 | # folder/subdir/newsubdir/{e_A,newfile.rs} |
| 352 | # folder/subdir/tweaked/{g,h,Makefile_SUB_A} |
| 353 | # folder/unchanged/<LOTS OF FILES> |
| 354 | # Commit B(-minimal): |
| 355 | # (modify both leaps, rename basename/ -> basename/subdir/, add |
| 356 | # newfile.{c,py}) |
| 357 | # general/{leap1_B, leap2_B} |
| 358 | # basename/subdir/{numbers_B, sequence_B, values_B} |
| 359 | # dir/{a,b,c,d,e_O,Makefile_TOP_O,newfile.c} |
| 360 | # dir/tweaked/{f,g,h,Makefile_SUB_O,newfile.py} |
| 361 | # dir/unchanged/<LOTS OF FILES> |
| 362 | # Expected: |
| 363 | # general/{jump1_Merged, jump2_Merged} |
| 364 | # basename/subdir/{numbers_Merged, sequence_Merged, values_Merged} |
| 365 | # folder/subdir/{a,b,c,d,Makefile_TOP_A,newfile.c} |
| 366 | # folder/subdir/newsubdir/e_A |
| 367 | # folder/subdir/tweaked/{g,h,Makefile_SUB_A,newfile.py} |
| 368 | # folder/unchanged/<LOTS OF FILES> |
| 369 | # |
| 370 | # Objects that need to be fetched: |
| 371 | # Rename detection: |
| 372 | # Side1 (O->A): |
| 373 | # Basename-matches rename detection only needs to fetch these objects: |
| 374 | # numbers_O, numbers_A |
| 375 | # sequence_O, sequence_A |
| 376 | # values_O, values_A |
| 377 | # Makefile_TOP_O, Makefile_TOP_A |
| 378 | # Makefile_SUB_O, Makefile_SUB_A |
| 379 | # e_O, e_A |
| 380 | # General rename detection only needs to fetch these objects: |
| 381 | # leap1_O, leap2_O |
| 382 | # jump1_A, jump2_A, newfile.rs |
| 383 | # (only need remaining relevant sources, but any relevant sources need |
| 384 | # to be matched against all possible unpaired destinations) |
| 385 | # Side2 (O->B): |
| 386 | # Basename-matches rename detection only needs to fetch these objects: |
| 387 | # numbers_B |
| 388 | # sequence_B |
| 389 | # values_B |
| 390 | # (because numbers_O, sequence_O, and values_O already fetched above) |
| 391 | # General rename detection only needs to fetch these objects: |
| 392 | # <None> |
| 393 | # Merge: |
| 394 | # 3-way content merge needs to grab these objects: |
| 395 | # leap1_B |
| 396 | # leap2_B |
| 397 | # Nothing else needs to fetch objects |
| 398 | # |
| 399 | # Summary: 4 fetches (1 for 6 objects, 1 for 8, 1 for 3, 1 for 2) |
| 400 | # |
| 401 | test_expect_success 'Objects downloaded with lots of renames and modifications' ' |
| 402 | test_setup_repo && |
| 403 | git clone --sparse --filter=blob:none "file://$(pwd)/server" objects-many && |
| 404 | ( |
| 405 | cd objects-many && |
| 406 | |
| 407 | git rev-list --objects --all --missing=print | |
| 408 | grep "^?" | sort >missing-objects-before && |
| 409 | |
| 410 | git checkout -q origin/A && |
| 411 | |
| 412 | GIT_TRACE2_PERF="$(pwd)/trace.output" git \ |
| 413 | -c merge.directoryRenames=true merge --no-stat \ |
| 414 | --no-progress origin/B-many && |
| 415 | |
| 416 | # Check the number of objects we reported we would fetch |
| 417 | cat >expect <<-EOF && |
| 418 | fetch_count:12 |
| 419 | fetch_count:5 |
| 420 | fetch_count:3 |
| 421 | fetch_count:2 |
| 422 | EOF |
| 423 | grep fetch_count trace.output | cut -d "|" -f 9 | tr -d " ." >actual && |
| 424 | test_cmp expect actual && |
| 425 | |
| 426 | # Check the number of fetch commands exec-ed by filtering trace to |
| 427 | # child_start events by the top-level program (2nd field == d0) |
| 428 | grep " d0 .* child_start .*fetch.negotiationAlgorithm" trace.output >fetches && |
| 429 | test_line_count = 4 fetches && |
| 430 | |
| 431 | git rev-list --objects --all --missing=print | |
| 432 | grep "^?" | sort >missing-objects-after && |
| 433 | comm -2 -3 missing-objects-before missing-objects-after >old && |
| 434 | comm -1 -3 missing-objects-before missing-objects-after >new && |
| 435 | # No new missing objects |
| 436 | test_must_be_empty new && |
| 437 | # Fetched 12 + 5 + 3 + 2 = 22 objects |
| 438 | test_line_count = 22 old |
| 439 | ) |
| 440 | ' |
| 441 | |
| 442 | test_done |