| 1 | #!/bin/sh |
| 2 | |
| 3 | test_description='pack-objects multi-pack reuse' |
| 4 | |
| 5 | . ./test-lib.sh |
| 6 | . "$TEST_DIRECTORY"/lib-bitmap.sh |
| 7 | . "$TEST_DIRECTORY"/lib-pack.sh |
| 8 | |
| 9 | GIT_TEST_MULTI_PACK_INDEX=0 |
| 10 | GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL=0 |
| 11 | |
| 12 | # The --path-walk option does not consider the preferred pack |
| 13 | # at all for reusing deltas, so this variable changes the |
| 14 | # behavior of this test, if enabled. |
| 15 | GIT_TEST_PACK_PATH_WALK=0 |
| 16 | export GIT_TEST_PACK_PATH_WALK |
| 17 | |
| 18 | objdir=.git/objects |
| 19 | packdir=$objdir/pack |
| 20 | |
| 21 | test_pack_reused () { |
| 22 | test_trace2_data pack-objects pack-reused "$1" |
| 23 | } |
| 24 | |
| 25 | test_packs_reused () { |
| 26 | test_trace2_data pack-objects packs-reused "$1" |
| 27 | } |
| 28 | |
| 29 | |
| 30 | # pack_position <object> </path/to/pack.idx |
| 31 | pack_position () { |
| 32 | git show-index >objects && |
| 33 | grep "$1" objects | cut -d" " -f1 |
| 34 | } |
| 35 | |
| 36 | # B as an OFS_DELTA against A at the given one-byte distance. |
| 37 | pack_obj_b_ofs_a () { |
| 38 | pack_obj "$B" "$A" >b-ref.tmp && |
| 39 | printf "\145" && |
| 40 | printf "\\$(printf "%03o" "$1")" && |
| 41 | dd if=b-ref.tmp bs=1 skip=$((1 + $(test_oid rawsz))) 2>/dev/null |
| 42 | } |
| 43 | |
| 44 | # test_pack_objects_reused_all <pack-reused> <packs-reused> |
| 45 | test_pack_objects_reused_all () { |
| 46 | : >trace2.txt && |
| 47 | GIT_TRACE2_EVENT="$PWD/trace2.txt" \ |
| 48 | git pack-objects --stdout --revs --all --delta-base-offset \ |
| 49 | >got.pack && |
| 50 | |
| 51 | test_pack_reused "$1" <trace2.txt && |
| 52 | test_packs_reused "$2" <trace2.txt && |
| 53 | |
| 54 | git index-pack --strict -o got.idx got.pack |
| 55 | } |
| 56 | |
| 57 | # test_pack_objects_reused <pack-reused> <packs-reused> |
| 58 | test_pack_objects_reused () { |
| 59 | : >trace2.txt && |
| 60 | GIT_TRACE2_EVENT="$PWD/trace2.txt" \ |
| 61 | git pack-objects --stdout --revs >got.pack && |
| 62 | |
| 63 | test_pack_reused "$1" <trace2.txt && |
| 64 | test_packs_reused "$2" <trace2.txt && |
| 65 | |
| 66 | git index-pack --strict -o got.idx got.pack |
| 67 | } |
| 68 | |
| 69 | test_expect_success 'preferred pack is reused for single-pack reuse' ' |
| 70 | test_config pack.allowPackReuse single && |
| 71 | git config set maintenance.auto false && |
| 72 | |
| 73 | for i in A B |
| 74 | do |
| 75 | test_commit "$i" && |
| 76 | git repack -d || return 1 |
| 77 | done && |
| 78 | |
| 79 | git multi-pack-index write --bitmap && |
| 80 | |
| 81 | test_pack_objects_reused_all 3 1 |
| 82 | ' |
| 83 | |
| 84 | test_expect_success 'multi-pack reuse is disabled by default' ' |
| 85 | test_pack_objects_reused_all 3 1 |
| 86 | ' |
| 87 | |
| 88 | test_expect_success 'feature.experimental implies multi-pack reuse' ' |
| 89 | test_config feature.experimental true && |
| 90 | |
| 91 | test_pack_objects_reused_all 6 2 |
| 92 | ' |
| 93 | |
| 94 | test_expect_success 'multi-pack reuse can be disabled with feature.experimental' ' |
| 95 | test_config feature.experimental true && |
| 96 | test_config pack.allowPackReuse single && |
| 97 | |
| 98 | test_pack_objects_reused_all 3 1 |
| 99 | ' |
| 100 | |
| 101 | test_expect_success 'enable multi-pack reuse' ' |
| 102 | git config pack.allowPackReuse multi |
| 103 | ' |
| 104 | |
| 105 | test_expect_success 'reuse all objects from subset of bitmapped packs' ' |
| 106 | test_commit C && |
| 107 | git repack -d && |
| 108 | |
| 109 | git multi-pack-index write --bitmap && |
| 110 | |
| 111 | cat >in <<-EOF && |
| 112 | $(git rev-parse C) |
| 113 | ^$(git rev-parse A) |
| 114 | EOF |
| 115 | |
| 116 | test_pack_objects_reused 6 2 <in |
| 117 | ' |
| 118 | |
| 119 | test_expect_success 'reuse all objects from all packs' ' |
| 120 | test_pack_objects_reused_all 9 3 |
| 121 | ' |
| 122 | |
| 123 | test_expect_success '--no-ref-delta reuses REF_DELTA-free bitmapped packs' ' |
| 124 | # Whole-word reuse is unavailable under --no-ref-delta, so reusing |
| 125 | # every object below exercises the per-object bitmap path. |
| 126 | : >trace2.txt && |
| 127 | GIT_TRACE2_EVENT="$PWD/trace2.txt" \ |
| 128 | git pack-objects --stdout --revs --all --delta-base-offset \ |
| 129 | --no-ref-delta >got.pack && |
| 130 | |
| 131 | test_pack_reused 9 <trace2.txt && |
| 132 | test_packs_reused 3 <trace2.txt && |
| 133 | |
| 134 | git index-pack --strict -o got.idx got.pack && |
| 135 | test-tool pack-deltas --list-deltas got.idx >deltas && |
| 136 | test_grep ! " REF_DELTA " deltas |
| 137 | ' |
| 138 | |
| 139 | test_expect_success 'reuse objects from first pack with middle gap' ' |
| 140 | for i in D E F |
| 141 | do |
| 142 | test_commit "$i" || return 1 |
| 143 | done && |
| 144 | |
| 145 | # Set "pack.window" to zero to ensure that we do not create any |
| 146 | # deltas, which could alter the amount of pack reuse we perform |
| 147 | # (if, for e.g., we are not sending one or more bases). |
| 148 | D="$(git -c pack.window=0 pack-objects --all --unpacked $packdir/pack)" && |
| 149 | |
| 150 | d_pos="$(pack_position $(git rev-parse D) <$packdir/pack-$D.idx)" && |
| 151 | e_pos="$(pack_position $(git rev-parse E) <$packdir/pack-$D.idx)" && |
| 152 | f_pos="$(pack_position $(git rev-parse F) <$packdir/pack-$D.idx)" && |
| 153 | |
| 154 | # commits F, E, and D, should appear in that order at the |
| 155 | # beginning of the pack |
| 156 | test $f_pos -lt $e_pos && |
| 157 | test $e_pos -lt $d_pos && |
| 158 | |
| 159 | # Ensure that the pack we are constructing sorts ahead of any |
| 160 | # other packs in lexical/bitmap order by choosing it as the |
| 161 | # preferred pack. |
| 162 | git multi-pack-index write --bitmap --preferred-pack="pack-$D.idx" && |
| 163 | |
| 164 | cat >in <<-EOF && |
| 165 | $(git rev-parse E) |
| 166 | ^$(git rev-parse D) |
| 167 | EOF |
| 168 | |
| 169 | test_pack_objects_reused 3 1 <in |
| 170 | ' |
| 171 | |
| 172 | test_expect_success 'reuse objects from middle pack with middle gap' ' |
| 173 | rm -fr $packdir/multi-pack-index* && |
| 174 | |
| 175 | # Ensure that the pack we are constructing sort into any |
| 176 | # position *but* the first one, by choosing a different pack as |
| 177 | # the preferred one. |
| 178 | git multi-pack-index write --bitmap --preferred-pack="pack-$A.idx" && |
| 179 | |
| 180 | cat >in <<-EOF && |
| 181 | $(git rev-parse E) |
| 182 | ^$(git rev-parse D) |
| 183 | EOF |
| 184 | |
| 185 | test_pack_objects_reused 3 1 <in |
| 186 | ' |
| 187 | |
| 188 | test_expect_success 'omit delta with uninteresting base (same pack)' ' |
| 189 | git repack -adk && |
| 190 | |
| 191 | test_seq 32 >f && |
| 192 | git add f && |
| 193 | test_tick && |
| 194 | git commit -m "delta" && |
| 195 | delta="$(git rev-parse HEAD)" && |
| 196 | |
| 197 | test_seq 64 >f && |
| 198 | test_tick && |
| 199 | git commit -a -m "base" && |
| 200 | base="$(git rev-parse HEAD)" && |
| 201 | |
| 202 | test_commit other && |
| 203 | |
| 204 | git repack -d && |
| 205 | |
| 206 | have_delta "$(git rev-parse $delta:f)" "$(git rev-parse $base:f)" && |
| 207 | |
| 208 | git multi-pack-index write --bitmap && |
| 209 | |
| 210 | cat >in <<-EOF && |
| 211 | $(git rev-parse other) |
| 212 | ^$base |
| 213 | EOF |
| 214 | |
| 215 | # We can only reuse the 3 objects corresponding to "other" from |
| 216 | # the latest pack. |
| 217 | # |
| 218 | # This is because even though we want "delta", we do not want |
| 219 | # "base", meaning that we have to inflate the delta/base-pair |
| 220 | # corresponding to the blob in commit "delta", which bypasses |
| 221 | # the pack-reuse mechanism. |
| 222 | # |
| 223 | # The remaining objects from the other pack are similarly not |
| 224 | # reused because their objects are on the uninteresting side of |
| 225 | # the query. |
| 226 | test_pack_objects_reused 3 1 <in |
| 227 | ' |
| 228 | |
| 229 | test_expect_success 'omit delta from uninteresting base (cross pack)' ' |
| 230 | cat >in <<-EOF && |
| 231 | $(git rev-parse $base) |
| 232 | ^$(git rev-parse $delta) |
| 233 | EOF |
| 234 | |
| 235 | P="$(git pack-objects --revs $packdir/pack <in)" && |
| 236 | |
| 237 | git multi-pack-index write --bitmap --preferred-pack="pack-$P.idx" && |
| 238 | |
| 239 | packs_nr="$(find $packdir -type f -name "pack-*.pack" | wc -l)" && |
| 240 | objects_nr="$(git rev-list --count --all --objects)" && |
| 241 | |
| 242 | test_pack_objects_reused_all $(($objects_nr - 1)) $packs_nr |
| 243 | ' |
| 244 | |
| 245 | test_expect_success 'non-omitted delta in MIDX preferred pack' ' |
| 246 | test_config pack.allowPackReuse single && |
| 247 | |
| 248 | cat >p1.objects <<-EOF && |
| 249 | $(git rev-parse $base) |
| 250 | ^$(git rev-parse $delta^) |
| 251 | EOF |
| 252 | cat >p2.objects <<-EOF && |
| 253 | $(git rev-parse F) |
| 254 | EOF |
| 255 | |
| 256 | p1="$(git pack-objects --revs $packdir/pack <p1.objects)" && |
| 257 | p2="$(git pack-objects --revs $packdir/pack <p2.objects)" && |
| 258 | |
| 259 | cat >in <<-EOF && |
| 260 | pack-$p1.idx |
| 261 | pack-$p2.idx |
| 262 | EOF |
| 263 | git multi-pack-index write --bitmap --stdin-packs \ |
| 264 | --preferred-pack=pack-$p1.pack <in && |
| 265 | |
| 266 | git show-index <$packdir/pack-$p1.idx >expect && |
| 267 | |
| 268 | test_pack_objects_reused_all $(wc -l <expect) 1 |
| 269 | ' |
| 270 | |
| 271 | test_expect_success 'duplicate objects' ' |
| 272 | git init duplicate-objects && |
| 273 | ( |
| 274 | cd duplicate-objects && |
| 275 | |
| 276 | git config pack.allowPackReuse multi && |
| 277 | |
| 278 | test_commit base && |
| 279 | |
| 280 | git repack -a && |
| 281 | |
| 282 | git rev-parse HEAD^{tree} >in && |
| 283 | p="$(git pack-objects $packdir/pack <in)" && |
| 284 | |
| 285 | git multi-pack-index write --bitmap --preferred-pack=pack-$p.idx && |
| 286 | |
| 287 | objects_nr="$(git rev-list --count --all --objects)" && |
| 288 | packs_nr="$(find $packdir -type f -name "pack-*.pack" | wc -l)" && |
| 289 | |
| 290 | test_pack_objects_reused_all $objects_nr $packs_nr |
| 291 | ) |
| 292 | ' |
| 293 | |
| 294 | test_expect_success 'duplicate objects with verbatim reuse' ' |
| 295 | git init duplicate-objects-verbatim && |
| 296 | ( |
| 297 | cd duplicate-objects-verbatim && |
| 298 | |
| 299 | git config pack.allowPackReuse multi && |
| 300 | |
| 301 | test_commit_bulk 64 && |
| 302 | |
| 303 | # take the first object from the main pack... |
| 304 | git show-index <$(ls $packdir/pack-*.idx) >obj.raw && |
| 305 | sort -nk1 <obj.raw | head -n1 | cut -d" " -f2 >in && |
| 306 | |
| 307 | # ...and create a separate pack containing just that object |
| 308 | p="$(git pack-objects $packdir/pack <in)" && |
| 309 | |
| 310 | git multi-pack-index write --bitmap --preferred-pack=pack-$p.idx && |
| 311 | |
| 312 | test_pack_objects_reused_all 192 2 |
| 313 | ) |
| 314 | ' |
| 315 | |
| 316 | test_expect_success 'reuse with intra-pack duplicate objects' ' |
| 317 | git init intra-pack-duplicate-objects && |
| 318 | ( |
| 319 | cd intra-pack-duplicate-objects && |
| 320 | |
| 321 | # Make enough objects to exercise whole-word reuse. |
| 322 | test_commit_bulk 20 && |
| 323 | test_commit --printf A a "\7\0" && |
| 324 | test_commit --printf B b "\7\76" && |
| 325 | |
| 326 | objects_nr=$(git rev-list --count --objects --all) && |
| 327 | git rev-list --objects --all | |
| 328 | cut -d" " -f1 >objects && |
| 329 | A=$(test_oid packlib_7_0) && |
| 330 | B=$(test_oid packlib_7_76) && |
| 331 | grep -v -e "^$A$" -e "^$B$" objects >rest && |
| 332 | pack_obj "$A" >a-full && |
| 333 | pack_obj "$B" >b-full && |
| 334 | while read oid |
| 335 | do |
| 336 | pack_obj "$oid" || exit 1 |
| 337 | done <rest >rest.entries && |
| 338 | { |
| 339 | # Arrange the pack as A, B, A, C..., so that physical |
| 340 | # positions diverge from MIDX pseudo-pack order. |
| 341 | pack_header $((objects_nr + 1)) && |
| 342 | cat a-full b-full a-full rest.entries |
| 343 | } >duplicate.pack && |
| 344 | pack_trailer duplicate.pack && |
| 345 | clear_packs && |
| 346 | git index-pack --stdin <duplicate.pack && |
| 347 | |
| 348 | git multi-pack-index write --bitmap && |
| 349 | git config pack.allowPackReuse single && |
| 350 | test_pack_objects_reused_all "$objects_nr" 1 && |
| 351 | rm -f got.idx && |
| 352 | test_env GIT_TEST_MIDX_READ_BTMP=false \ |
| 353 | test_pack_objects_reused_all 0 0 |
| 354 | ) |
| 355 | ' |
| 356 | |
| 357 | test_expect_success 'omit delta whose duplicate base is not selected' ' |
| 358 | ( |
| 359 | cd intra-pack-duplicate-objects && |
| 360 | |
| 361 | A=$(test_oid packlib_7_0) && |
| 362 | B=$(test_oid packlib_7_76) && |
| 363 | objects_nr=$(wc -l <objects) && |
| 364 | |
| 365 | a_size=$(wc -c <a-full) && |
| 366 | test "$((2 * a_size))" -lt 128 && |
| 367 | |
| 368 | # The .idx order of duplicate OIDs is unspecified. Try a delta |
| 369 | # against each copy and keep the pack whose base was omitted. |
| 370 | for distance in "$((2 * a_size))" "$a_size" |
| 371 | do |
| 372 | base_offset=$((12 + 2 * a_size - distance)) && |
| 373 | pack_obj_b_ofs_a "$distance" >b-delta && |
| 374 | { |
| 375 | pack_header "$((objects_nr + 1))" && |
| 376 | cat a-full a-full b-delta rest.entries |
| 377 | } >candidate.pack && |
| 378 | pack_trailer candidate.pack && |
| 379 | clear_packs && |
| 380 | git index-pack --stdin <candidate.pack && |
| 381 | git multi-pack-index write --bitmap || return 1 |
| 382 | |
| 383 | selected_offset=$( |
| 384 | test-tool read-midx --show-objects "$objdir" | |
| 385 | awk -v oid="$A" "\$1 == oid { print \$2 }" |
| 386 | ) || return 1 |
| 387 | test -n "$selected_offset" || return 1 |
| 388 | test "$selected_offset" = "$base_offset" || break |
| 389 | done && |
| 390 | test "$selected_offset" != "$base_offset" && |
| 391 | |
| 392 | test_pack_objects_reused_all "$((objects_nr - 1))" 1 |
| 393 | ) |
| 394 | ' |
| 395 | |
| 396 | test_done |