| 1 | #!/bin/sh |
| 2 | |
| 3 | test_description='handling of promisor remote advertisement' |
| 4 | |
| 5 | . ./test-lib.sh |
| 6 | |
| 7 | if ! test_have_prereq PERL_TEST_HELPERS |
| 8 | then |
| 9 | skip_all='skipping promisor remote capabilities tests; Perl not available' |
| 10 | test_done |
| 11 | fi |
| 12 | |
| 13 | GIT_TEST_MULTI_PACK_INDEX=0 |
| 14 | GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL=0 |
| 15 | |
| 16 | # Setup the repository with three commits, this way HEAD is always |
| 17 | # available and we can hide commit 1 or 2. |
| 18 | test_expect_success 'setup: create "template" repository' ' |
| 19 | git init template && |
| 20 | test_commit -C template 1 && |
| 21 | test_commit -C template 2 && |
| 22 | test_commit -C template 3 && |
| 23 | test-tool genrandom foo 10k >template/foo && |
| 24 | git -C template add foo && |
| 25 | git -C template commit -m foo |
| 26 | ' |
| 27 | |
| 28 | # A bare repo will act as a server repo with unpacked objects. |
| 29 | test_expect_success 'setup: create bare "server" repository' ' |
| 30 | git clone --bare --no-local template server && |
| 31 | mv server/objects/pack/pack-* . && |
| 32 | packfile=$(ls pack-*.pack) && |
| 33 | git -C server unpack-objects --strict <"$packfile" |
| 34 | ' |
| 35 | |
| 36 | check_missing_objects () { |
| 37 | git -C "$1" rev-list --objects --all --missing=print > all.txt && |
| 38 | perl -ne 'print if s/^[?]//' all.txt >missing.txt && |
| 39 | test_line_count = "$2" missing.txt && |
| 40 | if test "$2" -lt 2 |
| 41 | then |
| 42 | test "$3" = "$(cat missing.txt)" |
| 43 | else |
| 44 | test -f "$3" && |
| 45 | sort <"$3" >expected_sorted && |
| 46 | sort <missing.txt >actual_sorted && |
| 47 | test_cmp expected_sorted actual_sorted |
| 48 | fi |
| 49 | } |
| 50 | |
| 51 | initialize_server () { |
| 52 | count="$1" |
| 53 | missing_oids="$2" |
| 54 | |
| 55 | # Repack everything first |
| 56 | git -C server -c repack.writebitmaps=false repack -a -d && |
| 57 | |
| 58 | # Remove promisor file in case they exist, useful when reinitializing |
| 59 | rm -rf server/objects/pack/*.promisor && |
| 60 | |
| 61 | # Repack without the largest object and create a promisor pack on server |
| 62 | git -C server -c repack.writebitmaps=false repack -a -d \ |
| 63 | --filter=blob:limit=5k --filter-to="$(pwd)/pack" && |
| 64 | promisor_file=$(ls server/objects/pack/*.pack | sed "s/\.pack/.promisor/") && |
| 65 | >"$promisor_file" && |
| 66 | |
| 67 | # Check objects missing on the server |
| 68 | check_missing_objects server "$count" "$missing_oids" |
| 69 | } |
| 70 | |
| 71 | copy_to_lop () { |
| 72 | oid_path="$(test_oid_to_path $1)" && |
| 73 | path="server/objects/$oid_path" && |
| 74 | path2="lop/objects/$oid_path" && |
| 75 | mkdir -p $(dirname "$path2") && |
| 76 | cp "$path" "$path2" |
| 77 | } |
| 78 | |
| 79 | # On Windows, `pwd` returns a path like 'D:/foo/bar'. Prepend '/' to turn |
| 80 | # it into '/D:/foo/bar', which is what git expects in file:// URLs on Windows. |
| 81 | # On Unix, the path already starts with '/', so this is a no-op. |
| 82 | pwd_path=$(pwd) |
| 83 | case "$pwd_path" in |
| 84 | [a-zA-Z]:*) pwd_path="/$pwd_path" ;; |
| 85 | esac |
| 86 | |
| 87 | # Allowed characters: alphanumeric, standard path/URI (_ . ~ / : -), |
| 88 | # and those percent-encoded below (% space = , ;) |
| 89 | rest=$(printf "%s" "$pwd_path" | tr -d 'a-zA-Z0-9_.~/:% =,;-') |
| 90 | if test -n "$rest" |
| 91 | then |
| 92 | skip_all="PWD contains unsupported special characters" |
| 93 | test_done |
| 94 | fi |
| 95 | |
| 96 | TRASH_DIRECTORY_URL="file://$pwd_path" |
| 97 | |
| 98 | encoded_path=$(printf "%s" "$pwd_path" | |
| 99 | sed -e 's/%/%25/g' -e 's/ /%20/g' -e 's/=/%3D/g' \ |
| 100 | -e 's/;/%3B/g' -e 's/,/%2C/g') |
| 101 | |
| 102 | ENCODED_TRASH_DIRECTORY_URL="file://$encoded_path" |
| 103 | |
| 104 | test_expect_success "setup for testing promisor remote advertisement" ' |
| 105 | # Create another bare repo called "lop" (for Large Object Promisor) |
| 106 | git init --bare lop && |
| 107 | |
| 108 | # Copy the largest object from server to lop |
| 109 | obj="HEAD:foo" && |
| 110 | oid="$(git -C server rev-parse $obj)" && |
| 111 | copy_to_lop "$oid" && |
| 112 | |
| 113 | initialize_server 1 "$oid" && |
| 114 | |
| 115 | # Configure lop as promisor remote for server |
| 116 | git -C server remote add lop "$TRASH_DIRECTORY_URL/lop" && |
| 117 | git -C server config remote.lop.promisor true && |
| 118 | |
| 119 | git -C lop config uploadpack.allowFilter true && |
| 120 | git -C lop config uploadpack.allowAnySHA1InWant true && |
| 121 | git -C server config uploadpack.allowFilter true && |
| 122 | git -C server config uploadpack.allowAnySHA1InWant true |
| 123 | ' |
| 124 | |
| 125 | test_expect_success "clone with promisor.advertise set to 'true'" ' |
| 126 | git -C server config promisor.advertise true && |
| 127 | test_when_finished "rm -rf client" && |
| 128 | |
| 129 | # Clone from server to create a client |
| 130 | GIT_NO_LAZY_FETCH=0 git clone -c remote.lop.promisor=true \ |
| 131 | -c remote.lop.fetch="+refs/heads/*:refs/remotes/lop/*" \ |
| 132 | -c remote.lop.url="$TRASH_DIRECTORY_URL/lop" \ |
| 133 | -c promisor.acceptfromserver=All \ |
| 134 | --no-local --filter="blob:limit=5k" server client && |
| 135 | |
| 136 | # Check that the largest object is still missing on the server |
| 137 | check_missing_objects server 1 "$oid" |
| 138 | ' |
| 139 | |
| 140 | test_expect_success "clone with promisor.advertise set to 'false'" ' |
| 141 | git -C server config promisor.advertise false && |
| 142 | test_when_finished "rm -rf client" && |
| 143 | |
| 144 | # Clone from server to create a client |
| 145 | GIT_NO_LAZY_FETCH=0 git clone -c remote.lop.promisor=true \ |
| 146 | -c remote.lop.fetch="+refs/heads/*:refs/remotes/lop/*" \ |
| 147 | -c remote.lop.url="$TRASH_DIRECTORY_URL/lop" \ |
| 148 | -c promisor.acceptfromserver=All \ |
| 149 | --no-local --filter="blob:limit=5k" server client && |
| 150 | |
| 151 | # Check that the largest object is not missing on the server |
| 152 | check_missing_objects server 0 "" && |
| 153 | |
| 154 | # Reinitialize server so that the largest object is missing again |
| 155 | initialize_server 1 "$oid" |
| 156 | ' |
| 157 | |
| 158 | test_expect_success "clone with promisor.acceptfromserver set to 'None'" ' |
| 159 | git -C server config promisor.advertise true && |
| 160 | test_when_finished "rm -rf client" && |
| 161 | |
| 162 | # Clone from server to create a client |
| 163 | GIT_NO_LAZY_FETCH=0 git clone -c remote.lop.promisor=true \ |
| 164 | -c remote.lop.fetch="+refs/heads/*:refs/remotes/lop/*" \ |
| 165 | -c remote.lop.url="$TRASH_DIRECTORY_URL/lop" \ |
| 166 | -c promisor.acceptfromserver=None \ |
| 167 | --no-local --filter="blob:limit=5k" server client && |
| 168 | |
| 169 | # Check that the largest object is not missing on the server |
| 170 | check_missing_objects server 0 "" && |
| 171 | |
| 172 | # Reinitialize server so that the largest object is missing again |
| 173 | initialize_server 1 "$oid" |
| 174 | ' |
| 175 | |
| 176 | test_expect_success "init + fetch with promisor.advertise set to 'true'" ' |
| 177 | git -C server config promisor.advertise true && |
| 178 | test_when_finished "rm -rf client" && |
| 179 | |
| 180 | git init client && |
| 181 | git -C client config remote.lop.promisor true && |
| 182 | git -C client config remote.lop.fetch "+refs/heads/*:refs/remotes/lop/*" && |
| 183 | git -C client config remote.lop.url "$TRASH_DIRECTORY_URL/lop" && |
| 184 | git -C client config remote.server.url "$TRASH_DIRECTORY_URL/server" && |
| 185 | git -C client config remote.server.fetch "+refs/heads/*:refs/remotes/server/*" && |
| 186 | git -C client config promisor.acceptfromserver All && |
| 187 | GIT_NO_LAZY_FETCH=0 git -C client fetch --filter="blob:limit=5k" server && |
| 188 | |
| 189 | # Check that the largest object is still missing on the server |
| 190 | check_missing_objects server 1 "$oid" |
| 191 | ' |
| 192 | |
| 193 | test_expect_success "clone with two promisors but only one advertised" ' |
| 194 | git -C server config promisor.advertise true && |
| 195 | test_when_finished "rm -rf client unused_lop" && |
| 196 | |
| 197 | # Create a promisor that will be configured but not be used |
| 198 | git init --bare unused_lop && |
| 199 | |
| 200 | # Clone from server to create a client |
| 201 | GIT_TRACE="$(pwd)/trace" GIT_NO_LAZY_FETCH=0 git clone \ |
| 202 | -c remote.unused_lop.promisor=true \ |
| 203 | -c remote.unused_lop.fetch="+refs/heads/*:refs/remotes/unused_lop/*" \ |
| 204 | -c remote.unused_lop.url="$TRASH_DIRECTORY_URL/unused_lop" \ |
| 205 | -c remote.lop.promisor=true \ |
| 206 | -c remote.lop.fetch="+refs/heads/*:refs/remotes/lop/*" \ |
| 207 | -c remote.lop.url="$TRASH_DIRECTORY_URL/lop" \ |
| 208 | -c promisor.acceptfromserver=All \ |
| 209 | --no-local --filter="blob:limit=5k" server client && |
| 210 | |
| 211 | # Check that "unused_lop" appears before "lop" in the config |
| 212 | printf "remote.%s.promisor true\n" "unused_lop" "lop" "origin" >expect && |
| 213 | git -C client config get --all --show-names --regexp "^remote\..*\.promisor$" >actual && |
| 214 | test_cmp expect actual && |
| 215 | |
| 216 | # Check that "lop" was tried |
| 217 | test_grep " fetch lop " trace && |
| 218 | # Check that "unused_lop" was not contacted |
| 219 | # This means "lop", the accepted promisor, was tried first |
| 220 | test_grep ! " fetch unused_lop " trace && |
| 221 | |
| 222 | # Check that the largest object is still missing on the server |
| 223 | check_missing_objects server 1 "$oid" |
| 224 | ' |
| 225 | |
| 226 | test_expect_success "init + fetch two promisors but only one advertised" ' |
| 227 | git -C server config promisor.advertise true && |
| 228 | test_when_finished "rm -rf client unused_lop" && |
| 229 | |
| 230 | # Create a promisor that will be configured but not be used |
| 231 | git init --bare unused_lop && |
| 232 | |
| 233 | git init client && |
| 234 | git -C client config remote.unused_lop.promisor true && |
| 235 | git -C client config remote.unused_lop.fetch "+refs/heads/*:refs/remotes/unused_lop/*" && |
| 236 | git -C client config remote.unused_lop.url "$TRASH_DIRECTORY_URL/unused_lop" && |
| 237 | git -C client config remote.lop.promisor true && |
| 238 | git -C client config remote.lop.fetch "+refs/heads/*:refs/remotes/lop/*" && |
| 239 | git -C client config remote.lop.url "$TRASH_DIRECTORY_URL/lop" && |
| 240 | git -C client config remote.server.url "$TRASH_DIRECTORY_URL/server" && |
| 241 | git -C client config remote.server.fetch "+refs/heads/*:refs/remotes/server/*" && |
| 242 | git -C client config promisor.acceptfromserver All && |
| 243 | |
| 244 | # Check that "unused_lop" appears before "lop" in the config |
| 245 | printf "remote.%s.promisor true\n" "unused_lop" "lop" >expect && |
| 246 | git -C client config get --all --show-names --regexp "^remote\..*\.promisor$" >actual && |
| 247 | test_cmp expect actual && |
| 248 | |
| 249 | GIT_TRACE="$(pwd)/trace" GIT_NO_LAZY_FETCH=0 git -C client fetch --filter="blob:limit=5k" server && |
| 250 | |
| 251 | # Check that "lop" was tried |
| 252 | test_grep " fetch lop " trace && |
| 253 | # Check that "unused_lop" was not contacted |
| 254 | # This means "lop", the accepted promisor, was tried first |
| 255 | test_grep ! " fetch unused_lop " trace && |
| 256 | |
| 257 | # Check that the largest object is still missing on the server |
| 258 | check_missing_objects server 1 "$oid" |
| 259 | ' |
| 260 | |
| 261 | test_expect_success "clone with promisor.acceptfromserver set to 'KnownName'" ' |
| 262 | git -C server config promisor.advertise true && |
| 263 | test_when_finished "rm -rf client" && |
| 264 | |
| 265 | # Clone from server to create a client |
| 266 | GIT_NO_LAZY_FETCH=0 git clone -c remote.lop.promisor=true \ |
| 267 | -c remote.lop.fetch="+refs/heads/*:refs/remotes/lop/*" \ |
| 268 | -c remote.lop.url="$TRASH_DIRECTORY_URL/lop" \ |
| 269 | -c promisor.acceptfromserver=KnownName \ |
| 270 | --no-local --filter="blob:limit=5k" server client && |
| 271 | |
| 272 | # Check that the largest object is still missing on the server |
| 273 | check_missing_objects server 1 "$oid" |
| 274 | ' |
| 275 | |
| 276 | test_expect_success "clone with 'KnownName' and different remote names" ' |
| 277 | git -C server config promisor.advertise true && |
| 278 | test_when_finished "rm -rf client" && |
| 279 | |
| 280 | # Clone from server to create a client |
| 281 | GIT_NO_LAZY_FETCH=0 git clone -c remote.serverTwo.promisor=true \ |
| 282 | -c remote.serverTwo.fetch="+refs/heads/*:refs/remotes/lop/*" \ |
| 283 | -c remote.serverTwo.url="$TRASH_DIRECTORY_URL/lop" \ |
| 284 | -c promisor.acceptfromserver=KnownName \ |
| 285 | --no-local --filter="blob:limit=5k" server client && |
| 286 | |
| 287 | # Check that the largest object is not missing on the server |
| 288 | check_missing_objects server 0 "" && |
| 289 | |
| 290 | # Reinitialize server so that the largest object is missing again |
| 291 | initialize_server 1 "$oid" |
| 292 | ' |
| 293 | |
| 294 | test_expect_success "clone with 'KnownName' and missing URL in the config" ' |
| 295 | git -C server config promisor.advertise true && |
| 296 | test_when_finished "rm -rf client" && |
| 297 | |
| 298 | # Clone from server to create a client |
| 299 | # Lazy fetching by the client from the LOP will fail because of the |
| 300 | # missing URL in the client config, so the server will have to lazy |
| 301 | # fetch from the LOP. |
| 302 | GIT_NO_LAZY_FETCH=0 git clone -c remote.lop.promisor=true \ |
| 303 | -c promisor.acceptfromserver=KnownName \ |
| 304 | --no-local --filter="blob:limit=5k" server client && |
| 305 | |
| 306 | # Check that the largest object is not missing on the server |
| 307 | check_missing_objects server 0 "" && |
| 308 | |
| 309 | # Reinitialize server so that the largest object is missing again |
| 310 | initialize_server 1 "$oid" |
| 311 | ' |
| 312 | |
| 313 | test_expect_success "clone with promisor.acceptfromserver set to 'KnownUrl'" ' |
| 314 | git -C server config promisor.advertise true && |
| 315 | test_when_finished "rm -rf client" && |
| 316 | |
| 317 | # Clone from server to create a client |
| 318 | GIT_NO_LAZY_FETCH=0 git clone -c remote.lop.promisor=true \ |
| 319 | -c remote.lop.fetch="+refs/heads/*:refs/remotes/lop/*" \ |
| 320 | -c remote.lop.url="$TRASH_DIRECTORY_URL/lop" \ |
| 321 | -c promisor.acceptfromserver=KnownUrl \ |
| 322 | --no-local --filter="blob:limit=5k" server client && |
| 323 | |
| 324 | # Check that the largest object is still missing on the server |
| 325 | check_missing_objects server 1 "$oid" |
| 326 | ' |
| 327 | |
| 328 | test_expect_success "clone with 'KnownUrl' and different remote urls" ' |
| 329 | ln -s lop serverTwo && |
| 330 | |
| 331 | git -C server config promisor.advertise true && |
| 332 | test_when_finished "rm -rf client" && |
| 333 | |
| 334 | # Clone from server to create a client |
| 335 | GIT_NO_LAZY_FETCH=0 git clone -c remote.lop.promisor=true \ |
| 336 | -c remote.lop.fetch="+refs/heads/*:refs/remotes/lop/*" \ |
| 337 | -c remote.lop.url="$TRASH_DIRECTORY_URL/serverTwo" \ |
| 338 | -c promisor.acceptfromserver=KnownUrl \ |
| 339 | --no-local --filter="blob:limit=5k" server client && |
| 340 | |
| 341 | # Check that the largest object is not missing on the server |
| 342 | check_missing_objects server 0 "" && |
| 343 | |
| 344 | # Reinitialize server so that the largest object is missing again |
| 345 | initialize_server 1 "$oid" |
| 346 | ' |
| 347 | |
| 348 | test_expect_success "clone with 'KnownUrl' and url not configured on the server" ' |
| 349 | git -C server config promisor.advertise true && |
| 350 | test_when_finished "rm -rf client" && |
| 351 | |
| 352 | test_when_finished "git -C server config set remote.lop.url \"$TRASH_DIRECTORY_URL/lop\"" && |
| 353 | git -C server config unset remote.lop.url && |
| 354 | |
| 355 | # Clone from server to create a client |
| 356 | # It should fail because the client will reject the LOP as URLs are |
| 357 | # different, and the server cannot lazy fetch as the LOP URL is |
| 358 | # missing, so the remote name will be used instead which will fail. |
| 359 | test_must_fail env GIT_NO_LAZY_FETCH=0 git clone -c remote.lop.promisor=true \ |
| 360 | -c remote.lop.fetch="+refs/heads/*:refs/remotes/lop/*" \ |
| 361 | -c remote.lop.url="$TRASH_DIRECTORY_URL/lop" \ |
| 362 | -c promisor.acceptfromserver=KnownUrl \ |
| 363 | --no-local --filter="blob:limit=5k" server client && |
| 364 | |
| 365 | # Check that the largest object is still missing on the server |
| 366 | check_missing_objects server 1 "$oid" |
| 367 | ' |
| 368 | |
| 369 | test_expect_success "clone with 'KnownUrl' and empty url, so not advertised" ' |
| 370 | git -C server config promisor.advertise true && |
| 371 | test_when_finished "rm -rf client" && |
| 372 | |
| 373 | test_when_finished "git -C server config set remote.lop.url \"$TRASH_DIRECTORY_URL/lop\"" && |
| 374 | git -C server config set remote.lop.url "" && |
| 375 | |
| 376 | # Clone from server to create a client |
| 377 | # It should fail because the client will reject the LOP as an empty URL is |
| 378 | # not advertised, and the server cannot lazy fetch as the LOP URL is empty, |
| 379 | # so the remote name will be used instead which will fail. |
| 380 | test_must_fail env GIT_NO_LAZY_FETCH=0 git clone -c remote.lop.promisor=true \ |
| 381 | -c remote.lop.fetch="+refs/heads/*:refs/remotes/lop/*" \ |
| 382 | -c remote.lop.url="$TRASH_DIRECTORY_URL/lop" \ |
| 383 | -c promisor.acceptfromserver=KnownUrl \ |
| 384 | --no-local --filter="blob:limit=5k" server client && |
| 385 | |
| 386 | # Check that the largest object is still missing on the server |
| 387 | check_missing_objects server 1 "$oid" |
| 388 | ' |
| 389 | |
| 390 | test_expect_success "clone with 'None' but URL allowlisted" ' |
| 391 | git -C server config promisor.advertise true && |
| 392 | test_when_finished "rm -rf client" && |
| 393 | |
| 394 | GIT_NO_LAZY_FETCH=0 git clone -c remote.lop.promisor=true \ |
| 395 | -c remote.lop.fetch="+refs/heads/*:refs/remotes/lop/*" \ |
| 396 | -c remote.lop.url="$TRASH_DIRECTORY_URL/lop" \ |
| 397 | -c promisor.acceptfromserver=None \ |
| 398 | -c promisor.acceptFromServerUrl="$ENCODED_TRASH_DIRECTORY_URL/*" \ |
| 399 | --no-local --filter="blob:limit=5k" server client && |
| 400 | |
| 401 | # Check that the largest object is still missing on the server |
| 402 | check_missing_objects server 1 "$oid" |
| 403 | ' |
| 404 | |
| 405 | test_expect_success "clone with 'None' but URL not in allowlist" ' |
| 406 | git -C server config promisor.advertise true && |
| 407 | test_when_finished "rm -rf client" && |
| 408 | |
| 409 | GIT_NO_LAZY_FETCH=0 git clone -c remote.lop.promisor=true \ |
| 410 | -c remote.lop.fetch="+refs/heads/*:refs/remotes/lop/*" \ |
| 411 | -c remote.lop.url="$TRASH_DIRECTORY_URL/lop" \ |
| 412 | -c promisor.acceptfromserver=None \ |
| 413 | -c promisor.acceptFromServerUrl="https://example.com/*" \ |
| 414 | --no-local --filter="blob:limit=5k" server client && |
| 415 | |
| 416 | # Check that the largest object is not missing on the server |
| 417 | check_missing_objects server 0 "" && |
| 418 | |
| 419 | # Reinitialize server so that the largest object is missing again |
| 420 | initialize_server 1 "$oid" |
| 421 | ' |
| 422 | |
| 423 | test_expect_success "clone with 'None' but URL allowlisted in one pattern out of two" ' |
| 424 | git -C server config promisor.advertise true && |
| 425 | test_when_finished "rm -rf client" && |
| 426 | |
| 427 | GIT_NO_LAZY_FETCH=0 git clone -c remote.lop.promisor=true \ |
| 428 | -c remote.lop.fetch="+refs/heads/*:refs/remotes/lop/*" \ |
| 429 | -c remote.lop.url="$TRASH_DIRECTORY_URL/lop" \ |
| 430 | -c promisor.acceptfromserver=None \ |
| 431 | -c promisor.acceptFromServerUrl="https://example.com/*" \ |
| 432 | -c promisor.acceptFromServerUrl="$ENCODED_TRASH_DIRECTORY_URL/*" \ |
| 433 | --no-local --filter="blob:limit=5k" server client && |
| 434 | |
| 435 | # Check that the largest object is still missing on the server |
| 436 | check_missing_objects server 1 "$oid" |
| 437 | ' |
| 438 | |
| 439 | test_expect_success "clone with 'None', URL allowlisted, but client has different URL" ' |
| 440 | git -C server config promisor.advertise true && |
| 441 | test_when_finished "rm -rf client" && |
| 442 | |
| 443 | # The client configures "lop" with a different URL (serverTwo) than |
| 444 | # what the server advertises (lop). Even though the advertised URL |
| 445 | # matches the allowlist, the remote is rejected because the |
| 446 | # configured URL does not match the advertised one. |
| 447 | GIT_NO_LAZY_FETCH=0 git clone -c remote.lop.promisor=true \ |
| 448 | -c remote.lop.fetch="+refs/heads/*:refs/remotes/lop/*" \ |
| 449 | -c remote.lop.url="$TRASH_DIRECTORY_URL/serverTwo" \ |
| 450 | -c promisor.acceptfromserver=None \ |
| 451 | -c promisor.acceptFromServerUrl="$ENCODED_TRASH_DIRECTORY_URL/*" \ |
| 452 | --no-local --filter="blob:limit=5k" server client && |
| 453 | |
| 454 | # Check that the largest object is not missing on the server |
| 455 | check_missing_objects server 0 "" && |
| 456 | |
| 457 | # Reinitialize server so that the largest object is missing again |
| 458 | initialize_server 1 "$oid" |
| 459 | ' |
| 460 | |
| 461 | test_expect_success "clone with URL allowlisted and no remote already configured" ' |
| 462 | git -C server config promisor.advertise true && |
| 463 | test_when_finished "rm -rf client" && |
| 464 | test_when_finished "rm -f full_names" && |
| 465 | |
| 466 | GIT_NO_LAZY_FETCH=0 git clone \ |
| 467 | -c promisor.acceptfromserver=None \ |
| 468 | -c promisor.acceptFromServerUrl="$ENCODED_TRASH_DIRECTORY_URL/*" \ |
| 469 | --no-local --filter="blob:limit=5k" server client && |
| 470 | |
| 471 | # Check that exactly one remote has been auto-created, identified |
| 472 | # by "remote.<name>.advertisedAs" == "lop". |
| 473 | git -C client config get --all --show-names --regexp \ |
| 474 | "remote\..*\.advertisedas" >full_names && |
| 475 | test_line_count = 1 full_names && |
| 476 | REMOTE_NAME=$(sed "s/^remote\.\(.*\)\.advertisedas .*$/\1/" full_names) && |
| 477 | |
| 478 | # Check ".url" and ".promisor" values |
| 479 | printf "%s\n" "$TRASH_DIRECTORY_URL/lop" "true" >expect && |
| 480 | git -C client config "remote.$REMOTE_NAME.url" >actual && |
| 481 | git -C client config "remote.$REMOTE_NAME.promisor" >>actual && |
| 482 | test_cmp expect actual && |
| 483 | |
| 484 | # Check that the largest object is still missing on the server |
| 485 | check_missing_objects server 1 "$oid" |
| 486 | ' |
| 487 | |
| 488 | test_expect_success "clone with named URL allowlisted and no pre-configured remote" ' |
| 489 | git -C server config promisor.advertise true && |
| 490 | test_when_finished "rm -rf client" && |
| 491 | |
| 492 | GIT_NO_LAZY_FETCH=0 git clone \ |
| 493 | -c promisor.acceptfromserver=None \ |
| 494 | -c promisor.acceptFromServerUrl="cdn=$ENCODED_TRASH_DIRECTORY_URL/*" \ |
| 495 | --no-local --filter="blob:limit=5k" server client && |
| 496 | |
| 497 | # Check that a remote has been auto-created with the right "cdn" name and fields. |
| 498 | printf "%s\n" "$TRASH_DIRECTORY_URL/lop" "true" "lop" >expect && |
| 499 | git -C client config "remote.cdn.url" >actual && |
| 500 | git -C client config "remote.cdn.promisor" >>actual && |
| 501 | git -C client config "remote.cdn.advertisedAs" >>actual && |
| 502 | test_cmp expect actual && |
| 503 | |
| 504 | # Check that the largest object is still missing on the server |
| 505 | check_missing_objects server 1 "$oid" |
| 506 | ' |
| 507 | |
| 508 | test_expect_success "clone with URL allowlisted but colliding name" ' |
| 509 | git -C server config promisor.advertise true && |
| 510 | test_when_finished "rm -rf client" && |
| 511 | |
| 512 | GIT_NO_LAZY_FETCH=0 git clone -c remote.cdn.promisor=true \ |
| 513 | -c remote.cdn.fetch="+refs/heads/*:refs/remotes/lop/*" \ |
| 514 | -c remote.cdn.url="https://example.com/cdn" \ |
| 515 | -c promisor.acceptfromserver=None \ |
| 516 | -c promisor.acceptFromServerUrl="cdn=$ENCODED_TRASH_DIRECTORY_URL/*" \ |
| 517 | --no-local --filter="blob:limit=5k" server client && |
| 518 | |
| 519 | # Check that a remote has been auto-created with the right "cdn-1" name and fields. |
| 520 | printf "%s\n" "$TRASH_DIRECTORY_URL/lop" "true" "lop" >expect && |
| 521 | git -C client config "remote.cdn-1.url" >actual && |
| 522 | git -C client config "remote.cdn-1.promisor" >>actual && |
| 523 | git -C client config "remote.cdn-1.advertisedAs" >>actual && |
| 524 | test_cmp expect actual && |
| 525 | |
| 526 | # Check that the original "cdn" remote was not overwritten. |
| 527 | printf "%s\n" "https://example.com/cdn" "true" >expect && |
| 528 | git -C client config "remote.cdn.url" >actual && |
| 529 | git -C client config "remote.cdn.promisor" >>actual && |
| 530 | test_cmp expect actual && |
| 531 | |
| 532 | # Check that the largest object is still missing on the server |
| 533 | check_missing_objects server 1 "$oid" |
| 534 | ' |
| 535 | |
| 536 | test_expect_success "clone with URL allowlisted and reusable remote" ' |
| 537 | git -C server config promisor.advertise true && |
| 538 | test_when_finished "rm -rf client" && |
| 539 | |
| 540 | GIT_NO_LAZY_FETCH=0 git clone \ |
| 541 | -c remote.cdn.fetch="+refs/heads/*:refs/remotes/lop/*" \ |
| 542 | -c remote.cdn.url="$TRASH_DIRECTORY_URL/lop" \ |
| 543 | -c promisor.acceptfromserver=None \ |
| 544 | -c promisor.acceptFromServerUrl="cdn=$ENCODED_TRASH_DIRECTORY_URL/*" \ |
| 545 | --no-local --filter="blob:limit=5k" server client && |
| 546 | |
| 547 | # Check that the existing "cdn" remote has been properly updated. |
| 548 | printf "%s\n" "$TRASH_DIRECTORY_URL/lop" "true" "lop" "+refs/heads/*:refs/remotes/lop/*" >expect && |
| 549 | git -C client config "remote.cdn.url" >actual && |
| 550 | git -C client config "remote.cdn.promisor" >>actual && |
| 551 | git -C client config "remote.cdn.advertisedAs" >>actual && |
| 552 | git -C client config "remote.cdn.fetch" >>actual && |
| 553 | test_cmp expect actual && |
| 554 | |
| 555 | # Check that no new "cdn-1" remote has been created. |
| 556 | test_must_fail git -C client config "remote.cdn-1.url" && |
| 557 | |
| 558 | # Check that the largest object is still missing on the server |
| 559 | check_missing_objects server 1 "$oid" |
| 560 | ' |
| 561 | |
| 562 | test_expect_success "clone with invalid promisor.acceptFromServerUrl" ' |
| 563 | git -C server config promisor.advertise true && |
| 564 | test_when_finished "rm -rf client" && |
| 565 | |
| 566 | # As "bad name" contains a space, which is not a valid remote name, |
| 567 | # the pattern should be rejected with a warning and no remote created. |
| 568 | GIT_NO_LAZY_FETCH=0 git clone \ |
| 569 | -c promisor.acceptfromserver=None \ |
| 570 | -c "promisor.acceptFromServerUrl=bad name=https://example.com/*" \ |
| 571 | --no-local --filter="blob:limit=5k" server client 2>err && |
| 572 | |
| 573 | # Check that a warning was emitted |
| 574 | test_grep "invalid remote name '\''bad name'\''" err && |
| 575 | |
| 576 | # Check that no remote was auto-created |
| 577 | test_must_fail git -C client config get --regexp "remote\..*\.advertisedas" && |
| 578 | |
| 579 | # Check that the largest object is not missing on the server |
| 580 | check_missing_objects server 0 "" && |
| 581 | |
| 582 | # Reinitialize server so that the largest object is missing again |
| 583 | initialize_server 1 "$oid" |
| 584 | ' |
| 585 | |
| 586 | test_expect_success "clone with promisor.sendFields" ' |
| 587 | git -C server config promisor.advertise true && |
| 588 | test_when_finished "rm -rf client" && |
| 589 | |
| 590 | git -C server remote add otherLop "https://invalid.invalid" && |
| 591 | git -C server config remote.otherLop.token "fooBar" && |
| 592 | git -C server config remote.otherLop.stuff "baz" && |
| 593 | git -C server config remote.otherLop.partialCloneFilter "blob:limit=10k" && |
| 594 | test_when_finished "git -C server remote remove otherLop" && |
| 595 | test_config -C server promisor.sendFields "partialCloneFilter, token" && |
| 596 | test_when_finished "rm trace" && |
| 597 | |
| 598 | # Clone from server to create a client |
| 599 | GIT_TRACE_PACKET="$(pwd)/trace" GIT_NO_LAZY_FETCH=0 git clone \ |
| 600 | -c remote.lop.promisor=true \ |
| 601 | -c remote.lop.fetch="+refs/heads/*:refs/remotes/lop/*" \ |
| 602 | -c remote.lop.url="$TRASH_DIRECTORY_URL/lop" \ |
| 603 | -c promisor.acceptfromserver=All \ |
| 604 | --no-local --filter="blob:limit=5k" server client && |
| 605 | |
| 606 | # Check that fields are properly transmitted |
| 607 | PR1="name=lop,url=$ENCODED_TRASH_DIRECTORY_URL/lop,partialCloneFilter=blob:none" && |
| 608 | PR2="name=otherLop,url=https://invalid.invalid,partialCloneFilter=blob:limit=10k,token=fooBar" && |
| 609 | test_grep "clone< promisor-remote=$PR1;$PR2" trace && |
| 610 | test_grep "clone> promisor-remote=lop;otherLop" trace && |
| 611 | |
| 612 | # Check that the largest object is still missing on the server |
| 613 | check_missing_objects server 1 "$oid" |
| 614 | ' |
| 615 | |
| 616 | test_expect_success "clone with promisor.checkFields" ' |
| 617 | git -C server config promisor.advertise true && |
| 618 | test_when_finished "rm -rf client" && |
| 619 | |
| 620 | git -C server remote add otherLop "https://invalid.invalid" && |
| 621 | git -C server config remote.otherLop.token "fooBar" && |
| 622 | git -C server config remote.otherLop.stuff "baz" && |
| 623 | git -C server config remote.otherLop.partialCloneFilter "blob:limit=10k" && |
| 624 | test_when_finished "git -C server remote remove otherLop" && |
| 625 | test_config -C server promisor.sendFields "partialCloneFilter, token" && |
| 626 | test_when_finished "rm trace" && |
| 627 | |
| 628 | # Clone from server to create a client |
| 629 | GIT_TRACE_PACKET="$(pwd)/trace" GIT_NO_LAZY_FETCH=0 git clone \ |
| 630 | -c remote.lop.promisor=true \ |
| 631 | -c remote.lop.fetch="+refs/heads/*:refs/remotes/lop/*" \ |
| 632 | -c remote.lop.url="$TRASH_DIRECTORY_URL/lop" \ |
| 633 | -c remote.lop.partialCloneFilter="blob:none" \ |
| 634 | -c promisor.acceptfromserver=All \ |
| 635 | -c promisor.checkFields=partialcloneFilter \ |
| 636 | --no-local --filter="blob:limit=5k" server client && |
| 637 | |
| 638 | # Check that fields are properly transmitted |
| 639 | PR1="name=lop,url=$ENCODED_TRASH_DIRECTORY_URL/lop,partialCloneFilter=blob:none" && |
| 640 | PR2="name=otherLop,url=https://invalid.invalid,partialCloneFilter=blob:limit=10k,token=fooBar" && |
| 641 | test_grep "clone< promisor-remote=$PR1;$PR2" trace && |
| 642 | test_grep "clone> promisor-remote=lop" trace && |
| 643 | test_grep ! "clone> promisor-remote=lop;otherLop" trace && |
| 644 | |
| 645 | # Check that the largest object is still missing on the server |
| 646 | check_missing_objects server 1 "$oid" |
| 647 | ' |
| 648 | |
| 649 | test_expect_success "clone with promisor.storeFields=partialCloneFilter" ' |
| 650 | git -C server config promisor.advertise true && |
| 651 | test_when_finished "rm -rf client" && |
| 652 | |
| 653 | git -C server remote add otherLop "https://invalid.invalid" && |
| 654 | git -C server config remote.otherLop.token "fooBar" && |
| 655 | git -C server config remote.otherLop.stuff "baz" && |
| 656 | git -C server config remote.otherLop.partialCloneFilter "blob:limit=10k" && |
| 657 | test_when_finished "git -C server remote remove otherLop" && |
| 658 | |
| 659 | git -C server config remote.lop.token "fooXXX" && |
| 660 | git -C server config remote.lop.partialCloneFilter "blob:limit=8k" && |
| 661 | |
| 662 | test_config -C server promisor.sendFields "partialCloneFilter, token" && |
| 663 | test_when_finished "rm trace" && |
| 664 | |
| 665 | # Clone from server to create a client |
| 666 | GIT_TRACE_PACKET="$(pwd)/trace" GIT_NO_LAZY_FETCH=0 git clone \ |
| 667 | -c remote.lop.promisor=true \ |
| 668 | -c remote.lop.fetch="+refs/heads/*:refs/remotes/lop/*" \ |
| 669 | -c remote.lop.url="$TRASH_DIRECTORY_URL/lop" \ |
| 670 | -c remote.lop.token="fooYYY" \ |
| 671 | -c remote.lop.partialCloneFilter="blob:none" \ |
| 672 | -c promisor.acceptfromserver=All \ |
| 673 | -c promisor.storeFields=partialcloneFilter \ |
| 674 | --no-local --filter="blob:limit=5k" server client 2>err && |
| 675 | |
| 676 | # Check that the filter from the server is stored |
| 677 | echo "blob:limit=8k" >expected && |
| 678 | git -C client config remote.lop.partialCloneFilter >actual && |
| 679 | test_cmp expected actual && |
| 680 | |
| 681 | # Check that user is notified when the filter is stored |
| 682 | test_grep "Storing new filter from server for remote '\''lop'\''" err && |
| 683 | test_grep "'\''blob:none'\'' -> '\''blob:limit=8k'\''" err && |
| 684 | |
| 685 | # Check that the token from the server is NOT stored |
| 686 | echo "fooYYY" >expected && |
| 687 | git -C client config remote.lop.token >actual && |
| 688 | test_cmp expected actual && |
| 689 | test_grep ! "Storing new token from server" err && |
| 690 | |
| 691 | # Check that the filter for an unknown remote is NOT stored |
| 692 | test_must_fail git -C client config remote.otherLop.partialCloneFilter >actual && |
| 693 | |
| 694 | # Check that the largest object is still missing on the server |
| 695 | check_missing_objects server 1 "$oid" && |
| 696 | |
| 697 | # Change the configuration on the server and fetch from the client |
| 698 | git -C server config remote.lop.partialCloneFilter "blob:limit=7k" && |
| 699 | GIT_NO_LAZY_FETCH=0 git -C client fetch \ |
| 700 | --filter="blob:limit=5k" ../server 2>err && |
| 701 | |
| 702 | # Check that the fetch updated the configuration on the client |
| 703 | echo "blob:limit=7k" >expected && |
| 704 | git -C client config remote.lop.partialCloneFilter >actual && |
| 705 | test_cmp expected actual && |
| 706 | |
| 707 | # Check that user is notified when the new filter is stored |
| 708 | test_grep "Storing new filter from server for remote '\''lop'\''" err && |
| 709 | test_grep "'\''blob:limit=8k'\'' -> '\''blob:limit=7k'\''" err |
| 710 | ' |
| 711 | |
| 712 | test_expect_success "clone and fetch with --filter=auto" ' |
| 713 | git -C server config promisor.advertise true && |
| 714 | test_when_finished "rm -rf client trace" && |
| 715 | |
| 716 | git -C server config remote.lop.partialCloneFilter "blob:limit=9500" && |
| 717 | test_config -C server promisor.sendFields "partialCloneFilter" && |
| 718 | |
| 719 | GIT_TRACE_PACKET="$(pwd)/trace" GIT_NO_LAZY_FETCH=0 git clone \ |
| 720 | -c remote.lop.promisor=true \ |
| 721 | -c remote.lop.url="$TRASH_DIRECTORY_URL/lop" \ |
| 722 | -c promisor.acceptfromserver=All \ |
| 723 | --no-local --filter=auto server client 2>err && |
| 724 | |
| 725 | test_grep "filter blob:limit=9500" trace && |
| 726 | test_grep ! "filter auto" trace && |
| 727 | |
| 728 | # Verify "auto" is persisted in config |
| 729 | echo auto >expected && |
| 730 | git -C client config remote.origin.partialCloneFilter >actual && |
| 731 | test_cmp expected actual && |
| 732 | |
| 733 | # Check that the largest object is still missing on the server |
| 734 | check_missing_objects server 1 "$oid" && |
| 735 | |
| 736 | # Now change the filter on the server |
| 737 | git -C server config remote.lop.partialCloneFilter "blob:limit=5678" && |
| 738 | |
| 739 | # Get a new commit on the server to ensure "git fetch" actually runs fetch-pack |
| 740 | test_commit -C template new-commit && |
| 741 | git -C template push --all "$(pwd)/server" && |
| 742 | |
| 743 | # Perform a fetch WITH --filter=auto |
| 744 | rm -rf trace && |
| 745 | GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch --filter=auto && |
| 746 | |
| 747 | # Verify that the new filter was used |
| 748 | test_grep "filter blob:limit=5678" trace && |
| 749 | |
| 750 | # Check that the largest object is still missing on the server |
| 751 | check_missing_objects server 1 "$oid" && |
| 752 | |
| 753 | # Change the filter on the server again |
| 754 | git -C server config remote.lop.partialCloneFilter "blob:limit=5432" && |
| 755 | |
| 756 | # Get yet a new commit on the server to ensure fetch-pack runs |
| 757 | test_commit -C template yet-a-new-commit && |
| 758 | git -C template push --all "$(pwd)/server" && |
| 759 | |
| 760 | # Perform a fetch WITHOUT --filter=auto |
| 761 | # Relies on "auto" being persisted in the client config |
| 762 | rm -rf trace && |
| 763 | GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch && |
| 764 | |
| 765 | # Verify that the new filter was used |
| 766 | test_grep "filter blob:limit=5432" trace && |
| 767 | |
| 768 | # Check that the largest object is still missing on the server |
| 769 | check_missing_objects server 1 "$oid" |
| 770 | ' |
| 771 | |
| 772 | test_expect_success "clone with promisor.advertise set to 'true' but don't delete the client" ' |
| 773 | git -C server config promisor.advertise true && |
| 774 | |
| 775 | # Clone from server to create a client |
| 776 | GIT_NO_LAZY_FETCH=0 git clone -c remote.lop.promisor=true \ |
| 777 | -c remote.lop.fetch="+refs/heads/*:refs/remotes/lop/*" \ |
| 778 | -c remote.lop.url="$TRASH_DIRECTORY_URL/lop" \ |
| 779 | -c promisor.acceptfromserver=All \ |
| 780 | --no-local --filter="blob:limit=5k" server client && |
| 781 | |
| 782 | # Check that the largest object is still missing on the server |
| 783 | check_missing_objects server 1 "$oid" |
| 784 | ' |
| 785 | |
| 786 | test_expect_success "setup for subsequent fetches" ' |
| 787 | # Generate new commit with large blob |
| 788 | test-tool genrandom bar 10k >template/bar && |
| 789 | git -C template add bar && |
| 790 | git -C template commit -m bar && |
| 791 | |
| 792 | # Fetch new commit with large blob |
| 793 | git -C server fetch origin && |
| 794 | git -C server update-ref HEAD FETCH_HEAD && |
| 795 | git -C server rev-parse HEAD >expected_head && |
| 796 | |
| 797 | # Repack everything twice and remove .promisor files before |
| 798 | # each repack. This makes sure everything gets repacked |
| 799 | # into a single packfile. The second repack is necessary |
| 800 | # because the first one fetches from lop and creates a new |
| 801 | # packfile and its associated .promisor file. |
| 802 | |
| 803 | rm -f server/objects/pack/*.promisor && |
| 804 | git -C server -c repack.writebitmaps=false repack -a -d && |
| 805 | rm -f server/objects/pack/*.promisor && |
| 806 | git -C server -c repack.writebitmaps=false repack -a -d && |
| 807 | |
| 808 | # Unpack everything |
| 809 | rm pack-* && |
| 810 | mv server/objects/pack/pack-* . && |
| 811 | packfile=$(ls pack-*.pack) && |
| 812 | git -C server unpack-objects --strict <"$packfile" && |
| 813 | |
| 814 | # Copy new large object to lop |
| 815 | obj_bar="HEAD:bar" && |
| 816 | oid_bar="$(git -C server rev-parse $obj_bar)" && |
| 817 | copy_to_lop "$oid_bar" && |
| 818 | |
| 819 | # Reinitialize server so that the 2 largest objects are missing |
| 820 | printf "%s\n" "$oid" "$oid_bar" >expected_missing.txt && |
| 821 | initialize_server 2 expected_missing.txt && |
| 822 | |
| 823 | # Create one more client |
| 824 | cp -r client client2 |
| 825 | ' |
| 826 | |
| 827 | test_expect_success "subsequent fetch from a client when promisor.advertise is true" ' |
| 828 | git -C server config promisor.advertise true && |
| 829 | |
| 830 | GIT_NO_LAZY_FETCH=0 git -C client pull origin && |
| 831 | |
| 832 | git -C client rev-parse HEAD >actual && |
| 833 | test_cmp expected_head actual && |
| 834 | |
| 835 | cat client/bar >/dev/null && |
| 836 | |
| 837 | check_missing_objects server 2 expected_missing.txt |
| 838 | ' |
| 839 | |
| 840 | test_expect_success "subsequent fetch from a client when promisor.advertise is false" ' |
| 841 | git -C server config promisor.advertise false && |
| 842 | |
| 843 | GIT_NO_LAZY_FETCH=0 git -C client2 pull origin && |
| 844 | |
| 845 | git -C client2 rev-parse HEAD >actual && |
| 846 | test_cmp expected_head actual && |
| 847 | |
| 848 | cat client2/bar >/dev/null && |
| 849 | |
| 850 | check_missing_objects server 1 "$oid" |
| 851 | ' |
| 852 | |
| 853 | test_done |