http: avoid concurrent appends to partial packs
Pack requests stage downloads in a predictable partial-pack file so an interrupted transfer can be resumed. Both packfile URI and ordinary dumb HTTP requests use this staging path. Opening it in append mode forces each write to the current end of the file, so concurrent responses can append duplicate data and corrupt the pack. Open the partial pack read-write without O_APPEND and seek once to its current end. Each downloader then retains the offset matching the Range it requested. Because the staging key must uniquely identify immutable pack contents, overlapping responses write the same bytes at the same offsets instead of extending the file with duplicate data. MinGW's non-append O_RDWR open grants FILE_SHARE_DELETE only for an existing file. Create a missing partial pack exclusively, close it, and reopen it without O_CREAT so every retained descriptor permits another downloader to unlink the staging path. Duplicate that descriptor for index-pack instead of reopening the path after closing the stream; index-pack installs its own pack and the shared staging file is only unlinked, never renamed. Accept HTTP 416 when a partial pack is already complete and let index-pack validate its contents. Exercise resumed transfers, EOF ranges, overlapping 200 and 206 responses, and unlinking the staging path while index-pack still holds its descriptor. Clarify the staging-key documentation. Signed-off-by: Ted Nyman <tnyman@openai.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Ted Nyman committed
Jul 21, 2026 at 16:29 UTC
0125ac2f19d95a9b441a1ef5dc13d1059fbb0362
6 files changed
+289
-25
Documentation/git-http-fetch.adoc
+3
-2
@@ -48,8 +48,9 @@ commit-id::
48
line (which is not expected in
49
this case), 'git http-fetch' fetches the packfile directly at the given
50
URL and uses index-pack to generate corresponding .idx and .keep files.
51
- The hash is used to determine the name of the temporary file and is
52
- arbitrary. The output of index-pack is printed to stdout. Requires
51
+ The hash is used to determine the name of the temporary file. It need
52
+ not be the pack hash, but it must uniquely identify the pack contents
53
+ for resumption. The output of index-pack is printed to stdout. Requires
54
one or more --index-pack-arg options.
55
56
--index-pack-arg=<arg>::
http-fetch.c
+2
-1
@@ -70,7 +70,8 @@ static void fetch_single_packfile(struct object_id *packfile_hash,
70
71
if (start_active_slot(preq->slot)) {
72
run_active_slot(preq->slot);
73
- if (results.curl_result != CURLE_OK) {
73
+ if (results.curl_result != CURLE_OK &&
74
+ results.http_code != 416) {
75
struct url_info url;
76
char *nurl = url_normalize(preq->url, &url);
77
if (!nurl || !git_env_bool("GIT_TRACE_REDACT", 1)) {
http-push.c
+2
-1
@@ -595,7 +595,8 @@ static void finish_request(struct transfer_request *request)
595
596
} else if (request->state == RUN_FETCH_PACKED) {
597
int fail = 1;
598
- if (request->curl_result != CURLE_OK) {
598
+ if (request->curl_result != CURLE_OK &&
599
+ request->http_code != 416) {
600
fprintf(stderr, "Unable to get pack file %s\n%s",
601
request->url, curl_errorstr);
602
} else {
http-walker.c
+2
-1
@@ -451,7 +451,8 @@ static int http_fetch_pack(struct walker *walker, struct alt_base *repo,
451
452
if (start_active_slot(preq->slot)) {
453
run_active_slot(preq->slot);
454
- if (results.curl_result != CURLE_OK) {
454
+ if (results.curl_result != CURLE_OK &&
455
+ results.http_code != 416) {
456
error("Unable to get pack file %s\n%s", preq->url,
457
curl_errorstr);
458
goto abort;
http.c
+36
-20
@@ -2688,10 +2688,13 @@ int finish_http_pack_request(struct http_pack_request *preq)
2688
int tmpfile_fd;
2689
int ret = 0;
2690
2691
+ /* Another downloader may unlink the staging path while we index it. */
2692
+ tmpfile_fd = xdup(fileno(preq->packfile));
2693
fclose(preq->packfile);
2694
preq->packfile = NULL;
2693
-
2694
- tmpfile_fd = xopen(preq->tmpfile.buf, O_RDONLY);
2695
+ if (lseek(tmpfile_fd, 0, SEEK_SET) < 0)
2696
+ die_errno("unable to seek local file %s for pack",
2697
+ preq->tmpfile.buf);
2698
2699
ip.git_cmd = 1;
2700
ip.in = tmpfile_fd;
@@ -2704,13 +2707,8 @@ int finish_http_pack_request(struct http_pack_request *preq)
2707
else
2708
ip.no_stdout = 1;
2709
2707
- if (run_command(&ip)) {
2710
+ if (run_command(&ip))
2711
ret = -1;
2709
- goto cleanup;
2710
- }
2711
-
2712
-cleanup:
2713
- close(tmpfile_fd);
2712
unlink(preq->tmpfile.buf);
2713
return ret;
2714
}
@@ -2738,22 +2736,45 @@ struct http_pack_request *new_http_pack_request(
2736
struct http_pack_request *new_direct_http_pack_request(
2737
const unsigned char *packed_git_hash, char *url)
2738
{
2741
- off_t prev_posn = 0;
2739
+ off_t prev_posn;
2740
struct http_pack_request *preq;
2741
+ int fd;
2742
2743
CALLOC_ARRAY(preq, 1);
2744
strbuf_init(&preq->tmpfile, 0);
2746
-
2745
preq->url = url;
2746
2747
odb_pack_name(the_repository, &preq->tmpfile, packed_git_hash, "pack");
2748
strbuf_addstr(&preq->tmpfile, ".temp");
2751
- preq->packfile = fopen(preq->tmpfile.buf, "a");
2752
- if (!preq->packfile) {
2753
- error("Unable to open local file %s for pack",
2754
- preq->tmpfile.buf);
2749
+ /*
2750
+ * MinGW's non-append O_RDWR open grants FILE_SHARE_DELETE only for an
2751
+ * existing file; reopen a newly created file so others may unlink it.
2752
+ */
2753
+ for (;;) {
2754
+ fd = open(preq->tmpfile.buf, O_RDWR);
2755
+ if (fd >= 0 || errno != ENOENT)
2756
+ break;
2757
+ fd = open(preq->tmpfile.buf, O_RDWR | O_CREAT | O_EXCL, 0666);
2758
+ if (fd >= 0) {
2759
+ close(fd);
2760
+ continue;
2761
+ }
2762
+ if (errno != EEXIST)
2763
+ break;
2764
+ }
2765
+ if (fd < 0) {
2766
+ error_errno("unable to open local file %s for pack",
2767
+ preq->tmpfile.buf);
2768
goto abort;
2769
}
2770
+ prev_posn = lseek(fd, 0, SEEK_END);
2771
+ if (prev_posn < 0) {
2772
+ error_errno("unable to seek local file %s for pack",
2773
+ preq->tmpfile.buf);
2774
+ close(fd);
2775
+ goto abort;
2776
+ }
2777
+ preq->packfile = xfdopen(fd, "w");
2778
2779
preq->slot = get_active_slot();
2780
preq->headers = object_request_headers();
@@ -2762,12 +2783,7 @@ struct http_pack_request *new_direct_http_pack_request(
2783
curl_easy_setopt(preq->slot->curl, CURLOPT_URL, preq->url);
2784
curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER, preq->headers);
2785
2765
- /*
2766
- * If there is data present from a previous transfer attempt,
2767
- * resume where it left off
2768
- */
2769
- prev_posn = ftello(preq->packfile);
2770
- if (prev_posn>0) {
2786
+ if (prev_posn > 0) {
2787
if (http_is_verbose)
2788
fprintf(stderr,
2789
"Resuming fetch of pack %s at byte %"PRIuMAX"\n",
t/t5550-http-fetch-dumb.sh
+244
@@ -293,6 +293,250 @@ test_expect_success 'http-fetch --packfile' '
293
git -C packfileclient cat-file -e "$HASH"
294
'
295
296
+test_expect_success 'http-fetch --packfile resumes a partial download' '
297
+ git init packfileclient-resume &&
298
+ p=$(cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git &&
299
+ ls objects/pack/pack-*.pack) &&
300
+ tmpfile="packfileclient-resume/.git/objects/pack/pack-$ARBITRARY.pack.temp" &&
301
+ test_copy_bytes 64 <"$HTTPD_DOCUMENT_ROOT_PATH/repo_pack.git/$p" >"$tmpfile" &&
302
+ GIT_TRACE_CURL="$TRASH_DIRECTORY/resume.trace" \
303
+ git -C packfileclient-resume http-fetch --packfile="$ARBITRARY" \
304
+ --index-pack-arg=index-pack --index-pack-arg=--stdin \
305
+ --index-pack-arg=--keep \
306
+ "$HTTPD_URL/dumb/repo_pack.git/$p" >out &&
307
+ test_grep "Range: bytes=64-" resume.trace &&
308
+ test_path_is_missing "$tmpfile" &&
309
+ git -C packfileclient-resume cat-file -e "$HASH"
310
+'
311
+
312
+test_expect_success 'http-fetch --packfile permits unlink while indexing' '
313
+ git init packfileclient-unlink &&
314
+ p=$(cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git &&
315
+ ls objects/pack/pack-*.pack) &&
316
+ tmpfile="packfileclient-unlink/.git/objects/pack/pack-$ARBITRARY.pack.temp" &&
317
+ write_script git-unlink-index-pack <<-\EOF &&
318
+ test -f "$GIT_TEST_PACK_TEMP" || exit 1
319
+ rm "$GIT_TEST_PACK_TEMP" || exit 1
320
+ exec git index-pack "$@"
321
+ EOF
322
+ test_when_finished "rm -f git-unlink-index-pack" &&
323
+ PATH="$TRASH_DIRECTORY:$PATH" \
324
+ GIT_TEST_PACK_TEMP="$TRASH_DIRECTORY/$tmpfile" \
325
+ git -C packfileclient-unlink http-fetch --packfile="$ARBITRARY" \
326
+ --index-pack-arg=unlink-index-pack \
327
+ --index-pack-arg=--stdin --index-pack-arg=--keep \
328
+ "$HTTPD_URL/dumb/repo_pack.git/$p" >out &&
329
+ test_path_is_missing "$tmpfile" &&
330
+ git -C packfileclient-unlink cat-file -e "$HASH"
331
+'
332
+
333
+test_expect_success PIPE 'concurrent http-fetch --packfile accepts a complete partial' '
334
+ git init packfileclient-concurrent &&
335
+ p=$(cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git &&
336
+ ls objects/pack/pack-*.pack) &&
337
+ packhash=$(basename "$p" .pack) &&
338
+ packhash=${packhash#pack-} &&
339
+ tmpfile="packfileclient-concurrent/.git/objects/pack/pack-$packhash.pack.temp" &&
340
+ test_copy_bytes 64 <"$HTTPD_DOCUMENT_ROOT_PATH/repo_pack.git/$p" >"$tmpfile" &&
341
+ mkfifo first-ready first-continue &&
342
+ exec 8<>first-ready &&
343
+ exec 9<>first-continue &&
344
+ write_script git-wait-index-pack <<-\EOF &&
345
+ echo ready >"$GIT_TEST_WAIT_READY" &&
346
+ read continue <"$GIT_TEST_WAIT_CONTINUE" &&
347
+ exec git index-pack "$@"
348
+ EOF
349
+ {
350
+ (
351
+ if ! PATH="$TRASH_DIRECTORY:$PATH" \
352
+ GIT_TEST_WAIT_READY="$TRASH_DIRECTORY/first-ready" \
353
+ GIT_TEST_WAIT_CONTINUE="$TRASH_DIRECTORY/first-continue" \
354
+ GIT_TRACE_CURL="$TRASH_DIRECTORY/first.trace" \
355
+ git -C packfileclient-concurrent http-fetch --packfile="$packhash" \
356
+ --index-pack-arg=wait-index-pack \
357
+ --index-pack-arg=--stdin --index-pack-arg=--keep \
358
+ "$HTTPD_URL/dumb/repo_pack.git/$p" >first.out
359
+ then
360
+ echo failed >"$TRASH_DIRECTORY/first-ready" &&
361
+ exit 1
362
+ fi
363
+ ) &
364
+ first_pid=$!
365
+ } &&
366
+ test_when_finished "
367
+ echo continue >&9
368
+ kill $first_pid 2>/dev/null || :
369
+ wait $first_pid 2>/dev/null || :
370
+ exec 8>&-
371
+ exec 9>&-
372
+ rm -f first-ready first-continue git-wait-index-pack
373
+ " &&
374
+ read ready <&8 &&
375
+ test "$ready" = ready &&
376
+ GIT_TRACE_CURL="$TRASH_DIRECTORY/second.trace" \
377
+ git -C packfileclient-concurrent http-fetch --packfile="$packhash" \
378
+ --index-pack-arg=index-pack \
379
+ --index-pack-arg=--stdin --index-pack-arg=--keep \
380
+ "$HTTPD_URL/dumb/repo_pack.git/$p" >second.out &&
381
+ echo continue >&9 &&
382
+ wait "$first_pid" &&
383
+ printf "pack\t%s\n" "$packhash" >expect &&
384
+ test_cmp expect first.out &&
385
+ printf "keep\t%s\n" "$packhash" >expect &&
386
+ test_cmp expect second.out &&
387
+ test_grep "Range: bytes=64-" first.trace &&
388
+ test_grep "Range: bytes=[0-9]*-" second.trace &&
389
+ test_grep "HTTP/[0-9.]* 416" second.trace &&
390
+ test_path_is_missing "$tmpfile" &&
391
+ git -C packfileclient-concurrent cat-file -e "$HASH"
392
+'
393
+
394
+test_expect_success PERL,PIPE 'concurrent http-fetch --packfile cannot corrupt an overlapping download' '
395
+ git init packfileclient-overlap &&
396
+ blob=$(test-tool genrandom pack-overlap 2m |
397
+ git -C "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git \
398
+ hash-object -w --stdin) &&
399
+ packhash=$(printf "%s\n" "$blob" |
400
+ git -C "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git \
401
+ pack-objects "$TRASH_DIRECTORY/overlap-pack") &&
402
+ pack="$TRASH_DIRECTORY/overlap-pack-$packhash.pack" &&
403
+ tmpfile="packfileclient-overlap/.git/objects/pack/pack-$packhash.pack.temp" &&
404
+ mkfifo server-ready first-ready &&
405
+ exec 7<>server-ready &&
406
+ exec 8<>first-ready &&
407
+ write_script slow-pack-server "$PERL_PATH" <<-\EOF &&
408
+ use strict;
409
+ use warnings;
410
+ use IO::Socket::INET;
411
+
412
+ my ($packfile, $server_ready, $first_ready) = @ARGV;
413
+ open(my $in, "<:raw", $packfile) or die "open $packfile: $!";
414
+ my $pack = do { local $/; <$in> };
415
+ close($in) or die "close $packfile: $!";
416
+ my $server = IO::Socket::INET->new(LocalAddr => "127.0.0.1",
417
+ LocalPort => 0, Proto => "tcp", Listen => 2, ReuseAddr => 1)
418
+ or die "listen: $!";
419
+
420
+ sub signal_ready {
421
+ my ($file, $value) = @_;
422
+ open(my $out, ">", $file) or die "open $file: $!";
423
+ print $out "$value\n" or die "write $file: $!";
424
+ close($out) or die "close $file: $!";
425
+ }
426
+
427
+ sub write_all {
428
+ my ($out, $data) = @_;
429
+ my $offset = 0;
430
+ while ($offset < length($data)) {
431
+ my $written = syswrite($out, $data,
432
+ length($data) - $offset, $offset);
433
+ defined($written) && $written or die "write response: $!";
434
+ $offset += $written;
435
+ }
436
+ }
437
+
438
+ sub start_response {
439
+ my $out = $server->accept() or die "accept: $!";
440
+ <$out> or die "read request: $!";
441
+ my $start = 0;
442
+ while (<$out>) {
443
+ last if /^\r?\n$/;
444
+ $start = $1 if /^Range: bytes=(\d+)-/i;
445
+ }
446
+ $start < length($pack) or die "invalid range $start";
447
+ my $length = length($pack) - $start;
448
+ my $middle = int($length / 2);
449
+ my $status = $start ? "206 Partial Content" : "200 OK";
450
+ my $headers = "HTTP/1.1 $status\r\n" .
451
+ "Content-Length: $length\r\n" .
452
+ ($start ? "Content-Range: bytes $start-" .
453
+ (length($pack) - 1) . "/" . length($pack) . "\r\n" : "") .
454
+ "Connection: close\r\n\r\n";
455
+ write_all($out, $headers);
456
+ write_all($out, substr($pack, $start, $middle));
457
+ return ($out, $start + $middle);
458
+ }
459
+
460
+ signal_ready($server_ready, $server->sockport());
461
+ my ($first, $first_pos) = start_response();
462
+ signal_ready($first_ready, "ready");
463
+ my ($second, $second_pos) = start_response();
464
+ write_all($first, substr($pack, $first_pos));
465
+ write_all($second, substr($pack, $second_pos));
466
+ close($first) or die "close first response: $!";
467
+ close($second) or die "close second response: $!";
468
+ EOF
469
+ {
470
+ (
471
+ if ! "$TRASH_DIRECTORY/slow-pack-server" "$pack" \
472
+ "$TRASH_DIRECTORY/server-ready" \
473
+ "$TRASH_DIRECTORY/first-ready"
474
+ then
475
+ echo failed >"$TRASH_DIRECTORY/server-ready" &&
476
+ echo failed >"$TRASH_DIRECTORY/first-ready" &&
477
+ exit 1
478
+ fi
479
+ ) >server.log 2>&1 &
480
+ server_pid=$!
481
+ } &&
482
+ test_when_finished "
483
+ kill $server_pid 2>/dev/null || :
484
+ wait $server_pid 2>/dev/null || :
485
+ exec 7>&-
486
+ exec 8>&-
487
+ rm -f server-ready first-ready slow-pack-server
488
+ " &&
489
+ read port <&7 &&
490
+ url="http://127.0.0.1:$port/pack" &&
491
+ {
492
+ (
493
+ if ! GIT_TRACE_CURL="$TRASH_DIRECTORY/overlap-first.trace" \
494
+ GIT_TRACE_CURL_NO_DATA=1 \
495
+ git -C packfileclient-overlap http-fetch --packfile="$packhash" \
496
+ --index-pack-arg=index-pack \
497
+ --index-pack-arg=--stdin --index-pack-arg=--keep \
498
+ "$url" >first.out
499
+ then
500
+ echo failed >"$TRASH_DIRECTORY/first-ready" &&
501
+ exit 1
502
+ fi
503
+ ) &
504
+ first_pid=$!
505
+ } &&
506
+ test_when_finished "
507
+ kill $first_pid 2>/dev/null || :
508
+ wait $first_pid 2>/dev/null || :
509
+ " &&
510
+ read ready <&8 &&
511
+ test "$ready" = ready &&
512
+ test_path_is_file "$tmpfile" &&
513
+ test -s "$tmpfile" &&
514
+ {
515
+ GIT_TRACE_CURL="$TRASH_DIRECTORY/overlap-second.trace" \
516
+ GIT_TRACE_CURL_NO_DATA=1 \
517
+ git -C packfileclient-overlap http-fetch --packfile="$packhash" \
518
+ --index-pack-arg=index-pack \
519
+ --index-pack-arg=--stdin --index-pack-arg=--keep \
520
+ "$url" >second.out &
521
+ second_pid=$!
522
+ } &&
523
+ test_when_finished "
524
+ kill $second_pid 2>/dev/null || :
525
+ wait $second_pid 2>/dev/null || :
526
+ " &&
527
+ wait "$server_pid" &&
528
+ wait "$first_pid" &&
529
+ wait "$second_pid" &&
530
+ test_grep "HTTP/[0-9.]* 200" overlap-first.trace &&
531
+ test_grep "Range: bytes=[1-9][0-9]*-" overlap-second.trace &&
532
+ test_grep "HTTP/[0-9.]* 206" overlap-second.trace &&
533
+ printf "keep\t%s\npack\t%s\n" "$packhash" "$packhash" | sort >expect &&
534
+ sort first.out second.out >actual &&
535
+ test_cmp expect actual &&
536
+ test_path_is_missing "$tmpfile" &&
537
+ git -C packfileclient-overlap cat-file -e "$blob"
538
+'
539
+
540
test_expect_success 'fetch notices corrupt pack' '
541
cp -R "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad1.git &&
542
(cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad1.git &&