[PATCH] HTTP partial transfer support for object, pack, and index transfers
HTTP partial transfer support for object, pack, and index transfers [jc: this should not be placed in "master" -- it does not have any fixes requested on the list.] Signed-off-by: Nick Hengeveld <nickh@reactrix.com> Signed-off-by: Junio C Hamano <junkio@cox.net>
Nick Hengeveld committed
Sep 28, 2005 at 10:14 UTC
49a0f240f7be05728f97903efd97ad7898ff6d08
1 file changed
+161
-23
http-fetch.c
+161
-23
@@ -13,8 +13,12 @@
13
#define curl_global_init(a) do { /* nothing */ } while(0)
14
#endif
15
16
+#define PREV_BUF_SIZE 4096
17
+#define RANGE_HEADER_SIZE 30
18
+
19
static CURL *curl;
20
static struct curl_slist *no_pragma_header;
21
+static struct curl_slist *no_range_header;
22
static char curl_errorstr[CURL_ERROR_SIZE];
23
24
static char *initial_base;
@@ -87,12 +91,37 @@ void prefetch(unsigned char *sha1)
91
{
92
}
93
94
+int relink_or_rename(char *old, char *new) {
95
+ int ret;
96
+
97
+ ret = link(old, new);
98
+ if (ret < 0) {
99
+ /* Same Coda hack as in write_sha1_file(sha1_file.c) */
100
+ ret = errno;
101
+ if (ret == EXDEV && !rename(old, new))
102
+ return 0;
103
+ }
104
+ unlink(old);
105
+ if (ret) {
106
+ if (ret != EEXIST)
107
+ return ret;
108
+ }
109
+
110
+ return 0;
111
+}
112
+
113
static int got_alternates = 0;
114
115
static int fetch_index(struct alt_base *repo, unsigned char *sha1)
116
{
117
char *filename;
118
char *url;
119
+ char tmpfile[PATH_MAX];
120
+ int ret;
121
+ long prev_posn = 0;
122
+ char range[RANGE_HEADER_SIZE];
123
+ struct curl_slist *range_header = NULL;
124
+ CURLcode curl_result;
125
126
FILE *indexfile;
127
@@ -108,7 +137,8 @@ static int fetch_index(struct alt_base *repo, unsigned char *sha1)
137
repo->base, sha1_to_hex(sha1));
138
139
filename = sha1_pack_index_name(sha1);
111
- indexfile = fopen(filename, "w");
140
+ snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
141
+ indexfile = fopen(tmpfile, "a");
142
if (!indexfile)
143
return error("Unable to open local file %s for pack index",
144
filename);
@@ -119,13 +149,36 @@ static int fetch_index(struct alt_base *repo, unsigned char *sha1)
149
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, no_pragma_header);
150
curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errorstr);
151
122
- if (curl_easy_perform(curl)) {
152
+ /* If there is data present from a previous transfer attempt,
153
+ resume where it left off */
154
+ prev_posn = ftell(indexfile);
155
+ if (prev_posn>0) {
156
+ if (get_verbosely)
157
+ fprintf(stderr,
158
+ "Resuming fetch of index for pack %s at byte %ld\n",
159
+ sha1_to_hex(sha1), prev_posn);
160
+ sprintf(range, "Range: bytes=%ld-", prev_posn);
161
+ range_header = curl_slist_append(range_header, range);
162
+ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, range_header);
163
+ }
164
+
165
+ /* Clear out the Range: header after performing the request, so
166
+ other curl requests don't inherit inappropriate header data */
167
+ curl_result = curl_easy_perform(curl);
168
+ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, no_range_header);
169
+ if (curl_result != 0) {
170
fclose(indexfile);
171
return error("Unable to get pack index %s\n%s", url,
172
curl_errorstr);
173
}
174
175
fclose(indexfile);
176
+
177
+ ret = relink_or_rename(tmpfile, filename);
178
+ if (ret)
179
+ return error("unable to write index filename %s: %s",
180
+ filename, strerror(ret));
181
+
182
return 0;
183
}
184
@@ -306,6 +359,12 @@ static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
359
struct packed_git **lst;
360
FILE *packfile;
361
char *filename;
362
+ char tmpfile[PATH_MAX];
363
+ int ret;
364
+ long prev_posn = 0;
365
+ char range[RANGE_HEADER_SIZE];
366
+ struct curl_slist *range_header = NULL;
367
+ CURLcode curl_result;
368
369
if (fetch_indices(repo))
370
return -1;
@@ -325,7 +384,8 @@ static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
384
repo->base, sha1_to_hex(target->sha1));
385
386
filename = sha1_pack_name(target->sha1);
328
- packfile = fopen(filename, "w");
387
+ snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
388
+ packfile = fopen(tmpfile, "a");
389
if (!packfile)
390
return error("Unable to open local file %s for pack",
391
filename);
@@ -336,7 +396,24 @@ static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
396
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, no_pragma_header);
397
curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errorstr);
398
339
- if (curl_easy_perform(curl)) {
399
+ /* If there is data present from a previous transfer attempt,
400
+ resume where it left off */
401
+ prev_posn = ftell(packfile);
402
+ if (prev_posn>0) {
403
+ if (get_verbosely)
404
+ fprintf(stderr,
405
+ "Resuming fetch of pack %s at byte %ld\n",
406
+ sha1_to_hex(target->sha1), prev_posn);
407
+ sprintf(range, "Range: bytes=%ld-", prev_posn);
408
+ range_header = curl_slist_append(range_header, range);
409
+ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, range_header);
410
+ }
411
+
412
+ /* Clear out the Range: header after performing the request, so
413
+ other curl requests don't inherit inappropriate header data */
414
+ curl_result = curl_easy_perform(curl);
415
+ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, no_range_header);
416
+ if (curl_result != 0) {
417
fclose(packfile);
418
return error("Unable to get pack file %s\n%s", url,
419
curl_errorstr);
@@ -344,6 +421,11 @@ static int fetch_pack(struct alt_base *repo, unsigned char *sha1)
421
422
fclose(packfile);
423
424
+ ret = relink_or_rename(tmpfile, filename);
425
+ if (ret)
426
+ return error("unable to write pack filename %s: %s",
427
+ filename, strerror(ret));
428
+
429
lst = &repo->packs;
430
while (*lst != target)
431
lst = &((*lst)->next);
@@ -360,14 +442,29 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1)
442
char *filename = sha1_file_name(sha1);
443
unsigned char real_sha1[20];
444
char tmpfile[PATH_MAX];
445
+ char prevfile[PATH_MAX];
446
int ret;
447
char *url;
448
char *posn;
449
+ int prevlocal;
450
+ unsigned char prev_buf[PREV_BUF_SIZE];
451
+ ssize_t prev_read = 0;
452
+ long prev_posn = 0;
453
+ char range[RANGE_HEADER_SIZE];
454
+ struct curl_slist *range_header = NULL;
455
+ CURLcode curl_result;
456
+
457
+ snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
458
+ snprintf(prevfile, sizeof(prevfile), "%s.prev", filename);
459
+ unlink(prevfile);
460
+ rename(tmpfile, prevfile);
461
+ unlink(tmpfile);
462
+
463
+ local = open(tmpfile, O_WRONLY | O_CREAT | O_EXCL, 0666);
464
367
- snprintf(tmpfile, sizeof(tmpfile), "%s/obj_XXXXXX",
368
- get_object_directory());
465
+ /* Note: if another instance starts now, it will turn our new
466
+ tmpfile into its prevfile. */
467
370
- local = mkstemp(tmpfile);
468
if (local < 0)
469
return error("Couldn't create temporary file %s for %s: %s\n",
470
tmpfile, filename, strerror(errno));
@@ -396,8 +493,57 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1)
493
494
curl_easy_setopt(curl, CURLOPT_URL, url);
495
399
- if (curl_easy_perform(curl)) {
400
- unlink(filename);
496
+ /* If a previous temp file is present, process what was already
497
+ fetched. */
498
+ prevlocal = open(prevfile, O_RDONLY);
499
+ if (prevlocal != -1) {
500
+ do {
501
+ prev_read = read(prevlocal, prev_buf, PREV_BUF_SIZE);
502
+ if (prev_read>0) {
503
+ if (fwrite_sha1_file(prev_buf,
504
+ 1,
505
+ prev_read,
506
+ NULL) == prev_read) {
507
+ prev_posn += prev_read;
508
+ } else {
509
+ prev_read = -1;
510
+ }
511
+ }
512
+ } while (prev_read > 0);
513
+ close(prevlocal);
514
+ }
515
+ unlink(prevfile);
516
+
517
+ /* Reset inflate/SHA1 if there was an error reading the previous temp
518
+ file; also rewind to the beginning of the local file. */
519
+ if (prev_read == -1) {
520
+ memset(&stream, 0, sizeof(stream));
521
+ inflateInit(&stream);
522
+ SHA1_Init(&c);
523
+ if (prev_posn>0) {
524
+ prev_posn = 0;
525
+ lseek(local, SEEK_SET, 0);
526
+ }
527
+ }
528
+
529
+ /* If we have successfully processed data from a previous fetch
530
+ attempt, only fetch the data we don't already have. */
531
+ if (prev_posn>0) {
532
+ if (get_verbosely)
533
+ fprintf(stderr,
534
+ "Resuming fetch of object %s at byte %ld\n",
535
+ hex, prev_posn);
536
+ sprintf(range, "Range: bytes=%ld-", prev_posn);
537
+ range_header = curl_slist_append(range_header, range);
538
+ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, range_header);
539
+ }
540
+
541
+ /* Clear out the Range: header after performing the request, so
542
+ other curl requests don't inherit inappropriate header data */
543
+ curl_result = curl_easy_perform(curl);
544
+ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, no_range_header);
545
+ if (curl_result != 0) {
546
+ unlink(tmpfile);
547
return error("%s", curl_errorstr);
548
}
549
@@ -413,20 +559,11 @@ static int fetch_object(struct alt_base *repo, unsigned char *sha1)
559
unlink(tmpfile);
560
return error("File %s has bad hash\n", hex);
561
}
416
- ret = link(tmpfile, filename);
417
- if (ret < 0) {
418
- /* Same Coda hack as in write_sha1_file(sha1_file.c) */
419
- ret = errno;
420
- if (ret == EXDEV && !rename(tmpfile, filename))
421
- goto out;
422
- }
423
- unlink(tmpfile);
424
- if (ret) {
425
- if (ret != EEXIST)
426
- return error("unable to write sha1 filename %s: %s",
427
- filename, strerror(ret));
428
- }
429
- out:
562
+ ret = relink_or_rename(tmpfile, filename);
563
+ if (ret)
564
+ return error("unable to write sha1 filename %s: %s",
565
+ filename, strerror(ret));
566
+
567
pull_say("got %s\n", hex);
568
return 0;
569
}
@@ -519,6 +656,7 @@ int main(int argc, char **argv)
656
657
curl = curl_easy_init();
658
no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
659
+ no_range_header = curl_slist_append(no_range_header, "Range:");
660
661
curl_ssl_verify = getenv("GIT_SSL_NO_VERIFY") ? 0 : 1;
662
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, curl_ssl_verify);