Smart fetch over HTTP: client side
The git-remote-curl backend detects if the remote server supports the git-upload-pack service, and if so, runs git-fetch-pack locally in a pipe to generate the want/have commands. The advertisements from the server that were obtained during the discovery are passed into git-fetch-pack before the POST request starts, permitting server capability discovery and enablement. Common objects that are discovered are appended onto the request as have lines and are sent again on the next request. This allows the remote side to reinitialize its in-memory list of common objects during the next request. Because all requests are relatively short, below git-remote-curl's 1 MiB buffer limit, requests will use the standard Content-Length header and be valid HTTP/1.0 POST requests. This makes the fetch client more tolerant of proxy servers which don't support HTTP/1.1 or the chunked transfer encoding. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> CC: Daniel Barkalow <barkalow@iabervon.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Shawn O. Pearce committed
Oct 30, 2009 at 17:47 UTC
249b2004d8c9c58ed1ea1665dfd376af0312ed7e
3 files changed
+160
-22
builtin-fetch-pack.c
+93
-17
@@ -165,6 +165,24 @@ enum ack_type {
165
ACK_ready
166
};
167
168
+static void consume_shallow_list(int fd)
169
+{
170
+ if (args.stateless_rpc && args.depth > 0) {
171
+ /* If we sent a depth we will get back "duplicate"
172
+ * shallow and unshallow commands every time there
173
+ * is a block of have lines exchanged.
174
+ */
175
+ char line[1000];
176
+ while (packet_read_line(fd, line, sizeof(line))) {
177
+ if (!prefixcmp(line, "shallow "))
178
+ continue;
179
+ if (!prefixcmp(line, "unshallow "))
180
+ continue;
181
+ die("git fetch-pack: expected shallow list");
182
+ }
183
+ }
184
+}
185
+
186
static enum ack_type get_ack(int fd, unsigned char *result_sha1)
187
{
188
static char line[1000];
@@ -190,6 +208,15 @@ static enum ack_type get_ack(int fd, unsigned char *result_sha1)
208
die("git fetch_pack: expected ACK/NAK, got '%s'", line);
209
}
210
211
+static void send_request(int fd, struct strbuf *buf)
212
+{
213
+ if (args.stateless_rpc) {
214
+ send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
215
+ packet_flush(fd);
216
+ } else
217
+ safe_write(fd, buf->buf, buf->len);
218
+}
219
+
220
static int find_common(int fd[2], unsigned char *result_sha1,
221
struct ref *refs)
222
{
@@ -199,7 +226,10 @@ static int find_common(int fd[2], unsigned char *result_sha1,
226
unsigned in_vain = 0;
227
int got_continue = 0;
228
struct strbuf req_buf = STRBUF_INIT;
229
+ size_t state_len = 0;
230
231
+ if (args.stateless_rpc && multi_ack == 1)
232
+ die("--stateless-rpc requires multi_ack_detailed");
233
if (marked)
234
for_each_ref(clear_marks, NULL);
235
marked = 1;
@@ -256,13 +286,13 @@ static int find_common(int fd[2], unsigned char *result_sha1,
286
if (args.depth > 0)
287
packet_buf_write(&req_buf, "deepen %d", args.depth);
288
packet_buf_flush(&req_buf);
259
-
260
- safe_write(fd[1], req_buf.buf, req_buf.len);
289
+ state_len = req_buf.len;
290
291
if (args.depth > 0) {
292
char line[1024];
293
unsigned char sha1[20];
294
295
+ send_request(fd[1], &req_buf);
296
while (packet_read_line(fd[0], line, sizeof(line))) {
297
if (!prefixcmp(line, "shallow ")) {
298
if (get_sha1_hex(line + 8, sha1))
@@ -284,28 +314,40 @@ static int find_common(int fd[2], unsigned char *result_sha1,
314
}
315
die("expected shallow/unshallow, got %s", line);
316
}
317
+ } else if (!args.stateless_rpc)
318
+ send_request(fd[1], &req_buf);
319
+
320
+ if (!args.stateless_rpc) {
321
+ /* If we aren't using the stateless-rpc interface
322
+ * we don't need to retain the headers.
323
+ */
324
+ strbuf_setlen(&req_buf, 0);
325
+ state_len = 0;
326
}
327
328
flushes = 0;
329
retval = -1;
330
while ((sha1 = get_rev())) {
292
- packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
331
+ packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
332
if (args.verbose)
333
fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
334
in_vain++;
335
if (!(31 & ++count)) {
336
int ack;
337
299
- packet_flush(fd[1]);
338
+ packet_buf_flush(&req_buf);
339
+ send_request(fd[1], &req_buf);
340
+ strbuf_setlen(&req_buf, state_len);
341
flushes++;
342
343
/*
344
* We keep one window "ahead" of the other side, and
345
* will wait for an ACK only on the next one
346
*/
306
- if (count == 32)
347
+ if (!args.stateless_rpc && count == 32)
348
continue;
349
350
+ consume_shallow_list(fd[0]);
351
do {
352
ack = get_ack(fd[0], result_sha1);
353
if (args.verbose && ack)
@@ -322,6 +364,17 @@ static int find_common(int fd[2], unsigned char *result_sha1,
364
case ACK_continue: {
365
struct commit *commit =
366
lookup_commit(result_sha1);
367
+ if (args.stateless_rpc
368
+ && ack == ACK_common
369
+ && !(commit->object.flags & COMMON)) {
370
+ /* We need to replay the have for this object
371
+ * on the next RPC request so the peer knows
372
+ * it is in common with us.
373
+ */
374
+ const char *hex = sha1_to_hex(result_sha1);
375
+ packet_buf_write(&req_buf, "have %s\n", hex);
376
+ state_len = req_buf.len;
377
+ }
378
mark_common(commit, 0, 1);
379
retval = 0;
380
in_vain = 0;
@@ -339,7 +392,8 @@ static int find_common(int fd[2], unsigned char *result_sha1,
392
}
393
}
394
done:
342
- packet_write(fd[1], "done\n");
395
+ packet_buf_write(&req_buf, "done\n");
396
+ send_request(fd[1], &req_buf);
397
if (args.verbose)
398
fprintf(stderr, "done\n");
399
if (retval != 0) {
@@ -348,6 +402,7 @@ done:
402
}
403
strbuf_release(&req_buf);
404
405
+ consume_shallow_list(fd[0]);
406
while (flushes || multi_ack) {
407
int ack = get_ack(fd[0], result_sha1);
408
if (ack) {
@@ -672,6 +727,8 @@ static struct ref *do_fetch_pack(int fd[2],
727
*/
728
warning("no common commits");
729
730
+ if (args.stateless_rpc)
731
+ packet_flush(fd[1]);
732
if (get_pack(fd, pack_lockfile))
733
die("git fetch-pack: fetch failed.");
734
@@ -742,6 +799,8 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
799
struct ref *ref = NULL;
800
char *dest = NULL, **heads;
801
int fd[2];
802
+ char *pack_lockfile = NULL;
803
+ char **pack_lockfile_ptr = NULL;
804
struct child_process *conn;
805
806
nr_heads = 0;
@@ -791,6 +850,15 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
850
args.no_progress = 1;
851
continue;
852
}
853
+ if (!strcmp("--stateless-rpc", arg)) {
854
+ args.stateless_rpc = 1;
855
+ continue;
856
+ }
857
+ if (!strcmp("--lock-pack", arg)) {
858
+ args.lock_pack = 1;
859
+ pack_lockfile_ptr = &pack_lockfile;
860
+ continue;
861
+ }
862
usage(fetch_pack_usage);
863
}
864
dest = (char *)arg;
@@ -801,19 +869,27 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
869
if (!dest)
870
usage(fetch_pack_usage);
871
804
- conn = git_connect(fd, (char *)dest, args.uploadpack,
805
- args.verbose ? CONNECT_VERBOSE : 0);
806
- if (conn) {
807
- get_remote_heads(fd[0], &ref, 0, NULL, 0, NULL);
808
-
809
- ref = fetch_pack(&args, fd, conn, ref, dest, nr_heads, heads, NULL);
810
- close(fd[0]);
811
- close(fd[1]);
812
- if (finish_connect(conn))
813
- ref = NULL;
872
+ if (args.stateless_rpc) {
873
+ conn = NULL;
874
+ fd[0] = 0;
875
+ fd[1] = 1;
876
} else {
815
- ref = NULL;
877
+ conn = git_connect(fd, (char *)dest, args.uploadpack,
878
+ args.verbose ? CONNECT_VERBOSE : 0);
879
+ }
880
+
881
+ get_remote_heads(fd[0], &ref, 0, NULL, 0, NULL);
882
+
883
+ ref = fetch_pack(&args, fd, conn, ref, dest,
884
+ nr_heads, heads, pack_lockfile_ptr);
885
+ if (pack_lockfile) {
886
+ printf("lock %s\n", pack_lockfile);
887
+ fflush(stdout);
888
}
889
+ close(fd[0]);
890
+ close(fd[1]);
891
+ if (finish_connect(conn))
892
+ ref = NULL;
893
ret = !ref;
894
895
if (!ret && nr_heads) {
fetch-pack.h
+2
-1
@@ -13,7 +13,8 @@ struct fetch_pack_args
13
fetch_all:1,
14
verbose:1,
15
no_progress:1,
16
- include_tag:1;
16
+ include_tag:1,
17
+ stateless_rpc:1;
18
};
19
20
struct ref *fetch_pack(struct fetch_pack_args *args,
remote-curl.c
+65
-4
@@ -45,7 +45,7 @@ static int set_option(const char *name, const char *value)
45
options.progress = 0;
46
else
47
return -1;
48
- return 1 /* TODO implement later */;
48
+ return 0;
49
}
50
else if (!strcmp(name, "depth")) {
51
char *end;
@@ -53,7 +53,7 @@ static int set_option(const char *name, const char *value)
53
if (value == end || *end)
54
return -1;
55
options.depth = v;
56
- return 1 /* TODO implement later */;
56
+ return 0;
57
}
58
else if (!strcmp(name, "followtags")) {
59
if (!strcmp(value, "true"))
@@ -62,7 +62,7 @@ static int set_option(const char *name, const char *value)
62
options.followtags = 0;
63
else
64
return -1;
65
- return 1 /* TODO implement later */;
65
+ return 0;
66
}
67
else if (!strcmp(name, "dry-run")) {
68
if (!strcmp(value, "true"))
@@ -463,6 +463,8 @@ static int fetch_dumb(int nr_heads, struct ref **to_fetch)
463
char **targets = xmalloc(nr_heads * sizeof(char*));
464
int ret, i;
465
466
+ if (options.depth)
467
+ die("dumb http transport does not support --depth");
468
for (i = 0; i < nr_heads; i++)
469
targets[i] = xstrdup(sha1_to_hex(to_fetch[i]->old_sha1));
470
@@ -481,6 +483,65 @@ static int fetch_dumb(int nr_heads, struct ref **to_fetch)
483
return ret ? error("Fetch failed.") : 0;
484
}
485
486
+static int fetch_git(struct discovery *heads,
487
+ int nr_heads, struct ref **to_fetch)
488
+{
489
+ struct rpc_state rpc;
490
+ char *depth_arg = NULL;
491
+ const char **argv;
492
+ int argc = 0, i, err;
493
+
494
+ argv = xmalloc((15 + nr_heads) * sizeof(char*));
495
+ argv[argc++] = "fetch-pack";
496
+ argv[argc++] = "--stateless-rpc";
497
+ argv[argc++] = "--lock-pack";
498
+ if (options.followtags)
499
+ argv[argc++] = "--include-tag";
500
+ if (options.thin)
501
+ argv[argc++] = "--thin";
502
+ if (options.verbosity >= 3) {
503
+ argv[argc++] = "-v";
504
+ argv[argc++] = "-v";
505
+ }
506
+ if (!options.progress)
507
+ argv[argc++] = "--no-progress";
508
+ if (options.depth) {
509
+ struct strbuf buf = STRBUF_INIT;
510
+ strbuf_addf(&buf, "--depth=%lu", options.depth);
511
+ depth_arg = strbuf_detach(&buf, NULL);
512
+ argv[argc++] = depth_arg;
513
+ }
514
+ argv[argc++] = url;
515
+ for (i = 0; i < nr_heads; i++) {
516
+ struct ref *ref = to_fetch[i];
517
+ if (!ref->name || !*ref->name)
518
+ die("cannot fetch by sha1 over smart http");
519
+ argv[argc++] = ref->name;
520
+ }
521
+ argv[argc++] = NULL;
522
+
523
+ memset(&rpc, 0, sizeof(rpc));
524
+ rpc.service_name = "git-upload-pack",
525
+ rpc.argv = argv;
526
+
527
+ err = rpc_service(&rpc, heads);
528
+ if (rpc.result.len)
529
+ safe_write(1, rpc.result.buf, rpc.result.len);
530
+ strbuf_release(&rpc.result);
531
+ free(argv);
532
+ free(depth_arg);
533
+ return err;
534
+}
535
+
536
+static int fetch(int nr_heads, struct ref **to_fetch)
537
+{
538
+ struct discovery *d = discover_refs("git-upload-pack");
539
+ if (d->proto_git)
540
+ return fetch_git(d, nr_heads, to_fetch);
541
+ else
542
+ return fetch_dumb(nr_heads, to_fetch);
543
+}
544
+
545
static void parse_fetch(struct strbuf *buf)
546
{
547
struct ref **to_fetch = NULL;
@@ -523,7 +584,7 @@ static void parse_fetch(struct strbuf *buf)
584
break;
585
} while (1);
586
526
- if (fetch_dumb(nr_heads, to_fetch))
587
+ if (fetch(nr_heads, to_fetch))
588
exit(128); /* error already reported */
589
free_refs(list_head);
590
free(to_fetch);