@matheusbsbs / matheus-bagliano / commits / de1a2fdd38

Fork 0

Smart push over HTTP: client side

The git-remote-curl backend detects if the remote server supports the git-receive-pack service, and if so, runs git-send-pack in a pipe to dump the command and pack data as a single POST request. The advertisements from the server that were obtained during the discovery are passed into git-send-pack before the POST request starts. This permits git-send-pack to operate largely unmodified. For smaller packs (those under 1 MiB) a HTTP/1.0 POST with a Content-Length is used, permitting interaction with any server. The 1 MiB limit is arbitrary, but is sufficent to fit most deltas created by human authors against text sources with the occasional small binary file (e.g. few KiB icon image). The configuration option http.postBuffer can be used to increase (or shink) this buffer if the default is not sufficient. For larger packs which cannot be spooled entirely into the helper's memory space (due to http.postBuffer being too small), the POST request requires HTTP/1.1 and sets "Transfer-Encoding: chunked". This permits the client to upload an unknown amount of data in one HTTP transaction without needing to pregenerate the entire pack file locally. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> CC: Daniel Barkalow <barkalow@iabervon.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Shawn O. Pearce committed Oct 30, 2009 at 17:47 UTC de1a2fdd38b138c4e4fed6412783dcb74d63d2da

8 files changed +374 -15

Documentation/config.txt

@@ -1089,6 +1089,14 @@ http.maxRequests::

1089 How many HTTP requests to launch in parallel. Can be overridden

1090 by the 'GIT_HTTP_MAX_REQUESTS' environment variable. Default is 5.

1091

1092 +http.postBuffer::

1093 + Maximum size in bytes of the buffer used by smart HTTP

1094 + transports when POSTing data to the remote system.

1095 + For requests larger than this buffer size, HTTP/1.1 and

1096 + Transfer-Encoding: chunked is used to avoid creating a

1097 + massive pack file locally. Default is 1 MiB, which is

1098 + sufficient for most requests.

1099 +

1100 http.lowSpeedLimit, http.lowSpeedTime::

1101 If the HTTP transfer speed is less than 'http.lowSpeedLimit'

1102 for longer than 'http.lowSpeedTime' seconds, the transfer is aborted.

builtin-send-pack.c

+110 -6

@@ -2,9 +2,11 @@

2 #include "commit.h"

3 #include "refs.h"

4 #include "pkt-line.h"

5 +#include "sideband.h"

6 #include "run-command.h"

7 #include "remote.h"

8 #include "send-pack.h"

9 +#include "quote.h"

11 static const char send_pack_usage[] =

12 "git send-pack [--all | --mirror] [--dry-run] [--force] [--receive-pack=<git-receive-pack>] [--verbose] [--thin] [<host>:]<directory> [<ref>...]\n"

@@ -59,7 +61,7 @@ static int pack_objects(int fd, struct ref *refs, struct extra_have_objects *ext

61 memset(&po, 0, sizeof(po));

62 po.argv = argv;

63 po.in = -1;

62 - po.out = fd;

64 + po.out = args->stateless_rpc ? -1 : fd;

65 po.git_cmd = 1;

66 if (start_command(&po))

67 die_errno("git pack-objects failed");

@@ -83,6 +85,20 @@ static int pack_objects(int fd, struct ref *refs, struct extra_have_objects *ext

85 }

87 close(po.in);

88 +

89 + if (args->stateless_rpc) {

90 + char *buf = xmalloc(LARGE_PACKET_MAX);

91 + while (1) {

92 + ssize_t n = xread(po.out, buf, LARGE_PACKET_MAX);

93 + if (n <= 0)

94 + break;

95 + send_sideband(fd, -1, buf, n, LARGE_PACKET_MAX);

96 + }

97 + free(buf);

98 + close(po.out);

99 + po.out = -1;

100 + }

101 +

102 if (finish_command(&po))

103 return error("pack-objects died with strange error");

104 return 0;

@@ -303,6 +319,59 @@ static int refs_pushed(struct ref *ref)

319 return 0;

320 }

321

322 +static void print_helper_status(struct ref *ref)

323 +{

324 + struct strbuf buf = STRBUF_INIT;

325 +

326 + for (; ref; ref = ref->next) {

327 + const char *msg = NULL;

328 + const char *res;

329 +

330 + switch(ref->status) {

331 + case REF_STATUS_NONE:

332 + res = "error";

333 + msg = "no match";

334 + break;

335 +

336 + case REF_STATUS_OK:

337 + res = "ok";

338 + break;

339 +

340 + case REF_STATUS_UPTODATE:

341 + res = "ok";

342 + msg = "up to date";

343 + break;

344 +

345 + case REF_STATUS_REJECT_NONFASTFORWARD:

346 + res = "error";

347 + msg = "non-fast forward";

348 + break;

349 +

350 + case REF_STATUS_REJECT_NODELETE:

351 + case REF_STATUS_REMOTE_REJECT:

352 + res = "error";

353 + break;

354 +

355 + case REF_STATUS_EXPECTING_REPORT:

356 + default:

357 + continue;

358 + }

359 +

360 + strbuf_reset(&buf);

361 + strbuf_addf(&buf, "%s %s", res, ref->name);

362 + if (ref->remote_status)

363 + msg = ref->remote_status;

364 + if (msg) {

365 + strbuf_addch(&buf, ' ');

366 + quote_two_c_style(&buf, "", msg, 0);

367 + }

368 + strbuf_addch(&buf, '\n');

369 +

370 + safe_write(1, buf.buf, buf.len);

371 + }

372 + strbuf_release(&buf);

373 +}

374 +

375 int send_pack(struct send_pack_args *args,

376 int fd[], struct child_process *conn,

377 struct ref *remote_refs,

@@ -310,6 +379,7 @@ int send_pack(struct send_pack_args *args,

379 {

380 int in = fd[0];

381 int out = fd[1];

382 + struct strbuf req_buf = STRBUF_INIT;

383 struct ref *ref;

384 int new_refs;

385 int ask_for_status_report = 0;

@@ -391,14 +461,14 @@ int send_pack(struct send_pack_args *args,

461 char *new_hex = sha1_to_hex(ref->new_sha1);

462

463 if (ask_for_status_report) {

394 - packet_write(out, "%s %s %s%c%s",

464 + packet_buf_write(&req_buf, "%s %s %s%c%s",

465 old_hex, new_hex, ref->name, 0,

466 "report-status");

467 ask_for_status_report = 0;

468 expect_status_report = 1;

469 }

470 else

401 - packet_write(out, "%s %s %s",

471 + packet_buf_write(&req_buf, "%s %s %s",

472 old_hex, new_hex, ref->name);

473 }

474 ref->status = expect_status_report ?

@@ -406,7 +476,17 @@ int send_pack(struct send_pack_args *args,

476 REF_STATUS_OK;

477 }

478

409 - packet_flush(out);

479 + if (args->stateless_rpc) {

480 + if (!args->dry_run) {

481 + packet_buf_flush(&req_buf);

482 + send_sideband(out, -1, req_buf.buf, req_buf.len, LARGE_PACKET_MAX);

483 + }

484 + } else {

485 + safe_write(out, req_buf.buf, req_buf.len);

486 + packet_flush(out);

487 + }

488 + strbuf_release(&req_buf);

489 +

490 if (new_refs && !args->dry_run) {

491 if (pack_objects(out, remote_refs, extra_have, args) < 0) {

492 for (ref = remote_refs; ref; ref = ref->next)

@@ -414,11 +494,15 @@ int send_pack(struct send_pack_args *args,

494 return -1;

495 }

496 }

497 + if (args->stateless_rpc && !args->dry_run)

498 + packet_flush(out);

499

500 if (expect_status_report)

501 ret = receive_status(in, remote_refs);

502 else

503 ret = 0;

504 + if (args->stateless_rpc)

505 + packet_flush(out);

506

507 if (ret < 0)

508 return ret;

@@ -478,6 +562,7 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix)

562 struct extra_have_objects extra_have;

563 struct ref *remote_refs, *local_refs;

564 int ret;

565 + int helper_status = 0;

566 int send_all = 0;

567 const char *receivepack = "git-receive-pack";

568 int flags;

@@ -523,6 +608,14 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix)

608 args.use_thin_pack = 1;

609 continue;

610 }

611 + if (!strcmp(arg, "--stateless-rpc")) {

612 + args.stateless_rpc = 1;

613 + continue;

614 + }

615 + if (!strcmp(arg, "--helper-status")) {

616 + helper_status = 1;

617 + continue;

618 + }

619 usage(send_pack_usage);

620 }

621 if (!dest) {

@@ -551,7 +644,14 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix)

644 }

645 }

646

554 - conn = git_connect(fd, dest, receivepack, args.verbose ? CONNECT_VERBOSE : 0);

647 + if (args.stateless_rpc) {

648 + conn = NULL;

649 + fd[0] = 0;

650 + fd[1] = 1;

651 + } else {

652 + conn = git_connect(fd, dest, receivepack,

653 + args.verbose ? CONNECT_VERBOSE : 0);

654 + }

655

656 memset(&extra_have, 0, sizeof(extra_have));

657

@@ -575,12 +675,16 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix)

675

676 ret = send_pack(&args, fd, conn, remote_refs, &extra_have);

677

678 + if (helper_status)

679 + print_helper_status(remote_refs);

680 +

681 close(fd[1]);

682 close(fd[0]);

683

684 ret |= finish_connect(conn);

685

583 - print_push_status(dest, remote_refs);

686 + if (!helper_status)

687 + print_push_status(dest, remote_refs);

688

689 if (!args.dry_run && remote) {

690 struct ref *ref;

http.c

+10 -3

@@ -1,9 +1,11 @@

1 #include "http.h"

2 #include "pack.h"

3 +#include "sideband.h"

5 int data_received;

6 int active_requests;

7 int http_is_verbose;

8 +size_t http_post_buffer = 16 * LARGE_PACKET_MAX;

10 #ifdef USE_CURL_MULTI

11 static int max_requests = -1;

@@ -97,8 +99,6 @@ size_t fwrite_null(const void *ptr, size_t eltsize, size_t nmemb, void *strbuf)

99 return eltsize * nmemb;

100 }

101

100 -static void finish_active_slot(struct active_request_slot *slot);

101 -

102 #ifdef USE_CURL_MULTI

103 static void process_curl_messages(void)

104 {

@@ -174,6 +174,13 @@ static int http_options(const char *var, const char *value, void *cb)

174 if (!strcmp("http.proxy", var))

175 return git_config_string(&curl_http_proxy, var, value);

176

177 + if (!strcmp("http.postbuffer", var)) {

178 + http_post_buffer = git_config_int(var, value);

179 + if (http_post_buffer < LARGE_PACKET_MAX)

180 + http_post_buffer = LARGE_PACKET_MAX;

181 + return 0;

182 + }

183 +

184 /* Fall back on the default ones */

185 return git_default_config(var, value, cb);

186 }

@@ -638,7 +645,7 @@ void release_active_slot(struct active_request_slot *slot)

645 #endif

646 }

647

641 -static void finish_active_slot(struct active_request_slot *slot)

648 +void finish_active_slot(struct active_request_slot *slot)

649 {

650 closedown_active_slot(slot);

651 curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE, &slot->http_code);

http.h

@@ -79,6 +79,7 @@ extern curlioerr ioctl_buffer(CURL *handle, int cmd, void *clientp);

79 extern struct active_request_slot *get_active_slot(void);

80 extern int start_active_slot(struct active_request_slot *slot);

81 extern void run_active_slot(struct active_request_slot *slot);

82 +extern void finish_active_slot(struct active_request_slot *slot);

83 extern void finish_all_active_slots(void);

84 extern void release_active_slot(struct active_request_slot *slot);

@@ -94,6 +95,7 @@ extern void http_cleanup(void);

95 extern int data_received;

96 extern int active_requests;

97 extern int http_is_verbose;

98 +extern size_t http_post_buffer;

100 extern char curl_errorstr[CURL_ERROR_SIZE];

101

remote-curl.c

+233 -2

@@ -6,6 +6,7 @@

6 #include "exec_cmd.h"

7 #include "run-command.h"

8 #include "pkt-line.h"

9 +#include "sideband.h"

11 static struct remote *remote;

12 static const char *url;

@@ -16,7 +17,8 @@ struct options {

17 unsigned long depth;

18 unsigned progress : 1,

19 followtags : 1,

19 - dry_run : 1;

20 + dry_run : 1,

21 + thin : 1;

22 };

23 static struct options options;

@@ -274,6 +276,188 @@ static void output_refs(struct ref *refs)

276 free_refs(refs);

277 }

278

279 +struct rpc_state {

280 + const char *service_name;

281 + const char **argv;

282 + char *service_url;

283 + char *hdr_content_type;

284 + char *hdr_accept;

285 + char *buf;

286 + size_t alloc;

287 + size_t len;

288 + size_t pos;

289 + int in;

290 + int out;

291 + struct strbuf result;

292 +};

293 +

294 +static size_t rpc_out(void *ptr, size_t eltsize,

295 + size_t nmemb, void *buffer_)

296 +{

297 + size_t max = eltsize * nmemb;

298 + struct rpc_state *rpc = buffer_;

299 + size_t avail = rpc->len - rpc->pos;

300 +

301 + if (!avail) {

302 + avail = packet_read_line(rpc->out, rpc->buf, rpc->alloc);

303 + if (!avail)

304 + return 0;

305 + rpc->pos = 0;

306 + rpc->len = avail;

307 + }

308 +

309 + if (max < avail);

310 + avail = max;

311 + memcpy(ptr, rpc->buf + rpc->pos, avail);

312 + rpc->pos += avail;

313 + return avail;

314 +}

315 +

316 +static size_t rpc_in(const void *ptr, size_t eltsize,

317 + size_t nmemb, void *buffer_)

318 +{

319 + size_t size = eltsize * nmemb;

320 + struct rpc_state *rpc = buffer_;

321 + write_or_die(rpc->in, ptr, size);

322 + return size;

323 +}

324 +

325 +static int post_rpc(struct rpc_state *rpc)

326 +{

327 + struct active_request_slot *slot;

328 + struct slot_results results;

329 + struct curl_slist *headers = NULL;

330 + int err = 0, large_request = 0;

331 +

332 + /* Try to load the entire request, if we can fit it into the

333 + * allocated buffer space we can use HTTP/1.0 and avoid the

334 + * chunked encoding mess.

335 + */

336 + while (1) {

337 + size_t left = rpc->alloc - rpc->len;

338 + char *buf = rpc->buf + rpc->len;

339 + int n;

340 +

341 + if (left < LARGE_PACKET_MAX) {

342 + large_request = 1;

343 + break;

344 + }

345 +

346 + n = packet_read_line(rpc->out, buf, left);

347 + if (!n)

348 + break;

349 + rpc->len += n;

350 + }

351 +

352 + slot = get_active_slot();

353 + slot->results = &results;

354 +

355 + curl_easy_setopt(slot->curl, CURLOPT_POST, 1);

356 + curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);

357 + curl_easy_setopt(slot->curl, CURLOPT_URL, rpc->service_url);

358 +

359 + headers = curl_slist_append(headers, rpc->hdr_content_type);

360 + headers = curl_slist_append(headers, rpc->hdr_accept);

361 +

362 + if (large_request) {

363 + /* The request body is large and the size cannot be predicted.

364 + * We must use chunked encoding to send it.

365 + */

366 + headers = curl_slist_append(headers, "Expect: 100-continue");

367 + headers = curl_slist_append(headers, "Transfer-Encoding: chunked");

368 + curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, rpc_out);

369 + curl_easy_setopt(slot->curl, CURLOPT_INFILE, rpc);

370 + if (options.verbosity > 1) {

371 + fprintf(stderr, "POST %s (chunked)\n", rpc->service_name);

372 + fflush(stderr);

373 + }

374 +

375 + } else {

376 + /* We know the complete request size in advance, use the

377 + * more normal Content-Length approach.

378 + */

379 + curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, rpc->buf);

380 + curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, rpc->len);

381 + if (options.verbosity > 1) {

382 + fprintf(stderr, "POST %s (%lu bytes)\n",

383 + rpc->service_name, (unsigned long)rpc->len);

384 + fflush(stderr);

385 + }

386 + }

387 +

388 + curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, headers);

389 + curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, rpc_in);

390 + curl_easy_setopt(slot->curl, CURLOPT_FILE, rpc);

391 +

392 + slot->curl_result = curl_easy_perform(slot->curl);

393 + finish_active_slot(slot);

394 +

395 + if (results.curl_result != CURLE_OK) {

396 + err |= error("RPC failed; result=%d, HTTP code = %ld",

397 + results.curl_result, results.http_code);

398 + }

399 +

400 + curl_slist_free_all(headers);

401 + return err;

402 +}

403 +

404 +static int rpc_service(struct rpc_state *rpc, struct discovery *heads)

405 +{

406 + const char *svc = rpc->service_name;

407 + struct strbuf buf = STRBUF_INIT;

408 + struct child_process client;

409 + int err = 0;

410 +

411 + init_walker();

412 + memset(&client, 0, sizeof(client));

413 + client.in = -1;

414 + client.out = -1;

415 + client.git_cmd = 1;

416 + client.argv = rpc->argv;

417 + if (start_command(&client))

418 + exit(1);

419 + if (heads)

420 + write_or_die(client.in, heads->buf, heads->len);

421 +

422 + rpc->alloc = http_post_buffer;

423 + rpc->buf = xmalloc(rpc->alloc);

424 + rpc->in = client.in;

425 + rpc->out = client.out;

426 + strbuf_init(&rpc->result, 0);

427 +

428 + strbuf_addf(&buf, "%s/%s", url, svc);

429 + rpc->service_url = strbuf_detach(&buf, NULL);

430 +

431 + strbuf_addf(&buf, "Content-Type: application/x-%s-request", svc);

432 + rpc->hdr_content_type = strbuf_detach(&buf, NULL);

433 +

434 + strbuf_addf(&buf, "Accept: application/x-%s-response", svc);

435 + rpc->hdr_accept = strbuf_detach(&buf, NULL);

436 +

437 + while (!err) {

438 + int n = packet_read_line(rpc->out, rpc->buf, rpc->alloc);

439 + if (!n)

440 + break;

441 + rpc->pos = 0;

442 + rpc->len = n;

443 + err |= post_rpc(rpc);

444 + }

445 + strbuf_read(&rpc->result, client.out, 0);

446 +

447 + close(client.in);

448 + close(client.out);

449 + client.in = -1;

450 + client.out = -1;

451 +

452 + err |= finish_command(&client);

453 + free(rpc->service_url);

454 + free(rpc->hdr_content_type);

455 + free(rpc->hdr_accept);

456 + free(rpc->buf);

457 + strbuf_release(&buf);

458 + return err;

459 +}

460 +

461 static int fetch_dumb(int nr_heads, struct ref **to_fetch)

462 {

463 char **targets = xmalloc(nr_heads * sizeof(char*));

@@ -371,6 +555,52 @@ static int push_dav(int nr_spec, char **specs)

555 return 0;

556 }

557

558 +static int push_git(struct discovery *heads, int nr_spec, char **specs)

559 +{

560 + struct rpc_state rpc;

561 + const char **argv;

562 + int argc = 0, i, err;

563 +

564 + argv = xmalloc((10 + nr_spec) * sizeof(char*));

565 + argv[argc++] = "send-pack";

566 + argv[argc++] = "--stateless-rpc";

567 + argv[argc++] = "--helper-status";

568 + if (options.thin)

569 + argv[argc++] = "--thin";

570 + if (options.dry_run)

571 + argv[argc++] = "--dry-run";

572 + if (options.verbosity > 1)

573 + argv[argc++] = "--verbose";

574 + argv[argc++] = url;

575 + for (i = 0; i < nr_spec; i++)

576 + argv[argc++] = specs[i];

577 + argv[argc++] = NULL;

578 +

579 + memset(&rpc, 0, sizeof(rpc));

580 + rpc.service_name = "git-receive-pack",

581 + rpc.argv = argv;

582 +

583 + err = rpc_service(&rpc, heads);

584 + if (rpc.result.len)

585 + safe_write(1, rpc.result.buf, rpc.result.len);

586 + strbuf_release(&rpc.result);

587 + free(argv);

588 + return err;

589 +}

590 +

591 +static int push(int nr_spec, char **specs)

592 +{

593 + struct discovery *heads = discover_refs("git-receive-pack");

594 + int ret;

595 +

596 + if (heads->proto_git)

597 + ret = push_git(heads, nr_spec, specs);

598 + else

599 + ret = push_dav(nr_spec, specs);

600 + free_discovery(heads);

601 + return ret;

602 +}

603 +

604 static void parse_push(struct strbuf *buf)

605 {

606 char **specs = NULL;

@@ -391,7 +621,7 @@ static void parse_push(struct strbuf *buf)

621 break;

622 } while (1);

623

394 - if (push_dav(nr_spec, specs))

624 + if (push(nr_spec, specs))

625 exit(128); /* error already reported */

626 for (i = 0; i < nr_spec; i++)

627 free(specs[i]);

@@ -414,6 +644,7 @@ int main(int argc, const char **argv)

644

645 options.verbosity = 1;

646 options.progress = !!isatty(2);

647 + options.thin = 1;

648

649 remote = remote_get(argv[1]);

650

send-pack.h

+2 -1

@@ -8,7 +8,8 @@ struct send_pack_args {

8 force_update:1,

9 use_thin_pack:1,

10 use_ofs_delta:1,

11 - dry_run:1;

11 + dry_run:1,

12 + stateless_rpc:1;

13 };

15 int send_pack(struct send_pack_args *args,

sideband.c

+8 -3

@@ -135,9 +135,14 @@ ssize_t send_sideband(int fd, int band, const char *data, ssize_t sz, int packet

135 n = sz;

136 if (packet_max - 5 < n)

137 n = packet_max - 5;

138 - sprintf(hdr, "%04x", n + 5);

139 - hdr[4] = band;

140 - safe_write(fd, hdr, 5);

138 + if (0 <= band) {

139 + sprintf(hdr, "%04x", n + 5);

140 + hdr[4] = band;

141 + safe_write(fd, hdr, 5);

142 + } else {

143 + sprintf(hdr, "%04x", n + 4);

144 + safe_write(fd, hdr, 4);

145 + }

146 safe_write(fd, p, n);

147 p += n;

148 sz -= n;

transport.c

@@ -731,6 +731,7 @@ static int git_transport_push(struct transport *transport, struct ref *remote_re

731 NULL);

732 }

733

734 + memset(&args, 0, sizeof(args));

735 args.send_mirror = !!(flags & TRANSPORT_PUSH_MIRROR);

736 args.force_update = !!(flags & TRANSPORT_PUSH_FORCE);

737 args.use_thin_pack = data->thin;