send-pack: take refspecs over stdin
Pushing a large number of refs works over most transports, because we implement send-pack as an internal function. However, it can sometimes fail when pushing over http, because we have to spawn "git send-pack --stateless-rpc" to do the heavy lifting, and we pass each refspec on the command line. This can cause us to overflow the OS limits on the size of the command line for a large push. We can solve this by giving send-pack a --stdin option and using it from remote-curl. We already dealt with this on the fetch-pack side in 078b895 (fetch-pack: new --stdin option to read refs from stdin, 2012-04-02). The stdin option (and in particular, its use of packet-lines for stateless-rpc input) is modeled after that solution. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Jeff King committed
Aug 21, 2014 at 08:21 UTC
26be19ba8d8d2e7e3e288b395e7156d5b7af5140
5 files changed
+153
-2
Documentation/git-send-pack.txt
+12
-1
@@ -35,6 +35,16 @@ OPTIONS
35
Instead of explicitly specifying which refs to update,
36
update all heads that locally exist.
37
38
+--stdin::
39
+ Take the list of refs from stdin, one per line. If there
40
+ are refs specified on the command line in addition to this
41
+ option, then the refs from stdin are processed after those
42
+ on the command line.
43
++
44
+If '--stateless-rpc' is specified together with this option then
45
+the list of refs must be in packet format (pkt-line). Each ref must
46
+be in a separate packet, and the list must end with a flush packet.
47
+
48
--dry-run::
49
Do everything except actually send the updates.
50
@@ -77,7 +87,8 @@ this flag.
87
Without '--all' and without any '<ref>', the heads that exist
88
both on the local side and on the remote side are updated.
89
80
-When one or more '<ref>' are specified explicitly, it can be either a
90
+When one or more '<ref>' are specified explicitly (whether on the
91
+command line or via `--stdin`), it can be either a
92
single pattern, or a pair of such pattern separated by a colon
93
":" (this means that a ref name cannot have a colon in it). A
94
single pattern '<name>' is just a shorthand for '<name>:<name>'.
builtin/send-pack.c
+27
@@ -110,6 +110,7 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix)
110
int flags;
111
unsigned int reject_reasons;
112
int progress = -1;
113
+ int from_stdin = 0;
114
struct push_cas_option cas = {0};
115
116
argv++;
@@ -169,6 +170,10 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix)
170
args.stateless_rpc = 1;
171
continue;
172
}
173
+ if (!strcmp(arg, "--stdin")) {
174
+ from_stdin = 1;
175
+ continue;
176
+ }
177
if (!strcmp(arg, "--helper-status")) {
178
helper_status = 1;
179
continue;
@@ -201,6 +206,28 @@ int cmd_send_pack(int argc, const char **argv, const char *prefix)
206
}
207
if (!dest)
208
usage(send_pack_usage);
209
+
210
+ if (from_stdin) {
211
+ struct argv_array all_refspecs = ARGV_ARRAY_INIT;
212
+
213
+ for (i = 0; i < nr_refspecs; i++)
214
+ argv_array_push(&all_refspecs, refspecs[i]);
215
+
216
+ if (args.stateless_rpc) {
217
+ const char *buf;
218
+ while ((buf = packet_read_line(0, NULL)))
219
+ argv_array_push(&all_refspecs, buf);
220
+ } else {
221
+ struct strbuf line = STRBUF_INIT;
222
+ while (strbuf_getline(&line, stdin, '\n') != EOF)
223
+ argv_array_push(&all_refspecs, line.buf);
224
+ strbuf_release(&line);
225
+ }
226
+
227
+ refspecs = all_refspecs.argv;
228
+ nr_refspecs = all_refspecs.argc;
229
+ }
230
+
231
/*
232
* --all and --mirror are incompatible; neither makes sense
233
* with any refspecs.
remote-curl.c
+7
-1
@@ -863,6 +863,7 @@ static int push_git(struct discovery *heads, int nr_spec, char **specs)
863
int i, err;
864
struct argv_array args;
865
struct string_list_item *cas_option;
866
+ struct strbuf preamble = STRBUF_INIT;
867
868
argv_array_init(&args);
869
argv_array_pushl(&args, "send-pack", "--stateless-rpc", "--helper-status",
@@ -880,17 +881,22 @@ static int push_git(struct discovery *heads, int nr_spec, char **specs)
881
for_each_string_list_item(cas_option, &cas_options)
882
argv_array_push(&args, cas_option->string);
883
argv_array_push(&args, url.buf);
884
+
885
+ argv_array_push(&args, "--stdin");
886
for (i = 0; i < nr_spec; i++)
884
- argv_array_push(&args, specs[i]);
887
+ packet_buf_write(&preamble, "%s\n", specs[i]);
888
+ packet_buf_flush(&preamble);
889
890
memset(&rpc, 0, sizeof(rpc));
891
rpc.service_name = "git-receive-pack",
892
rpc.argv = args.argv;
893
+ rpc.stdin_preamble = &preamble;
894
895
err = rpc_service(&rpc, heads);
896
if (rpc.result.len)
897
write_or_die(1, rpc.result.buf, rpc.result.len);
898
strbuf_release(&rpc.result);
899
+ strbuf_release(&preamble);
900
argv_array_clear(&args);
901
return err;
902
}
t/t5408-send-pack-stdin.sh
new
+92
@@ -0,0 +1,92 @@
1
+#!/bin/sh
2
+
3
+test_description='send-pack --stdin tests'
4
+. ./test-lib.sh
5
+
6
+create_ref () {
7
+ tree=$(git write-tree) &&
8
+ test_tick &&
9
+ commit=$(echo "$1" | git commit-tree $tree) &&
10
+ git update-ref "$1" $commit
11
+}
12
+
13
+clear_remote () {
14
+ rm -rf remote.git &&
15
+ git init --bare remote.git
16
+}
17
+
18
+verify_push () {
19
+ git rev-parse "$1" >expect &&
20
+ git --git-dir=remote.git rev-parse "${2:-$1}" >actual &&
21
+ test_cmp expect actual
22
+}
23
+
24
+test_expect_success 'setup refs' '
25
+ cat >refs <<-\EOF &&
26
+ refs/heads/A
27
+ refs/heads/C
28
+ refs/tags/D
29
+ refs/heads/B
30
+ refs/tags/E
31
+ EOF
32
+ for i in $(cat refs); do
33
+ create_ref $i || return 1
34
+ done
35
+'
36
+
37
+# sanity check our setup
38
+test_expect_success 'refs on cmdline' '
39
+ clear_remote &&
40
+ git send-pack remote.git $(cat refs) &&
41
+ for i in $(cat refs); do
42
+ verify_push $i || return 1
43
+ done
44
+'
45
+
46
+test_expect_success 'refs over stdin' '
47
+ clear_remote &&
48
+ git send-pack remote.git --stdin <refs &&
49
+ for i in $(cat refs); do
50
+ verify_push $i || return 1
51
+ done
52
+'
53
+
54
+test_expect_success 'stdin lines are full refspecs' '
55
+ clear_remote &&
56
+ echo "A:other" >input &&
57
+ git send-pack remote.git --stdin <input &&
58
+ verify_push refs/heads/A refs/heads/other
59
+'
60
+
61
+test_expect_success 'stdin mixed with cmdline' '
62
+ clear_remote &&
63
+ echo A >input &&
64
+ git send-pack remote.git --stdin B <input &&
65
+ verify_push A &&
66
+ verify_push B
67
+'
68
+
69
+test_expect_success 'cmdline refs written in order' '
70
+ clear_remote &&
71
+ test_must_fail git send-pack remote.git A:foo B:foo &&
72
+ verify_push A foo
73
+'
74
+
75
+test_expect_success '--stdin refs come after cmdline' '
76
+ clear_remote &&
77
+ echo A:foo >input &&
78
+ test_must_fail git send-pack remote.git --stdin B:foo <input &&
79
+ verify_push B foo
80
+'
81
+
82
+test_expect_success 'refspecs and --mirror do not mix (cmdline)' '
83
+ clear_remote &&
84
+ test_must_fail git send-pack remote.git --mirror $(cat refs)
85
+'
86
+
87
+test_expect_success 'refspecs and --mirror do not mix (stdin)' '
88
+ clear_remote &&
89
+ test_must_fail git send-pack remote.git --mirror --stdin <refs
90
+'
91
+
92
+test_done
t/t5541-http-push-smart.sh
+15
@@ -323,5 +323,20 @@ test_expect_success 'push into half-auth-complete requires password' '
323
test_cmp expect actual
324
'
325
326
+run_with_limited_cmdline () {
327
+ (ulimit -s 128 && "$@")
328
+}
329
+
330
+test_lazy_prereq CMDLINE_LIMIT 'run_with_limited_cmdline true'
331
+
332
+test_expect_success CMDLINE_LIMIT 'push 2000 tags over http' '
333
+ sha1=$(git rev-parse HEAD) &&
334
+ test_seq 2000 |
335
+ sort |
336
+ sed "s|.*|$sha1 refs/tags/really-long-tag-name-&|" \
337
+ >.git/packed-refs &&
338
+ run_with_limited_cmdline git push --mirror
339
+'
340
+
341
stop_httpd
342
test_done