remote-helpers: Fetch more than one ref in a batch
Some network protocols (e.g. native git://) are able to fetch more than one ref at a time and reduce the overall transfer cost by combining the requests into a single exchange. Instead of feeding each fetch request one at a time to the helper, feed all of them at once so the helper can decide whether or not it should batch them. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> CC: Daniel Barkalow <barkalow@iabervon.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Shawn O. Pearce committed
Oct 30, 2009 at 17:47 UTC
292ce46b60e2c12450c5c21044acf9c41bd837df
3 files changed
+115
-26
Documentation/git-remote-helpers.txt
+10
-4
@@ -36,10 +36,16 @@ Commands are given by the caller on the helper's standard input, one per line.
36
complete list, outputs a blank line.
37
38
'fetch' <sha1> <name>::
39
- Fetches the given object, writing the necessary objects to the
40
- database. Outputs a blank line when the fetch is
41
- complete. Only objects which were reported in the ref list
42
- with a sha1 may be fetched this way.
39
+ Fetches the given object, writing the necessary objects
40
+ to the database. Fetch commands are sent in a batch, one
41
+ per line, and the batch is terminated with a blank line.
42
+ Outputs a single blank line when all fetch commands in the
43
+ same batch are complete. Only objects which were reported
44
+ in the ref list with a sha1 may be fetched this way.
45
++
46
+Optionally may output a 'lock <file>' line indicating a file under
47
+GIT_DIR/objects/pack which is keeping a pack until refs can be
48
+suitably updated.
49
+
50
Supported if the helper has the "fetch" capability.
51
remote-curl.c
+77
-11
@@ -87,6 +87,81 @@ static struct ref *get_refs(void)
87
return refs;
88
}
89
90
+static int fetch_dumb(int nr_heads, struct ref **to_fetch)
91
+{
92
+ char **targets = xmalloc(nr_heads * sizeof(char*));
93
+ int ret, i;
94
+
95
+ for (i = 0; i < nr_heads; i++)
96
+ targets[i] = xstrdup(sha1_to_hex(to_fetch[i]->old_sha1));
97
+
98
+ init_walker();
99
+ walker->get_all = 1;
100
+ walker->get_tree = 1;
101
+ walker->get_history = 1;
102
+ walker->get_verbosely = 0;
103
+ walker->get_recover = 0;
104
+ ret = walker_fetch(walker, nr_heads, targets, NULL, NULL);
105
+
106
+ for (i = 0; i < nr_heads; i++)
107
+ free(targets[i]);
108
+ free(targets);
109
+
110
+ return ret ? error("Fetch failed.") : 0;
111
+}
112
+
113
+static void parse_fetch(struct strbuf *buf)
114
+{
115
+ struct ref **to_fetch = NULL;
116
+ struct ref *list_head = NULL;
117
+ struct ref **list = &list_head;
118
+ int alloc_heads = 0, nr_heads = 0;
119
+
120
+ do {
121
+ if (!prefixcmp(buf->buf, "fetch ")) {
122
+ char *p = buf->buf + strlen("fetch ");
123
+ char *name;
124
+ struct ref *ref;
125
+ unsigned char old_sha1[20];
126
+
127
+ if (strlen(p) < 40 || get_sha1_hex(p, old_sha1))
128
+ die("protocol error: expected sha/ref, got %s'", p);
129
+ if (p[40] == ' ')
130
+ name = p + 41;
131
+ else if (!p[40])
132
+ name = "";
133
+ else
134
+ die("protocol error: expected sha/ref, got %s'", p);
135
+
136
+ ref = alloc_ref(name);
137
+ hashcpy(ref->old_sha1, old_sha1);
138
+
139
+ *list = ref;
140
+ list = &ref->next;
141
+
142
+ ALLOC_GROW(to_fetch, nr_heads + 1, alloc_heads);
143
+ to_fetch[nr_heads++] = ref;
144
+ }
145
+ else
146
+ die("http transport does not support %s", buf->buf);
147
+
148
+ strbuf_reset(buf);
149
+ if (strbuf_getline(buf, stdin, '\n') == EOF)
150
+ return;
151
+ if (!*buf->buf)
152
+ break;
153
+ } while (1);
154
+
155
+ if (fetch_dumb(nr_heads, to_fetch))
156
+ exit(128); /* error already reported */
157
+ free_refs(list_head);
158
+ free(to_fetch);
159
+
160
+ printf("\n");
161
+ fflush(stdout);
162
+ strbuf_reset(buf);
163
+}
164
+
165
int main(int argc, const char **argv)
166
{
167
struct strbuf buf = STRBUF_INIT;
@@ -110,17 +185,8 @@ int main(int argc, const char **argv)
185
if (strbuf_getline(&buf, stdin, '\n') == EOF)
186
break;
187
if (!prefixcmp(buf.buf, "fetch ")) {
113
- char *obj = buf.buf + strlen("fetch ");
114
- init_walker();
115
- walker->get_all = 1;
116
- walker->get_tree = 1;
117
- walker->get_history = 1;
118
- walker->get_verbosely = 0;
119
- walker->get_recover = 0;
120
- if (walker_fetch(walker, 1, &obj, NULL, NULL))
121
- die("Fetch failed.");
122
- printf("\n");
123
- fflush(stdout);
188
+ parse_fetch(&buf);
189
+
190
} else if (!strcmp(buf.buf, "list")) {
191
struct ref *refs = get_refs();
192
struct ref *posn;
transport-helper.c
+28
-11
@@ -10,6 +10,7 @@ struct helper_data
10
{
11
const char *name;
12
struct child_process *helper;
13
+ FILE *out;
14
unsigned fetch : 1;
15
};
16
@@ -18,7 +19,6 @@ static struct child_process *get_helper(struct transport *transport)
19
struct helper_data *data = transport->data;
20
struct strbuf buf = STRBUF_INIT;
21
struct child_process *helper;
21
- FILE *file;
22
23
if (data->helper)
24
return data->helper;
@@ -39,9 +39,9 @@ static struct child_process *get_helper(struct transport *transport)
39
40
write_str_in_full(helper->in, "capabilities\n");
41
42
- file = xfdopen(helper->out, "r");
42
+ data->out = xfdopen(helper->out, "r");
43
while (1) {
44
- if (strbuf_getline(&buf, file, '\n') == EOF)
44
+ if (strbuf_getline(&buf, data->out, '\n') == EOF)
45
exit(128); /* child died, message supplied already */
46
47
if (!*buf.buf)
@@ -58,6 +58,7 @@ static int disconnect_helper(struct transport *transport)
58
if (data->helper) {
59
write_str_in_full(data->helper->in, "\n");
60
close(data->helper->in);
61
+ fclose(data->out);
62
finish_command(data->helper);
63
free((char *)data->helper->argv[0]);
64
free(data->helper->argv);
@@ -70,8 +71,7 @@ static int disconnect_helper(struct transport *transport)
71
static int fetch_with_fetch(struct transport *transport,
72
int nr_heads, const struct ref **to_fetch)
73
{
73
- struct child_process *helper = get_helper(transport);
74
- FILE *file = xfdopen(helper->out, "r");
74
+ struct helper_data *data = transport->data;
75
int i;
76
struct strbuf buf = STRBUF_INIT;
77
@@ -82,12 +82,30 @@ static int fetch_with_fetch(struct transport *transport,
82
83
strbuf_addf(&buf, "fetch %s %s\n",
84
sha1_to_hex(posn->old_sha1), posn->name);
85
- write_in_full(helper->in, buf.buf, buf.len);
86
- strbuf_reset(&buf);
85
+ }
86
88
- if (strbuf_getline(&buf, file, '\n') == EOF)
87
+ strbuf_addch(&buf, '\n');
88
+ if (write_in_full(data->helper->in, buf.buf, buf.len) != buf.len)
89
+ die_errno("cannot send fetch to %s", data->name);
90
+
91
+ while (1) {
92
+ strbuf_reset(&buf);
93
+ if (strbuf_getline(&buf, data->out, '\n') == EOF)
94
exit(128); /* child died, message supplied already */
95
+
96
+ if (!prefixcmp(buf.buf, "lock ")) {
97
+ const char *name = buf.buf + 5;
98
+ if (transport->pack_lockfile)
99
+ warning("%s also locked %s", data->name, name);
100
+ else
101
+ transport->pack_lockfile = xstrdup(name);
102
+ }
103
+ else if (!buf.len)
104
+ break;
105
+ else
106
+ warning("%s unexpectedly said: '%s'", data->name, buf.buf);
107
}
108
+ strbuf_release(&buf);
109
return 0;
110
}
111
@@ -113,21 +131,20 @@ static int fetch(struct transport *transport,
131
132
static struct ref *get_refs_list(struct transport *transport, int for_push)
133
{
134
+ struct helper_data *data = transport->data;
135
struct child_process *helper;
136
struct ref *ret = NULL;
137
struct ref **tail = &ret;
138
struct ref *posn;
139
struct strbuf buf = STRBUF_INIT;
121
- FILE *file;
140
141
helper = get_helper(transport);
142
143
write_str_in_full(helper->in, "list\n");
144
127
- file = xfdopen(helper->out, "r");
145
while (1) {
146
char *eov, *eon;
130
- if (strbuf_getline(&buf, file, '\n') == EOF)
147
+ if (strbuf_getline(&buf, data->out, '\n') == EOF)
148
exit(128); /* child died, message supplied already */
149
150
if (!*buf.buf)