Add fetch methods to transport library.
Signed-off-by: Daniel Barkalow <barkalow@iabervon.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Daniel Barkalow committed
Sep 10, 2007 at 23:03 UTC
c29727d5241be0fee2265fbbe65d1b275b01dc1c
2 files changed
+302
-3
transport.c
+281
-3
@@ -1,6 +1,39 @@
1
#include "cache.h"
2
#include "transport.h"
3
#include "run-command.h"
4
+#include "http.h"
5
+#include "pkt-line.h"
6
+#include "fetch-pack.h"
7
+#include "walker.h"
8
+
9
+/* Generic functions for using commit walkers */
10
+
11
+static int fetch_objs_via_walker(const struct transport *transport,
12
+ int nr_objs, char **objs)
13
+{
14
+ char *dest = xstrdup(transport->url);
15
+ struct walker *walker = transport->data;
16
+
17
+ walker->get_all = 1;
18
+ walker->get_tree = 1;
19
+ walker->get_history = 1;
20
+ walker->get_verbosely = transport->verbose;
21
+ walker->get_recover = 0;
22
+
23
+ if (walker_fetch(walker, nr_objs, objs, NULL, dest))
24
+ die("Fetch failed.");
25
+
26
+ free(dest);
27
+ return 0;
28
+}
29
+
30
+static int disconnect_walker(struct transport *transport)
31
+{
32
+ struct walker *walker = transport->data;
33
+ if (walker)
34
+ walker_free(walker);
35
+ return 0;
36
+}
37
38
static const struct transport_ops rsync_transport;
39
@@ -36,9 +69,119 @@ static int curl_transport_push(struct transport *transport, int refspec_nr, cons
69
return !!err;
70
}
71
72
+#ifndef NO_CURL
73
+static int missing__target(int code, int result)
74
+{
75
+ return /* file:// URL -- do we ever use one??? */
76
+ (result == CURLE_FILE_COULDNT_READ_FILE) ||
77
+ /* http:// and https:// URL */
78
+ (code == 404 && result == CURLE_HTTP_RETURNED_ERROR) ||
79
+ /* ftp:// URL */
80
+ (code == 550 && result == CURLE_FTP_COULDNT_RETR_FILE)
81
+ ;
82
+}
83
+
84
+#define missing_target(a) missing__target((a)->http_code, (a)->curl_result)
85
+
86
+static struct ref *get_refs_via_curl(const struct transport *transport)
87
+{
88
+ struct buffer buffer;
89
+ char *data, *start, *mid;
90
+ char *ref_name;
91
+ char *refs_url;
92
+ int i = 0;
93
+
94
+ struct active_request_slot *slot;
95
+ struct slot_results results;
96
+
97
+ struct ref *refs = NULL;
98
+ struct ref *ref = NULL;
99
+ struct ref *last_ref = NULL;
100
+
101
+ data = xmalloc(4096);
102
+ buffer.size = 4096;
103
+ buffer.posn = 0;
104
+ buffer.buffer = data;
105
+
106
+ refs_url = xmalloc(strlen(transport->url) + 11);
107
+ sprintf(refs_url, "%s/info/refs", transport->url);
108
+
109
+ http_init();
110
+
111
+ slot = get_active_slot();
112
+ slot->results = &results;
113
+ curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
114
+ curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
115
+ curl_easy_setopt(slot->curl, CURLOPT_URL, refs_url);
116
+ curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
117
+ if (start_active_slot(slot)) {
118
+ run_active_slot(slot);
119
+ if (results.curl_result != CURLE_OK) {
120
+ if (missing_target(&results)) {
121
+ free(buffer.buffer);
122
+ return NULL;
123
+ } else {
124
+ free(buffer.buffer);
125
+ error("%s", curl_errorstr);
126
+ return NULL;
127
+ }
128
+ }
129
+ } else {
130
+ free(buffer.buffer);
131
+ error("Unable to start request");
132
+ return NULL;
133
+ }
134
+
135
+ http_cleanup();
136
+
137
+ data = buffer.buffer;
138
+ start = NULL;
139
+ mid = data;
140
+ while (i < buffer.posn) {
141
+ if (!start)
142
+ start = &data[i];
143
+ if (data[i] == '\t')
144
+ mid = &data[i];
145
+ if (data[i] == '\n') {
146
+ data[i] = 0;
147
+ ref_name = mid + 1;
148
+ ref = xmalloc(sizeof(struct ref) +
149
+ strlen(ref_name) + 1);
150
+ memset(ref, 0, sizeof(struct ref));
151
+ strcpy(ref->name, ref_name);
152
+ get_sha1_hex(start, ref->old_sha1);
153
+ if (!refs)
154
+ refs = ref;
155
+ if (last_ref)
156
+ last_ref->next = ref;
157
+ last_ref = ref;
158
+ start = NULL;
159
+ }
160
+ i++;
161
+ }
162
+
163
+ free(buffer.buffer);
164
+
165
+ return refs;
166
+}
167
+
168
+#else
169
+
170
+static struct ref *get_refs_via_curl(const struct transport *transport)
171
+{
172
+ die("Cannot fetch from '%s' without curl ...", transport->url);
173
+ return NULL;
174
+}
175
+
176
+#endif
177
+
178
static const struct transport_ops curl_transport = {
179
/* set_option */ NULL,
41
- /* push */ curl_transport_push
180
+ /* get_refs_list */ get_refs_via_curl,
181
+ /* fetch_refs */ NULL,
182
+ /* fetch_objs */ fetch_objs_via_walker,
183
+ /* push */ curl_transport_push,
184
+ /* disconnect */ disconnect_walker
185
};
186
187
static const struct transport_ops bundle_transport = {
@@ -46,7 +189,13 @@ static const struct transport_ops bundle_transport = {
189
190
struct git_transport_data {
191
unsigned thin : 1;
192
+ unsigned keep : 1;
193
+
194
+ int unpacklimit;
195
196
+ int depth;
197
+
198
+ const char *uploadpack;
199
const char *receivepack;
200
};
201
@@ -54,16 +203,86 @@ static int set_git_option(struct transport *connection,
203
const char *name, const char *value)
204
{
205
struct git_transport_data *data = connection->data;
57
- if (!strcmp(name, TRANS_OPT_RECEIVEPACK)) {
206
+ if (!strcmp(name, TRANS_OPT_UPLOADPACK)) {
207
+ data->uploadpack = value;
208
+ return 0;
209
+ } else if (!strcmp(name, TRANS_OPT_RECEIVEPACK)) {
210
data->receivepack = value;
211
return 0;
212
} else if (!strcmp(name, TRANS_OPT_THIN)) {
213
data->thin = !!value;
214
return 0;
215
+ } else if (!strcmp(name, TRANS_OPT_KEEP)) {
216
+ data->keep = !!value;
217
+ return 0;
218
+ } else if (!strcmp(name, TRANS_OPT_UNPACKLIMIT)) {
219
+ data->unpacklimit = atoi(value);
220
+ return 0;
221
+ } else if (!strcmp(name, TRANS_OPT_DEPTH)) {
222
+ if (!value)
223
+ data->depth = 0;
224
+ else
225
+ data->depth = atoi(value);
226
+ return 0;
227
}
228
return 1;
229
}
230
231
+static struct ref *get_refs_via_connect(const struct transport *transport)
232
+{
233
+ struct git_transport_data *data = transport->data;
234
+ struct ref *refs;
235
+ int fd[2];
236
+ pid_t pid;
237
+ char *dest = xstrdup(transport->url);
238
+
239
+ pid = git_connect(fd, dest, data->uploadpack, 0);
240
+
241
+ if (pid < 0)
242
+ die("Failed to connect to \"%s\"", transport->url);
243
+
244
+ get_remote_heads(fd[0], &refs, 0, NULL, 0);
245
+ packet_flush(fd[1]);
246
+
247
+ finish_connect(pid);
248
+
249
+ free(dest);
250
+
251
+ return refs;
252
+}
253
+
254
+static int fetch_refs_via_pack(const struct transport *transport,
255
+ int nr_heads, char **heads)
256
+{
257
+ struct git_transport_data *data = transport->data;
258
+ struct ref *refs;
259
+ char *dest = xstrdup(transport->url);
260
+ struct fetch_pack_args args;
261
+
262
+ args.uploadpack = data->uploadpack;
263
+ args.quiet = 0;
264
+ args.keep_pack = data->keep;
265
+ args.unpacklimit = data->unpacklimit;
266
+ args.use_thin_pack = data->thin;
267
+ args.fetch_all = 0;
268
+ args.verbose = transport->verbose;
269
+ args.depth = data->depth;
270
+ args.no_progress = 0;
271
+
272
+ setup_fetch_pack(&args);
273
+
274
+ refs = fetch_pack(dest, nr_heads, heads);
275
+
276
+ // ???? check that refs got everything?
277
+
278
+ /* free the memory used for the refs list ... */
279
+
280
+ free_refs(refs);
281
+
282
+ free(dest);
283
+ return 0;
284
+}
285
+
286
static int git_transport_push(struct transport *transport, int refspec_nr, const char **refspec, int flags) {
287
struct git_transport_data *data = transport->data;
288
const char **argv;
@@ -110,6 +329,9 @@ static int git_transport_push(struct transport *transport, int refspec_nr, const
329
330
static const struct transport_ops git_transport = {
331
/* set_option */ set_git_option,
332
+ /* get_refs_list */ get_refs_via_connect,
333
+ /* fetch_refs */ fetch_refs_via_pack,
334
+ /* fetch_objs */ NULL,
335
/* push */ git_transport_push
336
};
337
@@ -140,7 +362,10 @@ struct transport *transport_get(struct remote *remote, const char *url,
362
!prefixcmp(url, "ftp://")) {
363
ret = xmalloc(sizeof(*ret));
364
ret->ops = &curl_transport;
143
- ret->data = NULL;
365
+ if (fetch)
366
+ ret->data = get_http_walker(url);
367
+ else
368
+ ret->data = NULL;
369
} else if (is_local(url) && is_file(url)) {
370
ret = xmalloc(sizeof(*ret));
371
ret->data = NULL;
@@ -150,14 +375,19 @@ struct transport *transport_get(struct remote *remote, const char *url,
375
ret = xcalloc(1, sizeof(*ret));
376
ret->data = data;
377
data->thin = 1;
378
+ data->uploadpack = "git-upload-pack";
379
+ if (remote && remote->uploadpack)
380
+ data->uploadpack = remote->uploadpack;
381
data->receivepack = "git-receive-pack";
382
if (remote && remote->receivepack)
383
data->receivepack = remote->receivepack;
384
+ data->unpacklimit = -1;
385
ret->ops = &git_transport;
386
}
387
if (ret) {
388
ret->remote = remote;
389
ret->url = url;
390
+ ret->remote_refs = NULL;
391
ret->fetch = !!fetch;
392
}
393
return ret;
@@ -186,6 +416,54 @@ int transport_push(struct transport *transport,
416
return transport->ops->push(transport, refspec_nr, refspec, flags);
417
}
418
419
+struct ref *transport_get_remote_refs(struct transport *transport)
420
+{
421
+ if (!transport->remote_refs)
422
+ transport->remote_refs =
423
+ transport->ops->get_refs_list(transport);
424
+ return transport->remote_refs;
425
+}
426
+
427
+#define PACK_HEADS_CHUNK_COUNT 256
428
+
429
+int transport_fetch_refs(struct transport *transport, struct ref *refs)
430
+{
431
+ int i;
432
+ int nr_heads = 0;
433
+ char **heads = xmalloc(PACK_HEADS_CHUNK_COUNT * sizeof(char *));
434
+ struct ref *rm;
435
+ int use_objs = !transport->ops->fetch_refs;
436
+
437
+ for (rm = refs; rm; rm = rm->next) {
438
+ if (rm->peer_ref &&
439
+ !hashcmp(rm->peer_ref->old_sha1, rm->old_sha1))
440
+ continue;
441
+ if (use_objs) {
442
+ heads[nr_heads++] = xstrdup(sha1_to_hex(rm->old_sha1));
443
+ } else {
444
+ heads[nr_heads++] = xstrdup(rm->name);
445
+ }
446
+ if (nr_heads % PACK_HEADS_CHUNK_COUNT == 0)
447
+ heads = xrealloc(heads,
448
+ (nr_heads + PACK_HEADS_CHUNK_COUNT) *
449
+ sizeof(char *));
450
+ }
451
+
452
+ if (use_objs) {
453
+ if (transport->ops->fetch_objs(transport, nr_heads, heads))
454
+ return -1;
455
+ } else {
456
+ if (transport->ops->fetch_refs(transport, nr_heads, heads))
457
+ return -1;
458
+ }
459
+
460
+ /* free the memory used for the heads list ... */
461
+ for (i = 0; i < nr_heads; i++)
462
+ free(heads[i]);
463
+ free(heads);
464
+ return 0;
465
+}
466
+
467
int transport_disconnect(struct transport *transport)
468
{
469
int ret = 0;
transport.h
+21
@@ -29,6 +29,11 @@ struct transport_ops {
29
int (*set_option)(struct transport *connection, const char *name,
30
const char *value);
31
32
+ struct ref *(*get_refs_list)(const struct transport *transport);
33
+ int (*fetch_refs)(const struct transport *transport,
34
+ int nr_refs, char **refs);
35
+ int (*fetch_objs)(const struct transport *transport,
36
+ int nr_objs, char **objs);
37
int (*push)(struct transport *connection, int refspec_nr, const char **refspec, int flags);
38
39
int (*disconnect)(struct transport *connection);
@@ -40,12 +45,24 @@ struct transport *transport_get(struct remote *remote, const char *url,
45
46
/* Transport options which apply to git:// and scp-style URLs */
47
48
+/* The program to use on the remote side to send a pack */
49
+#define TRANS_OPT_UPLOADPACK "uploadpack"
50
+
51
/* The program to use on the remote side to receive a pack */
52
#define TRANS_OPT_RECEIVEPACK "receivepack"
53
54
/* Transfer the data as a thin pack if not null */
55
#define TRANS_OPT_THIN "thin"
56
57
+/* Keep the pack that was transferred if not null */
58
+#define TRANS_OPT_KEEP "keep"
59
+
60
+/* Unpack the objects if fewer than this number of objects are fetched */
61
+#define TRANS_OPT_UNPACKLIMIT "unpacklimit"
62
+
63
+/* Limit the depth of the fetch if not null */
64
+#define TRANS_OPT_DEPTH "depth"
65
+
66
/**
67
* Returns 0 if the option was used, non-zero otherwise. Prints a
68
* message to stderr if the option is not used.
@@ -56,6 +73,10 @@ int transport_set_option(struct transport *transport, const char *name,
73
int transport_push(struct transport *connection,
74
int refspec_nr, const char **refspec, int flags);
75
76
+struct ref *transport_get_remote_refs(struct transport *transport);
77
+
78
+int transport_fetch_refs(struct transport *transport, struct ref *refs);
79
+
80
int transport_disconnect(struct transport *transport);
81
82
#endif