http-fetch: support fetching packfiles by URL
Teach http-fetch the ability to download packfiles directly, given a URL, and to verify them. The http_pack_request suite has been augmented with a function that takes a URL directly. With this function, the hash is only used to determine the name of the temporary file. Signed-off-by: Jonathan Tan <jonathantanmy@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Jonathan Tan committed
Jun 10, 2020 at 13:57 UTC
8d5d2a34df4f82cd9cce913fa25f3a3c2c07d126
5 files changed
+123
-18
Documentation/git-http-fetch.txt
+8
-1
@@ -9,7 +9,7 @@ git-http-fetch - Download from a remote Git repository via HTTP
9
SYNOPSIS
10
--------
11
[verse]
12
-'git http-fetch' [-c] [-t] [-a] [-d] [-v] [-w filename] [--recover] [--stdin] <commit> <url>
12
+'git http-fetch' [-c] [-t] [-a] [-d] [-v] [-w filename] [--recover] [--stdin | --packfile=<hash> | <commit>] <url>
13
14
DESCRIPTION
15
-----------
@@ -40,6 +40,13 @@ commit-id::
40
41
<commit-id>['\t'<filename-as-in--w>]
42
43
+--packfile=<hash>::
44
+ Instead of a commit id on the command line (which is not expected in
45
+ this case), 'git http-fetch' fetches the packfile directly at the given
46
+ URL and uses index-pack to generate corresponding .idx and .keep files.
47
+ The hash is used to determine the name of the temporary file and is
48
+ arbitrary. The output of index-pack is printed to stdout.
49
+
50
--recover::
51
Verify that everything reachable from target is fetched. Used after
52
an earlier fetch is interrupted.
http-fetch.c
+53
-10
@@ -5,7 +5,7 @@
5
#include "walker.h"
6
7
static const char http_fetch_usage[] = "git http-fetch "
8
-"[-c] [-t] [-a] [-v] [--recover] [-w ref] [--stdin] commit-id url";
8
+"[-c] [-t] [-a] [-v] [--recover] [-w ref] [--stdin | --packfile=hash | commit-id] url";
9
10
static int fetch_using_walker(const char *raw_url, int get_verbosely,
11
int get_recover, int commits, char **commit_id,
@@ -43,6 +43,37 @@ static int fetch_using_walker(const char *raw_url, int get_verbosely,
43
return rc;
44
}
45
46
+static void fetch_single_packfile(struct object_id *packfile_hash,
47
+ const char *url) {
48
+ struct http_pack_request *preq;
49
+ struct slot_results results;
50
+ int ret;
51
+
52
+ http_init(NULL, url, 0);
53
+
54
+ preq = new_direct_http_pack_request(packfile_hash->hash, xstrdup(url));
55
+ if (preq == NULL)
56
+ die("couldn't create http pack request");
57
+ preq->slot->results = &results;
58
+ preq->generate_keep = 1;
59
+
60
+ if (start_active_slot(preq->slot)) {
61
+ run_active_slot(preq->slot);
62
+ if (results.curl_result != CURLE_OK) {
63
+ die("Unable to get pack file %s\n%s", preq->url,
64
+ curl_errorstr);
65
+ }
66
+ } else {
67
+ die("Unable to start request");
68
+ }
69
+
70
+ if ((ret = finish_http_pack_request(preq)))
71
+ die("finish_http_pack_request gave result %d", ret);
72
+
73
+ release_http_pack_request(preq);
74
+ http_cleanup();
75
+}
76
+
77
int cmd_main(int argc, const char **argv)
78
{
79
int commits_on_stdin = 0;
@@ -52,8 +83,12 @@ int cmd_main(int argc, const char **argv)
83
int arg = 1;
84
int get_verbosely = 0;
85
int get_recover = 0;
86
+ int packfile = 0;
87
+ struct object_id packfile_hash;
88
89
while (arg < argc && argv[arg][0] == '-') {
90
+ const char *p;
91
+
92
if (argv[arg][1] == 't') {
93
} else if (argv[arg][1] == 'c') {
94
} else if (argv[arg][1] == 'a') {
@@ -68,25 +103,33 @@ int cmd_main(int argc, const char **argv)
103
get_recover = 1;
104
} else if (!strcmp(argv[arg], "--stdin")) {
105
commits_on_stdin = 1;
106
+ } else if (skip_prefix(argv[arg], "--packfile=", &p)) {
107
+ const char *end;
108
+
109
+ packfile = 1;
110
+ if (parse_oid_hex(p, &packfile_hash, &end) || *end)
111
+ die(_("argument to --packfile must be a valid hash (got '%s')"), p);
112
}
113
arg++;
114
}
74
- if (argc != arg + 2 - commits_on_stdin)
115
+ if (argc != arg + 2 - (commits_on_stdin || packfile))
116
usage(http_fetch_usage);
76
- if (commits_on_stdin) {
77
- commits = walker_targets_stdin(&commit_id, &write_ref);
78
- } else {
79
- commit_id = (char **) &argv[arg++];
80
- commits = 1;
81
- }
117
118
setup_git_directory();
119
120
git_config(git_default_config, NULL);
121
87
- if (!argv[arg])
88
- BUG("must have one arg remaining");
122
+ if (packfile) {
123
+ fetch_single_packfile(&packfile_hash, argv[arg]);
124
+ return 0;
125
+ }
126
127
+ if (commits_on_stdin) {
128
+ commits = walker_targets_stdin(&commit_id, &write_ref);
129
+ } else {
130
+ commit_id = (char **) &argv[arg++];
131
+ commits = 1;
132
+ }
133
return fetch_using_walker(argv[arg], get_verbosely, get_recover,
134
commits, commit_id, write_ref,
135
commits_on_stdin);
http.c
+21
-7
@@ -2281,7 +2281,13 @@ int finish_http_pack_request(struct http_pack_request *preq)
2281
argv_array_push(&ip.args, "--stdin");
2282
ip.git_cmd = 1;
2283
ip.in = tmpfile_fd;
2284
- ip.no_stdout = 1;
2284
+ if (preq->generate_keep) {
2285
+ argv_array_pushf(&ip.args, "--keep=git %"PRIuMAX,
2286
+ (uintmax_t)getpid());
2287
+ ip.out = 0;
2288
+ } else {
2289
+ ip.no_stdout = 1;
2290
+ }
2291
2292
if (run_command(&ip)) {
2293
ret = -1;
@@ -2307,19 +2313,27 @@ void http_install_packfile(struct packed_git *p,
2313
}
2314
2315
struct http_pack_request *new_http_pack_request(
2310
- const unsigned char *packed_git_hash, const char *base_url)
2316
+ const unsigned char *packed_git_hash, const char *base_url) {
2317
+
2318
+ struct strbuf buf = STRBUF_INIT;
2319
+
2320
+ end_url_with_slash(&buf, base_url);
2321
+ strbuf_addf(&buf, "objects/pack/pack-%s.pack",
2322
+ hash_to_hex(packed_git_hash));
2323
+ return new_direct_http_pack_request(packed_git_hash,
2324
+ strbuf_detach(&buf, NULL));
2325
+}
2326
+
2327
+struct http_pack_request *new_direct_http_pack_request(
2328
+ const unsigned char *packed_git_hash, char *url)
2329
{
2330
off_t prev_posn = 0;
2313
- struct strbuf buf = STRBUF_INIT;
2331
struct http_pack_request *preq;
2332
2333
preq = xcalloc(1, sizeof(*preq));
2334
strbuf_init(&preq->tmpfile, 0);
2335
2319
- end_url_with_slash(&buf, base_url);
2320
- strbuf_addf(&buf, "objects/pack/pack-%s.pack",
2321
- hash_to_hex(packed_git_hash));
2322
- preq->url = strbuf_detach(&buf, NULL);
2336
+ preq->url = url;
2337
2338
strbuf_addf(&preq->tmpfile, "%s.temp", sha1_pack_name(packed_git_hash));
2339
preq->packfile = fopen(preq->tmpfile.buf, "a");
http.h
+11
@@ -216,6 +216,15 @@ int http_get_info_packs(const char *base_url,
216
217
struct http_pack_request {
218
char *url;
219
+
220
+ /*
221
+ * If this is true, finish_http_pack_request() will pass "--keep" to
222
+ * index-pack, resulting in the creation of a keep file, and will not
223
+ * suppress its stdout (that is, the "keep\t<hash>\n" line will be
224
+ * printed to stdout).
225
+ */
226
+ unsigned generate_keep : 1;
227
+
228
FILE *packfile;
229
struct strbuf tmpfile;
230
struct active_request_slot *slot;
@@ -223,6 +232,8 @@ struct http_pack_request {
232
233
struct http_pack_request *new_http_pack_request(
234
const unsigned char *packed_git_hash, const char *base_url);
235
+struct http_pack_request *new_direct_http_pack_request(
236
+ const unsigned char *packed_git_hash, char *url);
237
int finish_http_pack_request(struct http_pack_request *preq);
238
void release_http_pack_request(struct http_pack_request *preq);
239
t/t5550-http-fetch-dumb.sh
+30
@@ -199,6 +199,28 @@ test_expect_success 'fetch packed objects' '
199
git clone $HTTPD_URL/dumb/repo_pack.git
200
'
201
202
+test_expect_success 'http-fetch --packfile' '
203
+ # Arbitrary hash. Use rev-parse so that we get one of the correct
204
+ # length.
205
+ ARBITRARY=$(git -C "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git rev-parse HEAD) &&
206
+
207
+ git init packfileclient &&
208
+ p=$(cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git && ls objects/pack/pack-*.pack) &&
209
+ git -C packfileclient http-fetch --packfile=$ARBITRARY "$HTTPD_URL"/dumb/repo_pack.git/$p >out &&
210
+
211
+ grep "^keep.[0-9a-f]\{16,\}$" out &&
212
+ cut -c6- out >packhash &&
213
+
214
+ # Ensure that the expected files are generated
215
+ test -e "packfileclient/.git/objects/pack/pack-$(cat packhash).pack" &&
216
+ test -e "packfileclient/.git/objects/pack/pack-$(cat packhash).idx" &&
217
+ test -e "packfileclient/.git/objects/pack/pack-$(cat packhash).keep" &&
218
+
219
+ # Ensure that it has the HEAD of repo_pack, at least
220
+ HASH=$(git -C "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git rev-parse HEAD) &&
221
+ git -C packfileclient cat-file -e "$HASH"
222
+'
223
+
224
test_expect_success 'fetch notices corrupt pack' '
225
cp -R "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad1.git &&
226
(cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad1.git &&
@@ -214,6 +236,14 @@ test_expect_success 'fetch notices corrupt pack' '
236
)
237
'
238
239
+test_expect_success 'http-fetch --packfile with corrupt pack' '
240
+ rm -rf packfileclient &&
241
+ git init packfileclient &&
242
+ p=$(cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad1.git && ls objects/pack/pack-*.pack) &&
243
+ test_must_fail git -C packfileclient http-fetch --packfile \
244
+ "$HTTPD_URL"/dumb/repo_bad1.git/$p
245
+'
246
+
247
test_expect_success 'fetch notices corrupt idx' '
248
cp -R "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad2.git &&
249
(cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad2.git &&