@matheusbsbs / matheus-bagliano / commits / c23f592117

bundle-uri: fetch a list of bundles

When the content at a given bundle URI is not understood as a bundle (based on inspecting the initial content), then Git currently gives up and ignores that content. Independent bundle providers may want to split up the bundle content into multiple bundles, but still make them available from a single URI. Teach Git to attempt parsing the bundle URI content as a Git config file providing the key=value pairs for a bundle list. Git then looks at the mode of the list to see if ANY single bundle is sufficient or if ALL bundles are required. The content at the selected URIs are downloaded and the content is inspected again, creating a recursive process. To guard the recursion against malformed or malicious content, limit the recursion depth to a reasonable four for now. This can be converted to a configured value in the future if necessary. The value of four is twice as high as expected to be useful (a bundle list is unlikely to point to more bundle lists). To test this scenario, create an interesting bundle topology where three incremental bundles are built on top of a single full bundle. By using a merge commit, the two middle bundles are "independent" in that they do not require each other in order to unbundle themselves. They each only need the base bundle. The bundle containing the merge commit requires both of the middle bundles, though. This leads to some interesting decisions when unbundling, especially when we later implement heuristics that promote downloading bundles until the prerequisite commits are satisfied. Signed-off-by: Derrick Stolee <derrickstolee@github.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Derrick Stolee committed Oct 12, 2022 at 12:52 UTC c23f592117bac30765ca22545386c3e9304da803
3 files changed +448 -16
bundle-uri.c
+187 -16
@@ -37,6 +37,8 @@ static int clear_remote_bundle_info(struct remote_bundle_info *bundle,
37 {
38 FREE_AND_NULL(bundle->id);
39 FREE_AND_NULL(bundle->uri);
40 + FREE_AND_NULL(bundle->file);
41 + bundle->unbundled = 0;
42 return 0;
43 }
44
@@ -334,18 +336,117 @@ static int unbundle_from_file(struct repository *r, const char *file)
336 return result;
337 }
338
339 +struct bundle_list_context {
340 + struct repository *r;
341 + struct bundle_list *list;
342 + enum bundle_list_mode mode;
343 + int count;
344 + int depth;
345 +};
346 +
347 +/*
348 + * This early definition is necessary because we use indirect recursion:
349 + *
350 + * While iterating through a bundle list that was downloaded as part
351 + * of fetch_bundle_uri_internal(), iterator methods eventually call it
352 + * again, but with depth + 1.
353 + */
354 +static int fetch_bundle_uri_internal(struct repository *r,
355 + struct remote_bundle_info *bundle,
356 + int depth,
357 + struct bundle_list *list);
358 +
359 +static int download_bundle_to_file(struct remote_bundle_info *bundle, void *data)
360 +{
361 + int res;
362 + struct bundle_list_context *ctx = data;
363 +
364 + if (ctx->mode == BUNDLE_MODE_ANY && ctx->count)
365 + return 0;
366 +
367 + res = fetch_bundle_uri_internal(ctx->r, bundle, ctx->depth + 1, ctx->list);
368 +
369 + /*
370 + * Only increment count if the download succeeded. If our mode is
371 + * BUNDLE_MODE_ANY, then we will want to try other URIs in the
372 + * list in case they work instead.
373 + */
374 + if (!res)
375 + ctx->count++;
376 +
377 + /*
378 + * To be opportunistic as possible, we continue iterating and
379 + * download as many bundles as we can, so we can apply the ones
380 + * that work, even in BUNDLE_MODE_ALL mode.
381 + */
382 + return 0;
383 +}
384 +
385 +static int download_bundle_list(struct repository *r,
386 + struct bundle_list *local_list,
387 + struct bundle_list *global_list,
388 + int depth)
389 +{
390 + struct bundle_list_context ctx = {
391 + .r = r,
392 + .list = global_list,
393 + .depth = depth + 1,
394 + .mode = local_list->mode,
395 + };
396 +
397 + return for_all_bundles_in_list(local_list, download_bundle_to_file, &ctx);
398 +}
399 +
400 +static int fetch_bundle_list_in_config_format(struct repository *r,
401 + struct bundle_list *global_list,
402 + struct remote_bundle_info *bundle,
403 + int depth)
404 +{
405 + int result;
406 + struct bundle_list list_from_bundle;
407 +
408 + init_bundle_list(&list_from_bundle);
409 +
410 + if ((result = bundle_uri_parse_config_format(bundle->uri,
411 + bundle->file,
412 + &list_from_bundle)))
413 + goto cleanup;
414 +
415 + if (list_from_bundle.mode == BUNDLE_MODE_NONE) {
416 + warning(_("unrecognized bundle mode from URI '%s'"),
417 + bundle->uri);
418 + result = -1;
419 + goto cleanup;
420 + }
421 +
422 + if ((result = download_bundle_list(r, &list_from_bundle,
423 + global_list, depth)))
424 + goto cleanup;
425 +
426 +cleanup:
427 + clear_bundle_list(&list_from_bundle);
428 + return result;
429 +}
430 +
431 /**
432 * This limits the recursion on fetch_bundle_uri_internal() when following
433 * bundle lists.
434 */
435 static int max_bundle_uri_depth = 4;
436
437 +/**
438 + * Recursively download all bundles advertised at the given URI
439 + * to files. If the file is a bundle, then add it to the given
440 + * 'list'. Otherwise, expect a bundle list and recurse on the
441 + * URIs in that list according to the list mode (ANY or ALL).
442 + */
443 static int fetch_bundle_uri_internal(struct repository *r,
344 - const char *uri,
345 - int depth)
444 + struct remote_bundle_info *bundle,
445 + int depth,
446 + struct bundle_list *list)
447 {
448 int result = 0;
348 - char *filename;
449 + struct remote_bundle_info *bcopy;
450
451 if (depth >= max_bundle_uri_depth) {
452 warning(_("exceeded bundle URI recursion limit (%d)"),
@@ -353,36 +454,106 @@ static int fetch_bundle_uri_internal(struct repository *r,
454 return -1;
455 }
456
356 - if (!(filename = find_temp_filename())) {
457 + if (!bundle->file &&
458 + !(bundle->file = find_temp_filename())) {
459 result = -1;
460 goto cleanup;
461 }
462
361 - if ((result = copy_uri_to_file(filename, uri))) {
362 - warning(_("failed to download bundle from URI '%s'"), uri);
463 + if ((result = copy_uri_to_file(bundle->file, bundle->uri))) {
464 + warning(_("failed to download bundle from URI '%s'"), bundle->uri);
465 goto cleanup;
466 }
467
366 - if ((result = !is_bundle(filename, 0))) {
367 - warning(_("file at URI '%s' is not a bundle"), uri);
468 + if ((result = !is_bundle(bundle->file, 1))) {
469 + result = fetch_bundle_list_in_config_format(
470 + r, list, bundle, depth);
471 + if (result)
472 + warning(_("file at URI '%s' is not a bundle or bundle list"),
473 + bundle->uri);
474 goto cleanup;
475 }
476
371 - if ((result = unbundle_from_file(r, filename))) {
372 - warning(_("failed to unbundle bundle from URI '%s'"), uri);
373 - goto cleanup;
374 - }
477 + /* Copy the bundle and insert it into the global list. */
478 + CALLOC_ARRAY(bcopy, 1);
479 + bcopy->id = xstrdup(bundle->id);
480 + bcopy->file = xstrdup(bundle->file);
481 + hashmap_entry_init(&bcopy->ent, strhash(bcopy->id));
482 + hashmap_add(&list->bundles, &bcopy->ent);
483
484 cleanup:
377 - if (filename)
378 - unlink(filename);
379 - free(filename);
485 + if (result && bundle->file)
486 + unlink(bundle->file);
487 return result;
488 }
489
490 +/**
491 + * This loop iterator breaks the loop with nonzero return code on the
492 + * first successful unbundling of a bundle.
493 + */
494 +static int attempt_unbundle(struct remote_bundle_info *info, void *data)
495 +{
496 + struct repository *r = data;
497 +
498 + if (!info->file || info->unbundled)
499 + return 0;
500 +
501 + if (!unbundle_from_file(r, info->file)) {
502 + info->unbundled = 1;
503 + return 1;
504 + }
505 +
506 + return 0;
507 +}
508 +
509 +static int unbundle_all_bundles(struct repository *r,
510 + struct bundle_list *list)
511 +{
512 + /*
513 + * Iterate through all bundles looking for ones that can
514 + * successfully unbundle. If any succeed, then perhaps another
515 + * will succeed in the next attempt.
516 + *
517 + * Keep in mind that a non-zero result for the loop here means
518 + * the loop terminated early on a successful unbundling, which
519 + * signals that we can try again.
520 + */
521 + while (for_all_bundles_in_list(list, attempt_unbundle, r)) ;
522 +
523 + return 0;
524 +}
525 +
526 +static int unlink_bundle(struct remote_bundle_info *info, void *data)
527 +{
528 + if (info->file)
529 + unlink_or_warn(info->file);
530 + return 0;
531 +}
532 +
533 int fetch_bundle_uri(struct repository *r, const char *uri)
534 {
385 - return fetch_bundle_uri_internal(r, uri, 0);
535 + int result;
536 + struct bundle_list list;
537 + struct remote_bundle_info bundle = {
538 + .uri = xstrdup(uri),
539 + .id = xstrdup(""),
540 + };
541 +
542 + init_bundle_list(&list);
543 +
544 + /* If a bundle is added to this global list, then it is required. */
545 + list.mode = BUNDLE_MODE_ALL;
546 +
547 + if ((result = fetch_bundle_uri_internal(r, &bundle, 0, &list)))
548 + goto cleanup;
549 +
550 + result = unbundle_all_bundles(r, &list);
551 +
552 +cleanup:
553 + for_all_bundles_in_list(&list, unlink_bundle, NULL);
554 + clear_bundle_list(&list);
555 + clear_remote_bundle_info(&bundle, NULL);
556 + return result;
557 }
558
559 /**
bundle-uri.h
+13
@@ -28,6 +28,19 @@ struct remote_bundle_info {
28 * if there was no table of contents.
29 */
30 char *uri;
31 +
32 + /**
33 + * If the bundle has been downloaded, then 'file' is a
34 + * filename storing its contents. Otherwise, 'file' is
35 + * NULL.
36 + */
37 + char *file;
38 +
39 + /**
40 + * If the bundle has been unbundled successfully, then
41 + * this boolean is true.
42 + */
43 + unsigned unbundled:1;
44 };
45
46 #define REMOTE_BUNDLE_INFO_INIT { 0 }
t/t5558-clone-bundle-uri.sh
+248
@@ -41,6 +41,195 @@ test_expect_success 'clone with file:// bundle' '
41 test_cmp expect actual
42 '
43
44 +# To get interesting tests for bundle lists, we need to construct a
45 +# somewhat-interesting commit history.
46 +#
47 +# ---------------- bundle-4
48 +#
49 +# 4
50 +# / \
51 +# ----|---|------- bundle-3
52 +# | |
53 +# | 3
54 +# | |
55 +# ----|---|------- bundle-2
56 +# | |
57 +# 2 |
58 +# | |
59 +# ----|---|------- bundle-1
60 +# \ /
61 +# 1
62 +# |
63 +# (previous commits)
64 +test_expect_success 'construct incremental bundle list' '
65 + (
66 + cd clone-from &&
67 + git checkout -b base &&
68 + test_commit 1 &&
69 + git checkout -b left &&
70 + test_commit 2 &&
71 + git checkout -b right base &&
72 + test_commit 3 &&
73 + git checkout -b merge left &&
74 + git merge right -m "4" &&
75 +
76 + git bundle create bundle-1.bundle base &&
77 + git bundle create bundle-2.bundle base..left &&
78 + git bundle create bundle-3.bundle base..right &&
79 + git bundle create bundle-4.bundle merge --not left right
80 + )
81 +'
82 +
83 +test_expect_success 'clone bundle list (file, no heuristic)' '
84 + cat >bundle-list <<-EOF &&
85 + [bundle]
86 + version = 1
87 + mode = all
88 +
89 + [bundle "bundle-1"]
90 + uri = file://$(pwd)/clone-from/bundle-1.bundle
91 +
92 + [bundle "bundle-2"]
93 + uri = file://$(pwd)/clone-from/bundle-2.bundle
94 +
95 + [bundle "bundle-3"]
96 + uri = file://$(pwd)/clone-from/bundle-3.bundle
97 +
98 + [bundle "bundle-4"]
99 + uri = file://$(pwd)/clone-from/bundle-4.bundle
100 + EOF
101 +
102 + git clone --bundle-uri="file://$(pwd)/bundle-list" clone-from clone-list-file &&
103 + git -C clone-from for-each-ref --format="%(objectname)" >oids &&
104 + git -C clone-list-file cat-file --batch-check <oids &&
105 +
106 + git -C clone-list-file for-each-ref --format="%(refname)" >refs &&
107 + grep "refs/bundles/" refs >actual &&
108 + cat >expect <<-\EOF &&
109 + refs/bundles/base
110 + refs/bundles/left
111 + refs/bundles/merge
112 + refs/bundles/right
113 + EOF
114 + test_cmp expect actual
115 +'
116 +
117 +test_expect_success 'clone bundle list (file, all mode, some failures)' '
118 + cat >bundle-list <<-EOF &&
119 + [bundle]
120 + version = 1
121 + mode = all
122 +
123 + # Does not exist. Should be skipped.
124 + [bundle "bundle-0"]
125 + uri = file://$(pwd)/clone-from/bundle-0.bundle
126 +
127 + [bundle "bundle-1"]
128 + uri = file://$(pwd)/clone-from/bundle-1.bundle
129 +
130 + [bundle "bundle-2"]
131 + uri = file://$(pwd)/clone-from/bundle-2.bundle
132 +
133 + # No bundle-3 means bundle-4 will not apply.
134 +
135 + [bundle "bundle-4"]
136 + uri = file://$(pwd)/clone-from/bundle-4.bundle
137 +
138 + # Does not exist. Should be skipped.
139 + [bundle "bundle-5"]
140 + uri = file://$(pwd)/clone-from/bundle-5.bundle
141 + EOF
142 +
143 + GIT_TRACE2_PERF=1 \
144 + git clone --bundle-uri="file://$(pwd)/bundle-list" clone-from clone-all-some &&
145 + git -C clone-from for-each-ref --format="%(objectname)" >oids &&
146 + git -C clone-all-some cat-file --batch-check <oids &&
147 +
148 + git -C clone-all-some for-each-ref --format="%(refname)" >refs &&
149 + grep "refs/bundles/" refs >actual &&
150 + cat >expect <<-\EOF &&
151 + refs/bundles/base
152 + refs/bundles/left
153 + EOF
154 + test_cmp expect actual
155 +'
156 +
157 +test_expect_success 'clone bundle list (file, all mode, all failures)' '
158 + cat >bundle-list <<-EOF &&
159 + [bundle]
160 + version = 1
161 + mode = all
162 +
163 + # Does not exist. Should be skipped.
164 + [bundle "bundle-0"]
165 + uri = file://$(pwd)/clone-from/bundle-0.bundle
166 +
167 + # Does not exist. Should be skipped.
168 + [bundle "bundle-5"]
169 + uri = file://$(pwd)/clone-from/bundle-5.bundle
170 + EOF
171 +
172 + git clone --bundle-uri="file://$(pwd)/bundle-list" clone-from clone-all-fail &&
173 + git -C clone-from for-each-ref --format="%(objectname)" >oids &&
174 + git -C clone-all-fail cat-file --batch-check <oids &&
175 +
176 + git -C clone-all-fail for-each-ref --format="%(refname)" >refs &&
177 + ! grep "refs/bundles/" refs
178 +'
179 +
180 +test_expect_success 'clone bundle list (file, any mode)' '
181 + cat >bundle-list <<-EOF &&
182 + [bundle]
183 + version = 1
184 + mode = any
185 +
186 + # Does not exist. Should be skipped.
187 + [bundle "bundle-0"]
188 + uri = file://$(pwd)/clone-from/bundle-0.bundle
189 +
190 + [bundle "bundle-1"]
191 + uri = file://$(pwd)/clone-from/bundle-1.bundle
192 +
193 + # Does not exist. Should be skipped.
194 + [bundle "bundle-5"]
195 + uri = file://$(pwd)/clone-from/bundle-5.bundle
196 + EOF
197 +
198 + git clone --bundle-uri="file://$(pwd)/bundle-list" clone-from clone-any-file &&
199 + git -C clone-from for-each-ref --format="%(objectname)" >oids &&
200 + git -C clone-any-file cat-file --batch-check <oids &&
201 +
202 + git -C clone-any-file for-each-ref --format="%(refname)" >refs &&
203 + grep "refs/bundles/" refs >actual &&
204 + cat >expect <<-\EOF &&
205 + refs/bundles/base
206 + EOF
207 + test_cmp expect actual
208 +'
209 +
210 +test_expect_success 'clone bundle list (file, any mode, all failures)' '
211 + cat >bundle-list <<-EOF &&
212 + [bundle]
213 + version = 1
214 + mode = any
215 +
216 + # Does not exist. Should be skipped.
217 + [bundle "bundle-0"]
218 + uri = $HTTPD_URL/bundle-0.bundle
219 +
220 + # Does not exist. Should be skipped.
221 + [bundle "bundle-5"]
222 + uri = $HTTPD_URL/bundle-5.bundle
223 + EOF
224 +
225 + git clone --bundle-uri="file://$(pwd)/bundle-list" clone-from clone-any-fail &&
226 + git -C clone-from for-each-ref --format="%(objectname)" >oids &&
227 + git -C clone-any-fail cat-file --batch-check <oids &&
228 +
229 + git -C clone-any-fail for-each-ref --format="%(refname)" >refs &&
230 + ! grep "refs/bundles/" refs
231 +'
232 +
233 #########################################################################
234 # HTTP tests begin here
235
@@ -75,6 +264,65 @@ test_expect_success 'clone HTTP bundle' '
264 test_config -C clone-http log.excludedecoration refs/bundle/
265 '
266
267 +test_expect_success 'clone bundle list (HTTP, no heuristic)' '
268 + cp clone-from/bundle-*.bundle "$HTTPD_DOCUMENT_ROOT_PATH/" &&
269 + cat >"$HTTPD_DOCUMENT_ROOT_PATH/bundle-list" <<-EOF &&
270 + [bundle]
271 + version = 1
272 + mode = all
273 +
274 + [bundle "bundle-1"]
275 + uri = $HTTPD_URL/bundle-1.bundle
276 +
277 + [bundle "bundle-2"]
278 + uri = $HTTPD_URL/bundle-2.bundle
279 +
280 + [bundle "bundle-3"]
281 + uri = $HTTPD_URL/bundle-3.bundle
282 +
283 + [bundle "bundle-4"]
284 + uri = $HTTPD_URL/bundle-4.bundle
285 + EOF
286 +
287 + git clone --bundle-uri="$HTTPD_URL/bundle-list" clone-from clone-list-http &&
288 + git -C clone-from for-each-ref --format="%(objectname)" >oids &&
289 + git -C clone-list-http cat-file --batch-check <oids
290 +'
291 +
292 +test_expect_success 'clone bundle list (HTTP, any mode)' '
293 + cp clone-from/bundle-*.bundle "$HTTPD_DOCUMENT_ROOT_PATH/" &&
294 + cat >"$HTTPD_DOCUMENT_ROOT_PATH/bundle-list" <<-EOF &&
295 + [bundle]
296 + version = 1
297 + mode = any
298 +
299 + # Does not exist. Should be skipped.
300 + [bundle "bundle-0"]
301 + uri = $HTTPD_URL/bundle-0.bundle
302 +
303 + [bundle "bundle-1"]
304 + uri = $HTTPD_URL/bundle-1.bundle
305 +
306 + # Does not exist. Should be skipped.
307 + [bundle "bundle-5"]
308 + uri = $HTTPD_URL/bundle-5.bundle
309 + EOF
310 +
311 + git clone --bundle-uri="$HTTPD_URL/bundle-list" clone-from clone-any-http &&
312 + git -C clone-from for-each-ref --format="%(objectname)" >oids &&
313 + git -C clone-any-http cat-file --batch-check <oids &&
314 +
315 + git -C clone-list-file for-each-ref --format="%(refname)" >refs &&
316 + grep "refs/bundles/" refs >actual &&
317 + cat >expect <<-\EOF &&
318 + refs/bundles/base
319 + refs/bundles/left
320 + refs/bundles/merge
321 + refs/bundles/right
322 + EOF
323 + test_cmp expect actual
324 +'
325 +
326 # Do not add tests here unless they use the HTTP server, as they will
327 # not run unless the HTTP dependencies exist.
328