| 1 | #define USE_THE_REPOSITORY_VARIABLE |
| 2 | #define DISABLE_SIGN_COMPARE_WARNINGS |
| 3 | |
| 4 | #include "git-compat-util.h" |
| 5 | #include "bundle-uri.h" |
| 6 | #include "bundle.h" |
| 7 | #include "copy.h" |
| 8 | #include "gettext.h" |
| 9 | #include "refs.h" |
| 10 | #include "run-command.h" |
| 11 | #include "hashmap.h" |
| 12 | #include "pkt-line.h" |
| 13 | #include "config.h" |
| 14 | #include "fetch-pack.h" |
| 15 | #include "remote.h" |
| 16 | #include "trace2.h" |
| 17 | #include "odb.h" |
| 18 | |
| 19 | static struct { |
| 20 | enum bundle_list_heuristic heuristic; |
| 21 | const char *name; |
| 22 | } heuristics[BUNDLE_HEURISTIC__COUNT] = { |
| 23 | { BUNDLE_HEURISTIC_NONE, ""}, |
| 24 | { BUNDLE_HEURISTIC_CREATIONTOKEN, "creationToken" }, |
| 25 | }; |
| 26 | |
| 27 | static int compare_bundles(const void *hashmap_cmp_fn_data UNUSED, |
| 28 | const struct hashmap_entry *he1, |
| 29 | const struct hashmap_entry *he2, |
| 30 | const void *id) |
| 31 | { |
| 32 | const struct remote_bundle_info *e1 = |
| 33 | container_of(he1, const struct remote_bundle_info, ent); |
| 34 | const struct remote_bundle_info *e2 = |
| 35 | container_of(he2, const struct remote_bundle_info, ent); |
| 36 | |
| 37 | return strcmp(e1->id, id ? (const char *)id : e2->id); |
| 38 | } |
| 39 | |
| 40 | void init_bundle_list(struct bundle_list *list) |
| 41 | { |
| 42 | memset(list, 0, sizeof(*list)); |
| 43 | |
| 44 | /* Implied defaults. */ |
| 45 | list->mode = BUNDLE_MODE_ALL; |
| 46 | list->version = 1; |
| 47 | |
| 48 | hashmap_init(&list->bundles, compare_bundles, NULL, 0); |
| 49 | } |
| 50 | |
| 51 | static int clear_remote_bundle_info(struct remote_bundle_info *bundle, |
| 52 | void *data UNUSED) |
| 53 | { |
| 54 | FREE_AND_NULL(bundle->id); |
| 55 | FREE_AND_NULL(bundle->uri); |
| 56 | FREE_AND_NULL(bundle->file); |
| 57 | bundle->unbundled = 0; |
| 58 | return 0; |
| 59 | } |
| 60 | |
| 61 | void clear_bundle_list(struct bundle_list *list) |
| 62 | { |
| 63 | if (!list) |
| 64 | return; |
| 65 | |
| 66 | for_all_bundles_in_list(list, clear_remote_bundle_info, NULL); |
| 67 | hashmap_clear_and_free(&list->bundles, struct remote_bundle_info, ent); |
| 68 | free(list->baseURI); |
| 69 | } |
| 70 | |
| 71 | int for_all_bundles_in_list(struct bundle_list *list, |
| 72 | bundle_iterator iter, |
| 73 | void *data) |
| 74 | { |
| 75 | struct remote_bundle_info *info; |
| 76 | struct hashmap_iter i; |
| 77 | |
| 78 | hashmap_for_each_entry(&list->bundles, &i, info, ent) { |
| 79 | int result = iter(info, data); |
| 80 | |
| 81 | if (result) |
| 82 | return result; |
| 83 | } |
| 84 | |
| 85 | return 0; |
| 86 | } |
| 87 | |
| 88 | static int summarize_bundle(struct remote_bundle_info *info, void *data) |
| 89 | { |
| 90 | FILE *fp = data; |
| 91 | fprintf(fp, "[bundle \"%s\"]\n", info->id); |
| 92 | if (info->uri) |
| 93 | fprintf(fp, "\turi = %s\n", info->uri); |
| 94 | else |
| 95 | fprintf(fp, "\t# uri = (missing)\n"); |
| 96 | |
| 97 | if (info->creationToken) |
| 98 | fprintf(fp, "\tcreationToken = %"PRIu64"\n", info->creationToken); |
| 99 | return 0; |
| 100 | } |
| 101 | |
| 102 | void print_bundle_list(FILE *fp, struct bundle_list *list) |
| 103 | { |
| 104 | const char *mode; |
| 105 | |
| 106 | switch (list->mode) { |
| 107 | case BUNDLE_MODE_ALL: |
| 108 | mode = "all"; |
| 109 | break; |
| 110 | |
| 111 | case BUNDLE_MODE_ANY: |
| 112 | mode = "any"; |
| 113 | break; |
| 114 | |
| 115 | case BUNDLE_MODE_NONE: |
| 116 | default: |
| 117 | mode = "<unknown>"; |
| 118 | } |
| 119 | |
| 120 | fprintf(fp, "[bundle]\n"); |
| 121 | fprintf(fp, "\tversion = %d\n", list->version); |
| 122 | fprintf(fp, "\tmode = %s\n", mode); |
| 123 | |
| 124 | if (list->heuristic) { |
| 125 | int i; |
| 126 | for (i = 0; i < BUNDLE_HEURISTIC__COUNT; i++) { |
| 127 | if (heuristics[i].heuristic == list->heuristic) { |
| 128 | fprintf(fp, "\theuristic = %s\n", |
| 129 | heuristics[list->heuristic].name); |
| 130 | break; |
| 131 | } |
| 132 | } |
| 133 | } |
| 134 | |
| 135 | for_all_bundles_in_list(list, summarize_bundle, fp); |
| 136 | } |
| 137 | |
| 138 | /** |
| 139 | * Given a key-value pair, update the state of the given bundle list. |
| 140 | * Returns 0 if the key-value pair is understood. Returns -1 if the key |
| 141 | * is not understood or the value is malformed. |
| 142 | */ |
| 143 | static int bundle_list_update(const char *key, const char *value, |
| 144 | struct bundle_list *list) |
| 145 | { |
| 146 | struct strbuf id = STRBUF_INIT; |
| 147 | struct remote_bundle_info lookup = REMOTE_BUNDLE_INFO_INIT; |
| 148 | struct remote_bundle_info *bundle; |
| 149 | const char *subsection, *subkey; |
| 150 | size_t subsection_len; |
| 151 | |
| 152 | if (parse_config_key(key, "bundle", &subsection, &subsection_len, &subkey)) |
| 153 | return -1; |
| 154 | |
| 155 | if (!subsection_len) { |
| 156 | if (!strcmp(subkey, "version")) { |
| 157 | int version; |
| 158 | if (!git_parse_int(value, &version)) |
| 159 | return -1; |
| 160 | if (version != 1) |
| 161 | return -1; |
| 162 | |
| 163 | list->version = version; |
| 164 | return 0; |
| 165 | } |
| 166 | |
| 167 | if (!strcmp(subkey, "mode")) { |
| 168 | if (!strcmp(value, "all")) |
| 169 | list->mode = BUNDLE_MODE_ALL; |
| 170 | else if (!strcmp(value, "any")) |
| 171 | list->mode = BUNDLE_MODE_ANY; |
| 172 | else |
| 173 | return -1; |
| 174 | return 0; |
| 175 | } |
| 176 | |
| 177 | if (!strcmp(subkey, "heuristic")) { |
| 178 | int i; |
| 179 | for (i = 0; i < BUNDLE_HEURISTIC__COUNT; i++) { |
| 180 | if (heuristics[i].heuristic && |
| 181 | heuristics[i].name && |
| 182 | !strcmp(value, heuristics[i].name)) { |
| 183 | list->heuristic = heuristics[i].heuristic; |
| 184 | return 0; |
| 185 | } |
| 186 | } |
| 187 | |
| 188 | /* Ignore unknown heuristics. */ |
| 189 | return 0; |
| 190 | } |
| 191 | |
| 192 | /* Ignore other unknown global keys. */ |
| 193 | return 0; |
| 194 | } |
| 195 | |
| 196 | strbuf_add(&id, subsection, subsection_len); |
| 197 | |
| 198 | /* |
| 199 | * Check for an existing bundle with this <id>, or create one |
| 200 | * if necessary. |
| 201 | */ |
| 202 | lookup.id = id.buf; |
| 203 | hashmap_entry_init(&lookup.ent, strhash(lookup.id)); |
| 204 | if (!(bundle = hashmap_get_entry(&list->bundles, &lookup, ent, NULL))) { |
| 205 | CALLOC_ARRAY(bundle, 1); |
| 206 | bundle->id = strbuf_detach(&id, NULL); |
| 207 | hashmap_entry_init(&bundle->ent, strhash(bundle->id)); |
| 208 | hashmap_add(&list->bundles, &bundle->ent); |
| 209 | } |
| 210 | strbuf_release(&id); |
| 211 | |
| 212 | if (!strcmp(subkey, "uri")) { |
| 213 | if (bundle->uri) |
| 214 | return -1; |
| 215 | bundle->uri = relative_url(list->baseURI, value, NULL); |
| 216 | return 0; |
| 217 | } |
| 218 | |
| 219 | if (!strcmp(subkey, "creationtoken")) { |
| 220 | if (sscanf(value, "%"PRIu64, &bundle->creationToken) != 1) |
| 221 | warning(_("could not parse bundle list key %s with value '%s'"), |
| 222 | "creationToken", value); |
| 223 | return 0; |
| 224 | } |
| 225 | |
| 226 | /* |
| 227 | * At this point, we ignore any information that we don't |
| 228 | * understand, assuming it to be hints for a heuristic the client |
| 229 | * does not currently understand. |
| 230 | */ |
| 231 | return 0; |
| 232 | } |
| 233 | |
| 234 | static int config_to_bundle_list(const char *key, const char *value, |
| 235 | const struct config_context *ctx UNUSED, |
| 236 | void *data) |
| 237 | { |
| 238 | struct bundle_list *list = data; |
| 239 | return bundle_list_update(key, value, list); |
| 240 | } |
| 241 | |
| 242 | int bundle_uri_parse_config_format(const char *uri, |
| 243 | const char *filename, |
| 244 | struct bundle_list *list) |
| 245 | { |
| 246 | int result; |
| 247 | struct config_options opts = { |
| 248 | .error_action = CONFIG_ERROR_ERROR, |
| 249 | }; |
| 250 | |
| 251 | if (!list->baseURI) { |
| 252 | struct strbuf baseURI = STRBUF_INIT; |
| 253 | strbuf_addstr(&baseURI, uri); |
| 254 | |
| 255 | /* |
| 256 | * If the URI does not end with a trailing slash, then |
| 257 | * remove the filename portion of the path. This is |
| 258 | * important for relative URIs. |
| 259 | */ |
| 260 | strbuf_strip_file_from_path(&baseURI); |
| 261 | list->baseURI = strbuf_detach(&baseURI, NULL); |
| 262 | } |
| 263 | result = git_config_from_file_with_options(config_to_bundle_list, |
| 264 | filename, list, |
| 265 | CONFIG_SCOPE_UNKNOWN, |
| 266 | &opts); |
| 267 | |
| 268 | if (!result && list->mode == BUNDLE_MODE_NONE) { |
| 269 | warning(_("bundle list at '%s' has no mode"), uri); |
| 270 | result = 1; |
| 271 | } |
| 272 | |
| 273 | if (!result) { |
| 274 | struct hashmap_iter iter; |
| 275 | struct remote_bundle_info *bundle; |
| 276 | |
| 277 | hashmap_for_each_entry(&list->bundles, &iter, bundle, ent) { |
| 278 | if (!bundle->uri) { |
| 279 | error(_("bundle list at '%s': bundle '%s' has no uri"), |
| 280 | uri, bundle->id ? bundle->id : "<unknown>"); |
| 281 | result = 1; |
| 282 | } |
| 283 | } |
| 284 | } |
| 285 | |
| 286 | return result; |
| 287 | } |
| 288 | |
| 289 | static char *find_temp_filename(void) |
| 290 | { |
| 291 | int fd; |
| 292 | struct strbuf name = STRBUF_INIT; |
| 293 | /* |
| 294 | * Find a temporary filename that is available. This is briefly |
| 295 | * racy, but unlikely to collide. |
| 296 | */ |
| 297 | fd = odb_mkstemp(the_repository->objects, &name, |
| 298 | "bundles/tmp_uri_XXXXXX"); |
| 299 | if (fd < 0) { |
| 300 | warning(_("failed to create temporary file")); |
| 301 | return NULL; |
| 302 | } |
| 303 | |
| 304 | close(fd); |
| 305 | unlink(name.buf); |
| 306 | return strbuf_detach(&name, NULL); |
| 307 | } |
| 308 | |
| 309 | static int download_https_uri_to_file(const char *file, const char *uri) |
| 310 | { |
| 311 | int result = 0; |
| 312 | struct child_process cp = CHILD_PROCESS_INIT; |
| 313 | FILE *child_in = NULL, *child_out = NULL; |
| 314 | struct strbuf line = STRBUF_INIT; |
| 315 | int found_get = 0; |
| 316 | |
| 317 | /* |
| 318 | * The protocol we speak with git-remote-https(1) uses a space to |
| 319 | * separate between URI and file, so the URI itself must not contain a |
| 320 | * space. If it did, an adversary could change the location where the |
| 321 | * downloaded file is being written to. |
| 322 | * |
| 323 | * Similarly, we use newlines to separate commands from one another. |
| 324 | * Consequently, neither the URI nor the file must contain a newline or |
| 325 | * otherwise an adversary could inject arbitrary commands. |
| 326 | * |
| 327 | * TODO: Restricting newlines in the target paths may break valid |
| 328 | * usecases, even if those are a bit more on the esoteric side. |
| 329 | * If this ever becomes a problem we should probably think about |
| 330 | * alternatives. One alternative could be to use NUL-delimited |
| 331 | * requests in git-remote-http(1). Another alternative could be |
| 332 | * to use URL quoting. |
| 333 | */ |
| 334 | if (strpbrk(uri, " \n")) |
| 335 | return error("bundle-uri: URI is malformed: '%s'", file); |
| 336 | if (strchr(file, '\n')) |
| 337 | return error("bundle-uri: filename is malformed: '%s'", file); |
| 338 | |
| 339 | strvec_pushl(&cp.args, "git-remote-https", uri, NULL); |
| 340 | cp.err = -1; |
| 341 | cp.in = -1; |
| 342 | cp.out = -1; |
| 343 | |
| 344 | if (start_command(&cp)) |
| 345 | return 1; |
| 346 | |
| 347 | child_in = fdopen(cp.in, "w"); |
| 348 | if (!child_in) { |
| 349 | result = 1; |
| 350 | goto cleanup; |
| 351 | } |
| 352 | |
| 353 | child_out = fdopen(cp.out, "r"); |
| 354 | if (!child_out) { |
| 355 | result = 1; |
| 356 | goto cleanup; |
| 357 | } |
| 358 | |
| 359 | fprintf(child_in, "capabilities\n"); |
| 360 | fflush(child_in); |
| 361 | |
| 362 | while (!strbuf_getline(&line, child_out)) { |
| 363 | if (!line.len) |
| 364 | break; |
| 365 | if (!strcmp(line.buf, "get")) |
| 366 | found_get = 1; |
| 367 | } |
| 368 | strbuf_release(&line); |
| 369 | |
| 370 | if (!found_get) { |
| 371 | result = error(_("insufficient capabilities")); |
| 372 | goto cleanup; |
| 373 | } |
| 374 | |
| 375 | fprintf(child_in, "get %s %s\n\n", uri, file); |
| 376 | |
| 377 | cleanup: |
| 378 | if (child_in) |
| 379 | fclose(child_in); |
| 380 | if (finish_command(&cp)) |
| 381 | result = 1; |
| 382 | if (child_out) |
| 383 | fclose(child_out); |
| 384 | return result; |
| 385 | } |
| 386 | |
| 387 | static int copy_uri_to_file(const char *filename, const char *uri) |
| 388 | { |
| 389 | const char *out; |
| 390 | |
| 391 | if (starts_with(uri, "https:") || |
| 392 | starts_with(uri, "http:")) |
| 393 | return download_https_uri_to_file(filename, uri); |
| 394 | |
| 395 | if (skip_prefix(uri, "file://", &out)) |
| 396 | uri = out; |
| 397 | |
| 398 | /* Copy as a file */ |
| 399 | return copy_file(the_repository, filename, uri, 0); |
| 400 | } |
| 401 | |
| 402 | static int unbundle_from_file(struct repository *r, const char *file) |
| 403 | { |
| 404 | int result = 0; |
| 405 | int bundle_fd; |
| 406 | struct bundle_header header = BUNDLE_HEADER_INIT; |
| 407 | struct string_list_item *refname; |
| 408 | struct strbuf bundle_ref = STRBUF_INIT; |
| 409 | size_t bundle_prefix_len; |
| 410 | struct unbundle_opts opts = { |
| 411 | .flags = VERIFY_BUNDLE_QUIET | |
| 412 | (fetch_pack_fsck_objects() ? VERIFY_BUNDLE_FSCK : 0), |
| 413 | }; |
| 414 | |
| 415 | bundle_fd = read_bundle_header(file, &header); |
| 416 | if (bundle_fd < 0) { |
| 417 | result = 1; |
| 418 | goto cleanup; |
| 419 | } |
| 420 | |
| 421 | /* |
| 422 | * Skip the reachability walk here, since we will be adding |
| 423 | * a reachable ref pointing to the new tips, which will reach |
| 424 | * the prerequisite commits. |
| 425 | */ |
| 426 | result = unbundle(r, &header, bundle_fd, NULL, &opts); |
| 427 | if (result) { |
| 428 | result = 1; |
| 429 | goto cleanup; |
| 430 | } |
| 431 | |
| 432 | /* |
| 433 | * Convert all refs/heads/ from the bundle into refs/bundles/ |
| 434 | * in the local repository. |
| 435 | */ |
| 436 | strbuf_addstr(&bundle_ref, "refs/bundles/"); |
| 437 | bundle_prefix_len = bundle_ref.len; |
| 438 | |
| 439 | for_each_string_list_item(refname, &header.references) { |
| 440 | struct object_id *oid = refname->util; |
| 441 | struct object_id old_oid; |
| 442 | const char *branch_name; |
| 443 | int has_old; |
| 444 | |
| 445 | if (!skip_prefix(refname->string, "refs/", &branch_name)) |
| 446 | continue; |
| 447 | |
| 448 | strbuf_setlen(&bundle_ref, bundle_prefix_len); |
| 449 | strbuf_addstr(&bundle_ref, branch_name); |
| 450 | |
| 451 | has_old = !refs_read_ref(get_main_ref_store(the_repository), |
| 452 | bundle_ref.buf, &old_oid); |
| 453 | refs_update_ref(get_main_ref_store(the_repository), |
| 454 | "fetched bundle", bundle_ref.buf, oid, |
| 455 | has_old ? &old_oid : NULL, |
| 456 | 0, UPDATE_REFS_MSG_ON_ERR); |
| 457 | } |
| 458 | |
| 459 | cleanup: |
| 460 | strbuf_release(&bundle_ref); |
| 461 | bundle_header_release(&header); |
| 462 | return result; |
| 463 | } |
| 464 | |
| 465 | struct bundle_list_context { |
| 466 | struct repository *r; |
| 467 | struct bundle_list *list; |
| 468 | enum bundle_list_mode mode; |
| 469 | int count; |
| 470 | int depth; |
| 471 | }; |
| 472 | |
| 473 | /* |
| 474 | * This early definition is necessary because we use indirect recursion: |
| 475 | * |
| 476 | * While iterating through a bundle list that was downloaded as part |
| 477 | * of fetch_bundle_uri_internal(), iterator methods eventually call it |
| 478 | * again, but with depth + 1. |
| 479 | */ |
| 480 | static int fetch_bundle_uri_internal(struct repository *r, |
| 481 | struct remote_bundle_info *bundle, |
| 482 | int depth, |
| 483 | struct bundle_list *list); |
| 484 | |
| 485 | static int download_bundle_to_file(struct remote_bundle_info *bundle, void *data) |
| 486 | { |
| 487 | int res; |
| 488 | struct bundle_list_context *ctx = data; |
| 489 | |
| 490 | if (ctx->mode == BUNDLE_MODE_ANY && ctx->count) |
| 491 | return 0; |
| 492 | |
| 493 | res = fetch_bundle_uri_internal(ctx->r, bundle, ctx->depth + 1, ctx->list); |
| 494 | |
| 495 | /* |
| 496 | * Only increment count if the download succeeded. If our mode is |
| 497 | * BUNDLE_MODE_ANY, then we will want to try other URIs in the |
| 498 | * list in case they work instead. |
| 499 | */ |
| 500 | if (!res) |
| 501 | ctx->count++; |
| 502 | |
| 503 | /* |
| 504 | * To be opportunistic as possible, we continue iterating and |
| 505 | * download as many bundles as we can, so we can apply the ones |
| 506 | * that work, even in BUNDLE_MODE_ALL mode. |
| 507 | */ |
| 508 | return 0; |
| 509 | } |
| 510 | |
| 511 | struct bundles_for_sorting { |
| 512 | struct remote_bundle_info **items; |
| 513 | size_t alloc; |
| 514 | size_t nr; |
| 515 | }; |
| 516 | |
| 517 | static int append_bundle(struct remote_bundle_info *bundle, void *data) |
| 518 | { |
| 519 | struct bundles_for_sorting *list = data; |
| 520 | list->items[list->nr++] = bundle; |
| 521 | return 0; |
| 522 | } |
| 523 | |
| 524 | /** |
| 525 | * For use in QSORT() to get a list sorted by creationToken |
| 526 | * in decreasing order. |
| 527 | */ |
| 528 | static int compare_creation_token_decreasing(const void *va, const void *vb) |
| 529 | { |
| 530 | const struct remote_bundle_info * const *a = va; |
| 531 | const struct remote_bundle_info * const *b = vb; |
| 532 | |
| 533 | if ((*a)->creationToken > (*b)->creationToken) |
| 534 | return -1; |
| 535 | if ((*a)->creationToken < (*b)->creationToken) |
| 536 | return 1; |
| 537 | return 0; |
| 538 | } |
| 539 | |
| 540 | static int fetch_bundles_by_token(struct repository *r, |
| 541 | struct bundle_list *list) |
| 542 | { |
| 543 | int cur; |
| 544 | int move_direction = 0; |
| 545 | const char *creationTokenStr; |
| 546 | uint64_t maxCreationToken = 0, newMaxCreationToken = 0; |
| 547 | struct bundle_list_context ctx = { |
| 548 | .r = r, |
| 549 | .list = list, |
| 550 | .mode = list->mode, |
| 551 | }; |
| 552 | struct bundles_for_sorting bundles = { |
| 553 | .alloc = hashmap_get_size(&list->bundles), |
| 554 | }; |
| 555 | |
| 556 | ALLOC_ARRAY(bundles.items, bundles.alloc); |
| 557 | |
| 558 | for_all_bundles_in_list(list, append_bundle, &bundles); |
| 559 | |
| 560 | if (!bundles.nr) { |
| 561 | free(bundles.items); |
| 562 | return 0; |
| 563 | } |
| 564 | |
| 565 | QSORT(bundles.items, bundles.nr, compare_creation_token_decreasing); |
| 566 | |
| 567 | /* |
| 568 | * If fetch.bundleCreationToken exists, parses to a uint64t, and |
| 569 | * is not strictly smaller than the maximum creation token in the |
| 570 | * bundle list, then do not download any bundles. |
| 571 | */ |
| 572 | if (!repo_config_get_value(r, |
| 573 | "fetch.bundlecreationtoken", |
| 574 | &creationTokenStr)) { |
| 575 | if (sscanf(creationTokenStr, "%"PRIu64, &maxCreationToken) != 1) |
| 576 | maxCreationToken = 0; |
| 577 | if (bundles.items[0]->creationToken <= maxCreationToken) { |
| 578 | free(bundles.items); |
| 579 | return 0; |
| 580 | } |
| 581 | } |
| 582 | |
| 583 | /* |
| 584 | * Attempt to download and unbundle the minimum number of bundles by |
| 585 | * creationToken in decreasing order. If we fail to unbundle (after |
| 586 | * a successful download) then move to the next non-downloaded bundle |
| 587 | * and attempt downloading. Once we succeed in applying a bundle, |
| 588 | * move to the previous unapplied bundle and attempt to unbundle it |
| 589 | * again. |
| 590 | * |
| 591 | * In the case of a fresh clone, we will likely download all of the |
| 592 | * bundles before successfully unbundling the oldest one, then the |
| 593 | * rest of the bundles unbundle successfully in increasing order |
| 594 | * of creationToken. |
| 595 | * |
| 596 | * If there are existing objects, then this process may terminate |
| 597 | * early when all required commits from "new" bundles exist in the |
| 598 | * repo's object store. |
| 599 | */ |
| 600 | cur = 0; |
| 601 | while (cur >= 0 && cur < bundles.nr) { |
| 602 | struct remote_bundle_info *bundle = bundles.items[cur]; |
| 603 | |
| 604 | /* |
| 605 | * If we need to dig into bundles below the previous |
| 606 | * creation token value, then likely we are in an erroneous |
| 607 | * state due to missing or invalid bundles. Halt the process |
| 608 | * instead of continuing to download extra data. |
| 609 | */ |
| 610 | if (bundle->creationToken <= maxCreationToken) |
| 611 | break; |
| 612 | |
| 613 | if (!bundle->file) { |
| 614 | /* |
| 615 | * Not downloaded yet. Try downloading. |
| 616 | * |
| 617 | * Note that bundle->file is non-NULL if a download |
| 618 | * was attempted, even if it failed to download. |
| 619 | */ |
| 620 | if (fetch_bundle_uri_internal(ctx.r, bundle, ctx.depth + 1, ctx.list)) { |
| 621 | /* Mark as unbundled so we do not retry. */ |
| 622 | bundle->unbundled = 1; |
| 623 | |
| 624 | /* Try looking deeper in the list. */ |
| 625 | move_direction = 1; |
| 626 | goto move; |
| 627 | } |
| 628 | |
| 629 | /* We expect bundles when using creationTokens. */ |
| 630 | if (!is_bundle(bundle->file, 1)) { |
| 631 | warning(_("file downloaded from '%s' is not a bundle"), |
| 632 | bundle->uri); |
| 633 | break; |
| 634 | } |
| 635 | } |
| 636 | |
| 637 | if (bundle->file && !bundle->unbundled) { |
| 638 | /* |
| 639 | * This was downloaded, but not successfully |
| 640 | * unbundled. Try unbundling again. |
| 641 | */ |
| 642 | if (unbundle_from_file(ctx.r, bundle->file)) { |
| 643 | /* Try looking deeper in the list. */ |
| 644 | move_direction = 1; |
| 645 | } else { |
| 646 | /* |
| 647 | * Succeeded in unbundle. Retry bundles |
| 648 | * that previously failed to unbundle. |
| 649 | */ |
| 650 | move_direction = -1; |
| 651 | bundle->unbundled = 1; |
| 652 | |
| 653 | if (bundle->creationToken > newMaxCreationToken) |
| 654 | newMaxCreationToken = bundle->creationToken; |
| 655 | } |
| 656 | } |
| 657 | |
| 658 | /* |
| 659 | * Else case: downloaded and unbundled successfully. |
| 660 | * Skip this by moving in the same direction as the |
| 661 | * previous step. |
| 662 | */ |
| 663 | |
| 664 | move: |
| 665 | /* Move in the specified direction and repeat. */ |
| 666 | cur += move_direction; |
| 667 | } |
| 668 | |
| 669 | /* |
| 670 | * We succeed if the loop terminates because 'cur' drops below |
| 671 | * zero. The other case is that we terminate because 'cur' |
| 672 | * reaches the end of the list, so we have a failure no matter |
| 673 | * which bundles we apply from the list. |
| 674 | */ |
| 675 | if (cur < 0) { |
| 676 | struct strbuf value = STRBUF_INIT; |
| 677 | strbuf_addf(&value, "%"PRIu64"", newMaxCreationToken); |
| 678 | if (repo_config_set_multivar_gently(ctx.r, |
| 679 | "fetch.bundleCreationToken", |
| 680 | value.buf, NULL, 0)) |
| 681 | warning(_("failed to store maximum creation token")); |
| 682 | |
| 683 | strbuf_release(&value); |
| 684 | } |
| 685 | |
| 686 | free(bundles.items); |
| 687 | return cur >= 0; |
| 688 | } |
| 689 | |
| 690 | static int download_bundle_list(struct repository *r, |
| 691 | struct bundle_list *local_list, |
| 692 | struct bundle_list *global_list, |
| 693 | int depth) |
| 694 | { |
| 695 | struct bundle_list_context ctx = { |
| 696 | .r = r, |
| 697 | .list = global_list, |
| 698 | .depth = depth + 1, |
| 699 | .mode = local_list->mode, |
| 700 | }; |
| 701 | |
| 702 | return for_all_bundles_in_list(local_list, download_bundle_to_file, &ctx); |
| 703 | } |
| 704 | |
| 705 | static int fetch_bundle_list_in_config_format(struct repository *r, |
| 706 | struct bundle_list *global_list, |
| 707 | struct remote_bundle_info *bundle, |
| 708 | int depth) |
| 709 | { |
| 710 | int result; |
| 711 | struct bundle_list list_from_bundle; |
| 712 | |
| 713 | init_bundle_list(&list_from_bundle); |
| 714 | |
| 715 | if ((result = bundle_uri_parse_config_format(bundle->uri, |
| 716 | bundle->file, |
| 717 | &list_from_bundle))) |
| 718 | goto cleanup; |
| 719 | |
| 720 | if (list_from_bundle.mode == BUNDLE_MODE_NONE) { |
| 721 | warning(_("unrecognized bundle mode from URI '%s'"), |
| 722 | bundle->uri); |
| 723 | result = -1; |
| 724 | goto cleanup; |
| 725 | } |
| 726 | |
| 727 | /* |
| 728 | * If this list uses the creationToken heuristic, then the URIs |
| 729 | * it advertises are expected to be bundles, not nested lists. |
| 730 | * We can drop 'global_list' and 'depth'. |
| 731 | */ |
| 732 | if (list_from_bundle.heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN) { |
| 733 | result = fetch_bundles_by_token(r, &list_from_bundle); |
| 734 | global_list->heuristic = BUNDLE_HEURISTIC_CREATIONTOKEN; |
| 735 | } else if ((result = download_bundle_list(r, &list_from_bundle, |
| 736 | global_list, depth))) |
| 737 | goto cleanup; |
| 738 | |
| 739 | cleanup: |
| 740 | clear_bundle_list(&list_from_bundle); |
| 741 | return result; |
| 742 | } |
| 743 | |
| 744 | /** |
| 745 | * This limits the recursion on fetch_bundle_uri_internal() when following |
| 746 | * bundle lists. |
| 747 | */ |
| 748 | static int max_bundle_uri_depth = 4; |
| 749 | |
| 750 | /** |
| 751 | * Recursively download all bundles advertised at the given URI |
| 752 | * to files. If the file is a bundle, then add it to the given |
| 753 | * 'list'. Otherwise, expect a bundle list and recurse on the |
| 754 | * URIs in that list according to the list mode (ANY or ALL). |
| 755 | */ |
| 756 | static int fetch_bundle_uri_internal(struct repository *r, |
| 757 | struct remote_bundle_info *bundle, |
| 758 | int depth, |
| 759 | struct bundle_list *list) |
| 760 | { |
| 761 | int result = 0; |
| 762 | struct remote_bundle_info *bcopy; |
| 763 | |
| 764 | if (depth >= max_bundle_uri_depth) { |
| 765 | warning(_("exceeded bundle URI recursion limit (%d)"), |
| 766 | max_bundle_uri_depth); |
| 767 | return -1; |
| 768 | } |
| 769 | |
| 770 | if (!bundle->uri) { |
| 771 | error(_("bundle '%s' has no uri"), |
| 772 | bundle->id ? bundle->id : "<unknown>"); |
| 773 | return -1; |
| 774 | } |
| 775 | |
| 776 | if (!bundle->file && |
| 777 | !(bundle->file = find_temp_filename())) { |
| 778 | result = -1; |
| 779 | goto cleanup; |
| 780 | } |
| 781 | |
| 782 | if ((result = copy_uri_to_file(bundle->file, bundle->uri))) { |
| 783 | warning(_("failed to download bundle from URI '%s'"), bundle->uri); |
| 784 | goto cleanup; |
| 785 | } |
| 786 | |
| 787 | if ((result = !is_bundle(bundle->file, 1))) { |
| 788 | result = fetch_bundle_list_in_config_format( |
| 789 | r, list, bundle, depth); |
| 790 | if (result) |
| 791 | warning(_("file at URI '%s' is not a bundle or bundle list"), |
| 792 | bundle->uri); |
| 793 | goto cleanup; |
| 794 | } |
| 795 | |
| 796 | /* Copy the bundle and insert it into the global list. */ |
| 797 | CALLOC_ARRAY(bcopy, 1); |
| 798 | bcopy->id = xstrdup(bundle->id); |
| 799 | bcopy->file = xstrdup(bundle->file); |
| 800 | hashmap_entry_init(&bcopy->ent, strhash(bcopy->id)); |
| 801 | hashmap_add(&list->bundles, &bcopy->ent); |
| 802 | |
| 803 | cleanup: |
| 804 | if (result && bundle->file) |
| 805 | unlink(bundle->file); |
| 806 | return result; |
| 807 | } |
| 808 | |
| 809 | /** |
| 810 | * This loop iterator breaks the loop with nonzero return code on the |
| 811 | * first successful unbundling of a bundle. |
| 812 | */ |
| 813 | static int attempt_unbundle(struct remote_bundle_info *info, void *data) |
| 814 | { |
| 815 | struct repository *r = data; |
| 816 | |
| 817 | if (!info->file || info->unbundled) |
| 818 | return 0; |
| 819 | |
| 820 | if (!unbundle_from_file(r, info->file)) { |
| 821 | info->unbundled = 1; |
| 822 | return 1; |
| 823 | } |
| 824 | |
| 825 | return 0; |
| 826 | } |
| 827 | |
| 828 | static int unbundle_all_bundles(struct repository *r, |
| 829 | struct bundle_list *list) |
| 830 | { |
| 831 | /* |
| 832 | * Iterate through all bundles looking for ones that can |
| 833 | * successfully unbundle. If any succeed, then perhaps another |
| 834 | * will succeed in the next attempt. |
| 835 | * |
| 836 | * Keep in mind that a non-zero result for the loop here means |
| 837 | * the loop terminated early on a successful unbundling, which |
| 838 | * signals that we can try again. |
| 839 | */ |
| 840 | while (for_all_bundles_in_list(list, attempt_unbundle, r)) ; |
| 841 | |
| 842 | return 0; |
| 843 | } |
| 844 | |
| 845 | static int unlink_bundle(struct remote_bundle_info *info, void *data UNUSED) |
| 846 | { |
| 847 | if (info->file) |
| 848 | unlink_or_warn(info->file); |
| 849 | return 0; |
| 850 | } |
| 851 | |
| 852 | int fetch_bundle_uri(struct repository *r, const char *uri, |
| 853 | int *has_heuristic) |
| 854 | { |
| 855 | int result; |
| 856 | struct bundle_list list; |
| 857 | struct remote_bundle_info bundle = { |
| 858 | .uri = xstrdup(uri), |
| 859 | .id = xstrdup(""), |
| 860 | }; |
| 861 | |
| 862 | trace2_region_enter("fetch", "fetch-bundle-uri", the_repository); |
| 863 | |
| 864 | init_bundle_list(&list); |
| 865 | |
| 866 | /* |
| 867 | * Do not fetch an empty bundle URI. An empty bundle URI |
| 868 | * could signal that a configured bundle URI has been disabled. |
| 869 | */ |
| 870 | if (!*uri) { |
| 871 | result = 0; |
| 872 | goto cleanup; |
| 873 | } |
| 874 | |
| 875 | /* If a bundle is added to this global list, then it is required. */ |
| 876 | list.mode = BUNDLE_MODE_ALL; |
| 877 | |
| 878 | if ((result = fetch_bundle_uri_internal(r, &bundle, 0, &list))) |
| 879 | goto cleanup; |
| 880 | |
| 881 | result = unbundle_all_bundles(r, &list); |
| 882 | |
| 883 | cleanup: |
| 884 | if (has_heuristic) |
| 885 | *has_heuristic = (list.heuristic != BUNDLE_HEURISTIC_NONE); |
| 886 | for_all_bundles_in_list(&list, unlink_bundle, NULL); |
| 887 | clear_bundle_list(&list); |
| 888 | clear_remote_bundle_info(&bundle, NULL); |
| 889 | trace2_region_leave("fetch", "fetch-bundle-uri", the_repository); |
| 890 | return result; |
| 891 | } |
| 892 | |
| 893 | int fetch_bundle_list(struct repository *r, struct bundle_list *list) |
| 894 | { |
| 895 | int result; |
| 896 | struct bundle_list global_list; |
| 897 | |
| 898 | /* |
| 899 | * If the creationToken heuristic is used, then the URIs |
| 900 | * advertised by 'list' are not nested lists and instead |
| 901 | * direct bundles. We do not need to use global_list. |
| 902 | */ |
| 903 | if (list->heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN) |
| 904 | return fetch_bundles_by_token(r, list); |
| 905 | |
| 906 | init_bundle_list(&global_list); |
| 907 | |
| 908 | /* If a bundle is added to this global list, then it is required. */ |
| 909 | global_list.mode = BUNDLE_MODE_ALL; |
| 910 | |
| 911 | if ((result = download_bundle_list(r, list, &global_list, 0))) |
| 912 | goto cleanup; |
| 913 | |
| 914 | if (list->heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN) |
| 915 | result = fetch_bundles_by_token(r, list); |
| 916 | else |
| 917 | result = unbundle_all_bundles(r, &global_list); |
| 918 | |
| 919 | cleanup: |
| 920 | for_all_bundles_in_list(&global_list, unlink_bundle, NULL); |
| 921 | clear_bundle_list(&global_list); |
| 922 | return result; |
| 923 | } |
| 924 | |
| 925 | /** |
| 926 | * API for serve.c. |
| 927 | */ |
| 928 | |
| 929 | int bundle_uri_advertise(struct repository *r, struct strbuf *value UNUSED) |
| 930 | { |
| 931 | static int advertise_bundle_uri = -1; |
| 932 | |
| 933 | if (advertise_bundle_uri != -1) |
| 934 | goto cached; |
| 935 | |
| 936 | advertise_bundle_uri = 0; |
| 937 | repo_config_get_maybe_bool(r, "uploadpack.advertisebundleuris", &advertise_bundle_uri); |
| 938 | |
| 939 | cached: |
| 940 | return advertise_bundle_uri; |
| 941 | } |
| 942 | |
| 943 | static int config_to_packet_line(const char *key, const char *value, |
| 944 | const struct config_context *ctx UNUSED, |
| 945 | void *data) |
| 946 | { |
| 947 | struct packet_reader *writer = data; |
| 948 | |
| 949 | if (starts_with(key, "bundle.")) { |
| 950 | if (value && *value) |
| 951 | packet_write_fmt(writer->fd, "%s=%s", key, value); |
| 952 | else |
| 953 | warning(_("config '%s' has no value"), key); |
| 954 | } |
| 955 | |
| 956 | return 0; |
| 957 | } |
| 958 | |
| 959 | int bundle_uri_command(struct repository *r, |
| 960 | struct packet_reader *request) |
| 961 | { |
| 962 | struct packet_writer writer; |
| 963 | packet_writer_init(&writer, 1); |
| 964 | |
| 965 | while (packet_reader_read(request) == PACKET_READ_NORMAL) |
| 966 | die(_("bundle-uri: unexpected argument: '%s'"), request->line); |
| 967 | if (request->status != PACKET_READ_FLUSH) |
| 968 | die(_("bundle-uri: expected flush after arguments")); |
| 969 | |
| 970 | /* |
| 971 | * Read all "bundle.*" config lines to the client as key=value |
| 972 | * packet lines. |
| 973 | */ |
| 974 | repo_config(r, config_to_packet_line, &writer); |
| 975 | |
| 976 | packet_writer_flush(&writer); |
| 977 | |
| 978 | return 0; |
| 979 | } |
| 980 | |
| 981 | /** |
| 982 | * General API for {transport,connect}.c etc. |
| 983 | */ |
| 984 | int bundle_uri_parse_line(struct bundle_list *list, const char *line) |
| 985 | { |
| 986 | int result; |
| 987 | const char *equals; |
| 988 | struct strbuf key = STRBUF_INIT; |
| 989 | |
| 990 | if (!strlen(line)) |
| 991 | return error(_("bundle-uri: got an empty line")); |
| 992 | |
| 993 | equals = strchr(line, '='); |
| 994 | |
| 995 | if (!equals) |
| 996 | return error(_("bundle-uri: line is not of the form 'key=value'")); |
| 997 | if (line == equals || !*(equals + 1)) |
| 998 | return error(_("bundle-uri: line has empty key or value")); |
| 999 | |
| 1000 | strbuf_add(&key, line, equals - line); |
| 1001 | result = bundle_list_update(key.buf, equals + 1, list); |
| 1002 | strbuf_release(&key); |
| 1003 | |
| 1004 | return result; |
| 1005 | } |