| 1 | #include "git-compat-util.h" |
| 2 | #include "repack.h" |
| 3 | #include "hash.h" |
| 4 | #include "hex.h" |
| 5 | #include "lockfile.h" |
| 6 | #include "midx.h" |
| 7 | #include "odb.h" |
| 8 | #include "oidset.h" |
| 9 | #include "pack-bitmap.h" |
| 10 | #include "path.h" |
| 11 | #include "refs.h" |
| 12 | #include "run-command.h" |
| 13 | #include "tempfile.h" |
| 14 | #include "trace2.h" |
| 15 | |
| 16 | struct midx_snapshot_ref_data { |
| 17 | struct repository *repo; |
| 18 | struct tempfile *f; |
| 19 | struct oidset seen; |
| 20 | int preferred; |
| 21 | }; |
| 22 | |
| 23 | static int midx_snapshot_ref_one(const struct reference *ref, void *_data) |
| 24 | { |
| 25 | struct midx_snapshot_ref_data *data = _data; |
| 26 | const struct object_id *maybe_peeled = ref->oid; |
| 27 | struct object_id peeled; |
| 28 | |
| 29 | if (!reference_get_peeled_oid(data->repo, ref, &peeled)) |
| 30 | maybe_peeled = &peeled; |
| 31 | |
| 32 | if (oidset_insert(&data->seen, maybe_peeled)) |
| 33 | return 0; /* already seen */ |
| 34 | |
| 35 | if (odb_read_object_info(data->repo->objects, maybe_peeled, NULL) != OBJ_COMMIT) |
| 36 | return 0; |
| 37 | |
| 38 | fprintf(data->f->fp, "%s%s\n", data->preferred ? "+" : "", |
| 39 | oid_to_hex(maybe_peeled)); |
| 40 | |
| 41 | return 0; |
| 42 | } |
| 43 | |
| 44 | void midx_snapshot_refs(struct repository *repo, struct tempfile *f) |
| 45 | { |
| 46 | struct midx_snapshot_ref_data data; |
| 47 | |
| 48 | data.repo = repo; |
| 49 | data.f = f; |
| 50 | data.preferred = 0; |
| 51 | oidset_init(&data.seen, 0); |
| 52 | |
| 53 | if (!fdopen_tempfile(f, "w")) |
| 54 | die(_("could not open tempfile %s for writing"), |
| 55 | get_tempfile_path(f)); |
| 56 | |
| 57 | data.preferred = 1; |
| 58 | for_each_preferred_bitmap_tip(repo, midx_snapshot_ref_one, &data); |
| 59 | data.preferred = 0; |
| 60 | |
| 61 | refs_for_each_ref(get_main_ref_store(repo), |
| 62 | midx_snapshot_ref_one, &data); |
| 63 | |
| 64 | if (close_tempfile_gently(f)) { |
| 65 | int save_errno = errno; |
| 66 | delete_tempfile(&f); |
| 67 | errno = save_errno; |
| 68 | die_errno(_("could not close refs snapshot tempfile")); |
| 69 | } |
| 70 | |
| 71 | oidset_clear(&data.seen); |
| 72 | } |
| 73 | |
| 74 | static int midx_has_unknown_packs(struct string_list *include, |
| 75 | struct pack_geometry *geometry, |
| 76 | struct existing_packs *existing) |
| 77 | { |
| 78 | struct string_list_item *item; |
| 79 | |
| 80 | string_list_sort(include); |
| 81 | |
| 82 | for_each_string_list_item(item, &existing->midx_packs) { |
| 83 | const char *pack_name = item->string; |
| 84 | |
| 85 | /* |
| 86 | * Determine whether or not each MIDX'd pack from the existing |
| 87 | * MIDX (if any) is represented in the new MIDX. For each pack |
| 88 | * in the MIDX, it must either be: |
| 89 | * |
| 90 | * - In the "include" list of packs to be included in the new |
| 91 | * MIDX. Note this function is called before the include |
| 92 | * list is populated with any cruft pack(s). |
| 93 | * |
| 94 | * - Below the geometric split line (if using pack geometry), |
| 95 | * indicating that the pack won't be included in the new |
| 96 | * MIDX, but its contents were rolled up as part of the |
| 97 | * geometric repack. |
| 98 | * |
| 99 | * - In the existing non-kept packs list (if not using pack |
| 100 | * geometry), and marked as non-deleted. |
| 101 | */ |
| 102 | if (string_list_has_string(include, pack_name)) { |
| 103 | continue; |
| 104 | } else if (geometry) { |
| 105 | struct strbuf buf = STRBUF_INIT; |
| 106 | uint32_t j; |
| 107 | |
| 108 | for (j = 0; j < geometry->split; j++) { |
| 109 | strbuf_reset(&buf); |
| 110 | strbuf_addstr(&buf, pack_basename(geometry->pack[j])); |
| 111 | strbuf_strip_suffix(&buf, ".pack"); |
| 112 | strbuf_addstr(&buf, ".idx"); |
| 113 | |
| 114 | if (!strcmp(pack_name, buf.buf)) { |
| 115 | strbuf_release(&buf); |
| 116 | break; |
| 117 | } |
| 118 | } |
| 119 | |
| 120 | strbuf_release(&buf); |
| 121 | |
| 122 | if (j < geometry->split) |
| 123 | continue; |
| 124 | } else { |
| 125 | struct string_list_item *item; |
| 126 | |
| 127 | item = string_list_lookup(&existing->non_kept_packs, |
| 128 | pack_name); |
| 129 | if (item && !existing_pack_is_marked_for_deletion(item)) |
| 130 | continue; |
| 131 | } |
| 132 | |
| 133 | /* |
| 134 | * If we got to this point, the MIDX includes some pack that we |
| 135 | * don't know about. |
| 136 | */ |
| 137 | return 1; |
| 138 | } |
| 139 | |
| 140 | return 0; |
| 141 | } |
| 142 | |
| 143 | static void midx_included_packs(struct string_list *include, |
| 144 | struct repack_write_midx_opts *opts) |
| 145 | { |
| 146 | struct existing_packs *existing = opts->existing; |
| 147 | struct pack_geometry *geometry = opts->geometry; |
| 148 | struct string_list *names = opts->names; |
| 149 | struct string_list_item *item; |
| 150 | struct strbuf buf = STRBUF_INIT; |
| 151 | |
| 152 | for_each_string_list_item(item, &existing->kept_packs) { |
| 153 | strbuf_reset(&buf); |
| 154 | strbuf_addf(&buf, "%s.idx", item->string); |
| 155 | string_list_insert(include, buf.buf); |
| 156 | } |
| 157 | |
| 158 | for_each_string_list_item(item, names) { |
| 159 | strbuf_reset(&buf); |
| 160 | strbuf_addf(&buf, "pack-%s.idx", item->string); |
| 161 | string_list_insert(include, buf.buf); |
| 162 | } |
| 163 | |
| 164 | if (geometry->split_factor) { |
| 165 | uint32_t i; |
| 166 | |
| 167 | for (i = geometry->split; i < geometry->pack_nr; i++) { |
| 168 | struct packed_git *p = geometry->pack[i]; |
| 169 | |
| 170 | /* |
| 171 | * The multi-pack index never refers to packfiles part |
| 172 | * of an alternate object database, so we skip these. |
| 173 | * While git-multi-pack-index(1) would silently ignore |
| 174 | * them anyway, this allows us to skip executing the |
| 175 | * command completely when we have only non-local |
| 176 | * packfiles. |
| 177 | */ |
| 178 | if (!p->pack_local) |
| 179 | continue; |
| 180 | |
| 181 | strbuf_reset(&buf); |
| 182 | strbuf_addstr(&buf, pack_basename(p)); |
| 183 | strbuf_strip_suffix(&buf, ".pack"); |
| 184 | strbuf_addstr(&buf, ".idx"); |
| 185 | |
| 186 | string_list_insert(include, buf.buf); |
| 187 | } |
| 188 | } else { |
| 189 | for_each_string_list_item(item, &existing->non_kept_packs) { |
| 190 | if (existing_pack_is_marked_for_deletion(item)) |
| 191 | continue; |
| 192 | |
| 193 | strbuf_reset(&buf); |
| 194 | strbuf_addf(&buf, "%s.idx", item->string); |
| 195 | string_list_insert(include, buf.buf); |
| 196 | } |
| 197 | } |
| 198 | |
| 199 | if (opts->midx_must_contain_cruft || |
| 200 | midx_has_unknown_packs(include, geometry, existing)) { |
| 201 | /* |
| 202 | * If there are one or more unknown pack(s) present (see |
| 203 | * midx_has_unknown_packs() for what makes a pack |
| 204 | * "unknown") in the MIDX before the repack, keep them |
| 205 | * as they may be required to form a reachability |
| 206 | * closure if the MIDX is bitmapped. |
| 207 | * |
| 208 | * For example, a cruft pack can be required to form a |
| 209 | * reachability closure if the MIDX is bitmapped and one |
| 210 | * or more of the bitmap's selected commits reaches a |
| 211 | * once-cruft object that was later made reachable. |
| 212 | */ |
| 213 | for_each_string_list_item(item, &existing->cruft_packs) { |
| 214 | /* |
| 215 | * When doing a --geometric repack, there is no |
| 216 | * need to check for deleted packs, since we're |
| 217 | * by definition not doing an ALL_INTO_ONE |
| 218 | * repack (hence no packs will be deleted). |
| 219 | * Otherwise we must check for and exclude any |
| 220 | * packs which are enqueued for deletion. |
| 221 | * |
| 222 | * So we could omit the conditional below in the |
| 223 | * --geometric case, but doing so is unnecessary |
| 224 | * since no packs are marked as pending |
| 225 | * deletion (since we only call |
| 226 | * `existing_packs_mark_for_deletion()` when |
| 227 | * doing an all-into-one repack). |
| 228 | */ |
| 229 | if (existing_pack_is_marked_for_deletion(item)) |
| 230 | continue; |
| 231 | |
| 232 | strbuf_reset(&buf); |
| 233 | strbuf_addf(&buf, "%s.idx", item->string); |
| 234 | string_list_insert(include, buf.buf); |
| 235 | } |
| 236 | } else { |
| 237 | /* |
| 238 | * Modern versions of Git (with the appropriate |
| 239 | * configuration setting) will write new copies of |
| 240 | * once-cruft objects when doing a --geometric repack. |
| 241 | * |
| 242 | * If the MIDX has no cruft pack, new packs written |
| 243 | * during a --geometric repack will not rely on the |
| 244 | * cruft pack to form a reachability closure, so we can |
| 245 | * avoid including them in the MIDX in that case. |
| 246 | */ |
| 247 | ; |
| 248 | } |
| 249 | |
| 250 | strbuf_release(&buf); |
| 251 | } |
| 252 | |
| 253 | static void remove_redundant_bitmaps(struct string_list *include, |
| 254 | const char *packdir) |
| 255 | { |
| 256 | struct strbuf path = STRBUF_INIT; |
| 257 | struct string_list_item *item; |
| 258 | size_t packdir_len; |
| 259 | |
| 260 | strbuf_addstr(&path, packdir); |
| 261 | strbuf_addch(&path, '/'); |
| 262 | packdir_len = path.len; |
| 263 | |
| 264 | /* |
| 265 | * Remove any pack bitmaps corresponding to packs which are now |
| 266 | * included in the MIDX. |
| 267 | */ |
| 268 | for_each_string_list_item(item, include) { |
| 269 | strbuf_addstr(&path, item->string); |
| 270 | strbuf_strip_suffix(&path, ".idx"); |
| 271 | strbuf_addstr(&path, ".bitmap"); |
| 272 | |
| 273 | if (unlink(path.buf) && errno != ENOENT) |
| 274 | warning_errno(_("could not remove stale bitmap: %s"), |
| 275 | path.buf); |
| 276 | |
| 277 | strbuf_setlen(&path, packdir_len); |
| 278 | } |
| 279 | strbuf_release(&path); |
| 280 | } |
| 281 | |
| 282 | static void repack_prepare_midx_command(struct child_process *cmd, |
| 283 | struct repack_write_midx_opts *opts, |
| 284 | const char *subcommand) |
| 285 | { |
| 286 | cmd->git_cmd = 1; |
| 287 | |
| 288 | strvec_pushl(&cmd->args, "multi-pack-index", subcommand, NULL); |
| 289 | |
| 290 | if (opts->show_progress) |
| 291 | strvec_push(&cmd->args, "--progress"); |
| 292 | else |
| 293 | strvec_push(&cmd->args, "--no-progress"); |
| 294 | |
| 295 | if (opts->write_bitmaps) |
| 296 | strvec_push(&cmd->args, "--bitmap"); |
| 297 | } |
| 298 | |
| 299 | static int repack_fill_midx_stdin_packs(struct child_process *cmd, |
| 300 | struct string_list *include, |
| 301 | struct string_list *out) |
| 302 | { |
| 303 | struct strbuf in_buf = STRBUF_INIT; |
| 304 | struct strbuf out_buf = STRBUF_INIT; |
| 305 | struct string_list_item *item; |
| 306 | int ret; |
| 307 | |
| 308 | strvec_push(&cmd->args, "--stdin-packs"); |
| 309 | |
| 310 | for_each_string_list_item(item, include) |
| 311 | strbuf_addf(&in_buf, "%s\n", item->string); |
| 312 | |
| 313 | ret = pipe_command(cmd, in_buf.buf, in_buf.len, |
| 314 | out ? &out_buf : NULL, 0, NULL, 0); |
| 315 | |
| 316 | if (out) |
| 317 | string_list_split_f(out, out_buf.buf, "\n", -1, |
| 318 | STRING_LIST_SPLIT_NONEMPTY); |
| 319 | |
| 320 | strbuf_release(&in_buf); |
| 321 | strbuf_release(&out_buf); |
| 322 | |
| 323 | return ret; |
| 324 | } |
| 325 | |
| 326 | static int write_midx_included_packs(struct repack_write_midx_opts *opts) |
| 327 | { |
| 328 | struct child_process cmd = CHILD_PROCESS_INIT; |
| 329 | struct string_list include = STRING_LIST_INIT_DUP; |
| 330 | struct string_list_item *item; |
| 331 | struct packed_git *preferred = pack_geometry_preferred_pack(opts->geometry); |
| 332 | int ret = 0; |
| 333 | |
| 334 | midx_included_packs(&include, opts); |
| 335 | if (!include.nr) |
| 336 | goto done; |
| 337 | |
| 338 | repack_prepare_midx_command(&cmd, opts, "write"); |
| 339 | |
| 340 | if (preferred) |
| 341 | strvec_pushf(&cmd.args, "--preferred-pack=%s", |
| 342 | pack_basename(preferred)); |
| 343 | else if (opts->names->nr) { |
| 344 | /* The largest pack was repacked, meaning that either |
| 345 | * one or two packs exist depending on whether the |
| 346 | * repository has a cruft pack or not. |
| 347 | * |
| 348 | * Select the non-cruft one as preferred to encourage |
| 349 | * pack-reuse among packs containing reachable objects |
| 350 | * over unreachable ones. |
| 351 | * |
| 352 | * (Note we could write multiple packs here if |
| 353 | * `--max-pack-size` was given, but any one of them |
| 354 | * will suffice, so pick the first one.) |
| 355 | */ |
| 356 | for_each_string_list_item(item, opts->names) { |
| 357 | struct generated_pack *pack = item->util; |
| 358 | if (generated_pack_has_ext(pack, ".mtimes")) |
| 359 | continue; |
| 360 | |
| 361 | strvec_pushf(&cmd.args, "--preferred-pack=pack-%s.pack", |
| 362 | item->string); |
| 363 | break; |
| 364 | } |
| 365 | } else { |
| 366 | /* |
| 367 | * No packs were kept, and no packs were written. The |
| 368 | * only thing remaining are .keep packs (unless |
| 369 | * --pack-kept-objects was given). |
| 370 | * |
| 371 | * Set the `--preferred-pack` arbitrarily here. |
| 372 | */ |
| 373 | ; |
| 374 | } |
| 375 | |
| 376 | if (opts->refs_snapshot) |
| 377 | strvec_pushf(&cmd.args, "--refs-snapshot=%s", |
| 378 | opts->refs_snapshot); |
| 379 | |
| 380 | ret = repack_fill_midx_stdin_packs(&cmd, &include, NULL); |
| 381 | done: |
| 382 | if (!ret && opts->write_bitmaps) |
| 383 | remove_redundant_bitmaps(&include, opts->packdir); |
| 384 | |
| 385 | string_list_clear(&include, 0); |
| 386 | |
| 387 | return ret; |
| 388 | } |
| 389 | |
| 390 | struct midx_compaction_step { |
| 391 | union { |
| 392 | struct multi_pack_index *copy; |
| 393 | struct string_list write; |
| 394 | struct { |
| 395 | struct multi_pack_index *from; |
| 396 | struct multi_pack_index *to; |
| 397 | } compact; |
| 398 | } u; |
| 399 | |
| 400 | uint32_t objects_nr; |
| 401 | char *csum; |
| 402 | |
| 403 | enum { |
| 404 | MIDX_COMPACTION_STEP_UNKNOWN, |
| 405 | MIDX_COMPACTION_STEP_COPY, |
| 406 | MIDX_COMPACTION_STEP_WRITE, |
| 407 | MIDX_COMPACTION_STEP_COMPACT, |
| 408 | } type; |
| 409 | }; |
| 410 | |
| 411 | static const char *midx_compaction_step_base(const struct midx_compaction_step *step) |
| 412 | { |
| 413 | switch (step->type) { |
| 414 | case MIDX_COMPACTION_STEP_UNKNOWN: |
| 415 | BUG("cannot use UNKNOWN step as a base"); |
| 416 | case MIDX_COMPACTION_STEP_COPY: |
| 417 | return midx_get_checksum_hex(step->u.copy); |
| 418 | case MIDX_COMPACTION_STEP_WRITE: |
| 419 | BUG("cannot use WRITE step as a base"); |
| 420 | case MIDX_COMPACTION_STEP_COMPACT: |
| 421 | return midx_get_checksum_hex(step->u.compact.to); |
| 422 | default: |
| 423 | BUG("unhandled midx compaction step type %d", step->type); |
| 424 | } |
| 425 | } |
| 426 | |
| 427 | static int midx_compaction_step_exec_copy(struct midx_compaction_step *step) |
| 428 | { |
| 429 | step->csum = xstrdup(midx_get_checksum_hex(step->u.copy)); |
| 430 | return 0; |
| 431 | } |
| 432 | |
| 433 | static int midx_compaction_step_exec_write(struct midx_compaction_step *step, |
| 434 | struct repack_write_midx_opts *opts, |
| 435 | const char *base) |
| 436 | { |
| 437 | struct child_process cmd = CHILD_PROCESS_INIT; |
| 438 | struct string_list hash = STRING_LIST_INIT_DUP; |
| 439 | struct string_list_item *item; |
| 440 | const char *preferred_pack = NULL; |
| 441 | int ret = 0; |
| 442 | |
| 443 | if (!step->u.write.nr) { |
| 444 | ret = error(_("no packs to write MIDX during compaction")); |
| 445 | goto out; |
| 446 | } |
| 447 | |
| 448 | for_each_string_list_item(item, &step->u.write) { |
| 449 | if (item->util) |
| 450 | preferred_pack = item->string; |
| 451 | } |
| 452 | |
| 453 | repack_prepare_midx_command(&cmd, opts, "write"); |
| 454 | strvec_pushl(&cmd.args, "--incremental", "--no-write-chain-file", NULL); |
| 455 | strvec_pushf(&cmd.args, "--base=%s", base ? base : "none"); |
| 456 | |
| 457 | if (preferred_pack) { |
| 458 | struct strbuf buf = STRBUF_INIT; |
| 459 | |
| 460 | strbuf_addstr(&buf, preferred_pack); |
| 461 | strbuf_strip_suffix(&buf, ".idx"); |
| 462 | strbuf_addstr(&buf, ".pack"); |
| 463 | |
| 464 | strvec_pushf(&cmd.args, "--preferred-pack=%s", buf.buf); |
| 465 | |
| 466 | strbuf_release(&buf); |
| 467 | } |
| 468 | |
| 469 | ret = repack_fill_midx_stdin_packs(&cmd, &step->u.write, &hash); |
| 470 | if (hash.nr != 1) { |
| 471 | ret = error(_("expected exactly one line during MIDX write, " |
| 472 | "got: %"PRIuMAX), |
| 473 | (uintmax_t)hash.nr); |
| 474 | goto out; |
| 475 | } |
| 476 | |
| 477 | step->csum = xstrdup(hash.items[0].string); |
| 478 | |
| 479 | out: |
| 480 | string_list_clear(&hash, 0); |
| 481 | |
| 482 | return ret; |
| 483 | } |
| 484 | |
| 485 | static int midx_compaction_step_exec_compact(struct midx_compaction_step *step, |
| 486 | struct repack_write_midx_opts *opts) |
| 487 | { |
| 488 | struct child_process cmd = CHILD_PROCESS_INIT; |
| 489 | struct strbuf buf = STRBUF_INIT; |
| 490 | FILE *out = NULL; |
| 491 | int ret; |
| 492 | |
| 493 | repack_prepare_midx_command(&cmd, opts, "compact"); |
| 494 | strvec_pushl(&cmd.args, "--incremental", "--no-write-chain-file", |
| 495 | midx_get_checksum_hex(step->u.compact.from), |
| 496 | midx_get_checksum_hex(step->u.compact.to), NULL); |
| 497 | |
| 498 | cmd.out = -1; |
| 499 | |
| 500 | ret = start_command(&cmd); |
| 501 | if (ret) |
| 502 | goto out; |
| 503 | |
| 504 | out = xfdopen(cmd.out, "r"); |
| 505 | while (strbuf_getline_lf(&buf, out) != EOF) { |
| 506 | if (step->csum) { |
| 507 | ret = error(_("unexpected MIDX output: '%s'"), buf.buf); |
| 508 | fclose(out); |
| 509 | out = NULL; |
| 510 | finish_command(&cmd); |
| 511 | goto out; |
| 512 | } |
| 513 | step->csum = strbuf_detach(&buf, NULL); |
| 514 | } |
| 515 | |
| 516 | ret = finish_command(&cmd); |
| 517 | |
| 518 | out: |
| 519 | if (out) |
| 520 | fclose(out); |
| 521 | strbuf_release(&buf); |
| 522 | |
| 523 | return ret; |
| 524 | } |
| 525 | |
| 526 | static int midx_compaction_step_exec(struct midx_compaction_step *step, |
| 527 | struct repack_write_midx_opts *opts, |
| 528 | const char *base) |
| 529 | { |
| 530 | switch (step->type) { |
| 531 | case MIDX_COMPACTION_STEP_UNKNOWN: |
| 532 | BUG("cannot execute UNKNOWN midx compaction step"); |
| 533 | case MIDX_COMPACTION_STEP_COPY: |
| 534 | return midx_compaction_step_exec_copy(step); |
| 535 | case MIDX_COMPACTION_STEP_WRITE: |
| 536 | return midx_compaction_step_exec_write(step, opts, base); |
| 537 | case MIDX_COMPACTION_STEP_COMPACT: |
| 538 | return midx_compaction_step_exec_compact(step, opts); |
| 539 | default: |
| 540 | BUG("unhandled midx compaction step type %d", step->type); |
| 541 | } |
| 542 | } |
| 543 | |
| 544 | static void midx_compaction_step_release(struct midx_compaction_step *step) |
| 545 | { |
| 546 | if (step->type == MIDX_COMPACTION_STEP_WRITE) |
| 547 | string_list_clear(&step->u.write, 0); |
| 548 | free(step->csum); |
| 549 | } |
| 550 | |
| 551 | /* |
| 552 | * Build an append-only MIDX plan: a single WRITE step for the freshly |
| 553 | * written packs, plus COPY steps for every existing layer. No |
| 554 | * compaction or merging is performed. |
| 555 | */ |
| 556 | static void repack_make_midx_append_plan(struct repack_write_midx_opts *opts, |
| 557 | struct midx_compaction_step **steps_p, |
| 558 | size_t *steps_nr_p) |
| 559 | { |
| 560 | struct odb_source_files *files = odb_source_files_downcast(opts->existing->source); |
| 561 | struct multi_pack_index *m; |
| 562 | struct midx_compaction_step *steps = NULL; |
| 563 | struct midx_compaction_step *step; |
| 564 | size_t steps_nr = 0, steps_alloc = 0; |
| 565 | |
| 566 | odb_reprepare(opts->existing->repo->objects); |
| 567 | m = get_multi_pack_index(files->packed); |
| 568 | |
| 569 | if (opts->names->nr) { |
| 570 | struct strbuf buf = STRBUF_INIT; |
| 571 | uint32_t i; |
| 572 | |
| 573 | ALLOC_GROW(steps, st_add(steps_nr, 1), steps_alloc); |
| 574 | |
| 575 | step = &steps[steps_nr++]; |
| 576 | memset(step, 0, sizeof(*step)); |
| 577 | |
| 578 | step->type = MIDX_COMPACTION_STEP_WRITE; |
| 579 | string_list_init_dup(&step->u.write); |
| 580 | |
| 581 | for (i = 0; i < opts->names->nr; i++) { |
| 582 | strbuf_reset(&buf); |
| 583 | strbuf_addf(&buf, "pack-%s.idx", |
| 584 | opts->names->items[i].string); |
| 585 | string_list_append(&step->u.write, buf.buf); |
| 586 | } |
| 587 | |
| 588 | strbuf_release(&buf); |
| 589 | } |
| 590 | |
| 591 | for (; m; m = m->base_midx) { |
| 592 | ALLOC_GROW(steps, st_add(steps_nr, 1), steps_alloc); |
| 593 | |
| 594 | step = &steps[steps_nr++]; |
| 595 | memset(step, 0, sizeof(*step)); |
| 596 | |
| 597 | step->type = MIDX_COMPACTION_STEP_COPY; |
| 598 | step->u.copy = m; |
| 599 | step->objects_nr = m->num_objects; |
| 600 | } |
| 601 | |
| 602 | *steps_p = steps; |
| 603 | *steps_nr_p = steps_nr; |
| 604 | } |
| 605 | |
| 606 | static int repack_make_midx_compaction_plan(struct repack_write_midx_opts *opts, |
| 607 | struct midx_compaction_step **steps_p, |
| 608 | size_t *steps_nr_p) |
| 609 | { |
| 610 | struct odb_source_files *files = odb_source_files_downcast(opts->existing->source); |
| 611 | struct multi_pack_index *m; |
| 612 | struct midx_compaction_step *steps = NULL; |
| 613 | struct midx_compaction_step step = { 0 }; |
| 614 | struct strbuf buf = STRBUF_INIT; |
| 615 | size_t steps_nr = 0, steps_alloc = 0; |
| 616 | uint32_t i; |
| 617 | int ret = 0; |
| 618 | |
| 619 | trace2_region_enter("repack", "make_midx_compaction_plan", |
| 620 | opts->existing->repo); |
| 621 | |
| 622 | odb_reprepare(opts->existing->repo->objects); |
| 623 | m = get_multi_pack_index(files->packed); |
| 624 | |
| 625 | for (i = 0; m && i < m->num_packs + m->num_packs_in_base; i++) { |
| 626 | if (prepare_midx_pack(m, i)) { |
| 627 | ret = error(_("could not load pack %"PRIu32" from MIDX"), |
| 628 | i); |
| 629 | goto out; |
| 630 | } |
| 631 | } |
| 632 | |
| 633 | trace2_region_enter("repack", "steps:write", opts->existing->repo); |
| 634 | |
| 635 | /* |
| 636 | * The first MIDX in the resulting chain is always going to be |
| 637 | * new. |
| 638 | * |
| 639 | * At a minimum, it will include all of the newly written packs. |
| 640 | * If there is an existing MIDX whose tip layer contains packs |
| 641 | * that were repacked, it will also include any of its packs |
| 642 | * which were *not* rolled up as part of the geometric repack |
| 643 | * (if any), and the previous tip will be replaced. |
| 644 | * |
| 645 | * It may grow to include the packs from zero or more MIDXs from |
| 646 | * the old chain, beginning either at the old tip (if the MIDX |
| 647 | * was *not* rewritten) or the old tip's base MIDX layer |
| 648 | * (otherwise). |
| 649 | */ |
| 650 | step.type = MIDX_COMPACTION_STEP_WRITE; |
| 651 | string_list_init_dup(&step.u.write); |
| 652 | |
| 653 | for (i = 0; i < opts->names->nr; i++) { |
| 654 | strbuf_reset(&buf); |
| 655 | strbuf_addf(&buf, "pack-%s.idx", opts->names->items[i].string); |
| 656 | string_list_append(&step.u.write, buf.buf); |
| 657 | |
| 658 | trace2_data_string("repack", opts->existing->repo, |
| 659 | "include:fresh", |
| 660 | step.u.write.items[step.u.write.nr - 1].string); |
| 661 | } |
| 662 | for (i = 0; i < opts->geometry->split; i++) { |
| 663 | struct packed_git *p = opts->geometry->pack[i]; |
| 664 | if (unsigned_add_overflows(step.objects_nr, p->num_objects)) { |
| 665 | ret = error(_("too many objects in MIDX compaction step")); |
| 666 | goto out; |
| 667 | } |
| 668 | |
| 669 | step.objects_nr += p->num_objects; |
| 670 | } |
| 671 | trace2_data_intmax("repack", opts->existing->repo, |
| 672 | "include:fresh:objects_nr", |
| 673 | (uintmax_t)step.objects_nr); |
| 674 | |
| 675 | /* |
| 676 | * Now handle any existing packs which were *not* rewritten. |
| 677 | * |
| 678 | * The list of packs in opts->geometry only contains MIDX'd |
| 679 | * packs from the newest layer when that layer has more than |
| 680 | * 'repack.midxNewLayerThreshold' number of packs. |
| 681 | * |
| 682 | * If the MIDX tip was rewritten (that is, one or more of those |
| 683 | * packs appear below the split line), then add all packs above |
| 684 | * the split line to the new layer, as the old one is no longer |
| 685 | * usable. |
| 686 | * |
| 687 | * If the MIDX tip was not rewritten (that is, all MIDX'd packs |
| 688 | * from the youngest layer appear below the split line, or were |
| 689 | * not included in the geometric repack at all because there |
| 690 | * were too few of them), ignore them since we'll retain the |
| 691 | * existing layer as-is. |
| 692 | */ |
| 693 | for (i = opts->geometry->split; i < opts->geometry->pack_nr; i++) { |
| 694 | struct packed_git *p = opts->geometry->pack[i]; |
| 695 | struct string_list_item *item; |
| 696 | |
| 697 | strbuf_reset(&buf); |
| 698 | strbuf_addstr(&buf, pack_basename(p)); |
| 699 | strbuf_strip_suffix(&buf, ".pack"); |
| 700 | strbuf_addstr(&buf, ".idx"); |
| 701 | |
| 702 | if (p->multi_pack_index && |
| 703 | !opts->geometry->midx_tip_rewritten) { |
| 704 | trace2_data_string("repack", opts->existing->repo, |
| 705 | "exclude:unmodified", buf.buf); |
| 706 | continue; |
| 707 | } |
| 708 | |
| 709 | trace2_data_string("repack", opts->existing->repo, |
| 710 | "include:unmodified", buf.buf); |
| 711 | trace2_data_string("repack", opts->existing->repo, |
| 712 | "include:unmodified:midx", |
| 713 | p->multi_pack_index ? "true" : "false"); |
| 714 | |
| 715 | item = string_list_append(&step.u.write, buf.buf); |
| 716 | if (p->multi_pack_index || i == opts->geometry->pack_nr - 1) |
| 717 | item->util = (void *)1; /* mark as preferred */ |
| 718 | |
| 719 | if (unsigned_add_overflows(step.objects_nr, p->num_objects)) { |
| 720 | ret = error(_("too many objects in MIDX compaction step")); |
| 721 | goto out; |
| 722 | } |
| 723 | |
| 724 | step.objects_nr += p->num_objects; |
| 725 | } |
| 726 | trace2_data_intmax("repack", opts->existing->repo, |
| 727 | "include:unmodified:objects_nr", |
| 728 | (uintmax_t)step.objects_nr); |
| 729 | |
| 730 | /* |
| 731 | * If the MIDX tip was rewritten, then we no longer consider it |
| 732 | * a candidate for compaction, since it will not exist in the |
| 733 | * MIDX chain being built. |
| 734 | */ |
| 735 | if (opts->geometry->midx_tip_rewritten) |
| 736 | m = m->base_midx; |
| 737 | |
| 738 | trace2_data_string("repack", opts->existing->repo, "midx:rewrote-tip", |
| 739 | opts->geometry->midx_tip_rewritten ? "true" : "false"); |
| 740 | |
| 741 | trace2_region_enter("repack", "compact", opts->existing->repo); |
| 742 | |
| 743 | /* |
| 744 | * Compact additional MIDX layers into this proposed one until |
| 745 | * the merging condition is violated. |
| 746 | */ |
| 747 | while (m) { |
| 748 | uint32_t preferred_pack_idx; |
| 749 | |
| 750 | trace2_data_string("repack", opts->existing->repo, |
| 751 | "candidate", midx_get_checksum_hex(m)); |
| 752 | |
| 753 | if (step.objects_nr < m->num_objects / opts->midx_split_factor) { |
| 754 | /* |
| 755 | * Stop compacting MIDX layer as soon as the |
| 756 | * merged size is less than half the size of the |
| 757 | * next layer in the chain. |
| 758 | */ |
| 759 | trace2_data_string("repack", opts->existing->repo, |
| 760 | "compact", "violated"); |
| 761 | trace2_data_intmax("repack", opts->existing->repo, |
| 762 | "objects_nr", |
| 763 | (uintmax_t)step.objects_nr); |
| 764 | trace2_data_intmax("repack", opts->existing->repo, |
| 765 | "next_objects_nr", |
| 766 | (uintmax_t)m->num_objects); |
| 767 | trace2_data_intmax("repack", opts->existing->repo, |
| 768 | "split_factor", |
| 769 | (uintmax_t)opts->midx_split_factor); |
| 770 | |
| 771 | break; |
| 772 | } |
| 773 | |
| 774 | if (midx_preferred_pack(m, &preferred_pack_idx) < 0) { |
| 775 | ret = error(_("could not find preferred pack for MIDX " |
| 776 | "%s"), midx_get_checksum_hex(m)); |
| 777 | goto out; |
| 778 | } |
| 779 | |
| 780 | for (i = 0; i < m->num_packs; i++) { |
| 781 | struct string_list_item *item; |
| 782 | uint32_t pack_int_id = i + m->num_packs_in_base; |
| 783 | struct packed_git *p = nth_midxed_pack(m, pack_int_id); |
| 784 | |
| 785 | strbuf_reset(&buf); |
| 786 | strbuf_addstr(&buf, pack_basename(p)); |
| 787 | strbuf_strip_suffix(&buf, ".pack"); |
| 788 | strbuf_addstr(&buf, ".idx"); |
| 789 | |
| 790 | trace2_data_string("repack", opts->existing->repo, |
| 791 | "midx:pack", buf.buf); |
| 792 | |
| 793 | item = string_list_append(&step.u.write, buf.buf); |
| 794 | if (pack_int_id == preferred_pack_idx) |
| 795 | item->util = (void *)1; /* mark as preferred */ |
| 796 | } |
| 797 | |
| 798 | if (unsigned_add_overflows(step.objects_nr, m->num_objects)) { |
| 799 | ret = error(_("too many objects in MIDX compaction step")); |
| 800 | goto out; |
| 801 | } |
| 802 | step.objects_nr += m->num_objects; |
| 803 | |
| 804 | m = m->base_midx; |
| 805 | } |
| 806 | |
| 807 | if (step.u.write.nr > 0) { |
| 808 | /* |
| 809 | * As long as there is at least one new pack to write |
| 810 | * (and thus the MIDX is non-empty), add it to the plan. |
| 811 | */ |
| 812 | ALLOC_GROW(steps, steps_nr + 1, steps_alloc); |
| 813 | steps[steps_nr++] = step; |
| 814 | } |
| 815 | |
| 816 | trace2_data_intmax("repack", opts->existing->repo, |
| 817 | "step:objects_nr", (uintmax_t)step.objects_nr); |
| 818 | trace2_data_intmax("repack", opts->existing->repo, |
| 819 | "step:packs_nr", (uintmax_t)step.u.write.nr); |
| 820 | |
| 821 | trace2_region_leave("repack", "compact", opts->existing->repo); |
| 822 | trace2_region_leave("repack", "steps:write", opts->existing->repo); |
| 823 | |
| 824 | trace2_region_enter("repack", "steps:rest", opts->existing->repo); |
| 825 | |
| 826 | /* |
| 827 | * Then start over, repeat, and either compact or keep as-is |
| 828 | * each MIDX layer until we have exhausted the chain. |
| 829 | * |
| 830 | * Finally, evaluate the remainder of the chain (if any) and |
| 831 | * either compact a sequence of adjacent layers, or keep |
| 832 | * individual layers as-is according to the same merging |
| 833 | * condition as above. |
| 834 | */ |
| 835 | while (m) { |
| 836 | struct multi_pack_index *next = m; |
| 837 | |
| 838 | ALLOC_GROW(steps, steps_nr + 1, steps_alloc); |
| 839 | |
| 840 | memset(&step, 0, sizeof(step)); |
| 841 | step.type = MIDX_COMPACTION_STEP_UNKNOWN; |
| 842 | |
| 843 | trace2_region_enter("repack", "step", opts->existing->repo); |
| 844 | |
| 845 | trace2_data_string("repack", opts->existing->repo, |
| 846 | "from", midx_get_checksum_hex(m)); |
| 847 | |
| 848 | while (next) { |
| 849 | uint32_t proposed_objects_nr; |
| 850 | if (unsigned_add_overflows(step.objects_nr, next->num_objects)) { |
| 851 | ret = error(_("too many objects in MIDX compaction step")); |
| 852 | trace2_region_leave("repack", "step", opts->existing->repo); |
| 853 | goto out; |
| 854 | } |
| 855 | |
| 856 | proposed_objects_nr = step.objects_nr + next->num_objects; |
| 857 | |
| 858 | trace2_data_string("repack", opts->existing->repo, |
| 859 | "proposed", |
| 860 | midx_get_checksum_hex(next)); |
| 861 | trace2_data_intmax("repack", opts->existing->repo, |
| 862 | "proposed:objects_nr", |
| 863 | (uintmax_t)next->num_objects); |
| 864 | |
| 865 | if (!next->base_midx) { |
| 866 | /* |
| 867 | * If we are at the end of the MIDX |
| 868 | * chain, there is nothing to compact, |
| 869 | * so mark it and stop. |
| 870 | */ |
| 871 | step.objects_nr = proposed_objects_nr; |
| 872 | break; |
| 873 | } |
| 874 | |
| 875 | if (proposed_objects_nr < next->base_midx->num_objects / opts->midx_split_factor) { |
| 876 | /* |
| 877 | * If there is a MIDX following this |
| 878 | * one, but our accumulated size is less |
| 879 | * than half of its size, compacting |
| 880 | * them would violate the merging |
| 881 | * condition, so stop here. |
| 882 | */ |
| 883 | |
| 884 | trace2_data_string("repack", opts->existing->repo, |
| 885 | "compact:violated:at", |
| 886 | midx_get_checksum_hex(next->base_midx)); |
| 887 | trace2_data_intmax("repack", opts->existing->repo, |
| 888 | "compact:violated:at:objects_nr", |
| 889 | (uintmax_t)next->base_midx->num_objects); |
| 890 | break; |
| 891 | } |
| 892 | |
| 893 | /* |
| 894 | * Otherwise, it is OK to compact the next layer |
| 895 | * into this one. Do so, and then continue |
| 896 | * through the remainder of the chain. |
| 897 | */ |
| 898 | step.objects_nr = proposed_objects_nr; |
| 899 | trace2_data_intmax("repack", opts->existing->repo, |
| 900 | "step:objects_nr", |
| 901 | (uintmax_t)step.objects_nr); |
| 902 | next = next->base_midx; |
| 903 | } |
| 904 | |
| 905 | if (m == next) { |
| 906 | step.type = MIDX_COMPACTION_STEP_COPY; |
| 907 | step.u.copy = m; |
| 908 | |
| 909 | trace2_data_string("repack", opts->existing->repo, |
| 910 | "type", "copy"); |
| 911 | } else { |
| 912 | step.type = MIDX_COMPACTION_STEP_COMPACT; |
| 913 | step.u.compact.from = next; |
| 914 | step.u.compact.to = m; |
| 915 | |
| 916 | trace2_data_string("repack", opts->existing->repo, |
| 917 | "to", midx_get_checksum_hex(m)); |
| 918 | trace2_data_string("repack", opts->existing->repo, |
| 919 | "type", "compact"); |
| 920 | } |
| 921 | |
| 922 | m = next->base_midx; |
| 923 | steps[steps_nr++] = step; |
| 924 | trace2_region_leave("repack", "step", opts->existing->repo); |
| 925 | } |
| 926 | |
| 927 | trace2_region_leave("repack", "steps:rest", opts->existing->repo); |
| 928 | |
| 929 | out: |
| 930 | *steps_p = steps; |
| 931 | *steps_nr_p = steps_nr; |
| 932 | |
| 933 | strbuf_release(&buf); |
| 934 | |
| 935 | trace2_region_leave("repack", "make_midx_compaction_plan", |
| 936 | opts->existing->repo); |
| 937 | |
| 938 | return ret; |
| 939 | } |
| 940 | |
| 941 | static int write_midx_incremental(struct repack_write_midx_opts *opts) |
| 942 | { |
| 943 | struct odb_source_files *files = odb_source_files_downcast(opts->existing->source); |
| 944 | struct midx_compaction_step *steps = NULL; |
| 945 | struct strbuf lock_name = STRBUF_INIT; |
| 946 | struct lock_file lf; |
| 947 | struct strvec keep_hashes = STRVEC_INIT; |
| 948 | size_t steps_nr = 0; |
| 949 | size_t i; |
| 950 | int ret = 0; |
| 951 | |
| 952 | get_midx_chain_filename(files->packed, &lock_name); |
| 953 | if (safe_create_leading_directories(opts->existing->repo, |
| 954 | lock_name.buf)) |
| 955 | die_errno(_("unable to create leading directories of %s"), |
| 956 | lock_name.buf); |
| 957 | repo_hold_lock_file_for_update(opts->existing->repo, &lf, lock_name.buf, |
| 958 | LOCK_DIE_ON_ERROR); |
| 959 | |
| 960 | if (!fdopen_lock_file(&lf, "w")) { |
| 961 | ret = error_errno(_("unable to open multi-pack-index chain file")); |
| 962 | goto done; |
| 963 | } |
| 964 | |
| 965 | if (opts->geometry->split_factor) { |
| 966 | if (repack_make_midx_compaction_plan(opts, &steps, &steps_nr) < 0) { |
| 967 | ret = error(_("unable to generate compaction plan")); |
| 968 | goto done; |
| 969 | } |
| 970 | } else { |
| 971 | repack_make_midx_append_plan(opts, &steps, &steps_nr); |
| 972 | } |
| 973 | |
| 974 | for (i = 0; i < steps_nr; i++) { |
| 975 | struct midx_compaction_step *step = &steps[i]; |
| 976 | char *base = NULL; |
| 977 | |
| 978 | if (i + 1 < steps_nr) |
| 979 | base = xstrdup(midx_compaction_step_base(&steps[i + 1])); |
| 980 | |
| 981 | if (midx_compaction_step_exec(step, opts, base) < 0) { |
| 982 | ret = error(_("unable to execute compaction step %"PRIuMAX), |
| 983 | (uintmax_t)i); |
| 984 | free(base); |
| 985 | goto done; |
| 986 | } |
| 987 | |
| 988 | free(base); |
| 989 | } |
| 990 | |
| 991 | i = steps_nr; |
| 992 | while (i--) { |
| 993 | struct midx_compaction_step *step = &steps[i]; |
| 994 | if (!step->csum) |
| 995 | BUG("missing result for compaction step %"PRIuMAX, |
| 996 | (uintmax_t)i); |
| 997 | fprintf(get_lock_file_fp(&lf), "%s\n", step->csum); |
| 998 | strvec_push(&keep_hashes, step->csum); |
| 999 | } |
| 1000 | |
| 1001 | commit_lock_file(&lf); |
| 1002 | |
| 1003 | clear_incremental_midx_files(opts->existing->repo, &keep_hashes); |
| 1004 | |
| 1005 | done: |
| 1006 | strvec_clear(&keep_hashes); |
| 1007 | strbuf_release(&lock_name); |
| 1008 | for (i = 0; i < steps_nr; i++) |
| 1009 | midx_compaction_step_release(&steps[i]); |
| 1010 | free(steps); |
| 1011 | return ret; |
| 1012 | } |
| 1013 | |
| 1014 | int repack_write_midx(struct repack_write_midx_opts *opts) |
| 1015 | { |
| 1016 | switch (opts->mode) { |
| 1017 | case REPACK_WRITE_MIDX_NONE: |
| 1018 | BUG("write_midx mode is NONE?"); |
| 1019 | case REPACK_WRITE_MIDX_DEFAULT: |
| 1020 | return write_midx_included_packs(opts); |
| 1021 | case REPACK_WRITE_MIDX_INCREMENTAL: |
| 1022 | return write_midx_incremental(opts); |
| 1023 | default: |
| 1024 | BUG("unhandled write_midx mode: %d", opts->mode); |
| 1025 | } |
| 1026 | } |