midx-write.c: support reading an existing MIDX with `packs_to_include`

Avoid unconditionally copying all packs from an existing MIDX into a new MIDX by checking that packs added via `fill_packs_from_midx()` don't appear in the `to_include` set, if one was provided. Do so by calling `should_include_pack()` from both `add_pack_to_midx()` and `fill_packs_from_midx()`. In order to make this work, teach `should_include_pack()` a new "exclude_from_midx" parameter, which allows skipping the first check. This is done so that the caller in `fill_packs_from_midx()` doesn't reject all of the packs it provided since they appear in an existing MIDX by definition. The sum total of this change is that we are now able to read and reference objects in an existing MIDX even when given a non-NULL `packs_to_include`. This is a prerequisite step for incremental MIDXs, which need to load any existing MIDX (if one is present) in order to determine whether or not an object already appears in an earlier portion of the MIDX to avoid duplicating it across multiple portions. Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Taylor Blau committed May 29, 2024 at 18:55 UTC d6a8c58675a04abe2597179e3332bf031bd6e48a
1 file changed +11 -31
midx-write.c
+11 -31
@@ -101,27 +101,13 @@ struct write_midx_context {
101 };
102
103 static int should_include_pack(const struct write_midx_context *ctx,
104 - const char *file_name)
104 + const char *file_name,
105 + int exclude_from_midx)
106 {
106 - /*
107 - * Note that at most one of ctx->m and ctx->to_include are set,
108 - * so we are testing midx_contains_pack() and
109 - * string_list_has_string() independently (guarded by the
110 - * appropriate NULL checks).
111 - *
112 - * We could support passing to_include while reusing an existing
113 - * MIDX, but don't currently since the reuse process drags
114 - * forward all packs from an existing MIDX (without checking
115 - * whether or not they appear in the to_include list).
116 - *
117 - * If we added support for that, these next two conditional
118 - * should be performed independently (likely checking
119 - * to_include before the existing MIDX).
120 - */
121 - if (ctx->m && midx_contains_pack(ctx->m, file_name))
107 + if (exclude_from_midx && ctx->m && midx_contains_pack(ctx->m, file_name))
108 return 0;
123 - else if (ctx->to_include &&
124 - !string_list_has_string(ctx->to_include, file_name))
109 + if (ctx->to_include && !string_list_has_string(ctx->to_include,
110 + file_name))
111 return 0;
112 return 1;
113 }
@@ -135,7 +121,7 @@ static void add_pack_to_midx(const char *full_path, size_t full_path_len,
121 if (ends_with(file_name, ".idx")) {
122 display_progress(ctx->progress, ++ctx->pack_paths_checked);
123
138 - if (!should_include_pack(ctx, file_name))
124 + if (!should_include_pack(ctx, file_name, 1))
125 return;
126
127 ALLOC_GROW(ctx->info, ctx->nr + 1, ctx->alloc);
@@ -888,6 +874,9 @@ static int fill_packs_from_midx(struct write_midx_context *ctx,
874 uint32_t i;
875
876 for (i = 0; i < ctx->m->num_packs; i++) {
877 + if (!should_include_pack(ctx, ctx->m->pack_names[i], 0))
878 + continue;
879 +
880 ALLOC_GROW(ctx->info, ctx->nr + 1, ctx->alloc);
881
882 if (flags & MIDX_WRITE_REV_INDEX || preferred_pack_name) {
@@ -942,15 +931,7 @@ static int write_midx_internal(const char *object_dir,
931 die_errno(_("unable to create leading directories of %s"),
932 midx_name.buf);
933
945 - if (!packs_to_include) {
946 - /*
947 - * Only reference an existing MIDX when not filtering which
948 - * packs to include, since all packs and objects are copied
949 - * blindly from an existing MIDX if one is present.
950 - */
951 - ctx.m = lookup_multi_pack_index(the_repository, object_dir);
952 - }
953 -
934 + ctx.m = lookup_multi_pack_index(the_repository, object_dir);
935 if (ctx.m && !midx_checksum_valid(ctx.m)) {
936 warning(_("ignoring existing multi-pack-index; checksum mismatch"));
937 ctx.m = NULL;
@@ -959,6 +940,7 @@ static int write_midx_internal(const char *object_dir,
940 ctx.nr = 0;
941 ctx.alloc = ctx.m ? ctx.m->num_packs : 16;
942 ctx.info = NULL;
943 + ctx.to_include = packs_to_include;
944 ALLOC_ARRAY(ctx.info, ctx.alloc);
945
946 if (ctx.m && fill_packs_from_midx(&ctx, preferred_pack_name,
@@ -975,8 +957,6 @@ static int write_midx_internal(const char *object_dir,
957 else
958 ctx.progress = NULL;
959
978 - ctx.to_include = packs_to_include;
979 -
960 for_each_file_in_pack_dir(object_dir, add_pack_to_midx, &ctx);
961 stop_progress(&ctx.progress);
962