repack: 'write_midx_included_packs' API from the builtin
Now that we have sufficiently cleaned up the write_midx_included_packs() function, we can move it (along with the struct repack_write_midx_opts) out of the builtin, and into the repack.h header. Since this function (and the static ones that it depends on) are MIDX-specific details of the repacking process, move them to the repack-midx.c compilation unit instead of the general repack.c one. Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Taylor Blau committed
Oct 15, 2025 at 18:29 UTC
6d05eb135f67d2d45a0fbd110a32d28b1e28c95d
3 files changed
+307
-305
builtin/repack.c
-305
@@ -107,311 +107,6 @@ static int repack_config(const char *var, const char *value,
107
return git_default_config(var, value, ctx, cb);
108
}
109
110
-struct repack_write_midx_opts {
111
- struct existing_packs *existing;
112
- struct pack_geometry *geometry;
113
- struct string_list *names;
114
- const char *refs_snapshot;
115
- const char *packdir;
116
- int show_progress;
117
- int write_bitmaps;
118
- int midx_must_contain_cruft;
119
-};
120
-
121
-static int midx_has_unknown_packs(struct string_list *include,
122
- struct pack_geometry *geometry,
123
- struct existing_packs *existing)
124
-{
125
- struct string_list_item *item;
126
-
127
- string_list_sort(include);
128
-
129
- for_each_string_list_item(item, &existing->midx_packs) {
130
- const char *pack_name = item->string;
131
-
132
- /*
133
- * Determine whether or not each MIDX'd pack from the existing
134
- * MIDX (if any) is represented in the new MIDX. For each pack
135
- * in the MIDX, it must either be:
136
- *
137
- * - In the "include" list of packs to be included in the new
138
- * MIDX. Note this function is called before the include
139
- * list is populated with any cruft pack(s).
140
- *
141
- * - Below the geometric split line (if using pack geometry),
142
- * indicating that the pack won't be included in the new
143
- * MIDX, but its contents were rolled up as part of the
144
- * geometric repack.
145
- *
146
- * - In the existing non-kept packs list (if not using pack
147
- * geometry), and marked as non-deleted.
148
- */
149
- if (string_list_has_string(include, pack_name)) {
150
- continue;
151
- } else if (geometry) {
152
- struct strbuf buf = STRBUF_INIT;
153
- uint32_t j;
154
-
155
- for (j = 0; j < geometry->split; j++) {
156
- strbuf_reset(&buf);
157
- strbuf_addstr(&buf, pack_basename(geometry->pack[j]));
158
- strbuf_strip_suffix(&buf, ".pack");
159
- strbuf_addstr(&buf, ".idx");
160
-
161
- if (!strcmp(pack_name, buf.buf)) {
162
- strbuf_release(&buf);
163
- break;
164
- }
165
- }
166
-
167
- strbuf_release(&buf);
168
-
169
- if (j < geometry->split)
170
- continue;
171
- } else {
172
- struct string_list_item *item;
173
-
174
- item = string_list_lookup(&existing->non_kept_packs,
175
- pack_name);
176
- if (item && !existing_pack_is_marked_for_deletion(item))
177
- continue;
178
- }
179
-
180
- /*
181
- * If we got to this point, the MIDX includes some pack that we
182
- * don't know about.
183
- */
184
- return 1;
185
- }
186
-
187
- return 0;
188
-}
189
-
190
-static void midx_included_packs(struct string_list *include,
191
- struct repack_write_midx_opts *opts)
192
-{
193
- struct existing_packs *existing = opts->existing;
194
- struct pack_geometry *geometry = opts->geometry;
195
- struct string_list *names = opts->names;
196
- struct string_list_item *item;
197
- struct strbuf buf = STRBUF_INIT;
198
-
199
- for_each_string_list_item(item, &existing->kept_packs) {
200
- strbuf_reset(&buf);
201
- strbuf_addf(&buf, "%s.idx", item->string);
202
- string_list_insert(include, buf.buf);
203
- }
204
-
205
- for_each_string_list_item(item, names) {
206
- strbuf_reset(&buf);
207
- strbuf_addf(&buf, "pack-%s.idx", item->string);
208
- string_list_insert(include, buf.buf);
209
- }
210
-
211
- if (geometry->split_factor) {
212
- uint32_t i;
213
-
214
- for (i = geometry->split; i < geometry->pack_nr; i++) {
215
- struct packed_git *p = geometry->pack[i];
216
-
217
- /*
218
- * The multi-pack index never refers to packfiles part
219
- * of an alternate object database, so we skip these.
220
- * While git-multi-pack-index(1) would silently ignore
221
- * them anyway, this allows us to skip executing the
222
- * command completely when we have only non-local
223
- * packfiles.
224
- */
225
- if (!p->pack_local)
226
- continue;
227
-
228
- strbuf_reset(&buf);
229
- strbuf_addstr(&buf, pack_basename(p));
230
- strbuf_strip_suffix(&buf, ".pack");
231
- strbuf_addstr(&buf, ".idx");
232
-
233
- string_list_insert(include, buf.buf);
234
- }
235
- } else {
236
- for_each_string_list_item(item, &existing->non_kept_packs) {
237
- if (existing_pack_is_marked_for_deletion(item))
238
- continue;
239
-
240
- strbuf_reset(&buf);
241
- strbuf_addf(&buf, "%s.idx", item->string);
242
- string_list_insert(include, buf.buf);
243
- }
244
- }
245
-
246
- if (opts->midx_must_contain_cruft ||
247
- midx_has_unknown_packs(include, geometry, existing)) {
248
- /*
249
- * If there are one or more unknown pack(s) present (see
250
- * midx_has_unknown_packs() for what makes a pack
251
- * "unknown") in the MIDX before the repack, keep them
252
- * as they may be required to form a reachability
253
- * closure if the MIDX is bitmapped.
254
- *
255
- * For example, a cruft pack can be required to form a
256
- * reachability closure if the MIDX is bitmapped and one
257
- * or more of the bitmap's selected commits reaches a
258
- * once-cruft object that was later made reachable.
259
- */
260
- for_each_string_list_item(item, &existing->cruft_packs) {
261
- /*
262
- * When doing a --geometric repack, there is no
263
- * need to check for deleted packs, since we're
264
- * by definition not doing an ALL_INTO_ONE
265
- * repack (hence no packs will be deleted).
266
- * Otherwise we must check for and exclude any
267
- * packs which are enqueued for deletion.
268
- *
269
- * So we could omit the conditional below in the
270
- * --geometric case, but doing so is unnecessary
271
- * since no packs are marked as pending
272
- * deletion (since we only call
273
- * `existing_packs_mark_for_deletion()` when
274
- * doing an all-into-one repack).
275
- */
276
- if (existing_pack_is_marked_for_deletion(item))
277
- continue;
278
-
279
- strbuf_reset(&buf);
280
- strbuf_addf(&buf, "%s.idx", item->string);
281
- string_list_insert(include, buf.buf);
282
- }
283
- } else {
284
- /*
285
- * Modern versions of Git (with the appropriate
286
- * configuration setting) will write new copies of
287
- * once-cruft objects when doing a --geometric repack.
288
- *
289
- * If the MIDX has no cruft pack, new packs written
290
- * during a --geometric repack will not rely on the
291
- * cruft pack to form a reachability closure, so we can
292
- * avoid including them in the MIDX in that case.
293
- */
294
- ;
295
- }
296
-
297
- strbuf_release(&buf);
298
-}
299
-
300
-static void remove_redundant_bitmaps(struct string_list *include,
301
- const char *packdir)
302
-{
303
- struct strbuf path = STRBUF_INIT;
304
- struct string_list_item *item;
305
- size_t packdir_len;
306
-
307
- strbuf_addstr(&path, packdir);
308
- strbuf_addch(&path, '/');
309
- packdir_len = path.len;
310
-
311
- /*
312
- * Remove any pack bitmaps corresponding to packs which are now
313
- * included in the MIDX.
314
- */
315
- for_each_string_list_item(item, include) {
316
- strbuf_addstr(&path, item->string);
317
- strbuf_strip_suffix(&path, ".idx");
318
- strbuf_addstr(&path, ".bitmap");
319
-
320
- if (unlink(path.buf) && errno != ENOENT)
321
- warning_errno(_("could not remove stale bitmap: %s"),
322
- path.buf);
323
-
324
- strbuf_setlen(&path, packdir_len);
325
- }
326
- strbuf_release(&path);
327
-}
328
-
329
-static int write_midx_included_packs(struct repack_write_midx_opts *opts)
330
-{
331
- struct child_process cmd = CHILD_PROCESS_INIT;
332
- struct string_list include = STRING_LIST_INIT_DUP;
333
- struct string_list_item *item;
334
- struct packed_git *preferred = pack_geometry_preferred_pack(opts->geometry);
335
- FILE *in;
336
- int ret = 0;
337
-
338
- midx_included_packs(&include, opts);
339
- if (!include.nr)
340
- goto done;
341
-
342
- cmd.in = -1;
343
- cmd.git_cmd = 1;
344
-
345
- strvec_push(&cmd.args, "multi-pack-index");
346
- strvec_pushl(&cmd.args, "write", "--stdin-packs", NULL);
347
-
348
- if (opts->show_progress)
349
- strvec_push(&cmd.args, "--progress");
350
- else
351
- strvec_push(&cmd.args, "--no-progress");
352
-
353
- if (opts->write_bitmaps)
354
- strvec_push(&cmd.args, "--bitmap");
355
-
356
- if (preferred)
357
- strvec_pushf(&cmd.args, "--preferred-pack=%s",
358
- pack_basename(preferred));
359
- else if (opts->names->nr) {
360
- /* The largest pack was repacked, meaning that either
361
- * one or two packs exist depending on whether the
362
- * repository has a cruft pack or not.
363
- *
364
- * Select the non-cruft one as preferred to encourage
365
- * pack-reuse among packs containing reachable objects
366
- * over unreachable ones.
367
- *
368
- * (Note we could write multiple packs here if
369
- * `--max-pack-size` was given, but any one of them
370
- * will suffice, so pick the first one.)
371
- */
372
- for_each_string_list_item(item, opts->names) {
373
- struct generated_pack *pack = item->util;
374
- if (generated_pack_has_ext(pack, ".mtimes"))
375
- continue;
376
-
377
- strvec_pushf(&cmd.args, "--preferred-pack=pack-%s.pack",
378
- item->string);
379
- break;
380
- }
381
- } else {
382
- /*
383
- * No packs were kept, and no packs were written. The
384
- * only thing remaining are .keep packs (unless
385
- * --pack-kept-objects was given).
386
- *
387
- * Set the `--preferred-pack` arbitrarily here.
388
- */
389
- ;
390
- }
391
-
392
- if (opts->refs_snapshot)
393
- strvec_pushf(&cmd.args, "--refs-snapshot=%s",
394
- opts->refs_snapshot);
395
-
396
- ret = start_command(&cmd);
397
- if (ret)
398
- goto done;
399
-
400
- in = xfdopen(cmd.in, "w");
401
- for_each_string_list_item(item, &include)
402
- fprintf(in, "%s\n", item->string);
403
- fclose(in);
404
-
405
- ret = finish_command(&cmd);
406
-done:
407
- if (!ret && opts->write_bitmaps)
408
- remove_redundant_bitmaps(&include, opts->packdir);
409
-
410
- string_list_clear(&include, 0);
411
-
412
- return ret;
413
-}
414
-
110
static int finish_pack_objects_cmd(const struct git_hash_algo *algop,
111
struct child_process *cmd,
112
struct string_list *names,
repack-midx.c
+295
@@ -6,6 +6,7 @@
6
#include "oidset.h"
7
#include "pack-bitmap.h"
8
#include "refs.h"
9
+#include "run-command.h"
10
#include "tempfile.h"
11
12
struct midx_snapshot_ref_data {
@@ -75,3 +76,297 @@ void midx_snapshot_refs(struct repository *repo, struct tempfile *f)
76
77
oidset_clear(&data.seen);
78
}
79
+
80
+static int midx_has_unknown_packs(struct string_list *include,
81
+ struct pack_geometry *geometry,
82
+ struct existing_packs *existing)
83
+{
84
+ struct string_list_item *item;
85
+
86
+ string_list_sort(include);
87
+
88
+ for_each_string_list_item(item, &existing->midx_packs) {
89
+ const char *pack_name = item->string;
90
+
91
+ /*
92
+ * Determine whether or not each MIDX'd pack from the existing
93
+ * MIDX (if any) is represented in the new MIDX. For each pack
94
+ * in the MIDX, it must either be:
95
+ *
96
+ * - In the "include" list of packs to be included in the new
97
+ * MIDX. Note this function is called before the include
98
+ * list is populated with any cruft pack(s).
99
+ *
100
+ * - Below the geometric split line (if using pack geometry),
101
+ * indicating that the pack won't be included in the new
102
+ * MIDX, but its contents were rolled up as part of the
103
+ * geometric repack.
104
+ *
105
+ * - In the existing non-kept packs list (if not using pack
106
+ * geometry), and marked as non-deleted.
107
+ */
108
+ if (string_list_has_string(include, pack_name)) {
109
+ continue;
110
+ } else if (geometry) {
111
+ struct strbuf buf = STRBUF_INIT;
112
+ uint32_t j;
113
+
114
+ for (j = 0; j < geometry->split; j++) {
115
+ strbuf_reset(&buf);
116
+ strbuf_addstr(&buf, pack_basename(geometry->pack[j]));
117
+ strbuf_strip_suffix(&buf, ".pack");
118
+ strbuf_addstr(&buf, ".idx");
119
+
120
+ if (!strcmp(pack_name, buf.buf)) {
121
+ strbuf_release(&buf);
122
+ break;
123
+ }
124
+ }
125
+
126
+ strbuf_release(&buf);
127
+
128
+ if (j < geometry->split)
129
+ continue;
130
+ } else {
131
+ struct string_list_item *item;
132
+
133
+ item = string_list_lookup(&existing->non_kept_packs,
134
+ pack_name);
135
+ if (item && !existing_pack_is_marked_for_deletion(item))
136
+ continue;
137
+ }
138
+
139
+ /*
140
+ * If we got to this point, the MIDX includes some pack that we
141
+ * don't know about.
142
+ */
143
+ return 1;
144
+ }
145
+
146
+ return 0;
147
+}
148
+
149
+static void midx_included_packs(struct string_list *include,
150
+ struct repack_write_midx_opts *opts)
151
+{
152
+ struct existing_packs *existing = opts->existing;
153
+ struct pack_geometry *geometry = opts->geometry;
154
+ struct string_list *names = opts->names;
155
+ struct string_list_item *item;
156
+ struct strbuf buf = STRBUF_INIT;
157
+
158
+ for_each_string_list_item(item, &existing->kept_packs) {
159
+ strbuf_reset(&buf);
160
+ strbuf_addf(&buf, "%s.idx", item->string);
161
+ string_list_insert(include, buf.buf);
162
+ }
163
+
164
+ for_each_string_list_item(item, names) {
165
+ strbuf_reset(&buf);
166
+ strbuf_addf(&buf, "pack-%s.idx", item->string);
167
+ string_list_insert(include, buf.buf);
168
+ }
169
+
170
+ if (geometry->split_factor) {
171
+ uint32_t i;
172
+
173
+ for (i = geometry->split; i < geometry->pack_nr; i++) {
174
+ struct packed_git *p = geometry->pack[i];
175
+
176
+ /*
177
+ * The multi-pack index never refers to packfiles part
178
+ * of an alternate object database, so we skip these.
179
+ * While git-multi-pack-index(1) would silently ignore
180
+ * them anyway, this allows us to skip executing the
181
+ * command completely when we have only non-local
182
+ * packfiles.
183
+ */
184
+ if (!p->pack_local)
185
+ continue;
186
+
187
+ strbuf_reset(&buf);
188
+ strbuf_addstr(&buf, pack_basename(p));
189
+ strbuf_strip_suffix(&buf, ".pack");
190
+ strbuf_addstr(&buf, ".idx");
191
+
192
+ string_list_insert(include, buf.buf);
193
+ }
194
+ } else {
195
+ for_each_string_list_item(item, &existing->non_kept_packs) {
196
+ if (existing_pack_is_marked_for_deletion(item))
197
+ continue;
198
+
199
+ strbuf_reset(&buf);
200
+ strbuf_addf(&buf, "%s.idx", item->string);
201
+ string_list_insert(include, buf.buf);
202
+ }
203
+ }
204
+
205
+ if (opts->midx_must_contain_cruft ||
206
+ midx_has_unknown_packs(include, geometry, existing)) {
207
+ /*
208
+ * If there are one or more unknown pack(s) present (see
209
+ * midx_has_unknown_packs() for what makes a pack
210
+ * "unknown") in the MIDX before the repack, keep them
211
+ * as they may be required to form a reachability
212
+ * closure if the MIDX is bitmapped.
213
+ *
214
+ * For example, a cruft pack can be required to form a
215
+ * reachability closure if the MIDX is bitmapped and one
216
+ * or more of the bitmap's selected commits reaches a
217
+ * once-cruft object that was later made reachable.
218
+ */
219
+ for_each_string_list_item(item, &existing->cruft_packs) {
220
+ /*
221
+ * When doing a --geometric repack, there is no
222
+ * need to check for deleted packs, since we're
223
+ * by definition not doing an ALL_INTO_ONE
224
+ * repack (hence no packs will be deleted).
225
+ * Otherwise we must check for and exclude any
226
+ * packs which are enqueued for deletion.
227
+ *
228
+ * So we could omit the conditional below in the
229
+ * --geometric case, but doing so is unnecessary
230
+ * since no packs are marked as pending
231
+ * deletion (since we only call
232
+ * `existing_packs_mark_for_deletion()` when
233
+ * doing an all-into-one repack).
234
+ */
235
+ if (existing_pack_is_marked_for_deletion(item))
236
+ continue;
237
+
238
+ strbuf_reset(&buf);
239
+ strbuf_addf(&buf, "%s.idx", item->string);
240
+ string_list_insert(include, buf.buf);
241
+ }
242
+ } else {
243
+ /*
244
+ * Modern versions of Git (with the appropriate
245
+ * configuration setting) will write new copies of
246
+ * once-cruft objects when doing a --geometric repack.
247
+ *
248
+ * If the MIDX has no cruft pack, new packs written
249
+ * during a --geometric repack will not rely on the
250
+ * cruft pack to form a reachability closure, so we can
251
+ * avoid including them in the MIDX in that case.
252
+ */
253
+ ;
254
+ }
255
+
256
+ strbuf_release(&buf);
257
+}
258
+
259
+static void remove_redundant_bitmaps(struct string_list *include,
260
+ const char *packdir)
261
+{
262
+ struct strbuf path = STRBUF_INIT;
263
+ struct string_list_item *item;
264
+ size_t packdir_len;
265
+
266
+ strbuf_addstr(&path, packdir);
267
+ strbuf_addch(&path, '/');
268
+ packdir_len = path.len;
269
+
270
+ /*
271
+ * Remove any pack bitmaps corresponding to packs which are now
272
+ * included in the MIDX.
273
+ */
274
+ for_each_string_list_item(item, include) {
275
+ strbuf_addstr(&path, item->string);
276
+ strbuf_strip_suffix(&path, ".idx");
277
+ strbuf_addstr(&path, ".bitmap");
278
+
279
+ if (unlink(path.buf) && errno != ENOENT)
280
+ warning_errno(_("could not remove stale bitmap: %s"),
281
+ path.buf);
282
+
283
+ strbuf_setlen(&path, packdir_len);
284
+ }
285
+ strbuf_release(&path);
286
+}
287
+
288
+int write_midx_included_packs(struct repack_write_midx_opts *opts)
289
+{
290
+ struct child_process cmd = CHILD_PROCESS_INIT;
291
+ struct string_list include = STRING_LIST_INIT_DUP;
292
+ struct string_list_item *item;
293
+ struct packed_git *preferred = pack_geometry_preferred_pack(opts->geometry);
294
+ FILE *in;
295
+ int ret = 0;
296
+
297
+ midx_included_packs(&include, opts);
298
+ if (!include.nr)
299
+ goto done;
300
+
301
+ cmd.in = -1;
302
+ cmd.git_cmd = 1;
303
+
304
+ strvec_push(&cmd.args, "multi-pack-index");
305
+ strvec_pushl(&cmd.args, "write", "--stdin-packs", NULL);
306
+
307
+ if (opts->show_progress)
308
+ strvec_push(&cmd.args, "--progress");
309
+ else
310
+ strvec_push(&cmd.args, "--no-progress");
311
+
312
+ if (opts->write_bitmaps)
313
+ strvec_push(&cmd.args, "--bitmap");
314
+
315
+ if (preferred)
316
+ strvec_pushf(&cmd.args, "--preferred-pack=%s",
317
+ pack_basename(preferred));
318
+ else if (opts->names->nr) {
319
+ /* The largest pack was repacked, meaning that either
320
+ * one or two packs exist depending on whether the
321
+ * repository has a cruft pack or not.
322
+ *
323
+ * Select the non-cruft one as preferred to encourage
324
+ * pack-reuse among packs containing reachable objects
325
+ * over unreachable ones.
326
+ *
327
+ * (Note we could write multiple packs here if
328
+ * `--max-pack-size` was given, but any one of them
329
+ * will suffice, so pick the first one.)
330
+ */
331
+ for_each_string_list_item(item, opts->names) {
332
+ struct generated_pack *pack = item->util;
333
+ if (generated_pack_has_ext(pack, ".mtimes"))
334
+ continue;
335
+
336
+ strvec_pushf(&cmd.args, "--preferred-pack=pack-%s.pack",
337
+ item->string);
338
+ break;
339
+ }
340
+ } else {
341
+ /*
342
+ * No packs were kept, and no packs were written. The
343
+ * only thing remaining are .keep packs (unless
344
+ * --pack-kept-objects was given).
345
+ *
346
+ * Set the `--preferred-pack` arbitrarily here.
347
+ */
348
+ ;
349
+ }
350
+
351
+ if (opts->refs_snapshot)
352
+ strvec_pushf(&cmd.args, "--refs-snapshot=%s",
353
+ opts->refs_snapshot);
354
+
355
+ ret = start_command(&cmd);
356
+ if (ret)
357
+ goto done;
358
+
359
+ in = xfdopen(cmd.in, "w");
360
+ for_each_string_list_item(item, &include)
361
+ fprintf(in, "%s\n", item->string);
362
+ fclose(in);
363
+
364
+ ret = finish_command(&cmd);
365
+done:
366
+ if (!ret && opts->write_bitmaps)
367
+ remove_redundant_bitmaps(&include, opts->packdir);
368
+
369
+ string_list_clear(&include, 0);
370
+
371
+ return ret;
372
+}
repack.h
+12
@@ -101,6 +101,18 @@ void pack_geometry_release(struct pack_geometry *geometry);
101
102
struct tempfile;
103
104
+struct repack_write_midx_opts {
105
+ struct existing_packs *existing;
106
+ struct pack_geometry *geometry;
107
+ struct string_list *names;
108
+ const char *refs_snapshot;
109
+ const char *packdir;
110
+ int show_progress;
111
+ int write_bitmaps;
112
+ int midx_must_contain_cruft;
113
+};
114
+
115
void midx_snapshot_refs(struct repository *repo, struct tempfile *f);
116
+int write_midx_included_packs(struct repack_write_midx_opts *opts);
117
118
#endif /* REPACK_H */