gc: do not repack promisor packfiles
Teach gc to stop traversal at promisor objects, and to leave promisor packfiles alone. This has the effect of only repacking non-promisor packfiles, and preserves the distinction between promisor packfiles and non-promisor packfiles. Signed-off-by: Jonathan Tan <jonathantanmy@google.com> Signed-off-by: Jeff Hostetler <jeffhost@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Jonathan Tan committed
Dec 8, 2017 at 15:27 UTC
0c16cd499dee09b7fc6dd10cb6a476e96c147ef6
6 files changed
+114
-6
Documentation/git-pack-objects.txt
+11
@@ -255,6 +255,17 @@ a missing object is encountered. This is the default action.
255
The form '--missing=allow-any' will allow object traversal to continue
256
if a missing object is encountered. Missing objects will silently be
257
omitted from the results.
258
++
259
+The form '--missing=allow-promisor' is like 'allow-any', but will only
260
+allow object traversal to continue for EXPECTED promisor missing objects.
261
+Unexpected missing object will raise an error.
262
+
263
+--exclude-promisor-objects::
264
+ Omit objects that are known to be in the promisor remote. (This
265
+ option has the purpose of operating only on locally created objects,
266
+ so that when we repack, we still maintain a distinction between
267
+ locally created objects [without .promisor] and objects from the
268
+ promisor remote [with .promisor].) This is used with partial clone.
269
270
SEE ALSO
271
--------
builtin/gc.c
+3
@@ -458,6 +458,9 @@ int cmd_gc(int argc, const char **argv, const char *prefix)
458
argv_array_push(&prune, prune_expire);
459
if (quiet)
460
argv_array_push(&prune, "--no-progress");
461
+ if (repository_format_partial_clone)
462
+ argv_array_push(&prune,
463
+ "--exclude-promisor-objects");
464
if (run_command_v_opt(prune.argv, RUN_GIT_CMD))
465
return error(FAILED_RUN, prune.argv[0]);
466
}
builtin/pack-objects.c
+35
-2
@@ -75,6 +75,8 @@ static int use_bitmap_index = -1;
75
static int write_bitmap_index;
76
static uint16_t write_bitmap_options;
77
78
+static int exclude_promisor_objects;
79
+
80
static unsigned long delta_cache_size = 0;
81
static unsigned long max_delta_cache_size = 256 * 1024 * 1024;
82
static unsigned long cache_max_small_delta_size = 1000;
@@ -84,8 +86,9 @@ static unsigned long window_memory_limit = 0;
86
static struct list_objects_filter_options filter_options;
87
88
enum missing_action {
87
- MA_ERROR = 0, /* fail if any missing objects are encountered */
88
- MA_ALLOW_ANY, /* silently allow ALL missing objects */
89
+ MA_ERROR = 0, /* fail if any missing objects are encountered */
90
+ MA_ALLOW_ANY, /* silently allow ALL missing objects */
91
+ MA_ALLOW_PROMISOR, /* silently allow all missing PROMISOR objects */
92
};
93
static enum missing_action arg_missing_action;
94
static show_object_fn fn_show_object;
@@ -2577,6 +2580,20 @@ static void show_object__ma_allow_any(struct object *obj, const char *name, void
2580
show_object(obj, name, data);
2581
}
2582
2583
+static void show_object__ma_allow_promisor(struct object *obj, const char *name, void *data)
2584
+{
2585
+ assert(arg_missing_action == MA_ALLOW_PROMISOR);
2586
+
2587
+ /*
2588
+ * Quietly ignore EXPECTED missing objects. This avoids problems with
2589
+ * staging them now and getting an odd error later.
2590
+ */
2591
+ if (!has_object_file(&obj->oid) && is_promisor_object(&obj->oid))
2592
+ return;
2593
+
2594
+ show_object(obj, name, data);
2595
+}
2596
+
2597
static int option_parse_missing_action(const struct option *opt,
2598
const char *arg, int unset)
2599
{
@@ -2591,10 +2608,18 @@ static int option_parse_missing_action(const struct option *opt,
2608
2609
if (!strcmp(arg, "allow-any")) {
2610
arg_missing_action = MA_ALLOW_ANY;
2611
+ fetch_if_missing = 0;
2612
fn_show_object = show_object__ma_allow_any;
2613
return 0;
2614
}
2615
2616
+ if (!strcmp(arg, "allow-promisor")) {
2617
+ arg_missing_action = MA_ALLOW_PROMISOR;
2618
+ fetch_if_missing = 0;
2619
+ fn_show_object = show_object__ma_allow_promisor;
2620
+ return 0;
2621
+ }
2622
+
2623
die(_("invalid value for --missing"));
2624
return 0;
2625
}
@@ -3008,6 +3033,8 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
3033
{ OPTION_CALLBACK, 0, "missing", NULL, N_("action"),
3034
N_("handling for missing objects"), PARSE_OPT_NONEG,
3035
option_parse_missing_action },
3036
+ OPT_BOOL(0, "exclude-promisor-objects", &exclude_promisor_objects,
3037
+ N_("do not pack objects in promisor packfiles")),
3038
OPT_END(),
3039
};
3040
@@ -3053,6 +3080,12 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
3080
argv_array_push(&rp, "--unpacked");
3081
}
3082
3083
+ if (exclude_promisor_objects) {
3084
+ use_internal_rev_list = 1;
3085
+ fetch_if_missing = 0;
3086
+ argv_array_push(&rp, "--exclude-promisor-objects");
3087
+ }
3088
+
3089
if (!reuse_object)
3090
reuse_delta = 0;
3091
if (pack_compression_level == -1)
builtin/prune.c
+7
@@ -101,12 +101,15 @@ int cmd_prune(int argc, const char **argv, const char *prefix)
101
{
102
struct rev_info revs;
103
struct progress *progress = NULL;
104
+ int exclude_promisor_objects = 0;
105
const struct option options[] = {
106
OPT__DRY_RUN(&show_only, N_("do not remove, show only")),
107
OPT__VERBOSE(&verbose, N_("report pruned objects")),
108
OPT_BOOL(0, "progress", &show_progress, N_("show progress")),
109
OPT_EXPIRY_DATE(0, "expire", &expire,
110
N_("expire objects older than <time>")),
111
+ OPT_BOOL(0, "exclude-promisor-objects", &exclude_promisor_objects,
112
+ N_("limit traversal to objects outside promisor packfiles")),
113
OPT_END()
114
};
115
char *s;
@@ -139,6 +142,10 @@ int cmd_prune(int argc, const char **argv, const char *prefix)
142
show_progress = isatty(2);
143
if (show_progress)
144
progress = start_delayed_progress(_("Checking connectivity"), 0);
145
+ if (exclude_promisor_objects) {
146
+ fetch_if_missing = 0;
147
+ revs.exclude_promisor_objects = 1;
148
+ }
149
150
mark_reachable_objects(&revs, 1, expire, progress);
151
stop_progress(&progress);
builtin/repack.c
+6
-2
@@ -83,7 +83,8 @@ static void remove_pack_on_signal(int signo)
83
84
/*
85
* Adds all packs hex strings to the fname list, which do not
86
- * have a corresponding .keep file.
86
+ * have a corresponding .keep or .promisor file. These packs are not to
87
+ * be kept if we are going to pack everything into one file.
88
*/
89
static void get_non_kept_pack_filenames(struct string_list *fname_list)
90
{
@@ -101,7 +102,8 @@ static void get_non_kept_pack_filenames(struct string_list *fname_list)
102
103
fname = xmemdupz(e->d_name, len);
104
104
- if (!file_exists(mkpath("%s/%s.keep", packdir, fname)))
105
+ if (!file_exists(mkpath("%s/%s.keep", packdir, fname)) &&
106
+ !file_exists(mkpath("%s/%s.promisor", packdir, fname)))
107
string_list_append_nodup(fname_list, fname);
108
else
109
free(fname);
@@ -232,6 +234,8 @@ int cmd_repack(int argc, const char **argv, const char *prefix)
234
argv_array_push(&cmd.args, "--all");
235
argv_array_push(&cmd.args, "--reflog");
236
argv_array_push(&cmd.args, "--indexed-objects");
237
+ if (repository_format_partial_clone)
238
+ argv_array_push(&cmd.args, "--exclude-promisor-objects");
239
if (window)
240
argv_array_pushf(&cmd.args, "--window=%s", window);
241
if (window_memory)
t/t0410-partial-clone.sh
+52
-2
@@ -10,14 +10,16 @@ delete_object () {
10
11
pack_as_from_promisor () {
12
HASH=$(git -C repo pack-objects .git/objects/pack/pack) &&
13
- >repo/.git/objects/pack/pack-$HASH.promisor
13
+ >repo/.git/objects/pack/pack-$HASH.promisor &&
14
+ echo $HASH
15
}
16
17
promise_and_delete () {
18
HASH=$(git -C repo rev-parse "$1") &&
19
git -C repo tag -a -m message my_annotated_tag "$HASH" &&
20
git -C repo rev-parse my_annotated_tag | pack_as_from_promisor &&
20
- git -C repo tag -d my_annotated_tag &&
21
+ # tag -d prints a message to stdout, so redirect it
22
+ git -C repo tag -d my_annotated_tag >/dev/null &&
23
delete_object repo "$HASH"
24
}
25
@@ -261,6 +263,54 @@ test_expect_success 'rev-list accepts missing and promised objects on command li
263
git -C repo rev-list --exclude-promisor-objects --objects "$COMMIT" "$TREE" "$BLOB"
264
'
265
266
+test_expect_success 'gc does not repack promisor objects' '
267
+ rm -rf repo &&
268
+ test_create_repo repo &&
269
+ test_commit -C repo my_commit &&
270
+
271
+ TREE_HASH=$(git -C repo rev-parse HEAD^{tree}) &&
272
+ HASH=$(printf "$TREE_HASH\n" | pack_as_from_promisor) &&
273
+
274
+ git -C repo config core.repositoryformatversion 1 &&
275
+ git -C repo config extensions.partialclone "arbitrary string" &&
276
+ git -C repo gc &&
277
+
278
+ # Ensure that the promisor packfile still exists, and remove it
279
+ test -e repo/.git/objects/pack/pack-$HASH.pack &&
280
+ rm repo/.git/objects/pack/pack-$HASH.* &&
281
+
282
+ # Ensure that the single other pack contains the commit, but not the tree
283
+ ls repo/.git/objects/pack/pack-*.pack >packlist &&
284
+ test_line_count = 1 packlist &&
285
+ git verify-pack repo/.git/objects/pack/pack-*.pack -v >out &&
286
+ grep "$(git -C repo rev-parse HEAD)" out &&
287
+ ! grep "$TREE_HASH" out
288
+'
289
+
290
+test_expect_success 'gc stops traversal when a missing but promised object is reached' '
291
+ rm -rf repo &&
292
+ test_create_repo repo &&
293
+ test_commit -C repo my_commit &&
294
+
295
+ TREE_HASH=$(git -C repo rev-parse HEAD^{tree}) &&
296
+ HASH=$(promise_and_delete $TREE_HASH) &&
297
+
298
+ git -C repo config core.repositoryformatversion 1 &&
299
+ git -C repo config extensions.partialclone "arbitrary string" &&
300
+ git -C repo gc &&
301
+
302
+ # Ensure that the promisor packfile still exists, and remove it
303
+ test -e repo/.git/objects/pack/pack-$HASH.pack &&
304
+ rm repo/.git/objects/pack/pack-$HASH.* &&
305
+
306
+ # Ensure that the single other pack contains the commit, but not the tree
307
+ ls repo/.git/objects/pack/pack-*.pack >packlist &&
308
+ test_line_count = 1 packlist &&
309
+ git verify-pack repo/.git/objects/pack/pack-*.pack -v >out &&
310
+ grep "$(git -C repo rev-parse HEAD)" out &&
311
+ ! grep "$TREE_HASH" out
312
+'
313
+
314
LIB_HTTPD_PORT=12345 # default port, 410, cannot be used as non-root
315
. "$TEST_DIRECTORY"/lib-httpd.sh
316
start_httpd