@matheusbsbs / matheus-bagliano / commits / 826ae79fca

pack-refs: teach --exclude option to exclude refs from being packed

At GitLab, we have a system that creates ephemeral internal refs that don't live long before getting deleted. Having an option to exclude certain refs from a packed-refs file allows these internal references to be deleted much more efficiently. Add an --exclude option to the pack-refs builtin, and use the ref exclusions API to exclude certain refs from being packed into the final packed-refs file Signed-off-by: John Cai <johncai86@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

John Cai committed May 12, 2023 at 21:34 UTC 826ae79fca263bc2b70c54fddacb1603c5ebb9c6
11 files changed +69 -20
Documentation/git-pack-refs.txt
+11 -1
@@ -8,7 +8,7 @@ git-pack-refs - Pack heads and tags for efficient repository access
8 SYNOPSIS
9 --------
10 [verse]
11 -'git pack-refs' [--all] [--no-prune]
11 +'git pack-refs' [--all] [--no-prune] [--exclude <pattern>]
12
13 DESCRIPTION
14 -----------
@@ -60,6 +60,16 @@ with many branches of historical interests.
60 The command usually removes loose refs under `$GIT_DIR/refs`
61 hierarchy after packing them. This option tells it not to.
62
63 +--exclude <pattern>::
64 +
65 +Do not pack refs matching the given `glob(7)` pattern. Repetitions of this option
66 +accumulate exclusion patterns. Use `--no-exclude` to clear and reset the list of
67 +patterns. If a ref is already packed, including it with `--exclude` will not
68 +unpack it.
69 +
70 +When used with `--all`, pack only loose refs which do not match any of
71 +the provided `--exclude` patterns.
72 +
73
74 BUGS
75 ----
builtin/pack-refs.c
+16 -4
@@ -4,22 +4,34 @@
4 #include "parse-options.h"
5 #include "refs.h"
6 #include "repository.h"
7 +#include "revision.h"
8
9 static char const * const pack_refs_usage[] = {
9 - N_("git pack-refs [--all] [--no-prune]"),
10 + N_("git pack-refs [--all] [--no-prune] [--exclude <pattern>]"),
11 NULL
12 };
13
14 int cmd_pack_refs(int argc, const char **argv, const char *prefix)
15 {
16 unsigned int flags = PACK_REFS_PRUNE;
17 + static struct ref_exclusions excludes = REF_EXCLUSIONS_INIT;
18 + struct pack_refs_opts pack_refs_opts = {.exclusions = &excludes, .flags = flags};
19 + static struct string_list option_excluded_refs = STRING_LIST_INIT_NODUP;
20 + struct string_list_item *item;
21 +
22 struct option opts[] = {
17 - OPT_BIT(0, "all", &flags, N_("pack everything"), PACK_REFS_ALL),
18 - OPT_BIT(0, "prune", &flags, N_("prune loose refs (default)"), PACK_REFS_PRUNE),
23 + OPT_BIT(0, "all", &pack_refs_opts.flags, N_("pack everything"), PACK_REFS_ALL),
24 + OPT_BIT(0, "prune", &pack_refs_opts.flags, N_("prune loose refs (default)"), PACK_REFS_PRUNE),
25 + OPT_STRING_LIST(0, "exclude", &option_excluded_refs, N_("pattern"),
26 + N_("references to exclude")),
27 OPT_END(),
28 };
29 git_config(git_default_config, NULL);
30 if (parse_options(argc, argv, prefix, opts, pack_refs_usage, 0))
31 usage_with_options(pack_refs_usage, opts);
24 - return refs_pack_refs(get_main_ref_store(the_repository), flags);
32 +
33 + for_each_string_list_item(item, &option_excluded_refs)
34 + add_ref_exclusion(pack_refs_opts.exclusions, item->string);
35 +
36 + return refs_pack_refs(get_main_ref_store(the_repository), &pack_refs_opts);
37 }
refs.c
+2 -2
@@ -2132,9 +2132,9 @@ void base_ref_store_init(struct ref_store *refs, struct repository *repo,
2132 }
2133
2134 /* backend functions */
2135 -int refs_pack_refs(struct ref_store *refs, unsigned int flags)
2135 +int refs_pack_refs(struct ref_store *refs, struct pack_refs_opts *opts)
2136 {
2137 - return refs->be->pack_refs(refs, flags);
2137 + return refs->be->pack_refs(refs, opts);
2138 }
2139
2140 int peel_iterated_oid(const struct object_id *base, struct object_id *peeled)
refs.h
+6 -1
@@ -63,6 +63,11 @@ struct worktree;
63 #define RESOLVE_REF_NO_RECURSE 0x02
64 #define RESOLVE_REF_ALLOW_BAD_NAME 0x04
65
66 +struct pack_refs_opts {
67 + unsigned int flags;
68 + struct ref_exclusions *exclusions;
69 +};
70 +
71 const char *refs_resolve_ref_unsafe(struct ref_store *refs,
72 const char *refname,
73 int resolve_flags,
@@ -405,7 +410,7 @@ void warn_dangling_symrefs(FILE *fp, const char *msg_fmt,
410 * Write a packed-refs file for the current repository.
411 * flags: Combination of the above PACK_REFS_* flags.
412 */
408 -int refs_pack_refs(struct ref_store *refs, unsigned int flags);
413 +int refs_pack_refs(struct ref_store *refs, struct pack_refs_opts *opts);
414
415 /*
416 * Setup reflog before using. Fill in err and return -1 on failure.
refs/debug.c
+2 -2
@@ -122,10 +122,10 @@ static int debug_initial_transaction_commit(struct ref_store *refs,
122 return res;
123 }
124
125 -static int debug_pack_refs(struct ref_store *ref_store, unsigned int flags)
125 +static int debug_pack_refs(struct ref_store *ref_store, struct pack_refs_opts *opts)
126 {
127 struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store;
128 - int res = drefs->refs->be->pack_refs(drefs->refs, flags);
128 + int res = drefs->refs->be->pack_refs(drefs->refs, opts);
129 trace_printf_key(&trace_refs, "pack_refs: %d\n", res);
130 return res;
131 }
refs/files-backend.c
+10 -6
@@ -19,6 +19,7 @@
19 #include "../worktree.h"
20 #include "../wrapper.h"
21 #include "../write-or-die.h"
22 +#include "../revision.h"
23
24 /*
25 * This backend uses the following flags in `ref_update::flags` for
@@ -1173,15 +1174,18 @@ static void prune_refs(struct files_ref_store *refs, struct ref_to_prune **refs_
1174 */
1175 static int should_pack_ref(const char *refname,
1176 const struct object_id *oid, unsigned int ref_flags,
1176 - unsigned int pack_flags)
1177 + struct pack_refs_opts *opts)
1178 {
1179 /* Do not pack per-worktree refs: */
1180 if (parse_worktree_ref(refname, NULL, NULL, NULL) !=
1181 REF_WORKTREE_SHARED)
1182 return 0;
1183
1184 + if (ref_excluded(opts->exclusions, refname))
1185 + return 0;
1186 +
1187 /* Do not pack non-tags unless PACK_REFS_ALL is set: */
1184 - if (!(pack_flags & PACK_REFS_ALL) && !starts_with(refname, "refs/tags/"))
1188 + if (!(opts->flags & PACK_REFS_ALL) && !starts_with(refname, "refs/tags/"))
1189 return 0;
1190
1191 /* Do not pack symbolic refs: */
@@ -1195,7 +1199,8 @@ static int should_pack_ref(const char *refname,
1199 return 1;
1200 }
1201
1198 -static int files_pack_refs(struct ref_store *ref_store, unsigned int flags)
1202 +static int files_pack_refs(struct ref_store *ref_store,
1203 + struct pack_refs_opts *opts)
1204 {
1205 struct files_ref_store *refs =
1206 files_downcast(ref_store, REF_STORE_WRITE | REF_STORE_ODB,
@@ -1220,8 +1225,7 @@ static int files_pack_refs(struct ref_store *ref_store, unsigned int flags)
1225 * in the packed ref cache. If the reference should be
1226 * pruned, also add it to refs_to_prune.
1227 */
1223 - if (!should_pack_ref(iter->refname, iter->oid, iter->flags,
1224 - flags))
1228 + if (!should_pack_ref(iter->refname, iter->oid, iter->flags, opts))
1229 continue;
1230
1231 /*
@@ -1235,7 +1239,7 @@ static int files_pack_refs(struct ref_store *ref_store, unsigned int flags)
1239 iter->refname, err.buf);
1240
1241 /* Schedule the loose reference for pruning if requested. */
1238 - if ((flags & PACK_REFS_PRUNE)) {
1242 + if ((opts->flags & PACK_REFS_PRUNE)) {
1243 struct ref_to_prune *n;
1244 FLEX_ALLOC_STR(n, name, iter->refname);
1245 oidcpy(&n->oid, iter->oid);
refs/packed-backend.c
+1 -1
@@ -1576,7 +1576,7 @@ static int packed_delete_refs(struct ref_store *ref_store, const char *msg,
1576 }
1577
1578 static int packed_pack_refs(struct ref_store *ref_store UNUSED,
1579 - unsigned int flags UNUSED)
1579 + struct pack_refs_opts *pack_opts UNUSED)
1580 {
1581 /*
1582 * Packed refs are already packed. It might be that loose refs
refs/refs-internal.h
+2 -1
@@ -547,7 +547,8 @@ typedef int ref_transaction_commit_fn(struct ref_store *refs,
547 struct ref_transaction *transaction,
548 struct strbuf *err);
549
550 -typedef int pack_refs_fn(struct ref_store *ref_store, unsigned int flags);
550 +typedef int pack_refs_fn(struct ref_store *ref_store,
551 + struct pack_refs_opts *opts);
552 typedef int create_symref_fn(struct ref_store *ref_store,
553 const char *ref_target,
554 const char *refs_heads_master,
revision.h
+1 -1
@@ -86,7 +86,7 @@ struct rev_cmdline_info {
86 struct ref_exclusions {
87 /*
88 * Excluded refs is a list of wildmatch patterns. If any of the
89 - * patterns matches, the reference will be excluded.
89 + * patterns match, the reference will be excluded.
90 */
91 struct string_list excluded_refs;
92
t/helper/test-ref-store.c
+2 -1
@@ -116,8 +116,9 @@ static struct flag_definition pack_flags[] = { FLAG_DEF(PACK_REFS_PRUNE),
116 static int cmd_pack_refs(struct ref_store *refs, const char **argv)
117 {
118 unsigned int flags = arg_flags(*argv++, "flags", pack_flags);
119 + struct pack_refs_opts pack_opts = { .flags = flags };
120
120 - return refs_pack_refs(refs, flags);
121 + return refs_pack_refs(refs, &pack_opts);
122 }
123
124 static int cmd_create_symref(struct ref_store *refs, const char **argv)
t/t3210-pack-refs.sh
+16
@@ -108,6 +108,22 @@ test_expect_success \
108 git branch -d n/o/p &&
109 git branch n'
110
111 +test_expect_success 'test excluded refs are not packed' '
112 + git branch dont_pack1 &&
113 + git branch dont_pack2 &&
114 + git branch pack_this &&
115 + git pack-refs --all --exclude "refs/heads/dont_pack*" &&
116 + test -f .git/refs/heads/dont_pack1 &&
117 + test -f .git/refs/heads/dont_pack2 &&
118 + ! test -f .git/refs/heads/pack_this'
119 +
120 +test_expect_success 'test --no-exclude refs clears excluded refs' '
121 + git branch dont_pack3 &&
122 + git branch dont_pack4 &&
123 + git pack-refs --all --exclude "refs/heads/dont_pack*" --no-exclude &&
124 + ! test -f .git/refs/heads/dont_pack3 &&
125 + ! test -f .git/refs/heads/dont_pack4'
126 +
127 test_expect_success \
128 'see if up-to-date packed refs are preserved' \
129 'git branch q &&