@matheusbsbs / matheus-bagliano / commits / 25a0023f28

builtin/refs: new command to migrate ref storage formats

Introduce a new command that allows the user to migrate a repository between ref storage formats. This new command is implemented as part of a new git-refs(1) executable. This is due to two reasons: - There is no good place to put the migration logic in existing commands. git-maintenance(1) felt unwieldy, and git-pack-refs(1) is not the correct place to put it, either. - I had it in my mind to create a new low-level command for accessing refs for quite a while already. git-refs(1) is that command and can over time grow more functionality relating to refs. This should help discoverability by consolidating low-level access to refs into a single executable. As mentioned in the preceding commit that introduces the ref storage format migration logic, the new `git refs migrate` command still has a bunch of restrictions. These restrictions are documented accordingly. Signed-off-by: Patrick Steinhardt <ps@pks.im> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Patrick Steinhardt committed Jun 6, 2024 at 07:29 UTC 25a0023f28600102f54e7529c20da5928c3e9c75
8 files changed +384
.gitignore
+1
@@ -126,6 +126,7 @@
126 /git-rebase
127 /git-receive-pack
128 /git-reflog
129 +/git-refs
130 /git-remote
131 /git-remote-http
132 /git-remote-https
Documentation/git-refs.txt new
+61
@@ -0,0 +1,61 @@
1 +git-refs(1)
2 +===========
3 +
4 +NAME
5 +----
6 +git-refs - Low-level access to refs
7 +
8 +
9 +SYNOPSIS
10 +--------
11 +[verse]
12 +'git refs migrate' --ref-format=<format> [--dry-run]
13 +
14 +DESCRIPTION
15 +-----------
16 +
17 +This command provides low-level access to refs.
18 +
19 +COMMANDS
20 +--------
21 +
22 +migrate::
23 + Migrate ref store between different formats.
24 +
25 +OPTIONS
26 +-------
27 +
28 +The following options are specific to 'git refs migrate':
29 +
30 +--ref-format=<format>::
31 + The ref format to migrate the ref store to. Can be one of:
32 ++
33 +include::ref-storage-format.txt[]
34 +
35 +--dry-run::
36 + Perform the migration, but do not modify the repository. The migrated
37 + refs will be written into a separate directory that can be inspected
38 + separately. The name of the directory will be reported on stdout. This
39 + can be used to double check that the migration works as expected before
40 + performing the actual migration.
41 +
42 +KNOWN LIMITATIONS
43 +-----------------
44 +
45 +The ref format migration has several known limitations in its current form:
46 +
47 +* It is not possible to migrate repositories that have reflogs.
48 +
49 +* It is not possible to migrate repositories that have worktrees.
50 +
51 +* There is no way to block concurrent writes to the repository during an
52 + ongoing migration. Concurrent writes can lead to an inconsistent migrated
53 + state. Users are expected to block writes on a higher level. If your
54 + repository is registered for scheduled maintenance, it is recommended to
55 + unregister it first with git-maintenance(1).
56 +
57 +These limitations may eventually be lifted.
58 +
59 +GIT
60 +---
61 +Part of the linkgit:git[1] suite
Makefile
+1
@@ -1283,6 +1283,7 @@ BUILTIN_OBJS += builtin/read-tree.o
1283 BUILTIN_OBJS += builtin/rebase.o
1284 BUILTIN_OBJS += builtin/receive-pack.o
1285 BUILTIN_OBJS += builtin/reflog.o
1286 +BUILTIN_OBJS += builtin/refs.o
1287 BUILTIN_OBJS += builtin/remote-ext.o
1288 BUILTIN_OBJS += builtin/remote-fd.o
1289 BUILTIN_OBJS += builtin/remote.o
builtin.h
+1
@@ -207,6 +207,7 @@ int cmd_rebase(int argc, const char **argv, const char *prefix);
207 int cmd_rebase__interactive(int argc, const char **argv, const char *prefix);
208 int cmd_receive_pack(int argc, const char **argv, const char *prefix);
209 int cmd_reflog(int argc, const char **argv, const char *prefix);
210 +int cmd_refs(int argc, const char **argv, const char *prefix);
211 int cmd_remote(int argc, const char **argv, const char *prefix);
212 int cmd_remote_ext(int argc, const char **argv, const char *prefix);
213 int cmd_remote_fd(int argc, const char **argv, const char *prefix);
builtin/refs.c new
+75
@@ -0,0 +1,75 @@
1 +#include "builtin.h"
2 +#include "parse-options.h"
3 +#include "refs.h"
4 +#include "repository.h"
5 +#include "strbuf.h"
6 +
7 +#define REFS_MIGRATE_USAGE \
8 + N_("git refs migrate --ref-format=<format> [--dry-run]")
9 +
10 +static int cmd_refs_migrate(int argc, const char **argv, const char *prefix)
11 +{
12 + const char * const migrate_usage[] = {
13 + REFS_MIGRATE_USAGE,
14 + NULL,
15 + };
16 + const char *format_str = NULL;
17 + enum ref_storage_format format;
18 + unsigned int flags = 0;
19 + struct option options[] = {
20 + OPT_STRING_F(0, "ref-format", &format_str, N_("format"),
21 + N_("specify the reference format to convert to"),
22 + PARSE_OPT_NONEG),
23 + OPT_BIT(0, "dry-run", &flags,
24 + N_("perform a non-destructive dry-run"),
25 + REPO_MIGRATE_REF_STORAGE_FORMAT_DRYRUN),
26 + OPT_END(),
27 + };
28 + struct strbuf errbuf = STRBUF_INIT;
29 + int err;
30 +
31 + argc = parse_options(argc, argv, prefix, options, migrate_usage, 0);
32 + if (argc)
33 + usage(_("too many arguments"));
34 + if (!format_str)
35 + usage(_("missing --ref-format=<format>"));
36 +
37 + format = ref_storage_format_by_name(format_str);
38 + if (format == REF_STORAGE_FORMAT_UNKNOWN) {
39 + err = error(_("unknown ref storage format '%s'"), format_str);
40 + goto out;
41 + }
42 +
43 + if (the_repository->ref_storage_format == format) {
44 + err = error(_("repository already uses '%s' format"),
45 + ref_storage_format_to_name(format));
46 + goto out;
47 + }
48 +
49 + if (repo_migrate_ref_storage_format(the_repository, format, flags, &errbuf) < 0) {
50 + err = error("%s", errbuf.buf);
51 + goto out;
52 + }
53 +
54 + err = 0;
55 +
56 +out:
57 + strbuf_release(&errbuf);
58 + return err;
59 +}
60 +
61 +int cmd_refs(int argc, const char **argv, const char *prefix)
62 +{
63 + const char * const refs_usage[] = {
64 + REFS_MIGRATE_USAGE,
65 + NULL,
66 + };
67 + parse_opt_subcommand_fn *fn = NULL;
68 + struct option opts[] = {
69 + OPT_SUBCOMMAND("migrate", &fn, cmd_refs_migrate),
70 + OPT_END(),
71 + };
72 +
73 + argc = parse_options(argc, argv, prefix, opts, refs_usage, 0);
74 + return fn(argc, argv, prefix);
75 +}
command-list.txt
+1
@@ -157,6 +157,7 @@ git-read-tree plumbingmanipulators
157 git-rebase mainporcelain history
158 git-receive-pack synchelpers
159 git-reflog ancillarymanipulators complete
160 +git-refs ancillarymanipulators complete
161 git-remote ancillarymanipulators complete
162 git-repack ancillarymanipulators complete
163 git-replace ancillarymanipulators complete
git.c
+1
@@ -594,6 +594,7 @@ static struct cmd_struct commands[] = {
594 { "rebase", cmd_rebase, RUN_SETUP | NEED_WORK_TREE },
595 { "receive-pack", cmd_receive_pack },
596 { "reflog", cmd_reflog, RUN_SETUP },
597 + { "refs", cmd_refs, RUN_SETUP },
598 { "remote", cmd_remote, RUN_SETUP },
599 { "remote-ext", cmd_remote_ext, NO_PARSEOPT },
600 { "remote-fd", cmd_remote_fd, NO_PARSEOPT },
t/t1460-refs-migrate.sh new
+243
@@ -0,0 +1,243 @@
1 +#!/bin/sh
2 +
3 +test_description='migration of ref storage backends'
4 +
5 +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
6 +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
7 +
8 +TEST_PASSES_SANITIZE_LEAK=true
9 +. ./test-lib.sh
10 +
11 +test_migration () {
12 + git -C "$1" for-each-ref --include-root-refs \
13 + --format='%(refname) %(objectname) %(symref)' >expect &&
14 + git -C "$1" refs migrate --ref-format="$2" &&
15 + git -C "$1" for-each-ref --include-root-refs \
16 + --format='%(refname) %(objectname) %(symref)' >actual &&
17 + test_cmp expect actual &&
18 +
19 + git -C "$1" rev-parse --show-ref-format >actual &&
20 + echo "$2" >expect &&
21 + test_cmp expect actual
22 +}
23 +
24 +test_expect_success 'setup' '
25 + rm -rf .git &&
26 + # The migration does not yet support reflogs.
27 + git config --global core.logAllRefUpdates false
28 +'
29 +
30 +test_expect_success "superfluous arguments" '
31 + test_when_finished "rm -rf repo" &&
32 + git init repo &&
33 + test_must_fail git -C repo refs migrate foo 2>err &&
34 + cat >expect <<-EOF &&
35 + usage: too many arguments
36 + EOF
37 + test_cmp expect err
38 +'
39 +
40 +test_expect_success "missing ref storage format" '
41 + test_when_finished "rm -rf repo" &&
42 + git init repo &&
43 + test_must_fail git -C repo refs migrate 2>err &&
44 + cat >expect <<-EOF &&
45 + usage: missing --ref-format=<format>
46 + EOF
47 + test_cmp expect err
48 +'
49 +
50 +test_expect_success "unknown ref storage format" '
51 + test_when_finished "rm -rf repo" &&
52 + git init repo &&
53 + test_must_fail git -C repo refs migrate \
54 + --ref-format=unknown 2>err &&
55 + cat >expect <<-EOF &&
56 + error: unknown ref storage format ${SQ}unknown${SQ}
57 + EOF
58 + test_cmp expect err
59 +'
60 +
61 +ref_formats="files reftable"
62 +for from_format in $ref_formats
63 +do
64 + for to_format in $ref_formats
65 + do
66 + if test "$from_format" = "$to_format"
67 + then
68 + continue
69 + fi
70 +
71 + test_expect_success "$from_format: migration to same format fails" '
72 + test_when_finished "rm -rf repo" &&
73 + git init --ref-format=$from_format repo &&
74 + test_must_fail git -C repo refs migrate \
75 + --ref-format=$from_format 2>err &&
76 + cat >expect <<-EOF &&
77 + error: repository already uses ${SQ}$from_format${SQ} format
78 + EOF
79 + test_cmp expect err
80 + '
81 +
82 + test_expect_success "$from_format -> $to_format: migration with reflog fails" '
83 + test_when_finished "rm -rf repo" &&
84 + git init --ref-format=$from_format repo &&
85 + test_config -C repo core.logAllRefUpdates true &&
86 + test_commit -C repo logged &&
87 + test_must_fail git -C repo refs migrate \
88 + --ref-format=$to_format 2>err &&
89 + cat >expect <<-EOF &&
90 + error: migrating reflogs is not supported yet
91 + EOF
92 + test_cmp expect err
93 + '
94 +
95 + test_expect_success "$from_format -> $to_format: migration with worktree fails" '
96 + test_when_finished "rm -rf repo" &&
97 + git init --ref-format=$from_format repo &&
98 + git -C repo worktree add wt &&
99 + test_must_fail git -C repo refs migrate \
100 + --ref-format=$to_format 2>err &&
101 + cat >expect <<-EOF &&
102 + error: migrating repositories with worktrees is not supported yet
103 + EOF
104 + test_cmp expect err
105 + '
106 +
107 + test_expect_success "$from_format -> $to_format: unborn HEAD" '
108 + test_when_finished "rm -rf repo" &&
109 + git init --ref-format=$from_format repo &&
110 + test_migration repo "$to_format"
111 + '
112 +
113 + test_expect_success "$from_format -> $to_format: single ref" '
114 + test_when_finished "rm -rf repo" &&
115 + git init --ref-format=$from_format repo &&
116 + test_commit -C repo initial &&
117 + test_migration repo "$to_format"
118 + '
119 +
120 + test_expect_success "$from_format -> $to_format: bare repository" '
121 + test_when_finished "rm -rf repo repo.git" &&
122 + git init --ref-format=$from_format repo &&
123 + test_commit -C repo initial &&
124 + git clone --ref-format=$from_format --mirror repo repo.git &&
125 + test_migration repo.git "$to_format"
126 + '
127 +
128 + test_expect_success "$from_format -> $to_format: dangling symref" '
129 + test_when_finished "rm -rf repo" &&
130 + git init --ref-format=$from_format repo &&
131 + test_commit -C repo initial &&
132 + git -C repo symbolic-ref BROKEN_HEAD refs/heads/nonexistent &&
133 + test_migration repo "$to_format" &&
134 + echo refs/heads/nonexistent >expect &&
135 + git -C repo symbolic-ref BROKEN_HEAD >actual &&
136 + test_cmp expect actual
137 + '
138 +
139 + test_expect_success "$from_format -> $to_format: broken ref" '
140 + test_when_finished "rm -rf repo" &&
141 + git init --ref-format=$from_format repo &&
142 + test_commit -C repo initial &&
143 + test-tool -C repo ref-store main update-ref "" refs/heads/broken \
144 + "$(test_oid 001)" "$ZERO_OID" REF_SKIP_CREATE_REFLOG,REF_SKIP_OID_VERIFICATION &&
145 + test_migration repo "$to_format" &&
146 + test_oid 001 >expect &&
147 + git -C repo rev-parse refs/heads/broken >actual &&
148 + test_cmp expect actual
149 + '
150 +
151 + test_expect_success "$from_format -> $to_format: pseudo-refs" '
152 + test_when_finished "rm -rf repo" &&
153 + git init --ref-format=$from_format repo &&
154 + test_commit -C repo initial &&
155 + git -C repo update-ref FOO_HEAD HEAD &&
156 + test_migration repo "$to_format"
157 + '
158 +
159 + test_expect_success "$from_format -> $to_format: special refs are left alone" '
160 + test_when_finished "rm -rf repo" &&
161 + git init --ref-format=$from_format repo &&
162 + test_commit -C repo initial &&
163 + git -C repo rev-parse HEAD >repo/.git/MERGE_HEAD &&
164 + git -C repo rev-parse MERGE_HEAD &&
165 + test_migration repo "$to_format" &&
166 + test_path_is_file repo/.git/MERGE_HEAD
167 + '
168 +
169 + test_expect_success "$from_format -> $to_format: a bunch of refs" '
170 + test_when_finished "rm -rf repo" &&
171 + git init --ref-format=$from_format repo &&
172 +
173 + test_commit -C repo initial &&
174 + cat >input <<-EOF &&
175 + create FOO_HEAD HEAD
176 + create refs/heads/branch-1 HEAD
177 + create refs/heads/branch-2 HEAD
178 + create refs/heads/branch-3 HEAD
179 + create refs/heads/branch-4 HEAD
180 + create refs/tags/tag-1 HEAD
181 + create refs/tags/tag-2 HEAD
182 + EOF
183 + git -C repo update-ref --stdin <input &&
184 + test_migration repo "$to_format"
185 + '
186 +
187 + test_expect_success "$from_format -> $to_format: dry-run migration does not modify repository" '
188 + test_when_finished "rm -rf repo" &&
189 + git init --ref-format=$from_format repo &&
190 + test_commit -C repo initial &&
191 + git -C repo refs migrate --dry-run \
192 + --ref-format=$to_format >output &&
193 + grep "Finished dry-run migration of refs" output &&
194 + test_path_is_dir repo/.git/ref_migration.* &&
195 + echo $from_format >expect &&
196 + git -C repo rev-parse --show-ref-format >actual &&
197 + test_cmp expect actual
198 + '
199 + done
200 +done
201 +
202 +test_expect_success 'migrating from files format deletes backend files' '
203 + test_when_finished "rm -rf repo" &&
204 + git init --ref-format=files repo &&
205 + test_commit -C repo first &&
206 + git -C repo pack-refs --all &&
207 + test_commit -C repo second &&
208 + git -C repo update-ref ORIG_HEAD HEAD &&
209 + git -C repo rev-parse HEAD >repo/.git/FETCH_HEAD &&
210 +
211 + test_path_is_file repo/.git/HEAD &&
212 + test_path_is_file repo/.git/ORIG_HEAD &&
213 + test_path_is_file repo/.git/refs/heads/main &&
214 + test_path_is_file repo/.git/packed-refs &&
215 +
216 + test_migration repo reftable &&
217 +
218 + echo "ref: refs/heads/.invalid" >expect &&
219 + test_cmp expect repo/.git/HEAD &&
220 + echo "this repository uses the reftable format" >expect &&
221 + test_cmp expect repo/.git/refs/heads &&
222 + test_path_is_file repo/.git/FETCH_HEAD &&
223 + test_path_is_missing repo/.git/ORIG_HEAD &&
224 + test_path_is_missing repo/.git/refs/heads/main &&
225 + test_path_is_missing repo/.git/logs &&
226 + test_path_is_missing repo/.git/packed-refs
227 +'
228 +
229 +test_expect_success 'migrating from reftable format deletes backend files' '
230 + test_when_finished "rm -rf repo" &&
231 + git init --ref-format=reftable repo &&
232 + test_commit -C repo first &&
233 +
234 + test_path_is_dir repo/.git/reftable &&
235 + test_migration repo files &&
236 +
237 + test_path_is_missing repo/.git/reftable &&
238 + echo "ref: refs/heads/main" >expect &&
239 + test_cmp expect repo/.git/HEAD &&
240 + test_path_is_file repo/.git/refs/heads/main
241 +'
242 +
243 +test_done