promisor-remote: teach lazy-fetch in any repo
This is one step towards supporting partial clone submodules. Even after this patch, we will still lack partial clone submodules support, primarily because a lot of Git code that accesses submodule objects does so by adding their object stores as alternates, meaning that any lazy fetches that would occur in the submodule would be done based on the config of the superproject, not of the submodule. This also prevents testing of the functionality in this patch by user-facing commands. So for now, test this mechanism using a test helper. Besides that, there is some code that uses the wrapper functions like has_promisor_remote(). Those will need to be checked to see if they could support the non-wrapper functions instead (and thus support any repository, not just the_repository). Signed-off-by: Jonathan Tan <jonathantanmy@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com> Reviewed-by: Elijah Newren <newren@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Jonathan Tan committed
Jun 17, 2021 at 10:13 UTC
ef830cc4341260ef45ffe6c7164e23505d45a5a2
7 files changed
+76
-9
Makefile
+1
@@ -725,6 +725,7 @@ TEST_BUILTINS_OBJS += test-oidmap.o
725
TEST_BUILTINS_OBJS += test-online-cpus.o
726
TEST_BUILTINS_OBJS += test-parse-options.o
727
TEST_BUILTINS_OBJS += test-parse-pathspec-file.o
728
+TEST_BUILTINS_OBJS += test-partial-clone.o
729
TEST_BUILTINS_OBJS += test-path-utils.o
730
TEST_BUILTINS_OBJS += test-pcre2-config.o
731
TEST_BUILTINS_OBJS += test-pkt-line.o
object-file.c
+2
-5
@@ -1570,15 +1570,12 @@ static int do_oid_object_info_extended(struct repository *r,
1570
}
1571
1572
/* Check if it is a missing object */
1573
- if (fetch_if_missing && has_promisor_remote() &&
1574
- !already_retried && r == the_repository &&
1573
+ if (fetch_if_missing && repo_has_promisor_remote(r) &&
1574
+ !already_retried &&
1575
!(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) {
1576
/*
1577
* TODO Investigate checking promisor_remote_get_direct()
1578
* TODO return value and stopping on error here.
1579
- * TODO Pass a repository struct through
1580
- * promisor_remote_get_direct(), such that arbitrary
1581
- * repositories work.
1579
*/
1580
promisor_remote_get_direct(r, real, 1);
1581
already_retried = 1;
promisor-remote.c
+5
-4
@@ -10,7 +10,8 @@ struct promisor_remote_config {
10
struct promisor_remote **promisors_tail;
11
};
12
13
-static int fetch_objects(const char *remote_name,
13
+static int fetch_objects(struct repository *repo,
14
+ const char *remote_name,
15
const struct object_id *oids,
16
int oid_nr)
17
{
@@ -20,6 +21,8 @@ static int fetch_objects(const char *remote_name,
21
22
child.git_cmd = 1;
23
child.in = -1;
24
+ if (repo != the_repository)
25
+ prepare_other_repo_env(&child.env_array, repo->gitdir);
26
strvec_pushl(&child.args, "-c", "fetch.negotiationAlgorithm=noop",
27
"fetch", remote_name, "--no-tags",
28
"--no-write-fetch-head", "--recurse-submodules=no",
@@ -240,10 +243,8 @@ int promisor_remote_get_direct(struct repository *repo,
243
244
promisor_remote_init(repo);
245
243
- if (repo != the_repository)
244
- BUG("only the_repository is supported for now");
246
for (r = repo->promisor_remote_config->promisors; r; r = r->next) {
246
- if (fetch_objects(r->name, remaining_oids, remaining_nr) < 0) {
247
+ if (fetch_objects(repo, r->name, remaining_oids, remaining_nr) < 0) {
248
if (remaining_nr == 1)
249
continue;
250
remaining_nr = remove_fetched_oids(repo, &remaining_oids,
t/helper/test-partial-clone.c
new
+43
@@ -0,0 +1,43 @@
1
+#include "cache.h"
2
+#include "test-tool.h"
3
+#include "repository.h"
4
+#include "object-store.h"
5
+
6
+/*
7
+ * Prints the size of the object corresponding to the given hash in a specific
8
+ * gitdir. This is similar to "git -C gitdir cat-file -s", except that this
9
+ * exercises the code that accesses the object of an arbitrary repository that
10
+ * is not the_repository. ("git -C gitdir" makes it so that the_repository is
11
+ * the one in gitdir.)
12
+ */
13
+static void object_info(const char *gitdir, const char *oid_hex)
14
+{
15
+ struct repository r;
16
+ struct object_id oid;
17
+ unsigned long size;
18
+ struct object_info oi = {.sizep = &size};
19
+ const char *p;
20
+
21
+ if (repo_init(&r, gitdir, NULL))
22
+ die("could not init repo");
23
+ if (parse_oid_hex(oid_hex, &oid, &p))
24
+ die("could not parse oid");
25
+ if (oid_object_info_extended(&r, &oid, &oi, 0))
26
+ die("could not obtain object info");
27
+ printf("%d\n", (int) size);
28
+}
29
+
30
+int cmd__partial_clone(int argc, const char **argv)
31
+{
32
+ setup_git_directory();
33
+
34
+ if (argc < 4)
35
+ die("too few arguments");
36
+
37
+ if (!strcmp(argv[1], "object-info"))
38
+ object_info(argv[2], argv[3]);
39
+ else
40
+ die("invalid argument '%s'", argv[1]);
41
+
42
+ return 0;
43
+}
t/helper/test-tool.c
+1
@@ -46,6 +46,7 @@ static struct test_cmd cmds[] = {
46
{ "online-cpus", cmd__online_cpus },
47
{ "parse-options", cmd__parse_options },
48
{ "parse-pathspec-file", cmd__parse_pathspec_file },
49
+ { "partial-clone", cmd__partial_clone },
50
{ "path-utils", cmd__path_utils },
51
{ "pcre2-config", cmd__pcre2_config },
52
{ "pkt-line", cmd__pkt_line },
t/helper/test-tool.h
+1
@@ -35,6 +35,7 @@ int cmd__oidmap(int argc, const char **argv);
35
int cmd__online_cpus(int argc, const char **argv);
36
int cmd__parse_options(int argc, const char **argv);
37
int cmd__parse_pathspec_file(int argc, const char** argv);
38
+int cmd__partial_clone(int argc, const char **argv);
39
int cmd__path_utils(int argc, const char **argv);
40
int cmd__pcre2_config(int argc, const char **argv);
41
int cmd__pkt_line(int argc, const char **argv);
t/t0410-partial-clone.sh
+23
@@ -604,6 +604,29 @@ test_expect_success 'do not fetch when checking existence of tree we construct o
604
git -C repo cherry-pick side1
605
'
606
607
+test_expect_success 'lazy-fetch when accessing object not in the_repository' '
608
+ rm -rf full partial.git &&
609
+ test_create_repo full &&
610
+ test_commit -C full create-a-file file.txt &&
611
+
612
+ test_config -C full uploadpack.allowfilter 1 &&
613
+ test_config -C full uploadpack.allowanysha1inwant 1 &&
614
+ git clone --filter=blob:none --bare "file://$(pwd)/full" partial.git &&
615
+ FILE_HASH=$(git -C full rev-parse HEAD:file.txt) &&
616
+
617
+ # Sanity check that the file is missing
618
+ git -C partial.git rev-list --objects --missing=print HEAD >out &&
619
+ grep "[?]$FILE_HASH" out &&
620
+
621
+ git -C full cat-file -s "$FILE_HASH" >expect &&
622
+ test-tool partial-clone object-info partial.git "$FILE_HASH" >actual &&
623
+ test_cmp expect actual &&
624
+
625
+ # Sanity check that the file is now present
626
+ git -C partial.git rev-list --objects --missing=print HEAD >out &&
627
+ ! grep "[?]$FILE_HASH" out
628
+'
629
+
630
. "$TEST_DIRECTORY"/lib-httpd.sh
631
start_httpd
632