@matheusbsbs / matheus-bagliano / commits / a35e03dee0

submodule: lazily add submodule ODBs as alternates

Teach Git to add submodule ODBs as alternates to the object store of the_repository only upon the first access of an object not in the_repository, and not when add_submodule_odb() is called. This provides a means of gradually migrating from accessing a submodule's object through alternates to accessing a submodule's object by explicitly passing its repository object. Any Git command can declare that it might access submodule objects by calling add_submodule_odb() (as they do now), but the submodule ODBs themselves will not be added until needed, so individual commands and/or combinations of arguments can be migrated one by one. [The advantage of explicit repository-object passing is code clarity (it is clear which repository an object read is from), performance (there is no need to linearly search through all submodule ODBs whenever an object is accessed from any repository, whether superproject or submodule), and the possibility of future features like partial clone submodules (which right now is not possible because if an object is missing, we do not know which repository to lazy-fetch into).] This commit also introduces an environment variable that a test may set to make the actual registration of alternates fatal, in order to demonstrate that its codepaths do not need this registration. Signed-off-by: Jonathan Tan <jonathantanmy@google.com> Reviewed-by: Emily Shaffer <emilyshaffer@google.com> Reviewed-by: Matheus Tavares <matheus.bernardino@usp.br> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Jonathan Tan committed Aug 16, 2021 at 14:09 UTC a35e03dee0e8daa442227018ecd180ae1c1b39bc

4 files changed +41 -1

object-file.c

@@ -32,6 +32,7 @@

32 #include "packfile.h"

33 #include "object-store.h"

34 #include "promisor-remote.h"

35 +#include "submodule.h"

37 /* The maximum size for an object header. */

38 #define MAX_HEADER_LEN 32

@@ -1592,6 +1593,10 @@ static int do_oid_object_info_extended(struct repository *r,

1593 break;

1594 }

1595

1596 + if (register_all_submodule_odb_as_alternates())

1597 + /* We added some alternates; retry */

1598 + continue;

1599 +

1600 /* Check if it is a missing object */

1601 if (fetch_if_missing && repo_has_promisor_remote(r) &&

1602 !already_retried &&

submodule.c

+19 -1

@@ -165,6 +165,8 @@ void stage_updated_gitmodules(struct index_state *istate)

165 die(_("staging updated .gitmodules failed"));

166 }

167

168 +static struct string_list added_submodule_odb_paths = STRING_LIST_INIT_NODUP;

169 +

170 /* TODO: remove this function, use repo_submodule_init instead. */

171 int add_submodule_odb(const char *path)

172 {

@@ -178,12 +180,28 @@ int add_submodule_odb(const char *path)

180 ret = -1;

181 goto done;

182 }

181 - add_to_alternates_memory(objects_directory.buf);

183 + string_list_insert(&added_submodule_odb_paths,

184 + strbuf_detach(&objects_directory, NULL));

185 done:

186 strbuf_release(&objects_directory);

187 return ret;

188 }

189

190 +int register_all_submodule_odb_as_alternates(void)

191 +{

192 + int i;

193 + int ret = added_submodule_odb_paths.nr;

194 +

195 + for (i = 0; i < added_submodule_odb_paths.nr; i++)

196 + add_to_alternates_memory(added_submodule_odb_paths.items[i].string);

197 + if (ret) {

198 + string_list_clear(&added_submodule_odb_paths, 0);

199 + if (git_env_bool("GIT_TEST_FATAL_REGISTER_SUBMODULE_ODB", 0))

200 + BUG("register_all_submodule_odb_as_alternates() called");

201 + }

202 + return ret;

203 +}

204 +

205 void set_diffopt_flags_from_submodule_config(struct diff_options *diffopt,

206 const char *path)

207 {

submodule.h

@@ -97,7 +97,14 @@ int submodule_uses_gitfile(const char *path);

97 #define SUBMODULE_REMOVAL_IGNORE_IGNORED_UNTRACKED (1<<2)

98 int bad_to_remove_submodule(const char *path, unsigned flags);

100 +/*

101 + * Call add_submodule_odb() to add the submodule at the given path to a list.

102 + * When register_all_submodule_odb_as_alternates() is called, the object stores

103 + * of all submodules in that list will be added as alternates in

104 + * the_repository.

105 + */

106 int add_submodule_odb(const char *path);

107 +int register_all_submodule_odb_as_alternates(void);

108

109 /*

110 * Checks if there are submodule changes in a..b. If a is the null OID,

t/README

+10

@@ -448,6 +448,16 @@ GIT_TEST_CHECKOUT_WORKERS=<n> overrides the 'checkout.workers' setting

448 to <n> and 'checkout.thresholdForParallelism' to 0, forcing the

449 execution of the parallel-checkout code.

450

451 +GIT_TEST_FATAL_REGISTER_SUBMODULE_ODB=<boolean>, when true, makes

452 +registering submodule ODBs as alternates a fatal action. Support for

453 +this environment variable can be removed once the migration to

454 +explicitly providing repositories when accessing submodule objects is

455 +complete (in which case we might want to replace this with a trace2

456 +call so that users can make it visible if accessing submodule objects

457 +without an explicit repository still happens) or needs to be abandoned

458 +for whatever reason (in which case the migrated codepaths still retain

459 +their performance benefits).

460 +

461 Naming Tests

462 ------------

463