fetching submodules: respect `submodule.fetchJobs` config option
This allows to configure fetching and updating in parallel without having the command line option. This moved the responsibility to determine how many parallel processes to start from builtin/fetch to submodule.c as we need a way to communicate "The user did not specify the number of parallel processes in the command line options" in the builtin fetch. The submodule code takes care of the precedence (CLI > config > default). Reviewed-by: Jonathan Nieder <jrnieder@gmail.com> Signed-off-by: Stefan Beller <sbeller@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Stefan Beller committed
Feb 29, 2016 at 18:07 UTC
a028a1930c6b4b848e8fb47cc92c30b23d99a75e
5 files changed
+38
-2
Documentation/config.txt
+6
@@ -2646,6 +2646,12 @@ submodule.<name>.ignore::
2646
"--ignore-submodules" option. The 'git submodule' commands are not
2647
affected by this setting.
2648
2649
+submodule.fetchJobs::
2650
+ Specifies how many submodules are fetched/cloned at the same time.
2651
+ A positive integer allows up to that number of submodules fetched
2652
+ in parallel. A value of 0 will give some reasonable default.
2653
+ If unset, it defaults to 1.
2654
+
2655
tag.sort::
2656
This variable controls the sort ordering of tags when displayed by
2657
linkgit:git-tag[1]. Without the "--sort=<value>" option provided, the
builtin/fetch.c
+1
-1
@@ -37,7 +37,7 @@ static int prune = -1; /* unspecified */
37
static int all, append, dry_run, force, keep, multiple, update_head_ok, verbosity;
38
static int progress = -1, recurse_submodules = RECURSE_SUBMODULES_DEFAULT;
39
static int tags = TAGS_DEFAULT, unshallow, update_shallow;
40
-static int max_children = 1;
40
+static int max_children = -1;
41
static const char *depth;
42
static const char *upload_pack;
43
static struct strbuf default_rla = STRBUF_INIT;
submodule.c
+15
-1
@@ -15,6 +15,7 @@
15
#include "thread-utils.h"
16
17
static int config_fetch_recurse_submodules = RECURSE_SUBMODULES_ON_DEMAND;
18
+static int parallel_jobs = 1;
19
static struct string_list changed_submodule_paths;
20
static int initialized_fetch_ref_tips;
21
static struct sha1_array ref_tips_before_fetch;
@@ -169,7 +170,12 @@ void set_diffopt_flags_from_submodule_config(struct diff_options *diffopt,
170
171
int submodule_config(const char *var, const char *value, void *cb)
172
{
172
- if (starts_with(var, "submodule."))
173
+ if (!strcmp(var, "submodule.fetchjobs")) {
174
+ parallel_jobs = git_config_int(var, value);
175
+ if (parallel_jobs < 0)
176
+ die(_("negative values not allowed for submodule.fetchJobs"));
177
+ return 0;
178
+ } else if (starts_with(var, "submodule."))
179
return parse_submodule_config_option(var, value);
180
else if (!strcmp(var, "fetch.recursesubmodules")) {
181
config_fetch_recurse_submodules = parse_fetch_recurse_submodules_arg(var, value);
@@ -771,6 +777,9 @@ int fetch_populated_submodules(const struct argv_array *options,
777
argv_array_push(&spf.args, "--recurse-submodules-default");
778
/* default value, "--submodule-prefix" and its value are added later */
779
780
+ if (max_parallel_jobs < 0)
781
+ max_parallel_jobs = parallel_jobs;
782
+
783
calculate_changed_submodule_paths();
784
run_processes_parallel(max_parallel_jobs,
785
get_next_submodule,
@@ -1117,3 +1126,8 @@ void connect_work_tree_and_git_dir(const char *work_tree, const char *git_dir)
1126
strbuf_release(&rel_path);
1127
free((void *)real_work_tree);
1128
}
1129
+
1130
+int parallel_submodules(void)
1131
+{
1132
+ return parallel_jobs;
1133
+}
submodule.h
+2
@@ -26,6 +26,7 @@ struct submodule_update_strategy {
26
enum submodule_update_type type;
27
const char *command;
28
};
29
+#define SUBMODULE_UPDATE_STRATEGY_INIT {SM_UPDATE_UNSPECIFIED, NULL}
30
31
int is_staging_gitmodules_ok(void);
32
int update_path_in_gitmodules(const char *oldpath, const char *newpath);
@@ -57,5 +58,6 @@ int find_unpushed_submodules(unsigned char new_sha1[20], const char *remotes_nam
58
struct string_list *needs_pushing);
59
int push_unpushed_submodules(unsigned char new_sha1[20], const char *remotes_name);
60
void connect_work_tree_and_git_dir(const char *work_tree, const char *git_dir);
61
+int parallel_submodules(void);
62
63
#endif
t/t5526-fetch-submodules.sh
+14
@@ -471,4 +471,18 @@ test_expect_success "don't fetch submodule when newly recorded commits are alrea
471
test_i18ncmp expect.err actual.err
472
'
473
474
+test_expect_success 'fetching submodules respects parallel settings' '
475
+ git config fetch.recurseSubmodules true &&
476
+ (
477
+ cd downstream &&
478
+ GIT_TRACE=$(pwd)/trace.out git fetch --jobs 7 &&
479
+ grep "7 tasks" trace.out &&
480
+ git config submodule.fetchJobs 8 &&
481
+ GIT_TRACE=$(pwd)/trace.out git fetch &&
482
+ grep "8 tasks" trace.out &&
483
+ GIT_TRACE=$(pwd)/trace.out git fetch --jobs 9 &&
484
+ grep "9 tasks" trace.out
485
+ )
486
+'
487
+
488
test_done