submodules: allow parallel fetching, add tests and documentation
This enables the work of the previous patches. Signed-off-by: Stefan Beller <sbeller@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Stefan Beller committed
Dec 15, 2015 at 16:04 UTC
62104ba14af4845c6c1ba1dab05fad4a289d806f
6 files changed
+40
-4
Documentation/fetch-options.txt
+7
@@ -100,6 +100,13 @@ ifndef::git-pull[]
100
reference to a commit that isn't already in the local submodule
101
clone.
102
103
+-j::
104
+--jobs=<n>::
105
+ Number of parallel children to be used for fetching submodules.
106
+ Each will fetch from different submodules, such that fetching many
107
+ submodules will be faster. By default submodules will be fetched
108
+ one at a time.
109
+
110
--no-recurse-submodules::
111
Disable recursive fetching of submodules (this has the same effect as
112
using the '--recurse-submodules=no' option).
builtin/fetch.c
+5
-1
@@ -37,6 +37,7 @@ static int prune = -1; /* unspecified */
37
static int all, append, dry_run, force, keep, multiple, update_head_ok, verbosity;
38
static int progress = -1, recurse_submodules = RECURSE_SUBMODULES_DEFAULT;
39
static int tags = TAGS_DEFAULT, unshallow, update_shallow;
40
+static int max_children = 1;
41
static const char *depth;
42
static const char *upload_pack;
43
static struct strbuf default_rla = STRBUF_INIT;
@@ -99,6 +100,8 @@ static struct option builtin_fetch_options[] = {
100
N_("fetch all tags and associated objects"), TAGS_SET),
101
OPT_SET_INT('n', NULL, &tags,
102
N_("do not fetch all tags (--no-tags)"), TAGS_UNSET),
103
+ OPT_INTEGER('j', "jobs", &max_children,
104
+ N_("number of submodules fetched in parallel")),
105
OPT_BOOL('p', "prune", &prune,
106
N_("prune remote-tracking branches no longer on remote")),
107
{ OPTION_CALLBACK, 0, "recurse-submodules", NULL, N_("on-demand"),
@@ -1213,7 +1216,8 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
1216
result = fetch_populated_submodules(&options,
1217
submodule_prefix,
1218
recurse_submodules,
1216
- verbosity < 0);
1219
+ verbosity < 0,
1220
+ max_children);
1221
argv_array_clear(&options);
1222
}
1223
builtin/pull.c
+6
@@ -95,6 +95,7 @@ static int opt_force;
95
static char *opt_tags;
96
static char *opt_prune;
97
static char *opt_recurse_submodules;
98
+static char *max_children;
99
static int opt_dry_run;
100
static char *opt_keep;
101
static char *opt_depth;
@@ -178,6 +179,9 @@ static struct option pull_options[] = {
179
N_("on-demand"),
180
N_("control recursive fetching of submodules"),
181
PARSE_OPT_OPTARG),
182
+ OPT_PASSTHRU('j', "jobs", &max_children, N_("n"),
183
+ N_("number of submodules pulled in parallel"),
184
+ PARSE_OPT_OPTARG),
185
OPT_BOOL(0, "dry-run", &opt_dry_run,
186
N_("dry run")),
187
OPT_PASSTHRU('k', "keep", &opt_keep, NULL,
@@ -525,6 +529,8 @@ static int run_fetch(const char *repo, const char **refspecs)
529
argv_array_push(&args, opt_prune);
530
if (opt_recurse_submodules)
531
argv_array_push(&args, opt_recurse_submodules);
532
+ if (max_children)
533
+ argv_array_push(&args, max_children);
534
if (opt_dry_run)
535
argv_array_push(&args, "--dry-run");
536
if (opt_keep)
submodule.c
+1
-2
@@ -729,10 +729,9 @@ static int fetch_finish(int retvalue, struct child_process *cp,
729
730
int fetch_populated_submodules(const struct argv_array *options,
731
const char *prefix, int command_line_option,
732
- int quiet)
732
+ int quiet, int max_parallel_jobs)
733
{
734
int i;
735
- int max_parallel_jobs = 1;
735
struct submodule_parallel_fetch spf = SPF_INIT;
736
737
spf.work_tree = get_git_work_tree();
submodule.h
+1
-1
@@ -31,7 +31,7 @@ void set_config_fetch_recurse_submodules(int value);
31
void check_for_new_submodule_commits(unsigned char new_sha1[20]);
32
int fetch_populated_submodules(const struct argv_array *options,
33
const char *prefix, int command_line_option,
34
- int quiet);
34
+ int quiet, int max_parallel_jobs);
35
unsigned is_submodule_modified(const char *path, int ignore_untracked);
36
int submodule_uses_gitfile(const char *path);
37
int ok_to_remove_submodule(const char *path);
t/t5526-fetch-submodules.sh
+20
@@ -71,6 +71,17 @@ test_expect_success "fetch --recurse-submodules recurses into submodules" '
71
test_i18ncmp expect.err actual.err
72
'
73
74
+test_expect_success "fetch --recurse-submodules -j2 has the same output behaviour" '
75
+ add_upstream_commit &&
76
+ (
77
+ cd downstream &&
78
+ GIT_TRACE=$(pwd)/../trace.out git fetch --recurse-submodules -j2 2>../actual.err
79
+ ) &&
80
+ test_must_be_empty actual.out &&
81
+ test_i18ncmp expect.err actual.err &&
82
+ grep "2 tasks" trace.out
83
+'
84
+
85
test_expect_success "fetch alone only fetches superproject" '
86
add_upstream_commit &&
87
(
@@ -140,6 +151,15 @@ test_expect_success "--quiet propagates to submodules" '
151
! test -s actual.err
152
'
153
154
+test_expect_success "--quiet propagates to parallel submodules" '
155
+ (
156
+ cd downstream &&
157
+ git fetch --recurse-submodules -j 2 --quiet >../actual.out 2>../actual.err
158
+ ) &&
159
+ ! test -s actual.out &&
160
+ ! test -s actual.err
161
+'
162
+
163
test_expect_success "--dry-run propagates to submodules" '
164
add_upstream_commit &&
165
(