fetch: let --jobs=<n> parallelize --multiple, too
So far, `--jobs=<n>` only parallelizes submodule fetches/clones, not `--multiple` fetches, which is unintuitive, given that the option's name does not say anything about submodules in particular. Let's change that. With this patch, also fetches from multiple remotes are parallelized. For backwards-compatibility (and to prepare for a use case where submodule and multiple-remote fetches may need different parallelization limits), the config setting `submodule.fetchJobs` still only controls the submodule part of `git fetch`, while the newly-introduced setting `fetch.parallel` controls both (but can be overridden for submodules with `submodule.fetchJobs`). Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Johannes Schindelin committed
Oct 5, 2019 at 11:46 UTC
d54dea77dba081770fec7707110d8480ccaf9439
4 files changed
+137
-21
Documentation/config/fetch.txt
+10
@@ -68,3 +68,13 @@ fetch.showForcedUpdates::
68
Set to false to enable `--no-show-forced-updates` in
69
linkgit:git-fetch[1] and linkgit:git-pull[1] commands.
70
Defaults to true.
71
+
72
+fetch.parallel::
73
+ Specifies the maximal number of fetch operations to be run in parallel
74
+ at a time (submodules, or remotes when the `--multiple` option of
75
+ linkgit:git-fetch[1] is in effect).
76
++
77
+A value of 0 will give some reasonable default. If unset, it defaults to 1.
78
++
79
+For submodules, this setting can be overridden using the `submodule.fetchJobs`
80
+config setting.
Documentation/fetch-options.txt
+9
-4
@@ -160,10 +160,15 @@ ifndef::git-pull[]
160
161
-j::
162
--jobs=<n>::
163
- Number of parallel children to be used for fetching submodules.
164
- Each will fetch from different submodules, such that fetching many
165
- submodules will be faster. By default submodules will be fetched
166
- one at a time.
163
+ Number of parallel children to be used for all forms of fetching.
164
++
165
+If the `--multiple` option was specified, the different remotes will be fetched
166
+in parallel. If multiple submodules are fetched, they will be fetched in
167
+parallel. To control them independently, use the config settings
168
+`fetch.parallel` and `submodule.fetchJobs` (see linkgit:git-config[1]).
169
++
170
+Typically, parallel recursive and multi-remote fetches will be faster. By
171
+default fetches are performed sequentially, not in parallel.
172
173
--no-recurse-submodules::
174
Disable recursive fetching of submodules (this has the same effect as
builtin/fetch.c
+107
-17
@@ -54,7 +54,8 @@ static int all, append, dry_run, force, keep, multiple, update_head_ok, verbosit
54
static int progress = -1;
55
static int enable_auto_gc = 1;
56
static int tags = TAGS_DEFAULT, unshallow, update_shallow, deepen;
57
-static int max_children = 1;
57
+static int max_jobs = -1, submodule_fetch_jobs_config = -1;
58
+static int fetch_parallel_config = 1;
59
static enum transport_family family;
60
static const char *depth;
61
static const char *deepen_since;
@@ -96,13 +97,20 @@ static int git_fetch_config(const char *k, const char *v, void *cb)
97
}
98
99
if (!strcmp(k, "submodule.fetchjobs")) {
99
- max_children = parse_submodule_fetchjobs(k, v);
100
+ submodule_fetch_jobs_config = parse_submodule_fetchjobs(k, v);
101
return 0;
102
} else if (!strcmp(k, "fetch.recursesubmodules")) {
103
recurse_submodules = parse_fetch_recurse_submodules_arg(k, v);
104
return 0;
105
}
106
107
+ if (!strcmp(k, "fetch.parallel")) {
108
+ fetch_parallel_config = git_config_int(k, v);
109
+ if (fetch_parallel_config < 0)
110
+ die(_("fetch.parallel cannot be negative"));
111
+ return 0;
112
+ }
113
+
114
return git_default_config(k, v, cb);
115
}
116
@@ -134,7 +142,7 @@ static struct option builtin_fetch_options[] = {
142
N_("fetch all tags and associated objects"), TAGS_SET),
143
OPT_SET_INT('n', NULL, &tags,
144
N_("do not fetch all tags (--no-tags)"), TAGS_UNSET),
137
- OPT_INTEGER('j', "jobs", &max_children,
145
+ OPT_INTEGER('j', "jobs", &max_jobs,
146
N_("number of submodules fetched in parallel")),
147
OPT_BOOL('p', "prune", &prune,
148
N_("prune remote-tracking branches no longer on remote")),
@@ -1456,7 +1464,62 @@ static void add_options_to_argv(struct argv_array *argv)
1464
1465
}
1466
1459
-static int fetch_multiple(struct string_list *list)
1467
+/* Fetch multiple remotes in parallel */
1468
+
1469
+struct parallel_fetch_state {
1470
+ const char **argv;
1471
+ struct string_list *remotes;
1472
+ int next, result;
1473
+};
1474
+
1475
+static int fetch_next_remote(struct child_process *cp, struct strbuf *out,
1476
+ void *cb, void **task_cb)
1477
+{
1478
+ struct parallel_fetch_state *state = cb;
1479
+ char *remote;
1480
+
1481
+ if (state->next < 0 || state->next >= state->remotes->nr)
1482
+ return 0;
1483
+
1484
+ remote = state->remotes->items[state->next++].string;
1485
+ *task_cb = remote;
1486
+
1487
+ argv_array_pushv(&cp->args, state->argv);
1488
+ argv_array_push(&cp->args, remote);
1489
+ cp->git_cmd = 1;
1490
+
1491
+ if (verbosity >= 0)
1492
+ printf(_("Fetching %s\n"), remote);
1493
+
1494
+ return 1;
1495
+}
1496
+
1497
+static int fetch_failed_to_start(struct strbuf *out, void *cb, void *task_cb)
1498
+{
1499
+ struct parallel_fetch_state *state = cb;
1500
+ const char *remote = task_cb;
1501
+
1502
+ state->result = error(_("Could not fetch %s"), remote);
1503
+
1504
+ return 0;
1505
+}
1506
+
1507
+static int fetch_finished(int result, struct strbuf *out,
1508
+ void *cb, void *task_cb)
1509
+{
1510
+ struct parallel_fetch_state *state = cb;
1511
+ const char *remote = task_cb;
1512
+
1513
+ if (result) {
1514
+ strbuf_addf(out, _("could not fetch '%s' (exit code: %d)\n"),
1515
+ remote, result);
1516
+ state->result = -1;
1517
+ }
1518
+
1519
+ return 0;
1520
+}
1521
+
1522
+static int fetch_multiple(struct string_list *list, int max_children)
1523
{
1524
int i, result = 0;
1525
struct argv_array argv = ARGV_ARRAY_INIT;
@@ -1470,20 +1533,34 @@ static int fetch_multiple(struct string_list *list)
1533
argv_array_pushl(&argv, "fetch", "--append", "--no-auto-gc", NULL);
1534
add_options_to_argv(&argv);
1535
1473
- for (i = 0; i < list->nr; i++) {
1474
- const char *name = list->items[i].string;
1475
- argv_array_push(&argv, name);
1476
- if (verbosity >= 0)
1477
- printf(_("Fetching %s\n"), name);
1478
- if (run_command_v_opt(argv.argv, RUN_GIT_CMD)) {
1479
- error(_("Could not fetch %s"), name);
1480
- result = 1;
1536
+ if (max_children != 1 && list->nr != 1) {
1537
+ struct parallel_fetch_state state = { argv.argv, list, 0, 0 };
1538
+
1539
+ argv_array_push(&argv, "--end-of-options");
1540
+ result = run_processes_parallel_tr2(max_children,
1541
+ &fetch_next_remote,
1542
+ &fetch_failed_to_start,
1543
+ &fetch_finished,
1544
+ &state,
1545
+ "fetch", "parallel/fetch");
1546
+
1547
+ if (!result)
1548
+ result = state.result;
1549
+ } else
1550
+ for (i = 0; i < list->nr; i++) {
1551
+ const char *name = list->items[i].string;
1552
+ argv_array_push(&argv, name);
1553
+ if (verbosity >= 0)
1554
+ printf(_("Fetching %s\n"), name);
1555
+ if (run_command_v_opt(argv.argv, RUN_GIT_CMD)) {
1556
+ error(_("Could not fetch %s"), name);
1557
+ result = 1;
1558
+ }
1559
+ argv_array_pop(&argv);
1560
}
1482
- argv_array_pop(&argv);
1483
- }
1561
1562
argv_array_clear(&argv);
1486
- return result;
1563
+ return !!result;
1564
}
1565
1566
/*
@@ -1626,7 +1703,8 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
1703
for (i = 1; i < argc; i++)
1704
strbuf_addf(&default_rla, " %s", argv[i]);
1705
1629
- fetch_config_from_gitmodules(&max_children, &recurse_submodules);
1706
+ fetch_config_from_gitmodules(&submodule_fetch_jobs_config,
1707
+ &recurse_submodules);
1708
git_config(git_fetch_config, NULL);
1709
1710
argc = parse_options(argc, argv, prefix,
@@ -1692,15 +1770,27 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
1770
fetch_one_setup_partial(remote);
1771
result = fetch_one(remote, argc, argv, prune_tags_ok);
1772
} else {
1773
+ int max_children = max_jobs;
1774
+
1775
if (filter_options.choice)
1776
die(_("--filter can only be used with the remote "
1777
"configured in extensions.partialclone"));
1778
+
1779
+ if (max_children < 0)
1780
+ max_children = fetch_parallel_config;
1781
+
1782
/* TODO should this also die if we have a previous partial-clone? */
1699
- result = fetch_multiple(&list);
1783
+ result = fetch_multiple(&list, max_children);
1784
}
1785
1786
if (!result && (recurse_submodules != RECURSE_SUBMODULES_OFF)) {
1787
struct argv_array options = ARGV_ARRAY_INIT;
1788
+ int max_children = max_jobs;
1789
+
1790
+ if (max_children < 0)
1791
+ max_children = submodule_fetch_jobs_config;
1792
+ if (max_children < 0)
1793
+ max_children = fetch_parallel_config;
1794
1795
add_options_to_argv(&options);
1796
result = fetch_populated_submodules(the_repository,
t/t5514-fetch-multiple.sh
+11
@@ -183,4 +183,15 @@ test_expect_success 'git fetch --all --tags' '
183
test_cmp expect test8/output
184
'
185
186
+test_expect_success 'parallel' '
187
+ git remote add one ./bogus1 &&
188
+ git remote add two ./bogus2 &&
189
+
190
+ test_must_fail env GIT_TRACE="$PWD/trace" \
191
+ git fetch --jobs=2 --multiple one two 2>err &&
192
+ grep "preparing to run up to 2 tasks" trace &&
193
+ test_i18ngrep "could not fetch .one.*128" err &&
194
+ test_i18ngrep "could not fetch .two.*128" err
195
+'
196
+
197
test_done