git submodule update: have a dedicated helper for cloning
This introduces a new helper function in git submodule--helper which takes care of cloning all submodules, which we want to parallelize eventually. Some tests (such as empty URL, update_mode=none) are required in the helper to make the decision for cloning. These checks have been moved into the C function as well (no need to repeat them in the shell script). Reviewed-by: Jonathan Nieder <jrnieder@gmail.com> Signed-off-by: Stefan Beller <sbeller@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Stefan Beller committed
Feb 29, 2016 at 18:07 UTC
48308681b072a1d32e1361c255347324a8ad151e
2 files changed
+259
-34
builtin/submodule--helper.c
+246
@@ -255,6 +255,251 @@ static int module_clone(int argc, const char **argv, const char *prefix)
255
return 0;
256
}
257
258
+struct submodule_update_clone {
259
+ /* index into 'list', the list of submodules to look into for cloning */
260
+ int current;
261
+ struct module_list list;
262
+ unsigned warn_if_uninitialized : 1;
263
+
264
+ /* update parameter passed via commandline */
265
+ struct submodule_update_strategy update;
266
+
267
+ /* configuration parameters which are passed on to the children */
268
+ int quiet;
269
+ const char *reference;
270
+ const char *depth;
271
+ const char *recursive_prefix;
272
+ const char *prefix;
273
+
274
+ /* Machine-readable status lines to be consumed by git-submodule.sh */
275
+ struct string_list projectlines;
276
+
277
+ /* If we want to stop as fast as possible and return an error */
278
+ unsigned quickstop : 1;
279
+};
280
+#define SUBMODULE_UPDATE_CLONE_INIT {0, MODULE_LIST_INIT, 0, \
281
+ SUBMODULE_UPDATE_STRATEGY_INIT, 0, NULL, NULL, NULL, NULL, \
282
+ STRING_LIST_INIT_DUP, 0}
283
+
284
+/**
285
+ * Determine whether 'ce' needs to be cloned. If so, prepare the 'child' to
286
+ * run the clone. Returns 1 if 'ce' needs to be cloned, 0 otherwise.
287
+ */
288
+static int prepare_to_clone_next_submodule(const struct cache_entry *ce,
289
+ struct child_process *child,
290
+ struct submodule_update_clone *suc,
291
+ struct strbuf *out)
292
+{
293
+ const struct submodule *sub = NULL;
294
+ struct strbuf displaypath_sb = STRBUF_INIT;
295
+ struct strbuf sb = STRBUF_INIT;
296
+ const char *displaypath = NULL;
297
+ char *url = NULL;
298
+ int needs_cloning = 0;
299
+
300
+ if (ce_stage(ce)) {
301
+ if (suc->recursive_prefix)
302
+ strbuf_addf(&sb, "%s/%s", suc->recursive_prefix, ce->name);
303
+ else
304
+ strbuf_addf(&sb, "%s", ce->name);
305
+ strbuf_addf(out, _("Skipping unmerged submodule %s"), sb.buf);
306
+ strbuf_addch(out, '\n');
307
+ goto cleanup;
308
+ }
309
+
310
+ sub = submodule_from_path(null_sha1, ce->name);
311
+
312
+ if (suc->recursive_prefix)
313
+ displaypath = relative_path(suc->recursive_prefix,
314
+ ce->name, &displaypath_sb);
315
+ else
316
+ displaypath = ce->name;
317
+
318
+ if (suc->update.type == SM_UPDATE_NONE
319
+ || (suc->update.type == SM_UPDATE_UNSPECIFIED
320
+ && sub->update_strategy.type == SM_UPDATE_NONE)) {
321
+ strbuf_addf(out, _("Skipping submodule '%s'"), displaypath);
322
+ strbuf_addch(out, '\n');
323
+ goto cleanup;
324
+ }
325
+
326
+ /*
327
+ * Looking up the url in .git/config.
328
+ * We must not fall back to .gitmodules as we only want
329
+ * to process configured submodules.
330
+ */
331
+ strbuf_reset(&sb);
332
+ strbuf_addf(&sb, "submodule.%s.url", sub->name);
333
+ git_config_get_string(sb.buf, &url);
334
+ if (!url) {
335
+ /*
336
+ * Only mention uninitialized submodules when their
337
+ * path have been specified
338
+ */
339
+ if (suc->warn_if_uninitialized) {
340
+ strbuf_addf(out,
341
+ _("Submodule path '%s' not initialized"),
342
+ displaypath);
343
+ strbuf_addch(out, '\n');
344
+ strbuf_addstr(out,
345
+ _("Maybe you want to use 'update --init'?"));
346
+ strbuf_addch(out, '\n');
347
+ }
348
+ goto cleanup;
349
+ }
350
+
351
+ strbuf_reset(&sb);
352
+ strbuf_addf(&sb, "%s/.git", ce->name);
353
+ needs_cloning = !file_exists(sb.buf);
354
+
355
+ strbuf_reset(&sb);
356
+ strbuf_addf(&sb, "%06o %s %d %d\t%s\n", ce->ce_mode,
357
+ sha1_to_hex(ce->sha1), ce_stage(ce),
358
+ needs_cloning, ce->name);
359
+ string_list_append(&suc->projectlines, sb.buf);
360
+
361
+ if (!needs_cloning)
362
+ goto cleanup;
363
+
364
+ child->git_cmd = 1;
365
+ child->no_stdin = 1;
366
+ child->stdout_to_stderr = 1;
367
+ child->err = -1;
368
+ argv_array_push(&child->args, "submodule--helper");
369
+ argv_array_push(&child->args, "clone");
370
+ if (suc->quiet)
371
+ argv_array_push(&child->args, "--quiet");
372
+ if (suc->prefix)
373
+ argv_array_pushl(&child->args, "--prefix", suc->prefix, NULL);
374
+ argv_array_pushl(&child->args, "--path", sub->path, NULL);
375
+ argv_array_pushl(&child->args, "--name", sub->name, NULL);
376
+ argv_array_pushl(&child->args, "--url", url, NULL);
377
+ if (suc->reference)
378
+ argv_array_push(&child->args, suc->reference);
379
+ if (suc->depth)
380
+ argv_array_push(&child->args, suc->depth);
381
+
382
+cleanup:
383
+ free(url);
384
+ strbuf_reset(&displaypath_sb);
385
+ strbuf_reset(&sb);
386
+
387
+ return needs_cloning;
388
+}
389
+
390
+static int update_clone_get_next_task(struct child_process *child,
391
+ struct strbuf *err,
392
+ void *suc_cb,
393
+ void **void_task_cb)
394
+{
395
+ struct submodule_update_clone *suc = suc_cb;
396
+
397
+ for (; suc->current < suc->list.nr; suc->current++) {
398
+ const struct cache_entry *ce = suc->list.entries[suc->current];
399
+ if (prepare_to_clone_next_submodule(ce, child, suc, err)) {
400
+ suc->current++;
401
+ return 1;
402
+ }
403
+ }
404
+ return 0;
405
+}
406
+
407
+static int update_clone_start_failure(struct strbuf *err,
408
+ void *suc_cb,
409
+ void *void_task_cb)
410
+{
411
+ struct submodule_update_clone *suc = suc_cb;
412
+ suc->quickstop = 1;
413
+ return 1;
414
+}
415
+
416
+static int update_clone_task_finished(int result,
417
+ struct strbuf *err,
418
+ void *suc_cb,
419
+ void *void_task_cb)
420
+{
421
+ struct submodule_update_clone *suc = suc_cb;
422
+
423
+ if (!result)
424
+ return 0;
425
+
426
+ suc->quickstop = 1;
427
+ return 1;
428
+}
429
+
430
+static int update_clone(int argc, const char **argv, const char *prefix)
431
+{
432
+ const char *update = NULL;
433
+ struct string_list_item *item;
434
+ struct pathspec pathspec;
435
+ struct submodule_update_clone suc = SUBMODULE_UPDATE_CLONE_INIT;
436
+
437
+ struct option module_update_clone_options[] = {
438
+ OPT_STRING(0, "prefix", &prefix,
439
+ N_("path"),
440
+ N_("path into the working tree")),
441
+ OPT_STRING(0, "recursive-prefix", &suc.recursive_prefix,
442
+ N_("path"),
443
+ N_("path into the working tree, across nested "
444
+ "submodule boundaries")),
445
+ OPT_STRING(0, "update", &update,
446
+ N_("string"),
447
+ N_("rebase, merge, checkout or none")),
448
+ OPT_STRING(0, "reference", &suc.reference, N_("repo"),
449
+ N_("reference repository")),
450
+ OPT_STRING(0, "depth", &suc.depth, "<depth>",
451
+ N_("Create a shallow clone truncated to the "
452
+ "specified number of revisions")),
453
+ OPT__QUIET(&suc.quiet, N_("don't print cloning progress")),
454
+ OPT_END()
455
+ };
456
+
457
+ const char *const git_submodule_helper_usage[] = {
458
+ N_("git submodule--helper update_clone [--prefix=<path>] [<path>...]"),
459
+ NULL
460
+ };
461
+ suc.prefix = prefix;
462
+
463
+ argc = parse_options(argc, argv, prefix, module_update_clone_options,
464
+ git_submodule_helper_usage, 0);
465
+
466
+ if (update)
467
+ if (parse_submodule_update_strategy(update, &suc.update) < 0)
468
+ die(_("bad value for update parameter"));
469
+
470
+ if (module_list_compute(argc, argv, prefix, &pathspec, &suc.list) < 0)
471
+ return 1;
472
+
473
+ if (pathspec.nr)
474
+ suc.warn_if_uninitialized = 1;
475
+
476
+ /* Overlay the parsed .gitmodules file with .git/config */
477
+ gitmodules_config();
478
+ git_config(submodule_config, NULL);
479
+
480
+ run_processes_parallel(1,
481
+ update_clone_get_next_task,
482
+ update_clone_start_failure,
483
+ update_clone_task_finished,
484
+ &suc);
485
+
486
+ /*
487
+ * We saved the output and put it out all at once now.
488
+ * That means:
489
+ * - the listener does not have to interleave their (checkout)
490
+ * work with our fetching. The writes involved in a
491
+ * checkout involve more straightforward sequential I/O.
492
+ * - the listener can avoid doing any work if fetching failed.
493
+ */
494
+ if (suc.quickstop)
495
+ return 1;
496
+
497
+ for_each_string_list_item(item, &suc.projectlines)
498
+ utf8_fprintf(stdout, "%s", item->string);
499
+
500
+ return 0;
501
+}
502
+
503
struct cmd_struct {
504
const char *cmd;
505
int (*fn)(int, const char **, const char *);
@@ -264,6 +509,7 @@ static struct cmd_struct commands[] = {
509
{"list", module_list},
510
{"name", module_name},
511
{"clone", module_clone},
512
+ {"update-clone", update_clone}
513
};
514
515
int cmd_submodule__helper(int argc, const char **argv, const char *prefix)
git-submodule.sh
+13
-34
@@ -664,17 +664,20 @@ cmd_update()
664
cmd_init "--" "$@" || return
665
fi
666
667
- cloned_modules=
668
- git submodule--helper list --prefix "$wt_prefix" "$@" | {
667
+ {
668
+ git submodule--helper update-clone ${GIT_QUIET:+--quiet} \
669
+ ${wt_prefix:+--prefix "$wt_prefix"} \
670
+ ${prefix:+--recursive-prefix "$prefix"} \
671
+ ${update:+--update "$update"} \
672
+ ${reference:+--reference "$reference"} \
673
+ ${depth:+--depth "$depth"} \
674
+ "$@" || echo "#unmatched"
675
+ } | {
676
err=
670
- while read mode sha1 stage sm_path
677
+ while read mode sha1 stage just_cloned sm_path
678
do
679
die_if_unmatched "$mode"
673
- if test "$stage" = U
674
- then
675
- echo >&2 "Skipping unmerged submodule $prefix$sm_path"
676
- continue
677
- fi
680
+
681
name=$(git submodule--helper name "$sm_path") || exit
682
url=$(git config submodule."$name".url)
683
branch=$(get_submodule_config "$name" branch master)
@@ -691,27 +694,10 @@ cmd_update()
694
695
displaypath=$(relative_path "$prefix$sm_path")
696
694
- if test "$update_module" = "none"
695
- then
696
- echo >&2 "Skipping submodule '$displaypath'"
697
- continue
698
- fi
699
-
700
- if test -z "$url"
701
- then
702
- # Only mention uninitialized submodules when its
703
- # path have been specified
704
- test "$#" != "0" &&
705
- say >&2 "$(eval_gettext "Submodule path '\$displaypath' not initialized
706
-Maybe you want to use 'update --init'?")"
707
- continue
708
- fi
709
-
710
- if ! test -d "$sm_path"/.git && ! test -f "$sm_path"/.git
697
+ if test $just_cloned -eq 1
698
then
712
- git submodule--helper clone ${GIT_QUIET:+--quiet} --prefix "$prefix" --path "$sm_path" --name "$name" --url "$url" "$reference" "$depth" || exit
713
- cloned_modules="$cloned_modules;$name"
699
subsha1=
700
+ update_module=checkout
701
else
702
subsha1=$(clear_local_git_env; cd "$sm_path" &&
703
git rev-parse --verify HEAD) ||
@@ -751,13 +737,6 @@ Maybe you want to use 'update --init'?")"
737
die "$(eval_gettext "Unable to fetch in submodule path '\$displaypath'")"
738
fi
739
754
- # Is this something we just cloned?
755
- case ";$cloned_modules;" in
756
- *";$name;"*)
757
- # then there is no local change to integrate
758
- update_module=checkout ;;
759
- esac
760
-
740
must_die_on_failure=
741
case "$update_module" in
742
checkout)