fetch_populated_submodules: use new parallel job processing
In a later patch we enable parallel processing of submodules, this only adds the possibility for it. So this change should not change any user facing behavior. Signed-off-by: Stefan Beller <sbeller@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Stefan Beller committed
Dec 15, 2015 at 16:04 UTC
fe85ee6e23e642985004d37e63b4c0d0e5f30587
1 file changed
+98
-44
submodule.c
+98
-44
@@ -12,6 +12,7 @@
12
#include "sha1-array.h"
13
#include "argv-array.h"
14
#include "blob.h"
15
+#include "thread-utils.h"
16
17
static int config_fetch_recurse_submodules = RECURSE_SUBMODULES_ON_DEMAND;
18
static struct string_list changed_submodule_paths;
@@ -610,37 +611,28 @@ static void calculate_changed_submodule_paths(void)
611
initialized_fetch_ref_tips = 0;
612
}
613
613
-int fetch_populated_submodules(const struct argv_array *options,
614
- const char *prefix, int command_line_option,
615
- int quiet)
614
+struct submodule_parallel_fetch {
615
+ int count;
616
+ struct argv_array args;
617
+ const char *work_tree;
618
+ const char *prefix;
619
+ int command_line_option;
620
+ int quiet;
621
+ int result;
622
+};
623
+#define SPF_INIT {0, ARGV_ARRAY_INIT, NULL, NULL, 0, 0, 0}
624
+
625
+static int get_next_submodule(struct child_process *cp,
626
+ struct strbuf *err, void *data, void **task_cb)
627
{
617
- int i, result = 0;
618
- struct child_process cp = CHILD_PROCESS_INIT;
619
- struct argv_array argv = ARGV_ARRAY_INIT;
620
- const char *work_tree = get_git_work_tree();
621
- if (!work_tree)
622
- goto out;
623
-
624
- if (read_cache() < 0)
625
- die("index file corrupt");
626
-
627
- argv_array_push(&argv, "fetch");
628
- for (i = 0; i < options->argc; i++)
629
- argv_array_push(&argv, options->argv[i]);
630
- argv_array_push(&argv, "--recurse-submodules-default");
631
- /* default value, "--submodule-prefix" and its value are added later */
632
-
633
- cp.env = local_repo_env;
634
- cp.git_cmd = 1;
635
- cp.no_stdin = 1;
636
-
637
- calculate_changed_submodule_paths();
628
+ int ret = 0;
629
+ struct submodule_parallel_fetch *spf = data;
630
639
- for (i = 0; i < active_nr; i++) {
631
+ for (; spf->count < active_nr; spf->count++) {
632
struct strbuf submodule_path = STRBUF_INIT;
633
struct strbuf submodule_git_dir = STRBUF_INIT;
634
struct strbuf submodule_prefix = STRBUF_INIT;
643
- const struct cache_entry *ce = active_cache[i];
635
+ const struct cache_entry *ce = active_cache[spf->count];
636
const char *git_dir, *default_argv;
637
const struct submodule *submodule;
638
@@ -652,7 +644,7 @@ int fetch_populated_submodules(const struct argv_array *options,
644
submodule = submodule_from_name(null_sha1, ce->name);
645
646
default_argv = "yes";
655
- if (command_line_option == RECURSE_SUBMODULES_DEFAULT) {
647
+ if (spf->command_line_option == RECURSE_SUBMODULES_DEFAULT) {
648
if (submodule &&
649
submodule->fetch_recurse !=
650
RECURSE_SUBMODULES_NONE) {
@@ -675,40 +667,102 @@ int fetch_populated_submodules(const struct argv_array *options,
667
default_argv = "on-demand";
668
}
669
}
678
- } else if (command_line_option == RECURSE_SUBMODULES_ON_DEMAND) {
670
+ } else if (spf->command_line_option == RECURSE_SUBMODULES_ON_DEMAND) {
671
if (!unsorted_string_list_lookup(&changed_submodule_paths, ce->name))
672
continue;
673
default_argv = "on-demand";
674
}
675
684
- strbuf_addf(&submodule_path, "%s/%s", work_tree, ce->name);
676
+ strbuf_addf(&submodule_path, "%s/%s", spf->work_tree, ce->name);
677
strbuf_addf(&submodule_git_dir, "%s/.git", submodule_path.buf);
686
- strbuf_addf(&submodule_prefix, "%s%s/", prefix, ce->name);
678
+ strbuf_addf(&submodule_prefix, "%s%s/", spf->prefix, ce->name);
679
git_dir = read_gitfile(submodule_git_dir.buf);
680
if (!git_dir)
681
git_dir = submodule_git_dir.buf;
682
if (is_directory(git_dir)) {
691
- if (!quiet)
692
- fprintf(stderr, "Fetching submodule %s%s\n", prefix, ce->name);
693
- cp.dir = submodule_path.buf;
694
- argv_array_push(&argv, default_argv);
695
- argv_array_push(&argv, "--submodule-prefix");
696
- argv_array_push(&argv, submodule_prefix.buf);
697
- cp.argv = argv.argv;
698
- if (run_command(&cp))
699
- result = 1;
700
- argv_array_pop(&argv);
701
- argv_array_pop(&argv);
702
- argv_array_pop(&argv);
683
+ child_process_init(cp);
684
+ cp->dir = strbuf_detach(&submodule_path, NULL);
685
+ cp->env = local_repo_env;
686
+ cp->git_cmd = 1;
687
+ if (!spf->quiet)
688
+ strbuf_addf(err, "Fetching submodule %s%s\n",
689
+ spf->prefix, ce->name);
690
+ argv_array_init(&cp->args);
691
+ argv_array_pushv(&cp->args, spf->args.argv);
692
+ argv_array_push(&cp->args, default_argv);
693
+ argv_array_push(&cp->args, "--submodule-prefix");
694
+ argv_array_push(&cp->args, submodule_prefix.buf);
695
+ ret = 1;
696
}
697
strbuf_release(&submodule_path);
698
strbuf_release(&submodule_git_dir);
699
strbuf_release(&submodule_prefix);
700
+ if (ret) {
701
+ spf->count++;
702
+ return 1;
703
+ }
704
}
708
- argv_array_clear(&argv);
705
+ return 0;
706
+}
707
+
708
+static int fetch_start_failure(struct child_process *cp,
709
+ struct strbuf *err,
710
+ void *cb, void *task_cb)
711
+{
712
+ struct submodule_parallel_fetch *spf = cb;
713
+
714
+ spf->result = 1;
715
+
716
+ return 0;
717
+}
718
+
719
+static int fetch_finish(int retvalue, struct child_process *cp,
720
+ struct strbuf *err, void *cb, void *task_cb)
721
+{
722
+ struct submodule_parallel_fetch *spf = cb;
723
+
724
+ if (retvalue)
725
+ spf->result = 1;
726
+
727
+ return 0;
728
+}
729
+
730
+int fetch_populated_submodules(const struct argv_array *options,
731
+ const char *prefix, int command_line_option,
732
+ int quiet)
733
+{
734
+ int i;
735
+ int max_parallel_jobs = 1;
736
+ struct submodule_parallel_fetch spf = SPF_INIT;
737
+
738
+ spf.work_tree = get_git_work_tree();
739
+ spf.command_line_option = command_line_option;
740
+ spf.quiet = quiet;
741
+ spf.prefix = prefix;
742
+
743
+ if (!spf.work_tree)
744
+ goto out;
745
+
746
+ if (read_cache() < 0)
747
+ die("index file corrupt");
748
+
749
+ argv_array_push(&spf.args, "fetch");
750
+ for (i = 0; i < options->argc; i++)
751
+ argv_array_push(&spf.args, options->argv[i]);
752
+ argv_array_push(&spf.args, "--recurse-submodules-default");
753
+ /* default value, "--submodule-prefix" and its value are added later */
754
+
755
+ calculate_changed_submodule_paths();
756
+ run_processes_parallel(max_parallel_jobs,
757
+ get_next_submodule,
758
+ fetch_start_failure,
759
+ fetch_finish,
760
+ &spf);
761
+
762
+ argv_array_clear(&spf.args);
763
out:
764
string_list_clear(&changed_submodule_paths, 1);
711
- return result;
765
+ return spf.result;
766
}
767
768
unsigned is_submodule_modified(const char *path, int ignore_untracked)