submodule--helper: initial clone learns retry logic
Each submodule that is attempted to be cloned, will be retried once in case of failure after all other submodules were cloned. This helps to mitigate ephemeral server failures and increases chances of a reliable clone of a repo with hundreds of submodules immensely. Signed-off-by: Stefan Beller <sbeller@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Stefan Beller committed
Jun 9, 2016 at 17:35 UTC
665b35eccd39fefd714cb5c332277a6b94fd9386
1 file changed
+59
-7
builtin/submodule--helper.c
+59
-7
@@ -592,10 +592,14 @@ struct submodule_update_clone {
592
593
/* If we want to stop as fast as possible and return an error */
594
unsigned quickstop : 1;
595
+
596
+ /* failed clones to be retried again */
597
+ const struct cache_entry **failed_clones;
598
+ int failed_clones_nr, failed_clones_alloc;
599
};
600
#define SUBMODULE_UPDATE_CLONE_INIT {0, MODULE_LIST_INIT, 0, \
601
SUBMODULE_UPDATE_STRATEGY_INIT, 0, -1, NULL, NULL, NULL, NULL, \
598
- STRING_LIST_INIT_DUP, 0}
602
+ STRING_LIST_INIT_DUP, 0, NULL, 0, 0}
603
604
605
static void next_submodule_warn_missing(struct submodule_update_clone *suc,
@@ -720,23 +724,47 @@ cleanup:
724
static int update_clone_get_next_task(struct child_process *child,
725
struct strbuf *err,
726
void *suc_cb,
723
- void **void_task_cb)
727
+ void **idx_task_cb)
728
{
729
struct submodule_update_clone *suc = suc_cb;
730
+ const struct cache_entry *ce;
731
+ int index;
732
733
for (; suc->current < suc->list.nr; suc->current++) {
728
- const struct cache_entry *ce = suc->list.entries[suc->current];
734
+ ce = suc->list.entries[suc->current];
735
if (prepare_to_clone_next_submodule(ce, child, suc, err)) {
736
+ int *p = xmalloc(sizeof(*p));
737
+ *p = suc->current;
738
+ *idx_task_cb = p;
739
suc->current++;
740
return 1;
741
}
742
}
743
+
744
+ /*
745
+ * The loop above tried cloning each submodule once, now try the
746
+ * stragglers again, which we can imagine as an extension of the
747
+ * entry list.
748
+ */
749
+ index = suc->current - suc->list.nr;
750
+ if (index < suc->failed_clones_nr) {
751
+ int *p;
752
+ ce = suc->failed_clones[index];
753
+ if (!prepare_to_clone_next_submodule(ce, child, suc, err))
754
+ die("BUG: ce was a submodule before?");
755
+ p = xmalloc(sizeof(*p));
756
+ *p = suc->current;
757
+ *idx_task_cb = p;
758
+ suc->current ++;
759
+ return 1;
760
+ }
761
+
762
return 0;
763
}
764
765
static int update_clone_start_failure(struct strbuf *err,
766
void *suc_cb,
739
- void *void_task_cb)
767
+ void *idx_task_cb)
768
{
769
struct submodule_update_clone *suc = suc_cb;
770
suc->quickstop = 1;
@@ -746,15 +774,39 @@ static int update_clone_start_failure(struct strbuf *err,
774
static int update_clone_task_finished(int result,
775
struct strbuf *err,
776
void *suc_cb,
749
- void *void_task_cb)
777
+ void *idx_task_cb)
778
{
779
+ const struct cache_entry *ce;
780
struct submodule_update_clone *suc = suc_cb;
781
782
+ int *idxP = *(int**)idx_task_cb;
783
+ int idx = *idxP;
784
+ free(idxP);
785
+
786
if (!result)
787
return 0;
788
756
- suc->quickstop = 1;
757
- return 1;
789
+ if (idx < suc->list.nr) {
790
+ ce = suc->list.entries[idx];
791
+ strbuf_addf(err, _("Failed to clone '%s'. Retry scheduled"),
792
+ ce->name);
793
+ strbuf_addch(err, '\n');
794
+ ALLOC_GROW(suc->failed_clones,
795
+ suc->failed_clones_nr + 1,
796
+ suc->failed_clones_alloc);
797
+ suc->failed_clones[suc->failed_clones_nr++] = ce;
798
+ return 0;
799
+ } else {
800
+ idx = suc->current - suc->list.nr;
801
+ ce = suc->failed_clones[idx];
802
+ strbuf_addf(err, _("Failed to clone '%s' a second time, aborting"),
803
+ ce->name);
804
+ strbuf_addch(err, '\n');
805
+ suc->quickstop = 1;
806
+ return 1;
807
+ }
808
+
809
+ return 0;
810
}
811
812
static int update_clone(int argc, const char **argv, const char *prefix)