fetch/pull: recurse into submodules when necessary
To be able to access all commits of populated submodules referenced by the superproject it is sufficient to only then let "git fetch" recurse into a submodule when the new commits fetched in the superproject record new commits for it. Having these commits present is extremely useful when using the "--submodule" option to "git diff" (which is what "git gui" and "gitk" do since 1.6.6), as all submodule commits needed for creating a descriptive output can be accessed. Also merging submodule commits (added in 1.7.3) depends on the submodule commits in question being present to work. Last but not least this enables disconnected operation when using submodules, as all commits necessary for a successful "git submodule update -N" will have been fetched automatically. So we choose this mode as the default for fetch and pull. Before a new or changed ref from upstream is updated in update_local_ref() "git rev-list <new-sha1> --not --branches --remotes" is used to determine all newly fetched commits. These are then walked and diffed against their parent(s) to see if a submodule has been changed. If that is the case, its path is stored to be fetched after the superproject fetch is completed. Using the "--recurse-submodules" or the "--no-recurse-submodules" option disables the examination of the fetched refs because the result will be ignored anyway. There is currently no infrastructure for storing deleted and new submodules in the .git directory of the superproject. That's why fetch and pull for now only fetch submodules that are already checked out and are not renamed. In t7403 the "--no-recurse-submodules" argument had to be added to "git pull" to avoid failure because of the moved upstream submodule repo. Thanks-to: Jonathan Nieder <jrnieder@gmail.com> Thanks-to: Heiko Voigt <hvoigt@hvoigt.net> Signed-off-by: Jens Lehmann <Jens.Lehmann@web.de> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Jens Lehmann committed
Mar 6, 2011 at 23:10 UTC
88a21979c5717e3f37b9691e90b6dbf2b94c751a
6 files changed
+243
-17
Documentation/fetch-options.txt
+8
@@ -73,6 +73,14 @@ ifndef::git-pull[]
73
Prepend <path> to paths printed in informative messages
74
such as "Fetching submodule foo". This option is used
75
internally when recursing over submodules.
76
+
77
+--recurse-submodules-default=[yes|on-demand]::
78
+ This option is used internally to temporarily provide a
79
+ non-negative default value for the --recurse-submodules
80
+ option. All other methods of configuring fetch's submodule
81
+ recursion (such as settings in linkgit:gitmodules[5] and
82
+ linkgit:git-config[1]) override this option, as does
83
+ specifying --[no-]recurse-submodules directly.
84
endif::git-pull[]
85
86
-u::
builtin/fetch.c
+17
-9
@@ -28,12 +28,6 @@ enum {
28
TAGS_SET = 2
29
};
30
31
-enum {
32
- RECURSE_SUBMODULES_OFF = 0,
33
- RECURSE_SUBMODULES_DEFAULT = 1,
34
- RECURSE_SUBMODULES_ON = 2
35
-};
36
-
31
static int all, append, dry_run, force, keep, multiple, prune, update_head_ok, verbosity;
32
static int progress, recurse_submodules = RECURSE_SUBMODULES_DEFAULT;
33
static int tags = TAGS_DEFAULT;
@@ -42,6 +36,7 @@ static const char *upload_pack;
36
static struct strbuf default_rla = STRBUF_INIT;
37
static struct transport *transport;
38
static const char *submodule_prefix = "";
39
+static const char *recurse_submodules_default;
40
41
static struct option builtin_fetch_options[] = {
42
OPT__VERBOSITY(&verbosity),
@@ -73,6 +68,9 @@ static struct option builtin_fetch_options[] = {
68
"deepen history of shallow clone"),
69
{ OPTION_STRING, 0, "submodule-prefix", &submodule_prefix, "DIR",
70
"prepend this to submodule path output", PARSE_OPT_HIDDEN },
71
+ { OPTION_STRING, 0, "recurse-submodules-default",
72
+ &recurse_submodules_default, NULL,
73
+ "default mode for recursion", PARSE_OPT_HIDDEN },
74
OPT_END()
75
};
76
@@ -284,6 +282,9 @@ static int update_local_ref(struct ref *ref,
282
else {
283
msg = "storing head";
284
what = "[new branch]";
285
+ if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
286
+ (recurse_submodules != RECURSE_SUBMODULES_ON))
287
+ check_for_new_submodule_commits(ref->new_sha1);
288
}
289
290
r = s_update_ref(msg, ref, 0);
@@ -299,6 +300,9 @@ static int update_local_ref(struct ref *ref,
300
strcpy(quickref, find_unique_abbrev(current->object.sha1, DEFAULT_ABBREV));
301
strcat(quickref, "..");
302
strcat(quickref, find_unique_abbrev(ref->new_sha1, DEFAULT_ABBREV));
303
+ if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
304
+ (recurse_submodules != RECURSE_SUBMODULES_ON))
305
+ check_for_new_submodule_commits(ref->new_sha1);
306
r = s_update_ref("fast-forward", ref, 1);
307
sprintf(display, "%c %-*s %-*s -> %s%s", r ? '!' : ' ',
308
TRANSPORT_SUMMARY_WIDTH, quickref, REFCOL_WIDTH, remote,
@@ -310,6 +314,9 @@ static int update_local_ref(struct ref *ref,
314
strcpy(quickref, find_unique_abbrev(current->object.sha1, DEFAULT_ABBREV));
315
strcat(quickref, "...");
316
strcat(quickref, find_unique_abbrev(ref->new_sha1, DEFAULT_ABBREV));
317
+ if ((recurse_submodules != RECURSE_SUBMODULES_OFF) &&
318
+ (recurse_submodules != RECURSE_SUBMODULES_ON))
319
+ check_for_new_submodule_commits(ref->new_sha1);
320
r = s_update_ref("forced-update", ref, 1);
321
sprintf(display, "%c %-*s %-*s -> %s (%s)", r ? '!' : '+',
322
TRANSPORT_SUMMARY_WIDTH, quickref, REFCOL_WIDTH, remote,
@@ -949,9 +956,10 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
956
if (!result && (recurse_submodules != RECURSE_SUBMODULES_OFF)) {
957
const char *options[10];
958
int num_options = 0;
952
- /* Set recursion as default when we already are recursing */
953
- if (submodule_prefix[0])
954
- set_config_fetch_recurse_submodules(1);
959
+ if (recurse_submodules_default) {
960
+ int arg = parse_fetch_recurse_submodules_arg("--recurse-submodules-default", recurse_submodules_default);
961
+ set_config_fetch_recurse_submodules(arg);
962
+ }
963
gitmodules_config();
964
git_config(submodule_config, NULL);
965
add_options_to_argv(&num_options, options);
submodule.c
+99
-7
@@ -12,7 +12,8 @@
12
struct string_list config_name_for_path;
13
struct string_list config_fetch_recurse_submodules_for_name;
14
struct string_list config_ignore_for_name;
15
-static int config_fetch_recurse_submodules;
15
+static int config_fetch_recurse_submodules = RECURSE_SUBMODULES_ON_DEMAND;
16
+struct string_list changed_submodule_paths;
17
18
static int add_submodule_odb(const char *path)
19
{
@@ -152,6 +153,20 @@ void handle_ignore_submodules_arg(struct diff_options *diffopt,
153
die("bad --ignore-submodules argument: %s", arg);
154
}
155
156
+int parse_fetch_recurse_submodules_arg(const char *opt, const char *arg)
157
+{
158
+ switch (git_config_maybe_bool(opt, arg)) {
159
+ case 1:
160
+ return RECURSE_SUBMODULES_ON;
161
+ case 0:
162
+ return RECURSE_SUBMODULES_OFF;
163
+ default:
164
+ if (!strcmp(arg, "on-demand"))
165
+ return RECURSE_SUBMODULES_ON_DEMAND;
166
+ die("bad %s argument: %s", opt, arg);
167
+ }
168
+}
169
+
170
void show_submodule_summary(FILE *f, const char *path,
171
unsigned char one[20], unsigned char two[20],
172
unsigned dirty_submodule,
@@ -248,27 +263,95 @@ void set_config_fetch_recurse_submodules(int value)
263
config_fetch_recurse_submodules = value;
264
}
265
266
+static void submodule_collect_changed_cb(struct diff_queue_struct *q,
267
+ struct diff_options *options,
268
+ void *data)
269
+{
270
+ int i;
271
+ for (i = 0; i < q->nr; i++) {
272
+ struct diff_filepair *p = q->queue[i];
273
+ if (!S_ISGITLINK(p->two->mode))
274
+ continue;
275
+
276
+ if (S_ISGITLINK(p->one->mode)) {
277
+ /* NEEDSWORK: We should honor the name configured in
278
+ * the .gitmodules file of the commit we are examining
279
+ * here to be able to correctly follow submodules
280
+ * being moved around. */
281
+ struct string_list_item *path;
282
+ path = unsorted_string_list_lookup(&changed_submodule_paths, p->two->path);
283
+ if (!path)
284
+ string_list_append(&changed_submodule_paths, xstrdup(p->two->path));
285
+ } else {
286
+ /* Submodule is new or was moved here */
287
+ /* NEEDSWORK: When the .git directories of submodules
288
+ * live inside the superprojects .git directory some
289
+ * day we should fetch new submodules directly into
290
+ * that location too when config or options request
291
+ * that so they can be checked out from there. */
292
+ continue;
293
+ }
294
+ }
295
+}
296
+
297
+void check_for_new_submodule_commits(unsigned char new_sha1[20])
298
+{
299
+ struct rev_info rev;
300
+ struct commit *commit;
301
+ const char *argv[] = {NULL, NULL, "--not", "--all", NULL};
302
+ int argc = ARRAY_SIZE(argv) - 1;
303
+
304
+ init_revisions(&rev, NULL);
305
+ argv[1] = xstrdup(sha1_to_hex(new_sha1));
306
+ setup_revisions(argc, argv, &rev, NULL);
307
+ if (prepare_revision_walk(&rev))
308
+ die("revision walk setup failed");
309
+
310
+ /*
311
+ * Collect all submodules (whether checked out or not) for which new
312
+ * commits have been recorded upstream in "changed_submodule_paths".
313
+ */
314
+ while ((commit = get_revision(&rev))) {
315
+ struct commit_list *parent = commit->parents;
316
+ while (parent) {
317
+ struct diff_options diff_opts;
318
+ diff_setup(&diff_opts);
319
+ diff_opts.output_format |= DIFF_FORMAT_CALLBACK;
320
+ diff_opts.format_callback = submodule_collect_changed_cb;
321
+ if (diff_setup_done(&diff_opts) < 0)
322
+ die("diff_setup_done failed");
323
+ diff_tree_sha1(parent->item->object.sha1, commit->object.sha1, "", &diff_opts);
324
+ diffcore_std(&diff_opts);
325
+ diff_flush(&diff_opts);
326
+ parent = parent->next;
327
+ }
328
+ }
329
+ free((char *)argv[1]);
330
+}
331
+
332
int fetch_populated_submodules(int num_options, const char **options,
333
const char *prefix, int ignore_config,
334
int quiet)
335
{
255
- int i, result = 0, argc = 0;
336
+ int i, result = 0, argc = 0, default_argc;
337
struct child_process cp;
338
const char **argv;
339
struct string_list_item *name_for_path;
340
const char *work_tree = get_git_work_tree();
341
if (!work_tree)
261
- return 0;
342
+ goto out;
343
344
if (!the_index.initialized)
345
if (read_cache() < 0)
346
die("index file corrupt");
347
267
- /* 4: "fetch" (options) "--submodule-prefix" prefix NULL */
268
- argv = xcalloc(num_options + 4, sizeof(const char *));
348
+ /* 6: "fetch" (options) --recurse-submodules-default default "--submodule-prefix" prefix NULL */
349
+ argv = xcalloc(num_options + 6, sizeof(const char *));
350
argv[argc++] = "fetch";
351
for (i = 0; i < num_options; i++)
352
argv[argc++] = options[i];
353
+ argv[argc++] = "--recurse-submodules-default";
354
+ default_argc = argc++;
355
argv[argc++] = "--submodule-prefix";
356
357
memset(&cp, 0, sizeof(cp));
@@ -282,7 +365,7 @@ int fetch_populated_submodules(int num_options, const char **options,
365
struct strbuf submodule_git_dir = STRBUF_INIT;
366
struct strbuf submodule_prefix = STRBUF_INIT;
367
struct cache_entry *ce = active_cache[i];
285
- const char *git_dir, *name;
368
+ const char *git_dir, *name, *default_argv;
369
370
if (!S_ISGITLINK(ce->ce_mode))
371
continue;
@@ -292,6 +375,7 @@ int fetch_populated_submodules(int num_options, const char **options,
375
if (name_for_path)
376
name = name_for_path->util;
377
378
+ default_argv = "yes";
379
if (!ignore_config) {
380
struct string_list_item *fetch_recurse_submodules_option;
381
fetch_recurse_submodules_option = unsorted_string_list_lookup(&config_fetch_recurse_submodules_for_name, name);
@@ -299,8 +383,13 @@ int fetch_populated_submodules(int num_options, const char **options,
383
if (!fetch_recurse_submodules_option->util)
384
continue;
385
} else {
302
- if (!config_fetch_recurse_submodules)
386
+ if (config_fetch_recurse_submodules == RECURSE_SUBMODULES_OFF)
387
continue;
388
+ if (config_fetch_recurse_submodules == RECURSE_SUBMODULES_ON_DEMAND) {
389
+ if (!unsorted_string_list_lookup(&changed_submodule_paths, ce->name))
390
+ continue;
391
+ default_argv = "on-demand";
392
+ }
393
}
394
}
395
@@ -314,6 +403,7 @@ int fetch_populated_submodules(int num_options, const char **options,
403
if (!quiet)
404
printf("Fetching submodule %s%s\n", prefix, ce->name);
405
cp.dir = submodule_path.buf;
406
+ argv[default_argc] = default_argv;
407
argv[argc] = submodule_prefix.buf;
408
if (run_command(&cp))
409
result = 1;
@@ -323,6 +413,8 @@ int fetch_populated_submodules(int num_options, const char **options,
413
strbuf_release(&submodule_prefix);
414
}
415
free(argv);
416
+out:
417
+ string_list_clear(&changed_submodule_paths, 1);
418
return result;
419
}
420
submodule.h
+9
@@ -3,17 +3,26 @@
3
4
struct diff_options;
5
6
+enum {
7
+ RECURSE_SUBMODULES_ON_DEMAND = -1,
8
+ RECURSE_SUBMODULES_OFF = 0,
9
+ RECURSE_SUBMODULES_DEFAULT = 1,
10
+ RECURSE_SUBMODULES_ON = 2
11
+};
12
+
13
void set_diffopt_flags_from_submodule_config(struct diff_options *diffopt,
14
const char *path);
15
int submodule_config(const char *var, const char *value, void *cb);
16
void gitmodules_config();
17
int parse_submodule_config_option(const char *var, const char *value);
18
void handle_ignore_submodules_arg(struct diff_options *diffopt, const char *);
19
+int parse_fetch_recurse_submodules_arg(const char *opt, const char *arg);
20
void show_submodule_summary(FILE *f, const char *path,
21
unsigned char one[20], unsigned char two[20],
22
unsigned dirty_submodule,
23
const char *del, const char *add, const char *reset);
24
void set_config_fetch_recurse_submodules(int value);
25
+void check_for_new_submodule_commits(unsigned char new_sha1[20]);
26
int fetch_populated_submodules(int num_options, const char **options,
27
const char *prefix, int ignore_config,
28
int quiet);
t/t5526-fetch-submodules.sh
+109
@@ -192,4 +192,113 @@ test_expect_success "--no-recurse-submodules overrides config setting" '
192
! test -s actual.err
193
'
194
195
+test_expect_success "Recursion doesn't happen when no new commits are fetched in the superproject" '
196
+ (
197
+ cd downstream &&
198
+ (
199
+ cd submodule &&
200
+ git config --unset fetch.recurseSubmodules
201
+ ) &&
202
+ git config --unset fetch.recurseSubmodules
203
+ git fetch >../actual.out 2>../actual.err
204
+ ) &&
205
+ ! test -s actual.out &&
206
+ ! test -s actual.err
207
+'
208
+
209
+test_expect_success "Recursion stops when no new submodule commits are fetched" '
210
+ head1=$(git rev-parse --short HEAD) &&
211
+ git add submodule &&
212
+ git commit -m "new submodule" &&
213
+ head2=$(git rev-parse --short HEAD) &&
214
+ echo "Fetching submodule submodule" > expect.out.sub &&
215
+ echo "From $pwd/." > expect.err.sub &&
216
+ echo " $head1..$head2 master -> origin/master" >> expect.err.sub
217
+ head -2 expect.err >> expect.err.sub &&
218
+ (
219
+ cd downstream &&
220
+ git fetch >../actual.out 2>../actual.err
221
+ ) &&
222
+ test_cmp expect.err.sub actual.err &&
223
+ test_cmp expect.out.sub actual.out
224
+'
225
+
226
+test_expect_success "Recursion doesn't happen when new superproject commits don't change any submodules" '
227
+ add_upstream_commit &&
228
+ head1=$(git rev-parse --short HEAD) &&
229
+ echo a > file &&
230
+ git add file &&
231
+ git commit -m "new file" &&
232
+ head2=$(git rev-parse --short HEAD) &&
233
+ echo "From $pwd/." > expect.err.file &&
234
+ echo " $head1..$head2 master -> origin/master" >> expect.err.file &&
235
+ (
236
+ cd downstream &&
237
+ git fetch >../actual.out 2>../actual.err
238
+ ) &&
239
+ ! test -s actual.out &&
240
+ test_cmp expect.err.file actual.err
241
+'
242
+
243
+test_expect_success "Recursion picks up config in submodule" '
244
+ (
245
+ cd downstream &&
246
+ git fetch --recurse-submodules &&
247
+ (
248
+ cd submodule &&
249
+ git config fetch.recurseSubmodules true
250
+ )
251
+ ) &&
252
+ add_upstream_commit &&
253
+ head1=$(git rev-parse --short HEAD) &&
254
+ git add submodule &&
255
+ git commit -m "new submodule" &&
256
+ head2=$(git rev-parse --short HEAD) &&
257
+ echo "From $pwd/." > expect.err.sub &&
258
+ echo " $head1..$head2 master -> origin/master" >> expect.err.sub &&
259
+ cat expect.err >> expect.err.sub &&
260
+ (
261
+ cd downstream &&
262
+ git fetch >../actual.out 2>../actual.err &&
263
+ (
264
+ cd submodule &&
265
+ git config --unset fetch.recurseSubmodules
266
+ )
267
+ ) &&
268
+ test_cmp expect.err.sub actual.err &&
269
+ test_cmp expect.out actual.out
270
+'
271
+
272
+test_expect_success "Recursion picks up all submodules when necessary" '
273
+ add_upstream_commit &&
274
+ (
275
+ cd submodule &&
276
+ (
277
+ cd deepsubmodule &&
278
+ git fetch &&
279
+ git checkout -q FETCH_HEAD
280
+ ) &&
281
+ head1=$(git rev-parse --short HEAD^) &&
282
+ git add deepsubmodule &&
283
+ git commit -m "new deepsubmodule"
284
+ head2=$(git rev-parse --short HEAD) &&
285
+ echo "From $pwd/submodule" > ../expect.err.sub &&
286
+ echo " $head1..$head2 master -> origin/master" >> ../expect.err.sub
287
+ ) &&
288
+ head1=$(git rev-parse --short HEAD) &&
289
+ git add submodule &&
290
+ git commit -m "new submodule" &&
291
+ head2=$(git rev-parse --short HEAD) &&
292
+ echo "From $pwd/." > expect.err.2 &&
293
+ echo " $head1..$head2 master -> origin/master" >> expect.err.2 &&
294
+ cat expect.err.sub >> expect.err.2 &&
295
+ tail -2 expect.err >> expect.err.2 &&
296
+ (
297
+ cd downstream &&
298
+ git fetch >../actual.out 2>../actual.err
299
+ ) &&
300
+ test_cmp expect.err.2 actual.err &&
301
+ test_cmp expect.out actual.out
302
+'
303
+
304
test_done
t/t7403-submodule-sync.sh
+1
-1
@@ -52,7 +52,7 @@ test_expect_success 'change submodule url' '
52
53
test_expect_success '"git submodule sync" should update submodule URLs' '
54
(cd super-clone &&
55
- git pull &&
55
+ git pull --no-recurse-submodules &&
56
git submodule sync
57
) &&
58
test -d "$(git config -f super-clone/submodule/.git/config \