grep: optionally recurse into submodules
Allow grep to recognize submodules and recursively search for patterns in each submodule. This is done by forking off a process to recursively call grep on each submodule. The top level --super-prefix option is used to pass a path to the submodule which can in turn be used to prepend to output or in pathspec matching logic. Recursion only occurs for submodules which have been initialized and checked out by the parent project. If a submodule hasn't been initialized and checked out it is simply skipped. In order to support the existing multi-threading infrastructure in grep, output from each child process is captured in a strbuf so that it can be later printed to the console in an ordered fashion. To limit the number of theads that are created, each child process has half the number of threads as its parents (minimum of 1), otherwise we potentailly have a fork-bomb. Signed-off-by: Brandon Williams <bmwill@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Brandon Williams committed
Dec 16, 2016 at 11:03 UTC
0281e487fd913bd9a32a710f3109ff3002f3e4a9
4 files changed
+386
-20
Documentation/git-grep.txt
+5
@@ -26,6 +26,7 @@ SYNOPSIS
26
[--threads <num>]
27
[-f <file>] [-e] <pattern>
28
[--and|--or|--not|(|)|-e <pattern>...]
29
+ [--recurse-submodules]
30
[ [--[no-]exclude-standard] [--cached | --no-index | --untracked] | <tree>...]
31
[--] [<pathspec>...]
32
@@ -88,6 +89,10 @@ OPTIONS
89
mechanism. Only useful when searching files in the current directory
90
with `--no-index`.
91
92
+--recurse-submodules::
93
+ Recursively search in each submodule that has been initialized and
94
+ checked out in the repository.
95
+
96
-a::
97
--text::
98
Process binary files as if they were text.
builtin/grep.c
+281
-19
@@ -18,12 +18,20 @@
18
#include "quote.h"
19
#include "dir.h"
20
#include "pathspec.h"
21
+#include "submodule.h"
22
23
static char const * const grep_usage[] = {
24
N_("git grep [<options>] [-e] <pattern> [<rev>...] [[--] <path>...]"),
25
NULL
26
};
27
28
+static const char *super_prefix;
29
+static int recurse_submodules;
30
+static struct argv_array submodule_options = ARGV_ARRAY_INIT;
31
+
32
+static int grep_submodule_launch(struct grep_opt *opt,
33
+ const struct grep_source *gs);
34
+
35
#define GREP_NUM_THREADS_DEFAULT 8
36
static int num_threads;
37
@@ -174,7 +182,10 @@ static void *run(void *arg)
182
break;
183
184
opt->output_priv = w;
177
- hit |= grep_source(opt, &w->source);
185
+ if (w->source.type == GREP_SOURCE_SUBMODULE)
186
+ hit |= grep_submodule_launch(opt, &w->source);
187
+ else
188
+ hit |= grep_source(opt, &w->source);
189
grep_source_clear_data(&w->source);
190
work_done(w);
191
}
@@ -300,6 +311,10 @@ static int grep_sha1(struct grep_opt *opt, const unsigned char *sha1,
311
if (opt->relative && opt->prefix_length) {
312
quote_path_relative(filename + tree_name_len, opt->prefix, &pathbuf);
313
strbuf_insert(&pathbuf, 0, filename, tree_name_len);
314
+ } else if (super_prefix) {
315
+ strbuf_add(&pathbuf, filename, tree_name_len);
316
+ strbuf_addstr(&pathbuf, super_prefix);
317
+ strbuf_addstr(&pathbuf, filename + tree_name_len);
318
} else {
319
strbuf_addstr(&pathbuf, filename);
320
}
@@ -328,10 +343,13 @@ static int grep_file(struct grep_opt *opt, const char *filename)
343
{
344
struct strbuf buf = STRBUF_INIT;
345
331
- if (opt->relative && opt->prefix_length)
346
+ if (opt->relative && opt->prefix_length) {
347
quote_path_relative(filename, opt->prefix, &buf);
333
- else
348
+ } else {
349
+ if (super_prefix)
350
+ strbuf_addstr(&buf, super_prefix);
351
strbuf_addstr(&buf, filename);
352
+ }
353
354
#ifndef NO_PTHREADS
355
if (num_threads) {
@@ -378,31 +396,260 @@ static void run_pager(struct grep_opt *opt, const char *prefix)
396
exit(status);
397
}
398
381
-static int grep_cache(struct grep_opt *opt, const struct pathspec *pathspec, int cached)
399
+static void compile_submodule_options(const struct grep_opt *opt,
400
+ const struct pathspec *pathspec,
401
+ int cached, int untracked,
402
+ int opt_exclude, int use_index,
403
+ int pattern_type_arg)
404
+{
405
+ struct grep_pat *pattern;
406
+ int i;
407
+
408
+ if (recurse_submodules)
409
+ argv_array_push(&submodule_options, "--recurse-submodules");
410
+
411
+ if (cached)
412
+ argv_array_push(&submodule_options, "--cached");
413
+ if (!use_index)
414
+ argv_array_push(&submodule_options, "--no-index");
415
+ if (untracked)
416
+ argv_array_push(&submodule_options, "--untracked");
417
+ if (opt_exclude > 0)
418
+ argv_array_push(&submodule_options, "--exclude-standard");
419
+
420
+ if (opt->invert)
421
+ argv_array_push(&submodule_options, "-v");
422
+ if (opt->ignore_case)
423
+ argv_array_push(&submodule_options, "-i");
424
+ if (opt->word_regexp)
425
+ argv_array_push(&submodule_options, "-w");
426
+ switch (opt->binary) {
427
+ case GREP_BINARY_NOMATCH:
428
+ argv_array_push(&submodule_options, "-I");
429
+ break;
430
+ case GREP_BINARY_TEXT:
431
+ argv_array_push(&submodule_options, "-a");
432
+ break;
433
+ default:
434
+ break;
435
+ }
436
+ if (opt->allow_textconv)
437
+ argv_array_push(&submodule_options, "--textconv");
438
+ if (opt->max_depth != -1)
439
+ argv_array_pushf(&submodule_options, "--max-depth=%d",
440
+ opt->max_depth);
441
+ if (opt->linenum)
442
+ argv_array_push(&submodule_options, "-n");
443
+ if (!opt->pathname)
444
+ argv_array_push(&submodule_options, "-h");
445
+ if (!opt->relative)
446
+ argv_array_push(&submodule_options, "--full-name");
447
+ if (opt->name_only)
448
+ argv_array_push(&submodule_options, "-l");
449
+ if (opt->unmatch_name_only)
450
+ argv_array_push(&submodule_options, "-L");
451
+ if (opt->null_following_name)
452
+ argv_array_push(&submodule_options, "-z");
453
+ if (opt->count)
454
+ argv_array_push(&submodule_options, "-c");
455
+ if (opt->file_break)
456
+ argv_array_push(&submodule_options, "--break");
457
+ if (opt->heading)
458
+ argv_array_push(&submodule_options, "--heading");
459
+ if (opt->pre_context)
460
+ argv_array_pushf(&submodule_options, "--before-context=%d",
461
+ opt->pre_context);
462
+ if (opt->post_context)
463
+ argv_array_pushf(&submodule_options, "--after-context=%d",
464
+ opt->post_context);
465
+ if (opt->funcname)
466
+ argv_array_push(&submodule_options, "-p");
467
+ if (opt->funcbody)
468
+ argv_array_push(&submodule_options, "-W");
469
+ if (opt->all_match)
470
+ argv_array_push(&submodule_options, "--all-match");
471
+ if (opt->debug)
472
+ argv_array_push(&submodule_options, "--debug");
473
+ if (opt->status_only)
474
+ argv_array_push(&submodule_options, "-q");
475
+
476
+ switch (pattern_type_arg) {
477
+ case GREP_PATTERN_TYPE_BRE:
478
+ argv_array_push(&submodule_options, "-G");
479
+ break;
480
+ case GREP_PATTERN_TYPE_ERE:
481
+ argv_array_push(&submodule_options, "-E");
482
+ break;
483
+ case GREP_PATTERN_TYPE_FIXED:
484
+ argv_array_push(&submodule_options, "-F");
485
+ break;
486
+ case GREP_PATTERN_TYPE_PCRE:
487
+ argv_array_push(&submodule_options, "-P");
488
+ break;
489
+ case GREP_PATTERN_TYPE_UNSPECIFIED:
490
+ break;
491
+ }
492
+
493
+ for (pattern = opt->pattern_list; pattern != NULL;
494
+ pattern = pattern->next) {
495
+ switch (pattern->token) {
496
+ case GREP_PATTERN:
497
+ argv_array_pushf(&submodule_options, "-e%s",
498
+ pattern->pattern);
499
+ break;
500
+ case GREP_AND:
501
+ case GREP_OPEN_PAREN:
502
+ case GREP_CLOSE_PAREN:
503
+ case GREP_NOT:
504
+ case GREP_OR:
505
+ argv_array_push(&submodule_options, pattern->pattern);
506
+ break;
507
+ /* BODY and HEAD are not used by git-grep */
508
+ case GREP_PATTERN_BODY:
509
+ case GREP_PATTERN_HEAD:
510
+ break;
511
+ }
512
+ }
513
+
514
+ /*
515
+ * Limit number of threads for child process to use.
516
+ * This is to prevent potential fork-bomb behavior of git-grep as each
517
+ * submodule process has its own thread pool.
518
+ */
519
+ argv_array_pushf(&submodule_options, "--threads=%d",
520
+ (num_threads + 1) / 2);
521
+
522
+ /* Add Pathspecs */
523
+ argv_array_push(&submodule_options, "--");
524
+ for (i = 0; i < pathspec->nr; i++)
525
+ argv_array_push(&submodule_options,
526
+ pathspec->items[i].original);
527
+}
528
+
529
+/*
530
+ * Launch child process to grep contents of a submodule
531
+ */
532
+static int grep_submodule_launch(struct grep_opt *opt,
533
+ const struct grep_source *gs)
534
+{
535
+ struct child_process cp = CHILD_PROCESS_INIT;
536
+ int status, i;
537
+ struct work_item *w = opt->output_priv;
538
+
539
+ prepare_submodule_repo_env(&cp.env_array);
540
+
541
+ /* Add super prefix */
542
+ argv_array_pushf(&cp.args, "--super-prefix=%s%s/",
543
+ super_prefix ? super_prefix : "",
544
+ gs->name);
545
+ argv_array_push(&cp.args, "grep");
546
+
547
+ /* Add options */
548
+ for (i = 0; i < submodule_options.argc; i++)
549
+ argv_array_push(&cp.args, submodule_options.argv[i]);
550
+
551
+ cp.git_cmd = 1;
552
+ cp.dir = gs->path;
553
+
554
+ /*
555
+ * Capture output to output buffer and check the return code from the
556
+ * child process. A '0' indicates a hit, a '1' indicates no hit and
557
+ * anything else is an error.
558
+ */
559
+ status = capture_command(&cp, &w->out, 0);
560
+ if (status && (status != 1)) {
561
+ /* flush the buffer */
562
+ write_or_die(1, w->out.buf, w->out.len);
563
+ die("process for submodule '%s' failed with exit code: %d",
564
+ gs->name, status);
565
+ }
566
+
567
+ /* invert the return code to make a hit equal to 1 */
568
+ return !status;
569
+}
570
+
571
+/*
572
+ * Prep grep structures for a submodule grep
573
+ * sha1: the sha1 of the submodule or NULL if using the working tree
574
+ * filename: name of the submodule including tree name of parent
575
+ * path: location of the submodule
576
+ */
577
+static int grep_submodule(struct grep_opt *opt, const unsigned char *sha1,
578
+ const char *filename, const char *path)
579
+{
580
+ if (!is_submodule_initialized(path))
581
+ return 0;
582
+ if (!is_submodule_populated(path))
583
+ return 0;
584
+
585
+#ifndef NO_PTHREADS
586
+ if (num_threads) {
587
+ add_work(opt, GREP_SOURCE_SUBMODULE, filename, path, sha1);
588
+ return 0;
589
+ } else
590
+#endif
591
+ {
592
+ struct work_item w;
593
+ int hit;
594
+
595
+ grep_source_init(&w.source, GREP_SOURCE_SUBMODULE,
596
+ filename, path, sha1);
597
+ strbuf_init(&w.out, 0);
598
+ opt->output_priv = &w;
599
+ hit = grep_submodule_launch(opt, &w.source);
600
+
601
+ write_or_die(1, w.out.buf, w.out.len);
602
+
603
+ grep_source_clear(&w.source);
604
+ strbuf_release(&w.out);
605
+ return hit;
606
+ }
607
+}
608
+
609
+static int grep_cache(struct grep_opt *opt, const struct pathspec *pathspec,
610
+ int cached)
611
{
612
int hit = 0;
613
int nr;
614
+ struct strbuf name = STRBUF_INIT;
615
+ int name_base_len = 0;
616
+ if (super_prefix) {
617
+ name_base_len = strlen(super_prefix);
618
+ strbuf_addstr(&name, super_prefix);
619
+ }
620
+
621
read_cache();
622
623
for (nr = 0; nr < active_nr; nr++) {
624
const struct cache_entry *ce = active_cache[nr];
389
- if (!S_ISREG(ce->ce_mode))
390
- continue;
391
- if (!ce_path_match(ce, pathspec, NULL))
625
+ strbuf_setlen(&name, name_base_len);
626
+ strbuf_addstr(&name, ce->name);
627
+
628
+ if (S_ISREG(ce->ce_mode) &&
629
+ match_pathspec(pathspec, name.buf, name.len, 0, NULL,
630
+ S_ISDIR(ce->ce_mode) ||
631
+ S_ISGITLINK(ce->ce_mode))) {
632
+ /*
633
+ * If CE_VALID is on, we assume worktree file and its
634
+ * cache entry are identical, even if worktree file has
635
+ * been modified, so use cache version instead
636
+ */
637
+ if (cached || (ce->ce_flags & CE_VALID) ||
638
+ ce_skip_worktree(ce)) {
639
+ if (ce_stage(ce) || ce_intent_to_add(ce))
640
+ continue;
641
+ hit |= grep_sha1(opt, ce->oid.hash, ce->name,
642
+ 0, ce->name);
643
+ } else {
644
+ hit |= grep_file(opt, ce->name);
645
+ }
646
+ } else if (recurse_submodules && S_ISGITLINK(ce->ce_mode) &&
647
+ submodule_path_match(pathspec, name.buf, NULL)) {
648
+ hit |= grep_submodule(opt, NULL, ce->name, ce->name);
649
+ } else {
650
continue;
393
- /*
394
- * If CE_VALID is on, we assume worktree file and its cache entry
395
- * are identical, even if worktree file has been modified, so use
396
- * cache version instead
397
- */
398
- if (cached || (ce->ce_flags & CE_VALID) || ce_skip_worktree(ce)) {
399
- if (ce_stage(ce) || ce_intent_to_add(ce))
400
- continue;
401
- hit |= grep_sha1(opt, ce->oid.hash, ce->name, 0,
402
- ce->name);
651
}
404
- else
405
- hit |= grep_file(opt, ce->name);
652
+
653
if (ce_stage(ce)) {
654
do {
655
nr++;
@@ -413,6 +660,8 @@ static int grep_cache(struct grep_opt *opt, const struct pathspec *pathspec, int
660
if (hit && opt->status_only)
661
break;
662
}
663
+
664
+ strbuf_release(&name);
665
return hit;
666
}
667
@@ -651,6 +900,8 @@ int cmd_grep(int argc, const char **argv, const char *prefix)
900
N_("search in both tracked and untracked files")),
901
OPT_SET_INT(0, "exclude-standard", &opt_exclude,
902
N_("ignore files specified via '.gitignore'"), 1),
903
+ OPT_BOOL(0, "recurse-submodules", &recurse_submodules,
904
+ N_("recursivley search in each submodule")),
905
OPT_GROUP(""),
906
OPT_BOOL('v', "invert-match", &opt.invert,
907
N_("show non-matching lines")),
@@ -755,6 +1006,7 @@ int cmd_grep(int argc, const char **argv, const char *prefix)
1006
init_grep_defaults();
1007
git_config(grep_cmd_config, NULL);
1008
grep_init(&opt, prefix);
1009
+ super_prefix = get_super_prefix();
1010
1011
/*
1012
* If there is no -- then the paths must exist in the working
@@ -872,6 +1124,13 @@ int cmd_grep(int argc, const char **argv, const char *prefix)
1124
pathspec.max_depth = opt.max_depth;
1125
pathspec.recursive = 1;
1126
1127
+ if (recurse_submodules) {
1128
+ gitmodules_config();
1129
+ compile_submodule_options(&opt, &pathspec, cached, untracked,
1130
+ opt_exclude, use_index,
1131
+ pattern_type_arg);
1132
+ }
1133
+
1134
if (show_in_pager && (cached || list.nr))
1135
die(_("--open-files-in-pager only works on the worktree"));
1136
@@ -895,6 +1154,9 @@ int cmd_grep(int argc, const char **argv, const char *prefix)
1154
}
1155
}
1156
1157
+ if (recurse_submodules && (!use_index || untracked || list.nr))
1158
+ die(_("option not supported with --recurse-submodules."));
1159
+
1160
if (!show_in_pager && !opt.status_only)
1161
setup_pager();
1162
git.c
+1
-1
@@ -434,7 +434,7 @@ static struct cmd_struct commands[] = {
434
{ "fsck-objects", cmd_fsck, RUN_SETUP },
435
{ "gc", cmd_gc, RUN_SETUP },
436
{ "get-tar-commit-id", cmd_get_tar_commit_id },
437
- { "grep", cmd_grep, RUN_SETUP_GENTLY },
437
+ { "grep", cmd_grep, RUN_SETUP_GENTLY | SUPPORT_SUPER_PREFIX },
438
{ "hash-object", cmd_hash_object },
439
{ "help", cmd_help },
440
{ "index-pack", cmd_index_pack, RUN_SETUP_GENTLY },
t/t7814-grep-recurse-submodules.sh
new
+99
@@ -0,0 +1,99 @@
1
+#!/bin/sh
2
+
3
+test_description='Test grep recurse-submodules feature
4
+
5
+This test verifies the recurse-submodules feature correctly greps across
6
+submodules.
7
+'
8
+
9
+. ./test-lib.sh
10
+
11
+test_expect_success 'setup directory structure and submodule' '
12
+ echo "foobar" >a &&
13
+ mkdir b &&
14
+ echo "bar" >b/b &&
15
+ git add a b &&
16
+ git commit -m "add a and b" &&
17
+ git init submodule &&
18
+ echo "foobar" >submodule/a &&
19
+ git -C submodule add a &&
20
+ git -C submodule commit -m "add a" &&
21
+ git submodule add ./submodule &&
22
+ git commit -m "added submodule"
23
+'
24
+
25
+test_expect_success 'grep correctly finds patterns in a submodule' '
26
+ cat >expect <<-\EOF &&
27
+ a:foobar
28
+ b/b:bar
29
+ submodule/a:foobar
30
+ EOF
31
+
32
+ git grep -e "bar" --recurse-submodules >actual &&
33
+ test_cmp expect actual
34
+'
35
+
36
+test_expect_success 'grep and basic pathspecs' '
37
+ cat >expect <<-\EOF &&
38
+ submodule/a:foobar
39
+ EOF
40
+
41
+ git grep -e. --recurse-submodules -- submodule >actual &&
42
+ test_cmp expect actual
43
+'
44
+
45
+test_expect_success 'grep and nested submodules' '
46
+ git init submodule/sub &&
47
+ echo "foobar" >submodule/sub/a &&
48
+ git -C submodule/sub add a &&
49
+ git -C submodule/sub commit -m "add a" &&
50
+ git -C submodule submodule add ./sub &&
51
+ git -C submodule add sub &&
52
+ git -C submodule commit -m "added sub" &&
53
+ git add submodule &&
54
+ git commit -m "updated submodule" &&
55
+
56
+ cat >expect <<-\EOF &&
57
+ a:foobar
58
+ b/b:bar
59
+ submodule/a:foobar
60
+ submodule/sub/a:foobar
61
+ EOF
62
+
63
+ git grep -e "bar" --recurse-submodules >actual &&
64
+ test_cmp expect actual
65
+'
66
+
67
+test_expect_success 'grep and multiple patterns' '
68
+ cat >expect <<-\EOF &&
69
+ a:foobar
70
+ submodule/a:foobar
71
+ submodule/sub/a:foobar
72
+ EOF
73
+
74
+ git grep -e "bar" --and -e "foo" --recurse-submodules >actual &&
75
+ test_cmp expect actual
76
+'
77
+
78
+test_expect_success 'grep and multiple patterns' '
79
+ cat >expect <<-\EOF &&
80
+ b/b:bar
81
+ EOF
82
+
83
+ git grep -e "bar" --and --not -e "foo" --recurse-submodules >actual &&
84
+ test_cmp expect actual
85
+'
86
+
87
+test_incompatible_with_recurse_submodules ()
88
+{
89
+ test_expect_success "--recurse-submodules and $1 are incompatible" "
90
+ test_must_fail git grep -e. --recurse-submodules $1 2>actual &&
91
+ test_i18ngrep 'not supported with --recurse-submodules' actual
92
+ "
93
+}
94
+
95
+test_incompatible_with_recurse_submodules --untracked
96
+test_incompatible_with_recurse_submodules --no-index
97
+test_incompatible_with_recurse_submodules HEAD
98
+
99
+test_done