gitweb: Restructure projects list generation
Extract filtering out forks (which is done if 'forks' feature is enabled) into filter_forks_from_projects_list subroutine, and searching projects (via projects search form, or via content tags) into search_projects_list subroutine. Both are now run _before_ displaying projects, and not while printing; this allow to know upfront if there were any found projects. Gitweb now can and do print 'No such projects found' if user searches for phrase which does not correspond to any project (any repository). This also would allow splitting projects list into pages, if we so desire. Filtering out forks and marking repository (project) as having forks is now consolidated into one subroutine (special case of handling forks in git_get_projects_list only for $projects_list being file is now removed). Forks handling is also cleaned up and simplified. $pr->{'forks'} now contains un-filled list of forks; we can now also detect situation where the way for having forks is prepared, but there are no forks yet. Sorting projects got also refactored in a very straight way (just moving code) into sort_projects_list subroutine. The interaction between forks, content tags and searching is now made more explicit: searching whether by tag, or via search form turns off fork filtering (gitweb searches also forks, and will show all results). If 'ctags' feature is disabled, then searching by tag is too. The t9500 test now includes some basic test for 'forks' and 'ctags' features; the t9502 includes test checking if gitweb correctly filters out forks. Generating list of projects by scanning given directory is now also a bit simplified wrt. handling filtering; it is byproduct of extracting filtering forks to separate subroutine. While at it we now detect that there are no projects and respond with "404 No projects found" also for 'project_index' and 'opml' actions. Helped-by: Jonathan Nieder <jrnieder@gmail.com> Signed-off-by: Jakub Narebski <jnareb@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Jakub Narebski committed
Apr 29, 2011 at 19:51 UTC
12b1443c2cf41174311c81667022c6f3149aa542
3 files changed
+296
-77
gitweb/gitweb.perl
+173
-77
@@ -2651,21 +2651,23 @@ sub git_get_project_url_list {
2651
}
2652
2653
sub git_get_projects_list {
2654
- my ($filter) = @_;
2654
+ my $filter = shift || '';
2655
my @list;
2656
2657
- $filter ||= '';
2657
$filter =~ s/\.git$//;
2658
2660
- my $check_forks = gitweb_check_feature('forks');
2661
-
2659
if (-d $projects_list) {
2660
# search in directory
2664
- my $dir = $projects_list . ($filter ? "/$filter" : '');
2661
+ my $dir = $projects_list;
2662
# remove the trailing "/"
2663
$dir =~ s!/+$!!;
2667
- my $pfxlen = length("$dir");
2668
- my $pfxdepth = ($dir =~ tr!/!!);
2664
+ my $pfxlen = length("$projects_list");
2665
+ my $pfxdepth = ($projects_list =~ tr!/!!);
2666
+ # when filtering, search only given subdirectory
2667
+ if ($filter) {
2668
+ $dir .= "/$filter";
2669
+ $dir =~ s!/+$!!;
2670
+ }
2671
2672
File::Find::find({
2673
follow_fast => 1, # follow symbolic links
@@ -2680,14 +2682,14 @@ sub git_get_projects_list {
2682
# only directories can be git repositories
2683
return unless (-d $_);
2684
# don't traverse too deep (Find is super slow on os x)
2685
+ # $project_maxdepth excludes depth of $projectroot
2686
if (($File::Find::name =~ tr!/!!) - $pfxdepth > $project_maxdepth) {
2687
$File::Find::prune = 1;
2688
return;
2689
}
2690
2688
- my $subdir = substr($File::Find::name, $pfxlen + 1);
2691
+ my $path = substr($File::Find::name, $pfxlen + 1);
2692
# we check related file in $projectroot
2690
- my $path = ($filter ? "$filter/" : '') . $subdir;
2693
if (check_export_ok("$projectroot/$path")) {
2694
push @list, { path => $path };
2695
$File::Find::prune = 1;
@@ -2700,7 +2702,6 @@ sub git_get_projects_list {
2702
# 'git%2Fgit.git Linus+Torvalds'
2703
# 'libs%2Fklibc%2Fklibc.git H.+Peter+Anvin'
2704
# 'linux%2Fhotplug%2Fudev.git Greg+Kroah-Hartman'
2703
- my %paths;
2705
open my $fd, '<', $projects_list or return;
2706
PROJECT:
2707
while (my $line = <$fd>) {
@@ -2711,32 +2712,9 @@ sub git_get_projects_list {
2712
if (!defined $path) {
2713
next;
2714
}
2714
- if ($filter ne '') {
2715
- # looking for forks;
2716
- my $pfx = substr($path, 0, length($filter));
2717
- if ($pfx ne $filter) {
2718
- next PROJECT;
2719
- }
2720
- my $sfx = substr($path, length($filter));
2721
- if ($sfx !~ /^\/.*\.git$/) {
2722
- next PROJECT;
2723
- }
2724
- } elsif ($check_forks) {
2725
- PATH:
2726
- foreach my $filter (keys %paths) {
2727
- # looking for forks;
2728
- my $pfx = substr($path, 0, length($filter));
2729
- if ($pfx ne $filter) {
2730
- next PATH;
2731
- }
2732
- my $sfx = substr($path, length($filter));
2733
- if ($sfx !~ /^\/.*\.git$/) {
2734
- next PATH;
2735
- }
2736
- # is a fork, don't include it in
2737
- # the list
2738
- next PROJECT;
2739
- }
2715
+ # if $filter is rpovided, check if $path begins with $filter
2716
+ if ($filter && $path !~ m!^\Q$filter\E/!) {
2717
+ next;
2718
}
2719
if (check_export_ok("$projectroot/$path")) {
2720
my $pr = {
@@ -2744,8 +2722,6 @@ sub git_get_projects_list {
2722
owner => to_utf8($owner),
2723
};
2724
push @list, $pr;
2747
- (my $forks_path = $path) =~ s/\.git$//;
2748
- $paths{$forks_path}++;
2725
}
2726
}
2727
close $fd;
@@ -2753,6 +2729,98 @@ sub git_get_projects_list {
2729
return @list;
2730
}
2731
2732
+# written with help of Tree::Trie module (Perl Artistic License, GPL compatibile)
2733
+# as side effects it sets 'forks' field to list of forks for forked projects
2734
+sub filter_forks_from_projects_list {
2735
+ my $projects = shift;
2736
+
2737
+ my %trie; # prefix tree of directories (path components)
2738
+ # generate trie out of those directories that might contain forks
2739
+ foreach my $pr (@$projects) {
2740
+ my $path = $pr->{'path'};
2741
+ $path =~ s/\.git$//; # forks of 'repo.git' are in 'repo/' directory
2742
+ next if ($path =~ m!/$!); # skip non-bare repositories, e.g. 'repo/.git'
2743
+ next unless ($path); # skip '.git' repository: tests, git-instaweb
2744
+ next unless (-d $path); # containing directory exists
2745
+ $pr->{'forks'} = []; # there can be 0 or more forks of project
2746
+
2747
+ # add to trie
2748
+ my @dirs = split('/', $path);
2749
+ # walk the trie, until either runs out of components or out of trie
2750
+ my $ref = \%trie;
2751
+ while (scalar @dirs &&
2752
+ exists($ref->{$dirs[0]})) {
2753
+ $ref = $ref->{shift @dirs};
2754
+ }
2755
+ # create rest of trie structure from rest of components
2756
+ foreach my $dir (@dirs) {
2757
+ $ref = $ref->{$dir} = {};
2758
+ }
2759
+ # create end marker, store $pr as a data
2760
+ $ref->{''} = $pr if (!exists $ref->{''});
2761
+ }
2762
+
2763
+ # filter out forks, by finding shortest prefix match for paths
2764
+ my @filtered;
2765
+ PROJECT:
2766
+ foreach my $pr (@$projects) {
2767
+ # trie lookup
2768
+ my $ref = \%trie;
2769
+ DIR:
2770
+ foreach my $dir (split('/', $pr->{'path'})) {
2771
+ if (exists $ref->{''}) {
2772
+ # found [shortest] prefix, is a fork - skip it
2773
+ push @{$ref->{''}{'forks'}}, $pr;
2774
+ next PROJECT;
2775
+ }
2776
+ if (!exists $ref->{$dir}) {
2777
+ # not in trie, cannot have prefix, not a fork
2778
+ push @filtered, $pr;
2779
+ next PROJECT;
2780
+ }
2781
+ # If the dir is there, we just walk one step down the trie.
2782
+ $ref = $ref->{$dir};
2783
+ }
2784
+ # we ran out of trie
2785
+ # (shouldn't happen: it's either no match, or end marker)
2786
+ push @filtered, $pr;
2787
+ }
2788
+
2789
+ return @filtered;
2790
+}
2791
+
2792
+# note: fill_project_list_info must be run first,
2793
+# for 'descr_long' and 'ctags' to be filled
2794
+sub search_projects_list {
2795
+ my ($projlist, %opts) = @_;
2796
+ my $tagfilter = $opts{'tagfilter'};
2797
+ my $searchtext = $opts{'searchtext'};
2798
+
2799
+ return @$projlist
2800
+ unless ($tagfilter || $searchtext);
2801
+
2802
+ my @projects;
2803
+ PROJECT:
2804
+ foreach my $pr (@$projlist) {
2805
+
2806
+ if ($tagfilter) {
2807
+ next unless ref($pr->{'ctags'}) eq 'HASH';
2808
+ next unless
2809
+ grep { lc($_) eq lc($tagfilter) } keys %{$pr->{'ctags'}};
2810
+ }
2811
+
2812
+ if ($searchtext) {
2813
+ next unless
2814
+ $pr->{'path'} =~ /$searchtext/ ||
2815
+ $pr->{'descr_long'} =~ /$searchtext/;
2816
+ }
2817
+
2818
+ push @projects, $pr;
2819
+ }
2820
+
2821
+ return @projects;
2822
+}
2823
+
2824
our $gitweb_project_owner = undef;
2825
sub git_get_project_list_from_file {
2826
@@ -4742,7 +4810,7 @@ sub git_patchset_body {
4810
# project in the list, removing invalid projects from returned list
4811
# NOTE: modifies $projlist, but does not remove entries from it
4812
sub fill_project_list_info {
4745
- my ($projlist, $check_forks) = @_;
4813
+ my $projlist = shift;
4814
my @projects;
4815
4816
my $show_ctags = gitweb_check_feature('ctags');
@@ -4762,23 +4830,36 @@ sub fill_project_list_info {
4830
if (!defined $pr->{'owner'}) {
4831
$pr->{'owner'} = git_get_project_owner("$pr->{'path'}") || "";
4832
}
4765
- if ($check_forks) {
4766
- my $pname = $pr->{'path'};
4767
- if (($pname =~ s/\.git$//) &&
4768
- ($pname !~ /\/$/) &&
4769
- (-d "$projectroot/$pname")) {
4770
- $pr->{'forks'} = "-d $projectroot/$pname";
4771
- } else {
4772
- $pr->{'forks'} = 0;
4773
- }
4833
+ if ($show_ctags) {
4834
+ $pr->{'ctags'} = git_get_project_ctags($pr->{'path'});
4835
}
4775
- $show_ctags and $pr->{'ctags'} = git_get_project_ctags($pr->{'path'});
4836
push @projects, $pr;
4837
}
4838
4839
return @projects;
4840
}
4841
4842
+sub sort_projects_list {
4843
+ my ($projlist, $order) = @_;
4844
+ my @projects;
4845
+
4846
+ my %order_info = (
4847
+ project => { key => 'path', type => 'str' },
4848
+ descr => { key => 'descr_long', type => 'str' },
4849
+ owner => { key => 'owner', type => 'str' },
4850
+ age => { key => 'age', type => 'num' }
4851
+ );
4852
+ my $oi = $order_info{$order};
4853
+ return @$projlist unless defined $oi;
4854
+ if ($oi->{'type'} eq 'str') {
4855
+ @projects = sort {$a->{$oi->{'key'}} cmp $b->{$oi->{'key'}}} @$projlist;
4856
+ } else {
4857
+ @projects = sort {$a->{$oi->{'key'}} <=> $b->{$oi->{'key'}}} @$projlist;
4858
+ }
4859
+
4860
+ return @projects;
4861
+}
4862
+
4863
# print 'sort by' <th> element, generating 'sort by $name' replay link
4864
# if that order is not selected
4865
sub print_sort_th {
@@ -4805,28 +4886,39 @@ sub format_sort_th {
4886
sub git_project_list_body {
4887
# actually uses global variable $project
4888
my ($projlist, $order, $from, $to, $extra, $no_header) = @_;
4889
+ my @projects = @$projlist;
4890
4891
my $check_forks = gitweb_check_feature('forks');
4810
- my @projects = fill_project_list_info($projlist, $check_forks);
4892
+ my $show_ctags = gitweb_check_feature('ctags');
4893
+ my $tagfilter = $show_ctags ? $cgi->param('by_tag') : undef;
4894
+ $check_forks = undef
4895
+ if ($tagfilter || $searchtext);
4896
+
4897
+ # filtering out forks before filling info allows to do less work
4898
+ @projects = filter_forks_from_projects_list(\@projects)
4899
+ if ($check_forks);
4900
+ @projects = fill_project_list_info(\@projects);
4901
+ # searching projects require filling to be run before it
4902
+ @projects = search_projects_list(\@projects,
4903
+ 'searchtext' => $searchtext,
4904
+ 'tagfilter' => $tagfilter)
4905
+ if ($tagfilter || $searchtext);
4906
4907
$order ||= $default_projects_order;
4908
$from = 0 unless defined $from;
4909
$to = $#projects if (!defined $to || $#projects < $to);
4910
4816
- my %order_info = (
4817
- project => { key => 'path', type => 'str' },
4818
- descr => { key => 'descr_long', type => 'str' },
4819
- owner => { key => 'owner', type => 'str' },
4820
- age => { key => 'age', type => 'num' }
4821
- );
4822
- my $oi = $order_info{$order};
4823
- if ($oi->{'type'} eq 'str') {
4824
- @projects = sort {$a->{$oi->{'key'}} cmp $b->{$oi->{'key'}}} @projects;
4825
- } else {
4826
- @projects = sort {$a->{$oi->{'key'}} <=> $b->{$oi->{'key'}}} @projects;
4911
+ # short circuit
4912
+ if ($from > $to) {
4913
+ print "<center>\n".
4914
+ "<b>No such projects found</b><br />\n".
4915
+ "Click ".$cgi->a({-href=>href(project=>undef)},"here")." to view all projects<br />\n".
4916
+ "</center>\n<br />\n";
4917
+ return;
4918
}
4919
4829
- my $show_ctags = gitweb_check_feature('ctags');
4920
+ @projects = sort_projects_list(\@projects, $order);
4921
+
4922
if ($show_ctags) {
4923
my %ctags;
4924
foreach my $p (@projects) {
@@ -4852,32 +4944,26 @@ sub git_project_list_body {
4944
"</tr>\n";
4945
}
4946
my $alternate = 1;
4855
- my $tagfilter = $cgi->param('by_tag');
4947
for (my $i = $from; $i <= $to; $i++) {
4948
my $pr = $projects[$i];
4949
4859
- next if $tagfilter and $show_ctags and not grep { lc $_ eq lc $tagfilter } keys %{$pr->{'ctags'}};
4860
- next if $searchtext and not $pr->{'path'} =~ /$searchtext/
4861
- and not $pr->{'descr_long'} =~ /$searchtext/;
4862
- # Weed out forks or non-matching entries of search
4863
- if ($check_forks) {
4864
- my $forkbase = $project; $forkbase ||= ''; $forkbase =~ s#\.git$#/#;
4865
- $forkbase="^$forkbase" if $forkbase;
4866
- next if not $searchtext and not $tagfilter and $show_ctags
4867
- and $pr->{'path'} =~ m#$forkbase.*/.*#; # regexp-safe
4868
- }
4869
-
4950
if ($alternate) {
4951
print "<tr class=\"dark\">\n";
4952
} else {
4953
print "<tr class=\"light\">\n";
4954
}
4955
$alternate ^= 1;
4956
+
4957
if ($check_forks) {
4958
print "<td>";
4959
if ($pr->{'forks'}) {
4879
- print "<!-- $pr->{'forks'} -->\n";
4880
- print $cgi->a({-href => href(project=>$pr->{'path'}, action=>"forks")}, "+");
4960
+ my $nforks = scalar @{$pr->{'forks'}};
4961
+ if ($nforks > 0) {
4962
+ print $cgi->a({-href => href(project=>$pr->{'path'}, action=>"forks"),
4963
+ -title => "$nforks forks"}, "+");
4964
+ } else {
4965
+ print $cgi->span({-title => "$nforks forks"}, "+");
4966
+ }
4967
}
4968
print "</td>\n";
4969
}
@@ -5357,7 +5443,10 @@ sub git_forks {
5443
}
5444
5445
sub git_project_index {
5360
- my @projects = git_get_projects_list($project);
5446
+ my @projects = git_get_projects_list();
5447
+ if (!@projects) {
5448
+ die_error(404, "No projects found");
5449
+ }
5450
5451
print $cgi->header(
5452
-type => 'text/plain',
@@ -5399,7 +5488,11 @@ sub git_summary {
5488
my $check_forks = gitweb_check_feature('forks');
5489
5490
if ($check_forks) {
5491
+ # find forks of a project
5492
@forklist = git_get_projects_list($project);
5493
+ # filter out forks of forks
5494
+ @forklist = filter_forks_from_projects_list(\@forklist)
5495
+ if (@forklist);
5496
}
5497
5498
git_header_html();
@@ -7319,6 +7412,9 @@ sub git_atom {
7412
7413
sub git_opml {
7414
my @list = git_get_projects_list();
7415
+ if (!@list) {
7416
+ die_error(404, "No projects found");
7417
+ }
7418
7419
print $cgi->header(
7420
-type => 'text/xml',
t/t9500-gitweb-standalone-no-errors.sh
+49
@@ -595,4 +595,53 @@ test_expect_success HIGHLIGHT \
595
git commit -m "Add test.sh" &&
596
gitweb_run "p=.git;a=blob;f=test.sh"'
597
598
+# ----------------------------------------------------------------------
599
+# forks of projects
600
+
601
+cat >>gitweb_config.perl <<\EOF &&
602
+$feature{'forks'}{'default'} = [1];
603
+EOF
604
+
605
+test_expect_success \
606
+ 'forks: prepare' \
607
+ 'git init --bare foo.git &&
608
+ git --git-dir=foo.git --work-tree=. add file &&
609
+ git --git-dir=foo.git --work-tree=. commit -m "Initial commit" &&
610
+ echo "foo" > foo.git/description &&
611
+ mkdir -p foo &&
612
+ (cd foo &&
613
+ git clone --shared --bare ../foo.git foo-forked.git &&
614
+ echo "fork of foo" > foo-forked.git/description)'
615
+
616
+test_expect_success \
617
+ 'forks: projects list' \
618
+ 'gitweb_run'
619
+
620
+test_expect_success \
621
+ 'forks: forks action' \
622
+ 'gitweb_run "p=foo.git;a=forks"'
623
+
624
+# ----------------------------------------------------------------------
625
+# content tags (tag cloud)
626
+
627
+cat >>gitweb_config.perl <<-\EOF &&
628
+# we don't test _setting_ content tags, so any true value is good
629
+$feature{'ctags'}{'default'} = ['ctags_script.cgi'];
630
+EOF
631
+
632
+test_expect_success \
633
+ 'ctags: tag cloud in projects list' \
634
+ 'mkdir .git/ctags &&
635
+ echo "2" > .git/ctags/foo &&
636
+ echo "1" > .git/ctags/bar &&
637
+ gitweb_run'
638
+
639
+test_expect_success \
640
+ 'ctags: search projects by existing tag' \
641
+ 'gitweb_run "by_tag=foo"'
642
+
643
+test_expect_success \
644
+ 'ctags: search projects by non existent tag' \
645
+ 'gitweb_run "by_tag=non-existent"'
646
+
647
test_done
t/t9502-gitweb-standalone-parse-output.sh
+74
@@ -112,4 +112,78 @@ test_expect_success 'snapshot: hierarchical branch name (xx/test)' '
112
'
113
test_debug 'cat gitweb.headers'
114
115
+# ----------------------------------------------------------------------
116
+# forks of projects
117
+
118
+test_expect_success 'forks: setup' '
119
+ git init --bare foo.git &&
120
+ echo file > file &&
121
+ git --git-dir=foo.git --work-tree=. add file &&
122
+ git --git-dir=foo.git --work-tree=. commit -m "Initial commit" &&
123
+ echo "foo" > foo.git/description &&
124
+ git clone --bare foo.git foo.bar.git &&
125
+ echo "foo.bar" > foo.bar.git/description &&
126
+ git clone --bare foo.git foo_baz.git &&
127
+ echo "foo_baz" > foo_baz.git/description &&
128
+ rm -fr foo &&
129
+ mkdir -p foo &&
130
+ (
131
+ cd foo &&
132
+ git clone --shared --bare ../foo.git foo-forked.git &&
133
+ echo "fork of foo" > foo-forked.git/description
134
+ )
135
+'
136
+
137
+test_expect_success 'forks: not skipped unless "forks" feature enabled' '
138
+ gitweb_run "a=project_list" &&
139
+ grep -q ">\\.git<" gitweb.body &&
140
+ grep -q ">foo\\.git<" gitweb.body &&
141
+ grep -q ">foo_baz\\.git<" gitweb.body &&
142
+ grep -q ">foo\\.bar\\.git<" gitweb.body &&
143
+ grep -q ">foo_baz\\.git<" gitweb.body &&
144
+ grep -q ">foo/foo-forked\\.git<" gitweb.body &&
145
+ grep -q ">fork of .*<" gitweb.body
146
+'
147
+
148
+cat >>gitweb_config.perl <<\EOF &&
149
+$feature{'forks'}{'default'} = [1];
150
+EOF
151
+
152
+test_expect_success 'forks: forks skipped if "forks" feature enabled' '
153
+ gitweb_run "a=project_list" &&
154
+ grep -q ">\\.git<" gitweb.body &&
155
+ grep -q ">foo\\.git<" gitweb.body &&
156
+ grep -q ">foo_baz\\.git<" gitweb.body &&
157
+ grep -q ">foo\\.bar\\.git<" gitweb.body &&
158
+ grep -q ">foo_baz\\.git<" gitweb.body &&
159
+ grep -v ">foo/foo-forked\\.git<" gitweb.body &&
160
+ grep -v ">fork of .*<" gitweb.body
161
+'
162
+
163
+test_expect_success 'forks: "forks" action for forked repository' '
164
+ gitweb_run "p=foo.git;a=forks" &&
165
+ grep -q ">foo/foo-forked\\.git<" gitweb.body &&
166
+ grep -q ">fork of foo<" gitweb.body
167
+'
168
+
169
+test_expect_success 'forks: can access forked repository' '
170
+ gitweb_run "p=foo/foo-forked.git;a=summary" &&
171
+ grep -q "200 OK" gitweb.headers &&
172
+ grep -q ">fork of foo<" gitweb.body
173
+'
174
+
175
+test_expect_success 'forks: project_index lists all projects (incl. forks)' '
176
+ cat >expected <<-\EOF
177
+ .git
178
+ foo.bar.git
179
+ foo.git
180
+ foo/foo-forked.git
181
+ foo_baz.git
182
+ EOF
183
+ gitweb_run "a=project_index" &&
184
+ sed -e "s/ .*//" <gitweb.body | sort >actual &&
185
+ test_cmp expected actual
186
+'
187
+
188
+
189
test_done