gitweb: Handle non UTF-8 text better
gitweb assumes that everything is in UTF-8. If a text contains invalid UTF-8 character sequences, the text must be in a different encoding. This commit introduces $fallback_encoding which would be used as input encoding if gitweb encounters text with is not valid UTF-8. Add basic test for this in t/t9500-gitweb-standalone-no-errors.sh Signed-off-by: Martin Koegler <mkoegler@auto.tuwien.ac.at> Signed-off-by: Jakub Narebski <jnareb@gmail.com> Tested-by: Alexandre Julliard <julliard@winehq.org> Tested-by: Ismail Dönmez <ismail@pardus.org.tr> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Martin Koegler committed
Jun 3, 2007 at 17:42 UTC
00f429af7bfaa6a9141bc0ac30995d4fae24836a
2 files changed
+59
-10
gitweb/gitweb.perl
+31
-10
@@ -94,6 +94,13 @@ our $default_text_plain_charset = undef;
94
# (relative to the current git repository)
95
our $mimetypes_file = undef;
96
97
+# assume this charset if line contains non-UTF-8 characters;
98
+# it should be valid encoding (see Encoding::Supported(3pm) for list),
99
+# for which encoding all byte sequences are valid, for example
100
+# 'iso-8859-1' aka 'latin1' (it is decoded without checking, so it
101
+# could be even 'utf-8' for the old behavior)
102
+our $fallback_encoding = 'latin1';
103
+
104
# You define site-wide feature defaults here; override them with
105
# $GITWEB_CONFIG as necessary.
106
our %feature = (
@@ -602,6 +609,20 @@ sub validate_refname {
609
return $input;
610
}
611
612
+# decode sequences of octets in utf8 into Perl's internal form,
613
+# which is utf-8 with utf8 flag set if needed. gitweb writes out
614
+# in utf-8 thanks to "binmode STDOUT, ':utf8'" at beginning
615
+sub to_utf8 {
616
+ my $str = shift;
617
+ my $res;
618
+ eval { $res = decode_utf8($str, Encode::FB_CROAK); };
619
+ if (defined $res) {
620
+ return $res;
621
+ } else {
622
+ return decode($fallback_encoding, $str, Encode::FB_DEFAULT);
623
+ }
624
+}
625
+
626
# quote unsafe chars, but keep the slash, even when it's not
627
# correct, but quoted slashes look too horrible in bookmarks
628
sub esc_param {
@@ -626,7 +647,7 @@ sub esc_html ($;%) {
647
my $str = shift;
648
my %opts = @_;
649
629
- $str = decode_utf8($str);
650
+ $str = to_utf8($str);
651
$str = $cgi->escapeHTML($str);
652
if ($opts{'-nbsp'}) {
653
$str =~ s/ / /g;
@@ -640,7 +661,7 @@ sub esc_path {
661
my $str = shift;
662
my %opts = @_;
663
643
- $str = decode_utf8($str);
664
+ $str = to_utf8($str);
665
$str = $cgi->escapeHTML($str);
666
if ($opts{'-nbsp'}) {
667
$str =~ s/ / /g;
@@ -925,7 +946,7 @@ sub format_subject_html {
946
947
if (length($short) < length($long)) {
948
return $cgi->a({-href => $href, -class => "list subject",
928
- -title => decode_utf8($long)},
949
+ -title => to_utf8($long)},
950
esc_html($short) . $extra);
951
} else {
952
return $cgi->a({-href => $href, -class => "list subject"},
@@ -1239,7 +1260,7 @@ sub git_get_projects_list {
1260
if (check_export_ok("$projectroot/$path")) {
1261
my $pr = {
1262
path => $path,
1242
- owner => decode_utf8($owner),
1263
+ owner => to_utf8($owner),
1264
};
1265
push @list, $pr;
1266
(my $forks_path = $path) =~ s/\.git$//;
@@ -1269,7 +1290,7 @@ sub git_get_project_owner {
1290
$pr = unescape($pr);
1291
$ow = unescape($ow);
1292
if ($pr eq $project) {
1272
- $owner = decode_utf8($ow);
1293
+ $owner = to_utf8($ow);
1294
last;
1295
}
1296
}
@@ -1759,7 +1780,7 @@ sub get_file_owner {
1780
}
1781
my $owner = $gcos;
1782
$owner =~ s/[,;].*$//;
1762
- return decode_utf8($owner);
1783
+ return to_utf8($owner);
1784
}
1785
1786
## ......................................................................
@@ -1842,7 +1863,7 @@ sub git_header_html {
1863
1864
my $title = "$site_name";
1865
if (defined $project) {
1845
- $title .= " - " . decode_utf8($project);
1866
+ $title .= " - " . to_utf8($project);
1867
if (defined $action) {
1868
$title .= "/$action";
1869
if (defined $file_name) {
@@ -2116,7 +2137,7 @@ sub git_print_page_path {
2137
2138
print "<div class=\"page_path\">";
2139
print $cgi->a({-href => href(action=>"tree", hash_base=>$hb),
2119
- -title => 'tree root'}, decode_utf8("[$project]"));
2140
+ -title => 'tree root'}, to_utf8("[$project]"));
2141
print " / ";
2142
if (defined $name) {
2143
my @dirname = split '/', $name;
@@ -2936,7 +2957,7 @@ sub git_project_list_body {
2957
($pr->{'age'}, $pr->{'age_string'}) = @aa;
2958
if (!defined $pr->{'descr'}) {
2959
my $descr = git_get_project_description($pr->{'path'}) || "";
2939
- $pr->{'descr_long'} = decode_utf8($descr);
2960
+ $pr->{'descr_long'} = to_utf8($descr);
2961
$pr->{'descr'} = chop_str($descr, 25, 5);
2962
}
2963
if (!defined $pr->{'owner'}) {
@@ -3981,7 +4002,7 @@ sub git_snapshot {
4002
my $git = git_cmd_str();
4003
my $name = $project;
4004
$name =~ s/\047/\047\\\047\047/g;
3984
- my $filename = decode_utf8(basename($project));
4005
+ my $filename = to_utf8(basename($project));
4006
my $cmd;
4007
if ($suffix eq 'zip') {
4008
$filename .= "-$hash.$suffix";
t/t9500-gitweb-standalone-no-errors.sh
+28
@@ -487,4 +487,32 @@ test_expect_success \
487
'gitweb_run "p=.git;a=atom"'
488
test_debug 'cat gitweb.log'
489
490
+# ----------------------------------------------------------------------
491
+# encoding/decoding
492
+
493
+test_expect_success \
494
+ 'encode(commit): utf8' \
495
+ '. ../t3901-utf8.txt &&
496
+ echo "UTF-8" >> file &&
497
+ git add file &&
498
+ git commit -F ../t3900/1-UTF-8.txt &&
499
+ gitweb_run "p=.git;a=commit"'
500
+test_debug 'cat gitweb.log'
501
+
502
+test_expect_success \
503
+ 'encode(commit): iso-8859-1' \
504
+ '. ../t3901-8859-1.txt &&
505
+ echo "ISO-8859-1" >> file &&
506
+ git add file &&
507
+ git config i18n.commitencoding ISO-8859-1 &&
508
+ git commit -F ../t3900/ISO-8859-1.txt &&
509
+ git config --unset i18n.commitencoding &&
510
+ gitweb_run "p=.git;a=commit"'
511
+test_debug 'cat gitweb.log'
512
+
513
+test_expect_success \
514
+ 'encode(log): utf-8 and iso-8859-1' \
515
+ 'gitweb_run "p=.git;a=log"'
516
+test_debug 'cat gitweb.log'
517
+
518
test_done