@matheusbsbs / matheus-bagliano / commits / 00f429af7b
Fork 0

gitweb: Handle non UTF-8 text better

gitweb assumes that everything is in UTF-8. If a text contains invalid UTF-8 character sequences, the text must be in a different encoding. This commit introduces $fallback_encoding which would be used as input encoding if gitweb encounters text with is not valid UTF-8. Add basic test for this in t/t9500-gitweb-standalone-no-errors.sh Signed-off-by: Martin Koegler <mkoegler@auto.tuwien.ac.at> Signed-off-by: Jakub Narebski <jnareb@gmail.com> Tested-by: Alexandre Julliard <julliard@winehq.org> Tested-by: Ismail Dönmez <ismail@pardus.org.tr> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Martin Koegler committed Jun 3, 2007 at 17:42 UTC 00f429af7bfaa6a9141bc0ac30995d4fae24836a
2 files changed +59 -10
gitweb/gitweb.perl
+31 -10
@@ -94,6 +94,13 @@ our $default_text_plain_charset = undef;
94 # (relative to the current git repository)
95 our $mimetypes_file = undef;
96
97 +# assume this charset if line contains non-UTF-8 characters;
98 +# it should be valid encoding (see Encoding::Supported(3pm) for list),
99 +# for which encoding all byte sequences are valid, for example
100 +# 'iso-8859-1' aka 'latin1' (it is decoded without checking, so it
101 +# could be even 'utf-8' for the old behavior)
102 +our $fallback_encoding = 'latin1';
103 +
104 # You define site-wide feature defaults here; override them with
105 # $GITWEB_CONFIG as necessary.
106 our %feature = (
@@ -602,6 +609,20 @@ sub validate_refname {
609 return $input;
610 }
611
612 +# decode sequences of octets in utf8 into Perl's internal form,
613 +# which is utf-8 with utf8 flag set if needed. gitweb writes out
614 +# in utf-8 thanks to "binmode STDOUT, ':utf8'" at beginning
615 +sub to_utf8 {
616 + my $str = shift;
617 + my $res;
618 + eval { $res = decode_utf8($str, Encode::FB_CROAK); };
619 + if (defined $res) {
620 + return $res;
621 + } else {
622 + return decode($fallback_encoding, $str, Encode::FB_DEFAULT);
623 + }
624 +}
625 +
626 # quote unsafe chars, but keep the slash, even when it's not
627 # correct, but quoted slashes look too horrible in bookmarks
628 sub esc_param {
@@ -626,7 +647,7 @@ sub esc_html ($;%) {
647 my $str = shift;
648 my %opts = @_;
649
629 - $str = decode_utf8($str);
650 + $str = to_utf8($str);
651 $str = $cgi->escapeHTML($str);
652 if ($opts{'-nbsp'}) {
653 $str =~ s/ /&nbsp;/g;
@@ -640,7 +661,7 @@ sub esc_path {
661 my $str = shift;
662 my %opts = @_;
663
643 - $str = decode_utf8($str);
664 + $str = to_utf8($str);
665 $str = $cgi->escapeHTML($str);
666 if ($opts{'-nbsp'}) {
667 $str =~ s/ /&nbsp;/g;
@@ -925,7 +946,7 @@ sub format_subject_html {
946
947 if (length($short) < length($long)) {
948 return $cgi->a({-href => $href, -class => "list subject",
928 - -title => decode_utf8($long)},
949 + -title => to_utf8($long)},
950 esc_html($short) . $extra);
951 } else {
952 return $cgi->a({-href => $href, -class => "list subject"},
@@ -1239,7 +1260,7 @@ sub git_get_projects_list {
1260 if (check_export_ok("$projectroot/$path")) {
1261 my $pr = {
1262 path => $path,
1242 - owner => decode_utf8($owner),
1263 + owner => to_utf8($owner),
1264 };
1265 push @list, $pr;
1266 (my $forks_path = $path) =~ s/\.git$//;
@@ -1269,7 +1290,7 @@ sub git_get_project_owner {
1290 $pr = unescape($pr);
1291 $ow = unescape($ow);
1292 if ($pr eq $project) {
1272 - $owner = decode_utf8($ow);
1293 + $owner = to_utf8($ow);
1294 last;
1295 }
1296 }
@@ -1759,7 +1780,7 @@ sub get_file_owner {
1780 }
1781 my $owner = $gcos;
1782 $owner =~ s/[,;].*$//;
1762 - return decode_utf8($owner);
1783 + return to_utf8($owner);
1784 }
1785
1786 ## ......................................................................
@@ -1842,7 +1863,7 @@ sub git_header_html {
1863
1864 my $title = "$site_name";
1865 if (defined $project) {
1845 - $title .= " - " . decode_utf8($project);
1866 + $title .= " - " . to_utf8($project);
1867 if (defined $action) {
1868 $title .= "/$action";
1869 if (defined $file_name) {
@@ -2116,7 +2137,7 @@ sub git_print_page_path {
2137
2138 print "<div class=\"page_path\">";
2139 print $cgi->a({-href => href(action=>"tree", hash_base=>$hb),
2119 - -title => 'tree root'}, decode_utf8("[$project]"));
2140 + -title => 'tree root'}, to_utf8("[$project]"));
2141 print " / ";
2142 if (defined $name) {
2143 my @dirname = split '/', $name;
@@ -2936,7 +2957,7 @@ sub git_project_list_body {
2957 ($pr->{'age'}, $pr->{'age_string'}) = @aa;
2958 if (!defined $pr->{'descr'}) {
2959 my $descr = git_get_project_description($pr->{'path'}) || "";
2939 - $pr->{'descr_long'} = decode_utf8($descr);
2960 + $pr->{'descr_long'} = to_utf8($descr);
2961 $pr->{'descr'} = chop_str($descr, 25, 5);
2962 }
2963 if (!defined $pr->{'owner'}) {
@@ -3981,7 +4002,7 @@ sub git_snapshot {
4002 my $git = git_cmd_str();
4003 my $name = $project;
4004 $name =~ s/\047/\047\\\047\047/g;
3984 - my $filename = decode_utf8(basename($project));
4005 + my $filename = to_utf8(basename($project));
4006 my $cmd;
4007 if ($suffix eq 'zip') {
4008 $filename .= "-$hash.$suffix";
t/t9500-gitweb-standalone-no-errors.sh
+28
@@ -487,4 +487,32 @@ test_expect_success \
487 'gitweb_run "p=.git;a=atom"'
488 test_debug 'cat gitweb.log'
489
490 +# ----------------------------------------------------------------------
491 +# encoding/decoding
492 +
493 +test_expect_success \
494 + 'encode(commit): utf8' \
495 + '. ../t3901-utf8.txt &&
496 + echo "UTF-8" >> file &&
497 + git add file &&
498 + git commit -F ../t3900/1-UTF-8.txt &&
499 + gitweb_run "p=.git;a=commit"'
500 +test_debug 'cat gitweb.log'
501 +
502 +test_expect_success \
503 + 'encode(commit): iso-8859-1' \
504 + '. ../t3901-8859-1.txt &&
505 + echo "ISO-8859-1" >> file &&
506 + git add file &&
507 + git config i18n.commitencoding ISO-8859-1 &&
508 + git commit -F ../t3900/ISO-8859-1.txt &&
509 + git config --unset i18n.commitencoding &&
510 + gitweb_run "p=.git;a=commit"'
511 +test_debug 'cat gitweb.log'
512 +
513 +test_expect_success \
514 + 'encode(log): utf-8 and iso-8859-1' \
515 + 'gitweb_run "p=.git;a=log"'
516 +test_debug 'cat gitweb.log'
517 +
518 test_done