@matheusbsbs / matheus-bagliano / commits / 0382318424
Fork 0

contrib/git-svn: stabilize memory usage for big fetches

We should be safely able to import histories with thousands of revisions without hogging up lots of memory. With this, we lose the ability to autocorrect mistakes when people specify revisions in reverse, but it's probably no longer a problem since we only have one method of log parsing nowadays. I've added an extra check to ensure that revision numbers do increment. Also, increment the version number to 0.11.0. I really should just call it 1.0 soon... Signed-off-by: Eric Wong <normalperson@yhbt.net> Signed-off-by: Junio C Hamano <junkio@cox.net>

Eric Wong committed Mar 25, 2006 at 18:52 UTC 03823184247215fba1dd9c9c39659d08dea3bad7
1 file changed +63 -46
contrib/git-svn/git-svn.perl
+63 -46
@@ -8,7 +8,7 @@ use vars qw/ $AUTHOR $VERSION
8 $GIT_SVN_INDEX $GIT_SVN
9 $GIT_DIR $REV_DIR/;
10 $AUTHOR = 'Eric Wong <normalperson@yhbt.net>';
11 -$VERSION = '0.10.0';
11 +$VERSION = '0.11.0';
12 $GIT_DIR = $ENV{GIT_DIR} || "$ENV{PWD}/.git";
13 # make sure the svn binary gives consistent output between locales and TZs:
14 $ENV{TZ} = 'UTC';
@@ -217,9 +217,8 @@ sub fetch {
217 push @log_args, '--stop-on-copy' unless $_no_stop_copy;
218
219 my $svn_log = svn_log_raw(@log_args);
220 - @$svn_log = sort { $a->{revision} <=> $b->{revision} } @$svn_log;
220
222 - my $base = shift @$svn_log or croak "No base revision!\n";
221 + my $base = next_log_entry($svn_log) or croak "No base revision!\n";
222 my $last_commit = undef;
223 unless (-d $SVN_WC) {
224 svn_cmd_checkout($SVN_URL,$base->{revision},$SVN_WC);
@@ -234,18 +233,22 @@ sub fetch {
233 }
234 my @svn_up = qw(svn up);
235 push @svn_up, '--ignore-externals' unless $_no_ignore_ext;
237 - my $last_rev = $base->{revision};
238 - foreach my $log_msg (@$svn_log) {
239 - assert_svn_wc_clean($last_rev, $last_commit);
240 - $last_rev = $log_msg->{revision};
241 - sys(@svn_up,"-r$last_rev");
236 + my $last = $base;
237 + while (my $log_msg = next_log_entry($svn_log)) {
238 + assert_svn_wc_clean($last->{revision}, $last_commit);
239 + if ($last->{revision} >= $log_msg->{revision}) {
240 + croak "Out of order: last >= current: ",
241 + "$last->{revision} >= $log_msg->{revision}\n";
242 + }
243 + sys(@svn_up,"-r$log_msg->{revision}");
244 $last_commit = git_commit($log_msg, $last_commit, @parents);
245 + $last = $log_msg;
246 }
244 - assert_svn_wc_clean($last_rev, $last_commit);
247 + assert_svn_wc_clean($last->{revision}, $last_commit);
248 unless (-e "$GIT_DIR/refs/heads/master") {
249 sys(qw(git-update-ref refs/heads/master),$last_commit);
250 }
248 - return pop @$svn_log;
251 + return $last;
252 }
253
254 sub commit {
@@ -708,49 +711,61 @@ sub svn_commit_tree {
711 return fetch("$rev_committed=$commit")->{revision};
712 }
713
714 +# read the entire log into a temporary file (which is removed ASAP)
715 +# and store the file handle + parser state
716 sub svn_log_raw {
717 my (@log_args) = @_;
713 - my $pid = open my $log_fh,'-|';
718 + my $log_fh = IO::File->new_tmpfile or croak $!;
719 + my $pid = fork;
720 defined $pid or croak $!;
715 -
716 - if ($pid == 0) {
721 + if (!$pid) {
722 + open STDOUT, '>&', $log_fh or croak $!;
723 exec (qw(svn log), @log_args) or croak $!
724 }
725 + waitpid $pid, 0;
726 + croak if $?;
727 + seek $log_fh, 0, 0 or croak $!;
728 + return { state => 'sep', fh => $log_fh };
729 +}
730 +
731 +sub next_log_entry {
732 + my $log = shift; # retval of svn_log_raw()
733 + my $ret = undef;
734 + my $fh = $log->{fh};
735
720 - my @svn_log;
721 - my $state = 'sep';
722 - while (<$log_fh>) {
736 + while (<$fh>) {
737 chomp;
738 if (/^\-{72}$/) {
725 - if ($state eq 'msg') {
726 - if ($svn_log[$#svn_log]->{lines}) {
727 - $svn_log[$#svn_log]->{msg} .= $_."\n";
728 - unless(--$svn_log[$#svn_log]->{lines}) {
729 - $state = 'sep';
739 + if ($log->{state} eq 'msg') {
740 + if ($ret->{lines}) {
741 + $ret->{msg} .= $_."\n";
742 + unless(--$ret->{lines}) {
743 + $log->{state} = 'sep';
744 }
745 } else {
746 croak "Log parse error at: $_\n",
733 - $svn_log[$#svn_log]->{revision},
747 + $ret->{revision},
748 "\n";
749 }
750 next;
751 }
738 - if ($state ne 'sep') {
752 + if ($log->{state} ne 'sep') {
753 croak "Log parse error at: $_\n",
740 - "state: $state\n",
741 - $svn_log[$#svn_log]->{revision},
754 + "state: $log->{state}\n",
755 + $ret->{revision},
756 "\n";
757 }
744 - $state = 'rev';
758 + $log->{state} = 'rev';
759
760 # if we have an empty log message, put something there:
747 - if (@svn_log) {
748 - $svn_log[$#svn_log]->{msg} ||= "\n";
749 - delete $svn_log[$#svn_log]->{lines};
761 + if ($ret) {
762 + $ret->{msg} ||= "\n";
763 + delete $ret->{lines};
764 + return $ret;
765 }
766 next;
767 }
753 - if ($state eq 'rev' && s/^r(\d+)\s*\|\s*//) {
768 + if ($log->{state} eq 'rev' && s/^r(\d+)\s*\|\s*//) {
769 my $rev = $1;
770 my ($author, $date, $lines) = split(/\s*\|\s*/, $_, 3);
771 ($lines) = ($lines =~ /(\d+)/);
@@ -758,36 +773,34 @@ sub svn_log_raw {
773 /(\d{4})\-(\d\d)\-(\d\d)\s
774 (\d\d)\:(\d\d)\:(\d\d)\s([\-\+]\d+)/x)
775 or croak "Failed to parse date: $date\n";
761 - my %log_msg = ( revision => $rev,
776 + $ret = { revision => $rev,
777 date => "$tz $Y-$m-$d $H:$M:$S",
778 author => $author,
779 lines => $lines,
765 - msg => '' );
780 + msg => '' };
781 if (defined $_authors && ! defined $users{$author}) {
782 die "Author: $author not defined in ",
783 "$_authors file\n";
784 }
770 - push @svn_log, \%log_msg;
771 - $state = 'msg_start';
785 + $log->{state} = 'msg_start';
786 next;
787 }
788 # skip the first blank line of the message:
775 - if ($state eq 'msg_start' && /^$/) {
776 - $state = 'msg';
777 - } elsif ($state eq 'msg') {
778 - if ($svn_log[$#svn_log]->{lines}) {
779 - $svn_log[$#svn_log]->{msg} .= $_."\n";
780 - unless (--$svn_log[$#svn_log]->{lines}) {
781 - $state = 'sep';
789 + if ($log->{state} eq 'msg_start' && /^$/) {
790 + $log->{state} = 'msg';
791 + } elsif ($log->{state} eq 'msg') {
792 + if ($ret->{lines}) {
793 + $ret->{msg} .= $_."\n";
794 + unless (--$ret->{lines}) {
795 + $log->{state} = 'sep';
796 }
797 } else {
798 croak "Log parse error at: $_\n",
785 - $svn_log[$#svn_log]->{revision},"\n";
799 + $ret->{revision},"\n";
800 }
801 }
802 }
789 - close $log_fh or croak $?;
790 - return \@svn_log;
803 + return $ret;
804 }
805
806 sub svn_info {
@@ -1114,9 +1127,13 @@ __END__
1127
1128 Data structures:
1129
1117 -@svn_log = array of log_msg hashes
1130 +$svn_log hashref (as returned by svn_log_raw)
1131 +{
1132 + fh => file handle of the log file,
1133 + state => state of the log file parser (sep/msg/rev/msg_start...)
1134 +}
1135
1119 -$log_msg hash
1136 +$log_msg hashref as returned by next_log_entry($svn_log)
1137 {
1138 msg => 'whitespace-formatted log entry
1139 ', # trailing newline is preserved