contrib/git-svn: stabilize memory usage for big fetches
We should be safely able to import histories with thousands of revisions without hogging up lots of memory. With this, we lose the ability to autocorrect mistakes when people specify revisions in reverse, but it's probably no longer a problem since we only have one method of log parsing nowadays. I've added an extra check to ensure that revision numbers do increment. Also, increment the version number to 0.11.0. I really should just call it 1.0 soon... Signed-off-by: Eric Wong <normalperson@yhbt.net> Signed-off-by: Junio C Hamano <junkio@cox.net>
Eric Wong committed
Mar 25, 2006 at 18:52 UTC
03823184247215fba1dd9c9c39659d08dea3bad7
1 file changed
+63
-46
contrib/git-svn/git-svn.perl
+63
-46
@@ -8,7 +8,7 @@ use vars qw/ $AUTHOR $VERSION
8
$GIT_SVN_INDEX $GIT_SVN
9
$GIT_DIR $REV_DIR/;
10
$AUTHOR = 'Eric Wong <normalperson@yhbt.net>';
11
-$VERSION = '0.10.0';
11
+$VERSION = '0.11.0';
12
$GIT_DIR = $ENV{GIT_DIR} || "$ENV{PWD}/.git";
13
# make sure the svn binary gives consistent output between locales and TZs:
14
$ENV{TZ} = 'UTC';
@@ -217,9 +217,8 @@ sub fetch {
217
push @log_args, '--stop-on-copy' unless $_no_stop_copy;
218
219
my $svn_log = svn_log_raw(@log_args);
220
- @$svn_log = sort { $a->{revision} <=> $b->{revision} } @$svn_log;
220
222
- my $base = shift @$svn_log or croak "No base revision!\n";
221
+ my $base = next_log_entry($svn_log) or croak "No base revision!\n";
222
my $last_commit = undef;
223
unless (-d $SVN_WC) {
224
svn_cmd_checkout($SVN_URL,$base->{revision},$SVN_WC);
@@ -234,18 +233,22 @@ sub fetch {
233
}
234
my @svn_up = qw(svn up);
235
push @svn_up, '--ignore-externals' unless $_no_ignore_ext;
237
- my $last_rev = $base->{revision};
238
- foreach my $log_msg (@$svn_log) {
239
- assert_svn_wc_clean($last_rev, $last_commit);
240
- $last_rev = $log_msg->{revision};
241
- sys(@svn_up,"-r$last_rev");
236
+ my $last = $base;
237
+ while (my $log_msg = next_log_entry($svn_log)) {
238
+ assert_svn_wc_clean($last->{revision}, $last_commit);
239
+ if ($last->{revision} >= $log_msg->{revision}) {
240
+ croak "Out of order: last >= current: ",
241
+ "$last->{revision} >= $log_msg->{revision}\n";
242
+ }
243
+ sys(@svn_up,"-r$log_msg->{revision}");
244
$last_commit = git_commit($log_msg, $last_commit, @parents);
245
+ $last = $log_msg;
246
}
244
- assert_svn_wc_clean($last_rev, $last_commit);
247
+ assert_svn_wc_clean($last->{revision}, $last_commit);
248
unless (-e "$GIT_DIR/refs/heads/master") {
249
sys(qw(git-update-ref refs/heads/master),$last_commit);
250
}
248
- return pop @$svn_log;
251
+ return $last;
252
}
253
254
sub commit {
@@ -708,49 +711,61 @@ sub svn_commit_tree {
711
return fetch("$rev_committed=$commit")->{revision};
712
}
713
714
+# read the entire log into a temporary file (which is removed ASAP)
715
+# and store the file handle + parser state
716
sub svn_log_raw {
717
my (@log_args) = @_;
713
- my $pid = open my $log_fh,'-|';
718
+ my $log_fh = IO::File->new_tmpfile or croak $!;
719
+ my $pid = fork;
720
defined $pid or croak $!;
715
-
716
- if ($pid == 0) {
721
+ if (!$pid) {
722
+ open STDOUT, '>&', $log_fh or croak $!;
723
exec (qw(svn log), @log_args) or croak $!
724
}
725
+ waitpid $pid, 0;
726
+ croak if $?;
727
+ seek $log_fh, 0, 0 or croak $!;
728
+ return { state => 'sep', fh => $log_fh };
729
+}
730
+
731
+sub next_log_entry {
732
+ my $log = shift; # retval of svn_log_raw()
733
+ my $ret = undef;
734
+ my $fh = $log->{fh};
735
720
- my @svn_log;
721
- my $state = 'sep';
722
- while (<$log_fh>) {
736
+ while (<$fh>) {
737
chomp;
738
if (/^\-{72}$/) {
725
- if ($state eq 'msg') {
726
- if ($svn_log[$#svn_log]->{lines}) {
727
- $svn_log[$#svn_log]->{msg} .= $_."\n";
728
- unless(--$svn_log[$#svn_log]->{lines}) {
729
- $state = 'sep';
739
+ if ($log->{state} eq 'msg') {
740
+ if ($ret->{lines}) {
741
+ $ret->{msg} .= $_."\n";
742
+ unless(--$ret->{lines}) {
743
+ $log->{state} = 'sep';
744
}
745
} else {
746
croak "Log parse error at: $_\n",
733
- $svn_log[$#svn_log]->{revision},
747
+ $ret->{revision},
748
"\n";
749
}
750
next;
751
}
738
- if ($state ne 'sep') {
752
+ if ($log->{state} ne 'sep') {
753
croak "Log parse error at: $_\n",
740
- "state: $state\n",
741
- $svn_log[$#svn_log]->{revision},
754
+ "state: $log->{state}\n",
755
+ $ret->{revision},
756
"\n";
757
}
744
- $state = 'rev';
758
+ $log->{state} = 'rev';
759
760
# if we have an empty log message, put something there:
747
- if (@svn_log) {
748
- $svn_log[$#svn_log]->{msg} ||= "\n";
749
- delete $svn_log[$#svn_log]->{lines};
761
+ if ($ret) {
762
+ $ret->{msg} ||= "\n";
763
+ delete $ret->{lines};
764
+ return $ret;
765
}
766
next;
767
}
753
- if ($state eq 'rev' && s/^r(\d+)\s*\|\s*//) {
768
+ if ($log->{state} eq 'rev' && s/^r(\d+)\s*\|\s*//) {
769
my $rev = $1;
770
my ($author, $date, $lines) = split(/\s*\|\s*/, $_, 3);
771
($lines) = ($lines =~ /(\d+)/);
@@ -758,36 +773,34 @@ sub svn_log_raw {
773
/(\d{4})\-(\d\d)\-(\d\d)\s
774
(\d\d)\:(\d\d)\:(\d\d)\s([\-\+]\d+)/x)
775
or croak "Failed to parse date: $date\n";
761
- my %log_msg = ( revision => $rev,
776
+ $ret = { revision => $rev,
777
date => "$tz $Y-$m-$d $H:$M:$S",
778
author => $author,
779
lines => $lines,
765
- msg => '' );
780
+ msg => '' };
781
if (defined $_authors && ! defined $users{$author}) {
782
die "Author: $author not defined in ",
783
"$_authors file\n";
784
}
770
- push @svn_log, \%log_msg;
771
- $state = 'msg_start';
785
+ $log->{state} = 'msg_start';
786
next;
787
}
788
# skip the first blank line of the message:
775
- if ($state eq 'msg_start' && /^$/) {
776
- $state = 'msg';
777
- } elsif ($state eq 'msg') {
778
- if ($svn_log[$#svn_log]->{lines}) {
779
- $svn_log[$#svn_log]->{msg} .= $_."\n";
780
- unless (--$svn_log[$#svn_log]->{lines}) {
781
- $state = 'sep';
789
+ if ($log->{state} eq 'msg_start' && /^$/) {
790
+ $log->{state} = 'msg';
791
+ } elsif ($log->{state} eq 'msg') {
792
+ if ($ret->{lines}) {
793
+ $ret->{msg} .= $_."\n";
794
+ unless (--$ret->{lines}) {
795
+ $log->{state} = 'sep';
796
}
797
} else {
798
croak "Log parse error at: $_\n",
785
- $svn_log[$#svn_log]->{revision},"\n";
799
+ $ret->{revision},"\n";
800
}
801
}
802
}
789
- close $log_fh or croak $?;
790
- return \@svn_log;
803
+ return $ret;
804
}
805
806
sub svn_info {
@@ -1114,9 +1127,13 @@ __END__
1127
1128
Data structures:
1129
1117
-@svn_log = array of log_msg hashes
1130
+$svn_log hashref (as returned by svn_log_raw)
1131
+{
1132
+ fh => file handle of the log file,
1133
+ state => state of the log file parser (sep/msg/rev/msg_start...)
1134
+}
1135
1119
-$log_msg hash
1136
+$log_msg hashref as returned by next_log_entry($svn_log)
1137
{
1138
msg => 'whitespace-formatted log entry
1139
', # trailing newline is preserved