SVN import: Use one log call
One "svn log" (or its equivalent) per revision adds delay and server load. Instead, open two SVN connections -- one for the log, and one for the files. Positive side effect: Only those log entries which actually contain data are committed => no more empty commits. Also, change the "-l" option to set the maximum revision to be pulled, not the number of revisions. Signed-off-by: Junio C Hamano <junkio@cox.net>
Matthias Urlichs committed
Nov 29, 2005 at 08:13 UTC
034908047d8c46fb4e1323285b3b066567c7fadc
2 files changed
+21
-29
Documentation/git-svnimport.txt
+5
-8
@@ -10,7 +10,7 @@ git-svnimport - Import a SVN repository into git
10
SYNOPSIS
11
--------
12
'git-svnimport' [ -o <branch-for-HEAD> ] [ -h ] [ -v ] [ -d | -D ]
13
- [ -C <GIT_repository> ] [ -i ] [ -u ] [-l limit_nr_changes]
13
+ [ -C <GIT_repository> ] [ -i ] [ -u ] [-l limit_rev]
14
[ -b branch_subdir ] [ -t trunk_subdir ] [ -T tag_subdir ]
15
[ -s start_chg ] [ -m ] [ -M regex ]
16
<SVN_repository_URL> [ <path> ]
@@ -71,14 +71,11 @@ When importing incementally, you might need to edit the .git/svn2git file.
71
regex. It can be used with -m to also see the default regexes.
72
You must escape forward slashes.
73
74
--l <max_num_changes>::
75
- Limit the number of SVN changesets we pull before quitting.
76
- This option is necessary because the SVN library has serious memory
77
- leaks; the recommended value for nontrivial imports is 100.
74
+-l <max_rev>::
75
+ Specify a maximum revision number to pull.
76
79
- git-svnimport will still exit with a zero exit code. You can check
80
- the size of the file ".git/svn2git" to determine whether to call
81
- the importer again.
77
+ Formerly, this option controlled how many revisions to pull, due to
78
+ SVN memory leaks. (These have been worked around.)
79
80
-v::
81
Verbosity: let 'svnimport' report what it is doing.
git-svnimport.perl
+16
-21
@@ -35,7 +35,7 @@ our($opt_h,$opt_o,$opt_v,$opt_u,$opt_C,$opt_i,$opt_m,$opt_M,$opt_t,$opt_T,$opt_b
35
sub usage() {
36
print STDERR <<END;
37
Usage: ${\basename $0} # fetch/update GIT from SVN
38
- [-o branch-for-HEAD] [-h] [-v] [-l max_num_changes]
38
+ [-o branch-for-HEAD] [-h] [-v] [-l max_rev]
39
[-C GIT_repository] [-t tagname] [-T trunkname] [-b branchname]
40
[-d|-D] [-i] [-u] [-s start_chg] [-m] [-M regex] [SVN_URL]
41
END
@@ -126,8 +126,9 @@ sub file {
126
package main;
127
use URI;
128
129
-my $svn = $svn_url;
129
+our $svn = $svn_url;
130
$svn .= "/$svn_dir" if defined $svn_dir;
131
+my $svn2 = SVNconn->new($svn);
132
$svn = SVNconn->new($svn);
133
134
my $lwp_ua;
@@ -198,7 +199,7 @@ $ENV{GIT_INDEX_FILE} = $git_index;
199
my $maxnum = 0;
200
my $last_rev = "";
201
my $last_branch;
201
-my $current_rev = $opt_s-1;
202
+my $current_rev = $opt_s || 1;
203
unless(-d $git_dir) {
204
system("git-init-db");
205
die "Cannot init the GIT db at $git_tree: $?\n" if $?;
@@ -254,7 +255,7 @@ EOM
255
my($num,$branch,$ref) = split;
256
$branches{$branch}{$num} = $ref;
257
$branches{$branch}{"LAST"} = $ref;
257
- $current_rev = $num if $current_rev < $num;
258
+ $current_rev = $num+1 if $current_rev <= $num;
259
}
260
close($B);
261
}
@@ -708,17 +709,17 @@ sub commit {
709
print "DONE: $revision $dest $cid\n" if $opt_v;
710
}
711
711
-my ($changed_paths, $revision, $author, $date, $message, $pool) = @_;
712
-sub _commit_all {
713
- ($changed_paths, $revision, $author, $date, $message, $pool) = @_;
712
+sub commit_all {
713
+ # Recursive use of the SVN connection does not work
714
+ local $svn = $svn2;
715
+
716
+ my ($changed_paths, $revision, $author, $date, $message, $pool) = @_;
717
my %p;
718
while(my($path,$action) = each %$changed_paths) {
719
$p{$path} = [ $action->action,$action->copyfrom_path, $action->copyfrom_rev, $path ];
720
}
721
$changed_paths = \%p;
719
-}
722
721
-sub commit_all {
723
my %done;
724
my @col;
725
my $pref;
@@ -734,18 +735,12 @@ sub commit_all {
735
}
736
}
737
737
-while(++$current_rev <= $svn->{'maxrev'}) {
738
- if (defined $opt_l) {
739
- $opt_l--;
740
- if ($opt_l < 0) {
741
- last;
742
- }
743
- }
744
- my $pool=SVN::Pool->new;
745
- $svn->{'svn'}->get_log("/",$current_rev,$current_rev,1,1,1,\&_commit_all,$pool);
746
- $pool->clear;
747
- commit_all();
748
-}
738
+$opt_l = $svn->{'maxrev'} if not defined $opt_l or $opt_l > $svn->{'maxrev'};
739
+print "Fetching from $current_rev to $opt_l ...\n" if $opt_v;
740
+
741
+my $pool=SVN::Pool->new;
742
+$svn->{'svn'}->get_log("/",$current_rev,$opt_l,0,1,1,\&commit_all,$pool);
743
+$pool->clear;
744
745
746
unlink($git_index);