svn import: Add direct HTTP access
Some SVN repositories that are accessible through HTTP don't like when I retrieve files using SVN methods ("internal server error"). Therefore, I added an option to get the contents using (persistent) HTTP directly. This also reduces round-trip time, from two or three requests down to one. Also corrected error handling a bit. Signed-Off-By: Matthias Urlichs <smurf@smurf.noris.de>
Matthias Urlichs committed
Oct 11, 2005 at 18:13 UTC
25f6f325d7a8f7cb686a9ffd9fa2c00b3aa85a60
2 files changed
+80
-19
Documentation/git-svnimport.txt
+26
-2
@@ -9,10 +9,11 @@ git-svnimport - Import a SVN repository into git
9
10
SYNOPSIS
11
--------
12
-'git-svnimport' [ -o <branch-for-HEAD> ] [ -h ] [ -v ]
12
+'git-svnimport' [ -o <branch-for-HEAD> ] [ -h ] [ -v ] [ -d | -D ]
13
[ -C <GIT_repository> ] [ -i ] [ -u ] [-l limit_nr_changes]
14
[ -b branch_subdir ] [ -t trunk_subdir ] [ -T tag_subdir ]
15
- [ -s start_chg ] [ -m ] [ -M regex ] [ <SVN_repository_URL> ]
15
+ [ -s start_chg ] [ -m ] [ -M regex ]
16
+ <SVN_repository_URL> [ <path> ]
17
18
19
DESCRIPTION
@@ -82,9 +83,32 @@ When importing incementally, you might need to edit the .git/svn2git file.
83
-v::
84
Verbosity: let 'svnimport' report what it is doing.
85
86
+-d::
87
+ Use direct HTTP requests if possible. The "<path>" argument is used
88
+ only for retrieving the SVN logs; the path to the contents is
89
+ included in the SVN log.
90
+
91
+-D::
92
+ Use direct HTTP requests if possible. The "<path>" argument is used
93
+ for retrieving the logs, as well as for the contents.
94
++
95
+There's no safe way to automatically find out which of these options to
96
+use, so you need to try both. Usually, the one that's wrong will die
97
+with a 40x error pretty quickly.
98
+
99
<SVN_repository_URL>::
100
The URL of the SVN module you want to import. For local
101
repositories, use "file:///absolute/path".
102
++
103
+If you're using the "-d" or "-D" option, this is the URL of the SVN
104
+repository itself; it usually ends in "/svn".
105
+
106
+<SVN_repository_URL>::
107
+ The URL of the SVN module you want to import. For local
108
+ repositories, use "file:///absolute/path".
109
+
110
+<path>
111
+ The path to the module you want to check out.
112
113
-h::
114
Print a short usage message and exit.
git-svnimport.perl
+54
-17
@@ -30,26 +30,26 @@ die "Need CVN:Core 1.2.1 or better" if $SVN::Core::VERSION lt "1.2.1";
30
$SIG{'PIPE'}="IGNORE";
31
$ENV{'TZ'}="UTC";
32
33
-our($opt_h,$opt_o,$opt_v,$opt_u,$opt_C,$opt_i,$opt_m,$opt_M,$opt_t,$opt_T,$opt_b,$opt_s,$opt_l);
33
+our($opt_h,$opt_o,$opt_v,$opt_u,$opt_C,$opt_i,$opt_m,$opt_M,$opt_t,$opt_T,$opt_b,$opt_s,$opt_l,$opt_d,$opt_D);
34
35
sub usage() {
36
print STDERR <<END;
37
Usage: ${\basename $0} # fetch/update GIT from CVS
38
[-o branch-for-HEAD] [-h] [-v] [-l max_num_changes]
39
[-C GIT_repository] [-t tagname] [-T trunkname] [-b branchname]
40
- [-i] [-u] [-s start_chg] [-m] [-M regex] [SVN_URL]
40
+ [-d|-D] [-i] [-u] [-s start_chg] [-m] [-M regex] [SVN_URL]
41
END
42
exit(1);
43
}
44
45
-getopts("b:C:hil:mM:o:s:t:T:uv") or usage();
45
+getopts("b:C:dDhil:mM:o:s:t:T:uv") or usage();
46
usage if $opt_h;
47
48
my $tag_name = $opt_t || "tags";
49
my $trunk_name = $opt_T || "trunk";
50
my $branch_name = $opt_b || "branches";
51
52
-@ARGV == 1 or usage();
52
+@ARGV == 1 or @ARGV == 2 or usage();
53
54
$opt_o ||= "origin";
55
$opt_s ||= 1;
@@ -58,6 +58,7 @@ my $git_tree = $opt_C;
58
$git_tree ||= ".";
59
60
my $svn_url = $ARGV[0];
61
+my $svn_dir = $ARGV[1];
62
63
our @mergerx = ();
64
if ($opt_m) {
@@ -106,30 +107,44 @@ sub conn {
107
108
sub file {
109
my($self,$path,$rev) = @_;
109
- my $res;
110
111
my ($fh, $name) = tempfile('gitsvn.XXXXXX',
112
DIR => File::Spec->tmpdir(), UNLINK => 1);
113
114
print "... $rev $path ...\n" if $opt_v;
115
eval { $self->{'svn'}->get_file($path,$rev,$fh); };
116
- if ($@ and $@ !~ /Attempted to get checksum/) {
117
- # retry
118
- $self->conn();
119
- eval { $self->{'svn'}->get_file($path,$rev,$fh); };
120
- };
121
- return () if $@ and $@ !~ /Attempted to get checksum/;
122
- die $@ if $@;
116
+ if($@) {
117
+ return undef if $@ =~ /Attempted to get checksum/;
118
+ die $@;
119
+ }
120
close ($fh);
121
125
- return ($name, $res);
122
+ return $name;
123
}
124
128
-
125
package main;
126
+use URI;
127
131
-my $svn = SVNconn->new($svn_url);
128
+my $svn = $svn_url;
129
+$svn .= "/$svn_dir" if defined $svn_dir;
130
+$svn = SVNconn->new($svn);
131
132
+my $lwp_ua;
133
+if($opt_d or $opt_D) {
134
+ $svn_url = URI->new($svn_url)->canonical;
135
+ if($opt_D) {
136
+ $svn_dir =~ s#/*$#/#;
137
+ } else {
138
+ $svn_dir = "";
139
+ }
140
+ if ($svn_url->scheme eq "http") {
141
+ use LWP::UserAgent;
142
+ $lwp_ua = LWP::UserAgent->new(keep_alive => 1, requests_redirectable => []);
143
+ } else {
144
+ print STDERR "Warning: not HTTP; turning off direct file access\n";
145
+ $opt_d=0;
146
+ }
147
+}
148
149
sub pdate($) {
150
my($d) = @_;
@@ -258,8 +273,30 @@ sub get_file($$$) {
273
}
274
275
# now get it
261
- my ($name, $res) = eval { $svn->file($svnpath,$rev); };
262
- return () unless defined $name;
276
+ my $name;
277
+ if($opt_d) {
278
+ my($req,$res);
279
+
280
+ # /svn/!svn/bc/2/django/trunk/django-docs/build.py
281
+ my $url=$svn_url->clone();
282
+ $url->path($url->path."/!svn/bc/$rev/$svn_dir$svnpath");
283
+ print "Fetching $url...\n" if $opt_v;
284
+ $req = HTTP::Request->new(GET => $url);
285
+ $res = $lwp_ua->request($req);
286
+ if ($res->is_success) {
287
+ my $fh;
288
+ ($fh, $name) = tempfile('gitsvn.XXXXXX',
289
+ DIR => File::Spec->tmpdir(), UNLINK => 1);
290
+ print $fh $res->content;
291
+ close($fh) or die "Could not write $name: $!\n";
292
+ } else {
293
+ return undef if $res->code == 301; # directory?
294
+ die $res->status_line." at $url\n";
295
+ }
296
+ } else {
297
+ $name = $svn->file($svnpath,$rev);
298
+ return undef unless defined $name;
299
+ }
300
301
open my $F, '-|', "git-hash-object", "-w", $name
302
or die "Cannot create object: $!\n";