git-svn: allow multi-fetch to fetch things chronologically
Since single fetching is a special case of multi-fetch, share code with it and the fetch loop into Git::SVN::Ra since it uses a single Ra connection and multiple Git::SVN objects. Signed-off-by: Eric Wong <normalperson@yhbt.net>
Eric Wong committed
Jan 27, 2007 at 22:28 UTC
0af9c9f94ae8a327536679ec1976df65ecd64b6e
1 file changed
+120
-91
git-svn.perl
+120
-91
@@ -416,15 +416,11 @@ sub cmd_multi_init {
416
}
417
418
sub cmd_multi_fetch {
419
- my @gs;
420
- foreach (command(qw/config -l/)) {
421
- next unless m!^svn-remote\.(.+)\.fetch=
422
- \s*(.*)\s*:\s*refs/remotes/(.+)\s*$!x;
423
- my ($repo_id, $path, $ref_id) = ($1, $2, $3);
424
- push @gs, Git::SVN->new($ref_id, $repo_id, $path);
425
- }
426
- foreach (@gs) {
427
- $_->fetch;
419
+ my $remotes = Git::SVN::read_all_remotes();
420
+ foreach my $repo_id (sort keys %$remotes) {
421
+ my $url = $remotes->{$repo_id}->{url} or next;
422
+ my $fetch = $remotes->{$repo_id}->{fetch} or next;
423
+ Git::SVN::fetch_all($repo_id, $url, $fetch);
424
}
425
}
426
@@ -698,6 +694,28 @@ BEGIN {
694
svn:entry:committed-date/;
695
}
696
697
+sub fetch_all {
698
+ my ($repo_id, $url, $fetch) = @_;
699
+ my @gs;
700
+ my $ra = Git::SVN::Ra->new($url);
701
+ my $head = $ra->get_latest_revnum;
702
+ my $base = $head;
703
+ my $new_remote;
704
+ foreach my $p (sort keys %$fetch) {
705
+ my $gs = Git::SVN->new($fetch->{$p}, $repo_id, $p);
706
+ my $lr = $gs->last_rev;
707
+ if (defined $lr) {
708
+ $base = $lr if ($lr < $base);
709
+ } else {
710
+ $new_remote = 1;
711
+ }
712
+ push @gs, $gs;
713
+ }
714
+ $base = 0 if $new_remote;
715
+ return if (++$base > $head);
716
+ $ra->gs_fetch_loop_common($base, $head, @gs);
717
+}
718
+
719
sub read_all_remotes {
720
my $r = {};
721
foreach (grep { s/^svn-remote\.// } command(qw/config -l/)) {
@@ -981,16 +999,12 @@ sub assert_index_clean {
999
}
1000
1001
sub get_commit_parents {
984
- my ($self, $log_entry, @parents) = @_;
1002
+ my ($self, $log_entry) = @_;
1003
my (%seen, @ret, @tmp);
986
- # commit parents can be conditionally bound to a particular
987
- # svn revision via: "svn_revno=commit_sha1", filter them out here:
988
- foreach my $p (@parents) {
989
- next unless defined $p;
990
- if ($p =~ /^(\d+)=($::sha1_short)$/o) {
991
- push @tmp, $2 if $1 == $log_entry->{revision};
992
- } else {
993
- push @tmp, $p if $p =~ /^$::sha1_short$/o;
1004
+ # legacy support for 'set-tree'; this is only used by set_tree_cb:
1005
+ if (my $ip = $self->{inject_parents}) {
1006
+ if (my $commit = delete $ip->{$log_entry->{revision}}) {
1007
+ push @tmp, $commit;
1008
}
1009
}
1010
if (my $cur = ::verify_ref($self->refname.'^0')) {
@@ -1017,7 +1031,7 @@ sub full_url {
1031
}
1032
1033
sub do_git_commit {
1020
- my ($self, $log_entry, @parents) = @_;
1034
+ my ($self, $log_entry) = @_;
1035
if (my $c = $self->rev_db_get($log_entry->{revision})) {
1036
croak "$log_entry->{revision} = $c already exists! ",
1037
"Why are we refetching it?\n";
@@ -1037,7 +1051,7 @@ sub do_git_commit {
1051
die "Tree is not a valid sha1: $tree\n" if $tree !~ /^$::sha1$/o;
1052
1053
my @exec = ('git-commit-tree', $tree);
1040
- foreach ($self->get_commit_parents($log_entry, @parents)) {
1054
+ foreach ($self->get_commit_parents($log_entry)) {
1055
push @exec, '-p', $_;
1056
}
1057
defined(my $pid = open3(my $msg_fh, my $out_fh, '>&STDERR', @exec))
@@ -1291,40 +1305,7 @@ sub fetch {
1305
my ($last_rev, $last_commit) = $self->last_rev_commit;
1306
my ($base, $head) = $self->get_fetch_range($min_rev, $max_rev);
1307
return if ($base > $head);
1294
- if (defined $last_commit) {
1295
- $self->assert_index_clean($last_commit);
1296
- }
1297
- my $inc = 1000;
1298
- my ($min, $max) = ($base, $head < $base + $inc ? $head : $base + $inc);
1299
- my $err_handler = $SVN::Error::handler;
1300
- my $err;
1301
- $SVN::Error::handler = sub { ($err) = @_; skip_unknown_revs($err); } ;
1302
- while (1) {
1303
- my @revs;
1304
- $self->ra->get_log([$self->{path}], $min, $max, 0, 1, 1,
1305
- sub {
1306
- my ($paths, $rev) = @_;
1307
- push @revs, [ dup_changed_paths($paths), $rev ];
1308
- });
1309
- if (! @revs && $err && $max >= $head) {
1310
- print STDERR "Branch probably deleted:\n ",
1311
- $err->expanded_message,
1312
- "\nWill attempt to follow revisions ",
1313
- "r$min .. r$max",
1314
- "committed before the deletion\n";
1315
- @revs = map { [ undef, $_ ] } ($min .. $max);
1316
- }
1317
- foreach (@revs) {
1318
- if (my $log_entry = $self->do_fetch(@$_)) {
1319
- $self->do_git_commit($log_entry, @parents);
1320
- }
1321
- }
1322
- last if $max >= $head;
1323
- $min = $max + 1;
1324
- $max += $inc;
1325
- $max = $head if ($max > $head);
1326
- }
1327
- $SVN::Error::handler = $err_handler;
1308
+ $self->ra->gs_fetch_loop_common($base, $head, $self);
1309
}
1310
1311
sub set_tree_cb {
@@ -1335,7 +1316,8 @@ sub set_tree_cb {
1316
$log_entry->{author} = $author;
1317
$self->do_git_commit($log_entry, "$rev=$tree");
1318
} else {
1338
- $self->fetch(undef, undef, "$rev=$tree");
1319
+ $self->{inject_parents} = { $rev => $tree };
1320
+ $self->fetch(undef, undef);
1321
}
1322
}
1323
@@ -1358,42 +1340,6 @@ sub set_tree {
1340
}
1341
}
1342
1361
-sub skip_unknown_revs {
1362
- my ($err) = @_;
1363
- my $errno = $err->apr_err();
1364
- # Maybe the branch we're tracking didn't
1365
- # exist when the repo started, so it's
1366
- # not an error if it doesn't, just continue
1367
- #
1368
- # Wonderfully consistent library, eh?
1369
- # 160013 - svn:// and file://
1370
- # 175002 - http(s)://
1371
- # 175007 - http(s):// (this repo required authorization, too...)
1372
- # More codes may be discovered later...
1373
- if ($errno == 175007 || $errno == 175002 || $errno == 160013) {
1374
- return;
1375
- }
1376
- croak "Error from SVN, ($errno): ", $err->expanded_message,"\n";
1377
-}
1378
-
1379
-# svn_log_changed_path_t objects passed to get_log are likely to be
1380
-# overwritten even if only the refs are copied to an external variable,
1381
-# so we should dup the structures in their entirety. Using an externally
1382
-# passed pool (instead of our temporary and quickly cleared pool in
1383
-# Git::SVN::Ra) does not help matters at all...
1384
-sub dup_changed_paths {
1385
- my ($paths) = @_;
1386
- return undef unless $paths;
1387
- my %ret;
1388
- foreach my $p (keys %$paths) {
1389
- my $i = $paths->{$p};
1390
- my %s = map { $_ => $i->$_ }
1391
- qw/copyfrom_path copyfrom_rev action/;
1392
- $ret{$p} = \%s;
1393
- }
1394
- \%ret;
1395
-}
1396
-
1343
# rev_db:
1344
# Tie::File seems to be prone to offset errors if revisions get sparse,
1345
# it's not that fast, either. Tie::File is also not in Perl 5.6. So
@@ -2324,6 +2270,53 @@ sub gs_do_switch {
2270
$editor->{git_commit_ok};
2271
}
2272
2273
+sub gs_fetch_loop_common {
2274
+ my ($self, $base, $head, @gs) = @_;
2275
+ my $inc = 1000;
2276
+ my ($min, $max) = ($base, $head < $base + $inc ? $head : $base + $inc);
2277
+ my $err_handler = $SVN::Error::handler;
2278
+ my $err;
2279
+ $SVN::Error::handler = sub { ($err) = @_; skip_unknown_revs($err); };
2280
+ my @paths = @gs == 1 ? ($gs[0]->{path}) : ('');
2281
+ foreach my $gs (@gs) {
2282
+ if (my $last_commit = $gs->last_commit) {
2283
+ $gs->assert_index_clean($last_commit);
2284
+ }
2285
+ $gs->{path_regex} = qr/^\/\Q$gs->{path}\E\/?/;
2286
+ }
2287
+ while (1) {
2288
+ my @revs;
2289
+ $self->get_log(\@paths, $min, $max, 0, 1, 1,
2290
+ sub { push @revs, [ dup_changed_paths($_[0]), $_[1] ]; });
2291
+ if (! @revs && $err && $max >= $head) {
2292
+ print STDERR "Branch probably deleted:\n ",
2293
+ $err->expanded_message,
2294
+ "\nWill attempt to follow revisions ",
2295
+ "r$min .. r$max ",
2296
+ "committed before the deletion\n";
2297
+ @revs = map { [ undef, $_ ] } ($min .. $max);
2298
+ }
2299
+ foreach (@revs) {
2300
+ my ($paths, $r) = @$_;
2301
+ foreach my $gs (@gs) {
2302
+ if ($paths) {
2303
+ grep /$gs->{path_regex}/, keys %$paths
2304
+ or next;
2305
+ }
2306
+ next if defined $gs->rev_db_get($r);
2307
+ if (my $log_entry = $gs->do_fetch($paths, $r)) {
2308
+ $gs->do_git_commit($log_entry);
2309
+ }
2310
+ }
2311
+ }
2312
+ last if $max >= $head;
2313
+ $min = $max + 1;
2314
+ $max += $inc;
2315
+ $max = $head if ($max > $head);
2316
+ }
2317
+ $SVN::Error::handler = $err_handler;
2318
+}
2319
+
2320
sub minimize_url {
2321
my ($self) = @_;
2322
return $self->{url} if ($self->{url} eq $self->{repos_root});
@@ -2356,6 +2349,42 @@ sub can_do_switch {
2349
$can_do_switch;
2350
}
2351
2352
+sub skip_unknown_revs {
2353
+ my ($err) = @_;
2354
+ my $errno = $err->apr_err();
2355
+ # Maybe the branch we're tracking didn't
2356
+ # exist when the repo started, so it's
2357
+ # not an error if it doesn't, just continue
2358
+ #
2359
+ # Wonderfully consistent library, eh?
2360
+ # 160013 - svn:// and file://
2361
+ # 175002 - http(s)://
2362
+ # 175007 - http(s):// (this repo required authorization, too...)
2363
+ # More codes may be discovered later...
2364
+ if ($errno == 175007 || $errno == 175002 || $errno == 160013) {
2365
+ return;
2366
+ }
2367
+ die "Error from SVN, ($errno): ", $err->expanded_message,"\n";
2368
+}
2369
+
2370
+# svn_log_changed_path_t objects passed to get_log are likely to be
2371
+# overwritten even if only the refs are copied to an external variable,
2372
+# so we should dup the structures in their entirety. Using an externally
2373
+# passed pool (instead of our temporary and quickly cleared pool in
2374
+# Git::SVN::Ra) does not help matters at all...
2375
+sub dup_changed_paths {
2376
+ my ($paths) = @_;
2377
+ return undef unless $paths;
2378
+ my %ret;
2379
+ foreach my $p (keys %$paths) {
2380
+ my $i = $paths->{$p};
2381
+ my %s = map { $_ => $i->$_ }
2382
+ qw/copyfrom_path copyfrom_rev action/;
2383
+ $ret{$p} = \%s;
2384
+ }
2385
+ \%ret;
2386
+}
2387
+
2388
package Git::SVN::Log;
2389
use strict;
2390
use warnings;