@matheusbsbs / matheus-bagliano / commits / 0af9c9f94a
Fork 0

git-svn: allow multi-fetch to fetch things chronologically

Since single fetching is a special case of multi-fetch, share code with it and the fetch loop into Git::SVN::Ra since it uses a single Ra connection and multiple Git::SVN objects. Signed-off-by: Eric Wong <normalperson@yhbt.net>

Eric Wong committed Jan 27, 2007 at 22:28 UTC 0af9c9f94ae8a327536679ec1976df65ecd64b6e
1 file changed +120 -91
git-svn.perl
+120 -91
@@ -416,15 +416,11 @@ sub cmd_multi_init {
416 }
417
418 sub cmd_multi_fetch {
419 - my @gs;
420 - foreach (command(qw/config -l/)) {
421 - next unless m!^svn-remote\.(.+)\.fetch=
422 - \s*(.*)\s*:\s*refs/remotes/(.+)\s*$!x;
423 - my ($repo_id, $path, $ref_id) = ($1, $2, $3);
424 - push @gs, Git::SVN->new($ref_id, $repo_id, $path);
425 - }
426 - foreach (@gs) {
427 - $_->fetch;
419 + my $remotes = Git::SVN::read_all_remotes();
420 + foreach my $repo_id (sort keys %$remotes) {
421 + my $url = $remotes->{$repo_id}->{url} or next;
422 + my $fetch = $remotes->{$repo_id}->{fetch} or next;
423 + Git::SVN::fetch_all($repo_id, $url, $fetch);
424 }
425 }
426
@@ -698,6 +694,28 @@ BEGIN {
694 svn:entry:committed-date/;
695 }
696
697 +sub fetch_all {
698 + my ($repo_id, $url, $fetch) = @_;
699 + my @gs;
700 + my $ra = Git::SVN::Ra->new($url);
701 + my $head = $ra->get_latest_revnum;
702 + my $base = $head;
703 + my $new_remote;
704 + foreach my $p (sort keys %$fetch) {
705 + my $gs = Git::SVN->new($fetch->{$p}, $repo_id, $p);
706 + my $lr = $gs->last_rev;
707 + if (defined $lr) {
708 + $base = $lr if ($lr < $base);
709 + } else {
710 + $new_remote = 1;
711 + }
712 + push @gs, $gs;
713 + }
714 + $base = 0 if $new_remote;
715 + return if (++$base > $head);
716 + $ra->gs_fetch_loop_common($base, $head, @gs);
717 +}
718 +
719 sub read_all_remotes {
720 my $r = {};
721 foreach (grep { s/^svn-remote\.// } command(qw/config -l/)) {
@@ -981,16 +999,12 @@ sub assert_index_clean {
999 }
1000
1001 sub get_commit_parents {
984 - my ($self, $log_entry, @parents) = @_;
1002 + my ($self, $log_entry) = @_;
1003 my (%seen, @ret, @tmp);
986 - # commit parents can be conditionally bound to a particular
987 - # svn revision via: "svn_revno=commit_sha1", filter them out here:
988 - foreach my $p (@parents) {
989 - next unless defined $p;
990 - if ($p =~ /^(\d+)=($::sha1_short)$/o) {
991 - push @tmp, $2 if $1 == $log_entry->{revision};
992 - } else {
993 - push @tmp, $p if $p =~ /^$::sha1_short$/o;
1004 + # legacy support for 'set-tree'; this is only used by set_tree_cb:
1005 + if (my $ip = $self->{inject_parents}) {
1006 + if (my $commit = delete $ip->{$log_entry->{revision}}) {
1007 + push @tmp, $commit;
1008 }
1009 }
1010 if (my $cur = ::verify_ref($self->refname.'^0')) {
@@ -1017,7 +1031,7 @@ sub full_url {
1031 }
1032
1033 sub do_git_commit {
1020 - my ($self, $log_entry, @parents) = @_;
1034 + my ($self, $log_entry) = @_;
1035 if (my $c = $self->rev_db_get($log_entry->{revision})) {
1036 croak "$log_entry->{revision} = $c already exists! ",
1037 "Why are we refetching it?\n";
@@ -1037,7 +1051,7 @@ sub do_git_commit {
1051 die "Tree is not a valid sha1: $tree\n" if $tree !~ /^$::sha1$/o;
1052
1053 my @exec = ('git-commit-tree', $tree);
1040 - foreach ($self->get_commit_parents($log_entry, @parents)) {
1054 + foreach ($self->get_commit_parents($log_entry)) {
1055 push @exec, '-p', $_;
1056 }
1057 defined(my $pid = open3(my $msg_fh, my $out_fh, '>&STDERR', @exec))
@@ -1291,40 +1305,7 @@ sub fetch {
1305 my ($last_rev, $last_commit) = $self->last_rev_commit;
1306 my ($base, $head) = $self->get_fetch_range($min_rev, $max_rev);
1307 return if ($base > $head);
1294 - if (defined $last_commit) {
1295 - $self->assert_index_clean($last_commit);
1296 - }
1297 - my $inc = 1000;
1298 - my ($min, $max) = ($base, $head < $base + $inc ? $head : $base + $inc);
1299 - my $err_handler = $SVN::Error::handler;
1300 - my $err;
1301 - $SVN::Error::handler = sub { ($err) = @_; skip_unknown_revs($err); } ;
1302 - while (1) {
1303 - my @revs;
1304 - $self->ra->get_log([$self->{path}], $min, $max, 0, 1, 1,
1305 - sub {
1306 - my ($paths, $rev) = @_;
1307 - push @revs, [ dup_changed_paths($paths), $rev ];
1308 - });
1309 - if (! @revs && $err && $max >= $head) {
1310 - print STDERR "Branch probably deleted:\n ",
1311 - $err->expanded_message,
1312 - "\nWill attempt to follow revisions ",
1313 - "r$min .. r$max",
1314 - "committed before the deletion\n";
1315 - @revs = map { [ undef, $_ ] } ($min .. $max);
1316 - }
1317 - foreach (@revs) {
1318 - if (my $log_entry = $self->do_fetch(@$_)) {
1319 - $self->do_git_commit($log_entry, @parents);
1320 - }
1321 - }
1322 - last if $max >= $head;
1323 - $min = $max + 1;
1324 - $max += $inc;
1325 - $max = $head if ($max > $head);
1326 - }
1327 - $SVN::Error::handler = $err_handler;
1308 + $self->ra->gs_fetch_loop_common($base, $head, $self);
1309 }
1310
1311 sub set_tree_cb {
@@ -1335,7 +1316,8 @@ sub set_tree_cb {
1316 $log_entry->{author} = $author;
1317 $self->do_git_commit($log_entry, "$rev=$tree");
1318 } else {
1338 - $self->fetch(undef, undef, "$rev=$tree");
1319 + $self->{inject_parents} = { $rev => $tree };
1320 + $self->fetch(undef, undef);
1321 }
1322 }
1323
@@ -1358,42 +1340,6 @@ sub set_tree {
1340 }
1341 }
1342
1361 -sub skip_unknown_revs {
1362 - my ($err) = @_;
1363 - my $errno = $err->apr_err();
1364 - # Maybe the branch we're tracking didn't
1365 - # exist when the repo started, so it's
1366 - # not an error if it doesn't, just continue
1367 - #
1368 - # Wonderfully consistent library, eh?
1369 - # 160013 - svn:// and file://
1370 - # 175002 - http(s)://
1371 - # 175007 - http(s):// (this repo required authorization, too...)
1372 - # More codes may be discovered later...
1373 - if ($errno == 175007 || $errno == 175002 || $errno == 160013) {
1374 - return;
1375 - }
1376 - croak "Error from SVN, ($errno): ", $err->expanded_message,"\n";
1377 -}
1378 -
1379 -# svn_log_changed_path_t objects passed to get_log are likely to be
1380 -# overwritten even if only the refs are copied to an external variable,
1381 -# so we should dup the structures in their entirety. Using an externally
1382 -# passed pool (instead of our temporary and quickly cleared pool in
1383 -# Git::SVN::Ra) does not help matters at all...
1384 -sub dup_changed_paths {
1385 - my ($paths) = @_;
1386 - return undef unless $paths;
1387 - my %ret;
1388 - foreach my $p (keys %$paths) {
1389 - my $i = $paths->{$p};
1390 - my %s = map { $_ => $i->$_ }
1391 - qw/copyfrom_path copyfrom_rev action/;
1392 - $ret{$p} = \%s;
1393 - }
1394 - \%ret;
1395 -}
1396 -
1343 # rev_db:
1344 # Tie::File seems to be prone to offset errors if revisions get sparse,
1345 # it's not that fast, either. Tie::File is also not in Perl 5.6. So
@@ -2324,6 +2270,53 @@ sub gs_do_switch {
2270 $editor->{git_commit_ok};
2271 }
2272
2273 +sub gs_fetch_loop_common {
2274 + my ($self, $base, $head, @gs) = @_;
2275 + my $inc = 1000;
2276 + my ($min, $max) = ($base, $head < $base + $inc ? $head : $base + $inc);
2277 + my $err_handler = $SVN::Error::handler;
2278 + my $err;
2279 + $SVN::Error::handler = sub { ($err) = @_; skip_unknown_revs($err); };
2280 + my @paths = @gs == 1 ? ($gs[0]->{path}) : ('');
2281 + foreach my $gs (@gs) {
2282 + if (my $last_commit = $gs->last_commit) {
2283 + $gs->assert_index_clean($last_commit);
2284 + }
2285 + $gs->{path_regex} = qr/^\/\Q$gs->{path}\E\/?/;
2286 + }
2287 + while (1) {
2288 + my @revs;
2289 + $self->get_log(\@paths, $min, $max, 0, 1, 1,
2290 + sub { push @revs, [ dup_changed_paths($_[0]), $_[1] ]; });
2291 + if (! @revs && $err && $max >= $head) {
2292 + print STDERR "Branch probably deleted:\n ",
2293 + $err->expanded_message,
2294 + "\nWill attempt to follow revisions ",
2295 + "r$min .. r$max ",
2296 + "committed before the deletion\n";
2297 + @revs = map { [ undef, $_ ] } ($min .. $max);
2298 + }
2299 + foreach (@revs) {
2300 + my ($paths, $r) = @$_;
2301 + foreach my $gs (@gs) {
2302 + if ($paths) {
2303 + grep /$gs->{path_regex}/, keys %$paths
2304 + or next;
2305 + }
2306 + next if defined $gs->rev_db_get($r);
2307 + if (my $log_entry = $gs->do_fetch($paths, $r)) {
2308 + $gs->do_git_commit($log_entry);
2309 + }
2310 + }
2311 + }
2312 + last if $max >= $head;
2313 + $min = $max + 1;
2314 + $max += $inc;
2315 + $max = $head if ($max > $head);
2316 + }
2317 + $SVN::Error::handler = $err_handler;
2318 +}
2319 +
2320 sub minimize_url {
2321 my ($self) = @_;
2322 return $self->{url} if ($self->{url} eq $self->{repos_root});
@@ -2356,6 +2349,42 @@ sub can_do_switch {
2349 $can_do_switch;
2350 }
2351
2352 +sub skip_unknown_revs {
2353 + my ($err) = @_;
2354 + my $errno = $err->apr_err();
2355 + # Maybe the branch we're tracking didn't
2356 + # exist when the repo started, so it's
2357 + # not an error if it doesn't, just continue
2358 + #
2359 + # Wonderfully consistent library, eh?
2360 + # 160013 - svn:// and file://
2361 + # 175002 - http(s)://
2362 + # 175007 - http(s):// (this repo required authorization, too...)
2363 + # More codes may be discovered later...
2364 + if ($errno == 175007 || $errno == 175002 || $errno == 160013) {
2365 + return;
2366 + }
2367 + die "Error from SVN, ($errno): ", $err->expanded_message,"\n";
2368 +}
2369 +
2370 +# svn_log_changed_path_t objects passed to get_log are likely to be
2371 +# overwritten even if only the refs are copied to an external variable,
2372 +# so we should dup the structures in their entirety. Using an externally
2373 +# passed pool (instead of our temporary and quickly cleared pool in
2374 +# Git::SVN::Ra) does not help matters at all...
2375 +sub dup_changed_paths {
2376 + my ($paths) = @_;
2377 + return undef unless $paths;
2378 + my %ret;
2379 + foreach my $p (keys %$paths) {
2380 + my $i = $paths->{$p};
2381 + my %s = map { $_ => $i->$_ }
2382 + qw/copyfrom_path copyfrom_rev action/;
2383 + $ret{$p} = \%s;
2384 + }
2385 + \%ret;
2386 +}
2387 +
2388 package Git::SVN::Log;
2389 use strict;
2390 use warnings;