@matheusbsbs / matheus-bagliano / commits / 27a1a8014b
Fork 0

git-svn: enable delta transfers during fetches when using SVN:: libs

This should drastically reduce bandwidth used for network transfers. This is not enabled for file:// repositories by default because of the increased CPU usage and I/O needed. GIT_SVN_DELTA_FETCH may be set to a true value to enable or false (0) to disable delta transfers regardless of the repository type. Signed-off-by: Eric Wong <normalperson@yhbt.net> Signed-off-by: Junio C Hamano <junkio@cox.net>

Eric Wong committed Nov 27, 2006 at 21:44 UTC 27a1a8014b842c0d70fdc91c68dd361ca2dfb34c
1 file changed +189 -5
git-svn.perl
+189 -5
@@ -68,7 +68,7 @@ my ($_revision,$_stdin,$_no_ignore_ext,$_no_stop_copy,$_help,$_rmdir,$_edit,
68 $_limit, $_verbose, $_incremental, $_oneline, $_l_fmt, $_show_commit,
69 $_version, $_upgrade, $_authors, $_branch_all_refs, @_opt_m,
70 $_merge, $_strategy, $_dry_run, $_ignore_nodate, $_non_recursive,
71 - $_username, $_config_dir, $_no_auth_cache);
71 + $_username, $_config_dir, $_no_auth_cache, $_xfer_delta);
72 my (@_branch_from, %tree_map, %users, %rusers, %equiv);
73 my ($_svn_co_url_revs, $_svn_pg_peg_revs);
74 my @repo_path_split_cache;
@@ -2675,6 +2675,9 @@ sub libsvn_load {
2675 require SVN::Ra;
2676 require SVN::Delta;
2677 push @SVN::Git::Editor::ISA, 'SVN::Delta::Editor';
2678 + push @SVN::Git::Fetcher::ISA, 'SVN::Delta::Editor';
2679 + *SVN::Git::Fetcher::process_rm = *process_rm;
2680 + *SVN::Git::Fetcher::safe_qx = *safe_qx;
2681 my $kill_stupid_warnings = $SVN::Node::none.$SVN::Node::file.
2682 $SVN::Node::dir.$SVN::Node::unknown.
2683 $SVN::Node::none.$SVN::Node::file.
@@ -2827,6 +2830,13 @@ sub libsvn_connect {
2830 config => $config,
2831 pool => SVN::Pool->new,
2832 auth_provider_callbacks => $callbacks);
2833 +
2834 + my $df = $ENV{GIT_SVN_DELTA_FETCH};
2835 + if (defined $df) {
2836 + $_xfer_delta = $df;
2837 + } else {
2838 + $_xfer_delta = ($url =~ m#^file://#) ? undef : 1;
2839 + }
2840 $ra->{svn_path} = $url;
2841 $ra->{repos_root} = $ra->get_repos_root;
2842 $ra->{svn_path} =~ s#^\Q$ra->{repos_root}\E/*##;
@@ -2915,6 +2925,24 @@ sub process_rm {
2925 }
2926
2927 sub libsvn_fetch {
2928 + $_xfer_delta ? libsvn_fetch_delta(@_) : libsvn_fetch_full(@_);
2929 +}
2930 +
2931 +sub libsvn_fetch_delta {
2932 + my ($last_commit, $paths, $rev, $author, $date, $msg) = @_;
2933 + my $pool = SVN::Pool->new;
2934 + my $ed = SVN::Git::Fetcher->new({ c => $last_commit, ra => $SVN,
2935 + paths => $paths });
2936 + my $reporter = $SVN->do_update($rev, '', 1, $ed, $pool);
2937 + my @lock = $SVN::Core::VERSION ge '1.2.0' ? (undef) : ();
2938 + my (undef, $last_rev, undef) = cmt_metadata($last_commit);
2939 + $reporter->set_path('', $last_rev, 0, @lock, $pool);
2940 + $reporter->finish_report($pool);
2941 + $pool->clear;
2942 + libsvn_log_entry($rev, $author, $date, $msg, [$last_commit]);
2943 +}
2944 +
2945 +sub libsvn_fetch_full {
2946 my ($last_commit, $paths, $rev, $author, $date, $msg) = @_;
2947 open my $gui, '| git-update-index -z --index-info' or croak $!;
2948 my @amr;
@@ -3133,7 +3161,11 @@ sub libsvn_find_parent_branch {
3161 unlink $GIT_SVN_INDEX;
3162 print STDERR "Found branch parent: ($GIT_SVN) $parent\n";
3163 sys(qw/git-read-tree/, $parent);
3136 - return libsvn_fetch($parent, $paths, $rev,
3164 + # I can't seem to get do_switch() to work correctly with
3165 + # the SWIG interface (TypeError when passing switch_url...),
3166 + # so we'll unconditionally bypass the delta interface here
3167 + # for now
3168 + return libsvn_fetch_full($parent, $paths, $rev,
3169 $author, $date, $msg);
3170 }
3171 print STDERR "Nope, branch point not imported or unknown\n";
@@ -3153,9 +3185,19 @@ sub libsvn_new_tree {
3185 return $log_entry;
3186 }
3187 my ($paths, $rev, $author, $date, $msg) = @_;
3156 - open my $gui, '| git-update-index -z --index-info' or croak $!;
3157 - libsvn_traverse($gui, '', $SVN->{svn_path}, $rev);
3158 - close $gui or croak $?;
3188 + if ($_xfer_delta) {
3189 + my $pool = SVN::Pool->new;
3190 + my $ed = SVN::Git::Fetcher->new({paths => $paths, ra => $SVN});
3191 + my $reporter = $SVN->do_update($rev, '', 1, $ed, $pool);
3192 + my @lock = $SVN::Core::VERSION ge '1.2.0' ? (undef) : ();
3193 + $reporter->set_path('', $rev, 1, @lock, $pool);
3194 + $reporter->finish_report($pool);
3195 + $pool->clear;
3196 + } else {
3197 + open my $gui, '| git-update-index -z --index-info' or croak $!;
3198 + libsvn_traverse($gui, '', $SVN->{svn_path}, $rev);
3199 + close $gui or croak $?;
3200 + }
3201 return libsvn_log_entry($rev, $author, $date, $msg);
3202 }
3203
@@ -3325,6 +3367,148 @@ sub copy_remote_ref {
3367 "refs/remotes/$GIT_SVN on $origin\n";
3368 }
3369 }
3370 +package SVN::Git::Fetcher;
3371 +use vars qw/@ISA/;
3372 +use strict;
3373 +use warnings;
3374 +use Carp qw/croak/;
3375 +use IO::File qw//;
3376 +
3377 +# file baton members: path, mode_a, mode_b, pool, fh, blob, base
3378 +sub new {
3379 + my ($class, $git_svn) = @_;
3380 + my $self = SVN::Delta::Editor->new;
3381 + bless $self, $class;
3382 + open my $gui, '| git-update-index -z --index-info' or croak $!;
3383 + $self->{gui} = $gui;
3384 + $self->{c} = $git_svn->{c} if exists $git_svn->{c};
3385 + if (my $p = $git_svn->{paths} && $git_svn->{ra}) {
3386 + my $s = $git_svn->{ra}->{svn_path};
3387 + $s = length $s ? qr#^/\Q$s\E/# : qr#^/#;
3388 + $self->{paths} = { map { my $x = $_;
3389 + $x =~ s/$s//;
3390 + $x => $p->{$_} } keys %$p };
3391 + }
3392 + require Digest::MD5;
3393 + $self;
3394 +}
3395 +
3396 +sub delete_entry {
3397 + my ($self, $path, $rev, $pb) = @_;
3398 + process_rm($self->{gui}, $self->{c}, $path);
3399 + undef;
3400 +}
3401 +
3402 +sub open_file {
3403 + my ($self, $path, $pb, $rev) = @_;
3404 + my ($mode, $blob) = (safe_qx('git-ls-tree',$self->{c},'--',$path)
3405 + =~ /^(\d{6}) blob ([a-f\d]{40})\t/);
3406 + { path => $path, mode_a => $mode, mode_b => $mode, blob => $blob,
3407 + pool => SVN::Pool->new };
3408 +}
3409 +
3410 +sub add_file {
3411 + my ($self, $path, $pb, $cp_path, $cp_rev) = @_;
3412 + { path => $path, mode_a => 100644, mode_b => 100644,
3413 + pool => SVN::Pool->new };
3414 +}
3415 +
3416 +sub change_file_prop {
3417 + my ($self, $fb, $prop, $value) = @_;
3418 + if ($prop eq 'svn:executable') {
3419 + if ($fb->{mode_b} != 120000) {
3420 + $fb->{mode_b} = defined $value ? 100755 : 100644;
3421 + }
3422 + } elsif ($prop eq 'svn:special') {
3423 + $fb->{mode_b} = defined $value ? 120000 : 100644;
3424 + }
3425 + undef;
3426 +}
3427 +
3428 +sub apply_textdelta {
3429 + my ($self, $fb, $exp) = @_;
3430 + my $fh = IO::File->new_tmpfile;
3431 + $fh->autoflush(1);
3432 + # $fh gets auto-closed() by SVN::TxDelta::apply(),
3433 + # (but $base does not,) so dup() it for reading in close_file
3434 + open my $dup, '<&', $fh or croak $!;
3435 + my $base = IO::File->new_tmpfile;
3436 + $base->autoflush(1);
3437 + if ($fb->{blob}) {
3438 + defined (my $pid = fork) or croak $!;
3439 + if (!$pid) {
3440 + open STDOUT, '>&', $base or croak $!;
3441 + print STDOUT 'link ' if ($fb->{mode_a} == 120000);
3442 + exec qw/git-cat-file blob/, $fb->{blob} or croak $!;
3443 + }
3444 + waitpid $pid, 0;
3445 + croak $? if $?;
3446 +
3447 + if (defined $exp) {
3448 + seek $base, 0, 0 or croak $!;
3449 + my $md5 = Digest::MD5->new;
3450 + $md5->addfile($base);
3451 + my $got = $md5->hexdigest;
3452 + die "Checksum mismatch: $fb->{path} $fb->{blob}\n",
3453 + "expected: $exp\n",
3454 + " got: $got\n" if ($got ne $exp);
3455 + }
3456 + }
3457 + seek $base, 0, 0 or croak $!;
3458 + $fb->{fh} = $dup;
3459 + $fb->{base} = $base;
3460 + [ SVN::TxDelta::apply($base, $fh, undef, $fb->{path}, $fb->{pool}) ];
3461 +}
3462 +
3463 +sub close_file {
3464 + my ($self, $fb, $exp) = @_;
3465 + my $hash;
3466 + my $path = $fb->{path};
3467 + if (my $fh = $fb->{fh}) {
3468 + seek($fh, 0, 0) or croak $!;
3469 + my $md5 = Digest::MD5->new;
3470 + $md5->addfile($fh);
3471 + my $got = $md5->hexdigest;
3472 + die "Checksum mismatch: $path\n",
3473 + "expected: $exp\n got: $got\n" if ($got ne $exp);
3474 + seek($fh, 0, 0) or croak $!;
3475 + if ($fb->{mode_b} == 120000) {
3476 + read($fh, my $buf, 5) == 5 or croak $!;
3477 + $buf eq 'link ' or die "$path has mode 120000",
3478 + "but is not a link\n";
3479 + }
3480 + defined(my $pid = open my $out,'-|') or die "Can't fork: $!\n";
3481 + if (!$pid) {
3482 + open STDIN, '<&', $fh or croak $!;
3483 + exec qw/git-hash-object -w --stdin/ or croak $!;
3484 + }
3485 + chomp($hash = do { local $/; <$out> });
3486 + close $out or croak $!;
3487 + close $fh or croak $!;
3488 + $hash =~ /^[a-f\d]{40}$/ or die "not a sha1: $hash\n";
3489 + close $fb->{base} or croak $!;
3490 + } else {
3491 + $hash = $fb->{blob} or die "no blob information\n";
3492 + }
3493 + $fb->{pool}->clear;
3494 + my $gui = $self->{gui};
3495 + print $gui "$fb->{mode_b} $hash\t$path\0" or croak $!;
3496 + print "\t", $self->{paths}->{$path}->action,
3497 + "\t$path\n" if defined $self->{paths}->{$path};
3498 + undef;
3499 +}
3500 +
3501 +sub abort_edit {
3502 + my $self = shift;
3503 + close $self->{gui};
3504 + $self->SUPER::abort_edit(@_);
3505 +}
3506 +
3507 +sub close_edit {
3508 + my $self = shift;
3509 + close $self->{gui} or croak;
3510 + $self->SUPER::close_edit(@_);
3511 +}
3512
3513 package SVN::Git::Editor;
3514 use vars qw/@ISA/;