@matheusbsbs / matheus-bagliano / commits / 66ab84b99c
Fork 0

git-svn: reinstate old rev_db optimization in new rev_map

This reinstates an old optimization in .rev_db which stored the highest revision number we scanned, allowing us to avoid scanning the SVN log for those revisions again in a subsequent invocation. This means the last 24-byte record in a .rev_map file can be a 4-byte SVN revision number with 20-bytes of zeroes representing a non-existent commit. This record can and will be overwritten when a new commit iff the commit is all zeroes. Signed-off-by: Eric Wong <normalperson@yhbt.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Eric Wong committed Dec 8, 2007 at 23:27 UTC 66ab84b99cdff3d650c9458aa737916735aebfcc
1 file changed +57 -14
git-svn.perl
+57 -14
@@ -1904,7 +1904,7 @@ sub last_rev_commit {
1904 ($self->{last_rev}, $self->{last_commit}) = (undef, undef);
1905 return (undef, undef);
1906 }
1907 - my ($rev, $commit) = $self->rev_map_max;
1907 + my ($rev, $commit) = $self->rev_map_max(1);
1908 ($self->{last_rev}, $self->{last_commit}) = ($rev, $commit);
1909 return ($rev, $commit);
1910 }
@@ -2534,11 +2534,16 @@ sub rebuild {
2534 # * 4 bytes for the integer representing an SVN revision number
2535 # * 20 bytes representing the sha1 of a git commit
2536 # - No empty padding records like the old format
2537 +# (except the last record, which can be overwritten)
2538 # - new records are written append-only since SVN revision numbers
2539 # increase monotonically
2540 # - lookups on SVN revision number are done via a binary search
2540 -# - Piping the file to xxd(1) -c24 is a good way of dumping it for
2541 -# viewing or editing, should the need ever arise.
2541 +# - Piping the file to xxd -c24 is a good way of dumping it for
2542 +# viewing or editing (piped back through xxd -r), should the need
2543 +# ever arise.
2544 +# - The last record can be padding revision with an all-zero sha1
2545 +# This is used to optimize fetch performance when using multiple
2546 +# "fetch" directives in .git/config
2547 #
2548 # These files are disposable unless noMetadata or useSvmProps is set
2549
@@ -2548,16 +2553,32 @@ sub _rev_map_set {
2553 my $size = (stat($fh))[7];
2554 ($size % 24) == 0 or croak "inconsistent size: $size";
2555
2556 + my $wr_offset = 0;
2557 if ($size > 0) {
2558 sysseek($fh, -24, SEEK_END) or croak "seek: $!";
2559 my $read = sysread($fh, my $buf, 24) or croak "read: $!";
2560 $read == 24 or croak "read only $read bytes (!= 24)";
2561 my ($last_rev, $last_commit) = unpack(rev_map_fmt, $buf);
2556 - if ($last_rev >= $rev) {
2557 - croak "last_rev is higher!: $last_rev >= $rev";
2562 + if ($last_commit eq ('0' x40)) {
2563 + if ($size >= 48) {
2564 + sysseek($fh, -48, SEEK_END) or croak "seek: $!";
2565 + $read = sysread($fh, $buf, 24) or
2566 + croak "read: $!";
2567 + $read == 24 or
2568 + croak "read only $read bytes (!= 24)";
2569 + ($last_rev, $last_commit) =
2570 + unpack(rev_map_fmt, $buf);
2571 + if ($last_commit eq ('0' x40)) {
2572 + croak "inconsistent .rev_map\n";
2573 + }
2574 + }
2575 + if ($last_rev >= $rev) {
2576 + croak "last_rev is higher!: $last_rev >= $rev";
2577 + }
2578 + $wr_offset = -24;
2579 }
2580 }
2560 - sysseek($fh, 0, SEEK_END) or croak "seek: $!";
2581 + sysseek($fh, $wr_offset, SEEK_END) or croak "seek: $!";
2582 syswrite($fh, pack(rev_map_fmt, $rev, $commit), 24) == 24 or
2583 croak "write: $!";
2584 }
@@ -2599,7 +2620,7 @@ sub rev_map_set {
2620 "$db => $db_lock ($!)\n";
2621 }
2622
2602 - sysopen(my $fh, $db_lock, O_RDWR | O_APPEND | O_CREAT)
2623 + sysopen(my $fh, $db_lock, O_RDWR | O_CREAT)
2624 or croak "Couldn't open $db_lock: $!\n";
2625 _rev_map_set($fh, $rev, $commit);
2626 if ($sync) {
@@ -2622,25 +2643,40 @@ sub rev_map_set {
2643 }
2644 }
2645
2646 +# If want_commit, this will return an array of (rev, commit) where
2647 +# commit _must_ be a valid commit in the archive.
2648 +# Otherwise, it'll return the max revision (whether or not the
2649 +# commit is valid or just a 0x40 placeholder).
2650 sub rev_map_max {
2626 - my ($self) = @_;
2651 + my ($self, $want_commit) = @_;
2652 $self->rebuild;
2653 my $map_path = $self->map_path;
2629 - stat $map_path or return wantarray ? (0, undef) : 0;
2654 + stat $map_path or return $want_commit ? (0, undef) : 0;
2655 sysopen(my $fh, $map_path, O_RDONLY) or croak "open: $!";
2656 my $size = (stat($fh))[7];
2657 ($size % 24) == 0 or croak "inconsistent size: $size";
2658
2659 if ($size == 0) {
2660 close $fh or croak "close: $!";
2636 - return wantarray ? (0, undef) : 0;
2661 + return $want_commit ? (0, undef) : 0;
2662 }
2663
2639 - sysseek($fh, -24, SEEK_END);
2664 + sysseek($fh, -24, SEEK_END) or croak "seek: $!";
2665 sysread($fh, my $buf, 24) == 24 or croak "read: $!";
2641 - close $fh or croak "close: $!";
2666 my ($r, $c) = unpack(rev_map_fmt, $buf);
2643 - wantarray ? ($r, $c) : $r;
2667 + if ($want_commit && $c eq ('0' x40)) {
2668 + if ($size < 48) {
2669 + return $want_commit ? (0, undef) : 0;
2670 + }
2671 + sysseek($fh, -48, SEEK_END) or croak "seek: $!";
2672 + sysread($fh, $buf, 24) == 24 or croak "read: $!";
2673 + ($r, $c) = unpack(rev_map_fmt, $buf);
2674 + if ($c eq ('0'x40)) {
2675 + croak "Penultimate record is all-zeroes in $map_path";
2676 + }
2677 + }
2678 + close $fh or croak "close: $!";
2679 + $want_commit ? ($r, $c) : $r;
2680 }
2681
2682 sub rev_map_get {
@@ -2672,7 +2708,7 @@ sub rev_map_get {
2708 $u = $i - 24;
2709 } else { # $r == $rev
2710 close($fh) or croak "close: $!";
2675 - return $c;
2711 + return $c eq ('0' x 40) ? undef : $c;
2712 }
2713 }
2714 close($fh) or croak "close: $!";
@@ -3863,6 +3899,13 @@ sub gs_fetch_loop_common {
3899 $ra_invalid = undef;
3900 }
3901 }
3902 + # pre-fill the .rev_db since it'll eventually get filled in
3903 + # with '0' x40 if something new gets committed
3904 + foreach my $gs (@$gsv) {
3905 + next if $gs->rev_map_max >= $max;
3906 + next if defined $gs->rev_map_get($max);
3907 + $gs->rev_map_set($max, 0 x40);
3908 + }
3909 foreach my $g (@$globs) {
3910 my $k = "svn-remote.$g->{remote}.$g->{t}-maxRev";
3911 Git::SVN::tmp_config($k, $max);