git-svn: reinstate old rev_db optimization in new rev_map
This reinstates an old optimization in .rev_db which stored the highest revision number we scanned, allowing us to avoid scanning the SVN log for those revisions again in a subsequent invocation. This means the last 24-byte record in a .rev_map file can be a 4-byte SVN revision number with 20-bytes of zeroes representing a non-existent commit. This record can and will be overwritten when a new commit iff the commit is all zeroes. Signed-off-by: Eric Wong <normalperson@yhbt.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Eric Wong committed
Dec 8, 2007 at 23:27 UTC
66ab84b99cdff3d650c9458aa737916735aebfcc
1 file changed
+57
-14
git-svn.perl
+57
-14
@@ -1904,7 +1904,7 @@ sub last_rev_commit {
1904
($self->{last_rev}, $self->{last_commit}) = (undef, undef);
1905
return (undef, undef);
1906
}
1907
- my ($rev, $commit) = $self->rev_map_max;
1907
+ my ($rev, $commit) = $self->rev_map_max(1);
1908
($self->{last_rev}, $self->{last_commit}) = ($rev, $commit);
1909
return ($rev, $commit);
1910
}
@@ -2534,11 +2534,16 @@ sub rebuild {
2534
# * 4 bytes for the integer representing an SVN revision number
2535
# * 20 bytes representing the sha1 of a git commit
2536
# - No empty padding records like the old format
2537
+# (except the last record, which can be overwritten)
2538
# - new records are written append-only since SVN revision numbers
2539
# increase monotonically
2540
# - lookups on SVN revision number are done via a binary search
2540
-# - Piping the file to xxd(1) -c24 is a good way of dumping it for
2541
-# viewing or editing, should the need ever arise.
2541
+# - Piping the file to xxd -c24 is a good way of dumping it for
2542
+# viewing or editing (piped back through xxd -r), should the need
2543
+# ever arise.
2544
+# - The last record can be padding revision with an all-zero sha1
2545
+# This is used to optimize fetch performance when using multiple
2546
+# "fetch" directives in .git/config
2547
#
2548
# These files are disposable unless noMetadata or useSvmProps is set
2549
@@ -2548,16 +2553,32 @@ sub _rev_map_set {
2553
my $size = (stat($fh))[7];
2554
($size % 24) == 0 or croak "inconsistent size: $size";
2555
2556
+ my $wr_offset = 0;
2557
if ($size > 0) {
2558
sysseek($fh, -24, SEEK_END) or croak "seek: $!";
2559
my $read = sysread($fh, my $buf, 24) or croak "read: $!";
2560
$read == 24 or croak "read only $read bytes (!= 24)";
2561
my ($last_rev, $last_commit) = unpack(rev_map_fmt, $buf);
2556
- if ($last_rev >= $rev) {
2557
- croak "last_rev is higher!: $last_rev >= $rev";
2562
+ if ($last_commit eq ('0' x40)) {
2563
+ if ($size >= 48) {
2564
+ sysseek($fh, -48, SEEK_END) or croak "seek: $!";
2565
+ $read = sysread($fh, $buf, 24) or
2566
+ croak "read: $!";
2567
+ $read == 24 or
2568
+ croak "read only $read bytes (!= 24)";
2569
+ ($last_rev, $last_commit) =
2570
+ unpack(rev_map_fmt, $buf);
2571
+ if ($last_commit eq ('0' x40)) {
2572
+ croak "inconsistent .rev_map\n";
2573
+ }
2574
+ }
2575
+ if ($last_rev >= $rev) {
2576
+ croak "last_rev is higher!: $last_rev >= $rev";
2577
+ }
2578
+ $wr_offset = -24;
2579
}
2580
}
2560
- sysseek($fh, 0, SEEK_END) or croak "seek: $!";
2581
+ sysseek($fh, $wr_offset, SEEK_END) or croak "seek: $!";
2582
syswrite($fh, pack(rev_map_fmt, $rev, $commit), 24) == 24 or
2583
croak "write: $!";
2584
}
@@ -2599,7 +2620,7 @@ sub rev_map_set {
2620
"$db => $db_lock ($!)\n";
2621
}
2622
2602
- sysopen(my $fh, $db_lock, O_RDWR | O_APPEND | O_CREAT)
2623
+ sysopen(my $fh, $db_lock, O_RDWR | O_CREAT)
2624
or croak "Couldn't open $db_lock: $!\n";
2625
_rev_map_set($fh, $rev, $commit);
2626
if ($sync) {
@@ -2622,25 +2643,40 @@ sub rev_map_set {
2643
}
2644
}
2645
2646
+# If want_commit, this will return an array of (rev, commit) where
2647
+# commit _must_ be a valid commit in the archive.
2648
+# Otherwise, it'll return the max revision (whether or not the
2649
+# commit is valid or just a 0x40 placeholder).
2650
sub rev_map_max {
2626
- my ($self) = @_;
2651
+ my ($self, $want_commit) = @_;
2652
$self->rebuild;
2653
my $map_path = $self->map_path;
2629
- stat $map_path or return wantarray ? (0, undef) : 0;
2654
+ stat $map_path or return $want_commit ? (0, undef) : 0;
2655
sysopen(my $fh, $map_path, O_RDONLY) or croak "open: $!";
2656
my $size = (stat($fh))[7];
2657
($size % 24) == 0 or croak "inconsistent size: $size";
2658
2659
if ($size == 0) {
2660
close $fh or croak "close: $!";
2636
- return wantarray ? (0, undef) : 0;
2661
+ return $want_commit ? (0, undef) : 0;
2662
}
2663
2639
- sysseek($fh, -24, SEEK_END);
2664
+ sysseek($fh, -24, SEEK_END) or croak "seek: $!";
2665
sysread($fh, my $buf, 24) == 24 or croak "read: $!";
2641
- close $fh or croak "close: $!";
2666
my ($r, $c) = unpack(rev_map_fmt, $buf);
2643
- wantarray ? ($r, $c) : $r;
2667
+ if ($want_commit && $c eq ('0' x40)) {
2668
+ if ($size < 48) {
2669
+ return $want_commit ? (0, undef) : 0;
2670
+ }
2671
+ sysseek($fh, -48, SEEK_END) or croak "seek: $!";
2672
+ sysread($fh, $buf, 24) == 24 or croak "read: $!";
2673
+ ($r, $c) = unpack(rev_map_fmt, $buf);
2674
+ if ($c eq ('0'x40)) {
2675
+ croak "Penultimate record is all-zeroes in $map_path";
2676
+ }
2677
+ }
2678
+ close $fh or croak "close: $!";
2679
+ $want_commit ? ($r, $c) : $r;
2680
}
2681
2682
sub rev_map_get {
@@ -2672,7 +2708,7 @@ sub rev_map_get {
2708
$u = $i - 24;
2709
} else { # $r == $rev
2710
close($fh) or croak "close: $!";
2675
- return $c;
2711
+ return $c eq ('0' x 40) ? undef : $c;
2712
}
2713
}
2714
close($fh) or croak "close: $!";
@@ -3863,6 +3899,13 @@ sub gs_fetch_loop_common {
3899
$ra_invalid = undef;
3900
}
3901
}
3902
+ # pre-fill the .rev_db since it'll eventually get filled in
3903
+ # with '0' x40 if something new gets committed
3904
+ foreach my $gs (@$gsv) {
3905
+ next if $gs->rev_map_max >= $max;
3906
+ next if defined $gs->rev_map_get($max);
3907
+ $gs->rev_map_set($max, 0 x40);
3908
+ }
3909
foreach my $g (@$globs) {
3910
my $k = "svn-remote.$g->{remote}.$g->{t}-maxRev";
3911
Git::SVN::tmp_config($k, $max);