529
"$head history\n";
530
}
531
my $desired_revision = substr($revision_or_hash, 1);
532
- $result = $gs->rev_db_get($desired_revision);
532
+ $result = $gs->rev_map_get($desired_revision);
533
} else {
534
my (undef, $rev, undef) = cmt_metadata($revision_or_hash);
535
$result = $rev;
1128
if (defined $url && defined $rev) {
1129
next if $max{$url} and $max{$url} < $rev;
1130
if (my $gs = Git::SVN->find_by_url($url)) {
1131
- my $c = $gs->rev_db_get($rev);
1131
+ my $c = $gs->rev_map_get($rev);
1132
if ($c && $c eq $hash) {
1133
close $fh; # break the pipe
1134
return ($url, $rev, $uuid, $gs);
1135
} else {
1136
- $max{$url} ||= $gs->rev_db_max;
1136
+ $max{$url} ||= $gs->rev_map_max;
1137
}
1138
}
1139
}
1234
package Git::SVN;
1235
use strict;
1236
use warnings;
1237
+use Fcntl qw/:DEFAULT :seek/;
1238
+use constant rev_map_fmt => 'NH40';
1239
use vars qw/$default_repo_id $default_ref_id $_no_metadata $_follow_parent
1240
$_repack $_repack_flags $_use_svm_props $_head
1241
$_use_svnsync_props $no_reuse_existing $_minimize_url
1364
if ($fetch) {
1365
foreach my $p (sort keys %$fetch) {
1366
my $gs = Git::SVN->new($fetch->{$p}, $repo_id, $p);
1365
- my $lr = $gs->rev_db_max;
1367
+ my $lr = $gs->rev_map_max;
1368
if (defined $lr) {
1369
$base = $lr if ($lr < $base);
1370
}
1899
return ($rev, $c);
1900
}
1901
}
1900
- my $db_path = $self->db_path;
1901
- unless (-e $db_path) {
1902
+ my $map_path = $self->map_path;
1903
+ unless (-e $map_path) {
1904
($self->{last_rev}, $self->{last_commit}) = (undef, undef);
1905
return (undef, undef);
1906
}
1905
- my $offset = -41; # from tail
1906
- my $rl;
1907
- open my $fh, '<', $db_path or croak "$db_path not readable: $!\n";
1908
- sysseek($fh, $offset, 2); # don't care for errors
1909
- sysread($fh, $rl, 41) == 41 or return (undef, undef);
1910
- chomp $rl;
1911
- while (('0' x40) eq $rl && sysseek($fh, 0, 1) != 0) {
1912
- $offset -= 41;
1913
- sysseek($fh, $offset, 2); # don't care for errors
1914
- sysread($fh, $rl, 41) == 41 or return (undef, undef);
1915
- chomp $rl;
1916
- }
1917
- if ($c && $c ne $rl) {
1918
- die "$db_path and ", $self->refname,
1919
- " inconsistent!:\n$c != $rl\n";
1920
- }
1921
- my $rev = sysseek($fh, 0, 1) or croak $!;
1922
- $rev = ($rev - 41) / 41;
1923
- close $fh or croak $!;
1924
- ($self->{last_rev}, $self->{last_commit}) = ($rev, $c);
1925
- return ($rev, $c);
1907
+ my ($rev, $commit) = $self->rev_map_max;
1908
+ ($self->{last_rev}, $self->{last_commit}) = ($rev, $commit);
1909
+ return ($rev, $commit);
1910
}
1911
1912
sub get_fetch_range {
1913
my ($self, $min, $max) = @_;
1914
$max ||= $self->ra->get_latest_revnum;
1931
- $min ||= $self->rev_db_max;
1915
+ $min ||= $self->rev_map_max;
1916
(++$min, $max);
1917
}
1918
2057
" was r$lr, but we are about to fetch: ",
2058
"r$log_entry->{revision}!\n";
2059
}
2076
- if (my $c = $self->rev_db_get($log_entry->{revision})) {
2060
+ if (my $c = $self->rev_map_get($log_entry->{revision})) {
2061
croak "$log_entry->{revision} = $c already exists! ",
2062
"Why are we refetching it?\n";
2063
}
2100
die "Failed to commit, invalid sha1: $commit\n";
2101
}
2102
2119
- $self->rev_db_set($log_entry->{revision}, $commit, 1);
2103
+ $self->rev_map_set($log_entry->{revision}, $commit, 1);
2104
2105
$self->{last_rev} = $log_entry->{revision};
2106
$self->{last_commit} = $commit;
2107
print "r$log_entry->{revision}";
2108
if (defined $log_entry->{svm_revision}) {
2109
print " (\@$log_entry->{svm_revision})";
2126
- $self->rev_db_set($log_entry->{svm_revision}, $commit,
2110
+ $self->rev_map_set($log_entry->{svm_revision}, $commit,
2111
0, $self->svm_uuid);
2112
}
2113
print " = $commit ($self->{ref_id})\n";
2449
}
2450
}
2451
2452
+sub rebuild_from_rev_db {
2453
+ my ($self, $path) = @_;
2454
+ my $r = -1;
2455
+ open my $fh, '<', $path or croak "open: $!";
2456
+ while (<$fh>) {
2457
+ length($_) == 41 or croak "inconsistent size in ($_) != 41";
2458
+ chomp($_);
2459
+ ++$r;
2460
+ next if $_ eq ('0' x 40);
2461
+ $self->rev_map_set($r, $_);
2462
+ print "r$r = $_\n";
2463
+ }
2464
+ close $fh or croak "close: $!";
2465
+ unlink $path or croak "unlink: $!";
2466
+}
2467
+
2468
sub rebuild {
2469
my ($self) = @_;
2470
- my $db_path = $self->db_path;
2471
- return if (-e $db_path && ! -z $db_path);
2470
+ my $map_path = $self->map_path;
2471
+ return if (-e $map_path && ! -z $map_path);
2472
return unless ::verify_ref($self->refname.'^0');
2473
- if (-f $self->{db_root}) {
2474
- rename $self->{db_root}, $db_path or die
2475
- "rename $self->{db_root} => $db_path failed: $!\n";
2476
- my ($dir, $base) = ($db_path =~ m#^(.*?)/?([^/]+)$#);
2477
- symlink $base, $self->{db_root} or die
2478
- "symlink $base => $self->{db_root} failed: $!\n";
2473
+ if ($self->use_svm_props || $self->no_metadata) {
2474
+ my $rev_db = $self->rev_db_path;
2475
+ $self->rebuild_from_rev_db($rev_db);
2476
+ if ($self->use_svm_props) {
2477
+ my $svm_rev_db = $self->rev_db_path($self->svm_uuid);
2478
+ $self->rebuild_from_rev_db($svm_rev_db);
2479
+ }
2480
+ $self->unlink_rev_db_symlink;
2481
return;
2482
}
2481
- print "Rebuilding $db_path ...\n";
2482
- my ($log, $ctx) = command_output_pipe("log", '--no-color', $self->refname);
2483
- my $latest;
2483
+ print "Rebuilding $map_path ...\n";
2484
+ my ($log, $ctx) =
2485
+ command_output_pipe(qw/rev-list --pretty=raw --no-color --reverse/,
2486
+ $self->refname, '--');
2487
my $full_url = $self->full_url;
2488
remove_username($full_url);
2486
- my $svn_uuid;
2489
+ my $svn_uuid = $self->ra_uuid;
2490
my $c;
2491
while (<$log>) {
2492
if ( m{^commit ($::sha1)$} ) {
2502
2503
# if we merged or otherwise started elsewhere, this is
2504
# how we break out of it
2502
- if ((defined $svn_uuid && ($uuid ne $svn_uuid)) ||
2505
+ if (($uuid ne $svn_uuid) ||
2506
($full_url && $url && ($url ne $full_url))) {
2507
next;
2508
}
2506
- $latest ||= $rev;
2507
- $svn_uuid ||= $uuid;
2509
2509
- $self->rev_db_set($rev, $c);
2510
+ $self->rev_map_set($rev, $c);
2511
print "r$rev = $c\n";
2512
}
2513
command_close_pipe($log, $ctx);
2513
- print "Done rebuilding $db_path\n";
2514
+ print "Done rebuilding $map_path\n";
2515
+ my $rev_db_path = $self->rev_db_path;
2516
+ if (-f $self->rev_db_path) {
2517
+ unlink $self->rev_db_path or croak "unlink: $!";
2518
+ }
2519
+ $self->unlink_rev_db_symlink;
2520
}
2521
2516
-# rev_db:
2522
+# rev_map:
2523
# Tie::File seems to be prone to offset errors if revisions get sparse,
2524
# it's not that fast, either. Tie::File is also not in Perl 5.6. So
2525
# one of my favorite modules is out :< Next up would be one of the DBM
2520
-# modules, but I'm not sure which is most portable... So I'll just
2521
-# go with something that's plain-text, but still capable of
2522
-# being randomly accessed. So here's my ultra-simple fixed-width
2523
-# database. All records are 40 characters + "\n", so it's easy to seek
2524
-# to a revision: (41 * rev) is the byte offset.
2525
-# A record of 40 0s denotes an empty revision.
2526
-# And yes, it's still pretty fast (faster than Tie::File).
2526
+# modules, but I'm not sure which is most portable...
2527
+#
2528
+# This is the replacement for the rev_db format, which was too big
2529
+# and inefficient for large repositories with a lot of sparse history
2530
+# (mainly tags)
2531
+#
2532
+# The format is this:
2533
+# - 24 bytes for every record,
2534
+# * 4 bytes for the integer representing an SVN revision number
2535
+# * 20 bytes representing the sha1 of a git commit
2536
+# - No empty padding records like the old format
2537
+# - new records are written append-only since SVN revision numbers
2538
+# increase monotonically
2539
+# - lookups on SVN revision number are done via a binary search
2540
+# - Piping the file to xxd(1) -c24 is a good way of dumping it for
2541
+# viewing or editing, should the need ever arise.
2542
+#
2543
# These files are disposable unless noMetadata or useSvmProps is set
2544
2529
-sub _rev_db_set {
2545
+sub _rev_map_set {
2546
my ($fh, $rev, $commit) = @_;
2531
- my $offset = $rev * 41;
2532
- # assume that append is the common case:
2533
- seek $fh, 0, 2 or croak $!;
2534
- my $pos = tell $fh;
2535
- if ($pos < $offset) {
2536
- for (1 .. (($offset - $pos) / 41)) {
2537
- print $fh (('0' x 40),"\n") or croak $!;
2547
+
2548
+ my $size = (stat($fh))[7];
2549
+ ($size % 24) == 0 or croak "inconsistent size: $size";
2550
+
2551
+ if ($size > 0) {
2552
+ sysseek($fh, -24, SEEK_END) or croak "seek: $!";
2553
+ my $read = sysread($fh, my $buf, 24) or croak "read: $!";
2554
+ $read == 24 or croak "read only $read bytes (!= 24)";
2555
+ my ($last_rev, $last_commit) = unpack(rev_map_fmt, $buf);
2556
+ if ($last_rev >= $rev) {
2557
+ croak "last_rev is higher!: $last_rev >= $rev";
2558
}
2559
}
2540
- seek $fh, $offset, 0 or croak $!;
2541
- print $fh $commit,"\n" or croak $!;
2560
+ sysseek($fh, 0, SEEK_END) or croak "seek: $!";
2561
+ syswrite($fh, pack(rev_map_fmt, $rev, $commit), 24) == 24 or
2562
+ croak "write: $!";
2563
}
2564
2565
sub mkfile {
2572
}
2573
}
2574
2554
-sub rev_db_set {
2575
+sub rev_map_set {
2576
my ($self, $rev, $commit, $update_ref, $uuid) = @_;
2577
length $commit == 40 or die "arg3 must be a full SHA1 hexsum\n";
2557
- my $db = $self->db_path($uuid);
2578
+ my $db = $self->map_path($uuid);
2579
my $db_lock = "$db.lock";
2580
my $sig;
2581
if ($update_ref) {
2590
# and we can't afford to lose it because rebuild() won't work
2591
if ($self->use_svm_props || $self->no_metadata) {
2592
$sync = 1;
2572
- copy($db, $db_lock) or die "rev_db_set(@_): ",
2593
+ copy($db, $db_lock) or die "rev_map_set(@_): ",
2594
"Failed to copy: ",
2595
"$db => $db_lock ($!)\n";
2596
} else {
2576
- rename $db, $db_lock or die "rev_db_set(@_): ",
2597
+ rename $db, $db_lock or die "rev_map_set(@_): ",
2598
"Failed to rename: ",
2599
"$db => $db_lock ($!)\n";
2600
}
2580
- open my $fh, '+<', $db_lock or die "Couldn't open $db_lock: $!\n";
2581
- _rev_db_set($fh, $rev, $commit);
2601
+
2602
+ sysopen(my $fh, $db_lock, O_RDWR | O_APPEND | O_CREAT)
2603
+ or croak "Couldn't open $db_lock: $!\n";
2604
+ _rev_map_set($fh, $rev, $commit);
2605
if ($sync) {
2606
$fh->flush or die "Couldn't flush $db_lock: $!\n";
2607
$fh->sync or die "Couldn't sync $db_lock: $!\n";
2612
command_noisy('update-ref', '-m', "r$rev",
2613
$self->refname, $commit);
2614
}
2592
- rename $db_lock, $db or die "rev_db_set(@_): ", "Failed to rename: ",
2615
+ rename $db_lock, $db or die "rev_map_set(@_): ", "Failed to rename: ",
2616
"$db_lock => $db ($!)\n";
2617
delete $LOCKFILES{$db_lock};
2618
if ($update_ref) {
2622
}
2623
}
2624
2602
-sub rev_db_max {
2625
+sub rev_map_max {
2626
my ($self) = @_;
2627
$self->rebuild;
2605
- my $db_path = $self->db_path;
2606
- my @stat = stat $db_path or return 0;
2607
- ($stat[7] % 41) == 0 or die "$db_path inconsistent size: $stat[7]\n";
2608
- my $max = $stat[7] / 41;
2609
- (($max > 0) ? $max - 1 : 0);
2628
+ my $map_path = $self->map_path;
2629
+ stat $map_path or return wantarray ? (0, undef) : 0;
2630
+ sysopen(my $fh, $map_path, O_RDONLY) or croak "open: $!";
2631
+ my $size = (stat($fh))[7];
2632
+ ($size % 24) == 0 or croak "inconsistent size: $size";
2633
+
2634
+ if ($size == 0) {
2635
+ close $fh or croak "close: $!";
2636
+ return wantarray ? (0, undef) : 0;
2637
+ }
2638
+
2639
+ sysseek($fh, -24, SEEK_END);
2640
+ sysread($fh, my $buf, 24) == 24 or croak "read: $!";
2641
+ close $fh or croak "close: $!";
2642
+ my ($r, $c) = unpack(rev_map_fmt, $buf);
2643
+ wantarray ? ($r, $c) : $r;
2644
}
2645
2612
-sub rev_db_get {
2646
+sub rev_map_get {
2647
my ($self, $rev, $uuid) = @_;
2614
- my $ret;
2615
- my $offset = $rev * 41;
2616
- my $db_path = $self->db_path($uuid);
2617
- return undef unless -e $db_path;
2618
- open my $fh, '<', $db_path or croak $!;
2619
- if (sysseek($fh, $offset, 0) == $offset) {
2620
- my $read = sysread($fh, $ret, 40);
2621
- $ret = undef if ($read != 40 || $ret eq ('0'x40));
2648
+ my $map_path = $self->map_path($uuid);
2649
+ return undef unless -e $map_path;
2650
+
2651
+ sysopen(my $fh, $map_path, O_RDONLY) or croak "open: $!";
2652
+ my $size = (stat($fh))[7];
2653
+ ($size % 24) == 0 or croak "inconsistent size: $size";
2654
+
2655
+ if ($size == 0) {
2656
+ close $fh or croak "close: $fh";
2657
+ return undef;
2658
}
2623
- close $fh or croak $!;
2624
- $ret;
2659
+
2660
+ my ($l, $u) = (0, $size - 24);
2661
+ my ($r, $c, $buf);
2662
+
2663
+ while ($l <= $u) {
2664
+ my $i = int(($l/24 + $u/24) / 2) * 24;
2665
+ sysseek($fh, $i, SEEK_SET) or croak "seek: $!";
2666
+ sysread($fh, my $buf, 24) == 24 or croak "read: $!";
2667
+ my ($r, $c) = unpack('NH40', $buf);
2668
+
2669
+ if ($r < $rev) {
2670
+ $l = $i + 24;
2671
+ } elsif ($r > $rev) {
2672
+ $u = $i - 24;
2673
+ } else { # $r == $rev
2674
+ close($fh) or croak "close: $!";
2675
+ return $c;
2676
+ }
2677
+ }
2678
+ close($fh) or croak "close: $!";
2679
+ undef;
2680
}
2681
2682
# Finds the first svn revision that exists on (if $eq_ok is true) or
2688
--$rev unless $eq_ok;
2689
$min_rev ||= 1;
2690
while ($rev >= $min_rev) {
2636
- if (my $c = $self->rev_db_get($rev)) {
2691
+ if (my $c = $self->rev_map_get($rev)) {
2692
return ($rev, $c);
2693
}
2694
--$rev;
2703
sub find_rev_after {
2704
my ($self, $rev, $eq_ok, $max_rev) = @_;
2705
++$rev unless $eq_ok;
2651
- $max_rev ||= $self->rev_db_max();
2706
+ $max_rev ||= $self->rev_map_max;
2707
while ($rev <= $max_rev) {
2653
- if (my $c = $self->rev_db_get($rev)) {
2708
+ if (my $c = $self->rev_map_get($rev)) {
2709
return ($rev, $c);
2710
}
2711
++$rev;
2728
bless {
2729
ref_id => $ref_id, dir => $dir, index => "$dir/index",
2730
path => $path, config => "$ENV{GIT_DIR}/svn/config",
2676
- db_root => "$dir/.rev_db", repo_id => $repo_id }, $class;
2731
+ map_root => "$dir/.rev_map", repo_id => $repo_id }, $class;
2732
}
2733
2679
-sub db_path {
2734
+# for read-only access of old .rev_db formats
2735
+sub unlink_rev_db_symlink {
2736
+ my ($self) = @_;
2737
+ my $link = $self->rev_db_path;
2738
+ $link =~ s/\.[\w-]+$// or croak "missing UUID at the end of $link";
2739
+ if (-l $link) {
2740
+ unlink $link or croak "unlink: $link failed!";
2741
+ }
2742
+}
2743
+
2744
+sub rev_db_path {
2745
+ my ($self, $uuid) = @_;
2746
+ my $db_path = $self->map_path($uuid);
2747
+ $db_path =~ s{/\.rev_map\.}{/\.rev_db\.}
2748
+ or croak "map_path: $db_path does not contain '/.rev_map.' !";
2749
+ $db_path;
2750
+}
2751
+
2752
+# the new replacement for .rev_db
2753
+sub map_path {
2754
my ($self, $uuid) = @_;
2755
$uuid ||= $self->ra_uuid;
2682
- "$self->{db_root}.$uuid";
2756
+ "$self->{map_root}.$uuid";
2757
}
2758
2759
sub uri_encode {
3837
3838
foreach my $gs ($self->match_globs(\%exists, $paths,
3839
$globs, $r)) {
3766
- if ($gs->rev_db_max >= $r) {
3840
+ if ($gs->rev_map_max >= $r) {
3841
next;
3842
}
3843
next unless $gs->match_paths($paths, $r);
3863
$ra_invalid = undef;
3864
}
3865
}
3792
- # pre-fill the .rev_db since it'll eventually get filled in
3793
- # with '0' x40 if something new gets committed
3794
- foreach my $gs (@$gsv) {
3795
- next if defined $gs->rev_db_get($max);
3796
- $gs->rev_db_set($max, 0 x40);
3797
- }
3866
foreach my $g (@$globs) {
3867
my $k = "svn-remote.$g->{remote}.$g->{t}-maxRev";
3868
Git::SVN::tmp_config($k, $max);
4098
push @cmd, @log_opts;
4099
if (defined $r_max && $r_max == $r_min) {
4100
push @cmd, '--max-count=1';
4033
- if (my $c = $gs->rev_db_get($r_max)) {
4101
+ if (my $c = $gs->rev_map_get($r_max)) {
4102
push @cmd, $c;
4103
}
4104
} elsif (defined $r_max) {
4379
# --use-separate-remotes option in git-clone (now default)
4380
# - we do not automatically migrate to this (following
4381
# the example set by core git)
4382
+#
4383
+# v5 layout: .rev_db.$UUID => .rev_map.$UUID
4384
+# - newer, more-efficient format that uses 24-bytes per record
4385
+# with no filler space.
4386
+# - use xxd -c24 < .rev_map.$UUID to view and debug
4387
+# - This is a one-way migration, repositories updated to the
4388
+# new format will not be able to use old git-svn without
4389
+# rebuilding the .rev_db. Rebuilding the rev_db is not
4390
+# possible if noMetadata or useSvmProps are set; but should
4391
+# be no problem for users that use the (sensible) defaults.
4392
use strict;
4393
use warnings;
4394
use Carp qw/croak/;