git-svn: memoize _rev_list and rebuild
According to profile data, _rev_list and rebuild consume a large portion of time. Memoize the results of _rev_list and memoize rebuild internals to avoid subprocess invocation. When importing 15152 revisions on a LAN, time improved from 10 hours to 3-4 hours. Signed-off-by: lin zuojian <manjian2006@gmail.com> Signed-off-by: Eric Wong <normalperson@yhbt.net>
lin zuojian committed
Jan 23, 2014 at 10:15 UTC
ab0bcec9873f1fcef6c4b8825cc9e762c636ca9e
1 file changed
+38
-3
perl/Git/SVN.pm
+38
-3
@@ -1599,6 +1599,7 @@ sub tie_for_persistent_memoization {
1599
my %lookup_svn_merge_cache;
1600
my %check_cherry_pick_cache;
1601
my %has_no_changes_cache;
1602
+ my %_rev_list_cache;
1603
1604
tie_for_persistent_memoization(\%lookup_svn_merge_cache,
1605
"$cache_path/lookup_svn_merge");
@@ -1620,6 +1621,14 @@ sub tie_for_persistent_memoization {
1621
SCALAR_CACHE => ['HASH' => \%has_no_changes_cache],
1622
LIST_CACHE => 'FAULT',
1623
;
1624
+
1625
+ tie_for_persistent_memoization(\%_rev_list_cache,
1626
+ "$cache_path/_rev_list");
1627
+ memoize '_rev_list',
1628
+ SCALAR_CACHE => 'FAULT',
1629
+ LIST_CACHE => ['HASH' => \%_rev_list_cache],
1630
+ ;
1631
+
1632
}
1633
1634
sub unmemoize_svn_mergeinfo_functions {
@@ -1629,6 +1638,7 @@ sub tie_for_persistent_memoization {
1638
Memoize::unmemoize 'lookup_svn_merge';
1639
Memoize::unmemoize 'check_cherry_pick';
1640
Memoize::unmemoize 'has_no_changes';
1641
+ Memoize::unmemoize '_rev_list';
1642
}
1643
1644
sub clear_memoized_mergeinfo_caches {
@@ -1959,11 +1969,25 @@ sub rebuild_from_rev_db {
1969
unlink $path or croak "unlink: $!";
1970
}
1971
1972
+#define a global associate map to record rebuild status
1973
+my %rebuild_status;
1974
+#define a global associate map to record rebuild verify status
1975
+my %rebuild_verify_status;
1976
+
1977
sub rebuild {
1978
my ($self) = @_;
1979
my $map_path = $self->map_path;
1980
my $partial = (-e $map_path && ! -z $map_path);
1966
- return unless ::verify_ref($self->refname.'^0');
1981
+ my $verify_key = $self->refname.'^0';
1982
+ if (!$rebuild_verify_status{$verify_key}) {
1983
+ my $verify_result = ::verify_ref($verify_key);
1984
+ if ($verify_result) {
1985
+ $rebuild_verify_status{$verify_key} = 1;
1986
+ }
1987
+ }
1988
+ if (!$rebuild_verify_status{$verify_key}) {
1989
+ return;
1990
+ }
1991
if (!$partial && ($self->use_svm_props || $self->no_metadata)) {
1992
my $rev_db = $self->rev_db_path;
1993
$self->rebuild_from_rev_db($rev_db);
@@ -1977,10 +2001,21 @@ sub rebuild {
2001
print "Rebuilding $map_path ...\n" if (!$partial);
2002
my ($base_rev, $head) = ($partial ? $self->rev_map_max_norebuild(1) :
2003
(undef, undef));
2004
+ my $key_value = ($head ? "$head.." : "") . $self->refname;
2005
+ if (exists $rebuild_status{$key_value}) {
2006
+ print "Done rebuilding $map_path\n" if (!$partial || !$head);
2007
+ my $rev_db_path = $self->rev_db_path;
2008
+ if (-f $self->rev_db_path) {
2009
+ unlink $self->rev_db_path or croak "unlink: $!";
2010
+ }
2011
+ $self->unlink_rev_db_symlink;
2012
+ return;
2013
+ }
2014
my ($log, $ctx) =
1981
- command_output_pipe(qw/rev-list --pretty=raw --reverse/,
1982
- ($head ? "$head.." : "") . $self->refname,
2015
+ command_output_pipe(qw/rev-list --pretty=raw --reverse/,
2016
+ $key_value,
2017
'--');
2018
+ $rebuild_status{$key_value} = 1;
2019
my $metadata_url = $self->metadata_url;
2020
remove_username($metadata_url);
2021
my $svn_uuid = $self->rewrite_uuid || $self->ra_uuid;