git-remote-mediawiki: more efficient 'pull' in the best case
The only way to fetch new revisions from a wiki before this patch was to query each page for new revisions. This is good when tracking a small set of pages on a large wiki, but very inefficient when tracking many pages on a wiki with little activity. Implement a new strategy that queries the wiki for its last global revision, queries each new revision, and filter out pages that are not tracked. Signed-off-by: Simon Perrat <simon.perrat@ensimag.imag.fr> Signed-off-by: Simon CATHEBRAS <Simon.Cathebras@ensimag.imag.fr> Signed-off-by: Julien KHAYAT <Julien.Khayat@ensimag.imag.fr> Signed-off-by: Charles ROUSSEL <Charles.Roussel@ensimag.imag.fr> Signed-off-by: Guillaume SASDY <Guillaume.Sasdy@ensimag.imag.fr> Signed-off-by: Matthieu Moy <Matthieu.Moy@imag.fr> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Matthieu Moy committed
Jul 6, 2012 at 12:03 UTC
5a29217dda0625707a46f527d17489c0bff0c2c9
2 files changed
+110
-21
contrib/mw-to-git/git-remote-mediawiki
+93
-21
@@ -13,12 +13,9 @@
13
#
14
# Known limitations:
15
#
16
-# - Poor performance in the best case: it takes forever to check
17
-# whether we're up-to-date (on fetch or push) or to fetch a few
18
-# revisions from a large wiki, because we use exclusively a
19
-# page-based synchronization. We could switch to a wiki-wide
20
-# synchronization when the synchronization involves few revisions
21
-# but the wiki is large.
16
+# - Several strategies are provided to fetch modifications from the
17
+# wiki, but no automatic heuristics is provided, the user has
18
+# to understand and chose which strategy is appropriate for him.
19
#
20
# - Git renames could be turned into MediaWiki renames (see TODO
21
# below)
@@ -89,6 +86,21 @@ my $shallow_import = run_git("config --get --bool remote.". $remotename .".shall
86
chomp($shallow_import);
87
$shallow_import = ($shallow_import eq "true");
88
89
+# Fetch (clone and pull) by revisions instead of by pages. This behavior
90
+# is more efficient when we have a wiki with lots of pages and we fetch
91
+# the revisions quite often so that they concern only few pages.
92
+# Possible values:
93
+# - by_rev: perform one query per new revision on the remote wiki
94
+# - by_page: query each tracked page for new revision
95
+my $fetch_strategy = run_git("config --get remote.$remotename.fetchStrategy");
96
+unless ($fetch_strategy) {
97
+ $fetch_strategy = run_git("config --get mediawiki.fetchStrategy");
98
+}
99
+chomp($fetch_strategy);
100
+unless ($fetch_strategy) {
101
+ $fetch_strategy = "by_page";
102
+}
103
+
104
# Dumb push: don't update notes and mediawiki ref to reflect the last push.
105
#
106
# Configurable with mediawiki.dumbPush, or per-remote with
@@ -533,6 +545,26 @@ sub get_last_local_revision {
545
# Remember the timestamp corresponding to a revision id.
546
my %basetimestamps;
547
548
+# Get the last remote revision without taking in account which pages are
549
+# tracked or not. This function makes a single request to the wiki thus
550
+# avoid a loop onto all tracked pages. This is useful for the fetch-by-rev
551
+# option.
552
+sub get_last_global_remote_rev {
553
+ mw_connect_maybe();
554
+
555
+ my $query = {
556
+ action => 'query',
557
+ list => 'recentchanges',
558
+ prop => 'revisions',
559
+ rclimit => '1',
560
+ rcdir => 'older',
561
+ };
562
+ my $result = $mediawiki->api($query);
563
+ return $result->{query}->{recentchanges}[0]->{revid};
564
+}
565
+
566
+# Get the last remote revision concerning the tracked pages and the tracked
567
+# categories.
568
sub get_last_remote_revision {
569
mw_connect_maybe();
570
@@ -803,9 +835,6 @@ sub mw_import_ref {
835
836
mw_connect_maybe();
837
806
- my %pages_hash = get_mw_pages();
807
- my @pages = values(%pages_hash);
808
-
838
print STDERR "Searching revisions...\n";
839
my $last_local = get_last_local_revision();
840
my $fetch_from = $last_local + 1;
@@ -814,10 +843,35 @@ sub mw_import_ref {
843
} else {
844
print STDERR ", fetching from here.\n";
845
}
817
- my ($n, @revisions) = fetch_mw_revisions(\@pages, $fetch_from);
846
819
- # Creation of the fast-import stream
820
- print STDERR "Fetching & writing export data...\n";
847
+ my $n = 0;
848
+ if ($fetch_strategy eq "by_rev") {
849
+ print STDERR "Fetching & writing export data by revs...\n";
850
+ $n = mw_import_ref_by_revs($fetch_from);
851
+ } elsif ($fetch_strategy eq "by_page") {
852
+ print STDERR "Fetching & writing export data by pages...\n";
853
+ $n = mw_import_ref_by_pages($fetch_from);
854
+ } else {
855
+ print STDERR "fatal: invalid fetch strategy \"$fetch_strategy\".\n";
856
+ print STDERR "Check your configuration variables remote.$remotename.fetchStrategy and mediawiki.fetchStrategy\n";
857
+ exit 1;
858
+ }
859
+
860
+ if ($fetch_from == 1 && $n == 0) {
861
+ print STDERR "You appear to have cloned an empty MediaWiki.\n";
862
+ # Something has to be done remote-helper side. If nothing is done, an error is
863
+ # thrown saying that HEAD is refering to unknown object 0000000000000000000
864
+ # and the clone fails.
865
+ }
866
+}
867
+
868
+sub mw_import_ref_by_pages {
869
+
870
+ my $fetch_from = shift;
871
+ my %pages_hash = get_mw_pages();
872
+ my @pages = values(%pages_hash);
873
+
874
+ my ($n, @revisions) = fetch_mw_revisions(\@pages, $fetch_from);
875
876
@revisions = sort {$a->{revid} <=> $b->{revid}} @revisions;
877
my @revision_ids = map $_->{revid}, @revisions;
@@ -825,12 +879,26 @@ sub mw_import_ref {
879
return mw_import_revids($fetch_from, \@revision_ids, \%pages_hash);
880
}
881
882
+sub mw_import_ref_by_revs {
883
+
884
+ my $fetch_from = shift;
885
+ my %pages_hash = get_mw_pages();
886
+
887
+ my $last_remote = get_last_global_remote_rev();
888
+ my @revision_ids = $fetch_from..$last_remote;
889
+ return mw_import_revids($fetch_from, \@revision_ids, \%pages_hash);
890
+}
891
+
892
+# Import revisions given in second argument (array of integers).
893
+# Only pages appearing in the third argument (hash indexed by page titles)
894
+# will be imported.
895
sub mw_import_revids {
896
my $fetch_from = shift;
897
my $revision_ids = shift;
898
my $pages = shift;
899
900
my $n = 0;
901
+ my $n_actual = 0;
902
my $last_timestamp = 0; # Placeholer in case $rev->timestamp is undefined
903
904
foreach my $pagerevid (@$revision_ids) {
@@ -848,9 +916,20 @@ sub mw_import_revids {
916
my $result_page = $result_pages[0];
917
my $rev = $result_pages[0]->{revisions}->[0];
918
919
+ # Count page even if we skip it, since we display
920
+ # $n/$total and $total includes skipped pages.
921
$n++;
922
923
my $page_title = $result_page->{title};
924
+
925
+ if (!exists($pages->{$page_title})) {
926
+ print STDERR "$n/", scalar(@$revision_ids),
927
+ ": Skipping revision #$rev->{revid} of $page_title\n";
928
+ next;
929
+ }
930
+
931
+ $n_actual++;
932
+
933
my %commit;
934
$commit{author} = $rev->{user} || 'Anonymous';
935
$commit{comment} = $rev->{comment} || '*Empty MediaWiki Message*';
@@ -875,17 +954,10 @@ sub mw_import_revids {
954
# of a file do one common commit for both file and media page.
955
# Else do commit only for that page.
956
print STDERR "$n/", scalar(@$revision_ids), ": Revision #$rev->{revid} of $commit{title}\n";
878
- import_file_revision(\%commit, ($fetch_from == 1), $n, \%mediafile);
879
- }
880
-
881
- if ($fetch_from == 1 && $n == 0) {
882
- print STDERR "You appear to have cloned an empty MediaWiki.\n";
883
- # Something has to be done remote-helper side. If nothing is done, an error is
884
- # thrown saying that HEAD is refering to unknown object 0000000000000000000
885
- # and the clone fails.
957
+ import_file_revision(\%commit, ($fetch_from == 1), $n_actual, \%mediafile);
958
}
959
888
- return $n;
960
+ return $n_actual;
961
}
962
963
sub error_non_fast_forward {
contrib/mw-to-git/t/t9364-pull-by-rev.sh
new
+17
@@ -0,0 +1,17 @@
1
+#!/bin/sh
2
+
3
+test_description='Test the Git Mediawiki remote helper: git pull by revision'
4
+
5
+. ./test-gitmw-lib.sh
6
+. ./push-pull-tests.sh
7
+. $TEST_DIRECTORY/test-lib.sh
8
+
9
+test_check_precond
10
+
11
+test_expect_success 'configuration' '
12
+ git config --global mediawiki.fetchStrategy by_rev
13
+'
14
+
15
+test_push_pull
16
+
17
+test_done