git-remote-mediawiki: import "File:" attachments
Add the symmetrical feature to the "File:" export support in the previous patch. Download files from the wiki as needed, and feed them into the fast-import stream. Import both the file itself, and the corresponding description page. Signed-off-by: Pavel Volek <Pavel.Volek@ensimag.imag.fr> Signed-off-by: NGUYEN Kim Thuat <Kim-Thuat.Nguyen@ensimag.imag.fr> Signed-off-by: ROUCHER IGLESIAS Javier <roucherj@ensimag.imag.fr> Signed-off-by: Matthieu Moy <Matthieu.Moy@imag.fr> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Pavel Volek committed
Jun 27, 2012 at 16:21 UTC
6a9e55b0fc5df408c4399b56820aab4db3cadeda
1 file changed
+249
-8
contrib/mw-to-git/git-remote-mediawiki
+249
-8
@@ -13,9 +13,6 @@
13
#
14
# Known limitations:
15
#
16
-# - Only wiki pages are managed, no support for [[File:...]]
17
-# attachments.
18
-#
16
# - Poor performance in the best case: it takes forever to check
17
# whether we're up-to-date (on fetch or push) or to fetch a few
18
# revisions from a large wiki, because we use exclusively a
@@ -36,6 +33,7 @@
33
use strict;
34
use MediaWiki::API;
35
use DateTime::Format::ISO8601;
36
+use FileHandle;
37
38
# By default, use UTF-8 to communicate with Git and the user
39
binmode STDERR, ":utf8";
@@ -72,6 +70,11 @@ chomp(@tracked_pages);
70
my @tracked_categories = split(/[ \n]/, run_git("config --get-all remote.". $remotename .".categories"));
71
chomp(@tracked_categories);
72
73
+# Import media files too.
74
+my $import_media = run_git("config --get --bool remote.". $remotename .".mediaimport");
75
+chomp($import_media);
76
+$import_media = ($import_media eq "true");
77
+
78
my $wiki_login = run_git("config --get remote.". $remotename .".mwLogin");
79
# TODO: ideally, this should be able to read from keyboard, but we're
80
# inside a remote helper, so our stdin is connect to git, not to a
@@ -87,6 +90,9 @@ my $shallow_import = run_git("config --get --bool remote.". $remotename .".shall
90
chomp($shallow_import);
91
$shallow_import = ($shallow_import eq "true");
92
93
+# Cache for MediaWiki namespace ids.
94
+my %namespace_id;
95
+
96
# Dumb push: don't update notes and mediawiki ref to reflect the last push.
97
#
98
# Configurable with mediawiki.dumbPush, or per-remote with
@@ -363,6 +369,14 @@ sub get_mw_pages {
369
if (!$user_defined) {
370
get_mw_all_pages(\%pages);
371
}
372
+ if ($import_media) {
373
+ print STDERR "Getting media files for selected pages...\n";
374
+ if ($user_defined) {
375
+ get_linked_mediafiles(\%pages);
376
+ } else {
377
+ get_all_mediafiles(\%pages);
378
+ }
379
+ }
380
return values(%pages);
381
}
382
@@ -379,6 +393,152 @@ sub run_git {
393
}
394
395
396
+sub get_all_mediafiles {
397
+ my $pages = shift;
398
+ # Attach list of all pages for media files from the API,
399
+ # they are in a different namespace, only one namespace
400
+ # can be queried at the same moment
401
+ my $mw_pages = $mediawiki->list({
402
+ action => 'query',
403
+ list => 'allpages',
404
+ apnamespace => get_mw_namespace_id("File"),
405
+ aplimit => 'max'
406
+ });
407
+ if (!defined($mw_pages)) {
408
+ print STDERR "fatal: could not get the list of pages for media files.\n";
409
+ print STDERR "fatal: '$url' does not appear to be a mediawiki\n";
410
+ print STDERR "fatal: make sure '$url/api.php' is a valid page.\n";
411
+ exit 1;
412
+ }
413
+ foreach my $page (@{$mw_pages}) {
414
+ $pages->{$page->{title}} = $page;
415
+ }
416
+}
417
+
418
+sub get_linked_mediafiles {
419
+ my $pages = shift;
420
+ my @titles = map $_->{title}, values(%{$pages});
421
+
422
+ # The query is split in small batches because of the MW API limit of
423
+ # the number of links to be returned (500 links max).
424
+ my $batch = 10;
425
+ while (@titles) {
426
+ if ($#titles < $batch) {
427
+ $batch = $#titles;
428
+ }
429
+ my @slice = @titles[0..$batch];
430
+
431
+ # pattern 'page1|page2|...' required by the API
432
+ my $mw_titles = join('|', @slice);
433
+
434
+ # Media files could be included or linked from
435
+ # a page, get all related
436
+ my $query = {
437
+ action => 'query',
438
+ prop => 'links|images',
439
+ titles => $mw_titles,
440
+ plnamespace => get_mw_namespace_id("File"),
441
+ pllimit => 'max'
442
+ };
443
+ my $result = $mediawiki->api($query);
444
+
445
+ while (my ($id, $page) = each(%{$result->{query}->{pages}})) {
446
+ my @titles;
447
+ if (defined($page->{links})) {
448
+ my @link_titles = map $_->{title}, @{$page->{links}};
449
+ push(@titles, @link_titles);
450
+ }
451
+ if (defined($page->{images})) {
452
+ my @image_titles = map $_->{title}, @{$page->{images}};
453
+ push(@titles, @image_titles);
454
+ }
455
+ if (@titles) {
456
+ get_mw_first_pages(\@titles, \%{$pages});
457
+ }
458
+ }
459
+
460
+ @titles = @titles[($batch+1)..$#titles];
461
+ }
462
+}
463
+
464
+sub get_mw_mediafile_for_page_revision {
465
+ # Name of the file on Wiki, with the prefix.
466
+ my $mw_filename = shift;
467
+ my $timestamp = shift;
468
+ my %mediafile;
469
+
470
+ # Search if on MediaWiki exists a media file with given
471
+ # timestamp. In that case download the file.
472
+ my $query = {
473
+ action => 'query',
474
+ prop => 'imageinfo',
475
+ titles => $mw_filename,
476
+ iistart => $timestamp,
477
+ iiend => $timestamp,
478
+ iiprop => 'timestamp|archivename|url',
479
+ iilimit => 1
480
+ };
481
+ my $result = $mediawiki->api($query);
482
+
483
+ my ($fileid, $file) = each ( %{$result->{query}->{pages}} );
484
+ # If not defined it means there is no revision of the file for
485
+ # given timestamp.
486
+ if (defined($file->{imageinfo})) {
487
+ # Get real name of media file.
488
+ my $filename;
489
+ if (index($mw_filename, 'File:') == 0) {
490
+ $filename = substr $mw_filename, 5;
491
+ } else {
492
+ $filename = substr $mw_filename, 6;
493
+ }
494
+ $mediafile{title} = $filename;
495
+
496
+ my $fileinfo = pop(@{$file->{imageinfo}});
497
+ $mediafile{timestamp} = $fileinfo->{timestamp};
498
+ # If this is an old version of the file, the file has to be
499
+ # obtained from the archive. Otherwise it can be downloaded
500
+ # by MediaWiki API download() function.
501
+ if (defined($fileinfo->{archivename})) {
502
+ $mediafile{content} = download_mw_mediafile_from_archive($fileinfo->{url});
503
+ } else {
504
+ $mediafile{content} = download_mw_mediafile($mw_filename);
505
+ }
506
+ }
507
+ return %mediafile;
508
+}
509
+
510
+sub download_mw_mediafile_from_archive {
511
+ my $url = shift;
512
+ my $file;
513
+
514
+ my $ua = LWP::UserAgent->new;
515
+ my $response = $ua->get($url);
516
+ if ($response->code) {
517
+ $file = $response->decoded_content;
518
+ } else {
519
+ print STDERR "Error downloading a file from archive.\n";
520
+ }
521
+
522
+ return $file;
523
+}
524
+
525
+sub download_mw_mediafile {
526
+ my $filename = shift;
527
+
528
+ $mediawiki->{config}->{files_url} = $url;
529
+
530
+ my $file_content = $mediawiki->download( { title => $filename } );
531
+ if (!defined($file_content)) {
532
+ print STDERR "\tFile \'$filename\' could not be downloaded.\n";
533
+ exit 1;
534
+ } elsif ($file_content eq "") {
535
+ print STDERR "\tFile \'$filename\' does not exist on the wiki.\n";
536
+ exit 1;
537
+ } else {
538
+ return $file_content;
539
+ }
540
+}
541
+
542
sub get_last_local_revision {
543
# Get note regarding last mediawiki revision
544
my $note = run_git("notes --ref=$remotename/mediawiki show refs/mediawiki/$remotename/master 2>/dev/null");
@@ -482,6 +642,16 @@ sub literal_data {
642
print STDOUT "data ", bytes::length($content), "\n", $content;
643
}
644
645
+sub literal_data_raw {
646
+ # Output possibly binary content.
647
+ my ($content) = @_;
648
+ # Avoid confusion between size in bytes and in characters
649
+ utf8::downgrade($content);
650
+ binmode STDOUT, ":raw";
651
+ print STDOUT "data ", bytes::length($content), "\n", $content;
652
+ binmode STDOUT, ":utf8";
653
+}
654
+
655
sub mw_capabilities {
656
# Revisions are imported to the private namespace
657
# refs/mediawiki/$remotename/ by the helper and fetched into
@@ -569,6 +739,11 @@ sub import_file_revision {
739
my %commit = %{$commit};
740
my $full_import = shift;
741
my $n = shift;
742
+ my $mediafile = shift;
743
+ my %mediafile;
744
+ if ($mediafile) {
745
+ %mediafile = %{$mediafile};
746
+ }
747
748
my $title = $commit{title};
749
my $comment = $commit{comment};
@@ -588,6 +763,10 @@ sub import_file_revision {
763
if ($content ne DELETED_CONTENT) {
764
print STDOUT "M 644 inline $title.mw\n";
765
literal_data($content);
766
+ if (%mediafile) {
767
+ print STDOUT "M 644 inline $mediafile{title}\n";
768
+ literal_data_raw($mediafile{content});
769
+ }
770
print STDOUT "\n\n";
771
} else {
772
print STDOUT "D $title.mw\n";
@@ -683,12 +862,11 @@ sub mw_import_ref {
862
863
$n++;
864
865
+ my $page_title = $result->{query}->{pages}->{$pagerevid->{pageid}}->{title};
866
my %commit;
867
$commit{author} = $rev->{user} || 'Anonymous';
868
$commit{comment} = $rev->{comment} || '*Empty MediaWiki Message*';
689
- $commit{title} = mediawiki_smudge_filename(
690
- $result->{query}->{pages}->{$pagerevid->{pageid}}->{title}
691
- );
869
+ $commit{title} = mediawiki_smudge_filename($page_title);
870
$commit{mw_revision} = $pagerevid->{revid};
871
$commit{content} = mediawiki_smudge($rev->{'*'});
872
@@ -699,9 +877,25 @@ sub mw_import_ref {
877
}
878
$commit{date} = DateTime::Format::ISO8601->parse_datetime($last_timestamp);
879
702
- print STDERR "$n/", scalar(@revisions), ": Revision #$pagerevid->{revid} of $commit{title}\n";
880
+ # Differentiates classic pages and media files.
881
+ my @prefix = split(":", $page_title);
882
704
- import_file_revision(\%commit, ($fetch_from == 1), $n);
883
+ my %mediafile;
884
+ if ($prefix[0] eq "File" || $prefix[0] eq "Image") {
885
+ # The name of the file is the same as the media page.
886
+ my $filename = $page_title;
887
+ %mediafile = get_mw_mediafile_for_page_revision($filename, $rev->{timestamp});
888
+ }
889
+ # If this is a revision of the media page for new version
890
+ # of a file do one common commit for both file and media page.
891
+ # Else do commit only for that page.
892
+ print STDERR "$n/", scalar(@revisions), ": Revision #$pagerevid->{revid} of $commit{title}\n";
893
+ if (%mediafile) {
894
+ print STDERR "\tDownloading file $mediafile{title}, version $mediafile{timestamp}\n";
895
+ import_file_revision(\%commit, ($fetch_from == 1), $n, \%mediafile);
896
+ } else {
897
+ import_file_revision(\%commit, ($fetch_from == 1), $n);
898
+ }
899
}
900
901
if ($fetch_from == 1 && $n == 0) {
@@ -1006,3 +1200,50 @@ sub get_allowed_file_extensions {
1200
1201
return %hashFile;
1202
}
1203
+
1204
+# Return MediaWiki id for a canonical namespace name.
1205
+# Ex.: "File", "Project".
1206
+# Looks for the namespace id in the local configuration
1207
+# variables, if it is not found asks MW API.
1208
+sub get_mw_namespace_id {
1209
+ mw_connect_maybe();
1210
+ my $name = shift;
1211
+
1212
+ if (!exists $namespace_id{$name}) {
1213
+ # Look at configuration file, if the record for that namespace is
1214
+ # already stored. Namespaces are stored in form:
1215
+ # "Name_of_namespace:Id_namespace", ex.: "File:6".
1216
+ my @temp = split(/[ \n]/, run_git("config --get-all remote."
1217
+ . $remotename .".namespaces"));
1218
+ chomp(@temp);
1219
+ foreach my $ns (@temp) {
1220
+ my ($n, $s) = split(/:/, $ns);
1221
+ $namespace_id{$n} = $s;
1222
+ }
1223
+ }
1224
+
1225
+ if (!exists $namespace_id{$name}) {
1226
+ # NS not found => get namespace id from MW and store it in
1227
+ # configuration file.
1228
+ my $query = {
1229
+ action => 'query',
1230
+ meta => 'siteinfo',
1231
+ siprop => 'namespaces'
1232
+ };
1233
+ my $result = $mediawiki->api($query);
1234
+
1235
+ while (my ($id, $ns) = each(%{$result->{query}->{namespaces}})) {
1236
+ if (defined($ns->{canonical}) && ($ns->{canonical} eq $name)) {
1237
+ run_git("config --add remote.". $remotename
1238
+ .".namespaces ". $name .":". $ns->{id});
1239
+ $namespace_id{$name} = $ns->{id};
1240
+ }
1241
+ }
1242
+ }
1243
+
1244
+ if (exists $namespace_id{$name}) {
1245
+ return $namespace_id{$name};
1246
+ } else {
1247
+ die "No such namespace $name on MediaWiki.";
1248
+ }
1249
+}