archimport: safer log file parsing
Better logfile parsing, no longer confused by 'headers' after the first blank line. Re-enabled tag-reading with abrowse (baz and tla compatible) Remove need to quote args to external processes Signed-off-by: Eric Wong <normalperson@yhbt.net> Signed-off-by: Martin Langhoff <martin@catalyst.net.nz>
Eric Wong committed
Nov 23, 2005 at 23:53 UTC
6df896b50a8f1294d00e8da9d3662d9422dd0533
1 file changed
+111
-98
git-archimport.perl
+111
-98
@@ -140,10 +140,10 @@ sub do_abrowse {
140
$ps{type} = 's';
141
} elsif ($type =~ /\(.*import\)/) {
142
$ps{type} = 'i';
143
- } elsif ($type =~ m/\(tag.*\)/) {
143
+ } elsif ($type =~ m/\(tag.*?(\S+\@\S+).*?\)/) {
144
$ps{type} = 't';
145
# read which revision we've tagged when we parse the log
146
- #$ps{tag} = $1;
146
+ $ps{tag} = $1;
147
} else {
148
warn "Unknown type $type";
149
}
@@ -359,78 +359,73 @@ foreach my $ps (@psets) {
359
#
360
my $tree;
361
362
- my $commitlog = safe_pipe_capture($TLA,'cat-archive-log',$ps->{id});
362
+ my @commitlog = safe_pipe_capture($TLA,'cat-archive-log',$ps->{id});
363
die "Error in cat-archive-log: $!" if $?;
364
365
- # parselog will git-add/rm files
366
- # and generally prepare things for the commit
367
- # NOTE: parselog will shell-quote filenames!
368
- my ($sum, $msg, $add, $del, $mod, $ren) = parselog($commitlog);
369
- my $logmessage = "$sum\n$msg";
370
-
365
+ parselog($ps,\@commitlog);
366
367
# imports don't give us good info
368
# on added files. Shame on them
374
- if ($ps->{type} eq 'i' || $ps->{type} eq 't') {
375
- `find . -type f -print0 | grep -zv '^./$git_dir' | xargs -0 -l100 git-update-index --add`;
376
- `git-ls-files --deleted -z | xargs --no-run-if-empty -0 -l100 git-update-index --remove`;
369
+ if ($ps->{type} eq 'i' || $ps->{type} eq 't') {
370
+ system('git-ls-files --others -z | '.
371
+ 'git-update-index --add -z --stdin') == 0 or die "$! $?\n";
372
+ system('git-ls-files --deleted -z | '.
373
+ 'git-update-index --remove -z --stdin') == 0 or die "$! $?\n";
374
}
375
379
- if (@$add) {
376
+ # TODO: handle removed_directories and renamed_directories:
377
+
378
+ if (my $add = $ps->{new_files}) {
379
while (@$add) {
380
my @slice = splice(@$add, 0, 100);
382
- my $slice = join(' ', @slice);
383
- `git-update-index --add $slice`;
384
- die "Error in git-update-index --add: $!" if $?;
381
+ system('git-update-index','--add','--',@slice) == 0 or
382
+ die "Error in git-update-index --add: $! $?\n";
383
}
384
}
387
- if (@$del) {
388
- foreach my $file (@$del) {
389
- unlink $file or die "Problems deleting $file : $!";
390
- }
385
+
386
+ if (my $del = $ps->{removed_files}) {
387
+ unlink @$del;
388
while (@$del) {
389
my @slice = splice(@$del, 0, 100);
393
- my $slice = join(' ', @slice);
394
- `git-update-index --remove $slice`;
395
- die "Error in git-update-index --remove: $!" if $?;
390
+ system('git-update-index','--remove','--',@slice) == 0 or
391
+ die "Error in git-update-index --remove: $! $?\n";
392
}
393
}
398
- if (@$ren) { # renamed
394
+
395
+ if (my $ren = $ps->{renamed_files}) { # renamed
396
if (@$ren % 2) {
397
die "Odd number of entries in rename!?";
398
}
402
- ;
399
+
400
while (@$ren) {
404
- my $from = pop @$ren;
405
- my $to = pop @$ren;
401
+ my $from = shift @$ren;
402
+ my $to = shift @$ren;
403
404
unless (-d dirname($to)) {
405
mkpath(dirname($to)); # will die on err
406
}
410
- #print "moving $from $to";
411
- `mv $from $to`;
412
- die "Error renaming $from $to : $!" if $?;
413
- `git-update-index --remove $from`;
414
- die "Error in git-update-index --remove: $!" if $?;
415
- `git-update-index --add $to`;
416
- die "Error in git-update-index --add: $!" if $?;
407
+ print "moving $from $to";
408
+ rename($from, $to) or die "Error renaming '$from' '$to': $!\n";
409
+ system('git-update-index','--remove','--',$from) == 0 or
410
+ die "Error in git-update-index --remove: $! $?\n";
411
+ system('git-update-index','--add','--',$to) == 0 or
412
+ die "Error in git-update-index --add: $! $?\n";
413
}
414
415
}
420
- if (@$mod) { # must be _after_ renames
416
+
417
+ if (my $mod = $ps->{modified_files}) {
418
while (@$mod) {
419
my @slice = splice(@$mod, 0, 100);
423
- my $slice = join(' ', @slice);
424
- `git-update-index $slice`;
425
- die "Error in git-update-index: $!" if $?;
420
+ system('git-update-index','--',@slice) == 0 or
421
+ die "Error in git-update-index: $! $?\n";
422
}
423
}
428
-
424
+
425
# warn "errors when running git-update-index! $!";
426
$tree = `git-write-tree`;
427
die "cannot write tree $!" if $?;
428
chomp $tree;
433
-
429
430
#
431
# Who's your daddy?
@@ -464,13 +459,14 @@ foreach my $ps (@psets) {
459
$ENV{GIT_COMMITTER_EMAIL} = $ps->{email};
460
$ENV{GIT_COMMITTER_DATE} = $ps->{date};
461
467
- my ($pid, $commit_rh, $commit_wh);
468
- $commit_rh = 'commit_rh';
469
- $commit_wh = 'commit_wh';
470
-
471
- $pid = open2(*READER, *WRITER,'git-commit-tree',$tree,@par)
462
+ my $pid = open2(*READER, *WRITER,'git-commit-tree',$tree,@par)
463
or die $!;
473
- print WRITER $logmessage; # write
464
+ print WRITER $ps->{summary},"\n";
465
+ print WRITER $ps->{message},"\n";
466
+
467
+ # make it easy to backtrack and figure out which Arch revision this was:
468
+ print WRITER 'git-archimport-id: ',$ps->{id},"\n";
469
+
470
close WRITER;
471
my $commitid = <READER>; # read
472
chomp $commitid;
@@ -568,7 +564,9 @@ sub apply_cset {
564
565
566
# =for reference
571
-# A log entry looks like
567
+# notes: *-files/-directories keys cannot have spaces, they're always
568
+# pika-escaped. Everything after the first newline
569
+# A log entry looks like:
570
# Revision: moodle-org--moodle--1.3.3--patch-15
571
# Archive: arch-eduforge@catalyst.net.nz--2004
572
# Creator: Penny Leach <penny@catalyst.net.nz>
@@ -586,70 +584,85 @@ sub apply_cset {
584
# admin/editor.html backup/lib.php backup/restore.php
585
# New-patches: arch-eduforge@catalyst.net.nz--2004/moodle-org--moodle--1.3.3--patch-15
586
# Summary: Updating to latest from MOODLE_14_STABLE (1.4.5+)
587
+# summary can be multiline with a leading space just like the above fields
588
# Keywords:
589
#
590
# Updating yadda tadda tadda madda
591
sub parselog {
593
- my $log = shift;
594
- #print $log;
595
-
596
- my (@add, @del, @mod, @ren, @kw, $sum, $msg );
597
-
598
- if ($log =~ m/(?:\n|^)New-files:(.*?)(?=\n\w)/s ) {
599
- my $files = $1;
600
- @add = split(m/\s+/s, $files);
601
- }
602
-
603
- if ($log =~ m/(?:\n|^)Removed-files:(.*?)(?=\n\w)/s ) {
604
- my $files = $1;
605
- @del = split(m/\s+/s, $files);
606
- }
592
+ my ($ps, $log) = @_;
593
+ my $key = undef;
594
+
595
+ # headers we want that contain filenames:
596
+ my %want_headers = (
597
+ new_files => 1,
598
+ modified_files => 1,
599
+ renamed_files => 1,
600
+ renamed_directories => 1,
601
+ removed_files => 1,
602
+ removed_directories => 1,
603
+ );
604
608
- if ($log =~ m/(?:\n|^)Modified-files:(.*?)(?=\n\w)/s ) {
609
- my $files = $1;
610
- @mod = split(m/\s+/s, $files);
605
+ chomp (@$log);
606
+ while ($_ = shift @$log) {
607
+ if (/^Continuation-of:\s*(.*)/) {
608
+ $ps->{tag} = $1;
609
+ $key = undef;
610
+ } elsif (/^Summary:\s*(.*)$/ ) {
611
+ # summary can be multiline as long as it has a leading space
612
+ $ps->{summary} = [ $1 ];
613
+ $key = 'summary';
614
+ } elsif (/^Creator: (.*)\s*<([^\>]+)>/) {
615
+ $ps->{author} = $1;
616
+ $ps->{email} = $2;
617
+ $key = undef;
618
+ # any *-files or *-directories can be read here:
619
+ } elsif (/^([A-Z][a-z\-]+):\s*(.*)$/) {
620
+ my $val = $2;
621
+ $key = lc $1;
622
+ $key =~ tr/-/_/; # too lazy to quote :P
623
+ if ($want_headers{$key}) {
624
+ push @{$ps->{$key}}, split(/\s+/, $val);
625
+ } else {
626
+ $key = undef;
627
+ }
628
+ } elsif (/^$/) {
629
+ last; # remainder of @$log that didn't get shifted off is message
630
+ } elsif ($key) {
631
+ if (/^\s+(.*)$/) {
632
+ if ($key eq 'summary') {
633
+ push @{$ps->{$key}}, $1;
634
+ } else { # files/directories:
635
+ push @{$ps->{$key}}, split(/\s+/, $1);
636
+ }
637
+ } else {
638
+ $key = undef;
639
+ }
640
+ }
641
}
642
+
643
+ # post-processing:
644
+ $ps->{summary} = join("\n",@{$ps->{summary}})."\n";
645
+ $ps->{message} = join("\n",@$log);
646
613
- if ($log =~ m/(?:\n|^)Renamed-files:(.*?)(?=\n\w)/s ) {
614
- my $files = $1;
615
- @ren = split(m/\s+/s, $files);
616
- }
617
-
618
- $sum ='';
619
- if ($log =~ m/^Summary:(.+?)$/m ) {
620
- $sum = $1;
621
- $sum =~ s/^\s+//;
622
- $sum =~ s/\s+$//;
623
- }
624
-
625
- $msg = '';
626
- if ($log =~ m/\n\n(.+)$/s) {
627
- $msg = $1;
628
- $msg =~ s/^\s+//;
629
- $msg =~ s/\s+$//;
630
- }
631
-
632
-
633
- # cleanup the arrays
634
- foreach my $ref ( (\@add, \@del, \@mod, \@ren) ) {
635
- my @tmp = ();
636
- while (my $t = pop @$ref) {
637
- next unless length ($t);
638
- next if $t =~ m!\{arch\}/!;
639
- next if $t =~ m!\.arch-ids/!;
640
- next if $t =~ m!\.arch-inventory$!;
647
+ # skip Arch control files, unescape pika-escaped files
648
+ foreach my $k (keys %want_headers) {
649
+ next unless (defined $ps->{$k});
650
+ my @tmp;
651
+ foreach my $t (@{$ps->{$k}}) {
652
+ next unless length ($t);
653
+ next if $t =~ m!\{arch\}/!;
654
+ next if $t =~ m!\.arch-ids/!;
655
+ # should we skip this?
656
+ next if $t =~ m!\.arch-inventory$!;
657
# tla cat-archive-log will give us filenames with spaces as file\(sp)name - why?
658
# we can assume that any filename with \ indicates some pika escaping that we want to get rid of.
643
- if ($t =~ /\\/ ){
659
+ if ($t =~ /\\/ ){
660
$t = (safe_pipe_capture($TLA,'escape','--unescaped',$t))[0];
661
}
646
- push (@tmp, $t);
662
+ push @tmp, $t;
663
}
648
- @$ref = @tmp;
664
+ $ps->{$k} = \@tmp if scalar @tmp;
665
}
650
-
651
- #print Dumper [$sum, $msg, \@add, \@del, \@mod, \@ren];
652
- return ($sum, $msg, \@add, \@del, \@mod, \@ren);
666
}
667
668
# write/read a tag