@matheusbsbs / matheus-bagliano / commits / 6df896b50a
Fork 0

archimport: safer log file parsing

Better logfile parsing, no longer confused by 'headers' after the first blank line. Re-enabled tag-reading with abrowse (baz and tla compatible) Remove need to quote args to external processes Signed-off-by: Eric Wong <normalperson@yhbt.net> Signed-off-by: Martin Langhoff <martin@catalyst.net.nz>

Eric Wong committed Nov 23, 2005 at 23:53 UTC 6df896b50a8f1294d00e8da9d3662d9422dd0533
1 file changed +111 -98
git-archimport.perl
+111 -98
@@ -140,10 +140,10 @@ sub do_abrowse {
140 $ps{type} = 's';
141 } elsif ($type =~ /\(.*import\)/) {
142 $ps{type} = 'i';
143 - } elsif ($type =~ m/\(tag.*\)/) {
143 + } elsif ($type =~ m/\(tag.*?(\S+\@\S+).*?\)/) {
144 $ps{type} = 't';
145 # read which revision we've tagged when we parse the log
146 - #$ps{tag} = $1;
146 + $ps{tag} = $1;
147 } else {
148 warn "Unknown type $type";
149 }
@@ -359,78 +359,73 @@ foreach my $ps (@psets) {
359 #
360 my $tree;
361
362 - my $commitlog = safe_pipe_capture($TLA,'cat-archive-log',$ps->{id});
362 + my @commitlog = safe_pipe_capture($TLA,'cat-archive-log',$ps->{id});
363 die "Error in cat-archive-log: $!" if $?;
364
365 - # parselog will git-add/rm files
366 - # and generally prepare things for the commit
367 - # NOTE: parselog will shell-quote filenames!
368 - my ($sum, $msg, $add, $del, $mod, $ren) = parselog($commitlog);
369 - my $logmessage = "$sum\n$msg";
370 -
365 + parselog($ps,\@commitlog);
366
367 # imports don't give us good info
368 # on added files. Shame on them
374 - if ($ps->{type} eq 'i' || $ps->{type} eq 't') {
375 - `find . -type f -print0 | grep -zv '^./$git_dir' | xargs -0 -l100 git-update-index --add`;
376 - `git-ls-files --deleted -z | xargs --no-run-if-empty -0 -l100 git-update-index --remove`;
369 + if ($ps->{type} eq 'i' || $ps->{type} eq 't') {
370 + system('git-ls-files --others -z | '.
371 + 'git-update-index --add -z --stdin') == 0 or die "$! $?\n";
372 + system('git-ls-files --deleted -z | '.
373 + 'git-update-index --remove -z --stdin') == 0 or die "$! $?\n";
374 }
375
379 - if (@$add) {
376 + # TODO: handle removed_directories and renamed_directories:
377 +
378 + if (my $add = $ps->{new_files}) {
379 while (@$add) {
380 my @slice = splice(@$add, 0, 100);
382 - my $slice = join(' ', @slice);
383 - `git-update-index --add $slice`;
384 - die "Error in git-update-index --add: $!" if $?;
381 + system('git-update-index','--add','--',@slice) == 0 or
382 + die "Error in git-update-index --add: $! $?\n";
383 }
384 }
387 - if (@$del) {
388 - foreach my $file (@$del) {
389 - unlink $file or die "Problems deleting $file : $!";
390 - }
385 +
386 + if (my $del = $ps->{removed_files}) {
387 + unlink @$del;
388 while (@$del) {
389 my @slice = splice(@$del, 0, 100);
393 - my $slice = join(' ', @slice);
394 - `git-update-index --remove $slice`;
395 - die "Error in git-update-index --remove: $!" if $?;
390 + system('git-update-index','--remove','--',@slice) == 0 or
391 + die "Error in git-update-index --remove: $! $?\n";
392 }
393 }
398 - if (@$ren) { # renamed
394 +
395 + if (my $ren = $ps->{renamed_files}) { # renamed
396 if (@$ren % 2) {
397 die "Odd number of entries in rename!?";
398 }
402 - ;
399 +
400 while (@$ren) {
404 - my $from = pop @$ren;
405 - my $to = pop @$ren;
401 + my $from = shift @$ren;
402 + my $to = shift @$ren;
403
404 unless (-d dirname($to)) {
405 mkpath(dirname($to)); # will die on err
406 }
410 - #print "moving $from $to";
411 - `mv $from $to`;
412 - die "Error renaming $from $to : $!" if $?;
413 - `git-update-index --remove $from`;
414 - die "Error in git-update-index --remove: $!" if $?;
415 - `git-update-index --add $to`;
416 - die "Error in git-update-index --add: $!" if $?;
407 + print "moving $from $to";
408 + rename($from, $to) or die "Error renaming '$from' '$to': $!\n";
409 + system('git-update-index','--remove','--',$from) == 0 or
410 + die "Error in git-update-index --remove: $! $?\n";
411 + system('git-update-index','--add','--',$to) == 0 or
412 + die "Error in git-update-index --add: $! $?\n";
413 }
414
415 }
420 - if (@$mod) { # must be _after_ renames
416 +
417 + if (my $mod = $ps->{modified_files}) {
418 while (@$mod) {
419 my @slice = splice(@$mod, 0, 100);
423 - my $slice = join(' ', @slice);
424 - `git-update-index $slice`;
425 - die "Error in git-update-index: $!" if $?;
420 + system('git-update-index','--',@slice) == 0 or
421 + die "Error in git-update-index: $! $?\n";
422 }
423 }
428 -
424 +
425 # warn "errors when running git-update-index! $!";
426 $tree = `git-write-tree`;
427 die "cannot write tree $!" if $?;
428 chomp $tree;
433 -
429
430 #
431 # Who's your daddy?
@@ -464,13 +459,14 @@ foreach my $ps (@psets) {
459 $ENV{GIT_COMMITTER_EMAIL} = $ps->{email};
460 $ENV{GIT_COMMITTER_DATE} = $ps->{date};
461
467 - my ($pid, $commit_rh, $commit_wh);
468 - $commit_rh = 'commit_rh';
469 - $commit_wh = 'commit_wh';
470 -
471 - $pid = open2(*READER, *WRITER,'git-commit-tree',$tree,@par)
462 + my $pid = open2(*READER, *WRITER,'git-commit-tree',$tree,@par)
463 or die $!;
473 - print WRITER $logmessage; # write
464 + print WRITER $ps->{summary},"\n";
465 + print WRITER $ps->{message},"\n";
466 +
467 + # make it easy to backtrack and figure out which Arch revision this was:
468 + print WRITER 'git-archimport-id: ',$ps->{id},"\n";
469 +
470 close WRITER;
471 my $commitid = <READER>; # read
472 chomp $commitid;
@@ -568,7 +564,9 @@ sub apply_cset {
564
565
566 # =for reference
571 -# A log entry looks like
567 +# notes: *-files/-directories keys cannot have spaces, they're always
568 +# pika-escaped. Everything after the first newline
569 +# A log entry looks like:
570 # Revision: moodle-org--moodle--1.3.3--patch-15
571 # Archive: arch-eduforge@catalyst.net.nz--2004
572 # Creator: Penny Leach <penny@catalyst.net.nz>
@@ -586,70 +584,85 @@ sub apply_cset {
584 # admin/editor.html backup/lib.php backup/restore.php
585 # New-patches: arch-eduforge@catalyst.net.nz--2004/moodle-org--moodle--1.3.3--patch-15
586 # Summary: Updating to latest from MOODLE_14_STABLE (1.4.5+)
587 +# summary can be multiline with a leading space just like the above fields
588 # Keywords:
589 #
590 # Updating yadda tadda tadda madda
591 sub parselog {
593 - my $log = shift;
594 - #print $log;
595 -
596 - my (@add, @del, @mod, @ren, @kw, $sum, $msg );
597 -
598 - if ($log =~ m/(?:\n|^)New-files:(.*?)(?=\n\w)/s ) {
599 - my $files = $1;
600 - @add = split(m/\s+/s, $files);
601 - }
602 -
603 - if ($log =~ m/(?:\n|^)Removed-files:(.*?)(?=\n\w)/s ) {
604 - my $files = $1;
605 - @del = split(m/\s+/s, $files);
606 - }
592 + my ($ps, $log) = @_;
593 + my $key = undef;
594 +
595 + # headers we want that contain filenames:
596 + my %want_headers = (
597 + new_files => 1,
598 + modified_files => 1,
599 + renamed_files => 1,
600 + renamed_directories => 1,
601 + removed_files => 1,
602 + removed_directories => 1,
603 + );
604
608 - if ($log =~ m/(?:\n|^)Modified-files:(.*?)(?=\n\w)/s ) {
609 - my $files = $1;
610 - @mod = split(m/\s+/s, $files);
605 + chomp (@$log);
606 + while ($_ = shift @$log) {
607 + if (/^Continuation-of:\s*(.*)/) {
608 + $ps->{tag} = $1;
609 + $key = undef;
610 + } elsif (/^Summary:\s*(.*)$/ ) {
611 + # summary can be multiline as long as it has a leading space
612 + $ps->{summary} = [ $1 ];
613 + $key = 'summary';
614 + } elsif (/^Creator: (.*)\s*<([^\>]+)>/) {
615 + $ps->{author} = $1;
616 + $ps->{email} = $2;
617 + $key = undef;
618 + # any *-files or *-directories can be read here:
619 + } elsif (/^([A-Z][a-z\-]+):\s*(.*)$/) {
620 + my $val = $2;
621 + $key = lc $1;
622 + $key =~ tr/-/_/; # too lazy to quote :P
623 + if ($want_headers{$key}) {
624 + push @{$ps->{$key}}, split(/\s+/, $val);
625 + } else {
626 + $key = undef;
627 + }
628 + } elsif (/^$/) {
629 + last; # remainder of @$log that didn't get shifted off is message
630 + } elsif ($key) {
631 + if (/^\s+(.*)$/) {
632 + if ($key eq 'summary') {
633 + push @{$ps->{$key}}, $1;
634 + } else { # files/directories:
635 + push @{$ps->{$key}}, split(/\s+/, $1);
636 + }
637 + } else {
638 + $key = undef;
639 + }
640 + }
641 }
642 +
643 + # post-processing:
644 + $ps->{summary} = join("\n",@{$ps->{summary}})."\n";
645 + $ps->{message} = join("\n",@$log);
646
613 - if ($log =~ m/(?:\n|^)Renamed-files:(.*?)(?=\n\w)/s ) {
614 - my $files = $1;
615 - @ren = split(m/\s+/s, $files);
616 - }
617 -
618 - $sum ='';
619 - if ($log =~ m/^Summary:(.+?)$/m ) {
620 - $sum = $1;
621 - $sum =~ s/^\s+//;
622 - $sum =~ s/\s+$//;
623 - }
624 -
625 - $msg = '';
626 - if ($log =~ m/\n\n(.+)$/s) {
627 - $msg = $1;
628 - $msg =~ s/^\s+//;
629 - $msg =~ s/\s+$//;
630 - }
631 -
632 -
633 - # cleanup the arrays
634 - foreach my $ref ( (\@add, \@del, \@mod, \@ren) ) {
635 - my @tmp = ();
636 - while (my $t = pop @$ref) {
637 - next unless length ($t);
638 - next if $t =~ m!\{arch\}/!;
639 - next if $t =~ m!\.arch-ids/!;
640 - next if $t =~ m!\.arch-inventory$!;
647 + # skip Arch control files, unescape pika-escaped files
648 + foreach my $k (keys %want_headers) {
649 + next unless (defined $ps->{$k});
650 + my @tmp;
651 + foreach my $t (@{$ps->{$k}}) {
652 + next unless length ($t);
653 + next if $t =~ m!\{arch\}/!;
654 + next if $t =~ m!\.arch-ids/!;
655 + # should we skip this?
656 + next if $t =~ m!\.arch-inventory$!;
657 # tla cat-archive-log will give us filenames with spaces as file\(sp)name - why?
658 # we can assume that any filename with \ indicates some pika escaping that we want to get rid of.
643 - if ($t =~ /\\/ ){
659 + if ($t =~ /\\/ ){
660 $t = (safe_pipe_capture($TLA,'escape','--unescaped',$t))[0];
661 }
646 - push (@tmp, $t);
662 + push @tmp, $t;
663 }
648 - @$ref = @tmp;
664 + $ps->{$k} = \@tmp if scalar @tmp;
665 }
650 -
651 - #print Dumper [$sum, $msg, \@add, \@del, \@mod, \@ren];
652 - return ($sum, $msg, \@add, \@del, \@mod, \@ren);
666 }
667
668 # write/read a tag