@matheusbsbs / matheus-bagliano / commits / 3e525e6738
Fork 0

archimport: Add the accurate changeset applyer

And make it the default. This includes stats tracking to verbose mode Signed-off-by: Eric Wong <normalperson@yhbt.net> Signed-off-by: Martin Langhoff <martin@catalyst.net.nz>

Eric Wong committed Nov 23, 2005 at 23:55 UTC 3e525e673849393035a2639e17ff74e616a618b9
1 file changed +171 -28
git-archimport.perl
+171 -28
@@ -25,6 +25,9 @@ See man (1) git-archimport for more details.
25 - audit shell-escaping of filenames
26 - hide our private tags somewhere smarter
27 - find a way to make "cat *patches | patch" safe even when patchfiles are missing newlines
28 + - sort and apply patches by graphing ancestry relations instead of just
29 + relying in dates supplied in the changeset itself.
30 + tla ancestry-graph -m could be helpful here...
31
32 =head1 Devel tricks
33
@@ -66,18 +69,18 @@ my $git_dir = $ENV{"GIT_DIR"} || ".git";
69 $ENV{"GIT_DIR"} = $git_dir;
70 my $ptag_dir = "$git_dir/archimport/tags";
71
69 -our($opt_h,$opt_v,$opt_T,$opt_t,$opt_D,$opt_a,$opt_o);
72 +our($opt_h,$opt_f,$opt_v,$opt_T,$opt_t,$opt_D,$opt_a,$opt_o);
73
74 sub usage() {
75 print STDERR <<END;
76 Usage: ${\basename $0} # fetch/update GIT from Arch
74 - [ -o ] [ -h ] [ -v ] [ -T ] [ -a ] [ -D depth ] [ -t tempdir ]
77 + [ -f ] [ -o ] [ -h ] [ -v ] [ -T ] [ -a ] [ -D depth ] [ -t tempdir ]
78 repository/arch-branch [ repository/arch-branch] ...
79 END
80 exit(1);
81 }
82
80 -getopts("Thvat:D:") or usage();
83 +getopts("fThvat:D:") or usage();
84 usage if $opt_h;
85
86 @ARGV >= 1 or usage();
@@ -95,6 +98,10 @@ my %reachable = (); # Arch repositories we can access
98 my %unreachable = (); # Arch repositories we can't access :<
99 my @psets = (); # the collection
100 my %psets = (); # the collection, by name
101 +my %stats = ( # Track which strategy we used to import:
102 + get_tag => 0, replay => 0, get_new => 0, get_delta => 0,
103 + simple_changeset => 0, import_or_tag => 0
104 +);
105
106 my %rptags = (); # my reverse private tags
107 # to map a SHA1 to a commitid
@@ -288,29 +295,69 @@ sub old_style_branchname {
295
296 *git_branchname = $opt_o ? *old_style_branchname : *tree_dirname;
297
291 -# process patchsets
292 -foreach my $ps (@psets) {
293 - $ps->{branch} = git_branchname($ps->{id});
294 -
295 - #
296 - # ensure we have a clean state
297 - #
298 - if (`git-diff-files`) {
299 - die "Unclean tree when about to process $ps->{id} " .
300 - " - did we fail to commit cleanly before?";
301 - }
302 - die $! if $?;
298 +sub process_patchset_accurate {
299 + my $ps = shift;
300 +
301 + # switch to that branch if we're not already in that branch:
302 + if (-e "$git_dir/refs/heads/$ps->{branch}") {
303 + system('git-checkout','-f',$ps->{branch}) == 0 or die "$! $?\n";
304
304 - #
305 - # skip commits already in repo
306 - #
307 - if (ptag($ps->{id})) {
308 - $opt_v && print " * Skipping already imported: $ps->{id}\n";
309 - next;
305 + # remove any old stuff that got leftover:
306 + my $rm = safe_pipe_capture('git-ls-files','--others','-z');
307 + rmtree(split(/\0/,$rm)) if $rm;
308 }
309 +
310 + # Apply the import/changeset/merge into the working tree
311 + my $dir = sync_to_ps($ps);
312 + # read the new log entry:
313 + my @commitlog = safe_pipe_capture($TLA,'cat-log','-d',$dir,$ps->{id});
314 + die "Error in cat-log: $!" if $?;
315 + chomp @commitlog;
316 +
317 + # grab variables we want from the log, new fields get added to $ps:
318 + # (author, date, email, summary, message body ...)
319 + parselog($ps, \@commitlog);
320 +
321 + if ($ps->{id} =~ /--base-0$/ && $ps->{id} ne $psets[0]{id}) {
322 + # this should work when importing continuations
323 + if ($ps->{tag} && (my $branchpoint = eval { ptag($ps->{tag}) })) {
324 +
325 + # find where we are supposed to branch from
326 + system('git-checkout','-f','-b',$ps->{branch},
327 + $branchpoint) == 0 or die "$! $?\n";
328 +
329 + # remove any old stuff that got leftover:
330 + my $rm = safe_pipe_capture('git-ls-files','--others','-z');
331 + rmtree(split(/\0/,$rm)) if $rm;
332
312 - print " * Starting to work on $ps->{id}\n";
333 + # If we trust Arch with the fact that this is just
334 + # a tag, and it does not affect the state of the tree
335 + # then we just tag and move on
336 + tag($ps->{id}, $branchpoint);
337 + ptag($ps->{id}, $branchpoint);
338 + print " * Tagged $ps->{id} at $branchpoint\n";
339 + return 0;
340 + } else {
341 + warn "Tagging from unknown id unsupported\n" if $ps->{tag};
342 + }
343 + # allow multiple bases/imports here since Arch supports cherry-picks
344 + # from unrelated trees
345 + }
346 +
347 + # update the index with all the changes we got
348 + system('git-ls-files --others -z | '.
349 + 'git-update-index --add -z --stdin') == 0 or die "$! $?\n";
350 + system('git-ls-files --deleted -z | '.
351 + 'git-update-index --remove -z --stdin') == 0 or die "$! $?\n";
352 + system('git-ls-files -z | '.
353 + 'git-update-index -z --stdin') == 0 or die "$! $?\n";
354 + return 1;
355 +}
356
357 +# the native changeset processing strategy. This is very fast, but
358 +# does not handle permissions or any renames involving directories
359 +sub process_patchset_fast {
360 + my $ps = shift;
361 #
362 # create the branch if needed
363 #
@@ -338,7 +385,7 @@ foreach my $ps (@psets) {
385 tag($ps->{id}, $branchpoint);
386 ptag($ps->{id}, $branchpoint);
387 print " * Tagged $ps->{id} at $branchpoint\n";
341 - next;
388 + return 0;
389 }
390 die $! if $?;
391 }
@@ -348,16 +395,17 @@ foreach my $ps (@psets) {
395 #
396 if ($ps->{type} eq 'i' || $ps->{type} eq 't') {
397 apply_import($ps) or die $!;
398 + $stats{import_or_tag}++;
399 $import=0;
400 } elsif ($ps->{type} eq 's') {
401 apply_cset($ps);
402 + $stats{simple_changeset}++;
403 }
404
405 #
406 # prepare update git's index, based on what arch knows
407 # about the pset, resolve parents, etc
408 #
360 - my $tree;
409
410 my @commitlog = safe_pipe_capture($TLA,'cat-archive-log',$ps->{id});
411 die "Error in cat-archive-log: $!" if $?;
@@ -404,14 +452,13 @@ foreach my $ps (@psets) {
452 unless (-d dirname($to)) {
453 mkpath(dirname($to)); # will die on err
454 }
407 - print "moving $from $to";
455 + # print "moving $from $to";
456 rename($from, $to) or die "Error renaming '$from' '$to': $!\n";
457 system('git-update-index','--remove','--',$from) == 0 or
458 die "Error in git-update-index --remove: $! $?\n";
459 system('git-update-index','--add','--',$to) == 0 or
460 die "Error in git-update-index --add: $! $?\n";
461 }
414 -
462 }
463
464 if (my $mod = $ps->{modified_files}) {
@@ -421,9 +468,46 @@ foreach my $ps (@psets) {
468 die "Error in git-update-index: $! $?\n";
469 }
470 }
471 + return 1; # we successfully applied the changeset
472 +}
473 +
474 +if ($opt_f) {
475 + print "Will import patchsets using the fast strategy\n",
476 + "Renamed directories and permission changes will be missed\n";
477 + *process_patchset = *process_patchset_fast;
478 +} else {
479 + print "Using the default (accurate) import strategy.\n",
480 + "Things may be a bit slow\n";
481 + *process_patchset = *process_patchset_accurate;
482 +}
483
484 +foreach my $ps (@psets) {
485 + # process patchsets
486 + $ps->{branch} = git_branchname($ps->{id});
487 +
488 + #
489 + # ensure we have a clean state
490 + #
491 + if (my $dirty = `git-diff-files`) {
492 + die "Unclean tree when about to process $ps->{id} " .
493 + " - did we fail to commit cleanly before?\n$dirty";
494 + }
495 + die $! if $?;
496 +
497 + #
498 + # skip commits already in repo
499 + #
500 + if (ptag($ps->{id})) {
501 + $opt_v && print " * Skipping already imported: $ps->{id}\n";
502 + return 0;
503 + }
504 +
505 + print " * Starting to work on $ps->{id}\n";
506 +
507 + process_patchset($ps) or next;
508 +
509 # warn "errors when running git-update-index! $!";
426 - $tree = `git-write-tree`;
510 + my $tree = `git-write-tree`;
511 die "cannot write tree $!" if $?;
512 chomp $tree;
513
@@ -494,6 +578,65 @@ foreach my $ps (@psets) {
578 print " + commit $commitid\n";
579 $opt_v && print " + commit date is $ps->{date} \n";
580 $opt_v && print " + parents: ",join(' ',@par),"\n";
581 + if (my $dirty = `git-diff-files`) {
582 + die "22 Unclean tree when about to process $ps->{id} " .
583 + " - did we fail to commit cleanly before?\n$dirty";
584 + }
585 +}
586 +
587 +if ($opt_v) {
588 + foreach (sort keys %stats) {
589 + print" $_: $stats{$_}\n";
590 + }
591 +}
592 +exit 0;
593 +
594 +# used by the accurate strategy:
595 +sub sync_to_ps {
596 + my $ps = shift;
597 + my $tree_dir = $tmp.'/'.tree_dirname($ps->{id});
598 +
599 + $opt_v && print "sync_to_ps($ps->{id}) method: ";
600 +
601 + if (-d $tree_dir) {
602 + if ($ps->{type} eq 't') {
603 + $opt_v && print "get (tag)\n";
604 + # looks like a tag-only or (worse,) a mixed tags/changeset branch,
605 + # can't rely on replay to work correctly on these
606 + rmtree($tree_dir);
607 + safe_pipe_capture($TLA,'get','--no-pristine',$ps->{id},$tree_dir);
608 + $stats{get_tag}++;
609 + } else {
610 + my $tree_id = arch_tree_id($tree_dir);
611 + if ($ps->{parent_id} && ($ps->{parent_id} eq $tree_id)) {
612 + # the common case (hopefully)
613 + $opt_v && print "replay\n";
614 + safe_pipe_capture($TLA,'replay','-d',$tree_dir,$ps->{id});
615 + $stats{replay}++;
616 + } else {
617 + # getting one tree is usually faster than getting two trees
618 + # and applying the delta ...
619 + rmtree($tree_dir);
620 + $opt_v && print "apply-delta\n";
621 + safe_pipe_capture($TLA,'get','--no-pristine',
622 + $ps->{id},$tree_dir);
623 + $stats{get_delta}++;
624 + }
625 + }
626 + } else {
627 + # new branch work
628 + $opt_v && print "get (new tree)\n";
629 + safe_pipe_capture($TLA,'get','--no-pristine',$ps->{id},$tree_dir);
630 + $stats{get_new}++;
631 + }
632 +
633 + # added -I flag to rsync since we're going to fast! AIEEEEE!!!!
634 + system('rsync','-aI','--delete','--exclude',$git_dir,
635 +# '--exclude','.arch-inventory',
636 + '--exclude','.arch-ids','--exclude','{arch}',
637 + '--exclude','+*','--exclude',',*',
638 + "$tree_dir/",'./') == 0 or die "Cannot rsync $tree_dir: $! $?";
639 + return $tree_dir;
640 }
641
642 sub apply_import {
@@ -896,7 +1039,7 @@ sub safe_pipe_capture {
1039 @output = (<$child>);
1040 close $child or die join(' ',@_).": $! $?";
1041 } else {
899 - exec(@_) or die $?; # exec() can fail the executable can't be found
1042 + exec(@_) or die "$! $?"; # exec() can fail the executable can't be found
1043 }
1044 return wantarray ? @output : join('',@output);
1045 }