archimport: Add the accurate changeset applyer
And make it the default. This includes stats tracking to verbose mode Signed-off-by: Eric Wong <normalperson@yhbt.net> Signed-off-by: Martin Langhoff <martin@catalyst.net.nz>
Eric Wong committed
Nov 23, 2005 at 23:55 UTC
3e525e673849393035a2639e17ff74e616a618b9
1 file changed
+171
-28
git-archimport.perl
+171
-28
@@ -25,6 +25,9 @@ See man (1) git-archimport for more details.
25
- audit shell-escaping of filenames
26
- hide our private tags somewhere smarter
27
- find a way to make "cat *patches | patch" safe even when patchfiles are missing newlines
28
+ - sort and apply patches by graphing ancestry relations instead of just
29
+ relying in dates supplied in the changeset itself.
30
+ tla ancestry-graph -m could be helpful here...
31
32
=head1 Devel tricks
33
@@ -66,18 +69,18 @@ my $git_dir = $ENV{"GIT_DIR"} || ".git";
69
$ENV{"GIT_DIR"} = $git_dir;
70
my $ptag_dir = "$git_dir/archimport/tags";
71
69
-our($opt_h,$opt_v,$opt_T,$opt_t,$opt_D,$opt_a,$opt_o);
72
+our($opt_h,$opt_f,$opt_v,$opt_T,$opt_t,$opt_D,$opt_a,$opt_o);
73
74
sub usage() {
75
print STDERR <<END;
76
Usage: ${\basename $0} # fetch/update GIT from Arch
74
- [ -o ] [ -h ] [ -v ] [ -T ] [ -a ] [ -D depth ] [ -t tempdir ]
77
+ [ -f ] [ -o ] [ -h ] [ -v ] [ -T ] [ -a ] [ -D depth ] [ -t tempdir ]
78
repository/arch-branch [ repository/arch-branch] ...
79
END
80
exit(1);
81
}
82
80
-getopts("Thvat:D:") or usage();
83
+getopts("fThvat:D:") or usage();
84
usage if $opt_h;
85
86
@ARGV >= 1 or usage();
@@ -95,6 +98,10 @@ my %reachable = (); # Arch repositories we can access
98
my %unreachable = (); # Arch repositories we can't access :<
99
my @psets = (); # the collection
100
my %psets = (); # the collection, by name
101
+my %stats = ( # Track which strategy we used to import:
102
+ get_tag => 0, replay => 0, get_new => 0, get_delta => 0,
103
+ simple_changeset => 0, import_or_tag => 0
104
+);
105
106
my %rptags = (); # my reverse private tags
107
# to map a SHA1 to a commitid
@@ -288,29 +295,69 @@ sub old_style_branchname {
295
296
*git_branchname = $opt_o ? *old_style_branchname : *tree_dirname;
297
291
-# process patchsets
292
-foreach my $ps (@psets) {
293
- $ps->{branch} = git_branchname($ps->{id});
294
-
295
- #
296
- # ensure we have a clean state
297
- #
298
- if (`git-diff-files`) {
299
- die "Unclean tree when about to process $ps->{id} " .
300
- " - did we fail to commit cleanly before?";
301
- }
302
- die $! if $?;
298
+sub process_patchset_accurate {
299
+ my $ps = shift;
300
+
301
+ # switch to that branch if we're not already in that branch:
302
+ if (-e "$git_dir/refs/heads/$ps->{branch}") {
303
+ system('git-checkout','-f',$ps->{branch}) == 0 or die "$! $?\n";
304
304
- #
305
- # skip commits already in repo
306
- #
307
- if (ptag($ps->{id})) {
308
- $opt_v && print " * Skipping already imported: $ps->{id}\n";
309
- next;
305
+ # remove any old stuff that got leftover:
306
+ my $rm = safe_pipe_capture('git-ls-files','--others','-z');
307
+ rmtree(split(/\0/,$rm)) if $rm;
308
}
309
+
310
+ # Apply the import/changeset/merge into the working tree
311
+ my $dir = sync_to_ps($ps);
312
+ # read the new log entry:
313
+ my @commitlog = safe_pipe_capture($TLA,'cat-log','-d',$dir,$ps->{id});
314
+ die "Error in cat-log: $!" if $?;
315
+ chomp @commitlog;
316
+
317
+ # grab variables we want from the log, new fields get added to $ps:
318
+ # (author, date, email, summary, message body ...)
319
+ parselog($ps, \@commitlog);
320
+
321
+ if ($ps->{id} =~ /--base-0$/ && $ps->{id} ne $psets[0]{id}) {
322
+ # this should work when importing continuations
323
+ if ($ps->{tag} && (my $branchpoint = eval { ptag($ps->{tag}) })) {
324
+
325
+ # find where we are supposed to branch from
326
+ system('git-checkout','-f','-b',$ps->{branch},
327
+ $branchpoint) == 0 or die "$! $?\n";
328
+
329
+ # remove any old stuff that got leftover:
330
+ my $rm = safe_pipe_capture('git-ls-files','--others','-z');
331
+ rmtree(split(/\0/,$rm)) if $rm;
332
312
- print " * Starting to work on $ps->{id}\n";
333
+ # If we trust Arch with the fact that this is just
334
+ # a tag, and it does not affect the state of the tree
335
+ # then we just tag and move on
336
+ tag($ps->{id}, $branchpoint);
337
+ ptag($ps->{id}, $branchpoint);
338
+ print " * Tagged $ps->{id} at $branchpoint\n";
339
+ return 0;
340
+ } else {
341
+ warn "Tagging from unknown id unsupported\n" if $ps->{tag};
342
+ }
343
+ # allow multiple bases/imports here since Arch supports cherry-picks
344
+ # from unrelated trees
345
+ }
346
+
347
+ # update the index with all the changes we got
348
+ system('git-ls-files --others -z | '.
349
+ 'git-update-index --add -z --stdin') == 0 or die "$! $?\n";
350
+ system('git-ls-files --deleted -z | '.
351
+ 'git-update-index --remove -z --stdin') == 0 or die "$! $?\n";
352
+ system('git-ls-files -z | '.
353
+ 'git-update-index -z --stdin') == 0 or die "$! $?\n";
354
+ return 1;
355
+}
356
357
+# the native changeset processing strategy. This is very fast, but
358
+# does not handle permissions or any renames involving directories
359
+sub process_patchset_fast {
360
+ my $ps = shift;
361
#
362
# create the branch if needed
363
#
@@ -338,7 +385,7 @@ foreach my $ps (@psets) {
385
tag($ps->{id}, $branchpoint);
386
ptag($ps->{id}, $branchpoint);
387
print " * Tagged $ps->{id} at $branchpoint\n";
341
- next;
388
+ return 0;
389
}
390
die $! if $?;
391
}
@@ -348,16 +395,17 @@ foreach my $ps (@psets) {
395
#
396
if ($ps->{type} eq 'i' || $ps->{type} eq 't') {
397
apply_import($ps) or die $!;
398
+ $stats{import_or_tag}++;
399
$import=0;
400
} elsif ($ps->{type} eq 's') {
401
apply_cset($ps);
402
+ $stats{simple_changeset}++;
403
}
404
405
#
406
# prepare update git's index, based on what arch knows
407
# about the pset, resolve parents, etc
408
#
360
- my $tree;
409
410
my @commitlog = safe_pipe_capture($TLA,'cat-archive-log',$ps->{id});
411
die "Error in cat-archive-log: $!" if $?;
@@ -404,14 +452,13 @@ foreach my $ps (@psets) {
452
unless (-d dirname($to)) {
453
mkpath(dirname($to)); # will die on err
454
}
407
- print "moving $from $to";
455
+ # print "moving $from $to";
456
rename($from, $to) or die "Error renaming '$from' '$to': $!\n";
457
system('git-update-index','--remove','--',$from) == 0 or
458
die "Error in git-update-index --remove: $! $?\n";
459
system('git-update-index','--add','--',$to) == 0 or
460
die "Error in git-update-index --add: $! $?\n";
461
}
414
-
462
}
463
464
if (my $mod = $ps->{modified_files}) {
@@ -421,9 +468,46 @@ foreach my $ps (@psets) {
468
die "Error in git-update-index: $! $?\n";
469
}
470
}
471
+ return 1; # we successfully applied the changeset
472
+}
473
+
474
+if ($opt_f) {
475
+ print "Will import patchsets using the fast strategy\n",
476
+ "Renamed directories and permission changes will be missed\n";
477
+ *process_patchset = *process_patchset_fast;
478
+} else {
479
+ print "Using the default (accurate) import strategy.\n",
480
+ "Things may be a bit slow\n";
481
+ *process_patchset = *process_patchset_accurate;
482
+}
483
484
+foreach my $ps (@psets) {
485
+ # process patchsets
486
+ $ps->{branch} = git_branchname($ps->{id});
487
+
488
+ #
489
+ # ensure we have a clean state
490
+ #
491
+ if (my $dirty = `git-diff-files`) {
492
+ die "Unclean tree when about to process $ps->{id} " .
493
+ " - did we fail to commit cleanly before?\n$dirty";
494
+ }
495
+ die $! if $?;
496
+
497
+ #
498
+ # skip commits already in repo
499
+ #
500
+ if (ptag($ps->{id})) {
501
+ $opt_v && print " * Skipping already imported: $ps->{id}\n";
502
+ return 0;
503
+ }
504
+
505
+ print " * Starting to work on $ps->{id}\n";
506
+
507
+ process_patchset($ps) or next;
508
+
509
# warn "errors when running git-update-index! $!";
426
- $tree = `git-write-tree`;
510
+ my $tree = `git-write-tree`;
511
die "cannot write tree $!" if $?;
512
chomp $tree;
513
@@ -494,6 +578,65 @@ foreach my $ps (@psets) {
578
print " + commit $commitid\n";
579
$opt_v && print " + commit date is $ps->{date} \n";
580
$opt_v && print " + parents: ",join(' ',@par),"\n";
581
+ if (my $dirty = `git-diff-files`) {
582
+ die "22 Unclean tree when about to process $ps->{id} " .
583
+ " - did we fail to commit cleanly before?\n$dirty";
584
+ }
585
+}
586
+
587
+if ($opt_v) {
588
+ foreach (sort keys %stats) {
589
+ print" $_: $stats{$_}\n";
590
+ }
591
+}
592
+exit 0;
593
+
594
+# used by the accurate strategy:
595
+sub sync_to_ps {
596
+ my $ps = shift;
597
+ my $tree_dir = $tmp.'/'.tree_dirname($ps->{id});
598
+
599
+ $opt_v && print "sync_to_ps($ps->{id}) method: ";
600
+
601
+ if (-d $tree_dir) {
602
+ if ($ps->{type} eq 't') {
603
+ $opt_v && print "get (tag)\n";
604
+ # looks like a tag-only or (worse,) a mixed tags/changeset branch,
605
+ # can't rely on replay to work correctly on these
606
+ rmtree($tree_dir);
607
+ safe_pipe_capture($TLA,'get','--no-pristine',$ps->{id},$tree_dir);
608
+ $stats{get_tag}++;
609
+ } else {
610
+ my $tree_id = arch_tree_id($tree_dir);
611
+ if ($ps->{parent_id} && ($ps->{parent_id} eq $tree_id)) {
612
+ # the common case (hopefully)
613
+ $opt_v && print "replay\n";
614
+ safe_pipe_capture($TLA,'replay','-d',$tree_dir,$ps->{id});
615
+ $stats{replay}++;
616
+ } else {
617
+ # getting one tree is usually faster than getting two trees
618
+ # and applying the delta ...
619
+ rmtree($tree_dir);
620
+ $opt_v && print "apply-delta\n";
621
+ safe_pipe_capture($TLA,'get','--no-pristine',
622
+ $ps->{id},$tree_dir);
623
+ $stats{get_delta}++;
624
+ }
625
+ }
626
+ } else {
627
+ # new branch work
628
+ $opt_v && print "get (new tree)\n";
629
+ safe_pipe_capture($TLA,'get','--no-pristine',$ps->{id},$tree_dir);
630
+ $stats{get_new}++;
631
+ }
632
+
633
+ # added -I flag to rsync since we're going to fast! AIEEEEE!!!!
634
+ system('rsync','-aI','--delete','--exclude',$git_dir,
635
+# '--exclude','.arch-inventory',
636
+ '--exclude','.arch-ids','--exclude','{arch}',
637
+ '--exclude','+*','--exclude',',*',
638
+ "$tree_dir/",'./') == 0 or die "Cannot rsync $tree_dir: $! $?";
639
+ return $tree_dir;
640
}
641
642
sub apply_import {
@@ -896,7 +1039,7 @@ sub safe_pipe_capture {
1039
@output = (<$child>);
1040
close $child or die join(' ',@_).": $! $?";
1041
} else {
899
- exec(@_) or die $?; # exec() can fail the executable can't be found
1042
+ exec(@_) or die "$! $?"; # exec() can fail the executable can't be found
1043
}
1044
return wantarray ? @output : join('',@output);
1045
}