@matheusbsbs / matheus-bagliano / commits / 42f44b08bc
Fork 0

archimport: add -D <depth> and -a switch

add -D <depth> option to abrowse add -a switch to attempt to auto-register archives at mirrors.sourcecontrol.net (ML: Also removes some std libraries no longer in use) Signed-off-by: Eric Wong <normalperson@yhbt.net> Signed-off-by: Martin Langhoff <martin@catalyst.net.nz>

Eric Wong committed Nov 23, 2005 at 23:52 UTC 42f44b08bc1bf84a6e620cf7254b820a9656daca
1 file changed +135 -90
git-archimport.perl
+135 -90
@@ -53,14 +53,9 @@ and can contain multiple, unrelated branches.
53 use strict;
54 use warnings;
55 use Getopt::Std;
56 -use File::Spec;
57 -use File::Temp qw(tempfile tempdir);
56 +use File::Temp qw(tempdir);
57 use File::Path qw(mkpath rmtree);
58 use File::Basename qw(basename dirname);
60 -use Time::Local;
61 -use IO::Socket;
62 -use IO::Pipe;
63 -use POSIX qw(strftime dup2);
59 use Data::Dumper qw/ Dumper /;
60 use IPC::Open2;
61
@@ -71,27 +66,33 @@ my $git_dir = $ENV{"GIT_DIR"} || ".git";
66 $ENV{"GIT_DIR"} = $git_dir;
67 my $ptag_dir = "$git_dir/archimport/tags";
68
74 -our($opt_h,$opt_v, $opt_T,$opt_t,$opt_o);
69 +our($opt_h,$opt_v,$opt_T,$opt_t,$opt_D,$opt_a,$opt_o);
70
71 sub usage() {
72 print STDERR <<END;
73 Usage: ${\basename $0} # fetch/update GIT from Arch
79 - [ -o ] [ -h ] [ -v ] [ -T ] [ -t tempdir ]
74 + [ -o ] [ -h ] [ -v ] [ -T ] [ -a ] [ -D depth ] [ -t tempdir ]
75 repository/arch-branch [ repository/arch-branch] ...
76 END
77 exit(1);
78 }
79
85 -getopts("Thvt:") or usage();
80 +getopts("Thvat:D:") or usage();
81 usage if $opt_h;
82
83 @ARGV >= 1 or usage();
89 -my @arch_roots = @ARGV;
84 +# $arch_branches:
85 +# values associated with keys:
86 +# =1 - Arch version / git 'branch' detected via abrowse on a limit
87 +# >1 - Arch version / git 'branch' of an auxilliary branch we've merged
88 +my %arch_branches = map { $_ => 1 } @ARGV;
89
90 $ENV{'TMPDIR'} = $opt_t if $opt_t; # $ENV{TMPDIR} will affect tempdir() calls:
91 my $tmp = tempdir('git-archimport-XXXXXX', TMPDIR => 1, CLEANUP => 1);
92 $opt_v && print "+ Using $tmp as temporary directory\n";
93
94 +my %reachable = (); # Arch repositories we can access
95 +my %unreachable = (); # Arch repositories we can't access :<
96 my @psets = (); # the collection
97 my %psets = (); # the collection, by name
98
@@ -99,96 +100,112 @@ my %rptags = (); # my reverse private tags
100 # to map a SHA1 to a commitid
101 my $TLA = $ENV{'ARCH_CLIENT'} || 'tla';
102
102 -foreach my $root (@arch_roots) {
103 - my ($arepo, $abranch) = split(m!/!, $root);
104 - open ABROWSE, "tla abrowse -f -A $arepo --desc --merges $abranch |"
105 - or die "Problems with tla abrowse: $!";
103 +sub do_abrowse {
104 + my $stage = shift;
105 + while (my ($limit, $level) = each %arch_branches) {
106 + next unless $level == $stage;
107 +
108 + open ABROWSE, "$TLA abrowse -fkD --merges $limit |"
109 + or die "Problems with tla abrowse: $!";
110
107 - my %ps = (); # the current one
108 - my $mode = '';
109 - my $lastseen = '';
111 + my %ps = (); # the current one
112 + my $lastseen = '';
113
111 - while (<ABROWSE>) {
112 - chomp;
113 -
114 - # first record padded w 8 spaces
115 - if (s/^\s{8}\b//) {
114 + while (<ABROWSE>) {
115 + chomp;
116
117 - # store the record we just captured
118 - if (%ps) {
119 - my %temp = %ps; # break references
120 - push (@psets, \%temp);
121 - $psets{$temp{id}} = \%temp;
122 - %ps = ();
123 - }
124 -
125 - my ($id, $type) = split(m/\s{3}/, $_);
126 - $ps{id} = $id;
127 - $ps{repo} = $arepo;
128 -
129 - # deal with types
130 - if ($type =~ m/^\(simple changeset\)/) {
131 - $ps{type} = 's';
132 - } elsif ($type eq '(initial import)') {
133 - $ps{type} = 'i';
134 - } elsif ($type =~ m/^\(tag revision of (.+)\)/) {
135 - $ps{type} = 't';
136 - $ps{tag} = $1;
137 - } else {
138 - warn "Unknown type $type";
139 - }
140 - $lastseen = 'id';
141 - }
142 -
143 - if (s/^\s{10}//) {
144 - # 10 leading spaces or more
145 - # indicate commit metadata
146 -
147 - # date & author
148 - if ($lastseen eq 'id' && m/^\d{4}-\d{2}-\d{2}/) {
117 + # first record padded w 8 spaces
118 + if (s/^\s{8}\b//) {
119 + my ($id, $type) = split(m/\s+/, $_, 2);
120 +
121 + my %last_ps;
122 + # store the record we just captured
123 + if (%ps && !exists $psets{ $ps{id} }) {
124 + %last_ps = %ps; # break references
125 + push (@psets, \%last_ps);
126 + $psets{ $last_ps{id} } = \%last_ps;
127 + }
128
150 - my ($date, $authoremail) = split(m/\s{2,}/, $_);
151 - $ps{date} = $date;
152 - $ps{date} =~ s/\bGMT$//; # strip off trailign GMT
153 - if ($ps{date} =~ m/\b\w+$/) {
154 - warn 'Arch dates not in GMT?! - imported dates will be wrong';
129 + my $branch = extract_versionname($id);
130 + %ps = ( id => $id, branch => $branch );
131 + if (%last_ps && ($last_ps{branch} eq $branch)) {
132 + $ps{parent_id} = $last_ps{id};
133 + }
134 +
135 + $arch_branches{$branch} = 1;
136 + $lastseen = 'id';
137 +
138 + # deal with types (should work with baz or tla):
139 + if ($type =~ m/\(.*changeset\)/) {
140 + $ps{type} = 's';
141 + } elsif ($type =~ /\(.*import\)/) {
142 + $ps{type} = 'i';
143 + } elsif ($type =~ m/\(tag.*\)/) {
144 + $ps{type} = 't';
145 + # read which revision we've tagged when we parse the log
146 + #$ps{tag} = $1;
147 + } else {
148 + warn "Unknown type $type";
149 + }
150 +
151 + $arch_branches{$branch} = 1;
152 + $lastseen = 'id';
153 + } elsif (s/^\s{10}//) {
154 + # 10 leading spaces or more
155 + # indicate commit metadata
156 +
157 + # date
158 + if ($lastseen eq 'id' && m/^(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d)/){
159 + $ps{date} = $1;
160 + $lastseen = 'date';
161 + } elsif ($_ eq 'merges in:') {
162 + $ps{merges} = [];
163 + $lastseen = 'merges';
164 + } elsif ($lastseen eq 'merges' && s/^\s{2}//) {
165 + my $id = $_;
166 + push (@{$ps{merges}}, $id);
167 +
168 + # aggressive branch finding:
169 + if ($opt_D) {
170 + my $branch = extract_versionname($id);
171 + my $repo = extract_reponame($branch);
172 +
173 + if (archive_reachable($repo) &&
174 + !defined $arch_branches{$branch}) {
175 + $arch_branches{$branch} = $stage + 1;
176 + }
177 + }
178 + } else {
179 + warn "more metadata after merges!?: $_\n" unless /^\s*$/;
180 }
156 -
157 - $authoremail =~ m/^(.+)\s(\S+)$/;
158 - $ps{author} = $1;
159 - $ps{email} = $2;
160 -
161 - $lastseen = 'date';
162 -
163 - } elsif ($lastseen eq 'date') {
164 - # the only hint is position
165 - # subject is after date
166 - $ps{subj} = $_;
167 - $lastseen = 'subj';
168 -
169 - } elsif ($lastseen eq 'subj' && $_ eq 'merges in:') {
170 - $ps{merges} = [];
171 - $lastseen = 'merges';
172 -
173 - } elsif ($lastseen eq 'merges' && s/^\s{2}//) {
174 - push (@{$ps{merges}}, $_);
175 - } else {
176 - warn 'more metadata after merges!?';
181 }
178 -
182 }
180 - }
183
182 - if (%ps) {
183 - my %temp = %ps; # break references
184 - push (@psets, \%temp);
185 - $psets{ $temp{id} } = \%temp;
186 - %ps = ();
187 - }
188 - close ABROWSE;
184 + if (%ps && !exists $psets{ $ps{id} }) {
185 + my %temp = %ps; # break references
186 + if (@psets && $psets[$#psets]{branch} eq $ps{branch}) {
187 + $temp{parent_id} = $psets[$#psets]{id};
188 + }
189 + push (@psets, \%temp);
190 + $psets{ $temp{id} } = \%temp;
191 + }
192 +
193 + close ABROWSE or die "$TLA abrowse failed on $limit\n";
194 + }
195 } # end foreach $root
196
197 +do_abrowse(1);
198 +my $depth = 2;
199 +$opt_D ||= 0;
200 +while ($depth <= $opt_D) {
201 + do_abrowse($depth);
202 + $depth++;
203 +}
204 +
205 ## Order patches by time
206 +# FIXME see if we can find a more optimal way to do this by graphing
207 +# the ancestry data and walking it, that way we won't have to rely on
208 +# client-supplied dates
209 @psets = sort {$a->{date}.$b->{id} cmp $b->{date}.$b->{id}} @psets;
210
211 #print Dumper \@psets;
@@ -209,7 +226,7 @@ unless (-d $git_dir) { # initial import
226 }
227 } else { # progressing an import
228 # load the rptags
212 - opendir(DIR, "$git_dir/archimport/tags")
229 + opendir(DIR, $ptag_dir)
230 || die "can't opendir: $!";
231 while (my $file = readdir(DIR)) {
232 # skip non-interesting-files
@@ -829,6 +846,7 @@ sub find_parents {
846 }
847 }
848 }
849 +
850 @parents = ();
851 foreach (keys %parents) {
852 push @parents, '-p', ptag($_);
@@ -856,6 +874,7 @@ sub commitid2pset {
874 return $ps;
875 }
876
877 +
878 # an alterative to `command` that allows input to be passed as an array
879 # to work around shell problems with weird characters in arguments
880 sub safe_pipe_capture {
@@ -869,4 +888,30 @@ sub safe_pipe_capture {
888 return wantarray ? @output : join('',@output);
889 }
890
891 +# `tla logs -rf -d <dir> | head -n1` or `baz tree-id <dir>`
892 +sub arch_tree_id {
893 + my $dir = shift;
894 + chomp( my $ret = (safe_pipe_capture($TLA,'logs','-rf','-d',$dir))[0] );
895 + return $ret;
896 +}
897 +
898 +sub archive_reachable {
899 + my $archive = shift;
900 + return 1 if $reachable{$archive};
901 + return 0 if $unreachable{$archive};
902 +
903 + if (system "$TLA whereis-archive $archive >/dev/null") {
904 + if ($opt_a && (system($TLA,'register-archive',
905 + "http://mirrors.sourcecontrol.net/$archive") == 0)) {
906 + $reachable{$archive} = 1;
907 + return 1;
908 + }
909 + print STDERR "Archive is unreachable: $archive\n";
910 + $unreachable{$archive} = 1;
911 + return 0;
912 + } else {
913 + $reachable{$archive} = 1;
914 + return 1;
915 + }
916 +}
917