archimport: add -D <depth> and -a switch
add -D <depth> option to abrowse add -a switch to attempt to auto-register archives at mirrors.sourcecontrol.net (ML: Also removes some std libraries no longer in use) Signed-off-by: Eric Wong <normalperson@yhbt.net> Signed-off-by: Martin Langhoff <martin@catalyst.net.nz>
Eric Wong committed
Nov 23, 2005 at 23:52 UTC
42f44b08bc1bf84a6e620cf7254b820a9656daca
1 file changed
+135
-90
git-archimport.perl
+135
-90
@@ -53,14 +53,9 @@ and can contain multiple, unrelated branches.
53
use strict;
54
use warnings;
55
use Getopt::Std;
56
-use File::Spec;
57
-use File::Temp qw(tempfile tempdir);
56
+use File::Temp qw(tempdir);
57
use File::Path qw(mkpath rmtree);
58
use File::Basename qw(basename dirname);
60
-use Time::Local;
61
-use IO::Socket;
62
-use IO::Pipe;
63
-use POSIX qw(strftime dup2);
59
use Data::Dumper qw/ Dumper /;
60
use IPC::Open2;
61
@@ -71,27 +66,33 @@ my $git_dir = $ENV{"GIT_DIR"} || ".git";
66
$ENV{"GIT_DIR"} = $git_dir;
67
my $ptag_dir = "$git_dir/archimport/tags";
68
74
-our($opt_h,$opt_v, $opt_T,$opt_t,$opt_o);
69
+our($opt_h,$opt_v,$opt_T,$opt_t,$opt_D,$opt_a,$opt_o);
70
71
sub usage() {
72
print STDERR <<END;
73
Usage: ${\basename $0} # fetch/update GIT from Arch
79
- [ -o ] [ -h ] [ -v ] [ -T ] [ -t tempdir ]
74
+ [ -o ] [ -h ] [ -v ] [ -T ] [ -a ] [ -D depth ] [ -t tempdir ]
75
repository/arch-branch [ repository/arch-branch] ...
76
END
77
exit(1);
78
}
79
85
-getopts("Thvt:") or usage();
80
+getopts("Thvat:D:") or usage();
81
usage if $opt_h;
82
83
@ARGV >= 1 or usage();
89
-my @arch_roots = @ARGV;
84
+# $arch_branches:
85
+# values associated with keys:
86
+# =1 - Arch version / git 'branch' detected via abrowse on a limit
87
+# >1 - Arch version / git 'branch' of an auxilliary branch we've merged
88
+my %arch_branches = map { $_ => 1 } @ARGV;
89
90
$ENV{'TMPDIR'} = $opt_t if $opt_t; # $ENV{TMPDIR} will affect tempdir() calls:
91
my $tmp = tempdir('git-archimport-XXXXXX', TMPDIR => 1, CLEANUP => 1);
92
$opt_v && print "+ Using $tmp as temporary directory\n";
93
94
+my %reachable = (); # Arch repositories we can access
95
+my %unreachable = (); # Arch repositories we can't access :<
96
my @psets = (); # the collection
97
my %psets = (); # the collection, by name
98
@@ -99,96 +100,112 @@ my %rptags = (); # my reverse private tags
100
# to map a SHA1 to a commitid
101
my $TLA = $ENV{'ARCH_CLIENT'} || 'tla';
102
102
-foreach my $root (@arch_roots) {
103
- my ($arepo, $abranch) = split(m!/!, $root);
104
- open ABROWSE, "tla abrowse -f -A $arepo --desc --merges $abranch |"
105
- or die "Problems with tla abrowse: $!";
103
+sub do_abrowse {
104
+ my $stage = shift;
105
+ while (my ($limit, $level) = each %arch_branches) {
106
+ next unless $level == $stage;
107
+
108
+ open ABROWSE, "$TLA abrowse -fkD --merges $limit |"
109
+ or die "Problems with tla abrowse: $!";
110
107
- my %ps = (); # the current one
108
- my $mode = '';
109
- my $lastseen = '';
111
+ my %ps = (); # the current one
112
+ my $lastseen = '';
113
111
- while (<ABROWSE>) {
112
- chomp;
113
-
114
- # first record padded w 8 spaces
115
- if (s/^\s{8}\b//) {
114
+ while (<ABROWSE>) {
115
+ chomp;
116
117
- # store the record we just captured
118
- if (%ps) {
119
- my %temp = %ps; # break references
120
- push (@psets, \%temp);
121
- $psets{$temp{id}} = \%temp;
122
- %ps = ();
123
- }
124
-
125
- my ($id, $type) = split(m/\s{3}/, $_);
126
- $ps{id} = $id;
127
- $ps{repo} = $arepo;
128
-
129
- # deal with types
130
- if ($type =~ m/^\(simple changeset\)/) {
131
- $ps{type} = 's';
132
- } elsif ($type eq '(initial import)') {
133
- $ps{type} = 'i';
134
- } elsif ($type =~ m/^\(tag revision of (.+)\)/) {
135
- $ps{type} = 't';
136
- $ps{tag} = $1;
137
- } else {
138
- warn "Unknown type $type";
139
- }
140
- $lastseen = 'id';
141
- }
142
-
143
- if (s/^\s{10}//) {
144
- # 10 leading spaces or more
145
- # indicate commit metadata
146
-
147
- # date & author
148
- if ($lastseen eq 'id' && m/^\d{4}-\d{2}-\d{2}/) {
117
+ # first record padded w 8 spaces
118
+ if (s/^\s{8}\b//) {
119
+ my ($id, $type) = split(m/\s+/, $_, 2);
120
+
121
+ my %last_ps;
122
+ # store the record we just captured
123
+ if (%ps && !exists $psets{ $ps{id} }) {
124
+ %last_ps = %ps; # break references
125
+ push (@psets, \%last_ps);
126
+ $psets{ $last_ps{id} } = \%last_ps;
127
+ }
128
150
- my ($date, $authoremail) = split(m/\s{2,}/, $_);
151
- $ps{date} = $date;
152
- $ps{date} =~ s/\bGMT$//; # strip off trailign GMT
153
- if ($ps{date} =~ m/\b\w+$/) {
154
- warn 'Arch dates not in GMT?! - imported dates will be wrong';
129
+ my $branch = extract_versionname($id);
130
+ %ps = ( id => $id, branch => $branch );
131
+ if (%last_ps && ($last_ps{branch} eq $branch)) {
132
+ $ps{parent_id} = $last_ps{id};
133
+ }
134
+
135
+ $arch_branches{$branch} = 1;
136
+ $lastseen = 'id';
137
+
138
+ # deal with types (should work with baz or tla):
139
+ if ($type =~ m/\(.*changeset\)/) {
140
+ $ps{type} = 's';
141
+ } elsif ($type =~ /\(.*import\)/) {
142
+ $ps{type} = 'i';
143
+ } elsif ($type =~ m/\(tag.*\)/) {
144
+ $ps{type} = 't';
145
+ # read which revision we've tagged when we parse the log
146
+ #$ps{tag} = $1;
147
+ } else {
148
+ warn "Unknown type $type";
149
+ }
150
+
151
+ $arch_branches{$branch} = 1;
152
+ $lastseen = 'id';
153
+ } elsif (s/^\s{10}//) {
154
+ # 10 leading spaces or more
155
+ # indicate commit metadata
156
+
157
+ # date
158
+ if ($lastseen eq 'id' && m/^(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d)/){
159
+ $ps{date} = $1;
160
+ $lastseen = 'date';
161
+ } elsif ($_ eq 'merges in:') {
162
+ $ps{merges} = [];
163
+ $lastseen = 'merges';
164
+ } elsif ($lastseen eq 'merges' && s/^\s{2}//) {
165
+ my $id = $_;
166
+ push (@{$ps{merges}}, $id);
167
+
168
+ # aggressive branch finding:
169
+ if ($opt_D) {
170
+ my $branch = extract_versionname($id);
171
+ my $repo = extract_reponame($branch);
172
+
173
+ if (archive_reachable($repo) &&
174
+ !defined $arch_branches{$branch}) {
175
+ $arch_branches{$branch} = $stage + 1;
176
+ }
177
+ }
178
+ } else {
179
+ warn "more metadata after merges!?: $_\n" unless /^\s*$/;
180
}
156
-
157
- $authoremail =~ m/^(.+)\s(\S+)$/;
158
- $ps{author} = $1;
159
- $ps{email} = $2;
160
-
161
- $lastseen = 'date';
162
-
163
- } elsif ($lastseen eq 'date') {
164
- # the only hint is position
165
- # subject is after date
166
- $ps{subj} = $_;
167
- $lastseen = 'subj';
168
-
169
- } elsif ($lastseen eq 'subj' && $_ eq 'merges in:') {
170
- $ps{merges} = [];
171
- $lastseen = 'merges';
172
-
173
- } elsif ($lastseen eq 'merges' && s/^\s{2}//) {
174
- push (@{$ps{merges}}, $_);
175
- } else {
176
- warn 'more metadata after merges!?';
181
}
178
-
182
}
180
- }
183
182
- if (%ps) {
183
- my %temp = %ps; # break references
184
- push (@psets, \%temp);
185
- $psets{ $temp{id} } = \%temp;
186
- %ps = ();
187
- }
188
- close ABROWSE;
184
+ if (%ps && !exists $psets{ $ps{id} }) {
185
+ my %temp = %ps; # break references
186
+ if (@psets && $psets[$#psets]{branch} eq $ps{branch}) {
187
+ $temp{parent_id} = $psets[$#psets]{id};
188
+ }
189
+ push (@psets, \%temp);
190
+ $psets{ $temp{id} } = \%temp;
191
+ }
192
+
193
+ close ABROWSE or die "$TLA abrowse failed on $limit\n";
194
+ }
195
} # end foreach $root
196
197
+do_abrowse(1);
198
+my $depth = 2;
199
+$opt_D ||= 0;
200
+while ($depth <= $opt_D) {
201
+ do_abrowse($depth);
202
+ $depth++;
203
+}
204
+
205
## Order patches by time
206
+# FIXME see if we can find a more optimal way to do this by graphing
207
+# the ancestry data and walking it, that way we won't have to rely on
208
+# client-supplied dates
209
@psets = sort {$a->{date}.$b->{id} cmp $b->{date}.$b->{id}} @psets;
210
211
#print Dumper \@psets;
@@ -209,7 +226,7 @@ unless (-d $git_dir) { # initial import
226
}
227
} else { # progressing an import
228
# load the rptags
212
- opendir(DIR, "$git_dir/archimport/tags")
229
+ opendir(DIR, $ptag_dir)
230
|| die "can't opendir: $!";
231
while (my $file = readdir(DIR)) {
232
# skip non-interesting-files
@@ -829,6 +846,7 @@ sub find_parents {
846
}
847
}
848
}
849
+
850
@parents = ();
851
foreach (keys %parents) {
852
push @parents, '-p', ptag($_);
@@ -856,6 +874,7 @@ sub commitid2pset {
874
return $ps;
875
}
876
877
+
878
# an alterative to `command` that allows input to be passed as an array
879
# to work around shell problems with weird characters in arguments
880
sub safe_pipe_capture {
@@ -869,4 +888,30 @@ sub safe_pipe_capture {
888
return wantarray ? @output : join('',@output);
889
}
890
891
+# `tla logs -rf -d <dir> | head -n1` or `baz tree-id <dir>`
892
+sub arch_tree_id {
893
+ my $dir = shift;
894
+ chomp( my $ret = (safe_pipe_capture($TLA,'logs','-rf','-d',$dir))[0] );
895
+ return $ret;
896
+}
897
+
898
+sub archive_reachable {
899
+ my $archive = shift;
900
+ return 1 if $reachable{$archive};
901
+ return 0 if $unreachable{$archive};
902
+
903
+ if (system "$TLA whereis-archive $archive >/dev/null") {
904
+ if ($opt_a && (system($TLA,'register-archive',
905
+ "http://mirrors.sourcecontrol.net/$archive") == 0)) {
906
+ $reachable{$archive} = 1;
907
+ return 1;
908
+ }
909
+ print STDERR "Archive is unreachable: $archive\n";
910
+ $unreachable{$archive} = 1;
911
+ return 0;
912
+ } else {
913
+ $reachable{$archive} = 1;
914
+ return 1;
915
+ }
916
+}
917