A Perforce importer for git.
Signed-off-by: Sean Estabrooks <seanlkml@sympatico.ca> Signed-off-by: Junio C Hamano <junkio@cox.net>
Sean committed
May 24, 2006 at 18:04 UTC
1d84a60459a32fb1707ff7de4a013b5d9673f55d
2 files changed
+522
Documentation/git-p4import.txt
new
+165
@@ -0,0 +1,165 @@
1
+git-p4import(1)
2
+===============
3
+
4
+NAME
5
+----
6
+git-p4import - Import a Perforce repository into git
7
+
8
+
9
+SYNOPSIS
10
+--------
11
+`git-p4import` [-q|-v] [--authors <file>] [-t <timezone>] <//p4repo/path> <branch>
12
+
13
+`git-p4import` --stitch <//p4repo/path>
14
+
15
+`git-p4import`
16
+
17
+
18
+DESCRIPTION
19
+-----------
20
+Import a Perforce repository into an existing git repository. When
21
+a <//p4repo/path> and <branch> are specified a new branch with the
22
+given name will be created and the initial import will begin.
23
+
24
+Once the initial import is complete you can do an incremental import
25
+of new commits from the Perforce repository. You do this by checking
26
+out the appropriate git branch and then running `git-p4import` without
27
+any options.
28
+
29
+The standard p4 client is used to communicate with the Perforce
30
+repository; it must be configured correctly in order for `git-p4import`
31
+to operate (see below).
32
+
33
+
34
+OPTIONS
35
+-------
36
+-q::
37
+ Do not display any progress information.
38
+
39
+-v::
40
+ Give extra progress information.
41
+
42
+\--authors::
43
+ Specify an authors file containing a mapping of Perforce user
44
+ ids to full names and email addresses (see Notes below).
45
+
46
+\--stitch::
47
+ Import the contents of the given perforce branch into the
48
+ currently checked out git branch.
49
+
50
+\--log::
51
+ Store debugging information in the specified file.
52
+
53
+-t::
54
+ Specify that the remote repository is in the specified timezone.
55
+ Timezone must be in the format "US/Pacific" or "Europe/London"
56
+ etc. You only need to specify this once, it will be saved in
57
+ the git config file for the repository.
58
+
59
+<//p4repo/path>::
60
+ The Perforce path that will be imported into the specified branch.
61
+
62
+<branch>::
63
+ The new branch that will be created to hold the Perforce imports.
64
+
65
+
66
+P4 Client
67
+---------
68
+You must make the `p4` client command available in your $PATH and
69
+configure it to communicate with the target Perforce repository.
70
+Typically this means you must set the "$P4PORT" and "$P4CLIENT"
71
+environment variables.
72
+
73
+You must also configure a `p4` client "view" which maps the Perforce
74
+branch into the top level of your git repository, for example:
75
+
76
+------------
77
+Client: myhost
78
+
79
+Root: /home/sean/import
80
+
81
+Options: noallwrite clobber nocompress unlocked modtime rmdir
82
+
83
+View:
84
+ //public/jam/... //myhost/jam/...
85
+------------
86
+
87
+With the above `p4` client setup, you could import the "jam"
88
+perforce branch into a branch named "jammy", like so:
89
+
90
+------------
91
+$ mkdir -p /home/sean/import/jam
92
+$ cd /home/sean/import/jam
93
+$ git init-db
94
+$ git p4import //public/jam jammy
95
+------------
96
+
97
+
98
+Multiple Branches
99
+-----------------
100
+Note that by creating multiple "views" you can use `git-p4import`
101
+to import additional branches into the same git repository.
102
+However, the `p4` client has a limitation in that it silently
103
+ignores all but the last "view" that maps into the same local
104
+directory. So the following will *not* work:
105
+
106
+------------
107
+View:
108
+ //public/jam/... //myhost/jam/...
109
+ //public/other/... //myhost/jam/...
110
+ //public/guest/... //myhost/jam/...
111
+------------
112
+
113
+If you want more than one Perforce branch to be imported into the
114
+same directory you must employ a workaround. A simple option is
115
+to adjust your `p4` client before each import to only include a
116
+single view.
117
+
118
+Another option is to create multiple symlinks locally which all
119
+point to the same directory in your git repository and then use
120
+one per "view" instead of listing the actual directory.
121
+
122
+
123
+Tags
124
+----
125
+A git tag of the form p4/xx is created for every change imported from
126
+the Perforce repository where xx is the Perforce changeset number.
127
+Therefore after the import you can use git to access any commit by its
128
+Perforce number, eg. git show p4/327.
129
+
130
+The tag associated with the HEAD commit is also how `git-p4import`
131
+determines if their are new changes to incrementally import from the
132
+Perforce repository.
133
+
134
+If you import from a repository with many thousands of changes
135
+you will have an equal number of p4/xxxx git tags. Git tags can
136
+be expensive in terms of disk space and repository operations.
137
+If you don't need to perform further incremental imports, you
138
+may delete the tags.
139
+
140
+
141
+Notes
142
+-----
143
+You can interrupt the import (eg. ctrl-c) at any time and restart it
144
+without worry.
145
+
146
+Author information is automatically determined by querying the
147
+Perforce "users" table using the id associated with each change.
148
+However, if you want to manually supply these mappings you can do
149
+so with the "--authors" option. It accepts a file containing a list
150
+of mappings with each line containing one mapping in the format:
151
+
152
+------------
153
+ perforce_id = Full Name <email@address.com>
154
+------------
155
+
156
+
157
+Author
158
+------
159
+Written by Sean Estabrooks <seanlkml@sympatico.ca>
160
+
161
+
162
+GIT
163
+---
164
+Part of the gitlink:git[7] suite
165
+
git-p4import.py
new
+357
@@ -0,0 +1,357 @@
1
+#!/usr/bin/python
2
+#
3
+# This tool is copyright (c) 2006, Sean Estabrooks.
4
+# It is released under the Gnu Public License, version 2.
5
+#
6
+# Import Perforce branches into Git repositories.
7
+# Checking out the files is done by calling the standard p4
8
+# client which you must have properly configured yourself
9
+#
10
+
11
+import marshal
12
+import os
13
+import sys
14
+import time
15
+import getopt
16
+
17
+from signal import signal, \
18
+ SIGPIPE, SIGINT, SIG_DFL, \
19
+ default_int_handler
20
+
21
+signal(SIGPIPE, SIG_DFL)
22
+s = signal(SIGINT, SIG_DFL)
23
+if s != default_int_handler:
24
+ signal(SIGINT, s)
25
+
26
+
27
+def die(msg, *args):
28
+ for a in args:
29
+ msg = "%s %s" % (msg, a)
30
+ print "git-p4import fatal error:", msg
31
+ sys.exit(1)
32
+
33
+def usage():
34
+ print "USAGE: git-p4import [-q|-v] [--authors=<file>] [-t <timezone>] [//p4repo/path <branch>]"
35
+ sys.exit(1)
36
+
37
+verbosity = 1
38
+logfile = "/dev/null"
39
+ignore_warnings = False
40
+stitch = 0
41
+
42
+def report(level, msg, *args):
43
+ global verbosity
44
+ global logfile
45
+ for a in args:
46
+ msg = "%s %s" % (msg, a)
47
+ fd = open(logfile, "a")
48
+ fd.writelines(msg)
49
+ fd.close()
50
+ if level <= verbosity:
51
+ print msg
52
+
53
+class p4_command:
54
+ def __init__(self, _repopath):
55
+ try:
56
+ global logfile
57
+ self.userlist = {}
58
+ if _repopath[-1] == '/':
59
+ self.repopath = _repopath[:-1]
60
+ else:
61
+ self.repopath = _repopath
62
+ if self.repopath[-4:] != "/...":
63
+ self.repopath= "%s/..." % self.repopath
64
+ f=os.popen('p4 -V 2>>%s'%logfile, 'rb')
65
+ a = f.readlines()
66
+ if f.close():
67
+ raise
68
+ except:
69
+ die("Could not find the \"p4\" command")
70
+
71
+ def p4(self, cmd, *args):
72
+ global logfile
73
+ cmd = "%s %s" % (cmd, ' '.join(args))
74
+ report(2, "P4:", cmd)
75
+ f=os.popen('p4 -G %s 2>>%s' % (cmd,logfile), 'rb')
76
+ list = []
77
+ while 1:
78
+ try:
79
+ list.append(marshal.load(f))
80
+ except EOFError:
81
+ break
82
+ self.ret = f.close()
83
+ return list
84
+
85
+ def sync(self, id, force=False, trick=False, test=False):
86
+ if force:
87
+ ret = self.p4("sync -f %s@%s"%(self.repopath, id))[0]
88
+ elif trick:
89
+ ret = self.p4("sync -k %s@%s"%(self.repopath, id))[0]
90
+ elif test:
91
+ ret = self.p4("sync -n %s@%s"%(self.repopath, id))[0]
92
+ else:
93
+ ret = self.p4("sync %s@%s"%(self.repopath, id))[0]
94
+ if ret['code'] == "error":
95
+ data = ret['data'].upper()
96
+ if data.find('VIEW') > 0:
97
+ die("Perforce reports %s is not in client view"% self.repopath)
98
+ elif data.find('UP-TO-DATE') < 0:
99
+ die("Could not sync files from perforce", self.repopath)
100
+
101
+ def changes(self, since=0):
102
+ try:
103
+ list = []
104
+ for rec in self.p4("changes %s@%s,#head" % (self.repopath, since+1)):
105
+ list.append(rec['change'])
106
+ list.reverse()
107
+ return list
108
+ except:
109
+ return []
110
+
111
+ def authors(self, filename):
112
+ f=open(filename)
113
+ for l in f.readlines():
114
+ self.userlist[l[:l.find('=')].rstrip()] = \
115
+ (l[l.find('=')+1:l.find('<')].rstrip(),l[l.find('<')+1:l.find('>')])
116
+ f.close()
117
+ for f,e in self.userlist.items():
118
+ report(2, f, ":", e[0], " <", e[1], ">")
119
+
120
+ def _get_user(self, id):
121
+ if not self.userlist.has_key(id):
122
+ try:
123
+ user = self.p4("users", id)[0]
124
+ self.userlist[id] = (user['FullName'], user['Email'])
125
+ except:
126
+ self.userlist[id] = (id, "")
127
+ return self.userlist[id]
128
+
129
+ def _format_date(self, ticks):
130
+ symbol='+'
131
+ name = time.tzname[0]
132
+ offset = time.timezone
133
+ if ticks[8]:
134
+ name = time.tzname[1]
135
+ offset = time.altzone
136
+ if offset < 0:
137
+ offset *= -1
138
+ symbol = '-'
139
+ localo = "%s%02d%02d %s" % (symbol, offset / 3600, offset % 3600, name)
140
+ tickso = time.strftime("%a %b %d %H:%M:%S %Y", ticks)
141
+ return "%s %s" % (tickso, localo)
142
+
143
+ def where(self):
144
+ try:
145
+ return self.p4("where %s" % self.repopath)[-1]['path']
146
+ except:
147
+ return ""
148
+
149
+ def describe(self, num):
150
+ desc = self.p4("describe -s", num)[0]
151
+ self.msg = desc['desc']
152
+ self.author, self.email = self._get_user(desc['user'])
153
+ self.date = self._format_date(time.localtime(long(desc['time'])))
154
+ return self
155
+
156
+class git_command:
157
+ def __init__(self):
158
+ try:
159
+ self.version = self.git("--version")[0][12:].rstrip()
160
+ except:
161
+ die("Could not find the \"git\" command")
162
+ try:
163
+ self.gitdir = self.get_single("rev-parse --git-dir")
164
+ report(2, "gdir:", self.gitdir)
165
+ except:
166
+ die("Not a git repository... did you forget to \"git init-db\" ?")
167
+ try:
168
+ self.cdup = self.get_single("rev-parse --show-cdup")
169
+ if self.cdup != "":
170
+ os.chdir(self.cdup)
171
+ self.topdir = os.getcwd()
172
+ report(2, "topdir:", self.topdir)
173
+ except:
174
+ die("Could not find top git directory")
175
+
176
+ def git(self, cmd):
177
+ global logfile
178
+ report(2, "GIT:", cmd)
179
+ f=os.popen('git %s 2>>%s' % (cmd,logfile), 'rb')
180
+ r=f.readlines()
181
+ self.ret = f.close()
182
+ return r
183
+
184
+ def get_single(self, cmd):
185
+ return self.git(cmd)[0].rstrip()
186
+
187
+ def current_branch(self):
188
+ try:
189
+ testit = self.git("rev-parse --verify HEAD")[0]
190
+ return self.git("symbolic-ref HEAD")[0][11:].rstrip()
191
+ except:
192
+ return None
193
+
194
+ def get_config(self, variable):
195
+ try:
196
+ return self.git("repo-config --get %s" % variable)[0].rstrip()
197
+ except:
198
+ return None
199
+
200
+ def set_config(self, variable, value):
201
+ try:
202
+ self.git("repo-config %s %s"%(variable, value) )
203
+ except:
204
+ die("Could not set %s to " % variable, value)
205
+
206
+ def make_tag(self, name, head):
207
+ self.git("tag -f %s %s"%(name,head))
208
+
209
+ def top_change(self, branch):
210
+ try:
211
+ a=self.get_single("name-rev --tags refs/heads/%s" % branch)
212
+ loc = a.find(' tags/') + 6
213
+ if a[loc:loc+3] != "p4/":
214
+ raise
215
+ return int(a[loc+3:][:-2])
216
+ except:
217
+ return 0
218
+
219
+ def update_index(self):
220
+ self.git("ls-files -m -d -o -z | git update-index --add --remove -z --stdin")
221
+
222
+ def checkout(self, branch):
223
+ self.git("checkout %s" % branch)
224
+
225
+ def repoint_head(self, branch):
226
+ self.git("symbolic-ref HEAD refs/heads/%s" % branch)
227
+
228
+ def remove_files(self):
229
+ self.git("ls-files | xargs rm")
230
+
231
+ def clean_directories(self):
232
+ self.git("clean -d")
233
+
234
+ def fresh_branch(self, branch):
235
+ report(1, "Creating new branch", branch)
236
+ self.git("ls-files | xargs rm")
237
+ os.remove(".git/index")
238
+ self.repoint_head(branch)
239
+ self.git("clean -d")
240
+
241
+ def basedir(self):
242
+ return self.topdir
243
+
244
+ def commit(self, author, email, date, msg, id):
245
+ self.update_index()
246
+ fd=open(".msg", "w")
247
+ fd.writelines(msg)
248
+ fd.close()
249
+ try:
250
+ current = self.get_single("rev-parse --verify HEAD")
251
+ head = "-p HEAD"
252
+ except:
253
+ current = ""
254
+ head = ""
255
+ tree = self.get_single("write-tree")
256
+ for r,l in [('DATE',date),('NAME',author),('EMAIL',email)]:
257
+ os.environ['GIT_AUTHOR_%s'%r] = l
258
+ os.environ['GIT_COMMITTER_%s'%r] = l
259
+ commit = self.get_single("commit-tree %s %s < .msg" % (tree,head))
260
+ os.remove(".msg")
261
+ self.make_tag("p4/%s"%id, commit)
262
+ self.git("update-ref HEAD %s %s" % (commit, current) )
263
+
264
+
265
+try:
266
+ opts, args = getopt.getopt(sys.argv[1:], "qhvt:",
267
+ ["authors=","help","stitch=","timezone=","log=","ignore"])
268
+except getopt.GetoptError:
269
+ usage()
270
+
271
+for o, a in opts:
272
+ if o == "-q":
273
+ verbosity = 0
274
+ if o == "-v":
275
+ verbosity += 1
276
+ if o in ("--log"):
277
+ logfile = a
278
+ if o in ("-h", "--help"):
279
+ usage()
280
+ if o in ("--ignore"):
281
+ ignore_warnings = True
282
+
283
+git = git_command()
284
+branch=git.current_branch()
285
+
286
+for o, a in opts:
287
+ if o in ("-t", "--timezone"):
288
+ git.set_config("perforce.timezone", a)
289
+ if o in ("--stitch"):
290
+ git.set_config("perforce.%s.path" % branch, a)
291
+ stitch = 1
292
+
293
+if len(args) == 2:
294
+ branch = args[1]
295
+ git.checkout(branch)
296
+ if branch == git.current_branch():
297
+ die("Branch %s already exists!" % branch)
298
+ report(1, "Setting perforce to ", args[0])
299
+ git.set_config("perforce.%s.path" % branch, args[0])
300
+elif len(args) != 0:
301
+ die("You must specify the perforce //depot/path and git branch")
302
+
303
+p4path = git.get_config("perforce.%s.path" % branch)
304
+if p4path == None:
305
+ die("Do not know Perforce //depot/path for git branch", branch)
306
+
307
+p4 = p4_command(p4path)
308
+
309
+for o, a in opts:
310
+ if o in ("-a", "--authors"):
311
+ p4.authors(a)
312
+
313
+localdir = git.basedir()
314
+if p4.where()[:len(localdir)] != localdir:
315
+ report(1, "**WARNING** Appears p4 client is misconfigured")
316
+ report(1, " for sync from %s to %s" % (p4.repopath, localdir))
317
+ if ignore_warnings != True:
318
+ die("Reconfigure or use \"--ignore\" on command line")
319
+
320
+if stitch == 0:
321
+ top = git.top_change(branch)
322
+else:
323
+ top = 0
324
+changes = p4.changes(top)
325
+count = len(changes)
326
+if count == 0:
327
+ report(1, "Already up to date...")
328
+ sys.exit(0)
329
+
330
+ptz = git.get_config("perforce.timezone")
331
+if ptz:
332
+ report(1, "Setting timezone to", ptz)
333
+ os.environ['TZ'] = ptz
334
+ time.tzset()
335
+
336
+if stitch == 1:
337
+ git.remove_files()
338
+ git.clean_directories()
339
+ p4.sync(changes[0], force=True)
340
+elif top == 0 and branch != git.current_branch():
341
+ p4.sync(changes[0], test=True)
342
+ report(1, "Creating new initial commit");
343
+ git.fresh_branch(branch)
344
+ p4.sync(changes[0], force=True)
345
+else:
346
+ p4.sync(changes[0], trick=True)
347
+
348
+report(1, "processing %s changes from p4 (%s) to git (%s)" % (count, p4.repopath, branch))
349
+for id in changes:
350
+ report(1, "Importing changeset", id)
351
+ change = p4.describe(id)
352
+ p4.sync(id)
353
+ git.commit(change.author, change.email, change.date, change.msg, id)
354
+ if stitch == 1:
355
+ git.clean_directories()
356
+ stitch = 0
357
+