@matheusbsbs / matheus-bagliano / commits / 60df071c6c

git-p4: add initial support for RCS keywords

RCS keywords cause problems for git-p4 as perforce always expands them (if +k is set) and so when applying the patch, git reports that the files have been modified by both sides, when in fact they haven't. This change means that when git-p4 detects a problem applying a patch, it will check to see if keyword expansion could be the culprit. If it is, it strips the keywords in the p4 repository so that they match what git is expecting. It then has another go at applying the patch. This behaviour is enabled with a new git-p4 configuration option and is off by default. Acked-by: Pete Wyckoff <pw@padd.com> Signed-off-by: Luke Diamand <luke@diamand.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Luke Diamand committed Feb 23, 2012 at 07:51 UTC 60df071c6cce6ee3f92f62042da11c62fe113817
3 files changed +501 -10
Documentation/git-p4.txt
+5
@@ -483,6 +483,11 @@ git-p4.skipUserNameCheck::
483 user map, 'git p4' exits. This option can be used to force
484 submission regardless.
485
486 +git-p4.attemptRCSCleanup:
487 + If enabled, 'git p4 submit' will attempt to cleanup RCS keywords
488 + ($Header$, etc). These would otherwise cause merge conflicts and prevent
489 + the submit going ahead. This option should be considered experimental at
490 + present.
491
492 IMPLEMENTATION DETAILS
493 ----------------------
contrib/fast-import/git-p4
+108 -10
@@ -10,7 +10,7 @@
10
11 import optparse, sys, os, marshal, subprocess, shelve
12 import tempfile, getopt, os.path, time, platform
13 -import re
13 +import re, shutil
14
15 verbose = False
16
@@ -186,6 +186,47 @@ def split_p4_type(p4type):
186 mods = s[1]
187 return (base, mods)
188
189 +#
190 +# return the raw p4 type of a file (text, text+ko, etc)
191 +#
192 +def p4_type(file):
193 + results = p4CmdList(["fstat", "-T", "headType", file])
194 + return results[0]['headType']
195 +
196 +#
197 +# Given a type base and modifier, return a regexp matching
198 +# the keywords that can be expanded in the file
199 +#
200 +def p4_keywords_regexp_for_type(base, type_mods):
201 + if base in ("text", "unicode", "binary"):
202 + kwords = None
203 + if "ko" in type_mods:
204 + kwords = 'Id|Header'
205 + elif "k" in type_mods:
206 + kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
207 + else:
208 + return None
209 + pattern = r"""
210 + \$ # Starts with a dollar, followed by...
211 + (%s) # one of the keywords, followed by...
212 + (:[^$]+)? # possibly an old expansion, followed by...
213 + \$ # another dollar
214 + """ % kwords
215 + return pattern
216 + else:
217 + return None
218 +
219 +#
220 +# Given a file, return a regexp matching the possible
221 +# RCS keywords that will be expanded, or None for files
222 +# with kw expansion turned off.
223 +#
224 +def p4_keywords_regexp_for_file(file):
225 + if not os.path.exists(file):
226 + return None
227 + else:
228 + (type_base, type_mods) = split_p4_type(p4_type(file))
229 + return p4_keywords_regexp_for_type(type_base, type_mods)
230
231 def setP4ExecBit(file, mode):
232 # Reopens an already open file and changes the execute bit to match
@@ -753,6 +794,29 @@ class P4Submit(Command, P4UserMap):
794
795 return result
796
797 + def patchRCSKeywords(self, file, pattern):
798 + # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
799 + (handle, outFileName) = tempfile.mkstemp(dir='.')
800 + try:
801 + outFile = os.fdopen(handle, "w+")
802 + inFile = open(file, "r")
803 + regexp = re.compile(pattern, re.VERBOSE)
804 + for line in inFile.readlines():
805 + line = regexp.sub(r'$\1$', line)
806 + outFile.write(line)
807 + inFile.close()
808 + outFile.close()
809 + # Forcibly overwrite the original file
810 + os.unlink(file)
811 + shutil.move(outFileName, file)
812 + except:
813 + # cleanup our temporary file
814 + os.unlink(outFileName)
815 + print "Failed to strip RCS keywords in %s" % file
816 + raise
817 +
818 + print "Patched up RCS keywords in %s" % file
819 +
820 def p4UserForCommit(self,id):
821 # Return the tuple (perforce user,git email) for a given git commit id
822 self.getUserMapFromPerforceServer()
@@ -918,6 +982,7 @@ class P4Submit(Command, P4UserMap):
982 filesToDelete = set()
983 editedFiles = set()
984 filesToChangeExecBit = {}
985 +
986 for line in diff:
987 diff = parseDiffTreeEntry(line)
988 modifier = diff['status']
@@ -964,9 +1029,45 @@ class P4Submit(Command, P4UserMap):
1029 patchcmd = diffcmd + " | git apply "
1030 tryPatchCmd = patchcmd + "--check -"
1031 applyPatchCmd = patchcmd + "--check --apply -"
1032 + patch_succeeded = True
1033
1034 if os.system(tryPatchCmd) != 0:
1035 + fixed_rcs_keywords = False
1036 + patch_succeeded = False
1037 print "Unfortunately applying the change failed!"
1038 +
1039 + # Patch failed, maybe it's just RCS keyword woes. Look through
1040 + # the patch to see if that's possible.
1041 + if gitConfig("git-p4.attemptRCSCleanup","--bool") == "true":
1042 + file = None
1043 + pattern = None
1044 + kwfiles = {}
1045 + for file in editedFiles | filesToDelete:
1046 + # did this file's delta contain RCS keywords?
1047 + pattern = p4_keywords_regexp_for_file(file)
1048 +
1049 + if pattern:
1050 + # this file is a possibility...look for RCS keywords.
1051 + regexp = re.compile(pattern, re.VERBOSE)
1052 + for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1053 + if regexp.search(line):
1054 + if verbose:
1055 + print "got keyword match on %s in %s in %s" % (pattern, line, file)
1056 + kwfiles[file] = pattern
1057 + break
1058 +
1059 + for file in kwfiles:
1060 + if verbose:
1061 + print "zapping %s with %s" % (line,pattern)
1062 + self.patchRCSKeywords(file, kwfiles[file])
1063 + fixed_rcs_keywords = True
1064 +
1065 + if fixed_rcs_keywords:
1066 + print "Retrying the patch with RCS keywords cleaned up"
1067 + if os.system(tryPatchCmd) == 0:
1068 + patch_succeeded = True
1069 +
1070 + if not patch_succeeded:
1071 print "What do you want to do?"
1072 response = "x"
1073 while response != "s" and response != "a" and response != "w":
@@ -1585,15 +1686,12 @@ class P4Sync(Command, P4UserMap):
1686
1687 # Note that we do not try to de-mangle keywords on utf16 files,
1688 # even though in theory somebody may want that.
1588 - if type_base in ("text", "unicode", "binary"):
1589 - if "ko" in type_mods:
1590 - text = ''.join(contents)
1591 - text = re.sub(r'\$(Id|Header):[^$]*\$', r'$\1$', text)
1592 - contents = [ text ]
1593 - elif "k" in type_mods:
1594 - text = ''.join(contents)
1595 - text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$', r'$\1$', text)
1596 - contents = [ text ]
1689 + pattern = p4_keywords_regexp_for_type(type_base, type_mods)
1690 + if pattern:
1691 + regexp = re.compile(pattern, re.VERBOSE)
1692 + text = ''.join(contents)
1693 + text = regexp.sub(r'$\1$', text)
1694 + contents = [ text ]
1695
1696 self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
1697
t/t9810-git-p4-rcs.sh new
+388
@@ -0,0 +1,388 @@
1 +#!/bin/sh
2 +
3 +test_description='git-p4 rcs keywords'
4 +
5 +. ./lib-git-p4.sh
6 +
7 +test_expect_success 'start p4d' '
8 + start_p4d
9 +'
10 +
11 +#
12 +# Make one file with keyword lines at the top, and
13 +# enough plain text to be able to test modifications
14 +# far away from the keywords.
15 +#
16 +test_expect_success 'init depot' '
17 + (
18 + cd "$cli" &&
19 + cat <<-\EOF >filek &&
20 + $Id$
21 + /* $Revision$ */
22 + # $Change$
23 + line4
24 + line5
25 + line6
26 + line7
27 + line8
28 + EOF
29 + cp filek fileko &&
30 + sed -i "s/Revision/Revision: do not scrub me/" fileko
31 + cp fileko file_text &&
32 + sed -i "s/Id/Id: do not scrub me/" file_text
33 + p4 add -t text+k filek &&
34 + p4 submit -d "filek" &&
35 + p4 add -t text+ko fileko &&
36 + p4 submit -d "fileko" &&
37 + p4 add -t text file_text &&
38 + p4 submit -d "file_text"
39 + )
40 +'
41 +
42 +#
43 +# Generate these in a function to make it easy to use single quote marks.
44 +#
45 +write_scrub_scripts () {
46 + cat >"$TRASH_DIRECTORY/scrub_k.py" <<-\EOF &&
47 + import re, sys
48 + sys.stdout.write(re.sub(r'(?i)\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$', r'$\1$', sys.stdin.read()))
49 + EOF
50 + cat >"$TRASH_DIRECTORY/scrub_ko.py" <<-\EOF
51 + import re, sys
52 + sys.stdout.write(re.sub(r'(?i)\$(Id|Header):[^$]*\$', r'$\1$', sys.stdin.read()))
53 + EOF
54 +}
55 +
56 +test_expect_success 'scrub scripts' '
57 + write_scrub_scripts
58 +'
59 +
60 +#
61 +# Compare $cli/file to its scrubbed version, should be different.
62 +# Compare scrubbed $cli/file to $git/file, should be same.
63 +#
64 +scrub_k_check () {
65 + file="$1" &&
66 + scrub="$TRASH_DIRECTORY/$file" &&
67 + "$PYTHON_PATH" "$TRASH_DIRECTORY/scrub_k.py" <"$git/$file" >"$scrub" &&
68 + ! test_cmp "$cli/$file" "$scrub" &&
69 + test_cmp "$git/$file" "$scrub" &&
70 + rm "$scrub"
71 +}
72 +scrub_ko_check () {
73 + file="$1" &&
74 + scrub="$TRASH_DIRECTORY/$file" &&
75 + "$PYTHON_PATH" "$TRASH_DIRECTORY/scrub_ko.py" <"$git/$file" >"$scrub" &&
76 + ! test_cmp "$cli/$file" "$scrub" &&
77 + test_cmp "$git/$file" "$scrub" &&
78 + rm "$scrub"
79 +}
80 +
81 +#
82 +# Modify far away from keywords. If no RCS lines show up
83 +# in the diff, there is no conflict.
84 +#
85 +test_expect_success 'edit far away from RCS lines' '
86 + test_when_finished cleanup_git &&
87 + "$GITP4" clone --dest="$git" //depot &&
88 + (
89 + cd "$git" &&
90 + git config git-p4.skipSubmitEdit true &&
91 + sed -i "s/^line7/line7 edit/" filek &&
92 + git commit -m "filek line7 edit" filek &&
93 + "$GITP4" submit &&
94 + scrub_k_check filek
95 + )
96 +'
97 +
98 +#
99 +# Modify near the keywords. This will require RCS scrubbing.
100 +#
101 +test_expect_success 'edit near RCS lines' '
102 + test_when_finished cleanup_git &&
103 + "$GITP4" clone --dest="$git" //depot &&
104 + (
105 + cd "$git" &&
106 + git config git-p4.skipSubmitEdit true &&
107 + git config git-p4.attemptRCSCleanup true &&
108 + sed -i "s/^line4/line4 edit/" filek &&
109 + git commit -m "filek line4 edit" filek &&
110 + "$GITP4" submit &&
111 + scrub_k_check filek
112 + )
113 +'
114 +
115 +#
116 +# Modify the keywords themselves. This also will require RCS scrubbing.
117 +#
118 +test_expect_success 'edit keyword lines' '
119 + test_when_finished cleanup_git &&
120 + "$GITP4" clone --dest="$git" //depot &&
121 + (
122 + cd "$git" &&
123 + git config git-p4.skipSubmitEdit true &&
124 + git config git-p4.attemptRCSCleanup true &&
125 + sed -i "/Revision/d" filek &&
126 + git commit -m "filek remove Revision line" filek &&
127 + "$GITP4" submit &&
128 + scrub_k_check filek
129 + )
130 +'
131 +
132 +#
133 +# Scrubbing text+ko files should not alter all keywords, just Id, Header.
134 +#
135 +test_expect_success 'scrub ko files differently' '
136 + test_when_finished cleanup_git &&
137 + "$GITP4" clone --dest="$git" //depot &&
138 + (
139 + cd "$git" &&
140 + git config git-p4.skipSubmitEdit true &&
141 + git config git-p4.attemptRCSCleanup true &&
142 + sed -i "s/^line4/line4 edit/" fileko &&
143 + git commit -m "fileko line4 edit" fileko &&
144 + "$GITP4" submit &&
145 + scrub_ko_check fileko &&
146 + ! scrub_k_check fileko
147 + )
148 +'
149 +
150 +# hack; git-p4 submit should do it on its own
151 +test_expect_success 'cleanup after failure' '
152 + (
153 + cd "$cli" &&
154 + p4 revert ...
155 + )
156 +'
157 +
158 +#
159 +# Do not scrub anything but +k or +ko files. Sneak a change into
160 +# the cli file so that submit will get a conflict. Make sure that
161 +# scrubbing doesn't make a mess of things.
162 +#
163 +# Assumes that git-p4 exits leaving the p4 file open, with the
164 +# conflict-generating patch unapplied.
165 +#
166 +# This might happen only if the git repo is behind the p4 repo at
167 +# submit time, and there is a conflict.
168 +#
169 +test_expect_success 'do not scrub plain text' '
170 + test_when_finished cleanup_git &&
171 + "$GITP4" clone --dest="$git" //depot &&
172 + (
173 + cd "$git" &&
174 + git config git-p4.skipSubmitEdit true &&
175 + git config git-p4.attemptRCSCleanup true &&
176 + sed -i "s/^line4/line4 edit/" file_text &&
177 + git commit -m "file_text line4 edit" file_text &&
178 + (
179 + cd "$cli" &&
180 + p4 open file_text &&
181 + sed -i "s/^line5/line5 p4 edit/" file_text &&
182 + p4 submit -d "file5 p4 edit"
183 + ) &&
184 + ! "$GITP4" submit &&
185 + (
186 + # exepct something like:
187 + # file_text - file(s) not opened on this client
188 + # but not copious diff output
189 + cd "$cli" &&
190 + p4 diff file_text >wc &&
191 + test_line_count = 1 wc
192 + )
193 + )
194 +'
195 +
196 +# hack; git-p4 submit should do it on its own
197 +test_expect_success 'cleanup after failure 2' '
198 + (
199 + cd "$cli" &&
200 + p4 revert ...
201 + )
202 +'
203 +
204 +create_kw_file () {
205 + cat <<\EOF >"$1"
206 +/* A file
207 + Id: $Id$
208 + Revision: $Revision$
209 + File: $File$
210 + */
211 +int main(int argc, const char **argv) {
212 + return 0;
213 +}
214 +EOF
215 +}
216 +
217 +test_expect_success 'add kwfile' '
218 + (
219 + cd "$cli" &&
220 + echo file1 >file1 &&
221 + p4 add file1 &&
222 + p4 submit -d "file 1" &&
223 + create_kw_file kwfile1.c &&
224 + p4 add kwfile1.c &&
225 + p4 submit -d "Add rcw kw file" kwfile1.c
226 + )
227 +'
228 +
229 +p4_append_to_file () {
230 + f="$1" &&
231 + p4 edit -t ktext "$f" &&
232 + echo "/* $(date) */" >>"$f" &&
233 + p4 submit -d "appending a line in p4"
234 +}
235 +
236 +# Create some files with RCS keywords. If they get modified
237 +# elsewhere then the version number gets bumped which then
238 +# results in a merge conflict if we touch the RCS kw lines,
239 +# even though the change itself would otherwise apply cleanly.
240 +test_expect_success 'cope with rcs keyword expansion damage' '
241 + test_when_finished cleanup_git &&
242 + "$GITP4" clone --dest="$git" //depot &&
243 + (
244 + cd "$git" &&
245 + git config git-p4.skipSubmitEdit true &&
246 + git config git-p4.attemptRCSCleanup true &&
247 + (cd ../cli && p4_append_to_file kwfile1.c) &&
248 + old_lines=$(wc -l <kwfile1.c) &&
249 + perl -n -i -e "print unless m/Revision:/" kwfile1.c &&
250 + new_lines=$(wc -l <kwfile1.c) &&
251 + test $new_lines = $(($old_lines - 1)) &&
252 +
253 + git add kwfile1.c &&
254 + git commit -m "Zap an RCS kw line" &&
255 + "$GITP4" submit &&
256 + "$GITP4" rebase &&
257 + git diff p4/master &&
258 + "$GITP4" commit &&
259 + echo "try modifying in both" &&
260 + cd "$cli" &&
261 + p4 edit kwfile1.c &&
262 + echo "line from p4" >>kwfile1.c &&
263 + p4 submit -d "add a line in p4" kwfile1.c &&
264 + cd "$git" &&
265 + echo "line from git at the top" | cat - kwfile1.c >kwfile1.c.new &&
266 + mv kwfile1.c.new kwfile1.c &&
267 + git commit -m "Add line in git at the top" kwfile1.c &&
268 + "$GITP4" rebase &&
269 + "$GITP4" submit
270 + )
271 +'
272 +
273 +test_expect_success 'cope with rcs keyword file deletion' '
274 + test_when_finished cleanup_git &&
275 + (
276 + cd "$cli" &&
277 + echo "\$Revision\$" >kwdelfile.c &&
278 + p4 add -t ktext kwdelfile.c &&
279 + p4 submit -d "Add file to be deleted" &&
280 + cat kwdelfile.c &&
281 + grep 1 kwdelfile.c
282 + ) &&
283 + "$GITP4" clone --dest="$git" //depot &&
284 + (
285 + cd "$git" &&
286 + grep Revision kwdelfile.c &&
287 + git rm -f kwdelfile.c &&
288 + git commit -m "Delete a file containing RCS keywords" &&
289 + git config git-p4.skipSubmitEdit true &&
290 + git config git-p4.attemptRCSCleanup true &&
291 + "$GITP4" submit
292 + ) &&
293 + (
294 + cd "$cli" &&
295 + p4 sync &&
296 + ! test -f kwdelfile.c
297 + )
298 +'
299 +
300 +# If you add keywords in git of the form $Header$ then everything should
301 +# work fine without any special handling.
302 +test_expect_success 'Add keywords in git which match the default p4 values' '
303 + test_when_finished cleanup_git &&
304 + "$GITP4" clone --dest="$git" //depot &&
305 + (
306 + cd "$git" &&
307 + echo "NewKW: \$Revision\$" >>kwfile1.c &&
308 + git add kwfile1.c &&
309 + git commit -m "Adding RCS keywords in git" &&
310 + git config git-p4.skipSubmitEdit true &&
311 + git config git-p4.attemptRCSCleanup true &&
312 + "$GITP4" submit
313 + ) &&
314 + (
315 + cd "$cli" &&
316 + p4 sync &&
317 + test -f kwfile1.c &&
318 + grep "NewKW.*Revision.*[0-9]" kwfile1.c
319 +
320 + )
321 +'
322 +
323 +# If you add keywords in git of the form $Header:#1$ then things will fail
324 +# unless git-p4 takes steps to scrub the *git* commit.
325 +#
326 +test_expect_failure 'Add keywords in git which do not match the default p4 values' '
327 + test_when_finished cleanup_git &&
328 + "$GITP4" clone --dest="$git" //depot &&
329 + (
330 + cd "$git" &&
331 + echo "NewKW2: \$Revision:1\$" >>kwfile1.c &&
332 + git add kwfile1.c &&
333 + git commit -m "Adding RCS keywords in git" &&
334 + git config git-p4.skipSubmitEdit true &&
335 + git config git-p4.attemptRCSCleanup true &&
336 + "$GITP4" submit
337 + ) &&
338 + (
339 + cd "$cli" &&
340 + p4 sync &&
341 + grep "NewKW2.*Revision.*[0-9]" kwfile1.c
342 +
343 + )
344 +'
345 +
346 +# Check that the existing merge conflict handling still works.
347 +# Modify kwfile1.c in git, and delete in p4. We should be able
348 +# to skip the git commit.
349 +#
350 +test_expect_success 'merge conflict handling still works' '
351 + test_when_finished cleanup_git &&
352 + (
353 + cd "$cli" &&
354 + echo "Hello:\$Id\$" >merge2.c &&
355 + echo "World" >>merge2.c &&
356 + p4 add -t ktext merge2.c &&
357 + p4 submit -d "add merge test file"
358 + ) &&
359 + "$GITP4" clone --dest="$git" //depot &&
360 + (
361 + cd "$git" &&
362 + sed -e "/Hello/d" merge2.c >merge2.c.tmp &&
363 + mv merge2.c.tmp merge2.c &&
364 + git add merge2.c &&
365 + git commit -m "Modifying merge2.c"
366 + ) &&
367 + (
368 + cd "$cli" &&
369 + p4 delete merge2.c &&
370 + p4 submit -d "remove merge test file"
371 + ) &&
372 + (
373 + cd "$git" &&
374 + test -f merge2.c &&
375 + git config git-p4.skipSubmitEdit true &&
376 + git config git-p4.attemptRCSCleanup true &&
377 + !(echo "s" | "$GITP4" submit) &&
378 + git rebase --skip &&
379 + ! test -f merge2.c
380 + )
381 +'
382 +
383 +
384 +test_expect_success 'kill p4d' '
385 + kill_p4d
386 +'
387 +
388 +test_done