@matheusbsbs / matheus-bagliano / commits / 70c0d55349

git-p4: resolve RCS keywords in bytes not utf-8

RCS keywords are strings that are replaced with information from Perforce. Examples include $Date$, $Author$, $File$, $Change$ etc. Perforce resolves these by expanding them with their expanded values when files are synced, but Git's data model requires these expanded values to be converted back into their unexpanded form. Previously, git-p4.py would implement this behaviour through the use of regular expressions. However, the regular expression substitution was applied using decoded strings i.e. the content of incoming commit diffs was first decoded from bytes into UTF-8, processed with regular expressions, then converted back to bytes. Not only is this behaviour inefficient, but it is also a cause of a common issue caused by text files containing invalid UTF-8 data. For files created in Windows, CP1252 Smart Quote Characters (0x93 and 0x94) are seen fairly frequently. These codes are invalid in UTF-8, so if the script encountered any file containing them, on Python 2 the symbols will be corrupted, and on Python 3 the script will fail with an exception. This patch replaces this decoding/encoding with bytes object regular expressions, so that the substitution is performed directly upon the source data with no conversions. A test for smart quote handling has been added to the t9810-git-p4-rcs.sh test suite. Signed-off-by: Joel Holdsworth <jholdsworth@nvidia.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Joel Holdsworth committed Dec 16, 2021 at 13:46 UTC 70c0d55349a50707166f9fb9a9720ac1c0530217

2 files changed +23 -7

git-p4.py

+8 -7

@@ -56,8 +56,8 @@ defaultBlockSize = 1<<20

57 p4_access_checked = False

59 -re_ko_keywords = re.compile(r'\$(Id|Header)(:[^$\n]+)?\$')

59 +re_ko_keywords = re.compile(br'\$(Id|Header)(:[^$\n]+)?\$')

62 def p4_build_cmd(cmd):

63 """Build a suitable p4 command line.

@@ -1754,9 +1754,9 @@ class P4Submit(Command, P4UserMap):

1754 # Attempt to zap the RCS keywords in a p4 controlled file matching the given regex

1755 (handle, outFileName) = tempfile.mkstemp(dir='.')

1756 try:

1757 - with os.fdopen(handle, "w") as outFile, open(file, "r") as inFile:

1757 + with os.fdopen(handle, "wb") as outFile, open(file, "rb") as inFile:

1758 for line in inFile.readlines():

1759 - outFile.write(regexp.sub(r'$\1$', line))

1759 + outFile.write(regexp.sub(br'$\1$', line))

1760 # Forcibly overwrite the original file

1761 os.unlink(file)

1762 shutil.move(outFileName, file)

@@ -2089,7 +2089,9 @@ class P4Submit(Command, P4UserMap):

2089 regexp = p4_keywords_regexp_for_file(file)

2090 if regexp:

2091 # this file is a possibility...look for RCS keywords.

2092 - for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):

2092 + for line in read_pipe_lines(

2093 + ["git", "diff", "%s^..%s" % (id, id), file],

2094 + raw=True):

2095 if regexp.search(line):

2096 if verbose:

2097 print("got keyword match on %s in %s in %s" % (regex.pattern, line, file))

@@ -3020,8 +3022,7 @@ class P4Sync(Command, P4UserMap):

3022 # even though in theory somebody may want that.

3023 regexp = p4_keywords_regexp_for_type(type_base, type_mods)

3024 if regexp:

3023 - contents = [encode_text_stream(regexp.sub(

3024 - r'$\1$', ''.join(decode_text_stream(c) for c in contents)))]

3025 + contents = [regexp.sub(br'$\1$', c) for c in contents]

3026

3027 if self.largeFileSystem:

3028 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)

t/t9810-git-p4-rcs.sh

+15

@@ -4,6 +4,8 @@ test_description='git p4 rcs keywords'

5 . ./lib-git-p4.sh

7 +CP1252="\223\224"

8 +

9 test_expect_success 'start p4d' '

10 start_p4d

11 '

@@ -32,6 +34,9 @@ test_expect_success 'init depot' '

34 p4 submit -d "filek" &&

35 p4 add -t text+ko fileko &&

36 p4 submit -d "fileko" &&

37 + printf "$CP1252" >fileko_cp1252 &&

38 + p4 add -t text+ko fileko_cp1252 &&

39 + p4 submit -d "fileko_cp1252" &&

40 p4 add -t text file_text &&

41 p4 submit -d "file_text"

42 )

@@ -359,4 +364,14 @@ test_expect_failure 'Add keywords in git which do not match the default p4 value

364 )

365 '

366

367 +test_expect_success 'check cp1252 smart quote are preserved through RCS keyword processing' '

368 + test_when_finished cleanup_git &&

369 + git p4 clone --dest="$git" //depot &&

370 + (

371 + cd "$git" &&

372 + printf "$CP1252" >expect &&

373 + test_cmp_bin expect fileko_cp1252

374 + )

375 +'

376 +

377 test_done