@matheusbsbs / matheus-bagliano / commits / d38208a297

Fork 0

git-p4: convert path to unicode before processing them

P4 allows essentially arbitrary encoding for path data while we would perfer to be dealing only with unicode strings. Since path data need to survive round-trip back to p4, this patch implements the general policy that we store path data as-is, but decode them to unicode before doing any non-trivial processing. A new `decode_path()` method is provided that generally does the correct conversion, taking into account `git-p4.pathEncoding` configuration. For python2.7, path strings will be left as-is if it only contains ASCII characters. For python3, decoding is always done so that we have str objects. Signed-off-by: Yang Zhao <yang.zhao@skyboxlabs.com> Reviewed-by: Ben Keene <seraphire@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Yang Zhao committed Dec 13, 2019 at 15:52 UTC d38208a297e76bbfbfa8e485632c217aaafa9486

1 file changed +44 -25

git-p4.py

+44 -25

@@ -150,6 +150,21 @@ else:

150 def encode_text_stream(s):

151 return s.encode('utf_8') if isinstance(s, unicode) else s

152

153 +def decode_path(path):

154 + """Decode a given string (bytes or otherwise) using configured path encoding options

155 + """

156 + encoding = gitConfig('git-p4.pathEncoding') or 'utf_8'

157 + if bytes is not str:

158 + return path.decode(encoding, errors='replace') if isinstance(path, bytes) else path

159 + else:

160 + try:

161 + path.decode('ascii')

162 + except:

163 + path = path.decode(encoding, errors='replace')

164 + if verbose:

165 + print('Path with non-ASCII characters detected. Used {} to decode: {}'.format(encoding, path))

166 + return path

167 +

168 def write_pipe(c, stdin):

169 if verbose:

170 sys.stderr.write('Writing pipe: %s\n' % str(c))

@@ -697,7 +712,8 @@ def p4Where(depotPath):

712 if "depotFile" in entry:

713 # Search for the base client side depot path, as long as it starts with the branch's P4 path.

714 # The base path always ends with "/...".

700 - if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":

715 + entry_path = decode_path(entry['depotFile'])

716 + if entry_path.find(depotPath) == 0 and entry_path[-4:] == "/...":

717 output = entry

718 break

719 elif "data" in entry:

@@ -712,11 +728,11 @@ def p4Where(depotPath):

728 return ""

729 clientPath = ""

730 if "path" in output:

715 - clientPath = output.get("path")

731 + clientPath = decode_path(output['path'])

732 elif "data" in output:

733 data = output.get("data")

718 - lastSpace = data.rfind(" ")

719 - clientPath = data[lastSpace + 1:]

734 + lastSpace = data.rfind(b" ")

735 + clientPath = decode_path(data[lastSpace + 1:])

736

737 if clientPath.endswith("..."):

738 clientPath = clientPath[:-3]

@@ -2484,7 +2500,7 @@ class View(object):

2500

2501 def convert_client_path(self, clientFile):

2502 # chop off //client/ part to make it relative

2487 - if not clientFile.startswith(self.client_prefix):

2503 + if not decode_path(clientFile).startswith(self.client_prefix):

2504 die("No prefix '%s' on clientFile '%s'" %

2505 (self.client_prefix, clientFile))

2506 return clientFile[len(self.client_prefix):]

@@ -2493,7 +2509,7 @@ class View(object):

2509 """ Caching file paths by "p4 where" batch query """

2510

2511 # List depot file paths exclude that already cached

2496 - fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]

2512 + fileArgs = [f['path'] for f in files if decode_path(f['path']) not in self.client_spec_path_cache]

2513

2514 if len(fileArgs) == 0:

2515 return # All files in cache

@@ -2508,16 +2524,18 @@ class View(object):

2524 if "unmap" in res:

2525 # it will list all of them, but only one not unmap-ped

2526 continue

2527 + depot_path = decode_path(res['depotFile'])

2528 if gitConfigBool("core.ignorecase"):

2512 - res['depotFile'] = res['depotFile'].lower()

2513 - self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])

2529 + depot_path = depot_path.lower()

2530 + self.client_spec_path_cache[depot_path] = self.convert_client_path(res["clientFile"])

2531

2532 # not found files or unmap files set to ""

2533 for depotFile in fileArgs:

2534 + depotFile = decode_path(depotFile)

2535 if gitConfigBool("core.ignorecase"):

2536 depotFile = depotFile.lower()

2537 if depotFile not in self.client_spec_path_cache:

2520 - self.client_spec_path_cache[depotFile] = ""

2538 + self.client_spec_path_cache[depotFile] = b''

2539

2540 def map_in_client(self, depot_path):

2541 """Return the relative location in the client where this

@@ -2635,7 +2653,7 @@ class P4Sync(Command, P4UserMap):

2653 elif path.lower() == p.lower():

2654 return False

2655 for p in self.depotPaths:

2638 - if p4PathStartsWith(path, p):

2656 + if p4PathStartsWith(path, decode_path(p)):

2657 return True

2658 return False

2659

@@ -2644,7 +2662,7 @@ class P4Sync(Command, P4UserMap):

2662 fnum = 0

2663 while "depotFile%s" % fnum in commit:

2664 path = commit["depotFile%s" % fnum]

2647 - found = self.isPathWanted(path)

2665 + found = self.isPathWanted(decode_path(path))

2666 if not found:

2667 fnum = fnum + 1

2668 continue

@@ -2678,7 +2696,7 @@ class P4Sync(Command, P4UserMap):

2696 if self.useClientSpec:

2697 # branch detection moves files up a level (the branch name)

2698 # from what client spec interpretation gives

2681 - path = self.clientSpecDirs.map_in_client(path)

2699 + path = decode_path(self.clientSpecDirs.map_in_client(path))

2700 if self.detectBranches:

2701 for b in self.knownBranches:

2702 if p4PathStartsWith(path, b + "/"):

@@ -2712,14 +2730,15 @@ class P4Sync(Command, P4UserMap):

2730 branches = {}

2731 fnum = 0

2732 while "depotFile%s" % fnum in commit:

2715 - path = commit["depotFile%s" % fnum]

2733 + raw_path = commit["depotFile%s" % fnum]

2734 + path = decode_path(raw_path)

2735 found = self.isPathWanted(path)

2736 if not found:

2737 fnum = fnum + 1

2738 continue

2739

2740 file = {}

2722 - file["path"] = path

2741 + file["path"] = raw_path

2742 file["rev"] = commit["rev%s" % fnum]

2743 file["action"] = commit["action%s" % fnum]

2744 file["type"] = commit["type%s" % fnum]

@@ -2728,7 +2747,7 @@ class P4Sync(Command, P4UserMap):

2747 # start with the full relative path where this file would

2748 # go in a p4 client

2749 if self.useClientSpec:

2731 - relPath = self.clientSpecDirs.map_in_client(path)

2750 + relPath = decode_path(self.clientSpecDirs.map_in_client(path))

2751 else:

2752 relPath = self.stripRepoPath(path, self.depotPaths)

2753

@@ -2766,14 +2785,15 @@ class P4Sync(Command, P4UserMap):

2785 # - helper for streamP4Files

2786

2787 def streamOneP4File(self, file, contents):

2769 - relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)

2770 - relPath = self.encodeWithUTF8(relPath)

2788 + file_path = file['depotFile']

2789 + relPath = self.stripRepoPath(decode_path(file_path), self.branchPrefixes)

2790 +

2791 if verbose:

2792 if 'fileSize' in self.stream_file:

2793 size = int(self.stream_file['fileSize'])

2794 else:

2795 size = 0 # deleted files don't get a fileSize apparently

2776 - sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))

2796 + sys.stdout.write('\r%s --> %s (%i MB)\n' % (file_path, relPath, size/1024/1024))

2797 sys.stdout.flush()

2798

2799 (type_base, type_mods) = split_p4_type(file["type"])

@@ -2791,7 +2811,7 @@ class P4Sync(Command, P4UserMap):

2811 # to nothing. This causes p4 errors when checking out such

2812 # a change, and errors here too. Work around it by ignoring

2813 # the bad symlink; hopefully a future change fixes it.

2794 - print("\nIgnoring empty symlink in %s" % file['depotFile'])

2814 + print("\nIgnoring empty symlink in %s" % file_path)

2815 return

2816 elif data[-1] == '\n':

2817 contents = [data[:-1]]

@@ -2810,7 +2830,7 @@ class P4Sync(Command, P4UserMap):

2830 # just the native "NT" type.

2831 #

2832 try:

2813 - text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])

2833 + text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (decode_path(file['depotFile']), file['change'])], raw=True)

2834 except Exception as e:

2835 if 'Translation of file content failed' in str(e):

2836 type_base = 'binary'

@@ -2818,7 +2838,7 @@ class P4Sync(Command, P4UserMap):

2838 raise e

2839 else:

2840 if p4_version_string().find('/NT') >= 0:

2821 - text = text.replace('\r\n', '\n')

2841 + text = text.replace(b'\r\n', b'\n')

2842 contents = [ text ]

2843

2844 if type_base == "apple":

@@ -2849,8 +2869,7 @@ class P4Sync(Command, P4UserMap):

2869 self.writeToGitStream(git_mode, relPath, contents)

2870

2871 def streamOneP4Deletion(self, file):

2852 - relPath = self.stripRepoPath(file['path'], self.branchPrefixes)

2853 - relPath = self.encodeWithUTF8(relPath)

2872 + relPath = self.stripRepoPath(decode_path(file['path']), self.branchPrefixes)

2873 if verbose:

2874 sys.stdout.write("delete %s\n" % relPath)

2875 sys.stdout.flush()

@@ -3037,8 +3056,8 @@ class P4Sync(Command, P4UserMap):

3056 if self.clientSpecDirs:

3057 self.clientSpecDirs.update_client_spec_path_cache(files)

3058

3040 - files = [f for f in files

3041 - if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]

3059 + files = [f for (f, path) in ((f, decode_path(f['path'])) for f in files)

3060 + if self.inClientSpec(path) and self.hasBranchPrefix(path)]

3061

3062 if gitConfigBool('git-p4.keepEmptyCommits'):

3063 allow_empty = True