git-p4: add support for large file systems
Perforce repositories can contain large (binary) files. Migrating these repositories to Git generates very large local clones. External storage systems such as Git LFS [1], Git Fat [2], Git Media [3], git-annex [4] try to address this problem. Add a generic mechanism to detect large files based on extension, uncompressed size, and/or compressed size. [1] https://git-lfs.github.com/ [2] https://github.com/jedbrown/git-fat [3] https://github.com/alebedev/git-media [4] https://git-annex.branchable.com/ Signed-off-by: Lars Schneider <larsxschneider@gmail.com> Conflicts: Documentation/git-p4.txt git-p4.py Signed-off-by: Junio C Hamano <gitster@pobox.com>
Lars Schneider committed
Sep 26, 2015 at 09:55 UTC
a5db4b127bf86b9dbeb3cb079f4a9530ba313941
3 files changed
+353
-10
Documentation/git-p4.txt
+32
@@ -510,6 +510,38 @@ git-p4.useClientSpec::
510
option '--use-client-spec'. See the "CLIENT SPEC" section above.
511
This variable is a boolean, not the name of a p4 client.
512
513
+git-p4.largeFileSystem::
514
+ Specify the system that is used for large (binary) files. Please note
515
+ that large file systems do not support the 'git p4 submit' command.
516
+ Only Git LFS [1] is implemented right now. Download
517
+ and install the Git LFS command line extension to use this option
518
+ and configure it like this:
519
++
520
+-------------
521
+git config git-p4.largeFileSystem GitLFS
522
+-------------
523
++
524
+ [1] https://git-lfs.github.com/
525
+
526
+git-p4.largeFileExtensions::
527
+ All files matching a file extension in the list will be processed
528
+ by the large file system. Do not prefix the extensions with '.'.
529
+
530
+git-p4.largeFileThreshold::
531
+ All files with an uncompressed size exceeding the threshold will be
532
+ processed by the large file system. By default the threshold is
533
+ defined in bytes. Add the suffix k, m, or g to change the unit.
534
+
535
+git-p4.largeFileCompressedThreshold::
536
+ All files with a compressed size exceeding the threshold will be
537
+ processed by the large file system. This option might slow down
538
+ your clone/sync process. By default the threshold is defined in
539
+ bytes. Add the suffix k, m, or g to change the unit.
540
+
541
+git-p4.largeFilePush::
542
+ Boolean variable which defines if large files are automatically
543
+ pushed to a server.
544
+
545
Submit variables
546
~~~~~~~~~~~~~~~~
547
git-p4.detectRenames::
git-p4.py
+129
-10
@@ -22,6 +22,8 @@ import platform
22
import re
23
import shutil
24
import stat
25
+import zipfile
26
+import zlib
27
28
try:
29
from subprocess import CalledProcessError
@@ -932,6 +934,110 @@ def wildcard_present(path):
934
m = re.search("[*#@%]", path)
935
return m is not None
936
937
+class LargeFileSystem(object):
938
+ """Base class for large file system support."""
939
+
940
+ def __init__(self, writeToGitStream):
941
+ self.largeFiles = set()
942
+ self.writeToGitStream = writeToGitStream
943
+
944
+ def generatePointer(self, cloneDestination, contentFile):
945
+ """Return the content of a pointer file that is stored in Git instead of
946
+ the actual content."""
947
+ assert False, "Method 'generatePointer' required in " + self.__class__.__name__
948
+
949
+ def pushFile(self, localLargeFile):
950
+ """Push the actual content which is not stored in the Git repository to
951
+ a server."""
952
+ assert False, "Method 'pushFile' required in " + self.__class__.__name__
953
+
954
+ def hasLargeFileExtension(self, relPath):
955
+ return reduce(
956
+ lambda a, b: a or b,
957
+ [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
958
+ False
959
+ )
960
+
961
+ def generateTempFile(self, contents):
962
+ contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
963
+ for d in contents:
964
+ contentFile.write(d)
965
+ contentFile.close()
966
+ return contentFile.name
967
+
968
+ def exceedsLargeFileThreshold(self, relPath, contents):
969
+ if gitConfigInt('git-p4.largeFileThreshold'):
970
+ contentsSize = sum(len(d) for d in contents)
971
+ if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
972
+ return True
973
+ if gitConfigInt('git-p4.largeFileCompressedThreshold'):
974
+ contentsSize = sum(len(d) for d in contents)
975
+ if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
976
+ return False
977
+ contentTempFile = self.generateTempFile(contents)
978
+ compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
979
+ zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
980
+ zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
981
+ zf.close()
982
+ compressedContentsSize = zf.infolist()[0].compress_size
983
+ os.remove(contentTempFile)
984
+ os.remove(compressedContentFile.name)
985
+ if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
986
+ return True
987
+ return False
988
+
989
+ def addLargeFile(self, relPath):
990
+ self.largeFiles.add(relPath)
991
+
992
+ def removeLargeFile(self, relPath):
993
+ self.largeFiles.remove(relPath)
994
+
995
+ def isLargeFile(self, relPath):
996
+ return relPath in self.largeFiles
997
+
998
+ def processContent(self, git_mode, relPath, contents):
999
+ """Processes the content of git fast import. This method decides if a
1000
+ file is stored in the large file system and handles all necessary
1001
+ steps."""
1002
+ if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1003
+ contentTempFile = self.generateTempFile(contents)
1004
+ (git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1005
+
1006
+ # Move temp file to final location in large file system
1007
+ largeFileDir = os.path.dirname(localLargeFile)
1008
+ if not os.path.isdir(largeFileDir):
1009
+ os.makedirs(largeFileDir)
1010
+ shutil.move(contentTempFile, localLargeFile)
1011
+ self.addLargeFile(relPath)
1012
+ if gitConfigBool('git-p4.largeFilePush'):
1013
+ self.pushFile(localLargeFile)
1014
+ if verbose:
1015
+ sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1016
+ return (git_mode, contents)
1017
+
1018
+class MockLFS(LargeFileSystem):
1019
+ """Mock large file system for testing."""
1020
+
1021
+ def generatePointer(self, contentFile):
1022
+ """The pointer content is the original content prefixed with "pointer-".
1023
+ The local filename of the large file storage is derived from the file content.
1024
+ """
1025
+ with open(contentFile, 'r') as f:
1026
+ content = next(f)
1027
+ gitMode = '100644'
1028
+ pointerContents = 'pointer-' + content
1029
+ localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1030
+ return (gitMode, pointerContents, localLargeFile)
1031
+
1032
+ def pushFile(self, localLargeFile):
1033
+ """The remote filename of the large file storage is the same as the local
1034
+ one but in a different directory.
1035
+ """
1036
+ remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1037
+ if not os.path.exists(remotePath):
1038
+ os.makedirs(remotePath)
1039
+ shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1040
+
1041
class Command:
1042
def __init__(self):
1043
self.usage = "usage: %prog [options]"
@@ -1105,6 +1211,9 @@ class P4Submit(Command, P4UserMap):
1211
self.p4HasMoveCommand = p4_has_move_command()
1212
self.branch = None
1213
1214
+ if gitConfig('git-p4.largeFileSystem'):
1215
+ die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1216
+
1217
def check(self):
1218
if len(p4CmdList("opened ...")) > 0:
1219
die("You have files opened with perforce! Close them before starting the sync.")
@@ -2055,6 +2164,13 @@ class P4Sync(Command, P4UserMap):
2164
self.clientSpecDirs = None
2165
self.tempBranches = []
2166
self.tempBranchLocation = "git-p4-tmp"
2167
+ self.largeFileSystem = None
2168
+
2169
+ if gitConfig('git-p4.largeFileSystem'):
2170
+ largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2171
+ self.largeFileSystem = largeFileSystemConstructor(
2172
+ lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2173
+ )
2174
2175
if gitConfig("git-p4.syncFromOrigin") == "false":
2176
self.syncWithOrigin = False
@@ -2175,6 +2291,13 @@ class P4Sync(Command, P4UserMap):
2291
2292
return branches
2293
2294
+ def writeToGitStream(self, gitMode, relPath, contents):
2295
+ self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2296
+ self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2297
+ for d in contents:
2298
+ self.gitStream.write(d)
2299
+ self.gitStream.write('\n')
2300
+
2301
# output one file from the P4 stream
2302
# - helper for streamP4Files
2303
@@ -2245,17 +2368,10 @@ class P4Sync(Command, P4UserMap):
2368
text = regexp.sub(r'$\1$', text)
2369
contents = [ text ]
2370
2248
- self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
2371
+ if self.largeFileSystem:
2372
+ (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2373
2250
- # total length...
2251
- length = 0
2252
- for d in contents:
2253
- length = length + len(d)
2254
-
2255
- self.gitStream.write("data %d\n" % length)
2256
- for d in contents:
2257
- self.gitStream.write(d)
2258
- self.gitStream.write("\n")
2374
+ self.writeToGitStream(git_mode, relPath, contents)
2375
2376
def streamOneP4Deletion(self, file):
2377
relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
@@ -2264,6 +2380,9 @@ class P4Sync(Command, P4UserMap):
2380
sys.stdout.flush()
2381
self.gitStream.write("D %s\n" % relPath)
2382
2383
+ if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2384
+ self.largeFileSystem.removeLargeFile(relPath)
2385
+
2386
# handle another chunk of streaming data
2387
def streamP4FilesCb(self, marshalled):
2388
t/t9823-git-p4-mock-lfs.sh
new
+192
@@ -0,0 +1,192 @@
1
+#!/bin/sh
2
+
3
+test_description='Clone repositories and store files in Mock LFS'
4
+
5
+. ./lib-git-p4.sh
6
+
7
+test_file_is_not_in_mock_lfs () {
8
+ FILE="$1" &&
9
+ CONTENT="$2" &&
10
+ echo "$CONTENT" >expect_content &&
11
+ test_path_is_file "$FILE" &&
12
+ test_cmp expect_content "$FILE"
13
+}
14
+
15
+test_file_is_in_mock_lfs () {
16
+ FILE="$1" &&
17
+ CONTENT="$2" &&
18
+ LOCAL_STORAGE=".git/mock-storage/local/$CONTENT" &&
19
+ SERVER_STORAGE=".git/mock-storage/remote/$CONTENT" &&
20
+ echo "pointer-$CONTENT" >expect_pointer &&
21
+ echo "$CONTENT" >expect_content &&
22
+ test_path_is_file "$FILE" &&
23
+ test_path_is_file "$LOCAL_STORAGE" &&
24
+ test_path_is_file "$SERVER_STORAGE" &&
25
+ test_cmp expect_pointer "$FILE" &&
26
+ test_cmp expect_content "$LOCAL_STORAGE" &&
27
+ test_cmp expect_content "$SERVER_STORAGE"
28
+}
29
+
30
+test_file_is_deleted_in_mock_lfs () {
31
+ FILE="$1" &&
32
+ CONTENT="$2" &&
33
+ LOCAL_STORAGE=".git/mock-storage/local/$CONTENT" &&
34
+ SERVER_STORAGE=".git/mock-storage/remote/$CONTENT" &&
35
+ echo "pointer-$CONTENT" >expect_pointer &&
36
+ echo "$CONTENT" >expect_content &&
37
+ test_path_is_missing "$FILE" &&
38
+ test_path_is_file "$LOCAL_STORAGE" &&
39
+ test_path_is_file "$SERVER_STORAGE" &&
40
+ test_cmp expect_content "$LOCAL_STORAGE" &&
41
+ test_cmp expect_content "$SERVER_STORAGE"
42
+}
43
+
44
+test_file_count_in_dir () {
45
+ DIR="$1" &&
46
+ EXPECTED_COUNT="$2" &&
47
+ find "$DIR" -type f >actual &&
48
+ test_line_count = $EXPECTED_COUNT actual
49
+}
50
+
51
+test_expect_success 'start p4d' '
52
+ start_p4d
53
+'
54
+
55
+test_expect_success 'Create repo with binary files' '
56
+ client_view "//depot/... //client/..." &&
57
+ (
58
+ cd "$cli" &&
59
+
60
+ echo "content 1 txt 23 bytes" >file1.txt &&
61
+ p4 add file1.txt &&
62
+ echo "content 2-3 bin 25 bytes" >file2.dat &&
63
+ p4 add file2.dat &&
64
+ p4 submit -d "Add text and binary file" &&
65
+
66
+ mkdir "path with spaces" &&
67
+ echo "content 2-3 bin 25 bytes" >"path with spaces/file3.bin" &&
68
+ p4 add "path with spaces/file3.bin" &&
69
+ p4 submit -d "Add another binary file with same content and spaces in path" &&
70
+
71
+ echo "content 4 bin 26 bytes XX" >file4.bin &&
72
+ p4 add file4.bin &&
73
+ p4 submit -d "Add another binary file with different content"
74
+ )
75
+'
76
+
77
+test_expect_success 'Store files in Mock LFS based on size (>24 bytes)' '
78
+ client_view "//depot/... //client/..." &&
79
+ test_when_finished cleanup_git &&
80
+ (
81
+ cd "$git" &&
82
+ git init . &&
83
+ git config git-p4.useClientSpec true &&
84
+ git config git-p4.largeFileSystem MockLFS &&
85
+ git config git-p4.largeFileThreshold 24 &&
86
+ git config git-p4.largeFilePush True &&
87
+ git p4 clone --destination="$git" //depot@all &&
88
+
89
+ test_file_is_not_in_mock_lfs file1.txt "content 1 txt 23 bytes" &&
90
+ test_file_is_in_mock_lfs file2.dat "content 2-3 bin 25 bytes" &&
91
+ test_file_is_in_mock_lfs "path with spaces/file3.bin" "content 2-3 bin 25 bytes" &&
92
+ test_file_is_in_mock_lfs file4.bin "content 4 bin 26 bytes XX" &&
93
+
94
+ test_file_count_in_dir ".git/mock-storage/local" 2 &&
95
+ test_file_count_in_dir ".git/mock-storage/remote" 2
96
+ )
97
+'
98
+
99
+test_expect_success 'Store files in Mock LFS based on extension (dat)' '
100
+ client_view "//depot/... //client/..." &&
101
+ test_when_finished cleanup_git &&
102
+ (
103
+ cd "$git" &&
104
+ git init . &&
105
+ git config git-p4.useClientSpec true &&
106
+ git config git-p4.largeFileSystem MockLFS &&
107
+ git config git-p4.largeFileExtensions dat &&
108
+ git config git-p4.largeFilePush True &&
109
+ git p4 clone --destination="$git" //depot@all &&
110
+
111
+ test_file_is_not_in_mock_lfs file1.txt "content 1 txt 23 bytes" &&
112
+ test_file_is_in_mock_lfs file2.dat "content 2-3 bin 25 bytes" &&
113
+ test_file_is_not_in_mock_lfs "path with spaces/file3.bin" "content 2-3 bin 25 bytes" &&
114
+ test_file_is_not_in_mock_lfs file4.bin "content 4 bin 26 bytes XX" &&
115
+
116
+ test_file_count_in_dir ".git/mock-storage/local" 1 &&
117
+ test_file_count_in_dir ".git/mock-storage/remote" 1
118
+ )
119
+'
120
+
121
+test_expect_success 'Store files in Mock LFS based on extension (dat) and use git p4 sync and no client spec' '
122
+ test_when_finished cleanup_git &&
123
+ (
124
+ cd "$git" &&
125
+ git init &&
126
+ git config git-p4.useClientSpec true &&
127
+ git config git-p4.largeFileSystem MockLFS &&
128
+ git config git-p4.largeFileExtensions dat &&
129
+ git config git-p4.largeFilePush True &&
130
+ git p4 sync //depot &&
131
+ git checkout p4/master &&
132
+
133
+ test_file_is_not_in_mock_lfs file1.txt "content 1 txt 23 bytes" &&
134
+ test_file_is_in_mock_lfs file2.dat "content 2-3 bin 25 bytes" &&
135
+ test_file_is_not_in_mock_lfs "path with spaces/file3.bin" "content 2-3 bin 25 bytes" &&
136
+ test_file_is_not_in_mock_lfs file4.bin "content 4 bin 26 bytes XX" &&
137
+
138
+ test_file_count_in_dir ".git/mock-storage/local" 1 &&
139
+ test_file_count_in_dir ".git/mock-storage/remote" 1
140
+ )
141
+'
142
+
143
+test_expect_success 'Remove file from repo and store files in Mock LFS based on size (>24 bytes)' '
144
+ client_view "//depot/... //client/..." &&
145
+ (
146
+ cd "$cli" &&
147
+ p4 delete file4.bin &&
148
+ p4 submit -d "Remove file"
149
+ ) &&
150
+
151
+ client_view "//depot/... //client/..." &&
152
+ test_when_finished cleanup_git &&
153
+ (
154
+ cd "$git" &&
155
+ git init . &&
156
+ git config git-p4.useClientSpec true &&
157
+ git config git-p4.largeFileSystem MockLFS &&
158
+ git config git-p4.largeFileThreshold 24 &&
159
+ git config git-p4.largeFilePush True &&
160
+ git p4 clone --destination="$git" //depot@all &&
161
+
162
+ test_file_is_not_in_mock_lfs file1.txt "content 1 txt 23 bytes" &&
163
+ test_file_is_in_mock_lfs file2.dat "content 2-3 bin 25 bytes" &&
164
+ test_file_is_in_mock_lfs "path with spaces/file3.bin" "content 2-3 bin 25 bytes" &&
165
+ test_file_is_deleted_in_mock_lfs file4.bin "content 4 bin 26 bytes XX" &&
166
+
167
+ test_file_count_in_dir ".git/mock-storage/local" 2 &&
168
+ test_file_count_in_dir ".git/mock-storage/remote" 2
169
+ )
170
+'
171
+
172
+test_expect_success 'Run git p4 submit in repo configured with large file system' '
173
+ client_view "//depot/... //client/..." &&
174
+ test_when_finished cleanup_git &&
175
+ (
176
+ cd "$git" &&
177
+ git init . &&
178
+ git config git-p4.useClientSpec true &&
179
+ git config git-p4.largeFileSystem MockLFS &&
180
+ git config git-p4.largeFileThreshold 24 &&
181
+ git config git-p4.largeFilePush True &&
182
+ git p4 clone --destination="$git" //depot@all &&
183
+
184
+ test_must_fail git p4 submit
185
+ )
186
+'
187
+
188
+test_expect_success 'kill p4d' '
189
+ kill_p4d
190
+'
191
+
192
+test_done