@matheusbsbs / matheus-bagliano / commits / 3deed5e078

git-p4: auto-size the block

git-p4 originally would fetch changes in one query. On large repos this could fail because of the limits that Perforce imposes on the number of items returned and the number of queries in the database. To fix this, git-p4 learned to query changes in blocks of 512 changes, However, this can be very slow - if you have a few million changes, with each chunk taking about a second, it can be an hour or so. Although it's possible to tune this value manually with the "--changes-block-size" option, it's far from obvious to ordinary users that this is what needs doing. This change alters the block size dynamically by looking for the specific error messages returned from the Perforce server, and reducing the block size if the error is seen, either to the limit reported by the server, or to half the current block size. That means we can start out with a very large block size, and then let it automatically drop down to a value that works without error, while still failing correctly if some other error occurs. Signed-off-by: Luke Diamand <luke@diamand.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Luke Diamand committed Jun 8, 2018 at 21:32 UTC 3deed5e078f2998573589b1ca03d89f41ea346e9

2 files changed +29 -6

git-p4.py

+21 -6

@@ -47,8 +47,8 @@ verbose = False

47 # Only labels/tags matching this will be imported/exported

48 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'

50 -# Grab changes in blocks of this many revisions, unless otherwise requested

51 -defaultBlockSize = 512

50 +# The block size is reduced automatically if required

51 +defaultBlockSize = 1<<20

53 p4_access_checked = False

@@ -958,7 +958,8 @@ def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):

958 changes = set()

959

960 # Retrieve changes a block at a time, to prevent running

961 - # into a MaxResults/MaxScanRows error from the server.

961 + # into a MaxResults/MaxScanRows error from the server. If

962 + # we _do_ hit one of those errors, turn down the block size

963

964 while True:

965 cmd = ['changes']

@@ -972,10 +973,24 @@ def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):

973 for p in depotPaths:

974 cmd += ["%s...@%s" % (p, revisionRange)]

975

976 + # fetch the changes

977 + try:

978 + result = p4CmdList(cmd, errors_as_exceptions=True)

979 + except P4RequestSizeException as e:

980 + if not block_size:

981 + block_size = e.limit

982 + elif block_size > e.limit:

983 + block_size = e.limit

984 + else:

985 + block_size = max(2, block_size // 2)

986 +

987 + if verbose: print("block size error, retrying with block size {0}".format(block_size))

988 + continue

989 + except P4Exception as e:

990 + die('Error retrieving changes description ({0})'.format(e.p4ExitCode))

991 +

992 # Insert changes in chronological order

976 - for entry in reversed(p4CmdList(cmd)):

977 - if entry.has_key('p4ExitCode'):

978 - die('Error retrieving changes descriptions ({})'.format(entry['p4ExitCode']))

993 + for entry in reversed(result):

994 if not entry.has_key('change'):

995 continue

996 changes.add(int(entry['change']))

t/t9818-git-p4-block.sh

@@ -129,6 +129,7 @@ test_expect_success 'Create a repo with multiple depot paths' '

129 '

130

131 test_expect_success 'Clone repo with multiple depot paths' '

132 + test_when_finished cleanup_git &&

133 (

134 cd "$git" &&

135 git p4 clone --changes-block-size=4 //depot/pathA@all //depot/pathB@all \

@@ -138,6 +139,13 @@ test_expect_success 'Clone repo with multiple depot paths' '

139 )

140 '

141

142 +test_expect_success 'Clone repo with self-sizing block size' '

143 + test_when_finished cleanup_git &&

144 + git p4 clone --changes-block-size=1000000 //depot@all --destination="$git" &&

145 + git -C "$git" log --oneline >log &&

146 + test_line_count \> 10 log

147 +'

148 +

149 test_expect_success 'kill p4d' '

150 kill_p4d

151 '