@matheusbsbs / matheus-bagliano / commits / 8b4c0103a9

sha1_file: support lazily fetching missing objects

Teach sha1_file to fetch objects from the remote configured in extensions.partialclone whenever an object is requested but missing. The fetching of objects can be suppressed through a global variable. This is used by fsck and index-pack. However, by default, such fetching is not suppressed. This is meant as a temporary measure to ensure that all Git commands work in such a situation. Future patches will update some commands to either tolerate missing objects (without fetching them) or be more efficient in fetching them. In order to determine the code changes in sha1_file.c necessary, I investigated the following: (1) functions in sha1_file.c that take in a hash, without the user regarding how the object is stored (loose or packed) (2) functions in packfile.c (because I need to check callers that know about the loose/packed distinction and operate on both differently, and ensure that they can handle the concept of objects that are neither loose nor packed) (1) is handled by the modification to sha1_object_info_extended(). For (2), I looked at for_each_packed_object and others. For for_each_packed_object, the callers either already work or are fixed in this patch: - reachable - only to find recent objects - builtin/fsck - already knows about missing objects - builtin/cat-file - warning message added in this commit Callers of the other functions do not need to be changed: - parse_pack_index - http - indirectly from http_get_info_packs - find_pack_entry_one - this searches a single pack that is provided as an argument; the caller already knows (through other means) that the sought object is in a specific pack - find_sha1_pack - fast-import - appears to be an optimization to not store a file if it is already in a pack - http-walker - to search through a struct alt_base - http-push - to search through remote packs - has_sha1_pack - builtin/fsck - already knows about promisor objects - builtin/count-objects - informational purposes only (check if loose object is also packed) - builtin/prune-packed - check if object to be pruned is packed (if not, don't prune it) - revision - used to exclude packed objects if requested by user - diff - just for optimization Signed-off-by: Jonathan Tan <jonathantanmy@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Jonathan Tan committed Dec 8, 2017 at 15:27 UTC 8b4c0103a98239287176a537f7a04d9b07b49125

8 files changed +99 -8

builtin/cat-file.c

@@ -475,6 +475,8 @@ static int batch_objects(struct batch_options *opt)

475

476 for_each_loose_object(batch_loose_object, &sa, 0);

477 for_each_packed_object(batch_packed_object, &sa, 0);

478 + if (repository_format_partial_clone)

479 + warning("This repository has extensions.partialClone set. Some objects may not be loaded.");

480

481 cb.opt = opt;

482 cb.expand = &data;

builtin/fetch-pack.c

@@ -53,6 +53,8 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)

53 struct oid_array shallow = OID_ARRAY_INIT;

54 struct string_list deepen_not = STRING_LIST_INIT_DUP;

56 + fetch_if_missing = 0;

57 +

58 packet_trace_identity("fetch-pack");

60 memset(&args, 0, sizeof(args));

builtin/fsck.c

@@ -678,6 +678,9 @@ int cmd_fsck(int argc, const char **argv, const char *prefix)

678 int i;

679 struct alternate_object_database *alt;

680

681 + /* fsck knows how to handle missing promisor objects */

682 + fetch_if_missing = 0;

683 +

684 errors_found = 0;

685 check_replace_refs = 0;

686

builtin/index-pack.c

@@ -1657,6 +1657,12 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)

1657 unsigned foreign_nr = 1; /* zero is a "good" value, assume bad */

1658 int report_end_of_input = 0;

1659

1660 + /*

1661 + * index-pack never needs to fetch missing objects, since it only

1662 + * accesses the repo to do hash collision checks

1663 + */

1664 + fetch_if_missing = 0;

1665 +

1666 if (argc == 2 && !strcmp(argv[1], "-h"))

1667 usage(index_pack_usage);

1668

cache.h

@@ -1727,6 +1727,14 @@ struct object_info {

1727 #define OBJECT_INFO_QUICK 8

1728 extern int sha1_object_info_extended(const unsigned char *, struct object_info *, unsigned flags);

1729

1730 +/*

1731 + * Set this to 0 to prevent sha1_object_info_extended() from fetching missing

1732 + * blobs. This has a difference only if extensions.partialClone is set.

1733 + *

1734 + * Its default value is 1.

1735 + */

1736 +extern int fetch_if_missing;

1737 +

1738 /* Dumb servers support */

1739 extern int update_server_info(int);

1740

fetch-object.c

@@ -10,7 +10,9 @@ void fetch_object(const char *remote_name, const unsigned char *sha1)

10 struct remote *remote;

11 struct transport *transport;

12 struct ref *ref;

13 + int original_fetch_if_missing = fetch_if_missing;

15 + fetch_if_missing = 0;

16 remote = remote_get(remote_name);

17 if (!remote->url[0])

18 die(_("Remote with no URL"));

@@ -21,4 +23,5 @@ void fetch_object(const char *remote_name, const unsigned char *sha1)

23 transport_set_option(transport, TRANS_OPT_FROM_PROMISOR, "1");

24 transport_set_option(transport, TRANS_OPT_NO_DEPENDENTS, "1");

25 transport_fetch_refs(transport, ref);

26 + fetch_if_missing = original_fetch_if_missing;

27 }

sha1_file.c

+24 -8

@@ -29,6 +29,7 @@

29 #include "mergesort.h"

30 #include "quote.h"

31 #include "packfile.h"

32 +#include "fetch-object.h"

34 const unsigned char null_sha1[GIT_MAX_RAWSZ];

35 const struct object_id null_oid;

@@ -1144,6 +1145,8 @@ static int sha1_loose_object_info(const unsigned char *sha1,

1145 return (status < 0) ? status : 0;

1146 }

1147

1148 +int fetch_if_missing = 1;

1149 +

1150 int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi, unsigned flags)

1151 {

1152 static struct object_info blank_oi = OBJECT_INFO_INIT;

@@ -1152,6 +1155,7 @@ int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi,

1155 const unsigned char *real = (flags & OBJECT_INFO_LOOKUP_REPLACE) ?

1156 lookup_replace_object(sha1) :

1157 sha1;

1158 + int already_retried = 0;

1159

1160 if (!oi)

1161 oi = &blank_oi;

@@ -1176,19 +1180,32 @@ int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi,

1180 }

1181 }

1182

1179 - if (!find_pack_entry(real, &e)) {

1183 + while (1) {

1184 + if (find_pack_entry(real, &e))

1185 + break;

1186 +

1187 /* Most likely it's a loose object. */

1188 if (!sha1_loose_object_info(real, oi, flags))

1189 return 0;

1190

1191 /* Not a loose object; someone else may have just packed it. */

1185 - if (flags & OBJECT_INFO_QUICK) {

1186 - return -1;

1187 - } else {

1188 - reprepare_packed_git();

1189 - if (!find_pack_entry(real, &e))

1190 - return -1;

1192 + reprepare_packed_git();

1193 + if (find_pack_entry(real, &e))

1194 + break;

1195 +

1196 + /* Check if it is a missing object */

1197 + if (fetch_if_missing && repository_format_partial_clone &&

1198 + !already_retried) {

1199 + /*

1200 + * TODO Investigate haveing fetch_object() return

1201 + * TODO error/success and stopping the music here.

1202 + */

1203 + fetch_object(repository_format_partial_clone, real);

1204 + already_retried = 1;

1205 + continue;

1206 }

1207 +

1208 + return -1;

1209 }

1210

1211 if (oi == &blank_oi)

@@ -1197,7 +1214,6 @@ int sha1_object_info_extended(const unsigned char *sha1, struct object_info *oi,

1214 * information below, so return early.

1215 */

1216 return 0;

1200 -

1217 rtype = packed_object_info(e.p, e.offset, oi);

1218 if (rtype < 0) {

1219 mark_bad_packed_object(e.p, real);

t/t0410-partial-clone.sh

+51

@@ -138,4 +138,55 @@ test_expect_success 'missing CLI object, but promised, passes fsck' '

138 git -C repo fsck "$A"

139 '

140

141 +test_expect_success 'fetching of missing objects' '

142 + rm -rf repo &&

143 + test_create_repo server &&

144 + test_commit -C server foo &&

145 + git -C server repack -a -d --write-bitmap-index &&

146 +

147 + git clone "file://$(pwd)/server" repo &&

148 + HASH=$(git -C repo rev-parse foo) &&

149 + rm -rf repo/.git/objects/* &&

150 +

151 + git -C repo config core.repositoryformatversion 1 &&

152 + git -C repo config extensions.partialclone "origin" &&

153 + git -C repo cat-file -p "$HASH" &&

154 +

155 + # Ensure that the .promisor file is written, and check that its

156 + # associated packfile contains the object

157 + ls repo/.git/objects/pack/pack-*.promisor >promisorlist &&

158 + test_line_count = 1 promisorlist &&

159 + IDX=$(cat promisorlist | sed "s/promisor$/idx/") &&

160 + git verify-pack --verbose "$IDX" | grep "$HASH"

161 +'

162 +

163 +LIB_HTTPD_PORT=12345 # default port, 410, cannot be used as non-root

164 +. "$TEST_DIRECTORY"/lib-httpd.sh

165 +start_httpd

166 +

167 +test_expect_success 'fetching of missing objects from an HTTP server' '

168 + rm -rf repo &&

169 + SERVER="$HTTPD_DOCUMENT_ROOT_PATH/server" &&

170 + test_create_repo "$SERVER" &&

171 + test_commit -C "$SERVER" foo &&

172 + git -C "$SERVER" repack -a -d --write-bitmap-index &&

173 +

174 + git clone $HTTPD_URL/smart/server repo &&

175 + HASH=$(git -C repo rev-parse foo) &&

176 + rm -rf repo/.git/objects/* &&

177 +

178 + git -C repo config core.repositoryformatversion 1 &&

179 + git -C repo config extensions.partialclone "origin" &&

180 + git -C repo cat-file -p "$HASH" &&

181 +

182 + # Ensure that the .promisor file is written, and check that its

183 + # associated packfile contains the object

184 + ls repo/.git/objects/pack/pack-*.promisor >promisorlist &&

185 + test_line_count = 1 promisorlist &&

186 + IDX=$(cat promisorlist | sed "s/promisor$/idx/") &&

187 + git verify-pack --verbose "$IDX" | grep "$HASH"

188 +'

189 +

190 +stop_httpd

191 +

192 test_done