@matheusbsbs / matheus-bagliano / commits / 8d8c2a5aef

fsck: always compute USED flags for unreachable objects

The --connectivity-only option avoids opening every object, and instead just marks reachable objects with a flag and compares this to the set of all objects. This strategy is discussed in more detail in 3e3f8bd608 (fsck: prepare dummy objects for --connectivity-check, 2017-01-17). This means that we report _every_ unreachable object as dangling. Whereas in a full fsck, we'd have actually opened and parsed each of those unreachable objects, marking their child objects with the USED flag, to mean "this was mentioned by another object". And thus we can report only the tip of an unreachable segment of the object graph as dangling. You can see this difference with a trivial example: tree=$(git hash-object -t tree -w /dev/null) one=$(echo one | git commit-tree $tree) two=$(echo two | git commit-tree -p $one $tree) Running `git fsck` will report only $two as dangling, but with --connectivity-only, both commits (and the tree) are reported. Likewise, using --lost-found would write all three objects. We can make --connectivity-only work like the normal case by taking a separate pass over the unreachable objects, parsing them and marking objects they refer to as USED. That still avoids parsing any blobs, though we do pay the cost to access any unreachable commits and trees (which may or may not be noticeable, depending on how many you have). If neither --dangling nor --lost-found is in effect, then we can skip this step entirely, just like we do now. That makes "--connectivity-only --no-dangling" just as fast as the current "--connectivity-only". I.e., we do the correct thing always, but you can still tweak the options to make it faster if you don't care about dangling objects. Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Jeff King committed Mar 4, 2019 at 23:47 UTC 8d8c2a5aef0fd20a68271697e50412791c06d9b7

3 files changed +83 -2

Documentation/git-fsck.txt

@@ -69,6 +69,10 @@ index file, all SHA-1 references in `refs` namespace, and all reflogs

69 exist). This will detect corruption in commits and trees, but

70 not do any semantic checks (e.g., for format errors). Corruption

71 in blob objects will not be detected at all.

72 ++

73 +Unreachable tags, commits, and trees will also be accessed to find the

74 +tips of dangling segments of history. Use `--no-dangling` if you don't

75 +care about this output and want to speed it up further.

77 --strict::

78 Enable more strict checking, namely to catch a file mode

builtin/fsck.c

+62

@@ -216,6 +216,48 @@ static int mark_used(struct object *obj, int type, void *data, struct fsck_optio

216 return 0;

217 }

218

219 +static void mark_unreachable_referents(const struct object_id *oid)

220 +{

221 + struct fsck_options options = FSCK_OPTIONS_DEFAULT;

222 + struct object *obj = lookup_object(the_repository, oid->hash);

223 +

224 + if (!obj || !(obj->flags & HAS_OBJ))

225 + return; /* not part of our original set */

226 + if (obj->flags & REACHABLE)

227 + return; /* reachable objects already traversed */

228 +

229 + /*

230 + * Avoid passing OBJ_NONE to fsck_walk, which will parse the object

231 + * (and we want to avoid parsing blobs).

232 + */

233 + if (obj->type == OBJ_NONE) {

234 + enum object_type type = oid_object_info(the_repository,

235 + &obj->oid, NULL);

236 + if (type > 0)

237 + object_as_type(the_repository, obj, type, 0);

238 + }

239 +

240 + options.walk = mark_used;

241 + fsck_walk(obj, NULL, &options);

242 +}

243 +

244 +static int mark_loose_unreachable_referents(const struct object_id *oid,

245 + const char *path,

246 + void *data)

247 +{

248 + mark_unreachable_referents(oid);

249 + return 0;

250 +}

251 +

252 +static int mark_packed_unreachable_referents(const struct object_id *oid,

253 + struct packed_git *pack,

254 + uint32_t pos,

255 + void *data)

256 +{

257 + mark_unreachable_referents(oid);

258 + return 0;

259 +}

260 +

261 /*

262 * Check a single reachable object

263 */

@@ -328,6 +370,26 @@ static void check_connectivity(void)

370 /* Traverse the pending reachable objects */

371 traverse_reachable();

372

373 + /*

374 + * With --connectivity-only, we won't have actually opened and marked

375 + * unreachable objects with USED. Do that now to make --dangling, etc

376 + * accurate.

377 + */

378 + if (connectivity_only && (show_dangling || write_lost_and_found)) {

379 + /*

380 + * Even though we already have a "struct object" for each of

381 + * these in memory, we must not iterate over the internal

382 + * object hash as we do below. Our loop would potentially

383 + * resize the hash, making our iteration invalid.

384 + *

385 + * Instead, we'll just go back to the source list of objects,

386 + * and ignore any that weren't present in our earlier

387 + * traversal.

388 + */

389 + for_each_loose_object(mark_loose_unreachable_referents, NULL, 0);

390 + for_each_packed_object(mark_packed_unreachable_referents, NULL, 0);

391 + }

392 +

393 /* Look up all the requirements, warn about missing objects.. */

394 max = get_max_object_index();

395 if (verbose)

t/t1450-fsck.sh

+17 -2

@@ -740,7 +740,7 @@ test_expect_success 'fsck detects truncated loose object' '

740 # for each of type, we have one version which is referenced by another object

741 # (and so while unreachable, not dangling), and another variant which really is

742 # dangling.

743 -test_expect_success 'fsck notices dangling objects' '

743 +test_expect_success 'create dangling-object repository' '

744 git init dangling &&

745 (

746 cd dangling &&

@@ -751,12 +751,17 @@ test_expect_success 'fsck notices dangling objects' '

751 commit=$(git commit-tree $tree) &&

752 dcommit=$(git commit-tree -p $commit $tree) &&

753

754 - cat >expect <<-EOF &&

754 + cat >expect <<-EOF

755 dangling blob $dblob

756 dangling commit $dcommit

757 dangling tree $dtree

758 EOF

759 + )

760 +'

761

762 +test_expect_success 'fsck notices dangling objects' '

763 + (

764 + cd dangling &&

765 git fsck >actual &&

766 # the output order is non-deterministic, as it comes from a hash

767 sort <actual >actual.sorted &&

@@ -764,6 +769,16 @@ test_expect_success 'fsck notices dangling objects' '

769 )

770 '

771

772 +test_expect_success 'fsck --connectivity-only notices dangling objects' '

773 + (

774 + cd dangling &&

775 + git fsck --connectivity-only >actual &&

776 + # the output order is non-deterministic, as it comes from a hash

777 + sort <actual >actual.sorted &&

778 + test_i18ncmp expect actual.sorted

779 + )

780 +'

781 +

782 test_expect_success 'fsck $name notices bogus $name' '

783 test_must_fail git fsck bogus &&

784 test_must_fail git fsck $ZERO_OID