@matheusbsbs / matheus-bagliano / commits / 18b6b1b5c5

upload-pack.c: avoid enumerating hidden refs where possible

In a similar fashion as a previous commit, teach `upload-pack` to avoid enumerating hidden references where possible. Note, however, that there are certain cases where cannot avoid enumerating even hidden references, in particular when either of: - `uploadpack.allowTipSHA1InWant`, or - `uploadpack.allowReachableSHA1InWant` are set, corresponding to `ALLOW_TIP_SHA1` and `ALLOW_REACHABLE_SHA1`, respectively. When either of these bits are set, upload-pack's `is_our_ref()` function needs to consider the `HIDDEN_REF` bit of the referent's object flags. So we must visit all references, including the hidden ones, in order to mark their referents with the `HIDDEN_REF` bit. When neither `ALLOW_TIP_SHA1` nor `ALLOW_REACHABLE_SHA1` are set, the `is_our_ref()` function considers only the `OUR_REF` bit, and not the `HIDDEN_REF` one. `OUR_REF` is applied via `mark_our_ref()`, and only to objects at the tips of non-hidden references, so we do not need to visit hidden references in this case. When neither of those bits are set, `upload-pack` can potentially avoid enumerating a large number of references. In the same example as a previous commit (linux.git with one hidden reference per commit, "refs/pull/N"): $ printf 0000 >in $ hyperfine --warmup=1 \ 'git -c transfer.hideRefs=refs/pull upload-pack . <in' \ 'git.compile -c transfer.hideRefs=refs/pull -c uploadpack.allowTipSHA1InWant upload-pack . <in' \ 'git.compile -c transfer.hideRefs=refs/pull upload-pack . <in' Benchmark 1: git -c transfer.hideRefs=refs/pull upload-pack . <in Time (mean ± σ): 406.9 ms ± 1.1 ms [User: 357.3 ms, System: 49.5 ms] Range (min … max): 405.7 ms … 409.2 ms 10 runs Benchmark 2: git.compile -c transfer.hideRefs=refs/pull -c uploadpack.allowTipSHA1InWant upload-pack . <in Time (mean ± σ): 406.5 ms ± 1.3 ms [User: 356.5 ms, System: 49.9 ms] Range (min … max): 404.6 ms … 408.8 ms 10 runs Benchmark 3: git.compile -c transfer.hideRefs=refs/pull upload-pack . <in Time (mean ± σ): 4.7 ms ± 0.2 ms [User: 0.7 ms, System: 3.9 ms] Range (min … max): 4.3 ms … 6.1 ms 472 runs Summary 'git.compile -c transfer.hideRefs=refs/pull upload-pack . <in' ran 86.62 ± 4.33 times faster than 'git.compile -c transfer.hideRefs=refs/pull -c uploadpack.allowTipSHA1InWant upload-pack . <in' 86.70 ± 4.33 times faster than 'git -c transfer.hideRefs=refs/pull upload-pack . <in' As above, we must visit every reference when uploadPack.allowTipSHA1InWant is set. But when it is unset, we can visit far fewer references. Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Taylor Blau committed Jul 10, 2023 at 17:12 UTC 18b6b1b5c57bb4d1b2730b664fb1660b006ef7d3
1 file changed +31 -6
upload-pack.c
+31 -6
@@ -602,11 +602,36 @@ static int get_common_commits(struct upload_pack_data *data,
602 }
603 }
604
605 +static int allow_hidden_refs(enum allow_uor allow_uor)
606 +{
607 + if ((allow_uor & ALLOW_ANY_SHA1) == ALLOW_ANY_SHA1)
608 + return 1;
609 + return !(allow_uor & (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1));
610 +}
611 +
612 +static void for_each_namespaced_ref_1(each_ref_fn fn,
613 + struct upload_pack_data *data)
614 +{
615 + const char **excludes = NULL;
616 + /*
617 + * If `data->allow_uor` allows fetching hidden refs, we need to
618 + * mark all references (including hidden ones), to check in
619 + * `is_our_ref()` below.
620 + *
621 + * Otherwise, we only care about whether each reference's object
622 + * has the OUR_REF bit set or not, so do not need to visit
623 + * hidden references.
624 + */
625 + if (allow_hidden_refs(data->allow_uor))
626 + excludes = hidden_refs_to_excludes(&data->hidden_refs);
627 +
628 + for_each_namespaced_ref(excludes, fn, data);
629 +}
630 +
631 +
632 static int is_our_ref(struct object *o, enum allow_uor allow_uor)
633 {
607 - int allow_hidden_ref = (allow_uor &
608 - (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1));
609 - return o->flags & ((allow_hidden_ref ? HIDDEN_REF : 0) | OUR_REF);
634 + return o->flags & ((allow_hidden_refs(allow_uor) ? 0 : HIDDEN_REF) | OUR_REF);
635 }
636
637 /*
@@ -855,7 +880,7 @@ static void deepen(struct upload_pack_data *data, int depth)
880 * marked with OUR_REF.
881 */
882 head_ref_namespaced(check_ref, data);
858 - for_each_namespaced_ref(NULL, check_ref, data);
883 + for_each_namespaced_ref_1(check_ref, data);
884
885 get_reachable_list(data, &reachable_shallows);
886 result = get_shallow_commits(&reachable_shallows,
@@ -1386,7 +1411,7 @@ void upload_pack(const int advertise_refs, const int stateless_rpc,
1411 if (advertise_refs)
1412 data.no_done = 1;
1413 head_ref_namespaced(send_ref, &data);
1389 - for_each_namespaced_ref(NULL, send_ref, &data);
1414 + for_each_namespaced_ref_1(send_ref, &data);
1415 if (!data.sent_capabilities) {
1416 const char *refname = "capabilities^{}";
1417 write_v0_ref(&data, refname, refname, null_oid());
@@ -1400,7 +1425,7 @@ void upload_pack(const int advertise_refs, const int stateless_rpc,
1425 packet_flush(1);
1426 } else {
1427 head_ref_namespaced(check_ref, &data);
1403 - for_each_namespaced_ref(NULL, check_ref, &data);
1428 + for_each_namespaced_ref_1(check_ref, &data);
1429 }
1430
1431 if (!advertise_refs) {