builtin/log: prefetch necessary blobs for `git cherry`
In partial clones, `git cherry` fetches necessary blobs on-demand one at a time, which can be very slow. We would like to prefetch all necessary blobs upfront. To do so, we need to be able to first figure out which blobs are needed. `git cherry` does its work in a two-phase approach: first computing header-only IDs (based on file paths and modes), then falling back to full content-based IDs only when header-only IDs collide -- or, more accurately, whenever the oidhash() of the header-only object_ids collide. patch-ids.c handles this by creating an ids->patches hashmap that has all the data we need, but the problem is that any attempt to query the hashmap will invoke the patch_id_neq() function on any colliding objects, which causes the on-demand fetching. Insert a new prefetch_cherry_blobs() function before checking for collisions. Use a temporary replacement on the ids->patches.cmpfn in order to enumerate the blobs that would be needed without yet fetching them, and then fetch them all at once, then restore the old ids->patches.cmpfn. Signed-off-by: Elijah Newren <newren@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Elijah Newren committed
May 14, 2026 at 16:25 UTC
463c1bfc2b65357569055916e348e1bd9c7a5b25
2 files changed
+158
builtin/log.c
+131
@@ -21,10 +21,12 @@
21
#include "color.h"
22
#include "commit.h"
23
#include "diff.h"
24
+#include "diffcore.h"
25
#include "diff-merges.h"
26
#include "revision.h"
27
#include "log-tree.h"
28
#include "oid-array.h"
29
+#include "oidset.h"
30
#include "tag.h"
31
#include "reflog-walk.h"
32
#include "patch-ids.h"
@@ -43,9 +45,11 @@
45
#include "utf8.h"
46
47
#include "commit-reach.h"
48
+#include "promisor-remote.h"
49
#include "range-diff.h"
50
#include "tmp-objdir.h"
51
#include "tree.h"
52
+#include "userdiff.h"
53
#include "write-or-die.h"
54
55
#define MAIL_DEFAULT_WRAP 72
@@ -2602,6 +2606,131 @@ static void print_commit(char sign, struct commit *commit, int verbose,
2606
}
2607
}
2608
2609
+/*
2610
+ * Enumerate blob OIDs from a single commit's diff, inserting them into blobs.
2611
+ * Skips files whose userdiff driver explicitly declares binary status
2612
+ * (drv->binary > 0), since patch-ID uses oid_to_hex() for those and
2613
+ * never reads blob content. Use userdiff_find_by_path() since
2614
+ * diff_filespec_load_driver() is static in diff.c.
2615
+ *
2616
+ * Clean up with diff_queue_clear() (from diffcore.h).
2617
+ */
2618
+static void collect_diff_blob_oids(struct commit *commit,
2619
+ struct diff_options *opts,
2620
+ struct oidset *blobs)
2621
+{
2622
+ struct diff_queue_struct *q;
2623
+
2624
+ /*
2625
+ * Merge commits are filtered out by patch_id_defined() in patch-ids.c,
2626
+ * so we'll never be called with one.
2627
+ */
2628
+ assert(!commit->parents || !commit->parents->next);
2629
+
2630
+ if (commit->parents)
2631
+ diff_tree_oid(&commit->parents->item->object.oid,
2632
+ &commit->object.oid, "", opts);
2633
+ else
2634
+ diff_root_tree_oid(&commit->object.oid, "", opts);
2635
+ diffcore_std(opts);
2636
+
2637
+ q = &diff_queued_diff;
2638
+ for (int i = 0; i < q->nr; i++) {
2639
+ struct diff_filepair *p = q->queue[i];
2640
+ struct userdiff_driver *drv;
2641
+
2642
+ /* Skip binary files */
2643
+ drv = userdiff_find_by_path(opts->repo->index, p->one->path);
2644
+ if (drv && drv->binary > 0)
2645
+ continue;
2646
+
2647
+ if (DIFF_FILE_VALID(p->one) &&
2648
+ odb_read_object_info_extended(opts->repo->objects,
2649
+ &p->one->oid, NULL,
2650
+ OBJECT_INFO_FOR_PREFETCH))
2651
+ oidset_insert(blobs, &p->one->oid);
2652
+ if (DIFF_FILE_VALID(p->two) &&
2653
+ odb_read_object_info_extended(opts->repo->objects,
2654
+ &p->two->oid, NULL,
2655
+ OBJECT_INFO_FOR_PREFETCH))
2656
+ oidset_insert(blobs, &p->two->oid);
2657
+ }
2658
+ diff_queue_clear(q);
2659
+}
2660
+
2661
+static int always_match(const void *cmp_data UNUSED,
2662
+ const struct hashmap_entry *entry1 UNUSED,
2663
+ const struct hashmap_entry *entry2 UNUSED,
2664
+ const void *keydata UNUSED)
2665
+{
2666
+ return 0;
2667
+}
2668
+
2669
+/*
2670
+ * Prefetch blobs for git cherry in partial clones.
2671
+ *
2672
+ * Called between the revision walk (which builds the head-side
2673
+ * commit list) and the has_commit_patch_id() comparison loop.
2674
+ *
2675
+ * Uses a cmpfn-swap trick to avoid reading blobs: temporarily
2676
+ * replaces the hashmap's comparison function with a trivial
2677
+ * always-match function, so hashmap_get()/hashmap_get_next() match
2678
+ * any entry with the same oidhash bucket. These are the set of oids
2679
+ * that would trigger patch_id_neq() during normal lookup and cause
2680
+ * blobs to be read on demand, and we want to prefetch them all at
2681
+ * once instead.
2682
+ */
2683
+static void prefetch_cherry_blobs(struct repository *repo,
2684
+ struct commit_list *list,
2685
+ struct patch_ids *ids)
2686
+{
2687
+ struct oidset blobs = OIDSET_INIT;
2688
+ hashmap_cmp_fn original_cmpfn;
2689
+
2690
+ /* Exit if we're not in a partial clone */
2691
+ if (!repo_has_promisor_remote(repo))
2692
+ return;
2693
+
2694
+ /* Save original cmpfn, replace with always_match */
2695
+ original_cmpfn = ids->patches.cmpfn;
2696
+ ids->patches.cmpfn = always_match;
2697
+
2698
+ /* Find header-only collisions, gather blobs from those commits */
2699
+ for (struct commit_list *l = list; l; l = l->next) {
2700
+ struct commit *c = l->item;
2701
+ bool match_found = false;
2702
+ for (struct patch_id *cur = patch_id_iter_first(c, ids);
2703
+ cur;
2704
+ cur = patch_id_iter_next(cur, ids)) {
2705
+ match_found = true;
2706
+ collect_diff_blob_oids(cur->commit, &ids->diffopts,
2707
+ &blobs);
2708
+ }
2709
+ if (match_found)
2710
+ collect_diff_blob_oids(c, &ids->diffopts, &blobs);
2711
+ }
2712
+
2713
+ /* Restore original cmpfn */
2714
+ ids->patches.cmpfn = original_cmpfn;
2715
+
2716
+ /* If we have any blobs to fetch, fetch them */
2717
+ if (oidset_size(&blobs)) {
2718
+ struct oid_array to_fetch = OID_ARRAY_INIT;
2719
+ struct oidset_iter iter;
2720
+ const struct object_id *oid;
2721
+
2722
+ oidset_iter_init(&blobs, &iter);
2723
+ while ((oid = oidset_iter_next(&iter)))
2724
+ oid_array_append(&to_fetch, oid);
2725
+
2726
+ promisor_remote_get_direct(repo, to_fetch.oid, to_fetch.nr);
2727
+
2728
+ oid_array_clear(&to_fetch);
2729
+ }
2730
+
2731
+ oidset_clear(&blobs);
2732
+}
2733
+
2734
int cmd_cherry(int argc,
2735
const char **argv,
2736
const char *prefix,
@@ -2673,6 +2802,8 @@ int cmd_cherry(int argc,
2802
commit_list_insert(commit, &list);
2803
}
2804
2805
+ prefetch_cherry_blobs(the_repository, list, &ids);
2806
+
2807
for (struct commit_list *l = list; l; l = l->next) {
2808
char sign = '+';
2809
t/t3500-cherry.sh
+27
@@ -78,4 +78,31 @@ test_expect_success 'cherry ignores whitespace' '
78
test_cmp expect actual
79
'
80
81
+# Reuse the expect file from the previous test, in a partial clone
82
+test_expect_success 'cherry in partial clone does bulk prefetch' '
83
+ test_config uploadpack.allowfilter 1 &&
84
+ test_config uploadpack.allowanysha1inwant 1 &&
85
+ test_when_finished "rm -rf copy" &&
86
+
87
+ git clone --bare --filter=blob:none file://"$(pwd)" copy &&
88
+ (
89
+ cd copy &&
90
+ GIT_TRACE2_EVENT="$(pwd)/trace.output" git cherry upstream-with-space feature-without-space >actual &&
91
+ test_cmp ../expect actual &&
92
+
93
+ grep "child_start.*fetch.negotiationAlgorithm" trace.output >fetches &&
94
+ test_line_count = 1 fetches &&
95
+ test_trace2_data promisor fetch_count 4 <trace.output &&
96
+
97
+ # A second invocation should not refetch any blobs, since
98
+ # the prefetch is expected to filter out OIDs that are
99
+ # already present locally.
100
+ GIT_TRACE2_EVENT="$(pwd)/trace2.output" git cherry upstream-with-space feature-without-space >actual &&
101
+ test_cmp ../expect actual &&
102
+
103
+ ! grep "child_start.*fetch.negotiationAlgorithm" trace2.output &&
104
+ ! grep "\"key\":\"fetch_count\"" trace2.output
105
+ )
106
+'
107
+
108
test_done