for-each-ref: introduce a '--start-after' option
The `git-for-each-ref(1)` command is used to iterate over references present in a repository. In large repositories with millions of references, it would be optimal to paginate this output such that we can start iteration from a given reference. This would avoid having to iterate over all references from the beginning each time when paginating through results. The previous commit added 'seek' functionality to the reference backends. Utilize this and expose a '--start-after' option in 'git-for-each-ref(1)'. When used, the reference iteration seeks to the lexicographically next reference and iterates from there onward. This enables efficient pagination workflows, where the calling script can remember the last provided reference and use that as the starting point for the next set of references: git for-each-ref --count=100 git for-each-ref --count=100 --start-after=refs/heads/branch-100 git for-each-ref --count=100 --start-after=refs/heads/branch-200 Since the reference iterators only allow seeking to a specified marker via the `ref_iterator_seek()`, we introduce a helper function `start_ref_iterator_after()`, which seeks to next reference by simply adding (char) 1 to the marker. We must note that pagination always continues from the provided marker, as such any concurrent reference updates lexicographically behind the marker will not be output. Document the same. Signed-off-by: Karthik Nayak <karthik.188@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Karthik Nayak committed
Jul 15, 2025 at 13:28 UTC
dabecb9db2b25a32d72c90832f338849b665fdf9
5 files changed
+272
-19
Documentation/git-for-each-ref.adoc
+9
-1
@@ -14,7 +14,7 @@ SYNOPSIS
14
[--points-at=<object>]
15
[--merged[=<object>]] [--no-merged[=<object>]]
16
[--contains[=<object>]] [--no-contains[=<object>]]
17
- [--exclude=<pattern> ...]
17
+ [--exclude=<pattern> ...] [--start-after=<marker>]
18
19
DESCRIPTION
20
-----------
@@ -108,6 +108,14 @@ TAB %(refname)`.
108
--include-root-refs::
109
List root refs (HEAD and pseudorefs) apart from regular refs.
110
111
+--start-after=<marker>::
112
+ Allows paginating the output by skipping references up to and including the
113
+ specified marker. When paging, it should be noted that references may be
114
+ deleted, modified or added between invocations. Output will only yield those
115
+ references which follow the marker lexicographically. Output begins from the
116
+ first reference that would come after the marker alphabetically. Cannot be
117
+ used with general pattern matching or custom sort options.
118
+
119
FIELD NAMES
120
-----------
121
builtin/for-each-ref.c
+8
@@ -13,6 +13,7 @@ static char const * const for_each_ref_usage[] = {
13
N_("git for-each-ref [--points-at <object>]"),
14
N_("git for-each-ref [--merged [<commit>]] [--no-merged [<commit>]]"),
15
N_("git for-each-ref [--contains [<commit>]] [--no-contains [<commit>]]"),
16
+ N_("git for-each-ref [--start-after <marker>]"),
17
NULL
18
};
19
@@ -44,6 +45,7 @@ int cmd_for_each_ref(int argc,
45
OPT_GROUP(""),
46
OPT_INTEGER( 0 , "count", &format.array_opts.max_count, N_("show only <n> matched refs")),
47
OPT_STRING( 0 , "format", &format.format, N_("format"), N_("format to use for the output")),
48
+ OPT_STRING( 0 , "start-after", &filter.start_after, N_("start-start"), N_("start iteration after the provided marker")),
49
OPT__COLOR(&format.use_color, N_("respect format colors")),
50
OPT_REF_FILTER_EXCLUDE(&filter),
51
OPT_REF_SORT(&sorting_options),
@@ -79,6 +81,9 @@ int cmd_for_each_ref(int argc,
81
if (verify_ref_format(&format))
82
usage_with_options(for_each_ref_usage, opts);
83
84
+ if (filter.start_after && sorting_options.nr > 1)
85
+ die(_("cannot use --start-after with custom sort options"));
86
+
87
sorting = ref_sorting_options(&sorting_options);
88
ref_sorting_set_sort_flags_all(sorting, REF_SORTING_ICASE, icase);
89
filter.ignore_case = icase;
@@ -100,6 +105,9 @@ int cmd_for_each_ref(int argc,
105
filter.name_patterns = argv;
106
}
107
108
+ if (filter.start_after && filter.name_patterns && filter.name_patterns[0])
109
+ die(_("cannot use --start-after with patterns"));
110
+
111
if (include_root_refs)
112
flags |= FILTER_REFS_ROOT_REFS | FILTER_REFS_DETACHED_HEAD;
113
ref-filter.c
+60
-18
@@ -2683,6 +2683,41 @@ static int filter_exclude_match(struct ref_filter *filter, const char *refname)
2683
return match_pattern(filter->exclude.v, refname, filter->ignore_case);
2684
}
2685
2686
+/*
2687
+ * We need to seek to the reference right after a given marker but excluding any
2688
+ * matching references. So we seek to the lexicographically next reference.
2689
+ */
2690
+static int start_ref_iterator_after(struct ref_iterator *iter, const char *marker)
2691
+{
2692
+ struct strbuf sb = STRBUF_INIT;
2693
+ int ret;
2694
+
2695
+ strbuf_addstr(&sb, marker);
2696
+ strbuf_addch(&sb, 1);
2697
+
2698
+ ret = ref_iterator_seek(iter, sb.buf, 0);
2699
+
2700
+ strbuf_release(&sb);
2701
+ return ret;
2702
+}
2703
+
2704
+static int for_each_fullref_with_seek(struct ref_filter *filter, each_ref_fn cb,
2705
+ void *cb_data, unsigned int flags)
2706
+{
2707
+ struct ref_iterator *iter;
2708
+ int ret = 0;
2709
+
2710
+ iter = refs_ref_iterator_begin(get_main_ref_store(the_repository), "",
2711
+ NULL, 0, flags);
2712
+ if (filter->start_after)
2713
+ ret = start_ref_iterator_after(iter, filter->start_after);
2714
+
2715
+ if (ret)
2716
+ return ret;
2717
+
2718
+ return do_for_each_ref_iterator(iter, cb, cb_data);
2719
+}
2720
+
2721
/*
2722
* This is the same as for_each_fullref_in(), but it tries to iterate
2723
* only over the patterns we'll care about. Note that it _doesn't_ do a full
@@ -2694,8 +2729,8 @@ static int for_each_fullref_in_pattern(struct ref_filter *filter,
2729
{
2730
if (filter->kind & FILTER_REFS_ROOT_REFS) {
2731
/* In this case, we want to print all refs including root refs. */
2697
- return refs_for_each_include_root_refs(get_main_ref_store(the_repository),
2698
- cb, cb_data);
2732
+ return for_each_fullref_with_seek(filter, cb, cb_data,
2733
+ DO_FOR_EACH_INCLUDE_ROOT_REFS);
2734
}
2735
2736
if (!filter->match_as_path) {
@@ -2704,8 +2739,7 @@ static int for_each_fullref_in_pattern(struct ref_filter *filter,
2739
* prefixes like "refs/heads/" etc. are stripped off,
2740
* so we have to look at everything:
2741
*/
2707
- return refs_for_each_fullref_in(get_main_ref_store(the_repository),
2708
- "", NULL, cb, cb_data);
2742
+ return for_each_fullref_with_seek(filter, cb, cb_data, 0);
2743
}
2744
2745
if (filter->ignore_case) {
@@ -2714,14 +2748,12 @@ static int for_each_fullref_in_pattern(struct ref_filter *filter,
2748
* so just return everything and let the caller
2749
* sort it out.
2750
*/
2717
- return refs_for_each_fullref_in(get_main_ref_store(the_repository),
2718
- "", NULL, cb, cb_data);
2751
+ return for_each_fullref_with_seek(filter, cb, cb_data, 0);
2752
}
2753
2754
if (!filter->name_patterns[0]) {
2755
/* no patterns; we have to look at everything */
2723
- return refs_for_each_fullref_in(get_main_ref_store(the_repository),
2724
- "", filter->exclude.v, cb, cb_data);
2756
+ return for_each_fullref_with_seek(filter, cb, cb_data, 0);
2757
}
2758
2759
return refs_for_each_fullref_in_prefixes(get_main_ref_store(the_repository),
@@ -3189,6 +3221,7 @@ void filter_is_base(struct repository *r,
3221
3222
static int do_filter_refs(struct ref_filter *filter, unsigned int type, each_ref_fn fn, void *cb_data)
3223
{
3224
+ const char *prefix = NULL;
3225
int ret = 0;
3226
3227
filter->kind = type & FILTER_REFS_KIND_MASK;
@@ -3207,19 +3240,28 @@ static int do_filter_refs(struct ref_filter *filter, unsigned int type, each_ref
3240
* of filter_ref_kind().
3241
*/
3242
if (filter->kind == FILTER_REFS_BRANCHES)
3210
- ret = refs_for_each_fullref_in(get_main_ref_store(the_repository),
3211
- "refs/heads/", NULL,
3212
- fn, cb_data);
3243
+ prefix = "refs/heads/";
3244
else if (filter->kind == FILTER_REFS_REMOTES)
3214
- ret = refs_for_each_fullref_in(get_main_ref_store(the_repository),
3215
- "refs/remotes/", NULL,
3216
- fn, cb_data);
3245
+ prefix = "refs/remotes/";
3246
else if (filter->kind == FILTER_REFS_TAGS)
3218
- ret = refs_for_each_fullref_in(get_main_ref_store(the_repository),
3219
- "refs/tags/", NULL, fn,
3220
- cb_data);
3221
- else if (filter->kind & FILTER_REFS_REGULAR)
3247
+ prefix = "refs/tags/";
3248
+
3249
+ if (prefix) {
3250
+ struct ref_iterator *iter;
3251
+
3252
+ iter = refs_ref_iterator_begin(get_main_ref_store(the_repository),
3253
+ "", NULL, 0, 0);
3254
+
3255
+ if (filter->start_after)
3256
+ ret = start_ref_iterator_after(iter, filter->start_after);
3257
+ else if (prefix)
3258
+ ret = ref_iterator_seek(iter, prefix, 1);
3259
+
3260
+ if (!ret)
3261
+ ret = do_for_each_ref_iterator(iter, fn, cb_data);
3262
+ } else if (filter->kind & FILTER_REFS_REGULAR) {
3263
ret = for_each_fullref_in_pattern(filter, fn, cb_data);
3264
+ }
3265
3266
/*
3267
* When printing all ref types, HEAD is already included,
ref-filter.h
+1
@@ -64,6 +64,7 @@ struct ref_array {
64
65
struct ref_filter {
66
const char **name_patterns;
67
+ const char *start_after;
68
struct strvec exclude;
69
struct oid_array points_at;
70
struct commit_list *with_commit;
t/t6302-for-each-ref-filter.sh
+194
@@ -541,4 +541,198 @@ test_expect_success 'validate worktree atom' '
541
test_cmp expect actual
542
'
543
544
+test_expect_success 'start after with empty value' '
545
+ cat >expect <<-\EOF &&
546
+ refs/heads/main
547
+ refs/heads/main_worktree
548
+ refs/heads/side
549
+ refs/odd/spot
550
+ refs/tags/annotated-tag
551
+ refs/tags/doubly-annotated-tag
552
+ refs/tags/doubly-signed-tag
553
+ refs/tags/foo1.10
554
+ refs/tags/foo1.3
555
+ refs/tags/foo1.6
556
+ refs/tags/four
557
+ refs/tags/one
558
+ refs/tags/signed-tag
559
+ refs/tags/three
560
+ refs/tags/two
561
+ EOF
562
+ git for-each-ref --format="%(refname)" --start-after="" >actual &&
563
+ test_cmp expect actual
564
+'
565
+
566
+test_expect_success 'start after a specific reference' '
567
+ cat >expect <<-\EOF &&
568
+ refs/tags/annotated-tag
569
+ refs/tags/doubly-annotated-tag
570
+ refs/tags/doubly-signed-tag
571
+ refs/tags/foo1.10
572
+ refs/tags/foo1.3
573
+ refs/tags/foo1.6
574
+ refs/tags/four
575
+ refs/tags/one
576
+ refs/tags/signed-tag
577
+ refs/tags/three
578
+ refs/tags/two
579
+ EOF
580
+ git for-each-ref --format="%(refname)" --start-after=refs/odd/spot >actual &&
581
+ test_cmp expect actual
582
+'
583
+
584
+test_expect_success 'start after a specific reference with partial match' '
585
+ cat >expect <<-\EOF &&
586
+ refs/odd/spot
587
+ refs/tags/annotated-tag
588
+ refs/tags/doubly-annotated-tag
589
+ refs/tags/doubly-signed-tag
590
+ refs/tags/foo1.10
591
+ refs/tags/foo1.3
592
+ refs/tags/foo1.6
593
+ refs/tags/four
594
+ refs/tags/one
595
+ refs/tags/signed-tag
596
+ refs/tags/three
597
+ refs/tags/two
598
+ EOF
599
+ git for-each-ref --format="%(refname)" --start-after=refs/odd/sp >actual &&
600
+ test_cmp expect actual
601
+'
602
+
603
+test_expect_success 'start after, just behind a specific reference' '
604
+ cat >expect <<-\EOF &&
605
+ refs/odd/spot
606
+ refs/tags/annotated-tag
607
+ refs/tags/doubly-annotated-tag
608
+ refs/tags/doubly-signed-tag
609
+ refs/tags/foo1.10
610
+ refs/tags/foo1.3
611
+ refs/tags/foo1.6
612
+ refs/tags/four
613
+ refs/tags/one
614
+ refs/tags/signed-tag
615
+ refs/tags/three
616
+ refs/tags/two
617
+ EOF
618
+ git for-each-ref --format="%(refname)" --start-after=refs/odd/parrot >actual &&
619
+ test_cmp expect actual
620
+'
621
+
622
+test_expect_success 'start after with specific directory match' '
623
+ cat >expect <<-\EOF &&
624
+ refs/odd/spot
625
+ refs/tags/annotated-tag
626
+ refs/tags/doubly-annotated-tag
627
+ refs/tags/doubly-signed-tag
628
+ refs/tags/foo1.10
629
+ refs/tags/foo1.3
630
+ refs/tags/foo1.6
631
+ refs/tags/four
632
+ refs/tags/one
633
+ refs/tags/signed-tag
634
+ refs/tags/three
635
+ refs/tags/two
636
+ EOF
637
+ git for-each-ref --format="%(refname)" --start-after=refs/odd >actual &&
638
+ test_cmp expect actual
639
+'
640
+
641
+test_expect_success 'start after with specific directory and trailing slash' '
642
+ cat >expect <<-\EOF &&
643
+ refs/odd/spot
644
+ refs/tags/annotated-tag
645
+ refs/tags/doubly-annotated-tag
646
+ refs/tags/doubly-signed-tag
647
+ refs/tags/foo1.10
648
+ refs/tags/foo1.3
649
+ refs/tags/foo1.6
650
+ refs/tags/four
651
+ refs/tags/one
652
+ refs/tags/signed-tag
653
+ refs/tags/three
654
+ refs/tags/two
655
+ EOF
656
+ git for-each-ref --format="%(refname)" --start-after=refs/odd/ >actual &&
657
+ test_cmp expect actual
658
+'
659
+
660
+test_expect_success 'start after, just behind a specific directory' '
661
+ cat >expect <<-\EOF &&
662
+ refs/odd/spot
663
+ refs/tags/annotated-tag
664
+ refs/tags/doubly-annotated-tag
665
+ refs/tags/doubly-signed-tag
666
+ refs/tags/foo1.10
667
+ refs/tags/foo1.3
668
+ refs/tags/foo1.6
669
+ refs/tags/four
670
+ refs/tags/one
671
+ refs/tags/signed-tag
672
+ refs/tags/three
673
+ refs/tags/two
674
+ EOF
675
+ git for-each-ref --format="%(refname)" --start-after=refs/lost >actual &&
676
+ test_cmp expect actual
677
+'
678
+
679
+test_expect_success 'start after, overflow specific reference length' '
680
+ cat >expect <<-\EOF &&
681
+ refs/tags/annotated-tag
682
+ refs/tags/doubly-annotated-tag
683
+ refs/tags/doubly-signed-tag
684
+ refs/tags/foo1.10
685
+ refs/tags/foo1.3
686
+ refs/tags/foo1.6
687
+ refs/tags/four
688
+ refs/tags/one
689
+ refs/tags/signed-tag
690
+ refs/tags/three
691
+ refs/tags/two
692
+ EOF
693
+ git for-each-ref --format="%(refname)" --start-after=refs/odd/spotnew >actual &&
694
+ test_cmp expect actual
695
+'
696
+
697
+test_expect_success 'start after, overflow specific reference path' '
698
+ cat >expect <<-\EOF &&
699
+ refs/tags/annotated-tag
700
+ refs/tags/doubly-annotated-tag
701
+ refs/tags/doubly-signed-tag
702
+ refs/tags/foo1.10
703
+ refs/tags/foo1.3
704
+ refs/tags/foo1.6
705
+ refs/tags/four
706
+ refs/tags/one
707
+ refs/tags/signed-tag
708
+ refs/tags/three
709
+ refs/tags/two
710
+ EOF
711
+ git for-each-ref --format="%(refname)" --start-after=refs/odd/spot/new >actual &&
712
+ test_cmp expect actual
713
+'
714
+
715
+test_expect_success 'start after, last reference' '
716
+ cat >expect <<-\EOF &&
717
+ EOF
718
+ git for-each-ref --format="%(refname)" --start-after=refs/tags/two >actual &&
719
+ test_cmp expect actual
720
+'
721
+
722
+test_expect_success 'start after used with a pattern' '
723
+ cat >expect <<-\EOF &&
724
+ fatal: cannot use --start-after with patterns
725
+ EOF
726
+ test_must_fail git for-each-ref --format="%(refname)" --start-after=refs/odd/spot refs/tags 2>actual &&
727
+ test_cmp expect actual
728
+'
729
+
730
+test_expect_success 'start after used with custom sort order' '
731
+ cat >expect <<-\EOF &&
732
+ fatal: cannot use --start-after with custom sort options
733
+ EOF
734
+ test_must_fail git for-each-ref --format="%(refname)" --start-after=refs/odd/spot --sort=author 2>actual &&
735
+ test_cmp expect actual
736
+'
737
+
738
test_done