@matheusbsbs / matheus-bagliano / commits / f1b5d3da16

path-walk: support blob size limit filter

Extend the path-walk API to handle the 'blob:limit=<size>' object filter natively. This filter omits blobs whose size is equal to or greater than the given limit, matching the semantics used by the list-objects-filter machinery. When revs->filter.choice is LOFC_BLOB_LIMIT, the prepare_filters() method stores the limit value in info->blob_limit and clears the filter from revs. If the limit is zero, this degenerates to blob:none (all blobs excluded), so info->blobs is set to 0 instead. During walk_path(), blob batches are filtered before being delivered to the callback: each blob's size is checked via odb_read_object_info(), and only blobs strictly smaller than the limit are included. Blobs whose size cannot be determined (e.g. missing in a partial clone) are conservatively included, matching the existing filter behavior. Empty batches after filtering are skipped entirely. The check for inclusion in the path batch looks a little strange at first glance. We use odb_read_object_info() to read the object's size. Based on all of the assumptions to this point, this _should_ return OBJ_BLOB. Since we are focused on the size filter, we use a short-circuited OR (||) to skip the size check if that method returns a different object type. Notice that this inspection of object sizes requires the content to be present in the repository. The odb_read_object_info() call will download a missing blob on-demand. This means that the use of the path-walk API within 'git backfill' would not operate nicely with this filter type. The intention of that command is to download missing blobs in batches. Downloading objects one-by-one would go against the point. Update the validation in 'git backfill' to add its own compatibility check on top of path_walk_filter_compatible(). Add tests for blob:limit=0 (equivalent to blob:none) and blob:limit=3 (which exercises partial filtering within a batch where some blobs are kept and others are excluded). Co-authored-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Derrick Stolee <stolee@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Derrick Stolee committed May 22, 2026 at 18:24 UTC f1b5d3da163f90093ab4f1816df24be54e75a1d9
6 files changed +130 -6
Documentation/git-pack-objects.adoc
+1 -1
@@ -404,7 +404,7 @@ will be automatically changed to version `1`.
404 +
405 Incompatible with `--delta-islands`. The `--use-bitmap-index` option is
406 ignored in the presence of `--path-walk`. The `--path-walk` option
407 -supports the `--filter=<spec>` form `blob:none`.
407 +supports the `--filter=<spec>` forms `blob:none` and `blob:limit=<n>`.
408
409
410 DELTA ISLANDS
builtin/backfill.c
+2
@@ -98,6 +98,8 @@ static void reject_unsupported_rev_list_options(struct rev_info *revs)
98 "--diff-merges");
99 if (!path_walk_filter_compatible(&revs->filter))
100 die(_("cannot backfill with these filter options"));
101 + if (revs->filter.blob_limit_value)
102 + die(_("cannot backfill with blob size limits"));
103 }
104
105 static int do_backfill(struct backfill_context *ctx)
path-walk.c
+37 -4
@@ -10,6 +10,7 @@
10 #include "hex.h"
11 #include "list-objects.h"
12 #include "list-objects-filter-options.h"
13 +#include "odb.h"
14 #include "object.h"
15 #include "oid-array.h"
16 #include "path.h"
@@ -327,13 +328,35 @@ static int walk_path(struct path_walk_context *ctx,
328 /*
329 * Evaluate function pointer on this data, if requested.
330 * Ignore object type filters for tagged objects (path starts
330 - * with `/`).
331 + * with `/`), first for blobs and then other types.
332 */
332 - if ((list->type == OBJ_TREE && (ctx->info->trees || path_is_for_direct_objects(path))) ||
333 - (list->type == OBJ_BLOB && (ctx->info->blobs || path_is_for_direct_objects(path))) ||
334 - (list->type == OBJ_TAG && ctx->info->tags))
333 + if (list->type == OBJ_BLOB &&
334 + ctx->info->blob_limit &&
335 + !path_is_for_direct_objects(path)) {
336 + struct oid_array filtered = OID_ARRAY_INIT;
337 +
338 + for (size_t i = 0; i < list->oids.nr; i++) {
339 + unsigned long size;
340 +
341 + if (odb_read_object_info(ctx->repo->objects,
342 + &list->oids.oid[i],
343 + &size) != OBJ_BLOB ||
344 + size < ctx->info->blob_limit)
345 + oid_array_append(&filtered,
346 + &list->oids.oid[i]);
347 + }
348 +
349 + if (filtered.nr)
350 + ret = ctx->info->path_fn(path, &filtered, list->type,
351 + ctx->info->path_fn_data);
352 + oid_array_clear(&filtered);
353 + } else if (path_is_for_direct_objects(path) ||
354 + (list->type == OBJ_TREE && ctx->info->trees) ||
355 + (list->type == OBJ_BLOB && ctx->info->blobs) ||
356 + (list->type == OBJ_TAG && ctx->info->tags)) {
357 ret = ctx->info->path_fn(path, &list->oids, list->type,
358 ctx->info->path_fn_data);
359 + }
360
361 /* Expand data for children. */
362 if (list->type == OBJ_TREE) {
@@ -510,6 +533,16 @@ static int prepare_filters(struct path_walk_info *info,
533 }
534 return 1;
535
536 + case LOFC_BLOB_LIMIT:
537 + if (info) {
538 + if (!options->blob_limit_value)
539 + info->blobs = 0;
540 + else
541 + info->blob_limit = options->blob_limit_value;
542 + list_objects_filter_release(options);
543 + }
544 + return 1;
545 +
546 default:
547 error(_("object filter '%s' not supported by the path-walk API"),
548 list_objects_filter_spec(options));
path-walk.h
+7
@@ -47,6 +47,13 @@ struct path_walk_info {
47 int blobs;
48 int tags;
49
50 + /**
51 + * If non-zero, specifies a maximum blob size. Blobs with a
52 + * size equal to or greater than this limit will not be
53 + * emitted unless included in 'pending'.
54 + */
55 + unsigned long blob_limit;
56 +
57 /**
58 * When 'prune_all_uninteresting' is set and a path has all objects
59 * marked as UNINTERESTING, then the path-walk will not visit those
t/t5620-backfill.sh
+1 -1
@@ -20,7 +20,7 @@ test_expect_success 'backfill rejects incompatible filter options' '
20 test_grep "cannot backfill with these filter options" err &&
21
22 test_must_fail git backfill --objects --filter=blob:limit=10m 2>err &&
23 - test_grep "cannot backfill with these filter options" err
23 + test_grep "cannot backfill with blob size limits" err
24 '
25
26 # We create objects in the 'src' repo.
t/t6601-path-walk.sh
+82
@@ -477,4 +477,86 @@ test_expect_success 'topic only, blob:none filter' '
477 test_cmp_sorted expect out
478 '
479
480 +test_expect_success 'all, blob:limit=0 filter' '
481 + test-tool path-walk --filter=blob:limit=0 -- --all >out &&
482 +
483 + cat >expect <<-EOF &&
484 + 0:commit::$(git rev-parse topic)
485 + 0:commit::$(git rev-parse base)
486 + 0:commit::$(git rev-parse base~1)
487 + 0:commit::$(git rev-parse base~2)
488 + 1:tag:/tags:$(git rev-parse refs/tags/first)
489 + 1:tag:/tags:$(git rev-parse refs/tags/second.1)
490 + 1:tag:/tags:$(git rev-parse refs/tags/second.2)
491 + 1:tag:/tags:$(git rev-parse refs/tags/third)
492 + 1:tag:/tags:$(git rev-parse refs/tags/fourth)
493 + 1:tag:/tags:$(git rev-parse refs/tags/tree-tag)
494 + 1:tag:/tags:$(git rev-parse refs/tags/blob-tag)
495 + 2:blob:/tagged-blobs:$(git rev-parse refs/tags/blob-tag^{})
496 + 2:blob:/tagged-blobs:$(git rev-parse refs/tags/blob-tag2^{})
497 + 3:tree::$(git rev-parse topic^{tree})
498 + 3:tree::$(git rev-parse base^{tree})
499 + 3:tree::$(git rev-parse base~1^{tree})
500 + 3:tree::$(git rev-parse base~2^{tree})
501 + 3:tree::$(git rev-parse refs/tags/tree-tag^{})
502 + 3:tree::$(git rev-parse refs/tags/tree-tag2^{})
503 + 4:tree:a/:$(git rev-parse base:a)
504 + 5:tree:child/:$(git rev-parse refs/tags/tree-tag:child)
505 + 6:tree:left/:$(git rev-parse base:left)
506 + 6:tree:left/:$(git rev-parse base~2:left)
507 + 7:tree:right/:$(git rev-parse topic:right)
508 + 7:tree:right/:$(git rev-parse base~1:right)
509 + 7:tree:right/:$(git rev-parse base~2:right)
510 + blobs:2
511 + commits:4
512 + tags:7
513 + trees:13
514 + EOF
515 +
516 + test_cmp_sorted expect out
517 +'
518 +
519 +test_expect_success 'all, blob:limit=3 filter' '
520 + test-tool path-walk --filter=blob:limit=3 -- --all >out &&
521 +
522 + cat >expect <<-EOF &&
523 + 0:commit::$(git rev-parse topic)
524 + 0:commit::$(git rev-parse base)
525 + 0:commit::$(git rev-parse base~1)
526 + 0:commit::$(git rev-parse base~2)
527 + 1:tag:/tags:$(git rev-parse refs/tags/first)
528 + 1:tag:/tags:$(git rev-parse refs/tags/second.1)
529 + 1:tag:/tags:$(git rev-parse refs/tags/second.2)
530 + 1:tag:/tags:$(git rev-parse refs/tags/third)
531 + 1:tag:/tags:$(git rev-parse refs/tags/fourth)
532 + 1:tag:/tags:$(git rev-parse refs/tags/tree-tag)
533 + 1:tag:/tags:$(git rev-parse refs/tags/blob-tag)
534 + 2:blob:/tagged-blobs:$(git rev-parse refs/tags/blob-tag^{})
535 + 2:blob:/tagged-blobs:$(git rev-parse refs/tags/blob-tag2^{})
536 + 3:tree::$(git rev-parse topic^{tree})
537 + 3:tree::$(git rev-parse base^{tree})
538 + 3:tree::$(git rev-parse base~1^{tree})
539 + 3:tree::$(git rev-parse base~2^{tree})
540 + 3:tree::$(git rev-parse refs/tags/tree-tag^{})
541 + 3:tree::$(git rev-parse refs/tags/tree-tag2^{})
542 + 4:blob:a:$(git rev-parse base~2:a)
543 + 5:tree:a/:$(git rev-parse base:a)
544 + 6:tree:child/:$(git rev-parse refs/tags/tree-tag:child)
545 + 7:tree:left/:$(git rev-parse base:left)
546 + 7:tree:left/:$(git rev-parse base~2:left)
547 + 8:blob:left/b:$(git rev-parse base~2:left/b)
548 + 9:tree:right/:$(git rev-parse topic:right)
549 + 9:tree:right/:$(git rev-parse base~1:right)
550 + 9:tree:right/:$(git rev-parse base~2:right)
551 + 10:blob:right/c:$(git rev-parse base~2:right/c)
552 + 11:blob:right/d:$(git rev-parse base~1:right/d)
553 + blobs:6
554 + commits:4
555 + tags:7
556 + trees:13
557 + EOF
558 +
559 + test_cmp_sorted expect out
560 +'
561 +
562 test_done