@matheusbsbs / matheus-bagliano / commits / 4f3bd5606a
Fork 0

pack-bitmap: implement BLOB_NONE filtering

We can easily support BLOB_NONE filters with bitmaps. Since we know the types of all of the objects, we just need to clear the result bits of any blobs. Note two subtleties in the implementation (which I also called out in comments): - we have to include any blobs that were specifically asked for (and not reached through graph traversal) to match the non-bitmap version - we have to handle in-pack and "ext_index" objects separately. Arguably prepare_bitmap_walk() could be adding these ext_index objects to the type bitmaps. But it doesn't for now, so let's match the rest of the bitmap code here (it probably wouldn't be an efficiency improvement to do so since the cost of extending those bitmaps is about the same as our loop here, but it might make the code a bit simpler). Here are perf results for the new test on git.git: Test HEAD^ HEAD -------------------------------------------------------------------------------- 5310.9: rev-list count with blob:none 1.67(1.62+0.05) 0.22(0.21+0.02) -86.8% Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Jeff King committed Feb 14, 2020 at 13:22 UTC 4f3bd5606a02260274555f41fd7d6368f2bea1d8
3 files changed +93
pack-bitmap.c
+74
@@ -712,6 +712,73 @@ static int in_bitmapped_pack(struct bitmap_index *bitmap_git,
712 return 0;
713 }
714
715 +static struct bitmap *find_tip_blobs(struct bitmap_index *bitmap_git,
716 + struct object_list *tip_objects)
717 +{
718 + struct bitmap *result = bitmap_new();
719 + struct object_list *p;
720 +
721 + for (p = tip_objects; p; p = p->next) {
722 + int pos;
723 +
724 + if (p->item->type != OBJ_BLOB)
725 + continue;
726 +
727 + pos = bitmap_position(bitmap_git, &p->item->oid);
728 + if (pos < 0)
729 + continue;
730 +
731 + bitmap_set(result, pos);
732 + }
733 +
734 + return result;
735 +}
736 +
737 +static void filter_bitmap_blob_none(struct bitmap_index *bitmap_git,
738 + struct object_list *tip_objects,
739 + struct bitmap *to_filter)
740 +{
741 + struct eindex *eindex = &bitmap_git->ext_index;
742 + struct bitmap *tips;
743 + struct ewah_iterator it;
744 + eword_t mask;
745 + uint32_t i;
746 +
747 + /*
748 + * The non-bitmap version of this filter never removes
749 + * blobs which the other side specifically asked for,
750 + * so we must match that behavior.
751 + */
752 + tips = find_tip_blobs(bitmap_git, tip_objects);
753 +
754 + /*
755 + * We can use the blob type-bitmap to work in whole words
756 + * for the objects that are actually in the bitmapped packfile.
757 + */
758 + for (i = 0, init_type_iterator(&it, bitmap_git, OBJ_BLOB);
759 + i < to_filter->word_alloc && ewah_iterator_next(&mask, &it);
760 + i++) {
761 + if (i < tips->word_alloc)
762 + mask &= ~tips->words[i];
763 + to_filter->words[i] &= ~mask;
764 + }
765 +
766 + /*
767 + * Clear any blobs that weren't in the packfile (and so would not have
768 + * been caught by the loop above. We'll have to check them
769 + * individually.
770 + */
771 + for (i = 0; i < eindex->count; i++) {
772 + uint32_t pos = i + bitmap_git->pack->num_objects;
773 + if (eindex->objects[i]->type == OBJ_BLOB &&
774 + bitmap_get(to_filter, pos) &&
775 + !bitmap_get(tips, pos))
776 + bitmap_unset(to_filter, pos);
777 + }
778 +
779 + bitmap_free(tips);
780 +}
781 +
782 static int filter_bitmap(struct bitmap_index *bitmap_git,
783 struct object_list *tip_objects,
784 struct bitmap *to_filter,
@@ -720,6 +787,13 @@ static int filter_bitmap(struct bitmap_index *bitmap_git,
787 if (!filter || filter->choice == LOFC_DISABLED)
788 return 0;
789
790 + if (filter->choice == LOFC_BLOB_NONE) {
791 + if (bitmap_git)
792 + filter_bitmap_blob_none(bitmap_git, tip_objects,
793 + to_filter);
794 + return 0;
795 + }
796 +
797 /* filter choice not handled */
798 return -1;
799 }
t/perf/p5310-pack-bitmaps.sh
+5
@@ -47,6 +47,11 @@ test_perf 'rev-list (objects)' '
47 git rev-list --all --use-bitmap-index --objects >/dev/null
48 '
49
50 +test_perf 'rev-list count with blob:none' '
51 + git rev-list --use-bitmap-index --count --objects --all \
52 + --filter=blob:none >/dev/null
53 +'
54 +
55 test_expect_success 'create partial bitmap state' '
56 # pick a commit to represent the repo tip in the past
57 cutoff=$(git rev-list HEAD~100 -1) &&
t/t6113-rev-list-bitmap-filters.sh
+14
@@ -21,4 +21,18 @@ test_expect_success 'filters fallback to non-bitmap traversal' '
21 test_cmp expect actual
22 '
23
24 +test_expect_success 'blob:none filter' '
25 + git rev-list --objects --filter=blob:none HEAD >expect &&
26 + git rev-list --use-bitmap-index \
27 + --objects --filter=blob:none HEAD >actual &&
28 + test_bitmap_traversal expect actual
29 +'
30 +
31 +test_expect_success 'blob:none filter with specified blob' '
32 + git rev-list --objects --filter=blob:none HEAD HEAD:two.t >expect &&
33 + git rev-list --use-bitmap-index \
34 + --objects --filter=blob:none HEAD HEAD:two.t >actual &&
35 + test_bitmap_traversal expect actual
36 +'
37 +
38 test_done