@matheusbsbs / matheus-bagliano / commits / c528e17966

pack-bitmap: write multi-pack bitmaps

Write multi-pack bitmaps in the format described by Documentation/technical/bitmap-format.txt, inferring their presence with the absence of '--bitmap'. To write a multi-pack bitmap, this patch attempts to reuse as much of the existing machinery from pack-objects as possible. Specifically, the MIDX code prepares a packing_data struct that pretends as if a single packfile has been generated containing all of the objects contained within the MIDX. Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Taylor Blau committed Aug 31, 2021 at 16:52 UTC c528e179662ce1353c51f65e1b5895d02988c78c
4 files changed +215 -9
Documentation/git-multi-pack-index.txt
+11 -1
@@ -10,7 +10,7 @@ SYNOPSIS
10 --------
11 [verse]
12 'git multi-pack-index' [--object-dir=<dir>] [--[no-]progress]
13 - [--preferred-pack=<pack>] <subcommand>
13 + [--preferred-pack=<pack>] [--[no-]bitmap] <subcommand>
14
15 DESCRIPTION
16 -----------
@@ -42,6 +42,9 @@ write::
42 multiple packs contain the same object. `<pack>` must
43 contain at least one object. If not given, ties are
44 broken in favor of the pack with the lowest mtime.
45 +
46 + --[no-]bitmap::
47 + Control whether or not a multi-pack bitmap is written.
48 --
49
50 verify::
@@ -83,6 +86,13 @@ EXAMPLES
86 $ git multi-pack-index write
87 -----------------------------------------------
88
89 +* Write a MIDX file for the packfiles in the current .git folder with a
90 +corresponding bitmap.
91 ++
92 +-------------------------------------------------------------
93 +$ git multi-pack-index write --preferred-pack=<pack> --bitmap
94 +-------------------------------------------------------------
95 +
96 * Write a MIDX file for the packfiles in an alternate object store.
97 +
98 -----------------------------------------------
builtin/multi-pack-index.c
+2
@@ -68,6 +68,8 @@ static int cmd_multi_pack_index_write(int argc, const char **argv)
68 OPT_STRING(0, "preferred-pack", &opts.preferred_pack,
69 N_("preferred-pack"),
70 N_("pack for reuse when computing a multi-pack bitmap")),
71 + OPT_BIT(0, "bitmap", &opts.flags, N_("write multi-pack bitmap"),
72 + MIDX_WRITE_BITMAP | MIDX_WRITE_REV_INDEX),
73 OPT_END(),
74 };
75
midx.c
+201 -8
@@ -13,6 +13,10 @@
13 #include "repository.h"
14 #include "chunk-format.h"
15 #include "pack.h"
16 +#include "pack-bitmap.h"
17 +#include "refs.h"
18 +#include "revision.h"
19 +#include "list-objects.h"
20
21 #define MIDX_SIGNATURE 0x4d494458 /* "MIDX" */
22 #define MIDX_VERSION 1
@@ -893,6 +897,166 @@ static int midx_checksum_valid(struct multi_pack_index *m)
897 return hashfile_checksum_valid(m->data, m->data_len);
898 }
899
900 +static void prepare_midx_packing_data(struct packing_data *pdata,
901 + struct write_midx_context *ctx)
902 +{
903 + uint32_t i;
904 +
905 + memset(pdata, 0, sizeof(struct packing_data));
906 + prepare_packing_data(the_repository, pdata);
907 +
908 + for (i = 0; i < ctx->entries_nr; i++) {
909 + struct pack_midx_entry *from = &ctx->entries[ctx->pack_order[i]];
910 + struct object_entry *to = packlist_alloc(pdata, &from->oid);
911 +
912 + oe_set_in_pack(pdata, to,
913 + ctx->info[ctx->pack_perm[from->pack_int_id]].p);
914 + }
915 +}
916 +
917 +static int add_ref_to_pending(const char *refname,
918 + const struct object_id *oid,
919 + int flag, void *cb_data)
920 +{
921 + struct rev_info *revs = (struct rev_info*)cb_data;
922 + struct object *object;
923 +
924 + if ((flag & REF_ISSYMREF) && (flag & REF_ISBROKEN)) {
925 + warning("symbolic ref is dangling: %s", refname);
926 + return 0;
927 + }
928 +
929 + object = parse_object_or_die(oid, refname);
930 + if (object->type != OBJ_COMMIT)
931 + return 0;
932 +
933 + add_pending_object(revs, object, "");
934 + if (bitmap_is_preferred_refname(revs->repo, refname))
935 + object->flags |= NEEDS_BITMAP;
936 + return 0;
937 +}
938 +
939 +struct bitmap_commit_cb {
940 + struct commit **commits;
941 + size_t commits_nr, commits_alloc;
942 +
943 + struct write_midx_context *ctx;
944 +};
945 +
946 +static const struct object_id *bitmap_oid_access(size_t index,
947 + const void *_entries)
948 +{
949 + const struct pack_midx_entry *entries = _entries;
950 + return &entries[index].oid;
951 +}
952 +
953 +static void bitmap_show_commit(struct commit *commit, void *_data)
954 +{
955 + struct bitmap_commit_cb *data = _data;
956 + int pos = oid_pos(&commit->object.oid, data->ctx->entries,
957 + data->ctx->entries_nr,
958 + bitmap_oid_access);
959 + if (pos < 0)
960 + return;
961 +
962 + ALLOC_GROW(data->commits, data->commits_nr + 1, data->commits_alloc);
963 + data->commits[data->commits_nr++] = commit;
964 +}
965 +
966 +static struct commit **find_commits_for_midx_bitmap(uint32_t *indexed_commits_nr_p,
967 + struct write_midx_context *ctx)
968 +{
969 + struct rev_info revs;
970 + struct bitmap_commit_cb cb = {0};
971 +
972 + cb.ctx = ctx;
973 +
974 + repo_init_revisions(the_repository, &revs, NULL);
975 + setup_revisions(0, NULL, &revs, NULL);
976 + for_each_ref(add_ref_to_pending, &revs);
977 +
978 + /*
979 + * Skipping promisor objects here is intentional, since it only excludes
980 + * them from the list of reachable commits that we want to select from
981 + * when computing the selection of MIDX'd commits to receive bitmaps.
982 + *
983 + * Reachability bitmaps do require that their objects be closed under
984 + * reachability, but fetching any objects missing from promisors at this
985 + * point is too late. But, if one of those objects can be reached from
986 + * an another object that is included in the bitmap, then we will
987 + * complain later that we don't have reachability closure (and fail
988 + * appropriately).
989 + */
990 + fetch_if_missing = 0;
991 + revs.exclude_promisor_objects = 1;
992 +
993 + if (prepare_revision_walk(&revs))
994 + die(_("revision walk setup failed"));
995 +
996 + traverse_commit_list(&revs, bitmap_show_commit, NULL, &cb);
997 + if (indexed_commits_nr_p)
998 + *indexed_commits_nr_p = cb.commits_nr;
999 +
1000 + return cb.commits;
1001 +}
1002 +
1003 +static int write_midx_bitmap(char *midx_name, unsigned char *midx_hash,
1004 + struct write_midx_context *ctx,
1005 + unsigned flags)
1006 +{
1007 + struct packing_data pdata;
1008 + struct pack_idx_entry **index;
1009 + struct commit **commits = NULL;
1010 + uint32_t i, commits_nr;
1011 + char *bitmap_name = xstrfmt("%s-%s.bitmap", midx_name, hash_to_hex(midx_hash));
1012 + int ret;
1013 +
1014 + prepare_midx_packing_data(&pdata, ctx);
1015 +
1016 + commits = find_commits_for_midx_bitmap(&commits_nr, ctx);
1017 +
1018 + /*
1019 + * Build the MIDX-order index based on pdata.objects (which is already
1020 + * in MIDX order; c.f., 'midx_pack_order_cmp()' for the definition of
1021 + * this order).
1022 + */
1023 + ALLOC_ARRAY(index, pdata.nr_objects);
1024 + for (i = 0; i < pdata.nr_objects; i++)
1025 + index[i] = &pdata.objects[i].idx;
1026 +
1027 + bitmap_writer_show_progress(flags & MIDX_PROGRESS);
1028 + bitmap_writer_build_type_index(&pdata, index, pdata.nr_objects);
1029 +
1030 + /*
1031 + * bitmap_writer_finish expects objects in lex order, but pack_order
1032 + * gives us exactly that. use it directly instead of re-sorting the
1033 + * array.
1034 + *
1035 + * This changes the order of objects in 'index' between
1036 + * bitmap_writer_build_type_index and bitmap_writer_finish.
1037 + *
1038 + * The same re-ordering takes place in the single-pack bitmap code via
1039 + * write_idx_file(), which is called by finish_tmp_packfile(), which
1040 + * happens between bitmap_writer_build_type_index() and
1041 + * bitmap_writer_finish().
1042 + */
1043 + for (i = 0; i < pdata.nr_objects; i++)
1044 + index[ctx->pack_order[i]] = &pdata.objects[i].idx;
1045 +
1046 + bitmap_writer_select_commits(commits, commits_nr, -1);
1047 + ret = bitmap_writer_build(&pdata);
1048 + if (ret < 0)
1049 + goto cleanup;
1050 +
1051 + bitmap_writer_set_checksum(midx_hash);
1052 + bitmap_writer_finish(index, pdata.nr_objects, bitmap_name, 0);
1053 +
1054 +cleanup:
1055 + free(index);
1056 + free(bitmap_name);
1057 + return ret;
1058 +}
1059 +
1060 static int write_midx_internal(const char *object_dir,
1061 struct string_list *packs_to_drop,
1062 const char *preferred_pack_name,
@@ -941,7 +1105,7 @@ static int write_midx_internal(const char *object_dir,
1105
1106 ctx.info[ctx.nr].orig_pack_int_id = i;
1107 ctx.info[ctx.nr].pack_name = xstrdup(ctx.m->pack_names[i]);
944 - ctx.info[ctx.nr].p = NULL;
1108 + ctx.info[ctx.nr].p = ctx.m->packs[i];
1109 ctx.info[ctx.nr].expired = 0;
1110
1111 if (flags & MIDX_WRITE_REV_INDEX) {
@@ -975,8 +1139,26 @@ static int write_midx_internal(const char *object_dir,
1139 for_each_file_in_pack_dir(object_dir, add_pack_to_midx, &ctx);
1140 stop_progress(&ctx.progress);
1141
978 - if (ctx.m && ctx.nr == ctx.m->num_packs && !packs_to_drop)
979 - goto cleanup;
1142 + if (ctx.m && ctx.nr == ctx.m->num_packs && !packs_to_drop) {
1143 + struct bitmap_index *bitmap_git;
1144 + int bitmap_exists;
1145 + int want_bitmap = flags & MIDX_WRITE_BITMAP;
1146 +
1147 + bitmap_git = prepare_midx_bitmap_git(the_repository, ctx.m);
1148 + bitmap_exists = bitmap_git && bitmap_is_midx(bitmap_git);
1149 + free_bitmap_index(bitmap_git);
1150 +
1151 + if (bitmap_exists || !want_bitmap) {
1152 + /*
1153 + * The correct MIDX already exists, and so does a
1154 + * corresponding bitmap (or one wasn't requested).
1155 + */
1156 + if (!want_bitmap)
1157 + clear_midx_files_ext(object_dir, ".bitmap",
1158 + NULL);
1159 + goto cleanup;
1160 + }
1161 + }
1162
1163 if (preferred_pack_name) {
1164 int found = 0;
@@ -992,7 +1174,8 @@ static int write_midx_internal(const char *object_dir,
1174 if (!found)
1175 warning(_("unknown preferred pack: '%s'"),
1176 preferred_pack_name);
995 - } else if (ctx.nr && (flags & MIDX_WRITE_REV_INDEX)) {
1177 + } else if (ctx.nr &&
1178 + (flags & (MIDX_WRITE_REV_INDEX | MIDX_WRITE_BITMAP))) {
1179 struct packed_git *oldest = ctx.info[ctx.preferred_pack_idx].p;
1180 ctx.preferred_pack_idx = 0;
1181
@@ -1124,9 +1307,6 @@ static int write_midx_internal(const char *object_dir,
1307 hold_lock_file_for_update(&lk, midx_name, LOCK_DIE_ON_ERROR);
1308 f = hashfd(get_lock_file_fd(&lk), get_lock_file_path(&lk));
1309
1127 - if (ctx.m)
1128 - close_object_store(the_repository->objects);
1129 -
1310 if (ctx.nr - dropped_packs == 0) {
1311 error(_("no pack files to index."));
1312 result = 1;
@@ -1157,14 +1337,25 @@ static int write_midx_internal(const char *object_dir,
1337 finalize_hashfile(f, midx_hash, CSUM_FSYNC | CSUM_HASH_IN_STREAM);
1338 free_chunkfile(cf);
1339
1160 - if (flags & MIDX_WRITE_REV_INDEX)
1340 + if (flags & (MIDX_WRITE_REV_INDEX | MIDX_WRITE_BITMAP))
1341 ctx.pack_order = midx_pack_order(&ctx);
1342
1343 if (flags & MIDX_WRITE_REV_INDEX)
1344 write_midx_reverse_index(midx_name, midx_hash, &ctx);
1345 + if (flags & MIDX_WRITE_BITMAP) {
1346 + if (write_midx_bitmap(midx_name, midx_hash, &ctx, flags) < 0) {
1347 + error(_("could not write multi-pack bitmap"));
1348 + result = 1;
1349 + goto cleanup;
1350 + }
1351 + }
1352 +
1353 + if (ctx.m)
1354 + close_object_store(the_repository->objects);
1355
1356 commit_lock_file(&lk);
1357
1358 + clear_midx_files_ext(object_dir, ".bitmap", midx_hash);
1359 clear_midx_files_ext(object_dir, ".rev", midx_hash);
1360
1361 cleanup:
@@ -1181,6 +1372,7 @@ cleanup:
1372 free(ctx.pack_perm);
1373 free(ctx.pack_order);
1374 free(midx_name);
1375 +
1376 return result;
1377 }
1378
@@ -1241,6 +1433,7 @@ void clear_midx_file(struct repository *r)
1433 if (remove_path(midx))
1434 die(_("failed to clear multi-pack-index at %s"), midx);
1435
1436 + clear_midx_files_ext(r->objects->odb->path, ".bitmap", NULL);
1437 clear_midx_files_ext(r->objects->odb->path, ".rev", NULL);
1438
1439 free(midx);
midx.h
+1
@@ -41,6 +41,7 @@ struct multi_pack_index {
41
42 #define MIDX_PROGRESS (1 << 0)
43 #define MIDX_WRITE_REV_INDEX (1 << 1)
44 +#define MIDX_WRITE_BITMAP (1 << 2)
45
46 const unsigned char *get_midx_checksum(struct multi_pack_index *m);
47 char *get_midx_filename(const char *object_dir);