pack-revindex: write multi-pack reverse indexes
Implement the writing half of multi-pack reverse indexes. This is nothing more than the format describe a few patches ago, with a new set of helper functions that will be used to clear out stale .rev files corresponding to old MIDXs. Unfortunately, a very similar comparison function as the one implemented recently in pack-revindex.c is reimplemented here, this time accepting a MIDX-internal type. An effort to DRY these up would create more indirection and overhead than is necessary, so it isn't pursued here. Currently, there are no callers which pass the MIDX_WRITE_REV_INDEX flag, meaning that this is all dead code. But, that won't be the case for long, since subsequent patches will introduce the multi-pack bitmap, which will begin passing this field. (In midx.c:write_midx_internal(), the two adjacent if statements share a conditional, but are written separately since the first one will eventually also handle the MIDX_WRITE_BITMAP flag, which does not yet exist.) Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Taylor Blau committed
Mar 30, 2021 at 11:04 UTC
38ff7cabb6b8e51df78ce20c20632eba24265ee4
2 files changed
+116
midx.c
+115
@@ -12,6 +12,7 @@
12
#include "run-command.h"
13
#include "repository.h"
14
#include "chunk-format.h"
15
+#include "pack.h"
16
17
#define MIDX_SIGNATURE 0x4d494458 /* "MIDX" */
18
#define MIDX_VERSION 1
@@ -462,6 +463,7 @@ struct write_midx_context {
463
uint32_t entries_nr;
464
465
uint32_t *pack_perm;
466
+ uint32_t *pack_order;
467
unsigned large_offsets_needed:1;
468
uint32_t num_large_offsets;
469
@@ -816,6 +818,70 @@ static int write_midx_large_offsets(struct hashfile *f,
818
return 0;
819
}
820
821
+static int midx_pack_order_cmp(const void *va, const void *vb, void *_ctx)
822
+{
823
+ struct write_midx_context *ctx = _ctx;
824
+
825
+ struct pack_midx_entry *a = &ctx->entries[*(const uint32_t *)va];
826
+ struct pack_midx_entry *b = &ctx->entries[*(const uint32_t *)vb];
827
+
828
+ uint32_t perm_a = ctx->pack_perm[a->pack_int_id];
829
+ uint32_t perm_b = ctx->pack_perm[b->pack_int_id];
830
+
831
+ /* Sort objects in the preferred pack ahead of any others. */
832
+ if (a->preferred > b->preferred)
833
+ return -1;
834
+ if (a->preferred < b->preferred)
835
+ return 1;
836
+
837
+ /* Then, order objects by which packs they appear in. */
838
+ if (perm_a < perm_b)
839
+ return -1;
840
+ if (perm_a > perm_b)
841
+ return 1;
842
+
843
+ /* Then, disambiguate by their offset within each pack. */
844
+ if (a->offset < b->offset)
845
+ return -1;
846
+ if (a->offset > b->offset)
847
+ return 1;
848
+
849
+ return 0;
850
+}
851
+
852
+static uint32_t *midx_pack_order(struct write_midx_context *ctx)
853
+{
854
+ uint32_t *pack_order;
855
+ uint32_t i;
856
+
857
+ ALLOC_ARRAY(pack_order, ctx->entries_nr);
858
+ for (i = 0; i < ctx->entries_nr; i++)
859
+ pack_order[i] = i;
860
+ QSORT_S(pack_order, ctx->entries_nr, midx_pack_order_cmp, ctx);
861
+
862
+ return pack_order;
863
+}
864
+
865
+static void write_midx_reverse_index(char *midx_name, unsigned char *midx_hash,
866
+ struct write_midx_context *ctx)
867
+{
868
+ struct strbuf buf = STRBUF_INIT;
869
+ const char *tmp_file;
870
+
871
+ strbuf_addf(&buf, "%s-%s.rev", midx_name, hash_to_hex(midx_hash));
872
+
873
+ tmp_file = write_rev_file_order(NULL, ctx->pack_order, ctx->entries_nr,
874
+ midx_hash, WRITE_REV);
875
+
876
+ if (finalize_object_file(tmp_file, buf.buf))
877
+ die(_("cannot store reverse index file"));
878
+
879
+ strbuf_release(&buf);
880
+}
881
+
882
+static void clear_midx_files_ext(struct repository *r, const char *ext,
883
+ unsigned char *keep_hash);
884
+
885
static int write_midx_internal(const char *object_dir, struct multi_pack_index *m,
886
struct string_list *packs_to_drop,
887
const char *preferred_pack_name,
@@ -1001,6 +1067,14 @@ static int write_midx_internal(const char *object_dir, struct multi_pack_index *
1067
1068
finalize_hashfile(f, midx_hash, CSUM_FSYNC | CSUM_HASH_IN_STREAM);
1069
free_chunkfile(cf);
1070
+
1071
+ if (flags & MIDX_WRITE_REV_INDEX)
1072
+ ctx.pack_order = midx_pack_order(&ctx);
1073
+
1074
+ if (flags & MIDX_WRITE_REV_INDEX)
1075
+ write_midx_reverse_index(midx_name, midx_hash, &ctx);
1076
+ clear_midx_files_ext(the_repository, ".rev", midx_hash);
1077
+
1078
commit_lock_file(&lk);
1079
1080
cleanup:
@@ -1015,6 +1089,7 @@ cleanup:
1089
free(ctx.info);
1090
free(ctx.entries);
1091
free(ctx.pack_perm);
1092
+ free(ctx.pack_order);
1093
free(midx_name);
1094
return result;
1095
}
@@ -1027,6 +1102,44 @@ int write_midx_file(const char *object_dir,
1102
flags);
1103
}
1104
1105
+struct clear_midx_data {
1106
+ char *keep;
1107
+ const char *ext;
1108
+};
1109
+
1110
+static void clear_midx_file_ext(const char *full_path, size_t full_path_len,
1111
+ const char *file_name, void *_data)
1112
+{
1113
+ struct clear_midx_data *data = _data;
1114
+
1115
+ if (!(starts_with(file_name, "multi-pack-index-") &&
1116
+ ends_with(file_name, data->ext)))
1117
+ return;
1118
+ if (data->keep && !strcmp(data->keep, file_name))
1119
+ return;
1120
+
1121
+ if (unlink(full_path))
1122
+ die_errno(_("failed to remove %s"), full_path);
1123
+}
1124
+
1125
+static void clear_midx_files_ext(struct repository *r, const char *ext,
1126
+ unsigned char *keep_hash)
1127
+{
1128
+ struct clear_midx_data data;
1129
+ memset(&data, 0, sizeof(struct clear_midx_data));
1130
+
1131
+ if (keep_hash)
1132
+ data.keep = xstrfmt("multi-pack-index-%s%s",
1133
+ hash_to_hex(keep_hash), ext);
1134
+ data.ext = ext;
1135
+
1136
+ for_each_file_in_pack_dir(r->objects->odb->path,
1137
+ clear_midx_file_ext,
1138
+ &data);
1139
+
1140
+ free(data.keep);
1141
+}
1142
+
1143
void clear_midx_file(struct repository *r)
1144
{
1145
char *midx = get_midx_filename(r->objects->odb->path);
@@ -1039,6 +1152,8 @@ void clear_midx_file(struct repository *r)
1152
if (remove_path(midx))
1153
die(_("failed to clear multi-pack-index at %s"), midx);
1154
1155
+ clear_midx_files_ext(r, ".rev", NULL);
1156
+
1157
free(midx);
1158
}
1159
midx.h
+1
@@ -40,6 +40,7 @@ struct multi_pack_index {
40
};
41
42
#define MIDX_PROGRESS (1 << 0)
43
+#define MIDX_WRITE_REV_INDEX (1 << 1)
44
45
char *get_midx_rev_filename(struct multi_pack_index *m);
46