@leroysheep / rag-system
Code Commits Pulls CI/CD 82,037
test
bisect jch maint master next seen test todo
h 155 lines 5.65 KB
Raw
1 #ifndef MIDX_H
2 #define MIDX_H
3
4 #include "string-list.h"
5
6 struct object_id;
7 struct pack_entry;
8 struct repository;
9 struct bitmapped_pack;
10 struct git_hash_algo;
11 struct odb_source;
12 struct strvec;
13
14 #define MIDX_SIGNATURE 0x4d494458 /* "MIDX" */
15 #define MIDX_VERSION_V1 1
16 #define MIDX_VERSION_V2 2
17 #define MIDX_BYTE_FILE_VERSION 4
18 #define MIDX_BYTE_HASH_VERSION 5
19 #define MIDX_BYTE_NUM_CHUNKS 6
20 #define MIDX_BYTE_NUM_PACKS 8
21 #define MIDX_HEADER_SIZE 12
22
23 #define MIDX_CHUNK_ALIGNMENT 4
24 #define MIDX_CHUNKID_PACKNAMES 0x504e414d /* "PNAM" */
25 #define MIDX_CHUNKID_BITMAPPEDPACKS 0x42544d50 /* "BTMP" */
26 #define MIDX_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */
27 #define MIDX_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */
28 #define MIDX_CHUNKID_OBJECTOFFSETS 0x4f4f4646 /* "OOFF" */
29 #define MIDX_CHUNKID_LARGEOFFSETS 0x4c4f4646 /* "LOFF" */
30 #define MIDX_CHUNKID_REVINDEX 0x52494458 /* "RIDX" */
31 #define MIDX_CHUNKID_BASE 0x42415345 /* "BASE" */
32 #define MIDX_CHUNK_OFFSET_WIDTH (2 * sizeof(uint32_t))
33 #define MIDX_LARGE_OFFSET_NEEDED 0x80000000
34
35 #define GIT_TEST_MULTI_PACK_INDEX "GIT_TEST_MULTI_PACK_INDEX"
36 #define GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL \
37 "GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL"
38
39 struct multi_pack_index {
40 struct odb_source_packed *source;
41
42 const unsigned char *data;
43 size_t data_len;
44
45 const uint32_t *revindex_data;
46 const uint32_t *revindex_map;
47 size_t revindex_len;
48
49 uint32_t signature;
50 unsigned char version;
51 unsigned char hash_len;
52 unsigned char num_chunks;
53 uint32_t num_packs;
54 uint32_t num_objects;
55 int preferred_pack_idx;
56
57 int has_chain;
58
59 const unsigned char *chunk_pack_names;
60 size_t chunk_pack_names_len;
61 const uint32_t *chunk_bitmapped_packs;
62 size_t chunk_bitmapped_packs_len;
63 const uint32_t *chunk_oid_fanout;
64 const unsigned char *chunk_oid_lookup;
65 const unsigned char *chunk_object_offsets;
66 const unsigned char *chunk_large_offsets;
67 size_t chunk_large_offsets_len;
68 const unsigned char *chunk_revindex;
69 size_t chunk_revindex_len;
70
71 struct multi_pack_index *base_midx;
72 uint32_t num_objects_in_base;
73 uint32_t num_packs_in_base;
74
75 const char **pack_names;
76 size_t *pack_names_sorted;
77 struct packed_git **packs;
78 };
79
80 #define MIDX_PROGRESS (1 << 0)
81 #define MIDX_WRITE_REV_INDEX (1 << 1)
82 #define MIDX_WRITE_BITMAP (1 << 2)
83 #define MIDX_WRITE_BITMAP_HASH_CACHE (1 << 3)
84 #define MIDX_WRITE_BITMAP_LOOKUP_TABLE (1 << 4)
85 #define MIDX_WRITE_INCREMENTAL (1 << 5)
86 #define MIDX_WRITE_COMPACT (1 << 6)
87 #define MIDX_WRITE_NO_CHAIN (1 << 7)
88
89 #define MIDX_EXT_REV "rev"
90 #define MIDX_EXT_BITMAP "bitmap"
91 #define MIDX_EXT_MIDX "midx"
92
93 const char *midx_get_checksum_hex(const struct multi_pack_index *m) /* static buffer */;
94 const unsigned char *midx_get_checksum_hash(const struct multi_pack_index *m);
95 void get_midx_filename(struct odb_source_packed *source, struct strbuf *out);
96 void get_midx_filename_ext(struct odb_source_packed *source, struct strbuf *out,
97 const unsigned char *hash, const char *ext);
98 void get_midx_chain_dirname(struct odb_source_packed *source, struct strbuf *out);
99 void get_midx_chain_filename(struct odb_source_packed *source, struct strbuf *out);
100 void get_split_midx_filename_ext(struct odb_source_packed *source, struct strbuf *buf,
101 const unsigned char *hash, const char *ext);
102
103 struct multi_pack_index *get_multi_pack_index(struct odb_source_packed *source);
104 struct multi_pack_index *load_multi_pack_index(struct odb_source_packed *source);
105 int prepare_midx_pack(struct multi_pack_index *m, uint32_t pack_int_id);
106 struct packed_git *nth_midxed_pack(struct multi_pack_index *m,
107 uint32_t pack_int_id);
108 int nth_bitmapped_pack(struct multi_pack_index *m,
109 struct bitmapped_pack *bp, uint32_t pack_int_id);
110 int bsearch_one_midx(const struct object_id *oid, struct multi_pack_index *m,
111 uint32_t *result);
112 int bsearch_midx(const struct object_id *oid, struct multi_pack_index *m,
113 uint32_t *result);
114 int midx_has_oid(struct multi_pack_index *m, const struct object_id *oid);
115 off_t nth_midxed_offset(struct multi_pack_index *m, uint32_t pos);
116 uint32_t nth_midxed_pack_int_id(struct multi_pack_index *m, uint32_t pos);
117 struct object_id *nth_midxed_object_oid(struct object_id *oid,
118 struct multi_pack_index *m,
119 uint32_t n);
120 int fill_midx_entry(struct multi_pack_index *m, const struct object_id *oid, struct pack_entry *e);
121 int midx_contains_pack(struct multi_pack_index *m,
122 const char *idx_or_pack_name);
123 int midx_layer_contains_pack(struct multi_pack_index *m,
124 const char *idx_or_pack_name);
125 int midx_preferred_pack(struct multi_pack_index *m, uint32_t *pack_int_id);
126 int prepare_multi_pack_index_one(struct odb_source_packed *source);
127
128 /*
129 * Variant of write_midx_file which writes a MIDX containing only the packs
130 * specified in packs_to_include.
131 */
132 int write_midx_file(struct odb_source_packed *source,
133 const char *preferred_pack_name, const char *refs_snapshot,
134 const char *incremental_base, unsigned flags);
135 int write_midx_file_only(struct odb_source_packed *source,
136 struct string_list *packs_to_include,
137 const char *preferred_pack_name,
138 const char *refs_snapshot,
139 const char *incremental_base,
140 unsigned flags);
141 int write_midx_file_compact(struct odb_source_packed *source,
142 struct multi_pack_index *from,
143 struct multi_pack_index *to,
144 const char *incremental_base,
145 unsigned flags);
146 void clear_midx_file(struct repository *r);
147 void clear_incremental_midx_files(struct repository *r,
148 const struct strvec *keep_hashes);
149 int verify_midx_file(struct odb_source_packed *source, unsigned flags);
150 int expire_midx_packs(struct odb_source_packed *source, unsigned flags);
151 int midx_repack(struct odb_source_packed *source, size_t batch_size, unsigned flags);
152
153 void close_midx(struct multi_pack_index *m);
154
155 #endif