| 1 | #ifndef PACKFILE_H |
| 2 | #define PACKFILE_H |
| 3 | |
| 4 | #include "list.h" |
| 5 | #include "object.h" |
| 6 | #include "odb.h" |
| 7 | #include "odb/source-files.h" |
| 8 | #include "odb/source-packed.h" |
| 9 | #include "oidset.h" |
| 10 | #include "packfile-list.h" |
| 11 | #include "repository.h" |
| 12 | |
| 13 | /* in odb.h */ |
| 14 | struct object_info; |
| 15 | struct odb_read_stream; |
| 16 | |
| 17 | struct packed_git { |
| 18 | struct pack_window *windows; |
| 19 | off_t pack_size; |
| 20 | const void *index_data; |
| 21 | size_t index_size; |
| 22 | uint32_t num_objects; |
| 23 | size_t crc_offset; |
| 24 | struct oidset bad_objects; |
| 25 | int index_version; |
| 26 | time_t mtime; |
| 27 | int pack_fd; |
| 28 | int index; /* for builtin/pack-objects.c */ |
| 29 | unsigned pack_local:1, |
| 30 | pack_keep:1, |
| 31 | pack_keep_in_core:1, |
| 32 | pack_keep_in_core_open:1, |
| 33 | freshened:1, |
| 34 | do_not_close:1, |
| 35 | pack_promisor:1, |
| 36 | multi_pack_index:1, |
| 37 | is_cruft:1; |
| 38 | unsigned char hash[GIT_MAX_RAWSZ]; |
| 39 | struct revindex_entry *revindex; |
| 40 | const uint32_t *revindex_data; |
| 41 | const uint32_t *revindex_map; |
| 42 | size_t revindex_size; |
| 43 | /* |
| 44 | * mtimes_map points at the beginning of the memory mapped region of |
| 45 | * this pack's corresponding .mtimes file, and mtimes_size is the size |
| 46 | * of that .mtimes file |
| 47 | */ |
| 48 | const uint32_t *mtimes_map; |
| 49 | size_t mtimes_size; |
| 50 | |
| 51 | /* repo denotes the repository this packfile belongs to */ |
| 52 | struct repository *repo; |
| 53 | |
| 54 | /* something like ".git/objects/pack/xxxxx.pack" */ |
| 55 | char pack_name[FLEX_ARRAY]; /* more */ |
| 56 | }; |
| 57 | |
| 58 | /* |
| 59 | * Add the pack to the store so that contained objects become accessible via |
| 60 | * the store. This moves ownership into the store. |
| 61 | */ |
| 62 | void packfile_store_add_pack(struct odb_source_packed *store, |
| 63 | struct packed_git *pack); |
| 64 | |
| 65 | /* |
| 66 | * Get all packs managed by the given store, including packfiles that are |
| 67 | * referenced by multi-pack indices. |
| 68 | */ |
| 69 | struct packfile_list_entry *packfile_store_get_packs(struct odb_source_packed *store); |
| 70 | |
| 71 | struct repo_for_each_pack_data { |
| 72 | struct odb_source *source; |
| 73 | struct packfile_list_entry *entry; |
| 74 | }; |
| 75 | |
| 76 | static inline struct repo_for_each_pack_data repo_for_eack_pack_data_init(struct repository *repo) |
| 77 | { |
| 78 | struct repo_for_each_pack_data data = { 0 }; |
| 79 | |
| 80 | odb_prepare_alternates(repo->objects); |
| 81 | |
| 82 | for (struct odb_source *source = repo->objects->sources; source; source = source->next) { |
| 83 | struct odb_source_files *files = odb_source_files_downcast(source); |
| 84 | struct packfile_list_entry *entry = packfile_store_get_packs(files->packed); |
| 85 | if (!entry) |
| 86 | continue; |
| 87 | data.source = source; |
| 88 | data.entry = entry; |
| 89 | break; |
| 90 | } |
| 91 | |
| 92 | return data; |
| 93 | } |
| 94 | |
| 95 | static inline void repo_for_each_pack_data_next(struct repo_for_each_pack_data *data) |
| 96 | { |
| 97 | struct odb_source *source; |
| 98 | |
| 99 | data->entry = data->entry->next; |
| 100 | if (data->entry) |
| 101 | return; |
| 102 | |
| 103 | for (source = data->source->next; source; source = source->next) { |
| 104 | struct odb_source_files *files = odb_source_files_downcast(source); |
| 105 | struct packfile_list_entry *entry = packfile_store_get_packs(files->packed); |
| 106 | if (!entry) |
| 107 | continue; |
| 108 | data->source = source; |
| 109 | data->entry = entry; |
| 110 | return; |
| 111 | } |
| 112 | |
| 113 | data->source = NULL; |
| 114 | data->entry = NULL; |
| 115 | } |
| 116 | |
| 117 | /* |
| 118 | * Load and iterate through all packs of the given repository. This helper |
| 119 | * function will yield packfiles from all object sources connected to the |
| 120 | * repository. |
| 121 | */ |
| 122 | #define repo_for_each_pack(repo, p) \ |
| 123 | for (struct repo_for_each_pack_data eack_pack_data = repo_for_eack_pack_data_init(repo); \ |
| 124 | ((p) = (eack_pack_data.entry ? eack_pack_data.entry->pack : NULL)); \ |
| 125 | repo_for_each_pack_data_next(&eack_pack_data)) |
| 126 | |
| 127 | /* |
| 128 | * Open the packfile and add it to the store if it isn't yet known. Returns |
| 129 | * either the newly opened packfile or the preexisting packfile. Returns a |
| 130 | * `NULL` pointer in case the packfile could not be opened. |
| 131 | */ |
| 132 | struct packed_git *packfile_store_load_pack(struct odb_source_packed *store, |
| 133 | const char *idx_path, int local); |
| 134 | |
| 135 | enum kept_pack_type { |
| 136 | KEPT_PACK_ON_DISK = (1 << 0), |
| 137 | KEPT_PACK_IN_CORE = (1 << 1), |
| 138 | KEPT_PACK_IN_CORE_OPEN = (1 << 2), |
| 139 | }; |
| 140 | |
| 141 | /* |
| 142 | * Retrieve the cache of kept packs from the given packfile store. Accepts a |
| 143 | * combination of `kept_pack_type` flags. The cache is computed on demand and |
| 144 | * will be recomputed whenever the flags change. |
| 145 | */ |
| 146 | struct packed_git **packfile_store_get_kept_pack_cache(struct odb_source_packed *store, |
| 147 | unsigned flags); |
| 148 | |
| 149 | struct pack_window { |
| 150 | struct pack_window *next; |
| 151 | unsigned char *base; |
| 152 | off_t offset; |
| 153 | size_t len; |
| 154 | unsigned int last_used; |
| 155 | unsigned int inuse_cnt; |
| 156 | }; |
| 157 | |
| 158 | struct pack_entry { |
| 159 | off_t offset; |
| 160 | struct packed_git *p; |
| 161 | }; |
| 162 | |
| 163 | /* |
| 164 | * Generate the filename to be used for a pack file with checksum "sha1" and |
| 165 | * extension "ext". The result is written into the strbuf "buf", overwriting |
| 166 | * any existing contents. A pointer to buf->buf is returned as a convenience. |
| 167 | * |
| 168 | * Example: odb_pack_name(out, sha1, "idx") => ".git/objects/pack/pack-1234..idx" |
| 169 | */ |
| 170 | char *odb_pack_name(struct repository *r, struct strbuf *buf, |
| 171 | const unsigned char *hash, const char *ext); |
| 172 | |
| 173 | /* |
| 174 | * Return the basename of the packfile, omitting any containing directory |
| 175 | * (e.g., "pack-1234abcd[...].pack"). |
| 176 | */ |
| 177 | const char *pack_basename(struct packed_git *p); |
| 178 | |
| 179 | /* |
| 180 | * Parse the pack idx file found at idx_path and create a packed_git struct |
| 181 | * which can be used with find_pack_entry_one(). |
| 182 | * |
| 183 | * You probably don't want to use this function! It skips most of the normal |
| 184 | * sanity checks (including whether we even have the matching .pack file), |
| 185 | * and does not add the resulting packed_git struct to the internal list of |
| 186 | * packs. You probably want add_packed_git() instead. |
| 187 | */ |
| 188 | struct packed_git *parse_pack_index(struct repository *r, unsigned char *sha1, |
| 189 | const char *idx_path); |
| 190 | |
| 191 | typedef void each_file_in_pack_dir_fn(const char *full_path, size_t full_path_len, |
| 192 | const char *file_name, void *data); |
| 193 | void for_each_file_in_pack_subdir(const char *objdir, |
| 194 | const char *subdir, |
| 195 | each_file_in_pack_dir_fn fn, |
| 196 | void *data); |
| 197 | void for_each_file_in_pack_dir(const char *objdir, |
| 198 | each_file_in_pack_dir_fn fn, |
| 199 | void *data); |
| 200 | |
| 201 | /* |
| 202 | * Iterate over all accessible packed objects without respect to reachability. |
| 203 | * By default, this includes both local and alternate packs. |
| 204 | * |
| 205 | * Note that some objects may appear twice if they are found in multiple packs. |
| 206 | * Each pack is visited in an unspecified order. By default, objects within a |
| 207 | * pack are visited in pack-idx order (i.e., sorted by oid). |
| 208 | */ |
| 209 | typedef int each_packed_object_fn(const struct object_id *oid, |
| 210 | struct packed_git *pack, |
| 211 | uint32_t pos, |
| 212 | void *data); |
| 213 | int for_each_object_in_pack(struct packed_git *p, |
| 214 | each_packed_object_fn, void *data, |
| 215 | enum odb_for_each_object_flags flags); |
| 216 | |
| 217 | /* A hook to report invalid files in pack directory */ |
| 218 | #define PACKDIR_FILE_PACK 1 |
| 219 | #define PACKDIR_FILE_IDX 2 |
| 220 | #define PACKDIR_FILE_GARBAGE 4 |
| 221 | extern void (*report_garbage)(unsigned seen_bits, const char *path); |
| 222 | |
| 223 | void pack_report(struct repository *repo); |
| 224 | |
| 225 | /* |
| 226 | * mmap the index file for the specified packfile (if it is not |
| 227 | * already mmapped). Return 0 on success. |
| 228 | */ |
| 229 | int open_pack_index(struct packed_git *); |
| 230 | |
| 231 | /* |
| 232 | * munmap the index file for the specified packfile (if it is |
| 233 | * currently mmapped). |
| 234 | */ |
| 235 | void close_pack_index(struct packed_git *); |
| 236 | |
| 237 | int close_pack_fd(struct packed_git *p); |
| 238 | |
| 239 | uint32_t get_pack_fanout(struct packed_git *p, uint32_t value); |
| 240 | |
| 241 | struct object_database; |
| 242 | |
| 243 | unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned long *); |
| 244 | void close_pack_windows(struct packed_git *); |
| 245 | void close_pack(struct packed_git *); |
| 246 | void unuse_pack(struct pack_window **); |
| 247 | void clear_delta_base_cache(void); |
| 248 | struct packed_git *add_packed_git(struct repository *r, const char *path, |
| 249 | size_t path_len, int local); |
| 250 | |
| 251 | /* |
| 252 | * Unlink the .pack and associated extension files. |
| 253 | * Does not unlink if 'force_delete' is false and the pack-file is |
| 254 | * marked as ".keep". |
| 255 | */ |
| 256 | void unlink_pack_path(const char *pack_name, int force_delete); |
| 257 | |
| 258 | /* |
| 259 | * Make sure that a pointer access into an mmap'd index file is within bounds, |
| 260 | * and can provide at least 8 bytes of data. |
| 261 | * |
| 262 | * Note that this is only necessary for variable-length segments of the file |
| 263 | * (like the 64-bit extended offset table), as we compare the size to the |
| 264 | * fixed-length parts when we open the file. |
| 265 | */ |
| 266 | void check_pack_index_ptr(const struct packed_git *p, const void *ptr); |
| 267 | |
| 268 | /* |
| 269 | * Perform binary search on a pack-index for a given oid. Packfile is expected to |
| 270 | * have a valid pack-index. |
| 271 | * |
| 272 | * See 'bsearch_hash' for more information. |
| 273 | */ |
| 274 | int bsearch_pack(const struct object_id *oid, const struct packed_git *p, uint32_t *result); |
| 275 | |
| 276 | /* |
| 277 | * Write the oid of the nth object within the specified packfile into the first |
| 278 | * parameter. Open the index if it is not already open. Returns 0 on success, |
| 279 | * negative otherwise. |
| 280 | */ |
| 281 | int nth_packed_object_id(struct object_id *, struct packed_git *, uint32_t n); |
| 282 | |
| 283 | /* |
| 284 | * Return the offset of the nth object within the specified packfile. |
| 285 | * The index must already be opened. |
| 286 | */ |
| 287 | off_t nth_packed_object_offset(const struct packed_git *, uint32_t n); |
| 288 | |
| 289 | /* |
| 290 | * If the object named by oid is present in the specified packfile, |
| 291 | * return its offset within the packfile; otherwise, return 0. |
| 292 | */ |
| 293 | off_t find_pack_entry_one(const struct object_id *oid, struct packed_git *); |
| 294 | |
| 295 | int packfile_fill_entry(struct packed_git *p, |
| 296 | const struct object_id *oid, |
| 297 | struct pack_entry *e); |
| 298 | |
| 299 | int is_pack_valid(struct packed_git *); |
| 300 | void *unpack_entry(struct repository *r, struct packed_git *, off_t, |
| 301 | enum object_type *, size_t *); |
| 302 | unsigned long unpack_object_header_buffer(const unsigned char *buf, unsigned long len, enum object_type *type, size_t *sizep); |
| 303 | size_t get_size_from_delta(struct packed_git *, struct pack_window **, off_t); |
| 304 | int unpack_object_header(struct packed_git *, struct pack_window **, off_t *, size_t *); |
| 305 | off_t get_delta_base(struct packed_git *p, struct pack_window **w_curs, |
| 306 | off_t *curpos, enum object_type type, |
| 307 | off_t delta_obj_offset); |
| 308 | |
| 309 | int packfile_read_object_stream(struct odb_read_stream **out, |
| 310 | const struct object_id *oid, |
| 311 | struct packed_git *pack, |
| 312 | off_t offset); |
| 313 | |
| 314 | void release_pack_memory(size_t); |
| 315 | |
| 316 | /* global flag to enable extra checks when accessing packed objects */ |
| 317 | extern int do_check_packed_object_crc; |
| 318 | |
| 319 | /* |
| 320 | * Look up the object info for a specific offset in the packfile. |
| 321 | * Returns zero on success, a negative error code otherwise. |
| 322 | */ |
| 323 | int packed_object_info(struct odb_source_packed *source, |
| 324 | struct packed_git *pack, |
| 325 | off_t offset, struct object_info *); |
| 326 | int packed_object_info_with_index_pos(struct odb_source_packed *source, |
| 327 | struct packed_git *p, off_t obj_offset, |
| 328 | uint32_t *maybe_index_pos, struct object_info *oi); |
| 329 | |
| 330 | void mark_bad_packed_object(struct packed_git *, const struct object_id *); |
| 331 | const struct packed_git *has_packed_and_bad(struct repository *, const struct object_id *); |
| 332 | |
| 333 | int has_object_pack(struct repository *r, const struct object_id *oid); |
| 334 | int has_object_kept_pack(struct repository *r, const struct object_id *oid, |
| 335 | unsigned flags); |
| 336 | |
| 337 | /* |
| 338 | * Return 1 if an object in a promisor packfile is or refers to the given |
| 339 | * object, 0 otherwise. |
| 340 | */ |
| 341 | int is_promisor_object(struct repository *r, const struct object_id *oid); |
| 342 | |
| 343 | /* |
| 344 | * Expose a function for fuzz testing. |
| 345 | * |
| 346 | * load_idx() parses a block of memory as a packfile index and puts the results |
| 347 | * into a struct packed_git. |
| 348 | * |
| 349 | * This function should not be used directly. It is exposed here only so that we |
| 350 | * have a convenient entry-point for fuzz testing. For real uses, you should |
| 351 | * probably use open_pack_index() instead. |
| 352 | */ |
| 353 | int load_idx(const char *path, const unsigned int hashsz, void *idx_map, |
| 354 | size_t idx_size, struct packed_git *p); |
| 355 | |
| 356 | /* |
| 357 | * Parse a --pack_header option as accepted by index-pack and unpack-objects, |
| 358 | * turning it into the matching bytes we'd find in a pack. |
| 359 | */ |
| 360 | int parse_pack_header_option(const char *in, unsigned char *out, unsigned int *len); |
| 361 | |
| 362 | #endif |