| 1 | #ifndef PACKFILE_H |
| 2 | #define PACKFILE_H |
| 3 | |
| 4 | #include "list.h" |
| 5 | #include "object.h" |
| 6 | #include "odb.h" |
| 7 | #include "odb/source-files.h" |
| 8 | #include "oidset.h" |
| 9 | #include "repository.h" |
| 10 | #include "strmap.h" |
| 11 | |
| 12 | /* in odb.h */ |
| 13 | struct object_info; |
| 14 | struct odb_read_stream; |
| 15 | |
| 16 | struct packed_git { |
| 17 | struct pack_window *windows; |
| 18 | off_t pack_size; |
| 19 | const void *index_data; |
| 20 | size_t index_size; |
| 21 | uint32_t num_objects; |
| 22 | size_t crc_offset; |
| 23 | struct oidset bad_objects; |
| 24 | int index_version; |
| 25 | time_t mtime; |
| 26 | int pack_fd; |
| 27 | int index; /* for builtin/pack-objects.c */ |
| 28 | unsigned pack_local:1, |
| 29 | pack_keep:1, |
| 30 | pack_keep_in_core:1, |
| 31 | pack_keep_in_core_open:1, |
| 32 | freshened:1, |
| 33 | do_not_close:1, |
| 34 | pack_promisor:1, |
| 35 | multi_pack_index:1, |
| 36 | is_cruft:1; |
| 37 | unsigned char hash[GIT_MAX_RAWSZ]; |
| 38 | struct revindex_entry *revindex; |
| 39 | const uint32_t *revindex_data; |
| 40 | const uint32_t *revindex_map; |
| 41 | size_t revindex_size; |
| 42 | /* |
| 43 | * mtimes_map points at the beginning of the memory mapped region of |
| 44 | * this pack's corresponding .mtimes file, and mtimes_size is the size |
| 45 | * of that .mtimes file |
| 46 | */ |
| 47 | const uint32_t *mtimes_map; |
| 48 | size_t mtimes_size; |
| 49 | |
| 50 | /* repo denotes the repository this packfile belongs to */ |
| 51 | struct repository *repo; |
| 52 | |
| 53 | /* something like ".git/objects/pack/xxxxx.pack" */ |
| 54 | char pack_name[FLEX_ARRAY]; /* more */ |
| 55 | }; |
| 56 | |
| 57 | struct packfile_list { |
| 58 | struct packfile_list_entry *head, *tail; |
| 59 | }; |
| 60 | |
| 61 | struct packfile_list_entry { |
| 62 | struct packfile_list_entry *next; |
| 63 | struct packed_git *pack; |
| 64 | }; |
| 65 | |
| 66 | void packfile_list_clear(struct packfile_list *list); |
| 67 | void packfile_list_remove(struct packfile_list *list, struct packed_git *pack); |
| 68 | void packfile_list_prepend(struct packfile_list *list, struct packed_git *pack); |
| 69 | void packfile_list_append(struct packfile_list *list, struct packed_git *pack); |
| 70 | |
| 71 | /* |
| 72 | * Find the pack within the "packs" list whose index contains the object |
| 73 | * "oid". For general object lookups, you probably don't want this; use |
| 74 | * find_pack_entry() instead. |
| 75 | */ |
| 76 | struct packed_git *packfile_list_find_oid(struct packfile_list_entry *packs, |
| 77 | const struct object_id *oid); |
| 78 | |
| 79 | /* |
| 80 | * A store that manages packfiles for a given object database. |
| 81 | */ |
| 82 | struct packfile_store { |
| 83 | struct odb_source *source; |
| 84 | |
| 85 | /* |
| 86 | * The list of packfiles in the order in which they have been most |
| 87 | * recently used. |
| 88 | */ |
| 89 | struct packfile_list packs; |
| 90 | |
| 91 | /* |
| 92 | * Cache of packfiles which are marked as "kept", either because there |
| 93 | * is an on-disk ".keep" file or because they are marked as "kept" in |
| 94 | * memory. |
| 95 | * |
| 96 | * Should not be accessed directly, but via |
| 97 | * `packfile_store_get_kept_pack_cache()`. The list of packs gets |
| 98 | * invalidated when the stored flags and the flags passed to |
| 99 | * `packfile_store_get_kept_pack_cache()` mismatch. |
| 100 | */ |
| 101 | struct { |
| 102 | struct packed_git **packs; |
| 103 | unsigned flags; |
| 104 | } kept_cache; |
| 105 | |
| 106 | /* The multi-pack index that belongs to this specific packfile store. */ |
| 107 | struct multi_pack_index *midx; |
| 108 | |
| 109 | /* |
| 110 | * A map of packfile names to packed_git structs for tracking which |
| 111 | * packs have been loaded already. |
| 112 | */ |
| 113 | struct strmap packs_by_path; |
| 114 | |
| 115 | /* |
| 116 | * Whether packfiles have already been populated with this store's |
| 117 | * packs. |
| 118 | */ |
| 119 | bool initialized; |
| 120 | |
| 121 | /* |
| 122 | * Usually, packfiles will be reordered to the front of the `packs` |
| 123 | * list whenever an object is looked up via them. This has the effect |
| 124 | * that packs that contain a lot of accessed objects will be located |
| 125 | * towards the front. |
| 126 | * |
| 127 | * This is usually desirable, but there are exceptions. One exception |
| 128 | * is when the looking up multiple objects in a loop for each packfile. |
| 129 | * In that case, we may easily end up with an infinite loop as the |
| 130 | * packfiles get reordered to the front repeatedly. |
| 131 | * |
| 132 | * Setting this field to `true` thus disables these reorderings. |
| 133 | */ |
| 134 | bool skip_mru_updates; |
| 135 | }; |
| 136 | |
| 137 | /* |
| 138 | * Allocate and initialize a new empty packfile store for the given object |
| 139 | * database source. |
| 140 | */ |
| 141 | struct packfile_store *packfile_store_new(struct odb_source *source); |
| 142 | |
| 143 | /* |
| 144 | * Free the packfile store and all its associated state. All packfiles |
| 145 | * tracked by the store will be closed. |
| 146 | */ |
| 147 | void packfile_store_free(struct packfile_store *store); |
| 148 | |
| 149 | /* |
| 150 | * Close all packfiles associated with this store. The packfiles won't be |
| 151 | * free'd, so they can be re-opened at a later point in time. |
| 152 | */ |
| 153 | void packfile_store_close(struct packfile_store *store); |
| 154 | |
| 155 | /* |
| 156 | * Prepare the packfile store by loading packfiles and multi-pack indices for |
| 157 | * all alternates. This becomes a no-op if the store is already prepared. |
| 158 | * |
| 159 | * It shouldn't typically be necessary to call this function directly, as |
| 160 | * functions that access the store know to prepare it. |
| 161 | */ |
| 162 | void packfile_store_prepare(struct packfile_store *store); |
| 163 | |
| 164 | /* |
| 165 | * Clear the packfile caches and try to look up any new packfiles that have |
| 166 | * appeared since last preparing the packfiles store. |
| 167 | * |
| 168 | * This function must be called under the `odb_read_lock()`. |
| 169 | */ |
| 170 | void packfile_store_reprepare(struct packfile_store *store); |
| 171 | |
| 172 | /* |
| 173 | * Add the pack to the store so that contained objects become accessible via |
| 174 | * the store. This moves ownership into the store. |
| 175 | */ |
| 176 | void packfile_store_add_pack(struct packfile_store *store, |
| 177 | struct packed_git *pack); |
| 178 | |
| 179 | /* |
| 180 | * Get all packs managed by the given store, including packfiles that are |
| 181 | * referenced by multi-pack indices. |
| 182 | */ |
| 183 | struct packfile_list_entry *packfile_store_get_packs(struct packfile_store *store); |
| 184 | |
| 185 | struct repo_for_each_pack_data { |
| 186 | struct odb_source *source; |
| 187 | struct packfile_list_entry *entry; |
| 188 | }; |
| 189 | |
| 190 | static inline struct repo_for_each_pack_data repo_for_eack_pack_data_init(struct repository *repo) |
| 191 | { |
| 192 | struct repo_for_each_pack_data data = { 0 }; |
| 193 | |
| 194 | odb_prepare_alternates(repo->objects); |
| 195 | |
| 196 | for (struct odb_source *source = repo->objects->sources; source; source = source->next) { |
| 197 | struct odb_source_files *files = odb_source_files_downcast(source); |
| 198 | struct packfile_list_entry *entry = packfile_store_get_packs(files->packed); |
| 199 | if (!entry) |
| 200 | continue; |
| 201 | data.source = source; |
| 202 | data.entry = entry; |
| 203 | break; |
| 204 | } |
| 205 | |
| 206 | return data; |
| 207 | } |
| 208 | |
| 209 | static inline void repo_for_each_pack_data_next(struct repo_for_each_pack_data *data) |
| 210 | { |
| 211 | struct odb_source *source; |
| 212 | |
| 213 | data->entry = data->entry->next; |
| 214 | if (data->entry) |
| 215 | return; |
| 216 | |
| 217 | for (source = data->source->next; source; source = source->next) { |
| 218 | struct odb_source_files *files = odb_source_files_downcast(source); |
| 219 | struct packfile_list_entry *entry = packfile_store_get_packs(files->packed); |
| 220 | if (!entry) |
| 221 | continue; |
| 222 | data->source = source; |
| 223 | data->entry = entry; |
| 224 | return; |
| 225 | } |
| 226 | |
| 227 | data->source = NULL; |
| 228 | data->entry = NULL; |
| 229 | } |
| 230 | |
| 231 | /* |
| 232 | * Load and iterate through all packs of the given repository. This helper |
| 233 | * function will yield packfiles from all object sources connected to the |
| 234 | * repository. |
| 235 | */ |
| 236 | #define repo_for_each_pack(repo, p) \ |
| 237 | for (struct repo_for_each_pack_data eack_pack_data = repo_for_eack_pack_data_init(repo); \ |
| 238 | ((p) = (eack_pack_data.entry ? eack_pack_data.entry->pack : NULL)); \ |
| 239 | repo_for_each_pack_data_next(&eack_pack_data)) |
| 240 | |
| 241 | int packfile_store_read_object_stream(struct odb_read_stream **out, |
| 242 | struct packfile_store *store, |
| 243 | const struct object_id *oid); |
| 244 | |
| 245 | /* |
| 246 | * Try to read the object identified by its ID from the object store and |
| 247 | * populate the object info with its data. Returns 1 in case the object was |
| 248 | * not found, 0 if it was and read successfully, and a negative error code in |
| 249 | * case the object was corrupted. |
| 250 | */ |
| 251 | int packfile_store_read_object_info(struct packfile_store *store, |
| 252 | const struct object_id *oid, |
| 253 | struct object_info *oi, |
| 254 | enum object_info_flags flags); |
| 255 | |
| 256 | /* |
| 257 | * Open the packfile and add it to the store if it isn't yet known. Returns |
| 258 | * either the newly opened packfile or the preexisting packfile. Returns a |
| 259 | * `NULL` pointer in case the packfile could not be opened. |
| 260 | */ |
| 261 | struct packed_git *packfile_store_load_pack(struct packfile_store *store, |
| 262 | const char *idx_path, int local); |
| 263 | |
| 264 | int packfile_store_freshen_object(struct packfile_store *store, |
| 265 | const struct object_id *oid); |
| 266 | |
| 267 | enum kept_pack_type { |
| 268 | KEPT_PACK_ON_DISK = (1 << 0), |
| 269 | KEPT_PACK_IN_CORE = (1 << 1), |
| 270 | KEPT_PACK_IN_CORE_OPEN = (1 << 2), |
| 271 | }; |
| 272 | |
| 273 | /* |
| 274 | * Count the number objects contained in the given packfile store. If |
| 275 | * successful, the number of objects will be written to the `out` pointer. |
| 276 | * |
| 277 | * Return 0 on success, a negative error code otherwise. |
| 278 | */ |
| 279 | int packfile_store_count_objects(struct packfile_store *store, |
| 280 | enum odb_count_objects_flags flags, |
| 281 | unsigned long *out); |
| 282 | |
| 283 | /* |
| 284 | * Retrieve the cache of kept packs from the given packfile store. Accepts a |
| 285 | * combination of `kept_pack_type` flags. The cache is computed on demand and |
| 286 | * will be recomputed whenever the flags change. |
| 287 | */ |
| 288 | struct packed_git **packfile_store_get_kept_pack_cache(struct packfile_store *store, |
| 289 | unsigned flags); |
| 290 | |
| 291 | struct pack_window { |
| 292 | struct pack_window *next; |
| 293 | unsigned char *base; |
| 294 | off_t offset; |
| 295 | size_t len; |
| 296 | unsigned int last_used; |
| 297 | unsigned int inuse_cnt; |
| 298 | }; |
| 299 | |
| 300 | struct pack_entry { |
| 301 | off_t offset; |
| 302 | struct packed_git *p; |
| 303 | }; |
| 304 | |
| 305 | /* |
| 306 | * Generate the filename to be used for a pack file with checksum "sha1" and |
| 307 | * extension "ext". The result is written into the strbuf "buf", overwriting |
| 308 | * any existing contents. A pointer to buf->buf is returned as a convenience. |
| 309 | * |
| 310 | * Example: odb_pack_name(out, sha1, "idx") => ".git/objects/pack/pack-1234..idx" |
| 311 | */ |
| 312 | char *odb_pack_name(struct repository *r, struct strbuf *buf, |
| 313 | const unsigned char *hash, const char *ext); |
| 314 | |
| 315 | /* |
| 316 | * Return the basename of the packfile, omitting any containing directory |
| 317 | * (e.g., "pack-1234abcd[...].pack"). |
| 318 | */ |
| 319 | const char *pack_basename(struct packed_git *p); |
| 320 | |
| 321 | /* |
| 322 | * Parse the pack idx file found at idx_path and create a packed_git struct |
| 323 | * which can be used with find_pack_entry_one(). |
| 324 | * |
| 325 | * You probably don't want to use this function! It skips most of the normal |
| 326 | * sanity checks (including whether we even have the matching .pack file), |
| 327 | * and does not add the resulting packed_git struct to the internal list of |
| 328 | * packs. You probably want add_packed_git() instead. |
| 329 | */ |
| 330 | struct packed_git *parse_pack_index(struct repository *r, unsigned char *sha1, |
| 331 | const char *idx_path); |
| 332 | |
| 333 | typedef void each_file_in_pack_dir_fn(const char *full_path, size_t full_path_len, |
| 334 | const char *file_name, void *data); |
| 335 | void for_each_file_in_pack_subdir(const char *objdir, |
| 336 | const char *subdir, |
| 337 | each_file_in_pack_dir_fn fn, |
| 338 | void *data); |
| 339 | void for_each_file_in_pack_dir(const char *objdir, |
| 340 | each_file_in_pack_dir_fn fn, |
| 341 | void *data); |
| 342 | |
| 343 | /* |
| 344 | * Iterate over all accessible packed objects without respect to reachability. |
| 345 | * By default, this includes both local and alternate packs. |
| 346 | * |
| 347 | * Note that some objects may appear twice if they are found in multiple packs. |
| 348 | * Each pack is visited in an unspecified order. By default, objects within a |
| 349 | * pack are visited in pack-idx order (i.e., sorted by oid). |
| 350 | */ |
| 351 | typedef int each_packed_object_fn(const struct object_id *oid, |
| 352 | struct packed_git *pack, |
| 353 | uint32_t pos, |
| 354 | void *data); |
| 355 | int for_each_object_in_pack(struct packed_git *p, |
| 356 | each_packed_object_fn, void *data, |
| 357 | enum odb_for_each_object_flags flags); |
| 358 | |
| 359 | /* |
| 360 | * Iterate through all packed objects in the given packfile store and invoke |
| 361 | * the callback function for each of them. If an object info request is given, |
| 362 | * then the object info will be read for every individual object and passed to |
| 363 | * the callback as if `packfile_store_read_object_info()` was called for the |
| 364 | * object. |
| 365 | * |
| 366 | * The flags parameter is a combination of `odb_for_each_object_flags`. |
| 367 | */ |
| 368 | int packfile_store_for_each_object(struct packfile_store *store, |
| 369 | const struct object_info *request, |
| 370 | odb_for_each_object_cb cb, |
| 371 | void *cb_data, |
| 372 | const struct odb_for_each_object_options *opts); |
| 373 | |
| 374 | int packfile_store_find_abbrev_len(struct packfile_store *store, |
| 375 | const struct object_id *oid, |
| 376 | unsigned min_len, |
| 377 | unsigned *out); |
| 378 | |
| 379 | /* A hook to report invalid files in pack directory */ |
| 380 | #define PACKDIR_FILE_PACK 1 |
| 381 | #define PACKDIR_FILE_IDX 2 |
| 382 | #define PACKDIR_FILE_GARBAGE 4 |
| 383 | extern void (*report_garbage)(unsigned seen_bits, const char *path); |
| 384 | |
| 385 | void pack_report(struct repository *repo); |
| 386 | |
| 387 | /* |
| 388 | * mmap the index file for the specified packfile (if it is not |
| 389 | * already mmapped). Return 0 on success. |
| 390 | */ |
| 391 | int open_pack_index(struct packed_git *); |
| 392 | |
| 393 | /* |
| 394 | * munmap the index file for the specified packfile (if it is |
| 395 | * currently mmapped). |
| 396 | */ |
| 397 | void close_pack_index(struct packed_git *); |
| 398 | |
| 399 | int close_pack_fd(struct packed_git *p); |
| 400 | |
| 401 | uint32_t get_pack_fanout(struct packed_git *p, uint32_t value); |
| 402 | |
| 403 | struct object_database; |
| 404 | |
| 405 | unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned long *); |
| 406 | void close_pack_windows(struct packed_git *); |
| 407 | void close_pack(struct packed_git *); |
| 408 | void unuse_pack(struct pack_window **); |
| 409 | void clear_delta_base_cache(void); |
| 410 | struct packed_git *add_packed_git(struct repository *r, const char *path, |
| 411 | size_t path_len, int local); |
| 412 | |
| 413 | /* |
| 414 | * Unlink the .pack and associated extension files. |
| 415 | * Does not unlink if 'force_delete' is false and the pack-file is |
| 416 | * marked as ".keep". |
| 417 | */ |
| 418 | void unlink_pack_path(const char *pack_name, int force_delete); |
| 419 | |
| 420 | /* |
| 421 | * Make sure that a pointer access into an mmap'd index file is within bounds, |
| 422 | * and can provide at least 8 bytes of data. |
| 423 | * |
| 424 | * Note that this is only necessary for variable-length segments of the file |
| 425 | * (like the 64-bit extended offset table), as we compare the size to the |
| 426 | * fixed-length parts when we open the file. |
| 427 | */ |
| 428 | void check_pack_index_ptr(const struct packed_git *p, const void *ptr); |
| 429 | |
| 430 | /* |
| 431 | * Perform binary search on a pack-index for a given oid. Packfile is expected to |
| 432 | * have a valid pack-index. |
| 433 | * |
| 434 | * See 'bsearch_hash' for more information. |
| 435 | */ |
| 436 | int bsearch_pack(const struct object_id *oid, const struct packed_git *p, uint32_t *result); |
| 437 | |
| 438 | /* |
| 439 | * Write the oid of the nth object within the specified packfile into the first |
| 440 | * parameter. Open the index if it is not already open. Returns 0 on success, |
| 441 | * negative otherwise. |
| 442 | */ |
| 443 | int nth_packed_object_id(struct object_id *, struct packed_git *, uint32_t n); |
| 444 | |
| 445 | /* |
| 446 | * Return the offset of the nth object within the specified packfile. |
| 447 | * The index must already be opened. |
| 448 | */ |
| 449 | off_t nth_packed_object_offset(const struct packed_git *, uint32_t n); |
| 450 | |
| 451 | /* |
| 452 | * If the object named by oid is present in the specified packfile, |
| 453 | * return its offset within the packfile; otherwise, return 0. |
| 454 | */ |
| 455 | off_t find_pack_entry_one(const struct object_id *oid, struct packed_git *); |
| 456 | |
| 457 | int is_pack_valid(struct packed_git *); |
| 458 | void *unpack_entry(struct repository *r, struct packed_git *, off_t, |
| 459 | enum object_type *, size_t *); |
| 460 | unsigned long unpack_object_header_buffer(const unsigned char *buf, unsigned long len, enum object_type *type, size_t *sizep); |
| 461 | size_t get_size_from_delta(struct packed_git *, struct pack_window **, off_t); |
| 462 | int unpack_object_header(struct packed_git *, struct pack_window **, off_t *, size_t *); |
| 463 | off_t get_delta_base(struct packed_git *p, struct pack_window **w_curs, |
| 464 | off_t *curpos, enum object_type type, |
| 465 | off_t delta_obj_offset); |
| 466 | |
| 467 | int packfile_read_object_stream(struct odb_read_stream **out, |
| 468 | const struct object_id *oid, |
| 469 | struct packed_git *pack, |
| 470 | off_t offset); |
| 471 | |
| 472 | void release_pack_memory(size_t); |
| 473 | |
| 474 | /* global flag to enable extra checks when accessing packed objects */ |
| 475 | extern int do_check_packed_object_crc; |
| 476 | |
| 477 | /* |
| 478 | * Look up the object info for a specific offset in the packfile. |
| 479 | * Returns zero on success, a negative error code otherwise. |
| 480 | */ |
| 481 | int packed_object_info(struct packed_git *pack, |
| 482 | off_t offset, struct object_info *); |
| 483 | |
| 484 | void mark_bad_packed_object(struct packed_git *, const struct object_id *); |
| 485 | const struct packed_git *has_packed_and_bad(struct repository *, const struct object_id *); |
| 486 | |
| 487 | int has_object_pack(struct repository *r, const struct object_id *oid); |
| 488 | int has_object_kept_pack(struct repository *r, const struct object_id *oid, |
| 489 | unsigned flags); |
| 490 | |
| 491 | /* |
| 492 | * Return 1 if an object in a promisor packfile is or refers to the given |
| 493 | * object, 0 otherwise. |
| 494 | */ |
| 495 | int is_promisor_object(struct repository *r, const struct object_id *oid); |
| 496 | |
| 497 | /* |
| 498 | * Expose a function for fuzz testing. |
| 499 | * |
| 500 | * load_idx() parses a block of memory as a packfile index and puts the results |
| 501 | * into a struct packed_git. |
| 502 | * |
| 503 | * This function should not be used directly. It is exposed here only so that we |
| 504 | * have a convenient entry-point for fuzz testing. For real uses, you should |
| 505 | * probably use open_pack_index() instead. |
| 506 | */ |
| 507 | int load_idx(const char *path, const unsigned int hashsz, void *idx_map, |
| 508 | size_t idx_size, struct packed_git *p); |
| 509 | |
| 510 | /* |
| 511 | * Parse a --pack_header option as accepted by index-pack and unpack-objects, |
| 512 | * turning it into the matching bytes we'd find in a pack. |
| 513 | */ |
| 514 | int parse_pack_header_option(const char *in, unsigned char *out, unsigned int *len); |
| 515 | |
| 516 | #endif |