| 1 | #define USE_THE_REPOSITORY_VARIABLE |
| 2 | #define DISABLE_SIGN_COMPARE_WARNINGS |
| 3 | |
| 4 | #include "builtin.h" |
| 5 | #include "config.h" |
| 6 | #include "delta.h" |
| 7 | #include "environment.h" |
| 8 | #include "gettext.h" |
| 9 | #include "hex.h" |
| 10 | #include "pack.h" |
| 11 | #include "csum-file.h" |
| 12 | #include "blob.h" |
| 13 | #include "commit.h" |
| 14 | #include "tag.h" |
| 15 | #include "tree.h" |
| 16 | #include "progress.h" |
| 17 | #include "fsck.h" |
| 18 | #include "strbuf.h" |
| 19 | #include "thread-utils.h" |
| 20 | #include "packfile.h" |
| 21 | #include "pack-revindex.h" |
| 22 | #include "object-file.h" |
| 23 | #include "odb.h" |
| 24 | #include "odb/streaming.h" |
| 25 | #include "oid-array.h" |
| 26 | #include "oidset.h" |
| 27 | #include "path.h" |
| 28 | #include "replace-object.h" |
| 29 | #include "tree-walk.h" |
| 30 | #include "promisor-remote.h" |
| 31 | #include "run-command.h" |
| 32 | #include "setup.h" |
| 33 | #include "strvec.h" |
| 34 | |
| 35 | static const char index_pack_usage[] = |
| 36 | "git index-pack [-v] [-o <index-file>] [--keep | --keep=<msg>] [--[no-]rev-index] [--verify] [--strict[=<msg-id>=<severity>...]] [--fsck-objects[=<msg-id>=<severity>...]] (<pack-file> | --stdin [--fix-thin] [<pack-file>])"; |
| 37 | |
| 38 | struct object_entry { |
| 39 | struct pack_idx_entry idx; |
| 40 | size_t size; |
| 41 | unsigned char hdr_size; |
| 42 | signed char type; |
| 43 | signed char real_type; |
| 44 | }; |
| 45 | |
| 46 | struct object_stat { |
| 47 | unsigned delta_depth; |
| 48 | int base_object_no; |
| 49 | }; |
| 50 | |
| 51 | struct base_data { |
| 52 | /* Initialized by make_base(). */ |
| 53 | struct base_data *base; |
| 54 | struct object_entry *obj; |
| 55 | int ref_first, ref_last; |
| 56 | int ofs_first, ofs_last; |
| 57 | /* |
| 58 | * Threads should increment retain_data if they are about to call |
| 59 | * patch_delta() using this struct's data as a base, and decrement this |
| 60 | * when they are done. While retain_data is nonzero, this struct's data |
| 61 | * will not be freed even if the delta base cache limit is exceeded. |
| 62 | */ |
| 63 | int retain_data; |
| 64 | /* |
| 65 | * The number of direct children that have not been fully processed |
| 66 | * (entered work_head, entered done_head, left done_head). When this |
| 67 | * number reaches zero, this struct base_data can be freed. |
| 68 | */ |
| 69 | int children_remaining; |
| 70 | |
| 71 | /* Not initialized by make_base(). */ |
| 72 | struct list_head list; |
| 73 | void *data; |
| 74 | size_t size; |
| 75 | }; |
| 76 | |
| 77 | /* |
| 78 | * Stack of struct base_data that have unprocessed children. |
| 79 | * threaded_second_pass() uses this as a source of work (the other being the |
| 80 | * objects array). |
| 81 | * |
| 82 | * Guarded by work_mutex. |
| 83 | */ |
| 84 | static LIST_HEAD(work_head); |
| 85 | |
| 86 | /* |
| 87 | * Stack of struct base_data that have children, all of whom have been |
| 88 | * processed or are being processed, and at least one child is being processed. |
| 89 | * These struct base_data must be kept around until the last child is |
| 90 | * processed. |
| 91 | * |
| 92 | * Guarded by work_mutex. |
| 93 | */ |
| 94 | static LIST_HEAD(done_head); |
| 95 | |
| 96 | /* |
| 97 | * All threads share one delta base cache. |
| 98 | * |
| 99 | * base_cache_used is guarded by work_mutex, and base_cache_limit is read-only |
| 100 | * in a thread. |
| 101 | */ |
| 102 | static size_t base_cache_used; |
| 103 | static size_t base_cache_limit; |
| 104 | |
| 105 | struct thread_local_data { |
| 106 | pthread_t thread; |
| 107 | int pack_fd; |
| 108 | }; |
| 109 | |
| 110 | /* Remember to update object flag allocation in object.h */ |
| 111 | #define FLAG_LINK (1u<<20) |
| 112 | #define FLAG_CHECKED (1u<<21) |
| 113 | |
| 114 | struct ofs_delta_entry { |
| 115 | off_t offset; |
| 116 | int obj_no; |
| 117 | }; |
| 118 | |
| 119 | struct ref_delta_entry { |
| 120 | struct object_id oid; |
| 121 | int obj_no; |
| 122 | }; |
| 123 | |
| 124 | static struct object_entry *objects; |
| 125 | static struct object_stat *obj_stat; |
| 126 | static struct ofs_delta_entry *ofs_deltas; |
| 127 | static struct ref_delta_entry *ref_deltas; |
| 128 | static struct thread_local_data nothread_data; |
| 129 | static int nr_objects; |
| 130 | static int nr_ofs_deltas; |
| 131 | static int nr_ref_deltas; |
| 132 | static int ref_deltas_alloc; |
| 133 | static int nr_resolved_deltas; |
| 134 | static int nr_threads; |
| 135 | |
| 136 | static int from_stdin; |
| 137 | static int strict; |
| 138 | static int do_fsck_object; |
| 139 | static struct fsck_options fsck_options; |
| 140 | static int verbose; |
| 141 | static const char *progress_title; |
| 142 | static int show_resolving_progress; |
| 143 | static int show_stat; |
| 144 | static int check_self_contained_and_connected; |
| 145 | |
| 146 | static struct progress *progress; |
| 147 | |
| 148 | static unsigned char input_buffer[DEFAULT_IO_BUFFER_SIZE]; |
| 149 | static unsigned int input_offset, input_len; |
| 150 | static off_t consumed_bytes; |
| 151 | static off_t max_input_size; |
| 152 | static unsigned deepest_delta; |
| 153 | static struct git_hash_ctx input_ctx; |
| 154 | static uint32_t input_crc32; |
| 155 | static int input_fd, output_fd; |
| 156 | static const char *curr_pack; |
| 157 | |
| 158 | /* |
| 159 | * outgoing_links is guarded by read_mutex, and record_outgoing_links is |
| 160 | * read-only in a thread. |
| 161 | */ |
| 162 | static struct oidset outgoing_links = OIDSET_INIT; |
| 163 | static int record_outgoing_links; |
| 164 | |
| 165 | static struct thread_local_data *thread_data; |
| 166 | static int nr_dispatched; |
| 167 | static int threads_active; |
| 168 | |
| 169 | static pthread_mutex_t read_mutex; |
| 170 | #define read_lock() lock_mutex(&read_mutex) |
| 171 | #define read_unlock() unlock_mutex(&read_mutex) |
| 172 | |
| 173 | static pthread_mutex_t counter_mutex; |
| 174 | #define counter_lock() lock_mutex(&counter_mutex) |
| 175 | #define counter_unlock() unlock_mutex(&counter_mutex) |
| 176 | |
| 177 | static pthread_mutex_t work_mutex; |
| 178 | #define work_lock() lock_mutex(&work_mutex) |
| 179 | #define work_unlock() unlock_mutex(&work_mutex) |
| 180 | |
| 181 | static pthread_mutex_t deepest_delta_mutex; |
| 182 | #define deepest_delta_lock() lock_mutex(&deepest_delta_mutex) |
| 183 | #define deepest_delta_unlock() unlock_mutex(&deepest_delta_mutex) |
| 184 | |
| 185 | static pthread_key_t key; |
| 186 | |
| 187 | static inline void lock_mutex(pthread_mutex_t *mutex) |
| 188 | { |
| 189 | if (threads_active) |
| 190 | pthread_mutex_lock(mutex); |
| 191 | } |
| 192 | |
| 193 | static inline void unlock_mutex(pthread_mutex_t *mutex) |
| 194 | { |
| 195 | if (threads_active) |
| 196 | pthread_mutex_unlock(mutex); |
| 197 | } |
| 198 | |
| 199 | /* |
| 200 | * Mutex and conditional variable can't be statically-initialized on Windows. |
| 201 | */ |
| 202 | static void init_thread(void) |
| 203 | { |
| 204 | int i; |
| 205 | init_recursive_mutex(&read_mutex); |
| 206 | pthread_mutex_init(&counter_mutex, NULL); |
| 207 | pthread_mutex_init(&work_mutex, NULL); |
| 208 | if (show_stat) |
| 209 | pthread_mutex_init(&deepest_delta_mutex, NULL); |
| 210 | pthread_key_create(&key, NULL); |
| 211 | CALLOC_ARRAY(thread_data, nr_threads); |
| 212 | for (i = 0; i < nr_threads; i++) { |
| 213 | thread_data[i].pack_fd = xopen(curr_pack, O_RDONLY); |
| 214 | } |
| 215 | |
| 216 | threads_active = 1; |
| 217 | } |
| 218 | |
| 219 | static void cleanup_thread(void) |
| 220 | { |
| 221 | int i; |
| 222 | if (!threads_active) |
| 223 | return; |
| 224 | threads_active = 0; |
| 225 | pthread_mutex_destroy(&read_mutex); |
| 226 | pthread_mutex_destroy(&counter_mutex); |
| 227 | pthread_mutex_destroy(&work_mutex); |
| 228 | if (show_stat) |
| 229 | pthread_mutex_destroy(&deepest_delta_mutex); |
| 230 | for (i = 0; i < nr_threads; i++) |
| 231 | close(thread_data[i].pack_fd); |
| 232 | pthread_key_delete(key); |
| 233 | free(thread_data); |
| 234 | } |
| 235 | |
| 236 | static int mark_link(struct object *obj, enum object_type type, |
| 237 | void *data UNUSED, |
| 238 | struct fsck_options *options UNUSED) |
| 239 | { |
| 240 | if (!obj) |
| 241 | return -1; |
| 242 | |
| 243 | if (type != OBJ_ANY && obj->type != type) |
| 244 | die(_("object type mismatch at %s"), oid_to_hex(&obj->oid)); |
| 245 | |
| 246 | obj->flags |= FLAG_LINK; |
| 247 | return 0; |
| 248 | } |
| 249 | |
| 250 | /* The content of each linked object must have been checked |
| 251 | or it must be already present in the object database */ |
| 252 | static unsigned check_object(struct object *obj) |
| 253 | { |
| 254 | if (!obj) |
| 255 | return 0; |
| 256 | |
| 257 | if (!(obj->flags & FLAG_LINK)) |
| 258 | return 0; |
| 259 | |
| 260 | if (!(obj->flags & FLAG_CHECKED)) { |
| 261 | size_t size; |
| 262 | int type = odb_read_object_info(the_repository->objects, |
| 263 | &obj->oid, &size); |
| 264 | if (type <= 0) |
| 265 | die(_("did not receive expected object %s"), |
| 266 | oid_to_hex(&obj->oid)); |
| 267 | if (type != obj->type) |
| 268 | die(_("object %s: expected type %s, found %s"), |
| 269 | oid_to_hex(&obj->oid), |
| 270 | type_name(obj->type), type_name(type)); |
| 271 | obj->flags |= FLAG_CHECKED; |
| 272 | return 1; |
| 273 | } |
| 274 | |
| 275 | return 0; |
| 276 | } |
| 277 | |
| 278 | static unsigned check_objects(void) |
| 279 | { |
| 280 | unsigned i, max, foreign_nr = 0; |
| 281 | |
| 282 | max = get_max_object_index(the_repository); |
| 283 | |
| 284 | if (verbose) |
| 285 | progress = start_delayed_progress(the_repository, |
| 286 | _("Checking objects"), max); |
| 287 | |
| 288 | for (i = 0; i < max; i++) { |
| 289 | foreign_nr += check_object(get_indexed_object(the_repository, i)); |
| 290 | display_progress(progress, i + 1); |
| 291 | } |
| 292 | |
| 293 | stop_progress(&progress); |
| 294 | return foreign_nr; |
| 295 | } |
| 296 | |
| 297 | |
| 298 | /* Discard current buffer used content. */ |
| 299 | static void flush(void) |
| 300 | { |
| 301 | if (input_offset) { |
| 302 | if (output_fd >= 0) |
| 303 | write_or_die(output_fd, input_buffer, input_offset); |
| 304 | git_hash_update(&input_ctx, input_buffer, input_offset); |
| 305 | memmove(input_buffer, input_buffer + input_offset, input_len); |
| 306 | input_offset = 0; |
| 307 | } |
| 308 | } |
| 309 | |
| 310 | /* |
| 311 | * Make sure at least "min" bytes are available in the buffer, and |
| 312 | * return the pointer to the buffer. |
| 313 | */ |
| 314 | static void *fill(int min) |
| 315 | { |
| 316 | if (min <= input_len) |
| 317 | return input_buffer + input_offset; |
| 318 | if (min > sizeof(input_buffer)) |
| 319 | die(Q_("cannot fill %d byte", |
| 320 | "cannot fill %d bytes", |
| 321 | min), |
| 322 | min); |
| 323 | flush(); |
| 324 | do { |
| 325 | ssize_t ret = xread(input_fd, input_buffer + input_len, |
| 326 | sizeof(input_buffer) - input_len); |
| 327 | if (ret <= 0) { |
| 328 | if (!ret) |
| 329 | die(_("early EOF")); |
| 330 | die_errno(_("read error on input")); |
| 331 | } |
| 332 | input_len += ret; |
| 333 | if (from_stdin) |
| 334 | display_throughput(progress, consumed_bytes + input_len); |
| 335 | } while (input_len < min); |
| 336 | return input_buffer; |
| 337 | } |
| 338 | |
| 339 | static void use(int bytes) |
| 340 | { |
| 341 | if (bytes > input_len) |
| 342 | die(_("used more bytes than were available")); |
| 343 | input_crc32 = crc32(input_crc32, input_buffer + input_offset, bytes); |
| 344 | input_len -= bytes; |
| 345 | input_offset += bytes; |
| 346 | |
| 347 | /* make sure off_t is sufficiently large not to wrap */ |
| 348 | if (signed_add_overflows(consumed_bytes, bytes)) |
| 349 | die(_("pack too large for current definition of off_t")); |
| 350 | consumed_bytes += bytes; |
| 351 | if (max_input_size && consumed_bytes > max_input_size) { |
| 352 | struct strbuf size_limit = STRBUF_INIT; |
| 353 | strbuf_humanise_bytes(&size_limit, max_input_size); |
| 354 | die(_("pack exceeds maximum allowed size (%s)"), |
| 355 | size_limit.buf); |
| 356 | } |
| 357 | } |
| 358 | |
| 359 | static const char *open_pack_file(const char *pack_name) |
| 360 | { |
| 361 | if (from_stdin) { |
| 362 | input_fd = 0; |
| 363 | if (!pack_name) { |
| 364 | struct strbuf tmp_file = STRBUF_INIT; |
| 365 | output_fd = odb_mkstemp(the_repository->objects, &tmp_file, |
| 366 | "pack/tmp_pack_XXXXXX"); |
| 367 | pack_name = strbuf_detach(&tmp_file, NULL); |
| 368 | } else { |
| 369 | output_fd = xopen(pack_name, O_CREAT|O_EXCL|O_RDWR, 0600); |
| 370 | } |
| 371 | nothread_data.pack_fd = output_fd; |
| 372 | } else { |
| 373 | input_fd = xopen(pack_name, O_RDONLY); |
| 374 | output_fd = -1; |
| 375 | nothread_data.pack_fd = input_fd; |
| 376 | } |
| 377 | the_hash_algo->init_fn(&input_ctx); |
| 378 | return pack_name; |
| 379 | } |
| 380 | |
| 381 | static void parse_pack_header(void) |
| 382 | { |
| 383 | unsigned char *hdr = fill(sizeof(struct pack_header)); |
| 384 | |
| 385 | /* Header consistency check */ |
| 386 | if (get_be32(hdr) != PACK_SIGNATURE) |
| 387 | die(_("pack signature mismatch")); |
| 388 | hdr += 4; |
| 389 | if (!pack_version_ok_native(get_be32(hdr))) |
| 390 | die(_("pack version %"PRIu32" unsupported"), |
| 391 | get_be32(hdr)); |
| 392 | hdr += 4; |
| 393 | |
| 394 | nr_objects = get_be32(hdr); |
| 395 | use(sizeof(struct pack_header)); |
| 396 | } |
| 397 | |
| 398 | __attribute__((format (printf, 2, 3))) |
| 399 | static NORETURN void bad_object(off_t offset, const char *format, ...) |
| 400 | { |
| 401 | va_list params; |
| 402 | char buf[1024]; |
| 403 | |
| 404 | va_start(params, format); |
| 405 | vsnprintf(buf, sizeof(buf), format, params); |
| 406 | va_end(params); |
| 407 | die(_("pack has bad object at offset %"PRIuMAX": %s"), |
| 408 | (uintmax_t)offset, buf); |
| 409 | } |
| 410 | |
| 411 | static inline struct thread_local_data *get_thread_data(void) |
| 412 | { |
| 413 | if (HAVE_THREADS) { |
| 414 | if (threads_active) |
| 415 | return pthread_getspecific(key); |
| 416 | assert(!threads_active && |
| 417 | "This should only be reached when all threads are gone"); |
| 418 | } |
| 419 | return ¬hread_data; |
| 420 | } |
| 421 | |
| 422 | static void set_thread_data(struct thread_local_data *data) |
| 423 | { |
| 424 | if (threads_active) |
| 425 | pthread_setspecific(key, data); |
| 426 | } |
| 427 | |
| 428 | static void free_base_data(struct base_data *c) |
| 429 | { |
| 430 | if (c->data) { |
| 431 | FREE_AND_NULL(c->data); |
| 432 | base_cache_used -= c->size; |
| 433 | } |
| 434 | } |
| 435 | |
| 436 | static void prune_base_data(struct base_data *retain) |
| 437 | { |
| 438 | struct list_head *pos; |
| 439 | |
| 440 | if (base_cache_used <= base_cache_limit) |
| 441 | return; |
| 442 | |
| 443 | list_for_each_prev(pos, &done_head) { |
| 444 | struct base_data *b = list_entry(pos, struct base_data, list); |
| 445 | if (b->retain_data || b == retain) |
| 446 | continue; |
| 447 | if (b->data) { |
| 448 | free_base_data(b); |
| 449 | if (base_cache_used <= base_cache_limit) |
| 450 | return; |
| 451 | } |
| 452 | } |
| 453 | |
| 454 | list_for_each_prev(pos, &work_head) { |
| 455 | struct base_data *b = list_entry(pos, struct base_data, list); |
| 456 | if (b->retain_data || b == retain) |
| 457 | continue; |
| 458 | if (b->data) { |
| 459 | free_base_data(b); |
| 460 | if (base_cache_used <= base_cache_limit) |
| 461 | return; |
| 462 | } |
| 463 | } |
| 464 | } |
| 465 | |
| 466 | static int is_delta_type(enum object_type type) |
| 467 | { |
| 468 | return (type == OBJ_REF_DELTA || type == OBJ_OFS_DELTA); |
| 469 | } |
| 470 | |
| 471 | static void *unpack_entry_data(off_t offset, size_t size, |
| 472 | enum object_type type, struct object_id *oid) |
| 473 | { |
| 474 | static char fixed_buf[8192]; |
| 475 | int status; |
| 476 | git_zstream stream; |
| 477 | void *buf; |
| 478 | struct git_hash_ctx c; |
| 479 | char hdr[32]; |
| 480 | int hdrlen; |
| 481 | |
| 482 | if (!is_delta_type(type)) { |
| 483 | hdrlen = format_object_header(hdr, sizeof(hdr), type, size); |
| 484 | the_hash_algo->init_fn(&c); |
| 485 | git_hash_update(&c, hdr, hdrlen); |
| 486 | } else |
| 487 | oid = NULL; |
| 488 | if (type == OBJ_BLOB && |
| 489 | size > repo_settings_get_big_file_threshold(the_repository)) |
| 490 | buf = fixed_buf; |
| 491 | else |
| 492 | buf = xmallocz(size); |
| 493 | |
| 494 | memset(&stream, 0, sizeof(stream)); |
| 495 | git_inflate_init(&stream); |
| 496 | stream.next_out = buf; |
| 497 | stream.avail_out = buf == fixed_buf ? sizeof(fixed_buf) : size; |
| 498 | |
| 499 | do { |
| 500 | unsigned char *last_out = stream.next_out; |
| 501 | stream.next_in = fill(1); |
| 502 | stream.avail_in = input_len; |
| 503 | status = git_inflate(&stream, 0); |
| 504 | use(input_len - stream.avail_in); |
| 505 | if (oid) |
| 506 | git_hash_update(&c, last_out, stream.next_out - last_out); |
| 507 | if (buf == fixed_buf) { |
| 508 | stream.next_out = buf; |
| 509 | stream.avail_out = sizeof(fixed_buf); |
| 510 | } |
| 511 | } while (status == Z_OK); |
| 512 | if (stream.total_out != size || status != Z_STREAM_END) |
| 513 | bad_object(offset, _("inflate returned %d"), status); |
| 514 | git_inflate_end(&stream); |
| 515 | if (oid) |
| 516 | git_hash_final_oid(oid, &c); |
| 517 | return buf == fixed_buf ? NULL : buf; |
| 518 | } |
| 519 | |
| 520 | static void *unpack_raw_entry(struct object_entry *obj, |
| 521 | off_t *ofs_offset, |
| 522 | struct object_id *ref_oid, |
| 523 | struct object_id *oid) |
| 524 | { |
| 525 | unsigned char *p; |
| 526 | size_t size, c; |
| 527 | off_t base_offset; |
| 528 | unsigned shift; |
| 529 | void *data; |
| 530 | |
| 531 | obj->idx.offset = consumed_bytes; |
| 532 | input_crc32 = crc32(0, NULL, 0); |
| 533 | |
| 534 | p = fill(1); |
| 535 | c = *p; |
| 536 | use(1); |
| 537 | obj->type = (c >> 4) & 7; |
| 538 | size = (c & 15); |
| 539 | shift = 4; |
| 540 | while (c & 0x80) { |
| 541 | if ((bitsizeof(size_t) - 7) < shift) |
| 542 | die(_("object size too large for this platform")); |
| 543 | p = fill(1); |
| 544 | c = *p; |
| 545 | use(1); |
| 546 | size += (c & 0x7f) << shift; |
| 547 | shift += 7; |
| 548 | } |
| 549 | obj->size = size; |
| 550 | |
| 551 | switch (obj->type) { |
| 552 | case OBJ_REF_DELTA: |
| 553 | oidread(ref_oid, fill(the_hash_algo->rawsz), |
| 554 | the_repository->hash_algo); |
| 555 | use(the_hash_algo->rawsz); |
| 556 | break; |
| 557 | case OBJ_OFS_DELTA: |
| 558 | p = fill(1); |
| 559 | c = *p; |
| 560 | use(1); |
| 561 | base_offset = c & 127; |
| 562 | while (c & 128) { |
| 563 | base_offset += 1; |
| 564 | if (!base_offset || MSB(base_offset, 7)) |
| 565 | bad_object(obj->idx.offset, _("offset value overflow for delta base object")); |
| 566 | p = fill(1); |
| 567 | c = *p; |
| 568 | use(1); |
| 569 | base_offset = (base_offset << 7) + (c & 127); |
| 570 | } |
| 571 | *ofs_offset = obj->idx.offset - base_offset; |
| 572 | if (*ofs_offset <= 0 || *ofs_offset >= obj->idx.offset) |
| 573 | bad_object(obj->idx.offset, _("delta base offset is out of bound")); |
| 574 | break; |
| 575 | case OBJ_COMMIT: |
| 576 | case OBJ_TREE: |
| 577 | case OBJ_BLOB: |
| 578 | case OBJ_TAG: |
| 579 | break; |
| 580 | default: |
| 581 | bad_object(obj->idx.offset, _("unknown object type %d"), obj->type); |
| 582 | } |
| 583 | obj->hdr_size = consumed_bytes - obj->idx.offset; |
| 584 | |
| 585 | data = unpack_entry_data(obj->idx.offset, obj->size, obj->type, oid); |
| 586 | obj->idx.crc32 = input_crc32; |
| 587 | return data; |
| 588 | } |
| 589 | |
| 590 | static void *unpack_data(struct object_entry *obj, |
| 591 | int (*consume)(const unsigned char *, unsigned long, void *), |
| 592 | void *cb_data) |
| 593 | { |
| 594 | off_t from = obj[0].idx.offset + obj[0].hdr_size; |
| 595 | off_t len = obj[1].idx.offset - from; |
| 596 | unsigned char *data, *inbuf; |
| 597 | git_zstream stream; |
| 598 | int status; |
| 599 | |
| 600 | data = xmallocz(consume ? 64*1024 : obj->size); |
| 601 | inbuf = xmalloc((len < 64*1024) ? (int)len : 64*1024); |
| 602 | |
| 603 | memset(&stream, 0, sizeof(stream)); |
| 604 | git_inflate_init(&stream); |
| 605 | stream.next_out = data; |
| 606 | stream.avail_out = consume ? 64*1024 : obj->size; |
| 607 | |
| 608 | do { |
| 609 | ssize_t n = (len < 64*1024) ? (ssize_t)len : 64*1024; |
| 610 | n = xpread(get_thread_data()->pack_fd, inbuf, n, from); |
| 611 | if (n < 0) |
| 612 | die_errno(_("cannot pread pack file")); |
| 613 | if (!n) |
| 614 | die(Q_("premature end of pack file, %"PRIuMAX" byte missing", |
| 615 | "premature end of pack file, %"PRIuMAX" bytes missing", |
| 616 | len), |
| 617 | (uintmax_t)len); |
| 618 | from += n; |
| 619 | len -= n; |
| 620 | stream.next_in = inbuf; |
| 621 | stream.avail_in = n; |
| 622 | if (!consume) |
| 623 | status = git_inflate(&stream, 0); |
| 624 | else { |
| 625 | do { |
| 626 | status = git_inflate(&stream, 0); |
| 627 | if (consume(data, stream.next_out - data, cb_data)) { |
| 628 | free(inbuf); |
| 629 | free(data); |
| 630 | return NULL; |
| 631 | } |
| 632 | stream.next_out = data; |
| 633 | stream.avail_out = 64*1024; |
| 634 | } while (status == Z_OK && stream.avail_in); |
| 635 | } |
| 636 | } while (len && status == Z_OK && !stream.avail_in); |
| 637 | |
| 638 | /* This has been inflated OK when first encountered, so... */ |
| 639 | if (status != Z_STREAM_END || stream.total_out != obj->size) |
| 640 | die(_("serious inflate inconsistency")); |
| 641 | |
| 642 | git_inflate_end(&stream); |
| 643 | free(inbuf); |
| 644 | if (consume) { |
| 645 | FREE_AND_NULL(data); |
| 646 | } |
| 647 | return data; |
| 648 | } |
| 649 | |
| 650 | static void *get_data_from_pack(struct object_entry *obj) |
| 651 | { |
| 652 | return unpack_data(obj, NULL, NULL); |
| 653 | } |
| 654 | |
| 655 | static int compare_ofs_delta_bases(off_t offset1, off_t offset2, |
| 656 | enum object_type type1, |
| 657 | enum object_type type2) |
| 658 | { |
| 659 | int cmp = type1 - type2; |
| 660 | if (cmp) |
| 661 | return cmp; |
| 662 | return offset1 < offset2 ? -1 : |
| 663 | offset1 > offset2 ? 1 : |
| 664 | 0; |
| 665 | } |
| 666 | |
| 667 | static int find_ofs_delta(const off_t offset) |
| 668 | { |
| 669 | int first = 0, last = nr_ofs_deltas; |
| 670 | |
| 671 | while (first < last) { |
| 672 | int next = first + (last - first) / 2; |
| 673 | struct ofs_delta_entry *delta = &ofs_deltas[next]; |
| 674 | int cmp; |
| 675 | |
| 676 | cmp = compare_ofs_delta_bases(offset, delta->offset, |
| 677 | OBJ_OFS_DELTA, |
| 678 | objects[delta->obj_no].type); |
| 679 | if (!cmp) |
| 680 | return next; |
| 681 | if (cmp < 0) { |
| 682 | last = next; |
| 683 | continue; |
| 684 | } |
| 685 | first = next+1; |
| 686 | } |
| 687 | return -first-1; |
| 688 | } |
| 689 | |
| 690 | static void find_ofs_delta_children(off_t offset, |
| 691 | int *first_index, int *last_index) |
| 692 | { |
| 693 | int first = find_ofs_delta(offset); |
| 694 | int last = first; |
| 695 | int end = nr_ofs_deltas - 1; |
| 696 | |
| 697 | if (first < 0) { |
| 698 | *first_index = 0; |
| 699 | *last_index = -1; |
| 700 | return; |
| 701 | } |
| 702 | while (first > 0 && ofs_deltas[first - 1].offset == offset) |
| 703 | --first; |
| 704 | while (last < end && ofs_deltas[last + 1].offset == offset) |
| 705 | ++last; |
| 706 | *first_index = first; |
| 707 | *last_index = last; |
| 708 | } |
| 709 | |
| 710 | static int compare_ref_delta_bases(const struct object_id *oid1, |
| 711 | const struct object_id *oid2, |
| 712 | enum object_type type1, |
| 713 | enum object_type type2) |
| 714 | { |
| 715 | int cmp = type1 - type2; |
| 716 | if (cmp) |
| 717 | return cmp; |
| 718 | return oidcmp(oid1, oid2); |
| 719 | } |
| 720 | |
| 721 | static int find_ref_delta(const struct object_id *oid) |
| 722 | { |
| 723 | int first = 0, last = nr_ref_deltas; |
| 724 | |
| 725 | while (first < last) { |
| 726 | int next = first + (last - first) / 2; |
| 727 | struct ref_delta_entry *delta = &ref_deltas[next]; |
| 728 | int cmp; |
| 729 | |
| 730 | cmp = compare_ref_delta_bases(oid, &delta->oid, |
| 731 | OBJ_REF_DELTA, |
| 732 | objects[delta->obj_no].type); |
| 733 | if (!cmp) |
| 734 | return next; |
| 735 | if (cmp < 0) { |
| 736 | last = next; |
| 737 | continue; |
| 738 | } |
| 739 | first = next+1; |
| 740 | } |
| 741 | return -first-1; |
| 742 | } |
| 743 | |
| 744 | static void find_ref_delta_children(const struct object_id *oid, |
| 745 | int *first_index, int *last_index) |
| 746 | { |
| 747 | int first = find_ref_delta(oid); |
| 748 | int last = first; |
| 749 | int end = nr_ref_deltas - 1; |
| 750 | |
| 751 | if (first < 0) { |
| 752 | *first_index = 0; |
| 753 | *last_index = -1; |
| 754 | return; |
| 755 | } |
| 756 | while (first > 0 && oideq(&ref_deltas[first - 1].oid, oid)) |
| 757 | --first; |
| 758 | while (last < end && oideq(&ref_deltas[last + 1].oid, oid)) |
| 759 | ++last; |
| 760 | *first_index = first; |
| 761 | *last_index = last; |
| 762 | } |
| 763 | |
| 764 | struct compare_data { |
| 765 | struct object_entry *entry; |
| 766 | struct odb_read_stream *st; |
| 767 | unsigned char *buf; |
| 768 | unsigned long buf_size; |
| 769 | }; |
| 770 | |
| 771 | static int compare_objects(const unsigned char *buf, unsigned long size, |
| 772 | void *cb_data) |
| 773 | { |
| 774 | struct compare_data *data = cb_data; |
| 775 | |
| 776 | if (data->buf_size < size) { |
| 777 | free(data->buf); |
| 778 | data->buf = xmalloc(size); |
| 779 | data->buf_size = size; |
| 780 | } |
| 781 | |
| 782 | while (size) { |
| 783 | ssize_t len = odb_read_stream_read(data->st, data->buf, size); |
| 784 | if (len == 0) |
| 785 | die(_("SHA1 COLLISION FOUND WITH %s !"), |
| 786 | oid_to_hex(&data->entry->idx.oid)); |
| 787 | if (len < 0) |
| 788 | die(_("unable to read %s"), |
| 789 | oid_to_hex(&data->entry->idx.oid)); |
| 790 | if (memcmp(buf, data->buf, len)) |
| 791 | die(_("SHA1 COLLISION FOUND WITH %s !"), |
| 792 | oid_to_hex(&data->entry->idx.oid)); |
| 793 | size -= len; |
| 794 | buf += len; |
| 795 | } |
| 796 | return 0; |
| 797 | } |
| 798 | |
| 799 | static int check_collison(struct object_entry *entry) |
| 800 | { |
| 801 | struct compare_data data; |
| 802 | |
| 803 | if (entry->size <= repo_settings_get_big_file_threshold(the_repository) || |
| 804 | entry->type != OBJ_BLOB) |
| 805 | return -1; |
| 806 | |
| 807 | memset(&data, 0, sizeof(data)); |
| 808 | data.entry = entry; |
| 809 | data.st = odb_read_stream_open(the_repository->objects, &entry->idx.oid, NULL); |
| 810 | if (!data.st) |
| 811 | return -1; |
| 812 | if (data.st->size != entry->size || data.st->type != entry->type) |
| 813 | die(_("SHA1 COLLISION FOUND WITH %s !"), |
| 814 | oid_to_hex(&entry->idx.oid)); |
| 815 | unpack_data(entry, compare_objects, &data); |
| 816 | odb_read_stream_close(data.st); |
| 817 | free(data.buf); |
| 818 | return 0; |
| 819 | } |
| 820 | |
| 821 | static void record_outgoing_link(const struct object_id *oid) |
| 822 | { |
| 823 | oidset_insert(&outgoing_links, oid); |
| 824 | } |
| 825 | |
| 826 | static void maybe_record_name_entry(const struct name_entry *entry) |
| 827 | { |
| 828 | /* |
| 829 | * Checking only trees here results in a significantly faster packfile |
| 830 | * indexing, but the drawback is that if the packfile to be indexed |
| 831 | * references a local blob only directly (that is, never through a |
| 832 | * local tree), that local blob is in danger of being garbage |
| 833 | * collected. Such a situation may arise if we push local commits, |
| 834 | * including one with a change to a blob in the root tree, and then the |
| 835 | * server incorporates them into its main branch through a "rebase" or |
| 836 | * "squash" merge strategy, and then we fetch the new main branch from |
| 837 | * the server. |
| 838 | * |
| 839 | * This situation has not been observed yet - we have only noticed |
| 840 | * missing commits, not missing trees or blobs. (In fact, if it were |
| 841 | * believed that only missing commits are problematic, one could argue |
| 842 | * that we should also exclude trees during the outgoing link check; |
| 843 | * but it is safer to include them.) |
| 844 | * |
| 845 | * Due to the rarity of the situation (it has not been observed to |
| 846 | * happen in real life), and because the "penalty" in such a situation |
| 847 | * is merely to refetch the missing blob when it's needed (and this |
| 848 | * happens only once - when refetched, the blob goes into a promisor |
| 849 | * pack, so it won't be GC-ed, the tradeoff seems worth it. |
| 850 | */ |
| 851 | if (S_ISDIR(entry->mode)) |
| 852 | record_outgoing_link(&entry->oid); |
| 853 | } |
| 854 | |
| 855 | static void do_record_outgoing_links(struct object *obj) |
| 856 | { |
| 857 | if (obj->type == OBJ_TREE) { |
| 858 | struct tree *tree = (struct tree *)obj; |
| 859 | struct tree_desc desc; |
| 860 | struct name_entry entry; |
| 861 | if (init_tree_desc_gently(&desc, &tree->object.oid, |
| 862 | tree->buffer, tree->size, 0)) |
| 863 | /* |
| 864 | * Error messages are given when packs are |
| 865 | * verified, so do not print any here. |
| 866 | */ |
| 867 | return; |
| 868 | while (tree_entry_gently(&desc, &entry)) |
| 869 | maybe_record_name_entry(&entry); |
| 870 | } else if (obj->type == OBJ_COMMIT) { |
| 871 | struct commit *commit = (struct commit *) obj; |
| 872 | struct commit_list *parents = commit->parents; |
| 873 | |
| 874 | record_outgoing_link(get_commit_tree_oid(commit)); |
| 875 | for (; parents; parents = parents->next) |
| 876 | record_outgoing_link(&parents->item->object.oid); |
| 877 | } else if (obj->type == OBJ_TAG) { |
| 878 | struct tag *tag = (struct tag *) obj; |
| 879 | record_outgoing_link(get_tagged_oid(tag)); |
| 880 | } |
| 881 | } |
| 882 | |
| 883 | static void sha1_object(const void *data, struct object_entry *obj_entry, |
| 884 | unsigned long size, enum object_type type, |
| 885 | const struct object_id *oid) |
| 886 | { |
| 887 | void *new_data = NULL; |
| 888 | int collision_test_needed = 0; |
| 889 | |
| 890 | assert(data || obj_entry); |
| 891 | |
| 892 | if (startup_info->have_repository) { |
| 893 | read_lock(); |
| 894 | collision_test_needed = odb_has_object(the_repository->objects, oid, |
| 895 | ODB_HAS_OBJECT_FETCH_PROMISOR); |
| 896 | read_unlock(); |
| 897 | } |
| 898 | |
| 899 | if (collision_test_needed && !data) { |
| 900 | read_lock(); |
| 901 | if (!check_collison(obj_entry)) |
| 902 | collision_test_needed = 0; |
| 903 | read_unlock(); |
| 904 | } |
| 905 | if (collision_test_needed) { |
| 906 | void *has_data; |
| 907 | enum object_type has_type; |
| 908 | size_t has_size; |
| 909 | read_lock(); |
| 910 | has_type = odb_read_object_info(the_repository->objects, oid, &has_size); |
| 911 | if (has_type < 0) |
| 912 | die(_("cannot read existing object info %s"), oid_to_hex(oid)); |
| 913 | if (has_type != type || has_size != size) |
| 914 | die(_("SHA1 COLLISION FOUND WITH %s !"), oid_to_hex(oid)); |
| 915 | has_data = odb_read_object(the_repository->objects, oid, |
| 916 | &has_type, &has_size); |
| 917 | read_unlock(); |
| 918 | if (!data) |
| 919 | data = new_data = get_data_from_pack(obj_entry); |
| 920 | if (!has_data) |
| 921 | die(_("cannot read existing object %s"), oid_to_hex(oid)); |
| 922 | if (size != has_size || type != has_type || |
| 923 | memcmp(data, has_data, size) != 0) |
| 924 | die(_("SHA1 COLLISION FOUND WITH %s !"), oid_to_hex(oid)); |
| 925 | free(has_data); |
| 926 | } |
| 927 | |
| 928 | if (strict || do_fsck_object || record_outgoing_links) { |
| 929 | read_lock(); |
| 930 | if (type == OBJ_BLOB) { |
| 931 | struct blob *blob = lookup_blob(the_repository, oid); |
| 932 | if (blob) |
| 933 | blob->object.flags |= FLAG_CHECKED; |
| 934 | else |
| 935 | die(_("invalid blob object %s"), oid_to_hex(oid)); |
| 936 | if (do_fsck_object && |
| 937 | fsck_object(&blob->object, (void *)data, size, &fsck_options)) |
| 938 | die(_("fsck error in packed object")); |
| 939 | } else { |
| 940 | struct object *obj; |
| 941 | int eaten; |
| 942 | void *buf = (void *) data; |
| 943 | |
| 944 | assert(data && "data can only be NULL for large _blobs_"); |
| 945 | |
| 946 | /* |
| 947 | * we do not need to free the memory here, as the |
| 948 | * buf is deleted by the caller. |
| 949 | */ |
| 950 | obj = parse_object_buffer(the_repository, oid, type, |
| 951 | size, buf, |
| 952 | &eaten); |
| 953 | if (!obj) |
| 954 | die(_("invalid %s"), type_name(type)); |
| 955 | if (do_fsck_object && |
| 956 | fsck_object(obj, buf, size, &fsck_options)) |
| 957 | die(_("fsck error in packed object")); |
| 958 | if (strict && fsck_walk(obj, NULL, &fsck_options)) |
| 959 | die(_("Not all child objects of %s are reachable"), oid_to_hex(&obj->oid)); |
| 960 | if (record_outgoing_links) |
| 961 | do_record_outgoing_links(obj); |
| 962 | |
| 963 | if (obj->type == OBJ_TREE) { |
| 964 | struct tree *item = (struct tree *) obj; |
| 965 | item->buffer = NULL; |
| 966 | obj->parsed = 0; |
| 967 | } |
| 968 | if (obj->type == OBJ_COMMIT) { |
| 969 | struct commit *commit = (struct commit *) obj; |
| 970 | if (detach_commit_buffer(commit, NULL) != data) |
| 971 | BUG("parse_object_buffer transmogrified our buffer"); |
| 972 | } |
| 973 | obj->flags |= FLAG_CHECKED; |
| 974 | } |
| 975 | read_unlock(); |
| 976 | } |
| 977 | |
| 978 | free(new_data); |
| 979 | } |
| 980 | |
| 981 | /* |
| 982 | * Ensure that this node has been reconstructed and return its contents. |
| 983 | * |
| 984 | * In the typical and best case, this node would already be reconstructed |
| 985 | * (through the invocation to resolve_delta() in threaded_second_pass()) and it |
| 986 | * would not be pruned. However, if pruning of this node was necessary due to |
| 987 | * reaching delta_base_cache_limit, this function will find the closest |
| 988 | * ancestor with reconstructed data that has not been pruned (or if there is |
| 989 | * none, the ultimate base object), and reconstruct each node in the delta |
| 990 | * chain in order to generate the reconstructed data for this node. |
| 991 | */ |
| 992 | static void *get_base_data(struct base_data *c) |
| 993 | { |
| 994 | if (!c->data) { |
| 995 | struct object_entry *obj = c->obj; |
| 996 | struct base_data **delta = NULL; |
| 997 | int delta_nr = 0, delta_alloc = 0; |
| 998 | |
| 999 | while (is_delta_type(c->obj->type) && !c->data) { |
| 1000 | ALLOC_GROW(delta, delta_nr + 1, delta_alloc); |
| 1001 | delta[delta_nr++] = c; |
| 1002 | c = c->base; |
| 1003 | } |
| 1004 | if (!delta_nr) { |
| 1005 | c->data = get_data_from_pack(obj); |
| 1006 | c->size = obj->size; |
| 1007 | base_cache_used += c->size; |
| 1008 | prune_base_data(c); |
| 1009 | } |
| 1010 | for (; delta_nr > 0; delta_nr--) { |
| 1011 | void *base, *raw; |
| 1012 | c = delta[delta_nr - 1]; |
| 1013 | obj = c->obj; |
| 1014 | base = get_base_data(c->base); |
| 1015 | raw = get_data_from_pack(obj); |
| 1016 | c->data = patch_delta( |
| 1017 | base, c->base->size, |
| 1018 | raw, obj->size, |
| 1019 | &c->size); |
| 1020 | free(raw); |
| 1021 | if (!c->data) |
| 1022 | bad_object(obj->idx.offset, _("failed to apply delta")); |
| 1023 | base_cache_used += c->size; |
| 1024 | prune_base_data(c); |
| 1025 | } |
| 1026 | free(delta); |
| 1027 | } |
| 1028 | return c->data; |
| 1029 | } |
| 1030 | |
| 1031 | static struct base_data *make_base(struct object_entry *obj, |
| 1032 | struct base_data *parent) |
| 1033 | { |
| 1034 | struct base_data *base = xcalloc(1, sizeof(struct base_data)); |
| 1035 | base->base = parent; |
| 1036 | base->obj = obj; |
| 1037 | find_ref_delta_children(&obj->idx.oid, |
| 1038 | &base->ref_first, &base->ref_last); |
| 1039 | find_ofs_delta_children(obj->idx.offset, |
| 1040 | &base->ofs_first, &base->ofs_last); |
| 1041 | base->children_remaining = base->ref_last - base->ref_first + |
| 1042 | base->ofs_last - base->ofs_first + 2; |
| 1043 | return base; |
| 1044 | } |
| 1045 | |
| 1046 | static struct base_data *resolve_delta(struct object_entry *delta_obj, |
| 1047 | struct base_data *base) |
| 1048 | { |
| 1049 | void *delta_data, *result_data; |
| 1050 | struct base_data *result; |
| 1051 | size_t result_size; |
| 1052 | |
| 1053 | if (show_stat) { |
| 1054 | int i = delta_obj - objects; |
| 1055 | int j = base->obj - objects; |
| 1056 | obj_stat[i].delta_depth = obj_stat[j].delta_depth + 1; |
| 1057 | deepest_delta_lock(); |
| 1058 | if (deepest_delta < obj_stat[i].delta_depth) |
| 1059 | deepest_delta = obj_stat[i].delta_depth; |
| 1060 | deepest_delta_unlock(); |
| 1061 | obj_stat[i].base_object_no = j; |
| 1062 | } |
| 1063 | delta_data = get_data_from_pack(delta_obj); |
| 1064 | assert(base->data); |
| 1065 | result_data = patch_delta(base->data, base->size, |
| 1066 | delta_data, delta_obj->size, &result_size); |
| 1067 | free(delta_data); |
| 1068 | if (!result_data) |
| 1069 | bad_object(delta_obj->idx.offset, _("failed to apply delta")); |
| 1070 | hash_object_file(the_hash_algo, result_data, result_size, |
| 1071 | delta_obj->real_type, &delta_obj->idx.oid); |
| 1072 | sha1_object(result_data, NULL, result_size, delta_obj->real_type, |
| 1073 | &delta_obj->idx.oid); |
| 1074 | |
| 1075 | result = make_base(delta_obj, base); |
| 1076 | result->data = result_data; |
| 1077 | result->size = result_size; |
| 1078 | |
| 1079 | counter_lock(); |
| 1080 | nr_resolved_deltas++; |
| 1081 | counter_unlock(); |
| 1082 | |
| 1083 | return result; |
| 1084 | } |
| 1085 | |
| 1086 | static int compare_ofs_delta_entry(const void *a, const void *b) |
| 1087 | { |
| 1088 | const struct ofs_delta_entry *delta_a = a; |
| 1089 | const struct ofs_delta_entry *delta_b = b; |
| 1090 | |
| 1091 | return delta_a->offset < delta_b->offset ? -1 : |
| 1092 | delta_a->offset > delta_b->offset ? 1 : |
| 1093 | 0; |
| 1094 | } |
| 1095 | |
| 1096 | static int compare_ref_delta_entry(const void *a, const void *b) |
| 1097 | { |
| 1098 | const struct ref_delta_entry *delta_a = a; |
| 1099 | const struct ref_delta_entry *delta_b = b; |
| 1100 | |
| 1101 | return oidcmp(&delta_a->oid, &delta_b->oid); |
| 1102 | } |
| 1103 | |
| 1104 | static void *threaded_second_pass(void *data) |
| 1105 | { |
| 1106 | if (data) |
| 1107 | set_thread_data(data); |
| 1108 | for (;;) { |
| 1109 | struct base_data *parent = NULL; |
| 1110 | struct object_entry *child_obj = NULL; |
| 1111 | struct base_data *child = NULL; |
| 1112 | |
| 1113 | counter_lock(); |
| 1114 | display_progress(progress, nr_resolved_deltas); |
| 1115 | counter_unlock(); |
| 1116 | |
| 1117 | work_lock(); |
| 1118 | if (list_empty(&work_head)) { |
| 1119 | /* |
| 1120 | * Take an object from the object array. |
| 1121 | */ |
| 1122 | while (nr_dispatched < nr_objects && |
| 1123 | is_delta_type(objects[nr_dispatched].type)) |
| 1124 | nr_dispatched++; |
| 1125 | if (nr_dispatched >= nr_objects) { |
| 1126 | work_unlock(); |
| 1127 | break; |
| 1128 | } |
| 1129 | child_obj = &objects[nr_dispatched++]; |
| 1130 | } else { |
| 1131 | /* |
| 1132 | * Peek at the top of the stack, and take a child from |
| 1133 | * it. |
| 1134 | */ |
| 1135 | parent = list_first_entry(&work_head, struct base_data, |
| 1136 | list); |
| 1137 | |
| 1138 | while (parent->ref_first <= parent->ref_last) { |
| 1139 | int offset = ref_deltas[parent->ref_first++].obj_no; |
| 1140 | child_obj = objects + offset; |
| 1141 | if (child_obj->real_type != OBJ_REF_DELTA) { |
| 1142 | child_obj = NULL; |
| 1143 | continue; |
| 1144 | } |
| 1145 | child_obj->real_type = parent->obj->real_type; |
| 1146 | break; |
| 1147 | } |
| 1148 | |
| 1149 | if (!child_obj && parent->ofs_first <= parent->ofs_last) { |
| 1150 | child_obj = objects + |
| 1151 | ofs_deltas[parent->ofs_first++].obj_no; |
| 1152 | assert(child_obj->real_type == OBJ_OFS_DELTA); |
| 1153 | child_obj->real_type = parent->obj->real_type; |
| 1154 | } |
| 1155 | |
| 1156 | if (parent->ref_first > parent->ref_last && |
| 1157 | parent->ofs_first > parent->ofs_last) { |
| 1158 | /* |
| 1159 | * This parent has run out of children, so move |
| 1160 | * it to done_head. |
| 1161 | */ |
| 1162 | list_del(&parent->list); |
| 1163 | list_add(&parent->list, &done_head); |
| 1164 | } |
| 1165 | |
| 1166 | /* |
| 1167 | * Ensure that the parent has data, since we will need |
| 1168 | * it later. |
| 1169 | * |
| 1170 | * NEEDSWORK: If parent data needs to be reloaded, this |
| 1171 | * prolongs the time that the current thread spends in |
| 1172 | * the mutex. A mitigating factor is that parent data |
| 1173 | * needs to be reloaded only if the delta base cache |
| 1174 | * limit is exceeded, so in the typical case, this does |
| 1175 | * not happen. |
| 1176 | */ |
| 1177 | get_base_data(parent); |
| 1178 | parent->retain_data++; |
| 1179 | } |
| 1180 | work_unlock(); |
| 1181 | |
| 1182 | if (child_obj) { |
| 1183 | if (parent) { |
| 1184 | child = resolve_delta(child_obj, parent); |
| 1185 | if (!child->children_remaining) |
| 1186 | FREE_AND_NULL(child->data); |
| 1187 | } else{ |
| 1188 | child = make_base(child_obj, NULL); |
| 1189 | if (child->children_remaining) { |
| 1190 | /* |
| 1191 | * Since this child has its own delta children, |
| 1192 | * we will need this data in the future. |
| 1193 | * Inflate now so that future iterations will |
| 1194 | * have access to this object's data while |
| 1195 | * outside the work mutex. |
| 1196 | */ |
| 1197 | child->data = get_data_from_pack(child_obj); |
| 1198 | child->size = child_obj->size; |
| 1199 | } |
| 1200 | } |
| 1201 | } |
| 1202 | |
| 1203 | work_lock(); |
| 1204 | if (parent) |
| 1205 | parent->retain_data--; |
| 1206 | |
| 1207 | if (child && child->data) { |
| 1208 | /* |
| 1209 | * This child has its own children, so add it to |
| 1210 | * work_head. |
| 1211 | */ |
| 1212 | list_add(&child->list, &work_head); |
| 1213 | base_cache_used += child->size; |
| 1214 | prune_base_data(NULL); |
| 1215 | } else if (child) { |
| 1216 | /* |
| 1217 | * This child does not have its own children. It may be |
| 1218 | * the last descendant of its ancestors; free those |
| 1219 | * that we can. |
| 1220 | */ |
| 1221 | struct base_data *p = parent; |
| 1222 | |
| 1223 | while (p) { |
| 1224 | struct base_data *next_p; |
| 1225 | |
| 1226 | p->children_remaining--; |
| 1227 | if (p->children_remaining) |
| 1228 | break; |
| 1229 | |
| 1230 | next_p = p->base; |
| 1231 | free_base_data(p); |
| 1232 | list_del(&p->list); |
| 1233 | free(p); |
| 1234 | |
| 1235 | p = next_p; |
| 1236 | } |
| 1237 | FREE_AND_NULL(child); |
| 1238 | } |
| 1239 | work_unlock(); |
| 1240 | } |
| 1241 | return NULL; |
| 1242 | } |
| 1243 | |
| 1244 | /* |
| 1245 | * First pass: |
| 1246 | * - find locations of all objects; |
| 1247 | * - calculate SHA1 of all non-delta objects; |
| 1248 | * - remember base (SHA1 or offset) for all deltas. |
| 1249 | */ |
| 1250 | static void parse_pack_objects(unsigned char *hash) |
| 1251 | { |
| 1252 | int i, nr_delays = 0; |
| 1253 | struct ofs_delta_entry *ofs_delta = ofs_deltas; |
| 1254 | struct object_id ref_delta_oid; |
| 1255 | struct stat st; |
| 1256 | struct git_hash_ctx tmp_ctx; |
| 1257 | |
| 1258 | if (verbose) |
| 1259 | progress = start_progress( |
| 1260 | the_repository, |
| 1261 | progress_title ? progress_title : |
| 1262 | from_stdin ? _("Receiving objects") : _("Indexing objects"), |
| 1263 | nr_objects); |
| 1264 | for (i = 0; i < nr_objects; i++) { |
| 1265 | struct object_entry *obj = &objects[i]; |
| 1266 | void *data = unpack_raw_entry(obj, &ofs_delta->offset, |
| 1267 | &ref_delta_oid, |
| 1268 | &obj->idx.oid); |
| 1269 | obj->real_type = obj->type; |
| 1270 | if (obj->type == OBJ_OFS_DELTA) { |
| 1271 | nr_ofs_deltas++; |
| 1272 | ofs_delta->obj_no = i; |
| 1273 | ofs_delta++; |
| 1274 | } else if (obj->type == OBJ_REF_DELTA) { |
| 1275 | ALLOC_GROW(ref_deltas, nr_ref_deltas + 1, ref_deltas_alloc); |
| 1276 | oidcpy(&ref_deltas[nr_ref_deltas].oid, &ref_delta_oid); |
| 1277 | ref_deltas[nr_ref_deltas].obj_no = i; |
| 1278 | nr_ref_deltas++; |
| 1279 | } else if (!data) { |
| 1280 | /* large blobs, check later */ |
| 1281 | obj->real_type = OBJ_BAD; |
| 1282 | nr_delays++; |
| 1283 | } else |
| 1284 | sha1_object(data, NULL, obj->size, obj->type, |
| 1285 | &obj->idx.oid); |
| 1286 | free(data); |
| 1287 | display_progress(progress, i+1); |
| 1288 | } |
| 1289 | objects[i].idx.offset = consumed_bytes; |
| 1290 | stop_progress(&progress); |
| 1291 | |
| 1292 | /* Check pack integrity */ |
| 1293 | flush(); |
| 1294 | the_hash_algo->init_fn(&tmp_ctx); |
| 1295 | git_hash_clone(&tmp_ctx, &input_ctx); |
| 1296 | git_hash_final(hash, &tmp_ctx); |
| 1297 | if (!hasheq(fill(the_hash_algo->rawsz), hash, the_repository->hash_algo)) |
| 1298 | die(_("pack is corrupted (SHA1 mismatch)")); |
| 1299 | use(the_hash_algo->rawsz); |
| 1300 | |
| 1301 | /* If input_fd is a file, we should have reached its end now. */ |
| 1302 | if (fstat(input_fd, &st)) |
| 1303 | die_errno(_("cannot fstat packfile")); |
| 1304 | if (S_ISREG(st.st_mode) && |
| 1305 | lseek(input_fd, 0, SEEK_CUR) - input_len != st.st_size) |
| 1306 | die(_("pack has junk at the end")); |
| 1307 | |
| 1308 | for (i = 0; i < nr_objects; i++) { |
| 1309 | struct object_entry *obj = &objects[i]; |
| 1310 | if (obj->real_type != OBJ_BAD) |
| 1311 | continue; |
| 1312 | obj->real_type = obj->type; |
| 1313 | sha1_object(NULL, obj, obj->size, obj->type, |
| 1314 | &obj->idx.oid); |
| 1315 | nr_delays--; |
| 1316 | } |
| 1317 | if (nr_delays) |
| 1318 | die(_("confusion beyond insanity in parse_pack_objects()")); |
| 1319 | } |
| 1320 | |
| 1321 | /* |
| 1322 | * Second pass: |
| 1323 | * - for all non-delta objects, look if it is used as a base for |
| 1324 | * deltas; |
| 1325 | * - if used as a base, uncompress the object and apply all deltas, |
| 1326 | * recursively checking if the resulting object is used as a base |
| 1327 | * for some more deltas. |
| 1328 | */ |
| 1329 | static void resolve_deltas(struct pack_idx_option *opts) |
| 1330 | { |
| 1331 | int i; |
| 1332 | |
| 1333 | if (!nr_ofs_deltas && !nr_ref_deltas) |
| 1334 | return; |
| 1335 | |
| 1336 | /* Sort deltas by base SHA1/offset for fast searching */ |
| 1337 | QSORT(ofs_deltas, nr_ofs_deltas, compare_ofs_delta_entry); |
| 1338 | QSORT(ref_deltas, nr_ref_deltas, compare_ref_delta_entry); |
| 1339 | |
| 1340 | if (verbose || show_resolving_progress) |
| 1341 | progress = start_progress(the_repository, |
| 1342 | _("Resolving deltas"), |
| 1343 | nr_ref_deltas + nr_ofs_deltas); |
| 1344 | |
| 1345 | nr_dispatched = 0; |
| 1346 | base_cache_limit = opts->delta_base_cache_limit * nr_threads; |
| 1347 | if (nr_threads > 1 || getenv("GIT_FORCE_THREADS")) { |
| 1348 | init_thread(); |
| 1349 | for (i = 0; i < nr_threads; i++) { |
| 1350 | int ret = pthread_create(&thread_data[i].thread, NULL, |
| 1351 | threaded_second_pass, thread_data + i); |
| 1352 | if (ret) |
| 1353 | die(_("unable to create thread: %s"), |
| 1354 | strerror(ret)); |
| 1355 | } |
| 1356 | for (i = 0; i < nr_threads; i++) |
| 1357 | pthread_join(thread_data[i].thread, NULL); |
| 1358 | cleanup_thread(); |
| 1359 | return; |
| 1360 | } |
| 1361 | threaded_second_pass(¬hread_data); |
| 1362 | } |
| 1363 | |
| 1364 | /* |
| 1365 | * Third pass: |
| 1366 | * - append objects to convert thin pack to full pack if required |
| 1367 | * - write the final pack hash |
| 1368 | */ |
| 1369 | static void fix_unresolved_deltas(struct hashfile *f); |
| 1370 | static void conclude_pack(int fix_thin_pack, const char *curr_pack, unsigned char *pack_hash) |
| 1371 | { |
| 1372 | if (nr_ref_deltas + nr_ofs_deltas == nr_resolved_deltas) { |
| 1373 | stop_progress(&progress); |
| 1374 | /* Flush remaining pack final hash. */ |
| 1375 | flush(); |
| 1376 | return; |
| 1377 | } |
| 1378 | |
| 1379 | if (fix_thin_pack) { |
| 1380 | struct hashfile *f; |
| 1381 | unsigned char read_hash[GIT_MAX_RAWSZ], tail_hash[GIT_MAX_RAWSZ]; |
| 1382 | struct strbuf msg = STRBUF_INIT; |
| 1383 | int nr_unresolved = nr_ofs_deltas + nr_ref_deltas - nr_resolved_deltas; |
| 1384 | int nr_objects_initial = nr_objects; |
| 1385 | if (nr_unresolved <= 0) |
| 1386 | die(_("confusion beyond insanity")); |
| 1387 | REALLOC_ARRAY(objects, nr_objects + nr_unresolved + 1); |
| 1388 | memset(objects + nr_objects + 1, 0, |
| 1389 | nr_unresolved * sizeof(*objects)); |
| 1390 | f = hashfd(the_repository->hash_algo, output_fd, curr_pack); |
| 1391 | fix_unresolved_deltas(f); |
| 1392 | strbuf_addf(&msg, Q_("completed with %d local object", |
| 1393 | "completed with %d local objects", |
| 1394 | nr_objects - nr_objects_initial), |
| 1395 | nr_objects - nr_objects_initial); |
| 1396 | stop_progress_msg(&progress, msg.buf); |
| 1397 | strbuf_release(&msg); |
| 1398 | finalize_hashfile(f, tail_hash, FSYNC_COMPONENT_PACK, 0); |
| 1399 | hashcpy(read_hash, pack_hash, the_repository->hash_algo); |
| 1400 | fixup_pack_header_footer(the_hash_algo, output_fd, pack_hash, |
| 1401 | curr_pack, nr_objects, |
| 1402 | read_hash, consumed_bytes-the_hash_algo->rawsz); |
| 1403 | if (!hasheq(read_hash, tail_hash, the_repository->hash_algo)) |
| 1404 | die(_("Unexpected tail checksum for %s " |
| 1405 | "(disk corruption?)"), curr_pack); |
| 1406 | } |
| 1407 | if (nr_ofs_deltas + nr_ref_deltas != nr_resolved_deltas) |
| 1408 | die(Q_("pack has %d unresolved delta", |
| 1409 | "pack has %d unresolved deltas", |
| 1410 | nr_ofs_deltas + nr_ref_deltas - nr_resolved_deltas), |
| 1411 | nr_ofs_deltas + nr_ref_deltas - nr_resolved_deltas); |
| 1412 | } |
| 1413 | |
| 1414 | static int write_compressed(struct hashfile *f, void *in, unsigned int size) |
| 1415 | { |
| 1416 | git_zstream stream; |
| 1417 | int status; |
| 1418 | unsigned char outbuf[4096]; |
| 1419 | struct repo_config_values *cfg = repo_config_values(the_repository); |
| 1420 | |
| 1421 | git_deflate_init(&stream, cfg->zlib_compression_level); |
| 1422 | stream.next_in = in; |
| 1423 | stream.avail_in = size; |
| 1424 | |
| 1425 | do { |
| 1426 | stream.next_out = outbuf; |
| 1427 | stream.avail_out = sizeof(outbuf); |
| 1428 | status = git_deflate(&stream, Z_FINISH); |
| 1429 | hashwrite(f, outbuf, sizeof(outbuf) - stream.avail_out); |
| 1430 | } while (status == Z_OK); |
| 1431 | |
| 1432 | if (status != Z_STREAM_END) |
| 1433 | die(_("unable to deflate appended object (%d)"), status); |
| 1434 | size = stream.total_out; |
| 1435 | git_deflate_end(&stream); |
| 1436 | return size; |
| 1437 | } |
| 1438 | |
| 1439 | static struct object_entry *append_obj_to_pack(struct hashfile *f, |
| 1440 | const unsigned char *sha1, void *buf, |
| 1441 | unsigned long size, enum object_type type) |
| 1442 | { |
| 1443 | struct object_entry *obj = &objects[nr_objects++]; |
| 1444 | unsigned char header[10]; |
| 1445 | unsigned long s = size; |
| 1446 | int n = 0; |
| 1447 | unsigned char c = (type << 4) | (s & 15); |
| 1448 | s >>= 4; |
| 1449 | while (s) { |
| 1450 | header[n++] = c | 0x80; |
| 1451 | c = s & 0x7f; |
| 1452 | s >>= 7; |
| 1453 | } |
| 1454 | header[n++] = c; |
| 1455 | crc32_begin(f); |
| 1456 | hashwrite(f, header, n); |
| 1457 | obj[0].size = size; |
| 1458 | obj[0].hdr_size = n; |
| 1459 | obj[0].type = type; |
| 1460 | obj[0].real_type = type; |
| 1461 | obj[1].idx.offset = obj[0].idx.offset + n; |
| 1462 | obj[1].idx.offset += write_compressed(f, buf, size); |
| 1463 | obj[0].idx.crc32 = crc32_end(f); |
| 1464 | hashflush(f); |
| 1465 | oidread(&obj->idx.oid, sha1, the_repository->hash_algo); |
| 1466 | return obj; |
| 1467 | } |
| 1468 | |
| 1469 | static int delta_pos_compare(const void *_a, const void *_b) |
| 1470 | { |
| 1471 | struct ref_delta_entry *a = *(struct ref_delta_entry **)_a; |
| 1472 | struct ref_delta_entry *b = *(struct ref_delta_entry **)_b; |
| 1473 | return a->obj_no - b->obj_no; |
| 1474 | } |
| 1475 | |
| 1476 | static void fix_unresolved_deltas(struct hashfile *f) |
| 1477 | { |
| 1478 | struct ref_delta_entry **sorted_by_pos; |
| 1479 | int i; |
| 1480 | |
| 1481 | /* |
| 1482 | * Since many unresolved deltas may well be themselves base objects |
| 1483 | * for more unresolved deltas, we really want to include the |
| 1484 | * smallest number of base objects that would cover as much delta |
| 1485 | * as possible by picking the |
| 1486 | * trunc deltas first, allowing for other deltas to resolve without |
| 1487 | * additional base objects. Since most base objects are to be found |
| 1488 | * before deltas depending on them, a good heuristic is to start |
| 1489 | * resolving deltas in the same order as their position in the pack. |
| 1490 | */ |
| 1491 | ALLOC_ARRAY(sorted_by_pos, nr_ref_deltas); |
| 1492 | for (i = 0; i < nr_ref_deltas; i++) |
| 1493 | sorted_by_pos[i] = &ref_deltas[i]; |
| 1494 | QSORT(sorted_by_pos, nr_ref_deltas, delta_pos_compare); |
| 1495 | |
| 1496 | if (repo_has_promisor_remote(the_repository)) { |
| 1497 | /* |
| 1498 | * Prefetch the delta bases. |
| 1499 | */ |
| 1500 | struct oid_array to_fetch = OID_ARRAY_INIT; |
| 1501 | for (i = 0; i < nr_ref_deltas; i++) { |
| 1502 | struct ref_delta_entry *d = sorted_by_pos[i]; |
| 1503 | if (!odb_read_object_info_extended(the_repository->objects, |
| 1504 | &d->oid, NULL, |
| 1505 | OBJECT_INFO_FOR_PREFETCH)) |
| 1506 | continue; |
| 1507 | oid_array_append(&to_fetch, &d->oid); |
| 1508 | } |
| 1509 | promisor_remote_get_direct(the_repository, |
| 1510 | to_fetch.oid, to_fetch.nr); |
| 1511 | oid_array_clear(&to_fetch); |
| 1512 | } |
| 1513 | |
| 1514 | for (i = 0; i < nr_ref_deltas; i++) { |
| 1515 | struct ref_delta_entry *d = sorted_by_pos[i]; |
| 1516 | enum object_type type; |
| 1517 | void *data; |
| 1518 | size_t size; |
| 1519 | |
| 1520 | if (objects[d->obj_no].real_type != OBJ_REF_DELTA) |
| 1521 | continue; |
| 1522 | data = odb_read_object(the_repository->objects, &d->oid, |
| 1523 | &type, &size); |
| 1524 | if (!data) |
| 1525 | continue; |
| 1526 | |
| 1527 | if (check_object_signature(the_repository, &d->oid, data, size, |
| 1528 | type) < 0) |
| 1529 | die(_("local object %s is corrupt"), oid_to_hex(&d->oid)); |
| 1530 | |
| 1531 | /* |
| 1532 | * Add this as an object to the objects array and call |
| 1533 | * threaded_second_pass() (which will pick up the added |
| 1534 | * object). |
| 1535 | */ |
| 1536 | append_obj_to_pack(f, d->oid.hash, data, size, type); |
| 1537 | free(data); |
| 1538 | threaded_second_pass(NULL); |
| 1539 | |
| 1540 | display_progress(progress, nr_resolved_deltas); |
| 1541 | } |
| 1542 | free(sorted_by_pos); |
| 1543 | } |
| 1544 | |
| 1545 | static const char *derive_filename(const char *pack_name, const char *strip, |
| 1546 | const char *suffix, struct strbuf *buf) |
| 1547 | { |
| 1548 | size_t len; |
| 1549 | if (!strip_suffix(pack_name, strip, &len) || !len || |
| 1550 | pack_name[len - 1] != '.') |
| 1551 | die(_("packfile name '%s' does not end with '.%s'"), |
| 1552 | pack_name, strip); |
| 1553 | strbuf_add(buf, pack_name, len); |
| 1554 | strbuf_addstr(buf, suffix); |
| 1555 | return buf->buf; |
| 1556 | } |
| 1557 | |
| 1558 | static void write_special_file(const char *suffix, const char *msg, |
| 1559 | const char *pack_name, const unsigned char *hash, |
| 1560 | const char **report) |
| 1561 | { |
| 1562 | struct strbuf name_buf = STRBUF_INIT; |
| 1563 | const char *filename; |
| 1564 | int fd; |
| 1565 | int msg_len = strlen(msg); |
| 1566 | |
| 1567 | if (pack_name) |
| 1568 | filename = derive_filename(pack_name, "pack", suffix, &name_buf); |
| 1569 | else |
| 1570 | filename = odb_pack_name(the_repository, &name_buf, hash, suffix); |
| 1571 | |
| 1572 | fd = safe_create_file_with_leading_directories(the_repository, filename); |
| 1573 | if (fd < 0) { |
| 1574 | if (errno != EEXIST) |
| 1575 | die_errno(_("cannot write %s file '%s'"), |
| 1576 | suffix, filename); |
| 1577 | } else { |
| 1578 | if (msg_len > 0) { |
| 1579 | write_or_die(fd, msg, msg_len); |
| 1580 | write_or_die(fd, "\n", 1); |
| 1581 | } |
| 1582 | if (close(fd) != 0) |
| 1583 | die_errno(_("cannot close written %s file '%s'"), |
| 1584 | suffix, filename); |
| 1585 | if (report) |
| 1586 | *report = suffix; |
| 1587 | } |
| 1588 | strbuf_release(&name_buf); |
| 1589 | } |
| 1590 | |
| 1591 | static void rename_tmp_packfile(const char **final_name, |
| 1592 | const char *curr_name, |
| 1593 | struct strbuf *name, unsigned char *hash, |
| 1594 | const char *ext, int make_read_only_if_same) |
| 1595 | { |
| 1596 | if (!*final_name || strcmp(*final_name, curr_name)) { |
| 1597 | if (!*final_name) |
| 1598 | *final_name = odb_pack_name(the_repository, name, hash, ext); |
| 1599 | if (finalize_object_file(the_repository, curr_name, *final_name)) |
| 1600 | die(_("unable to rename temporary '*.%s' file to '%s'"), |
| 1601 | ext, *final_name); |
| 1602 | } else if (make_read_only_if_same) { |
| 1603 | chmod(*final_name, 0444); |
| 1604 | } |
| 1605 | } |
| 1606 | |
| 1607 | static void final(const char *final_pack_name, const char *curr_pack_name, |
| 1608 | const char *final_index_name, const char *curr_index_name, |
| 1609 | const char *final_rev_index_name, const char *curr_rev_index_name, |
| 1610 | const char *keep_msg, const char *promisor_msg, |
| 1611 | unsigned char *hash) |
| 1612 | { |
| 1613 | const char *report = "pack"; |
| 1614 | struct strbuf pack_name = STRBUF_INIT; |
| 1615 | struct strbuf index_name = STRBUF_INIT; |
| 1616 | struct strbuf rev_index_name = STRBUF_INIT; |
| 1617 | |
| 1618 | if (!from_stdin) { |
| 1619 | close(input_fd); |
| 1620 | } else { |
| 1621 | fsync_component_or_die(FSYNC_COMPONENT_PACK, output_fd, curr_pack_name); |
| 1622 | if (close(output_fd)) |
| 1623 | die_errno(_("error while closing pack file")); |
| 1624 | } |
| 1625 | |
| 1626 | if (keep_msg) |
| 1627 | write_special_file("keep", keep_msg, final_pack_name, hash, |
| 1628 | &report); |
| 1629 | if (promisor_msg) |
| 1630 | write_special_file("promisor", promisor_msg, final_pack_name, |
| 1631 | hash, NULL); |
| 1632 | |
| 1633 | rename_tmp_packfile(&final_pack_name, curr_pack_name, &pack_name, |
| 1634 | hash, "pack", from_stdin); |
| 1635 | if (curr_rev_index_name) |
| 1636 | rename_tmp_packfile(&final_rev_index_name, curr_rev_index_name, |
| 1637 | &rev_index_name, hash, "rev", 1); |
| 1638 | rename_tmp_packfile(&final_index_name, curr_index_name, &index_name, |
| 1639 | hash, "idx", 1); |
| 1640 | |
| 1641 | if (do_fsck_object && startup_info->have_repository) { |
| 1642 | struct odb_source_files *files = |
| 1643 | odb_source_files_downcast(the_repository->objects->sources); |
| 1644 | packfile_store_load_pack(files->packed, final_index_name, 0); |
| 1645 | } |
| 1646 | |
| 1647 | if (!from_stdin) { |
| 1648 | printf("%s\n", hash_to_hex(hash)); |
| 1649 | } else { |
| 1650 | struct strbuf buf = STRBUF_INIT; |
| 1651 | |
| 1652 | strbuf_addf(&buf, "%s\t%s\n", report, hash_to_hex(hash)); |
| 1653 | write_or_die(1, buf.buf, buf.len); |
| 1654 | strbuf_release(&buf); |
| 1655 | |
| 1656 | /* Write the last part of the buffer to stdout */ |
| 1657 | write_in_full(1, input_buffer + input_offset, input_len); |
| 1658 | } |
| 1659 | |
| 1660 | strbuf_release(&rev_index_name); |
| 1661 | strbuf_release(&index_name); |
| 1662 | strbuf_release(&pack_name); |
| 1663 | } |
| 1664 | |
| 1665 | static int git_index_pack_config(const char *k, const char *v, |
| 1666 | const struct config_context *ctx, void *cb) |
| 1667 | { |
| 1668 | struct pack_idx_option *opts = cb; |
| 1669 | |
| 1670 | if (!strcmp(k, "pack.indexversion")) { |
| 1671 | opts->version = git_config_int(k, v, ctx->kvi); |
| 1672 | if (opts->version > 2) |
| 1673 | die(_("bad pack.indexVersion=%"PRIu32), opts->version); |
| 1674 | return 0; |
| 1675 | } |
| 1676 | if (!strcmp(k, "pack.threads")) { |
| 1677 | nr_threads = git_config_int(k, v, ctx->kvi); |
| 1678 | if (nr_threads < 0) |
| 1679 | die(_("invalid number of threads specified (%d)"), |
| 1680 | nr_threads); |
| 1681 | if (!HAVE_THREADS && nr_threads != 1) { |
| 1682 | warning(_("no threads support, ignoring %s"), k); |
| 1683 | nr_threads = 1; |
| 1684 | } |
| 1685 | return 0; |
| 1686 | } |
| 1687 | if (!strcmp(k, "pack.writereverseindex")) { |
| 1688 | if (git_config_bool(k, v)) |
| 1689 | opts->flags |= WRITE_REV; |
| 1690 | else |
| 1691 | opts->flags &= ~WRITE_REV; |
| 1692 | } |
| 1693 | if (!strcmp(k, "core.deltabasecachelimit")) { |
| 1694 | opts->delta_base_cache_limit = git_config_ulong(k, v, ctx->kvi); |
| 1695 | return 0; |
| 1696 | } |
| 1697 | return git_default_config(k, v, ctx, cb); |
| 1698 | } |
| 1699 | |
| 1700 | static int cmp_uint32(const void *a_, const void *b_) |
| 1701 | { |
| 1702 | uint32_t a = *((uint32_t *)a_); |
| 1703 | uint32_t b = *((uint32_t *)b_); |
| 1704 | |
| 1705 | return (a < b) ? -1 : (a != b); |
| 1706 | } |
| 1707 | |
| 1708 | static void read_v2_anomalous_offsets(struct packed_git *p, |
| 1709 | struct pack_idx_option *opts) |
| 1710 | { |
| 1711 | const uint32_t *idx1, *idx2; |
| 1712 | uint32_t i; |
| 1713 | |
| 1714 | /* The address of the 4-byte offset table */ |
| 1715 | idx1 = (((const uint32_t *)((const uint8_t *)p->index_data + p->crc_offset)) |
| 1716 | + (size_t)p->num_objects /* CRC32 table */ |
| 1717 | ); |
| 1718 | |
| 1719 | /* The address of the 8-byte offset table */ |
| 1720 | idx2 = idx1 + p->num_objects; |
| 1721 | |
| 1722 | for (i = 0; i < p->num_objects; i++) { |
| 1723 | uint32_t off = ntohl(idx1[i]); |
| 1724 | if (!(off & 0x80000000)) |
| 1725 | continue; |
| 1726 | off = off & 0x7fffffff; |
| 1727 | check_pack_index_ptr(p, &idx2[off * 2]); |
| 1728 | if (idx2[off * 2]) |
| 1729 | continue; |
| 1730 | /* |
| 1731 | * The real offset is ntohl(idx2[off * 2]) in high 4 |
| 1732 | * octets, and ntohl(idx2[off * 2 + 1]) in low 4 |
| 1733 | * octets. But idx2[off * 2] is Zero!!! |
| 1734 | */ |
| 1735 | ALLOC_GROW(opts->anomaly, opts->anomaly_nr + 1, opts->anomaly_alloc); |
| 1736 | opts->anomaly[opts->anomaly_nr++] = ntohl(idx2[off * 2 + 1]); |
| 1737 | } |
| 1738 | |
| 1739 | QSORT(opts->anomaly, opts->anomaly_nr, cmp_uint32); |
| 1740 | } |
| 1741 | |
| 1742 | static void read_idx_option(struct pack_idx_option *opts, const char *pack_name) |
| 1743 | { |
| 1744 | struct packed_git *p = add_packed_git(the_repository, pack_name, |
| 1745 | strlen(pack_name), 1); |
| 1746 | |
| 1747 | if (!p) |
| 1748 | die(_("Cannot open existing pack file '%s'"), pack_name); |
| 1749 | if (open_pack_index(p)) |
| 1750 | die(_("Cannot open existing pack idx file for '%s'"), pack_name); |
| 1751 | |
| 1752 | /* Read the attributes from the existing idx file */ |
| 1753 | opts->version = p->index_version; |
| 1754 | |
| 1755 | if (opts->version == 2) |
| 1756 | read_v2_anomalous_offsets(p, opts); |
| 1757 | |
| 1758 | /* |
| 1759 | * Get rid of the idx file as we do not need it anymore. |
| 1760 | * NEEDSWORK: extract this bit from free_pack_by_name() in |
| 1761 | * object-file.c, perhaps? It shouldn't matter very much as we |
| 1762 | * know we haven't installed this pack (hence we never have |
| 1763 | * read anything from it). |
| 1764 | */ |
| 1765 | close_pack_index(p); |
| 1766 | free(p); |
| 1767 | } |
| 1768 | |
| 1769 | static void show_pack_info(int stat_only) |
| 1770 | { |
| 1771 | int i, baseobjects = nr_objects - nr_ref_deltas - nr_ofs_deltas; |
| 1772 | unsigned long *chain_histogram = NULL; |
| 1773 | |
| 1774 | if (deepest_delta) |
| 1775 | CALLOC_ARRAY(chain_histogram, deepest_delta); |
| 1776 | |
| 1777 | for (i = 0; i < nr_objects; i++) { |
| 1778 | struct object_entry *obj = &objects[i]; |
| 1779 | |
| 1780 | if (is_delta_type(obj->type)) |
| 1781 | chain_histogram[obj_stat[i].delta_depth - 1]++; |
| 1782 | if (stat_only) |
| 1783 | continue; |
| 1784 | printf("%s %-6s %"PRIuMAX" %"PRIuMAX" %"PRIuMAX, |
| 1785 | oid_to_hex(&obj->idx.oid), |
| 1786 | type_name(obj->real_type), (uintmax_t)obj->size, |
| 1787 | (uintmax_t)(obj[1].idx.offset - obj->idx.offset), |
| 1788 | (uintmax_t)obj->idx.offset); |
| 1789 | if (is_delta_type(obj->type)) { |
| 1790 | struct object_entry *bobj = &objects[obj_stat[i].base_object_no]; |
| 1791 | printf(" %u %s", obj_stat[i].delta_depth, |
| 1792 | oid_to_hex(&bobj->idx.oid)); |
| 1793 | } |
| 1794 | putchar('\n'); |
| 1795 | } |
| 1796 | |
| 1797 | if (baseobjects) |
| 1798 | printf_ln(Q_("non delta: %d object", |
| 1799 | "non delta: %d objects", |
| 1800 | baseobjects), |
| 1801 | baseobjects); |
| 1802 | for (i = 0; i < deepest_delta; i++) { |
| 1803 | if (!chain_histogram[i]) |
| 1804 | continue; |
| 1805 | printf_ln(Q_("chain length = %d: %lu object", |
| 1806 | "chain length = %d: %lu objects", |
| 1807 | chain_histogram[i]), |
| 1808 | i + 1, |
| 1809 | chain_histogram[i]); |
| 1810 | } |
| 1811 | free(chain_histogram); |
| 1812 | } |
| 1813 | |
| 1814 | static void repack_local_links(void) |
| 1815 | { |
| 1816 | struct child_process cmd = CHILD_PROCESS_INIT; |
| 1817 | FILE *out; |
| 1818 | struct strbuf line = STRBUF_INIT; |
| 1819 | struct oidset_iter iter; |
| 1820 | struct object_id *oid; |
| 1821 | char *base_name = NULL; |
| 1822 | |
| 1823 | if (!oidset_size(&outgoing_links)) |
| 1824 | return; |
| 1825 | |
| 1826 | oidset_iter_init(&outgoing_links, &iter); |
| 1827 | while ((oid = oidset_iter_next(&iter))) { |
| 1828 | struct object_info info = OBJECT_INFO_INIT; |
| 1829 | if (odb_read_object_info_extended(the_repository->objects, oid, &info, 0)) |
| 1830 | /* Missing; assume it is a promisor object */ |
| 1831 | continue; |
| 1832 | if (info.whence == OI_PACKED && info.u.packed.pack->pack_promisor) |
| 1833 | continue; |
| 1834 | |
| 1835 | if (!cmd.args.nr) { |
| 1836 | base_name = mkpathdup( |
| 1837 | "%s/pack/pack", |
| 1838 | repo_get_object_directory(the_repository)); |
| 1839 | strvec_push(&cmd.args, "pack-objects"); |
| 1840 | strvec_push(&cmd.args, |
| 1841 | "--exclude-promisor-objects-best-effort"); |
| 1842 | strvec_push(&cmd.args, base_name); |
| 1843 | cmd.git_cmd = 1; |
| 1844 | cmd.in = -1; |
| 1845 | cmd.out = -1; |
| 1846 | if (start_command(&cmd)) |
| 1847 | die(_("could not start pack-objects to repack local links")); |
| 1848 | } |
| 1849 | |
| 1850 | if (write_in_full(cmd.in, oid_to_hex(oid), the_hash_algo->hexsz) < 0 || |
| 1851 | write_in_full(cmd.in, "\n", 1) < 0) |
| 1852 | die(_("failed to feed local object to pack-objects")); |
| 1853 | } |
| 1854 | |
| 1855 | if (!cmd.args.nr) |
| 1856 | return; |
| 1857 | |
| 1858 | close(cmd.in); |
| 1859 | |
| 1860 | out = xfdopen(cmd.out, "r"); |
| 1861 | while (strbuf_getline_lf(&line, out) != EOF) { |
| 1862 | unsigned char binary[GIT_MAX_RAWSZ]; |
| 1863 | if (line.len != the_hash_algo->hexsz || |
| 1864 | !hex_to_bytes(binary, line.buf, line.len)) |
| 1865 | die(_("index-pack: Expecting full hex object ID lines only from pack-objects.")); |
| 1866 | |
| 1867 | /* |
| 1868 | * pack-objects creates the .pack and .idx files, but not the |
| 1869 | * .promisor file. Create the .promisor file, which is empty. |
| 1870 | */ |
| 1871 | write_special_file("promisor", "", NULL, binary, NULL); |
| 1872 | } |
| 1873 | |
| 1874 | fclose(out); |
| 1875 | if (finish_command(&cmd)) |
| 1876 | die(_("could not finish pack-objects to repack local links")); |
| 1877 | strbuf_release(&line); |
| 1878 | free(base_name); |
| 1879 | } |
| 1880 | |
| 1881 | int cmd_index_pack(int argc, |
| 1882 | const char **argv, |
| 1883 | const char *prefix, |
| 1884 | struct repository *repo UNUSED) |
| 1885 | { |
| 1886 | int i, fix_thin_pack = 0, verify = 0, stat_only = 0, rev_index; |
| 1887 | const char *curr_index; |
| 1888 | char *curr_rev_index = NULL; |
| 1889 | const char *index_name = NULL, *pack_name = NULL, *rev_index_name = NULL; |
| 1890 | const char *keep_msg = NULL; |
| 1891 | const char *promisor_msg = NULL; |
| 1892 | struct strbuf index_name_buf = STRBUF_INIT; |
| 1893 | struct strbuf rev_index_name_buf = STRBUF_INIT; |
| 1894 | struct pack_idx_entry **idx_objects; |
| 1895 | struct pack_idx_option opts; |
| 1896 | unsigned char pack_hash[GIT_MAX_RAWSZ]; |
| 1897 | unsigned foreign_nr = 1; /* zero is a "good" value, assume bad */ |
| 1898 | int report_end_of_input = 0; |
| 1899 | int hash_algo = 0; |
| 1900 | |
| 1901 | /* |
| 1902 | * index-pack never needs to fetch missing objects except when |
| 1903 | * REF_DELTA bases are missing (which are explicitly handled). It only |
| 1904 | * accesses the repo to do hash collision checks and to check which |
| 1905 | * REF_DELTA bases need to be fetched. |
| 1906 | */ |
| 1907 | fetch_if_missing = 0; |
| 1908 | |
| 1909 | show_usage_if_asked(argc, argv, index_pack_usage); |
| 1910 | |
| 1911 | disable_replace_refs(); |
| 1912 | |
| 1913 | fsck_options_init(&fsck_options, the_repository, FSCK_OPTIONS_MISSING_GITMODULES); |
| 1914 | fsck_options.walk = mark_link; |
| 1915 | |
| 1916 | reset_pack_idx_option(&opts); |
| 1917 | opts.flags |= WRITE_REV; |
| 1918 | repo_config(the_repository, git_index_pack_config, &opts); |
| 1919 | if (prefix && chdir(prefix)) |
| 1920 | die(_("Cannot come back to cwd")); |
| 1921 | |
| 1922 | if (git_env_bool(GIT_TEST_NO_WRITE_REV_INDEX, 0)) |
| 1923 | rev_index = 0; |
| 1924 | else |
| 1925 | rev_index = !!(opts.flags & (WRITE_REV_VERIFY | WRITE_REV)); |
| 1926 | |
| 1927 | for (i = 1; i < argc; i++) { |
| 1928 | const char *arg = argv[i]; |
| 1929 | |
| 1930 | if (*arg == '-') { |
| 1931 | if (!strcmp(arg, "--stdin")) { |
| 1932 | from_stdin = 1; |
| 1933 | } else if (!strcmp(arg, "--fix-thin")) { |
| 1934 | fix_thin_pack = 1; |
| 1935 | } else if (skip_to_optional_arg(arg, "--strict", &arg)) { |
| 1936 | strict = 1; |
| 1937 | do_fsck_object = 1; |
| 1938 | fsck_set_msg_types(&fsck_options, arg); |
| 1939 | } else if (!strcmp(arg, "--check-self-contained-and-connected")) { |
| 1940 | strict = 1; |
| 1941 | check_self_contained_and_connected = 1; |
| 1942 | } else if (skip_to_optional_arg(arg, "--fsck-objects", &arg)) { |
| 1943 | do_fsck_object = 1; |
| 1944 | fsck_set_msg_types(&fsck_options, arg); |
| 1945 | } else if (!strcmp(arg, "--verify")) { |
| 1946 | verify = 1; |
| 1947 | } else if (!strcmp(arg, "--verify-stat")) { |
| 1948 | verify = 1; |
| 1949 | show_stat = 1; |
| 1950 | } else if (!strcmp(arg, "--verify-stat-only")) { |
| 1951 | verify = 1; |
| 1952 | show_stat = 1; |
| 1953 | stat_only = 1; |
| 1954 | } else if (skip_to_optional_arg(arg, "--keep", &keep_msg)) { |
| 1955 | ; /* nothing to do */ |
| 1956 | } else if (skip_to_optional_arg(arg, "--promisor", &promisor_msg)) { |
| 1957 | record_outgoing_links = 1; |
| 1958 | } else if (starts_with(arg, "--threads=")) { |
| 1959 | char *end; |
| 1960 | nr_threads = strtoul(arg+10, &end, 0); |
| 1961 | if (!arg[10] || *end || nr_threads < 0) |
| 1962 | usage(index_pack_usage); |
| 1963 | if (!HAVE_THREADS && nr_threads != 1) { |
| 1964 | warning(_("no threads support, ignoring %s"), arg); |
| 1965 | nr_threads = 1; |
| 1966 | } |
| 1967 | } else if (skip_prefix(arg, "--pack_header=", &arg)) { |
| 1968 | if (parse_pack_header_option(arg, |
| 1969 | input_buffer, |
| 1970 | &input_len) < 0) |
| 1971 | die(_("bad --pack_header: %s"), arg); |
| 1972 | } else if (!strcmp(arg, "-v")) { |
| 1973 | verbose = 1; |
| 1974 | } else if (!strcmp(arg, "--progress-title")) { |
| 1975 | if (progress_title || (i+1) >= argc) |
| 1976 | usage(index_pack_usage); |
| 1977 | progress_title = argv[++i]; |
| 1978 | } else if (!strcmp(arg, "--show-resolving-progress")) { |
| 1979 | show_resolving_progress = 1; |
| 1980 | } else if (!strcmp(arg, "--report-end-of-input")) { |
| 1981 | report_end_of_input = 1; |
| 1982 | } else if (!strcmp(arg, "-o")) { |
| 1983 | if (index_name || (i+1) >= argc) |
| 1984 | usage(index_pack_usage); |
| 1985 | index_name = argv[++i]; |
| 1986 | } else if (starts_with(arg, "--index-version=")) { |
| 1987 | char *c; |
| 1988 | opts.version = strtoul(arg + 16, &c, 10); |
| 1989 | if (opts.version > 2) |
| 1990 | die(_("bad %s"), arg); |
| 1991 | if (*c == ',') |
| 1992 | opts.off32_limit = strtoul(c+1, &c, 0); |
| 1993 | if (*c || opts.off32_limit & 0x80000000) |
| 1994 | die(_("bad %s"), arg); |
| 1995 | } else if (skip_prefix(arg, "--max-input-size=", &arg)) { |
| 1996 | max_input_size = strtoumax(arg, NULL, 10); |
| 1997 | } else if (skip_prefix(arg, "--object-format=", &arg)) { |
| 1998 | hash_algo = hash_algo_by_name(arg); |
| 1999 | if (hash_algo == GIT_HASH_UNKNOWN) |
| 2000 | die(_("unknown hash algorithm '%s'"), arg); |
| 2001 | repo_set_hash_algo(the_repository, hash_algo); |
| 2002 | } else if (!strcmp(arg, "--rev-index")) { |
| 2003 | rev_index = 1; |
| 2004 | } else if (!strcmp(arg, "--no-rev-index")) { |
| 2005 | rev_index = 0; |
| 2006 | } else |
| 2007 | usage(index_pack_usage); |
| 2008 | continue; |
| 2009 | } |
| 2010 | |
| 2011 | if (pack_name) |
| 2012 | usage(index_pack_usage); |
| 2013 | pack_name = arg; |
| 2014 | } |
| 2015 | |
| 2016 | if (!pack_name && !from_stdin) |
| 2017 | usage(index_pack_usage); |
| 2018 | if (fix_thin_pack && !from_stdin) |
| 2019 | die(_("the option '%s' requires '%s'"), "--fix-thin", "--stdin"); |
| 2020 | if (promisor_msg && pack_name) |
| 2021 | die(_("--promisor cannot be used with a pack name")); |
| 2022 | if (from_stdin && !startup_info->have_repository) |
| 2023 | die(_("--stdin requires a git repository")); |
| 2024 | if (from_stdin && hash_algo) |
| 2025 | die(_("options '%s' and '%s' cannot be used together"), "--object-format", "--stdin"); |
| 2026 | if (!index_name && pack_name) |
| 2027 | index_name = derive_filename(pack_name, "pack", "idx", &index_name_buf); |
| 2028 | |
| 2029 | /* |
| 2030 | * Packfiles and indices do not carry enough information to be able to |
| 2031 | * identify their object hash. So when we are neither in a repository |
| 2032 | * nor has the user told us which object hash to use we have no other |
| 2033 | * choice but to guess the object hash. |
| 2034 | */ |
| 2035 | if (!the_repository->hash_algo) |
| 2036 | repo_set_hash_algo(the_repository, GIT_HASH_DEFAULT); |
| 2037 | |
| 2038 | opts.flags &= ~(WRITE_REV | WRITE_REV_VERIFY); |
| 2039 | if (rev_index) { |
| 2040 | opts.flags |= verify ? WRITE_REV_VERIFY : WRITE_REV; |
| 2041 | if (index_name) |
| 2042 | rev_index_name = derive_filename(index_name, |
| 2043 | "idx", "rev", |
| 2044 | &rev_index_name_buf); |
| 2045 | } |
| 2046 | |
| 2047 | if (verify) { |
| 2048 | if (!index_name) |
| 2049 | die(_("--verify with no packfile name given")); |
| 2050 | read_idx_option(&opts, index_name); |
| 2051 | opts.flags |= WRITE_IDX_VERIFY | WRITE_IDX_STRICT; |
| 2052 | } |
| 2053 | if (strict) |
| 2054 | opts.flags |= WRITE_IDX_STRICT; |
| 2055 | |
| 2056 | if (HAVE_THREADS && !nr_threads) { |
| 2057 | nr_threads = online_cpus(); |
| 2058 | /* |
| 2059 | * Experiments show that going above 20 threads doesn't help, |
| 2060 | * no matter how many cores you have. Below that, we tend to |
| 2061 | * max at half the number of online_cpus(), presumably because |
| 2062 | * half of those are hyperthreads rather than full cores. We'll |
| 2063 | * never reduce the level below "3", though, to match a |
| 2064 | * historical value that nobody complained about. |
| 2065 | */ |
| 2066 | if (nr_threads < 4) |
| 2067 | ; /* too few cores to consider capping */ |
| 2068 | else if (nr_threads < 6) |
| 2069 | nr_threads = 3; /* historic cap */ |
| 2070 | else if (nr_threads < 40) |
| 2071 | nr_threads /= 2; |
| 2072 | else |
| 2073 | nr_threads = 20; /* hard cap */ |
| 2074 | } |
| 2075 | |
| 2076 | curr_pack = open_pack_file(pack_name); |
| 2077 | parse_pack_header(); |
| 2078 | CALLOC_ARRAY(objects, st_add(nr_objects, 1)); |
| 2079 | if (show_stat) |
| 2080 | CALLOC_ARRAY(obj_stat, st_add(nr_objects, 1)); |
| 2081 | CALLOC_ARRAY(ofs_deltas, nr_objects); |
| 2082 | parse_pack_objects(pack_hash); |
| 2083 | if (report_end_of_input) |
| 2084 | write_in_full(2, "\0", 1); |
| 2085 | resolve_deltas(&opts); |
| 2086 | conclude_pack(fix_thin_pack, curr_pack, pack_hash); |
| 2087 | free(ofs_deltas); |
| 2088 | free(ref_deltas); |
| 2089 | if (strict) |
| 2090 | foreign_nr = check_objects(); |
| 2091 | |
| 2092 | if (show_stat) |
| 2093 | show_pack_info(stat_only); |
| 2094 | |
| 2095 | ALLOC_ARRAY(idx_objects, nr_objects); |
| 2096 | for (i = 0; i < nr_objects; i++) |
| 2097 | idx_objects[i] = &objects[i].idx; |
| 2098 | curr_index = write_idx_file(the_repository, index_name, idx_objects, |
| 2099 | nr_objects, &opts, pack_hash); |
| 2100 | if (rev_index) |
| 2101 | curr_rev_index = write_rev_file(the_repository, rev_index_name, |
| 2102 | idx_objects, nr_objects, |
| 2103 | pack_hash, opts.flags); |
| 2104 | free(idx_objects); |
| 2105 | |
| 2106 | if (!verify) |
| 2107 | final(pack_name, curr_pack, |
| 2108 | index_name, curr_index, |
| 2109 | rev_index_name, curr_rev_index, |
| 2110 | keep_msg, promisor_msg, |
| 2111 | pack_hash); |
| 2112 | else |
| 2113 | close(input_fd); |
| 2114 | |
| 2115 | if (do_fsck_object) { |
| 2116 | /* |
| 2117 | * We cannot perform queued consistency checks when running |
| 2118 | * outside of a repository because those require us to read |
| 2119 | * from the object database, which is uninitialized. |
| 2120 | * |
| 2121 | * TODO: we may eventually set up an in-memory object database, |
| 2122 | * which would allow us to perform these queued checks. |
| 2123 | */ |
| 2124 | if (!startup_info->have_repository && |
| 2125 | fsck_has_queued_checks(&fsck_options)) |
| 2126 | die(_("cannot perform queued object checks outside " |
| 2127 | "of a repository")); |
| 2128 | |
| 2129 | if (fsck_finish(&fsck_options)) |
| 2130 | die(_("fsck error in pack objects")); |
| 2131 | } |
| 2132 | |
| 2133 | free(opts.anomaly); |
| 2134 | free(objects); |
| 2135 | strbuf_release(&index_name_buf); |
| 2136 | strbuf_release(&rev_index_name_buf); |
| 2137 | if (!pack_name) |
| 2138 | free((void *) curr_pack); |
| 2139 | if (!index_name) |
| 2140 | free((void *) curr_index); |
| 2141 | free(curr_rev_index); |
| 2142 | |
| 2143 | repack_local_links(); |
| 2144 | |
| 2145 | /* |
| 2146 | * Let the caller know this pack is not self contained |
| 2147 | */ |
| 2148 | if (check_self_contained_and_connected && foreign_nr) |
| 2149 | return 1; |
| 2150 | |
| 2151 | return 0; |
| 2152 | } |