@matheusbsbs / matheus-bagliano / commits / 9bc879c1ce
Fork 0

Refactor how we open pack files to prepare for multiple windows.

To efficiently support mmaping of multiple regions of the same pack file we want to keep the pack's file descriptor open while we are actively working with that pack. So we are now keeping that file descriptor in packed_git.pack_fd and closing it only after we unmap the last window. This is going to increase the number of file descriptors that are in use at once, however that will be bounded by the total number of pack files present and therefore should not be very high. It is a small tradeoff which we may need to revisit after some testing can be done on various repositories and systems. For code clarity we also want to seperate out the implementation of how we open a pack file from the implementation which locates a suitable window (or makes a new one) from the given pack file. Since this is a rather large delta I'm taking advantage of doing it now, in a fairly isolated change. When we open a pack file we need to examine the header and trailer without having a mmap in place, as we may only need to mmap the middle section of this particular pack. Consequently the verification code has been refactored to make use of the new read_or_die function. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>

Shawn O. Pearce committed Dec 23, 2006 at 02:34 UTC 9bc879c1ced505089e2a1e420d32599bb15b35b5
2 files changed +47 -40
cache.h
+1
@@ -351,6 +351,7 @@ extern struct packed_git {
351 unsigned int *index_base;
352 off_t index_size;
353 off_t pack_size;
354 + int pack_fd;
355 int pack_local;
356 unsigned char sha1[20];
357 /* something like ".git/objects/pack/xxxxx.pack" */
sha1_file.c
+46 -40
@@ -465,6 +465,8 @@ static int unuse_one_packed_git(void)
465 munmap(lru->windows->base, lru->windows->len);
466 free(lru->windows);
467 lru->windows = NULL;
468 + close(p->pack_fd);
469 + p->pack_fd = -1;
470 return 1;
471 }
472
@@ -474,62 +476,64 @@ void unuse_packed_git(struct packed_git *p)
476 p->windows->inuse_cnt--;
477 }
478
477 -int use_packed_git(struct packed_git *p)
479 +static void open_packed_git(struct packed_git *p)
480 {
481 + struct stat st;
482 + struct pack_header hdr;
483 + unsigned char sha1[20];
484 + unsigned char *idx_sha1;
485 +
486 + p->pack_fd = open(p->pack_name, O_RDONLY);
487 + if (p->pack_fd < 0 || fstat(p->pack_fd, &st))
488 + die("packfile %s cannot be opened", p->pack_name);
489 +
490 + /* If we created the struct before we had the pack we lack size. */
491 if (!p->pack_size) {
480 - struct stat st;
481 - /* We created the struct before we had the pack */
482 - stat(p->pack_name, &st);
492 if (!S_ISREG(st.st_mode))
493 die("packfile %s not a regular file", p->pack_name);
494 p->pack_size = st.st_size;
486 - }
495 + } else if (p->pack_size != st.st_size)
496 + die("packfile %s size changed", p->pack_name);
497 +
498 + /* Verify we recognize this pack file format. */
499 + read_or_die(p->pack_fd, &hdr, sizeof(hdr));
500 + if (hdr.hdr_signature != htonl(PACK_SIGNATURE))
501 + die("file %s is not a GIT packfile", p->pack_name);
502 + if (!pack_version_ok(hdr.hdr_version))
503 + die("packfile %s is version %u and not supported"
504 + " (try upgrading GIT to a newer version)",
505 + p->pack_name, ntohl(hdr.hdr_version));
506 +
507 + /* Verify the pack matches its index. */
508 + if (num_packed_objects(p) != ntohl(hdr.hdr_entries))
509 + die("packfile %s claims to have %u objects"
510 + " while index size indicates %u objects",
511 + p->pack_name, ntohl(hdr.hdr_entries),
512 + num_packed_objects(p));
513 + if (lseek(p->pack_fd, p->pack_size - sizeof(sha1), SEEK_SET) == -1)
514 + die("end of packfile %s is unavailable", p->pack_name);
515 + read_or_die(p->pack_fd, sha1, sizeof(sha1));
516 + idx_sha1 = ((unsigned char *)p->index_base) + p->index_size - 40;
517 + if (hashcmp(sha1, idx_sha1))
518 + die("packfile %s does not match index", p->pack_name);
519 +}
520 +
521 +int use_packed_git(struct packed_git *p)
522 +{
523 + if (p->pack_fd == -1)
524 + open_packed_git(p);
525 if (!p->windows) {
488 - int fd;
489 - struct stat st;
526 struct pack_window *win;
491 - struct pack_header *hdr;
527
528 pack_mapped += p->pack_size;
529 while (packed_git_limit < pack_mapped && unuse_one_packed_git())
530 ; /* nothing */
496 - fd = open(p->pack_name, O_RDONLY);
497 - if (fd < 0)
498 - die("packfile %s cannot be opened", p->pack_name);
499 - if (fstat(fd, &st)) {
500 - close(fd);
501 - die("packfile %s cannot be opened", p->pack_name);
502 - }
503 - if (st.st_size != p->pack_size)
504 - die("packfile %s size mismatch.", p->pack_name);
531 win = xcalloc(1, sizeof(*win));
532 win->len = p->pack_size;
507 - win->base = mmap(NULL, p->pack_size, PROT_READ, MAP_PRIVATE, fd, 0);
508 - close(fd);
533 + win->base = mmap(NULL, p->pack_size, PROT_READ, MAP_PRIVATE, p->pack_fd, 0);
534 if (win->base == MAP_FAILED)
535 die("packfile %s cannot be mapped.", p->pack_name);
536 p->windows = win;
512 -
513 - /* Check if we understand this pack file. If we don't we're
514 - * likely too old to handle it.
515 - */
516 - hdr = (struct pack_header*)win->base;
517 - if (hdr->hdr_signature != htonl(PACK_SIGNATURE))
518 - die("packfile %s isn't actually a pack.", p->pack_name);
519 - if (!pack_version_ok(hdr->hdr_version))
520 - die("packfile %s is version %i and not supported"
521 - " (try upgrading GIT to a newer version)",
522 - p->pack_name, ntohl(hdr->hdr_version));
523 -
524 - /* Check if the pack file matches with the index file.
525 - * this is cheap.
526 - */
527 - if (hashcmp((unsigned char *)(p->index_base) +
528 - p->index_size - 40,
529 - p->windows->base +
530 - p->pack_size - 20)) {
531 - die("packfile %s does not match index.", p->pack_name);
532 - }
537 }
538 p->windows->last_used = pack_used_ctr++;
539 p->windows->inuse_cnt++;
@@ -563,6 +567,7 @@ struct packed_git *add_packed_git(char *path, int path_len, int local)
567 p->index_base = idx_map;
568 p->next = NULL;
569 p->windows = NULL;
570 + p->pack_fd = -1;
571 p->pack_local = local;
572 if ((path_len > 44) && !get_sha1_hex(path + path_len - 44, sha1))
573 hashcpy(p->sha1, sha1);
@@ -594,6 +599,7 @@ struct packed_git *parse_pack_index_file(const unsigned char *sha1, char *idx_pa
599 p->index_base = idx_map;
600 p->next = NULL;
601 p->windows = NULL;
602 + p->pack_fd = -1;
603 hashcpy(p->sha1, sha1);
604 return p;
605 }