@matheusbsbs / matheus-bagliano / commits / d079837eee
Fork 0

Lazily open pack index files on demand

In some repository configurations the user may have many packfiles, but all of the recent commits/trees/tags/blobs are likely to be in the most recent packfile (the one with the newest mtime). It is therefore common to be able to complete an entire operation by accessing only one packfile, even if there are 25 packfiles available to the repository. Rather than opening and mmaping the corresponding .idx file for every pack found, we now only open and map the .idx when we suspect there might be an object of interest in there. Of course we cannot known in advance which packfile contains an object, so we still need to scan the entire packed_git list to locate anything. But odds are users want to access objects in the most recently created packfiles first, and that may be all they ever need for the current operation. Junio observed in b867092f that placing recent packfiles before older ones can slightly improve access times for recent objects, without degrading it for historical object access. This change improves upon Junio's observations by trying even harder to avoid the .idx files that we won't need. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>

Shawn O. Pearce committed May 26, 2007 at 01:24 UTC d079837eeeadc37d266113a1fd2deb0a01aaee91
5 files changed +49 -6
builtin-count-objects.c
+2
@@ -111,6 +111,8 @@ int cmd_count_objects(int ac, const char **av, const char *prefix)
111 for (p = packed_git; p; p = p->next) {
112 if (!p->pack_local)
113 continue;
114 + if (!p->index_data && open_pack_index(p))
115 + continue;
116 packed += p->num_objects;
117 num_pack++;
118 }
cache.h
+2 -1
@@ -485,10 +485,11 @@ extern struct packed_git *find_sha1_pack(const unsigned char *sha1,
485 struct packed_git *packs);
486
487 extern void pack_report(void);
488 +extern int open_pack_index(struct packed_git *);
489 extern unsigned char* use_pack(struct packed_git *, struct pack_window **, off_t, unsigned int *);
490 extern void unuse_pack(struct pack_window **);
491 extern struct packed_git *add_packed_git(const char *, int, int);
491 -extern const unsigned char *nth_packed_object_sha1(const struct packed_git *, uint32_t);
492 +extern const unsigned char *nth_packed_object_sha1(struct packed_git *, uint32_t);
493 extern off_t find_pack_entry_one(const unsigned char *, struct packed_git *);
494 extern void *unpack_entry(struct packed_git *, off_t, enum object_type *, unsigned long *);
495 extern unsigned long unpack_object_header_gently(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep);
pack-check.c
+7 -2
@@ -128,12 +128,17 @@ static void show_pack_info(struct packed_git *p)
128
129 int verify_pack(struct packed_git *p, int verbose)
130 {
131 - off_t index_size = p->index_size;
132 - const unsigned char *index_base = p->index_data;
131 + off_t index_size;
132 + const unsigned char *index_base;
133 SHA_CTX ctx;
134 unsigned char sha1[20];
135 int ret;
136
137 + if (open_pack_index(p))
138 + return error("packfile %s index not opened", p->pack_name);
139 + index_size = p->index_size;
140 + index_base = p->index_data;
141 +
142 ret = 0;
143 /* Verify SHA1 sum of the index file */
144 SHA1_Init(&ctx);
pack-redundant.c
+3
@@ -550,6 +550,9 @@ static struct pack_list * add_pack(struct packed_git *p)
550 l.pack = p;
551 llist_init(&l.all_objects);
552
553 + if (!p->index_data && open_pack_index(p))
554 + return NULL;
555 +
556 base = p->index_data;
557 base += 256 * 4 + ((p->index_version < 2) ? 4 : 8);
558 step = (p->index_version < 2) ? 24 : 20;
sha1_file.c
+35 -3
@@ -530,6 +530,21 @@ static int check_packed_git_idx(const char *path, struct packed_git *p)
530 return 0;
531 }
532
533 +int open_pack_index (struct packed_git *p)
534 +{
535 + char *idx_name;
536 + int ret;
537 +
538 + if (p->index_data)
539 + return 0;
540 +
541 + idx_name = xstrdup(p->pack_name);
542 + strcpy(idx_name + strlen(idx_name) - strlen(".pack"), ".idx");
543 + ret = check_packed_git_idx(idx_name, p);
544 + free(idx_name);
545 + return ret;
546 +}
547 +
548 static void scan_windows(struct packed_git *p,
549 struct packed_git **lru_p,
550 struct pack_window **lru_w,
@@ -605,6 +620,9 @@ static int open_packed_git_1(struct packed_git *p)
620 unsigned char *idx_sha1;
621 long fd_flag;
622
623 + if (!p->index_data && open_pack_index(p))
624 + return error("packfile %s index unavailable", p->pack_name);
625 +
626 p->pack_fd = open(p->pack_name, O_RDONLY);
627 if (p->pack_fd < 0 || fstat(p->pack_fd, &st))
628 return -1;
@@ -757,8 +775,7 @@ struct packed_git *add_packed_git(const char *path, int path_len, int local)
775 return NULL;
776 memcpy(p->pack_name, path, path_len);
777 strcpy(p->pack_name + path_len, ".pack");
760 - if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode) ||
761 - check_packed_git_idx(path, p)) {
778 + if (stat(p->pack_name, &st) || !S_ISREG(st.st_mode)) {
779 free(p);
780 return NULL;
781 }
@@ -766,6 +783,10 @@ struct packed_git *add_packed_git(const char *path, int path_len, int local)
783 /* ok, it looks sane as far as we can check without
784 * actually mapping the pack file.
785 */
786 + p->index_version = 0;
787 + p->index_data = NULL;
788 + p->index_size = 0;
789 + p->num_objects = 0;
790 p->pack_size = st.st_size;
791 p->next = NULL;
792 p->windows = NULL;
@@ -1572,10 +1593,15 @@ void *unpack_entry(struct packed_git *p, off_t obj_offset,
1593 return data;
1594 }
1595
1575 -const unsigned char *nth_packed_object_sha1(const struct packed_git *p,
1596 +const unsigned char *nth_packed_object_sha1(struct packed_git *p,
1597 uint32_t n)
1598 {
1599 const unsigned char *index = p->index_data;
1600 + if (!index) {
1601 + if (open_pack_index(p))
1602 + return NULL;
1603 + index = p->index_data;
1604 + }
1605 if (n >= p->num_objects)
1606 return NULL;
1607 index += 4 * 256;
@@ -1612,6 +1638,12 @@ off_t find_pack_entry_one(const unsigned char *sha1,
1638 const unsigned char *index = p->index_data;
1639 unsigned hi, lo;
1640
1641 + if (!index) {
1642 + if (open_pack_index(p))
1643 + return 0;
1644 + level1_ofs = p->index_data;
1645 + index = p->index_data;
1646 + }
1647 if (p->index_version > 1) {
1648 level1_ofs += 2;
1649 index += 8;