@matheusbsbs / matheus-bagliano / commits / c41ee586dc
Fork 0

Refactor packed_git to prepare for sliding mmap windows.

The idea behind the sliding mmap window pack reader implementation is to have multiple mmap regions active against the same pack file, thereby allowing the process to mmap in only the active/hot sections of the pack and reduce overall virtual address space usage. To implement this we need to refactor the mmap related data (pack_base, pack_use_cnt) out of struct packed_git and move them into a new struct pack_window. We are refactoring the code to support a single struct pack_window per packfile, thereby emulating the prior behavior of mmap'ing the entire pack file. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>

Shawn O. Pearce committed Dec 23, 2006 at 02:33 UTC c41ee586dc95b757cdff4deae10a30a691ba758b
4 files changed +40 -34
builtin-pack-objects.c
+2 -2
@@ -438,7 +438,7 @@ static unsigned long write_object(struct sha1file *f,
438 }
439
440 use_packed_git(p);
441 - buf = (char *) p->pack_base
441 + buf = p->windows->base
442 + entry->in_pack_offset
443 + entry->in_pack_header_size;
444 datalen = find_packed_object_size(p, entry->in_pack_offset)
@@ -943,7 +943,7 @@ static void check_object(struct object_entry *entry)
943 struct object_entry *base_entry = NULL;
944
945 use_packed_git(p);
946 - buf = p->pack_base;
946 + buf = p->windows->base;
947 buf += entry->in_pack_offset;
948
949 /* We want in_pack_type even if we do not reuse delta.
cache.h
+10 -3
@@ -336,14 +336,21 @@ extern struct alternate_object_database {
336 } *alt_odb_list;
337 extern void prepare_alt_odb(void);
338
339 +struct pack_window {
340 + struct pack_window *next;
341 + unsigned char *base;
342 + off_t offset;
343 + size_t len;
344 + unsigned int last_used;
345 + unsigned int inuse_cnt;
346 +};
347 +
348 extern struct packed_git {
349 struct packed_git *next;
350 unsigned long index_size;
351 unsigned long pack_size;
352 + struct pack_window *windows;
353 unsigned int *index_base;
344 - void *pack_base;
345 - unsigned int pack_last_used;
346 - unsigned int pack_use_cnt;
354 int pack_local;
355 unsigned char sha1[20];
356 /* something like ".git/objects/pack/xxxxx.pack" */
pack-check.c
+3 -3
@@ -13,7 +13,8 @@ static int verify_packfile(struct packed_git *p)
13 int nr_objects, err, i;
14
15 /* Header consistency check */
16 - hdr = p->pack_base;
16 + pack_base = p->windows->base;
17 + hdr = (struct pack_header*)pack_base;
18 if (hdr->hdr_signature != htonl(PACK_SIGNATURE))
19 return error("Packfile %s signature mismatch", p->pack_name);
20 if (!pack_version_ok(hdr->hdr_version))
@@ -26,7 +27,6 @@ static int verify_packfile(struct packed_git *p)
27 num_packed_objects(p));
28
29 SHA1_Init(&ctx);
29 - pack_base = p->pack_base;
30 SHA1_Update(&ctx, pack_base, pack_size - 20);
31 SHA1_Final(sha1, &ctx);
32 if (hashcmp(sha1, (unsigned char *)pack_base + pack_size - 20))
@@ -78,7 +78,7 @@ static void show_pack_info(struct packed_git *p)
78 int nr_objects, i;
79 unsigned int chain_histogram[MAX_CHAIN];
80
81 - hdr = p->pack_base;
81 + hdr = (struct pack_header*)p->windows->base;
82 nr_objects = ntohl(hdr->hdr_entries);
83 memset(chain_histogram, 0, sizeof(chain_histogram));
84
sha1_file.c
+25 -26
@@ -455,21 +455,23 @@ static int unuse_one_packed_git(void)
455 struct packed_git *p, *lru = NULL;
456
457 for (p = packed_git; p; p = p->next) {
458 - if (p->pack_use_cnt || !p->pack_base)
458 + if (!p->windows || p->windows->inuse_cnt)
459 continue;
460 - if (!lru || p->pack_last_used < lru->pack_last_used)
460 + if (!lru || p->windows->last_used < lru->windows->last_used)
461 lru = p;
462 }
463 if (!lru)
464 return 0;
465 - munmap(lru->pack_base, lru->pack_size);
466 - lru->pack_base = NULL;
465 + munmap(lru->windows->base, lru->windows->len);
466 + free(lru->windows);
467 + lru->windows = NULL;
468 return 1;
469 }
470
471 void unuse_packed_git(struct packed_git *p)
472 {
472 - p->pack_use_cnt--;
473 + if (p->windows)
474 + p->windows->inuse_cnt--;
475 }
476
477 int use_packed_git(struct packed_git *p)
@@ -482,10 +484,10 @@ int use_packed_git(struct packed_git *p)
484 die("packfile %s not a regular file", p->pack_name);
485 p->pack_size = st.st_size;
486 }
485 - if (!p->pack_base) {
487 + if (!p->windows) {
488 int fd;
489 struct stat st;
488 - void *map;
490 + struct pack_window *win;
491 struct pack_header *hdr;
492
493 pack_mapped += p->pack_size;
@@ -500,16 +502,18 @@ int use_packed_git(struct packed_git *p)
502 }
503 if (st.st_size != p->pack_size)
504 die("packfile %s size mismatch.", p->pack_name);
503 - map = mmap(NULL, p->pack_size, PROT_READ, MAP_PRIVATE, fd, 0);
505 + win = xcalloc(1, sizeof(*win));
506 + win->len = p->pack_size;
507 + win->base = mmap(NULL, p->pack_size, PROT_READ, MAP_PRIVATE, fd, 0);
508 close(fd);
505 - if (map == MAP_FAILED)
509 + if (win->base == MAP_FAILED)
510 die("packfile %s cannot be mapped.", p->pack_name);
507 - p->pack_base = map;
511 + p->windows = win;
512
513 /* Check if we understand this pack file. If we don't we're
514 * likely too old to handle it.
515 */
512 - hdr = map;
516 + hdr = (struct pack_header*)win->base;
517 if (hdr->hdr_signature != htonl(PACK_SIGNATURE))
518 die("packfile %s isn't actually a pack.", p->pack_name);
519 if (!pack_version_ok(hdr->hdr_version))
@@ -522,13 +526,13 @@ int use_packed_git(struct packed_git *p)
526 */
527 if (hashcmp((unsigned char *)(p->index_base) +
528 p->index_size - 40,
525 - (unsigned char *)p->pack_base +
529 + p->windows->base +
530 p->pack_size - 20)) {
531 die("packfile %s does not match index.", p->pack_name);
532 }
533 }
530 - p->pack_last_used = pack_used_ctr++;
531 - p->pack_use_cnt++;
534 + p->windows->last_used = pack_used_ctr++;
535 + p->windows->inuse_cnt++;
536 return 0;
537 }
538
@@ -558,9 +562,7 @@ struct packed_git *add_packed_git(char *path, int path_len, int local)
562 p->pack_size = st.st_size;
563 p->index_base = idx_map;
564 p->next = NULL;
561 - p->pack_base = NULL;
562 - p->pack_last_used = 0;
563 - p->pack_use_cnt = 0;
565 + p->windows = NULL;
566 p->pack_local = local;
567 if ((path_len > 44) && !get_sha1_hex(path + path_len - 44, sha1))
568 hashcpy(p->sha1, sha1);
@@ -591,9 +593,7 @@ struct packed_git *parse_pack_index_file(const unsigned char *sha1, char *idx_pa
593 p->pack_size = 0;
594 p->index_base = idx_map;
595 p->next = NULL;
594 - p->pack_base = NULL;
595 - p->pack_last_used = 0;
596 - p->pack_use_cnt = 0;
596 + p->windows = NULL;
597 hashcpy(p->sha1, sha1);
598 return p;
599 }
@@ -882,7 +882,7 @@ static unsigned long get_delta_base(struct packed_git *p,
882 unsigned long delta_obj_offset,
883 unsigned long *base_obj_offset)
884 {
885 - unsigned char *base_info = (unsigned char *) p->pack_base + offset;
885 + unsigned char *base_info = p->windows->base + offset;
886 unsigned long base_offset;
887
888 /* there must be at least 20 bytes left regardless of delta type */
@@ -949,7 +949,7 @@ static int packed_delta_info(struct packed_git *p,
949
950 memset(&stream, 0, sizeof(stream));
951
952 - stream.next_in = (unsigned char *) p->pack_base + offset;
952 + stream.next_in = p->windows->base + offset;
953 stream.avail_in = p->pack_size - offset;
954 stream.next_out = delta_head;
955 stream.avail_out = sizeof(delta_head);
@@ -984,8 +984,7 @@ static unsigned long unpack_object_header(struct packed_git *p, unsigned long of
984 if (p->pack_size <= offset)
985 die("object offset outside of pack file");
986
987 - used = unpack_object_header_gently((unsigned char *)p->pack_base +
988 - offset,
987 + used = unpack_object_header_gently(p->windows->base + offset,
988 p->pack_size - offset, type, sizep);
989 if (!used)
990 die("object offset outside of pack file");
@@ -1031,7 +1030,7 @@ void packed_object_info_detail(struct packed_git *p,
1030 if (p->pack_size <= offset + 20)
1031 die("pack file %s records an incomplete delta base",
1032 p->pack_name);
1034 - next_sha1 = (unsigned char *) p->pack_base + offset;
1033 + next_sha1 = p->windows->base + offset;
1034 if (*delta_chain_length == 0)
1035 hashcpy(base_sha1, next_sha1);
1036 offset = find_pack_entry_one(next_sha1, p);
@@ -1081,7 +1080,7 @@ static void *unpack_compressed_entry(struct packed_git *p,
1080 buffer = xmalloc(size + 1);
1081 buffer[size] = 0;
1082 memset(&stream, 0, sizeof(stream));
1084 - stream.next_in = (unsigned char*)p->pack_base + offset;
1083 + stream.next_in = p->windows->base + offset;
1084 stream.avail_in = p->pack_size - offset;
1085 stream.next_out = buffer;
1086 stream.avail_out = size;