Refactor packed_git to prepare for sliding mmap windows.
The idea behind the sliding mmap window pack reader implementation is to have multiple mmap regions active against the same pack file, thereby allowing the process to mmap in only the active/hot sections of the pack and reduce overall virtual address space usage. To implement this we need to refactor the mmap related data (pack_base, pack_use_cnt) out of struct packed_git and move them into a new struct pack_window. We are refactoring the code to support a single struct pack_window per packfile, thereby emulating the prior behavior of mmap'ing the entire pack file. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>
Shawn O. Pearce committed
Dec 23, 2006 at 02:33 UTC
c41ee586dc95b757cdff4deae10a30a691ba758b
4 files changed
+40
-34
builtin-pack-objects.c
+2
-2
@@ -438,7 +438,7 @@ static unsigned long write_object(struct sha1file *f,
438
}
439
440
use_packed_git(p);
441
- buf = (char *) p->pack_base
441
+ buf = p->windows->base
442
+ entry->in_pack_offset
443
+ entry->in_pack_header_size;
444
datalen = find_packed_object_size(p, entry->in_pack_offset)
@@ -943,7 +943,7 @@ static void check_object(struct object_entry *entry)
943
struct object_entry *base_entry = NULL;
944
945
use_packed_git(p);
946
- buf = p->pack_base;
946
+ buf = p->windows->base;
947
buf += entry->in_pack_offset;
948
949
/* We want in_pack_type even if we do not reuse delta.
cache.h
+10
-3
@@ -336,14 +336,21 @@ extern struct alternate_object_database {
336
} *alt_odb_list;
337
extern void prepare_alt_odb(void);
338
339
+struct pack_window {
340
+ struct pack_window *next;
341
+ unsigned char *base;
342
+ off_t offset;
343
+ size_t len;
344
+ unsigned int last_used;
345
+ unsigned int inuse_cnt;
346
+};
347
+
348
extern struct packed_git {
349
struct packed_git *next;
350
unsigned long index_size;
351
unsigned long pack_size;
352
+ struct pack_window *windows;
353
unsigned int *index_base;
344
- void *pack_base;
345
- unsigned int pack_last_used;
346
- unsigned int pack_use_cnt;
354
int pack_local;
355
unsigned char sha1[20];
356
/* something like ".git/objects/pack/xxxxx.pack" */
pack-check.c
+3
-3
@@ -13,7 +13,8 @@ static int verify_packfile(struct packed_git *p)
13
int nr_objects, err, i;
14
15
/* Header consistency check */
16
- hdr = p->pack_base;
16
+ pack_base = p->windows->base;
17
+ hdr = (struct pack_header*)pack_base;
18
if (hdr->hdr_signature != htonl(PACK_SIGNATURE))
19
return error("Packfile %s signature mismatch", p->pack_name);
20
if (!pack_version_ok(hdr->hdr_version))
@@ -26,7 +27,6 @@ static int verify_packfile(struct packed_git *p)
27
num_packed_objects(p));
28
29
SHA1_Init(&ctx);
29
- pack_base = p->pack_base;
30
SHA1_Update(&ctx, pack_base, pack_size - 20);
31
SHA1_Final(sha1, &ctx);
32
if (hashcmp(sha1, (unsigned char *)pack_base + pack_size - 20))
@@ -78,7 +78,7 @@ static void show_pack_info(struct packed_git *p)
78
int nr_objects, i;
79
unsigned int chain_histogram[MAX_CHAIN];
80
81
- hdr = p->pack_base;
81
+ hdr = (struct pack_header*)p->windows->base;
82
nr_objects = ntohl(hdr->hdr_entries);
83
memset(chain_histogram, 0, sizeof(chain_histogram));
84
sha1_file.c
+25
-26
@@ -455,21 +455,23 @@ static int unuse_one_packed_git(void)
455
struct packed_git *p, *lru = NULL;
456
457
for (p = packed_git; p; p = p->next) {
458
- if (p->pack_use_cnt || !p->pack_base)
458
+ if (!p->windows || p->windows->inuse_cnt)
459
continue;
460
- if (!lru || p->pack_last_used < lru->pack_last_used)
460
+ if (!lru || p->windows->last_used < lru->windows->last_used)
461
lru = p;
462
}
463
if (!lru)
464
return 0;
465
- munmap(lru->pack_base, lru->pack_size);
466
- lru->pack_base = NULL;
465
+ munmap(lru->windows->base, lru->windows->len);
466
+ free(lru->windows);
467
+ lru->windows = NULL;
468
return 1;
469
}
470
471
void unuse_packed_git(struct packed_git *p)
472
{
472
- p->pack_use_cnt--;
473
+ if (p->windows)
474
+ p->windows->inuse_cnt--;
475
}
476
477
int use_packed_git(struct packed_git *p)
@@ -482,10 +484,10 @@ int use_packed_git(struct packed_git *p)
484
die("packfile %s not a regular file", p->pack_name);
485
p->pack_size = st.st_size;
486
}
485
- if (!p->pack_base) {
487
+ if (!p->windows) {
488
int fd;
489
struct stat st;
488
- void *map;
490
+ struct pack_window *win;
491
struct pack_header *hdr;
492
493
pack_mapped += p->pack_size;
@@ -500,16 +502,18 @@ int use_packed_git(struct packed_git *p)
502
}
503
if (st.st_size != p->pack_size)
504
die("packfile %s size mismatch.", p->pack_name);
503
- map = mmap(NULL, p->pack_size, PROT_READ, MAP_PRIVATE, fd, 0);
505
+ win = xcalloc(1, sizeof(*win));
506
+ win->len = p->pack_size;
507
+ win->base = mmap(NULL, p->pack_size, PROT_READ, MAP_PRIVATE, fd, 0);
508
close(fd);
505
- if (map == MAP_FAILED)
509
+ if (win->base == MAP_FAILED)
510
die("packfile %s cannot be mapped.", p->pack_name);
507
- p->pack_base = map;
511
+ p->windows = win;
512
513
/* Check if we understand this pack file. If we don't we're
514
* likely too old to handle it.
515
*/
512
- hdr = map;
516
+ hdr = (struct pack_header*)win->base;
517
if (hdr->hdr_signature != htonl(PACK_SIGNATURE))
518
die("packfile %s isn't actually a pack.", p->pack_name);
519
if (!pack_version_ok(hdr->hdr_version))
@@ -522,13 +526,13 @@ int use_packed_git(struct packed_git *p)
526
*/
527
if (hashcmp((unsigned char *)(p->index_base) +
528
p->index_size - 40,
525
- (unsigned char *)p->pack_base +
529
+ p->windows->base +
530
p->pack_size - 20)) {
531
die("packfile %s does not match index.", p->pack_name);
532
}
533
}
530
- p->pack_last_used = pack_used_ctr++;
531
- p->pack_use_cnt++;
534
+ p->windows->last_used = pack_used_ctr++;
535
+ p->windows->inuse_cnt++;
536
return 0;
537
}
538
@@ -558,9 +562,7 @@ struct packed_git *add_packed_git(char *path, int path_len, int local)
562
p->pack_size = st.st_size;
563
p->index_base = idx_map;
564
p->next = NULL;
561
- p->pack_base = NULL;
562
- p->pack_last_used = 0;
563
- p->pack_use_cnt = 0;
565
+ p->windows = NULL;
566
p->pack_local = local;
567
if ((path_len > 44) && !get_sha1_hex(path + path_len - 44, sha1))
568
hashcpy(p->sha1, sha1);
@@ -591,9 +593,7 @@ struct packed_git *parse_pack_index_file(const unsigned char *sha1, char *idx_pa
593
p->pack_size = 0;
594
p->index_base = idx_map;
595
p->next = NULL;
594
- p->pack_base = NULL;
595
- p->pack_last_used = 0;
596
- p->pack_use_cnt = 0;
596
+ p->windows = NULL;
597
hashcpy(p->sha1, sha1);
598
return p;
599
}
@@ -882,7 +882,7 @@ static unsigned long get_delta_base(struct packed_git *p,
882
unsigned long delta_obj_offset,
883
unsigned long *base_obj_offset)
884
{
885
- unsigned char *base_info = (unsigned char *) p->pack_base + offset;
885
+ unsigned char *base_info = p->windows->base + offset;
886
unsigned long base_offset;
887
888
/* there must be at least 20 bytes left regardless of delta type */
@@ -949,7 +949,7 @@ static int packed_delta_info(struct packed_git *p,
949
950
memset(&stream, 0, sizeof(stream));
951
952
- stream.next_in = (unsigned char *) p->pack_base + offset;
952
+ stream.next_in = p->windows->base + offset;
953
stream.avail_in = p->pack_size - offset;
954
stream.next_out = delta_head;
955
stream.avail_out = sizeof(delta_head);
@@ -984,8 +984,7 @@ static unsigned long unpack_object_header(struct packed_git *p, unsigned long of
984
if (p->pack_size <= offset)
985
die("object offset outside of pack file");
986
987
- used = unpack_object_header_gently((unsigned char *)p->pack_base +
988
- offset,
987
+ used = unpack_object_header_gently(p->windows->base + offset,
988
p->pack_size - offset, type, sizep);
989
if (!used)
990
die("object offset outside of pack file");
@@ -1031,7 +1030,7 @@ void packed_object_info_detail(struct packed_git *p,
1030
if (p->pack_size <= offset + 20)
1031
die("pack file %s records an incomplete delta base",
1032
p->pack_name);
1034
- next_sha1 = (unsigned char *) p->pack_base + offset;
1033
+ next_sha1 = p->windows->base + offset;
1034
if (*delta_chain_length == 0)
1035
hashcpy(base_sha1, next_sha1);
1036
offset = find_pack_entry_one(next_sha1, p);
@@ -1081,7 +1080,7 @@ static void *unpack_compressed_entry(struct packed_git *p,
1080
buffer = xmalloc(size + 1);
1081
buffer[size] = 0;
1082
memset(&stream, 0, sizeof(stream));
1084
- stream.next_in = (unsigned char*)p->pack_base + offset;
1083
+ stream.next_in = p->windows->base + offset;
1084
stream.avail_in = p->pack_size - offset;
1085
stream.next_out = buffer;
1086
stream.avail_out = size;