@matheusbsbs / matheus-bagliano / commits / 60bb8b1453
Fork 0

Fully activate the sliding window pack access.

This finally turns on the sliding window behavior for packfile data access by mapping limited size windows and chaining them under the packed_git->windows list. We consider a given byte offset to be within the window only if there would be at least 20 bytes (one hash worth of data) accessible after the requested offset. This range selection relates to the contract that use_pack() makes with its callers, allowing them to access one hash or one object header without needing to call use_pack() for every byte of data obtained. In the worst case scenario we will map the same page of data twice into memory: once at the end of one window and once again at the start of the next window. This duplicate page mapping will happen only when an object header or a delta base reference is spanned over the end of a window and is always limited to just one page of duplication, as no sane operating system will ever have a page size smaller than a hash. I am assuming that the possible wasted page of virtual address space is going to perform faster than the alternatives, which would be to copy the object header or ref delta into a temporary buffer prior to parsing, or to check the window range on every byte during header parsing. We may decide to revisit this decision in the future since this is just a gut instinct decision and has not actually been proven out by experimental testing. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>

Shawn O. Pearce committed Dec 23, 2006 at 02:34 UTC 60bb8b1453e2f93d13e7bb44e8e46c085d2dd752
5 files changed +76 -13
Documentation/config.txt
+11
@@ -118,6 +118,17 @@ core.legacyheaders::
118 database directly (where the "http://" and "rsync://" protocols
119 count as direct access).
120
121 +core.packedGitWindowSize::
122 + Number of bytes of a pack file to map into memory in a
123 + single mapping operation. Larger window sizes may allow
124 + your system to process a smaller number of large pack files
125 + more quickly. Smaller window sizes will negatively affect
126 + performance due to increased calls to the opreating system's
127 + memory manager, but may improve performance when accessing
128 + a large number of large pack files. Default is 32 MiB,
129 + which should be reasonable for all users/operating systems.
130 + You probably do not need to adjust this value.
131 +
132 core.packedGitLimit::
133 Maximum number of bytes to map simultaneously into memory
134 from pack files. If Git needs to access more than this many
cache.h
+1
@@ -196,6 +196,7 @@ extern int warn_ambiguous_refs;
196 extern int shared_repository;
197 extern const char *apply_default_whitespace;
198 extern int zlib_compression_level;
199 +extern size_t packed_git_window_size;
200 extern size_t packed_git_limit;
201
202 #define GIT_REPO_VERSION 0
config.c
+10
@@ -298,6 +298,16 @@ int git_default_config(const char *var, const char *value)
298 return 0;
299 }
300
301 + if (!strcmp(var, "core.packedgitwindowsize")) {
302 + int pgsz = getpagesize();
303 + packed_git_window_size = git_config_int(var, value);
304 + packed_git_window_size /= pgsz;
305 + if (!packed_git_window_size)
306 + packed_git_window_size = 1;
307 + packed_git_window_size *= pgsz;
308 + return 0;
309 + }
310 +
311 if (!strcmp(var, "core.packedgitlimit")) {
312 packed_git_limit = git_config_int(var, value);
313 return 0;
environment.c
+1
@@ -23,6 +23,7 @@ char *git_log_output_encoding;
23 int shared_repository = PERM_UMASK;
24 const char *apply_default_whitespace;
25 int zlib_compression_level = Z_DEFAULT_COMPRESSION;
26 +size_t packed_git_window_size = 32 * 1024 * 1024;
27 size_t packed_git_limit = 256 * 1024 * 1024;
28 int pager_in_use;
29 int pager_use_color = 1;
sha1_file.c
+53 -13
@@ -397,8 +397,9 @@ static char *find_sha1_file(const unsigned char *sha1, struct stat *st)
397 return NULL;
398 }
399
400 -static int pack_used_ctr;
401 -static unsigned long pack_mapped;
400 +static unsigned int pack_used_ctr;
401 +static size_t pack_mapped;
402 +static size_t page_size;
403 struct packed_git *packed_git;
404
405 static int check_packed_git_idx(const char *path, unsigned long *idx_size_,
@@ -536,31 +537,70 @@ static void open_packed_git(struct packed_git *p)
537 die("packfile %s does not match index", p->pack_name);
538 }
539
540 +static int in_window(struct pack_window *win, unsigned long offset)
541 +{
542 + /* We must promise at least 20 bytes (one hash) after the
543 + * offset is available from this window, otherwise the offset
544 + * is not actually in this window and a different window (which
545 + * has that one hash excess) must be used. This is to support
546 + * the object header and delta base parsing routines below.
547 + */
548 + off_t win_off = win->offset;
549 + return win_off <= offset
550 + && (offset + 20) <= (win_off + win->len);
551 +}
552 +
553 unsigned char* use_pack(struct packed_git *p,
554 struct pack_window **w_cursor,
555 unsigned long offset,
556 unsigned int *left)
557 {
544 - struct pack_window *win = p->windows;
558 + struct pack_window *win = *w_cursor;
559
560 if (p->pack_fd == -1)
561 open_packed_git(p);
548 - if (!win) {
549 - pack_mapped += p->pack_size;
550 - while (packed_git_limit < pack_mapped && unuse_one_window())
551 - ; /* nothing */
552 - win = xcalloc(1, sizeof(*win));
553 - win->len = p->pack_size;
554 - win->base = mmap(NULL, p->pack_size, PROT_READ, MAP_PRIVATE, p->pack_fd, 0);
555 - if (win->base == MAP_FAILED)
556 - die("packfile %s cannot be mapped.", p->pack_name);
557 - p->windows = win;
562 +
563 + /* Since packfiles end in a hash of their content and its
564 + * pointless to ask for an offset into the middle of that
565 + * hash, and the in_window function above wouldn't match
566 + * don't allow an offset too close to the end of the file.
567 + */
568 + if (offset > (p->pack_size - 20))
569 + die("offset beyond end of packfile (truncated pack?)");
570 +
571 + if (!win || !in_window(win, offset)) {
572 + if (win)
573 + win->inuse_cnt--;
574 + for (win = p->windows; win; win = win->next) {
575 + if (in_window(win, offset))
576 + break;
577 + }
578 + if (!win) {
579 + if (!page_size)
580 + page_size = getpagesize();
581 + win = xcalloc(1, sizeof(*win));
582 + win->offset = (offset / page_size) * page_size;
583 + win->len = p->pack_size - win->offset;
584 + if (win->len > packed_git_window_size)
585 + win->len = packed_git_window_size;
586 + pack_mapped += win->len;
587 + while (packed_git_limit < pack_mapped && unuse_one_window())
588 + ; /* nothing */
589 + win->base = mmap(NULL, win->len,
590 + PROT_READ, MAP_PRIVATE,
591 + p->pack_fd, win->offset);
592 + if (win->base == MAP_FAILED)
593 + die("packfile %s cannot be mapped.", p->pack_name);
594 + win->next = p->windows;
595 + p->windows = win;
596 + }
597 }
598 if (win != *w_cursor) {
599 win->last_used = pack_used_ctr++;
600 win->inuse_cnt++;
601 *w_cursor = win;
602 }
603 + offset -= win->offset;
604 if (left)
605 *left = win->len - offset;
606 return win->base + offset;