@matheusbsbs / matheus-bagliano / commits / d1efefa46f
Fork 0

Actually handle some-low memory conditions

Tim Ansell discovered his Debian server didn't permit git-daemon to use as much memory as it needed to handle cloning a project with a 128 MiB packfile. Filtering the strace provided by Tim of the rev-list child showed this gem of a sequence: open("./objects/pack/pack-*.pack", O_RDONLY|O_LARGEFILE <unfinished ...> <... open resumed> ) = 5 OK, so the packfile is fd 5... mmap2(NULL, 33554432, PROT_READ, MAP_PRIVATE, 5, 0 <unfinished ...> <... mmap2 resumed> ) = 0xb5e2d000 and we mapped one 32 MiB window from it at position 0... mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...> <... mmap2 resumed> ) = -1 ENOMEM (Cannot allocate memory) And we asked for another window further into the file. But got denied. In Tim's case this was due to a resource limit on the git-daemon process, and its children. Now where are we in the code? We're down inside use_pack(), after we have called unuse_one_window() enough times to make sure we stay within our allowed maximum window size. However since we didn't unmap the prior window at 0xb5e2d000 we aren't exceeding the current limit (which probably was just the defaults). But we're actually down inside xmmap()... So we release the window we do have (by calling release_pack_memory), assuming there is some memory pressure... munmap(0xb5e2d000, 33554432 <unfinished ...> <... munmap resumed> ) = 0 close(5 <unfinished ...> <... close resumed> ) = 0 And that was the last window in this packfile. So we closed it. Way to go us. Our xmmap did not expect release_pack_memory to close the fd its about to map... mmap2(NULL, 31020635, PROT_READ, MAP_PRIVATE, 5, 0x6000 <unfinished ...> <... mmap2 resumed> ) = -1 EBADF (Bad file descriptor) And so the Linux kernel happily tells us f' off. write(2, "fatal: ", 7 <unfinished ...> <... write resumed> ) = 7 write(2, "Out of memory? mmap failed: Bad "..., 47 <unfinished ...> <... write resumed> ) = 47 And we report the bad file descriptor error, and not the ENOMEM, and die, claiming we are out of memory. But actually that mmap should have succeeded, as we had enough memory for that window, seeing as how we released the prior one. Originally when I developed the sliding window mmap feature I had this exact same bug in fast-import, and I dealt with it by handing in the struct packed_git* we want to open the new window for, as the caller wasn't prepared to reopen the packfile if unuse_one_window closed it. The same is true here from xmmap, but the caller doesn't have the struct packed_git* handy. So I'm using the file descriptor instead to perform the same test. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>

Shawn O. Pearce committed Apr 25, 2007 at 04:02 UTC d1efefa46fda6bb68bcd73a5e532eef98ef28a1d
2 files changed +11 -11
git-compat-util.h
+6 -6
@@ -156,13 +156,13 @@ extern size_t gitstrlcpy(char *, const char *, size_t);
156 extern uintmax_t gitstrtoumax(const char *, char **, int);
157 #endif
158
159 -extern void release_pack_memory(size_t);
159 +extern void release_pack_memory(size_t, int);
160
161 static inline char* xstrdup(const char *str)
162 {
163 char *ret = strdup(str);
164 if (!ret) {
165 - release_pack_memory(strlen(str) + 1);
165 + release_pack_memory(strlen(str) + 1, -1);
166 ret = strdup(str);
167 if (!ret)
168 die("Out of memory, strdup failed");
@@ -176,7 +176,7 @@ static inline void *xmalloc(size_t size)
176 if (!ret && !size)
177 ret = malloc(1);
178 if (!ret) {
179 - release_pack_memory(size);
179 + release_pack_memory(size, -1);
180 ret = malloc(size);
181 if (!ret && !size)
182 ret = malloc(1);
@@ -195,7 +195,7 @@ static inline void *xrealloc(void *ptr, size_t size)
195 if (!ret && !size)
196 ret = realloc(ptr, 1);
197 if (!ret) {
198 - release_pack_memory(size);
198 + release_pack_memory(size, -1);
199 ret = realloc(ptr, size);
200 if (!ret && !size)
201 ret = realloc(ptr, 1);
@@ -211,7 +211,7 @@ static inline void *xcalloc(size_t nmemb, size_t size)
211 if (!ret && (!nmemb || !size))
212 ret = calloc(1, 1);
213 if (!ret) {
214 - release_pack_memory(nmemb * size);
214 + release_pack_memory(nmemb * size, -1);
215 ret = calloc(nmemb, size);
216 if (!ret && (!nmemb || !size))
217 ret = calloc(1, 1);
@@ -228,7 +228,7 @@ static inline void *xmmap(void *start, size_t length,
228 if (ret == MAP_FAILED) {
229 if (!length)
230 return NULL;
231 - release_pack_memory(length);
231 + release_pack_memory(length, fd);
232 ret = mmap(start, length, prot, flags, fd, offset);
233 if (ret == MAP_FAILED)
234 die("Out of memory? mmap failed: %s", strerror(errno));
sha1_file.c
+5 -5
@@ -516,7 +516,7 @@ static void scan_windows(struct packed_git *p,
516 }
517 }
518
519 -static int unuse_one_window(struct packed_git *current)
519 +static int unuse_one_window(struct packed_git *current, int keep_fd)
520 {
521 struct packed_git *p, *lru_p = NULL;
522 struct pack_window *lru_w = NULL, *lru_l = NULL;
@@ -532,7 +532,7 @@ static int unuse_one_window(struct packed_git *current)
532 lru_l->next = lru_w->next;
533 else {
534 lru_p->windows = lru_w->next;
535 - if (!lru_p->windows && lru_p != current) {
535 + if (!lru_p->windows && lru_p->pack_fd != keep_fd) {
536 close(lru_p->pack_fd);
537 lru_p->pack_fd = -1;
538 }
@@ -544,10 +544,10 @@ static int unuse_one_window(struct packed_git *current)
544 return 0;
545 }
546
547 -void release_pack_memory(size_t need)
547 +void release_pack_memory(size_t need, int fd)
548 {
549 size_t cur = pack_mapped;
550 - while (need >= (cur - pack_mapped) && unuse_one_window(NULL))
550 + while (need >= (cur - pack_mapped) && unuse_one_window(NULL, fd))
551 ; /* nothing */
552 }
553
@@ -680,7 +680,7 @@ unsigned char* use_pack(struct packed_git *p,
680 win->len = (size_t)len;
681 pack_mapped += win->len;
682 while (packed_git_limit < pack_mapped
683 - && unuse_one_window(p))
683 + && unuse_one_window(p, p->pack_fd))
684 ; /* nothing */
685 win->base = xmmap(NULL, win->len,
686 PROT_READ, MAP_PRIVATE,