@matheusbsbs / matheus-bagliano / commits / c7934306d1

Fork 0

Limit file descriptors used by packs

Rather than using 'errno == EMFILE' after a failed open() call to indicate the process is out of file descriptors and an LRU pack window should be closed, place a hard upper limit on the number of open packs based on the actual rlimit of the process. By using a hard upper limit that is below the rlimit of the current process it is not necessary to check for EMFILE on every single fd-allocating system call. Instead reserving 25 file descriptors makes it safe to assume the system call won't fail due to being over the filedescriptor limit. Here 25 is chosen as a WAG, but considers 3 for stdin/stdout/stderr, and at least a few for other Git code to operate on temporary files. An additional 20 is reserved as it is not known what the C library needs to perform other services on Git's behalf, such as nsswitch or name resolution. This fixes a case where running `git gc --auto` in a repository with more than 1024 packs (but an rlimit of 1024 open fds) fails due to the temporary output file not being able to allocate a file descriptor. The output file is opened by pack-objects after object enumeration and delta compression are done, both of which have already opened all of the packs and fully populated the file descriptor table. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Shawn O. Pearce committed Feb 28, 2011 at 12:52 UTC c7934306d1504325d26950f35759ad478772e9c3

1 file changed +30 -13

sha1_file.c

+30 -13

@@ -417,6 +417,8 @@ static unsigned int pack_used_ctr;

417 static unsigned int pack_mmap_calls;

418 static unsigned int peak_pack_open_windows;

419 static unsigned int pack_open_windows;

420 +static unsigned int pack_open_fds;

421 +static unsigned int pack_max_fds;

422 static size_t peak_pack_mapped;

423 static size_t pack_mapped;

424 struct packed_git *packed_git;

@@ -596,6 +598,7 @@ static int unuse_one_window(struct packed_git *current, int keep_fd)

598 lru_p->windows = lru_w->next;

599 if (!lru_p->windows && lru_p->pack_fd != keep_fd) {

600 close(lru_p->pack_fd);

601 + pack_open_fds--;

602 lru_p->pack_fd = -1;

603 }

604 }

@@ -680,8 +683,10 @@ void free_pack_by_name(const char *pack_name)

683 if (strcmp(pack_name, p->pack_name) == 0) {

684 clear_delta_base_cache();

685 close_pack_windows(p);

683 - if (p->pack_fd != -1)

686 + if (p->pack_fd != -1) {

687 close(p->pack_fd);

688 + pack_open_fds--;

689 + }

690 close_pack_index(p);

691 free(p->bad_object_sha1);

692 *pp = p->next;

@@ -707,9 +712,29 @@ static int open_packed_git_1(struct packed_git *p)

712 if (!p->index_data && open_pack_index(p))

713 return error("packfile %s index unavailable", p->pack_name);

714

715 + if (!pack_max_fds) {

716 + struct rlimit lim;

717 + unsigned int max_fds;

718 +

719 + if (getrlimit(RLIMIT_NOFILE, &lim))

720 + die_errno("cannot get RLIMIT_NOFILE");

721 +

722 + max_fds = lim.rlim_cur;

723 +

724 + /* Save 3 for stdin/stdout/stderr, 22 for work */

725 + if (25 < max_fds)

726 + pack_max_fds = max_fds - 25;

727 + else

728 + pack_max_fds = 1;

729 + }

730 +

731 + while (pack_max_fds <= pack_open_fds && unuse_one_window(NULL, -1))

732 + ; /* nothing */

733 +

734 p->pack_fd = git_open_noatime(p->pack_name, p);

735 if (p->pack_fd < 0 || fstat(p->pack_fd, &st))

736 return -1;

737 + pack_open_fds++;

738

739 /* If we created the struct before we had the pack we lack size. */

740 if (!p->pack_size) {

@@ -761,6 +786,7 @@ static int open_packed_git(struct packed_git *p)

786 return 0;

787 if (p->pack_fd != -1) {

788 close(p->pack_fd);

789 + pack_open_fds--;

790 p->pack_fd = -1;

791 }

792 return -1;

@@ -918,6 +944,9 @@ struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path)

944

945 void install_packed_git(struct packed_git *pack)

946 {

947 + if (pack->pack_fd != -1)

948 + pack_open_fds++;

949 +

950 pack->next = packed_git;

951 packed_git = pack;

952 }

@@ -935,8 +964,6 @@ static void prepare_packed_git_one(char *objdir, int local)

964 sprintf(path, "%s/pack", objdir);

965 len = strlen(path);

966 dir = opendir(path);

938 - while (!dir && errno == EMFILE && unuse_one_window(NULL, -1))

939 - dir = opendir(path);

967 if (!dir) {

968 if (errno != ENOENT)

969 error("unable to open object pack directory: %s: %s",

@@ -1092,14 +1119,6 @@ static int git_open_noatime(const char *name, struct packed_git *p)

1119 if (fd >= 0)

1120 return fd;

1121

1095 - /* Might the failure be insufficient file descriptors? */

1096 - if (errno == EMFILE) {

1097 - if (unuse_one_window(p, -1))

1098 - continue;

1099 - else

1100 - return -1;

1101 - }

1102 -

1122 /* Might the failure be due to O_NOATIME? */

1123 if (errno != ENOENT && sha1_file_open_flag) {

1124 sha1_file_open_flag = 0;

@@ -2359,8 +2378,6 @@ static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen,

2378

2379 filename = sha1_file_name(sha1);

2380 fd = create_tmpfile(tmpfile, sizeof(tmpfile), filename);

2362 - while (fd < 0 && errno == EMFILE && unuse_one_window(NULL, -1))

2363 - fd = create_tmpfile(tmpfile, sizeof(tmpfile), filename);

2381 if (fd < 0) {

2382 if (errno == EACCES)

2383 return error("insufficient permission for adding an object to repository database %s\n", get_object_directory());