Work around EMFILE when there are too many pack files
When opening any files in the object database, release unused pack windows if the open(2) syscall fails due to EMFILE (too many open files in this process). This allows Git to degrade gracefully on a repository with thousands of pack files, and a commit stored in a loose object in the middle of the history. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Shawn O. Pearce committed
Nov 1, 2010 at 15:54 UTC
f2e872aa5eeb74f429632e14d0f7fa2ed828b65a
1 file changed
+27
-16
sha1_file.c
+27
-16
@@ -35,7 +35,7 @@ static size_t sz_fmt(size_t s) { return s; }
35
36
const unsigned char null_sha1[20];
37
38
-static int git_open_noatime(const char *name);
38
+static int git_open_noatime(const char *name, struct packed_git *p);
39
40
int safe_create_leading_directories(char *path)
41
{
@@ -300,7 +300,7 @@ static void read_info_alternates(const char * relative_base, int depth)
300
int fd;
301
302
sprintf(path, "%s/%s", relative_base, alt_file_name);
303
- fd = git_open_noatime(path);
303
+ fd = git_open_noatime(path, NULL);
304
if (fd < 0)
305
return;
306
if (fstat(fd, &st) || (st.st_size == 0)) {
@@ -413,7 +413,7 @@ static int check_packed_git_idx(const char *path, struct packed_git *p)
413
struct pack_idx_header *hdr;
414
size_t idx_size;
415
uint32_t version, nr, i, *index;
416
- int fd = git_open_noatime(path);
416
+ int fd = git_open_noatime(path, p);
417
struct stat st;
418
419
if (fd < 0)
@@ -657,9 +657,7 @@ static int open_packed_git_1(struct packed_git *p)
657
if (!p->index_data && open_pack_index(p))
658
return error("packfile %s index unavailable", p->pack_name);
659
660
- p->pack_fd = git_open_noatime(p->pack_name);
661
- while (p->pack_fd < 0 && errno == EMFILE && unuse_one_window(p, -1))
662
- p->pack_fd = git_open_noatime(p->pack_name);
660
+ p->pack_fd = git_open_noatime(p->pack_name, p);
661
if (p->pack_fd < 0 || fstat(p->pack_fd, &st))
662
return -1;
663
@@ -876,7 +874,7 @@ static void prepare_packed_git_one(char *objdir, int local)
874
sprintf(path, "%s/pack", objdir);
875
len = strlen(path);
876
dir = opendir(path);
879
- while (!dir && errno == EMFILE && unuse_one_window(packed_git, -1))
877
+ while (!dir && errno == EMFILE && unuse_one_window(NULL, -1))
878
dir = opendir(path);
879
if (!dir) {
880
if (errno != ENOENT)
@@ -1024,18 +1022,31 @@ int check_sha1_signature(const unsigned char *sha1, void *map, unsigned long siz
1022
return hashcmp(sha1, real_sha1) ? -1 : 0;
1023
}
1024
1027
-static int git_open_noatime(const char *name)
1025
+static int git_open_noatime(const char *name, struct packed_git *p)
1026
{
1027
static int sha1_file_open_flag = O_NOATIME;
1030
- int fd = open(name, O_RDONLY | sha1_file_open_flag);
1028
1032
- /* Might the failure be due to O_NOATIME? */
1033
- if (fd < 0 && errno != ENOENT && sha1_file_open_flag) {
1034
- fd = open(name, O_RDONLY);
1029
+ for (;;) {
1030
+ int fd = open(name, O_RDONLY | sha1_file_open_flag);
1031
if (fd >= 0)
1032
+ return fd;
1033
+
1034
+ /* Might the failure be insufficient file descriptors? */
1035
+ if (errno == EMFILE) {
1036
+ if (unuse_one_window(p, -1))
1037
+ continue;
1038
+ else
1039
+ return -1;
1040
+ }
1041
+
1042
+ /* Might the failure be due to O_NOATIME? */
1043
+ if (errno != ENOENT && sha1_file_open_flag) {
1044
sha1_file_open_flag = 0;
1045
+ continue;
1046
+ }
1047
+
1048
+ return -1;
1049
}
1038
- return fd;
1050
}
1051
1052
static int open_sha1_file(const unsigned char *sha1)
@@ -1044,7 +1055,7 @@ static int open_sha1_file(const unsigned char *sha1)
1055
char *name = sha1_file_name(sha1);
1056
struct alternate_object_database *alt;
1057
1047
- fd = git_open_noatime(name);
1058
+ fd = git_open_noatime(name, NULL);
1059
if (fd >= 0)
1060
return fd;
1061
@@ -1053,7 +1064,7 @@ static int open_sha1_file(const unsigned char *sha1)
1064
for (alt = alt_odb_list; alt; alt = alt->next) {
1065
name = alt->name;
1066
fill_sha1_path(name, sha1);
1056
- fd = git_open_noatime(alt->base);
1067
+ fd = git_open_noatime(alt->base, NULL);
1068
if (fd >= 0)
1069
return fd;
1070
}
@@ -2314,7 +2325,7 @@ static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen,
2325
2326
filename = sha1_file_name(sha1);
2327
fd = create_tmpfile(tmpfile, sizeof(tmpfile), filename);
2317
- while (fd < 0 && errno == EMFILE && unuse_one_window(packed_git, -1))
2328
+ while (fd < 0 && errno == EMFILE && unuse_one_window(NULL, -1))
2329
fd = create_tmpfile(tmpfile, sizeof(tmpfile), filename);
2330
if (fd < 0) {
2331
if (errno == EACCES)