sha1_file.c: Don't retain open fds on small packs
If a pack file is small enough that its entire contents fits within one mmap window, mmap the file and then immediately close its file descriptor. This reduces the number of file descriptors that are needed to read from repositories with many tiny pack files, such as one that has received 1000 pushes (and created 1000 small pack files) since its last repack. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Shawn O. Pearce committed
Mar 2, 2011 at 10:01 UTC
d131b7afea58f47721dafd013ab6aff4adfc42dd
3 files changed
+39
-6
cache.h
+2
-1
@@ -899,7 +899,8 @@ extern struct packed_git {
899
time_t mtime;
900
int pack_fd;
901
unsigned pack_local:1,
902
- pack_keep:1;
902
+ pack_keep:1,
903
+ do_not_close:1;
904
unsigned char sha1[20];
905
/* something like ".git/objects/pack/xxxxx.pack" */
906
char pack_name[FLEX_ARRAY]; /* more */
fast-import.c
+1
@@ -871,6 +871,7 @@ static void start_packfile(void)
871
p = xcalloc(1, sizeof(*p) + strlen(tmpfile) + 2);
872
strcpy(p->pack_name, tmpfile);
873
p->pack_fd = pack_fd;
874
+ p->do_not_close = 1;
875
pack_file = sha1fd(pack_fd, p->pack_name);
876
877
hdr.hdr_signature = htonl(PACK_SIGNATURE);
sha1_file.c
+36
-5
@@ -596,7 +596,8 @@ static int unuse_one_window(struct packed_git *current, int keep_fd)
596
lru_l->next = lru_w->next;
597
else {
598
lru_p->windows = lru_w->next;
599
- if (!lru_p->windows && lru_p->pack_fd != keep_fd) {
599
+ if (!lru_p->windows && lru_p->pack_fd != -1
600
+ && lru_p->pack_fd != keep_fd) {
601
close(lru_p->pack_fd);
602
pack_open_fds--;
603
lru_p->pack_fd = -1;
@@ -812,14 +813,13 @@ unsigned char *use_pack(struct packed_git *p,
813
{
814
struct pack_window *win = *w_cursor;
815
815
- if (p->pack_fd == -1 && open_packed_git(p))
816
- die("packfile %s cannot be accessed", p->pack_name);
817
-
816
/* Since packfiles end in a hash of their content and it's
817
* pointless to ask for an offset into the middle of that
818
* hash, and the in_window function above wouldn't match
819
* don't allow an offset too close to the end of the file.
820
*/
821
+ if (!p->pack_size && p->pack_fd == -1 && open_packed_git(p))
822
+ die("packfile %s cannot be accessed", p->pack_name);
823
if (offset > (p->pack_size - 20))
824
die("offset beyond end of packfile (truncated pack?)");
825
@@ -833,6 +833,10 @@ unsigned char *use_pack(struct packed_git *p,
833
if (!win) {
834
size_t window_align = packed_git_window_size / 2;
835
off_t len;
836
+
837
+ if (p->pack_fd == -1 && open_packed_git(p))
838
+ die("packfile %s cannot be accessed", p->pack_name);
839
+
840
win = xcalloc(1, sizeof(*win));
841
win->offset = (offset / window_align) * window_align;
842
len = p->pack_size - win->offset;
@@ -850,6 +854,12 @@ unsigned char *use_pack(struct packed_git *p,
854
die("packfile %s cannot be mapped: %s",
855
p->pack_name,
856
strerror(errno));
857
+ if (!win->offset && win->len == p->pack_size
858
+ && !p->do_not_close) {
859
+ close(p->pack_fd);
860
+ pack_open_fds--;
861
+ p->pack_fd = -1;
862
+ }
863
pack_mmap_calls++;
864
pack_open_windows++;
865
if (pack_mapped > peak_pack_mapped)
@@ -1950,6 +1960,27 @@ off_t find_pack_entry_one(const unsigned char *sha1,
1960
return 0;
1961
}
1962
1963
+static int is_pack_valid(struct packed_git *p)
1964
+{
1965
+ /* An already open pack is known to be valid. */
1966
+ if (p->pack_fd != -1)
1967
+ return 1;
1968
+
1969
+ /* If the pack has one window completely covering the
1970
+ * file size, the pack is known to be valid even if
1971
+ * the descriptor is not currently open.
1972
+ */
1973
+ if (p->windows) {
1974
+ struct pack_window *w = p->windows;
1975
+
1976
+ if (!w->offset && w->len == p->pack_size)
1977
+ return 1;
1978
+ }
1979
+
1980
+ /* Force the pack to open to prove its valid. */
1981
+ return !open_packed_git(p);
1982
+}
1983
+
1984
static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e)
1985
{
1986
static struct packed_git *last_found = (void *)1;
@@ -1979,7 +2010,7 @@ static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e)
2010
* it may have been deleted since the index
2011
* was loaded!
2012
*/
1982
- if (p->pack_fd == -1 && open_packed_git(p)) {
2013
+ if (!is_pack_valid(p)) {
2014
error("packfile %s cannot be accessed", p->pack_name);
2015
goto next;
2016
}