@matheusbsbs / matheus-bagliano / commits / 9892bebafe
Fork 0

sha1_file: don't malloc the whole compressed result when writing out objects

There is no real advantage to malloc the whole output buffer and deflate the data in a single pass when writing loose objects. That is like only 1% faster while using more memory, especially with large files where memory usage is far more. It is best to deflate and write the data out in small chunks reusing the same memory instead. For example, using 'git add' on a few large files averaging 40 MB ... Before: 21.45user 1.10system 0:22.57elapsed 99%CPU (0avgtext+0avgdata 0maxresident)k 0inputs+828040outputs (0major+142640minor)pagefaults 0swaps After: 21.50user 1.25system 0:22.76elapsed 99%CPU (0avgtext+0avgdata 0maxresident)k 0inputs+828040outputs (0major+104408minor)pagefaults 0swaps While the runtime stayed relatively the same, the number of minor page faults went down significantly. Signed-off-by: Nicolas Pitre <nico@fluxnic.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Nicolas Pitre committed Feb 20, 2010 at 23:27 UTC 9892bebafe0865d8f4f3f18d60a1cfa2d1447cd7
1 file changed +10 -14
sha1_file.c
+10 -14
@@ -2281,8 +2281,7 @@ static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen,
2281 void *buf, unsigned long len, time_t mtime)
2282 {
2283 int fd, ret;
2284 - size_t size;
2285 - unsigned char *compressed;
2284 + unsigned char compressed[4096];
2285 z_stream stream;
2286 char *filename;
2287 static char tmpfile[PATH_MAX];
@@ -2301,12 +2300,8 @@ static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen,
2300 /* Set it up */
2301 memset(&stream, 0, sizeof(stream));
2302 deflateInit(&stream, zlib_compression_level);
2304 - size = 8 + deflateBound(&stream, len+hdrlen);
2305 - compressed = xmalloc(size);
2306 -
2307 - /* Compress it */
2303 stream.next_out = compressed;
2309 - stream.avail_out = size;
2304 + stream.avail_out = sizeof(compressed);
2305
2306 /* First header.. */
2307 stream.next_in = (unsigned char *)hdr;
@@ -2317,20 +2312,21 @@ static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen,
2312 /* Then the data itself.. */
2313 stream.next_in = buf;
2314 stream.avail_in = len;
2320 - ret = deflate(&stream, Z_FINISH);
2315 + do {
2316 + ret = deflate(&stream, Z_FINISH);
2317 + if (write_buffer(fd, compressed, stream.next_out - compressed) < 0)
2318 + die("unable to write sha1 file");
2319 + stream.next_out = compressed;
2320 + stream.avail_out = sizeof(compressed);
2321 + } while (ret == Z_OK);
2322 +
2323 if (ret != Z_STREAM_END)
2324 die("unable to deflate new object %s (%d)", sha1_to_hex(sha1), ret);
2323 -
2325 ret = deflateEnd(&stream);
2326 if (ret != Z_OK)
2327 die("deflateEnd on object %s failed (%d)", sha1_to_hex(sha1), ret);
2328
2328 - size = stream.total_out;
2329 -
2330 - if (write_buffer(fd, compressed, size) < 0)
2331 - die("unable to write sha1 file");
2329 close_sha1_file(fd);
2333 - free(compressed);
2330
2331 if (mtime) {
2332 struct utimbuf utb;