@matheusbsbs / matheus-bagliano / commits / 18bdec1118
Fork 0

Limit the size of the new delta_base_cache

The new configuration variable core.deltaBaseCacheLimit allows the user to control how much memory they are willing to give to Git for caching base objects of deltas. This is not normally meant to be a user tweakable knob; the "out of the box" settings are meant to be suitable for almost all workloads. We default to 16 MiB under the assumption that the cache is not meant to consume all of the user's available memory, and that the cache's main purpose was to cache trees, for faster path limiters during revision traversal. Since trees tend to be relatively small objects, this relatively small limit should still allow a large number of objects. On the other hand we don't want the cache to start storing 200 different versions of a 200 MiB blob, as this could easily blow the entire address space of a 32 bit process. We evict OBJ_BLOB from the cache first (credit goes to Junio) as we want to favor OBJ_TREE within the cache. These are the objects that have the highest inflate() startup penalty, as they tend to be small and thus don't have that much of a chance to ammortize that penalty over the entire data. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <junkio@cox.net>

Shawn O. Pearce committed Mar 19, 2007 at 01:14 UTC 18bdec1118df92649b70ce126aff2f147deecad5
5 files changed +46 -4
Documentation/config.txt
+13
@@ -240,6 +240,19 @@ the largest projects. You probably do not need to adjust this value.
240 +
241 Common unit suffixes of 'k', 'm', or 'g' are supported.
242
243 +core.deltaBaseCacheLimit::
244 + Maximum number of bytes to reserve for caching base objects
245 + that multiple deltafied objects reference. By storing the
246 + entire decompressed base objects in a cache Git is able
247 + to avoid unpacking and decompressing frequently used base
248 + objects multiple times.
249 ++
250 +Default is 16 MiB on all platforms. This should be reasonable
251 +for all users/operating systems, except on the largest projects.
252 +You probably do not need to adjust this value.
253 ++
254 +Common unit suffixes of 'k', 'm', or 'g' are supported.
255 +
256 alias.*::
257 Command aliases for the gitlink:git[1] command wrapper - e.g.
258 after defining "alias.last = cat-file commit HEAD", the invocation
cache.h
+1
@@ -228,6 +228,7 @@ extern const char *apply_default_whitespace;
228 extern int zlib_compression_level;
229 extern size_t packed_git_window_size;
230 extern size_t packed_git_limit;
231 +extern size_t delta_base_cache_limit;
232 extern int auto_crlf;
233
234 #define GIT_REPO_VERSION 0
config.c
+5
@@ -331,6 +331,11 @@ int git_default_config(const char *var, const char *value)
331 return 0;
332 }
333
334 + if (!strcmp(var, "core.deltabasecachelimit")) {
335 + delta_base_cache_limit = git_config_int(var, value);
336 + return 0;
337 + }
338 +
339 if (!strcmp(var, "core.autocrlf")) {
340 if (value && !strcasecmp(value, "input")) {
341 auto_crlf = -1;
environment.c
+1
@@ -27,6 +27,7 @@ const char *apply_default_whitespace;
27 int zlib_compression_level = Z_DEFAULT_COMPRESSION;
28 size_t packed_git_window_size = DEFAULT_PACKED_GIT_WINDOW_SIZE;
29 size_t packed_git_limit = DEFAULT_PACKED_GIT_LIMIT;
30 +size_t delta_base_cache_limit = 16 * 1024 * 1024;
31 int pager_in_use;
32 int pager_use_color = 1;
33 int auto_crlf = 0; /* 1: both ways, -1: only when adding git objects */
sha1_file.c
+26 -4
@@ -1354,6 +1354,7 @@ static void *unpack_compressed_entry(struct packed_git *p,
1354
1355 #define MAX_DELTA_CACHE (256)
1356
1357 +static size_t delta_base_cached;
1358 static struct delta_base_cache_entry {
1359 struct packed_git *p;
1360 off_t base_offset;
@@ -1384,8 +1385,10 @@ static void *cache_or_unpack_entry(struct packed_git *p, off_t base_offset,
1385 return unpack_entry(p, base_offset, type, base_size);
1386
1387 found_cache_entry:
1387 - if (!keep_cache)
1388 + if (!keep_cache) {
1389 ent->data = NULL;
1390 + delta_base_cached -= ent->size;
1391 + }
1392 else {
1393 ret = xmalloc(ent->size + 1);
1394 memcpy(ret, ent->data, ent->size);
@@ -1396,14 +1399,33 @@ found_cache_entry:
1399 return ret;
1400 }
1401
1402 +static inline void release_delta_base_cache(struct delta_base_cache_entry *ent)
1403 +{
1404 + if (ent->data) {
1405 + free(ent->data);
1406 + ent->data = NULL;
1407 + delta_base_cached -= ent->size;
1408 + }
1409 +}
1410 +
1411 static void add_delta_base_cache(struct packed_git *p, off_t base_offset,
1412 void *base, unsigned long base_size, enum object_type type)
1413 {
1402 - unsigned long hash = pack_entry_hash(p, base_offset);
1414 + unsigned long i, hash = pack_entry_hash(p, base_offset);
1415 struct delta_base_cache_entry *ent = delta_base_cache + hash;
1416
1405 - if (ent->data)
1406 - free(ent->data);
1417 + release_delta_base_cache(ent);
1418 + delta_base_cached += base_size;
1419 + for (i = 0; delta_base_cached > delta_base_cache_limit
1420 + && i < ARRAY_SIZE(delta_base_cache); i++) {
1421 + struct delta_base_cache_entry *f = delta_base_cache + i;
1422 + if (f->type == OBJ_BLOB)
1423 + release_delta_base_cache(f);
1424 + }
1425 + for (i = 0; delta_base_cached > delta_base_cache_limit
1426 + && i < ARRAY_SIZE(delta_base_cache); i++)
1427 + release_delta_base_cache(delta_base_cache + i);
1428 +
1429 ent->p = p;
1430 ent->base_offset = base_offset;
1431 ent->type = type;