@matheusbsbs / matheus-bagliano / commits / a97773ce7e
Fork 0

Add pack-objects window memory usage limit

This adds an option (--window-memory=N) and configuration variable (pack.windowMemory = N) to limit the memory size of the pack-objects delta search window. This works by removing the oldest unpacked objects whenever the total size goes above the limit. It will always leave at least one object, though, so as not to completely eliminate the possibility of computing deltas. This is an extra limit on top of the normal window size (--window=N); the window will not dynamically grow above the fixed number of entries specified to fill the memory limit. With this, repacking a repository with a mix of large and small objects is possible even with a very large window. Cleaner and correct circular buffer handling courtesy of Nicolas Pitre. Signed-off-by: Brian Downing <bdowning@lavos.net> Acked-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Brian Downing committed Jul 12, 2007 at 08:07 UTC a97773ce7e4fb635615d3b363f144d8c8a780fea
1 file changed +44 -8
builtin-pack-objects.c
+44 -8
@@ -16,8 +16,9 @@
16 #include "progress.h"
17
18 static const char pack_usage[] = "\
19 -git-pack-objects [{ -q | --progress | --all-progress }] [--max-pack-size=N] \n\
20 - [--local] [--incremental] [--window=N] [--depth=N] \n\
19 +git-pack-objects [{ -q | --progress | --all-progress }] \n\
20 + [--max-pack-size=N] [--local] [--incremental] \n\
21 + [--window=N] [--window-memory=N] [--depth=N] \n\
22 [--no-reuse-delta] [--no-reuse-object] [--delta-base-offset] \n\
23 [--non-empty] [--revs [--unpacked | --all]*] [--reflog] \n\
24 [--stdout | base-name] [<ref-list | <object-list]";
@@ -79,6 +80,9 @@ static unsigned long delta_cache_size = 0;
80 static unsigned long max_delta_cache_size = 0;
81 static unsigned long cache_max_small_delta_size = 1000;
82
83 +static unsigned long window_memory_usage = 0;
84 +static unsigned long window_memory_limit = 0;
85 +
86 /*
87 * The object names in objects array are hashed with this hashtable,
88 * to help looking up the entry by object name.
@@ -1357,12 +1361,14 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
1361 if (sz != trg_size)
1362 die("object %s inconsistent object length (%lu vs %lu)",
1363 sha1_to_hex(trg_entry->idx.sha1), sz, trg_size);
1364 + window_memory_usage += sz;
1365 }
1366 if (!src->data) {
1367 src->data = read_sha1_file(src_entry->idx.sha1, &type, &sz);
1368 if (sz != src_size)
1369 die("object %s inconsistent object length (%lu vs %lu)",
1370 sha1_to_hex(src_entry->idx.sha1), sz, src_size);
1371 + window_memory_usage += sz;
1372 }
1373 if (!src->index) {
1374 src->index = create_delta_index(src->data, src_size);
@@ -1372,6 +1378,7 @@ static int try_delta(struct unpacked *trg, struct unpacked *src,
1378 warning("suboptimal pack - out of memory");
1379 return 0;
1380 }
1381 + window_memory_usage += sizeof_delta_index(src->index);
1382 }
1383
1384 delta_buf = create_delta(src->index, trg->data, trg_size, &delta_size, max_size);
@@ -1414,9 +1421,22 @@ static unsigned int check_delta_limit(struct object_entry *me, unsigned int n)
1421 return m;
1422 }
1423
1424 +static void free_unpacked(struct unpacked *n)
1425 +{
1426 + window_memory_usage -= sizeof_delta_index(n->index);
1427 + free_delta_index(n->index);
1428 + n->index = NULL;
1429 + if (n->data) {
1430 + free(n->data);
1431 + n->data = NULL;
1432 + window_memory_usage -= n->entry->size;
1433 + }
1434 + n->entry = NULL;
1435 +}
1436 +
1437 static void find_deltas(struct object_entry **list, int window, int depth)
1438 {
1419 - uint32_t i = nr_objects, idx = 0, processed = 0;
1439 + uint32_t i = nr_objects, idx = 0, count = 0, processed = 0;
1440 unsigned int array_size = window * sizeof(struct unpacked);
1441 struct unpacked *array;
1442 int max_depth;
@@ -1451,12 +1471,17 @@ static void find_deltas(struct object_entry **list, int window, int depth)
1471 if (entry->no_try_delta)
1472 continue;
1473
1454 - free_delta_index(n->index);
1455 - n->index = NULL;
1456 - free(n->data);
1457 - n->data = NULL;
1474 + free_unpacked(n);
1475 n->entry = entry;
1476
1477 + while (window_memory_limit &&
1478 + window_memory_usage > window_memory_limit &&
1479 + count > 1) {
1480 + uint32_t tail = (idx + window - count) % window;
1481 + free_unpacked(array + tail);
1482 + count--;
1483 + }
1484 +
1485 /*
1486 * If the current object is at pack edge, take the depth the
1487 * objects that depend on the current object into account
@@ -1491,6 +1516,8 @@ static void find_deltas(struct object_entry **list, int window, int depth)
1516
1517 next:
1518 idx++;
1519 + if (count + 1 < window)
1520 + count++;
1521 if (idx >= window)
1522 idx = 0;
1523 } while (i > 0);
@@ -1529,7 +1556,11 @@ static int git_pack_config(const char *k, const char *v)
1556 window = git_config_int(k, v);
1557 return 0;
1558 }
1532 - if(!strcmp(k, "pack.depth")) {
1559 + if (!strcmp(k, "pack.windowmemory")) {
1560 + window_memory_limit = git_config_ulong(k, v);
1561 + return 0;
1562 + }
1563 + if (!strcmp(k, "pack.depth")) {
1564 depth = git_config_int(k, v);
1565 return 0;
1566 }
@@ -1705,6 +1736,11 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix)
1736 usage(pack_usage);
1737 continue;
1738 }
1739 + if (!prefixcmp(arg, "--window-memory=")) {
1740 + if (!git_parse_ulong(arg+16, &window_memory_limit))
1741 + usage(pack_usage);
1742 + continue;
1743 + }
1744 if (!prefixcmp(arg, "--depth=")) {
1745 char *end;
1746 depth = strtoul(arg+8, &end, 0);