@matheusbsbs / matheus-bagliano / commits / 417305764a

index-pack: reduce object_entry size to save memory

For each object in the input pack, we need one struct object_entry. On x86-64, this struct is 64 bytes long. Although: - The 8 bytes for delta_depth and base_object_no are only useful when show_stat is set. And it's never set unless someone is debugging. - The three fields hdr_size, type and real_type take 4 bytes each even though they never use more than 4 bits. By moving delta_depth and base_object_no out of struct object_entry and make the other 3 fields one byte long instead of 4, we shrink 25% of this struct. On a 3.4M object repo (*) that's about 53MB. The saving is less impressive compared to index-pack memory use for basic bookkeeping (**), about 16%. (*) linux-2.6.git already has 4M objects as of v3.19-rc7 so this is not an unrealistic number of objects that we have to deal with. (**) 3.4M * (sizeof(object_entry) + sizeof(delta_entry)) = 311MB Brought-up-by: Matthew Sporleder <msporleder@gmail.com> Signed-off-by: Nguyễn Thái Ngọc Duy <pclouds@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Nguyễn Thái Ngọc Duy committed Feb 26, 2015 at 17:52 UTC 417305764a90eaf100faaa9ca46cd76ed6b3bcb6
1 file changed +19 -11
builtin/index-pack.c
+19 -11
@@ -18,9 +18,12 @@ static const char index_pack_usage[] =
18 struct object_entry {
19 struct pack_idx_entry idx;
20 unsigned long size;
21 - unsigned int hdr_size;
22 - enum object_type type;
23 - enum object_type real_type;
21 + unsigned char hdr_size;
22 + signed char type;
23 + signed char real_type;
24 +};
25 +
26 +struct object_stat {
27 unsigned delta_depth;
28 int base_object_no;
29 };
@@ -64,6 +67,7 @@ struct delta_entry {
67 };
68
69 static struct object_entry *objects;
70 +static struct object_stat *obj_stat;
71 static struct delta_entry *deltas;
72 static struct thread_local nothread_data;
73 static int nr_objects;
@@ -873,13 +877,15 @@ static void resolve_delta(struct object_entry *delta_obj,
877 void *base_data, *delta_data;
878
879 if (show_stat) {
876 - delta_obj->delta_depth = base->obj->delta_depth + 1;
880 + int i = delta_obj - objects;
881 + int j = base->obj - objects;
882 + obj_stat[i].delta_depth = obj_stat[j].delta_depth + 1;
883 deepest_delta_lock();
878 - if (deepest_delta < delta_obj->delta_depth)
879 - deepest_delta = delta_obj->delta_depth;
884 + if (deepest_delta < obj_stat[i].delta_depth)
885 + deepest_delta = obj_stat[i].delta_depth;
886 deepest_delta_unlock();
887 + obj_stat[i].base_object_no = j;
888 }
882 - delta_obj->base_object_no = base->obj - objects;
889 delta_data = get_data_from_pack(delta_obj);
890 base_data = get_base_data(base);
891 result->obj = delta_obj;
@@ -902,7 +908,7 @@ static void resolve_delta(struct object_entry *delta_obj,
908 * "want"; if so, swap in "set" and return true. Otherwise, leave it untouched
909 * and return false.
910 */
905 -static int compare_and_swap_type(enum object_type *type,
911 +static int compare_and_swap_type(signed char *type,
912 enum object_type want,
913 enum object_type set)
914 {
@@ -1499,7 +1505,7 @@ static void show_pack_info(int stat_only)
1505 struct object_entry *obj = &objects[i];
1506
1507 if (is_delta_type(obj->type))
1502 - chain_histogram[obj->delta_depth - 1]++;
1508 + chain_histogram[obj_stat[i].delta_depth - 1]++;
1509 if (stat_only)
1510 continue;
1511 printf("%s %-6s %lu %lu %"PRIuMAX,
@@ -1508,8 +1514,8 @@ static void show_pack_info(int stat_only)
1514 (unsigned long)(obj[1].idx.offset - obj->idx.offset),
1515 (uintmax_t)obj->idx.offset);
1516 if (is_delta_type(obj->type)) {
1511 - struct object_entry *bobj = &objects[obj->base_object_no];
1512 - printf(" %u %s", obj->delta_depth, sha1_to_hex(bobj->idx.sha1));
1517 + struct object_entry *bobj = &objects[obj_stat[i].base_object_no];
1518 + printf(" %u %s", obj_stat[i].delta_depth, sha1_to_hex(bobj->idx.sha1));
1519 }
1520 putchar('\n');
1521 }
@@ -1672,6 +1678,8 @@ int cmd_index_pack(int argc, const char **argv, const char *prefix)
1678 curr_pack = open_pack_file(pack_name);
1679 parse_pack_header();
1680 objects = xcalloc(nr_objects + 1, sizeof(struct object_entry));
1681 + if (show_stat)
1682 + obj_stat = xcalloc(nr_objects + 1, sizeof(struct object_stat));
1683 deltas = xcalloc(nr_objects, sizeof(struct delta_entry));
1684 parse_pack_objects(pack_sha1);
1685 resolve_deltas();