@matheusbsbs / matheus-bagliano / commits / 96c47d1466

fast-import: introduce mem_pool type

Introduce the mem_pool type which encapsulates all the information necessary to manage a pool of memory. This change moves the existing variables in fast-import used to support the global memory pool to use this structure. It also renames variables that are no longer used by memory pools to reflect their more scoped usage. These changes allow for the multiple instances of a memory pool to exist and be reused outside of fast-import. In a future commit the mem_pool type will be moved to its own file. Signed-off-by: Jameson Miller <jamill@microsoft.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Jameson Miller committed Apr 11, 2018 at 18:37 UTC 96c47d1466d2d11e2028b79fdc0bcaf7a4bb7345

1 file changed +52 -29

fast-import.c

+52 -29

@@ -216,6 +216,19 @@ struct mp_block {

216 uintmax_t space[FLEX_ARRAY]; /* more */

217 };

218

219 +struct mem_pool {

220 + struct mp_block *mp_block;

221 +

222 + /*

223 + * The amount of available memory to grow the pool by.

224 + * This size does not include the overhead for the mp_block.

225 + */

226 + size_t block_alloc;

227 +

228 + /* The total amount of memory allocated by the pool. */

229 + size_t pool_alloc;

230 +};

231 +

232 struct atom_str {

233 struct atom_str *next_atom;

234 unsigned short str_len;

@@ -304,9 +317,8 @@ static int global_argc;

317 static const char **global_argv;

318

319 /* Memory pools */

307 -static size_t mem_pool_alloc = 2*1024*1024 - sizeof(struct mp_block);

308 -static size_t total_allocd;

309 -static struct mp_block *mp_block_head;

320 +static struct mem_pool fi_mem_pool = {NULL, 2*1024*1024 -

321 + sizeof(struct mp_block), 0 };

322

323 /* Atom management */

324 static unsigned int atom_table_sz = 4451;

@@ -341,6 +353,7 @@ static unsigned int tree_entry_alloc = 1000;

353 static void *avail_tree_entry;

354 static unsigned int avail_tree_table_sz = 100;

355 static struct avail_tree_content **avail_tree_table;

356 +static size_t tree_entry_allocd;

357 static struct strbuf old_tree = STRBUF_INIT;

358 static struct strbuf new_tree = STRBUF_INIT;

359

@@ -634,7 +647,21 @@ static unsigned int hc_str(const char *s, size_t len)

647 return r;

648 }

649

637 -static void *pool_alloc(size_t len)

650 +static struct mp_block *mem_pool_alloc_block(struct mem_pool *mem_pool, size_t block_alloc)

651 +{

652 + struct mp_block *p;

653 +

654 + mem_pool->pool_alloc += sizeof(struct mp_block) + block_alloc;

655 + p = xmalloc(st_add(sizeof(struct mp_block), block_alloc));

656 + p->next_block = mem_pool->mp_block;

657 + p->next_free = (char *)p->space;

658 + p->end = p->next_free + block_alloc;

659 + mem_pool->mp_block = p;

660 +

661 + return p;

662 +}

663 +

664 +static void *mem_pool_alloc(struct mem_pool *mem_pool, size_t len)

665 {

666 struct mp_block *p;

667 void *r;

@@ -643,21 +670,17 @@ static void *pool_alloc(size_t len)

670 if (len & (sizeof(uintmax_t) - 1))

671 len += sizeof(uintmax_t) - (len & (sizeof(uintmax_t) - 1));

672

646 - for (p = mp_block_head; p; p = p->next_block)

647 - if ((p->end - p->next_free >= len))

673 + for (p = mem_pool->mp_block; p; p = p->next_block)

674 + if (p->end - p->next_free >= len)

675 break;

676

677 if (!p) {

651 - if (len >= (mem_pool_alloc/2)) {

652 - total_allocd += len;

678 + if (len >= (mem_pool->block_alloc / 2)) {

679 + mem_pool->pool_alloc += len;

680 return xmalloc(len);

681 }

655 - total_allocd += sizeof(struct mp_block) + mem_pool_alloc;

656 - p = xmalloc(st_add(sizeof(struct mp_block), mem_pool_alloc));

657 - p->next_block = mp_block_head;

658 - p->next_free = (char *) p->space;

659 - p->end = p->next_free + mem_pool_alloc;

660 - mp_block_head = p;

682 +

683 + p = mem_pool_alloc_block(mem_pool, mem_pool->block_alloc);

684 }

685

686 r = p->next_free;

@@ -665,10 +688,10 @@ static void *pool_alloc(size_t len)

688 return r;

689 }

690

668 -static void *pool_calloc(size_t count, size_t size)

691 +static void *mem_pool_calloc(struct mem_pool *mem_pool, size_t count, size_t size)

692 {

670 - size_t len = count * size;

671 - void *r = pool_alloc(len);

693 + size_t len = st_mult(count, size);

694 + void *r = mem_pool_alloc(mem_pool, len);

695 memset(r, 0, len);

696 return r;

697 }

@@ -676,7 +699,7 @@ static void *pool_calloc(size_t count, size_t size)

699 static char *pool_strdup(const char *s)

700 {

701 size_t len = strlen(s) + 1;

679 - char *r = pool_alloc(len);

702 + char *r = mem_pool_alloc(&fi_mem_pool, len);

703 memcpy(r, s, len);

704 return r;

705 }

@@ -685,7 +708,7 @@ static void insert_mark(uintmax_t idnum, struct object_entry *oe)

708 {

709 struct mark_set *s = marks;

710 while ((idnum >> s->shift) >= 1024) {

688 - s = pool_calloc(1, sizeof(struct mark_set));

711 + s = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct mark_set));

712 s->shift = marks->shift + 10;

713 s->data.sets[0] = marks;

714 marks = s;

@@ -694,7 +717,7 @@ static void insert_mark(uintmax_t idnum, struct object_entry *oe)

717 uintmax_t i = idnum >> s->shift;

718 idnum -= i << s->shift;

719 if (!s->data.sets[i]) {

697 - s->data.sets[i] = pool_calloc(1, sizeof(struct mark_set));

720 + s->data.sets[i] = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct mark_set));

721 s->data.sets[i]->shift = s->shift - 10;

722 }

723 s = s->data.sets[i];

@@ -732,7 +755,7 @@ static struct atom_str *to_atom(const char *s, unsigned short len)

755 if (c->str_len == len && !strncmp(s, c->str_dat, len))

756 return c;

757

735 - c = pool_alloc(sizeof(struct atom_str) + len + 1);

758 + c = mem_pool_alloc(&fi_mem_pool, sizeof(struct atom_str) + len + 1);

759 c->str_len = len;

760 memcpy(c->str_dat, s, len);

761 c->str_dat[len] = 0;

@@ -763,7 +786,7 @@ static struct branch *new_branch(const char *name)

786 if (check_refname_format(name, REFNAME_ALLOW_ONELEVEL))

787 die("Branch name doesn't conform to GIT standards: %s", name);

788

766 - b = pool_calloc(1, sizeof(struct branch));

789 + b = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct branch));

790 b->name = pool_strdup(name);

791 b->table_next_branch = branch_table[hc];

792 b->branch_tree.versions[0].mode = S_IFDIR;

@@ -799,7 +822,7 @@ static struct tree_content *new_tree_content(unsigned int cnt)

822 avail_tree_table[hc] = f->next_avail;

823 } else {

824 cnt = cnt & 7 ? ((cnt / 8) + 1) * 8 : cnt;

802 - f = pool_alloc(sizeof(*t) + sizeof(t->entries[0]) * cnt);

825 + f = mem_pool_alloc(&fi_mem_pool, sizeof(*t) + sizeof(t->entries[0]) * cnt);

826 f->entry_capacity = cnt;

827 }

828

@@ -844,7 +867,7 @@ static struct tree_entry *new_tree_entry(void)

867

868 if (!avail_tree_entry) {

869 unsigned int n = tree_entry_alloc;

847 - total_allocd += n * sizeof(struct tree_entry);

870 + tree_entry_allocd += n * sizeof(struct tree_entry);

871 ALLOC_ARRAY(e, n);

872 avail_tree_entry = e;

873 while (n-- > 1) {

@@ -2862,7 +2885,7 @@ static void parse_new_tag(const char *arg)

2885 enum object_type type;

2886 const char *v;

2887

2865 - t = pool_alloc(sizeof(struct tag));

2888 + t = mem_pool_alloc(&fi_mem_pool, sizeof(struct tag));

2889 memset(t, 0, sizeof(struct tag));

2890 t->name = pool_strdup(arg);

2891 if (last_tag)

@@ -3461,12 +3484,12 @@ int cmd_main(int argc, const char **argv)

3484 atom_table = xcalloc(atom_table_sz, sizeof(struct atom_str*));

3485 branch_table = xcalloc(branch_table_sz, sizeof(struct branch*));

3486 avail_tree_table = xcalloc(avail_tree_table_sz, sizeof(struct avail_tree_content*));

3464 - marks = pool_calloc(1, sizeof(struct mark_set));

3487 + marks = mem_pool_calloc(&fi_mem_pool, 1, sizeof(struct mark_set));

3488

3489 global_argc = argc;

3490 global_argv = argv;

3491

3469 - rc_free = pool_alloc(cmd_save * sizeof(*rc_free));

3492 + rc_free = mem_pool_alloc(&fi_mem_pool, cmd_save * sizeof(*rc_free));

3493 for (i = 0; i < (cmd_save - 1); i++)

3494 rc_free[i].next = &rc_free[i + 1];

3495 rc_free[cmd_save - 1].next = NULL;

@@ -3541,8 +3564,8 @@ int cmd_main(int argc, const char **argv)

3564 fprintf(stderr, "Total branches: %10lu (%10lu loads )\n", branch_count, branch_load_count);

3565 fprintf(stderr, " marks: %10" PRIuMAX " (%10" PRIuMAX " unique )\n", (((uintmax_t)1) << marks->shift) * 1024, marks_set_count);

3566 fprintf(stderr, " atoms: %10u\n", atom_cnt);

3544 - fprintf(stderr, "Memory total: %10" PRIuMAX " KiB\n", (total_allocd + alloc_count*sizeof(struct object_entry))/1024);

3545 - fprintf(stderr, " pools: %10lu KiB\n", (unsigned long)(total_allocd/1024));

3567 + fprintf(stderr, "Memory total: %10" PRIuMAX " KiB\n", (tree_entry_allocd + fi_mem_pool.pool_alloc + alloc_count*sizeof(struct object_entry))/1024);

3568 + fprintf(stderr, " pools: %10lu KiB\n", (unsigned long)((tree_entry_allocd + fi_mem_pool.pool_alloc) /1024));

3569 fprintf(stderr, " objects: %10" PRIuMAX " KiB\n", (alloc_count*sizeof(struct object_entry))/1024);

3570 fprintf(stderr, "---------------------------------------------------------------------\n");

3571 pack_report();