@matheusbsbs / matheus-bagliano / commits / 0ea9f045f4
Fork 0

Use uintmax_t for marks in fast-import.

If a frontend wants to use a mark per file revision and per commit and is doing a truly huge import (such as a 32 GiB SVN repository) we may need more than 2**32 unique mark values, especially if the frontend is unable (or unwilling) to recycle mark values. For mark idnums we should use the largest unsigned integer type available, hoping that will be at least 64 bits when we are compiled as a 64 bit executable. This way we may consume huge amounts of memory storing our mark table, but we'll at least be able to process the entire import without failing. Signed-off-by: Shawn O. Pearce <spearce@spearce.org>

Shawn O. Pearce committed Jan 16, 2007 at 00:33 UTC 0ea9f045f4eaa1d37c6b318d9d6849a4f447b997
2 files changed +48 -44
fast-import.c
+47 -44
@@ -215,18 +215,20 @@ struct hash_list
215 unsigned char sha1[20];
216 };
217
218 -/* Stats and misc. counters */
218 +/* Configured limits on output */
219 static unsigned long max_depth = 10;
220 -static unsigned long max_objects = -1;
220 static unsigned long max_packsize = -1;
222 -static unsigned long alloc_count;
221 +static uintmax_t max_objects = -1;
222 +
223 +/* Stats and misc. counters */
224 +static uintmax_t alloc_count;
225 +static uintmax_t object_count;
226 +static uintmax_t marks_set_count;
227 +static uintmax_t object_count_by_type[1 << TYPE_BITS];
228 +static uintmax_t duplicate_count_by_type[1 << TYPE_BITS];
229 +static uintmax_t delta_count_by_type[1 << TYPE_BITS];
230 static unsigned long branch_count;
231 static unsigned long branch_load_count;
225 -static unsigned long object_count;
226 -static unsigned long marks_set_count;
227 -static unsigned long object_count_by_type[1 << TYPE_BITS];
228 -static unsigned long duplicate_count_by_type[1 << TYPE_BITS];
229 -static unsigned long delta_count_by_type[1 << TYPE_BITS];
232
233 /* Memory pools */
234 static size_t mem_pool_alloc = 2*1024*1024 - sizeof(struct mem_pool);
@@ -279,7 +281,7 @@ static struct tag *last_tag;
281
282 /* Input stream parsing */
283 static struct strbuf command_buf;
282 -static unsigned long next_mark;
284 +static uintmax_t next_mark;
285 static struct dbuf new_data;
286 static FILE* branch_log;
287
@@ -406,7 +408,7 @@ static void size_dbuf(struct dbuf *b, size_t maxlen)
408 b->buffer = xmalloc(b->capacity);
409 }
410
409 -static void insert_mark(unsigned long idnum, struct object_entry *oe)
411 +static void insert_mark(uintmax_t idnum, struct object_entry *oe)
412 {
413 struct mark_set *s = marks;
414 while ((idnum >> s->shift) >= 1024) {
@@ -416,7 +418,7 @@ static void insert_mark(unsigned long idnum, struct object_entry *oe)
418 marks = s;
419 }
420 while (s->shift) {
419 - unsigned long i = idnum >> s->shift;
421 + uintmax_t i = idnum >> s->shift;
422 idnum -= i << s->shift;
423 if (!s->data.sets[i]) {
424 s->data.sets[i] = pool_calloc(1, sizeof(struct mark_set));
@@ -429,14 +431,14 @@ static void insert_mark(unsigned long idnum, struct object_entry *oe)
431 s->data.marked[idnum] = oe;
432 }
433
432 -static struct object_entry* find_mark(unsigned long idnum)
434 +static struct object_entry* find_mark(uintmax_t idnum)
435 {
434 - unsigned long orig_idnum = idnum;
436 + uintmax_t orig_idnum = idnum;
437 struct mark_set *s = marks;
438 struct object_entry *oe = NULL;
439 if ((idnum >> s->shift) < 1024) {
440 while (s && s->shift) {
439 - unsigned long i = idnum >> s->shift;
441 + uintmax_t i = idnum >> s->shift;
442 idnum -= i << s->shift;
443 s = s->data.sets[i];
444 }
@@ -444,7 +446,7 @@ static struct object_entry* find_mark(unsigned long idnum)
446 oe = s->data.marked[idnum];
447 }
448 if (!oe)
447 - die("mark :%lu not declared", orig_idnum);
449 + die("mark :%ju not declared", orig_idnum);
450 return oe;
451 }
452
@@ -781,7 +783,7 @@ static int store_object(
783 size_t datlen,
784 struct last_object *last,
785 unsigned char *sha1out,
784 - unsigned long mark)
786 + uintmax_t mark)
787 {
788 void *out, *delta;
789 struct object_entry *e;
@@ -1225,10 +1227,10 @@ static void dump_tags()
1227 }
1228
1229 static void dump_marks_helper(FILE *f,
1228 - unsigned long base,
1230 + uintmax_t base,
1231 struct mark_set *m)
1232 {
1231 - int k;
1233 + uintmax_t k;
1234 if (m->shift) {
1235 for (k = 0; k < 1024; k++) {
1236 if (m->data.sets[k])
@@ -1238,7 +1240,7 @@ static void dump_marks_helper(FILE *f,
1240 } else {
1241 for (k = 0; k < 1024; k++) {
1242 if (m->data.marked[k])
1241 - fprintf(f, ":%lu %s\n", base + k,
1243 + fprintf(f, ":%ju %s\n", base + k,
1244 sha1_to_hex(m->data.marked[k]->sha1));
1245 }
1246 }
@@ -1262,7 +1264,7 @@ static void read_next_command()
1264 static void cmd_mark()
1265 {
1266 if (!strncmp("mark :", command_buf.buf, 6)) {
1265 - next_mark = strtoul(command_buf.buf + 6, NULL, 10);
1267 + next_mark = strtoumax(command_buf.buf + 6, NULL, 10);
1268 read_next_command();
1269 }
1270 else
@@ -1375,7 +1377,7 @@ static void file_change_m(struct branch *b)
1377
1378 if (*p == ':') {
1379 char *x;
1378 - oe = find_mark(strtoul(p + 1, &x, 10));
1380 + oe = find_mark(strtoumax(p + 1, &x, 10));
1381 hashcpy(sha1, oe->sha1);
1382 p = x;
1383 } else {
@@ -1458,12 +1460,12 @@ static void cmd_from(struct branch *b)
1460 hashcpy(b->branch_tree.versions[0].sha1, t);
1461 hashcpy(b->branch_tree.versions[1].sha1, t);
1462 } else if (*from == ':') {
1461 - unsigned long idnum = strtoul(from + 1, NULL, 10);
1463 + uintmax_t idnum = strtoumax(from + 1, NULL, 10);
1464 struct object_entry *oe = find_mark(idnum);
1465 unsigned long size;
1466 char *buf;
1467 if (oe->type != OBJ_COMMIT)
1466 - die("Mark :%lu not a commit", idnum);
1468 + die("Mark :%ju not a commit", idnum);
1469 hashcpy(b->sha1, oe->sha1);
1470 buf = gfi_unpack_entry(oe, &size);
1471 if (!buf || size < 46)
@@ -1521,10 +1523,10 @@ static struct hash_list* cmd_merge(unsigned int *count)
1523 if (s)
1524 hashcpy(n->sha1, s->sha1);
1525 else if (*from == ':') {
1524 - unsigned long idnum = strtoul(from + 1, NULL, 10);
1526 + uintmax_t idnum = strtoumax(from + 1, NULL, 10);
1527 struct object_entry *oe = find_mark(idnum);
1528 if (oe->type != OBJ_COMMIT)
1527 - die("Mark :%lu not a commit", idnum);
1529 + die("Mark :%ju not a commit", idnum);
1530 hashcpy(n->sha1, oe->sha1);
1531 } else if (get_sha1(from, n->sha1))
1532 die("Invalid ref name or SHA1 expression: %s", from);
@@ -1650,7 +1652,7 @@ static void cmd_new_commit()
1652 fputc('"', branch_log);
1653 } else
1654 fprintf(branch_log, "%s", b->name);
1653 - fprintf(branch_log," :%lu %s\n",next_mark,sha1_to_hex(b->sha1));
1655 + fprintf(branch_log," :%ju %s\n",next_mark,sha1_to_hex(b->sha1));
1656 }
1657 }
1658
@@ -1665,7 +1667,7 @@ static void cmd_new_tag()
1667 void *msg;
1668 size_t msglen;
1669 struct tag *t;
1668 - unsigned long from_mark = 0;
1670 + uintmax_t from_mark = 0;
1671 unsigned char sha1[20];
1672
1673 /* Obtain the new tag name from the rest of our command */
@@ -1704,10 +1706,10 @@ static void cmd_new_tag()
1706 if (s) {
1707 hashcpy(sha1, s->sha1);
1708 } else if (*from == ':') {
1707 - from_mark = strtoul(from + 1, NULL, 10);
1709 + from_mark = strtoumax(from + 1, NULL, 10);
1710 struct object_entry *oe = find_mark(from_mark);
1711 if (oe->type != OBJ_COMMIT)
1710 - die("Mark :%lu not a commit", from_mark);
1712 + die("Mark :%ju not a commit", from_mark);
1713 hashcpy(sha1, oe->sha1);
1714 } else if (!get_sha1(from, sha1)) {
1715 unsigned long size;
@@ -1758,7 +1760,7 @@ static void cmd_new_tag()
1760 fputc('"', branch_log);
1761 } else
1762 fprintf(branch_log, "%s", t->name);
1761 - fprintf(branch_log," :%lu %s\n",from_mark,sha1_to_hex(t->sha1));
1763 + fprintf(branch_log," :%ju %s\n",from_mark,sha1_to_hex(t->sha1));
1764 }
1765 }
1766
@@ -1806,8 +1808,8 @@ static const char fast_import_usage[] =
1808 int main(int argc, const char **argv)
1809 {
1810 int i;
1809 - unsigned long est_obj_cnt = object_entry_alloc;
1810 - unsigned long duplicate_count;
1811 + uintmax_t est_obj_cnt = object_entry_alloc;
1812 + uintmax_t duplicate_count;
1813
1814 setup_ident();
1815 git_config(git_default_config);
@@ -1818,11 +1820,11 @@ int main(int argc, const char **argv)
1820 if (*a != '-' || !strcmp(a, "--"))
1821 break;
1822 else if (!strncmp(a, "--objects=", 10))
1821 - est_obj_cnt = strtoul(a + 10, NULL, 0);
1823 + est_obj_cnt = strtoumax(a + 10, NULL, 0);
1824 else if (!strncmp(a, "--max-objects-per-pack=", 23))
1823 - max_objects = strtoul(a + 23, NULL, 0);
1825 + max_objects = strtoumax(a + 23, NULL, 0);
1826 else if (!strncmp(a, "--max-pack-size=", 16))
1825 - max_packsize = strtoul(a + 16, NULL, 0) * 1024 * 1024;
1827 + max_packsize = strtoumax(a + 16, NULL, 0) * 1024 * 1024;
1828 else if (!strncmp(a, "--depth=", 8))
1829 max_depth = strtoul(a + 8, NULL, 0);
1830 else if (!strncmp(a, "--active-branches=", 18))
@@ -1875,23 +1877,24 @@ int main(int argc, const char **argv)
1877 if (branch_log)
1878 fclose(branch_log);
1879
1880 + duplicate_count = 0;
1881 for (i = 0; i < ARRAY_SIZE(duplicate_count_by_type); i++)
1882 duplicate_count += duplicate_count_by_type[i];
1883
1884 fprintf(stderr, "%s statistics:\n", argv[0]);
1885 fprintf(stderr, "---------------------------------------------------------------------\n");
1883 - fprintf(stderr, "Alloc'd objects: %10lu (%10lu overflow )\n", alloc_count, alloc_count - est_obj_cnt);
1884 - fprintf(stderr, "Total objects: %10lu (%10lu duplicates )\n", object_count, duplicate_count);
1885 - fprintf(stderr, " blobs : %10lu (%10lu duplicates %10lu deltas)\n", object_count_by_type[OBJ_BLOB], duplicate_count_by_type[OBJ_BLOB], delta_count_by_type[OBJ_BLOB]);
1886 - fprintf(stderr, " trees : %10lu (%10lu duplicates %10lu deltas)\n", object_count_by_type[OBJ_TREE], duplicate_count_by_type[OBJ_TREE], delta_count_by_type[OBJ_TREE]);
1887 - fprintf(stderr, " commits: %10lu (%10lu duplicates %10lu deltas)\n", object_count_by_type[OBJ_COMMIT], duplicate_count_by_type[OBJ_COMMIT], delta_count_by_type[OBJ_COMMIT]);
1888 - fprintf(stderr, " tags : %10lu (%10lu duplicates %10lu deltas)\n", object_count_by_type[OBJ_TAG], duplicate_count_by_type[OBJ_TAG], delta_count_by_type[OBJ_TAG]);
1886 + fprintf(stderr, "Alloc'd objects: %10ju (%10ju overflow )\n", alloc_count, alloc_count - est_obj_cnt);
1887 + fprintf(stderr, "Total objects: %10ju (%10ju duplicates )\n", object_count, duplicate_count);
1888 + fprintf(stderr, " blobs : %10ju (%10ju duplicates %10ju deltas)\n", object_count_by_type[OBJ_BLOB], duplicate_count_by_type[OBJ_BLOB], delta_count_by_type[OBJ_BLOB]);
1889 + fprintf(stderr, " trees : %10ju (%10ju duplicates %10ju deltas)\n", object_count_by_type[OBJ_TREE], duplicate_count_by_type[OBJ_TREE], delta_count_by_type[OBJ_TREE]);
1890 + fprintf(stderr, " commits: %10ju (%10ju duplicates %10ju deltas)\n", object_count_by_type[OBJ_COMMIT], duplicate_count_by_type[OBJ_COMMIT], delta_count_by_type[OBJ_COMMIT]);
1891 + fprintf(stderr, " tags : %10ju (%10ju duplicates %10ju deltas)\n", object_count_by_type[OBJ_TAG], duplicate_count_by_type[OBJ_TAG], delta_count_by_type[OBJ_TAG]);
1892 fprintf(stderr, "Total branches: %10lu (%10lu loads )\n", branch_count, branch_load_count);
1890 - fprintf(stderr, " marks: %10u (%10lu unique )\n", (1 << marks->shift) * 1024, marks_set_count);
1893 + fprintf(stderr, " marks: %10ju (%10ju unique )\n", (((uintmax_t)1) << marks->shift) * 1024, marks_set_count);
1894 fprintf(stderr, " atoms: %10u\n", atom_cnt);
1892 - fprintf(stderr, "Memory total: %10lu KiB\n", (total_allocd + alloc_count*sizeof(struct object_entry))/1024);
1895 + fprintf(stderr, "Memory total: %10ju KiB\n", (total_allocd + alloc_count*sizeof(struct object_entry))/1024);
1896 fprintf(stderr, " pools: %10lu KiB\n", total_allocd/1024);
1894 - fprintf(stderr, " objects: %10lu KiB\n", (alloc_count*sizeof(struct object_entry))/1024);
1897 + fprintf(stderr, " objects: %10ju KiB\n", (alloc_count*sizeof(struct object_entry))/1024);
1898 fprintf(stderr, "---------------------------------------------------------------------\n");
1899 pack_report();
1900 fprintf(stderr, "---------------------------------------------------------------------\n");
git-compat-util.h
+1
@@ -28,6 +28,7 @@
28 #include <string.h>
29 #include <errno.h>
30 #include <limits.h>
31 +#include <inttypes.h>
32 #include <sys/param.h>
33 #include <sys/types.h>
34 #include <dirent.h>