Use uintmax_t for marks in fast-import.
If a frontend wants to use a mark per file revision and per commit and is doing a truly huge import (such as a 32 GiB SVN repository) we may need more than 2**32 unique mark values, especially if the frontend is unable (or unwilling) to recycle mark values. For mark idnums we should use the largest unsigned integer type available, hoping that will be at least 64 bits when we are compiled as a 64 bit executable. This way we may consume huge amounts of memory storing our mark table, but we'll at least be able to process the entire import without failing. Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Shawn O. Pearce committed
Jan 16, 2007 at 00:33 UTC
0ea9f045f4eaa1d37c6b318d9d6849a4f447b997
2 files changed
+48
-44
fast-import.c
+47
-44
@@ -215,18 +215,20 @@ struct hash_list
215
unsigned char sha1[20];
216
};
217
218
-/* Stats and misc. counters */
218
+/* Configured limits on output */
219
static unsigned long max_depth = 10;
220
-static unsigned long max_objects = -1;
220
static unsigned long max_packsize = -1;
222
-static unsigned long alloc_count;
221
+static uintmax_t max_objects = -1;
222
+
223
+/* Stats and misc. counters */
224
+static uintmax_t alloc_count;
225
+static uintmax_t object_count;
226
+static uintmax_t marks_set_count;
227
+static uintmax_t object_count_by_type[1 << TYPE_BITS];
228
+static uintmax_t duplicate_count_by_type[1 << TYPE_BITS];
229
+static uintmax_t delta_count_by_type[1 << TYPE_BITS];
230
static unsigned long branch_count;
231
static unsigned long branch_load_count;
225
-static unsigned long object_count;
226
-static unsigned long marks_set_count;
227
-static unsigned long object_count_by_type[1 << TYPE_BITS];
228
-static unsigned long duplicate_count_by_type[1 << TYPE_BITS];
229
-static unsigned long delta_count_by_type[1 << TYPE_BITS];
232
233
/* Memory pools */
234
static size_t mem_pool_alloc = 2*1024*1024 - sizeof(struct mem_pool);
@@ -279,7 +281,7 @@ static struct tag *last_tag;
281
282
/* Input stream parsing */
283
static struct strbuf command_buf;
282
-static unsigned long next_mark;
284
+static uintmax_t next_mark;
285
static struct dbuf new_data;
286
static FILE* branch_log;
287
@@ -406,7 +408,7 @@ static void size_dbuf(struct dbuf *b, size_t maxlen)
408
b->buffer = xmalloc(b->capacity);
409
}
410
409
-static void insert_mark(unsigned long idnum, struct object_entry *oe)
411
+static void insert_mark(uintmax_t idnum, struct object_entry *oe)
412
{
413
struct mark_set *s = marks;
414
while ((idnum >> s->shift) >= 1024) {
@@ -416,7 +418,7 @@ static void insert_mark(unsigned long idnum, struct object_entry *oe)
418
marks = s;
419
}
420
while (s->shift) {
419
- unsigned long i = idnum >> s->shift;
421
+ uintmax_t i = idnum >> s->shift;
422
idnum -= i << s->shift;
423
if (!s->data.sets[i]) {
424
s->data.sets[i] = pool_calloc(1, sizeof(struct mark_set));
@@ -429,14 +431,14 @@ static void insert_mark(unsigned long idnum, struct object_entry *oe)
431
s->data.marked[idnum] = oe;
432
}
433
432
-static struct object_entry* find_mark(unsigned long idnum)
434
+static struct object_entry* find_mark(uintmax_t idnum)
435
{
434
- unsigned long orig_idnum = idnum;
436
+ uintmax_t orig_idnum = idnum;
437
struct mark_set *s = marks;
438
struct object_entry *oe = NULL;
439
if ((idnum >> s->shift) < 1024) {
440
while (s && s->shift) {
439
- unsigned long i = idnum >> s->shift;
441
+ uintmax_t i = idnum >> s->shift;
442
idnum -= i << s->shift;
443
s = s->data.sets[i];
444
}
@@ -444,7 +446,7 @@ static struct object_entry* find_mark(unsigned long idnum)
446
oe = s->data.marked[idnum];
447
}
448
if (!oe)
447
- die("mark :%lu not declared", orig_idnum);
449
+ die("mark :%ju not declared", orig_idnum);
450
return oe;
451
}
452
@@ -781,7 +783,7 @@ static int store_object(
783
size_t datlen,
784
struct last_object *last,
785
unsigned char *sha1out,
784
- unsigned long mark)
786
+ uintmax_t mark)
787
{
788
void *out, *delta;
789
struct object_entry *e;
@@ -1225,10 +1227,10 @@ static void dump_tags()
1227
}
1228
1229
static void dump_marks_helper(FILE *f,
1228
- unsigned long base,
1230
+ uintmax_t base,
1231
struct mark_set *m)
1232
{
1231
- int k;
1233
+ uintmax_t k;
1234
if (m->shift) {
1235
for (k = 0; k < 1024; k++) {
1236
if (m->data.sets[k])
@@ -1238,7 +1240,7 @@ static void dump_marks_helper(FILE *f,
1240
} else {
1241
for (k = 0; k < 1024; k++) {
1242
if (m->data.marked[k])
1241
- fprintf(f, ":%lu %s\n", base + k,
1243
+ fprintf(f, ":%ju %s\n", base + k,
1244
sha1_to_hex(m->data.marked[k]->sha1));
1245
}
1246
}
@@ -1262,7 +1264,7 @@ static void read_next_command()
1264
static void cmd_mark()
1265
{
1266
if (!strncmp("mark :", command_buf.buf, 6)) {
1265
- next_mark = strtoul(command_buf.buf + 6, NULL, 10);
1267
+ next_mark = strtoumax(command_buf.buf + 6, NULL, 10);
1268
read_next_command();
1269
}
1270
else
@@ -1375,7 +1377,7 @@ static void file_change_m(struct branch *b)
1377
1378
if (*p == ':') {
1379
char *x;
1378
- oe = find_mark(strtoul(p + 1, &x, 10));
1380
+ oe = find_mark(strtoumax(p + 1, &x, 10));
1381
hashcpy(sha1, oe->sha1);
1382
p = x;
1383
} else {
@@ -1458,12 +1460,12 @@ static void cmd_from(struct branch *b)
1460
hashcpy(b->branch_tree.versions[0].sha1, t);
1461
hashcpy(b->branch_tree.versions[1].sha1, t);
1462
} else if (*from == ':') {
1461
- unsigned long idnum = strtoul(from + 1, NULL, 10);
1463
+ uintmax_t idnum = strtoumax(from + 1, NULL, 10);
1464
struct object_entry *oe = find_mark(idnum);
1465
unsigned long size;
1466
char *buf;
1467
if (oe->type != OBJ_COMMIT)
1466
- die("Mark :%lu not a commit", idnum);
1468
+ die("Mark :%ju not a commit", idnum);
1469
hashcpy(b->sha1, oe->sha1);
1470
buf = gfi_unpack_entry(oe, &size);
1471
if (!buf || size < 46)
@@ -1521,10 +1523,10 @@ static struct hash_list* cmd_merge(unsigned int *count)
1523
if (s)
1524
hashcpy(n->sha1, s->sha1);
1525
else if (*from == ':') {
1524
- unsigned long idnum = strtoul(from + 1, NULL, 10);
1526
+ uintmax_t idnum = strtoumax(from + 1, NULL, 10);
1527
struct object_entry *oe = find_mark(idnum);
1528
if (oe->type != OBJ_COMMIT)
1527
- die("Mark :%lu not a commit", idnum);
1529
+ die("Mark :%ju not a commit", idnum);
1530
hashcpy(n->sha1, oe->sha1);
1531
} else if (get_sha1(from, n->sha1))
1532
die("Invalid ref name or SHA1 expression: %s", from);
@@ -1650,7 +1652,7 @@ static void cmd_new_commit()
1652
fputc('"', branch_log);
1653
} else
1654
fprintf(branch_log, "%s", b->name);
1653
- fprintf(branch_log," :%lu %s\n",next_mark,sha1_to_hex(b->sha1));
1655
+ fprintf(branch_log," :%ju %s\n",next_mark,sha1_to_hex(b->sha1));
1656
}
1657
}
1658
@@ -1665,7 +1667,7 @@ static void cmd_new_tag()
1667
void *msg;
1668
size_t msglen;
1669
struct tag *t;
1668
- unsigned long from_mark = 0;
1670
+ uintmax_t from_mark = 0;
1671
unsigned char sha1[20];
1672
1673
/* Obtain the new tag name from the rest of our command */
@@ -1704,10 +1706,10 @@ static void cmd_new_tag()
1706
if (s) {
1707
hashcpy(sha1, s->sha1);
1708
} else if (*from == ':') {
1707
- from_mark = strtoul(from + 1, NULL, 10);
1709
+ from_mark = strtoumax(from + 1, NULL, 10);
1710
struct object_entry *oe = find_mark(from_mark);
1711
if (oe->type != OBJ_COMMIT)
1710
- die("Mark :%lu not a commit", from_mark);
1712
+ die("Mark :%ju not a commit", from_mark);
1713
hashcpy(sha1, oe->sha1);
1714
} else if (!get_sha1(from, sha1)) {
1715
unsigned long size;
@@ -1758,7 +1760,7 @@ static void cmd_new_tag()
1760
fputc('"', branch_log);
1761
} else
1762
fprintf(branch_log, "%s", t->name);
1761
- fprintf(branch_log," :%lu %s\n",from_mark,sha1_to_hex(t->sha1));
1763
+ fprintf(branch_log," :%ju %s\n",from_mark,sha1_to_hex(t->sha1));
1764
}
1765
}
1766
@@ -1806,8 +1808,8 @@ static const char fast_import_usage[] =
1808
int main(int argc, const char **argv)
1809
{
1810
int i;
1809
- unsigned long est_obj_cnt = object_entry_alloc;
1810
- unsigned long duplicate_count;
1811
+ uintmax_t est_obj_cnt = object_entry_alloc;
1812
+ uintmax_t duplicate_count;
1813
1814
setup_ident();
1815
git_config(git_default_config);
@@ -1818,11 +1820,11 @@ int main(int argc, const char **argv)
1820
if (*a != '-' || !strcmp(a, "--"))
1821
break;
1822
else if (!strncmp(a, "--objects=", 10))
1821
- est_obj_cnt = strtoul(a + 10, NULL, 0);
1823
+ est_obj_cnt = strtoumax(a + 10, NULL, 0);
1824
else if (!strncmp(a, "--max-objects-per-pack=", 23))
1823
- max_objects = strtoul(a + 23, NULL, 0);
1825
+ max_objects = strtoumax(a + 23, NULL, 0);
1826
else if (!strncmp(a, "--max-pack-size=", 16))
1825
- max_packsize = strtoul(a + 16, NULL, 0) * 1024 * 1024;
1827
+ max_packsize = strtoumax(a + 16, NULL, 0) * 1024 * 1024;
1828
else if (!strncmp(a, "--depth=", 8))
1829
max_depth = strtoul(a + 8, NULL, 0);
1830
else if (!strncmp(a, "--active-branches=", 18))
@@ -1875,23 +1877,24 @@ int main(int argc, const char **argv)
1877
if (branch_log)
1878
fclose(branch_log);
1879
1880
+ duplicate_count = 0;
1881
for (i = 0; i < ARRAY_SIZE(duplicate_count_by_type); i++)
1882
duplicate_count += duplicate_count_by_type[i];
1883
1884
fprintf(stderr, "%s statistics:\n", argv[0]);
1885
fprintf(stderr, "---------------------------------------------------------------------\n");
1883
- fprintf(stderr, "Alloc'd objects: %10lu (%10lu overflow )\n", alloc_count, alloc_count - est_obj_cnt);
1884
- fprintf(stderr, "Total objects: %10lu (%10lu duplicates )\n", object_count, duplicate_count);
1885
- fprintf(stderr, " blobs : %10lu (%10lu duplicates %10lu deltas)\n", object_count_by_type[OBJ_BLOB], duplicate_count_by_type[OBJ_BLOB], delta_count_by_type[OBJ_BLOB]);
1886
- fprintf(stderr, " trees : %10lu (%10lu duplicates %10lu deltas)\n", object_count_by_type[OBJ_TREE], duplicate_count_by_type[OBJ_TREE], delta_count_by_type[OBJ_TREE]);
1887
- fprintf(stderr, " commits: %10lu (%10lu duplicates %10lu deltas)\n", object_count_by_type[OBJ_COMMIT], duplicate_count_by_type[OBJ_COMMIT], delta_count_by_type[OBJ_COMMIT]);
1888
- fprintf(stderr, " tags : %10lu (%10lu duplicates %10lu deltas)\n", object_count_by_type[OBJ_TAG], duplicate_count_by_type[OBJ_TAG], delta_count_by_type[OBJ_TAG]);
1886
+ fprintf(stderr, "Alloc'd objects: %10ju (%10ju overflow )\n", alloc_count, alloc_count - est_obj_cnt);
1887
+ fprintf(stderr, "Total objects: %10ju (%10ju duplicates )\n", object_count, duplicate_count);
1888
+ fprintf(stderr, " blobs : %10ju (%10ju duplicates %10ju deltas)\n", object_count_by_type[OBJ_BLOB], duplicate_count_by_type[OBJ_BLOB], delta_count_by_type[OBJ_BLOB]);
1889
+ fprintf(stderr, " trees : %10ju (%10ju duplicates %10ju deltas)\n", object_count_by_type[OBJ_TREE], duplicate_count_by_type[OBJ_TREE], delta_count_by_type[OBJ_TREE]);
1890
+ fprintf(stderr, " commits: %10ju (%10ju duplicates %10ju deltas)\n", object_count_by_type[OBJ_COMMIT], duplicate_count_by_type[OBJ_COMMIT], delta_count_by_type[OBJ_COMMIT]);
1891
+ fprintf(stderr, " tags : %10ju (%10ju duplicates %10ju deltas)\n", object_count_by_type[OBJ_TAG], duplicate_count_by_type[OBJ_TAG], delta_count_by_type[OBJ_TAG]);
1892
fprintf(stderr, "Total branches: %10lu (%10lu loads )\n", branch_count, branch_load_count);
1890
- fprintf(stderr, " marks: %10u (%10lu unique )\n", (1 << marks->shift) * 1024, marks_set_count);
1893
+ fprintf(stderr, " marks: %10ju (%10ju unique )\n", (((uintmax_t)1) << marks->shift) * 1024, marks_set_count);
1894
fprintf(stderr, " atoms: %10u\n", atom_cnt);
1892
- fprintf(stderr, "Memory total: %10lu KiB\n", (total_allocd + alloc_count*sizeof(struct object_entry))/1024);
1895
+ fprintf(stderr, "Memory total: %10ju KiB\n", (total_allocd + alloc_count*sizeof(struct object_entry))/1024);
1896
fprintf(stderr, " pools: %10lu KiB\n", total_allocd/1024);
1894
- fprintf(stderr, " objects: %10lu KiB\n", (alloc_count*sizeof(struct object_entry))/1024);
1897
+ fprintf(stderr, " objects: %10ju KiB\n", (alloc_count*sizeof(struct object_entry))/1024);
1898
fprintf(stderr, "---------------------------------------------------------------------\n");
1899
pack_report();
1900
fprintf(stderr, "---------------------------------------------------------------------\n");
git-compat-util.h
+1
@@ -28,6 +28,7 @@
28
#include <string.h>
29
#include <errno.h>
30
#include <limits.h>
31
+#include <inttypes.h>
32
#include <sys/param.h>
33
#include <sys/types.h>
34
#include <dirent.h>