Add string-specific memory pool
Intern strings so they can be compared by address and stored without wasting space. This library uses the macros in the obj_pool.h and trp.h to create a memory pool for strings and expose an API for handling them. [rr: added API docs] [jn: with some API simplifications, new documentation and tests] Signed-off-by: David Barr <david.barr@cordelta.com> Signed-off-by: Ramkumar Ramachandra <artagnon@gmail.com> Signed-off-by: Jonathan Nieder <jrnieder@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
David Barr committed
Aug 9, 2010 at 17:34 UTC
1d73b52f5ba4184de6acf474f14668001304a10c
7 files changed
+210
-3
.gitignore
+1
@@ -173,6 +173,7 @@
173
/test-run-command
174
/test-sha1
175
/test-sigchain
176
+/test-string-pool
177
/test-treap
178
/common-cmds.h
179
*.tar.gz
Makefile
+6
-3
@@ -415,6 +415,7 @@ TEST_PROGRAMS_NEED_X += test-path-utils
415
TEST_PROGRAMS_NEED_X += test-run-command
416
TEST_PROGRAMS_NEED_X += test-sha1
417
TEST_PROGRAMS_NEED_X += test-sigchain
418
+TEST_PROGRAMS_NEED_X += test-string-pool
419
TEST_PROGRAMS_NEED_X += test-treap
420
TEST_PROGRAMS_NEED_X += test-index-version
421
@@ -1742,7 +1743,7 @@ ifndef NO_CURL
1743
endif
1744
XDIFF_OBJS = xdiff/xdiffi.o xdiff/xprepare.o xdiff/xutils.o xdiff/xemit.o \
1745
xdiff/xmerge.o xdiff/xpatience.o
1745
-VCSSVN_OBJS =
1746
+VCSSVN_OBJS = vcs-svn/string_pool.o
1747
OBJECTS := $(GIT_OBJS) $(XDIFF_OBJS) $(VCSSVN_OBJS)
1748
1749
dep_files := $(foreach f,$(OBJECTS),$(dir $f).depend/$(notdir $f).d)
@@ -1867,7 +1868,7 @@ xdiff-interface.o $(XDIFF_OBJS): \
1868
xdiff/xutils.h xdiff/xprepare.h xdiff/xdiffi.h xdiff/xemit.h
1869
1870
$(VCSSVN_OBJS): \
1870
- vcs-svn/obj_pool.h vcs-svn/trp.h
1871
+ vcs-svn/obj_pool.h vcs-svn/trp.h vcs-svn/string_pool.h
1872
endif
1873
1874
exec_cmd.s exec_cmd.o: EXTRA_CPPFLAGS = \
@@ -2018,10 +2019,12 @@ test-delta$X: diff-delta.o patch-delta.o
2019
2020
test-parse-options$X: parse-options.o
2021
2022
+test-string-pool$X: vcs-svn/lib.a
2023
+
2024
.PRECIOUS: $(TEST_OBJS)
2025
2026
test-%$X: test-%.o $(GITLIBS)
2024
- $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) $(LIBS)
2027
+ $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) $(filter %.a,$^) $(LIBS)
2028
2029
check-sha1:: test-sha1$X
2030
./test-sha1.sh
t/t0080-vcs-svn.sh
+16
@@ -76,6 +76,22 @@ test_expect_success 'obj pool: high-water mark' '
76
test_cmp expected actual
77
'
78
79
+test_expect_success 'string pool' '
80
+ echo a does not equal b >expected.differ &&
81
+ echo a equals a >expected.match &&
82
+ echo equals equals equals >expected.matchmore &&
83
+
84
+ test-string-pool "a,--b" >actual.differ &&
85
+ test-string-pool "a,a" >actual.match &&
86
+ test-string-pool "equals-equals" >actual.matchmore &&
87
+ test_must_fail test-string-pool a,a,a &&
88
+ test_must_fail test-string-pool a &&
89
+
90
+ test_cmp expected.differ actual.differ &&
91
+ test_cmp expected.match actual.match &&
92
+ test_cmp expected.matchmore actual.matchmore
93
+'
94
+
95
test_expect_success 'treap sort' '
96
cat <<-\EOF >unsorted &&
97
68
test-string-pool.c
new
+31
@@ -0,0 +1,31 @@
1
+/*
2
+ * test-string-pool.c: code to exercise the svn importer's string pool
3
+ */
4
+
5
+#include "git-compat-util.h"
6
+#include "vcs-svn/string_pool.h"
7
+
8
+int main(int argc, char *argv[])
9
+{
10
+ const uint32_t unequal = pool_intern("does not equal");
11
+ const uint32_t equal = pool_intern("equals");
12
+ uint32_t buf[3];
13
+ uint32_t n;
14
+
15
+ if (argc != 2)
16
+ usage("test-string-pool <string>,<string>");
17
+
18
+ n = pool_tok_seq(3, buf, ",-", argv[1]);
19
+ if (n >= 3)
20
+ die("too many strings");
21
+ if (n <= 1)
22
+ die("too few strings");
23
+
24
+ buf[2] = buf[1];
25
+ buf[1] = (buf[0] == buf[2]) ? equal : unequal;
26
+ pool_print_seq(3, buf, ' ', stdout);
27
+ fputc('\n', stdout);
28
+
29
+ pool_reset();
30
+ return 0;
31
+}
vcs-svn/string_pool.c
new
+102
@@ -0,0 +1,102 @@
1
+/*
2
+ * Licensed under a two-clause BSD-style license.
3
+ * See LICENSE for details.
4
+ */
5
+
6
+#include "git-compat-util.h"
7
+#include "trp.h"
8
+#include "obj_pool.h"
9
+#include "string_pool.h"
10
+
11
+static struct trp_root tree = { ~0 };
12
+
13
+struct node {
14
+ uint32_t offset;
15
+ struct trp_node children;
16
+};
17
+
18
+/* Two memory pools: one for struct node, and another for strings */
19
+obj_pool_gen(node, struct node, 4096)
20
+obj_pool_gen(string, char, 4096)
21
+
22
+static char *node_value(struct node *node)
23
+{
24
+ return node ? string_pointer(node->offset) : NULL;
25
+}
26
+
27
+static int node_cmp(struct node *a, struct node *b)
28
+{
29
+ return strcmp(node_value(a), node_value(b));
30
+}
31
+
32
+/* Build a Treap from the node structure (a trp_node w/ offset) */
33
+trp_gen(static, tree_, struct node, children, node, node_cmp);
34
+
35
+const char *pool_fetch(uint32_t entry)
36
+{
37
+ return node_value(node_pointer(entry));
38
+}
39
+
40
+uint32_t pool_intern(const char *key)
41
+{
42
+ /* Canonicalize key */
43
+ struct node *match = NULL, *node;
44
+ uint32_t key_len;
45
+ if (key == NULL)
46
+ return ~0;
47
+ key_len = strlen(key) + 1;
48
+ node = node_pointer(node_alloc(1));
49
+ node->offset = string_alloc(key_len);
50
+ strcpy(node_value(node), key);
51
+ match = tree_search(&tree, node);
52
+ if (!match) {
53
+ tree_insert(&tree, node);
54
+ } else {
55
+ node_free(1);
56
+ string_free(key_len);
57
+ node = match;
58
+ }
59
+ return node_offset(node);
60
+}
61
+
62
+uint32_t pool_tok_r(char *str, const char *delim, char **saveptr)
63
+{
64
+ char *token = strtok_r(str, delim, saveptr);
65
+ return token ? pool_intern(token) : ~0;
66
+}
67
+
68
+void pool_print_seq(uint32_t len, uint32_t *seq, char delim, FILE *stream)
69
+{
70
+ uint32_t i;
71
+ for (i = 0; i < len && ~seq[i]; i++) {
72
+ fputs(pool_fetch(seq[i]), stream);
73
+ if (i < len - 1 && ~seq[i + 1])
74
+ fputc(delim, stream);
75
+ }
76
+}
77
+
78
+uint32_t pool_tok_seq(uint32_t sz, uint32_t *seq, const char *delim, char *str)
79
+{
80
+ char *context = NULL;
81
+ uint32_t token = ~0;
82
+ uint32_t length;
83
+
84
+ if (sz == 0)
85
+ return ~0;
86
+ if (str)
87
+ token = pool_tok_r(str, delim, &context);
88
+ for (length = 0; length < sz; length++) {
89
+ seq[length] = token;
90
+ if (token == ~0)
91
+ return length;
92
+ token = pool_tok_r(NULL, delim, &context);
93
+ }
94
+ seq[sz - 1] = ~0;
95
+ return sz;
96
+}
97
+
98
+void pool_reset(void)
99
+{
100
+ node_reset();
101
+ string_reset();
102
+}
vcs-svn/string_pool.h
new
+11
@@ -0,0 +1,11 @@
1
+#ifndef STRING_POOL_H_
2
+#define STRING_POOL_H_
3
+
4
+uint32_t pool_intern(const char *key);
5
+const char *pool_fetch(uint32_t entry);
6
+uint32_t pool_tok_r(char *str, const char *delim, char **saveptr);
7
+void pool_print_seq(uint32_t len, uint32_t *seq, char delim, FILE *stream);
8
+uint32_t pool_tok_seq(uint32_t sz, uint32_t *seq, const char *delim, char *str);
9
+void pool_reset(void);
10
+
11
+#endif
vcs-svn/string_pool.txt
new
+43
@@ -0,0 +1,43 @@
1
+string_pool API
2
+===============
3
+
4
+The string_pool API provides facilities for replacing strings
5
+with integer keys that can be more easily compared and stored.
6
+The facilities are designed so that one could teach Git without
7
+too much trouble to store the information needed for these keys to
8
+remain valid over multiple executions.
9
+
10
+Functions
11
+---------
12
+
13
+pool_intern::
14
+ Include a string in the string pool and get its key.
15
+ If that string is already in the pool, retrieves its
16
+ existing key.
17
+
18
+pool_fetch::
19
+ Retrieve the string associated to a given key.
20
+
21
+pool_tok_r::
22
+ Extract the key of the next token from a string.
23
+ Interface mimics strtok_r.
24
+
25
+pool_print_seq::
26
+ Print a sequence of strings named by key to a file, using the
27
+ specified delimiter to separate them.
28
+
29
+ If NULL (key ~0) appears in the sequence, the sequence ends
30
+ early.
31
+
32
+pool_tok_seq::
33
+ Split a string into tokens, storing the keys of segments
34
+ into a caller-provided array.
35
+
36
+ Unless sz is 0, the array will always be ~0-terminated.
37
+ If there is not enough room for all the tokens, the
38
+ array holds as many tokens as fit in the entries before
39
+ the terminating ~0. Return value is the index after the
40
+ last token, or sz if the tokens did not fit.
41
+
42
+pool_reset::
43
+ Deallocate storage for the string pool.