Add stream helper library
This library provides thread-unsafe fgets()- and fread()-like functions where the caller does not have to supply a buffer. It maintains a couple of static buffers and provides an API to use them. [rr: allow input from files other than stdin] [jn: with tests, documentation, and error handling improvements] Signed-off-by: David Barr <david.barr@cordelta.com> Signed-off-by: Ramkumar Ramachandra <artagnon@gmail.com> Signed-off-by: Jonathan Nieder <jrnieder@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
David Barr committed
Aug 9, 2010 at 17:39 UTC
3bbaec00a8ffc6ea7e71c3b707851fe663d93a45
7 files changed
+274
-2
.gitignore
+1
@@ -166,6 +166,7 @@
166
/test-dump-cache-tree
167
/test-genrandom
168
/test-index-version
169
+/test-line-buffer
170
/test-match-trees
171
/test-obj-pool
172
/test-parse-options
Makefile
+6
-2
@@ -408,6 +408,7 @@ TEST_PROGRAMS_NEED_X += test-date
408
TEST_PROGRAMS_NEED_X += test-delta
409
TEST_PROGRAMS_NEED_X += test-dump-cache-tree
410
TEST_PROGRAMS_NEED_X += test-genrandom
411
+TEST_PROGRAMS_NEED_X += test-line-buffer
412
TEST_PROGRAMS_NEED_X += test-match-trees
413
TEST_PROGRAMS_NEED_X += test-obj-pool
414
TEST_PROGRAMS_NEED_X += test-parse-options
@@ -1743,7 +1744,7 @@ ifndef NO_CURL
1744
endif
1745
XDIFF_OBJS = xdiff/xdiffi.o xdiff/xprepare.o xdiff/xutils.o xdiff/xemit.o \
1746
xdiff/xmerge.o xdiff/xpatience.o
1746
-VCSSVN_OBJS = vcs-svn/string_pool.o
1747
+VCSSVN_OBJS = vcs-svn/string_pool.o vcs-svn/line_buffer.o
1748
OBJECTS := $(GIT_OBJS) $(XDIFF_OBJS) $(VCSSVN_OBJS)
1749
1750
dep_files := $(foreach f,$(OBJECTS),$(dir $f).depend/$(notdir $f).d)
@@ -1868,7 +1869,8 @@ xdiff-interface.o $(XDIFF_OBJS): \
1869
xdiff/xutils.h xdiff/xprepare.h xdiff/xdiffi.h xdiff/xemit.h
1870
1871
$(VCSSVN_OBJS): \
1871
- vcs-svn/obj_pool.h vcs-svn/trp.h vcs-svn/string_pool.h
1872
+ vcs-svn/obj_pool.h vcs-svn/trp.h vcs-svn/string_pool.h \
1873
+ vcs-svn/line_buffer.h
1874
endif
1875
1876
exec_cmd.s exec_cmd.o: EXTRA_CPPFLAGS = \
@@ -2017,6 +2019,8 @@ test-date$X: date.o ctype.o
2019
2020
test-delta$X: diff-delta.o patch-delta.o
2021
2022
+test-line-buffer$X: vcs-svn/lib.a
2023
+
2024
test-parse-options$X: parse-options.o
2025
2026
test-string-pool$X: vcs-svn/lib.a
t/t0080-vcs-svn.sh
+54
@@ -76,6 +76,60 @@ test_expect_success 'obj pool: high-water mark' '
76
test_cmp expected actual
77
'
78
79
+test_expect_success 'line buffer' '
80
+ echo HELLO >expected1 &&
81
+ printf "%s\n" "" HELLO >expected2 &&
82
+ echo >expected3 &&
83
+ printf "%s\n" "" Q | q_to_nul >expected4 &&
84
+ printf "%s\n" foo "" >expected5 &&
85
+ printf "%s\n" "" foo >expected6 &&
86
+
87
+ test-line-buffer <<-\EOF >actual1 &&
88
+ 5
89
+ HELLO
90
+ EOF
91
+
92
+ test-line-buffer <<-\EOF >actual2 &&
93
+ 0
94
+
95
+ 5
96
+ HELLO
97
+ EOF
98
+
99
+ q_to_nul <<-\EOF |
100
+ 1
101
+ Q
102
+ EOF
103
+ test-line-buffer >actual3 &&
104
+
105
+ q_to_nul <<-\EOF |
106
+ 0
107
+
108
+ 1
109
+ Q
110
+ EOF
111
+ test-line-buffer >actual4 &&
112
+
113
+ test-line-buffer <<-\EOF >actual5 &&
114
+ 5
115
+ foo
116
+ EOF
117
+
118
+ test-line-buffer <<-\EOF >actual6 &&
119
+ 0
120
+
121
+ 5
122
+ foo
123
+ EOF
124
+
125
+ test_cmp expected1 actual1 &&
126
+ test_cmp expected2 actual2 &&
127
+ test_cmp expected3 actual3 &&
128
+ test_cmp expected4 actual4 &&
129
+ test_cmp expected5 actual5 &&
130
+ test_cmp expected6 actual6
131
+'
132
+
133
test_expect_success 'string pool' '
134
echo a does not equal b >expected.differ &&
135
echo a equals a >expected.match &&
test-line-buffer.c
new
+46
@@ -0,0 +1,46 @@
1
+/*
2
+ * test-line-buffer.c: code to exercise the svn importer's input helper
3
+ *
4
+ * Input format:
5
+ * number NL
6
+ * (number bytes) NL
7
+ * number NL
8
+ * ...
9
+ */
10
+
11
+#include "git-compat-util.h"
12
+#include "vcs-svn/line_buffer.h"
13
+
14
+static uint32_t strtouint32(const char *s)
15
+{
16
+ char *end;
17
+ uintmax_t n = strtoumax(s, &end, 10);
18
+ if (*s == '\0' || *end != '\0')
19
+ die("invalid count: %s", s);
20
+ return (uint32_t) n;
21
+}
22
+
23
+int main(int argc, char *argv[])
24
+{
25
+ char *s;
26
+
27
+ if (argc != 1)
28
+ usage("test-line-buffer < input.txt");
29
+ if (buffer_init(NULL))
30
+ die_errno("open error");
31
+ while ((s = buffer_read_line())) {
32
+ s = buffer_read_string(strtouint32(s));
33
+ fputs(s, stdout);
34
+ fputc('\n', stdout);
35
+ buffer_skip_bytes(1);
36
+ if (!(s = buffer_read_line()))
37
+ break;
38
+ buffer_copy_bytes(strtouint32(s) + 1);
39
+ }
40
+ if (buffer_deinit())
41
+ die("input error");
42
+ if (ferror(stdout))
43
+ die("output error");
44
+ buffer_reset();
45
+ return 0;
46
+}
vcs-svn/line_buffer.c
new
+97
@@ -0,0 +1,97 @@
1
+/*
2
+ * Licensed under a two-clause BSD-style license.
3
+ * See LICENSE for details.
4
+ */
5
+
6
+#include "git-compat-util.h"
7
+#include "line_buffer.h"
8
+#include "obj_pool.h"
9
+
10
+#define LINE_BUFFER_LEN 10000
11
+#define COPY_BUFFER_LEN 4096
12
+
13
+/* Create memory pool for char sequence of known length */
14
+obj_pool_gen(blob, char, 4096)
15
+
16
+static char line_buffer[LINE_BUFFER_LEN];
17
+static char byte_buffer[COPY_BUFFER_LEN];
18
+static FILE *infile;
19
+
20
+int buffer_init(const char *filename)
21
+{
22
+ infile = filename ? fopen(filename, "r") : stdin;
23
+ if (!infile)
24
+ return -1;
25
+ return 0;
26
+}
27
+
28
+int buffer_deinit(void)
29
+{
30
+ int err;
31
+ if (infile == stdin)
32
+ return ferror(infile);
33
+ err = ferror(infile);
34
+ err |= fclose(infile);
35
+ return err;
36
+}
37
+
38
+/* Read a line without trailing newline. */
39
+char *buffer_read_line(void)
40
+{
41
+ char *end;
42
+ if (!fgets(line_buffer, sizeof(line_buffer), infile))
43
+ /* Error or data exhausted. */
44
+ return NULL;
45
+ end = line_buffer + strlen(line_buffer);
46
+ if (end[-1] == '\n')
47
+ end[-1] = '\0';
48
+ else if (feof(infile))
49
+ ; /* No newline at end of file. That's fine. */
50
+ else
51
+ /*
52
+ * Line was too long.
53
+ * There is probably a saner way to deal with this,
54
+ * but for now let's return an error.
55
+ */
56
+ return NULL;
57
+ return line_buffer;
58
+}
59
+
60
+char *buffer_read_string(uint32_t len)
61
+{
62
+ char *s;
63
+ blob_free(blob_pool.size);
64
+ s = blob_pointer(blob_alloc(len + 1));
65
+ s[fread(s, 1, len, infile)] = '\0';
66
+ return ferror(infile) ? NULL : s;
67
+}
68
+
69
+void buffer_copy_bytes(uint32_t len)
70
+{
71
+ uint32_t in;
72
+ while (len > 0 && !feof(infile) && !ferror(infile)) {
73
+ in = len < COPY_BUFFER_LEN ? len : COPY_BUFFER_LEN;
74
+ in = fread(byte_buffer, 1, in, infile);
75
+ len -= in;
76
+ fwrite(byte_buffer, 1, in, stdout);
77
+ if (ferror(stdout)) {
78
+ buffer_skip_bytes(len);
79
+ return;
80
+ }
81
+ }
82
+}
83
+
84
+void buffer_skip_bytes(uint32_t len)
85
+{
86
+ uint32_t in;
87
+ while (len > 0 && !feof(infile) && !ferror(infile)) {
88
+ in = len < COPY_BUFFER_LEN ? len : COPY_BUFFER_LEN;
89
+ in = fread(byte_buffer, 1, in, infile);
90
+ len -= in;
91
+ }
92
+}
93
+
94
+void buffer_reset(void)
95
+{
96
+ blob_reset();
97
+}
vcs-svn/line_buffer.h
new
+12
@@ -0,0 +1,12 @@
1
+#ifndef LINE_BUFFER_H_
2
+#define LINE_BUFFER_H_
3
+
4
+int buffer_init(const char *filename);
5
+int buffer_deinit(void);
6
+char *buffer_read_line(void);
7
+char *buffer_read_string(uint32_t len);
8
+void buffer_copy_bytes(uint32_t len);
9
+void buffer_skip_bytes(uint32_t len);
10
+void buffer_reset(void);
11
+
12
+#endif
vcs-svn/line_buffer.txt
new
+58
@@ -0,0 +1,58 @@
1
+line_buffer API
2
+===============
3
+
4
+The line_buffer library provides a convenient interface for
5
+mostly-line-oriented input.
6
+
7
+Each line is not permitted to exceed 10000 bytes. The provided
8
+functions are not thread-safe or async-signal-safe, and like
9
+`fgets()`, they generally do not function correctly if interrupted
10
+by a signal without SA_RESTART set.
11
+
12
+Calling sequence
13
+----------------
14
+
15
+The calling program:
16
+
17
+ - specifies a file to read with `buffer_init`
18
+ - processes input with `buffer_read_line`, `buffer_read_string`,
19
+ `buffer_skip_bytes`, and `buffer_copy_bytes`
20
+ - closes the file with `buffer_deinit`, perhaps to start over and
21
+ read another file.
22
+
23
+Before exiting, the caller can use `buffer_reset` to deallocate
24
+resources for the benefit of profiling tools.
25
+
26
+Functions
27
+---------
28
+
29
+`buffer_init`::
30
+ Open the named file for input. If filename is NULL,
31
+ start reading from stdin. On failure, returns -1 (with
32
+ errno indicating the nature of the failure).
33
+
34
+`buffer_deinit`::
35
+ Stop reading from the current file (closing it unless
36
+ it was stdin). Returns nonzero if `fclose` fails or
37
+ the error indicator was set.
38
+
39
+`buffer_read_line`::
40
+ Read a line and strip off the trailing newline.
41
+ On failure or end of file, returns NULL.
42
+
43
+`buffer_read_string`::
44
+ Read `len` characters of input or up to the end of the
45
+ file, whichever comes first. Returns NULL on error.
46
+ Returns whatever characters were read (possibly "")
47
+ for end of file.
48
+
49
+`buffer_copy_bytes`::
50
+ Read `len` bytes of input and dump them to the standard output
51
+ stream. Returns early for error or end of file.
52
+
53
+`buffer_skip_bytes`::
54
+ Discards `len` bytes from the input stream (stopping early
55
+ if necessary because of an error or eof).
56
+
57
+`buffer_reset`::
58
+ Deallocates non-static buffers.