Add streaming filter API
This introduces an API to plug custom filters to an input stream. The caller gets get_stream_filter("path") to obtain an appropriate filter for the path, and then uses it when opening an input stream via open_istream(). After that, the caller can read from the stream with read_istream(), and close it with close_istream(), just like an unfiltered stream. This only adds a "null" filter that is a pass-thru filter, but later changes can add LF-to-CRLF and other filters, and the callers of the streaming API do not have to change. Signed-off-by: Junio C Hamano <gitster@pobox.com>
Junio C Hamano committed
May 20, 2011 at 14:33 UTC
b6691092d707860019bbab80eaaf9173ada10586
5 files changed
+209
-16
convert.c
+77
-7
@@ -814,12 +814,69 @@ int renormalize_buffer(const char *path, const char *src, size_t len, struct str
814
return ret | convert_to_git(path, src, len, dst, 0);
815
}
816
817
+/*****************************************************************
818
+ *
819
+ * Streaming converison support
820
+ *
821
+ *****************************************************************/
822
+
823
+typedef int (*filter_fn)(struct stream_filter *,
824
+ const char *input, size_t *isize_p,
825
+ char *output, size_t *osize_p);
826
+typedef void (*free_fn)(struct stream_filter *);
827
+
828
+struct stream_filter_vtbl {
829
+ filter_fn filter;
830
+ free_fn free;
831
+};
832
+
833
+struct stream_filter {
834
+ struct stream_filter_vtbl *vtbl;
835
+};
836
+
837
+static int null_filter_fn(struct stream_filter *filter,
838
+ const char *input, size_t *isize_p,
839
+ char *output, size_t *osize_p)
840
+{
841
+ size_t count = *isize_p;
842
+ if (*osize_p < count)
843
+ count = *osize_p;
844
+ if (count) {
845
+ memmove(output, input, count);
846
+ *isize_p -= count;
847
+ *osize_p -= count;
848
+ }
849
+ return 0;
850
+}
851
+
852
+static void null_free_fn(struct stream_filter *filter)
853
+{
854
+ ; /* nothing -- null instances are shared */
855
+}
856
+
857
+static struct stream_filter_vtbl null_vtbl = {
858
+ null_filter_fn,
859
+ null_free_fn,
860
+};
861
+
862
+static struct stream_filter null_filter_singleton = {
863
+ &null_vtbl,
864
+};
865
+
866
+int is_null_stream_filter(struct stream_filter *filter)
867
+{
868
+ return filter == &null_filter_singleton;
869
+}
870
+
871
/*
818
- * You would be crazy to set CRLF, smuge/clean or ident to
819
- * a large binary blob you would want us not to slurp into
820
- * the memory!
872
+ * Return an appropriately constructed filter for the path, or NULL if
873
+ * the contents cannot be filtered without reading the whole thing
874
+ * in-core.
875
+ *
876
+ * Note that you would be crazy to set CRLF, smuge/clean or ident to a
877
+ * large binary blob you would want us not to slurp into the memory!
878
*/
822
-int can_bypass_conversion(const char *path)
879
+struct stream_filter *get_stream_filter(const char *path, const unsigned char *sha1)
880
{
881
struct conv_attrs ca;
882
enum crlf_action crlf_action;
@@ -828,11 +885,24 @@ int can_bypass_conversion(const char *path)
885
886
if (ca.ident ||
887
(ca.drv && (ca.drv->smudge || ca.drv->clean)))
831
- return 0;
888
+ return NULL;
889
890
crlf_action = input_crlf_action(ca.crlf_action, ca.eol_attr);
891
if ((crlf_action == CRLF_BINARY) || (crlf_action == CRLF_INPUT) ||
892
(crlf_action == CRLF_GUESS && auto_crlf == AUTO_CRLF_FALSE))
836
- return 1;
837
- return 0;
893
+ return &null_filter_singleton;
894
+
895
+ return NULL;
896
+}
897
+
898
+void free_stream_filter(struct stream_filter *filter)
899
+{
900
+ filter->vtbl->free(filter);
901
+}
902
+
903
+int stream_filter(struct stream_filter *filter,
904
+ const char *input, size_t *isize_p,
905
+ char *output, size_t *osize_p)
906
+{
907
+ return filter->vtbl->filter(filter, input, isize_p, output, osize_p);
908
}
convert.h
+22
-1
@@ -40,5 +40,26 @@ extern int convert_to_working_tree(const char *path, const char *src,
40
size_t len, struct strbuf *dst);
41
extern int renormalize_buffer(const char *path, const char *src, size_t len,
42
struct strbuf *dst);
43
-extern int can_bypass_conversion(const char *path);
43
+
44
+/*****************************************************************
45
+ *
46
+ * Streaming converison support
47
+ *
48
+ *****************************************************************/
49
+
50
+struct stream_filter; /* opaque */
51
+
52
+extern struct stream_filter *get_stream_filter(const char *path, const unsigned char *);
53
+extern void free_stream_filter(struct stream_filter *);
54
+extern int is_null_stream_filter(struct stream_filter *);
55
+
56
+/*
57
+ * Use as much input up to *isize_p and fill output up to *osize_p;
58
+ * update isize_p and osize_p to indicate how much buffer space was
59
+ * consumed and filled. Return 0 on success, non-zero on error.
60
+ */
61
+extern int stream_filter(struct stream_filter *,
62
+ const char *input, size_t *isize_p,
63
+ char *output, size_t *osize_p);
64
+
65
#endif /* CONVERT_H */
entry.c
+10
-6
@@ -116,6 +116,7 @@ static int fstat_output(int fd, const struct checkout *state, struct stat *st)
116
}
117
118
static int streaming_write_entry(struct cache_entry *ce, char *path,
119
+ struct stream_filter *filter,
120
const struct checkout *state, int to_tempfile,
121
int *fstat_done, struct stat *statbuf)
122
{
@@ -126,7 +127,7 @@ static int streaming_write_entry(struct cache_entry *ce, char *path,
127
ssize_t kept = 0;
128
int fd = -1;
129
129
- st = open_istream(ce->sha1, &type, &sz);
130
+ st = open_istream(ce->sha1, &type, &sz, filter);
131
if (!st)
132
return -1;
133
if (type != OBJ_BLOB)
@@ -186,11 +187,14 @@ static int write_entry(struct cache_entry *ce, char *path, const struct checkout
187
size_t wrote, newsize = 0;
188
struct stat st;
189
189
- if ((ce_mode_s_ifmt == S_IFREG) &&
190
- can_bypass_conversion(path) &&
191
- !streaming_write_entry(ce, path, state, to_tempfile,
192
- &fstat_done, &st))
193
- goto finish;
190
+ if (ce_mode_s_ifmt == S_IFREG) {
191
+ struct stream_filter *filter = get_stream_filter(path, ce->sha1);
192
+ if (filter &&
193
+ !streaming_write_entry(ce, path, filter,
194
+ state, to_tempfile,
195
+ &fstat_done, &st))
196
+ goto finish;
197
+ }
198
199
switch (ce_mode_s_ifmt) {
200
case S_IFREG:
streaming.c
+99
-1
@@ -41,6 +41,9 @@ struct stream_vtbl {
41
static open_method_decl(incore);
42
static open_method_decl(loose);
43
static open_method_decl(pack_non_delta);
44
+static struct git_istream *attach_stream_filter(struct git_istream *st,
45
+ struct stream_filter *filter);
46
+
47
48
static open_istream_fn open_istream_tbl[] = {
49
open_istream_incore,
@@ -48,6 +51,17 @@ static open_istream_fn open_istream_tbl[] = {
51
open_istream_pack_non_delta,
52
};
53
54
+#define FILTER_BUFFER (1024*16)
55
+
56
+struct filtered_istream {
57
+ struct git_istream *upstream;
58
+ struct stream_filter *filter;
59
+ char ibuf[FILTER_BUFFER];
60
+ char obuf[FILTER_BUFFER];
61
+ int i_end, i_ptr;
62
+ int o_end, o_ptr;
63
+};
64
+
65
struct git_istream {
66
const struct stream_vtbl *vtbl;
67
unsigned long size; /* inflated size of full object */
@@ -72,6 +86,8 @@ struct git_istream {
86
struct packed_git *pack;
87
off_t pos;
88
} in_pack;
89
+
90
+ struct filtered_istream filtered;
91
} u;
92
};
93
@@ -112,7 +128,8 @@ static enum input_source istream_source(const unsigned char *sha1,
128
129
struct git_istream *open_istream(const unsigned char *sha1,
130
enum object_type *type,
115
- unsigned long *size)
131
+ unsigned long *size,
132
+ struct stream_filter *filter)
133
{
134
struct git_istream *st;
135
struct object_info oi;
@@ -129,6 +146,14 @@ struct git_istream *open_istream(const unsigned char *sha1,
146
return NULL;
147
}
148
}
149
+ if (st && filter) {
150
+ /* Add "&& !is_null_stream_filter(filter)" for performance */
151
+ struct git_istream *nst = attach_stream_filter(st, filter);
152
+ if (!nst)
153
+ close_istream(st);
154
+ st = nst;
155
+ }
156
+
157
*size = st->size;
158
return st;
159
}
@@ -147,6 +172,79 @@ static void close_deflated_stream(struct git_istream *st)
172
}
173
174
175
+/*****************************************************************
176
+ *
177
+ * Filtered stream
178
+ *
179
+ *****************************************************************/
180
+
181
+static close_method_decl(filtered)
182
+{
183
+ free_stream_filter(st->u.filtered.filter);
184
+ return close_istream(st->u.filtered.upstream);
185
+}
186
+
187
+static read_method_decl(filtered)
188
+{
189
+ struct filtered_istream *fs = &(st->u.filtered);
190
+ size_t filled = 0;
191
+
192
+ while (sz) {
193
+ /* do we already have filtered output? */
194
+ if (fs->o_ptr < fs->o_end) {
195
+ size_t to_move = fs->o_end - fs->o_ptr;
196
+ if (sz < to_move)
197
+ to_move = sz;
198
+ memcpy(buf + filled, fs->obuf + fs->o_ptr, to_move);
199
+ fs->o_ptr += to_move;
200
+ sz -= to_move;
201
+ filled += to_move;
202
+ continue;
203
+ }
204
+ fs->o_end = fs->o_ptr = 0;
205
+
206
+ /* do we have anything to feed the filter with? */
207
+ if (fs->i_ptr < fs->i_end) {
208
+ size_t to_feed = fs->i_end - fs->i_ptr;
209
+ size_t to_receive = FILTER_BUFFER;
210
+ if (stream_filter(fs->filter,
211
+ fs->ibuf + fs->i_ptr, &to_feed,
212
+ fs->obuf, &to_receive))
213
+ return -1;
214
+ fs->i_ptr = fs->i_end - to_feed;
215
+ fs->o_end = FILTER_BUFFER - to_receive;
216
+ continue;
217
+ }
218
+ fs->i_end = fs->i_ptr = 0;
219
+
220
+ /* refill the input from the upstream */
221
+ fs->i_end = read_istream(fs->upstream, fs->ibuf, FILTER_BUFFER);
222
+ if (fs->i_end <= 0)
223
+ break;
224
+ }
225
+ return filled;
226
+}
227
+
228
+static struct stream_vtbl filtered_vtbl = {
229
+ close_istream_filtered,
230
+ read_istream_filtered,
231
+};
232
+
233
+static struct git_istream *attach_stream_filter(struct git_istream *st,
234
+ struct stream_filter *filter)
235
+{
236
+ struct git_istream *ifs = xmalloc(sizeof(*ifs));
237
+ struct filtered_istream *fs = &(ifs->u.filtered);
238
+
239
+ ifs->vtbl = &filtered_vtbl;
240
+ fs->upstream = st;
241
+ fs->filter = filter;
242
+ fs->i_end = fs->i_ptr = 0;
243
+ fs->o_end = fs->o_ptr = 0;
244
+ ifs->size = -1; /* unknown */
245
+ return ifs;
246
+}
247
+
248
/*****************************************************************
249
*
250
* Loose object stream
streaming.h
+1
-1
@@ -8,7 +8,7 @@
8
/* opaque */
9
struct git_istream;
10
11
-extern struct git_istream *open_istream(const unsigned char *, enum object_type *, unsigned long *);
11
+extern struct git_istream *open_istream(const unsigned char *, enum object_type *, unsigned long *, struct stream_filter *);
12
extern int close_istream(struct git_istream *);
13
extern ssize_t read_istream(struct git_istream *, char *, size_t);
14