streaming: a new API to read from the object store
Given an object name, use open_istream() to get a git_istream handle that you can read_istream() from as if you are using read(2) to read the contents of the object, and close it with close_istream() when you are done. Currently, we do not do anything fancy--it just calls read_sha1_file() and keeps the contents in memory as a whole, and carve it out as you request with read_istream(). Signed-off-by: Junio C Hamano <gitster@pobox.com>
Junio C Hamano committed
May 11, 2011 at 19:30 UTC
46bf043807cc5d8986f41139a8c28491f613c5e0
3 files changed
+210
Makefile
+2
@@ -552,6 +552,7 @@ LIB_H += sha1-lookup.h
552
LIB_H += sideband.h
553
LIB_H += sigchain.h
554
LIB_H += strbuf.h
555
+LIB_H += streaming.h
556
LIB_H += string-list.h
557
LIB_H += submodule.h
558
LIB_H += tag.h
@@ -657,6 +658,7 @@ LIB_OBJS += shallow.o
658
LIB_OBJS += sideband.o
659
LIB_OBJS += sigchain.o
660
LIB_OBJS += strbuf.o
661
+LIB_OBJS += streaming.o
662
LIB_OBJS += string-list.o
663
LIB_OBJS += submodule.o
664
LIB_OBJS += symlinks.o
streaming.c
new
+193
@@ -0,0 +1,193 @@
1
+/*
2
+ * Copyright (c) 2011, Google Inc.
3
+ */
4
+#include "cache.h"
5
+#include "streaming.h"
6
+
7
+enum input_source {
8
+ stream_error = -1,
9
+ incore = 0,
10
+ loose = 1,
11
+ pack_non_delta = 2
12
+};
13
+
14
+typedef int (*open_istream_fn)(struct git_istream *,
15
+ struct object_info *,
16
+ const unsigned char *,
17
+ enum object_type *);
18
+typedef int (*close_istream_fn)(struct git_istream *);
19
+typedef ssize_t (*read_istream_fn)(struct git_istream *, char *, size_t);
20
+
21
+struct stream_vtbl {
22
+ close_istream_fn close;
23
+ read_istream_fn read;
24
+};
25
+
26
+#define open_method_decl(name) \
27
+ int open_istream_ ##name \
28
+ (struct git_istream *st, struct object_info *oi, \
29
+ const unsigned char *sha1, \
30
+ enum object_type *type)
31
+
32
+#define close_method_decl(name) \
33
+ int close_istream_ ##name \
34
+ (struct git_istream *st)
35
+
36
+#define read_method_decl(name) \
37
+ ssize_t read_istream_ ##name \
38
+ (struct git_istream *st, char *buf, size_t sz)
39
+
40
+/* forward declaration */
41
+static open_method_decl(incore);
42
+static open_method_decl(loose);
43
+static open_method_decl(pack_non_delta);
44
+
45
+static open_istream_fn open_istream_tbl[] = {
46
+ open_istream_incore,
47
+ open_istream_loose,
48
+ open_istream_pack_non_delta,
49
+};
50
+
51
+struct git_istream {
52
+ const struct stream_vtbl *vtbl;
53
+ unsigned long size; /* inflated size of full object */
54
+
55
+ union {
56
+ struct {
57
+ char *buf; /* from read_object() */
58
+ unsigned long read_ptr;
59
+ } incore;
60
+
61
+ struct {
62
+ int fd; /* open for reading */
63
+ /* NEEDSWORK: what else? */
64
+ } loose;
65
+
66
+ struct {
67
+ int fd; /* open for reading */
68
+ /* NEEDSWORK: what else? */
69
+ } in_pack;
70
+ } u;
71
+};
72
+
73
+int close_istream(struct git_istream *st)
74
+{
75
+ return st->vtbl->close(st);
76
+}
77
+
78
+ssize_t read_istream(struct git_istream *st, char *buf, size_t sz)
79
+{
80
+ return st->vtbl->read(st, buf, sz);
81
+}
82
+
83
+static enum input_source istream_source(const unsigned char *sha1,
84
+ enum object_type *type,
85
+ struct object_info *oi)
86
+{
87
+ unsigned long size;
88
+ int status;
89
+
90
+ oi->sizep = &size;
91
+ status = sha1_object_info_extended(sha1, oi);
92
+ if (status < 0)
93
+ return stream_error;
94
+ *type = status;
95
+
96
+ switch (oi->whence) {
97
+ case OI_LOOSE:
98
+ return loose;
99
+ case OI_PACKED:
100
+ if (!oi->u.packed.is_delta && big_file_threshold <= size)
101
+ return pack_non_delta;
102
+ /* fallthru */
103
+ default:
104
+ return incore;
105
+ }
106
+}
107
+
108
+struct git_istream *open_istream(const unsigned char *sha1,
109
+ enum object_type *type,
110
+ unsigned long *size)
111
+{
112
+ struct git_istream *st;
113
+ struct object_info oi;
114
+ const unsigned char *real = lookup_replace_object(sha1);
115
+ enum input_source src = istream_source(real, type, &oi);
116
+
117
+ if (src < 0)
118
+ return NULL;
119
+
120
+ st = xmalloc(sizeof(*st));
121
+ if (open_istream_tbl[src](st, &oi, real, type)) {
122
+ if (open_istream_incore(st, &oi, real, type)) {
123
+ free(st);
124
+ return NULL;
125
+ }
126
+ }
127
+ *size = st->size;
128
+ return st;
129
+}
130
+
131
+/*****************************************************************
132
+ *
133
+ * Loose object stream
134
+ *
135
+ *****************************************************************/
136
+
137
+static open_method_decl(loose)
138
+{
139
+ return -1; /* for now */
140
+}
141
+
142
+
143
+/*****************************************************************
144
+ *
145
+ * Non-delta packed object stream
146
+ *
147
+ *****************************************************************/
148
+
149
+static open_method_decl(pack_non_delta)
150
+{
151
+ return -1; /* for now */
152
+}
153
+
154
+
155
+/*****************************************************************
156
+ *
157
+ * In-core stream
158
+ *
159
+ *****************************************************************/
160
+
161
+static close_method_decl(incore)
162
+{
163
+ free(st->u.incore.buf);
164
+ return 0;
165
+}
166
+
167
+static read_method_decl(incore)
168
+{
169
+ size_t read_size = sz;
170
+ size_t remainder = st->size - st->u.incore.read_ptr;
171
+
172
+ if (remainder <= read_size)
173
+ read_size = remainder;
174
+ if (read_size) {
175
+ memcpy(buf, st->u.incore.buf + st->u.incore.read_ptr, read_size);
176
+ st->u.incore.read_ptr += read_size;
177
+ }
178
+ return read_size;
179
+}
180
+
181
+static struct stream_vtbl incore_vtbl = {
182
+ close_istream_incore,
183
+ read_istream_incore,
184
+};
185
+
186
+static open_method_decl(incore)
187
+{
188
+ st->u.incore.buf = read_sha1_file_extended(sha1, type, &st->size, 0);
189
+ st->u.incore.read_ptr = 0;
190
+ st->vtbl = &incore_vtbl;
191
+
192
+ return st->u.incore.buf ? 0 : -1;
193
+}
streaming.h
new
+15
@@ -0,0 +1,15 @@
1
+/*
2
+ * Copyright (c) 2011, Google Inc.
3
+ */
4
+#ifndef STREAMING_H
5
+#define STREAMING_H 1
6
+#include "cache.h"
7
+
8
+/* opaque */
9
+struct git_istream;
10
+
11
+extern struct git_istream *open_istream(const unsigned char *, enum object_type *, unsigned long *);
12
+extern int close_istream(struct git_istream *);
13
+extern ssize_t read_istream(struct git_istream *, char *, size_t);
14
+
15
+#endif /* STREAMING_H */