@matheusbsbs / matheus-bagliano / commits / 727a935a71

odb/source-loose: wire up `read_object_stream()` callback

Move `odb_source_loose_read_object_stream()` and its associated helpers from "object-file.c" into "odb/source-loose.c" and wire it up as the `read_object_stream()` callback of the loose source. As part of the move we are also forced to expose a couple of functions from "object-file.h" that parse object headers in a somewhat-generic way, as those functions are now used by both subsystems. Signed-off-by: Patrick Steinhardt <ps@pks.im> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Patrick Steinhardt committed Jun 1, 2026 at 10:20 UTC 727a935a71c29524c936520d8aba4de7098f7566
4 files changed +222 -200
object-file.c
+6 -194
@@ -164,28 +164,6 @@ int stream_object_signature(struct repository *r,
164 return !oideq(oid, &real_oid) ? -1 : 0;
165 }
166
167 -/*
168 - * Find "oid" as a loose object in given source, open the object and return its
169 - * file descriptor. Returns the file descriptor on success, negative on failure.
170 - *
171 - * The "path" out-parameter will give the path of the object we found (if any).
172 - * Note that it may point to static storage and is only valid until another
173 - * call to stat_loose_object().
174 - */
175 -static int open_loose_object(struct odb_source_loose *loose,
176 - const struct object_id *oid, const char **path)
177 -{
178 - static struct strbuf buf = STRBUF_INIT;
179 - int fd;
180 -
181 - *path = odb_loose_path(&loose->files->base, &buf, oid);
182 - fd = git_open(*path);
183 - if (fd >= 0)
184 - return fd;
185 -
186 - return -1;
187 -}
188 -
167 static int quick_has_loose(struct odb_source_loose *loose,
168 const struct object_id *oid)
169 {
@@ -215,42 +193,11 @@ static void *map_fd(int fd, const char *path, unsigned long *size)
193 return map;
194 }
195
218 -static void *odb_source_loose_map_object(struct odb_source *source,
219 - const struct object_id *oid,
220 - unsigned long *size)
221 -{
222 - struct odb_source_files *files = odb_source_files_downcast(source);
223 - const char *p;
224 - int fd = open_loose_object(files->loose, oid, &p);
225 -
226 - if (fd < 0)
227 - return NULL;
228 - return map_fd(fd, p, size);
229 -}
230 -
231 -enum unpack_loose_header_result {
232 - ULHR_OK,
233 - ULHR_BAD,
234 - ULHR_TOO_LONG,
235 -};
236 -
237 -/**
238 - * unpack_loose_header() initializes the data stream needed to unpack
239 - * a loose object header.
240 - *
241 - * Returns:
242 - *
243 - * - ULHR_OK on success
244 - * - ULHR_BAD on error
245 - * - ULHR_TOO_LONG if the header was too long
246 - *
247 - * It will only parse up to MAX_HEADER_LEN bytes.
248 - */
249 -static enum unpack_loose_header_result unpack_loose_header(git_zstream *stream,
250 - unsigned char *map,
251 - unsigned long mapsize,
252 - void *buffer,
253 - unsigned long bufsiz)
196 +enum unpack_loose_header_result unpack_loose_header(git_zstream *stream,
197 + unsigned char *map,
198 + unsigned long mapsize,
199 + void *buffer,
200 + unsigned long bufsiz)
201 {
202 int status;
203
@@ -340,7 +287,7 @@ static void *unpack_loose_rest(git_zstream *stream,
287 * too permissive for what we want to check. So do an anal
288 * object header parse by hand.
289 */
343 -static int parse_loose_header(const char *hdr, struct object_info *oi)
290 +int parse_loose_header(const char *hdr, struct object_info *oi)
291 {
292 const char *type_buf = hdr;
293 size_t size;
@@ -2170,138 +2117,3 @@ struct odb_transaction *odb_transaction_files_begin(struct odb_source *source)
2117
2118 return &transaction->base;
2119 }
2173 -
2174 -struct odb_loose_read_stream {
2175 - struct odb_read_stream base;
2176 - git_zstream z;
2177 - enum {
2178 - ODB_LOOSE_READ_STREAM_INUSE,
2179 - ODB_LOOSE_READ_STREAM_DONE,
2180 - ODB_LOOSE_READ_STREAM_ERROR,
2181 - } z_state;
2182 - void *mapped;
2183 - unsigned long mapsize;
2184 - char hdr[32];
2185 - int hdr_avail;
2186 - int hdr_used;
2187 -};
2188 -
2189 -static ssize_t read_istream_loose(struct odb_read_stream *_st, char *buf, size_t sz)
2190 -{
2191 - struct odb_loose_read_stream *st =
2192 - container_of(_st, struct odb_loose_read_stream, base);
2193 - size_t total_read = 0;
2194 -
2195 - switch (st->z_state) {
2196 - case ODB_LOOSE_READ_STREAM_DONE:
2197 - return 0;
2198 - case ODB_LOOSE_READ_STREAM_ERROR:
2199 - return -1;
2200 - default:
2201 - break;
2202 - }
2203 -
2204 - if (st->hdr_used < st->hdr_avail) {
2205 - size_t to_copy = st->hdr_avail - st->hdr_used;
2206 - if (sz < to_copy)
2207 - to_copy = sz;
2208 - memcpy(buf, st->hdr + st->hdr_used, to_copy);
2209 - st->hdr_used += to_copy;
2210 - total_read += to_copy;
2211 - }
2212 -
2213 - while (total_read < sz) {
2214 - int status;
2215 -
2216 - st->z.next_out = (unsigned char *)buf + total_read;
2217 - st->z.avail_out = sz - total_read;
2218 - status = git_inflate(&st->z, Z_FINISH);
2219 -
2220 - total_read = st->z.next_out - (unsigned char *)buf;
2221 -
2222 - if (status == Z_STREAM_END) {
2223 - git_inflate_end(&st->z);
2224 - st->z_state = ODB_LOOSE_READ_STREAM_DONE;
2225 - break;
2226 - }
2227 - if (status != Z_OK && (status != Z_BUF_ERROR || total_read < sz)) {
2228 - git_inflate_end(&st->z);
2229 - st->z_state = ODB_LOOSE_READ_STREAM_ERROR;
2230 - return -1;
2231 - }
2232 - }
2233 - return total_read;
2234 -}
2235 -
2236 -static int close_istream_loose(struct odb_read_stream *_st)
2237 -{
2238 - struct odb_loose_read_stream *st =
2239 - container_of(_st, struct odb_loose_read_stream, base);
2240 -
2241 - if (st->z_state == ODB_LOOSE_READ_STREAM_INUSE)
2242 - git_inflate_end(&st->z);
2243 - munmap(st->mapped, st->mapsize);
2244 - return 0;
2245 -}
2246 -
2247 -int odb_source_loose_read_object_stream(struct odb_read_stream **out,
2248 - struct odb_source *source,
2249 - const struct object_id *oid)
2250 -{
2251 - struct object_info oi = OBJECT_INFO_INIT;
2252 - struct odb_loose_read_stream *st;
2253 - unsigned long mapsize;
2254 - unsigned long size_ul;
2255 - void *mapped;
2256 -
2257 - mapped = odb_source_loose_map_object(source, oid, &mapsize);
2258 - if (!mapped)
2259 - return -1;
2260 -
2261 - /*
2262 - * Note: we must allocate this structure early even though we may still
2263 - * fail. This is because we need to initialize the zlib stream, and it
2264 - * is not possible to copy the stream around after the fact because it
2265 - * has self-referencing pointers.
2266 - */
2267 - CALLOC_ARRAY(st, 1);
2268 -
2269 - switch (unpack_loose_header(&st->z, mapped, mapsize, st->hdr,
2270 - sizeof(st->hdr))) {
2271 - case ULHR_OK:
2272 - break;
2273 - case ULHR_BAD:
2274 - case ULHR_TOO_LONG:
2275 - goto error;
2276 - }
2277 -
2278 - /*
2279 - * object_info.sizep is unsigned long* (32-bit on Windows), but
2280 - * st->base.size is size_t (64-bit). Use temporary variable.
2281 - * Note: loose objects >4GB would still truncate here, but such
2282 - * large loose objects are uncommon (they'd normally be packed).
2283 - */
2284 - oi.sizep = &size_ul;
2285 - oi.typep = &st->base.type;
2286 -
2287 - if (parse_loose_header(st->hdr, &oi) < 0 || st->base.type < 0)
2288 - goto error;
2289 - st->base.size = size_ul;
2290 -
2291 - st->mapped = mapped;
2292 - st->mapsize = mapsize;
2293 - st->hdr_used = strlen(st->hdr) + 1;
2294 - st->hdr_avail = st->z.total_out;
2295 - st->z_state = ODB_LOOSE_READ_STREAM_INUSE;
2296 - st->base.close = close_istream_loose;
2297 - st->base.read = read_istream_loose;
2298 -
2299 - *out = &st->base;
2300 -
2301 - return 0;
2302 -error:
2303 - git_inflate_end(&st->z);
2304 - munmap(mapped, mapsize);
2305 - free(st);
2306 - return -1;
2307 -}
object-file.h
+26 -5
@@ -18,13 +18,8 @@ int index_fd(struct index_state *istate, struct object_id *oid, int fd, struct s
18 int index_path(struct index_state *istate, struct object_id *oid, const char *path, struct stat *st, unsigned flags);
19
20 struct object_info;
21 -struct odb_read_stream;
21 struct odb_source;
22
24 -int odb_source_loose_read_object_stream(struct odb_read_stream **out,
25 - struct odb_source *source,
26 - const struct object_id *oid);
27 -
23 /*
24 * Return true iff an object database source has a loose object
25 * with the specified name. This function does not respect replace
@@ -199,6 +194,32 @@ int read_object_info_from_path(struct odb_source_loose *loose,
194 struct object_info *oi,
195 enum object_info_flags flags);
196
197 +enum unpack_loose_header_result {
198 + ULHR_OK,
199 + ULHR_BAD,
200 + ULHR_TOO_LONG,
201 +};
202 +
203 +/**
204 + * unpack_loose_header() initializes the data stream needed to unpack
205 + * a loose object header.
206 + *
207 + * Returns:
208 + *
209 + * - ULHR_OK on success
210 + * - ULHR_BAD on error
211 + * - ULHR_TOO_LONG if the header was too long
212 + *
213 + * It will only parse up to MAX_HEADER_LEN bytes.
214 + */
215 +enum unpack_loose_header_result unpack_loose_header(git_zstream *stream,
216 + unsigned char *map,
217 + unsigned long mapsize,
218 + void *buffer,
219 + unsigned long bufsiz);
220 +
221 +int parse_loose_header(const char *hdr, struct object_info *oi);
222 +
223 struct odb_transaction;
224
225 /*
odb/source-files.c
+1 -1
@@ -67,7 +67,7 @@ static int odb_source_files_read_object_stream(struct odb_read_stream **out,
67 {
68 struct odb_source_files *files = odb_source_files_downcast(source);
69 if (!packfile_store_read_object_stream(out, files->packed, oid) ||
70 - !odb_source_loose_read_object_stream(out, source, oid))
70 + !odb_source_read_object_stream(out, &files->loose->base, oid))
71 return 0;
72 return -1;
73 }
odb/source-loose.c
+189
@@ -1,11 +1,13 @@
1 #include "git-compat-util.h"
2 #include "abspath.h"
3 #include "chdir-notify.h"
4 +#include "gettext.h"
5 #include "loose.h"
6 #include "object-file.h"
7 #include "odb.h"
8 #include "odb/source-files.h"
9 #include "odb/source-loose.h"
10 +#include "odb/streaming.h"
11 #include "oidtree.h"
12 #include "strbuf.h"
13
@@ -30,6 +32,192 @@ static int odb_source_loose_read_object_info(struct odb_source *source,
32 return read_object_info_from_path(loose, buf.buf, oid, oi, flags);
33 }
34
35 +/*
36 + * Find "oid" as a loose object in given source, open the object and return its
37 + * file descriptor. Returns the file descriptor on success, negative on failure.
38 + *
39 + * The "path" out-parameter will give the path of the object we found (if any).
40 + * Note that it may point to static storage and is only valid until another
41 + * call to open_loose_object().
42 + */
43 +static int open_loose_object(struct odb_source_loose *loose,
44 + const struct object_id *oid, const char **path)
45 +{
46 + static struct strbuf buf = STRBUF_INIT;
47 + int fd;
48 +
49 + *path = odb_loose_path(&loose->base, &buf, oid);
50 + fd = git_open(*path);
51 + if (fd >= 0)
52 + return fd;
53 +
54 + return -1;
55 +}
56 +
57 +static void *odb_source_loose_map_object(struct odb_source_loose *loose,
58 + const struct object_id *oid,
59 + unsigned long *size)
60 +{
61 + const char *p;
62 + int fd = open_loose_object(loose, oid, &p);
63 + void *map = NULL;
64 + struct stat st;
65 +
66 + if (fd < 0)
67 + return NULL;
68 +
69 + if (!fstat(fd, &st)) {
70 + *size = xsize_t(st.st_size);
71 + if (!*size) {
72 + /* mmap() is forbidden on empty files */
73 + error(_("object file %s is empty"), p);
74 + goto out;
75 + }
76 +
77 + map = xmmap(NULL, *size, PROT_READ, MAP_PRIVATE, fd, 0);
78 + }
79 +
80 +out:
81 + close(fd);
82 + return map;
83 +}
84 +
85 +struct odb_loose_read_stream {
86 + struct odb_read_stream base;
87 + git_zstream z;
88 + enum {
89 + ODB_LOOSE_READ_STREAM_INUSE,
90 + ODB_LOOSE_READ_STREAM_DONE,
91 + ODB_LOOSE_READ_STREAM_ERROR,
92 + } z_state;
93 + void *mapped;
94 + unsigned long mapsize;
95 + char hdr[32];
96 + int hdr_avail;
97 + int hdr_used;
98 +};
99 +
100 +static ssize_t read_istream_loose(struct odb_read_stream *_st, char *buf, size_t sz)
101 +{
102 + struct odb_loose_read_stream *st =
103 + container_of(_st, struct odb_loose_read_stream, base);
104 + size_t total_read = 0;
105 +
106 + switch (st->z_state) {
107 + case ODB_LOOSE_READ_STREAM_DONE:
108 + return 0;
109 + case ODB_LOOSE_READ_STREAM_ERROR:
110 + return -1;
111 + default:
112 + break;
113 + }
114 +
115 + if (st->hdr_used < st->hdr_avail) {
116 + size_t to_copy = st->hdr_avail - st->hdr_used;
117 + if (sz < to_copy)
118 + to_copy = sz;
119 + memcpy(buf, st->hdr + st->hdr_used, to_copy);
120 + st->hdr_used += to_copy;
121 + total_read += to_copy;
122 + }
123 +
124 + while (total_read < sz) {
125 + int status;
126 +
127 + st->z.next_out = (unsigned char *)buf + total_read;
128 + st->z.avail_out = sz - total_read;
129 + status = git_inflate(&st->z, Z_FINISH);
130 +
131 + total_read = st->z.next_out - (unsigned char *)buf;
132 +
133 + if (status == Z_STREAM_END) {
134 + git_inflate_end(&st->z);
135 + st->z_state = ODB_LOOSE_READ_STREAM_DONE;
136 + break;
137 + }
138 + if (status != Z_OK && (status != Z_BUF_ERROR || total_read < sz)) {
139 + git_inflate_end(&st->z);
140 + st->z_state = ODB_LOOSE_READ_STREAM_ERROR;
141 + return -1;
142 + }
143 + }
144 + return total_read;
145 +}
146 +
147 +static int close_istream_loose(struct odb_read_stream *_st)
148 +{
149 + struct odb_loose_read_stream *st =
150 + container_of(_st, struct odb_loose_read_stream, base);
151 +
152 + if (st->z_state == ODB_LOOSE_READ_STREAM_INUSE)
153 + git_inflate_end(&st->z);
154 + munmap(st->mapped, st->mapsize);
155 + return 0;
156 +}
157 +
158 +static int odb_source_loose_read_object_stream(struct odb_read_stream **out,
159 + struct odb_source *source,
160 + const struct object_id *oid)
161 +{
162 + struct odb_source_loose *loose = odb_source_loose_downcast(source);
163 + struct object_info oi = OBJECT_INFO_INIT;
164 + struct odb_loose_read_stream *st;
165 + unsigned long mapsize;
166 + unsigned long size_ul;
167 + void *mapped;
168 +
169 + mapped = odb_source_loose_map_object(loose, oid, &mapsize);
170 + if (!mapped)
171 + return -1;
172 +
173 + /*
174 + * Note: we must allocate this structure early even though we may still
175 + * fail. This is because we need to initialize the zlib stream, and it
176 + * is not possible to copy the stream around after the fact because it
177 + * has self-referencing pointers.
178 + */
179 + CALLOC_ARRAY(st, 1);
180 +
181 + switch (unpack_loose_header(&st->z, mapped, mapsize, st->hdr,
182 + sizeof(st->hdr))) {
183 + case ULHR_OK:
184 + break;
185 + case ULHR_BAD:
186 + case ULHR_TOO_LONG:
187 + goto error;
188 + }
189 +
190 + /*
191 + * object_info.sizep is unsigned long* (32-bit on Windows), but
192 + * st->base.size is size_t (64-bit). Use temporary variable.
193 + * Note: loose objects >4GB would still truncate here, but such
194 + * large loose objects are uncommon (they'd normally be packed).
195 + */
196 + oi.sizep = &size_ul;
197 + oi.typep = &st->base.type;
198 +
199 + if (parse_loose_header(st->hdr, &oi) < 0 || st->base.type < 0)
200 + goto error;
201 + st->base.size = size_ul;
202 +
203 + st->mapped = mapped;
204 + st->mapsize = mapsize;
205 + st->hdr_used = strlen(st->hdr) + 1;
206 + st->hdr_avail = st->z.total_out;
207 + st->z_state = ODB_LOOSE_READ_STREAM_INUSE;
208 + st->base.close = close_istream_loose;
209 + st->base.read = read_istream_loose;
210 +
211 + *out = &st->base;
212 +
213 + return 0;
214 +error:
215 + git_inflate_end(&st->z);
216 + munmap(mapped, mapsize);
217 + free(st);
218 + return -1;
219 +}
220 +
221 static void odb_source_loose_clear_cache(struct odb_source_loose *loose)
222 {
223 oidtree_clear(loose->cache);
@@ -84,6 +272,7 @@ struct odb_source_loose *odb_source_loose_new(struct odb_source_files *files)
272 loose->base.close = odb_source_loose_close;
273 loose->base.reprepare = odb_source_loose_reprepare;
274 loose->base.read_object_info = odb_source_loose_read_object_info;
275 + loose->base.read_object_stream = odb_source_loose_read_object_stream;
276
277 if (!is_absolute_path(loose->base.path))
278 chdir_notify_register(NULL, odb_source_loose_reparent, loose);