@matheusbsbs / matheus-bagliano / commits / 46bc0e731a

reftable: read reftable files

This supports reading a single reftable file. The commit introduces an abstract iterator type, which captures the usecases both of reading individual refs, and iterating over a segment of the ref namespace. Signed-off-by: Han-Wen Nienhuys <hanwen@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Han-Wen Nienhuys committed Oct 7, 2021 at 20:25 UTC 46bc0e731a70716485bc3c8bdaf7e6e6f88073a9
6 files changed +1231
Makefile
+2
@@ -2460,7 +2460,9 @@ REFTABLE_OBJS += reftable/basics.o
2460 REFTABLE_OBJS += reftable/error.o
2461 REFTABLE_OBJS += reftable/block.o
2462 REFTABLE_OBJS += reftable/blocksource.o
2463 +REFTABLE_OBJS += reftable/iter.o
2464 REFTABLE_OBJS += reftable/publicbasics.o
2465 +REFTABLE_OBJS += reftable/reader.o
2466 REFTABLE_OBJS += reftable/record.o
2467 REFTABLE_OBJS += reftable/refname.o
2468 REFTABLE_OBJS += reftable/generic.o
reftable/iter.c new
+194
@@ -0,0 +1,194 @@
1 +/*
2 +Copyright 2020 Google LLC
3 +
4 +Use of this source code is governed by a BSD-style
5 +license that can be found in the LICENSE file or at
6 +https://developers.google.com/open-source/licenses/bsd
7 +*/
8 +
9 +#include "iter.h"
10 +
11 +#include "system.h"
12 +
13 +#include "block.h"
14 +#include "generic.h"
15 +#include "constants.h"
16 +#include "reader.h"
17 +#include "reftable-error.h"
18 +
19 +int iterator_is_null(struct reftable_iterator *it)
20 +{
21 + return !it->ops;
22 +}
23 +
24 +static void filtering_ref_iterator_close(void *iter_arg)
25 +{
26 + struct filtering_ref_iterator *fri = iter_arg;
27 + strbuf_release(&fri->oid);
28 + reftable_iterator_destroy(&fri->it);
29 +}
30 +
31 +static int filtering_ref_iterator_next(void *iter_arg,
32 + struct reftable_record *rec)
33 +{
34 + struct filtering_ref_iterator *fri = iter_arg;
35 + struct reftable_ref_record *ref = rec->data;
36 + int err = 0;
37 + while (1) {
38 + err = reftable_iterator_next_ref(&fri->it, ref);
39 + if (err != 0) {
40 + break;
41 + }
42 +
43 + if (fri->double_check) {
44 + struct reftable_iterator it = { NULL };
45 +
46 + err = reftable_table_seek_ref(&fri->tab, &it,
47 + ref->refname);
48 + if (err == 0) {
49 + err = reftable_iterator_next_ref(&it, ref);
50 + }
51 +
52 + reftable_iterator_destroy(&it);
53 +
54 + if (err < 0) {
55 + break;
56 + }
57 +
58 + if (err > 0) {
59 + continue;
60 + }
61 + }
62 +
63 + if (ref->value_type == REFTABLE_REF_VAL2 &&
64 + (!memcmp(fri->oid.buf, ref->value.val2.target_value,
65 + fri->oid.len) ||
66 + !memcmp(fri->oid.buf, ref->value.val2.value,
67 + fri->oid.len)))
68 + return 0;
69 +
70 + if (ref->value_type == REFTABLE_REF_VAL1 &&
71 + !memcmp(fri->oid.buf, ref->value.val1, fri->oid.len)) {
72 + return 0;
73 + }
74 + }
75 +
76 + reftable_ref_record_release(ref);
77 + return err;
78 +}
79 +
80 +static struct reftable_iterator_vtable filtering_ref_iterator_vtable = {
81 + .next = &filtering_ref_iterator_next,
82 + .close = &filtering_ref_iterator_close,
83 +};
84 +
85 +void iterator_from_filtering_ref_iterator(struct reftable_iterator *it,
86 + struct filtering_ref_iterator *fri)
87 +{
88 + assert(!it->ops);
89 + it->iter_arg = fri;
90 + it->ops = &filtering_ref_iterator_vtable;
91 +}
92 +
93 +static void indexed_table_ref_iter_close(void *p)
94 +{
95 + struct indexed_table_ref_iter *it = p;
96 + block_iter_close(&it->cur);
97 + reftable_block_done(&it->block_reader.block);
98 + reftable_free(it->offsets);
99 + strbuf_release(&it->oid);
100 +}
101 +
102 +static int indexed_table_ref_iter_next_block(struct indexed_table_ref_iter *it)
103 +{
104 + uint64_t off;
105 + int err = 0;
106 + if (it->offset_idx == it->offset_len) {
107 + it->is_finished = 1;
108 + return 1;
109 + }
110 +
111 + reftable_block_done(&it->block_reader.block);
112 +
113 + off = it->offsets[it->offset_idx++];
114 + err = reader_init_block_reader(it->r, &it->block_reader, off,
115 + BLOCK_TYPE_REF);
116 + if (err < 0) {
117 + return err;
118 + }
119 + if (err > 0) {
120 + /* indexed block does not exist. */
121 + return REFTABLE_FORMAT_ERROR;
122 + }
123 + block_reader_start(&it->block_reader, &it->cur);
124 + return 0;
125 +}
126 +
127 +static int indexed_table_ref_iter_next(void *p, struct reftable_record *rec)
128 +{
129 + struct indexed_table_ref_iter *it = p;
130 + struct reftable_ref_record *ref = rec->data;
131 +
132 + while (1) {
133 + int err = block_iter_next(&it->cur, rec);
134 + if (err < 0) {
135 + return err;
136 + }
137 +
138 + if (err > 0) {
139 + err = indexed_table_ref_iter_next_block(it);
140 + if (err < 0) {
141 + return err;
142 + }
143 +
144 + if (it->is_finished) {
145 + return 1;
146 + }
147 + continue;
148 + }
149 + /* BUG */
150 + if (!memcmp(it->oid.buf, ref->value.val2.target_value,
151 + it->oid.len) ||
152 + !memcmp(it->oid.buf, ref->value.val2.value, it->oid.len)) {
153 + return 0;
154 + }
155 + }
156 +}
157 +
158 +int new_indexed_table_ref_iter(struct indexed_table_ref_iter **dest,
159 + struct reftable_reader *r, uint8_t *oid,
160 + int oid_len, uint64_t *offsets, int offset_len)
161 +{
162 + struct indexed_table_ref_iter empty = INDEXED_TABLE_REF_ITER_INIT;
163 + struct indexed_table_ref_iter *itr =
164 + reftable_calloc(sizeof(struct indexed_table_ref_iter));
165 + int err = 0;
166 +
167 + *itr = empty;
168 + itr->r = r;
169 + strbuf_add(&itr->oid, oid, oid_len);
170 +
171 + itr->offsets = offsets;
172 + itr->offset_len = offset_len;
173 +
174 + err = indexed_table_ref_iter_next_block(itr);
175 + if (err < 0) {
176 + reftable_free(itr);
177 + } else {
178 + *dest = itr;
179 + }
180 + return err;
181 +}
182 +
183 +static struct reftable_iterator_vtable indexed_table_ref_iter_vtable = {
184 + .next = &indexed_table_ref_iter_next,
185 + .close = &indexed_table_ref_iter_close,
186 +};
187 +
188 +void iterator_from_indexed_table_ref_iter(struct reftable_iterator *it,
189 + struct indexed_table_ref_iter *itr)
190 +{
191 + assert(!it->ops);
192 + it->iter_arg = itr;
193 + it->ops = &indexed_table_ref_iter_vtable;
194 +}
reftable/iter.h new
+69
@@ -0,0 +1,69 @@
1 +/*
2 +Copyright 2020 Google LLC
3 +
4 +Use of this source code is governed by a BSD-style
5 +license that can be found in the LICENSE file or at
6 +https://developers.google.com/open-source/licenses/bsd
7 +*/
8 +
9 +#ifndef ITER_H
10 +#define ITER_H
11 +
12 +#include "system.h"
13 +#include "block.h"
14 +#include "record.h"
15 +
16 +#include "reftable-iterator.h"
17 +#include "reftable-generic.h"
18 +
19 +/* Returns true for a zeroed out iterator, such as the one returned from
20 + * iterator_destroy. */
21 +int iterator_is_null(struct reftable_iterator *it);
22 +
23 +/* iterator that produces only ref records that point to `oid` */
24 +struct filtering_ref_iterator {
25 + int double_check;
26 + struct reftable_table tab;
27 + struct strbuf oid;
28 + struct reftable_iterator it;
29 +};
30 +#define FILTERING_REF_ITERATOR_INIT \
31 + { \
32 + .oid = STRBUF_INIT \
33 + }
34 +
35 +void iterator_from_filtering_ref_iterator(struct reftable_iterator *,
36 + struct filtering_ref_iterator *);
37 +
38 +/* iterator that produces only ref records that point to `oid`,
39 + * but using the object index.
40 + */
41 +struct indexed_table_ref_iter {
42 + struct reftable_reader *r;
43 + struct strbuf oid;
44 +
45 + /* mutable */
46 + uint64_t *offsets;
47 +
48 + /* Points to the next offset to read. */
49 + int offset_idx;
50 + int offset_len;
51 + struct block_reader block_reader;
52 + struct block_iter cur;
53 + int is_finished;
54 +};
55 +
56 +#define INDEXED_TABLE_REF_ITER_INIT \
57 + { \
58 + .cur = { .last_key = STRBUF_INIT }, .oid = STRBUF_INIT, \
59 + }
60 +
61 +void iterator_from_indexed_table_ref_iter(struct reftable_iterator *it,
62 + struct indexed_table_ref_iter *itr);
63 +
64 +/* Takes ownership of `offsets` */
65 +int new_indexed_table_ref_iter(struct indexed_table_ref_iter **dest,
66 + struct reftable_reader *r, uint8_t *oid,
67 + int oid_len, uint64_t *offsets, int offset_len);
68 +
69 +#endif
reftable/reader.c new
+801
@@ -0,0 +1,801 @@
1 +/*
2 +Copyright 2020 Google LLC
3 +
4 +Use of this source code is governed by a BSD-style
5 +license that can be found in the LICENSE file or at
6 +https://developers.google.com/open-source/licenses/bsd
7 +*/
8 +
9 +#include "reader.h"
10 +
11 +#include "system.h"
12 +#include "block.h"
13 +#include "constants.h"
14 +#include "generic.h"
15 +#include "iter.h"
16 +#include "record.h"
17 +#include "reftable-error.h"
18 +#include "reftable-generic.h"
19 +#include "tree.h"
20 +
21 +uint64_t block_source_size(struct reftable_block_source *source)
22 +{
23 + return source->ops->size(source->arg);
24 +}
25 +
26 +int block_source_read_block(struct reftable_block_source *source,
27 + struct reftable_block *dest, uint64_t off,
28 + uint32_t size)
29 +{
30 + int result = source->ops->read_block(source->arg, dest, off, size);
31 + dest->source = *source;
32 + return result;
33 +}
34 +
35 +void block_source_close(struct reftable_block_source *source)
36 +{
37 + if (!source->ops) {
38 + return;
39 + }
40 +
41 + source->ops->close(source->arg);
42 + source->ops = NULL;
43 +}
44 +
45 +static struct reftable_reader_offsets *
46 +reader_offsets_for(struct reftable_reader *r, uint8_t typ)
47 +{
48 + switch (typ) {
49 + case BLOCK_TYPE_REF:
50 + return &r->ref_offsets;
51 + case BLOCK_TYPE_LOG:
52 + return &r->log_offsets;
53 + case BLOCK_TYPE_OBJ:
54 + return &r->obj_offsets;
55 + }
56 + abort();
57 +}
58 +
59 +static int reader_get_block(struct reftable_reader *r,
60 + struct reftable_block *dest, uint64_t off,
61 + uint32_t sz)
62 +{
63 + if (off >= r->size)
64 + return 0;
65 +
66 + if (off + sz > r->size) {
67 + sz = r->size - off;
68 + }
69 +
70 + return block_source_read_block(&r->source, dest, off, sz);
71 +}
72 +
73 +uint32_t reftable_reader_hash_id(struct reftable_reader *r)
74 +{
75 + return r->hash_id;
76 +}
77 +
78 +const char *reader_name(struct reftable_reader *r)
79 +{
80 + return r->name;
81 +}
82 +
83 +static int parse_footer(struct reftable_reader *r, uint8_t *footer,
84 + uint8_t *header)
85 +{
86 + uint8_t *f = footer;
87 + uint8_t first_block_typ;
88 + int err = 0;
89 + uint32_t computed_crc;
90 + uint32_t file_crc;
91 +
92 + if (memcmp(f, "REFT", 4)) {
93 + err = REFTABLE_FORMAT_ERROR;
94 + goto done;
95 + }
96 + f += 4;
97 +
98 + if (memcmp(footer, header, header_size(r->version))) {
99 + err = REFTABLE_FORMAT_ERROR;
100 + goto done;
101 + }
102 +
103 + f++;
104 + r->block_size = get_be24(f);
105 +
106 + f += 3;
107 + r->min_update_index = get_be64(f);
108 + f += 8;
109 + r->max_update_index = get_be64(f);
110 + f += 8;
111 +
112 + if (r->version == 1) {
113 + r->hash_id = GIT_SHA1_FORMAT_ID;
114 + } else {
115 + r->hash_id = get_be32(f);
116 + switch (r->hash_id) {
117 + case GIT_SHA1_FORMAT_ID:
118 + break;
119 + case GIT_SHA256_FORMAT_ID:
120 + break;
121 + default:
122 + err = REFTABLE_FORMAT_ERROR;
123 + goto done;
124 + }
125 + f += 4;
126 + }
127 +
128 + r->ref_offsets.index_offset = get_be64(f);
129 + f += 8;
130 +
131 + r->obj_offsets.offset = get_be64(f);
132 + f += 8;
133 +
134 + r->object_id_len = r->obj_offsets.offset & ((1 << 5) - 1);
135 + r->obj_offsets.offset >>= 5;
136 +
137 + r->obj_offsets.index_offset = get_be64(f);
138 + f += 8;
139 + r->log_offsets.offset = get_be64(f);
140 + f += 8;
141 + r->log_offsets.index_offset = get_be64(f);
142 + f += 8;
143 +
144 + computed_crc = crc32(0, footer, f - footer);
145 + file_crc = get_be32(f);
146 + f += 4;
147 + if (computed_crc != file_crc) {
148 + err = REFTABLE_FORMAT_ERROR;
149 + goto done;
150 + }
151 +
152 + first_block_typ = header[header_size(r->version)];
153 + r->ref_offsets.is_present = (first_block_typ == BLOCK_TYPE_REF);
154 + r->ref_offsets.offset = 0;
155 + r->log_offsets.is_present = (first_block_typ == BLOCK_TYPE_LOG ||
156 + r->log_offsets.offset > 0);
157 + r->obj_offsets.is_present = r->obj_offsets.offset > 0;
158 + err = 0;
159 +done:
160 + return err;
161 +}
162 +
163 +int init_reader(struct reftable_reader *r, struct reftable_block_source *source,
164 + const char *name)
165 +{
166 + struct reftable_block footer = { NULL };
167 + struct reftable_block header = { NULL };
168 + int err = 0;
169 + uint64_t file_size = block_source_size(source);
170 +
171 + /* Need +1 to read type of first block. */
172 + uint32_t read_size = header_size(2) + 1; /* read v2 because it's larger. */
173 + memset(r, 0, sizeof(struct reftable_reader));
174 +
175 + if (read_size > file_size) {
176 + err = REFTABLE_FORMAT_ERROR;
177 + goto done;
178 + }
179 +
180 + err = block_source_read_block(source, &header, 0, read_size);
181 + if (err != read_size) {
182 + err = REFTABLE_IO_ERROR;
183 + goto done;
184 + }
185 +
186 + if (memcmp(header.data, "REFT", 4)) {
187 + err = REFTABLE_FORMAT_ERROR;
188 + goto done;
189 + }
190 + r->version = header.data[4];
191 + if (r->version != 1 && r->version != 2) {
192 + err = REFTABLE_FORMAT_ERROR;
193 + goto done;
194 + }
195 +
196 + r->size = file_size - footer_size(r->version);
197 + r->source = *source;
198 + r->name = xstrdup(name);
199 + r->hash_id = 0;
200 +
201 + err = block_source_read_block(source, &footer, r->size,
202 + footer_size(r->version));
203 + if (err != footer_size(r->version)) {
204 + err = REFTABLE_IO_ERROR;
205 + goto done;
206 + }
207 +
208 + err = parse_footer(r, footer.data, header.data);
209 +done:
210 + reftable_block_done(&footer);
211 + reftable_block_done(&header);
212 + return err;
213 +}
214 +
215 +struct table_iter {
216 + struct reftable_reader *r;
217 + uint8_t typ;
218 + uint64_t block_off;
219 + struct block_iter bi;
220 + int is_finished;
221 +};
222 +#define TABLE_ITER_INIT \
223 + { \
224 + .bi = {.last_key = STRBUF_INIT } \
225 + }
226 +
227 +static void table_iter_copy_from(struct table_iter *dest,
228 + struct table_iter *src)
229 +{
230 + dest->r = src->r;
231 + dest->typ = src->typ;
232 + dest->block_off = src->block_off;
233 + dest->is_finished = src->is_finished;
234 + block_iter_copy_from(&dest->bi, &src->bi);
235 +}
236 +
237 +static int table_iter_next_in_block(struct table_iter *ti,
238 + struct reftable_record *rec)
239 +{
240 + int res = block_iter_next(&ti->bi, rec);
241 + if (res == 0 && reftable_record_type(rec) == BLOCK_TYPE_REF) {
242 + ((struct reftable_ref_record *)rec->data)->update_index +=
243 + ti->r->min_update_index;
244 + }
245 +
246 + return res;
247 +}
248 +
249 +static void table_iter_block_done(struct table_iter *ti)
250 +{
251 + if (!ti->bi.br) {
252 + return;
253 + }
254 + reftable_block_done(&ti->bi.br->block);
255 + FREE_AND_NULL(ti->bi.br);
256 +
257 + ti->bi.last_key.len = 0;
258 + ti->bi.next_off = 0;
259 +}
260 +
261 +static int32_t extract_block_size(uint8_t *data, uint8_t *typ, uint64_t off,
262 + int version)
263 +{
264 + int32_t result = 0;
265 +
266 + if (off == 0) {
267 + data += header_size(version);
268 + }
269 +
270 + *typ = data[0];
271 + if (reftable_is_block_type(*typ)) {
272 + result = get_be24(data + 1);
273 + }
274 + return result;
275 +}
276 +
277 +int reader_init_block_reader(struct reftable_reader *r, struct block_reader *br,
278 + uint64_t next_off, uint8_t want_typ)
279 +{
280 + int32_t guess_block_size = r->block_size ? r->block_size :
281 + DEFAULT_BLOCK_SIZE;
282 + struct reftable_block block = { NULL };
283 + uint8_t block_typ = 0;
284 + int err = 0;
285 + uint32_t header_off = next_off ? 0 : header_size(r->version);
286 + int32_t block_size = 0;
287 +
288 + if (next_off >= r->size)
289 + return 1;
290 +
291 + err = reader_get_block(r, &block, next_off, guess_block_size);
292 + if (err < 0)
293 + return err;
294 +
295 + block_size = extract_block_size(block.data, &block_typ, next_off,
296 + r->version);
297 + if (block_size < 0)
298 + return block_size;
299 +
300 + if (want_typ != BLOCK_TYPE_ANY && block_typ != want_typ) {
301 + reftable_block_done(&block);
302 + return 1;
303 + }
304 +
305 + if (block_size > guess_block_size) {
306 + reftable_block_done(&block);
307 + err = reader_get_block(r, &block, next_off, block_size);
308 + if (err < 0) {
309 + return err;
310 + }
311 + }
312 +
313 + return block_reader_init(br, &block, header_off, r->block_size,
314 + hash_size(r->hash_id));
315 +}
316 +
317 +static int table_iter_next_block(struct table_iter *dest,
318 + struct table_iter *src)
319 +{
320 + uint64_t next_block_off = src->block_off + src->bi.br->full_block_size;
321 + struct block_reader br = { 0 };
322 + int err = 0;
323 +
324 + dest->r = src->r;
325 + dest->typ = src->typ;
326 + dest->block_off = next_block_off;
327 +
328 + err = reader_init_block_reader(src->r, &br, next_block_off, src->typ);
329 + if (err > 0) {
330 + dest->is_finished = 1;
331 + return 1;
332 + }
333 + if (err != 0)
334 + return err;
335 + else {
336 + struct block_reader *brp =
337 + reftable_malloc(sizeof(struct block_reader));
338 + *brp = br;
339 +
340 + dest->is_finished = 0;
341 + block_reader_start(brp, &dest->bi);
342 + }
343 + return 0;
344 +}
345 +
346 +static int table_iter_next(struct table_iter *ti, struct reftable_record *rec)
347 +{
348 + if (reftable_record_type(rec) != ti->typ)
349 + return REFTABLE_API_ERROR;
350 +
351 + while (1) {
352 + struct table_iter next = TABLE_ITER_INIT;
353 + int err = 0;
354 + if (ti->is_finished) {
355 + return 1;
356 + }
357 +
358 + err = table_iter_next_in_block(ti, rec);
359 + if (err <= 0) {
360 + return err;
361 + }
362 +
363 + err = table_iter_next_block(&next, ti);
364 + if (err != 0) {
365 + ti->is_finished = 1;
366 + }
367 + table_iter_block_done(ti);
368 + if (err != 0) {
369 + return err;
370 + }
371 + table_iter_copy_from(ti, &next);
372 + block_iter_close(&next.bi);
373 + }
374 +}
375 +
376 +static int table_iter_next_void(void *ti, struct reftable_record *rec)
377 +{
378 + return table_iter_next(ti, rec);
379 +}
380 +
381 +static void table_iter_close(void *p)
382 +{
383 + struct table_iter *ti = p;
384 + table_iter_block_done(ti);
385 + block_iter_close(&ti->bi);
386 +}
387 +
388 +static struct reftable_iterator_vtable table_iter_vtable = {
389 + .next = &table_iter_next_void,
390 + .close = &table_iter_close,
391 +};
392 +
393 +static void iterator_from_table_iter(struct reftable_iterator *it,
394 + struct table_iter *ti)
395 +{
396 + assert(!it->ops);
397 + it->iter_arg = ti;
398 + it->ops = &table_iter_vtable;
399 +}
400 +
401 +static int reader_table_iter_at(struct reftable_reader *r,
402 + struct table_iter *ti, uint64_t off,
403 + uint8_t typ)
404 +{
405 + struct block_reader br = { 0 };
406 + struct block_reader *brp = NULL;
407 +
408 + int err = reader_init_block_reader(r, &br, off, typ);
409 + if (err != 0)
410 + return err;
411 +
412 + brp = reftable_malloc(sizeof(struct block_reader));
413 + *brp = br;
414 + ti->r = r;
415 + ti->typ = block_reader_type(brp);
416 + ti->block_off = off;
417 + block_reader_start(brp, &ti->bi);
418 + return 0;
419 +}
420 +
421 +static int reader_start(struct reftable_reader *r, struct table_iter *ti,
422 + uint8_t typ, int index)
423 +{
424 + struct reftable_reader_offsets *offs = reader_offsets_for(r, typ);
425 + uint64_t off = offs->offset;
426 + if (index) {
427 + off = offs->index_offset;
428 + if (off == 0) {
429 + return 1;
430 + }
431 + typ = BLOCK_TYPE_INDEX;
432 + }
433 +
434 + return reader_table_iter_at(r, ti, off, typ);
435 +}
436 +
437 +static int reader_seek_linear(struct reftable_reader *r, struct table_iter *ti,
438 + struct reftable_record *want)
439 +{
440 + struct reftable_record rec =
441 + reftable_new_record(reftable_record_type(want));
442 + struct strbuf want_key = STRBUF_INIT;
443 + struct strbuf got_key = STRBUF_INIT;
444 + struct table_iter next = TABLE_ITER_INIT;
445 + int err = -1;
446 +
447 + reftable_record_key(want, &want_key);
448 +
449 + while (1) {
450 + err = table_iter_next_block(&next, ti);
451 + if (err < 0)
452 + goto done;
453 +
454 + if (err > 0) {
455 + break;
456 + }
457 +
458 + err = block_reader_first_key(next.bi.br, &got_key);
459 + if (err < 0)
460 + goto done;
461 +
462 + if (strbuf_cmp(&got_key, &want_key) > 0) {
463 + table_iter_block_done(&next);
464 + break;
465 + }
466 +
467 + table_iter_block_done(ti);
468 + table_iter_copy_from(ti, &next);
469 + }
470 +
471 + err = block_iter_seek(&ti->bi, &want_key);
472 + if (err < 0)
473 + goto done;
474 + err = 0;
475 +
476 +done:
477 + block_iter_close(&next.bi);
478 + reftable_record_destroy(&rec);
479 + strbuf_release(&want_key);
480 + strbuf_release(&got_key);
481 + return err;
482 +}
483 +
484 +static int reader_seek_indexed(struct reftable_reader *r,
485 + struct reftable_iterator *it,
486 + struct reftable_record *rec)
487 +{
488 + struct reftable_index_record want_index = { .last_key = STRBUF_INIT };
489 + struct reftable_record want_index_rec = { NULL };
490 + struct reftable_index_record index_result = { .last_key = STRBUF_INIT };
491 + struct reftable_record index_result_rec = { NULL };
492 + struct table_iter index_iter = TABLE_ITER_INIT;
493 + struct table_iter next = TABLE_ITER_INIT;
494 + int err = 0;
495 +
496 + reftable_record_key(rec, &want_index.last_key);
497 + reftable_record_from_index(&want_index_rec, &want_index);
498 + reftable_record_from_index(&index_result_rec, &index_result);
499 +
500 + err = reader_start(r, &index_iter, reftable_record_type(rec), 1);
501 + if (err < 0)
502 + goto done;
503 +
504 + err = reader_seek_linear(r, &index_iter, &want_index_rec);
505 + while (1) {
506 + err = table_iter_next(&index_iter, &index_result_rec);
507 + table_iter_block_done(&index_iter);
508 + if (err != 0)
509 + goto done;
510 +
511 + err = reader_table_iter_at(r, &next, index_result.offset, 0);
512 + if (err != 0)
513 + goto done;
514 +
515 + err = block_iter_seek(&next.bi, &want_index.last_key);
516 + if (err < 0)
517 + goto done;
518 +
519 + if (next.typ == reftable_record_type(rec)) {
520 + err = 0;
521 + break;
522 + }
523 +
524 + if (next.typ != BLOCK_TYPE_INDEX) {
525 + err = REFTABLE_FORMAT_ERROR;
526 + break;
527 + }
528 +
529 + table_iter_copy_from(&index_iter, &next);
530 + }
531 +
532 + if (err == 0) {
533 + struct table_iter empty = TABLE_ITER_INIT;
534 + struct table_iter *malloced =
535 + reftable_calloc(sizeof(struct table_iter));
536 + *malloced = empty;
537 + table_iter_copy_from(malloced, &next);
538 + iterator_from_table_iter(it, malloced);
539 + }
540 +done:
541 + block_iter_close(&next.bi);
542 + table_iter_close(&index_iter);
543 + reftable_record_release(&want_index_rec);
544 + reftable_record_release(&index_result_rec);
545 + return err;
546 +}
547 +
548 +static int reader_seek_internal(struct reftable_reader *r,
549 + struct reftable_iterator *it,
550 + struct reftable_record *rec)
551 +{
552 + struct reftable_reader_offsets *offs =
553 + reader_offsets_for(r, reftable_record_type(rec));
554 + uint64_t idx = offs->index_offset;
555 + struct table_iter ti = TABLE_ITER_INIT;
556 + int err = 0;
557 + if (idx > 0)
558 + return reader_seek_indexed(r, it, rec);
559 +
560 + err = reader_start(r, &ti, reftable_record_type(rec), 0);
561 + if (err < 0)
562 + return err;
563 + err = reader_seek_linear(r, &ti, rec);
564 + if (err < 0)
565 + return err;
566 + else {
567 + struct table_iter *p =
568 + reftable_malloc(sizeof(struct table_iter));
569 + *p = ti;
570 + iterator_from_table_iter(it, p);
571 + }
572 +
573 + return 0;
574 +}
575 +
576 +static int reader_seek(struct reftable_reader *r, struct reftable_iterator *it,
577 + struct reftable_record *rec)
578 +{
579 + uint8_t typ = reftable_record_type(rec);
580 +
581 + struct reftable_reader_offsets *offs = reader_offsets_for(r, typ);
582 + if (!offs->is_present) {
583 + iterator_set_empty(it);
584 + return 0;
585 + }
586 +
587 + return reader_seek_internal(r, it, rec);
588 +}
589 +
590 +int reftable_reader_seek_ref(struct reftable_reader *r,
591 + struct reftable_iterator *it, const char *name)
592 +{
593 + struct reftable_ref_record ref = {
594 + .refname = (char *)name,
595 + };
596 + struct reftable_record rec = { NULL };
597 + reftable_record_from_ref(&rec, &ref);
598 + return reader_seek(r, it, &rec);
599 +}
600 +
601 +int reftable_reader_seek_log_at(struct reftable_reader *r,
602 + struct reftable_iterator *it, const char *name,
603 + uint64_t update_index)
604 +{
605 + struct reftable_log_record log = {
606 + .refname = (char *)name,
607 + .update_index = update_index,
608 + };
609 + struct reftable_record rec = { NULL };
610 + reftable_record_from_log(&rec, &log);
611 + return reader_seek(r, it, &rec);
612 +}
613 +
614 +int reftable_reader_seek_log(struct reftable_reader *r,
615 + struct reftable_iterator *it, const char *name)
616 +{
617 + uint64_t max = ~((uint64_t)0);
618 + return reftable_reader_seek_log_at(r, it, name, max);
619 +}
620 +
621 +void reader_close(struct reftable_reader *r)
622 +{
623 + block_source_close(&r->source);
624 + FREE_AND_NULL(r->name);
625 +}
626 +
627 +int reftable_new_reader(struct reftable_reader **p,
628 + struct reftable_block_source *src, char const *name)
629 +{
630 + struct reftable_reader *rd =
631 + reftable_calloc(sizeof(struct reftable_reader));
632 + int err = init_reader(rd, src, name);
633 + if (err == 0) {
634 + *p = rd;
635 + } else {
636 + block_source_close(src);
637 + reftable_free(rd);
638 + }
639 + return err;
640 +}
641 +
642 +void reftable_reader_free(struct reftable_reader *r)
643 +{
644 + reader_close(r);
645 + reftable_free(r);
646 +}
647 +
648 +static int reftable_reader_refs_for_indexed(struct reftable_reader *r,
649 + struct reftable_iterator *it,
650 + uint8_t *oid)
651 +{
652 + struct reftable_obj_record want = {
653 + .hash_prefix = oid,
654 + .hash_prefix_len = r->object_id_len,
655 + };
656 + struct reftable_record want_rec = { NULL };
657 + struct reftable_iterator oit = { NULL };
658 + struct reftable_obj_record got = { NULL };
659 + struct reftable_record got_rec = { NULL };
660 + int err = 0;
661 + struct indexed_table_ref_iter *itr = NULL;
662 +
663 + /* Look through the reverse index. */
664 + reftable_record_from_obj(&want_rec, &want);
665 + err = reader_seek(r, &oit, &want_rec);
666 + if (err != 0)
667 + goto done;
668 +
669 + /* read out the reftable_obj_record */
670 + reftable_record_from_obj(&got_rec, &got);
671 + err = iterator_next(&oit, &got_rec);
672 + if (err < 0)
673 + goto done;
674 +
675 + if (err > 0 ||
676 + memcmp(want.hash_prefix, got.hash_prefix, r->object_id_len)) {
677 + /* didn't find it; return empty iterator */
678 + iterator_set_empty(it);
679 + err = 0;
680 + goto done;
681 + }
682 +
683 + err = new_indexed_table_ref_iter(&itr, r, oid, hash_size(r->hash_id),
684 + got.offsets, got.offset_len);
685 + if (err < 0)
686 + goto done;
687 + got.offsets = NULL;
688 + iterator_from_indexed_table_ref_iter(it, itr);
689 +
690 +done:
691 + reftable_iterator_destroy(&oit);
692 + reftable_record_release(&got_rec);
693 + return err;
694 +}
695 +
696 +static int reftable_reader_refs_for_unindexed(struct reftable_reader *r,
697 + struct reftable_iterator *it,
698 + uint8_t *oid)
699 +{
700 + struct table_iter ti_empty = TABLE_ITER_INIT;
701 + struct table_iter *ti = reftable_calloc(sizeof(struct table_iter));
702 + struct filtering_ref_iterator *filter = NULL;
703 + struct filtering_ref_iterator empty = FILTERING_REF_ITERATOR_INIT;
704 + int oid_len = hash_size(r->hash_id);
705 + int err;
706 +
707 + *ti = ti_empty;
708 + err = reader_start(r, ti, BLOCK_TYPE_REF, 0);
709 + if (err < 0) {
710 + reftable_free(ti);
711 + return err;
712 + }
713 +
714 + filter = reftable_malloc(sizeof(struct filtering_ref_iterator));
715 + *filter = empty;
716 +
717 + strbuf_add(&filter->oid, oid, oid_len);
718 + reftable_table_from_reader(&filter->tab, r);
719 + filter->double_check = 0;
720 + iterator_from_table_iter(&filter->it, ti);
721 +
722 + iterator_from_filtering_ref_iterator(it, filter);
723 + return 0;
724 +}
725 +
726 +int reftable_reader_refs_for(struct reftable_reader *r,
727 + struct reftable_iterator *it, uint8_t *oid)
728 +{
729 + if (r->obj_offsets.is_present)
730 + return reftable_reader_refs_for_indexed(r, it, oid);
731 + return reftable_reader_refs_for_unindexed(r, it, oid);
732 +}
733 +
734 +uint64_t reftable_reader_max_update_index(struct reftable_reader *r)
735 +{
736 + return r->max_update_index;
737 +}
738 +
739 +uint64_t reftable_reader_min_update_index(struct reftable_reader *r)
740 +{
741 + return r->min_update_index;
742 +}
743 +
744 +/* generic table interface. */
745 +
746 +static int reftable_reader_seek_void(void *tab, struct reftable_iterator *it,
747 + struct reftable_record *rec)
748 +{
749 + return reader_seek(tab, it, rec);
750 +}
751 +
752 +static uint32_t reftable_reader_hash_id_void(void *tab)
753 +{
754 + return reftable_reader_hash_id(tab);
755 +}
756 +
757 +static uint64_t reftable_reader_min_update_index_void(void *tab)
758 +{
759 + return reftable_reader_min_update_index(tab);
760 +}
761 +
762 +static uint64_t reftable_reader_max_update_index_void(void *tab)
763 +{
764 + return reftable_reader_max_update_index(tab);
765 +}
766 +
767 +static struct reftable_table_vtable reader_vtable = {
768 + .seek_record = reftable_reader_seek_void,
769 + .hash_id = reftable_reader_hash_id_void,
770 + .min_update_index = reftable_reader_min_update_index_void,
771 + .max_update_index = reftable_reader_max_update_index_void,
772 +};
773 +
774 +void reftable_table_from_reader(struct reftable_table *tab,
775 + struct reftable_reader *reader)
776 +{
777 + assert(!tab->ops);
778 + tab->ops = &reader_vtable;
779 + tab->table_arg = reader;
780 +}
781 +
782 +
783 +int reftable_reader_print_file(const char *tablename)
784 +{
785 + struct reftable_block_source src = { NULL };
786 + int err = reftable_block_source_from_file(&src, tablename);
787 + struct reftable_reader *r = NULL;
788 + struct reftable_table tab = { NULL };
789 + if (err < 0)
790 + goto done;
791 +
792 + err = reftable_new_reader(&r, &src, tablename);
793 + if (err < 0)
794 + goto done;
795 +
796 + reftable_table_from_reader(&tab, r);
797 + err = reftable_table_print(&tab);
798 +done:
799 + reftable_reader_free(r);
800 + return err;
801 +}
reftable/reader.h new
+64
@@ -0,0 +1,64 @@
1 +/*
2 +Copyright 2020 Google LLC
3 +
4 +Use of this source code is governed by a BSD-style
5 +license that can be found in the LICENSE file or at
6 +https://developers.google.com/open-source/licenses/bsd
7 +*/
8 +
9 +#ifndef READER_H
10 +#define READER_H
11 +
12 +#include "block.h"
13 +#include "record.h"
14 +#include "reftable-iterator.h"
15 +#include "reftable-reader.h"
16 +
17 +uint64_t block_source_size(struct reftable_block_source *source);
18 +
19 +int block_source_read_block(struct reftable_block_source *source,
20 + struct reftable_block *dest, uint64_t off,
21 + uint32_t size);
22 +void block_source_close(struct reftable_block_source *source);
23 +
24 +/* metadata for a block type */
25 +struct reftable_reader_offsets {
26 + int is_present;
27 + uint64_t offset;
28 + uint64_t index_offset;
29 +};
30 +
31 +/* The state for reading a reftable file. */
32 +struct reftable_reader {
33 + /* for convience, associate a name with the instance. */
34 + char *name;
35 + struct reftable_block_source source;
36 +
37 + /* Size of the file, excluding the footer. */
38 + uint64_t size;
39 +
40 + /* 'sha1' for SHA1, 's256' for SHA-256 */
41 + uint32_t hash_id;
42 +
43 + uint32_t block_size;
44 + uint64_t min_update_index;
45 + uint64_t max_update_index;
46 + /* Length of the OID keys in the 'o' section */
47 + int object_id_len;
48 + int version;
49 +
50 + struct reftable_reader_offsets ref_offsets;
51 + struct reftable_reader_offsets obj_offsets;
52 + struct reftable_reader_offsets log_offsets;
53 +};
54 +
55 +int init_reader(struct reftable_reader *r, struct reftable_block_source *source,
56 + const char *name);
57 +void reader_close(struct reftable_reader *r);
58 +const char *reader_name(struct reftable_reader *r);
59 +
60 +/* initialize a block reader to read from `r` */
61 +int reader_init_block_reader(struct reftable_reader *r, struct block_reader *br,
62 + uint64_t next_off, uint8_t want_typ);
63 +
64 +#endif
reftable/reftable-reader.h new
+101
@@ -0,0 +1,101 @@
1 +/*
2 + Copyright 2020 Google LLC
3 +
4 + Use of this source code is governed by a BSD-style
5 + license that can be found in the LICENSE file or at
6 + https://developers.google.com/open-source/licenses/bsd
7 +*/
8 +
9 +#ifndef REFTABLE_READER_H
10 +#define REFTABLE_READER_H
11 +
12 +#include "reftable-iterator.h"
13 +#include "reftable-blocksource.h"
14 +
15 +/*
16 + * Reading single tables
17 + *
18 + * The follow routines are for reading single files. For an
19 + * application-level interface, skip ahead to struct
20 + * reftable_merged_table and struct reftable_stack.
21 + */
22 +
23 +/* The reader struct is a handle to an open reftable file. */
24 +struct reftable_reader;
25 +
26 +/* Generic table. */
27 +struct reftable_table;
28 +
29 +/* reftable_new_reader opens a reftable for reading. If successful,
30 + * returns 0 code and sets pp. The name is used for creating a
31 + * stack. Typically, it is the basename of the file. The block source
32 + * `src` is owned by the reader, and is closed on calling
33 + * reftable_reader_destroy(). On error, the block source `src` is
34 + * closed as well.
35 + */
36 +int reftable_new_reader(struct reftable_reader **pp,
37 + struct reftable_block_source *src, const char *name);
38 +
39 +/* reftable_reader_seek_ref returns an iterator where 'name' would be inserted
40 + in the table. To seek to the start of the table, use name = "".
41 +
42 + example:
43 +
44 + struct reftable_reader *r = NULL;
45 + int err = reftable_new_reader(&r, &src, "filename");
46 + if (err < 0) { ... }
47 + struct reftable_iterator it = {0};
48 + err = reftable_reader_seek_ref(r, &it, "refs/heads/master");
49 + if (err < 0) { ... }
50 + struct reftable_ref_record ref = {0};
51 + while (1) {
52 + err = reftable_iterator_next_ref(&it, &ref);
53 + if (err > 0) {
54 + break;
55 + }
56 + if (err < 0) {
57 + ..error handling..
58 + }
59 + ..found..
60 + }
61 + reftable_iterator_destroy(&it);
62 + reftable_ref_record_release(&ref);
63 +*/
64 +int reftable_reader_seek_ref(struct reftable_reader *r,
65 + struct reftable_iterator *it, const char *name);
66 +
67 +/* returns the hash ID used in this table. */
68 +uint32_t reftable_reader_hash_id(struct reftable_reader *r);
69 +
70 +/* seek to logs for the given name, older than update_index. To seek to the
71 + start of the table, use name = "".
72 +*/
73 +int reftable_reader_seek_log_at(struct reftable_reader *r,
74 + struct reftable_iterator *it, const char *name,
75 + uint64_t update_index);
76 +
77 +/* seek to newest log entry for given name. */
78 +int reftable_reader_seek_log(struct reftable_reader *r,
79 + struct reftable_iterator *it, const char *name);
80 +
81 +/* closes and deallocates a reader. */
82 +void reftable_reader_free(struct reftable_reader *);
83 +
84 +/* return an iterator for the refs pointing to `oid`. */
85 +int reftable_reader_refs_for(struct reftable_reader *r,
86 + struct reftable_iterator *it, uint8_t *oid);
87 +
88 +/* return the max_update_index for a table */
89 +uint64_t reftable_reader_max_update_index(struct reftable_reader *r);
90 +
91 +/* return the min_update_index for a table */
92 +uint64_t reftable_reader_min_update_index(struct reftable_reader *r);
93 +
94 +/* creates a generic table from a file reader. */
95 +void reftable_table_from_reader(struct reftable_table *tab,
96 + struct reftable_reader *reader);
97 +
98 +/* print table onto stdout for debugging. */
99 +int reftable_reader_print_file(const char *tablename);
100 +
101 +#endif