@matheusbsbs / matheus-bagliano / commits / 1ae2b8cda8

reftable: add merged table view

This adds an abstract, read-only interface to the ref database. This primitive is used to construct the read view of the ref database (the read view is constructed by merging several *.ref files). It also provides the mechanism to provide a unified view of the refs in the main repository and the per-worktree refs. Signed-off-by: Han-Wen Nienhuys <hanwen@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Han-Wen Nienhuys committed Oct 7, 2021 at 20:25 UTC 1ae2b8cda84c4c662d8f60898e034d9643f097b6
6 files changed +943
Makefile
+2
@@ -2462,6 +2462,7 @@ REFTABLE_OBJS += reftable/block.o
2462 REFTABLE_OBJS += reftable/blocksource.o
2463 REFTABLE_OBJS += reftable/iter.o
2464 REFTABLE_OBJS += reftable/publicbasics.o
2465 +REFTABLE_OBJS += reftable/merged.o
2466 REFTABLE_OBJS += reftable/pq.o
2467 REFTABLE_OBJS += reftable/reader.o
2468 REFTABLE_OBJS += reftable/record.o
@@ -2473,6 +2474,7 @@ REFTABLE_OBJS += reftable/writer.o
2474
2475 REFTABLE_TEST_OBJS += reftable/basics_test.o
2476 REFTABLE_TEST_OBJS += reftable/block_test.o
2477 +REFTABLE_TEST_OBJS += reftable/merged_test.o
2478 REFTABLE_TEST_OBJS += reftable/pq_test.o
2479 REFTABLE_TEST_OBJS += reftable/record_test.o
2480 REFTABLE_TEST_OBJS += reftable/readwrite_test.o
reftable/merged.c new
+362
@@ -0,0 +1,362 @@
1 +/*
2 +Copyright 2020 Google LLC
3 +
4 +Use of this source code is governed by a BSD-style
5 +license that can be found in the LICENSE file or at
6 +https://developers.google.com/open-source/licenses/bsd
7 +*/
8 +
9 +#include "merged.h"
10 +
11 +#include "constants.h"
12 +#include "iter.h"
13 +#include "pq.h"
14 +#include "reader.h"
15 +#include "record.h"
16 +#include "generic.h"
17 +#include "reftable-merged.h"
18 +#include "reftable-error.h"
19 +#include "system.h"
20 +
21 +static int merged_iter_init(struct merged_iter *mi)
22 +{
23 + int i = 0;
24 + for (i = 0; i < mi->stack_len; i++) {
25 + struct reftable_record rec = reftable_new_record(mi->typ);
26 + int err = iterator_next(&mi->stack[i], &rec);
27 + if (err < 0) {
28 + return err;
29 + }
30 +
31 + if (err > 0) {
32 + reftable_iterator_destroy(&mi->stack[i]);
33 + reftable_record_destroy(&rec);
34 + } else {
35 + struct pq_entry e = {
36 + .rec = rec,
37 + .index = i,
38 + };
39 + merged_iter_pqueue_add(&mi->pq, e);
40 + }
41 + }
42 +
43 + return 0;
44 +}
45 +
46 +static void merged_iter_close(void *p)
47 +{
48 + struct merged_iter *mi = p;
49 + int i = 0;
50 + merged_iter_pqueue_release(&mi->pq);
51 + for (i = 0; i < mi->stack_len; i++) {
52 + reftable_iterator_destroy(&mi->stack[i]);
53 + }
54 + reftable_free(mi->stack);
55 +}
56 +
57 +static int merged_iter_advance_nonnull_subiter(struct merged_iter *mi,
58 + size_t idx)
59 +{
60 + struct reftable_record rec = reftable_new_record(mi->typ);
61 + struct pq_entry e = {
62 + .rec = rec,
63 + .index = idx,
64 + };
65 + int err = iterator_next(&mi->stack[idx], &rec);
66 + if (err < 0)
67 + return err;
68 +
69 + if (err > 0) {
70 + reftable_iterator_destroy(&mi->stack[idx]);
71 + reftable_record_destroy(&rec);
72 + return 0;
73 + }
74 +
75 + merged_iter_pqueue_add(&mi->pq, e);
76 + return 0;
77 +}
78 +
79 +static int merged_iter_advance_subiter(struct merged_iter *mi, size_t idx)
80 +{
81 + if (iterator_is_null(&mi->stack[idx]))
82 + return 0;
83 + return merged_iter_advance_nonnull_subiter(mi, idx);
84 +}
85 +
86 +static int merged_iter_next_entry(struct merged_iter *mi,
87 + struct reftable_record *rec)
88 +{
89 + struct strbuf entry_key = STRBUF_INIT;
90 + struct pq_entry entry = { 0 };
91 + int err = 0;
92 +
93 + if (merged_iter_pqueue_is_empty(mi->pq))
94 + return 1;
95 +
96 + entry = merged_iter_pqueue_remove(&mi->pq);
97 + err = merged_iter_advance_subiter(mi, entry.index);
98 + if (err < 0)
99 + return err;
100 +
101 + /*
102 + One can also use reftable as datacenter-local storage, where the ref
103 + database is maintained in globally consistent database (eg.
104 + CockroachDB or Spanner). In this scenario, replication delays together
105 + with compaction may cause newer tables to contain older entries. In
106 + such a deployment, the loop below must be changed to collect all
107 + entries for the same key, and return new the newest one.
108 + */
109 + reftable_record_key(&entry.rec, &entry_key);
110 + while (!merged_iter_pqueue_is_empty(mi->pq)) {
111 + struct pq_entry top = merged_iter_pqueue_top(mi->pq);
112 + struct strbuf k = STRBUF_INIT;
113 + int err = 0, cmp = 0;
114 +
115 + reftable_record_key(&top.rec, &k);
116 +
117 + cmp = strbuf_cmp(&k, &entry_key);
118 + strbuf_release(&k);
119 +
120 + if (cmp > 0) {
121 + break;
122 + }
123 +
124 + merged_iter_pqueue_remove(&mi->pq);
125 + err = merged_iter_advance_subiter(mi, top.index);
126 + if (err < 0) {
127 + return err;
128 + }
129 + reftable_record_destroy(&top.rec);
130 + }
131 +
132 + reftable_record_copy_from(rec, &entry.rec, hash_size(mi->hash_id));
133 + reftable_record_destroy(&entry.rec);
134 + strbuf_release(&entry_key);
135 + return 0;
136 +}
137 +
138 +static int merged_iter_next(struct merged_iter *mi, struct reftable_record *rec)
139 +{
140 + while (1) {
141 + int err = merged_iter_next_entry(mi, rec);
142 + if (err == 0 && mi->suppress_deletions &&
143 + reftable_record_is_deletion(rec)) {
144 + continue;
145 + }
146 +
147 + return err;
148 + }
149 +}
150 +
151 +static int merged_iter_next_void(void *p, struct reftable_record *rec)
152 +{
153 + struct merged_iter *mi = p;
154 + if (merged_iter_pqueue_is_empty(mi->pq))
155 + return 1;
156 +
157 + return merged_iter_next(mi, rec);
158 +}
159 +
160 +static struct reftable_iterator_vtable merged_iter_vtable = {
161 + .next = &merged_iter_next_void,
162 + .close = &merged_iter_close,
163 +};
164 +
165 +static void iterator_from_merged_iter(struct reftable_iterator *it,
166 + struct merged_iter *mi)
167 +{
168 + assert(!it->ops);
169 + it->iter_arg = mi;
170 + it->ops = &merged_iter_vtable;
171 +}
172 +
173 +int reftable_new_merged_table(struct reftable_merged_table **dest,
174 + struct reftable_table *stack, int n,
175 + uint32_t hash_id)
176 +{
177 + struct reftable_merged_table *m = NULL;
178 + uint64_t last_max = 0;
179 + uint64_t first_min = 0;
180 + int i = 0;
181 + for (i = 0; i < n; i++) {
182 + uint64_t min = reftable_table_min_update_index(&stack[i]);
183 + uint64_t max = reftable_table_max_update_index(&stack[i]);
184 +
185 + if (reftable_table_hash_id(&stack[i]) != hash_id) {
186 + return REFTABLE_FORMAT_ERROR;
187 + }
188 + if (i == 0 || min < first_min) {
189 + first_min = min;
190 + }
191 + if (i == 0 || max > last_max) {
192 + last_max = max;
193 + }
194 + }
195 +
196 + m = reftable_calloc(sizeof(struct reftable_merged_table));
197 + m->stack = stack;
198 + m->stack_len = n;
199 + m->min = first_min;
200 + m->max = last_max;
201 + m->hash_id = hash_id;
202 + *dest = m;
203 + return 0;
204 +}
205 +
206 +/* clears the list of subtable, without affecting the readers themselves. */
207 +void merged_table_release(struct reftable_merged_table *mt)
208 +{
209 + FREE_AND_NULL(mt->stack);
210 + mt->stack_len = 0;
211 +}
212 +
213 +void reftable_merged_table_free(struct reftable_merged_table *mt)
214 +{
215 + if (!mt) {
216 + return;
217 + }
218 + merged_table_release(mt);
219 + reftable_free(mt);
220 +}
221 +
222 +uint64_t
223 +reftable_merged_table_max_update_index(struct reftable_merged_table *mt)
224 +{
225 + return mt->max;
226 +}
227 +
228 +uint64_t
229 +reftable_merged_table_min_update_index(struct reftable_merged_table *mt)
230 +{
231 + return mt->min;
232 +}
233 +
234 +static int reftable_table_seek_record(struct reftable_table *tab,
235 + struct reftable_iterator *it,
236 + struct reftable_record *rec)
237 +{
238 + return tab->ops->seek_record(tab->table_arg, it, rec);
239 +}
240 +
241 +static int merged_table_seek_record(struct reftable_merged_table *mt,
242 + struct reftable_iterator *it,
243 + struct reftable_record *rec)
244 +{
245 + struct reftable_iterator *iters = reftable_calloc(
246 + sizeof(struct reftable_iterator) * mt->stack_len);
247 + struct merged_iter merged = {
248 + .stack = iters,
249 + .typ = reftable_record_type(rec),
250 + .hash_id = mt->hash_id,
251 + .suppress_deletions = mt->suppress_deletions,
252 + };
253 + int n = 0;
254 + int err = 0;
255 + int i = 0;
256 + for (i = 0; i < mt->stack_len && err == 0; i++) {
257 + int e = reftable_table_seek_record(&mt->stack[i], &iters[n],
258 + rec);
259 + if (e < 0) {
260 + err = e;
261 + }
262 + if (e == 0) {
263 + n++;
264 + }
265 + }
266 + if (err < 0) {
267 + int i = 0;
268 + for (i = 0; i < n; i++) {
269 + reftable_iterator_destroy(&iters[i]);
270 + }
271 + reftable_free(iters);
272 + return err;
273 + }
274 +
275 + merged.stack_len = n;
276 + err = merged_iter_init(&merged);
277 + if (err < 0) {
278 + merged_iter_close(&merged);
279 + return err;
280 + } else {
281 + struct merged_iter *p =
282 + reftable_malloc(sizeof(struct merged_iter));
283 + *p = merged;
284 + iterator_from_merged_iter(it, p);
285 + }
286 + return 0;
287 +}
288 +
289 +int reftable_merged_table_seek_ref(struct reftable_merged_table *mt,
290 + struct reftable_iterator *it,
291 + const char *name)
292 +{
293 + struct reftable_ref_record ref = {
294 + .refname = (char *)name,
295 + };
296 + struct reftable_record rec = { NULL };
297 + reftable_record_from_ref(&rec, &ref);
298 + return merged_table_seek_record(mt, it, &rec);
299 +}
300 +
301 +int reftable_merged_table_seek_log_at(struct reftable_merged_table *mt,
302 + struct reftable_iterator *it,
303 + const char *name, uint64_t update_index)
304 +{
305 + struct reftable_log_record log = {
306 + .refname = (char *)name,
307 + .update_index = update_index,
308 + };
309 + struct reftable_record rec = { NULL };
310 + reftable_record_from_log(&rec, &log);
311 + return merged_table_seek_record(mt, it, &rec);
312 +}
313 +
314 +int reftable_merged_table_seek_log(struct reftable_merged_table *mt,
315 + struct reftable_iterator *it,
316 + const char *name)
317 +{
318 + uint64_t max = ~((uint64_t)0);
319 + return reftable_merged_table_seek_log_at(mt, it, name, max);
320 +}
321 +
322 +uint32_t reftable_merged_table_hash_id(struct reftable_merged_table *mt)
323 +{
324 + return mt->hash_id;
325 +}
326 +
327 +static int reftable_merged_table_seek_void(void *tab,
328 + struct reftable_iterator *it,
329 + struct reftable_record *rec)
330 +{
331 + return merged_table_seek_record(tab, it, rec);
332 +}
333 +
334 +static uint32_t reftable_merged_table_hash_id_void(void *tab)
335 +{
336 + return reftable_merged_table_hash_id(tab);
337 +}
338 +
339 +static uint64_t reftable_merged_table_min_update_index_void(void *tab)
340 +{
341 + return reftable_merged_table_min_update_index(tab);
342 +}
343 +
344 +static uint64_t reftable_merged_table_max_update_index_void(void *tab)
345 +{
346 + return reftable_merged_table_max_update_index(tab);
347 +}
348 +
349 +static struct reftable_table_vtable merged_table_vtable = {
350 + .seek_record = reftable_merged_table_seek_void,
351 + .hash_id = reftable_merged_table_hash_id_void,
352 + .min_update_index = reftable_merged_table_min_update_index_void,
353 + .max_update_index = reftable_merged_table_max_update_index_void,
354 +};
355 +
356 +void reftable_table_from_merged_table(struct reftable_table *tab,
357 + struct reftable_merged_table *merged)
358 +{
359 + assert(!tab->ops);
360 + tab->ops = &merged_table_vtable;
361 + tab->table_arg = merged;
362 +}
reftable/merged.h new
+38
@@ -0,0 +1,38 @@
1 +/*
2 +Copyright 2020 Google LLC
3 +
4 +Use of this source code is governed by a BSD-style
5 +license that can be found in the LICENSE file or at
6 +https://developers.google.com/open-source/licenses/bsd
7 +*/
8 +
9 +#ifndef MERGED_H
10 +#define MERGED_H
11 +
12 +#include "pq.h"
13 +
14 +struct reftable_merged_table {
15 + struct reftable_table *stack;
16 + size_t stack_len;
17 + uint32_t hash_id;
18 +
19 + /* If unset, produce deletions. This is useful for compaction. For the
20 + * full stack, deletions should be produced. */
21 + int suppress_deletions;
22 +
23 + uint64_t min;
24 + uint64_t max;
25 +};
26 +
27 +struct merged_iter {
28 + struct reftable_iterator *stack;
29 + uint32_t hash_id;
30 + size_t stack_len;
31 + uint8_t typ;
32 + int suppress_deletions;
33 + struct merged_iter_pqueue pq;
34 +};
35 +
36 +void merged_table_release(struct reftable_merged_table *mt);
37 +
38 +#endif
reftable/merged_test.c new
+468
@@ -0,0 +1,468 @@
1 +/*
2 +Copyright 2020 Google LLC
3 +
4 +Use of this source code is governed by a BSD-style
5 +license that can be found in the LICENSE file or at
6 +https://developers.google.com/open-source/licenses/bsd
7 +*/
8 +
9 +#include "merged.h"
10 +
11 +#include "system.h"
12 +
13 +#include "basics.h"
14 +#include "blocksource.h"
15 +#include "constants.h"
16 +#include "reader.h"
17 +#include "record.h"
18 +#include "test_framework.h"
19 +#include "reftable-merged.h"
20 +#include "reftable-tests.h"
21 +#include "reftable-generic.h"
22 +#include "reftable-writer.h"
23 +
24 +static void write_test_table(struct strbuf *buf,
25 + struct reftable_ref_record refs[], int n)
26 +{
27 + int min = 0xffffffff;
28 + int max = 0;
29 + int i = 0;
30 + int err;
31 +
32 + struct reftable_write_options opts = {
33 + .block_size = 256,
34 + };
35 + struct reftable_writer *w = NULL;
36 + for (i = 0; i < n; i++) {
37 + uint64_t ui = refs[i].update_index;
38 + if (ui > max) {
39 + max = ui;
40 + }
41 + if (ui < min) {
42 + min = ui;
43 + }
44 + }
45 +
46 + w = reftable_new_writer(&strbuf_add_void, buf, &opts);
47 + reftable_writer_set_limits(w, min, max);
48 +
49 + for (i = 0; i < n; i++) {
50 + uint64_t before = refs[i].update_index;
51 + int n = reftable_writer_add_ref(w, &refs[i]);
52 + EXPECT(n == 0);
53 + EXPECT(before == refs[i].update_index);
54 + }
55 +
56 + err = reftable_writer_close(w);
57 + EXPECT_ERR(err);
58 +
59 + reftable_writer_free(w);
60 +}
61 +
62 +static void write_test_log_table(struct strbuf *buf,
63 + struct reftable_log_record logs[], int n,
64 + uint64_t update_index)
65 +{
66 + int i = 0;
67 + int err;
68 +
69 + struct reftable_write_options opts = {
70 + .block_size = 256,
71 + .exact_log_message = 1,
72 + };
73 + struct reftable_writer *w = NULL;
74 + w = reftable_new_writer(&strbuf_add_void, buf, &opts);
75 + reftable_writer_set_limits(w, update_index, update_index);
76 +
77 + for (i = 0; i < n; i++) {
78 + int err = reftable_writer_add_log(w, &logs[i]);
79 + EXPECT_ERR(err);
80 + }
81 +
82 + err = reftable_writer_close(w);
83 + EXPECT_ERR(err);
84 +
85 + reftable_writer_free(w);
86 +}
87 +
88 +static struct reftable_merged_table *
89 +merged_table_from_records(struct reftable_ref_record **refs,
90 + struct reftable_block_source **source,
91 + struct reftable_reader ***readers, int *sizes,
92 + struct strbuf *buf, int n)
93 +{
94 + int i = 0;
95 + struct reftable_merged_table *mt = NULL;
96 + int err;
97 + struct reftable_table *tabs =
98 + reftable_calloc(n * sizeof(struct reftable_table));
99 + *readers = reftable_calloc(n * sizeof(struct reftable_reader *));
100 + *source = reftable_calloc(n * sizeof(**source));
101 + for (i = 0; i < n; i++) {
102 + write_test_table(&buf[i], refs[i], sizes[i]);
103 + block_source_from_strbuf(&(*source)[i], &buf[i]);
104 +
105 + err = reftable_new_reader(&(*readers)[i], &(*source)[i],
106 + "name");
107 + EXPECT_ERR(err);
108 + reftable_table_from_reader(&tabs[i], (*readers)[i]);
109 + }
110 +
111 + err = reftable_new_merged_table(&mt, tabs, n, GIT_SHA1_FORMAT_ID);
112 + EXPECT_ERR(err);
113 + return mt;
114 +}
115 +
116 +static void readers_destroy(struct reftable_reader **readers, size_t n)
117 +{
118 + int i = 0;
119 + for (; i < n; i++)
120 + reftable_reader_free(readers[i]);
121 + reftable_free(readers);
122 +}
123 +
124 +static void test_merged_between(void)
125 +{
126 + uint8_t hash1[GIT_SHA1_RAWSZ] = { 1, 2, 3, 0 };
127 +
128 + struct reftable_ref_record r1[] = { {
129 + .refname = "b",
130 + .update_index = 1,
131 + .value_type = REFTABLE_REF_VAL1,
132 + .value.val1 = hash1,
133 + } };
134 + struct reftable_ref_record r2[] = { {
135 + .refname = "a",
136 + .update_index = 2,
137 + .value_type = REFTABLE_REF_DELETION,
138 + } };
139 +
140 + struct reftable_ref_record *refs[] = { r1, r2 };
141 + int sizes[] = { 1, 1 };
142 + struct strbuf bufs[2] = { STRBUF_INIT, STRBUF_INIT };
143 + struct reftable_block_source *bs = NULL;
144 + struct reftable_reader **readers = NULL;
145 + struct reftable_merged_table *mt =
146 + merged_table_from_records(refs, &bs, &readers, sizes, bufs, 2);
147 + int i;
148 + struct reftable_ref_record ref = { NULL };
149 + struct reftable_iterator it = { NULL };
150 + int err = reftable_merged_table_seek_ref(mt, &it, "a");
151 + EXPECT_ERR(err);
152 +
153 + err = reftable_iterator_next_ref(&it, &ref);
154 + EXPECT_ERR(err);
155 + EXPECT(ref.update_index == 2);
156 + reftable_ref_record_release(&ref);
157 + reftable_iterator_destroy(&it);
158 + readers_destroy(readers, 2);
159 + reftable_merged_table_free(mt);
160 + for (i = 0; i < ARRAY_SIZE(bufs); i++) {
161 + strbuf_release(&bufs[i]);
162 + }
163 + reftable_free(bs);
164 +}
165 +
166 +static void test_merged(void)
167 +{
168 + uint8_t hash1[GIT_SHA1_RAWSZ] = { 1 };
169 + uint8_t hash2[GIT_SHA1_RAWSZ] = { 2 };
170 + struct reftable_ref_record r1[] = {
171 + {
172 + .refname = "a",
173 + .update_index = 1,
174 + .value_type = REFTABLE_REF_VAL1,
175 + .value.val1 = hash1,
176 + },
177 + {
178 + .refname = "b",
179 + .update_index = 1,
180 + .value_type = REFTABLE_REF_VAL1,
181 + .value.val1 = hash1,
182 + },
183 + {
184 + .refname = "c",
185 + .update_index = 1,
186 + .value_type = REFTABLE_REF_VAL1,
187 + .value.val1 = hash1,
188 + }
189 + };
190 + struct reftable_ref_record r2[] = { {
191 + .refname = "a",
192 + .update_index = 2,
193 + .value_type = REFTABLE_REF_DELETION,
194 + } };
195 + struct reftable_ref_record r3[] = {
196 + {
197 + .refname = "c",
198 + .update_index = 3,
199 + .value_type = REFTABLE_REF_VAL1,
200 + .value.val1 = hash2,
201 + },
202 + {
203 + .refname = "d",
204 + .update_index = 3,
205 + .value_type = REFTABLE_REF_VAL1,
206 + .value.val1 = hash1,
207 + },
208 + };
209 +
210 + struct reftable_ref_record want[] = {
211 + r2[0],
212 + r1[1],
213 + r3[0],
214 + r3[1],
215 + };
216 +
217 + struct reftable_ref_record *refs[] = { r1, r2, r3 };
218 + int sizes[3] = { 3, 1, 2 };
219 + struct strbuf bufs[3] = { STRBUF_INIT, STRBUF_INIT, STRBUF_INIT };
220 + struct reftable_block_source *bs = NULL;
221 + struct reftable_reader **readers = NULL;
222 + struct reftable_merged_table *mt =
223 + merged_table_from_records(refs, &bs, &readers, sizes, bufs, 3);
224 +
225 + struct reftable_iterator it = { NULL };
226 + int err = reftable_merged_table_seek_ref(mt, &it, "a");
227 + struct reftable_ref_record *out = NULL;
228 + size_t len = 0;
229 + size_t cap = 0;
230 + int i = 0;
231 +
232 + EXPECT_ERR(err);
233 + EXPECT(reftable_merged_table_hash_id(mt) == GIT_SHA1_FORMAT_ID);
234 + EXPECT(reftable_merged_table_min_update_index(mt) == 1);
235 +
236 + while (len < 100) { /* cap loops/recursion. */
237 + struct reftable_ref_record ref = { NULL };
238 + int err = reftable_iterator_next_ref(&it, &ref);
239 + if (err > 0) {
240 + break;
241 + }
242 + if (len == cap) {
243 + cap = 2 * cap + 1;
244 + out = reftable_realloc(
245 + out, sizeof(struct reftable_ref_record) * cap);
246 + }
247 + out[len++] = ref;
248 + }
249 + reftable_iterator_destroy(&it);
250 +
251 + EXPECT(ARRAY_SIZE(want) == len);
252 + for (i = 0; i < len; i++) {
253 + EXPECT(reftable_ref_record_equal(&want[i], &out[i],
254 + GIT_SHA1_RAWSZ));
255 + }
256 + for (i = 0; i < len; i++) {
257 + reftable_ref_record_release(&out[i]);
258 + }
259 + reftable_free(out);
260 +
261 + for (i = 0; i < 3; i++) {
262 + strbuf_release(&bufs[i]);
263 + }
264 + readers_destroy(readers, 3);
265 + reftable_merged_table_free(mt);
266 + reftable_free(bs);
267 +}
268 +
269 +static struct reftable_merged_table *
270 +merged_table_from_log_records(struct reftable_log_record **logs,
271 + struct reftable_block_source **source,
272 + struct reftable_reader ***readers, int *sizes,
273 + struct strbuf *buf, int n)
274 +{
275 + int i = 0;
276 + struct reftable_merged_table *mt = NULL;
277 + int err;
278 + struct reftable_table *tabs =
279 + reftable_calloc(n * sizeof(struct reftable_table));
280 + *readers = reftable_calloc(n * sizeof(struct reftable_reader *));
281 + *source = reftable_calloc(n * sizeof(**source));
282 + for (i = 0; i < n; i++) {
283 + write_test_log_table(&buf[i], logs[i], sizes[i], i + 1);
284 + block_source_from_strbuf(&(*source)[i], &buf[i]);
285 +
286 + err = reftable_new_reader(&(*readers)[i], &(*source)[i],
287 + "name");
288 + EXPECT_ERR(err);
289 + reftable_table_from_reader(&tabs[i], (*readers)[i]);
290 + }
291 +
292 + err = reftable_new_merged_table(&mt, tabs, n, GIT_SHA1_FORMAT_ID);
293 + EXPECT_ERR(err);
294 + return mt;
295 +}
296 +
297 +static void test_merged_logs(void)
298 +{
299 + uint8_t hash1[GIT_SHA1_RAWSZ] = { 1 };
300 + uint8_t hash2[GIT_SHA1_RAWSZ] = { 2 };
301 + uint8_t hash3[GIT_SHA1_RAWSZ] = { 3 };
302 + struct reftable_log_record r1[] = {
303 + {
304 + .refname = "a",
305 + .update_index = 2,
306 + .value_type = REFTABLE_LOG_UPDATE,
307 + .value.update = {
308 + .old_hash = hash2,
309 + /* deletion */
310 + .name = "jane doe",
311 + .email = "jane@invalid",
312 + .message = "message2",
313 + }
314 + },
315 + {
316 + .refname = "a",
317 + .update_index = 1,
318 + .value_type = REFTABLE_LOG_UPDATE,
319 + .value.update = {
320 + .old_hash = hash1,
321 + .new_hash = hash2,
322 + .name = "jane doe",
323 + .email = "jane@invalid",
324 + .message = "message1",
325 + }
326 + },
327 + };
328 + struct reftable_log_record r2[] = {
329 + {
330 + .refname = "a",
331 + .update_index = 3,
332 + .value_type = REFTABLE_LOG_UPDATE,
333 + .value.update = {
334 + .new_hash = hash3,
335 + .name = "jane doe",
336 + .email = "jane@invalid",
337 + .message = "message3",
338 + }
339 + },
340 + };
341 + struct reftable_log_record r3[] = {
342 + {
343 + .refname = "a",
344 + .update_index = 2,
345 + .value_type = REFTABLE_LOG_DELETION,
346 + },
347 + };
348 + struct reftable_log_record want[] = {
349 + r2[0],
350 + r3[0],
351 + r1[1],
352 + };
353 +
354 + struct reftable_log_record *logs[] = { r1, r2, r3 };
355 + int sizes[3] = { 2, 1, 1 };
356 + struct strbuf bufs[3] = { STRBUF_INIT, STRBUF_INIT, STRBUF_INIT };
357 + struct reftable_block_source *bs = NULL;
358 + struct reftable_reader **readers = NULL;
359 + struct reftable_merged_table *mt = merged_table_from_log_records(
360 + logs, &bs, &readers, sizes, bufs, 3);
361 +
362 + struct reftable_iterator it = { NULL };
363 + int err = reftable_merged_table_seek_log(mt, &it, "a");
364 + struct reftable_log_record *out = NULL;
365 + size_t len = 0;
366 + size_t cap = 0;
367 + int i = 0;
368 +
369 + EXPECT_ERR(err);
370 + EXPECT(reftable_merged_table_hash_id(mt) == GIT_SHA1_FORMAT_ID);
371 + EXPECT(reftable_merged_table_min_update_index(mt) == 1);
372 +
373 + while (len < 100) { /* cap loops/recursion. */
374 + struct reftable_log_record log = { NULL };
375 + int err = reftable_iterator_next_log(&it, &log);
376 + if (err > 0) {
377 + break;
378 + }
379 + if (len == cap) {
380 + cap = 2 * cap + 1;
381 + out = reftable_realloc(
382 + out, sizeof(struct reftable_log_record) * cap);
383 + }
384 + out[len++] = log;
385 + }
386 + reftable_iterator_destroy(&it);
387 +
388 + EXPECT(ARRAY_SIZE(want) == len);
389 + for (i = 0; i < len; i++) {
390 + EXPECT(reftable_log_record_equal(&want[i], &out[i],
391 + GIT_SHA1_RAWSZ));
392 + }
393 +
394 + err = reftable_merged_table_seek_log_at(mt, &it, "a", 2);
395 + EXPECT_ERR(err);
396 + reftable_log_record_release(&out[0]);
397 + err = reftable_iterator_next_log(&it, &out[0]);
398 + EXPECT_ERR(err);
399 + EXPECT(reftable_log_record_equal(&out[0], &r3[0], GIT_SHA1_RAWSZ));
400 + reftable_iterator_destroy(&it);
401 +
402 + for (i = 0; i < len; i++) {
403 + reftable_log_record_release(&out[i]);
404 + }
405 + reftable_free(out);
406 +
407 + for (i = 0; i < 3; i++) {
408 + strbuf_release(&bufs[i]);
409 + }
410 + readers_destroy(readers, 3);
411 + reftable_merged_table_free(mt);
412 + reftable_free(bs);
413 +}
414 +
415 +static void test_default_write_opts(void)
416 +{
417 + struct reftable_write_options opts = { 0 };
418 + struct strbuf buf = STRBUF_INIT;
419 + struct reftable_writer *w =
420 + reftable_new_writer(&strbuf_add_void, &buf, &opts);
421 +
422 + struct reftable_ref_record rec = {
423 + .refname = "master",
424 + .update_index = 1,
425 + };
426 + int err;
427 + struct reftable_block_source source = { NULL };
428 + struct reftable_table *tab = reftable_calloc(sizeof(*tab) * 1);
429 + uint32_t hash_id;
430 + struct reftable_reader *rd = NULL;
431 + struct reftable_merged_table *merged = NULL;
432 +
433 + reftable_writer_set_limits(w, 1, 1);
434 +
435 + err = reftable_writer_add_ref(w, &rec);
436 + EXPECT_ERR(err);
437 +
438 + err = reftable_writer_close(w);
439 + EXPECT_ERR(err);
440 + reftable_writer_free(w);
441 +
442 + block_source_from_strbuf(&source, &buf);
443 +
444 + err = reftable_new_reader(&rd, &source, "filename");
445 + EXPECT_ERR(err);
446 +
447 + hash_id = reftable_reader_hash_id(rd);
448 + EXPECT(hash_id == GIT_SHA1_FORMAT_ID);
449 +
450 + reftable_table_from_reader(&tab[0], rd);
451 + err = reftable_new_merged_table(&merged, tab, 1, GIT_SHA1_FORMAT_ID);
452 + EXPECT_ERR(err);
453 +
454 + reftable_reader_free(rd);
455 + reftable_merged_table_free(merged);
456 + strbuf_release(&buf);
457 +}
458 +
459 +/* XXX test refs_for(oid) */
460 +
461 +int merged_test_main(int argc, const char *argv[])
462 +{
463 + RUN_TEST(test_merged_logs);
464 + RUN_TEST(test_merged_between);
465 + RUN_TEST(test_merged);
466 + RUN_TEST(test_default_write_opts);
467 + return 0;
468 +}
reftable/reftable-merged.h new
+72
@@ -0,0 +1,72 @@
1 +/*
2 +Copyright 2020 Google LLC
3 +
4 +Use of this source code is governed by a BSD-style
5 +license that can be found in the LICENSE file or at
6 +https://developers.google.com/open-source/licenses/bsd
7 +*/
8 +
9 +#ifndef REFTABLE_MERGED_H
10 +#define REFTABLE_MERGED_H
11 +
12 +#include "reftable-iterator.h"
13 +
14 +/*
15 + * Merged tables
16 + *
17 + * A ref database kept in a sequence of table files. The merged_table presents a
18 + * unified view to reading (seeking, iterating) a sequence of immutable tables.
19 + *
20 + * The merged tables are on purpose kept disconnected from their actual storage
21 + * (eg. files on disk), because it is useful to merge tables aren't files. For
22 + * example, the per-workspace and global ref namespace can be implemented as a
23 + * merged table of two stacks of file-backed reftables.
24 + */
25 +
26 +/* A merged table is implements seeking/iterating over a stack of tables. */
27 +struct reftable_merged_table;
28 +
29 +/* A generic reftable; see below. */
30 +struct reftable_table;
31 +
32 +/* reftable_new_merged_table creates a new merged table. It takes ownership of
33 + the stack array.
34 +*/
35 +int reftable_new_merged_table(struct reftable_merged_table **dest,
36 + struct reftable_table *stack, int n,
37 + uint32_t hash_id);
38 +
39 +/* returns an iterator positioned just before 'name' */
40 +int reftable_merged_table_seek_ref(struct reftable_merged_table *mt,
41 + struct reftable_iterator *it,
42 + const char *name);
43 +
44 +/* returns an iterator for log entry, at given update_index */
45 +int reftable_merged_table_seek_log_at(struct reftable_merged_table *mt,
46 + struct reftable_iterator *it,
47 + const char *name, uint64_t update_index);
48 +
49 +/* like reftable_merged_table_seek_log_at but look for the newest entry. */
50 +int reftable_merged_table_seek_log(struct reftable_merged_table *mt,
51 + struct reftable_iterator *it,
52 + const char *name);
53 +
54 +/* returns the max update_index covered by this merged table. */
55 +uint64_t
56 +reftable_merged_table_max_update_index(struct reftable_merged_table *mt);
57 +
58 +/* returns the min update_index covered by this merged table. */
59 +uint64_t
60 +reftable_merged_table_min_update_index(struct reftable_merged_table *mt);
61 +
62 +/* releases memory for the merged_table */
63 +void reftable_merged_table_free(struct reftable_merged_table *m);
64 +
65 +/* return the hash ID of the merged table. */
66 +uint32_t reftable_merged_table_hash_id(struct reftable_merged_table *m);
67 +
68 +/* create a generic table from reftable_merged_table */
69 +void reftable_table_from_merged_table(struct reftable_table *tab,
70 + struct reftable_merged_table *table);
71 +
72 +#endif
t/helper/test-reftable.c
+1
@@ -5,6 +5,7 @@ int cmd__reftable(int argc, const char **argv)
5 {
6 basics_test_main(argc, argv);
7 block_test_main(argc, argv);
8 + merged_test_main(argc, argv);
9 pq_test_main(argc, argv);
10 record_test_main(argc, argv);
11 readwrite_test_main(argc, argv);