@leroysheep / rag-system
Code Commits Pulls CI/CD 82,027
next
bisect jch maint master next seen test todo
h 196 lines 6.41 KB
Raw
1 /*
2 * Copyright 2020 Google LLC
3 *
4 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://developers.google.com/open-source/licenses/bsd
7 */
8
9 #ifndef REFTABLE_STACK_H
10 #define REFTABLE_STACK_H
11
12 #include "reftable-system.h"
13 #include "reftable-writer.h"
14
15 /*
16 * The stack presents an interface to a mutable sequence of reftables.
17
18 * A stack can be mutated by pushing a table to the top of the stack.
19
20 * The reftable_stack automatically compacts files on disk to ensure good
21 * amortized performance.
22 *
23 * For windows and other platforms that cannot have open files as rename
24 * destinations, concurrent access from multiple processes needs the rand()
25 * random seed to be randomized.
26 */
27 struct reftable_stack;
28
29 /* Options related to opening a stack. */
30 struct reftable_stack_options {
31 /*
32 * 4-byte identifier ("sha1", "s256") of the hash. Defaults to SHA1 if
33 * unset.
34 */
35 enum reftable_hash hash_id;
36
37 /*
38 * Callback function to execute whenever the stack is being reloaded.
39 * This can be used e.g. to discard cached information that relies on
40 * the old stack's data. The payload data will be passed as argument to
41 * the callback.
42 */
43 void (*on_reload)(void *payload);
44 void *on_reload_payload;
45
46 int suppress_deletions;
47 };
48
49 /* open a new reftable stack. The tables along with the table list will be
50 * stored in 'dir'. Typically, this should be .git/reftables.
51 */
52 int reftable_new_stack(struct reftable_stack **dest, const char *dir,
53 const struct reftable_stack_options *opts);
54
55 /* returns the update_index at which a next table should be written. */
56 uint64_t reftable_stack_next_update_index(struct reftable_stack *st);
57
58 /* holds a transaction to add tables at the top of a stack. */
59 struct reftable_addition;
60
61 enum {
62 /*
63 * Reload the stack when the stack is out-of-date after locking it.
64 */
65 REFTABLE_STACK_NEW_ADDITION_RELOAD = (1 << 0),
66 };
67
68 /*
69 * returns a new transaction to add reftables to the given stack. As a side
70 * effect, the ref database is locked. Accepts REFTABLE_STACK_NEW_ADDITION_*
71 * flags.
72 */
73 int reftable_stack_new_addition(struct reftable_addition **dest,
74 struct reftable_stack *st,
75 const struct reftable_write_options *opts,
76 unsigned int flags);
77
78 /* Adds a reftable to transaction. */
79 int reftable_addition_add(struct reftable_addition *add,
80 int (*write_table)(struct reftable_writer *wr,
81 void *arg),
82 void *arg);
83
84 /* Commits the transaction, releasing the lock. After calling this,
85 * reftable_addition_destroy should still be called.
86 */
87 int reftable_addition_commit(struct reftable_addition *add);
88
89 /* Release all non-committed data from the transaction, and deallocate the
90 * transaction. Releases the lock if held. */
91 void reftable_addition_destroy(struct reftable_addition *add);
92
93 /*
94 * Add a new table to the stack. The write_table function must call
95 * reftable_writer_set_limits, add refs and return an error value.
96 * The flags are passed through to `reftable_stack_new_addition()`.
97 */
98 int reftable_stack_add(struct reftable_stack *st,
99 int (*write_table)(struct reftable_writer *wr,
100 void *write_arg),
101 void *write_arg,
102 const struct reftable_write_options *opts,
103 unsigned flags);
104
105 struct reftable_iterator;
106
107 /*
108 * Initialize an iterator for the merged tables contained in the stack that can
109 * be used to iterate through refs. The iterator is valid until the next reload
110 * or write.
111 */
112 int reftable_stack_init_ref_iterator(struct reftable_stack *st,
113 struct reftable_iterator *it);
114
115 /*
116 * Initialize an iterator for the merged tables contained in the stack that can
117 * be used to iterate through logs. The iterator is valid until the next reload
118 * or write.
119 */
120 int reftable_stack_init_log_iterator(struct reftable_stack *st,
121 struct reftable_iterator *it);
122
123 /* returns the merged_table for seeking. This table is valid until the
124 * next write or reload, and should not be closed or deleted.
125 */
126 struct reftable_merged_table *
127 reftable_stack_merged_table(struct reftable_stack *st);
128
129 /* frees all resources associated with the stack. */
130 void reftable_stack_destroy(struct reftable_stack *st);
131
132 /* Reloads the stack if necessary. This is very cheap to run if the stack was up
133 * to date */
134 int reftable_stack_reload(struct reftable_stack *st);
135
136 /* Policy for expiring reflog entries. */
137 struct reftable_log_expiry_config {
138 /* Drop entries older than this timestamp */
139 uint64_t time;
140
141 /* Drop older entries */
142 uint64_t min_update_index;
143 };
144
145 /* compacts all reftables into a giant table. Expire reflog entries if config is
146 * non-NULL */
147 int reftable_stack_compact_all(struct reftable_stack *st,
148 const struct reftable_write_options *opts,
149 struct reftable_log_expiry_config *config);
150
151 /*
152 * Check if compaction is required.
153 *
154 * When `use_heuristics` is false, check if all tables can be compacted to a
155 * single table. If true, use heuristics to determine if the tables need to be
156 * compacted to maintain geometric progression.
157 */
158 int reftable_stack_compaction_required(struct reftable_stack *st,
159 const struct reftable_write_options *opts,
160 bool use_heuristics,
161 bool *required);
162
163 /* heuristically compact unbalanced table stack. */
164 int reftable_stack_auto_compact(struct reftable_stack *st,
165 const struct reftable_write_options *opts);
166
167 /* delete stale .ref tables. */
168 int reftable_stack_clean(struct reftable_stack *st);
169
170 /* convenience function to read a single ref. Returns < 0 for error, 0 for
171 * success, and 1 if ref not found. */
172 int reftable_stack_read_ref(struct reftable_stack *st, const char *refname,
173 struct reftable_ref_record *ref);
174
175 /* convenience function to read a single log. Returns < 0 for error, 0 for
176 * success, and 1 if ref not found. */
177 int reftable_stack_read_log(struct reftable_stack *st, const char *refname,
178 struct reftable_log_record *log);
179
180 /* statistics on past compactions. */
181 struct reftable_compaction_stats {
182 uint64_t bytes; /* total number of bytes written */
183 uint64_t entries_written; /* total number of entries written, including
184 failures. */
185 int attempts; /* how often we tried to compact */
186 int failures; /* failures happen on concurrent updates */
187 };
188
189 /* return statistics for compaction up till now. */
190 struct reftable_compaction_stats *
191 reftable_stack_compaction_stats(struct reftable_stack *st);
192
193 /* Return the hash of the stack. */
194 enum reftable_hash reftable_stack_hash_id(struct reftable_stack *st);
195
196 #endif