@leroysheep / rag-system
Code Commits Pulls CI/CD 81,295
bisect
bisect jch maint master next seen test todo
h 170 lines 5.64 KB
Raw
1 /*
2 * Copyright 2020 Google LLC
3 *
4 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://developers.google.com/open-source/licenses/bsd
7 */
8
9 #ifndef REFTABLE_STACK_H
10 #define REFTABLE_STACK_H
11
12 #include "reftable-system.h"
13 #include "reftable-writer.h"
14
15 /*
16 * The stack presents an interface to a mutable sequence of reftables.
17
18 * A stack can be mutated by pushing a table to the top of the stack.
19
20 * The reftable_stack automatically compacts files on disk to ensure good
21 * amortized performance.
22 *
23 * For windows and other platforms that cannot have open files as rename
24 * destinations, concurrent access from multiple processes needs the rand()
25 * random seed to be randomized.
26 */
27 struct reftable_stack;
28
29 /* open a new reftable stack. The tables along with the table list will be
30 * stored in 'dir'. Typically, this should be .git/reftables.
31 */
32 int reftable_new_stack(struct reftable_stack **dest, const char *dir,
33 const struct reftable_write_options *opts);
34
35 /* returns the update_index at which a next table should be written. */
36 uint64_t reftable_stack_next_update_index(struct reftable_stack *st);
37
38 /* holds a transaction to add tables at the top of a stack. */
39 struct reftable_addition;
40
41 enum {
42 /*
43 * Reload the stack when the stack is out-of-date after locking it.
44 */
45 REFTABLE_STACK_NEW_ADDITION_RELOAD = (1 << 0),
46 };
47
48 /*
49 * returns a new transaction to add reftables to the given stack. As a side
50 * effect, the ref database is locked. Accepts REFTABLE_STACK_NEW_ADDITION_*
51 * flags.
52 */
53 int reftable_stack_new_addition(struct reftable_addition **dest,
54 struct reftable_stack *st,
55 unsigned int flags);
56
57 /* Adds a reftable to transaction. */
58 int reftable_addition_add(struct reftable_addition *add,
59 int (*write_table)(struct reftable_writer *wr,
60 void *arg),
61 void *arg);
62
63 /* Commits the transaction, releasing the lock. After calling this,
64 * reftable_addition_destroy should still be called.
65 */
66 int reftable_addition_commit(struct reftable_addition *add);
67
68 /* Release all non-committed data from the transaction, and deallocate the
69 * transaction. Releases the lock if held. */
70 void reftable_addition_destroy(struct reftable_addition *add);
71
72 /*
73 * Add a new table to the stack. The write_table function must call
74 * reftable_writer_set_limits, add refs and return an error value.
75 * The flags are passed through to `reftable_stack_new_addition()`.
76 */
77 int reftable_stack_add(struct reftable_stack *st,
78 int (*write_table)(struct reftable_writer *wr,
79 void *write_arg),
80 void *write_arg, unsigned flags);
81
82 struct reftable_iterator;
83
84 /*
85 * Initialize an iterator for the merged tables contained in the stack that can
86 * be used to iterate through refs. The iterator is valid until the next reload
87 * or write.
88 */
89 int reftable_stack_init_ref_iterator(struct reftable_stack *st,
90 struct reftable_iterator *it);
91
92 /*
93 * Initialize an iterator for the merged tables contained in the stack that can
94 * be used to iterate through logs. The iterator is valid until the next reload
95 * or write.
96 */
97 int reftable_stack_init_log_iterator(struct reftable_stack *st,
98 struct reftable_iterator *it);
99
100 /* returns the merged_table for seeking. This table is valid until the
101 * next write or reload, and should not be closed or deleted.
102 */
103 struct reftable_merged_table *
104 reftable_stack_merged_table(struct reftable_stack *st);
105
106 /* frees all resources associated with the stack. */
107 void reftable_stack_destroy(struct reftable_stack *st);
108
109 /* Reloads the stack if necessary. This is very cheap to run if the stack was up
110 * to date */
111 int reftable_stack_reload(struct reftable_stack *st);
112
113 /* Policy for expiring reflog entries. */
114 struct reftable_log_expiry_config {
115 /* Drop entries older than this timestamp */
116 uint64_t time;
117
118 /* Drop older entries */
119 uint64_t min_update_index;
120 };
121
122 /* compacts all reftables into a giant table. Expire reflog entries if config is
123 * non-NULL */
124 int reftable_stack_compact_all(struct reftable_stack *st,
125 struct reftable_log_expiry_config *config);
126
127 /*
128 * Check if compaction is required.
129 *
130 * When `use_heuristics` is false, check if all tables can be compacted to a
131 * single table. If true, use heuristics to determine if the tables need to be
132 * compacted to maintain geometric progression.
133 */
134 int reftable_stack_compaction_required(struct reftable_stack *st,
135 bool use_heuristics,
136 bool *required);
137
138 /* heuristically compact unbalanced table stack. */
139 int reftable_stack_auto_compact(struct reftable_stack *st);
140
141 /* delete stale .ref tables. */
142 int reftable_stack_clean(struct reftable_stack *st);
143
144 /* convenience function to read a single ref. Returns < 0 for error, 0 for
145 * success, and 1 if ref not found. */
146 int reftable_stack_read_ref(struct reftable_stack *st, const char *refname,
147 struct reftable_ref_record *ref);
148
149 /* convenience function to read a single log. Returns < 0 for error, 0 for
150 * success, and 1 if ref not found. */
151 int reftable_stack_read_log(struct reftable_stack *st, const char *refname,
152 struct reftable_log_record *log);
153
154 /* statistics on past compactions. */
155 struct reftable_compaction_stats {
156 uint64_t bytes; /* total number of bytes written */
157 uint64_t entries_written; /* total number of entries written, including
158 failures. */
159 int attempts; /* how often we tried to compact */
160 int failures; /* failures happen on concurrent updates */
161 };
162
163 /* return statistics for compaction up till now. */
164 struct reftable_compaction_stats *
165 reftable_stack_compaction_stats(struct reftable_stack *st);
166
167 /* Return the hash of the stack. */
168 enum reftable_hash reftable_stack_hash_id(struct reftable_stack *st);
169
170 #endif