@leroysheep / rag-system
Code Commits Pulls CI/CD 81,841
master
bisect jch maint master next seen test todo
h 168 lines 5.27 KB
Raw
1 /*
2 * Copyright 2020 Google LLC
3 *
4 * Use of this source code is governed by a BSD-style
5 * license that can be found in the LICENSE file or at
6 * https://developers.google.com/open-source/licenses/bsd
7 */
8
9 #ifndef REFTABLE_WRITER_H
10 #define REFTABLE_WRITER_H
11
12 #include "reftable-system.h"
13 #include "reftable-record.h"
14
15 /* Writing single reftables */
16
17 /* reftable_write_options sets options for writing a single reftable. */
18 struct reftable_write_options {
19 /* boolean: do not pad out blocks to block size. */
20 unsigned unpadded : 1;
21
22 /* the blocksize. Should be less than 2^24. */
23 uint32_t block_size;
24
25 /* boolean: do not generate a SHA1 => ref index. */
26 unsigned skip_index_objects : 1;
27
28 /* how often to write complete keys in each block. */
29 uint16_t restart_interval;
30
31 /* Default mode for creating files. If unset, use 0666 (+umask) */
32 unsigned int default_permissions;
33
34 /* boolean: copy log messages exactly. If unset, check that the message
35 * is a single line, and add '\n' if missing.
36 */
37 unsigned exact_log_message : 1;
38
39 /* boolean: Prevent auto-compaction of tables. */
40 unsigned disable_auto_compact : 1;
41
42 /*
43 * Geometric sequence factor used by auto-compaction to decide which
44 * tables to compact. Defaults to 2 if unset.
45 */
46 uint8_t auto_compaction_factor;
47
48 /*
49 * The number of milliseconds to wait when trying to lock "tables.list".
50 * Note that this does not apply to locking individual tables, as these
51 * should only ever be locked when already holding the "tables.list"
52 * lock.
53 *
54 * Passing 0 will fail immediately when the file is locked, passing a
55 * negative value will cause us to block indefinitely.
56 */
57 long lock_timeout_ms;
58 };
59
60 /* reftable_block_stats holds statistics for a single block type */
61 struct reftable_block_stats {
62 /* total number of entries written */
63 int entries;
64 /* total number of key restarts */
65 uint32_t restarts;
66 /* total number of blocks */
67 int blocks;
68 /* total number of index blocks */
69 int index_blocks;
70 /* depth of the index */
71 int max_index_level;
72
73 /* offset of the first block for this type */
74 uint64_t offset;
75 /* offset of the top level index block for this type, or 0 if not
76 * present */
77 uint64_t index_offset;
78 };
79
80 /* stats holds overall statistics for a single reftable */
81 struct reftable_stats {
82 /* total number of blocks written. */
83 int blocks;
84 /* stats for ref data */
85 struct reftable_block_stats ref_stats;
86 /* stats for the SHA1 to ref map. */
87 struct reftable_block_stats obj_stats;
88 /* stats for index blocks */
89 struct reftable_block_stats idx_stats;
90 /* stats for log blocks */
91 struct reftable_block_stats log_stats;
92
93 /* disambiguation length of shortened object IDs. */
94 int object_id_len;
95 };
96
97 struct reftable_writer;
98
99 /* Create a new writer. */
100 int reftable_writer_new(struct reftable_writer **out,
101 ssize_t (*writer_func)(void *, const void *, size_t),
102 int (*flush_func)(void *),
103 void *writer_arg, enum reftable_hash hash_id,
104 const struct reftable_write_options *opts);
105
106 /*
107 * Set the range of update indices for the records we will add. When writing a
108 * table into a stack, the min should be at least
109 * reftable_stack_next_update_index(), or REFTABLE_API_ERROR is returned.
110 *
111 * For transactional updates to a stack, typically min==max, and the
112 * update_index can be obtained by inspeciting the stack. When converting an
113 * existing ref database into a single reftable, this would be a range of
114 * update-index timestamps.
115 *
116 * The function should be called before adding any records to the writer. If not
117 * it will fail with REFTABLE_API_ERROR.
118 */
119 int reftable_writer_set_limits(struct reftable_writer *w, uint64_t min,
120 uint64_t max);
121
122 /*
123 Add a reftable_ref_record. The record should have names that come after
124 already added records.
125
126 The update_index must be within the limits set by
127 reftable_writer_set_limits(), or REFTABLE_API_ERROR is returned. It is an
128 REFTABLE_API_ERROR error to write a ref record after a log record.
129 */
130 int reftable_writer_add_ref(struct reftable_writer *w,
131 struct reftable_ref_record *ref);
132
133 /*
134 Convenience function to add multiple reftable_ref_records; the function sorts
135 the records before adding them, reordering the records array passed in.
136 */
137 int reftable_writer_add_refs(struct reftable_writer *w,
138 struct reftable_ref_record *refs, size_t n);
139
140 /*
141 adds reftable_log_records. Log records are keyed by (refname, decreasing
142 update_index). The key for the record added must come after the already added
143 log records.
144 */
145 int reftable_writer_add_log(struct reftable_writer *w,
146 struct reftable_log_record *log);
147
148 /*
149 Convenience function to add multiple reftable_log_records; the function sorts
150 the records before adding them, reordering records array passed in.
151 */
152 int reftable_writer_add_logs(struct reftable_writer *w,
153 struct reftable_log_record *logs, size_t n);
154
155 /* reftable_writer_close finalizes the reftable. The writer is retained so
156 * statistics can be inspected. */
157 int reftable_writer_close(struct reftable_writer *w);
158
159 /* writer_stats returns the statistics on the reftable being written.
160
161 This struct becomes invalid when the writer is freed.
162 */
163 const struct reftable_stats *reftable_writer_stats(struct reftable_writer *w);
164
165 /* reftable_writer_free deallocates memory for the writer */
166 void reftable_writer_free(struct reftable_writer *w);
167
168 #endif