refs/reftable: allow configuring block size
Add a new option `reftable.blockSize` that allows the user to control the block size used by the reftable library. Signed-off-by: Patrick Steinhardt <ps@pks.im> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Patrick Steinhardt committed
May 13, 2024 at 10:18 UTC
831b366c243ecaa450289d76926bb687c746d495
4 files changed
+118
-1
Documentation/config.txt
+2
@@ -498,6 +498,8 @@ include::config/rebase.txt[]
498
499
include::config/receive.txt[]
500
501
+include::config/reftable.txt[]
502
+
503
include::config/remote.txt[]
504
505
include::config/remotes.txt[]
Documentation/config/reftable.txt
new
+14
@@ -0,0 +1,14 @@
1
+reftable.blockSize::
2
+ The size in bytes used by the reftable backend when writing blocks.
3
+ The block size is determined by the writer, and does not have to be a
4
+ power of 2. The block size must be larger than the longest reference
5
+ name or log entry used in the repository, as references cannot span
6
+ blocks.
7
++
8
+Powers of two that are friendly to the virtual memory system or
9
+filesystem (such as 4kB or 8kB) are recommended. Larger sizes (64kB) can
10
+yield better compression, with a possible increased cost incurred by
11
+readers during access.
12
++
13
+The largest block size is `16777215` bytes (15.99 MiB). The default value is
14
+`4096` bytes (4kB). A value of `0` will use the default value.
refs/reftable-backend.c
+30
-1
@@ -1,6 +1,7 @@
1
#include "../git-compat-util.h"
2
#include "../abspath.h"
3
#include "../chdir-notify.h"
4
+#include "../config.h"
5
#include "../environment.h"
6
#include "../gettext.h"
7
#include "../hash.h"
@@ -228,6 +229,22 @@ done:
229
return ret;
230
}
231
232
+static int reftable_be_config(const char *var, const char *value,
233
+ const struct config_context *ctx,
234
+ void *_opts)
235
+{
236
+ struct reftable_write_options *opts = _opts;
237
+
238
+ if (!strcmp(var, "reftable.blocksize")) {
239
+ unsigned long block_size = git_config_ulong(var, value, ctx->kvi);
240
+ if (block_size > 16777215)
241
+ die("reftable block size cannot exceed 16MB");
242
+ opts->block_size = block_size;
243
+ }
244
+
245
+ return 0;
246
+}
247
+
248
static struct ref_store *reftable_be_init(struct repository *repo,
249
const char *gitdir,
250
unsigned int store_flags)
@@ -243,12 +260,24 @@ static struct ref_store *reftable_be_init(struct repository *repo,
260
base_ref_store_init(&refs->base, repo, gitdir, &refs_be_reftable);
261
strmap_init(&refs->worktree_stacks);
262
refs->store_flags = store_flags;
246
- refs->write_options.block_size = 4096;
263
+
264
refs->write_options.hash_id = repo->hash_algo->format_id;
265
refs->write_options.default_permissions = calc_shared_perm(0666 & ~mask);
266
refs->write_options.disable_auto_compact =
267
!git_env_bool("GIT_TEST_REFTABLE_AUTOCOMPACTION", 1);
268
269
+ git_config(reftable_be_config, &refs->write_options);
270
+
271
+ /*
272
+ * It is somewhat unfortunate that we have to mirror the default block
273
+ * size of the reftable library here. But given that the write options
274
+ * wouldn't be updated by the library here, and given that we require
275
+ * the proper block size to trim reflog message so that they fit, we
276
+ * must set up a proper value here.
277
+ */
278
+ if (!refs->write_options.block_size)
279
+ refs->write_options.block_size = 4096;
280
+
281
/*
282
* Set up the main reftable stack that is hosted in GIT_COMMON_DIR.
283
* This stack contains both the shared and the main worktree refs.
t/t0613-reftable-write-options.sh
+72
@@ -99,4 +99,76 @@ test_expect_success 'many refs results in multiple blocks' '
99
)
100
'
101
102
+test_expect_success 'tiny block size leads to error' '
103
+ test_when_finished "rm -rf repo" &&
104
+ git init repo &&
105
+ (
106
+ cd repo &&
107
+ test_commit initial &&
108
+ cat >expect <<-EOF &&
109
+ error: unable to compact stack: entry too large
110
+ EOF
111
+ test_must_fail git -c reftable.blockSize=50 pack-refs 2>err &&
112
+ test_cmp expect err
113
+ )
114
+'
115
+
116
+test_expect_success 'small block size leads to multiple ref blocks' '
117
+ test_config_global core.logAllRefUpdates false &&
118
+ test_when_finished "rm -rf repo" &&
119
+ git init repo &&
120
+ (
121
+ cd repo &&
122
+ test_commit A &&
123
+ test_commit B &&
124
+ git -c reftable.blockSize=100 pack-refs &&
125
+
126
+ cat >expect <<-EOF &&
127
+ header:
128
+ block_size: 100
129
+ ref:
130
+ - length: 53
131
+ restarts: 1
132
+ - length: 74
133
+ restarts: 1
134
+ - length: 38
135
+ restarts: 1
136
+ EOF
137
+ test-tool dump-reftable -b .git/reftable/*.ref >actual &&
138
+ test_cmp expect actual
139
+ )
140
+'
141
+
142
+test_expect_success 'small block size fails with large reflog message' '
143
+ test_when_finished "rm -rf repo" &&
144
+ git init repo &&
145
+ (
146
+ cd repo &&
147
+ test_commit A &&
148
+ perl -e "print \"a\" x 500" >logmsg &&
149
+ cat >expect <<-EOF &&
150
+ fatal: update_ref failed for ref ${SQ}refs/heads/logme${SQ}: reftable: transaction failure: entry too large
151
+ EOF
152
+ test_must_fail git -c reftable.blockSize=100 \
153
+ update-ref -m "$(cat logmsg)" refs/heads/logme HEAD 2>err &&
154
+ test_cmp expect err
155
+ )
156
+'
157
+
158
+test_expect_success 'block size exceeding maximum supported size' '
159
+ test_config_global core.logAllRefUpdates false &&
160
+ test_when_finished "rm -rf repo" &&
161
+ git init repo &&
162
+ (
163
+ cd repo &&
164
+ test_commit A &&
165
+ test_commit B &&
166
+ cat >expect <<-EOF &&
167
+ fatal: reftable block size cannot exceed 16MB
168
+ EOF
169
+ test_must_fail git -c reftable.blockSize=16777216 pack-refs 2>err &&
170
+ test_cmp expect err
171
+ )
172
+'
173
+
174
test_done