@matheusbsbs / matheus-bagliano / commits / 66ed011bf7

reftable/writer: optimize allocations by using a scratch buffer

Both `writer_add_record()` and `reftable_writer_add_ref()` get executed for every single ref record we're adding to the reftable writer. And as both functions use a local buffer to write data, the allocations we have to do here add up during larger transactions. Refactor the code to use a scratch buffer part of the `reftable_writer` itself such that we can reuse it. This signifcantly reduces the number of allocations during large transactions, e.g. when migrating refs from the "files" backend to the "reftable" backend. Before this change: HEAP SUMMARY: in use at exit: 80,048 bytes in 49 blocks total heap usage: 5,032,171 allocs, 5,032,122 frees, 418,792,092 bytes allocated After this change: HEAP SUMMARY: in use at exit: 80,048 bytes in 49 blocks total heap usage: 3,025,864 allocs, 3,025,815 frees, 372,746,291 bytes allocated This also translate into a small speedup: Benchmark 1: migrate files:reftable (refcount = 1000000, revision = HEAD~) Time (mean ± σ): 827.2 ms ± 16.5 ms [User: 689.4 ms, System: 124.9 ms] Range (min … max): 809.0 ms … 924.7 ms 50 runs Benchmark 2: migrate files:reftable (refcount = 1000000, revision = HEAD) Time (mean ± σ): 813.6 ms ± 11.6 ms [User: 679.0 ms, System: 123.4 ms] Range (min … max): 786.7 ms … 833.5 ms 50 runs Summary migrate files:reftable (refcount = 1000000, revision = HEAD) ran 1.02 ± 0.02 times faster than migrate files:reftable (refcount = 1000000, revision = HEAD~) Signed-off-by: Patrick Steinhardt <ps@pks.im> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Patrick Steinhardt committed Nov 20, 2024 at 08:51 UTC 66ed011bf73e6503059a44698f83fd8a6b0f7726

2 files changed +12 -12

reftable/writer.c

+11 -12

@@ -148,6 +148,7 @@ int reftable_writer_new(struct reftable_writer **out,

148

149 reftable_buf_init(&wp->block_writer_data.last_key);

150 reftable_buf_init(&wp->last_key);

151 + reftable_buf_init(&wp->buf);

152 REFTABLE_CALLOC_ARRAY(wp->block, opts.block_size);

153 if (!wp->block) {

154 reftable_free(wp);

@@ -180,6 +181,7 @@ static void writer_release(struct reftable_writer *w)

181 w->block_writer = NULL;

182 writer_clear_index(w);

183 reftable_buf_release(&w->last_key);

184 + reftable_buf_release(&w->buf);

185 }

186 }

187

@@ -249,20 +251,19 @@ static int writer_index_hash(struct reftable_writer *w, struct reftable_buf *has

251 static int writer_add_record(struct reftable_writer *w,

252 struct reftable_record *rec)

253 {

252 - struct reftable_buf key = REFTABLE_BUF_INIT;

254 int err;

255

255 - err = reftable_record_key(rec, &key);

256 + err = reftable_record_key(rec, &w->buf);

257 if (err < 0)

258 goto done;

259

259 - if (reftable_buf_cmp(&w->last_key, &key) >= 0) {

260 + if (reftable_buf_cmp(&w->last_key, &w->buf) >= 0) {

261 err = REFTABLE_API_ERROR;

262 goto done;

263 }

264

265 reftable_buf_reset(&w->last_key);

265 - err = reftable_buf_add(&w->last_key, key.buf, key.len);

266 + err = reftable_buf_add(&w->last_key, w->buf.buf, w->buf.len);

267 if (err < 0)

268 goto done;

269

@@ -312,7 +313,6 @@ static int writer_add_record(struct reftable_writer *w,

313 }

314

315 done:

315 - reftable_buf_release(&key);

316 return err;

317 }

318

@@ -325,7 +325,6 @@ int reftable_writer_add_ref(struct reftable_writer *w,

325 .ref = *ref

326 },

327 };

328 - struct reftable_buf buf = REFTABLE_BUF_INIT;

328 int err;

329

330 if (!ref->refname ||

@@ -340,24 +339,25 @@ int reftable_writer_add_ref(struct reftable_writer *w,

339 goto out;

340

341 if (!w->opts.skip_index_objects && reftable_ref_record_val1(ref)) {

343 - err = reftable_buf_add(&buf, (char *)reftable_ref_record_val1(ref),

342 + reftable_buf_reset(&w->buf);

343 + err = reftable_buf_add(&w->buf, (char *)reftable_ref_record_val1(ref),

344 hash_size(w->opts.hash_id));

345 if (err < 0)

346 goto out;

347

348 - err = writer_index_hash(w, &buf);

348 + err = writer_index_hash(w, &w->buf);

349 if (err < 0)

350 goto out;

351 }

352

353 if (!w->opts.skip_index_objects && reftable_ref_record_val2(ref)) {

354 - reftable_buf_reset(&buf);

355 - err = reftable_buf_add(&buf, reftable_ref_record_val2(ref),

354 + reftable_buf_reset(&w->buf);

355 + err = reftable_buf_add(&w->buf, reftable_ref_record_val2(ref),

356 hash_size(w->opts.hash_id));

357 if (err < 0)

358 goto out;

359

360 - err = writer_index_hash(w, &buf);

360 + err = writer_index_hash(w, &w->buf);

361 if (err < 0)

362 goto out;

363 }

@@ -365,7 +365,6 @@ int reftable_writer_add_ref(struct reftable_writer *w,

365 err = 0;

366

367 out:

368 - reftable_buf_release(&buf);

368 return err;

369 }

370

reftable/writer.h

@@ -20,6 +20,7 @@ struct reftable_writer {

20 void *write_arg;

21 int pending_padding;

22 struct reftable_buf last_key;

23 + struct reftable_buf buf;

25 /* offset of next block to write. */

26 uint64_t next;