@matheusbsbs / matheus-bagliano / commits / d94ac23d3b

reftable/block: optimize allocations by using scratch buffer

The block writer needs to compute the key for every record that one adds to the writer. The buffer for this key is stored on the stack and thus reallocated on every call to `block_writer_add()`, which is inefficient. Refactor the code so that we store the buffer in the `block_writer` struct itself so that we can reuse it. This reduces the number of allocations when writing many refs, e.g. when migrating one million refs from the "files" backend to the "reftable backend. Before this change: HEAP SUMMARY: in use at exit: 80,048 bytes in 49 blocks total heap usage: 3,025,864 allocs, 3,025,815 frees, 372,746,291 bytes allocated After this change: HEAP SUMMARY: in use at exit: 80,048 bytes in 49 blocks total heap usage: 2,013,250 allocs, 2,013,201 frees, 347,543,583 bytes allocated Signed-off-by: Patrick Steinhardt <ps@pks.im> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Patrick Steinhardt committed Nov 20, 2024 at 08:51 UTC d94ac23d3b9e223c6cacfdab7047d3d4efae3218

2 files changed +6 -8

reftable/block.c

+5 -8

@@ -110,24 +110,21 @@ int block_writer_add(struct block_writer *w, struct reftable_record *rec)

110 .buf = w->block + w->next,

111 .len = w->block_size - w->next,

112 };

113 -

113 struct string_view start = out;

115 -

114 int is_restart = 0;

117 - struct reftable_buf key = REFTABLE_BUF_INIT;

115 int n = 0;

116 int err;

117

121 - err = reftable_record_key(rec, &key);

118 + err = reftable_record_key(rec, &w->buf);

119 if (err < 0)

120 goto done;

121

125 - if (!key.len) {

122 + if (!w->buf.len) {

123 err = REFTABLE_API_ERROR;

124 goto done;

125 }

126

130 - n = reftable_encode_key(&is_restart, out, last, key,

127 + n = reftable_encode_key(&is_restart, out, last, w->buf,

128 reftable_record_val_type(rec));

129 if (n < 0) {

130 err = -1;

@@ -143,9 +140,8 @@ int block_writer_add(struct block_writer *w, struct reftable_record *rec)

140 string_view_consume(&out, n);

141

142 err = block_writer_register_restart(w, start.len - out.len, is_restart,

146 - &key);

143 + &w->buf);

144 done:

148 - reftable_buf_release(&key);

145 return err;

146 }

147

@@ -569,6 +565,7 @@ void block_writer_release(struct block_writer *bw)

565 REFTABLE_FREE_AND_NULL(bw->zstream);

566 REFTABLE_FREE_AND_NULL(bw->restarts);

567 REFTABLE_FREE_AND_NULL(bw->compressed);

568 + reftable_buf_release(&bw->buf);

569 reftable_buf_release(&bw->last_key);

570 /* the block is not owned. */

571 }

reftable/block.h

@@ -39,6 +39,7 @@ struct block_writer {

39 uint32_t restart_cap;

41 struct reftable_buf last_key;

42 + struct reftable_buf buf;

43 int entries;

44 };