pack-write.c: prepare to write 'pack-*.rev' files
This patch prepares for callers to be able to write reverse index files to disk. It adds the necessary machinery to write a format-compliant .rev file from within 'write_rev_file()', which is called from 'finish_tmp_packfile()'. Similar to the process by which the reverse index is computed in memory, these new paths also have to sort a list of objects by their offsets within a packfile. These new paths use a qsort() (as opposed to a radix sort), since our specialized radix sort requires a full revindex_entry struct per object, which is more memory than we need to allocate. The qsort is obviously slower, but the theoretical slowdown would require a repository with a large amount of objects, likely implying that the time spent in, say, pack-objects during a repack would dominate the overall runtime. Signed-off-by: Taylor Blau <me@ttaylorr.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Taylor Blau committed
Jan 25, 2021 at 18:37 UTC
8ef50d9958f7be21e38026433d30f72521b4de47
2 files changed
+123
-1
pack-write.c
+119
-1
@@ -167,6 +167,113 @@ const char *write_idx_file(const char *index_name, struct pack_idx_entry **objec
167
return index_name;
168
}
169
170
+static int pack_order_cmp(const void *va, const void *vb, void *ctx)
171
+{
172
+ struct pack_idx_entry **objects = ctx;
173
+
174
+ off_t oa = objects[*(uint32_t*)va]->offset;
175
+ off_t ob = objects[*(uint32_t*)vb]->offset;
176
+
177
+ if (oa < ob)
178
+ return -1;
179
+ if (oa > ob)
180
+ return 1;
181
+ return 0;
182
+}
183
+
184
+static void write_rev_header(struct hashfile *f)
185
+{
186
+ uint32_t oid_version;
187
+ switch (hash_algo_by_ptr(the_hash_algo)) {
188
+ case GIT_HASH_SHA1:
189
+ oid_version = 1;
190
+ break;
191
+ case GIT_HASH_SHA256:
192
+ oid_version = 2;
193
+ break;
194
+ default:
195
+ die("write_rev_header: unknown hash version");
196
+ }
197
+
198
+ hashwrite_be32(f, RIDX_SIGNATURE);
199
+ hashwrite_be32(f, RIDX_VERSION);
200
+ hashwrite_be32(f, oid_version);
201
+}
202
+
203
+static void write_rev_index_positions(struct hashfile *f,
204
+ struct pack_idx_entry **objects,
205
+ uint32_t nr_objects)
206
+{
207
+ uint32_t *pack_order;
208
+ uint32_t i;
209
+
210
+ ALLOC_ARRAY(pack_order, nr_objects);
211
+ for (i = 0; i < nr_objects; i++)
212
+ pack_order[i] = i;
213
+ QSORT_S(pack_order, nr_objects, pack_order_cmp, objects);
214
+
215
+ for (i = 0; i < nr_objects; i++)
216
+ hashwrite_be32(f, pack_order[i]);
217
+
218
+ free(pack_order);
219
+}
220
+
221
+static void write_rev_trailer(struct hashfile *f, const unsigned char *hash)
222
+{
223
+ hashwrite(f, hash, the_hash_algo->rawsz);
224
+}
225
+
226
+const char *write_rev_file(const char *rev_name,
227
+ struct pack_idx_entry **objects,
228
+ uint32_t nr_objects,
229
+ const unsigned char *hash,
230
+ unsigned flags)
231
+{
232
+ struct hashfile *f;
233
+ int fd;
234
+
235
+ if ((flags & WRITE_REV) && (flags & WRITE_REV_VERIFY))
236
+ die(_("cannot both write and verify reverse index"));
237
+
238
+ if (flags & WRITE_REV) {
239
+ if (!rev_name) {
240
+ struct strbuf tmp_file = STRBUF_INIT;
241
+ fd = odb_mkstemp(&tmp_file, "pack/tmp_rev_XXXXXX");
242
+ rev_name = strbuf_detach(&tmp_file, NULL);
243
+ } else {
244
+ unlink(rev_name);
245
+ fd = open(rev_name, O_CREAT|O_EXCL|O_WRONLY, 0600);
246
+ if (fd < 0)
247
+ die_errno("unable to create '%s'", rev_name);
248
+ }
249
+ f = hashfd(fd, rev_name);
250
+ } else if (flags & WRITE_REV_VERIFY) {
251
+ struct stat statbuf;
252
+ if (stat(rev_name, &statbuf)) {
253
+ if (errno == ENOENT) {
254
+ /* .rev files are optional */
255
+ return NULL;
256
+ } else
257
+ die_errno(_("could not stat: %s"), rev_name);
258
+ }
259
+ f = hashfd_check(rev_name);
260
+ } else
261
+ return NULL;
262
+
263
+ write_rev_header(f);
264
+
265
+ write_rev_index_positions(f, objects, nr_objects);
266
+ write_rev_trailer(f, hash);
267
+
268
+ if (rev_name && adjust_shared_perm(rev_name) < 0)
269
+ die(_("failed to make %s readable"), rev_name);
270
+
271
+ finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_CLOSE |
272
+ ((flags & WRITE_IDX_VERIFY) ? 0 : CSUM_FSYNC));
273
+
274
+ return rev_name;
275
+}
276
+
277
off_t write_pack_header(struct hashfile *f, uint32_t nr_entries)
278
{
279
struct pack_header hdr;
@@ -342,7 +449,7 @@ void finish_tmp_packfile(struct strbuf *name_buffer,
449
struct pack_idx_option *pack_idx_opts,
450
unsigned char hash[])
451
{
345
- const char *idx_tmp_name;
452
+ const char *idx_tmp_name, *rev_tmp_name = NULL;
453
int basename_len = name_buffer->len;
454
455
if (adjust_shared_perm(pack_tmp_name))
@@ -353,6 +460,9 @@ void finish_tmp_packfile(struct strbuf *name_buffer,
460
if (adjust_shared_perm(idx_tmp_name))
461
die_errno("unable to make temporary index file readable");
462
463
+ rev_tmp_name = write_rev_file(NULL, written_list, nr_written, hash,
464
+ pack_idx_opts->flags);
465
+
466
strbuf_addf(name_buffer, "%s.pack", hash_to_hex(hash));
467
468
if (rename(pack_tmp_name, name_buffer->buf))
@@ -366,6 +476,14 @@ void finish_tmp_packfile(struct strbuf *name_buffer,
476
477
strbuf_setlen(name_buffer, basename_len);
478
479
+ if (rev_tmp_name) {
480
+ strbuf_addf(name_buffer, "%s.rev", hash_to_hex(hash));
481
+ if (rename(rev_tmp_name, name_buffer->buf))
482
+ die_errno("unable to rename temporary reverse-index file");
483
+ }
484
+
485
+ strbuf_setlen(name_buffer, basename_len);
486
+
487
free((void *)idx_tmp_name);
488
}
489
pack.h
+4
@@ -42,6 +42,8 @@ struct pack_idx_option {
42
/* flag bits */
43
#define WRITE_IDX_VERIFY 01 /* verify only, do not write the idx file */
44
#define WRITE_IDX_STRICT 02
45
+#define WRITE_REV 04
46
+#define WRITE_REV_VERIFY 010
47
48
uint32_t version;
49
uint32_t off32_limit;
@@ -91,6 +93,8 @@ struct ref;
93
94
void write_promisor_file(const char *promisor_name, struct ref **sought, int nr_sought);
95
96
+const char *write_rev_file(const char *rev_name, struct pack_idx_entry **objects, uint32_t nr_objects, const unsigned char *hash, unsigned flags);
97
+
98
/*
99
* The "hdr" output buffer should be at least this big, which will handle sizes
100
* up to 2^67.