object-file: relocate ODB transaction code
The bulk-checkin subsystem provides various functions to manage ODB transactions. Apart from {begin,end}_odb_transaction(), these functions are only used by the object-file subsystem to manage aspects of a transaction implementation specific to the files object source. Relocate all the transaction code in bulk-checkin to object-file. This simplifies the exposed transaction interface by reducing it to only {begin,end}_odb_transaction(). Function and type names are adjusted in the subsequent commit to better fit the new location. Signed-off-by: Justin Tobler <jltobler@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Justin Tobler committed
Sep 16, 2025 at 13:29 UTC
78839e9cdead363d10190a009783c2d18149cc54
11 files changed
+410
-453
Makefile
-1
@@ -974,7 +974,6 @@ LIB_OBJS += blame.o
974
LIB_OBJS += blob.o
975
LIB_OBJS += bloom.o
976
LIB_OBJS += branch.o
977
-LIB_OBJS += bulk-checkin.o
977
LIB_OBJS += bundle-uri.o
978
LIB_OBJS += bundle.o
979
LIB_OBJS += cache-tree.o
builtin/add.c
+1
-1
@@ -14,13 +14,13 @@
14
#include "gettext.h"
15
#include "pathspec.h"
16
#include "run-command.h"
17
+#include "object-file.h"
18
#include "parse-options.h"
19
#include "path.h"
20
#include "preload-index.h"
21
#include "diff.h"
22
#include "read-cache.h"
23
#include "revision.h"
23
-#include "bulk-checkin.h"
24
#include "strvec.h"
25
#include "submodule.h"
26
#include "add-interactive.h"
builtin/unpack-objects.c
-1
@@ -2,7 +2,6 @@
2
#define DISABLE_SIGN_COMPARE_WARNINGS
3
4
#include "builtin.h"
5
-#include "bulk-checkin.h"
5
#include "config.h"
6
#include "environment.h"
7
#include "gettext.h"
builtin/update-index.c
-1
@@ -8,7 +8,6 @@
8
#define DISABLE_SIGN_COMPARE_WARNINGS
9
10
#include "builtin.h"
11
-#include "bulk-checkin.h"
11
#include "config.h"
12
#include "environment.h"
13
#include "gettext.h"
bulk-checkin.c
deleted
-393
@@ -1,393 +0,0 @@
1
-/*
2
- * Copyright (c) 2011, Google Inc.
3
- */
4
-
5
-#define USE_THE_REPOSITORY_VARIABLE
6
-
7
-#include "git-compat-util.h"
8
-#include "bulk-checkin.h"
9
-#include "environment.h"
10
-#include "gettext.h"
11
-#include "hex.h"
12
-#include "lockfile.h"
13
-#include "repository.h"
14
-#include "csum-file.h"
15
-#include "pack.h"
16
-#include "strbuf.h"
17
-#include "tmp-objdir.h"
18
-#include "packfile.h"
19
-#include "object-file.h"
20
-#include "odb.h"
21
-
22
-struct bulk_checkin_packfile {
23
- char *pack_tmp_name;
24
- struct hashfile *f;
25
- off_t offset;
26
- struct pack_idx_option pack_idx_opts;
27
-
28
- struct pack_idx_entry **written;
29
- uint32_t alloc_written;
30
- uint32_t nr_written;
31
-};
32
-
33
-struct odb_transaction {
34
- struct object_database *odb;
35
-
36
- struct tmp_objdir *objdir;
37
- struct bulk_checkin_packfile packfile;
38
-};
39
-
40
-static void finish_tmp_packfile(struct odb_transaction *transaction,
41
- struct strbuf *basename,
42
- unsigned char hash[])
43
-{
44
- struct bulk_checkin_packfile *state = &transaction->packfile;
45
- struct repository *repo = transaction->odb->repo;
46
- char *idx_tmp_name = NULL;
47
-
48
- stage_tmp_packfiles(repo, basename, state->pack_tmp_name,
49
- state->written, state->nr_written, NULL,
50
- &state->pack_idx_opts, hash, &idx_tmp_name);
51
- rename_tmp_packfile_idx(repo, basename, &idx_tmp_name);
52
-
53
- free(idx_tmp_name);
54
-}
55
-
56
-static void flush_bulk_checkin_packfile(struct odb_transaction *transaction)
57
-{
58
- struct bulk_checkin_packfile *state = &transaction->packfile;
59
- struct repository *repo = transaction->odb->repo;
60
- unsigned char hash[GIT_MAX_RAWSZ];
61
- struct strbuf packname = STRBUF_INIT;
62
-
63
- if (!state->f)
64
- return;
65
-
66
- if (state->nr_written == 0) {
67
- close(state->f->fd);
68
- free_hashfile(state->f);
69
- unlink(state->pack_tmp_name);
70
- goto clear_exit;
71
- } else if (state->nr_written == 1) {
72
- finalize_hashfile(state->f, hash, FSYNC_COMPONENT_PACK,
73
- CSUM_HASH_IN_STREAM | CSUM_FSYNC | CSUM_CLOSE);
74
- } else {
75
- int fd = finalize_hashfile(state->f, hash, FSYNC_COMPONENT_PACK, 0);
76
- fixup_pack_header_footer(repo->hash_algo, fd, hash, state->pack_tmp_name,
77
- state->nr_written, hash,
78
- state->offset);
79
- close(fd);
80
- }
81
-
82
- strbuf_addf(&packname, "%s/pack/pack-%s.",
83
- repo_get_object_directory(transaction->odb->repo),
84
- hash_to_hex_algop(hash, repo->hash_algo));
85
-
86
- finish_tmp_packfile(transaction, &packname, hash);
87
- for (uint32_t i = 0; i < state->nr_written; i++)
88
- free(state->written[i]);
89
-
90
-clear_exit:
91
- free(state->pack_tmp_name);
92
- free(state->written);
93
- memset(state, 0, sizeof(*state));
94
-
95
- strbuf_release(&packname);
96
- /* Make objects we just wrote available to ourselves */
97
- reprepare_packed_git(repo);
98
-}
99
-
100
-/*
101
- * Cleanup after batch-mode fsync_object_files.
102
- */
103
-static void flush_batch_fsync(struct odb_transaction *transaction)
104
-{
105
- struct strbuf temp_path = STRBUF_INIT;
106
- struct tempfile *temp;
107
-
108
- if (!transaction->objdir)
109
- return;
110
-
111
- /*
112
- * Issue a full hardware flush against a temporary file to ensure
113
- * that all objects are durable before any renames occur. The code in
114
- * fsync_loose_object_bulk_checkin has already issued a writeout
115
- * request, but it has not flushed any writeback cache in the storage
116
- * hardware or any filesystem logs. This fsync call acts as a barrier
117
- * to ensure that the data in each new object file is durable before
118
- * the final name is visible.
119
- */
120
- strbuf_addf(&temp_path, "%s/bulk_fsync_XXXXXX",
121
- repo_get_object_directory(transaction->odb->repo));
122
- temp = xmks_tempfile(temp_path.buf);
123
- fsync_or_die(get_tempfile_fd(temp), get_tempfile_path(temp));
124
- delete_tempfile(&temp);
125
- strbuf_release(&temp_path);
126
-
127
- /*
128
- * Make the object files visible in the primary ODB after their data is
129
- * fully durable.
130
- */
131
- tmp_objdir_migrate(transaction->objdir);
132
- transaction->objdir = NULL;
133
-}
134
-
135
-static int already_written(struct odb_transaction *transaction,
136
- struct object_id *oid)
137
-{
138
- /* The object may already exist in the repository */
139
- if (odb_has_object(transaction->odb, oid,
140
- HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR))
141
- return 1;
142
-
143
- /* Might want to keep the list sorted */
144
- for (uint32_t i = 0; i < transaction->packfile.nr_written; i++)
145
- if (oideq(&transaction->packfile.written[i]->oid, oid))
146
- return 1;
147
-
148
- /* This is a new object we need to keep */
149
- return 0;
150
-}
151
-
152
-/*
153
- * Read the contents from fd for size bytes, streaming it to the
154
- * packfile in state while updating the hash in ctx. Signal a failure
155
- * by returning a negative value when the resulting pack would exceed
156
- * the pack size limit and this is not the first object in the pack,
157
- * so that the caller can discard what we wrote from the current pack
158
- * by truncating it and opening a new one. The caller will then call
159
- * us again after rewinding the input fd.
160
- *
161
- * The already_hashed_to pointer is kept untouched by the caller to
162
- * make sure we do not hash the same byte when we are called
163
- * again. This way, the caller does not have to checkpoint its hash
164
- * status before calling us just in case we ask it to call us again
165
- * with a new pack.
166
- */
167
-static int stream_blob_to_pack(struct bulk_checkin_packfile *state,
168
- struct git_hash_ctx *ctx, off_t *already_hashed_to,
169
- int fd, size_t size, const char *path,
170
- unsigned flags)
171
-{
172
- git_zstream s;
173
- unsigned char ibuf[16384];
174
- unsigned char obuf[16384];
175
- unsigned hdrlen;
176
- int status = Z_OK;
177
- int write_object = (flags & INDEX_WRITE_OBJECT);
178
- off_t offset = 0;
179
-
180
- git_deflate_init(&s, pack_compression_level);
181
-
182
- hdrlen = encode_in_pack_object_header(obuf, sizeof(obuf), OBJ_BLOB, size);
183
- s.next_out = obuf + hdrlen;
184
- s.avail_out = sizeof(obuf) - hdrlen;
185
-
186
- while (status != Z_STREAM_END) {
187
- if (size && !s.avail_in) {
188
- size_t rsize = size < sizeof(ibuf) ? size : sizeof(ibuf);
189
- ssize_t read_result = read_in_full(fd, ibuf, rsize);
190
- if (read_result < 0)
191
- die_errno("failed to read from '%s'", path);
192
- if ((size_t)read_result != rsize)
193
- die("failed to read %u bytes from '%s'",
194
- (unsigned)rsize, path);
195
- offset += rsize;
196
- if (*already_hashed_to < offset) {
197
- size_t hsize = offset - *already_hashed_to;
198
- if (rsize < hsize)
199
- hsize = rsize;
200
- if (hsize)
201
- git_hash_update(ctx, ibuf, hsize);
202
- *already_hashed_to = offset;
203
- }
204
- s.next_in = ibuf;
205
- s.avail_in = rsize;
206
- size -= rsize;
207
- }
208
-
209
- status = git_deflate(&s, size ? 0 : Z_FINISH);
210
-
211
- if (!s.avail_out || status == Z_STREAM_END) {
212
- if (write_object) {
213
- size_t written = s.next_out - obuf;
214
-
215
- /* would we bust the size limit? */
216
- if (state->nr_written &&
217
- pack_size_limit_cfg &&
218
- pack_size_limit_cfg < state->offset + written) {
219
- git_deflate_abort(&s);
220
- return -1;
221
- }
222
-
223
- hashwrite(state->f, obuf, written);
224
- state->offset += written;
225
- }
226
- s.next_out = obuf;
227
- s.avail_out = sizeof(obuf);
228
- }
229
-
230
- switch (status) {
231
- case Z_OK:
232
- case Z_BUF_ERROR:
233
- case Z_STREAM_END:
234
- continue;
235
- default:
236
- die("unexpected deflate failure: %d", status);
237
- }
238
- }
239
- git_deflate_end(&s);
240
- return 0;
241
-}
242
-
243
-/* Lazily create backing packfile for the state */
244
-static void prepare_to_stream(struct odb_transaction *transaction,
245
- unsigned flags)
246
-{
247
- struct bulk_checkin_packfile *state = &transaction->packfile;
248
- if (!(flags & INDEX_WRITE_OBJECT) || state->f)
249
- return;
250
-
251
- state->f = create_tmp_packfile(transaction->odb->repo,
252
- &state->pack_tmp_name);
253
- reset_pack_idx_option(&state->pack_idx_opts);
254
-
255
- /* Pretend we are going to write only one object */
256
- state->offset = write_pack_header(state->f, 1);
257
- if (!state->offset)
258
- die_errno("unable to write pack header");
259
-}
260
-
261
-int index_blob_bulk_checkin(struct odb_transaction *transaction,
262
- struct object_id *result_oid, int fd, size_t size,
263
- const char *path, unsigned flags)
264
-{
265
- struct bulk_checkin_packfile *state = &transaction->packfile;
266
- off_t seekback, already_hashed_to;
267
- struct git_hash_ctx ctx;
268
- unsigned char obuf[16384];
269
- unsigned header_len;
270
- struct hashfile_checkpoint checkpoint;
271
- struct pack_idx_entry *idx = NULL;
272
-
273
- seekback = lseek(fd, 0, SEEK_CUR);
274
- if (seekback == (off_t) -1)
275
- return error("cannot find the current offset");
276
-
277
- header_len = format_object_header((char *)obuf, sizeof(obuf),
278
- OBJ_BLOB, size);
279
- transaction->odb->repo->hash_algo->init_fn(&ctx);
280
- git_hash_update(&ctx, obuf, header_len);
281
-
282
- /* Note: idx is non-NULL when we are writing */
283
- if ((flags & INDEX_WRITE_OBJECT) != 0) {
284
- CALLOC_ARRAY(idx, 1);
285
-
286
- prepare_to_stream(transaction, flags);
287
- hashfile_checkpoint_init(state->f, &checkpoint);
288
- }
289
-
290
- already_hashed_to = 0;
291
-
292
- while (1) {
293
- prepare_to_stream(transaction, flags);
294
- if (idx) {
295
- hashfile_checkpoint(state->f, &checkpoint);
296
- idx->offset = state->offset;
297
- crc32_begin(state->f);
298
- }
299
- if (!stream_blob_to_pack(state, &ctx, &already_hashed_to,
300
- fd, size, path, flags))
301
- break;
302
- /*
303
- * Writing this object to the current pack will make
304
- * it too big; we need to truncate it, start a new
305
- * pack, and write into it.
306
- */
307
- if (!idx)
308
- BUG("should not happen");
309
- hashfile_truncate(state->f, &checkpoint);
310
- state->offset = checkpoint.offset;
311
- flush_bulk_checkin_packfile(transaction);
312
- if (lseek(fd, seekback, SEEK_SET) == (off_t) -1)
313
- return error("cannot seek back");
314
- }
315
- git_hash_final_oid(result_oid, &ctx);
316
- if (!idx)
317
- return 0;
318
-
319
- idx->crc32 = crc32_end(state->f);
320
- if (already_written(transaction, result_oid)) {
321
- hashfile_truncate(state->f, &checkpoint);
322
- state->offset = checkpoint.offset;
323
- free(idx);
324
- } else {
325
- oidcpy(&idx->oid, result_oid);
326
- ALLOC_GROW(state->written,
327
- state->nr_written + 1,
328
- state->alloc_written);
329
- state->written[state->nr_written++] = idx;
330
- }
331
- return 0;
332
-}
333
-
334
-void prepare_loose_object_bulk_checkin(struct odb_transaction *transaction)
335
-{
336
- /*
337
- * We lazily create the temporary object directory
338
- * the first time an object might be added, since
339
- * callers may not know whether any objects will be
340
- * added at the time they call begin_odb_transaction.
341
- */
342
- if (!transaction || transaction->objdir)
343
- return;
344
-
345
- transaction->objdir = tmp_objdir_create(transaction->odb->repo, "bulk-fsync");
346
- if (transaction->objdir)
347
- tmp_objdir_replace_primary_odb(transaction->objdir, 0);
348
-}
349
-
350
-void fsync_loose_object_bulk_checkin(struct odb_transaction *transaction,
351
- int fd, const char *filename)
352
-{
353
- /*
354
- * If we have an active ODB transaction, we issue a call that
355
- * cleans the filesystem page cache but avoids a hardware flush
356
- * command. Later on we will issue a single hardware flush
357
- * before renaming the objects to their final names as part of
358
- * flush_batch_fsync.
359
- */
360
- if (!transaction || !transaction->objdir ||
361
- git_fsync(fd, FSYNC_WRITEOUT_ONLY) < 0) {
362
- if (errno == ENOSYS)
363
- warning(_("core.fsyncMethod = batch is unsupported on this platform"));
364
- fsync_or_die(fd, filename);
365
- }
366
-}
367
-
368
-struct odb_transaction *begin_odb_transaction(struct object_database *odb)
369
-{
370
- if (odb->transaction)
371
- return NULL;
372
-
373
- CALLOC_ARRAY(odb->transaction, 1);
374
- odb->transaction->odb = odb;
375
-
376
- return odb->transaction;
377
-}
378
-
379
-void end_odb_transaction(struct odb_transaction *transaction)
380
-{
381
- if (!transaction)
382
- return;
383
-
384
- /*
385
- * Ensure the transaction ending matches the pending transaction.
386
- */
387
- ASSERT(transaction == transaction->odb->transaction);
388
-
389
- flush_batch_fsync(transaction);
390
- flush_bulk_checkin_packfile(transaction);
391
- transaction->odb->transaction = NULL;
392
- free(transaction);
393
-}
bulk-checkin.h
deleted
-52
@@ -1,52 +0,0 @@
1
-/*
2
- * Copyright (c) 2011, Google Inc.
3
- */
4
-#ifndef BULK_CHECKIN_H
5
-#define BULK_CHECKIN_H
6
-
7
-#include "object.h"
8
-#include "odb.h"
9
-
10
-struct odb_transaction;
11
-
12
-void prepare_loose_object_bulk_checkin(struct odb_transaction *transaction);
13
-void fsync_loose_object_bulk_checkin(struct odb_transaction *transaction,
14
- int fd, const char *filename);
15
-
16
-/*
17
- * This writes the specified object to a packfile. Objects written here
18
- * during the same transaction are written to the same packfile. The
19
- * packfile is not flushed until the transaction is flushed. The caller
20
- * is expected to ensure a valid transaction is setup for objects to be
21
- * recorded to.
22
- *
23
- * This also bypasses the usual "convert-to-git" dance, and that is on
24
- * purpose. We could write a streaming version of the converting
25
- * functions and insert that before feeding the data to fast-import
26
- * (or equivalent in-core API described above). However, that is
27
- * somewhat complicated, as we do not know the size of the filter
28
- * result, which we need to know beforehand when writing a git object.
29
- * Since the primary motivation for trying to stream from the working
30
- * tree file and to avoid mmaping it in core is to deal with large
31
- * binary blobs, they generally do not want to get any conversion, and
32
- * callers should avoid this code path when filters are requested.
33
- */
34
-int index_blob_bulk_checkin(struct odb_transaction *transaction,
35
- struct object_id *oid, int fd, size_t size,
36
- const char *path, unsigned flags);
37
-
38
-/*
39
- * Tell the object database to optimize for adding
40
- * multiple objects. end_odb_transaction must be called
41
- * to make new objects visible. If a transaction is already
42
- * pending, NULL is returned.
43
- */
44
-struct odb_transaction *begin_odb_transaction(struct object_database *odb);
45
-
46
-/*
47
- * Tell the object database to make any objects from the
48
- * current transaction visible.
49
- */
50
-void end_odb_transaction(struct odb_transaction *transaction);
51
-
52
-#endif
cache-tree.c
-1
@@ -8,7 +8,6 @@
8
#include "tree.h"
9
#include "tree-walk.h"
10
#include "cache-tree.h"
11
-#include "bulk-checkin.h"
11
#include "object-file.h"
12
#include "odb.h"
13
#include "read-cache-ll.h"
meson.build
-1
@@ -287,7 +287,6 @@ libgit_sources = [
287
'blob.c',
288
'bloom.c',
289
'branch.c',
290
- 'bulk-checkin.c',
290
'bundle-uri.c',
291
'bundle.c',
292
'cache-tree.c',
object-file.c
+393
-1
@@ -10,7 +10,6 @@
10
#define USE_THE_REPOSITORY_VARIABLE
11
12
#include "git-compat-util.h"
13
-#include "bulk-checkin.h"
13
#include "convert.h"
14
#include "dir.h"
15
#include "environment.h"
@@ -28,6 +27,8 @@
27
#include "read-cache-ll.h"
28
#include "setup.h"
29
#include "streaming.h"
30
+#include "tempfile.h"
31
+#include "tmp-objdir.h"
32
33
/* The maximum size for an object header. */
34
#define MAX_HEADER_LEN 32
@@ -666,6 +667,93 @@ void hash_object_file(const struct git_hash_algo *algo, const void *buf,
667
write_object_file_prepare(algo, buf, len, type, oid, hdr, &hdrlen);
668
}
669
670
+struct bulk_checkin_packfile {
671
+ char *pack_tmp_name;
672
+ struct hashfile *f;
673
+ off_t offset;
674
+ struct pack_idx_option pack_idx_opts;
675
+
676
+ struct pack_idx_entry **written;
677
+ uint32_t alloc_written;
678
+ uint32_t nr_written;
679
+};
680
+
681
+struct odb_transaction {
682
+ struct object_database *odb;
683
+
684
+ struct tmp_objdir *objdir;
685
+ struct bulk_checkin_packfile packfile;
686
+};
687
+
688
+static void prepare_loose_object_bulk_checkin(struct odb_transaction *transaction)
689
+{
690
+ /*
691
+ * We lazily create the temporary object directory
692
+ * the first time an object might be added, since
693
+ * callers may not know whether any objects will be
694
+ * added at the time they call begin_odb_transaction.
695
+ */
696
+ if (!transaction || transaction->objdir)
697
+ return;
698
+
699
+ transaction->objdir = tmp_objdir_create(transaction->odb->repo, "bulk-fsync");
700
+ if (transaction->objdir)
701
+ tmp_objdir_replace_primary_odb(transaction->objdir, 0);
702
+}
703
+
704
+static void fsync_loose_object_bulk_checkin(struct odb_transaction *transaction,
705
+ int fd, const char *filename)
706
+{
707
+ /*
708
+ * If we have an active ODB transaction, we issue a call that
709
+ * cleans the filesystem page cache but avoids a hardware flush
710
+ * command. Later on we will issue a single hardware flush
711
+ * before renaming the objects to their final names as part of
712
+ * flush_batch_fsync.
713
+ */
714
+ if (!transaction || !transaction->objdir ||
715
+ git_fsync(fd, FSYNC_WRITEOUT_ONLY) < 0) {
716
+ if (errno == ENOSYS)
717
+ warning(_("core.fsyncMethod = batch is unsupported on this platform"));
718
+ fsync_or_die(fd, filename);
719
+ }
720
+}
721
+
722
+/*
723
+ * Cleanup after batch-mode fsync_object_files.
724
+ */
725
+static void flush_batch_fsync(struct odb_transaction *transaction)
726
+{
727
+ struct strbuf temp_path = STRBUF_INIT;
728
+ struct tempfile *temp;
729
+
730
+ if (!transaction->objdir)
731
+ return;
732
+
733
+ /*
734
+ * Issue a full hardware flush against a temporary file to ensure
735
+ * that all objects are durable before any renames occur. The code in
736
+ * fsync_loose_object_bulk_checkin has already issued a writeout
737
+ * request, but it has not flushed any writeback cache in the storage
738
+ * hardware or any filesystem logs. This fsync call acts as a barrier
739
+ * to ensure that the data in each new object file is durable before
740
+ * the final name is visible.
741
+ */
742
+ strbuf_addf(&temp_path, "%s/bulk_fsync_XXXXXX",
743
+ repo_get_object_directory(transaction->odb->repo));
744
+ temp = xmks_tempfile(temp_path.buf);
745
+ fsync_or_die(get_tempfile_fd(temp), get_tempfile_path(temp));
746
+ delete_tempfile(&temp);
747
+ strbuf_release(&temp_path);
748
+
749
+ /*
750
+ * Make the object files visible in the primary ODB after their data is
751
+ * fully durable.
752
+ */
753
+ tmp_objdir_migrate(transaction->objdir);
754
+ transaction->objdir = NULL;
755
+}
756
+
757
/* Finalize a file on disk, and close it. */
758
static void close_loose_object(struct odb_source *source,
759
int fd, const char *filename)
@@ -1243,6 +1331,283 @@ static int index_core(struct index_state *istate,
1331
return ret;
1332
}
1333
1334
+static int already_written(struct odb_transaction *transaction,
1335
+ struct object_id *oid)
1336
+{
1337
+ /* The object may already exist in the repository */
1338
+ if (odb_has_object(transaction->odb, oid,
1339
+ HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR))
1340
+ return 1;
1341
+
1342
+ /* Might want to keep the list sorted */
1343
+ for (uint32_t i = 0; i < transaction->packfile.nr_written; i++)
1344
+ if (oideq(&transaction->packfile.written[i]->oid, oid))
1345
+ return 1;
1346
+
1347
+ /* This is a new object we need to keep */
1348
+ return 0;
1349
+}
1350
+
1351
+/* Lazily create backing packfile for the state */
1352
+static void prepare_to_stream(struct odb_transaction *transaction,
1353
+ unsigned flags)
1354
+{
1355
+ struct bulk_checkin_packfile *state = &transaction->packfile;
1356
+ if (!(flags & INDEX_WRITE_OBJECT) || state->f)
1357
+ return;
1358
+
1359
+ state->f = create_tmp_packfile(transaction->odb->repo,
1360
+ &state->pack_tmp_name);
1361
+ reset_pack_idx_option(&state->pack_idx_opts);
1362
+
1363
+ /* Pretend we are going to write only one object */
1364
+ state->offset = write_pack_header(state->f, 1);
1365
+ if (!state->offset)
1366
+ die_errno("unable to write pack header");
1367
+}
1368
+
1369
+/*
1370
+ * Read the contents from fd for size bytes, streaming it to the
1371
+ * packfile in state while updating the hash in ctx. Signal a failure
1372
+ * by returning a negative value when the resulting pack would exceed
1373
+ * the pack size limit and this is not the first object in the pack,
1374
+ * so that the caller can discard what we wrote from the current pack
1375
+ * by truncating it and opening a new one. The caller will then call
1376
+ * us again after rewinding the input fd.
1377
+ *
1378
+ * The already_hashed_to pointer is kept untouched by the caller to
1379
+ * make sure we do not hash the same byte when we are called
1380
+ * again. This way, the caller does not have to checkpoint its hash
1381
+ * status before calling us just in case we ask it to call us again
1382
+ * with a new pack.
1383
+ */
1384
+static int stream_blob_to_pack(struct bulk_checkin_packfile *state,
1385
+ struct git_hash_ctx *ctx, off_t *already_hashed_to,
1386
+ int fd, size_t size, const char *path,
1387
+ unsigned flags)
1388
+{
1389
+ git_zstream s;
1390
+ unsigned char ibuf[16384];
1391
+ unsigned char obuf[16384];
1392
+ unsigned hdrlen;
1393
+ int status = Z_OK;
1394
+ int write_object = (flags & INDEX_WRITE_OBJECT);
1395
+ off_t offset = 0;
1396
+
1397
+ git_deflate_init(&s, pack_compression_level);
1398
+
1399
+ hdrlen = encode_in_pack_object_header(obuf, sizeof(obuf), OBJ_BLOB, size);
1400
+ s.next_out = obuf + hdrlen;
1401
+ s.avail_out = sizeof(obuf) - hdrlen;
1402
+
1403
+ while (status != Z_STREAM_END) {
1404
+ if (size && !s.avail_in) {
1405
+ size_t rsize = size < sizeof(ibuf) ? size : sizeof(ibuf);
1406
+ ssize_t read_result = read_in_full(fd, ibuf, rsize);
1407
+ if (read_result < 0)
1408
+ die_errno("failed to read from '%s'", path);
1409
+ if ((size_t)read_result != rsize)
1410
+ die("failed to read %u bytes from '%s'",
1411
+ (unsigned)rsize, path);
1412
+ offset += rsize;
1413
+ if (*already_hashed_to < offset) {
1414
+ size_t hsize = offset - *already_hashed_to;
1415
+ if (rsize < hsize)
1416
+ hsize = rsize;
1417
+ if (hsize)
1418
+ git_hash_update(ctx, ibuf, hsize);
1419
+ *already_hashed_to = offset;
1420
+ }
1421
+ s.next_in = ibuf;
1422
+ s.avail_in = rsize;
1423
+ size -= rsize;
1424
+ }
1425
+
1426
+ status = git_deflate(&s, size ? 0 : Z_FINISH);
1427
+
1428
+ if (!s.avail_out || status == Z_STREAM_END) {
1429
+ if (write_object) {
1430
+ size_t written = s.next_out - obuf;
1431
+
1432
+ /* would we bust the size limit? */
1433
+ if (state->nr_written &&
1434
+ pack_size_limit_cfg &&
1435
+ pack_size_limit_cfg < state->offset + written) {
1436
+ git_deflate_abort(&s);
1437
+ return -1;
1438
+ }
1439
+
1440
+ hashwrite(state->f, obuf, written);
1441
+ state->offset += written;
1442
+ }
1443
+ s.next_out = obuf;
1444
+ s.avail_out = sizeof(obuf);
1445
+ }
1446
+
1447
+ switch (status) {
1448
+ case Z_OK:
1449
+ case Z_BUF_ERROR:
1450
+ case Z_STREAM_END:
1451
+ continue;
1452
+ default:
1453
+ die("unexpected deflate failure: %d", status);
1454
+ }
1455
+ }
1456
+ git_deflate_end(&s);
1457
+ return 0;
1458
+}
1459
+
1460
+static void finish_tmp_packfile(struct odb_transaction *transaction,
1461
+ struct strbuf *basename,
1462
+ unsigned char hash[])
1463
+{
1464
+ struct bulk_checkin_packfile *state = &transaction->packfile;
1465
+ struct repository *repo = transaction->odb->repo;
1466
+ char *idx_tmp_name = NULL;
1467
+
1468
+ stage_tmp_packfiles(repo, basename, state->pack_tmp_name,
1469
+ state->written, state->nr_written, NULL,
1470
+ &state->pack_idx_opts, hash, &idx_tmp_name);
1471
+ rename_tmp_packfile_idx(repo, basename, &idx_tmp_name);
1472
+
1473
+ free(idx_tmp_name);
1474
+}
1475
+
1476
+static void flush_bulk_checkin_packfile(struct odb_transaction *transaction)
1477
+{
1478
+ struct bulk_checkin_packfile *state = &transaction->packfile;
1479
+ struct repository *repo = transaction->odb->repo;
1480
+ unsigned char hash[GIT_MAX_RAWSZ];
1481
+ struct strbuf packname = STRBUF_INIT;
1482
+
1483
+ if (!state->f)
1484
+ return;
1485
+
1486
+ if (state->nr_written == 0) {
1487
+ close(state->f->fd);
1488
+ free_hashfile(state->f);
1489
+ unlink(state->pack_tmp_name);
1490
+ goto clear_exit;
1491
+ } else if (state->nr_written == 1) {
1492
+ finalize_hashfile(state->f, hash, FSYNC_COMPONENT_PACK,
1493
+ CSUM_HASH_IN_STREAM | CSUM_FSYNC | CSUM_CLOSE);
1494
+ } else {
1495
+ int fd = finalize_hashfile(state->f, hash, FSYNC_COMPONENT_PACK, 0);
1496
+ fixup_pack_header_footer(repo->hash_algo, fd, hash, state->pack_tmp_name,
1497
+ state->nr_written, hash,
1498
+ state->offset);
1499
+ close(fd);
1500
+ }
1501
+
1502
+ strbuf_addf(&packname, "%s/pack/pack-%s.",
1503
+ repo_get_object_directory(transaction->odb->repo),
1504
+ hash_to_hex_algop(hash, repo->hash_algo));
1505
+
1506
+ finish_tmp_packfile(transaction, &packname, hash);
1507
+ for (uint32_t i = 0; i < state->nr_written; i++)
1508
+ free(state->written[i]);
1509
+
1510
+clear_exit:
1511
+ free(state->pack_tmp_name);
1512
+ free(state->written);
1513
+ memset(state, 0, sizeof(*state));
1514
+
1515
+ strbuf_release(&packname);
1516
+ /* Make objects we just wrote available to ourselves */
1517
+ reprepare_packed_git(repo);
1518
+}
1519
+
1520
+/*
1521
+ * This writes the specified object to a packfile. Objects written here
1522
+ * during the same transaction are written to the same packfile. The
1523
+ * packfile is not flushed until the transaction is flushed. The caller
1524
+ * is expected to ensure a valid transaction is setup for objects to be
1525
+ * recorded to.
1526
+ *
1527
+ * This also bypasses the usual "convert-to-git" dance, and that is on
1528
+ * purpose. We could write a streaming version of the converting
1529
+ * functions and insert that before feeding the data to fast-import
1530
+ * (or equivalent in-core API described above). However, that is
1531
+ * somewhat complicated, as we do not know the size of the filter
1532
+ * result, which we need to know beforehand when writing a git object.
1533
+ * Since the primary motivation for trying to stream from the working
1534
+ * tree file and to avoid mmaping it in core is to deal with large
1535
+ * binary blobs, they generally do not want to get any conversion, and
1536
+ * callers should avoid this code path when filters are requested.
1537
+ */
1538
+static int index_blob_bulk_checkin(struct odb_transaction *transaction,
1539
+ struct object_id *result_oid, int fd, size_t size,
1540
+ const char *path, unsigned flags)
1541
+{
1542
+ struct bulk_checkin_packfile *state = &transaction->packfile;
1543
+ off_t seekback, already_hashed_to;
1544
+ struct git_hash_ctx ctx;
1545
+ unsigned char obuf[16384];
1546
+ unsigned header_len;
1547
+ struct hashfile_checkpoint checkpoint;
1548
+ struct pack_idx_entry *idx = NULL;
1549
+
1550
+ seekback = lseek(fd, 0, SEEK_CUR);
1551
+ if (seekback == (off_t)-1)
1552
+ return error("cannot find the current offset");
1553
+
1554
+ header_len = format_object_header((char *)obuf, sizeof(obuf),
1555
+ OBJ_BLOB, size);
1556
+ transaction->odb->repo->hash_algo->init_fn(&ctx);
1557
+ git_hash_update(&ctx, obuf, header_len);
1558
+
1559
+ /* Note: idx is non-NULL when we are writing */
1560
+ if ((flags & INDEX_WRITE_OBJECT) != 0) {
1561
+ CALLOC_ARRAY(idx, 1);
1562
+
1563
+ prepare_to_stream(transaction, flags);
1564
+ hashfile_checkpoint_init(state->f, &checkpoint);
1565
+ }
1566
+
1567
+ already_hashed_to = 0;
1568
+
1569
+ while (1) {
1570
+ prepare_to_stream(transaction, flags);
1571
+ if (idx) {
1572
+ hashfile_checkpoint(state->f, &checkpoint);
1573
+ idx->offset = state->offset;
1574
+ crc32_begin(state->f);
1575
+ }
1576
+ if (!stream_blob_to_pack(state, &ctx, &already_hashed_to,
1577
+ fd, size, path, flags))
1578
+ break;
1579
+ /*
1580
+ * Writing this object to the current pack will make
1581
+ * it too big; we need to truncate it, start a new
1582
+ * pack, and write into it.
1583
+ */
1584
+ if (!idx)
1585
+ BUG("should not happen");
1586
+ hashfile_truncate(state->f, &checkpoint);
1587
+ state->offset = checkpoint.offset;
1588
+ flush_bulk_checkin_packfile(transaction);
1589
+ if (lseek(fd, seekback, SEEK_SET) == (off_t)-1)
1590
+ return error("cannot seek back");
1591
+ }
1592
+ git_hash_final_oid(result_oid, &ctx);
1593
+ if (!idx)
1594
+ return 0;
1595
+
1596
+ idx->crc32 = crc32_end(state->f);
1597
+ if (already_written(transaction, result_oid)) {
1598
+ hashfile_truncate(state->f, &checkpoint);
1599
+ state->offset = checkpoint.offset;
1600
+ free(idx);
1601
+ } else {
1602
+ oidcpy(&idx->oid, result_oid);
1603
+ ALLOC_GROW(state->written,
1604
+ state->nr_written + 1,
1605
+ state->alloc_written);
1606
+ state->written[state->nr_written++] = idx;
1607
+ }
1608
+ return 0;
1609
+}
1610
+
1611
int index_fd(struct index_state *istate, struct object_id *oid,
1612
int fd, struct stat *st,
1613
enum object_type type, const char *path, unsigned flags)
@@ -1609,3 +1974,30 @@ out:
1974
munmap(map, mapsize);
1975
return ret;
1976
}
1977
+
1978
+struct odb_transaction *begin_odb_transaction(struct object_database *odb)
1979
+{
1980
+ if (odb->transaction)
1981
+ return NULL;
1982
+
1983
+ CALLOC_ARRAY(odb->transaction, 1);
1984
+ odb->transaction->odb = odb;
1985
+
1986
+ return odb->transaction;
1987
+}
1988
+
1989
+void end_odb_transaction(struct odb_transaction *transaction)
1990
+{
1991
+ if (!transaction)
1992
+ return;
1993
+
1994
+ /*
1995
+ * Ensure the transaction ending matches the pending transaction.
1996
+ */
1997
+ ASSERT(transaction == transaction->odb->transaction);
1998
+
1999
+ flush_batch_fsync(transaction);
2000
+ flush_bulk_checkin_packfile(transaction);
2001
+ transaction->odb->transaction = NULL;
2002
+ free(transaction);
2003
+}
object-file.h
+16
@@ -218,4 +218,20 @@ int read_loose_object(struct repository *repo,
218
void **contents,
219
struct object_info *oi);
220
221
+struct odb_transaction;
222
+
223
+/*
224
+ * Tell the object database to optimize for adding
225
+ * multiple objects. end_odb_transaction must be called
226
+ * to make new objects visible. If a transaction is already
227
+ * pending, NULL is returned.
228
+ */
229
+struct odb_transaction *begin_odb_transaction(struct object_database *odb);
230
+
231
+/*
232
+ * Tell the object database to make any objects from the
233
+ * current transaction visible.
234
+ */
235
+void end_odb_transaction(struct odb_transaction *transaction);
236
+
237
#endif /* OBJECT_FILE_H */
read-cache.c
-1
@@ -8,7 +8,6 @@
8
#define DISABLE_SIGN_COMPARE_WARNINGS
9
10
#include "git-compat-util.h"
11
-#include "bulk-checkin.h"
11
#include "config.h"
12
#include "date.h"
13
#include "diff.h"