object-file: move logic to write loose objects
The logic to write loose objects is split up across "object-file.c" and "odb/source-loose.c". This split is somewhat weird, but it is the result of two things: - `force_object_loose()` used to reach into internals of how exactly we write objects. - The logic of writing objects is intertwined with potentially starting a transaction. We have refactored `force_object_loose()` over preceding commits to work via generic interfaces now, so this reason doesn't exist anymore. But the second reason still does, as our management of "files" transactions and their ad-hoc creation is still very messy. This area definitely requires further work, and that work is indeed ongoing. That being said, we can already move the writing logic into the "loose" backend rather easily. All we have to do is to expose two functions that relate to the transactions. Expose these two functions and move the writing logic into the "loose" backend accordingly so that it becomes more self-contained. Note that this requires us to drop a reference to `the_repository` in favor of using the source's repository in `start_loose_object_common()`. Signed-off-by: Patrick Steinhardt <ps@pks.im> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Patrick Steinhardt committed
Jul 17, 2026 at 11:32 UTC
810f8b203303f27543537d3ceadc90b065ed9c8f
3 files changed
+357
-379
object-file.c
+3
-357
@@ -491,7 +491,7 @@ struct odb_transaction_files {
491
const char *prefix;
492
};
493
494
-static int odb_transaction_files_prepare(struct odb_transaction *base)
494
+int odb_transaction_files_prepare(struct odb_transaction *base)
495
{
496
struct odb_transaction_files *transaction =
497
container_of_or_null(base, struct odb_transaction_files, base);
@@ -514,8 +514,8 @@ static int odb_transaction_files_prepare(struct odb_transaction *base)
514
return 0;
515
}
516
517
-static void odb_transaction_files_fsync(struct odb_transaction *base,
518
- int fd, const char *filename)
517
+void odb_transaction_files_fsync(struct odb_transaction *base,
518
+ int fd, const char *filename)
519
{
520
struct odb_transaction_files *transaction =
521
container_of_or_null(base, struct odb_transaction_files, base);
@@ -539,360 +539,6 @@ static void odb_transaction_files_fsync(struct odb_transaction *base,
539
}
540
}
541
542
-/* Finalize a file on disk, and close it. */
543
-static void close_loose_object(struct odb_source_loose *loose,
544
- int fd, const char *filename)
545
-{
546
- if (loose->base.will_destroy)
547
- goto out;
548
-
549
- if (batch_fsync_enabled(FSYNC_COMPONENT_LOOSE_OBJECT))
550
- odb_transaction_files_fsync(loose->base.odb->transaction, fd, filename);
551
- else if (fsync_object_files > 0)
552
- fsync_or_die(fd, filename);
553
- else
554
- fsync_component_or_die(FSYNC_COMPONENT_LOOSE_OBJECT, fd,
555
- filename);
556
-
557
-out:
558
- if (close(fd) != 0)
559
- die_errno(_("error when closing loose object file"));
560
-}
561
-
562
-/* Size of directory component, including the ending '/' */
563
-static inline int directory_size(const char *filename)
564
-{
565
- const char *s = strrchr(filename, '/');
566
- if (!s)
567
- return 0;
568
- return s - filename + 1;
569
-}
570
-
571
-/*
572
- * This creates a temporary file in the same directory as the final
573
- * 'filename'
574
- *
575
- * We want to avoid cross-directory filename renames, because those
576
- * can have problems on various filesystems (FAT, NFS, Coda).
577
- */
578
-static int create_tmpfile(struct repository *repo,
579
- struct strbuf *tmp, const char *filename)
580
-{
581
- int fd, dirlen = directory_size(filename);
582
-
583
- strbuf_reset(tmp);
584
- strbuf_add(tmp, filename, dirlen);
585
- strbuf_addstr(tmp, "tmp_obj_XXXXXX");
586
- fd = git_mkstemp_mode(tmp->buf, 0444);
587
- if (fd < 0 && dirlen && errno == ENOENT) {
588
- /*
589
- * Make sure the directory exists; note that the contents
590
- * of the buffer are undefined after mkstemp returns an
591
- * error, so we have to rewrite the whole buffer from
592
- * scratch.
593
- */
594
- strbuf_reset(tmp);
595
- strbuf_add(tmp, filename, dirlen - 1);
596
- if (mkdir(tmp->buf, 0777) && errno != EEXIST)
597
- return -1;
598
- if (adjust_shared_perm(repo, tmp->buf))
599
- return -1;
600
-
601
- /* Try again */
602
- strbuf_addstr(tmp, "/tmp_obj_XXXXXX");
603
- fd = git_mkstemp_mode(tmp->buf, 0444);
604
- }
605
- return fd;
606
-}
607
-
608
-/**
609
- * Common steps for loose object writers to start writing loose
610
- * objects:
611
- *
612
- * - Create tmpfile for the loose object.
613
- * - Setup zlib stream for compression.
614
- * - Start to feed header to zlib stream.
615
- *
616
- * Returns a "fd", which should later be provided to
617
- * end_loose_object_common().
618
- */
619
-static int start_loose_object_common(struct odb_source_loose *loose,
620
- struct strbuf *tmp_file,
621
- const char *filename, unsigned flags,
622
- git_zstream *stream,
623
- unsigned char *buf, size_t buflen,
624
- struct git_hash_ctx *c, struct git_hash_ctx *compat_c,
625
- char *hdr, int hdrlen)
626
-{
627
- const struct git_hash_algo *algo = loose->base.odb->repo->hash_algo;
628
- const struct git_hash_algo *compat = loose->base.odb->repo->compat_hash_algo;
629
- int fd;
630
- struct repo_config_values *cfg = repo_config_values(the_repository);
631
-
632
- fd = create_tmpfile(loose->base.odb->repo, tmp_file, filename);
633
- if (fd < 0) {
634
- if (flags & ODB_WRITE_OBJECT_SILENT)
635
- return -1;
636
- else if (errno == EACCES)
637
- return error(_("insufficient permission for adding "
638
- "an object to repository database %s"),
639
- loose->base.path);
640
- else
641
- return error_errno(
642
- _("unable to create temporary file"));
643
- }
644
-
645
- /* Setup zlib stream for compression */
646
- git_deflate_init(stream, cfg->zlib_compression_level);
647
- stream->next_out = buf;
648
- stream->avail_out = buflen;
649
- git_hash_init(c, algo);
650
- if (compat && compat_c)
651
- git_hash_init(compat_c, compat);
652
-
653
- /* Start to feed header to zlib stream */
654
- stream->next_in = (unsigned char *)hdr;
655
- stream->avail_in = hdrlen;
656
- while (git_deflate(stream, 0) == Z_OK)
657
- ; /* nothing */
658
- git_hash_update(c, hdr, hdrlen);
659
- if (compat && compat_c)
660
- git_hash_update(compat_c, hdr, hdrlen);
661
-
662
- return fd;
663
-}
664
-
665
-/**
666
- * Common steps for the inner git_deflate() loop for writing loose
667
- * objects. Returns what git_deflate() returns.
668
- */
669
-static int write_loose_object_common(struct odb_source_loose *loose,
670
- struct git_hash_ctx *c, struct git_hash_ctx *compat_c,
671
- git_zstream *stream, const int flush,
672
- unsigned char *in0, const int fd,
673
- unsigned char *compressed,
674
- const size_t compressed_len)
675
-{
676
- const struct git_hash_algo *compat = loose->base.odb->repo->compat_hash_algo;
677
- int ret;
678
-
679
- ret = git_deflate(stream, flush ? Z_FINISH : 0);
680
- git_hash_update(c, in0, stream->next_in - in0);
681
- if (compat && compat_c)
682
- git_hash_update(compat_c, in0, stream->next_in - in0);
683
- if (write_in_full(fd, compressed, stream->next_out - compressed) < 0)
684
- die_errno(_("unable to write loose object file"));
685
- stream->next_out = compressed;
686
- stream->avail_out = compressed_len;
687
-
688
- return ret;
689
-}
690
-
691
-/**
692
- * Common steps for loose object writers to end writing loose objects:
693
- *
694
- * - End the compression of zlib stream.
695
- * - Get the calculated oid to "oid".
696
- */
697
-static int end_loose_object_common(struct odb_source_loose *loose,
698
- struct git_hash_ctx *c, struct git_hash_ctx *compat_c,
699
- git_zstream *stream, struct object_id *oid,
700
- struct object_id *compat_oid)
701
-{
702
- const struct git_hash_algo *compat = loose->base.odb->repo->compat_hash_algo;
703
- int ret;
704
-
705
- ret = git_deflate_end_gently(stream);
706
- if (ret != Z_OK)
707
- return ret;
708
- git_hash_final_oid(oid, c);
709
- if (compat && compat_c)
710
- git_hash_final_oid(compat_oid, compat_c);
711
-
712
- return Z_OK;
713
-}
714
-
715
-int write_loose_object(struct odb_source_loose *loose,
716
- const struct object_id *oid, char *hdr,
717
- int hdrlen, const void *buf, unsigned long len,
718
- const time_t *mtime, unsigned flags)
719
-{
720
- int fd, ret;
721
- unsigned char compressed[4096];
722
- git_zstream stream;
723
- struct git_hash_ctx c;
724
- struct object_id parano_oid;
725
- static struct strbuf tmp_file = STRBUF_INIT;
726
- static struct strbuf filename = STRBUF_INIT;
727
-
728
- if (batch_fsync_enabled(FSYNC_COMPONENT_LOOSE_OBJECT))
729
- odb_transaction_files_prepare(loose->base.odb->transaction);
730
-
731
- odb_loose_path(loose, &filename, oid);
732
-
733
- fd = start_loose_object_common(loose, &tmp_file, filename.buf, flags,
734
- &stream, compressed, sizeof(compressed),
735
- &c, NULL, hdr, hdrlen);
736
- if (fd < 0)
737
- return -1;
738
-
739
- /* Then the data itself.. */
740
- stream.next_in = (void *)buf;
741
- stream.avail_in = len;
742
- do {
743
- unsigned char *in0 = stream.next_in;
744
-
745
- ret = write_loose_object_common(loose, &c, NULL, &stream, 1, in0, fd,
746
- compressed, sizeof(compressed));
747
- } while (ret == Z_OK);
748
-
749
- if (ret != Z_STREAM_END)
750
- die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
751
- ret);
752
- ret = end_loose_object_common(loose, &c, NULL, &stream, ¶no_oid, NULL);
753
- if (ret != Z_OK)
754
- die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
755
- ret);
756
- if (!oideq(oid, ¶no_oid))
757
- die(_("confused by unstable object source data for %s"),
758
- oid_to_hex(oid));
759
-
760
- close_loose_object(loose, fd, tmp_file.buf);
761
-
762
- if (mtime) {
763
- struct utimbuf utb = {
764
- .actime = *mtime,
765
- .modtime = *mtime,
766
- };
767
-
768
- if (utime(tmp_file.buf, &utb) < 0 &&
769
- !(flags & ODB_WRITE_OBJECT_SILENT))
770
- warning_errno(_("failed utime() on %s"), tmp_file.buf);
771
- }
772
-
773
- return finalize_object_file_flags(loose->base.odb->repo, tmp_file.buf, filename.buf,
774
- FOF_SKIP_COLLISION_CHECK);
775
-}
776
-
777
-int odb_source_loose_write_stream(struct odb_source_loose *loose,
778
- struct odb_write_stream *in_stream, size_t len,
779
- struct object_id *oid)
780
-{
781
- const struct git_hash_algo *compat = loose->base.odb->repo->compat_hash_algo;
782
- struct object_id compat_oid;
783
- int fd, ret, err = 0, flush = 0;
784
- unsigned char compressed[4096];
785
- git_zstream stream;
786
- struct git_hash_ctx c, compat_c;
787
- struct strbuf tmp_file = STRBUF_INIT;
788
- struct strbuf filename = STRBUF_INIT;
789
- unsigned char buf[8192];
790
- int dirlen;
791
- char hdr[MAX_HEADER_LEN];
792
- int hdrlen;
793
-
794
- if (batch_fsync_enabled(FSYNC_COMPONENT_LOOSE_OBJECT))
795
- odb_transaction_files_prepare(loose->base.odb->transaction);
796
-
797
- /* Since oid is not determined, save tmp file to odb path. */
798
- strbuf_addf(&filename, "%s/", loose->base.path);
799
- hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, len);
800
-
801
- /*
802
- * Common steps for write_loose_object and stream_loose_object to
803
- * start writing loose objects:
804
- *
805
- * - Create tmpfile for the loose object.
806
- * - Setup zlib stream for compression.
807
- * - Start to feed header to zlib stream.
808
- */
809
- fd = start_loose_object_common(loose, &tmp_file, filename.buf, 0,
810
- &stream, compressed, sizeof(compressed),
811
- &c, &compat_c, hdr, hdrlen);
812
- if (fd < 0) {
813
- err = -1;
814
- goto cleanup;
815
- }
816
-
817
- /* Then the data itself.. */
818
- do {
819
- unsigned char *in0 = stream.next_in;
820
-
821
- if (!stream.avail_in && !in_stream->is_finished) {
822
- ssize_t read_len = odb_write_stream_read(in_stream, buf,
823
- sizeof(buf));
824
- if (read_len < 0) {
825
- close(fd);
826
- err = -1;
827
- goto cleanup;
828
- }
829
-
830
- stream.avail_in = read_len;
831
- stream.next_in = buf;
832
- in0 = buf;
833
- /* All data has been read. */
834
- if (in_stream->is_finished)
835
- flush = 1;
836
- }
837
- ret = write_loose_object_common(loose, &c, &compat_c, &stream, flush, in0, fd,
838
- compressed, sizeof(compressed));
839
- /*
840
- * Unlike write_loose_object(), we do not have the entire
841
- * buffer. If we get Z_BUF_ERROR due to too few input bytes,
842
- * then we'll replenish them in the next input_stream->read()
843
- * call when we loop.
844
- */
845
- } while (ret == Z_OK || ret == Z_BUF_ERROR);
846
-
847
- if (stream.total_in != len + hdrlen)
848
- die(_("write stream object %"PRIuMAX" != %"PRIuMAX), (uintmax_t)stream.total_in,
849
- (uintmax_t)len + hdrlen);
850
-
851
- /*
852
- * Common steps for write_loose_object and stream_loose_object to
853
- * end writing loose object:
854
- *
855
- * - End the compression of zlib stream.
856
- * - Get the calculated oid.
857
- */
858
- if (ret != Z_STREAM_END)
859
- die(_("unable to stream deflate new object (%d)"), ret);
860
- ret = end_loose_object_common(loose, &c, &compat_c, &stream, oid, &compat_oid);
861
- if (ret != Z_OK)
862
- die(_("deflateEnd on stream object failed (%d)"), ret);
863
- close_loose_object(loose, fd, tmp_file.buf);
864
-
865
- if (odb_freshen_object(loose->base.odb, oid)) {
866
- unlink_or_warn(tmp_file.buf);
867
- goto cleanup;
868
- }
869
- odb_loose_path(loose, &filename, oid);
870
-
871
- /* We finally know the object path, and create the missing dir. */
872
- dirlen = directory_size(filename.buf);
873
- if (dirlen) {
874
- struct strbuf dir = STRBUF_INIT;
875
- strbuf_add(&dir, filename.buf, dirlen);
876
-
877
- if (safe_create_dir_in_gitdir(loose->base.odb->repo, dir.buf) &&
878
- errno != EEXIST) {
879
- err = error_errno(_("unable to create directory %s"), dir.buf);
880
- strbuf_release(&dir);
881
- goto cleanup;
882
- }
883
- strbuf_release(&dir);
884
- }
885
-
886
- err = finalize_object_file_flags(loose->base.odb->repo, tmp_file.buf, filename.buf,
887
- FOF_SKIP_COLLISION_CHECK);
888
- if (!err && compat)
889
- err = repo_add_loose_object_map(loose, oid, &compat_oid);
890
-cleanup:
891
- strbuf_release(&tmp_file);
892
- strbuf_release(&filename);
893
- return err;
894
-}
895
-
542
/*
543
* We can't use the normal fsck_error_function() for index_mem(),
544
* because we don't yet have a valid oid for it to report. Instead,
object-file.h
+4
-18
@@ -24,20 +24,6 @@ int index_path(struct index_state *istate, struct object_id *oid, const char *pa
24
struct object_info;
25
struct odb_source;
26
27
-/*
28
- * Write the given stream into the loose object source. The only difference
29
- * from the generic implementation of this function is that we don't perform an
30
- * object existence check here.
31
- *
32
- * TODO: We should stop exposing this function altogether and move it into
33
- * "odb/source-loose.c". This requires a couple of refactorings though to make
34
- * `force_object_loose()` generic and is thus postponed to a later point in
35
- * time.
36
- */
37
-int odb_source_loose_write_stream(struct odb_source_loose *source,
38
- struct odb_write_stream *stream, size_t len,
39
- struct object_id *oid);
40
-
27
/*
28
* Put in `buf` the name of the file in the local object database that
29
* would be used to store a loose object with the specified oid.
@@ -131,10 +117,6 @@ int finalize_object_file_flags(struct repository *repo,
117
void hash_object_file(const struct git_hash_algo *algo, const void *buf,
118
size_t len, enum object_type type,
119
struct object_id *oid);
134
-int write_loose_object(struct odb_source_loose *loose,
135
- const struct object_id *oid, char *hdr,
136
- int hdrlen, const void *buf, unsigned long len,
137
- const time_t *mtime, unsigned flags);
120
121
/* Helper to check and "touch" a file */
122
int check_and_freshen_file(const char *fn, int freshen,
@@ -195,4 +177,8 @@ int odb_transaction_files_begin(struct odb_source *source,
177
struct odb_transaction **out,
178
enum odb_transaction_flags flags);
179
180
+int odb_transaction_files_prepare(struct odb_transaction *base);
181
+void odb_transaction_files_fsync(struct odb_transaction *base,
182
+ int fd, const char *filename);
183
+
184
#endif /* OBJECT_FILE_H */
odb/source-loose.c
+350
-4
@@ -11,8 +11,11 @@
11
#include "odb/source-loose.h"
12
#include "odb/streaming.h"
13
#include "oidtree.h"
14
+#include "path.h"
15
#include "repository.h"
16
#include "strbuf.h"
17
+#include "tempfile.h"
18
+#include "write-or-die.h"
19
20
static int append_loose_object(const struct object_id *oid,
21
const char *path UNUSED,
@@ -583,6 +586,241 @@ static int odb_source_loose_freshen_object(struct odb_source *source,
586
return !!check_and_freshen_file(path.buf, 1, mtime);
587
}
588
589
+/* Finalize a file on disk, and close it. */
590
+static void close_loose_object(struct odb_source_loose *loose,
591
+ int fd, const char *filename)
592
+{
593
+ if (loose->base.will_destroy)
594
+ goto out;
595
+
596
+ if (batch_fsync_enabled(FSYNC_COMPONENT_LOOSE_OBJECT))
597
+ odb_transaction_files_fsync(loose->base.odb->transaction, fd, filename);
598
+ else if (fsync_object_files > 0)
599
+ fsync_or_die(fd, filename);
600
+ else
601
+ fsync_component_or_die(FSYNC_COMPONENT_LOOSE_OBJECT, fd,
602
+ filename);
603
+
604
+out:
605
+ if (close(fd) != 0)
606
+ die_errno(_("error when closing loose object file"));
607
+}
608
+
609
+/* Size of directory component, including the ending '/' */
610
+static inline int directory_size(const char *filename)
611
+{
612
+ const char *s = strrchr(filename, '/');
613
+ if (!s)
614
+ return 0;
615
+ return s - filename + 1;
616
+}
617
+
618
+/*
619
+ * This creates a temporary file in the same directory as the final
620
+ * 'filename'
621
+ *
622
+ * We want to avoid cross-directory filename renames, because those
623
+ * can have problems on various filesystems (FAT, NFS, Coda).
624
+ */
625
+static int create_tmpfile(struct repository *repo,
626
+ struct strbuf *tmp, const char *filename)
627
+{
628
+ int fd, dirlen = directory_size(filename);
629
+
630
+ strbuf_reset(tmp);
631
+ strbuf_add(tmp, filename, dirlen);
632
+ strbuf_addstr(tmp, "tmp_obj_XXXXXX");
633
+ fd = git_mkstemp_mode(tmp->buf, 0444);
634
+ if (fd < 0 && dirlen && errno == ENOENT) {
635
+ /*
636
+ * Make sure the directory exists; note that the contents
637
+ * of the buffer are undefined after mkstemp returns an
638
+ * error, so we have to rewrite the whole buffer from
639
+ * scratch.
640
+ */
641
+ strbuf_reset(tmp);
642
+ strbuf_add(tmp, filename, dirlen - 1);
643
+ if (mkdir(tmp->buf, 0777) && errno != EEXIST)
644
+ return -1;
645
+ if (adjust_shared_perm(repo, tmp->buf))
646
+ return -1;
647
+
648
+ /* Try again */
649
+ strbuf_addstr(tmp, "/tmp_obj_XXXXXX");
650
+ fd = git_mkstemp_mode(tmp->buf, 0444);
651
+ }
652
+ return fd;
653
+}
654
+
655
+/**
656
+ * Common steps for loose object writers to start writing loose
657
+ * objects:
658
+ *
659
+ * - Create tmpfile for the loose object.
660
+ * - Setup zlib stream for compression.
661
+ * - Start to feed header to zlib stream.
662
+ *
663
+ * Returns a "fd", which should later be provided to
664
+ * end_loose_object_common().
665
+ */
666
+static int start_loose_object_common(struct odb_source_loose *loose,
667
+ struct strbuf *tmp_file,
668
+ const char *filename, unsigned flags,
669
+ git_zstream *stream,
670
+ unsigned char *buf, size_t buflen,
671
+ struct git_hash_ctx *c, struct git_hash_ctx *compat_c,
672
+ char *hdr, int hdrlen)
673
+{
674
+ const struct git_hash_algo *algo = loose->base.odb->repo->hash_algo;
675
+ const struct git_hash_algo *compat = loose->base.odb->repo->compat_hash_algo;
676
+ int fd;
677
+ struct repo_config_values *cfg = repo_config_values(loose->base.odb->repo);
678
+
679
+ fd = create_tmpfile(loose->base.odb->repo, tmp_file, filename);
680
+ if (fd < 0) {
681
+ if (flags & ODB_WRITE_OBJECT_SILENT)
682
+ return -1;
683
+ else if (errno == EACCES)
684
+ return error(_("insufficient permission for adding "
685
+ "an object to repository database %s"),
686
+ loose->base.path);
687
+ else
688
+ return error_errno(
689
+ _("unable to create temporary file"));
690
+ }
691
+
692
+ /* Setup zlib stream for compression */
693
+ git_deflate_init(stream, cfg->zlib_compression_level);
694
+ stream->next_out = buf;
695
+ stream->avail_out = buflen;
696
+ git_hash_init(c, algo);
697
+ if (compat && compat_c)
698
+ git_hash_init(compat_c, compat);
699
+
700
+ /* Start to feed header to zlib stream */
701
+ stream->next_in = (unsigned char *)hdr;
702
+ stream->avail_in = hdrlen;
703
+ while (git_deflate(stream, 0) == Z_OK)
704
+ ; /* nothing */
705
+ git_hash_update(c, hdr, hdrlen);
706
+ if (compat && compat_c)
707
+ git_hash_update(compat_c, hdr, hdrlen);
708
+
709
+ return fd;
710
+}
711
+
712
+/**
713
+ * Common steps for the inner git_deflate() loop for writing loose
714
+ * objects. Returns what git_deflate() returns.
715
+ */
716
+static int write_loose_object_common(struct odb_source_loose *loose,
717
+ struct git_hash_ctx *c, struct git_hash_ctx *compat_c,
718
+ git_zstream *stream, const int flush,
719
+ unsigned char *in0, const int fd,
720
+ unsigned char *compressed,
721
+ const size_t compressed_len)
722
+{
723
+ const struct git_hash_algo *compat = loose->base.odb->repo->compat_hash_algo;
724
+ int ret;
725
+
726
+ ret = git_deflate(stream, flush ? Z_FINISH : 0);
727
+ git_hash_update(c, in0, stream->next_in - in0);
728
+ if (compat && compat_c)
729
+ git_hash_update(compat_c, in0, stream->next_in - in0);
730
+ if (write_in_full(fd, compressed, stream->next_out - compressed) < 0)
731
+ die_errno(_("unable to write loose object file"));
732
+ stream->next_out = compressed;
733
+ stream->avail_out = compressed_len;
734
+
735
+ return ret;
736
+}
737
+
738
+/**
739
+ * Common steps for loose object writers to end writing loose objects:
740
+ *
741
+ * - End the compression of zlib stream.
742
+ * - Get the calculated oid to "oid".
743
+ */
744
+static int end_loose_object_common(struct odb_source_loose *loose,
745
+ struct git_hash_ctx *c, struct git_hash_ctx *compat_c,
746
+ git_zstream *stream, struct object_id *oid,
747
+ struct object_id *compat_oid)
748
+{
749
+ const struct git_hash_algo *compat = loose->base.odb->repo->compat_hash_algo;
750
+ int ret;
751
+
752
+ ret = git_deflate_end_gently(stream);
753
+ if (ret != Z_OK)
754
+ return ret;
755
+ git_hash_final_oid(oid, c);
756
+ if (compat && compat_c)
757
+ git_hash_final_oid(compat_oid, compat_c);
758
+
759
+ return Z_OK;
760
+}
761
+
762
+static int write_loose_object(struct odb_source_loose *loose,
763
+ const struct object_id *oid, char *hdr,
764
+ int hdrlen, const void *buf, unsigned long len,
765
+ const time_t *mtime, unsigned flags)
766
+{
767
+ int fd, ret;
768
+ unsigned char compressed[4096];
769
+ git_zstream stream;
770
+ struct git_hash_ctx c;
771
+ struct object_id parano_oid;
772
+ static struct strbuf tmp_file = STRBUF_INIT;
773
+ static struct strbuf filename = STRBUF_INIT;
774
+
775
+ if (batch_fsync_enabled(FSYNC_COMPONENT_LOOSE_OBJECT))
776
+ odb_transaction_files_prepare(loose->base.odb->transaction);
777
+
778
+ odb_loose_path(loose, &filename, oid);
779
+
780
+ fd = start_loose_object_common(loose, &tmp_file, filename.buf, flags,
781
+ &stream, compressed, sizeof(compressed),
782
+ &c, NULL, hdr, hdrlen);
783
+ if (fd < 0)
784
+ return -1;
785
+
786
+ /* Then the data itself.. */
787
+ stream.next_in = (void *)buf;
788
+ stream.avail_in = len;
789
+ do {
790
+ unsigned char *in0 = stream.next_in;
791
+
792
+ ret = write_loose_object_common(loose, &c, NULL, &stream, 1, in0, fd,
793
+ compressed, sizeof(compressed));
794
+ } while (ret == Z_OK);
795
+
796
+ if (ret != Z_STREAM_END)
797
+ die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
798
+ ret);
799
+ ret = end_loose_object_common(loose, &c, NULL, &stream, ¶no_oid, NULL);
800
+ if (ret != Z_OK)
801
+ die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
802
+ ret);
803
+ if (!oideq(oid, ¶no_oid))
804
+ die(_("confused by unstable object source data for %s"),
805
+ oid_to_hex(oid));
806
+
807
+ close_loose_object(loose, fd, tmp_file.buf);
808
+
809
+ if (mtime) {
810
+ struct utimbuf utb = {
811
+ .actime = *mtime,
812
+ .modtime = *mtime,
813
+ };
814
+
815
+ if (utime(tmp_file.buf, &utb) < 0 &&
816
+ !(flags & ODB_WRITE_OBJECT_SILENT))
817
+ warning_errno(_("failed utime() on %s"), tmp_file.buf);
818
+ }
819
+
820
+ return finalize_object_file_flags(loose->base.odb->repo, tmp_file.buf, filename.buf,
821
+ FOF_SKIP_COLLISION_CHECK);
822
+}
823
+
824
static int odb_source_loose_write_object(struct odb_source *source,
825
const void *buf, size_t len,
826
enum object_type type,
@@ -611,12 +849,120 @@ static int odb_source_loose_write_object_stream(struct odb_source *source,
849
size_t len,
850
struct object_id *oid)
851
{
852
+ struct odb_source_loose *loose = odb_source_loose_downcast(source);
853
+ const struct git_hash_algo *compat = loose->base.odb->repo->compat_hash_algo;
854
+ struct object_id compat_oid;
855
+ int fd, ret, err = 0, flush = 0;
856
+ unsigned char compressed[4096];
857
+ git_zstream stream;
858
+ struct git_hash_ctx c, compat_c;
859
+ struct strbuf tmp_file = STRBUF_INIT;
860
+ struct strbuf filename = STRBUF_INIT;
861
+ unsigned char buf[8192];
862
+ int dirlen;
863
+ char hdr[MAX_HEADER_LEN];
864
+ int hdrlen;
865
+
866
+ if (batch_fsync_enabled(FSYNC_COMPONENT_LOOSE_OBJECT))
867
+ odb_transaction_files_prepare(loose->base.odb->transaction);
868
+
869
+ /* Since oid is not determined, save tmp file to odb path. */
870
+ strbuf_addf(&filename, "%s/", loose->base.path);
871
+ hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, len);
872
+
873
/*
615
- * TODO: the implementation should be moved here, see the comment on
616
- * the called function in "object-file.h".
874
+ * Common steps for write_loose_object and stream_loose_object to
875
+ * start writing loose objects:
876
+ *
877
+ * - Create tmpfile for the loose object.
878
+ * - Setup zlib stream for compression.
879
+ * - Start to feed header to zlib stream.
880
*/
618
- struct odb_source_loose *loose = odb_source_loose_downcast(source);
619
- return odb_source_loose_write_stream(loose, in_stream, len, oid);
881
+ fd = start_loose_object_common(loose, &tmp_file, filename.buf, 0,
882
+ &stream, compressed, sizeof(compressed),
883
+ &c, &compat_c, hdr, hdrlen);
884
+ if (fd < 0) {
885
+ err = -1;
886
+ goto cleanup;
887
+ }
888
+
889
+ /* Then the data itself.. */
890
+ do {
891
+ unsigned char *in0 = stream.next_in;
892
+
893
+ if (!stream.avail_in && !in_stream->is_finished) {
894
+ ssize_t read_len = odb_write_stream_read(in_stream, buf,
895
+ sizeof(buf));
896
+ if (read_len < 0) {
897
+ close(fd);
898
+ err = -1;
899
+ goto cleanup;
900
+ }
901
+
902
+ stream.avail_in = read_len;
903
+ stream.next_in = buf;
904
+ in0 = buf;
905
+ /* All data has been read. */
906
+ if (in_stream->is_finished)
907
+ flush = 1;
908
+ }
909
+ ret = write_loose_object_common(loose, &c, &compat_c, &stream, flush, in0, fd,
910
+ compressed, sizeof(compressed));
911
+ /*
912
+ * Unlike write_loose_object(), we do not have the entire
913
+ * buffer. If we get Z_BUF_ERROR due to too few input bytes,
914
+ * then we'll replenish them in the next input_stream->read()
915
+ * call when we loop.
916
+ */
917
+ } while (ret == Z_OK || ret == Z_BUF_ERROR);
918
+
919
+ if (stream.total_in != len + hdrlen)
920
+ die(_("write stream object %"PRIuMAX" != %"PRIuMAX), (uintmax_t)stream.total_in,
921
+ (uintmax_t)len + hdrlen);
922
+
923
+ /*
924
+ * Common steps for write_loose_object and stream_loose_object to
925
+ * end writing loose object:
926
+ *
927
+ * - End the compression of zlib stream.
928
+ * - Get the calculated oid.
929
+ */
930
+ if (ret != Z_STREAM_END)
931
+ die(_("unable to stream deflate new object (%d)"), ret);
932
+ ret = end_loose_object_common(loose, &c, &compat_c, &stream, oid, &compat_oid);
933
+ if (ret != Z_OK)
934
+ die(_("deflateEnd on stream object failed (%d)"), ret);
935
+ close_loose_object(loose, fd, tmp_file.buf);
936
+
937
+ if (odb_freshen_object(loose->base.odb, oid)) {
938
+ unlink_or_warn(tmp_file.buf);
939
+ goto cleanup;
940
+ }
941
+ odb_loose_path(loose, &filename, oid);
942
+
943
+ /* We finally know the object path, and create the missing dir. */
944
+ dirlen = directory_size(filename.buf);
945
+ if (dirlen) {
946
+ struct strbuf dir = STRBUF_INIT;
947
+ strbuf_add(&dir, filename.buf, dirlen);
948
+
949
+ if (safe_create_dir_in_gitdir(loose->base.odb->repo, dir.buf) &&
950
+ errno != EEXIST) {
951
+ err = error_errno(_("unable to create directory %s"), dir.buf);
952
+ strbuf_release(&dir);
953
+ goto cleanup;
954
+ }
955
+ strbuf_release(&dir);
956
+ }
957
+
958
+ err = finalize_object_file_flags(loose->base.odb->repo, tmp_file.buf, filename.buf,
959
+ FOF_SKIP_COLLISION_CHECK);
960
+ if (!err && compat)
961
+ err = repo_add_loose_object_map(loose, oid, &compat_oid);
962
+cleanup:
963
+ strbuf_release(&tmp_file);
964
+ strbuf_release(&filename);
965
+ return err;
966
}
967
968
static int odb_source_loose_begin_transaction(struct odb_source *source UNUSED,