object-file.c: refactor write_loose_object() to several steps

When writing a large blob using "write_loose_object()", we have to pass a buffer with the whole content of the blob, and this behavior will consume lots of memory and may cause OOM. We will introduce a stream version function ("stream_loose_object()") in later commit to resolve this issue. Before introducing that streaming function, do some refactoring on "write_loose_object()" to reuse code for both versions. Rewrite "write_loose_object()" as follows: 1. Figure out a path for the (temp) object file. This step is only used in "write_loose_object()". 2. Move common steps for starting to write loose objects into a new function "start_loose_object_common()". 3. Compress data. 4. Move common steps for ending zlib stream into a new function "end_loose_object_common()". 5. Close fd and finalize the object file. Helped-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com> Helped-by: Jiang Xin <zhiyou.jx@alibaba-inc.com> Signed-off-by: Han Xin <chiyutianyi@gmail.com> Signed-off-by: Ævar Arnfjörð Bjarmason <avarab@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Han Xin committed Jun 11, 2022 at 10:44 UTC 97a9db6ffb13a49dfe0565b65275b935f4e4d620
1 file changed +74 -24
object-file.c
+74 -24
@@ -1951,6 +1951,74 @@ static int create_tmpfile(struct strbuf *tmp, const char *filename)
1951 return fd;
1952 }
1953
1954 +/**
1955 + * Common steps for loose object writers to start writing loose
1956 + * objects:
1957 + *
1958 + * - Create tmpfile for the loose object.
1959 + * - Setup zlib stream for compression.
1960 + * - Start to feed header to zlib stream.
1961 + *
1962 + * Returns a "fd", which should later be provided to
1963 + * end_loose_object_common().
1964 + */
1965 +static int start_loose_object_common(struct strbuf *tmp_file,
1966 + const char *filename, unsigned flags,
1967 + git_zstream *stream,
1968 + unsigned char *buf, size_t buflen,
1969 + git_hash_ctx *c,
1970 + char *hdr, int hdrlen)
1971 +{
1972 + int fd;
1973 +
1974 + fd = create_tmpfile(tmp_file, filename);
1975 + if (fd < 0) {
1976 + if (flags & HASH_SILENT)
1977 + return -1;
1978 + else if (errno == EACCES)
1979 + return error(_("insufficient permission for adding "
1980 + "an object to repository database %s"),
1981 + get_object_directory());
1982 + else
1983 + return error_errno(
1984 + _("unable to create temporary file"));
1985 + }
1986 +
1987 + /* Setup zlib stream for compression */
1988 + git_deflate_init(stream, zlib_compression_level);
1989 + stream->next_out = buf;
1990 + stream->avail_out = buflen;
1991 + the_hash_algo->init_fn(c);
1992 +
1993 + /* Start to feed header to zlib stream */
1994 + stream->next_in = (unsigned char *)hdr;
1995 + stream->avail_in = hdrlen;
1996 + while (git_deflate(stream, 0) == Z_OK)
1997 + ; /* nothing */
1998 + the_hash_algo->update_fn(c, hdr, hdrlen);
1999 +
2000 + return fd;
2001 +}
2002 +
2003 +/**
2004 + * Common steps for loose object writers to end writing loose objects:
2005 + *
2006 + * - End the compression of zlib stream.
2007 + * - Get the calculated oid to "oid".
2008 + */
2009 +static int end_loose_object_common(git_hash_ctx *c, git_zstream *stream,
2010 + struct object_id *oid)
2011 +{
2012 + int ret;
2013 +
2014 + ret = git_deflate_end_gently(stream);
2015 + if (ret != Z_OK)
2016 + return ret;
2017 + the_hash_algo->final_oid_fn(oid, c);
2018 +
2019 + return Z_OK;
2020 +}
2021 +
2022 static int write_loose_object(const struct object_id *oid, char *hdr,
2023 int hdrlen, const void *buf, unsigned long len,
2024 time_t mtime, unsigned flags)
@@ -1968,28 +2036,11 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
2036
2037 loose_object_path(the_repository, &filename, oid);
2038
1971 - fd = create_tmpfile(&tmp_file, filename.buf);
1972 - if (fd < 0) {
1973 - if (flags & HASH_SILENT)
1974 - return -1;
1975 - else if (errno == EACCES)
1976 - return error(_("insufficient permission for adding an object to repository database %s"), get_object_directory());
1977 - else
1978 - return error_errno(_("unable to create temporary file"));
1979 - }
1980 -
1981 - /* Set it up */
1982 - git_deflate_init(&stream, zlib_compression_level);
1983 - stream.next_out = compressed;
1984 - stream.avail_out = sizeof(compressed);
1985 - the_hash_algo->init_fn(&c);
1986 -
1987 - /* First header.. */
1988 - stream.next_in = (unsigned char *)hdr;
1989 - stream.avail_in = hdrlen;
1990 - while (git_deflate(&stream, 0) == Z_OK)
1991 - ; /* nothing */
1992 - the_hash_algo->update_fn(&c, hdr, hdrlen);
2039 + fd = start_loose_object_common(&tmp_file, filename.buf, flags,
2040 + &stream, compressed, sizeof(compressed),
2041 + &c, hdr, hdrlen);
2042 + if (fd < 0)
2043 + return -1;
2044
2045 /* Then the data itself.. */
2046 stream.next_in = (void *)buf;
@@ -2007,11 +2058,10 @@ static int write_loose_object(const struct object_id *oid, char *hdr,
2058 if (ret != Z_STREAM_END)
2059 die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),
2060 ret);
2010 - ret = git_deflate_end_gently(&stream);
2061 + ret = end_loose_object_common(&c, &stream, &parano_oid);
2062 if (ret != Z_OK)
2063 die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),
2064 ret);
2014 - the_hash_algo->final_oid_fn(&parano_oid, &c);
2065 if (!oideq(oid, &parano_oid))
2066 die(_("confused by unstable object source data for %s"),
2067 oid_to_hex(oid));