@matheusbsbs / matheus-bagliano / commits / 63a6745a07

object-file: update the loose object map when writing loose objects

To implement SHA1 compatibility on SHA256 repositories the loose object map needs to be updated whenver a loose object is written. Updating the loose object map this way allows git to support the old hash algorithm in constant time. The functions write_loose_object, and stream_loose_object are the only two functions that write to the loose object store. Update stream_loose_object to compute the compatibiilty hash, update the loose object, and then call repo_add_loose_object_map to update the loose object map. Update write_object_file_flags to convert the object into it's compatibility encoding, hash the compatibility encoding, write the object, and then update the loose object map. Update force_object_loose to lookup the hash of the compatibility encoding, write the loose object, and then update the loose object map. Update write_object_file_literally to convert the object into it's compatibility hash encoding, hash the compatibility enconding, write the object, and then update the loose object map, when the type string is a known type. For objects with an unknown type this results in a partially broken repository, as the objects are not mapped. The point of write_object_file_literally is to generate a partially broken repository for testing. For testing skipping writing the loose object map is much more useful than refusing to write the broken object at all. Except that the loose objects are updated before the loose object map I have not done any analysis to see how robust this scheme is in the event of failure. Signed-off-by: "Eric W. Biederman" <ebiederm@xmission.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Eric W. Biederman committed Oct 1, 2023 at 21:40 UTC 63a6745a07c2202d16d580156661d42e00b9762f

1 file changed +95 -18

object-file.c

+95 -18

@@ -43,6 +43,8 @@

43 #include "setup.h"

44 #include "submodule.h"

45 #include "fsck.h"

46 +#include "loose.h"

47 +#include "object-file-convert.h"

49 /* The maximum size for an object header. */

50 #define MAX_HEADER_LEN 32

@@ -1952,9 +1954,12 @@ static int start_loose_object_common(struct strbuf *tmp_file,

1954 const char *filename, unsigned flags,

1955 git_zstream *stream,

1956 unsigned char *buf, size_t buflen,

1955 - git_hash_ctx *c,

1957 + git_hash_ctx *c, git_hash_ctx *compat_c,

1958 char *hdr, int hdrlen)

1959 {

1960 + struct repository *repo = the_repository;

1961 + const struct git_hash_algo *algo = repo->hash_algo;

1962 + const struct git_hash_algo *compat = repo->compat_hash_algo;

1963 int fd;

1964

1965 fd = create_tmpfile(tmp_file, filename);

@@ -1974,14 +1979,18 @@ static int start_loose_object_common(struct strbuf *tmp_file,

1979 git_deflate_init(stream, zlib_compression_level);

1980 stream->next_out = buf;

1981 stream->avail_out = buflen;

1977 - the_hash_algo->init_fn(c);

1982 + algo->init_fn(c);

1983 + if (compat && compat_c)

1984 + compat->init_fn(compat_c);

1985

1986 /* Start to feed header to zlib stream */

1987 stream->next_in = (unsigned char *)hdr;

1988 stream->avail_in = hdrlen;

1989 while (git_deflate(stream, 0) == Z_OK)

1990 ; /* nothing */

1984 - the_hash_algo->update_fn(c, hdr, hdrlen);

1991 + algo->update_fn(c, hdr, hdrlen);

1992 + if (compat && compat_c)

1993 + compat->update_fn(compat_c, hdr, hdrlen);

1994

1995 return fd;

1996 }

@@ -1990,16 +1999,21 @@ static int start_loose_object_common(struct strbuf *tmp_file,

1999 * Common steps for the inner git_deflate() loop for writing loose

2000 * objects. Returns what git_deflate() returns.

2001 */

1993 -static int write_loose_object_common(git_hash_ctx *c,

2002 +static int write_loose_object_common(git_hash_ctx *c, git_hash_ctx *compat_c,

2003 git_zstream *stream, const int flush,

2004 unsigned char *in0, const int fd,

2005 unsigned char *compressed,

2006 const size_t compressed_len)

2007 {

2008 + struct repository *repo = the_repository;

2009 + const struct git_hash_algo *algo = repo->hash_algo;

2010 + const struct git_hash_algo *compat = repo->compat_hash_algo;

2011 int ret;

2012

2013 ret = git_deflate(stream, flush ? Z_FINISH : 0);

2002 - the_hash_algo->update_fn(c, in0, stream->next_in - in0);

2014 + algo->update_fn(c, in0, stream->next_in - in0);

2015 + if (compat && compat_c)

2016 + compat->update_fn(compat_c, in0, stream->next_in - in0);

2017 if (write_in_full(fd, compressed, stream->next_out - compressed) < 0)

2018 die_errno(_("unable to write loose object file"));

2019 stream->next_out = compressed;

@@ -2014,15 +2028,21 @@ static int write_loose_object_common(git_hash_ctx *c,

2028 * - End the compression of zlib stream.

2029 * - Get the calculated oid to "oid".

2030 */

2017 -static int end_loose_object_common(git_hash_ctx *c, git_zstream *stream,

2018 - struct object_id *oid)

2031 +static int end_loose_object_common(git_hash_ctx *c, git_hash_ctx *compat_c,

2032 + git_zstream *stream, struct object_id *oid,

2033 + struct object_id *compat_oid)

2034 {

2035 + struct repository *repo = the_repository;

2036 + const struct git_hash_algo *algo = repo->hash_algo;

2037 + const struct git_hash_algo *compat = repo->compat_hash_algo;

2038 int ret;

2039

2040 ret = git_deflate_end_gently(stream);

2041 if (ret != Z_OK)

2042 return ret;

2025 - the_hash_algo->final_oid_fn(oid, c);

2043 + algo->final_oid_fn(oid, c);

2044 + if (compat && compat_c)

2045 + compat->final_oid_fn(compat_oid, compat_c);

2046

2047 return Z_OK;

2048 }

@@ -2046,7 +2066,7 @@ static int write_loose_object(const struct object_id *oid, char *hdr,

2066

2067 fd = start_loose_object_common(&tmp_file, filename.buf, flags,

2068 &stream, compressed, sizeof(compressed),

2049 - &c, hdr, hdrlen);

2069 + &c, NULL, hdr, hdrlen);

2070 if (fd < 0)

2071 return -1;

2072

@@ -2056,14 +2076,14 @@ static int write_loose_object(const struct object_id *oid, char *hdr,

2076 do {

2077 unsigned char *in0 = stream.next_in;

2078

2059 - ret = write_loose_object_common(&c, &stream, 1, in0, fd,

2079 + ret = write_loose_object_common(&c, NULL, &stream, 1, in0, fd,

2080 compressed, sizeof(compressed));

2081 } while (ret == Z_OK);

2082

2083 if (ret != Z_STREAM_END)

2084 die(_("unable to deflate new object %s (%d)"), oid_to_hex(oid),

2085 ret);

2066 - ret = end_loose_object_common(&c, &stream, &parano_oid);

2086 + ret = end_loose_object_common(&c, NULL, &stream, &parano_oid, NULL);

2087 if (ret != Z_OK)

2088 die(_("deflateEnd on object %s failed (%d)"), oid_to_hex(oid),

2089 ret);

@@ -2108,10 +2128,12 @@ static int freshen_packed_object(const struct object_id *oid)

2128 int stream_loose_object(struct input_stream *in_stream, size_t len,

2129 struct object_id *oid)

2130 {

2131 + const struct git_hash_algo *compat = the_repository->compat_hash_algo;

2132 + struct object_id compat_oid;

2133 int fd, ret, err = 0, flush = 0;

2134 unsigned char compressed[4096];

2135 git_zstream stream;

2114 - git_hash_ctx c;

2136 + git_hash_ctx c, compat_c;

2137 struct strbuf tmp_file = STRBUF_INIT;

2138 struct strbuf filename = STRBUF_INIT;

2139 int dirlen;

@@ -2135,7 +2157,7 @@ int stream_loose_object(struct input_stream *in_stream, size_t len,

2157 */

2158 fd = start_loose_object_common(&tmp_file, filename.buf, 0,

2159 &stream, compressed, sizeof(compressed),

2138 - &c, hdr, hdrlen);

2160 + &c, &compat_c, hdr, hdrlen);

2161 if (fd < 0) {

2162 err = -1;

2163 goto cleanup;

@@ -2153,7 +2175,7 @@ int stream_loose_object(struct input_stream *in_stream, size_t len,

2175 if (in_stream->is_finished)

2176 flush = 1;

2177 }

2156 - ret = write_loose_object_common(&c, &stream, flush, in0, fd,

2178 + ret = write_loose_object_common(&c, &compat_c, &stream, flush, in0, fd,

2179 compressed, sizeof(compressed));

2180 /*

2181 * Unlike write_loose_object(), we do not have the entire

@@ -2176,7 +2198,7 @@ int stream_loose_object(struct input_stream *in_stream, size_t len,

2198 */

2199 if (ret != Z_STREAM_END)

2200 die(_("unable to stream deflate new object (%d)"), ret);

2179 - ret = end_loose_object_common(&c, &stream, oid);

2201 + ret = end_loose_object_common(&c, &compat_c, &stream, oid, &compat_oid);

2202 if (ret != Z_OK)

2203 die(_("deflateEnd on stream object failed (%d)"), ret);

2204 close_loose_object(fd, tmp_file.buf);

@@ -2203,6 +2225,8 @@ int stream_loose_object(struct input_stream *in_stream, size_t len,

2225 }

2226

2227 err = finalize_object_file(tmp_file.buf, filename.buf);

2228 + if (!err && compat)

2229 + err = repo_add_loose_object_map(the_repository, oid, &compat_oid);

2230 cleanup:

2231 strbuf_release(&tmp_file);

2232 strbuf_release(&filename);

@@ -2213,17 +2237,38 @@ int write_object_file_flags(const void *buf, unsigned long len,

2237 enum object_type type, struct object_id *oid,

2238 unsigned flags)

2239 {

2240 + struct repository *repo = the_repository;

2241 + const struct git_hash_algo *algo = repo->hash_algo;

2242 + const struct git_hash_algo *compat = repo->compat_hash_algo;

2243 + struct object_id compat_oid;

2244 char hdr[MAX_HEADER_LEN];

2245 int hdrlen = sizeof(hdr);

2246

2247 + /* Generate compat_oid */

2248 + if (compat) {

2249 + if (type == OBJ_BLOB)

2250 + hash_object_file(compat, buf, len, type, &compat_oid);

2251 + else {

2252 + struct strbuf converted = STRBUF_INIT;

2253 + convert_object_file(&converted, algo, compat,

2254 + buf, len, type, 0);

2255 + hash_object_file(compat, converted.buf, converted.len,

2256 + type, &compat_oid);

2257 + strbuf_release(&converted);

2258 + }

2259 + }

2260 +

2261 /* Normally if we have it in the pack then we do not bother writing

2262 * it out into .git/objects/??/?{38} file.

2263 */

2222 - write_object_file_prepare(the_hash_algo, buf, len, type, oid, hdr,

2223 - &hdrlen);

2264 + write_object_file_prepare(algo, buf, len, type, oid, hdr, &hdrlen);

2265 if (freshen_packed_object(oid) || freshen_loose_object(oid))

2266 return 0;

2226 - return write_loose_object(oid, hdr, hdrlen, buf, len, 0, flags);

2267 + if (write_loose_object(oid, hdr, hdrlen, buf, len, 0, flags))

2268 + return -1;

2269 + if (compat)

2270 + return repo_add_loose_object_map(repo, oid, &compat_oid);

2271 + return 0;

2272 }

2273

2274 int write_object_file_literally(const void *buf, unsigned long len,

@@ -2231,7 +2276,27 @@ int write_object_file_literally(const void *buf, unsigned long len,

2276 unsigned flags)

2277 {

2278 char *header;

2279 + struct repository *repo = the_repository;

2280 + const struct git_hash_algo *algo = repo->hash_algo;

2281 + const struct git_hash_algo *compat = repo->compat_hash_algo;

2282 + struct object_id compat_oid;

2283 int hdrlen, status = 0;

2284 + int compat_type = -1;

2285 +

2286 + if (compat) {

2287 + compat_type = type_from_string_gently(type, -1, 1);

2288 + if (compat_type == OBJ_BLOB)

2289 + hash_object_file(compat, buf, len, compat_type,

2290 + &compat_oid);

2291 + else if (compat_type != -1) {

2292 + struct strbuf converted = STRBUF_INIT;

2293 + convert_object_file(&converted, algo, compat,

2294 + buf, len, compat_type, 0);

2295 + hash_object_file(compat, converted.buf, converted.len,

2296 + compat_type, &compat_oid);

2297 + strbuf_release(&converted);

2298 + }

2299 + }

2300

2301 /* type string, SP, %lu of the length plus NUL must fit this */

2302 hdrlen = strlen(type) + MAX_HEADER_LEN;

@@ -2244,6 +2309,8 @@ int write_object_file_literally(const void *buf, unsigned long len,

2309 if (freshen_packed_object(oid) || freshen_loose_object(oid))

2310 goto cleanup;

2311 status = write_loose_object(oid, header, hdrlen, buf, len, 0, 0);

2312 + if (compat_type != -1)

2313 + return repo_add_loose_object_map(repo, oid, &compat_oid);

2314

2315 cleanup:

2316 free(header);

@@ -2252,9 +2319,12 @@ cleanup:

2319

2320 int force_object_loose(const struct object_id *oid, time_t mtime)

2321 {

2322 + struct repository *repo = the_repository;

2323 + const struct git_hash_algo *compat = repo->compat_hash_algo;

2324 void *buf;

2325 unsigned long len;

2326 struct object_info oi = OBJECT_INFO_INIT;

2327 + struct object_id compat_oid;

2328 enum object_type type;

2329 char hdr[MAX_HEADER_LEN];

2330 int hdrlen;

@@ -2267,8 +2337,15 @@ int force_object_loose(const struct object_id *oid, time_t mtime)

2337 oi.contentp = &buf;

2338 if (oid_object_info_extended(the_repository, oid, &oi, 0))

2339 return error(_("cannot read object for %s"), oid_to_hex(oid));

2340 + if (compat) {

2341 + if (repo_oid_to_algop(repo, oid, compat, &compat_oid))

2342 + return error(_("cannot map object %s to %s"),

2343 + oid_to_hex(oid), compat->name);

2344 + }

2345 hdrlen = format_object_header(hdr, sizeof(hdr), type, len);

2346 ret = write_loose_object(oid, hdr, hdrlen, buf, len, mtime, 0);

2347 + if (!ret && compat)

2348 + ret = repo_add_loose_object_map(the_repository, oid, &compat_oid);

2349 free(buf);

2350

2351 return ret;