sha1_file: remove recursion in unpack_entry
Similar to the recursion in packed_object_info(), this leads to problems on stack-space-constrained systems in the presence of long delta chains. We proceed in three phases: 1. Dig through the delta chain, saving each delta object's offsets and size on an ad-hoc stack. 2. Unpack the base object at the bottom. 3. Unpack and apply the deltas from the stack. Signed-off-by: Thomas Rast <trast@student.ethz.ch> Signed-off-by: Junio C Hamano <gitster@pobox.com>
Thomas Rast committed
Mar 27, 2013 at 21:03 UTC
abe601bba52ee6b0bf89d282aa1c3ef5fd89cbb0
1 file changed
+150
-81
sha1_file.c
+150
-81
@@ -1864,68 +1864,6 @@ static void add_delta_base_cache(struct packed_git *p, off_t base_offset,
1864
static void *read_object(const unsigned char *sha1, enum object_type *type,
1865
unsigned long *size);
1866
1867
-static void *unpack_delta_entry(struct packed_git *p,
1868
- struct pack_window **w_curs,
1869
- off_t curpos,
1870
- unsigned long delta_size,
1871
- off_t obj_offset,
1872
- enum object_type *type,
1873
- unsigned long *sizep)
1874
-{
1875
- void *delta_data, *result, *base;
1876
- unsigned long base_size;
1877
- off_t base_offset;
1878
-
1879
- base_offset = get_delta_base(p, w_curs, &curpos, *type, obj_offset);
1880
- if (!base_offset) {
1881
- error("failed to validate delta base reference "
1882
- "at offset %"PRIuMAX" from %s",
1883
- (uintmax_t)curpos, p->pack_name);
1884
- return NULL;
1885
- }
1886
- unuse_pack(w_curs);
1887
- base = cache_or_unpack_entry(p, base_offset, &base_size, type, 0);
1888
- if (!base) {
1889
- /*
1890
- * We're probably in deep shit, but let's try to fetch
1891
- * the required base anyway from another pack or loose.
1892
- * This is costly but should happen only in the presence
1893
- * of a corrupted pack, and is better than failing outright.
1894
- */
1895
- struct revindex_entry *revidx;
1896
- const unsigned char *base_sha1;
1897
- revidx = find_pack_revindex(p, base_offset);
1898
- if (!revidx)
1899
- return NULL;
1900
- base_sha1 = nth_packed_object_sha1(p, revidx->nr);
1901
- error("failed to read delta base object %s"
1902
- " at offset %"PRIuMAX" from %s",
1903
- sha1_to_hex(base_sha1), (uintmax_t)base_offset,
1904
- p->pack_name);
1905
- mark_bad_packed_object(p, base_sha1);
1906
- base = read_object(base_sha1, type, &base_size);
1907
- if (!base)
1908
- return NULL;
1909
- }
1910
-
1911
- delta_data = unpack_compressed_entry(p, w_curs, curpos, delta_size);
1912
- if (!delta_data) {
1913
- error("failed to unpack compressed delta "
1914
- "at offset %"PRIuMAX" from %s",
1915
- (uintmax_t)curpos, p->pack_name);
1916
- free(base);
1917
- return NULL;
1918
- }
1919
- result = patch_delta(base, base_size,
1920
- delta_data, delta_size,
1921
- sizep);
1922
- if (!result)
1923
- die("failed to apply delta");
1924
- free(delta_data);
1925
- add_delta_base_cache(p, base_offset, base, base_size, *type);
1926
- return result;
1927
-}
1928
-
1867
static void write_pack_access_log(struct packed_git *p, off_t obj_offset)
1868
{
1869
static FILE *log_file;
@@ -1946,48 +1884,179 @@ static void write_pack_access_log(struct packed_git *p, off_t obj_offset)
1884
1885
int do_check_packed_object_crc;
1886
1887
+#define UNPACK_ENTRY_STACK_PREALLOC 64
1888
+struct unpack_entry_stack_ent {
1889
+ off_t obj_offset;
1890
+ off_t curpos;
1891
+ unsigned long size;
1892
+};
1893
+
1894
void *unpack_entry(struct packed_git *p, off_t obj_offset,
1950
- enum object_type *type, unsigned long *sizep)
1895
+ enum object_type *final_type, unsigned long *final_size)
1896
{
1897
struct pack_window *w_curs = NULL;
1898
off_t curpos = obj_offset;
1954
- void *data;
1899
+ void *data = NULL;
1900
+ unsigned long size;
1901
+ enum object_type type;
1902
+ struct unpack_entry_stack_ent small_delta_stack[UNPACK_ENTRY_STACK_PREALLOC];
1903
+ struct unpack_entry_stack_ent *delta_stack = small_delta_stack;
1904
+ int delta_stack_nr = 0, delta_stack_alloc = UNPACK_ENTRY_STACK_PREALLOC;
1905
+ int base_from_cache = 0;
1906
1907
if (log_pack_access)
1908
write_pack_access_log(p, obj_offset);
1909
1959
- if (do_check_packed_object_crc && p->index_version > 1) {
1960
- struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
1961
- unsigned long len = revidx[1].offset - obj_offset;
1962
- if (check_pack_crc(p, &w_curs, obj_offset, len, revidx->nr)) {
1963
- const unsigned char *sha1 =
1964
- nth_packed_object_sha1(p, revidx->nr);
1965
- error("bad packed object CRC for %s",
1966
- sha1_to_hex(sha1));
1967
- mark_bad_packed_object(p, sha1);
1968
- unuse_pack(&w_curs);
1969
- return NULL;
1910
+ /* PHASE 1: drill down to the innermost base object */
1911
+ for (;;) {
1912
+ off_t base_offset;
1913
+ int i;
1914
+ struct delta_base_cache_entry *ent;
1915
+
1916
+ if (do_check_packed_object_crc && p->index_version > 1) {
1917
+ struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
1918
+ unsigned long len = revidx[1].offset - obj_offset;
1919
+ if (check_pack_crc(p, &w_curs, obj_offset, len, revidx->nr)) {
1920
+ const unsigned char *sha1 =
1921
+ nth_packed_object_sha1(p, revidx->nr);
1922
+ error("bad packed object CRC for %s",
1923
+ sha1_to_hex(sha1));
1924
+ mark_bad_packed_object(p, sha1);
1925
+ unuse_pack(&w_curs);
1926
+ return NULL;
1927
+ }
1928
+ }
1929
+
1930
+ ent = get_delta_base_cache_entry(p, curpos);
1931
+ if (eq_delta_base_cache_entry(ent, p, curpos)) {
1932
+ type = ent->type;
1933
+ data = ent->data;
1934
+ size = ent->size;
1935
+ clear_delta_base_cache_entry(ent);
1936
+ base_from_cache = 1;
1937
+ break;
1938
+ }
1939
+
1940
+ type = unpack_object_header(p, &w_curs, &curpos, &size);
1941
+ if (type != OBJ_OFS_DELTA && type != OBJ_REF_DELTA)
1942
+ break;
1943
+
1944
+ base_offset = get_delta_base(p, &w_curs, &curpos, type, obj_offset);
1945
+ if (!base_offset) {
1946
+ error("failed to validate delta base reference "
1947
+ "at offset %"PRIuMAX" from %s",
1948
+ (uintmax_t)curpos, p->pack_name);
1949
+ /* bail to phase 2, in hopes of recovery */
1950
+ data = NULL;
1951
+ break;
1952
}
1953
+
1954
+ /* push object, proceed to base */
1955
+ if (delta_stack_nr >= delta_stack_alloc
1956
+ && delta_stack == small_delta_stack) {
1957
+ delta_stack_alloc = alloc_nr(delta_stack_nr);
1958
+ delta_stack = xmalloc(sizeof(*delta_stack)*delta_stack_alloc);
1959
+ memcpy(delta_stack, small_delta_stack,
1960
+ sizeof(*delta_stack)*delta_stack_nr);
1961
+ } else {
1962
+ ALLOC_GROW(delta_stack, delta_stack_nr+1, delta_stack_alloc);
1963
+ }
1964
+ i = delta_stack_nr++;
1965
+ delta_stack[i].obj_offset = obj_offset;
1966
+ delta_stack[i].curpos = curpos;
1967
+ delta_stack[i].size = size;
1968
+
1969
+ curpos = obj_offset = base_offset;
1970
}
1971
1973
- *type = unpack_object_header(p, &w_curs, &curpos, sizep);
1974
- switch (*type) {
1972
+ /* PHASE 2: handle the base */
1973
+ switch (type) {
1974
case OBJ_OFS_DELTA:
1975
case OBJ_REF_DELTA:
1977
- data = unpack_delta_entry(p, &w_curs, curpos, *sizep,
1978
- obj_offset, type, sizep);
1976
+ if (data)
1977
+ die("BUG in unpack_entry: left loop at a valid delta");
1978
break;
1979
case OBJ_COMMIT:
1980
case OBJ_TREE:
1981
case OBJ_BLOB:
1982
case OBJ_TAG:
1984
- data = unpack_compressed_entry(p, &w_curs, curpos, *sizep);
1983
+ if (!base_from_cache)
1984
+ data = unpack_compressed_entry(p, &w_curs, curpos, size);
1985
break;
1986
default:
1987
data = NULL;
1988
error("unknown object type %i at offset %"PRIuMAX" in %s",
1989
- *type, (uintmax_t)obj_offset, p->pack_name);
1989
+ type, (uintmax_t)obj_offset, p->pack_name);
1990
}
1991
+
1992
+ /* PHASE 3: apply deltas in order */
1993
+
1994
+ /* invariants:
1995
+ * 'data' holds the base data, or NULL if there was corruption
1996
+ */
1997
+ while (delta_stack_nr) {
1998
+ void *delta_data;
1999
+ void *base = data;
2000
+ unsigned long delta_size, base_size = size;
2001
+ int i;
2002
+
2003
+ data = NULL;
2004
+
2005
+ if (base)
2006
+ add_delta_base_cache(p, obj_offset, base, base_size, type);
2007
+
2008
+ if (!base) {
2009
+ /*
2010
+ * We're probably in deep shit, but let's try to fetch
2011
+ * the required base anyway from another pack or loose.
2012
+ * This is costly but should happen only in the presence
2013
+ * of a corrupted pack, and is better than failing outright.
2014
+ */
2015
+ struct revindex_entry *revidx;
2016
+ const unsigned char *base_sha1;
2017
+ revidx = find_pack_revindex(p, obj_offset);
2018
+ if (revidx) {
2019
+ base_sha1 = nth_packed_object_sha1(p, revidx->nr);
2020
+ error("failed to read delta base object %s"
2021
+ " at offset %"PRIuMAX" from %s",
2022
+ sha1_to_hex(base_sha1), (uintmax_t)obj_offset,
2023
+ p->pack_name);
2024
+ mark_bad_packed_object(p, base_sha1);
2025
+ base = read_object(base_sha1, &type, &base_size);
2026
+ }
2027
+ }
2028
+
2029
+ i = --delta_stack_nr;
2030
+ obj_offset = delta_stack[i].obj_offset;
2031
+ curpos = delta_stack[i].curpos;
2032
+ delta_size = delta_stack[i].size;
2033
+
2034
+ if (!base)
2035
+ continue;
2036
+
2037
+ delta_data = unpack_compressed_entry(p, &w_curs, curpos, delta_size);
2038
+
2039
+ if (!delta_data) {
2040
+ error("failed to unpack compressed delta "
2041
+ "at offset %"PRIuMAX" from %s",
2042
+ (uintmax_t)curpos, p->pack_name);
2043
+ free(base);
2044
+ data = NULL;
2045
+ continue;
2046
+ }
2047
+
2048
+ data = patch_delta(base, base_size,
2049
+ delta_data, delta_size,
2050
+ &size);
2051
+ if (!data)
2052
+ die("failed to apply delta");
2053
+
2054
+ free (delta_data);
2055
+ }
2056
+
2057
+ *final_type = type;
2058
+ *final_size = size;
2059
+
2060
unuse_pack(&w_curs);
2061
return data;
2062
}