@matheusbsbs / matheus-bagliano / commits / f1d8130be0

pack: move clear_delta_base_cache(), packed_object_info(), unpack_entry()

Both sha1_file.c and packfile.c now need read_object(), so a copy of read_object() was created in packfile.c. This patch makes both mark_bad_packed_object() and has_packed_and_bad() global. Unlike most of the other patches in this series, these 2 functions need to remain global. Signed-off-by: Jonathan Tan <jonathantanmy@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Jonathan Tan committed Aug 18, 2017 at 15:20 UTC f1d8130be0a7229b215b76eac54c561a0124bc99
4 files changed +685 -670
cache.h
-7
@@ -1187,9 +1187,6 @@ extern void *map_sha1_file(const unsigned char *sha1, unsigned long *size);
1187 extern int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz);
1188 extern int parse_sha1_header(const char *hdr, unsigned long *sizep);
1189
1190 -/* global flag to enable extra checks when accessing packed objects */
1191 -extern int do_check_packed_object_crc;
1192 -
1190 extern int check_sha1_signature(const unsigned char *sha1, void *buf, unsigned long size, const char *type);
1191
1192 extern int finalize_object_file(const char *tmpfile, const char *filename);
@@ -1623,8 +1620,6 @@ extern int odb_mkstemp(struct strbuf *template, const char *pattern);
1620 */
1621 extern int odb_pack_keep(const char *name);
1622
1626 -extern void clear_delta_base_cache(void);
1627 -
1623 /*
1624 * Make sure that a pointer access into an mmap'd index file is within bounds,
1625 * and can provide at least 8 bytes of data.
@@ -1662,7 +1657,6 @@ extern off_t nth_packed_object_offset(const struct packed_git *, uint32_t n);
1657 extern off_t find_pack_entry_one(const unsigned char *sha1, struct packed_git *);
1658
1659 extern int is_pack_valid(struct packed_git *);
1665 -extern void *unpack_entry(struct packed_git *, off_t, enum object_type *, unsigned long *);
1660
1661 /*
1662 * Iterate over the files in the loose-object parts of the object
@@ -1773,7 +1767,6 @@ struct object_info {
1767 /* Do not retry packed storage after checking packed and loose storage */
1768 #define OBJECT_INFO_QUICK 8
1769 extern int sha1_object_info_extended(const unsigned char *, struct object_info *, unsigned flags);
1776 -extern int packed_object_info(struct packed_git *pack, off_t offset, struct object_info *);
1770
1771 /* Dumb servers support */
1772 extern int update_server_info(int);
packfile.c
+661
@@ -5,6 +5,8 @@
5 #include "mergesort.h"
6 #include "packfile.h"
7 #include "delta.h"
8 +#include "list.h"
9 +#include "streaming.h"
10
11 char *odb_pack_name(struct strbuf *buf,
12 const unsigned char *sha1,
@@ -975,3 +977,662 @@ int unpack_object_header(struct packed_git *p,
977
978 return type;
979 }
980 +
981 +void mark_bad_packed_object(struct packed_git *p, const unsigned char *sha1)
982 +{
983 + unsigned i;
984 + for (i = 0; i < p->num_bad_objects; i++)
985 + if (!hashcmp(sha1, p->bad_object_sha1 + GIT_SHA1_RAWSZ * i))
986 + return;
987 + p->bad_object_sha1 = xrealloc(p->bad_object_sha1,
988 + st_mult(GIT_MAX_RAWSZ,
989 + st_add(p->num_bad_objects, 1)));
990 + hashcpy(p->bad_object_sha1 + GIT_SHA1_RAWSZ * p->num_bad_objects, sha1);
991 + p->num_bad_objects++;
992 +}
993 +
994 +const struct packed_git *has_packed_and_bad(const unsigned char *sha1)
995 +{
996 + struct packed_git *p;
997 + unsigned i;
998 +
999 + for (p = packed_git; p; p = p->next)
1000 + for (i = 0; i < p->num_bad_objects; i++)
1001 + if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i))
1002 + return p;
1003 + return NULL;
1004 +}
1005 +
1006 +static off_t get_delta_base(struct packed_git *p,
1007 + struct pack_window **w_curs,
1008 + off_t *curpos,
1009 + enum object_type type,
1010 + off_t delta_obj_offset)
1011 +{
1012 + unsigned char *base_info = use_pack(p, w_curs, *curpos, NULL);
1013 + off_t base_offset;
1014 +
1015 + /* use_pack() assured us we have [base_info, base_info + 20)
1016 + * as a range that we can look at without walking off the
1017 + * end of the mapped window. Its actually the hash size
1018 + * that is assured. An OFS_DELTA longer than the hash size
1019 + * is stupid, as then a REF_DELTA would be smaller to store.
1020 + */
1021 + if (type == OBJ_OFS_DELTA) {
1022 + unsigned used = 0;
1023 + unsigned char c = base_info[used++];
1024 + base_offset = c & 127;
1025 + while (c & 128) {
1026 + base_offset += 1;
1027 + if (!base_offset || MSB(base_offset, 7))
1028 + return 0; /* overflow */
1029 + c = base_info[used++];
1030 + base_offset = (base_offset << 7) + (c & 127);
1031 + }
1032 + base_offset = delta_obj_offset - base_offset;
1033 + if (base_offset <= 0 || base_offset >= delta_obj_offset)
1034 + return 0; /* out of bound */
1035 + *curpos += used;
1036 + } else if (type == OBJ_REF_DELTA) {
1037 + /* The base entry _must_ be in the same pack */
1038 + base_offset = find_pack_entry_one(base_info, p);
1039 + *curpos += 20;
1040 + } else
1041 + die("I am totally screwed");
1042 + return base_offset;
1043 +}
1044 +
1045 +/*
1046 + * Like get_delta_base above, but we return the sha1 instead of the pack
1047 + * offset. This means it is cheaper for REF deltas (we do not have to do
1048 + * the final object lookup), but more expensive for OFS deltas (we
1049 + * have to load the revidx to convert the offset back into a sha1).
1050 + */
1051 +static const unsigned char *get_delta_base_sha1(struct packed_git *p,
1052 + struct pack_window **w_curs,
1053 + off_t curpos,
1054 + enum object_type type,
1055 + off_t delta_obj_offset)
1056 +{
1057 + if (type == OBJ_REF_DELTA) {
1058 + unsigned char *base = use_pack(p, w_curs, curpos, NULL);
1059 + return base;
1060 + } else if (type == OBJ_OFS_DELTA) {
1061 + struct revindex_entry *revidx;
1062 + off_t base_offset = get_delta_base(p, w_curs, &curpos,
1063 + type, delta_obj_offset);
1064 +
1065 + if (!base_offset)
1066 + return NULL;
1067 +
1068 + revidx = find_pack_revindex(p, base_offset);
1069 + if (!revidx)
1070 + return NULL;
1071 +
1072 + return nth_packed_object_sha1(p, revidx->nr);
1073 + } else
1074 + return NULL;
1075 +}
1076 +
1077 +static int retry_bad_packed_offset(struct packed_git *p, off_t obj_offset)
1078 +{
1079 + int type;
1080 + struct revindex_entry *revidx;
1081 + const unsigned char *sha1;
1082 + revidx = find_pack_revindex(p, obj_offset);
1083 + if (!revidx)
1084 + return OBJ_BAD;
1085 + sha1 = nth_packed_object_sha1(p, revidx->nr);
1086 + mark_bad_packed_object(p, sha1);
1087 + type = sha1_object_info(sha1, NULL);
1088 + if (type <= OBJ_NONE)
1089 + return OBJ_BAD;
1090 + return type;
1091 +}
1092 +
1093 +#define POI_STACK_PREALLOC 64
1094 +
1095 +static enum object_type packed_to_object_type(struct packed_git *p,
1096 + off_t obj_offset,
1097 + enum object_type type,
1098 + struct pack_window **w_curs,
1099 + off_t curpos)
1100 +{
1101 + off_t small_poi_stack[POI_STACK_PREALLOC];
1102 + off_t *poi_stack = small_poi_stack;
1103 + int poi_stack_nr = 0, poi_stack_alloc = POI_STACK_PREALLOC;
1104 +
1105 + while (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
1106 + off_t base_offset;
1107 + unsigned long size;
1108 + /* Push the object we're going to leave behind */
1109 + if (poi_stack_nr >= poi_stack_alloc && poi_stack == small_poi_stack) {
1110 + poi_stack_alloc = alloc_nr(poi_stack_nr);
1111 + ALLOC_ARRAY(poi_stack, poi_stack_alloc);
1112 + memcpy(poi_stack, small_poi_stack, sizeof(off_t)*poi_stack_nr);
1113 + } else {
1114 + ALLOC_GROW(poi_stack, poi_stack_nr+1, poi_stack_alloc);
1115 + }
1116 + poi_stack[poi_stack_nr++] = obj_offset;
1117 + /* If parsing the base offset fails, just unwind */
1118 + base_offset = get_delta_base(p, w_curs, &curpos, type, obj_offset);
1119 + if (!base_offset)
1120 + goto unwind;
1121 + curpos = obj_offset = base_offset;
1122 + type = unpack_object_header(p, w_curs, &curpos, &size);
1123 + if (type <= OBJ_NONE) {
1124 + /* If getting the base itself fails, we first
1125 + * retry the base, otherwise unwind */
1126 + type = retry_bad_packed_offset(p, base_offset);
1127 + if (type > OBJ_NONE)
1128 + goto out;
1129 + goto unwind;
1130 + }
1131 + }
1132 +
1133 + switch (type) {
1134 + case OBJ_BAD:
1135 + case OBJ_COMMIT:
1136 + case OBJ_TREE:
1137 + case OBJ_BLOB:
1138 + case OBJ_TAG:
1139 + break;
1140 + default:
1141 + error("unknown object type %i at offset %"PRIuMAX" in %s",
1142 + type, (uintmax_t)obj_offset, p->pack_name);
1143 + type = OBJ_BAD;
1144 + }
1145 +
1146 +out:
1147 + if (poi_stack != small_poi_stack)
1148 + free(poi_stack);
1149 + return type;
1150 +
1151 +unwind:
1152 + while (poi_stack_nr) {
1153 + obj_offset = poi_stack[--poi_stack_nr];
1154 + type = retry_bad_packed_offset(p, obj_offset);
1155 + if (type > OBJ_NONE)
1156 + goto out;
1157 + }
1158 + type = OBJ_BAD;
1159 + goto out;
1160 +}
1161 +
1162 +static struct hashmap delta_base_cache;
1163 +static size_t delta_base_cached;
1164 +
1165 +static LIST_HEAD(delta_base_cache_lru);
1166 +
1167 +struct delta_base_cache_key {
1168 + struct packed_git *p;
1169 + off_t base_offset;
1170 +};
1171 +
1172 +struct delta_base_cache_entry {
1173 + struct hashmap hash;
1174 + struct delta_base_cache_key key;
1175 + struct list_head lru;
1176 + void *data;
1177 + unsigned long size;
1178 + enum object_type type;
1179 +};
1180 +
1181 +static unsigned int pack_entry_hash(struct packed_git *p, off_t base_offset)
1182 +{
1183 + unsigned int hash;
1184 +
1185 + hash = (unsigned int)(intptr_t)p + (unsigned int)base_offset;
1186 + hash += (hash >> 8) + (hash >> 16);
1187 + return hash;
1188 +}
1189 +
1190 +static struct delta_base_cache_entry *
1191 +get_delta_base_cache_entry(struct packed_git *p, off_t base_offset)
1192 +{
1193 + struct hashmap_entry entry;
1194 + struct delta_base_cache_key key;
1195 +
1196 + if (!delta_base_cache.cmpfn)
1197 + return NULL;
1198 +
1199 + hashmap_entry_init(&entry, pack_entry_hash(p, base_offset));
1200 + key.p = p;
1201 + key.base_offset = base_offset;
1202 + return hashmap_get(&delta_base_cache, &entry, &key);
1203 +}
1204 +
1205 +static int delta_base_cache_key_eq(const struct delta_base_cache_key *a,
1206 + const struct delta_base_cache_key *b)
1207 +{
1208 + return a->p == b->p && a->base_offset == b->base_offset;
1209 +}
1210 +
1211 +static int delta_base_cache_hash_cmp(const void *unused_cmp_data,
1212 + const void *va, const void *vb,
1213 + const void *vkey)
1214 +{
1215 + const struct delta_base_cache_entry *a = va, *b = vb;
1216 + const struct delta_base_cache_key *key = vkey;
1217 + if (key)
1218 + return !delta_base_cache_key_eq(&a->key, key);
1219 + else
1220 + return !delta_base_cache_key_eq(&a->key, &b->key);
1221 +}
1222 +
1223 +static int in_delta_base_cache(struct packed_git *p, off_t base_offset)
1224 +{
1225 + return !!get_delta_base_cache_entry(p, base_offset);
1226 +}
1227 +
1228 +/*
1229 + * Remove the entry from the cache, but do _not_ free the associated
1230 + * entry data. The caller takes ownership of the "data" buffer, and
1231 + * should copy out any fields it wants before detaching.
1232 + */
1233 +static void detach_delta_base_cache_entry(struct delta_base_cache_entry *ent)
1234 +{
1235 + hashmap_remove(&delta_base_cache, ent, &ent->key);
1236 + list_del(&ent->lru);
1237 + delta_base_cached -= ent->size;
1238 + free(ent);
1239 +}
1240 +
1241 +static void *cache_or_unpack_entry(struct packed_git *p, off_t base_offset,
1242 + unsigned long *base_size, enum object_type *type)
1243 +{
1244 + struct delta_base_cache_entry *ent;
1245 +
1246 + ent = get_delta_base_cache_entry(p, base_offset);
1247 + if (!ent)
1248 + return unpack_entry(p, base_offset, type, base_size);
1249 +
1250 + if (type)
1251 + *type = ent->type;
1252 + if (base_size)
1253 + *base_size = ent->size;
1254 + return xmemdupz(ent->data, ent->size);
1255 +}
1256 +
1257 +static inline void release_delta_base_cache(struct delta_base_cache_entry *ent)
1258 +{
1259 + free(ent->data);
1260 + detach_delta_base_cache_entry(ent);
1261 +}
1262 +
1263 +void clear_delta_base_cache(void)
1264 +{
1265 + struct list_head *lru, *tmp;
1266 + list_for_each_safe(lru, tmp, &delta_base_cache_lru) {
1267 + struct delta_base_cache_entry *entry =
1268 + list_entry(lru, struct delta_base_cache_entry, lru);
1269 + release_delta_base_cache(entry);
1270 + }
1271 +}
1272 +
1273 +static void add_delta_base_cache(struct packed_git *p, off_t base_offset,
1274 + void *base, unsigned long base_size, enum object_type type)
1275 +{
1276 + struct delta_base_cache_entry *ent = xmalloc(sizeof(*ent));
1277 + struct list_head *lru, *tmp;
1278 +
1279 + delta_base_cached += base_size;
1280 +
1281 + list_for_each_safe(lru, tmp, &delta_base_cache_lru) {
1282 + struct delta_base_cache_entry *f =
1283 + list_entry(lru, struct delta_base_cache_entry, lru);
1284 + if (delta_base_cached <= delta_base_cache_limit)
1285 + break;
1286 + release_delta_base_cache(f);
1287 + }
1288 +
1289 + ent->key.p = p;
1290 + ent->key.base_offset = base_offset;
1291 + ent->type = type;
1292 + ent->data = base;
1293 + ent->size = base_size;
1294 + list_add_tail(&ent->lru, &delta_base_cache_lru);
1295 +
1296 + if (!delta_base_cache.cmpfn)
1297 + hashmap_init(&delta_base_cache, delta_base_cache_hash_cmp, NULL, 0);
1298 + hashmap_entry_init(ent, pack_entry_hash(p, base_offset));
1299 + hashmap_add(&delta_base_cache, ent);
1300 +}
1301 +
1302 +int packed_object_info(struct packed_git *p, off_t obj_offset,
1303 + struct object_info *oi)
1304 +{
1305 + struct pack_window *w_curs = NULL;
1306 + unsigned long size;
1307 + off_t curpos = obj_offset;
1308 + enum object_type type;
1309 +
1310 + /*
1311 + * We always get the representation type, but only convert it to
1312 + * a "real" type later if the caller is interested.
1313 + */
1314 + if (oi->contentp) {
1315 + *oi->contentp = cache_or_unpack_entry(p, obj_offset, oi->sizep,
1316 + &type);
1317 + if (!*oi->contentp)
1318 + type = OBJ_BAD;
1319 + } else {
1320 + type = unpack_object_header(p, &w_curs, &curpos, &size);
1321 + }
1322 +
1323 + if (!oi->contentp && oi->sizep) {
1324 + if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
1325 + off_t tmp_pos = curpos;
1326 + off_t base_offset = get_delta_base(p, &w_curs, &tmp_pos,
1327 + type, obj_offset);
1328 + if (!base_offset) {
1329 + type = OBJ_BAD;
1330 + goto out;
1331 + }
1332 + *oi->sizep = get_size_from_delta(p, &w_curs, tmp_pos);
1333 + if (*oi->sizep == 0) {
1334 + type = OBJ_BAD;
1335 + goto out;
1336 + }
1337 + } else {
1338 + *oi->sizep = size;
1339 + }
1340 + }
1341 +
1342 + if (oi->disk_sizep) {
1343 + struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
1344 + *oi->disk_sizep = revidx[1].offset - obj_offset;
1345 + }
1346 +
1347 + if (oi->typep || oi->typename) {
1348 + enum object_type ptot;
1349 + ptot = packed_to_object_type(p, obj_offset, type, &w_curs,
1350 + curpos);
1351 + if (oi->typep)
1352 + *oi->typep = ptot;
1353 + if (oi->typename) {
1354 + const char *tn = typename(ptot);
1355 + if (tn)
1356 + strbuf_addstr(oi->typename, tn);
1357 + }
1358 + if (ptot < 0) {
1359 + type = OBJ_BAD;
1360 + goto out;
1361 + }
1362 + }
1363 +
1364 + if (oi->delta_base_sha1) {
1365 + if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
1366 + const unsigned char *base;
1367 +
1368 + base = get_delta_base_sha1(p, &w_curs, curpos,
1369 + type, obj_offset);
1370 + if (!base) {
1371 + type = OBJ_BAD;
1372 + goto out;
1373 + }
1374 +
1375 + hashcpy(oi->delta_base_sha1, base);
1376 + } else
1377 + hashclr(oi->delta_base_sha1);
1378 + }
1379 +
1380 + oi->whence = in_delta_base_cache(p, obj_offset) ? OI_DBCACHED :
1381 + OI_PACKED;
1382 +
1383 +out:
1384 + unuse_pack(&w_curs);
1385 + return type;
1386 +}
1387 +
1388 +static void *unpack_compressed_entry(struct packed_git *p,
1389 + struct pack_window **w_curs,
1390 + off_t curpos,
1391 + unsigned long size)
1392 +{
1393 + int st;
1394 + git_zstream stream;
1395 + unsigned char *buffer, *in;
1396 +
1397 + buffer = xmallocz_gently(size);
1398 + if (!buffer)
1399 + return NULL;
1400 + memset(&stream, 0, sizeof(stream));
1401 + stream.next_out = buffer;
1402 + stream.avail_out = size + 1;
1403 +
1404 + git_inflate_init(&stream);
1405 + do {
1406 + in = use_pack(p, w_curs, curpos, &stream.avail_in);
1407 + stream.next_in = in;
1408 + st = git_inflate(&stream, Z_FINISH);
1409 + if (!stream.avail_out)
1410 + break; /* the payload is larger than it should be */
1411 + curpos += stream.next_in - in;
1412 + } while (st == Z_OK || st == Z_BUF_ERROR);
1413 + git_inflate_end(&stream);
1414 + if ((st != Z_STREAM_END) || stream.total_out != size) {
1415 + free(buffer);
1416 + return NULL;
1417 + }
1418 +
1419 + return buffer;
1420 +}
1421 +
1422 +static void write_pack_access_log(struct packed_git *p, off_t obj_offset)
1423 +{
1424 + static struct trace_key pack_access = TRACE_KEY_INIT(PACK_ACCESS);
1425 + trace_printf_key(&pack_access, "%s %"PRIuMAX"\n",
1426 + p->pack_name, (uintmax_t)obj_offset);
1427 +}
1428 +
1429 +int do_check_packed_object_crc;
1430 +
1431 +#define UNPACK_ENTRY_STACK_PREALLOC 64
1432 +struct unpack_entry_stack_ent {
1433 + off_t obj_offset;
1434 + off_t curpos;
1435 + unsigned long size;
1436 +};
1437 +
1438 +static void *read_object(const unsigned char *sha1, enum object_type *type,
1439 + unsigned long *size)
1440 +{
1441 + struct object_info oi = OBJECT_INFO_INIT;
1442 + void *content;
1443 + oi.typep = type;
1444 + oi.sizep = size;
1445 + oi.contentp = &content;
1446 +
1447 + if (sha1_object_info_extended(sha1, &oi, 0) < 0)
1448 + return NULL;
1449 + return content;
1450 +}
1451 +
1452 +void *unpack_entry(struct packed_git *p, off_t obj_offset,
1453 + enum object_type *final_type, unsigned long *final_size)
1454 +{
1455 + struct pack_window *w_curs = NULL;
1456 + off_t curpos = obj_offset;
1457 + void *data = NULL;
1458 + unsigned long size;
1459 + enum object_type type;
1460 + struct unpack_entry_stack_ent small_delta_stack[UNPACK_ENTRY_STACK_PREALLOC];
1461 + struct unpack_entry_stack_ent *delta_stack = small_delta_stack;
1462 + int delta_stack_nr = 0, delta_stack_alloc = UNPACK_ENTRY_STACK_PREALLOC;
1463 + int base_from_cache = 0;
1464 +
1465 + write_pack_access_log(p, obj_offset);
1466 +
1467 + /* PHASE 1: drill down to the innermost base object */
1468 + for (;;) {
1469 + off_t base_offset;
1470 + int i;
1471 + struct delta_base_cache_entry *ent;
1472 +
1473 + ent = get_delta_base_cache_entry(p, curpos);
1474 + if (ent) {
1475 + type = ent->type;
1476 + data = ent->data;
1477 + size = ent->size;
1478 + detach_delta_base_cache_entry(ent);
1479 + base_from_cache = 1;
1480 + break;
1481 + }
1482 +
1483 + if (do_check_packed_object_crc && p->index_version > 1) {
1484 + struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
1485 + off_t len = revidx[1].offset - obj_offset;
1486 + if (check_pack_crc(p, &w_curs, obj_offset, len, revidx->nr)) {
1487 + const unsigned char *sha1 =
1488 + nth_packed_object_sha1(p, revidx->nr);
1489 + error("bad packed object CRC for %s",
1490 + sha1_to_hex(sha1));
1491 + mark_bad_packed_object(p, sha1);
1492 + data = NULL;
1493 + goto out;
1494 + }
1495 + }
1496 +
1497 + type = unpack_object_header(p, &w_curs, &curpos, &size);
1498 + if (type != OBJ_OFS_DELTA && type != OBJ_REF_DELTA)
1499 + break;
1500 +
1501 + base_offset = get_delta_base(p, &w_curs, &curpos, type, obj_offset);
1502 + if (!base_offset) {
1503 + error("failed to validate delta base reference "
1504 + "at offset %"PRIuMAX" from %s",
1505 + (uintmax_t)curpos, p->pack_name);
1506 + /* bail to phase 2, in hopes of recovery */
1507 + data = NULL;
1508 + break;
1509 + }
1510 +
1511 + /* push object, proceed to base */
1512 + if (delta_stack_nr >= delta_stack_alloc
1513 + && delta_stack == small_delta_stack) {
1514 + delta_stack_alloc = alloc_nr(delta_stack_nr);
1515 + ALLOC_ARRAY(delta_stack, delta_stack_alloc);
1516 + memcpy(delta_stack, small_delta_stack,
1517 + sizeof(*delta_stack)*delta_stack_nr);
1518 + } else {
1519 + ALLOC_GROW(delta_stack, delta_stack_nr+1, delta_stack_alloc);
1520 + }
1521 + i = delta_stack_nr++;
1522 + delta_stack[i].obj_offset = obj_offset;
1523 + delta_stack[i].curpos = curpos;
1524 + delta_stack[i].size = size;
1525 +
1526 + curpos = obj_offset = base_offset;
1527 + }
1528 +
1529 + /* PHASE 2: handle the base */
1530 + switch (type) {
1531 + case OBJ_OFS_DELTA:
1532 + case OBJ_REF_DELTA:
1533 + if (data)
1534 + die("BUG: unpack_entry: left loop at a valid delta");
1535 + break;
1536 + case OBJ_COMMIT:
1537 + case OBJ_TREE:
1538 + case OBJ_BLOB:
1539 + case OBJ_TAG:
1540 + if (!base_from_cache)
1541 + data = unpack_compressed_entry(p, &w_curs, curpos, size);
1542 + break;
1543 + default:
1544 + data = NULL;
1545 + error("unknown object type %i at offset %"PRIuMAX" in %s",
1546 + type, (uintmax_t)obj_offset, p->pack_name);
1547 + }
1548 +
1549 + /* PHASE 3: apply deltas in order */
1550 +
1551 + /* invariants:
1552 + * 'data' holds the base data, or NULL if there was corruption
1553 + */
1554 + while (delta_stack_nr) {
1555 + void *delta_data;
1556 + void *base = data;
1557 + void *external_base = NULL;
1558 + unsigned long delta_size, base_size = size;
1559 + int i;
1560 +
1561 + data = NULL;
1562 +
1563 + if (base)
1564 + add_delta_base_cache(p, obj_offset, base, base_size, type);
1565 +
1566 + if (!base) {
1567 + /*
1568 + * We're probably in deep shit, but let's try to fetch
1569 + * the required base anyway from another pack or loose.
1570 + * This is costly but should happen only in the presence
1571 + * of a corrupted pack, and is better than failing outright.
1572 + */
1573 + struct revindex_entry *revidx;
1574 + const unsigned char *base_sha1;
1575 + revidx = find_pack_revindex(p, obj_offset);
1576 + if (revidx) {
1577 + base_sha1 = nth_packed_object_sha1(p, revidx->nr);
1578 + error("failed to read delta base object %s"
1579 + " at offset %"PRIuMAX" from %s",
1580 + sha1_to_hex(base_sha1), (uintmax_t)obj_offset,
1581 + p->pack_name);
1582 + mark_bad_packed_object(p, base_sha1);
1583 + base = read_object(base_sha1, &type, &base_size);
1584 + external_base = base;
1585 + }
1586 + }
1587 +
1588 + i = --delta_stack_nr;
1589 + obj_offset = delta_stack[i].obj_offset;
1590 + curpos = delta_stack[i].curpos;
1591 + delta_size = delta_stack[i].size;
1592 +
1593 + if (!base)
1594 + continue;
1595 +
1596 + delta_data = unpack_compressed_entry(p, &w_curs, curpos, delta_size);
1597 +
1598 + if (!delta_data) {
1599 + error("failed to unpack compressed delta "
1600 + "at offset %"PRIuMAX" from %s",
1601 + (uintmax_t)curpos, p->pack_name);
1602 + data = NULL;
1603 + free(external_base);
1604 + continue;
1605 + }
1606 +
1607 + data = patch_delta(base, base_size,
1608 + delta_data, delta_size,
1609 + &size);
1610 +
1611 + /*
1612 + * We could not apply the delta; warn the user, but keep going.
1613 + * Our failure will be noticed either in the next iteration of
1614 + * the loop, or if this is the final delta, in the caller when
1615 + * we return NULL. Those code paths will take care of making
1616 + * a more explicit warning and retrying with another copy of
1617 + * the object.
1618 + */
1619 + if (!data)
1620 + error("failed to apply delta");
1621 +
1622 + free(delta_data);
1623 + free(external_base);
1624 + }
1625 +
1626 + if (final_type)
1627 + *final_type = type;
1628 + if (final_size)
1629 + *final_size = size;
1630 +
1631 +out:
1632 + unuse_pack(&w_curs);
1633 +
1634 + if (delta_stack != small_delta_stack)
1635 + free(delta_stack);
1636 +
1637 + return data;
1638 +}
packfile.h
+10
@@ -60,8 +60,10 @@ extern unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t
60 extern void close_pack_windows(struct packed_git *);
61 extern void close_all_packs(void);
62 extern void unuse_pack(struct pack_window **);
63 +extern void clear_delta_base_cache(void);
64 extern struct packed_git *add_packed_git(const char *path, size_t path_len, int local);
65
66 +extern void *unpack_entry(struct packed_git *, off_t, enum object_type *, unsigned long *);
67 extern unsigned long unpack_object_header_buffer(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep);
68 extern unsigned long get_size_from_delta(struct packed_git *, struct pack_window **, off_t);
69 extern int unpack_object_header(struct packed_git *, struct pack_window **, off_t *, unsigned long *);
@@ -70,4 +72,12 @@ extern void release_pack_memory(size_t);
72
73 extern int open_packed_git(struct packed_git *p);
74
75 +/* global flag to enable extra checks when accessing packed objects */
76 +extern int do_check_packed_object_crc;
77 +
78 +extern int packed_object_info(struct packed_git *pack, off_t offset, struct object_info *);
79 +
80 +extern void mark_bad_packed_object(struct packed_git *p, const unsigned char *sha1);
81 +extern const struct packed_git *has_packed_and_bad(const unsigned char *sha1);
82 +
83 #endif
sha1_file.c
+14 -663
@@ -719,32 +719,6 @@ void *xmmap(void *start, size_t length,
719 return ret;
720 }
721
722 -static void mark_bad_packed_object(struct packed_git *p,
723 - const unsigned char *sha1)
724 -{
725 - unsigned i;
726 - for (i = 0; i < p->num_bad_objects; i++)
727 - if (!hashcmp(sha1, p->bad_object_sha1 + GIT_SHA1_RAWSZ * i))
728 - return;
729 - p->bad_object_sha1 = xrealloc(p->bad_object_sha1,
730 - st_mult(GIT_MAX_RAWSZ,
731 - st_add(p->num_bad_objects, 1)));
732 - hashcpy(p->bad_object_sha1 + GIT_SHA1_RAWSZ * p->num_bad_objects, sha1);
733 - p->num_bad_objects++;
734 -}
735 -
736 -static const struct packed_git *has_packed_and_bad(const unsigned char *sha1)
737 -{
738 - struct packed_git *p;
739 - unsigned i;
740 -
741 - for (p = packed_git; p; p = p->next)
742 - for (i = 0; i < p->num_bad_objects; i++)
743 - if (!hashcmp(sha1, p->bad_object_sha1 + 20 * i))
744 - return p;
745 - return NULL;
746 -}
747 -
722 /*
723 * With an in-core object data in "map", rehash it to make sure the
724 * object name actually matches "sha1" to detect object corruption.
@@ -1101,629 +1075,6 @@ int parse_sha1_header(const char *hdr, unsigned long *sizep)
1075 return parse_sha1_header_extended(hdr, &oi, 0);
1076 }
1077
1104 -static off_t get_delta_base(struct packed_git *p,
1105 - struct pack_window **w_curs,
1106 - off_t *curpos,
1107 - enum object_type type,
1108 - off_t delta_obj_offset)
1109 -{
1110 - unsigned char *base_info = use_pack(p, w_curs, *curpos, NULL);
1111 - off_t base_offset;
1112 -
1113 - /* use_pack() assured us we have [base_info, base_info + 20)
1114 - * as a range that we can look at without walking off the
1115 - * end of the mapped window. Its actually the hash size
1116 - * that is assured. An OFS_DELTA longer than the hash size
1117 - * is stupid, as then a REF_DELTA would be smaller to store.
1118 - */
1119 - if (type == OBJ_OFS_DELTA) {
1120 - unsigned used = 0;
1121 - unsigned char c = base_info[used++];
1122 - base_offset = c & 127;
1123 - while (c & 128) {
1124 - base_offset += 1;
1125 - if (!base_offset || MSB(base_offset, 7))
1126 - return 0; /* overflow */
1127 - c = base_info[used++];
1128 - base_offset = (base_offset << 7) + (c & 127);
1129 - }
1130 - base_offset = delta_obj_offset - base_offset;
1131 - if (base_offset <= 0 || base_offset >= delta_obj_offset)
1132 - return 0; /* out of bound */
1133 - *curpos += used;
1134 - } else if (type == OBJ_REF_DELTA) {
1135 - /* The base entry _must_ be in the same pack */
1136 - base_offset = find_pack_entry_one(base_info, p);
1137 - *curpos += 20;
1138 - } else
1139 - die("I am totally screwed");
1140 - return base_offset;
1141 -}
1142 -
1143 -/*
1144 - * Like get_delta_base above, but we return the sha1 instead of the pack
1145 - * offset. This means it is cheaper for REF deltas (we do not have to do
1146 - * the final object lookup), but more expensive for OFS deltas (we
1147 - * have to load the revidx to convert the offset back into a sha1).
1148 - */
1149 -static const unsigned char *get_delta_base_sha1(struct packed_git *p,
1150 - struct pack_window **w_curs,
1151 - off_t curpos,
1152 - enum object_type type,
1153 - off_t delta_obj_offset)
1154 -{
1155 - if (type == OBJ_REF_DELTA) {
1156 - unsigned char *base = use_pack(p, w_curs, curpos, NULL);
1157 - return base;
1158 - } else if (type == OBJ_OFS_DELTA) {
1159 - struct revindex_entry *revidx;
1160 - off_t base_offset = get_delta_base(p, w_curs, &curpos,
1161 - type, delta_obj_offset);
1162 -
1163 - if (!base_offset)
1164 - return NULL;
1165 -
1166 - revidx = find_pack_revindex(p, base_offset);
1167 - if (!revidx)
1168 - return NULL;
1169 -
1170 - return nth_packed_object_sha1(p, revidx->nr);
1171 - } else
1172 - return NULL;
1173 -}
1174 -
1175 -static int retry_bad_packed_offset(struct packed_git *p, off_t obj_offset)
1176 -{
1177 - int type;
1178 - struct revindex_entry *revidx;
1179 - const unsigned char *sha1;
1180 - revidx = find_pack_revindex(p, obj_offset);
1181 - if (!revidx)
1182 - return OBJ_BAD;
1183 - sha1 = nth_packed_object_sha1(p, revidx->nr);
1184 - mark_bad_packed_object(p, sha1);
1185 - type = sha1_object_info(sha1, NULL);
1186 - if (type <= OBJ_NONE)
1187 - return OBJ_BAD;
1188 - return type;
1189 -}
1190 -
1191 -#define POI_STACK_PREALLOC 64
1192 -
1193 -static enum object_type packed_to_object_type(struct packed_git *p,
1194 - off_t obj_offset,
1195 - enum object_type type,
1196 - struct pack_window **w_curs,
1197 - off_t curpos)
1198 -{
1199 - off_t small_poi_stack[POI_STACK_PREALLOC];
1200 - off_t *poi_stack = small_poi_stack;
1201 - int poi_stack_nr = 0, poi_stack_alloc = POI_STACK_PREALLOC;
1202 -
1203 - while (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
1204 - off_t base_offset;
1205 - unsigned long size;
1206 - /* Push the object we're going to leave behind */
1207 - if (poi_stack_nr >= poi_stack_alloc && poi_stack == small_poi_stack) {
1208 - poi_stack_alloc = alloc_nr(poi_stack_nr);
1209 - ALLOC_ARRAY(poi_stack, poi_stack_alloc);
1210 - memcpy(poi_stack, small_poi_stack, sizeof(off_t)*poi_stack_nr);
1211 - } else {
1212 - ALLOC_GROW(poi_stack, poi_stack_nr+1, poi_stack_alloc);
1213 - }
1214 - poi_stack[poi_stack_nr++] = obj_offset;
1215 - /* If parsing the base offset fails, just unwind */
1216 - base_offset = get_delta_base(p, w_curs, &curpos, type, obj_offset);
1217 - if (!base_offset)
1218 - goto unwind;
1219 - curpos = obj_offset = base_offset;
1220 - type = unpack_object_header(p, w_curs, &curpos, &size);
1221 - if (type <= OBJ_NONE) {
1222 - /* If getting the base itself fails, we first
1223 - * retry the base, otherwise unwind */
1224 - type = retry_bad_packed_offset(p, base_offset);
1225 - if (type > OBJ_NONE)
1226 - goto out;
1227 - goto unwind;
1228 - }
1229 - }
1230 -
1231 - switch (type) {
1232 - case OBJ_BAD:
1233 - case OBJ_COMMIT:
1234 - case OBJ_TREE:
1235 - case OBJ_BLOB:
1236 - case OBJ_TAG:
1237 - break;
1238 - default:
1239 - error("unknown object type %i at offset %"PRIuMAX" in %s",
1240 - type, (uintmax_t)obj_offset, p->pack_name);
1241 - type = OBJ_BAD;
1242 - }
1243 -
1244 -out:
1245 - if (poi_stack != small_poi_stack)
1246 - free(poi_stack);
1247 - return type;
1248 -
1249 -unwind:
1250 - while (poi_stack_nr) {
1251 - obj_offset = poi_stack[--poi_stack_nr];
1252 - type = retry_bad_packed_offset(p, obj_offset);
1253 - if (type > OBJ_NONE)
1254 - goto out;
1255 - }
1256 - type = OBJ_BAD;
1257 - goto out;
1258 -}
1259 -
1260 -static struct hashmap delta_base_cache;
1261 -static size_t delta_base_cached;
1262 -
1263 -static LIST_HEAD(delta_base_cache_lru);
1264 -
1265 -struct delta_base_cache_key {
1266 - struct packed_git *p;
1267 - off_t base_offset;
1268 -};
1269 -
1270 -struct delta_base_cache_entry {
1271 - struct hashmap hash;
1272 - struct delta_base_cache_key key;
1273 - struct list_head lru;
1274 - void *data;
1275 - unsigned long size;
1276 - enum object_type type;
1277 -};
1278 -
1279 -static unsigned int pack_entry_hash(struct packed_git *p, off_t base_offset)
1280 -{
1281 - unsigned int hash;
1282 -
1283 - hash = (unsigned int)(intptr_t)p + (unsigned int)base_offset;
1284 - hash += (hash >> 8) + (hash >> 16);
1285 - return hash;
1286 -}
1287 -
1288 -static struct delta_base_cache_entry *
1289 -get_delta_base_cache_entry(struct packed_git *p, off_t base_offset)
1290 -{
1291 - struct hashmap_entry entry;
1292 - struct delta_base_cache_key key;
1293 -
1294 - if (!delta_base_cache.cmpfn)
1295 - return NULL;
1296 -
1297 - hashmap_entry_init(&entry, pack_entry_hash(p, base_offset));
1298 - key.p = p;
1299 - key.base_offset = base_offset;
1300 - return hashmap_get(&delta_base_cache, &entry, &key);
1301 -}
1302 -
1303 -static int delta_base_cache_key_eq(const struct delta_base_cache_key *a,
1304 - const struct delta_base_cache_key *b)
1305 -{
1306 - return a->p == b->p && a->base_offset == b->base_offset;
1307 -}
1308 -
1309 -static int delta_base_cache_hash_cmp(const void *unused_cmp_data,
1310 - const void *va, const void *vb,
1311 - const void *vkey)
1312 -{
1313 - const struct delta_base_cache_entry *a = va, *b = vb;
1314 - const struct delta_base_cache_key *key = vkey;
1315 - if (key)
1316 - return !delta_base_cache_key_eq(&a->key, key);
1317 - else
1318 - return !delta_base_cache_key_eq(&a->key, &b->key);
1319 -}
1320 -
1321 -static int in_delta_base_cache(struct packed_git *p, off_t base_offset)
1322 -{
1323 - return !!get_delta_base_cache_entry(p, base_offset);
1324 -}
1325 -
1326 -/*
1327 - * Remove the entry from the cache, but do _not_ free the associated
1328 - * entry data. The caller takes ownership of the "data" buffer, and
1329 - * should copy out any fields it wants before detaching.
1330 - */
1331 -static void detach_delta_base_cache_entry(struct delta_base_cache_entry *ent)
1332 -{
1333 - hashmap_remove(&delta_base_cache, ent, &ent->key);
1334 - list_del(&ent->lru);
1335 - delta_base_cached -= ent->size;
1336 - free(ent);
1337 -}
1338 -
1339 -static void *cache_or_unpack_entry(struct packed_git *p, off_t base_offset,
1340 - unsigned long *base_size, enum object_type *type)
1341 -{
1342 - struct delta_base_cache_entry *ent;
1343 -
1344 - ent = get_delta_base_cache_entry(p, base_offset);
1345 - if (!ent)
1346 - return unpack_entry(p, base_offset, type, base_size);
1347 -
1348 - if (type)
1349 - *type = ent->type;
1350 - if (base_size)
1351 - *base_size = ent->size;
1352 - return xmemdupz(ent->data, ent->size);
1353 -}
1354 -
1355 -static inline void release_delta_base_cache(struct delta_base_cache_entry *ent)
1356 -{
1357 - free(ent->data);
1358 - detach_delta_base_cache_entry(ent);
1359 -}
1360 -
1361 -void clear_delta_base_cache(void)
1362 -{
1363 - struct list_head *lru, *tmp;
1364 - list_for_each_safe(lru, tmp, &delta_base_cache_lru) {
1365 - struct delta_base_cache_entry *entry =
1366 - list_entry(lru, struct delta_base_cache_entry, lru);
1367 - release_delta_base_cache(entry);
1368 - }
1369 -}
1370 -
1371 -static void add_delta_base_cache(struct packed_git *p, off_t base_offset,
1372 - void *base, unsigned long base_size, enum object_type type)
1373 -{
1374 - struct delta_base_cache_entry *ent = xmalloc(sizeof(*ent));
1375 - struct list_head *lru, *tmp;
1376 -
1377 - delta_base_cached += base_size;
1378 -
1379 - list_for_each_safe(lru, tmp, &delta_base_cache_lru) {
1380 - struct delta_base_cache_entry *f =
1381 - list_entry(lru, struct delta_base_cache_entry, lru);
1382 - if (delta_base_cached <= delta_base_cache_limit)
1383 - break;
1384 - release_delta_base_cache(f);
1385 - }
1386 -
1387 - ent->key.p = p;
1388 - ent->key.base_offset = base_offset;
1389 - ent->type = type;
1390 - ent->data = base;
1391 - ent->size = base_size;
1392 - list_add_tail(&ent->lru, &delta_base_cache_lru);
1393 -
1394 - if (!delta_base_cache.cmpfn)
1395 - hashmap_init(&delta_base_cache, delta_base_cache_hash_cmp, NULL, 0);
1396 - hashmap_entry_init(ent, pack_entry_hash(p, base_offset));
1397 - hashmap_add(&delta_base_cache, ent);
1398 -}
1399 -
1400 -int packed_object_info(struct packed_git *p, off_t obj_offset,
1401 - struct object_info *oi)
1402 -{
1403 - struct pack_window *w_curs = NULL;
1404 - unsigned long size;
1405 - off_t curpos = obj_offset;
1406 - enum object_type type;
1407 -
1408 - /*
1409 - * We always get the representation type, but only convert it to
1410 - * a "real" type later if the caller is interested.
1411 - */
1412 - if (oi->contentp) {
1413 - *oi->contentp = cache_or_unpack_entry(p, obj_offset, oi->sizep,
1414 - &type);
1415 - if (!*oi->contentp)
1416 - type = OBJ_BAD;
1417 - } else {
1418 - type = unpack_object_header(p, &w_curs, &curpos, &size);
1419 - }
1420 -
1421 - if (!oi->contentp && oi->sizep) {
1422 - if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
1423 - off_t tmp_pos = curpos;
1424 - off_t base_offset = get_delta_base(p, &w_curs, &tmp_pos,
1425 - type, obj_offset);
1426 - if (!base_offset) {
1427 - type = OBJ_BAD;
1428 - goto out;
1429 - }
1430 - *oi->sizep = get_size_from_delta(p, &w_curs, tmp_pos);
1431 - if (*oi->sizep == 0) {
1432 - type = OBJ_BAD;
1433 - goto out;
1434 - }
1435 - } else {
1436 - *oi->sizep = size;
1437 - }
1438 - }
1439 -
1440 - if (oi->disk_sizep) {
1441 - struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
1442 - *oi->disk_sizep = revidx[1].offset - obj_offset;
1443 - }
1444 -
1445 - if (oi->typep || oi->typename) {
1446 - enum object_type ptot;
1447 - ptot = packed_to_object_type(p, obj_offset, type, &w_curs,
1448 - curpos);
1449 - if (oi->typep)
1450 - *oi->typep = ptot;
1451 - if (oi->typename) {
1452 - const char *tn = typename(ptot);
1453 - if (tn)
1454 - strbuf_addstr(oi->typename, tn);
1455 - }
1456 - if (ptot < 0) {
1457 - type = OBJ_BAD;
1458 - goto out;
1459 - }
1460 - }
1461 -
1462 - if (oi->delta_base_sha1) {
1463 - if (type == OBJ_OFS_DELTA || type == OBJ_REF_DELTA) {
1464 - const unsigned char *base;
1465 -
1466 - base = get_delta_base_sha1(p, &w_curs, curpos,
1467 - type, obj_offset);
1468 - if (!base) {
1469 - type = OBJ_BAD;
1470 - goto out;
1471 - }
1472 -
1473 - hashcpy(oi->delta_base_sha1, base);
1474 - } else
1475 - hashclr(oi->delta_base_sha1);
1476 - }
1477 -
1478 - oi->whence = in_delta_base_cache(p, obj_offset) ? OI_DBCACHED :
1479 - OI_PACKED;
1480 -
1481 -out:
1482 - unuse_pack(&w_curs);
1483 - return type;
1484 -}
1485 -
1486 -static void *unpack_compressed_entry(struct packed_git *p,
1487 - struct pack_window **w_curs,
1488 - off_t curpos,
1489 - unsigned long size)
1490 -{
1491 - int st;
1492 - git_zstream stream;
1493 - unsigned char *buffer, *in;
1494 -
1495 - buffer = xmallocz_gently(size);
1496 - if (!buffer)
1497 - return NULL;
1498 - memset(&stream, 0, sizeof(stream));
1499 - stream.next_out = buffer;
1500 - stream.avail_out = size + 1;
1501 -
1502 - git_inflate_init(&stream);
1503 - do {
1504 - in = use_pack(p, w_curs, curpos, &stream.avail_in);
1505 - stream.next_in = in;
1506 - st = git_inflate(&stream, Z_FINISH);
1507 - if (!stream.avail_out)
1508 - break; /* the payload is larger than it should be */
1509 - curpos += stream.next_in - in;
1510 - } while (st == Z_OK || st == Z_BUF_ERROR);
1511 - git_inflate_end(&stream);
1512 - if ((st != Z_STREAM_END) || stream.total_out != size) {
1513 - free(buffer);
1514 - return NULL;
1515 - }
1516 -
1517 - return buffer;
1518 -}
1519 -
1520 -static void *read_object(const unsigned char *sha1, enum object_type *type,
1521 - unsigned long *size);
1522 -
1523 -static void write_pack_access_log(struct packed_git *p, off_t obj_offset)
1524 -{
1525 - static struct trace_key pack_access = TRACE_KEY_INIT(PACK_ACCESS);
1526 - trace_printf_key(&pack_access, "%s %"PRIuMAX"\n",
1527 - p->pack_name, (uintmax_t)obj_offset);
1528 -}
1529 -
1530 -int do_check_packed_object_crc;
1531 -
1532 -#define UNPACK_ENTRY_STACK_PREALLOC 64
1533 -struct unpack_entry_stack_ent {
1534 - off_t obj_offset;
1535 - off_t curpos;
1536 - unsigned long size;
1537 -};
1538 -
1539 -void *unpack_entry(struct packed_git *p, off_t obj_offset,
1540 - enum object_type *final_type, unsigned long *final_size)
1541 -{
1542 - struct pack_window *w_curs = NULL;
1543 - off_t curpos = obj_offset;
1544 - void *data = NULL;
1545 - unsigned long size;
1546 - enum object_type type;
1547 - struct unpack_entry_stack_ent small_delta_stack[UNPACK_ENTRY_STACK_PREALLOC];
1548 - struct unpack_entry_stack_ent *delta_stack = small_delta_stack;
1549 - int delta_stack_nr = 0, delta_stack_alloc = UNPACK_ENTRY_STACK_PREALLOC;
1550 - int base_from_cache = 0;
1551 -
1552 - write_pack_access_log(p, obj_offset);
1553 -
1554 - /* PHASE 1: drill down to the innermost base object */
1555 - for (;;) {
1556 - off_t base_offset;
1557 - int i;
1558 - struct delta_base_cache_entry *ent;
1559 -
1560 - ent = get_delta_base_cache_entry(p, curpos);
1561 - if (ent) {
1562 - type = ent->type;
1563 - data = ent->data;
1564 - size = ent->size;
1565 - detach_delta_base_cache_entry(ent);
1566 - base_from_cache = 1;
1567 - break;
1568 - }
1569 -
1570 - if (do_check_packed_object_crc && p->index_version > 1) {
1571 - struct revindex_entry *revidx = find_pack_revindex(p, obj_offset);
1572 - off_t len = revidx[1].offset - obj_offset;
1573 - if (check_pack_crc(p, &w_curs, obj_offset, len, revidx->nr)) {
1574 - const unsigned char *sha1 =
1575 - nth_packed_object_sha1(p, revidx->nr);
1576 - error("bad packed object CRC for %s",
1577 - sha1_to_hex(sha1));
1578 - mark_bad_packed_object(p, sha1);
1579 - data = NULL;
1580 - goto out;
1581 - }
1582 - }
1583 -
1584 - type = unpack_object_header(p, &w_curs, &curpos, &size);
1585 - if (type != OBJ_OFS_DELTA && type != OBJ_REF_DELTA)
1586 - break;
1587 -
1588 - base_offset = get_delta_base(p, &w_curs, &curpos, type, obj_offset);
1589 - if (!base_offset) {
1590 - error("failed to validate delta base reference "
1591 - "at offset %"PRIuMAX" from %s",
1592 - (uintmax_t)curpos, p->pack_name);
1593 - /* bail to phase 2, in hopes of recovery */
1594 - data = NULL;
1595 - break;
1596 - }
1597 -
1598 - /* push object, proceed to base */
1599 - if (delta_stack_nr >= delta_stack_alloc
1600 - && delta_stack == small_delta_stack) {
1601 - delta_stack_alloc = alloc_nr(delta_stack_nr);
1602 - ALLOC_ARRAY(delta_stack, delta_stack_alloc);
1603 - memcpy(delta_stack, small_delta_stack,
1604 - sizeof(*delta_stack)*delta_stack_nr);
1605 - } else {
1606 - ALLOC_GROW(delta_stack, delta_stack_nr+1, delta_stack_alloc);
1607 - }
1608 - i = delta_stack_nr++;
1609 - delta_stack[i].obj_offset = obj_offset;
1610 - delta_stack[i].curpos = curpos;
1611 - delta_stack[i].size = size;
1612 -
1613 - curpos = obj_offset = base_offset;
1614 - }
1615 -
1616 - /* PHASE 2: handle the base */
1617 - switch (type) {
1618 - case OBJ_OFS_DELTA:
1619 - case OBJ_REF_DELTA:
1620 - if (data)
1621 - die("BUG: unpack_entry: left loop at a valid delta");
1622 - break;
1623 - case OBJ_COMMIT:
1624 - case OBJ_TREE:
1625 - case OBJ_BLOB:
1626 - case OBJ_TAG:
1627 - if (!base_from_cache)
1628 - data = unpack_compressed_entry(p, &w_curs, curpos, size);
1629 - break;
1630 - default:
1631 - data = NULL;
1632 - error("unknown object type %i at offset %"PRIuMAX" in %s",
1633 - type, (uintmax_t)obj_offset, p->pack_name);
1634 - }
1635 -
1636 - /* PHASE 3: apply deltas in order */
1637 -
1638 - /* invariants:
1639 - * 'data' holds the base data, or NULL if there was corruption
1640 - */
1641 - while (delta_stack_nr) {
1642 - void *delta_data;
1643 - void *base = data;
1644 - void *external_base = NULL;
1645 - unsigned long delta_size, base_size = size;
1646 - int i;
1647 -
1648 - data = NULL;
1649 -
1650 - if (base)
1651 - add_delta_base_cache(p, obj_offset, base, base_size, type);
1652 -
1653 - if (!base) {
1654 - /*
1655 - * We're probably in deep shit, but let's try to fetch
1656 - * the required base anyway from another pack or loose.
1657 - * This is costly but should happen only in the presence
1658 - * of a corrupted pack, and is better than failing outright.
1659 - */
1660 - struct revindex_entry *revidx;
1661 - const unsigned char *base_sha1;
1662 - revidx = find_pack_revindex(p, obj_offset);
1663 - if (revidx) {
1664 - base_sha1 = nth_packed_object_sha1(p, revidx->nr);
1665 - error("failed to read delta base object %s"
1666 - " at offset %"PRIuMAX" from %s",
1667 - sha1_to_hex(base_sha1), (uintmax_t)obj_offset,
1668 - p->pack_name);
1669 - mark_bad_packed_object(p, base_sha1);
1670 - base = read_object(base_sha1, &type, &base_size);
1671 - external_base = base;
1672 - }
1673 - }
1674 -
1675 - i = --delta_stack_nr;
1676 - obj_offset = delta_stack[i].obj_offset;
1677 - curpos = delta_stack[i].curpos;
1678 - delta_size = delta_stack[i].size;
1679 -
1680 - if (!base)
1681 - continue;
1682 -
1683 - delta_data = unpack_compressed_entry(p, &w_curs, curpos, delta_size);
1684 -
1685 - if (!delta_data) {
1686 - error("failed to unpack compressed delta "
1687 - "at offset %"PRIuMAX" from %s",
1688 - (uintmax_t)curpos, p->pack_name);
1689 - data = NULL;
1690 - free(external_base);
1691 - continue;
1692 - }
1693 -
1694 - data = patch_delta(base, base_size,
1695 - delta_data, delta_size,
1696 - &size);
1697 -
1698 - /*
1699 - * We could not apply the delta; warn the user, but keep going.
1700 - * Our failure will be noticed either in the next iteration of
1701 - * the loop, or if this is the final delta, in the caller when
1702 - * we return NULL. Those code paths will take care of making
1703 - * a more explicit warning and retrying with another copy of
1704 - * the object.
1705 - */
1706 - if (!data)
1707 - error("failed to apply delta");
1708 -
1709 - free(delta_data);
1710 - free(external_base);
1711 - }
1712 -
1713 - if (final_type)
1714 - *final_type = type;
1715 - if (final_size)
1716 - *final_size = size;
1717 -
1718 -out:
1719 - unuse_pack(&w_curs);
1720 -
1721 - if (delta_stack != small_delta_stack)
1722 - free(delta_stack);
1723 -
1724 - return data;
1725 -}
1726 -
1078 const unsigned char *nth_packed_object_sha1(struct packed_git *p,
1079 uint32_t n)
1080 {
@@ -2083,6 +1434,20 @@ int sha1_object_info(const unsigned char *sha1, unsigned long *sizep)
1434 return type;
1435 }
1436
1437 +static void *read_object(const unsigned char *sha1, enum object_type *type,
1438 + unsigned long *size)
1439 +{
1440 + struct object_info oi = OBJECT_INFO_INIT;
1441 + void *content;
1442 + oi.typep = type;
1443 + oi.sizep = size;
1444 + oi.contentp = &content;
1445 +
1446 + if (sha1_object_info_extended(sha1, &oi, 0) < 0)
1447 + return NULL;
1448 + return content;
1449 +}
1450 +
1451 int pretend_sha1_file(void *buf, unsigned long len, enum object_type type,
1452 unsigned char *sha1)
1453 {
@@ -2101,20 +1466,6 @@ int pretend_sha1_file(void *buf, unsigned long len, enum object_type type,
1466 return 0;
1467 }
1468
2104 -static void *read_object(const unsigned char *sha1, enum object_type *type,
2105 - unsigned long *size)
2106 -{
2107 - struct object_info oi = OBJECT_INFO_INIT;
2108 - void *content;
2109 - oi.typep = type;
2110 - oi.sizep = size;
2111 - oi.contentp = &content;
2112 -
2113 - if (sha1_object_info_extended(sha1, &oi, 0) < 0)
2114 - return NULL;
2115 - return content;
2116 -}
2117 -
1469 /*
1470 * This function dies on corrupt objects; the callers who want to
1471 * deal with them should arrange to call read_object() and give error