@matheusbsbs / matheus-bagliano / commits / eb32d236df
Fork 0

introduce delta objects with offset to base

This adds a new object, namely OBJ_OFS_DELTA, renames OBJ_DELTA to OBJ_REF_DELTA to better make the distinction between those two delta objects, and adds support for the handling of those new delta objects in sha1_file.c only. The OBJ_OFS_DELTA contains a relative offset from the delta object's position in a pack instead of the 20-byte SHA1 reference to identify the base object. Since the base is likely to be not so far away, the relative offset is more likely to have a smaller encoding on average than an absolute offset. And for those delta objects the base must always be stored first because there is no way to know the distance of later objects when streaming a pack. Hence this relative offset is always meant to be negative. The offset encoding is slightly denser than the one used for object size -- credits to <linux@horizon.com> (whoever this is) for bringing it to my attention. This allows for pack size reduction between 3.2% (Linux-2.6) to over 5% (linux-historic). Runtime pack access should be faster too since delta replay does skip a search in the pack index for each delta in a chain. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <junkio@cox.net>

Nicolas Pitre committed Sep 21, 2006 at 00:06 UTC eb32d236df0c16b936b04f0c5402addb61cdb311
5 files changed +82 -45
builtin-pack-objects.c
+8 -8
@@ -232,7 +232,7 @@ static int encode_header(enum object_type type, unsigned long size, unsigned cha
232 int n = 1;
233 unsigned char c;
234
235 - if (type < OBJ_COMMIT || type > OBJ_DELTA)
235 + if (type < OBJ_COMMIT || type > OBJ_REF_DELTA)
236 die("bad type %d", type);
237
238 c = (type << 4) | (size & 15);
@@ -297,7 +297,7 @@ static int revalidate_pack_entry(struct object_entry *entry, unsigned char *data
297 used = unpack_object_header_gently(data, len, &type, &size);
298 if (!used)
299 return -1;
300 - if (type == OBJ_DELTA)
300 + if (type == OBJ_REF_DELTA)
301 used += 20; /* skip base object name */
302 data += used;
303 len -= used;
@@ -340,7 +340,7 @@ static unsigned long write_object(struct sha1file *f,
340 obj_type = entry->type;
341 if (! entry->in_pack)
342 to_reuse = 0; /* can't reuse what we don't have */
343 - else if (obj_type == OBJ_DELTA)
343 + else if (obj_type == OBJ_REF_DELTA)
344 to_reuse = 1; /* check_object() decided it for us */
345 else if (obj_type != entry->in_pack_type)
346 to_reuse = 0; /* pack has delta which is unusable */
@@ -380,7 +380,7 @@ static unsigned long write_object(struct sha1file *f,
380 if (entry->delta) {
381 buf = delta_against(buf, size, entry);
382 size = entry->delta_size;
383 - obj_type = OBJ_DELTA;
383 + obj_type = OBJ_REF_DELTA;
384 }
385 /*
386 * The object header is a byte of 'type' followed by zero or
@@ -409,11 +409,11 @@ static unsigned long write_object(struct sha1file *f,
409 sha1write(f, buf, datalen);
410 unuse_packed_git(p);
411 hdrlen = 0; /* not really */
412 - if (obj_type == OBJ_DELTA)
412 + if (obj_type == OBJ_REF_DELTA)
413 reused_delta++;
414 reused++;
415 }
416 - if (obj_type == OBJ_DELTA)
416 + if (obj_type == OBJ_REF_DELTA)
417 written_delta++;
418 written++;
419 return hdrlen + datalen;
@@ -916,7 +916,7 @@ static void check_object(struct object_entry *entry)
916 * delta.
917 */
918 if (!no_reuse_delta &&
919 - entry->in_pack_type == OBJ_DELTA &&
919 + entry->in_pack_type == OBJ_REF_DELTA &&
920 (base_entry = locate_object_entry(base)) &&
921 (!base_entry->preferred_base)) {
922
@@ -929,7 +929,7 @@ static void check_object(struct object_entry *entry)
929 /* uncompressed size of the delta data */
930 entry->size = entry->delta_size = size;
931 entry->delta = base_entry;
932 - entry->type = OBJ_DELTA;
932 + entry->type = OBJ_REF_DELTA;
933
934 entry->delta_sibling = base_entry->delta_child;
935 base_entry->delta_child = entry;
builtin-unpack-objects.c
+1 -1
@@ -241,7 +241,7 @@ static void unpack_one(unsigned nr, unsigned total)
241 case OBJ_TAG:
242 unpack_non_delta_entry(type, size);
243 return;
244 - case OBJ_DELTA:
244 + case OBJ_REF_DELTA:
245 unpack_delta_entry(size);
246 return;
247 default:
cache.h
+3 -2
@@ -274,8 +274,9 @@ enum object_type {
274 OBJ_TREE = 2,
275 OBJ_BLOB = 3,
276 OBJ_TAG = 4,
277 - /* 5/6 for future expansion */
278 - OBJ_DELTA = 7,
277 + /* 5 for future expansion */
278 + OBJ_OFS_DELTA = 6,
279 + OBJ_REF_DELTA = 7,
280 OBJ_BAD,
281 };
282
index-pack.c
+4 -4
@@ -158,7 +158,7 @@ static void *unpack_raw_entry(unsigned long offset,
158 }
159
160 switch (type) {
161 - case OBJ_DELTA:
161 + case OBJ_REF_DELTA:
162 if (pos + 20 >= pack_limit)
163 bad_object(offset, "object extends past end of pack");
164 hashcpy(delta_base, pack_base + pos);
@@ -301,7 +301,7 @@ static void parse_pack_objects(void)
301 data = unpack_raw_entry(offset, &obj->type, &data_size,
302 base_sha1, &offset);
303 obj->real_type = obj->type;
304 - if (obj->type == OBJ_DELTA) {
304 + if (obj->type == OBJ_REF_DELTA) {
305 struct delta_entry *delta = &deltas[nr_deltas++];
306 delta->obj = obj;
307 hashcpy(delta->base_sha1, base_sha1);
@@ -328,7 +328,7 @@ static void parse_pack_objects(void)
328 struct object_entry *obj = &objects[i];
329 int j, first, last;
330
331 - if (obj->type == OBJ_DELTA)
331 + if (obj->type == OBJ_REF_DELTA)
332 continue;
333 if (find_deltas_based_on_sha1(obj->sha1, &first, &last))
334 continue;
@@ -341,7 +341,7 @@ static void parse_pack_objects(void)
341
342 /* Check for unresolved deltas */
343 for (i = 0; i < nr_deltas; i++) {
344 - if (deltas[i].obj->real_type == OBJ_DELTA)
344 + if (deltas[i].obj->real_type == OBJ_REF_DELTA)
345 die("packfile '%s' has unresolved deltas", pack_name);
346 }
347 }
sha1_file.c
+66 -30
@@ -883,26 +883,61 @@ void * unpack_sha1_file(void *map, unsigned long mapsize, char *type, unsigned l
883 return unpack_sha1_rest(&stream, hdr, *size);
884 }
885
886 +static unsigned long get_delta_base(struct packed_git *p,
887 + unsigned long offset,
888 + enum object_type kind,
889 + unsigned long delta_obj_offset,
890 + unsigned long *base_obj_offset)
891 +{
892 + unsigned char *base_info = (unsigned char *) p->pack_base + offset;
893 + unsigned long base_offset;
894 +
895 + /* there must be at least 20 bytes left regardless of delta type */
896 + if (p->pack_size <= offset + 20)
897 + die("truncated pack file");
898 +
899 + if (kind == OBJ_OFS_DELTA) {
900 + unsigned used = 0;
901 + unsigned char c = base_info[used++];
902 + base_offset = c & 127;
903 + while (c & 128) {
904 + base_offset += 1;
905 + if (!base_offset || base_offset & ~(~0UL >> 7))
906 + die("offset value overflow for delta base object");
907 + c = base_info[used++];
908 + base_offset = (base_offset << 7) + (c & 127);
909 + }
910 + base_offset = delta_obj_offset - base_offset;
911 + if (base_offset >= delta_obj_offset)
912 + die("delta base offset out of bound");
913 + offset += used;
914 + } else if (kind == OBJ_REF_DELTA) {
915 + /* The base entry _must_ be in the same pack */
916 + base_offset = find_pack_entry_one(base_info, p);
917 + if (!base_offset)
918 + die("failed to find delta-pack base object %s",
919 + sha1_to_hex(base_info));
920 + offset += 20;
921 + } else
922 + die("I am totally screwed");
923 + *base_obj_offset = base_offset;
924 + return offset;
925 +}
926 +
927 /* forward declaration for a mutually recursive function */
928 static int packed_object_info(struct packed_git *p, unsigned long offset,
929 char *type, unsigned long *sizep);
930
931 static int packed_delta_info(struct packed_git *p,
932 unsigned long offset,
933 + enum object_type kind,
934 + unsigned long obj_offset,
935 char *type,
936 unsigned long *sizep)
937 {
938 unsigned long base_offset;
896 - unsigned char *base_sha1 = (unsigned char *) p->pack_base + offset;
939
898 - if (p->pack_size < offset + 20)
899 - die("truncated pack file");
900 - /* The base entry _must_ be in the same pack */
901 - base_offset = find_pack_entry_one(base_sha1, p);
902 - if (!base_offset)
903 - die("failed to find delta-pack base object %s",
904 - sha1_to_hex(base_sha1));
905 - offset += 20;
940 + offset = get_delta_base(p, offset, kind, obj_offset, &base_offset);
941
942 /* We choose to only get the type of the base object and
943 * ignore potentially corrupt pack file that expects the delta
@@ -975,7 +1010,7 @@ int check_reuse_pack_delta(struct packed_git *p, unsigned long offset,
1010 use_packed_git(p);
1011 ptr = offset;
1012 ptr = unpack_object_header(p, ptr, kindp, sizep);
978 - if (*kindp != OBJ_DELTA)
1013 + if (*kindp != OBJ_REF_DELTA)
1014 goto done;
1015 hashcpy(base, (unsigned char *) p->pack_base + ptr);
1016 status = 0;
@@ -992,11 +1027,12 @@ void packed_object_info_detail(struct packed_git *p,
1027 unsigned int *delta_chain_length,
1028 unsigned char *base_sha1)
1029 {
995 - unsigned long val;
1030 + unsigned long obj_offset, val;
1031 unsigned char *next_sha1;
1032 enum object_type kind;
1033
1034 *delta_chain_length = 0;
1035 + obj_offset = offset;
1036 offset = unpack_object_header(p, offset, &kind, size);
1037
1038 for (;;) {
@@ -1011,7 +1047,13 @@ void packed_object_info_detail(struct packed_git *p,
1047 strcpy(type, type_names[kind]);
1048 *store_size = 0; /* notyet */
1049 return;
1014 - case OBJ_DELTA:
1050 + case OBJ_OFS_DELTA:
1051 + get_delta_base(p, offset, kind, obj_offset, &offset);
1052 + if (*delta_chain_length == 0) {
1053 + /* TODO: find base_sha1 as pointed by offset */
1054 + }
1055 + break;
1056 + case OBJ_REF_DELTA:
1057 if (p->pack_size <= offset + 20)
1058 die("pack file %s records an incomplete delta base",
1059 p->pack_name);
@@ -1021,6 +1063,7 @@ void packed_object_info_detail(struct packed_git *p,
1063 offset = find_pack_entry_one(next_sha1, p);
1064 break;
1065 }
1066 + obj_offset = offset;
1067 offset = unpack_object_header(p, offset, &kind, &val);
1068 (*delta_chain_length)++;
1069 }
@@ -1029,15 +1072,15 @@ void packed_object_info_detail(struct packed_git *p,
1072 static int packed_object_info(struct packed_git *p, unsigned long offset,
1073 char *type, unsigned long *sizep)
1074 {
1032 - unsigned long size;
1075 + unsigned long size, obj_offset = offset;
1076 enum object_type kind;
1077
1078 offset = unpack_object_header(p, offset, &kind, &size);
1079
1037 - if (kind == OBJ_DELTA)
1038 - return packed_delta_info(p, offset, type, sizep);
1039 -
1080 switch (kind) {
1081 + case OBJ_OFS_DELTA:
1082 + case OBJ_REF_DELTA:
1083 + return packed_delta_info(p, offset, kind, obj_offset, type, sizep);
1084 case OBJ_COMMIT:
1085 case OBJ_TREE:
1086 case OBJ_BLOB:
@@ -1083,23 +1126,15 @@ static void *unpack_compressed_entry(struct packed_git *p,
1126 static void *unpack_delta_entry(struct packed_git *p,
1127 unsigned long offset,
1128 unsigned long delta_size,
1129 + enum object_type kind,
1130 + unsigned long obj_offset,
1131 char *type,
1132 unsigned long *sizep)
1133 {
1134 void *delta_data, *result, *base;
1135 unsigned long result_size, base_size, base_offset;
1091 - unsigned char *base_sha1;
1092 -
1093 - if (p->pack_size < offset + 20)
1094 - die("truncated pack file");
1095 - /* The base entry _must_ be in the same pack */
1096 - base_sha1 = (unsigned char*)p->pack_base + offset;
1097 - base_offset = find_pack_entry_one(base_sha1, p);
1098 - if (!base_offset)
1099 - die("failed to find delta-pack base object %s",
1100 - sha1_to_hex(base_sha1));
1101 - offset += 20;
1136
1137 + offset = get_delta_base(p, offset, kind, obj_offset, &base_offset);
1138 base = unpack_entry_gently(p, base_offset, type, &base_size);
1139 if (!base)
1140 die("failed to read delta base object at %lu from %s",
@@ -1136,13 +1171,14 @@ static void *unpack_entry(struct pack_entry *entry,
1171 void *unpack_entry_gently(struct packed_git *p, unsigned long offset,
1172 char *type, unsigned long *sizep)
1173 {
1139 - unsigned long size;
1174 + unsigned long size, obj_offset = offset;
1175 enum object_type kind;
1176
1177 offset = unpack_object_header(p, offset, &kind, &size);
1178 switch (kind) {
1144 - case OBJ_DELTA:
1145 - return unpack_delta_entry(p, offset, size, type, sizep);
1179 + case OBJ_OFS_DELTA:
1180 + case OBJ_REF_DELTA:
1181 + return unpack_delta_entry(p, offset, size, kind, obj_offset, type, sizep);
1182 case OBJ_COMMIT:
1183 case OBJ_TREE:
1184 case OBJ_BLOB: