@matheusbsbs / matheus-bagliano / commits / 5c49c11686
Fork 0

pack-objects: better check_object() performances

With large amount of objects, check_object() is really trashing the pack sliding map and the filesystem cache. It has a completely random access pattern especially with old objects where delta replay jumps back and forth all over the pack. This patch improves things by: 1) sorting objects by their offset in pack before calling check_object() so the pack access pattern is linear; 2) recording the object type at add_object_entry() time since it is already known in most cases; 3) recording the pack offset even for preferred_base objects; 4) avoid calling sha1_object_info() if all possible. This limits pack accesses to the bare minimum and makes them perfectly linear. In the process check_object() was made more clear (to me at least). Note: I thought about walking the sorted_by_offset list backward in get_object_details() so if a pack happens to be larger than the available file cache, then the cache would have been populated with useful data from the beginning of the pack already when find_deltas() is called. Strangely, testing (on Linux) showed absolutely no performance difference. Signed-off-by: Nicolas Pitre <nico@cam.org> Signed-off-by: Junio C Hamano <junkio@cox.net>

Nicolas Pitre committed Apr 16, 2007 at 12:32 UTC 5c49c11686df9d1c27a194349d0b2092e6446f42
1 file changed +126 -80
builtin-pack-objects.c
+126 -80
@@ -813,7 +813,8 @@ static unsigned name_hash(const char *name)
813 return hash;
814 }
815
816 -static int add_object_entry(const unsigned char *sha1, unsigned hash, int exclude)
816 +static int add_object_entry(const unsigned char *sha1, enum object_type type,
817 + unsigned hash, int exclude)
818 {
819 struct object_entry *entry;
820 struct packed_git *p, *found_pack = NULL;
@@ -831,19 +832,19 @@ static int add_object_entry(const unsigned char *sha1, unsigned hash, int exclud
832 return 0;
833 }
834
834 - if (!exclude) {
835 - for (p = packed_git; p; p = p->next) {
836 - off_t offset = find_pack_entry_one(sha1, p);
837 - if (offset) {
838 - if (incremental)
839 - return 0;
840 - if (local && !p->pack_local)
841 - return 0;
842 - if (!found_pack) {
843 - found_offset = offset;
844 - found_pack = p;
845 - }
835 + for (p = packed_git; p; p = p->next) {
836 + off_t offset = find_pack_entry_one(sha1, p);
837 + if (offset) {
838 + if (!found_pack) {
839 + found_offset = offset;
840 + found_pack = p;
841 }
842 + if (exclude)
843 + break;
844 + if (incremental)
845 + return 0;
846 + if (local && !p->pack_local)
847 + return 0;
848 }
849 }
850
@@ -856,6 +857,8 @@ static int add_object_entry(const unsigned char *sha1, unsigned hash, int exclud
857 memset(entry, 0, sizeof(*entry));
858 hashcpy(entry->sha1, sha1);
859 entry->hash = hash;
860 + if (type)
861 + entry->type = type;
862 if (exclude)
863 entry->preferred_base = 1;
864 else
@@ -1008,7 +1011,9 @@ static void add_pbase_object(struct tree_desc *tree,
1011 return;
1012 if (name[cmplen] != '/') {
1013 unsigned hash = name_hash(fullname);
1011 - add_object_entry(entry.sha1, hash, 1);
1014 + add_object_entry(entry.sha1,
1015 + S_ISDIR(entry.mode) ? OBJ_TREE : OBJ_BLOB,
1016 + hash, 1);
1017 return;
1018 }
1019 if (S_ISDIR(entry.mode)) {
@@ -1079,7 +1084,7 @@ static void add_preferred_base_object(const char *name, unsigned hash)
1084 cmplen = name_cmp_len(name);
1085 for (it = pbase_tree; it; it = it->next) {
1086 if (cmplen == 0) {
1082 - add_object_entry(it->pcache.sha1, 0, 1);
1087 + add_object_entry(it->pcache.sha1, OBJ_TREE, 0, 1);
1088 }
1089 else {
1090 struct tree_desc tree;
@@ -1121,87 +1126,105 @@ static void add_preferred_base(unsigned char *sha1)
1126
1127 static void check_object(struct object_entry *entry)
1128 {
1124 - if (entry->in_pack && !entry->preferred_base) {
1129 + if (entry->in_pack) {
1130 struct packed_git *p = entry->in_pack;
1131 struct pack_window *w_curs = NULL;
1127 - unsigned long size, used;
1132 + const unsigned char *base_ref = NULL;
1133 + struct object_entry *base_entry;
1134 + unsigned long used, used_0;
1135 unsigned int avail;
1129 - unsigned char *buf;
1130 - struct object_entry *base_entry = NULL;
1136 + off_t ofs;
1137 + unsigned char *buf, c;
1138
1139 buf = use_pack(p, &w_curs, entry->in_pack_offset, &avail);
1140
1134 - /* We want in_pack_type even if we do not reuse delta.
1141 + /*
1142 + * We want in_pack_type even if we do not reuse delta.
1143 * There is no point not reusing non-delta representations.
1144 */
1145 used = unpack_object_header_gently(buf, avail,
1138 - &entry->in_pack_type, &size);
1146 + &entry->in_pack_type,
1147 + &entry->size);
1148
1140 - /* Check if it is delta, and the base is also an object
1141 - * we are going to pack. If so we will reuse the existing
1142 - * delta.
1149 + /*
1150 + * Determine if this is a delta and if so whether we can
1151 + * reuse it or not. Otherwise let's find out as cheaply as
1152 + * possible what the actual type and size for this object is.
1153 */
1144 - if (!no_reuse_delta) {
1145 - unsigned char c;
1146 - const unsigned char *base_name;
1147 - off_t ofs;
1148 - unsigned long used_0;
1149 - /* there is at least 20 bytes left in the pack */
1150 - switch (entry->in_pack_type) {
1151 - case OBJ_REF_DELTA:
1152 - base_name = use_pack(p, &w_curs,
1153 - entry->in_pack_offset + used, NULL);
1154 - used += 20;
1155 - break;
1156 - case OBJ_OFS_DELTA:
1157 - buf = use_pack(p, &w_curs,
1158 - entry->in_pack_offset + used, NULL);
1159 - used_0 = 0;
1160 - c = buf[used_0++];
1161 - ofs = c & 127;
1162 - while (c & 128) {
1163 - ofs += 1;
1164 - if (!ofs || MSB(ofs, 7))
1165 - die("delta base offset overflow in pack for %s",
1166 - sha1_to_hex(entry->sha1));
1167 - c = buf[used_0++];
1168 - ofs = (ofs << 7) + (c & 127);
1169 - }
1170 - if (ofs >= entry->in_pack_offset)
1171 - die("delta base offset out of bound for %s",
1154 + switch (entry->in_pack_type) {
1155 + default:
1156 + /* Not a delta hence we've already got all we need. */
1157 + entry->type = entry->in_pack_type;
1158 + entry->in_pack_header_size = used;
1159 + unuse_pack(&w_curs);
1160 + return;
1161 + case OBJ_REF_DELTA:
1162 + if (!no_reuse_delta && !entry->preferred_base)
1163 + base_ref = use_pack(p, &w_curs,
1164 + entry->in_pack_offset + used, NULL);
1165 + entry->in_pack_header_size = used + 20;
1166 + break;
1167 + case OBJ_OFS_DELTA:
1168 + buf = use_pack(p, &w_curs,
1169 + entry->in_pack_offset + used, NULL);
1170 + used_0 = 0;
1171 + c = buf[used_0++];
1172 + ofs = c & 127;
1173 + while (c & 128) {
1174 + ofs += 1;
1175 + if (!ofs || MSB(ofs, 7))
1176 + die("delta base offset overflow in pack for %s",
1177 sha1_to_hex(entry->sha1));
1173 - ofs = entry->in_pack_offset - ofs;
1174 - base_name = find_packed_object_name(p, ofs);
1175 - used += used_0;
1176 - break;
1177 - default:
1178 - base_name = NULL;
1178 + c = buf[used_0++];
1179 + ofs = (ofs << 7) + (c & 127);
1180 }
1180 - if (base_name)
1181 - base_entry = locate_object_entry(base_name);
1181 + if (ofs >= entry->in_pack_offset)
1182 + die("delta base offset out of bound for %s",
1183 + sha1_to_hex(entry->sha1));
1184 + ofs = entry->in_pack_offset - ofs;
1185 + if (!no_reuse_delta && !entry->preferred_base)
1186 + base_ref = find_packed_object_name(p, ofs);
1187 + entry->in_pack_header_size = used + used_0;
1188 + break;
1189 }
1183 - unuse_pack(&w_curs);
1184 - entry->in_pack_header_size = used;
1190
1186 - if (base_entry) {
1187 -
1188 - /* Depth value does not matter - find_deltas()
1189 - * will never consider reused delta as the
1190 - * base object to deltify other objects
1191 - * against, in order to avoid circular deltas.
1191 + if (base_ref && (base_entry = locate_object_entry(base_ref))) {
1192 + /*
1193 + * If base_ref was set above that means we wish to
1194 + * reuse delta data, and we even found that base
1195 + * in the list of objects we want to pack. Goodie!
1196 + *
1197 + * Depth value does not matter - find_deltas() will
1198 + * never consider reused delta as the base object to
1199 + * deltify other objects against, in order to avoid
1200 + * circular deltas.
1201 */
1193 -
1194 - /* uncompressed size of the delta data */
1195 - entry->size = size;
1196 - entry->delta = base_entry;
1202 entry->type = entry->in_pack_type;
1198 -
1203 + entry->delta = base_entry;
1204 entry->delta_sibling = base_entry->delta_child;
1205 base_entry->delta_child = entry;
1206 + unuse_pack(&w_curs);
1207 + return;
1208 + }
1209
1210 + if (entry->type) {
1211 + /*
1212 + * This must be a delta and we already know what the
1213 + * final object type is. Let's extract the actual
1214 + * object size from the delta header.
1215 + */
1216 + entry->size = get_size_from_delta(p, &w_curs,
1217 + entry->in_pack_offset + entry->in_pack_header_size);
1218 + unuse_pack(&w_curs);
1219 return;
1220 }
1204 - /* Otherwise we would do the usual */
1221 +
1222 + /*
1223 + * No choice but to fall back to the recursive delta walk
1224 + * with sha1_object_info() to find about the object type
1225 + * at this point...
1226 + */
1227 + unuse_pack(&w_curs);
1228 }
1229
1230 entry->type = sha1_object_info(entry->sha1, &entry->size);
@@ -1210,14 +1233,37 @@ static void check_object(struct object_entry *entry)
1233 sha1_to_hex(entry->sha1));
1234 }
1235
1236 +static int pack_offset_sort(const void *_a, const void *_b)
1237 +{
1238 + const struct object_entry *a = *(struct object_entry **)_a;
1239 + const struct object_entry *b = *(struct object_entry **)_b;
1240 +
1241 + /* avoid filesystem trashing with loose objects */
1242 + if (!a->in_pack && !b->in_pack)
1243 + return hashcmp(a->sha1, b->sha1);
1244 +
1245 + if (a->in_pack < b->in_pack)
1246 + return -1;
1247 + if (a->in_pack > b->in_pack)
1248 + return 1;
1249 + return a->in_pack_offset < b->in_pack_offset ? -1 :
1250 + (a->in_pack_offset > b->in_pack_offset);
1251 +}
1252 +
1253 static void get_object_details(void)
1254 {
1255 uint32_t i;
1216 - struct object_entry *entry;
1256 + struct object_entry **sorted_by_offset;
1257 +
1258 + sorted_by_offset = xcalloc(nr_objects, sizeof(struct object_entry *));
1259 + for (i = 0; i < nr_objects; i++)
1260 + sorted_by_offset[i] = objects + i;
1261 + qsort(sorted_by_offset, nr_objects, sizeof(*sorted_by_offset), pack_offset_sort);
1262
1263 prepare_pack_ix();
1219 - for (i = 0, entry = objects; i < nr_objects; i++, entry++)
1220 - check_object(entry);
1264 + for (i = 0; i < nr_objects; i++)
1265 + check_object(sorted_by_offset[i]);
1266 + free(sorted_by_offset);
1267 }
1268
1269 static int type_size_sort(const void *_a, const void *_b)
@@ -1520,20 +1566,20 @@ static void read_object_list_from_stdin(void)
1566
1567 hash = name_hash(line+41);
1568 add_preferred_base_object(line+41, hash);
1523 - add_object_entry(sha1, hash, 0);
1569 + add_object_entry(sha1, 0, hash, 0);
1570 }
1571 }
1572
1573 static void show_commit(struct commit *commit)
1574 {
1529 - add_object_entry(commit->object.sha1, 0, 0);
1575 + add_object_entry(commit->object.sha1, OBJ_COMMIT, 0, 0);
1576 }
1577
1578 static void show_object(struct object_array_entry *p)
1579 {
1580 unsigned hash = name_hash(p->name);
1581 add_preferred_base_object(p->name, hash);
1536 - add_object_entry(p->item->sha1, hash, 0);
1582 + add_object_entry(p->item->sha1, p->item->type, hash, 0);
1583 }
1584
1585 static void show_edge(struct commit *commit)