@matheusbsbs / matheus-bagliano / commits / 92392b4a45

Fork 0

index-pack: Honor core.deltaBaseCacheLimit when resolving deltas

If we are trying to resolve deltas for a long delta chain composed of multi-megabyte objects we can easily run into requiring 500M+ of memory to hold each object in the chain on the call stack while we recurse into the dependent objects and resolve them. We now use a simple delta cache that discards objects near the bottom of the call stack first, as they are the most least recently used objects in this current delta chain. If we recurse out of a chain we may find the base object is no longer available, as it was free'd to keep memory under the deltaBaseCacheLimit. In such cases we must unpack the base object again, which will require recursing back to the root of the top of the delta chain as we released that root first. The astute reader will probably realize that we can still exceed the delta base cache limit, but this happens only if the most recent base plus the delta plus the inflated dependent sum up to more than the base cache limit. Due to the way patch_delta is currently implemented we cannot operate in less memory anyway. Signed-off-by: Shawn O. Pearce <spearce@spearce.org> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Shawn O. Pearce committed Jul 15, 2008 at 04:45 UTC 92392b4a4530056918174988200d7c10286a4acd

1 file changed +46 -2

index-pack.c

+46 -2

@@ -52,6 +52,7 @@ struct delta_entry

52 static struct object_entry *objects;

53 static struct delta_entry *deltas;

54 static struct base_data *base_cache;

55 +static size_t base_cache_used;

56 static int nr_objects;

57 static int nr_deltas;

58 static int nr_resolved_deltas;

@@ -219,6 +220,20 @@ static void bad_object(unsigned long offset, const char *format, ...)

220 die("pack has bad object at offset %lu: %s", offset, buf);

221 }

222

223 +static void prune_base_data(struct base_data *retain)

224 +{

225 + struct base_data *b = base_cache;

226 + for (b = base_cache;

227 + base_cache_used > delta_base_cache_limit && b;

228 + b = b->child) {

229 + if (b->data && b != retain) {

230 + free(b->data);

231 + b->data = NULL;

232 + base_cache_used -= b->size;

233 + }

234 + }

235 +}

236 +

237 static void link_base_data(struct base_data *base, struct base_data *c)

238 {

239 if (base)

@@ -228,6 +243,8 @@ static void link_base_data(struct base_data *base, struct base_data *c)

243

244 c->base = base;

245 c->child = NULL;

246 + base_cache_used += c->size;

247 + prune_base_data(c);

248 }

249

250 static void unlink_base_data(struct base_data *c)

@@ -237,7 +254,10 @@ static void unlink_base_data(struct base_data *c)

254 base->child = NULL;

255 else

256 base_cache = NULL;

240 - free(c->data);

257 + if (c->data) {

258 + free(c->data);

259 + base_cache_used -= c->size;

260 + }

261 }

262

263 static void *unpack_entry_data(unsigned long offset, unsigned long size)

@@ -455,6 +475,30 @@ static void sha1_object(const void *data, unsigned long size,

475 }

476 }

477

478 +static void *get_base_data(struct base_data *c)

479 +{

480 + if (!c->data) {

481 + struct object_entry *obj = c->obj;

482 +

483 + if (obj->type == OBJ_REF_DELTA || obj->type == OBJ_OFS_DELTA) {

484 + void *base = get_base_data(c->base);

485 + void *raw = get_data_from_pack(obj);

486 + c->data = patch_delta(

487 + base, c->base->size,

488 + raw, obj->size,

489 + &c->size);

490 + free(raw);

491 + if (!c->data)

492 + bad_object(obj->idx.offset, "failed to apply delta");

493 + } else

494 + c->data = get_data_from_pack(obj);

495 +

496 + base_cache_used += c->size;

497 + prune_base_data(c);

498 + }

499 + return c->data;

500 +}

501 +

502 static void resolve_delta(struct object_entry *delta_obj,

503 struct base_data *base_obj, enum object_type type)

504 {

@@ -467,7 +511,7 @@ static void resolve_delta(struct object_entry *delta_obj,

511 delta_obj->real_type = type;

512 delta_data = get_data_from_pack(delta_obj);

513 delta_size = delta_obj->size;

470 - result.data = patch_delta(base_obj->data, base_obj->size,

514 + result.data = patch_delta(get_base_data(base_obj), base_obj->size,

515 delta_data, delta_size,

516 &result.size);

517 free(delta_data);