Thin pack - create packfile with missing delta base.
This goes together with "rev-list --object-edge" change, to feed pack-objects list of edge commits in addition to the usual object list. Upon seeing such list, pack-objects loosens the usual "self contained delta" constraints, and can produce delta against blobs and trees contained in the edge commits without storing the delta base objects themselves. The resulting packfile is not usable in .git/object/packs, but is a good way to implement "delta-only" transfer. Signed-off-by: Junio C Hamano <junkio@cox.net>
Junio C Hamano committed
Feb 19, 2006 at 14:47 UTC
7a979d99bafae3576d9480007f958eed0c9e0278
1 file changed
+219
-81
pack-objects.c
+219
-81
@@ -3,6 +3,7 @@
3
#include "delta.h"
4
#include "pack.h"
5
#include "csum-file.h"
6
+#include "diff.h"
7
#include <sys/time.h>
8
9
static const char pack_usage[] = "git-pack-objects [-q] [--no-reuse-delta] [--non-empty] [--local] [--incremental] [--window=N] [--depth=N] {--stdout | base-name} < object-list";
@@ -26,6 +27,13 @@ struct object_entry {
27
struct object_entry *delta_sibling; /* other deltified objects who
28
* uses the same base as me
29
*/
30
+ int preferred_base; /* we do not pack this, but is encouraged to
31
+ * be used as the base objectto delta huge
32
+ * objects against.
33
+ */
34
+ int based_on_preferred; /* current delta candidate is a preferred
35
+ * one, or delta against a preferred one.
36
+ */
37
};
38
39
/*
@@ -48,7 +56,7 @@ static int local = 0;
56
static int incremental = 0;
57
static struct object_entry **sorted_by_sha, **sorted_by_type;
58
static struct object_entry *objects = NULL;
51
-static int nr_objects = 0, nr_alloc = 0;
59
+static int nr_objects = 0, nr_alloc = 0, nr_result = 0;
60
static const char *base_name;
61
static unsigned char pack_file_sha1[20];
62
static int progress = 1;
@@ -229,7 +237,8 @@ static int encode_header(enum object_type type, unsigned long size, unsigned cha
237
return n;
238
}
239
232
-static unsigned long write_object(struct sha1file *f, struct object_entry *entry)
240
+static unsigned long write_object(struct sha1file *f,
241
+ struct object_entry *entry)
242
{
243
unsigned long size;
244
char type[10];
@@ -239,6 +248,9 @@ static unsigned long write_object(struct sha1file *f, struct object_entry *entry
248
enum object_type obj_type;
249
int to_reuse = 0;
250
251
+ if (entry->preferred_base)
252
+ return 0;
253
+
254
obj_type = entry->type;
255
if (! entry->in_pack)
256
to_reuse = 0; /* can't reuse what we don't have */
@@ -326,10 +338,11 @@ static void write_pack_file(void)
338
if (!base_name)
339
f = sha1fd(1, "<stdout>");
340
else
329
- f = sha1create("%s-%s.%s", base_name, sha1_to_hex(object_list_sha1), "pack");
341
+ f = sha1create("%s-%s.%s", base_name,
342
+ sha1_to_hex(object_list_sha1), "pack");
343
hdr.hdr_signature = htonl(PACK_SIGNATURE);
344
hdr.hdr_version = htonl(PACK_VERSION);
332
- hdr.hdr_entries = htonl(nr_objects);
345
+ hdr.hdr_entries = htonl(nr_result);
346
sha1write(f, &hdr, sizeof(hdr));
347
offset = sizeof(hdr);
348
for (i = 0; i < nr_objects; i++)
@@ -341,9 +354,10 @@ static void write_pack_file(void)
354
static void write_index_file(void)
355
{
356
int i;
344
- struct sha1file *f = sha1create("%s-%s.%s", base_name, sha1_to_hex(object_list_sha1), "idx");
357
+ struct sha1file *f = sha1create("%s-%s.%s", base_name,
358
+ sha1_to_hex(object_list_sha1), "idx");
359
struct object_entry **list = sorted_by_sha;
346
- struct object_entry **last = list + nr_objects;
360
+ struct object_entry **last = list + nr_result;
361
unsigned int array[256];
362
363
/*
@@ -368,7 +382,7 @@ static void write_index_file(void)
382
* Write the actual SHA1 entries..
383
*/
384
list = sorted_by_sha;
371
- for (i = 0; i < nr_objects; i++) {
385
+ for (i = 0; i < nr_result; i++) {
386
struct object_entry *entry = *list++;
387
unsigned int offset = htonl(entry->offset);
388
sha1write(f, &offset, 4);
@@ -378,27 +392,87 @@ static void write_index_file(void)
392
sha1close(f, NULL, 1);
393
}
394
381
-static int add_object_entry(unsigned char *sha1, unsigned int hash)
395
+static int locate_object_entry_hash(const unsigned char *sha1)
396
+{
397
+ int i;
398
+ unsigned int ui;
399
+ memcpy(&ui, sha1, sizeof(unsigned int));
400
+ i = ui % object_ix_hashsz;
401
+ while (0 < object_ix[i]) {
402
+ if (!memcmp(sha1, objects[object_ix[i]-1].sha1, 20))
403
+ return i;
404
+ if (++i == object_ix_hashsz)
405
+ i = 0;
406
+ }
407
+ return -1 - i;
408
+}
409
+
410
+static struct object_entry *locate_object_entry(const unsigned char *sha1)
411
+{
412
+ int i;
413
+
414
+ if (!object_ix_hashsz)
415
+ return NULL;
416
+
417
+ i = locate_object_entry_hash(sha1);
418
+ if (0 <= i)
419
+ return &objects[object_ix[i]-1];
420
+ return NULL;
421
+}
422
+
423
+static void rehash_objects(void)
424
{
425
+ int i;
426
+ struct object_entry *oe;
427
+
428
+ object_ix_hashsz = nr_objects * 3;
429
+ if (object_ix_hashsz < 1024)
430
+ object_ix_hashsz = 1024;
431
+ object_ix = xrealloc(object_ix, sizeof(int) * object_ix_hashsz);
432
+ object_ix = memset(object_ix, 0, sizeof(int) * object_ix_hashsz);
433
+ for (i = 0, oe = objects; i < nr_objects; i++, oe++) {
434
+ int ix = locate_object_entry_hash(oe->sha1);
435
+ if (0 <= ix)
436
+ continue;
437
+ ix = -1 - ix;
438
+ object_ix[ix] = i + 1;
439
+ }
440
+}
441
+
442
+static int add_object_entry(const unsigned char *sha1, const char *name, int exclude)
443
+{
444
+ unsigned int hash = 0;
445
unsigned int idx = nr_objects;
446
struct object_entry *entry;
447
struct packed_git *p;
448
unsigned int found_offset = 0;
449
struct packed_git *found_pack = NULL;
388
-
389
- for (p = packed_git; p; p = p->next) {
390
- struct pack_entry e;
391
- if (find_pack_entry_one(sha1, &e, p)) {
392
- if (incremental)
393
- return 0;
394
- if (local && !p->pack_local)
395
- return 0;
396
- if (!found_pack) {
397
- found_offset = e.offset;
398
- found_pack = e.p;
450
+ int ix;
451
+
452
+ if (!exclude) {
453
+ for (p = packed_git; p; p = p->next) {
454
+ struct pack_entry e;
455
+ if (find_pack_entry_one(sha1, &e, p)) {
456
+ if (incremental)
457
+ return 0;
458
+ if (local && !p->pack_local)
459
+ return 0;
460
+ if (!found_pack) {
461
+ found_offset = e.offset;
462
+ found_pack = e.p;
463
+ }
464
}
465
}
466
}
467
+ if ((entry = locate_object_entry(sha1)) != NULL)
468
+ goto already_added;
469
+
470
+ while (*name) {
471
+ unsigned char c = *name++;
472
+ if (isspace(c))
473
+ continue;
474
+ hash = hash * 11 + c;
475
+ }
476
477
if (idx >= nr_alloc) {
478
unsigned int needed = (idx + 1024) * 3 / 2;
@@ -406,45 +480,79 @@ static int add_object_entry(unsigned char *sha1, unsigned int hash)
480
nr_alloc = needed;
481
}
482
entry = objects + idx;
483
+ nr_objects = idx + 1;
484
memset(entry, 0, sizeof(*entry));
485
memcpy(entry->sha1, sha1, 20);
486
entry->hash = hash;
412
- if (found_pack) {
413
- entry->in_pack = found_pack;
414
- entry->in_pack_offset = found_offset;
487
+
488
+ if (object_ix_hashsz * 3 <= nr_objects * 4)
489
+ rehash_objects();
490
+ else {
491
+ ix = locate_object_entry_hash(entry->sha1);
492
+ if (0 <= ix)
493
+ die("internal error in object hashing.");
494
+ object_ix[-1 - ix] = idx + 1;
495
+ }
496
+
497
+ already_added:
498
+ if (exclude)
499
+ entry->preferred_base = 1;
500
+ else {
501
+ if (found_pack) {
502
+ entry->in_pack = found_pack;
503
+ entry->in_pack_offset = found_offset;
504
+ }
505
}
416
- nr_objects = idx+1;
506
return 1;
507
}
508
420
-static int locate_object_entry_hash(unsigned char *sha1)
509
+static void add_pbase_tree(struct tree_desc *tree)
510
{
422
- int i;
423
- unsigned int ui;
424
- memcpy(&ui, sha1, sizeof(unsigned int));
425
- i = ui % object_ix_hashsz;
426
- while (0 < object_ix[i]) {
427
- if (!memcmp(sha1, objects[object_ix[i]-1].sha1, 20))
428
- return i;
429
- if (++i == object_ix_hashsz)
430
- i = 0;
511
+ while (tree->size) {
512
+ const unsigned char *sha1;
513
+ const char *name;
514
+ unsigned mode;
515
+ unsigned long size;
516
+ char type[20];
517
+
518
+ sha1 = tree_entry_extract(tree, &name, &mode);
519
+ update_tree_entry(tree);
520
+ if (!has_sha1_file(sha1))
521
+ continue;
522
+ if (sha1_object_info(sha1, type, &size))
523
+ continue;
524
+ add_object_entry(sha1, name, 1);
525
+ if (!strcmp(type, "tree")) {
526
+ struct tree_desc sub;
527
+ void *elem;
528
+ elem = read_sha1_file(sha1, type, &sub.size);
529
+ sub.buf = elem;
530
+ if (sub.buf) {
531
+ add_pbase_tree(&sub);
532
+ free(elem);
533
+ }
534
+ }
535
}
432
- return -1 - i;
536
}
537
435
-static struct object_entry *locate_object_entry(unsigned char *sha1)
538
+static void add_preferred_base(unsigned char *sha1)
539
{
437
- int i = locate_object_entry_hash(sha1);
438
- if (0 <= i)
439
- return &objects[object_ix[i]-1];
440
- return NULL;
540
+ struct tree_desc tree;
541
+ void *elem;
542
+ elem = read_object_with_reference(sha1, "tree", &tree.size, NULL);
543
+ tree.buf = elem;
544
+ if (!tree.buf)
545
+ return;
546
+ add_object_entry(sha1, "", 1);
547
+ add_pbase_tree(&tree);
548
+ free(elem);
549
}
550
551
static void check_object(struct object_entry *entry)
552
{
553
char type[20];
554
447
- if (entry->in_pack) {
555
+ if (entry->in_pack && !entry->preferred_base) {
556
unsigned char base[20];
557
unsigned long size;
558
struct object_entry *base_entry;
@@ -463,7 +571,8 @@ static void check_object(struct object_entry *entry)
571
*/
572
if (!no_reuse_delta &&
573
entry->in_pack_type == OBJ_DELTA &&
466
- (base_entry = locate_object_entry(base))) {
574
+ (base_entry = locate_object_entry(base)) &&
575
+ (!base_entry->preferred_base)) {
576
577
/* Depth value does not matter - find_deltas()
578
* will never consider reused delta as the
@@ -501,25 +610,6 @@ static void check_object(struct object_entry *entry)
610
sha1_to_hex(entry->sha1), type);
611
}
612
504
-static void hash_objects(void)
505
-{
506
- int i;
507
- struct object_entry *oe;
508
-
509
- object_ix_hashsz = nr_objects * 2;
510
- object_ix = xcalloc(sizeof(int), object_ix_hashsz);
511
- for (i = 0, oe = objects; i < nr_objects; i++, oe++) {
512
- int ix = locate_object_entry_hash(oe->sha1);
513
- if (0 <= ix) {
514
- error("the same object '%s' added twice",
515
- sha1_to_hex(oe->sha1));
516
- continue;
517
- }
518
- ix = -1 - ix;
519
- object_ix[ix] = i + 1;
520
- }
521
-}
522
-
613
static unsigned int check_delta_limit(struct object_entry *me, unsigned int n)
614
{
615
struct object_entry *child = me->delta_child;
@@ -538,7 +628,6 @@ static void get_object_details(void)
628
int i;
629
struct object_entry *entry;
630
541
- hash_objects();
631
prepare_pack_ix();
632
for (i = 0, entry = objects; i < nr_objects; i++, entry++)
633
check_object(entry);
@@ -576,6 +665,24 @@ static int sha1_sort(const struct object_entry *a, const struct object_entry *b)
665
return memcmp(a->sha1, b->sha1, 20);
666
}
667
668
+static struct object_entry **create_final_object_list()
669
+{
670
+ struct object_entry **list;
671
+ int i, j;
672
+
673
+ for (i = nr_result = 0; i < nr_objects; i++)
674
+ if (!objects[i].preferred_base)
675
+ nr_result++;
676
+ list = xmalloc(nr_result * sizeof(struct object_entry *));
677
+ for (i = j = 0; i < nr_objects; i++) {
678
+ if (!objects[i].preferred_base)
679
+ list[j++] = objects + i;
680
+ }
681
+ current_sort = sha1_sort;
682
+ qsort(list, nr_result, sizeof(struct object_entry *), sort_comparator);
683
+ return list;
684
+}
685
+
686
static int type_size_sort(const struct object_entry *a, const struct object_entry *b)
687
{
688
if (a->type < b->type)
@@ -586,6 +693,10 @@ static int type_size_sort(const struct object_entry *a, const struct object_entr
693
return -1;
694
if (a->hash > b->hash)
695
return 1;
696
+ if (a->preferred_base < b->preferred_base)
697
+ return -1;
698
+ if (a->preferred_base > b->preferred_base)
699
+ return 1;
700
if (a->size < b->size)
701
return -1;
702
if (a->size > b->size)
@@ -610,6 +721,8 @@ static int try_delta(struct unpacked *cur, struct unpacked *old, unsigned max_de
721
{
722
struct object_entry *cur_entry = cur->entry;
723
struct object_entry *old_entry = old->entry;
724
+ int old_preferred = (old_entry->preferred_base ||
725
+ old_entry->based_on_preferred);
726
unsigned long size, oldsize, delta_size, sizediff;
727
long max_size;
728
void *delta_buf;
@@ -618,9 +731,15 @@ static int try_delta(struct unpacked *cur, struct unpacked *old, unsigned max_de
731
if (cur_entry->type != old_entry->type)
732
return -1;
733
621
- /* If the current object is at edge, take the depth the objects
622
- * that depend on the current object into account -- otherwise
623
- * they would become too deep.
734
+ /* We do not compute delta to *create* objects we are not
735
+ * going to pack.
736
+ */
737
+ if (cur_entry->preferred_base)
738
+ return -1;
739
+
740
+ /* If the current object is at pack edge, take the depth the
741
+ * objects that depend on the current object into account --
742
+ * otherwise they would become too deep.
743
*/
744
if (cur_entry->delta_child) {
745
if (max_depth <= cur_entry->delta_limit)
@@ -646,8 +765,27 @@ static int try_delta(struct unpacked *cur, struct unpacked *old, unsigned max_de
765
* delete).
766
*/
767
max_size = size / 2 - 20;
649
- if (cur_entry->delta)
650
- max_size = cur_entry->delta_size-1;
768
+ if (cur_entry->delta) {
769
+ if (cur_entry->based_on_preferred) {
770
+ if (old_preferred)
771
+ max_size = cur_entry->delta_size-1;
772
+ else
773
+ /* trying with non-preferred one when we
774
+ * already have a delta based on preferred
775
+ * one is pointless.
776
+ */
777
+ return 0;
778
+ }
779
+ else if (!old_preferred)
780
+ max_size = cur_entry->delta_size-1;
781
+ else
782
+ /* otherwise... even if delta with a
783
+ * preferred one produces a bigger result than
784
+ * what we currently have, which is based on a
785
+ * non-preferred one, it is OK.
786
+ */
787
+ ;
788
+ }
789
if (sizediff >= max_size)
790
return -1;
791
delta_buf = diff_delta(old->data, oldsize,
@@ -657,6 +795,7 @@ static int try_delta(struct unpacked *cur, struct unpacked *old, unsigned max_de
795
cur_entry->delta = old_entry;
796
cur_entry->delta_size = delta_size;
797
cur_entry->depth = old_entry->depth + 1;
798
+ cur_entry->based_on_preferred = old_preferred;
799
free(delta_buf);
800
return 0;
801
}
@@ -722,7 +861,7 @@ static void find_deltas(struct object_entry **list, int window, int depth)
861
static void prepare_pack(int window, int depth)
862
{
863
if (progress)
725
- fprintf(stderr, "Packing %d objects", nr_objects);
864
+ fprintf(stderr, "Packing %d objects", nr_result);
865
get_object_details();
866
if (progress)
867
fputc('.', stderr);
@@ -861,8 +1000,6 @@ int main(int argc, char **argv)
1000
gettimeofday(&prev_tv, NULL);
1001
}
1002
while (fgets(line, sizeof(line), stdin) != NULL) {
864
- unsigned int hash;
865
- char *p;
1003
unsigned char sha1[20];
1004
1005
if (progress && (eye_candy <= nr_objects)) {
@@ -881,31 +1018,32 @@ int main(int argc, char **argv)
1018
}
1019
eye_candy += eye_candy_incr;
1020
}
1021
+ if (line[0] == '-') {
1022
+ if (get_sha1_hex(line+1, sha1))
1023
+ die("expected edge sha1, got garbage:\n %s",
1024
+ line+1);
1025
+ add_preferred_base(sha1);
1026
+ continue;
1027
+ }
1028
if (get_sha1_hex(line, sha1))
1029
die("expected sha1, got garbage:\n %s", line);
886
- hash = 0;
887
- p = line+40;
888
- while (*p) {
889
- unsigned char c = *p++;
890
- if (isspace(c))
891
- continue;
892
- hash = hash * 11 + c;
893
- }
894
- add_object_entry(sha1, hash);
1030
+ add_object_entry(sha1, line+40, 0);
1031
}
1032
if (progress)
1033
fprintf(stderr, "Done counting %d objects.\n", nr_objects);
1034
if (non_empty && !nr_objects)
1035
return 0;
1036
901
- sorted_by_sha = create_sorted_list(sha1_sort);
1037
+ sorted_by_sha = create_final_object_list();
1038
SHA1_Init(&ctx);
1039
list = sorted_by_sha;
904
- for (i = 0; i < nr_objects; i++) {
1040
+ for (i = 0; i < nr_result; i++) {
1041
struct object_entry *entry = *list++;
1042
SHA1_Update(&ctx, entry->sha1, 20);
1043
}
1044
SHA1_Final(object_list_sha1, &ctx);
1045
+ if (progress && (nr_objects != nr_result))
1046
+ fprintf(stderr, "Result has %d objects.\n", nr_result);
1047
1048
if (reuse_cached_pack(object_list_sha1, pack_to_stdout))
1049
;
@@ -918,6 +1056,6 @@ int main(int argc, char **argv)
1056
}
1057
if (progress)
1058
fprintf(stderr, "Total %d, written %d (delta %d), reused %d (delta %d)\n",
921
- nr_objects, written, written_delta, reused, reused_delta);
1059
+ nr_result, written, written_delta, reused, reused_delta);
1060
return 0;
1061
}