@matheusbsbs / matheus-bagliano / commits / 0660626caf
Fork 0

binary diff: further updates.

This updates the user interface and generated diff data format. * "diff --binary" is used to signal that we want an e-mailable binary patch. It implies --full-index and -p. * "apply --allow-binary-replacement" acquired a short synonym "apply --binary". * After the "GIT binary patch\n" header line there is a token to record which binary patch mechanism was used, so that we can extend it later. Currently there are two mechanisms defined: "literal" and "delta". The former records the deflated postimage and the latter records the deflated delta from the preimage to postimage. For purely implementation convenience, I added the deflated length after these "literal/delta" tokens (otherwise the decoding side needs to guess and reallocate the buffer while inflating). Improvement patches are very welcomed. Signed-off-by: Junio C Hamano <junkio@cox.net>

Junio C Hamano committed May 5, 2006 at 02:41 UTC 0660626caff6ac3745cd7b7908a2ca540141a2ec
5 files changed +304 -96
apply.c
+97 -33
@@ -114,6 +114,9 @@ struct patch {
114 char *new_name, *old_name, *def_name;
115 unsigned int old_mode, new_mode;
116 int is_rename, is_copy, is_new, is_delete, is_binary;
117 +#define BINARY_DELTA_DEFLATED 1
118 +#define BINARY_LITERAL_DEFLATED 2
119 + unsigned long deflate_origlen;
120 int lines_added, lines_deleted;
121 int score;
122 struct fragment *fragments;
@@ -969,9 +972,11 @@ static inline int metadata_changes(struct patch *patch)
972
973 static int parse_binary(char *buffer, unsigned long size, struct patch *patch)
974 {
972 - /* We have read "GIT binary patch\n"; what follows is a
973 - * sequence of 'length-byte' followed by base-85 encoded
974 - * delta data.
975 + /* We have read "GIT binary patch\n"; what follows is a line
976 + * that says the patch method (currently, either "deflated
977 + * literal" or "deflated delta") and the length of data before
978 + * deflating; a sequence of 'length-byte' followed by base-85
979 + * encoded data follows.
980 *
981 * Each 5-byte sequence of base-85 encodes up to 4 bytes,
982 * and we would limit the patch line to 66 characters,
@@ -982,11 +987,27 @@ static int parse_binary(char *buffer, unsigned long size, struct patch *patch)
987 */
988 int llen, used;
989 struct fragment *fragment;
985 - char *delta = NULL;
990 + char *data = NULL;
991
987 - patch->is_binary = 1;
992 patch->fragments = fragment = xcalloc(1, sizeof(*fragment));
989 - used = 0;
993 +
994 + /* Grab the type of patch */
995 + llen = linelen(buffer, size);
996 + used = llen;
997 + linenr++;
998 +
999 + if (!strncmp(buffer, "delta ", 6)) {
1000 + patch->is_binary = BINARY_DELTA_DEFLATED;
1001 + patch->deflate_origlen = strtoul(buffer + 6, NULL, 10);
1002 + }
1003 + else if (!strncmp(buffer, "literal ", 8)) {
1004 + patch->is_binary = BINARY_LITERAL_DEFLATED;
1005 + patch->deflate_origlen = strtoul(buffer + 8, NULL, 10);
1006 + }
1007 + else
1008 + return error("unrecognized binary patch at line %d: %.*s",
1009 + linenr-1, llen-1, buffer);
1010 + buffer += llen;
1011 while (1) {
1012 int byte_length, max_byte_length, newsize;
1013 llen = linelen(buffer, size);
@@ -1015,8 +1036,8 @@ static int parse_binary(char *buffer, unsigned long size, struct patch *patch)
1036 byte_length <= max_byte_length - 4)
1037 goto corrupt;
1038 newsize = fragment->size + byte_length;
1018 - delta = xrealloc(delta, newsize);
1019 - if (decode_85(delta + fragment->size,
1039 + data = xrealloc(data, newsize);
1040 + if (decode_85(data + fragment->size,
1041 buffer + 1,
1042 byte_length))
1043 goto corrupt;
@@ -1024,7 +1045,7 @@ static int parse_binary(char *buffer, unsigned long size, struct patch *patch)
1045 buffer += llen;
1046 size -= llen;
1047 }
1027 - fragment->patch = delta;
1048 + fragment->patch = data;
1049 return used;
1050 corrupt:
1051 return error("corrupt binary patch at line %d: %.*s",
@@ -1425,6 +1446,61 @@ static int apply_one_fragment(struct buffer_desc *desc, struct fragment *frag)
1446 return offset;
1447 }
1448
1449 +static char *inflate_it(const void *data, unsigned long size,
1450 + unsigned long inflated_size)
1451 +{
1452 + z_stream stream;
1453 + void *out;
1454 + int st;
1455 +
1456 + memset(&stream, 0, sizeof(stream));
1457 +
1458 + stream.next_in = (unsigned char *)data;
1459 + stream.avail_in = size;
1460 + stream.next_out = out = xmalloc(inflated_size);
1461 + stream.avail_out = inflated_size;
1462 + inflateInit(&stream);
1463 + st = inflate(&stream, Z_FINISH);
1464 + if ((st != Z_STREAM_END) || stream.total_out != inflated_size) {
1465 + free(out);
1466 + return NULL;
1467 + }
1468 + return out;
1469 +}
1470 +
1471 +static int apply_binary_fragment(struct buffer_desc *desc, struct patch *patch)
1472 +{
1473 + unsigned long dst_size;
1474 + struct fragment *fragment = patch->fragments;
1475 + void *data;
1476 + void *result;
1477 +
1478 + data = inflate_it(fragment->patch, fragment->size,
1479 + patch->deflate_origlen);
1480 + if (!data)
1481 + return error("corrupt patch data");
1482 + switch (patch->is_binary) {
1483 + case BINARY_DELTA_DEFLATED:
1484 + result = patch_delta(desc->buffer, desc->size,
1485 + data,
1486 + patch->deflate_origlen,
1487 + &dst_size);
1488 + free(desc->buffer);
1489 + desc->buffer = result;
1490 + free(data);
1491 + break;
1492 + case BINARY_LITERAL_DEFLATED:
1493 + free(desc->buffer);
1494 + desc->buffer = data;
1495 + dst_size = patch->deflate_origlen;
1496 + break;
1497 + }
1498 + if (!desc->buffer)
1499 + return -1;
1500 + desc->size = desc->alloc = dst_size;
1501 + return 0;
1502 +}
1503 +
1504 static int apply_binary(struct buffer_desc *desc, struct patch *patch)
1505 {
1506 const char *name = patch->old_name ? patch->old_name : patch->new_name;
@@ -1466,18 +1542,20 @@ static int apply_binary(struct buffer_desc *desc, struct patch *patch)
1542 "'%s' but it is not empty", name);
1543 }
1544
1469 - if (desc->buffer) {
1545 + get_sha1_hex(patch->new_sha1_prefix, sha1);
1546 + if (!memcmp(sha1, null_sha1, 20)) {
1547 free(desc->buffer);
1548 desc->alloc = desc->size = 0;
1472 - }
1473 - get_sha1_hex(patch->new_sha1_prefix, sha1);
1474 - if (!memcmp(sha1, null_sha1, 20))
1549 + desc->buffer = NULL;
1550 return 0; /* deletion patch */
1551 + }
1552
1553 if (has_sha1_file(sha1)) {
1554 + /* We already have the postimage */
1555 char type[10];
1556 unsigned long size;
1557
1558 + free(desc->buffer);
1559 desc->buffer = read_sha1_file(sha1, type, &size);
1560 if (!desc->buffer)
1561 return error("the necessary postimage %s for "
@@ -1486,28 +1564,13 @@ static int apply_binary(struct buffer_desc *desc, struct patch *patch)
1564 desc->alloc = desc->size = size;
1565 }
1566 else {
1489 - char type[10];
1490 - unsigned long src_size, dst_size;
1491 - void *src;
1492 -
1493 - get_sha1_hex(patch->old_sha1_prefix, sha1);
1494 - src = read_sha1_file(sha1, type, &src_size);
1495 - if (!src)
1496 - return error("the necessary preimage %s for "
1497 - "'%s' cannot be read",
1498 - patch->old_sha1_prefix, name);
1499 -
1500 - /* patch->fragment->patch has the delta data and
1501 - * we should apply it to the preimage.
1567 + /* We have verified desc matches the preimage;
1568 + * apply the patch data to it, which is stored
1569 + * in the patch->fragments->{patch,size}.
1570 */
1503 - desc->buffer = patch_delta(src, src_size,
1504 - (void*) patch->fragments->patch,
1505 - patch->fragments->size,
1506 - &dst_size);
1507 - if (!desc->buffer)
1571 + if (apply_binary_fragment(desc, patch))
1572 return error("binary patch does not apply to '%s'",
1573 name);
1510 - desc->size = desc->alloc = dst_size;
1574
1575 /* verify that the result matches */
1576 write_sha1_file_prepare(desc->buffer, desc->size, blob_type,
@@ -2102,7 +2165,8 @@ int main(int argc, char **argv)
2165 diffstat = 1;
2166 continue;
2167 }
2105 - if (!strcmp(arg, "--allow-binary-replacement")) {
2168 + if (!strcmp(arg, "--allow-binary-replacement") ||
2169 + !strcmp(arg, "--binary")) {
2170 allow_binary_replacement = 1;
2171 continue;
2172 }
base85.c new
+134
@@ -0,0 +1,134 @@
1 +#include "cache.h"
2 +
3 +#undef DEBUG_85
4 +
5 +#ifdef DEBUG_85
6 +#define say(a) fprintf(stderr, a)
7 +#define say1(a,b) fprintf(stderr, a, b)
8 +#define say2(a,b,c) fprintf(stderr, a, b, c)
9 +#else
10 +#define say(a) do {} while(0)
11 +#define say1(a,b) do {} while(0)
12 +#define say2(a,b,c) do {} while(0)
13 +#endif
14 +
15 +static const char en85[] = {
16 + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
17 + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J',
18 + 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T',
19 + 'U', 'V', 'W', 'X', 'Y', 'Z',
20 + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j',
21 + 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't',
22 + 'u', 'v', 'w', 'x', 'y', 'z',
23 + '!', '#', '$', '%', '&', '(', ')', '*', '+', '-',
24 + ';', '<', '=', '>', '?', '@', '^', '_', '`', '{',
25 + '|', '}', '~'
26 +};
27 +
28 +static char de85[256];
29 +static void prep_base85(void)
30 +{
31 + int i;
32 + if (de85['Z'])
33 + return;
34 + for (i = 0; i < ARRAY_SIZE(en85); i++) {
35 + int ch = en85[i];
36 + de85[ch] = i + 1;
37 + }
38 +}
39 +
40 +int decode_85(char *dst, char *buffer, int len)
41 +{
42 + prep_base85();
43 +
44 + say2("decode 85 <%.*s>", len/4*5, buffer);
45 + while (len) {
46 + unsigned acc = 0;
47 + int cnt;
48 + for (cnt = 0; cnt < 5; cnt++, buffer++) {
49 + int ch = *((unsigned char *)buffer);
50 + int de = de85[ch];
51 + if (!de)
52 + return error("invalid base85 alphabet %c", ch);
53 + de--;
54 + if (cnt == 4) {
55 + /*
56 + * Detect overflow. The largest
57 + * 5-letter possible is "|NsC0" to
58 + * encode 0xffffffff, and "|NsC" gives
59 + * 0x03030303 at this point (i.e.
60 + * 0xffffffff = 0x03030303 * 85).
61 + */
62 + if (0x03030303 < acc ||
63 + (0x03030303 == acc && de))
64 + error("invalid base85 sequence %.5s",
65 + buffer-3);
66 + }
67 + acc = acc * 85 + de;
68 + say1(" <%08x>", acc);
69 + }
70 + say1(" %08x", acc);
71 + for (cnt = 0; cnt < 4 && len; cnt++, len--) {
72 + *dst++ = (acc >> 24) & 0xff;
73 + acc = acc << 8;
74 + }
75 + }
76 + say("\n");
77 +
78 + return 0;
79 +}
80 +
81 +void encode_85(char *buf, unsigned char *data, int bytes)
82 +{
83 + prep_base85();
84 +
85 + say("encode 85");
86 + while (bytes) {
87 + unsigned acc = 0;
88 + int cnt;
89 + for (cnt = 0; cnt < 4 && bytes; cnt++, bytes--) {
90 + int ch = *data++;
91 + acc |= ch << ((3-cnt)*8);
92 + }
93 + say1(" %08x", acc);
94 + for (cnt = 0; cnt < 5; cnt++) {
95 + int val = acc % 85;
96 + acc /= 85;
97 + buf[4-cnt] = en85[val];
98 + }
99 + buf += 5;
100 + }
101 + say("\n");
102 +
103 + *buf = 0;
104 +}
105 +
106 +#ifdef DEBUG_85
107 +int main(int ac, char **av)
108 +{
109 + char buf[1024];
110 +
111 + if (!strcmp(av[1], "-e")) {
112 + int len = strlen(av[2]);
113 + encode_85(buf, av[2], len);
114 + if (len <= 26) len = len + 'A' - 1;
115 + else len = len + 'a' - 26 + 1;
116 + printf("encoded: %c%s\n", len, buf);
117 + return 0;
118 + }
119 + if (!strcmp(av[1], "-d")) {
120 + int len = *av[2];
121 + if ('A' <= len && len <= 'Z') len = len - 'A' + 1;
122 + else len = len - 'a' + 26 + 1;
123 + decode_85(buf, av[2]+1, len);
124 + printf("decoded: %.*s\n", len, buf);
125 + return 0;
126 + }
127 + if (!strcmp(av[1], "-t")) {
128 + char t[4] = { -1,-1,-1,-1 };
129 + encode_85(buf, t, 4);
130 + printf("encoded: D%s\n", buf);
131 + return 0;
132 + }
133 +}
134 +#endif
cache.h
+1
@@ -365,5 +365,6 @@ extern void setup_pager(void);
365
366 /* base85 */
367 int decode_85(char *dst, char *line, int linelen);
368 +void encode_85(char *buf, unsigned char *data, int bytes);
369
370 #endif /* CACHE_H */
diff.c
+71 -63
@@ -392,78 +392,78 @@ static void show_stats(struct diffstat_t* data)
392 total_files, adds, dels);
393 }
394
395 -static void *encode_delta_size(void *data, unsigned long size)
395 +static unsigned char *deflate_it(char *data,
396 + unsigned long size,
397 + unsigned long *result_size)
398 {
397 - unsigned char *cp = data;
398 - *cp++ = size;
399 - size >>= 7;
400 - while (size) {
401 - cp[-1] |= 0x80;
402 - *cp++ = size;
403 - size >>= 7;
404 - }
405 - return cp;
399 + int bound;
400 + unsigned char *deflated;
401 + z_stream stream;
402 +
403 + memset(&stream, 0, sizeof(stream));
404 + deflateInit(&stream, Z_BEST_COMPRESSION);
405 + bound = deflateBound(&stream, size);
406 + deflated = xmalloc(bound);
407 + stream.next_out = deflated;
408 + stream.avail_out = bound;
409 +
410 + stream.next_in = (unsigned char *)data;
411 + stream.avail_in = size;
412 + while (deflate(&stream, Z_FINISH) == Z_OK)
413 + ; /* nothing */
414 + deflateEnd(&stream);
415 + *result_size = stream.total_out;
416 + return deflated;
417 }
418
408 -static void *safe_diff_delta(const unsigned char *src, unsigned long src_size,
409 - const unsigned char *dst, unsigned long dst_size,
410 - unsigned long *delta_size)
419 +static void emit_binary_diff(mmfile_t *one, mmfile_t *two)
420 {
412 - unsigned long bufsize;
413 - unsigned char *data;
414 - unsigned char *cp;
415 -
416 - if (src_size && dst_size)
417 - return diff_delta(src, src_size, dst, dst_size, delta_size, 0);
421 + void *cp;
422 + void *delta;
423 + void *deflated;
424 + void *data;
425 + unsigned long orig_size;
426 + unsigned long delta_size;
427 + unsigned long deflate_size;
428 + unsigned long data_size;
429
419 - /* diff-delta does not like to do delta with empty, so
420 - * we do that by hand here. Sigh...
430 + printf("GIT binary patch\n");
431 + /* We could do deflated delta, or we could do just deflated two,
432 + * whichever is smaller.
433 */
422 -
423 - if (!src_size)
424 - /* literal copy can be done only 127-byte at a time.
425 - */
426 - bufsize = dst_size + (dst_size / 127) + 40;
427 - else
428 - bufsize = 40;
429 - data = xmalloc(bufsize);
430 - cp = encode_delta_size(data, src_size);
431 - cp = encode_delta_size(cp, dst_size);
432 -
433 - if (dst_size) {
434 - /* copy out literally */
435 - while (dst_size) {
436 - int sz = (127 < dst_size) ? 127 : dst_size;
437 - *cp++ = sz;
438 - dst_size -= sz;
439 - while (sz) {
440 - *cp++ = *dst++;
441 - sz--;
442 - }
434 + delta = NULL;
435 + deflated = deflate_it(two->ptr, two->size, &deflate_size);
436 + if (one->size && two->size) {
437 + delta = diff_delta(one->ptr, one->size,
438 + two->ptr, two->size,
439 + &delta_size, deflate_size);
440 + if (delta) {
441 + void *to_free = delta;
442 + orig_size = delta_size;
443 + delta = deflate_it(delta, delta_size, &delta_size);
444 + free(to_free);
445 }
446 }
445 - *delta_size = (cp - data);
446 - return data;
447 -}
447
449 -static void emit_binary_diff(mmfile_t *one, mmfile_t *two)
450 -{
451 - void *delta, *cp;
452 - unsigned long delta_size;
448 + if (delta && delta_size < deflate_size) {
449 + printf("delta %lu\n", orig_size);
450 + free(deflated);
451 + data = delta;
452 + data_size = delta_size;
453 + }
454 + else {
455 + printf("literal %lu\n", two->size);
456 + free(delta);
457 + data = deflated;
458 + data_size = deflate_size;
459 + }
460
454 - printf("GIT binary patch\n");
455 - delta = safe_diff_delta(one->ptr, one->size,
456 - two->ptr, two->size,
457 - &delta_size);
458 - if (!delta)
459 - die("unable to generate binary diff");
460 -
461 - /* emit delta encoded in base85 */
462 - cp = delta;
463 - while (delta_size) {
464 - int bytes = (52 < delta_size) ? 52 : delta_size;
461 + /* emit data encoded in base85 */
462 + cp = data;
463 + while (data_size) {
464 + int bytes = (52 < data_size) ? 52 : data_size;
465 char line[70];
466 - delta_size -= bytes;
466 + data_size -= bytes;
467 if (bytes <= 26)
468 line[0] = bytes + 'A' - 1;
469 else
@@ -473,7 +473,7 @@ static void emit_binary_diff(mmfile_t *one, mmfile_t *two)
473 puts(line);
474 }
475 printf("\n");
476 - free(delta);
476 + free(data);
477 }
478
479 #define FIRST_FEW_BYTES 8000
@@ -538,7 +538,11 @@ static void builtin_diff(const char *name_a,
538 die("unable to read files to diff");
539
540 if (mmfile_is_binary(&mf1) || mmfile_is_binary(&mf2)) {
541 - if (o->full_index)
541 + /* Quite common confusing case */
542 + if (mf1.size == mf2.size &&
543 + !memcmp(mf1.ptr, mf2.ptr, mf1.size))
544 + goto free_ab_and_return;
545 + if (o->binary)
546 emit_binary_diff(&mf1, &mf2);
547 else
548 printf("Binary files %s and %s differ\n",
@@ -1239,6 +1243,10 @@ int diff_opt_parse(struct diff_options *options, const char **av, int ac)
1243 options->rename_limit = strtoul(arg+2, NULL, 10);
1244 else if (!strcmp(arg, "--full-index"))
1245 options->full_index = 1;
1246 + else if (!strcmp(arg, "--binary")) {
1247 + options->output_format = DIFF_FORMAT_PATCH;
1248 + options->full_index = options->binary = 1;
1249 + }
1250 else if (!strcmp(arg, "--name-only"))
1251 options->output_format = DIFF_FORMAT_NAME;
1252 else if (!strcmp(arg, "--name-status"))
diff.h
+1
@@ -28,6 +28,7 @@ struct diff_options {
28 with_raw:1,
29 with_stat:1,
30 tree_in_recursive:1,
31 + binary:1,
32 full_index:1,
33 silent_on_remove:1,
34 find_copies_harder:1;