split the diff-delta interface
This patch splits the diff-delta interface into index creation and delta generation. A wrapper is provided to preserve the diff-delta() call. This will allow for an optimization in pack-objects.c where the source object could be fixed and a full window of objects tentatively tried against that same source object without recomputing the source index each time. This patch only restructure things, plus a couple cleanups for good measure. There is no performance change yet. Signed-off-by: Nicolas Pitre <nico@cam.org>
Nicolas Pitre committed
Apr 24, 2006 at 23:07 UTC
08abe669c05521499149dbf84fedefb04a8fa34d
3 files changed
+155
-92
delta.h
+68
-7
@@ -1,12 +1,73 @@
1
#ifndef DELTA_H
2
#define DELTA_H
3
4
-/* handling of delta buffers */
5
-extern void *diff_delta(void *from_buf, unsigned long from_size,
6
- void *to_buf, unsigned long to_size,
7
- unsigned long *delta_size, unsigned long max_size);
8
-extern void *patch_delta(void *src_buf, unsigned long src_size,
9
- void *delta_buf, unsigned long delta_size,
4
+/* opaque object for delta index */
5
+struct delta_index;
6
+
7
+/*
8
+ * create_delta_index: compute index data from given buffer
9
+ *
10
+ * This returns a pointer to a struct delta_index that should be passed to
11
+ * subsequent create_delta() calls, or to free_delta_index(). A NULL pointer
12
+ * is returned on failure. The given buffer must not be freed nor altered
13
+ * before free_delta_index() is called. The returned pointer must be freed
14
+ * using free_delta_index().
15
+ */
16
+extern struct delta_index *
17
+create_delta_index(const void *buf, unsigned long bufsize);
18
+
19
+/*
20
+ * free_delta_index: free the index created by create_delta_index()
21
+ */
22
+extern void free_delta_index(struct delta_index *index);
23
+
24
+/*
25
+ * create_delta: create a delta from given index for the given buffer
26
+ *
27
+ * This function may be called multiple times with different buffers using
28
+ * the same delta_index pointer. If max_delta_size is non-zero and the
29
+ * resulting delta is to be larger than max_delta_size then NULL is returned.
30
+ * On success, a non-NULL pointer to the buffer with the delta data is
31
+ * returned and *delta_size is updated with its size. The returned buffer
32
+ * must be freed by the caller.
33
+ */
34
+extern void *
35
+create_delta(const struct delta_index *index,
36
+ const void *buf, unsigned long bufsize,
37
+ unsigned long *delta_size, unsigned long max_delta_size);
38
+
39
+/*
40
+ * diff_delta: create a delta from source buffer to target buffer
41
+ *
42
+ * If max_delta_size is non-zero and the resulting delta is to be larger
43
+ * than max_delta_size then NULL is returned. On success, a non-NULL
44
+ * pointer to the buffer with the delta data is returned and *delta_size is
45
+ * updated with its size. The returned buffer must be freed by the caller.
46
+ */
47
+static inline void *
48
+diff_delta(const void *src_buf, unsigned long src_bufsize,
49
+ const void *trg_buf, unsigned long trg_bufsize,
50
+ unsigned long *delta_size, unsigned long max_delta_size)
51
+{
52
+ struct delta_index *index = create_delta_index(src_buf, src_bufsize);
53
+ if (index) {
54
+ void *delta = create_delta(index, trg_buf, trg_bufsize,
55
+ delta_size, max_delta_size);
56
+ free_delta_index(index);
57
+ return delta;
58
+ }
59
+ return NULL;
60
+}
61
+
62
+/*
63
+ * patch_delta: recreate target buffer given source buffer and delta data
64
+ *
65
+ * On success, a non-NULL pointer to the target buffer is returned and
66
+ * *trg_bufsize is updated with its size. On failure a NULL pointer is
67
+ * returned. The returned buffer must be freed by the caller.
68
+ */
69
+extern void *patch_delta(const void *src_buf, unsigned long src_size,
70
+ const void *delta_buf, unsigned long delta_size,
71
unsigned long *dst_size);
72
73
/* the smallest possible delta size is 4 bytes */
@@ -14,7 +75,7 @@ extern void *patch_delta(void *src_buf, unsigned long src_size,
75
76
/*
77
* This must be called twice on the delta data buffer, first to get the
17
- * expected reference buffer size, and again to get the result buffer size.
78
+ * expected source buffer size, and again to get the target buffer size.
79
*/
80
static inline unsigned long get_delta_hdr_size(const unsigned char **datap,
81
const unsigned char *top)
diff-delta.c
+85
-83
@@ -27,53 +27,70 @@
27
/* block size: min = 16, max = 64k, power of 2 */
28
#define BLK_SIZE 16
29
30
-#define MIN(a, b) ((a) < (b) ? (a) : (b))
30
+/* maximum hash entry list for the same hash bucket */
31
+#define HASH_LIMIT 64
32
33
#define GR_PRIME 0x9e370001
34
#define HASH(v, shift) (((unsigned int)(v) * GR_PRIME) >> (shift))
35
35
-struct index {
36
+struct index_entry {
37
const unsigned char *ptr;
38
unsigned int val;
38
- struct index *next;
39
+ struct index_entry *next;
40
};
41
41
-static struct index ** delta_index(const unsigned char *buf,
42
- unsigned long bufsize,
43
- unsigned long trg_bufsize,
44
- unsigned int *hash_shift)
42
+struct delta_index {
43
+ const void *src_buf;
44
+ unsigned long src_size;
45
+ unsigned int hash_shift;
46
+ struct index_entry *hash[0];
47
+};
48
+
49
+struct delta_index * create_delta_index(const void *buf, unsigned long bufsize)
50
{
46
- unsigned int i, hsize, hshift, hlimit, entries, *hash_count;
47
- const unsigned char *data;
48
- struct index *entry, **hash;
51
+ unsigned int i, hsize, hshift, entries, *hash_count;
52
+ const unsigned char *data, *buffer = buf;
53
+ struct delta_index *index;
54
+ struct index_entry *entry, **hash;
55
void *mem;
56
57
+ if (!buf || !bufsize)
58
+ return NULL;
59
+
60
/* determine index hash size */
61
entries = bufsize / BLK_SIZE;
62
hsize = entries / 4;
63
for (i = 4; (1 << i) < hsize && i < 31; i++);
64
hsize = 1 << i;
65
hshift = 32 - i;
57
- *hash_shift = hshift;
66
67
/* allocate lookup index */
60
- mem = malloc(hsize * sizeof(*hash) + entries * sizeof(*entry));
68
+ mem = malloc(sizeof(*index) +
69
+ sizeof(*hash) * hsize +
70
+ sizeof(*entry) * entries);
71
if (!mem)
72
return NULL;
73
+ index = mem;
74
+ mem = index + 1;
75
hash = mem;
64
- entry = mem + hsize * sizeof(*hash);
76
+ mem = hash + hsize;
77
+ entry = mem;
78
+
79
+ index->src_buf = buf;
80
+ index->src_size = bufsize;
81
+ index->hash_shift = hshift;
82
memset(hash, 0, hsize * sizeof(*hash));
83
84
/* allocate an array to count hash entries */
85
hash_count = calloc(hsize, sizeof(*hash_count));
86
if (!hash_count) {
70
- free(hash);
87
+ free(index);
88
return NULL;
89
}
90
91
/* then populate the index */
75
- data = buf + entries * BLK_SIZE - BLK_SIZE;
76
- while (data >= buf) {
92
+ data = buffer + entries * BLK_SIZE - BLK_SIZE;
93
+ while (data >= buffer) {
94
unsigned int val = adler32(0, data, BLK_SIZE);
95
i = HASH(val, hshift);
96
entry->ptr = data;
@@ -91,27 +108,18 @@ static struct index ** delta_index(const unsigned char *buf,
108
* bucket that would bring us to O(m*n) computing costs (m and n
109
* corresponding to reference and target buffer sizes).
110
*
94
- * The more the target buffer is large, the more it is important to
95
- * have small entry lists for each hash buckets. With such a limit
96
- * the cost is bounded to something more like O(m+n).
97
- */
98
- hlimit = (1 << 26) / trg_bufsize;
99
- if (hlimit < 4*BLK_SIZE)
100
- hlimit = 4*BLK_SIZE;
101
-
102
- /*
103
- * Now make sure none of the hash buckets has more entries than
111
+ * Make sure none of the hash buckets has more entries than
112
* we're willing to test. Otherwise we cull the entry list
113
* uniformly to still preserve a good repartition across
114
* the reference buffer.
115
*/
116
for (i = 0; i < hsize; i++) {
109
- if (hash_count[i] < hlimit)
117
+ if (hash_count[i] < HASH_LIMIT)
118
continue;
119
entry = hash[i];
120
do {
113
- struct index *keep = entry;
114
- int skip = hash_count[i] / hlimit / 2;
121
+ struct index_entry *keep = entry;
122
+ int skip = hash_count[i] / HASH_LIMIT / 2;
123
do {
124
entry = entry->next;
125
} while(--skip && entry);
@@ -120,7 +128,12 @@ static struct index ** delta_index(const unsigned char *buf,
128
}
129
free(hash_count);
130
123
- return hash;
131
+ return index;
132
+}
133
+
134
+void free_delta_index(struct delta_index *index)
135
+{
136
+ free(index);
137
}
138
139
/* provide the size of the copy opcode given the block offset and size */
@@ -131,21 +144,17 @@ static struct index ** delta_index(const unsigned char *buf,
144
/* the maximum size for any opcode */
145
#define MAX_OP_SIZE COPYOP_SIZE(0xffffffff, 0xffffffff)
146
134
-void *diff_delta(void *from_buf, unsigned long from_size,
135
- void *to_buf, unsigned long to_size,
136
- unsigned long *delta_size,
137
- unsigned long max_size)
147
+void *
148
+create_delta(const struct delta_index *index,
149
+ const void *trg_buf, unsigned long trg_size,
150
+ unsigned long *delta_size, unsigned long max_size)
151
{
152
unsigned int i, outpos, outsize, hash_shift;
153
int inscnt;
154
const unsigned char *ref_data, *ref_top, *data, *top;
155
unsigned char *out;
143
- struct index *entry, **hash;
156
145
- if (!from_size || !to_size)
146
- return NULL;
147
- hash = delta_index(from_buf, from_size, to_size, &hash_shift);
148
- if (!hash)
157
+ if (!trg_buf || !trg_size)
158
return NULL;
159
160
outpos = 0;
@@ -153,60 +162,55 @@ void *diff_delta(void *from_buf, unsigned long from_size,
162
if (max_size && outsize >= max_size)
163
outsize = max_size + MAX_OP_SIZE + 1;
164
out = malloc(outsize);
156
- if (!out) {
157
- free(hash);
165
+ if (!out)
166
return NULL;
159
- }
160
-
161
- ref_data = from_buf;
162
- ref_top = from_buf + from_size;
163
- data = to_buf;
164
- top = to_buf + to_size;
167
168
/* store reference buffer size */
167
- out[outpos++] = from_size;
168
- from_size >>= 7;
169
- while (from_size) {
170
- out[outpos - 1] |= 0x80;
171
- out[outpos++] = from_size;
172
- from_size >>= 7;
169
+ i = index->src_size;
170
+ while (i >= 0x80) {
171
+ out[outpos++] = i | 0x80;
172
+ i >>= 7;
173
}
174
+ out[outpos++] = i;
175
176
/* store target buffer size */
176
- out[outpos++] = to_size;
177
- to_size >>= 7;
178
- while (to_size) {
179
- out[outpos - 1] |= 0x80;
180
- out[outpos++] = to_size;
181
- to_size >>= 7;
177
+ i = trg_size;
178
+ while (i >= 0x80) {
179
+ out[outpos++] = i | 0x80;
180
+ i >>= 7;
181
}
182
+ out[outpos++] = i;
183
184
+ ref_data = index->src_buf;
185
+ ref_top = ref_data + index->src_size;
186
+ data = trg_buf;
187
+ top = trg_buf + trg_size;
188
+ hash_shift = index->hash_shift;
189
inscnt = 0;
190
191
while (data < top) {
192
unsigned int moff = 0, msize = 0;
188
- if (data + BLK_SIZE <= top) {
189
- unsigned int val = adler32(0, data, BLK_SIZE);
190
- i = HASH(val, hash_shift);
191
- for (entry = hash[i]; entry; entry = entry->next) {
192
- const unsigned char *ref = entry->ptr;
193
- const unsigned char *src = data;
194
- unsigned int ref_size = ref_top - ref;
195
- if (entry->val != val)
196
- continue;
197
- if (ref_size > top - src)
198
- ref_size = top - src;
199
- if (ref_size > 0x10000)
200
- ref_size = 0x10000;
201
- if (ref_size <= msize)
202
- break;
203
- while (ref_size-- && *src++ == *ref)
204
- ref++;
205
- if (msize < ref - entry->ptr) {
206
- /* this is our best match so far */
207
- msize = ref - entry->ptr;
208
- moff = entry->ptr - ref_data;
209
- }
193
+ struct index_entry *entry;
194
+ unsigned int val = adler32(0, data, BLK_SIZE);
195
+ i = HASH(val, hash_shift);
196
+ for (entry = index->hash[i]; entry; entry = entry->next) {
197
+ const unsigned char *ref = entry->ptr;
198
+ const unsigned char *src = data;
199
+ unsigned int ref_size = ref_top - ref;
200
+ if (entry->val != val)
201
+ continue;
202
+ if (ref_size > top - src)
203
+ ref_size = top - src;
204
+ if (ref_size > 0x10000)
205
+ ref_size = 0x10000;
206
+ if (ref_size <= msize)
207
+ break;
208
+ while (ref_size-- && *src++ == *ref)
209
+ ref++;
210
+ if (msize < ref - entry->ptr) {
211
+ /* this is our best match so far */
212
+ msize = ref - entry->ptr;
213
+ moff = entry->ptr - ref_data;
214
}
215
}
216
@@ -271,7 +275,6 @@ void *diff_delta(void *from_buf, unsigned long from_size,
275
out = realloc(out, outsize);
276
if (!out) {
277
free(tmp);
274
- free(hash);
278
return NULL;
279
}
280
}
@@ -280,7 +283,6 @@ void *diff_delta(void *from_buf, unsigned long from_size,
283
if (inscnt)
284
out[outpos - inscnt - 1] = inscnt;
285
283
- free(hash);
286
*delta_size = outpos;
287
return out;
288
}
patch-delta.c
+2
-2
@@ -13,8 +13,8 @@
13
#include <string.h>
14
#include "delta.h"
15
16
-void *patch_delta(void *src_buf, unsigned long src_size,
17
- void *delta_buf, unsigned long delta_size,
16
+void *patch_delta(const void *src_buf, unsigned long src_size,
17
+ const void *delta_buf, unsigned long delta_size,
18
unsigned long *dst_size)
19
{
20
const unsigned char *data, *top;