@matheusbsbs / matheus-bagliano / commits / fd2bbdd238
Fork 0

Clean-up Geert's similarity fingerprint code.

This splits Geert's similarity fingerprint code into main program and fingerprinting function. The next step would be to try using this in pack-objects.c::try_delta() -- which would be a good evaluation. Signed-off-by: Junio C Hamano <junkio@cox.net>

Junio C Hamano committed Apr 15, 2006 at 01:32 UTC fd2bbdd2386ea0c558aba95711ef53c4552a6146
7 files changed +521
.gitignore
+1
@@ -123,6 +123,7 @@ git-write-tree
123 git-core-*/?*
124 test-date
125 test-delta
126 +test-gsimm
127 common-cmds.h
128 *.tar.gz
129 *.dsc
Makefile
+4
@@ -606,6 +606,9 @@ test-date$X: test-date.c date.o ctype.o
606 test-delta$X: test-delta.c diff-delta.o patch-delta.o
607 $(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $^ -lz
608
609 +test-gsimm$X: test-gsimm.c gsimm.o rabinpoly.o
610 + $(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $^
611 +
612 check:
613 for i in *.c; do sparse $(ALL_CFLAGS) $(SPARSE_FLAGS) $$i || exit; done
614
@@ -654,6 +657,7 @@ clean:
657 rm -f *.o mozilla-sha1/*.o arm/*.o ppc/*.o compat/*.o xdiff/*.o \
658 $(LIB_FILE) $(XDIFF_LIB)
659 rm -f $(ALL_PROGRAMS) git$X
660 + rm -f test-date$X test-delta$X test-gsimm$X
661 rm -f *.spec *.pyc *.pyo */*.pyc */*.pyo common-cmds.h TAGS tags
662 rm -rf $(GIT_TARNAME)
663 rm -f $(GIT_TARNAME).tar.gz git-core_$(GIT_VERSION)-*.tar.gz
gsimm.c new
+94
@@ -0,0 +1,94 @@
1 +#include "rabinpoly.h"
2 +#include "gsimm.h"
3 +
4 +/* Has to be power of two. Since the Rabin hash only has 63
5 + usable bits, the number of hashes is limited to 32.
6 + Lower powers of two could be used for speeding up processing
7 + of very large files. */
8 +#define NUM_HASHES_PER_CHAR 32
9 +
10 +/* Size of cache used to eliminate duplicate substrings.
11 + Make small enough to comfortably fit in L1 cache. */
12 +#define DUP_CACHE_SIZE 256
13 +
14 +/* For the final counting, do not count each bit individually, but
15 + group them. Must be power of two, at most NUM_HASHES_PER_CHAR.
16 + However, larger sizes result in higher cache usage. Use 8 bits
17 + per group for efficient processing of large files on fast machines
18 + with decent caches, or 4 bits for faster processing of small files
19 + and for machines with small caches. */
20 +#define GROUP_BITS 4
21 +#define GROUP_COUNTERS (1<<GROUP_BITS)
22 +
23 +static void freq_to_md(u_char *md, int *freq)
24 +{ int j, k;
25 +
26 + for (j = 0; j < MD_LENGTH; j++)
27 + { u_char ch = 0;
28 +
29 + for (k = 0; k < 8; k++) ch = 2*ch + (freq[8*j+k] > 0);
30 + md[j] = ch;
31 + }
32 + bzero (freq, sizeof(freq[0]) * MD_BITS);
33 +}
34 +
35 +void gb_simm_process(u_char *data, unsigned len, u_char *md)
36 +{ size_t j = 0;
37 + u_int32_t ofs;
38 + u_int32_t dup_cache[DUP_CACHE_SIZE];
39 + u_int32_t count [MD_BITS * (GROUP_COUNTERS/GROUP_BITS)];
40 + int freq[MD_BITS];
41 +
42 + bzero (freq, sizeof(freq[0]) * MD_BITS);
43 + bzero (dup_cache, DUP_CACHE_SIZE * sizeof (u_int32_t));
44 + bzero (count, (MD_BITS * (GROUP_COUNTERS/GROUP_BITS) * sizeof (u_int32_t)));
45 +
46 + /* Ignore incomplete substrings */
47 + while (j < len && j < RABIN_WINDOW_SIZE) rabin_slide8 (data[j++]);
48 +
49 + while (j < len)
50 + { u_int64_t hash;
51 + u_int32_t ofs, sum;
52 + u_char idx;
53 + int k;
54 +
55 + hash = rabin_slide8 (data[j++]);
56 +
57 + /* In order to update a much larger frequency table
58 + with only 32 bits of checksum, randomly select a
59 + part of the table to update. The selection should
60 + only depend on the content of the represented data,
61 + and be independent of the bits used for the update.
62 +
63 + Instead of updating 32 individual counters, process
64 + the checksum in MD_BITS / GROUP_BITS groups of
65 + GROUP_BITS bits, and count the frequency of each bit pattern.
66 + */
67 +
68 + idx = (hash >> 32);
69 + sum = (u_int32_t) hash;
70 + ofs = idx % (MD_BITS / NUM_HASHES_PER_CHAR) * NUM_HASHES_PER_CHAR;
71 + idx %= DUP_CACHE_SIZE;
72 + if (dup_cache[idx] != sum)
73 + { dup_cache[idx] = sum;
74 + for (k = 0; k < NUM_HASHES_PER_CHAR / GROUP_BITS; k++)
75 + { count[ofs * GROUP_COUNTERS / GROUP_BITS + (sum % GROUP_COUNTERS)]++;
76 + ofs += GROUP_BITS;
77 + sum >>= GROUP_BITS;
78 + } } }
79 +
80 + /* Distribute the occurrences of each bit group over the frequency table. */
81 + for (ofs = 0; ofs < MD_BITS; ofs += GROUP_BITS)
82 + { int j;
83 + for (j = 0; j < GROUP_COUNTERS; j++)
84 + { int k;
85 + for (k = 0; k < GROUP_BITS; k++)
86 + { freq[ofs + k] += ((1<<k) & j)
87 + ? count[ofs * GROUP_COUNTERS / GROUP_BITS + j]
88 + : -count[ofs * GROUP_COUNTERS / GROUP_BITS + j];
89 + } } }
90 +
91 + if (md)
92 + { rabin_reset();
93 + freq_to_md (md, freq);
94 +} }
gsimm.h new
+28
@@ -0,0 +1,28 @@
1 +#ifndef GSIMM_H
2 +#define GSIMM_H
3 +
4 +/* Length of file message digest (MD) in bytes. Longer MD's are
5 + better, but increase processing time for diminishing returns.
6 + Must be multiple of NUM_HASHES_PER_CHAR / 8, and at least 24
7 + for good results
8 +*/
9 +#define MD_LENGTH 32
10 +#define MD_BITS (MD_LENGTH * 8)
11 +
12 +/* The MIN_FILE_SIZE indicates the absolute minimal file size that
13 + can be processed. As indicated above, the first and last
14 + RABIN_WINDOW_SIZE - 1 bytes are skipped.
15 + In order to get at least an average of 12 samples
16 + per bit in the final message digest, require at least 3 * MD_LENGTH
17 + complete windows in the file. */
18 +#define MIN_FILE_SIZE (3 * MD_LENGTH + 2 * (RABIN_WINDOW_SIZE - 1))
19 +
20 +/* Limit matching algorithm to files less than 256 MB, so we can use
21 + 32 bit integers everywhere without fear of overflow. For larger
22 + files we should add logic to mmap the file by piece and accumulate
23 + the frequency counts. */
24 +#define MAX_FILE_SIZE (256*1024*1024 - 1)
25 +
26 +void gb_simm_process(u_char *data, unsigned len, u_char *md);
27 +
28 +#endif
rabinpoly.c new
+154
@@ -0,0 +1,154 @@
1 +/*
2 + *
3 + * Copyright (C) 1999 David Mazieres (dm@uun.org)
4 + *
5 + * This program is free software; you can redistribute it and/or
6 + * modify it under the terms of the GNU General Public License as
7 + * published by the Free Software Foundation; either version 2, or (at
8 + * your option) any later version.
9 + *
10 + * This program is distributed in the hope that it will be useful, but
11 + * WITHOUT ANY WARRANTY; without even the implied warranty of
12 + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 + * General Public License for more details.
14 + *
15 + * You should have received a copy of the GNU General Public License
16 + * along with this program; if not, write to the Free Software
17 + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
18 + * USA
19 + *
20 + */
21 +
22 + /* Faster generic_fls */
23 + /* (c) 2002, D.Phillips and Sistina Software */
24 +
25 +#include "rabinpoly.h"
26 +#define MSB64 0x8000000000000000ULL
27 +
28 +static inline unsigned fls8(unsigned n)
29 +{
30 + return n & 0xf0?
31 + n & 0xc0? (n >> 7) + 7: (n >> 5) + 5:
32 + n & 0x0c? (n >> 3) + 3: n - ((n + 1) >> 2);
33 +}
34 +
35 +static inline unsigned fls16(unsigned n)
36 +{
37 + return n & 0xff00? fls8(n >> 8) + 8: fls8(n);
38 +}
39 +
40 +static inline unsigned fls32(unsigned n)
41 +{
42 + return n & 0xffff0000? fls16(n >> 16) + 16: fls16(n);
43 +}
44 +
45 +static inline unsigned fls64(unsigned long long n) /* should be u64 */
46 +{
47 + return n & 0xffffffff00000000ULL? fls32(n >> 32) + 32: fls32(n);
48 +}
49 +
50 +
51 +static u_int64_t polymod (u_int64_t nh, u_int64_t nl, u_int64_t d);
52 +static void polymult (u_int64_t *php, u_int64_t *plp,
53 + u_int64_t x, u_int64_t y);
54 +static u_int64_t polymmult (u_int64_t x, u_int64_t y, u_int64_t d);
55 +
56 +static u_int64_t poly = 0xb15e234bd3792f63ull; // Actual polynomial
57 +static u_int64_t T[256]; // Lookup table for mod
58 +static int shift;
59 +
60 +u_int64_t append8 (u_int64_t p, u_char m)
61 +{ return ((p << 8) | m) ^ T[p >> shift];
62 +}
63 +
64 +static u_int64_t
65 +polymod (u_int64_t nh, u_int64_t nl, u_int64_t d)
66 +{ int i, k;
67 + assert (d);
68 + k = fls64 (d) - 1;
69 + d <<= 63 - k;
70 +
71 + if (nh) {
72 + if (nh & MSB64)
73 + nh ^= d;
74 + for (i = 62; i >= 0; i--)
75 + if (nh & 1ULL << i) {
76 + nh ^= d >> (63 - i);
77 + nl ^= d << (i + 1);
78 + }
79 + }
80 + for (i = 63; i >= k; i--)
81 + if (nl & 1ULL << i)
82 + nl ^= d >> (63 - i);
83 + return nl;
84 +}
85 +
86 +static void
87 +polymult (u_int64_t *php, u_int64_t *plp, u_int64_t x, u_int64_t y)
88 +{ int i;
89 + u_int64_t ph = 0, pl = 0;
90 + if (x & 1)
91 + pl = y;
92 + for (i = 1; i < 64; i++)
93 + if (x & (1ULL << i)) {
94 + ph ^= y >> (64 - i);
95 + pl ^= y << i;
96 + }
97 + if (php)
98 + *php = ph;
99 + if (plp)
100 + *plp = pl;
101 +}
102 +
103 +static u_int64_t
104 +polymmult (u_int64_t x, u_int64_t y, u_int64_t d)
105 +{
106 + u_int64_t h, l;
107 + polymult (&h, &l, x, y);
108 + return polymod (h, l, d);
109 +}
110 +
111 +static int size = RABIN_WINDOW_SIZE;
112 +static u_int64_t fingerprint = 0;
113 +static int bufpos = -1;
114 +static u_int64_t U[256];
115 +static u_char buf[RABIN_WINDOW_SIZE];
116 +
117 +void rabin_init ()
118 +{ u_int64_t sizeshift = 1;
119 + u_int64_t T1;
120 + int xshift;
121 + int i, j;
122 + assert (poly >= 0x100);
123 + xshift = fls64 (poly) - 1;
124 + shift = xshift - 8;
125 + T1 = polymod (0, 1ULL << xshift, poly);
126 + for (j = 0; j < 256; j++)
127 + T[j] = polymmult (j, T1, poly) | ((u_int64_t) j << xshift);
128 +
129 + for (i = 1; i < size; i++)
130 + sizeshift = append8 (sizeshift, 0);
131 + for (i = 0; i < 256; i++)
132 + U[i] = polymmult (i, sizeshift, poly);
133 + bzero (buf, sizeof (buf));
134 +}
135 +
136 +void
137 +rabin_reset ()
138 +{ rabin_init();
139 + fingerprint = 0;
140 + bzero (buf, sizeof (buf));
141 +}
142 +
143 +u_int64_t
144 +rabin_slide8 (u_char m)
145 +{ u_char om;
146 + if (++bufpos >= size) bufpos = 0;
147 +
148 + om = buf[bufpos];
149 + buf[bufpos] = m;
150 + fingerprint = append8 (fingerprint ^ U[om], m);
151 +
152 + return fingerprint;
153 +}
154 +
rabinpoly.h new
+31
@@ -0,0 +1,31 @@
1 +/*
2 + *
3 + * Copyright (C) 2000 David Mazieres (dm@uun.org)
4 + *
5 + * This program is free software; you can redistribute it and/or
6 + * modify it under the terms of the GNU General Public License as
7 + * published by the Free Software Foundation; either version 2, or (at
8 + * your option) any later version.
9 + *
10 + * This program is distributed in the hope that it will be useful, but
11 + * WITHOUT ANY WARRANTY; without even the implied warranty of
12 + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 + * General Public License for more details.
14 + *
15 + * You should have received a copy of the GNU General Public License
16 + * along with this program; if not, write to the Free Software
17 + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
18 + * USA
19 + *
20 + * Translated to C and simplified by Geert Bosch (bosch@gnat.com)
21 + */
22 +
23 +#include <assert.h>
24 +#include <strings.h>
25 +#include <sys/types.h>
26 +
27 +#ifndef RABIN_WINDOW_SIZE
28 +#define RABIN_WINDOW_SIZE 48
29 +#endif
30 +void rabin_reset();
31 +u_int64_t rabin_slide8(u_char c);
test-gsimm.c new
+209
@@ -0,0 +1,209 @@
1 +#include <unistd.h>
2 +#include <stdlib.h>
3 +#include <fcntl.h>
4 +#include <libgen.h>
5 +#include <stdio.h>
6 +#include <assert.h>
7 +#include <math.h>
8 +#include <string.h>
9 +#include <sys/types.h>
10 +#include <sys/stat.h>
11 +#include <sys/mman.h>
12 +
13 +#include "rabinpoly.h"
14 +#include "gsimm.h"
15 +
16 +#define MIN(x,y) ((y)<(x) ? (y) : (x))
17 +#define MAX(x,y) ((y)>(x) ? (y) : (x))
18 +
19 +/* The RABIN_WINDOW_SIZE is the size of fingerprint window used by
20 + Rabin algorithm. This is not a modifiable parameter.
21 +
22 + The first RABIN_WINDOW_SIZE - 1 bytes are skipped, in order to ensure
23 + fingerprints are good hashes. This does somewhat reduce the
24 + influence of the first few bytes in the file (they're part of
25 + fewer windows, like the last few bytes), but that actually isn't
26 + so bad as files often start with fixed content that may bias comparisons.
27 +*/
28 +
29 +typedef struct fileinfo
30 +{ char *name;
31 + size_t length;
32 + u_char md[MD_LENGTH];
33 + int match;
34 +} File;
35 +
36 +int flag_verbose = 0;
37 +int flag_debug = 0;
38 +char *flag_relative = 0;
39 +
40 +char cmd[12] = " ...";
41 +char md_strbuf[MD_LENGTH * 2 + 1];
42 +u_char relative_md [MD_LENGTH];
43 +
44 +File *file;
45 +int file_count;
46 +size_t file_bytes;
47 +
48 +char hex[17] = "0123456789abcdef";
49 +
50 +void usage()
51 +{ fprintf (stderr, "usage: %s [-dhvw] [-r fingerprint] file ...\n", cmd);
52 + fprintf (stderr, " -d\tdebug output, repeate for more verbosity\n");
53 + fprintf (stderr, " -h\tshow this usage information\n");
54 + fprintf (stderr, " -r\tshow distance relative to fingerprint "
55 + "(%u hex digits)\n", MD_LENGTH * 2);
56 + fprintf (stderr, " -v\tverbose output, repeat for even more verbosity\n");
57 + fprintf (stderr, " -w\tenable warnings for suspect statistics\n");
58 + exit (1);
59 +}
60 +
61 +int dist (u_char *l, u_char *r)
62 +{ int j, k;
63 + int d = 0;
64 +
65 + for (j = 0; j < MD_LENGTH; j++)
66 + { u_char ch = l[j] ^ r[j];
67 +
68 + for (k = 0; k < 8; k++) d += ((ch & (1<<k)) > 0);
69 + }
70 +
71 + return d;
72 +}
73 +
74 +char *md_to_str(u_char *md)
75 +{ int j;
76 +
77 + for (j = 0; j < MD_LENGTH; j++)
78 + { u_char ch = md[j];
79 +
80 + md_strbuf[j*2] = hex[ch >> 4];
81 + md_strbuf[j*2+1] = hex[ch & 0xF];
82 + }
83 +
84 + md_strbuf[j*2] = 0;
85 + return md_strbuf;
86 +}
87 +
88 +void process_file (char *name)
89 +{ int fd;
90 + struct stat fs;
91 + u_char *data;
92 + File *fi = file+file_count;;
93 +
94 + fd = open (name, O_RDONLY, 0);
95 + if (fd < 0)
96 + { perror (name);
97 + exit (2);
98 + }
99 +
100 + if (fstat (fd, &fs))
101 + { perror (name);
102 + exit (2);
103 + }
104 +
105 + if (fs.st_size >= MIN_FILE_SIZE
106 + && fs.st_size <= MAX_FILE_SIZE)
107 + { fi->length = fs.st_size;
108 + fi->name = name;
109 +
110 + data = (u_char *) mmap (0, fs.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
111 +
112 + if (data == (u_char *) -1)
113 + { perror (name);
114 + exit (2);
115 + }
116 +
117 + gb_simm_process (data, fs.st_size, fi->md);
118 + if (flag_relative)
119 + { int d = dist (fi->md, relative_md);
120 + double sim = 1.0 - MIN (1.0, (double) (d) / (MD_LENGTH * 4 - 1));
121 + fprintf (stdout, "%s %llu %u %s %u %3.1f\n",
122 + md_to_str (fi->md), (long long unsigned) 0,
123 + (unsigned) fs.st_size, name,
124 + d, 100.0 * sim);
125 + }
126 + else
127 + {
128 + fprintf (stdout, "%s %llu %u %s\n",
129 + md_to_str (fi->md), (long long unsigned) 0,
130 + (unsigned) fs.st_size, name);
131 + }
132 + munmap (data, fs.st_size);
133 + file_bytes += fs.st_size;
134 + file_count++;
135 + } else if (flag_verbose)
136 + { fprintf (stdout, "skipping %s (size %llu)\n", name, (long long unsigned) fs.st_size); }
137 +
138 + close (fd);
139 +}
140 +
141 +u_char *str_to_md(char *str, u_char *md)
142 +{ int j;
143 +
144 + if (!md || !str) return 0;
145 +
146 + bzero (md, MD_LENGTH);
147 +
148 + for (j = 0; j < MD_LENGTH * 2; j++)
149 + { char ch = str[j];
150 +
151 + if (ch >= '0' && ch <= '9')
152 + { md [j/2] = (md [j/2] << 4) + (ch - '0');
153 + }
154 + else
155 + { ch |= 32;
156 +
157 + if (ch < 'a' || ch > 'f') break;
158 + md [j/2] = (md[j/2] << 4) + (ch - 'a' + 10);
159 + } }
160 +
161 + return (j != MD_LENGTH * 2 || str[j] != 0) ? 0 : md;
162 +}
163 +
164 +int main (int argc, char *argv[])
165 +{ int ch, j;
166 +
167 + strncpy (cmd, basename (argv[0]), 8);
168 +
169 + while ((ch = getopt(argc, argv, "dhr:vw")) != -1)
170 + { switch (ch)
171 + { case 'd': flag_debug++;
172 + break;
173 + case 'r': if (!optarg)
174 + { fprintf (stderr, "%s: missing argument for -r\n", cmd);
175 + return 1;
176 + }
177 + if (str_to_md (optarg, relative_md)) flag_relative = optarg;
178 + else
179 + { fprintf (stderr, "%s: not a valid fingerprint\n", optarg);
180 + return 1;
181 + }
182 + break;
183 + case 'v': flag_verbose++;
184 + break;
185 + case 'w': break;
186 + default : usage();
187 + return (ch != 'h');
188 + } }
189 +
190 + argc -= optind;
191 + argv += optind;
192 +
193 + if (argc == 0) usage();
194 +
195 + rabin_reset ();
196 + if (flag_verbose && flag_relative)
197 + { fprintf (stdout, "distances are relative to %s\n", flag_relative);
198 + }
199 +
200 + file = (File *) calloc (argc, sizeof (File));
201 +
202 + for (j = 0; j < argc; j++) process_file (argv[j]);
203 +
204 + if (flag_verbose)
205 + { fprintf (stdout, "%li bytes in %i files\n", (long) file_bytes, file_count);
206 + }
207 +
208 + return 0;
209 +}