@matheusbsbs / matheus-bagliano / commits / 9a305b67f8
Fork 0

Geert's similarity

Define a function to compute similarity score 0.0<=score<=1.0 Signed-off-by: Junio C Hamano <junkio@cox.net>

Junio C Hamano committed Apr 16, 2006 at 21:07 UTC 9a305b67f8055503a743e67f628400f094c169ee
3 files changed +39 -24
gsimm.c
+29
@@ -1,3 +1,4 @@
1 +#include <string.h>
2 #include "rabinpoly.h"
3 #include "gsimm.h"
4
@@ -32,6 +33,29 @@ static void freq_to_md(u_char *md, int *freq)
33 bzero (freq, sizeof(freq[0]) * MD_BITS);
34 }
35
36 +static int dist (u_char *l, u_char *r)
37 +{ int j, k;
38 + int d = 0;
39 +
40 + for (j = 0; j < MD_LENGTH; j++)
41 + { u_char ch = l[j] ^ r[j];
42 +
43 + for (k = 0; k < 8; k++) d += ((ch & (1<<k)) > 0);
44 + }
45 +
46 + return d;
47 +}
48 +
49 +double gb_simm_score(u_char *l, u_char *r)
50 +{
51 + int d = dist(l, r);
52 + double sim = (double) (d) / (MD_LENGTH * 4 - 1);
53 + if (1.0 < sim)
54 + return 0;
55 + else
56 + return 1.0 - sim;
57 +}
58 +
59 void gb_simm_process(u_char *data, unsigned len, u_char *md)
60 { size_t j = 0;
61 u_int32_t ofs;
@@ -39,6 +63,11 @@ void gb_simm_process(u_char *data, unsigned len, u_char *md)
63 u_int32_t count [MD_BITS * (GROUP_COUNTERS/GROUP_BITS)];
64 int freq[MD_BITS];
65
66 + if (len < GB_SIMM_MIN_FILE_SIZE || GB_SIMM_MAX_FILE_SIZE < len) {
67 + memset(md, 0, MD_LENGTH);
68 + return;
69 + }
70 +
71 bzero (freq, sizeof(freq[0]) * MD_BITS);
72 bzero (dup_cache, DUP_CACHE_SIZE * sizeof (u_int32_t));
73 bzero (count, (MD_BITS * (GROUP_COUNTERS/GROUP_BITS) * sizeof (u_int32_t)));
gsimm.h
+3 -2
@@ -15,14 +15,15 @@
15 In order to get at least an average of 12 samples
16 per bit in the final message digest, require at least 3 * MD_LENGTH
17 complete windows in the file. */
18 -#define MIN_FILE_SIZE (3 * MD_LENGTH + 2 * (RABIN_WINDOW_SIZE - 1))
18 +#define GB_SIMM_MIN_FILE_SIZE (3 * MD_LENGTH + 2 * (RABIN_WINDOW_SIZE - 1))
19
20 /* Limit matching algorithm to files less than 256 MB, so we can use
21 32 bit integers everywhere without fear of overflow. For larger
22 files we should add logic to mmap the file by piece and accumulate
23 the frequency counts. */
24 -#define MAX_FILE_SIZE (256*1024*1024 - 1)
24 +#define GB_SIMM_MAX_FILE_SIZE (256*1024*1024 - 1)
25
26 void gb_simm_process(u_char *data, unsigned len, u_char *md);
27 +double gb_simm_score(u_char *l, u_char *r);
28
29 #endif
test-gsimm.c
+7 -22
@@ -58,19 +58,6 @@ void usage()
58 exit (1);
59 }
60
61 -int dist (u_char *l, u_char *r)
62 -{ int j, k;
63 - int d = 0;
64 -
65 - for (j = 0; j < MD_LENGTH; j++)
66 - { u_char ch = l[j] ^ r[j];
67 -
68 - for (k = 0; k < 8; k++) d += ((ch & (1<<k)) > 0);
69 - }
70 -
71 - return d;
72 -}
73 -
61 char *md_to_str(u_char *md)
62 { int j;
63
@@ -102,8 +89,8 @@ void process_file (char *name)
89 exit (2);
90 }
91
105 - if (fs.st_size >= MIN_FILE_SIZE
106 - && fs.st_size <= MAX_FILE_SIZE)
92 + if (fs.st_size >= GB_SIMM_MIN_FILE_SIZE
93 + && fs.st_size <= GB_SIMM_MAX_FILE_SIZE)
94 { fi->length = fs.st_size;
95 fi->name = name;
96
@@ -116,13 +103,11 @@ void process_file (char *name)
103
104 gb_simm_process (data, fs.st_size, fi->md);
105 if (flag_relative)
119 - { int d = dist (fi->md, relative_md);
120 - double sim = 1.0 - MIN (1.0, (double) (d) / (MD_LENGTH * 4 - 1));
121 - fprintf (stdout, "%s %llu %u %s %u %3.1f\n",
122 - md_to_str (fi->md), (long long unsigned) 0,
123 - (unsigned) fs.st_size, name,
124 - d, 100.0 * sim);
125 - }
106 + fprintf (stdout, "%s %llu %u %s %u %3.1f\n",
107 + md_to_str (fi->md), (long long unsigned) 0,
108 + (unsigned) fs.st_size, name,
109 + (unsigned) 0,
110 + 100.0 * gb_simm_score(fi->md, relative_md));
111 else
112 {
113 fprintf (stdout, "%s %llu %u %s\n",