@matheusbsbs / matheus-bagliano / commits / a872275098

teach fast-export an --anonymize option

Sometimes users want to report a bug they experience on their repository, but they are not at liberty to share the contents of the repository. It would be useful if they could produce a repository that has a similar shape to its history and tree, but without leaking any information. This "anonymized" repository could then be shared with developers (assuming it still replicates the original problem). This patch implements an "--anonymize" option to fast-export, which generates a stream that can recreate such a repository. Producing a single stream makes it easy for the caller to verify that they are not leaking any useful information. You can get an overview of what will be shared by running a command like: git fast-export --anonymize --all | perl -pe 's/\d+/X/g' | sort -u | less which will show every unique line we generate, modulo any numbers (each anonymized token is assigned a number, like "User 0", and we replace it consistently in the output). In addition to anonymizing, this produces test cases that are relatively small (compared to the original repository) and fast to generate (compared to using filter-branch, or modifying the output of fast-export yourself). Here are numbers for git.git: $ time git fast-export --anonymize --all \ --tag-of-filtered-object=drop >output real 0m2.883s user 0m2.828s sys 0m0.052s $ gzip output $ ls -lh output.gz | awk '{print $5}' 2.9M Signed-off-by: Jeff King <peff@peff.net> Signed-off-by: Junio C Hamano <gitster@pobox.com>

Jeff King committed Aug 27, 2014 at 13:01 UTC a8722750985a53cc502a66ae3d68a9e42c7fdb98

3 files changed +407 -11

Documentation/git-fast-export.txt

@@ -105,6 +105,12 @@ marks the same across runs.

105 in the commit (as opposed to just listing the files which are

106 different from the commit's first parent).

107

108 +--anonymize::

109 + Replace all refnames, paths, blob contents, commit and tag

110 + messages, names, and email addresses in the output with

111 + anonymized data, while still retaining the shape of history and

112 + of the stored tree.

113 +

114 --refspec::

115 Apply the specified refspec to each ref exported. Multiple of them can

116 be specified.

builtin/fast-export.c

+289 -11

@@ -18,6 +18,7 @@

18 #include "parse-options.h"

19 #include "quote.h"

20 #include "remote.h"

21 +#include "blob.h"

23 static const char *fast_export_usage[] = {

24 N_("git fast-export [rev-list-opts]"),

@@ -34,6 +35,7 @@ static int full_tree;

35 static struct string_list extra_refs = STRING_LIST_INIT_NODUP;

36 static struct refspec *refspecs;

37 static int refspecs_nr;

38 +static int anonymize;

40 static int parse_opt_signed_tag_mode(const struct option *opt,

41 const char *arg, int unset)

@@ -81,6 +83,76 @@ static int has_unshown_parent(struct commit *commit)

83 return 0;

84 }

86 +struct anonymized_entry {

87 + struct hashmap_entry hash;

88 + const char *orig;

89 + size_t orig_len;

90 + const char *anon;

91 + size_t anon_len;

92 +};

93 +

94 +static int anonymized_entry_cmp(const void *va, const void *vb,

95 + const void *data)

96 +{

97 + const struct anonymized_entry *a = va, *b = vb;

98 + return a->orig_len != b->orig_len ||

99 + memcmp(a->orig, b->orig, a->orig_len);

100 +}

101 +

102 +/*

103 + * Basically keep a cache of X->Y so that we can repeatedly replace

104 + * the same anonymized string with another. The actual generation

105 + * is farmed out to the generate function.

106 + */

107 +static const void *anonymize_mem(struct hashmap *map,

108 + void *(*generate)(const void *, size_t *),

109 + const void *orig, size_t *len)

110 +{

111 + struct anonymized_entry key, *ret;

112 +

113 + if (!map->cmpfn)

114 + hashmap_init(map, anonymized_entry_cmp, 0);

115 +

116 + hashmap_entry_init(&key, memhash(orig, *len));

117 + key.orig = orig;

118 + key.orig_len = *len;

119 + ret = hashmap_get(map, &key, NULL);

120 +

121 + if (!ret) {

122 + ret = xmalloc(sizeof(*ret));

123 + hashmap_entry_init(&ret->hash, key.hash.hash);

124 + ret->orig = xstrdup(orig);

125 + ret->orig_len = *len;

126 + ret->anon = generate(orig, len);

127 + ret->anon_len = *len;

128 + hashmap_put(map, ret);

129 + }

130 +

131 + *len = ret->anon_len;

132 + return ret->anon;

133 +}

134 +

135 +/*

136 + * We anonymize each component of a path individually,

137 + * so that paths a/b and a/c will share a common root.

138 + * The paths are cached via anonymize_mem so that repeated

139 + * lookups for "a" will yield the same value.

140 + */

141 +static void anonymize_path(struct strbuf *out, const char *path,

142 + struct hashmap *map,

143 + void *(*generate)(const void *, size_t *))

144 +{

145 + while (*path) {

146 + const char *end_of_component = strchrnul(path, '/');

147 + size_t len = end_of_component - path;

148 + const char *c = anonymize_mem(map, generate, path, &len);

149 + strbuf_add(out, c, len);

150 + path = end_of_component;

151 + if (*path)

152 + strbuf_addch(out, *path++);

153 + }

154 +}

155 +

156 /* Since intptr_t is C99, we do not use it here */

157 static inline uint32_t *mark_to_ptr(uint32_t mark)

158 {

@@ -119,6 +191,26 @@ static void show_progress(void)

191 printf("progress %d objects\n", counter);

192 }

193

194 +/*

195 + * Ideally we would want some transformation of the blob data here

196 + * that is unreversible, but would still be the same size and have

197 + * the same data relationship to other blobs (so that we get the same

198 + * delta and packing behavior as the original). But the first and last

199 + * requirements there are probably mutually exclusive, so let's take

200 + * the easy way out for now, and just generate arbitrary content.

201 + *

202 + * There's no need to cache this result with anonymize_mem, since

203 + * we already handle blob content caching with marks.

204 + */

205 +static char *anonymize_blob(unsigned long *size)

206 +{

207 + static int counter;

208 + struct strbuf out = STRBUF_INIT;

209 + strbuf_addf(&out, "anonymous blob %d", counter++);

210 + *size = out.len;

211 + return strbuf_detach(&out, NULL);

212 +}

213 +

214 static void export_blob(const unsigned char *sha1)

215 {

216 unsigned long size;

@@ -137,12 +229,19 @@ static void export_blob(const unsigned char *sha1)

229 if (object && object->flags & SHOWN)

230 return;

231

140 - buf = read_sha1_file(sha1, &type, &size);

141 - if (!buf)

142 - die ("Could not read blob %s", sha1_to_hex(sha1));

143 - if (check_sha1_signature(sha1, buf, size, typename(type)) < 0)

144 - die("sha1 mismatch in blob %s", sha1_to_hex(sha1));

145 - object = parse_object_buffer(sha1, type, size, buf, &eaten);

232 + if (anonymize) {

233 + buf = anonymize_blob(&size);

234 + object = (struct object *)lookup_blob(sha1);

235 + eaten = 0;

236 + } else {

237 + buf = read_sha1_file(sha1, &type, &size);

238 + if (!buf)

239 + die ("Could not read blob %s", sha1_to_hex(sha1));

240 + if (check_sha1_signature(sha1, buf, size, typename(type)) < 0)

241 + die("sha1 mismatch in blob %s", sha1_to_hex(sha1));

242 + object = parse_object_buffer(sha1, type, size, buf, &eaten);

243 + }

244 +

245 if (!object)

246 die("Could not read blob %s", sha1_to_hex(sha1));

247

@@ -190,7 +289,7 @@ static int depth_first(const void *a_, const void *b_)

289 return (a->status == 'R') - (b->status == 'R');

290 }

291

193 -static void print_path(const char *path)

292 +static void print_path_1(const char *path)

293 {

294 int need_quote = quote_c_style(path, NULL, NULL, 0);

295 if (need_quote)

@@ -201,6 +300,43 @@ static void print_path(const char *path)

300 printf("%s", path);

301 }

302

303 +static void *anonymize_path_component(const void *path, size_t *len)

304 +{

305 + static int counter;

306 + struct strbuf out = STRBUF_INIT;

307 + strbuf_addf(&out, "path%d", counter++);

308 + return strbuf_detach(&out, len);

309 +}

310 +

311 +static void print_path(const char *path)

312 +{

313 + if (!anonymize)

314 + print_path_1(path);

315 + else {

316 + static struct hashmap paths;

317 + static struct strbuf anon = STRBUF_INIT;

318 +

319 + anonymize_path(&anon, path, &paths, anonymize_path_component);

320 + print_path_1(anon.buf);

321 + strbuf_reset(&anon);

322 + }

323 +}

324 +

325 +static void *generate_fake_sha1(const void *old, size_t *len)

326 +{

327 + static uint32_t counter = 1; /* avoid null sha1 */

328 + unsigned char *out = xcalloc(20, 1);

329 + put_be32(out + 16, counter++);

330 + return out;

331 +}

332 +

333 +static const unsigned char *anonymize_sha1(const unsigned char *sha1)

334 +{

335 + static struct hashmap sha1s;

336 + size_t len = 20;

337 + return anonymize_mem(&sha1s, generate_fake_sha1, sha1, &len);

338 +}

339 +

340 static void show_filemodify(struct diff_queue_struct *q,

341 struct diff_options *options, void *data)

342 {

@@ -245,7 +381,9 @@ static void show_filemodify(struct diff_queue_struct *q,

381 */

382 if (no_data || S_ISGITLINK(spec->mode))

383 printf("M %06o %s ", spec->mode,

248 - sha1_to_hex(spec->sha1));

384 + sha1_to_hex(anonymize ?

385 + anonymize_sha1(spec->sha1) :

386 + spec->sha1));

387 else {

388 struct object *object = lookup_object(spec->sha1);

389 printf("M %06o :%d ", spec->mode,

@@ -279,6 +417,114 @@ static const char *find_encoding(const char *begin, const char *end)

417 return bol;

418 }

419

420 +static void *anonymize_ref_component(const void *old, size_t *len)

421 +{

422 + static int counter;

423 + struct strbuf out = STRBUF_INIT;

424 + strbuf_addf(&out, "ref%d", counter++);

425 + return strbuf_detach(&out, len);

426 +}

427 +

428 +static const char *anonymize_refname(const char *refname)

429 +{

430 + /*

431 + * If any of these prefixes is found, we will leave it intact

432 + * so that tags remain tags and so forth.

433 + */

434 + static const char *prefixes[] = {

435 + "refs/heads/",

436 + "refs/tags/",

437 + "refs/remotes/",

438 + "refs/"

439 + };

440 + static struct hashmap refs;

441 + static struct strbuf anon = STRBUF_INIT;

442 + int i;

443 +

444 + /*

445 + * We also leave "master" as a special case, since it does not reveal

446 + * anything interesting.

447 + */

448 + if (!strcmp(refname, "refs/heads/master"))

449 + return refname;

450 +

451 + strbuf_reset(&anon);

452 + for (i = 0; i < ARRAY_SIZE(prefixes); i++) {

453 + if (skip_prefix(refname, prefixes[i], &refname)) {

454 + strbuf_addstr(&anon, prefixes[i]);

455 + break;

456 + }

457 + }

458 +

459 + anonymize_path(&anon, refname, &refs, anonymize_ref_component);

460 + return anon.buf;

461 +}

462 +

463 +/*

464 + * We do not even bother to cache commit messages, as they are unlikely

465 + * to be repeated verbatim, and it is not that interesting when they are.

466 + */

467 +static char *anonymize_commit_message(const char *old)

468 +{

469 + static int counter;

470 + return xstrfmt("subject %d\n\nbody\n", counter++);

471 +}

472 +

473 +static struct hashmap idents;

474 +static void *anonymize_ident(const void *old, size_t *len)

475 +{

476 + static int counter;

477 + struct strbuf out = STRBUF_INIT;

478 + strbuf_addf(&out, "User %d <user%d@example.com>", counter, counter);

479 + counter++;

480 + return strbuf_detach(&out, len);

481 +}

482 +

483 +/*

484 + * Our strategy here is to anonymize the names and email addresses,

485 + * but keep timestamps intact, as they influence things like traversal

486 + * order (and by themselves should not be too revealing).

487 + */

488 +static void anonymize_ident_line(const char **beg, const char **end)

489 +{

490 + static struct strbuf buffers[] = { STRBUF_INIT, STRBUF_INIT };

491 + static unsigned which_buffer;

492 +

493 + struct strbuf *out;

494 + struct ident_split split;

495 + const char *end_of_header;

496 +

497 + out = &buffers[which_buffer++];

498 + which_buffer %= ARRAY_SIZE(buffers);

499 + strbuf_reset(out);

500 +

501 + /* skip "committer", "author", "tagger", etc */

502 + end_of_header = strchr(*beg, ' ');

503 + if (!end_of_header)

504 + die("BUG: malformed line fed to anonymize_ident_line: %.*s",

505 + (int)(*end - *beg), *beg);

506 + end_of_header++;

507 + strbuf_add(out, *beg, end_of_header - *beg);

508 +

509 + if (!split_ident_line(&split, end_of_header, *end - end_of_header) &&

510 + split.date_begin) {

511 + const char *ident;

512 + size_t len;

513 +

514 + len = split.mail_end - split.name_begin;

515 + ident = anonymize_mem(&idents, anonymize_ident,

516 + split.name_begin, &len);

517 + strbuf_add(out, ident, len);

518 + strbuf_addch(out, ' ');

519 + strbuf_add(out, split.date_begin, split.tz_end - split.date_begin);

520 + } else {

521 + strbuf_addstr(out, "Malformed Ident <malformed@example.com> 0 -0000");

522 + }

523 +

524 + *beg = out->buf;

525 + *end = out->buf + out->len;

526 +}

527 +

528 static void handle_commit(struct commit *commit, struct rev_info *rev)

529 {

530 int saved_output_format = rev->diffopt.output_format;

@@ -287,6 +533,7 @@ static void handle_commit(struct commit *commit, struct rev_info *rev)

533 const char *encoding, *message;

534 char *reencoded = NULL;

535 struct commit_list *p;

536 + const char *refname;

537 int i;

538

539 rev->diffopt.output_format = DIFF_FORMAT_CALLBACK;

@@ -326,13 +573,22 @@ static void handle_commit(struct commit *commit, struct rev_info *rev)

573 if (!S_ISGITLINK(diff_queued_diff.queue[i]->two->mode))

574 export_blob(diff_queued_diff.queue[i]->two->sha1);

575

576 + refname = commit->util;

577 + if (anonymize) {

578 + refname = anonymize_refname(refname);

579 + anonymize_ident_line(&committer, &committer_end);

580 + anonymize_ident_line(&author, &author_end);

581 + }

582 +

583 mark_next_object(&commit->object);

330 - if (!is_encoding_utf8(encoding))

584 + if (anonymize)

585 + reencoded = anonymize_commit_message(message);

586 + else if (!is_encoding_utf8(encoding))

587 reencoded = reencode_string(message, "UTF-8", encoding);

588 if (!commit->parents)

333 - printf("reset %s\n", (const char*)commit->util);

589 + printf("reset %s\n", refname);

590 printf("commit %s\nmark :%"PRIu32"\n%.*s\n%.*s\ndata %u\n%s",

335 - (const char *)commit->util, last_idnum,

591 + refname, last_idnum,

592 (int)(author_end - author), author,

593 (int)(committer_end - committer), committer,

594 (unsigned)(reencoded

@@ -363,6 +619,14 @@ static void handle_commit(struct commit *commit, struct rev_info *rev)

619 show_progress();

620 }

621

622 +static void *anonymize_tag(const void *old, size_t *len)

623 +{

624 + static int counter;

625 + struct strbuf out = STRBUF_INIT;

626 + strbuf_addf(&out, "tag message %d", counter++);

627 + return strbuf_detach(&out, len);

628 +}

629 +

630 static void handle_tail(struct object_array *commits, struct rev_info *revs)

631 {

632 struct commit *commit;

@@ -419,6 +683,17 @@ static void handle_tag(const char *name, struct tag *tag)

683 } else {

684 tagger++;

685 tagger_end = strchrnul(tagger, '\n');

686 + if (anonymize)

687 + anonymize_ident_line(&tagger, &tagger_end);

688 + }

689 +

690 + if (anonymize) {

691 + name = anonymize_refname(name);

692 + if (message) {

693 + static struct hashmap tags;

694 + message = anonymize_mem(&tags, anonymize_tag,

695 + message, &message_size);

696 + }

697 }

698

699 /* handle signed tags */

@@ -584,6 +859,8 @@ static void handle_tags_and_duplicates(void)

859 handle_tag(name, (struct tag *)object);

860 break;

861 case OBJ_COMMIT:

862 + if (anonymize)

863 + name = anonymize_refname(name);

864 /* create refs pointing to already seen commits */

865 commit = (struct commit *)object;

866 printf("reset %s\nfrom :%d\n\n", name,

@@ -719,6 +996,7 @@ int cmd_fast_export(int argc, const char **argv, const char *prefix)

996 OPT_BOOL(0, "no-data", &no_data, N_("Skip output of blob data")),

997 OPT_STRING_LIST(0, "refspec", &refspecs_list, N_("refspec"),

998 N_("Apply refspec to exported refs")),

999 + OPT_BOOL(0, "anonymize", &anonymize, N_("anonymize output")),

1000 OPT_END()

1001 };

1002

t/t9351-fast-export-anonymize.sh new

+112

@@ -0,0 +1,112 @@

1 +#!/bin/sh

2 +

3 +test_description='basic tests for fast-export --anonymize'

4 +. ./test-lib.sh

5 +

6 +test_expect_success 'setup simple repo' '

7 + test_commit base &&

8 + test_commit foo &&

9 + git checkout -b other HEAD^ &&

10 + mkdir subdir &&

11 + test_commit subdir/bar &&

12 + test_commit subdir/xyzzy &&

13 + git tag -m "annotated tag" mytag

14 +'

15 +

16 +test_expect_success 'export anonymized stream' '

17 + git fast-export --anonymize --all >stream

18 +'

19 +

20 +# this also covers commit messages

21 +test_expect_success 'stream omits path names' '

22 + ! grep base stream &&

23 + ! grep foo stream &&

24 + ! grep subdir stream &&

25 + ! grep bar stream &&

26 + ! grep xyzzy stream

27 +'

28 +

29 +test_expect_success 'stream allows master as refname' '

30 + grep master stream

31 +'

32 +

33 +test_expect_success 'stream omits other refnames' '

34 + ! grep other stream &&

35 + ! grep mytag stream

36 +'

37 +

38 +test_expect_success 'stream omits identities' '

39 + ! grep "$GIT_COMMITTER_NAME" stream &&

40 + ! grep "$GIT_COMMITTER_EMAIL" stream &&

41 + ! grep "$GIT_AUTHOR_NAME" stream &&

42 + ! grep "$GIT_AUTHOR_EMAIL" stream

43 +'

44 +

45 +test_expect_success 'stream omits tag message' '

46 + ! grep "annotated tag" stream

47 +'

48 +

49 +# NOTE: we chdir to the new, anonymized repository

50 +# after this. All further tests should assume this.

51 +test_expect_success 'import stream to new repository' '

52 + git init new &&

53 + cd new &&

54 + git fast-import <../stream

55 +'

56 +

57 +test_expect_success 'result has two branches' '

58 + git for-each-ref --format="%(refname)" refs/heads >branches &&

59 + test_line_count = 2 branches &&

60 + other_branch=$(grep -v refs/heads/master branches)

61 +'

62 +

63 +test_expect_success 'repo has original shape and timestamps' '

64 + shape () {

65 + git log --format="%m %ct" --left-right --boundary "$@"

66 + } &&

67 + (cd .. && shape master...other) >expect &&

68 + shape master...$other_branch >actual &&

69 + test_cmp expect actual

70 +'

71 +

72 +test_expect_success 'root tree has original shape' '

73 + # the output entries are not necessarily in the same

74 + # order, but we know at least that we will have one tree

75 + # and one blob, so just check the sorted order

76 + cat >expect <<-\EOF &&

77 + blob

78 + tree

79 + EOF

80 + git ls-tree $other_branch >root &&

81 + cut -d" " -f2 <root | sort >actual &&

82 + test_cmp expect actual

83 +'

84 +

85 +test_expect_success 'paths in subdir ended up in one tree' '

86 + cat >expect <<-\EOF &&

87 + blob

88 + blob

89 + EOF

90 + tree=$(grep tree root | cut -f2) &&

91 + git ls-tree $other_branch:$tree >tree &&

92 + cut -d" " -f2 <tree >actual &&

93 + test_cmp expect actual

94 +'

95 +

96 +test_expect_success 'tag points to branch tip' '

97 + git rev-parse $other_branch >expect &&

98 + git for-each-ref --format="%(*objectname)" | grep . >actual &&

99 + test_cmp expect actual

100 +'

101 +

102 +test_expect_success 'idents are shared' '

103 + git log --all --format="%an <%ae>" >authors &&

104 + sort -u authors >unique &&

105 + test_line_count = 1 unique &&

106 + git log --all --format="%cn <%ce>" >committers &&

107 + sort -u committers >unique &&

108 + test_line_count = 1 unique &&

109 + ! test_cmp authors committers

110 +'

111 +

112 +test_done