@matheusbsbs / matheus-bagliano / commits / 21746aa34f
Fork 0

SVN dump parser

svndump parses data that is in SVN dumpfile format produced by `svnadmin dump` with the help of line_buffer and uses repo_tree and fast_export to emit a git fast-import stream. Based roughly on com.hydrografix.svndump 0.92 from the SvnToCCase project at <http://svn2cc.sarovar.org/>, by Stefan Hegny and others. [rr: allow input from files other than stdin] [jn: with test, more error reporting] Signed-off-by: David Barr <david.barr@cordelta.com> Signed-off-by: Ramkumar Ramachandra <artagnon@gmail.com> Signed-off-by: Jonathan Nieder <jrnieder@gmail.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>

David Barr committed Aug 9, 2010 at 17:55 UTC 21746aa34fc99d2c73634bc9829387c27c109dbe
8 files changed +372 -2
.gitignore
+1
@@ -175,6 +175,7 @@
175 /test-sha1
176 /test-sigchain
177 /test-string-pool
178 +/test-svn-fe
179 /test-treap
180 /common-cmds.h
181 *.tar.gz
Makefile
+6 -2
@@ -417,6 +417,7 @@ TEST_PROGRAMS_NEED_X += test-run-command
417 TEST_PROGRAMS_NEED_X += test-sha1
418 TEST_PROGRAMS_NEED_X += test-sigchain
419 TEST_PROGRAMS_NEED_X += test-string-pool
420 +TEST_PROGRAMS_NEED_X += test-svn-fe
421 TEST_PROGRAMS_NEED_X += test-treap
422 TEST_PROGRAMS_NEED_X += test-index-version
423
@@ -1745,7 +1746,7 @@ endif
1746 XDIFF_OBJS = xdiff/xdiffi.o xdiff/xprepare.o xdiff/xutils.o xdiff/xemit.o \
1747 xdiff/xmerge.o xdiff/xpatience.o
1748 VCSSVN_OBJS = vcs-svn/string_pool.o vcs-svn/line_buffer.o \
1748 - vcs-svn/repo_tree.o vcs-svn/fast_export.o
1749 + vcs-svn/repo_tree.o vcs-svn/fast_export.o vcs-svn/svndump.o
1750 OBJECTS := $(GIT_OBJS) $(XDIFF_OBJS) $(VCSSVN_OBJS)
1751
1752 dep_files := $(foreach f,$(OBJECTS),$(dir $f).depend/$(notdir $f).d)
@@ -1871,7 +1872,8 @@ xdiff-interface.o $(XDIFF_OBJS): \
1872
1873 $(VCSSVN_OBJS): \
1874 vcs-svn/obj_pool.h vcs-svn/trp.h vcs-svn/string_pool.h \
1874 - vcs-svn/line_buffer.h vcs-svn/repo_tree.h vcs-svn/fast_export.h
1875 + vcs-svn/line_buffer.h vcs-svn/repo_tree.h vcs-svn/fast_export.h \
1876 + vcs-svn/svndump.h
1877 endif
1878
1879 exec_cmd.s exec_cmd.o: EXTRA_CPPFLAGS = \
@@ -2026,6 +2028,8 @@ test-parse-options$X: parse-options.o
2028
2029 test-string-pool$X: vcs-svn/lib.a
2030
2031 +test-svn-fe$X: vcs-svn/lib.a
2032 +
2033 .PRECIOUS: $(TEST_OBJS)
2034
2035 test-%$X: test-%.o $(GITLIBS)
contrib/svn-fe/svn-fe.c
+1
@@ -10,6 +10,7 @@ int main(int argc, char **argv)
10 {
11 svndump_init(NULL);
12 svndump_read((argc > 1) ? argv[1] : NULL);
13 + svndump_deinit();
14 svndump_reset();
15 return 0;
16 }
t/t9010-svn-fe.sh new
+32
@@ -0,0 +1,32 @@
1 +#!/bin/sh
2 +
3 +test_description='check svn dumpfile importer'
4 +
5 +. ./lib-git-svn.sh
6 +
7 +test_dump() {
8 + label=$1
9 + dump=$2
10 + test_expect_success "$dump" '
11 + svnadmin create "$label-svn" &&
12 + svnadmin load "$label-svn" < "$TEST_DIRECTORY/$dump" &&
13 + svn_cmd export "file://$(pwd)/$label-svn" "$label-svnco" &&
14 + git init "$label-git" &&
15 + test-svn-fe "$TEST_DIRECTORY/$dump" >"$label.fe" &&
16 + (
17 + cd "$label-git" &&
18 + git fast-import < ../"$label.fe"
19 + ) &&
20 + (
21 + cd "$label-svnco" &&
22 + git init &&
23 + git add . &&
24 + git fetch "../$label-git" master &&
25 + git diff --exit-code FETCH_HEAD
26 + )
27 + '
28 +}
29 +
30 +test_dump simple t9111/svnsync.dump
31 +
32 +test_done
test-svn-fe.c new
+17
@@ -0,0 +1,17 @@
1 +/*
2 + * test-svn-fe: Code to exercise the svn import lib
3 + */
4 +
5 +#include "git-compat-util.h"
6 +#include "vcs-svn/svndump.h"
7 +
8 +int main(int argc, char *argv[])
9 +{
10 + if (argc != 2)
11 + usage("test-svn-fe <file>");
12 + svndump_init(argv[1]);
13 + svndump_read(NULL);
14 + svndump_deinit();
15 + svndump_reset();
16 + return 0;
17 +}
vcs-svn/LICENSE
+4
@@ -4,6 +4,10 @@ All rights reserved.
4 Copyright (C) 2008 Jason Evans <jasone@canonware.com>.
5 All rights reserved.
6
7 +Copyright (C) 2005 Stefan Hegny, hydrografix Consulting GmbH,
8 +Frankfurt/Main, Germany
9 +and others, see http://svn2cc.sarovar.org
10 +
11 Redistribution and use in source and binary forms, with or without
12 modification, are permitted provided that the following conditions
13 are met:
vcs-svn/svndump.c new
+302
@@ -0,0 +1,302 @@
1 +/*
2 + * Parse and rearrange a svnadmin dump.
3 + * Create the dump with:
4 + * svnadmin dump --incremental -r<startrev>:<endrev> <repository> >outfile
5 + *
6 + * Licensed under a two-clause BSD-style license.
7 + * See LICENSE for details.
8 + */
9 +
10 +#include "cache.h"
11 +#include "repo_tree.h"
12 +#include "fast_export.h"
13 +#include "line_buffer.h"
14 +#include "obj_pool.h"
15 +#include "string_pool.h"
16 +
17 +#define NODEACT_REPLACE 4
18 +#define NODEACT_DELETE 3
19 +#define NODEACT_ADD 2
20 +#define NODEACT_CHANGE 1
21 +#define NODEACT_UNKNOWN 0
22 +
23 +#define DUMP_CTX 0
24 +#define REV_CTX 1
25 +#define NODE_CTX 2
26 +
27 +#define LENGTH_UNKNOWN (~0)
28 +#define DATE_RFC2822_LEN 31
29 +
30 +/* Create memory pool for log messages */
31 +obj_pool_gen(log, char, 4096)
32 +
33 +static char* log_copy(uint32_t length, char *log)
34 +{
35 + char *buffer;
36 + log_free(log_pool.size);
37 + buffer = log_pointer(log_alloc(length));
38 + strncpy(buffer, log, length);
39 + return buffer;
40 +}
41 +
42 +static struct {
43 + uint32_t action, propLength, textLength, srcRev, srcMode, mark, type;
44 + uint32_t src[REPO_MAX_PATH_DEPTH], dst[REPO_MAX_PATH_DEPTH];
45 +} node_ctx;
46 +
47 +static struct {
48 + uint32_t revision, author;
49 + unsigned long timestamp;
50 + char *log;
51 +} rev_ctx;
52 +
53 +static struct {
54 + uint32_t uuid, url;
55 +} dump_ctx;
56 +
57 +static struct {
58 + uint32_t svn_log, svn_author, svn_date, svn_executable, svn_special, uuid,
59 + revision_number, node_path, node_kind, node_action,
60 + node_copyfrom_path, node_copyfrom_rev, text_content_length,
61 + prop_content_length, content_length;
62 +} keys;
63 +
64 +static void reset_node_ctx(char *fname)
65 +{
66 + node_ctx.type = 0;
67 + node_ctx.action = NODEACT_UNKNOWN;
68 + node_ctx.propLength = LENGTH_UNKNOWN;
69 + node_ctx.textLength = LENGTH_UNKNOWN;
70 + node_ctx.src[0] = ~0;
71 + node_ctx.srcRev = 0;
72 + node_ctx.srcMode = 0;
73 + pool_tok_seq(REPO_MAX_PATH_DEPTH, node_ctx.dst, "/", fname);
74 + node_ctx.mark = 0;
75 +}
76 +
77 +static void reset_rev_ctx(uint32_t revision)
78 +{
79 + rev_ctx.revision = revision;
80 + rev_ctx.timestamp = 0;
81 + rev_ctx.log = NULL;
82 + rev_ctx.author = ~0;
83 +}
84 +
85 +static void reset_dump_ctx(uint32_t url)
86 +{
87 + dump_ctx.url = url;
88 + dump_ctx.uuid = ~0;
89 +}
90 +
91 +static void init_keys(void)
92 +{
93 + keys.svn_log = pool_intern("svn:log");
94 + keys.svn_author = pool_intern("svn:author");
95 + keys.svn_date = pool_intern("svn:date");
96 + keys.svn_executable = pool_intern("svn:executable");
97 + keys.svn_special = pool_intern("svn:special");
98 + keys.uuid = pool_intern("UUID");
99 + keys.revision_number = pool_intern("Revision-number");
100 + keys.node_path = pool_intern("Node-path");
101 + keys.node_kind = pool_intern("Node-kind");
102 + keys.node_action = pool_intern("Node-action");
103 + keys.node_copyfrom_path = pool_intern("Node-copyfrom-path");
104 + keys.node_copyfrom_rev = pool_intern("Node-copyfrom-rev");
105 + keys.text_content_length = pool_intern("Text-content-length");
106 + keys.prop_content_length = pool_intern("Prop-content-length");
107 + keys.content_length = pool_intern("Content-length");
108 +}
109 +
110 +static void read_props(void)
111 +{
112 + uint32_t len;
113 + uint32_t key = ~0;
114 + char *val = NULL;
115 + char *t;
116 + while ((t = buffer_read_line()) && strcmp(t, "PROPS-END")) {
117 + if (!strncmp(t, "K ", 2)) {
118 + len = atoi(&t[2]);
119 + key = pool_intern(buffer_read_string(len));
120 + buffer_read_line();
121 + } else if (!strncmp(t, "V ", 2)) {
122 + len = atoi(&t[2]);
123 + val = buffer_read_string(len);
124 + if (key == keys.svn_log) {
125 + /* Value length excludes terminating nul. */
126 + rev_ctx.log = log_copy(len + 1, val);
127 + } else if (key == keys.svn_author) {
128 + rev_ctx.author = pool_intern(val);
129 + } else if (key == keys.svn_date) {
130 + if (parse_date_basic(val, &rev_ctx.timestamp, NULL))
131 + fprintf(stderr, "Invalid timestamp: %s\n", val);
132 + } else if (key == keys.svn_executable) {
133 + node_ctx.type = REPO_MODE_EXE;
134 + } else if (key == keys.svn_special) {
135 + node_ctx.type = REPO_MODE_LNK;
136 + }
137 + key = ~0;
138 + buffer_read_line();
139 + }
140 + }
141 +}
142 +
143 +static void handle_node(void)
144 +{
145 + if (node_ctx.propLength != LENGTH_UNKNOWN && node_ctx.propLength)
146 + read_props();
147 +
148 + if (node_ctx.srcRev)
149 + node_ctx.srcMode = repo_copy(node_ctx.srcRev, node_ctx.src, node_ctx.dst);
150 +
151 + if (node_ctx.textLength != LENGTH_UNKNOWN &&
152 + node_ctx.type != REPO_MODE_DIR)
153 + node_ctx.mark = next_blob_mark();
154 +
155 + if (node_ctx.action == NODEACT_DELETE) {
156 + repo_delete(node_ctx.dst);
157 + } else if (node_ctx.action == NODEACT_CHANGE ||
158 + node_ctx.action == NODEACT_REPLACE) {
159 + if (node_ctx.action == NODEACT_REPLACE &&
160 + node_ctx.type == REPO_MODE_DIR)
161 + repo_replace(node_ctx.dst, node_ctx.mark);
162 + else if (node_ctx.propLength != LENGTH_UNKNOWN)
163 + repo_modify(node_ctx.dst, node_ctx.type, node_ctx.mark);
164 + else if (node_ctx.textLength != LENGTH_UNKNOWN)
165 + node_ctx.srcMode = repo_replace(node_ctx.dst, node_ctx.mark);
166 + } else if (node_ctx.action == NODEACT_ADD) {
167 + if (node_ctx.srcRev && node_ctx.propLength != LENGTH_UNKNOWN)
168 + repo_modify(node_ctx.dst, node_ctx.type, node_ctx.mark);
169 + else if (node_ctx.srcRev && node_ctx.textLength != LENGTH_UNKNOWN)
170 + node_ctx.srcMode = repo_replace(node_ctx.dst, node_ctx.mark);
171 + else if ((node_ctx.type == REPO_MODE_DIR && !node_ctx.srcRev) ||
172 + node_ctx.textLength != LENGTH_UNKNOWN)
173 + repo_add(node_ctx.dst, node_ctx.type, node_ctx.mark);
174 + }
175 +
176 + if (node_ctx.propLength == LENGTH_UNKNOWN && node_ctx.srcMode)
177 + node_ctx.type = node_ctx.srcMode;
178 +
179 + if (node_ctx.mark)
180 + fast_export_blob(node_ctx.type, node_ctx.mark, node_ctx.textLength);
181 + else if (node_ctx.textLength != LENGTH_UNKNOWN)
182 + buffer_skip_bytes(node_ctx.textLength);
183 +}
184 +
185 +static void handle_revision(void)
186 +{
187 + if (rev_ctx.revision)
188 + repo_commit(rev_ctx.revision, rev_ctx.author, rev_ctx.log,
189 + dump_ctx.uuid, dump_ctx.url, rev_ctx.timestamp);
190 +}
191 +
192 +void svndump_read(const char *url)
193 +{
194 + char *val;
195 + char *t;
196 + uint32_t active_ctx = DUMP_CTX;
197 + uint32_t len;
198 + uint32_t key;
199 +
200 + reset_dump_ctx(pool_intern(url));
201 + while ((t = buffer_read_line())) {
202 + val = strstr(t, ": ");
203 + if (!val)
204 + continue;
205 + *val++ = '\0';
206 + *val++ = '\0';
207 + key = pool_intern(t);
208 +
209 + if (key == keys.uuid) {
210 + dump_ctx.uuid = pool_intern(val);
211 + } else if (key == keys.revision_number) {
212 + if (active_ctx == NODE_CTX)
213 + handle_node();
214 + if (active_ctx != DUMP_CTX)
215 + handle_revision();
216 + active_ctx = REV_CTX;
217 + reset_rev_ctx(atoi(val));
218 + } else if (key == keys.node_path) {
219 + if (active_ctx == NODE_CTX)
220 + handle_node();
221 + active_ctx = NODE_CTX;
222 + reset_node_ctx(val);
223 + } else if (key == keys.node_kind) {
224 + if (!strcmp(val, "dir"))
225 + node_ctx.type = REPO_MODE_DIR;
226 + else if (!strcmp(val, "file"))
227 + node_ctx.type = REPO_MODE_BLB;
228 + else
229 + fprintf(stderr, "Unknown node-kind: %s\n", val);
230 + } else if (key == keys.node_action) {
231 + if (!strcmp(val, "delete")) {
232 + node_ctx.action = NODEACT_DELETE;
233 + } else if (!strcmp(val, "add")) {
234 + node_ctx.action = NODEACT_ADD;
235 + } else if (!strcmp(val, "change")) {
236 + node_ctx.action = NODEACT_CHANGE;
237 + } else if (!strcmp(val, "replace")) {
238 + node_ctx.action = NODEACT_REPLACE;
239 + } else {
240 + fprintf(stderr, "Unknown node-action: %s\n", val);
241 + node_ctx.action = NODEACT_UNKNOWN;
242 + }
243 + } else if (key == keys.node_copyfrom_path) {
244 + pool_tok_seq(REPO_MAX_PATH_DEPTH, node_ctx.src, "/", val);
245 + } else if (key == keys.node_copyfrom_rev) {
246 + node_ctx.srcRev = atoi(val);
247 + } else if (key == keys.text_content_length) {
248 + node_ctx.textLength = atoi(val);
249 + } else if (key == keys.prop_content_length) {
250 + node_ctx.propLength = atoi(val);
251 + } else if (key == keys.content_length) {
252 + len = atoi(val);
253 + buffer_read_line();
254 + if (active_ctx == REV_CTX) {
255 + read_props();
256 + } else if (active_ctx == NODE_CTX) {
257 + handle_node();
258 + active_ctx = REV_CTX;
259 + } else {
260 + fprintf(stderr, "Unexpected content length header: %d\n", len);
261 + buffer_skip_bytes(len);
262 + }
263 + }
264 + }
265 + if (active_ctx == NODE_CTX)
266 + handle_node();
267 + if (active_ctx != DUMP_CTX)
268 + handle_revision();
269 +}
270 +
271 +void svndump_init(const char *filename)
272 +{
273 + buffer_init(filename);
274 + repo_init();
275 + reset_dump_ctx(~0);
276 + reset_rev_ctx(0);
277 + reset_node_ctx(NULL);
278 + init_keys();
279 +}
280 +
281 +void svndump_deinit(void)
282 +{
283 + log_reset();
284 + repo_reset();
285 + reset_dump_ctx(~0);
286 + reset_rev_ctx(0);
287 + reset_node_ctx(NULL);
288 + if (buffer_deinit())
289 + fprintf(stderr, "Input error\n");
290 + if (ferror(stdout))
291 + fprintf(stderr, "Output error\n");
292 +}
293 +
294 +void svndump_reset(void)
295 +{
296 + log_reset();
297 + buffer_reset();
298 + repo_reset();
299 + reset_dump_ctx(~0);
300 + reset_rev_ctx(0);
301 + reset_node_ctx(NULL);
302 +}
vcs-svn/svndump.h new
+9
@@ -0,0 +1,9 @@
1 +#ifndef SVNDUMP_H_
2 +#define SVNDUMP_H_
3 +
4 +void svndump_init(const char *filename);
5 +void svndump_read(const char *url);
6 +void svndump_deinit(void);
7 +void svndump_reset(void);
8 +
9 +#endif