vcs-svn: set up channel to read fast-import cat-blob response
Set up some plumbing: teach the svndump lib to pass a file descriptor number to the fast_export lib, representing where cat-blob/ls responses can be read from, and add a get_response_line helper function to the fast_export lib to read a line from that file. Unfortunately this means that svn-fe needs file descriptor 3 to be redirected from somewhere (preferrably the cat-blob stream of a fast-import backend); otherwise it will fail: $ svndump <path> | svn-fe fatal: cannot read from file descriptor 3: Bad file descriptor For the moment, "svn-fe 3</dev/null" works as a workaround but it will not work for very long. A fast-import backend that can retrieve old commits is needed in order to be able to fulfill svn "Node-copyfrom-rev" requests that refer to revs from a previous run. [jn: with new change description] Based-on-patch-by: Jonathan Nieder <jrnieder@gmail.com> Signed-off-by: David Barr <david.barr@cordelta.com> Signed-off-by: Jonathan Nieder <jrnieder@gmail.com>
David Barr committed
Mar 5, 2011 at 13:30 UTC
41529bbce407fbf1a925cfbc7c1aa30064f66ae7
5 files changed
+109
-52
contrib/svn-fe/svn-fe.txt
+5
-1
@@ -7,7 +7,11 @@ svn-fe - convert an SVN "dumpfile" to a fast-import stream
7
8
SYNOPSIS
9
--------
10
-svnadmin dump --incremental REPO | svn-fe [url] | git fast-import
10
+[verse]
11
+mkfifo backchannel &&
12
+svnadmin dump --incremental REPO |
13
+ svn-fe [url] 3<backchannel |
14
+ git fast-import --cat-blob-fd=3 3>backchannel
15
16
DESCRIPTION
17
-----------
t/t9010-svn-fe.sh
+67
-51
@@ -5,8 +5,26 @@ test_description='check svn dumpfile importer'
5
. ./test-lib.sh
6
7
reinit_git () {
8
+ if ! test_declared_prereq PIPE
9
+ then
10
+ echo >&4 "reinit_git: need to declare PIPE prerequisite"
11
+ return 127
12
+ fi
13
rm -fr .git &&
9
- git init
14
+ rm -f stream backflow &&
15
+ git init &&
16
+ mkfifo stream backflow
17
+}
18
+
19
+try_dump () {
20
+ input=$1 &&
21
+ maybe_fail=${2:+test_$2} &&
22
+
23
+ {
24
+ $maybe_fail test-svn-fe "$input" >stream 3<backflow &
25
+ } &&
26
+ git fast-import --cat-blob-fd=3 <stream 3>backflow &&
27
+ wait $!
28
}
29
30
properties () {
@@ -35,21 +53,27 @@ text_no_props () {
53
54
>empty
55
38
-test_expect_success 'empty dump' '
56
+test_expect_success 'setup: have pipes?' '
57
+ rm -f frob &&
58
+ if mkfifo frob
59
+ then
60
+ test_set_prereq PIPE
61
+ fi
62
+'
63
+
64
+test_expect_success PIPE 'empty dump' '
65
reinit_git &&
66
echo "SVN-fs-dump-format-version: 2" >input &&
41
- test-svn-fe input >stream &&
42
- git fast-import <stream
67
+ try_dump input
68
'
69
45
-test_expect_success 'v4 dumps not supported' '
70
+test_expect_success PIPE 'v4 dumps not supported' '
71
reinit_git &&
72
echo "SVN-fs-dump-format-version: 4" >v4.dump &&
48
- test_must_fail test-svn-fe v4.dump >stream &&
49
- test_cmp empty stream
73
+ try_dump v4.dump must_fail
74
'
75
52
-test_expect_failure 'empty revision' '
76
+test_expect_failure PIPE 'empty revision' '
77
reinit_git &&
78
printf "rev <nobody, nobody@local>: %s\n" "" "" >expect &&
79
cat >emptyrev.dump <<-\EOF &&
@@ -64,13 +88,12 @@ test_expect_failure 'empty revision' '
88
Content-length: 0
89
90
EOF
67
- test-svn-fe emptyrev.dump >stream &&
68
- git fast-import <stream &&
91
+ try_dump emptyrev.dump &&
92
git log -p --format="rev <%an, %ae>: %s" HEAD >actual &&
93
test_cmp expect actual
94
'
95
73
-test_expect_success 'empty properties' '
96
+test_expect_success PIPE 'empty properties' '
97
reinit_git &&
98
printf "rev <nobody, nobody@local>: %s\n" "" "" >expect &&
99
cat >emptyprop.dump <<-\EOF &&
@@ -88,13 +111,12 @@ test_expect_success 'empty properties' '
111
112
PROPS-END
113
EOF
91
- test-svn-fe emptyprop.dump >stream &&
92
- git fast-import <stream &&
114
+ try_dump emptyprop.dump &&
115
git log -p --format="rev <%an, %ae>: %s" HEAD >actual &&
116
test_cmp expect actual
117
'
118
97
-test_expect_success 'author name and commit message' '
119
+test_expect_success PIPE 'author name and commit message' '
120
reinit_git &&
121
echo "<author@example.com, author@example.com@local>" >expect.author &&
122
cat >message <<-\EOF &&
@@ -121,15 +143,14 @@ test_expect_success 'author name and commit message' '
143
echo &&
144
cat props
145
} >log.dump &&
124
- test-svn-fe log.dump >stream &&
125
- git fast-import <stream &&
146
+ try_dump log.dump &&
147
git log -p --format="%B" HEAD >actual.log &&
148
git log --format="<%an, %ae>" >actual.author &&
149
test_cmp message actual.log &&
150
test_cmp expect.author actual.author
151
'
152
132
-test_expect_success 'unsupported properties are ignored' '
153
+test_expect_success PIPE 'unsupported properties are ignored' '
154
reinit_git &&
155
echo author >expect &&
156
cat >extraprop.dump <<-\EOF &&
@@ -149,13 +170,12 @@ test_expect_success 'unsupported properties are ignored' '
170
author
171
PROPS-END
172
EOF
152
- test-svn-fe extraprop.dump >stream &&
153
- git fast-import <stream &&
173
+ try_dump extraprop.dump &&
174
git log -p --format=%an HEAD >actual &&
175
test_cmp expect actual
176
'
177
158
-test_expect_failure 'timestamp and empty file' '
178
+test_expect_failure PIPE 'timestamp and empty file' '
179
echo author@example.com >expect.author &&
180
echo 1999-01-01 >expect.date &&
181
echo file >expect.files &&
@@ -186,8 +206,7 @@ test_expect_failure 'timestamp and empty file' '
206
207
EOF
208
} >emptyfile.dump &&
189
- test-svn-fe emptyfile.dump >stream &&
190
- git fast-import <stream &&
209
+ try_dump emptyfile.dump &&
210
git log --format=%an HEAD >actual.author &&
211
git log --date=short --format=%ad HEAD >actual.date &&
212
git ls-tree -r --name-only HEAD >actual.files &&
@@ -198,7 +217,7 @@ test_expect_failure 'timestamp and empty file' '
217
test_cmp empty file
218
'
219
201
-test_expect_success 'directory with files' '
220
+test_expect_success PIPE 'directory with files' '
221
reinit_git &&
222
printf "%s\n" directory/file1 directory/file2 >expect.files &&
223
echo hi >hi &&
@@ -242,8 +261,7 @@ test_expect_success 'directory with files' '
261
EOF
262
text_no_props hi
263
} >directory.dump &&
245
- test-svn-fe directory.dump >stream &&
246
- git fast-import <stream &&
264
+ try_dump directory.dump &&
265
266
git ls-tree -r --name-only HEAD >actual.files &&
267
git checkout HEAD directory &&
@@ -252,7 +270,8 @@ test_expect_success 'directory with files' '
270
test_cmp hi directory/file2
271
'
272
255
-test_expect_success 'node without action' '
273
+test_expect_success PIPE 'node without action' '
274
+ reinit_git &&
275
cat >inaction.dump <<-\EOF &&
276
SVN-fs-dump-format-version: 3
277
@@ -269,10 +288,11 @@ test_expect_success 'node without action' '
288
289
PROPS-END
290
EOF
272
- test_must_fail test-svn-fe inaction.dump
291
+ try_dump inaction.dump must_fail
292
'
293
275
-test_expect_success 'action: add node without text' '
294
+test_expect_success PIPE 'action: add node without text' '
295
+ reinit_git &&
296
cat >textless.dump <<-\EOF &&
297
SVN-fs-dump-format-version: 3
298
@@ -290,10 +310,10 @@ test_expect_success 'action: add node without text' '
310
311
PROPS-END
312
EOF
293
- test_must_fail test-svn-fe textless.dump
313
+ try_dump textless.dump must_fail
314
'
315
296
-test_expect_failure 'change file mode but keep old content' '
316
+test_expect_failure PIPE 'change file mode but keep old content' '
317
reinit_git &&
318
cat >expect <<-\EOF &&
319
OBJID
@@ -356,8 +376,7 @@ test_expect_failure 'change file mode but keep old content' '
376
377
PROPS-END
378
EOF
359
- test-svn-fe filemode.dump >stream &&
360
- git fast-import <stream &&
379
+ try_dump filemode.dump &&
380
{
381
git rev-list HEAD |
382
git diff-tree --root --stdin |
@@ -370,7 +389,7 @@ test_expect_failure 'change file mode but keep old content' '
389
test_cmp hello actual.target
390
'
391
373
-test_expect_success 'change file mode and reiterate content' '
392
+test_expect_success PIPE 'change file mode and reiterate content' '
393
reinit_git &&
394
cat >expect <<-\EOF &&
395
OBJID
@@ -382,7 +401,7 @@ test_expect_success 'change file mode and reiterate content' '
401
EOF
402
echo "link hello" >expect.blob &&
403
echo hello >hello &&
385
- cat >filemode.dump <<-\EOF &&
404
+ cat >filemode2.dump <<-\EOF &&
405
SVN-fs-dump-format-version: 3
406
407
Revision-number: 1
@@ -437,8 +456,7 @@ test_expect_success 'change file mode and reiterate content' '
456
PROPS-END
457
link hello
458
EOF
440
- test-svn-fe filemode.dump >stream &&
441
- git fast-import <stream &&
459
+ try_dump filemode2.dump &&
460
{
461
git rev-list HEAD |
462
git diff-tree --root --stdin |
@@ -451,7 +469,8 @@ test_expect_success 'change file mode and reiterate content' '
469
test_cmp hello actual.target
470
'
471
454
-test_expect_success 'deltas not supported' '
472
+test_expect_success PIPE 'deltas not supported' '
473
+ reinit_git &&
474
{
475
# (old) h + (inline) ello + (old) \n
476
printf "SVNQ%b%b%s" "Q\003\006\005\004" "\001Q\0204\001\002" "ello" |
@@ -511,10 +530,10 @@ test_expect_success 'deltas not supported' '
530
echo PROPS-END &&
531
cat delta
532
} >delta.dump &&
514
- test_must_fail test-svn-fe delta.dump
533
+ test_must_fail try_dump delta.dump
534
'
535
517
-test_expect_success 'property deltas supported' '
536
+test_expect_success PIPE 'property deltas supported' '
537
reinit_git &&
538
cat >expect <<-\EOF &&
539
OBJID
@@ -570,8 +589,7 @@ test_expect_success 'property deltas supported' '
589
PROPS-END
590
EOF
591
} >propdelta.dump &&
573
- test-svn-fe propdelta.dump >stream &&
574
- git fast-import <stream &&
592
+ try_dump propdelta.dump &&
593
{
594
git rev-list HEAD |
595
git diff-tree --stdin |
@@ -580,7 +598,7 @@ test_expect_success 'property deltas supported' '
598
test_cmp expect actual
599
'
600
583
-test_expect_success 'properties on /' '
601
+test_expect_success PIPE 'properties on /' '
602
reinit_git &&
603
cat <<-\EOF >expect &&
604
OBJID
@@ -625,8 +643,7 @@ test_expect_success 'properties on /' '
643
644
PROPS-END
645
EOF
628
- test-svn-fe changeroot.dump >stream &&
629
- git fast-import <stream &&
646
+ try_dump changeroot.dump &&
647
{
648
git rev-list HEAD |
649
git diff-tree --root --always --stdin |
@@ -635,7 +652,7 @@ test_expect_success 'properties on /' '
652
test_cmp expect actual
653
'
654
638
-test_expect_success 'deltas for typechange' '
655
+test_expect_success PIPE 'deltas for typechange' '
656
reinit_git &&
657
cat >expect <<-\EOF &&
658
OBJID
@@ -711,8 +728,7 @@ test_expect_success 'deltas for typechange' '
728
PROPS-END
729
link testing 321
730
EOF
714
- test-svn-fe deleteprop.dump >stream &&
715
- git fast-import <stream &&
731
+ try_dump deleteprop.dump &&
732
{
733
git rev-list HEAD |
734
git diff-tree --root --stdin |
@@ -736,12 +752,12 @@ test_expect_success 'set up svn repo' '
752
fi
753
'
754
739
-test_expect_success SVNREPO 't9135/svn.dump' '
740
- git init simple-git &&
741
- test-svn-fe "$TEST_DIRECTORY/t9135/svn.dump" >simple.fe &&
755
+test_expect_success SVNREPO,PIPE 't9135/svn.dump' '
756
+ mkdir -p simple-git &&
757
(
758
cd simple-git &&
744
- git fast-import <../simple.fe
759
+ reinit_git &&
760
+ try_dump "$TEST_DIRECTORY/t9135/svn.dump"
761
) &&
762
(
763
cd simple-svnco &&
vcs-svn/fast_export.c
+28
@@ -12,6 +12,24 @@
12
#define MAX_GITSVN_LINE_LEN 4096
13
14
static uint32_t first_commit_done;
15
+static struct line_buffer report_buffer = LINE_BUFFER_INIT;
16
+
17
+void fast_export_init(int fd)
18
+{
19
+ if (buffer_fdinit(&report_buffer, fd))
20
+ die_errno("cannot read from file descriptor %d", fd);
21
+}
22
+
23
+void fast_export_deinit(void)
24
+{
25
+ if (buffer_deinit(&report_buffer))
26
+ die_errno("error closing fast-import feedback stream");
27
+}
28
+
29
+void fast_export_reset(void)
30
+{
31
+ buffer_reset(&report_buffer);
32
+}
33
34
void fast_export_delete(uint32_t depth, uint32_t *path)
35
{
@@ -63,6 +81,16 @@ void fast_export_commit(uint32_t revision, uint32_t author, char *log,
81
printf("progress Imported commit %"PRIu32".\n\n", revision);
82
}
83
84
+static const char *get_response_line(void)
85
+{
86
+ const char *line = buffer_read_line(&report_buffer);
87
+ if (line)
88
+ return line;
89
+ if (buffer_ferror(&report_buffer))
90
+ die_errno("error reading from fast-import");
91
+ die("unexpected end of fast-import feedback");
92
+}
93
+
94
void fast_export_blob(uint32_t mode, uint32_t mark, uint32_t len, struct line_buffer *input)
95
{
96
if (mode == REPO_MODE_LNK) {
vcs-svn/fast_export.h
+4
@@ -3,6 +3,10 @@
3
4
#include "line_buffer.h"
5
6
+void fast_export_init(int fd);
7
+void fast_export_deinit(void);
8
+void fast_export_reset(void);
9
+
10
void fast_export_delete(uint32_t depth, uint32_t *path);
11
void fast_export_modify(uint32_t depth, uint32_t *path, uint32_t mode,
12
uint32_t mark);
vcs-svn/svndump.c
+5
@@ -14,6 +14,8 @@
14
#include "obj_pool.h"
15
#include "string_pool.h"
16
17
+#define REPORT_FILENO 3
18
+
19
#define NODEACT_REPLACE 4
20
#define NODEACT_DELETE 3
21
#define NODEACT_ADD 2
@@ -367,6 +369,7 @@ int svndump_init(const char *filename)
369
if (buffer_init(&input, filename))
370
return error("cannot open %s: %s", filename, strerror(errno));
371
repo_init();
372
+ fast_export_init(REPORT_FILENO);
373
reset_dump_ctx(~0);
374
reset_rev_ctx(0);
375
reset_node_ctx(NULL);
@@ -377,6 +380,7 @@ int svndump_init(const char *filename)
380
void svndump_deinit(void)
381
{
382
log_reset();
383
+ fast_export_deinit();
384
repo_reset();
385
reset_dump_ctx(~0);
386
reset_rev_ctx(0);
@@ -390,6 +394,7 @@ void svndump_deinit(void)
394
void svndump_reset(void)
395
{
396
log_reset();
397
+ fast_export_reset();
398
buffer_reset(&input);
399
repo_reset();
400
reset_dump_ctx(~0);