Extracted TAR archive building/reading code out of 'ipfs get'
Matt Bell committed
Jan 24, 2015 at 00:26 UTC
fe48ae2dd6b63baaeb50d13fa9bc590301fe77ca
3 files changed
+343
-294
core/commands/get.go
+35
-294
@@ -1,25 +1,18 @@
1
package commands
2
3
import (
4
- "archive/tar"
5
- "bytes"
4
"compress/gzip"
5
"errors"
6
"fmt"
7
"io"
8
"os"
11
- p "path"
12
- fp "path/filepath"
9
"strings"
14
- "sync"
10
11
cmds "github.com/jbenet/go-ipfs/commands"
12
core "github.com/jbenet/go-ipfs/core"
18
- dag "github.com/jbenet/go-ipfs/merkledag"
19
- uio "github.com/jbenet/go-ipfs/unixfs/io"
20
- upb "github.com/jbenet/go-ipfs/unixfs/pb"
13
+ tar "github.com/jbenet/go-ipfs/thirdparty/tar"
14
+ utar "github.com/jbenet/go-ipfs/unixfs/tar"
15
22
- proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
16
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb"
17
)
18
@@ -64,22 +57,19 @@ may also specify the level of compression by specifying '-l=<1-9>'.
57
return
58
}
59
67
- compress, _, _ := req.Option("compress").Bool()
68
- compressionLevel, found, _ := req.Option("compression-level").Int()
69
- if !found {
70
- if compress {
71
- compressionLevel = gzip.DefaultCompression
72
- } else {
73
- compressionLevel = gzip.NoCompression
74
- }
75
- } else {
76
- if compressionLevel < 1 || compressionLevel > 9 {
77
- res.SetError(ErrInvalidCompressionLevel, cmds.ErrClient)
78
- return
79
- }
60
+ cmprs, _, _ := req.Option("compress").Bool()
61
+ cmplvl, cmplvlFound, _ := req.Option("compression-level").Int()
62
+ switch {
63
+ case !cmprs:
64
+ cmplvl = gzip.NoCompression
65
+ case cmprs && !cmplvlFound:
66
+ cmplvl = gzip.DefaultCompression
67
+ case cmprs && cmplvlFound && (cmplvl < 1 || cmplvl > 9):
68
+ res.SetError(ErrInvalidCompressionLevel, cmds.ErrClient)
69
+ return
70
}
71
82
- reader, err := get(node, req.Arguments()[0], compressionLevel)
72
+ reader, err := get(node, req.Arguments()[0], cmplvl)
73
if err != nil {
74
res.SetError(err, cmds.ErrNormal)
75
return
@@ -87,7 +77,7 @@ may also specify the level of compression by specifying '-l=<1-9>'.
77
res.SetOutput(reader)
78
},
79
PostRun: func(req cmds.Request, res cmds.Response) {
90
- reader := res.Output().(io.Reader)
80
+ outReader := res.Output().(io.Reader)
81
res.SetOutput(nil)
82
83
outPath, _, _ := req.Option("output").String()
@@ -95,15 +85,17 @@ may also specify the level of compression by specifying '-l=<1-9>'.
85
outPath = req.Arguments()[0]
86
}
87
98
- compress, _, _ := req.Option("compress").Bool()
99
- compressionLevel, found, _ := req.Option("compression-level").Int()
100
- compress = (compress && (compressionLevel > 0 || !found)) || compressionLevel > 0
88
+ cmprs, _, _ := req.Option("compress").Bool()
89
+ cmplvl, _, _ := req.Option("compression-level").Int()
90
+ if !cmprs {
91
+ cmprs = cmplvl > 0
92
+ }
93
94
if archive, _, _ := req.Option("archive").Bool(); archive {
95
if !strings.HasSuffix(outPath, ".tar") {
96
outPath += ".tar"
97
}
106
- if compress {
98
+ if cmprs {
99
outPath += ".gz"
100
}
101
fmt.Printf("Saving archive to %s\n", outPath)
@@ -117,7 +109,7 @@ may also specify the level of compression by specifying '-l=<1-9>'.
109
110
bar := pb.New(0).SetUnits(pb.U_BYTES)
111
bar.Output = os.Stderr
120
- pbReader := bar.NewProxyReader(reader)
112
+ pbReader := bar.NewProxyReader(outReader)
113
bar.Start()
114
defer bar.Finish()
115
@@ -136,291 +128,40 @@ may also specify the level of compression by specifying '-l=<1-9>'.
128
bar := pb.New(0).SetUnits(pb.U_BYTES)
129
bar.Output = os.Stderr
130
139
- preexisting := true
140
- pathIsDir := false
141
- if stat, err := os.Stat(outPath); err != nil && os.IsNotExist(err) {
142
- preexisting = false
143
- } else if err != nil {
144
- res.SetError(err, cmds.ErrNormal)
145
- return
146
- } else if stat.IsDir() {
147
- pathIsDir = true
148
- }
149
-
150
- var tarReader *tar.Reader
151
- if compress {
152
- gzipReader, err := gzip.NewReader(reader)
131
+ // wrap the reader with the progress bar proxy reader
132
+ // if the output is compressed, also wrap it in a gzip.Reader
133
+ var reader io.Reader
134
+ if cmprs {
135
+ gzipReader, err := gzip.NewReader(outReader)
136
if err != nil {
137
res.SetError(err, cmds.ErrNormal)
138
return
139
}
140
defer gzipReader.Close()
158
- pbReader := bar.NewProxyReader(gzipReader)
159
- tarReader = tar.NewReader(pbReader)
141
+ reader = bar.NewProxyReader(gzipReader)
142
} else {
161
- pbReader := bar.NewProxyReader(reader)
162
- tarReader = tar.NewReader(pbReader)
143
+ reader = bar.NewProxyReader(outReader)
144
}
145
146
bar.Start()
147
defer bar.Finish()
148
168
- for i := 0; ; i++ {
169
- header, err := tarReader.Next()
170
- if err != nil && err != io.EOF {
171
- res.SetError(err, cmds.ErrNormal)
172
- return
173
- }
174
- if header == nil || err == io.EOF {
175
- break
176
- }
177
-
178
- if header.Typeflag == tar.TypeDir {
179
- pathElements := strings.Split(header.Name, "/")
180
- if !preexisting {
181
- pathElements = pathElements[1:]
182
- }
183
- path := fp.Join(pathElements...)
184
- path = fp.Join(outPath, path)
185
- if i == 0 {
186
- outPath = path
187
- }
188
-
189
- err = os.MkdirAll(path, 0755)
190
- if err != nil {
191
- res.SetError(err, cmds.ErrNormal)
192
- return
193
- }
194
- continue
195
- }
196
-
197
- var path string
198
- if i == 0 {
199
- if preexisting {
200
- if !pathIsDir {
201
- res.SetError(os.ErrExist, cmds.ErrNormal)
202
- return
203
- }
204
- path = fp.Join(outPath, header.Name)
205
- } else {
206
- path = outPath
207
- }
208
- } else {
209
- pathElements := strings.Split(header.Name, "/")[1:]
210
- path = fp.Join(pathElements...)
211
- path = fp.Join(outPath, path)
212
- }
213
-
214
- file, err := os.Create(path)
215
- if err != nil {
216
- res.SetError(err, cmds.ErrNormal)
217
- return
218
- }
219
-
220
- _, err = io.Copy(file, tarReader)
221
- if err != nil {
222
- res.SetError(err, cmds.ErrNormal)
223
- return
224
- }
225
-
226
- err = file.Close()
227
- if err != nil {
228
- res.SetError(err, cmds.ErrNormal)
229
- return
230
- }
149
+ extractor := &tar.Extractor{outPath}
150
+ err := extractor.Extract(reader)
151
+ if err != nil {
152
+ res.SetError(err, cmds.ErrNormal)
153
}
154
},
155
}
156
157
func getCheckOptions(req cmds.Request) error {
236
- compressionLevel, found, _ := req.Option("compression-level").Int()
237
- if found && (compressionLevel < 1 || compressionLevel > 9) {
158
+ cmplvl, found, _ := req.Option("compression-level").Int()
159
+ if found && (cmplvl < 1 || cmplvl > 9) {
160
return ErrInvalidCompressionLevel
161
}
162
return nil
163
}
164
165
func get(node *core.IpfsNode, path string, compression int) (io.Reader, error) {
244
- buf := NewBufReadWriter()
245
-
246
- go func() {
247
- err := copyFilesAsTar(node, buf, path, compression)
248
- if err != nil {
249
- log.Error(err)
250
- return
251
- }
252
- }()
253
-
254
- return buf, nil
255
-}
256
-
257
-func copyFilesAsTar(node *core.IpfsNode, buf *bufReadWriter, path string, compression int) error {
258
- var gzipWriter *gzip.Writer
259
- var writer *tar.Writer
260
- var err error
261
- if compression != gzip.NoCompression {
262
- gzipWriter, err = gzip.NewWriterLevel(buf, compression)
263
- if err != nil {
264
- return err
265
- }
266
- writer = tar.NewWriter(gzipWriter)
267
- } else {
268
- writer = tar.NewWriter(buf)
269
- }
270
-
271
- err = _copyFilesAsTar(node, writer, buf, path, nil)
272
- if err != nil {
273
- return err
274
- }
275
-
276
- buf.mutex.Lock()
277
- err = writer.Close()
278
- if err != nil {
279
- return err
280
- }
281
- if gzipWriter != nil {
282
- err = gzipWriter.Close()
283
- if err != nil {
284
- return err
285
- }
286
- }
287
- buf.Close()
288
- buf.mutex.Unlock()
289
- buf.Signal()
290
- return nil
291
-}
292
-
293
-func _copyFilesAsTar(node *core.IpfsNode, writer *tar.Writer, buf *bufReadWriter, path string, dagnode *dag.Node) error {
294
- var err error
295
- if dagnode == nil {
296
- dagnode, err = node.Resolver.ResolvePath(path)
297
- if err != nil {
298
- return err
299
- }
300
- }
301
-
302
- pb := new(upb.Data)
303
- err = proto.Unmarshal(dagnode.Data, pb)
304
- if err != nil {
305
- return err
306
- }
307
-
308
- if pb.GetType() == upb.Data_Directory {
309
- buf.mutex.Lock()
310
- err = writer.WriteHeader(&tar.Header{
311
- Name: path,
312
- Typeflag: tar.TypeDir,
313
- Mode: 0777,
314
- // TODO: set mode, dates, etc. when added to unixFS
315
- })
316
- buf.mutex.Unlock()
317
- if err != nil {
318
- return err
319
- }
320
-
321
- for _, link := range dagnode.Links {
322
- err := _copyFilesAsTar(node, writer, buf, p.Join(path, link.Name), link.Node)
323
- if err != nil {
324
- return err
325
- }
326
- }
327
-
328
- return nil
329
- }
330
-
331
- buf.mutex.Lock()
332
- err = writer.WriteHeader(&tar.Header{
333
- Name: path,
334
- Size: int64(pb.GetFilesize()),
335
- Typeflag: tar.TypeReg,
336
- Mode: 0644,
337
- // TODO: set mode, dates, etc. when added to unixFS
338
- })
339
- buf.mutex.Unlock()
340
- if err != nil {
341
- return err
342
- }
343
-
344
- reader, err := uio.NewDagReader(dagnode, node.DAG)
345
- if err != nil {
346
- return err
347
- }
348
-
349
- _, err = syncCopy(writer, reader, buf)
350
- if err != nil {
351
- return err
352
- }
353
-
354
- return nil
355
-}
356
-
357
-type bufReadWriter struct {
358
- buf bytes.Buffer
359
- closed bool
360
- signalChan chan struct{}
361
- mutex *sync.Mutex
362
-}
363
-
364
-func NewBufReadWriter() *bufReadWriter {
365
- return &bufReadWriter{
366
- signalChan: make(chan struct{}),
367
- mutex: &sync.Mutex{},
368
- }
369
-}
370
-
371
-func (i *bufReadWriter) Read(p []byte) (int, error) {
372
- <-i.signalChan
373
- i.mutex.Lock()
374
- defer i.mutex.Unlock()
375
-
376
- if i.buf.Len() == 0 {
377
- if i.closed {
378
- return 0, io.EOF
379
- }
380
- return 0, nil
381
- }
382
-
383
- n, err := i.buf.Read(p)
384
- if err == io.EOF && !i.closed || i.buf.Len() > 0 {
385
- return n, nil
386
- }
387
- return n, err
388
-}
389
-
390
-func (i *bufReadWriter) Write(p []byte) (int, error) {
391
- return i.buf.Write(p)
392
-}
393
-
394
-func (i *bufReadWriter) Signal() {
395
- i.signalChan <- struct{}{}
396
-}
397
-
398
-func (i *bufReadWriter) Close() error {
399
- i.closed = true
400
- return nil
401
-}
402
-
403
-func syncCopy(writer io.Writer, reader io.Reader, buf *bufReadWriter) (int64, error) {
404
- written := int64(0)
405
- copyBuf := make([]byte, 32*1024)
406
- for {
407
- nr, err := reader.Read(copyBuf)
408
- if nr > 0 {
409
- buf.mutex.Lock()
410
- nw, err := writer.Write(copyBuf[:nr])
411
- buf.mutex.Unlock()
412
- if err != nil {
413
- return written, err
414
- }
415
- written += int64(nw)
416
- buf.Signal()
417
- }
418
- if err == io.EOF {
419
- break
420
- }
421
- if err != nil {
422
- return written, err
423
- }
424
- }
425
- return written, nil
166
+ return utar.NewReader(path, node.DAG, node.Resolver, compression)
167
}
thirdparty/tar/extractor.go
new
+108
@@ -0,0 +1,108 @@
1
+package tar
2
+
3
+import (
4
+ "archive/tar"
5
+ "io"
6
+ "os"
7
+ fp "path/filepath"
8
+ "strings"
9
+)
10
+
11
+type Extractor struct {
12
+ Path string
13
+}
14
+
15
+func (te *Extractor) Extract(reader io.Reader) error {
16
+ tarReader := tar.NewReader(reader)
17
+
18
+ // Check if the output path already exists, so we know whether we should
19
+ // create our output with that name, or if we should put the output inside
20
+ // a preexisting directory
21
+ exists := true
22
+ pathIsDir := false
23
+ if stat, err := os.Stat(te.Path); err != nil && os.IsNotExist(err) {
24
+ exists = false
25
+ } else if err != nil {
26
+ return err
27
+ } else if stat.IsDir() {
28
+ pathIsDir = true
29
+ }
30
+
31
+ // files come recursively in order (i == 0 is root directory)
32
+ for i := 0; ; i++ {
33
+ header, err := tarReader.Next()
34
+ if err != nil && err != io.EOF {
35
+ return err
36
+ }
37
+ if header == nil || err == io.EOF {
38
+ break
39
+ }
40
+
41
+ if header.Typeflag == tar.TypeDir {
42
+ err = te.extractDir(header, i, exists)
43
+ if err != nil {
44
+ return err
45
+ }
46
+ continue
47
+ }
48
+
49
+ err = te.extractFile(header, tarReader, i, exists, pathIsDir)
50
+ if err != nil {
51
+ return err
52
+ }
53
+ }
54
+ return nil
55
+}
56
+
57
+func (te *Extractor) extractDir(h *tar.Header, depth int, exists bool) error {
58
+ pathElements := strings.Split(h.Name, "/")
59
+ if !exists {
60
+ pathElements = pathElements[1:]
61
+ }
62
+ path := fp.Join(pathElements...)
63
+ path = fp.Join(te.Path, path)
64
+ if depth == 0 {
65
+ // if this is the root root directory, use it as the output path for remaining files
66
+ te.Path = path
67
+ }
68
+
69
+ err := os.MkdirAll(path, 0755)
70
+ if err != nil {
71
+ return err
72
+ }
73
+
74
+ return nil
75
+}
76
+
77
+func (te *Extractor) extractFile(h *tar.Header, r *tar.Reader, depth int, exists bool, pathIsDir bool) error {
78
+ var path string
79
+ if depth == 0 {
80
+ // if depth is 0, this is the only file (we aren't 'ipfs get'ing a directory)
81
+ switch {
82
+ case exists && !pathIsDir:
83
+ return os.ErrExist
84
+ case exists && pathIsDir:
85
+ path = fp.Join(te.Path, h.Name)
86
+ case !exists:
87
+ path = te.Path
88
+ }
89
+ } else {
90
+ // we are outputting a directory, this file is inside of it
91
+ pathElements := strings.Split(h.Name, "/")[1:]
92
+ path = fp.Join(pathElements...)
93
+ path = fp.Join(te.Path, path)
94
+ }
95
+
96
+ file, err := os.Create(path)
97
+ if err != nil {
98
+ return err
99
+ }
100
+ defer file.Close()
101
+
102
+ _, err = io.Copy(file, r)
103
+ if err != nil {
104
+ return err
105
+ }
106
+
107
+ return nil
108
+}
unixfs/tar/reader.go
new
+200
@@ -0,0 +1,200 @@
1
+package tar
2
+
3
+import (
4
+ "archive/tar"
5
+ "bytes"
6
+ "compress/gzip"
7
+ "io"
8
+ p "path"
9
+
10
+ mdag "github.com/jbenet/go-ipfs/merkledag"
11
+ path "github.com/jbenet/go-ipfs/path"
12
+ uio "github.com/jbenet/go-ipfs/unixfs/io"
13
+ upb "github.com/jbenet/go-ipfs/unixfs/pb"
14
+
15
+ proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
16
+)
17
+
18
+type Reader struct {
19
+ buf bytes.Buffer
20
+ closed bool
21
+ signalChan chan struct{}
22
+ dag mdag.DAGService
23
+ resolver *path.Resolver
24
+ writer *tar.Writer
25
+ gzipWriter *gzip.Writer
26
+ err error
27
+}
28
+
29
+func NewReader(path string, dag mdag.DAGService, resolver *path.Resolver, compression int) (*Reader, error) {
30
+ reader := &Reader{
31
+ signalChan: make(chan struct{}),
32
+ dag: dag,
33
+ resolver: resolver,
34
+ }
35
+
36
+ var err error
37
+ if compression != gzip.NoCompression {
38
+ reader.gzipWriter, err = gzip.NewWriterLevel(&reader.buf, compression)
39
+ if err != nil {
40
+ return nil, err
41
+ }
42
+ reader.writer = tar.NewWriter(reader.gzipWriter)
43
+ } else {
44
+ reader.writer = tar.NewWriter(&reader.buf)
45
+ }
46
+
47
+ dagnode, err := resolver.ResolvePath(path)
48
+ if err != nil {
49
+ return nil, err
50
+ }
51
+
52
+ // writeToBuf will write the data to the buffer, and will signal when there
53
+ // is new data to read
54
+ go reader.writeToBuf(dagnode, path, 0)
55
+
56
+ return reader, nil
57
+}
58
+
59
+func (i *Reader) writeToBuf(dagnode *mdag.Node, path string, depth int) {
60
+ pb := new(upb.Data)
61
+ err := proto.Unmarshal(dagnode.Data, pb)
62
+ if err != nil {
63
+ i.emitError(err)
64
+ return
65
+ }
66
+
67
+ if depth == 0 {
68
+ defer i.close()
69
+ }
70
+
71
+ if pb.GetType() == upb.Data_Directory {
72
+ err = i.writer.WriteHeader(&tar.Header{
73
+ Name: path,
74
+ Typeflag: tar.TypeDir,
75
+ Mode: 0777,
76
+ // TODO: set mode, dates, etc. when added to unixFS
77
+ })
78
+ if err != nil {
79
+ i.emitError(err)
80
+ return
81
+ }
82
+
83
+ for _, link := range dagnode.Links {
84
+ childNode, err := link.GetNode(i.dag)
85
+ if err != nil {
86
+ i.emitError(err)
87
+ return
88
+ }
89
+ i.writeToBuf(childNode, p.Join(path, link.Name), depth+1)
90
+ }
91
+ return
92
+ }
93
+
94
+ err = i.writer.WriteHeader(&tar.Header{
95
+ Name: path,
96
+ Size: int64(pb.GetFilesize()),
97
+ Typeflag: tar.TypeReg,
98
+ Mode: 0644,
99
+ // TODO: set mode, dates, etc. when added to unixFS
100
+ })
101
+ if err != nil {
102
+ i.emitError(err)
103
+ return
104
+ }
105
+
106
+ reader, err := uio.NewDagReader(dagnode, i.dag)
107
+ if err != nil {
108
+ i.emitError(err)
109
+ return
110
+ }
111
+
112
+ err = i.syncCopy(reader)
113
+ if err != nil {
114
+ i.emitError(err)
115
+ return
116
+ }
117
+}
118
+
119
+func (i *Reader) Read(p []byte) (int, error) {
120
+ // wait for the goroutine that is writing data to the buffer to tell us
121
+ // there is something to read
122
+ if !i.closed {
123
+ <-i.signalChan
124
+ }
125
+
126
+ if i.err != nil {
127
+ return 0, i.err
128
+ }
129
+
130
+ if !i.closed {
131
+ defer i.signal()
132
+ }
133
+
134
+ if i.buf.Len() == 0 {
135
+ if i.closed {
136
+ return 0, io.EOF
137
+ }
138
+ return 0, nil
139
+ }
140
+
141
+ n, err := i.buf.Read(p)
142
+ if err == io.EOF && !i.closed || i.buf.Len() > 0 {
143
+ return n, nil
144
+ }
145
+
146
+ return n, err
147
+}
148
+
149
+func (i *Reader) signal() {
150
+ i.signalChan <- struct{}{}
151
+}
152
+
153
+func (i *Reader) emitError(err error) {
154
+ i.err = err
155
+ i.signal()
156
+}
157
+
158
+func (i *Reader) close() {
159
+ i.closed = true
160
+ i.flush()
161
+}
162
+
163
+func (i *Reader) flush() {
164
+ defer i.signal()
165
+ err := i.writer.Close()
166
+ if err != nil {
167
+ i.emitError(err)
168
+ return
169
+ }
170
+ if i.gzipWriter != nil {
171
+ err = i.gzipWriter.Close()
172
+ if err != nil {
173
+ i.emitError(err)
174
+ return
175
+ }
176
+ }
177
+}
178
+
179
+func (i *Reader) syncCopy(reader io.Reader) error {
180
+ buf := make([]byte, 32*1024)
181
+ for {
182
+ nr, err := reader.Read(buf)
183
+ if nr > 0 {
184
+ _, err := i.writer.Write(buf[:nr])
185
+ if err != nil {
186
+ return err
187
+ }
188
+ i.signal()
189
+ // wait for Read to finish reading
190
+ <-i.signalChan
191
+ }
192
+ if err == io.EOF {
193
+ break
194
+ }
195
+ if err != nil {
196
+ return err
197
+ }
198
+ }
199
+ return nil
200
+}