Decompose DagArchive from unixfs tar
License: MIT Signed-off-by: rht <rhtbot@gmail.com>
rht committed
Aug 20, 2015 at 14:53 UTC
9f0c8134cb4c6a1a769437c685b39b233759d172
3 files changed
+85
-71
core/commands/get.go
+2
-2
@@ -15,7 +15,7 @@ import (
15
core "github.com/ipfs/go-ipfs/core"
16
path "github.com/ipfs/go-ipfs/path"
17
tar "github.com/ipfs/go-ipfs/thirdparty/tar"
18
- utar "github.com/ipfs/go-ipfs/unixfs/tar"
18
+ uarchive "github.com/ipfs/go-ipfs/unixfs/archive"
19
)
20
21
var ErrInvalidCompressionLevel = errors.New("Compression level must be between 1 and 9")
@@ -70,7 +70,7 @@ may also specify the level of compression by specifying '-l=<1-9>'.
70
}
71
72
archive, _, _ := req.Option("archive").Bool()
73
- reader, err := utar.DagArchive(ctx, dn, p.String(), node.DAG, archive, cmplvl)
73
+ reader, err := uarchive.DagArchive(ctx, dn, p.String(), node.DAG, archive, cmplvl)
74
if err != nil {
75
res.SetError(err, cmds.ErrNormal)
76
return
unixfs/archive/archive.go
new
+83
@@ -0,0 +1,83 @@
1
+package archive
2
+
3
+import (
4
+ "bufio"
5
+ "compress/gzip"
6
+ "io"
7
+ "path"
8
+
9
+ cxt "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
10
+
11
+ mdag "github.com/ipfs/go-ipfs/merkledag"
12
+ tar "github.com/ipfs/go-ipfs/unixfs/archive/tar"
13
+ uio "github.com/ipfs/go-ipfs/unixfs/io"
14
+)
15
+
16
+// DefaultBufSize is the buffer size for gets. for now, 1MB, which is ~4 blocks.
17
+// TODO: does this need to be configurable?
18
+var DefaultBufSize = 1048576
19
+
20
+// DagArchive is equivalent to `ipfs getdag $hash | maybe_tar | maybe_gzip`
21
+func DagArchive(ctx cxt.Context, nd *mdag.Node, name string, dag mdag.DAGService, archive bool, compression int) (io.Reader, error) {
22
+
23
+ _, filename := path.Split(name)
24
+
25
+ // need to connect a writer to a reader
26
+ piper, pipew := io.Pipe()
27
+
28
+ // use a buffered writer to parallelize task
29
+ bufw := bufio.NewWriterSize(pipew, DefaultBufSize)
30
+
31
+ // compression determines whether to use gzip compression.
32
+ var maybeGzw io.Writer
33
+ if compression != gzip.NoCompression {
34
+ var err error
35
+ maybeGzw, err = gzip.NewWriterLevel(bufw, compression)
36
+ if err != nil {
37
+ return nil, err
38
+ }
39
+ } else {
40
+ maybeGzw = bufw
41
+ }
42
+
43
+ if !archive && compression != gzip.NoCompression {
44
+ // the case when the node is a file
45
+ dagr, err := uio.NewDagReader(ctx, nd, dag)
46
+ if err != nil {
47
+ pipew.CloseWithError(err)
48
+ return nil, err
49
+ }
50
+
51
+ go func() {
52
+ if _, err := dagr.WriteTo(maybeGzw); err != nil {
53
+ pipew.CloseWithError(err)
54
+ return
55
+ }
56
+ pipew.Close() // everything seems to be ok.
57
+ }()
58
+ } else {
59
+ // the case for 1. archive, and 2. not archived and not compressed, in which tar is used anyway as a transport format
60
+
61
+ // construct the tar writer
62
+ w, err := tar.NewWriter(ctx, dag, archive, compression, maybeGzw)
63
+ if err != nil {
64
+ return nil, err
65
+ }
66
+
67
+ go func() {
68
+ // write all the nodes recursively
69
+ if err := w.WriteNode(nd, filename); err != nil {
70
+ pipew.CloseWithError(err)
71
+ return
72
+ }
73
+ if err := bufw.Flush(); err != nil {
74
+ pipew.CloseWithError(err)
75
+ return
76
+ }
77
+ w.Close()
78
+ pipew.Close() // everything seems to be ok.
79
+ }()
80
+ }
81
+
82
+ return piper, nil
83
+}
unixfs/archive/tar/writer.go
renamed
-69
@@ -2,8 +2,6 @@ package tar
2
3
import (
4
"archive/tar"
5
- "bufio"
6
- "compress/gzip"
5
"io"
6
"path"
7
"time"
@@ -17,73 +15,6 @@ import (
15
upb "github.com/ipfs/go-ipfs/unixfs/pb"
16
)
17
20
-// DefaultBufSize is the buffer size for gets. for now, 1MB, which is ~4 blocks.
21
-// TODO: does this need to be configurable?
22
-var DefaultBufSize = 1048576
23
-
24
-// DagArchive is equivalent to `ipfs getdag $hash | maybe_tar | maybe_gzip`
25
-func DagArchive(ctx cxt.Context, nd *mdag.Node, name string, dag mdag.DAGService, archive bool, compression int) (io.Reader, error) {
26
-
27
- _, filename := path.Split(name)
28
-
29
- // need to connect a writer to a reader
30
- piper, pipew := io.Pipe()
31
-
32
- // use a buffered writer to parallelize task
33
- bufw := bufio.NewWriterSize(pipew, DefaultBufSize)
34
-
35
- // compression determines whether to use gzip compression.
36
- var maybeGzw io.Writer
37
- if compression != gzip.NoCompression {
38
- var err error
39
- maybeGzw, err = gzip.NewWriterLevel(bufw, compression)
40
- if err != nil {
41
- return nil, err
42
- }
43
- } else {
44
- maybeGzw = bufw
45
- }
46
-
47
- // construct the tar writer
48
- w, err := NewWriter(ctx, dag, archive, compression, maybeGzw)
49
- if err != nil {
50
- return nil, err
51
- }
52
-
53
- // write all the nodes recursively
54
- go func() {
55
- if !archive && compression != gzip.NoCompression {
56
- // the case when the node is a file
57
- dagr, err := uio.NewDagReader(w.ctx, nd, w.Dag)
58
- if err != nil {
59
- pipew.CloseWithError(err)
60
- return
61
- }
62
-
63
- if _, err := dagr.WriteTo(maybeGzw); err != nil {
64
- pipew.CloseWithError(err)
65
- return
66
- }
67
- } else {
68
- // the case for 1. archive, and 2. not archived and not compressed, in which tar is used anyway as a transport format
69
- if err := w.WriteNode(nd, filename); err != nil {
70
- pipew.CloseWithError(err)
71
- return
72
- }
73
- }
74
-
75
- if err := bufw.Flush(); err != nil {
76
- pipew.CloseWithError(err)
77
- return
78
- }
79
-
80
- w.Close()
81
- pipew.Close() // everything seems to be ok.
82
- }()
83
-
84
- return piper, nil
85
-}
86
-
18
// Writer is a utility structure that helps to write
19
// unixfs merkledag nodes as a tar archive format.
20
// It wraps any io.Writer.