first pass at a tar importer
License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com>
Jeromy committed
Sep 9, 2015 at 20:20 UTC
5464020c5884030fbab3e423bb6dc9ed226098d2
3 files changed
+299
core/commands/root.go
+1
@@ -108,6 +108,7 @@ var rootSubcommands = map[string]*cmds.Command{
108
"resolve": ResolveCmd,
109
"stats": StatsCmd,
110
"swarm": SwarmCmd,
111
+ "tar": TarCmd,
112
"tour": tourCmd,
113
"file": unixfs.UnixFSCmd,
114
"update": UpdateCmd,
core/commands/tar.go
new
+113
@@ -0,0 +1,113 @@
1
+package commands
2
+
3
+import (
4
+ "io"
5
+ "strings"
6
+
7
+ cmds "github.com/ipfs/go-ipfs/commands"
8
+ core "github.com/ipfs/go-ipfs/core"
9
+ path "github.com/ipfs/go-ipfs/path"
10
+ tar "github.com/ipfs/go-ipfs/tar"
11
+)
12
+
13
+var TarCmd = &cmds.Command{
14
+ Helptext: cmds.HelpText{
15
+ Tagline: "utility functions for tar files in ipfs",
16
+ },
17
+
18
+ Subcommands: map[string]*cmds.Command{
19
+ "add": tarAddCmd,
20
+ "cat": tarCatCmd,
21
+ },
22
+}
23
+
24
+var tarAddCmd = &cmds.Command{
25
+ Helptext: cmds.HelpText{
26
+ Tagline: "import a tar file into ipfs",
27
+ ShortDescription: `
28
+'ipfs tar add' will parse a tar file and create a merkledag structure to represent it.
29
+`,
30
+ },
31
+
32
+ Arguments: []cmds.Argument{
33
+ cmds.FileArg("file", true, false, "tar file to add").EnableStdin(),
34
+ },
35
+ Run: func(req cmds.Request, res cmds.Response) {
36
+ nd, err := req.InvocContext().GetNode()
37
+ if err != nil {
38
+ res.SetError(err, cmds.ErrNormal)
39
+ return
40
+ }
41
+
42
+ fi, err := req.Files().NextFile()
43
+ if err != nil {
44
+ res.SetError(err, cmds.ErrNormal)
45
+ return
46
+ }
47
+
48
+ node, err := tar.ImportTar(fi, nd.DAG)
49
+ if err != nil {
50
+ res.SetError(err, cmds.ErrNormal)
51
+ return
52
+ }
53
+
54
+ k, err := node.Key()
55
+ if err != nil {
56
+ res.SetError(err, cmds.ErrNormal)
57
+ return
58
+ }
59
+
60
+ fi.FileName()
61
+ res.SetOutput(&AddedObject{
62
+ Name: fi.FileName(),
63
+ Hash: k.B58String(),
64
+ })
65
+ },
66
+ Type: AddedObject{},
67
+ Marshalers: cmds.MarshalerMap{
68
+ cmds.Text: func(res cmds.Response) (io.Reader, error) {
69
+ o := res.Output().(*AddedObject)
70
+ return strings.NewReader(o.Hash), nil
71
+ },
72
+ },
73
+}
74
+
75
+var tarCatCmd = &cmds.Command{
76
+ Helptext: cmds.HelpText{
77
+ Tagline: "export a tar file from ipfs",
78
+ ShortDescription: `
79
+'ipfs tar cat' will export a tar file from a previously imported one in ipfs
80
+`,
81
+ },
82
+
83
+ Arguments: []cmds.Argument{
84
+ cmds.StringArg("path", true, false, "ipfs path of archive to export").EnableStdin(),
85
+ },
86
+ Run: func(req cmds.Request, res cmds.Response) {
87
+ nd, err := req.InvocContext().GetNode()
88
+ if err != nil {
89
+ res.SetError(err, cmds.ErrNormal)
90
+ return
91
+ }
92
+
93
+ p, err := path.ParsePath(req.Arguments()[0])
94
+ if err != nil {
95
+ res.SetError(err, cmds.ErrNormal)
96
+ return
97
+ }
98
+
99
+ root, err := core.Resolve(req.Context(), nd, p)
100
+ if err != nil {
101
+ res.SetError(err, cmds.ErrNormal)
102
+ return
103
+ }
104
+
105
+ r, err := tar.ExportTar(req.Context(), root, nd.DAG)
106
+ if err != nil {
107
+ res.SetError(err, cmds.ErrNormal)
108
+ return
109
+ }
110
+
111
+ res.SetOutput(r)
112
+ },
113
+}
tar/format.go
new
+185
@@ -0,0 +1,185 @@
1
+package tarfmt
2
+
3
+import (
4
+ "archive/tar"
5
+ "bytes"
6
+ "errors"
7
+ "io"
8
+ "io/ioutil"
9
+
10
+ importer "github.com/ipfs/go-ipfs/importer"
11
+ chunk "github.com/ipfs/go-ipfs/importer/chunk"
12
+ dag "github.com/ipfs/go-ipfs/merkledag"
13
+ uio "github.com/ipfs/go-ipfs/unixfs/io"
14
+
15
+ context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
16
+)
17
+
18
+var blockSize = 512
19
+var zeroBlock = make([]byte, blockSize)
20
+
21
+func marshalHeader(h *tar.Header) ([]byte, error) {
22
+ buf := new(bytes.Buffer)
23
+ w := tar.NewWriter(buf)
24
+ err := w.WriteHeader(h)
25
+ if err != nil {
26
+ return nil, err
27
+ }
28
+ return buf.Bytes(), nil
29
+}
30
+
31
+func ImportTar(r io.Reader, ds dag.DAGService) (*dag.Node, error) {
32
+ rall, err := ioutil.ReadAll(r)
33
+ if err != nil {
34
+ return nil, err
35
+ }
36
+
37
+ r = bytes.NewReader(rall)
38
+
39
+ tr := tar.NewReader(r)
40
+
41
+ root := new(dag.Node)
42
+ root.Data = []byte("ipfs/tar")
43
+
44
+ for {
45
+ h, err := tr.Next()
46
+ if err != nil {
47
+ if err == io.EOF {
48
+ break
49
+ }
50
+ return nil, err
51
+ }
52
+
53
+ header := new(dag.Node)
54
+
55
+ headerBytes, err := marshalHeader(h)
56
+ if err != nil {
57
+ return nil, err
58
+ }
59
+
60
+ header.Data = headerBytes
61
+
62
+ if h.Size > 0 {
63
+ spl := chunk.NewRabin(tr, uint64(chunk.DefaultBlockSize))
64
+ nd, err := importer.BuildDagFromReader(ds, spl, nil)
65
+ if err != nil {
66
+ return nil, err
67
+ }
68
+
69
+ err = header.AddNodeLinkClean("data", nd)
70
+ if err != nil {
71
+ return nil, err
72
+ }
73
+ }
74
+
75
+ _, err = ds.Add(header)
76
+ if err != nil {
77
+ return nil, err
78
+ }
79
+
80
+ err = root.AddNodeLinkClean(h.Name, header)
81
+ if err != nil {
82
+ return nil, err
83
+ }
84
+ }
85
+
86
+ _, err = ds.Add(root)
87
+ if err != nil {
88
+ return nil, err
89
+ }
90
+
91
+ return root, nil
92
+}
93
+
94
+type tarReader struct {
95
+ links []*dag.Link
96
+ ds dag.DAGService
97
+
98
+ hdrBuf *bytes.Reader
99
+ fileRead *countReader
100
+ pad int
101
+
102
+ ctx context.Context
103
+}
104
+
105
+func (tr *tarReader) Read(b []byte) (int, error) {
106
+ if tr.hdrBuf != nil {
107
+ n, err := tr.hdrBuf.Read(b)
108
+ if err == io.EOF {
109
+ tr.hdrBuf = nil
110
+ return n, nil
111
+ }
112
+ return n, err
113
+ }
114
+ if tr.fileRead != nil {
115
+ n, err := tr.fileRead.Read(b)
116
+ if err == io.EOF {
117
+ nr := tr.fileRead.n
118
+ tr.pad = (blockSize - (nr % blockSize)) % blockSize
119
+ tr.fileRead.Close()
120
+ tr.fileRead = nil
121
+ return n, nil
122
+ }
123
+ return n, err
124
+ }
125
+ if tr.pad > 0 {
126
+ n := copy(b, zeroBlock[:tr.pad])
127
+ tr.pad -= n
128
+ return n, nil
129
+ }
130
+
131
+ if len(tr.links) == 0 {
132
+ return 0, io.EOF
133
+ }
134
+
135
+ next := tr.links[0]
136
+ tr.links = tr.links[1:]
137
+
138
+ headerNd, err := next.GetNode(tr.ctx, tr.ds)
139
+ if err != nil {
140
+ return 0, err
141
+ }
142
+
143
+ tr.hdrBuf = bytes.NewReader(headerNd.Data)
144
+ if len(headerNd.Links) > 0 {
145
+ data, err := headerNd.Links[0].GetNode(tr.ctx, tr.ds)
146
+ if err != nil {
147
+ return 0, err
148
+ }
149
+
150
+ dr, err := uio.NewDagReader(tr.ctx, data, tr.ds)
151
+ if err != nil {
152
+ return 0, err
153
+ }
154
+
155
+ tr.fileRead = &countReader{r: dr}
156
+ }
157
+
158
+ return tr.Read(b)
159
+}
160
+
161
+func ExportTar(ctx context.Context, root *dag.Node, ds dag.DAGService) (io.Reader, error) {
162
+ if string(root.Data) != "ipfs/tar" {
163
+ return nil, errors.New("not an ipfs tarchive")
164
+ }
165
+ return &tarReader{
166
+ links: root.Links,
167
+ ds: ds,
168
+ ctx: ctx,
169
+ }, nil
170
+}
171
+
172
+type countReader struct {
173
+ r io.ReadCloser
174
+ n int
175
+}
176
+
177
+func (r *countReader) Read(b []byte) (int, error) {
178
+ n, err := r.r.Read(b)
179
+ r.n += n
180
+ return n, err
181
+}
182
+
183
+func (r *countReader) Close() error {
184
+ return r.r.Close()
185
+}