sharding directory entries down to not create a massive root node
License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com>
Jeromy committed
Sep 10, 2015 at 12:34 UTC
678e74bdae51faa7d07bf5b1ba2abd783367a27c
1 file changed
+53
-10
tar/format.go
+53
-10
@@ -6,15 +6,20 @@ import (
6
"errors"
7
"io"
8
"io/ioutil"
9
+ "strings"
10
11
importer "github.com/ipfs/go-ipfs/importer"
12
chunk "github.com/ipfs/go-ipfs/importer/chunk"
13
dag "github.com/ipfs/go-ipfs/merkledag"
14
+ dagutil "github.com/ipfs/go-ipfs/merkledag/utils"
15
uio "github.com/ipfs/go-ipfs/unixfs/io"
16
+ u "github.com/ipfs/go-ipfs/util"
17
18
context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
19
)
20
21
+var log = u.Logger("tarfmt")
22
+
23
var blockSize = 512
24
var zeroBlock = make([]byte, blockSize)
25
@@ -41,6 +46,8 @@ func ImportTar(r io.Reader, ds dag.DAGService) (*dag.Node, error) {
46
root := new(dag.Node)
47
root.Data = []byte("ipfs/tar")
48
49
+ e := dagutil.NewDagEditor(ds, root)
50
+
51
for {
52
h, err := tr.Next()
53
if err != nil {
@@ -77,12 +84,14 @@ func ImportTar(r io.Reader, ds dag.DAGService) (*dag.Node, error) {
84
return nil, err
85
}
86
80
- err = root.AddNodeLinkClean(h.Name, header)
87
+ path := escapePath(h.Name)
88
+ err = e.InsertNodeAtPath(context.Background(), path, header, func() *dag.Node { return new(dag.Node) })
89
if err != nil {
90
return nil, err
91
}
92
}
93
94
+ root = e.GetNode()
95
_, err = ds.Add(root)
96
if err != nil {
97
return nil, err
@@ -91,18 +100,30 @@ func ImportTar(r io.Reader, ds dag.DAGService) (*dag.Node, error) {
100
return root, nil
101
}
102
103
+// adds a '-' to the beginning of each path element so we can use 'data' as a
104
+// special link in the structure without having to worry about
105
+func escapePath(path string) string {
106
+ elems := strings.Split(strings.Trim(path, "/"), "/")
107
+ for i, e := range elems {
108
+ elems[i] = "-" + e
109
+ }
110
+ return strings.Join(elems, "/")
111
+}
112
+
113
type tarReader struct {
114
links []*dag.Link
115
ds dag.DAGService
116
98
- hdrBuf *bytes.Reader
99
- fileRead *countReader
100
- pad int
117
+ childRead *tarReader
118
+ hdrBuf *bytes.Reader
119
+ fileRead *countReader
120
+ pad int
121
122
ctx context.Context
123
}
124
125
func (tr *tarReader) Read(b []byte) (int, error) {
126
+ // if we have a header to be read, it takes priority
127
if tr.hdrBuf != nil {
128
n, err := tr.hdrBuf.Read(b)
129
if err == io.EOF {
@@ -111,6 +132,18 @@ func (tr *tarReader) Read(b []byte) (int, error) {
132
}
133
return n, err
134
}
135
+
136
+ // no header remaining, check for recursive
137
+ if tr.childRead != nil {
138
+ n, err := tr.childRead.Read(b)
139
+ if err == io.EOF {
140
+ tr.childRead = nil
141
+ return n, nil
142
+ }
143
+ return n, err
144
+ }
145
+
146
+ // check for filedata to be read
147
if tr.fileRead != nil {
148
n, err := tr.fileRead.Read(b)
149
if err == io.EOF {
@@ -122,6 +155,8 @@ func (tr *tarReader) Read(b []byte) (int, error) {
155
}
156
return n, err
157
}
158
+
159
+ // filedata reads must be padded out to 512 byte offsets
160
if tr.pad > 0 {
161
n := copy(b, zeroBlock[:tr.pad])
162
tr.pad -= n
@@ -141,18 +176,26 @@ func (tr *tarReader) Read(b []byte) (int, error) {
176
}
177
178
tr.hdrBuf = bytes.NewReader(headerNd.Data)
144
- if len(headerNd.Links) > 0 {
145
- data, err := headerNd.Links[0].GetNode(tr.ctx, tr.ds)
146
- if err != nil {
147
- return 0, err
148
- }
179
150
- dr, err := uio.NewDagReader(tr.ctx, data, tr.ds)
180
+ dataNd, err := headerNd.GetLinkedNode(tr.ctx, tr.ds, "data")
181
+ if err != nil && err != dag.ErrNotFound {
182
+ return 0, err
183
+ }
184
+
185
+ if err == nil {
186
+ dr, err := uio.NewDagReader(tr.ctx, dataNd, tr.ds)
187
if err != nil {
188
+ log.Error("dagreader error: ", err)
189
return 0, err
190
}
191
192
tr.fileRead = &countReader{r: dr}
193
+ } else if len(headerNd.Links) > 0 {
194
+ tr.childRead = &tarReader{
195
+ links: headerNd.Links,
196
+ ds: tr.ds,
197
+ ctx: tr.ctx,
198
+ }
199
}
200
201
return tr.Read(b)