Fix sharding memory growth, and fix resolver for unixfs paths
License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com>
Jeromy committed
Apr 30, 2017 at 13:37 UTC
0597a04924c8e456ea47d7236b8296ed00c4de45
5 files changed
+62
-23
core/coreunix/cat.go
+6
-1
@@ -9,7 +9,12 @@ import (
9
)
10
11
func Cat(ctx context.Context, n *core.IpfsNode, pstr string) (uio.DagReader, error) {
12
- dagNode, err := core.Resolve(ctx, n.Namesys, n.Resolver, path.Path(pstr))
12
+ r := &path.Resolver{
13
+ DAG: n.DAG,
14
+ ResolveOnce: uio.ResolveUnixfsOnce,
15
+ }
16
+
17
+ dagNode, err := core.Resolve(ctx, n.Namesys, r, path.Path(pstr))
18
if err != nil {
19
return nil, err
20
}
path/resolver.go
+2
-2
@@ -163,7 +163,7 @@ func (s *Resolver) ResolveLinks(ctx context.Context, ndd node.Node, names []stri
163
ctx, cancel = context.WithTimeout(ctx, time.Minute)
164
defer cancel()
165
166
- lnk, rest, err := nd.ResolveLink(names)
166
+ lnk, err := s.ResolveOnce(ctx, s.DAG, nd, names[0])
167
if err == dag.ErrLinkNotFound {
168
return result, ErrNoLink{Name: names[0], Node: nd.Cid()}
169
} else if err != nil {
@@ -177,7 +177,7 @@ func (s *Resolver) ResolveLinks(ctx context.Context, ndd node.Node, names []stri
177
178
nd = nextnode
179
result = append(result, nextnode)
180
- names = rest
180
+ names = names[1:]
181
}
182
return result, nil
183
}
test/sharness/t0260-sharding-flag.sh
+5
@@ -68,6 +68,11 @@ test_add_large_dir_v1() {
68
echo "$exphash" > sharddir_exp &&
69
test_cmp sharddir_exp sharddir_out
70
'
71
+
72
+ test_expect_success "can access a path under the dir" '
73
+ ipfs cat "$exphash/file20" > file20_out &&
74
+ test_cmp testdata/file20 file20_out
75
+ '
76
}
77
78
# this hash implies both the directory and the leaf entries are CIDv1
unixfs/hamt/hamt.go
+45
-18
@@ -62,7 +62,7 @@ type HamtShard struct {
62
63
// child can either be another shard, or a leaf node value
64
type child interface {
65
- Node() (node.Node, error)
65
+ Link() (*node.Link, error)
66
Label() string
67
}
68
@@ -144,12 +144,12 @@ func (ds *HamtShard) Node() (node.Node, error) {
144
cindex := ds.indexForBitPos(i)
145
ch := ds.children[cindex]
146
if ch != nil {
147
- cnd, err := ch.Node()
147
+ clnk, err := ch.Link()
148
if err != nil {
149
return nil, err
150
}
151
152
- err = out.AddNodeLinkClean(ds.linkNamePrefix(i)+ch.Label(), cnd)
152
+ err = out.AddRawLink(ds.linkNamePrefix(i)+ch.Label(), clnk)
153
if err != nil {
154
return nil, err
155
}
@@ -188,10 +188,10 @@ func (ds *HamtShard) Node() (node.Node, error) {
188
189
type shardValue struct {
190
key string
191
- val node.Node
191
+ val *node.Link
192
}
193
194
-func (sv *shardValue) Node() (node.Node, error) {
194
+func (sv *shardValue) Link() (*node.Link, error) {
195
return sv.val, nil
196
}
197
@@ -214,7 +214,18 @@ func (ds *HamtShard) Label() string {
214
// Set sets 'name' = nd in the HAMT
215
func (ds *HamtShard) Set(ctx context.Context, name string, nd node.Node) error {
216
hv := &hashBits{b: hash([]byte(name))}
217
- return ds.modifyValue(ctx, hv, name, nd)
217
+ _, err := ds.dserv.Add(nd)
218
+ if err != nil {
219
+ return err
220
+ }
221
+
222
+ lnk, err := node.MakeLink(nd)
223
+ if err != nil {
224
+ return err
225
+ }
226
+ lnk.Name = ds.linkNamePrefix(0) + name
227
+
228
+ return ds.modifyValue(ctx, hv, name, lnk)
229
}
230
231
// Remove deletes the named entry if it exists, this operation is idempotent.
@@ -226,13 +237,16 @@ func (ds *HamtShard) Remove(ctx context.Context, name string) error {
237
func (ds *HamtShard) Find(ctx context.Context, name string) (node.Node, error) {
238
hv := &hashBits{b: hash([]byte(name))}
239
229
- var out node.Node
240
+ var out *node.Link
241
err := ds.getValue(ctx, hv, name, func(sv *shardValue) error {
242
out = sv.val
243
return nil
244
})
245
+ if err != nil {
246
+ return nil, err
247
+ }
248
235
- return out, err
249
+ return ds.dserv.Get(ctx, out.Cid)
250
}
251
252
// getChild returns the i'th child of this shard. If it is cached in the
@@ -291,9 +305,10 @@ func (ds *HamtShard) loadChild(ctx context.Context, i int) (child, error) {
305
306
c = cds
307
} else {
308
+ lnk2 := *lnk
309
c = &shardValue{
310
key: lnk.Name[ds.maxpadlen:],
296
- val: nd,
311
+ val: &lnk2,
312
}
313
}
314
@@ -305,16 +320,32 @@ func (ds *HamtShard) setChild(i int, c child) {
320
ds.children[i] = c
321
}
322
308
-func (ds *HamtShard) insertChild(idx int, key string, val node.Node) error {
309
- if val == nil {
323
+func (ds *HamtShard) Link() (*node.Link, error) {
324
+ nd, err := ds.Node()
325
+ if err != nil {
326
+ return nil, err
327
+ }
328
+
329
+ _, err = ds.dserv.Add(nd)
330
+ if err != nil {
331
+ return nil, err
332
+ }
333
+
334
+ return node.MakeLink(nd)
335
+}
336
+
337
+func (ds *HamtShard) insertChild(idx int, key string, lnk *node.Link) error {
338
+ if lnk == nil {
339
return os.ErrNotExist
340
}
341
342
i := ds.indexForBitPos(idx)
343
ds.bitfield.SetBit(ds.bitfield, idx, 1)
344
+
345
+ lnk.Name = ds.linkNamePrefix(idx) + key
346
sv := &shardValue{
347
key: key,
317
- val: val,
348
+ val: lnk,
349
}
350
351
ds.children = append(ds.children[:i], append([]child{sv}, ds.children[i:]...)...)
@@ -370,11 +401,7 @@ func (ds *HamtShard) EnumLinks(ctx context.Context) ([]*node.Link, error) {
401
402
func (ds *HamtShard) ForEachLink(ctx context.Context, f func(*node.Link) error) error {
403
return ds.walkTrie(ctx, func(sv *shardValue) error {
373
- lnk, err := node.MakeLink(sv.val)
374
- if err != nil {
375
- return err
376
- }
377
-
404
+ lnk := sv.val
405
lnk.Name = sv.key
406
407
return f(lnk)
@@ -414,7 +441,7 @@ func (ds *HamtShard) walkTrie(ctx context.Context, cb func(*shardValue) error) e
441
return nil
442
}
443
417
-func (ds *HamtShard) modifyValue(ctx context.Context, hv *hashBits, key string, val node.Node) error {
444
+func (ds *HamtShard) modifyValue(ctx context.Context, hv *hashBits, key string, val *node.Link) error {
445
idx := hv.Next(ds.tableSizeLg2)
446
447
if ds.bitfield.Bit(idx) != 1 {
unixfs/io/dirbuilder.go
+4
-2
@@ -48,10 +48,12 @@ func NewDirectory(dserv mdag.DAGService) *Directory {
48
return db
49
}
50
51
+var ErrNotADir = fmt.Errorf("merkledag node was not a directory or shard")
52
+
53
func NewDirectoryFromNode(dserv mdag.DAGService, nd node.Node) (*Directory, error) {
54
pbnd, ok := nd.(*mdag.ProtoNode)
55
if !ok {
54
- return nil, mdag.ErrNotProtobuf
56
+ return nil, ErrNotADir
57
}
58
59
pbd, err := format.FromBytes(pbnd.Data())
@@ -76,7 +78,7 @@ func NewDirectoryFromNode(dserv mdag.DAGService, nd node.Node) (*Directory, erro
78
shard: shard,
79
}, nil
80
default:
79
- return nil, fmt.Errorf("merkledag node was not a directory or shard")
81
+ return nil, ErrNotADir
82
}
83
}
84