add global config switch for sharding
License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com>
Jeromy committed
Mar 22, 2017 at 16:09 UTC
c4c665395a8ef92360f4251bbb3df712538035f6
11 files changed
+124
-39
core/builder.go
+4
@@ -18,6 +18,7 @@ import (
18
pin "github.com/ipfs/go-ipfs/pin"
19
repo "github.com/ipfs/go-ipfs/repo"
20
cfg "github.com/ipfs/go-ipfs/repo/config"
21
+ uio "github.com/ipfs/go-ipfs/unixfs/io"
22
23
ci "gx/ipfs/QmPGxZ1DP2w45WcogpW1h43BvseXbfke9N91qotpoQcUeS/go-libp2p-crypto"
24
ds "gx/ipfs/QmRWDav6mzWseLWeYfVd5fvUKiVe9xNH29YfMF438fG364/go-datastore"
@@ -175,6 +176,9 @@ func setupNode(ctx context.Context, n *IpfsNode, cfg *BuildCfg) error {
176
return err
177
}
178
179
+ // TEMP: setting global sharding switch here
180
+ uio.UseHAMTSharding = conf.Experimental.ShardingEnabled
181
+
182
opts.HasBloomFilterSize = conf.Datastore.BloomFilterSize
183
if !cfg.Permament {
184
opts.HasBloomFilterSize = 0
core/commands/ls.go
+15
-2
@@ -105,11 +105,24 @@ The JSON output contains type information.
105
106
output := make([]LsObject, len(req.Arguments()))
107
for i, dagnode := range dagnodes {
108
+ dir, err := uio.NewDirectoryFromNode(nd.DAG, dagnode)
109
+ if err != nil {
110
+ res.SetError(err, cmds.ErrNormal)
111
+ return
112
+ }
113
+
114
+ links, err := dir.Links()
115
+ if err != nil {
116
+ res.SetError(err, cmds.ErrNormal)
117
+ return
118
+ }
119
+
120
output[i] = LsObject{
121
Hash: paths[i],
110
- Links: make([]LsLink, len(dagnode.Links())),
122
+ Links: make([]LsLink, len(links)),
123
}
112
- for j, link := range dagnode.Links() {
124
+
125
+ for j, link := range links {
126
t := unixfspb.Data_DataType(-1)
127
128
linkNode, err := link.GetNode(req.Context(), dserv)
core/coreunix/add.go
+6
-3
@@ -190,15 +190,18 @@ func (adder *Adder) PinRoot() error {
190
func (adder *Adder) Finalize() (node.Node, error) {
191
root := adder.mr.GetValue()
192
193
- // cant just call adder.RootNode() here as we need the name for printing
194
- rootNode, err := root.GetNode()
193
+ err := root.Flush()
194
if err != nil {
195
return nil, err
196
}
197
198
var name string
199
if !adder.Wrap {
201
- name = rootNode.Links()[0].Name
200
+ children, err := root.(*mfs.Directory).ListNames()
201
+ if err != nil {
202
+ return nil, err
203
+ }
204
+ name = children[0]
205
206
dir, ok := adder.mr.GetValue().(*mfs.Directory)
207
if !ok {
mfs/dir.go
+3
-6
@@ -59,7 +59,7 @@ func NewDirectory(ctx context.Context, name string, node node.Node, parent child
59
60
// closeChild updates the child by the given name to the dag node 'nd'
61
// and changes its own dag node
62
-func (d *Directory) closeChild(name string, nd *dag.ProtoNode, sync bool) error {
62
+func (d *Directory) closeChild(name string, nd node.Node, sync bool) error {
63
mynd, err := d.closeChildUpdate(name, nd, sync)
64
if err != nil {
65
return err
@@ -72,7 +72,7 @@ func (d *Directory) closeChild(name string, nd *dag.ProtoNode, sync bool) error
72
}
73
74
// closeChildUpdate is the portion of closeChild that needs to be locked around
75
-func (d *Directory) closeChildUpdate(name string, nd *dag.ProtoNode, sync bool) (*dag.ProtoNode, error) {
75
+func (d *Directory) closeChildUpdate(name string, nd node.Node, sync bool) (*dag.ProtoNode, error) {
76
d.lock.Lock()
77
defer d.lock.Unlock()
78
@@ -329,13 +329,10 @@ func (d *Directory) Unlink(name string) error {
329
}
330
331
func (d *Directory) Flush() error {
332
- d.lock.Lock()
333
- nd, err := d.flushCurrentNode()
332
+ nd, err := d.GetNode()
333
if err != nil {
335
- d.lock.Unlock()
334
return err
335
}
338
- d.lock.Unlock()
336
337
return d.parent.closeChild(d.name, nd, true)
338
}
mfs/mfs_test.go
+1
-1
@@ -752,7 +752,7 @@ func TestMfsHugeDir(t *testing.T) {
752
defer cancel()
753
_, rt := setupRoot(ctx, t)
754
755
- for i := 0; i < 100000; i++ {
755
+ for i := 0; i < 10000; i++ {
756
err := Mkdir(rt, fmt.Sprintf("/dir%d", i), false, false)
757
if err != nil {
758
t.Fatal(err)
mfs/system.go
+5
-4
@@ -12,6 +12,7 @@ package mfs
12
import (
13
"context"
14
"errors"
15
+ "fmt"
16
"sync"
17
"time"
18
@@ -30,7 +31,7 @@ var log = logging.Logger("mfs")
31
var ErrIsDirectory = errors.New("error: is a directory")
32
33
type childCloser interface {
33
- closeChild(string, *dag.ProtoNode, bool) error
34
+ closeChild(string, node.Node, bool) error
35
}
36
37
type NodeType int
@@ -87,7 +88,7 @@ func NewRoot(parent context.Context, ds dag.DAGService, node *dag.ProtoNode, pf
88
}
89
90
switch pbn.GetType() {
90
- case ft.TDirectory:
91
+ case ft.TDirectory, ft.THAMTShard:
92
rval, err := NewDirectory(parent, node.String(), node, root, ds)
93
if err != nil {
94
return nil, err
@@ -101,7 +102,7 @@ func NewRoot(parent context.Context, ds dag.DAGService, node *dag.ProtoNode, pf
102
}
103
root.val = fi
104
default:
104
- panic("unrecognized! (NYI)")
105
+ return nil, fmt.Errorf("unrecognized unixfs type: %s", pbn.GetType())
106
}
107
return root, nil
108
}
@@ -124,7 +125,7 @@ func (kr *Root) Flush() error {
125
126
// closeChild implements the childCloser interface, and signals to the publisher that
127
// there are changes ready to be published
127
-func (kr *Root) closeChild(name string, nd *dag.ProtoNode, sync bool) error {
128
+func (kr *Root) closeChild(name string, nd node.Node, sync bool) error {
129
c, err := kr.dserv.Add(nd)
130
if err != nil {
131
return err
repo/config/experiments.go
+1
@@ -2,4 +2,5 @@ package config
2
3
type Experiments struct {
4
FilestoreEnabled bool
5
+ ShardingEnabled bool
6
}
test/sharness/t0040-add-and-cat.sh
-18
@@ -207,20 +207,6 @@ test_add_named_pipe() {
207
'
208
}
209
210
-test_add_sharded_dir() {
211
- mkdir testdata
212
- for i in `seq 2000`
213
- do
214
- echo $i > testdata/file$i
215
- done
216
-
217
- test_expect_success "ipfs add on very large directory succeeds" '
218
- ipfs add -r -q testdata | tail -n1 > sharddir_out &&
219
- echo QmSCJD1KYLhVVHqBK3YyXuoEqHt7vggyJhzoFYbT8v1XYL > sharddir_exp &&
220
- test_cmp sharddir_exp sharddir_out
221
- '
222
-}
223
-
210
test_add_pwd_is_symlink() {
211
test_expect_success "ipfs add -r adds directory content when ./ is symlink" '
212
mkdir hellodir &&
@@ -453,8 +439,6 @@ test_kill_ipfs_daemon
439
440
test_add_cat_file
441
456
-test_add_sharded_dir
457
-
442
test_add_cat_raw
443
444
test_expect_success "ipfs add --only-hash succeeds" '
@@ -475,8 +459,6 @@ test_launch_ipfs_daemon --offline
459
460
test_add_cat_file
461
478
-test_add_sharded_dir
479
-
462
test_kill_ipfs_daemon
463
464
test_done
test/sharness/t0250-files-api.sh
+17
-3
@@ -51,9 +51,9 @@ test_sharding() {
51
ipfs files mkdir /foo
52
'
53
54
- test_expect_success "can make 1100 files in a directory" '
54
+ test_expect_success "can make 100 files in a directory" '
55
printf "" > list_exp_raw
56
- for i in `seq 1100`
56
+ for i in `seq 100`
57
do
58
echo $i | ipfs files write --create /foo/file$i
59
echo file$i >> list_exp_raw
@@ -71,6 +71,12 @@ test_sharding() {
71
echo "65" > file_exp &&
72
test_cmp file_out file_exp
73
'
74
+
75
+ test_expect_success "output object was really sharded" '
76
+ ipfs files stat --hash /foo > expected_foo_hash &&
77
+ echo QmPkwLJTYZRGPJ8Lazr9qPdrLmswPtUjaDbEpmR9jEh1se > actual_foo_hash &&
78
+ test_cmp expected_foo_hash actual_foo_hash
79
+ '
80
}
81
82
test_files_api() {
@@ -508,7 +514,7 @@ test_files_api() {
514
}
515
516
# test offline and online
511
-#test_files_api
517
+test_files_api
518
519
test_expect_success "clean up objects from previous test run" '
520
ipfs repo gc
@@ -518,6 +524,14 @@ test_launch_ipfs_daemon
524
525
ONLINE=1 # set online flag so tests can easily tell
526
test_files_api
527
+test_kill_ipfs_daemon
528
+
529
+test_expect_success "enable sharding in config" '
530
+ ipfs config --json Experimental.ShardingEnabled true
531
+'
532
+
533
+test_launch_ipfs_daemon
534
test_sharding
535
test_kill_ipfs_daemon
536
+
537
test_done
test/sharness/t0260-sharding-flag.sh
new
+58
@@ -0,0 +1,58 @@
1
+#!/bin/sh
2
+#
3
+# Copyright (c) 2014 Christian Couder
4
+# MIT Licensed; see the LICENSE file in this repository.
5
+#
6
+
7
+test_description="Test global enable sharding flag"
8
+
9
+. lib/test-lib.sh
10
+
11
+test_expect_success "set up test data" '
12
+ mkdir testdata
13
+ for i in `seq 2000`
14
+ do
15
+ echo $i > testdata/file$i
16
+ done
17
+'
18
+
19
+test_add_large_dir() {
20
+ exphash="$1"
21
+ test_expect_success "ipfs add on very large directory succeeds" '
22
+ ipfs add -r -q testdata | tail -n1 > sharddir_out &&
23
+ echo "$exphash" > sharddir_exp &&
24
+ test_cmp sharddir_exp sharddir_out
25
+ '
26
+}
27
+
28
+test_init_ipfs
29
+
30
+UNSHARDED="QmavrTrQG4VhoJmantURAYuw3bowq3E2WcvP36NRQDAC1N"
31
+test_add_large_dir "$UNSHARDED"
32
+
33
+test_launch_ipfs_daemon
34
+
35
+test_add_large_dir "$UNSHARDED"
36
+
37
+test_kill_ipfs_daemon
38
+
39
+test_expect_success "enable sharding" '
40
+ ipfs config --json Experimental.ShardingEnabled true
41
+'
42
+
43
+SHARDED="QmSCJD1KYLhVVHqBK3YyXuoEqHt7vggyJhzoFYbT8v1XYL"
44
+test_add_large_dir "$SHARDED"
45
+
46
+test_launch_ipfs_daemon
47
+
48
+test_add_large_dir "$SHARDED"
49
+
50
+test_kill_ipfs_daemon
51
+
52
+test_expect_success "sharded and unsharded output look the same" '
53
+ ipfs ls "$SHARDED" | sort > sharded_out &&
54
+ ipfs ls "$UNSHARDED" | sort > unsharded_out &&
55
+ test_cmp sharded_out unsharded_out
56
+'
57
+
58
+test_done
unixfs/io/dirbuilder.go
+14
-2
@@ -17,6 +17,10 @@ import (
17
// result in the node being restructured into a sharded object.
18
var ShardSplitThreshold = 1000
19
20
+// UseHAMTSharding is a global flag that signifies whether or not to use the
21
+// HAMT sharding scheme for directory creation
22
+var UseHAMTSharding = false
23
+
24
// DefaultShardWidth is the default value used for hamt sharding width.
25
var DefaultShardWidth = 256
26
@@ -31,7 +35,15 @@ type Directory struct {
35
func NewDirectory(dserv mdag.DAGService) *Directory {
36
db := new(Directory)
37
db.dserv = dserv
34
- db.dirnode = format.EmptyDirNode()
38
+ if UseHAMTSharding {
39
+ s, err := hamt.NewHamtShard(dserv, DefaultShardWidth)
40
+ if err != nil {
41
+ panic(err) // will only panic if DefaultShardWidth is a bad value
42
+ }
43
+ db.shard = s
44
+ } else {
45
+ db.dirnode = format.EmptyDirNode()
46
+ }
47
return db
48
}
49
@@ -70,7 +82,7 @@ func NewDirectoryFromNode(dserv mdag.DAGService, nd node.Node) (*Directory, erro
82
// AddChild adds a (name, key)-pair to the root node.
83
func (d *Directory) AddChild(ctx context.Context, name string, nd node.Node) error {
84
if d.shard == nil {
73
- if len(d.dirnode.Links()) < ShardSplitThreshold {
85
+ if !UseHAMTSharding {
86
_ = d.dirnode.RemoveNodeLink(name)
87
return d.dirnode.AddNodeLinkClean(name, nd)
88
}