refactoring unixfs node and importer/helpers to match
Jeromy committed
Mar 6, 2015 at 11:20 UTC
0e9772c57ef97f39a8f318857e32ab1dd1b63225
4 files changed
+103
-26
importer/helpers/dagbuilder.go
+7
-2
@@ -78,6 +78,11 @@ func (db *DagBuilderHelper) Next() []byte {
78
return d
79
}
80
81
+// GetDagServ returns the dagservice object this Helper is using
82
+func (db *DagBuilderHelper) GetDagServ() dag.DAGService {
83
+ return db.dserv
84
+}
85
+
86
// FillNodeLayer will add datanodes as children to the give node until
87
// at most db.indirSize ndoes are added
88
//
@@ -86,7 +91,7 @@ func (db *DagBuilderHelper) FillNodeLayer(node *UnixfsNode) error {
91
92
// while we have room AND we're not done
93
for node.NumChildren() < db.maxlinks && !db.Done() {
89
- child := NewUnixfsNode()
94
+ child := NewUnixfsBlock()
95
96
if err := db.FillNodeWithData(child); err != nil {
97
return err
@@ -110,7 +115,7 @@ func (db *DagBuilderHelper) FillNodeWithData(node *UnixfsNode) error {
115
return ErrSizeLimitExceeded
116
}
117
113
- node.setData(data)
118
+ node.SetData(data)
119
return nil
120
}
121
importer/helpers/helpers.go
+40
-3
@@ -38,20 +38,51 @@ var ErrSizeLimitExceeded = fmt.Errorf("object size limit exceeded")
38
// of unixfs DAG trees
39
type UnixfsNode struct {
40
node *dag.Node
41
- ufmt *ft.MultiBlock
41
+ ufmt *ft.FSNode
42
}
43
44
+// NewUnixfsNode creates a new Unixfs node to represent a file
45
func NewUnixfsNode() *UnixfsNode {
46
return &UnixfsNode{
47
node: new(dag.Node),
47
- ufmt: new(ft.MultiBlock),
48
+ ufmt: &ft.FSNode{Type: ft.TFile},
49
}
50
}
51
52
+// NewUnixfsBlock creates a new Unixfs node to represent a raw data block
53
+func NewUnixfsBlock() *UnixfsNode {
54
+ return &UnixfsNode{
55
+ node: new(dag.Node),
56
+ ufmt: &ft.FSNode{Type: ft.TRaw},
57
+ }
58
+}
59
+
60
+// NewUnixfsNodeFromDag reconstructs a Unixfs node from a given dag node
61
+func NewUnixfsNodeFromDag(nd *dag.Node) (*UnixfsNode, error) {
62
+ mb, err := ft.FSNodeFromBytes(nd.Data)
63
+ if err != nil {
64
+ return nil, err
65
+ }
66
+
67
+ return &UnixfsNode{
68
+ node: nd,
69
+ ufmt: mb,
70
+ }, nil
71
+}
72
+
73
func (n *UnixfsNode) NumChildren() int {
74
return n.ufmt.NumChildren()
75
}
76
77
+func (n *UnixfsNode) GetChild(i int, ds dag.DAGService) (*UnixfsNode, error) {
78
+ nd, err := n.node.Links[i].GetNode(ds)
79
+ if err != nil {
80
+ return nil, err
81
+ }
82
+
83
+ return NewUnixfsNodeFromDag(nd)
84
+}
85
+
86
// addChild will add the given UnixfsNode as a child of the receiver.
87
// the passed in DagBuilderHelper is used to store the child node an
88
// pin it locally so it doesnt get lost
@@ -83,7 +114,13 @@ func (n *UnixfsNode) AddChild(child *UnixfsNode, db *DagBuilderHelper) error {
114
return nil
115
}
116
86
-func (n *UnixfsNode) setData(data []byte) {
117
+// Removes the child node at the given index
118
+func (n *UnixfsNode) RemoveChild(index int) {
119
+ n.ufmt.RemoveBlockSize(index)
120
+ n.node.Links = append(n.node.Links[:index], n.node.Links[index+1:]...)
121
+}
122
+
123
+func (n *UnixfsNode) SetData(data []byte) {
124
n.ufmt.Data = data
125
}
126
unixfs/format.go
+50
-16
@@ -9,6 +9,13 @@ import (
9
pb "github.com/jbenet/go-ipfs/unixfs/pb"
10
)
11
12
+const (
13
+ TRaw = pb.Data_Raw
14
+ TFile = pb.Data_File
15
+ TDirectory = pb.Data_Directory
16
+ TMetadata = pb.Data_Metadata
17
+)
18
+
19
var ErrMalformedFileFormat = errors.New("malformed data in file format")
20
var ErrInvalidDirLocation = errors.New("found directory node in unexpected place")
21
var ErrUnrecognizedType = errors.New("unrecognized node type")
@@ -98,33 +105,60 @@ func DataSize(data []byte) (uint64, error) {
105
}
106
}
107
101
-type MultiBlock struct {
102
- Data []byte
108
+type FSNode struct {
109
+ Data []byte
110
+
111
+ // total data size for each child
112
blocksizes []uint64
104
- subtotal uint64
113
+
114
+ // running sum of blocksizes
115
+ subtotal uint64
116
+
117
+ // node type of this node
118
+ Type pb.Data_DataType
119
+}
120
+
121
+func FSNodeFromBytes(b []byte) (*FSNode, error) {
122
+ pbn := new(pb.Data)
123
+ err := proto.Unmarshal(b, pbn)
124
+ if err != nil {
125
+ return nil, err
126
+ }
127
+
128
+ n := new(FSNode)
129
+ n.Data = pbn.Data
130
+ n.blocksizes = pbn.Blocksizes
131
+ n.subtotal = pbn.GetFilesize() - uint64(len(n.Data))
132
+ n.Type = pbn.GetType()
133
+ return n, nil
134
+}
135
+
136
+// AddBlockSize adds the size of the next child block of this node
137
+func (n *FSNode) AddBlockSize(s uint64) {
138
+ n.subtotal += s
139
+ n.blocksizes = append(n.blocksizes, s)
140
}
141
107
-func (mb *MultiBlock) AddBlockSize(s uint64) {
108
- mb.subtotal += s
109
- mb.blocksizes = append(mb.blocksizes, s)
142
+func (n *FSNode) RemoveBlockSize(i int) {
143
+ n.subtotal -= n.blocksizes[i]
144
+ n.blocksizes = append(n.blocksizes[:i], n.blocksizes[i+1:]...)
145
}
146
112
-func (mb *MultiBlock) GetBytes() ([]byte, error) {
147
+func (n *FSNode) GetBytes() ([]byte, error) {
148
pbn := new(pb.Data)
114
- t := pb.Data_File
115
- pbn.Type = &t
116
- pbn.Filesize = proto.Uint64(uint64(len(mb.Data)) + mb.subtotal)
117
- pbn.Blocksizes = mb.blocksizes
118
- pbn.Data = mb.Data
149
+ pbn.Type = &n.Type
150
+ pbn.Filesize = proto.Uint64(uint64(len(n.Data)) + n.subtotal)
151
+ pbn.Blocksizes = n.blocksizes
152
+ pbn.Data = n.Data
153
return proto.Marshal(pbn)
154
}
155
122
-func (mb *MultiBlock) FileSize() uint64 {
123
- return uint64(len(mb.Data)) + mb.subtotal
156
+func (n *FSNode) FileSize() uint64 {
157
+ return uint64(len(n.Data)) + n.subtotal
158
}
159
126
-func (mb *MultiBlock) NumChildren() int {
127
- return len(mb.blocksizes)
160
+func (n *FSNode) NumChildren() int {
161
+ return len(n.blocksizes)
162
}
163
164
type Metadata struct {
unixfs/format_test.go
+6
-5
@@ -7,15 +7,16 @@ import (
7
pb "github.com/jbenet/go-ipfs/unixfs/pb"
8
)
9
10
-func TestMultiBlock(t *testing.T) {
11
- mbf := new(MultiBlock)
10
+func TestFSNode(t *testing.T) {
11
+ fsn := new(FSNode)
12
+ fsn.Type = TFile
13
for i := 0; i < 15; i++ {
13
- mbf.AddBlockSize(100)
14
+ fsn.AddBlockSize(100)
15
}
16
16
- mbf.Data = make([]byte, 128)
17
+ fsn.Data = make([]byte, 128)
18
18
- b, err := mbf.GetBytes()
19
+ b, err := fsn.GetBytes()
20
if err != nil {
21
t.Fatal(err)
22
}