working on dag modification structures, factored out the data format into an importer subpackage and added more ipns tests
Jeromy committed
Oct 6, 2014 at 03:42 UTC
5592030ed39dbeaefddbaeddb90abbce104891e7
16 files changed
+454
-105
core/commands/add.go
+2
-1
@@ -10,6 +10,7 @@ import (
10
11
"github.com/jbenet/go-ipfs/core"
12
"github.com/jbenet/go-ipfs/importer"
13
+ ft "github.com/jbenet/go-ipfs/importer/format"
14
dag "github.com/jbenet/go-ipfs/merkledag"
15
u "github.com/jbenet/go-ipfs/util"
16
)
@@ -75,7 +76,7 @@ func AddPath(n *core.IpfsNode, fpath string, depth int) (*dag.Node, error) {
76
}
77
78
func addDir(n *core.IpfsNode, fpath string, depth int) (*dag.Node, error) {
78
- tree := &dag.Node{Data: dag.FolderPBData()}
79
+ tree := &dag.Node{Data: ft.FolderPBData()}
80
81
files, err := ioutil.ReadDir(fpath)
82
if err != nil {
fuse/ipns/ipns_test.go
+91
@@ -63,6 +63,7 @@ func setupIpnsTest(t *testing.T, node *core.IpfsNode) (*core.IpfsNode, *fstest.M
63
return node, mnt
64
}
65
66
+// Test writing a file and reading it back
67
func TestIpnsBasicIO(t *testing.T) {
68
_, mnt := setupIpnsTest(t, nil)
69
defer mnt.Close()
@@ -80,6 +81,7 @@ func TestIpnsBasicIO(t *testing.T) {
81
}
82
}
83
84
+// Test to make sure file changes persist over mounts of ipns
85
func TestFilePersistence(t *testing.T) {
86
node, mnt := setupIpnsTest(t, nil)
87
@@ -104,6 +106,7 @@ func TestFilePersistence(t *testing.T) {
106
}
107
}
108
109
+// Test to make sure the filesystem reports file sizes correctly
110
func TestFileSizeReporting(t *testing.T) {
111
_, mnt := setupIpnsTest(t, nil)
112
defer mnt.Close()
@@ -120,3 +123,91 @@ func TestFileSizeReporting(t *testing.T) {
123
t.Fatal("Read incorrect size from stat!")
124
}
125
}
126
+
127
+// Test to make sure you cant create multiple entries with the same name
128
+func TestDoubleEntryFailure(t *testing.T) {
129
+ _, mnt := setupIpnsTest(t, nil)
130
+ defer mnt.Close()
131
+
132
+ dname := mnt.Dir + "/local/thisisadir"
133
+ err := os.Mkdir(dname, 0777)
134
+ if err != nil {
135
+ t.Fatal(err)
136
+ }
137
+
138
+ err = os.Mkdir(dname, 0777)
139
+ if err == nil {
140
+ t.Fatal("Should have gotten error one creating new directory.")
141
+ }
142
+}
143
+
144
+func TestAppendFile(t *testing.T) {
145
+ _, mnt := setupIpnsTest(t, nil)
146
+ defer mnt.Close()
147
+
148
+ fname := mnt.Dir + "/local/file"
149
+ data := writeFile(t, 1300, fname)
150
+
151
+ fi, err := os.OpenFile(fname, os.O_RDWR|os.O_APPEND, 0666)
152
+ if err != nil {
153
+ t.Fatal(err)
154
+ }
155
+
156
+ nudata := randBytes(500)
157
+
158
+ n, err := fi.Write(nudata)
159
+ if err != nil {
160
+ t.Fatal(err)
161
+ }
162
+ err = fi.Close()
163
+ if err != nil {
164
+ t.Fatal(err)
165
+ }
166
+
167
+ if n != len(nudata) {
168
+ t.Fatal("Failed to write enough bytes.")
169
+ }
170
+
171
+ data = append(data, nudata...)
172
+
173
+ rbuf, err := ioutil.ReadFile(fname)
174
+ if err != nil {
175
+ t.Fatal(err)
176
+ }
177
+ if !bytes.Equal(rbuf, data) {
178
+ t.Fatal("Data inconsistent!")
179
+ }
180
+}
181
+
182
+// Test writing a medium sized file one byte at a time
183
+func TestMultiWrite(t *testing.T) {
184
+ _, mnt := setupIpnsTest(t, nil)
185
+ defer mnt.Close()
186
+
187
+ fpath := mnt.Dir + "/local/file"
188
+ fi, err := os.Create(fpath)
189
+ if err != nil {
190
+ t.Fatal(err)
191
+ }
192
+
193
+ data := randBytes(1001)
194
+ for i := 0; i < len(data); i++ {
195
+ n, err := fi.Write(data[i : i+1])
196
+ if err != nil {
197
+ t.Fatal(err)
198
+ }
199
+ if n != 1 {
200
+ t.Fatal("Somehow wrote the wrong number of bytes! (n != 1)")
201
+ }
202
+ }
203
+ fi.Close()
204
+
205
+ rbuf, err := ioutil.ReadFile(fpath)
206
+ if err != nil {
207
+ t.Fatal(err)
208
+ }
209
+
210
+ if !bytes.Equal(rbuf, data) {
211
+ t.Fatal("File on disk did not match bytes written")
212
+ }
213
+}
fuse/ipns/ipns_unix.go
+15
-8
@@ -14,6 +14,7 @@ import (
14
"github.com/jbenet/go-ipfs/core"
15
ci "github.com/jbenet/go-ipfs/crypto"
16
imp "github.com/jbenet/go-ipfs/importer"
17
+ ft "github.com/jbenet/go-ipfs/importer/format"
18
mdag "github.com/jbenet/go-ipfs/merkledag"
19
u "github.com/jbenet/go-ipfs/util"
20
)
@@ -77,7 +78,7 @@ func CreateRoot(n *core.IpfsNode, keys []ci.PrivKey, ipfsroot string) (*Root, er
78
pointsTo, err := n.Namesys.Resolve(name)
79
if err != nil {
80
log.Warning("Could not resolve value for local ipns entry, providing empty dir")
80
- nd.Nd = &mdag.Node{Data: mdag.FolderPBData()}
81
+ nd.Nd = &mdag.Node{Data: ft.FolderPBData()}
82
root.LocalDirs[name] = nd
83
continue
84
}
@@ -199,14 +200,14 @@ type Node struct {
200
Ipfs *core.IpfsNode
201
Nd *mdag.Node
202
fd *mdag.DagReader
202
- cached *mdag.PBData
203
+ cached *ft.PBData
204
205
// For writing
206
writerBuf WriteAtBuf
207
}
208
209
func (s *Node) loadData() error {
209
- s.cached = new(mdag.PBData)
210
+ s.cached = new(ft.PBData)
211
return proto.Unmarshal(s.Nd.Data, s.cached)
212
}
213
@@ -216,10 +217,10 @@ func (s *Node) Attr() fuse.Attr {
217
s.loadData()
218
}
219
switch s.cached.GetType() {
219
- case mdag.PBData_Directory:
220
+ case ft.PBData_Directory:
221
return fuse.Attr{Mode: os.ModeDir | 0555}
221
- case mdag.PBData_File, mdag.PBData_Raw:
222
- size, err := s.Nd.DataSize()
222
+ case ft.PBData_File, ft.PBData_Raw:
223
+ size, err := ft.DataSize(s.Nd.Data)
224
if err != nil {
225
log.Error("Error getting size of file: %s", err)
226
size = 0
@@ -414,7 +415,7 @@ func (n *Node) Fsync(req *fuse.FsyncRequest, intr fs.Intr) fuse.Error {
415
416
func (n *Node) Mkdir(req *fuse.MkdirRequest, intr fs.Intr) (fs.Node, fuse.Error) {
417
log.Debug("Got mkdir request!")
417
- dagnd := &mdag.Node{Data: mdag.FolderPBData()}
418
+ dagnd := &mdag.Node{Data: ft.FolderPBData()}
419
nnode := n.Nd.Copy()
420
nnode.AddNodeLink(req.Name, dagnd)
421
@@ -448,6 +449,12 @@ func (n *Node) Mkdir(req *fuse.MkdirRequest, intr fs.Intr) (fs.Node, fuse.Error)
449
func (n *Node) Open(req *fuse.OpenRequest, resp *fuse.OpenResponse, intr fs.Intr) (fs.Handle, fuse.Error) {
450
//log.Debug("[%s] Received open request! flags = %s", n.name, req.Flags.String())
451
//TODO: check open flags and truncate if necessary
452
+ if req.Flags&fuse.OpenTruncate != 0 {
453
+ log.Warning("Need to truncate file!")
454
+ }
455
+ if req.Flags&fuse.OpenAppend != 0 {
456
+ log.Warning("Need to append to file!")
457
+ }
458
return n, nil
459
}
460
@@ -460,7 +467,7 @@ func (n *Node) Create(req *fuse.CreateRequest, resp *fuse.CreateResponse, intr f
467
log.Debug("Got create request: %s", req.Name)
468
469
// New 'empty' file
463
- nd := &mdag.Node{Data: mdag.FilePBData(nil, 0)}
470
+ nd := &mdag.Node{Data: ft.FilePBData(nil, 0)}
471
child := n.makeChild(req.Name, nd)
472
473
nnode := n.Nd.Copy()
fuse/readonly/readonly_unix.go
+5
-4
@@ -19,6 +19,7 @@ import (
19
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse"
20
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs"
21
core "github.com/jbenet/go-ipfs/core"
22
+ ft "github.com/jbenet/go-ipfs/importer/format"
23
mdag "github.com/jbenet/go-ipfs/merkledag"
24
u "github.com/jbenet/go-ipfs/util"
25
)
@@ -79,11 +80,11 @@ type Node struct {
80
Ipfs *core.IpfsNode
81
Nd *mdag.Node
82
fd *mdag.DagReader
82
- cached *mdag.PBData
83
+ cached *ft.PBData
84
}
85
86
func (s *Node) loadData() error {
86
- s.cached = new(mdag.PBData)
87
+ s.cached = new(ft.PBData)
88
return proto.Unmarshal(s.Nd.Data, s.cached)
89
}
90
@@ -94,9 +95,9 @@ func (s *Node) Attr() fuse.Attr {
95
s.loadData()
96
}
97
switch s.cached.GetType() {
97
- case mdag.PBData_Directory:
98
+ case ft.PBData_Directory:
99
return fuse.Attr{Mode: os.ModeDir | 0555}
99
- case mdag.PBData_File, mdag.PBData_Raw:
100
+ case ft.PBData_File, ft.PBData_Raw:
101
size, _ := s.Nd.Size()
102
return fuse.Attr{
103
Mode: 0444,
importer/dagwriter/dagwriter.go
new
+80
@@ -0,0 +1,80 @@
1
+package dagwriter
2
+
3
+import (
4
+ imp "github.com/jbenet/go-ipfs/importer"
5
+ ft "github.com/jbenet/go-ipfs/importer/format"
6
+ dag "github.com/jbenet/go-ipfs/merkledag"
7
+ "github.com/jbenet/go-ipfs/util"
8
+)
9
+
10
+var log = util.Logger("dagwriter")
11
+
12
+type DagWriter struct {
13
+ dagserv *dag.DAGService
14
+ node *dag.Node
15
+ totalSize int64
16
+ splChan chan []byte
17
+ done chan struct{}
18
+ splitter imp.StreamSplitter
19
+ seterr error
20
+}
21
+
22
+func NewDagWriter(ds *dag.DAGService, splitter imp.StreamSplitter) *DagWriter {
23
+ dw := new(DagWriter)
24
+ dw.dagserv = ds
25
+ dw.splChan = make(chan []byte, 8)
26
+ dw.splitter = splitter
27
+ dw.done = make(chan struct{})
28
+ go dw.startSplitter()
29
+ return dw
30
+}
31
+
32
+func (dw *DagWriter) startSplitter() {
33
+ blkchan := dw.splitter.Split(dw.splChan)
34
+ first := <-blkchan
35
+ root := new(dag.Node)
36
+ fileSize := uint64(0)
37
+ for blkData := range blkchan {
38
+ fileSize += uint64(len(blkData))
39
+ node := &dag.Node{Data: ft.WrapData(blkData)}
40
+ _, err := dw.dagserv.Add(node)
41
+ if err != nil {
42
+ dw.seterr = err
43
+ log.Critical("Got error adding created node to dagservice: %s", err)
44
+ return
45
+ }
46
+ err = root.AddNodeLinkClean("", node)
47
+ if err != nil {
48
+ dw.seterr = err
49
+ log.Critical("Got error adding created node to root node: %s", err)
50
+ return
51
+ }
52
+ }
53
+ root.Data = ft.FilePBData(first, fileSize)
54
+ _, err := dw.dagserv.Add(root)
55
+ if err != nil {
56
+ dw.seterr = err
57
+ log.Critical("Got error adding created node to dagservice: %s", err)
58
+ return
59
+ }
60
+ dw.node = root
61
+ dw.done <- struct{}{}
62
+}
63
+
64
+func (dw *DagWriter) Write(b []byte) (int, error) {
65
+ if dw.seterr != nil {
66
+ return 0, dw.seterr
67
+ }
68
+ dw.splChan <- b
69
+ return len(b), nil
70
+}
71
+
72
+func (dw *DagWriter) Close() error {
73
+ close(dw.splChan)
74
+ <-dw.done
75
+ return nil
76
+}
77
+
78
+func (dw *DagWriter) GetNode() *dag.Node {
79
+ return dw.node
80
+}
importer/dagwriter/dagwriter_test.go
new
+102
@@ -0,0 +1,102 @@
1
+package dagwriter
2
+
3
+import (
4
+ "testing"
5
+
6
+ "io"
7
+
8
+ ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
9
+ bs "github.com/jbenet/go-ipfs/blockservice"
10
+ imp "github.com/jbenet/go-ipfs/importer"
11
+ mdag "github.com/jbenet/go-ipfs/merkledag"
12
+)
13
+
14
+type datasource struct {
15
+ i int
16
+}
17
+
18
+func (d *datasource) Read(b []byte) (int, error) {
19
+ for i, _ := range b {
20
+ b[i] = byte(d.i % 256)
21
+ d.i++
22
+ }
23
+ return len(b), nil
24
+}
25
+
26
+func (d *datasource) Matches(t *testing.T, r io.Reader, length int) bool {
27
+ b := make([]byte, 100)
28
+ i := 0
29
+ for {
30
+ n, err := r.Read(b)
31
+ if err != nil && err != io.EOF {
32
+ t.Fatal(err)
33
+ }
34
+ for _, v := range b[:n] {
35
+ if v != byte(i%256) {
36
+ t.Fatalf("Buffers differed at byte: %d (%d != %d)", i, v, (i % 256))
37
+ }
38
+ i++
39
+ }
40
+ if err == io.EOF {
41
+ break
42
+ }
43
+ }
44
+ if i != length {
45
+ t.Fatalf("Incorrect length. (%d != %d)", i, length)
46
+ }
47
+ return true
48
+}
49
+
50
+func TestDagWriter(t *testing.T) {
51
+ dstore := ds.NewMapDatastore()
52
+ bserv, err := bs.NewBlockService(dstore, nil)
53
+ if err != nil {
54
+ t.Fatal(err)
55
+ }
56
+ dag := &mdag.DAGService{bserv}
57
+ dw := NewDagWriter(dag, &imp.SizeSplitter2{4096})
58
+
59
+ nbytes := int64(1024 * 1024 * 2)
60
+ n, err := io.CopyN(dw, &datasource{}, nbytes)
61
+ if err != nil {
62
+ t.Fatal(err)
63
+ }
64
+
65
+ if n != nbytes {
66
+ t.Fatal("Copied incorrect amount of bytes!")
67
+ }
68
+
69
+ dw.Close()
70
+
71
+ node := dw.GetNode()
72
+ read, err := mdag.NewDagReader(node, dag)
73
+ if err != nil {
74
+ t.Fatal(err)
75
+ }
76
+
77
+ d := &datasource{}
78
+ if !d.Matches(t, read, int(nbytes)) {
79
+ t.Fatal("Failed to validate!")
80
+ }
81
+}
82
+
83
+func TestMassiveWrite(t *testing.T) {
84
+ t.SkipNow()
85
+ dstore := ds.NewNullDatastore()
86
+ bserv, err := bs.NewBlockService(dstore, nil)
87
+ if err != nil {
88
+ t.Fatal(err)
89
+ }
90
+ dag := &mdag.DAGService{bserv}
91
+ dw := NewDagWriter(dag, &imp.SizeSplitter2{4096})
92
+
93
+ nbytes := int64(1024 * 1024 * 1024 * 16)
94
+ n, err := io.CopyN(dw, &datasource{}, nbytes)
95
+ if err != nil {
96
+ t.Fatal(err)
97
+ }
98
+ if n != nbytes {
99
+ t.Fatal("Incorrect copy size.")
100
+ }
101
+ dw.Close()
102
+}
importer/format/Makefile
new
+5
@@ -0,0 +1,5 @@
1
+all: data.pb.go
2
+
3
+data.pb.go: data.proto
4
+ protoc --go_out=. data.proto
5
+
importer/format/data.pb.go
renamed
+12
-4
@@ -3,7 +3,7 @@
3
// DO NOT EDIT!
4
5
/*
6
-Package merkledag is a generated protocol buffer package.
6
+Package format is a generated protocol buffer package.
7
8
It is generated from these files:
9
data.proto
@@ -11,7 +11,7 @@ It is generated from these files:
11
It has these top-level messages:
12
PBData
13
*/
14
-package merkledag
14
+package format
15
16
import proto "code.google.com/p/goprotobuf/proto"
17
import math "math"
@@ -57,9 +57,10 @@ func (x *PBData_DataType) UnmarshalJSON(data []byte) error {
57
}
58
59
type PBData struct {
60
- Type *PBData_DataType `protobuf:"varint,1,req,enum=merkledag.PBData_DataType" json:"Type,omitempty"`
60
+ Type *PBData_DataType `protobuf:"varint,1,req,enum=format.PBData_DataType" json:"Type,omitempty"`
61
Data []byte `protobuf:"bytes,2,opt" json:"Data,omitempty"`
62
Filesize *uint64 `protobuf:"varint,3,opt,name=filesize" json:"filesize,omitempty"`
63
+ Blocksizes []uint64 `protobuf:"varint,4,rep,name=blocksizes" json:"blocksizes,omitempty"`
64
XXX_unrecognized []byte `json:"-"`
65
}
66
@@ -88,6 +89,13 @@ func (m *PBData) GetFilesize() uint64 {
89
return 0
90
}
91
92
+func (m *PBData) GetBlocksizes() []uint64 {
93
+ if m != nil {
94
+ return m.Blocksizes
95
+ }
96
+ return nil
97
+}
98
+
99
func init() {
92
- proto.RegisterEnum("merkledag.PBData_DataType", PBData_DataType_name, PBData_DataType_value)
100
+ proto.RegisterEnum("format.PBData_DataType", PBData_DataType_name, PBData_DataType_value)
101
}
importer/format/data.proto
renamed
+2
-1
@@ -1,4 +1,4 @@
1
-package merkledag;
1
+package format;
2
3
message PBData {
4
enum DataType {
@@ -10,4 +10,5 @@ message PBData {
10
required DataType Type = 1;
11
optional bytes Data = 2;
12
optional uint64 filesize = 3;
13
+ repeated uint64 blocksizes = 4;
14
}
importer/format/format.go
new
+71
@@ -0,0 +1,71 @@
1
+// Package format implements a data format for files in the ipfs filesystem
2
+// It is not the only format in ipfs, but it is the one that the filesystem assumes
3
+package format
4
+
5
+import (
6
+ "errors"
7
+
8
+ "code.google.com/p/goprotobuf/proto"
9
+)
10
+
11
+func FilePBData(data []byte, totalsize uint64) []byte {
12
+ pbfile := new(PBData)
13
+ typ := PBData_File
14
+ pbfile.Type = &typ
15
+ pbfile.Data = data
16
+ pbfile.Filesize = proto.Uint64(totalsize)
17
+
18
+ data, err := proto.Marshal(pbfile)
19
+ if err != nil {
20
+ //this really shouldnt happen, i promise
21
+ panic(err)
22
+ }
23
+ return data
24
+}
25
+
26
+func FolderPBData() []byte {
27
+ pbfile := new(PBData)
28
+ typ := PBData_Directory
29
+ pbfile.Type = &typ
30
+
31
+ data, err := proto.Marshal(pbfile)
32
+ if err != nil {
33
+ //this really shouldnt happen, i promise
34
+ panic(err)
35
+ }
36
+ return data
37
+}
38
+
39
+func WrapData(b []byte) []byte {
40
+ pbdata := new(PBData)
41
+ typ := PBData_Raw
42
+ pbdata.Data = b
43
+ pbdata.Type = &typ
44
+
45
+ out, err := proto.Marshal(pbdata)
46
+ if err != nil {
47
+ // This shouldnt happen. seriously.
48
+ panic(err)
49
+ }
50
+
51
+ return out
52
+}
53
+
54
+func DataSize(data []byte) (uint64, error) {
55
+ pbdata := new(PBData)
56
+ err := proto.Unmarshal(data, pbdata)
57
+ if err != nil {
58
+ return 0, err
59
+ }
60
+
61
+ switch pbdata.GetType() {
62
+ case PBData_Directory:
63
+ return 0, errors.New("Cant get data size of directory!")
64
+ case PBData_File:
65
+ return pbdata.GetFilesize(), nil
66
+ case PBData_Raw:
67
+ return uint64(len(pbdata.GetData())), nil
68
+ default:
69
+ return 0, errors.New("Unrecognized node data type!")
70
+ }
71
+}
importer/importer.go
+3
-2
@@ -5,6 +5,7 @@ import (
5
"io"
6
"os"
7
8
+ ft "github.com/jbenet/go-ipfs/importer/format"
9
dag "github.com/jbenet/go-ipfs/merkledag"
10
)
11
@@ -34,7 +35,7 @@ func NewDagFromReaderWithSplitter(r io.Reader, spl BlockSplitter) (*dag.Node, er
35
totalsize := uint64(len(first))
36
for blk := range blkChan {
37
totalsize += uint64(len(blk))
37
- child := &dag.Node{Data: dag.WrapData(blk)}
38
+ child := &dag.Node{Data: ft.WrapData(blk)}
39
err := root.AddNodeLink(fmt.Sprintf("%d", i), child)
40
if err != nil {
41
return nil, err
@@ -42,7 +43,7 @@ func NewDagFromReaderWithSplitter(r io.Reader, spl BlockSplitter) (*dag.Node, er
43
i++
44
}
45
45
- root.Data = dag.FilePBData(first, totalsize)
46
+ root.Data = ft.FilePBData(first, totalsize)
47
return root, nil
48
}
49
importer/splitting.go
+29
@@ -6,10 +6,16 @@ import (
6
u "github.com/jbenet/go-ipfs/util"
7
)
8
9
+// OLD
10
type BlockSplitter interface {
11
Split(io.Reader) chan []byte
12
}
13
14
+// NEW
15
+type StreamSplitter interface {
16
+ Split(chan []byte) chan []byte
17
+}
18
+
19
type SizeSplitter struct {
20
Size int
21
}
@@ -39,3 +45,26 @@ func (ss *SizeSplitter) Split(r io.Reader) chan []byte {
45
}()
46
return out
47
}
48
+
49
+type SizeSplitter2 struct {
50
+ Size int
51
+}
52
+
53
+func (ss *SizeSplitter2) Split(in chan []byte) chan []byte {
54
+ out := make(chan []byte)
55
+ go func() {
56
+ defer close(out)
57
+ var buf []byte
58
+ for b := range in {
59
+ buf = append(buf, b...)
60
+ for len(buf) > ss.Size {
61
+ out <- buf[:ss.Size]
62
+ buf = buf[ss.Size:]
63
+ }
64
+ }
65
+ if len(buf) > 0 {
66
+ out <- buf
67
+ }
68
+ }()
69
+ return out
70
+}
merkledag/Makefile
+1
-4
@@ -1,11 +1,8 @@
1
2
-all: node.pb.go data.pb.go
2
+all: node.pb.go
3
4
node.pb.go: node.proto
5
protoc --gogo_out=. --proto_path=../../../../:/usr/local/opt/protobuf/include:. $<
6
7
-data.pb.go: data.proto
8
- protoc --go_out=. data.proto
9
-
7
clean:
8
rm node.pb.go
merkledag/dagreader.go
+9
-8
@@ -6,6 +6,7 @@ import (
6
"io"
7
8
proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
9
+ ft "github.com/jbenet/go-ipfs/importer/format"
10
u "github.com/jbenet/go-ipfs/util"
11
)
12
@@ -20,21 +21,21 @@ type DagReader struct {
21
}
22
23
func NewDagReader(n *Node, serv *DAGService) (io.Reader, error) {
23
- pb := new(PBData)
24
+ pb := new(ft.PBData)
25
err := proto.Unmarshal(n.Data, pb)
26
if err != nil {
27
return nil, err
28
}
29
switch pb.GetType() {
29
- case PBData_Directory:
30
+ case ft.PBData_Directory:
31
return nil, ErrIsDir
31
- case PBData_File:
32
+ case ft.PBData_File:
33
return &DagReader{
34
node: n,
35
serv: serv,
36
buf: bytes.NewBuffer(pb.GetData()),
37
}, nil
37
- case PBData_Raw:
38
+ case ft.PBData_Raw:
39
return bytes.NewBuffer(pb.GetData()), nil
40
default:
41
panic("Unrecognized node type!")
@@ -54,7 +55,7 @@ func (dr *DagReader) precalcNextBuf() error {
55
}
56
nxt = nxtNode
57
}
57
- pb := new(PBData)
58
+ pb := new(ft.PBData)
59
err := proto.Unmarshal(nxt.Data, pb)
60
if err != nil {
61
return err
@@ -62,13 +63,13 @@ func (dr *DagReader) precalcNextBuf() error {
63
dr.position++
64
65
switch pb.GetType() {
65
- case PBData_Directory:
66
+ case ft.PBData_Directory:
67
panic("Why is there a directory under a file?")
67
- case PBData_File:
68
+ case ft.PBData_File:
69
//TODO: this *should* work, needs testing first
70
//return NewDagReader(nxt, dr.serv)
71
panic("Not yet handling different layers of indirection!")
71
- case PBData_Raw:
72
+ case ft.PBData_Raw:
73
dr.buf = bytes.NewBuffer(pb.GetData())
74
return nil
75
default:
merkledag/merkledag.go
+25
-72
@@ -1,11 +1,8 @@
1
package merkledag
2
3
import (
4
- "errors"
4
"fmt"
5
7
- proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
8
-
6
mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
7
blocks "github.com/jbenet/go-ipfs/blocks"
8
bserv "github.com/jbenet/go-ipfs/blockservice"
@@ -37,6 +34,9 @@ type Link struct {
34
// cumulative size of target object
35
Size uint64
36
37
+ // cumulative size of data stored in object
38
+ DataSize uint64
39
+
40
// multihash of the target object
41
Hash mh.Multihash
42
@@ -46,14 +46,28 @@ type Link struct {
46
47
// AddNodeLink adds a link to another node.
48
func (n *Node) AddNodeLink(name string, that *Node) error {
49
- // DEBUG CODE
50
- for _, l := range n.Links {
51
- if l.Name == name {
52
- panic("Trying to add child that already exists!")
53
- }
49
+ s, err := that.Size()
50
+ if err != nil {
51
+ return err
52
}
55
- //
53
54
+ h, err := that.Multihash()
55
+ if err != nil {
56
+ return err
57
+ }
58
+
59
+ n.Links = append(n.Links, &Link{
60
+ Name: name,
61
+ Size: s,
62
+ Hash: h,
63
+ Node: that,
64
+ })
65
+ return nil
66
+}
67
+
68
+// AddNodeLink adds a link to another node. without keeping a reference to
69
+// the child node
70
+func (n *Node) AddNodeLinkClean(name string, that *Node) error {
71
s, err := that.Size()
72
if err != nil {
73
return err
@@ -68,7 +82,6 @@ func (n *Node) AddNodeLink(name string, that *Node) error {
82
Name: name,
83
Size: s,
84
Hash: h,
71
- Node: that,
85
})
86
return nil
87
}
@@ -83,6 +96,8 @@ func (n *Node) RemoveNodeLink(name string) error {
96
return u.ErrNotFound
97
}
98
99
+// Copy returns a copy of the node.
100
+// NOTE: does not make copies of Node objects in the links.
101
func (n *Node) Copy() *Node {
102
nnode := new(Node)
103
nnode.Data = make([]byte, len(n.Data))
@@ -108,25 +123,6 @@ func (n *Node) Size() (uint64, error) {
123
return s, nil
124
}
125
111
-func (n *Node) DataSize() (uint64, error) {
112
- pbdata := new(PBData)
113
- err := proto.Unmarshal(n.Data, pbdata)
114
- if err != nil {
115
- return 0, err
116
- }
117
-
118
- switch pbdata.GetType() {
119
- case PBData_Directory:
120
- return 0, errors.New("Cant get data size of directory!")
121
- case PBData_File:
122
- return pbdata.GetFilesize(), nil
123
- case PBData_Raw:
124
- return uint64(len(pbdata.GetData())), nil
125
- default:
126
- return 0, errors.New("Unrecognized node data type!")
127
- }
128
-}
129
-
126
// Multihash hashes the encoded data of this node.
127
func (n *Node) Multihash() (mh.Multihash, error) {
128
b, err := n.Encoded(false)
@@ -230,46 +226,3 @@ func (n *DAGService) Get(k u.Key) (*Node, error) {
226
227
return Decoded(b.Data)
228
}
233
-
234
-func FilePBData(data []byte, totalsize uint64) []byte {
235
- pbfile := new(PBData)
236
- typ := PBData_File
237
- pbfile.Type = &typ
238
- pbfile.Data = data
239
- pbfile.Filesize = proto.Uint64(totalsize)
240
-
241
- data, err := proto.Marshal(pbfile)
242
- if err != nil {
243
- //this really shouldnt happen, i promise
244
- panic(err)
245
- }
246
- return data
247
-}
248
-
249
-func FolderPBData() []byte {
250
- pbfile := new(PBData)
251
- typ := PBData_Directory
252
- pbfile.Type = &typ
253
-
254
- data, err := proto.Marshal(pbfile)
255
- if err != nil {
256
- //this really shouldnt happen, i promise
257
- panic(err)
258
- }
259
- return data
260
-}
261
-
262
-func WrapData(b []byte) []byte {
263
- pbdata := new(PBData)
264
- typ := PBData_Raw
265
- pbdata.Data = b
266
- pbdata.Type = &typ
267
-
268
- out, err := proto.Marshal(pbdata)
269
- if err != nil {
270
- // This shouldnt happen. seriously.
271
- panic(err)
272
- }
273
-
274
- return out
275
-}
merkledag/merkledag_test.go
+2
-1
@@ -2,8 +2,9 @@ package merkledag
2
3
import (
4
"fmt"
5
- u "github.com/jbenet/go-ipfs/util"
5
"testing"
6
+
7
+ u "github.com/jbenet/go-ipfs/util"
8
)
9
10
func TestNode(t *testing.T) {