fixed data size reporting
Jeromy committed
Oct 5, 2014 at 01:22 UTC
a13baff33d6a856fce02f939a6c4ade349884869
7 files changed
+69
-41
fuse/ipns/ipns_test.go
+21
-10
@@ -18,7 +18,7 @@ func randBytes(size int) []byte {
18
return b
19
}
20
21
-func writeFile(t *testing.T, size int, path string) ([]byte, error) {
21
+func writeFile(t *testing.T, size int, path string) []byte {
22
data := randBytes(size)
23
fi, err := os.Create(path)
24
if err != nil {
@@ -39,7 +39,7 @@ func writeFile(t *testing.T, size int, path string) ([]byte, error) {
39
t.Fatal(err)
40
}
41
42
- return data, nil
42
+ return data
43
}
44
45
func setupIpnsTest(t *testing.T, node *core.IpfsNode) (*core.IpfsNode, *fstest.Mount) {
@@ -68,10 +68,7 @@ func TestIpnsBasicIO(t *testing.T) {
68
defer mnt.Close()
69
70
fname := mnt.Dir + "/local/testfile"
71
- data, err := writeFile(t, 12345, fname)
72
- if err != nil {
73
- t.Fatal(err)
74
- }
71
+ data := writeFile(t, 12345, fname)
72
73
rbuf, err := ioutil.ReadFile(fname)
74
if err != nil {
@@ -87,10 +84,7 @@ func TestFilePersistence(t *testing.T) {
84
node, mnt := setupIpnsTest(t, nil)
85
86
fname := "/local/atestfile"
90
- data, err := writeFile(t, 127, mnt.Dir+fname)
91
- if err != nil {
92
- t.Fatal(err)
93
- }
87
+ data := writeFile(t, 127, mnt.Dir+fname)
88
89
// Wait for publish: TODO: make publish happen faster in tests
90
time.Sleep(time.Millisecond * 40)
@@ -109,3 +103,20 @@ func TestFilePersistence(t *testing.T) {
103
t.Fatalf("File data changed between mounts! sizes differ: %d != %d", len(data), len(rbuf))
104
}
105
}
106
+
107
+func TestFileSizeReporting(t *testing.T) {
108
+ _, mnt := setupIpnsTest(t, nil)
109
+ defer mnt.Close()
110
+
111
+ fname := mnt.Dir + "/local/sizecheck"
112
+ data := writeFile(t, 5555, fname)
113
+
114
+ finfo, err := os.Stat(fname)
115
+ if err != nil {
116
+ t.Fatal(err)
117
+ }
118
+
119
+ if finfo.Size() != int64(len(data)) {
120
+ t.Fatal("Read incorrect size from stat!")
121
+ }
122
+}
fuse/ipns/ipns_unix.go
+7
-4
@@ -219,10 +219,14 @@ func (s *Node) Attr() fuse.Attr {
219
case mdag.PBData_Directory:
220
return fuse.Attr{Mode: os.ModeDir | 0555}
221
case mdag.PBData_File, mdag.PBData_Raw:
222
- size, _ := s.Nd.Size()
222
+ size, err := s.Nd.DataSize()
223
+ if err != nil {
224
+ log.Error("Error getting size of file: %s", err)
225
+ size = 0
226
+ }
227
return fuse.Attr{
228
Mode: 0666,
225
- Size: uint64(size),
229
+ Size: size,
230
Blocks: uint64(len(s.Nd.Links)),
231
}
232
default:
@@ -323,7 +327,6 @@ func (n *Node) Flush(req *fuse.FlushRequest, intr fs.Intr) fuse.Error {
327
// folder, bad things would happen.
328
buf := bytes.NewReader(n.writerBuf.Bytes())
329
newNode, err := imp.NewDagFromReader(buf)
326
- log.Debug("flush: new data = %v", newNode.Data)
330
if err != nil {
331
log.Critical("error creating dag from writerBuf: %s", err)
332
return err
@@ -457,7 +460,7 @@ func (n *Node) Create(req *fuse.CreateRequest, resp *fuse.CreateResponse, intr f
460
log.Debug("Got create request: %s", req.Name)
461
462
// New 'empty' file
460
- nd := &mdag.Node{Data: mdag.FilePBData(nil)}
463
+ nd := &mdag.Node{Data: mdag.FilePBData(nil, 0)}
464
child := n.makeChild(req.Name, nd)
465
466
nnode := n.Nd.Copy()
importer/importer.go
+4
-22
@@ -28,10 +28,12 @@ func NewDagFromReader(r io.Reader) (*dag.Node, error) {
28
func NewDagFromReaderWithSplitter(r io.Reader, spl BlockSplitter) (*dag.Node, error) {
29
blkChan := spl.Split(r)
30
first := <-blkChan
31
- root := &dag.Node{Data: dag.FilePBData(first)}
31
+ root := &dag.Node{}
32
33
i := 0
34
+ totalsize := uint64(len(first))
35
for blk := range blkChan {
36
+ totalsize += uint64(len(blk))
37
child := &dag.Node{Data: dag.WrapData(blk)}
38
err := root.AddNodeLink(fmt.Sprintf("%d", i), child)
39
if err != nil {
@@ -40,6 +42,7 @@ func NewDagFromReaderWithSplitter(r io.Reader, spl BlockSplitter) (*dag.Node, er
42
i++
43
}
44
45
+ root.Data = dag.FilePBData(first, totalsize)
46
return root, nil
47
}
48
@@ -62,24 +65,3 @@ func NewDagFromFile(fpath string) (*dag.Node, error) {
65
66
return NewDagFromReader(f)
67
}
65
-
66
-// TODO: this needs a better name
67
-func NewDagInNode(r io.Reader, n *dag.Node) error {
68
- n.Links = nil
69
-
70
- blkChan := DefaultSplitter.Split(r)
71
- first := <-blkChan
72
- n.Data = dag.FilePBData(first)
73
-
74
- i := 0
75
- for blk := range blkChan {
76
- child := &dag.Node{Data: dag.WrapData(blk)}
77
- err := n.AddNodeLink(fmt.Sprintf("%d", i), child)
78
- if err != nil {
79
- return err
80
- }
81
- i++
82
- }
83
-
84
- return nil
85
-}
merkledag/data.pb.go
+9
-1
@@ -13,7 +13,7 @@ It has these top-level messages:
13
*/
14
package merkledag
15
16
-import proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
16
+import proto "code.google.com/p/goprotobuf/proto"
17
import math "math"
18
19
// Reference imports to suppress errors if they are not otherwise used.
@@ -59,6 +59,7 @@ func (x *PBData_DataType) UnmarshalJSON(data []byte) error {
59
type PBData struct {
60
Type *PBData_DataType `protobuf:"varint,1,req,enum=merkledag.PBData_DataType" json:"Type,omitempty"`
61
Data []byte `protobuf:"bytes,2,opt" json:"Data,omitempty"`
62
+ Filesize *uint64 `protobuf:"varint,3,opt,name=filesize" json:"filesize,omitempty"`
63
XXX_unrecognized []byte `json:"-"`
64
}
65
@@ -80,6 +81,13 @@ func (m *PBData) GetData() []byte {
81
return nil
82
}
83
84
+func (m *PBData) GetFilesize() uint64 {
85
+ if m != nil && m.Filesize != nil {
86
+ return *m.Filesize
87
+ }
88
+ return 0
89
+}
90
+
91
func init() {
92
proto.RegisterEnum("merkledag.PBData_DataType", PBData_DataType_name, PBData_DataType_value)
93
}
merkledag/data.proto
+1
@@ -9,4 +9,5 @@ message PBData {
9
10
required DataType Type = 1;
11
optional bytes Data = 2;
12
+ optional uint64 filesize = 3;
13
}
merkledag/merkledag.go
+22
-1
@@ -1,6 +1,7 @@
1
package merkledag
2
3
import (
4
+ "errors"
5
"fmt"
6
7
proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
@@ -107,6 +108,25 @@ func (n *Node) Size() (uint64, error) {
108
return s, nil
109
}
110
111
+func (n *Node) DataSize() (uint64, error) {
112
+ pbdata := new(PBData)
113
+ err := proto.Unmarshal(n.Data, pbdata)
114
+ if err != nil {
115
+ return 0, err
116
+ }
117
+
118
+ switch pbdata.GetType() {
119
+ case PBData_Directory:
120
+ return 0, errors.New("Cant get data size of directory!")
121
+ case PBData_File:
122
+ return pbdata.GetFilesize(), nil
123
+ case PBData_Raw:
124
+ return uint64(len(pbdata.GetData())), nil
125
+ default:
126
+ return 0, errors.New("Unrecognized node data type!")
127
+ }
128
+}
129
+
130
// Multihash hashes the encoded data of this node.
131
func (n *Node) Multihash() (mh.Multihash, error) {
132
b, err := n.Encoded(false)
@@ -211,11 +231,12 @@ func (n *DAGService) Get(k u.Key) (*Node, error) {
231
return Decoded(b.Data)
232
}
233
214
-func FilePBData(data []byte) []byte {
234
+func FilePBData(data []byte, totalsize uint64) []byte {
235
pbfile := new(PBData)
236
typ := PBData_File
237
pbfile.Type = &typ
238
pbfile.Data = data
239
+ pbfile.Filesize = proto.Uint64(totalsize)
240
241
data, err := proto.Marshal(pbfile)
242
if err != nil {
util/util.go
+5
-3
@@ -122,9 +122,11 @@ func SetupLogging() {
122
*/
123
logging.SetFormatter(logging.MustStringFormatter(LogFormat))
124
125
- for _, n := range loggers {
126
- logging.SetLevel(logging.ERROR, n)
127
- }
125
+ /*
126
+ for _, n := range loggers {
127
+ logging.SetLevel(logging.ERROR, n)
128
+ }
129
+ */
130
}
131
132
// Logger retrieves a particular logger + initializes it at a particular level