implement metadata node for unixfs and other
Jeromy committed
Feb 2, 2015 at 22:46 UTC
b79fffc96585982871edb9e85cc53cde876171c7
7 files changed
+205
-1
core/coreunix/metadata.go
new
+37
@@ -0,0 +1,37 @@
1
+package coreunix
2
+
3
+import (
4
+ dag "github.com/jbenet/go-ipfs/merkledag"
5
+ ft "github.com/jbenet/go-ipfs/unixfs"
6
+ u "github.com/jbenet/go-ipfs/util"
7
+)
8
+
9
+func AddMetadataTo(key u.Key, m *ft.Metadata, dserv dag.DAGService) (u.Key, error) {
10
+ nd, err := dserv.Get(key)
11
+ if err != nil {
12
+ return "", err
13
+ }
14
+
15
+ mdnode := new(dag.Node)
16
+ mdata, err := ft.BytesForMetadata(m)
17
+ if err != nil {
18
+ return "", err
19
+ }
20
+
21
+ mdnode.Data = mdata
22
+ err = mdnode.AddNodeLinkClean("file", nd)
23
+ if err != nil {
24
+ return "", err
25
+ }
26
+
27
+ return dserv.Add(mdnode)
28
+}
29
+
30
+func Metadata(key u.Key, dserv dag.DAGService) (*ft.Metadata, error) {
31
+ nd, err := dserv.Get(key)
32
+ if err != nil {
33
+ return nil, err
34
+ }
35
+
36
+ return ft.MetadataFromBytes(nd.Data)
37
+}
core/coreunix/metadata_test.go
new
+82
@@ -0,0 +1,82 @@
1
+package coreunix
2
+
3
+import (
4
+ "bytes"
5
+ "io/ioutil"
6
+ "testing"
7
+
8
+ context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
9
+ ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
10
+ dssync "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync"
11
+ bstore "github.com/jbenet/go-ipfs/blocks/blockstore"
12
+ bserv "github.com/jbenet/go-ipfs/blockservice"
13
+ offline "github.com/jbenet/go-ipfs/exchange/offline"
14
+ importer "github.com/jbenet/go-ipfs/importer"
15
+ chunk "github.com/jbenet/go-ipfs/importer/chunk"
16
+ merkledag "github.com/jbenet/go-ipfs/merkledag"
17
+ ft "github.com/jbenet/go-ipfs/unixfs"
18
+ uio "github.com/jbenet/go-ipfs/unixfs/io"
19
+ u "github.com/jbenet/go-ipfs/util"
20
+)
21
+
22
+func getDagservAndPinner(t *testing.T) merkledag.DAGService {
23
+ db := dssync.MutexWrap(ds.NewMapDatastore())
24
+ bs := bstore.NewBlockstore(db)
25
+ blockserv, err := bserv.New(bs, offline.Exchange(bs))
26
+ if err != nil {
27
+ t.Fatal(err)
28
+ }
29
+ return merkledag.NewDAGService(blockserv)
30
+}
31
+
32
+func TestMetadata(t *testing.T) {
33
+ // Make some random node
34
+ ds := getDagservAndPinner(t)
35
+ data := make([]byte, 1000)
36
+ u.NewTimeSeededRand().Read(data)
37
+ r := bytes.NewReader(data)
38
+ nd, err := importer.BuildDagFromReader(r, ds, nil, chunk.DefaultSplitter)
39
+ if err != nil {
40
+ t.Fatal(err)
41
+ }
42
+
43
+ k, err := nd.Key()
44
+ if err != nil {
45
+ t.Fatal(err)
46
+ }
47
+
48
+ m := new(ft.Metadata)
49
+ m.MimeType = "THIS IS A TEST"
50
+
51
+ mdk, err := AddMetadataTo(k, m, ds)
52
+ if err != nil {
53
+ t.Fatal(err)
54
+ }
55
+
56
+ rec, err := Metadata(mdk, ds)
57
+ if err != nil {
58
+ t.Fatal(err)
59
+ }
60
+ if rec.MimeType != m.MimeType {
61
+ t.Fatalf("something went wrong in conversion: '%s' != '%s'", rec.MimeType, m.MimeType)
62
+ }
63
+
64
+ retnode, err := ds.Get(mdk)
65
+ if err != nil {
66
+ t.Fatal(err)
67
+ }
68
+
69
+ ndr, err := uio.NewDagReader(context.TODO(), retnode, ds)
70
+ if err != nil {
71
+ t.Fatal(err)
72
+ }
73
+
74
+ out, err := ioutil.ReadAll(ndr)
75
+ if err != nil {
76
+ t.Fatal(err)
77
+ }
78
+
79
+ if !bytes.Equal(out, data) {
80
+ t.Fatal("read incorrect data")
81
+ }
82
+}
unixfs/format.go
+45
@@ -126,3 +126,48 @@ func (mb *MultiBlock) FileSize() uint64 {
126
func (mb *MultiBlock) NumChildren() int {
127
return len(mb.blocksizes)
128
}
129
+
130
+type Metadata struct {
131
+ MimeType string
132
+ Size uint64
133
+}
134
+
135
+func MetadataFromBytes(b []byte) (*Metadata, error) {
136
+ pbd := new(pb.Data)
137
+ err := proto.Unmarshal(b, pbd)
138
+ if err != nil {
139
+ return nil, err
140
+ }
141
+ if pbd.GetType() != pb.Data_Metadata {
142
+ return nil, errors.New("incorrect node type")
143
+ }
144
+
145
+ pbm := new(pb.Metadata)
146
+ err = proto.Unmarshal(pbd.Data, pbm)
147
+ if err != nil {
148
+ return nil, err
149
+ }
150
+ md := new(Metadata)
151
+ md.MimeType = pbm.GetMimeType()
152
+ return md, nil
153
+}
154
+
155
+func (m *Metadata) Bytes() ([]byte, error) {
156
+ pbm := new(pb.Metadata)
157
+ pbm.MimeType = &m.MimeType
158
+ return proto.Marshal(pbm)
159
+}
160
+
161
+func BytesForMetadata(m *Metadata) ([]byte, error) {
162
+ pbd := new(pb.Data)
163
+ pbd.Filesize = proto.Uint64(m.Size)
164
+ typ := pb.Data_Metadata
165
+ pbd.Type = &typ
166
+ mdd, err := m.Bytes()
167
+ if err != nil {
168
+ return nil, err
169
+ }
170
+
171
+ pbd.Data = mdd
172
+ return proto.Marshal(pbd)
173
+}
unixfs/io/dagreader.go
+11
@@ -70,6 +70,15 @@ func NewDagReader(ctx context.Context, n *mdag.Node, serv mdag.DAGService) (*Dag
70
fallthrough
71
case ftpb.Data_File:
72
return newDataFileReader(ctx, n, pb, serv), nil
73
+ case ftpb.Data_Metadata:
74
+ if len(n.Links) == 0 {
75
+ return nil, errors.New("incorrectly formatted metadata object")
76
+ }
77
+ child, err := n.Links[0].GetNode(serv)
78
+ if err != nil {
79
+ return nil, err
80
+ }
81
+ return NewDagReader(ctx, child, serv)
82
default:
83
return nil, ft.ErrUnrecognizedType
84
}
@@ -118,6 +127,8 @@ func (dr *DagReader) precalcNextBuf() error {
127
case ftpb.Data_Raw:
128
dr.buf = NewRSNCFromBytes(pb.GetData())
129
return nil
130
+ case ftpb.Data_Metadata:
131
+ return errors.New("Shouldnt have had metadata object inside file")
132
default:
133
return ft.ErrUnrecognizedType
134
}
unixfs/pb/unixfs.pb.go
+21
-1
@@ -10,10 +10,11 @@ It is generated from these files:
10
11
It has these top-level messages:
12
Data
13
+ Metadata
14
*/
15
package unixfs_pb
16
16
-import proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/gogoprotobuf/proto"
17
+import proto "code.google.com/p/gogoprotobuf/proto"
18
import math "math"
19
20
// Reference imports to suppress errors if they are not otherwise used.
@@ -26,17 +27,20 @@ const (
27
Data_Raw Data_DataType = 0
28
Data_Directory Data_DataType = 1
29
Data_File Data_DataType = 2
30
+ Data_Metadata Data_DataType = 3
31
)
32
33
var Data_DataType_name = map[int32]string{
34
0: "Raw",
35
1: "Directory",
36
2: "File",
37
+ 3: "Metadata",
38
}
39
var Data_DataType_value = map[string]int32{
40
"Raw": 0,
41
"Directory": 1,
42
"File": 2,
43
+ "Metadata": 3,
44
}
45
46
func (x Data_DataType) Enum() *Data_DataType {
@@ -96,6 +100,22 @@ func (m *Data) GetBlocksizes() []uint64 {
100
return nil
101
}
102
103
+type Metadata struct {
104
+ MimeType *string `protobuf:"bytes,1,req" json:"MimeType,omitempty"`
105
+ XXX_unrecognized []byte `json:"-"`
106
+}
107
+
108
+func (m *Metadata) Reset() { *m = Metadata{} }
109
+func (m *Metadata) String() string { return proto.CompactTextString(m) }
110
+func (*Metadata) ProtoMessage() {}
111
+
112
+func (m *Metadata) GetMimeType() string {
113
+ if m != nil && m.MimeType != nil {
114
+ return *m.MimeType
115
+ }
116
+ return ""
117
+}
118
+
119
func init() {
120
proto.RegisterEnum("unixfs.pb.Data_DataType", Data_DataType_name, Data_DataType_value)
121
}
unixfs/pb/unixfs.proto
+5
@@ -5,6 +5,7 @@ message Data {
5
Raw = 0;
6
Directory = 1;
7
File = 2;
8
+ Metadata = 3;
9
}
10
11
required DataType Type = 1;
@@ -12,3 +13,7 @@ message Data {
13
optional uint64 filesize = 3;
14
repeated uint64 blocksizes = 4;
15
}
16
+
17
+message Metadata {
18
+ required string MimeType = 1;
19
+}
util/key.go
+4
@@ -23,6 +23,10 @@ func (k Key) Pretty() string {
23
return k.B58String()
24
}
25
26
+func (k Key) ToMultihash() mh.Multihash {
27
+ return mh.Multihash(k)
28
+}
29
+
30
// B58String returns Key in a b58 encoded string
31
func (k Key) B58String() string {
32
return B58KeyEncode(k)