@cryptotaxi247 / kubo / commits / c62b039e2

delete unixfs code...

License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com>

Jeromy committed Jul 31, 2018 at 01:13 UTC c62b039e2b759251c38db4dc287d34ccc684fc91
32 files changed -7603
Rules.mk
-3
@@ -47,9 +47,6 @@ ifneq ($(filter coverage% clean distclean,$(MAKECMDGOALS)),)
47 include $(dir)/Rules.mk
48 endif
49
50 -dir := unixfs/pb
51 -include $(dir)/Rules.mk
52 -
50 dir := pin/internal/pb
51 include $(dir)/Rules.mk
52
unixfs/archive/archive.go deleted
-108
@@ -1,108 +0,0 @@
1 -// Package archive provides utilities to archive and compress a [Unixfs] DAG.
2 -package archive
3 -
4 -import (
5 - "bufio"
6 - "compress/gzip"
7 - "context"
8 - "io"
9 - "path"
10 -
11 - tar "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/archive/tar"
12 - uio "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/io"
13 -
14 - ipld "gx/ipfs/QmZtNq8dArGfnpCZfx2pUNY7UcjGhVp5qqwQ4hH6mpTMRQ/go-ipld-format"
15 -)
16 -
17 -// DefaultBufSize is the buffer size for gets. for now, 1MB, which is ~4 blocks.
18 -// TODO: does this need to be configurable?
19 -var DefaultBufSize = 1048576
20 -
21 -type identityWriteCloser struct {
22 - w io.Writer
23 -}
24 -
25 -func (i *identityWriteCloser) Write(p []byte) (int, error) {
26 - return i.w.Write(p)
27 -}
28 -
29 -func (i *identityWriteCloser) Close() error {
30 - return nil
31 -}
32 -
33 -// DagArchive is equivalent to `ipfs getdag $hash | maybe_tar | maybe_gzip`
34 -func DagArchive(ctx context.Context, nd ipld.Node, name string, dag ipld.DAGService, archive bool, compression int) (io.Reader, error) {
35 -
36 - cleaned := path.Clean(name)
37 - _, filename := path.Split(cleaned)
38 -
39 - // need to connect a writer to a reader
40 - piper, pipew := io.Pipe()
41 - checkErrAndClosePipe := func(err error) bool {
42 - if err != nil {
43 - pipew.CloseWithError(err)
44 - return true
45 - }
46 - return false
47 - }
48 -
49 - // use a buffered writer to parallelize task
50 - bufw := bufio.NewWriterSize(pipew, DefaultBufSize)
51 -
52 - // compression determines whether to use gzip compression.
53 - maybeGzw, err := newMaybeGzWriter(bufw, compression)
54 - if checkErrAndClosePipe(err) {
55 - return nil, err
56 - }
57 -
58 - closeGzwAndPipe := func() {
59 - if err := maybeGzw.Close(); checkErrAndClosePipe(err) {
60 - return
61 - }
62 - if err := bufw.Flush(); checkErrAndClosePipe(err) {
63 - return
64 - }
65 - pipew.Close() // everything seems to be ok.
66 - }
67 -
68 - if !archive && compression != gzip.NoCompression {
69 - // the case when the node is a file
70 - dagr, err := uio.NewDagReader(ctx, nd, dag)
71 - if checkErrAndClosePipe(err) {
72 - return nil, err
73 - }
74 -
75 - go func() {
76 - if _, err := dagr.WriteTo(maybeGzw); checkErrAndClosePipe(err) {
77 - return
78 - }
79 - closeGzwAndPipe() // everything seems to be ok
80 - }()
81 - } else {
82 - // the case for 1. archive, and 2. not archived and not compressed, in which tar is used anyway as a transport format
83 -
84 - // construct the tar writer
85 - w, err := tar.NewWriter(ctx, dag, maybeGzw)
86 - if checkErrAndClosePipe(err) {
87 - return nil, err
88 - }
89 -
90 - go func() {
91 - // write all the nodes recursively
92 - if err := w.WriteNode(nd, filename); checkErrAndClosePipe(err) {
93 - return
94 - }
95 - w.Close() // close tar writer
96 - closeGzwAndPipe() // everything seems to be ok
97 - }()
98 - }
99 -
100 - return piper, nil
101 -}
102 -
103 -func newMaybeGzWriter(w io.Writer, compression int) (io.WriteCloser, error) {
104 - if compression != gzip.NoCompression {
105 - return gzip.NewWriterLevel(w, compression)
106 - }
107 - return &identityWriteCloser{w}, nil
108 -}
unixfs/archive/tar/writer.go deleted
-143
@@ -1,143 +0,0 @@
1 -// Package tar provides functionality to write a unixfs merkledag
2 -// as a tar archive.
3 -package tar
4 -
5 -import (
6 - "archive/tar"
7 - "context"
8 - "fmt"
9 - "io"
10 - "path"
11 - "time"
12 -
13 - mdag "gx/ipfs/QmRy4Qk9hbgFX9NGJRm8rBThrA8PZhNCitMgeRYyZ67s59/go-merkledag"
14 - ft "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs"
15 - uio "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/io"
16 - upb "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/pb"
17 -
18 - ipld "gx/ipfs/QmZtNq8dArGfnpCZfx2pUNY7UcjGhVp5qqwQ4hH6mpTMRQ/go-ipld-format"
19 -)
20 -
21 -// Writer is a utility structure that helps to write
22 -// unixfs merkledag nodes as a tar archive format.
23 -// It wraps any io.Writer.
24 -type Writer struct {
25 - Dag ipld.DAGService
26 - TarW *tar.Writer
27 -
28 - ctx context.Context
29 -}
30 -
31 -// NewWriter wraps given io.Writer.
32 -func NewWriter(ctx context.Context, dag ipld.DAGService, w io.Writer) (*Writer, error) {
33 - return &Writer{
34 - Dag: dag,
35 - TarW: tar.NewWriter(w),
36 - ctx: ctx,
37 - }, nil
38 -}
39 -
40 -func (w *Writer) writeDir(nd *mdag.ProtoNode, fpath string) error {
41 - dir, err := uio.NewDirectoryFromNode(w.Dag, nd)
42 - if err != nil {
43 - return err
44 - }
45 - if err := writeDirHeader(w.TarW, fpath); err != nil {
46 - return err
47 - }
48 -
49 - return dir.ForEachLink(w.ctx, func(l *ipld.Link) error {
50 - child, err := w.Dag.Get(w.ctx, l.Cid)
51 - if err != nil {
52 - return err
53 - }
54 - npath := path.Join(fpath, l.Name)
55 - return w.WriteNode(child, npath)
56 - })
57 -}
58 -
59 -func (w *Writer) writeFile(nd *mdag.ProtoNode, fsNode *ft.FSNode, fpath string) error {
60 - if err := writeFileHeader(w.TarW, fpath, fsNode.FileSize()); err != nil {
61 - return err
62 - }
63 -
64 - dagr := uio.NewPBFileReader(w.ctx, nd, fsNode, w.Dag)
65 - if _, err := dagr.WriteTo(w.TarW); err != nil {
66 - return err
67 - }
68 - w.TarW.Flush()
69 - return nil
70 -}
71 -
72 -// WriteNode adds a node to the archive.
73 -func (w *Writer) WriteNode(nd ipld.Node, fpath string) error {
74 - switch nd := nd.(type) {
75 - case *mdag.ProtoNode:
76 - fsNode, err := ft.FSNodeFromBytes(nd.Data())
77 - if err != nil {
78 - return err
79 - }
80 -
81 - switch fsNode.Type() {
82 - case upb.Data_Metadata:
83 - fallthrough
84 - case upb.Data_Directory, upb.Data_HAMTShard:
85 - return w.writeDir(nd, fpath)
86 - case upb.Data_Raw:
87 - fallthrough
88 - case upb.Data_File:
89 - return w.writeFile(nd, fsNode, fpath)
90 - case upb.Data_Symlink:
91 - return writeSymlinkHeader(w.TarW, string(fsNode.Data()), fpath)
92 - default:
93 - return ft.ErrUnrecognizedType
94 - }
95 - case *mdag.RawNode:
96 - if err := writeFileHeader(w.TarW, fpath, uint64(len(nd.RawData()))); err != nil {
97 - return err
98 - }
99 -
100 - if _, err := w.TarW.Write(nd.RawData()); err != nil {
101 - return err
102 - }
103 - w.TarW.Flush()
104 - return nil
105 - default:
106 - return fmt.Errorf("nodes of type %T are not supported in unixfs", nd)
107 - }
108 -}
109 -
110 -// Close closes the tar writer.
111 -func (w *Writer) Close() error {
112 - return w.TarW.Close()
113 -}
114 -
115 -func writeDirHeader(w *tar.Writer, fpath string) error {
116 - return w.WriteHeader(&tar.Header{
117 - Name: fpath,
118 - Typeflag: tar.TypeDir,
119 - Mode: 0777,
120 - ModTime: time.Now(),
121 - // TODO: set mode, dates, etc. when added to unixFS
122 - })
123 -}
124 -
125 -func writeFileHeader(w *tar.Writer, fpath string, size uint64) error {
126 - return w.WriteHeader(&tar.Header{
127 - Name: fpath,
128 - Size: int64(size),
129 - Typeflag: tar.TypeReg,
130 - Mode: 0644,
131 - ModTime: time.Now(),
132 - // TODO: set mode, dates, etc. when added to unixFS
133 - })
134 -}
135 -
136 -func writeSymlinkHeader(w *tar.Writer, target, fpath string) error {
137 - return w.WriteHeader(&tar.Header{
138 - Name: fpath,
139 - Linkname: target,
140 - Mode: 0777,
141 - Typeflag: tar.TypeSymlink,
142 - })
143 -}
unixfs/hamt/hamt.go deleted
-530
@@ -1,530 +0,0 @@
1 -// Package hamt implements a Hash Array Mapped Trie over ipfs merkledag nodes.
2 -// It is implemented mostly as described in the wikipedia article on HAMTs,
3 -// however the table size is variable (usually 256 in our usages) as opposed to
4 -// 32 as suggested in the article. The hash function used is currently
5 -// Murmur3, but this value is configurable (the datastructure reports which
6 -// hash function its using).
7 -//
8 -// The one algorithmic change we implement that is not mentioned in the
9 -// wikipedia article is the collapsing of empty shards.
10 -// Given the following tree: ( '[' = shards, '{' = values )
11 -// [ 'A' ] -> [ 'B' ] -> { "ABC" }
12 -// | L-> { "ABD" }
13 -// L-> { "ASDF" }
14 -// If we simply removed "ABC", we would end up with a tree where shard 'B' only
15 -// has a single child. This causes two issues, the first, is that now we have
16 -// an extra lookup required to get to "ABD". The second issue is that now we
17 -// have a tree that contains only "ABD", but is not the same tree that we would
18 -// get by simply inserting "ABD" into a new tree. To address this, we always
19 -// check for empty shard nodes upon deletion and prune them to maintain a
20 -// consistent tree, independent of insertion order.
21 -package hamt
22 -
23 -import (
24 - "context"
25 - "fmt"
26 - "os"
27 -
28 - dag "gx/ipfs/QmRy4Qk9hbgFX9NGJRm8rBThrA8PZhNCitMgeRYyZ67s59/go-merkledag"
29 - format "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs"
30 - upb "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/pb"
31 -
32 - bitfield "gx/ipfs/QmTbBs3Y3u5F69XNJzdnnc6SP5GKgcXxCDzx6w8m6piVRT/go-bitfield"
33 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
34 - proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
35 - ipld "gx/ipfs/QmZtNq8dArGfnpCZfx2pUNY7UcjGhVp5qqwQ4hH6mpTMRQ/go-ipld-format"
36 - "gx/ipfs/QmfJHywXQu98UeZtGJBQrPAR6AtmDjjbe3qjTo9piXHPnx/murmur3"
37 -)
38 -
39 -const (
40 - // HashMurmur3 is the multiformats identifier for Murmur3
41 - HashMurmur3 uint64 = 0x22
42 -)
43 -
44 -// A Shard represents the HAMT. It should be initialized with NewShard().
45 -type Shard struct {
46 - nd *dag.ProtoNode
47 -
48 - bitfield bitfield.Bitfield
49 -
50 - children []child
51 -
52 - tableSize int
53 - tableSizeLg2 int
54 -
55 - prefix *cid.Prefix
56 - hashFunc uint64
57 -
58 - prefixPadStr string
59 - maxpadlen int
60 -
61 - dserv ipld.DAGService
62 -}
63 -
64 -// child can either be another shard, or a leaf node value
65 -type child interface {
66 - Link() (*ipld.Link, error)
67 - Label() string
68 -}
69 -
70 -// NewShard creates a new, empty HAMT shard with the given size.
71 -func NewShard(dserv ipld.DAGService, size int) (*Shard, error) {
72 - ds, err := makeShard(dserv, size)
73 - if err != nil {
74 - return nil, err
75 - }
76 -
77 - ds.nd = new(dag.ProtoNode)
78 - ds.hashFunc = HashMurmur3
79 - return ds, nil
80 -}
81 -
82 -func makeShard(ds ipld.DAGService, size int) (*Shard, error) {
83 - lg2s, err := logtwo(size)
84 - if err != nil {
85 - return nil, err
86 - }
87 - maxpadding := fmt.Sprintf("%X", size-1)
88 - return &Shard{
89 - tableSizeLg2: lg2s,
90 - prefixPadStr: fmt.Sprintf("%%0%dX", len(maxpadding)),
91 - maxpadlen: len(maxpadding),
92 - bitfield: bitfield.NewBitfield(size),
93 - tableSize: size,
94 - dserv: ds,
95 - }, nil
96 -}
97 -
98 -// NewHamtFromDag creates new a HAMT shard from the given DAG.
99 -func NewHamtFromDag(dserv ipld.DAGService, nd ipld.Node) (*Shard, error) {
100 - pbnd, ok := nd.(*dag.ProtoNode)
101 - if !ok {
102 - return nil, dag.ErrNotProtobuf
103 - }
104 -
105 - pbd, err := format.FromBytes(pbnd.Data())
106 - if err != nil {
107 - return nil, err
108 - }
109 -
110 - if pbd.GetType() != upb.Data_HAMTShard {
111 - return nil, fmt.Errorf("node was not a dir shard")
112 - }
113 -
114 - if pbd.GetHashType() != HashMurmur3 {
115 - return nil, fmt.Errorf("only murmur3 supported as hash function")
116 - }
117 -
118 - ds, err := makeShard(dserv, int(pbd.GetFanout()))
119 - if err != nil {
120 - return nil, err
121 - }
122 -
123 - ds.nd = pbnd.Copy().(*dag.ProtoNode)
124 - ds.children = make([]child, len(pbnd.Links()))
125 - ds.bitfield.SetBytes(pbd.GetData())
126 - ds.hashFunc = pbd.GetHashType()
127 - ds.prefix = &ds.nd.Prefix
128 -
129 - return ds, nil
130 -}
131 -
132 -// SetPrefix sets the CID Prefix
133 -func (ds *Shard) SetPrefix(prefix *cid.Prefix) {
134 - ds.prefix = prefix
135 -}
136 -
137 -// Prefix gets the CID Prefix, may be nil if unset
138 -func (ds *Shard) Prefix() *cid.Prefix {
139 - return ds.prefix
140 -}
141 -
142 -// Node serializes the HAMT structure into a merkledag node with unixfs formatting
143 -func (ds *Shard) Node() (ipld.Node, error) {
144 - out := new(dag.ProtoNode)
145 - out.SetPrefix(ds.prefix)
146 -
147 - cindex := 0
148 - // TODO: optimized 'for each set bit'
149 - for i := 0; i < ds.tableSize; i++ {
150 - if !ds.bitfield.Bit(i) {
151 - continue
152 - }
153 -
154 - ch := ds.children[cindex]
155 - if ch != nil {
156 - clnk, err := ch.Link()
157 - if err != nil {
158 - return nil, err
159 - }
160 -
161 - err = out.AddRawLink(ds.linkNamePrefix(i)+ch.Label(), clnk)
162 - if err != nil {
163 - return nil, err
164 - }
165 - } else {
166 - // child unloaded, just copy in link with updated name
167 - lnk := ds.nd.Links()[cindex]
168 - label := lnk.Name[ds.maxpadlen:]
169 -
170 - err := out.AddRawLink(ds.linkNamePrefix(i)+label, lnk)
171 - if err != nil {
172 - return nil, err
173 - }
174 - }
175 - cindex++
176 - }
177 -
178 - typ := upb.Data_HAMTShard
179 - data, err := proto.Marshal(&upb.Data{
180 - Type: &typ,
181 - Fanout: proto.Uint64(uint64(ds.tableSize)),
182 - HashType: proto.Uint64(HashMurmur3),
183 - Data: ds.bitfield.Bytes(),
184 - })
185 - if err != nil {
186 - return nil, err
187 - }
188 -
189 - out.SetData(data)
190 -
191 - err = ds.dserv.Add(context.TODO(), out)
192 - if err != nil {
193 - return nil, err
194 - }
195 -
196 - return out, nil
197 -}
198 -
199 -type shardValue struct {
200 - key string
201 - val *ipld.Link
202 -}
203 -
204 -// Link returns a link to this node
205 -func (sv *shardValue) Link() (*ipld.Link, error) {
206 - return sv.val, nil
207 -}
208 -
209 -func (sv *shardValue) Label() string {
210 - return sv.key
211 -}
212 -
213 -func hash(val []byte) []byte {
214 - h := murmur3.New64()
215 - h.Write(val)
216 - return h.Sum(nil)
217 -}
218 -
219 -// Label for Shards is the empty string, this is used to differentiate them from
220 -// value entries
221 -func (ds *Shard) Label() string {
222 - return ""
223 -}
224 -
225 -// Set sets 'name' = nd in the HAMT
226 -func (ds *Shard) Set(ctx context.Context, name string, nd ipld.Node) error {
227 - hv := &hashBits{b: hash([]byte(name))}
228 - err := ds.dserv.Add(ctx, nd)
229 - if err != nil {
230 - return err
231 - }
232 -
233 - lnk, err := ipld.MakeLink(nd)
234 - if err != nil {
235 - return err
236 - }
237 - lnk.Name = ds.linkNamePrefix(0) + name
238 -
239 - return ds.modifyValue(ctx, hv, name, lnk)
240 -}
241 -
242 -// Remove deletes the named entry if it exists, this operation is idempotent.
243 -func (ds *Shard) Remove(ctx context.Context, name string) error {
244 - hv := &hashBits{b: hash([]byte(name))}
245 - return ds.modifyValue(ctx, hv, name, nil)
246 -}
247 -
248 -// Find searches for a child node by 'name' within this hamt
249 -func (ds *Shard) Find(ctx context.Context, name string) (*ipld.Link, error) {
250 - hv := &hashBits{b: hash([]byte(name))}
251 -
252 - var out *ipld.Link
253 - err := ds.getValue(ctx, hv, name, func(sv *shardValue) error {
254 - out = sv.val
255 - return nil
256 - })
257 - if err != nil {
258 - return nil, err
259 - }
260 -
261 - return out, nil
262 -}
263 -
264 -// getChild returns the i'th child of this shard. If it is cached in the
265 -// children array, it will return it from there. Otherwise, it loads the child
266 -// node from disk.
267 -func (ds *Shard) getChild(ctx context.Context, i int) (child, error) {
268 - if i >= len(ds.children) || i < 0 {
269 - return nil, fmt.Errorf("invalid index passed to getChild (likely corrupt bitfield)")
270 - }
271 -
272 - if len(ds.children) != len(ds.nd.Links()) {
273 - return nil, fmt.Errorf("inconsistent lengths between children array and Links array")
274 - }
275 -
276 - c := ds.children[i]
277 - if c != nil {
278 - return c, nil
279 - }
280 -
281 - return ds.loadChild(ctx, i)
282 -}
283 -
284 -// loadChild reads the i'th child node of this shard from disk and returns it
285 -// as a 'child' interface
286 -func (ds *Shard) loadChild(ctx context.Context, i int) (child, error) {
287 - lnk := ds.nd.Links()[i]
288 - if len(lnk.Name) < ds.maxpadlen {
289 - return nil, fmt.Errorf("invalid link name '%s'", lnk.Name)
290 - }
291 -
292 - var c child
293 - if len(lnk.Name) == ds.maxpadlen {
294 - nd, err := lnk.GetNode(ctx, ds.dserv)
295 - if err != nil {
296 - return nil, err
297 - }
298 - cds, err := NewHamtFromDag(ds.dserv, nd)
299 - if err != nil {
300 - return nil, err
301 - }
302 -
303 - c = cds
304 - } else {
305 - lnk2 := *lnk
306 - c = &shardValue{
307 - key: lnk.Name[ds.maxpadlen:],
308 - val: &lnk2,
309 - }
310 - }
311 -
312 - ds.children[i] = c
313 - return c, nil
314 -}
315 -
316 -func (ds *Shard) setChild(i int, c child) {
317 - ds.children[i] = c
318 -}
319 -
320 -// Link returns a merklelink to this shard node
321 -func (ds *Shard) Link() (*ipld.Link, error) {
322 - nd, err := ds.Node()
323 - if err != nil {
324 - return nil, err
325 - }
326 -
327 - err = ds.dserv.Add(context.TODO(), nd)
328 - if err != nil {
329 - return nil, err
330 - }
331 -
332 - return ipld.MakeLink(nd)
333 -}
334 -
335 -func (ds *Shard) insertChild(idx int, key string, lnk *ipld.Link) error {
336 - if lnk == nil {
337 - return os.ErrNotExist
338 - }
339 -
340 - i := ds.indexForBitPos(idx)
341 - ds.bitfield.SetBit(idx)
342 -
343 - lnk.Name = ds.linkNamePrefix(idx) + key
344 - sv := &shardValue{
345 - key: key,
346 - val: lnk,
347 - }
348 -
349 - ds.children = append(ds.children[:i], append([]child{sv}, ds.children[i:]...)...)
350 - ds.nd.SetLinks(append(ds.nd.Links()[:i], append([]*ipld.Link{nil}, ds.nd.Links()[i:]...)...))
351 - return nil
352 -}
353 -
354 -func (ds *Shard) rmChild(i int) error {
355 - if i < 0 || i >= len(ds.children) || i >= len(ds.nd.Links()) {
356 - return fmt.Errorf("hamt: attempted to remove child with out of range index")
357 - }
358 -
359 - copy(ds.children[i:], ds.children[i+1:])
360 - ds.children = ds.children[:len(ds.children)-1]
361 -
362 - copy(ds.nd.Links()[i:], ds.nd.Links()[i+1:])
363 - ds.nd.SetLinks(ds.nd.Links()[:len(ds.nd.Links())-1])
364 -
365 - return nil
366 -}
367 -
368 -func (ds *Shard) getValue(ctx context.Context, hv *hashBits, key string, cb func(*shardValue) error) error {
369 - idx := hv.Next(ds.tableSizeLg2)
370 - if ds.bitfield.Bit(int(idx)) {
371 - cindex := ds.indexForBitPos(idx)
372 -
373 - child, err := ds.getChild(ctx, cindex)
374 - if err != nil {
375 - return err
376 - }
377 -
378 - switch child := child.(type) {
379 - case *Shard:
380 - return child.getValue(ctx, hv, key, cb)
381 - case *shardValue:
382 - if child.key == key {
383 - return cb(child)
384 - }
385 - }
386 - }
387 -
388 - return os.ErrNotExist
389 -}
390 -
391 -// EnumLinks collects all links in the Shard.
392 -func (ds *Shard) EnumLinks(ctx context.Context) ([]*ipld.Link, error) {
393 - var links []*ipld.Link
394 - err := ds.ForEachLink(ctx, func(l *ipld.Link) error {
395 - links = append(links, l)
396 - return nil
397 - })
398 - return links, err
399 -}
400 -
401 -// ForEachLink walks the Shard and calls the given function.
402 -func (ds *Shard) ForEachLink(ctx context.Context, f func(*ipld.Link) error) error {
403 - return ds.walkTrie(ctx, func(sv *shardValue) error {
404 - lnk := sv.val
405 - lnk.Name = sv.key
406 -
407 - return f(lnk)
408 - })
409 -}
410 -
411 -func (ds *Shard) walkTrie(ctx context.Context, cb func(*shardValue) error) error {
412 - for idx := range ds.children {
413 - c, err := ds.getChild(ctx, idx)
414 - if err != nil {
415 - return err
416 - }
417 -
418 - switch c := c.(type) {
419 - case *shardValue:
420 - if err := cb(c); err != nil {
421 - return err
422 - }
423 -
424 - case *Shard:
425 - if err := c.walkTrie(ctx, cb); err != nil {
426 - return err
427 - }
428 - default:
429 - return fmt.Errorf("unexpected child type: %#v", c)
430 - }
431 - }
432 - return nil
433 -}
434 -
435 -func (ds *Shard) modifyValue(ctx context.Context, hv *hashBits, key string, val *ipld.Link) error {
436 - idx := hv.Next(ds.tableSizeLg2)
437 -
438 - if !ds.bitfield.Bit(idx) {
439 - return ds.insertChild(idx, key, val)
440 - }
441 -
442 - cindex := ds.indexForBitPos(idx)
443 -
444 - child, err := ds.getChild(ctx, cindex)
445 - if err != nil {
446 - return err
447 - }
448 -
449 - switch child := child.(type) {
450 - case *Shard:
451 - err := child.modifyValue(ctx, hv, key, val)
452 - if err != nil {
453 - return err
454 - }
455 -
456 - if val == nil {
457 - switch len(child.children) {
458 - case 0:
459 - // empty sub-shard, prune it
460 - // Note: this shouldnt normally ever happen
461 - // in the event of another implementation creates flawed
462 - // structures, this will help to normalize them.
463 - ds.bitfield.UnsetBit(idx)
464 - return ds.rmChild(cindex)
465 - case 1:
466 - nchild, ok := child.children[0].(*shardValue)
467 - if ok {
468 - // sub-shard with a single value element, collapse it
469 - ds.setChild(cindex, nchild)
470 - }
471 - return nil
472 - }
473 - }
474 -
475 - return nil
476 - case *shardValue:
477 - if child.key == key {
478 - // value modification
479 - if val == nil {
480 - ds.bitfield.UnsetBit(idx)
481 - return ds.rmChild(cindex)
482 - }
483 -
484 - child.val = val
485 - return nil
486 - }
487 -
488 - if val == nil {
489 - return os.ErrNotExist
490 - }
491 -
492 - // replace value with another shard, one level deeper
493 - ns, err := NewShard(ds.dserv, ds.tableSize)
494 - if err != nil {
495 - return err
496 - }
497 - ns.prefix = ds.prefix
498 - chhv := &hashBits{
499 - b: hash([]byte(child.key)),
500 - consumed: hv.consumed,
501 - }
502 -
503 - err = ns.modifyValue(ctx, hv, key, val)
504 - if err != nil {
505 - return err
506 - }
507 -
508 - err = ns.modifyValue(ctx, chhv, child.key, child.val)
509 - if err != nil {
510 - return err
511 - }
512 -
513 - ds.setChild(cindex, ns)
514 - return nil
515 - default:
516 - return fmt.Errorf("unexpected type for child: %#v", child)
517 - }
518 -}
519 -
520 -// indexForBitPos returns the index within the collapsed array corresponding to
521 -// the given bit in the bitset. The collapsed array contains only one entry
522 -// per bit set in the bitfield, and this function is used to map the indices.
523 -func (ds *Shard) indexForBitPos(bp int) int {
524 - return ds.bitfield.OnesBefore(bp)
525 -}
526 -
527 -// linkNamePrefix takes in the bitfield index of an entry and returns its hex prefix
528 -func (ds *Shard) linkNamePrefix(idx int) string {
529 - return fmt.Sprintf(ds.prefixPadStr, idx)
530 -}
unixfs/hamt/hamt_stress_test.go deleted
-291
@@ -1,291 +0,0 @@
1 -package hamt
2 -
3 -import (
4 - "context"
5 - "fmt"
6 - "math/rand"
7 - "os"
8 - "testing"
9 - "time"
10 -
11 - mdtest "gx/ipfs/QmRy4Qk9hbgFX9NGJRm8rBThrA8PZhNCitMgeRYyZ67s59/go-merkledag/test"
12 - ft "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs"
13 -
14 - ipld "gx/ipfs/QmZtNq8dArGfnpCZfx2pUNY7UcjGhVp5qqwQ4hH6mpTMRQ/go-ipld-format"
15 -)
16 -
17 -func getNames(prefix string, count int) []string {
18 - out := make([]string, count)
19 - for i := 0; i < count; i++ {
20 - out[i] = fmt.Sprintf("%s%d", prefix, i)
21 - }
22 - return out
23 -}
24 -
25 -const (
26 - opAdd = iota
27 - opDel
28 - opFind
29 -)
30 -
31 -type testOp struct {
32 - Op int
33 - Val string
34 -}
35 -
36 -func stringArrToSet(arr []string) map[string]bool {
37 - out := make(map[string]bool)
38 - for _, s := range arr {
39 - out[s] = true
40 - }
41 - return out
42 -}
43 -
44 -// generate two different random sets of operations to result in the same
45 -// ending directory (same set of entries at the end) and execute each of them
46 -// in turn, then compare to ensure the output is the same on each.
47 -func TestOrderConsistency(t *testing.T) {
48 - seed := time.Now().UnixNano()
49 - t.Logf("using seed = %d", seed)
50 - ds := mdtest.Mock()
51 -
52 - shardWidth := 1024
53 -
54 - keep := getNames("good", 4000)
55 - temp := getNames("tempo", 6000)
56 -
57 - ops := genOpSet(seed, keep, temp)
58 - s, err := executeOpSet(t, ds, shardWidth, ops)
59 - if err != nil {
60 - t.Fatal(err)
61 - }
62 -
63 - err = validateOpSetCompletion(t, s, keep, temp)
64 - if err != nil {
65 - t.Fatal(err)
66 - }
67 -
68 - ops2 := genOpSet(seed+1000, keep, temp)
69 - s2, err := executeOpSet(t, ds, shardWidth, ops2)
70 - if err != nil {
71 - t.Fatal(err)
72 - }
73 -
74 - err = validateOpSetCompletion(t, s2, keep, temp)
75 - if err != nil {
76 - t.Fatal(err)
77 - }
78 -
79 - nd, err := s.Node()
80 - if err != nil {
81 - t.Fatal(err)
82 - }
83 -
84 - nd2, err := s2.Node()
85 - if err != nil {
86 - t.Fatal(err)
87 - }
88 -
89 - k := nd.Cid()
90 - k2 := nd2.Cid()
91 -
92 - if !k.Equals(k2) {
93 - t.Fatal("got different results: ", k, k2)
94 - }
95 -}
96 -
97 -func validateOpSetCompletion(t *testing.T, s *Shard, keep, temp []string) error {
98 - ctx := context.TODO()
99 - for _, n := range keep {
100 - _, err := s.Find(ctx, n)
101 - if err != nil {
102 - return fmt.Errorf("couldnt find %s: %s", n, err)
103 - }
104 - }
105 -
106 - for _, n := range temp {
107 - _, err := s.Find(ctx, n)
108 - if err != os.ErrNotExist {
109 - return fmt.Errorf("expected not to find: %s", err)
110 - }
111 - }
112 -
113 - return nil
114 -}
115 -
116 -func executeOpSet(t *testing.T, ds ipld.DAGService, width int, ops []testOp) (*Shard, error) {
117 - ctx := context.TODO()
118 - s, err := NewShard(ds, width)
119 - if err != nil {
120 - return nil, err
121 - }
122 -
123 - e := ft.EmptyDirNode()
124 - ds.Add(ctx, e)
125 -
126 - for _, o := range ops {
127 - switch o.Op {
128 - case opAdd:
129 - err := s.Set(ctx, o.Val, e)
130 - if err != nil {
131 - return nil, fmt.Errorf("inserting %s: %s", o.Val, err)
132 - }
133 - case opDel:
134 - err := s.Remove(ctx, o.Val)
135 - if err != nil {
136 - return nil, fmt.Errorf("deleting %s: %s", o.Val, err)
137 - }
138 - case opFind:
139 - _, err := s.Find(ctx, o.Val)
140 - if err != nil {
141 - return nil, fmt.Errorf("finding %s: %s", o.Val, err)
142 - }
143 - }
144 - }
145 -
146 - return s, nil
147 -}
148 -
149 -func genOpSet(seed int64, keep, temp []string) []testOp {
150 - tempset := stringArrToSet(temp)
151 -
152 - allnames := append(keep, temp...)
153 - shuffle(seed, allnames)
154 -
155 - var todel []string
156 -
157 - var ops []testOp
158 -
159 - for {
160 - n := len(allnames) + len(todel)
161 - if n == 0 {
162 - return ops
163 - }
164 -
165 - rn := rand.Intn(n)
166 -
167 - if rn < len(allnames) {
168 - next := allnames[0]
169 - allnames = allnames[1:]
170 - ops = append(ops, testOp{
171 - Op: opAdd,
172 - Val: next,
173 - })
174 -
175 - if tempset[next] {
176 - todel = append(todel, next)
177 - }
178 - } else {
179 - shuffle(seed+100, todel)
180 - next := todel[0]
181 - todel = todel[1:]
182 -
183 - ops = append(ops, testOp{
184 - Op: opDel,
185 - Val: next,
186 - })
187 - }
188 - }
189 -}
190 -
191 -// executes the given op set with a repl to allow easier debugging
192 -/*func debugExecuteOpSet(ds node.DAGService, width int, ops []testOp) (*Shard, error) {
193 -
194 - s, err := NewShard(ds, width)
195 - if err != nil {
196 - return nil, err
197 - }
198 -
199 - e := ft.EmptyDirNode()
200 - ds.Add(e)
201 - ctx := context.TODO()
202 -
203 - run := 0
204 -
205 - opnames := map[int]string{
206 - opAdd: "add",
207 - opDel: "del",
208 - }
209 -
210 -mainloop:
211 - for i := 0; i < len(ops); i++ {
212 - o := ops[i]
213 -
214 - fmt.Printf("Op %d: %s %s\n", i, opnames[o.Op], o.Val)
215 - for run == 0 {
216 - cmd := readCommand()
217 - parts := strings.Split(cmd, " ")
218 - switch parts[0] {
219 - case "":
220 - run = 1
221 - case "find":
222 - _, err := s.Find(ctx, parts[1])
223 - if err == nil {
224 - fmt.Println("success")
225 - } else {
226 - fmt.Println(err)
227 - }
228 - case "run":
229 - if len(parts) > 1 {
230 - n, err := strconv.Atoi(parts[1])
231 - if err != nil {
232 - panic(err)
233 - }
234 -
235 - run = n
236 - } else {
237 - run = -1
238 - }
239 - case "lookop":
240 - for k = 0; k < len(ops); k++ {
241 - if ops[k].Val == parts[1] {
242 - fmt.Printf(" Op %d: %s %s\n", k, opnames[ops[k].Op], parts[1])
243 - }
244 - }
245 - case "restart":
246 - var err error
247 - s, err = NewShard(ds, width)
248 - if err != nil {
249 - panic(err)
250 - }
251 - i = -1
252 - continue mainloop
253 - case "print":
254 - nd, err := s.Node()
255 - if err != nil {
256 - panic(err)
257 - }
258 - printDag(ds, nd.(*dag.ProtoNode), 0)
259 - }
260 - }
261 - run--
262 -
263 - switch o.Op {
264 - case opAdd:
265 - err := s.Set(ctx, o.Val, e)
266 - if err != nil {
267 - return nil, fmt.Errorf("inserting %s: %s", o.Val, err)
268 - }
269 - case opDel:
270 - fmt.Println("deleting: ", o.Val)
271 - err := s.Remove(ctx, o.Val)
272 - if err != nil {
273 - return nil, fmt.Errorf("deleting %s: %s", o.Val, err)
274 - }
275 - case opFind:
276 - _, err := s.Find(ctx, o.Val)
277 - if err != nil {
278 - return nil, fmt.Errorf("finding %s: %s", o.Val, err)
279 - }
280 - }
281 - }
282 -
283 - return s, nil
284 -}
285 -
286 -func readCommand() string {
287 - fmt.Print("> ")
288 - scan := bufio.NewScanner(os.Stdin)
289 - scan.Scan()
290 - return scan.Text()
291 -}*/
unixfs/hamt/hamt_test.go deleted
-610
@@ -1,610 +0,0 @@
1 -package hamt
2 -
3 -import (
4 - "context"
5 - "fmt"
6 - "math/rand"
7 - "os"
8 - "sort"
9 - "testing"
10 - "time"
11 -
12 - "github.com/ipfs/go-ipfs/dagutils"
13 - dag "gx/ipfs/QmRy4Qk9hbgFX9NGJRm8rBThrA8PZhNCitMgeRYyZ67s59/go-merkledag"
14 - mdtest "gx/ipfs/QmRy4Qk9hbgFX9NGJRm8rBThrA8PZhNCitMgeRYyZ67s59/go-merkledag/test"
15 - ft "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs"
16 -
17 - ipld "gx/ipfs/QmZtNq8dArGfnpCZfx2pUNY7UcjGhVp5qqwQ4hH6mpTMRQ/go-ipld-format"
18 -)
19 -
20 -func shuffle(seed int64, arr []string) {
21 - r := rand.New(rand.NewSource(seed))
22 - for i := 0; i < len(arr); i++ {
23 - a := r.Intn(len(arr))
24 - b := r.Intn(len(arr))
25 - arr[a], arr[b] = arr[b], arr[a]
26 - }
27 -}
28 -
29 -func makeDir(ds ipld.DAGService, size int) ([]string, *Shard, error) {
30 - return makeDirWidth(ds, size, 256)
31 -}
32 -
33 -func makeDirWidth(ds ipld.DAGService, size, width int) ([]string, *Shard, error) {
34 - ctx := context.Background()
35 -
36 - s, _ := NewShard(ds, width)
37 -
38 - var dirs []string
39 - for i := 0; i < size; i++ {
40 - dirs = append(dirs, fmt.Sprintf("DIRNAME%d", i))
41 - }
42 -
43 - shuffle(time.Now().UnixNano(), dirs)
44 -
45 - for i := 0; i < len(dirs); i++ {
46 - nd := ft.EmptyDirNode()
47 - ds.Add(ctx, nd)
48 - err := s.Set(ctx, dirs[i], nd)
49 - if err != nil {
50 - return nil, nil, err
51 - }
52 - }
53 -
54 - return dirs, s, nil
55 -}
56 -
57 -func assertLink(s *Shard, name string, found bool) error {
58 - _, err := s.Find(context.Background(), name)
59 - switch err {
60 - case os.ErrNotExist:
61 - if found {
62 - return err
63 - }
64 -
65 - return nil
66 - case nil:
67 - if found {
68 - return nil
69 - }
70 -
71 - return fmt.Errorf("expected not to find link named %s", name)
72 - default:
73 - return err
74 - }
75 -}
76 -
77 -func assertSerializationWorks(ds ipld.DAGService, s *Shard) error {
78 - ctx, cancel := context.WithCancel(context.Background())
79 - defer cancel()
80 - nd, err := s.Node()
81 - if err != nil {
82 - return err
83 - }
84 -
85 - nds, err := NewHamtFromDag(ds, nd)
86 - if err != nil {
87 - return err
88 - }
89 -
90 - linksA, err := s.EnumLinks(ctx)
91 - if err != nil {
92 - return err
93 - }
94 -
95 - linksB, err := nds.EnumLinks(ctx)
96 - if err != nil {
97 - return err
98 - }
99 -
100 - if len(linksA) != len(linksB) {
101 - return fmt.Errorf("links arrays are different sizes")
102 - }
103 -
104 - for i, a := range linksA {
105 - b := linksB[i]
106 - if a.Name != b.Name {
107 - return fmt.Errorf("links names mismatch")
108 - }
109 -
110 - if a.Cid.String() != b.Cid.String() {
111 - return fmt.Errorf("link hashes dont match")
112 - }
113 -
114 - if a.Size != b.Size {
115 - return fmt.Errorf("link sizes not the same")
116 - }
117 - }
118 -
119 - return nil
120 -}
121 -
122 -func TestBasicSet(t *testing.T) {
123 - ds := mdtest.Mock()
124 - for _, w := range []int{128, 256, 512, 1024, 2048, 4096} {
125 - t.Run(fmt.Sprintf("BasicSet%d", w), func(t *testing.T) {
126 - names, s, err := makeDirWidth(ds, 1000, w)
127 - if err != nil {
128 - t.Fatal(err)
129 - }
130 - ctx := context.Background()
131 -
132 - for _, d := range names {
133 - _, err := s.Find(ctx, d)
134 - if err != nil {
135 - t.Fatal(err)
136 - }
137 - }
138 - })
139 - }
140 -}
141 -
142 -func TestDirBuilding(t *testing.T) {
143 - ds := mdtest.Mock()
144 - _, _ = NewShard(ds, 256)
145 -
146 - _, s, err := makeDir(ds, 200)
147 - if err != nil {
148 - t.Fatal(err)
149 - }
150 -
151 - nd, err := s.Node()
152 - if err != nil {
153 - t.Fatal(err)
154 - }
155 -
156 - //printDag(ds, nd, 0)
157 -
158 - k := nd.Cid()
159 -
160 - if k.String() != "QmY89TkSEVHykWMHDmyejSWFj9CYNtvzw4UwnT9xbc4Zjc" {
161 - t.Fatalf("output didnt match what we expected (got %s)", k.String())
162 - }
163 -}
164 -
165 -func TestShardReload(t *testing.T) {
166 - ds := mdtest.Mock()
167 - _, _ = NewShard(ds, 256)
168 - ctx, cancel := context.WithCancel(context.Background())
169 - defer cancel()
170 -
171 - _, s, err := makeDir(ds, 200)
172 - if err != nil {
173 - t.Fatal(err)
174 - }
175 -
176 - nd, err := s.Node()
177 - if err != nil {
178 - t.Fatal(err)
179 - }
180 -
181 - nds, err := NewHamtFromDag(ds, nd)
182 - if err != nil {
183 - t.Fatal(err)
184 - }
185 -
186 - lnks, err := nds.EnumLinks(ctx)
187 - if err != nil {
188 - t.Fatal(err)
189 - }
190 -
191 - if len(lnks) != 200 {
192 - t.Fatal("not enough links back")
193 - }
194 -
195 - _, err = nds.Find(ctx, "DIRNAME50")
196 - if err != nil {
197 - t.Fatal(err)
198 - }
199 -
200 - // Now test roundtrip marshal with no operations
201 -
202 - nds, err = NewHamtFromDag(ds, nd)
203 - if err != nil {
204 - t.Fatal(err)
205 - }
206 -
207 - ond, err := nds.Node()
208 - if err != nil {
209 - t.Fatal(err)
210 - }
211 -
212 - outk := ond.Cid()
213 - ndk := nd.Cid()
214 -
215 - if !outk.Equals(ndk) {
216 - printDiff(ds, nd.(*dag.ProtoNode), ond.(*dag.ProtoNode))
217 - t.Fatal("roundtrip serialization failed")
218 - }
219 -}
220 -
221 -func TestRemoveElems(t *testing.T) {
222 - ds := mdtest.Mock()
223 - dirs, s, err := makeDir(ds, 500)
224 - if err != nil {
225 - t.Fatal(err)
226 - }
227 - ctx := context.Background()
228 -
229 - for i := 0; i < 100; i++ {
230 - err := s.Remove(ctx, fmt.Sprintf("NOTEXIST%d", rand.Int()))
231 - if err != os.ErrNotExist {
232 - t.Fatal("shouldnt be able to remove things that don't exist")
233 - }
234 - }
235 -
236 - for _, d := range dirs {
237 - _, err := s.Find(ctx, d)
238 - if err != nil {
239 - t.Fatal(err)
240 - }
241 - }
242 -
243 - shuffle(time.Now().UnixNano(), dirs)
244 -
245 - for _, d := range dirs {
246 - err := s.Remove(ctx, d)
247 - if err != nil {
248 - t.Fatal(err)
249 - }
250 - }
251 -
252 - nd, err := s.Node()
253 - if err != nil {
254 - t.Fatal(err)
255 - }
256 -
257 - if len(nd.Links()) > 0 {
258 - t.Fatal("shouldnt have any links here")
259 - }
260 -
261 - err = s.Remove(ctx, "doesnt exist")
262 - if err != os.ErrNotExist {
263 - t.Fatal("expected error does not exist")
264 - }
265 -}
266 -
267 -func TestSetAfterMarshal(t *testing.T) {
268 - ds := mdtest.Mock()
269 - _, s, err := makeDir(ds, 300)
270 - if err != nil {
271 - t.Fatal(err)
272 - }
273 - ctx := context.Background()
274 -
275 - nd, err := s.Node()
276 - if err != nil {
277 - t.Fatal(err)
278 - }
279 -
280 - nds, err := NewHamtFromDag(ds, nd)
281 - if err != nil {
282 - t.Fatal(err)
283 - }
284 -
285 - empty := ft.EmptyDirNode()
286 - for i := 0; i < 100; i++ {
287 - err := nds.Set(ctx, fmt.Sprintf("moredirs%d", i), empty)
288 - if err != nil {
289 - t.Fatal(err)
290 - }
291 - }
292 -
293 - links, err := nds.EnumLinks(ctx)
294 - if err != nil {
295 - t.Fatal(err)
296 - }
297 -
298 - if len(links) != 400 {
299 - t.Fatal("expected 400 links")
300 - }
301 -
302 - err = assertSerializationWorks(ds, nds)
303 - if err != nil {
304 - t.Fatal(err)
305 - }
306 -}
307 -
308 -func TestDuplicateAddShard(t *testing.T) {
309 - ds := mdtest.Mock()
310 - dir, _ := NewShard(ds, 256)
311 - nd := new(dag.ProtoNode)
312 - ctx := context.Background()
313 -
314 - err := dir.Set(ctx, "test", nd)
315 - if err != nil {
316 - t.Fatal(err)
317 - }
318 -
319 - err = dir.Set(ctx, "test", nd)
320 - if err != nil {
321 - t.Fatal(err)
322 - }
323 -
324 - lnks, err := dir.EnumLinks(ctx)
325 - if err != nil {
326 - t.Fatal(err)
327 - }
328 -
329 - if len(lnks) != 1 {
330 - t.Fatal("expected only one link")
331 - }
332 -}
333 -
334 -func TestLoadFailsFromNonShard(t *testing.T) {
335 - ds := mdtest.Mock()
336 - nd := ft.EmptyDirNode()
337 -
338 - _, err := NewHamtFromDag(ds, nd)
339 - if err == nil {
340 - t.Fatal("expected dir shard creation to fail when given normal directory")
341 - }
342 -
343 - nd = new(dag.ProtoNode)
344 -
345 - _, err = NewHamtFromDag(ds, nd)
346 - if err == nil {
347 - t.Fatal("expected dir shard creation to fail when given normal directory")
348 - }
349 -}
350 -
351 -func TestFindNonExisting(t *testing.T) {
352 - ds := mdtest.Mock()
353 - _, s, err := makeDir(ds, 100)
354 - if err != nil {
355 - t.Fatal(err)
356 - }
357 - ctx := context.Background()
358 -
359 - for i := 0; i < 200; i++ {
360 - _, err := s.Find(ctx, fmt.Sprintf("notfound%d", i))
361 - if err != os.ErrNotExist {
362 - t.Fatal("expected ErrNotExist")
363 - }
364 - }
365 -}
366 -
367 -func TestRemoveElemsAfterMarshal(t *testing.T) {
368 - ds := mdtest.Mock()
369 - dirs, s, err := makeDir(ds, 30)
370 - if err != nil {
371 - t.Fatal(err)
372 - }
373 - ctx := context.Background()
374 -
375 - sort.Strings(dirs)
376 -
377 - err = s.Remove(ctx, dirs[0])
378 - if err != nil {
379 - t.Fatal(err)
380 - }
381 -
382 - out, err := s.Find(ctx, dirs[0])
383 - if err == nil {
384 - t.Fatal("expected error, got: ", out)
385 - }
386 -
387 - nd, err := s.Node()
388 - if err != nil {
389 - t.Fatal(err)
390 - }
391 -
392 - nds, err := NewHamtFromDag(ds, nd)
393 - if err != nil {
394 - t.Fatal(err)
395 - }
396 -
397 - _, err = nds.Find(ctx, dirs[0])
398 - if err == nil {
399 - t.Fatal("expected not to find ", dirs[0])
400 - }
401 -
402 - for _, d := range dirs[1:] {
403 - _, err := nds.Find(ctx, d)
404 - if err != nil {
405 - t.Fatal("could not find expected link after unmarshaling")
406 - }
407 - }
408 -
409 - for _, d := range dirs[1:] {
410 - err := nds.Remove(ctx, d)
411 - if err != nil {
412 - t.Fatal(err)
413 - }
414 - }
415 -
416 - links, err := nds.EnumLinks(ctx)
417 - if err != nil {
418 - t.Fatal(err)
419 - }
420 -
421 - if len(links) != 0 {
422 - t.Fatal("expected all links to be removed")
423 - }
424 -
425 - err = assertSerializationWorks(ds, nds)
426 - if err != nil {
427 - t.Fatal(err)
428 - }
429 -}
430 -
431 -func TestBitfieldIndexing(t *testing.T) {
432 - ds := mdtest.Mock()
433 - s, _ := NewShard(ds, 256)
434 -
435 - set := func(i int) {
436 - s.bitfield.SetBit(i)
437 - }
438 -
439 - assert := func(i int, val int) {
440 - if s.indexForBitPos(i) != val {
441 - t.Fatalf("expected index %d to be %d", i, val)
442 - }
443 - }
444 -
445 - assert(50, 0)
446 - set(4)
447 - set(5)
448 - set(60)
449 -
450 - assert(10, 2)
451 - set(3)
452 - assert(10, 3)
453 - assert(1, 0)
454 -
455 - assert(100, 4)
456 - set(50)
457 - assert(45, 3)
458 - set(100)
459 - assert(100, 5)
460 -}
461 -
462 -// test adding a sharded directory node as the child of another directory node.
463 -// if improperly implemented, the parent hamt may assume the child is a part of
464 -// itself.
465 -func TestSetHamtChild(t *testing.T) {
466 - ctx := context.Background()
467 -
468 - ds := mdtest.Mock()
469 - s, _ := NewShard(ds, 256)
470 -
471 - e := ft.EmptyDirNode()
472 - ds.Add(ctx, e)
473 -
474 - err := s.Set(ctx, "bar", e)
475 - if err != nil {
476 - t.Fatal(err)
477 - }
478 -
479 - snd, err := s.Node()
480 - if err != nil {
481 - t.Fatal(err)
482 - }
483 -
484 - _, ns, err := makeDir(ds, 50)
485 - if err != nil {
486 - t.Fatal(err)
487 - }
488 -
489 - err = ns.Set(ctx, "foo", snd)
490 - if err != nil {
491 - t.Fatal(err)
492 - }
493 -
494 - nsnd, err := ns.Node()
495 - if err != nil {
496 - t.Fatal(err)
497 - }
498 -
499 - hs, err := NewHamtFromDag(ds, nsnd)
500 - if err != nil {
501 - t.Fatal(err)
502 - }
503 -
504 - err = assertLink(hs, "bar", false)
505 - if err != nil {
506 - t.Fatal(err)
507 - }
508 -
509 - err = assertLink(hs, "foo", true)
510 - if err != nil {
511 - t.Fatal(err)
512 - }
513 -}
514 -
515 -func printDiff(ds ipld.DAGService, a, b *dag.ProtoNode) {
516 - diff, err := dagutils.Diff(context.TODO(), ds, a, b)
517 - if err != nil {
518 - panic(err)
519 - }
520 -
521 - for _, d := range diff {
522 - fmt.Println(d)
523 - }
524 -}
525 -
526 -func BenchmarkHAMTWalk(b *testing.B) {
527 - ctx := context.Background()
528 -
529 - ds := mdtest.Mock()
530 - sh, _ := NewShard(ds, 256)
531 - nd, err := sh.Node()
532 - if err != nil {
533 - b.Fatal(err)
534 - }
535 -
536 - err = ds.Add(ctx, nd)
537 - if err != nil {
538 - b.Fatal(err)
539 - }
540 - ds.Add(ctx, ft.EmptyDirNode())
541 -
542 - s, err := NewHamtFromDag(ds, nd)
543 - if err != nil {
544 - b.Fatal(err)
545 - }
546 -
547 - for j := 0; j < 1000; j++ {
548 - err = s.Set(ctx, fmt.Sprintf("%d", j), ft.EmptyDirNode())
549 - if err != nil {
550 - b.Fatal(err)
551 - }
552 - }
553 -
554 - for i := 0; i < b.N; i++ {
555 - cnt := 0
556 - err = s.ForEachLink(ctx, func(l *ipld.Link) error {
557 - cnt++
558 - return nil
559 - })
560 - if err != nil {
561 - b.Fatal(err)
562 - }
563 - if cnt < 1000 {
564 - b.Fatal("expected 100 children")
565 - }
566 - }
567 -}
568 -
569 -func BenchmarkHAMTSet(b *testing.B) {
570 - ctx := context.Background()
571 -
572 - ds := mdtest.Mock()
573 - sh, _ := NewShard(ds, 256)
574 - nd, err := sh.Node()
575 - if err != nil {
576 - b.Fatal(err)
577 - }
578 -
579 - err = ds.Add(ctx, nd)
580 - if err != nil {
581 - b.Fatal(err)
582 - }
583 - ds.Add(ctx, ft.EmptyDirNode())
584 -
585 - for i := 0; i < b.N; i++ {
586 - s, err := NewHamtFromDag(ds, nd)
587 - if err != nil {
588 - b.Fatal(err)
589 - }
590 -
591 - err = s.Set(context.TODO(), fmt.Sprint(i), ft.EmptyDirNode())
592 - if err != nil {
593 - b.Fatal(err)
594 - }
595 -
596 - out, err := s.Node()
597 - if err != nil {
598 - b.Fatal(err)
599 - }
600 -
601 - nd = out
602 - }
603 -}
604 -
605 -func TestHamtBadSize(t *testing.T) {
606 - _, err := NewShard(nil, 7)
607 - if err == nil {
608 - t.Fatal("should have failed to construct hamt with bad size")
609 - }
610 -}
unixfs/hamt/util.go deleted
-52
@@ -1,52 +0,0 @@
1 -package hamt
2 -
3 -import (
4 - "fmt"
5 - "math/bits"
6 -)
7 -
8 -// hashBits is a helper that allows the reading of the 'next n bits' as an integer.
9 -type hashBits struct {
10 - b []byte
11 - consumed int
12 -}
13 -
14 -func mkmask(n int) byte {
15 - return (1 << uint(n)) - 1
16 -}
17 -
18 -// Next returns the next 'i' bits of the hashBits value as an integer
19 -func (hb *hashBits) Next(i int) int {
20 - curbi := hb.consumed / 8
21 - leftb := 8 - (hb.consumed % 8)
22 -
23 - curb := hb.b[curbi]
24 - if i == leftb {
25 - out := int(mkmask(i) & curb)
26 - hb.consumed += i
27 - return out
28 - } else if i < leftb {
29 - a := curb & mkmask(leftb) // mask out the high bits we don't want
30 - b := a & ^mkmask(leftb-i) // mask out the low bits we don't want
31 - c := b >> uint(leftb-i) // shift whats left down
32 - hb.consumed += i
33 - return int(c)
34 - } else {
35 - out := int(mkmask(leftb) & curb)
36 - out <<= uint(i - leftb)
37 - hb.consumed += leftb
38 - out += hb.Next(i - leftb)
39 - return out
40 - }
41 -}
42 -
43 -func logtwo(v int) (int, error) {
44 - if v <= 0 {
45 - return 0, fmt.Errorf("hamt size should be a power of two")
46 - }
47 - lg2 := bits.TrailingZeros(uint(v))
48 - if 1<<uint(lg2) != v {
49 - return 0, fmt.Errorf("hamt size should be a power of two")
50 - }
51 - return lg2, nil
52 -}
unixfs/hamt/util_test.go deleted
-45
@@ -1,45 +0,0 @@
1 -package hamt
2 -
3 -import (
4 - "testing"
5 -)
6 -
7 -func TestHashBitsEvenSizes(t *testing.T) {
8 - buf := []byte{255, 127, 79, 45, 116, 99, 35, 17}
9 - hb := hashBits{b: buf}
10 -
11 - for _, v := range buf {
12 - if hb.Next(8) != int(v) {
13 - t.Fatal("got wrong numbers back")
14 - }
15 - }
16 -}
17 -
18 -func TestHashBitsUneven(t *testing.T) {
19 - buf := []byte{255, 127, 79, 45, 116, 99, 35, 17}
20 - hb := hashBits{b: buf}
21 -
22 - v := hb.Next(4)
23 - if v != 15 {
24 - t.Fatal("should have gotten 15: ", v)
25 - }
26 -
27 - v = hb.Next(4)
28 - if v != 15 {
29 - t.Fatal("should have gotten 15: ", v)
30 - }
31 -
32 - if v := hb.Next(3); v != 3 {
33 - t.Fatalf("expected 3, but got %b", v)
34 - }
35 - if v := hb.Next(3); v != 7 {
36 - t.Fatalf("expected 7, but got %b", v)
37 - }
38 - if v := hb.Next(3); v != 6 {
39 - t.Fatalf("expected 6, but got %b", v)
40 - }
41 -
42 - if v := hb.Next(15); v != 20269 {
43 - t.Fatalf("expected 20269, but got %b (%d)", v, v)
44 - }
45 -}
unixfs/importer/balanced/balanced_test.go deleted
-334
@@ -1,334 +0,0 @@
1 -package balanced
2 -
3 -import (
4 - "bytes"
5 - "context"
6 - "fmt"
7 - "io"
8 - "io/ioutil"
9 - mrand "math/rand"
10 - "testing"
11 -
12 - dag "gx/ipfs/QmRy4Qk9hbgFX9NGJRm8rBThrA8PZhNCitMgeRYyZ67s59/go-merkledag"
13 - mdtest "gx/ipfs/QmRy4Qk9hbgFX9NGJRm8rBThrA8PZhNCitMgeRYyZ67s59/go-merkledag/test"
14 - h "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/importer/helpers"
15 - uio "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/io"
16 -
17 - u "gx/ipfs/QmPdKqUcHGFdeSpvjVoaTRPPstGif9GBZb5Q56RVw9o69A/go-ipfs-util"
18 - chunker "gx/ipfs/QmVDjhUMtkRskBFAVNwyXuLSKbeAya7JKPnzAxMKDaK4x4/go-ipfs-chunker"
19 - ipld "gx/ipfs/QmZtNq8dArGfnpCZfx2pUNY7UcjGhVp5qqwQ4hH6mpTMRQ/go-ipld-format"
20 -)
21 -
22 -// TODO: extract these tests and more as a generic layout test suite
23 -
24 -func buildTestDag(ds ipld.DAGService, spl chunker.Splitter) (*dag.ProtoNode, error) {
25 - dbp := h.DagBuilderParams{
26 - Dagserv: ds,
27 - Maxlinks: h.DefaultLinksPerBlock,
28 - }
29 -
30 - nd, err := Layout(dbp.New(spl))
31 - if err != nil {
32 - return nil, err
33 - }
34 -
35 - return nd.(*dag.ProtoNode), nil
36 -}
37 -
38 -func getTestDag(t *testing.T, ds ipld.DAGService, size int64, blksize int64) (*dag.ProtoNode, []byte) {
39 - data := make([]byte, size)
40 - u.NewTimeSeededRand().Read(data)
41 - r := bytes.NewReader(data)
42 -
43 - nd, err := buildTestDag(ds, chunker.NewSizeSplitter(r, blksize))
44 - if err != nil {
45 - t.Fatal(err)
46 - }
47 -
48 - return nd, data
49 -}
50 -
51 -//Test where calls to read are smaller than the chunk size
52 -func TestSizeBasedSplit(t *testing.T) {
53 - if testing.Short() {
54 - t.SkipNow()
55 - }
56 -
57 - testFileConsistency(t, 32*512, 512)
58 - testFileConsistency(t, 32*4096, 4096)
59 -
60 - // Uneven offset
61 - testFileConsistency(t, 31*4095, 4096)
62 -}
63 -
64 -func testFileConsistency(t *testing.T, nbytes int64, blksize int64) {
65 - ds := mdtest.Mock()
66 - nd, should := getTestDag(t, ds, nbytes, blksize)
67 -
68 - r, err := uio.NewDagReader(context.Background(), nd, ds)
69 - if err != nil {
70 - t.Fatal(err)
71 - }
72 -
73 - dagrArrComp(t, r, should)
74 -}
75 -
76 -func TestBuilderConsistency(t *testing.T) {
77 - testFileConsistency(t, 100000, chunker.DefaultBlockSize)
78 -}
79 -
80 -func TestNoChunking(t *testing.T) {
81 - ds := mdtest.Mock()
82 -
83 - nd, should := getTestDag(t, ds, 1000, 2000)
84 - r, err := uio.NewDagReader(context.Background(), nd, ds)
85 - if err != nil {
86 - t.Fatal(err)
87 - }
88 -
89 - dagrArrComp(t, r, should)
90 -}
91 -
92 -func TestTwoChunks(t *testing.T) {
93 - ds := mdtest.Mock()
94 -
95 - nd, should := getTestDag(t, ds, 2000, 1000)
96 - r, err := uio.NewDagReader(context.Background(), nd, ds)
97 - if err != nil {
98 - t.Fatal(err)
99 - }
100 -
101 - dagrArrComp(t, r, should)
102 -}
103 -
104 -func arrComp(a, b []byte) error {
105 - if len(a) != len(b) {
106 - return fmt.Errorf("arrays differ in length. %d != %d", len(a), len(b))
107 - }
108 - for i, v := range a {
109 - if v != b[i] {
110 - return fmt.Errorf("arrays differ at index: %d", i)
111 - }
112 - }
113 - return nil
114 -}
115 -
116 -func dagrArrComp(t *testing.T, r io.Reader, should []byte) {
117 - out, err := ioutil.ReadAll(r)
118 - if err != nil {
119 - t.Fatal(err)
120 - }
121 -
122 - if err := arrComp(out, should); err != nil {
123 - t.Fatal(err)
124 - }
125 -}
126 -
127 -func TestIndirectBlocks(t *testing.T) {
128 - ds := mdtest.Mock()
129 - dag, buf := getTestDag(t, ds, 1024*1024, 512)
130 -
131 - reader, err := uio.NewDagReader(context.Background(), dag, ds)
132 - if err != nil {
133 - t.Fatal(err)
134 - }
135 -
136 - out, err := ioutil.ReadAll(reader)
137 - if err != nil {
138 - t.Fatal(err)
139 - }
140 -
141 - if !bytes.Equal(out, buf) {
142 - t.Fatal("Not equal!")
143 - }
144 -}
145 -
146 -func TestSeekingBasic(t *testing.T) {
147 - nbytes := int64(10 * 1024)
148 - ds := mdtest.Mock()
149 - nd, should := getTestDag(t, ds, nbytes, 500)
150 -
151 - rs, err := uio.NewDagReader(context.Background(), nd, ds)
152 - if err != nil {
153 - t.Fatal(err)
154 - }
155 -
156 - start := int64(4000)
157 - n, err := rs.Seek(start, io.SeekStart)
158 - if err != nil {
159 - t.Fatal(err)
160 - }
161 - if n != start {
162 - t.Fatal("Failed to seek to correct offset")
163 - }
164 -
165 - dagrArrComp(t, rs, should[start:])
166 -}
167 -
168 -func TestSeekToBegin(t *testing.T) {
169 - ds := mdtest.Mock()
170 - nd, should := getTestDag(t, ds, 10*1024, 500)
171 -
172 - rs, err := uio.NewDagReader(context.Background(), nd, ds)
173 - if err != nil {
174 - t.Fatal(err)
175 - }
176 -
177 - n, err := io.CopyN(ioutil.Discard, rs, 1024*4)
178 - if err != nil {
179 - t.Fatal(err)
180 - }
181 - if n != 4096 {
182 - t.Fatal("Copy didnt copy enough bytes")
183 - }
184 -
185 - seeked, err := rs.Seek(0, io.SeekStart)
186 - if err != nil {
187 - t.Fatal(err)
188 - }
189 - if seeked != 0 {
190 - t.Fatal("Failed to seek to beginning")
191 - }
192 -
193 - dagrArrComp(t, rs, should)
194 -}
195 -
196 -func TestSeekToAlmostBegin(t *testing.T) {
197 - ds := mdtest.Mock()
198 - nd, should := getTestDag(t, ds, 10*1024, 500)
199 -
200 - rs, err := uio.NewDagReader(context.Background(), nd, ds)
201 - if err != nil {
202 - t.Fatal(err)
203 - }
204 -
205 - n, err := io.CopyN(ioutil.Discard, rs, 1024*4)
206 - if err != nil {
207 - t.Fatal(err)
208 - }
209 - if n != 4096 {
210 - t.Fatal("Copy didnt copy enough bytes")
211 - }
212 -
213 - seeked, err := rs.Seek(1, io.SeekStart)
214 - if err != nil {
215 - t.Fatal(err)
216 - }
217 - if seeked != 1 {
218 - t.Fatal("Failed to seek to almost beginning")
219 - }
220 -
221 - dagrArrComp(t, rs, should[1:])
222 -}
223 -
224 -func TestSeekEnd(t *testing.T) {
225 - nbytes := int64(50 * 1024)
226 - ds := mdtest.Mock()
227 - nd, _ := getTestDag(t, ds, nbytes, 500)
228 -
229 - rs, err := uio.NewDagReader(context.Background(), nd, ds)
230 - if err != nil {
231 - t.Fatal(err)
232 - }
233 -
234 - seeked, err := rs.Seek(0, io.SeekEnd)
235 - if err != nil {
236 - t.Fatal(err)
237 - }
238 - if seeked != nbytes {
239 - t.Fatal("Failed to seek to end")
240 - }
241 -}
242 -
243 -func TestSeekEndSingleBlockFile(t *testing.T) {
244 - nbytes := int64(100)
245 - ds := mdtest.Mock()
246 - nd, _ := getTestDag(t, ds, nbytes, 5000)
247 -
248 - rs, err := uio.NewDagReader(context.Background(), nd, ds)
249 - if err != nil {
250 - t.Fatal(err)
251 - }
252 -
253 - seeked, err := rs.Seek(0, io.SeekEnd)
254 - if err != nil {
255 - t.Fatal(err)
256 - }
257 - if seeked != nbytes {
258 - t.Fatal("Failed to seek to end")
259 - }
260 -}
261 -
262 -func TestSeekingStress(t *testing.T) {
263 - nbytes := int64(1024 * 1024)
264 - ds := mdtest.Mock()
265 - nd, should := getTestDag(t, ds, nbytes, 1000)
266 -
267 - rs, err := uio.NewDagReader(context.Background(), nd, ds)
268 - if err != nil {
269 - t.Fatal(err)
270 - }
271 -
272 - testbuf := make([]byte, nbytes)
273 - for i := 0; i < 50; i++ {
274 - offset := mrand.Intn(int(nbytes))
275 - l := int(nbytes) - offset
276 - n, err := rs.Seek(int64(offset), io.SeekStart)
277 - if err != nil {
278 - t.Fatal(err)
279 - }
280 - if n != int64(offset) {
281 - t.Fatal("Seek failed to move to correct position")
282 - }
283 -
284 - nread, err := rs.Read(testbuf[:l])
285 - if err != nil {
286 - t.Fatal(err)
287 - }
288 - if nread != l {
289 - t.Fatal("Failed to read enough bytes")
290 - }
291 -
292 - err = arrComp(testbuf[:l], should[offset:offset+l])
293 - if err != nil {
294 - t.Fatal(err)
295 - }
296 - }
297 -
298 -}
299 -
300 -func TestSeekingConsistency(t *testing.T) {
301 - nbytes := int64(128 * 1024)
302 - ds := mdtest.Mock()
303 - nd, should := getTestDag(t, ds, nbytes, 500)
304 -
305 - rs, err := uio.NewDagReader(context.Background(), nd, ds)
306 - if err != nil {
307 - t.Fatal(err)
308 - }
309 -
310 - out := make([]byte, nbytes)
311 -
312 - for coff := nbytes - 4096; coff >= 0; coff -= 4096 {
313 - t.Log(coff)
314 - n, err := rs.Seek(coff, io.SeekStart)
315 - if err != nil {
316 - t.Fatal(err)
317 - }
318 - if n != coff {
319 - t.Fatal("wasnt able to seek to the right position")
320 - }
321 - nread, err := rs.Read(out[coff : coff+4096])
322 - if err != nil {
323 - t.Fatal(err)
324 - }
325 - if nread != 4096 {
326 - t.Fatal("didnt read the correct number of bytes")
327 - }
328 - }
329 -
330 - err = arrComp(out, should)
331 - if err != nil {
332 - t.Fatal(err)
333 - }
334 -}
unixfs/importer/balanced/builder.go deleted
-255
@@ -1,255 +0,0 @@
1 -// Package balanced provides methods to build balanced DAGs, which are generalistic
2 -// DAGs in which all leaves (nodes representing chunks of data) are at the same
3 -// distance from the root. Nodes can have only a maximum number of children; to be
4 -// able to store more leaf data nodes balanced DAGs are extended by increasing its
5 -// depth (and having more intermediary nodes).
6 -//
7 -// Internal nodes are always represented by UnixFS nodes (of type `File`) encoded
8 -// inside DAG nodes (see the `go-unixfs` package for details of UnixFS). In
9 -// contrast, leaf nodes with data have multiple possible representations: UnixFS
10 -// nodes as above, raw nodes with just the file data (no format) and Filestore
11 -// nodes (that directly link to the file on disk using a format stored on a raw
12 -// node, see the `go-ipfs/filestore` package for details of Filestore.)
13 -//
14 -// In the case the entire file fits into just one node it will be formatted as a
15 -// (single) leaf node (without parent) with the possible representations already
16 -// mentioned. This is the only scenario where the root can be of a type different
17 -// that the UnixFS node.
18 -//
19 -// +-------------+
20 -// | Root 4 |
21 -// +-------------+
22 -// |
23 -// +--------------------------+----------------------------+
24 -// | |
25 -// +-------------+ +-------------+
26 -// | Node 2 | | Node 5 |
27 -// +-------------+ +-------------+
28 -// | |
29 -// +-------------+-------------+ +-------------+
30 -// | | |
31 -// +-------------+ +-------------+ +-------------+
32 -// | Node 1 | | Node 3 | | Node 6 |
33 -// +-------------+ +-------------+ +-------------+
34 -// | | |
35 -// +------+------+ +------+------+ +------+
36 -// | | | | |
37 -// +=========+ +=========+ +=========+ +=========+ +=========+
38 -// | Chunk 1 | | Chunk 2 | | Chunk 3 | | Chunk 4 | | Chunk 5 |
39 -// +=========+ +=========+ +=========+ +=========+ +=========+
40 -//
41 -package balanced
42 -
43 -import (
44 - "errors"
45 -
46 - ft "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs"
47 - h "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/importer/helpers"
48 -
49 - ipld "gx/ipfs/QmZtNq8dArGfnpCZfx2pUNY7UcjGhVp5qqwQ4hH6mpTMRQ/go-ipld-format"
50 -)
51 -
52 -// Layout builds a balanced DAG layout. In a balanced DAG of depth 1, leaf nodes
53 -// with data are added to a single `root` until the maximum number of links is
54 -// reached. Then, to continue adding more data leaf nodes, a `newRoot` is created
55 -// pointing to the old `root` (which will now become and intermediary node),
56 -// increasing the depth of the DAG to 2. This will increase the maximum number of
57 -// data leaf nodes the DAG can have (`Maxlinks() ^ depth`). The `fillNodeRec`
58 -// function will add more intermediary child nodes to `newRoot` (which already has
59 -// `root` as child) that in turn will have leaf nodes with data added to them.
60 -// After that process is completed (the maximum number of links is reached),
61 -// `fillNodeRec` will return and the loop will be repeated: the `newRoot` created
62 -// will become the old `root` and a new root will be created again to increase the
63 -// depth of the DAG. The process is repeated until there is no more data to add
64 -// (i.e. the DagBuilderHelper’s Done() function returns true).
65 -//
66 -// The nodes are filled recursively, so the DAG is built from the bottom up. Leaf
67 -// nodes are created first using the chunked file data and its size. The size is
68 -// then bubbled up to the parent (internal) node, which aggregates all the sizes of
69 -// its children and bubbles that combined size up to its parent, and so on up to
70 -// the root. This way, a balanced DAG acts like a B-tree when seeking to a byte
71 -// offset in the file the graph represents: each internal node uses the file size
72 -// of its children as an index when seeking.
73 -//
74 -// `Layout` creates a root and hands it off to be filled:
75 -//
76 -// +-------------+
77 -// | Root 1 |
78 -// +-------------+
79 -// |
80 -// ( fillNodeRec fills in the )
81 -// ( chunks on the root. )
82 -// |
83 -// +------+------+
84 -// | |
85 -// + - - - - + + - - - - +
86 -// | Chunk 1 | | Chunk 2 |
87 -// + - - - - + + - - - - +
88 -//
89 -// ↓
90 -// When the root is full but there's more data...
91 -// ↓
92 -//
93 -// +-------------+
94 -// | Root 1 |
95 -// +-------------+
96 -// |
97 -// +------+------+
98 -// | |
99 -// +=========+ +=========+ + - - - - +
100 -// | Chunk 1 | | Chunk 2 | | Chunk 3 |
101 -// +=========+ +=========+ + - - - - +
102 -//
103 -// ↓
104 -// ...Layout's job is to create a new root.
105 -// ↓
106 -//
107 -// +-------------+
108 -// | Root 2 |
109 -// +-------------+
110 -// |
111 -// +-------------+ - - - - - - - - +
112 -// | |
113 -// +-------------+ ( fillNodeRec creates the )
114 -// | Node 1 | ( branch that connects )
115 -// +-------------+ ( "Root 2" to "Chunk 3." )
116 -// | |
117 -// +------+------+ + - - - - -+
118 -// | | |
119 -// +=========+ +=========+ + - - - - +
120 -// | Chunk 1 | | Chunk 2 | | Chunk 3 |
121 -// +=========+ +=========+ + - - - - +
122 -//
123 -func Layout(db *h.DagBuilderHelper) (ipld.Node, error) {
124 - if db.Done() {
125 - // No data, return just an empty node.
126 - root, err := db.NewLeafNode(nil)
127 - if err != nil {
128 - return nil, err
129 - }
130 - // This works without Filestore support (`ProcessFileStore`).
131 - // TODO: Why? Is there a test case missing?
132 -
133 - return db.AddNodeAndClose(root)
134 - }
135 -
136 - // The first `root` will be a single leaf node with data
137 - // (corner case), after that subsequent `root` nodes will
138 - // always be internal nodes (with a depth > 0) that can
139 - // be handled by the loop.
140 - root, fileSize, err := db.NewLeafDataNode()
141 - if err != nil {
142 - return nil, err
143 - }
144 -
145 - // Each time a DAG of a certain `depth` is filled (because it
146 - // has reached its maximum capacity of `db.Maxlinks()` per node)
147 - // extend it by making it a sub-DAG of a bigger DAG with `depth+1`.
148 - for depth := 1; !db.Done(); depth++ {
149 -
150 - // Add the old `root` as a child of the `newRoot`.
151 - newRoot := db.NewFSNodeOverDag(ft.TFile)
152 - newRoot.AddChild(root, fileSize, db)
153 -
154 - // Fill the `newRoot` (that has the old `root` already as child)
155 - // and make it the current `root` for the next iteration (when
156 - // it will become "old").
157 - root, fileSize, err = fillNodeRec(db, newRoot, depth)
158 - if err != nil {
159 - return nil, err
160 - }
161 - }
162 -
163 - return db.AddNodeAndClose(root)
164 -}
165 -
166 -// fillNodeRec will "fill" the given internal (non-leaf) `node` with data by
167 -// adding child nodes to it, either leaf data nodes (if `depth` is 1) or more
168 -// internal nodes with higher depth (and calling itself recursively on them
169 -// until *they* are filled with data). The data to fill the node with is
170 -// provided by DagBuilderHelper.
171 -//
172 -// `node` represents a (sub-)DAG root that is being filled. If called recursively,
173 -// it is `nil`, a new node is created. If it has been called from `Layout` (see
174 -// diagram below) it points to the new root (that increases the depth of the DAG),
175 -// it already has a child (the old root). New children will be added to this new
176 -// root, and those children will in turn be filled (calling `fillNodeRec`
177 -// recursively).
178 -//
179 -// +-------------+
180 -// | `node` |
181 -// | (new root) |
182 -// +-------------+
183 -// |
184 -// +-------------+ - - - - - - + - - - - - - - - - - - +
185 -// | | |
186 -// +--------------+ + - - - - - + + - - - - - +
187 -// | (old root) | | new child | | |
188 -// +--------------+ + - - - - - + + - - - - - +
189 -// | | |
190 -// +------+------+ + - - + - - - +
191 -// | | | |
192 -// +=========+ +=========+ + - - - - + + - - - - +
193 -// | Chunk 1 | | Chunk 2 | | Chunk 3 | | Chunk 4 |
194 -// +=========+ +=========+ + - - - - + + - - - - +
195 -//
196 -// The `node` to be filled uses the `FSNodeOverDag` abstraction that allows adding
197 -// child nodes without packing/unpacking the UnixFS layer node (having an internal
198 -// `ft.FSNode` cache).
199 -//
200 -// It returns the `ipld.Node` representation of the passed `node` filled with
201 -// children and the `nodeFileSize` with the total size of the file chunk (leaf)
202 -// nodes stored under this node (parent nodes store this to enable efficient
203 -// seeking through the DAG when reading data later).
204 -//
205 -// warning: **children** pinned indirectly, but input node IS NOT pinned.
206 -func fillNodeRec(db *h.DagBuilderHelper, node *h.FSNodeOverDag, depth int) (filledNode ipld.Node, nodeFileSize uint64, err error) {
207 - if depth < 1 {
208 - return nil, 0, errors.New("attempt to fillNode at depth < 1")
209 - }
210 -
211 - if node == nil {
212 - node = db.NewFSNodeOverDag(ft.TFile)
213 - }
214 -
215 - // Child node created on every iteration to add to parent `node`.
216 - // It can be a leaf node or another internal node.
217 - var childNode ipld.Node
218 - // File size from the child node needed to update the `FSNode`
219 - // in `node` when adding the child.
220 - var childFileSize uint64
221 -
222 - // While we have room and there is data available to be added.
223 - for node.NumChildren() < db.Maxlinks() && !db.Done() {
224 -
225 - if depth == 1 {
226 - // Base case: add leaf node with data.
227 - childNode, childFileSize, err = db.NewLeafDataNode()
228 - if err != nil {
229 - return nil, 0, err
230 - }
231 - } else {
232 - // Recursion case: create an internal node to in turn keep
233 - // descending in the DAG and adding child nodes to it.
234 - childNode, childFileSize, err = fillNodeRec(db, nil, depth-1)
235 - if err != nil {
236 - return nil, 0, err
237 - }
238 - }
239 -
240 - err = node.AddChild(childNode, childFileSize, db)
241 - if err != nil {
242 - return nil, 0, err
243 - }
244 - }
245 -
246 - nodeFileSize = node.FileSize()
247 -
248 - // Get the final `dag.ProtoNode` with the `FSNode` data encoded inside.
249 - filledNode, err = node.Commit()
250 - if err != nil {
251 - return nil, 0, err
252 - }
253 -
254 - return filledNode, nodeFileSize, nil
255 -}
unixfs/importer/helpers/dagbuilder.go deleted
-458
@@ -1,458 +0,0 @@
1 -package helpers
2 -
3 -import (
4 - "context"
5 - "io"
6 - "os"
7 -
8 - dag "gx/ipfs/QmRy4Qk9hbgFX9NGJRm8rBThrA8PZhNCitMgeRYyZ67s59/go-merkledag"
9 - ft "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs"
10 - pb "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/pb"
11 -
12 - pi "gx/ipfs/QmSHjPDw8yNgLZ7cBfX7w3Smn7PHwYhNEpd4LHQQxUg35L/go-ipfs-posinfo"
13 - chunker "gx/ipfs/QmVDjhUMtkRskBFAVNwyXuLSKbeAya7JKPnzAxMKDaK4x4/go-ipfs-chunker"
14 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
15 - ipld "gx/ipfs/QmZtNq8dArGfnpCZfx2pUNY7UcjGhVp5qqwQ4hH6mpTMRQ/go-ipld-format"
16 - files "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit/files"
17 -)
18 -
19 -// DagBuilderHelper wraps together a bunch of objects needed to
20 -// efficiently create unixfs dag trees
21 -type DagBuilderHelper struct {
22 - dserv ipld.DAGService
23 - spl chunker.Splitter
24 - recvdErr error
25 - rawLeaves bool
26 - nextData []byte // the next item to return.
27 - maxlinks int
28 - batch *ipld.Batch
29 - prefix *cid.Prefix
30 -
31 - // Filestore support variables.
32 - // ----------------------------
33 - // TODO: Encapsulate in `FilestoreNode` (which is basically what they are).
34 - //
35 - // Besides having the path this variable (if set) is used as a flag
36 - // to indicate that Filestore should be used.
37 - fullPath string
38 - stat os.FileInfo
39 - // Keeps track of the current file size added to the DAG (used in
40 - // the balanced builder). It is assumed that the `DagBuilderHelper`
41 - // is not reused to construct another DAG, but a new one (with a
42 - // zero `offset`) is created.
43 - offset uint64
44 -}
45 -
46 -// DagBuilderParams wraps configuration options to create a DagBuilderHelper
47 -// from a chunker.Splitter.
48 -type DagBuilderParams struct {
49 - // Maximum number of links per intermediate node
50 - Maxlinks int
51 -
52 - // RawLeaves signifies that the importer should use raw ipld nodes as leaves
53 - // instead of using the unixfs TRaw type
54 - RawLeaves bool
55 -
56 - // CID Prefix to use if set
57 - Prefix *cid.Prefix
58 -
59 - // DAGService to write blocks to (required)
60 - Dagserv ipld.DAGService
61 -
62 - // NoCopy signals to the chunker that it should track fileinfo for
63 - // filestore adds
64 - NoCopy bool
65 -
66 - // URL if non-empty (and NoCopy is also true) indicates that the
67 - // file will not be stored in the datastore but instead retrieved
68 - // from this location via the urlstore.
69 - URL string
70 -}
71 -
72 -// New generates a new DagBuilderHelper from the given params and a given
73 -// chunker.Splitter as data source.
74 -func (dbp *DagBuilderParams) New(spl chunker.Splitter) *DagBuilderHelper {
75 - db := &DagBuilderHelper{
76 - dserv: dbp.Dagserv,
77 - spl: spl,
78 - rawLeaves: dbp.RawLeaves,
79 - prefix: dbp.Prefix,
80 - maxlinks: dbp.Maxlinks,
81 - batch: ipld.NewBatch(context.TODO(), dbp.Dagserv),
82 - }
83 - if fi, ok := spl.Reader().(files.FileInfo); dbp.NoCopy && ok {
84 - db.fullPath = fi.AbsPath()
85 - db.stat = fi.Stat()
86 - }
87 -
88 - if dbp.URL != "" && dbp.NoCopy {
89 - db.fullPath = dbp.URL
90 - }
91 - return db
92 -}
93 -
94 -// prepareNext consumes the next item from the splitter and puts it
95 -// in the nextData field. it is idempotent-- if nextData is full
96 -// it will do nothing.
97 -func (db *DagBuilderHelper) prepareNext() {
98 - // if we already have data waiting to be consumed, we're ready
99 - if db.nextData != nil || db.recvdErr != nil {
100 - return
101 - }
102 -
103 - db.nextData, db.recvdErr = db.spl.NextBytes()
104 - if db.recvdErr == io.EOF {
105 - db.recvdErr = nil
106 - }
107 -}
108 -
109 -// Done returns whether or not we're done consuming the incoming data.
110 -func (db *DagBuilderHelper) Done() bool {
111 - // ensure we have an accurate perspective on data
112 - // as `done` this may be called before `next`.
113 - db.prepareNext() // idempotent
114 - if db.recvdErr != nil {
115 - return false
116 - }
117 - return db.nextData == nil
118 -}
119 -
120 -// Next returns the next chunk of data to be inserted into the dag
121 -// if it returns nil, that signifies that the stream is at an end, and
122 -// that the current building operation should finish.
123 -func (db *DagBuilderHelper) Next() ([]byte, error) {
124 - db.prepareNext() // idempotent
125 - d := db.nextData
126 - db.nextData = nil // signal we've consumed it
127 - if db.recvdErr != nil {
128 - return nil, db.recvdErr
129 - }
130 - return d, nil
131 -}
132 -
133 -// GetDagServ returns the dagservice object this Helper is using
134 -func (db *DagBuilderHelper) GetDagServ() ipld.DAGService {
135 - return db.dserv
136 -}
137 -
138 -// NewUnixfsNode creates a new Unixfs node to represent a file.
139 -func (db *DagBuilderHelper) NewUnixfsNode() *UnixfsNode {
140 - n := &UnixfsNode{
141 - node: new(dag.ProtoNode),
142 - ufmt: ft.NewFSNode(ft.TFile),
143 - }
144 - n.SetPrefix(db.prefix)
145 - return n
146 -}
147 -
148 -// GetPrefix returns the internal `cid.Prefix` set in the builder.
149 -func (db *DagBuilderHelper) GetPrefix() *cid.Prefix {
150 - return db.prefix
151 -}
152 -
153 -// NewLeaf creates a leaf node filled with data. If rawLeaves is
154 -// defined than a raw leaf will be returned. Otherwise, if data is
155 -// nil the type field will be TRaw (for backwards compatibility), if
156 -// data is defined (but possibly empty) the type field will be TRaw.
157 -func (db *DagBuilderHelper) NewLeaf(data []byte) (*UnixfsNode, error) {
158 - if len(data) > BlockSizeLimit {
159 - return nil, ErrSizeLimitExceeded
160 - }
161 -
162 - if db.rawLeaves {
163 - if db.prefix == nil {
164 - return &UnixfsNode{
165 - rawnode: dag.NewRawNode(data),
166 - raw: true,
167 - }, nil
168 - }
169 - rawnode, err := dag.NewRawNodeWPrefix(data, *db.prefix)
170 - if err != nil {
171 - return nil, err
172 - }
173 - return &UnixfsNode{
174 - rawnode: rawnode,
175 - raw: true,
176 - }, nil
177 - }
178 -
179 - if data == nil {
180 - return db.NewUnixfsNode(), nil
181 - }
182 -
183 - blk := db.newUnixfsBlock()
184 - blk.SetData(data)
185 - return blk, nil
186 -}
187 -
188 -// NewLeafNode is a variation from `NewLeaf` (see its description) that
189 -// returns an `ipld.Node` instead.
190 -func (db *DagBuilderHelper) NewLeafNode(data []byte) (ipld.Node, error) {
191 - if len(data) > BlockSizeLimit {
192 - return nil, ErrSizeLimitExceeded
193 - }
194 -
195 - if db.rawLeaves {
196 - // Encapsulate the data in a raw node.
197 - if db.prefix == nil {
198 - return dag.NewRawNode(data), nil
199 - }
200 - rawnode, err := dag.NewRawNodeWPrefix(data, *db.prefix)
201 - if err != nil {
202 - return nil, err
203 - }
204 - return rawnode, nil
205 - }
206 -
207 - // Encapsulate the data in UnixFS node (instead of a raw node).
208 - fsNodeOverDag := db.NewFSNodeOverDag(ft.TFile)
209 - fsNodeOverDag.SetFileData(data)
210 - node, err := fsNodeOverDag.Commit()
211 - if err != nil {
212 - return nil, err
213 - }
214 - // TODO: Encapsulate this sequence of calls into a function that
215 - // just returns the final `ipld.Node` avoiding going through
216 - // `FSNodeOverDag`.
217 - // TODO: Using `TFile` for backwards-compatibility, a bug in the
218 - // balanced builder was causing the leaf nodes to be generated
219 - // with this type instead of `TRaw`, the one that should be used
220 - // (like the trickle builder does).
221 - // (See https://github.com/ipfs/go-ipfs/pull/5120.)
222 -
223 - return node, nil
224 -}
225 -
226 -// newUnixfsBlock creates a new Unixfs node to represent a raw data block
227 -func (db *DagBuilderHelper) newUnixfsBlock() *UnixfsNode {
228 - n := &UnixfsNode{
229 - node: new(dag.ProtoNode),
230 - ufmt: ft.NewFSNode(ft.TRaw),
231 - }
232 - n.SetPrefix(db.prefix)
233 - return n
234 -}
235 -
236 -// FillNodeLayer will add datanodes as children to the give node until
237 -// at most db.indirSize nodes are added.
238 -func (db *DagBuilderHelper) FillNodeLayer(node *UnixfsNode) error {
239 -
240 - // while we have room AND we're not done
241 - for node.NumChildren() < db.maxlinks && !db.Done() {
242 - child, err := db.GetNextDataNode()
243 - if err != nil {
244 - return err
245 - }
246 -
247 - if err := node.AddChild(child, db); err != nil {
248 - return err
249 - }
250 - }
251 -
252 - return nil
253 -}
254 -
255 -// GetNextDataNode builds a UnixFsNode with the data obtained from the
256 -// Splitter, given the constraints (BlockSizeLimit, RawLeaves) specified
257 -// when creating the DagBuilderHelper.
258 -func (db *DagBuilderHelper) GetNextDataNode() (*UnixfsNode, error) {
259 - data, err := db.Next()
260 - if err != nil {
261 - return nil, err
262 - }
263 -
264 - if data == nil { // we're done!
265 - return nil, nil
266 - }
267 -
268 - return db.NewLeaf(data)
269 -}
270 -
271 -// NewLeafDataNode is a variation of `GetNextDataNode` that returns
272 -// an `ipld.Node` instead. It builds the `node` with the data obtained
273 -// from the Splitter and returns it with the `dataSize` (that will be
274 -// used to keep track of the DAG file size). The size of the data is
275 -// computed here because after that it will be hidden by `NewLeafNode`
276 -// inside a generic `ipld.Node` representation.
277 -func (db *DagBuilderHelper) NewLeafDataNode() (node ipld.Node, dataSize uint64, err error) {
278 - fileData, err := db.Next()
279 - if err != nil {
280 - return nil, 0, err
281 - }
282 - dataSize = uint64(len(fileData))
283 -
284 - // Create a new leaf node containing the file chunk data.
285 - node, err = db.NewLeafNode(fileData)
286 - if err != nil {
287 - return nil, 0, err
288 - }
289 -
290 - // Convert this leaf to a `FilestoreNode` if needed.
291 - node = db.ProcessFileStore(node, dataSize)
292 -
293 - return node, dataSize, nil
294 -}
295 -
296 -// ProcessFileStore generates, if Filestore is being used, the
297 -// `FilestoreNode` representation of the `ipld.Node` that
298 -// contains the file data. If Filestore is not being used just
299 -// return the same node to continue with its addition to the DAG.
300 -//
301 -// The `db.offset` is updated at this point (instead of when
302 -// `NewLeafDataNode` is called, both work in tandem but the
303 -// offset is more related to this function).
304 -func (db *DagBuilderHelper) ProcessFileStore(node ipld.Node, dataSize uint64) ipld.Node {
305 - // Check if Filestore is being used.
306 - if db.fullPath != "" {
307 - // Check if the node is actually a raw node (needed for
308 - // Filestore support).
309 - if _, ok := node.(*dag.RawNode); ok {
310 - fn := &pi.FilestoreNode{
311 - Node: node,
312 - PosInfo: &pi.PosInfo{
313 - Offset: db.offset,
314 - FullPath: db.fullPath,
315 - Stat: db.stat,
316 - },
317 - }
318 -
319 - // Update `offset` with the size of the data generated by `db.Next`.
320 - db.offset += dataSize
321 -
322 - return fn
323 - }
324 - }
325 -
326 - // Filestore is not used, return the same `node` argument.
327 - return node
328 -}
329 -
330 -// Add sends a node to the DAGService, and returns it.
331 -func (db *DagBuilderHelper) Add(node *UnixfsNode) (ipld.Node, error) {
332 - dn, err := node.GetDagNode()
333 - if err != nil {
334 - return nil, err
335 - }
336 -
337 - err = db.dserv.Add(context.TODO(), dn)
338 - if err != nil {
339 - return nil, err
340 - }
341 -
342 - return dn, nil
343 -}
344 -
345 -// Maxlinks returns the configured maximum number for links
346 -// for nodes built with this helper.
347 -func (db *DagBuilderHelper) Maxlinks() int {
348 - return db.maxlinks
349 -}
350 -
351 -// Close has the DAGService perform a batch Commit operation.
352 -// It should be called at the end of the building process to make
353 -// sure all data is persisted.
354 -func (db *DagBuilderHelper) Close() error {
355 - return db.batch.Commit()
356 -}
357 -
358 -// AddNodeAndClose adds the last `ipld.Node` from the DAG and
359 -// closes the builder. It returns the same `node` passed as
360 -// argument.
361 -func (db *DagBuilderHelper) AddNodeAndClose(node ipld.Node) (ipld.Node, error) {
362 - err := db.batch.Add(node)
363 - if err != nil {
364 - return nil, err
365 - }
366 -
367 - err = db.Close()
368 - if err != nil {
369 - return nil, err
370 - }
371 -
372 - return node, nil
373 -}
374 -
375 -// FSNodeOverDag encapsulates an `unixfs.FSNode` that will be stored in a
376 -// `dag.ProtoNode`. Instead of just having a single `ipld.Node` that
377 -// would need to be constantly (un)packed to access and modify its
378 -// internal `FSNode` in the process of creating a UnixFS DAG, this
379 -// structure stores an `FSNode` cache to manipulate it (add child nodes)
380 -// directly , and only when the node has reached its final (immutable) state
381 -// (signaled by calling `Commit()`) is it committed to a single (indivisible)
382 -// `ipld.Node`.
383 -//
384 -// It is used mainly for internal (non-leaf) nodes, and for some
385 -// representations of data leaf nodes (that don't use raw nodes or
386 -// Filestore).
387 -//
388 -// It aims to replace the `UnixfsNode` structure which encapsulated too
389 -// many possible node state combinations.
390 -//
391 -// TODO: Revisit the name.
392 -type FSNodeOverDag struct {
393 - dag *dag.ProtoNode
394 - file *ft.FSNode
395 -}
396 -
397 -// NewFSNodeOverDag creates a new `dag.ProtoNode` and `ft.FSNode`
398 -// decoupled from one onther (and will continue in that way until
399 -// `Commit` is called), with `fsNodeType` specifying the type of
400 -// the UnixFS layer node (either `File` or `Raw`).
401 -func (db *DagBuilderHelper) NewFSNodeOverDag(fsNodeType pb.Data_DataType) *FSNodeOverDag {
402 - node := new(FSNodeOverDag)
403 - node.dag = new(dag.ProtoNode)
404 - node.dag.SetPrefix(db.GetPrefix())
405 -
406 - node.file = ft.NewFSNode(fsNodeType)
407 -
408 - return node
409 -}
410 -
411 -// AddChild adds a `child` `ipld.Node` to both node layers. The
412 -// `dag.ProtoNode` creates a link to the child node while the
413 -// `ft.FSNode` stores its file size (that is, not the size of the
414 -// node but the size of the file data that it is storing at the
415 -// UnixFS layer). The child is also stored in the `DAGService`.
416 -func (n *FSNodeOverDag) AddChild(child ipld.Node, fileSize uint64, db *DagBuilderHelper) error {
417 - err := n.dag.AddNodeLink("", child)
418 - if err != nil {
419 - return err
420 - }
421 -
422 - n.file.AddBlockSize(fileSize)
423 -
424 - return db.batch.Add(child)
425 -}
426 -
427 -// Commit unifies (resolves) the cache nodes into a single `ipld.Node`
428 -// that represents them: the `ft.FSNode` is encoded inside the
429 -// `dag.ProtoNode`.
430 -//
431 -// TODO: Evaluate making it read-only after committing.
432 -func (n *FSNodeOverDag) Commit() (ipld.Node, error) {
433 - fileData, err := n.file.GetBytes()
434 - if err != nil {
435 - return nil, err
436 - }
437 - n.dag.SetData(fileData)
438 -
439 - return n.dag, nil
440 -}
441 -
442 -// NumChildren returns the number of children of the `ft.FSNode`.
443 -func (n *FSNodeOverDag) NumChildren() int {
444 - return n.file.NumChildren()
445 -}
446 -
447 -// FileSize returns the `Filesize` attribute from the underlying
448 -// representation of the `ft.FSNode`.
449 -func (n *FSNodeOverDag) FileSize() uint64 {
450 - return n.file.FileSize()
451 -}
452 -
453 -// SetFileData stores the `fileData` in the `ft.FSNode`. It
454 -// should be used only when `FSNodeOverDag` represents a leaf
455 -// node (internal nodes don't carry data, just file sizes).
456 -func (n *FSNodeOverDag) SetFileData(fileData []byte) {
457 - n.file.SetData(fileData)
458 -}
unixfs/importer/helpers/helpers.go deleted
-173
@@ -1,173 +0,0 @@
1 -package helpers
2 -
3 -import (
4 - "context"
5 - "fmt"
6 - "os"
7 -
8 - dag "gx/ipfs/QmRy4Qk9hbgFX9NGJRm8rBThrA8PZhNCitMgeRYyZ67s59/go-merkledag"
9 - ft "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs"
10 -
11 - pi "gx/ipfs/QmSHjPDw8yNgLZ7cBfX7w3Smn7PHwYhNEpd4LHQQxUg35L/go-ipfs-posinfo"
12 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
13 - ipld "gx/ipfs/QmZtNq8dArGfnpCZfx2pUNY7UcjGhVp5qqwQ4hH6mpTMRQ/go-ipld-format"
14 -)
15 -
16 -// BlockSizeLimit specifies the maximum size an imported block can have.
17 -var BlockSizeLimit = 1048576 // 1 MB
18 -
19 -// rough estimates on expected sizes
20 -var roughLinkBlockSize = 1 << 13 // 8KB
21 -var roughLinkSize = 34 + 8 + 5 // sha256 multihash + size + no name + protobuf framing
22 -
23 -// DefaultLinksPerBlock governs how the importer decides how many links there
24 -// will be per block. This calculation is based on expected distributions of:
25 -// * the expected distribution of block sizes
26 -// * the expected distribution of link sizes
27 -// * desired access speed
28 -// For now, we use:
29 -//
30 -// var roughLinkBlockSize = 1 << 13 // 8KB
31 -// var roughLinkSize = 288 // sha256 + framing + name
32 -// var DefaultLinksPerBlock = (roughLinkBlockSize / roughLinkSize)
33 -//
34 -// See calc_test.go
35 -var DefaultLinksPerBlock = roughLinkBlockSize / roughLinkSize
36 -
37 -// ErrSizeLimitExceeded signals that a block is larger than BlockSizeLimit.
38 -var ErrSizeLimitExceeded = fmt.Errorf("object size limit exceeded")
39 -
40 -// UnixfsNode is a struct created to aid in the generation
41 -// of unixfs DAG trees
42 -type UnixfsNode struct {
43 - raw bool
44 - rawnode *dag.RawNode
45 - node *dag.ProtoNode
46 - ufmt *ft.FSNode
47 - posInfo *pi.PosInfo
48 -}
49 -
50 -// NewUnixfsNodeFromDag reconstructs a Unixfs node from a given dag node
51 -func NewUnixfsNodeFromDag(nd *dag.ProtoNode) (*UnixfsNode, error) {
52 - mb, err := ft.FSNodeFromBytes(nd.Data())
53 - if err != nil {
54 - return nil, err
55 - }
56 -
57 - return &UnixfsNode{
58 - node: nd,
59 - ufmt: mb,
60 - }, nil
61 -}
62 -
63 -// SetPrefix sets the CID Prefix
64 -func (n *UnixfsNode) SetPrefix(prefix *cid.Prefix) {
65 - n.node.SetPrefix(prefix)
66 -}
67 -
68 -// NumChildren returns the number of children referenced by this UnixfsNode.
69 -func (n *UnixfsNode) NumChildren() int {
70 - return n.ufmt.NumChildren()
71 -}
72 -
73 -// GetChild gets the ith child of this node from the given DAGService.
74 -func (n *UnixfsNode) GetChild(ctx context.Context, i int, ds ipld.DAGService) (*UnixfsNode, error) {
75 - nd, err := n.node.Links()[i].GetNode(ctx, ds)
76 - if err != nil {
77 - return nil, err
78 - }
79 -
80 - pbn, ok := nd.(*dag.ProtoNode)
81 - if !ok {
82 - return nil, dag.ErrNotProtobuf
83 - }
84 -
85 - return NewUnixfsNodeFromDag(pbn)
86 -}
87 -
88 -// AddChild adds the given UnixfsNode as a child of the receiver.
89 -// The passed in DagBuilderHelper is used to store the child node an
90 -// pin it locally so it doesnt get lost.
91 -func (n *UnixfsNode) AddChild(child *UnixfsNode, db *DagBuilderHelper) error {
92 - n.ufmt.AddBlockSize(child.FileSize())
93 -
94 - childnode, err := child.GetDagNode()
95 - if err != nil {
96 - return err
97 - }
98 -
99 - // Add a link to this node without storing a reference to the memory
100 - // This way, we avoid nodes building up and consuming all of our RAM
101 - err = n.node.AddNodeLink("", childnode)
102 - if err != nil {
103 - return err
104 - }
105 -
106 - err = db.batch.Add(childnode)
107 -
108 - return err
109 -}
110 -
111 -// RemoveChild deletes the child node at the given index.
112 -func (n *UnixfsNode) RemoveChild(index int, dbh *DagBuilderHelper) {
113 - n.ufmt.RemoveBlockSize(index)
114 - n.node.SetLinks(append(n.node.Links()[:index], n.node.Links()[index+1:]...))
115 -}
116 -
117 -// SetData stores data in this node.
118 -func (n *UnixfsNode) SetData(data []byte) {
119 - n.ufmt.SetData(data)
120 -}
121 -
122 -// FileSize returns the total file size of this tree (including children)
123 -// In the case of raw nodes, it returns the length of the
124 -// raw data.
125 -func (n *UnixfsNode) FileSize() uint64 {
126 - if n.raw {
127 - return uint64(len(n.rawnode.RawData()))
128 - }
129 - return n.ufmt.FileSize()
130 -}
131 -
132 -// SetPosInfo sets information about the offset of the data of this node in a
133 -// filesystem file.
134 -func (n *UnixfsNode) SetPosInfo(offset uint64, fullPath string, stat os.FileInfo) {
135 - n.posInfo = &pi.PosInfo{
136 - Offset: offset,
137 - FullPath: fullPath,
138 - Stat: stat,
139 - }
140 -}
141 -
142 -// GetDagNode fills out the proper formatting for the unixfs node
143 -// inside of a DAG node and returns the dag node.
144 -func (n *UnixfsNode) GetDagNode() (ipld.Node, error) {
145 - nd, err := n.getBaseDagNode()
146 - if err != nil {
147 - return nil, err
148 - }
149 -
150 - if n.posInfo != nil {
151 - if rn, ok := nd.(*dag.RawNode); ok {
152 - return &pi.FilestoreNode{
153 - Node: rn,
154 - PosInfo: n.posInfo,
155 - }, nil
156 - }
157 - }
158 -
159 - return nd, nil
160 -}
161 -
162 -func (n *UnixfsNode) getBaseDagNode() (ipld.Node, error) {
163 - if n.raw {
164 - return n.rawnode, nil
165 - }
166 -
167 - data, err := n.ufmt.GetBytes()
168 - if err != nil {
169 - return nil, err
170 - }
171 - n.node.SetData(data)
172 - return n.node, nil
173 -}
unixfs/importer/importer.go deleted
-34
@@ -1,34 +0,0 @@
1 -// Package importer implements utilities used to create IPFS DAGs from files
2 -// and readers.
3 -package importer
4 -
5 -import (
6 - chunker "gx/ipfs/QmVDjhUMtkRskBFAVNwyXuLSKbeAya7JKPnzAxMKDaK4x4/go-ipfs-chunker"
7 - ipld "gx/ipfs/QmZtNq8dArGfnpCZfx2pUNY7UcjGhVp5qqwQ4hH6mpTMRQ/go-ipld-format"
8 -
9 - bal "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/importer/balanced"
10 - h "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/importer/helpers"
11 - trickle "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/importer/trickle"
12 -)
13 -
14 -// BuildDagFromReader creates a DAG given a DAGService and a Splitter
15 -// implementation (Splitters are io.Readers), using a Balanced layout.
16 -func BuildDagFromReader(ds ipld.DAGService, spl chunker.Splitter) (ipld.Node, error) {
17 - dbp := h.DagBuilderParams{
18 - Dagserv: ds,
19 - Maxlinks: h.DefaultLinksPerBlock,
20 - }
21 -
22 - return bal.Layout(dbp.New(spl))
23 -}
24 -
25 -// BuildTrickleDagFromReader creates a DAG given a DAGService and a Splitter
26 -// implementation (Splitters are io.Readers), using a Trickle Layout.
27 -func BuildTrickleDagFromReader(ds ipld.DAGService, spl chunker.Splitter) (ipld.Node, error) {
28 - dbp := h.DagBuilderParams{
29 - Dagserv: ds,
30 - Maxlinks: h.DefaultLinksPerBlock,
31 - }
32 -
33 - return trickle.Layout(dbp.New(spl))
34 -}
unixfs/importer/importer_test.go deleted
-118
@@ -1,118 +0,0 @@
1 -package importer
2 -
3 -import (
4 - "bytes"
5 - "context"
6 - "io"
7 - "io/ioutil"
8 - "testing"
9 -
10 - mdtest "gx/ipfs/QmRy4Qk9hbgFX9NGJRm8rBThrA8PZhNCitMgeRYyZ67s59/go-merkledag/test"
11 - uio "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/io"
12 -
13 - u "gx/ipfs/QmPdKqUcHGFdeSpvjVoaTRPPstGif9GBZb5Q56RVw9o69A/go-ipfs-util"
14 - chunker "gx/ipfs/QmVDjhUMtkRskBFAVNwyXuLSKbeAya7JKPnzAxMKDaK4x4/go-ipfs-chunker"
15 - ipld "gx/ipfs/QmZtNq8dArGfnpCZfx2pUNY7UcjGhVp5qqwQ4hH6mpTMRQ/go-ipld-format"
16 -)
17 -
18 -func getBalancedDag(t testing.TB, size int64, blksize int64) (ipld.Node, ipld.DAGService) {
19 - ds := mdtest.Mock()
20 - r := io.LimitReader(u.NewTimeSeededRand(), size)
21 - nd, err := BuildDagFromReader(ds, chunker.NewSizeSplitter(r, blksize))
22 - if err != nil {
23 - t.Fatal(err)
24 - }
25 - return nd, ds
26 -}
27 -
28 -func getTrickleDag(t testing.TB, size int64, blksize int64) (ipld.Node, ipld.DAGService) {
29 - ds := mdtest.Mock()
30 - r := io.LimitReader(u.NewTimeSeededRand(), size)
31 - nd, err := BuildTrickleDagFromReader(ds, chunker.NewSizeSplitter(r, blksize))
32 - if err != nil {
33 - t.Fatal(err)
34 - }
35 - return nd, ds
36 -}
37 -
38 -func TestBalancedDag(t *testing.T) {
39 - ds := mdtest.Mock()
40 - buf := make([]byte, 10000)
41 - u.NewTimeSeededRand().Read(buf)
42 - r := bytes.NewReader(buf)
43 -
44 - nd, err := BuildDagFromReader(ds, chunker.DefaultSplitter(r))
45 - if err != nil {
46 - t.Fatal(err)
47 - }
48 -
49 - dr, err := uio.NewDagReader(context.Background(), nd, ds)
50 - if err != nil {
51 - t.Fatal(err)
52 - }
53 -
54 - out, err := ioutil.ReadAll(dr)
55 - if err != nil {
56 - t.Fatal(err)
57 - }
58 -
59 - if !bytes.Equal(out, buf) {
60 - t.Fatal("bad read")
61 - }
62 -}
63 -
64 -func BenchmarkBalancedReadSmallBlock(b *testing.B) {
65 - b.StopTimer()
66 - nbytes := int64(10000000)
67 - nd, ds := getBalancedDag(b, nbytes, 4096)
68 -
69 - b.SetBytes(nbytes)
70 - b.StartTimer()
71 - runReadBench(b, nd, ds)
72 -}
73 -
74 -func BenchmarkTrickleReadSmallBlock(b *testing.B) {
75 - b.StopTimer()
76 - nbytes := int64(10000000)
77 - nd, ds := getTrickleDag(b, nbytes, 4096)
78 -
79 - b.SetBytes(nbytes)
80 - b.StartTimer()
81 - runReadBench(b, nd, ds)
82 -}
83 -
84 -func BenchmarkBalancedReadFull(b *testing.B) {
85 - b.StopTimer()
86 - nbytes := int64(10000000)
87 - nd, ds := getBalancedDag(b, nbytes, chunker.DefaultBlockSize)
88 -
89 - b.SetBytes(nbytes)
90 - b.StartTimer()
91 - runReadBench(b, nd, ds)
92 -}
93 -
94 -func BenchmarkTrickleReadFull(b *testing.B) {
95 - b.StopTimer()
96 - nbytes := int64(10000000)
97 - nd, ds := getTrickleDag(b, nbytes, chunker.DefaultBlockSize)
98 -
99 - b.SetBytes(nbytes)
100 - b.StartTimer()
101 - runReadBench(b, nd, ds)
102 -}
103 -
104 -func runReadBench(b *testing.B, nd ipld.Node, ds ipld.DAGService) {
105 - for i := 0; i < b.N; i++ {
106 - ctx, cancel := context.WithCancel(context.Background())
107 - read, err := uio.NewDagReader(ctx, nd, ds)
108 - if err != nil {
109 - b.Fatal(err)
110 - }
111 -
112 - _, err = read.WriteTo(ioutil.Discard)
113 - if err != nil && err != io.EOF {
114 - b.Fatal(err)
115 - }
116 - cancel()
117 - }
118 -}
unixfs/importer/trickle/trickle_test.go deleted
-640
@@ -1,640 +0,0 @@
1 -package trickle
2 -
3 -import (
4 - "bytes"
5 - "context"
6 - "fmt"
7 - "io"
8 - "io/ioutil"
9 - mrand "math/rand"
10 - "testing"
11 -
12 - merkledag "gx/ipfs/QmRy4Qk9hbgFX9NGJRm8rBThrA8PZhNCitMgeRYyZ67s59/go-merkledag"
13 - mdtest "gx/ipfs/QmRy4Qk9hbgFX9NGJRm8rBThrA8PZhNCitMgeRYyZ67s59/go-merkledag/test"
14 - ft "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs"
15 - h "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/importer/helpers"
16 - uio "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/io"
17 -
18 - u "gx/ipfs/QmPdKqUcHGFdeSpvjVoaTRPPstGif9GBZb5Q56RVw9o69A/go-ipfs-util"
19 - chunker "gx/ipfs/QmVDjhUMtkRskBFAVNwyXuLSKbeAya7JKPnzAxMKDaK4x4/go-ipfs-chunker"
20 - ipld "gx/ipfs/QmZtNq8dArGfnpCZfx2pUNY7UcjGhVp5qqwQ4hH6mpTMRQ/go-ipld-format"
21 -)
22 -
23 -type UseRawLeaves bool
24 -
25 -const (
26 - ProtoBufLeaves UseRawLeaves = false
27 - RawLeaves UseRawLeaves = true
28 -)
29 -
30 -func runBothSubtests(t *testing.T, tfunc func(*testing.T, UseRawLeaves)) {
31 - t.Run("leaves=ProtoBuf", func(t *testing.T) { tfunc(t, ProtoBufLeaves) })
32 - t.Run("leaves=Raw", func(t *testing.T) { tfunc(t, RawLeaves) })
33 -}
34 -
35 -func buildTestDag(ds ipld.DAGService, spl chunker.Splitter, rawLeaves UseRawLeaves) (*merkledag.ProtoNode, error) {
36 - dbp := h.DagBuilderParams{
37 - Dagserv: ds,
38 - Maxlinks: h.DefaultLinksPerBlock,
39 - RawLeaves: bool(rawLeaves),
40 - }
41 -
42 - nd, err := Layout(dbp.New(spl))
43 - if err != nil {
44 - return nil, err
45 - }
46 -
47 - pbnd, ok := nd.(*merkledag.ProtoNode)
48 - if !ok {
49 - return nil, merkledag.ErrNotProtobuf
50 - }
51 -
52 - return pbnd, VerifyTrickleDagStructure(pbnd, VerifyParams{
53 - Getter: ds,
54 - Direct: dbp.Maxlinks,
55 - LayerRepeat: layerRepeat,
56 - RawLeaves: bool(rawLeaves),
57 - })
58 -}
59 -
60 -//Test where calls to read are smaller than the chunk size
61 -func TestSizeBasedSplit(t *testing.T) {
62 - runBothSubtests(t, testSizeBasedSplit)
63 -}
64 -
65 -func testSizeBasedSplit(t *testing.T, rawLeaves UseRawLeaves) {
66 - if testing.Short() {
67 - t.SkipNow()
68 - }
69 - bs := chunker.SizeSplitterGen(512)
70 - testFileConsistency(t, bs, 32*512, rawLeaves)
71 -
72 - bs = chunker.SizeSplitterGen(4096)
73 - testFileConsistency(t, bs, 32*4096, rawLeaves)
74 -
75 - // Uneven offset
76 - testFileConsistency(t, bs, 31*4095, rawLeaves)
77 -}
78 -
79 -func dup(b []byte) []byte {
80 - o := make([]byte, len(b))
81 - copy(o, b)
82 - return o
83 -}
84 -
85 -func testFileConsistency(t *testing.T, bs chunker.SplitterGen, nbytes int, rawLeaves UseRawLeaves) {
86 - should := make([]byte, nbytes)
87 - u.NewTimeSeededRand().Read(should)
88 -
89 - read := bytes.NewReader(should)
90 - ds := mdtest.Mock()
91 - nd, err := buildTestDag(ds, bs(read), rawLeaves)
92 - if err != nil {
93 - t.Fatal(err)
94 - }
95 -
96 - r, err := uio.NewDagReader(context.Background(), nd, ds)
97 - if err != nil {
98 - t.Fatal(err)
99 - }
100 -
101 - out, err := ioutil.ReadAll(r)
102 - if err != nil {
103 - t.Fatal(err)
104 - }
105 -
106 - err = arrComp(out, should)
107 - if err != nil {
108 - t.Fatal(err)
109 - }
110 -}
111 -
112 -func TestBuilderConsistency(t *testing.T) {
113 - runBothSubtests(t, testBuilderConsistency)
114 -}
115 -
116 -func testBuilderConsistency(t *testing.T, rawLeaves UseRawLeaves) {
117 - nbytes := 100000
118 - buf := new(bytes.Buffer)
119 - io.CopyN(buf, u.NewTimeSeededRand(), int64(nbytes))
120 - should := dup(buf.Bytes())
121 - dagserv := mdtest.Mock()
122 - nd, err := buildTestDag(dagserv, chunker.DefaultSplitter(buf), rawLeaves)
123 - if err != nil {
124 - t.Fatal(err)
125 - }
126 - r, err := uio.NewDagReader(context.Background(), nd, dagserv)
127 - if err != nil {
128 - t.Fatal(err)
129 - }
130 -
131 - out, err := ioutil.ReadAll(r)
132 - if err != nil {
133 - t.Fatal(err)
134 - }
135 -
136 - err = arrComp(out, should)
137 - if err != nil {
138 - t.Fatal(err)
139 - }
140 -}
141 -
142 -func arrComp(a, b []byte) error {
143 - if len(a) != len(b) {
144 - return fmt.Errorf("arrays differ in length. %d != %d", len(a), len(b))
145 - }
146 - for i, v := range a {
147 - if v != b[i] {
148 - return fmt.Errorf("arrays differ at index: %d", i)
149 - }
150 - }
151 - return nil
152 -}
153 -
154 -func TestIndirectBlocks(t *testing.T) {
155 - runBothSubtests(t, testIndirectBlocks)
156 -}
157 -
158 -func testIndirectBlocks(t *testing.T, rawLeaves UseRawLeaves) {
159 - splitter := chunker.SizeSplitterGen(512)
160 - nbytes := 1024 * 1024
161 - buf := make([]byte, nbytes)
162 - u.NewTimeSeededRand().Read(buf)
163 -
164 - read := bytes.NewReader(buf)
165 -
166 - ds := mdtest.Mock()
167 - dag, err := buildTestDag(ds, splitter(read), rawLeaves)
168 - if err != nil {
169 - t.Fatal(err)
170 - }
171 -
172 - reader, err := uio.NewDagReader(context.Background(), dag, ds)
173 - if err != nil {
174 - t.Fatal(err)
175 - }
176 -
177 - out, err := ioutil.ReadAll(reader)
178 - if err != nil {
179 - t.Fatal(err)
180 - }
181 -
182 - if !bytes.Equal(out, buf) {
183 - t.Fatal("Not equal!")
184 - }
185 -}
186 -
187 -func TestSeekingBasic(t *testing.T) {
188 - runBothSubtests(t, testSeekingBasic)
189 -}
190 -
191 -func testSeekingBasic(t *testing.T, rawLeaves UseRawLeaves) {
192 - nbytes := int64(10 * 1024)
193 - should := make([]byte, nbytes)
194 - u.NewTimeSeededRand().Read(should)
195 -
196 - read := bytes.NewReader(should)
197 - ds := mdtest.Mock()
198 - nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 512), rawLeaves)
199 - if err != nil {
200 - t.Fatal(err)
201 - }
202 -
203 - rs, err := uio.NewDagReader(context.Background(), nd, ds)
204 - if err != nil {
205 - t.Fatal(err)
206 - }
207 -
208 - start := int64(4000)
209 - n, err := rs.Seek(start, io.SeekStart)
210 - if err != nil {
211 - t.Fatal(err)
212 - }
213 - if n != start {
214 - t.Fatal("Failed to seek to correct offset")
215 - }
216 -
217 - out, err := ioutil.ReadAll(rs)
218 - if err != nil {
219 - t.Fatal(err)
220 - }
221 -
222 - err = arrComp(out, should[start:])
223 - if err != nil {
224 - t.Fatal(err)
225 - }
226 -}
227 -
228 -func TestSeekToBegin(t *testing.T) {
229 - runBothSubtests(t, testSeekToBegin)
230 -}
231 -
232 -func testSeekToBegin(t *testing.T, rawLeaves UseRawLeaves) {
233 - nbytes := int64(10 * 1024)
234 - should := make([]byte, nbytes)
235 - u.NewTimeSeededRand().Read(should)
236 -
237 - read := bytes.NewReader(should)
238 - ds := mdtest.Mock()
239 - nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 500), rawLeaves)
240 - if err != nil {
241 - t.Fatal(err)
242 - }
243 -
244 - rs, err := uio.NewDagReader(context.Background(), nd, ds)
245 - if err != nil {
246 - t.Fatal(err)
247 - }
248 -
249 - n, err := io.CopyN(ioutil.Discard, rs, 1024*4)
250 - if err != nil {
251 - t.Fatal(err)
252 - }
253 - if n != 4096 {
254 - t.Fatal("Copy didnt copy enough bytes")
255 - }
256 -
257 - seeked, err := rs.Seek(0, io.SeekStart)
258 - if err != nil {
259 - t.Fatal(err)
260 - }
261 - if seeked != 0 {
262 - t.Fatal("Failed to seek to beginning")
263 - }
264 -
265 - out, err := ioutil.ReadAll(rs)
266 - if err != nil {
267 - t.Fatal(err)
268 - }
269 -
270 - err = arrComp(out, should)
271 - if err != nil {
272 - t.Fatal(err)
273 - }
274 -}
275 -
276 -func TestSeekToAlmostBegin(t *testing.T) {
277 - runBothSubtests(t, testSeekToAlmostBegin)
278 -}
279 -
280 -func testSeekToAlmostBegin(t *testing.T, rawLeaves UseRawLeaves) {
281 - nbytes := int64(10 * 1024)
282 - should := make([]byte, nbytes)
283 - u.NewTimeSeededRand().Read(should)
284 -
285 - read := bytes.NewReader(should)
286 - ds := mdtest.Mock()
287 - nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 500), rawLeaves)
288 - if err != nil {
289 - t.Fatal(err)
290 - }
291 -
292 - rs, err := uio.NewDagReader(context.Background(), nd, ds)
293 - if err != nil {
294 - t.Fatal(err)
295 - }
296 -
297 - n, err := io.CopyN(ioutil.Discard, rs, 1024*4)
298 - if err != nil {
299 - t.Fatal(err)
300 - }
301 - if n != 4096 {
302 - t.Fatal("Copy didnt copy enough bytes")
303 - }
304 -
305 - seeked, err := rs.Seek(1, io.SeekStart)
306 - if err != nil {
307 - t.Fatal(err)
308 - }
309 - if seeked != 1 {
310 - t.Fatal("Failed to seek to almost beginning")
311 - }
312 -
313 - out, err := ioutil.ReadAll(rs)
314 - if err != nil {
315 - t.Fatal(err)
316 - }
317 -
318 - err = arrComp(out, should[1:])
319 - if err != nil {
320 - t.Fatal(err)
321 - }
322 -}
323 -
324 -func TestSeekEnd(t *testing.T) {
325 - runBothSubtests(t, testSeekEnd)
326 -}
327 -
328 -func testSeekEnd(t *testing.T, rawLeaves UseRawLeaves) {
329 - nbytes := int64(50 * 1024)
330 - should := make([]byte, nbytes)
331 - u.NewTimeSeededRand().Read(should)
332 -
333 - read := bytes.NewReader(should)
334 - ds := mdtest.Mock()
335 - nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 500), rawLeaves)
336 - if err != nil {
337 - t.Fatal(err)
338 - }
339 -
340 - rs, err := uio.NewDagReader(context.Background(), nd, ds)
341 - if err != nil {
342 - t.Fatal(err)
343 - }
344 -
345 - seeked, err := rs.Seek(0, io.SeekEnd)
346 - if err != nil {
347 - t.Fatal(err)
348 - }
349 - if seeked != nbytes {
350 - t.Fatal("Failed to seek to end")
351 - }
352 -}
353 -
354 -func TestSeekEndSingleBlockFile(t *testing.T) {
355 - runBothSubtests(t, testSeekEndSingleBlockFile)
356 -}
357 -
358 -func testSeekEndSingleBlockFile(t *testing.T, rawLeaves UseRawLeaves) {
359 - nbytes := int64(100)
360 - should := make([]byte, nbytes)
361 - u.NewTimeSeededRand().Read(should)
362 -
363 - read := bytes.NewReader(should)
364 - ds := mdtest.Mock()
365 - nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 5000), rawLeaves)
366 - if err != nil {
367 - t.Fatal(err)
368 - }
369 -
370 - rs, err := uio.NewDagReader(context.Background(), nd, ds)
371 - if err != nil {
372 - t.Fatal(err)
373 - }
374 -
375 - seeked, err := rs.Seek(0, io.SeekEnd)
376 - if err != nil {
377 - t.Fatal(err)
378 - }
379 - if seeked != nbytes {
380 - t.Fatal("Failed to seek to end")
381 - }
382 -}
383 -
384 -func TestSeekingStress(t *testing.T) {
385 - runBothSubtests(t, testSeekingStress)
386 -}
387 -
388 -func testSeekingStress(t *testing.T, rawLeaves UseRawLeaves) {
389 - nbytes := int64(1024 * 1024)
390 - should := make([]byte, nbytes)
391 - u.NewTimeSeededRand().Read(should)
392 -
393 - read := bytes.NewReader(should)
394 - ds := mdtest.Mock()
395 - nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 1000), rawLeaves)
396 - if err != nil {
397 - t.Fatal(err)
398 - }
399 -
400 - rs, err := uio.NewDagReader(context.Background(), nd, ds)
401 - if err != nil {
402 - t.Fatal(err)
403 - }
404 -
405 - testbuf := make([]byte, nbytes)
406 - for i := 0; i < 50; i++ {
407 - offset := mrand.Intn(int(nbytes))
408 - l := int(nbytes) - offset
409 - n, err := rs.Seek(int64(offset), io.SeekStart)
410 - if err != nil {
411 - t.Fatal(err)
412 - }
413 - if n != int64(offset) {
414 - t.Fatal("Seek failed to move to correct position")
415 - }
416 -
417 - nread, err := rs.Read(testbuf[:l])
418 - if err != nil {
419 - t.Fatal(err)
420 - }
421 - if nread != l {
422 - t.Fatal("Failed to read enough bytes")
423 - }
424 -
425 - err = arrComp(testbuf[:l], should[offset:offset+l])
426 - if err != nil {
427 - t.Fatal(err)
428 - }
429 - }
430 -
431 -}
432 -
433 -func TestSeekingConsistency(t *testing.T) {
434 - runBothSubtests(t, testSeekingConsistency)
435 -}
436 -
437 -func testSeekingConsistency(t *testing.T, rawLeaves UseRawLeaves) {
438 - nbytes := int64(128 * 1024)
439 - should := make([]byte, nbytes)
440 - u.NewTimeSeededRand().Read(should)
441 -
442 - read := bytes.NewReader(should)
443 - ds := mdtest.Mock()
444 - nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 500), rawLeaves)
445 - if err != nil {
446 - t.Fatal(err)
447 - }
448 -
449 - rs, err := uio.NewDagReader(context.Background(), nd, ds)
450 - if err != nil {
451 - t.Fatal(err)
452 - }
453 -
454 - out := make([]byte, nbytes)
455 -
456 - for coff := nbytes - 4096; coff >= 0; coff -= 4096 {
457 - t.Log(coff)
458 - n, err := rs.Seek(coff, io.SeekStart)
459 - if err != nil {
460 - t.Fatal(err)
461 - }
462 - if n != coff {
463 - t.Fatal("wasnt able to seek to the right position")
464 - }
465 - nread, err := rs.Read(out[coff : coff+4096])
466 - if err != nil {
467 - t.Fatal(err)
468 - }
469 - if nread != 4096 {
470 - t.Fatal("didnt read the correct number of bytes")
471 - }
472 - }
473 -
474 - err = arrComp(out, should)
475 - if err != nil {
476 - t.Fatal(err)
477 - }
478 -}
479 -
480 -func TestAppend(t *testing.T) {
481 - runBothSubtests(t, testAppend)
482 -}
483 -
484 -func testAppend(t *testing.T, rawLeaves UseRawLeaves) {
485 - nbytes := int64(128 * 1024)
486 - should := make([]byte, nbytes)
487 - u.NewTimeSeededRand().Read(should)
488 -
489 - // Reader for half the bytes
490 - read := bytes.NewReader(should[:nbytes/2])
491 - ds := mdtest.Mock()
492 - nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 500), rawLeaves)
493 - if err != nil {
494 - t.Fatal(err)
495 - }
496 -
497 - dbp := &h.DagBuilderParams{
498 - Dagserv: ds,
499 - Maxlinks: h.DefaultLinksPerBlock,
500 - RawLeaves: bool(rawLeaves),
501 - }
502 -
503 - r := bytes.NewReader(should[nbytes/2:])
504 -
505 - ctx := context.Background()
506 - nnode, err := Append(ctx, nd, dbp.New(chunker.NewSizeSplitter(r, 500)))
507 - if err != nil {
508 - t.Fatal(err)
509 - }
510 -
511 - err = VerifyTrickleDagStructure(nnode, VerifyParams{
512 - Getter: ds,
513 - Direct: dbp.Maxlinks,
514 - LayerRepeat: layerRepeat,
515 - RawLeaves: bool(rawLeaves),
516 - })
517 - if err != nil {
518 - t.Fatal(err)
519 - }
520 -
521 - fread, err := uio.NewDagReader(ctx, nnode, ds)
522 - if err != nil {
523 - t.Fatal(err)
524 - }
525 -
526 - out, err := ioutil.ReadAll(fread)
527 - if err != nil {
528 - t.Fatal(err)
529 - }
530 -
531 - err = arrComp(out, should)
532 - if err != nil {
533 - t.Fatal(err)
534 - }
535 -}
536 -
537 -// This test appends one byte at a time to an empty file
538 -func TestMultipleAppends(t *testing.T) {
539 - runBothSubtests(t, testMultipleAppends)
540 -}
541 -
542 -func testMultipleAppends(t *testing.T, rawLeaves UseRawLeaves) {
543 - ds := mdtest.Mock()
544 -
545 - // TODO: fix small size appends and make this number bigger
546 - nbytes := int64(1000)
547 - should := make([]byte, nbytes)
548 - u.NewTimeSeededRand().Read(should)
549 -
550 - read := bytes.NewReader(nil)
551 - nd, err := buildTestDag(ds, chunker.NewSizeSplitter(read, 500), rawLeaves)
552 - if err != nil {
553 - t.Fatal(err)
554 - }
555 -
556 - dbp := &h.DagBuilderParams{
557 - Dagserv: ds,
558 - Maxlinks: 4,
559 - RawLeaves: bool(rawLeaves),
560 - }
561 -
562 - spl := chunker.SizeSplitterGen(500)
563 -
564 - ctx := context.Background()
565 - for i := 0; i < len(should); i++ {
566 -
567 - nnode, err := Append(ctx, nd, dbp.New(spl(bytes.NewReader(should[i:i+1]))))
568 - if err != nil {
569 - t.Fatal(err)
570 - }
571 -
572 - err = VerifyTrickleDagStructure(nnode, VerifyParams{
573 - Getter: ds,
574 - Direct: dbp.Maxlinks,
575 - LayerRepeat: layerRepeat,
576 - RawLeaves: bool(rawLeaves),
577 - })
578 - if err != nil {
579 - t.Fatal(err)
580 - }
581 -
582 - fread, err := uio.NewDagReader(ctx, nnode, ds)
583 - if err != nil {
584 - t.Fatal(err)
585 - }
586 -
587 - out, err := ioutil.ReadAll(fread)
588 - if err != nil {
589 - t.Fatal(err)
590 - }
591 -
592 - err = arrComp(out, should[:i+1])
593 - if err != nil {
594 - t.Fatal(err)
595 - }
596 - }
597 -}
598 -
599 -func TestAppendSingleBytesToEmpty(t *testing.T) {
600 - ds := mdtest.Mock()
601 -
602 - data := []byte("AB")
603 -
604 - nd := new(merkledag.ProtoNode)
605 - nd.SetData(ft.FilePBData(nil, 0))
606 -
607 - dbp := &h.DagBuilderParams{
608 - Dagserv: ds,
609 - Maxlinks: 4,
610 - }
611 -
612 - spl := chunker.SizeSplitterGen(500)
613 -
614 - ctx := context.Background()
615 - nnode, err := Append(ctx, nd, dbp.New(spl(bytes.NewReader(data[:1]))))
616 - if err != nil {
617 - t.Fatal(err)
618 - }
619 -
620 - nnode, err = Append(ctx, nnode, dbp.New(spl(bytes.NewReader(data[1:]))))
621 - if err != nil {
622 - t.Fatal(err)
623 - }
624 -
625 - fread, err := uio.NewDagReader(ctx, nnode, ds)
626 - if err != nil {
627 - t.Fatal(err)
628 - }
629 -
630 - out, err := ioutil.ReadAll(fread)
631 - if err != nil {
632 - t.Fatal(err)
633 - }
634 -
635 - fmt.Println(out, data)
636 - err = arrComp(out, data)
637 - if err != nil {
638 - t.Fatal(err)
639 - }
640 -}
unixfs/importer/trickle/trickledag.go deleted
-366
@@ -1,366 +0,0 @@
1 -// Package trickle allows to build trickle DAGs.
2 -// In this type of DAG, non-leave nodes are first filled
3 -// with data leaves, and then incorporate "layers" of subtrees
4 -// as additional links.
5 -//
6 -// Each layer is a trickle sub-tree and is limited by an increasing
7 -// maximum depth. Thus, the nodes first layer
8 -// can only hold leaves (depth 1) but subsequent layers can grow deeper.
9 -// By default, this module places 4 nodes per layer (that is, 4 subtrees
10 -// of the same maximum depth before increasing it).
11 -//
12 -// Trickle DAGs are very good for sequentially reading data, as the
13 -// first data leaves are directly reachable from the root and those
14 -// coming next are always nearby. They are
15 -// suited for things like streaming applications.
16 -package trickle
17 -
18 -import (
19 - "context"
20 - "errors"
21 - "fmt"
22 -
23 - dag "gx/ipfs/QmRy4Qk9hbgFX9NGJRm8rBThrA8PZhNCitMgeRYyZ67s59/go-merkledag"
24 - ft "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs"
25 - h "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/importer/helpers"
26 -
27 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
28 - ipld "gx/ipfs/QmZtNq8dArGfnpCZfx2pUNY7UcjGhVp5qqwQ4hH6mpTMRQ/go-ipld-format"
29 -)
30 -
31 -// layerRepeat specifies how many times to append a child tree of a
32 -// given depth. Higher values increase the width of a given node, which
33 -// improves seek speeds.
34 -const layerRepeat = 4
35 -
36 -// Layout builds a new DAG with the trickle format using the provided
37 -// DagBuilderHelper. See the module's description for a more detailed
38 -// explanation.
39 -func Layout(db *h.DagBuilderHelper) (ipld.Node, error) {
40 - root := db.NewUnixfsNode()
41 - if err := fillTrickleRec(db, root, -1); err != nil {
42 - return nil, err
43 - }
44 -
45 - out, err := db.Add(root)
46 - if err != nil {
47 - return nil, err
48 - }
49 -
50 - if err := db.Close(); err != nil {
51 - return nil, err
52 - }
53 -
54 - return out, nil
55 -}
56 -
57 -// fillTrickleRec creates a trickle (sub-)tree with an optional maximum specified depth
58 -// in the case maxDepth is greater than zero, or with unlimited depth otherwise
59 -// (where the DAG builder will signal the end of data to end the function).
60 -func fillTrickleRec(db *h.DagBuilderHelper, node *h.UnixfsNode, maxDepth int) error {
61 - // Always do this, even in the base case
62 - if err := db.FillNodeLayer(node); err != nil {
63 - return err
64 - }
65 -
66 - for depth := 1; ; depth++ {
67 - // Apply depth limit only if the parameter is set (> 0).
68 - if maxDepth > 0 && depth == maxDepth {
69 - return nil
70 - }
71 - for layer := 0; layer < layerRepeat; layer++ {
72 - if db.Done() {
73 - return nil
74 - }
75 -
76 - nextChild := db.NewUnixfsNode()
77 - if err := fillTrickleRec(db, nextChild, depth); err != nil {
78 - return err
79 - }
80 -
81 - if err := node.AddChild(nextChild, db); err != nil {
82 - return err
83 - }
84 - }
85 - }
86 -}
87 -
88 -// Append appends the data in `db` to the dag, using the Trickledag format
89 -func Append(ctx context.Context, basen ipld.Node, db *h.DagBuilderHelper) (out ipld.Node, errOut error) {
90 - base, ok := basen.(*dag.ProtoNode)
91 - if !ok {
92 - return nil, dag.ErrNotProtobuf
93 - }
94 -
95 - defer func() {
96 - if errOut == nil {
97 - if err := db.Close(); err != nil {
98 - errOut = err
99 - }
100 - }
101 - }()
102 -
103 - // Convert to unixfs node for working with easily
104 - ufsn, err := h.NewUnixfsNodeFromDag(base)
105 - if err != nil {
106 - return nil, err
107 - }
108 -
109 - // Get depth of this 'tree'
110 - n, layerProgress := trickleDepthInfo(ufsn, db.Maxlinks())
111 - if n == 0 {
112 - // If direct blocks not filled...
113 - if err := db.FillNodeLayer(ufsn); err != nil {
114 - return nil, err
115 - }
116 -
117 - if db.Done() {
118 - return ufsn.GetDagNode()
119 - }
120 -
121 - // If continuing, our depth has increased by one
122 - n++
123 - }
124 -
125 - // Last child in this node may not be a full tree, lets file it up
126 - if err := appendFillLastChild(ctx, ufsn, n-1, layerProgress, db); err != nil {
127 - return nil, err
128 - }
129 -
130 - // after appendFillLastChild, our depth is now increased by one
131 - if !db.Done() {
132 - n++
133 - }
134 -
135 - // Now, continue filling out tree like normal
136 - for i := n; !db.Done(); i++ {
137 - for j := 0; j < layerRepeat && !db.Done(); j++ {
138 - next := db.NewUnixfsNode()
139 - err := fillTrickleRec(db, next, i)
140 - if err != nil {
141 - return nil, err
142 - }
143 -
144 - err = ufsn.AddChild(next, db)
145 - if err != nil {
146 - return nil, err
147 - }
148 - }
149 - }
150 -
151 - return ufsn.GetDagNode()
152 -}
153 -
154 -// appendFillLastChild will take in an incomplete trickledag node (uncomplete meaning, not full) and
155 -// fill it out to the specified depth with blocks from the given DagBuilderHelper
156 -func appendFillLastChild(ctx context.Context, ufsn *h.UnixfsNode, depth int, layerFill int, db *h.DagBuilderHelper) error {
157 - if ufsn.NumChildren() <= db.Maxlinks() {
158 - return nil
159 - }
160 - // Recursive step, grab last child
161 - last := ufsn.NumChildren() - 1
162 - lastChild, err := ufsn.GetChild(ctx, last, db.GetDagServ())
163 - if err != nil {
164 - return err
165 - }
166 -
167 - // Fill out last child (may not be full tree)
168 - nchild, err := appendRec(ctx, lastChild, db, depth-1)
169 - if err != nil {
170 - return err
171 - }
172 -
173 - // Update changed child in parent node
174 - ufsn.RemoveChild(last, db)
175 - err = ufsn.AddChild(nchild, db)
176 - if err != nil {
177 - return err
178 - }
179 -
180 - // Partially filled depth layer
181 - if layerFill != 0 {
182 - for ; layerFill < layerRepeat && !db.Done(); layerFill++ {
183 - next := db.NewUnixfsNode()
184 - err := fillTrickleRec(db, next, depth)
185 - if err != nil {
186 - return err
187 - }
188 -
189 - err = ufsn.AddChild(next, db)
190 - if err != nil {
191 - return err
192 - }
193 - }
194 - }
195 -
196 - return nil
197 -}
198 -
199 -// recursive call for Append
200 -func appendRec(ctx context.Context, ufsn *h.UnixfsNode, db *h.DagBuilderHelper, depth int) (*h.UnixfsNode, error) {
201 - if depth == 0 || db.Done() {
202 - return ufsn, nil
203 - }
204 -
205 - // Get depth of this 'tree'
206 - n, layerProgress := trickleDepthInfo(ufsn, db.Maxlinks())
207 - if n == 0 {
208 - // If direct blocks not filled...
209 - if err := db.FillNodeLayer(ufsn); err != nil {
210 - return nil, err
211 - }
212 - n++
213 - }
214 -
215 - // If at correct depth, no need to continue
216 - if n == depth {
217 - return ufsn, nil
218 - }
219 -
220 - if err := appendFillLastChild(ctx, ufsn, n, layerProgress, db); err != nil {
221 - return nil, err
222 - }
223 -
224 - // after appendFillLastChild, our depth is now increased by one
225 - if !db.Done() {
226 - n++
227 - }
228 -
229 - // Now, continue filling out tree like normal
230 - for i := n; i < depth && !db.Done(); i++ {
231 - for j := 0; j < layerRepeat && !db.Done(); j++ {
232 - next := db.NewUnixfsNode()
233 - if err := fillTrickleRec(db, next, i); err != nil {
234 - return nil, err
235 - }
236 -
237 - if err := ufsn.AddChild(next, db); err != nil {
238 - return nil, err
239 - }
240 - }
241 - }
242 -
243 - return ufsn, nil
244 -}
245 -
246 -func trickleDepthInfo(node *h.UnixfsNode, maxlinks int) (int, int) {
247 - n := node.NumChildren()
248 - if n < maxlinks {
249 - return 0, 0
250 - }
251 -
252 - return ((n - maxlinks) / layerRepeat) + 1, (n - maxlinks) % layerRepeat
253 -}
254 -
255 -// VerifyParams is used by VerifyTrickleDagStructure
256 -type VerifyParams struct {
257 - Getter ipld.NodeGetter
258 - Direct int
259 - LayerRepeat int
260 - Prefix *cid.Prefix
261 - RawLeaves bool
262 -}
263 -
264 -// VerifyTrickleDagStructure checks that the given dag matches exactly the trickle dag datastructure
265 -// layout
266 -func VerifyTrickleDagStructure(nd ipld.Node, p VerifyParams) error {
267 - return verifyTDagRec(nd, -1, p)
268 -}
269 -
270 -// Recursive call for verifying the structure of a trickledag
271 -func verifyTDagRec(n ipld.Node, depth int, p VerifyParams) error {
272 - codec := cid.DagProtobuf
273 - if depth == 0 {
274 - if len(n.Links()) > 0 {
275 - return errors.New("expected direct block")
276 - }
277 - // zero depth dag is raw data block
278 - switch nd := n.(type) {
279 - case *dag.ProtoNode:
280 - pbn, err := ft.FromBytes(nd.Data())
281 - if err != nil {
282 - return err
283 - }
284 -
285 - if pbn.GetType() != ft.TRaw {
286 - return errors.New("expected raw block")
287 - }
288 -
289 - if p.RawLeaves {
290 - return errors.New("expected raw leaf, got a protobuf node")
291 - }
292 - case *dag.RawNode:
293 - if !p.RawLeaves {
294 - return errors.New("expected protobuf node as leaf")
295 - }
296 - codec = cid.Raw
297 - default:
298 - return errors.New("expected ProtoNode or RawNode")
299 - }
300 - }
301 -
302 - // verify prefix
303 - if p.Prefix != nil {
304 - prefix := n.Cid().Prefix()
305 - expect := *p.Prefix // make a copy
306 - expect.Codec = uint64(codec)
307 - if codec == cid.Raw && expect.Version == 0 {
308 - expect.Version = 1
309 - }
310 - if expect.MhLength == -1 {
311 - expect.MhLength = prefix.MhLength
312 - }
313 - if prefix != expect {
314 - return fmt.Errorf("unexpected cid prefix: expected: %v; got %v", expect, prefix)
315 - }
316 - }
317 -
318 - if depth == 0 {
319 - return nil
320 - }
321 -
322 - nd, ok := n.(*dag.ProtoNode)
323 - if !ok {
324 - return errors.New("expected ProtoNode")
325 - }
326 -
327 - // Verify this is a branch node
328 - pbn, err := ft.FromBytes(nd.Data())
329 - if err != nil {
330 - return err
331 - }
332 -
333 - if pbn.GetType() != ft.TFile {
334 - return fmt.Errorf("expected file as branch node, got: %s", pbn.GetType())
335 - }
336 -
337 - if len(pbn.Data) > 0 {
338 - return errors.New("branch node should not have data")
339 - }
340 -
341 - for i := 0; i < len(nd.Links()); i++ {
342 - child, err := nd.Links()[i].GetNode(context.TODO(), p.Getter)
343 - if err != nil {
344 - return err
345 - }
346 -
347 - if i < p.Direct {
348 - // Direct blocks
349 - err := verifyTDagRec(child, 0, p)
350 - if err != nil {
351 - return err
352 - }
353 - } else {
354 - // Recursive trickle dags
355 - rdepth := ((i - p.Direct) / p.LayerRepeat) + 1
356 - if rdepth >= depth && depth > 0 {
357 - return errors.New("child dag was too deep")
358 - }
359 - err := verifyTDagRec(child, rdepth, p)
360 - if err != nil {
361 - return err
362 - }
363 - }
364 - }
365 - return nil
366 -}
unixfs/io/bufdagreader.go deleted
-39
@@ -1,39 +0,0 @@
1 -package io
2 -
3 -import (
4 - "bytes"
5 - "context"
6 -)
7 -
8 -// BufDagReader implements a DagReader that reads from a byte slice
9 -// using a bytes.Reader. It is used for RawNodes.
10 -type BufDagReader struct {
11 - *bytes.Reader
12 -}
13 -
14 -// NewBufDagReader returns a DAG reader for the given byte slice.
15 -// BufDagReader is used to read RawNodes.
16 -func NewBufDagReader(b []byte) *BufDagReader {
17 - return &BufDagReader{bytes.NewReader(b)}
18 -}
19 -
20 -var _ DagReader = (*BufDagReader)(nil)
21 -
22 -// Close is a nop.
23 -func (*BufDagReader) Close() error {
24 - return nil
25 -}
26 -
27 -// CtxReadFull reads the slice onto b.
28 -func (rd *BufDagReader) CtxReadFull(ctx context.Context, b []byte) (int, error) {
29 - return rd.Read(b)
30 -}
31 -
32 -// Size returns the size of the buffer.
33 -func (rd *BufDagReader) Size() uint64 {
34 - s := rd.Reader.Size()
35 - if s < 0 {
36 - panic("size smaller than 0 (impossible!!)")
37 - }
38 - return uint64(s)
39 -}
unixfs/io/dagreader.go deleted
-79
@@ -1,79 +0,0 @@
1 -package io
2 -
3 -import (
4 - "context"
5 - "errors"
6 - "io"
7 -
8 - mdag "gx/ipfs/QmRy4Qk9hbgFX9NGJRm8rBThrA8PZhNCitMgeRYyZ67s59/go-merkledag"
9 - ft "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs"
10 - ftpb "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/pb"
11 -
12 - ipld "gx/ipfs/QmZtNq8dArGfnpCZfx2pUNY7UcjGhVp5qqwQ4hH6mpTMRQ/go-ipld-format"
13 -)
14 -
15 -// Common errors
16 -var (
17 - ErrIsDir = errors.New("this dag node is a directory")
18 - ErrCantReadSymlinks = errors.New("cannot currently read symlinks")
19 - ErrUnkownNodeType = errors.New("unknown node type")
20 -)
21 -
22 -// A DagReader provides read-only read and seek acess to a unixfs file.
23 -// Different implementations of readers are used for the different
24 -// types of unixfs/protobuf-encoded nodes.
25 -type DagReader interface {
26 - ReadSeekCloser
27 - Size() uint64
28 - CtxReadFull(context.Context, []byte) (int, error)
29 -}
30 -
31 -// A ReadSeekCloser implements interfaces to read, copy, seek and close.
32 -type ReadSeekCloser interface {
33 - io.Reader
34 - io.Seeker
35 - io.Closer
36 - io.WriterTo
37 -}
38 -
39 -// NewDagReader creates a new reader object that reads the data represented by
40 -// the given node, using the passed in DAGService for data retrieval
41 -func NewDagReader(ctx context.Context, n ipld.Node, serv ipld.NodeGetter) (DagReader, error) {
42 - switch n := n.(type) {
43 - case *mdag.RawNode:
44 - return NewBufDagReader(n.RawData()), nil
45 - case *mdag.ProtoNode:
46 - fsNode, err := ft.FSNodeFromBytes(n.Data())
47 - if err != nil {
48 - return nil, err
49 - }
50 -
51 - switch fsNode.Type() {
52 - case ftpb.Data_Directory, ftpb.Data_HAMTShard:
53 - // Dont allow reading directories
54 - return nil, ErrIsDir
55 - case ftpb.Data_File, ftpb.Data_Raw:
56 - return NewPBFileReader(ctx, n, fsNode, serv), nil
57 - case ftpb.Data_Metadata:
58 - if len(n.Links()) == 0 {
59 - return nil, errors.New("incorrectly formatted metadata object")
60 - }
61 - child, err := n.Links()[0].GetNode(ctx, serv)
62 - if err != nil {
63 - return nil, err
64 - }
65 -
66 - childpb, ok := child.(*mdag.ProtoNode)
67 - if !ok {
68 - return nil, mdag.ErrNotProtobuf
69 - }
70 - return NewDagReader(ctx, childpb, serv)
71 - case ftpb.Data_Symlink:
72 - return nil, ErrCantReadSymlinks
73 - default:
74 - return nil, ft.ErrUnrecognizedType
75 - }
76 - default:
77 - return nil, ErrUnkownNodeType
78 - }
79 -}
unixfs/io/dagreader_test.go deleted
-347
@@ -1,347 +0,0 @@
1 -package io
2 -
3 -import (
4 - "bytes"
5 - "io"
6 - "io/ioutil"
7 - "math/rand"
8 - "strings"
9 - "testing"
10 -
11 - mdag "gx/ipfs/QmRy4Qk9hbgFX9NGJRm8rBThrA8PZhNCitMgeRYyZ67s59/go-merkledag"
12 - "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs"
13 -
14 - context "context"
15 -
16 - testu "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/test"
17 -)
18 -
19 -func TestBasicRead(t *testing.T) {
20 - dserv := testu.GetDAGServ()
21 - inbuf, node := testu.GetRandomNode(t, dserv, 1024, testu.UseProtoBufLeaves)
22 - ctx, closer := context.WithCancel(context.Background())
23 - defer closer()
24 -
25 - reader, err := NewDagReader(ctx, node, dserv)
26 - if err != nil {
27 - t.Fatal(err)
28 - }
29 -
30 - outbuf, err := ioutil.ReadAll(reader)
31 - if err != nil {
32 - t.Fatal(err)
33 - }
34 -
35 - err = testu.ArrComp(inbuf, outbuf)
36 - if err != nil {
37 - t.Fatal(err)
38 - }
39 -}
40 -
41 -func TestSeekAndRead(t *testing.T) {
42 - dserv := testu.GetDAGServ()
43 - inbuf := make([]byte, 256)
44 - for i := 0; i <= 255; i++ {
45 - inbuf[i] = byte(i)
46 - }
47 -
48 - node := testu.GetNode(t, dserv, inbuf, testu.UseProtoBufLeaves)
49 - ctx, closer := context.WithCancel(context.Background())
50 - defer closer()
51 -
52 - reader, err := NewDagReader(ctx, node, dserv)
53 - if err != nil {
54 - t.Fatal(err)
55 - }
56 -
57 - for i := 255; i >= 0; i-- {
58 - reader.Seek(int64(i), io.SeekStart)
59 -
60 - if getOffset(reader) != int64(i) {
61 - t.Fatal("expected offset to be increased by one after read")
62 - }
63 -
64 - out := readByte(t, reader)
65 -
66 - if int(out) != i {
67 - t.Fatalf("read %d at index %d, expected %d", out, i, i)
68 - }
69 -
70 - if getOffset(reader) != int64(i+1) {
71 - t.Fatal("expected offset to be increased by one after read")
72 - }
73 - }
74 -}
75 -
76 -func TestSeekAndReadLarge(t *testing.T) {
77 - dserv := testu.GetDAGServ()
78 - inbuf := make([]byte, 20000)
79 - rand.Read(inbuf)
80 -
81 - node := testu.GetNode(t, dserv, inbuf, testu.UseProtoBufLeaves)
82 - ctx, closer := context.WithCancel(context.Background())
83 - defer closer()
84 -
85 - reader, err := NewDagReader(ctx, node, dserv)
86 - if err != nil {
87 - t.Fatal(err)
88 - }
89 -
90 - _, err = reader.Seek(10000, io.SeekStart)
91 - if err != nil {
92 - t.Fatal(err)
93 - }
94 -
95 - buf := make([]byte, 100)
96 - _, err = io.ReadFull(reader, buf)
97 - if err != nil {
98 - t.Fatal(err)
99 - }
100 -
101 - if !bytes.Equal(buf, inbuf[10000:10100]) {
102 - t.Fatal("seeked read failed")
103 - }
104 -
105 - pbdr := reader.(*PBDagReader)
106 - var count int
107 - for i, p := range pbdr.promises {
108 - if i > 20 && i < 30 {
109 - if p == nil {
110 - t.Fatal("expected index to be not nil: ", i)
111 - }
112 - count++
113 - } else {
114 - if p != nil {
115 - t.Fatal("expected index to be nil: ", i)
116 - }
117 - }
118 - }
119 - // -1 because we read some and it cleared one
120 - if count != preloadSize-1 {
121 - t.Fatalf("expected %d preloaded promises, got %d", preloadSize-1, count)
122 - }
123 -}
124 -
125 -func TestReadAndCancel(t *testing.T) {
126 - dserv := testu.GetDAGServ()
127 - inbuf := make([]byte, 20000)
128 - rand.Read(inbuf)
129 -
130 - node := testu.GetNode(t, dserv, inbuf, testu.UseProtoBufLeaves)
131 - ctx, closer := context.WithCancel(context.Background())
132 - defer closer()
133 -
134 - reader, err := NewDagReader(ctx, node, dserv)
135 - if err != nil {
136 - t.Fatal(err)
137 - }
138 -
139 - ctx, cancel := context.WithCancel(context.Background())
140 - buf := make([]byte, 100)
141 - _, err = reader.CtxReadFull(ctx, buf)
142 - if err != nil {
143 - t.Fatal(err)
144 - }
145 - if !bytes.Equal(buf, inbuf[0:100]) {
146 - t.Fatal("read failed")
147 - }
148 - cancel()
149 -
150 - b, err := ioutil.ReadAll(reader)
151 - if err != nil {
152 - t.Fatal(err)
153 - }
154 -
155 - if !bytes.Equal(inbuf[100:], b) {
156 - t.Fatal("buffers not equal")
157 - }
158 -}
159 -
160 -func TestRelativeSeek(t *testing.T) {
161 - dserv := testu.GetDAGServ()
162 - ctx, closer := context.WithCancel(context.Background())
163 - defer closer()
164 -
165 - inbuf := make([]byte, 1024)
166 -
167 - for i := 0; i < 256; i++ {
168 - inbuf[i*4] = byte(i)
169 - }
170 -
171 - inbuf[1023] = 1 // force the reader to be 1024 bytes
172 - node := testu.GetNode(t, dserv, inbuf, testu.UseProtoBufLeaves)
173 -
174 - reader, err := NewDagReader(ctx, node, dserv)
175 - if err != nil {
176 - t.Fatal(err)
177 - }
178 -
179 - for i := 0; i < 256; i++ {
180 - if getOffset(reader) != int64(i*4) {
181 - t.Fatalf("offset should be %d, was %d", i*4, getOffset(reader))
182 - }
183 - out := readByte(t, reader)
184 - if int(out) != i {
185 - t.Fatalf("expected to read: %d at %d, read %d", i, getOffset(reader)-1, out)
186 - }
187 - if i != 255 {
188 - _, err := reader.Seek(3, io.SeekCurrent)
189 - if err != nil {
190 - t.Fatal(err)
191 - }
192 - }
193 - }
194 -
195 - _, err = reader.Seek(4, io.SeekEnd)
196 - if err != nil {
197 - t.Fatal(err)
198 - }
199 -
200 - for i := 0; i < 256; i++ {
201 - if getOffset(reader) != int64(1020-i*4) {
202 - t.Fatalf("offset should be %d, was %d", 1020-i*4, getOffset(reader))
203 - }
204 - out := readByte(t, reader)
205 - if int(out) != 255-i {
206 - t.Fatalf("expected to read: %d at %d, read %d", 255-i, getOffset(reader)-1, out)
207 - }
208 - reader.Seek(-5, io.SeekCurrent) // seek 4 bytes but we read one byte every time so 5 bytes
209 - }
210 -
211 -}
212 -
213 -func TestTypeFailures(t *testing.T) {
214 - dserv := testu.GetDAGServ()
215 - ctx, closer := context.WithCancel(context.Background())
216 - defer closer()
217 -
218 - node := unixfs.EmptyDirNode()
219 - if _, err := NewDagReader(ctx, node, dserv); err != ErrIsDir {
220 - t.Fatalf("excepted to get %v, got %v", ErrIsDir, err)
221 - }
222 -
223 - data, err := unixfs.SymlinkData("/somelink")
224 - if err != nil {
225 - t.Fatal(err)
226 - }
227 - node = mdag.NodeWithData(data)
228 -
229 - if _, err := NewDagReader(ctx, node, dserv); err != ErrCantReadSymlinks {
230 - t.Fatalf("excepted to get %v, got %v", ErrCantReadSymlinks, err)
231 - }
232 -}
233 -
234 -func TestBadPBData(t *testing.T) {
235 - dserv := testu.GetDAGServ()
236 - ctx, closer := context.WithCancel(context.Background())
237 - defer closer()
238 -
239 - node := mdag.NodeWithData([]byte{42})
240 - _, err := NewDagReader(ctx, node, dserv)
241 - if err == nil {
242 - t.Fatal("excepted error, got nil")
243 - }
244 -}
245 -
246 -func TestMetadataNode(t *testing.T) {
247 - ctx, closer := context.WithCancel(context.Background())
248 - defer closer()
249 -
250 - dserv := testu.GetDAGServ()
251 - rdata, rnode := testu.GetRandomNode(t, dserv, 512, testu.UseProtoBufLeaves)
252 - err := dserv.Add(ctx, rnode)
253 - if err != nil {
254 - t.Fatal(err)
255 - }
256 -
257 - data, err := unixfs.BytesForMetadata(&unixfs.Metadata{
258 - MimeType: "text",
259 - Size: 125,
260 - })
261 - if err != nil {
262 - t.Fatal(err)
263 - }
264 - node := mdag.NodeWithData(data)
265 -
266 - _, err = NewDagReader(ctx, node, dserv)
267 - if err == nil {
268 - t.Fatal("expected an error")
269 - }
270 - if !strings.Contains(err.Error(), "incorrectly formatted") {
271 - t.Fatal("expected different error")
272 - }
273 -
274 - node.AddNodeLink("", rnode)
275 -
276 - reader, err := NewDagReader(ctx, node, dserv)
277 - if err != nil {
278 - t.Fatal(err)
279 - }
280 - readdata, err := ioutil.ReadAll(reader)
281 - if err != nil {
282 - t.Fatal(err)
283 - }
284 - if err := testu.ArrComp(rdata, readdata); err != nil {
285 - t.Fatal(err)
286 - }
287 -}
288 -
289 -func TestWriteTo(t *testing.T) {
290 - dserv := testu.GetDAGServ()
291 - inbuf, node := testu.GetRandomNode(t, dserv, 1024, testu.UseProtoBufLeaves)
292 - ctx, closer := context.WithCancel(context.Background())
293 - defer closer()
294 -
295 - reader, err := NewDagReader(ctx, node, dserv)
296 - if err != nil {
297 - t.Fatal(err)
298 - }
299 -
300 - outbuf := new(bytes.Buffer)
301 - reader.WriteTo(outbuf)
302 -
303 - err = testu.ArrComp(inbuf, outbuf.Bytes())
304 - if err != nil {
305 - t.Fatal(err)
306 - }
307 -
308 -}
309 -
310 -func TestReaderSzie(t *testing.T) {
311 - dserv := testu.GetDAGServ()
312 - size := int64(1024)
313 - _, node := testu.GetRandomNode(t, dserv, size, testu.UseProtoBufLeaves)
314 - ctx, closer := context.WithCancel(context.Background())
315 - defer closer()
316 -
317 - reader, err := NewDagReader(ctx, node, dserv)
318 - if err != nil {
319 - t.Fatal(err)
320 - }
321 -
322 - if reader.Size() != uint64(size) {
323 - t.Fatal("wrong reader size")
324 - }
325 -}
326 -
327 -func readByte(t testing.TB, reader DagReader) byte {
328 - out := make([]byte, 1)
329 - c, err := reader.Read(out)
330 -
331 - if c != 1 {
332 - t.Fatal("reader should have read just one byte")
333 - }
334 - if err != nil {
335 - t.Fatal(err)
336 - }
337 -
338 - return out[0]
339 -}
340 -
341 -func getOffset(reader DagReader) int64 {
342 - offset, err := reader.Seek(0, io.SeekCurrent)
343 - if err != nil {
344 - panic("failed to retrieve offset: " + err.Error())
345 - }
346 - return offset
347 -}
unixfs/io/directory.go deleted
-257
@@ -1,257 +0,0 @@
1 -package io
2 -
3 -import (
4 - "context"
5 - "fmt"
6 - "os"
7 -
8 - mdag "gx/ipfs/QmRy4Qk9hbgFX9NGJRm8rBThrA8PZhNCitMgeRYyZ67s59/go-merkledag"
9 - format "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs"
10 - hamt "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/hamt"
11 -
12 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
13 - ipld "gx/ipfs/QmZtNq8dArGfnpCZfx2pUNY7UcjGhVp5qqwQ4hH6mpTMRQ/go-ipld-format"
14 -)
15 -
16 -// UseHAMTSharding is a global flag that signifies whether or not to use the
17 -// HAMT sharding scheme for directory creation
18 -var UseHAMTSharding = false
19 -
20 -// DefaultShardWidth is the default value used for hamt sharding width.
21 -var DefaultShardWidth = 256
22 -
23 -// Directory defines a UnixFS directory. It is used for creating, reading and
24 -// editing directories. It allows to work with different directory schemes,
25 -// like the basic or the HAMT implementation.
26 -//
27 -// It just allows to perform explicit edits on a single directory, working with
28 -// directory trees is out of its scope, they are managed by the MFS layer
29 -// (which is the main consumer of this interface).
30 -type Directory interface {
31 -
32 - // SetPrefix sets the CID prefix of the root node.
33 - SetPrefix(*cid.Prefix)
34 -
35 - // AddChild adds a (name, key) pair to the root node.
36 - AddChild(context.Context, string, ipld.Node) error
37 -
38 - // ForEachLink applies the given function to Links in the directory.
39 - ForEachLink(context.Context, func(*ipld.Link) error) error
40 -
41 - // Links returns the all the links in the directory node.
42 - Links(context.Context) ([]*ipld.Link, error)
43 -
44 - // Find returns the root node of the file named 'name' within this directory.
45 - // In the case of HAMT-directories, it will traverse the tree.
46 - Find(context.Context, string) (ipld.Node, error)
47 -
48 - // RemoveChild removes the child with the given name.
49 - RemoveChild(context.Context, string) error
50 -
51 - // GetNode returns the root of this directory.
52 - GetNode() (ipld.Node, error)
53 -
54 - // GetPrefix returns the CID Prefix used.
55 - GetPrefix() *cid.Prefix
56 -}
57 -
58 -// TODO: Evaluate removing `dserv` from this layer and providing it in MFS.
59 -// (The functions should in that case add a `DAGService` argument.)
60 -
61 -// BasicDirectory is the basic implementation of `Directory`. All the entries
62 -// are stored in a single node.
63 -type BasicDirectory struct {
64 - node *mdag.ProtoNode
65 - dserv ipld.DAGService
66 -}
67 -
68 -// HAMTDirectory is the HAMT implementation of `Directory`.
69 -// (See package `hamt` for more information.)
70 -type HAMTDirectory struct {
71 - shard *hamt.Shard
72 - dserv ipld.DAGService
73 -}
74 -
75 -// NewDirectory returns a Directory. It needs a `DAGService` to add the children.
76 -func NewDirectory(dserv ipld.DAGService) Directory {
77 - if UseHAMTSharding {
78 - dir := new(HAMTDirectory)
79 - s, err := hamt.NewShard(dserv, DefaultShardWidth)
80 - if err != nil {
81 - panic(err) // will only panic if DefaultShardWidth is a bad value
82 - }
83 - dir.shard = s
84 - dir.dserv = dserv
85 - return dir
86 - }
87 -
88 - dir := new(BasicDirectory)
89 - dir.node = format.EmptyDirNode()
90 - dir.dserv = dserv
91 - return dir
92 -}
93 -
94 -// ErrNotADir implies that the given node was not a unixfs directory
95 -var ErrNotADir = fmt.Errorf("merkledag node was not a directory or shard")
96 -
97 -// NewDirectoryFromNode loads a unixfs directory from the given IPLD node and
98 -// DAGService.
99 -func NewDirectoryFromNode(dserv ipld.DAGService, node ipld.Node) (Directory, error) {
100 - protoBufNode, ok := node.(*mdag.ProtoNode)
101 - if !ok {
102 - return nil, ErrNotADir
103 - }
104 -
105 - fsNode, err := format.FSNodeFromBytes(protoBufNode.Data())
106 - if err != nil {
107 - return nil, err
108 - }
109 -
110 - switch fsNode.Type() {
111 - case format.TDirectory:
112 - return &BasicDirectory{
113 - dserv: dserv,
114 - node: protoBufNode.Copy().(*mdag.ProtoNode),
115 - }, nil
116 - case format.THAMTShard:
117 - shard, err := hamt.NewHamtFromDag(dserv, node)
118 - if err != nil {
119 - return nil, err
120 - }
121 - return &HAMTDirectory{
122 - dserv: dserv,
123 - shard: shard,
124 - }, nil
125 - }
126 -
127 - return nil, ErrNotADir
128 -}
129 -
130 -// SetPrefix implements the `Directory` interface.
131 -func (d *BasicDirectory) SetPrefix(prefix *cid.Prefix) {
132 - d.node.SetPrefix(prefix)
133 -}
134 -
135 -// AddChild implements the `Directory` interface. It adds (or replaces)
136 -// a link to the given `node` under `name`.
137 -func (d *BasicDirectory) AddChild(ctx context.Context, name string, node ipld.Node) error {
138 - d.node.RemoveNodeLink(name)
139 - // Remove old link (if it existed), don't check a potential `ErrNotFound`.
140 -
141 - return d.node.AddNodeLink(name, node)
142 -}
143 -
144 -// ForEachLink implements the `Directory` interface.
145 -func (d *BasicDirectory) ForEachLink(ctx context.Context, f func(*ipld.Link) error) error {
146 - for _, l := range d.node.Links() {
147 - if err := f(l); err != nil {
148 - return err
149 - }
150 - }
151 - return nil
152 -}
153 -
154 -// Links implements the `Directory` interface.
155 -func (d *BasicDirectory) Links(ctx context.Context) ([]*ipld.Link, error) {
156 - return d.node.Links(), nil
157 -}
158 -
159 -// Find implements the `Directory` interface.
160 -func (d *BasicDirectory) Find(ctx context.Context, name string) (ipld.Node, error) {
161 - lnk, err := d.node.GetNodeLink(name)
162 - if err == mdag.ErrLinkNotFound {
163 - err = os.ErrNotExist
164 - }
165 - if err != nil {
166 - return nil, err
167 - }
168 -
169 - return d.dserv.Get(ctx, lnk.Cid)
170 -}
171 -
172 -// RemoveChild implements the `Directory` interface.
173 -func (d *BasicDirectory) RemoveChild(ctx context.Context, name string) error {
174 - return d.node.RemoveNodeLink(name)
175 -}
176 -
177 -// GetNode implements the `Directory` interface.
178 -func (d *BasicDirectory) GetNode() (ipld.Node, error) {
179 - return d.node, nil
180 -}
181 -
182 -// GetPrefix implements the `Directory` interface.
183 -func (d *BasicDirectory) GetPrefix() *cid.Prefix {
184 - return &d.node.Prefix
185 -}
186 -
187 -// SwitchToSharding returns a HAMT implementation of this directory.
188 -func (d *BasicDirectory) SwitchToSharding(ctx context.Context) (Directory, error) {
189 - hamtDir := new(HAMTDirectory)
190 - hamtDir.dserv = d.dserv
191 -
192 - shard, err := hamt.NewShard(d.dserv, DefaultShardWidth)
193 - if err != nil {
194 - return nil, err
195 - }
196 - shard.SetPrefix(&d.node.Prefix)
197 - hamtDir.shard = shard
198 -
199 - for _, lnk := range d.node.Links() {
200 - node, err := d.dserv.Get(ctx, lnk.Cid)
201 - if err != nil {
202 - return nil, err
203 - }
204 -
205 - err = hamtDir.shard.Set(ctx, lnk.Name, node)
206 - if err != nil {
207 - return nil, err
208 - }
209 - }
210 -
211 - return hamtDir, nil
212 -}
213 -
214 -// SetPrefix implements the `Directory` interface.
215 -func (d *HAMTDirectory) SetPrefix(prefix *cid.Prefix) {
216 - d.shard.SetPrefix(prefix)
217 -}
218 -
219 -// AddChild implements the `Directory` interface.
220 -func (d *HAMTDirectory) AddChild(ctx context.Context, name string, nd ipld.Node) error {
221 - return d.shard.Set(ctx, name, nd)
222 -}
223 -
224 -// ForEachLink implements the `Directory` interface.
225 -func (d *HAMTDirectory) ForEachLink(ctx context.Context, f func(*ipld.Link) error) error {
226 - return d.shard.ForEachLink(ctx, f)
227 -}
228 -
229 -// Links implements the `Directory` interface.
230 -func (d *HAMTDirectory) Links(ctx context.Context) ([]*ipld.Link, error) {
231 - return d.shard.EnumLinks(ctx)
232 -}
233 -
234 -// Find implements the `Directory` interface. It will traverse the tree.
235 -func (d *HAMTDirectory) Find(ctx context.Context, name string) (ipld.Node, error) {
236 - lnk, err := d.shard.Find(ctx, name)
237 - if err != nil {
238 - return nil, err
239 - }
240 -
241 - return lnk.GetNode(ctx, d.dserv)
242 -}
243 -
244 -// RemoveChild implements the `Directory` interface.
245 -func (d *HAMTDirectory) RemoveChild(ctx context.Context, name string) error {
246 - return d.shard.Remove(ctx, name)
247 -}
248 -
249 -// GetNode implements the `Directory` interface.
250 -func (d *HAMTDirectory) GetNode() (ipld.Node, error) {
251 - return d.shard.Node()
252 -}
253 -
254 -// GetPrefix implements the `Directory` interface.
255 -func (d *HAMTDirectory) GetPrefix() *cid.Prefix {
256 - return d.shard.Prefix()
257 -}
unixfs/io/directory_test.go deleted
-158
@@ -1,158 +0,0 @@
1 -package io
2 -
3 -import (
4 - "context"
5 - "fmt"
6 - "testing"
7 -
8 - mdtest "gx/ipfs/QmRy4Qk9hbgFX9NGJRm8rBThrA8PZhNCitMgeRYyZ67s59/go-merkledag/test"
9 - ft "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs"
10 -)
11 -
12 -func TestEmptyNode(t *testing.T) {
13 - n := ft.EmptyDirNode()
14 - if len(n.Links()) != 0 {
15 - t.Fatal("empty node should have 0 links")
16 - }
17 -}
18 -
19 -func TestDirectoryGrowth(t *testing.T) {
20 - ds := mdtest.Mock()
21 - dir := NewDirectory(ds)
22 - ctx := context.Background()
23 -
24 - d := ft.EmptyDirNode()
25 - ds.Add(ctx, d)
26 -
27 - nelems := 10000
28 -
29 - for i := 0; i < nelems; i++ {
30 - err := dir.AddChild(ctx, fmt.Sprintf("dir%d", i), d)
31 - if err != nil {
32 - t.Fatal(err)
33 - }
34 - }
35 -
36 - _, err := dir.GetNode()
37 - if err != nil {
38 - t.Fatal(err)
39 - }
40 -
41 - links, err := dir.Links(ctx)
42 - if err != nil {
43 - t.Fatal(err)
44 - }
45 -
46 - if len(links) != nelems {
47 - t.Fatal("didnt get right number of elements")
48 - }
49 -
50 - dirc := d.Cid()
51 -
52 - names := make(map[string]bool)
53 - for _, l := range links {
54 - names[l.Name] = true
55 - if !l.Cid.Equals(dirc) {
56 - t.Fatal("link wasnt correct")
57 - }
58 - }
59 -
60 - for i := 0; i < nelems; i++ {
61 - dn := fmt.Sprintf("dir%d", i)
62 - if !names[dn] {
63 - t.Fatal("didnt find directory: ", dn)
64 - }
65 -
66 - _, err := dir.Find(context.Background(), dn)
67 - if err != nil {
68 - t.Fatal(err)
69 - }
70 - }
71 -}
72 -
73 -func TestDuplicateAddDir(t *testing.T) {
74 - ds := mdtest.Mock()
75 - dir := NewDirectory(ds)
76 - ctx := context.Background()
77 - nd := ft.EmptyDirNode()
78 -
79 - err := dir.AddChild(ctx, "test", nd)
80 - if err != nil {
81 - t.Fatal(err)
82 - }
83 -
84 - err = dir.AddChild(ctx, "test", nd)
85 - if err != nil {
86 - t.Fatal(err)
87 - }
88 -
89 - lnks, err := dir.Links(ctx)
90 - if err != nil {
91 - t.Fatal(err)
92 - }
93 -
94 - if len(lnks) != 1 {
95 - t.Fatal("expected only one link")
96 - }
97 -}
98 -
99 -func TestDirBuilder(t *testing.T) {
100 - ds := mdtest.Mock()
101 - dir := NewDirectory(ds)
102 - ctx := context.Background()
103 -
104 - child := ft.EmptyDirNode()
105 - err := ds.Add(ctx, child)
106 - if err != nil {
107 - t.Fatal(err)
108 - }
109 -
110 - count := 5000
111 -
112 - for i := 0; i < count; i++ {
113 - err := dir.AddChild(ctx, fmt.Sprintf("entry %d", i), child)
114 - if err != nil {
115 - t.Fatal(err)
116 - }
117 - }
118 -
119 - dirnd, err := dir.GetNode()
120 - if err != nil {
121 - t.Fatal(err)
122 - }
123 -
124 - links, err := dir.Links(ctx)
125 - if err != nil {
126 - t.Fatal(err)
127 - }
128 -
129 - if len(links) != count {
130 - t.Fatal("not enough links dawg", len(links), count)
131 - }
132 -
133 - adir, err := NewDirectoryFromNode(ds, dirnd)
134 - if err != nil {
135 - t.Fatal(err)
136 - }
137 -
138 - links, err = adir.Links(ctx)
139 - if err != nil {
140 - t.Fatal(err)
141 - }
142 -
143 - names := make(map[string]bool)
144 - for _, lnk := range links {
145 - names[lnk.Name] = true
146 - }
147 -
148 - for i := 0; i < count; i++ {
149 - n := fmt.Sprintf("entry %d", i)
150 - if !names[n] {
151 - t.Fatal("COULDNT FIND: ", n)
152 - }
153 - }
154 -
155 - if len(links) != count {
156 - t.Fatal("wrong number of links", len(links), count)
157 - }
158 -}
unixfs/io/doc.go deleted
-3
@@ -1,3 +0,0 @@
1 -// Package io implements convenience objects for working with the ipfs
2 -// unixfs data format.
3 -package io
unixfs/io/pbdagreader.go deleted
-328
@@ -1,328 +0,0 @@
1 -package io
2 -
3 -import (
4 - "context"
5 - "errors"
6 - "fmt"
7 - "io"
8 -
9 - mdag "gx/ipfs/QmRy4Qk9hbgFX9NGJRm8rBThrA8PZhNCitMgeRYyZ67s59/go-merkledag"
10 - ft "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs"
11 - ftpb "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/pb"
12 -
13 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
14 - ipld "gx/ipfs/QmZtNq8dArGfnpCZfx2pUNY7UcjGhVp5qqwQ4hH6mpTMRQ/go-ipld-format"
15 -)
16 -
17 -// PBDagReader provides a way to easily read the data contained in a dag.
18 -type PBDagReader struct {
19 - serv ipld.NodeGetter
20 -
21 - // UnixFS file (it should be of type `Data_File` or `Data_Raw` only).
22 - file *ft.FSNode
23 -
24 - // the current data buffer to be read from
25 - // will either be a bytes.Reader or a child DagReader
26 - buf ReadSeekCloser
27 -
28 - // NodePromises for each of 'nodes' child links
29 - promises []*ipld.NodePromise
30 -
31 - // the cid of each child of the current node
32 - links []*cid.Cid
33 -
34 - // the index of the child link currently being read from
35 - linkPosition int
36 -
37 - // current offset for the read head within the 'file'
38 - offset int64
39 -
40 - // Our context
41 - ctx context.Context
42 -
43 - // context cancel for children
44 - cancel func()
45 -}
46 -
47 -var _ DagReader = (*PBDagReader)(nil)
48 -
49 -// NewPBFileReader constructs a new PBFileReader.
50 -func NewPBFileReader(ctx context.Context, n *mdag.ProtoNode, file *ft.FSNode, serv ipld.NodeGetter) *PBDagReader {
51 - fctx, cancel := context.WithCancel(ctx)
52 - curLinks := getLinkCids(n)
53 - return &PBDagReader{
54 - serv: serv,
55 - buf: NewBufDagReader(file.Data()),
56 - promises: make([]*ipld.NodePromise, len(curLinks)),
57 - links: curLinks,
58 - ctx: fctx,
59 - cancel: cancel,
60 - file: file,
61 - }
62 -}
63 -
64 -const preloadSize = 10
65 -
66 -func (dr *PBDagReader) preload(ctx context.Context, beg int) {
67 - end := beg + preloadSize
68 - if end >= len(dr.links) {
69 - end = len(dr.links)
70 - }
71 -
72 - copy(dr.promises[beg:], ipld.GetNodes(ctx, dr.serv, dr.links[beg:end]))
73 -}
74 -
75 -// precalcNextBuf follows the next link in line and loads it from the
76 -// DAGService, setting the next buffer to read from
77 -func (dr *PBDagReader) precalcNextBuf(ctx context.Context) error {
78 - if dr.buf != nil {
79 - dr.buf.Close() // Just to make sure
80 - dr.buf = nil
81 - }
82 -
83 - if dr.linkPosition >= len(dr.promises) {
84 - return io.EOF
85 - }
86 -
87 - // If we drop to <= preloadSize/2 preloading nodes, preload the next 10.
88 - for i := dr.linkPosition; i < dr.linkPosition+preloadSize/2 && i < len(dr.promises); i++ {
89 - // TODO: check if canceled.
90 - if dr.promises[i] == nil {
91 - dr.preload(ctx, i)
92 - break
93 - }
94 - }
95 -
96 - nxt, err := dr.promises[dr.linkPosition].Get(ctx)
97 - dr.promises[dr.linkPosition] = nil
98 - switch err {
99 - case nil:
100 - case context.DeadlineExceeded, context.Canceled:
101 - err = ctx.Err()
102 - if err != nil {
103 - return ctx.Err()
104 - }
105 - // In this case, the context used to *preload* the node has been canceled.
106 - // We need to retry the load with our context and we might as
107 - // well preload some extra nodes while we're at it.
108 - //
109 - // Note: When using `Read`, this code will never execute as
110 - // `Read` will use the global context. It only runs if the user
111 - // explicitly reads with a custom context (e.g., by calling
112 - // `CtxReadFull`).
113 - dr.preload(ctx, dr.linkPosition)
114 - nxt, err = dr.promises[dr.linkPosition].Get(ctx)
115 - dr.promises[dr.linkPosition] = nil
116 - if err != nil {
117 - return err
118 - }
119 - default:
120 - return err
121 - }
122 -
123 - dr.linkPosition++
124 -
125 - return dr.loadBufNode(nxt)
126 -}
127 -
128 -func (dr *PBDagReader) loadBufNode(node ipld.Node) error {
129 - switch node := node.(type) {
130 - case *mdag.ProtoNode:
131 - fsNode, err := ft.FSNodeFromBytes(node.Data())
132 - if err != nil {
133 - return fmt.Errorf("incorrectly formatted protobuf: %s", err)
134 - }
135 -
136 - switch fsNode.Type() {
137 - case ftpb.Data_File:
138 - dr.buf = NewPBFileReader(dr.ctx, node, fsNode, dr.serv)
139 - return nil
140 - case ftpb.Data_Raw:
141 - dr.buf = NewBufDagReader(fsNode.Data())
142 - return nil
143 - default:
144 - return fmt.Errorf("found %s node in unexpected place", fsNode.Type().String())
145 - }
146 - case *mdag.RawNode:
147 - dr.buf = NewBufDagReader(node.RawData())
148 - return nil
149 - default:
150 - return ErrUnkownNodeType
151 - }
152 -}
153 -
154 -func getLinkCids(n ipld.Node) []*cid.Cid {
155 - links := n.Links()
156 - out := make([]*cid.Cid, 0, len(links))
157 - for _, l := range links {
158 - out = append(out, l.Cid)
159 - }
160 - return out
161 -}
162 -
163 -// Size return the total length of the data from the DAG structured file.
164 -func (dr *PBDagReader) Size() uint64 {
165 - return dr.file.FileSize()
166 -}
167 -
168 -// Read reads data from the DAG structured file
169 -func (dr *PBDagReader) Read(b []byte) (int, error) {
170 - return dr.CtxReadFull(dr.ctx, b)
171 -}
172 -
173 -// CtxReadFull reads data from the DAG structured file
174 -func (dr *PBDagReader) CtxReadFull(ctx context.Context, b []byte) (int, error) {
175 - if dr.buf == nil {
176 - if err := dr.precalcNextBuf(ctx); err != nil {
177 - return 0, err
178 - }
179 - }
180 -
181 - // If no cached buffer, load one
182 - total := 0
183 - for {
184 - // Attempt to fill bytes from cached buffer
185 - n, err := io.ReadFull(dr.buf, b[total:])
186 - total += n
187 - dr.offset += int64(n)
188 - switch err {
189 - // io.EOF will happen is dr.buf had noting more to read (n == 0)
190 - case io.EOF, io.ErrUnexpectedEOF:
191 - // do nothing
192 - case nil:
193 - return total, nil
194 - default:
195 - return total, err
196 - }
197 -
198 - // if we are not done with the output buffer load next block
199 - err = dr.precalcNextBuf(ctx)
200 - if err != nil {
201 - return total, err
202 - }
203 - }
204 -}
205 -
206 -// WriteTo writes to the given writer.
207 -func (dr *PBDagReader) WriteTo(w io.Writer) (int64, error) {
208 - if dr.buf == nil {
209 - if err := dr.precalcNextBuf(dr.ctx); err != nil {
210 - return 0, err
211 - }
212 - }
213 -
214 - // If no cached buffer, load one
215 - total := int64(0)
216 - for {
217 - // Attempt to write bytes from cached buffer
218 - n, err := dr.buf.WriteTo(w)
219 - total += n
220 - dr.offset += n
221 - if err != nil {
222 - if err != io.EOF {
223 - return total, err
224 - }
225 - }
226 -
227 - // Otherwise, load up the next block
228 - err = dr.precalcNextBuf(dr.ctx)
229 - if err != nil {
230 - if err == io.EOF {
231 - return total, nil
232 - }
233 - return total, err
234 - }
235 - }
236 -}
237 -
238 -// Close closes the reader.
239 -func (dr *PBDagReader) Close() error {
240 - dr.cancel()
241 - return nil
242 -}
243 -
244 -// Seek implements io.Seeker, and will seek to a given offset in the file
245 -// interface matches standard unix seek
246 -// TODO: check if we can do relative seeks, to reduce the amount of dagreader
247 -// recreations that need to happen.
248 -func (dr *PBDagReader) Seek(offset int64, whence int) (int64, error) {
249 - switch whence {
250 - case io.SeekStart:
251 - if offset < 0 {
252 - return -1, errors.New("invalid offset")
253 - }
254 - if offset == dr.offset {
255 - return offset, nil
256 - }
257 -
258 - // left represents the number of bytes remaining to seek to (from beginning)
259 - left := offset
260 - if int64(len(dr.file.Data())) >= offset {
261 - // Close current buf to close potential child dagreader
262 - if dr.buf != nil {
263 - dr.buf.Close()
264 - }
265 - dr.buf = NewBufDagReader(dr.file.Data()[offset:])
266 -
267 - // start reading links from the beginning
268 - dr.linkPosition = 0
269 - dr.offset = offset
270 - return offset, nil
271 - }
272 -
273 - // skip past root block data
274 - left -= int64(len(dr.file.Data()))
275 -
276 - // iterate through links and find where we need to be
277 - for i := 0; i < dr.file.NumChildren(); i++ {
278 - if dr.file.BlockSize(i) > uint64(left) {
279 - dr.linkPosition = i
280 - break
281 - } else {
282 - left -= int64(dr.file.BlockSize(i))
283 - }
284 - }
285 -
286 - // start sub-block request
287 - err := dr.precalcNextBuf(dr.ctx)
288 - if err != nil {
289 - return 0, err
290 - }
291 -
292 - // set proper offset within child readseeker
293 - n, err := dr.buf.Seek(left, io.SeekStart)
294 - if err != nil {
295 - return -1, err
296 - }
297 -
298 - // sanity
299 - left -= n
300 - if left != 0 {
301 - return -1, errors.New("failed to seek properly")
302 - }
303 - dr.offset = offset
304 - return offset, nil
305 - case io.SeekCurrent:
306 - // TODO: be smarter here
307 - if offset == 0 {
308 - return dr.offset, nil
309 - }
310 -
311 - noffset := dr.offset + offset
312 - return dr.Seek(noffset, io.SeekStart)
313 - case io.SeekEnd:
314 - noffset := int64(dr.file.FileSize()) - offset
315 - n, err := dr.Seek(noffset, io.SeekStart)
316 -
317 - // Return negative number if we can't figure out the file size. Using io.EOF
318 - // for this seems to be good(-enough) solution as it's only returned by
319 - // precalcNextBuf when we step out of file range.
320 - // This is needed for gateway to function properly
321 - if err == io.EOF && dr.file.Type() == ftpb.Data_File {
322 - return -1, nil
323 - }
324 - return n, err
325 - default:
326 - return 0, errors.New("invalid whence")
327 - }
328 -}
unixfs/io/resolve.go deleted
-58
@@ -1,58 +0,0 @@
1 -package io
2 -
3 -import (
4 - "context"
5 -
6 - dag "gx/ipfs/QmRy4Qk9hbgFX9NGJRm8rBThrA8PZhNCitMgeRYyZ67s59/go-merkledag"
7 - ft "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs"
8 - hamt "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/hamt"
9 -
10 - ipld "gx/ipfs/QmZtNq8dArGfnpCZfx2pUNY7UcjGhVp5qqwQ4hH6mpTMRQ/go-ipld-format"
11 -)
12 -
13 -// ResolveUnixfsOnce resolves a single hop of a path through a graph in a
14 -// unixfs context. This includes handling traversing sharded directories.
15 -func ResolveUnixfsOnce(ctx context.Context, ds ipld.NodeGetter, nd ipld.Node, names []string) (*ipld.Link, []string, error) {
16 - switch nd := nd.(type) {
17 - case *dag.ProtoNode:
18 - upb, err := ft.FromBytes(nd.Data())
19 - if err != nil {
20 - // Not a unixfs node, use standard object traversal code
21 - lnk, err := nd.GetNodeLink(names[0])
22 - if err != nil {
23 - return nil, nil, err
24 - }
25 -
26 - return lnk, names[1:], nil
27 - }
28 -
29 - switch upb.GetType() {
30 - case ft.THAMTShard:
31 - rods := dag.NewReadOnlyDagService(ds)
32 - s, err := hamt.NewHamtFromDag(rods, nd)
33 - if err != nil {
34 - return nil, nil, err
35 - }
36 -
37 - out, err := s.Find(ctx, names[0])
38 - if err != nil {
39 - return nil, nil, err
40 - }
41 -
42 - return out, names[1:], nil
43 - default:
44 - lnk, err := nd.GetNodeLink(names[0])
45 - if err != nil {
46 - return nil, nil, err
47 - }
48 -
49 - return lnk, names[1:], nil
50 - }
51 - default:
52 - lnk, rest, err := nd.ResolveLink(names)
53 - if err != nil {
54 - return nil, nil, err
55 - }
56 - return lnk, rest, nil
57 - }
58 -}
unixfs/mod/dagmodifier.go deleted
-592
@@ -1,592 +0,0 @@
1 -// Package mod provides DAG modification utilities to, for example,
2 -// insert additional nodes in a unixfs DAG or truncate them.
3 -package mod
4 -
5 -import (
6 - "bytes"
7 - "context"
8 - "errors"
9 - "io"
10 -
11 - mdag "gx/ipfs/QmRy4Qk9hbgFX9NGJRm8rBThrA8PZhNCitMgeRYyZ67s59/go-merkledag"
12 - ft "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs"
13 - help "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/importer/helpers"
14 - trickle "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/importer/trickle"
15 - uio "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/io"
16 -
17 - chunker "gx/ipfs/QmVDjhUMtkRskBFAVNwyXuLSKbeAya7JKPnzAxMKDaK4x4/go-ipfs-chunker"
18 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
19 - proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
20 - ipld "gx/ipfs/QmZtNq8dArGfnpCZfx2pUNY7UcjGhVp5qqwQ4hH6mpTMRQ/go-ipld-format"
21 -)
22 -
23 -// Common errors
24 -var (
25 - ErrSeekFail = errors.New("failed to seek properly")
26 - ErrUnrecognizedWhence = errors.New("unrecognized whence")
27 - ErrNotUnixfs = errors.New("dagmodifier only supports unixfs nodes (proto or raw)")
28 -)
29 -
30 -// 2MB
31 -var writebufferSize = 1 << 21
32 -
33 -// DagModifier is the only struct licensed and able to correctly
34 -// perform surgery on a DAG 'file'
35 -// Dear god, please rename this to something more pleasant
36 -type DagModifier struct {
37 - dagserv ipld.DAGService
38 - curNode ipld.Node
39 -
40 - splitter chunker.SplitterGen
41 - ctx context.Context
42 - readCancel func()
43 -
44 - writeStart uint64
45 - curWrOff uint64
46 - wrBuf *bytes.Buffer
47 -
48 - Prefix cid.Prefix
49 - RawLeaves bool
50 -
51 - read uio.DagReader
52 -}
53 -
54 -// NewDagModifier returns a new DagModifier, the Cid prefix for newly
55 -// created nodes will be inhered from the passed in node. If the Cid
56 -// version if not 0 raw leaves will also be enabled. The Prefix and
57 -// RawLeaves options can be overridden by changing them after the call.
58 -func NewDagModifier(ctx context.Context, from ipld.Node, serv ipld.DAGService, spl chunker.SplitterGen) (*DagModifier, error) {
59 - switch from.(type) {
60 - case *mdag.ProtoNode, *mdag.RawNode:
61 - // ok
62 - default:
63 - return nil, ErrNotUnixfs
64 - }
65 -
66 - prefix := from.Cid().Prefix()
67 - prefix.Codec = cid.DagProtobuf
68 - rawLeaves := false
69 - if prefix.Version > 0 {
70 - rawLeaves = true
71 - }
72 -
73 - return &DagModifier{
74 - curNode: from.Copy(),
75 - dagserv: serv,
76 - splitter: spl,
77 - ctx: ctx,
78 - Prefix: prefix,
79 - RawLeaves: rawLeaves,
80 - }, nil
81 -}
82 -
83 -// WriteAt will modify a dag file in place
84 -func (dm *DagModifier) WriteAt(b []byte, offset int64) (int, error) {
85 - // TODO: this is currently VERY inefficient
86 - // each write that happens at an offset other than the current one causes a
87 - // flush to disk, and dag rewrite
88 - if offset == int64(dm.writeStart) && dm.wrBuf != nil {
89 - // If we would overwrite the previous write
90 - if len(b) >= dm.wrBuf.Len() {
91 - dm.wrBuf.Reset()
92 - }
93 - } else if uint64(offset) != dm.curWrOff {
94 - size, err := dm.Size()
95 - if err != nil {
96 - return 0, err
97 - }
98 - if offset > size {
99 - err := dm.expandSparse(offset - size)
100 - if err != nil {
101 - return 0, err
102 - }
103 - }
104 -
105 - err = dm.Sync()
106 - if err != nil {
107 - return 0, err
108 - }
109 - dm.writeStart = uint64(offset)
110 - }
111 -
112 - return dm.Write(b)
113 -}
114 -
115 -// A reader that just returns zeros
116 -type zeroReader struct{}
117 -
118 -func (zr zeroReader) Read(b []byte) (int, error) {
119 - for i := range b {
120 - b[i] = 0
121 - }
122 - return len(b), nil
123 -}
124 -
125 -// expandSparse grows the file with zero blocks of 4096
126 -// A small blocksize is chosen to aid in deduplication
127 -func (dm *DagModifier) expandSparse(size int64) error {
128 - r := io.LimitReader(zeroReader{}, size)
129 - spl := chunker.NewSizeSplitter(r, 4096)
130 - nnode, err := dm.appendData(dm.curNode, spl)
131 - if err != nil {
132 - return err
133 - }
134 - err = dm.dagserv.Add(dm.ctx, nnode)
135 - return err
136 -}
137 -
138 -// Write continues writing to the dag at the current offset
139 -func (dm *DagModifier) Write(b []byte) (int, error) {
140 - if dm.read != nil {
141 - dm.read = nil
142 - }
143 - if dm.wrBuf == nil {
144 - dm.wrBuf = new(bytes.Buffer)
145 - }
146 -
147 - n, err := dm.wrBuf.Write(b)
148 - if err != nil {
149 - return n, err
150 - }
151 - dm.curWrOff += uint64(n)
152 - if dm.wrBuf.Len() > writebufferSize {
153 - err := dm.Sync()
154 - if err != nil {
155 - return n, err
156 - }
157 - }
158 - return n, nil
159 -}
160 -
161 -// Size returns the Filesize of the node
162 -func (dm *DagModifier) Size() (int64, error) {
163 - fileSize, err := fileSize(dm.curNode)
164 - if err != nil {
165 - return 0, err
166 - }
167 - if dm.wrBuf != nil && int64(dm.wrBuf.Len())+int64(dm.writeStart) > int64(fileSize) {
168 - return int64(dm.wrBuf.Len()) + int64(dm.writeStart), nil
169 - }
170 - return int64(fileSize), nil
171 -}
172 -
173 -func fileSize(n ipld.Node) (uint64, error) {
174 - switch nd := n.(type) {
175 - case *mdag.ProtoNode:
176 - f, err := ft.FromBytes(nd.Data())
177 - if err != nil {
178 - return 0, err
179 - }
180 - return f.GetFilesize(), nil
181 - case *mdag.RawNode:
182 - return uint64(len(nd.RawData())), nil
183 - default:
184 - return 0, ErrNotUnixfs
185 - }
186 -}
187 -
188 -// Sync writes changes to this dag to disk
189 -func (dm *DagModifier) Sync() error {
190 - // No buffer? Nothing to do
191 - if dm.wrBuf == nil {
192 - return nil
193 - }
194 -
195 - // If we have an active reader, kill it
196 - if dm.read != nil {
197 - dm.read = nil
198 - dm.readCancel()
199 - }
200 -
201 - // Number of bytes we're going to write
202 - buflen := dm.wrBuf.Len()
203 -
204 - // overwrite existing dag nodes
205 - thisc, err := dm.modifyDag(dm.curNode, dm.writeStart)
206 - if err != nil {
207 - return err
208 - }
209 -
210 - dm.curNode, err = dm.dagserv.Get(dm.ctx, thisc)
211 - if err != nil {
212 - return err
213 - }
214 -
215 - // need to write past end of current dag
216 - if dm.wrBuf.Len() > 0 {
217 - dm.curNode, err = dm.appendData(dm.curNode, dm.splitter(dm.wrBuf))
218 - if err != nil {
219 - return err
220 - }
221 -
222 - err = dm.dagserv.Add(dm.ctx, dm.curNode)
223 - if err != nil {
224 - return err
225 - }
226 - }
227 -
228 - dm.writeStart += uint64(buflen)
229 -
230 - dm.wrBuf = nil
231 - return nil
232 -}
233 -
234 -// modifyDag writes the data in 'dm.wrBuf' over the data in 'node' starting at 'offset'
235 -// returns the new key of the passed in node.
236 -func (dm *DagModifier) modifyDag(n ipld.Node, offset uint64) (*cid.Cid, error) {
237 - // If we've reached a leaf node.
238 - if len(n.Links()) == 0 {
239 - switch nd0 := n.(type) {
240 - case *mdag.ProtoNode:
241 - f, err := ft.FromBytes(nd0.Data())
242 - if err != nil {
243 - return nil, err
244 - }
245 -
246 - _, err = dm.wrBuf.Read(f.Data[offset:])
247 - if err != nil && err != io.EOF {
248 - return nil, err
249 - }
250 -
251 - // Update newly written node..
252 - b, err := proto.Marshal(f)
253 - if err != nil {
254 - return nil, err
255 - }
256 -
257 - nd := new(mdag.ProtoNode)
258 - nd.SetData(b)
259 - nd.SetPrefix(&nd0.Prefix)
260 - err = dm.dagserv.Add(dm.ctx, nd)
261 - if err != nil {
262 - return nil, err
263 - }
264 -
265 - return nd.Cid(), nil
266 - case *mdag.RawNode:
267 - origData := nd0.RawData()
268 - bytes := make([]byte, len(origData))
269 -
270 - // copy orig data up to offset
271 - copy(bytes, origData[:offset])
272 -
273 - // copy in new data
274 - n, err := dm.wrBuf.Read(bytes[offset:])
275 - if err != nil && err != io.EOF {
276 - return nil, err
277 - }
278 -
279 - // copy remaining data
280 - offsetPlusN := int(offset) + n
281 - if offsetPlusN < len(origData) {
282 - copy(bytes[offsetPlusN:], origData[offsetPlusN:])
283 - }
284 -
285 - nd, err := mdag.NewRawNodeWPrefix(bytes, nd0.Cid().Prefix())
286 - if err != nil {
287 - return nil, err
288 - }
289 - err = dm.dagserv.Add(dm.ctx, nd)
290 - if err != nil {
291 - return nil, err
292 - }
293 -
294 - return nd.Cid(), nil
295 - }
296 - }
297 -
298 - node, ok := n.(*mdag.ProtoNode)
299 - if !ok {
300 - return nil, ErrNotUnixfs
301 - }
302 -
303 - f, err := ft.FromBytes(node.Data())
304 - if err != nil {
305 - return nil, err
306 - }
307 -
308 - var cur uint64
309 - for i, bs := range f.GetBlocksizes() {
310 - // We found the correct child to write into
311 - if cur+bs > offset {
312 - child, err := node.Links()[i].GetNode(dm.ctx, dm.dagserv)
313 - if err != nil {
314 - return nil, err
315 - }
316 -
317 - k, err := dm.modifyDag(child, offset-cur)
318 - if err != nil {
319 - return nil, err
320 - }
321 -
322 - node.Links()[i].Cid = k
323 -
324 - // Recache serialized node
325 - _, err = node.EncodeProtobuf(true)
326 - if err != nil {
327 - return nil, err
328 - }
329 -
330 - if dm.wrBuf.Len() == 0 {
331 - // No more bytes to write!
332 - break
333 - }
334 - offset = cur + bs
335 - }
336 - cur += bs
337 - }
338 -
339 - err = dm.dagserv.Add(dm.ctx, node)
340 - return node.Cid(), err
341 -}
342 -
343 -// appendData appends the blocks from the given chan to the end of this dag
344 -func (dm *DagModifier) appendData(nd ipld.Node, spl chunker.Splitter) (ipld.Node, error) {
345 - switch nd := nd.(type) {
346 - case *mdag.ProtoNode, *mdag.RawNode:
347 - dbp := &help.DagBuilderParams{
348 - Dagserv: dm.dagserv,
349 - Maxlinks: help.DefaultLinksPerBlock,
350 - Prefix: &dm.Prefix,
351 - RawLeaves: dm.RawLeaves,
352 - }
353 - return trickle.Append(dm.ctx, nd, dbp.New(spl))
354 - default:
355 - return nil, ErrNotUnixfs
356 - }
357 -}
358 -
359 -// Read data from this dag starting at the current offset
360 -func (dm *DagModifier) Read(b []byte) (int, error) {
361 - err := dm.readPrep()
362 - if err != nil {
363 - return 0, err
364 - }
365 -
366 - n, err := dm.read.Read(b)
367 - dm.curWrOff += uint64(n)
368 - return n, err
369 -}
370 -
371 -func (dm *DagModifier) readPrep() error {
372 - err := dm.Sync()
373 - if err != nil {
374 - return err
375 - }
376 -
377 - if dm.read == nil {
378 - ctx, cancel := context.WithCancel(dm.ctx)
379 - dr, err := uio.NewDagReader(ctx, dm.curNode, dm.dagserv)
380 - if err != nil {
381 - cancel()
382 - return err
383 - }
384 -
385 - i, err := dr.Seek(int64(dm.curWrOff), io.SeekStart)
386 - if err != nil {
387 - cancel()
388 - return err
389 - }
390 -
391 - if i != int64(dm.curWrOff) {
392 - cancel()
393 - return ErrSeekFail
394 - }
395 -
396 - dm.readCancel = cancel
397 - dm.read = dr
398 - }
399 -
400 - return nil
401 -}
402 -
403 -// CtxReadFull reads data from this dag starting at the current offset
404 -func (dm *DagModifier) CtxReadFull(ctx context.Context, b []byte) (int, error) {
405 - err := dm.readPrep()
406 - if err != nil {
407 - return 0, err
408 - }
409 -
410 - n, err := dm.read.CtxReadFull(ctx, b)
411 - dm.curWrOff += uint64(n)
412 - return n, err
413 -}
414 -
415 -// GetNode gets the modified DAG Node
416 -func (dm *DagModifier) GetNode() (ipld.Node, error) {
417 - err := dm.Sync()
418 - if err != nil {
419 - return nil, err
420 - }
421 - return dm.curNode.Copy(), nil
422 -}
423 -
424 -// HasChanges returned whether or not there are unflushed changes to this dag
425 -func (dm *DagModifier) HasChanges() bool {
426 - return dm.wrBuf != nil
427 -}
428 -
429 -// Seek modifies the offset according to whence. See unixfs/io for valid whence
430 -// values.
431 -func (dm *DagModifier) Seek(offset int64, whence int) (int64, error) {
432 - err := dm.Sync()
433 - if err != nil {
434 - return 0, err
435 - }
436 -
437 - fisize, err := dm.Size()
438 - if err != nil {
439 - return 0, err
440 - }
441 -
442 - var newoffset uint64
443 - switch whence {
444 - case io.SeekCurrent:
445 - newoffset = dm.curWrOff + uint64(offset)
446 - case io.SeekStart:
447 - newoffset = uint64(offset)
448 - case io.SeekEnd:
449 - newoffset = uint64(fisize) - uint64(offset)
450 - default:
451 - return 0, ErrUnrecognizedWhence
452 - }
453 -
454 - if int64(newoffset) > fisize {
455 - if err := dm.expandSparse(int64(newoffset) - fisize); err != nil {
456 - return 0, err
457 - }
458 - }
459 - dm.curWrOff = newoffset
460 - dm.writeStart = newoffset
461 -
462 - if dm.read != nil {
463 - _, err = dm.read.Seek(offset, whence)
464 - if err != nil {
465 - return 0, err
466 - }
467 - }
468 -
469 - return int64(dm.curWrOff), nil
470 -}
471 -
472 -// Truncate truncates the current Node to 'size' and replaces it with the
473 -// new one.
474 -func (dm *DagModifier) Truncate(size int64) error {
475 - err := dm.Sync()
476 - if err != nil {
477 - return err
478 - }
479 -
480 - realSize, err := dm.Size()
481 - if err != nil {
482 - return err
483 - }
484 - if size == int64(realSize) {
485 - return nil
486 - }
487 -
488 - // Truncate can also be used to expand the file
489 - if size > int64(realSize) {
490 - return dm.expandSparse(int64(size) - realSize)
491 - }
492 -
493 - nnode, err := dagTruncate(dm.ctx, dm.curNode, uint64(size), dm.dagserv)
494 - if err != nil {
495 - return err
496 - }
497 -
498 - err = dm.dagserv.Add(dm.ctx, nnode)
499 - if err != nil {
500 - return err
501 - }
502 -
503 - dm.curNode = nnode
504 - return nil
505 -}
506 -
507 -// dagTruncate truncates the given node to 'size' and returns the modified Node
508 -func dagTruncate(ctx context.Context, n ipld.Node, size uint64, ds ipld.DAGService) (ipld.Node, error) {
509 - if len(n.Links()) == 0 {
510 - switch nd := n.(type) {
511 - case *mdag.ProtoNode:
512 - // TODO: this can likely be done without marshaling and remarshaling
513 - pbn, err := ft.FromBytes(nd.Data())
514 - if err != nil {
515 - return nil, err
516 - }
517 - nd.SetData(ft.WrapData(pbn.Data[:size]))
518 - return nd, nil
519 - case *mdag.RawNode:
520 - return mdag.NewRawNodeWPrefix(nd.RawData()[:size], nd.Cid().Prefix())
521 - }
522 - }
523 -
524 - nd, ok := n.(*mdag.ProtoNode)
525 - if !ok {
526 - return nil, ErrNotUnixfs
527 - }
528 -
529 - var cur uint64
530 - end := 0
531 - var modified ipld.Node
532 - ndata, err := ft.FSNodeFromBytes(nd.Data())
533 - if err != nil {
534 - return nil, err
535 - }
536 - // Reset the block sizes of the node to adjust them
537 - // with the new values of the truncated children.
538 - ndata.RemoveAllBlockSizes()
539 - for i, lnk := range nd.Links() {
540 - child, err := lnk.GetNode(ctx, ds)
541 - if err != nil {
542 - return nil, err
543 - }
544 -
545 - childsize, err := fileSize(child)
546 - if err != nil {
547 - return nil, err
548 - }
549 -
550 - // found the child we want to cut
551 - if size < cur+childsize {
552 - nchild, err := dagTruncate(ctx, child, size-cur, ds)
553 - if err != nil {
554 - return nil, err
555 - }
556 -
557 - ndata.AddBlockSize(size - cur)
558 -
559 - modified = nchild
560 - end = i
561 - break
562 - }
563 - cur += childsize
564 - ndata.AddBlockSize(childsize)
565 - }
566 -
567 - err = ds.Add(ctx, modified)
568 - if err != nil {
569 - return nil, err
570 - }
571 -
572 - nd.SetLinks(nd.Links()[:end])
573 - err = nd.AddNodeLink("", modified)
574 - if err != nil {
575 - return nil, err
576 - }
577 -
578 - d, err := ndata.GetBytes()
579 - if err != nil {
580 - return nil, err
581 - }
582 - // Save the new block sizes to the original node.
583 - nd.SetData(d)
584 -
585 - // invalidate cache and recompute serialized data
586 - _, err = nd.EncodeProtobuf(true)
587 - if err != nil {
588 - return nil, err
589 - }
590 -
591 - return nd, nil
592 -}
unixfs/mod/dagmodifier_test.go deleted
-803
@@ -1,803 +0,0 @@
1 -package mod
2 -
3 -import (
4 - "context"
5 - "fmt"
6 - "io"
7 - "io/ioutil"
8 - "testing"
9 -
10 - h "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/importer/helpers"
11 - trickle "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/importer/trickle"
12 -
13 - uio "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/io"
14 - testu "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/test"
15 -
16 - u "gx/ipfs/QmPdKqUcHGFdeSpvjVoaTRPPstGif9GBZb5Q56RVw9o69A/go-ipfs-util"
17 -)
18 -
19 -func testModWrite(t *testing.T, beg, size uint64, orig []byte, dm *DagModifier, opts testu.NodeOpts) []byte {
20 - newdata := make([]byte, size)
21 - r := u.NewTimeSeededRand()
22 - r.Read(newdata)
23 -
24 - if size+beg > uint64(len(orig)) {
25 - orig = append(orig, make([]byte, (size+beg)-uint64(len(orig)))...)
26 - }
27 - copy(orig[beg:], newdata)
28 -
29 - nmod, err := dm.WriteAt(newdata, int64(beg))
30 - if err != nil {
31 - t.Fatal(err)
32 - }
33 -
34 - if nmod != int(size) {
35 - t.Fatalf("Mod length not correct! %d != %d", nmod, size)
36 - }
37 -
38 - verifyNode(t, orig, dm, opts)
39 -
40 - return orig
41 -}
42 -
43 -func verifyNode(t *testing.T, orig []byte, dm *DagModifier, opts testu.NodeOpts) {
44 - nd, err := dm.GetNode()
45 - if err != nil {
46 - t.Fatal(err)
47 - }
48 -
49 - err = trickle.VerifyTrickleDagStructure(nd, trickle.VerifyParams{
50 - Getter: dm.dagserv,
51 - Direct: h.DefaultLinksPerBlock,
52 - LayerRepeat: 4,
53 - Prefix: &opts.Prefix,
54 - RawLeaves: opts.RawLeavesUsed,
55 - })
56 - if err != nil {
57 - t.Fatal(err)
58 - }
59 -
60 - rd, err := uio.NewDagReader(context.Background(), nd, dm.dagserv)
61 - if err != nil {
62 - t.Fatal(err)
63 - }
64 -
65 - after, err := ioutil.ReadAll(rd)
66 - if err != nil {
67 - t.Fatal(err)
68 - }
69 -
70 - err = testu.ArrComp(after, orig)
71 - if err != nil {
72 - t.Fatal(err)
73 - }
74 -}
75 -
76 -func runAllSubtests(t *testing.T, tfunc func(*testing.T, testu.NodeOpts)) {
77 - t.Run("opts=ProtoBufLeaves", func(t *testing.T) { tfunc(t, testu.UseProtoBufLeaves) })
78 - t.Run("opts=RawLeaves", func(t *testing.T) { tfunc(t, testu.UseRawLeaves) })
79 - t.Run("opts=CidV1", func(t *testing.T) { tfunc(t, testu.UseCidV1) })
80 - t.Run("opts=Blake2b256", func(t *testing.T) { tfunc(t, testu.UseBlake2b256) })
81 -}
82 -
83 -func TestDagModifierBasic(t *testing.T) {
84 - runAllSubtests(t, testDagModifierBasic)
85 -}
86 -func testDagModifierBasic(t *testing.T, opts testu.NodeOpts) {
87 - dserv := testu.GetDAGServ()
88 - b, n := testu.GetRandomNode(t, dserv, 50000, opts)
89 - ctx, cancel := context.WithCancel(context.Background())
90 - defer cancel()
91 -
92 - dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
93 - if err != nil {
94 - t.Fatal(err)
95 - }
96 - if opts.ForceRawLeaves {
97 - dagmod.RawLeaves = true
98 - }
99 -
100 - // Within zero block
101 - beg := uint64(15)
102 - length := uint64(60)
103 -
104 - t.Log("Testing mod within zero block")
105 - b = testModWrite(t, beg, length, b, dagmod, opts)
106 -
107 - // Within bounds of existing file
108 - beg = 1000
109 - length = 4000
110 - t.Log("Testing mod within bounds of existing multiblock file.")
111 - b = testModWrite(t, beg, length, b, dagmod, opts)
112 -
113 - // Extend bounds
114 - beg = 49500
115 - length = 4000
116 -
117 - t.Log("Testing mod that extends file.")
118 - b = testModWrite(t, beg, length, b, dagmod, opts)
119 -
120 - // "Append"
121 - beg = uint64(len(b))
122 - length = 3000
123 - t.Log("Testing pure append")
124 - _ = testModWrite(t, beg, length, b, dagmod, opts)
125 -
126 - // Verify reported length
127 - node, err := dagmod.GetNode()
128 - if err != nil {
129 - t.Fatal(err)
130 - }
131 -
132 - size, err := fileSize(node)
133 - if err != nil {
134 - t.Fatal(err)
135 - }
136 -
137 - expected := uint64(50000 + 3500 + 3000)
138 - if size != expected {
139 - t.Fatalf("Final reported size is incorrect [%d != %d]", size, expected)
140 - }
141 -}
142 -
143 -func TestMultiWrite(t *testing.T) {
144 - runAllSubtests(t, testMultiWrite)
145 -}
146 -func testMultiWrite(t *testing.T, opts testu.NodeOpts) {
147 - dserv := testu.GetDAGServ()
148 - n := testu.GetEmptyNode(t, dserv, opts)
149 -
150 - ctx, cancel := context.WithCancel(context.Background())
151 - defer cancel()
152 -
153 - dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
154 - if err != nil {
155 - t.Fatal(err)
156 - }
157 - if opts.ForceRawLeaves {
158 - dagmod.RawLeaves = true
159 - }
160 -
161 - data := make([]byte, 4000)
162 - u.NewTimeSeededRand().Read(data)
163 -
164 - for i := 0; i < len(data); i++ {
165 - n, err := dagmod.WriteAt(data[i:i+1], int64(i))
166 - if err != nil {
167 - t.Fatal(err)
168 - }
169 - if n != 1 {
170 - t.Fatal("Somehow wrote the wrong number of bytes! (n != 1)")
171 - }
172 -
173 - size, err := dagmod.Size()
174 - if err != nil {
175 - t.Fatal(err)
176 - }
177 -
178 - if size != int64(i+1) {
179 - t.Fatal("Size was reported incorrectly")
180 - }
181 - }
182 -
183 - verifyNode(t, data, dagmod, opts)
184 -}
185 -
186 -func TestMultiWriteAndFlush(t *testing.T) {
187 - runAllSubtests(t, testMultiWriteAndFlush)
188 -}
189 -func testMultiWriteAndFlush(t *testing.T, opts testu.NodeOpts) {
190 - dserv := testu.GetDAGServ()
191 - n := testu.GetEmptyNode(t, dserv, opts)
192 -
193 - ctx, cancel := context.WithCancel(context.Background())
194 - defer cancel()
195 -
196 - dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
197 - if err != nil {
198 - t.Fatal(err)
199 - }
200 - if opts.ForceRawLeaves {
201 - dagmod.RawLeaves = true
202 - }
203 -
204 - data := make([]byte, 20)
205 - u.NewTimeSeededRand().Read(data)
206 -
207 - for i := 0; i < len(data); i++ {
208 - n, err := dagmod.WriteAt(data[i:i+1], int64(i))
209 - if err != nil {
210 - t.Fatal(err)
211 - }
212 - if n != 1 {
213 - t.Fatal("Somehow wrote the wrong number of bytes! (n != 1)")
214 - }
215 - err = dagmod.Sync()
216 - if err != nil {
217 - t.Fatal(err)
218 - }
219 - }
220 -
221 - verifyNode(t, data, dagmod, opts)
222 -}
223 -
224 -func TestWriteNewFile(t *testing.T) {
225 - runAllSubtests(t, testWriteNewFile)
226 -}
227 -func testWriteNewFile(t *testing.T, opts testu.NodeOpts) {
228 - dserv := testu.GetDAGServ()
229 - n := testu.GetEmptyNode(t, dserv, opts)
230 -
231 - ctx, cancel := context.WithCancel(context.Background())
232 - defer cancel()
233 -
234 - dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
235 - if err != nil {
236 - t.Fatal(err)
237 - }
238 - if opts.ForceRawLeaves {
239 - dagmod.RawLeaves = true
240 - }
241 -
242 - towrite := make([]byte, 2000)
243 - u.NewTimeSeededRand().Read(towrite)
244 -
245 - nw, err := dagmod.Write(towrite)
246 - if err != nil {
247 - t.Fatal(err)
248 - }
249 - if nw != len(towrite) {
250 - t.Fatal("Wrote wrong amount")
251 - }
252 -
253 - verifyNode(t, towrite, dagmod, opts)
254 -}
255 -
256 -func TestMultiWriteCoal(t *testing.T) {
257 - runAllSubtests(t, testMultiWriteCoal)
258 -}
259 -func testMultiWriteCoal(t *testing.T, opts testu.NodeOpts) {
260 - dserv := testu.GetDAGServ()
261 - n := testu.GetEmptyNode(t, dserv, opts)
262 -
263 - ctx, cancel := context.WithCancel(context.Background())
264 - defer cancel()
265 -
266 - dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
267 - if err != nil {
268 - t.Fatal(err)
269 - }
270 - if opts.ForceRawLeaves {
271 - dagmod.RawLeaves = true
272 - }
273 -
274 - data := make([]byte, 1000)
275 - u.NewTimeSeededRand().Read(data)
276 -
277 - for i := 0; i < len(data); i++ {
278 - n, err := dagmod.WriteAt(data[:i+1], 0)
279 - if err != nil {
280 - fmt.Println("FAIL AT ", i)
281 - t.Fatal(err)
282 - }
283 - if n != i+1 {
284 - t.Fatal("Somehow wrote the wrong number of bytes! (n != 1)")
285 - }
286 -
287 - }
288 -
289 - verifyNode(t, data, dagmod, opts)
290 -}
291 -
292 -func TestLargeWriteChunks(t *testing.T) {
293 - runAllSubtests(t, testLargeWriteChunks)
294 -}
295 -func testLargeWriteChunks(t *testing.T, opts testu.NodeOpts) {
296 - dserv := testu.GetDAGServ()
297 - n := testu.GetEmptyNode(t, dserv, opts)
298 -
299 - ctx, cancel := context.WithCancel(context.Background())
300 - defer cancel()
301 -
302 - dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
303 - if err != nil {
304 - t.Fatal(err)
305 - }
306 - if opts.ForceRawLeaves {
307 - dagmod.RawLeaves = true
308 - }
309 -
310 - wrsize := 1000
311 - datasize := 10000000
312 - data := make([]byte, datasize)
313 -
314 - u.NewTimeSeededRand().Read(data)
315 -
316 - for i := 0; i < datasize/wrsize; i++ {
317 - n, err := dagmod.WriteAt(data[i*wrsize:(i+1)*wrsize], int64(i*wrsize))
318 - if err != nil {
319 - t.Fatal(err)
320 - }
321 - if n != wrsize {
322 - t.Fatal("failed to write buffer")
323 - }
324 - }
325 -
326 - _, err = dagmod.Seek(0, io.SeekStart)
327 - if err != nil {
328 - t.Fatal(err)
329 - }
330 -
331 - out, err := ioutil.ReadAll(dagmod)
332 - if err != nil {
333 - t.Fatal(err)
334 - }
335 -
336 - if err = testu.ArrComp(out, data); err != nil {
337 - t.Fatal(err)
338 - }
339 -}
340 -
341 -func TestDagTruncate(t *testing.T) {
342 - runAllSubtests(t, testDagTruncate)
343 -}
344 -func testDagTruncate(t *testing.T, opts testu.NodeOpts) {
345 - dserv := testu.GetDAGServ()
346 - b, n := testu.GetRandomNode(t, dserv, 50000, opts)
347 - ctx, cancel := context.WithCancel(context.Background())
348 - defer cancel()
349 -
350 - dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
351 - if err != nil {
352 - t.Fatal(err)
353 - }
354 - if opts.ForceRawLeaves {
355 - dagmod.RawLeaves = true
356 - }
357 -
358 - err = dagmod.Truncate(12345)
359 - if err != nil {
360 - t.Fatal(err)
361 - }
362 - size, err := dagmod.Size()
363 - if err != nil {
364 - t.Fatal(err)
365 - }
366 -
367 - if size != 12345 {
368 - t.Fatal("size was incorrect!")
369 - }
370 -
371 - _, err = dagmod.Seek(0, io.SeekStart)
372 - if err != nil {
373 - t.Fatal(err)
374 - }
375 -
376 - out, err := ioutil.ReadAll(dagmod)
377 - if err != nil {
378 - t.Fatal(err)
379 - }
380 -
381 - if err = testu.ArrComp(out, b[:12345]); err != nil {
382 - t.Fatal(err)
383 - }
384 -
385 - err = dagmod.Truncate(10)
386 - if err != nil {
387 - t.Fatal(err)
388 - }
389 -
390 - size, err = dagmod.Size()
391 - if err != nil {
392 - t.Fatal(err)
393 - }
394 -
395 - if size != 10 {
396 - t.Fatal("size was incorrect!")
397 - }
398 -
399 - err = dagmod.Truncate(0)
400 - if err != nil {
401 - t.Fatal(err)
402 - }
403 -
404 - size, err = dagmod.Size()
405 - if err != nil {
406 - t.Fatal(err)
407 - }
408 -
409 - if size != 0 {
410 - t.Fatal("size was incorrect!")
411 - }
412 -}
413 -
414 -// TestDagTruncateSameSize tests that a DAG truncated
415 -// to the same size (i.e., doing nothing) doesn't modify
416 -// the DAG (its hash).
417 -func TestDagTruncateSameSize(t *testing.T) {
418 - runAllSubtests(t, testDagTruncateSameSize)
419 -}
420 -func testDagTruncateSameSize(t *testing.T, opts testu.NodeOpts) {
421 - dserv := testu.GetDAGServ()
422 - _, n := testu.GetRandomNode(t, dserv, 50000, opts)
423 - ctx, cancel := context.WithCancel(context.Background())
424 - defer cancel()
425 -
426 - dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
427 - if err != nil {
428 - t.Fatal(err)
429 - }
430 - // Copied from `TestDagTruncate`.
431 -
432 - size, err := dagmod.Size()
433 - if err != nil {
434 - t.Fatal(err)
435 - }
436 -
437 - err = dagmod.Truncate(size)
438 - if err != nil {
439 - t.Fatal(err)
440 - }
441 -
442 - modifiedNode, err := dagmod.GetNode()
443 - if err != nil {
444 - t.Fatal(err)
445 - }
446 -
447 - if modifiedNode.Cid().Equals(n.Cid()) == false {
448 - t.Fatal("the node has been modified!")
449 - }
450 -}
451 -
452 -func TestSparseWrite(t *testing.T) {
453 - runAllSubtests(t, testSparseWrite)
454 -}
455 -func testSparseWrite(t *testing.T, opts testu.NodeOpts) {
456 - dserv := testu.GetDAGServ()
457 - n := testu.GetEmptyNode(t, dserv, opts)
458 - ctx, cancel := context.WithCancel(context.Background())
459 - defer cancel()
460 -
461 - dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
462 - if err != nil {
463 - t.Fatal(err)
464 - }
465 - if opts.ForceRawLeaves {
466 - dagmod.RawLeaves = true
467 - }
468 -
469 - buf := make([]byte, 5000)
470 - u.NewTimeSeededRand().Read(buf[2500:])
471 -
472 - wrote, err := dagmod.WriteAt(buf[2500:], 2500)
473 - if err != nil {
474 - t.Fatal(err)
475 - }
476 -
477 - if wrote != 2500 {
478 - t.Fatal("incorrect write amount")
479 - }
480 -
481 - _, err = dagmod.Seek(0, io.SeekStart)
482 - if err != nil {
483 - t.Fatal(err)
484 - }
485 -
486 - out, err := ioutil.ReadAll(dagmod)
487 - if err != nil {
488 - t.Fatal(err)
489 - }
490 -
491 - if err = testu.ArrComp(out, buf); err != nil {
492 - t.Fatal(err)
493 - }
494 -}
495 -
496 -func TestSeekPastEndWrite(t *testing.T) {
497 - runAllSubtests(t, testSeekPastEndWrite)
498 -}
499 -func testSeekPastEndWrite(t *testing.T, opts testu.NodeOpts) {
500 - dserv := testu.GetDAGServ()
501 - n := testu.GetEmptyNode(t, dserv, opts)
502 - ctx, cancel := context.WithCancel(context.Background())
503 - defer cancel()
504 -
505 - dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
506 - if err != nil {
507 - t.Fatal(err)
508 - }
509 - if opts.ForceRawLeaves {
510 - dagmod.RawLeaves = true
511 - }
512 -
513 - buf := make([]byte, 5000)
514 - u.NewTimeSeededRand().Read(buf[2500:])
515 -
516 - nseek, err := dagmod.Seek(2500, io.SeekStart)
517 - if err != nil {
518 - t.Fatal(err)
519 - }
520 -
521 - if nseek != 2500 {
522 - t.Fatal("failed to seek")
523 - }
524 -
525 - wrote, err := dagmod.Write(buf[2500:])
526 - if err != nil {
527 - t.Fatal(err)
528 - }
529 -
530 - if wrote != 2500 {
531 - t.Fatal("incorrect write amount")
532 - }
533 -
534 - _, err = dagmod.Seek(0, io.SeekStart)
535 - if err != nil {
536 - t.Fatal(err)
537 - }
538 -
539 - out, err := ioutil.ReadAll(dagmod)
540 - if err != nil {
541 - t.Fatal(err)
542 - }
543 -
544 - if err = testu.ArrComp(out, buf); err != nil {
545 - t.Fatal(err)
546 - }
547 -}
548 -
549 -func TestRelativeSeek(t *testing.T) {
550 - runAllSubtests(t, testRelativeSeek)
551 -}
552 -func testRelativeSeek(t *testing.T, opts testu.NodeOpts) {
553 - dserv := testu.GetDAGServ()
554 - n := testu.GetEmptyNode(t, dserv, opts)
555 - ctx, cancel := context.WithCancel(context.Background())
556 - defer cancel()
557 -
558 - dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
559 - if err != nil {
560 - t.Fatal(err)
561 - }
562 - if opts.ForceRawLeaves {
563 - dagmod.RawLeaves = true
564 - }
565 -
566 - for i := 0; i < 64; i++ {
567 - dagmod.Write([]byte{byte(i)})
568 - if _, err := dagmod.Seek(1, io.SeekCurrent); err != nil {
569 - t.Fatal(err)
570 - }
571 - }
572 -
573 - out, err := ioutil.ReadAll(dagmod)
574 - if err != nil {
575 - t.Fatal(err)
576 - }
577 -
578 - for i, v := range out {
579 - if v != 0 && i/2 != int(v) {
580 - t.Errorf("expected %d, at index %d, got %d", i/2, i, v)
581 - }
582 - }
583 -}
584 -
585 -func TestInvalidSeek(t *testing.T) {
586 - runAllSubtests(t, testInvalidSeek)
587 -}
588 -func testInvalidSeek(t *testing.T, opts testu.NodeOpts) {
589 - dserv := testu.GetDAGServ()
590 - n := testu.GetEmptyNode(t, dserv, opts)
591 - ctx, cancel := context.WithCancel(context.Background())
592 - defer cancel()
593 -
594 - dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
595 - if err != nil {
596 - t.Fatal(err)
597 - }
598 - if opts.ForceRawLeaves {
599 - dagmod.RawLeaves = true
600 - }
601 -
602 - _, err = dagmod.Seek(10, -10)
603 -
604 - if err != ErrUnrecognizedWhence {
605 - t.Fatal(err)
606 - }
607 -}
608 -
609 -func TestEndSeek(t *testing.T) {
610 - runAllSubtests(t, testEndSeek)
611 -}
612 -func testEndSeek(t *testing.T, opts testu.NodeOpts) {
613 - dserv := testu.GetDAGServ()
614 -
615 - n := testu.GetEmptyNode(t, dserv, opts)
616 - ctx, cancel := context.WithCancel(context.Background())
617 - defer cancel()
618 -
619 - dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
620 - if err != nil {
621 - t.Fatal(err)
622 - }
623 - if opts.ForceRawLeaves {
624 - dagmod.RawLeaves = true
625 - }
626 -
627 - _, err = dagmod.Write(make([]byte, 100))
628 - if err != nil {
629 - t.Fatal(err)
630 - }
631 -
632 - offset, err := dagmod.Seek(0, io.SeekCurrent)
633 - if err != nil {
634 - t.Fatal(err)
635 - }
636 - if offset != 100 {
637 - t.Fatal("expected the relative seek 0 to return current location")
638 - }
639 -
640 - offset, err = dagmod.Seek(0, io.SeekStart)
641 - if err != nil {
642 - t.Fatal(err)
643 - }
644 - if offset != 0 {
645 - t.Fatal("expected the absolute seek to set offset at 0")
646 - }
647 -
648 - offset, err = dagmod.Seek(0, io.SeekEnd)
649 - if err != nil {
650 - t.Fatal(err)
651 - }
652 - if offset != 100 {
653 - t.Fatal("expected the end seek to set offset at end")
654 - }
655 -}
656 -
657 -func TestReadAndSeek(t *testing.T) {
658 - runAllSubtests(t, testReadAndSeek)
659 -}
660 -func testReadAndSeek(t *testing.T, opts testu.NodeOpts) {
661 - dserv := testu.GetDAGServ()
662 -
663 - n := testu.GetEmptyNode(t, dserv, opts)
664 - ctx, cancel := context.WithCancel(context.Background())
665 - defer cancel()
666 -
667 - dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
668 - if err != nil {
669 - t.Fatal(err)
670 - }
671 - if opts.ForceRawLeaves {
672 - dagmod.RawLeaves = true
673 - }
674 -
675 - writeBuf := []byte{0, 1, 2, 3, 4, 5, 6, 7}
676 - dagmod.Write(writeBuf)
677 -
678 - if !dagmod.HasChanges() {
679 - t.Fatal("there are changes, this should be true")
680 - }
681 -
682 - readBuf := make([]byte, 4)
683 - offset, err := dagmod.Seek(0, io.SeekStart)
684 - if offset != 0 {
685 - t.Fatal("expected offset to be 0")
686 - }
687 - if err != nil {
688 - t.Fatal(err)
689 - }
690 -
691 - // read 0,1,2,3
692 - c, err := dagmod.Read(readBuf)
693 - if err != nil {
694 - t.Fatal(err)
695 - }
696 - if c != 4 {
697 - t.Fatalf("expected length of 4 got %d", c)
698 - }
699 -
700 - for i := byte(0); i < 4; i++ {
701 - if readBuf[i] != i {
702 - t.Fatalf("wrong value %d [at index %d]", readBuf[i], i)
703 - }
704 - }
705 -
706 - // skip 4
707 - _, err = dagmod.Seek(1, io.SeekCurrent)
708 - if err != nil {
709 - t.Fatalf("error: %s, offset %d, reader offset %d", err, dagmod.curWrOff, getOffset(dagmod.read))
710 - }
711 -
712 - //read 5,6,7
713 - readBuf = make([]byte, 3)
714 - c, err = dagmod.Read(readBuf)
715 - if err != nil {
716 - t.Fatal(err)
717 - }
718 - if c != 3 {
719 - t.Fatalf("expected length of 3 got %d", c)
720 - }
721 -
722 - for i := byte(0); i < 3; i++ {
723 - if readBuf[i] != i+5 {
724 - t.Fatalf("wrong value %d [at index %d]", readBuf[i], i)
725 - }
726 -
727 - }
728 -
729 -}
730 -
731 -func TestCtxRead(t *testing.T) {
732 - runAllSubtests(t, testCtxRead)
733 -}
734 -func testCtxRead(t *testing.T, opts testu.NodeOpts) {
735 - dserv := testu.GetDAGServ()
736 -
737 - n := testu.GetEmptyNode(t, dserv, opts)
738 - ctx, cancel := context.WithCancel(context.Background())
739 - defer cancel()
740 -
741 - dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
742 - if err != nil {
743 - t.Fatal(err)
744 - }
745 - if opts.ForceRawLeaves {
746 - dagmod.RawLeaves = true
747 - }
748 -
749 - _, err = dagmod.Write([]byte{0, 1, 2, 3, 4, 5, 6, 7})
750 - if err != nil {
751 - t.Fatal(err)
752 - }
753 - dagmod.Seek(0, io.SeekStart)
754 -
755 - readBuf := make([]byte, 4)
756 - _, err = dagmod.CtxReadFull(ctx, readBuf)
757 - if err != nil {
758 - t.Fatal(err)
759 - }
760 - err = testu.ArrComp(readBuf, []byte{0, 1, 2, 3})
761 - if err != nil {
762 - t.Fatal(err)
763 - }
764 - // TODO(Kubuxu): context cancel case, I will do it after I figure out dagreader tests,
765 - // because this is exacelly the same.
766 -}
767 -
768 -func BenchmarkDagmodWrite(b *testing.B) {
769 - b.StopTimer()
770 - dserv := testu.GetDAGServ()
771 - n := testu.GetEmptyNode(b, dserv, testu.UseProtoBufLeaves)
772 - ctx, cancel := context.WithCancel(context.Background())
773 - defer cancel()
774 -
775 - wrsize := 4096
776 -
777 - dagmod, err := NewDagModifier(ctx, n, dserv, testu.SizeSplitterGen(512))
778 - if err != nil {
779 - b.Fatal(err)
780 - }
781 -
782 - buf := make([]byte, b.N*wrsize)
783 - u.NewTimeSeededRand().Read(buf)
784 - b.StartTimer()
785 - b.SetBytes(int64(wrsize))
786 - for i := 0; i < b.N; i++ {
787 - n, err := dagmod.Write(buf[i*wrsize : (i+1)*wrsize])
788 - if err != nil {
789 - b.Fatal(err)
790 - }
791 - if n != wrsize {
792 - b.Fatal("Wrote bad size")
793 - }
794 - }
795 -}
796 -
797 -func getOffset(reader uio.DagReader) int64 {
798 - offset, err := reader.Seek(0, io.SeekCurrent)
799 - if err != nil {
800 - panic("failed to retrieve offset: " + err.Error())
801 - }
802 - return offset
803 -}
unixfs/pb/Rules.mk deleted
-8
@@ -1,8 +0,0 @@
1 -include mk/header.mk
2 -
3 -PB_$(d) = $(wildcard $(d)/*.proto)
4 -TGTS_$(d) = $(PB_$(d):.proto=.pb.go)
5 -
6 -#DEPS_GO += $(TGTS_$(d))
7 -
8 -include mk/footer.mk
unixfs/pb/unixfs.pb.go deleted
-147
@@ -1,147 +0,0 @@
1 -// Code generated by protoc-gen-gogo.
2 -// source: unixfs.proto
3 -// DO NOT EDIT!
4 -
5 -/*
6 -Package unixfs_pb is a generated protocol buffer package.
7 -
8 -It is generated from these files:
9 - unixfs.proto
10 -
11 -It has these top-level messages:
12 - Data
13 - Metadata
14 -*/
15 -package unixfs_pb
16 -
17 -import proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
18 -import fmt "fmt"
19 -import math "math"
20 -
21 -// Reference imports to suppress errors if they are not otherwise used.
22 -var _ = proto.Marshal
23 -var _ = fmt.Errorf
24 -var _ = math.Inf
25 -
26 -type Data_DataType int32
27 -
28 -const (
29 - Data_Raw Data_DataType = 0
30 - Data_Directory Data_DataType = 1
31 - Data_File Data_DataType = 2
32 - Data_Metadata Data_DataType = 3
33 - Data_Symlink Data_DataType = 4
34 - Data_HAMTShard Data_DataType = 5
35 -)
36 -
37 -var Data_DataType_name = map[int32]string{
38 - 0: "Raw",
39 - 1: "Directory",
40 - 2: "File",
41 - 3: "Metadata",
42 - 4: "Symlink",
43 - 5: "HAMTShard",
44 -}
45 -var Data_DataType_value = map[string]int32{
46 - "Raw": 0,
47 - "Directory": 1,
48 - "File": 2,
49 - "Metadata": 3,
50 - "Symlink": 4,
51 - "HAMTShard": 5,
52 -}
53 -
54 -func (x Data_DataType) Enum() *Data_DataType {
55 - p := new(Data_DataType)
56 - *p = x
57 - return p
58 -}
59 -func (x Data_DataType) String() string {
60 - return proto.EnumName(Data_DataType_name, int32(x))
61 -}
62 -func (x *Data_DataType) UnmarshalJSON(data []byte) error {
63 - value, err := proto.UnmarshalJSONEnum(Data_DataType_value, data, "Data_DataType")
64 - if err != nil {
65 - return err
66 - }
67 - *x = Data_DataType(value)
68 - return nil
69 -}
70 -
71 -type Data struct {
72 - Type *Data_DataType `protobuf:"varint,1,req,name=Type,enum=unixfs.pb.Data_DataType" json:"Type,omitempty"`
73 - Data []byte `protobuf:"bytes,2,opt,name=Data" json:"Data,omitempty"`
74 - Filesize *uint64 `protobuf:"varint,3,opt,name=filesize" json:"filesize,omitempty"`
75 - Blocksizes []uint64 `protobuf:"varint,4,rep,name=blocksizes" json:"blocksizes,omitempty"`
76 - HashType *uint64 `protobuf:"varint,5,opt,name=hashType" json:"hashType,omitempty"`
77 - Fanout *uint64 `protobuf:"varint,6,opt,name=fanout" json:"fanout,omitempty"`
78 - XXX_unrecognized []byte `json:"-"`
79 -}
80 -
81 -func (m *Data) Reset() { *m = Data{} }
82 -func (m *Data) String() string { return proto.CompactTextString(m) }
83 -func (*Data) ProtoMessage() {}
84 -
85 -func (m *Data) GetType() Data_DataType {
86 - if m != nil && m.Type != nil {
87 - return *m.Type
88 - }
89 - return Data_Raw
90 -}
91 -
92 -func (m *Data) GetData() []byte {
93 - if m != nil {
94 - return m.Data
95 - }
96 - return nil
97 -}
98 -
99 -func (m *Data) GetFilesize() uint64 {
100 - if m != nil && m.Filesize != nil {
101 - return *m.Filesize
102 - }
103 - return 0
104 -}
105 -
106 -func (m *Data) GetBlocksizes() []uint64 {
107 - if m != nil {
108 - return m.Blocksizes
109 - }
110 - return nil
111 -}
112 -
113 -func (m *Data) GetHashType() uint64 {
114 - if m != nil && m.HashType != nil {
115 - return *m.HashType
116 - }
117 - return 0
118 -}
119 -
120 -func (m *Data) GetFanout() uint64 {
121 - if m != nil && m.Fanout != nil {
122 - return *m.Fanout
123 - }
124 - return 0
125 -}
126 -
127 -type Metadata struct {
128 - MimeType *string `protobuf:"bytes,1,opt,name=MimeType" json:"MimeType,omitempty"`
129 - XXX_unrecognized []byte `json:"-"`
130 -}
131 -
132 -func (m *Metadata) Reset() { *m = Metadata{} }
133 -func (m *Metadata) String() string { return proto.CompactTextString(m) }
134 -func (*Metadata) ProtoMessage() {}
135 -
136 -func (m *Metadata) GetMimeType() string {
137 - if m != nil && m.MimeType != nil {
138 - return *m.MimeType
139 - }
140 - return ""
141 -}
142 -
143 -func init() {
144 - proto.RegisterType((*Data)(nil), "unixfs.pb.Data")
145 - proto.RegisterType((*Metadata)(nil), "unixfs.pb.Metadata")
146 - proto.RegisterEnum("unixfs.pb.Data_DataType", Data_DataType_name, Data_DataType_value)
147 -}
unixfs/pb/unixfs.proto deleted
-24
@@ -1,24 +0,0 @@
1 -package unixfs.pb;
2 -
3 -message Data {
4 - enum DataType {
5 - Raw = 0;
6 - Directory = 1;
7 - File = 2;
8 - Metadata = 3;
9 - Symlink = 4;
10 - HAMTShard = 5;
11 - }
12 -
13 - required DataType Type = 1;
14 - optional bytes Data = 2;
15 - optional uint64 filesize = 3;
16 - repeated uint64 blocksizes = 4;
17 -
18 - optional uint64 hashType = 5;
19 - optional uint64 fanout = 6;
20 -}
21 -
22 -message Metadata {
23 - optional string MimeType = 1;
24 -}
unixfs/test/utils.go deleted
-135
@@ -1,135 +0,0 @@
1 -package testu
2 -
3 -import (
4 - "bytes"
5 - "context"
6 - "fmt"
7 - "io"
8 - "io/ioutil"
9 - "testing"
10 -
11 - mdag "gx/ipfs/QmRy4Qk9hbgFX9NGJRm8rBThrA8PZhNCitMgeRYyZ67s59/go-merkledag"
12 - mdagmock "gx/ipfs/QmRy4Qk9hbgFX9NGJRm8rBThrA8PZhNCitMgeRYyZ67s59/go-merkledag/test"
13 - ft "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs"
14 - h "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/importer/helpers"
15 - trickle "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/importer/trickle"
16 -
17 - u "gx/ipfs/QmPdKqUcHGFdeSpvjVoaTRPPstGif9GBZb5Q56RVw9o69A/go-ipfs-util"
18 - mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash"
19 - chunker "gx/ipfs/QmVDjhUMtkRskBFAVNwyXuLSKbeAya7JKPnzAxMKDaK4x4/go-ipfs-chunker"
20 - cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
21 - ipld "gx/ipfs/QmZtNq8dArGfnpCZfx2pUNY7UcjGhVp5qqwQ4hH6mpTMRQ/go-ipld-format"
22 -)
23 -
24 -// SizeSplitterGen creates a generator.
25 -func SizeSplitterGen(size int64) chunker.SplitterGen {
26 - return func(r io.Reader) chunker.Splitter {
27 - return chunker.NewSizeSplitter(r, size)
28 - }
29 -}
30 -
31 -// GetDAGServ returns a mock DAGService.
32 -func GetDAGServ() ipld.DAGService {
33 - return mdagmock.Mock()
34 -}
35 -
36 -// NodeOpts is used by GetNode, GetEmptyNode and GetRandomNode
37 -type NodeOpts struct {
38 - Prefix cid.Prefix
39 - // ForceRawLeaves if true will force the use of raw leaves
40 - ForceRawLeaves bool
41 - // RawLeavesUsed is true if raw leaves or either implicitly or explicitly enabled
42 - RawLeavesUsed bool
43 -}
44 -
45 -// Some shorthands for NodeOpts.
46 -var (
47 - UseProtoBufLeaves = NodeOpts{Prefix: mdag.V0CidPrefix()}
48 - UseRawLeaves = NodeOpts{Prefix: mdag.V0CidPrefix(), ForceRawLeaves: true, RawLeavesUsed: true}
49 - UseCidV1 = NodeOpts{Prefix: mdag.V1CidPrefix(), RawLeavesUsed: true}
50 - UseBlake2b256 NodeOpts
51 -)
52 -
53 -func init() {
54 - UseBlake2b256 = UseCidV1
55 - UseBlake2b256.Prefix.MhType = mh.Names["blake2b-256"]
56 - UseBlake2b256.Prefix.MhLength = -1
57 -}
58 -
59 -// GetNode returns a unixfs file node with the specified data.
60 -func GetNode(t testing.TB, dserv ipld.DAGService, data []byte, opts NodeOpts) ipld.Node {
61 - in := bytes.NewReader(data)
62 -
63 - dbp := h.DagBuilderParams{
64 - Dagserv: dserv,
65 - Maxlinks: h.DefaultLinksPerBlock,
66 - Prefix: &opts.Prefix,
67 - RawLeaves: opts.RawLeavesUsed,
68 - }
69 -
70 - node, err := trickle.Layout(dbp.New(SizeSplitterGen(500)(in)))
71 - if err != nil {
72 - t.Fatal(err)
73 - }
74 -
75 - return node
76 -}
77 -
78 -// GetEmptyNode returns an empty unixfs file node.
79 -func GetEmptyNode(t testing.TB, dserv ipld.DAGService, opts NodeOpts) ipld.Node {
80 - return GetNode(t, dserv, []byte{}, opts)
81 -}
82 -
83 -// GetRandomNode returns a random unixfs file node.
84 -func GetRandomNode(t testing.TB, dserv ipld.DAGService, size int64, opts NodeOpts) ([]byte, ipld.Node) {
85 - in := io.LimitReader(u.NewTimeSeededRand(), size)
86 - buf, err := ioutil.ReadAll(in)
87 - if err != nil {
88 - t.Fatal(err)
89 - }
90 -
91 - node := GetNode(t, dserv, buf, opts)
92 - return buf, node
93 -}
94 -
95 -// ArrComp checks if two byte slices are the same.
96 -func ArrComp(a, b []byte) error {
97 - if len(a) != len(b) {
98 - return fmt.Errorf("arrays differ in length. %d != %d", len(a), len(b))
99 - }
100 - for i, v := range a {
101 - if v != b[i] {
102 - return fmt.Errorf("arrays differ at index: %d", i)
103 - }
104 - }
105 - return nil
106 -}
107 -
108 -// PrintDag pretty-prints the given dag to stdout.
109 -func PrintDag(nd *mdag.ProtoNode, ds ipld.DAGService, indent int) {
110 - pbd, err := ft.FromBytes(nd.Data())
111 - if err != nil {
112 - panic(err)
113 - }
114 -
115 - for i := 0; i < indent; i++ {
116 - fmt.Print(" ")
117 - }
118 - fmt.Printf("{size = %d, type = %s, children = %d", pbd.GetFilesize(), pbd.GetType().String(), len(pbd.GetBlocksizes()))
119 - if len(nd.Links()) > 0 {
120 - fmt.Println()
121 - }
122 - for _, lnk := range nd.Links() {
123 - child, err := lnk.GetNode(context.Background(), ds)
124 - if err != nil {
125 - panic(err)
126 - }
127 - PrintDag(child.(*mdag.ProtoNode), ds, indent+1)
128 - }
129 - if len(nd.Links()) > 0 {
130 - for i := 0; i < indent; i++ {
131 - fmt.Print(" ")
132 - }
133 - }
134 - fmt.Println("}")
135 -}
unixfs/unixfs.go deleted
-305
@@ -1,305 +0,0 @@
1 -// Package unixfs implements a data format for files in the IPFS filesystem It
2 -// is not the only format in ipfs, but it is the one that the filesystem
3 -// assumes
4 -package unixfs
5 -
6 -import (
7 - "errors"
8 -
9 - proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
10 -
11 - dag "gx/ipfs/QmRy4Qk9hbgFX9NGJRm8rBThrA8PZhNCitMgeRYyZ67s59/go-merkledag"
12 - pb "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/pb"
13 -)
14 -
15 -// Shorthands for protobuffer types
16 -const (
17 - TRaw = pb.Data_Raw
18 - TFile = pb.Data_File
19 - TDirectory = pb.Data_Directory
20 - TMetadata = pb.Data_Metadata
21 - TSymlink = pb.Data_Symlink
22 - THAMTShard = pb.Data_HAMTShard
23 -)
24 -
25 -// Common errors
26 -var (
27 - ErrMalformedFileFormat = errors.New("malformed data in file format")
28 - ErrUnrecognizedType = errors.New("unrecognized node type")
29 -)
30 -
31 -// FromBytes unmarshals a byte slice as protobuf Data.
32 -func FromBytes(data []byte) (*pb.Data, error) {
33 - pbdata := new(pb.Data)
34 - err := proto.Unmarshal(data, pbdata)
35 - if err != nil {
36 - return nil, err
37 - }
38 - return pbdata, nil
39 -}
40 -
41 -// FilePBData creates a protobuf File with the given
42 -// byte slice and returns the marshaled protobuf bytes representing it.
43 -func FilePBData(data []byte, totalsize uint64) []byte {
44 - pbfile := new(pb.Data)
45 - typ := pb.Data_File
46 - pbfile.Type = &typ
47 - pbfile.Data = data
48 - pbfile.Filesize = proto.Uint64(totalsize)
49 -
50 - data, err := proto.Marshal(pbfile)
51 - if err != nil {
52 - // This really shouldnt happen, i promise
53 - // The only failure case for marshal is if required fields
54 - // are not filled out, and they all are. If the proto object
55 - // gets changed and nobody updates this function, the code
56 - // should panic due to programmer error
57 - panic(err)
58 - }
59 - return data
60 -}
61 -
62 -//FolderPBData returns Bytes that represent a Directory.
63 -func FolderPBData() []byte {
64 - pbfile := new(pb.Data)
65 - typ := pb.Data_Directory
66 - pbfile.Type = &typ
67 -
68 - data, err := proto.Marshal(pbfile)
69 - if err != nil {
70 - //this really shouldnt happen, i promise
71 - panic(err)
72 - }
73 - return data
74 -}
75 -
76 -//WrapData marshals raw bytes into a `Data_Raw` type protobuf message.
77 -func WrapData(b []byte) []byte {
78 - pbdata := new(pb.Data)
79 - typ := pb.Data_Raw
80 - pbdata.Data = b
81 - pbdata.Type = &typ
82 - pbdata.Filesize = proto.Uint64(uint64(len(b)))
83 -
84 - out, err := proto.Marshal(pbdata)
85 - if err != nil {
86 - // This shouldnt happen. seriously.
87 - panic(err)
88 - }
89 -
90 - return out
91 -}
92 -
93 -//SymlinkData returns a `Data_Symlink` protobuf message for the path you specify.
94 -func SymlinkData(path string) ([]byte, error) {
95 - pbdata := new(pb.Data)
96 - typ := pb.Data_Symlink
97 - pbdata.Data = []byte(path)
98 - pbdata.Type = &typ
99 -
100 - out, err := proto.Marshal(pbdata)
101 - if err != nil {
102 - return nil, err
103 - }
104 -
105 - return out, nil
106 -}
107 -
108 -// UnwrapData unmarshals a protobuf messages and returns the contents.
109 -func UnwrapData(data []byte) ([]byte, error) {
110 - pbdata := new(pb.Data)
111 - err := proto.Unmarshal(data, pbdata)
112 - if err != nil {
113 - return nil, err
114 - }
115 - return pbdata.GetData(), nil
116 -}
117 -
118 -// DataSize returns the size of the contents in protobuf wrapped slice.
119 -// For raw data it simply provides the length of it. For Data_Files, it
120 -// will return the associated filesize. Note that Data_Directories will
121 -// return an error.
122 -func DataSize(data []byte) (uint64, error) {
123 - pbdata := new(pb.Data)
124 - err := proto.Unmarshal(data, pbdata)
125 - if err != nil {
126 - return 0, err
127 - }
128 -
129 - switch pbdata.GetType() {
130 - case pb.Data_Directory:
131 - return 0, errors.New("can't get data size of directory")
132 - case pb.Data_File:
133 - return pbdata.GetFilesize(), nil
134 - case pb.Data_Raw:
135 - return uint64(len(pbdata.GetData())), nil
136 - default:
137 - return 0, errors.New("unrecognized node data type")
138 - }
139 -}
140 -
141 -// An FSNode represents a filesystem object using the UnixFS specification.
142 -//
143 -// The `NewFSNode` constructor should be used instead of just calling `new(FSNode)`
144 -// to guarantee that the required (`Type` and `Filesize`) fields in the `format`
145 -// structure are initialized before marshaling (in `GetBytes()`).
146 -type FSNode struct {
147 -
148 - // UnixFS format defined as a protocol buffers message.
149 - format pb.Data
150 -}
151 -
152 -// FSNodeFromBytes unmarshal a protobuf message onto an FSNode.
153 -func FSNodeFromBytes(b []byte) (*FSNode, error) {
154 - n := new(FSNode)
155 - err := proto.Unmarshal(b, &n.format)
156 - if err != nil {
157 - return nil, err
158 - }
159 -
160 - return n, nil
161 -}
162 -
163 -// NewFSNode creates a new FSNode structure with the given `dataType`.
164 -//
165 -// It initializes the (required) `Type` field (that doesn't have a `Set()`
166 -// accessor so it must be specified at creation), otherwise the `Marshal()`
167 -// method in `GetBytes()` would fail (`required field "Type" not set`).
168 -//
169 -// It also initializes the `Filesize` pointer field to ensure its value
170 -// is never nil before marshaling, this is not a required field but it is
171 -// done to be backwards compatible with previous `go-ipfs` versions hash.
172 -// (If it wasn't initialized there could be cases where `Filesize` could
173 -// have been left at nil, when the `FSNode` was created but no data or
174 -// child nodes were set to adjust it, as is the case in `NewLeaf()`.)
175 -func NewFSNode(dataType pb.Data_DataType) *FSNode {
176 - n := new(FSNode)
177 - n.format.Type = &dataType
178 -
179 - // Initialize by `Filesize` by updating it with a dummy (zero) value.
180 - n.UpdateFilesize(0)
181 -
182 - return n
183 -}
184 -
185 -// AddBlockSize adds the size of the next child block of this node
186 -func (n *FSNode) AddBlockSize(s uint64) {
187 - n.UpdateFilesize(int64(s))
188 - n.format.Blocksizes = append(n.format.Blocksizes, s)
189 -}
190 -
191 -// RemoveBlockSize removes the given child block's size.
192 -func (n *FSNode) RemoveBlockSize(i int) {
193 - n.UpdateFilesize(-int64(n.format.Blocksizes[i]))
194 - n.format.Blocksizes = append(n.format.Blocksizes[:i], n.format.Blocksizes[i+1:]...)
195 -}
196 -
197 -// BlockSize returns the block size indexed by `i`.
198 -// TODO: Evaluate if this function should be bounds checking.
199 -func (n *FSNode) BlockSize(i int) uint64 {
200 - return n.format.Blocksizes[i]
201 -}
202 -
203 -// RemoveAllBlockSizes removes all the child block sizes of this node.
204 -func (n *FSNode) RemoveAllBlockSizes() {
205 - n.format.Blocksizes = []uint64{}
206 - n.format.Filesize = proto.Uint64(uint64(len(n.Data())))
207 -}
208 -
209 -// GetBytes marshals this node as a protobuf message.
210 -func (n *FSNode) GetBytes() ([]byte, error) {
211 - return proto.Marshal(&n.format)
212 -}
213 -
214 -// FileSize returns the total size of this tree. That is, the size of
215 -// the data in this node plus the size of all its children.
216 -func (n *FSNode) FileSize() uint64 {
217 - return n.format.GetFilesize()
218 -}
219 -
220 -// NumChildren returns the number of child blocks of this node
221 -func (n *FSNode) NumChildren() int {
222 - return len(n.format.Blocksizes)
223 -}
224 -
225 -// Data retrieves the `Data` field from the internal `format`.
226 -func (n *FSNode) Data() []byte {
227 - return n.format.GetData()
228 -}
229 -
230 -// SetData sets the `Data` field from the internal `format`
231 -// updating its `Filesize`.
232 -func (n *FSNode) SetData(newData []byte) {
233 - n.UpdateFilesize(int64(len(newData) - len(n.Data())))
234 - n.format.Data = newData
235 -}
236 -
237 -// UpdateFilesize updates the `Filesize` field from the internal `format`
238 -// by a signed difference (`filesizeDiff`).
239 -// TODO: Add assert to check for `Filesize` > 0?
240 -func (n *FSNode) UpdateFilesize(filesizeDiff int64) {
241 - n.format.Filesize = proto.Uint64(uint64(
242 - int64(n.format.GetFilesize()) + filesizeDiff))
243 -}
244 -
245 -// Type retrieves the `Type` field from the internal `format`.
246 -func (n *FSNode) Type() pb.Data_DataType {
247 - return n.format.GetType()
248 -}
249 -
250 -// Metadata is used to store additional FSNode information.
251 -type Metadata struct {
252 - MimeType string
253 - Size uint64
254 -}
255 -
256 -// MetadataFromBytes Unmarshals a protobuf Data message into Metadata.
257 -// The provided slice should have been encoded with BytesForMetadata().
258 -func MetadataFromBytes(b []byte) (*Metadata, error) {
259 - pbd := new(pb.Data)
260 - err := proto.Unmarshal(b, pbd)
261 - if err != nil {
262 - return nil, err
263 - }
264 - if pbd.GetType() != pb.Data_Metadata {
265 - return nil, errors.New("incorrect node type")
266 - }
267 -
268 - pbm := new(pb.Metadata)
269 - err = proto.Unmarshal(pbd.Data, pbm)
270 - if err != nil {
271 - return nil, err
272 - }
273 - md := new(Metadata)
274 - md.MimeType = pbm.GetMimeType()
275 - return md, nil
276 -}
277 -
278 -// Bytes marshals Metadata as a protobuf message of Metadata type.
279 -func (m *Metadata) Bytes() ([]byte, error) {
280 - pbm := new(pb.Metadata)
281 - pbm.MimeType = &m.MimeType
282 - return proto.Marshal(pbm)
283 -}
284 -
285 -// BytesForMetadata wraps the given Metadata as a profobuf message of Data type,
286 -// setting the DataType to Metadata. The wrapped bytes are itself the
287 -// result of calling m.Bytes().
288 -func BytesForMetadata(m *Metadata) ([]byte, error) {
289 - pbd := new(pb.Data)
290 - pbd.Filesize = proto.Uint64(m.Size)
291 - typ := pb.Data_Metadata
292 - pbd.Type = &typ
293 - mdd, err := m.Bytes()
294 - if err != nil {
295 - return nil, err
296 - }
297 -
298 - pbd.Data = mdd
299 - return proto.Marshal(pbd)
300 -}
301 -
302 -// EmptyDirNode creates an empty folder Protonode.
303 -func EmptyDirNode() *dag.ProtoNode {
304 - return dag.NodeWithData(FolderPBData())
305 -}
unixfs/unixfs_test.go deleted
-160
@@ -1,160 +0,0 @@
1 -package unixfs
2 -
3 -import (
4 - "bytes"
5 - "testing"
6 -
7 - proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
8 -
9 - pb "gx/ipfs/QmSaz8Qg77gGqvDvLKeSAY7ivDEnramSWF6T7TcRwFpHtP/go-unixfs/pb"
10 -)
11 -
12 -func TestFSNode(t *testing.T) {
13 - fsn := NewFSNode(TFile)
14 - for i := 0; i < 16; i++ {
15 - fsn.AddBlockSize(100)
16 - }
17 - fsn.RemoveBlockSize(15)
18 -
19 - fsn.SetData(make([]byte, 128))
20 -
21 - b, err := fsn.GetBytes()
22 - if err != nil {
23 - t.Fatal(err)
24 - }
25 -
26 - pbn := new(pb.Data)
27 - err = proto.Unmarshal(b, pbn)
28 - if err != nil {
29 - t.Fatal(err)
30 - }
31 -
32 - ds, err := DataSize(b)
33 - if err != nil {
34 - t.Fatal(err)
35 - }
36 - nKids := fsn.NumChildren()
37 - if nKids != 15 {
38 - t.Fatal("Wrong number of child nodes")
39 - }
40 -
41 - if ds != (100*15)+128 {
42 - t.Fatal("Datasize calculations incorrect!")
43 - }
44 -
45 - nfsn, err := FSNodeFromBytes(b)
46 - if err != nil {
47 - t.Fatal(err)
48 - }
49 -
50 - if nfsn.FileSize() != (100*15)+128 {
51 - t.Fatal("fsNode FileSize calculations incorrect")
52 - }
53 -}
54 -
55 -func TestPBdataTools(t *testing.T) {
56 - raw := []byte{0x00, 0x01, 0x02, 0x17, 0xA1}
57 - rawPB := WrapData(raw)
58 -
59 - pbDataSize, err := DataSize(rawPB)
60 - if err != nil {
61 - t.Fatal(err)
62 - }
63 -
64 - same := len(raw) == int(pbDataSize)
65 - if !same {
66 - t.Fatal("WrapData changes the size of data.")
67 - }
68 -
69 - rawPBBytes, err := UnwrapData(rawPB)
70 - if err != nil {
71 - t.Fatal(err)
72 - }
73 -
74 - same = bytes.Equal(raw, rawPBBytes)
75 - if !same {
76 - t.Fatal("Unwrap failed to produce the correct wrapped data.")
77 - }
78 -
79 - rawPBdata, err := FromBytes(rawPB)
80 - if err != nil {
81 - t.Fatal(err)
82 - }
83 -
84 - isRaw := rawPBdata.GetType() == TRaw
85 - if !isRaw {
86 - t.Fatal("WrapData does not create pb.Data_Raw!")
87 - }
88 -
89 - catFile := []byte("Mr_Meowgie.gif")
90 - catPBfile := FilePBData(catFile, 17)
91 - catSize, err := DataSize(catPBfile)
92 - if catSize != 17 {
93 - t.Fatal("FilePBData is the wrong size.")
94 - }
95 - if err != nil {
96 - t.Fatal(err)
97 - }
98 -
99 - dirPB := FolderPBData()
100 - dir, err := FromBytes(dirPB)
101 - isDir := dir.GetType() == TDirectory
102 - if !isDir {
103 - t.Fatal("FolderPBData does not create a directory!")
104 - }
105 - if err != nil {
106 - t.Fatal(err)
107 - }
108 - _, dirErr := DataSize(dirPB)
109 - if dirErr == nil {
110 - t.Fatal("DataSize didn't throw an error when taking the size of a directory.")
111 - }
112 -
113 - catSym, err := SymlinkData("/ipfs/adad123123/meowgie.gif")
114 - if err != nil {
115 - t.Fatal(err)
116 - }
117 -
118 - catSymPB, err := FromBytes(catSym)
119 - isSym := catSymPB.GetType() == TSymlink
120 - if !isSym {
121 - t.Fatal("Failed to make a Symlink.")
122 - }
123 - if err != nil {
124 - t.Fatal(err)
125 - }
126 -
127 - _, sizeErr := DataSize(catSym)
128 - if sizeErr == nil {
129 - t.Fatal("DataSize didn't throw an error when taking the size of a Symlink.")
130 - }
131 -
132 -}
133 -
134 -func TestMetadata(t *testing.T) {
135 - meta := &Metadata{
136 - MimeType: "audio/aiff",
137 - Size: 12345,
138 - }
139 -
140 - _, err := meta.Bytes()
141 - if err != nil {
142 - t.Fatal(err)
143 - }
144 -
145 - metaPB, err := BytesForMetadata(meta)
146 - if err != nil {
147 - t.Fatal(err)
148 - }
149 -
150 - meta, err = MetadataFromBytes(metaPB)
151 - if err != nil {
152 - t.Fatal(err)
153 - }
154 -
155 - mimeAiff := meta.MimeType == "audio/aiff"
156 - if !mimeAiff {
157 - t.Fatal("Metadata does not Marshal and Unmarshal properly!")
158 - }
159 -
160 -}