Rework package structure for unixfs and subpackage
cc @jbenet
Jeromy committed
Oct 8, 2014 at 21:14 UTC
093c8fb04d0f272af661f4ba8c383a082306770f
19 files changed
+81
-66
core/commands/add.go
+1
-1
@@ -10,8 +10,8 @@ import (
10
11
"github.com/jbenet/go-ipfs/core"
12
"github.com/jbenet/go-ipfs/importer"
13
- ft "github.com/jbenet/go-ipfs/importer/format"
13
dag "github.com/jbenet/go-ipfs/merkledag"
14
+ ft "github.com/jbenet/go-ipfs/unixfs"
15
u "github.com/jbenet/go-ipfs/util"
16
)
17
core/commands/cat.go
+2
-2
@@ -5,7 +5,7 @@ import (
5
"io"
6
7
"github.com/jbenet/go-ipfs/core"
8
- mdag "github.com/jbenet/go-ipfs/merkledag"
8
+ uio "github.com/jbenet/go-ipfs/unixfs/io"
9
)
10
11
func Cat(n *core.IpfsNode, args []string, opts map[string]interface{}, out io.Writer) error {
@@ -15,7 +15,7 @@ func Cat(n *core.IpfsNode, args []string, opts map[string]interface{}, out io.Wr
15
return fmt.Errorf("catFile error: %v", err)
16
}
17
18
- read, err := mdag.NewDagReader(dagnode, n.DAG)
18
+ read, err := uio.NewDagReader(dagnode, n.DAG)
19
if err != nil {
20
return fmt.Errorf("cat error: %v", err)
21
}
fuse/ipns/ipns_unix.go
+8
-7
@@ -1,6 +1,7 @@
1
package ipns
2
3
import (
4
+ "errors"
5
"io/ioutil"
6
"os"
7
"path/filepath"
@@ -12,10 +13,10 @@ import (
13
14
"github.com/jbenet/go-ipfs/core"
15
ci "github.com/jbenet/go-ipfs/crypto"
15
- imp "github.com/jbenet/go-ipfs/importer"
16
- dt "github.com/jbenet/go-ipfs/importer/dagwriter"
17
- ft "github.com/jbenet/go-ipfs/importer/format"
16
+ "github.com/jbenet/go-ipfs/importer/chunk"
17
mdag "github.com/jbenet/go-ipfs/merkledag"
18
+ ft "github.com/jbenet/go-ipfs/unixfs"
19
+ uio "github.com/jbenet/go-ipfs/unixfs/io"
20
u "github.com/jbenet/go-ipfs/util"
21
)
22
@@ -204,7 +205,7 @@ type Node struct {
205
206
Ipfs *core.IpfsNode
207
Nd *mdag.Node
207
- dagMod *dt.DagModifier
208
+ dagMod *uio.DagModifier
209
cached *ft.PBData
210
}
211
@@ -293,7 +294,7 @@ func (s *Node) ReadDir(intr fs.Intr) ([]fuse.Dirent, fuse.Error) {
294
// ReadAll reads the object data as file data
295
func (s *Node) ReadAll(intr fs.Intr) ([]byte, fuse.Error) {
296
log.Debug("ipns: ReadAll [%s]", s.name)
296
- r, err := mdag.NewDagReader(s.Nd, s.Ipfs.DAG)
297
+ r, err := uio.NewDagReader(s.Nd, s.Ipfs.DAG)
298
if err != nil {
299
return nil, err
300
}
@@ -312,7 +313,7 @@ func (n *Node) Write(req *fuse.WriteRequest, resp *fuse.WriteResponse, intr fs.I
313
314
if n.dagMod == nil {
315
// Create a DagModifier to allow us to change the existing dag node
315
- dmod, err := dt.NewDagModifier(n.Nd, n.Ipfs.DAG, imp.DefaultSplitter)
316
+ dmod, err := uio.NewDagModifier(n.Nd, n.Ipfs.DAG, chunk.DefaultSplitter)
317
if err != nil {
318
log.Error("Error creating dag modifier: %s", err)
319
return err
@@ -541,7 +542,7 @@ func (n *Node) Rename(req *fuse.RenameRequest, newDir fs.Node, intr fs.Intr) fus
542
}
543
default:
544
log.Critical("Unknown node type for rename target dir!")
544
- return err
545
+ return errors.New("Unknown fs node type!")
546
}
547
return nil
548
}
fuse/readonly/readonly_unix.go
+4
-3
@@ -19,8 +19,9 @@ import (
19
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse"
20
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs"
21
core "github.com/jbenet/go-ipfs/core"
22
- ft "github.com/jbenet/go-ipfs/importer/format"
22
mdag "github.com/jbenet/go-ipfs/merkledag"
23
+ ft "github.com/jbenet/go-ipfs/unixfs"
24
+ uio "github.com/jbenet/go-ipfs/unixfs/io"
25
u "github.com/jbenet/go-ipfs/util"
26
)
27
@@ -79,7 +80,7 @@ func (*Root) ReadDir(intr fs.Intr) ([]fuse.Dirent, fuse.Error) {
80
type Node struct {
81
Ipfs *core.IpfsNode
82
Nd *mdag.Node
82
- fd *mdag.DagReader
83
+ fd *uio.DagReader
84
cached *ft.PBData
85
}
86
@@ -143,7 +144,7 @@ func (s *Node) ReadDir(intr fs.Intr) ([]fuse.Dirent, fuse.Error) {
144
// ReadAll reads the object data as file data
145
func (s *Node) ReadAll(intr fs.Intr) ([]byte, fuse.Error) {
146
u.DOut("Read node.\n")
146
- r, err := mdag.NewDagReader(s.Nd, s.Ipfs.DAG)
147
+ r, err := uio.NewDagReader(s.Nd, s.Ipfs.DAG)
148
if err != nil {
149
return nil, err
150
}
importer/chunk/rabin.go
renamed
+1
-1
@@ -1,4 +1,4 @@
1
-package importer
1
+package chunk
2
3
import (
4
"bufio"
importer/chunk/splitting.go
renamed
+10
-2
@@ -1,6 +1,14 @@
1
-package importer
1
+package chunk
2
3
-import "io"
3
+import (
4
+ "io"
5
+
6
+ "github.com/jbenet/go-ipfs/util"
7
+)
8
+
9
+var log = util.Logger("chunk")
10
+
11
+var DefaultSplitter = &SizeSplitter{1024 * 512}
12
13
type BlockSplitter interface {
14
Split(r io.Reader) chan []byte
importer/importer.go
+4
-5
@@ -5,8 +5,9 @@ import (
5
"io"
6
"os"
7
8
- ft "github.com/jbenet/go-ipfs/importer/format"
8
+ "github.com/jbenet/go-ipfs/importer/chunk"
9
dag "github.com/jbenet/go-ipfs/merkledag"
10
+ ft "github.com/jbenet/go-ipfs/unixfs"
11
"github.com/jbenet/go-ipfs/util"
12
)
13
@@ -18,18 +19,16 @@ var BlockSizeLimit = int64(1048576) // 1 MB
19
// ErrSizeLimitExceeded signals that a block is larger than BlockSizeLimit.
20
var ErrSizeLimitExceeded = fmt.Errorf("object size limit exceeded")
21
21
-var DefaultSplitter = &SizeSplitter{1024 * 512}
22
-
22
// todo: incremental construction with an ipfs node. dumping constructed
23
// objects into the datastore, to avoid buffering all in memory
24
25
// NewDagFromReader constructs a Merkle DAG from the given io.Reader.
26
// size required for block construction.
27
func NewDagFromReader(r io.Reader) (*dag.Node, error) {
29
- return NewDagFromReaderWithSplitter(r, DefaultSplitter)
28
+ return NewDagFromReaderWithSplitter(r, chunk.DefaultSplitter)
29
}
30
32
-func NewDagFromReaderWithSplitter(r io.Reader, spl BlockSplitter) (*dag.Node, error) {
31
+func NewDagFromReaderWithSplitter(r io.Reader, spl chunk.BlockSplitter) (*dag.Node, error) {
32
blkChan := spl.Split(r)
33
first := <-blkChan
34
root := &dag.Node{}
importer/importer_test.go
+11
-7
@@ -9,9 +9,13 @@ import (
9
"os"
10
"testing"
11
12
- dag "github.com/jbenet/go-ipfs/merkledag"
12
+ "github.com/jbenet/go-ipfs/importer/chunk"
13
+ uio "github.com/jbenet/go-ipfs/unixfs/io"
14
)
15
16
+// NOTE:
17
+// These tests tests a combination of unixfs/io/dagreader and importer/chunk.
18
+// Maybe split them up somehow?
19
func TestBuildDag(t *testing.T) {
20
td := os.TempDir()
21
fi, err := os.Create(td + "/tmpfi")
@@ -34,9 +38,9 @@ func TestBuildDag(t *testing.T) {
38
39
//Test where calls to read are smaller than the chunk size
40
func TestSizeBasedSplit(t *testing.T) {
37
- bs := &SizeSplitter{512}
41
+ bs := &chunk.SizeSplitter{512}
42
testFileConsistency(t, bs, 32*512)
39
- bs = &SizeSplitter{4096}
43
+ bs = &chunk.SizeSplitter{4096}
44
testFileConsistency(t, bs, 32*4096)
45
46
// Uneven offset
@@ -49,7 +53,7 @@ func dup(b []byte) []byte {
53
return o
54
}
55
52
-func testFileConsistency(t *testing.T, bs BlockSplitter, nbytes int) {
56
+func testFileConsistency(t *testing.T, bs chunk.BlockSplitter, nbytes int) {
57
buf := new(bytes.Buffer)
58
io.CopyN(buf, rand.Reader, int64(nbytes))
59
should := dup(buf.Bytes())
@@ -57,7 +61,7 @@ func testFileConsistency(t *testing.T, bs BlockSplitter, nbytes int) {
61
if err != nil {
62
t.Fatal(err)
63
}
60
- r, err := dag.NewDagReader(nd, nil)
64
+ r, err := uio.NewDagReader(nd, nil)
65
if err != nil {
66
t.Fatal(err)
67
}
@@ -86,14 +90,14 @@ func arrComp(a, b []byte) error {
90
}
91
92
func TestMaybeRabinConsistency(t *testing.T) {
89
- testFileConsistency(t, NewMaybeRabin(4096), 256*4096)
93
+ testFileConsistency(t, chunk.NewMaybeRabin(4096), 256*4096)
94
}
95
96
func TestRabinBlockSize(t *testing.T) {
97
buf := new(bytes.Buffer)
98
nbytes := 1024 * 1024
99
io.CopyN(buf, rand.Reader, int64(nbytes))
96
- rab := NewMaybeRabin(4096)
100
+ rab := chunk.NewMaybeRabin(4096)
101
blkch := rab.Split(buf)
102
103
var blocks [][]byte
server/http/ipfs.go
+2
-1
@@ -6,6 +6,7 @@ import (
6
core "github.com/jbenet/go-ipfs/core"
7
"github.com/jbenet/go-ipfs/importer"
8
dag "github.com/jbenet/go-ipfs/merkledag"
9
+ uio "github.com/jbenet/go-ipfs/unixfs/io"
10
u "github.com/jbenet/go-ipfs/util"
11
)
12
@@ -33,5 +34,5 @@ func (i *ipfsHandler) AddNodeToDAG(nd *dag.Node) (u.Key, error) {
34
}
35
36
func (i *ipfsHandler) NewDagReader(nd *dag.Node) (io.Reader, error) {
36
- return dag.NewDagReader(nd, i.node.DAG)
37
+ return uio.NewDagReader(nd, i.node.DAG)
38
}
unixfs/Makefile
renamed
unixfs/data.pb.go
renamed
+5
-5
@@ -3,7 +3,7 @@
3
// DO NOT EDIT!
4
5
/*
6
-Package format is a generated protocol buffer package.
6
+Package unixfs is a generated protocol buffer package.
7
8
It is generated from these files:
9
data.proto
@@ -11,9 +11,9 @@ It is generated from these files:
11
It has these top-level messages:
12
PBData
13
*/
14
-package format
14
+package unixfs
15
16
-import proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
16
+import proto "code.google.com/p/goprotobuf/proto"
17
import math "math"
18
19
// Reference imports to suppress errors if they are not otherwise used.
@@ -57,7 +57,7 @@ func (x *PBData_DataType) UnmarshalJSON(data []byte) error {
57
}
58
59
type PBData struct {
60
- Type *PBData_DataType `protobuf:"varint,1,req,enum=format.PBData_DataType" json:"Type,omitempty"`
60
+ Type *PBData_DataType `protobuf:"varint,1,req,enum=unixfs.PBData_DataType" json:"Type,omitempty"`
61
Data []byte `protobuf:"bytes,2,opt" json:"Data,omitempty"`
62
Filesize *uint64 `protobuf:"varint,3,opt,name=filesize" json:"filesize,omitempty"`
63
Blocksizes []uint64 `protobuf:"varint,4,rep,name=blocksizes" json:"blocksizes,omitempty"`
@@ -97,5 +97,5 @@ func (m *PBData) GetBlocksizes() []uint64 {
97
}
98
99
func init() {
100
- proto.RegisterEnum("format.PBData_DataType", PBData_DataType_name, PBData_DataType_value)
100
+ proto.RegisterEnum("unixfs.PBData_DataType", PBData_DataType_name, PBData_DataType_value)
101
}
unixfs/data.proto
renamed
+1
-1
@@ -1,4 +1,4 @@
1
-package format;
1
+package unixfs;
2
3
message PBData {
4
enum DataType {
unixfs/format.go
renamed
+1
-1
@@ -1,6 +1,6 @@
1
// Package format implements a data format for files in the ipfs filesystem
2
// It is not the only format in ipfs, but it is the one that the filesystem assumes
3
-package format
3
+package unixfs
4
5
import (
6
"errors"
unixfs/format_test.go
renamed
+1
-1
@@ -1,4 +1,4 @@
1
-package format
1
+package unixfs
2
3
import (
4
"testing"
unixfs/io/dagmodifier.go
renamed
+6
-6
@@ -1,4 +1,4 @@
1
-package dagwriter
1
+package io
2
3
import (
4
"bytes"
@@ -6,9 +6,9 @@ import (
6
7
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
8
9
- imp "github.com/jbenet/go-ipfs/importer"
10
- ft "github.com/jbenet/go-ipfs/importer/format"
9
+ "github.com/jbenet/go-ipfs/importer/chunk"
10
mdag "github.com/jbenet/go-ipfs/merkledag"
11
+ ft "github.com/jbenet/go-ipfs/unixfs"
12
u "github.com/jbenet/go-ipfs/util"
13
)
14
@@ -20,10 +20,10 @@ type DagModifier struct {
20
curNode *mdag.Node
21
22
pbdata *ft.PBData
23
- splitter imp.BlockSplitter
23
+ splitter chunk.BlockSplitter
24
}
25
26
-func NewDagModifier(from *mdag.Node, serv *mdag.DAGService, spl imp.BlockSplitter) (*DagModifier, error) {
26
+func NewDagModifier(from *mdag.Node, serv *mdag.DAGService, spl chunk.BlockSplitter) (*DagModifier, error) {
27
pbd, err := ft.FromBytes(from.Data)
28
if err != nil {
29
return nil, err
@@ -172,7 +172,7 @@ func (dm *DagModifier) WriteAt(b []byte, offset uint64) (int, error) {
172
173
// splitBytes uses a splitterFunc to turn a large array of bytes
174
// into many smaller arrays of bytes
175
-func splitBytes(b []byte, spl imp.BlockSplitter) [][]byte {
175
+func splitBytes(b []byte, spl chunk.BlockSplitter) [][]byte {
176
out := spl.Split(bytes.NewReader(b))
177
var arr [][]byte
178
for blk := range out {
unixfs/io/dagmodifier_test.go
renamed
+7
-7
@@ -1,4 +1,4 @@
1
-package dagwriter
1
+package io
2
3
import (
4
"fmt"
@@ -8,9 +8,9 @@ import (
8
9
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/op/go-logging"
10
bs "github.com/jbenet/go-ipfs/blockservice"
11
- imp "github.com/jbenet/go-ipfs/importer"
12
- ft "github.com/jbenet/go-ipfs/importer/format"
11
+ "github.com/jbenet/go-ipfs/importer/chunk"
12
mdag "github.com/jbenet/go-ipfs/merkledag"
13
+ ft "github.com/jbenet/go-ipfs/unixfs"
14
u "github.com/jbenet/go-ipfs/util"
15
16
ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
@@ -26,7 +26,7 @@ func getMockDagServ(t *testing.T) *mdag.DAGService {
26
}
27
28
func getNode(t *testing.T, dserv *mdag.DAGService, size int64) ([]byte, *mdag.Node) {
29
- dw := NewDagWriter(dserv, &imp.SizeSplitter{500})
29
+ dw := NewDagWriter(dserv, &chunk.SizeSplitter{500})
30
31
n, err := io.CopyN(dw, u.NewFastRand(), size)
32
if err != nil {
@@ -39,7 +39,7 @@ func getNode(t *testing.T, dserv *mdag.DAGService, size int64) ([]byte, *mdag.No
39
dw.Close()
40
node := dw.GetNode()
41
42
- dr, err := mdag.NewDagReader(node, dserv)
42
+ dr, err := NewDagReader(node, dserv)
43
if err != nil {
44
t.Fatal(err)
45
}
@@ -76,7 +76,7 @@ func testModWrite(t *testing.T, beg, size uint64, orig []byte, dm *DagModifier)
76
t.Fatal(err)
77
}
78
79
- rd, err := mdag.NewDagReader(nd, dm.dagserv)
79
+ rd, err := NewDagReader(nd, dm.dagserv)
80
if err != nil {
81
t.Fatal(err)
82
}
@@ -99,7 +99,7 @@ func TestDagModifierBasic(t *testing.T) {
99
dserv := getMockDagServ(t)
100
b, n := getNode(t, dserv, 50000)
101
102
- dagmod, err := NewDagModifier(n, dserv, &imp.SizeSplitter{512})
102
+ dagmod, err := NewDagModifier(n, dserv, &chunk.SizeSplitter{512})
103
if err != nil {
104
t.Fatal(err)
105
}
unixfs/io/dagreader.go
renamed
+6
-5
@@ -1,4 +1,4 @@
1
-package merkledag
1
+package io
2
3
import (
4
"bytes"
@@ -6,7 +6,8 @@ import (
6
"io"
7
8
proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
9
- ft "github.com/jbenet/go-ipfs/importer/format"
9
+ mdag "github.com/jbenet/go-ipfs/merkledag"
10
+ ft "github.com/jbenet/go-ipfs/unixfs"
11
u "github.com/jbenet/go-ipfs/util"
12
)
13
@@ -14,15 +15,15 @@ var ErrIsDir = errors.New("this dag node is a directory")
15
16
// DagReader provides a way to easily read the data contained in a dag.
17
type DagReader struct {
17
- serv *DAGService
18
- node *Node
18
+ serv *mdag.DAGService
19
+ node *mdag.Node
20
position int
21
buf *bytes.Buffer
22
}
23
24
// NewDagReader creates a new reader object that reads the data represented by the given
25
// node, using the passed in DAGService for data retreival
25
-func NewDagReader(n *Node, serv *DAGService) (io.Reader, error) {
26
+func NewDagReader(n *mdag.Node, serv *mdag.DAGService) (io.Reader, error) {
27
pb := new(ft.PBData)
28
err := proto.Unmarshal(n.Data, pb)
29
if err != nil {
unixfs/io/dagwriter.go
renamed
+5
-5
@@ -1,9 +1,9 @@
1
-package dagwriter
1
+package io
2
3
import (
4
- imp "github.com/jbenet/go-ipfs/importer"
5
- ft "github.com/jbenet/go-ipfs/importer/format"
4
+ "github.com/jbenet/go-ipfs/importer/chunk"
5
dag "github.com/jbenet/go-ipfs/merkledag"
6
+ ft "github.com/jbenet/go-ipfs/unixfs"
7
"github.com/jbenet/go-ipfs/util"
8
)
9
@@ -15,11 +15,11 @@ type DagWriter struct {
15
totalSize int64
16
splChan chan []byte
17
done chan struct{}
18
- splitter imp.BlockSplitter
18
+ splitter chunk.BlockSplitter
19
seterr error
20
}
21
22
-func NewDagWriter(ds *dag.DAGService, splitter imp.BlockSplitter) *DagWriter {
22
+func NewDagWriter(ds *dag.DAGService, splitter chunk.BlockSplitter) *DagWriter {
23
dw := new(DagWriter)
24
dw.dagserv = ds
25
dw.splChan = make(chan []byte, 8)
unixfs/io/dagwriter_test.go
renamed
+6
-6
@@ -1,4 +1,4 @@
1
-package dagwriter
1
+package io
2
3
import (
4
"testing"
@@ -7,7 +7,7 @@ import (
7
8
ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/datastore.go"
9
bs "github.com/jbenet/go-ipfs/blockservice"
10
- imp "github.com/jbenet/go-ipfs/importer"
10
+ chunk "github.com/jbenet/go-ipfs/importer/chunk"
11
mdag "github.com/jbenet/go-ipfs/merkledag"
12
)
13
@@ -54,7 +54,7 @@ func TestDagWriter(t *testing.T) {
54
t.Fatal(err)
55
}
56
dag := &mdag.DAGService{bserv}
57
- dw := NewDagWriter(dag, &imp.SizeSplitter{4096})
57
+ dw := NewDagWriter(dag, &chunk.SizeSplitter{4096})
58
59
nbytes := int64(1024 * 1024 * 2)
60
n, err := io.CopyN(dw, &datasource{}, nbytes)
@@ -69,7 +69,7 @@ func TestDagWriter(t *testing.T) {
69
dw.Close()
70
71
node := dw.GetNode()
72
- read, err := mdag.NewDagReader(node, dag)
72
+ read, err := NewDagReader(node, dag)
73
if err != nil {
74
t.Fatal(err)
75
}
@@ -88,7 +88,7 @@ func TestMassiveWrite(t *testing.T) {
88
t.Fatal(err)
89
}
90
dag := &mdag.DAGService{bserv}
91
- dw := NewDagWriter(dag, &imp.SizeSplitter{4096})
91
+ dw := NewDagWriter(dag, &chunk.SizeSplitter{4096})
92
93
nbytes := int64(1024 * 1024 * 1024 * 16)
94
n, err := io.CopyN(dw, &datasource{}, nbytes)
@@ -113,7 +113,7 @@ func BenchmarkDagWriter(b *testing.B) {
113
nbytes := int64(100000)
114
for i := 0; i < b.N; i++ {
115
b.SetBytes(nbytes)
116
- dw := NewDagWriter(dag, &imp.SizeSplitter{4096})
116
+ dw := NewDagWriter(dag, &chunk.SizeSplitter{4096})
117
n, err := io.CopyN(dw, &datasource{}, nbytes)
118
if err != nil {
119
b.Fatal(err)