files2.0: updates for file type split
License: MIT Signed-off-by: Łukasz Magiera <magik6k@gmail.com>
Łukasz Magiera committed
Nov 19, 2018 at 03:24 UTC
adc74907554e7018b5a173e0621ada07f73d04c9
21 files changed
+275
-210
cmd/ipfs/init.go
+7
-4
@@ -85,13 +85,16 @@ environment variable:
85
86
f := req.Files
87
if f != nil {
88
- _, confFile, err := f.NextFile()
89
- if err != nil {
90
- return err
88
+ it, _ := req.Files.Entries()
89
+ if !it.Next() && it.Err() != nil {
90
+ return it.Err()
91
+ }
92
+ if it.File() == nil {
93
+ return fmt.Errorf("expected a regular file")
94
}
95
96
conf = &config.Config{}
94
- if err := json.NewDecoder(confFile).Decode(conf); err != nil {
97
+ if err := json.NewDecoder(it.File()).Decode(conf); err != nil {
98
return err
99
}
100
}
core/commands/add.go
-1
@@ -11,7 +11,6 @@ import (
11
"github.com/ipfs/go-ipfs/core/coreapi/interface/options"
12
13
pb "gx/ipfs/QmPtj12fdwuAqj9sBSTNUxBNu8kCGNp8b3o8yUzMm5GHpq/pb"
14
- files "gx/ipfs/QmXWZCd8jfaHmt4UDSnjKmGcrQMw95bDGWqEeVLVJjoANX/go-ipfs-files"
14
cmds "gx/ipfs/QmaAP56JAwdjwisPTu4yx17whcjTr6y5JCSCF77Y1rahWV/go-ipfs-cmds"
15
cmdkit "gx/ipfs/Qmde5VP1qUkyQXKCfmEUA7bP64V2HAptbJ7phuPp7jXWwg/go-ipfs-cmdkit"
16
mh "gx/ipfs/QmerPMzPk1mJVowm8KgmoknWa4yCYvvugMPsgWmDNUvDLW/go-multihash"
core/commands/block.go
+7
-4
@@ -153,9 +153,12 @@ than 'sha2-256' or format to anything other than 'v0' will result in CIDv1.
153
return err
154
}
155
156
- _, file, err := req.Files.NextFile()
157
- if err != nil {
158
- return err
156
+ it, _ := req.Files.Entries()
157
+ if !it.Next() && it.Err() != nil {
158
+ return it.Err()
159
+ }
160
+ if it.File() == nil {
161
+ return fmt.Errorf("expected a regular file")
162
}
163
164
mhtype, _ := req.Options[mhtypeOptionName].(string)
@@ -178,7 +181,7 @@ than 'sha2-256' or format to anything other than 'v0' will result in CIDv1.
181
}
182
}
183
181
- p, err := api.Block().Put(req.Context, file, options.Block.Hash(mhtval, mhlen), options.Block.Format(format))
184
+ p, err := api.Block().Put(req.Context, it.File(), options.Block.Hash(mhtval, mhlen), options.Block.Format(format))
185
if err != nil {
186
return err
187
}
core/commands/cat.go
+4
-2
@@ -9,6 +9,7 @@ import (
9
"github.com/ipfs/go-ipfs/core/commands/cmdenv"
10
"github.com/ipfs/go-ipfs/core/coreapi/interface"
11
12
+ "gx/ipfs/QmXWZCd8jfaHmt4UDSnjKmGcrQMw95bDGWqEeVLVJjoANX/go-ipfs-files"
13
cmds "gx/ipfs/QmaAP56JAwdjwisPTu4yx17whcjTr6y5JCSCF77Y1rahWV/go-ipfs-cmds"
14
"gx/ipfs/Qmde5VP1qUkyQXKCfmEUA7bP64V2HAptbJ7phuPp7jXWwg/go-ipfs-cmdkit"
15
)
@@ -122,12 +123,13 @@ func cat(ctx context.Context, api iface.CoreAPI, paths []string, offset int64, m
123
return nil, 0, err
124
}
125
125
- file, err := api.Unixfs().Get(ctx, fpath)
126
+ f, err := api.Unixfs().Get(ctx, fpath)
127
if err != nil {
128
return nil, 0, err
129
}
130
130
- if file.IsDirectory() {
131
+ file, ok := f.(files.File)
132
+ if !ok {
133
return nil, 0, iface.ErrIsDir
134
}
135
core/commands/config.go
+7
-3
@@ -280,10 +280,14 @@ can't be undone.
280
}
281
defer r.Close()
282
283
- file, err := req.Files.NextFile()
284
- if err != nil {
285
- return err
283
+ it, _ := req.Files.Entries()
284
+ if !it.Next() && it.Err() != nil {
285
+ return it.Err()
286
+ }
287
+ if it.File() == nil {
288
+ return fmt.Errorf("expected a regular file")
289
}
290
+ file := it.File()
291
defer file.Close()
292
293
return replaceConfig(r, file)
core/commands/dag/dag.go
+8
-9
@@ -92,16 +92,12 @@ into an object of the specified format.
92
defer nd.Blockstore.PinLock().Unlock()
93
}
94
95
- for {
96
- _, file, err := req.Files.NextFile()
97
- if err == io.EOF {
98
- // Finished the list of files.
99
- break
100
- } else if err != nil {
101
- return err
95
+ it, _ := req.Files.Entries()
96
+ for it.Next() {
97
+ if it.File() == nil {
98
+ return fmt.Errorf("expected a regular file")
99
}
103
-
104
- nds, err := coredag.ParseInputs(ienc, format, file, mhType, -1)
100
+ nds, err := coredag.ParseInputs(ienc, format, it.File(), mhType, -1)
101
if err != nil {
102
return err
103
}
@@ -122,6 +118,9 @@ into an object of the specified format.
118
return err
119
}
120
}
121
+ if it.Err() != nil {
122
+ return err
123
+ }
124
125
if err := b.Commit(); err != nil {
126
return err
core/commands/files.go
+7
-4
@@ -769,12 +769,15 @@ stat' on the file or any of its ancestors.
769
return err
770
}
771
772
- _, input, err := req.Files.NextFile()
773
- if err != nil {
774
- return err
772
+ it, _ := req.Files.Entries()
773
+ if !it.Next() && it.Err() != nil {
774
+ return it.Err()
775
+ }
776
+ if it.File() == nil {
777
+ return fmt.Errorf("expected a regular file")
778
}
779
777
- var r io.Reader = input
780
+ var r io.Reader = it.File()
781
if countfound {
782
r = io.LimitReader(r, int64(count))
783
}
core/commands/object/object.go
+7
-4
@@ -391,9 +391,12 @@ And then run:
391
return err
392
}
393
394
- _, input, err := req.Files.NextFile()
395
- if err != nil && err != io.EOF {
396
- return err
394
+ it, _ := req.Files.Entries()
395
+ if !it.Next() && it.Err() != nil {
396
+ return it.Err()
397
+ }
398
+ if it.File() == nil {
399
+ return fmt.Errorf("expected a regular file")
400
}
401
402
inputenc, _ := req.Options["inputenc"].(string)
@@ -411,7 +414,7 @@ And then run:
414
return err
415
}
416
414
- p, err := api.Object().Put(req.Context, input,
417
+ p, err := api.Object().Put(req.Context, it.File(),
418
options.Object.DataType(datafieldenc),
419
options.Object.InputEnc(inputenc),
420
options.Object.Pin(dopin))
core/commands/object/patch.go
+17
-11
@@ -4,12 +4,12 @@ import (
4
"fmt"
5
"io"
6
7
- cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
7
+ "github.com/ipfs/go-ipfs/core/commands/cmdenv"
8
coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
9
"github.com/ipfs/go-ipfs/core/coreapi/interface/options"
10
11
- cmds "gx/ipfs/QmaAP56JAwdjwisPTu4yx17whcjTr6y5JCSCF77Y1rahWV/go-ipfs-cmds"
12
- cmdkit "gx/ipfs/Qmde5VP1qUkyQXKCfmEUA7bP64V2HAptbJ7phuPp7jXWwg/go-ipfs-cmdkit"
11
+ "gx/ipfs/QmaAP56JAwdjwisPTu4yx17whcjTr6y5JCSCF77Y1rahWV/go-ipfs-cmds"
12
+ "gx/ipfs/Qmde5VP1qUkyQXKCfmEUA7bP64V2HAptbJ7phuPp7jXWwg/go-ipfs-cmdkit"
13
)
14
15
var ObjectPatchCmd = &cmds.Command{
@@ -60,12 +60,15 @@ the limit will not be respected by the network.
60
return err
61
}
62
63
- _, data, err := req.Files.NextFile()
64
- if err != nil {
65
- return err
63
+ it, _ := req.Files.Entries()
64
+ if !it.Next() && it.Err() != nil {
65
+ return it.Err()
66
+ }
67
+ if it.File() == nil {
68
+ return fmt.Errorf("expected a regular file")
69
}
70
68
- p, err := api.Object().AppendData(req.Context, root, data)
71
+ p, err := api.Object().AppendData(req.Context, root, it.File())
72
if err != nil {
73
return err
74
}
@@ -107,12 +110,15 @@ Example:
110
return err
111
}
112
110
- _, data, err := req.Files.NextFile()
111
- if err != nil {
112
- return err
113
+ it, _ := req.Files.Entries()
114
+ if !it.Next() && it.Err() != nil {
115
+ return it.Err()
116
+ }
117
+ if it.File() == nil {
118
+ return fmt.Errorf("expected a regular file")
119
}
120
115
- p, err := api.Object().SetData(req.Context, root, data)
121
+ p, err := api.Object().SetData(req.Context, root, it.File())
122
if err != nil {
123
return err
124
}
core/commands/tar.go
+8
-5
@@ -44,12 +44,15 @@ represent it.
44
return err
45
}
46
47
- name, fi, err := req.Files.NextFile()
48
- if err != nil {
49
- return err
47
+ it, _ := req.Files.Entries()
48
+ if !it.Next() && it.Err() != nil {
49
+ return it.Err()
50
+ }
51
+ if it.File() == nil {
52
+ return fmt.Errorf("expected a regular file")
53
}
54
52
- node, err := tar.ImportTar(req.Context, fi, nd.DAG)
55
+ node, err := tar.ImportTar(req.Context, it.File(), nd.DAG)
56
if err != nil {
57
return err
58
}
@@ -57,7 +60,7 @@ represent it.
60
c := node.Cid()
61
62
return cmds.EmitOnce(res, &coreiface.AddEvent{
60
- Name: name,
63
+ Name: it.Name(),
64
Hash: c.String(),
65
})
66
},
core/coreapi/interface/unixfs.go
+2
-2
@@ -23,13 +23,13 @@ type UnixfsAPI interface {
23
// Add imports the data from the reader into merkledag file
24
//
25
// TODO: a long useful comment on how to use this for many different scenarios
26
- Add(context.Context, files.File, ...options.UnixfsAddOption) (ResolvedPath, error)
26
+ Add(context.Context, files.Node, ...options.UnixfsAddOption) (ResolvedPath, error)
27
28
// Get returns a read-only handle to a file tree referenced by a path
29
//
30
// Note that some implementations of this API may apply the specified context
31
// to operations performed on the returned file
32
- Get(context.Context, Path) (files.File, error)
32
+ Get(context.Context, Path) (files.Node, error)
33
34
// Ls returns the list of links in a directory
35
Ls(context.Context, Path) ([]*ipld.Link, error)
core/coreapi/unixfile.go
+67
-41
@@ -3,7 +3,6 @@ package coreapi
3
import (
4
"context"
5
"errors"
6
- "io"
6
7
files "gx/ipfs/QmXWZCd8jfaHmt4UDSnjKmGcrQMw95bDGWqEeVLVJjoANX/go-ipfs-files"
8
ft "gx/ipfs/Qmbvw7kpSM2p6rbQ57WGRhhqNfCiNGW6EKH4xgHLw4bsnB/go-unixfs"
@@ -21,90 +20,114 @@ const prefetchFiles = 4
20
type ufsDirectory struct {
21
ctx context.Context
22
dserv ipld.DAGService
23
+ dir uio.Directory
24
+}
25
26
+type ufsIterator struct {
27
+ ctx context.Context
28
files chan *ipld.Link
29
+ dserv ipld.DAGService
30
+
31
+ curName string
32
+ curFile files.Node
33
+
34
+ err error
35
}
36
28
-func (d *ufsDirectory) Close() error {
29
- return files.ErrNotReader
37
+func (it *ufsIterator) Name() string {
38
+ return it.curName
39
}
40
32
-func (d *ufsDirectory) Read(_ []byte) (int, error) {
33
- return 0, files.ErrNotReader
41
+func (it *ufsIterator) Node() files.Node {
42
+ return it.curFile
43
}
44
36
-func (d *ufsDirectory) IsDirectory() bool {
37
- return true
45
+func (it *ufsIterator) File() files.File {
46
+ f, _ := it.curFile.(files.File)
47
+ return f
48
}
49
40
-func (d *ufsDirectory) NextFile() (string, files.File, error) {
41
- l, ok := <-d.files
50
+func (it *ufsIterator) Dir() files.Directory {
51
+ d, _ := it.curFile.(files.Directory)
52
+ return d
53
+}
54
+
55
+func (it *ufsIterator) Next() bool {
56
+ l, ok := <-it.files
57
if !ok {
43
- return "", nil, io.EOF
58
+ return false
59
}
60
46
- nd, err := l.GetNode(d.ctx, d.dserv)
61
+ it.curFile = nil
62
+
63
+ nd, err := l.GetNode(it.ctx, it.dserv)
64
if err != nil {
48
- return "", nil, err
65
+ it.err = err
66
+ return false
67
}
68
51
- f, err := newUnixfsFile(d.ctx, d.dserv, nd, d)
52
- return l.Name, f, err
69
+ it.curName = l.Name
70
+ it.curFile, it.err = newUnixfsFile(it.ctx, it.dserv, nd)
71
+ return it.err == nil
72
}
73
55
-func (d *ufsDirectory) Size() (int64, error) {
56
- return 0, files.ErrNotReader
74
+func (it *ufsIterator) Err() error {
75
+ return it.err
76
}
77
59
-func (d *ufsDirectory) Seek(offset int64, whence int) (int64, error) {
60
- return 0, files.ErrNotReader
78
+func (d *ufsDirectory) Close() error {
79
+ return nil
80
}
81
63
-type ufsFile struct {
64
- uio.DagReader
82
+func (d *ufsDirectory) Entries() (files.DirIterator, error) {
83
+ fileCh := make(chan *ipld.Link, prefetchFiles)
84
+ go func() {
85
+ d.dir.ForEachLink(d.ctx, func(link *ipld.Link) error {
86
+ select {
87
+ case fileCh <- link:
88
+ case <-d.ctx.Done():
89
+ return d.ctx.Err()
90
+ }
91
+ return nil
92
+ })
93
+
94
+ close(fileCh)
95
+ }()
96
+
97
+ return &ufsIterator{
98
+ ctx: d.ctx,
99
+ files: fileCh,
100
+ dserv: d.dserv,
101
+ }, nil
102
}
103
67
-func (f *ufsFile) IsDirectory() bool {
68
- return false
104
+func (d *ufsDirectory) Size() (int64, error) {
105
+ return 0, files.ErrNotSupported
106
}
107
71
-func (f *ufsFile) NextFile() (string, files.File, error) {
72
- return "", nil, files.ErrNotDirectory
108
+type ufsFile struct {
109
+ uio.DagReader
110
}
111
112
func (f *ufsFile) Size() (int64, error) {
113
return int64(f.DagReader.Size()), nil
114
}
115
79
-func newUnixfsDir(ctx context.Context, dserv ipld.DAGService, nd ipld.Node) (files.File, error) {
116
+func newUnixfsDir(ctx context.Context, dserv ipld.DAGService, nd ipld.Node) (files.Directory, error) {
117
dir, err := uio.NewDirectoryFromNode(dserv, nd)
118
if err != nil {
119
return nil, err
120
}
121
85
- fileCh := make(chan *ipld.Link, prefetchFiles)
86
- go func() {
87
- dir.ForEachLink(ctx, func(link *ipld.Link) error {
88
- select {
89
- case fileCh <- link:
90
- case <-ctx.Done():
91
- return ctx.Err()
92
- }
93
- return nil
94
- })
95
-
96
- close(fileCh)
97
- }()
98
-
122
return &ufsDirectory{
123
ctx: ctx,
124
dserv: dserv,
125
103
- files: fileCh,
126
+ dir: dir,
127
}, nil
128
}
129
107
-func newUnixfsFile(ctx context.Context, dserv ipld.DAGService, nd ipld.Node, parent files.File) (files.File, error) {
130
+func newUnixfsFile(ctx context.Context, dserv ipld.DAGService, nd ipld.Node) (files.Node, error) {
131
switch dn := nd.(type) {
132
case *dag.ProtoNode:
133
fsn, err := ft.FSNodeFromBytes(dn.Data())
@@ -129,3 +152,6 @@ func newUnixfsFile(ctx context.Context, dserv ipld.DAGService, nd ipld.Node, par
152
DagReader: dr,
153
}, nil
154
}
155
+
156
+var _ files.Directory = &ufsDirectory{}
157
+var _ files.File = &ufsFile{}
core/coreapi/unixfs.go
+3
-3
@@ -28,7 +28,7 @@ type UnixfsAPI CoreAPI
28
29
// Add builds a merkledag node from a reader, adds it to the blockstore,
30
// and returns the key representing that node.
31
-func (api *UnixfsAPI) Add(ctx context.Context, files files.File, opts ...options.UnixfsAddOption) (coreiface.ResolvedPath, error) {
31
+func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options.UnixfsAddOption) (coreiface.ResolvedPath, error) {
32
settings, prefix, err := options.UnixfsAddOptions(opts...)
33
if err != nil {
34
return nil, err
@@ -133,7 +133,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.File, opts ...options
133
return coreiface.IpfsPath(nd.Cid()), nil
134
}
135
136
-func (api *UnixfsAPI) Get(ctx context.Context, p coreiface.Path) (files.File, error) {
136
+func (api *UnixfsAPI) Get(ctx context.Context, p coreiface.Path) (files.Node, error) {
137
ses := api.core().getSession(ctx)
138
139
nd, err := ses.ResolveNode(ctx, p)
@@ -141,7 +141,7 @@ func (api *UnixfsAPI) Get(ctx context.Context, p coreiface.Path) (files.File, er
141
return nil, err
142
}
143
144
- return newUnixfsFile(ctx, ses.dag, nd, nil)
144
+ return newUnixfsFile(ctx, ses.dag, nd)
145
}
146
147
// Ls returns the contents of an IPFS or IPNS object(s) at path p, with the format:
core/coreapi/unixfs_test.go
+75
-66
@@ -134,36 +134,36 @@ func makeAPI(ctx context.Context) (*core.IpfsNode, coreiface.CoreAPI, error) {
134
return nd[0], api[0], nil
135
}
136
137
-func strFile(data string) func() files.File {
138
- return func() files.File {
137
+func strFile(data string) func() files.Node {
138
+ return func() files.Node {
139
return files.NewReaderFile(ioutil.NopCloser(strings.NewReader(data)), nil)
140
}
141
}
142
143
-func twoLevelDir() func() files.File {
144
- return func() files.File {
145
- return files.NewSliceFile([]files.FileEntry{{
146
- Name: "abc", File: files.NewSliceFile([]files.FileEntry{
147
- {Name: "def", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader("world")), nil)},
148
- })},
143
+func twoLevelDir() func() files.Node {
144
+ return func() files.Node {
145
+ return files.NewSliceFile([]files.DirEntry{
146
+ files.FileEntry("abc", files.NewSliceFile([]files.DirEntry{
147
+ files.FileEntry("def", files.NewReaderFile(ioutil.NopCloser(strings.NewReader("world")), nil)),
148
+ })),
149
150
- {Name: "bar", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello2")), nil)},
151
- {Name: "foo", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello1")), nil)},
150
+ files.FileEntry("bar", files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello2")), nil)),
151
+ files.FileEntry("foo", files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello1")), nil)),
152
})
153
}
154
}
155
156
-func flatDir() files.File {
157
- return files.NewSliceFile([]files.FileEntry{
158
- {Name: "bar", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello2")), nil)},
159
- {Name: "foo", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello1")), nil)},
156
+func flatDir() files.Node {
157
+ return files.NewSliceFile([]files.DirEntry{
158
+ files.FileEntry("bar", files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello2")), nil)),
159
+ files.FileEntry("foo", files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello1")), nil)),
160
})
161
}
162
163
-func wrapped(name string) func(f files.File) files.File {
164
- return func(f files.File) files.File {
165
- return files.NewSliceFile([]files.FileEntry{
166
- {Name: name, File: f},
163
+func wrapped(name string) func(f files.Node) files.Node {
164
+ return func(f files.Node) files.Node {
165
+ return files.NewSliceFile([]files.DirEntry{
166
+ files.FileEntry(name, f),
167
})
168
}
169
}
@@ -177,8 +177,8 @@ func TestAdd(t *testing.T) {
177
178
cases := []struct {
179
name string
180
- data func() files.File
181
- expect func(files.File) files.File
180
+ data func() files.Node
181
+ expect func(files.Node) files.Node
182
183
path string
184
err string
@@ -295,7 +295,7 @@ func TestAdd(t *testing.T) {
295
{
296
name: "addWrapped",
297
path: "/ipfs/QmVE9rNpj5doj7XHzp5zMUxD7BJgXEqx4pe3xZ3JBReWHE",
298
- data: func() files.File {
298
+ data: func() files.Node {
299
return files.NewReaderFile(ioutil.NopCloser(strings.NewReader(helloStr)), nil)
300
},
301
wrap: "foo",
@@ -305,7 +305,7 @@ func TestAdd(t *testing.T) {
305
{
306
name: "addNotWrappedDirFile",
307
path: hello,
308
- data: func() files.File {
308
+ data: func() files.Node {
309
return files.NewReaderFile(ioutil.NopCloser(strings.NewReader(helloStr)), nil)
310
},
311
wrap: "foo",
@@ -313,12 +313,12 @@ func TestAdd(t *testing.T) {
313
{
314
name: "stdinWrapped",
315
path: "/ipfs/QmU3r81oZycjHS9oaSHw37ootMFuFUw1DvMLKXPsezdtqU",
316
- data: func() files.File {
316
+ data: func() files.Node {
317
return files.NewReaderFile(ioutil.NopCloser(strings.NewReader(helloStr)), nil)
318
},
319
- expect: func(files.File) files.File {
320
- return files.NewSliceFile([]files.FileEntry{
321
- {Name: "QmQy2Dw4Wk7rdJKjThjYXzfFJNaRKRHhHP5gHHXroJMYxk", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader(helloStr)), nil)},
319
+ expect: func(files.Node) files.Node {
320
+ return files.NewSliceFile([]files.DirEntry{
321
+ files.FileEntry("QmQy2Dw4Wk7rdJKjThjYXzfFJNaRKRHhHP5gHHXroJMYxk", files.NewReaderFile(ioutil.NopCloser(strings.NewReader(helloStr)), nil)),
322
})
323
},
324
opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true)},
@@ -326,7 +326,7 @@ func TestAdd(t *testing.T) {
326
{
327
name: "stdinNamed",
328
path: "/ipfs/QmQ6cGBmb3ZbdrQW1MRm1RJnYnaxCqfssz7CrTa9NEhQyS",
329
- data: func() files.File {
329
+ data: func() files.Node {
330
rf, err := files.NewReaderPathFile(os.Stdin.Name(), ioutil.NopCloser(strings.NewReader(helloStr)), nil)
331
if err != nil {
332
panic(err)
@@ -334,9 +334,9 @@ func TestAdd(t *testing.T) {
334
335
return rf
336
},
337
- expect: func(files.File) files.File {
338
- return files.NewSliceFile([]files.FileEntry{
339
- {Name: "test", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader(helloStr)), nil)},
337
+ expect: func(files.Node) files.Node {
338
+ return files.NewSliceFile([]files.DirEntry{
339
+ files.FileEntry("test", files.NewReaderFile(ioutil.NopCloser(strings.NewReader(helloStr)), nil)),
340
})
341
},
342
opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true), options.Unixfs.StdinName("test")},
@@ -360,11 +360,11 @@ func TestAdd(t *testing.T) {
360
// hidden
361
{
362
name: "hiddenFiles",
363
- data: func() files.File {
364
- return files.NewSliceFile([]files.FileEntry{
365
- {Name: ".bar", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello2")), nil)},
366
- {Name: "bar", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello2")), nil)},
367
- {Name: "foo", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello1")), nil)},
363
+ data: func() files.Node {
364
+ return files.NewSliceFile([]files.DirEntry{
365
+ files.FileEntry(".bar", files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello2")), nil)),
366
+ files.FileEntry("bar", files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello2")), nil)),
367
+ files.FileEntry("foo", files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello1")), nil)),
368
})
369
},
370
wrap: "t",
@@ -373,7 +373,7 @@ func TestAdd(t *testing.T) {
373
},
374
{
375
name: "hiddenFileAlwaysAdded",
376
- data: func() files.File {
376
+ data: func() files.Node {
377
return files.NewReaderFile(ioutil.NopCloser(strings.NewReader(helloStr)), nil)
378
},
379
wrap: ".foo",
@@ -381,14 +381,14 @@ func TestAdd(t *testing.T) {
381
},
382
{
383
name: "hiddenFilesNotAdded",
384
- data: func() files.File {
385
- return files.NewSliceFile([]files.FileEntry{
386
- {Name: ".bar", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello2")), nil)},
387
- {Name: "bar", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello2")), nil)},
388
- {Name: "foo", File: files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello1")), nil)},
384
+ data: func() files.Node {
385
+ return files.NewSliceFile([]files.DirEntry{
386
+ files.FileEntry(".bar", files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello2")), nil)),
387
+ files.FileEntry("bar", files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello2")), nil)),
388
+ files.FileEntry("foo", files.NewReaderFile(ioutil.NopCloser(strings.NewReader("hello1")), nil)),
389
})
390
},
391
- expect: func(files.File) files.File {
391
+ expect: func(files.Node) files.Node {
392
return flatDir()
393
},
394
wrap: "t",
@@ -431,7 +431,7 @@ func TestAdd(t *testing.T) {
431
},
432
{
433
name: "progress1M",
434
- data: func() files.File {
434
+ data: func() files.Node {
435
r := bytes.NewReader(bytes.Repeat([]byte{0}, 1000000))
436
return files.NewReaderFile(ioutil.NopCloser(r), nil)
437
},
@@ -457,8 +457,8 @@ func TestAdd(t *testing.T) {
457
458
data := testCase.data()
459
if testCase.wrap != "" {
460
- data = files.NewSliceFile([]files.FileEntry{
461
- {Name: testCase.wrap, File: data},
460
+ data = files.NewSliceFile([]files.DirEntry{
461
+ files.FileEntry(testCase.wrap, data),
462
})
463
}
464
@@ -533,9 +533,12 @@ func TestAdd(t *testing.T) {
533
534
// compare file structure with Unixfs().Get
535
536
- var cmpFile func(origName string, orig files.File, gotName string, got files.File)
537
- cmpFile = func(origName string, orig files.File, gotName string, got files.File) {
538
- if orig.IsDirectory() != got.IsDirectory() {
536
+ var cmpFile func(origName string, orig files.Node, gotName string, got files.Node)
537
+ cmpFile = func(origName string, orig files.Node, gotName string, got files.Node) {
538
+ _, origDir := orig.(files.Directory)
539
+ _, gotDir := got.(files.Directory)
540
+
541
+ if origDir != gotDir {
542
t.Fatal("file type mismatch")
543
}
544
@@ -543,16 +546,16 @@ func TestAdd(t *testing.T) {
546
t.Errorf("file name mismatch, orig='%s', got='%s'", origName, gotName)
547
}
548
546
- if !orig.IsDirectory() {
549
+ if !gotDir {
550
defer orig.Close()
551
defer got.Close()
552
550
- do, err := ioutil.ReadAll(orig)
553
+ do, err := ioutil.ReadAll(orig.(files.File))
554
if err != nil {
555
t.Fatal(err)
556
}
557
555
- dg, err := ioutil.ReadAll(got)
558
+ dg, err := ioutil.ReadAll(got.(files.File))
559
if err != nil {
560
t.Fatal(err)
561
}
@@ -564,21 +567,28 @@ func TestAdd(t *testing.T) {
567
return
568
}
569
567
- for {
568
- origName, origFile, err := orig.NextFile()
569
- gotName, gotFile, err2 := got.NextFile()
570
+ origIt, _ := orig.(files.Directory).Entries()
571
+ gotIt, _ := got.(files.Directory).Entries()
572
571
- if err != nil {
572
- if err == io.EOF && err2 == io.EOF {
573
- break
573
+ for {
574
+ if origIt.Next() {
575
+ if !gotIt.Next() {
576
+ t.Fatal("gotIt out of entries before origIt")
577
}
575
- t.Fatal(err)
576
- }
577
- if err2 != nil {
578
- t.Fatal(err)
578
+ } else {
579
+ if gotIt.Next() {
580
+ t.Fatal("origIt out of entries before gotIt")
581
+ }
582
+ break
583
}
584
581
- cmpFile(origName, origFile, gotName, gotFile)
585
+ cmpFile(origIt.Name(), origIt.Node(), gotIt.Name(), gotIt.Node())
586
+ }
587
+ if origIt.Err() != nil {
588
+ t.Fatal(origIt.Err())
589
+ }
590
+ if gotIt.Err() != nil {
591
+ t.Fatal(gotIt.Err())
592
}
593
}
594
@@ -667,7 +677,7 @@ func TestGetEmptyFile(t *testing.T) {
677
}
678
679
buf := make([]byte, 1) // non-zero so that Read() actually tries to read
670
- n, err := io.ReadFull(r, buf)
680
+ n, err := io.ReadFull(r.(files.File), buf)
681
if err != nil && err != io.EOF {
682
t.Error(err)
683
}
@@ -703,9 +713,8 @@ func TestGetDir(t *testing.T) {
713
t.Error(err)
714
}
715
706
- _, err = r.Read(make([]byte, 2))
707
- if err != files.ErrNotReader {
708
- t.Fatalf("expected ErrIsDir, got: %s", err)
716
+ if _, ok := r.(files.Directory); !ok {
717
+ t.Fatalf("expected a directory")
718
}
719
}
720
core/corehttp/gateway_handler.go
+16
-12
@@ -172,10 +172,7 @@ func (i *gatewayHandler) getOrHeadHandler(ctx context.Context, w http.ResponseWr
172
return
173
}
174
175
- dir := dr.IsDirectory()
176
- if !dir {
177
- defer dr.Close()
178
- }
175
+ defer dr.Close()
176
177
// Check etag send back to us
178
etag := "\"" + resolvedPath.Cid().String() + "\""
@@ -240,14 +237,14 @@ func (i *gatewayHandler) getOrHeadHandler(ctx context.Context, w http.ResponseWr
237
// TODO: break this out when we split /ipfs /ipns routes.
238
modtime := time.Now()
239
243
- if strings.HasPrefix(urlPath, ipfsPathPrefix) && !dir {
244
- w.Header().Set("Cache-Control", "public, max-age=29030400, immutable")
240
+ if f, ok := dr.(files.File); ok {
241
+ if strings.HasPrefix(urlPath, ipfsPathPrefix) {
242
+ w.Header().Set("Cache-Control", "public, max-age=29030400, immutable")
243
246
- // set modtime to a really long time ago, since files are immutable and should stay cached
247
- modtime = time.Unix(1, 0)
248
- }
244
+ // set modtime to a really long time ago, since files are immutable and should stay cached
245
+ modtime = time.Unix(1, 0)
246
+ }
247
250
- if !dir {
248
urlFilename := r.URL.Query().Get("filename")
249
var name string
250
if urlFilename != "" {
@@ -256,8 +253,9 @@ func (i *gatewayHandler) getOrHeadHandler(ctx context.Context, w http.ResponseWr
253
} else {
254
name = getFilename(urlPath)
255
}
259
- i.serveFile(w, r, name, modtime, dr)
256
+ i.serveFile(w, r, name, modtime, f)
257
return
258
+
259
}
260
261
nd, err := i.api.ResolveNode(ctx, resolvedPath)
@@ -290,8 +288,14 @@ func (i *gatewayHandler) getOrHeadHandler(ctx context.Context, w http.ResponseWr
288
}
289
defer dr.Close()
290
291
+ f, ok := dr.(files.File)
292
+ if !ok {
293
+ internalWebError(w, files.ErrNotReader)
294
+ return
295
+ }
296
+
297
// write to request
294
- http.ServeContent(w, r, "index.html", modtime, dr)
298
+ http.ServeContent(w, r, "index.html", modtime, f)
299
return
300
default:
301
internalWebError(w, err)
core/coreunix/add.go
+31
-31
@@ -2,6 +2,7 @@ package coreunix
2
3
import (
4
"context"
5
+ "errors"
6
"fmt"
7
"io"
8
"io/ioutil"
@@ -400,7 +401,7 @@ func (adder *Adder) addNode(node ipld.Node, path string) error {
401
}
402
403
// AddAllAndPin adds the given request's files and pin them.
403
-func (adder *Adder) AddAllAndPin(file files.File) (ipld.Node, error) {
404
+func (adder *Adder) AddAllAndPin(file files.Node) (ipld.Node, error) {
405
if adder.Pin {
406
adder.unlocker = adder.blockstore.PinLock()
407
}
@@ -410,23 +411,23 @@ func (adder *Adder) AddAllAndPin(file files.File) (ipld.Node, error) {
411
}
412
}()
413
413
- switch {
414
- case file.IsDirectory():
414
+ switch tf := file.(type) {
415
+ case files.Directory:
416
// Iterate over each top-level file and add individually. Otherwise the
417
// single files.File f is treated as a directory, affecting hidden file
418
// semantics.
418
- for {
419
- name, f, err := file.NextFile()
420
- if err == io.EOF {
421
- // Finished the list of files.
422
- break
423
- } else if err != nil {
424
- return nil, err
425
- }
426
- if err := adder.addFile(name, f); err != nil {
419
+ it, err := tf.Entries()
420
+ if err != nil {
421
+ return nil, err
422
+ }
423
+ for it.Next() {
424
+ if err := adder.addFile(it.Name(), it.Node()); err != nil {
425
return nil, err
426
}
427
}
428
+ if it.Err() != nil {
429
+ return nil, it.Err()
430
+ }
431
break
432
default:
433
if err := adder.addFile("", file); err != nil {
@@ -447,7 +448,7 @@ func (adder *Adder) AddAllAndPin(file files.File) (ipld.Node, error) {
448
return nd, adder.PinRoot()
449
}
450
450
-func (adder *Adder) addFile(path string, file files.File) error {
451
+func (adder *Adder) addFile(path string, file files.Node) error {
452
err := adder.maybePauseForGC()
453
if err != nil {
454
return err
@@ -467,8 +468,8 @@ func (adder *Adder) addFile(path string, file files.File) error {
468
}
469
adder.liveNodes++
470
470
- if file.IsDirectory() {
471
- return adder.addDir(path, file)
471
+ if dir, ok := file.(files.Directory); ok {
472
+ return adder.addDir(path, dir)
473
}
474
475
// case for symlink
@@ -491,9 +492,12 @@ func (adder *Adder) addFile(path string, file files.File) error {
492
// case for regular file
493
// if the progress flag was specified, wrap the file so that we can send
494
// progress updates to the client (over the output channel)
494
- var reader io.Reader = file
495
+ reader, ok := file.(io.Reader)
496
+ if !ok {
497
+ return errors.New("file doesn't support reading")
498
+ }
499
if adder.Progress {
496
- rdr := &progressReader{file: file, path: path, out: adder.Out}
500
+ rdr := &progressReader{file: reader, path: path, out: adder.Out}
501
if fi, ok := file.(files.FileInfo); ok {
502
reader = &progressReader2{rdr, fi}
503
} else {
@@ -517,7 +521,7 @@ func (adder *Adder) addFile(path string, file files.File) error {
521
return adder.addNode(dagnode, path)
522
}
523
520
-func (adder *Adder) addDir(path string, dir files.File) error {
524
+func (adder *Adder) addDir(path string, dir files.Directory) error {
525
log.Infof("adding directory: %s", path)
526
527
mr, err := adder.mfsRoot()
@@ -533,27 +537,23 @@ func (adder *Adder) addDir(path string, dir files.File) error {
537
return err
538
}
539
536
- for {
537
- name, file, err := dir.NextFile()
538
- if err != nil && err != io.EOF {
539
- return err
540
- }
541
- if file == nil {
542
- break
543
- }
544
-
545
- fpath := gopath.Join(path, name)
540
+ it, _ := dir.Entries()
541
+ for it.Next() {
542
+ fpath := gopath.Join(path, it.Name())
543
544
// Skip hidden files when adding recursively, unless Hidden is enabled.
548
- if files.IsHidden(fpath, file) && !adder.Hidden {
545
+ if files.IsHidden(fpath, it.Node()) && !adder.Hidden {
546
log.Infof("%s is hidden, skipping", fpath)
547
continue
548
}
552
- err = adder.addFile(fpath, file)
549
+ err = adder.addFile(fpath, it.Node())
550
if err != nil {
551
return err
552
}
553
}
554
+ if it.Err() != nil {
555
+ return it.Err()
556
+ }
557
558
return nil
559
}
@@ -616,7 +616,7 @@ func getOutput(dagnode ipld.Node) (*Object, error) {
616
}
617
618
type progressReader struct {
619
- file files.File
619
+ file io.Reader
620
path string
621
out chan<- interface{}
622
bytes int64
core/coreunix/add_test.go
+4
-4
@@ -81,10 +81,10 @@ func TestAddGCLive(t *testing.T) {
81
datad := ioutil.NopCloser(bytes.NewBufferString("testfileD"))
82
rfd := files.NewReaderFile(datad, nil)
83
84
- slf := files.NewSliceFile([]files.FileEntry{
85
- {File: rfa, Name: "a"},
86
- {File: hangfile, Name: "b"},
87
- {File: rfd, Name: "d"},
84
+ slf := files.NewSliceFile([]files.DirEntry{
85
+ files.FileEntry("a", rfa),
86
+ files.FileEntry("b", hangfile),
87
+ files.FileEntry("d", rfd),
88
})
89
90
addDone := make(chan struct{})
fuse/readonly/ipfs_test.go
+2
-1
@@ -22,6 +22,7 @@ import (
22
ci "gx/ipfs/QmPuhRE325DR8ChNcFtgd6F1eANCHy1oohXZPpYop4xsK6/go-testutil/ci"
23
chunker "gx/ipfs/QmR4QQVkBZsZENRjYFVi8dEtPL3daZRNKk24m4r6WKJHNm/go-ipfs-chunker"
24
fstest "gx/ipfs/QmSJBsmLP1XMjv8hxYg2rUMdPDB7YUpyBo9idjrJ6Cmq6F/fuse/fs/fstestutil"
25
+ files "gx/ipfs/QmXWZCd8jfaHmt4UDSnjKmGcrQMw95bDGWqEeVLVJjoANX/go-ipfs-files"
26
importer "gx/ipfs/Qmbvw7kpSM2p6rbQ57WGRhhqNfCiNGW6EKH4xgHLw4bsnB/go-unixfs/importer"
27
uio "gx/ipfs/Qmbvw7kpSM2p6rbQ57WGRhhqNfCiNGW6EKH4xgHLw4bsnB/go-unixfs/io"
28
ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format"
@@ -180,7 +181,7 @@ func TestIpfsStressRead(t *testing.T) {
181
errs <- err
182
}
183
183
- data, err := ioutil.ReadAll(read)
184
+ data, err := ioutil.ReadAll(read.(files.File))
185
if err != nil {
186
errs <- err
187
}
test/integration/addcat_test.go
+1
-1
@@ -154,7 +154,7 @@ func DirectAddCat(data []byte, conf testutil.LatencyConfig) error {
154
155
// verify
156
bufout := new(bytes.Buffer)
157
- io.Copy(bufout, readerCatted)
157
+ io.Copy(bufout, readerCatted.(io.Reader))
158
if 0 != bytes.Compare(bufout.Bytes(), data) {
159
return errors.New("catted data does not match added data")
160
}
test/integration/bench_cat_test.go
+1
-1
@@ -101,7 +101,7 @@ func benchCat(b *testing.B, data []byte, conf testutil.LatencyConfig) error {
101
102
// verify
103
bufout := new(bytes.Buffer)
104
- io.Copy(bufout, readerCatted)
104
+ io.Copy(bufout, readerCatted.(io.Reader))
105
if 0 != bytes.Compare(bufout.Bytes(), data) {
106
return errors.New("catted data does not match added data")
107
}
test/integration/three_legged_cat_test.go
+1
-1
@@ -133,7 +133,7 @@ func RunThreeLeggedCat(data []byte, conf testutil.LatencyConfig) error {
133
134
// verify
135
bufout := new(bytes.Buffer)
136
- io.Copy(bufout, readerCatted)
136
+ io.Copy(bufout, readerCatted.(io.Reader))
137
if 0 != bytes.Compare(bufout.Bytes(), data) {
138
return errors.New("catted data does not match added data")
139
}