Flatten multipart file transfers
License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com>
Jeromy committed
Dec 5, 2015 at 19:20 UTC
d892661f3e7974cebdc9fd6f910ed741446f8944
11 files changed
+75
-73
commands/cli/parse.go
+15
-5
@@ -44,7 +44,17 @@ func Parse(input []string, stdin *os.File, root *cmds.Command) (cmds.Request, *c
44
}
45
}
46
47
- stringArgs, fileArgs, err := parseArgs(stringVals, stdin, cmd.Arguments, recursive, root)
47
+ // if '--hidden' is provided, enumerate hidden paths
48
+ hiddenOpt := req.Option("hidden")
49
+ hidden := false
50
+ if hiddenOpt != nil {
51
+ hidden, _, err = hiddenOpt.Bool()
52
+ if err != nil {
53
+ return req, nil, nil, u.ErrCast()
54
+ }
55
+ }
56
+
57
+ stringArgs, fileArgs, err := parseArgs(stringVals, stdin, cmd.Arguments, recursive, hidden, root)
58
if err != nil {
59
return req, cmd, path, err
60
}
@@ -223,7 +233,7 @@ func parseOpts(args []string, root *cmds.Command) (
233
return
234
}
235
226
-func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursive bool, root *cmds.Command) ([]string, []files.File, error) {
236
+func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursive, hidden bool, root *cmds.Command) ([]string, []files.File, error) {
237
// ignore stdin on Windows
238
if runtime.GOOS == "windows" {
239
stdin = nil
@@ -308,7 +318,7 @@ func parseArgs(inputs []string, stdin *os.File, argDefs []cmds.Argument, recursi
318
// treat stringArg values as file paths
319
fpath := inputs[0]
320
inputs = inputs[1:]
311
- file, err := appendFile(fpath, argDef, recursive)
321
+ file, err := appendFile(fpath, argDef, recursive, hidden)
322
if err != nil {
323
return nil, nil, err
324
}
@@ -389,7 +399,7 @@ func appendStdinAsString(args []string, stdin *os.File) ([]string, *os.File, err
399
const notRecursiveFmtStr = "'%s' is a directory, use the '-%s' flag to specify directories"
400
const dirNotSupportedFmtStr = "Invalid path '%s', argument '%s' does not support directories"
401
392
-func appendFile(fpath string, argDef *cmds.Argument, recursive bool) (files.File, error) {
402
+func appendFile(fpath string, argDef *cmds.Argument, recursive, hidden bool) (files.File, error) {
403
fpath = filepath.ToSlash(filepath.Clean(fpath))
404
405
if fpath == "." {
@@ -414,7 +424,7 @@ func appendFile(fpath string, argDef *cmds.Argument, recursive bool) (files.File
424
}
425
}
426
417
- return files.NewSerialFile(path.Base(fpath), fpath, stat)
427
+ return files.NewSerialFile(path.Base(fpath), fpath, hidden, stat)
428
}
429
430
// isTerminal returns true if stdin is a Stdin pipe (e.g. `cat file | ipfs`),
commands/files/multipartfile.go
+12
-18
@@ -1,10 +1,10 @@
1
package files
2
3
import (
4
+ "io"
5
"io/ioutil"
6
"mime"
7
"mime/multipart"
7
- "net/http"
8
"net/url"
9
)
10
@@ -12,7 +12,8 @@ const (
12
multipartFormdataType = "multipart/form-data"
13
multipartMixedType = "multipart/mixed"
14
15
- applicationSymlink = "application/symlink"
15
+ applicationDirectory = "application/x-directory"
16
+ applicationSymlink = "application/symlink"
17
18
contentTypeHeader = "Content-Type"
19
)
@@ -45,40 +46,33 @@ func NewFileFromPart(part *multipart.Part) (File, error) {
46
}, nil
47
}
48
48
- var params map[string]string
49
var err error
50
- f.Mediatype, params, err = mime.ParseMediaType(contentType)
50
+ f.Mediatype, _, err = mime.ParseMediaType(contentType)
51
if err != nil {
52
return nil, err
53
}
54
55
- if f.IsDirectory() {
56
- boundary, found := params["boundary"]
57
- if !found {
58
- return nil, http.ErrMissingBoundary
59
- }
60
-
61
- f.Reader = multipart.NewReader(part, boundary)
62
- }
63
-
55
return f, nil
56
}
57
58
func (f *MultipartFile) IsDirectory() bool {
68
- return f.Mediatype == multipartFormdataType || f.Mediatype == multipartMixedType
59
+ return f.Mediatype == multipartFormdataType || f.Mediatype == applicationDirectory
60
}
61
62
func (f *MultipartFile) NextFile() (File, error) {
63
if !f.IsDirectory() {
64
return nil, ErrNotDirectory
65
}
66
+ if f.Reader != nil {
67
+ part, err := f.Reader.NextPart()
68
+ if err != nil {
69
+ return nil, err
70
+ }
71
76
- part, err := f.Reader.NextPart()
77
- if err != nil {
78
- return nil, err
72
+ return NewFileFromPart(part)
73
}
74
81
- return NewFileFromPart(part)
75
+ return nil, io.EOF
76
}
77
78
func (f *MultipartFile) FileName() string {
commands/files/serialfile.go
+15
-4
@@ -6,6 +6,7 @@ import (
6
"io/ioutil"
7
"os"
8
"path/filepath"
9
+ "strings"
10
"syscall"
11
)
12
@@ -18,9 +19,10 @@ type serialFile struct {
19
files []os.FileInfo
20
stat os.FileInfo
21
current *File
22
+ hidden bool
23
}
24
23
-func NewSerialFile(name, path string, stat os.FileInfo) (File, error) {
25
+func NewSerialFile(name, path string, hidden bool, stat os.FileInfo) (File, error) {
26
switch mode := stat.Mode(); {
27
case mode.IsRegular():
28
file, err := os.Open(path)
@@ -35,7 +37,7 @@ func NewSerialFile(name, path string, stat os.FileInfo) (File, error) {
37
if err != nil {
38
return nil, err
39
}
38
- return &serialFile{name, path, contents, stat, nil}, nil
40
+ return &serialFile{name, path, contents, stat, nil, hidden}, nil
41
case mode&os.ModeSymlink != 0:
42
target, err := os.Readlink(path)
43
if err != nil {
@@ -68,6 +70,15 @@ func (f *serialFile) NextFile() (File, error) {
70
stat := f.files[0]
71
f.files = f.files[1:]
72
73
+ for !f.hidden && strings.HasPrefix(stat.Name(), ".") {
74
+ if len(f.files) == 0 {
75
+ return nil, io.EOF
76
+ }
77
+
78
+ stat = f.files[0]
79
+ f.files = f.files[1:]
80
+ }
81
+
82
// open the next file
83
fileName := filepath.ToSlash(filepath.Join(f.name, stat.Name()))
84
filePath := filepath.ToSlash(filepath.Join(f.path, stat.Name()))
@@ -75,7 +86,7 @@ func (f *serialFile) NextFile() (File, error) {
86
// recursively call the constructor on the next file
87
// if it's a regular file, we will open it as a ReaderFile
88
// if it's a directory, files in it will be opened serially
78
- sf, err := NewSerialFile(fileName, filePath, stat)
89
+ sf, err := NewSerialFile(fileName, filePath, f.hidden, stat)
90
if err != nil {
91
return nil, err
92
}
@@ -94,7 +105,7 @@ func (f *serialFile) FullPath() string {
105
}
106
107
func (f *serialFile) Read(p []byte) (int, error) {
97
- return 0, ErrNotReader
108
+ return 0, io.EOF
109
}
110
111
func (f *serialFile) Close() error {
commands/files/slicefile.go
+1
-1
@@ -41,7 +41,7 @@ func (f *SliceFile) FullPath() string {
41
}
42
43
func (f *SliceFile) Read(p []byte) (int, error) {
44
- return 0, ErrNotReader
44
+ return 0, io.EOF
45
}
46
47
func (f *SliceFile) Close() error {
commands/http/client.go
+2
-3
@@ -13,7 +13,6 @@ import (
13
"strings"
14
15
cmds "github.com/ipfs/go-ipfs/commands"
16
- path "github.com/ipfs/go-ipfs/path"
16
config "github.com/ipfs/go-ipfs/repo/config"
17
18
context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
@@ -86,8 +85,8 @@ func (c *client) Send(req cmds.Request) (cmds.Response, error) {
85
reader = fileReader
86
}
87
89
- pth := path.Join(req.Path())
90
- url := fmt.Sprintf(ApiUrlFormat, c.serverAddress, ApiPath, pth, query)
88
+ path := strings.Join(req.Path(), "/")
89
+ url := fmt.Sprintf(ApiUrlFormat, c.serverAddress, ApiPath, path, query)
90
91
httpReq, err := http.NewRequest("POST", url, reader)
92
if err != nil {
commands/http/multifilereader.go
+24
-17
@@ -17,7 +17,7 @@ import (
17
type MultiFileReader struct {
18
io.Reader
19
20
- files files.File
20
+ files []files.File
21
currentFile io.Reader
22
buf bytes.Buffer
23
mpWriter *multipart.Writer
@@ -34,7 +34,7 @@ type MultiFileReader struct {
34
// if `form` is false, the Content-Type will be 'multipart/mixed'.
35
func NewMultiFileReader(file files.File, form bool) *MultiFileReader {
36
mfr := &MultiFileReader{
37
- files: file,
37
+ files: []files.File{file},
38
form: form,
39
mutex: &sync.Mutex{},
40
}
@@ -54,34 +54,41 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) {
54
55
// if the current file isn't set, advance to the next file
56
if mfr.currentFile == nil {
57
- file, err := mfr.files.NextFile()
58
- if err == io.EOF {
59
- mfr.mpWriter.Close()
60
- mfr.closed = true
61
- } else if err != nil {
62
- return 0, err
57
+ var file files.File
58
+ for file == nil {
59
+ if len(mfr.files) == 0 {
60
+ mfr.mpWriter.Close()
61
+ mfr.closed = true
62
+ return mfr.buf.Read(buf)
63
+ }
64
+
65
+ nextfile, err := mfr.files[len(mfr.files)-1].NextFile()
66
+ if err == io.EOF {
67
+ mfr.files = mfr.files[:len(mfr.files)-1]
68
+ continue
69
+ } else if err != nil {
70
+ return 0, err
71
+ }
72
+
73
+ file = nextfile
74
}
75
76
// handle starting a new file part
77
if !mfr.closed {
78
79
var contentType string
69
- if s, ok := file.(*files.Symlink); ok {
70
- mfr.currentFile = s
71
-
80
+ if _, ok := file.(*files.Symlink); ok {
81
contentType = "application/symlink"
82
} else if file.IsDirectory() {
74
- // if file is a directory, create a multifilereader from it
75
- // (using 'multipart/mixed')
76
- nmfr := NewMultiFileReader(file, false)
77
- mfr.currentFile = nmfr
78
- contentType = fmt.Sprintf("multipart/mixed; boundary=%s", nmfr.Boundary())
83
+ mfr.files = append(mfr.files, file)
84
+ contentType = "application/x-directory"
85
} else {
86
// otherwise, use the file as a reader to read its contents
81
- mfr.currentFile = file
87
contentType = "application/octet-stream"
88
}
89
90
+ mfr.currentFile = file
91
+
92
// write the boundary and headers
93
header := make(textproto.MIMEHeader)
94
filename := url.QueryEscape(file.FileName())
core/commands/add.go
+1
-20
@@ -2,7 +2,6 @@ package commands
2
3
import (
4
"fmt"
5
- "io"
5
6
"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb"
7
"github.com/ipfs/go-ipfs/core/coreunix"
@@ -147,26 +146,8 @@ remains to be implemented.
146
fileAdder.Pin = dopin
147
fileAdder.Silent = silent
148
150
- // addAllFiles loops over a convenience slice file to
151
- // add each file individually. e.g. 'ipfs add a b c'
152
- addAllFiles := func(sliceFile files.File) error {
153
- for {
154
- file, err := sliceFile.NextFile()
155
- if err != nil && err != io.EOF {
156
- return err
157
- }
158
- if file == nil {
159
- return nil // done
160
- }
161
-
162
- if err := fileAdder.AddFile(file); err != nil {
163
- return err
164
- }
165
- }
166
- }
167
-
149
addAllAndPin := func(f files.File) error {
169
- if err := addAllFiles(f); err != nil {
150
+ if err := fileAdder.AddFile(f); err != nil {
151
return err
152
}
153
core/coreunix/add.go
+2
-2
@@ -255,7 +255,7 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) {
255
return "", err
256
}
257
258
- f, err := files.NewSerialFile(root, root, stat)
258
+ f, err := files.NewSerialFile(root, root, false, stat)
259
if err != nil {
260
return "", err
261
}
@@ -354,7 +354,7 @@ func (adder *Adder) addFile(file files.File) error {
354
355
switch {
356
case files.IsHidden(file) && !adder.Hidden:
357
- log.Debugf("%s is hidden, skipping", file.FileName())
357
+ log.Infof("%s is hidden, skipping", file.FileName())
358
return &hiddenFileError{file.FileName()}
359
case file.IsDirectory():
360
return adder.addDir(file)
exchange/bitswap/workers.go
+1
-1
@@ -89,7 +89,7 @@ func (bs *Bitswap) provideWorker(px process.Process) {
89
defer cancel()
90
91
if err := bs.network.Provide(ctx, k); err != nil {
92
- log.Error(err)
92
+ log.Warning(err)
93
}
94
}
95
importer/importer.go
+1
-1
@@ -29,7 +29,7 @@ func BuildDagFromFile(fpath string, ds dag.DAGService) (*dag.Node, error) {
29
return nil, fmt.Errorf("`%s` is a directory", fpath)
30
}
31
32
- f, err := files.NewSerialFile(fpath, fpath, stat)
32
+ f, err := files.NewSerialFile(fpath, fpath, false, stat)
33
if err != nil {
34
return nil, err
35
}
mfs/ops.go
+1
-1
@@ -102,7 +102,7 @@ func PutNode(r *Root, path string, nd *dag.Node) error {
102
// intermediary directories as needed if 'parents' is set to true
103
func Mkdir(r *Root, pth string, parents bool) error {
104
if pth == "" {
105
- panic("empty path")
105
+ return nil
106
}
107
parts := path.SplitList(pth)
108
if parts[0] == "" {