filestore: add URLStore
License: MIT Signed-off-by: Jakub Sztandera <kubuxu@protonmail.ch>
Jakub Sztandera committed
Oct 6, 2017 at 14:17 UTC
1a835202b23983fb74ded8d365a63219eb427e4f
14 files changed
+186
-31
core/commands/commands_test.go
+2
@@ -210,6 +210,8 @@ func TestCommands(t *testing.T) {
210
"/tar/add",
211
"/tar/cat",
212
"/update",
213
+ "/urlstore",
214
+ "/urlstore/add",
215
"/version",
216
}
217
core/commands/root.go
+2
-1
@@ -5,12 +5,12 @@ import (
5
"strings"
6
7
oldcmds "github.com/ipfs/go-ipfs/commands"
8
+ lgc "github.com/ipfs/go-ipfs/commands/legacy"
9
dag "github.com/ipfs/go-ipfs/core/commands/dag"
10
e "github.com/ipfs/go-ipfs/core/commands/e"
11
ocmd "github.com/ipfs/go-ipfs/core/commands/object"
12
unixfs "github.com/ipfs/go-ipfs/core/commands/unixfs"
13
13
- lgc "github.com/ipfs/go-ipfs/commands/legacy"
14
"gx/ipfs/QmNueRyPRQiV7PUEpnP4GgGLuK1rKQLaRW7sfPvUetYig1/go-ipfs-cmds"
15
logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log"
16
"gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit"
@@ -136,6 +136,7 @@ var rootSubcommands = map[string]*cmds.Command{
136
"tar": lgc.NewCommand(TarCmd),
137
"file": lgc.NewCommand(unixfs.UnixFSCmd),
138
"update": lgc.NewCommand(ExternalBinary()),
139
+ "urlstore": lgc.NewCommand(UrlStoreCmd),
140
"version": lgc.NewCommand(VersionCmd),
141
"shutdown": lgc.NewCommand(daemonShutdownCmd),
142
}
core/commands/urlstore.go
new
+84
@@ -0,0 +1,84 @@
1
+package commands
2
+
3
+import (
4
+ "fmt"
5
+ "io"
6
+ "net/http"
7
+ "strings"
8
+
9
+ cmds "github.com/ipfs/go-ipfs/commands"
10
+ balanced "github.com/ipfs/go-ipfs/importer/balanced"
11
+ ihelper "github.com/ipfs/go-ipfs/importer/helpers"
12
+
13
+ mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash"
14
+ chunk "gx/ipfs/QmXnzH7wowyLZy8XJxxaQCVTgLMcDXdMBznmsrmQWCyiQV/go-ipfs-chunker"
15
+ cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
16
+ cmdkit "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit"
17
+)
18
+
19
+var UrlStoreCmd = &cmds.Command{
20
+
21
+ Subcommands: map[string]*cmds.Command{
22
+ "add": urlAdd,
23
+ },
24
+}
25
+
26
+var urlAdd = &cmds.Command{
27
+ Arguments: []cmdkit.Argument{
28
+ cmdkit.StringArg("url", true, false, "URL to add to IPFS"),
29
+ },
30
+ Type: BlockStat{},
31
+
32
+ Run: func(req cmds.Request, res cmds.Response) {
33
+ url := req.Arguments()[0]
34
+ n, err := req.InvocContext().GetNode()
35
+ if err != nil {
36
+ res.SetError(err, cmdkit.ErrNormal)
37
+ return
38
+ }
39
+
40
+ hreq, err := http.NewRequest("GET", url, nil)
41
+ if err != nil {
42
+ res.SetError(err, cmdkit.ErrNormal)
43
+ return
44
+ }
45
+
46
+ hres, err := http.DefaultClient.Do(hreq)
47
+ if err != nil {
48
+ res.SetError(err, cmdkit.ErrNormal)
49
+ return
50
+ }
51
+ if hres.StatusCode != http.StatusOK {
52
+ res.SetError(fmt.Errorf("expected code 200, got: %d", hres.StatusCode), cmdkit.ErrNormal)
53
+ return
54
+ }
55
+
56
+ chk := chunk.NewSizeSplitter(hres.Body, chunk.DefaultBlockSize)
57
+ prefix := cid.NewPrefixV1(cid.DagProtobuf, mh.SHA2_256)
58
+ dbp := &ihelper.DagBuilderParams{
59
+ Dagserv: n.DAG,
60
+ RawLeaves: true,
61
+ Maxlinks: ihelper.DefaultLinksPerBlock,
62
+ NoCopy: true,
63
+ Prefix: &prefix,
64
+ URL: url,
65
+ }
66
+
67
+ blc, err := balanced.Layout(dbp.New(chk))
68
+ if err != nil {
69
+ res.SetError(err, cmdkit.ErrNormal)
70
+ return
71
+ }
72
+
73
+ res.SetOutput(BlockStat{
74
+ Key: blc.Cid().String(),
75
+ Size: int(hres.ContentLength),
76
+ })
77
+ },
78
+ Marshalers: cmds.MarshalerMap{
79
+ cmds.Text: func(res cmds.Response) (io.Reader, error) {
80
+ bs := res.Output().(*BlockStat)
81
+ return strings.NewReader(bs.Key + "\n"), nil
82
+ },
83
+ },
84
+}
core/coreunix/add.go
+1
-1
@@ -19,12 +19,12 @@ import (
19
"github.com/ipfs/go-ipfs/pin"
20
unixfs "github.com/ipfs/go-ipfs/unixfs"
21
22
- posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo"
22
ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format"
23
chunker "gx/ipfs/QmXnzH7wowyLZy8XJxxaQCVTgLMcDXdMBznmsrmQWCyiQV/go-ipfs-chunker"
24
cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
25
logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log"
26
files "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit/files"
27
+ posinfo "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo"
28
bstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore"
29
)
30
core/coreunix/add_test.go
+1
-1
@@ -18,9 +18,9 @@ import (
18
"github.com/ipfs/go-ipfs/repo/config"
19
20
blocks "gx/ipfs/QmTRCUvZLiir12Qr6MV3HKfKMHX8Nf1Vddn6t2g5nsQSb9/go-block-format"
21
- pi "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo"
21
cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
22
files "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit/files"
23
+ pi "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo"
24
blockstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore"
25
datastore "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore"
26
syncds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore/sync"
filestore/filestore.go
+1
-1
@@ -11,9 +11,9 @@ import (
11
"context"
12
13
blocks "gx/ipfs/QmTRCUvZLiir12Qr6MV3HKfKMHX8Nf1Vddn6t2g5nsQSb9/go-block-format"
14
- posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo"
14
cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
15
logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log"
16
+ posinfo "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo"
17
blockstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore"
18
dsq "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore/query"
19
)
filestore/filestore_test.go
+1
-1
@@ -9,8 +9,8 @@ import (
9
10
dag "github.com/ipfs/go-ipfs/merkledag"
11
12
- posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo"
12
cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
13
+ posinfo "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo"
14
blockstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore"
15
ds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore"
16
)
filestore/fsrefstore.go
+64
-12
@@ -4,6 +4,7 @@ import (
4
"context"
5
"fmt"
6
"io"
7
+ "net/http"
8
"os"
9
"path/filepath"
10
@@ -12,8 +13,8 @@ import (
13
dshelp "gx/ipfs/QmNP2u7bofwUQptHQGPfabGWtTCbxhNLSZKqbf1uzsup9V/go-ipfs-ds-help"
14
proto "gx/ipfs/QmT6n4mspWYEya864BhCUJEgyxiRfmiSY9ruQwTUNpRKaM/protobuf/proto"
15
blocks "gx/ipfs/QmTRCUvZLiir12Qr6MV3HKfKMHX8Nf1Vddn6t2g5nsQSb9/go-block-format"
15
- posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo"
16
cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
17
+ posinfo "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo"
18
blockstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore"
19
ds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore"
20
dsns "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore/namespace"
@@ -111,7 +112,6 @@ func (f *FileManager) Get(c *cid.Cid) (blocks.Block, error) {
112
if err != nil {
113
return nil, err
114
}
114
-
115
out, err := f.readDataObj(c, dobj)
116
if err != nil {
117
return nil, err
@@ -120,6 +120,14 @@ func (f *FileManager) Get(c *cid.Cid) (blocks.Block, error) {
120
return blocks.NewBlockWithCid(out, c)
121
}
122
123
+func (f *FileManager) readDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) {
124
+ if !d.GetURL() {
125
+ return f.readFileDataObj(c, d)
126
+ } else {
127
+ return f.readURLDataObj(c, d)
128
+ }
129
+}
130
+
131
func (f *FileManager) getDataObj(c *cid.Cid) (*pb.DataObj, error) {
132
o, err := f.ds.Get(dshelp.CidToDsKey(c))
133
switch err {
@@ -148,8 +156,7 @@ func unmarshalDataObj(o interface{}) (*pb.DataObj, error) {
156
return &dobj, nil
157
}
158
151
-// reads and verifies the block
152
-func (f *FileManager) readDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) {
159
+func (f *FileManager) readFileDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) {
160
p := filepath.FromSlash(d.GetFilePath())
161
abspath := filepath.Join(f.root, p)
162
@@ -187,6 +194,46 @@ func (f *FileManager) readDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) {
194
return outbuf, nil
195
}
196
197
+// reads and verifies the block from URL
198
+func (f *FileManager) readURLDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) {
199
+
200
+ req, err := http.NewRequest("GET", d.GetFilePath(), nil)
201
+ if err != nil {
202
+ return nil, err
203
+ }
204
+
205
+ req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", d.GetOffset(), d.GetOffset()+d.GetSize_()-1))
206
+
207
+ res, err := http.DefaultClient.Do(req)
208
+ if err != nil {
209
+ return nil, err
210
+ }
211
+ if res.StatusCode != http.StatusPartialContent {
212
+ return nil, fmt.Errorf("expected HTTP 206 got %d", res.StatusCode)
213
+ }
214
+
215
+ outbuf := make([]byte, d.GetSize_())
216
+ _, err = io.ReadFull(res.Body, outbuf)
217
+ if err == io.EOF || err == io.ErrUnexpectedEOF {
218
+ return nil, &CorruptReferenceError{StatusFileChanged, err}
219
+ } else if err != nil {
220
+ return nil, &CorruptReferenceError{StatusFileError, err}
221
+ }
222
+ res.Body.Close()
223
+
224
+ outcid, err := c.Prefix().Sum(outbuf)
225
+ if err != nil {
226
+ return nil, err
227
+ }
228
+
229
+ if !c.Equals(outcid) {
230
+ return nil, &CorruptReferenceError{StatusFileChanged,
231
+ fmt.Errorf("data in file did not match. %s offset %d", d.GetFilePath(), d.GetOffset())}
232
+ }
233
+
234
+ return outbuf, nil
235
+}
236
+
237
// Has returns if the FileManager is storing a block reference. It does not
238
// validate the data, nor checks if the reference is valid.
239
func (f *FileManager) Has(c *cid.Cid) (bool, error) {
@@ -209,16 +256,21 @@ func (f *FileManager) Put(b *posinfo.FilestoreNode) error {
256
func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error {
257
var dobj pb.DataObj
258
212
- if !filepath.HasPrefix(b.PosInfo.FullPath, f.root) {
213
- return fmt.Errorf("cannot add filestore references outside ipfs root (%s)", f.root)
214
- }
259
+ if !b.PosInfo.IsURL {
260
+ if !filepath.HasPrefix(b.PosInfo.FullPath, f.root) {
261
+ return fmt.Errorf("cannot add filestore references outside ipfs root (%s)", f.root)
262
+ }
263
216
- p, err := filepath.Rel(f.root, b.PosInfo.FullPath)
217
- if err != nil {
218
- return err
219
- }
264
+ p, err := filepath.Rel(f.root, b.PosInfo.FullPath)
265
+ if err != nil {
266
+ return err
267
+ }
268
221
- dobj.FilePath = proto.String(filepath.ToSlash(p))
269
+ dobj.FilePath = proto.String(filepath.ToSlash(p))
270
+ } else {
271
+ dobj.FilePath = proto.String(b.PosInfo.FullPath)
272
+ dobj.URL = proto.Bool(true)
273
+ }
274
dobj.Offset = proto.Uint64(b.PosInfo.Offset)
275
dobj.Size_ = proto.Uint64(uint64(len(b.RawData())))
276
filestore/pb/Makefile
+5
-7
@@ -1,10 +1,8 @@
1
-PB = $(wildcard *.proto)
2
-GO = $(PB:.proto=.pb.go)
1
+include mk/header.mk
2
4
-all: $(GO)
3
+PB_$(d) = $(wildcard $(d)/*.proto)
4
+TGTS_$(d) = $(PB_$(d):.proto=.pb.go)
5
6
-%.pb.go: %.proto
7
- protoc --gogo_out=. $<
6
+#DEPS_GO += $(TGTS_$(d))
7
9
-clean:
10
- rm *.pb.go
8
+include mk/footer.mk
filestore/pb/dataobj.pb.go
+10
-2
@@ -1,12 +1,12 @@
1
// Code generated by protoc-gen-gogo.
2
-// source: dataobj.proto
2
+// source: filestore/pb/dataobj.proto
3
// DO NOT EDIT!
4
5
/*
6
Package datastore_pb is a generated protocol buffer package.
7
8
It is generated from these files:
9
- dataobj.proto
9
+ filestore/pb/dataobj.proto
10
11
It has these top-level messages:
12
DataObj
@@ -26,6 +26,7 @@ type DataObj struct {
26
FilePath *string `protobuf:"bytes,1,opt,name=FilePath" json:"FilePath,omitempty"`
27
Offset *uint64 `protobuf:"varint,2,opt,name=Offset" json:"Offset,omitempty"`
28
Size_ *uint64 `protobuf:"varint,3,opt,name=Size" json:"Size,omitempty"`
29
+ URL *bool `protobuf:"varint,4,opt,name=URL" json:"URL,omitempty"`
30
XXX_unrecognized []byte `json:"-"`
31
}
32
@@ -54,6 +55,13 @@ func (m *DataObj) GetSize_() uint64 {
55
return 0
56
}
57
58
+func (m *DataObj) GetURL() bool {
59
+ if m != nil && m.URL != nil {
60
+ return *m.URL
61
+ }
62
+ return false
63
+}
64
+
65
func init() {
66
proto.RegisterType((*DataObj)(nil), "datastore.pb.DataObj")
67
}
filestore/pb/dataobj.proto
+1
@@ -4,4 +4,5 @@ message DataObj {
4
optional string FilePath = 1;
5
optional uint64 Offset = 2;
6
optional uint64 Size = 3;
7
+ optional bool URL = 4;
8
}
importer/helpers/dagbuilder.go
+9
-1
@@ -25,6 +25,7 @@ type DagBuilderHelper struct {
25
maxlinks int
26
batch *ipld.Batch
27
fullPath string
28
+ isUrl bool
29
stat os.FileInfo
30
prefix *cid.Prefix
31
}
@@ -48,6 +49,8 @@ type DagBuilderParams struct {
49
// NoCopy signals to the chunker that it should track fileinfo for
50
// filestore adds
51
NoCopy bool
52
+
53
+ URL string
54
}
55
56
// New generates a new DagBuilderHelper from the given params and a given
@@ -65,6 +68,11 @@ func (dbp *DagBuilderParams) New(spl chunker.Splitter) *DagBuilderHelper {
68
db.fullPath = fi.AbsPath()
69
db.stat = fi.Stat()
70
}
71
+
72
+ if dbp.URL != "" {
73
+ db.fullPath = dbp.URL
74
+ db.isUrl = true
75
+ }
76
return db
77
}
78
@@ -206,7 +214,7 @@ func (db *DagBuilderHelper) GetNextDataNode() (*UnixfsNode, error) {
214
// from the DagBuilderHelper.
215
func (db *DagBuilderHelper) SetPosInfo(node *UnixfsNode, offset uint64) {
216
if db.fullPath != "" {
209
- node.SetPosInfo(offset, db.fullPath, db.stat)
217
+ node.SetPosInfo(offset, db.fullPath, db.stat, db.isUrl)
218
}
219
}
220
importer/helpers/helpers.go
+3
-2
@@ -8,9 +8,9 @@ import (
8
dag "github.com/ipfs/go-ipfs/merkledag"
9
ft "github.com/ipfs/go-ipfs/unixfs"
10
11
- pi "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo"
11
ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format"
12
cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
13
+ pi "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo"
14
)
15
16
// BlockSizeLimit specifies the maximum size an imported block can have.
@@ -142,11 +142,12 @@ func (n *UnixfsNode) FileSize() uint64 {
142
143
// SetPosInfo sets information about the offset of the data of this node in a
144
// filesystem file.
145
-func (n *UnixfsNode) SetPosInfo(offset uint64, fullPath string, stat os.FileInfo) {
145
+func (n *UnixfsNode) SetPosInfo(offset uint64, fullPath string, stat os.FileInfo, isUrl bool) {
146
n.posInfo = &pi.PosInfo{
147
Offset: offset,
148
FullPath: fullPath,
149
Stat: stat,
150
+ IsURL: isUrl,
151
}
152
}
153
package.json
+2
-2
@@ -422,9 +422,9 @@
422
},
423
{
424
"author": "hector",
425
- "hash": "QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe",
425
+ "hash": "QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC",
426
"name": "go-ipfs-posinfo",
427
- "version": "0.0.2"
427
+ "version": "0.0.3"
428
},
429
{
430
"author": "hsanjuan",