Add config option to enable urlstore.
License: MIT Signed-off-by: Kevin Atkinson <k@kevina.org>
Kevin Atkinson committed
Jun 23, 2018 at 17:03 UTC
b53a1b3022bdb5a0b480ecb4fb87826db73fdc7e
6 files changed
+45
-5
core/builder.go
+1
-1
@@ -213,7 +213,7 @@ func setupNode(ctx context.Context, n *IpfsNode, cfg *BuildCfg) error {
213
n.GCLocker = bstore.NewGCLocker()
214
n.Blockstore = bstore.NewGCBlockstore(cbs, n.GCLocker)
215
216
- if conf.Experimental.FilestoreEnabled {
216
+ if conf.Experimental.FilestoreEnabled || conf.Experimental.UrlstoreEnabled {
217
// hash security
218
n.Filestore = filestore.NewFilestore(cbs, n.Repo.FileManager())
219
n.Blockstore = bstore.NewGCBlockstore(n.Filestore, n.GCLocker)
docs/experimental-features.md
+21
@@ -17,6 +17,7 @@ the above issue.
17
- [go-multiplex stream muxer](#go-multiplex-stream-muxer)
18
- [Raw leaves for unixfs files](#raw-leaves-for-unixfs-files)
19
- [ipfs filestore](#ipfs-filestore)
20
+- [ipfs urlstore](#ipfs-urlstore)
21
- [BadgerDB datastore](#badger-datastore)
22
- [Private Networks](#private-networks)
23
- [ipfs p2p](#ipfs-p2p)
@@ -164,6 +165,26 @@ And then pass the `--nocopy` flag when running `ipfs add`
165
166
---
167
168
+## ipfs urlstore
169
+Allows ipfs to retrieve blocks contents via a url instead of storing it in the datastore
170
+
171
+### State
172
+experimental.
173
+
174
+### In Version
175
+???.
176
+
177
+### How to enable
178
+Modify your ipfs config:
179
+```
180
+ipfs config --json Experimental.UrlstoreEnabled true
181
+```
182
+
183
+### Road to being a real feature
184
+???.
185
+
186
+---
187
+
188
## Private Networks
189
190
Allows ipfs to only connect to other peers who have a shared secret key.
filestore/filestore_test.go
+1
@@ -23,6 +23,7 @@ func newTestFilestore(t *testing.T) (string, *Filestore) {
23
t.Fatal(err)
24
}
25
fm := NewFileManager(mds, testdir)
26
+ fm.AllowFiles = true
27
28
bs := blockstore.NewBlockstore(mds)
29
fstore := NewFilestore(bs, fm)
filestore/fsrefstore.go
+18
-3
@@ -29,8 +29,10 @@ var FilestorePrefix = ds.NewKey("filestore")
29
// to the actual location of the block data in the filesystem
30
// (a path and an offset).
31
type FileManager struct {
32
- ds ds.Batching
33
- root string
32
+ AllowFiles bool
33
+ AllowUrls bool
34
+ ds ds.Batching
35
+ root string
36
}
37
38
// CorruptReferenceError implements the error interface.
@@ -52,7 +54,7 @@ func (c CorruptReferenceError) Error() string {
54
// datastore and root. All FilestoreNodes paths are relative to the
55
// root path given here, which is prepended for any operations.
56
func NewFileManager(ds ds.Batching, root string) *FileManager {
55
- return &FileManager{dsns.Wrap(ds, FilestorePrefix), root}
57
+ return &FileManager{ds: dsns.Wrap(ds, FilestorePrefix), root: root}
58
}
59
60
// AllKeysChan returns a channel from which to read the keys stored in
@@ -157,6 +159,10 @@ func unmarshalDataObj(o interface{}) (*pb.DataObj, error) {
159
}
160
161
func (f *FileManager) readFileDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) {
162
+ if !f.AllowFiles {
163
+ return nil, fmt.Errorf("filestore not enabled")
164
+ }
165
+
166
p := filepath.FromSlash(d.GetFilePath())
167
abspath := filepath.Join(f.root, p)
168
@@ -196,6 +202,9 @@ func (f *FileManager) readFileDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error)
202
203
// reads and verifies the block from URL
204
func (f *FileManager) readURLDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) {
205
+ if !f.AllowUrls {
206
+ return nil, fmt.Errorf("urlstore not enabled")
207
+ }
208
209
req, err := http.NewRequest("GET", d.GetFilePath(), nil)
210
if err != nil {
@@ -257,6 +266,9 @@ func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error {
266
var dobj pb.DataObj
267
268
if !IsURL(b.PosInfo.FullPath) {
269
+ if !f.AllowFiles {
270
+ return fmt.Errorf("filestore not enabled")
271
+ }
272
if !filepath.HasPrefix(b.PosInfo.FullPath, f.root) {
273
return fmt.Errorf("cannot add filestore references outside ipfs root (%s)", f.root)
274
}
@@ -268,6 +280,9 @@ func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error {
280
281
dobj.FilePath = proto.String(filepath.ToSlash(p))
282
} else {
283
+ if !f.AllowUrls {
284
+ return fmt.Errorf("urlstore not enabled")
285
+ }
286
dobj.FilePath = proto.String(b.PosInfo.FullPath)
287
}
288
dobj.Offset = proto.Uint64(b.PosInfo.Offset)
repo/config/experiments.go
+1
@@ -2,6 +2,7 @@ package config
2
3
type Experiments struct {
4
FilestoreEnabled bool
5
+ UrlstoreEnabled bool
6
ShardingEnabled bool
7
Libp2pStreamMounting bool
8
}
repo/fsrepo/fsrepo.go
+3
-1
@@ -175,8 +175,10 @@ func open(repoPath string) (repo.Repo, error) {
175
return nil, err
176
}
177
178
- if r.config.Experimental.FilestoreEnabled {
178
+ if r.config.Experimental.FilestoreEnabled || r.config.Experimental.UrlstoreEnabled {
179
r.filemgr = filestore.NewFileManager(r.ds, filepath.Dir(r.path))
180
+ r.filemgr.AllowFiles = r.config.Experimental.FilestoreEnabled
181
+ r.filemgr.AllowUrls = r.config.Experimental.UrlstoreEnabled
182
}
183
184
keepLocked = true