@cryptotaxi247 / kubo / commits / 696a0f039

Simplify code: use prefix instead of flag to determine if a url

License: MIT Signed-off-by: Kevin Atkinson <k@kevina.org>

Kevin Atkinson committed Jun 22, 2018 at 21:39 UTC 696a0f039e464f4c8b8db6dfa78c4480d5e268fd
11 files changed +39 -32
core/coreunix/add.go
+1 -1
@@ -19,12 +19,12 @@ import (
19 "github.com/ipfs/go-ipfs/pin"
20 unixfs "github.com/ipfs/go-ipfs/unixfs"
21
22 + posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo"
23 ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format"
24 chunker "gx/ipfs/QmXnzH7wowyLZy8XJxxaQCVTgLMcDXdMBznmsrmQWCyiQV/go-ipfs-chunker"
25 cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
26 logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log"
27 files "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit/files"
27 - posinfo "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo"
28 bstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore"
29 )
30
core/coreunix/add_test.go
+1 -1
@@ -18,9 +18,9 @@ import (
18 "github.com/ipfs/go-ipfs/repo/config"
19
20 blocks "gx/ipfs/QmTRCUvZLiir12Qr6MV3HKfKMHX8Nf1Vddn6t2g5nsQSb9/go-block-format"
21 + pi "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo"
22 cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
23 files "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit/files"
23 - pi "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo"
24 blockstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore"
25 datastore "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore"
26 syncds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore/sync"
filestore/filestore.go
+1 -1
@@ -11,9 +11,9 @@ import (
11 "context"
12
13 blocks "gx/ipfs/QmTRCUvZLiir12Qr6MV3HKfKMHX8Nf1Vddn6t2g5nsQSb9/go-block-format"
14 + posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo"
15 cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
16 logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log"
16 - posinfo "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo"
17 blockstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore"
18 dsq "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore/query"
19 )
filestore/filestore_test.go
+13 -1
@@ -9,8 +9,8 @@ import (
9
10 dag "github.com/ipfs/go-ipfs/merkledag"
11
12 + posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo"
13 cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
13 - posinfo "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo"
14 blockstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore"
15 ds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore"
16 )
@@ -162,3 +162,15 @@ func TestDeletes(t *testing.T) {
162 }
163 }
164 }
165 +
166 +func TestIsURL(t *testing.T) {
167 + if !IsURL("http://www.example.com") {
168 + t.Fatal("IsURL failed: http://www.example.com")
169 + }
170 + if !IsURL("https://www.example.com") {
171 + t.Fatal("IsURL failed: https://www.example.com")
172 + }
173 + if IsURL("adir/afile") {
174 + t.Fatal("IsURL recognized non-url")
175 + }
176 +}
filestore/fsrefstore.go
+9 -4
@@ -13,8 +13,8 @@ import (
13 dshelp "gx/ipfs/QmNP2u7bofwUQptHQGPfabGWtTCbxhNLSZKqbf1uzsup9V/go-ipfs-ds-help"
14 proto "gx/ipfs/QmT6n4mspWYEya864BhCUJEgyxiRfmiSY9ruQwTUNpRKaM/protobuf/proto"
15 blocks "gx/ipfs/QmTRCUvZLiir12Qr6MV3HKfKMHX8Nf1Vddn6t2g5nsQSb9/go-block-format"
16 + posinfo "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo"
17 cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
17 - posinfo "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo"
18 blockstore "gx/ipfs/QmdpuJBPBZ6sLPj9BQpn3Rpi38BT2cF1QMiUfyzNWeySW4/go-ipfs-blockstore"
19 ds "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore"
20 dsns "gx/ipfs/QmeiCcJfDW1GJnWUArudsv5rQsihpi4oyddPhdqo3CfX6i/go-datastore/namespace"
@@ -121,7 +121,7 @@ func (f *FileManager) Get(c *cid.Cid) (blocks.Block, error) {
121 }
122
123 func (f *FileManager) readDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) {
124 - if !d.GetURL() {
124 + if !IsURL(d.GetFilePath()) {
125 return f.readFileDataObj(c, d)
126 } else {
127 return f.readURLDataObj(c, d)
@@ -256,7 +256,7 @@ func (f *FileManager) Put(b *posinfo.FilestoreNode) error {
256 func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error {
257 var dobj pb.DataObj
258
259 - if !b.PosInfo.IsURL {
259 + if !IsURL(b.PosInfo.FullPath) {
260 if !filepath.HasPrefix(b.PosInfo.FullPath, f.root) {
261 return fmt.Errorf("cannot add filestore references outside ipfs root (%s)", f.root)
262 }
@@ -269,7 +269,6 @@ func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error {
269 dobj.FilePath = proto.String(filepath.ToSlash(p))
270 } else {
271 dobj.FilePath = proto.String(b.PosInfo.FullPath)
272 - dobj.URL = proto.Bool(true)
272 }
273 dobj.Offset = proto.Uint64(b.PosInfo.Offset)
274 dobj.Size_ = proto.Uint64(uint64(len(b.RawData())))
@@ -298,3 +297,9 @@ func (f *FileManager) PutMany(bs []*posinfo.FilestoreNode) error {
297
298 return batch.Commit()
299 }
300 +
301 +func IsURL(str string) bool {
302 + return (len(str) > 7 && str[0] == 'h' && str[1] == 't' && str[2] == 't' && str[3] == 'p') &&
303 + ((len(str) > 8 && str[4] == 's' && str[5] == ':' && str[6] == '/' && str[7] == '/') ||
304 + (str[4] == ':' && str[5] == '/' && str[6] == '/'))
305 +}
filestore/pb/Makefile
+7 -5
@@ -1,8 +1,10 @@
1 -include mk/header.mk
1 +PB = $(wildcard *.proto)
2 +GO = $(PB:.proto=.pb.go)
3
3 -PB_$(d) = $(wildcard $(d)/*.proto)
4 -TGTS_$(d) = $(PB_$(d):.proto=.pb.go)
4 +all: $(GO)
5
6 -#DEPS_GO += $(TGTS_$(d))
6 +%.pb.go: %.proto
7 + protoc --gogo_out=. $<
8
8 -include mk/footer.mk
9 +clean:
10 + rm *.pb.go
filestore/pb/dataobj.pb.go
+2 -10
@@ -1,12 +1,12 @@
1 // Code generated by protoc-gen-gogo.
2 -// source: filestore/pb/dataobj.proto
2 +// source: dataobj.proto
3 // DO NOT EDIT!
4
5 /*
6 Package datastore_pb is a generated protocol buffer package.
7
8 It is generated from these files:
9 - filestore/pb/dataobj.proto
9 + dataobj.proto
10
11 It has these top-level messages:
12 DataObj
@@ -26,7 +26,6 @@ type DataObj struct {
26 FilePath *string `protobuf:"bytes,1,opt,name=FilePath" json:"FilePath,omitempty"`
27 Offset *uint64 `protobuf:"varint,2,opt,name=Offset" json:"Offset,omitempty"`
28 Size_ *uint64 `protobuf:"varint,3,opt,name=Size" json:"Size,omitempty"`
29 - URL *bool `protobuf:"varint,4,opt,name=URL" json:"URL,omitempty"`
29 XXX_unrecognized []byte `json:"-"`
30 }
31
@@ -55,13 +54,6 @@ func (m *DataObj) GetSize_() uint64 {
54 return 0
55 }
56
58 -func (m *DataObj) GetURL() bool {
59 - if m != nil && m.URL != nil {
60 - return *m.URL
61 - }
62 - return false
63 -}
64 -
57 func init() {
58 proto.RegisterType((*DataObj)(nil), "datastore.pb.DataObj")
59 }
filestore/pb/dataobj.proto
-1
@@ -4,5 +4,4 @@ message DataObj {
4 optional string FilePath = 1;
5 optional uint64 Offset = 2;
6 optional uint64 Size = 3;
7 - optional bool URL = 4;
7 }
importer/helpers/dagbuilder.go
+1 -3
@@ -25,7 +25,6 @@ type DagBuilderHelper struct {
25 maxlinks int
26 batch *ipld.Batch
27 fullPath string
28 - isUrl bool
28 stat os.FileInfo
29 prefix *cid.Prefix
30 }
@@ -71,7 +70,6 @@ func (dbp *DagBuilderParams) New(spl chunker.Splitter) *DagBuilderHelper {
70
71 if dbp.URL != "" {
72 db.fullPath = dbp.URL
74 - db.isUrl = true
73 }
74 return db
75 }
@@ -214,7 +212,7 @@ func (db *DagBuilderHelper) GetNextDataNode() (*UnixfsNode, error) {
212 // from the DagBuilderHelper.
213 func (db *DagBuilderHelper) SetPosInfo(node *UnixfsNode, offset uint64) {
214 if db.fullPath != "" {
217 - node.SetPosInfo(offset, db.fullPath, db.stat, db.isUrl)
215 + node.SetPosInfo(offset, db.fullPath, db.stat)
216 }
217 }
218
importer/helpers/helpers.go
+2 -3
@@ -8,9 +8,9 @@ import (
8 dag "github.com/ipfs/go-ipfs/merkledag"
9 ft "github.com/ipfs/go-ipfs/unixfs"
10
11 + pi "gx/ipfs/QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe/go-ipfs-posinfo"
12 ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format"
13 cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
13 - pi "gx/ipfs/QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC/go-ipfs-posinfo"
14 )
15
16 // BlockSizeLimit specifies the maximum size an imported block can have.
@@ -142,12 +142,11 @@ func (n *UnixfsNode) FileSize() uint64 {
142
143 // SetPosInfo sets information about the offset of the data of this node in a
144 // filesystem file.
145 -func (n *UnixfsNode) SetPosInfo(offset uint64, fullPath string, stat os.FileInfo, isUrl bool) {
145 +func (n *UnixfsNode) SetPosInfo(offset uint64, fullPath string, stat os.FileInfo) {
146 n.posInfo = &pi.PosInfo{
147 Offset: offset,
148 FullPath: fullPath,
149 Stat: stat,
150 - IsURL: isUrl,
150 }
151 }
152
package.json
+2 -2
@@ -422,9 +422,9 @@
422 },
423 {
424 "author": "hector",
425 - "hash": "QmdGSfmN4wWNXVs2XiwHbpjnUikJ7HyrTJNHyYGdodyJDC",
425 + "hash": "QmUWsXLvYYDAaoAt9TPZpFX4ffHHMg46AHrz1ZLTN5ABbe",
426 "name": "go-ipfs-posinfo",
427 - "version": "0.0.3"
427 + "version": "0.0.2"
428 },
429 {
430 "author": "hsanjuan",