coreapi unixfs: pin/local/hash-only options
License: MIT Signed-off-by: Łukasz Magiera <magik6k@gmail.com>
Łukasz Magiera committed
Sep 20, 2018 at 23:05 UTC
49946c69c4f3e12faa2d8c68116c9f863f666e9a
3 files changed
+138
-4
core/coreapi/interface/options/unixfs.go
+29
@@ -21,6 +21,10 @@ type UnixfsAddSettings struct {
21
22
Chunker string
23
Layout Layout
24
+
25
+ Pin bool
26
+ OnlyHash bool
27
+ Local bool
28
}
29
30
type UnixfsAddOption func(*UnixfsAddSettings) error
@@ -36,6 +40,10 @@ func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, error) {
40
41
Chunker: "size-262144",
42
Layout: BalancedLayout,
43
+
44
+ Pin: false,
45
+ OnlyHash: false,
46
+ Local: false,
47
}
48
49
for _, opt := range opts {
@@ -94,3 +102,24 @@ func (unixfsOpts) Layout(layout Layout) UnixfsAddOption {
102
return nil
103
}
104
}
105
+
106
+func (unixfsOpts) Pin(pin bool) UnixfsAddOption {
107
+ return func(settings *UnixfsAddSettings) error {
108
+ settings.Pin = pin
109
+ return nil
110
+ }
111
+}
112
+
113
+func (unixfsOpts) HashOnly(hashOnly bool) UnixfsAddOption {
114
+ return func(settings *UnixfsAddSettings) error {
115
+ settings.OnlyHash = hashOnly
116
+ return nil
117
+ }
118
+}
119
+
120
+func (unixfsOpts) Local(local bool) UnixfsAddOption {
121
+ return func(settings *UnixfsAddSettings) error {
122
+ settings.Local = local
123
+ return nil
124
+ }
125
+}
core/coreapi/unixfs.go
+53
-4
@@ -4,18 +4,25 @@ import (
4
"context"
5
"errors"
6
"fmt"
7
+ "github.com/ipfs/go-ipfs/core"
8
"io"
9
10
coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
11
"github.com/ipfs/go-ipfs/core/coreapi/interface/options"
12
"github.com/ipfs/go-ipfs/core/coreunix"
13
13
- uio "gx/ipfs/QmPL8bYtbACcSFFiSr4s2du7Na382NxRADR8hC7D9FkEA2/go-unixfs/io"
14
mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash"
15
cidutil "gx/ipfs/QmQJSeE3CX4zos9qeaG8EhecEK9zvrTEfTG84J8C5NVRwt/go-cidutil"
16
+ offline "gx/ipfs/QmR5miWuikPxWyUrzMYJVmFUcD44pGdtc98h9Qsbp4YcJw/go-ipfs-exchange-offline"
17
"gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files"
18
+ ft "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs"
19
+ uio "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs/io"
20
+ mfs "gx/ipfs/QmahrY1adY4wvtYEtoGjpZ2GUohTyukrkMkwUR9ytRjTG2/go-mfs"
21
dag "gx/ipfs/QmcBoNcAP6qDjgRBew7yjvCqHq7p5jMstE44jPUBWBxzsV/go-merkledag"
22
+ dagtest "gx/ipfs/QmcBoNcAP6qDjgRBew7yjvCqHq7p5jMstE44jPUBWBxzsV/go-merkledag/test"
23
+ blockservice "gx/ipfs/QmcRecCZWM2NZfCQrCe97Ch3Givv8KKEP82tGUDntzdLFe/go-blockservice"
24
ipld "gx/ipfs/QmdDXJs4axxefSPgK6Y1QhpJWKuDPnGJiqgq4uncb4rFHL/go-ipld-format"
25
+ bstore "gx/ipfs/QmdriVJgKx4JADRgh3cYPXqXmsa1A45SvFki1nDWHhQNtC/go-ipfs-blockstore"
26
)
27
28
type UnixfsAPI CoreAPI
@@ -59,7 +66,33 @@ func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options.
66
prefix.MhType = settings.MhType
67
prefix.MhLength = -1
68
62
- fileAdder, err := coreunix.NewAdder(ctx, api.node.Pinning, api.node.Blockstore, api.node.DAG)
69
+ n := api.node
70
+ if settings.OnlyHash {
71
+ nilnode, err := core.NewNode(ctx, &core.BuildCfg{
72
+ //TODO: need this to be true or all files
73
+ // hashed will be stored in memory!
74
+ NilRepo: true,
75
+ })
76
+ if err != nil {
77
+ return nil, err
78
+ }
79
+ n = nilnode
80
+ }
81
+
82
+ addblockstore := n.Blockstore
83
+ //if !(fscache || nocopy) {
84
+ addblockstore = bstore.NewGCBlockstore(n.BaseBlocks, n.GCLocker)
85
+ //}
86
+
87
+ exch := n.Exchange
88
+ if settings.Local {
89
+ exch = offline.Exchange(addblockstore)
90
+ }
91
+
92
+ bserv := blockservice.New(addblockstore, exch) // hash security 001
93
+ dserv := dag.NewDAGService(bserv)
94
+
95
+ fileAdder, err := coreunix.NewAdder(ctx, n.Pinning, n.Blockstore, dserv)
96
if err != nil {
97
return nil, err
98
}
@@ -68,7 +101,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options.
101
//fileAdder.Progress = progress
102
//fileAdder.Hidden = hidden
103
//fileAdder.Wrap = wrap
71
- //fileAdder.Pin = dopin
104
+ fileAdder.Pin = settings.Pin
105
fileAdder.Silent = true
106
fileAdder.RawLeaves = settings.RawLeaves
107
//fileAdder.NoCopy = nocopy
@@ -91,6 +124,19 @@ func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options.
124
}
125
}
126
127
+ if settings.OnlyHash {
128
+ md := dagtest.Mock()
129
+ emptyDirNode := ft.EmptyDirNode()
130
+ // Use the same prefix for the "empty" MFS root as for the file adder.
131
+ emptyDirNode.SetCidBuilder(fileAdder.CidBuilder)
132
+ mr, err := mfs.NewRoot(ctx, md, emptyDirNode, nil)
133
+ if err != nil {
134
+ return nil, err
135
+ }
136
+
137
+ fileAdder.SetMfsRoot(mr)
138
+ }
139
+
140
err = fileAdder.AddFile(files.NewReaderFile("", "", r, nil))
141
if err != nil {
142
return nil, err
@@ -101,8 +147,11 @@ func (api *UnixfsAPI) Add(ctx context.Context, r io.ReadCloser, opts ...options.
147
return nil, err
148
}
149
104
- return coreiface.IpfsPath(nd.Cid()), err
150
+ if settings.Pin {
151
+ err = fileAdder.PinRoot()
152
+ }
153
154
+ return coreiface.IpfsPath(nd.Cid()), err
155
}
156
157
// Cat returns the data contained by an IPFS or IPNS object(s) at path `p`.
core/coreapi/unixfs_test.go
+56
@@ -204,6 +204,13 @@ func TestAdd(t *testing.T) {
204
path: "/ipfs/QmNNhDGttafX3M1wKWixGre6PrLFGjnoPEDXjBYpTv93HP",
205
opts: []options.UnixfsAddOption{options.Unixfs.Chunker("size-4"), options.Unixfs.Layout(options.TrickleLeyout)},
206
},
207
+ // Local
208
+ {
209
+ name: "addLocal", // better cases in sharness
210
+ data: helloStr,
211
+ path: hello,
212
+ opts: []options.UnixfsAddOption{options.Unixfs.Local(true)},
213
+ },
214
}
215
216
for _, testCase := range cases {
@@ -244,6 +251,55 @@ func TestAdd(t *testing.T) {
251
}
252
}
253
254
+func TestAddPinned(t *testing.T) {
255
+ ctx := context.Background()
256
+ _, api, err := makeAPI(ctx)
257
+ if err != nil {
258
+ t.Error(err)
259
+ }
260
+
261
+ str := strings.NewReader(helloStr)
262
+ _, err = api.Unixfs().Add(ctx, ioutil.NopCloser(str), options.Unixfs.Pin(true))
263
+ if err != nil {
264
+ t.Error(err)
265
+ }
266
+
267
+ pins, err := api.Pin().Ls(ctx)
268
+ if len(pins) != 1 {
269
+ t.Fatalf("expected 1 pin, got %d", len(pins))
270
+ }
271
+
272
+ if pins[0].Path().String() != "/ipld/QmQy2Dw4Wk7rdJKjThjYXzfFJNaRKRHhHP5gHHXroJMYxk" {
273
+ t.Fatalf("got unexpected pin: %s", pins[0].Path().String())
274
+ }
275
+}
276
+
277
+func TestAddHashOnly(t *testing.T) {
278
+ ctx := context.Background()
279
+ _, api, err := makeAPI(ctx)
280
+ if err != nil {
281
+ t.Error(err)
282
+ }
283
+
284
+ str := strings.NewReader(helloStr)
285
+ p, err := api.Unixfs().Add(ctx, ioutil.NopCloser(str), options.Unixfs.HashOnly(true))
286
+ if err != nil {
287
+ t.Error(err)
288
+ }
289
+
290
+ if p.String() != hello {
291
+ t.Errorf("unxepected path: %s", p.String())
292
+ }
293
+
294
+ _, err = api.Block().Get(ctx, p)
295
+ if err == nil {
296
+ t.Fatal("expected an error")
297
+ }
298
+ if err.Error() != "blockservice: key not found" {
299
+ t.Errorf("unxepected error: %s", err.Error())
300
+ }
301
+}
302
+
303
func TestCatEmptyFile(t *testing.T) {
304
ctx := context.Background()
305
node, api, err := makeAPI(ctx)