add remote pinning policy for mfs (#7798)
* remote pinning service MFS policy * update go-ipfs-config * hardening secret sanitization in `ipfs config` commands Co-authored-by: Adin Schmahmann <adin.schmahmann@gmail.com> Co-authored-by: Marcin Rataj <lidel@lidel.org>
Petar Maymounkov committed
Jan 28, 2021 at 15:58 UTC
8794928c31437f0bcf877326674b268f9528ae42
10 files changed
+641
-106
cmd/ipfs/daemon.go
+3
@@ -439,6 +439,9 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
439
// initialize metrics collector
440
prometheus.MustRegister(&corehttp.IpfsNodeCollector{Node: node})
441
442
+ // start MFS pinning thread
443
+ startPinMFS(daemonConfigPollInterval, cctx, &ipfsPinMFSNode{node})
444
+
445
// The daemon is *finally* ready.
446
fmt.Printf("Daemon is ready\n")
447
notifyReady()
cmd/ipfs/pinmfs.go
new
+266
@@ -0,0 +1,266 @@
1
+package main
2
+
3
+import (
4
+ "context"
5
+ "fmt"
6
+ "time"
7
+
8
+ "github.com/libp2p/go-libp2p-core/host"
9
+ peer "github.com/libp2p/go-libp2p-core/peer"
10
+
11
+ cid "github.com/ipfs/go-cid"
12
+ ipld "github.com/ipfs/go-ipld-format"
13
+ logging "github.com/ipfs/go-log"
14
+ pinclient "github.com/ipfs/go-pinning-service-http-client"
15
+
16
+ config "github.com/ipfs/go-ipfs-config"
17
+ "github.com/ipfs/go-ipfs/core"
18
+)
19
+
20
+// mfslog is the logger for remote mfs pinning
21
+var mfslog = logging.Logger("remotepinning/mfs")
22
+
23
+type lastPin struct {
24
+ Time time.Time
25
+ ServiceName string
26
+ ServiceConfig config.RemotePinningService
27
+ CID cid.Cid
28
+}
29
+
30
+func (x lastPin) IsValid() bool {
31
+ return x != lastPin{}
32
+}
33
+
34
+const daemonConfigPollInterval = time.Minute / 2
35
+const defaultRepinInterval = 5 * time.Minute
36
+
37
+type pinMFSContext interface {
38
+ Context() context.Context
39
+ GetConfigNoCache() (*config.Config, error)
40
+}
41
+
42
+type pinMFSNode interface {
43
+ RootNode() (ipld.Node, error)
44
+ Identity() peer.ID
45
+ PeerHost() host.Host
46
+}
47
+
48
+type ipfsPinMFSNode struct {
49
+ node *core.IpfsNode
50
+}
51
+
52
+func (x *ipfsPinMFSNode) RootNode() (ipld.Node, error) {
53
+ return x.node.FilesRoot.GetDirectory().GetNode()
54
+}
55
+
56
+func (x *ipfsPinMFSNode) Identity() peer.ID {
57
+ return x.node.Identity
58
+}
59
+
60
+func (x *ipfsPinMFSNode) PeerHost() host.Host {
61
+ return x.node.PeerHost
62
+}
63
+
64
+func startPinMFS(configPollInterval time.Duration, cctx pinMFSContext, node pinMFSNode) {
65
+ errCh := make(chan error)
66
+ go pinMFSOnChange(configPollInterval, cctx, node, errCh)
67
+ go func() {
68
+ for {
69
+ select {
70
+ case err, isOpen := <-errCh:
71
+ if !isOpen {
72
+ return
73
+ }
74
+ mfslog.Errorf("%v", err)
75
+ case <-cctx.Context().Done():
76
+ return
77
+ }
78
+ }
79
+ }()
80
+}
81
+
82
+func pinMFSOnChange(configPollInterval time.Duration, cctx pinMFSContext, node pinMFSNode, errCh chan<- error) {
83
+ defer close(errCh)
84
+
85
+ var tmo *time.Timer
86
+ defer func() {
87
+ if tmo != nil {
88
+ tmo.Stop()
89
+ }
90
+ }()
91
+
92
+ lastPins := map[string]lastPin{}
93
+ for {
94
+ // polling sleep
95
+ if tmo == nil {
96
+ tmo = time.NewTimer(configPollInterval)
97
+ } else {
98
+ tmo.Reset(configPollInterval)
99
+ }
100
+ select {
101
+ case <-cctx.Context().Done():
102
+ return
103
+ case <-tmo.C:
104
+ }
105
+
106
+ // reread the config, which may have changed in the meantime
107
+ cfg, err := cctx.GetConfigNoCache()
108
+ if err != nil {
109
+ select {
110
+ case errCh <- fmt.Errorf("pinning reading config (%v)", err):
111
+ case <-cctx.Context().Done():
112
+ return
113
+ }
114
+ continue
115
+ }
116
+ mfslog.Debugf("pinning loop is awake, %d remote services", len(cfg.Pinning.RemoteServices))
117
+
118
+ // get the most recent MFS root cid
119
+ rootNode, err := node.RootNode()
120
+ if err != nil {
121
+ select {
122
+ case errCh <- fmt.Errorf("pinning reading MFS root (%v)", err):
123
+ case <-cctx.Context().Done():
124
+ return
125
+ }
126
+ continue
127
+ }
128
+ rootCid := rootNode.Cid()
129
+
130
+ // pin to all remote services in parallel
131
+ pinAllMFS(cctx.Context(), node, cfg, rootCid, lastPins, errCh)
132
+ }
133
+}
134
+
135
+// pinAllMFS pins on all remote services in parallel to overcome DoS attacks.
136
+func pinAllMFS(ctx context.Context, node pinMFSNode, cfg *config.Config, rootCid cid.Cid, lastPins map[string]lastPin, errCh chan<- error) {
137
+ ch := make(chan lastPin, len(cfg.Pinning.RemoteServices))
138
+ for svcName_, svcConfig_ := range cfg.Pinning.RemoteServices {
139
+ // skip services where MFS is not enabled
140
+ svcName, svcConfig := svcName_, svcConfig_
141
+ mfslog.Debugf("pinning considering service %s for mfs pinning", svcName)
142
+ if !svcConfig.Policies.MFS.Enable {
143
+ mfslog.Debugf("pinning service %s is not enabled", svcName)
144
+ ch <- lastPin{}
145
+ continue
146
+ }
147
+ // read mfs pin interval for this service
148
+ var repinInterval time.Duration
149
+ if svcConfig.Policies.MFS.RepinInterval == "" {
150
+ repinInterval = defaultRepinInterval
151
+ } else {
152
+ var err error
153
+ repinInterval, err = time.ParseDuration(svcConfig.Policies.MFS.RepinInterval)
154
+ if err != nil {
155
+ select {
156
+ case errCh <- fmt.Errorf("remote pinning service %s has invalid MFS.RepinInterval (%v)", svcName, err):
157
+ case <-ctx.Done():
158
+ }
159
+ ch <- lastPin{}
160
+ continue
161
+ }
162
+ }
163
+
164
+ // do nothing, if MFS has not changed since last pin on the exact same service or waiting for MFS.RepinInterval
165
+ if last, ok := lastPins[svcName]; ok {
166
+ if last.ServiceConfig == svcConfig && (last.CID == rootCid || time.Since(last.Time) < repinInterval) {
167
+ if last.CID == rootCid {
168
+ mfslog.Debugf("pinning MFS root to %s: pin for %s exists since %s, skipping", svcName, rootCid, last.Time.String())
169
+ } else {
170
+ mfslog.Debugf("pinning MFS root to %s: skipped due to MFS.RepinInterval=%s (remaining: %s)", svcName, repinInterval.String(), (repinInterval - time.Since(last.Time)).String())
171
+ }
172
+ ch <- lastPin{}
173
+ continue
174
+ }
175
+ }
176
+
177
+ mfslog.Debugf("pinning MFS root %s to %s", rootCid, svcName)
178
+ go func() {
179
+ if r, err := pinMFS(ctx, node, rootCid, svcName, svcConfig); err != nil {
180
+ select {
181
+ case errCh <- fmt.Errorf("pinning MFS root %s to %s (%v)", rootCid, svcName, err):
182
+ case <-ctx.Done():
183
+ }
184
+ ch <- lastPin{}
185
+ } else {
186
+ ch <- r
187
+ }
188
+ }()
189
+ }
190
+ for i := 0; i < len(cfg.Pinning.RemoteServices); i++ {
191
+ if x := <-ch; x.IsValid() {
192
+ lastPins[x.ServiceName] = x
193
+ }
194
+ }
195
+}
196
+
197
+func pinMFS(
198
+ ctx context.Context,
199
+ node pinMFSNode,
200
+ cid cid.Cid,
201
+ svcName string,
202
+ svcConfig config.RemotePinningService,
203
+) (lastPin, error) {
204
+ c := pinclient.NewClient(svcConfig.API.Endpoint, svcConfig.API.Key)
205
+
206
+ pinName := svcConfig.Policies.MFS.PinName
207
+ if pinName == "" {
208
+ pinName = fmt.Sprintf("policy/%s/mfs", node.Identity().String())
209
+ }
210
+
211
+ // check if MFS pin exists (across all possible states) and inspect its CID
212
+ pinStatuses := []pinclient.Status{pinclient.StatusQueued, pinclient.StatusPinning, pinclient.StatusPinned, pinclient.StatusFailed}
213
+ lsPinCh, lsErrCh := c.Ls(ctx, pinclient.PinOpts.FilterName(pinName), pinclient.PinOpts.FilterStatus(pinStatuses...))
214
+ existingRequestID := "" // is there any pre-existing MFS pin with pinName (for any CID)?
215
+ alreadyPinned := false // is CID for current MFS already pinned?
216
+ pinTime := time.Now().UTC()
217
+ for ps := range lsPinCh {
218
+ existingRequestID = ps.GetRequestId()
219
+ if ps.GetPin().GetCid() == cid && ps.GetStatus() != pinclient.StatusFailed {
220
+ alreadyPinned = true
221
+ pinTime = ps.GetCreated().UTC()
222
+ break
223
+ }
224
+ }
225
+ for range lsPinCh { // in case the prior loop exits early
226
+ }
227
+ if err := <-lsErrCh; err != nil {
228
+ return lastPin{}, fmt.Errorf("error while listing remote pins: %v", err)
229
+ }
230
+
231
+ // CID of the current MFS root is already pinned, nothing to do
232
+ if alreadyPinned {
233
+ mfslog.Debugf("pinning MFS to %s: pin for %s exists since %s, skipping", svcName, cid, pinTime.String())
234
+ return lastPin{Time: pinTime, ServiceName: svcName, ServiceConfig: svcConfig, CID: cid}, nil
235
+ }
236
+
237
+ // Prepare Pin.name
238
+ addOpts := []pinclient.AddOption{pinclient.PinOpts.WithName(pinName)}
239
+
240
+ // Prepare Pin.origins
241
+ // Add own multiaddrs to the 'origins' array, so Pinning Service can
242
+ // use that as a hint and connect back to us (if possible)
243
+ if node.PeerHost() != nil {
244
+ addrs, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(node.PeerHost()))
245
+ if err != nil {
246
+ return lastPin{}, err
247
+ }
248
+ addOpts = append(addOpts, pinclient.PinOpts.WithOrigins(addrs...))
249
+ }
250
+
251
+ // Create or replace pin for MFS root
252
+ if existingRequestID != "" {
253
+ mfslog.Debugf("pinning to %s: replacing existing MFS root pin with %s", svcName, cid)
254
+ _, err := c.Replace(ctx, existingRequestID, cid, addOpts...)
255
+ if err != nil {
256
+ return lastPin{}, err
257
+ }
258
+ } else {
259
+ mfslog.Debugf("pinning to %s: creating a new MFS root pin for %s", svcName, cid)
260
+ _, err := c.Add(ctx, cid, addOpts...)
261
+ if err != nil {
262
+ return lastPin{}, err
263
+ }
264
+ }
265
+ return lastPin{Time: pinTime, ServiceName: svcName, ServiceConfig: svcConfig, CID: cid}, nil
266
+}
cmd/ipfs/pinmfs_test.go
new
+179
@@ -0,0 +1,179 @@
1
+package main
2
+
3
+import (
4
+ "context"
5
+ "fmt"
6
+ "strings"
7
+ "testing"
8
+ "time"
9
+
10
+ config "github.com/ipfs/go-ipfs-config"
11
+ ipld "github.com/ipfs/go-ipld-format"
12
+ "github.com/ipfs/go-merkledag"
13
+ "github.com/libp2p/go-libp2p-core/host"
14
+ peer "github.com/libp2p/go-libp2p-core/peer"
15
+)
16
+
17
+type testPinMFSContext struct {
18
+ ctx context.Context
19
+ cfg *config.Config
20
+ err error
21
+}
22
+
23
+func (x *testPinMFSContext) Context() context.Context {
24
+ return x.ctx
25
+}
26
+
27
+func (x *testPinMFSContext) GetConfigNoCache() (*config.Config, error) {
28
+ return x.cfg, x.err
29
+}
30
+
31
+type testPinMFSNode struct {
32
+ err error
33
+}
34
+
35
+func (x *testPinMFSNode) RootNode() (ipld.Node, error) {
36
+ return merkledag.NewRawNode([]byte{0x01}), x.err
37
+}
38
+
39
+func (x *testPinMFSNode) Identity() peer.ID {
40
+ return peer.ID("test_id")
41
+}
42
+
43
+func (x *testPinMFSNode) PeerHost() host.Host {
44
+ return nil
45
+}
46
+
47
+var testConfigPollInterval = time.Second
48
+
49
+func isErrorSimilar(e1, e2 error) bool {
50
+ switch {
51
+ case e1 == nil && e2 == nil:
52
+ return true
53
+ case e1 != nil && e2 == nil:
54
+ return false
55
+ case e1 == nil && e2 != nil:
56
+ return false
57
+ default:
58
+ return strings.Contains(e1.Error(), e2.Error()) || strings.Contains(e2.Error(), e1.Error())
59
+ }
60
+}
61
+
62
+func TestPinMFSConfigError(t *testing.T) {
63
+ ctx := &testPinMFSContext{
64
+ ctx: context.Background(),
65
+ cfg: nil,
66
+ err: fmt.Errorf("couldn't read config"),
67
+ }
68
+ node := &testPinMFSNode{}
69
+ errCh := make(chan error)
70
+ go pinMFSOnChange(testConfigPollInterval, ctx, node, errCh)
71
+ if !isErrorSimilar(<-errCh, ctx.err) {
72
+ t.Errorf("error did not propagate")
73
+ }
74
+ if !isErrorSimilar(<-errCh, ctx.err) {
75
+ t.Errorf("error did not propagate")
76
+ }
77
+}
78
+
79
+func TestPinMFSRootNodeError(t *testing.T) {
80
+ ctx := &testPinMFSContext{
81
+ ctx: context.Background(),
82
+ cfg: &config.Config{
83
+ Pinning: config.Pinning{},
84
+ },
85
+ err: nil,
86
+ }
87
+ node := &testPinMFSNode{
88
+ err: fmt.Errorf("cannot create root node"),
89
+ }
90
+ errCh := make(chan error)
91
+ go pinMFSOnChange(testConfigPollInterval, ctx, node, errCh)
92
+ if !isErrorSimilar(<-errCh, node.err) {
93
+ t.Errorf("error did not propagate")
94
+ }
95
+ if !isErrorSimilar(<-errCh, node.err) {
96
+ t.Errorf("error did not propagate")
97
+ }
98
+}
99
+
100
+func TestPinMFSService(t *testing.T) {
101
+ cfg_invalid_interval := &config.Config{
102
+ Pinning: config.Pinning{
103
+ RemoteServices: map[string]config.RemotePinningService{
104
+ "disabled": {
105
+ Policies: config.RemotePinningServicePolicies{
106
+ MFS: config.RemotePinningServiceMFSPolicy{
107
+ Enable: false,
108
+ },
109
+ },
110
+ },
111
+ "invalid_interval": {
112
+ Policies: config.RemotePinningServicePolicies{
113
+ MFS: config.RemotePinningServiceMFSPolicy{
114
+ Enable: true,
115
+ RepinInterval: "INVALID_INTERVAL",
116
+ },
117
+ },
118
+ },
119
+ },
120
+ },
121
+ }
122
+ cfg_valid_unnamed := &config.Config{
123
+ Pinning: config.Pinning{
124
+ RemoteServices: map[string]config.RemotePinningService{
125
+ "valid_unnamed": {
126
+ Policies: config.RemotePinningServicePolicies{
127
+ MFS: config.RemotePinningServiceMFSPolicy{
128
+ Enable: true,
129
+ PinName: "",
130
+ RepinInterval: "2s",
131
+ },
132
+ },
133
+ },
134
+ },
135
+ },
136
+ }
137
+ cfg_valid_named := &config.Config{
138
+ Pinning: config.Pinning{
139
+ RemoteServices: map[string]config.RemotePinningService{
140
+ "valid_named": {
141
+ Policies: config.RemotePinningServicePolicies{
142
+ MFS: config.RemotePinningServiceMFSPolicy{
143
+ Enable: true,
144
+ PinName: "pin_name",
145
+ RepinInterval: "2s",
146
+ },
147
+ },
148
+ },
149
+ },
150
+ },
151
+ }
152
+ testPinMFSServiceWithError(t, cfg_invalid_interval, "remote pinning service invalid_interval has invalid MFS.RepinInterval")
153
+ testPinMFSServiceWithError(t, cfg_valid_unnamed, "error while listing remote pins: empty response from remote pinning service")
154
+ testPinMFSServiceWithError(t, cfg_valid_named, "error while listing remote pins: empty response from remote pinning service")
155
+}
156
+
157
+func testPinMFSServiceWithError(t *testing.T, cfg *config.Config, expectedErrorPrefix string) {
158
+ goctx, cancel := context.WithCancel(context.Background())
159
+ ctx := &testPinMFSContext{
160
+ ctx: goctx,
161
+ cfg: cfg,
162
+ err: nil,
163
+ }
164
+ node := &testPinMFSNode{
165
+ err: nil,
166
+ }
167
+ errCh := make(chan error)
168
+ go pinMFSOnChange(testConfigPollInterval, ctx, node, errCh)
169
+ defer cancel()
170
+ // first pass through the pinning loop
171
+ err := <-errCh
172
+ if !strings.Contains((err).Error(), expectedErrorPrefix) {
173
+ t.Errorf("expecting error containing %q", expectedErrorPrefix)
174
+ }
175
+ // second pass through the pinning loop
176
+ if !strings.Contains((err).Error(), expectedErrorPrefix) {
177
+ t.Errorf("expecting error containing %q", expectedErrorPrefix)
178
+ }
179
+}
commands/context.go
+5
-1
@@ -10,7 +10,7 @@ import (
10
coreapi "github.com/ipfs/go-ipfs/core/coreapi"
11
loader "github.com/ipfs/go-ipfs/plugin/loader"
12
13
- "github.com/ipfs/go-ipfs-cmds"
13
+ cmds "github.com/ipfs/go-ipfs-cmds"
14
config "github.com/ipfs/go-ipfs-config"
15
logging "github.com/ipfs/go-log"
16
coreiface "github.com/ipfs/interface-go-ipfs-core"
@@ -48,6 +48,10 @@ func (c *Context) GetConfig() (*config.Config, error) {
48
return c.config, err
49
}
50
51
+func (c *Context) GetConfigNoCache() (*config.Config, error) {
52
+ return c.LoadConfig(c.ConfigRoot)
53
+}
54
+
55
// GetNode returns the node of the current Command execution
56
// context. It may construct it with the provided function.
57
func (c *Context) GetNode() (*core.IpfsNode, error) {
core/commands/config.go
+103
-77
@@ -36,8 +36,6 @@ const (
36
configDryRunOptionName = "dry-run"
37
)
38
39
-var tryRemoteServiceApiErr = errors.New("cannot show or change pinning services through this API (try: ipfs pin remote service --help)")
40
-
39
var ConfigCmd = &cmds.Command{
40
Helptext: cmds.HelpText{
41
Tagline: "Get and set ipfs config values.",
@@ -90,8 +88,8 @@ Set the value of the 'Datastore.Path' key:
88
89
// Temporary fix until we move ApiKey secrets out of the config file
90
// (remote services are a map, so more advanced blocking is required)
93
- if blocked := inBlockedScope(key, config.RemoteServicesSelector); blocked {
94
- return tryRemoteServiceApiErr
91
+ if blocked := matchesGlobPrefix(key, config.PinningConcealSelector); blocked {
92
+ return errors.New("cannot show or change pinning services credentials")
93
}
94
95
cfgRoot, err := cmdenv.GetConfigRoot(env)
@@ -148,22 +146,31 @@ Set the value of the 'Datastore.Path' key:
146
Type: ConfigField{},
147
}
148
151
-// Returns bool to indicate if tested key is in the blocked scope.
152
-// (scope includes parent, direct, and child match)
153
-func inBlockedScope(testKey string, blockedScope string) bool {
154
- blockedScope = strings.ToLower(blockedScope)
155
- roots := strings.Split(strings.ToLower(testKey), ".")
156
- var scope []string
157
- for _, name := range roots {
158
- scope := append(scope, name)
159
- impactedKey := strings.Join(scope, ".")
160
- // blockedScope=foo.bar.BLOCKED should return true
161
- // for parent and child impactedKeys: foo.bar and foo.bar.BLOCKED.subkey
162
- if strings.HasPrefix(impactedKey, blockedScope) || strings.HasPrefix(blockedScope, impactedKey) {
163
- return true
149
+// matchesGlobPrefix returns true if and only if the key matches the glob.
150
+// The key is a sequence of string "parts", separated by commas.
151
+// The glob is a sequence of string "patterns".
152
+// matchesGlobPrefix tries to match all of the first K parts to the first K patterns, respectively,
153
+// where K is the length of the shorter of key or glob.
154
+// A pattern matches a part if and only if the pattern is "*" or the lowercase pattern equals the lowercase part.
155
+//
156
+// For example:
157
+// matchesGlobPrefix("foo.bar", []string{"*", "bar", "baz"}) returns true
158
+// matchesGlobPrefix("foo.bar.baz", []string{"*", "bar"}) returns true
159
+// matchesGlobPrefix("foo.bar", []string{"baz", "*"}) returns false
160
+func matchesGlobPrefix(key string, glob []string) bool {
161
+ k := strings.Split(key, ".")
162
+ for i, g := range glob {
163
+ if i >= len(k) {
164
+ break
165
+ }
166
+ if g == "*" {
167
+ continue
168
+ }
169
+ if !strings.EqualFold(k[i], g) {
170
+ return false
171
}
172
}
166
- return false
173
+ return true
174
}
175
176
var configShowCmd = &cmds.Command{
@@ -173,7 +180,7 @@ var configShowCmd = &cmds.Command{
180
NOTE: For security reasons, this command will omit your private key and remote services. If you would like to make a full backup of your config (private key included), you must copy the config file from your repo.
181
`,
182
},
176
- Type: map[string]interface{}{},
183
+ Type: make(map[string]interface{}),
184
Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
185
cfgRoot, err := cmdenv.GetConfigRoot(env)
186
if err != nil {
@@ -196,12 +203,12 @@ NOTE: For security reasons, this command will omit your private key and remote s
203
return err
204
}
205
199
- err = scrubValue(cfg, []string{config.IdentityTag, config.PrivKeyTag})
206
+ cfg, err = scrubValue(cfg, []string{config.IdentityTag, config.PrivKeyTag})
207
if err != nil {
208
return err
209
}
210
204
- err = scrubOptionalValue(cfg, []string{config.PinningTag, config.RemoteServicesTag})
211
+ cfg, err = scrubOptionalValue(cfg, config.PinningConcealSelector)
212
if err != nil {
213
return err
214
}
@@ -222,54 +229,49 @@ NOTE: For security reasons, this command will omit your private key and remote s
229
}
230
231
// Scrubs value and returns error if missing
225
-func scrubValue(m map[string]interface{}, key []string) error {
226
- return scrub(m, key, false)
232
+func scrubValue(m map[string]interface{}, key []string) (map[string]interface{}, error) {
233
+ return scrubMapInternal(m, key, false)
234
}
235
236
// Scrubs value and returns no error if missing
230
-func scrubOptionalValue(m map[string]interface{}, key []string) error {
231
- return scrub(m, key, true)
237
+func scrubOptionalValue(m map[string]interface{}, key []string) (map[string]interface{}, error) {
238
+ return scrubMapInternal(m, key, true)
239
}
240
234
-func scrub(m map[string]interface{}, key []string, okIfMissing bool) error {
235
- find := func(m map[string]interface{}, k string) (string, interface{}, bool) {
236
- lckey := strings.ToLower(k)
237
- for mkey, val := range m {
238
- lcmkey := strings.ToLower(mkey)
239
- if lckey == lcmkey {
240
- return mkey, val, true
241
- }
242
- }
243
- return "", nil, false
241
+func scrubEither(u interface{}, key []string, okIfMissing bool) (interface{}, error) {
242
+ m, ok := u.(map[string]interface{})
243
+ if ok {
244
+ return scrubMapInternal(m, key, okIfMissing)
245
}
246
+ return scrubValueInternal(m, key, okIfMissing)
247
+}
248
246
- cur := m
247
- for _, k := range key[:len(key)-1] {
248
- foundk, val, ok := find(cur, k)
249
- if !ok && !okIfMissing {
250
- return errors.New("failed to find specified key")
251
- }
252
-
253
- if foundk != k {
254
- // case mismatch, calling this an error
255
- return fmt.Errorf("case mismatch in config, expected %q but got %q", k, foundk)
256
- }
257
-
258
- mval, mok := val.(map[string]interface{})
259
- if !mok {
260
- return fmt.Errorf("%s was not a map", foundk)
261
- }
262
-
263
- cur = mval
249
+func scrubValueInternal(v interface{}, key []string, okIfMissing bool) (interface{}, error) {
250
+ if v == nil && !okIfMissing {
251
+ return nil, errors.New("failed to find specified key")
252
}
253
+ return nil, nil
254
+}
255
266
- todel, _, ok := find(cur, key[len(key)-1])
267
- if !ok && !okIfMissing {
268
- return fmt.Errorf("%s, not found", strings.Join(key, "."))
256
+func scrubMapInternal(m map[string]interface{}, key []string, okIfMissing bool) (map[string]interface{}, error) {
257
+ if len(key) == 0 {
258
+ return make(map[string]interface{}), nil // delete value
259
}
270
-
271
- delete(cur, todel)
272
- return nil
260
+ n := map[string]interface{}{}
261
+ for k, v := range m {
262
+ if key[0] == "*" || strings.EqualFold(key[0], k) {
263
+ u, err := scrubEither(v, key[1:], okIfMissing)
264
+ if err != nil {
265
+ return nil, err
266
+ }
267
+ if u != nil {
268
+ n[k] = u
269
+ }
270
+ } else {
271
+ n[k] = v
272
+ }
273
+ }
274
+ return n, nil
275
}
276
277
var configEditCmd = &cmds.Command{
@@ -421,7 +423,7 @@ func scrubPrivKey(cfg *config.Config) (map[string]interface{}, error) {
423
return nil, err
424
}
425
424
- err = scrubValue(cfgMap, []string{config.IdentityTag, config.PrivKeyTag})
426
+ cfgMap, err = scrubValue(cfgMap, []string{config.IdentityTag, config.PrivKeyTag})
427
if err != nil {
428
return nil, err
429
}
@@ -503,14 +505,14 @@ func editConfig(filename string) error {
505
}
506
507
func replaceConfig(r repo.Repo, file io.Reader) error {
506
- var cfg config.Config
507
- if err := json.NewDecoder(file).Decode(&cfg); err != nil {
508
+ var newCfg config.Config
509
+ if err := json.NewDecoder(file).Decode(&newCfg); err != nil {
510
return errors.New("failed to decode file as config")
511
}
512
513
// Handle Identity.PrivKey (secret)
514
513
- if len(cfg.Identity.PrivKey) != 0 {
515
+ if len(newCfg.Identity.PrivKey) != 0 {
516
return errors.New("setting private key with API is not supported")
517
}
518
@@ -524,33 +526,57 @@ func replaceConfig(r repo.Repo, file io.Reader) error {
526
return errors.New("private key in config was not a string")
527
}
528
527
- cfg.Identity.PrivKey = pkstr
529
+ newCfg.Identity.PrivKey = pkstr
530
+
531
+ // Handle Pinning.RemoteServices (API.Key of each service is a secret)
532
529
- // Handle Pinning.RemoteServices (ApiKey of each service is secret)
530
- // Note: these settings are opt-in and may be missing
533
+ newServices := newCfg.Pinning.RemoteServices
534
+ oldServices, err := getRemotePinningServices(r)
535
+ if err != nil {
536
+ return fmt.Errorf("failed to load remote pinning services info (%v)", err)
537
+ }
538
532
- if len(cfg.Pinning.RemoteServices) != 0 {
533
- return tryRemoteServiceApiErr
539
+ // fail fast if service lists are obviously different
540
+ if len(newServices) != len(oldServices) {
541
+ return errors.New("cannot add or remove remote pinning services with 'config replace'")
542
}
543
536
- // detect if existing config has any remote services defined..
537
- if remoteServicesTag, err := getConfig(r, config.RemoteServicesSelector); err == nil {
544
+ // re-apply API details and confirm every modified service already existed
545
+ for name, oldSvc := range oldServices {
546
+ if newSvc, hadSvc := newServices[name]; hadSvc {
547
+ // fail if input changes any of API details
548
+ // (interop with config show: allow Endpoint as long it did not change)
549
+ if len(newSvc.API.Key) != 0 || (len(newSvc.API.Endpoint) != 0 && newSvc.API.Endpoint != oldSvc.API.Endpoint) {
550
+ return errors.New("cannot change remote pinning services api info with `config replace`")
551
+ }
552
+ // re-apply API details and store service in updated config
553
+ newSvc.API = oldSvc.API
554
+ newCfg.Pinning.RemoteServices[name] = newSvc
555
+ } else {
556
+ // error on service rm attempt
557
+ return errors.New("cannot add or remove remote pinning services with 'config replace'")
558
+ }
559
+ }
560
+
561
+ return r.SetConfig(&newCfg)
562
+}
563
+
564
+func getRemotePinningServices(r repo.Repo) (map[string]config.RemotePinningService, error) {
565
+ var oldServices map[string]config.RemotePinningService
566
+ if remoteServicesTag, err := getConfig(r, config.RemoteServicesPath); err == nil {
567
// seems that golang cannot type assert map[string]interface{} to map[string]config.RemotePinningService
568
// so we have to manually copy the data :-|
569
if val, ok := remoteServicesTag.Value.(map[string]interface{}); ok {
541
- var services map[string]config.RemotePinningService
570
jsonString, err := json.Marshal(val)
571
if err != nil {
544
- return fmt.Errorf("failed to replace config while preserving %s: %s", config.RemoteServicesSelector, err)
572
+ return nil, err
573
}
546
- err = json.Unmarshal(jsonString, &services)
574
+ err = json.Unmarshal(jsonString, &oldServices)
575
if err != nil {
548
- return fmt.Errorf("failed to replace config while preserving %s: %s", config.RemoteServicesSelector, err)
576
+ return nil, err
577
}
550
- // .. if so, apply them on top of the new config
551
- cfg.Pinning.RemoteServices = services
578
}
579
}
580
+ return oldServices, nil
581
555
- return r.SetConfig(&cfg)
582
}
core/commands/config_test.go
new
+17
@@ -0,0 +1,17 @@
1
+package commands
2
+
3
+import "testing"
4
+
5
+func TestScrubMapInternalDelete(t *testing.T) {
6
+ m, err := scrubMapInternal(nil, nil, true)
7
+ if err != nil {
8
+ t.Error(err)
9
+ }
10
+ if m == nil {
11
+ t.Errorf("expecting an empty map, got nil")
12
+ }
13
+ if len(m) != 0 {
14
+ t.Errorf("expecting an empty map, got a non-empty map")
15
+
16
+ }
17
+}
core/commands/pin/remotepin.go
+5
-4
@@ -470,10 +470,11 @@ TIP:
470
}
471
472
cfg.Pinning.RemoteServices[name] = config.RemotePinningService{
473
- Api: config.RemotePinningServiceApi{
473
+ API: config.RemotePinningServiceAPI{
474
Endpoint: endpoint,
475
Key: key,
476
},
477
+ Policies: config.RemotePinningServicePolicies{},
478
}
479
480
return repo.SetConfig(cfg)
@@ -562,7 +563,7 @@ TIP: pass '--enc=json' for more useful JSON output.
563
services := cfg.Pinning.RemoteServices
564
result := PinServicesList{make([]ServiceDetails, 0, len(services))}
565
for svcName, svcConfig := range services {
565
- svcDetails := ServiceDetails{svcName, svcConfig.Api.Endpoint, nil}
566
+ svcDetails := ServiceDetails{svcName, svcConfig.API.Endpoint, nil}
567
568
// if --pin-count is passed, we try to fetch pin numbers from remote service
569
if req.Options[pinServiceStatOptionName].(bool) {
@@ -738,11 +739,11 @@ func getRemotePinServiceInfo(env cmds.Environment, name string) (endpoint, key s
739
if !present {
740
return "", "", fmt.Errorf("service not known")
741
}
741
- endpoint, err = normalizeEndpoint(service.Api.Endpoint)
742
+ endpoint, err = normalizeEndpoint(service.API.Endpoint)
743
if err != nil {
744
return "", "", err
745
}
745
- return endpoint, service.Api.Key, nil
746
+ return endpoint, service.API.Key, nil
747
}
748
749
func normalizeEndpoint(endpoint string) (string, error) {
go.mod
+1
-1
@@ -32,7 +32,7 @@ require (
32
github.com/ipfs/go-ipfs-blockstore v0.1.4
33
github.com/ipfs/go-ipfs-chunker v0.0.5
34
github.com/ipfs/go-ipfs-cmds v0.6.0
35
- github.com/ipfs/go-ipfs-config v0.11.0
35
+ github.com/ipfs/go-ipfs-config v0.12.0
36
github.com/ipfs/go-ipfs-ds-help v0.1.1
37
github.com/ipfs/go-ipfs-exchange-interface v0.0.1
38
github.com/ipfs/go-ipfs-exchange-offline v0.0.1
go.sum
+2
-2
@@ -329,8 +329,8 @@ github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7Na
329
github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8=
330
github.com/ipfs/go-ipfs-cmds v0.6.0 h1:yAxdowQZzoFKjcLI08sXVNnqVj3jnABbf9smrPQmBsw=
331
github.com/ipfs/go-ipfs-cmds v0.6.0/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk=
332
-github.com/ipfs/go-ipfs-config v0.11.0 h1:w4t2pz415Gtg6MTUKAq06C7ezC59/Us+k3+n1Tje+wg=
333
-github.com/ipfs/go-ipfs-config v0.11.0/go.mod h1:Ei/FLgHGTdPyqCPK0oPCwGTe8VSnsjJjx7HZqUb6Ry0=
332
+github.com/ipfs/go-ipfs-config v0.12.0 h1:wxqN3ohBlis1EkhkzIKuF+XLx4YNn9rNpiSOYw3DFZc=
333
+github.com/ipfs/go-ipfs-config v0.12.0/go.mod h1:Ei/FLgHGTdPyqCPK0oPCwGTe8VSnsjJjx7HZqUb6Ry0=
334
github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
335
github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ=
336
github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
test/sharness/t0700-remotepin.sh
+60
-21
@@ -34,7 +34,8 @@ test_expect_success "creating test user on remote pinning service" '
34
ipfs pin remote service add test_pin_svc ${TEST_PIN_SVC} ${TEST_PIN_SVC_KEY} &&
35
ipfs pin remote service add test_invalid_key_svc ${TEST_PIN_SVC} fake_api_key &&
36
ipfs pin remote service add test_invalid_url_path_svc ${TEST_PIN_SVC}/invalid-path fake_api_key &&
37
- ipfs pin remote service add test_invalid_url_dns_svc https://invalid-service.example.com fake_api_key
37
+ ipfs pin remote service add test_invalid_url_dns_svc https://invalid-service.example.com fake_api_key &&
38
+ ipfs pin remote service add test_pin_mfs_svc ${TEST_PIN_SVC} ${TEST_PIN_SVC_KEY}
39
'
40
41
# add a service with a invalid endpoint
@@ -51,23 +52,60 @@ test_expect_success "test 'ipfs pin remote service ls'" '
52
grep -q test_invalid_url_dns_svc ls_out
53
'
54
54
-# SECURITY of access tokens in Api.Key fields:
55
-# Pinning.RemoteServices includes Api.Key, and we give it the same treatment
55
+test_expect_success "test enabling mfs pinning" '
56
+ ipfs config --json Pinning.RemoteServices.test_pin_mfs_svc.Policies.MFS.RepinInterval \"10s\" &&
57
+ ipfs config --json Pinning.RemoteServices.test_pin_mfs_svc.Policies.MFS.PinName \"mfs_test_pin\" &&
58
+ ipfs config --json Pinning.RemoteServices.test_pin_mfs_svc.Policies.MFS.Enable true &&
59
+ ipfs config --json Pinning.RemoteServices.test_pin_mfs_svc.Policies.MFS.RepinInterval > repin_interval &&
60
+ ipfs config --json Pinning.RemoteServices.test_pin_mfs_svc.Policies.MFS.PinName > pin_name &&
61
+ ipfs config --json Pinning.RemoteServices.test_pin_mfs_svc.Policies.MFS.Enable > enable &&
62
+ echo 10s > expected_repin_interval &&
63
+ echo mfs_test_pin > expected_pin_name &&
64
+ echo true > expected_enable &&
65
+ test_cmp repin_interval expected_repin_interval &&
66
+ test_cmp pin_name expected_pin_name &&
67
+ test_cmp enable expected_enable
68
+'
69
+
70
+# expect PIN to be created
71
+test_expect_success "verify MFS root is being pinned" '
72
+ ipfs files cp /ipfs/bafkqaaa /mfs-pinning-test-$(date +%s.%N) &&
73
+ ipfs files flush &&
74
+ sleep 31 &&
75
+ ipfs files stat / --enc=json | jq -r .Hash > mfs_cid &&
76
+ ipfs pin remote ls --service=test_pin_mfs_svc --name=mfs_test_pin --status=queued,pinning,pinned,failed --enc=json | tee ls_out | jq -r .Cid > pin_cid &&
77
+ cat mfs_cid ls_out &&
78
+ test_cmp mfs_cid pin_cid
79
+'
80
+
81
+# expect existing PIN to be replaced
82
+test_expect_success "verify MFS root is being repinned on CID change" '
83
+ ipfs files cp /ipfs/bafkqaaa /mfs-pinning-repin-test-$(date +%s.%N) &&
84
+ ipfs files flush &&
85
+ sleep 31 &&
86
+ ipfs files stat / --enc=json | jq -r .Hash > mfs_cid &&
87
+ ipfs pin remote ls --service=test_pin_mfs_svc --name=mfs_test_pin --status=queued,pinning,pinned,failed --enc=json | tee ls_out | jq -r .Cid > pin_cid &&
88
+ cat mfs_cid ls_out &&
89
+ test_cmp mfs_cid pin_cid
90
+'
91
+
92
+# SECURITY of access tokens in API.Key fields:
93
+# Pinning.RemoteServices includes API.Key, and we give it the same treatment
94
# as Identity.PrivKey to prevent exposing it on the network
95
96
test_expect_success "'ipfs config Pinning' fails" '
97
test_expect_code 1 ipfs config Pinning 2>&1 > config_out
98
'
61
-test_expect_success "output does not include Api.Key" '
99
+test_expect_success "output does not include API.Key" '
100
test_expect_code 1 grep -q Key config_out
101
'
102
65
-test_expect_success "'ipfs config Pinning.RemoteServices.test_pin_svc.Api.Key' fails" '
66
- test_expect_code 1 ipfs config Pinning.RemoteServices.test_pin_svc.Api.Key 2> config_out
103
+test_expect_success "'ipfs config Pinning.RemoteServices.test_pin_svc.API.Key' fails" '
104
+ test_expect_code 1 ipfs config Pinning.RemoteServices.test_pin_svc.API.Key 2> config_out
105
'
106
107
test_expect_success "output includes meaningful error" '
70
- echo "Error: cannot show or change pinning services through this API (try: ipfs pin remote service --help)" > config_exp &&
108
+ echo "Error: cannot show or change pinning services credentials" > config_exp &&
109
test_cmp config_exp config_out
110
'
111
@@ -78,29 +116,30 @@ test_expect_success "output includes meaningful error" '
116
test_cmp config_exp config_out
117
'
118
81
-test_expect_success "'ipfs config show' doesn't include RemoteServices" '
82
- ipfs config show > show_config &&
83
- test_expect_code 1 grep RemoteServices show_config
119
+test_expect_success "'ipfs config show' does not include Pinning.RemoteServices[*].API.Key" '
120
+ ipfs config show | tee show_config | jq -r .Pinning.RemoteServices > remote_services &&
121
+ test_expect_code 1 grep \"Key\" remote_services &&
122
+ test_expect_code 1 grep fake_api_key show_config &&
123
+ test_expect_code 1 grep "$TEST_PIN_SVC_KEY" show_config
124
'
125
86
-test_expect_success "'ipfs config replace' injects remote services back" '
87
- test_expect_code 1 grep -q -E "test_.+_svc" show_config &&
126
+test_expect_success "'ipfs config replace' injects Pinning.RemoteServices[*].API.Key back" '
127
+ test_expect_code 1 grep fake_api_key show_config &&
128
+ test_expect_code 1 grep "$TEST_PIN_SVC_KEY" show_config &&
129
ipfs config replace show_config &&
89
- test_expect_code 0 grep -q test_pin_svc "$IPFS_PATH/config" &&
90
- test_expect_code 0 grep -q test_invalid_key_svc "$IPFS_PATH/config" &&
91
- test_expect_code 0 grep -q test_invalid_url_path_svc "$IPFS_PATH/config" &&
92
- test_expect_code 0 grep -q test_invalid_url_dns_svc "$IPFS_PATH/config"
130
+ test_expect_code 0 grep fake_api_key "$IPFS_PATH/config" &&
131
+ test_expect_code 0 grep "$TEST_PIN_SVC_KEY" "$IPFS_PATH/config"
132
'
133
134
# note: we remove Identity.PrivKey to ensure error is triggered by Pinning.RemoteServices
96
-test_expect_success "'ipfs config replace' with remote services errors out" '
97
- jq -M "del(.Identity.PrivKey)" "$IPFS_PATH/config" | jq ".Pinning += { RemoteServices: {\"foo\": {} }}" > new_config &&
135
+test_expect_success "'ipfs config replace' with Pinning.RemoteServices[*].API.Key errors out" '
136
+ jq -M "del(.Identity.PrivKey)" "$IPFS_PATH/config" | jq ".Pinning += { RemoteServices: {\"myservice\": {\"API\": {\"Endpoint\": \"https://example.com/psa\", \"Key\": \"mysecret\"}}}}" > new_config &&
137
test_expect_code 1 ipfs config replace - < new_config 2> replace_out
138
'
100
-test_expect_success "output includes meaningful error" '
101
- echo "Error: cannot show or change pinning services through this API (try: ipfs pin remote service --help)" > replace_expected &&
139
+test_expect_success "output includes meaningful error" "
140
+ echo \"Error: cannot add or remove remote pinning services with 'config replace'\" > replace_expected &&
141
test_cmp replace_out replace_expected
103
-'
142
+"
143
144
# /SECURITY
145