master
go 461 lines 16.4 KB
Raw
1 package cli
2
3 import (
4 "errors"
5 "fmt"
6 "net"
7 "net/http"
8 "testing"
9 "time"
10
11 "github.com/google/uuid"
12 "github.com/ipfs/go-test/random"
13 "github.com/ipfs/kubo/test/cli/harness"
14 "github.com/ipfs/kubo/test/cli/testutils/pinningservice"
15 "github.com/stretchr/testify/assert"
16 "github.com/stretchr/testify/require"
17 "github.com/tidwall/gjson"
18 "github.com/tidwall/sjson"
19 )
20
21 func runPinningService(t *testing.T, authToken string) (*pinningservice.PinningService, string) {
22 svc := pinningservice.New()
23 router := pinningservice.NewRouter(authToken, svc)
24 server := &http.Server{Handler: router}
25 listener, err := net.Listen("tcp", "127.0.0.1:0")
26 require.NoError(t, err)
27 go func() {
28 err := server.Serve(listener)
29 if err != nil && !errors.Is(err, net.ErrClosed) && !errors.Is(err, http.ErrServerClosed) {
30 t.Logf("Serve error: %s", err)
31 }
32 }()
33 t.Cleanup(func() { listener.Close() })
34
35 return svc, fmt.Sprintf("http://%s/api/v1", listener.Addr().String())
36 }
37
38 func TestRemotePinning(t *testing.T) {
39 t.Parallel()
40 authToken := "testauthtoken"
41
42 t.Run("MFS pinning", func(t *testing.T) {
43 t.Parallel()
44 node := harness.NewT(t).NewNode().Init()
45 node.Runner.Env["MFS_PIN_POLL_INTERVAL"] = "10ms"
46
47 _, svcURL := runPinningService(t, authToken)
48 node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
49 node.IPFS("config", "--json", "Pinning.RemoteServices.svc.Policies.MFS.RepinInterval", `"1s"`)
50 node.IPFS("config", "--json", "Pinning.RemoteServices.svc.Policies.MFS.PinName", `"test_pin"`)
51 node.IPFS("config", "--json", "Pinning.RemoteServices.svc.Policies.MFS.Enable", "true")
52
53 node.StartDaemon()
54 t.Cleanup(func() { node.StopDaemon() })
55
56 node.IPFS("files", "cp", "/ipfs/bafkqaaa", "/mfs-pinning-test-"+uuid.NewString())
57 node.IPFS("files", "flush")
58 res := node.IPFS("files", "stat", "/", "--enc=json")
59 hash := gjson.Get(res.Stdout.String(), "Hash").Str
60
61 assert.Eventually(t,
62 func() bool {
63 res = node.IPFS("pin", "remote", "ls",
64 "--service=svc",
65 "--name=test_pin",
66 "--status=queued,pinning,pinned,failed",
67 "--enc=json",
68 )
69 pinnedHash := gjson.Get(res.Stdout.String(), "Cid").Str
70 return hash == pinnedHash
71 },
72 10*time.Second,
73 10*time.Millisecond,
74 )
75
76 t.Run("MFS root is repinned on CID change", func(t *testing.T) {
77 node.IPFS("files", "cp", "/ipfs/bafkqaaa", "/mfs-pinning-repin-test-"+uuid.NewString())
78 node.IPFS("files", "flush")
79 res = node.IPFS("files", "stat", "/", "--enc=json")
80 hash := gjson.Get(res.Stdout.String(), "Hash").Str
81 assert.Eventually(t,
82 func() bool {
83 res := node.IPFS("pin", "remote", "ls",
84 "--service=svc",
85 "--name=test_pin",
86 "--status=queued,pinning,pinned,failed",
87 "--enc=json",
88 )
89 pinnedHash := gjson.Get(res.Stdout.String(), "Cid").Str
90 return hash == pinnedHash
91 },
92 10*time.Second,
93 10*time.Millisecond,
94 )
95 })
96 })
97
98 // Pinning.RemoteServices includes API.Key, so we give it the same treatment
99 // as Identity,PrivKey to prevent exposing it on the network
100 t.Run("access token security", func(t *testing.T) {
101 t.Parallel()
102 node := harness.NewT(t).NewNode().Init()
103 node.IPFS("pin", "remote", "service", "add", "1", "http://example1.com", "testkey")
104 res := node.RunIPFS("config", "Pinning")
105 assert.Equal(t, 1, res.ExitCode())
106 assert.Contains(t, res.Stderr.String(), "cannot show or change pinning services credentials")
107 assert.NotContains(t, res.Stdout.String(), "testkey")
108
109 res = node.RunIPFS("config", "Pinning.RemoteServices.1.API.Key")
110 assert.Equal(t, 1, res.ExitCode())
111 assert.Contains(t, res.Stderr.String(), "cannot show or change pinning services credentials")
112 assert.NotContains(t, res.Stdout.String(), "testkey")
113
114 configShow := node.RunIPFS("config", "show").Stdout.String()
115 assert.NotContains(t, configShow, "testkey")
116
117 t.Run("re-injecting config with 'ipfs config replace' preserves the API keys", func(t *testing.T) {
118 node.WriteBytes("config-show", []byte(configShow))
119 node.IPFS("config", "replace", "config-show")
120 assert.Contains(t, node.ReadFile(node.ConfigFile()), "testkey")
121 })
122
123 t.Run("injecting config with 'ipfs config replace' with API keys returns an error", func(t *testing.T) {
124 // remove Identity.PrivKey to ensure error is triggered by Pinning.RemoteServices
125 configJSON := MustVal(sjson.Delete(configShow, "Identity.PrivKey"))
126 configJSON = MustVal(sjson.Set(configJSON, "Pinning.RemoteServices.1.API.Key", "testkey"))
127 node.WriteBytes("new-config", []byte(configJSON))
128 res := node.RunIPFS("config", "replace", "new-config")
129 assert.Equal(t, 1, res.ExitCode())
130 assert.Contains(t, res.Stderr.String(), "cannot change remote pinning services api info with `config replace`")
131 })
132 })
133
134 t.Run("pin remote service ls --stat", func(t *testing.T) {
135 t.Parallel()
136 node := harness.NewT(t).NewNode().Init().StartDaemon()
137 defer node.StopDaemon()
138
139 _, svcURL := runPinningService(t, authToken)
140
141 node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
142 node.IPFS("pin", "remote", "service", "add", "invalid-svc", svcURL+"/invalidpath", authToken)
143
144 res := node.IPFS("pin", "remote", "service", "ls", "--stat")
145 assert.Contains(t, res.Stdout.String(), " 0/0/0/0")
146
147 stats := node.IPFS("pin", "remote", "service", "ls", "--stat", "--enc=json").Stdout.String()
148 assert.Equal(t, "valid", gjson.Get(stats, `RemoteServices.#(Service == "svc").Stat.Status`).Str)
149 assert.Equal(t, "invalid", gjson.Get(stats, `RemoteServices.#(Service == "invalid-svc").Stat.Status`).Str)
150
151 // no --stat returns no stat obj
152 t.Run("no --stat returns no stat obj", func(t *testing.T) {
153 res := node.IPFS("pin", "remote", "service", "ls", "--enc=json")
154 assert.False(t, gjson.Get(res.Stdout.String(), `RemoteServices.#(Service == "svc").Stat`).Exists())
155 })
156 })
157
158 t.Run("adding service with invalid URL fails", func(t *testing.T) {
159 t.Parallel()
160 node := harness.NewT(t).NewNode().Init().StartDaemon()
161 defer node.StopDaemon()
162
163 res := node.RunIPFS("pin", "remote", "service", "add", "svc", "invalid-service.example.com", "key")
164 assert.Equal(t, 1, res.ExitCode())
165 assert.Contains(t, res.Stderr.String(), "service endpoint must be a valid HTTP URL")
166
167 res = node.RunIPFS("pin", "remote", "service", "add", "svc", "xyz://invalid-service.example.com", "key")
168 assert.Equal(t, 1, res.ExitCode())
169 assert.Contains(t, res.Stderr.String(), "service endpoint must be a valid HTTP URL")
170 })
171
172 t.Run("unauthorized pinning service calls fail", func(t *testing.T) {
173 t.Parallel()
174 node := harness.NewT(t).NewNode().Init().StartDaemon()
175 defer node.StopDaemon()
176 _, svcURL := runPinningService(t, authToken)
177
178 node.IPFS("pin", "remote", "service", "add", "svc", svcURL, "othertoken")
179
180 res := node.RunIPFS("pin", "remote", "ls", "--service=svc")
181 assert.Equal(t, 1, res.ExitCode())
182 assert.Contains(t, res.Stderr.String(), "access denied")
183 })
184
185 t.Run("pinning service calls fail when there is a wrong path", func(t *testing.T) {
186 t.Parallel()
187 node := harness.NewT(t).NewNode().Init().StartDaemon()
188 defer node.StopDaemon()
189 _, svcURL := runPinningService(t, authToken)
190 node.IPFS("pin", "remote", "service", "add", "svc", svcURL+"/invalid-path", authToken)
191
192 res := node.RunIPFS("pin", "remote", "ls", "--service=svc")
193 assert.Equal(t, 1, res.ExitCode())
194 assert.Contains(t, res.Stderr.String(), "404")
195 })
196
197 t.Run("pinning service calls fail when DNS resolution fails", func(t *testing.T) {
198 t.Parallel()
199 node := harness.NewT(t).NewNode().Init().StartDaemon()
200 defer node.StopDaemon()
201 node.IPFS("pin", "remote", "service", "add", "svc", "https://invalid-service.example.com", authToken)
202
203 res := node.RunIPFS("pin", "remote", "ls", "--service=svc")
204 assert.Equal(t, 1, res.ExitCode())
205 assert.Contains(t, res.Stderr.String(), "no such host")
206 })
207
208 t.Run("pin remote service rm", func(t *testing.T) {
209 t.Parallel()
210 node := harness.NewT(t).NewNode().Init().StartDaemon()
211 defer node.StopDaemon()
212 node.IPFS("pin", "remote", "service", "add", "svc", "https://example.com", authToken)
213 node.IPFS("pin", "remote", "service", "rm", "svc")
214 res := node.IPFS("pin", "remote", "service", "ls")
215 assert.NotContains(t, res.Stdout.String(), "svc")
216 })
217
218 t.Run("remote pinning", func(t *testing.T) {
219 t.Parallel()
220
221 verifyStatus := func(node *harness.Node, name, hash, status string) {
222 resJSON := node.IPFS("pin", "remote", "ls",
223 "--service=svc",
224 "--enc=json",
225 "--name="+name,
226 "--status="+status,
227 ).Stdout.String()
228
229 assert.Equal(t, status, gjson.Get(resJSON, "Status").Str)
230 assert.Equal(t, hash, gjson.Get(resJSON, "Cid").Str)
231 assert.Equal(t, name, gjson.Get(resJSON, "Name").Str)
232 }
233
234 t.Run("'ipfs pin remote add --background=true'", func(t *testing.T) {
235 node := harness.NewT(t).NewNode().Init().StartDaemon()
236 defer node.StopDaemon()
237 svc, svcURL := runPinningService(t, authToken)
238 node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
239
240 // retain a ptr to the pin that's in the DB so we can directly mutate its status
241 // to simulate async work
242 pinCh := make(chan *pinningservice.PinStatus, 1)
243 svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) {
244 pinCh <- pin
245 }
246
247 hash := node.IPFSAddStr("foo")
248 node.IPFS("pin", "remote", "add",
249 "--background=true",
250 "--service=svc",
251 "--name=pin1",
252 hash,
253 )
254
255 pin := <-pinCh
256
257 transitionStatus := func(status string) {
258 pin.M.Lock()
259 pin.Status = status
260 pin.M.Unlock()
261 }
262
263 verifyStatus(node, "pin1", hash, "queued")
264
265 transitionStatus("pinning")
266 verifyStatus(node, "pin1", hash, "pinning")
267
268 transitionStatus("pinned")
269 verifyStatus(node, "pin1", hash, "pinned")
270
271 transitionStatus("failed")
272 verifyStatus(node, "pin1", hash, "failed")
273 })
274
275 t.Run("'ipfs pin remote add --background=false'", func(t *testing.T) {
276 t.Parallel()
277 node := harness.NewT(t).NewNode().Init().StartDaemon()
278 defer node.StopDaemon()
279 svc, svcURL := runPinningService(t, authToken)
280 node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
281
282 svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) {
283 pin.M.Lock()
284 defer pin.M.Unlock()
285 pin.Status = "pinned"
286 }
287 hash := node.IPFSAddStr("foo")
288 node.IPFS("pin", "remote", "add",
289 "--background=false",
290 "--service=svc",
291 "--name=pin2",
292 hash,
293 )
294 verifyStatus(node, "pin2", hash, "pinned")
295 })
296
297 t.Run("'ipfs pin remote ls' with multiple statuses", func(t *testing.T) {
298 t.Parallel()
299 node := harness.NewT(t).NewNode().Init().StartDaemon()
300 defer node.StopDaemon()
301 svc, svcURL := runPinningService(t, authToken)
302 node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
303
304 hash := node.IPFSAddStr("foo")
305 desiredStatuses := map[string]string{
306 "pin-queued": "queued",
307 "pin-pinning": "pinning",
308 "pin-pinned": "pinned",
309 "pin-failed": "failed",
310 }
311 var pins []*pinningservice.PinStatus
312 svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) {
313 pin.M.Lock()
314 defer pin.M.Unlock()
315 pins = append(pins, pin)
316 // this must be "pinned" for the 'pin remote add' command to return
317 // after 'pin remote add', we change the status to its real status
318 pin.Status = "pinned"
319 }
320
321 for pinName := range desiredStatuses {
322 node.IPFS("pin", "remote", "add",
323 "--service=svc",
324 "--name="+pinName,
325 hash,
326 )
327 }
328 for _, pin := range pins {
329 pin.M.Lock()
330 pin.Status = desiredStatuses[pin.Pin.Name]
331 pin.M.Unlock()
332 }
333
334 res := node.IPFS("pin", "remote", "ls",
335 "--service=svc",
336 "--status=queued,pinning,pinned,failed",
337 "--enc=json",
338 )
339 actualStatuses := map[string]string{}
340 for _, line := range res.Stdout.Lines() {
341 name := gjson.Get(line, "Name").Str
342 status := gjson.Get(line, "Status").Str
343 // drop statuses of other pins we didn't add
344 if _, ok := desiredStatuses[name]; ok {
345 actualStatuses[name] = status
346 }
347 }
348 assert.Equal(t, desiredStatuses, actualStatuses)
349 })
350
351 t.Run("'ipfs pin remote ls' by CID", func(t *testing.T) {
352 t.Parallel()
353 node := harness.NewT(t).NewNode().Init().StartDaemon()
354 defer node.StopDaemon()
355 svc, svcURL := runPinningService(t, authToken)
356 node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
357
358 transitionedCh := make(chan struct{}, 1)
359 svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) {
360 pin.M.Lock()
361 defer pin.M.Unlock()
362 pin.Status = "pinned"
363 transitionedCh <- struct{}{}
364 }
365 hash := node.IPFSAddStr(string(random.Bytes(1000)))
366 node.IPFS("pin", "remote", "add", "--background=false", "--service=svc", hash)
367 <-transitionedCh
368 res := node.IPFS("pin", "remote", "ls", "--service=svc", "--cid="+hash, "--enc=json").Stdout.String()
369 assert.Contains(t, res, hash)
370 })
371
372 t.Run("'ipfs pin remote rm --name' without --force when multiple pins match", func(t *testing.T) {
373 t.Parallel()
374 node := harness.NewT(t).NewNode().Init().StartDaemon()
375 defer node.StopDaemon()
376 svc, svcURL := runPinningService(t, authToken)
377 node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
378
379 svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) {
380 pin.M.Lock()
381 defer pin.M.Unlock()
382 pin.Status = "pinned"
383 }
384 hash := node.IPFSAddStr(string(random.Bytes(1000)))
385 node.IPFS("pin", "remote", "add", "--service=svc", "--name=force-test-name", hash)
386 node.IPFS("pin", "remote", "add", "--service=svc", "--name=force-test-name", hash)
387
388 t.Run("fails", func(t *testing.T) {
389 res := node.RunIPFS("pin", "remote", "rm", "--service=svc", "--name=force-test-name")
390 assert.Equal(t, 1, res.ExitCode())
391 assert.Contains(t, res.Stderr.String(), "Error: multiple remote pins are matching this query, add --force to confirm the bulk removal")
392 })
393
394 t.Run("matching pins are not removed", func(t *testing.T) {
395 lines := node.IPFS("pin", "remote", "ls", "--service=svc", "--name=force-test-name").Stdout.Lines()
396 assert.Contains(t, lines[0], "force-test-name")
397 assert.Contains(t, lines[1], "force-test-name")
398 })
399 })
400
401 t.Run("'ipfs pin remote rm --name --force' remove multiple pins", func(t *testing.T) {
402 t.Parallel()
403 node := harness.NewT(t).NewNode().Init().StartDaemon()
404 defer node.StopDaemon()
405 svc, svcURL := runPinningService(t, authToken)
406 node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
407
408 svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) {
409 pin.M.Lock()
410 defer pin.M.Unlock()
411 pin.Status = "pinned"
412 }
413 hash := node.IPFSAddStr(string(random.Bytes(1000)))
414 node.IPFS("pin", "remote", "add", "--service=svc", "--name=force-test-name", hash)
415 node.IPFS("pin", "remote", "add", "--service=svc", "--name=force-test-name", hash)
416
417 node.IPFS("pin", "remote", "rm", "--service=svc", "--name=force-test-name", "--force")
418 out := node.IPFS("pin", "remote", "ls", "--service=svc", "--name=force-test-name").Stdout.Trimmed()
419 assert.Empty(t, out)
420 })
421
422 t.Run("'ipfs pin remote rm --force' removes all pins", func(t *testing.T) {
423 t.Parallel()
424 node := harness.NewT(t).NewNode().Init().StartDaemon()
425 defer node.StopDaemon()
426 svc, svcURL := runPinningService(t, authToken)
427 node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
428
429 svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) {
430 pin.M.Lock()
431 defer pin.M.Unlock()
432 pin.Status = "pinned"
433 }
434 for i := range 4 {
435 hash := node.IPFSAddStr(string(random.Bytes(1000)))
436 name := fmt.Sprintf("--name=%d", i)
437 node.IPFS("pin", "remote", "add", "--service=svc", "--name="+name, hash)
438 }
439
440 lines := node.IPFS("pin", "remote", "ls", "--service=svc").Stdout.Lines()
441 assert.Len(t, lines, 4)
442
443 node.IPFS("pin", "remote", "rm", "--service=svc", "--force")
444
445 lines = node.IPFS("pin", "remote", "ls", "--service=svc").Stdout.Lines()
446 assert.Len(t, lines, 0)
447 })
448 })
449
450 t.Run("'ipfs pin remote add' shows a warning message when offline", func(t *testing.T) {
451 t.Parallel()
452 node := harness.NewT(t).NewNode().Init()
453 _, svcURL := runPinningService(t, authToken)
454 node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
455
456 hash := node.IPFSAddStr(string(random.Bytes(1000)))
457 res := node.IPFS("pin", "remote", "add", "--service=svc", "--background", hash)
458 warningMsg := "WARNING: the local node is offline and remote pinning may fail if there is no other provider for this CID"
459 assert.Contains(t, res.Stdout.String(), warningMsg)
460 })
461 }