test: port remote pinning tests to Go (#9720)
This also means that rb-pinning-service-api is no longer required for running remote pinning tests. This alone saves at least 3 minutes in test runtime in CI because we don't need to checkout the repo, build the Docker image, run it, etc. Instead this implements a simple pinning service in Go that the test runs in-process, with a callback that can be used to control the async behavior of the pinning service (e.g. simulate work happening asynchronously like transitioning from "queued" -> "pinning" -> "pinned"). This also adds an environment variable to Kubo to control the MFS remote pin polling interval, so that we don't have to wait 30 seconds in the test for MFS changes to be repinned. This is purely for tests so I don't think we should document this. This entire test suite runs in around 2.5 sec on my laptop, compared to the existing 3+ minutes in CI.
Gus Eggert committed
Mar 30, 2023 at 07:46 UTC
a24cfb89a509aa3a8dd95be363f2cbb2d4c8e692
9 files changed
+901
-347
.github/workflows/sharness.yml
-14
@@ -29,24 +29,10 @@ jobs:
29
path: kubo
30
- name: Install missing tools
31
run: sudo apt install -y socat net-tools fish libxml2-utils
32
- - name: Checkout IPFS Pinning Service API
33
- uses: actions/checkout@v3
34
- with:
35
- repository: ipfs-shipyard/rb-pinning-service-api
36
- ref: 773c3adbb421c551d2d89288abac3e01e1f7c3a8
37
- path: rb-pinning-service-api
38
- # TODO: check if docker compose (not docker-compose) is available on default gh runners
39
- - name: Start IPFS Pinning Service API
40
- run: |
41
- (for i in {1..3}; do docker compose pull && break || sleep 5; done) &&
42
- docker compose up -d
43
- working-directory: rb-pinning-service-api
32
- name: Restore Go Cache
33
uses: protocol/cache-go-action@v1
34
with:
35
name: ${{ github.job }}
48
- - name: Find IPFS Pinning Service API address
49
- run: echo "TEST_DOCKER_HOST=$(ip -4 addr show docker0 | grep -Po 'inet \K[\d.]+')" >> $GITHUB_ENV
36
- uses: actions/cache@v3
37
with:
38
path: test/sharness/lib/dependencies
cmd/ipfs/pinmfs.go
+14
-1
@@ -3,6 +3,7 @@ package main
3
import (
4
"context"
5
"fmt"
6
+ "os"
7
"time"
8
9
"github.com/libp2p/go-libp2p/core/host"
@@ -31,7 +32,19 @@ func (x lastPin) IsValid() bool {
32
return x != lastPin{}
33
}
34
34
-const daemonConfigPollInterval = time.Minute / 2
35
+var daemonConfigPollInterval = time.Minute / 2
36
+
37
+func init() {
38
+ // this environment variable is solely for testing, use at your own risk
39
+ if pollDurStr := os.Getenv("MFS_PIN_POLL_INTERVAL"); pollDurStr != "" {
40
+ d, err := time.ParseDuration(pollDurStr)
41
+ if err != nil {
42
+ mfslog.Error("error parsing MFS_PIN_POLL_INTERVAL, using default:", err)
43
+ }
44
+ daemonConfigPollInterval = d
45
+ }
46
+}
47
+
48
const defaultRepinInterval = 5 * time.Minute
49
50
type pinMFSContext interface {
go.mod
+5
@@ -43,6 +43,7 @@ require (
43
github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
44
github.com/jbenet/go-temp-err-catcher v0.1.0
45
github.com/jbenet/goprocess v0.1.4
46
+ github.com/julienschmidt/httprouter v1.3.0
47
github.com/libp2p/go-doh-resolver v0.4.0
48
github.com/libp2p/go-libp2p v0.26.4
49
github.com/libp2p/go-libp2p-http v0.4.0
@@ -67,6 +68,8 @@ require (
68
github.com/prometheus/client_golang v1.14.0
69
github.com/stretchr/testify v1.8.2
70
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
71
+ github.com/tidwall/gjson v1.14.4
72
+ github.com/tidwall/sjson v1.2.5
73
github.com/whyrusleeping/go-sysinfo v0.0.0-20190219211824-4a357d4b90b1
74
github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7
75
go.opencensus.io v0.24.0
@@ -198,6 +201,8 @@ require (
201
github.com/samber/lo v1.36.0 // indirect
202
github.com/spaolacci/murmur3 v1.1.0 // indirect
203
github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e // indirect
204
+ github.com/tidwall/match v1.1.1 // indirect
205
+ github.com/tidwall/pretty v1.2.0 // indirect
206
github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect
207
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
208
github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect
go.sum
+10
@@ -495,6 +495,7 @@ github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVY
495
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
496
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
497
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
498
+github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
499
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
500
github.com/kami-zh/go-capturer v0.0.0-20171211120116-e492ea43421d/go.mod h1:P2viExyCEfeWGU259JnaQ34Inuec4R38JCyBx2edgD0=
501
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
@@ -858,6 +859,15 @@ github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cb
859
github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e h1:T5PdfK/M1xyrHwynxMIVMWLS7f/qHwfslZphxtGnw7s=
860
github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g=
861
github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M=
862
+github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
863
+github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
864
+github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
865
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
866
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
867
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
868
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
869
+github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
870
+github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
871
github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
872
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
873
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
test/cli/harness/node.go
+17
@@ -76,6 +76,23 @@ func (n *Node) WriteBytes(filename string, b []byte) {
76
}
77
}
78
79
+// ReadFile reads the specific file. If it is relative, it is relative the node's root dir.
80
+func (n *Node) ReadFile(filename string) string {
81
+ f := filename
82
+ if !filepath.IsAbs(filename) {
83
+ f = filepath.Join(n.Dir, filename)
84
+ }
85
+ b, err := os.ReadFile(f)
86
+ if err != nil {
87
+ panic(err)
88
+ }
89
+ return string(b)
90
+}
91
+
92
+func (n *Node) ConfigFile() string {
93
+ return filepath.Join(n.Dir, "config")
94
+}
95
+
96
func (n *Node) ReadConfig() *config.Config {
97
cfg, err := serial.Load(filepath.Join(n.Dir, "config"))
98
if err != nil {
test/cli/must.go
new
+8
@@ -0,0 +1,8 @@
1
+package cli
2
+
3
+func MustVal[V any](val V, err error) V {
4
+ if err != nil {
5
+ panic(err)
6
+ }
7
+ return val
8
+}
test/cli/pinning_remote_test.go
new
+446
@@ -0,0 +1,446 @@
1
+package cli
2
+
3
+import (
4
+ "errors"
5
+ "fmt"
6
+ "net"
7
+ "net/http"
8
+ "testing"
9
+ "time"
10
+
11
+ "github.com/google/uuid"
12
+ "github.com/ipfs/kubo/test/cli/harness"
13
+ "github.com/ipfs/kubo/test/cli/testutils"
14
+ "github.com/ipfs/kubo/test/cli/testutils/pinningservice"
15
+ "github.com/stretchr/testify/assert"
16
+ "github.com/stretchr/testify/require"
17
+ "github.com/tidwall/gjson"
18
+ "github.com/tidwall/sjson"
19
+)
20
+
21
+func runPinningService(t *testing.T, authToken string) (*pinningservice.PinningService, string) {
22
+ svc := pinningservice.New()
23
+ router := pinningservice.NewRouter(authToken, svc)
24
+ server := &http.Server{Handler: router}
25
+ listener, err := net.Listen("tcp", "127.0.0.1:0")
26
+ require.NoError(t, err)
27
+ go func() {
28
+ err := server.Serve(listener)
29
+ if err != nil && !errors.Is(err, net.ErrClosed) && !errors.Is(err, http.ErrServerClosed) {
30
+ t.Logf("Serve error: %s", err)
31
+ }
32
+ }()
33
+ t.Cleanup(func() { listener.Close() })
34
+
35
+ return svc, fmt.Sprintf("http://%s/api/v1", listener.Addr().String())
36
+}
37
+
38
+func TestRemotePinning(t *testing.T) {
39
+ t.Parallel()
40
+ authToken := "testauthtoken"
41
+
42
+ t.Run("MFS pinning", func(t *testing.T) {
43
+ t.Parallel()
44
+ node := harness.NewT(t).NewNode().Init()
45
+ node.Runner.Env["MFS_PIN_POLL_INTERVAL"] = "10ms"
46
+
47
+ _, svcURL := runPinningService(t, authToken)
48
+ node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
49
+ node.IPFS("config", "--json", "Pinning.RemoteServices.svc.Policies.MFS.RepinInterval", `"1s"`)
50
+ node.IPFS("config", "--json", "Pinning.RemoteServices.svc.Policies.MFS.PinName", `"test_pin"`)
51
+ node.IPFS("config", "--json", "Pinning.RemoteServices.svc.Policies.MFS.Enable", "true")
52
+
53
+ node.StartDaemon()
54
+
55
+ node.IPFS("files", "cp", "/ipfs/bafkqaaa", "/mfs-pinning-test-"+uuid.NewString())
56
+ node.IPFS("files", "flush")
57
+ res := node.IPFS("files", "stat", "/", "--enc=json")
58
+ hash := gjson.Get(res.Stdout.String(), "Hash").Str
59
+
60
+ assert.Eventually(t,
61
+ func() bool {
62
+ res = node.IPFS("pin", "remote", "ls",
63
+ "--service=svc",
64
+ "--name=test_pin",
65
+ "--status=queued,pinning,pinned,failed",
66
+ "--enc=json",
67
+ )
68
+ pinnedHash := gjson.Get(res.Stdout.String(), "Cid").Str
69
+ return hash == pinnedHash
70
+ },
71
+ 10*time.Second,
72
+ 10*time.Millisecond,
73
+ )
74
+
75
+ t.Run("MFS root is repinned on CID change", func(t *testing.T) {
76
+ node.IPFS("files", "cp", "/ipfs/bafkqaaa", "/mfs-pinning-repin-test-"+uuid.NewString())
77
+ node.IPFS("files", "flush")
78
+ res = node.IPFS("files", "stat", "/", "--enc=json")
79
+ hash := gjson.Get(res.Stdout.String(), "Hash").Str
80
+ assert.Eventually(t,
81
+ func() bool {
82
+ res := node.IPFS("pin", "remote", "ls",
83
+ "--service=svc",
84
+ "--name=test_pin",
85
+ "--status=queued,pinning,pinned,failed",
86
+ "--enc=json",
87
+ )
88
+ pinnedHash := gjson.Get(res.Stdout.String(), "Cid").Str
89
+ return hash == pinnedHash
90
+ },
91
+ 10*time.Second,
92
+ 10*time.Millisecond,
93
+ )
94
+ })
95
+ })
96
+
97
+ // Pinning.RemoteServices includes API.Key, so we give it the same treatment
98
+ // as Identity,PrivKey to prevent exposing it on the network
99
+ t.Run("access token security", func(t *testing.T) {
100
+ t.Parallel()
101
+ node := harness.NewT(t).NewNode().Init()
102
+ node.IPFS("pin", "remote", "service", "add", "1", "http://example1.com", "testkey")
103
+ res := node.RunIPFS("config", "Pinning")
104
+ assert.Equal(t, 1, res.ExitCode())
105
+ assert.Contains(t, res.Stderr.String(), "cannot show or change pinning services credentials")
106
+ assert.NotContains(t, res.Stdout.String(), "testkey")
107
+
108
+ res = node.RunIPFS("config", "Pinning.RemoteServices.1.API.Key")
109
+ assert.Equal(t, 1, res.ExitCode())
110
+ assert.Contains(t, res.Stderr.String(), "cannot show or change pinning services credentials")
111
+ assert.NotContains(t, res.Stdout.String(), "testkey")
112
+
113
+ configShow := node.RunIPFS("config", "show").Stdout.String()
114
+ assert.NotContains(t, configShow, "testkey")
115
+
116
+ t.Run("re-injecting config with 'ipfs config replace' preserves the API keys", func(t *testing.T) {
117
+ node.WriteBytes("config-show", []byte(configShow))
118
+ node.IPFS("config", "replace", "config-show")
119
+ assert.Contains(t, node.ReadFile(node.ConfigFile()), "testkey")
120
+ })
121
+
122
+ t.Run("injecting config with 'ipfs config replace' with API keys returns an error", func(t *testing.T) {
123
+ // remove Identity.PrivKey to ensure error is triggered by Pinning.RemoteServices
124
+ configJSON := MustVal(sjson.Delete(configShow, "Identity.PrivKey"))
125
+ configJSON = MustVal(sjson.Set(configJSON, "Pinning.RemoteServices.1.API.Key", "testkey"))
126
+ node.WriteBytes("new-config", []byte(configJSON))
127
+ res := node.RunIPFS("config", "replace", "new-config")
128
+ assert.Equal(t, 1, res.ExitCode())
129
+ assert.Contains(t, res.Stderr.String(), "cannot change remote pinning services api info with `config replace`")
130
+ })
131
+ })
132
+
133
+ t.Run("pin remote service ls --stat", func(t *testing.T) {
134
+ t.Parallel()
135
+ node := harness.NewT(t).NewNode().Init().StartDaemon()
136
+ _, svcURL := runPinningService(t, authToken)
137
+
138
+ node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
139
+ node.IPFS("pin", "remote", "service", "add", "invalid-svc", svcURL+"/invalidpath", authToken)
140
+
141
+ res := node.IPFS("pin", "remote", "service", "ls", "--stat")
142
+ assert.Contains(t, res.Stdout.String(), " 0/0/0/0")
143
+
144
+ stats := node.IPFS("pin", "remote", "service", "ls", "--stat", "--enc=json").Stdout.String()
145
+ assert.Equal(t, "valid", gjson.Get(stats, `RemoteServices.#(Service == "svc").Stat.Status`).Str)
146
+ assert.Equal(t, "invalid", gjson.Get(stats, `RemoteServices.#(Service == "invalid-svc").Stat.Status`).Str)
147
+
148
+ // no --stat returns no stat obj
149
+ t.Run("no --stat returns no stat obj", func(t *testing.T) {
150
+ res := node.IPFS("pin", "remote", "service", "ls", "--enc=json")
151
+ assert.False(t, gjson.Get(res.Stdout.String(), `RemoteServices.#(Service == "svc").Stat`).Exists())
152
+ })
153
+ })
154
+
155
+ t.Run("adding service with invalid URL fails", func(t *testing.T) {
156
+ t.Parallel()
157
+ node := harness.NewT(t).NewNode().Init().StartDaemon()
158
+
159
+ res := node.RunIPFS("pin", "remote", "service", "add", "svc", "invalid-service.example.com", "key")
160
+ assert.Equal(t, 1, res.ExitCode())
161
+ assert.Contains(t, res.Stderr.String(), "service endpoint must be a valid HTTP URL")
162
+
163
+ res = node.RunIPFS("pin", "remote", "service", "add", "svc", "xyz://invalid-service.example.com", "key")
164
+ assert.Equal(t, 1, res.ExitCode())
165
+ assert.Contains(t, res.Stderr.String(), "service endpoint must be a valid HTTP URL")
166
+ })
167
+
168
+ t.Run("unauthorized pinning service calls fail", func(t *testing.T) {
169
+ t.Parallel()
170
+ node := harness.NewT(t).NewNode().Init().StartDaemon()
171
+ _, svcURL := runPinningService(t, authToken)
172
+
173
+ node.IPFS("pin", "remote", "service", "add", "svc", svcURL, "othertoken")
174
+
175
+ res := node.RunIPFS("pin", "remote", "ls", "--service=svc")
176
+ assert.Equal(t, 1, res.ExitCode())
177
+ assert.Contains(t, res.Stderr.String(), "access denied")
178
+ })
179
+
180
+ t.Run("pinning service calls fail when there is a wrong path", func(t *testing.T) {
181
+ t.Parallel()
182
+ node := harness.NewT(t).NewNode().Init().StartDaemon()
183
+ _, svcURL := runPinningService(t, authToken)
184
+ node.IPFS("pin", "remote", "service", "add", "svc", svcURL+"/invalid-path", authToken)
185
+
186
+ res := node.RunIPFS("pin", "remote", "ls", "--service=svc")
187
+ assert.Equal(t, 1, res.ExitCode())
188
+ assert.Contains(t, res.Stderr.String(), "404")
189
+ })
190
+
191
+ t.Run("pinning service calls fail when DNS resolution fails", func(t *testing.T) {
192
+ t.Parallel()
193
+ node := harness.NewT(t).NewNode().Init().StartDaemon()
194
+ node.IPFS("pin", "remote", "service", "add", "svc", "https://invalid-service.example.com", authToken)
195
+
196
+ res := node.RunIPFS("pin", "remote", "ls", "--service=svc")
197
+ assert.Equal(t, 1, res.ExitCode())
198
+ assert.Contains(t, res.Stderr.String(), "no such host")
199
+ })
200
+
201
+ t.Run("pin remote service rm", func(t *testing.T) {
202
+ t.Parallel()
203
+ node := harness.NewT(t).NewNode().Init().StartDaemon()
204
+ node.IPFS("pin", "remote", "service", "add", "svc", "https://example.com", authToken)
205
+ node.IPFS("pin", "remote", "service", "rm", "svc")
206
+ res := node.IPFS("pin", "remote", "service", "ls")
207
+ assert.NotContains(t, res.Stdout.String(), "svc")
208
+ })
209
+
210
+ t.Run("remote pinning", func(t *testing.T) {
211
+ t.Parallel()
212
+
213
+ verifyStatus := func(node *harness.Node, name, hash, status string) {
214
+ resJSON := node.IPFS("pin", "remote", "ls",
215
+ "--service=svc",
216
+ "--enc=json",
217
+ "--name="+name,
218
+ "--status="+status,
219
+ ).Stdout.String()
220
+
221
+ assert.Equal(t, status, gjson.Get(resJSON, "Status").Str)
222
+ assert.Equal(t, hash, gjson.Get(resJSON, "Cid").Str)
223
+ assert.Equal(t, name, gjson.Get(resJSON, "Name").Str)
224
+ }
225
+
226
+ t.Run("'ipfs pin remote add --background=true'", func(t *testing.T) {
227
+ node := harness.NewT(t).NewNode().Init().StartDaemon()
228
+ svc, svcURL := runPinningService(t, authToken)
229
+ node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
230
+
231
+ // retain a ptr to the pin that's in the DB so we can directly mutate its status
232
+ // to simulate async work
233
+ pinCh := make(chan *pinningservice.PinStatus, 1)
234
+ svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) {
235
+ pinCh <- pin
236
+ }
237
+
238
+ hash := node.IPFSAddStr("foo")
239
+ node.IPFS("pin", "remote", "add",
240
+ "--background=true",
241
+ "--service=svc",
242
+ "--name=pin1",
243
+ hash,
244
+ )
245
+
246
+ pin := <-pinCh
247
+
248
+ transitionStatus := func(status string) {
249
+ pin.M.Lock()
250
+ pin.Status = status
251
+ pin.M.Unlock()
252
+ }
253
+
254
+ verifyStatus(node, "pin1", hash, "queued")
255
+
256
+ transitionStatus("pinning")
257
+ verifyStatus(node, "pin1", hash, "pinning")
258
+
259
+ transitionStatus("pinned")
260
+ verifyStatus(node, "pin1", hash, "pinned")
261
+
262
+ transitionStatus("failed")
263
+ verifyStatus(node, "pin1", hash, "failed")
264
+ })
265
+
266
+ t.Run("'ipfs pin remote add --background=false'", func(t *testing.T) {
267
+ t.Parallel()
268
+ node := harness.NewT(t).NewNode().Init().StartDaemon()
269
+ svc, svcURL := runPinningService(t, authToken)
270
+ node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
271
+
272
+ svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) {
273
+ pin.M.Lock()
274
+ defer pin.M.Unlock()
275
+ pin.Status = "pinned"
276
+ }
277
+ hash := node.IPFSAddStr("foo")
278
+ node.IPFS("pin", "remote", "add",
279
+ "--background=false",
280
+ "--service=svc",
281
+ "--name=pin2",
282
+ hash,
283
+ )
284
+ verifyStatus(node, "pin2", hash, "pinned")
285
+ })
286
+
287
+ t.Run("'ipfs pin remote ls' with multiple statuses", func(t *testing.T) {
288
+ t.Parallel()
289
+ node := harness.NewT(t).NewNode().Init().StartDaemon()
290
+ svc, svcURL := runPinningService(t, authToken)
291
+ node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
292
+
293
+ hash := node.IPFSAddStr("foo")
294
+ desiredStatuses := map[string]string{
295
+ "pin-queued": "queued",
296
+ "pin-pinning": "pinning",
297
+ "pin-pinned": "pinned",
298
+ "pin-failed": "failed",
299
+ }
300
+ var pins []*pinningservice.PinStatus
301
+ svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) {
302
+ pin.M.Lock()
303
+ defer pin.M.Unlock()
304
+ pins = append(pins, pin)
305
+ // this must be "pinned" for the 'pin remote add' command to return
306
+ // after 'pin remote add', we change the status to its real status
307
+ pin.Status = "pinned"
308
+ }
309
+
310
+ for pinName := range desiredStatuses {
311
+ node.IPFS("pin", "remote", "add",
312
+ "--service=svc",
313
+ "--name="+pinName,
314
+ hash,
315
+ )
316
+ }
317
+ for _, pin := range pins {
318
+ pin.M.Lock()
319
+ pin.Status = desiredStatuses[pin.Pin.Name]
320
+ pin.M.Unlock()
321
+ }
322
+
323
+ res := node.IPFS("pin", "remote", "ls",
324
+ "--service=svc",
325
+ "--status=queued,pinning,pinned,failed",
326
+ "--enc=json",
327
+ )
328
+ actualStatuses := map[string]string{}
329
+ for _, line := range res.Stdout.Lines() {
330
+ name := gjson.Get(line, "Name").Str
331
+ status := gjson.Get(line, "Status").Str
332
+ // drop statuses of other pins we didn't add
333
+ if _, ok := desiredStatuses[name]; ok {
334
+ actualStatuses[name] = status
335
+ }
336
+ }
337
+ assert.Equal(t, desiredStatuses, actualStatuses)
338
+ })
339
+
340
+ t.Run("'ipfs pin remote ls' by CID", func(t *testing.T) {
341
+ t.Parallel()
342
+ node := harness.NewT(t).NewNode().Init().StartDaemon()
343
+ svc, svcURL := runPinningService(t, authToken)
344
+ node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
345
+
346
+ transitionedCh := make(chan struct{}, 1)
347
+ svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) {
348
+ pin.M.Lock()
349
+ defer pin.M.Unlock()
350
+ pin.Status = "pinned"
351
+ transitionedCh <- struct{}{}
352
+ }
353
+ hash := node.IPFSAddStr(string(testutils.RandomBytes(1000)))
354
+ node.IPFS("pin", "remote", "add", "--background=false", "--service=svc", hash)
355
+ <-transitionedCh
356
+ res := node.IPFS("pin", "remote", "ls", "--service=svc", "--cid="+hash, "--enc=json").Stdout.String()
357
+ assert.Contains(t, res, hash)
358
+ })
359
+
360
+ t.Run("'ipfs pin remote rm --name' without --force when multiple pins match", func(t *testing.T) {
361
+ t.Parallel()
362
+ node := harness.NewT(t).NewNode().Init().StartDaemon()
363
+ svc, svcURL := runPinningService(t, authToken)
364
+ node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
365
+
366
+ svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) {
367
+ pin.M.Lock()
368
+ defer pin.M.Unlock()
369
+ pin.Status = "pinned"
370
+ }
371
+ hash := node.IPFSAddStr(string(testutils.RandomBytes(1000)))
372
+ node.IPFS("pin", "remote", "add", "--service=svc", "--name=force-test-name", hash)
373
+ node.IPFS("pin", "remote", "add", "--service=svc", "--name=force-test-name", hash)
374
+
375
+ t.Run("fails", func(t *testing.T) {
376
+ res := node.RunIPFS("pin", "remote", "rm", "--service=svc", "--name=force-test-name")
377
+ assert.Equal(t, 1, res.ExitCode())
378
+ assert.Contains(t, res.Stderr.String(), "Error: multiple remote pins are matching this query, add --force to confirm the bulk removal")
379
+ })
380
+
381
+ t.Run("matching pins are not removed", func(t *testing.T) {
382
+ lines := node.IPFS("pin", "remote", "ls", "--service=svc", "--name=force-test-name").Stdout.Lines()
383
+ assert.Contains(t, lines[0], "force-test-name")
384
+ assert.Contains(t, lines[1], "force-test-name")
385
+ })
386
+ })
387
+
388
+ t.Run("'ipfs pin remote rm --name --force' remove multiple pins", func(t *testing.T) {
389
+ t.Parallel()
390
+ node := harness.NewT(t).NewNode().Init().StartDaemon()
391
+ svc, svcURL := runPinningService(t, authToken)
392
+ node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
393
+
394
+ svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) {
395
+ pin.M.Lock()
396
+ defer pin.M.Unlock()
397
+ pin.Status = "pinned"
398
+ }
399
+ hash := node.IPFSAddStr(string(testutils.RandomBytes(1000)))
400
+ node.IPFS("pin", "remote", "add", "--service=svc", "--name=force-test-name", hash)
401
+ node.IPFS("pin", "remote", "add", "--service=svc", "--name=force-test-name", hash)
402
+
403
+ node.IPFS("pin", "remote", "rm", "--service=svc", "--name=force-test-name", "--force")
404
+ out := node.IPFS("pin", "remote", "ls", "--service=svc", "--name=force-test-name").Stdout.Trimmed()
405
+ assert.Empty(t, out)
406
+ })
407
+
408
+ t.Run("'ipfs pin remote rm --force' removes all pins", func(t *testing.T) {
409
+ t.Parallel()
410
+ node := harness.NewT(t).NewNode().Init().StartDaemon()
411
+ svc, svcURL := runPinningService(t, authToken)
412
+ node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
413
+
414
+ svc.PinAdded = func(req *pinningservice.AddPinRequest, pin *pinningservice.PinStatus) {
415
+ pin.M.Lock()
416
+ defer pin.M.Unlock()
417
+ pin.Status = "pinned"
418
+ }
419
+ for i := 0; i < 4; i++ {
420
+ hash := node.IPFSAddStr(string(testutils.RandomBytes(1000)))
421
+ name := fmt.Sprintf("--name=%d", i)
422
+ node.IPFS("pin", "remote", "add", "--service=svc", "--name="+name, hash)
423
+ }
424
+
425
+ lines := node.IPFS("pin", "remote", "ls", "--service=svc").Stdout.Lines()
426
+ assert.Len(t, lines, 4)
427
+
428
+ node.IPFS("pin", "remote", "rm", "--service=svc", "--force")
429
+
430
+ lines = node.IPFS("pin", "remote", "ls", "--service=svc").Stdout.Lines()
431
+ assert.Len(t, lines, 0)
432
+ })
433
+ })
434
+
435
+ t.Run("'ipfs pin remote add' shows a warning message when offline", func(t *testing.T) {
436
+ t.Parallel()
437
+ node := harness.NewT(t).NewNode().Init()
438
+ _, svcURL := runPinningService(t, authToken)
439
+ node.IPFS("pin", "remote", "service", "add", "svc", svcURL, authToken)
440
+
441
+ hash := node.IPFSAddStr(string(testutils.RandomBytes(1000)))
442
+ res := node.IPFS("pin", "remote", "add", "--service=svc", "--background", hash)
443
+ warningMsg := "WARNING: the local node is offline and remote pinning may fail if there is no other provider for this CID"
444
+ assert.Contains(t, res.Stdout.String(), warningMsg)
445
+ })
446
+}
test/cli/testutils/pinningservice/pinning.go
new
+401
@@ -0,0 +1,401 @@
1
+package pinningservice
2
+
3
+import (
4
+ "encoding/json"
5
+ "fmt"
6
+ "net/http"
7
+ "reflect"
8
+ "strconv"
9
+ "strings"
10
+ "sync"
11
+ "time"
12
+
13
+ "github.com/google/uuid"
14
+ "github.com/julienschmidt/httprouter"
15
+)
16
+
17
+func NewRouter(authToken string, svc *PinningService) http.Handler {
18
+ router := httprouter.New()
19
+ router.GET("/api/v1/pins", svc.listPins)
20
+ router.POST("/api/v1/pins", svc.addPin)
21
+ router.GET("/api/v1/pins/:requestID", svc.getPin)
22
+ router.POST("/api/v1/pins/:requestID", svc.replacePin)
23
+ router.DELETE("/api/v1/pins/:requestID", svc.removePin)
24
+
25
+ handler := authHandler(authToken, router)
26
+
27
+ return handler
28
+}
29
+
30
+func authHandler(authToken string, delegate http.Handler) http.Handler {
31
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
32
+ authz := r.Header.Get("Authorization")
33
+ if !strings.HasPrefix(authz, "Bearer ") {
34
+ errResp(w, "invalid authorization token, must start with 'Bearer '", "", http.StatusBadRequest)
35
+ return
36
+ }
37
+
38
+ token := strings.TrimPrefix(authz, "Bearer ")
39
+ if token != authToken {
40
+ errResp(w, "access denied", "", http.StatusUnauthorized)
41
+ return
42
+ }
43
+
44
+ delegate.ServeHTTP(w, r)
45
+ })
46
+}
47
+
48
+func New() *PinningService {
49
+ return &PinningService{
50
+ PinAdded: func(*AddPinRequest, *PinStatus) {},
51
+ }
52
+}
53
+
54
+// PinningService is a basic pinning service that implements the Remote Pinning API, for testing Kubo's integration with remote pinning services.
55
+// Pins are not persisted, they are just kept in-memory, and this provides callbacks for controlling the behavior of the pinning service.
56
+type PinningService struct {
57
+ m sync.Mutex
58
+ // PinAdded is a callback that is invoked after a new pin is added via the API.
59
+ PinAdded func(*AddPinRequest, *PinStatus)
60
+ pins []*PinStatus
61
+}
62
+
63
+type Pin struct {
64
+ CID string `json:"cid"`
65
+ Name string `json:"name"`
66
+ Origins []string `json:"origins"`
67
+ Meta map[string]interface{} `json:"meta"`
68
+}
69
+
70
+type PinStatus struct {
71
+ M sync.Mutex
72
+ RequestID string
73
+ Status string
74
+ Created time.Time
75
+ Pin Pin
76
+ Delegates []string
77
+ Info map[string]interface{}
78
+}
79
+
80
+func (p *PinStatus) MarshalJSON() ([]byte, error) {
81
+ type pinStatusJSON struct {
82
+ RequestID string `json:"requestid"`
83
+ Status string `json:"status"`
84
+ Created time.Time `json:"created"`
85
+ Pin Pin `json:"pin"`
86
+ Delegates []string `json:"delegates"`
87
+ Info map[string]interface{} `json:"info"`
88
+ }
89
+ // lock the pin before marshaling it to protect against data races while marshaling
90
+ p.M.Lock()
91
+ pinJSON := pinStatusJSON{
92
+ RequestID: p.RequestID,
93
+ Status: p.Status,
94
+ Created: p.Created,
95
+ Pin: p.Pin,
96
+ Delegates: p.Delegates,
97
+ Info: p.Info,
98
+ }
99
+ p.M.Unlock()
100
+ return json.Marshal(pinJSON)
101
+}
102
+
103
+func (p *PinStatus) Clone() PinStatus {
104
+ return PinStatus{
105
+ RequestID: p.RequestID,
106
+ Status: p.Status,
107
+ Created: p.Created,
108
+ Pin: p.Pin,
109
+ Delegates: p.Delegates,
110
+ Info: p.Info,
111
+ }
112
+}
113
+
114
+const (
115
+ matchExact = "exact"
116
+ matchIExact = "iexact"
117
+ matchPartial = "partial"
118
+ matchIPartial = "ipartial"
119
+
120
+ statusQueued = "queued"
121
+ statusPinning = "pinning"
122
+ statusPinned = "pinned"
123
+ statusFailed = "failed"
124
+
125
+ timeLayout = "2006-01-02T15:04:05.999Z"
126
+)
127
+
128
+func errResp(w http.ResponseWriter, reason, details string, statusCode int) {
129
+ type errorObj struct {
130
+ Reason string `json:"reason"`
131
+ Details string `json:"details"`
132
+ }
133
+ type errorResp struct {
134
+ Error errorObj `json:"error"`
135
+ }
136
+ resp := errorResp{
137
+ Error: errorObj{
138
+ Reason: reason,
139
+ Details: details,
140
+ },
141
+ }
142
+ writeJSON(w, resp, statusCode)
143
+}
144
+
145
+func writeJSON(w http.ResponseWriter, val any, statusCode int) {
146
+ b, err := json.Marshal(val)
147
+ if err != nil {
148
+ w.Header().Set("Content-Type", "text/plain")
149
+ errResp(w, fmt.Sprintf("marshaling response: %s", err), "", http.StatusInternalServerError)
150
+ return
151
+ }
152
+ w.Header().Set("Content-Type", "application/json")
153
+ w.WriteHeader(statusCode)
154
+ _, _ = w.Write(b)
155
+}
156
+
157
+type AddPinRequest struct {
158
+ CID string `json:"cid"`
159
+ Name string `json:"name"`
160
+ Origins []string `json:"origins"`
161
+ Meta map[string]interface{} `json:"meta"`
162
+}
163
+
164
+func (p *PinningService) addPin(writer http.ResponseWriter, req *http.Request, params httprouter.Params) {
165
+ var addReq AddPinRequest
166
+ err := json.NewDecoder(req.Body).Decode(&addReq)
167
+ if err != nil {
168
+ errResp(writer, fmt.Sprintf("unmarshaling req: %s", err), "", http.StatusBadRequest)
169
+ return
170
+ }
171
+
172
+ pin := &PinStatus{
173
+ RequestID: uuid.NewString(),
174
+ Status: statusQueued,
175
+ Created: time.Now(),
176
+ Pin: Pin(addReq),
177
+ }
178
+
179
+ p.m.Lock()
180
+ p.pins = append(p.pins, pin)
181
+ p.m.Unlock()
182
+
183
+ writeJSON(writer, &pin, http.StatusAccepted)
184
+ p.PinAdded(&addReq, pin)
185
+}
186
+
187
+type ListPinsResponse struct {
188
+ Count int `json:"count"`
189
+ Results []*PinStatus `json:"results"`
190
+}
191
+
192
+func (p *PinningService) listPins(writer http.ResponseWriter, req *http.Request, params httprouter.Params) {
193
+ q := req.URL.Query()
194
+
195
+ cidStr := q.Get("cid")
196
+ name := q.Get("name")
197
+ match := q.Get("match")
198
+ status := q.Get("status")
199
+ beforeStr := q.Get("before")
200
+ afterStr := q.Get("after")
201
+ limitStr := q.Get("limit")
202
+ metaStr := q.Get("meta")
203
+
204
+ if limitStr == "" {
205
+ limitStr = "10"
206
+ }
207
+ limit, err := strconv.Atoi(limitStr)
208
+ if err != nil {
209
+ errResp(writer, fmt.Sprintf("parsing limit: %s", err), "", http.StatusBadRequest)
210
+ return
211
+ }
212
+
213
+ var cids []string
214
+ if cidStr != "" {
215
+ cids = strings.Split(cidStr, ",")
216
+ }
217
+
218
+ var statuses []string
219
+ if status != "" {
220
+ statuses = strings.Split(status, ",")
221
+ }
222
+
223
+ p.m.Lock()
224
+ defer p.m.Unlock()
225
+ var pins []*PinStatus
226
+ for _, pinStatus := range p.pins {
227
+ // clone it so we can immediately release the lock
228
+ pinStatus.M.Lock()
229
+ clonedPS := pinStatus.Clone()
230
+ pinStatus.M.Unlock()
231
+
232
+ // cid
233
+ var matchesCID bool
234
+ if len(cids) == 0 {
235
+ matchesCID = true
236
+ } else {
237
+ for _, cid := range cids {
238
+ if cid == clonedPS.Pin.CID {
239
+ matchesCID = true
240
+ }
241
+ }
242
+ }
243
+ if !matchesCID {
244
+ continue
245
+ }
246
+
247
+ // name
248
+ if match == "" {
249
+ match = matchExact
250
+ }
251
+ if name != "" {
252
+ switch match {
253
+ case matchExact:
254
+ if name != clonedPS.Pin.Name {
255
+ continue
256
+ }
257
+ case matchIExact:
258
+ if !strings.EqualFold(name, clonedPS.Pin.Name) {
259
+ continue
260
+ }
261
+ case matchPartial:
262
+ if !strings.Contains(clonedPS.Pin.Name, name) {
263
+ continue
264
+ }
265
+ case matchIPartial:
266
+ if !strings.Contains(strings.ToLower(clonedPS.Pin.Name), strings.ToLower(name)) {
267
+ continue
268
+ }
269
+ default:
270
+ errResp(writer, fmt.Sprintf("unknown match %q", match), "", http.StatusBadRequest)
271
+ return
272
+ }
273
+ }
274
+
275
+ // status
276
+ var matchesStatus bool
277
+ if len(statuses) == 0 {
278
+ statuses = []string{statusPinned}
279
+ }
280
+ for _, status := range statuses {
281
+ if status == clonedPS.Status {
282
+ matchesStatus = true
283
+ }
284
+ }
285
+ if !matchesStatus {
286
+ continue
287
+ }
288
+
289
+ // before
290
+ if beforeStr != "" {
291
+ before, err := time.Parse(timeLayout, beforeStr)
292
+ if err != nil {
293
+ errResp(writer, fmt.Sprintf("parsing before: %s", err), "", http.StatusBadRequest)
294
+ return
295
+ }
296
+ if !clonedPS.Created.Before(before) {
297
+ continue
298
+ }
299
+ }
300
+
301
+ // after
302
+ if afterStr != "" {
303
+ after, err := time.Parse(timeLayout, afterStr)
304
+ if err != nil {
305
+ errResp(writer, fmt.Sprintf("parsing before: %s", err), "", http.StatusBadRequest)
306
+ return
307
+ }
308
+ if !clonedPS.Created.After(after) {
309
+ continue
310
+ }
311
+ }
312
+
313
+ // meta
314
+ if metaStr != "" {
315
+ meta := map[string]interface{}{}
316
+ err := json.Unmarshal([]byte(metaStr), &meta)
317
+ if err != nil {
318
+ errResp(writer, fmt.Sprintf("parsing meta: %s", err), "", http.StatusBadRequest)
319
+ return
320
+ }
321
+ var matchesMeta bool
322
+ for k, v := range meta {
323
+ pinV, contains := clonedPS.Pin.Meta[k]
324
+ if !contains || !reflect.DeepEqual(pinV, v) {
325
+ matchesMeta = false
326
+ break
327
+ }
328
+ }
329
+ if !matchesMeta {
330
+ continue
331
+ }
332
+ }
333
+
334
+ // add the original pin status, not the cloned one
335
+ pins = append(pins, pinStatus)
336
+
337
+ if len(pins) == limit {
338
+ break
339
+ }
340
+ }
341
+
342
+ out := ListPinsResponse{
343
+ Count: len(pins),
344
+ Results: pins,
345
+ }
346
+ writeJSON(writer, out, http.StatusOK)
347
+}
348
+
349
+func (p *PinningService) getPin(writer http.ResponseWriter, req *http.Request, params httprouter.Params) {
350
+ requestID := params.ByName("requestID")
351
+ p.m.Lock()
352
+ defer p.m.Unlock()
353
+ for _, pin := range p.pins {
354
+ if pin.RequestID == requestID {
355
+ writeJSON(writer, pin, http.StatusOK)
356
+ return
357
+ }
358
+ }
359
+ errResp(writer, "", "", http.StatusNotFound)
360
+}
361
+
362
+func (p *PinningService) replacePin(writer http.ResponseWriter, req *http.Request, params httprouter.Params) {
363
+ requestID := params.ByName("requestID")
364
+
365
+ var replaceReq Pin
366
+ err := json.NewDecoder(req.Body).Decode(&replaceReq)
367
+ if err != nil {
368
+ errResp(writer, fmt.Sprintf("decoding request: %s", err), "", http.StatusBadRequest)
369
+ return
370
+ }
371
+
372
+ p.m.Lock()
373
+ defer p.m.Unlock()
374
+ for _, pin := range p.pins {
375
+ if pin.RequestID == requestID {
376
+ pin.M.Lock()
377
+ pin.Pin = replaceReq
378
+ pin.M.Unlock()
379
+ writer.WriteHeader(http.StatusAccepted)
380
+ return
381
+ }
382
+ }
383
+ errResp(writer, "", "", http.StatusNotFound)
384
+}
385
+
386
+func (p *PinningService) removePin(writer http.ResponseWriter, req *http.Request, params httprouter.Params) {
387
+ requestID := params.ByName("requestID")
388
+
389
+ p.m.Lock()
390
+ defer p.m.Unlock()
391
+
392
+ for i, pin := range p.pins {
393
+ if pin.RequestID == requestID {
394
+ p.pins = append(p.pins[0:i], p.pins[i+1:]...)
395
+ writer.WriteHeader(http.StatusAccepted)
396
+ return
397
+ }
398
+ }
399
+
400
+ errResp(writer, "", "", http.StatusNotFound)
401
+}
test/sharness/t0700-remotepin.sh
deleted
-332
@@ -1,332 +0,0 @@
1
-#!/usr/bin/env bash
2
-
3
-test_description="Test ipfs remote pinning operations"
4
-
5
-. lib/test-lib.sh
6
-
7
-if [ -z ${TEST_DOCKER_HOST+x} ]; then
8
- # TODO: set up instead of skipping?
9
- skip_all='Skipping pinning service integration tests: missing TEST_DOCKER_HOST, remote pinning service not available'
10
- test_done
11
-fi
12
-
13
-# daemon running in online mode to ensure Pin.origins/PinStatus.delegates work
14
-test_init_ipfs
15
-test_launch_ipfs_daemon
16
-
17
-# create user on pinning service
18
-TEST_PIN_SVC="http://${TEST_DOCKER_HOST}:5000/api/v1"
19
-TEST_PIN_SVC_KEY=$(curl -s -X POST "$TEST_PIN_SVC/users" -d email="go-ipfs-sharness@ipfs.example.com" | jq --raw-output .access_token)
20
-
21
-# pin remote service add|ls|rm
22
-
23
-# confirm empty service list response has proper json struct
24
-# https://github.com/ipfs/go-ipfs/pull/7829
25
-test_expect_success "test 'ipfs pin remote service ls' JSON on empty list" '
26
- ipfs pin remote service ls --stat --enc=json | tee empty_ls_out &&
27
- echo "{\"RemoteServices\":[]}" > exp_ls_out &&
28
- test_cmp exp_ls_out empty_ls_out
29
-'
30
-
31
-# add valid and invalid services
32
-test_expect_success "creating test user on remote pinning service" '
33
- echo CI host IP address ${TEST_PIN_SVC} &&
34
- ipfs pin remote service add test_pin_svc ${TEST_PIN_SVC} ${TEST_PIN_SVC_KEY} &&
35
- ipfs pin remote service add test_invalid_key_svc ${TEST_PIN_SVC} fake_api_key &&
36
- ipfs pin remote service add test_invalid_url_path_svc ${TEST_PIN_SVC}/invalid-path fake_api_key &&
37
- ipfs pin remote service add test_invalid_url_dns_svc https://invalid-service.example.com fake_api_key &&
38
- ipfs pin remote service add test_pin_mfs_svc ${TEST_PIN_SVC} ${TEST_PIN_SVC_KEY}
39
-'
40
-
41
-# add a service with a invalid endpoint
42
-test_expect_success "adding remote service with invalid endpoint" '
43
- test_expect_code 1 ipfs pin remote service add test_endpoint_no_protocol invalid-service.example.com fake_api_key &&
44
- test_expect_code 1 ipfs pin remote service add test_endpoint_bad_protocol xyz://invalid-service.example.com fake_api_key
45
-'
46
-
47
-test_expect_success "test 'ipfs pin remote service ls'" '
48
- ipfs pin remote service ls | tee ls_out &&
49
- grep -q test_pin_svc ls_out &&
50
- grep -q test_invalid_key_svc ls_out &&
51
- grep -q test_invalid_url_path_svc ls_out &&
52
- grep -q test_invalid_url_dns_svc ls_out
53
-'
54
-
55
-test_expect_success "test enabling mfs pinning" '
56
- ipfs config --json Pinning.RemoteServices.test_pin_mfs_svc.Policies.MFS.RepinInterval \"10s\" &&
57
- ipfs config --json Pinning.RemoteServices.test_pin_mfs_svc.Policies.MFS.PinName \"mfs_test_pin\" &&
58
- ipfs config --json Pinning.RemoteServices.test_pin_mfs_svc.Policies.MFS.Enable true &&
59
- ipfs config --json Pinning.RemoteServices.test_pin_mfs_svc.Policies.MFS.RepinInterval > repin_interval &&
60
- ipfs config --json Pinning.RemoteServices.test_pin_mfs_svc.Policies.MFS.PinName > pin_name &&
61
- ipfs config --json Pinning.RemoteServices.test_pin_mfs_svc.Policies.MFS.Enable > enable &&
62
- echo 10s > expected_repin_interval &&
63
- echo mfs_test_pin > expected_pin_name &&
64
- echo true > expected_enable &&
65
- test_cmp repin_interval expected_repin_interval &&
66
- test_cmp pin_name expected_pin_name &&
67
- test_cmp enable expected_enable
68
-'
69
-
70
-# expect PIN to be created
71
-test_expect_success "verify MFS root is being pinned" '
72
- ipfs files cp /ipfs/bafkqaaa /mfs-pinning-test-$(date +%s.%N) &&
73
- ipfs files flush &&
74
- sleep 31 &&
75
- ipfs files stat / --enc=json | jq -r .Hash > mfs_cid &&
76
- ipfs pin remote ls --service=test_pin_mfs_svc --name=mfs_test_pin --status=queued,pinning,pinned,failed --enc=json | tee ls_out | jq -r .Cid > pin_cid &&
77
- cat mfs_cid ls_out &&
78
- test_cmp mfs_cid pin_cid
79
-'
80
-
81
-# expect existing PIN to be replaced
82
-test_expect_success "verify MFS root is being repinned on CID change" '
83
- ipfs files cp /ipfs/bafkqaaa /mfs-pinning-repin-test-$(date +%s.%N) &&
84
- ipfs files flush &&
85
- sleep 31 &&
86
- ipfs files stat / --enc=json | jq -r .Hash > mfs_cid &&
87
- ipfs pin remote ls --service=test_pin_mfs_svc --name=mfs_test_pin --status=queued,pinning,pinned,failed --enc=json | tee ls_out | jq -r .Cid > pin_cid &&
88
- cat mfs_cid ls_out &&
89
- test_cmp mfs_cid pin_cid
90
-'
91
-
92
-# SECURITY of access tokens in API.Key fields:
93
-# Pinning.RemoteServices includes API.Key, and we give it the same treatment
94
-# as Identity.PrivKey to prevent exposing it on the network
95
-
96
-test_expect_success "'ipfs config Pinning' fails" '
97
- test_expect_code 1 ipfs config Pinning 2>&1 > config_out
98
-'
99
-test_expect_success "output does not include API.Key" '
100
- test_expect_code 1 grep -q Key config_out
101
-'
102
-
103
-test_expect_success "'ipfs config Pinning.RemoteServices.test_pin_svc.API.Key' fails" '
104
- test_expect_code 1 ipfs config Pinning.RemoteServices.test_pin_svc.API.Key 2> config_out
105
-'
106
-
107
-test_expect_success "output includes meaningful error" '
108
- echo "Error: cannot show or change pinning services credentials" > config_exp &&
109
- test_cmp config_exp config_out
110
-'
111
-
112
-test_expect_success "'ipfs config Pinning.RemoteServices.test_pin_svc' fails" '
113
- test_expect_code 1 ipfs config Pinning.RemoteServices.test_pin_svc 2> config_out
114
-'
115
-test_expect_success "output includes meaningful error" '
116
- test_cmp config_exp config_out
117
-'
118
-
119
-test_expect_success "'ipfs config show' does not include Pinning.RemoteServices[*].API.Key" '
120
- ipfs config show | tee show_config | jq -r .Pinning.RemoteServices > remote_services &&
121
- test_expect_code 1 grep \"Key\" remote_services &&
122
- test_expect_code 1 grep fake_api_key show_config &&
123
- test_expect_code 1 grep "$TEST_PIN_SVC_KEY" show_config
124
-'
125
-
126
-test_expect_success "'ipfs config replace' injects Pinning.RemoteServices[*].API.Key back" '
127
- test_expect_code 1 grep fake_api_key show_config &&
128
- test_expect_code 1 grep "$TEST_PIN_SVC_KEY" show_config &&
129
- ipfs config replace show_config &&
130
- test_expect_code 0 grep fake_api_key "$IPFS_PATH/config" &&
131
- test_expect_code 0 grep "$TEST_PIN_SVC_KEY" "$IPFS_PATH/config"
132
-'
133
-
134
-# note: we remove Identity.PrivKey to ensure error is triggered by Pinning.RemoteServices
135
-test_expect_success "'ipfs config replace' with Pinning.RemoteServices[*].API.Key errors out" '
136
- jq -M "del(.Identity.PrivKey)" "$IPFS_PATH/config" | jq ".Pinning += { RemoteServices: {\"myservice\": {\"API\": {\"Endpoint\": \"https://example.com/psa\", \"Key\": \"mysecret\"}}}}" > new_config &&
137
- test_expect_code 1 ipfs config replace - < new_config 2> replace_out
138
-'
139
-test_expect_success "output includes meaningful error" "
140
- echo \"Error: cannot add or remove remote pinning services with 'config replace'\" > replace_expected &&
141
- test_cmp replace_out replace_expected
142
-"
143
-
144
-# /SECURITY
145
-
146
-test_expect_success "pin remote service ls --stat' returns numbers for a valid service" '
147
- ipfs pin remote service ls --stat | grep -E "^test_pin_svc.+[0-9]+/[0-9]+/[0-9]+/[0-9]+$"
148
-'
149
-
150
-test_expect_success "pin remote service ls --enc=json --stat' returns valid status" "
151
- ipfs pin remote service ls --stat --enc=json | jq --raw-output '.RemoteServices[] | select(.Service == \"test_pin_svc\") | .Stat.Status' | tee stat_out &&
152
- echo valid > stat_expected &&
153
- test_cmp stat_out stat_expected
154
-"
155
-
156
-test_expect_success "pin remote service ls --stat' returns invalid status for invalid service" '
157
- ipfs pin remote service ls --stat | grep -E "^test_invalid_url_path_svc.+invalid$"
158
-'
159
-
160
-test_expect_success "pin remote service ls --enc=json --stat' returns invalid status" "
161
- ipfs pin remote service ls --stat --enc=json | jq --raw-output '.RemoteServices[] | select(.Service == \"test_invalid_url_path_svc\") | .Stat.Status' | tee stat_out &&
162
- echo invalid > stat_expected &&
163
- test_cmp stat_out stat_expected
164
-"
165
-
166
-test_expect_success "pin remote service ls --enc=json' (without --stat) returns no Stat object" "
167
- ipfs pin remote service ls --enc=json | jq --raw-output '.RemoteServices[] | select(.Service == \"test_invalid_url_path_svc\") | .Stat' | tee stat_out &&
168
- echo null > stat_expected &&
169
- test_cmp stat_out stat_expected
170
-"
171
-
172
-test_expect_success "check connection to the test pinning service" '
173
- ipfs pin remote ls --service=test_pin_svc --enc=json
174
-'
175
-
176
-test_expect_success "unauthorized pinning service calls fail" '
177
- test_expect_code 1 ipfs pin remote ls --service=test_invalid_key_svc
178
-'
179
-
180
-test_expect_success "misconfigured pinning service calls fail (wrong path)" '
181
- test_expect_code 1 ipfs pin remote ls --service=test_invalid_url_path_svc
182
-'
183
-
184
-test_expect_success "misconfigured pinning service calls fail (dns error)" '
185
- test_expect_code 1 ipfs pin remote ls --service=test_invalid_url_dns_svc
186
-'
187
-
188
-# pin remote service rm
189
-
190
-test_expect_success "remove pinning service" '
191
- ipfs pin remote service rm test_invalid_key_svc &&
192
- ipfs pin remote service rm test_invalid_url_path_svc &&
193
- ipfs pin remote service rm test_invalid_url_dns_svc
194
-'
195
-
196
-test_expect_success "verify pinning service removal works" '
197
- ipfs pin remote service ls | tee ls_out &&
198
- test_expect_code 1 grep test_invalid_key_svc ls_out &&
199
- test_expect_code 1 grep test_invalid_url_path_svc ls_out &&
200
- test_expect_code 1 grep test_invalid_url_dns_svc ls_out
201
-'
202
-
203
-# pin remote add
204
-
205
-# we leverage the fact that inlined CID can be pinned instantly on the remote service
206
-# (https://github.com/ipfs-shipyard/rb-pinning-service-api/issues/8)
207
-# below test ensures that assumption is correct (before we proceed to actual tests)
208
-test_expect_success "verify that default add (implicit --background=false) works with data inlined in CID" '
209
- ipfs pin remote add --service=test_pin_svc --name=inlined_null bafkqaaa &&
210
- ipfs pin remote ls --service=test_pin_svc --enc=json --name=inlined_null --status=pinned | jq --raw-output .Status | tee ls_out &&
211
- grep -q "pinned" ls_out
212
-'
213
-
214
-test_remote_pins() {
215
- BASE=$1
216
- if [ -n "$BASE" ]; then
217
- BASE_ARGS="--cid-base=$BASE"
218
- fi
219
-
220
- # note: HAS_MISSING is not inlined nor imported to IPFS on purpose, to reliably test 'queued' state
221
- test_expect_success "create some hashes using base $BASE" '
222
- export HASH_A=$(echo -n "A @ $(date +%s.%N)" | ipfs add $BASE_ARGS -q --inline --inline-limit 1000 --pin=false) &&
223
- export HASH_B=$(echo -n "B @ $(date +%s.%N)" | ipfs add $BASE_ARGS -q --inline --inline-limit 1000 --pin=false) &&
224
- export HASH_C=$(echo -n "C @ $(date +%s.%N)" | ipfs add $BASE_ARGS -q --inline --inline-limit 1000 --pin=false) &&
225
- export HASH_MISSING=$(echo "MISSING FROM IPFS @ $(date +%s.%N)" | ipfs add $BASE_ARGS -q --only-hash) &&
226
- echo "A: $HASH_A" &&
227
- echo "B: $HASH_B" &&
228
- echo "C: $HASH_C" &&
229
- echo "M: $HASH_MISSING"
230
- '
231
-
232
- test_expect_success "'ipfs pin remote add --background=true'" '
233
- ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_a $HASH_A
234
- '
235
-
236
- test_expect_success "verify background add worked (instantly pinned variant)" '
237
- ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_a | tee ls_out &&
238
- test_expect_code 0 grep -q name_a ls_out &&
239
- test_expect_code 0 grep -q $HASH_A ls_out
240
- '
241
-
242
- test_expect_success "'ipfs pin remote add --background=true' with CID that is not available" '
243
- test_expect_code 0 ipfs pin remote add --background=true --service=test_pin_svc --enc=json $BASE_ARGS --name=name_m $HASH_MISSING
244
- '
245
-
246
- test_expect_success "verify background add worked (queued variant)" '
247
- ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_m --status=queued,pinning | tee ls_out &&
248
- test_expect_code 0 grep -q name_m ls_out &&
249
- test_expect_code 0 grep -q $HASH_MISSING ls_out
250
- '
251
-
252
- test_expect_success "'ipfs pin remote add --background=false'" '
253
- test_expect_code 0 ipfs pin remote add --background=false --service=test_pin_svc --enc=json $BASE_ARGS --name=name_b $HASH_B
254
- '
255
-
256
- test_expect_success "verify foreground add worked" '
257
- ipfs pin remote ls --service=test_pin_svc --enc=json $ID_B | tee ls_out &&
258
- test_expect_code 0 grep -q name_b ls_out &&
259
- test_expect_code 0 grep -q pinned ls_out &&
260
- test_expect_code 0 grep -q $HASH_B ls_out
261
- '
262
-
263
- test_expect_success "'ipfs pin remote ls' for existing pins by multiple statuses" '
264
- ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued,pinning,pinned,failed | tee ls_out &&
265
- test_expect_code 0 grep -q $HASH_A ls_out &&
266
- test_expect_code 0 grep -q $HASH_B ls_out &&
267
- test_expect_code 0 grep -q $HASH_MISSING ls_out
268
- '
269
-
270
- test_expect_success "'ipfs pin remote ls' for existing pins by CID" '
271
- ipfs pin remote ls --service=test_pin_svc --enc=json --cid=$HASH_B | tee ls_out &&
272
- test_expect_code 0 grep -q $HASH_B ls_out
273
- '
274
-
275
- test_expect_success "'ipfs pin remote ls' for existing pins by name" '
276
- ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_a | tee ls_out &&
277
- test_expect_code 0 grep -q $HASH_A ls_out
278
- '
279
-
280
- test_expect_success "'ipfs pin remote ls' for ongoing pins by status" '
281
- ipfs pin remote ls --service=test_pin_svc --status=queued,pinning | tee ls_out &&
282
- test_expect_code 0 grep -q $HASH_MISSING ls_out
283
- '
284
-
285
- # --force is required only when more than a single match is found,
286
- # so we add second pin with the same name (but different CID) to simulate that scenario
287
- test_expect_success "'ipfs pin remote rm --name' fails without --force when matching multiple pins" '
288
- test_expect_code 0 ipfs pin remote add --service=test_pin_svc --enc=json $BASE_ARGS --name=name_b $HASH_C &&
289
- test_expect_code 1 ipfs pin remote rm --service=test_pin_svc --name=name_b 2> rm_out &&
290
- echo "Error: multiple remote pins are matching this query, add --force to confirm the bulk removal" > rm_expected &&
291
- test_cmp rm_out rm_expected
292
- '
293
-
294
- test_expect_success "'ipfs pin remote rm --name' without --force did not remove matching pins" '
295
- ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_b | jq --raw-output .Cid | tee ls_out &&
296
- test_expect_code 0 grep -q $HASH_B ls_out &&
297
- test_expect_code 0 grep -q $HASH_C ls_out
298
- '
299
-
300
- test_expect_success "'ipfs pin remote rm --name' with --force removes all matching pins" '
301
- test_expect_code 0 ipfs pin remote rm --service=test_pin_svc --name=name_b --force &&
302
- ipfs pin remote ls --service=test_pin_svc --enc=json --name=name_b | jq --raw-output .Cid | tee ls_out &&
303
- test_expect_code 1 grep -q $HASH_B ls_out &&
304
- test_expect_code 1 grep -q $HASH_C ls_out
305
- '
306
-
307
- test_expect_success "'ipfs pin remote rm --force' removes all pinned items" '
308
- ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued,pinning,pinned,failed | jq --raw-output .Cid | tee ls_out &&
309
- test_expect_code 0 grep -q $HASH_A ls_out &&
310
- test_expect_code 0 grep -q $HASH_MISSING ls_out &&
311
- ipfs pin remote rm --service=test_pin_svc --status=queued,pinning,pinned,failed --force &&
312
- ipfs pin remote ls --service=test_pin_svc --enc=json --status=queued,pinning,pinned,failed | jq --raw-output .Cid | tee ls_out &&
313
- test_expect_code 1 grep -q $HASH_A ls_out &&
314
- test_expect_code 1 grep -q $HASH_MISSING ls_out
315
- '
316
-
317
-}
318
-
319
-test_remote_pins ""
320
-
321
-test_kill_ipfs_daemon
322
-
323
-WARNINGMESSAGE="WARNING: the local node is offline and remote pinning may fail if there is no other provider for this CID"
324
-
325
-test_expect_success "'ipfs pin remote add' shows the warning message while offline" '
326
- test_expect_code 0 ipfs pin remote add --service=test_pin_svc --background $BASE_ARGS --name=name_a $HASH_A > actual &&
327
- test_expect_code 0 grep -q "$WARNINGMESSAGE" actual
328
-'
329
-
330
-test_done
331
-
332
-# vim: ts=2 sw=2 sts=2 et: