| 1 | package pin |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "fmt" |
| 6 | "io" |
| 7 | "os" |
| 8 | "sort" |
| 9 | "strings" |
| 10 | "text/tabwriter" |
| 11 | "time" |
| 12 | |
| 13 | neturl "net/url" |
| 14 | gopath "path" |
| 15 | |
| 16 | "golang.org/x/sync/errgroup" |
| 17 | |
| 18 | pinclient "github.com/ipfs/boxo/pinning/remote/client" |
| 19 | cid "github.com/ipfs/go-cid" |
| 20 | cidenc "github.com/ipfs/go-cidutil/cidenc" |
| 21 | cmds "github.com/ipfs/go-ipfs-cmds" |
| 22 | logging "github.com/ipfs/go-log/v2" |
| 23 | config "github.com/ipfs/kubo/config" |
| 24 | "github.com/ipfs/kubo/core/commands/cmdenv" |
| 25 | "github.com/ipfs/kubo/core/commands/cmdutils" |
| 26 | fsrepo "github.com/ipfs/kubo/repo/fsrepo" |
| 27 | "github.com/libp2p/go-libp2p/core/host" |
| 28 | peer "github.com/libp2p/go-libp2p/core/peer" |
| 29 | ) |
| 30 | |
| 31 | var log = logging.Logger("core/commands/cmdenv") |
| 32 | |
| 33 | var remotePinCmd = &cmds.Command{ |
| 34 | Helptext: cmds.HelpText{ |
| 35 | Tagline: "Pin (and unpin) objects to remote pinning service.", |
| 36 | }, |
| 37 | |
| 38 | Subcommands: map[string]*cmds.Command{ |
| 39 | "add": addRemotePinCmd, |
| 40 | "ls": listRemotePinCmd, |
| 41 | "rm": rmRemotePinCmd, |
| 42 | "service": remotePinServiceCmd, |
| 43 | }, |
| 44 | } |
| 45 | |
| 46 | var remotePinServiceCmd = &cmds.Command{ |
| 47 | Helptext: cmds.HelpText{ |
| 48 | Tagline: "Configure remote pinning services.", |
| 49 | }, |
| 50 | |
| 51 | Subcommands: map[string]*cmds.Command{ |
| 52 | "add": addRemotePinServiceCmd, |
| 53 | "ls": lsRemotePinServiceCmd, |
| 54 | "rm": rmRemotePinServiceCmd, |
| 55 | }, |
| 56 | } |
| 57 | |
| 58 | const ( |
| 59 | pinNameOptionName = "name" |
| 60 | pinCIDsOptionName = "cid" |
| 61 | pinStatusOptionName = "status" |
| 62 | pinServiceNameOptionName = "service" |
| 63 | pinServiceNameArgName = pinServiceNameOptionName |
| 64 | pinServiceEndpointArgName = "endpoint" |
| 65 | pinServiceKeyArgName = "key" |
| 66 | pinServiceStatOptionName = "stat" |
| 67 | pinBackgroundOptionName = "background" |
| 68 | pinForceOptionName = "force" |
| 69 | ) |
| 70 | |
| 71 | type RemotePinOutput struct { |
| 72 | Status string |
| 73 | Cid string |
| 74 | Name string |
| 75 | } |
| 76 | |
| 77 | func toRemotePinOutput(ps pinclient.PinStatusGetter, enc cidenc.Encoder) RemotePinOutput { |
| 78 | return RemotePinOutput{ |
| 79 | Name: ps.GetPin().GetName(), |
| 80 | Status: ps.GetStatus().String(), |
| 81 | Cid: enc.Encode(ps.GetPin().GetCid()), |
| 82 | } |
| 83 | } |
| 84 | |
| 85 | func printRemotePinDetails(w io.Writer, out *RemotePinOutput) { |
| 86 | tw := tabwriter.NewWriter(w, 0, 0, 1, ' ', 0) |
| 87 | defer tw.Flush() |
| 88 | fw := func(k string, v string) { |
| 89 | fmt.Fprintf(tw, "%s:\t%s\n", k, v) |
| 90 | } |
| 91 | fw("CID", out.Cid) |
| 92 | fw("Name", out.Name) |
| 93 | fw("Status", out.Status) |
| 94 | } |
| 95 | |
| 96 | // remote pin commands |
| 97 | |
| 98 | var pinServiceNameOption = cmds.StringOption(pinServiceNameOptionName, "Name of the remote pinning service to use (mandatory).") |
| 99 | |
| 100 | var addRemotePinCmd = &cmds.Command{ |
| 101 | Helptext: cmds.HelpText{ |
| 102 | Tagline: "Pin object to remote pinning service.", |
| 103 | ShortDescription: "Asks remote pinning service to pin an IPFS object from a given path.", |
| 104 | LongDescription: ` |
| 105 | Asks remote pinning service to pin an IPFS object from a given path or a CID. |
| 106 | |
| 107 | To pin CID 'bafkqaaa' to service named 'mysrv' under a pin named 'mypin': |
| 108 | |
| 109 | $ ipfs pin remote add --service=mysrv --name=mypin bafkqaaa |
| 110 | |
| 111 | The above command will block until remote service returns 'pinned' status, |
| 112 | which may take time depending on the size and available providers of the pinned |
| 113 | data. |
| 114 | |
| 115 | If you prefer to not wait for pinning confirmation and return immediately |
| 116 | after remote service confirms 'queued' status, add the '--background' flag: |
| 117 | |
| 118 | $ ipfs pin remote add --service=mysrv --name=mypin --background bafkqaaa |
| 119 | |
| 120 | Status of background pin requests can be inspected with the 'ls' command. |
| 121 | |
| 122 | To list all pins for the CID across all statuses: |
| 123 | |
| 124 | $ ipfs pin remote ls --service=mysrv --cid=bafkqaaa --status=queued \ |
| 125 | --status=pinning --status=pinned --status=failed |
| 126 | |
| 127 | NOTE: a comma-separated notation is supported in CLI for convenience: |
| 128 | |
| 129 | $ ipfs pin remote ls --service=mysrv --cid=bafkqaaa --status=queued,pinning,pinned,failed |
| 130 | |
| 131 | `, |
| 132 | }, |
| 133 | |
| 134 | Arguments: []cmds.Argument{ |
| 135 | cmds.StringArg("ipfs-path", true, false, "CID or Path to be pinned."), |
| 136 | }, |
| 137 | Options: []cmds.Option{ |
| 138 | pinServiceNameOption, |
| 139 | cmds.StringOption(pinNameOptionName, "An optional name for the pin."), |
| 140 | cmds.BoolOption(pinBackgroundOptionName, "Add to the queue on the remote service and return immediately (does not wait for pinned status).").WithDefault(false), |
| 141 | }, |
| 142 | Type: RemotePinOutput{}, |
| 143 | Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { |
| 144 | ctx, cancel := context.WithCancel(req.Context) |
| 145 | defer cancel() |
| 146 | |
| 147 | enc, err := cmdenv.GetCidEncoder(req) |
| 148 | if err != nil { |
| 149 | return err |
| 150 | } |
| 151 | |
| 152 | // Get remote service |
| 153 | c, err := getRemotePinServiceFromRequest(req, env) |
| 154 | if err != nil { |
| 155 | return err |
| 156 | } |
| 157 | |
| 158 | // Prepare value for Pin.cid |
| 159 | if len(req.Arguments) != 1 { |
| 160 | return fmt.Errorf("expecting one CID argument") |
| 161 | } |
| 162 | api, err := cmdenv.GetApi(env, req) |
| 163 | if err != nil { |
| 164 | return err |
| 165 | } |
| 166 | p, err := cmdutils.PathOrCidPath(req.Arguments[0]) |
| 167 | if err != nil { |
| 168 | return err |
| 169 | } |
| 170 | |
| 171 | rp, _, err := api.ResolvePath(ctx, p) |
| 172 | if err != nil { |
| 173 | return err |
| 174 | } |
| 175 | |
| 176 | // Prepare Pin.name |
| 177 | opts := []pinclient.AddOption{} |
| 178 | if name, nameFound := req.Options[pinNameOptionName]; nameFound { |
| 179 | nameStr := name.(string) |
| 180 | // Validate pin name |
| 181 | if err := cmdutils.ValidatePinName(nameStr); err != nil { |
| 182 | return err |
| 183 | } |
| 184 | opts = append(opts, pinclient.PinOpts.WithName(nameStr)) |
| 185 | } |
| 186 | |
| 187 | // Prepare Pin.origins |
| 188 | // If CID in blockstore, add own multiaddrs to the 'origins' array |
| 189 | // so pinning service can use that as a hint and connect back to us. |
| 190 | node, err := cmdenv.GetNode(env) |
| 191 | if err != nil { |
| 192 | return err |
| 193 | } |
| 194 | |
| 195 | isInBlockstore, err := node.Blockstore.Has(req.Context, rp.RootCid()) |
| 196 | if err != nil { |
| 197 | return err |
| 198 | } |
| 199 | |
| 200 | if isInBlockstore && node.PeerHost != nil { |
| 201 | addrs, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(node.PeerHost)) |
| 202 | if err != nil { |
| 203 | return err |
| 204 | } |
| 205 | opts = append(opts, pinclient.PinOpts.WithOrigins(addrs...)) |
| 206 | } else if isInBlockstore && !node.IsOnline && cmds.GetEncoding(req, cmds.Text) == cmds.Text { |
| 207 | fmt.Fprintf(os.Stdout, "WARNING: the local node is offline and remote pinning may fail if there is no other provider for this CID\n") |
| 208 | } |
| 209 | |
| 210 | // Execute remote pin request |
| 211 | // TODO: fix panic when pinning service is down |
| 212 | ps, err := c.Add(ctx, rp.RootCid(), opts...) |
| 213 | if err != nil { |
| 214 | return err |
| 215 | } |
| 216 | |
| 217 | // Act on PinStatus.delegates |
| 218 | // If Pinning Service returned any delegates, proactively try to |
| 219 | // connect to them to facilitate data exchange without waiting for DHT |
| 220 | // lookup |
| 221 | for _, d := range ps.GetDelegates() { |
| 222 | // TODO: confirm this works as expected |
| 223 | p, err := peer.AddrInfoFromP2pAddr(d) |
| 224 | if err != nil { |
| 225 | return err |
| 226 | } |
| 227 | if err := api.Swarm().Connect(ctx, *p); err != nil { |
| 228 | log.Infof("error connecting to remote pin delegate %v : %w", d, err) |
| 229 | } |
| 230 | } |
| 231 | |
| 232 | // Block unless --background=true is passed |
| 233 | if !req.Options[pinBackgroundOptionName].(bool) { |
| 234 | const pinWaitTime = 500 * time.Millisecond |
| 235 | var timer *time.Timer |
| 236 | requestID := ps.GetRequestId() |
| 237 | for { |
| 238 | ps, err = c.GetStatusByID(ctx, requestID) |
| 239 | if err != nil { |
| 240 | return fmt.Errorf("failed to check pin status for requestid=%q due to error: %v", requestID, err) |
| 241 | } |
| 242 | if ps.GetRequestId() != requestID { |
| 243 | return fmt.Errorf("failed to check pin status for requestid=%q, remote service sent unexpected requestid=%q", requestID, ps.GetRequestId()) |
| 244 | } |
| 245 | s := ps.GetStatus() |
| 246 | if s == pinclient.StatusPinned { |
| 247 | break |
| 248 | } |
| 249 | if s == pinclient.StatusFailed { |
| 250 | return fmt.Errorf("remote service failed to pin requestid=%q", requestID) |
| 251 | } |
| 252 | if timer == nil { |
| 253 | timer = time.NewTimer(pinWaitTime) |
| 254 | } else { |
| 255 | timer.Reset(pinWaitTime) |
| 256 | } |
| 257 | select { |
| 258 | case <-timer.C: |
| 259 | case <-ctx.Done(): |
| 260 | timer.Stop() |
| 261 | return fmt.Errorf("waiting for pin interrupted, requestid=%q remains on remote service", requestID) |
| 262 | } |
| 263 | } |
| 264 | } |
| 265 | |
| 266 | return res.Emit(toRemotePinOutput(ps, enc)) |
| 267 | }, |
| 268 | Encoders: cmds.EncoderMap{ |
| 269 | cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *RemotePinOutput) error { |
| 270 | printRemotePinDetails(w, out) |
| 271 | return nil |
| 272 | }), |
| 273 | }, |
| 274 | } |
| 275 | |
| 276 | var listRemotePinCmd = &cmds.Command{ |
| 277 | Helptext: cmds.HelpText{ |
| 278 | Tagline: "List objects pinned to remote pinning service.", |
| 279 | ShortDescription: ` |
| 280 | Returns a list of objects that are pinned to a remote pinning service. |
| 281 | `, |
| 282 | LongDescription: ` |
| 283 | Returns a list of objects that are pinned to a remote pinning service. |
| 284 | |
| 285 | NOTE: By default, it will only show matching objects in 'pinned' state. |
| 286 | Pass '--status=queued,pinning,pinned,failed' to list pins in all states. |
| 287 | `, |
| 288 | }, |
| 289 | |
| 290 | Arguments: []cmds.Argument{}, |
| 291 | Options: []cmds.Option{ |
| 292 | pinServiceNameOption, |
| 293 | cmds.StringOption(pinNameOptionName, "Return pins with names that contain the value provided (case-sensitive, exact match)."), |
| 294 | cmds.DelimitedStringsOption(",", pinCIDsOptionName, "Return pins for the specified CIDs (comma-separated)."), |
| 295 | cmds.DelimitedStringsOption(",", pinStatusOptionName, "Return pins with the specified statuses (queued,pinning,pinned,failed).").WithDefault([]string{"pinned"}), |
| 296 | }, |
| 297 | Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { |
| 298 | c, err := getRemotePinServiceFromRequest(req, env) |
| 299 | if err != nil { |
| 300 | return err |
| 301 | } |
| 302 | |
| 303 | enc, err := cmdenv.GetCidEncoder(req) |
| 304 | if err != nil { |
| 305 | return err |
| 306 | } |
| 307 | |
| 308 | ctx, cancel := context.WithCancel(req.Context) |
| 309 | defer cancel() |
| 310 | |
| 311 | psCh := make(chan pinclient.PinStatusGetter) |
| 312 | lsErr := make(chan error, 1) |
| 313 | go func() { |
| 314 | lsErr <- lsRemote(ctx, req, c, psCh) |
| 315 | }() |
| 316 | for ps := range psCh { |
| 317 | if err := res.Emit(toRemotePinOutput(ps, enc)); err != nil { |
| 318 | return err |
| 319 | } |
| 320 | } |
| 321 | |
| 322 | return <-lsErr |
| 323 | }, |
| 324 | Type: RemotePinOutput{}, |
| 325 | Encoders: cmds.EncoderMap{ |
| 326 | cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *RemotePinOutput) error { |
| 327 | // pin remote ls produces a flat output similar to legacy pin ls |
| 328 | fmt.Fprintf(w, "%s\t%s\t%s\n", out.Cid, out.Status, cmdenv.EscNonPrint(out.Name)) |
| 329 | return nil |
| 330 | }), |
| 331 | }, |
| 332 | } |
| 333 | |
| 334 | // Executes GET /pins/?query-with-filters |
| 335 | func lsRemote(ctx context.Context, req *cmds.Request, c *pinclient.Client, out chan<- pinclient.PinStatusGetter) error { |
| 336 | opts := []pinclient.LsOption{} |
| 337 | if name, nameFound := req.Options[pinNameOptionName]; nameFound { |
| 338 | nameStr := name.(string) |
| 339 | // Validate name filter |
| 340 | if err := cmdutils.ValidatePinName(nameStr); err != nil { |
| 341 | close(out) |
| 342 | return err |
| 343 | } |
| 344 | opts = append(opts, pinclient.PinOpts.FilterName(nameStr)) |
| 345 | } |
| 346 | |
| 347 | if cidsRaw, cidsFound := req.Options[pinCIDsOptionName]; cidsFound { |
| 348 | cidsRawArr := cidsRaw.([]string) |
| 349 | var parsedCIDs []cid.Cid |
| 350 | for _, rawCID := range cidsRawArr { |
| 351 | parsedCID, err := cid.Decode(rawCID) |
| 352 | if err != nil { |
| 353 | close(out) |
| 354 | return fmt.Errorf("CID %q cannot be parsed: %v", rawCID, err) |
| 355 | } |
| 356 | parsedCIDs = append(parsedCIDs, parsedCID) |
| 357 | } |
| 358 | opts = append(opts, pinclient.PinOpts.FilterCIDs(parsedCIDs...)) |
| 359 | } |
| 360 | if statusRaw, statusFound := req.Options[pinStatusOptionName]; statusFound { |
| 361 | statusRawArr := statusRaw.([]string) |
| 362 | var parsedStatuses []pinclient.Status |
| 363 | for _, rawStatus := range statusRawArr { |
| 364 | s := pinclient.Status(rawStatus) |
| 365 | if s.String() == string(pinclient.StatusUnknown) { |
| 366 | close(out) |
| 367 | return fmt.Errorf("status %q is not valid", rawStatus) |
| 368 | } |
| 369 | parsedStatuses = append(parsedStatuses, s) |
| 370 | } |
| 371 | opts = append(opts, pinclient.PinOpts.FilterStatus(parsedStatuses...)) |
| 372 | } |
| 373 | |
| 374 | return c.Ls(ctx, out, opts...) |
| 375 | } |
| 376 | |
| 377 | var rmRemotePinCmd = &cmds.Command{ |
| 378 | Helptext: cmds.HelpText{ |
| 379 | Tagline: "Remove pins from remote pinning service.", |
| 380 | ShortDescription: "Removes the remote pin, allowing it to be garbage-collected if needed.", |
| 381 | LongDescription: ` |
| 382 | Removes remote pins, allowing them to be garbage-collected if needed. |
| 383 | |
| 384 | This command accepts the same search query parameters as 'ls', and it is good |
| 385 | practice to execute 'ls' before 'rm' to confirm the list of pins to be removed. |
| 386 | |
| 387 | To remove a single pin for a specific CID: |
| 388 | |
| 389 | $ ipfs pin remote ls --service=mysrv --cid=bafkqaaa |
| 390 | $ ipfs pin remote rm --service=mysrv --cid=bafkqaaa |
| 391 | |
| 392 | When more than one pin matches the query on the remote service, an error is |
| 393 | returned. To confirm the removal of multiple pins, pass '--force': |
| 394 | |
| 395 | $ ipfs pin remote ls --service=mysrv --name=popular-name |
| 396 | $ ipfs pin remote rm --service=mysrv --name=popular-name --force |
| 397 | |
| 398 | NOTE: When no '--status' is passed, implicit '--status=pinned' is used. |
| 399 | To list and then remove all pending pin requests, pass an explicit status list: |
| 400 | |
| 401 | $ ipfs pin remote ls --service=mysrv --status=queued,pinning,failed |
| 402 | $ ipfs pin remote rm --service=mysrv --status=queued,pinning,failed --force |
| 403 | |
| 404 | `, |
| 405 | }, |
| 406 | |
| 407 | Arguments: []cmds.Argument{}, |
| 408 | Options: []cmds.Option{ |
| 409 | pinServiceNameOption, |
| 410 | cmds.StringOption(pinNameOptionName, "Remove pins with names that contain provided value (case-sensitive, exact match)."), |
| 411 | cmds.DelimitedStringsOption(",", pinCIDsOptionName, "Remove pins for the specified CIDs."), |
| 412 | cmds.DelimitedStringsOption(",", pinStatusOptionName, "Remove pins with the specified statuses (queued,pinning,pinned,failed).").WithDefault([]string{"pinned"}), |
| 413 | cmds.BoolOption(pinForceOptionName, "Allow removal of multiple pins matching the query without additional confirmation.").WithDefault(false), |
| 414 | }, |
| 415 | Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { |
| 416 | c, err := getRemotePinServiceFromRequest(req, env) |
| 417 | if err != nil { |
| 418 | return err |
| 419 | } |
| 420 | |
| 421 | rmIDs := []string{} |
| 422 | if len(req.Arguments) != 0 { |
| 423 | return fmt.Errorf("unexpected argument %q", req.Arguments[0]) |
| 424 | } |
| 425 | |
| 426 | psCh := make(chan pinclient.PinStatusGetter) |
| 427 | errCh := make(chan error, 1) |
| 428 | ctx, cancel := context.WithCancel(req.Context) |
| 429 | defer cancel() |
| 430 | |
| 431 | go func() { |
| 432 | errCh <- lsRemote(ctx, req, c, psCh) |
| 433 | }() |
| 434 | for ps := range psCh { |
| 435 | rmIDs = append(rmIDs, ps.GetRequestId()) |
| 436 | } |
| 437 | if err = <-errCh; err != nil { |
| 438 | return fmt.Errorf("error while listing remote pins: %v", err) |
| 439 | } |
| 440 | |
| 441 | if len(rmIDs) > 1 && !req.Options[pinForceOptionName].(bool) { |
| 442 | return fmt.Errorf("multiple remote pins are matching this query, add --force to confirm the bulk removal") |
| 443 | } |
| 444 | |
| 445 | for _, rmID := range rmIDs { |
| 446 | if err = c.DeleteByID(ctx, rmID); err != nil { |
| 447 | return fmt.Errorf("removing pin identified by requestid=%q failed: %v", rmID, err) |
| 448 | } |
| 449 | } |
| 450 | return nil |
| 451 | }, |
| 452 | } |
| 453 | |
| 454 | // remote service commands |
| 455 | |
| 456 | var addRemotePinServiceCmd = &cmds.Command{ |
| 457 | Helptext: cmds.HelpText{ |
| 458 | Tagline: "Add remote pinning service.", |
| 459 | ShortDescription: "Add credentials for access to a remote pinning service.", |
| 460 | LongDescription: ` |
| 461 | Add credentials for access to a remote pinning service and store them in the |
| 462 | config under Pinning.RemoteServices map. |
| 463 | |
| 464 | TIP: |
| 465 | |
| 466 | To add services and test them by fetching pin count stats: |
| 467 | |
| 468 | $ ipfs pin remote service add goodsrv https://pin-api.example.com secret-key |
| 469 | $ ipfs pin remote service add badsrv https://bad-api.example.com invalid-key |
| 470 | $ ipfs pin remote service ls --stat |
| 471 | goodsrv https://pin-api.example.com 0/0/0/0 |
| 472 | badsrv https://bad-api.example.com invalid |
| 473 | |
| 474 | `, |
| 475 | }, |
| 476 | Arguments: []cmds.Argument{ |
| 477 | cmds.StringArg(pinServiceNameArgName, true, false, "Service name."), |
| 478 | cmds.StringArg(pinServiceEndpointArgName, true, false, "Service endpoint."), |
| 479 | cmds.StringArg(pinServiceKeyArgName, true, false, "Service key."), |
| 480 | }, |
| 481 | Type: nil, |
| 482 | Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { |
| 483 | cfgRoot, err := cmdenv.GetConfigRoot(env) |
| 484 | if err != nil { |
| 485 | return err |
| 486 | } |
| 487 | repo, err := fsrepo.Open(cfgRoot) |
| 488 | if err != nil { |
| 489 | return err |
| 490 | } |
| 491 | defer repo.Close() |
| 492 | |
| 493 | if len(req.Arguments) < 3 { |
| 494 | return fmt.Errorf("expecting three arguments: service name, endpoint and key") |
| 495 | } |
| 496 | |
| 497 | name := req.Arguments[0] |
| 498 | endpoint, err := normalizeEndpoint(req.Arguments[1]) |
| 499 | if err != nil { |
| 500 | return err |
| 501 | } |
| 502 | key := req.Arguments[2] |
| 503 | |
| 504 | cfg, err := repo.Config() |
| 505 | if err != nil { |
| 506 | return err |
| 507 | } |
| 508 | if cfg.Pinning.RemoteServices != nil { |
| 509 | if _, present := cfg.Pinning.RemoteServices[name]; present { |
| 510 | return fmt.Errorf("service already present") |
| 511 | } |
| 512 | } else { |
| 513 | cfg.Pinning.RemoteServices = map[string]config.RemotePinningService{} |
| 514 | } |
| 515 | |
| 516 | cfg.Pinning.RemoteServices[name] = config.RemotePinningService{ |
| 517 | API: config.RemotePinningServiceAPI{ |
| 518 | Endpoint: endpoint, |
| 519 | Key: key, |
| 520 | }, |
| 521 | Policies: config.RemotePinningServicePolicies{}, |
| 522 | } |
| 523 | |
| 524 | return repo.SetConfig(cfg) |
| 525 | }, |
| 526 | } |
| 527 | |
| 528 | var rmRemotePinServiceCmd = &cmds.Command{ |
| 529 | Helptext: cmds.HelpText{ |
| 530 | Tagline: "Remove remote pinning service.", |
| 531 | ShortDescription: "Remove credentials for access to a remote pinning service.", |
| 532 | }, |
| 533 | Arguments: []cmds.Argument{ |
| 534 | cmds.StringArg(pinServiceNameOptionName, true, false, "Name of remote pinning service to remove."), |
| 535 | }, |
| 536 | Options: []cmds.Option{}, |
| 537 | Type: nil, |
| 538 | Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { |
| 539 | cfgRoot, err := cmdenv.GetConfigRoot(env) |
| 540 | if err != nil { |
| 541 | return err |
| 542 | } |
| 543 | repo, err := fsrepo.Open(cfgRoot) |
| 544 | if err != nil { |
| 545 | return err |
| 546 | } |
| 547 | defer repo.Close() |
| 548 | |
| 549 | if len(req.Arguments) != 1 { |
| 550 | return fmt.Errorf("expecting one argument: name") |
| 551 | } |
| 552 | name := req.Arguments[0] |
| 553 | |
| 554 | cfg, err := repo.Config() |
| 555 | if err != nil { |
| 556 | return err |
| 557 | } |
| 558 | if cfg.Pinning.RemoteServices != nil { |
| 559 | delete(cfg.Pinning.RemoteServices, name) |
| 560 | } |
| 561 | return repo.SetConfig(cfg) |
| 562 | }, |
| 563 | } |
| 564 | |
| 565 | var lsRemotePinServiceCmd = &cmds.Command{ |
| 566 | Helptext: cmds.HelpText{ |
| 567 | Tagline: "List remote pinning services.", |
| 568 | ShortDescription: "List remote pinning services.", |
| 569 | LongDescription: ` |
| 570 | List remote pinning services. |
| 571 | |
| 572 | By default, only a name and an endpoint are listed; however, one can pass |
| 573 | '--stat' to test each endpoint by fetching pin counts for each state: |
| 574 | |
| 575 | $ ipfs pin remote service ls --stat |
| 576 | goodsrv https://pin-api.example.com 0/0/0/0 |
| 577 | badsrv https://bad-api.example.com invalid |
| 578 | |
| 579 | TIP: pass '--enc=json' for more useful JSON output. |
| 580 | `, |
| 581 | }, |
| 582 | Arguments: []cmds.Argument{}, |
| 583 | Options: []cmds.Option{ |
| 584 | cmds.BoolOption(pinServiceStatOptionName, "Try to fetch and display current pin count on remote service (queued/pinning/pinned/failed).").WithDefault(false), |
| 585 | }, |
| 586 | Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { |
| 587 | ctx, cancel := context.WithCancel(req.Context) |
| 588 | defer cancel() |
| 589 | |
| 590 | cfgRoot, err := cmdenv.GetConfigRoot(env) |
| 591 | if err != nil { |
| 592 | return err |
| 593 | } |
| 594 | repo, err := fsrepo.Open(cfgRoot) |
| 595 | if err != nil { |
| 596 | return err |
| 597 | } |
| 598 | defer repo.Close() |
| 599 | |
| 600 | cfg, err := repo.Config() |
| 601 | if err != nil { |
| 602 | return err |
| 603 | } |
| 604 | if cfg.Pinning.RemoteServices == nil { |
| 605 | return cmds.EmitOnce(res, &PinServicesList{make([]ServiceDetails, 0)}) |
| 606 | } |
| 607 | services := cfg.Pinning.RemoteServices |
| 608 | result := PinServicesList{make([]ServiceDetails, 0, len(services))} |
| 609 | for svcName, svcConfig := range services { |
| 610 | svcDetails := ServiceDetails{svcName, svcConfig.API.Endpoint, nil} |
| 611 | |
| 612 | // if --pin-count is passed, we try to fetch pin numbers from remote service |
| 613 | if req.Options[pinServiceStatOptionName].(bool) { |
| 614 | lsRemotePinCount := func(ctx context.Context, env cmds.Environment, svcName string) (*PinCount, error) { |
| 615 | c, err := getRemotePinService(env, svcName) |
| 616 | if err != nil { |
| 617 | return nil, err |
| 618 | } |
| 619 | // we only care about total count, so requesting smallest batch |
| 620 | batch := pinclient.PinOpts.Limit(1) |
| 621 | fs := pinclient.PinOpts.FilterStatus |
| 622 | |
| 623 | statuses := []pinclient.Status{ |
| 624 | pinclient.StatusQueued, |
| 625 | pinclient.StatusPinning, |
| 626 | pinclient.StatusPinned, |
| 627 | pinclient.StatusFailed, |
| 628 | } |
| 629 | |
| 630 | g, ctx := errgroup.WithContext(ctx) |
| 631 | pc := &PinCount{} |
| 632 | |
| 633 | for _, s := range statuses { |
| 634 | status := s // lol https://golang.org/doc/faq#closures_and_goroutines |
| 635 | g.Go(func() error { |
| 636 | _, n, err := c.LsBatchSync(ctx, batch, fs(status)) |
| 637 | if err != nil { |
| 638 | return err |
| 639 | } |
| 640 | switch status { |
| 641 | case pinclient.StatusQueued: |
| 642 | pc.Queued = n |
| 643 | case pinclient.StatusPinning: |
| 644 | pc.Pinning = n |
| 645 | case pinclient.StatusPinned: |
| 646 | pc.Pinned = n |
| 647 | case pinclient.StatusFailed: |
| 648 | pc.Failed = n |
| 649 | } |
| 650 | return nil |
| 651 | }) |
| 652 | } |
| 653 | if err := g.Wait(); err != nil { |
| 654 | return nil, err |
| 655 | } |
| 656 | |
| 657 | return pc, nil |
| 658 | } |
| 659 | |
| 660 | pinCount, err := lsRemotePinCount(ctx, env, svcName) |
| 661 | |
| 662 | // PinCount is present only if we were able to fetch counts. |
| 663 | // We don't want to break listing of services so this is best-effort. |
| 664 | // (verbose err is returned by 'pin remote ls', if needed) |
| 665 | svcDetails.Stat = &Stat{} |
| 666 | if err == nil { |
| 667 | svcDetails.Stat.Status = "valid" |
| 668 | svcDetails.Stat.PinCount = pinCount |
| 669 | } else { |
| 670 | svcDetails.Stat.Status = "invalid" |
| 671 | } |
| 672 | } |
| 673 | result.RemoteServices = append(result.RemoteServices, svcDetails) |
| 674 | } |
| 675 | sort.Sort(result) |
| 676 | return cmds.EmitOnce(res, &result) |
| 677 | }, |
| 678 | Type: PinServicesList{}, |
| 679 | Encoders: cmds.EncoderMap{ |
| 680 | cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, list *PinServicesList) error { |
| 681 | tw := tabwriter.NewWriter(w, 1, 2, 1, ' ', 0) |
| 682 | withStat := req.Options[pinServiceStatOptionName].(bool) |
| 683 | for _, s := range list.RemoteServices { |
| 684 | if withStat { |
| 685 | stat := s.Stat.Status |
| 686 | pc := s.Stat.PinCount |
| 687 | if s.Stat.PinCount != nil { |
| 688 | stat = fmt.Sprintf("%d/%d/%d/%d", pc.Queued, pc.Pinning, pc.Pinned, pc.Failed) |
| 689 | } |
| 690 | fmt.Fprintf(tw, "%s\t%s\t%s\n", s.Service, s.ApiEndpoint, stat) |
| 691 | } else { |
| 692 | fmt.Fprintf(tw, "%s\t%s\n", s.Service, s.ApiEndpoint) |
| 693 | } |
| 694 | } |
| 695 | tw.Flush() |
| 696 | return nil |
| 697 | }), |
| 698 | }, |
| 699 | } |
| 700 | |
| 701 | type ServiceDetails struct { |
| 702 | Service string |
| 703 | ApiEndpoint string //nolint |
| 704 | Stat *Stat `json:",omitempty"` // present only when --stat not passed |
| 705 | } |
| 706 | |
| 707 | type Stat struct { |
| 708 | Status string |
| 709 | PinCount *PinCount `json:",omitempty"` // missing when --stat is passed but the service is offline |
| 710 | } |
| 711 | |
| 712 | type PinCount struct { |
| 713 | Queued int |
| 714 | Pinning int |
| 715 | Pinned int |
| 716 | Failed int |
| 717 | } |
| 718 | |
| 719 | // Struct returned by ipfs pin remote service ls --enc=json | jq |
| 720 | type PinServicesList struct { |
| 721 | RemoteServices []ServiceDetails |
| 722 | } |
| 723 | |
| 724 | func (l PinServicesList) Len() int { |
| 725 | return len(l.RemoteServices) |
| 726 | } |
| 727 | |
| 728 | func (l PinServicesList) Swap(i, j int) { |
| 729 | s := l.RemoteServices |
| 730 | s[i], s[j] = s[j], s[i] |
| 731 | } |
| 732 | |
| 733 | func (l PinServicesList) Less(i, j int) bool { |
| 734 | s := l.RemoteServices |
| 735 | return s[i].Service < s[j].Service |
| 736 | } |
| 737 | |
| 738 | func getRemotePinServiceFromRequest(req *cmds.Request, env cmds.Environment) (*pinclient.Client, error) { |
| 739 | service, serviceFound := req.Options[pinServiceNameOptionName] |
| 740 | if !serviceFound { |
| 741 | return nil, fmt.Errorf("a service name must be passed") |
| 742 | } |
| 743 | |
| 744 | serviceStr := service.(string) |
| 745 | var err error |
| 746 | c, err := getRemotePinService(env, serviceStr) |
| 747 | if err != nil { |
| 748 | return nil, err |
| 749 | } |
| 750 | |
| 751 | return c, nil |
| 752 | } |
| 753 | |
| 754 | func getRemotePinService(env cmds.Environment, name string) (*pinclient.Client, error) { |
| 755 | if name == "" { |
| 756 | return nil, fmt.Errorf("remote pinning service name not specified") |
| 757 | } |
| 758 | endpoint, key, err := getRemotePinServiceInfo(env, name) |
| 759 | if err != nil { |
| 760 | return nil, err |
| 761 | } |
| 762 | return pinclient.NewClient(endpoint, key), nil |
| 763 | } |
| 764 | |
| 765 | func getRemotePinServiceInfo(env cmds.Environment, name string) (endpoint, key string, err error) { |
| 766 | cfgRoot, err := cmdenv.GetConfigRoot(env) |
| 767 | if err != nil { |
| 768 | return "", "", err |
| 769 | } |
| 770 | repo, err := fsrepo.Open(cfgRoot) |
| 771 | if err != nil { |
| 772 | return "", "", err |
| 773 | } |
| 774 | defer repo.Close() |
| 775 | cfg, err := repo.Config() |
| 776 | if err != nil { |
| 777 | return "", "", err |
| 778 | } |
| 779 | if cfg.Pinning.RemoteServices == nil { |
| 780 | return "", "", fmt.Errorf("service not known") |
| 781 | } |
| 782 | service, present := cfg.Pinning.RemoteServices[name] |
| 783 | if !present { |
| 784 | return "", "", fmt.Errorf("service not known") |
| 785 | } |
| 786 | endpoint, err = normalizeEndpoint(service.API.Endpoint) |
| 787 | if err != nil { |
| 788 | return "", "", err |
| 789 | } |
| 790 | return endpoint, service.API.Key, nil |
| 791 | } |
| 792 | |
| 793 | func normalizeEndpoint(endpoint string) (string, error) { |
| 794 | uri, err := neturl.ParseRequestURI(endpoint) |
| 795 | if err != nil || !(uri.Scheme == "http" || uri.Scheme == "https") { |
| 796 | return "", fmt.Errorf("service endpoint must be a valid HTTP URL") |
| 797 | } |
| 798 | |
| 799 | // cleanup trailing and duplicate slashes (https://github.com/ipfs/kubo/issues/7826) |
| 800 | uri.Path = gopath.Clean(uri.Path) |
| 801 | uri.Path = strings.TrimSuffix(uri.Path, ".") |
| 802 | uri.Path = strings.TrimSuffix(uri.Path, "/") |
| 803 | |
| 804 | // remove any query params |
| 805 | if uri.RawQuery != "" { |
| 806 | return "", fmt.Errorf("service endpoint should be provided without any query parameters") |
| 807 | } |
| 808 | |
| 809 | if strings.HasSuffix(uri.Path, "/pins") { |
| 810 | return "", fmt.Errorf("service endpoint should be provided without the /pins suffix") |
| 811 | } |
| 812 | |
| 813 | return uri.String(), nil |
| 814 | } |