| 1 | package pin |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "fmt" |
| 7 | "io" |
| 8 | "os" |
| 9 | "time" |
| 10 | |
| 11 | "github.com/dustin/go-humanize" |
| 12 | bserv "github.com/ipfs/boxo/blockservice" |
| 13 | offline "github.com/ipfs/boxo/exchange/offline" |
| 14 | dag "github.com/ipfs/boxo/ipld/merkledag" |
| 15 | pin "github.com/ipfs/boxo/pinning/pinner" |
| 16 | verifcid "github.com/ipfs/boxo/verifcid" |
| 17 | cid "github.com/ipfs/go-cid" |
| 18 | cidenc "github.com/ipfs/go-cidutil/cidenc" |
| 19 | cmds "github.com/ipfs/go-ipfs-cmds" |
| 20 | coreiface "github.com/ipfs/kubo/core/coreiface" |
| 21 | options "github.com/ipfs/kubo/core/coreiface/options" |
| 22 | |
| 23 | config "github.com/ipfs/kubo/config" |
| 24 | core "github.com/ipfs/kubo/core" |
| 25 | cmdenv "github.com/ipfs/kubo/core/commands/cmdenv" |
| 26 | "github.com/ipfs/kubo/core/commands/cmdutils" |
| 27 | e "github.com/ipfs/kubo/core/commands/e" |
| 28 | ) |
| 29 | |
| 30 | var PinCmd = &cmds.Command{ |
| 31 | Helptext: cmds.HelpText{ |
| 32 | Tagline: "Pin (and unpin) objects to local storage.", |
| 33 | }, |
| 34 | |
| 35 | Subcommands: map[string]*cmds.Command{ |
| 36 | "add": addPinCmd, |
| 37 | "rm": rmPinCmd, |
| 38 | "ls": listPinCmd, |
| 39 | "verify": verifyPinCmd, |
| 40 | "update": updatePinCmd, |
| 41 | "remote": remotePinCmd, |
| 42 | }, |
| 43 | } |
| 44 | |
| 45 | type PinOutput struct { |
| 46 | Pins []string |
| 47 | } |
| 48 | |
| 49 | type AddPinOutput struct { |
| 50 | Pins []string `json:",omitempty"` |
| 51 | Progress int `json:",omitempty"` |
| 52 | Bytes uint64 `json:",omitempty"` |
| 53 | } |
| 54 | |
| 55 | const ( |
| 56 | pinRecursiveOptionName = "recursive" |
| 57 | pinProgressOptionName = "progress" |
| 58 | fastProvideRootOptionName = "fast-provide-root" |
| 59 | fastProvideDAGOptionName = "fast-provide-dag" |
| 60 | fastProvideWaitOptionName = "fast-provide-wait" |
| 61 | ) |
| 62 | |
| 63 | var addPinCmd = &cmds.Command{ |
| 64 | Helptext: cmds.HelpText{ |
| 65 | Tagline: "Pin objects to local storage.", |
| 66 | ShortDescription: "Stores an IPFS object(s) from a given path locally to disk.", |
| 67 | LongDescription: ` |
| 68 | Create a pin for the given object, protecting resolved CID from being garbage |
| 69 | collected. |
| 70 | |
| 71 | An optional name can be provided, and read back via 'ipfs pin ls --names'. |
| 72 | |
| 73 | Be mindful of defaults: |
| 74 | |
| 75 | Default pin type is 'recursive' (entire DAG). |
| 76 | Pass -r=false to create a direct pin for a single block. |
| 77 | Use 'pin ls -t recursive' to only list roots of recursively pinned DAGs |
| 78 | (significantly faster when many big DAGs are pinned recursively) |
| 79 | |
| 80 | Default pin name is empty. Pass '--name' to 'pin add' to set one |
| 81 | and use 'pin ls --names' to see it. Pinning a second time with a different |
| 82 | name will update the name of the pin. |
| 83 | |
| 84 | If daemon is running, any missing blocks will be retrieved from the network. |
| 85 | It may take some time. Pass '--progress' to track the progress. |
| 86 | `, |
| 87 | }, |
| 88 | |
| 89 | Arguments: []cmds.Argument{ |
| 90 | cmds.StringArg("ipfs-path", true, true, "Path to object(s) to be pinned.").EnableStdin(), |
| 91 | }, |
| 92 | Options: []cmds.Option{ |
| 93 | cmds.BoolOption(pinRecursiveOptionName, "r", "Recursively pin the object linked to by the specified object(s).").WithDefault(true), |
| 94 | cmds.StringOption(pinNameOptionName, "n", "An optional name for created pin(s)."), |
| 95 | cmds.BoolOption(pinProgressOptionName, "Show progress"), |
| 96 | cmds.BoolOption(fastProvideRootOptionName, "Immediately provide root CID to DHT after pinning. Default: Import.FastProvideRoot"), |
| 97 | cmds.BoolOption(fastProvideDAGOptionName, "Walk and provide the full DAG according to Provide.Strategy after pinning. Default: Import.FastProvideDAG"), |
| 98 | cmds.BoolOption(fastProvideWaitOptionName, "Block until the immediate provide completes. Default: Import.FastProvideWait"), |
| 99 | }, |
| 100 | Type: AddPinOutput{}, |
| 101 | Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { |
| 102 | api, err := cmdenv.GetApi(env, req) |
| 103 | if err != nil { |
| 104 | return err |
| 105 | } |
| 106 | |
| 107 | // set recursive flag |
| 108 | recursive, _ := req.Options[pinRecursiveOptionName].(bool) |
| 109 | name, _ := req.Options[pinNameOptionName].(string) |
| 110 | showProgress, _ := req.Options[pinProgressOptionName].(bool) |
| 111 | |
| 112 | // Validate pin name |
| 113 | if err := cmdutils.ValidatePinName(name); err != nil { |
| 114 | return err |
| 115 | } |
| 116 | |
| 117 | if err := req.ParseBodyArgs(); err != nil { |
| 118 | return err |
| 119 | } |
| 120 | |
| 121 | enc, err := cmdenv.GetCidEncoder(req) |
| 122 | if err != nil { |
| 123 | return err |
| 124 | } |
| 125 | |
| 126 | nd, fpRoot, fpDAG, fpWait := resolveFastProvideFlags(req, env) |
| 127 | |
| 128 | if !showProgress { |
| 129 | added, err := pinAddMany(req.Context, api, enc, req.Arguments, recursive, name) |
| 130 | if err != nil { |
| 131 | return err |
| 132 | } |
| 133 | |
| 134 | fastProvideAfterPin(req, nd, fpRoot, fpDAG, fpWait, added) |
| 135 | return cmds.EmitOnce(res, &AddPinOutput{Pins: added}) |
| 136 | } |
| 137 | |
| 138 | v := new(dag.ProgressTracker) |
| 139 | ctx := v.DeriveContext(req.Context) |
| 140 | |
| 141 | type pinResult struct { |
| 142 | pins []string |
| 143 | err error |
| 144 | } |
| 145 | |
| 146 | ch := make(chan pinResult, 1) |
| 147 | go func() { |
| 148 | added, err := pinAddMany(ctx, api, enc, req.Arguments, recursive, name) |
| 149 | ch <- pinResult{pins: added, err: err} |
| 150 | }() |
| 151 | |
| 152 | ticker := time.NewTicker(500 * time.Millisecond) |
| 153 | defer ticker.Stop() |
| 154 | |
| 155 | for { |
| 156 | select { |
| 157 | case val := <-ch: |
| 158 | if val.err != nil { |
| 159 | return val.err |
| 160 | } |
| 161 | |
| 162 | fastProvideAfterPin(req, nd, fpRoot, fpDAG, fpWait, val.pins) |
| 163 | |
| 164 | if ps := v.ProgressStat(); ps.Nodes != 0 { |
| 165 | if err := res.Emit(&AddPinOutput{Progress: ps.Nodes, Bytes: ps.Bytes}); err != nil { |
| 166 | return err |
| 167 | } |
| 168 | } |
| 169 | return res.Emit(&AddPinOutput{Pins: val.pins}) |
| 170 | case <-ticker.C: |
| 171 | ps := v.ProgressStat() |
| 172 | if err := res.Emit(&AddPinOutput{Progress: ps.Nodes, Bytes: ps.Bytes}); err != nil { |
| 173 | return err |
| 174 | } |
| 175 | case <-ctx.Done(): |
| 176 | log.Error(ctx.Err()) |
| 177 | return ctx.Err() |
| 178 | } |
| 179 | } |
| 180 | }, |
| 181 | Encoders: cmds.EncoderMap{ |
| 182 | cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *AddPinOutput) error { |
| 183 | rec, found := req.Options["recursive"].(bool) |
| 184 | var pintype string |
| 185 | if rec || !found { |
| 186 | pintype = "recursively" |
| 187 | } else { |
| 188 | pintype = "directly" |
| 189 | } |
| 190 | |
| 191 | for _, k := range out.Pins { |
| 192 | fmt.Fprintf(w, "pinned %s %s\n", k, pintype) |
| 193 | } |
| 194 | |
| 195 | return nil |
| 196 | }), |
| 197 | }, |
| 198 | PostRun: cmds.PostRunMap{ |
| 199 | cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { |
| 200 | for { |
| 201 | v, err := res.Next() |
| 202 | if err != nil { |
| 203 | if err == io.EOF { |
| 204 | return nil |
| 205 | } |
| 206 | return err |
| 207 | } |
| 208 | |
| 209 | out, ok := v.(*AddPinOutput) |
| 210 | if !ok { |
| 211 | return e.TypeErr(out, v) |
| 212 | } |
| 213 | if out.Pins == nil { |
| 214 | // this can only happen if the progress option is set |
| 215 | fmt.Fprintf(os.Stderr, "Fetched/Processed %d nodes (%s)\r", out.Progress, humanize.Bytes(out.Bytes)) |
| 216 | } else { |
| 217 | err = re.Emit(out) |
| 218 | if err != nil { |
| 219 | return err |
| 220 | } |
| 221 | } |
| 222 | } |
| 223 | }, |
| 224 | }, |
| 225 | } |
| 226 | |
| 227 | func pinAddMany(ctx context.Context, api coreiface.CoreAPI, enc cidenc.Encoder, paths []string, recursive bool, name string) ([]string, error) { |
| 228 | added := make([]string, len(paths)) |
| 229 | for i, b := range paths { |
| 230 | p, err := cmdutils.PathOrCidPath(b) |
| 231 | if err != nil { |
| 232 | return nil, err |
| 233 | } |
| 234 | |
| 235 | rp, _, err := api.ResolvePath(ctx, p) |
| 236 | if err != nil { |
| 237 | return nil, err |
| 238 | } |
| 239 | |
| 240 | if err := api.Pin().Add(ctx, rp, options.Pin.Recursive(recursive), options.Pin.Name(name)); err != nil { |
| 241 | return nil, err |
| 242 | } |
| 243 | added[i] = enc.Encode(rp.RootCid()) |
| 244 | } |
| 245 | |
| 246 | return added, nil |
| 247 | } |
| 248 | |
| 249 | // resolveFastProvideFlags resolves --fast-provide-root, --fast-provide-dag, |
| 250 | // and --fast-provide-wait from CLI flags, falling back to config defaults. |
| 251 | // Returns the node for use by fastProvideAfterPin. |
| 252 | func resolveFastProvideFlags(req *cmds.Request, env cmds.Environment) (nd *core.IpfsNode, root, dag, wait bool) { |
| 253 | nd, err := cmdenv.GetNode(env) |
| 254 | if err != nil { |
| 255 | return nil, config.DefaultFastProvideRoot, config.DefaultFastProvideDAG, config.DefaultFastProvideWait |
| 256 | } |
| 257 | cfg, err := nd.Repo.Config() |
| 258 | if err != nil { |
| 259 | return nd, config.DefaultFastProvideRoot, config.DefaultFastProvideDAG, config.DefaultFastProvideWait |
| 260 | } |
| 261 | fpRoot, fpRootSet := req.Options[fastProvideRootOptionName].(bool) |
| 262 | fpDAG, fpDAGSet := req.Options[fastProvideDAGOptionName].(bool) |
| 263 | fpWait, fpWaitSet := req.Options[fastProvideWaitOptionName].(bool) |
| 264 | root = config.ResolveBoolFromConfig(fpRoot, fpRootSet, cfg.Import.FastProvideRoot, config.DefaultFastProvideRoot) |
| 265 | dag = config.ResolveBoolFromConfig(fpDAG, fpDAGSet, cfg.Import.FastProvideDAG, config.DefaultFastProvideDAG) |
| 266 | wait = config.ResolveBoolFromConfig(fpWait, fpWaitSet, cfg.Import.FastProvideWait, config.DefaultFastProvideWait) |
| 267 | return nd, root, dag, wait |
| 268 | } |
| 269 | |
| 270 | // fastProvideAfterPin handles both root and DAG providing after a |
| 271 | // successful pin operation. Best-effort: errors are logged but do not |
| 272 | // fail the pin command. |
| 273 | func fastProvideAfterPin(req *cmds.Request, nd *core.IpfsNode, fpRoot, fpDAG, fpWait bool, encodedCIDs []string) { |
| 274 | if !fpRoot && !fpDAG { |
| 275 | return |
| 276 | } |
| 277 | cfg, err := nd.Repo.Config() |
| 278 | if err != nil { |
| 279 | return |
| 280 | } |
| 281 | var cidList []cid.Cid |
| 282 | for _, s := range encodedCIDs { |
| 283 | c, err := cid.Decode(s) |
| 284 | if err != nil { |
| 285 | continue |
| 286 | } |
| 287 | cidList = append(cidList, c) |
| 288 | } |
| 289 | |
| 290 | if fpDAG { |
| 291 | // DAG walk includes the root CID (DFS pre-order emits it |
| 292 | // first), so a separate root provide is not needed. |
| 293 | // Single call with all roots shares one bloom tracker. |
| 294 | cmdenv.ExecuteFastProvideDAG( |
| 295 | req.Context, |
| 296 | nd.Context(), |
| 297 | cidList, |
| 298 | nd.ProvidingStrategy, |
| 299 | nd.Blockstore, |
| 300 | nd.Provider, |
| 301 | fpWait, |
| 302 | uint(cfg.Provide.BloomFPRate.WithDefault(config.DefaultProvideBloomFPRate)), |
| 303 | 0, // block count unknown; bloom chain auto-grows |
| 304 | ) |
| 305 | } else if fpRoot { |
| 306 | for _, c := range cidList { |
| 307 | if err := cmdenv.ExecuteFastProvideRoot( |
| 308 | req.Context, nd, cfg, c, |
| 309 | fpWait, |
| 310 | true, // isPinned |
| 311 | true, // isPinnedRoot |
| 312 | false, // isMFS |
| 313 | ); err != nil { |
| 314 | log.Errorf("fast provide root after pin: %s", err) |
| 315 | } |
| 316 | } |
| 317 | } |
| 318 | } |
| 319 | |
| 320 | var rmPinCmd = &cmds.Command{ |
| 321 | Helptext: cmds.HelpText{ |
| 322 | Tagline: "Remove object from pin-list.", |
| 323 | ShortDescription: ` |
| 324 | Removes the pin from the given object allowing it to be garbage |
| 325 | collected if needed. (By default, recursively. Use -r=false for direct pins.) |
| 326 | `, |
| 327 | LongDescription: ` |
| 328 | Removes the pin from the given object allowing it to be garbage |
| 329 | collected if needed. (By default, recursively. Use -r=false for direct pins.) |
| 330 | |
| 331 | A pin may not be removed because the specified object is not pinned or pinned |
| 332 | indirectly. To determine if the object is pinned indirectly, use the command: |
| 333 | ipfs pin ls -t indirect <cid> |
| 334 | `, |
| 335 | }, |
| 336 | |
| 337 | Arguments: []cmds.Argument{ |
| 338 | cmds.StringArg("ipfs-path", true, true, "Path to object(s) to be unpinned.").EnableStdin(), |
| 339 | }, |
| 340 | Options: []cmds.Option{ |
| 341 | cmds.BoolOption(pinRecursiveOptionName, "r", "Recursively unpin the object linked to by the specified object(s).").WithDefault(true), |
| 342 | }, |
| 343 | Type: PinOutput{}, |
| 344 | Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { |
| 345 | api, err := cmdenv.GetApi(env, req) |
| 346 | if err != nil { |
| 347 | return err |
| 348 | } |
| 349 | |
| 350 | // set recursive flag |
| 351 | recursive, _ := req.Options[pinRecursiveOptionName].(bool) |
| 352 | |
| 353 | if err := req.ParseBodyArgs(); err != nil { |
| 354 | return err |
| 355 | } |
| 356 | |
| 357 | enc, err := cmdenv.GetCidEncoder(req) |
| 358 | if err != nil { |
| 359 | return err |
| 360 | } |
| 361 | |
| 362 | pins := make([]string, 0, len(req.Arguments)) |
| 363 | for _, b := range req.Arguments { |
| 364 | p, err := cmdutils.PathOrCidPath(b) |
| 365 | if err != nil { |
| 366 | return err |
| 367 | } |
| 368 | |
| 369 | rp, _, err := api.ResolvePath(req.Context, p) |
| 370 | if err != nil { |
| 371 | return err |
| 372 | } |
| 373 | |
| 374 | id := enc.Encode(rp.RootCid()) |
| 375 | pins = append(pins, id) |
| 376 | if err := api.Pin().Rm(req.Context, rp, options.Pin.RmRecursive(recursive)); err != nil { |
| 377 | return err |
| 378 | } |
| 379 | } |
| 380 | |
| 381 | return cmds.EmitOnce(res, &PinOutput{pins}) |
| 382 | }, |
| 383 | Encoders: cmds.EncoderMap{ |
| 384 | cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *PinOutput) error { |
| 385 | for _, k := range out.Pins { |
| 386 | fmt.Fprintf(w, "unpinned %s\n", k) |
| 387 | } |
| 388 | |
| 389 | return nil |
| 390 | }), |
| 391 | }, |
| 392 | } |
| 393 | |
| 394 | const ( |
| 395 | pinTypeOptionName = "type" |
| 396 | pinQuietOptionName = "quiet" |
| 397 | pinStreamOptionName = "stream" |
| 398 | pinNamesOptionName = "names" |
| 399 | ) |
| 400 | |
| 401 | var listPinCmd = &cmds.Command{ |
| 402 | Helptext: cmds.HelpText{ |
| 403 | Tagline: "List objects pinned to local storage.", |
| 404 | ShortDescription: ` |
| 405 | Returns a list of objects that are pinned locally. |
| 406 | By default, all pinned objects are returned, but the '--type' flag or |
| 407 | arguments can restrict that to a specific pin type or to some specific objects |
| 408 | respectively. |
| 409 | `, |
| 410 | LongDescription: ` |
| 411 | Returns a list of objects that are pinned locally. |
| 412 | |
| 413 | By default, all pinned objects are returned, but the '--type' flag or |
| 414 | arguments can restrict that to a specific pin type or to some specific objects |
| 415 | respectively. |
| 416 | |
| 417 | Use --type=<type> to specify the type of pinned keys to list. |
| 418 | Valid values are: |
| 419 | * "direct": pin that specific object. |
| 420 | * "recursive": pin that specific object, and indirectly pin all its |
| 421 | descendants |
| 422 | * "indirect": pinned indirectly by an ancestor (like a refcount) |
| 423 | * "all" |
| 424 | |
| 425 | By default, pin names are not included (returned as empty). |
| 426 | Pass '--names' flag to return pin names (set with '--name' from 'pin add'). |
| 427 | |
| 428 | With arguments, the command fails if any of the arguments is not a pinned |
| 429 | object. And if --type=<type> is additionally used, the command will also fail |
| 430 | if any of the arguments is not of the specified type. |
| 431 | |
| 432 | Example: |
| 433 | $ echo "hello" | ipfs add -q |
| 434 | QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN |
| 435 | $ ipfs pin ls |
| 436 | QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN recursive |
| 437 | # now remove the pin, and repin it directly |
| 438 | $ ipfs pin rm QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN |
| 439 | unpinned QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN |
| 440 | $ ipfs pin add -r=false QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN |
| 441 | pinned QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN directly |
| 442 | $ ipfs pin ls --type=direct |
| 443 | QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN direct |
| 444 | $ ipfs pin ls QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN |
| 445 | QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN direct |
| 446 | `, |
| 447 | }, |
| 448 | |
| 449 | Arguments: []cmds.Argument{ |
| 450 | cmds.StringArg("ipfs-path", false, true, "Path to object(s) to be listed."), |
| 451 | }, |
| 452 | Options: []cmds.Option{ |
| 453 | cmds.StringOption(pinTypeOptionName, "t", "The type of pinned keys to list. Can be \"direct\", \"indirect\", \"recursive\", or \"all\".").WithDefault("all"), |
| 454 | cmds.BoolOption(pinQuietOptionName, "q", "Output only the CIDs of pins."), |
| 455 | cmds.StringOption(pinNameOptionName, "n", "Limit returned pins to ones with names that contain the value provided (case-sensitive, partial match). Implies --names=true."), |
| 456 | cmds.BoolOption(pinStreamOptionName, "s", "Enable streaming of pins as they are discovered."), |
| 457 | cmds.BoolOption(pinNamesOptionName, "Include pin names in the output (slower, disabled by default)."), |
| 458 | }, |
| 459 | Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { |
| 460 | api, err := cmdenv.GetApi(env, req) |
| 461 | if err != nil { |
| 462 | return err |
| 463 | } |
| 464 | |
| 465 | n, err := cmdenv.GetNode(env) |
| 466 | if err != nil { |
| 467 | return err |
| 468 | } |
| 469 | |
| 470 | if n.Pinning == nil { |
| 471 | return fmt.Errorf("pinning service not available") |
| 472 | } |
| 473 | |
| 474 | typeStr, _ := req.Options[pinTypeOptionName].(string) |
| 475 | stream, _ := req.Options[pinStreamOptionName].(bool) |
| 476 | displayNames, _ := req.Options[pinNamesOptionName].(bool) |
| 477 | name, _ := req.Options[pinNameOptionName].(string) |
| 478 | |
| 479 | // Validate name filter |
| 480 | if err := cmdutils.ValidatePinName(name); err != nil { |
| 481 | return err |
| 482 | } |
| 483 | |
| 484 | mode, ok := pin.StringToMode(typeStr) |
| 485 | if !ok { |
| 486 | return fmt.Errorf("invalid type '%s', must be one of {direct, indirect, recursive, all}", typeStr) |
| 487 | } |
| 488 | |
| 489 | // For backward compatibility, we accumulate the pins in the same output type as before. |
| 490 | var emit func(PinLsOutputWrapper) error |
| 491 | lgcList := map[string]PinLsType{} |
| 492 | if !stream { |
| 493 | emit = func(v PinLsOutputWrapper) error { |
| 494 | lgcList[v.PinLsObject.Cid] = PinLsType{Type: v.PinLsObject.Type, Name: v.PinLsObject.Name} |
| 495 | return nil |
| 496 | } |
| 497 | } else { |
| 498 | emit = func(v PinLsOutputWrapper) error { |
| 499 | return res.Emit(v) |
| 500 | } |
| 501 | } |
| 502 | |
| 503 | if len(req.Arguments) > 0 { |
| 504 | err = pinLsKeys(req, mode, displayNames || name != "", n.Pinning, api, emit) |
| 505 | } else { |
| 506 | err = pinLsAll(req, typeStr, displayNames || name != "", name, api, emit) |
| 507 | } |
| 508 | if err != nil { |
| 509 | return err |
| 510 | } |
| 511 | |
| 512 | if !stream { |
| 513 | return cmds.EmitOnce(res, PinLsOutputWrapper{ |
| 514 | PinLsList: PinLsList{Keys: lgcList}, |
| 515 | }) |
| 516 | } |
| 517 | |
| 518 | return nil |
| 519 | }, |
| 520 | Type: PinLsOutputWrapper{}, |
| 521 | Encoders: cmds.EncoderMap{ |
| 522 | cmds.JSON: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out PinLsOutputWrapper) error { |
| 523 | stream, _ := req.Options[pinStreamOptionName].(bool) |
| 524 | |
| 525 | enc := json.NewEncoder(w) |
| 526 | |
| 527 | if stream { |
| 528 | return enc.Encode(out.PinLsObject) |
| 529 | } |
| 530 | |
| 531 | return enc.Encode(out.PinLsList) |
| 532 | }), |
| 533 | cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out PinLsOutputWrapper) error { |
| 534 | quiet, _ := req.Options[pinQuietOptionName].(bool) |
| 535 | stream, _ := req.Options[pinStreamOptionName].(bool) |
| 536 | |
| 537 | if stream { |
| 538 | if quiet { |
| 539 | fmt.Fprintf(w, "%s\n", out.PinLsObject.Cid) |
| 540 | } else if out.PinLsObject.Name == "" { |
| 541 | fmt.Fprintf(w, "%s %s\n", out.PinLsObject.Cid, out.PinLsObject.Type) |
| 542 | } else { |
| 543 | fmt.Fprintf(w, "%s %s %s\n", out.PinLsObject.Cid, out.PinLsObject.Type, out.PinLsObject.Name) |
| 544 | } |
| 545 | return nil |
| 546 | } |
| 547 | |
| 548 | for k, v := range out.PinLsList.Keys { |
| 549 | if quiet { |
| 550 | fmt.Fprintf(w, "%s\n", k) |
| 551 | } else if v.Name == "" { |
| 552 | fmt.Fprintf(w, "%s %s\n", k, v.Type) |
| 553 | } else { |
| 554 | fmt.Fprintf(w, "%s %s %s\n", k, v.Type, v.Name) |
| 555 | } |
| 556 | } |
| 557 | |
| 558 | return nil |
| 559 | }), |
| 560 | }, |
| 561 | } |
| 562 | |
| 563 | // PinLsOutputWrapper is the output type of the pin ls command. |
| 564 | // Pin ls needs to output two different type depending on if it's streamed or not. |
| 565 | // We use this to bypass the cmds lib refusing to have interface{} |
| 566 | type PinLsOutputWrapper struct { |
| 567 | PinLsList |
| 568 | PinLsObject |
| 569 | } |
| 570 | |
| 571 | // PinLsList is a set of pins with their type |
| 572 | type PinLsList struct { |
| 573 | Keys map[string]PinLsType `json:",omitempty"` |
| 574 | } |
| 575 | |
| 576 | // PinLsType contains the type of a pin |
| 577 | type PinLsType struct { |
| 578 | Type string |
| 579 | Name string |
| 580 | } |
| 581 | |
| 582 | // PinLsObject contains the description of a pin |
| 583 | type PinLsObject struct { |
| 584 | Cid string `json:",omitempty"` |
| 585 | Name string `json:",omitempty"` |
| 586 | Type string `json:",omitempty"` |
| 587 | } |
| 588 | |
| 589 | func pinLsKeys(req *cmds.Request, mode pin.Mode, displayNames bool, pinner pin.Pinner, api coreiface.CoreAPI, emit func(value PinLsOutputWrapper) error) error { |
| 590 | enc, err := cmdenv.GetCidEncoder(req) |
| 591 | if err != nil { |
| 592 | return err |
| 593 | } |
| 594 | |
| 595 | // Collect CIDs to check |
| 596 | cids := make([]cid.Cid, 0, len(req.Arguments)) |
| 597 | for _, p := range req.Arguments { |
| 598 | p, err := cmdutils.PathOrCidPath(p) |
| 599 | if err != nil { |
| 600 | return err |
| 601 | } |
| 602 | |
| 603 | rp, _, err := api.ResolvePath(req.Context, p) |
| 604 | if err != nil { |
| 605 | return err |
| 606 | } |
| 607 | |
| 608 | cids = append(cids, rp.RootCid()) |
| 609 | } |
| 610 | |
| 611 | // Check pins using the new type-specific method |
| 612 | pinned, err := pinner.CheckIfPinnedWithType(req.Context, mode, displayNames, cids...) |
| 613 | if err != nil { |
| 614 | return err |
| 615 | } |
| 616 | |
| 617 | // Process results |
| 618 | for i, p := range pinned { |
| 619 | if !p.Pinned() { |
| 620 | return fmt.Errorf("path '%s' is not pinned", req.Arguments[i]) |
| 621 | } |
| 622 | |
| 623 | pinType, _ := pin.ModeToString(p.Mode) |
| 624 | if p.Mode == pin.Indirect && p.Via.Defined() { |
| 625 | pinType = "indirect through " + enc.Encode(p.Via) |
| 626 | } |
| 627 | |
| 628 | err = emit(PinLsOutputWrapper{ |
| 629 | PinLsObject: PinLsObject{ |
| 630 | Type: pinType, |
| 631 | Cid: enc.Encode(cids[i]), |
| 632 | Name: p.Name, |
| 633 | }, |
| 634 | }) |
| 635 | if err != nil { |
| 636 | return err |
| 637 | } |
| 638 | } |
| 639 | |
| 640 | return nil |
| 641 | } |
| 642 | |
| 643 | func pinLsAll(req *cmds.Request, typeStr string, detailed bool, name string, api coreiface.CoreAPI, emit func(value PinLsOutputWrapper) error) error { |
| 644 | enc, err := cmdenv.GetCidEncoder(req) |
| 645 | if err != nil { |
| 646 | return err |
| 647 | } |
| 648 | |
| 649 | _, ok := pin.StringToMode(typeStr) |
| 650 | if !ok { |
| 651 | return fmt.Errorf("invalid type '%s', must be one of {direct, indirect, recursive, all}", typeStr) |
| 652 | } |
| 653 | |
| 654 | opt, err := options.Pin.Ls.Type(typeStr) |
| 655 | if err != nil { |
| 656 | return err |
| 657 | } |
| 658 | |
| 659 | pins := make(chan coreiface.Pin) |
| 660 | lsErr := make(chan error, 1) |
| 661 | lsCtx, cancel := context.WithCancel(req.Context) |
| 662 | defer cancel() |
| 663 | |
| 664 | go func() { |
| 665 | lsErr <- api.Pin().Ls(lsCtx, pins, opt, options.Pin.Ls.Detailed(detailed), options.Pin.Ls.Name(name)) |
| 666 | }() |
| 667 | |
| 668 | for p := range pins { |
| 669 | err = emit(PinLsOutputWrapper{ |
| 670 | PinLsObject: PinLsObject{ |
| 671 | Type: p.Type(), |
| 672 | Name: p.Name(), |
| 673 | Cid: enc.Encode(p.Path().RootCid()), |
| 674 | }, |
| 675 | }) |
| 676 | if err != nil { |
| 677 | return err |
| 678 | } |
| 679 | } |
| 680 | return <-lsErr |
| 681 | } |
| 682 | |
| 683 | const ( |
| 684 | pinUnpinOptionName = "unpin" |
| 685 | ) |
| 686 | |
| 687 | var updatePinCmd = &cmds.Command{ |
| 688 | Helptext: cmds.HelpText{ |
| 689 | Tagline: "Update a recursive pin.", |
| 690 | ShortDescription: ` |
| 691 | Efficiently pins a new object based on differences from an existing one and, |
| 692 | by default, removes the old pin. |
| 693 | |
| 694 | This command is useful when the new pin contains many similarities or is a |
| 695 | derivative of an existing one, particularly for large objects. This allows a more |
| 696 | efficient DAG-traversal which fully skips already-pinned branches from the old |
| 697 | object. As a requirement, the old object needs to be an existing recursive |
| 698 | pin. |
| 699 | `, |
| 700 | }, |
| 701 | |
| 702 | Arguments: []cmds.Argument{ |
| 703 | cmds.StringArg("from-path", true, false, "Path to old object."), |
| 704 | cmds.StringArg("to-path", true, false, "Path to a new object to be pinned."), |
| 705 | }, |
| 706 | Options: []cmds.Option{ |
| 707 | cmds.BoolOption(pinUnpinOptionName, "Remove the old pin.").WithDefault(true), |
| 708 | cmds.BoolOption(fastProvideRootOptionName, "Immediately provide new root CID to DHT after update. Default: Import.FastProvideRoot"), |
| 709 | cmds.BoolOption(fastProvideDAGOptionName, "Walk and provide the full DAG according to Provide.Strategy after update. Default: Import.FastProvideDAG"), |
| 710 | cmds.BoolOption(fastProvideWaitOptionName, "Block until the immediate provide completes. Default: Import.FastProvideWait"), |
| 711 | }, |
| 712 | Type: PinOutput{}, |
| 713 | Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { |
| 714 | api, err := cmdenv.GetApi(env, req) |
| 715 | if err != nil { |
| 716 | return err |
| 717 | } |
| 718 | |
| 719 | enc, err := cmdenv.GetCidEncoder(req) |
| 720 | if err != nil { |
| 721 | return err |
| 722 | } |
| 723 | |
| 724 | unpin, _ := req.Options[pinUnpinOptionName].(bool) |
| 725 | |
| 726 | fromPath, err := cmdutils.PathOrCidPath(req.Arguments[0]) |
| 727 | if err != nil { |
| 728 | return err |
| 729 | } |
| 730 | |
| 731 | toPath, err := cmdutils.PathOrCidPath(req.Arguments[1]) |
| 732 | if err != nil { |
| 733 | return err |
| 734 | } |
| 735 | |
| 736 | // Resolve the paths ahead of time so we can return the actual CIDs |
| 737 | from, _, err := api.ResolvePath(req.Context, fromPath) |
| 738 | if err != nil { |
| 739 | return err |
| 740 | } |
| 741 | to, _, err := api.ResolvePath(req.Context, toPath) |
| 742 | if err != nil { |
| 743 | return err |
| 744 | } |
| 745 | |
| 746 | err = api.Pin().Update(req.Context, from, to, options.Pin.Unpin(unpin)) |
| 747 | if err != nil { |
| 748 | return err |
| 749 | } |
| 750 | |
| 751 | nd, fpRoot, fpDAG, fpWait := resolveFastProvideFlags(req, env) |
| 752 | fastProvideAfterPin(req, nd, fpRoot, fpDAG, fpWait, []string{enc.Encode(to.RootCid())}) |
| 753 | |
| 754 | return cmds.EmitOnce(res, &PinOutput{Pins: []string{enc.Encode(from.RootCid()), enc.Encode(to.RootCid())}}) |
| 755 | }, |
| 756 | Encoders: cmds.EncoderMap{ |
| 757 | cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *PinOutput) error { |
| 758 | fmt.Fprintf(w, "updated %s to %s\n", out.Pins[0], out.Pins[1]) |
| 759 | return nil |
| 760 | }), |
| 761 | }, |
| 762 | } |
| 763 | |
| 764 | const ( |
| 765 | pinVerboseOptionName = "verbose" |
| 766 | ) |
| 767 | |
| 768 | var verifyPinCmd = &cmds.Command{ |
| 769 | Helptext: cmds.HelpText{ |
| 770 | Tagline: "Verify that recursive pins are complete.", |
| 771 | }, |
| 772 | Options: []cmds.Option{ |
| 773 | cmds.BoolOption(pinVerboseOptionName, "Also write the hashes of non-broken pins."), |
| 774 | cmds.BoolOption(pinQuietOptionName, "q", "Write just hashes of broken pins."), |
| 775 | }, |
| 776 | Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { |
| 777 | n, err := cmdenv.GetNode(env) |
| 778 | if err != nil { |
| 779 | return err |
| 780 | } |
| 781 | |
| 782 | verbose, _ := req.Options[pinVerboseOptionName].(bool) |
| 783 | quiet, _ := req.Options[pinQuietOptionName].(bool) |
| 784 | |
| 785 | if verbose && quiet { |
| 786 | return fmt.Errorf("the --verbose and --quiet options can not be used at the same time") |
| 787 | } |
| 788 | |
| 789 | enc, err := cmdenv.GetCidEncoder(req) |
| 790 | if err != nil { |
| 791 | return err |
| 792 | } |
| 793 | |
| 794 | opts := pinVerifyOpts{ |
| 795 | explain: !quiet, |
| 796 | includeOk: verbose, |
| 797 | } |
| 798 | out, err := pinVerify(req.Context, n, opts, enc) |
| 799 | if err != nil { |
| 800 | return err |
| 801 | } |
| 802 | return res.Emit(out) |
| 803 | }, |
| 804 | Type: PinVerifyRes{}, |
| 805 | Encoders: cmds.EncoderMap{ |
| 806 | cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *PinVerifyRes) error { |
| 807 | quiet, _ := req.Options[pinQuietOptionName].(bool) |
| 808 | |
| 809 | if quiet && !out.Ok { |
| 810 | fmt.Fprintf(w, "%s\n", out.Cid) |
| 811 | } else if !quiet { |
| 812 | out.Format(w) |
| 813 | } |
| 814 | |
| 815 | return nil |
| 816 | }), |
| 817 | }, |
| 818 | } |
| 819 | |
| 820 | // PinVerifyRes is the result returned for each pin checked in "pin verify" |
| 821 | type PinVerifyRes struct { |
| 822 | Cid string `json:",omitempty"` |
| 823 | Err string `json:",omitempty"` |
| 824 | PinStatus |
| 825 | } |
| 826 | |
| 827 | // PinStatus is part of PinVerifyRes, do not use directly |
| 828 | type PinStatus struct { |
| 829 | Ok bool `json:",omitempty"` |
| 830 | BadNodes []BadNode `json:",omitempty"` |
| 831 | } |
| 832 | |
| 833 | // BadNode is used in PinVerifyRes |
| 834 | type BadNode struct { |
| 835 | Cid string |
| 836 | Err string |
| 837 | } |
| 838 | |
| 839 | type pinVerifyOpts struct { |
| 840 | explain bool |
| 841 | includeOk bool |
| 842 | } |
| 843 | |
| 844 | // FIXME: this implementation is duplicated sith core/coreapi.PinAPI.Verify, remove this one and exclusively rely on CoreAPI. |
| 845 | func pinVerify(ctx context.Context, n *core.IpfsNode, opts pinVerifyOpts, enc cidenc.Encoder) (<-chan any, error) { |
| 846 | visited := make(map[cid.Cid]PinStatus) |
| 847 | |
| 848 | bs := n.Blocks.Blockstore() |
| 849 | DAG := dag.NewDAGService(bserv.New(bs, offline.Exchange(bs))) |
| 850 | getLinks := dag.GetLinksWithDAG(DAG) |
| 851 | |
| 852 | var checkPin func(root cid.Cid) PinStatus |
| 853 | checkPin = func(root cid.Cid) PinStatus { |
| 854 | key := root |
| 855 | if status, ok := visited[key]; ok { |
| 856 | return status |
| 857 | } |
| 858 | |
| 859 | if err := verifcid.ValidateCid(verifcid.DefaultAllowlist, root); err != nil { |
| 860 | status := PinStatus{Ok: false} |
| 861 | if opts.explain { |
| 862 | status.BadNodes = []BadNode{{Cid: enc.Encode(key), Err: err.Error()}} |
| 863 | } |
| 864 | visited[key] = status |
| 865 | return status |
| 866 | } |
| 867 | |
| 868 | links, err := getLinks(ctx, root) |
| 869 | if err != nil { |
| 870 | status := PinStatus{Ok: false} |
| 871 | if opts.explain { |
| 872 | status.BadNodes = []BadNode{{Cid: enc.Encode(key), Err: err.Error()}} |
| 873 | } |
| 874 | visited[key] = status |
| 875 | return status |
| 876 | } |
| 877 | |
| 878 | status := PinStatus{Ok: true} |
| 879 | for _, lnk := range links { |
| 880 | res := checkPin(lnk.Cid) |
| 881 | if !res.Ok { |
| 882 | status.Ok = false |
| 883 | status.BadNodes = append(status.BadNodes, res.BadNodes...) |
| 884 | } |
| 885 | } |
| 886 | |
| 887 | visited[key] = status |
| 888 | return status |
| 889 | } |
| 890 | |
| 891 | out := make(chan any) |
| 892 | go func() { |
| 893 | defer close(out) |
| 894 | for p := range n.Pinning.RecursiveKeys(ctx, false) { |
| 895 | if p.Err != nil { |
| 896 | out <- PinVerifyRes{Err: p.Err.Error()} |
| 897 | return |
| 898 | } |
| 899 | pinStatus := checkPin(p.Pin.Key) |
| 900 | if !pinStatus.Ok || opts.includeOk { |
| 901 | select { |
| 902 | case out <- PinVerifyRes{Cid: enc.Encode(p.Pin.Key), PinStatus: pinStatus}: |
| 903 | case <-ctx.Done(): |
| 904 | return |
| 905 | } |
| 906 | } |
| 907 | } |
| 908 | }() |
| 909 | |
| 910 | return out, nil |
| 911 | } |
| 912 | |
| 913 | // Format formats PinVerifyRes |
| 914 | func (r PinVerifyRes) Format(out io.Writer) { |
| 915 | if r.Err != "" { |
| 916 | fmt.Fprintf(out, "error: %s\n", r.Err) |
| 917 | return |
| 918 | } |
| 919 | |
| 920 | if r.Ok { |
| 921 | fmt.Fprintf(out, "%s ok\n", r.Cid) |
| 922 | return |
| 923 | } |
| 924 | |
| 925 | fmt.Fprintf(out, "%s broken\n", r.Cid) |
| 926 | for _, e := range r.BadNodes { |
| 927 | fmt.Fprintf(out, " %s: %s\n", e.Cid, e.Err) |
| 928 | } |
| 929 | } |