| 1 | package dagcmd |
| 2 | |
| 3 | import ( |
| 4 | "encoding/csv" |
| 5 | "encoding/json" |
| 6 | "fmt" |
| 7 | "io" |
| 8 | "path" |
| 9 | |
| 10 | "github.com/dustin/go-humanize" |
| 11 | "github.com/ipfs/kubo/core/commands/cmdenv" |
| 12 | "github.com/ipfs/kubo/core/commands/cmdutils" |
| 13 | |
| 14 | cid "github.com/ipfs/go-cid" |
| 15 | cidenc "github.com/ipfs/go-cidutil/cidenc" |
| 16 | cmds "github.com/ipfs/go-ipfs-cmds" |
| 17 | ) |
| 18 | |
| 19 | const ( |
| 20 | pinRootsOptionName = "pin-roots" |
| 21 | progressOptionName = "progress" |
| 22 | silentOptionName = "silent" |
| 23 | statsOptionName = "stats" |
| 24 | fastProvideRootOptionName = "fast-provide-root" |
| 25 | fastProvideDAGOptionName = "fast-provide-dag" |
| 26 | fastProvideWaitOptionName = "fast-provide-wait" |
| 27 | localOnlyOptionName = "local-only" |
| 28 | ) |
| 29 | |
| 30 | // DagCmd provides a subset of commands for interacting with ipld dag objects |
| 31 | var DagCmd = &cmds.Command{ |
| 32 | Helptext: cmds.HelpText{ |
| 33 | Tagline: "Interact with IPLD DAG objects.", |
| 34 | ShortDescription: ` |
| 35 | 'ipfs dag' is used for creating and manipulating DAG objects/hierarchies. |
| 36 | |
| 37 | This subcommand is intended to deprecate and replace |
| 38 | the existing 'ipfs object' command moving forward. |
| 39 | `, |
| 40 | }, |
| 41 | Subcommands: map[string]*cmds.Command{ |
| 42 | "put": DagPutCmd, |
| 43 | "get": DagGetCmd, |
| 44 | "resolve": DagResolveCmd, |
| 45 | "import": DagImportCmd, |
| 46 | "export": DagExportCmd, |
| 47 | "stat": DagStatCmd, |
| 48 | }, |
| 49 | } |
| 50 | |
| 51 | // OutputObject is the output type of 'dag put' command |
| 52 | type OutputObject struct { |
| 53 | Cid cid.Cid |
| 54 | } |
| 55 | |
| 56 | // ResolveOutput is the output type of 'dag resolve' command |
| 57 | type ResolveOutput struct { |
| 58 | Cid cid.Cid |
| 59 | RemPath string |
| 60 | } |
| 61 | |
| 62 | type CarImportStats struct { |
| 63 | BlockCount uint64 |
| 64 | BlockBytesCount uint64 |
| 65 | } |
| 66 | |
| 67 | // CarImportOutput is the output type of the 'dag import' commands |
| 68 | type CarImportOutput struct { |
| 69 | Root *RootMeta `json:",omitempty"` |
| 70 | Stats *CarImportStats `json:",omitempty"` |
| 71 | } |
| 72 | |
| 73 | // RootMeta is the metadata for a root pinning response |
| 74 | type RootMeta struct { |
| 75 | Cid cid.Cid |
| 76 | PinErrorMsg string |
| 77 | } |
| 78 | |
| 79 | // DagPutCmd is a command for adding a dag node |
| 80 | var DagPutCmd = &cmds.Command{ |
| 81 | Helptext: cmds.HelpText{ |
| 82 | Tagline: "Add a DAG node to IPFS.", |
| 83 | ShortDescription: ` |
| 84 | 'ipfs dag put' accepts input from a file or stdin and parses it |
| 85 | into an object of the specified format. |
| 86 | `, |
| 87 | }, |
| 88 | Arguments: []cmds.Argument{ |
| 89 | cmds.FileArg("object data", true, true, "The object to put").EnableStdin(), |
| 90 | }, |
| 91 | Options: []cmds.Option{ |
| 92 | cmds.StringOption("store-codec", "Codec that the stored object will be encoded with").WithDefault("dag-cbor"), |
| 93 | cmds.StringOption("input-codec", "Codec that the input object is encoded in").WithDefault("dag-json"), |
| 94 | cmds.BoolOption("pin", "Pin this object when adding."), |
| 95 | cmds.StringOption("hash", "Hash function to use"), |
| 96 | cmdutils.AllowBigBlockOption, |
| 97 | }, |
| 98 | Run: dagPut, |
| 99 | Type: OutputObject{}, |
| 100 | Encoders: cmds.EncoderMap{ |
| 101 | cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *OutputObject) error { |
| 102 | enc, err := cmdenv.GetCidEncoder(req) |
| 103 | if err != nil { |
| 104 | return err |
| 105 | } |
| 106 | fmt.Fprintln(w, enc.Encode(out.Cid)) |
| 107 | return nil |
| 108 | }), |
| 109 | }, |
| 110 | } |
| 111 | |
| 112 | // DagGetCmd is a command for getting a dag node from IPFS |
| 113 | var DagGetCmd = &cmds.Command{ |
| 114 | Helptext: cmds.HelpText{ |
| 115 | Tagline: "Get a DAG node from IPFS.", |
| 116 | ShortDescription: ` |
| 117 | 'ipfs dag get' fetches a DAG node from IPFS and prints it out in the specified |
| 118 | format. |
| 119 | `, |
| 120 | }, |
| 121 | Arguments: []cmds.Argument{ |
| 122 | cmds.StringArg("ref", true, false, "The object to get").EnableStdin(), |
| 123 | }, |
| 124 | Options: []cmds.Option{ |
| 125 | cmds.StringOption("output-codec", "Format that the object will be encoded as.").WithDefault("dag-json"), |
| 126 | }, |
| 127 | Run: dagGet, |
| 128 | } |
| 129 | |
| 130 | // DagResolveCmd returns address of highest block within a path and a path remainder |
| 131 | var DagResolveCmd = &cmds.Command{ |
| 132 | Helptext: cmds.HelpText{ |
| 133 | Tagline: "Resolve IPLD block.", |
| 134 | ShortDescription: ` |
| 135 | 'ipfs dag resolve' fetches a DAG node from IPFS, prints its address and remaining path. |
| 136 | `, |
| 137 | }, |
| 138 | Arguments: []cmds.Argument{ |
| 139 | cmds.StringArg("ref", true, false, "The path to resolve").EnableStdin(), |
| 140 | }, |
| 141 | Run: dagResolve, |
| 142 | Encoders: cmds.EncoderMap{ |
| 143 | cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *ResolveOutput) error { |
| 144 | var ( |
| 145 | enc cidenc.Encoder |
| 146 | err error |
| 147 | ) |
| 148 | switch { |
| 149 | case !cmdenv.CidBaseDefined(req): |
| 150 | // Not specified, check the path. |
| 151 | enc, err = cmdenv.CidEncoderFromPath(req.Arguments[0]) |
| 152 | if err == nil { |
| 153 | break |
| 154 | } |
| 155 | // Nope, fallback on the default. |
| 156 | fallthrough |
| 157 | default: |
| 158 | enc, err = cmdenv.GetCidEncoder(req) |
| 159 | if err != nil { |
| 160 | return err |
| 161 | } |
| 162 | } |
| 163 | p := enc.Encode(out.Cid) |
| 164 | if out.RemPath != "" { |
| 165 | p = path.Join(p, out.RemPath) |
| 166 | } |
| 167 | |
| 168 | fmt.Fprint(w, p) |
| 169 | return nil |
| 170 | }), |
| 171 | }, |
| 172 | Type: ResolveOutput{}, |
| 173 | } |
| 174 | |
| 175 | // DagImportCmd is a command for importing a car to ipfs |
| 176 | var DagImportCmd = &cmds.Command{ |
| 177 | Helptext: cmds.HelpText{ |
| 178 | Tagline: "Import the contents of .car files", |
| 179 | ShortDescription: ` |
| 180 | 'ipfs dag import' imports all blocks present in supplied .car |
| 181 | ( Content Address aRchive ) files, recursively pinning any roots |
| 182 | specified in the CAR file headers, unless --pin-roots is set to false. |
| 183 | |
| 184 | Note: |
| 185 | This command will import all blocks in the CAR file, not just those |
| 186 | reachable from the specified roots. However, these other blocks will |
| 187 | not be pinned and may be garbage collected later. |
| 188 | |
| 189 | The pinning of the roots happens after all car files are processed, |
| 190 | permitting import of DAGs spanning multiple files. |
| 191 | |
| 192 | Pinning takes place in offline-mode exclusively, one root at a time. |
| 193 | If the combination of blocks from the imported CAR files and what is |
| 194 | currently present in the blockstore does not represent a complete DAG, |
| 195 | pinning of that individual root will fail. |
| 196 | |
| 197 | Use --local-only to import a partial CAR (e.g. from 'dag export |
| 198 | --local-only'). --local-only implies --pin-roots=false because a partial |
| 199 | CAR has no full DAG to pin. |
| 200 | |
| 201 | FAST PROVIDE OPTIMIZATION: |
| 202 | |
| 203 | Root CIDs from CAR headers are immediately provided to the DHT in addition |
| 204 | to the regular provide queue, allowing other peers to discover your content |
| 205 | right away. This complements the sweep provider, which efficiently provides |
| 206 | all blocks according to Provide.Strategy over time. |
| 207 | |
| 208 | By default, the provide happens in the background without blocking the |
| 209 | command. Use --fast-provide-wait to wait for the provide to complete, or |
| 210 | --fast-provide-root=false to skip it. Works even with --pin-roots=false. |
| 211 | Automatically skipped when DHT is not available. |
| 212 | |
| 213 | Maximum supported CAR version: 2 |
| 214 | Specification of CAR formats: https://ipld.io/specs/transport/car/ |
| 215 | `, |
| 216 | }, |
| 217 | Arguments: []cmds.Argument{ |
| 218 | cmds.FileArg("path", true, true, "The path of a .car file.").EnableStdin(), |
| 219 | }, |
| 220 | Options: []cmds.Option{ |
| 221 | cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing. Default: true."), |
| 222 | cmds.BoolOption(localOnlyOptionName, "Import a partial CAR (e.g. from 'dag export --local-only'). Implies --pin-roots=false."), |
| 223 | cmds.BoolOption(silentOptionName, "No output."), |
| 224 | cmds.BoolOption(statsOptionName, "Output stats."), |
| 225 | cmds.BoolOption(fastProvideRootOptionName, "Immediately provide root CIDs to DHT in addition to regular queue, for faster discovery. Default: Import.FastProvideRoot"), |
| 226 | cmds.BoolOption(fastProvideDAGOptionName, "Walk and provide the full DAG according to Provide.Strategy after import. Default: Import.FastProvideDAG"), |
| 227 | cmds.BoolOption(fastProvideWaitOptionName, "Block until the immediate provide completes before returning. Default: Import.FastProvideWait"), |
| 228 | cmdutils.AllowBigBlockOption, |
| 229 | }, |
| 230 | Type: CarImportOutput{}, |
| 231 | Run: dagImport, |
| 232 | Encoders: cmds.EncoderMap{ |
| 233 | cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, event *CarImportOutput) error { |
| 234 | silent, _ := req.Options[silentOptionName].(bool) |
| 235 | if silent { |
| 236 | return nil |
| 237 | } |
| 238 | |
| 239 | // event should have only one of `Root` or `Stats` set, not both |
| 240 | if event.Root == nil { |
| 241 | if event.Stats == nil { |
| 242 | return fmt.Errorf("unexpected message from DAG import") |
| 243 | } |
| 244 | stats, _ := req.Options[statsOptionName].(bool) |
| 245 | if stats { |
| 246 | fmt.Fprintf(w, "Imported %d blocks (%d bytes)\n", event.Stats.BlockCount, event.Stats.BlockBytesCount) |
| 247 | } |
| 248 | return nil |
| 249 | } |
| 250 | |
| 251 | if event.Stats != nil { |
| 252 | return fmt.Errorf("unexpected message from DAG import") |
| 253 | } |
| 254 | |
| 255 | enc, err := cmdenv.GetCidEncoder(req) |
| 256 | if err != nil { |
| 257 | return err |
| 258 | } |
| 259 | |
| 260 | if event.Root.PinErrorMsg != "" { |
| 261 | return fmt.Errorf("pinning root %q FAILED: %s", enc.Encode(event.Root.Cid), event.Root.PinErrorMsg) |
| 262 | } |
| 263 | |
| 264 | event.Root.PinErrorMsg = "success" |
| 265 | |
| 266 | _, err = fmt.Fprintf( |
| 267 | w, |
| 268 | "Pinned root\t%s\t%s\n", |
| 269 | enc.Encode(event.Root.Cid), |
| 270 | event.Root.PinErrorMsg, |
| 271 | ) |
| 272 | return err |
| 273 | }), |
| 274 | }, |
| 275 | } |
| 276 | |
| 277 | // DagExportCmd is a command for exporting an ipfs dag to a car |
| 278 | var DagExportCmd = &cmds.Command{ |
| 279 | Helptext: cmds.HelpText{ |
| 280 | Tagline: "Streams the selected DAG as a .car stream on stdout.", |
| 281 | ShortDescription: ` |
| 282 | 'ipfs dag export' fetches a DAG and streams it out as a well-formed .car file. |
| 283 | Note that at present only single root selections / .car files are supported. |
| 284 | The output of blocks happens in strict DAG-traversal, first-seen, order. |
| 285 | CAR file follows the CARv1 format: https://ipld.io/specs/transport/car/carv1/ |
| 286 | |
| 287 | Use --local-only for a best-effort export from the local blockstore: blocks |
| 288 | that are missing or unreadable locally (and their subtrees) are skipped, so |
| 289 | the resulting CAR is partial. --local-only implies --offline. |
| 290 | `, |
| 291 | HTTP: &cmds.HTTPHelpText{ |
| 292 | ResponseContentType: "application/vnd.ipld.car", |
| 293 | }, |
| 294 | }, |
| 295 | Arguments: []cmds.Argument{ |
| 296 | cmds.StringArg("root", true, false, "CID of a root to recursively export").EnableStdin(), |
| 297 | }, |
| 298 | Options: []cmds.Option{ |
| 299 | cmds.BoolOption(progressOptionName, "p", "Stream progress data. Defaults to true when stderr is a terminal."), |
| 300 | cmds.BoolOption(localOnlyOptionName, "Best-effort export of locally-available blocks; missing or unreadable blocks (and their subtrees) are skipped. Implies --offline."), |
| 301 | }, |
| 302 | Run: dagExport, |
| 303 | PostRun: cmds.PostRunMap{ |
| 304 | cmds.CLI: finishCLIExport, |
| 305 | }, |
| 306 | } |
| 307 | |
| 308 | // DagStat is a dag stat command response. Cid is stored as a |
| 309 | // pre-encoded string (via GetCidEncoder in the Run handler) so that |
| 310 | // --cid-base is respected and no custom MarshalJSON is needed. |
| 311 | type DagStat struct { |
| 312 | Cid string `json:"Cid"` |
| 313 | Size uint64 `json:",omitempty"` |
| 314 | NumBlocks int64 `json:",omitempty"` |
| 315 | } |
| 316 | |
| 317 | type DagStatSummary struct { |
| 318 | redundantSize uint64 `json:"-"` |
| 319 | UniqueBlocks int `json:",omitempty"` |
| 320 | TotalSize uint64 `json:",omitempty"` |
| 321 | SharedSize uint64 `json:",omitempty"` |
| 322 | Ratio float32 `json:",omitempty"` |
| 323 | DagStatsArray []*DagStat `json:"DagStats,omitempty"` |
| 324 | } |
| 325 | |
| 326 | func (s *DagStatSummary) String() string { |
| 327 | return fmt.Sprintf("Total Size: %d (%s)\nUnique Blocks: %d\nShared Size: %d (%s)\nRatio: %f", |
| 328 | s.TotalSize, humanize.Bytes(s.TotalSize), |
| 329 | s.UniqueBlocks, |
| 330 | s.SharedSize, humanize.Bytes(s.SharedSize), |
| 331 | s.Ratio) |
| 332 | } |
| 333 | |
| 334 | func (s *DagStatSummary) incrementTotalSize(size uint64) { |
| 335 | s.TotalSize += size |
| 336 | } |
| 337 | |
| 338 | func (s *DagStatSummary) incrementRedundantSize(size uint64) { |
| 339 | s.redundantSize += size |
| 340 | } |
| 341 | |
| 342 | func (s *DagStatSummary) appendStats(stats *DagStat) { |
| 343 | s.DagStatsArray = append(s.DagStatsArray, stats) |
| 344 | } |
| 345 | |
| 346 | func (s *DagStatSummary) calculateSummary() { |
| 347 | s.Ratio = float32(s.redundantSize) / float32(s.TotalSize) |
| 348 | s.SharedSize = s.redundantSize - s.TotalSize |
| 349 | } |
| 350 | |
| 351 | // DagStatCmd is a command for getting size information about an ipfs-stored dag |
| 352 | var DagStatCmd = &cmds.Command{ |
| 353 | Helptext: cmds.HelpText{ |
| 354 | Tagline: "Gets stats for a DAG.", |
| 355 | ShortDescription: ` |
| 356 | 'ipfs dag stat' fetches a DAG and returns various statistics about it. |
| 357 | Statistics include size and number of blocks. |
| 358 | |
| 359 | Note: This command skips duplicate blocks in reporting both size and the number of blocks |
| 360 | `, |
| 361 | }, |
| 362 | Arguments: []cmds.Argument{ |
| 363 | cmds.StringArg("root", true, true, "CID of a DAG root to get statistics for").EnableStdin(), |
| 364 | }, |
| 365 | Options: []cmds.Option{ |
| 366 | cmds.BoolOption(progressOptionName, "p", "Stream progress data. Defaults to true when stderr is a terminal."), |
| 367 | }, |
| 368 | Run: dagStat, |
| 369 | Type: DagStatSummary{}, |
| 370 | PostRun: cmds.PostRunMap{ |
| 371 | cmds.CLI: finishCLIStat, |
| 372 | }, |
| 373 | Encoders: cmds.EncoderMap{ |
| 374 | cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, event *DagStatSummary) error { |
| 375 | fmt.Fprintln(w) |
| 376 | csvWriter := csv.NewWriter(w) |
| 377 | csvWriter.Comma = '\t' |
| 378 | cidSpacing := len(event.DagStatsArray[0].Cid) |
| 379 | header := []string{fmt.Sprintf("%-*s", cidSpacing, "CID"), fmt.Sprintf("%-15s", "Blocks"), "Size"} |
| 380 | if err := csvWriter.Write(header); err != nil { |
| 381 | return err |
| 382 | } |
| 383 | for _, dagStat := range event.DagStatsArray { |
| 384 | numBlocksStr := fmt.Sprint(dagStat.NumBlocks) |
| 385 | err := csvWriter.Write([]string{ |
| 386 | dagStat.Cid, |
| 387 | fmt.Sprintf("%-15s", numBlocksStr), |
| 388 | fmt.Sprint(dagStat.Size), |
| 389 | }) |
| 390 | if err != nil { |
| 391 | return err |
| 392 | } |
| 393 | } |
| 394 | csvWriter.Flush() |
| 395 | fmt.Fprint(w, "\nSummary\n") |
| 396 | _, err := fmt.Fprintf( |
| 397 | w, |
| 398 | "%v\n", |
| 399 | event, |
| 400 | ) |
| 401 | fmt.Fprint(w, "\n\n") |
| 402 | return err |
| 403 | }), |
| 404 | cmds.JSON: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, event *DagStatSummary) error { |
| 405 | return json.NewEncoder(w).Encode(event) |
| 406 | }), |
| 407 | }, |
| 408 | } |