master
go 675 lines 20.5 KB
Raw
1 package commands
2
3 import (
4 "context"
5 "errors"
6 "fmt"
7 "io"
8 "runtime"
9 "strings"
10 "sync"
11 "text/tabwriter"
12 "time"
13
14 oldcmds "github.com/ipfs/kubo/commands"
15 cmdenv "github.com/ipfs/kubo/core/commands/cmdenv"
16 coreiface "github.com/ipfs/kubo/core/coreiface"
17 corerepo "github.com/ipfs/kubo/core/corerepo"
18 fsrepo "github.com/ipfs/kubo/repo/fsrepo"
19 "github.com/ipfs/kubo/repo/fsrepo/migrations"
20
21 humanize "github.com/dustin/go-humanize"
22 bstore "github.com/ipfs/boxo/blockstore"
23 "github.com/ipfs/boxo/path"
24 cid "github.com/ipfs/go-cid"
25 cmds "github.com/ipfs/go-ipfs-cmds"
26 )
27
28 type RepoVersion struct {
29 Version string
30 }
31
32 var RepoCmd = &cmds.Command{
33 Helptext: cmds.HelpText{
34 Tagline: "Manipulate the IPFS repo.",
35 ShortDescription: `
36 'ipfs repo' is a plumbing command used to manipulate the repo.
37 `,
38 },
39
40 Subcommands: map[string]*cmds.Command{
41 "stat": repoStatCmd,
42 "gc": repoGcCmd,
43 "version": repoVersionCmd,
44 "verify": repoVerifyCmd,
45 "migrate": repoMigrateCmd,
46 "ls": RefsLocalCmd,
47 },
48 }
49
50 // GcResult is the result returned by "repo gc" command.
51 type GcResult struct {
52 Key cid.Cid
53 Error string `json:",omitempty"`
54 }
55
56 const (
57 repoStreamErrorsOptionName = "stream-errors"
58 repoQuietOptionName = "quiet"
59 repoSilentOptionName = "silent"
60 repoAllowDowngradeOptionName = "allow-downgrade"
61 repoToVersionOptionName = "to"
62 )
63
64 var repoGcCmd = &cmds.Command{
65 Helptext: cmds.HelpText{
66 Tagline: "Perform a garbage collection sweep on the repo.",
67 ShortDescription: `
68 'ipfs repo gc' is a plumbing command that will sweep the local
69 set of stored objects and remove ones that are not pinned in
70 order to reclaim hard disk space.
71 `,
72 },
73 Options: []cmds.Option{
74 cmds.BoolOption(repoStreamErrorsOptionName, "Stream errors."),
75 cmds.BoolOption(repoQuietOptionName, "q", "Write minimal output."),
76 cmds.BoolOption(repoSilentOptionName, "Write no output."),
77 },
78 Run: func(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment) error {
79 n, err := cmdenv.GetNode(env)
80 if err != nil {
81 return err
82 }
83
84 silent, _ := req.Options[repoSilentOptionName].(bool)
85 streamErrors, _ := req.Options[repoStreamErrorsOptionName].(bool)
86
87 gcOutChan := corerepo.GarbageCollectAsync(n, req.Context)
88
89 if streamErrors {
90 errs := false
91 for res := range gcOutChan {
92 if res.Error != nil {
93 if err := re.Emit(&GcResult{Error: res.Error.Error()}); err != nil {
94 return err
95 }
96 errs = true
97 } else {
98 if err := re.Emit(&GcResult{Key: res.KeyRemoved}); err != nil {
99 return err
100 }
101 }
102 }
103 if errs {
104 return errors.New("encountered errors during gc run")
105 }
106 } else {
107 err := corerepo.CollectResult(req.Context, gcOutChan, func(k cid.Cid) {
108 if silent {
109 return
110 }
111 // Nothing to do with this error, really. This
112 // most likely means that the client is gone but
113 // we still need to let the GC finish.
114 _ = re.Emit(&GcResult{Key: k})
115 })
116 if err != nil {
117 return err
118 }
119 }
120
121 return nil
122 },
123 Type: GcResult{},
124 Encoders: cmds.EncoderMap{
125 cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, gcr *GcResult) error {
126 quiet, _ := req.Options[repoQuietOptionName].(bool)
127 silent, _ := req.Options[repoSilentOptionName].(bool)
128
129 if silent {
130 return nil
131 }
132
133 if gcr.Error != "" {
134 _, err := fmt.Fprintf(w, "Error: %s\n", gcr.Error)
135 return err
136 }
137
138 prefix := "removed "
139 if quiet {
140 prefix = ""
141 }
142
143 _, err := fmt.Fprintf(w, "%s%s\n", prefix, gcr.Key)
144 return err
145 }),
146 },
147 }
148
149 const (
150 repoSizeOnlyOptionName = "size-only"
151 repoHumanOptionName = "human"
152 )
153
154 var repoStatCmd = &cmds.Command{
155 Helptext: cmds.HelpText{
156 Tagline: "Get stats for the currently used repo.",
157 ShortDescription: `
158 'ipfs repo stat' provides information about the local set of
159 stored objects. It outputs:
160
161 RepoSize int Size in bytes that the repo is currently taking.
162 StorageMax string Maximum datastore size (from configuration)
163 NumObjects int Number of objects in the local repo.
164 RepoPath string The path to the repo being currently used.
165 Version string The repo version.
166 `,
167 },
168 Options: []cmds.Option{
169 cmds.BoolOption(repoSizeOnlyOptionName, "s", "Only report RepoSize and StorageMax."),
170 cmds.BoolOption(repoHumanOptionName, "H", "Print sizes in human readable format (e.g., 1K 234M 2G)"),
171 },
172 Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
173 n, err := cmdenv.GetNode(env)
174 if err != nil {
175 return err
176 }
177
178 sizeOnly, _ := req.Options[repoSizeOnlyOptionName].(bool)
179 if sizeOnly {
180 sizeStat, err := corerepo.RepoSize(req.Context, n)
181 if err != nil {
182 return err
183 }
184 return cmds.EmitOnce(res, &corerepo.Stat{
185 SizeStat: sizeStat,
186 })
187 }
188
189 stat, err := corerepo.RepoStat(req.Context, n)
190 if err != nil {
191 return err
192 }
193
194 return cmds.EmitOnce(res, &stat)
195 },
196 Type: &corerepo.Stat{},
197 Encoders: cmds.EncoderMap{
198 cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, stat *corerepo.Stat) error {
199 wtr := tabwriter.NewWriter(w, 0, 0, 1, ' ', 0)
200 defer wtr.Flush()
201
202 human, _ := req.Options[repoHumanOptionName].(bool)
203 sizeOnly, _ := req.Options[repoSizeOnlyOptionName].(bool)
204
205 printSize := func(name string, size uint64) {
206 sizeStr := fmt.Sprintf("%d", size)
207 if human {
208 sizeStr = humanize.Bytes(size)
209 }
210
211 fmt.Fprintf(wtr, "%s:\t%s\n", name, sizeStr)
212 }
213
214 if !sizeOnly {
215 fmt.Fprintf(wtr, "NumObjects:\t%d\n", stat.NumObjects)
216 }
217
218 printSize("RepoSize", stat.RepoSize)
219 printSize("StorageMax", stat.StorageMax)
220
221 if !sizeOnly {
222 fmt.Fprintf(wtr, "RepoPath:\t%s\n", stat.RepoPath)
223 fmt.Fprintf(wtr, "Version:\t%s\n", stat.Version)
224 }
225
226 return nil
227 }),
228 },
229 }
230
231 // VerifyProgress reports verification progress to the user.
232 // It contains either a message about a corrupt block or a progress counter.
233 type VerifyProgress struct {
234 Msg string // Message about a corrupt/healed block (empty for valid blocks)
235 Progress int // Number of blocks processed so far
236 }
237
238 // verifyState represents the state of a block after verification.
239 // States track both the verification result and any remediation actions taken.
240 type verifyState int
241
242 const (
243 verifyStateValid verifyState = iota // Block is valid and uncorrupted
244 verifyStateCorrupt // Block is corrupt, no action taken
245 verifyStateCorruptRemoved // Block was corrupt and successfully removed
246 verifyStateCorruptRemoveFailed // Block was corrupt but removal failed
247 verifyStateCorruptHealed // Block was corrupt, removed, and successfully re-fetched
248 verifyStateCorruptHealFailed // Block was corrupt and removed, but re-fetching failed
249 )
250
251 const (
252 // verifyWorkerMultiplier determines worker pool size relative to CPU count.
253 // Since block verification is I/O-bound (disk reads + potential network fetches),
254 // we use more workers than CPU cores to maximize throughput.
255 verifyWorkerMultiplier = 2
256 )
257
258 // verifyResult contains the outcome of verifying a single block.
259 // It includes the block's CID, its verification state, and an optional
260 // human-readable message describing what happened.
261 type verifyResult struct {
262 cid cid.Cid // CID of the block that was verified
263 state verifyState // Final state after verification and any remediation
264 msg string // Human-readable message (empty for valid blocks)
265 }
266
267 // verifyWorkerRun processes CIDs from the keys channel, verifying their integrity.
268 // If shouldDrop is true, corrupt blocks are removed from the blockstore.
269 // If shouldHeal is true (implies shouldDrop), removed blocks are re-fetched from the network.
270 // The api parameter must be non-nil when shouldHeal is true.
271 // healTimeout specifies the maximum time to wait for each block heal (0 = no timeout).
272 func verifyWorkerRun(ctx context.Context, wg *sync.WaitGroup, keys <-chan cid.Cid, results chan<- *verifyResult, bs bstore.Blockstore, api coreiface.CoreAPI, shouldDrop, shouldHeal bool, healTimeout time.Duration) {
273 defer wg.Done()
274
275 sendResult := func(r *verifyResult) bool {
276 select {
277 case results <- r:
278 return true
279 case <-ctx.Done():
280 return false
281 }
282 }
283
284 for k := range keys {
285 _, err := bs.Get(ctx, k)
286 if err != nil {
287 // Block is corrupt
288 result := &verifyResult{cid: k, state: verifyStateCorrupt}
289
290 if !shouldDrop {
291 result.msg = fmt.Sprintf("block %s was corrupt (%s)", k, err)
292 if !sendResult(result) {
293 return
294 }
295 continue
296 }
297
298 // Try to delete
299 if delErr := bs.DeleteBlock(ctx, k); delErr != nil {
300 result.state = verifyStateCorruptRemoveFailed
301 result.msg = fmt.Sprintf("block %s was corrupt (%s), failed to remove (%s)", k, err, delErr)
302 if !sendResult(result) {
303 return
304 }
305 continue
306 }
307
308 if !shouldHeal {
309 result.state = verifyStateCorruptRemoved
310 result.msg = fmt.Sprintf("block %s was corrupt (%s), removed", k, err)
311 if !sendResult(result) {
312 return
313 }
314 continue
315 }
316
317 // Try to heal by re-fetching from network (api is guaranteed non-nil here)
318 healCtx := ctx
319 var healCancel context.CancelFunc
320 if healTimeout > 0 {
321 healCtx, healCancel = context.WithTimeout(ctx, healTimeout)
322 }
323
324 if _, healErr := api.Block().Get(healCtx, path.FromCid(k)); healErr != nil {
325 result.state = verifyStateCorruptHealFailed
326 result.msg = fmt.Sprintf("block %s was corrupt (%s), removed, failed to heal (%s)", k, err, healErr)
327 } else {
328 result.state = verifyStateCorruptHealed
329 result.msg = fmt.Sprintf("block %s was corrupt (%s), removed, healed", k, err)
330 }
331
332 if healCancel != nil {
333 healCancel()
334 }
335
336 if !sendResult(result) {
337 return
338 }
339 continue
340 }
341
342 // Block is valid
343 if !sendResult(&verifyResult{cid: k, state: verifyStateValid}) {
344 return
345 }
346 }
347 }
348
349 // verifyResultChan creates a channel of verification results by spawning multiple worker goroutines
350 // to process blocks in parallel. It returns immediately with a channel that will receive results.
351 func verifyResultChan(ctx context.Context, keys <-chan cid.Cid, bs bstore.Blockstore, api coreiface.CoreAPI, shouldDrop, shouldHeal bool, healTimeout time.Duration) <-chan *verifyResult {
352 results := make(chan *verifyResult)
353
354 go func() {
355 defer close(results)
356
357 var wg sync.WaitGroup
358
359 for i := 0; i < runtime.NumCPU()*verifyWorkerMultiplier; i++ {
360 wg.Add(1)
361 go verifyWorkerRun(ctx, &wg, keys, results, bs, api, shouldDrop, shouldHeal, healTimeout)
362 }
363
364 wg.Wait()
365 }()
366
367 return results
368 }
369
370 var repoVerifyCmd = &cmds.Command{
371 Helptext: cmds.HelpText{
372 Tagline: "Verify all blocks in repo are not corrupted.",
373 ShortDescription: `
374 'ipfs repo verify' checks integrity of all blocks in the local datastore.
375 Each block is read and validated against its CID to ensure data integrity.
376
377 Without any flags, this is a SAFE, read-only check that only reports corrupt
378 blocks without modifying the repository. This can be used as a "dry run" to
379 preview what --drop or --heal would do.
380
381 Use --drop to remove corrupt blocks, or --heal to remove and re-fetch from
382 the network.
383
384 Examples:
385 ipfs repo verify # safe read-only check, reports corrupt blocks
386 ipfs repo verify --drop # remove corrupt blocks
387 ipfs repo verify --heal # remove and re-fetch corrupt blocks
388
389 Exit Codes:
390 0: All blocks are valid, OR all corrupt blocks were successfully remediated
391 (with --drop or --heal)
392 1: Corrupt blocks detected (without flags), OR remediation failed (block
393 removal or healing failed with --drop or --heal)
394
395 Note: --heal requires the daemon to be running in online mode with network
396 connectivity to nodes that have the missing blocks. Make sure the daemon is
397 online and connected to other peers. Healing will attempt to re-fetch each
398 corrupt block from the network after removing it. If a block cannot be found
399 on the network, it will remain deleted.
400
401 WARNING: Both --drop and --heal are DESTRUCTIVE operations that permanently
402 delete corrupt blocks from your repository. Once deleted, blocks cannot be
403 recovered unless --heal successfully fetches them from the network. Blocks
404 that cannot be healed will remain permanently deleted. Always backup your
405 repository before using these options.
406 `,
407 },
408 Options: []cmds.Option{
409 cmds.BoolOption("drop", "Remove corrupt blocks from datastore (destructive operation)."),
410 cmds.BoolOption("heal", "Remove corrupt blocks and re-fetch from network (destructive operation, implies --drop)."),
411 cmds.StringOption("heal-timeout", "Maximum time to wait for each block heal (e.g., \"30s\"). Only applies with --heal.").WithDefault("30s"),
412 },
413 Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
414 nd, err := cmdenv.GetNode(env)
415 if err != nil {
416 return err
417 }
418
419 drop, _ := req.Options["drop"].(bool)
420 heal, _ := req.Options["heal"].(bool)
421
422 if heal {
423 drop = true // heal implies drop
424 }
425
426 // Parse and validate heal-timeout
427 timeoutStr, _ := req.Options["heal-timeout"].(string)
428 healTimeout, err := time.ParseDuration(timeoutStr)
429 if err != nil {
430 return fmt.Errorf("invalid heal-timeout: %w", err)
431 }
432 if healTimeout < 0 {
433 return errors.New("heal-timeout must be >= 0")
434 }
435
436 // Check online mode and API availability for healing operation
437 var api coreiface.CoreAPI
438 if heal {
439 if !nd.IsOnline {
440 return ErrNotOnline
441 }
442 api, err = cmdenv.GetApi(env, req)
443 if err != nil {
444 return err
445 }
446 if api == nil {
447 return fmt.Errorf("healing requested but API is not available - make sure daemon is online and connected to other peers")
448 }
449 }
450
451 bs := &bstore.ValidatingBlockstore{Blockstore: bstore.NewBlockstore(nd.Repo.Datastore())}
452
453 keys, err := bs.AllKeysChan(req.Context)
454 if err != nil {
455 log.Error(err)
456 return err
457 }
458
459 results := verifyResultChan(req.Context, keys, bs, api, drop, heal, healTimeout)
460
461 // Track statistics for each type of outcome
462 var corrupted, removed, removeFailed, healed, healFailed int
463 var i int
464
465 for result := range results {
466 // Update counters based on the block's final state
467 switch result.state {
468 case verifyStateCorrupt:
469 // Block is corrupt but no action was taken (--drop not specified)
470 corrupted++
471 case verifyStateCorruptRemoved:
472 // Block was corrupt and successfully removed (--drop specified)
473 corrupted++
474 removed++
475 case verifyStateCorruptRemoveFailed:
476 // Block was corrupt but couldn't be removed
477 corrupted++
478 removeFailed++
479 case verifyStateCorruptHealed:
480 // Block was corrupt, removed, and successfully re-fetched (--heal specified)
481 corrupted++
482 removed++
483 healed++
484 case verifyStateCorruptHealFailed:
485 // Block was corrupt and removed, but re-fetching failed
486 corrupted++
487 removed++
488 healFailed++
489 default:
490 // verifyStateValid blocks are not counted (they're the expected case)
491 }
492
493 // Emit progress message for corrupt blocks
494 if result.state != verifyStateValid && result.msg != "" {
495 if err := res.Emit(&VerifyProgress{Msg: result.msg}); err != nil {
496 return err
497 }
498 }
499
500 i++
501 if err := res.Emit(&VerifyProgress{Progress: i}); err != nil {
502 return err
503 }
504 }
505
506 if err := req.Context.Err(); err != nil {
507 return err
508 }
509
510 if corrupted > 0 {
511 // Build a summary of what happened with corrupt blocks
512 summary := fmt.Sprintf("verify complete, %d blocks corrupt", corrupted)
513 if removed > 0 {
514 summary += fmt.Sprintf(", %d removed", removed)
515 }
516 if removeFailed > 0 {
517 summary += fmt.Sprintf(", %d failed to remove", removeFailed)
518 }
519 if healed > 0 {
520 summary += fmt.Sprintf(", %d healed", healed)
521 }
522 if healFailed > 0 {
523 summary += fmt.Sprintf(", %d failed to heal", healFailed)
524 }
525
526 // Determine success/failure based on operation mode
527 shouldFail := false
528
529 if !drop {
530 // Detection-only mode: always fail if corruption found
531 shouldFail = true
532 } else if heal {
533 // Heal mode: fail if any removal or heal failed
534 shouldFail = (removeFailed > 0 || healFailed > 0)
535 } else {
536 // Drop mode: fail if any removal failed
537 shouldFail = (removeFailed > 0)
538 }
539
540 if shouldFail {
541 return errors.New(summary)
542 }
543
544 // Success: emit summary as a message instead of error
545 return res.Emit(&VerifyProgress{Msg: summary})
546 }
547
548 return res.Emit(&VerifyProgress{Msg: "verify complete, all blocks validated."})
549 },
550 Type: &VerifyProgress{},
551 Encoders: cmds.EncoderMap{
552 cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, obj *VerifyProgress) error {
553 if strings.Contains(obj.Msg, "was corrupt") {
554 fmt.Fprintln(w, obj.Msg)
555 return nil
556 }
557
558 if obj.Msg != "" {
559 if len(obj.Msg) < 20 {
560 obj.Msg += " "
561 }
562 fmt.Fprintln(w, obj.Msg)
563 return nil
564 }
565
566 fmt.Fprintf(w, "%d blocks processed.\r", obj.Progress)
567 return nil
568 }),
569 },
570 }
571
572 var repoVersionCmd = &cmds.Command{
573 Helptext: cmds.HelpText{
574 Tagline: "Show the repo version.",
575 ShortDescription: `
576 'ipfs repo version' returns the current repo version.
577 `,
578 },
579
580 Options: []cmds.Option{
581 cmds.BoolOption(repoQuietOptionName, "q", "Write minimal output."),
582 },
583 Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
584 return cmds.EmitOnce(res, &RepoVersion{
585 Version: fmt.Sprint(fsrepo.RepoVersion),
586 })
587 },
588 Type: RepoVersion{},
589 Encoders: cmds.EncoderMap{
590 cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *RepoVersion) error {
591 quiet, _ := req.Options[repoQuietOptionName].(bool)
592
593 if quiet {
594 fmt.Fprintf(w, "fs-repo@%s\n", out.Version)
595 } else {
596 fmt.Fprintf(w, "ipfs repo version fs-repo@%s\n", out.Version)
597 }
598 return nil
599 }),
600 },
601 }
602
603 var repoMigrateCmd = &cmds.Command{
604 Helptext: cmds.HelpText{
605 Tagline: "Apply repository migrations to a specific version.",
606 ShortDescription: `
607 'ipfs repo migrate' applies repository migrations to bring the repository
608 to a specific version. By default, migrates to the latest version supported
609 by this IPFS binary.
610
611 Examples:
612 ipfs repo migrate # Migrate to latest version
613 ipfs repo migrate --to=17 # Migrate to version 17
614 ipfs repo migrate --to=16 --allow-downgrade # Downgrade to version 16
615
616 WARNING: Downgrading a repository may cause data loss and requires using
617 an older IPFS binary that supports the target version. After downgrading,
618 you must use an IPFS implementation compatible with that repository version.
619
620 Repository versions 16+ use embedded migrations for faster, more reliable
621 migration. Versions below 16 require external migration tools.
622 `,
623 },
624 Options: []cmds.Option{
625 cmds.IntOption(repoToVersionOptionName, "Target repository version").WithDefault(fsrepo.RepoVersion),
626 cmds.BoolOption(repoAllowDowngradeOptionName, "Allow downgrading to a lower repo version"),
627 },
628 NoRemote: true,
629 // SetDoesNotUseRepo(true) might seem counter-intuitive since migrations
630 // do access the repo, but it's correct - we need direct filesystem access
631 // without going through the daemon. Migrations handle their own locking.
632 Extra: CreateCmdExtras(SetDoesNotUseRepo(true)),
633 Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
634 cctx := env.(*oldcmds.Context)
635 allowDowngrade, _ := req.Options[repoAllowDowngradeOptionName].(bool)
636 targetVersion, _ := req.Options[repoToVersionOptionName].(int)
637
638 // Get current repo version
639 currentVersion, err := migrations.RepoVersion(cctx.ConfigRoot)
640 if err != nil {
641 return fmt.Errorf("could not get current repo version: %w", err)
642 }
643
644 // Check if migration is needed
645 if currentVersion == targetVersion {
646 fmt.Printf("Repository is already at version %d.\n", targetVersion)
647 return nil
648 }
649
650 // Validate downgrade request
651 if targetVersion < currentVersion && !allowDowngrade {
652 return fmt.Errorf("downgrade from version %d to %d requires --allow-downgrade flag", currentVersion, targetVersion)
653 }
654
655 fmt.Printf("Migrating repository from version %d to %d...\n", currentVersion, targetVersion)
656
657 // Use hybrid migration strategy that intelligently combines external and embedded migrations
658 // Use req.Context instead of cctx.Context() to avoid opening the repo before migrations run,
659 // which would acquire the lock that migrations need
660 err = migrations.RunHybridMigrations(req.Context, targetVersion, cctx.ConfigRoot, allowDowngrade)
661 if err != nil {
662 fmt.Println("Repository migration failed:")
663 fmt.Printf(" %s\n", err)
664 fmt.Println("If you think this is a bug, please file an issue and include this whole log output.")
665 fmt.Println(" https://github.com/ipfs/kubo")
666 return err
667 }
668
669 fmt.Printf("Repository successfully migrated to version %d.\n", targetVersion)
670 if targetVersion < fsrepo.RepoVersion {
671 fmt.Println("WARNING: After downgrading, you must use an IPFS binary compatible with this repository version.")
672 }
673 return nil
674 },
675 }