@cryptotaxi247 / kubo / commits / c7eda21d6

test: verifyWorkerRun and helptext (#11063)

Marcin Rataj committed Nov 17, 2025 at 18:51 UTC c7eda21d686325e3d050f693adcf74b2b6e296ae
4 files changed +1011 -24
core/commands/repo.go
+253 -24
@@ -5,20 +5,22 @@ import (
5 "errors"
6 "fmt"
7 "io"
8 - "os"
8 "runtime"
9 "strings"
10 "sync"
11 "text/tabwriter"
12 + "time"
13
14 oldcmds "github.com/ipfs/kubo/commands"
15 cmdenv "github.com/ipfs/kubo/core/commands/cmdenv"
16 + coreiface "github.com/ipfs/kubo/core/coreiface"
17 corerepo "github.com/ipfs/kubo/core/corerepo"
18 fsrepo "github.com/ipfs/kubo/repo/fsrepo"
19 "github.com/ipfs/kubo/repo/fsrepo/migrations"
20
21 humanize "github.com/dustin/go-humanize"
22 bstore "github.com/ipfs/boxo/blockstore"
23 + "github.com/ipfs/boxo/path"
24 cid "github.com/ipfs/go-cid"
25 cmds "github.com/ipfs/go-ipfs-cmds"
26 )
@@ -226,45 +228,137 @@ Version string The repo version.
228 },
229 }
230
231 +// VerifyProgress reports verification progress to the user.
232 +// It contains either a message about a corrupt block or a progress counter.
233 type VerifyProgress struct {
230 - Msg string
231 - Progress int
234 + Msg string // Message about a corrupt/healed block (empty for valid blocks)
235 + Progress int // Number of blocks processed so far
236 }
237
234 -func verifyWorkerRun(ctx context.Context, wg *sync.WaitGroup, keys <-chan cid.Cid, results chan<- string, bs bstore.Blockstore) {
238 +// verifyState represents the state of a block after verification.
239 +// States track both the verification result and any remediation actions taken.
240 +type verifyState int
241 +
242 +const (
243 + verifyStateValid verifyState = iota // Block is valid and uncorrupted
244 + verifyStateCorrupt // Block is corrupt, no action taken
245 + verifyStateCorruptRemoved // Block was corrupt and successfully removed
246 + verifyStateCorruptRemoveFailed // Block was corrupt but removal failed
247 + verifyStateCorruptHealed // Block was corrupt, removed, and successfully re-fetched
248 + verifyStateCorruptHealFailed // Block was corrupt and removed, but re-fetching failed
249 +)
250 +
251 +const (
252 + // verifyWorkerMultiplier determines worker pool size relative to CPU count.
253 + // Since block verification is I/O-bound (disk reads + potential network fetches),
254 + // we use more workers than CPU cores to maximize throughput.
255 + verifyWorkerMultiplier = 2
256 +)
257 +
258 +// verifyResult contains the outcome of verifying a single block.
259 +// It includes the block's CID, its verification state, and an optional
260 +// human-readable message describing what happened.
261 +type verifyResult struct {
262 + cid cid.Cid // CID of the block that was verified
263 + state verifyState // Final state after verification and any remediation
264 + msg string // Human-readable message (empty for valid blocks)
265 +}
266 +
267 +// verifyWorkerRun processes CIDs from the keys channel, verifying their integrity.
268 +// If shouldDrop is true, corrupt blocks are removed from the blockstore.
269 +// If shouldHeal is true (implies shouldDrop), removed blocks are re-fetched from the network.
270 +// The api parameter must be non-nil when shouldHeal is true.
271 +// healTimeout specifies the maximum time to wait for each block heal (0 = no timeout).
272 +func verifyWorkerRun(ctx context.Context, wg *sync.WaitGroup, keys <-chan cid.Cid, results chan<- *verifyResult, bs bstore.Blockstore, api coreiface.CoreAPI, shouldDrop, shouldHeal bool, healTimeout time.Duration) {
273 defer wg.Done()
274
275 + sendResult := func(r *verifyResult) bool {
276 + select {
277 + case results <- r:
278 + return true
279 + case <-ctx.Done():
280 + return false
281 + }
282 + }
283 +
284 for k := range keys {
285 _, err := bs.Get(ctx, k)
286 if err != nil {
240 - select {
241 - case results <- fmt.Sprintf("block %s was corrupt (%s)", k, err):
242 - case <-ctx.Done():
243 - return
287 + // Block is corrupt
288 + result := &verifyResult{cid: k, state: verifyStateCorrupt}
289 +
290 + if !shouldDrop {
291 + result.msg = fmt.Sprintf("block %s was corrupt (%s)", k, err)
292 + if !sendResult(result) {
293 + return
294 + }
295 + continue
296 + }
297 +
298 + // Try to delete
299 + if delErr := bs.DeleteBlock(ctx, k); delErr != nil {
300 + result.state = verifyStateCorruptRemoveFailed
301 + result.msg = fmt.Sprintf("block %s was corrupt (%s), failed to remove (%s)", k, err, delErr)
302 + if !sendResult(result) {
303 + return
304 + }
305 + continue
306 + }
307 +
308 + if !shouldHeal {
309 + result.state = verifyStateCorruptRemoved
310 + result.msg = fmt.Sprintf("block %s was corrupt (%s), removed", k, err)
311 + if !sendResult(result) {
312 + return
313 + }
314 + continue
315 }
316
317 + // Try to heal by re-fetching from network (api is guaranteed non-nil here)
318 + healCtx := ctx
319 + var healCancel context.CancelFunc
320 + if healTimeout > 0 {
321 + healCtx, healCancel = context.WithTimeout(ctx, healTimeout)
322 + }
323 +
324 + if _, healErr := api.Block().Get(healCtx, path.FromCid(k)); healErr != nil {
325 + result.state = verifyStateCorruptHealFailed
326 + result.msg = fmt.Sprintf("block %s was corrupt (%s), removed, failed to heal (%s)", k, err, healErr)
327 + } else {
328 + result.state = verifyStateCorruptHealed
329 + result.msg = fmt.Sprintf("block %s was corrupt (%s), removed, healed", k, err)
330 + }
331 +
332 + if healCancel != nil {
333 + healCancel()
334 + }
335 +
336 + if !sendResult(result) {
337 + return
338 + }
339 continue
340 }
341
249 - select {
250 - case results <- "":
251 - case <-ctx.Done():
342 + // Block is valid
343 + if !sendResult(&verifyResult{cid: k, state: verifyStateValid}) {
344 return
345 }
346 }
347 }
348
257 -func verifyResultChan(ctx context.Context, keys <-chan cid.Cid, bs bstore.Blockstore) <-chan string {
258 - results := make(chan string)
349 +// verifyResultChan creates a channel of verification results by spawning multiple worker goroutines
350 +// to process blocks in parallel. It returns immediately with a channel that will receive results.
351 +func verifyResultChan(ctx context.Context, keys <-chan cid.Cid, bs bstore.Blockstore, api coreiface.CoreAPI, shouldDrop, shouldHeal bool, healTimeout time.Duration) <-chan *verifyResult {
352 + results := make(chan *verifyResult)
353
354 go func() {
355 defer close(results)
356
357 var wg sync.WaitGroup
358
265 - for i := 0; i < runtime.NumCPU()*2; i++ {
359 + for i := 0; i < runtime.NumCPU()*verifyWorkerMultiplier; i++ {
360 wg.Add(1)
267 - go verifyWorkerRun(ctx, &wg, keys, results, bs)
361 + go verifyWorkerRun(ctx, &wg, keys, results, bs, api, shouldDrop, shouldHeal, healTimeout)
362 }
363
364 wg.Wait()
@@ -276,6 +370,45 @@ func verifyResultChan(ctx context.Context, keys <-chan cid.Cid, bs bstore.Blocks
370 var repoVerifyCmd = &cmds.Command{
371 Helptext: cmds.HelpText{
372 Tagline: "Verify all blocks in repo are not corrupted.",
373 + ShortDescription: `
374 +'ipfs repo verify' checks integrity of all blocks in the local datastore.
375 +Each block is read and validated against its CID to ensure data integrity.
376 +
377 +Without any flags, this is a SAFE, read-only check that only reports corrupt
378 +blocks without modifying the repository. This can be used as a "dry run" to
379 +preview what --drop or --heal would do.
380 +
381 +Use --drop to remove corrupt blocks, or --heal to remove and re-fetch from
382 +the network.
383 +
384 +Examples:
385 + ipfs repo verify # safe read-only check, reports corrupt blocks
386 + ipfs repo verify --drop # remove corrupt blocks
387 + ipfs repo verify --heal # remove and re-fetch corrupt blocks
388 +
389 +Exit Codes:
390 + 0: All blocks are valid, OR all corrupt blocks were successfully remediated
391 + (with --drop or --heal)
392 + 1: Corrupt blocks detected (without flags), OR remediation failed (block
393 + removal or healing failed with --drop or --heal)
394 +
395 +Note: --heal requires the daemon to be running in online mode with network
396 +connectivity to nodes that have the missing blocks. Make sure the daemon is
397 +online and connected to other peers. Healing will attempt to re-fetch each
398 +corrupt block from the network after removing it. If a block cannot be found
399 +on the network, it will remain deleted.
400 +
401 +WARNING: Both --drop and --heal are DESTRUCTIVE operations that permanently
402 +delete corrupt blocks from your repository. Once deleted, blocks cannot be
403 +recovered unless --heal successfully fetches them from the network. Blocks
404 +that cannot be healed will remain permanently deleted. Always backup your
405 +repository before using these options.
406 +`,
407 + },
408 + Options: []cmds.Option{
409 + cmds.BoolOption("drop", "Remove corrupt blocks from datastore (destructive operation)."),
410 + cmds.BoolOption("heal", "Remove corrupt blocks and re-fetch from network (destructive operation, implies --drop)."),
411 + cmds.StringOption("heal-timeout", "Maximum time to wait for each block heal (e.g., \"30s\"). Only applies with --heal.").WithDefault("30s"),
412 },
413 Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
414 nd, err := cmdenv.GetNode(env)
@@ -283,6 +416,38 @@ var repoVerifyCmd = &cmds.Command{
416 return err
417 }
418
419 + drop, _ := req.Options["drop"].(bool)
420 + heal, _ := req.Options["heal"].(bool)
421 +
422 + if heal {
423 + drop = true // heal implies drop
424 + }
425 +
426 + // Parse and validate heal-timeout
427 + timeoutStr, _ := req.Options["heal-timeout"].(string)
428 + healTimeout, err := time.ParseDuration(timeoutStr)
429 + if err != nil {
430 + return fmt.Errorf("invalid heal-timeout: %w", err)
431 + }
432 + if healTimeout < 0 {
433 + return errors.New("heal-timeout must be >= 0")
434 + }
435 +
436 + // Check online mode and API availability for healing operation
437 + var api coreiface.CoreAPI
438 + if heal {
439 + if !nd.IsOnline {
440 + return ErrNotOnline
441 + }
442 + api, err = cmdenv.GetApi(env, req)
443 + if err != nil {
444 + return err
445 + }
446 + if api == nil {
447 + return fmt.Errorf("healing requested but API is not available - make sure daemon is online and connected to other peers")
448 + }
449 + }
450 +
451 bs := &bstore.ValidatingBlockstore{Blockstore: bstore.NewBlockstore(nd.Repo.Datastore())}
452
453 keys, err := bs.AllKeysChan(req.Context)
@@ -291,17 +456,47 @@ var repoVerifyCmd = &cmds.Command{
456 return err
457 }
458
294 - results := verifyResultChan(req.Context, keys, bs)
459 + results := verifyResultChan(req.Context, keys, bs, api, drop, heal, healTimeout)
460
296 - var fails int
461 + // Track statistics for each type of outcome
462 + var corrupted, removed, removeFailed, healed, healFailed int
463 var i int
298 - for msg := range results {
299 - if msg != "" {
300 - if err := res.Emit(&VerifyProgress{Msg: msg}); err != nil {
464 +
465 + for result := range results {
466 + // Update counters based on the block's final state
467 + switch result.state {
468 + case verifyStateCorrupt:
469 + // Block is corrupt but no action was taken (--drop not specified)
470 + corrupted++
471 + case verifyStateCorruptRemoved:
472 + // Block was corrupt and successfully removed (--drop specified)
473 + corrupted++
474 + removed++
475 + case verifyStateCorruptRemoveFailed:
476 + // Block was corrupt but couldn't be removed
477 + corrupted++
478 + removeFailed++
479 + case verifyStateCorruptHealed:
480 + // Block was corrupt, removed, and successfully re-fetched (--heal specified)
481 + corrupted++
482 + removed++
483 + healed++
484 + case verifyStateCorruptHealFailed:
485 + // Block was corrupt and removed, but re-fetching failed
486 + corrupted++
487 + removed++
488 + healFailed++
489 + default:
490 + // verifyStateValid blocks are not counted (they're the expected case)
491 + }
492 +
493 + // Emit progress message for corrupt blocks
494 + if result.state != verifyStateValid && result.msg != "" {
495 + if err := res.Emit(&VerifyProgress{Msg: result.msg}); err != nil {
496 return err
497 }
303 - fails++
498 }
499 +
500 i++
501 if err := res.Emit(&VerifyProgress{Progress: i}); err != nil {
502 return err
@@ -312,8 +507,42 @@ var repoVerifyCmd = &cmds.Command{
507 return err
508 }
509
315 - if fails != 0 {
316 - return errors.New("verify complete, some blocks were corrupt")
510 + if corrupted > 0 {
511 + // Build a summary of what happened with corrupt blocks
512 + summary := fmt.Sprintf("verify complete, %d blocks corrupt", corrupted)
513 + if removed > 0 {
514 + summary += fmt.Sprintf(", %d removed", removed)
515 + }
516 + if removeFailed > 0 {
517 + summary += fmt.Sprintf(", %d failed to remove", removeFailed)
518 + }
519 + if healed > 0 {
520 + summary += fmt.Sprintf(", %d healed", healed)
521 + }
522 + if healFailed > 0 {
523 + summary += fmt.Sprintf(", %d failed to heal", healFailed)
524 + }
525 +
526 + // Determine success/failure based on operation mode
527 + shouldFail := false
528 +
529 + if !drop {
530 + // Detection-only mode: always fail if corruption found
531 + shouldFail = true
532 + } else if heal {
533 + // Heal mode: fail if any removal or heal failed
534 + shouldFail = (removeFailed > 0 || healFailed > 0)
535 + } else {
536 + // Drop mode: fail if any removal failed
537 + shouldFail = (removeFailed > 0)
538 + }
539 +
540 + if shouldFail {
541 + return errors.New(summary)
542 + }
543 +
544 + // Success: emit summary as a message instead of error
545 + return res.Emit(&VerifyProgress{Msg: summary})
546 }
547
548 return res.Emit(&VerifyProgress{Msg: "verify complete, all blocks validated."})
@@ -322,7 +551,7 @@ var repoVerifyCmd = &cmds.Command{
551 Encoders: cmds.EncoderMap{
552 cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, obj *VerifyProgress) error {
553 if strings.Contains(obj.Msg, "was corrupt") {
325 - fmt.Fprintln(os.Stdout, obj.Msg)
554 + fmt.Fprintln(w, obj.Msg)
555 return nil
556 }
557
core/commands/repo_verify_test.go new
+371
@@ -0,0 +1,371 @@
1 +//go:build go1.25
2 +
3 +package commands
4 +
5 +// This file contains unit tests for the --heal-timeout flag functionality
6 +// using testing/synctest to avoid waiting for real timeouts.
7 +//
8 +// End-to-end tests for the full 'ipfs repo verify' command (including --drop
9 +// and --heal flags) are located in test/cli/repo_verify_test.go.
10 +
11 +import (
12 + "bytes"
13 + "context"
14 + "errors"
15 + "io"
16 + "sync"
17 + "testing"
18 + "testing/synctest"
19 + "time"
20 +
21 + blocks "github.com/ipfs/go-block-format"
22 + "github.com/ipfs/go-cid"
23 + ipld "github.com/ipfs/go-ipld-format"
24 + coreiface "github.com/ipfs/kubo/core/coreiface"
25 + "github.com/ipfs/kubo/core/coreiface/options"
26 + "github.com/stretchr/testify/assert"
27 + "github.com/stretchr/testify/require"
28 +
29 + "github.com/ipfs/boxo/path"
30 +)
31 +
32 +func TestVerifyWorkerHealTimeout(t *testing.T) {
33 + t.Run("heal succeeds before timeout", func(t *testing.T) {
34 + synctest.Test(t, func(t *testing.T) {
35 + const healTimeout = 5 * time.Second
36 + testCID := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi")
37 +
38 + // Setup channels
39 + keys := make(chan cid.Cid, 1)
40 + keys <- testCID
41 + close(keys)
42 + results := make(chan *verifyResult, 1)
43 +
44 + // Mock blockstore that returns error (simulating corruption)
45 + mockBS := &mockBlockstore{
46 + getError: errors.New("corrupt block"),
47 + }
48 +
49 + // Mock API where Block().Get() completes before timeout
50 + mockAPI := &mockCoreAPI{
51 + blockAPI: &mockBlockAPI{
52 + getDelay: 2 * time.Second, // Less than healTimeout
53 + data: []byte("healed data"),
54 + },
55 + }
56 +
57 + var wg sync.WaitGroup
58 + wg.Add(1)
59 +
60 + // Run worker
61 + go verifyWorkerRun(t.Context(), &wg, keys, results, mockBS, mockAPI, true, true, healTimeout)
62 +
63 + // Advance time past the mock delay but before timeout
64 + time.Sleep(3 * time.Second)
65 + synctest.Wait()
66 +
67 + wg.Wait()
68 + close(results)
69 +
70 + // Verify heal succeeded
71 + result := <-results
72 + require.NotNil(t, result)
73 + assert.Equal(t, verifyStateCorruptHealed, result.state)
74 + assert.Contains(t, result.msg, "healed")
75 + })
76 + })
77 +
78 + t.Run("heal fails due to timeout", func(t *testing.T) {
79 + synctest.Test(t, func(t *testing.T) {
80 + const healTimeout = 2 * time.Second
81 + testCID := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi")
82 +
83 + // Setup channels
84 + keys := make(chan cid.Cid, 1)
85 + keys <- testCID
86 + close(keys)
87 + results := make(chan *verifyResult, 1)
88 +
89 + // Mock blockstore that returns error (simulating corruption)
90 + mockBS := &mockBlockstore{
91 + getError: errors.New("corrupt block"),
92 + }
93 +
94 + // Mock API where Block().Get() takes longer than healTimeout
95 + mockAPI := &mockCoreAPI{
96 + blockAPI: &mockBlockAPI{
97 + getDelay: 5 * time.Second, // More than healTimeout
98 + data: []byte("healed data"),
99 + },
100 + }
101 +
102 + var wg sync.WaitGroup
103 + wg.Add(1)
104 +
105 + // Run worker
106 + go verifyWorkerRun(t.Context(), &wg, keys, results, mockBS, mockAPI, true, true, healTimeout)
107 +
108 + // Advance time past timeout
109 + time.Sleep(3 * time.Second)
110 + synctest.Wait()
111 +
112 + wg.Wait()
113 + close(results)
114 +
115 + // Verify heal failed due to timeout
116 + result := <-results
117 + require.NotNil(t, result)
118 + assert.Equal(t, verifyStateCorruptHealFailed, result.state)
119 + assert.Contains(t, result.msg, "failed to heal")
120 + assert.Contains(t, result.msg, "context deadline exceeded")
121 + })
122 + })
123 +
124 + t.Run("heal with zero timeout still attempts heal", func(t *testing.T) {
125 + synctest.Test(t, func(t *testing.T) {
126 + const healTimeout = 0 // Zero timeout means no timeout
127 + testCID := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi")
128 +
129 + // Setup channels
130 + keys := make(chan cid.Cid, 1)
131 + keys <- testCID
132 + close(keys)
133 + results := make(chan *verifyResult, 1)
134 +
135 + // Mock blockstore that returns error (simulating corruption)
136 + mockBS := &mockBlockstore{
137 + getError: errors.New("corrupt block"),
138 + }
139 +
140 + // Mock API that succeeds quickly
141 + mockAPI := &mockCoreAPI{
142 + blockAPI: &mockBlockAPI{
143 + getDelay: 100 * time.Millisecond,
144 + data: []byte("healed data"),
145 + },
146 + }
147 +
148 + var wg sync.WaitGroup
149 + wg.Add(1)
150 +
151 + // Run worker
152 + go verifyWorkerRun(t.Context(), &wg, keys, results, mockBS, mockAPI, true, true, healTimeout)
153 +
154 + // Advance time to let heal complete
155 + time.Sleep(200 * time.Millisecond)
156 + synctest.Wait()
157 +
158 + wg.Wait()
159 + close(results)
160 +
161 + // Verify heal succeeded even with zero timeout
162 + result := <-results
163 + require.NotNil(t, result)
164 + assert.Equal(t, verifyStateCorruptHealed, result.state)
165 + assert.Contains(t, result.msg, "healed")
166 + })
167 + })
168 +
169 + t.Run("multiple blocks with different timeout outcomes", func(t *testing.T) {
170 + synctest.Test(t, func(t *testing.T) {
171 + const healTimeout = 3 * time.Second
172 + testCID1 := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi")
173 + testCID2 := cid.MustParse("bafybeihvvulpp4evxj7x7armbqcyg6uezzuig6jp3lktpbovlqfkjtgyby")
174 +
175 + // Setup channels
176 + keys := make(chan cid.Cid, 2)
177 + keys <- testCID1
178 + keys <- testCID2
179 + close(keys)
180 + results := make(chan *verifyResult, 2)
181 +
182 + // Mock blockstore that always returns error (all blocks corrupt)
183 + mockBS := &mockBlockstore{
184 + getError: errors.New("corrupt block"),
185 + }
186 +
187 + // Create two mock block APIs with different delays
188 + // We'll need to alternate which one gets used
189 + // For simplicity, use one that succeeds fast
190 + mockAPI := &mockCoreAPI{
191 + blockAPI: &mockBlockAPI{
192 + getDelay: 1 * time.Second, // Less than healTimeout - will succeed
193 + data: []byte("healed data"),
194 + },
195 + }
196 +
197 + var wg sync.WaitGroup
198 + wg.Add(2) // Two workers
199 +
200 + // Run two workers
201 + go verifyWorkerRun(t.Context(), &wg, keys, results, mockBS, mockAPI, true, true, healTimeout)
202 + go verifyWorkerRun(t.Context(), &wg, keys, results, mockBS, mockAPI, true, true, healTimeout)
203 +
204 + // Advance time to let both complete
205 + time.Sleep(2 * time.Second)
206 + synctest.Wait()
207 +
208 + wg.Wait()
209 + close(results)
210 +
211 + // Collect results
212 + var healedCount int
213 + for result := range results {
214 + if result.state == verifyStateCorruptHealed {
215 + healedCount++
216 + }
217 + }
218 +
219 + // Both should heal successfully (both under timeout)
220 + assert.Equal(t, 2, healedCount)
221 + })
222 + })
223 +
224 + t.Run("valid block is not healed", func(t *testing.T) {
225 + synctest.Test(t, func(t *testing.T) {
226 + const healTimeout = 5 * time.Second
227 + testCID := cid.MustParse("bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi")
228 +
229 + // Setup channels
230 + keys := make(chan cid.Cid, 1)
231 + keys <- testCID
232 + close(keys)
233 + results := make(chan *verifyResult, 1)
234 +
235 + // Mock blockstore that returns valid block (no error)
236 + mockBS := &mockBlockstore{
237 + block: blocks.NewBlock([]byte("valid data")),
238 + }
239 +
240 + // Mock API (won't be called since block is valid)
241 + mockAPI := &mockCoreAPI{
242 + blockAPI: &mockBlockAPI{},
243 + }
244 +
245 + var wg sync.WaitGroup
246 + wg.Add(1)
247 +
248 + // Run worker with heal enabled
249 + go verifyWorkerRun(t.Context(), &wg, keys, results, mockBS, mockAPI, false, true, healTimeout)
250 +
251 + synctest.Wait()
252 +
253 + wg.Wait()
254 + close(results)
255 +
256 + // Verify block is marked valid, not healed
257 + result := <-results
258 + require.NotNil(t, result)
259 + assert.Equal(t, verifyStateValid, result.state)
260 + assert.Empty(t, result.msg)
261 + })
262 + })
263 +}
264 +
265 +// mockBlockstore implements a minimal blockstore for testing
266 +type mockBlockstore struct {
267 + getError error
268 + block blocks.Block
269 +}
270 +
271 +func (m *mockBlockstore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
272 + if m.getError != nil {
273 + return nil, m.getError
274 + }
275 + return m.block, nil
276 +}
277 +
278 +func (m *mockBlockstore) DeleteBlock(ctx context.Context, c cid.Cid) error {
279 + return nil
280 +}
281 +
282 +func (m *mockBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) {
283 + return m.block != nil, nil
284 +}
285 +
286 +func (m *mockBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
287 + if m.block != nil {
288 + return len(m.block.RawData()), nil
289 + }
290 + return 0, errors.New("block not found")
291 +}
292 +
293 +func (m *mockBlockstore) Put(ctx context.Context, b blocks.Block) error {
294 + return nil
295 +}
296 +
297 +func (m *mockBlockstore) PutMany(ctx context.Context, bs []blocks.Block) error {
298 + return nil
299 +}
300 +
301 +func (m *mockBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
302 + return nil, errors.New("not implemented")
303 +}
304 +
305 +func (m *mockBlockstore) HashOnRead(enabled bool) {
306 +}
307 +
308 +// mockBlockAPI implements BlockAPI for testing
309 +type mockBlockAPI struct {
310 + getDelay time.Duration
311 + getError error
312 + data []byte
313 +}
314 +
315 +func (m *mockBlockAPI) Get(ctx context.Context, p path.Path) (io.Reader, error) {
316 + if m.getDelay > 0 {
317 + select {
318 + case <-time.After(m.getDelay):
319 + // Delay completed
320 + case <-ctx.Done():
321 + return nil, ctx.Err()
322 + }
323 + }
324 + if m.getError != nil {
325 + return nil, m.getError
326 + }
327 + return bytes.NewReader(m.data), nil
328 +}
329 +
330 +func (m *mockBlockAPI) Put(ctx context.Context, r io.Reader, opts ...options.BlockPutOption) (coreiface.BlockStat, error) {
331 + return nil, errors.New("not implemented")
332 +}
333 +
334 +func (m *mockBlockAPI) Rm(ctx context.Context, p path.Path, opts ...options.BlockRmOption) error {
335 + return errors.New("not implemented")
336 +}
337 +
338 +func (m *mockBlockAPI) Stat(ctx context.Context, p path.Path) (coreiface.BlockStat, error) {
339 + return nil, errors.New("not implemented")
340 +}
341 +
342 +// mockCoreAPI implements minimal CoreAPI for testing
343 +type mockCoreAPI struct {
344 + blockAPI *mockBlockAPI
345 +}
346 +
347 +func (m *mockCoreAPI) Block() coreiface.BlockAPI {
348 + return m.blockAPI
349 +}
350 +
351 +func (m *mockCoreAPI) Unixfs() coreiface.UnixfsAPI { return nil }
352 +func (m *mockCoreAPI) Dag() coreiface.APIDagService { return nil }
353 +func (m *mockCoreAPI) Name() coreiface.NameAPI { return nil }
354 +func (m *mockCoreAPI) Key() coreiface.KeyAPI { return nil }
355 +func (m *mockCoreAPI) Pin() coreiface.PinAPI { return nil }
356 +func (m *mockCoreAPI) Object() coreiface.ObjectAPI { return nil }
357 +func (m *mockCoreAPI) Swarm() coreiface.SwarmAPI { return nil }
358 +func (m *mockCoreAPI) PubSub() coreiface.PubSubAPI { return nil }
359 +func (m *mockCoreAPI) Routing() coreiface.RoutingAPI { return nil }
360 +
361 +func (m *mockCoreAPI) ResolvePath(ctx context.Context, p path.Path) (path.ImmutablePath, []string, error) {
362 + return path.ImmutablePath{}, nil, errors.New("not implemented")
363 +}
364 +
365 +func (m *mockCoreAPI) ResolveNode(ctx context.Context, p path.Path) (ipld.Node, error) {
366 + return nil, errors.New("not implemented")
367 +}
368 +
369 +func (m *mockCoreAPI) WithOptions(...options.ApiOption) (coreiface.CoreAPI, error) {
370 + return nil, errors.New("not implemented")
371 +}
test/cli/repo_verify_test.go new
+384
@@ -0,0 +1,384 @@
1 +package cli
2 +
3 +import (
4 + "fmt"
5 + "os"
6 + "path/filepath"
7 + "strings"
8 + "testing"
9 +
10 + "github.com/ipfs/kubo/test/cli/harness"
11 + "github.com/stretchr/testify/assert"
12 + "github.com/stretchr/testify/require"
13 +)
14 +
15 +// Well-known block file names in flatfs blockstore that should not be corrupted during testing.
16 +// Flatfs stores each block as a separate .data file on disk.
17 +const (
18 + // emptyFileFlatfsFilename is the flatfs filename for an empty UnixFS file block
19 + emptyFileFlatfsFilename = "CIQL7TG2PB52XIZLLHDYIUFMHUQLMMZWBNBZSLDXFCPZ5VDNQQ2WDZQ"
20 + // emptyDirFlatfsFilename is the flatfs filename for an empty UnixFS directory block.
21 + // This block has special handling and may be served from memory even when corrupted on disk.
22 + emptyDirFlatfsFilename = "CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y"
23 +)
24 +
25 +// getEligibleFlatfsBlockFiles returns flatfs block files (*.data) that are safe to corrupt in tests.
26 +// Filters out well-known blocks (empty file/dir) that cause test flakiness.
27 +//
28 +// Note: This helper is specific to the flatfs blockstore implementation where each block
29 +// is stored as a separate file on disk under blocks/*/*.data.
30 +func getEligibleFlatfsBlockFiles(t *testing.T, node *harness.Node) []string {
31 + blockFiles, err := filepath.Glob(filepath.Join(node.Dir, "blocks", "*", "*.data"))
32 + require.NoError(t, err)
33 + require.NotEmpty(t, blockFiles, "no flatfs block files found")
34 +
35 + var eligible []string
36 + for _, f := range blockFiles {
37 + name := filepath.Base(f)
38 + if !strings.Contains(name, emptyFileFlatfsFilename) &&
39 + !strings.Contains(name, emptyDirFlatfsFilename) {
40 + eligible = append(eligible, f)
41 + }
42 + }
43 + return eligible
44 +}
45 +
46 +// corruptRandomBlock corrupts a random block file in the flatfs blockstore.
47 +// Returns the path to the corrupted file.
48 +func corruptRandomBlock(t *testing.T, node *harness.Node) string {
49 + eligible := getEligibleFlatfsBlockFiles(t, node)
50 + require.NotEmpty(t, eligible, "no eligible blocks to corrupt")
51 +
52 + toCorrupt := eligible[0]
53 + err := os.WriteFile(toCorrupt, []byte("corrupted data"), 0644)
54 + require.NoError(t, err)
55 +
56 + return toCorrupt
57 +}
58 +
59 +// corruptMultipleBlocks corrupts multiple block files in the flatfs blockstore.
60 +// Returns the paths to the corrupted files.
61 +func corruptMultipleBlocks(t *testing.T, node *harness.Node, count int) []string {
62 + eligible := getEligibleFlatfsBlockFiles(t, node)
63 + require.GreaterOrEqual(t, len(eligible), count, "not enough eligible blocks to corrupt")
64 +
65 + var corrupted []string
66 + for i := 0; i < count && i < len(eligible); i++ {
67 + err := os.WriteFile(eligible[i], []byte(fmt.Sprintf("corrupted data %d", i)), 0644)
68 + require.NoError(t, err)
69 + corrupted = append(corrupted, eligible[i])
70 + }
71 +
72 + return corrupted
73 +}
74 +
75 +func TestRepoVerify(t *testing.T) {
76 + t.Run("healthy repo passes", func(t *testing.T) {
77 + t.Parallel()
78 + node := harness.NewT(t).NewNode().Init()
79 + node.IPFS("add", "-q", "--raw-leaves=false", "-r", node.IPFSBin)
80 +
81 + res := node.IPFS("repo", "verify")
82 + assert.Contains(t, res.Stdout.String(), "all blocks validated")
83 + })
84 +
85 + t.Run("detects corruption", func(t *testing.T) {
86 + t.Parallel()
87 + node := harness.NewT(t).NewNode().Init()
88 + node.IPFSAddStr("test content")
89 +
90 + corruptRandomBlock(t, node)
91 +
92 + res := node.RunIPFS("repo", "verify")
93 + assert.Equal(t, 1, res.ExitCode())
94 + assert.Contains(t, res.Stdout.String(), "was corrupt")
95 + assert.Contains(t, res.Stderr.String(), "1 blocks corrupt")
96 + })
97 +
98 + t.Run("drop removes corrupt blocks", func(t *testing.T) {
99 + t.Parallel()
100 + node := harness.NewT(t).NewNode().Init()
101 + cid := node.IPFSAddStr("test content")
102 +
103 + corruptRandomBlock(t, node)
104 +
105 + res := node.RunIPFS("repo", "verify", "--drop")
106 + assert.Equal(t, 0, res.ExitCode(), "should exit 0 when all corrupt blocks removed successfully")
107 + output := res.Stdout.String()
108 + assert.Contains(t, output, "1 blocks corrupt")
109 + assert.Contains(t, output, "1 removed")
110 +
111 + // Verify block is gone
112 + res = node.RunIPFS("block", "stat", cid)
113 + assert.NotEqual(t, 0, res.ExitCode())
114 + })
115 +
116 + t.Run("heal requires online mode", func(t *testing.T) {
117 + t.Parallel()
118 + node := harness.NewT(t).NewNode().Init()
119 + node.IPFSAddStr("test content")
120 +
121 + corruptRandomBlock(t, node)
122 +
123 + res := node.RunIPFS("repo", "verify", "--heal")
124 + assert.NotEqual(t, 0, res.ExitCode())
125 + assert.Contains(t, res.Stderr.String(), "online mode")
126 + })
127 +
128 + t.Run("heal repairs from network", func(t *testing.T) {
129 + t.Parallel()
130 + nodes := harness.NewT(t).NewNodes(2).Init()
131 + nodes.StartDaemons().Connect()
132 + defer nodes.StopDaemons()
133 +
134 + // Add content to node 0
135 + cid := nodes[0].IPFSAddStr("test content for healing")
136 +
137 + // Wait for it to appear on node 1
138 + nodes[1].IPFS("block", "get", cid)
139 +
140 + // Corrupt on node 1
141 + corruptRandomBlock(t, nodes[1])
142 +
143 + // Heal should restore from node 0
144 + res := nodes[1].RunIPFS("repo", "verify", "--heal")
145 + assert.Equal(t, 0, res.ExitCode(), "should exit 0 when all corrupt blocks healed successfully")
146 + output := res.Stdout.String()
147 +
148 + // Should report corruption and healing with specific counts
149 + assert.Contains(t, output, "1 blocks corrupt")
150 + assert.Contains(t, output, "1 removed")
151 + assert.Contains(t, output, "1 healed")
152 +
153 + // Verify block is restored
154 + nodes[1].IPFS("block", "stat", cid)
155 + })
156 +
157 + t.Run("healed blocks contain correct data", func(t *testing.T) {
158 + t.Parallel()
159 + nodes := harness.NewT(t).NewNodes(2).Init()
160 + nodes.StartDaemons().Connect()
161 + defer nodes.StopDaemons()
162 +
163 + // Add specific content to node 0
164 + testContent := "this is the exact content that should be healed correctly"
165 + cid := nodes[0].IPFSAddStr(testContent)
166 +
167 + // Fetch to node 1 and verify the content is correct initially
168 + nodes[1].IPFS("block", "get", cid)
169 + res := nodes[1].IPFS("cat", cid)
170 + assert.Equal(t, testContent, res.Stdout.String())
171 +
172 + // Corrupt on node 1
173 + corruptRandomBlock(t, nodes[1])
174 +
175 + // Heal the corruption
176 + res = nodes[1].RunIPFS("repo", "verify", "--heal")
177 + assert.Equal(t, 0, res.ExitCode(), "should exit 0 when all corrupt blocks healed successfully")
178 + output := res.Stdout.String()
179 + assert.Contains(t, output, "1 blocks corrupt")
180 + assert.Contains(t, output, "1 removed")
181 + assert.Contains(t, output, "1 healed")
182 +
183 + // Verify the healed content matches the original exactly
184 + res = nodes[1].IPFS("cat", cid)
185 + assert.Equal(t, testContent, res.Stdout.String(), "healed content should match original")
186 +
187 + // Also verify via block get that the raw block data is correct
188 + block0 := nodes[0].IPFS("block", "get", cid)
189 + block1 := nodes[1].IPFS("block", "get", cid)
190 + assert.Equal(t, block0.Stdout.String(), block1.Stdout.String(), "raw block data should match")
191 + })
192 +
193 + t.Run("multiple corrupt blocks", func(t *testing.T) {
194 + t.Parallel()
195 + node := harness.NewT(t).NewNode().Init()
196 +
197 + // Create 20 blocks
198 + for i := 0; i < 20; i++ {
199 + node.IPFSAddStr(strings.Repeat("test content ", i+1))
200 + }
201 +
202 + // Corrupt 5 blocks
203 + corruptMultipleBlocks(t, node, 5)
204 +
205 + // Verify detects all corruptions
206 + res := node.RunIPFS("repo", "verify")
207 + assert.Equal(t, 1, res.ExitCode())
208 + // Error summary is in stderr
209 + assert.Contains(t, res.Stderr.String(), "5 blocks corrupt")
210 +
211 + // Test with --drop
212 + res = node.RunIPFS("repo", "verify", "--drop")
213 + assert.Equal(t, 0, res.ExitCode(), "should exit 0 when all corrupt blocks removed successfully")
214 + assert.Contains(t, res.Stdout.String(), "5 blocks corrupt")
215 + assert.Contains(t, res.Stdout.String(), "5 removed")
216 + })
217 +
218 + t.Run("empty repository", func(t *testing.T) {
219 + t.Parallel()
220 + node := harness.NewT(t).NewNode().Init()
221 +
222 + // Verify empty repo passes
223 + res := node.IPFS("repo", "verify")
224 + assert.Equal(t, 0, res.ExitCode())
225 + assert.Contains(t, res.Stdout.String(), "all blocks validated")
226 +
227 + // Should work with --drop and --heal too
228 + res = node.IPFS("repo", "verify", "--drop")
229 + assert.Equal(t, 0, res.ExitCode())
230 + assert.Contains(t, res.Stdout.String(), "all blocks validated")
231 + })
232 +
233 + t.Run("partial heal success", func(t *testing.T) {
234 + t.Parallel()
235 + nodes := harness.NewT(t).NewNodes(2).Init()
236 +
237 + // Start both nodes and connect them
238 + nodes.StartDaemons().Connect()
239 + defer nodes.StopDaemons()
240 +
241 + // Add 5 blocks to node 0, pin them to keep available
242 + cid1 := nodes[0].IPFSAddStr("content available for healing 1")
243 + cid2 := nodes[0].IPFSAddStr("content available for healing 2")
244 + cid3 := nodes[0].IPFSAddStr("content available for healing 3")
245 + cid4 := nodes[0].IPFSAddStr("content available for healing 4")
246 + cid5 := nodes[0].IPFSAddStr("content available for healing 5")
247 +
248 + // Pin these on node 0 to ensure they stay available
249 + nodes[0].IPFS("pin", "add", cid1)
250 + nodes[0].IPFS("pin", "add", cid2)
251 + nodes[0].IPFS("pin", "add", cid3)
252 + nodes[0].IPFS("pin", "add", cid4)
253 + nodes[0].IPFS("pin", "add", cid5)
254 +
255 + // Node 1 fetches these blocks
256 + nodes[1].IPFS("block", "get", cid1)
257 + nodes[1].IPFS("block", "get", cid2)
258 + nodes[1].IPFS("block", "get", cid3)
259 + nodes[1].IPFS("block", "get", cid4)
260 + nodes[1].IPFS("block", "get", cid5)
261 +
262 + // Now remove some blocks from node 0 to simulate partial availability
263 + nodes[0].IPFS("pin", "rm", cid3)
264 + nodes[0].IPFS("pin", "rm", cid4)
265 + nodes[0].IPFS("pin", "rm", cid5)
266 + nodes[0].IPFS("repo", "gc")
267 +
268 + // Verify node 1 is still connected
269 + peers := nodes[1].IPFS("swarm", "peers")
270 + require.Contains(t, peers.Stdout.String(), nodes[0].PeerID().String())
271 +
272 + // Corrupt 5 blocks on node 1
273 + corruptMultipleBlocks(t, nodes[1], 5)
274 +
275 + // Heal should partially succeed (only cid1 and cid2 available from node 0)
276 + res := nodes[1].RunIPFS("repo", "verify", "--heal")
277 + assert.Equal(t, 1, res.ExitCode())
278 +
279 + // Should show mixed results with specific counts in stderr
280 + errOutput := res.Stderr.String()
281 + assert.Contains(t, errOutput, "5 blocks corrupt")
282 + assert.Contains(t, errOutput, "5 removed")
283 + // Only cid1 and cid2 are available for healing, cid3-5 were GC'd
284 + assert.Contains(t, errOutput, "2 healed")
285 + assert.Contains(t, errOutput, "3 failed to heal")
286 + })
287 +
288 + t.Run("heal with block not available on network", func(t *testing.T) {
289 + t.Parallel()
290 + nodes := harness.NewT(t).NewNodes(2).Init()
291 +
292 + // Start both nodes and connect
293 + nodes.StartDaemons().Connect()
294 + defer nodes.StopDaemons()
295 +
296 + // Add unique content only to node 1
297 + nodes[1].IPFSAddStr("unique content that exists nowhere else")
298 +
299 + // Ensure nodes are connected
300 + peers := nodes[1].IPFS("swarm", "peers")
301 + require.Contains(t, peers.Stdout.String(), nodes[0].PeerID().String())
302 +
303 + // Corrupt the block on node 1
304 + corruptRandomBlock(t, nodes[1])
305 +
306 + // Heal should fail - node 0 doesn't have this content
307 + res := nodes[1].RunIPFS("repo", "verify", "--heal")
308 + assert.Equal(t, 1, res.ExitCode())
309 +
310 + // Should report heal failure with specific counts in stderr
311 + errOutput := res.Stderr.String()
312 + assert.Contains(t, errOutput, "1 blocks corrupt")
313 + assert.Contains(t, errOutput, "1 removed")
314 + assert.Contains(t, errOutput, "1 failed to heal")
315 + })
316 +
317 + t.Run("large repository scale test", func(t *testing.T) {
318 + t.Parallel()
319 + node := harness.NewT(t).NewNode().Init()
320 +
321 + // Create 1000 small blocks
322 + for i := 0; i < 1000; i++ {
323 + node.IPFSAddStr(fmt.Sprintf("content-%d", i))
324 + }
325 +
326 + // Corrupt 10 blocks
327 + corruptMultipleBlocks(t, node, 10)
328 +
329 + // Verify handles large repos efficiently
330 + res := node.RunIPFS("repo", "verify")
331 + assert.Equal(t, 1, res.ExitCode())
332 +
333 + // Should report exactly 10 corrupt blocks in stderr
334 + assert.Contains(t, res.Stderr.String(), "10 blocks corrupt")
335 +
336 + // Test --drop at scale
337 + res = node.RunIPFS("repo", "verify", "--drop")
338 + assert.Equal(t, 0, res.ExitCode(), "should exit 0 when all corrupt blocks removed successfully")
339 + output := res.Stdout.String()
340 + assert.Contains(t, output, "10 blocks corrupt")
341 + assert.Contains(t, output, "10 removed")
342 + })
343 +
344 + t.Run("drop with partial removal failures", func(t *testing.T) {
345 + t.Parallel()
346 + node := harness.NewT(t).NewNode().Init()
347 +
348 + // Create several blocks
349 + for i := 0; i < 5; i++ {
350 + node.IPFSAddStr(fmt.Sprintf("content for removal test %d", i))
351 + }
352 +
353 + // Corrupt 3 blocks
354 + corruptedFiles := corruptMultipleBlocks(t, node, 3)
355 + require.Len(t, corruptedFiles, 3)
356 +
357 + // Make one of the corrupted files read-only to simulate removal failure
358 + err := os.Chmod(corruptedFiles[0], 0400) // read-only
359 + require.NoError(t, err)
360 + defer func() { _ = os.Chmod(corruptedFiles[0], 0644) }() // cleanup
361 +
362 + // Also make the directory read-only to prevent deletion
363 + blockDir := filepath.Dir(corruptedFiles[0])
364 + originalPerm, err := os.Stat(blockDir)
365 + require.NoError(t, err)
366 + err = os.Chmod(blockDir, 0500) // read+execute only, no write
367 + require.NoError(t, err)
368 + defer func() { _ = os.Chmod(blockDir, originalPerm.Mode()) }() // cleanup
369 +
370 + // Try to drop - should fail because at least one block can't be removed
371 + res := node.RunIPFS("repo", "verify", "--drop")
372 + assert.Equal(t, 1, res.ExitCode(), "should exit 1 when some blocks fail to remove")
373 +
374 + // Restore permissions for verification
375 + _ = os.Chmod(blockDir, originalPerm.Mode())
376 + _ = os.Chmod(corruptedFiles[0], 0644)
377 +
378 + // Should report both successes and failures with specific counts
379 + errOutput := res.Stderr.String()
380 + assert.Contains(t, errOutput, "3 blocks corrupt")
381 + assert.Contains(t, errOutput, "2 removed")
382 + assert.Contains(t, errOutput, "1 failed to remove")
383 + })
384 +}
test/sharness/t0086-repo-verify.sh
+3
@@ -3,6 +3,9 @@
3 # Copyright (c) 2016 Jeromy Johnson
4 # MIT Licensed; see the LICENSE file in this repository.
5 #
6 +# NOTE: This is a legacy sharness test kept for compatibility.
7 +# New tests for 'ipfs repo verify' should be added to test/cli/repo_verify_test.go
8 +#
9
10 test_description="Test ipfs repo fsck"
11