| 1 | package cli |
| 2 | |
| 3 | import ( |
| 4 | "fmt" |
| 5 | "os" |
| 6 | "path/filepath" |
| 7 | "strings" |
| 8 | "testing" |
| 9 | |
| 10 | "github.com/ipfs/kubo/test/cli/harness" |
| 11 | "github.com/stretchr/testify/assert" |
| 12 | "github.com/stretchr/testify/require" |
| 13 | ) |
| 14 | |
| 15 | // Well-known block file names in flatfs blockstore that should not be corrupted during testing. |
| 16 | // Flatfs stores each block as a separate .data file on disk. |
| 17 | const ( |
| 18 | // emptyFileFlatfsFilename is the flatfs filename for an empty UnixFS file block |
| 19 | emptyFileFlatfsFilename = "CIQL7TG2PB52XIZLLHDYIUFMHUQLMMZWBNBZSLDXFCPZ5VDNQQ2WDZQ" |
| 20 | // emptyDirFlatfsFilename is the flatfs filename for an empty UnixFS directory block. |
| 21 | // This block has special handling and may be served from memory even when corrupted on disk. |
| 22 | emptyDirFlatfsFilename = "CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y" |
| 23 | ) |
| 24 | |
| 25 | // getEligibleFlatfsBlockFiles returns flatfs block files (*.data) that are safe to corrupt in tests. |
| 26 | // Filters out well-known blocks (empty file/dir) that cause test flakiness. |
| 27 | // |
| 28 | // Note: This helper is specific to the flatfs blockstore implementation where each block |
| 29 | // is stored as a separate file on disk under blocks/*/*.data. |
| 30 | func getEligibleFlatfsBlockFiles(t *testing.T, node *harness.Node) []string { |
| 31 | blockFiles, err := filepath.Glob(filepath.Join(node.Dir, "blocks", "*", "*.data")) |
| 32 | require.NoError(t, err) |
| 33 | require.NotEmpty(t, blockFiles, "no flatfs block files found") |
| 34 | |
| 35 | var eligible []string |
| 36 | for _, f := range blockFiles { |
| 37 | name := filepath.Base(f) |
| 38 | if !strings.Contains(name, emptyFileFlatfsFilename) && |
| 39 | !strings.Contains(name, emptyDirFlatfsFilename) { |
| 40 | eligible = append(eligible, f) |
| 41 | } |
| 42 | } |
| 43 | return eligible |
| 44 | } |
| 45 | |
| 46 | // corruptRandomBlock corrupts a random block file in the flatfs blockstore. |
| 47 | // Returns the path to the corrupted file. |
| 48 | func corruptRandomBlock(t *testing.T, node *harness.Node) string { |
| 49 | eligible := getEligibleFlatfsBlockFiles(t, node) |
| 50 | require.NotEmpty(t, eligible, "no eligible blocks to corrupt") |
| 51 | |
| 52 | toCorrupt := eligible[0] |
| 53 | err := os.WriteFile(toCorrupt, []byte("corrupted data"), 0644) |
| 54 | require.NoError(t, err) |
| 55 | |
| 56 | return toCorrupt |
| 57 | } |
| 58 | |
| 59 | // corruptMultipleBlocks corrupts multiple block files in the flatfs blockstore. |
| 60 | // Returns the paths to the corrupted files. |
| 61 | func corruptMultipleBlocks(t *testing.T, node *harness.Node, count int) []string { |
| 62 | eligible := getEligibleFlatfsBlockFiles(t, node) |
| 63 | require.GreaterOrEqual(t, len(eligible), count, "not enough eligible blocks to corrupt") |
| 64 | |
| 65 | var corrupted []string |
| 66 | for i := 0; i < count && i < len(eligible); i++ { |
| 67 | err := os.WriteFile(eligible[i], fmt.Appendf(nil, "corrupted data %d", i), 0644) |
| 68 | require.NoError(t, err) |
| 69 | corrupted = append(corrupted, eligible[i]) |
| 70 | } |
| 71 | |
| 72 | return corrupted |
| 73 | } |
| 74 | |
| 75 | func TestRepoVerify(t *testing.T) { |
| 76 | t.Run("healthy repo passes", func(t *testing.T) { |
| 77 | t.Parallel() |
| 78 | node := harness.NewT(t).NewNode().Init() |
| 79 | node.IPFS("add", "-q", "--raw-leaves=false", "-r", node.IPFSBin) |
| 80 | |
| 81 | res := node.IPFS("repo", "verify") |
| 82 | assert.Contains(t, res.Stdout.String(), "all blocks validated") |
| 83 | }) |
| 84 | |
| 85 | t.Run("detects corruption", func(t *testing.T) { |
| 86 | t.Parallel() |
| 87 | node := harness.NewT(t).NewNode().Init() |
| 88 | node.IPFSAddStr("test content") |
| 89 | |
| 90 | corruptRandomBlock(t, node) |
| 91 | |
| 92 | res := node.RunIPFS("repo", "verify") |
| 93 | assert.Equal(t, 1, res.ExitCode()) |
| 94 | assert.Contains(t, res.Stdout.String(), "was corrupt") |
| 95 | assert.Contains(t, res.Stderr.String(), "1 blocks corrupt") |
| 96 | }) |
| 97 | |
| 98 | t.Run("drop removes corrupt blocks", func(t *testing.T) { |
| 99 | t.Parallel() |
| 100 | node := harness.NewT(t).NewNode().Init() |
| 101 | cid := node.IPFSAddStr("test content") |
| 102 | |
| 103 | corruptRandomBlock(t, node) |
| 104 | |
| 105 | res := node.RunIPFS("repo", "verify", "--drop") |
| 106 | assert.Equal(t, 0, res.ExitCode(), "should exit 0 when all corrupt blocks removed successfully") |
| 107 | output := res.Stdout.String() |
| 108 | assert.Contains(t, output, "1 blocks corrupt") |
| 109 | assert.Contains(t, output, "1 removed") |
| 110 | |
| 111 | // Verify block is gone |
| 112 | res = node.RunIPFS("block", "stat", cid) |
| 113 | assert.NotEqual(t, 0, res.ExitCode()) |
| 114 | }) |
| 115 | |
| 116 | t.Run("heal requires online mode", func(t *testing.T) { |
| 117 | t.Parallel() |
| 118 | node := harness.NewT(t).NewNode().Init() |
| 119 | node.IPFSAddStr("test content") |
| 120 | |
| 121 | corruptRandomBlock(t, node) |
| 122 | |
| 123 | res := node.RunIPFS("repo", "verify", "--heal") |
| 124 | assert.NotEqual(t, 0, res.ExitCode()) |
| 125 | assert.Contains(t, res.Stderr.String(), "online mode") |
| 126 | }) |
| 127 | |
| 128 | t.Run("heal repairs from network", func(t *testing.T) { |
| 129 | t.Parallel() |
| 130 | nodes := harness.NewT(t).NewNodes(2).Init() |
| 131 | nodes.StartDaemons().Connect() |
| 132 | defer nodes.StopDaemons() |
| 133 | |
| 134 | // Add content to node 0 |
| 135 | cid := nodes[0].IPFSAddStr("test content for healing") |
| 136 | |
| 137 | // Wait for it to appear on node 1 |
| 138 | nodes[1].IPFS("block", "get", cid) |
| 139 | |
| 140 | // Corrupt on node 1 |
| 141 | corruptRandomBlock(t, nodes[1]) |
| 142 | |
| 143 | // Heal should restore from node 0 |
| 144 | res := nodes[1].RunIPFS("repo", "verify", "--heal") |
| 145 | assert.Equal(t, 0, res.ExitCode(), "should exit 0 when all corrupt blocks healed successfully") |
| 146 | output := res.Stdout.String() |
| 147 | |
| 148 | // Should report corruption and healing with specific counts |
| 149 | assert.Contains(t, output, "1 blocks corrupt") |
| 150 | assert.Contains(t, output, "1 removed") |
| 151 | assert.Contains(t, output, "1 healed") |
| 152 | |
| 153 | // Verify block is restored |
| 154 | nodes[1].IPFS("block", "stat", cid) |
| 155 | }) |
| 156 | |
| 157 | t.Run("healed blocks contain correct data", func(t *testing.T) { |
| 158 | t.Parallel() |
| 159 | nodes := harness.NewT(t).NewNodes(2).Init() |
| 160 | nodes.StartDaemons().Connect() |
| 161 | defer nodes.StopDaemons() |
| 162 | |
| 163 | // Add specific content to node 0 |
| 164 | testContent := "this is the exact content that should be healed correctly" |
| 165 | cid := nodes[0].IPFSAddStr(testContent) |
| 166 | |
| 167 | // Fetch to node 1 and verify the content is correct initially |
| 168 | nodes[1].IPFS("block", "get", cid) |
| 169 | res := nodes[1].IPFS("cat", cid) |
| 170 | assert.Equal(t, testContent, res.Stdout.String()) |
| 171 | |
| 172 | // Corrupt on node 1 |
| 173 | corruptRandomBlock(t, nodes[1]) |
| 174 | |
| 175 | // Heal the corruption |
| 176 | res = nodes[1].RunIPFS("repo", "verify", "--heal") |
| 177 | assert.Equal(t, 0, res.ExitCode(), "should exit 0 when all corrupt blocks healed successfully") |
| 178 | output := res.Stdout.String() |
| 179 | assert.Contains(t, output, "1 blocks corrupt") |
| 180 | assert.Contains(t, output, "1 removed") |
| 181 | assert.Contains(t, output, "1 healed") |
| 182 | |
| 183 | // Verify the healed content matches the original exactly |
| 184 | res = nodes[1].IPFS("cat", cid) |
| 185 | assert.Equal(t, testContent, res.Stdout.String(), "healed content should match original") |
| 186 | |
| 187 | // Also verify via block get that the raw block data is correct |
| 188 | block0 := nodes[0].IPFS("block", "get", cid) |
| 189 | block1 := nodes[1].IPFS("block", "get", cid) |
| 190 | assert.Equal(t, block0.Stdout.String(), block1.Stdout.String(), "raw block data should match") |
| 191 | }) |
| 192 | |
| 193 | t.Run("multiple corrupt blocks", func(t *testing.T) { |
| 194 | t.Parallel() |
| 195 | node := harness.NewT(t).NewNode().Init() |
| 196 | |
| 197 | // Create 20 blocks |
| 198 | for i := range 20 { |
| 199 | node.IPFSAddStr(strings.Repeat("test content ", i+1)) |
| 200 | } |
| 201 | |
| 202 | // Corrupt 5 blocks |
| 203 | corruptMultipleBlocks(t, node, 5) |
| 204 | |
| 205 | // Verify detects all corruptions |
| 206 | res := node.RunIPFS("repo", "verify") |
| 207 | assert.Equal(t, 1, res.ExitCode()) |
| 208 | // Error summary is in stderr |
| 209 | assert.Contains(t, res.Stderr.String(), "5 blocks corrupt") |
| 210 | |
| 211 | // Test with --drop |
| 212 | res = node.RunIPFS("repo", "verify", "--drop") |
| 213 | assert.Equal(t, 0, res.ExitCode(), "should exit 0 when all corrupt blocks removed successfully") |
| 214 | assert.Contains(t, res.Stdout.String(), "5 blocks corrupt") |
| 215 | assert.Contains(t, res.Stdout.String(), "5 removed") |
| 216 | }) |
| 217 | |
| 218 | t.Run("empty repository", func(t *testing.T) { |
| 219 | t.Parallel() |
| 220 | node := harness.NewT(t).NewNode().Init() |
| 221 | |
| 222 | // Verify empty repo passes |
| 223 | res := node.IPFS("repo", "verify") |
| 224 | assert.Equal(t, 0, res.ExitCode()) |
| 225 | assert.Contains(t, res.Stdout.String(), "all blocks validated") |
| 226 | |
| 227 | // Should work with --drop and --heal too |
| 228 | res = node.IPFS("repo", "verify", "--drop") |
| 229 | assert.Equal(t, 0, res.ExitCode()) |
| 230 | assert.Contains(t, res.Stdout.String(), "all blocks validated") |
| 231 | }) |
| 232 | |
| 233 | t.Run("partial heal success", func(t *testing.T) { |
| 234 | t.Parallel() |
| 235 | nodes := harness.NewT(t).NewNodes(2).Init() |
| 236 | |
| 237 | // Start both nodes and connect them |
| 238 | nodes.StartDaemons().Connect() |
| 239 | defer nodes.StopDaemons() |
| 240 | |
| 241 | // Add 5 blocks to node 0, pin them to keep available |
| 242 | cid1 := nodes[0].IPFSAddStr("content available for healing 1") |
| 243 | cid2 := nodes[0].IPFSAddStr("content available for healing 2") |
| 244 | cid3 := nodes[0].IPFSAddStr("content available for healing 3") |
| 245 | cid4 := nodes[0].IPFSAddStr("content available for healing 4") |
| 246 | cid5 := nodes[0].IPFSAddStr("content available for healing 5") |
| 247 | |
| 248 | // Pin these on node 0 to ensure they stay available |
| 249 | nodes[0].IPFS("pin", "add", cid1) |
| 250 | nodes[0].IPFS("pin", "add", cid2) |
| 251 | nodes[0].IPFS("pin", "add", cid3) |
| 252 | nodes[0].IPFS("pin", "add", cid4) |
| 253 | nodes[0].IPFS("pin", "add", cid5) |
| 254 | |
| 255 | // Node 1 fetches these blocks |
| 256 | nodes[1].IPFS("block", "get", cid1) |
| 257 | nodes[1].IPFS("block", "get", cid2) |
| 258 | nodes[1].IPFS("block", "get", cid3) |
| 259 | nodes[1].IPFS("block", "get", cid4) |
| 260 | nodes[1].IPFS("block", "get", cid5) |
| 261 | |
| 262 | // Now remove some blocks from node 0 to simulate partial availability |
| 263 | nodes[0].IPFS("pin", "rm", cid3) |
| 264 | nodes[0].IPFS("pin", "rm", cid4) |
| 265 | nodes[0].IPFS("pin", "rm", cid5) |
| 266 | nodes[0].IPFS("repo", "gc") |
| 267 | |
| 268 | // Verify node 1 is still connected |
| 269 | peers := nodes[1].IPFS("swarm", "peers") |
| 270 | require.Contains(t, peers.Stdout.String(), nodes[0].PeerID().String()) |
| 271 | |
| 272 | // Corrupt 5 blocks on node 1 |
| 273 | corruptMultipleBlocks(t, nodes[1], 5) |
| 274 | |
| 275 | // Heal should partially succeed (only cid1 and cid2 available from node 0) |
| 276 | res := nodes[1].RunIPFS("repo", "verify", "--heal") |
| 277 | assert.Equal(t, 1, res.ExitCode()) |
| 278 | |
| 279 | // Should show mixed results with specific counts in stderr |
| 280 | errOutput := res.Stderr.String() |
| 281 | assert.Contains(t, errOutput, "5 blocks corrupt") |
| 282 | assert.Contains(t, errOutput, "5 removed") |
| 283 | // Only cid1 and cid2 are available for healing, cid3-5 were GC'd |
| 284 | assert.Contains(t, errOutput, "2 healed") |
| 285 | assert.Contains(t, errOutput, "3 failed to heal") |
| 286 | }) |
| 287 | |
| 288 | t.Run("heal with block not available on network", func(t *testing.T) { |
| 289 | t.Parallel() |
| 290 | nodes := harness.NewT(t).NewNodes(2).Init() |
| 291 | |
| 292 | // Start both nodes and connect |
| 293 | nodes.StartDaemons().Connect() |
| 294 | defer nodes.StopDaemons() |
| 295 | |
| 296 | // Add unique content only to node 1 |
| 297 | nodes[1].IPFSAddStr("unique content that exists nowhere else") |
| 298 | |
| 299 | // Ensure nodes are connected |
| 300 | peers := nodes[1].IPFS("swarm", "peers") |
| 301 | require.Contains(t, peers.Stdout.String(), nodes[0].PeerID().String()) |
| 302 | |
| 303 | // Corrupt the block on node 1 |
| 304 | corruptRandomBlock(t, nodes[1]) |
| 305 | |
| 306 | // Heal should fail - node 0 doesn't have this content |
| 307 | res := nodes[1].RunIPFS("repo", "verify", "--heal") |
| 308 | assert.Equal(t, 1, res.ExitCode()) |
| 309 | |
| 310 | // Should report heal failure with specific counts in stderr |
| 311 | errOutput := res.Stderr.String() |
| 312 | assert.Contains(t, errOutput, "1 blocks corrupt") |
| 313 | assert.Contains(t, errOutput, "1 removed") |
| 314 | assert.Contains(t, errOutput, "1 failed to heal") |
| 315 | }) |
| 316 | |
| 317 | t.Run("large repository scale test", func(t *testing.T) { |
| 318 | t.Parallel() |
| 319 | node := harness.NewT(t).NewNode().Init() |
| 320 | |
| 321 | // Create 1000 small blocks |
| 322 | for i := range 1000 { |
| 323 | node.IPFSAddStr(fmt.Sprintf("content-%d", i)) |
| 324 | } |
| 325 | |
| 326 | // Corrupt 10 blocks |
| 327 | corruptMultipleBlocks(t, node, 10) |
| 328 | |
| 329 | // Verify handles large repos efficiently |
| 330 | res := node.RunIPFS("repo", "verify") |
| 331 | assert.Equal(t, 1, res.ExitCode()) |
| 332 | |
| 333 | // Should report exactly 10 corrupt blocks in stderr |
| 334 | assert.Contains(t, res.Stderr.String(), "10 blocks corrupt") |
| 335 | |
| 336 | // Test --drop at scale |
| 337 | res = node.RunIPFS("repo", "verify", "--drop") |
| 338 | assert.Equal(t, 0, res.ExitCode(), "should exit 0 when all corrupt blocks removed successfully") |
| 339 | output := res.Stdout.String() |
| 340 | assert.Contains(t, output, "10 blocks corrupt") |
| 341 | assert.Contains(t, output, "10 removed") |
| 342 | }) |
| 343 | |
| 344 | t.Run("drop with partial removal failures", func(t *testing.T) { |
| 345 | t.Parallel() |
| 346 | node := harness.NewT(t).NewNode().Init() |
| 347 | |
| 348 | // Create several blocks |
| 349 | for i := range 5 { |
| 350 | node.IPFSAddStr(fmt.Sprintf("content for removal test %d", i)) |
| 351 | } |
| 352 | |
| 353 | // Corrupt 3 blocks |
| 354 | corruptedFiles := corruptMultipleBlocks(t, node, 3) |
| 355 | require.Len(t, corruptedFiles, 3) |
| 356 | |
| 357 | // Make one of the corrupted files read-only to simulate removal failure |
| 358 | err := os.Chmod(corruptedFiles[0], 0400) // read-only |
| 359 | require.NoError(t, err) |
| 360 | defer func() { _ = os.Chmod(corruptedFiles[0], 0644) }() // cleanup |
| 361 | |
| 362 | // Also make the directory read-only to prevent deletion |
| 363 | blockDir := filepath.Dir(corruptedFiles[0]) |
| 364 | originalPerm, err := os.Stat(blockDir) |
| 365 | require.NoError(t, err) |
| 366 | err = os.Chmod(blockDir, 0500) // read+execute only, no write |
| 367 | require.NoError(t, err) |
| 368 | defer func() { _ = os.Chmod(blockDir, originalPerm.Mode()) }() // cleanup |
| 369 | |
| 370 | // Try to drop - should fail because at least one block can't be removed |
| 371 | res := node.RunIPFS("repo", "verify", "--drop") |
| 372 | assert.Equal(t, 1, res.ExitCode(), "should exit 1 when some blocks fail to remove") |
| 373 | |
| 374 | // Restore permissions for verification |
| 375 | _ = os.Chmod(blockDir, originalPerm.Mode()) |
| 376 | _ = os.Chmod(corruptedFiles[0], 0644) |
| 377 | |
| 378 | // Should report both successes and failures with specific counts |
| 379 | errOutput := res.Stderr.String() |
| 380 | assert.Contains(t, errOutput, "3 blocks corrupt") |
| 381 | assert.Contains(t, errOutput, "2 removed") |
| 382 | assert.Contains(t, errOutput, "1 failed to remove") |
| 383 | }) |
| 384 | } |