master
go 506 lines 19.1 KB
Raw
1 package migrations
2
3 // NOTE: These mixed migration tests validate the transition from old Kubo versions that used external
4 // migration binaries to the latest version with embedded migrations. This ensures users can upgrade
5 // from very old installations (v15) to the latest version seamlessly.
6 //
7 // The tests verify hybrid migration paths:
8 // - Forward: external binary (15→16) + embedded migrations (16→latest)
9 // - Backward: embedded migrations (latest→16) + external binary (16→15)
10 //
11 // This confirms compatibility between the old external migration system and the new embedded system.
12 //
13 // To run these tests successfully:
14 // export PATH="$(pwd)/cmd/ipfs:$PATH"
15 // go test ./test/cli/migrations/
16
17 import (
18 "bufio"
19 "context"
20 "encoding/json"
21 "fmt"
22 "io"
23 "os"
24 "os/exec"
25 "path/filepath"
26 "runtime"
27 "slices"
28 "strings"
29 "testing"
30 "time"
31
32 ipfs "github.com/ipfs/kubo"
33 "github.com/ipfs/kubo/test/cli/harness"
34 "github.com/stretchr/testify/require"
35 )
36
37 // TestMixedMigration15ToLatest tests migration from old Kubo (v15 with external migrations)
38 // to the latest version using a hybrid approach: external binary for 15→16, then embedded
39 // migrations for 16→latest. This ensures backward compatibility for users upgrading from
40 // very old Kubo installations.
41 func TestMixedMigration15ToLatest(t *testing.T) {
42 t.Parallel()
43
44 // Test mixed migration from v15 to latest (combines external 15→16 + embedded 16→latest)
45 t.Run("daemon migrate: mixed 15 to latest", testDaemonMigration15ToLatest)
46 t.Run("repo migrate: mixed 15 to latest", testRepoMigration15ToLatest)
47 }
48
49 // TestMixedMigrationLatestTo15Downgrade tests downgrading from the latest version back to v15
50 // using a hybrid approach: embedded migrations for latest→16, then external binary for 16→15.
51 // This ensures the migration system works bidirectionally for recovery scenarios.
52 func TestMixedMigrationLatestTo15Downgrade(t *testing.T) {
53 t.Parallel()
54
55 // Test reverse hybrid migration from latest to v15 (embedded latest→16 + external 16→15)
56 t.Run("repo migrate: reverse hybrid latest to 15", testRepoReverseHybridMigrationLatestTo15)
57 }
58
59 func testDaemonMigration15ToLatest(t *testing.T) {
60 // TEST: Migration from v15 to latest using 'ipfs daemon --migrate'
61 // This tests the mixed migration path: external binary (15→16) + embedded (16→latest)
62 node := setupStaticV15Repo(t)
63
64 // Create mock migration binary for 15→16 (16→17 will use embedded migration)
65 mockBinDir := createMockMigrationBinary(t, "15", "16")
66 customPath := buildCustomPath(mockBinDir)
67
68 configPath := filepath.Join(node.Dir, "config")
69 versionPath := filepath.Join(node.Dir, "version")
70
71 // Verify starting conditions
72 versionData, err := os.ReadFile(versionPath)
73 require.NoError(t, err)
74 require.Equal(t, "15", strings.TrimSpace(string(versionData)), "Should start at version 15")
75
76 // Read original config to verify preservation of key fields
77 var originalConfig map[string]any
78 configData, err := os.ReadFile(configPath)
79 require.NoError(t, err)
80 require.NoError(t, json.Unmarshal(configData, &originalConfig))
81
82 originalPeerID := getNestedValue(originalConfig, "Identity.PeerID")
83
84 // Run dual migration using daemon --migrate
85 stdoutOutput, migrationSuccess := runDaemonWithLegacyMigrationMonitoring(t, node, customPath)
86
87 // Debug output
88 t.Logf("Daemon output:\n%s", stdoutOutput)
89
90 // Verify hybrid migration was successful
91 require.True(t, migrationSuccess, "Hybrid migration should have been successful")
92 require.Contains(t, stdoutOutput, "Phase 1: External migration from v15 to v16", "Should detect external migration phase")
93 // Verify each embedded migration step from 16 to latest
94 verifyMigrationSteps(t, stdoutOutput, 16, ipfs.RepoVersion, true)
95 require.Contains(t, stdoutOutput, fmt.Sprintf("Phase 2: Embedded migration from v16 to v%d", ipfs.RepoVersion), "Should detect embedded migration phase")
96 require.Contains(t, stdoutOutput, "Hybrid migration completed successfully", "Should confirm hybrid migration completion")
97
98 // Verify final version is latest
99 versionData, err = os.ReadFile(versionPath)
100 require.NoError(t, err)
101 latestVersion := fmt.Sprintf("%d", ipfs.RepoVersion)
102 require.Equal(t, latestVersion, strings.TrimSpace(string(versionData)), "Version should be updated to latest")
103
104 // Verify config is still valid JSON and key fields preserved
105 var finalConfig map[string]any
106 configData, err = os.ReadFile(configPath)
107 require.NoError(t, err)
108 require.NoError(t, json.Unmarshal(configData, &finalConfig), "Config should remain valid JSON")
109
110 // Verify essential fields preserved
111 finalPeerID := getNestedValue(finalConfig, "Identity.PeerID")
112 require.Equal(t, originalPeerID, finalPeerID, "Identity.PeerID should be preserved")
113
114 // Verify bootstrap exists (may be modified by 16→17 migration)
115 finalBootstrap := getNestedValue(finalConfig, "Bootstrap")
116 require.NotNil(t, finalBootstrap, "Bootstrap should exist after migration")
117
118 // Verify AutoConf was added by 16→17 migration
119 autoConf := getNestedValue(finalConfig, "AutoConf")
120 require.NotNil(t, autoConf, "AutoConf should be added by 16→17 migration")
121 }
122
123 func testRepoMigration15ToLatest(t *testing.T) {
124 // TEST: Migration from v15 to latest using 'ipfs repo migrate'
125 // Comparison test to verify repo migrate produces same results as daemon migrate
126 node := setupStaticV15Repo(t)
127
128 // Create mock migration binary for 15→16 (16→17 will use embedded migration)
129 mockBinDir := createMockMigrationBinary(t, "15", "16")
130 customPath := buildCustomPath(mockBinDir)
131
132 configPath := filepath.Join(node.Dir, "config")
133 versionPath := filepath.Join(node.Dir, "version")
134
135 // Verify starting version
136 versionData, err := os.ReadFile(versionPath)
137 require.NoError(t, err)
138 require.Equal(t, "15", strings.TrimSpace(string(versionData)), "Should start at version 15")
139
140 // Run migration using 'ipfs repo migrate' with custom PATH
141 result := runMigrationWithCustomPath(node, customPath, "repo", "migrate")
142 require.Empty(t, result.Stderr.String(), "Migration should succeed without errors")
143
144 // Verify final version is latest
145 versionData, err = os.ReadFile(versionPath)
146 require.NoError(t, err)
147 latestVersion := fmt.Sprintf("%d", ipfs.RepoVersion)
148 require.Equal(t, latestVersion, strings.TrimSpace(string(versionData)), "Version should be updated to latest")
149
150 // Verify config is valid JSON
151 var finalConfig map[string]any
152 configData, err := os.ReadFile(configPath)
153 require.NoError(t, err)
154 require.NoError(t, json.Unmarshal(configData, &finalConfig), "Config should remain valid JSON")
155
156 // Verify essential fields exist
157 require.NotNil(t, getNestedValue(finalConfig, "Identity.PeerID"), "Identity.PeerID should exist")
158 require.NotNil(t, getNestedValue(finalConfig, "Bootstrap"), "Bootstrap should exist")
159 require.NotNil(t, getNestedValue(finalConfig, "AutoConf"), "AutoConf should be added")
160 }
161
162 // setupStaticV15Repo creates a test node using static v15 repo fixture
163 // This ensures tests remain stable and validates migration from very old repos
164 func setupStaticV15Repo(t *testing.T) *harness.Node {
165 // Get path to static v15 repo fixture
166 v15FixturePath := "testdata/v15-repo"
167
168 // Create temporary test directory using Go's testing temp dir
169 tmpDir := t.TempDir()
170
171 // Use the built binary (should be in PATH)
172 node := harness.BuildNode("ipfs", tmpDir, 0)
173
174 // Copy static fixture to test directory
175 cloneStaticRepoFixture(t, v15FixturePath, node.Dir)
176
177 return node
178 }
179
180 // runDaemonWithLegacyMigrationMonitoring monitors for hybrid migration patterns
181 func runDaemonWithLegacyMigrationMonitoring(t *testing.T, node *harness.Node, customPath string) (string, bool) {
182 // Monitor for hybrid migration completion - use "Hybrid migration completed successfully" as success pattern
183 stdoutOutput, daemonStarted := runDaemonWithMigrationMonitoringCustomEnv(t, node, "Using hybrid migration strategy", "Hybrid migration completed successfully", map[string]string{
184 "PATH": customPath, // Pass custom PATH with our mock binaries
185 })
186
187 // Check for hybrid migration patterns in output
188 hasHybridStart := strings.Contains(stdoutOutput, "Using hybrid migration strategy")
189 hasPhase1 := strings.Contains(stdoutOutput, "Phase 1: External migration from v15 to v16")
190 hasPhase2 := strings.Contains(stdoutOutput, fmt.Sprintf("Phase 2: Embedded migration from v16 to v%d", ipfs.RepoVersion))
191 hasHybridSuccess := strings.Contains(stdoutOutput, "Hybrid migration completed successfully")
192
193 // Success requires daemon to start and hybrid migration patterns to be detected
194 hybridMigrationSuccess := daemonStarted && hasHybridStart && hasPhase1 && hasPhase2 && hasHybridSuccess
195
196 return stdoutOutput, hybridMigrationSuccess
197 }
198
199 // runDaemonWithMigrationMonitoringCustomEnv is like runDaemonWithMigrationMonitoring but allows custom environment
200 func runDaemonWithMigrationMonitoringCustomEnv(t *testing.T, node *harness.Node, migrationPattern, successPattern string, extraEnv map[string]string) (string, bool) {
201 // Create context with timeout as safety net
202 ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
203 defer cancel()
204
205 // Set up daemon command with output monitoring
206 cmd := exec.CommandContext(ctx, node.IPFSBin, "daemon", "--migrate")
207 cmd.Dir = node.Dir
208
209 // Set environment (especially IPFS_PATH)
210 for k, v := range node.Runner.Env {
211 cmd.Env = append(cmd.Env, k+"="+v)
212 }
213
214 // Add extra environment variables (like PATH with mock binaries)
215 for k, v := range extraEnv {
216 cmd.Env = append(cmd.Env, k+"="+v)
217 }
218
219 // Set up pipes for output monitoring
220 stdout, err := cmd.StdoutPipe()
221 require.NoError(t, err)
222 stderr, err := cmd.StderrPipe()
223 require.NoError(t, err)
224
225 // Start the daemon
226 require.NoError(t, cmd.Start())
227
228 // Monitor output from both streams
229 var outputBuffer strings.Builder
230 done := make(chan bool)
231 migrationStarted := false
232 migrationCompleted := false
233
234 go func() {
235 scanner := bufio.NewScanner(io.MultiReader(stdout, stderr))
236 for scanner.Scan() {
237 line := scanner.Text()
238 outputBuffer.WriteString(line + "\n")
239
240 // Check for migration start
241 if strings.Contains(line, migrationPattern) {
242 migrationStarted = true
243 }
244
245 // Check for migration completion
246 if strings.Contains(line, successPattern) {
247 migrationCompleted = true
248 }
249
250 // Check for daemon ready
251 if strings.Contains(line, "Daemon is ready") {
252 done <- true
253 return
254 }
255 }
256 done <- false
257 }()
258
259 // Wait for daemon to be ready or timeout
260 daemonReady := false
261 select {
262 case ready := <-done:
263 daemonReady = ready
264 case <-ctx.Done():
265 t.Log("Daemon startup timed out")
266 }
267
268 // Stop the daemon using ipfs shutdown command for graceful shutdown
269 if cmd.Process != nil {
270 shutdownCmd := exec.Command(node.IPFSBin, "shutdown")
271 shutdownCmd.Dir = node.Dir
272 for k, v := range node.Runner.Env {
273 shutdownCmd.Env = append(shutdownCmd.Env, k+"="+v)
274 }
275
276 if err := shutdownCmd.Run(); err != nil {
277 // If graceful shutdown fails, force kill
278 _ = cmd.Process.Kill()
279 }
280
281 // Wait for process to exit
282 _ = cmd.Wait()
283 }
284
285 return outputBuffer.String(), daemonReady && migrationStarted && migrationCompleted
286 }
287
288 // buildCustomPath creates a custom PATH with mock migration binaries prepended.
289 // This is necessary for test isolation when running tests in parallel with t.Parallel().
290 // Without isolated PATH handling, parallel tests can interfere with each other through
291 // global PATH modifications, causing tests to download real migration binaries instead
292 // of using the test mocks.
293 func buildCustomPath(mockBinDirs ...string) string {
294 // Prepend mock directories to ensure they're found first
295 pathElements := append(mockBinDirs, os.Getenv("PATH"))
296 return strings.Join(pathElements, string(filepath.ListSeparator))
297 }
298
299 // runMigrationWithCustomPath runs a migration command with a custom PATH environment.
300 // This ensures the migration uses our mock binaries instead of downloading real ones.
301 func runMigrationWithCustomPath(node *harness.Node, customPath string, args ...string) *harness.RunResult {
302 return node.Runner.Run(harness.RunRequest{
303 Path: node.IPFSBin,
304 Args: args,
305 CmdOpts: []harness.CmdOpt{
306 func(cmd *exec.Cmd) {
307 // Remove existing PATH entries using slices.DeleteFunc
308 cmd.Env = slices.DeleteFunc(cmd.Env, func(s string) bool {
309 return strings.HasPrefix(s, "PATH=")
310 })
311 // Add custom PATH
312 cmd.Env = append(cmd.Env, "PATH="+customPath)
313 },
314 },
315 })
316 }
317
318 // createMockMigrationBinary creates a platform-agnostic Go binary for migration testing.
319 // Returns the directory containing the binary to be added to PATH.
320 func createMockMigrationBinary(t *testing.T, fromVer, toVer string) string {
321 // Create bin directory for migration binaries
322 binDir := t.TempDir()
323
324 // Create Go source for mock migration binary
325 scriptName := fmt.Sprintf("fs-repo-%s-to-%s", fromVer, toVer)
326 sourceFile := filepath.Join(binDir, scriptName+".go")
327 binaryPath := filepath.Join(binDir, scriptName)
328 if runtime.GOOS == "windows" {
329 binaryPath += ".exe"
330 }
331
332 // Generate minimal mock migration binary code
333 goSource := fmt.Sprintf(`package main
334 import ("fmt"; "os"; "path/filepath"; "strings"; "time")
335 func main() {
336 var path string
337 var revert bool
338 for _, a := range os.Args[1:] {
339 if strings.HasPrefix(a, "-path=") { path = a[6:] }
340 if a == "-revert" { revert = true }
341 }
342 if path == "" { fmt.Fprintln(os.Stderr, "missing -path="); os.Exit(1) }
343
344 from, to := "%s", "%s"
345 if revert { from, to = to, from }
346 fmt.Printf("fake applying %%s-to-%%s repo migration\n", from, to)
347
348 // Create and immediately remove lock file to simulate proper locking behavior
349 lockPath := filepath.Join(path, "repo.lock")
350 lockFile, err := os.Create(lockPath)
351 if err != nil && !os.IsExist(err) {
352 fmt.Fprintf(os.Stderr, "Error creating lock: %%v\n", err)
353 os.Exit(1)
354 }
355 if lockFile != nil {
356 lockFile.Close()
357 defer os.Remove(lockPath)
358 }
359
360 // Small delay to simulate migration work
361 time.Sleep(10 * time.Millisecond)
362
363 if err := os.WriteFile(filepath.Join(path, "version"), []byte(to), 0644); err != nil {
364 fmt.Fprintf(os.Stderr, "Error: %%v\n", err)
365 os.Exit(1)
366 }
367 }`, fromVer, toVer)
368
369 require.NoError(t, os.WriteFile(sourceFile, []byte(goSource), 0644))
370
371 // Compile the Go binary
372 cmd := exec.Command("go", "build", "-o", binaryPath, sourceFile)
373 cmd.Env = append(os.Environ(), "CGO_ENABLED=0") // Ensure static binary
374 require.NoError(t, cmd.Run())
375
376 // Verify the binary exists and is executable
377 _, err := os.Stat(binaryPath)
378 require.NoError(t, err, "Mock binary should exist")
379
380 // Return the bin directory to be added to PATH
381 return binDir
382 }
383
384 // expectedMigrationSteps generates the expected migration step strings for a version range.
385 // For forward migrations (from < to), it returns strings like "Running embedded migration fs-repo-16-to-17"
386 // For reverse migrations (from > to), it returns strings for the reverse path.
387 func expectedMigrationSteps(from, to int, forward bool) []string {
388 var steps []string
389
390 if forward {
391 // Forward migration: increment by 1 each step
392 for v := from; v < to; v++ {
393 migrationName := fmt.Sprintf("fs-repo-%d-to-%d", v, v+1)
394 steps = append(steps, fmt.Sprintf("Running embedded migration %s", migrationName))
395 }
396 } else {
397 // Reverse migration: decrement by 1 each step
398 for v := from; v > to; v-- {
399 migrationName := fmt.Sprintf("fs-repo-%d-to-%d", v, v-1)
400 steps = append(steps, fmt.Sprintf("Running reverse migration %s", migrationName))
401 }
402 }
403
404 return steps
405 }
406
407 // verifyMigrationSteps checks that all expected migration steps appear in the output
408 func verifyMigrationSteps(t *testing.T, output string, from, to int, forward bool) {
409 steps := expectedMigrationSteps(from, to, forward)
410 for _, step := range steps {
411 require.Contains(t, output, step, "Migration output should contain: %s", step)
412 }
413 }
414
415 // getNestedValue retrieves a nested value from a config map using dot notation
416 func getNestedValue(config map[string]any, path string) any {
417 parts := strings.Split(path, ".")
418 current := any(config)
419
420 for _, part := range parts {
421 switch v := current.(type) {
422 case map[string]any:
423 current = v[part]
424 default:
425 return nil
426 }
427 if current == nil {
428 return nil
429 }
430 }
431
432 return current
433 }
434
435 func testRepoReverseHybridMigrationLatestTo15(t *testing.T) {
436 // TEST: Reverse hybrid migration from latest to v15 using 'ipfs repo migrate --to=15 --allow-downgrade'
437 // This tests reverse hybrid migration: embedded (17→16) + external (16→15)
438
439 // Start with v15 fixture and migrate forward to latest to create proper backup files
440 node := setupStaticV15Repo(t)
441
442 // Create mock migration binaries for both forward and reverse migrations
443 mockBinDirs := []string{
444 createMockMigrationBinary(t, "15", "16"), // for forward migration
445 createMockMigrationBinary(t, "16", "15"), // for downgrade
446 }
447 customPath := buildCustomPath(mockBinDirs...)
448
449 configPath := filepath.Join(node.Dir, "config")
450 versionPath := filepath.Join(node.Dir, "version")
451
452 // Step 1: Forward migration from v15 to latest to create backup files
453 t.Logf("Step 1: Forward migration v15 → v%d", ipfs.RepoVersion)
454 result := runMigrationWithCustomPath(node, customPath, "repo", "migrate")
455
456 // Debug: print the output to see what happened
457 t.Logf("Forward migration stdout:\n%s", result.Stdout.String())
458 t.Logf("Forward migration stderr:\n%s", result.Stderr.String())
459
460 require.Empty(t, result.Stderr.String(), "Forward migration should succeed without errors")
461
462 // Verify we're at latest version after forward migration
463 versionData, err := os.ReadFile(versionPath)
464 require.NoError(t, err)
465 latestVersion := fmt.Sprintf("%d", ipfs.RepoVersion)
466 require.Equal(t, latestVersion, strings.TrimSpace(string(versionData)), "Should be at latest version after forward migration")
467
468 // Read config after forward migration to use as baseline for downgrade
469 var latestConfig map[string]any
470 configData, err := os.ReadFile(configPath)
471 require.NoError(t, err)
472 require.NoError(t, json.Unmarshal(configData, &latestConfig))
473
474 originalPeerID := getNestedValue(latestConfig, "Identity.PeerID")
475
476 // Step 2: Reverse hybrid migration from latest to v15
477 t.Logf("Step 2: Reverse hybrid migration v%d → v15", ipfs.RepoVersion)
478 result = runMigrationWithCustomPath(node, customPath, "repo", "migrate", "--to=15", "--allow-downgrade")
479 require.Empty(t, result.Stderr.String(), "Reverse hybrid migration should succeed without errors")
480
481 // Debug output
482 t.Logf("Downgrade migration output:\n%s", result.Stdout.String())
483
484 // Verify final version is 15
485 versionData, err = os.ReadFile(versionPath)
486 require.NoError(t, err)
487 require.Equal(t, "15", strings.TrimSpace(string(versionData)), "Version should be updated to 15")
488
489 // Verify config is still valid JSON and key fields preserved
490 var finalConfig map[string]any
491 configData, err = os.ReadFile(configPath)
492 require.NoError(t, err)
493 require.NoError(t, json.Unmarshal(configData, &finalConfig), "Config should remain valid JSON")
494
495 // Verify essential fields preserved
496 finalPeerID := getNestedValue(finalConfig, "Identity.PeerID")
497 require.Equal(t, originalPeerID, finalPeerID, "Identity.PeerID should be preserved")
498
499 // Verify bootstrap exists (may be modified by migrations)
500 finalBootstrap := getNestedValue(finalConfig, "Bootstrap")
501 require.NotNil(t, finalBootstrap, "Bootstrap should exist after migration")
502
503 // AutoConf should be removed by the downgrade (was added in 16→17)
504 autoConf := getNestedValue(finalConfig, "AutoConf")
505 require.Nil(t, autoConf, "AutoConf should be removed by downgrade to v15")
506 }