Automatically download and run migrations if needed
License: MIT Signed-off-by: Jeromy <why@ipfs.io>
Jeromy committed
Jul 1, 2016 at 16:10 UTC
a3bd3bc3641f9ee23d7a3b64058ef2cedc8d0e78
7 files changed
+382
-16
core/commands/repo.go
+1
-1
@@ -320,7 +320,7 @@ var repoVersionCmd = &cmds.Command{
320
},
321
Run: func(req cmds.Request, res cmds.Response) {
322
res.SetOutput(&RepoVersion{
323
- Version: fsrepo.RepoVersion,
323
+ Version: fmt.Sprint(fsrepo.RepoVersion),
324
})
325
},
326
Type: RepoVersion{},
core/commands/version.go
+1
-1
@@ -35,7 +35,7 @@ var VersionCmd = &cmds.Command{
35
res.SetOutput(&VersionOutput{
36
Version: config.CurrentVersionNumber,
37
Commit: config.CurrentCommit,
38
- Repo: fsrepo.RepoVersion,
38
+ Repo: fmt.Sprint(fsrepo.RepoVersion),
39
System: runtime.GOARCH + "/" + runtime.GOOS, //TODO: Precise version here
40
Golang: runtime.Version(),
41
})
core/corerepo/stat.go
+3
-1
@@ -1,6 +1,8 @@
1
package corerepo
2
3
import (
4
+ "fmt"
5
+
6
"github.com/ipfs/go-ipfs/core"
7
fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
8
context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
@@ -40,6 +42,6 @@ func RepoStat(n *core.IpfsNode, ctx context.Context) (*Stat, error) {
42
NumObjects: count,
43
RepoSize: usage,
44
RepoPath: path,
43
- Version: "fs-repo@" + fsrepo.RepoVersion,
45
+ Version: fmt.Sprintf("fs-repo@%d", fsrepo.RepoVersion),
46
}, nil
47
}
repo/fsrepo/fsrepo.go
+23
-3
@@ -26,7 +26,7 @@ import (
26
var log = logging.Logger("fsrepo")
27
28
// version number that we are currently expecting to see
29
-var RepoVersion = "4"
29
+var RepoVersion = 4
30
31
var migrationInstructions = `See https://github.com/ipfs/fs-repo-migrations/blob/master/run.md
32
Sorry for the inconvenience. In the future, these will run automatically.`
@@ -36,6 +36,12 @@ Program version is: %s
36
Please run the ipfs migration tool before continuing.
37
` + migrationInstructions
38
39
+var programTooLowMessage = `Your programs version (%d) is lower than your repos (%d).
40
+Please update ipfs to a version that supports the existing repo, or run
41
+a migration in reverse.
42
+
43
+See https://github.com/ipfs/fs-repo-migrations/blob/master/run.md for details.`
44
+
45
var (
46
ErrNoVersion = errors.New("no version file found, please run 0-to-1 migration tool.\n" + migrationInstructions)
47
ErrOldRepo = errors.New("ipfs repo found in old '~/.go-ipfs' location, please run migration tool.\n" + migrationInstructions)
@@ -134,8 +140,22 @@ func open(repoPath string) (repo.Repo, error) {
140
return nil, err
141
}
142
137
- if ver != RepoVersion {
138
- return nil, fmt.Errorf(errIncorrectRepoFmt, ver, RepoVersion)
143
+ if RepoVersion > ver {
144
+ r.lockfile.Close()
145
+
146
+ err := mfsr.TryMigrating(RepoVersion)
147
+ if err != nil {
148
+ return nil, err
149
+ }
150
+
151
+ r.lockfile, err = lockfile.Lock(r.path)
152
+ if err != nil {
153
+ return nil, fmt.Errorf("reacquiring lock: %s", err)
154
+ }
155
+
156
+ } else if ver > RepoVersion {
157
+ // program version too low for existing repo
158
+ return nil, fmt.Errorf(programTooLowMessage, RepoVersion, ver)
159
}
160
161
// check repo path, then check all constituent parts.
repo/fsrepo/migrations/mfsr.go
+33
-10
@@ -5,6 +5,7 @@ import (
5
"io/ioutil"
6
"os"
7
"path"
8
+ "strconv"
9
"strings"
10
)
11
@@ -16,27 +17,26 @@ func (rp RepoPath) VersionFile() string {
17
return path.Join(string(rp), VersionFile)
18
}
19
19
-func (rp RepoPath) Version() (string, error) {
20
+func (rp RepoPath) Version() (int, error) {
21
if rp == "" {
21
- return "", fmt.Errorf("invalid repo path \"%s\"", rp)
22
+ return 0, fmt.Errorf("invalid repo path \"%s\"", rp)
23
}
24
25
fn := rp.VersionFile()
26
if _, err := os.Stat(fn); os.IsNotExist(err) {
26
- return "", VersionFileNotFound(rp)
27
+ return 0, VersionFileNotFound(rp)
28
}
29
30
c, err := ioutil.ReadFile(fn)
31
if err != nil {
31
- return "", err
32
+ return 0, err
33
}
34
34
- s := string(c)
35
- s = strings.TrimSpace(s)
36
- return s, nil
35
+ s := strings.TrimSpace(string(c))
36
+ return strconv.Atoi(s)
37
}
38
39
-func (rp RepoPath) CheckVersion(version string) error {
39
+func (rp RepoPath) CheckVersion(version int) error {
40
v, err := rp.Version()
41
if err != nil {
42
return err
@@ -49,9 +49,9 @@ func (rp RepoPath) CheckVersion(version string) error {
49
return nil
50
}
51
52
-func (rp RepoPath) WriteVersion(version string) error {
52
+func (rp RepoPath) WriteVersion(version int) error {
53
fn := rp.VersionFile()
54
- return ioutil.WriteFile(fn, []byte(version+"\n"), 0644)
54
+ return ioutil.WriteFile(fn, []byte(fmt.Sprintf("%d\n", version)), 0644)
55
}
56
57
type VersionFileNotFound string
@@ -59,3 +59,26 @@ type VersionFileNotFound string
59
func (v VersionFileNotFound) Error() string {
60
return "no version file in repo at " + string(v)
61
}
62
+
63
+func TryMigrating(tovers int) error {
64
+ if !YesNoPrompt("run migrations automatically? [y/n]") {
65
+ return fmt.Errorf("please run the migrations manually")
66
+ }
67
+
68
+ return RunMigration(tovers)
69
+}
70
+
71
+func YesNoPrompt(prompt string) bool {
72
+ var s string
73
+ for {
74
+ fmt.Printf("%s ", prompt)
75
+ fmt.Scanf("%s", &s)
76
+ switch s {
77
+ case "y", "Y":
78
+ return true
79
+ case "n", "N":
80
+ return false
81
+ }
82
+ fmt.Println("Please press either 'y' or 'n'")
83
+ }
84
+}
repo/fsrepo/migrations/migrations.go
new
+220
@@ -0,0 +1,220 @@
1
+package mfsr
2
+
3
+import (
4
+ "bufio"
5
+ "fmt"
6
+ "io"
7
+ "io/ioutil"
8
+ "net/http"
9
+ "os"
10
+ "os/exec"
11
+ "path/filepath"
12
+ "runtime"
13
+ "strconv"
14
+ "strings"
15
+)
16
+
17
+var DistPath = "https://ipfs.io/ipns/dist.ipfs.io"
18
+
19
+const migrations = "fs-repo-migrations"
20
+
21
+func RunMigration(newv int) error {
22
+ migrateBin := "fs-repo-migrations"
23
+ fmt.Println(" => checking for migrations binary...")
24
+ _, err := exec.LookPath(migrateBin)
25
+ if err == nil {
26
+ // check to make sure migrations binary supports our target version
27
+ err = verifyMigrationSupportsVersion(migrateBin, newv)
28
+ }
29
+
30
+ if err != nil {
31
+ fmt.Println(" => usable migrations not found on system, fetching...")
32
+ loc, err := GetMigrations()
33
+ if err != nil {
34
+ return err
35
+ }
36
+
37
+ err = verifyMigrationSupportsVersion(loc, newv)
38
+ if err != nil {
39
+ return fmt.Errorf("could not find migrations binary that supports version %d", newv)
40
+ }
41
+
42
+ migrateBin = loc
43
+ }
44
+
45
+ cmd := exec.Command(migrateBin, "-to", fmt.Sprint(newv), "-y")
46
+ cmd.Stdout = os.Stdout
47
+ cmd.Stderr = os.Stderr
48
+
49
+ fmt.Printf(" => running migration: '%s -to %d -y'\n\n", migrateBin, newv)
50
+
51
+ err = cmd.Run()
52
+ if err != nil {
53
+ return fmt.Errorf("migration failed: %s", err)
54
+ }
55
+
56
+ fmt.Println(" => migrations binary completed successfully")
57
+
58
+ return nil
59
+}
60
+
61
+func GetMigrations() (string, error) {
62
+ latest, err := GetLatestVersion(DistPath, migrations)
63
+ if err != nil {
64
+ return "", fmt.Errorf("getting latest version of fs-repo-migrations: %s", err)
65
+ }
66
+
67
+ dir, err := ioutil.TempDir("", "go-ipfs-migrate")
68
+ if err != nil {
69
+ return "", fmt.Errorf("tempdir: %s", err)
70
+ }
71
+
72
+ out := filepath.Join(dir, migrations)
73
+
74
+ err = GetBinaryForVersion(migrations, migrations, DistPath, latest, out)
75
+ if err != nil {
76
+ fmt.Printf(" => error getting migrations binary: %s\n", err)
77
+ fmt.Println(" => could not find or install fs-repo-migrations, please manually install it")
78
+ return "", fmt.Errorf("failed to find migrations binary")
79
+ }
80
+
81
+ err = os.Chmod(out, 0755)
82
+ if err != nil {
83
+ return "", err
84
+ }
85
+
86
+ return out, nil
87
+}
88
+
89
+func verifyMigrationSupportsVersion(fsrbin string, vn int) error {
90
+ sn, err := migrationsVersion(fsrbin)
91
+ if err != nil {
92
+ return err
93
+ }
94
+
95
+ if sn >= vn {
96
+ return nil
97
+ }
98
+
99
+ return fmt.Errorf("migrations binary doesnt support version %d", vn)
100
+}
101
+
102
+func migrationsVersion(bin string) (int, error) {
103
+ out, err := exec.Command(bin, "-v").CombinedOutput()
104
+ if err != nil {
105
+ return 0, fmt.Errorf("failed to check migrations version: %s", err)
106
+ }
107
+
108
+ vs := strings.Trim(string(out), " \n\t")
109
+ vn, err := strconv.Atoi(vs)
110
+ if err != nil {
111
+ return 0, fmt.Errorf("migrations binary version check did not return a number")
112
+ }
113
+
114
+ return vn, nil
115
+}
116
+
117
+func GetVersions(ipfspath, dist string) ([]string, error) {
118
+ rc, err := httpFetch(ipfspath + "/" + dist + "/versions")
119
+ if err != nil {
120
+ return nil, err
121
+ }
122
+ defer rc.Close()
123
+
124
+ var out []string
125
+ scan := bufio.NewScanner(rc)
126
+ for scan.Scan() {
127
+ out = append(out, scan.Text())
128
+ }
129
+
130
+ return out, nil
131
+}
132
+
133
+func GetLatestVersion(ipfspath, dist string) (string, error) {
134
+ vs, err := GetVersions(ipfspath, dist)
135
+ if err != nil {
136
+ return "", err
137
+ }
138
+ var latest string
139
+ for i := len(vs) - 1; i >= 0; i-- {
140
+ if !strings.Contains(vs[i], "-dev") {
141
+ latest = vs[i]
142
+ break
143
+ }
144
+ }
145
+ if latest == "" {
146
+ return "", fmt.Errorf("couldnt find a non dev version in the list")
147
+ }
148
+ return vs[len(vs)-1], nil
149
+}
150
+
151
+func httpGet(url string) (*http.Response, error) {
152
+ req, err := http.NewRequest("GET", url, nil)
153
+ if err != nil {
154
+ return nil, fmt.Errorf("http.NewRequest error: %s", err)
155
+ }
156
+
157
+ req.Header.Set("User-Agent", "go-ipfs")
158
+
159
+ resp, err := http.DefaultClient.Do(req)
160
+ if err != nil {
161
+ return nil, fmt.Errorf("http.DefaultClient.Do error: %s", err)
162
+ }
163
+
164
+ return resp, nil
165
+}
166
+
167
+func httpFetch(url string) (io.ReadCloser, error) {
168
+ fmt.Printf("fetching url: %s\n", url)
169
+ resp, err := httpGet(url)
170
+ if err != nil {
171
+ return nil, err
172
+ }
173
+
174
+ if resp.StatusCode >= 400 {
175
+ mes, err := ioutil.ReadAll(resp.Body)
176
+ if err != nil {
177
+ return nil, fmt.Errorf("error reading error body: %s", err)
178
+ }
179
+
180
+ return nil, fmt.Errorf("%s: %s", resp.Status, string(mes))
181
+ }
182
+
183
+ return resp.Body, nil
184
+}
185
+
186
+func GetBinaryForVersion(distname, binnom, root, vers, out string) error {
187
+ dir, err := ioutil.TempDir("", "go-ipfs-auto-migrate")
188
+ if err != nil {
189
+ return err
190
+ }
191
+
192
+ var archive string
193
+ switch runtime.GOOS {
194
+ case "windows":
195
+ archive = "zip"
196
+ default:
197
+ archive = "tar.gz"
198
+ }
199
+ finame := fmt.Sprintf("%s_%s_%s-%s.%s", distname, vers, runtime.GOOS, runtime.GOARCH, archive)
200
+ distpath := fmt.Sprintf("%s/%s/%s/%s", root, distname, vers, finame)
201
+
202
+ data, err := httpFetch(distpath)
203
+ if err != nil {
204
+ return err
205
+ }
206
+
207
+ arcpath := filepath.Join(dir, finame)
208
+ fi, err := os.Create(arcpath)
209
+ if err != nil {
210
+ return err
211
+ }
212
+
213
+ _, err = io.Copy(fi, data)
214
+ if err != nil {
215
+ return err
216
+ }
217
+ fi.Close()
218
+
219
+ return unpackArchive(distname, binnom, arcpath, out, archive)
220
+}
repo/fsrepo/migrations/unpack.go
new
+101
@@ -0,0 +1,101 @@
1
+package mfsr
2
+
3
+import (
4
+ "archive/tar"
5
+ "archive/zip"
6
+ "compress/gzip"
7
+ "fmt"
8
+ "io"
9
+ "os"
10
+)
11
+
12
+func unpackArchive(dist, binnom, path, out, atype string) error {
13
+ switch atype {
14
+ case "zip":
15
+ return unpackZip(dist, binnom, path, out)
16
+ case "tar.gz":
17
+ return unpackTgz(dist, binnom, path, out)
18
+ default:
19
+ return fmt.Errorf("unrecognized archive type: %s", atype)
20
+ }
21
+}
22
+
23
+func unpackTgz(dist, binnom, path, out string) error {
24
+ fi, err := os.Open(path)
25
+ if err != nil {
26
+ return err
27
+ }
28
+ defer fi.Close()
29
+
30
+ gzr, err := gzip.NewReader(fi)
31
+ if err != nil {
32
+ return err
33
+ }
34
+
35
+ defer gzr.Close()
36
+
37
+ var bin io.Reader
38
+ tarr := tar.NewReader(gzr)
39
+
40
+loop:
41
+ for {
42
+ th, err := tarr.Next()
43
+ switch err {
44
+ default:
45
+ return err
46
+ case io.EOF:
47
+ break loop
48
+ case nil:
49
+ // continue
50
+ }
51
+
52
+ if th.Name == dist+"/"+binnom {
53
+ bin = tarr
54
+ break
55
+ }
56
+ }
57
+
58
+ if bin == nil {
59
+ return fmt.Errorf("no binary found in downloaded archive")
60
+ }
61
+
62
+ return writeToPath(bin, out)
63
+}
64
+
65
+func writeToPath(rc io.Reader, out string) error {
66
+ binfi, err := os.Create(out)
67
+ if err != nil {
68
+ return fmt.Errorf("error opening tmp bin path '%s': %s", out, err)
69
+ }
70
+ defer binfi.Close()
71
+
72
+ _, err = io.Copy(binfi, rc)
73
+ if err != nil {
74
+ return err
75
+ }
76
+
77
+ return nil
78
+}
79
+
80
+func unpackZip(dist, binnom, path, out string) error {
81
+ zipr, err := zip.OpenReader(path)
82
+ if err != nil {
83
+ return fmt.Errorf("error opening zipreader: %s", err)
84
+ }
85
+
86
+ defer zipr.Close()
87
+
88
+ var bin io.ReadCloser
89
+ for _, fis := range zipr.File {
90
+ if fis.Name == dist+"/"+binnom+".exe" {
91
+ rc, err := fis.Open()
92
+ if err != nil {
93
+ return fmt.Errorf("error extracting binary from archive: %s", err)
94
+ }
95
+
96
+ bin = rc
97
+ }
98
+ }
99
+
100
+ return writeToPath(bin, out)
101
+}