@cryptotaxi247 / kubo / commits / e37d08f21

Migration downloads individual migration binaries

The code in this PR finds the necessary mirgations, downloads the latest version of them from the distribution site, unpacks the executables, and runs the migrations in order. This code is also used to build the ipfs-update tool and the fs-repo-migrations tool. Note: the fs-repo-migrations tool is only used to run stand-alone migrations now and is not used by either go-ipfs or ipfs-update. Additional utility is provided by this PR, that is not specific to migrations: - Find local ipfs directory - Get current repo version - Check for ipfs daemon availability - Get version information about any distribution on distribution site - Fetch and unpack any binary executable over ipfs or http

gammazero committed Dec 16, 2020 at 18:47 UTC e37d08f2104cbf5d5c639a791404b31570a27a7d
15 files changed +1189 -362
cmd/ipfs/daemon.go
+2 -2
@@ -26,7 +26,7 @@ import (
26 libp2p "github.com/ipfs/go-ipfs/core/node/libp2p"
27 nodeMount "github.com/ipfs/go-ipfs/fuse/node"
28 fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
29 - migrate "github.com/ipfs/go-ipfs/repo/fsrepo/migrations"
29 + "github.com/ipfs/go-ipfs/repo/fsrepo/migrations"
30 sockets "github.com/libp2p/go-socket-activation"
31
32 cmds "github.com/ipfs/go-ipfs-cmds"
@@ -288,7 +288,7 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
288 return fmt.Errorf("fs-repo requires migration")
289 }
290
291 - err = migrate.RunMigration(fsrepo.RepoVersion)
291 + err = migrations.RunMigration(cctx.Context(), fsrepo.RepoVersion, "")
292 if err != nil {
293 fmt.Println("The migrations of fs-repo failed:")
294 fmt.Printf(" %s\n", err)
go.mod
+1
@@ -6,6 +6,7 @@ require (
6 github.com/blang/semver v3.5.1+incompatible
7 github.com/bren2010/proquint v0.0.0-20160323162903-38337c27106d
8 github.com/cheggaaa/pb v1.0.29
9 + github.com/coreos/go-semver v0.3.0
10 github.com/coreos/go-systemd/v22 v22.1.0
11 github.com/dustin/go-humanize v1.0.0
12 github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302
go.sum
+2
@@ -87,6 +87,7 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+
87 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
88 github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
89 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
90 +github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U=
91 github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo=
92 github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30=
93 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
@@ -1029,6 +1030,7 @@ github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9 h1:Y1/FEOpaCpD2
1030 github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4=
1031 github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 h1:E9S12nwJwEOXe2d6gT6qxdvqMnNq+VnSsKPgm2ZZNds=
1032 github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI=
1033 +github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c/go.mod h1:xxcJeBb7SIUl/Wzkz1eVKJE/CB34YNrqX2TQI6jY9zs=
1034 github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b h1:wA3QeTsaAXybLL2kb2cKhCAQTHgYTMwuI8lBlJSv5V8=
1035 github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b/go.mod h1:xT1Y5p2JR2PfSZihE0s4mjdJaRGp1waCTf5JzhQLBck=
1036 github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow=
repo/fsrepo/fsrepo.go
+3 -3
@@ -14,7 +14,7 @@ import (
14 keystore "github.com/ipfs/go-ipfs/keystore"
15 repo "github.com/ipfs/go-ipfs/repo"
16 "github.com/ipfs/go-ipfs/repo/common"
17 - mfsr "github.com/ipfs/go-ipfs/repo/fsrepo/migrations"
17 + "github.com/ipfs/go-ipfs/repo/fsrepo/migrations"
18 dir "github.com/ipfs/go-ipfs/thirdparty/dir"
19
20 ds "github.com/ipfs/go-datastore"
@@ -142,7 +142,7 @@ func open(repoPath string) (repo.Repo, error) {
142 }()
143
144 // Check version, and error out if not matching
145 - ver, err := mfsr.RepoPath(r.path).Version()
145 + ver, err := migrations.RepoVersion(r.path)
146 if err != nil {
147 if os.IsNotExist(err) {
148 return nil, ErrNoVersion
@@ -291,7 +291,7 @@ func Init(repoPath string, conf *config.Config) error {
291 return err
292 }
293
294 - if err := mfsr.RepoPath(repoPath).WriteVersion(RepoVersion); err != nil {
294 + if err := migrations.WriteRepoVersion(repoPath, RepoVersion); err != nil {
295 return err
296 }
297
repo/fsrepo/migrations/fetch.go new
+268
@@ -0,0 +1,268 @@
1 +package migrations
2 +
3 +import (
4 + "bufio"
5 + "bytes"
6 + "context"
7 + "fmt"
8 + "io"
9 + "io/ioutil"
10 + "net/http"
11 + "os"
12 + "os/exec"
13 + "path"
14 + "runtime"
15 + "strings"
16 +)
17 +
18 +const (
19 + // Distribution
20 + gatewayURL = "https://ipfs.io"
21 + ipfsDist = "/ipns/dist.ipfs.io"
22 + // ipfsDist = "/ipfs/QmYRLRDKobvg1AXTGeK5Xk6ntWTsjGiHbyNKhWfz7koGpa"
23 +
24 + // Maximum download size
25 + fetchSizeLimit = 1024 * 1024 * 512
26 +)
27 +
28 +type limitReadCloser struct {
29 + io.Reader
30 + io.Closer
31 +}
32 +
33 +var ipfsDistPath string
34 +
35 +func init() {
36 + SetIpfsDistPath("")
37 +}
38 +
39 +// SetIpfsDistPath sets the ipfs path to the distribution site. If an empty
40 +// string is given, then the path is set using the IPFS_DIST_PATH environ
41 +// veriable, or the default value if that is not defined.
42 +func SetIpfsDistPath(distPath string) {
43 + if distPath != "" {
44 + ipfsDistPath = distPath
45 + return
46 + }
47 +
48 + if dist := os.Getenv("IPFS_DIST_PATH"); dist != "" {
49 + ipfsDistPath = dist
50 + } else {
51 + ipfsDistPath = ipfsDist
52 + }
53 +}
54 +
55 +// FetchBinary downloads an archive from the distribution site and unpacks it.
56 +//
57 +// The base name of the archive file, inside the distribution directory on
58 +// distribution site, may differ from the distribution name. If it does, then
59 +// specify arcName.
60 +//
61 +// The base name of the binary inside the archive may differ from the base
62 +// archive name. If it does, then specify binName. For example, the following
63 +// is needed because the archive "go-ipfs_v0.7.0_linux-amd64.tar.gz" contains a
64 +// binary named "ipfs"
65 +//
66 +// FetchBinary(ctx, "go-ipfs", "v0.7.0", "go-ipfs", "ipfs", tmpDir)
67 +//
68 +// If out is a directory, then the binary is written to that directory with the
69 +// same name it has inside the archive. Otherwise, the binary file is written
70 +// to the file named by out.
71 +func FetchBinary(ctx context.Context, dist, ver, arcName, binName, out string) (string, error) {
72 + // If archive base name not specified, then it is same as dist.
73 + if arcName == "" {
74 + arcName = dist
75 + }
76 + // If binary base name is not specified, then it is same as archive base name.
77 + if binName == "" {
78 + binName = arcName
79 + }
80 +
81 + // Name of binary that exists inside archive
82 + binName = ExeName(binName)
83 +
84 + // Return error if file exists or stat fails for reason other than not
85 + // exists. If out is a directory, then write extracted binary to that dir.
86 + fi, err := os.Stat(out)
87 + if !os.IsNotExist(err) {
88 + if err != nil {
89 + return "", err
90 + }
91 + if !fi.IsDir() {
92 + return "", &os.PathError{
93 + Op: "FetchBinary",
94 + Path: out,
95 + Err: os.ErrExist,
96 + }
97 + }
98 + // out exists and is a directory, so compose final name
99 + out = path.Join(out, binName)
100 + }
101 +
102 + // Create temp directory to store download
103 + tmpDir, err := ioutil.TempDir("", arcName)
104 + if err != nil {
105 + return "", err
106 + }
107 + defer os.RemoveAll(tmpDir)
108 +
109 + atype := "tar.gz"
110 + if runtime.GOOS == "windows" {
111 + atype = "zip"
112 + }
113 +
114 + arcName = makeArchiveName(arcName, ver, atype)
115 + arcIpfsPath := makeIpfsPath(dist, ver, arcName)
116 +
117 + // Create a file to write the archive data to
118 + arcPath := path.Join(tmpDir, arcName)
119 + arcFile, err := os.Create(arcPath)
120 + if err != nil {
121 + return "", err
122 + }
123 + defer arcFile.Close()
124 +
125 + // Open connection to download archive from ipfs path
126 + rc, err := fetch(ctx, arcIpfsPath)
127 + if err != nil {
128 + return "", err
129 + }
130 + defer rc.Close()
131 +
132 + // Write download data
133 + _, err = io.Copy(arcFile, rc)
134 + if err != nil {
135 + return "", err
136 + }
137 + arcFile.Close()
138 +
139 + // Unpack the archive and write binary to out
140 + err = unpackArchive(arcPath, atype, dist, binName, out)
141 + if err != nil {
142 + return "", err
143 + }
144 +
145 + // Set mode of binary to executable
146 + err = os.Chmod(out, 0755)
147 + if err != nil {
148 + return "", err
149 + }
150 +
151 + return out, nil
152 +}
153 +
154 +// fetch attempts to fetch the file at the given ipfs path, first using the
155 +// local ipfs api if available, then using http. Returns io.ReadCloser on
156 +// success, which caller must close.
157 +func fetch(ctx context.Context, ipfsPath string) (io.ReadCloser, error) {
158 + // Try fetching via ipfs daemon
159 + rc, err := ipfsFetch(ctx, ipfsPath)
160 + if err == nil {
161 + // Transferred using local ipfs daemon
162 + return rc, nil
163 + }
164 + // Try fetching via HTTP
165 + return httpFetch(ctx, gatewayURL+ipfsPath)
166 +}
167 +
168 +// ipfsFetch attempts to fetch the file at the given ipfs path using the local
169 +// ipfs api. Returns io.ReadCloser on success, which caller must close.
170 +func ipfsFetch(ctx context.Context, ipfsPath string) (io.ReadCloser, error) {
171 + sh, _, err := ApiShell("")
172 + if err != nil {
173 + return nil, err
174 + }
175 +
176 + resp, err := sh.Request("cat", ipfsPath).Send(ctx)
177 + if err != nil {
178 + return nil, err
179 + }
180 + if resp.Error != nil {
181 + return nil, resp.Error
182 + }
183 +
184 + return newLimitReadCloser(resp.Output, fetchSizeLimit), nil
185 +}
186 +
187 +// httpFetch attempts to fetch the file at the given URL. Returns
188 +// io.ReadCloser on success, which caller must close.
189 +func httpFetch(ctx context.Context, url string) (io.ReadCloser, error) {
190 + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
191 + if err != nil {
192 + return nil, fmt.Errorf("http.NewRequest error: %s", err)
193 + }
194 +
195 + req.Header.Set("User-Agent", "go-ipfs")
196 +
197 + resp, err := http.DefaultClient.Do(req)
198 + if err != nil {
199 + return nil, fmt.Errorf("http.DefaultClient.Do error: %s", err)
200 + }
201 +
202 + if resp.StatusCode >= 400 {
203 + defer resp.Body.Close()
204 + mes, err := ioutil.ReadAll(resp.Body)
205 + if err != nil {
206 + return nil, fmt.Errorf("error reading error body: %s", err)
207 + }
208 + return nil, fmt.Errorf("GET %s error: %s: %s", url, resp.Status, string(mes))
209 + }
210 +
211 + return newLimitReadCloser(resp.Body, fetchSizeLimit), nil
212 +}
213 +
214 +func newLimitReadCloser(rc io.ReadCloser, limit int64) io.ReadCloser {
215 + return limitReadCloser{
216 + Reader: io.LimitReader(rc, limit),
217 + Closer: rc,
218 + }
219 +}
220 +
221 +// osWithVariant returns the OS name with optional variant.
222 +// Currently returns either runtime.GOOS, or "linux-musl".
223 +func osWithVariant() (string, error) {
224 + if runtime.GOOS != "linux" {
225 + return runtime.GOOS, nil
226 + }
227 +
228 + // ldd outputs the system's kind of libc.
229 + // - on standard ubuntu: ldd (Ubuntu GLIBC 2.23-0ubuntu5) 2.23
230 + // - on alpine: musl libc (x86_64)
231 + //
232 + // we use the combined stdout+stderr,
233 + // because ldd --version prints differently on different OSes.
234 + // - on standard ubuntu: stdout
235 + // - on alpine: stderr (it probably doesn't know the --version flag)
236 + //
237 + // we suppress non-zero exit codes (see last point about alpine).
238 + out, err := exec.Command("sh", "-c", "ldd --version || true").CombinedOutput()
239 + if err != nil {
240 + return "", err
241 + }
242 +
243 + // now just see if we can find "musl" somewhere in the output
244 + scan := bufio.NewScanner(bytes.NewBuffer(out))
245 + for scan.Scan() {
246 + if strings.Contains(scan.Text(), "musl") {
247 + return "linux-musl", nil
248 + }
249 + }
250 +
251 + return "linux", nil
252 +}
253 +
254 +// makeArchiveName composes the name of a migration binary archive.
255 +//
256 +// The archive name is in the format: name_version_osv-GOARCH.atype
257 +// Example: ipfs-10-to-11_v1.8.0_darwin-amd64.tar.gz
258 +func makeArchiveName(name, ver, atype string) string {
259 + return fmt.Sprintf("%s_%s_%s-%s.%s", name, ver, runtime.GOOS, runtime.GOARCH, atype)
260 +}
261 +
262 +// makeIpfsPath composes the name ipfs path location to download a migration
263 +// binary from the distribution site.
264 +//
265 +// The ipfs path format: distBaseCID/rootdir/version/name/archive
266 +func makeIpfsPath(dist, ver, arcName string) string {
267 + return fmt.Sprintf("%s/%s/%s/%s", ipfsDistPath, dist, ver, arcName)
268 +}
repo/fsrepo/migrations/fetch_test.go new
+129
@@ -0,0 +1,129 @@
1 +package migrations
2 +
3 +import (
4 + "bufio"
5 + "context"
6 + "io/ioutil"
7 + "os"
8 + "path"
9 + "strings"
10 + "testing"
11 +)
12 +
13 +func TestHttpFetch(t *testing.T) {
14 + ctx, cancel := context.WithCancel(context.Background())
15 + defer cancel()
16 +
17 + url := gatewayURL + path.Join(ipfsDistPath, distFSRM, distVersions)
18 + rc, err := httpFetch(ctx, url)
19 + if err != nil {
20 + t.Fatal(err)
21 + }
22 + defer rc.Close()
23 +
24 + var out []string
25 + scan := bufio.NewScanner(rc)
26 + for scan.Scan() {
27 + out = append(out, scan.Text())
28 + }
29 + err = scan.Err()
30 + if err != nil {
31 + t.Fatal("could not read versions:", err)
32 + }
33 +
34 + if len(out) < 14 {
35 + t.Fatal("do not get all expected data")
36 + }
37 + if out[0] != "v1.0.0" {
38 + t.Fatal("expected v1.0.0 as first line, got", out[0])
39 + }
40 +
41 + // Check bad URL
42 + url = gatewayURL + path.Join(ipfsDistPath, distFSRM, "no_such_file")
43 + rc, err = httpFetch(ctx, url)
44 + if err == nil || !strings.Contains(err.Error(), "404") {
45 + t.Fatal("expected error 404")
46 + }
47 +}
48 +
49 +func TestIpfsFetch(t *testing.T) {
50 + _, err := ApiEndpoint("")
51 + if err != nil {
52 + t.Skip("skipped - local ipfs daemon not available")
53 + }
54 +
55 + ctx, cancel := context.WithCancel(context.Background())
56 + defer cancel()
57 +
58 + url := path.Join(ipfsDistPath, distFSRM, distVersions)
59 + rc, err := ipfsFetch(ctx, url)
60 + if err != nil {
61 + t.Fatal(err)
62 + }
63 + defer rc.Close()
64 +
65 + var out []string
66 + scan := bufio.NewScanner(rc)
67 + for scan.Scan() {
68 + out = append(out, scan.Text())
69 + }
70 + err = scan.Err()
71 + if err != nil {
72 + t.Fatal("could not read versions:", err)
73 + }
74 +
75 + if len(out) < 14 {
76 + t.Fatal("do not get all expected data")
77 + }
78 + if out[0] != "v1.0.0" {
79 + t.Fatal("expected v1.0.0 as first line, got", out[0])
80 + }
81 +
82 + // Check bad URL
83 + url = path.Join(ipfsDistPath, distFSRM, "no_such_file")
84 + rc, err = ipfsFetch(ctx, url)
85 + if err == nil || !strings.Contains(err.Error(), "no link") {
86 + t.Fatal("expected 'no link' error, got:", err)
87 + }
88 +}
89 +
90 +func TestFetchBinary(t *testing.T) {
91 + tmpDir, err := ioutil.TempDir("", "fetchtest")
92 + if err != nil {
93 + panic(err)
94 + }
95 + defer os.RemoveAll(tmpDir)
96 +
97 + ctx, cancel := context.WithCancel(context.Background())
98 + defer cancel()
99 +
100 + vers, err := DistVersions(ctx, distFSRM, false)
101 + if err != nil {
102 + t.Fatal(err)
103 + }
104 + t.Log("latest version of", distFSRM, "is", vers[len(vers)-1])
105 +
106 + bin, err := FetchBinary(ctx, distFSRM, vers[0], distFSRM, distFSRM, tmpDir)
107 + if err != nil {
108 + t.Fatal(err)
109 + }
110 +
111 + fi, err := os.Stat(bin)
112 + if os.IsNotExist(err) {
113 + t.Error("expected file to exist:", bin)
114 + }
115 +
116 + t.Log("downloaded and unpacked", fi.Size(), "byte file:", fi.Name())
117 +
118 + bin, err = FetchBinary(ctx, "go-ipfs", "v0.3.5", "go-ipfs", "ipfs", tmpDir)
119 + if err != nil {
120 + t.Fatal(err)
121 + }
122 +
123 + fi, err = os.Stat(bin)
124 + if os.IsNotExist(err) {
125 + t.Error("expected file to exist:", bin)
126 + }
127 +
128 + t.Log("downloaded and unpacked", fi.Size(), "byte file:", fi.Name())
129 +}
repo/fsrepo/migrations/ipfsdir.go new
+198
@@ -0,0 +1,198 @@
1 +package migrations
2 +
3 +import (
4 + "errors"
5 + "fmt"
6 + "io/ioutil"
7 + "os"
8 + "path"
9 + "strconv"
10 + "strings"
11 + "time"
12 +
13 + api "github.com/ipfs/go-ipfs-api"
14 + "github.com/mitchellh/go-homedir"
15 +)
16 +
17 +const (
18 + envIpfsPath = "IPFS_PATH"
19 + versionFile = "version"
20 +
21 + // Local IPFS API
22 + apiFile = "api"
23 + shellUpTimeout = 2 * time.Second
24 +)
25 +
26 +var (
27 + disableDirCache bool
28 + ipfsDirCache string
29 + ipfsDirCacheKey string
30 +)
31 +
32 +// ApiEndpoint reads the api file from the local ipfs install directory and
33 +// returns the address:port read from the file. If the ipfs directory is not
34 +// specified then the default location is used.
35 +func ApiEndpoint(ipfsDir string) (string, error) {
36 + ipfsDir, err := checkIpfsDir(ipfsDir)
37 + if err != nil {
38 + return "", err
39 + }
40 + apiPath := path.Join(ipfsDir, apiFile)
41 +
42 + apiData, err := ioutil.ReadFile(apiPath)
43 + if err != nil {
44 + return "", err
45 + }
46 +
47 + val := strings.TrimSpace(string(apiData))
48 + parts := strings.Split(val, "/")
49 + if len(parts) != 5 {
50 + return "", fmt.Errorf("incorrectly formatted api string: %q", val)
51 + }
52 +
53 + return parts[2] + ":" + parts[4], nil
54 +}
55 +
56 +// ApiShell creates a new ipfs api shell and checks that it is up. If the shell
57 +// is available, then the shell and ipfs version are returned.
58 +func ApiShell(ipfsDir string) (*api.Shell, string, error) {
59 + apiEp, err := ApiEndpoint("")
60 + if err != nil {
61 + return nil, "", err
62 + }
63 + sh := api.NewShell(apiEp)
64 + sh.SetTimeout(shellUpTimeout)
65 + ver, _, err := sh.Version()
66 + if err != nil {
67 + return nil, "", errors.New("ipfs api shell not up")
68 + }
69 + sh.SetTimeout(0)
70 + return sh, ver, nil
71 +}
72 +
73 +// Returns the path of the default ipfs directory.
74 +func IpfsDir() (string, error) {
75 + return checkIpfsDir("")
76 +}
77 +
78 +// RepoVersion returns the version of the repo in the ipfs directory. If the
79 +// ipfs directory is not specified then the default location is used.
80 +func RepoVersion(ipfsDir string) (int, error) {
81 + ipfsDir, err := checkIpfsDir(ipfsDir)
82 + if err != nil {
83 + return 0, err
84 + }
85 + return repoVersion(ipfsDir)
86 +}
87 +
88 +// WriteRepoVersion writes the specified repo version to the repo located in
89 +// ipfsDir. If ipfsDir is not specified, then the default location is used.
90 +func WriteRepoVersion(ipfsDir string, version int) error {
91 + ipfsDir, err := checkIpfsDir(ipfsDir)
92 + if err != nil {
93 + return err
94 + }
95 +
96 + vFilePath := path.Join(ipfsDir, versionFile)
97 + return ioutil.WriteFile(vFilePath, []byte(fmt.Sprintf("%d\n", version)), 0644)
98 +}
99 +
100 +// CacheIpfsDir enables or disables caching the location of the ipfs directory.
101 +// Enabled by default, this avoids subsequent search for and check of the same
102 +// ipfs directory. Disabling the cache may be useful if the location of the
103 +// ipfs directory is expected to change.
104 +func CacheIpfsDir(enable bool) {
105 + if !enable {
106 + disableDirCache = true
107 + ipfsDirCache = ""
108 + ipfsDirCacheKey = ""
109 + homedir.DisableCache = true
110 + homedir.Reset()
111 + } else {
112 + homedir.DisableCache = false
113 + disableDirCache = false
114 + }
115 +}
116 +
117 +func repoVersion(ipfsDir string) (int, error) {
118 + c, err := ioutil.ReadFile(path.Join(ipfsDir, versionFile))
119 + if err != nil {
120 + if os.IsNotExist(err) {
121 + // IPFS directory exists without version file, so version 0
122 + return 0, nil
123 + }
124 + return 0, fmt.Errorf("cannot read repo version file: %s", err)
125 + }
126 +
127 + ver, err := strconv.Atoi(strings.TrimSpace(string(c)))
128 + if err != nil {
129 + return 0, errors.New("invalid data in repo version file")
130 + }
131 + return ver, nil
132 +}
133 +
134 +func checkIpfsDir(dir string) (string, error) {
135 + if dir == ipfsDirCacheKey && ipfsDirCache != "" {
136 + return ipfsDirCache, nil
137 + }
138 +
139 + var (
140 + err error
141 + found string
142 + )
143 + if dir == "" {
144 + found, err = findIpfsDir()
145 + if err != nil {
146 + return "", fmt.Errorf("could not find ipfs directory: %s", err)
147 + }
148 + } else {
149 + found, err = homedir.Expand(dir)
150 + if err != nil {
151 + return "", err
152 + }
153 +
154 + _, err = os.Stat(found)
155 + if err != nil {
156 + return "", err
157 + }
158 + }
159 +
160 + if !disableDirCache {
161 + ipfsDirCacheKey = dir
162 + ipfsDirCache = found
163 + }
164 +
165 + return found, nil
166 +}
167 +
168 +func findIpfsDir() (string, error) {
169 + ipfspath := os.Getenv(envIpfsPath)
170 + if ipfspath != "" {
171 + expandedPath, err := homedir.Expand(ipfspath)
172 + if err != nil {
173 + return "", err
174 + }
175 + return expandedPath, nil
176 + }
177 +
178 + home, err := homedir.Dir()
179 + if err != nil {
180 + return "", err
181 + }
182 + if home == "" {
183 + return "", errors.New("could not determine IPFS_PATH, home dir not set")
184 + }
185 +
186 + for _, dir := range []string{".go-ipfs", ".ipfs"} {
187 + defaultDir := path.Join(home, dir)
188 + _, err = os.Stat(defaultDir)
189 + if err == nil {
190 + return defaultDir, nil
191 + }
192 + if !os.IsNotExist(err) {
193 + return "", err
194 + }
195 + }
196 +
197 + return "", err
198 +}
repo/fsrepo/migrations/ipfsdir_test.go new
+156
@@ -0,0 +1,156 @@
1 +package migrations
2 +
3 +import (
4 + "io/ioutil"
5 + "os"
6 + "path"
7 + "testing"
8 +)
9 +
10 +var (
11 + fakeHome string
12 + fakeIpfs string
13 +)
14 +
15 +func init() {
16 + CacheIpfsDir(false)
17 +}
18 +
19 +func TestRepoDir(t *testing.T) {
20 + var err error
21 + fakeHome, err = ioutil.TempDir("", "testhome")
22 + if err != nil {
23 + panic(err)
24 + }
25 + defer os.RemoveAll(fakeHome)
26 +
27 + os.Setenv("HOME", fakeHome)
28 + fakeIpfs = path.Join(fakeHome, ".ipfs")
29 +
30 + t.Run("testFindIpfsDir", testFindIpfsDir)
31 + t.Run("testCheckIpfsDir", testCheckIpfsDir)
32 + t.Run("testRepoVersion", testRepoVersion)
33 +}
34 +
35 +func testFindIpfsDir(t *testing.T) {
36 + _, err := findIpfsDir()
37 + if err == nil {
38 + t.Fatal("expected error when no .ipfs directory to find")
39 + }
40 +
41 + err = os.Mkdir(fakeIpfs, os.ModePerm)
42 + if err != nil {
43 + panic(err)
44 + }
45 +
46 + dir, err := findIpfsDir()
47 + if err != nil {
48 + t.Fatal(err)
49 + }
50 + if dir != fakeIpfs {
51 + t.Fatal("wrong ipfs directory:", dir)
52 + }
53 +
54 + os.Setenv("IPFS_PATH", "~/.ipfs")
55 + dir, err = findIpfsDir()
56 + if err != nil {
57 + t.Fatal(err)
58 + }
59 + if dir != fakeIpfs {
60 + t.Fatal("wrong ipfs directory:", dir)
61 + }
62 +}
63 +
64 +func testCheckIpfsDir(t *testing.T) {
65 + _, err := checkIpfsDir("~/no_such_dir")
66 + if err == nil {
67 + t.Fatal("expected error from nonexistent directory")
68 + }
69 +
70 + dir, err := checkIpfsDir("~/.ipfs")
71 + if err != nil {
72 + t.Fatal(err)
73 + }
74 + if dir != fakeIpfs {
75 + t.Fatal("wrong ipfs directory:", dir)
76 + }
77 +}
78 +
79 +func testRepoVersion(t *testing.T) {
80 + ver, err := RepoVersion(fakeIpfs)
81 + if err != nil {
82 + t.Fatal(err)
83 + }
84 + if ver != 0 {
85 + t.Fatal("expected version 0 when no version file")
86 + }
87 +
88 + testVer := 42
89 + err = WriteRepoVersion(fakeIpfs, testVer)
90 + if err != nil {
91 + t.Fatal(err)
92 + }
93 +
94 + ver, err = RepoVersion(fakeIpfs)
95 + if err != nil {
96 + t.Fatal(err)
97 + }
98 + if ver != testVer {
99 + t.Fatalf("expected version %d, got %d", testVer, ver)
100 + }
101 +}
102 +
103 +func TestApiEndpoint(t *testing.T) {
104 + var err error
105 + fakeHome, err = ioutil.TempDir("", "testhome")
106 + if err != nil {
107 + panic(err)
108 + }
109 + defer os.RemoveAll(fakeHome)
110 + defer os.Unsetenv("HOME")
111 +
112 + os.Setenv("HOME", fakeHome)
113 + fakeIpfs = path.Join(fakeHome, ".ipfs")
114 +
115 + err = os.Mkdir(fakeIpfs, os.ModePerm)
116 + if err != nil {
117 + panic(err)
118 + }
119 +
120 + _, err = ApiEndpoint("")
121 + if err == nil {
122 + t.Fatal("expected error when missing api file")
123 + }
124 +
125 + apiPath := path.Join(fakeIpfs, apiFile)
126 + err = ioutil.WriteFile(apiPath, []byte("bad-data"), 0644)
127 + if err != nil {
128 + panic(err)
129 + }
130 +
131 + _, err = ApiEndpoint("")
132 + if err == nil {
133 + t.Fatal("expected error when bad data")
134 + }
135 +
136 + err = ioutil.WriteFile(apiPath, []byte("/ip4/127.0.0.1/tcp/5001"), 0644)
137 + if err != nil {
138 + panic(err)
139 + }
140 +
141 + val, err := ApiEndpoint("")
142 + if err != nil {
143 + t.Fatal(err)
144 + }
145 + if val != "127.0.0.1:5001" {
146 + t.Fatal("got unexpected value:", val)
147 + }
148 +
149 + val2, err := ApiEndpoint(fakeIpfs)
150 + if err != nil {
151 + t.Fatal(err)
152 + }
153 + if val2 != val {
154 + t.Fatal("expected", val, "got", val2)
155 + }
156 +}
repo/fsrepo/migrations/mfsr.go deleted
-55
@@ -1,55 +0,0 @@
1 -package mfsr
2 -
3 -import (
4 - "fmt"
5 - "io/ioutil"
6 - "os"
7 - "path"
8 - "strconv"
9 - "strings"
10 -)
11 -
12 -const VersionFile = "version"
13 -
14 -type RepoPath string
15 -
16 -func (rp RepoPath) VersionFile() string {
17 - return path.Join(string(rp), VersionFile)
18 -}
19 -
20 -func (rp RepoPath) Version() (int, error) {
21 - if rp == "" {
22 - return 0, fmt.Errorf("invalid repo path \"%s\"", rp)
23 - }
24 -
25 - fn := rp.VersionFile()
26 - if _, err := os.Stat(fn); err != nil {
27 - return 0, err
28 - }
29 -
30 - c, err := ioutil.ReadFile(fn)
31 - if err != nil {
32 - return 0, err
33 - }
34 -
35 - s := strings.TrimSpace(string(c))
36 - return strconv.Atoi(s)
37 -}
38 -
39 -func (rp RepoPath) CheckVersion(version int) error {
40 - v, err := rp.Version()
41 - if err != nil {
42 - return err
43 - }
44 -
45 - if v != version {
46 - return fmt.Errorf("versions differ (expected: %d, actual:%d)", version, v)
47 - }
48 -
49 - return nil
50 -}
51 -
52 -func (rp RepoPath) WriteVersion(version int) error {
53 - fn := rp.VersionFile()
54 - return ioutil.WriteFile(fn, []byte(fmt.Sprintf("%d\n", version)), 0644)
55 -}
repo/fsrepo/migrations/mfsr_test.go deleted
-43
@@ -1,43 +0,0 @@
1 -package mfsr
2 -
3 -import (
4 - "io/ioutil"
5 - "os"
6 - "strconv"
7 - "testing"
8 -
9 - "github.com/ipfs/go-ipfs/thirdparty/assert"
10 -)
11 -
12 -func testVersionFile(v string, t *testing.T) (rp RepoPath) {
13 - name, err := ioutil.TempDir("", v)
14 - if err != nil {
15 - t.Fatal(err)
16 - }
17 - rp = RepoPath(name)
18 - return rp
19 -}
20 -
21 -func TestVersion(t *testing.T) {
22 - rp := RepoPath("")
23 - _, err := rp.Version()
24 - assert.Err(err, t, "Should throw an error when path is bad,")
25 -
26 - rp = RepoPath("/path/to/nowhere")
27 - _, err = rp.Version()
28 - if !os.IsNotExist(err) {
29 - t.Fatalf("Should throw an `IsNotExist` error when file doesn't exist: %v", err)
30 - }
31 -
32 - fsrepoV := 5
33 -
34 - rp = testVersionFile(strconv.Itoa(fsrepoV), t)
35 - _, err = rp.Version()
36 - assert.Err(err, t, "Bad VersionFile")
37 -
38 - assert.Nil(rp.WriteVersion(fsrepoV), t, "Trouble writing version")
39 -
40 - assert.Nil(rp.CheckVersion(fsrepoV), t, "Trouble checking the version")
41 -
42 - assert.Err(rp.CheckVersion(1), t, "Should throw an error for the wrong version.")
43 -}
repo/fsrepo/migrations/migrations.go
+142 -221
@@ -1,282 +1,203 @@
1 -package mfsr
1 +package migrations
2
3 import (
4 - "bufio"
5 - "bytes"
4 + "context"
5 "fmt"
7 - "io"
6 "io/ioutil"
9 - "net/http"
7 + "log"
8 "os"
9 "os/exec"
12 - "path/filepath"
10 + "path"
11 "runtime"
14 - "strconv"
12 "strings"
13 + "sync"
14 )
15
18 -var DistPath = "https://ipfs.io/ipfs/QmYRLRDKobvg1AXTGeK5Xk6ntWTsjGiHbyNKhWfz7koGpa"
16 +const (
17 + // Migrations distribution
18 + distMigsRoot = "go-ipfs-repo-migrations"
19 + distFSRM = "fs-repo-migrations"
20 +)
21
20 -func init() {
21 - if dist := os.Getenv("IPFS_DIST_PATH"); dist != "" {
22 - DistPath = dist
22 +// RunMigration finds, downloads, and runs the individual migrations needed to
23 +// migrate the repo from its current version to the target version.
24 +func RunMigration(ctx context.Context, targetVer int, ipfsDir string) error {
25 + ipfsDir, err := checkIpfsDir(ipfsDir)
26 + if err != nil {
27 + return err
28 }
24 -}
25 -
26 -const migrations = "fs-repo-migrations"
27 -
28 -func migrationsBinName() string {
29 - switch runtime.GOOS {
30 - case "windows":
31 - return migrations + ".exe"
32 - default:
33 - return migrations
29 + fromVer, err := repoVersion(ipfsDir)
30 + if err != nil {
31 + return fmt.Errorf("could not get repo version: %s", err)
32 + }
33 + if fromVer == targetVer {
34 + // repo already at target version number
35 + return nil
36 }
35 -}
36 -
37 -func RunMigration(newv int) error {
38 - migrateBin := migrationsBinName()
37
40 - fmt.Println(" => Looking for suitable fs-repo-migrations binary.")
38 + log.Print("Looking for suitable migration binaries.")
39
42 - var err error
43 - migrateBin, err = exec.LookPath(migrateBin)
44 - if err == nil {
45 - // check to make sure migrations binary supports our target version
46 - err = verifyMigrationSupportsVersion(migrateBin, newv)
40 + migrations, binPaths, err := findMigrations(ctx, fromVer, targetVer)
41 + if err != nil {
42 + return err
43 }
44
49 - if err != nil {
50 - fmt.Println(" => None found, downloading.")
45 + // Download migrations that were not found
46 + if len(binPaths) < len(migrations) {
47 + missing := make([]string, 0, len(migrations)-len(binPaths))
48 + for _, mig := range migrations {
49 + if _, ok := binPaths[mig]; !ok {
50 + missing = append(missing, mig)
51 + }
52 + }
53 +
54 + log.Print("Need", len(missing), "migrations, downloading.")
55
52 - loc, err := GetMigrations()
56 + tmpDir, err := ioutil.TempDir("", "migrations")
57 if err != nil {
54 - fmt.Println(" => Failed to download fs-repo-migrations.")
58 return err
59 }
60 + defer os.RemoveAll(tmpDir)
61
58 - err = verifyMigrationSupportsVersion(loc, newv)
62 + fetched, err := fetchMigrations(ctx, missing, tmpDir)
63 if err != nil {
60 - return fmt.Errorf("no fs-repo-migration binary found for version %d: %s", newv, err)
64 + log.Print("Failed to download migrations.")
65 + return err
66 + }
67 + for i := range missing {
68 + binPaths[missing[i]] = fetched[i]
69 }
62 -
63 - migrateBin = loc
70 }
71
66 - cmd := exec.Command(migrateBin, "-to", fmt.Sprint(newv), "-y")
67 - cmd.Stdout = os.Stdout
68 - cmd.Stderr = os.Stderr
69 -
70 - fmt.Printf(" => Running: %s -to %d -y\n", migrateBin, newv)
71 -
72 - err = cmd.Run()
73 - if err != nil {
74 - fmt.Printf(" => Failed: %s -to %d -y\n", migrateBin, newv)
75 - return fmt.Errorf("migration failed: %s", err)
72 + var revert bool
73 + if fromVer > targetVer {
74 + revert = true
75 }
77 -
78 - fmt.Printf(" => Success: fs-repo has been migrated to version %d.\n", newv)
76 + for _, migration := range migrations {
77 + log.Println("Running migration", migration, "...")
78 + err = runMigration(ctx, binPaths[migration], ipfsDir, revert)
79 + if err != nil {
80 + return fmt.Errorf("migration %s failed: %s", migration, err)
81 + }
82 + }
83 + log.Printf("Success: fs-repo migrated to version %d.\n", targetVer)
84
85 return nil
86 }
87
83 -func GetMigrations() (string, error) {
84 - latest, err := GetLatestVersion(DistPath, migrations)
85 - if err != nil {
86 - return "", fmt.Errorf("failed to find latest fs-repo-migrations: %s", err)
87 - }
88 -
89 - dir, err := ioutil.TempDir("", "go-ipfs-migrate")
90 - if err != nil {
91 - return "", fmt.Errorf("failed to create fs-repo-migrations tempdir: %s", err)
92 - }
93 -
94 - out := filepath.Join(dir, migrationsBinName())
95 -
96 - err = GetBinaryForVersion(migrations, migrations, DistPath, latest, out)
97 - if err != nil {
98 - return "", fmt.Errorf("failed to download latest fs-repo-migrations: %s", err)
99 - }
100 -
101 - err = os.Chmod(out, 0755)
88 +func NeedMigration(target int) (bool, error) {
89 + vnum, err := RepoVersion("")
90 if err != nil {
103 - return "", err
91 + return false, fmt.Errorf("could not get repo version: %s", err)
92 }
93
106 - return out, nil
94 + return vnum != target, nil
95 }
96
109 -func verifyMigrationSupportsVersion(fsrbin string, vn int) error {
110 - sn, err := migrationsVersion(fsrbin)
111 - if err != nil {
112 - return err
97 +func ExeName(name string) string {
98 + if runtime.GOOS == "windows" {
99 + return name + ".exe"
100 }
114 -
115 - if sn >= vn {
116 - return nil
117 - }
118 -
119 - return fmt.Errorf("migrations binary doesn't support version %d: %s", vn, fsrbin)
101 + return name
102 }
103
122 -func migrationsVersion(bin string) (int, error) {
123 - out, err := exec.Command(bin, "-v").CombinedOutput()
124 - if err != nil {
125 - return 0, fmt.Errorf("failed to check migrations version: %s", err)
126 - }
127 -
128 - vs := strings.Trim(string(out), " \n\t")
129 - vn, err := strconv.Atoi(vs)
130 - if err != nil {
131 - return 0, fmt.Errorf("migrations binary version check did not return a number: %s", err)
132 - }
133 -
134 - return vn, nil
104 +func migrationName(from, to int) string {
105 + return fmt.Sprintf("ipfs-%d-to-%d", from, to)
106 }
107
137 -func GetVersions(ipfspath, dist string) ([]string, error) {
138 - rc, err := httpFetch(ipfspath + "/" + dist + "/versions")
139 - if err != nil {
140 - return nil, err
141 - }
142 - defer rc.Close()
143 -
144 - var out []string
145 - scan := bufio.NewScanner(rc)
146 - for scan.Scan() {
147 - out = append(out, scan.Text())
108 +// findMigrations returns a list of migrations, ordered from first to last
109 +// migration to apply, and a map of locations migration binaries of any
110 +// migrations that were found.
111 +func findMigrations(ctx context.Context, from, to int) ([]string, map[string]string, error) {
112 + step := 1
113 + count := to - from
114 + if from > to {
115 + step = -1
116 + count = from - to
117 }
118
150 - return out, nil
151 -}
119 + migrations := make([]string, 0, count)
120 + binPaths := make(map[string]string, count)
121
153 -func GetLatestVersion(ipfspath, dist string) (string, error) {
154 - vs, err := GetVersions(ipfspath, dist)
155 - if err != nil {
156 - return "", err
157 - }
158 - var latest string
159 - for i := len(vs) - 1; i >= 0; i-- {
160 - if !strings.Contains(vs[i], "-dev") {
161 - latest = vs[i]
162 - break
122 + for cur := from; cur != to; cur += step {
123 + if ctx.Err() != nil {
124 + return nil, nil, ctx.Err()
125 }
164 - }
165 - if latest == "" {
166 - return "", fmt.Errorf("couldn't find a non dev version in the list")
167 - }
168 - return vs[len(vs)-1], nil
169 -}
170 -
171 -func httpGet(url string) (*http.Response, error) {
172 - req, err := http.NewRequest(http.MethodGet, url, nil)
173 - if err != nil {
174 - return nil, fmt.Errorf("http.NewRequest error: %s", err)
175 - }
176 -
177 - req.Header.Set("User-Agent", "go-ipfs")
178 -
179 - resp, err := http.DefaultClient.Do(req)
180 - if err != nil {
181 - return nil, fmt.Errorf("http.DefaultClient.Do error: %s", err)
182 - }
183 -
184 - return resp, nil
185 -}
186 -
187 -func httpFetch(url string) (io.ReadCloser, error) {
188 - resp, err := httpGet(url)
189 - if err != nil {
190 - return nil, err
191 - }
192 -
193 - if resp.StatusCode >= 400 {
194 - mes, err := ioutil.ReadAll(resp.Body)
126 + migName := migrationName(cur, cur+step)
127 + migrations = append(migrations, migName)
128 + bin, err := exec.LookPath(migName)
129 if err != nil {
196 - return nil, fmt.Errorf("error reading error body: %s", err)
130 + continue
131 }
198 -
199 - return nil, fmt.Errorf("GET %s error: %s: %s", url, resp.Status, string(mes))
132 + binPaths[migName] = bin
133 }
201 -
202 - return resp.Body, nil
134 + return migrations, binPaths, nil
135 }
136
205 -func GetBinaryForVersion(distname, binnom, root, vers, out string) error {
206 - dir, err := ioutil.TempDir("", "go-ipfs-auto-migrate")
207 - if err != nil {
208 - return err
137 +func runMigration(ctx context.Context, binPath, ipfsDir string, revert bool) error {
138 + pathArg := fmt.Sprintf("-path=%s", ipfsDir)
139 + var cmd *exec.Cmd
140 + if revert {
141 + log.Println(" => Running:", binPath, pathArg, "-verbose=true -revert")
142 + cmd = exec.CommandContext(ctx, binPath, pathArg, "-verbose=true", "-revert")
143 + } else {
144 + log.Println(" => Running:", binPath, pathArg, "-verbose=true")
145 + cmd = exec.CommandContext(ctx, binPath, pathArg, "-verbose=true")
146 }
147 + cmd.Stdout = os.Stdout
148 + cmd.Stderr = os.Stderr
149 + return cmd.Run()
150 +}
151
211 - var archive string
212 - switch runtime.GOOS {
213 - case "windows":
214 - archive = "zip"
215 - binnom += ".exe"
216 - default:
217 - archive = "tar.gz"
218 - }
152 +// fetchMigrations downloads the requested migrations, and returns a slice with
153 +// the paths of each binary, in the same order as in needed.
154 +func fetchMigrations(ctx context.Context, needed []string, destDir string) ([]string, error) {
155 osv, err := osWithVariant()
156 if err != nil {
221 - return err
157 + return nil, err
158 }
223 -
159 if osv == "linux-musl" {
225 - return fmt.Errorf("linux-musl not supported, you must build the binary from source for your platform")
226 - }
227 -
228 - finame := fmt.Sprintf("%s_%s_%s-%s.%s", distname, vers, osv, runtime.GOARCH, archive)
229 - distpath := fmt.Sprintf("%s/%s/%s/%s", root, distname, vers, finame)
230 -
231 - data, err := httpFetch(distpath)
232 - if err != nil {
233 - return err
234 - }
235 -
236 - arcpath := filepath.Join(dir, finame)
237 - fi, err := os.Create(arcpath)
238 - if err != nil {
239 - return err
240 - }
241 -
242 - _, err = io.Copy(fi, data)
243 - if err != nil {
244 - return err
245 - }
246 - fi.Close()
247 -
248 - return unpackArchive(distname, binnom, arcpath, out, archive)
249 -}
250 -
251 -// osWithVariant returns the OS name with optional variant.
252 -// Currently returns either runtime.GOOS, or "linux-musl".
253 -func osWithVariant() (string, error) {
254 - if runtime.GOOS != "linux" {
255 - return runtime.GOOS, nil
256 - }
257 -
258 - // ldd outputs the system's kind of libc.
259 - // - on standard ubuntu: ldd (Ubuntu GLIBC 2.23-0ubuntu5) 2.23
260 - // - on alpine: musl libc (x86_64)
261 - //
262 - // we use the combined stdout+stderr,
263 - // because ldd --version prints differently on different OSes.
264 - // - on standard ubuntu: stdout
265 - // - on alpine: stderr (it probably doesn't know the --version flag)
266 - //
267 - // we suppress non-zero exit codes (see last point about alpine).
268 - out, err := exec.Command("sh", "-c", "ldd --version || true").CombinedOutput()
269 - if err != nil {
270 - return "", err
160 + return nil, fmt.Errorf("linux-musl not supported, you must build the binary from source for your platform")
161 + }
162 +
163 + var wg sync.WaitGroup
164 + wg.Add(len(needed))
165 + bins := make([]string, len(needed))
166 + // Download and unpack all requested migrations concurrently.
167 + for i, name := range needed {
168 + log.Printf("Downloading migration: %s...", name)
169 + go func(i int, name string) {
170 + defer wg.Done()
171 + distDir := path.Join(distMigsRoot, name)
172 + ver, err := LatestDistVersion(ctx, distDir)
173 + if err != nil {
174 + log.Printf("could not get latest version of migration %s: %s", name, err)
175 + return
176 + }
177 + loc, err := FetchBinary(ctx, distDir, ver, name, name, destDir)
178 + if err != nil {
179 + log.Printf("could not download %s: %s", name, err)
180 + return
181 + }
182 + log.Printf("Downloaded and unpacked migration: %s", loc)
183 + bins[i] = loc
184 + }(i, name)
185 + }
186 + wg.Wait()
187 +
188 + var fails []string
189 + for i := range bins {
190 + if bins[i] == "" {
191 + fails = append(fails, needed[i])
192 + }
193 }
272 -
273 - // now just see if we can find "musl" somewhere in the output
274 - scan := bufio.NewScanner(bytes.NewBuffer(out))
275 - for scan.Scan() {
276 - if strings.Contains(scan.Text(), "musl") {
277 - return "linux-musl", nil
194 + if len(fails) != 0 {
195 + err = fmt.Errorf("failed to download migrations: %s", strings.Join(fails, " "))
196 + if ctx.Err() != nil {
197 + err = fmt.Errorf("%s, %s", ctx.Err(), err)
198 }
199 + return nil, err
200 }
201
281 - return "linux", nil
202 + return bins, nil
203 }
repo/fsrepo/migrations/migrations_test.go new
+93
@@ -0,0 +1,93 @@
1 +package migrations
2 +
3 +import (
4 + "context"
5 + "io/ioutil"
6 + "os"
7 + "path"
8 + "testing"
9 +)
10 +
11 +func TestFindMigrations(t *testing.T) {
12 + tmpDir, err := ioutil.TempDir("", "migratetest")
13 + if err != nil {
14 + panic(err)
15 + }
16 + defer os.RemoveAll(tmpDir)
17 +
18 + ctx, cancel := context.WithCancel(context.Background())
19 + defer cancel()
20 +
21 + migs, bins, err := findMigrations(ctx, 0, 5)
22 + if err != nil {
23 + t.Fatal(err)
24 + }
25 + if len(bins) != 0 {
26 + t.Fatal("should not have found migrations")
27 + }
28 +
29 + for i := 1; i < 6; i++ {
30 + createFakeBin(i-1, i, tmpDir)
31 + }
32 +
33 + origPath := os.Getenv("PATH")
34 + os.Setenv("PATH", tmpDir)
35 + defer os.Setenv("PATH", origPath)
36 +
37 + migs, bins, err = findMigrations(ctx, 0, 5)
38 + if err != nil {
39 + t.Fatal(err)
40 + }
41 + if len(migs) != 5 {
42 + t.Fatal("expected 5 migrations")
43 + }
44 + if len(bins) != len(migs) {
45 + t.Fatal("missing", len(migs)-len(bins), "migrations")
46 + }
47 +
48 + os.Remove(bins[migs[2]])
49 +
50 + migs, bins, err = findMigrations(ctx, 0, 5)
51 + if err != nil {
52 + t.Fatal(err)
53 + }
54 + if len(bins) != len(migs)-1 {
55 + t.Fatal("should be missing one migration bin")
56 + }
57 +}
58 +
59 +func TestFetchMigrations(t *testing.T) {
60 + t.Skip("skip - migrations not available on distribution site yet")
61 +
62 + tmpDir, err := ioutil.TempDir("", "migratetest")
63 + if err != nil {
64 + panic(err)
65 + }
66 + defer os.RemoveAll(tmpDir)
67 +
68 + ctx, cancel := context.WithCancel(context.Background())
69 + defer cancel()
70 +
71 + needed := []string{"ipfs-1-to-2", "ipfs-2-to-3"}
72 + fetched, err := fetchMigrations(ctx, needed, tmpDir)
73 + if err != nil {
74 + t.Fatal(err)
75 + }
76 +
77 + for _, bin := range fetched {
78 + _, err = os.Stat(bin)
79 + if os.IsNotExist(err) {
80 + t.Error("expected file to exist:", bin)
81 + }
82 + }
83 +}
84 +
85 +func createFakeBin(from, to int, tmpDir string) {
86 + migPath := path.Join(tmpDir, ExeName(migrationName(from, to)))
87 + emptyFile, err := os.Create(migPath)
88 + if err != nil {
89 + panic(err)
90 + }
91 + emptyFile.Close()
92 + os.Chmod(migPath, 0755)
93 +}
repo/fsrepo/migrations/unpack.go
+42 -38
@@ -1,90 +1,82 @@
1 -package mfsr
1 +package migrations
2
3 import (
4 "archive/tar"
5 "archive/zip"
6 "compress/gzip"
7 + "errors"
8 "fmt"
9 "io"
10 "os"
11 )
12
12 -func unpackArchive(dist, binnom, path, out, atype string) error {
13 +func unpackArchive(arcPath, atype, root, name, out string) error {
14 + var err error
15 switch atype {
14 - case "zip":
15 - return unpackZip(dist, binnom, path, out)
16 case "tar.gz":
17 - return unpackTgz(dist, binnom, path, out)
17 + err = unpackTgz(arcPath, root, name, out)
18 + case "zip":
19 + err = unpackZip(arcPath, root, name, out)
20 default:
19 - return fmt.Errorf("unrecognized archive type: %s", atype)
21 + err = fmt.Errorf("unrecognized archive type: %s", atype)
22 + }
23 + if err != nil {
24 + return err
25 }
26 + os.Remove(arcPath)
27 + return nil
28 }
29
23 -func unpackTgz(dist, binnom, path, out string) error {
24 - fi, err := os.Open(path)
30 +func unpackTgz(arcPath, root, name, out string) error {
31 + fi, err := os.Open(arcPath)
32 if err != nil {
26 - return err
33 + return fmt.Errorf("cannot open archive file: %s", err)
34 }
35 defer fi.Close()
36
37 gzr, err := gzip.NewReader(fi)
38 if err != nil {
32 - return err
39 + return fmt.Errorf("error opening gzip reader: %s", err)
40 }
34 -
41 defer gzr.Close()
42
43 var bin io.Reader
44 tarr := tar.NewReader(gzr)
45
40 -loop:
46 + lookFor := root + "/" + name
47 for {
48 th, err := tarr.Next()
43 - switch err {
44 - default:
45 - return err
46 - case io.EOF:
47 - break loop
48 - case nil:
49 - // continue
49 + if err != nil {
50 + if err == io.EOF {
51 + break
52 + }
53 + return fmt.Errorf("cannot read archive: %s", err)
54 }
55
52 - if th.Name == dist+"/"+binnom {
56 + if th.Name == lookFor {
57 bin = tarr
58 break
59 }
60 }
61
62 if bin == nil {
59 - return fmt.Errorf("no binary found in downloaded archive")
63 + return errors.New("no binary found in archive")
64 }
65
66 return writeToPath(bin, out)
67 }
68
65 -func writeToPath(rc io.Reader, out string) error {
66 - binfi, err := os.Create(out)
67 - if err != nil {
68 - return fmt.Errorf("error opening tmp bin path '%s': %s", out, err)
69 - }
70 - defer binfi.Close()
71 -
72 - _, err = io.Copy(binfi, rc)
73 -
74 - return err
75 -}
76 -
77 -func unpackZip(dist, binnom, path, out string) error {
78 - zipr, err := zip.OpenReader(path)
69 +func unpackZip(arcPath, root, name, out string) error {
70 + zipr, err := zip.OpenReader(arcPath)
71 if err != nil {
80 - return fmt.Errorf("error opening zipreader: %s", err)
72 + return fmt.Errorf("error opening zip reader: %s", err)
73 }
82 -
74 defer zipr.Close()
75
76 + lookFor := root + "/" + name
77 var bin io.ReadCloser
78 for _, fis := range zipr.File {
87 - if fis.Name == dist+"/"+binnom {
79 + if fis.Name == lookFor {
80 rc, err := fis.Open()
81 if err != nil {
82 return fmt.Errorf("error extracting binary from archive: %s", err)
@@ -96,3 +88,15 @@ func unpackZip(dist, binnom, path, out string) error {
88
89 return writeToPath(bin, out)
90 }
91 +
92 +func writeToPath(rc io.Reader, out string) error {
93 + binfi, err := os.Create(out)
94 + if err != nil {
95 + return fmt.Errorf("error opening tmp bin path '%s': %s", out, err)
96 + }
97 + defer binfi.Close()
98 +
99 + _, err = io.Copy(binfi, rc)
100 +
101 + return err
102 +}
repo/fsrepo/migrations/versions.go new
+90
@@ -0,0 +1,90 @@
1 +package migrations
2 +
3 +import (
4 + "bufio"
5 + "context"
6 + "errors"
7 + "fmt"
8 + "os/exec"
9 + "path"
10 + "sort"
11 + "strconv"
12 + "strings"
13 +
14 + "github.com/coreos/go-semver/semver"
15 +)
16 +
17 +const distVersions = "versions"
18 +
19 +// LatestDistVersion returns the latest version, of the specified distribution,
20 +// that is available on the distribution site.
21 +func LatestDistVersion(ctx context.Context, dist string) (string, error) {
22 + vs, err := DistVersions(ctx, dist, false)
23 + if err != nil {
24 + return "", err
25 + }
26 +
27 + for i := len(vs) - 1; i >= 0; i-- {
28 + ver := vs[i]
29 + if !strings.Contains(ver, "-dev") {
30 + return ver, nil
31 + }
32 + }
33 + return "", errors.New("could not find a non dev version")
34 +}
35 +
36 +// DistVersions returns all versions of the specified distribution, that are
37 +// available on the distriburion site. List is in ascending order, unless
38 +// sortDesc is true.
39 +func DistVersions(ctx context.Context, dist string, sortDesc bool) ([]string, error) {
40 + rc, err := fetch(ctx, path.Join(ipfsDistPath, dist, distVersions))
41 + if err != nil {
42 + return nil, err
43 + }
44 + defer rc.Close()
45 +
46 + prefix := "v"
47 + var vers []*semver.Version
48 +
49 + scan := bufio.NewScanner(rc)
50 + for scan.Scan() {
51 + ver, err := semver.NewVersion(strings.TrimLeft(scan.Text(), prefix))
52 + if err != nil {
53 + continue
54 + }
55 + vers = append(vers, ver)
56 + }
57 + err = scan.Err()
58 + if err != nil {
59 + return nil, fmt.Errorf("could not read versions: %s", err)
60 + }
61 +
62 + if sortDesc {
63 + sort.Sort(sort.Reverse(semver.Versions(vers)))
64 + } else {
65 + sort.Sort(semver.Versions(vers))
66 + }
67 +
68 + out := make([]string, len(vers))
69 + for i := range vers {
70 + out[i] = prefix + vers[i].String()
71 + }
72 +
73 + return out, nil
74 +}
75 +
76 +// IpfsRepoVersion returns the repo version required by the ipfs daemon
77 +func IpfsRepoVersion(ctx context.Context) (int, error) {
78 + out, err := exec.CommandContext(ctx, "ipfs", "version", "--repo").CombinedOutput()
79 + if err != nil {
80 + return 0, fmt.Errorf("%s: %s", err, string(out))
81 + }
82 +
83 + verStr := strings.TrimSpace(string(out))
84 + ver, err := strconv.Atoi(verStr)
85 + if err != nil {
86 + return 0, fmt.Errorf("repo version is not an integer: %s", verStr)
87 + }
88 +
89 + return ver, nil
90 +}
repo/fsrepo/migrations/versions_test.go new
+63
@@ -0,0 +1,63 @@
1 +package migrations
2 +
3 +import (
4 + "context"
5 + "os/exec"
6 + "testing"
7 +
8 + "github.com/coreos/go-semver/semver"
9 +)
10 +
11 +const testDist = "go-ipfs"
12 +
13 +func TestDistVersions(t *testing.T) {
14 + ctx, cancel := context.WithCancel(context.Background())
15 + defer cancel()
16 +
17 + vers, err := DistVersions(ctx, testDist, true)
18 + if err != nil {
19 + t.Fatal(err)
20 + }
21 + if len(vers) == 0 {
22 + t.Fatal("no versions of", testDist)
23 + }
24 + t.Log("There are", len(vers), "versions of", testDist)
25 + t.Log("Latest 5 are:", vers[:5])
26 +}
27 +
28 +func TestLatestDistVersion(t *testing.T) {
29 + ctx, cancel := context.WithCancel(context.Background())
30 + defer cancel()
31 +
32 + latest, err := LatestDistVersion(ctx, testDist)
33 + if err != nil {
34 + t.Fatal(err)
35 + }
36 + if len(latest) < 6 {
37 + t.Fatal("latest version string too short", latest)
38 + }
39 + _, err = semver.NewVersion(latest[1:])
40 + if err != nil {
41 + t.Fatal("latest version has invalid format:", latest)
42 + }
43 + t.Log("Latest version of", testDist, "is", latest)
44 +}
45 +
46 +func TestIpfsRepoVersion(t *testing.T) {
47 + _, err := exec.LookPath("ipfs")
48 + if err != nil {
49 + t.Skip("ipfs not available")
50 + }
51 +
52 + ctx, cancel := context.WithCancel(context.Background())
53 + defer cancel()
54 +
55 + ipfsRepoVer, err := IpfsRepoVersion(ctx)
56 + if err != nil {
57 + t.Fatal("Could not get required repo version:", err)
58 + }
59 + if ipfsRepoVer < 1 {
60 + t.Fatal("Invalid repo version")
61 + }
62 + t.Log("IPFS repo version:", ipfsRepoVer)
63 +}