Review changes
gammazero committed
Mar 15, 2021 at 17:13 UTC
077266d3bd1912be5f6335c94ea69ecae25aad04
9 files changed
+79
-216
cmd/ipfs/daemon.go
+1
-2
@@ -288,9 +288,8 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
288
return fmt.Errorf("fs-repo requires migration")
289
}
290
291
- fetcher := migrations.NewHttpFetcher()
291
// Fetch migrations from current distribution, or location from environ
293
- fetcher.SetDistPath(migrations.GetDistPathEnv(migrations.CurrentIpfsDist))
292
+ fetcher := migrations.NewHttpFetcher(migrations.GetDistPathEnv(migrations.CurrentIpfsDist), "", "go-ipfs", 0)
293
err = migrations.RunMigration(cctx.Context(), fetcher, fsrepo.RepoVersion, "", false)
294
if err != nil {
295
fmt.Println("The migrations of fs-repo failed:")
repo/fsrepo/migrations/fetch_test.go
+40
-32
@@ -52,14 +52,10 @@ func createFakeArchive(name string, archZip bool, w io.Writer) {
52
}
53
}
54
55
-func TestSetDistPath(t *testing.T) {
56
- f1 := NewHttpFetcher()
57
- f2 := NewHttpFetcher()
58
- mf := NewMultiFetcher(f1, f2)
59
-
55
+func TestGetDistPath(t *testing.T) {
56
os.Unsetenv(envIpfsDistPath)
61
- mf.SetDistPath(GetDistPathEnv(""))
62
- if f1.distPath != IpnsIpfsDist {
57
+ distPath := GetDistPathEnv("")
58
+ if distPath != IpnsIpfsDist {
59
t.Error("did not set default dist path")
60
}
61
@@ -72,28 +68,18 @@ func TestSetDistPath(t *testing.T) {
68
os.Unsetenv(envIpfsDistPath)
69
}()
70
75
- mf.SetDistPath(GetDistPathEnv(""))
76
- if f1.distPath != testDist {
71
+ distPath = GetDistPathEnv("")
72
+ if distPath != testDist {
73
t.Error("did not set dist path from environ")
74
}
79
- if f2.distPath != testDist {
80
- t.Error("did not set dist path from environ")
81
- }
82
-
83
- mf.SetDistPath(GetDistPathEnv("ignored"))
84
- if f1.distPath != testDist {
85
- t.Error("did not set dist path from environ")
86
- }
87
- if f2.distPath != testDist {
75
+ distPath = GetDistPathEnv("ignored")
76
+ if distPath != testDist {
77
t.Error("did not set dist path from environ")
78
}
79
80
testDist = "/unit/test/dist2"
92
- mf.SetDistPath(testDist)
93
- if f1.distPath != testDist {
94
- t.Error("did not set dist path")
95
- }
96
- if f2.distPath != testDist {
81
+ fetcher := NewHttpFetcher(testDist, "", "", 0)
82
+ if fetcher.distPath != testDist {
83
t.Error("did not set dist path")
84
}
85
}
@@ -102,13 +88,10 @@ func TestHttpFetch(t *testing.T) {
88
ctx, cancel := context.WithCancel(context.Background())
89
defer cancel()
90
105
- fetcher := NewHttpFetcher()
91
ts := createTestServer()
92
defer ts.Close()
108
- err := fetcher.SetGateway(ts.URL)
109
- if err != nil {
110
- panic(err)
111
- }
93
+
94
+ fetcher := NewHttpFetcher("", ts.URL, "", 0)
95
96
rc, err := fetcher.Fetch(ctx, "/versions")
97
if err != nil {
@@ -150,12 +133,10 @@ func TestFetchBinary(t *testing.T) {
133
ctx, cancel := context.WithCancel(context.Background())
134
defer cancel()
135
153
- fetcher := NewHttpFetcher()
136
ts := createTestServer()
137
defer ts.Close()
156
- if err = fetcher.SetGateway(ts.URL); err != nil {
157
- panic(err)
158
- }
138
+
139
+ fetcher := NewHttpFetcher("", ts.URL, "", 0)
140
141
vers, err := DistVersions(ctx, fetcher, distFSRM, false)
142
if err != nil {
@@ -234,3 +215,30 @@ func TestFetchBinary(t *testing.T) {
215
t.Error("expected 'no binary found in archive' error")
216
}
217
}
218
+
219
+func TestMultiFetcher(t *testing.T) {
220
+ ctx, cancel := context.WithCancel(context.Background())
221
+ defer cancel()
222
+
223
+ ts := createTestServer()
224
+ defer ts.Close()
225
+
226
+ badFetcher := NewHttpFetcher("", "bad-url", "", 0)
227
+ fetcher := NewHttpFetcher("", ts.URL, "", 0)
228
+
229
+ mf := NewMultiFetcher(badFetcher, fetcher)
230
+
231
+ rc, err := mf.Fetch(ctx, "/versions")
232
+ if err != nil {
233
+ t.Fatal(err)
234
+ }
235
+ defer rc.Close()
236
+
237
+ vers, err := ioutil.ReadAll(rc)
238
+ if err != nil {
239
+ t.Fatal("could not read versions:", err)
240
+ }
241
+ if len(vers) < 45 {
242
+ fmt.Println("unexpected more data")
243
+ }
244
+}
repo/fsrepo/migrations/fetcher.go
-13
@@ -4,7 +4,6 @@ import (
4
"context"
5
"io"
6
"os"
7
- "strings"
7
)
8
9
const (
@@ -21,8 +20,6 @@ type Fetcher interface {
20
// Fetch attempts to fetch the file at the given ipfs path.
21
// Returns io.ReadCloser on success, which caller must close.
22
Fetch(ctx context.Context, filePath string) (io.ReadCloser, error)
24
- // SetDistPath sets the path to the distribution site for a Fetcher
25
- SetDistPath(distPath string)
23
}
24
25
// MultiFetcher holds multiple Fetchers and provides a Fetch that tries each
@@ -59,16 +56,6 @@ func (f *MultiFetcher) Fetch(ctx context.Context, ipfsPath string) (rc io.ReadCl
56
return
57
}
58
62
-// SetDistPath sets the path to the distribution site for all fetchers
63
-func (f *MultiFetcher) SetDistPath(distPath string) {
64
- if !strings.HasPrefix(distPath, "/") {
65
- distPath = "/" + distPath
66
- }
67
- for _, fetcher := range f.fetchers {
68
- fetcher.SetDistPath(distPath)
69
- }
70
-}
71
-
59
// NewLimitReadCloser returns a new io.ReadCloser with the reader wrappen in a
60
// io.LimitedReader limited to reading the amount specified.
61
func NewLimitReadCloser(rc io.ReadCloser, limit int64) io.ReadCloser {
repo/fsrepo/migrations/httpfetcher.go
+29
-26
@@ -6,7 +6,6 @@ import (
6
"io"
7
"io/ioutil"
8
"net/http"
9
- "net/url"
9
"path"
10
"strings"
11
)
@@ -18,43 +17,45 @@ const (
17
18
// HttpFetcher fetches files over HTTP
19
type HttpFetcher struct {
21
- gateway string
22
- distPath string
23
- limit int64
20
+ distPath string
21
+ gateway string
22
+ limit int64
23
+ userAgent string
24
}
25
26
var _ Fetcher = (*HttpFetcher)(nil)
27
28
// NewHttpFetcher creates a new HttpFetcher
29
-func NewHttpFetcher() *HttpFetcher {
30
- return &HttpFetcher{
31
- gateway: defaultGatewayURL,
29
+//
30
+// Specifying "" for distPath sets the default IPNS path.
31
+// Specifying "" for gateway sets the default.
32
+// Specifying 0 for fetchLimit sets the default, -1 means no limit.
33
+func NewHttpFetcher(distPath, gateway, userAgent string, fetchLimit int64) *HttpFetcher {
34
+ f := &HttpFetcher{
35
distPath: IpnsIpfsDist,
36
+ gateway: defaultGatewayURL,
37
limit: defaultFetchLimit,
38
}
35
-}
39
37
-// SetGateway sets the gateway URL
38
-func (f *HttpFetcher) SetGateway(gatewayURL string) error {
39
- gwURL, err := url.Parse(gatewayURL)
40
- if err != nil {
41
- return err
40
+ if distPath != "" {
41
+ if !strings.HasPrefix(distPath, "/") {
42
+ distPath = "/" + distPath
43
+ }
44
+ f.distPath = distPath
45
}
43
- f.gateway = gwURL.String()
44
- return nil
45
-}
46
47
-// SetDistPath sets the path to the distribution site.
48
-func (f *HttpFetcher) SetDistPath(distPath string) {
49
- if !strings.HasPrefix(distPath, "/") {
50
- distPath = "/" + distPath
47
+ if gateway != "" {
48
+ f.gateway = strings.TrimRight(gateway, "/")
49
}
52
- f.distPath = distPath
53
-}
50
55
-// SetFetchLimit sets the download size limit. A value of 0 means no limit.
56
-func (f *HttpFetcher) SetFetchLimit(limit int64) {
57
- f.limit = limit
51
+ if fetchLimit != 0 {
52
+ if fetchLimit == -1 {
53
+ fetchLimit = 0
54
+ }
55
+ f.limit = fetchLimit
56
+ }
57
+
58
+ return f
59
}
60
61
// Fetch attempts to fetch the file at the given path, from the distribution
@@ -68,7 +69,9 @@ func (f *HttpFetcher) Fetch(ctx context.Context, filePath string) (io.ReadCloser
69
return nil, fmt.Errorf("http.NewRequest error: %s", err)
70
}
71
71
- req.Header.Set("User-Agent", "go-ipfs")
72
+ if f.userAgent != "" {
73
+ req.Header.Set("User-Agent", f.userAgent)
74
+ }
75
76
resp, err := http.DefaultClient.Do(req)
77
if err != nil {
repo/fsrepo/migrations/ipfsdir.go
-27
@@ -16,39 +16,12 @@ const (
16
envIpfsPath = "IPFS_PATH"
17
defIpfsDir = ".ipfs"
18
versionFile = "version"
19
-
20
- // Local IPFS API
21
- apiFile = "api"
19
)
20
21
func init() {
22
homedir.DisableCache = true
23
}
24
28
-// ApiEndpoint reads the api file from the local ipfs install directory and
29
-// returns the address:port read from the file. If the ipfs directory is not
30
-// specified then the default location is used.
31
-func ApiEndpoint(ipfsDir string) (string, error) {
32
- ipfsDir, err := CheckIpfsDir(ipfsDir)
33
- if err != nil {
34
- return "", err
35
- }
36
- apiPath := path.Join(ipfsDir, apiFile)
37
-
38
- apiData, err := ioutil.ReadFile(apiPath)
39
- if err != nil {
40
- return "", err
41
- }
42
-
43
- val := strings.TrimSpace(string(apiData))
44
- parts := strings.Split(val, "/")
45
- if len(parts) != 5 {
46
- return "", fmt.Errorf("incorrectly formatted api string: %q", val)
47
- }
48
-
49
- return parts[2] + ":" + parts[4], nil
50
-}
51
-
25
// IpfsDir returns the path of the ipfs directory. If dir specified, then
26
// returns the expanded version dir. If dir is "", then return the directory
27
// set by IPFS_PATH, or if IPFS_PATH is not set, then return the default
repo/fsrepo/migrations/ipfsdir_test.go
-55
@@ -158,58 +158,3 @@ func testRepoVersion(t *testing.T) {
158
t.Fatal(err)
159
}
160
}
161
-
162
-func TestApiEndpoint(t *testing.T) {
163
- var err error
164
- fakeHome, err = ioutil.TempDir("", "testhome")
165
- if err != nil {
166
- panic(err)
167
- }
168
- defer os.RemoveAll(fakeHome)
169
- defer os.Unsetenv("HOME")
170
-
171
- os.Setenv("HOME", fakeHome)
172
- fakeIpfs = path.Join(fakeHome, ".ipfs")
173
-
174
- err = os.Mkdir(fakeIpfs, os.ModePerm)
175
- if err != nil {
176
- panic(err)
177
- }
178
-
179
- _, err = ApiEndpoint("")
180
- if err == nil {
181
- t.Fatal("expected error when missing api file")
182
- }
183
-
184
- apiPath := path.Join(fakeIpfs, apiFile)
185
- err = ioutil.WriteFile(apiPath, []byte("bad-data"), 0644)
186
- if err != nil {
187
- panic(err)
188
- }
189
-
190
- _, err = ApiEndpoint("")
191
- if err == nil {
192
- t.Fatal("expected error when bad data")
193
- }
194
-
195
- err = ioutil.WriteFile(apiPath, []byte("/ip4/127.0.0.1/tcp/5001"), 0644)
196
- if err != nil {
197
- panic(err)
198
- }
199
-
200
- val, err := ApiEndpoint("")
201
- if err != nil {
202
- t.Fatal(err)
203
- }
204
- if val != "127.0.0.1:5001" {
205
- t.Fatal("got unexpected value:", val)
206
- }
207
-
208
- val2, err := ApiEndpoint(fakeIpfs)
209
- if err != nil {
210
- t.Fatal(err)
211
- }
212
- if val2 != val {
213
- t.Fatal("expected", val, "got", val2)
214
- }
215
-}
repo/fsrepo/migrations/migrations_test.go
+5
-14
@@ -115,14 +115,9 @@ func TestFetchMigrations(t *testing.T) {
115
ctx, cancel := context.WithCancel(context.Background())
116
defer cancel()
117
118
- fetcher := NewHttpFetcher()
119
- fetcher.SetDistPath(CurrentIpfsDist)
118
ts := createTestServer()
119
defer ts.Close()
122
- err := fetcher.SetGateway(ts.URL)
123
- if err != nil {
124
- panic(err)
125
- }
120
+ fetcher := NewHttpFetcher(CurrentIpfsDist, ts.URL, "", 0)
121
122
tmpDir, err := ioutil.TempDir("", "migratetest")
123
if err != nil {
@@ -166,16 +161,12 @@ func TestRunMigrations(t *testing.T) {
161
t.Fatal(err)
162
}
163
169
- ctx, cancel := context.WithCancel(context.Background())
170
- defer cancel()
171
-
172
- fetcher := NewHttpFetcher()
173
- fetcher.SetDistPath(CurrentIpfsDist)
164
ts := createTestServer()
165
defer ts.Close()
176
- if err = fetcher.SetGateway(ts.URL); err != nil {
177
- panic(err)
178
- }
166
+ fetcher := NewHttpFetcher(CurrentIpfsDist, ts.URL, "", 0)
167
+
168
+ ctx, cancel := context.WithCancel(context.Background())
169
+ defer cancel()
170
171
targetVer := 9
172
repo/fsrepo/migrations/versions.go
-18
@@ -5,10 +5,8 @@ import (
5
"context"
6
"errors"
7
"fmt"
8
- "os/exec"
8
"path"
9
"sort"
11
- "strconv"
10
"strings"
11
12
"github.com/coreos/go-semver/semver"
@@ -75,19 +73,3 @@ func DistVersions(ctx context.Context, fetcher Fetcher, dist string, sortDesc bo
73
74
return out, nil
75
}
78
-
79
-// IpfsRepoVersion returns the repo version required by the ipfs daemon
80
-func IpfsRepoVersion(ctx context.Context) (int, error) {
81
- out, err := exec.CommandContext(ctx, "ipfs", "version", "--repo").CombinedOutput()
82
- if err != nil {
83
- return 0, fmt.Errorf("%s: %s", err, string(out))
84
- }
85
-
86
- verStr := strings.TrimSpace(string(out))
87
- ver, err := strconv.Atoi(verStr)
88
- if err != nil {
89
- return 0, fmt.Errorf("repo version is not an integer: %s", verStr)
90
- }
91
-
92
- return ver, nil
93
-}
repo/fsrepo/migrations/versions_test.go
+4
-29
@@ -2,7 +2,6 @@ package migrations
2
3
import (
4
"context"
5
- "os/exec"
5
"testing"
6
7
"github.com/coreos/go-semver/semver"
@@ -14,13 +13,9 @@ func TestDistVersions(t *testing.T) {
13
ctx, cancel := context.WithCancel(context.Background())
14
defer cancel()
15
17
- fetcher := NewHttpFetcher()
16
ts := createTestServer()
17
defer ts.Close()
20
- err := fetcher.SetGateway(ts.URL)
21
- if err != nil {
22
- panic(err)
23
- }
18
+ fetcher := NewHttpFetcher("", ts.URL, "", 0)
19
20
vers, err := DistVersions(ctx, fetcher, testDist, true)
21
if err != nil {
@@ -37,10 +32,9 @@ func TestLatestDistVersion(t *testing.T) {
32
ctx, cancel := context.WithCancel(context.Background())
33
defer cancel()
34
40
- fetcher := NewHttpFetcher()
41
- //ts := createTestServer()
42
- //defer ts.Close()
43
- //fetcher.SetGateway(ts.URL)
35
+ ts := createTestServer()
36
+ defer ts.Close()
37
+ fetcher := NewHttpFetcher("", ts.URL, "", 0)
38
39
latest, err := LatestDistVersion(ctx, fetcher, testDist, false)
40
if err != nil {
@@ -55,22 +49,3 @@ func TestLatestDistVersion(t *testing.T) {
49
}
50
t.Log("Latest version of", testDist, "is", latest)
51
}
58
-
59
-func TestIpfsRepoVersion(t *testing.T) {
60
- _, err := exec.LookPath("ipfs")
61
- if err != nil {
62
- t.Skip("ipfs not available")
63
- }
64
-
65
- ctx, cancel := context.WithCancel(context.Background())
66
- defer cancel()
67
-
68
- ipfsRepoVer, err := IpfsRepoVersion(ctx)
69
- if err != nil {
70
- t.Fatal("Could not get required repo version:", err)
71
- }
72
- if ipfsRepoVer < 1 {
73
- t.Fatal("Invalid repo version")
74
- }
75
- t.Log("IPFS repo version:", ipfsRepoVer)
76
-}