Fix error reading zip when archive not found
- Udate path to IPFS dist - Improve test coverage
gammazero committed
Jan 27, 2021 at 20:32 UTC
e205b664a2da1994c06fbc8e4bdf433e75115f7f
6 files changed
+382
-9
repo/fsrepo/migrations/fetch.go
+3
-1
@@ -16,6 +16,8 @@ import (
16
)
17
18
const (
19
+ envIpfsDistPath = "IPFS_DIST_PATH"
20
+
21
// Distribution
22
gatewayURL = "https://ipfs.io"
23
ipfsDist = "/ipns/dist.ipfs.io"
@@ -45,7 +47,7 @@ func SetIpfsDistPath(distPath string) {
47
return
48
}
49
48
- if dist := os.Getenv("IPFS_DIST_PATH"); dist != "" {
50
+ if dist := os.Getenv(envIpfsDistPath); dist != "" {
51
ipfsDistPath = dist
52
} else {
53
ipfsDistPath = ipfsDist
repo/fsrepo/migrations/fetch_test.go
+84
-2
@@ -10,6 +10,35 @@ import (
10
"testing"
11
)
12
13
+func TestSetIpfsDistPath(t *testing.T) {
14
+ os.Unsetenv(envIpfsDistPath)
15
+ SetIpfsDistPath("")
16
+ if ipfsDistPath != ipfsDist {
17
+ t.Error("did not set default dist path")
18
+ }
19
+
20
+ testDist := "/unit/test/dist"
21
+ err := os.Setenv(envIpfsDistPath, testDist)
22
+ if err != nil {
23
+ panic(err)
24
+ }
25
+ defer func() {
26
+ os.Unsetenv(envIpfsDistPath)
27
+ SetIpfsDistPath("")
28
+ }()
29
+
30
+ SetIpfsDistPath("")
31
+ if ipfsDistPath != testDist {
32
+ t.Error("did not set dist path from environ")
33
+ }
34
+
35
+ testDist = "/unit/test/dist2"
36
+ SetIpfsDistPath(testDist)
37
+ if ipfsDistPath != testDist {
38
+ t.Error("did not set dist path")
39
+ }
40
+}
41
+
42
func TestHttpFetch(t *testing.T) {
43
ctx, cancel := context.WithCancel(context.Background())
44
defer cancel()
@@ -39,11 +68,24 @@ func TestHttpFetch(t *testing.T) {
68
}
69
70
// Check bad URL
71
+ _, err = httpFetch(ctx, "")
72
+ if err == nil {
73
+ t.Fatal("expected error")
74
+ }
75
+
76
+ // Check unreachable URL
77
+ _, err = httpFetch(ctx, "http://127.0.0.123:65510")
78
+ if err == nil || !strings.HasSuffix(err.Error(), "connection refused") {
79
+ t.Fatal("expected 'connection refused' error")
80
+ }
81
+
82
+ // Check not found
83
url = gatewayURL + path.Join(ipfsDistPath, distFSRM, "no_such_file")
84
_, err = httpFetch(ctx, url)
85
if err == nil || !strings.Contains(err.Error(), "404") {
86
t.Fatal("expected error 404")
87
}
88
+
89
}
90
91
func TestIpfsFetch(t *testing.T) {
@@ -103,7 +145,7 @@ func TestFetchBinary(t *testing.T) {
145
}
146
t.Log("latest version of", distFSRM, "is", vers[len(vers)-1])
147
106
- bin, err := FetchBinary(ctx, distFSRM, vers[0], distFSRM, distFSRM, tmpDir)
148
+ bin, err := FetchBinary(ctx, distFSRM, vers[0], distFSRM, "", tmpDir)
149
if err != nil {
150
t.Fatal(err)
151
}
@@ -115,7 +157,7 @@ func TestFetchBinary(t *testing.T) {
157
158
t.Log("downloaded and unpacked", fi.Size(), "byte file:", fi.Name())
159
118
- bin, err = FetchBinary(ctx, "go-ipfs", "v0.3.5", "go-ipfs", "ipfs", tmpDir)
160
+ bin, err = FetchBinary(ctx, "go-ipfs", "v0.3.5", "", "ipfs", tmpDir)
161
if err != nil {
162
t.Fatal(err)
163
}
@@ -126,4 +168,44 @@ func TestFetchBinary(t *testing.T) {
168
}
169
170
t.Log("downloaded and unpacked", fi.Size(), "byte file:", fi.Name())
171
+
172
+ // Check error is destination already exists and is not directory
173
+ _, err = FetchBinary(ctx, "go-ipfs", "v0.3.5", "", "ipfs", bin)
174
+ if !os.IsExist(err) {
175
+ t.Fatal("expected 'exists' error")
176
+ }
177
+
178
+ // Check error creating temp download directory
179
+ err = os.Chmod(tmpDir, 0555)
180
+ if err != nil {
181
+ panic(err)
182
+ }
183
+ err = os.Setenv("TMPDIR", tmpDir)
184
+ if err != nil {
185
+ panic(err)
186
+ }
187
+ _, err = FetchBinary(ctx, "go-ipfs", "v0.3.5", "", "ipfs", tmpDir)
188
+ if !os.IsPermission(err) {
189
+ t.Error("expected 'permission'error")
190
+ }
191
+ err = os.Setenv("TMPDIR", "/tmp")
192
+ if err != nil {
193
+ panic(err)
194
+ }
195
+ err = os.Chmod(tmpDir, 0755)
196
+ if err != nil {
197
+ panic(err)
198
+ }
199
+
200
+ // Check error if failure to fetch due to bad dist
201
+ _, err = FetchBinary(ctx, "no-such-dist", "v0.3.5", "", "ipfs", tmpDir)
202
+ if err == nil || !strings.Contains(err.Error(), "Not Found") {
203
+ t.Error("expected 'Not Found' error")
204
+ }
205
+
206
+ // Check error if failure to unpack archive
207
+ _, err = FetchBinary(ctx, "go-ipfs", "v0.3.5", "", "not-such-bin", tmpDir)
208
+ if err == nil || err.Error() != "no binary found in archive" {
209
+ t.Error("expected 'no binary found in archive' error")
210
+ }
211
}
repo/fsrepo/migrations/ipfsdir_test.go
+77
-5
@@ -23,12 +23,12 @@ func TestRepoDir(t *testing.T) {
23
os.Setenv("HOME", fakeHome)
24
fakeIpfs = path.Join(fakeHome, ".ipfs")
25
26
- t.Run("testFindIpfsDir", testFindIpfsDir)
26
+ t.Run("testIpfsDir", testIpfsDir)
27
t.Run("testCheckIpfsDir", testCheckIpfsDir)
28
t.Run("testRepoVersion", testRepoVersion)
29
}
30
31
-func testFindIpfsDir(t *testing.T) {
31
+func testIpfsDir(t *testing.T) {
32
_, err := CheckIpfsDir("")
33
if err == nil {
34
t.Fatal("expected error when no .ipfs directory to find")
@@ -47,7 +47,7 @@ func testFindIpfsDir(t *testing.T) {
47
t.Fatal("wrong ipfs directory:", dir)
48
}
49
50
- os.Setenv("IPFS_PATH", "~/.ipfs")
50
+ os.Setenv(envIpfsPath, "~/.ipfs")
51
dir, err = IpfsDir("")
52
if err != nil {
53
t.Fatal(err)
@@ -55,10 +55,46 @@ func testFindIpfsDir(t *testing.T) {
55
if dir != fakeIpfs {
56
t.Fatal("wrong ipfs directory:", dir)
57
}
58
+
59
+ _, err = IpfsDir("~somesuer/foo")
60
+ if err == nil {
61
+ t.Fatal("expected error with user-specific home dir")
62
+ }
63
+
64
+ err = os.Setenv(envIpfsPath, "~somesuer/foo")
65
+ if err != nil {
66
+ panic(err)
67
+ }
68
+ _, err = IpfsDir("~somesuer/foo")
69
+ if err == nil {
70
+ t.Fatal("expected error with user-specific home dir")
71
+ }
72
+ err = os.Unsetenv(envIpfsPath)
73
+ if err != nil {
74
+ panic(err)
75
+ }
76
+
77
+ dir, err = IpfsDir("~/.ipfs")
78
+ if err != nil {
79
+ t.Fatal(err)
80
+ }
81
+ if dir != fakeIpfs {
82
+ t.Fatal("wrong ipfs directory:", dir)
83
+ }
84
+
85
+ _, err = IpfsDir("")
86
+ if err != nil {
87
+ t.Fatal(err)
88
+ }
89
}
90
91
func testCheckIpfsDir(t *testing.T) {
61
- _, err := CheckIpfsDir("~/no_such_dir")
92
+ _, err := CheckIpfsDir("~somesuer/foo")
93
+ if err == nil {
94
+ t.Fatal("expected error with user-specific home dir")
95
+ }
96
+
97
+ _, err = CheckIpfsDir("~/no_such_dir")
98
if err == nil {
99
t.Fatal("expected error from nonexistent directory")
100
}
@@ -73,7 +109,13 @@ func testCheckIpfsDir(t *testing.T) {
109
}
110
111
func testRepoVersion(t *testing.T) {
76
- _, err := RepoVersion(fakeIpfs)
112
+ badDir := "~somesuer/foo"
113
+ _, err := RepoVersion(badDir)
114
+ if err == nil {
115
+ t.Fatal("expected error with user-specific home dir")
116
+ }
117
+
118
+ _, err = RepoVersion(fakeIpfs)
119
if !os.IsNotExist(err) {
120
t.Fatal("expected not-exist error")
121
}
@@ -92,6 +134,29 @@ func testRepoVersion(t *testing.T) {
134
if ver != testVer {
135
t.Fatalf("expected version %d, got %d", testVer, ver)
136
}
137
+
138
+ err = WriteRepoVersion(badDir, testVer)
139
+ if err == nil {
140
+ t.Fatal("expected error with user-specific home dir")
141
+ }
142
+
143
+ ipfsDir, err := IpfsDir(fakeIpfs)
144
+ if err != nil {
145
+ t.Fatal(err)
146
+ }
147
+ vFilePath := path.Join(ipfsDir, versionFile)
148
+ err = ioutil.WriteFile(vFilePath, []byte("bad-version-data\n"), 0644)
149
+ if err != nil {
150
+ panic(err)
151
+ }
152
+ _, err = RepoVersion(fakeIpfs)
153
+ if err == nil || err.Error() != "invalid data in repo version file" {
154
+ t.Fatal("expected 'invalid data' error")
155
+ }
156
+ err = WriteRepoVersion(fakeIpfs, testVer)
157
+ if err != nil {
158
+ t.Fatal(err)
159
+ }
160
}
161
162
func TestApiEndpoint(t *testing.T) {
@@ -147,4 +212,11 @@ func TestApiEndpoint(t *testing.T) {
212
if val2 != val {
213
t.Fatal("expected", val, "got", val2)
214
}
215
+
216
+ _, _, err = ApiShell(fakeIpfs)
217
+ if err != nil {
218
+ if err.Error() != "ipfs api shell not up" {
219
+ t.Fatal("expected 'ipfs api shell not up' error")
220
+ }
221
+ }
222
}
repo/fsrepo/migrations/migrations_test.go
+1
-1
@@ -116,7 +116,7 @@ func TestFetchMigrations(t *testing.T) {
116
ctx, cancel := context.WithCancel(context.Background())
117
defer cancel()
118
119
- SetIpfsDistPath("/ipfs/QmdFVsmD668ijuBFJwjyXdP5Sq44a5bPNAq3nnQF77kpyJ")
119
+ SetIpfsDistPath("/ipfs/QmXt92hFRuvQgFhgHoaMxC4wLFcvKsCywQPTNmPYCGfEV4")
120
_, err := LatestDistVersion(ctx, "ipfs-1-to-2")
121
if err != nil {
122
if strings.Contains(err.Error(), http.StatusText(http.StatusNotFound)) {
repo/fsrepo/migrations/unpack.go
+5
@@ -83,9 +83,14 @@ func unpackZip(arcPath, root, name, out string) error {
83
}
84
85
bin = rc
86
+ break
87
}
88
}
89
90
+ if bin == nil {
91
+ return errors.New("no binary found in archive")
92
+ }
93
+
94
return writeToPath(bin, out)
95
}
96
repo/fsrepo/migrations/unpack_test.go
new
+212
@@ -0,0 +1,212 @@
1
+package migrations
2
+
3
+import (
4
+ "archive/tar"
5
+ "archive/zip"
6
+ "bufio"
7
+ "compress/gzip"
8
+ "io/ioutil"
9
+ "os"
10
+ "path"
11
+ "strings"
12
+ "testing"
13
+)
14
+
15
+func TestUnpackArchive(t *testing.T) {
16
+ // Check unrecognized archive type
17
+ err := unpackArchive("", "no-arch-type", "", "", "")
18
+ if err == nil || err.Error() != "unrecognized archive type: no-arch-type" {
19
+ t.Fatal("expected 'unrecognized archive type' error")
20
+ }
21
+
22
+ // Test cannot open errors
23
+ err = unpackArchive("no-archive", "tar.gz", "", "", "")
24
+ if err == nil || !strings.HasPrefix(err.Error(), "cannot open archive file") {
25
+ t.Fatal("expected 'cannot open' error, got:", err)
26
+ }
27
+ err = unpackArchive("no-archive", "zip", "", "", "")
28
+ if err == nil || !strings.HasPrefix(err.Error(), "error opening zip reader") {
29
+ t.Fatal("expected 'cannot open' error, got:", err)
30
+ }
31
+}
32
+
33
+func TestUnpackTgz(t *testing.T) {
34
+ tmpDir, err := ioutil.TempDir("", "testunpacktgz")
35
+ if err != nil {
36
+ panic(err)
37
+ }
38
+ defer os.RemoveAll(tmpDir)
39
+
40
+ badTarGzip := path.Join(tmpDir, "bad.tar.gz")
41
+ err = ioutil.WriteFile(badTarGzip, []byte("bad-data\n"), 0644)
42
+ if err != nil {
43
+ panic(err)
44
+ }
45
+ err = unpackTgz(badTarGzip, "", "abc", "abc")
46
+ if err == nil || !strings.HasPrefix(err.Error(), "error opening gzip reader") {
47
+ t.Fatal("expected error opening gzip reader, got:", err)
48
+ }
49
+
50
+ testTarGzip := path.Join(tmpDir, "test.tar.gz")
51
+ testData := "some data"
52
+ err = writeTarGzip(testTarGzip, "testroot", "testfile", testData)
53
+ if err != nil {
54
+ panic(err)
55
+ }
56
+
57
+ out := path.Join(tmpDir, "out.txt")
58
+
59
+ // Test looking for file that is not in archive
60
+ err = unpackTgz(testTarGzip, "testroot", "abc", out)
61
+ if err == nil || err.Error() != "no binary found in archive" {
62
+ t.Fatal("expected 'no binary found in archive' error, got:", err)
63
+ }
64
+
65
+ // Test that unpack works.
66
+ err = unpackTgz(testTarGzip, "testroot", "testfile", out)
67
+ if err != nil {
68
+ t.Fatal(err)
69
+ }
70
+
71
+ fi, err := os.Stat(out)
72
+ if err != nil {
73
+ t.Fatal(err)
74
+ }
75
+ if fi.Size() != int64(len(testData)) {
76
+ t.Fatal("unpacked file size is", fi.Size(), "expected", len(testData))
77
+ }
78
+
79
+}
80
+
81
+func TestUnpackZip(t *testing.T) {
82
+ tmpDir, err := ioutil.TempDir("", "testunpackzip")
83
+ if err != nil {
84
+ panic(err)
85
+ }
86
+ defer os.RemoveAll(tmpDir)
87
+
88
+ badZip := path.Join(tmpDir, "bad.zip")
89
+ err = ioutil.WriteFile(badZip, []byte("bad-data\n"), 0644)
90
+ if err != nil {
91
+ panic(err)
92
+ }
93
+ err = unpackZip(badZip, "", "abc", "abc")
94
+ if err == nil || !strings.HasPrefix(err.Error(), "error opening zip reader") {
95
+ t.Fatal("expected error opening zip reader, got:", err)
96
+ }
97
+
98
+ testZip := path.Join(tmpDir, "test.zip")
99
+ testData := "some data"
100
+ err = writeZip(testZip, "testroot", "testfile", testData)
101
+ if err != nil {
102
+ panic(err)
103
+ }
104
+
105
+ out := path.Join(tmpDir, "out.txt")
106
+
107
+ // Test looking for file that is not in archive
108
+ err = unpackZip(testZip, "testroot", "abc", out)
109
+ if err == nil || err.Error() != "no binary found in archive" {
110
+ t.Fatal("expected 'no binary found in archive' error, got:", err)
111
+ }
112
+
113
+ // Test that unpack works.
114
+ err = unpackZip(testZip, "testroot", "testfile", out)
115
+ if err != nil {
116
+ t.Fatal(err)
117
+ }
118
+
119
+ fi, err := os.Stat(out)
120
+ if err != nil {
121
+ t.Fatal(err)
122
+ }
123
+ if fi.Size() != int64(len(testData)) {
124
+ t.Fatal("unpacked file size is", fi.Size(), "expected", len(testData))
125
+ }
126
+}
127
+
128
+func writeTarGzip(archName, root, fileName, data string) error {
129
+ archFile, err := os.Create(archName)
130
+ if err != nil {
131
+ return err
132
+ }
133
+ defer archFile.Close()
134
+ wr := bufio.NewWriter(archFile)
135
+
136
+ // gzip writer writes to buffer
137
+ gzw := gzip.NewWriter(wr)
138
+ defer gzw.Close()
139
+ // tar writer writes to gzip
140
+ tw := tar.NewWriter(gzw)
141
+ defer tw.Close()
142
+
143
+ if fileName != "" {
144
+ hdr := &tar.Header{
145
+ Name: path.Join(root, fileName),
146
+ Mode: 0600,
147
+ Size: int64(len(data)),
148
+ }
149
+ // Write header
150
+ if err = tw.WriteHeader(hdr); err != nil {
151
+ return err
152
+ }
153
+ // Write file body
154
+ if _, err := tw.Write([]byte(data)); err != nil {
155
+ return err
156
+ }
157
+ }
158
+
159
+ if err = tw.Close(); err != nil {
160
+ return err
161
+ }
162
+ // Close gzip writer; finish writing gzip data to buffer
163
+ if err = gzw.Close(); err != nil {
164
+ return err
165
+ }
166
+ // Flush buffered data to file
167
+ if err = wr.Flush(); err != nil {
168
+ return err
169
+ }
170
+ // Close tar file
171
+ if err = archFile.Close(); err != nil {
172
+ return err
173
+ }
174
+ return nil
175
+}
176
+
177
+func writeZip(archName, root, fileName, data string) error {
178
+ archFile, err := os.Create(archName)
179
+ if err != nil {
180
+ return err
181
+ }
182
+ defer archFile.Close()
183
+ wr := bufio.NewWriter(archFile)
184
+
185
+ zw := zip.NewWriter(wr)
186
+ defer zw.Close()
187
+
188
+ // Write file name
189
+ f, err := zw.Create(path.Join(root, fileName))
190
+ if err != nil {
191
+ return err
192
+ }
193
+ // Write file data
194
+ _, err = f.Write([]byte(data))
195
+ if err != nil {
196
+ return err
197
+ }
198
+
199
+ // Close zip writer
200
+ if err = zw.Close(); err != nil {
201
+ return err
202
+ }
203
+ // Flush buffered data to file
204
+ if err = wr.Flush(); err != nil {
205
+ return err
206
+ }
207
+ // Close zip file
208
+ if err = archFile.Close(); err != nil {
209
+ return err
210
+ }
211
+ return nil
212
+}