repo/fsrepo/migrations: verified HTTP migrations (#10324)
Henrique Dias committed
Feb 19, 2024 at 14:20 UTC
595e1ba26857a3f694d5fb122045867b1e946dca
10 files changed
+444
-100
docs/changelogs/v0.27.md
+5
@@ -10,6 +10,7 @@
10
- [IPNS resolver cache's TTL can now be configured](#ipns-resolver-caches-ttl-can-now-be-configured)
11
- [RPC client: deprecated DHT API, added Routing API](#rpc-client-deprecated-dht-api-added-routing-api)
12
- [Deprecated DHT commands removed from `/api/v0/dht`](#deprecated-dht-commands-removed-from-apiv0dht)
13
+ - [Repository migrations are now trustless](#repository-migrations-are-now-trustless)
14
- [📝 Changelog](#-changelog)
15
- [👨👩👧👦 Contributors](#-contributors)
16
@@ -37,6 +38,10 @@ In the next version, all DHT deprecated methods will be removed from the Go RPC
38
39
All the DHT commands that were deprecated for over a year were finally removed from `/api/v0/dht`. Users should switch to modern `/api/v0/routing` which works with [both Amino DHT and Delegated Routers](https://github.com/ipfs/kubo/blob/master/docs/config.md#routing).
40
41
+#### Repository migrations are now trustless
42
+
43
+Kubo now only uses [trustless requests](https://specs.ipfs.tech/http-gateways/trustless-gateway/) (e.g., CAR files) when downloading repository migrations via HTTP. This further strengthens Kubo by not delegating trust to public gateways. The migration binaries are locally verified before being executed.
44
+
45
### 📝 Changelog
46
47
### 👨👩👧👦 Contributors
docs/gateway.md
+8
-4
@@ -12,14 +12,18 @@ Kubo's Gateway implementation follows [ipfs/specs: Specification for HTTP Gatewa
12
13
By default, Kubo nodes run
14
a [path gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#path-gateway) at `http://127.0.0.1:8080/`
15
-and a [subdomain gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway) at `http://localhost:8080/`
15
+and a [subdomain gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway) at `http://localhost:8080/`.
16
+Both support [trustless responses](https://docs.ipfs.tech/reference/http/gateway/#trustless-verifiable-retrieval) as opt-in via `Accept` header.
17
18
Additional listening addresses and gateway behaviors can be set in the [config](#configuration) file.
19
20
### Public gateways
21
21
-Protocol Labs provides a public gateway at `https://ipfs.io` (path) and `https://dweb.link` (subdomain).
22
-If you've ever seen a link in the form `https://ipfs.io/ipfs/Qm...`, that's being served from *our* gateway.
22
+Protocol Labs provides a public gateway at
23
+`https://ipfs.io` ([path](https://specs.ipfs.tech/http-gateways/path-gateway/)),
24
+`https://dweb.link` ([subdomain](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway)),
25
+and `https://trustless-gateway.link` ([trustless](https://specs.ipfs.tech/http-gateways/trustless-gateway/) only).
26
+If you've ever seen a link in the form `https://ipfs.io/ipfs/Qm...`, that's being served from a *public goods* gateway.
27
28
There is a list of third-party public gateways provided by the IPFS community at https://ipfs.github.io/public-gateway-checker/
29
@@ -105,7 +109,7 @@ This is a rough equivalent of `ipfs dag export`.
109
110
## Deprecated Subset of RPC API
111
108
-For legacy reasons, the gateway port exposes a small subset of RPC API under `/api/v0/`.
112
+For legacy reasons, some gateways may expose a small subset of RPC API under `/api/v0/`.
113
While this read-only API exposes a read-only, "safe" subset of the normal API,
114
it is deprecated and should not be used for greenfield projects.
115
go.mod
+1
@@ -136,6 +136,7 @@ require (
136
github.com/ipfs/go-bitfield v1.1.0 // indirect
137
github.com/ipfs/go-blockservice v0.5.0 // indirect
138
github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect
139
+ github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect
140
github.com/ipfs/go-ipfs-delay v0.0.1 // indirect
141
github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
142
github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect
go.sum
+3
@@ -330,11 +330,13 @@ github.com/ipfs/boxo v0.17.1-0.20240206084652-79cb4e2886d7/go.mod h1:pIZgTWdm3k3
330
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
331
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
332
github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=
333
+github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
334
github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk=
335
github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs=
336
github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM=
337
github.com/ipfs/go-blockservice v0.5.0 h1:B2mwhhhVQl2ntW2EIpaWPwSCxSuqr5fFA93Ms4bYLEY=
338
github.com/ipfs/go-blockservice v0.5.0/go.mod h1:W6brZ5k20AehbmERplmERn8o2Ni3ZZubvAxaIUeaT6w=
339
+github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
340
github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
341
github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M=
342
github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog=
@@ -367,6 +369,7 @@ github.com/ipfs/go-ipfs-blockstore v1.3.0 h1:m2EXaWgwTzAfsmt5UdJ7Is6l4gJcaM/A12X
369
github.com/ipfs/go-ipfs-blockstore v1.3.0/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE=
370
github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ=
371
github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8=
372
+github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8=
373
github.com/ipfs/go-ipfs-cmds v0.10.0 h1:ZB4+RgYaH4UARfJY0uLKl5UXgApqnRjKbuCiJVcErYk=
374
github.com/ipfs/go-ipfs-cmds v0.10.0/go.mod h1:sX5d7jkCft9XLPnkgEfXY0z2UBOB5g6fh/obBS0enJE=
375
github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
repo/fsrepo/migrations/fetch_test.go
+13
-65
@@ -5,56 +5,13 @@ import (
5
"bytes"
6
"context"
7
"fmt"
8
- "io"
9
- "net/http"
10
- "net/http/httptest"
8
"os"
12
- "path"
9
"path/filepath"
10
"runtime"
11
"strings"
12
"testing"
13
)
14
19
-func createTestServer() *httptest.Server {
20
- reqHandler := func(w http.ResponseWriter, r *http.Request) {
21
- defer r.Body.Close()
22
- if strings.Contains(r.URL.Path, "not-here") {
23
- http.NotFound(w, r)
24
- } else if strings.HasSuffix(r.URL.Path, "versions") {
25
- fmt.Fprint(w, "v1.0.0\nv1.1.0\nv1.1.2\nv2.0.0-rc1\n2.0.0\nv2.0.1\n")
26
- } else if strings.HasSuffix(r.URL.Path, ".tar.gz") {
27
- createFakeArchive(r.URL.Path, false, w)
28
- } else if strings.HasSuffix(r.URL.Path, "zip") {
29
- createFakeArchive(r.URL.Path, true, w)
30
- } else {
31
- http.NotFound(w, r)
32
- }
33
- }
34
- return httptest.NewServer(http.HandlerFunc(reqHandler))
35
-}
36
-
37
-func createFakeArchive(name string, archZip bool, w io.Writer) {
38
- fileName := strings.Split(path.Base(name), "_")[0]
39
- root := path.Base(path.Dir(path.Dir(name)))
40
-
41
- // Simulate fetching go-ipfs, which has "ipfs" as the name in the archive.
42
- if fileName == "go-ipfs" {
43
- fileName = "ipfs"
44
- }
45
- fileName = ExeName(fileName)
46
-
47
- var err error
48
- if archZip {
49
- err = writeZip(root, fileName, "FAKE DATA", w)
50
- } else {
51
- err = writeTarGzip(root, fileName, "FAKE DATA", w)
52
- }
53
- if err != nil {
54
- panic(err)
55
- }
56
-}
57
-
15
func TestGetDistPath(t *testing.T) {
16
os.Unsetenv(envIpfsDistPath)
17
distPath := GetDistPathEnv("")
@@ -91,12 +48,9 @@ func TestHttpFetch(t *testing.T) {
48
ctx, cancel := context.WithCancel(context.Background())
49
defer cancel()
50
94
- ts := createTestServer()
95
- defer ts.Close()
96
-
97
- fetcher := NewHttpFetcher("", ts.URL, "", 0)
51
+ fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0)
52
99
- out, err := fetcher.Fetch(ctx, "/versions")
53
+ out, err := fetcher.Fetch(ctx, "/kubo/versions")
54
if err != nil {
55
t.Fatal(err)
56
}
@@ -120,7 +74,7 @@ func TestHttpFetch(t *testing.T) {
74
75
// Check not found
76
_, err = fetcher.Fetch(ctx, "/no_such_file")
123
- if err == nil || !strings.Contains(err.Error(), "404") {
77
+ if err == nil || !strings.Contains(err.Error(), "no link") {
78
t.Fatal("expected error 404")
79
}
80
}
@@ -131,10 +85,7 @@ func TestFetchBinary(t *testing.T) {
85
ctx, cancel := context.WithCancel(context.Background())
86
defer cancel()
87
134
- ts := createTestServer()
135
- defer ts.Close()
136
-
137
- fetcher := NewHttpFetcher("", ts.URL, "", 0)
88
+ fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0)
89
90
vers, err := DistVersions(ctx, fetcher, distFSRM, false)
91
if err != nil {
@@ -154,7 +105,7 @@ func TestFetchBinary(t *testing.T) {
105
106
t.Log("downloaded and unpacked", fi.Size(), "byte file:", fi.Name())
107
157
- bin, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", tmpDir)
108
+ bin, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "ipfs", tmpDir)
109
if err != nil {
110
t.Fatal(err)
111
}
@@ -167,12 +118,12 @@ func TestFetchBinary(t *testing.T) {
118
t.Log("downloaded and unpacked", fi.Size(), "byte file:", fi.Name())
119
120
// Check error is destination already exists and is not directory
170
- _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", bin)
121
+ _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "ipfs", bin)
122
if !os.IsExist(err) {
123
t.Fatal("expected 'exists' error, got", err)
124
}
125
175
- _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", tmpDir)
126
+ _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "ipfs", tmpDir)
127
if !os.IsExist(err) {
128
t.Error("expected 'exists' error, got:", err)
129
}
@@ -192,7 +143,7 @@ func TestFetchBinary(t *testing.T) {
143
if err != nil {
144
panic(err)
145
}
195
- _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "ipfs", tmpDir)
146
+ _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "ipfs", tmpDir)
147
if !os.IsPermission(err) {
148
t.Error("expected 'permission' error, got:", err)
149
}
@@ -207,13 +158,13 @@ func TestFetchBinary(t *testing.T) {
158
}
159
160
// Check error if failure to fetch due to bad dist
210
- _, err = FetchBinary(ctx, fetcher, "not-here", "v0.3.5", "ipfs", tmpDir)
211
- if err == nil || !strings.Contains(err.Error(), "Not Found") {
161
+ _, err = FetchBinary(ctx, fetcher, "not-here", "v1.0.0", "ipfs", tmpDir)
162
+ if err == nil || !strings.Contains(err.Error(), "no link") {
163
t.Error("expected 'Not Found' error, got:", err)
164
}
165
166
// Check error if failure to unpack archive
216
- _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v0.3.5", "not-such-bin", tmpDir)
167
+ _, err = FetchBinary(ctx, fetcher, "go-ipfs", "v1.0.0", "not-such-bin", tmpDir)
168
if err == nil || err.Error() != "no binary found in archive" {
169
t.Error("expected 'no binary found in archive' error")
170
}
@@ -223,15 +174,12 @@ func TestMultiFetcher(t *testing.T) {
174
ctx, cancel := context.WithCancel(context.Background())
175
defer cancel()
176
226
- ts := createTestServer()
227
- defer ts.Close()
228
-
177
badFetcher := NewHttpFetcher("", "bad-url", "", 0)
230
- fetcher := NewHttpFetcher("", ts.URL, "", 0)
178
+ fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0)
179
180
mf := NewMultiFetcher(badFetcher, fetcher)
181
234
- vers, err := mf.Fetch(ctx, "/versions")
182
+ vers, err := mf.Fetch(ctx, "/kubo/versions")
183
if err != nil {
184
t.Fatal(err)
185
}
repo/fsrepo/migrations/httpfetcher.go
+165
-12
@@ -2,21 +2,40 @@ package migrations
2
3
import (
4
"context"
5
+ "errors"
6
"fmt"
7
"io"
8
"net/http"
8
- "path"
9
+ gopath "path"
10
"strings"
11
+
12
+ "github.com/ipfs/boxo/blockservice"
13
+ "github.com/ipfs/boxo/blockstore"
14
+ "github.com/ipfs/boxo/exchange/offline"
15
+ bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice"
16
+ files "github.com/ipfs/boxo/files"
17
+ "github.com/ipfs/boxo/ipld/merkledag"
18
+ unixfile "github.com/ipfs/boxo/ipld/unixfs/file"
19
+ "github.com/ipfs/boxo/ipns"
20
+ "github.com/ipfs/boxo/namesys"
21
+ "github.com/ipfs/boxo/path"
22
+ "github.com/ipfs/boxo/path/resolver"
23
+ "github.com/ipfs/go-datastore"
24
+ dssync "github.com/ipfs/go-datastore/sync"
25
+ "github.com/ipfs/go-unixfsnode"
26
+ gocarv2 "github.com/ipld/go-car/v2"
27
+ dagpb "github.com/ipld/go-codec-dagpb"
28
+ madns "github.com/multiformats/go-multiaddr-dns"
29
)
30
31
const (
32
// default is different name than ipfs.io which is being blocked by some ISPs
14
- defaultGatewayURL = "https://dweb.link"
33
+ defaultGatewayURL = "https://trustless-gateway.link"
34
// Default maximum download size.
35
defaultFetchLimit = 1024 * 1024 * 512
36
)
37
19
-// HttpFetcher fetches files over HTTP.
38
+// HttpFetcher fetches files over HTTP using verifiable CAR archives.
39
type HttpFetcher struct { //nolint
40
distPath string
41
gateway string
@@ -26,7 +45,7 @@ type HttpFetcher struct { //nolint
45
46
var _ Fetcher = (*HttpFetcher)(nil)
47
29
-// NewHttpFetcher creates a new HttpFetcher
48
+// NewHttpFetcher creates a new [HttpFetcher].
49
//
50
// Specifying "" for distPath sets the default IPNS path.
51
// Specifying "" for gateway sets the default.
@@ -62,13 +81,89 @@ func NewHttpFetcher(distPath, gateway, userAgent string, fetchLimit int64) *Http
81
// Fetch attempts to fetch the file at the given path, from the distribution
82
// site configured for this HttpFetcher.
83
func (f *HttpFetcher) Fetch(ctx context.Context, filePath string) ([]byte, error) {
65
- gwURL := f.gateway + path.Join(f.distPath, filePath)
66
- fmt.Printf("Fetching with HTTP: %q\n", gwURL)
84
+ imPath, err := f.resolvePath(ctx, gopath.Join(f.distPath, filePath))
85
+ if err != nil {
86
+ return nil, fmt.Errorf("path could not be resolved: %w", err)
87
+ }
88
+
89
+ rc, err := f.httpRequest(ctx, imPath, "application/vnd.ipld.car")
90
+ if err != nil {
91
+ return nil, fmt.Errorf("failed to fetch CAR: %w", err)
92
+ }
93
+
94
+ return carStreamToFileBytes(ctx, rc, imPath)
95
+}
96
+
97
+func (f *HttpFetcher) Close() error {
98
+ return nil
99
+}
100
+
101
+func (f *HttpFetcher) resolvePath(ctx context.Context, pathStr string) (path.ImmutablePath, error) {
102
+ p, err := path.NewPath(pathStr)
103
+ if err != nil {
104
+ return path.ImmutablePath{}, fmt.Errorf("path is invalid: %w", err)
105
+ }
106
+
107
+ for p.Mutable() {
108
+ // Download IPNS record and verify through the gateway, or resolve the
109
+ // DNSLink with the default DNS resolver.
110
+ name, err := ipns.NameFromString(p.Segments()[1])
111
+ if err == nil {
112
+ p, err = f.resolveIPNS(ctx, name)
113
+ } else {
114
+ p, err = f.resolveDNSLink(ctx, p)
115
+ }
116
68
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, gwURL, nil)
117
+ if err != nil {
118
+ return path.ImmutablePath{}, err
119
+ }
120
+ }
121
+
122
+ return path.NewImmutablePath(p)
123
+}
124
+
125
+func (f *HttpFetcher) resolveIPNS(ctx context.Context, name ipns.Name) (path.Path, error) {
126
+ rc, err := f.httpRequest(ctx, name.AsPath(), "application/vnd.ipfs.ipns-record")
127
+ if err != nil {
128
+ return path.ImmutablePath{}, err
129
+ }
130
+
131
+ rc = NewLimitReadCloser(rc, int64(ipns.MaxRecordSize))
132
+ rawRecord, err := io.ReadAll(rc)
133
+ if err != nil {
134
+ return path.ImmutablePath{}, err
135
+ }
136
+
137
+ rec, err := ipns.UnmarshalRecord(rawRecord)
138
+ if err != nil {
139
+ return path.ImmutablePath{}, err
140
+ }
141
+
142
+ err = ipns.ValidateWithName(rec, name)
143
+ if err != nil {
144
+ return path.ImmutablePath{}, err
145
+ }
146
+
147
+ return rec.Value()
148
+}
149
+
150
+func (f *HttpFetcher) resolveDNSLink(ctx context.Context, p path.Path) (path.Path, error) {
151
+ dnsResolver := namesys.NewDNSResolver(madns.DefaultResolver.LookupTXT)
152
+ res, err := dnsResolver.Resolve(ctx, p)
153
+ if err != nil {
154
+ return nil, err
155
+ }
156
+ return res.Path, nil
157
+}
158
+
159
+func (f *HttpFetcher) httpRequest(ctx context.Context, p path.Path, accept string) (io.ReadCloser, error) {
160
+ url := f.gateway + p.String()
161
+ fmt.Printf("Fetching with HTTP: %q\n", url)
162
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
163
if err != nil {
164
return nil, fmt.Errorf("http.NewRequest error: %w", err)
165
}
166
+ req.Header.Set("Accept", accept)
167
168
if f.userAgent != "" {
169
req.Header.Set("User-Agent", f.userAgent)
@@ -85,7 +180,7 @@ func (f *HttpFetcher) Fetch(ctx context.Context, filePath string) ([]byte, error
180
if err != nil {
181
return nil, fmt.Errorf("error reading error body: %w", err)
182
}
88
- return nil, fmt.Errorf("GET %s error: %s: %s", gwURL, resp.Status, string(mes))
183
+ return nil, fmt.Errorf("GET %s error: %s: %s", url, resp.Status, string(mes))
184
}
185
186
var rc io.ReadCloser
@@ -94,11 +189,69 @@ func (f *HttpFetcher) Fetch(ctx context.Context, filePath string) ([]byte, error
189
} else {
190
rc = resp.Body
191
}
97
- defer rc.Close()
192
99
- return io.ReadAll(rc)
193
+ return rc, nil
194
}
195
102
-func (f *HttpFetcher) Close() error {
103
- return nil
196
+func carStreamToFileBytes(ctx context.Context, r io.ReadCloser, imPath path.ImmutablePath) ([]byte, error) {
197
+ defer r.Close()
198
+
199
+ // Create temporary block datastore and dag service.
200
+ dataStore := dssync.MutexWrap(datastore.NewMapDatastore())
201
+ blockStore := blockstore.NewBlockstore(dataStore)
202
+ blockService := blockservice.New(blockStore, offline.Exchange(blockStore))
203
+ dagService := merkledag.NewDAGService(blockService)
204
+
205
+ defer dagService.Blocks.Close()
206
+ defer dataStore.Close()
207
+
208
+ // Create CAR reader
209
+ car, err := gocarv2.NewBlockReader(r)
210
+ if err != nil {
211
+ fmt.Println(err)
212
+ return nil, fmt.Errorf("error creating car reader: %s", err)
213
+ }
214
+
215
+ // Add all blocks to the blockstore.
216
+ for {
217
+ block, err := car.Next()
218
+ if err != nil && err != io.EOF {
219
+ return nil, fmt.Errorf("error reading block from car: %s", err)
220
+ } else if block == nil {
221
+ break
222
+ }
223
+
224
+ err = blockStore.Put(ctx, block)
225
+ if err != nil {
226
+ return nil, fmt.Errorf("error putting block in blockstore: %s", err)
227
+ }
228
+ }
229
+
230
+ fetcherCfg := bsfetcher.NewFetcherConfig(blockService)
231
+ fetcherCfg.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser)
232
+ fetcher := fetcherCfg.WithReifier(unixfsnode.Reify)
233
+ resolver := resolver.NewBasicResolver(fetcher)
234
+
235
+ cid, _, err := resolver.ResolveToLastNode(ctx, imPath)
236
+ if err != nil {
237
+ return nil, fmt.Errorf("failed to resolve: %w", err)
238
+ }
239
+
240
+ nd, err := dagService.Get(ctx, cid)
241
+ if err != nil {
242
+ return nil, fmt.Errorf("failed to resolve: %w", err)
243
+ }
244
+
245
+ // Make UnixFS file out of the node.
246
+ uf, err := unixfile.NewUnixfsFile(ctx, dagService, nd)
247
+ if err != nil {
248
+ return nil, fmt.Errorf("error building unixfs file: %s", err)
249
+ }
250
+
251
+ // Check if it's a file and return.
252
+ if f, ok := uf.(files.File); ok {
253
+ return io.ReadAll(f)
254
+ }
255
+
256
+ return nil, errors.New("unexpected unixfs node type")
257
}
repo/fsrepo/migrations/migrations_test.go
+2
-6
@@ -110,9 +110,7 @@ func TestFetchMigrations(t *testing.T) {
110
ctx, cancel := context.WithCancel(context.Background())
111
defer cancel()
112
113
- ts := createTestServer()
114
- defer ts.Close()
115
- fetcher := NewHttpFetcher(CurrentIpfsDist, ts.URL, "", 0)
113
+ fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0)
114
115
tmpDir := t.TempDir()
116
@@ -162,9 +160,7 @@ func TestRunMigrations(t *testing.T) {
160
t.Fatal(err)
161
}
162
165
- ts := createTestServer()
166
- defer ts.Close()
167
- fetcher := NewHttpFetcher(CurrentIpfsDist, ts.URL, "", 0)
163
+ fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0)
164
165
ctx, cancel := context.WithCancel(context.Background())
166
defer cancel()
repo/fsrepo/migrations/setup_test.go
new
+233
@@ -0,0 +1,233 @@
1
+package migrations
2
+
3
+import (
4
+ "bytes"
5
+ "context"
6
+ "fmt"
7
+ "io"
8
+ "net/http/httptest"
9
+ "os"
10
+ "path"
11
+ "path/filepath"
12
+ "strings"
13
+ "testing"
14
+
15
+ "github.com/ipfs/boxo/blockservice"
16
+ "github.com/ipfs/boxo/exchange/offline"
17
+ "github.com/ipfs/boxo/gateway"
18
+ blocks "github.com/ipfs/go-block-format"
19
+ "github.com/ipfs/go-cid"
20
+ "github.com/ipfs/go-unixfsnode/data/builder"
21
+ "github.com/ipld/go-car/v2"
22
+ carblockstore "github.com/ipld/go-car/v2/blockstore"
23
+ "github.com/ipld/go-ipld-prime"
24
+ cidlink "github.com/ipld/go-ipld-prime/linking/cid"
25
+ "github.com/multiformats/go-multicodec"
26
+ "github.com/multiformats/go-multihash"
27
+)
28
+
29
+var (
30
+ testIpfsDist string
31
+ testServer *httptest.Server
32
+)
33
+
34
+func TestMain(m *testing.M) {
35
+ // Setup test data
36
+ testDataDir := makeTestData()
37
+ defer os.RemoveAll(testDataDir)
38
+
39
+ testCar := makeTestCar(testDataDir)
40
+ defer os.RemoveAll(testCar)
41
+
42
+ // Setup test gateway
43
+ fd := setupTestGateway(testCar)
44
+ defer fd.Close()
45
+
46
+ // Run tests
47
+ os.Exit(m.Run())
48
+}
49
+
50
+func makeTestData() string {
51
+ tempDir, err := os.MkdirTemp("", "kubo-migrations-test-*")
52
+ if err != nil {
53
+ panic(err)
54
+ }
55
+
56
+ versions := []string{"v1.0.0", "v1.1.0", "v1.1.2", "v2.0.0-rc1", "2.0.0", "v2.0.1"}
57
+ packages := []string{"kubo", "go-ipfs", "fs-repo-migrations", "fs-repo-1-to-2", "fs-repo-2-to-3", "fs-repo-9-to-10", "fs-repo-10-to-11"}
58
+
59
+ // Generate fake data
60
+ for _, name := range packages {
61
+ err = os.MkdirAll(filepath.Join(tempDir, name), 0777)
62
+ if err != nil {
63
+ panic(err)
64
+ }
65
+
66
+ err = os.WriteFile(filepath.Join(tempDir, name, "versions"), []byte(strings.Join(versions, "\n")+"\n"), 0666)
67
+ if err != nil {
68
+ panic(err)
69
+ }
70
+
71
+ for _, version := range versions {
72
+ filename, archName := makeArchivePath(name, name, version, "tar.gz")
73
+ createFakeArchive(filepath.Join(tempDir, filename), archName, false)
74
+
75
+ filename, archName = makeArchivePath(name, name, version, "zip")
76
+ createFakeArchive(filepath.Join(tempDir, filename), archName, true)
77
+ }
78
+ }
79
+
80
+ return tempDir
81
+}
82
+
83
+func createFakeArchive(archName, name string, archZip bool) {
84
+ err := os.MkdirAll(filepath.Dir(archName), 0777)
85
+ if err != nil {
86
+ panic(err)
87
+ }
88
+
89
+ fileName := strings.Split(path.Base(name), "_")[0]
90
+ root := fileName
91
+
92
+ // Simulate fetching go-ipfs, which has "ipfs" as the name in the archive.
93
+ if fileName == "go-ipfs" || fileName == "kubo" {
94
+ fileName = "ipfs"
95
+ }
96
+ fileName = ExeName(fileName)
97
+
98
+ if archZip {
99
+ err = writeZipFile(archName, root, fileName, "FAKE DATA")
100
+ } else {
101
+ err = writeTarGzipFile(archName, root, fileName, "FAKE DATA")
102
+ }
103
+ if err != nil {
104
+ panic(err)
105
+ }
106
+}
107
+
108
+// makeTestCar makes a CAR file with the directory [testData]. This code is mostly
109
+// sourced from https://github.com/ipld/go-car/blob/1e2f0bd2c44ee31f48a8f602b25b5671cc0c4687/cmd/car/create.go
110
+func makeTestCar(testData string) string {
111
+ // make a cid with the right length that we eventually will patch with the root.
112
+ hasher, err := multihash.GetHasher(multihash.SHA2_256)
113
+ if err != nil {
114
+ panic(err)
115
+ }
116
+ digest := hasher.Sum([]byte{})
117
+ hash, err := multihash.Encode(digest, multihash.SHA2_256)
118
+ if err != nil {
119
+ panic(err)
120
+ }
121
+ proxyRoot := cid.NewCidV1(uint64(multicodec.DagPb), hash)
122
+
123
+ // Make CAR file
124
+ fd, err := os.CreateTemp("", "kubo-migrations-test-*.car")
125
+ if err != nil {
126
+ panic(err)
127
+ }
128
+ defer fd.Close()
129
+ filename := fd.Name()
130
+
131
+ rw, err := carblockstore.OpenReadWriteFile(fd, []cid.Cid{proxyRoot}, carblockstore.WriteAsCarV1(true))
132
+ if err != nil {
133
+ panic(err)
134
+ }
135
+ defer rw.Close()
136
+
137
+ ctx := context.Background()
138
+
139
+ ls := cidlink.DefaultLinkSystem()
140
+ ls.TrustedStorage = true
141
+ ls.StorageReadOpener = func(_ ipld.LinkContext, l ipld.Link) (io.Reader, error) {
142
+ cl, ok := l.(cidlink.Link)
143
+ if !ok {
144
+ return nil, fmt.Errorf("not a cidlink")
145
+ }
146
+ blk, err := rw.Get(ctx, cl.Cid)
147
+ if err != nil {
148
+ return nil, err
149
+ }
150
+ return bytes.NewBuffer(blk.RawData()), nil
151
+ }
152
+ ls.StorageWriteOpener = func(_ ipld.LinkContext) (io.Writer, ipld.BlockWriteCommitter, error) {
153
+ buf := bytes.NewBuffer(nil)
154
+ return buf, func(l ipld.Link) error {
155
+ cl, ok := l.(cidlink.Link)
156
+ if !ok {
157
+ return fmt.Errorf("not a cidlink")
158
+ }
159
+ blk, err := blocks.NewBlockWithCid(buf.Bytes(), cl.Cid)
160
+ if err != nil {
161
+ return err
162
+ }
163
+ return rw.Put(ctx, blk)
164
+ }, nil
165
+ }
166
+
167
+ l, _, err := builder.BuildUnixFSRecursive(testData, &ls)
168
+ if err != nil {
169
+ panic(err)
170
+ }
171
+
172
+ rcl, ok := l.(cidlink.Link)
173
+ if !ok {
174
+ panic(fmt.Errorf("could not interpret %s", l))
175
+ }
176
+
177
+ if err := rw.Finalize(); err != nil {
178
+ panic(err)
179
+ }
180
+ // re-open/finalize with the final root.
181
+ err = car.ReplaceRootsInFile(filename, []cid.Cid{rcl.Cid})
182
+ if err != nil {
183
+ panic(err)
184
+ }
185
+
186
+ return filename
187
+}
188
+
189
+func setupTestGateway(testCar string) io.Closer {
190
+ blockService, roots, fd, err := newBlockServiceFromCAR(testCar)
191
+ if err != nil {
192
+ panic(err)
193
+ }
194
+
195
+ if len(roots) != 1 {
196
+ panic("expected car with 1 root")
197
+ }
198
+
199
+ backend, err := gateway.NewBlocksBackend(blockService)
200
+ if err != nil {
201
+ panic(err)
202
+ }
203
+ conf := gateway.Config{
204
+ NoDNSLink: false,
205
+ DeserializedResponses: false,
206
+ }
207
+
208
+ testIpfsDist = "/ipfs/" + roots[0].String()
209
+ testServer = httptest.NewServer(gateway.NewHandler(conf, backend))
210
+
211
+ return fd
212
+}
213
+
214
+func newBlockServiceFromCAR(filepath string) (blockservice.BlockService, []cid.Cid, io.Closer, error) {
215
+ r, err := os.Open(filepath)
216
+ if err != nil {
217
+ return nil, nil, nil, err
218
+ }
219
+
220
+ bs, err := carblockstore.NewReadOnly(r, nil)
221
+ if err != nil {
222
+ _ = r.Close()
223
+ return nil, nil, nil, err
224
+ }
225
+
226
+ roots, err := bs.Roots()
227
+ if err != nil {
228
+ return nil, nil, nil, err
229
+ }
230
+
231
+ blockService := blockservice.New(bs, offline.Exchange(bs))
232
+ return blockService, roots, r, nil
233
+}
repo/fsrepo/migrations/versions_test.go
+2
-6
@@ -13,9 +13,7 @@ func TestDistVersions(t *testing.T) {
13
ctx, cancel := context.WithCancel(context.Background())
14
defer cancel()
15
16
- ts := createTestServer()
17
- defer ts.Close()
18
- fetcher := NewHttpFetcher("", ts.URL, "", 0)
16
+ fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0)
17
18
vers, err := DistVersions(ctx, fetcher, testDist, true)
19
if err != nil {
@@ -32,9 +30,7 @@ func TestLatestDistVersion(t *testing.T) {
30
ctx, cancel := context.WithCancel(context.Background())
31
defer cancel()
32
35
- ts := createTestServer()
36
- defer ts.Close()
37
- fetcher := NewHttpFetcher("", ts.URL, "", 0)
33
+ fetcher := NewHttpFetcher(testIpfsDist, testServer.URL, "", 0)
34
35
latest, err := LatestDistVersion(ctx, fetcher, testDist, false)
36
if err != nil {
test/sharness/t0003-docker-migrate.sh
+12
-7
@@ -36,15 +36,20 @@ test_expect_success "configure migration sources" '
36
ipfs config --json Migration.DownloadSources "[\"http://127.0.0.1:17233\"]"
37
'
38
39
-test_expect_success "make repo be version 4" '
40
- echo 4 > "$IPFS_PATH/version"
41
-'
42
-
39
test_expect_success "setup http response" '
40
+ mkdir migration &&
41
+ echo "v1.1.1" > migration/versions &&
42
+ mkdir -p migration/fs-repo-6-to-7 &&
43
+ echo "v1.1.1" > migration/fs-repo-6-to-7/versions &&
44
+ CID=$(ipfs add -r -Q migration) &&
45
echo "HTTP/1.1 200 OK" > vers_resp &&
45
- echo "Content-Length: 7" >> vers_resp &&
46
+ echo "Content-Type: application/vnd.ipld.car" >> vers_resp &&
47
echo "" >> vers_resp &&
47
- echo "v1.1.1" >> vers_resp
48
+ ipfs dag export $CID >> vers_resp
49
+'
50
+
51
+test_expect_success "make repo be version 4" '
52
+ echo 4 > "$IPFS_PATH/version"
53
'
54
55
test_expect_success "startup fake dists server" '
@@ -53,7 +58,7 @@ test_expect_success "startup fake dists server" '
58
'
59
60
test_expect_success "docker image runs" '
56
- DOC_ID=$(docker run -d -v "$IPFS_PATH":/data/ipfs --net=host "$IMAGE_TAG")
61
+ DOC_ID=$(docker run -d -v "$IPFS_PATH":/data/ipfs -e IPFS_DIST_PATH=/ipfs/$CID --net=host "$IMAGE_TAG")
62
'
63
64
test_expect_success "docker container tries to pull migrations from netcat" '