@cryptotaxi247 / kubo / commits / 14649aa8b

refactor: new go-libipfs/gateway API, deprecate Gateway.Writable (#9616)

Henrique Dias committed Feb 2, 2023 at 02:50 UTC 14649aa8ba8d7612ce9e35bba776fe7e7498b343
11 files changed +376 -18
cmd/ipfs/daemon.go
+6 -2
@@ -162,7 +162,7 @@ Headers.
162 cmds.StringOption(initProfileOptionKwd, "Configuration profiles to apply for --init. See ipfs init --help for more"),
163 cmds.StringOption(routingOptionKwd, "Overrides the routing option").WithDefault(routingOptionDefaultKwd),
164 cmds.BoolOption(mountKwd, "Mounts IPFS to the filesystem using FUSE (experimental)"),
165 - cmds.BoolOption(writableKwd, "Enable writing objects (with POST, PUT and DELETE)"),
165 + cmds.BoolOption(writableKwd, "Enable legacy Gateway.Writable (deprecated)"),
166 cmds.StringOption(ipfsMountKwd, "Path to the mountpoint for IPFS (if using --mount). Defaults to config setting."),
167 cmds.StringOption(ipnsMountKwd, "Path to the mountpoint for IPNS (if using --mount). Defaults to config setting."),
168 cmds.BoolOption(unrestrictedAPIAccessKwd, "Allow API access to unlisted hashes"),
@@ -791,7 +791,11 @@ func serveHTTPGateway(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, e
791
792 writable, writableOptionFound := req.Options[writableKwd].(bool)
793 if !writableOptionFound {
794 - writable = cfg.Gateway.Writable
794 + writable = cfg.Gateway.Writable.WithDefault(false)
795 + }
796 +
797 + if writable {
798 + log.Error("serveHTTPGateway: legacy Gateway.Writable is DEPRECATED and will be removed or changed in future versions. If you are still using this, provide feedback in https://github.com/ipfs/specs/issues/375")
799 }
800
801 listeners, err := sockets.TakeListeners("io.ipfs.gateway")
config/gateway.go
+3 -3
@@ -38,9 +38,9 @@ type Gateway struct {
38 // should be redirected.
39 RootRedirect string
40
41 - // Writable enables PUT/POST request handling by this gateway. Usually,
42 - // writing is done through the API, not the gateway.
43 - Writable bool
41 + // DEPRECATED: Enables legacy PUT/POST request handling.
42 + // Modern replacement tracked in https://github.com/ipfs/specs/issues/375
43 + Writable Flag `json:",omitempty"`
44
45 // PathPrefixes was removed: https://github.com/ipfs/go-ipfs/issues/7702
46 PathPrefixes []string
config/init.go
-1
@@ -65,7 +65,6 @@ func InitWithIdentity(identity Identity) (*Config, error) {
65
66 Gateway: Gateway{
67 RootRedirect: "",
68 - Writable: false,
68 NoFetch: false,
69 PathPrefixes: []string{},
70 HTTPHeaders: map[string][]string{
core/corehttp/gateway.go
+88 -5
@@ -1,12 +1,19 @@
1 package corehttp
2
3 import (
4 + "context"
5 "fmt"
6 + "io"
7 "net"
8 "net/http"
9
10 + cid "github.com/ipfs/go-cid"
11 + "github.com/ipfs/go-libipfs/blocks"
12 + "github.com/ipfs/go-libipfs/files"
13 "github.com/ipfs/go-libipfs/gateway"
14 + iface "github.com/ipfs/interface-go-ipfs-core"
15 options "github.com/ipfs/interface-go-ipfs-core/options"
16 + "github.com/ipfs/interface-go-ipfs-core/path"
17 version "github.com/ipfs/kubo"
18 core "github.com/ipfs/kubo/core"
19 coreapi "github.com/ipfs/kubo/core/coreapi"
@@ -38,15 +45,45 @@ func GatewayOption(writable bool, paths ...string) ServeOption {
45 return nil, err
46 }
47
41 - gateway := gateway.NewHandler(gateway.Config{
42 - Headers: headers,
43 - Writable: writable,
44 - }, api, offlineAPI)
48 + gatewayConfig := gateway.Config{
49 + Headers: headers,
50 + }
51 +
52 + gatewayAPI := &gatewayAPI{
53 + api: api,
54 + offlineAPI: offlineAPI,
55 + }
56
57 + gateway := gateway.NewHandler(gatewayConfig, gatewayAPI)
58 gateway = otelhttp.NewHandler(gateway, "Gateway.Request")
59
60 + var writableGateway *writableGatewayHandler
61 + if writable {
62 + writableGateway = &writableGatewayHandler{
63 + config: &gatewayConfig,
64 + api: api,
65 + }
66 + }
67 +
68 for _, p := range paths {
49 - mux.Handle(p+"/", gateway)
69 + mux.HandleFunc(p+"/", func(w http.ResponseWriter, r *http.Request) {
70 + if writable {
71 + switch r.Method {
72 + case http.MethodPost:
73 + writableGateway.postHandler(w, r)
74 + case http.MethodDelete:
75 + writableGateway.deleteHandler(w, r)
76 + case http.MethodPut:
77 + writableGateway.putHandler(w, r)
78 + default:
79 + gateway.ServeHTTP(w, r)
80 + }
81 +
82 + return
83 + }
84 +
85 + gateway.ServeHTTP(w, r)
86 + })
87 }
88 return mux, nil
89 }
@@ -62,3 +99,49 @@ func VersionOption() ServeOption {
99 return mux, nil
100 }
101 }
102 +
103 +type gatewayAPI struct {
104 + api iface.CoreAPI
105 + offlineAPI iface.CoreAPI
106 +}
107 +
108 +func (gw *gatewayAPI) GetUnixFsNode(ctx context.Context, pth path.Resolved) (files.Node, error) {
109 + return gw.api.Unixfs().Get(ctx, pth)
110 +}
111 +
112 +func (gw *gatewayAPI) LsUnixFsDir(ctx context.Context, pth path.Resolved) (<-chan iface.DirEntry, error) {
113 + // Optimization: use Unixfs.Ls without resolving children, but using the
114 + // cumulative DAG size as the file size. This allows for a fast listing
115 + // while keeping a good enough Size field.
116 + return gw.api.Unixfs().Ls(ctx, pth,
117 + options.Unixfs.ResolveChildren(false),
118 + options.Unixfs.UseCumulativeSize(true),
119 + )
120 +}
121 +
122 +func (gw *gatewayAPI) GetBlock(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
123 + r, err := gw.api.Block().Get(ctx, path.IpfsPath(cid))
124 + if err != nil {
125 + return nil, err
126 + }
127 +
128 + data, err := io.ReadAll(r)
129 + if err != nil {
130 + return nil, err
131 + }
132 +
133 + return blocks.NewBlockWithCid(data, cid)
134 +}
135 +
136 +func (gw *gatewayAPI) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) {
137 + return gw.api.Routing().Get(ctx, "/ipns/"+c.String())
138 +}
139 +
140 +func (gw *gatewayAPI) IsCached(ctx context.Context, pth path.Path) bool {
141 + _, err := gw.offlineAPI.Block().Stat(ctx, pth)
142 + return err == nil
143 +}
144 +
145 +func (gw *gatewayAPI) ResolvePath(ctx context.Context, pth path.Path) (path.Resolved, error) {
146 + return gw.api.ResolvePath(ctx, pth)
147 +}
core/corehttp/gateway_writable.go new
+265
@@ -0,0 +1,265 @@
1 +package corehttp
2 +
3 +import (
4 + "context"
5 + "fmt"
6 + "net/http"
7 + "os"
8 + gopath "path"
9 +
10 + cid "github.com/ipfs/go-cid"
11 + ipld "github.com/ipfs/go-ipld-format"
12 + "github.com/ipfs/go-libipfs/files"
13 + "github.com/ipfs/go-libipfs/gateway"
14 + dag "github.com/ipfs/go-merkledag"
15 + "github.com/ipfs/go-mfs"
16 + path "github.com/ipfs/go-path"
17 + "github.com/ipfs/go-path/resolver"
18 + iface "github.com/ipfs/interface-go-ipfs-core"
19 + routing "github.com/libp2p/go-libp2p/core/routing"
20 +)
21 +
22 +const (
23 + ipfsPathPrefix = "/ipfs/"
24 +)
25 +
26 +type writableGatewayHandler struct {
27 + api iface.CoreAPI
28 + config *gateway.Config
29 +}
30 +
31 +func (i *writableGatewayHandler) addUserHeaders(w http.ResponseWriter) {
32 + for k, v := range i.config.Headers {
33 + w.Header()[k] = v
34 + }
35 +}
36 +
37 +func (i *writableGatewayHandler) postHandler(w http.ResponseWriter, r *http.Request) {
38 + p, err := i.api.Unixfs().Add(r.Context(), files.NewReaderFile(r.Body))
39 + if err != nil {
40 + internalWebError(w, err)
41 + return
42 + }
43 +
44 + i.addUserHeaders(w) // ok, _now_ write user's headers.
45 + w.Header().Set("IPFS-Hash", p.Cid().String())
46 + log.Debugw("CID created, http redirect", "from", r.URL, "to", p, "status", http.StatusCreated)
47 + http.Redirect(w, r, p.String(), http.StatusCreated)
48 +}
49 +
50 +func (i *writableGatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) {
51 + ctx := r.Context()
52 + ds := i.api.Dag()
53 +
54 + // Parse the path
55 + rootCid, newPath, err := parseIpfsPath(r.URL.Path)
56 + if err != nil {
57 + webError(w, "WritableGateway: failed to parse the path", err, http.StatusBadRequest)
58 + return
59 + }
60 + if newPath == "" || newPath == "/" {
61 + http.Error(w, "WritableGateway: empty path", http.StatusBadRequest)
62 + return
63 + }
64 + newDirectory, newFileName := gopath.Split(newPath)
65 +
66 + // Resolve the old root.
67 +
68 + rnode, err := ds.Get(ctx, rootCid)
69 + if err != nil {
70 + webError(w, "WritableGateway: Could not create DAG from request", err, http.StatusInternalServerError)
71 + return
72 + }
73 +
74 + pbnd, ok := rnode.(*dag.ProtoNode)
75 + if !ok {
76 + webError(w, "Cannot read non protobuf nodes through gateway", dag.ErrNotProtobuf, http.StatusBadRequest)
77 + return
78 + }
79 +
80 + // Create the new file.
81 + newFilePath, err := i.api.Unixfs().Add(ctx, files.NewReaderFile(r.Body))
82 + if err != nil {
83 + webError(w, "WritableGateway: could not create DAG from request", err, http.StatusInternalServerError)
84 + return
85 + }
86 +
87 + newFile, err := ds.Get(ctx, newFilePath.Cid())
88 + if err != nil {
89 + webError(w, "WritableGateway: failed to resolve new file", err, http.StatusInternalServerError)
90 + return
91 + }
92 +
93 + // Patch the new file into the old root.
94 +
95 + root, err := mfs.NewRoot(ctx, ds, pbnd, nil)
96 + if err != nil {
97 + webError(w, "WritableGateway: failed to create MFS root", err, http.StatusBadRequest)
98 + return
99 + }
100 +
101 + if newDirectory != "" {
102 + err := mfs.Mkdir(root, newDirectory, mfs.MkdirOpts{Mkparents: true, Flush: false})
103 + if err != nil {
104 + webError(w, "WritableGateway: failed to create MFS directory", err, http.StatusInternalServerError)
105 + return
106 + }
107 + }
108 + dirNode, err := mfs.Lookup(root, newDirectory)
109 + if err != nil {
110 + webError(w, "WritableGateway: failed to lookup directory", err, http.StatusInternalServerError)
111 + return
112 + }
113 + dir, ok := dirNode.(*mfs.Directory)
114 + if !ok {
115 + http.Error(w, "WritableGateway: target directory is not a directory", http.StatusBadRequest)
116 + return
117 + }
118 + err = dir.Unlink(newFileName)
119 + switch err {
120 + case os.ErrNotExist, nil:
121 + default:
122 + webError(w, "WritableGateway: failed to replace existing file", err, http.StatusBadRequest)
123 + return
124 + }
125 + err = dir.AddChild(newFileName, newFile)
126 + if err != nil {
127 + webError(w, "WritableGateway: failed to link file into directory", err, http.StatusInternalServerError)
128 + return
129 + }
130 + nnode, err := root.GetDirectory().GetNode()
131 + if err != nil {
132 + webError(w, "WritableGateway: failed to finalize", err, http.StatusInternalServerError)
133 + return
134 + }
135 + newcid := nnode.Cid()
136 +
137 + i.addUserHeaders(w) // ok, _now_ write user's headers.
138 + w.Header().Set("IPFS-Hash", newcid.String())
139 +
140 + redirectURL := gopath.Join(ipfsPathPrefix, newcid.String(), newPath)
141 + log.Debugw("CID replaced, redirect", "from", r.URL, "to", redirectURL, "status", http.StatusCreated)
142 + http.Redirect(w, r, redirectURL, http.StatusCreated)
143 +}
144 +
145 +func (i *writableGatewayHandler) deleteHandler(w http.ResponseWriter, r *http.Request) {
146 + ctx := r.Context()
147 +
148 + // parse the path
149 +
150 + rootCid, newPath, err := parseIpfsPath(r.URL.Path)
151 + if err != nil {
152 + webError(w, "WritableGateway: failed to parse the path", err, http.StatusBadRequest)
153 + return
154 + }
155 + if newPath == "" || newPath == "/" {
156 + http.Error(w, "WritableGateway: empty path", http.StatusBadRequest)
157 + return
158 + }
159 + directory, filename := gopath.Split(newPath)
160 +
161 + // lookup the root
162 +
163 + rootNodeIPLD, err := i.api.Dag().Get(ctx, rootCid)
164 + if err != nil {
165 + webError(w, "WritableGateway: failed to resolve root CID", err, http.StatusInternalServerError)
166 + return
167 + }
168 + rootNode, ok := rootNodeIPLD.(*dag.ProtoNode)
169 + if !ok {
170 + http.Error(w, "WritableGateway: empty path", http.StatusInternalServerError)
171 + return
172 + }
173 +
174 + // construct the mfs root
175 +
176 + root, err := mfs.NewRoot(ctx, i.api.Dag(), rootNode, nil)
177 + if err != nil {
178 + webError(w, "WritableGateway: failed to construct the MFS root", err, http.StatusBadRequest)
179 + return
180 + }
181 +
182 + // lookup the parent directory
183 +
184 + parentNode, err := mfs.Lookup(root, directory)
185 + if err != nil {
186 + webError(w, "WritableGateway: failed to look up parent", err, http.StatusInternalServerError)
187 + return
188 + }
189 +
190 + parent, ok := parentNode.(*mfs.Directory)
191 + if !ok {
192 + http.Error(w, "WritableGateway: parent is not a directory", http.StatusInternalServerError)
193 + return
194 + }
195 +
196 + // delete the file
197 +
198 + switch parent.Unlink(filename) {
199 + case nil, os.ErrNotExist:
200 + default:
201 + webError(w, "WritableGateway: failed to remove file", err, http.StatusInternalServerError)
202 + return
203 + }
204 +
205 + nnode, err := root.GetDirectory().GetNode()
206 + if err != nil {
207 + webError(w, "WritableGateway: failed to finalize", err, http.StatusInternalServerError)
208 + return
209 + }
210 + ncid := nnode.Cid()
211 +
212 + i.addUserHeaders(w) // ok, _now_ write user's headers.
213 + w.Header().Set("IPFS-Hash", ncid.String())
214 +
215 + redirectURL := gopath.Join(ipfsPathPrefix+ncid.String(), directory)
216 + // note: StatusCreated is technically correct here as we created a new resource.
217 + log.Debugw("CID deleted, redirect", "from", r.RequestURI, "to", redirectURL, "status", http.StatusCreated)
218 + http.Redirect(w, r, redirectURL, http.StatusCreated)
219 +}
220 +
221 +func parseIpfsPath(p string) (cid.Cid, string, error) {
222 + rootPath, err := path.ParsePath(p)
223 + if err != nil {
224 + return cid.Cid{}, "", err
225 + }
226 +
227 + // Check the path.
228 + rsegs := rootPath.Segments()
229 + if rsegs[0] != "ipfs" {
230 + return cid.Cid{}, "", fmt.Errorf("WritableGateway: only ipfs paths supported")
231 + }
232 +
233 + rootCid, err := cid.Decode(rsegs[1])
234 + if err != nil {
235 + return cid.Cid{}, "", err
236 + }
237 +
238 + return rootCid, path.Join(rsegs[2:]), nil
239 +}
240 +
241 +func webError(w http.ResponseWriter, message string, err error, defaultCode int) {
242 + if _, ok := err.(resolver.ErrNoLink); ok {
243 + webErrorWithCode(w, message, err, http.StatusNotFound)
244 + } else if err == routing.ErrNotFound {
245 + webErrorWithCode(w, message, err, http.StatusNotFound)
246 + } else if ipld.IsNotFound(err) {
247 + webErrorWithCode(w, message, err, http.StatusNotFound)
248 + } else if err == context.DeadlineExceeded {
249 + webErrorWithCode(w, message, err, http.StatusRequestTimeout)
250 + } else {
251 + webErrorWithCode(w, message, err, defaultCode)
252 + }
253 +}
254 +
255 +func webErrorWithCode(w http.ResponseWriter, message string, err error, code int) {
256 + http.Error(w, fmt.Sprintf("%s: %s", message, err), code)
257 + if code >= 500 {
258 + log.Warnf("server error: %s: %s", message, err)
259 + }
260 +}
261 +
262 +// return a 500 error and log
263 +func internalWebError(w http.ResponseWriter, err error) {
264 + webErrorWithCode(w, "internalWebError", err, http.StatusInternalServerError)
265 +}
docs/config.md
+4 -1
@@ -682,7 +682,10 @@ Type: `string` (url)
682
683 ### `Gateway.Writable`
684
685 -A boolean to configure whether the gateway is writeable or not.
685 +**DEPRECATED**: Enables legacy PUT/POST request handling.
686 +
687 +This API is not standardized, and should not be used for new projects.
688 +We are working on a modern replacement. IPIP can be tracked in [ipfs/specs#375](https://github.com/ipfs/specs/issues/375).
689
690 Default: `false`
691
docs/examples/kubo-as-a-library/go.mod
+1 -1
@@ -7,7 +7,7 @@ go 1.18
7 replace github.com/ipfs/kubo => ./../../..
8
9 require (
10 - github.com/ipfs/go-libipfs v0.4.1-0.20230130233950-a005a5006496
10 + github.com/ipfs/go-libipfs v0.4.1-0.20230202010411-6399b73f974c
11 github.com/ipfs/interface-go-ipfs-core v0.10.0
12 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
13 github.com/libp2p/go-libp2p v0.24.2
docs/examples/kubo-as-a-library/go.sum
+2 -2
@@ -548,8 +548,8 @@ github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2
548 github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg=
549 github.com/ipfs/go-ipns v0.3.0 h1:ai791nTgVo+zTuq2bLvEGmWP1M0A6kGTXUsgv/Yq67A=
550 github.com/ipfs/go-ipns v0.3.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24=
551 -github.com/ipfs/go-libipfs v0.4.1-0.20230130233950-a005a5006496 h1:RVI31GQCFODREpasIFyVFkS6PjJT2bMwr/Bgr9Ryql4=
552 -github.com/ipfs/go-libipfs v0.4.1-0.20230130233950-a005a5006496/go.mod h1:AAPvZADZ80i+QhGCWNWCsx8IGY0t9C+IBEngLeYtySY=
551 +github.com/ipfs/go-libipfs v0.4.1-0.20230202010411-6399b73f974c h1:Z8GrWoG3VZWj0RvHnzKlyIyXh8sgCIw62O9t3jnhqyk=
552 +github.com/ipfs/go-libipfs v0.4.1-0.20230202010411-6399b73f974c/go.mod h1:S5wg08D/FkeYxeMf8adgt6Mi6ttbA7kSFcQYlmeGHMU=
553 github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
554 github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk=
555 github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A=
go.mod
+1 -1
@@ -45,7 +45,7 @@ require (
45 github.com/ipfs/go-ipld-git v0.1.1
46 github.com/ipfs/go-ipld-legacy v0.1.1
47 github.com/ipfs/go-ipns v0.3.0
48 - github.com/ipfs/go-libipfs v0.4.1-0.20230130233950-a005a5006496
48 + github.com/ipfs/go-libipfs v0.4.1-0.20230202010411-6399b73f974c
49 github.com/ipfs/go-log v1.0.5
50 github.com/ipfs/go-log/v2 v2.5.1
51 github.com/ipfs/go-merkledag v0.9.0
go.sum
+2 -2
@@ -570,8 +570,8 @@ github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2
570 github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg=
571 github.com/ipfs/go-ipns v0.3.0 h1:ai791nTgVo+zTuq2bLvEGmWP1M0A6kGTXUsgv/Yq67A=
572 github.com/ipfs/go-ipns v0.3.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24=
573 -github.com/ipfs/go-libipfs v0.4.1-0.20230130233950-a005a5006496 h1:RVI31GQCFODREpasIFyVFkS6PjJT2bMwr/Bgr9Ryql4=
574 -github.com/ipfs/go-libipfs v0.4.1-0.20230130233950-a005a5006496/go.mod h1:AAPvZADZ80i+QhGCWNWCsx8IGY0t9C+IBEngLeYtySY=
573 +github.com/ipfs/go-libipfs v0.4.1-0.20230202010411-6399b73f974c h1:Z8GrWoG3VZWj0RvHnzKlyIyXh8sgCIw62O9t3jnhqyk=
574 +github.com/ipfs/go-libipfs v0.4.1-0.20230202010411-6399b73f974c/go.mod h1:S5wg08D/FkeYxeMf8adgt6Mi6ttbA7kSFcQYlmeGHMU=
575 github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM=
576 github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk=
577 github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A=
test/sharness/t0111-gateway-writeable.sh
+4
@@ -34,6 +34,10 @@ test_expect_success "ipfs daemon up" '
34 test_fsh cat poll_apierr || test_fsh cat poll_apiout
35 '
36
37 +test_expect_success "deprecation notice is printed when Gateway.Writable=true" '
38 + test_should_contain "legacy Gateway.Writable is DEPRECATED and will be removed or changed in future versions. If you are still using this, provide feedback in https://github.com/ipfs/specs/issues/375" daemon_err
39 +'
40 +
41 test_expect_success "HTTP gateway gives access to sample file" '
42 curl -s -o welcome "http://$GWAY_ADDR/ipfs/$HASH_WELCOME_DOCS/readme" &&
43 grep "Hello and Welcome to IPFS!" welcome