@cryptotaxi247 / kubo / commits / f7aa1204b

feat: add gateway to http over libp2p

Jorropo committed Aug 30, 2023 at 18:47 UTC f7aa1204b177231a3797576e1c01e560bacf66d9
11 files changed +96 -32
cmd/ipfs/daemon.go
+47 -6
@@ -15,6 +15,9 @@ import (
15
16 multierror "github.com/hashicorp/go-multierror"
17
18 + options "github.com/ipfs/boxo/coreiface/options"
19 + cmds "github.com/ipfs/go-ipfs-cmds"
20 + mprome "github.com/ipfs/go-metrics-prometheus"
21 version "github.com/ipfs/kubo"
22 utilmain "github.com/ipfs/kubo/cmd/ipfs/util"
23 oldcmds "github.com/ipfs/kubo/commands"
@@ -30,14 +33,12 @@ import (
33 fsrepo "github.com/ipfs/kubo/repo/fsrepo"
34 "github.com/ipfs/kubo/repo/fsrepo/migrations"
35 "github.com/ipfs/kubo/repo/fsrepo/migrations/ipfsfetcher"
36 + goprocess "github.com/jbenet/goprocess"
37 p2pcrypto "github.com/libp2p/go-libp2p/core/crypto"
38 pnet "github.com/libp2p/go-libp2p/core/pnet"
39 + "github.com/libp2p/go-libp2p/core/protocol"
40 + p2phttp "github.com/libp2p/go-libp2p/p2p/http"
41 sockets "github.com/libp2p/go-socket-activation"
36 -
37 - options "github.com/ipfs/boxo/coreiface/options"
38 - cmds "github.com/ipfs/go-ipfs-cmds"
39 - mprome "github.com/ipfs/go-metrics-prometheus"
40 - goprocess "github.com/jbenet/goprocess"
42 ma "github.com/multiformats/go-multiaddr"
43 manet "github.com/multiformats/go-multiaddr/net"
44 prometheus "github.com/prometheus/client_golang/prometheus"
@@ -551,6 +552,12 @@ take effect.
552 return err
553 }
554
555 + // add trustless gateway over libp2p
556 + p2pGwErrc, err := serveTrustlessGatewayOverLibp2p(cctx)
557 + if err != nil {
558 + return err
559 + }
560 +
561 // Add ipfs version info to prometheus metrics
562 ipfsInfoMetric := promauto.NewGaugeVec(prometheus.GaugeOpts{
563 Name: "ipfs_info",
@@ -617,7 +624,7 @@ take effect.
624 // collect long-running errors and block for shutdown
625 // TODO(cryptix): our fuse currently doesn't follow this pattern for graceful shutdown
626 var errs error
620 - for err := range merge(apiErrc, gwErrc, gcErrc) {
627 + for err := range merge(apiErrc, gwErrc, gcErrc, p2pGwErrc) {
628 if err != nil {
629 errs = multierror.Append(errs, err)
630 }
@@ -899,6 +906,40 @@ func serveHTTPGateway(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, e
906 return errc, nil
907 }
908
909 +const gatewayProtocolID protocol.ID = "/ipfs-gateway" // FIXME: specify https://github.com/ipfs/specs/issues/433
910 +
911 +func serveTrustlessGatewayOverLibp2p(cctx *oldcmds.Context) (<-chan error, error) {
912 + opts := []corehttp.ServeOption{
913 + corehttp.MetricsCollectionOption("libp2p-gateway"),
914 + corehttp.TrustlessGatewayOption(),
915 + corehttp.VersionOption(),
916 + }
917 +
918 + node, err := cctx.ConstructNode()
919 + if err != nil {
920 + return nil, fmt.Errorf("serveHTTPGateway: ConstructNode() failed: %s", err)
921 + }
922 +
923 + handler, err := corehttp.MakeHandler(node, nil, opts...)
924 + if err != nil {
925 + return nil, err
926 + }
927 +
928 + h := p2phttp.Host{
929 + StreamHost: node.PeerHost,
930 + }
931 +
932 + h.SetHTTPHandler(gatewayProtocolID, handler)
933 +
934 + errc := make(chan error, 1)
935 + go func() {
936 + defer close(errc)
937 + errc <- h.Serve()
938 + }()
939 +
940 + return errc, nil
941 +}
942 +
943 // collects options and opens the fuse mountpoint.
944 func mountFuse(req *cmds.Request, cctx *oldcmds.Context) error {
945 cfg, err := cctx.GetConfig()
core/corehttp/commands.go
+2 -2
@@ -166,7 +166,7 @@ func CommandsROOption(cctx oldcmds.Context) ServeOption {
166 func CheckVersionOption() ServeOption {
167 daemonVersion := version.ApiVersion
168
169 - return ServeOption(func(n *core.IpfsNode, l net.Listener, parent *http.ServeMux) (*http.ServeMux, error) {
169 + return func(n *core.IpfsNode, l net.Listener, parent *http.ServeMux) (*http.ServeMux, error) {
170 mux := http.NewServeMux()
171 parent.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
172 if strings.HasPrefix(r.URL.Path, APIPath) {
@@ -188,5 +188,5 @@ func CheckVersionOption() ServeOption {
188 })
189
190 return mux, nil
191 - })
191 + }
192 }
core/corehttp/corehttp.go
+3 -3
@@ -31,9 +31,9 @@ const shutdownTimeout = 30 * time.Second
31 // initially passed in if not.
32 type ServeOption func(*core.IpfsNode, net.Listener, *http.ServeMux) (*http.ServeMux, error)
33
34 -// makeHandler turns a list of ServeOptions into a http.Handler that implements
34 +// MakeHandler turns a list of ServeOptions into a http.Handler that implements
35 // all of the given options, in order.
36 -func makeHandler(n *core.IpfsNode, l net.Listener, options ...ServeOption) (http.Handler, error) {
36 +func MakeHandler(n *core.IpfsNode, l net.Listener, options ...ServeOption) (http.Handler, error) {
37 topMux := http.NewServeMux()
38 mux := topMux
39 for _, option := range options {
@@ -86,7 +86,7 @@ func Serve(node *core.IpfsNode, lis net.Listener, options ...ServeOption) error
86 // make sure we close this no matter what.
87 defer lis.Close()
88
89 - handler, err := makeHandler(node, lis, options...)
89 + handler, err := MakeHandler(node, lis, options...)
90 if err != nil {
91 return err
92 }
core/corehttp/gateway.go
+25 -2
@@ -41,7 +41,7 @@ func GatewayOption(paths ...string) ServeOption {
41 handler = otelhttp.NewHandler(handler, "Gateway")
42
43 for _, p := range paths {
44 - mux.HandleFunc(p+"/", handler.ServeHTTP)
44 + mux.Handle(p+"/", handler)
45 }
46
47 return mux, nil
@@ -61,7 +61,7 @@ func HostnameOption() ServeOption {
61 }
62
63 childMux := http.NewServeMux()
64 - mux.HandleFunc("/", gateway.NewHostnameHandler(config, backend, childMux).ServeHTTP)
64 + mux.Handle("/", gateway.NewHostnameHandler(config, backend, childMux))
65 return childMux, nil
66 }
67 }
@@ -76,6 +76,29 @@ func VersionOption() ServeOption {
76 }
77 }
78
79 +func TrustlessGatewayOption() ServeOption {
80 + return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
81 + config, err := getGatewayConfig(n)
82 + if err != nil {
83 + return nil, err
84 + }
85 +
86 + bserv := blockservice.New(n.Blocks.Blockstore(), offline.Exchange(n.Blocks.Blockstore()))
87 +
88 + backend, err := gateway.NewBlocksBackend(bserv)
89 + if err != nil {
90 + return nil, err
91 + }
92 +
93 + handler := gateway.NewHandler(config, &offlineGatewayErrWrapper{gwimpl: backend})
94 + handler = otelhttp.NewHandler(handler, "Libp2p-Gateway")
95 +
96 + mux.Handle("/ipfs/", handler)
97 +
98 + return mux, nil
99 + }
100 +}
101 +
102 func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) {
103 cfg, err := n.Repo.Config()
104 if err != nil {
core/corehttp/gateway_test.go
+1 -1
@@ -121,7 +121,7 @@ func newTestServerAndNode(t *testing.T, ns mockNamesys) (*httptest.Server, iface
121 ts := httptest.NewServer(dh)
122 t.Cleanup(func() { ts.Close() })
123
124 - dh.Handler, err = makeHandler(n,
124 + dh.Handler, err = MakeHandler(n,
125 ts.Listener,
126 HostnameOption(),
127 GatewayOption("/ipfs", "/ipns"),
docs/examples/kubo-as-a-library/go.mod
+2 -2
@@ -9,7 +9,7 @@ replace github.com/ipfs/kubo => ./../../..
9 require (
10 github.com/ipfs/boxo v0.12.1-0.20230825151903-13569468babd
11 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
12 - github.com/libp2p/go-libp2p v0.30.0
12 + github.com/libp2p/go-libp2p v0.31.0
13 github.com/multiformats/go-multiaddr v0.11.0
14 )
15
@@ -148,7 +148,7 @@ require (
148 github.com/prometheus/procfs v0.11.1 // indirect
149 github.com/quic-go/qpack v0.4.0 // indirect
150 github.com/quic-go/qtls-go1-20 v0.3.3 // indirect
151 - github.com/quic-go/quic-go v0.38.0 // indirect
151 + github.com/quic-go/quic-go v0.38.1 // indirect
152 github.com/quic-go/webtransport-go v0.5.3 // indirect
153 github.com/raulk/go-watchdog v1.3.0 // indirect
154 github.com/samber/lo v1.36.0 // indirect
docs/examples/kubo-as-a-library/go.sum
+4 -4
@@ -458,8 +458,8 @@ github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZ
458 github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs=
459 github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM=
460 github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro=
461 -github.com/libp2p/go-libp2p v0.30.0 h1:9EZwFtJPFBcs/yJTnP90TpN1hgrT/EsFfM+OZuwV87U=
462 -github.com/libp2p/go-libp2p v0.30.0/go.mod h1:nr2g5V7lfftwgiJ78/HrID+pwvayLyqKCEirT2Y3Byg=
461 +github.com/libp2p/go-libp2p v0.31.0 h1:LFShhP8F6xthWiBBq3euxbKjZsoRajVEyBS9snfHxYg=
462 +github.com/libp2p/go-libp2p v0.31.0/go.mod h1:W/FEK1c/t04PbRH3fA9i5oucu5YcgrG0JVoBWT1B7Eg=
463 github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s=
464 github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w=
465 github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g=
@@ -640,8 +640,8 @@ github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo=
640 github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A=
641 github.com/quic-go/qtls-go1-20 v0.3.3 h1:17/glZSLI9P9fDAeyCHBFSWSqJcwx1byhLwP5eUIDCM=
642 github.com/quic-go/qtls-go1-20 v0.3.3/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
643 -github.com/quic-go/quic-go v0.38.0 h1:T45lASr5q/TrVwt+jrVccmqHhPL2XuSyoCLVCpfOSLc=
644 -github.com/quic-go/quic-go v0.38.0/go.mod h1:MPCuRq7KBK2hNcfKj/1iD1BGuN3eAYMeNxp3T42LRUg=
643 +github.com/quic-go/quic-go v0.38.1 h1:M36YWA5dEhEeT+slOu/SwMEucbYd0YFidxG3KlGPZaE=
644 +github.com/quic-go/quic-go v0.38.1/go.mod h1:ijnZM7JsFIkp4cRyjxJNIzdSfCLmUMg9wdyhGmg+SN4=
645 github.com/quic-go/webtransport-go v0.5.3 h1:5XMlzemqB4qmOlgIus5zB45AcZ2kCgCy2EptUrfOPWU=
646 github.com/quic-go/webtransport-go v0.5.3/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU=
647 github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk=
go.mod
+2 -2
@@ -45,7 +45,7 @@ require (
45 github.com/jbenet/goprocess v0.1.4
46 github.com/julienschmidt/httprouter v1.3.0
47 github.com/libp2p/go-doh-resolver v0.4.0
48 - github.com/libp2p/go-libp2p v0.30.0
48 + github.com/libp2p/go-libp2p v0.31.0
49 github.com/libp2p/go-libp2p-http v0.5.0
50 github.com/libp2p/go-libp2p-kad-dht v0.24.2
51 github.com/libp2p/go-libp2p-kbucket v0.6.3
@@ -192,7 +192,7 @@ require (
192 github.com/prometheus/statsd_exporter v0.22.7 // indirect
193 github.com/quic-go/qpack v0.4.0 // indirect
194 github.com/quic-go/qtls-go1-20 v0.3.3 // indirect
195 - github.com/quic-go/quic-go v0.38.0 // indirect
195 + github.com/quic-go/quic-go v0.38.1 // indirect
196 github.com/quic-go/webtransport-go v0.5.3 // indirect
197 github.com/raulk/go-watchdog v1.3.0 // indirect
198 github.com/rs/cors v1.7.0 // indirect
go.sum
+4 -4
@@ -515,8 +515,8 @@ github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZ
515 github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs=
516 github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM=
517 github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro=
518 -github.com/libp2p/go-libp2p v0.30.0 h1:9EZwFtJPFBcs/yJTnP90TpN1hgrT/EsFfM+OZuwV87U=
519 -github.com/libp2p/go-libp2p v0.30.0/go.mod h1:nr2g5V7lfftwgiJ78/HrID+pwvayLyqKCEirT2Y3Byg=
518 +github.com/libp2p/go-libp2p v0.31.0 h1:LFShhP8F6xthWiBBq3euxbKjZsoRajVEyBS9snfHxYg=
519 +github.com/libp2p/go-libp2p v0.31.0/go.mod h1:W/FEK1c/t04PbRH3fA9i5oucu5YcgrG0JVoBWT1B7Eg=
520 github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s=
521 github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w=
522 github.com/libp2p/go-libp2p-core v0.2.4/go.mod h1:STh4fdfa5vDYr0/SzYYeqnt+E6KfEV5VxfIrm0bcI0g=
@@ -746,8 +746,8 @@ github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo=
746 github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A=
747 github.com/quic-go/qtls-go1-20 v0.3.3 h1:17/glZSLI9P9fDAeyCHBFSWSqJcwx1byhLwP5eUIDCM=
748 github.com/quic-go/qtls-go1-20 v0.3.3/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
749 -github.com/quic-go/quic-go v0.38.0 h1:T45lASr5q/TrVwt+jrVccmqHhPL2XuSyoCLVCpfOSLc=
750 -github.com/quic-go/quic-go v0.38.0/go.mod h1:MPCuRq7KBK2hNcfKj/1iD1BGuN3eAYMeNxp3T42LRUg=
749 +github.com/quic-go/quic-go v0.38.1 h1:M36YWA5dEhEeT+slOu/SwMEucbYd0YFidxG3KlGPZaE=
750 +github.com/quic-go/quic-go v0.38.1/go.mod h1:ijnZM7JsFIkp4cRyjxJNIzdSfCLmUMg9wdyhGmg+SN4=
751 github.com/quic-go/webtransport-go v0.5.3 h1:5XMlzemqB4qmOlgIus5zB45AcZ2kCgCy2EptUrfOPWU=
752 github.com/quic-go/webtransport-go v0.5.3/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU=
753 github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk=
test/dependencies/go.mod
+2 -2
@@ -19,7 +19,7 @@ require (
19 github.com/ipld/go-ipld-prime v0.21.0
20 github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
21 github.com/jbenet/go-random-files v0.0.0-20190219210431-31b3f20ebded
22 - github.com/libp2p/go-libp2p v0.30.0
22 + github.com/libp2p/go-libp2p v0.31.0
23 github.com/multiformats/go-multiaddr v0.11.0
24 github.com/multiformats/go-multihash v0.2.3
25 gotest.tools/gotestsum v0.4.2
@@ -224,7 +224,7 @@ require (
224 github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect
225 github.com/quic-go/qpack v0.4.0 // indirect
226 github.com/quic-go/qtls-go1-20 v0.3.3 // indirect
227 - github.com/quic-go/quic-go v0.38.0 // indirect
227 + github.com/quic-go/quic-go v0.38.1 // indirect
228 github.com/quic-go/webtransport-go v0.5.3 // indirect
229 github.com/raulk/go-watchdog v1.3.0 // indirect
230 github.com/russross/blackfriday/v2 v2.1.0 // indirect
test/dependencies/go.sum
+4 -4
@@ -542,8 +542,8 @@ github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38y
542 github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic=
543 github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM=
544 github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro=
545 -github.com/libp2p/go-libp2p v0.30.0 h1:9EZwFtJPFBcs/yJTnP90TpN1hgrT/EsFfM+OZuwV87U=
546 -github.com/libp2p/go-libp2p v0.30.0/go.mod h1:nr2g5V7lfftwgiJ78/HrID+pwvayLyqKCEirT2Y3Byg=
545 +github.com/libp2p/go-libp2p v0.31.0 h1:LFShhP8F6xthWiBBq3euxbKjZsoRajVEyBS9snfHxYg=
546 +github.com/libp2p/go-libp2p v0.31.0/go.mod h1:W/FEK1c/t04PbRH3fA9i5oucu5YcgrG0JVoBWT1B7Eg=
547 github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s=
548 github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w=
549 github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0=
@@ -742,8 +742,8 @@ github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo=
742 github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A=
743 github.com/quic-go/qtls-go1-20 v0.3.3 h1:17/glZSLI9P9fDAeyCHBFSWSqJcwx1byhLwP5eUIDCM=
744 github.com/quic-go/qtls-go1-20 v0.3.3/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k=
745 -github.com/quic-go/quic-go v0.38.0 h1:T45lASr5q/TrVwt+jrVccmqHhPL2XuSyoCLVCpfOSLc=
746 -github.com/quic-go/quic-go v0.38.0/go.mod h1:MPCuRq7KBK2hNcfKj/1iD1BGuN3eAYMeNxp3T42LRUg=
745 +github.com/quic-go/quic-go v0.38.1 h1:M36YWA5dEhEeT+slOu/SwMEucbYd0YFidxG3KlGPZaE=
746 +github.com/quic-go/quic-go v0.38.1/go.mod h1:ijnZM7JsFIkp4cRyjxJNIzdSfCLmUMg9wdyhGmg+SN4=
747 github.com/quic-go/webtransport-go v0.5.3 h1:5XMlzemqB4qmOlgIus5zB45AcZ2kCgCy2EptUrfOPWU=
748 github.com/quic-go/webtransport-go v0.5.3/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU=
749 github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk=