@cryptotaxi247 / kubo / commits / 2fd55d198

integrate experimental AcceleratedDHTClient

The experimental AcceleratedDHTClient can be enabled from the config When enabled it modifies the output of the `ipfs stats dht` command.

Adin Schmahmann committed Apr 5, 2021 at 12:48 UTC 2fd55d198cb2dec4caf1170955a9934cbb3b64e1
9 files changed +190 -48
core/commands/dht.go
+34 -22
@@ -45,6 +45,12 @@ const (
45 dhtVerboseOptionName = "verbose"
46 )
47
48 +// kademlia extends the routing interface with a command to get the peers closest to the target
49 +type kademlia interface {
50 + routing.Routing
51 + GetClosestPeers(ctx context.Context, key string) ([]peer.ID, error)
52 +}
53 +
54 var queryDhtCmd = &cmds.Command{
55 Helptext: cmds.HelpText{
56 Tagline: "Find the closest Peer IDs to a given Peer ID by querying the DHT.",
@@ -63,7 +69,7 @@ var queryDhtCmd = &cmds.Command{
69 return err
70 }
71
66 - if nd.DHT == nil {
72 + if nd.DHTClient == nil {
73 return ErrNotDHT
74 }
75
@@ -73,40 +79,46 @@ var queryDhtCmd = &cmds.Command{
79 }
80
81 ctx, cancel := context.WithCancel(req.Context)
82 + defer cancel()
83 ctx, events := routing.RegisterForQueryEvents(ctx)
84
78 - dht := nd.DHT.WAN
79 - if !nd.DHT.WANActive() {
80 - dht = nd.DHT.LAN
85 + client := nd.DHTClient
86 + if client == nd.DHT {
87 + client = nd.DHT.WAN
88 + if !nd.DHT.WANActive() {
89 + client = nd.DHT.LAN
90 + }
91 }
92
83 - errCh := make(chan error, 1)
84 - go func() {
85 - defer close(errCh)
86 - defer cancel()
87 - closestPeers, err := dht.GetClosestPeers(ctx, string(id))
88 - if closestPeers != nil {
89 - for p := range closestPeers {
93 + if d, ok := client.(kademlia); !ok {
94 + return fmt.Errorf("dht client does not support GetClosestPeers")
95 + } else {
96 + errCh := make(chan error, 1)
97 + go func() {
98 + defer close(errCh)
99 + defer cancel()
100 + closestPeers, err := d.GetClosestPeers(ctx, string(id))
101 + for _, p := range closestPeers {
102 routing.PublishQueryEvent(ctx, &routing.QueryEvent{
103 ID: p,
104 Type: routing.FinalPeer,
105 })
106 }
95 - }
107
97 - if err != nil {
98 - errCh <- err
99 - return
100 - }
101 - }()
108 + if err != nil {
109 + errCh <- err
110 + return
111 + }
112 + }()
113
103 - for e := range events {
104 - if err := res.Emit(e); err != nil {
105 - return err
114 + for e := range events {
115 + if err := res.Emit(e); err != nil {
116 + return err
117 + }
118 }
107 - }
119
109 - return <-errCh
120 + return <-errCh
121 + }
122 },
123 Encoders: cmds.EncoderMap{
124 cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *routing.QueryEvent) error {
core/commands/stat_dht.go
+53 -1
@@ -12,6 +12,7 @@ import (
12 "github.com/libp2p/go-libp2p-core/network"
13 pstore "github.com/libp2p/go-libp2p-core/peerstore"
14 dht "github.com/libp2p/go-libp2p-kad-dht"
15 + "github.com/libp2p/go-libp2p-kad-dht/fullrt"
16 kbucket "github.com/libp2p/go-libp2p-kbucket"
17 )
18
@@ -43,7 +44,8 @@ This interface is not stable and may change from release to release.
44 `,
45 },
46 Arguments: []cmds.Argument{
46 - cmds.StringArg("dht", false, true, "The DHT whose table should be listed (wan or lan). Defaults to both."),
47 + cmds.StringArg("dht", false, true, "The DHT whose table should be listed (wanserver, lanserver, wan, lan). "+
48 + "wan and lan refer to client routing tables. When using the experimental DHT client only WAN is supported. Defaults to wan and lan."),
49 },
50 Options: []cmds.Option{},
51 Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
@@ -67,12 +69,62 @@ This interface is not stable and may change from release to release.
69 dhts = []string{"wan", "lan"}
70 }
71
72 + dhttypeloop:
73 for _, name := range dhts {
74 var dht *dht.IpfsDHT
75 +
76 + var separateClient bool
77 + if nd.DHTClient != nd.DHT {
78 + separateClient = true
79 + }
80 +
81 switch name {
82 case "wan":
83 + if separateClient {
84 + client, ok := nd.DHTClient.(*fullrt.FullRT)
85 + if !ok {
86 + return cmds.Errorf(cmds.ErrClient, "could not generate stats for the WAN DHT client type")
87 + }
88 + peerMap := client.Stat()
89 + buckets := make([]dhtBucket, 1)
90 + b := &dhtBucket{}
91 + for _, p := range peerMap {
92 + info := dhtPeerInfo{ID: p.String()}
93 +
94 + if ver, err := nd.Peerstore.Get(p, "AgentVersion"); err == nil {
95 + info.AgentVersion, _ = ver.(string)
96 + } else if err == pstore.ErrNotFound {
97 + // ignore
98 + } else {
99 + // this is a bug, usually.
100 + log.Errorw(
101 + "failed to get agent version from peerstore",
102 + "error", err,
103 + )
104 + }
105 +
106 + info.Connected = nd.PeerHost.Network().Connectedness(p) == network.Connected
107 + b.Peers = append(b.Peers, info)
108 + }
109 + buckets[0] = *b
110 +
111 + if err := res.Emit(dhtStat{
112 + Name: name,
113 + Buckets: buckets,
114 + }); err != nil {
115 + return err
116 + }
117 + continue dhttypeloop
118 + }
119 + fallthrough
120 + case "wanserver":
121 dht = nd.DHT.WAN
122 case "lan":
123 + if separateClient {
124 + return cmds.Errorf(cmds.ErrClient, "no LAN client found")
125 + }
126 + fallthrough
127 + case "lanserver":
128 dht = nd.DHT.LAN
129 default:
130 return cmds.Errorf(cmds.ErrClient, "unknown dht type: %s", name)
core/core.go
+5 -2
@@ -98,8 +98,11 @@ type IpfsNode struct {
98
99 PubSub *pubsub.PubSub `optional:"true"`
100 PSRouter *psrouter.PubsubValueStore `optional:"true"`
101 - DHT *ddht.DHT `optional:"true"`
102 - P2P *p2p.P2P `optional:"true"`
101 +
102 + DHT *ddht.DHT `optional:"true"`
103 + DHTClient routing.Routing `name:"dhtc" optional:"true"`
104 +
105 + P2P *p2p.P2P `optional:"true"`
106
107 Process goprocess.Process
108 ctx context.Context
core/node/groups.go
+1 -1
@@ -139,7 +139,7 @@ func LibP2P(bcfg *BuildCfg, cfg *config.Config) fx.Option {
139 fx.Provide(libp2p.Security(!bcfg.DisableEncryptedConnections, cfg.Swarm.Transports)),
140
141 fx.Provide(libp2p.Routing),
142 - fx.Provide(libp2p.BaseRouting),
142 + fx.Provide(libp2p.BaseRouting(cfg.Experimental.AcceleratedDHTClient)),
143 maybeProvide(libp2p.PubsubRouter, bcfg.getOpt("ipnsps")),
144
145 maybeProvide(libp2p.BandwidthCounter, !cfg.Swarm.DisableBandwidthMetrics),
core/node/libp2p/host.go
+1 -1
@@ -34,7 +34,7 @@ type P2PHostOut struct {
34 fx.Out
35
36 Host host.Host
37 - Routing BaseIpfsRouting
37 + Routing routing.Routing `name:"initialrouting"`
38 }
39
40 func Host(mctx helpers.MetricsCtx, lc fx.Lifecycle, params P2PHostIn) (out P2PHostOut, err error) {
core/node/libp2p/routing.go
+83 -14
@@ -7,9 +7,12 @@ import (
7
8 "github.com/ipfs/go-ipfs/core/node/helpers"
9
10 + "github.com/ipfs/go-ipfs/repo"
11 host "github.com/libp2p/go-libp2p-core/host"
12 routing "github.com/libp2p/go-libp2p-core/routing"
13 + dht "github.com/libp2p/go-libp2p-kad-dht"
14 ddht "github.com/libp2p/go-libp2p-kad-dht/dual"
15 + "github.com/libp2p/go-libp2p-kad-dht/fullrt"
16 "github.com/libp2p/go-libp2p-pubsub"
17 namesys "github.com/libp2p/go-libp2p-pubsub-router"
18 record "github.com/libp2p/go-libp2p-record"
@@ -32,23 +35,89 @@ type p2pRouterOut struct {
35 Router Router `group:"routers"`
36 }
37
35 -func BaseRouting(lc fx.Lifecycle, in BaseIpfsRouting) (out p2pRouterOut, dr *ddht.DHT) {
36 - if dht, ok := in.(*ddht.DHT); ok {
37 - dr = dht
38 +type processInitialRoutingIn struct {
39 + fx.In
40 +
41 + Router routing.Routing `name:"initialrouting"`
42 +
43 + // For setting up experimental DHT client
44 + Host host.Host
45 + Repo repo.Repo
46 + Validator record.Validator
47 +}
48 +
49 +type processInitialRoutingOut struct {
50 + fx.Out
51 +
52 + Router Router `group:"routers"`
53 + DHT *ddht.DHT
54 + DHTClient routing.Routing `name:"dhtc"`
55 + BaseRT BaseIpfsRouting
56 +}
57
39 - lc.Append(fx.Hook{
40 - OnStop: func(ctx context.Context) error {
41 - return dr.Close()
58 +func BaseRouting(experimentalDHTClient bool) interface{} {
59 + return func(mctx helpers.MetricsCtx, lc fx.Lifecycle, in processInitialRoutingIn) (out processInitialRoutingOut, err error) {
60 + var dr *ddht.DHT
61 + if dht, ok := in.Router.(*ddht.DHT); ok {
62 + dr = dht
63 +
64 + lc.Append(fx.Hook{
65 + OnStop: func(ctx context.Context) error {
66 + return dr.Close()
67 + },
68 + })
69 + }
70 +
71 + if dr != nil && experimentalDHTClient {
72 + cfg, err := in.Repo.Config()
73 + if err != nil {
74 + return out, err
75 + }
76 + bspeers, err := cfg.BootstrapPeers()
77 + if err != nil {
78 + return out, err
79 + }
80 +
81 + expClient, err := fullrt.NewFullRT(in.Host,
82 + dht.DefaultPrefix,
83 + fullrt.DHTOption(
84 + dht.Validator(in.Validator),
85 + dht.Datastore(in.Repo.Datastore()),
86 + dht.BootstrapPeers(bspeers...),
87 + dht.BucketSize(20),
88 + ),
89 + )
90 + if err != nil {
91 + return out, err
92 + }
93 +
94 + lc.Append(fx.Hook{
95 + OnStop: func(ctx context.Context) error {
96 + return expClient.Close()
97 + },
98 + })
99 +
100 + return processInitialRoutingOut{
101 + Router: Router{
102 + Routing: expClient,
103 + Priority: 1000,
104 + },
105 + DHT: dr,
106 + DHTClient: expClient,
107 + BaseRT: expClient,
108 + }, nil
109 + }
110 +
111 + return processInitialRoutingOut{
112 + Router: Router{
113 + Priority: 1000,
114 + Routing: in.Router,
115 },
43 - })
116 + DHT: dr,
117 + DHTClient: dr,
118 + BaseRT: in.Router,
119 + }, nil
120 }
45 -
46 - return p2pRouterOut{
47 - Router: Router{
48 - Priority: 1000,
49 - Routing: in,
50 - },
51 - }, dr
121 }
122
123 type p2pOnlineRoutingIn struct {
core/node/libp2p/routingopt.go
-1
@@ -2,7 +2,6 @@ package libp2p
2
3 import (
4 "context"
5 -
5 "github.com/ipfs/go-datastore"
6 host "github.com/libp2p/go-libp2p-core/host"
7 "github.com/libp2p/go-libp2p-core/peer"
go.mod
+2 -2
@@ -29,7 +29,7 @@ require (
29 github.com/ipfs/go-ipfs-blockstore v0.1.4
30 github.com/ipfs/go-ipfs-chunker v0.0.5
31 github.com/ipfs/go-ipfs-cmds v0.6.0
32 - github.com/ipfs/go-ipfs-config v0.13.0
32 + github.com/ipfs/go-ipfs-config v0.14.0
33 github.com/ipfs/go-ipfs-exchange-interface v0.0.1
34 github.com/ipfs/go-ipfs-exchange-offline v0.0.1
35 github.com/ipfs/go-ipfs-files v0.0.8
@@ -65,7 +65,7 @@ require (
65 github.com/libp2p/go-libp2p-core v0.8.5
66 github.com/libp2p/go-libp2p-discovery v0.5.0
67 github.com/libp2p/go-libp2p-http v0.2.0
68 - github.com/libp2p/go-libp2p-kad-dht v0.11.1
68 + github.com/libp2p/go-libp2p-kad-dht v0.12.0
69 github.com/libp2p/go-libp2p-kbucket v0.4.7
70 github.com/libp2p/go-libp2p-loggables v0.1.0
71 github.com/libp2p/go-libp2p-mplex v0.4.1
go.sum
+11 -4
@@ -271,8 +271,9 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf
271 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
272 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
273 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
274 -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
274 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
275 +github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
276 +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
277 github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
278 github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
279 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
@@ -411,8 +412,8 @@ github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7Na
412 github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8=
413 github.com/ipfs/go-ipfs-cmds v0.6.0 h1:yAxdowQZzoFKjcLI08sXVNnqVj3jnABbf9smrPQmBsw=
414 github.com/ipfs/go-ipfs-cmds v0.6.0/go.mod h1:ZgYiWVnCk43ChwoH8hAmI1IRbuVtq3GSTHwtRB/Kqhk=
414 -github.com/ipfs/go-ipfs-config v0.13.0 h1:ZH3dTmkVR9TTFBIbfWnFNC1JdwHbj8F0ryiaIFo7U/o=
415 -github.com/ipfs/go-ipfs-config v0.13.0/go.mod h1:Ei/FLgHGTdPyqCPK0oPCwGTe8VSnsjJjx7HZqUb6Ry0=
415 +github.com/ipfs/go-ipfs-config v0.14.0 h1:KijwGU788UycqPWv4GxzyfyN6EtfJjjDRzd/wSA86VU=
416 +github.com/ipfs/go-ipfs-config v0.14.0/go.mod h1:Ei/FLgHGTdPyqCPK0oPCwGTe8VSnsjJjx7HZqUb6Ry0=
417 github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
418 github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ=
419 github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw=
@@ -677,8 +678,10 @@ github.com/libp2p/go-libp2p-interface-connmgr v0.0.1/go.mod h1:GarlRLH0LdeWcLnYM
678 github.com/libp2p/go-libp2p-interface-connmgr v0.0.4/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k=
679 github.com/libp2p/go-libp2p-interface-connmgr v0.0.5/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k=
680 github.com/libp2p/go-libp2p-interface-pnet v0.0.1/go.mod h1:el9jHpQAXK5dnTpKA4yfCNBZXvrzdOU75zz+C6ryp3k=
680 -github.com/libp2p/go-libp2p-kad-dht v0.11.1 h1:FsriVQhOUZpCotWIjyFSjEDNJmUzuMma/RyyTDZanwc=
681 github.com/libp2p/go-libp2p-kad-dht v0.11.1/go.mod h1:5ojtR2acDPqh/jXf5orWy8YGb8bHQDS+qeDcoscL/PI=
682 +github.com/libp2p/go-libp2p-kad-dht v0.12.0 h1:R5vvp8kuXjsyDE/HEMKgM8XIwlRsP5BdAZexM+tJxdU=
683 +github.com/libp2p/go-libp2p-kad-dht v0.12.0/go.mod h1:zdQYru1c7dnluMpZls4i9Fj2TwYXS7YyDkJ1Yahv0w0=
684 +github.com/libp2p/go-libp2p-kbucket v0.3.1/go.mod h1:oyjT5O7tS9CQurok++ERgc46YLwEpuGoFq9ubvoUOio=
685 github.com/libp2p/go-libp2p-kbucket v0.4.7 h1:spZAcgxifvFZHBD8tErvppbnNiKA5uokDu3CV7axu70=
686 github.com/libp2p/go-libp2p-kbucket v0.4.7/go.mod h1:XyVo99AfQH0foSf176k4jY1xUJ2+jUJIZCSDm7r2YKk=
687 github.com/libp2p/go-libp2p-loggables v0.0.1/go.mod h1:lDipDlBNYbpyqyPX/KcoO+eq0sJYEVR2JgOexcivchg=
@@ -777,6 +780,8 @@ github.com/libp2p/go-libp2p-transport-upgrader v0.3.0/go.mod h1:i+SKzbRnvXdVbU3D
780 github.com/libp2p/go-libp2p-transport-upgrader v0.4.0/go.mod h1:J4ko0ObtZSmgn5BX5AmegP+dK3CSnU2lMCKsSq/EY0s=
781 github.com/libp2p/go-libp2p-transport-upgrader v0.4.2 h1:4JsnbfJzgZeRS9AWN7B9dPqn/LY/HoQTlO9gtdJTIYM=
782 github.com/libp2p/go-libp2p-transport-upgrader v0.4.2/go.mod h1:NR8ne1VwfreD5VIWIU62Agt/J18ekORFU/j1i2y8zvk=
783 +github.com/libp2p/go-libp2p-xor v0.0.0-20200501025846-71e284145d58 h1:GcTNu27BMpOTtMnQqun03+kbtHA1qTxJ/J8cZRRYu2k=
784 +github.com/libp2p/go-libp2p-xor v0.0.0-20200501025846-71e284145d58/go.mod h1:AYjOiqJIdcmI4SXE2ouKQuFrUbE5myv8txWaB2pl4TI=
785 github.com/libp2p/go-libp2p-yamux v0.1.2/go.mod h1:xUoV/RmYkg6BW/qGxA9XJyg+HzXFYkeXbnhjmnYzKp8=
786 github.com/libp2p/go-libp2p-yamux v0.1.3/go.mod h1:VGSQVrqkh6y4nm0189qqxMtvyBft44MOYYPpYKXiVt4=
787 github.com/libp2p/go-libp2p-yamux v0.2.0/go.mod h1:Db2gU+XfLpm6E4rG5uGCFX6uXA8MEXOxFcRoXUODaK8=
@@ -1208,6 +1213,7 @@ github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX
1213 github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
1214 github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
1215 github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
1216 +github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE=
1217 github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
1218 github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
1219 github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w=
@@ -1312,6 +1318,7 @@ golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPh
1318 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
1319 golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
1320 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
1321 +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
1322 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
1323 golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf h1:B2n+Zi5QeYRDAEodEu72OS36gmTWjgpXr2+cWcBW90o=
1324 golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=