@cryptotaxi247 / kubo / commits / 225dbe6c0

feat: periodic version check and json config (#10438)

Co-authored-by: Lucas Molas <schomatis@gmail.com> Co-authored-by: Marcin Rataj <lidel@lidel.org>

Patryk committed Jul 24, 2024 at 23:42 UTC 225dbe6c0340527071a162913e4b64a64df9be32
10 files changed +315 -10
cmd/ipfs/kubo/daemon.go
+54 -3
@@ -1,9 +1,11 @@
1 package kubo
2
3 import (
4 + "context"
5 "errors"
6 _ "expvar"
7 "fmt"
8 + "math"
9 "net"
10 "net/http"
11 _ "net/http/pprof"
@@ -438,9 +440,11 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
440 return fmt.Errorf("unrecognized routing option: %s", routingOption)
441 }
442
441 - agentVersionSuffixString, _ := req.Options[agentVersionSuffix].(string)
442 - if agentVersionSuffixString != "" {
443 - version.SetUserAgentSuffix(agentVersionSuffixString)
443 + // Set optional agent version suffix
444 + versionSuffixFromCli, _ := req.Options[agentVersionSuffix].(string)
445 + versionSuffix := cfg.Version.AgentSuffix.WithDefault(versionSuffixFromCli)
446 + if versionSuffix != "" {
447 + version.SetUserAgentSuffix(versionSuffix)
448 }
449
450 node, err := core.NewNode(req.Context, ncfg)
@@ -610,6 +614,15 @@ take effect.
614 }
615 if len(peers) == 0 {
616 log.Error("failed to bootstrap (no peers found): consider updating Bootstrap or Peering section of your config")
617 + } else {
618 + // After 1 minute we should have enough peers
619 + // to run informed version check
620 + startVersionChecker(
621 + cctx.Context(),
622 + node,
623 + cfg.Version.SwarmCheckEnabled.WithDefault(true),
624 + cfg.Version.SwarmCheckPercentThreshold.WithDefault(config.DefaultSwarmCheckPercentThreshold),
625 + )
626 }
627 })
628 }
@@ -1056,3 +1069,41 @@ func printVersion() {
1069 fmt.Printf("System version: %s\n", runtime.GOARCH+"/"+runtime.GOOS)
1070 fmt.Printf("Golang version: %s\n", runtime.Version())
1071 }
1072 +
1073 +func startVersionChecker(ctx context.Context, nd *core.IpfsNode, enabled bool, percentThreshold int64) {
1074 + if !enabled {
1075 + return
1076 + }
1077 + ticker := time.NewTicker(time.Hour)
1078 + defer ticker.Stop()
1079 + go func() {
1080 + for {
1081 + o, err := commands.DetectNewKuboVersion(nd, percentThreshold)
1082 + if err != nil {
1083 + // The version check is best-effort, and may fail in custom
1084 + // configurations that do not run standard WAN DHT. If it
1085 + // errors here, no point in spamming logs: og once and exit.
1086 + log.Errorw("initial version check failed, will not be run again", "error", err)
1087 + return
1088 + }
1089 + if o.UpdateAvailable {
1090 + newerPercent := fmt.Sprintf("%.0f%%", math.Round(float64(o.WithGreaterVersion)/float64(o.PeersSampled)*100))
1091 + log.Errorf(`
1092 +⚠️ A NEW VERSION OF KUBO DETECTED
1093 +
1094 +This Kubo node is running an outdated version (%s).
1095 +%s of the sampled Kubo peers are running a higher version.
1096 +Visit https://github.com/ipfs/kubo/releases or https://dist.ipfs.tech/#kubo and update to version %s or later.`,
1097 + o.RunningVersion, newerPercent, o.GreatestVersion)
1098 + }
1099 + select {
1100 + case <-ctx.Done():
1101 + return
1102 + case <-nd.Process.Closing():
1103 + return
1104 + case <-ticker.C:
1105 + continue
1106 + }
1107 + }
1108 + }()
1109 +}
config/config.go
+1
@@ -37,6 +37,7 @@ type Config struct {
37 Plugins Plugins
38 Pinning Pinning
39 Import Import
40 + Version Version
41
42 Internal Internal // experimental/unstable options
43 }
config/version.go new
+14
@@ -0,0 +1,14 @@
1 +package config
2 +
3 +const DefaultSwarmCheckPercentThreshold = 5
4 +
5 +// Version allows controling things like custom user agent and update checks.
6 +type Version struct {
7 + // Optional suffix to the AgentVersion presented by `ipfs id` and exposed
8 + // via libp2p identify protocol.
9 + AgentSuffix *OptionalString `json:",omitempty"`
10 +
11 + // Detect when to warn about new version when observed via libp2p identify
12 + SwarmCheckEnabled Flag `json:",omitempty"`
13 + SwarmCheckPercentThreshold *OptionalInteger `json:",omitempty"`
14 +}
core/commands/commands_test.go
+1
@@ -199,6 +199,7 @@ func TestCommands(t *testing.T) {
199 "/swarm/resources",
200 "/update",
201 "/version",
202 + "/version/check",
203 "/version/deps",
204 }
205
core/commands/version.go
+174 -7
@@ -5,17 +5,25 @@ import (
5 "fmt"
6 "io"
7 "runtime/debug"
8 + "strings"
9
9 - version "github.com/ipfs/kubo"
10 -
10 + versioncmp "github.com/hashicorp/go-version"
11 cmds "github.com/ipfs/go-ipfs-cmds"
12 + version "github.com/ipfs/kubo"
13 + "github.com/ipfs/kubo/config"
14 + "github.com/ipfs/kubo/core"
15 + "github.com/ipfs/kubo/core/commands/cmdenv"
16 + "github.com/libp2p/go-libp2p-kad-dht/fullrt"
17 + peer "github.com/libp2p/go-libp2p/core/peer"
18 + pstore "github.com/libp2p/go-libp2p/core/peerstore"
19 )
20
21 const (
15 - versionNumberOptionName = "number"
16 - versionCommitOptionName = "commit"
17 - versionRepoOptionName = "repo"
18 - versionAllOptionName = "all"
22 + versionNumberOptionName = "number"
23 + versionCommitOptionName = "commit"
24 + versionRepoOptionName = "repo"
25 + versionAllOptionName = "all"
26 + versionCheckThresholdOptionName = "min-percent"
27 )
28
29 var VersionCmd = &cmds.Command{
@@ -24,7 +32,8 @@ var VersionCmd = &cmds.Command{
32 ShortDescription: "Returns the current version of IPFS and exits.",
33 },
34 Subcommands: map[string]*cmds.Command{
27 - "deps": depsVersionCommand,
35 + "deps": depsVersionCommand,
36 + "check": checkVersionCommand,
37 },
38
39 Options: []cmds.Option{
@@ -130,3 +139,161 @@ Print out all dependencies and their versions.`,
139 }),
140 },
141 }
142 +
143 +const DefaultMinimalVersionFraction = 0.05 // 5%
144 +
145 +type VersionCheckOutput struct {
146 + UpdateAvailable bool
147 + RunningVersion string
148 + GreatestVersion string
149 + PeersSampled int
150 + WithGreaterVersion int
151 +}
152 +
153 +var checkVersionCommand = &cmds.Command{
154 + Helptext: cmds.HelpText{
155 + Tagline: "Checks Kubo version against connected peers.",
156 + ShortDescription: `
157 +This command uses the libp2p identify protocol to check the 'AgentVersion'
158 +of connected peers and see if the Kubo version we're running is outdated.
159 +
160 +Peers with an AgentVersion that doesn't start with 'kubo/' are ignored.
161 +'UpdateAvailable' is set to true only if the 'min-fraction' criteria are met.
162 +
163 +The 'ipfs daemon' does the same check regularly and logs when a new version
164 +is available. You can stop these regular checks by setting
165 +Version.SwarmCheckEnabled:false in the config.
166 +`,
167 + },
168 + Options: []cmds.Option{
169 + cmds.IntOption(versionCheckThresholdOptionName, "t", "Percentage (1-100) of sampled peers with the new Kubo version needed to trigger an update warning.").WithDefault(config.DefaultSwarmCheckPercentThreshold),
170 + },
171 + Type: VersionCheckOutput{},
172 +
173 + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
174 + nd, err := cmdenv.GetNode(env)
175 + if err != nil {
176 + return err
177 + }
178 +
179 + if !nd.IsOnline {
180 + return ErrNotOnline
181 + }
182 +
183 + minPercent, _ := req.Options[versionCheckThresholdOptionName].(int64)
184 + output, err := DetectNewKuboVersion(nd, minPercent)
185 + if err != nil {
186 + return err
187 + }
188 +
189 + if err := cmds.EmitOnce(res, output); err != nil {
190 + return err
191 + }
192 + return nil
193 + },
194 +}
195 +
196 +// DetectNewKuboVersion observers kubo version reported by other peers via
197 +// libp2p identify protocol and notifies when threshold fraction of seen swarm
198 +// is running updated Kubo. It is used by RPC and CLI at 'ipfs version check'
199 +// and also periodically when 'ipfs daemon' is running.
200 +func DetectNewKuboVersion(nd *core.IpfsNode, minPercent int64) (VersionCheckOutput, error) {
201 + ourVersion, err := versioncmp.NewVersion(version.CurrentVersionNumber)
202 + if err != nil {
203 + return VersionCheckOutput{}, fmt.Errorf("could not parse our own version %q: %w",
204 + version.CurrentVersionNumber, err)
205 + }
206 + // MAJOR.MINOR.PATCH without any suffix
207 + ourVersion = ourVersion.Core()
208 +
209 + greatestVersionSeen := ourVersion
210 + totalPeersSampled := 1 // Us (and to avoid division-by-zero edge case)
211 + withGreaterVersion := 0
212 +
213 + recordPeerVersion := func(agentVersion string) {
214 + // We process the version as is it assembled in GetUserAgentVersion
215 + segments := strings.Split(agentVersion, "/")
216 + if len(segments) < 2 {
217 + return
218 + }
219 + if segments[0] != "kubo" {
220 + return
221 + }
222 + versionNumber := segments[1] // As in our CurrentVersionNumber
223 +
224 + peerVersion, err := versioncmp.NewVersion(versionNumber)
225 + if err != nil {
226 + // Do not error on invalid remote versions, just ignore
227 + return
228 + }
229 +
230 + // Ignore prerelases and development releases (-dev, -rcX)
231 + if peerVersion.Metadata() != "" || peerVersion.Prerelease() != "" {
232 + return
233 + }
234 +
235 + // MAJOR.MINOR.PATCH without any suffix
236 + peerVersion = peerVersion.Core()
237 +
238 + // Valid peer version number
239 + totalPeersSampled += 1
240 + if ourVersion.LessThan(peerVersion) {
241 + withGreaterVersion += 1
242 + }
243 + if peerVersion.GreaterThan(greatestVersionSeen) {
244 + greatestVersionSeen = peerVersion
245 + }
246 + }
247 +
248 + processPeerstoreEntry := func(id peer.ID) {
249 + if v, err := nd.Peerstore.Get(id, "AgentVersion"); err == nil {
250 + recordPeerVersion(v.(string))
251 + } else if errors.Is(err, pstore.ErrNotFound) { // ignore noop
252 + } else { // a bug, usually.
253 + log.Errorw("failed to get agent version from peerstore", "error", err)
254 + }
255 + }
256 +
257 + // Amino DHT client keeps information about previously seen peers
258 + if nd.DHTClient != nd.DHT && nd.DHTClient != nil {
259 + client, ok := nd.DHTClient.(*fullrt.FullRT)
260 + if !ok {
261 + return VersionCheckOutput{}, errors.New("could not perform version check due to missing or incompatible DHT configuration")
262 + }
263 + for _, p := range client.Stat() {
264 + processPeerstoreEntry(p)
265 + }
266 + } else if nd.DHT != nil && nd.DHT.WAN != nil {
267 + for _, pi := range nd.DHT.WAN.RoutingTable().GetPeerInfos() {
268 + processPeerstoreEntry(pi.Id)
269 + }
270 + } else if nd.DHT != nil && nd.DHT.LAN != nil {
271 + for _, pi := range nd.DHT.LAN.RoutingTable().GetPeerInfos() {
272 + processPeerstoreEntry(pi.Id)
273 + }
274 + } else {
275 + return VersionCheckOutput{}, errors.New("could not perform version check due to missing or incompatible DHT configuration")
276 + }
277 +
278 + if minPercent < 1 || minPercent > 100 {
279 + if minPercent == 0 {
280 + minPercent = config.DefaultSwarmCheckPercentThreshold
281 + } else {
282 + return VersionCheckOutput{}, errors.New("Version.SwarmCheckPercentThreshold must be between 1 and 100")
283 + }
284 + }
285 +
286 + minFraction := float64(minPercent) / 100.0
287 +
288 + // UpdateAvailable flag is set only if minFraction was reached
289 + greaterFraction := float64(withGreaterVersion) / float64(totalPeersSampled)
290 +
291 + // Gathered metric are returned every time
292 + return VersionCheckOutput{
293 + UpdateAvailable: (greaterFraction >= minFraction),
294 + RunningVersion: ourVersion.String(),
295 + GreatestVersion: greatestVersionSeen.String(),
296 + PeersSampled: totalPeersSampled,
297 + WithGreaterVersion: withGreaterVersion,
298 + }, nil
299 +}
docs/changelogs/v0.30.md
+17
@@ -6,6 +6,8 @@
6
7 - [Overview](#overview)
8 - [🔦 Highlights](#-highlights)
9 + - [Automated `ipfs version check`](#automated-ipfs-version-check)
10 + - [Version Suffix Configuration](#version-suffix-configuration)
11 - [📝 Changelog](#-changelog)
12 - [👨‍👩‍👧‍👦 Contributors](#-contributors)
13
@@ -13,6 +15,21 @@
15
16 ### 🔦 Highlights
17
18 +#### Automated `ipfs version check`
19 +
20 +Kubo now performs privacy-preserving version checks using the [libp2p identify protocol](https://github.com/libp2p/specs/blob/master/identify/README.md) on peers detected by the Amino DHT client.
21 +If more than 5% of Kubo peers seen by your node are running a newer version, you will receive a log message notification.
22 +
23 +- For manual checks, refer to `ipfs version check --help` for details.
24 +- To disable automated checks, set [`Version.SwarmCheckEnabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#versionswarmcheckenabled) to `false`.
25 +
26 +#### Version Suffix Configuration
27 +
28 +Defining the optional agent version suffix is now simpler. The [`Version.AgentSuffix`](https://github.com/ipfs/kubo/blob/master/docs/config.md#agentsuffix) value from the Kubo config takes precedence over any value provided via `ipfs daemon --agent-version-suffix` (which is still supported).
29 +
30 +> [!NOTE]
31 +> Setting a custom version suffix helps with ecosystem analysis, such as Amino DHT reports published at https://stats.ipfs.network
32 +
33 ### 📝 Changelog
34
35 ### 👨‍👩‍👧‍👦 Contributors
docs/config.md
+40
@@ -180,6 +180,10 @@ config file at runtime.
180 - [`Import.UnixFSRawLeaves`](#importunixfsrawleaves)
181 - [`Import.UnixFSChunker`](#importunixfschunker)
182 - [`Import.HashFunction`](#importhashfunction)
183 + - [`Version`](#version)
184 + - [`Version.AgentSuffix`](#versionagentsuffix)
185 + - [`Version.SwarmCheckEnabled`](#versionswarmcheckenabled)
186 + - [`Version.SwarmCheckPercentThreshold`](#versionswarmcheckpercentthreshold)
187
188 ## Profiles
189
@@ -2435,3 +2439,39 @@ The default hash function. Commands affected: `ipfs add`, `ipfs block put`, `ipf
2439 Default: `sha2-256`
2440
2441 Type: `optionalString`
2442 +
2443 +## `Version`
2444 +
2445 +Options to configure agent version announced to the swarm, and leveraging
2446 +other peers version for detecting when there is time to update.
2447 +
2448 +### `Version.AgentSuffix`
2449 +
2450 +Optional suffix to the AgentVersion presented by `ipfs id` and exposed via [libp2p identify protocol](https://github.com/libp2p/specs/blob/master/identify/README.md#agentversion).
2451 +
2452 +The value from config takes precedence over value passed via `ipfs daemon --agent-version-suffix`.
2453 +
2454 +> [!NOTE]
2455 +> Setting a custom version suffix helps with ecosystem analysis, such as Amino DHT reports published at https://stats.ipfs.network
2456 +
2457 +Default: `""` (no suffix, or value from `ipfs daemon --agent-version-suffix=`)
2458 +
2459 +Type: `optionalString`
2460 +
2461 +### `Version.SwarmCheckEnabled`
2462 +
2463 +Observe the AgentVersion of swarm peers and log warning when
2464 +`SwarmCheckPercentThreshold` of peers runs version higher than this node.
2465 +
2466 +Default: `true`
2467 +
2468 +Type: `flag`
2469 +
2470 +### `Version.SwarmCheckPercentThreshold`
2471 +
2472 +Control the percentage of `kubo/` peers running new version required to
2473 +trigger update warning.
2474 +
2475 +Default: `5`
2476 +
2477 +Type: `optionalInteger` (1-100)
go.mod
+1
@@ -15,6 +15,7 @@ require (
15 github.com/fsnotify/fsnotify v1.6.0
16 github.com/google/uuid v1.6.0
17 github.com/hashicorp/go-multierror v1.1.1
18 + github.com/hashicorp/go-version v1.6.0
19 github.com/ipfs-shipyard/nopfs v0.0.12
20 github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c
21 github.com/ipfs/boxo v0.21.0
go.sum
+2
@@ -310,6 +310,8 @@ github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY
310 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
311 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
312 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
313 +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
314 +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
315 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
316 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
317 github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c=
test/sharness/t0026-id.sh
+11
@@ -65,5 +65,16 @@ iptb stop
65
66 test_kill_ipfs_daemon
67
68 +# Version.AgentSuffix overrides --agent-version-suffix (local, offline)
69 +test_expect_success "setting Version.AgentSuffix in config" '
70 + ipfs config Version.AgentSuffix json-config-suffix
71 +'
72 +test_launch_ipfs_daemon --agent-version-suffix=ignored-cli-suffix
73 +test_expect_success "checking AgentVersion with suffix set via JSON config" '
74 + test_id_compute_agent json-config-suffix > expected-agent-version &&
75 + ipfs id -f "<aver>\n" > actual-agent-version &&
76 + test_cmp expected-agent-version actual-agent-version
77 +'
78 +test_kill_ipfs_daemon
79
80 test_done