@cryptotaxi247 / netdata-1 / commits / e5e900d77

Port ipfs from python to Go (#18070)

* port ipfs to go * format testdata * fix imports * updates --------- Co-authored-by: ilyam8 <ilya@netdata.cloud>

Fotis Voutsas committed Jul 5, 2024 at 08:24 UTC e5e900d77207f442259a94235a739caca538b71d

26 files changed +1299 -645

CMakeLists.txt

-2

@@ -2858,7 +2858,6 @@ install(FILES

2858 src/collectors/python.d.plugin/go_expvar/go_expvar.conf

2859 src/collectors/python.d.plugin/haproxy/haproxy.conf

2860 src/collectors/python.d.plugin/icecast/icecast.conf

2861 - src/collectors/python.d.plugin/ipfs/ipfs.conf

2861 src/collectors/python.d.plugin/memcached/memcached.conf

2862 src/collectors/python.d.plugin/monit/monit.conf

2863 src/collectors/python.d.plugin/nsd/nsd.conf

@@ -2898,7 +2897,6 @@ install(FILES

2897 src/collectors/python.d.plugin/go_expvar/go_expvar.chart.py

2898 src/collectors/python.d.plugin/haproxy/haproxy.chart.py

2899 src/collectors/python.d.plugin/icecast/icecast.chart.py

2901 - src/collectors/python.d.plugin/ipfs/ipfs.chart.py

2900 src/collectors/python.d.plugin/memcached/memcached.chart.py

2901 src/collectors/python.d.plugin/monit/monit.chart.py

2902 src/collectors/python.d.plugin/nsd/nsd.chart.py

src/collectors/python.d.plugin/ipfs/README.md deleted

-1

@@ -1 +0,0 @@

1 -integrations/ipfs.md

\ No newline at end of file

src/collectors/python.d.plugin/ipfs/integrations/ipfs.md deleted

-236

@@ -1,236 +0,0 @@

1 -<!--startmeta

2 -custom_edit_url: "https://github.com/netdata/netdata/edit/master/src/collectors/python.d.plugin/ipfs/README.md"

3 -meta_yaml: "https://github.com/netdata/netdata/edit/master/src/collectors/python.d.plugin/ipfs/metadata.yaml"

4 -sidebar_label: "IPFS"

5 -learn_status: "Published"

6 -learn_rel_path: "Collecting Metrics/Storage, Mount Points and Filesystems"

7 -most_popular: False

8 -message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE COLLECTOR'S metadata.yaml FILE"

9 -endmeta-->

10 -

11 -# IPFS

12 -

13 -

14 -<img src="https://netdata.cloud/img/ipfs.svg" width="150"/>

15 -

16 -

17 -Plugin: python.d.plugin

18 -Module: ipfs

19 -

20 -<img src="https://img.shields.io/badge/maintained%20by-Netdata-%2300ab44" />

21 -

22 -## Overview

23 -

24 -This collector monitors IPFS server metrics about its quality and performance.

25 -

26 -It connects to an http endpoint of the IPFS server to collect the metrics

27 -

28 -This collector is supported on all platforms.

29 -

30 -This collector supports collecting metrics from multiple instances of this integration, including remote instances.

31 -

32 -

33 -### Default Behavior

34 -

35 -#### Auto-Detection

36 -

37 -If the endpoint is accessible by the Agent, netdata will autodetect it

38 -

39 -#### Limits

40 -

41 -Calls to the following endpoints are disabled due to IPFS bugs:

42 -

43 -/api/v0/stats/repo (https://github.com/ipfs/go-ipfs/issues/3874)

44 -/api/v0/pin/ls (https://github.com/ipfs/go-ipfs/issues/7528)

45 -

46 -

47 -#### Performance Impact

48 -

49 -The default configuration for this integration is not expected to impose a significant performance impact on the system.

50 -

51 -

52 -## Metrics

53 -

54 -Metrics grouped by *scope*.

55 -

56 -The scope defines the instance that the metric belongs to. An instance is uniquely identified by a set of labels.

57 -

58 -

59 -

60 -### Per IPFS instance

61 -

62 -These metrics refer to the entire monitored application.

63 -

64 -This scope has no labels.

65 -

66 -Metrics:

67 -

68 -| Metric | Dimensions | Unit |

69 -|:------|:----------|:----|

70 -| ipfs.bandwidth | in, out | kilobits/s |

71 -| ipfs.peers | peers | peers |

72 -| ipfs.repo_size | avail, size | GiB |

73 -| ipfs.repo_objects | objects, pinned, recursive_pins | objects |

74 -

75 -

76 -

77 -## Alerts

78 -

79 -

80 -The following alerts are available:

81 -

82 -| Alert name | On metric | Description |

83 -|:------------|:----------|:------------|

84 -| [ ipfs_datastore_usage ](https://github.com/netdata/netdata/blob/master/src/health/health.d/ipfs.conf) | ipfs.repo_size | IPFS datastore utilization |

85 -

86 -

87 -## Setup

88 -

89 -### Prerequisites

90 -

91 -No action required.

92 -

93 -### Configuration

94 -

95 -#### File

96 -

97 -The configuration file name for this integration is `python.d/ipfs.conf`.

98 -

99 -

100 -You can edit the configuration file using the `edit-config` script from the

101 -Netdata [config directory](/docs/netdata-agent/configuration/README.md#the-netdata-config-directory).

102 -

103 -```bash

104 -cd /etc/netdata 2>/dev/null || cd /opt/netdata/etc/netdata

105 -sudo ./edit-config python.d/ipfs.conf

106 -```

107 -#### Options

108 -

109 -There are 2 sections:

110 -

111 -* Global variables

112 -* One or more JOBS that can define multiple different instances to monitor.

113 -

114 -The following options can be defined globally: priority, penalty, autodetection_retry, update_every, but can also be defined per JOB to override the global values.

115 -

116 -Additionally, the following collapsed table contains all the options that can be configured inside a JOB definition.

117 -

118 -Every configuration JOB starts with a `job_name` value which will appear in the dashboard, unless a `name` parameter is specified.

119 -

120 -

121 -<details open><summary></summary>

122 -

124 -|:----|:-----------|:-------|:--------:|

125 -| update_every | Sets the default data collection frequency. | 5 | no |

126 -| priority | Controls the order of charts at the netdata dashboard. | 60000 | no |

127 -| autodetection_retry | Sets the job re-check interval in seconds. | 0 | no |

128 -| penalty | Indicates whether to apply penalty to update_every in case of failures. | yes | no |

130 -| url | URL to the IPFS API | no | yes |

131 -| repoapi | Collect repo metrics. | no | no |

132 -| pinapi | Set status of IPFS pinned object polling. | no | no |

133 -

134 -</details>

135 -

136 -#### Examples

137 -

138 -##### Basic (default out-of-the-box)

139 -

140 -A basic example configuration, one job will run at a time. Autodetect mechanism uses it by default.

141 -

142 -```yaml

143 -localhost:

144 - name: 'local'

145 - url: 'http://localhost:5001'

146 - repoapi: no

147 - pinapi: no

148 -

149 -```

150 -##### Multi-instance

151 -

152 -> **Note**: When you define multiple jobs, their names must be unique.

153 -

154 -Collecting metrics from local and remote instances.

155 -

156 -

157 -<details open><summary>Config</summary>

158 -

159 -```yaml

160 -localhost:

161 - name: 'local'

162 - url: 'http://localhost:5001'

163 - repoapi: no

164 - pinapi: no

165 -

166 -remote_host:

167 - name: 'remote'

168 - url: 'http://192.0.2.1:5001'

169 - repoapi: no

170 - pinapi: no

171 -

172 -```

173 -</details>

174 -

175 -

176 -

177 -## Troubleshooting

178 -

179 -### Debug Mode

180 -

181 -To troubleshoot issues with the `ipfs` collector, run the `python.d.plugin` with the debug option enabled. The output

182 -should give you clues as to why the collector isn't working.

183 -

184 -- Navigate to the `plugins.d` directory, usually at `/usr/libexec/netdata/plugins.d/`. If that's not the case on

185 - your system, open `netdata.conf` and look for the `plugins` setting under `[directories]`.

186 -

187 - ```bash

188 - cd /usr/libexec/netdata/plugins.d/

189 - ```

190 -

191 -- Switch to the `netdata` user.

192 -

193 - ```bash

194 - sudo -u netdata -s

195 - ```

196 -

197 -- Run the `python.d.plugin` to debug the collector:

198 -

199 - ```bash

200 - ./python.d.plugin ipfs debug trace

201 - ```

202 -

203 -### Getting Logs

204 -

205 -If you're encountering problems with the `ipfs` collector, follow these steps to retrieve logs and identify potential issues:

206 -

207 -- **Run the command** specific to your system (systemd, non-systemd, or Docker container).

208 -- **Examine the output** for any warnings or error messages that might indicate issues. These messages should provide clues about the root cause of the problem.

209 -

210 -#### System with systemd

211 -

212 -Use the following command to view logs generated since the last Netdata service restart:

213 -

214 -```bash

215 -journalctl _SYSTEMD_INVOCATION_ID="$(systemctl show --value --property=InvocationID netdata)" --namespace=netdata --grep ipfs

216 -```

217 -

218 -#### System without systemd

219 -

220 -Locate the collector log file, typically at `/var/log/netdata/collector.log`, and use `grep` to filter for collector's name:

221 -

222 -```bash

223 -grep ipfs /var/log/netdata/collector.log

224 -```

225 -

226 -**Note**: This method shows logs from all restarts. Focus on the **latest entries** for troubleshooting current issues.

227 -

228 -#### Docker Container

229 -

230 -If your Netdata runs in a Docker container named "netdata" (replace if different), use this command:

231 -

232 -```bash

233 -docker logs netdata 2>&1 | grep ipfs

234 -```

235 -

236 -

src/collectors/python.d.plugin/ipfs/ipfs.chart.py deleted

-149

@@ -1,149 +0,0 @@

1 -# -*- coding: utf-8 -*-

2 -# Description: IPFS netdata python.d module

3 -# Authors: davidak

4 -# SPDX-License-Identifier: GPL-3.0-or-later

5 -

6 -import json

7 -

8 -from bases.FrameworkServices.UrlService import UrlService

9 -

10 -ORDER = [

11 - 'bandwidth',

12 - 'peers',

13 - 'repo_size',

14 - 'repo_objects',

15 -]

16 -

17 -CHARTS = {

18 - 'bandwidth': {

19 - 'options': [None, 'IPFS Bandwidth', 'kilobits/s', 'Bandwidth', 'ipfs.bandwidth', 'line'],

20 - 'lines': [

21 - ['in', None, 'absolute', 8, 1000],

22 - ['out', None, 'absolute', -8, 1000]

23 - ]

24 - },

25 - 'peers': {

26 - 'options': [None, 'IPFS Peers', 'peers', 'Peers', 'ipfs.peers', 'line'],

27 - 'lines': [

28 - ['peers', None, 'absolute']

29 - ]

30 - },

31 - 'repo_size': {

32 - 'options': [None, 'IPFS Repo Size', 'GiB', 'Size', 'ipfs.repo_size', 'area'],

33 - 'lines': [

34 - ['avail', None, 'absolute', 1, 1 << 30],

35 - ['size', None, 'absolute', 1, 1 << 30],

36 - ]

37 - },

38 - 'repo_objects': {

39 - 'options': [None, 'IPFS Repo Objects', 'objects', 'Objects', 'ipfs.repo_objects', 'line'],

40 - 'lines': [

41 - ['objects', None, 'absolute', 1, 1],

42 - ['pinned', None, 'absolute', 1, 1],

43 - ['recursive_pins', None, 'absolute', 1, 1]

44 - ]

45 - }

46 -}

47 -

48 -SI_zeroes = {

49 - 'k': 3,

50 - 'm': 6,

51 - 'g': 9,

52 - 't': 12,

53 - 'p': 15,

54 - 'e': 18,

55 - 'z': 21,

56 - 'y': 24

57 -}

58 -

59 -

60 -class Service(UrlService):

61 - def __init__(self, configuration=None, name=None):

62 - UrlService.__init__(self, configuration=configuration, name=name)

63 - self.order = ORDER

64 - self.definitions = CHARTS

65 - self.baseurl = self.configuration.get('url', 'http://localhost:5001')

66 - self.method = "POST"

67 - self.do_pinapi = self.configuration.get('pinapi')

68 - self.do_repoapi = self.configuration.get('repoapi')

69 - self.__storage_max = None

70 -

71 - def _get_json(self, sub_url):

72 - """

73 - :return: json decoding of the specified url

74 - """

75 - self.url = self.baseurl + sub_url

76 - try:

77 - return json.loads(self._get_raw_data())

78 - except (TypeError, ValueError):

79 - return dict()

80 -

81 - @staticmethod

82 - def _recursive_pins(keys):

83 - return sum(1 for k in keys if keys[k]['Type'] == b'recursive')

84 -

85 - @staticmethod

86 - def _dehumanize(store_max):

87 - # convert from '10Gb' to 10000000000

88 - if not isinstance(store_max, int):

89 - store_max = store_max.lower()

90 - if store_max.endswith('b'):

91 - val, units = store_max[:-2], store_max[-2]

92 - if units in SI_zeroes:

93 - val += '0' * SI_zeroes[units]

94 - store_max = val

95 - try:

96 - store_max = int(store_max)

97 - except (TypeError, ValueError):

98 - store_max = None

99 - return store_max

100 -

101 - def _storagemax(self, store_cfg):

102 - if self.__storage_max is None:

103 - self.__storage_max = self._dehumanize(store_cfg)

104 - return self.__storage_max

105 -

106 - def _get_data(self):

107 - """

108 - Get data from API

109 - :return: dict

110 - """

111 - # suburl : List of (result-key, original-key, transform-func)

112 - cfg = {

113 - '/api/v0/stats/bw':

114 - [

115 - ('in', 'RateIn', int),

116 - ('out', 'RateOut', int),

117 - ],

118 - '/api/v0/swarm/peers':

119 - [

120 - ('peers', 'Peers', len),

121 - ],

122 - }

123 - if self.do_repoapi:

124 - cfg.update({

125 - '/api/v0/stats/repo':

126 - [

127 - ('size', 'RepoSize', int),

128 - ('objects', 'NumObjects', int),

129 - ('avail', 'StorageMax', self._storagemax),

130 - ],

131 - })

132 -

133 - if self.do_pinapi:

134 - cfg.update({

135 - '/api/v0/pin/ls':

136 - [

137 - ('pinned', 'Keys', len),

138 - ('recursive_pins', 'Keys', self._recursive_pins),

139 - ]

140 - })

141 - r = dict()

142 - for suburl in cfg:

143 - in_json = self._get_json(suburl)

144 - for new_key, orig_key, xmute in cfg[suburl]:

145 - try:

146 - r[new_key] = xmute(in_json[orig_key])

147 - except Exception as error:

148 - self.debug(error)

149 - return r or None

src/collectors/python.d.plugin/ipfs/ipfs.conf deleted

-82

@@ -1,82 +0,0 @@

1 -# netdata python.d.plugin configuration for ipfs

2 -#

3 -# This file is in YaML format. Generally the format is:

4 -#

5 -# name: value

6 -#

7 -# There are 2 sections:

8 -# - global variables

9 -# - one or more JOBS

10 -#

11 -# JOBS allow you to collect values from multiple sources.

12 -# Each source will have its own set of charts.

13 -#

14 -# JOB parameters have to be indented (using spaces only, example below).

15 -

16 -# ----------------------------------------------------------------------

17 -# Global Variables

18 -# These variables set the defaults for all JOBs, however each JOB

19 -# may define its own, overriding the defaults.

20 -

21 -# update_every sets the default data collection frequency.

22 -# If unset, the python.d.plugin default is used.

23 -# update_every: 1

24 -

25 -# priority controls the order of charts at the netdata dashboard.

26 -# Lower numbers move the charts towards the top of the page.

27 -# If unset, the default for python.d.plugin is used.

28 -# priority: 60000

29 -

30 -# penalty indicates whether to apply penalty to update_every in case of failures.

31 -# Penalty will increase every 5 failed updates in a row. Maximum penalty is 10 minutes.

32 -# penalty: yes

33 -

34 -# autodetection_retry sets the job re-check interval in seconds.

35 -# The job is not deleted if check fails.

36 -# Attempts to start the job are made once every autodetection_retry.

37 -# This feature is disabled by default.

38 -# autodetection_retry: 0

39 -

40 -# ----------------------------------------------------------------------

41 -# JOBS (data collection sources)

42 -#

43 -# The default JOBS share the same *name*. JOBS with the same name

44 -# are mutually exclusive. Only one of them will be allowed running at

45 -# any time. This allows autodetection to try several alternatives and

46 -# pick the one that works.

47 -#

48 -# Any number of jobs is supported.

49 -#

50 -# All python.d.plugin JOBS (for all its modules) support a set of

51 -# predefined parameters. These are:

52 -#

53 -# job_name:

54 -# name: myname # the JOB's name as it will appear at the

55 -# # dashboard (by default is the job_name)

56 -# # JOBs sharing a name are mutually exclusive

57 -# update_every: 1 # the JOB's data collection frequency

58 -# priority: 60000 # the JOB's order on the dashboard

59 -# penalty: yes # the JOB's penalty

60 -# autodetection_retry: 0 # the JOB's re-check interval in seconds

61 -#

62 -# Additionally to the above, ipfs also supports the following:

63 -#

64 -# url: 'URL' # URL to the IPFS API

65 -# repoapi: no # Collect repo metrics

66 -# # Currently defaults to disabled due to IPFS Bug

67 -# # https://github.com/ipfs/go-ipfs/issues/7528

68 -# # resulting in very high CPU Usage

69 -# pinapi: no # Set status of IPFS pinned object polling

70 -# # Currently defaults to disabled due to IPFS Bug

71 -# # https://github.com/ipfs/go-ipfs/issues/3874

72 -# # resulting in very high CPU Usage

73 -#

74 -# ----------------------------------------------------------------------

75 -# AUTO-DETECTION JOBS

76 -# only one of them will run (they have the same name)

77 -

78 -localhost:

79 - name: 'local'

80 - url: 'http://localhost:5001'

81 - repoapi: no

82 - pinapi: no

src/collectors/python.d.plugin/ipfs/metadata.yaml deleted

-172

@@ -1,172 +0,0 @@

1 -plugin_name: python.d.plugin

2 -modules:

3 - - meta:

4 - plugin_name: python.d.plugin

5 - module_name: ipfs

6 - monitored_instance:

7 - name: IPFS

8 - link: "https://ipfs.tech/"

9 - categories:

10 - - data-collection.storage-mount-points-and-filesystems

11 - icon_filename: "ipfs.svg"

12 - related_resources:

13 - integrations:

14 - list: []

15 - info_provided_to_referring_integrations:

16 - description: ""

17 - keywords: []

18 - most_popular: false

19 - overview:

20 - data_collection:

21 - metrics_description: "This collector monitors IPFS server metrics about its quality and performance."

22 - method_description: "It connects to an http endpoint of the IPFS server to collect the metrics"

23 - supported_platforms:

24 - include: []

25 - exclude: []

26 - multi_instance: true

27 - additional_permissions:

28 - description: ""

29 - default_behavior:

30 - auto_detection:

31 - description: "If the endpoint is accessible by the Agent, netdata will autodetect it"

32 - limits:

33 - description: |

34 - Calls to the following endpoints are disabled due to IPFS bugs:

35 -

36 - /api/v0/stats/repo (https://github.com/ipfs/go-ipfs/issues/3874)

37 - /api/v0/pin/ls (https://github.com/ipfs/go-ipfs/issues/7528)

38 - performance_impact:

39 - description: ""

40 - setup:

41 - prerequisites:

42 - list: []

43 - configuration:

44 - file:

45 - name: "python.d/ipfs.conf"

46 - options:

47 - description: |

48 - There are 2 sections:

49 -

50 - * Global variables

51 - * One or more JOBS that can define multiple different instances to monitor.

52 -

53 - The following options can be defined globally: priority, penalty, autodetection_retry, update_every, but can also be defined per JOB to override the global values.

54 -

55 - Additionally, the following collapsed table contains all the options that can be configured inside a JOB definition.

56 -

57 - Every configuration JOB starts with a `job_name` value which will appear in the dashboard, unless a `name` parameter is specified.

58 - folding:

59 - title: ""

60 - enabled: true

61 - list:

62 - - name: update_every

63 - description: Sets the default data collection frequency.

64 - default_value: 5

65 - required: false

66 - - name: priority

67 - description: Controls the order of charts at the netdata dashboard.

68 - default_value: 60000

69 - required: false

70 - - name: autodetection_retry

71 - description: Sets the job re-check interval in seconds.

72 - default_value: 0

73 - required: false

74 - - name: penalty

75 - description: Indicates whether to apply penalty to update_every in case of failures.

76 - default_value: yes

77 - required: false

78 - - name: name

79 - description: The JOB's name as it will appear at the dashboard (by default is the job_name)

80 - default_value: job_name

81 - required: false

82 - - name: url

83 - description: URL to the IPFS API

84 - default_value: no

85 - required: true

86 - - name: repoapi

87 - description: Collect repo metrics.

88 - default_value: no

89 - required: false

90 - - name: pinapi

91 - description: Set status of IPFS pinned object polling.

92 - default_value: no

93 - required: false

94 - examples:

95 - folding:

96 - enabled: true

97 - title: "Config"

98 - list:

99 - - name: Basic (default out-of-the-box)

100 - description: A basic example configuration, one job will run at a time. Autodetect mechanism uses it by default.

101 - folding:

102 - enabled: false

103 - config: |

104 - localhost:

105 - name: 'local'

106 - url: 'http://localhost:5001'

107 - repoapi: no

108 - pinapi: no

109 - - name: Multi-instance

110 - description: |

111 - > **Note**: When you define multiple jobs, their names must be unique.

112 -

113 - Collecting metrics from local and remote instances.

114 - config: |

115 - localhost:

116 - name: 'local'

117 - url: 'http://localhost:5001'

118 - repoapi: no

119 - pinapi: no

120 -

121 - remote_host:

122 - name: 'remote'

123 - url: 'http://192.0.2.1:5001'

124 - repoapi: no

125 - pinapi: no

126 - troubleshooting:

127 - problems:

128 - list: []

129 - alerts:

130 - - name: ipfs_datastore_usage

131 - link: https://github.com/netdata/netdata/blob/master/src/health/health.d/ipfs.conf

132 - metric: ipfs.repo_size

133 - info: IPFS datastore utilization

134 - metrics:

135 - folding:

136 - title: Metrics

137 - enabled: false

138 - description: ""

139 - availability: []

140 - scopes:

141 - - name: global

142 - description: "These metrics refer to the entire monitored application."

143 - labels: []

144 - metrics:

145 - - name: ipfs.bandwidth

146 - description: IPFS Bandwidth

147 - unit: "kilobits/s"

148 - chart_type: line

149 - dimensions:

150 - - name: in

151 - - name: out

152 - - name: ipfs.peers

153 - description: IPFS Peers

154 - unit: "peers"

155 - chart_type: line

156 - dimensions:

157 - - name: peers

158 - - name: ipfs.repo_size

159 - description: IPFS Repo Size

160 - unit: "GiB"

161 - chart_type: area

162 - dimensions:

163 - - name: avail

164 - - name: size

165 - - name: ipfs.repo_objects

166 - description: IPFS Repo Objects

167 - unit: "objects"

168 - chart_type: line

169 - dimensions:

170 - - name: objects

171 - - name: pinned

172 - - name: recursive_pins

src/collectors/python.d.plugin/python.d.conf

+1 -1

@@ -40,7 +40,6 @@ example: no

40 go_expvar: no

41 # haproxy: yes

42 # icecast: yes

43 -# ipfs: yes

43 # memcached: yes

44 # monit: yes

45 # nvidia_smi: yes

@@ -73,6 +72,7 @@ fail2ban: no # Removed (replaced with go.d/fail2ban).

72 freeradius: no # Removed (replaced with go.d/freeradius).

73 hddtemp: no # Removed (replaced with go.d/hddtemp).

74 hpssa: no # Removed (replaced with go.d/hpssa).

75 +ipfs: no # Removed (replaced with go.d/ipfs).

76 litespeed: no # Removed (replaced with go.d/litespeed).

77 megacli: no # Removed (replaced with go.d/megacli).

78 mongodb: no # Removed (replaced with go.d/mongodb).

src/go/plugin/go.d/README.md

@@ -82,6 +82,7 @@ see the appropriate collector readme.

82 | [hpssa](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/hpssa) | HPE Smart Array |

83 | [httpcheck](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/httpcheck) | Any HTTP Endpoint |

84 | [intelgpu](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/intelgpu) | Intel integrated GPU |

85 +| [ipfs](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/ipfs) | IPFS |

86 | [isc_dhcpd](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/isc_dhcpd) | ISC DHCP |

87 | [k8s_kubelet](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/k8s_kubelet) | Kubelet |

88 | [k8s_kubeproxy](https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/k8s_kubeproxy) | Kube-proxy |

src/go/plugin/go.d/config/go.d.conf

@@ -47,6 +47,7 @@ modules:

47 # hpssa: yes

48 # httpcheck: yes

49 # intelgpu: yes

50 +# ipfs: yes

51 # isc_dhcpd: yes

52 # k8s_kubelet: yes

53 # k8s_kubeproxy: yes

src/go/plugin/go.d/config/go.d/ipfs.conf new

@@ -0,0 +1,6 @@

1 +## All available configuration options, their descriptions and default values:

2 +## https://github.com/netdata/netdata/tree/master/src/go/plugin/go.d/modules/ipfs#readme

3 +

4 +#jobs:

5 +# - name: local

6 +# url: http://localhost:5001

src/go/plugin/go.d/config/go.d/sd/docker.conf

@@ -38,6 +38,8 @@ classify:

38 expr: '{{ or (eq .PrivatePort "5984") (match "sp" .Image "couchdb couchdb:*") }}'

39 - tags: "elasticsearch"

40 expr: '{{ or (eq .PrivatePort "9200") (match "sp" .Image "elasticsearch elasticsearch:* */elasticsearch */elasticsearch:* */opensearch */opensearch:*") }}'

41 + - tags: "ipfs"

42 + expr: '{{ and (eq .PrivatePort "5001") (match "sp" .Image "ipfs/kubo ipfs/kubo:*") }}'

43 - tags: "lighttpd"

44 expr: '{{ match "sp" .Image "*/lighttpd */lighttpd:*" }}'

45 - tags: "mongodb"

@@ -115,6 +117,11 @@ compose:

117 username: admin

118 password: admin

119 {{ end -}}

120 + - selector: "ipfs"

121 + template: |

122 + module: ipfs

123 + name: docker_{{.Name}}

124 + url: http://{{.Address}}

125 - selector: "lighttpd"

126 template: |

127 module: lighttpd

src/go/plugin/go.d/config/go.d/sd/net_listeners.conf

@@ -58,6 +58,8 @@ classify:

58 expr: '{{ and (eq .Port "9870") (eq .Comm "hadoop") }}'

59 - tags: "hdfs_datanode"

60 expr: '{{ and (eq .Port "9864") (eq .Comm "hadoop") }}'

61 + - tags: "ipfs"

62 + expr: '{{ and (eq .Port "5001") (eq .Comm "ipfs") }}'

63 - tags: "kubelet"

64 expr: '{{ and (eq .Port "10250" "10255") (eq .Comm "kubelet") }}'

65 - tags: "kubeproxy"

@@ -252,6 +254,11 @@ compose:

254 module: hdfs

255 name: datanode_local

256 url: http://{{.Address}}/jmx

257 + - selector: "ipfs"

258 + template: |

259 + module: ipfs

260 + name: local

261 + url: http://{{.Address}}

262 - selector: "kubelet"

263 template: |

264 module: k8s_kubelet

src/go/plugin/go.d/modules/init.go

@@ -37,6 +37,7 @@ import (

37 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/modules/hpssa"

38 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/modules/httpcheck"

39 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/modules/intelgpu"

40 + _ "github.com/netdata/netdata/go/plugins/plugin/go.d/modules/ipfs"

41 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/modules/isc_dhcpd"

42 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/modules/k8s_kubelet"

43 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/modules/k8s_kubeproxy"

src/go/plugin/go.d/modules/ipfs/charts.go new

+105

@@ -0,0 +1,105 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +package ipfs

4 +

5 +import (

6 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"

7 +)

8 +

9 +const (

10 + prioBandwidth = module.Priority + iota

11 + prioSwarmPeers

12 + prioDatastoreSpaceUtilization

13 + prioRepoSize

14 + prioRepoObj

15 + prioRepoPinnedObj

16 +)

17 +

18 +var charts = module.Charts{

19 + bandwidthChart.Copy(),

20 + peersChart.Copy(),

21 + datastoreUtilizationChart.Copy(),

22 + repoSizeChart.Copy(),

23 + repoObjChart.Copy(),

24 + repoPinnedObjChart.Copy(),

25 +}

26 +

27 +var (

28 + bandwidthChart = module.Chart{

29 + ID: "bandwidth",

30 + Title: "IPFS Bandwidth",

31 + Units: "bytes/s",

32 + Fam: "bandwidth",

33 + Ctx: "ipfs.bandwidth",

34 + Type: module.Area,

35 + Priority: prioBandwidth,

36 + Dims: module.Dims{

37 + {ID: "in", Algo: module.Incremental},

38 + {ID: "out", Mul: -1, Algo: module.Incremental},

39 + },

40 + }

41 +

42 + peersChart = module.Chart{

43 + ID: "peers",

44 + Title: "IPFS Peers",

45 + Units: "peers",

46 + Fam: "peers",

47 + Ctx: "ipfs.peers",

48 + Type: module.Line,

49 + Priority: prioSwarmPeers,

50 + Dims: module.Dims{

51 + {ID: "peers"},

52 + },

53 + }

54 +

55 + datastoreUtilizationChart = module.Chart{

56 + ID: "datastore_space_utilization",

57 + Title: "IPFS Datastore Space Utilization",

58 + Units: "percent",

59 + Fam: "size",

60 + Ctx: "ipfs.datastore_space_utilization",

61 + Type: module.Line,

62 + Priority: prioDatastoreSpaceUtilization,

63 + Dims: module.Dims{

64 + {ID: "used_percent", Name: "used"},

65 + },

66 + }

67 + repoSizeChart = module.Chart{

68 + ID: "repo_size",

69 + Title: "IPFS Repo Size",

70 + Units: "bytes",

71 + Fam: "size",

72 + Ctx: "ipfs.repo_size",

73 + Type: module.Line,

74 + Priority: prioRepoSize,

75 + Dims: module.Dims{

76 + {ID: "size"},

77 + },

78 + }

79 +

80 + repoObjChart = module.Chart{

81 + ID: "repo_objects",

82 + Title: "IPFS Repo Objects",

83 + Units: "objects",

84 + Fam: "objects",

85 + Ctx: "ipfs.repo_objects",

86 + Type: module.Line,

87 + Priority: prioRepoObj,

88 + Dims: module.Dims{

89 + {ID: "objects"},

90 + },

91 + }

92 + repoPinnedObjChart = module.Chart{

93 + ID: "repo_pinned_objects",

94 + Title: "IPFS Repo Pinned Objects",

95 + Units: "objects",

96 + Fam: "objects",

97 + Ctx: "ipfs.repo_pinned_objects",

98 + Type: module.Line,

99 + Priority: prioRepoPinnedObj,

100 + Dims: module.Dims{

101 + {ID: "pinned"},

102 + {ID: "recursive_pins"},

103 + },

104 + }

105 +)

src/go/plugin/go.d/modules/ipfs/collect.go new

+217

@@ -0,0 +1,217 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +package ipfs

4 +

5 +import (

6 + "encoding/json"

7 + "fmt"

8 + "io"

9 + "net/http"

10 +

11 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/web"

12 +)

13 +

14 +type (

15 + ipfsStatsBw struct {

16 + TotalIn int64 `json:"TotalIn"`

17 + TotalOut int64 `json:"TotalOut"`

18 + RateIn *float64 `json:"RateIn"`

19 + RateOut *float64 `json:"RateOut"`

20 + }

21 + ipfsStatsRepo struct {

22 + RepoSize int64 `json:"RepoSize"`

23 + StorageMax int64 `json:"StorageMax"`

24 + NumObjects int64 `json:"NumObjects"`

25 + }

26 + ipfsSwarmPeers struct {

27 + Peers []any `json:"Peers"`

28 + }

29 + ipfsPinsLs struct {

30 + Keys map[string]struct {

31 + Type string `json:"type"`

32 + } `json:"Keys"`

33 + }

34 +)

35 +

36 +const (

37 + urlPathStatsBandwidth = "/api/v0/stats/bw" // https://docs.ipfs.tech/reference/kubo/rpc/#api-v0-stats-bw

38 + urlPathStatsRepo = "/api/v0/stats/repo" // https://docs.ipfs.tech/reference/kubo/rpc/#api-v0-stats-repo

39 + urlPathSwarmPeers = "/api/v0/swarm/peers" // https://docs.ipfs.tech/reference/kubo/rpc/#api-v0-swarm-peers

40 + urlPathPinLs = "/api/v0/pin/ls" // https://docs.ipfs.tech/reference/kubo/rpc/#api-v0-pin-ls

41 +)

42 +

43 +func (ip *IPFS) collect() (map[string]int64, error) {

44 + mx := make(map[string]int64)

45 +

46 + if err := ip.collectStatsBandwidth(mx); err != nil {

47 + return nil, err

48 + }

49 + if err := ip.collectSwarmPeers(mx); err != nil {

50 + return nil, err

51 + }

52 + if ip.QueryRepoApi {

53 + // https://github.com/netdata/netdata/pull/9687

54 + // TODO: collect by default with "size-only"

55 + // https://github.com/ipfs/kubo/issues/7528#issuecomment-657398332

56 + if err := ip.collectStatsRepo(mx); err != nil {

57 + return nil, err

58 + }

59 + }

60 + if ip.QueryPinApi {

61 + if err := ip.collectPinLs(mx); err != nil {

62 + return nil, err

63 + }

64 + }

65 +

66 + return mx, nil

67 +}

68 +

69 +func (ip *IPFS) collectStatsBandwidth(mx map[string]int64) error {

70 + stats, err := ip.queryStatsBandwidth()

71 + if err != nil {

72 + return err

73 + }

74 +

75 + mx["in"] = stats.TotalIn

76 + mx["out"] = stats.TotalOut

77 +

78 + return nil

79 +}

80 +

81 +func (ip *IPFS) collectSwarmPeers(mx map[string]int64) error {

82 + stats, err := ip.querySwarmPeers()

83 + if err != nil {

84 + return err

85 + }

86 +

87 + mx["peers"] = int64(len(stats.Peers))

88 +

89 + return nil

90 +}

91 +

92 +func (ip *IPFS) collectStatsRepo(mx map[string]int64) error {

93 + stats, err := ip.queryStatsRepo()

94 + if err != nil {

95 + return err

96 + }

97 +

98 + mx["used_percent"] = 0

99 + if stats.StorageMax > 0 {

100 + mx["used_percent"] = stats.RepoSize * 100 / stats.StorageMax

101 + }

102 + mx["size"] = stats.RepoSize

103 + mx["objects"] = stats.NumObjects

104 +

105 + return nil

106 +}

107 +

108 +func (ip *IPFS) collectPinLs(mx map[string]int64) error {

109 + stats, err := ip.queryPinLs()

110 + if err != nil {

111 + return err

112 + }

113 +

114 + var n int64

115 + for _, v := range stats.Keys {

116 + if v.Type == "recursive" {

117 + n++

118 + }

119 + }

120 +

121 + mx["pinned"] = int64(len(stats.Keys))

122 + mx["recursive_pins"] = n

123 +

124 + return nil

125 +}

126 +

127 +func (ip *IPFS) queryStatsBandwidth() (*ipfsStatsBw, error) {

128 + req, err := web.NewHTTPRequest(ip.Request)

129 + if err != nil {

130 + return nil, err

131 + }

132 +

133 + req.URL.Path = urlPathStatsBandwidth

134 +

135 + var stats ipfsStatsBw

136 + if err := ip.doOKDecode(req, &stats); err != nil {

137 + return nil, err

138 + }

139 +

140 + if stats.RateIn == nil || stats.RateOut == nil {

141 + return nil, fmt.Errorf("unexpected response: not ipfs data")

142 + }

143 +

144 + return &stats, nil

145 +}

146 +

147 +func (ip *IPFS) querySwarmPeers() (*ipfsSwarmPeers, error) {

148 + req, err := web.NewHTTPRequest(ip.Request)

149 + if err != nil {

150 + return nil, err

151 + }

152 +

153 + req.URL.Path = urlPathSwarmPeers

154 +

155 + var stats ipfsSwarmPeers

156 + if err := ip.doOKDecode(req, &stats); err != nil {

157 + return nil, err

158 + }

159 +

160 + return &stats, nil

161 +}

162 +

163 +func (ip *IPFS) queryStatsRepo() (*ipfsStatsRepo, error) {

164 + req, err := web.NewHTTPRequest(ip.Request)

165 + if err != nil {

166 + return nil, err

167 + }

168 +

169 + req.URL.Path = urlPathStatsRepo

170 +

171 + var stats ipfsStatsRepo

172 + if err := ip.doOKDecode(req, &stats); err != nil {

173 + return nil, err

174 + }

175 +

176 + return &stats, nil

177 +}

178 +

179 +func (ip *IPFS) queryPinLs() (*ipfsPinsLs, error) {

180 + req, err := web.NewHTTPRequest(ip.Request)

181 + if err != nil {

182 + return nil, err

183 + }

184 +

185 + req.URL.Path = urlPathPinLs

186 +

187 + var stats ipfsPinsLs

188 + if err := ip.doOKDecode(req, &stats); err != nil {

189 + return nil, err

190 + }

191 +

192 + return &stats, nil

193 +}

194 +

195 +func (ip *IPFS) doOKDecode(req *http.Request, in interface{}) error {

196 + resp, err := ip.httpClient.Do(req)

197 + if err != nil {

198 + return fmt.Errorf("error on HTTP request '%s': %v", req.URL, err)

199 + }

200 + defer closeBody(resp)

201 +

202 + if resp.StatusCode != http.StatusOK {

203 + return fmt.Errorf("'%s' returned HTTP status code: %d", req.URL, resp.StatusCode)

204 + }

205 +

206 + if err := json.NewDecoder(resp.Body).Decode(in); err != nil {

207 + return fmt.Errorf("error on decoding response from '%s': %v", req.URL, err)

208 + }

209 + return nil

210 +}

211 +

212 +func closeBody(resp *http.Response) {

213 + if resp != nil && resp.Body != nil {

214 + _, _ = io.Copy(io.Discard, resp.Body)

215 + _ = resp.Body.Close()

216 + }

217 +}

src/go/plugin/go.d/modules/ipfs/config_schema.json new

+195

@@ -0,0 +1,195 @@

1 +{

2 + "jsonSchema": {

3 + "$schema": "http://json-schema.org/draft-07/schema#",

4 + "title": "IPFS collector configuration.",

5 + "type": "object",

6 + "properties": {

7 + "update_every": {

8 + "title": "Update every",

9 + "description": "Data collection interval, measured in seconds.",

10 + "type": "integer",

11 + "minimum": 1,

12 + "default": 1

13 + },

14 + "url": {

15 + "title": "URL",

16 + "description": "The base URL where the IPFS API can be accessed.",

17 + "type": "string",

18 + "default": "http://127.0.0.1:5001",

19 + "format": "uri"

20 + },

21 + "timeout": {

22 + "title": "Timeout",

23 + "description": "The timeout in seconds for the HTTP request.",

24 + "type": "number",

25 + "minimum": 0.5,

26 + "default": 1

27 + },

28 + "not_follow_redirects": {

29 + "title": "Not follow redirects",

30 + "description": "If set, the client will not follow HTTP redirects automatically.",

31 + "type": "boolean"

32 + },

33 + "repoapi": {

34 + "title": "Query Repo API",

35 + "description": "Enables querying the [/api/v0/stats/repo](https://docs.ipfs.tech/reference/kubo/rpc/#api-v0-repo-stat) endpoint for repository statistics. **Disabled by default** due to potential high CPU usage.",

36 + "type": "boolean"

37 + },

38 + "pinapi": {

39 + "title": "Query Pin API",

40 + "description": "Enables querying the [/api/v0/pin/ls](https://docs.ipfs.tech/reference/kubo/rpc/#api-v0-pin-ls) endpoint to retrieve a list of all pinned objects. **Consider enabling only if necessary**.",

41 + "type": "boolean"

42 + },

43 + "username": {

44 + "title": "Username",

45 + "description": "The username for basic authentication.",

46 + "type": "string",

47 + "sensitive": true

48 + },

49 + "password": {

50 + "title": "Password",

51 + "description": "The password for basic authentication.",

52 + "type": "string",

53 + "sensitive": true

54 + },

55 + "proxy_url": {

56 + "title": "Proxy URL",

57 + "description": "The URL of the proxy server.",

58 + "type": "string"

59 + },

60 + "proxy_username": {

61 + "title": "Proxy username",

62 + "description": "The username for proxy authentication.",

63 + "type": "string",

64 + "sensitive": true

65 + },

66 + "proxy_password": {

67 + "title": "Proxy password",

68 + "description": "The password for proxy authentication.",

69 + "type": "string",

70 + "sensitive": true

71 + },

72 + "headers": {

73 + "title": "Headers",

74 + "description": "Additional HTTP headers to include in the request.",

75 + "type": [

76 + "object",

77 + "null"

78 + ],

79 + "additionalProperties": {

80 + "type": "string"

81 + }

82 + },

83 + "tls_skip_verify": {

84 + "title": "Skip TLS verification",

85 + "description": "If set, TLS certificate verification will be skipped.",

86 + "type": "boolean"

87 + },

88 + "tls_ca": {

89 + "title": "TLS CA",

90 + "description": "The path to the CA certificate file for TLS verification.",

91 + "type": "string",

92 + "pattern": "^$|^/"

93 + },

94 + "tls_cert": {

95 + "title": "TLS certificate",

96 + "description": "The path to the client certificate file for TLS authentication.",

97 + "type": "string",

98 + "pattern": "^$|^/"

99 + },

100 + "tls_key": {

101 + "title": "TLS key",

102 + "description": "The path to the client key file for TLS authentication.",

103 + "type": "string",

104 + "pattern": "^$|^/"

105 + },

106 + "body": {

107 + "title": "Body",

108 + "type": "string"

109 + },

110 + "method": {

111 + "title": "Method",

112 + "type": "string"

113 + }

114 + },

115 + "required": [

116 + "url"

117 + ],

118 + "additionalProperties": false,

119 + "patternProperties": {

120 + "^name$": {}

121 + }

122 + },

123 + "uiSchema": {

124 + "uiOptions": {

125 + "fullPage": true

126 + },

127 + "body": {

128 + "ui:widget": "hidden"

129 + },

130 + "method": {

131 + "ui:widget": "hidden"

132 + },

133 + "timeout": {

134 + "ui:help": "Accepts decimals for precise control (e.g., type 1.5 for 1.5 seconds)."

135 + },

136 + "repoapi": {

137 + "ui:help": "This endpoint retrieves the number of objects in the local repository, which is not cached and can be computationally expensive for IPFS to calculate, especially with frequent collection intervals. See [#7528](https://github.com/ipfs/go-ipfs/issues/7528)."

138 + },

139 + "pinapi": {

140 + "ui:help": "Performance may decrease as the number of pinned objects grows, as the entire list needs to be retrieved. See [#3874](https://github.com/ipfs/go-ipfs/issues/3874)."

141 + },

142 + "password": {

143 + "ui:widget": "password"

144 + },

145 + "proxy_password": {

146 + "ui:widget": "password"

147 + },

148 + "ui:flavour": "tabs",

149 + "ui:options": {

150 + "tabs": [

151 + {

152 + "title": "Base",

153 + "fields": [

154 + "update_every",

155 + "url",

156 + "timeout",

157 + "repoapi",

158 + "pinapi",

159 + "not_follow_redirects"

160 + ]

161 + },

162 + {

163 + "title": "Auth",

164 + "fields": [

165 + "username",

166 + "password"

167 + ]

168 + },

169 + {

170 + "title": "TLS",

171 + "fields": [

172 + "tls_skip_verify",

173 + "tls_ca",

174 + "tls_cert",

175 + "tls_key"

176 + ]

177 + },

178 + {

179 + "title": "Proxy",

180 + "fields": [

181 + "proxy_url",

182 + "proxy_username",

183 + "proxy_password"

184 + ]

185 + },

186 + {

187 + "title": "Headers",

188 + "fields": [

189 + "headers"

190 + ]

191 + }

192 + ]

193 + }

194 + }

195 +}

src/go/plugin/go.d/modules/ipfs/ipfs.go new

+128

@@ -0,0 +1,128 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +package ipfs

4 +

5 +import (

6 + _ "embed"

7 + "errors"

8 + "net/http"

9 + "time"

10 +

11 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"

12 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/web"

13 +)

14 +

15 +//go:embed "config_schema.json"

16 +var configSchema string

17 +

18 +func init() {

19 + module.Register("ipfs", module.Creator{

20 + JobConfigSchema: configSchema,

21 + Create: func() module.Module { return New() },

22 + Config: func() any { return &Config{} },

23 + })

24 +}

25 +

26 +func New() *IPFS {

27 + return &IPFS{

28 + Config: Config{

29 + HTTP: web.HTTP{

30 + Request: web.Request{

31 + URL: "http://127.0.0.1:5001",

32 + Method: http.MethodPost,

33 + },

34 + Client: web.Client{

35 + Timeout: web.Duration(time.Second * 1),

36 + },

37 + },

38 + QueryRepoApi: false,

39 + QueryPinApi: false,

40 + },

41 + charts: charts.Copy(),

42 + }

43 +}

44 +

45 +type Config struct {

46 + UpdateEvery int `yaml:"update_every,omitempty" json:"update_every"`

47 + web.HTTP `yaml:",inline" json:""`

48 + QueryPinApi bool `yaml:"pinapi" json:"pinapi"`

49 + QueryRepoApi bool `yaml:"repoapi" json:"repoapi"`

50 +}

51 +

52 +type IPFS struct {

53 + module.Base

54 + Config `yaml:",inline" json:""`

55 +

56 + charts *module.Charts

57 +

58 + httpClient *http.Client

59 +}

60 +

61 +func (ip *IPFS) Configuration() any {

62 + return ip.Config

63 +}

64 +

65 +func (ip *IPFS) Init() error {

66 + if ip.URL == "" {

67 + ip.Error("URL not set")

68 + return errors.New("url not set")

69 + }

70 +

71 + client, err := web.NewHTTPClient(ip.Client)

72 + if err != nil {

73 + ip.Error(err)

74 + return err

75 + }

76 + ip.httpClient = client

77 +

78 + if !ip.QueryPinApi {

79 + _ = ip.Charts().Remove(repoPinnedObjChart.ID)

80 + }

81 + if !ip.QueryRepoApi {

82 + _ = ip.Charts().Remove(datastoreUtilizationChart.ID)

83 + _ = ip.Charts().Remove(repoSizeChart.ID)

84 + _ = ip.Charts().Remove(repoObjChart.ID)

85 + }

86 +

87 + ip.Debugf("using URL %s", ip.URL)

88 + ip.Debugf("using timeout: %s", ip.Timeout)

89 +

90 + return nil

91 +}

92 +

93 +func (ip *IPFS) Check() error {

94 + mx, err := ip.collect()

95 + if err != nil {

96 + ip.Error(err)

97 + return err

98 + }

99 +

100 + if len(mx) == 0 {

101 + return errors.New("no metrics collected")

102 + }

103 +

104 + return nil

105 +}

106 +

107 +func (ip *IPFS) Charts() *module.Charts {

108 + return ip.charts

109 +}

110 +

111 +func (ip *IPFS) Collect() map[string]int64 {

112 + mx, err := ip.collect()

113 + if err != nil {

114 + ip.Error(err)

115 + }

116 +

117 + if len(mx) == 0 {

118 + return nil

119 + }

120 +

121 + return mx

122 +}

123 +

124 +func (ip *IPFS) Cleanup() {

125 + if ip.httpClient != nil {

126 + ip.httpClient.CloseIdleConnections()

127 + }

128 +}

src/go/plugin/go.d/modules/ipfs/ipfs_test.go new

+279

@@ -0,0 +1,279 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +package ipfs

4 +

5 +import (

6 + "net/http"

7 + "net/http/httptest"

8 + "os"

9 + "testing"

10 +

11 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"

12 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/web"

13 +

14 + "github.com/stretchr/testify/assert"

15 + "github.com/stretchr/testify/require"

16 +)

17 +

18 +var (

19 + dataConfigJSON, _ = os.ReadFile("testdata/config.json")

20 + dataConfigYAML, _ = os.ReadFile("testdata/config.yaml")

21 +

22 + apiv0PinLsData, _ = os.ReadFile("testdata/api_v0_pin_ls.json")

23 + apiv0StatsBwData, _ = os.ReadFile("testdata/api_v0_stats_bw.json")

24 + apiv0StatsRepoData, _ = os.ReadFile("testdata/api_v0_stats_repo.json")

25 + apiv0SwarmPeersData, _ = os.ReadFile("testdata/api_v0_swarm_peers.json")

26 +)

27 +

28 +func Test_testDataIsValid(t *testing.T) {

29 + for name, data := range map[string][]byte{

30 + "dataConfigJSON": dataConfigJSON,

31 + "dataConfigYAML": dataConfigYAML,

32 + "apiv0PinLsData": apiv0PinLsData,

33 + "apiv0StatsBwData": apiv0StatsBwData,

34 + "apiv0StatsRepoData": apiv0StatsRepoData,

35 + "apiv0SwarmPeersData": apiv0SwarmPeersData,

36 + } {

37 + require.NotNil(t, data, name)

38 + }

39 +}

40 +

41 +func TestIPFS_ConfigurationSerialize(t *testing.T) {

42 + module.TestConfigurationSerialize(t, &IPFS{}, dataConfigJSON, dataConfigYAML)

43 +}

44 +

45 +func TestIPFS_Init(t *testing.T) {

46 + tests := map[string]struct {

47 + wantFail bool

48 + config Config

49 + }{

50 + "success with default": {

51 + wantFail: false,

52 + config: New().Config,

53 + },

54 + "fail when URL not set": {

55 + wantFail: true,

56 + config: Config{

57 + HTTP: web.HTTP{

58 + Request: web.Request{URL: ""},

59 + },

60 + },

61 + },

62 + }

63 +

64 + for name, test := range tests {

65 + t.Run(name, func(t *testing.T) {

66 + ipfs := New()

67 + ipfs.Config = test.config

68 +

69 + if test.wantFail {

70 + assert.Error(t, ipfs.Init())

71 + } else {

72 + assert.NoError(t, ipfs.Init())

73 + }

74 + })

75 + }

76 +}

77 +

78 +func TestIPFS_Charts(t *testing.T) {

79 + assert.NotNil(t, New().Charts())

80 +}

81 +

82 +func TestIPFS_Check(t *testing.T) {

83 + tests := map[string]struct {

84 + wantFail bool

85 + prepare func(t *testing.T) (*IPFS, func())

86 + }{

87 + "success default config": {

88 + wantFail: false,

89 + prepare: prepareCaseOkDefault,

90 + },

91 + "success all queries enabled": {

92 + wantFail: false,

93 + prepare: prepareCaseOkDefault,

94 + },

95 + "fails on unexpected json response": {

96 + wantFail: true,

97 + prepare: prepareCaseUnexpectedJsonResponse,

98 + },

99 + "fails on invalid format response": {

100 + wantFail: true,

101 + prepare: prepareCaseInvalidFormatResponse,

102 + },

103 + "fails on connection refused": {

104 + wantFail: true,

105 + prepare: prepareCaseConnectionRefused,

106 + },

107 + }

108 +

109 + for name, test := range tests {

110 + t.Run(name, func(t *testing.T) {

111 + ipfs, cleanup := test.prepare(t)

112 + defer cleanup()

113 +

114 + if test.wantFail {

115 + assert.Error(t, ipfs.Check())

116 + } else {

117 + assert.NoError(t, ipfs.Check())

118 + }

119 + })

120 + }

121 +}

122 +

123 +func TestIPFS_Collect(t *testing.T) {

124 + tests := map[string]struct {

125 + prepare func(t *testing.T) (*IPFS, func())

126 + wantMetrics map[string]int64

127 + }{

128 + "success default config": {

129 + prepare: prepareCaseOkDefault,

130 + wantMetrics: map[string]int64{

131 + "in": 20113594,

132 + "out": 3113852,

133 + "peers": 6,

134 + },

135 + },

136 + "success all queries enabled": {

137 + prepare: prepareCaseOkAllQueriesEnabled,

138 + wantMetrics: map[string]int64{

139 + "avail": 10000000000,

140 + "in": 20113594,

141 + "objects": 1,

142 + "out": 3113852,

143 + "peers": 6,

144 + "pinned": 1,

145 + "recursive_pins": 1,

146 + "size": 25495,

147 + "used_percent": 0,

148 + },

149 + },

150 + "fails on unexpected json response": {

151 + prepare: prepareCaseUnexpectedJsonResponse,

152 + },

153 + "fails on invalid format response": {

154 + prepare: prepareCaseInvalidFormatResponse,

155 + },

156 + "fails on connection refused": {

157 + prepare: prepareCaseConnectionRefused,

158 + },

159 + }

160 +

161 + for name, test := range tests {

162 + t.Run(name, func(t *testing.T) {

163 + ipfs, cleanup := test.prepare(t)

164 + defer cleanup()

165 +

166 + mx := ipfs.Collect()

167 +

168 + require.Equal(t, test.wantMetrics, mx)

169 + if len(test.wantMetrics) > 0 {

170 + testMetricsHasAllChartsDims(t, ipfs, mx)

171 + }

172 + })

173 + }

174 +}

175 +

176 +func testMetricsHasAllChartsDims(t *testing.T, ipfs *IPFS, mx map[string]int64) {

177 + for _, chart := range *ipfs.Charts() {

178 + if chart.Obsolete {

179 + continue

180 + }

181 + for _, dim := range chart.Dims {

182 + _, ok := mx[dim.ID]

183 + assert.Truef(t, ok, "collected metrics has no data for dim '%s' chart '%s'", dim.ID, chart.ID)

184 + }

185 + for _, v := range chart.Vars {

186 + _, ok := mx[v.ID]

187 + assert.Truef(t, ok, "collected metrics has no data for var '%s' chart '%s'", v.ID, chart.ID)

188 + }

189 + }

190 +}

191 +

192 +func prepareCaseOkDefault(t *testing.T) (*IPFS, func()) {

193 + t.Helper()

194 + srv := httptest.NewServer(http.HandlerFunc(

195 + func(w http.ResponseWriter, r *http.Request) {

196 + switch r.URL.Path {

197 + case urlPathStatsBandwidth:

198 + _, _ = w.Write(apiv0StatsBwData)

199 + case urlPathStatsRepo:

200 + _, _ = w.Write(apiv0StatsRepoData)

201 + case urlPathSwarmPeers:

202 + _, _ = w.Write(apiv0SwarmPeersData)

203 + case urlPathPinLs:

204 + _, _ = w.Write(apiv0PinLsData)

205 + default:

206 + w.WriteHeader(http.StatusNotFound)

207 + }

208 + }))

209 +

210 + ipfs := New()

211 + ipfs.URL = srv.URL

212 + require.NoError(t, ipfs.Init())

213 +

214 + return ipfs, srv.Close

215 +}

216 +

217 +func prepareCaseOkAllQueriesEnabled(t *testing.T) (*IPFS, func()) {

218 + t.Helper()

219 + ipfs, cleanup := prepareCaseOkDefault(t)

220 +

221 + ipfs.QueryRepoApi = true

222 + ipfs.QueryPinApi = true

223 +

224 + return ipfs, cleanup

225 +}

226 +

227 +func prepareCaseUnexpectedJsonResponse(t *testing.T) (*IPFS, func()) {

228 + t.Helper()

229 + resp := `

230 +{

231 + "elephant": {

232 + "burn": false,

233 + "mountain": true,

234 + "fog": false,

235 + "skin": -1561907625,

236 + "burst": "anyway",

237 + "shadow": 1558616893

238 + },

239 + "start": "ever",

240 + "base": 2093056027,

241 + "mission": -2007590351,

242 + "victory": 999053756,

243 + "die": false

244 +}

245 +`

246 + srv := httptest.NewServer(http.HandlerFunc(

247 + func(w http.ResponseWriter, r *http.Request) {

248 + _, _ = w.Write([]byte(resp))

249 + }))

250 +

251 + ipfs := New()

252 + ipfs.URL = srv.URL

253 + require.NoError(t, ipfs.Init())

254 +

255 + return ipfs, srv.Close

256 +}

257 +

258 +func prepareCaseInvalidFormatResponse(t *testing.T) (*IPFS, func()) {

259 + t.Helper()

260 + srv := httptest.NewServer(http.HandlerFunc(

261 + func(w http.ResponseWriter, r *http.Request) {

262 + _, _ = w.Write([]byte("hello and\n goodbye"))

263 + }))

264 +

265 + ipfs := New()

266 + ipfs.URL = srv.URL

267 + require.NoError(t, ipfs.Init())

268 +

269 + return ipfs, srv.Close

270 +}

271 +

272 +func prepareCaseConnectionRefused(t *testing.T) (*IPFS, func()) {

273 + t.Helper()

274 + ipfs := New()

275 + ipfs.URL = "http://127.0.0.1:65001"

276 + require.NoError(t, ipfs.Init())

277 +

278 + return ipfs, func() {}

279 +}

src/go/plugin/go.d/modules/ipfs/metadata.yaml new

+217

@@ -0,0 +1,217 @@

1 +plugin_name: go.d.plugin

2 +modules:

3 + - meta:

4 + id: collector-go.d.plugin-ipfs

5 + plugin_name: go.d.plugin

6 + module_name: ipfs

7 + monitored_instance:

8 + name: IPFS

9 + link: "https://ipfs.tech/"

10 + categories:

11 + - data-collection.storage-mount-points-and-filesystems

12 + icon_filename: "ipfs.svg"

13 + related_resources:

14 + integrations:

15 + list: []

16 + info_provided_to_referring_integrations:

17 + description: ""

18 + keywords:

19 + - ipfs

20 + - filesystem

21 + most_popular: false

22 + overview:

23 + data_collection:

24 + metrics_description: "This collector monitors IPFS daemon health and network activity."

25 + method_description: |

26 + It uses [RPC API](https://docs.ipfs.tech/reference/kubo/rpc/) to collect metrics.

27 +

28 + Used endpoints:

29 +

30 + - [/api/v0/stats/bw](https://docs.ipfs.tech/reference/kubo/rpc/#api-v0-stats-bw)

31 + - [/api/v0/swarm/peers](https://docs.ipfs.tech/reference/kubo/rpc/#api-v0-swarm-peers)

32 + - [/api/v0/stats/repo](https://docs.ipfs.tech/reference/kubo/rpc/#api-v0-stats-repo)

33 + - [/api/v0/pin/ls](https://docs.ipfs.tech/reference/kubo/rpc/#api-v0-pin-ls)

34 + supported_platforms:

35 + include: []

36 + exclude: []

37 + multi_instance: true

38 + additional_permissions:

39 + description: ""

40 + default_behavior:

41 + auto_detection:

42 + description: |

43 + By default, it detects IPFS instances running on localhost that are listening on port 5001.

44 + limits:

45 + description: ""

46 + performance_impact:

47 + description: |

48 + Calls to the following endpoints are disabled by default due to IPFS bugs:

49 +

50 + - /api/v0/stats/repo ([#7528](https://github.com/ipfs/go-ipfs/issues/7528)).

51 + - /api/v0/pin/ls ([#3874](https://github.com/ipfs/go-ipfs/issues/3874)).

52 +

53 + **Disabled by default** due to potential high CPU usage. Consider enabling only if necessary.

54 + setup:

55 + prerequisites:

56 + list: []

57 + configuration:

58 + file:

59 + name: go.d/ipfs.conf

60 + options:

61 + description: |

62 + The following options can be defined globally: update_every, autodetection_retry.

63 + folding:

64 + title: ""

65 + enabled: true

66 + list:

67 + - name: update_every

68 + description: Data collection frequency.

69 + default_value: 1

70 + required: false

71 + - name: autodetection_retry

72 + description: Recheck interval in seconds. Zero means no recheck will be scheduled.

73 + default_value: 0

74 + required: false

75 + - name: repoapi

76 + description: Enables querying the [/api/v0/stats/repo](https://docs.ipfs.tech/reference/kubo/rpc/#api-v0-repo-stat) endpoint for repository statistics.

77 + default_value: false

78 + required: false

79 + - name: pinapi

80 + description: Enables querying the [/api/v0/pin/ls](https://docs.ipfs.tech/reference/kubo/rpc/#api-v0-pin-ls) endpoint to retrieve a list of all pinned objects.

81 + default_value: false

82 + required: false

83 + - name: url

84 + description: Server URL.

85 + default_value: http://127.0.0.1:5001

86 + required: true

87 + - name: timeout

88 + description: HTTP request timeout.

89 + default_value: 1

90 + required: false

91 + - name: username

92 + description: Username for basic HTTP authentication.

93 + default_value: ""

94 + required: false

95 + - name: password

96 + description: Password for basic HTTP authentication.

97 + default_value: ""

98 + required: false

99 + - name: proxy_url

100 + description: Proxy URL.

101 + default_value: ""

102 + required: false

103 + - name: proxy_username

104 + description: Username for proxy basic HTTP authentication.

105 + default_value: ""

106 + required: false

107 + - name: proxy_password

108 + description: Password for proxy basic HTTP authentication.

109 + default_value: ""

110 + required: false

111 + - name: method

112 + description: HTTP request method.

113 + default_value: POST

114 + required: false

115 + - name: body

116 + description: HTTP request body.

117 + default_value: ""

118 + required: false

119 + - name: headers

120 + description: HTTP request headers.

121 + default_value: ""

122 + required: false

123 + - name: not_follow_redirects

124 + description: Redirect handling policy. Controls whether the client follows redirects.

125 + default_value: false

126 + required: false

127 + - name: tls_skip_verify

128 + description: Server certificate chain and hostname validation policy. Controls whether the client performs this check.

129 + default_value: false

130 + required: false

131 + - name: tls_ca

132 + description: Certification authority that the client uses when verifying the server's certificates.

133 + default_value: ""

134 + required: false

135 + - name: tls_cert

136 + description: Client TLS certificate.

137 + default_value: ""

138 + required: false

139 + - name: tls_key

140 + description: Client TLS key.

141 + default_value: ""

142 + required: false

143 + examples:

144 + folding:

145 + enabled: true

146 + title: "Config"

147 + list:

148 + - name: Basic

149 + description: A basic example configuration.

150 + folding:

151 + enabled: false

152 + config: |

153 + jobs:

154 + - name: local

155 + url: http://127.0.0.1:5001

156 + - name: Multi-instance

157 + description: |

158 + > **Note**: When you define multiple jobs, their names must be unique.

159 +

160 + Collecting metrics from local and remote instances.

161 + config: |

162 + jobs:

163 + - name: local

164 + url: http://127.0.0.1:5001

165 +

166 + - name: remote

167 + url: http://192.0.2.1:5001

168 + troubleshooting:

169 + problems:

170 + list: []

171 + alerts:

172 + - name: ipfs_datastore_usage

173 + link: https://github.com/netdata/netdata/blob/master/src/health/health.d/ipfs.conf

174 + metric: ipfs.datastore_space_utilization

175 + info: IPFS datastore utilization

176 + metrics:

177 + folding:

178 + title: Metrics

179 + enabled: false

180 + description: ""

181 + availability: []

182 + scopes:

183 + - name: global

184 + description: "These metrics refer to the entire monitored application."

185 + labels: []

186 + metrics:

187 + - name: ipfs.bandwidth

188 + description: IPFS Bandwidth

189 + unit: "bytes/s"

190 + chart_type: line

191 + dimensions:

192 + - name: in

193 + - name: out

194 + - name: ipfs.peers

195 + description: IPFS Peers

196 + unit: "peers"

197 + chart_type: line

198 + dimensions:

199 + - name: peers

200 + - name: ipfs.repo_size

201 + description: IPFS Repo Size

202 + unit: "GiB"

203 + chart_type: area

204 + dimensions:

205 + - name: size

206 + - name: ipfs.repo_objects

207 + description: IPFS Repo Objects

208 + unit: "objects"

209 + chart_type: line

210 + dimensions:

211 + - name: objects

212 + - name: ipfs.repo_pinned_objects

213 + description: IPFS Repo Pinned Objects

214 + unit: "objects"

215 + chart_type: line

216 + dimensions:

217 + - name: recursive_pins

src/go/plugin/go.d/modules/ipfs/testdata/api_v0_pin_ls.json new

@@ -0,0 +1,8 @@

1 +{

2 + "Keys": {

3 + "k1i2m3c4h5i6key": {

4 + "Type": "recursive",

5 + "Name": ""

6 + }

7 + }

8 +}

src/go/plugin/go.d/modules/ipfs/testdata/api_v0_stats_bw.json new

@@ -0,0 +1,6 @@

1 +{

2 + "TotalIn": 20113594,

3 + "TotalOut": 3113852,

4 + "RateIn": 1623.2181369394084,

5 + "RateOut": 0.13743234792898051

6 +}

src/go/plugin/go.d/modules/ipfs/testdata/api_v0_stats_repo.json new

@@ -0,0 +1,7 @@

1 +{

2 + "RepoSize": 25495,

3 + "StorageMax": 10000000000,

4 + "NumObjects": 1,

5 + "RepoPath": "/home/fotis/.ipfs",

6 + "Version": "fs-repo@15"

7 +}

src/go/plugin/go.d/modules/ipfs/testdata/api_v0_swarm_peers.json new

+70

@@ -0,0 +1,70 @@

1 +{

2 + "Peers": [

3 + {

4 + "Addr": "/ip4/1/tcp/27963",

5 + "Peer": "a",

6 + "Identify": {

7 + "ID": "",

8 + "PublicKey": "",

9 + "Addresses": null,

10 + "AgentVersion": "",

11 + "Protocols": null

12 + }

13 + },

14 + {

15 + "Addr": "/ip4/1/udp/4001/quic-v1",

16 + "Peer": "b",

17 + "Identify": {

18 + "ID": "",

19 + "PublicKey": "",

20 + "Addresses": null,

21 + "AgentVersion": "",

22 + "Protocols": null

23 + }

24 + },

25 + {

26 + "Addr": "/ip4/1/udp/4001/quic-v1/p2p/12D3KooWCqocoHdBANn2hH5acYAU4NdjEeBqERYk1MMTX49s1syY/p2p-circuit",

27 + "Peer": "c",

28 + "Identify": {

29 + "ID": "",

30 + "PublicKey": "",

31 + "Addresses": null,

32 + "AgentVersion": "",

33 + "Protocols": null

34 + }

35 + },

36 + {

37 + "Addr": "/ip4/1/tcp/4001",

38 + "Peer": "c",

39 + "Identify": {

40 + "ID": "",

41 + "PublicKey": "",

42 + "Addresses": null,

43 + "AgentVersion": "",

44 + "Protocols": null

45 + }

46 + },

47 + {

48 + "Addr": "/ip4/1/udp/33556/quic-v1",

49 + "Peer": "e",

50 + "Identify": {

51 + "ID": "",

52 + "PublicKey": "",

53 + "Addresses": null,

54 + "AgentVersion": "",

55 + "Protocols": null

56 + }

57 + },

58 + {

59 + "Addr": "/ip6/1::1/udp/4001/quic-v1",

60 + "Peer": "f",

61 + "Identify": {

62 + "ID": "",

63 + "PublicKey": "",

64 + "Addresses": null,

65 + "AgentVersion": "",

66 + "Protocols": null

67 + }

68 + }

69 + ]

70 +}

src/go/plugin/go.d/modules/ipfs/testdata/config.json new

+22

@@ -0,0 +1,22 @@

1 +{

2 + "update_every": 123,

3 + "url": "ok",

4 + "body": "ok",

5 + "method": "ok",

6 + "headers": {

7 + "ok": "ok"

8 + },

9 + "username": "ok",

10 + "password": "ok",

11 + "proxy_url": "ok",

12 + "proxy_username": "ok",

13 + "proxy_password": "ok",

14 + "timeout": 123.123,

15 + "not_follow_redirects": true,

16 + "tls_ca": "ok",

17 + "tls_cert": "ok",

18 + "tls_key": "ok",

19 + "tls_skip_verify": true,

20 + "pinapi": false,

21 + "repoapi": false

22 +}

src/go/plugin/go.d/modules/ipfs/testdata/config.yaml new

+19

@@ -0,0 +1,19 @@

1 +update_every: 123

2 +url: "ok"

3 +body: "ok"

4 +method: "ok"

5 +headers:

6 + ok: "ok"

7 +username: "ok"

8 +password: "ok"

9 +proxy_url: "ok"

10 +proxy_username: "ok"

11 +proxy_password: "ok"

12 +timeout: 123.123

13 +not_follow_redirects: yes

14 +tls_ca: "ok"

15 +tls_cert: "ok"

16 +tls_key: "ok"

17 +tls_skip_verify: yes

18 +pinapi: no

19 +repoapi: no

src/health/health.d/ipfs.conf

+2 -2

@@ -1,10 +1,10 @@

2 template: ipfs_datastore_usage

3 - on: ipfs.repo_size

3 + on: ipfs.datastore_space_utilization

4 class: Utilization

5 type: Data Sharing

6 component: IPFS

7 - calc: $size * 100 / $avail

7 + calc: $used

8 units: %

9 every: 10s

10 warn: $this > (($status >= $WARNING) ? (80) : (90))