Add @l2isbad suggestions and improve pool method
Luis Eduardo committed
Dec 23, 2017 at 11:05 UTC
37ec3474b943fb7e1c3bcca2a034188e7f05e722
2 files changed
+112
-59
conf.d/python.d/ceph.conf
+73
-2
@@ -1,5 +1,76 @@
1
-update_every : 10
1
+# netdata python.d.plugin configuration for ceph stats
2
+#
3
+# This file is in YaML format. Generally the format is:
4
+#
5
+# name: value
6
+#
7
+# There are 2 sections:
8
+# - global variables
9
+# - one or more JOBS
10
+#
11
+# JOBS allow you to collect values from multiple sources.
12
+# Each source will have its own set of charts.
13
+#
14
+# JOB parameters have to be indented (using spaces only, example below).
15
+
16
+# ----------------------------------------------------------------------
17
+# Global Variables
18
+# These variables set the defaults for all JOBs, however each JOB
19
+# may define its own, overriding the defaults.
20
+
21
+# update_every sets the default data collection frequency.
22
+# If unset, the python.d.plugin default is used.
23
+# update_every: 10
24
+
25
+# priority controls the order of charts at the netdata dashboard.
26
+# Lower numbers move the charts towards the top of the page.
27
+# If unset, the default for python.d.plugin is used.
28
+# priority: 60000
29
+
30
+# retries sets the number of retries to be made in case of failures.
31
+# If unset, the default for python.d.plugin is used.
32
+# Attempts to restore the service are made once every update_every
33
+# and only if the module has collected values in the past.
34
+# retries: 60
35
+
36
+# autodetection_retry sets the job re-check interval in seconds.
37
+# The job is not deleted if check fails.
38
+# Attempts to start the job are made once every autodetection_retry.
39
+# This feature is disabled by default.
40
+# autodetection_retry: 0
41
+
42
+# ----------------------------------------------------------------------
43
+# JOBS (data collection sources)
44
+#
45
+# The default JOBS share the same *name*. JOBS with the same name
46
+# are mutually exclusive. Only one of them will be allowed running at
47
+# any time. This allows autodetection to try several alternatives and
48
+# pick the one that works.
49
+#
50
+# Any number of jobs is supported.
51
+#
52
+# All python.d.plugin JOBS (for all its modules) support a set of
53
+# predefined parameters. These are:
54
+#
55
+# job_name:
56
+# name: myname # the JOB's name as it will appear at the
57
+# # dashboard (by default is the job_name)
58
+# # JOBs sharing a name are mutually exclusive
59
+# update_every: 10 # the JOB's data collection frequency
60
+# priority: 60000 # the JOB's order on the dashboard
61
+# retries: 60 # the JOB's number of restoration attempts
62
+# autodetection_retry: 0 # the JOB's re-check interval in seconds
63
+#
64
+# Additionally to the above, ceph plugin also supports the following:
65
+#
66
+# config_file: 'config_file' # Ceph config file.
67
+# keyring_file: 'keyring_file' # Ceph keyring file. netdata user must be added into ceph group
68
+# # and keyring file must be read group permission.
69
+# ----------------------------------------------------------------------
70
+# AUTO-DETECTION JOBS
71
+# only one of them will run (they have the same name)
72
+#
73
local:
74
config_file: '/etc/ceph/ceph.conf'
75
keyring_file: '/etc/ceph/ceph.client.admin.keyring'
5
- presentation_data: 'KB'
\ No newline at end of file
76
+
python.d/ceph.chart.py
+39
-57
@@ -2,38 +2,44 @@
2
# Description: ceph netdata python.d module
3
# Author: Luis Eduardo (lets00)
4
5
-import rados
5
+try:
6
+ import rados
7
+ CEPH = True
8
+except ImportError:
9
+ CEPH = False
10
+
11
import json
12
from bases.FrameworkServices.SimpleService import SimpleService
13
14
+# default module values (can be overridden per job in `config`)
15
+update_every = 10
16
+priority = 60000
17
+retries = 60
18
+
19
ORDER = ['general', 'pool_usage', 'pool_objects', 'osd_usage']
20
21
CHARTS = {
22
'general': {
13
- 'options': [None, 'Ceph General Options', 'usage', 'general', 'ceph.general', 'line'],
14
- 'lines': []
23
+ 'options': [None, 'Ceph General Options', 'KB', 'general', 'ceph.general', 'stacked'],
24
+ 'lines': [
25
+ ['general_usage', 'Usage', 'absolute', 1, 1024],
26
+ ['general_available', 'Available', 'absolute', 1, 1024]
27
+ ]
28
},
29
'pool_usage': {
17
- 'options': [None, 'Ceph Pools', 'usage', 'pool usage', 'ceph.pool_usage', 'line'],
30
+ 'options': [None, 'Ceph Pools', 'KB', 'pool', 'ceph.pool_usage', 'line'],
31
'lines': []
32
},
33
'pool_objects': {
21
- 'options': [None, 'Ceph Pools', 'objects', 'number of objects', 'ceph.pool_objects', 'line'],
34
+ 'options': [None, 'Ceph Pools', 'objects', 'pool', 'ceph.pool_objects', 'line'],
35
'lines': []
36
},
37
'osd_usage': {
25
- 'options': [None, 'Ceph OSDs', 'usage', 'osd usage', 'ceph.osd_usage', 'line'],
38
+ 'options': [None, 'Ceph OSDs', 'KB', 'osd', 'ceph.osd_usage', 'line'],
39
'lines': []
40
}
41
}
42
30
-PRESENTATION_MAP = {
31
- 'KB': 1,
32
- 'MB': 1024,
33
- 'GB': 1024**2,
34
- 'TB': 1024**3
35
-}
36
-
43
44
class Service(SimpleService):
45
def __init__(self, configuration=None, name=None):
@@ -42,8 +48,6 @@ class Service(SimpleService):
48
self.definitions = CHARTS
49
self.config_file = self.configuration.get('config_file')
50
self.keyring_file = self.configuration.get('keyring_file')
45
- self.presentation_data = self.configuration.get('presentation_data')
46
- self.presentation_value = None
51
self.cluster = rados.Rados(conffile=self.config_file,
52
conf=dict(keyring=self.keyring_file))
53
@@ -52,15 +56,12 @@ class Service(SimpleService):
56
Checks module
57
:return:
58
"""
59
+ if not CEPH:
60
+ self.error('rados module is needed to use ceph.chart.py')
61
+ return False
62
if not (self.config_file and self.keyring_file):
63
self.error('config_file and/or keyring_file is not defined')
64
return False
58
-
59
- # Set default presentation_data to GB if it does not defined and declare
60
- # presentation_value with PRESENTATION_MAP value.
61
- if not self.presentation_data:
62
- self.presentation_data = 'GB'
63
- self.presentation_value = PRESENTATION_MAP[self.presentation_data]
65
try:
66
self.cluster.connect()
67
except Exception as error:
@@ -71,38 +72,22 @@ class Service(SimpleService):
72
73
def create_definitions(self):
74
"""
74
- Create dynamically charts options, add presentation data/value
75
+ Create dynamically charts options
76
set on YAML file
77
:return: None
78
"""
78
- # Options
79
- self.definitions['general']['options'] = [None, 'Ceph General Options',
80
- 'usage ({})'.format(self.presentation_data),
81
- 'general', 'ceph.general', 'areastack']
82
- self.definitions['pool_usage']['options'] = [None, 'Ceph Pools',
83
- 'usage ({})'.format(self.presentation_data),
84
- 'pool usage', 'ceph.pool_usage', 'line']
85
- self.definitions['osd_usage']['options'] = [None, 'Ceph OSDs',
86
- 'usage ({})'.format(self.presentation_data),
87
- 'osd usage', 'ceph.osd_usage', 'line']
88
-
79
# Lines
90
- self.definitions['general']['lines'].append(['general_total', 'Total',
91
- 'absolute', 1, self.presentation_value])
92
- self.definitions['general']['lines'].append(['general_usage', 'Usage',
93
- 'absolute', 1, self.presentation_value])
94
- self.definitions['general']['lines'].append(['general_available', 'Available',
95
- 'absolute', 1, self.presentation_value])
96
-
97
- for pool in self.cluster.list_pools():
98
- self.definitions['pool_usage']['lines'].append([pool, pool,
99
- 'absolute', 1, self.presentation_value])
100
- self.definitions['pool_objects']['lines'].append(["obj_{}".format(pool), pool,
80
+ for pool in self._get_df()['pools']:
81
+ self.definitions['pool_usage']['lines'].append([pool['name'],
82
+ pool['name'],
83
+ 'absolute'])
84
+ self.definitions['pool_objects']['lines'].append(["obj_{}".format(pool['name']),
85
+ pool['name'],
86
'absolute'])
102
-
103
- for osd in self._get_osds_df()['nodes']:
104
- self.definitions['osd_usage']['lines'].append([osd['name'], osd['name'],
105
- 'absolute', 1, self.presentation_value])
87
+ for osd in self._get_osd_df()['nodes']:
88
+ self.definitions['osd_usage']['lines'].append([osd['name'],
89
+ osd['name'],
90
+ 'absolute'])
91
92
def get_data(self):
93
"""
@@ -124,11 +109,9 @@ class Service(SimpleService):
109
Get ceph's general usage
110
:return: dict
111
"""
127
- info = self.cluster.get_cluster_stats()
128
- # Values are get in bytes, convert it
129
- return {'general_total': float(info['kb']),
130
- 'general_usage': float(info['kb_used']),
131
- 'general_available': float(info['kb_avail'])
112
+ info = self._get_df()['stats']
113
+ return {'general_usage': info['total_used_bytes'],
114
+ 'general_available': info['total_avail_bytes']
115
}
116
117
def _get_pool_usage(self):
@@ -138,7 +121,7 @@ class Service(SimpleService):
121
"""
122
pool_usage = {}
123
for df in self._get_df()['pools']:
141
- pool_usage[df['name']] = df['stats']['bytes_used']
124
+ pool_usage[df['name']] = df['stats']['kb_used']
125
return pool_usage
126
127
def _get_pool_objects(self):
@@ -152,13 +135,12 @@ class Service(SimpleService):
135
return pool_objects
136
137
def _get_osd_usage(self):
155
- # TODO: FIX ME
138
"""
139
Process raw data into osd dict information to get osd usage
140
:return: A osd dict with osd name's key and usage bytes' value
141
"""
142
osd_usage = {}
161
- for osd in self._get_osds_df()['nodes']:
143
+ for osd in self._get_osd_df()['nodes']:
144
osd_usage[osd['name']] = float(osd['kb_used'])
145
return osd_usage
146
@@ -172,7 +154,7 @@ class Service(SimpleService):
154
'format': 'json'
155
}), '')[1])
156
175
- def _get_osds_df(self):
157
+ def _get_osd_df(self):
158
"""
159
Get ceph osd df output
160
:return: ceph osd df --format json