mysql: collect galera cluster metrics (#6962)
* mysql: collect galera cluster metrics * mysql: readme update * mysql: add galera cluster size and state alarms
Ilya Mashchenko committed
Oct 3, 2019 at 19:14 UTC
bfae683664b488df91940a81760459db2b18983c
4 files changed
+316
-52
collectors/python.d.plugin/mysql/README.md
+37
-5
@@ -241,12 +241,12 @@ It will produce following charts (if data is available):
241
- sql
242
- io
243
244
-42. **Replicated Writesets** in writesets/s
244
+42. **Galera Replicated Writesets** in writesets/s
245
246
- rx
247
- tx
248
249
-43. **Replicated Bytes** in KiB/s
249
+43. **Galera Replicated Bytes** in KiB/s
250
251
- rx
252
- tx
@@ -256,16 +256,48 @@ It will produce following charts (if data is available):
256
- rx
257
- tx
258
259
-45. **Replication Conflicts** in transactions
259
+45. **Galera Replication Conflicts** in transactions
260
261
- bf aborts
262
- cert fails
263
264
-46. **Flow Control** in ms
264
+46. **Galera Flow Control** in ms
265
266
- paused
267
268
-47. **Users CPU time** in percentage
268
+47. **Galera Cluster Status** in status
269
+
270
+ - status
271
+
272
+48. **Galera Cluster State** in state
273
+
274
+ - state
275
+
276
+49. **Galera Number of Nodes in the Cluster** in num
277
+
278
+ - nodes
279
+
280
+50. **Galera Total Weight of the Current Members in the Cluster** in weight
281
+
282
+ - weight
283
+
284
+51. **Galera Whether the Node is Connected to the Cluster** in boolean
285
+
286
+ - connected
287
+
288
+52. **Galera Whether the Node is Ready to Accept Queries** in boolean
289
+
290
+ - ready
291
+
292
+53. **Galera Open Transactions** in num
293
+
294
+ - open transactions
295
+
296
+54. **Galera Total Number of WSRep (applier/rollbacker) Threads** in num
297
+
298
+ - threads
299
+
300
+55. **Users CPU time** in percentage
301
302
- users
303
collectors/python.d.plugin/mysql/mysql.chart.py
+212
-47
@@ -117,6 +117,14 @@ GLOBAL_STATS = [
117
'Connection_errors_peer_address',
118
'Connection_errors_select',
119
'Connection_errors_tcpwrap',
120
+ 'Com_delete',
121
+ 'Com_insert',
122
+ 'Com_select',
123
+ 'Com_update',
124
+ 'Com_replace'
125
+]
126
+
127
+GALERA_STATS = [
128
'wsrep_local_recv_queue',
129
'wsrep_local_send_queue',
130
'wsrep_received',
@@ -126,11 +134,14 @@ GLOBAL_STATS = [
134
'wsrep_local_bf_aborts',
135
'wsrep_local_cert_failures',
136
'wsrep_flow_control_paused_ns',
129
- 'Com_delete',
130
- 'Com_insert',
131
- 'Com_select',
132
- 'Com_update',
133
- 'Com_replace'
137
+ 'wsrep_cluster_weight',
138
+ 'wsrep_cluster_size',
139
+ 'wsrep_cluster_status',
140
+ 'wsrep_local_state',
141
+ 'wsrep_open_transactions',
142
+ 'wsrep_connected',
143
+ 'wsrep_ready',
144
+ 'wsrep_thread_count'
145
]
146
147
@@ -216,7 +227,15 @@ ORDER = [
227
'galera_queue',
228
'galera_conflicts',
229
'galera_flow_control',
219
- 'userstats_cpu'
230
+ 'galera_cluster_status',
231
+ 'galera_cluster_state',
232
+ 'galera_cluster_size',
233
+ 'galera_cluster_weight',
234
+ 'galera_connected',
235
+ 'galera_ready',
236
+ 'galera_open_transactions',
237
+ 'galera_thread_count',
238
+ 'userstats_cpu',
239
]
240
241
CHARTS = {
@@ -594,6 +613,58 @@ CHARTS = {
613
['wsrep_flow_control_paused_ns', 'paused', 'incremental', 1, 1000000],
614
]
615
},
616
+ 'galera_cluster_status': {
617
+ 'options': [None, 'Cluster Component Status', 'status', 'galera', 'mysql.galera_cluster_status', 'line'],
618
+ 'lines': [
619
+ ['wsrep_cluster_status', 'status', 'absolute'],
620
+ ]
621
+ },
622
+ 'galera_cluster_state': {
623
+ 'options': [None, 'Cluster Component State', 'state', 'galera', 'mysql.galera_cluster_state', 'line'],
624
+ 'lines': [
625
+ ['wsrep_local_state', 'state', 'absolute'],
626
+ ]
627
+ },
628
+ 'galera_cluster_size': {
629
+ 'options': [None, 'Number of Nodes in the Cluster', 'num', 'galera', 'mysql.galera_cluster_size', 'line'],
630
+ 'lines': [
631
+ ['wsrep_cluster_size', 'nodes', 'absolute'],
632
+ ]
633
+ },
634
+ 'galera_cluster_weight': {
635
+ 'options': [None, 'The Total Weight of the Current Members in the Cluster', 'weight', 'galera',
636
+ 'mysql.galera_cluster_weight', 'line'],
637
+ 'lines': [
638
+ ['wsrep_cluster_weight', 'weight', 'absolute'],
639
+ ]
640
+ },
641
+ 'galera_connected': {
642
+ 'options': [None, 'Whether the Node is Connected to the Cluster', 'boolean', 'galera',
643
+ 'mysql.galera_connected', 'line'],
644
+ 'lines': [
645
+ ['wsrep_connected', 'connected', 'absolute'],
646
+ ]
647
+ },
648
+ 'galera_ready': {
649
+ 'options': [None, 'Whether the Node is Ready to Accept Queries', 'boolean', 'galera',
650
+ 'mysql.galera_ready', 'line'],
651
+ 'lines': [
652
+ ['wsrep_ready', 'ready', 'absolute'],
653
+ ]
654
+ },
655
+ 'galera_open_transactions': {
656
+ 'options': [None, 'Open Transactions', 'num', 'galera', 'mysql.galera_open_transactions', 'line'],
657
+ 'lines': [
658
+ ['wsrep_open_transactions', 'open transactions', 'absolute'],
659
+ ]
660
+ },
661
+ 'galera_thread_count': {
662
+ 'options': [None, 'Total Number of WSRep (applier/rollbacker) Threads', 'num', 'galera',
663
+ 'mysql.galera_thread_count', 'line'],
664
+ 'lines': [
665
+ ['wsrep_thread_count', 'threads', 'absolute'],
666
+ ]
667
+ },
668
'userstats_cpu': {
669
'options': [None, 'Users CPU time', 'percentage', 'userstats', 'mysql.userstats_cpu', 'stacked'],
670
'lines': []
@@ -663,6 +734,59 @@ def userstats_chart_template(name):
734
DEFAULT_REPL_CHANNEL = ''
735
736
737
+# Write Set REPlication
738
+# https://galeracluster.com/library/documentation/galera-status-variables.html
739
+# https://www.percona.com/doc/percona-xtradb-cluster/LATEST/wsrep-status-index.html
740
+class WSRepDataConverter:
741
+ unknown_value = -1
742
+
743
+ def convert(self, key, value):
744
+ if key == 'wsrep_connected':
745
+ return self.convert_connected(value)
746
+ elif key == 'wsrep_ready':
747
+ return self.convert_ready(value)
748
+ elif key == 'wsrep_cluster_status':
749
+ return self.convert_cluster_status(value)
750
+ return value
751
+
752
+ def convert_connected(self, value):
753
+ # https://www.percona.com/doc/percona-xtradb-cluster/LATEST/wsrep-status-index.html#wsrep_connected
754
+ if value == 'OFF':
755
+ return 0
756
+ if value == 'ON':
757
+ return 1
758
+ return self.unknown_value
759
+
760
+ def convert_ready(self, value):
761
+ # https://www.percona.com/doc/percona-xtradb-cluster/LATEST/wsrep-status-index.html#wsrep_ready
762
+ if value == 'OFF':
763
+ return 0
764
+ if value == 'ON':
765
+ return 1
766
+ return self.unknown_value
767
+
768
+ def convert_cluster_status(self, value):
769
+ # https://www.percona.com/doc/percona-xtradb-cluster/LATEST/wsrep-status-index.html#wsrep_cluster_status
770
+ # https://github.com/codership/wsrep-API/blob/eab2d5d5a31672c0b7d116ef1629ff18392fd7d0/wsrep_api.h
771
+ # typedef enum wsrep_view_status {
772
+ # WSREP_VIEW_PRIMARY, //!< primary group configuration (quorum present)
773
+ # WSREP_VIEW_NON_PRIMARY, //!< non-primary group configuration (quorum lost)
774
+ # WSREP_VIEW_DISCONNECTED, //!< not connected to group, retrying.
775
+ # WSREP_VIEW_MAX
776
+ # } wsrep_view_status_t;
777
+ value = value.lower()
778
+ if value == 'primary':
779
+ return 0
780
+ elif value == 'non-primary':
781
+ return 1
782
+ elif value == 'disconnected':
783
+ return 2
784
+ return self.unknown_value
785
+
786
+
787
+wsrep_converter = WSRepDataConverter()
788
+
789
+
790
class Service(MySQLService):
791
def __init__(self, configuration=None, name=None):
792
MySQLService.__init__(self, configuration=configuration, name=name)
@@ -686,12 +810,9 @@ class Service(MySQLService):
810
data = dict()
811
812
if 'global_status' in raw_data:
689
- global_status = dict(raw_data['global_status'][0])
690
- for key in GLOBAL_STATS:
691
- if key in global_status:
692
- data[key] = global_status[key]
693
- if 'Threads_created' in data and 'Connections' in data:
694
- data['Thread_cache_misses'] = round(int(data['Threads_created']) / float(data['Connections']) * 10000)
813
+ global_status = self.get_global_status(raw_data['global_status'])
814
+ if global_status:
815
+ data.update(global_status)
816
817
if 'slave_status' in raw_data:
818
status = self.get_slave_status(raw_data['slave_status'])
@@ -712,6 +833,52 @@ class Service(MySQLService):
833
834
return data or None
835
836
+ @staticmethod
837
+ def convert_wsrep(key, value):
838
+ return wsrep_converter.convert(key, value)
839
+
840
+ def get_global_status(self, raw_global_status):
841
+ # (
842
+ # (
843
+ # ('Aborted_clients', '18'),
844
+ # ('Aborted_connects', '33'),
845
+ # ('Access_denied_errors', '80'),
846
+ # ('Acl_column_grants', '0'),
847
+ # ('Acl_database_grants', '0'),
848
+ # ('Acl_function_grants', '0'),
849
+ # ('wsrep_ready', 'OFF'),
850
+ # ('wsrep_rollbacker_thread_count', '0'),
851
+ # ('wsrep_thread_count', '0')
852
+ # ),
853
+ # (
854
+ # ('Variable_name', 253, 60, 64, 64, 0, 0),
855
+ # ('Value', 253, 48, 2048, 2048, 0, 0),
856
+ # )
857
+ # )
858
+ rows = raw_global_status[0]
859
+ if not rows:
860
+ return
861
+
862
+ global_status = dict(rows)
863
+ data = dict()
864
+
865
+ for key in GLOBAL_STATS:
866
+ if key not in global_status:
867
+ continue
868
+ value = global_status[key]
869
+ data[key] = value
870
+
871
+ for key in GALERA_STATS:
872
+ if key not in global_status:
873
+ continue
874
+ value = global_status[key]
875
+ value = self.convert_wsrep(key, value)
876
+ data[key] = value
877
+
878
+ if 'Threads_created' in data and 'Connections' in data:
879
+ data['Thread_cache_misses'] = round(int(data['Threads_created']) / float(data['Connections']) * 10000)
880
+ return data
881
+
882
def get_slave_status(self, slave_status_data):
883
rows, description = slave_status_data[0], slave_status_data[1]
884
description_keys = [v[0] for v in description]
@@ -742,41 +909,39 @@ class Service(MySQLService):
909
self.add_new_charts(slave_status_chart_template, name)
910
911
def get_userstats(self, raw_data):
745
- # raw_data['user_statistics'] contains the following data structure:
746
- # (
747
- # (
748
- # ('netdata', 42L, 0L, 1264L, 3.111252999999968, 2.968510299999994, 110267L, 19741424L, 0L, 0L, 1265L, 0L,
749
- # 0L, 0L, 3L, 0L, 1301L, 0L, 0L, 7633L, 0L, 83L, 44L, 0L, 0L),
750
- # ('root', 60L, 0L, 184L, 0.22856499999999966, 0.1601419999999998, 11605L, 1516513L, 0L, 9L, 220L, 0L, 2L, 1L,
751
- # 6L, 4L,127L, 0L, 0L, 45L, 0L, 45L, 0L, 0L, 0L)
752
- # ),
753
- # (
754
- # ('User', 253, 9, 128, 128, 0, 0),
755
- # ('Total_connections', 3, 2, 11, 11, 0, 0),
756
- # ('Concurrent_connections', 3, 1, 11, 11, 0, 0),
757
- # ('Connected_time', 3, 4, 11, 11, 0, 0),
758
- # ('Busy_time', 5, 21, 21, 21, 31, 0),
759
- # ('Cpu_time', 5, 18, 21, 21, 31, 0),
760
- # ('Bytes_received', 8, 6, 21, 21, 0, 0),
761
- # ('Bytes_sent', 8, 8, 21, 21, 0, 0),
762
- # ('Binlog_bytes_written', 8, 1, 21, 21, 0, 0),
763
- # ('Rows_read', 8, 1, 21, 21, 0, 0),
764
- # ('Rows_sent', 8, 4, 21, 21, 0, 0),
765
- # ('Rows_deleted', 8, 1, 21, 21, 0, 0),
766
- # ('Rows_inserted', 8, 1, 21, 21, 0, 0),
767
- # ('Rows_updated', 8, 1, 21, 21, 0, 0),
768
- # ('Select_commands', 8, 1, 21, 21, 0, 0),
769
- # ('Update_commands', 8, 1, 21, 21, 0, 0),
770
- # ('Other_commands', 8, 4, 21, 21, 0, 0),
771
- # ('Commit_transactions', 8, 1, 21, 21, 0, 0),
772
- # ('Rollback_transactions', 8, 1, 21, 21, 0, 0),
773
- # ('Denied_connections', 8, 4, 21, 21, 0, 0),
774
- # ('Lost_connections', 8, 1, 21, 21, 0, 0),
775
- # ('Access_denied', 8, 2, 21, 21, 0, 0),
776
- # ('Empty_queries', 8, 2, 21, 21, 0, 0),
777
- # ('Total_ssl_connections', 8, 1, 21, 21, 0, 0),
778
- # ('Max_statement_time_exceeded', 8, 1, 21, 21, 0, 0)),
779
- # )
912
+ # (
913
+ # (
914
+ # ('netdata', 1L, 0L, 60L, 0.15842499999999984, 0.15767439999999996, 5206L, 963957L, 0L, 0L,
915
+ # 61L, 0L, 0L, 0L, 0L, 0L, 62L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L),
916
+ # ),
917
+ # (
918
+ # ('User', 253, 7, 128, 128, 0, 0),
919
+ # ('Total_connections', 3, 2, 11, 11, 0, 0),
920
+ # ('Concurrent_connections', 3, 1, 11, 11, 0, 0),
921
+ # ('Connected_time', 3, 2, 11, 11, 0, 0),
922
+ # ('Busy_time', 5, 20, 21, 21, 31, 0),
923
+ # ('Cpu_time', 5, 20, 21, 21, 31, 0),
924
+ # ('Bytes_received', 8, 4, 21, 21, 0, 0),
925
+ # ('Bytes_sent', 8, 6, 21, 21, 0, 0),
926
+ # ('Binlog_bytes_written', 8, 1, 21, 21, 0, 0),
927
+ # ('Rows_read', 8, 1, 21, 21, 0, 0),
928
+ # ('Rows_sent', 8, 2, 21, 21, 0, 0),
929
+ # ('Rows_deleted', 8, 1, 21, 21, 0, 0),
930
+ # ('Rows_inserted', 8, 1, 21, 21, 0, 0),
931
+ # ('Rows_updated', 8, 1, 21, 21, 0, 0),
932
+ # ('Select_commands', 8, 2, 21, 21, 0, 0),
933
+ # ('Update_commands', 8, 1, 21, 21, 0, 0),
934
+ # ('Other_commands', 8, 2, 21, 21, 0, 0),
935
+ # ('Commit_transactions', 8, 1, 21, 21, 0, 0),
936
+ # ('Rollback_transactions', 8, 1, 21, 21, 0, 0),
937
+ # ('Denied_connections', 8, 1, 21, 21, 0, 0),
938
+ # ('Lost_connections', 8, 1, 21, 21, 0, 0),
939
+ # ('Access_denied', 8, 1, 21, 21, 0, 0),
940
+ # ('Empty_queries', 8, 2, 21, 21, 0, 0),
941
+ # ('Total_ssl_connections', 8, 1, 21, 21, 0, 0),
942
+ # ('Max_statement_time_exceeded', 8, 1, 21, 21, 0, 0)
943
+ # )
944
+ # )
945
data = dict()
946
userstats_vars = [e[0] for e in raw_data['user_statistics'][1]]
947
for i, _ in enumerate(raw_data['user_statistics'][0]):
health/health.d/mysql.conf
+34
@@ -98,3 +98,37 @@ template: mysql_replication_lag
98
info: the number of seconds mysql replication is behind this master
99
to: dba
100
101
+
102
+# -----------------------------------------------------------------------------
103
+# galera cluster size
104
+
105
+template: mysql_galera_cluster_size_max_2m
106
+ on: mysql.galera_cluster_size
107
+ lookup: max -2m absolute
108
+ units: nodes
109
+ every: 10s
110
+ info: max cluster size 2 minute
111
+ to: dba
112
+
113
+template: mysql_galera_cluster_size
114
+ on: mysql.galera_cluster_size
115
+ calc: $nodes
116
+ units: nodes
117
+ every: 10s
118
+ warn: $this > $mysql_galera_cluster_size_max_2m
119
+ crit: $this < $mysql_galera_cluster_size_max_2m
120
+ delay: up 20s down 5m multiplier 1.5 max 1h
121
+ info: cluster size has changed
122
+ to: dba
123
+
124
+# galera node state
125
+
126
+template: mysql_galera_cluster_state
127
+ on: mysql.galera_cluster_state
128
+ calc: $state
129
+ every: 10s
130
+ warn: $this < 4
131
+ crit: $this < 2
132
+ delay: up 30s down 5m multiplier 1.5 max 1h
133
+ info: node state (0: undefined, 1: joining, 2: donor/desynced, 3: joined, 4: synced)
134
+ to: dba
web/gui/dashboard_info.js
+33
@@ -1260,6 +1260,39 @@ netdataDashboard.context = {
1260
info: 'A deadlock happens when two or more transactions mutually hold and request for locks, creating a cycle of dependencies. For more information about <a href="https://dev.mysql.com/doc/refman/5.7/en/innodb-deadlocks-handling.html" target="_blank">how to minimize and handle deadlocks</a>.'
1261
},
1262
1263
+ 'mysql.galera_cluster_status': {
1264
+ info:
1265
+ '<code>-1</code>: unknown, ' +
1266
+ '<code>0</code>: primary (primary group configuration, quorum present), ' +
1267
+ '<code>1</code>: non-primary (non-primary group configuration, quorum lost), ' +
1268
+ '<code>2</code>: disconnected(not connected to group, retrying).'
1269
+ },
1270
+
1271
+ 'mysql.galera_cluster_state': {
1272
+ info:
1273
+ '<code>0</code>: undefined, ' +
1274
+ '<code>1</code>: joining, ' +
1275
+ '<code>2</code>: donor/desynced, ' +
1276
+ '<code>3</code>: joined, ' +
1277
+ '<code>4</code>: synced.'
1278
+ },
1279
+
1280
+ 'mysql.galera_cluster_weight': {
1281
+ info: 'The value is counted as a sum of <code>pc.weight</code> of the nodes in the current Primary Component.'
1282
+ },
1283
+
1284
+ 'mysql.galera_connected': {
1285
+ info: '<code>0</code> means that the node has not yet connected to any of the cluster components. ' +
1286
+ 'This may be due to misconfiguration.'
1287
+ },
1288
+
1289
+ 'mysql.open_transactions': {
1290
+ info: 'The number of locally running transactions which have been registered inside the wsrep provider. ' +
1291
+ 'This means transactions which have made operations which have caused write set population to happen. ' +
1292
+ 'Transactions which are read only are not counted.'
1293
+ },
1294
+
1295
+
1296
// ------------------------------------------------------------------------
1297
// POSTGRESQL
1298