mysql collector: slave_status charts per replication channel (#6597)
* slave_status charts per replication channel
Ilya Mashchenko committed
Aug 6, 2019 at 16:48 UTC
a22b7e9a8112481aac83343473eb87f9d678ecd9
1 file changed
+113
-53
collectors/python.d.plugin/mysql/mysql.chart.py
+113
-53
@@ -601,6 +601,33 @@ CHARTS = {
601
}
602
603
604
+def slave_status_chart_template(channel_name):
605
+ order = [
606
+ 'slave_behind_{0}'.format(channel_name),
607
+ 'slave_status_{0}'.format(channel_name)
608
+ ]
609
+
610
+ charts = {
611
+ order[0]: {
612
+ 'options': [None, 'Slave Behind Seconds Channel {0}'.format(channel_name),
613
+ 'seconds', 'slave', 'mysql.slave_behind', 'line'],
614
+ 'lines': [
615
+ ['Seconds_Behind_Master_{0}'.format(channel_name), 'seconds', 'absolute']
616
+ ]
617
+ },
618
+ order[1]: {
619
+ 'options': [None, 'Slave Status Channel {0}'.format(channel_name),
620
+ 'status', 'slave', 'mysql.slave_status', 'line'],
621
+ 'lines': [
622
+ ['Slave_SQL_Running_{0}'.format(channel_name), 'sql_running', 'absolute'],
623
+ ['Slave_IO_Running_{0}'.format(channel_name), 'io_running', 'absolute']
624
+ ]
625
+ },
626
+ }
627
+
628
+ return order, charts
629
+
630
+
631
def userstats_chart_template(name):
632
order = [
633
'userstats_rows_{0}'.format(name),
@@ -632,6 +659,10 @@ def userstats_chart_template(name):
659
return order, charts
660
661
662
+# https://dev.mysql.com/doc/refman/8.0/en/replication-channels.html
663
+DEFAULT_REPL_CHANNEL = ''
664
+
665
+
666
class Service(MySQLService):
667
def __init__(self, configuration=None, name=None):
668
MySQLService.__init__(self, configuration=configuration, name=name)
@@ -643,6 +674,7 @@ class Service(MySQLService):
674
variables=QUERY_VARIABLES,
675
user_statistics=QUERY_USER_STATISTICS,
676
)
677
+ self.repl_channels = [DEFAULT_REPL_CHANNEL]
678
679
def _get_data(self):
680
@@ -651,29 +683,25 @@ class Service(MySQLService):
683
if not raw_data:
684
return None
685
654
- to_netdata = dict()
686
+ data = dict()
687
688
if 'global_status' in raw_data:
689
global_status = dict(raw_data['global_status'][0])
690
for key in GLOBAL_STATS:
691
if key in global_status:
660
- to_netdata[key] = global_status[key]
661
- if 'Threads_created' in to_netdata and 'Connections' in to_netdata:
662
- to_netdata['Thread_cache_misses'] = round(int(to_netdata['Threads_created'])
663
- / float(to_netdata['Connections']) * 10000)
692
+ data[key] = global_status[key]
693
+ if 'Threads_created' in data and 'Connections' in data:
694
+ data['Thread_cache_misses'] = round(int(data['Threads_created']) / float(data['Connections']) * 10000)
695
696
if 'slave_status' in raw_data:
666
- if raw_data['slave_status'][0]:
667
- slave_raw_data = dict(zip([e[0] for e in raw_data['slave_status'][1]], raw_data['slave_status'][0][0]))
668
- for key, func in SLAVE_STATS:
669
- if key in slave_raw_data:
670
- to_netdata[key] = func(slave_raw_data[key])
671
- else:
672
- self.queries.pop('slave_status')
697
+ status = self.get_slave_status(raw_data['slave_status'])
698
+ data.update(status)
699
+ else:
700
+ self.queries.pop('slave_status')
701
702
if 'user_statistics' in raw_data:
703
if raw_data['user_statistics'][0]:
676
- to_netdata.update(self.get_userstats(raw_data))
704
+ data.update(self.get_userstats(raw_data))
705
else:
706
self.queries.pop('user_statistics')
707
@@ -681,46 +709,75 @@ class Service(MySQLService):
709
variables = dict(raw_data['variables'][0])
710
for key in VARIABLES:
711
if key in variables:
684
- to_netdata[key] = variables[key]
685
-
686
- return to_netdata or None
687
-
688
- # raw_data['user_statistics'] contains the following data structure:
689
- # (
690
- # (
691
- # ('netdata', 42L, 0L, 1264L, 3.111252999999968, 2.968510299999994, 110267L, 19741424L, 0L, 0L, 1265L, 0L,
692
- # 0L, 0L, 3L, 0L, 1301L, 0L, 0L, 7633L, 0L, 83L, 44L, 0L, 0L),
693
- # ('root', 60L, 0L, 184L, 0.22856499999999966, 0.1601419999999998, 11605L, 1516513L, 0L, 9L, 220L, 0L, 2L, 1L,
694
- # 6L, 4L,127L, 0L, 0L, 45L, 0L, 45L, 0L, 0L, 0L)
695
- # ),
696
- # (
697
- # ('User', 253, 9, 128, 128, 0, 0),
698
- # ('Total_connections', 3, 2, 11, 11, 0, 0),
699
- # ('Concurrent_connections', 3, 1, 11, 11, 0, 0),
700
- # ('Connected_time', 3, 4, 11, 11, 0, 0),
701
- # ('Busy_time', 5, 21, 21, 21, 31, 0),
702
- # ('Cpu_time', 5, 18, 21, 21, 31, 0),
703
- # ('Bytes_received', 8, 6, 21, 21, 0, 0),
704
- # ('Bytes_sent', 8, 8, 21, 21, 0, 0),
705
- # ('Binlog_bytes_written', 8, 1, 21, 21, 0, 0),
706
- # ('Rows_read', 8, 1, 21, 21, 0, 0),
707
- # ('Rows_sent', 8, 4, 21, 21, 0, 0),
708
- # ('Rows_deleted', 8, 1, 21, 21, 0, 0),
709
- # ('Rows_inserted', 8, 1, 21, 21, 0, 0),
710
- # ('Rows_updated', 8, 1, 21, 21, 0, 0),
711
- # ('Select_commands', 8, 1, 21, 21, 0, 0),
712
- # ('Update_commands', 8, 1, 21, 21, 0, 0),
713
- # ('Other_commands', 8, 4, 21, 21, 0, 0),
714
- # ('Commit_transactions', 8, 1, 21, 21, 0, 0),
715
- # ('Rollback_transactions', 8, 1, 21, 21, 0, 0),
716
- # ('Denied_connections', 8, 4, 21, 21, 0, 0),
717
- # ('Lost_connections', 8, 1, 21, 21, 0, 0),
718
- # ('Access_denied', 8, 2, 21, 21, 0, 0),
719
- # ('Empty_queries', 8, 2, 21, 21, 0, 0),
720
- # ('Total_ssl_connections', 8, 1, 21, 21, 0, 0),
721
- # ('Max_statement_time_exceeded', 8, 1, 21, 21, 0, 0)),
722
- # )
712
+ data[key] = variables[key]
713
+
714
+ return data or None
715
+
716
+ def get_slave_status(self, slave_status_data):
717
+ rows, description = slave_status_data[0], slave_status_data[1]
718
+ description_keys = [v[0] for v in description]
719
+ if not rows:
720
+ return
721
+
722
+ data = dict()
723
+ for row in rows:
724
+ slave_data = dict(zip(description_keys, row))
725
+ channel_name = slave_data.get('Channel_Name', DEFAULT_REPL_CHANNEL)
726
+
727
+ if channel_name not in self.repl_channels and len(self.charts) > 0:
728
+ self.add_repl_channel_charts(channel_name)
729
+ self.repl_channels.append(channel_name)
730
+
731
+ for key, func in SLAVE_STATS:
732
+ if key not in slave_data:
733
+ continue
734
+
735
+ value = slave_data[key]
736
+ if channel_name:
737
+ key = '{0}_{1}'.format(key, channel_name)
738
+ data[key] = func(value)
739
+
740
+ return data
741
+
742
+ def add_repl_channel_charts(self, name):
743
+ self.add_new_charts(slave_status_chart_template, name)
744
+
745
def get_userstats(self, raw_data):
746
+ # raw_data['user_statistics'] contains the following data structure:
747
+ # (
748
+ # (
749
+ # ('netdata', 42L, 0L, 1264L, 3.111252999999968, 2.968510299999994, 110267L, 19741424L, 0L, 0L, 1265L, 0L,
750
+ # 0L, 0L, 3L, 0L, 1301L, 0L, 0L, 7633L, 0L, 83L, 44L, 0L, 0L),
751
+ # ('root', 60L, 0L, 184L, 0.22856499999999966, 0.1601419999999998, 11605L, 1516513L, 0L, 9L, 220L, 0L, 2L, 1L,
752
+ # 6L, 4L,127L, 0L, 0L, 45L, 0L, 45L, 0L, 0L, 0L)
753
+ # ),
754
+ # (
755
+ # ('User', 253, 9, 128, 128, 0, 0),
756
+ # ('Total_connections', 3, 2, 11, 11, 0, 0),
757
+ # ('Concurrent_connections', 3, 1, 11, 11, 0, 0),
758
+ # ('Connected_time', 3, 4, 11, 11, 0, 0),
759
+ # ('Busy_time', 5, 21, 21, 21, 31, 0),
760
+ # ('Cpu_time', 5, 18, 21, 21, 31, 0),
761
+ # ('Bytes_received', 8, 6, 21, 21, 0, 0),
762
+ # ('Bytes_sent', 8, 8, 21, 21, 0, 0),
763
+ # ('Binlog_bytes_written', 8, 1, 21, 21, 0, 0),
764
+ # ('Rows_read', 8, 1, 21, 21, 0, 0),
765
+ # ('Rows_sent', 8, 4, 21, 21, 0, 0),
766
+ # ('Rows_deleted', 8, 1, 21, 21, 0, 0),
767
+ # ('Rows_inserted', 8, 1, 21, 21, 0, 0),
768
+ # ('Rows_updated', 8, 1, 21, 21, 0, 0),
769
+ # ('Select_commands', 8, 1, 21, 21, 0, 0),
770
+ # ('Update_commands', 8, 1, 21, 21, 0, 0),
771
+ # ('Other_commands', 8, 4, 21, 21, 0, 0),
772
+ # ('Commit_transactions', 8, 1, 21, 21, 0, 0),
773
+ # ('Rollback_transactions', 8, 1, 21, 21, 0, 0),
774
+ # ('Denied_connections', 8, 4, 21, 21, 0, 0),
775
+ # ('Lost_connections', 8, 1, 21, 21, 0, 0),
776
+ # ('Access_denied', 8, 2, 21, 21, 0, 0),
777
+ # ('Empty_queries', 8, 2, 21, 21, 0, 0),
778
+ # ('Total_ssl_connections', 8, 1, 21, 21, 0, 0),
779
+ # ('Max_statement_time_exceeded', 8, 1, 21, 21, 0, 0)),
780
+ # )
781
data = dict()
782
userstats_vars = [e[0] for e in raw_data['user_statistics'][1]]
783
for i, _ in enumerate(raw_data['user_statistics'][0]):
@@ -742,7 +799,10 @@ class Service(MySQLService):
799
self.charts['userstats_cpu'].add_dimension(['userstats_{0}_Cpu_time'.format(name), name, 'incremental', 100, 1])
800
801
def create_new_userstats_charts(self, tube):
745
- order, charts = userstats_chart_template(tube)
802
+ self.add_new_charts(userstats_chart_template, tube)
803
+
804
+ def add_new_charts(self, template, *params):
805
+ order, charts = template(*params)
806
807
for chart_name in order:
808
params = [chart_name] + charts[chart_name]['options']