postgres add ssl connection support (#5661)
* add ssl connection support * 9.1 is deprecated, use current doc * make sslmode mandatory if any other ssl option is set + minor * ssl params check fix
Ilya Mashchenko committed
Mar 26, 2019 at 08:52 UTC
51c75a78c6ec9ae2da8cd308e620fc8a6639da33
2 files changed
+126
-80
collectors/python.d.plugin/postgres/postgres.chart.py
+117
-78
@@ -23,27 +23,41 @@ DEFAULT_CONNECT_TIMEOUT = 2 # seconds
23
DEFAULT_STATEMENT_TIMEOUT = 5000 # ms
24
25
26
-WAL = 'WAL'
27
-ARCHIVE = 'ARCHIVE'
28
-BACKENDS = 'BACKENDS'
29
-TABLE_STATS = 'TABLE_STATS'
30
-INDEX_STATS = 'INDEX_STATS'
31
-DATABASE = 'DATABASE'
32
-BGWRITER = 'BGWRITER'
33
-LOCKS = 'LOCKS'
34
-DATABASES = 'DATABASES'
35
-STANDBY = 'STANDBY'
36
-REPLICATION_SLOT = 'REPLICATION_SLOT'
37
-STANDBY_DELTA = 'STANDBY_DELTA'
38
-REPSLOT_FILES = 'REPSLOT_FILES'
39
-IF_SUPERUSER = 'IF_SUPERUSER'
40
-SERVER_VERSION = 'SERVER_VERSION'
41
-AUTOVACUUM = 'AUTOVACUUM'
42
-DIFF_LSN = 'DIFF_LSN'
43
-WAL_WRITES = 'WAL_WRITES'
26
+CONN_PARAM_HOST = 'host'
27
+CONN_PARAM_PORT = 'port'
28
+CONN_PARAM_DATABASE = 'database'
29
+CONN_PARAM_USER = 'user'
30
+CONN_PARAM_PASSWORD = 'password'
31
+CONN_PARAM_CONN_TIMEOUT = 'connect_timeout'
32
+CONN_PARAM_STATEMENT_TIMEOUT = 'statement_timeout'
33
+CONN_PARAM_SSL_MODE = 'sslmode'
34
+CONN_PARAM_SSL_ROOT_CERT = 'sslrootcert'
35
+CONN_PARAM_SSL_CRL = 'sslcrl'
36
+CONN_PARAM_SSL_CERT = 'sslcert'
37
+CONN_PARAM_SSL_KEY = 'sslkey'
38
+
39
+
40
+QUERY_NAME_WAL = 'WAL'
41
+QUERY_NAME_ARCHIVE = 'ARCHIVE'
42
+QUERY_NAME_BACKENDS = 'BACKENDS'
43
+QUERY_NAME_TABLE_STATS = 'TABLE_STATS'
44
+QUERY_NAME_INDEX_STATS = 'INDEX_STATS'
45
+QUERY_NAME_DATABASE = 'DATABASE'
46
+QUERY_NAME_BGWRITER = 'BGWRITER'
47
+QUERY_NAME_LOCKS = 'LOCKS'
48
+QUERY_NAME_DATABASES = 'DATABASES'
49
+QUERY_NAME_STANDBY = 'STANDBY'
50
+QUERY_NAME_REPLICATION_SLOT = 'REPLICATION_SLOT'
51
+QUERY_NAME_STANDBY_DELTA = 'STANDBY_DELTA'
52
+QUERY_NAME_REPSLOT_FILES = 'REPSLOT_FILES'
53
+QUERY_NAME_IF_SUPERUSER = 'IF_SUPERUSER'
54
+QUERY_NAME_SERVER_VERSION = 'SERVER_VERSION'
55
+QUERY_NAME_AUTOVACUUM = 'AUTOVACUUM'
56
+QUERY_NAME_DIFF_LSN = 'DIFF_LSN'
57
+QUERY_NAME_WAL_WRITES = 'WAL_WRITES'
58
59
METRICS = {
46
- DATABASE: [
60
+ QUERY_NAME_DATABASE: [
61
'connections',
62
'xact_commit',
63
'xact_rollback',
@@ -59,32 +73,32 @@ METRICS = {
73
'temp_bytes',
74
'size'
75
],
62
- BACKENDS: [
76
+ QUERY_NAME_BACKENDS: [
77
'backends_active',
78
'backends_idle'
79
],
66
- INDEX_STATS: [
80
+ QUERY_NAME_INDEX_STATS: [
81
'index_count',
82
'index_size'
83
],
70
- TABLE_STATS: [
84
+ QUERY_NAME_TABLE_STATS: [
85
'table_size',
86
'table_count'
87
],
74
- WAL: [
88
+ QUERY_NAME_WAL: [
89
'written_wal',
90
'recycled_wal',
91
'total_wal'
92
],
79
- WAL_WRITES: [
93
+ QUERY_NAME_WAL_WRITES: [
94
'wal_writes'
95
],
82
- ARCHIVE: [
96
+ QUERY_NAME_ARCHIVE: [
97
'ready_count',
98
'done_count',
99
'file_count'
100
],
87
- BGWRITER: [
101
+ QUERY_NAME_BGWRITER: [
102
'checkpoint_scheduled',
103
'checkpoint_requested',
104
'buffers_checkpoint',
@@ -94,7 +108,7 @@ METRICS = {
108
'buffers_alloc',
109
'buffers_backend_fsync'
110
],
97
- LOCKS: [
111
+ QUERY_NAME_LOCKS: [
112
'ExclusiveLock',
113
'RowShareLock',
114
'SIReadLock',
@@ -105,20 +119,20 @@ METRICS = {
119
'ShareLock',
120
'RowExclusiveLock'
121
],
108
- AUTOVACUUM: [
122
+ QUERY_NAME_AUTOVACUUM: [
123
'analyze',
124
'vacuum_analyze',
125
'vacuum',
126
'vacuum_freeze',
127
'brin_summarize'
128
],
115
- STANDBY_DELTA: [
129
+ QUERY_NAME_STANDBY_DELTA: [
130
'sent_delta',
131
'write_delta',
132
'flush_delta',
133
'replay_delta'
134
],
121
- REPSLOT_FILES: [
135
+ QUERY_NAME_REPSLOT_FILES: [
136
'replslot_wal_keep',
137
'replslot_files'
138
]
@@ -518,47 +532,47 @@ SELECT
532
533
534
def query_factory(name, version=NO_VERSION):
521
- if name == BACKENDS:
535
+ if name == QUERY_NAME_BACKENDS:
536
return QUERY_BACKEND[DEFAULT]
523
- elif name == TABLE_STATS:
537
+ elif name == QUERY_NAME_TABLE_STATS:
538
return QUERY_TABLE_STATS[DEFAULT]
525
- elif name == INDEX_STATS:
539
+ elif name == QUERY_NAME_INDEX_STATS:
540
return QUERY_INDEX_STATS[DEFAULT]
527
- elif name == DATABASE:
541
+ elif name == QUERY_NAME_DATABASE:
542
return QUERY_DATABASE[DEFAULT]
529
- elif name == BGWRITER:
543
+ elif name == QUERY_NAME_BGWRITER:
544
return QUERY_BGWRITER[DEFAULT]
531
- elif name == LOCKS:
545
+ elif name == QUERY_NAME_LOCKS:
546
return QUERY_LOCKS[DEFAULT]
533
- elif name == DATABASES:
547
+ elif name == QUERY_NAME_DATABASES:
548
return QUERY_DATABASES[DEFAULT]
535
- elif name == STANDBY:
549
+ elif name == QUERY_NAME_STANDBY:
550
return QUERY_STANDBY[DEFAULT]
537
- elif name == REPLICATION_SLOT:
551
+ elif name == QUERY_NAME_REPLICATION_SLOT:
552
return QUERY_REPLICATION_SLOT[DEFAULT]
539
- elif name == IF_SUPERUSER:
553
+ elif name == QUERY_NAME_IF_SUPERUSER:
554
return QUERY_SUPERUSER[DEFAULT]
541
- elif name == SERVER_VERSION:
555
+ elif name == QUERY_NAME_SERVER_VERSION:
556
return QUERY_SHOW_VERSION[DEFAULT]
543
- elif name == AUTOVACUUM:
557
+ elif name == QUERY_NAME_AUTOVACUUM:
558
return QUERY_AUTOVACUUM[DEFAULT]
545
- elif name == WAL:
559
+ elif name == QUERY_NAME_WAL:
560
if version < 100000:
561
return QUERY_WAL[V96]
562
return QUERY_WAL[DEFAULT]
549
- elif name == ARCHIVE:
563
+ elif name == QUERY_NAME_ARCHIVE:
564
if version < 100000:
565
return QUERY_ARCHIVE[V96]
566
return QUERY_ARCHIVE[DEFAULT]
553
- elif name == STANDBY_DELTA:
567
+ elif name == QUERY_NAME_STANDBY_DELTA:
568
if version < 100000:
569
return QUERY_STANDBY_DELTA[V96]
570
return QUERY_STANDBY_DELTA[DEFAULT]
557
- elif name == REPSLOT_FILES:
571
+ elif name == QUERY_NAME_REPSLOT_FILES:
572
if version < 110000:
573
return QUERY_REPSLOT_FILES[V10]
574
return QUERY_REPSLOT_FILES[DEFAULT]
561
- elif name == DIFF_LSN:
575
+ elif name == QUERY_NAME_DIFF_LSN:
576
if version < 100000:
577
return QUERY_DIFF_LSN[V96]
578
return QUERY_DIFF_LSN[DEFAULT]
@@ -794,6 +808,7 @@ class Service(SimpleService):
808
self.databases_to_poll = configuration.pop('database_poll', None)
809
self.configuration = configuration
810
self.conn = None
811
+ self.conn_params = dict()
812
self.server_version = None
813
self.is_superuser = False
814
self.alive = False
@@ -806,26 +821,44 @@ class Service(SimpleService):
821
def reconnect(self):
822
return self.connect()
823
809
- def connect(self):
810
- if self.conn:
811
- self.conn.close()
812
- self.conn = None
813
-
824
+ def build_conn_params(self):
825
conf = self.configuration
826
827
params = {
817
- 'host': conf.get('host'),
818
- 'port': conf.get('port', DEFAULT_PORT),
819
- 'database': conf.get('database'),
820
- 'user': conf.get('user', DEFAULT_USER),
821
- 'password': conf.get('password'),
822
- 'connect_timeout': conf.get('connect_timeout', DEFAULT_CONNECT_TIMEOUT),
828
+ CONN_PARAM_HOST: conf.get(CONN_PARAM_HOST),
829
+ CONN_PARAM_PORT: conf.get(CONN_PARAM_PORT, DEFAULT_PORT),
830
+ CONN_PARAM_DATABASE: conf.get(CONN_PARAM_DATABASE),
831
+ CONN_PARAM_USER: conf.get(CONN_PARAM_USER, DEFAULT_USER),
832
+ CONN_PARAM_PASSWORD: conf.get(CONN_PARAM_PASSWORD),
833
+ CONN_PARAM_CONN_TIMEOUT: conf.get(CONN_PARAM_CONN_TIMEOUT, DEFAULT_CONNECT_TIMEOUT),
834
'options': '-c statement_timeout={0}'.format(
824
- conf.get('statement_timeout', DEFAULT_STATEMENT_TIMEOUT)),
835
+ conf.get(CONN_PARAM_STATEMENT_TIMEOUT, DEFAULT_STATEMENT_TIMEOUT)),
836
}
837
838
+ # https://www.postgresql.org/docs/current/libpq-ssl.html
839
+ ssl_params = dict(
840
+ (k, v) for k, v in {
841
+ CONN_PARAM_SSL_MODE: conf.get(CONN_PARAM_SSL_MODE),
842
+ CONN_PARAM_SSL_ROOT_CERT: conf.get(CONN_PARAM_SSL_ROOT_CERT),
843
+ CONN_PARAM_SSL_CRL: conf.get(CONN_PARAM_SSL_CRL),
844
+ CONN_PARAM_SSL_CERT: conf.get(CONN_PARAM_SSL_CERT),
845
+ CONN_PARAM_SSL_KEY: conf.get(CONN_PARAM_SSL_KEY),
846
+ }.items() if v)
847
+
848
+ if CONN_PARAM_SSL_MODE not in ssl_params and len(ssl_params) > 0:
849
+ raise ValueError("mandatory 'sslmode' param is missing, please set")
850
+
851
+ params.update(ssl_params)
852
+
853
+ return params
854
+
855
+ def connect(self):
856
+ if self.conn:
857
+ self.conn.close()
858
+ self.conn = None
859
+
860
try:
828
- self.conn = psycopg2.connect(**params)
861
+ self.conn = psycopg2.connect(**self.conn_params)
862
self.conn.set_isolation_level(extensions.ISOLATION_LEVEL_AUTOCOMMIT)
863
self.conn.set_session(readonly=True)
864
except OperationalError as error:
@@ -841,8 +874,14 @@ class Service(SimpleService):
874
self.error("'python-psycopg2' package is needed to use postgres module")
875
return False
876
877
+ try:
878
+ self.conn_params = self.build_conn_params()
879
+ except ValueError as error:
880
+ self.error('error on creating connection params : {0}', error)
881
+ return False
882
+
883
if not self.connect():
845
- self.error('failed to connect to {0}'.format(hide_password(self.configuration)))
884
+ self.error('failed to connect to {0}'.format(hide_password(self.conn_params)))
885
return False
886
887
try:
@@ -904,51 +943,51 @@ class Service(SimpleService):
943
def check_queries(self):
944
cursor = self.conn.cursor()
945
907
- self.server_version = detect_server_version(cursor, query_factory(SERVER_VERSION))
946
+ self.server_version = detect_server_version(cursor, query_factory(QUERY_NAME_SERVER_VERSION))
947
self.debug('server version: {0}'.format(self.server_version))
948
910
- self.is_superuser = check_if_superuser(cursor, query_factory(IF_SUPERUSER))
949
+ self.is_superuser = check_if_superuser(cursor, query_factory(QUERY_NAME_IF_SUPERUSER))
950
self.debug('superuser: {0}'.format(self.is_superuser))
951
913
- self.databases = discover(cursor, query_factory(DATABASES))
952
+ self.databases = discover(cursor, query_factory(QUERY_NAME_DATABASES))
953
self.debug('discovered databases {0}'.format(self.databases))
954
if self.databases_to_poll:
955
to_poll = self.databases_to_poll.split()
956
self.databases = [db for db in self.databases if db in to_poll] or self.databases
957
919
- self.secondaries = discover(cursor, query_factory(STANDBY))
958
+ self.secondaries = discover(cursor, query_factory(QUERY_NAME_STANDBY))
959
self.debug('discovered secondaries: {0}'.format(self.secondaries))
960
961
if self.server_version >= 94000:
923
- self.replication_slots = discover(cursor, query_factory(REPLICATION_SLOT))
962
+ self.replication_slots = discover(cursor, query_factory(QUERY_NAME_REPLICATION_SLOT))
963
self.debug('discovered replication slots: {0}'.format(self.replication_slots))
964
965
cursor.close()
966
967
def populate_queries(self):
929
- self.queries[query_factory(DATABASE)] = METRICS[DATABASE]
930
- self.queries[query_factory(BACKENDS)] = METRICS[BACKENDS]
931
- self.queries[query_factory(LOCKS)] = METRICS[LOCKS]
932
- self.queries[query_factory(BGWRITER)] = METRICS[BGWRITER]
933
- self.queries[query_factory(DIFF_LSN, self.server_version)] = METRICS[WAL_WRITES]
934
- self.queries[query_factory(STANDBY_DELTA, self.server_version)] = METRICS[STANDBY_DELTA]
968
+ self.queries[query_factory(QUERY_NAME_DATABASE)] = METRICS[QUERY_NAME_DATABASE]
969
+ self.queries[query_factory(QUERY_NAME_BACKENDS)] = METRICS[QUERY_NAME_BACKENDS]
970
+ self.queries[query_factory(QUERY_NAME_LOCKS)] = METRICS[QUERY_NAME_LOCKS]
971
+ self.queries[query_factory(QUERY_NAME_BGWRITER)] = METRICS[QUERY_NAME_BGWRITER]
972
+ self.queries[query_factory(QUERY_NAME_DIFF_LSN, self.server_version)] = METRICS[QUERY_NAME_WAL_WRITES]
973
+ self.queries[query_factory(QUERY_NAME_STANDBY_DELTA, self.server_version)] = METRICS[QUERY_NAME_STANDBY_DELTA]
974
975
if self.do_index_stats:
937
- self.queries[query_factory(INDEX_STATS)] = METRICS[INDEX_STATS]
976
+ self.queries[query_factory(QUERY_NAME_INDEX_STATS)] = METRICS[QUERY_NAME_INDEX_STATS]
977
if self.do_table_stats:
939
- self.queries[query_factory(TABLE_STATS)] = METRICS[TABLE_STATS]
978
+ self.queries[query_factory(QUERY_NAME_TABLE_STATS)] = METRICS[QUERY_NAME_TABLE_STATS]
979
980
if self.is_superuser:
942
- self.queries[query_factory(ARCHIVE, self.server_version)] = METRICS[ARCHIVE]
981
+ self.queries[query_factory(QUERY_NAME_ARCHIVE, self.server_version)] = METRICS[QUERY_NAME_ARCHIVE]
982
983
if self.server_version >= 90400:
945
- self.queries[query_factory(WAL, self.server_version)] = METRICS[WAL]
984
+ self.queries[query_factory(QUERY_NAME_WAL, self.server_version)] = METRICS[QUERY_NAME_WAL]
985
986
if self.server_version >= 100000:
948
- self.queries[query_factory(REPSLOT_FILES, self.server_version)] = METRICS[REPSLOT_FILES]
987
+ self.queries[query_factory(QUERY_NAME_REPSLOT_FILES, self.server_version)] = METRICS[QUERY_NAME_REPSLOT_FILES]
988
989
if self.server_version >= 90400:
951
- self.queries[query_factory(AUTOVACUUM)] = METRICS[AUTOVACUUM]
990
+ self.queries[query_factory(QUERY_NAME_AUTOVACUUM)] = METRICS[QUERY_NAME_AUTOVACUUM]
991
992
def create_dynamic_charts(self):
993
for database_name in self.databases[::-1]:
collectors/python.d.plugin/postgres/postgres.conf
+9
-2
@@ -68,8 +68,15 @@
68
# password : 'example_pass'
69
# host : 'localhost'
70
# port : 5432
71
-# connect_timeout : 2 # in seconds, default is 2
72
-# statement_timeout : 2000 # in ms, default is 2000
71
+# connect_timeout : 2 # in seconds, default is 2
72
+# statement_timeout : 2000 # in ms, default is 2000
73
+#
74
+# SSL connection parameters (https://www.postgresql.org/docs/current/libpq-ssl.html)
75
+# sslmode : mode # one of [disable, allow, prefer, require, verify-ca, verify-full]
76
+# sslrootcert : path/to/rootcert # the location of the root certificate file
77
+# sslcrl : path/to/crl # the location of the CRL file
78
+# sslcert : path/to/cert # the location of the client certificate file
79
+# sslkey : path/to/key # the location of the client key file
80
#
81
# Additionally, the following options allow selective disabling of charts
82
#