Adding more postgres metrics (#11169)
* Adding more postgres metrics * standby lag time * average blocking transactions
Filip Plata committed
May 24, 2021 at 13:34 UTC
ca571a05f797009c87ae9b5819c1aab35c0aecb4
2 files changed
+122
-6
collectors/python.d.plugin/postgres/README.md
+17
@@ -68,6 +68,23 @@ Following charts are drawn:
68
69
- locks
70
71
+12. **Standby delta** KB
72
+
73
+ - sent delta
74
+ - write delta
75
+ - flush delta
76
+ - replay delta
77
+
78
+13. **Standby lag** seconds
79
+
80
+ - write lag
81
+ - flush lag
82
+ - replay lag
83
+
84
+14. **Average number of blocking transactions in db** processes
85
+
86
+ - blocking
87
+
88
## Configuration
89
90
Edit the `python.d/postgres.conf` configuration file using `edit-config` from the Netdata [config
collectors/python.d.plugin/postgres/postgres.chart.py
+105
-6
@@ -45,10 +45,12 @@ QUERY_NAME_INDEX_STATS = 'INDEX_STATS'
45
QUERY_NAME_DATABASE = 'DATABASE'
46
QUERY_NAME_BGWRITER = 'BGWRITER'
47
QUERY_NAME_LOCKS = 'LOCKS'
48
+QUERY_NAME_BLOCKERS = 'BLOCKERS'
49
QUERY_NAME_DATABASES = 'DATABASES'
50
QUERY_NAME_STANDBY = 'STANDBY'
51
QUERY_NAME_REPLICATION_SLOT = 'REPLICATION_SLOT'
52
QUERY_NAME_STANDBY_DELTA = 'STANDBY_DELTA'
53
+QUERY_NAME_STANDBY_LAG = 'STANDBY_LAG'
54
QUERY_NAME_REPSLOT_FILES = 'REPSLOT_FILES'
55
QUERY_NAME_IF_SUPERUSER = 'IF_SUPERUSER'
56
QUERY_NAME_SERVER_VERSION = 'SERVER_VERSION'
@@ -123,6 +125,9 @@ METRICS = {
125
'ShareLock',
126
'RowExclusiveLock'
127
],
128
+ QUERY_NAME_BLOCKERS: [
129
+ 'blocking_pids_avg'
130
+ ],
131
QUERY_NAME_AUTOVACUUM: [
132
'analyze',
133
'vacuum_analyze',
@@ -136,6 +141,11 @@ METRICS = {
141
'flush_delta',
142
'replay_delta'
143
],
144
+ QUERY_NAME_STANDBY_LAG: [
145
+ 'write_lag',
146
+ 'flush_lag',
147
+ 'replay_lag'
148
+ ],
149
QUERY_NAME_REPSLOT_FILES: [
150
'replslot_wal_keep',
151
'replslot_files'
@@ -263,7 +273,7 @@ FROM (
273
FROM pg_catalog.pg_stat_activity
274
WHERE backend_type IN ('client backend', 'background worker')
275
UNION ALL
266
- SELECT 'r', COUNT(1)
276
+ SELECT 'r', COUNT(1)
277
FROM pg_catalog.pg_stat_replication
278
) as s;
279
""",
@@ -277,7 +287,7 @@ FROM (
287
FROM pg_catalog.pg_stat_activity
288
WHERE query NOT LIKE 'autovacuum: %%'
289
UNION ALL
280
- SELECT 'r', COUNT(1)
290
+ SELECT 'r', COUNT(1)
291
FROM pg_catalog.pg_stat_replication
292
) as s;
293
""",
@@ -291,7 +301,7 @@ FROM (
301
FROM pg_catalog.pg_stat_activity
302
WHERE current_query NOT LIKE 'autovacuum: %%'
303
UNION ALL
294
- SELECT 'r', COUNT(1)
304
+ SELECT 'r', COUNT(1)
305
FROM pg_catalog.pg_stat_replication
306
) as s;
307
""",
@@ -386,6 +396,48 @@ ORDER BY datname, mode;
396
""",
397
}
398
399
+QUERY_BLOCKERS = {
400
+ DEFAULT: """
401
+WITH B AS (
402
+SELECT DISTINCT
403
+ pg_database.datname as database_name,
404
+ pg_locks.pid,
405
+ cardinality(pg_blocking_pids(pg_locks.pid)) AS blocking_pids
406
+FROM pg_locks
407
+INNER JOIN pg_database ON pg_database.oid = pg_locks.database
408
+WHERE NOT pg_locks.granted)
409
+SELECT database_name, AVG(blocking_pids) AS blocking_pids_avg
410
+FROM B
411
+GROUP BY database_name
412
+""",
413
+ V96: """
414
+WITH B AS (
415
+SELECT DISTINCT
416
+ pg_database.datname as database_name,
417
+ blocked_locks.pid AS blocked_pid,
418
+ COUNT(blocking_locks.pid) AS blocking_pids
419
+FROM pg_catalog.pg_locks blocked_locks
420
+INNER JOIN pg_database ON pg_database.oid = blocked_locks.database
421
+JOIN pg_catalog.pg_locks blocking_locks
422
+ ON blocking_locks.locktype = blocked_locks.locktype
423
+ AND blocking_locks.database IS NOT DISTINCT FROM blocked_locks.database
424
+ AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation
425
+ AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page
426
+ AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple
427
+ AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid
428
+ AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid
429
+ AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid
430
+ AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid
431
+ AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid
432
+ AND blocking_locks.pid != blocked_locks.pid
433
+WHERE NOT blocked_locks.GRANTED
434
+GROUP BY database_name, blocked_pid)
435
+SELECT database_name, AVG(blocking_pids) AS blocking_pids_avg
436
+FROM B
437
+GROUP BY database_name
438
+"""
439
+}
440
+
441
QUERY_DATABASES = {
442
DEFAULT: """
443
SELECT
@@ -478,6 +530,18 @@ WHERE application_name IS NOT NULL;
530
""",
531
}
532
533
+QUERY_STANDBY_LAG = {
534
+ DEFAULT: """
535
+SELECT
536
+ application_name,
537
+ COALESCE(EXTRACT(EPOCH FROM write_lag)::bigint, 0) AS write_lag,
538
+ COALESCE(EXTRACT(EPOCH FROM flush_lag)::bigint, 0) AS flush_lag,
539
+ COALESCE(EXTRACT(EPOCH FROM replay_lag)::bigint, 0) AS replay_lag
540
+FROM pg_stat_replication
541
+WHERE application_name IS NOT NULL;
542
+"""
543
+}
544
+
545
QUERY_REPSLOT_FILES = {
546
DEFAULT: """
547
WITH wal_size AS (
@@ -632,6 +696,10 @@ def query_factory(name, version=NO_VERSION):
696
return QUERY_BGWRITER[DEFAULT]
697
elif name == QUERY_NAME_LOCKS:
698
return QUERY_LOCKS[DEFAULT]
699
+ elif name == QUERY_NAME_BLOCKERS:
700
+ if version < 90600:
701
+ return QUERY_BLOCKERS[V96]
702
+ return QUERY_BLOCKERS[DEFAULT]
703
elif name == QUERY_NAME_DATABASES:
704
return QUERY_DATABASES[DEFAULT]
705
elif name == QUERY_NAME_STANDBY:
@@ -656,6 +724,8 @@ def query_factory(name, version=NO_VERSION):
724
if version < 100000:
725
return QUERY_STANDBY_DELTA[V96]
726
return QUERY_STANDBY_DELTA[DEFAULT]
727
+ elif name == QUERY_NAME_STANDBY_LAG:
728
+ return QUERY_STANDBY_LAG[DEFAULT]
729
elif name == QUERY_NAME_REPSLOT_FILES:
730
if version < 110000:
731
return QUERY_REPSLOT_FILES[V10]
@@ -676,6 +746,7 @@ ORDER = [
746
'db_stat_tuple_write',
747
'db_stat_transactions',
748
'db_stat_connections',
749
+ 'db_stat_blocking_pids_avg',
750
'database_size',
751
'backend_process',
752
'backend_usage',
@@ -695,6 +766,7 @@ ORDER = [
766
'stat_bgwriter_maxwritten',
767
'replication_slot',
768
'standby_delta',
769
+ 'standby_lag',
770
'autovacuum'
771
]
772
@@ -752,6 +824,13 @@ CHARTS = {
824
['temp_files', 'files', 'incremental']
825
]
826
},
827
+ 'db_stat_blocking_pids_avg': {
828
+ 'options': [None, 'Average number of blocking transactions in db', 'processes', 'db statistics',
829
+ 'postgres.db_stat_blocking_pids_avg', 'line'],
830
+ 'lines': [
831
+ ['blocking_pids_avg', 'blocking', 'absolute']
832
+ ]
833
+ },
834
'database_size': {
835
'options': [None, 'Database size', 'MiB', 'database size', 'postgres.db_size', 'stacked'],
836
'lines': [
@@ -884,6 +963,14 @@ CHARTS = {
963
['replay_delta', 'replay delta', 'absolute', 1, 1024]
964
]
965
},
966
+ 'standby_lag': {
967
+ 'options': [None, 'Standby lag', 'seconds', 'replication lag', 'postgres.standby_lag', 'line'],
968
+ 'lines': [
969
+ ['write_lag', 'write lag', 'absolute'],
970
+ ['flush_lag', 'flush lag', 'absolute'],
971
+ ['replay_lag', 'replay lag', 'absolute']
972
+ ]
973
+ },
974
'replication_slot': {
975
'options': [None, 'Replication slot files', 'files', 'replication slot', 'postgres.replication_slot', 'line'],
976
'lines': [
@@ -1073,6 +1160,7 @@ class Service(SimpleService):
1160
self.queries[query_factory(QUERY_NAME_BGWRITER)] = METRICS[QUERY_NAME_BGWRITER]
1161
self.queries[query_factory(QUERY_NAME_DIFF_LSN, self.server_version)] = METRICS[QUERY_NAME_WAL_WRITES]
1162
self.queries[query_factory(QUERY_NAME_STANDBY_DELTA, self.server_version)] = METRICS[QUERY_NAME_STANDBY_DELTA]
1163
+ self.queries[query_factory(QUERY_NAME_BLOCKERS, self.server_version)] = METRICS[QUERY_NAME_BLOCKERS]
1164
1165
if self.do_index_stats:
1166
self.queries[query_factory(QUERY_NAME_INDEX_STATS)] = METRICS[QUERY_NAME_INDEX_STATS]
@@ -1092,6 +1180,9 @@ class Service(SimpleService):
1180
if self.server_version >= 90400:
1181
self.queries[query_factory(QUERY_NAME_AUTOVACUUM)] = METRICS[QUERY_NAME_AUTOVACUUM]
1182
1183
+ if self.server_version >= 100000:
1184
+ self.queries[query_factory(QUERY_NAME_STANDBY_LAG)] = METRICS[QUERY_NAME_STANDBY_LAG]
1185
+
1186
def create_dynamic_charts(self):
1187
for database_name in self.databases[::-1]:
1188
dim = [
@@ -1116,11 +1207,19 @@ class Service(SimpleService):
1207
)
1208
1209
for application_name in self.secondaries[::-1]:
1119
- add_replication_delta_chart(
1210
+ add_replication_standby_chart(
1211
order=self.order,
1212
definitions=self.definitions,
1213
name='standby_delta',
1214
application_name=application_name,
1215
+ chart_family='replication delta',
1216
+ )
1217
+ add_replication_standby_chart(
1218
+ order=self.order,
1219
+ definitions=self.definitions,
1220
+ name='standby_lag',
1221
+ application_name=application_name,
1222
+ chart_family='replication lag',
1223
)
1224
1225
for slot_name in self.replication_slots[::-1]:
@@ -1199,7 +1298,7 @@ def add_database_stat_chart(order, definitions, name, database_name):
1298
'lines': create_lines(database_name, chart_template['lines'])}
1299
1300
1202
-def add_replication_delta_chart(order, definitions, name, application_name):
1301
+def add_replication_standby_chart(order, definitions, name, application_name, chart_family):
1302
def create_lines(standby, lines):
1303
result = list()
1304
for line in lines:
@@ -1213,7 +1312,7 @@ def add_replication_delta_chart(order, definitions, name, application_name):
1312
order.insert(position, chart_name)
1313
name, title, units, _, context, chart_type = chart_template['options']
1314
definitions[chart_name] = {
1216
- 'options': [name, title + ': ' + application_name, units, 'replication delta', context, chart_type],
1315
+ 'options': [name, title + ': ' + application_name, units, chart_family, context, chart_type],
1316
'lines': create_lines(application_name, chart_template['lines'])}
1317
1318