add clickhouse alerts (#17760)
Ilya Mashchenko committed
May 27, 2024 at 21:10 UTC
a89905ad3acf80d8c2c441f9fff338e34b47695b
2 files changed
+187
-1
src/go/collectors/go.d.plugin/modules/clickhouse/metadata.yaml
+47
-1
@@ -173,7 +173,47 @@ modules:
173
troubleshooting:
174
problems:
175
list: []
176
- alerts: []
176
+ alerts:
177
+ - name: clickhouse_restarted
178
+ metric: clickhouse.uptime
179
+ info: ClickHouse has recently been restarted
180
+ link: https://github.com/netdata/netdata/blob/master/src/health/health.d/clickhouse.conf
181
+ - name: clickhouse_queries_preempted
182
+ metric: clickhouse.queries_preempted
183
+ info: ClickHouse has queries that are stopped and waiting due to priority setting
184
+ link: https://github.com/netdata/netdata/blob/master/src/health/health.d/clickhouse.conf
185
+ - name: clickhouse_long_running_query
186
+ metric: clickhouse.longest_running_query_time
187
+ info: ClickHouse has a long-running query exceeding the threshold
188
+ link: https://github.com/netdata/netdata/blob/master/src/health/health.d/clickhouse.conf
189
+ - name: clickhouse_rejected_inserts
190
+ metric: clickhouse.rejected_inserts
191
+ info: ClickHouse has INSERT queries that are rejected due to high number of active data parts for partition in a MergeTree
192
+ link: https://github.com/netdata/netdata/blob/master/src/health/health.d/clickhouse.conf
193
+ - name: clickhouse_delayed_inserts
194
+ metric: clickhouse.delayed_inserts
195
+ info: ClickHouse has INSERT queries that are throttled due to high number of active data parts for partition in a MergeTree
196
+ link: https://github.com/netdata/netdata/blob/master/src/health/health.d/clickhouse.conf
197
+ - name: clickhouse_replication_lag
198
+ metric: clickhouse.replicas_max_absolute_delay
199
+ info: ClickHouse is experiencing replication lag greater than 5 minutes
200
+ link: https://github.com/netdata/netdata/blob/master/src/health/health.d/clickhouse.conf
201
+ - name: clickhouse_replicated_readonly_tables
202
+ metric: clickhouse.replicated_readonly_tables
203
+ info: ClickHouse has replicated tables in readonly state due to ZooKeeper session loss/startup without ZooKeeper configured
204
+ link: https://github.com/netdata/netdata/blob/master/src/health/health.d/clickhouse.conf
205
+ - name: clickhouse_max_part_count_for_partition
206
+ metric: clickhouse.max_part_count_for_partition
207
+ info: ClickHouse high number of parts per partition
208
+ link: https://github.com/netdata/netdata/blob/master/src/health/health.d/clickhouse.conf
209
+ - name: clickhouse_distributed_connections_failures
210
+ metric: clickhouse.distributed_connections_fail_exhausted_retries
211
+ info: ClickHouse has failed distributed connections after exhausting all retry attempts
212
+ link: https://github.com/netdata/netdata/blob/master/src/health/health.d/clickhouse.conf
213
+ - name: clickhouse_distributed_files_to_insert
214
+ metric: clickhouse.distributed_files_to_insert
215
+ info: ClickHouse high number of pending files to process for asynchronous insertion into Distributed tables
216
+ link: https://github.com/netdata/netdata/blob/master/src/health/health.d/clickhouse.conf
217
metrics:
218
folding:
219
title: Metrics
@@ -252,6 +292,12 @@ modules:
292
chart_type: line
293
dimensions:
294
- name: mem_limit_exceeded
295
+ - name: clickhouse.longest_running_query_time
296
+ description: Longest running query time
297
+ unit: seconds
298
+ chart_type: line
299
+ dimensions:
300
+ - name: longest_query_time
301
- name: clickhouse.queries_latency
302
description: Queries latency
303
unit: microseconds
src/health/health.d/clickhouse.conf
new
+140
@@ -0,0 +1,140 @@
1
+# you can disable an alarm notification by setting the 'to' line to: silent
2
+
3
+ template: clickhouse_restarted
4
+ on: clickhouse.uptime
5
+ class: Error
6
+ type: Database
7
+component: ClickHouse
8
+ calc: $uptime
9
+ units: seconds
10
+ every: 10s
11
+ warn: $this > 1 AND $this < 180
12
+ summary: ClickHouse restart detected
13
+ info: ClickHouse has recently been restarted
14
+ to: silent
15
+
16
+ template: clickhouse_queries_preempted
17
+ on: clickhouse.queries_preempted
18
+ class: Workload
19
+ type: Database
20
+component: ClickHouse
21
+ lookup: max -1m unaligned
22
+ units: preempted_queries
23
+ every: 10s
24
+ warn: $this > 0
25
+ delay: down 5m multiplier 1.5 max 1h
26
+ summary: ClickHouse preempted queries detected
27
+ info: ClickHouse has queries that are stopped and waiting due to priority setting
28
+ to: dba
29
+
30
+ template: clickhouse_long_running_query
31
+ on: clickhouse.longest_running_query_time
32
+ class: Latency
33
+ type: Database
34
+component: ClickHouse
35
+ lookup: max -1m unaligned
36
+ units: seconds
37
+ every: 10s
38
+ warn: $this > (($status >= $WARNING) ? (300) : (600))
39
+ delay: down 5m multiplier 1.5 max 1h
40
+ summary: ClickHouse long-running query detected
41
+ info: ClickHouse has a long-running query exceeding the threshold
42
+ to: dba
43
+
44
+ template: clickhouse_rejected_inserts
45
+ on: clickhouse.rejected_inserts
46
+ class: Workload
47
+ type: Database
48
+component: ClickHouse
49
+ lookup: sum -1m unaligned
50
+ units: rejected_inserts
51
+ every: 10s
52
+ warn: $this > 0
53
+ delay: down 5m multiplier 1.5 max 1h
54
+ summary: ClickHouse rejected INSERT queries detected
55
+ info: ClickHouse has INSERT queries that are rejected due to high number of active data parts for partition in a MergeTree
56
+ to: dba
57
+
58
+ template: clickhouse_delayed_inserts
59
+ on: clickhouse.delayed_inserts
60
+ class: Workload
61
+ type: Database
62
+component: ClickHouse
63
+ lookup: sum -1m unaligned
64
+ units: delayed_inserts
65
+ every: 10s
66
+ warn: $this > 0
67
+ delay: down 5m multiplier 1.5 max 1h
68
+ summary: ClickHouse delayed INSERT queries detected
69
+ info: ClickHouse has INSERT queries that are throttled due to high number of active data parts for partition in a MergeTree
70
+ to: silent
71
+
72
+ template: clickhouse_replication_lag
73
+ on: clickhouse.replicas_max_absolute_delay
74
+ class: Workload
75
+ type: Database
76
+component: ClickHouse
77
+ lookup: avg -1m unaligned
78
+ units: seconds
79
+ every: 10s
80
+ warn: $this > (($status >= $WARNING) ? (250) : (300))
81
+ delay: down 5m multiplier 1.5 max 1h
82
+ summary: ClickHouse high replication lag detected
83
+ info: ClickHouse is experiencing replication lag greater than 5 minutes
84
+ to: dba
85
+
86
+ template: clickhouse_replicated_readonly_tables
87
+ on: clickhouse.replicated_readonly_tables
88
+ class: Error
89
+ type: Database
90
+component: ClickHouse
91
+ lookup: max -1m unaligned
92
+ units: readonly_tables
93
+ every: 10s
94
+ warn: $this > 0
95
+ delay: down 5m multiplier 1.5 max 1h
96
+ summary: ClickHouse replicated tables in readonly state detected
97
+ info: ClickHouse has replicated tables in readonly state due to ZooKeeper session loss/startup without ZooKeeper configured
98
+ to: dba
99
+
100
+ template: clickhouse_max_part_count_for_partition
101
+ on: clickhouse.max_part_count_for_partition
102
+ class: Workload
103
+ type: Database
104
+component: ClickHouse
105
+ lookup: avg -1m unaligned
106
+ units: parts
107
+ every: 10s
108
+ warn: $this > (($status >= $WARNING) ? (200) : (300))
109
+ delay: down 5m multiplier 1.5 max 1h
110
+ summary: ClickHouse high parts/partition detected
111
+ info: ClickHouse high number of parts per partition
112
+ to: dba
113
+
114
+ template: clickhouse_distributed_connections_failures
115
+ on: clickhouse.distributed_connections_fail_exhausted_retries
116
+ class: Error
117
+ type: Database
118
+component: ClickHouse
119
+ lookup: sum -1m unaligned
120
+ units: failures
121
+ every: 10s
122
+ warn: $this > 0
123
+ delay: down 5m multiplier 1.5 max 1h
124
+ summary: ClickHouse distributed connections failures detected
125
+ info: ClickHouse has failed distributed connections after exhausting all retry attempts
126
+ to: dba
127
+
128
+ template: clickhouse_distributed_files_to_insert
129
+ on: clickhouse.distributed_files_to_insert
130
+ class: Workload
131
+ type: Database
132
+component: ClickHouse
133
+ lookup: max -1m unaligned
134
+ units: files
135
+ every: 10s
136
+ warn: $this > (($status >= $WARNING) ? (40) : (80))
137
+ delay: down 5m multiplier 1.5 max 1h
138
+ summary: ClickHouse high files to insert detected
139
+ info: ClickHouse high number of pending files to process for asynchronous insertion into Distributed tables
140
+ to: silent