| 1 | # you can disable an alarm notification by setting the 'to' line to: silent |
| 2 | |
| 3 | template: clickhouse_restarted |
| 4 | on: clickhouse.uptime |
| 5 | class: Error |
| 6 | type: Database |
| 7 | component: ClickHouse |
| 8 | calc: $uptime |
| 9 | units: seconds |
| 10 | every: 10s |
| 11 | warn: $this > 1 AND $this < 180 |
| 12 | summary: ClickHouse restart detected |
| 13 | info: ClickHouse has recently been restarted |
| 14 | to: silent |
| 15 | |
| 16 | template: clickhouse_queries_preempted |
| 17 | on: clickhouse.queries_preempted |
| 18 | class: Workload |
| 19 | type: Database |
| 20 | component: ClickHouse |
| 21 | lookup: max -1m unaligned |
| 22 | units: preempted_queries |
| 23 | every: 10s |
| 24 | warn: $this > 0 |
| 25 | delay: down 5m multiplier 1.5 max 1h |
| 26 | summary: ClickHouse preempted queries detected |
| 27 | info: ClickHouse has queries that are stopped and waiting due to priority setting |
| 28 | to: dba |
| 29 | |
| 30 | template: clickhouse_long_running_query |
| 31 | on: clickhouse.longest_running_query_time |
| 32 | class: Latency |
| 33 | type: Database |
| 34 | component: ClickHouse |
| 35 | lookup: max -1m unaligned |
| 36 | units: seconds |
| 37 | every: 10s |
| 38 | warn: $this > (($status >= $WARNING) ? (300) : (600)) |
| 39 | delay: down 5m multiplier 1.5 max 1h |
| 40 | summary: ClickHouse long-running query detected |
| 41 | info: ClickHouse has a long-running query exceeding the threshold |
| 42 | to: dba |
| 43 | |
| 44 | template: clickhouse_rejected_inserts |
| 45 | on: clickhouse.rejected_inserts |
| 46 | class: Workload |
| 47 | type: Database |
| 48 | component: ClickHouse |
| 49 | lookup: sum -1m unaligned |
| 50 | units: rejected_inserts |
| 51 | every: 10s |
| 52 | warn: $this > 0 |
| 53 | delay: down 5m multiplier 1.5 max 1h |
| 54 | summary: ClickHouse rejected INSERT queries detected |
| 55 | info: ClickHouse has INSERT queries that are rejected due to high number of active data parts for partition in a MergeTree |
| 56 | to: dba |
| 57 | |
| 58 | template: clickhouse_delayed_inserts |
| 59 | on: clickhouse.delayed_inserts |
| 60 | class: Workload |
| 61 | type: Database |
| 62 | component: ClickHouse |
| 63 | lookup: sum -1m unaligned |
| 64 | units: delayed_inserts |
| 65 | every: 10s |
| 66 | warn: $this > 0 |
| 67 | delay: down 5m multiplier 1.5 max 1h |
| 68 | summary: ClickHouse delayed INSERT queries detected |
| 69 | info: ClickHouse has INSERT queries that are throttled due to high number of active data parts for partition in a MergeTree |
| 70 | to: silent |
| 71 | |
| 72 | template: clickhouse_replication_lag |
| 73 | on: clickhouse.replicas_max_absolute_delay |
| 74 | class: Workload |
| 75 | type: Database |
| 76 | component: ClickHouse |
| 77 | lookup: avg -1m unaligned |
| 78 | units: seconds |
| 79 | every: 10s |
| 80 | warn: $this > (($status >= $WARNING) ? (250) : (300)) |
| 81 | delay: down 5m multiplier 1.5 max 1h |
| 82 | summary: ClickHouse high replication lag detected |
| 83 | info: ClickHouse is experiencing replication lag greater than 5 minutes |
| 84 | to: dba |
| 85 | |
| 86 | template: clickhouse_replicated_readonly_tables |
| 87 | on: clickhouse.replicated_readonly_tables |
| 88 | class: Error |
| 89 | type: Database |
| 90 | component: ClickHouse |
| 91 | lookup: max -1m unaligned |
| 92 | units: readonly_tables |
| 93 | every: 10s |
| 94 | warn: $this > 0 |
| 95 | delay: down 5m multiplier 1.5 max 1h |
| 96 | summary: ClickHouse replicated tables in readonly state detected |
| 97 | info: ClickHouse replicated tables are read-only due to startup without ZooKeeper configured or ZooKeeper session loss. |
| 98 | to: dba |
| 99 | |
| 100 | template: clickhouse_max_part_count_for_partition |
| 101 | on: clickhouse.max_part_count_for_partition |
| 102 | class: Workload |
| 103 | type: Database |
| 104 | component: ClickHouse |
| 105 | lookup: avg -1m unaligned |
| 106 | units: parts |
| 107 | every: 10s |
| 108 | warn: $this > (($status >= $WARNING) ? (200) : (300)) |
| 109 | delay: down 5m multiplier 1.5 max 1h |
| 110 | summary: ClickHouse high parts/partition detected |
| 111 | info: ClickHouse high number of parts per partition |
| 112 | to: dba |
| 113 | |
| 114 | template: clickhouse_distributed_connections_failures |
| 115 | on: clickhouse.distributed_connections_fail_exhausted_retries |
| 116 | class: Error |
| 117 | type: Database |
| 118 | component: ClickHouse |
| 119 | lookup: sum -1m unaligned |
| 120 | units: failures |
| 121 | every: 10s |
| 122 | warn: $this > 0 |
| 123 | delay: down 5m multiplier 1.5 max 1h |
| 124 | summary: ClickHouse distributed connections failures detected |
| 125 | info: ClickHouse has failed distributed connections after exhausting all retry attempts |
| 126 | to: dba |
| 127 | |
| 128 | template: clickhouse_distributed_files_to_insert |
| 129 | on: clickhouse.distributed_files_to_insert |
| 130 | class: Workload |
| 131 | type: Database |
| 132 | component: ClickHouse |
| 133 | lookup: max -1m unaligned |
| 134 | units: files |
| 135 | every: 10s |
| 136 | warn: $this > (($status >= $WARNING) ? (40) : (80)) |
| 137 | delay: down 5m multiplier 1.5 max 1h |
| 138 | summary: ClickHouse high files to insert detected |
| 139 | info: ClickHouse high number of pending files to process for asynchronous insertion into Distributed tables |
| 140 | to: silent |