| 1 | # you can disable an alarm notification by setting the 'to' line to: silent |
| 2 | |
| 3 | # --- Cluster Health --- |
| 4 | |
| 5 | template: am_data_explorer_keep_alive |
| 6 | on: azure_monitor.data_explorer.keep_alive |
| 7 | class: Availability |
| 8 | type: Database |
| 9 | component: Data Explorer |
| 10 | lookup: average -5m unaligned of average |
| 11 | units: count |
| 12 | every: 1m |
| 13 | crit: $this < (($status == $CRITICAL) ? (1) : (0.5)) |
| 14 | delay: down 5m multiplier 1.5 max 1h |
| 15 | summary: Data Explorer keep alive on ${label:resource_name} |
| 16 | info: Cluster keep-alive health signal for Azure Data Explorer cluster ${label:resource_name} \ |
| 17 | in ${label:resource_group} (${label:region}). \ |
| 18 | A value below 1 indicates the cluster is not responding properly. |
| 19 | to: sysadmin |
| 20 | |
| 21 | template: am_data_explorer_cpu |
| 22 | on: azure_monitor.data_explorer.cpu_utilization |
| 23 | class: Utilization |
| 24 | type: Database |
| 25 | component: Data Explorer |
| 26 | lookup: average -5m unaligned of average |
| 27 | units: percentage |
| 28 | every: 1m |
| 29 | warn: $this > (($status >= $WARNING) ? (70) : (80)) |
| 30 | crit: $this > (($status == $CRITICAL) ? (80) : (95)) |
| 31 | delay: down 5m multiplier 1.5 max 1h |
| 32 | summary: Data Explorer CPU on ${label:resource_name} |
| 33 | info: Average CPU utilization of Azure Data Explorer cluster ${label:resource_name} \ |
| 34 | in ${label:resource_group} (${label:region}) |
| 35 | to: sysadmin |
| 36 | |
| 37 | template: am_data_explorer_ingestion_utilization |
| 38 | on: azure_monitor.data_explorer.utilization |
| 39 | class: Utilization |
| 40 | type: Database |
| 41 | component: Data Explorer |
| 42 | lookup: average -5m unaligned of ingestion |
| 43 | units: percentage |
| 44 | every: 1m |
| 45 | warn: $this > (($status >= $WARNING) ? (70) : (80)) |
| 46 | crit: $this > (($status == $CRITICAL) ? (80) : (95)) |
| 47 | delay: down 5m multiplier 1.5 max 1h |
| 48 | summary: Data Explorer ingestion utilization on ${label:resource_name} |
| 49 | info: Average ingestion utilization of Azure Data Explorer cluster ${label:resource_name} \ |
| 50 | in ${label:resource_group} (${label:region}). \ |
| 51 | High utilization indicates the cluster is approaching ingestion capacity. |
| 52 | to: sysadmin |
| 53 | |
| 54 | template: am_data_explorer_cache_utilization |
| 55 | on: azure_monitor.data_explorer.utilization |
| 56 | class: Utilization |
| 57 | type: Database |
| 58 | component: Data Explorer |
| 59 | lookup: average -5m unaligned of cache |
| 60 | units: percentage |
| 61 | every: 1m |
| 62 | warn: $this > (($status >= $WARNING) ? (75) : (80)) |
| 63 | crit: $this > (($status == $CRITICAL) ? (85) : (95)) |
| 64 | delay: down 5m multiplier 1.5 max 1h |
| 65 | summary: Data Explorer cache utilization on ${label:resource_name} |
| 66 | info: Average cache utilization factor of Azure Data Explorer cluster ${label:resource_name} \ |
| 67 | in ${label:resource_group} (${label:region}). \ |
| 68 | High cache utilization may cause queries to read from cold storage. |
| 69 | to: sysadmin |
| 70 | |
| 71 | template: am_data_explorer_throttled_commands |
| 72 | on: azure_monitor.data_explorer.throttled_commands |
| 73 | class: Errors |
| 74 | type: Database |
| 75 | component: Data Explorer |
| 76 | lookup: average -5m unaligned of total |
| 77 | units: commands/s |
| 78 | every: 1m |
| 79 | warn: $this > (($status >= $WARNING) ? (1) : (5)) |
| 80 | crit: $this > (($status == $CRITICAL) ? (5) : (20)) |
| 81 | delay: down 5m multiplier 1.5 max 1h |
| 82 | summary: Data Explorer throttled commands on ${label:resource_name} |
| 83 | info: Rate of throttled commands on Azure Data Explorer cluster ${label:resource_name} \ |
| 84 | in ${label:resource_group} (${label:region}). \ |
| 85 | Throttling indicates the cluster is overloaded. |
| 86 | to: sysadmin |
| 87 | |
| 88 | # --- Query Performance --- |
| 89 | |
| 90 | template: am_data_explorer_query_duration |
| 91 | on: azure_monitor.data_explorer.query_duration |
| 92 | class: Latency |
| 93 | type: Database |
| 94 | component: Data Explorer |
| 95 | lookup: average -5m unaligned of average |
| 96 | units: milliseconds |
| 97 | every: 1m |
| 98 | warn: $this > (($status >= $WARNING) ? (15000) : (30000)) |
| 99 | crit: $this > (($status == $CRITICAL) ? (30000) : (60000)) |
| 100 | delay: down 5m multiplier 1.5 max 1h |
| 101 | summary: Data Explorer query duration on ${label:resource_name} |
| 102 | info: Average query duration on Azure Data Explorer cluster ${label:resource_name} \ |
| 103 | in ${label:resource_group} (${label:region}) |
| 104 | to: sysadmin |
| 105 | |
| 106 | template: am_data_explorer_throttled_queries |
| 107 | on: azure_monitor.data_explorer.throttled_queries |
| 108 | class: Errors |
| 109 | type: Database |
| 110 | component: Data Explorer |
| 111 | lookup: average -5m unaligned of total |
| 112 | units: queries/s |
| 113 | every: 1m |
| 114 | warn: $this > (($status >= $WARNING) ? (1) : (5)) |
| 115 | crit: $this > (($status == $CRITICAL) ? (5) : (20)) |
| 116 | delay: down 5m multiplier 1.5 max 1h |
| 117 | summary: Data Explorer throttled queries on ${label:resource_name} |
| 118 | info: Rate of throttled queries on Azure Data Explorer cluster ${label:resource_name} \ |
| 119 | in ${label:resource_group} (${label:region}). \ |
| 120 | Throttling indicates insufficient query capacity. |
| 121 | to: sysadmin |
| 122 | |
| 123 | # --- Ingestion Health --- |
| 124 | |
| 125 | template: am_data_explorer_ingestion_latency |
| 126 | on: azure_monitor.data_explorer.ingestion_latency |
| 127 | class: Latency |
| 128 | type: Database |
| 129 | component: Data Explorer |
| 130 | lookup: average -5m unaligned of average |
| 131 | units: seconds |
| 132 | every: 1m |
| 133 | warn: $this > (($status >= $WARNING) ? (300) : (600)) |
| 134 | crit: $this > (($status == $CRITICAL) ? (600) : (1800)) |
| 135 | delay: down 5m multiplier 1.5 max 1h |
| 136 | summary: Data Explorer ingestion latency on ${label:resource_name} |
| 137 | info: Average ingestion latency on Azure Data Explorer cluster ${label:resource_name} \ |
| 138 | in ${label:resource_group} (${label:region}). \ |
| 139 | High latency means data takes longer to become queryable. |
| 140 | to: sysadmin |
| 141 | |
| 142 | template: am_data_explorer_events_dropped |
| 143 | on: azure_monitor.data_explorer.events |
| 144 | class: Errors |
| 145 | type: Database |
| 146 | component: Data Explorer |
| 147 | lookup: average -5m unaligned of dropped |
| 148 | units: events/s |
| 149 | every: 1m |
| 150 | warn: $this > (($status >= $WARNING) ? (0) : (1)) |
| 151 | crit: $this > (($status == $CRITICAL) ? (1) : (10)) |
| 152 | delay: down 5m multiplier 1.5 max 1h |
| 153 | summary: Data Explorer events dropped on ${label:resource_name} |
| 154 | info: Rate of dropped ingestion events on Azure Data Explorer cluster ${label:resource_name} \ |
| 155 | in ${label:resource_group} (${label:region}). \ |
| 156 | Dropped events indicate data loss during ingestion. |
| 157 | to: sysadmin |
| 158 | |
| 159 | template: am_data_explorer_blobs_dropped |
| 160 | on: azure_monitor.data_explorer.blobs |
| 161 | class: Errors |
| 162 | type: Database |
| 163 | component: Data Explorer |
| 164 | lookup: average -5m unaligned of dropped |
| 165 | units: blobs/s |
| 166 | every: 1m |
| 167 | warn: $this > (($status >= $WARNING) ? (0) : (1)) |
| 168 | crit: $this > (($status == $CRITICAL) ? (1) : (10)) |
| 169 | delay: down 5m multiplier 1.5 max 1h |
| 170 | summary: Data Explorer blobs dropped on ${label:resource_name} |
| 171 | info: Rate of dropped blobs during ingestion on Azure Data Explorer cluster ${label:resource_name} \ |
| 172 | in ${label:resource_group} (${label:region}). \ |
| 173 | Dropped blobs indicate data loss during ingestion. |
| 174 | to: sysadmin |
| 175 | |
| 176 | template: am_data_explorer_ingestion_queue_length |
| 177 | on: azure_monitor.data_explorer.ingestion_queue |
| 178 | class: Workload |
| 179 | type: Database |
| 180 | component: Data Explorer |
| 181 | lookup: average -5m unaligned of length |
| 182 | units: messages |
| 183 | every: 1m |
| 184 | warn: $this > (($status >= $WARNING) ? (500) : (1000)) |
| 185 | crit: $this > (($status == $CRITICAL) ? (1000) : (5000)) |
| 186 | delay: down 5m multiplier 1.5 max 1h |
| 187 | summary: Data Explorer ingestion queue on ${label:resource_name} |
| 188 | info: Average ingestion queue length on Azure Data Explorer cluster ${label:resource_name} \ |
| 189 | in ${label:resource_group} (${label:region}). \ |
| 190 | A growing queue indicates ingestion is not keeping up with incoming data. |
| 191 | to: sysadmin |
| 192 | |
| 193 | template: am_data_explorer_queue_oldest_message |
| 194 | on: azure_monitor.data_explorer.queue_oldest_message |
| 195 | class: Latency |
| 196 | type: Database |
| 197 | component: Data Explorer |
| 198 | lookup: average -5m unaligned of age |
| 199 | units: seconds |
| 200 | every: 1m |
| 201 | warn: $this > (($status >= $WARNING) ? (300) : (600)) |
| 202 | crit: $this > (($status == $CRITICAL) ? (600) : (1800)) |
| 203 | delay: down 5m multiplier 1.5 max 1h |
| 204 | summary: Data Explorer queue oldest message age on ${label:resource_name} |
| 205 | info: Age of the oldest message in the ingestion queue of Azure Data Explorer cluster ${label:resource_name} \ |
| 206 | in ${label:resource_group} (${label:region}). \ |
| 207 | Old messages indicate a significant ingestion backlog. |
| 208 | to: sysadmin |
| 209 | |
| 210 | # --- Export Health --- |
| 211 | |
| 212 | template: am_data_explorer_export_utilization |
| 213 | on: azure_monitor.data_explorer.export_utilization |
| 214 | class: Utilization |
| 215 | type: Database |
| 216 | component: Data Explorer |
| 217 | lookup: average -5m unaligned of maximum |
| 218 | units: percentage |
| 219 | every: 1m |
| 220 | warn: $this > (($status >= $WARNING) ? (70) : (80)) |
| 221 | crit: $this > (($status == $CRITICAL) ? (80) : (95)) |
| 222 | delay: down 5m multiplier 1.5 max 1h |
| 223 | summary: Data Explorer export utilization on ${label:resource_name} |
| 224 | info: Export utilization of Azure Data Explorer cluster ${label:resource_name} \ |
| 225 | in ${label:resource_group} (${label:region}). \ |
| 226 | High export utilization may cause export jobs to be delayed or fail. |
| 227 | to: sysadmin |
| 228 | |
| 229 | template: am_data_explorer_continuous_export_pending |
| 230 | on: azure_monitor.data_explorer.continuous_export_pending |
| 231 | class: Workload |
| 232 | type: Database |
| 233 | component: Data Explorer |
| 234 | lookup: average -5m unaligned of maximum |
| 235 | units: jobs |
| 236 | every: 1m |
| 237 | warn: $this != nan AND $this > (($status >= $WARNING) ? (5) : (10)) |
| 238 | crit: $this != nan AND $this > (($status == $CRITICAL) ? (10) : (50)) |
| 239 | delay: down 5m multiplier 1.5 max 1h |
| 240 | summary: Data Explorer continuous export pending on ${label:resource_name} |
| 241 | info: Number of pending continuous export jobs on Azure Data Explorer cluster ${label:resource_name} \ |
| 242 | in ${label:resource_group} (${label:region}) |
| 243 | to: sysadmin |
| 244 | |
| 245 | template: am_data_explorer_continuous_export_lateness |
| 246 | on: azure_monitor.data_explorer.continuous_export_lateness |
| 247 | class: Latency |
| 248 | type: Database |
| 249 | component: Data Explorer |
| 250 | lookup: average -5m unaligned of maximum |
| 251 | units: minutes |
| 252 | every: 1m |
| 253 | warn: $this != nan AND $this > (($status >= $WARNING) ? (10) : (30)) |
| 254 | crit: $this != nan AND $this > (($status == $CRITICAL) ? (30) : (60)) |
| 255 | delay: down 5m multiplier 1.5 max 1h |
| 256 | summary: Data Explorer continuous export lateness on ${label:resource_name} |
| 257 | info: Maximum continuous export lateness on Azure Data Explorer cluster ${label:resource_name} \ |
| 258 | in ${label:resource_group} (${label:region}). \ |
| 259 | High lateness means exported data is significantly behind real-time. |
| 260 | to: sysadmin |
| 261 | |
| 262 | # --- Streaming Ingest --- |
| 263 | |
| 264 | template: am_data_explorer_streaming_ingest_utilization |
| 265 | on: azure_monitor.data_explorer.streaming_ingest_utilization |
| 266 | class: Utilization |
| 267 | type: Database |
| 268 | component: Data Explorer |
| 269 | lookup: average -5m unaligned of average |
| 270 | units: percentage |
| 271 | every: 1m |
| 272 | warn: $this != nan AND $this > (($status >= $WARNING) ? (70) : (80)) |
| 273 | crit: $this != nan AND $this > (($status == $CRITICAL) ? (80) : (95)) |
| 274 | delay: down 5m multiplier 1.5 max 1h |
| 275 | summary: Data Explorer streaming ingest utilization on ${label:resource_name} |
| 276 | info: Average streaming ingest utilization on Azure Data Explorer cluster ${label:resource_name} \ |
| 277 | in ${label:resource_group} (${label:region}). \ |
| 278 | High utilization indicates the cluster is approaching streaming ingest capacity. |
| 279 | to: sysadmin |
| 280 | |
| 281 | template: am_data_explorer_streaming_ingest_duration |
| 282 | on: azure_monitor.data_explorer.streaming_ingest_duration |
| 283 | class: Latency |
| 284 | type: Database |
| 285 | component: Data Explorer |
| 286 | lookup: average -5m unaligned of average |
| 287 | units: milliseconds |
| 288 | every: 1m |
| 289 | warn: $this != nan AND $this > (($status >= $WARNING) ? (500) : (1000)) |
| 290 | crit: $this != nan AND $this > (($status == $CRITICAL) ? (1000) : (5000)) |
| 291 | delay: down 5m multiplier 1.5 max 1h |
| 292 | summary: Data Explorer streaming ingest duration on ${label:resource_name} |
| 293 | info: Average streaming ingest duration on Azure Data Explorer cluster ${label:resource_name} \ |
| 294 | in ${label:resource_group} (${label:region}) |
| 295 | to: sysadmin |
| 296 | |
| 297 | # --- Materialized Views --- |
| 298 | |
| 299 | template: am_data_explorer_materialized_view_health |
| 300 | on: azure_monitor.data_explorer.materialized_view_health |
| 301 | class: Availability |
| 302 | type: Database |
| 303 | component: Data Explorer |
| 304 | lookup: average -5m unaligned of health |
| 305 | units: status |
| 306 | every: 1m |
| 307 | crit: $this != nan AND $this < 1 |
| 308 | delay: down 5m multiplier 1.5 max 1h |
| 309 | summary: Data Explorer materialized view health on ${label:resource_name} |
| 310 | info: Materialized view health status on Azure Data Explorer cluster ${label:resource_name} \ |
| 311 | in ${label:resource_group} (${label:region}). \ |
| 312 | A value below 1 indicates the materialized view is unhealthy. |
| 313 | to: sysadmin |
| 314 | |
| 315 | template: am_data_explorer_materialized_view_age |
| 316 | on: azure_monitor.data_explorer.materialized_view_age |
| 317 | class: Latency |
| 318 | type: Database |
| 319 | component: Data Explorer |
| 320 | lookup: average -5m unaligned of minutes |
| 321 | units: minutes |
| 322 | every: 1m |
| 323 | warn: $this != nan AND $this > (($status >= $WARNING) ? (30) : (60)) |
| 324 | crit: $this != nan AND $this > (($status == $CRITICAL) ? (60) : (120)) |
| 325 | delay: down 5m multiplier 1.5 max 1h |
| 326 | summary: Data Explorer materialized view age on ${label:resource_name} |
| 327 | info: Age of the materialized view on Azure Data Explorer cluster ${label:resource_name} \ |
| 328 | in ${label:resource_group} (${label:region}). \ |
| 329 | High age means the view is significantly behind the source data. |
| 330 | to: sysadmin |
| 331 | |
| 332 | template: am_data_explorer_materialized_view_data_loss |
| 333 | on: azure_monitor.data_explorer.materialized_view_data_loss |
| 334 | class: Errors |
| 335 | type: Database |
| 336 | component: Data Explorer |
| 337 | lookup: max -5m unaligned of maximum |
| 338 | units: status |
| 339 | every: 1m |
| 340 | crit: $this != nan AND $this > 0 |
| 341 | delay: down 5m multiplier 1.5 max 1h |
| 342 | summary: Data Explorer materialized view data loss on ${label:resource_name} |
| 343 | info: Materialized view is reporting data loss on Azure Data Explorer cluster ${label:resource_name} \ |
| 344 | in ${label:resource_group} (${label:region}). \ |
| 345 | This indicates potential data inconsistency in the view. |
| 346 | to: sysadmin |
| 347 | |
| 348 | # --- Follower Latency --- |
| 349 | |
| 350 | template: am_data_explorer_follower_latency |
| 351 | on: azure_monitor.data_explorer.follower_latency |
| 352 | class: Latency |
| 353 | type: Database |
| 354 | component: Data Explorer |
| 355 | lookup: average -5m unaligned of average |
| 356 | units: milliseconds |
| 357 | every: 1m |
| 358 | warn: $this != nan AND $this > (($status >= $WARNING) ? (30000) : (60000)) |
| 359 | crit: $this != nan AND $this > (($status == $CRITICAL) ? (60000) : (300000)) |
| 360 | delay: down 5m multiplier 1.5 max 1h |
| 361 | summary: Data Explorer follower latency on ${label:resource_name} |
| 362 | info: Average follower replication latency on Azure Data Explorer cluster ${label:resource_name} \ |
| 363 | in ${label:resource_group} (${label:region}). \ |
| 364 | High latency means follower databases are behind the leader. |
| 365 | to: sysadmin |