| 1 | |
| 2 | # Kubernetes API Server health alerts |
| 3 | |
| 4 | # ----------------------------------------------------------------------------- |
| 5 | # Request Rate (for alarm thresholds) |
| 6 | |
| 7 | template: k8s_apiserver_1m_requests |
| 8 | on: k8s_apiserver.requests_total |
| 9 | class: Workload |
| 10 | type: Kubernetes |
| 11 | component: API Server |
| 12 | lookup: sum -1m unaligned of requests |
| 13 | calc: ($this == 0)?(1):($this) |
| 14 | units: requests |
| 15 | every: 10s |
| 16 | info: number of API server requests in the last minute |
| 17 | |
| 18 | # ----------------------------------------------------------------------------- |
| 19 | # Dropped Requests |
| 20 | |
| 21 | template: k8s_apiserver_dropped_requests |
| 22 | on: k8s_apiserver.requests_dropped |
| 23 | class: Errors |
| 24 | type: Kubernetes |
| 25 | component: API Server |
| 26 | lookup: sum -1m unaligned of dropped |
| 27 | units: requests |
| 28 | every: 10s |
| 29 | warn: $this > 0 |
| 30 | crit: $this > 10 |
| 31 | delay: up 1m down 5m multiplier 1.5 max 1h |
| 32 | summary: K8s API server dropped requests |
| 33 | info: API server dropped requests in the last minute due to overload |
| 34 | to: sysadmin |
| 35 | |
| 36 | # ----------------------------------------------------------------------------- |
| 37 | # Error Rate (5xx responses) |
| 38 | |
| 39 | template: k8s_apiserver_5xx_errors |
| 40 | on: k8s_apiserver.requests_by_code |
| 41 | class: Errors |
| 42 | type: Kubernetes |
| 43 | component: API Server |
| 44 | lookup: sum -1m unaligned of 500,501,502,503,504 |
| 45 | calc: $this * 100 / $k8s_apiserver_1m_requests |
| 46 | units: % |
| 47 | every: 10s |
| 48 | warn: ($k8s_apiserver_1m_requests > 60) ? ($this > (($status >= $WARNING) ? ( 1 ) : ( 2 )) ) : ( 0 ) |
| 49 | crit: ($k8s_apiserver_1m_requests > 60) ? ($this > (($status == $CRITICAL) ? ( 2 ) : ( 5 )) ) : ( 0 ) |
| 50 | delay: up 2m down 15m multiplier 1.5 max 1h |
| 51 | summary: K8s API server error rate |
| 52 | info: Percentage of 5xx server error responses over the last minute |
| 53 | to: sysadmin |
| 54 | |
| 55 | # ----------------------------------------------------------------------------- |
| 56 | # Client Errors (4xx responses - excluding 401/403 which are normal auth) |
| 57 | |
| 58 | template: k8s_apiserver_4xx_errors |
| 59 | on: k8s_apiserver.requests_by_code |
| 60 | class: Errors |
| 61 | type: Kubernetes |
| 62 | component: API Server |
| 63 | lookup: sum -1m unaligned of 400,404,405,408,409,410,422,429 |
| 64 | calc: $this * 100 / $k8s_apiserver_1m_requests |
| 65 | units: % |
| 66 | every: 10s |
| 67 | warn: ($k8s_apiserver_1m_requests > 60) ? ($this > (($status >= $WARNING) ? ( 10 ) : ( 20 )) ) : ( 0 ) |
| 68 | delay: up 2m down 15m multiplier 1.5 max 1h |
| 69 | summary: K8s API server client errors |
| 70 | info: Percentage of 4xx client error responses over the last minute (excluding 401/403) |
| 71 | to: sysadmin |
| 72 | |
| 73 | # ----------------------------------------------------------------------------- |
| 74 | # Request Latency |
| 75 | |
| 76 | template: k8s_apiserver_latency_10m |
| 77 | on: k8s_apiserver.request_latency |
| 78 | class: Latency |
| 79 | type: Kubernetes |
| 80 | component: API Server |
| 81 | lookup: average -10m unaligned of p99 |
| 82 | units: milliseconds |
| 83 | every: 30s |
| 84 | info: average API server request latency (p99) over the last 10 minutes |
| 85 | |
| 86 | template: k8s_apiserver_latency_high |
| 87 | on: k8s_apiserver.request_latency |
| 88 | class: Latency |
| 89 | type: Kubernetes |
| 90 | component: API Server |
| 91 | lookup: average -1m unaligned of p99 |
| 92 | units: milliseconds |
| 93 | every: 10s |
| 94 | green: 500 |
| 95 | red: 2000 |
| 96 | warn: ($k8s_apiserver_1m_requests > 60) ? ($this > $green && $this > ($k8s_apiserver_latency_10m * 2)) : ( 0 ) |
| 97 | crit: ($k8s_apiserver_1m_requests > 60) ? ($this > $red && $this > ($k8s_apiserver_latency_10m * 4)) : ( 0 ) |
| 98 | delay: down 15m multiplier 1.5 max 1h |
| 99 | summary: K8s API server latency |
| 100 | info: API server request latency (p99) over the last minute |
| 101 | options: no-clear-notification |
| 102 | to: sysadmin |
| 103 | |
| 104 | # ----------------------------------------------------------------------------- |
| 105 | # Inflight Requests |
| 106 | |
| 107 | template: k8s_apiserver_inflight_mutating |
| 108 | on: k8s_apiserver.inflight_requests |
| 109 | class: Utilization |
| 110 | type: Kubernetes |
| 111 | component: API Server |
| 112 | lookup: average -1m unaligned of mutating |
| 113 | units: requests |
| 114 | every: 10s |
| 115 | warn: $this > 400 |
| 116 | crit: $this > 600 |
| 117 | delay: up 1m down 5m multiplier 1.5 max 1h |
| 118 | summary: K8s API server mutating inflight |
| 119 | info: Average number of mutating requests currently in flight over the last minute \ |
| 120 | (default limit is 600) |
| 121 | to: sysadmin |
| 122 | |
| 123 | template: k8s_apiserver_inflight_readonly |
| 124 | on: k8s_apiserver.inflight_requests |
| 125 | class: Utilization |
| 126 | type: Kubernetes |
| 127 | component: API Server |
| 128 | lookup: average -1m unaligned of read_only |
| 129 | units: requests |
| 130 | every: 10s |
| 131 | warn: $this > 800 |
| 132 | crit: $this > 1000 |
| 133 | delay: up 1m down 5m multiplier 1.5 max 1h |
| 134 | summary: K8s API server read-only inflight |
| 135 | info: Average number of read-only requests currently in flight over the last minute \ |
| 136 | (default limit is 1000) |
| 137 | to: sysadmin |
| 138 | |
| 139 | # ----------------------------------------------------------------------------- |
| 140 | # Workqueue Depth |
| 141 | |
| 142 | template: k8s_apiserver_workqueue_depth |
| 143 | on: k8s_apiserver.workqueue_depth |
| 144 | class: Utilization |
| 145 | type: Kubernetes |
| 146 | component: API Server |
| 147 | lookup: average -1m unaligned of depth |
| 148 | units: items |
| 149 | every: 10s |
| 150 | warn: $this > 100 |
| 151 | crit: $this > 500 |
| 152 | delay: up 2m down 10m multiplier 1.5 max 1h |
| 153 | summary: K8s API server workqueue depth |
| 154 | info: Average depth of controller work queue over the last minute |
| 155 | to: sysadmin |
| 156 | |
| 157 | # ----------------------------------------------------------------------------- |
| 158 | # Process Resources |
| 159 | |
| 160 | template: k8s_apiserver_goroutines |
| 161 | on: k8s_apiserver.goroutines |
| 162 | class: Utilization |
| 163 | type: Kubernetes |
| 164 | component: API Server |
| 165 | lookup: average -5m unaligned of goroutines |
| 166 | units: goroutines |
| 167 | every: 30s |
| 168 | warn: $this > 10000 |
| 169 | crit: $this > 20000 |
| 170 | delay: up 5m down 15m multiplier 1.5 max 1h |
| 171 | summary: K8s API server goroutines |
| 172 | info: Average number of goroutines over the last 5 minutes |
| 173 | to: sysadmin |
| 174 | |
| 175 | template: k8s_apiserver_open_fds_utilization |
| 176 | on: k8s_apiserver.open_fds |
| 177 | class: Utilization |
| 178 | type: Kubernetes |
| 179 | component: API Server |
| 180 | calc: ($max > 0) ? ($open * 100 / $max) : (0) |
| 181 | units: % |
| 182 | every: 30s |
| 183 | warn: $this > 80 |
| 184 | crit: $this > 95 |
| 185 | delay: up 2m down 15m multiplier 1.5 max 1h |
| 186 | summary: K8s API server file descriptor utilization |
| 187 | info: Percentage of file descriptors used by the API server |
| 188 | to: sysadmin |