| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | package elasticsearch |
| 4 | |
| 5 | import ( |
| 6 | "fmt" |
| 7 | "strings" |
| 8 | |
| 9 | "github.com/netdata/netdata/go/plugins/plugin/framework/collectorapi" |
| 10 | ) |
| 11 | |
| 12 | const ( |
| 13 | prioNodeIndicesIndexingOps = collectorapi.Priority + iota |
| 14 | prioNodeIndicesIndexingOpsCurrent |
| 15 | prioNodeIndicesIndexingOpsTime |
| 16 | prioNodeIndicesSearchOps |
| 17 | prioNodeIndicesSearchOpsCurrent |
| 18 | prioNodeIndicesSearchOpsTime |
| 19 | prioNodeIndicesRefreshOps |
| 20 | prioNodeIndicesRefreshOpsTime |
| 21 | prioNodeIndicesFlushOps |
| 22 | prioNodeIndicesFlushOpsTime |
| 23 | prioNodeIndicesFieldDataMemoryUsage |
| 24 | prioNodeIndicesFieldDataEvictions |
| 25 | prioNodeIndicesSegmentsCount |
| 26 | prioNodeIndicesSegmentsMemoryUsageTotal |
| 27 | prioNodeIndicesSegmentsMemoryUsage |
| 28 | prioNodeIndicesTransLogOps |
| 29 | prioNodeIndexTransLogSize |
| 30 | prioNodeFileDescriptors |
| 31 | prioNodeJVMMemHeap |
| 32 | prioNodeJVMMemHeapBytes |
| 33 | prioNodeJVMBufferPoolsCount |
| 34 | prioNodeJVMBufferPoolDirectMemory |
| 35 | prioNodeJVMBufferPoolMappedMemory |
| 36 | prioNodeJVMGCCount |
| 37 | prioNodeJVMGCTime |
| 38 | prioNodeThreadPoolQueued |
| 39 | prioNodeThreadPoolRejected |
| 40 | prioNodeClusterCommunicationPackets |
| 41 | prioNodeClusterCommunication |
| 42 | prioNodeHTTPConnections |
| 43 | prioNodeBreakersTrips |
| 44 | |
| 45 | prioClusterStatus |
| 46 | prioClusterNodesCount |
| 47 | prioClusterShardsCount |
| 48 | prioClusterPendingTasks |
| 49 | prioClusterInFlightFetchesCount |
| 50 | |
| 51 | prioClusterIndicesCount |
| 52 | prioClusterIndicesShardsCount |
| 53 | prioClusterIndicesDocsCount |
| 54 | prioClusterIndicesStoreSize |
| 55 | prioClusterIndicesQueryCache |
| 56 | prioClusterNodesByRoleCount |
| 57 | |
| 58 | prioNodeIndexHealth |
| 59 | prioNodeIndexShardsCount |
| 60 | prioNodeIndexDocsCount |
| 61 | prioNodeIndexStoreSize |
| 62 | ) |
| 63 | |
| 64 | var nodeChartsTmpl = collectorapi.Charts{ |
| 65 | nodeIndicesIndexingOpsChartTmpl.Copy(), |
| 66 | nodeIndicesIndexingOpsCurrentChartTmpl.Copy(), |
| 67 | nodeIndicesIndexingOpsTimeChartTmpl.Copy(), |
| 68 | |
| 69 | nodeIndicesSearchOpsChartTmpl.Copy(), |
| 70 | nodeIndicesSearchOpsCurrentChartTmpl.Copy(), |
| 71 | nodeIndicesSearchOpsTimeChartTmpl.Copy(), |
| 72 | |
| 73 | nodeIndicesRefreshOpsChartTmpl.Copy(), |
| 74 | nodeIndicesRefreshOpsTimeChartTmpl.Copy(), |
| 75 | |
| 76 | nodeIndicesFlushOpsChartTmpl.Copy(), |
| 77 | nodeIndicesFlushOpsTimeChartTmpl.Copy(), |
| 78 | |
| 79 | nodeIndicesFieldDataMemoryUsageChartTmpl.Copy(), |
| 80 | nodeIndicesFieldDataEvictionsChartTmpl.Copy(), |
| 81 | |
| 82 | nodeIndicesSegmentsCountChartTmpl.Copy(), |
| 83 | nodeIndicesSegmentsMemoryUsageTotalChartTmpl.Copy(), |
| 84 | nodeIndicesSegmentsMemoryUsageChartTmpl.Copy(), |
| 85 | |
| 86 | nodeIndicesTransLogOpsChartTmpl.Copy(), |
| 87 | nodeIndexTransLogSizeChartTmpl.Copy(), |
| 88 | |
| 89 | nodeFileDescriptorsChartTmpl.Copy(), |
| 90 | |
| 91 | nodeJVMMemHeapChartTmpl.Copy(), |
| 92 | nodeJVMMemHeapBytesChartTmpl.Copy(), |
| 93 | nodeJVMBufferPoolsCountChartTmpl.Copy(), |
| 94 | nodeJVMBufferPoolDirectMemoryChartTmpl.Copy(), |
| 95 | nodeJVMBufferPoolMappedMemoryChartTmpl.Copy(), |
| 96 | nodeJVMGCCountChartTmpl.Copy(), |
| 97 | nodeJVMGCTimeChartTmpl.Copy(), |
| 98 | |
| 99 | nodeThreadPoolQueuedChartTmpl.Copy(), |
| 100 | nodeThreadPoolRejectedChartTmpl.Copy(), |
| 101 | |
| 102 | nodeClusterCommunicationPacketsChartTmpl.Copy(), |
| 103 | nodeClusterCommunicationChartTmpl.Copy(), |
| 104 | |
| 105 | nodeHTTPConnectionsChartTmpl.Copy(), |
| 106 | |
| 107 | nodeBreakersTripsChartTmpl.Copy(), |
| 108 | } |
| 109 | |
| 110 | var ( |
| 111 | nodeIndicesIndexingOpsChartTmpl = collectorapi.Chart{ |
| 112 | ID: "node_%s_cluster_%s_indices_indexing_operations", |
| 113 | Title: "Indexing Operations", |
| 114 | Units: "operations/s", |
| 115 | Fam: "indices indexing", |
| 116 | Ctx: "elasticsearch.node_indices_indexing", |
| 117 | Priority: prioNodeIndicesIndexingOps, |
| 118 | Dims: collectorapi.Dims{ |
| 119 | {ID: "node_%s_indices_indexing_index_total", Name: "index", Algo: collectorapi.Incremental}, |
| 120 | }, |
| 121 | } |
| 122 | nodeIndicesIndexingOpsCurrentChartTmpl = collectorapi.Chart{ |
| 123 | ID: "node_%s_cluster_%s_indices_indexing_operations_current", |
| 124 | Title: "Indexing Operations Current", |
| 125 | Units: "operations", |
| 126 | Fam: "indices indexing", |
| 127 | Ctx: "elasticsearch.node_indices_indexing_current", |
| 128 | Priority: prioNodeIndicesIndexingOpsCurrent, |
| 129 | Dims: collectorapi.Dims{ |
| 130 | {ID: "node_%s_indices_indexing_index_current", Name: "index"}, |
| 131 | }, |
| 132 | } |
| 133 | nodeIndicesIndexingOpsTimeChartTmpl = collectorapi.Chart{ |
| 134 | ID: "node_%s_cluster_%s_indices_indexing_operations_time", |
| 135 | Title: "Time Spent On Indexing Operations", |
| 136 | Units: "milliseconds", |
| 137 | Fam: "indices indexing", |
| 138 | Ctx: "elasticsearch.node_indices_indexing_time", |
| 139 | Priority: prioNodeIndicesIndexingOpsTime, |
| 140 | Dims: collectorapi.Dims{ |
| 141 | {ID: "node_%s_indices_indexing_index_time_in_millis", Name: "index", Algo: collectorapi.Incremental}, |
| 142 | }, |
| 143 | } |
| 144 | |
| 145 | nodeIndicesSearchOpsChartTmpl = collectorapi.Chart{ |
| 146 | ID: "node_%s_cluster_%s_indices_search_operations", |
| 147 | Title: "Search Operations", |
| 148 | Units: "operations/s", |
| 149 | Fam: "indices search", |
| 150 | Ctx: "elasticsearch.node_indices_search", |
| 151 | Type: collectorapi.Stacked, |
| 152 | Priority: prioNodeIndicesSearchOps, |
| 153 | Dims: collectorapi.Dims{ |
| 154 | {ID: "node_%s_indices_search_query_total", Name: "queries", Algo: collectorapi.Incremental}, |
| 155 | {ID: "node_%s_indices_search_fetch_total", Name: "fetches", Algo: collectorapi.Incremental}, |
| 156 | }, |
| 157 | } |
| 158 | nodeIndicesSearchOpsCurrentChartTmpl = collectorapi.Chart{ |
| 159 | ID: "node_%s_cluster_%s_indices_search_operations_current", |
| 160 | Title: "Search Operations Current", |
| 161 | Units: "operations", |
| 162 | Fam: "indices search", |
| 163 | Ctx: "elasticsearch.node_indices_search_current", |
| 164 | Type: collectorapi.Stacked, |
| 165 | Priority: prioNodeIndicesSearchOpsCurrent, |
| 166 | Dims: collectorapi.Dims{ |
| 167 | {ID: "node_%s_indices_search_query_current", Name: "queries"}, |
| 168 | {ID: "node_%s_indices_search_fetch_current", Name: "fetches"}, |
| 169 | }, |
| 170 | } |
| 171 | nodeIndicesSearchOpsTimeChartTmpl = collectorapi.Chart{ |
| 172 | ID: "node_%s_cluster_%s_indices_search_operations_time", |
| 173 | Title: "Time Spent On Search Operations", |
| 174 | Units: "milliseconds", |
| 175 | Fam: "indices search", |
| 176 | Ctx: "elasticsearch.node_indices_search_time", |
| 177 | Type: collectorapi.Stacked, |
| 178 | Priority: prioNodeIndicesSearchOpsTime, |
| 179 | Dims: collectorapi.Dims{ |
| 180 | {ID: "node_%s_indices_search_query_time_in_millis", Name: "query", Algo: collectorapi.Incremental}, |
| 181 | {ID: "node_%s_indices_search_fetch_time_in_millis", Name: "fetch", Algo: collectorapi.Incremental}, |
| 182 | }, |
| 183 | } |
| 184 | |
| 185 | nodeIndicesRefreshOpsChartTmpl = collectorapi.Chart{ |
| 186 | ID: "node_%s_cluster_%s_indices_refresh_operations", |
| 187 | Title: "Refresh Operations", |
| 188 | Units: "operations/s", |
| 189 | Fam: "indices refresh", |
| 190 | Ctx: "elasticsearch.node_indices_refresh", |
| 191 | Priority: prioNodeIndicesRefreshOps, |
| 192 | Dims: collectorapi.Dims{ |
| 193 | {ID: "node_%s_indices_refresh_total", Name: "refresh", Algo: collectorapi.Incremental}, |
| 194 | }, |
| 195 | } |
| 196 | nodeIndicesRefreshOpsTimeChartTmpl = collectorapi.Chart{ |
| 197 | ID: "node_%s_cluster_%s_indices_refresh_operations_time", |
| 198 | Title: "Time Spent On Refresh Operations", |
| 199 | Units: "milliseconds", |
| 200 | Fam: "indices refresh", |
| 201 | Ctx: "elasticsearch.node_indices_refresh_time", |
| 202 | Priority: prioNodeIndicesRefreshOpsTime, |
| 203 | Dims: collectorapi.Dims{ |
| 204 | {ID: "node_%s_indices_refresh_total_time_in_millis", Name: "refresh", Algo: collectorapi.Incremental}, |
| 205 | }, |
| 206 | } |
| 207 | |
| 208 | nodeIndicesFlushOpsChartTmpl = collectorapi.Chart{ |
| 209 | ID: "node_%s_cluster_%s_indices_flush_operations", |
| 210 | Title: "Flush Operations", |
| 211 | Units: "operations/s", |
| 212 | Fam: "indices flush", |
| 213 | Ctx: "elasticsearch.node_indices_flush", |
| 214 | Priority: prioNodeIndicesFlushOps, |
| 215 | Dims: collectorapi.Dims{ |
| 216 | {ID: "node_%s_indices_flush_total", Name: "flush", Algo: collectorapi.Incremental}, |
| 217 | }, |
| 218 | } |
| 219 | nodeIndicesFlushOpsTimeChartTmpl = collectorapi.Chart{ |
| 220 | ID: "node_%s_cluster_%s_indices_flush_operations_time", |
| 221 | Title: "Time Spent On Flush Operations", |
| 222 | Units: "milliseconds", |
| 223 | Fam: "indices flush", |
| 224 | Ctx: "elasticsearch.node_indices_flush_time", |
| 225 | Priority: prioNodeIndicesFlushOpsTime, |
| 226 | Dims: collectorapi.Dims{ |
| 227 | {ID: "node_%s_indices_flush_total_time_in_millis", Name: "flush", Algo: collectorapi.Incremental}, |
| 228 | }, |
| 229 | } |
| 230 | |
| 231 | nodeIndicesFieldDataMemoryUsageChartTmpl = collectorapi.Chart{ |
| 232 | ID: "node_%s_cluster_%s_indices_fielddata_memory_usage", |
| 233 | Title: "Fielddata Cache Memory Usage", |
| 234 | Units: "bytes", |
| 235 | Fam: "indices fielddata", |
| 236 | Ctx: "elasticsearch.node_indices_fielddata_memory_usage", |
| 237 | Type: collectorapi.Area, |
| 238 | Priority: prioNodeIndicesFieldDataMemoryUsage, |
| 239 | Dims: collectorapi.Dims{ |
| 240 | {ID: "node_%s_indices_fielddata_memory_size_in_bytes", Name: "used"}, |
| 241 | }, |
| 242 | } |
| 243 | nodeIndicesFieldDataEvictionsChartTmpl = collectorapi.Chart{ |
| 244 | ID: "node_%s_cluster_%s_indices_fielddata_evictions", |
| 245 | Title: "Fielddata Evictions", |
| 246 | Units: "operations/s", |
| 247 | Fam: "indices fielddata", |
| 248 | Ctx: "elasticsearch.node_indices_fielddata_evictions", |
| 249 | Priority: prioNodeIndicesFieldDataEvictions, |
| 250 | Dims: collectorapi.Dims{ |
| 251 | {ID: "node_%s_indices_fielddata_evictions", Name: "evictions", Algo: collectorapi.Incremental}, |
| 252 | }, |
| 253 | } |
| 254 | |
| 255 | nodeIndicesSegmentsCountChartTmpl = collectorapi.Chart{ |
| 256 | ID: "node_%s_cluster_%s_indices_segments_count", |
| 257 | Title: "Segments Count", |
| 258 | Units: "segments", |
| 259 | Fam: "indices segments", |
| 260 | Ctx: "elasticsearch.node_indices_segments_count", |
| 261 | Priority: prioNodeIndicesSegmentsCount, |
| 262 | Dims: collectorapi.Dims{ |
| 263 | {ID: "node_%s_indices_segments_count", Name: "segments"}, |
| 264 | }, |
| 265 | } |
| 266 | nodeIndicesSegmentsMemoryUsageTotalChartTmpl = collectorapi.Chart{ |
| 267 | ID: "node_%s_cluster_%s_indices_segments_memory_usage_total", |
| 268 | Title: "Segments Memory Usage Total", |
| 269 | Units: "bytes", |
| 270 | Fam: "indices segments", |
| 271 | Ctx: "elasticsearch.node_indices_segments_memory_usage_total", |
| 272 | Priority: prioNodeIndicesSegmentsMemoryUsageTotal, |
| 273 | Dims: collectorapi.Dims{ |
| 274 | {ID: "node_%s_indices_segments_memory_in_bytes", Name: "used"}, |
| 275 | }, |
| 276 | } |
| 277 | nodeIndicesSegmentsMemoryUsageChartTmpl = collectorapi.Chart{ |
| 278 | ID: "node_%s_cluster_%s_indices_segments_memory_usage", |
| 279 | Title: "Segments Memory Usage", |
| 280 | Units: "bytes", |
| 281 | Fam: "indices segments", |
| 282 | Ctx: "elasticsearch.node_indices_segments_memory_usage", |
| 283 | Type: collectorapi.Stacked, |
| 284 | Priority: prioNodeIndicesSegmentsMemoryUsage, |
| 285 | Dims: collectorapi.Dims{ |
| 286 | {ID: "node_%s_indices_segments_terms_memory_in_bytes", Name: "terms"}, |
| 287 | {ID: "node_%s_indices_segments_stored_fields_memory_in_bytes", Name: "stored_fields"}, |
| 288 | {ID: "node_%s_indices_segments_term_vectors_memory_in_bytes", Name: "term_vectors"}, |
| 289 | {ID: "node_%s_indices_segments_norms_memory_in_bytes", Name: "norms"}, |
| 290 | {ID: "node_%s_indices_segments_points_memory_in_bytes", Name: "points"}, |
| 291 | {ID: "node_%s_indices_segments_doc_values_memory_in_bytes", Name: "doc_values"}, |
| 292 | {ID: "node_%s_indices_segments_index_writer_memory_in_bytes", Name: "index_writer"}, |
| 293 | {ID: "node_%s_indices_segments_version_map_memory_in_bytes", Name: "version_map"}, |
| 294 | {ID: "node_%s_indices_segments_fixed_bit_set_memory_in_bytes", Name: "fixed_bit_set"}, |
| 295 | }, |
| 296 | } |
| 297 | |
| 298 | nodeIndicesTransLogOpsChartTmpl = collectorapi.Chart{ |
| 299 | ID: "node_%s_cluster_%s_indices_translog_operations", |
| 300 | Title: "Translog Operations", |
| 301 | Units: "operations", |
| 302 | Fam: "indices translog", |
| 303 | Ctx: "elasticsearch.node_indices_translog_operations", |
| 304 | Type: collectorapi.Area, |
| 305 | Priority: prioNodeIndicesTransLogOps, |
| 306 | Dims: collectorapi.Dims{ |
| 307 | {ID: "node_%s_indices_translog_operations", Name: "total"}, |
| 308 | {ID: "node_%s_indices_translog_uncommitted_operations", Name: "uncommitted"}, |
| 309 | }, |
| 310 | } |
| 311 | nodeIndexTransLogSizeChartTmpl = collectorapi.Chart{ |
| 312 | ID: "node_%s_cluster_%s_index_translog_size", |
| 313 | Title: "Translog Size", |
| 314 | Units: "bytes", |
| 315 | Fam: "indices translog", |
| 316 | Ctx: "elasticsearch.node_indices_translog_size", |
| 317 | Type: collectorapi.Area, |
| 318 | Priority: prioNodeIndexTransLogSize, |
| 319 | Dims: collectorapi.Dims{ |
| 320 | {ID: "node_%s_indices_translog_size_in_bytes", Name: "total"}, |
| 321 | {ID: "node_%s_indices_translog_uncommitted_size_in_bytes", Name: "uncommitted"}, |
| 322 | }, |
| 323 | } |
| 324 | |
| 325 | nodeFileDescriptorsChartTmpl = collectorapi.Chart{ |
| 326 | ID: "node_%s_cluster_%s_file_descriptors", |
| 327 | Title: "Process File Descriptors", |
| 328 | Units: "fd", |
| 329 | Fam: "process", |
| 330 | Ctx: "elasticsearch.node_file_descriptors", |
| 331 | Priority: prioNodeFileDescriptors, |
| 332 | Dims: collectorapi.Dims{ |
| 333 | {ID: "node_%s_process_open_file_descriptors", Name: "open"}, |
| 334 | }, |
| 335 | } |
| 336 | |
| 337 | nodeJVMMemHeapChartTmpl = collectorapi.Chart{ |
| 338 | ID: "node_%s_cluster_%s_jvm_mem_heap", |
| 339 | Title: "JVM Heap Percentage Currently in Use", |
| 340 | Units: "percentage", |
| 341 | Fam: "jvm", |
| 342 | Ctx: "elasticsearch.node_jvm_heap", |
| 343 | Type: collectorapi.Area, |
| 344 | Priority: prioNodeJVMMemHeap, |
| 345 | Dims: collectorapi.Dims{ |
| 346 | {ID: "node_%s_jvm_mem_heap_used_percent", Name: "inuse"}, |
| 347 | }, |
| 348 | } |
| 349 | nodeJVMMemHeapBytesChartTmpl = collectorapi.Chart{ |
| 350 | ID: "node_%s_cluster_%s_jvm_mem_heap_bytes", |
| 351 | Title: "JVM Heap Commit And Usage", |
| 352 | Units: "bytes", |
| 353 | Fam: "jvm", |
| 354 | Ctx: "elasticsearch.node_jvm_heap_bytes", |
| 355 | Type: collectorapi.Area, |
| 356 | Priority: prioNodeJVMMemHeapBytes, |
| 357 | Dims: collectorapi.Dims{ |
| 358 | {ID: "node_%s_jvm_mem_heap_committed_in_bytes", Name: "committed"}, |
| 359 | {ID: "node_%s_jvm_mem_heap_used_in_bytes", Name: "used"}, |
| 360 | }, |
| 361 | } |
| 362 | nodeJVMBufferPoolsCountChartTmpl = collectorapi.Chart{ |
| 363 | ID: "node_%s_cluster_%s_jvm_buffer_pools_count", |
| 364 | Title: "JVM Buffer Pools Count", |
| 365 | Units: "pools", |
| 366 | Fam: "jvm", |
| 367 | Ctx: "elasticsearch.node_jvm_buffer_pools_count", |
| 368 | Priority: prioNodeJVMBufferPoolsCount, |
| 369 | Dims: collectorapi.Dims{ |
| 370 | {ID: "node_%s_jvm_buffer_pools_direct_count", Name: "direct"}, |
| 371 | {ID: "node_%s_jvm_buffer_pools_mapped_count", Name: "mapped"}, |
| 372 | }, |
| 373 | } |
| 374 | nodeJVMBufferPoolDirectMemoryChartTmpl = collectorapi.Chart{ |
| 375 | ID: "node_%s_cluster_%s_jvm_buffer_pool_direct_memory", |
| 376 | Title: "JVM Buffer Pool Direct Memory", |
| 377 | Units: "bytes", |
| 378 | Fam: "jvm", |
| 379 | Ctx: "elasticsearch.node_jvm_buffer_pool_direct_memory", |
| 380 | Type: collectorapi.Area, |
| 381 | Priority: prioNodeJVMBufferPoolDirectMemory, |
| 382 | Dims: collectorapi.Dims{ |
| 383 | {ID: "node_%s_jvm_buffer_pools_direct_total_capacity_in_bytes", Name: "total"}, |
| 384 | {ID: "node_%s_jvm_buffer_pools_direct_used_in_bytes", Name: "used"}, |
| 385 | }, |
| 386 | } |
| 387 | nodeJVMBufferPoolMappedMemoryChartTmpl = collectorapi.Chart{ |
| 388 | ID: "node_%s_cluster_%s_jvm_buffer_pool_mapped_memory", |
| 389 | Title: "JVM Buffer Pool Mapped Memory", |
| 390 | Units: "bytes", |
| 391 | Fam: "jvm", |
| 392 | Ctx: "elasticsearch.node_jvm_buffer_pool_mapped_memory", |
| 393 | Type: collectorapi.Area, |
| 394 | Priority: prioNodeJVMBufferPoolMappedMemory, |
| 395 | Dims: collectorapi.Dims{ |
| 396 | {ID: "node_%s_jvm_buffer_pools_mapped_total_capacity_in_bytes", Name: "total"}, |
| 397 | {ID: "node_%s_jvm_buffer_pools_mapped_used_in_bytes", Name: "used"}, |
| 398 | }, |
| 399 | } |
| 400 | nodeJVMGCCountChartTmpl = collectorapi.Chart{ |
| 401 | ID: "node_%s_cluster_%s_jvm_gc_count", |
| 402 | Title: "JVM Garbage Collections", |
| 403 | Units: "gc/s", |
| 404 | Fam: "jvm", |
| 405 | Ctx: "elasticsearch.node_jvm_gc_count", |
| 406 | Type: collectorapi.Stacked, |
| 407 | Priority: prioNodeJVMGCCount, |
| 408 | Dims: collectorapi.Dims{ |
| 409 | {ID: "node_%s_jvm_gc_collectors_young_collection_count", Name: "young", Algo: collectorapi.Incremental}, |
| 410 | {ID: "node_%s_jvm_gc_collectors_old_collection_count", Name: "old", Algo: collectorapi.Incremental}, |
| 411 | }, |
| 412 | } |
| 413 | nodeJVMGCTimeChartTmpl = collectorapi.Chart{ |
| 414 | ID: "node_%s_cluster_%s_jvm_gc_time", |
| 415 | Title: "JVM Time Spent On Garbage Collections", |
| 416 | Units: "milliseconds", |
| 417 | Fam: "jvm", |
| 418 | Ctx: "elasticsearch.node_jvm_gc_time", |
| 419 | Type: collectorapi.Stacked, |
| 420 | Priority: prioNodeJVMGCTime, |
| 421 | Dims: collectorapi.Dims{ |
| 422 | {ID: "node_%s_jvm_gc_collectors_young_collection_time_in_millis", Name: "young", Algo: collectorapi.Incremental}, |
| 423 | {ID: "node_%s_jvm_gc_collectors_old_collection_time_in_millis", Name: "old", Algo: collectorapi.Incremental}, |
| 424 | }, |
| 425 | } |
| 426 | |
| 427 | nodeThreadPoolQueuedChartTmpl = collectorapi.Chart{ |
| 428 | ID: "node_%s_cluster_%s_thread_pool_queued", |
| 429 | Title: "Thread Pool Queued Threads Count", |
| 430 | Units: "threads", |
| 431 | Fam: "thread pool", |
| 432 | Ctx: "elasticsearch.node_thread_pool_queued", |
| 433 | Type: collectorapi.Stacked, |
| 434 | Priority: prioNodeThreadPoolQueued, |
| 435 | Dims: collectorapi.Dims{ |
| 436 | {ID: "node_%s_thread_pool_generic_queue", Name: "generic"}, |
| 437 | {ID: "node_%s_thread_pool_search_queue", Name: "search"}, |
| 438 | {ID: "node_%s_thread_pool_search_throttled_queue", Name: "search_throttled"}, |
| 439 | {ID: "node_%s_thread_pool_get_queue", Name: "get"}, |
| 440 | {ID: "node_%s_thread_pool_analyze_queue", Name: "analyze"}, |
| 441 | {ID: "node_%s_thread_pool_write_queue", Name: "write"}, |
| 442 | {ID: "node_%s_thread_pool_snapshot_queue", Name: "snapshot"}, |
| 443 | {ID: "node_%s_thread_pool_warmer_queue", Name: "warmer"}, |
| 444 | {ID: "node_%s_thread_pool_refresh_queue", Name: "refresh"}, |
| 445 | {ID: "node_%s_thread_pool_listener_queue", Name: "listener"}, |
| 446 | {ID: "node_%s_thread_pool_fetch_shard_started_queue", Name: "fetch_shard_started"}, |
| 447 | {ID: "node_%s_thread_pool_fetch_shard_store_queue", Name: "fetch_shard_store"}, |
| 448 | {ID: "node_%s_thread_pool_flush_queue", Name: "flush"}, |
| 449 | {ID: "node_%s_thread_pool_force_merge_queue", Name: "force_merge"}, |
| 450 | {ID: "node_%s_thread_pool_management_queue", Name: "management"}, |
| 451 | }, |
| 452 | } |
| 453 | nodeThreadPoolRejectedChartTmpl = collectorapi.Chart{ |
| 454 | ID: "node_%s_cluster_%s_thread_pool_rejected", |
| 455 | Title: "Thread Pool Rejected Threads Count", |
| 456 | Units: "threads", |
| 457 | Fam: "thread pool", |
| 458 | Ctx: "elasticsearch.node_thread_pool_rejected", |
| 459 | Type: collectorapi.Stacked, |
| 460 | Priority: prioNodeThreadPoolRejected, |
| 461 | Dims: collectorapi.Dims{ |
| 462 | {ID: "node_%s_thread_pool_generic_rejected", Name: "generic"}, |
| 463 | {ID: "node_%s_thread_pool_search_rejected", Name: "search"}, |
| 464 | {ID: "node_%s_thread_pool_search_throttled_rejected", Name: "search_throttled"}, |
| 465 | {ID: "node_%s_thread_pool_get_rejected", Name: "get"}, |
| 466 | {ID: "node_%s_thread_pool_analyze_rejected", Name: "analyze"}, |
| 467 | {ID: "node_%s_thread_pool_write_rejected", Name: "write"}, |
| 468 | {ID: "node_%s_thread_pool_snapshot_rejected", Name: "snapshot"}, |
| 469 | {ID: "node_%s_thread_pool_warmer_rejected", Name: "warmer"}, |
| 470 | {ID: "node_%s_thread_pool_refresh_rejected", Name: "refresh"}, |
| 471 | {ID: "node_%s_thread_pool_listener_rejected", Name: "listener"}, |
| 472 | {ID: "node_%s_thread_pool_fetch_shard_started_rejected", Name: "fetch_shard_started"}, |
| 473 | {ID: "node_%s_thread_pool_fetch_shard_store_rejected", Name: "fetch_shard_store"}, |
| 474 | {ID: "node_%s_thread_pool_flush_rejected", Name: "flush"}, |
| 475 | {ID: "node_%s_thread_pool_force_merge_rejected", Name: "force_merge"}, |
| 476 | {ID: "node_%s_thread_pool_management_rejected", Name: "management"}, |
| 477 | }, |
| 478 | } |
| 479 | |
| 480 | nodeClusterCommunicationPacketsChartTmpl = collectorapi.Chart{ |
| 481 | ID: "node_%s_cluster_%s_cluster_communication_packets", |
| 482 | Title: "Node Cluster Communication", |
| 483 | Units: "pps", |
| 484 | Fam: "transport", |
| 485 | Ctx: "elasticsearch.node_cluster_communication_packets", |
| 486 | Priority: prioNodeClusterCommunicationPackets, |
| 487 | Dims: collectorapi.Dims{ |
| 488 | {ID: "node_%s_transport_rx_count", Name: "received", Algo: collectorapi.Incremental}, |
| 489 | {ID: "node_%s_transport_tx_count", Name: "sent", Mul: -1, Algo: collectorapi.Incremental}, |
| 490 | }, |
| 491 | } |
| 492 | nodeClusterCommunicationChartTmpl = collectorapi.Chart{ |
| 493 | ID: "node_%s_cluster_%s_cluster_communication_traffic", |
| 494 | Title: "Cluster Communication Bandwidth", |
| 495 | Units: "bytes/s", |
| 496 | Fam: "transport", |
| 497 | Ctx: "elasticsearch.node_cluster_communication_traffic", |
| 498 | Priority: prioNodeClusterCommunication, |
| 499 | Dims: collectorapi.Dims{ |
| 500 | {ID: "node_%s_transport_rx_size_in_bytes", Name: "received", Algo: collectorapi.Incremental}, |
| 501 | {ID: "node_%s_transport_tx_size_in_bytes", Name: "sent", Mul: -1, Algo: collectorapi.Incremental}, |
| 502 | }, |
| 503 | } |
| 504 | |
| 505 | nodeHTTPConnectionsChartTmpl = collectorapi.Chart{ |
| 506 | ID: "node_%s_cluster_%s_http_connections", |
| 507 | Title: "HTTP Connections", |
| 508 | Units: "connections", |
| 509 | Fam: "http", |
| 510 | Ctx: "elasticsearch.node_http_connections", |
| 511 | Priority: prioNodeHTTPConnections, |
| 512 | Dims: collectorapi.Dims{ |
| 513 | {ID: "node_%s_http_current_open", Name: "open"}, |
| 514 | }, |
| 515 | } |
| 516 | |
| 517 | nodeBreakersTripsChartTmpl = collectorapi.Chart{ |
| 518 | ID: "node_%s_cluster_%s_breakers_trips", |
| 519 | Title: "Circuit Breaker Trips Count", |
| 520 | Units: "trips/s", |
| 521 | Fam: "circuit breakers", |
| 522 | Ctx: "elasticsearch.node_breakers_trips", |
| 523 | Type: collectorapi.Stacked, |
| 524 | Priority: prioNodeBreakersTrips, |
| 525 | Dims: collectorapi.Dims{ |
| 526 | {ID: "node_%s_breakers_request_tripped", Name: "requests", Algo: collectorapi.Incremental}, |
| 527 | {ID: "node_%s_breakers_fielddata_tripped", Name: "fielddata", Algo: collectorapi.Incremental}, |
| 528 | {ID: "node_%s_breakers_in_flight_requests_tripped", Name: "in_flight_requests", Algo: collectorapi.Incremental}, |
| 529 | {ID: "node_%s_breakers_model_inference_tripped", Name: "model_inference", Algo: collectorapi.Incremental}, |
| 530 | {ID: "node_%s_breakers_accounting_tripped", Name: "accounting", Algo: collectorapi.Incremental}, |
| 531 | {ID: "node_%s_breakers_parent_tripped", Name: "parent", Algo: collectorapi.Incremental}, |
| 532 | }, |
| 533 | } |
| 534 | ) |
| 535 | |
| 536 | var clusterHealthChartsTmpl = collectorapi.Charts{ |
| 537 | clusterStatusChartTmpl.Copy(), |
| 538 | clusterNodesCountChartTmpl.Copy(), |
| 539 | clusterShardsCountChartTmpl.Copy(), |
| 540 | clusterPendingTasksChartTmpl.Copy(), |
| 541 | clusterInFlightFetchesCountChartTmpl.Copy(), |
| 542 | } |
| 543 | |
| 544 | var ( |
| 545 | clusterStatusChartTmpl = collectorapi.Chart{ |
| 546 | ID: "cluster_%s_status", |
| 547 | Title: "Cluster Status", |
| 548 | Units: "status", |
| 549 | Fam: "cluster health", |
| 550 | Ctx: "elasticsearch.cluster_health_status", |
| 551 | Priority: prioClusterStatus, |
| 552 | Dims: collectorapi.Dims{ |
| 553 | {ID: "cluster_status_green", Name: "green"}, |
| 554 | {ID: "cluster_status_red", Name: "red"}, |
| 555 | {ID: "cluster_status_yellow", Name: "yellow"}, |
| 556 | }, |
| 557 | } |
| 558 | clusterNodesCountChartTmpl = collectorapi.Chart{ |
| 559 | ID: "cluster_%s_number_of_nodes", |
| 560 | Title: "Cluster Nodes Count", |
| 561 | Units: "nodes", |
| 562 | Fam: "cluster health", |
| 563 | Ctx: "elasticsearch.cluster_number_of_nodes", |
| 564 | Priority: prioClusterNodesCount, |
| 565 | Dims: collectorapi.Dims{ |
| 566 | {ID: "cluster_number_of_nodes", Name: "nodes"}, |
| 567 | {ID: "cluster_number_of_data_nodes", Name: "data_nodes"}, |
| 568 | }, |
| 569 | } |
| 570 | clusterShardsCountChartTmpl = collectorapi.Chart{ |
| 571 | ID: "cluster_%s_shards_count", |
| 572 | Title: "Cluster Shards Count", |
| 573 | Units: "shards", |
| 574 | Fam: "cluster health", |
| 575 | Ctx: "elasticsearch.cluster_shards_count", |
| 576 | Priority: prioClusterShardsCount, |
| 577 | Dims: collectorapi.Dims{ |
| 578 | {ID: "cluster_active_primary_shards", Name: "active_primary"}, |
| 579 | {ID: "cluster_active_shards", Name: "active"}, |
| 580 | {ID: "cluster_relocating_shards", Name: "relocating"}, |
| 581 | {ID: "cluster_initializing_shards", Name: "initializing"}, |
| 582 | {ID: "cluster_unassigned_shards", Name: "unassigned"}, |
| 583 | {ID: "cluster_delayed_unassigned_shards", Name: "delayed_unassigned"}, |
| 584 | }, |
| 585 | } |
| 586 | clusterPendingTasksChartTmpl = collectorapi.Chart{ |
| 587 | ID: "cluster_%s_pending_tasks", |
| 588 | Title: "Cluster Pending Tasks", |
| 589 | Units: "tasks", |
| 590 | Fam: "cluster health", |
| 591 | Ctx: "elasticsearch.cluster_pending_tasks", |
| 592 | Priority: prioClusterPendingTasks, |
| 593 | Dims: collectorapi.Dims{ |
| 594 | {ID: "cluster_number_of_pending_tasks", Name: "pending"}, |
| 595 | }, |
| 596 | } |
| 597 | clusterInFlightFetchesCountChartTmpl = collectorapi.Chart{ |
| 598 | ID: "cluster_%s_number_of_in_flight_fetch", |
| 599 | Title: "Cluster Unfinished Fetches", |
| 600 | Units: "fetches", |
| 601 | Fam: "cluster health", |
| 602 | Ctx: "elasticsearch.cluster_number_of_in_flight_fetch", |
| 603 | Priority: prioClusterInFlightFetchesCount, |
| 604 | Dims: collectorapi.Dims{ |
| 605 | {ID: "cluster_number_of_in_flight_fetch", Name: "in_flight_fetch"}, |
| 606 | }, |
| 607 | } |
| 608 | ) |
| 609 | |
| 610 | var clusterStatsChartsTmpl = collectorapi.Charts{ |
| 611 | clusterIndicesCountChartTmpl.Copy(), |
| 612 | clusterIndicesShardsCountChartTmpl.Copy(), |
| 613 | clusterIndicesDocsCountChartTmpl.Copy(), |
| 614 | clusterIndicesStoreSizeChartTmpl.Copy(), |
| 615 | clusterIndicesQueryCacheChartTmpl.Copy(), |
| 616 | clusterNodesByRoleCountChartTmpl.Copy(), |
| 617 | } |
| 618 | |
| 619 | var ( |
| 620 | clusterIndicesCountChartTmpl = collectorapi.Chart{ |
| 621 | ID: "cluster_%s_indices_count", |
| 622 | Title: "Cluster Indices Count", |
| 623 | Units: "indices", |
| 624 | Fam: "cluster stats", |
| 625 | Ctx: "elasticsearch.cluster_indices_count", |
| 626 | Priority: prioClusterIndicesCount, |
| 627 | Dims: collectorapi.Dims{ |
| 628 | {ID: "cluster_indices_count", Name: "indices"}, |
| 629 | }, |
| 630 | } |
| 631 | clusterIndicesShardsCountChartTmpl = collectorapi.Chart{ |
| 632 | ID: "cluster_%s_indices_shards_count", |
| 633 | Title: "Cluster Indices Shards Count", |
| 634 | Units: "shards", |
| 635 | Fam: "cluster stats", |
| 636 | Ctx: "elasticsearch.cluster_indices_shards_count", |
| 637 | Priority: prioClusterIndicesShardsCount, |
| 638 | Dims: collectorapi.Dims{ |
| 639 | {ID: "cluster_indices_shards_total", Name: "total"}, |
| 640 | {ID: "cluster_indices_shards_primaries", Name: "primaries"}, |
| 641 | {ID: "cluster_indices_shards_replication", Name: "replication"}, |
| 642 | }, |
| 643 | } |
| 644 | clusterIndicesDocsCountChartTmpl = collectorapi.Chart{ |
| 645 | ID: "cluster_%s_indices_docs_count", |
| 646 | Title: "Cluster Indices Docs Count", |
| 647 | Units: "docs", |
| 648 | Fam: "cluster stats", |
| 649 | Ctx: "elasticsearch.cluster_indices_docs_count", |
| 650 | Priority: prioClusterIndicesDocsCount, |
| 651 | Dims: collectorapi.Dims{ |
| 652 | {ID: "cluster_indices_docs_count", Name: "docs"}, |
| 653 | }, |
| 654 | } |
| 655 | clusterIndicesStoreSizeChartTmpl = collectorapi.Chart{ |
| 656 | ID: "cluster_%s_indices_store_size", |
| 657 | Title: "Cluster Indices Store Size", |
| 658 | Units: "bytes", |
| 659 | Fam: "cluster stats", |
| 660 | Ctx: "elasticsearch.cluster_indices_store_size", |
| 661 | Priority: prioClusterIndicesStoreSize, |
| 662 | Dims: collectorapi.Dims{ |
| 663 | {ID: "cluster_indices_store_size_in_bytes", Name: "size"}, |
| 664 | }, |
| 665 | } |
| 666 | clusterIndicesQueryCacheChartTmpl = collectorapi.Chart{ |
| 667 | ID: "cluster_%s_indices_query_cache", |
| 668 | Title: "Cluster Indices Query Cache", |
| 669 | Units: "events/s", |
| 670 | Fam: "cluster stats", |
| 671 | Ctx: "elasticsearch.cluster_indices_query_cache", |
| 672 | Type: collectorapi.Stacked, |
| 673 | Priority: prioClusterIndicesQueryCache, |
| 674 | Dims: collectorapi.Dims{ |
| 675 | {ID: "cluster_indices_query_cache_hit_count", Name: "hit", Algo: collectorapi.Incremental}, |
| 676 | {ID: "cluster_indices_query_cache_miss_count", Name: "miss", Algo: collectorapi.Incremental}, |
| 677 | }, |
| 678 | } |
| 679 | clusterNodesByRoleCountChartTmpl = collectorapi.Chart{ |
| 680 | ID: "cluster_%s_nodes_by_role_count", |
| 681 | Title: "Cluster Nodes By Role Count", |
| 682 | Units: "nodes", |
| 683 | Fam: "cluster stats", |
| 684 | Ctx: "elasticsearch.cluster_nodes_by_role_count", |
| 685 | Priority: prioClusterNodesByRoleCount, |
| 686 | Dims: collectorapi.Dims{ |
| 687 | {ID: "cluster_nodes_count_coordinating_only", Name: "coordinating_only"}, |
| 688 | {ID: "cluster_nodes_count_data", Name: "data"}, |
| 689 | {ID: "cluster_nodes_count_data_cold", Name: "data_cold"}, |
| 690 | {ID: "cluster_nodes_count_data_content", Name: "data_content"}, |
| 691 | {ID: "cluster_nodes_count_data_frozen", Name: "data_frozen"}, |
| 692 | {ID: "cluster_nodes_count_data_hot", Name: "data_hot"}, |
| 693 | {ID: "cluster_nodes_count_data_warm", Name: "data_warm"}, |
| 694 | {ID: "cluster_nodes_count_ingest", Name: "ingest"}, |
| 695 | {ID: "cluster_nodes_count_master", Name: "master"}, |
| 696 | {ID: "cluster_nodes_count_ml", Name: "ml"}, |
| 697 | {ID: "cluster_nodes_count_remote_cluster_client", Name: "remote_cluster_client"}, |
| 698 | {ID: "cluster_nodes_count_voting_only", Name: "voting_only"}, |
| 699 | }, |
| 700 | } |
| 701 | ) |
| 702 | |
| 703 | var nodeIndexChartsTmpl = collectorapi.Charts{ |
| 704 | nodeIndexHealthChartTmpl.Copy(), |
| 705 | nodeIndexShardsCountChartTmpl.Copy(), |
| 706 | nodeIndexDocsCountChartTmpl.Copy(), |
| 707 | nodeIndexStoreSizeChartTmpl.Copy(), |
| 708 | } |
| 709 | |
| 710 | var ( |
| 711 | nodeIndexHealthChartTmpl = collectorapi.Chart{ |
| 712 | ID: "node_index_%s_cluster_%s_health", |
| 713 | Title: "Index Health", |
| 714 | Units: "status", |
| 715 | Fam: "index stats", |
| 716 | Ctx: "elasticsearch.node_index_health", |
| 717 | Priority: prioNodeIndexHealth, |
| 718 | Dims: collectorapi.Dims{ |
| 719 | {ID: "node_index_%s_stats_health_green", Name: "green"}, |
| 720 | {ID: "node_index_%s_stats_health_red", Name: "red"}, |
| 721 | {ID: "node_index_%s_stats_health_yellow", Name: "yellow"}, |
| 722 | }, |
| 723 | } |
| 724 | nodeIndexShardsCountChartTmpl = collectorapi.Chart{ |
| 725 | ID: "node_index_%s_cluster_%s_shards_count", |
| 726 | Title: "Index Shards Count", |
| 727 | Units: "shards", |
| 728 | Fam: "index stats", |
| 729 | Ctx: "elasticsearch.node_index_shards_count", |
| 730 | Priority: prioNodeIndexShardsCount, |
| 731 | Dims: collectorapi.Dims{ |
| 732 | {ID: "node_index_%s_stats_shards_count", Name: "shards"}, |
| 733 | }, |
| 734 | } |
| 735 | nodeIndexDocsCountChartTmpl = collectorapi.Chart{ |
| 736 | ID: "node_index_%s_cluster_%s_docs_count", |
| 737 | Title: "Index Docs Count", |
| 738 | Units: "docs", |
| 739 | Fam: "index stats", |
| 740 | Ctx: "elasticsearch.node_index_docs_count", |
| 741 | Priority: prioNodeIndexDocsCount, |
| 742 | Dims: collectorapi.Dims{ |
| 743 | {ID: "node_index_%s_stats_docs_count", Name: "docs"}, |
| 744 | }, |
| 745 | } |
| 746 | nodeIndexStoreSizeChartTmpl = collectorapi.Chart{ |
| 747 | ID: "node_index_%s_cluster_%s_store_size", |
| 748 | Title: "Index Store Size", |
| 749 | Units: "bytes", |
| 750 | Fam: "index stats", |
| 751 | Ctx: "elasticsearch.node_index_store_size", |
| 752 | Priority: prioNodeIndexStoreSize, |
| 753 | Dims: collectorapi.Dims{ |
| 754 | {ID: "node_index_%s_stats_store_size_in_bytes", Name: "store_size"}, |
| 755 | }, |
| 756 | } |
| 757 | ) |
| 758 | |
| 759 | func (c *Collector) addClusterStatsCharts() { |
| 760 | charts := clusterStatsChartsTmpl.Copy() |
| 761 | |
| 762 | for _, chart := range *charts { |
| 763 | chart.ID = fmt.Sprintf(chart.ID, c.clusterName) |
| 764 | chart.Labels = []collectorapi.Label{ |
| 765 | {Key: "cluster_name", Value: c.clusterName}, |
| 766 | } |
| 767 | } |
| 768 | |
| 769 | if err := c.charts.Add(*charts...); err != nil { |
| 770 | c.Warning(err) |
| 771 | } |
| 772 | } |
| 773 | |
| 774 | func (c *Collector) addClusterHealthCharts() { |
| 775 | charts := clusterHealthChartsTmpl.Copy() |
| 776 | |
| 777 | for _, chart := range *charts { |
| 778 | chart.ID = fmt.Sprintf(chart.ID, c.clusterName) |
| 779 | chart.Labels = []collectorapi.Label{ |
| 780 | {Key: "cluster_name", Value: c.clusterName}, |
| 781 | } |
| 782 | } |
| 783 | |
| 784 | if err := c.charts.Add(*charts...); err != nil { |
| 785 | c.Warning(err) |
| 786 | } |
| 787 | } |
| 788 | |
| 789 | func (c *Collector) addNodeCharts(nodeID string, node *esNodeStats) { |
| 790 | charts := nodeChartsTmpl.Copy() |
| 791 | |
| 792 | for _, chart := range *charts { |
| 793 | chart.ID = fmt.Sprintf(chart.ID, nodeID, c.clusterName) |
| 794 | chart.Labels = []collectorapi.Label{ |
| 795 | {Key: "cluster_name", Value: c.clusterName}, |
| 796 | {Key: "node_name", Value: node.Name}, |
| 797 | {Key: "host", Value: node.Host}, |
| 798 | } |
| 799 | for _, dim := range chart.Dims { |
| 800 | dim.ID = fmt.Sprintf(dim.ID, nodeID) |
| 801 | } |
| 802 | } |
| 803 | |
| 804 | if err := c.Charts().Add(*charts...); err != nil { |
| 805 | c.Warning(err) |
| 806 | } |
| 807 | } |
| 808 | |
| 809 | func (c *Collector) removeNodeCharts(nodeID string) { |
| 810 | px := fmt.Sprintf("node_%s_cluster_%s_", nodeID, c.clusterName) |
| 811 | c.removeCharts(px) |
| 812 | } |
| 813 | |
| 814 | func (c *Collector) addIndexCharts(index string) { |
| 815 | charts := nodeIndexChartsTmpl.Copy() |
| 816 | |
| 817 | for _, chart := range *charts { |
| 818 | chart.ID = fmt.Sprintf(chart.ID, index, c.clusterName) |
| 819 | chart.Labels = []collectorapi.Label{ |
| 820 | {Key: "cluster_name", Value: c.clusterName}, |
| 821 | {Key: "index", Value: index}, |
| 822 | } |
| 823 | for _, dim := range chart.Dims { |
| 824 | dim.ID = fmt.Sprintf(dim.ID, index) |
| 825 | } |
| 826 | } |
| 827 | |
| 828 | if err := c.Charts().Add(*charts...); err != nil { |
| 829 | c.Warning(err) |
| 830 | } |
| 831 | } |
| 832 | |
| 833 | func (c *Collector) removeIndexCharts(index string) { |
| 834 | px := fmt.Sprintf("node_index_%s_cluster_%s_", index, c.clusterName) |
| 835 | c.removeCharts(px) |
| 836 | } |
| 837 | |
| 838 | func (c *Collector) removeCharts(prefix string) { |
| 839 | for _, chart := range *c.Charts() { |
| 840 | if strings.HasPrefix(chart.ID, prefix) { |
| 841 | chart.MarkRemove() |
| 842 | chart.MarkNotCreated() |
| 843 | } |
| 844 | } |
| 845 | } |