master
yaml 798 lines 32.1 KB
Raw
1 plugin_name: go.d.plugin
2 modules:
3 - &module
4 meta: &meta
5 id: collector-go.d.plugin-elasticsearch
6 module_name: elasticsearch
7 plugin_name: go.d.plugin
8 monitored_instance:
9 name: Elasticsearch
10 link: https://www.elastic.co/elasticsearch/
11 icon_filename: elasticsearch.svg
12 categories:
13 - data-collection.databases
14 keywords:
15 - elastic
16 - elasticsearch
17 - opensearch
18 - search engine
19 related_resources:
20 integrations:
21 list:
22 - plugin_name: apps.plugin
23 module_name: apps
24 - plugin_name: cgroups.plugin
25 module_name: /sys/fs/cgroup
26 monitored_instance_name: Containers
27 info_provided_to_referring_integrations:
28 description: ""
29 overview:
30 data_collection:
31 metrics_description: |
32 This collector monitors the performance and health of the Elasticsearch cluster.
33 method_description: |
34 It uses [Cluster APIs](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster.html) to collect metrics.
35
36 Used endpoints:
37
38 | Endpoint | Description | API |
39 |------------------------|----------------------|-------------------------------------------------------------------------------------------------------------|
40 | `/` | Node info | |
41 | `/_nodes/stats` | Nodes metrics | [Nodes stats API](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-nodes-stats.html) |
42 | `/_nodes/_local/stats` | Local node metrics | [Nodes stats API](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-nodes-stats.html) |
43 | `/_cluster/health` | Cluster health stats | [Cluster health API](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html) |
44 | `/_cluster/stats` | Cluster metrics | [Cluster stats API](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-stats.html) |
45 supported_platforms:
46 include: []
47 exclude: []
48 multi_instance: true
49 additional_permissions:
50 description: ""
51 default_behavior:
52 auto_detection:
53 description: |
54 By default, it detects instances running on localhost by attempting to connect to port 9200:
55
56 - http://127.0.0.1:9200
57 - https://127.0.0.1:9200
58 limits:
59 description: |
60 By default, this collector monitors only the node it is connected to. To monitor all cluster nodes, set the `cluster_mode` configuration option to `yes`.
61 performance_impact:
62 description: ""
63 setup:
64 prerequisites:
65 list: []
66 configuration:
67 file:
68 name: "go.d/elasticsearch.conf"
69 options:
70 description: |
71 The following options can be defined globally: update_every, autodetection_retry.
72 folding:
73 title: Config options
74 enabled: true
75 list:
76 - name: update_every
77 description: Data collection interval (seconds).
78 default_value: 5
79 required: false
80 group: Collection
81 - name: autodetection_retry
82 description: Autodetection retry interval (seconds). Set 0 to disable.
83 default_value: 0
84 required: false
85 group: Collection
86
87 - name: url
88 description: Target endpoint URL.
89 default_value: http://127.0.0.1:9200
90 required: true
91 group: Target
92 - name: timeout
93 description: HTTP request timeout (seconds).
94 default_value: 2
95 required: false
96 group: Target
97
98 - name: cluster_mode
99 description: Collect metrics for all nodes in the cluster (yes) or only the local node (no).
100 default_value: no
101 required: false
102 group: Metrics Selection
103 - name: collect_node_stats
104 description: Collect node metrics.
105 default_value: yes
106 required: false
107 group: Metrics Selection
108 - name: collect_cluster_health
109 description: Collect cluster health metrics.
110 default_value: yes
111 required: false
112 group: Metrics Selection
113 - name: collect_cluster_stats
114 description: Collect cluster stats metrics.
115 default_value: yes
116 required: false
117 group: Metrics Selection
118 - name: collect_indices_stats
119 description: Collect index metrics.
120 default_value: no
121 required: false
122 group: Metrics Selection
123
124 - name: username
125 description: Username for Basic HTTP authentication.
126 default_value: ""
127 required: false
128 group: HTTP Auth
129 - name: password
130 description: Password for Basic HTTP authentication.
131 default_value: ""
132 required: false
133 group: HTTP Auth
134 - name: bearer_token_file
135 description: "Path to a file containing a bearer token (used for `Authorization: Bearer`)."
136 default_value: ""
137 required: false
138 group: HTTP Auth
139
140 - name: tls_skip_verify
141 description: Skip TLS certificate and hostname verification (insecure).
142 default_value: no
143 required: false
144 group: TLS
145 - name: tls_ca
146 description: Path to CA bundle used to validate the server certificate.
147 default_value: ""
148 required: false
149 group: TLS
150 - name: tls_cert
151 description: Path to client TLS certificate (for mTLS).
152 default_value: ""
153 required: false
154 group: TLS
155 - name: tls_key
156 description: Path to client TLS private key (for mTLS).
157 default_value: ""
158 required: false
159 group: TLS
160
161 - name: proxy_url
162 description: HTTP proxy URL.
163 default_value: ""
164 required: false
165 group: Proxy
166 - name: proxy_username
167 description: Username for proxy Basic HTTP authentication.
168 default_value: ""
169 required: false
170 group: Proxy
171 - name: proxy_password
172 description: Password for proxy Basic HTTP authentication.
173 default_value: ""
174 required: false
175 group: Proxy
176
177 - name: method
178 description: HTTP method to use.
179 default_value: "GET"
180 required: false
181 group: Request
182 - name: body
183 description: Request body (e.g., for POST/PUT).
184 default_value: ""
185 required: false
186 group: Request
187 - name: headers
188 description: "Additional HTTP headers (one per line as key: value)."
189 default_value: ""
190 required: false
191 group: Request
192 - name: not_follow_redirects
193 description: Do not follow HTTP redirects.
194 default_value: no
195 required: false
196 group: Request
197 - name: force_http2
198 description: Force HTTP/2 (including h2c over TCP).
199 default_value: no
200 required: false
201 group: Request
202
203 - name: functions.top_queries.disabled
204 description: Disable the [top-queries](#top-queries) function.
205 default_value: false
206 required: false
207 group: Functions
208 - name: functions.top_queries.timeout
209 description: Query timeout (seconds). Uses collector timeout if not set.
210 default_value: ""
211 required: false
212 group: Functions
213 - name: functions.top_queries.limit
214 description: Maximum number of queries to return.
215 default_value: 500
216 required: false
217 group: Functions
218
219 - name: vnode
220 description: Associates this data collection job with a [Virtual Node](https://learn.netdata.cloud/docs/netdata-agent/configuration/organize-systems-metrics-and-alerts#virtual-nodes).
221 default_value: ""
222 required: false
223 group: Virtual Node
224 examples:
225 folding:
226 title: Config
227 enabled: true
228 list:
229 - name: Basic single node mode
230 description: A basic example configuration.
231 folding:
232 enabled: false
233 config: |
234 jobs:
235 - name: local
236 url: http://127.0.0.1:9200
237 - name: Cluster mode
238 description: Cluster mode example configuration.
239 config: |
240 jobs:
241 - name: local
242 url: http://127.0.0.1:9200
243 cluster_mode: yes
244 - name: HTTP authentication
245 description: Basic HTTP authentication.
246 config: |
247 jobs:
248 - name: local
249 url: http://127.0.0.1:9200
250 username: username
251 password: password
252 - name: HTTPS with self-signed certificate
253 description: Elasticsearch with enabled HTTPS and self-signed certificate.
254 config: |
255 jobs:
256 - name: local
257 url: https://127.0.0.1:9200
258 tls_skip_verify: yes
259 - name: Multi-instance
260 description: |
261 > **Note**: When you define multiple jobs, their names must be unique.
262
263 Collecting metrics from local and remote instances.
264 config: |
265 jobs:
266 - name: local
267 url: http://127.0.0.1:9200
268
269 - name: remote
270 url: http://192.0.2.1:9200
271 troubleshooting:
272 problems:
273 list: []
274 alerts:
275 - name: elasticsearch_node_indices_search_time_query
276 link: https://github.com/netdata/netdata/blob/master/src/health/health.d/elasticsearch.conf
277 metric: elasticsearch.node_indices_search_time
278 info: search performance is degraded, queries run slowly.
279 - name: elasticsearch_node_indices_search_time_fetch
280 link: https://github.com/netdata/netdata/blob/master/src/health/health.d/elasticsearch.conf
281 metric: elasticsearch.node_indices_search_time
282 info: search performance is degraded, fetches run slowly.
283 - name: elasticsearch_cluster_health_status_red
284 link: https://github.com/netdata/netdata/blob/master/src/health/health.d/elasticsearch.conf
285 metric: elasticsearch.cluster_health_status
286 info: cluster health status is red.
287 - name: elasticsearch_cluster_health_status_yellow
288 link: https://github.com/netdata/netdata/blob/master/src/health/health.d/elasticsearch.conf
289 metric: elasticsearch.cluster_health_status
290 info: cluster health status is yellow.
291 - name: elasticsearch_node_index_health_red
292 link: https://github.com/netdata/netdata/blob/master/src/health/health.d/elasticsearch.conf
293 metric: elasticsearch.node_index_health
294 info: node index $label:index health status is red.
295 functions:
296 description: |
297 This collector exposes real-time functions for interactive troubleshooting in the Live tab.
298 list:
299 - id: top-queries
300 name: Top Queries
301 description: |
302 Retrieves currently running search tasks from the Elasticsearch [Tasks API](https://www.elastic.co/guide/en/elasticsearch/reference/current/tasks.html).
303
304 This function queries the `/_tasks` endpoint filtered for search actions (`*search`), providing a real-time snapshot of all active search operations across all nodes in the cluster.
305
306 Use cases:
307 - Identify long-running search queries that may be impacting cluster performance
308 - Monitor active search workload distribution across cluster nodes
309 - Debug slow or stuck search operations in real-time
310 parameters:
311 - id: __sort
312 name: Filter By
313 description: Select the primary sort column. Options include running time, start time, and task ID. Defaults to running time to show longest-running searches first.
314 type: select
315 required: true
316 default: runningTime
317 options: []
318 returns:
319 description: Real-time snapshot of currently executing search tasks across all cluster nodes. Each row represents a single active search operation.
320 columns:
321 - name: Task ID
322 type: string
323 unit: ""
324 visibility: hidden
325 description: "Unique identifier for the task in format `nodeId:taskId`. Can be used with the Task Management API to cancel long-running tasks."
326 - name: Node ID
327 type: string
328 unit: ""
329 description: "Internal identifier of the node executing this search task."
330 - name: Node Name
331 type: string
332 unit: ""
333 description: "Human-readable name of the node executing the search. Useful for identifying workload distribution across the cluster."
334 - name: Action
335 type: string
336 unit: ""
337 description: "The search action being performed (e.g., `indices:data/read/search`). Indicates the type of search operation."
338 - name: Type
339 type: string
340 unit: ""
341 visibility: hidden
342 description: "Task type classification (typically `transport` for search tasks)."
343 - name: Description
344 type: string
345 unit: ""
346 description: "Detailed description of the search task including indices being searched and query details. Truncated to 4096 characters."
347 - name: Start Time
348 type: timestamp
349 unit: ""
350 description: "Timestamp when the search task started executing."
351 - name: Running Time
352 type: duration
353 unit: "milliseconds"
354 description: "Time elapsed since the search started. High values indicate long-running searches that may need investigation or cancellation."
355 - name: Cancellable
356 type: boolean
357 unit: ""
358 visibility: hidden
359 description: "Whether the task supports cancellation via the Task Management API."
360 - name: Cancelled
361 type: boolean
362 unit: ""
363 visibility: hidden
364 description: "Whether a cancellation request has been issued for this task."
365 performance: |
366 Queries the `/_tasks` API filtered for search actions:<br/>• Lightweight operation with minimal cluster overhead<br/>• Returns only currently active search tasks, typically a small result set
367 security: |
368 Task descriptions may contain query details including potentially sensitive information:<br/>• Index names and search patterns<br/>• Query terms and filter values<br/>• Access should be restricted to authorized personnel only
369 prerequisites:
370 list:
371 - title: Ensure access to Tasks API
372 description: |
373 The user must have appropriate privileges to access the Tasks API.
374
375 1. For secured clusters, grant the `monitor` or `manage` cluster privilege:
376
377 ```json
378 {
379 "cluster": ["monitor"]
380 }
381 ```
382
383 2. Verify access to the Tasks API:
384
385 ```bash
386 curl -u user:password "http://localhost:9200/_tasks?actions=*search"
387 ```
388
389 :::info
390
391 - The Tasks API returns only currently running tasks; completed tasks are not stored
392 - Search tasks can be cancelled using `POST /_tasks/{task_id}/_cancel` if they are cancellable
393 - Works with both Elasticsearch and OpenSearch clusters
394
395 :::
396 availability: |
397 Available when:<br/>• The collector has successfully connected to Elasticsearch/OpenSearch<br/>• The user has `monitor` or `manage` cluster privileges<br/>• Returns HTTP 503 if collector is still initializing<br/>• Returns HTTP 500 if the Tasks API query fails<br/>• Returns HTTP 504 if the query times out
398 require_cloud: true
399 metrics:
400 folding:
401 title: Metrics
402 enabled: false
403 description: ""
404 availability: []
405 scopes:
406 - name: node
407 description: These metrics refer to the cluster node.
408 labels:
409 - name: cluster_name
410 description: |
411 Name of the cluster. Based on the [Cluster name setting](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#cluster-name).
412 - name: node_name
413 description: |
414 Human-readable identifier for the node. Based on the [Node name setting](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#node-name).
415 - name: host
416 description: |
417 Network host for the node, based on the [Network host setting](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#network.host).
418 metrics:
419 - name: elasticsearch.node_indices_indexing
420 description: Indexing Operations
421 unit: operations/s
422 chart_type: line
423 dimensions:
424 - name: index
425 - name: elasticsearch.node_indices_indexing_current
426 description: Indexing Operations Current
427 unit: operations
428 chart_type: line
429 dimensions:
430 - name: index
431 - name: elasticsearch.node_indices_indexing_time
432 description: Time Spent On Indexing Operations
433 unit: milliseconds
434 chart_type: line
435 dimensions:
436 - name: index
437 - name: elasticsearch.node_indices_search
438 description: Search Operations
439 unit: operations/s
440 chart_type: stacked
441 dimensions:
442 - name: queries
443 - name: fetches
444 - name: elasticsearch.node_indices_search_current
445 description: Search Operations Current
446 unit: operations
447 chart_type: stacked
448 dimensions:
449 - name: queries
450 - name: fetches
451 - name: elasticsearch.node_indices_search_time
452 description: node_indices_search_time
453 unit: milliseconds
454 chart_type: stacked
455 dimensions:
456 - name: queries
457 - name: fetches
458 - name: elasticsearch.node_indices_refresh
459 description: Refresh Operations
460 unit: operations/s
461 chart_type: line
462 dimensions:
463 - name: refresh
464 - name: elasticsearch.node_indices_refresh_time
465 description: Time Spent On Refresh Operations
466 unit: milliseconds
467 chart_type: line
468 dimensions:
469 - name: refresh
470 - name: elasticsearch.node_indices_flush
471 description: Flush Operations
472 unit: operations/s
473 chart_type: line
474 dimensions:
475 - name: flush
476 - name: elasticsearch.node_indices_flush_time
477 description: Time Spent On Flush Operations
478 unit: milliseconds
479 chart_type: line
480 dimensions:
481 - name: flush
482 - name: elasticsearch.node_indices_fielddata_memory_usage
483 description: Fielddata Cache Memory Usage
484 unit: bytes
485 chart_type: area
486 dimensions:
487 - name: used
488 - name: elasticsearch.node_indices_fielddata_evictions
489 description: Fielddata Evictions
490 unit: operations/s
491 chart_type: line
492 dimensions:
493 - name: evictions
494 - name: elasticsearch.node_indices_segments_count
495 description: Segments Count
496 unit: segments
497 chart_type: line
498 dimensions:
499 - name: segments
500 - name: elasticsearch.node_indices_segments_memory_usage_total
501 description: Segments Memory Usage Total
502 unit: bytes
503 chart_type: line
504 dimensions:
505 - name: used
506 - name: elasticsearch.node_indices_segments_memory_usage
507 description: Segments Memory Usage
508 unit: bytes
509 chart_type: stacked
510 dimensions:
511 - name: terms
512 - name: stored_fields
513 - name: term_vectors
514 - name: norms
515 - name: points
516 - name: doc_values
517 - name: index_writer
518 - name: version_map
519 - name: fixed_bit_set
520 - name: elasticsearch.node_indices_translog_operations
521 description: Translog Operations
522 unit: operations
523 chart_type: area
524 dimensions:
525 - name: total
526 - name: uncommitted
527 - name: elasticsearch.node_indices_translog_size
528 description: Translog Size
529 unit: bytes
530 chart_type: area
531 dimensions:
532 - name: total
533 - name: uncommitted
534 - name: elasticsearch.node_file_descriptors
535 description: Process File Descriptors
536 unit: fd
537 chart_type: line
538 dimensions:
539 - name: open
540 - name: elasticsearch.node_jvm_heap
541 description: JVM Heap Percentage Currently in Use
542 unit: percentage
543 chart_type: area
544 dimensions:
545 - name: inuse
546 - name: elasticsearch.node_jvm_heap_bytes
547 description: JVM Heap Commit And Usage
548 unit: bytes
549 chart_type: area
550 dimensions:
551 - name: committed
552 - name: used
553 - name: elasticsearch.node_jvm_buffer_pools_count
554 description: JVM Buffer Pools Count
555 unit: pools
556 chart_type: line
557 dimensions:
558 - name: direct
559 - name: mapped
560 - name: elasticsearch.node_jvm_buffer_pool_direct_memory
561 description: JVM Buffer Pool Direct Memory
562 unit: bytes
563 chart_type: area
564 dimensions:
565 - name: total
566 - name: used
567 - name: elasticsearch.node_jvm_buffer_pool_mapped_memory
568 description: JVM Buffer Pool Mapped Memory
569 unit: bytes
570 chart_type: area
571 dimensions:
572 - name: total
573 - name: used
574 - name: elasticsearch.node_jvm_gc_count
575 description: JVM Garbage Collections
576 unit: gc/s
577 chart_type: stacked
578 dimensions:
579 - name: young
580 - name: old
581 - name: elasticsearch.node_jvm_gc_time
582 description: JVM Time Spent On Garbage Collections
583 unit: milliseconds
584 chart_type: stacked
585 dimensions:
586 - name: young
587 - name: old
588 - name: elasticsearch.node_thread_pool_queued
589 description: Thread Pool Queued Threads Count
590 unit: threads
591 chart_type: stacked
592 dimensions:
593 - name: generic
594 - name: search
595 - name: search_throttled
596 - name: get
597 - name: analyze
598 - name: write
599 - name: snapshot
600 - name: warmer
601 - name: refresh
602 - name: listener
603 - name: fetch_shard_started
604 - name: fetch_shard_store
605 - name: flush
606 - name: force_merge
607 - name: management
608 - name: elasticsearch.node_thread_pool_rejected
609 description: Thread Pool Rejected Threads Count
610 unit: threads
611 chart_type: stacked
612 dimensions:
613 - name: generic
614 - name: search
615 - name: search_throttled
616 - name: get
617 - name: analyze
618 - name: write
619 - name: snapshot
620 - name: warmer
621 - name: refresh
622 - name: listener
623 - name: fetch_shard_started
624 - name: fetch_shard_store
625 - name: flush
626 - name: force_merge
627 - name: management
628 - name: elasticsearch.node_cluster_communication_packets
629 description: Cluster Communication
630 unit: pps
631 chart_type: line
632 dimensions:
633 - name: received
634 - name: sent
635 - name: elasticsearch.node_cluster_communication_traffic
636 description: Cluster Communication Bandwidth
637 unit: bytes/s
638 chart_type: line
639 dimensions:
640 - name: received
641 - name: sent
642 - name: elasticsearch.node_http_connections
643 description: HTTP Connections
644 unit: connections
645 chart_type: line
646 dimensions:
647 - name: open
648 - name: elasticsearch.node_breakers_trips
649 description: Circuit Breaker Trips Count
650 unit: trips/s
651 chart_type: stacked
652 dimensions:
653 - name: requests
654 - name: fielddata
655 - name: in_flight_requests
656 - name: model_inference
657 - name: accounting
658 - name: parent
659 - name: cluster
660 description: These metrics refer to the cluster.
661 labels:
662 - name: cluster_name
663 description: |
664 Name of the cluster. Based on the [Cluster name setting](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#cluster-name).
665 metrics:
666 - name: elasticsearch.cluster_health_status
667 description: Cluster Status
668 unit: status
669 chart_type: line
670 dimensions:
671 - name: green
672 - name: yellow
673 - name: red
674 - name: elasticsearch.cluster_number_of_nodes
675 description: Cluster Nodes Count
676 unit: nodes
677 chart_type: line
678 dimensions:
679 - name: nodes
680 - name: data_nodes
681 - name: elasticsearch.cluster_shards_count
682 description: Cluster Shards Count
683 unit: shards
684 chart_type: line
685 dimensions:
686 - name: active_primary
687 - name: active
688 - name: relocating
689 - name: initializing
690 - name: unassigned
691 - name: delayed_unaasigned
692 - name: elasticsearch.cluster_pending_tasks
693 description: Cluster Pending Tasks
694 unit: tasks
695 chart_type: line
696 dimensions:
697 - name: pending
698 - name: elasticsearch.cluster_number_of_in_flight_fetch
699 description: Cluster Unfinished Fetches
700 unit: fetches
701 chart_type: line
702 dimensions:
703 - name: in_flight_fetch
704 - name: elasticsearch.cluster_indices_count
705 description: Cluster Indices Count
706 unit: indices
707 chart_type: line
708 dimensions:
709 - name: indices
710 - name: elasticsearch.cluster_indices_shards_count
711 description: Cluster Indices Shards Count
712 unit: shards
713 chart_type: line
714 dimensions:
715 - name: total
716 - name: primaries
717 - name: replication
718 - name: elasticsearch.cluster_indices_docs_count
719 description: Cluster Indices Docs Count
720 unit: docs
721 chart_type: line
722 dimensions:
723 - name: docs
724 - name: elasticsearch.cluster_indices_store_size
725 description: Cluster Indices Store Size
726 unit: bytes
727 chart_type: line
728 dimensions:
729 - name: size
730 - name: elasticsearch.cluster_indices_query_cache
731 description: Cluster Indices Query Cache
732 unit: events/s
733 chart_type: line
734 dimensions:
735 - name: hit
736 - name: miss
737 - name: elasticsearch.cluster_nodes_by_role_count
738 description: Cluster Nodes By Role Count
739 unit: nodes
740 chart_type: line
741 dimensions:
742 - name: coordinating_only
743 - name: data
744 - name: data_cold
745 - name: data_content
746 - name: data_frozen
747 - name: data_hot
748 - name: data_warm
749 - name: ingest
750 - name: master
751 - name: ml
752 - name: remote_cluster_client
753 - name: voting_only
754 - name: index
755 description: These metrics refer to the index.
756 labels:
757 - name: cluster_name
758 description: |
759 Name of the cluster. Based on the [Cluster name setting](https://www.elastic.co/guide/en/elasticsearch/reference/current/important-settings.html#cluster-name).
760 - name: index
761 description: Name of the index.
762 metrics:
763 - name: elasticsearch.node_index_health
764 description: Index Health
765 unit: status
766 chart_type: line
767 dimensions:
768 - name: green
769 - name: yellow
770 - name: red
771 - name: elasticsearch.node_index_shards_count
772 description: Index Shards Count
773 unit: shards
774 chart_type: line
775 dimensions:
776 - name: shards
777 - name: elasticsearch.node_index_docs_count
778 description: Index Docs Count
779 unit: docs
780 chart_type: line
781 dimensions:
782 - name: docs
783 - name: elasticsearch.node_index_store_size
784 description: Index Store Size
785 unit: bytes
786 chart_type: line
787 dimensions:
788 - name: store_size
789 - <<: *module
790 meta:
791 <<: *meta
792 id: collector-go.d.plugin-opensearch
793 monitored_instance:
794 name: OpenSearch
795 link: https://opensearch.org/
796 icon_filename: opensearch.svg
797 categories:
798 - data-collection.databases