| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | package pulsar |
| 4 | |
| 5 | /* |
| 6 | Architecture: |
| 7 | - https://pulsar.apache.org/docs/en/concepts-overview/ |
| 8 | |
| 9 | Terminology: |
| 10 | - https://pulsar.apache.org/docs/en/reference-terminology/ |
| 11 | |
| 12 | Deploy Monitoring: |
| 13 | - http://pulsar.apache.org/docs/en/deploy-monitoring/ |
| 14 | |
| 15 | Metrics Reference: |
| 16 | - https://github.com/apache/pulsar/blob/master/site2/docs/reference-metrics.md |
| 17 | |
| 18 | REST API |
| 19 | - http://pulsar.apache.org/admin-rest-api/?version=master |
| 20 | |
| 21 | Grafana Dashboards: |
| 22 | - https://github.com/apache/pulsar/tree/master/docker/grafana/dashboards |
| 23 | |
| 24 | Stats in the source code: |
| 25 | - https://github.com/apache/pulsar/blob/master/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/ |
| 26 | - https://github.com/apache/pulsar/tree/master/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus |
| 27 | |
| 28 | If !'exposeTopicLevelMetricsInPrometheus: |
| 29 | - https://github.com/apache/pulsar/blob/master/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java |
| 30 | else: |
| 31 | - https://github.com/apache/pulsar/blob/master/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java |
| 32 | |
| 33 | Metrics updates parameters: |
| 34 | - statsUpdateFrequencyInSecs=60 |
| 35 | - statsUpdateInitialDelayInSecs=60 |
| 36 | |
| 37 | Metrics Exposing: |
| 38 | - Namespace : 'exposeTopicLevelMetricsInPrometheus' is set to false. |
| 39 | - Replication : 'replicationMetricsEnabled' is enabled. |
| 40 | - Topic : 'exposeTopicLevelMetricsInPrometheus' is set to true. |
| 41 | - Subscription: 'exposeTopicLevelMetricsInPrometheus' is set to true |
| 42 | - Consumer : 'exposeTopicLevelMetricsInPrometheus' and 'exposeConsumerLevelMetricsInPrometheus' are set to true. |
| 43 | - Publisher : 'exposePublisherStats' is set to true. RESP API option. (/admin/v2/broker-stats/topics) |
| 44 | */ |
| 45 | |
| 46 | /* |
| 47 | TODO: |
| 48 | Unused broker metrics: |
| 49 | - "pulsar_storage_backlog_size" : ?? is estimated total unconsumed or backlog size in bytes for the managed ledger, without accounting for replicas. |
| 50 | - "pulsar_storage_offloaded_size" : ?? is the size of all ledgers offloaded to 2nd tier storage. |
| 51 | - "pulsar_storage_backlog_quota_limit" : ?? is the total amount of the data in this topic that limit the backlog quota. |
| 52 | - "pulsar_in_bytes_total" : use "pulsar_throughput_in" for the same data. |
| 53 | - "pulsar_in_messages_total" : use "pulsar_rate_in" for the same data. |
| 54 | - "pulsar_subscription_unacked_messages" : negative values (https://github.com/apache/pulsar/issues/6510) |
| 55 | - "pulsar_subscription_back_log" : to detailed, we have summary per topic. Part of "pulsar_msg_backlog" (msgBacklog). |
| 56 | - "pulsar_subscription_msg_rate_out" : to detailed, we have summary per topic. Part of "pulsar_rate_out". |
| 57 | - "pulsar_subscription_msg_throughput_out": to detailed, we have summary per topic. Part of "pulsar_throughput_out". |
| 58 | |
| 59 | + All Consumer metrics (for each namespace, topic, subscription). |
| 60 | + JVM metrics. |
| 61 | + Zookeeper metrics. |
| 62 | + Bookkeeper metrics. |
| 63 | |
| 64 | Hardcoded update interval? (60) |
| 65 | - pulsar_storage_write_latency_le_* |
| 66 | - pulsar_entry_size_le_* |
| 67 | */ |
| 68 | |
| 69 | /* |
| 70 | https://github.com/apache/pulsar/blob/master/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java |
| 71 | Zero metrics which always present (labels: cluster): |
| 72 | - "pulsar_topics_count" |
| 73 | - "pulsar_subscriptions_count" |
| 74 | - "pulsar_producers_count" |
| 75 | - "pulsar_consumers_count" |
| 76 | - "pulsar_rate_in" |
| 77 | - "pulsar_rate_out" |
| 78 | - "pulsar_throughput_in" |
| 79 | - "pulsar_throughput_out" |
| 80 | - "pulsar_storage_size" |
| 81 | - "pulsar_storage_write_rate" |
| 82 | - "pulsar_storage_read_rate" |
| 83 | - "pulsar_msg_backlog" |
| 84 | */ |
| 85 | |
| 86 | const ( |
| 87 | // Namespace metrics (labels: namespace) |
| 88 | metricPulsarTopicsCount = "pulsar_topics_count" |
| 89 | // Namespace, Topic metrics (labels: namespace || namespace, topic) |
| 90 | metricPulsarSubscriptionsCount = "pulsar_subscriptions_count" |
| 91 | metricPulsarProducersCount = "pulsar_producers_count" |
| 92 | metricPulsarConsumersCount = "pulsar_consumers_count" |
| 93 | metricPulsarRateIn = "pulsar_rate_in" |
| 94 | metricPulsarRateOut = "pulsar_rate_out" |
| 95 | metricPulsarThroughputIn = "pulsar_throughput_in" |
| 96 | metricPulsarThroughputOut = "pulsar_throughput_out" |
| 97 | metricPulsarStorageSize = "pulsar_storage_size" |
| 98 | metricPulsarStorageWriteRate = "pulsar_storage_write_rate" // exposed with labels only if there is Bookie |
| 99 | metricPulsarStorageReadRate = "pulsar_storage_read_rate" // exposed with labels only if there is Bookie |
| 100 | metricPulsarMsgBacklog = "pulsar_msg_backlog" // has 'remote_cluster' label if no topic stats |
| 101 | // pulsar_storage_write_latency_le_* |
| 102 | // pulsar_entry_size_le_* |
| 103 | |
| 104 | // Subscriptions metrics (labels: namespace, topic, subscription) |
| 105 | metricPulsarSubscriptionDelayed = "pulsar_subscription_delayed" // Number of delayed messages currently being tracked |
| 106 | metricPulsarSubscriptionMsgRateRedeliver = "pulsar_subscription_msg_rate_redeliver" |
| 107 | metricPulsarSubscriptionBlockedOnUnackedMessages = "pulsar_subscription_blocked_on_unacked_messages" |
| 108 | |
| 109 | // Replication metrics (labels: namespace, remote_cluster || namespace, topic, remote_cluster) |
| 110 | // Exposed only when replication is enabled. |
| 111 | metricPulsarReplicationRateIn = "pulsar_replication_rate_in" |
| 112 | metricPulsarReplicationRateOut = "pulsar_replication_rate_out" |
| 113 | metricPulsarReplicationThroughputIn = "pulsar_replication_throughput_in" |
| 114 | metricPulsarReplicationThroughputOut = "pulsar_replication_throughput_out" |
| 115 | metricPulsarReplicationBacklog = "pulsar_replication_backlog" |
| 116 | ) |