@cryptotaxi247 / CoPilot / commits / 7907aa03

676 feature request display total index size per customer in indicesmanagement (#687)

* feat: add endpoint to fetch total indices size per customer * feat: add component to display total indices size per customer * feat: enhance customer indices display with popover for index details * feat: enable index selection in CustomerIndicesSize component * precommit fixes * lint fixes * chore: update CURRENT_VERSION to 0.1.43

taylor_socfortress committed Feb 10, 2026 at 09:49 UTC 7907aa03e4c5a0375cd17689c425b632b5c5996b
7 files changed +454 -69
backend/app/connectors/wazuh_indexer/routes/monitoring.py
+31
@@ -6,12 +6,14 @@ from fastapi import Security
6
7 from app.auth.utils import AuthHandler
8 from app.connectors.wazuh_indexer.schema.monitoring import ClusterHealthResponse
9 +from app.connectors.wazuh_indexer.schema.monitoring import CustomerIndicesSizeResponse
10 from app.connectors.wazuh_indexer.schema.monitoring import IndicesStatsResponse
11 from app.connectors.wazuh_indexer.schema.monitoring import NodeAllocationResponse
12 from app.connectors.wazuh_indexer.schema.monitoring import ShardsResponse
13
14 # from app.connectors.wazuh_indexer.schema import WazuhIndexerResponse, WazuhIndexerListResponse
15 from app.connectors.wazuh_indexer.services.monitoring import cluster_healthcheck
16 +from app.connectors.wazuh_indexer.services.monitoring import indices_size_per_customer
17 from app.connectors.wazuh_indexer.services.monitoring import indices_stats
18 from app.connectors.wazuh_indexer.services.monitoring import node_allocation
19 from app.connectors.wazuh_indexer.services.monitoring import (
@@ -101,6 +103,35 @@ async def get_indices_stats() -> Union[IndicesStatsResponse, HTTPException]:
103 raise HTTPException(status_code=500, detail="Failed to retrieve indices stats.")
104
105
106 +@wazuh_indexer_router.get(
107 + "/indices/size-per-customer",
108 + response_model=CustomerIndicesSizeResponse,
109 + description="Fetch Wazuh Indexer indices size aggregated per customer",
110 + dependencies=[Security(AuthHandler().require_any_scope("admin", "analyst"))],
111 +)
112 +async def get_indices_size_per_customer() -> Union[CustomerIndicesSizeResponse, HTTPException]:
113 + """
114 + Fetch Wazuh Indexer indices size per customer.
115 +
116 + This endpoint retrieves the total indices size aggregated per customer,
117 + where customer is extracted from index names (e.g., wazuh-copilot_37 -> copilot).
118 +
119 + Returns:
120 + CustomerIndicesSizeResponse: A Pydantic model representing the indices size per customer.
121 +
122 + Raises:
123 + HTTPException: An exception with a 500 status code is raised if the data cannot be retrieved.
124 + """
125 + try:
126 + response = await indices_size_per_customer()
127 + return response
128 + except Exception as e:
129 + raise HTTPException(
130 + status_code=500,
131 + detail=f"Failed to retrieve indices size per customer: {str(e)}",
132 + )
133 +
134 +
135 @wazuh_indexer_router.get(
136 "/shards",
137 response_model=ShardsResponse,
backend/app/connectors/wazuh_indexer/schema/monitoring.py
+14
@@ -78,3 +78,17 @@ class ShardsResponse(BaseModel):
78 shards: Optional[List[Shards]]
79 message: str
80 success: bool
81 +
82 +
83 +class CustomerIndicesSize(BaseModel):
84 + customer: str
85 + total_size_bytes: int
86 + total_size_human: str
87 + index_count: int
88 + indices: List[str]
89 +
90 +
91 +class CustomerIndicesSizeResponse(BaseModel):
92 + customer_sizes: Optional[List[CustomerIndicesSize]]
93 + message: str
94 + success: bool
backend/app/connectors/wazuh_indexer/services/monitoring.py
+127
@@ -1,3 +1,4 @@
1 +import re
2 from typing import Dict
3 from typing import Union
4
@@ -5,6 +6,8 @@ from loguru import logger
6
7 from app.connectors.wazuh_indexer.schema.monitoring import ClusterHealth
8 from app.connectors.wazuh_indexer.schema.monitoring import ClusterHealthResponse
9 +from app.connectors.wazuh_indexer.schema.monitoring import CustomerIndicesSize
10 +from app.connectors.wazuh_indexer.schema.monitoring import CustomerIndicesSizeResponse
11 from app.connectors.wazuh_indexer.schema.monitoring import IndicesStats
12 from app.connectors.wazuh_indexer.schema.monitoring import IndicesStatsResponse
13 from app.connectors.wazuh_indexer.schema.monitoring import NodeAllocation
@@ -153,3 +156,127 @@ async def output_shard_number_to_be_set_based_on_nodes() -> int:
156 logger.error(f"Shards check failed with error: {e}")
157 e = f"Shards check failed with error: {e}"
158 raise Exception(str(e))
159 +
160 +
161 +def parse_size_to_bytes(size_str: str) -> int:
162 + """
163 + Convert a human-readable size string to bytes.
164 + Handles formats like '1.2gb', '500mb', '100kb', '1024b'.
165 + """
166 + if not size_str or size_str == "Store size not found":
167 + return 0
168 +
169 + size_str = size_str.lower().strip()
170 +
171 + # Define multipliers
172 + multipliers = {
173 + "b": 1,
174 + "kb": 1024,
175 + "mb": 1024**2,
176 + "gb": 1024**3,
177 + "tb": 1024**4,
178 + }
179 +
180 + # Match number and unit
181 + match = re.match(r"^([\d.]+)\s*([a-z]+)$", size_str)
182 + if match:
183 + value = float(match.group(1))
184 + unit = match.group(2)
185 + return int(value * multipliers.get(unit, 1))
186 +
187 + # Try to parse as pure number (bytes)
188 + try:
189 + return int(float(size_str))
190 + except ValueError:
191 + return 0
192 +
193 +
194 +def bytes_to_human_readable(size_bytes: int) -> str:
195 + """Convert bytes to human-readable format."""
196 + for unit in ["b", "kb", "mb", "gb", "tb"]:
197 + if abs(size_bytes) < 1024.0:
198 + return f"{size_bytes:.2f}{unit}"
199 + size_bytes /= 1024.0
200 + return f"{size_bytes:.2f}pb"
201 +
202 +
203 +def extract_customer_from_index(index_name: str) -> str:
204 + """
205 + Extract customer name from index name.
206 + Pattern: after dash or underscore, before the next underscore or end.
207 + Examples:
208 + - wazuh-copilot_37 -> copilot
209 + - dev-taylor_37 -> taylor
210 + - wazuh-509dine2v_0 -> 509dine2v
211 + """
212 + # Match pattern: prefix-customer_suffix or prefix_customer_suffix
213 + match = re.match(r"^[^-_]+-([^_]+)_", index_name)
214 + if match:
215 + return match.group(1)
216 +
217 + # Fallback: try underscore as first separator
218 + match = re.match(r"^[^_]+_([^_]+)_", index_name)
219 + if match:
220 + return match.group(1)
221 +
222 + return "unknown"
223 +
224 +
225 +async def indices_size_per_customer() -> Union[CustomerIndicesSizeResponse, Dict[str, str]]:
226 + """
227 + Returns the total indices size aggregated per customer.
228 +
229 + Returns:
230 + CustomerIndicesSizeResponse: A Pydantic model containing the indices size per customer.
231 +
232 + Raises:
233 + Exception: An exception is raised if the indices stats cannot be retrieved.
234 + """
235 + logger.info("Collecting Wazuh Indexer indices size per customer")
236 + es_client = await create_wazuh_indexer_client("Wazuh-Indexer")
237 + try:
238 + raw_indices_stats_data = es_client.cat.indices(format="json")
239 +
240 + formatted_indices_stats_data = await format_indices_stats(raw_indices_stats_data)
241 +
242 + # Aggregate by customer
243 + customer_data: Dict[str, Dict] = {}
244 +
245 + for index_data in formatted_indices_stats_data:
246 + index_name = index_data.get("index", "")
247 + store_size = index_data.get("store_size", "0b")
248 +
249 + customer = extract_customer_from_index(index_name)
250 + size_bytes = parse_size_to_bytes(store_size)
251 +
252 + if customer not in customer_data:
253 + customer_data[customer] = {
254 + "total_size_bytes": 0,
255 + "index_count": 0,
256 + "indices": [],
257 + }
258 +
259 + customer_data[customer]["total_size_bytes"] += size_bytes
260 + customer_data[customer]["index_count"] += 1
261 + customer_data[customer]["indices"].append(index_name)
262 +
263 + # Convert to response models
264 + customer_sizes = [
265 + CustomerIndicesSize(
266 + customer=customer,
267 + total_size_bytes=data["total_size_bytes"],
268 + total_size_human=bytes_to_human_readable(data["total_size_bytes"]),
269 + index_count=data["index_count"],
270 + indices=data["indices"],
271 + )
272 + for customer, data in sorted(customer_data.items())
273 + ]
274 +
275 + return CustomerIndicesSizeResponse(
276 + customer_sizes=customer_sizes,
277 + success=True,
278 + message="Successfully collected Wazuh Indexer indices size per customer",
279 + )
280 + except Exception as e:
281 + logger.error(f"Indices size per customer check failed with error: {e}")
282 + raise Exception(f"Indices size per customer check failed with error: {e}")
backend/app/version/services/version.py
+1 -1
@@ -7,7 +7,7 @@ from loguru import logger
7 from packaging.version import Version
8
9 # Current version - update this with each release
10 -CURRENT_VERSION = "0.1.42"
10 +CURRENT_VERSION = "0.1.43"
11 VERSION_CHECK_URL = "https://api.github.com/repos/socfortress/CoPilot/releases/latest"
12
13
frontend/src/api/endpoints/wazuh/indices.ts
+14 -1
@@ -14,5 +14,18 @@ export default {
14 },
15 getClusterHealth() {
16 return HttpClient.get<FlaskBaseResponse & { cluster_health: ClusterHealth }>("/wazuh_indexer/health")
17 - }
17 + },
18 + getIndicesSizePerCustomer() {
19 + return HttpClient.get<{
20 + customer_sizes: {
21 + customer: string
22 + total_size_bytes: number
23 + total_size_human: string
24 + index_count: number
25 + indices: string[]
26 + }[]
27 + message: string
28 + success: boolean
29 + }>(`/wazuh_indexer/indices/size-per-customer`)
30 +}
31 }
frontend/src/components/indices/CustomerIndicesSize.vue new
+195
@@ -0,0 +1,195 @@
1 +<template>
2 + <n-card title="Storage per Customer" :bordered="bordered">
3 + <n-spin :show="loading">
4 + <div v-if="customerSizes && customerSizes.length > 0" class="customer-list">
5 + <div v-for="customer in customerSizes" :key="customer.customer" class="customer-item">
6 + <div class="customer-header">
7 + <div class="customer-name">
8 + <n-text strong>{{ customer.customer }}</n-text>
9 + <n-popover trigger="click" placement="bottom" :width="300">
10 + <template #trigger>
11 + <n-tag
12 + size="small"
13 + :bordered="false"
14 + type="info"
15 + class="clickable-tag"
16 + >
17 + {{ customer.index_count }} {{ customer.index_count === 1 ? "index" : "indices" }}
18 + </n-tag>
19 + </template>
20 + <div class="indices-popover">
21 + <div class="popover-header">
22 + <n-text strong>Indices for {{ customer.customer }}</n-text>
23 + </div>
24 + <n-scrollbar style="max-height: 200px">
25 + <div class="indices-list">
26 + <div
27 + v-for="index in customer.indices"
28 + :key="index"
29 + class="index-item"
30 + @click="selectIndex(index)"
31 + >
32 + <n-text code class="index-link">{{ index }}</n-text>
33 + </div>
34 + </div>
35 + </n-scrollbar>
36 + </div>
37 + </n-popover>
38 + </div>
39 + <n-text class="customer-size">{{ customer.total_size_human }}</n-text>
40 + </div>
41 + <n-progress
42 + type="line"
43 + :percentage="getPercentage(customer.total_size_bytes)"
44 + :show-indicator="false"
45 + :height="8"
46 + :border-radius="4"
47 + :color="getProgressColor(customer.total_size_bytes)"
48 + />
49 + </div>
50 + </div>
51 + <n-empty v-else-if="!loading" description="No customer data available" />
52 + </n-spin>
53 + </n-card>
54 +</template>
55 +
56 +<script lang="ts" setup>
57 +import { NCard, NEmpty, NPopover, NProgress, NScrollbar, NSpin, NTag, NText, useMessage, useThemeVars } from "naive-ui"
58 +import { computed, onBeforeMount, ref } from "vue"
59 +import Api from "@/api"
60 +
61 +interface CustomerIndicesSize {
62 + customer: string
63 + total_size_bytes: number
64 + total_size_human: string
65 + index_count: number
66 + indices: string[]
67 +}
68 +
69 +defineProps<{
70 + bordered?: boolean
71 +}>()
72 +
73 +const emit = defineEmits<{
74 + (e: "click", value: string): void
75 +}>()
76 +
77 +const message = useMessage()
78 +const themeVars = useThemeVars()
79 +
80 +const loading = ref(false)
81 +const customerSizes = ref<CustomerIndicesSize[]>([])
82 +
83 +const maxSize = computed(() => {
84 + if (!customerSizes.value.length) return 0
85 + return Math.max(...customerSizes.value.map(c => c.total_size_bytes))
86 +})
87 +
88 +function getPercentage(sizeBytes: number): number {
89 + if (!maxSize.value) return 0
90 + return Math.round((sizeBytes / maxSize.value) * 100)
91 +}
92 +
93 +function getProgressColor(sizeBytes: number): string {
94 + const percentage = getPercentage(sizeBytes)
95 + if (percentage >= 80) return themeVars.value.errorColor
96 + if (percentage >= 60) return themeVars.value.warningColor
97 + return themeVars.value.primaryColor
98 +}
99 +
100 +function selectIndex(indexName: string) {
101 + emit("click", indexName)
102 +}
103 +
104 +function getCustomerIndicesSize() {
105 + loading.value = true
106 +
107 + Api.wazuh.indices
108 + .getIndicesSizePerCustomer()
109 + .then(res => {
110 + if (res.data.success) {
111 + customerSizes.value = res.data.customer_sizes || []
112 + } else {
113 + message.error(res.data?.message || "An error occurred. Please try again later.")
114 + }
115 + })
116 + .catch(err => {
117 + message.error(err.response?.data?.message || "Failed to retrieve customer indices size.")
118 + })
119 + .finally(() => {
120 + loading.value = false
121 + })
122 +}
123 +
124 +onBeforeMount(() => {
125 + getCustomerIndicesSize()
126 +})
127 +</script>
128 +
129 +<style lang="scss" scoped>
130 +.customer-list {
131 + display: flex;
132 + flex-direction: column;
133 + gap: 16px;
134 + max-height: 400px;
135 + overflow-y: auto;
136 +
137 + .customer-item {
138 + .customer-header {
139 + display: flex;
140 + justify-content: space-between;
141 + align-items: center;
142 + margin-bottom: 6px;
143 +
144 + .customer-name {
145 + display: flex;
146 + align-items: center;
147 + gap: 8px;
148 +
149 + .clickable-tag {
150 + cursor: pointer;
151 + transition: opacity 0.2s;
152 +
153 + &:hover {
154 + opacity: 0.8;
155 + }
156 + }
157 + }
158 +
159 + .customer-size {
160 + font-weight: 600;
161 + font-family: var(--font-family-mono);
162 + }
163 + }
164 + }
165 +}
166 +
167 +.indices-popover {
168 + .popover-header {
169 + margin-bottom: 8px;
170 + padding-bottom: 8px;
171 + border-bottom: 1px solid var(--border-color);
172 + }
173 +
174 + .indices-list {
175 + display: flex;
176 + flex-direction: column;
177 + gap: 4px;
178 +
179 + .index-item {
180 + padding: 4px 0;
181 + cursor: pointer;
182 + border-radius: 4px;
183 + transition: background-color 0.2s;
184 +
185 + &:hover {
186 + background-color: var(--hover-color);
187 + }
188 +
189 + .index-link {
190 + cursor: pointer;
191 + }
192 + }
193 + }
194 +}
195 +</style>
frontend/src/views/Indices.vue
+72 -67
@@ -19,6 +19,10 @@
19 </div>
20 </div>
21
22 + <div class="section">
23 + <CustomerIndicesSize @click="setIndex" />
24 + </div>
25 +
26 <n-card class="section overflow-hidden" content-style="padding:0">
27 <div class="columns column-1200 flex gap-0!">
28 <div class="col basis-2/5">
@@ -39,6 +43,7 @@ import { defineAsyncComponent, onBeforeMount, ref } from "vue"
43 import { useRoute } from "vue-router"
44 import Api from "@/api"
45 import ClusterHealth from "@/components/indices/ClusterHealth.vue"
46 +import CustomerIndicesSize from "@/components/indices/CustomerIndicesSize.vue"
47 import Details from "@/components/indices/Details.vue"
48 import IndicesMarquee from "@/components/indices/Marquee.vue"
49 import NodeAllocation from "@/components/indices/NodeAllocation.vue"
@@ -54,84 +59,84 @@ const currentIndex = ref<IndexStats | null>(null)
59 const requestedIndex = ref<string | null>(null)
60
61 function setIndex(index: IndexStats | string) {
57 - if (typeof index === "string") {
58 - const indexStats = indices.value?.find(o => o.index === index) || null
59 - indexStats && (currentIndex.value = indexStats)
60 - } else {
61 - currentIndex.value = index
62 - }
62 + if (typeof index === "string") {
63 + const indexStats = indices.value?.find(o => o.index === index) || null
64 + indexStats && (currentIndex.value = indexStats)
65 + } else {
66 + currentIndex.value = index
67 + }
68 }
69
70 function getIndices(cb?: () => void) {
66 - loadingIndex.value = true
67 -
68 - Api.wazuh.indices
69 - .getIndices()
70 - .then(res => {
71 - if (res.data.success) {
72 - indices.value = res.data.indices_stats
73 -
74 - if (cb) cb()
75 - } else {
76 - message.error(res.data?.message || "An error occurred. Please try again later.")
77 - }
78 - })
79 - .catch(err => {
80 - if (err.response?.status === 401) {
81 - message.error(
82 - err.response?.data?.message ||
83 - "Wazuh-Indexer returned Unauthorized. Please check your connector credentials."
84 - )
85 - } else if (err.response?.status === 404) {
86 - message.error(err.response?.data?.message || "No indices were found.")
87 - } else {
88 - message.error(err.response?.data?.message || "An error occurred. Please try again later.")
89 - }
90 - })
91 - .finally(() => {
92 - loadingIndex.value = false
93 - })
71 + loadingIndex.value = true
72 +
73 + Api.wazuh.indices
74 + .getIndices()
75 + .then(res => {
76 + if (res.data.success) {
77 + indices.value = res.data.indices_stats
78 +
79 + if (cb) cb()
80 + } else {
81 + message.error(res.data?.message || "An error occurred. Please try again later.")
82 + }
83 + })
84 + .catch(err => {
85 + if (err.response?.status === 401) {
86 + message.error(
87 + err.response?.data?.message ||
88 + "Wazuh-Indexer returned Unauthorized. Please check your connector credentials."
89 + )
90 + } else if (err.response?.status === 404) {
91 + message.error(err.response?.data?.message || "No indices were found.")
92 + } else {
93 + message.error(err.response?.data?.message || "An error occurred. Please try again later.")
94 + }
95 + })
96 + .finally(() => {
97 + loadingIndex.value = false
98 + })
99 }
100
101 onBeforeMount(() => {
97 - if (route.query?.index_name) {
98 - requestedIndex.value = route.query.index_name.toString()
99 - }
102 + if (route.query?.index_name) {
103 + requestedIndex.value = route.query.index_name.toString()
104 + }
105
101 - getIndices(() => {
102 - requestedIndex.value && setIndex(requestedIndex.value)
103 - })
106 + getIndices(() => {
107 + requestedIndex.value && setIndex(requestedIndex.value)
108 + })
109 })
110 </script>
111
112 <style lang="scss" scoped>
113 .page {
109 - .section {
110 - margin-bottom: calc(var(--spacing) * 6);
111 -
112 - .columns {
113 - display: flex;
114 - gap: calc(var(--spacing) * 6);
115 -
116 - .stretchy {
117 - height: 100%;
118 - }
119 - }
120 - }
121 -
122 - @media (max-width: 1000px) {
123 - .section {
124 - .columns {
125 - flex-direction: column;
126 - }
127 - }
128 - }
129 - @media (max-width: 1200px) {
130 - .section {
131 - .columns.column-1200 {
132 - flex-direction: column;
133 - }
134 - }
135 - }
114 + .section {
115 + margin-bottom: calc(var(--spacing) * 6);
116 +
117 + .columns {
118 + display: flex;
119 + gap: calc(var(--spacing) * 6);
120 +
121 + .stretchy {
122 + height: 100%;
123 + }
124 + }
125 + }
126 +
127 + @media (max-width: 1000px) {
128 + .section {
129 + .columns {
130 + flex-direction: column;
131 + }
132 + }
133 + }
134 + @media (max-width: 1200px) {
135 + .section {
136 + .columns.column-1200 {
137 + flex-direction: column;
138 + }
139 + }
140 + }
141 }
142 </style>