@cryptotaxi247 / CoPilot / commits / 0aca182c

feat(event-sources): SOC portal column config UI + render (issue #833 slice 2/3) (#869)

* feat(event-sources): SOC portal column config UI + render (issue #833 slice 2/3) Adds the configure-columns flow to the SOC portal's event search view. Builds on slice 1 (#868) which landed the displayed_columns storage on the EventSources table. What's in this slice: - New ColumnConfigModal.vue — modal that lets a SOC user pick fields from the source's available mappings, edit the column header label, reorder via up/down buttons, and remove. "Reset to defaults" clears the saved layout (sends null) so the table falls back to the hardcoded defaults. Save calls PUT /siem/event_sources/{id} with {displayed_columns: [...] | null}. - EventSearch.vue — new "Columns" button in the filters bar (gated on a selected event source). The hardcoded baseColumns array is preserved as defaultColumns and used as the fallback. The active columns computed builds from selectedEventSource.displayed_columns when present, otherwise falls back to defaults. Cell rendering walks the dotted field path (agent.name, data.win.eventdata.targetUserName) and formats arrays/objects sensibly. - Types: new DisplayColumn interface; EventSource.displayed_columns is now DisplayColumn[] | null. - API: EventSourceCreatePayload and EventSourceUpdatePayload accept the new field. No customer-portal changes — slice 3 will mirror the render logic (read-only) on the customer-facing app. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * fix(event-sources): dump DisplayColumn instances to dicts before JSON storage PUT /siem/event_sources/{id} with non-null displayed_columns 500'd: sqlalchemy.exc.StatementError: (builtins.TypeError) Object of type DisplayColumn is not JSON serializable EventSourceUpdate.displayed_columns deserializes incoming JSON into DisplayColumn Pydantic instances. update_from_model was assigning the list of instances directly to the SQLAlchemy JSON column, and the column's json_serializer (stdlib json.dumps) doesn't know how to handle Pydantic models. model_dump() each entry first. Latent slice-1 bug exposed by slice 2 actually exercising the update path. The create path already worked because create_event_source calls EventSources(**model_dump()), which recursively flattens nested Pydantic. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * fix(event-sources): use exclude_unset for true partial updates PUT /siem/event_sources/{id} with only displayed_columns set was nulling every other column: pymysql.err.IntegrityError: (1048, "Column 'name' cannot be null") EventSourceUpdate's other fields default to None on the Pydantic side. update_from_model was unconditionally setting every attribute via hasattr, so partial updates clobbered the rest of the row to None and hit the NOT NULL constraint on `name` first. Switch to Pydantic 2's `model_dump(exclude_unset=True)` so only fields the client actually sent get applied. Side benefit: `model_dump()` recursively flattens DisplayColumn instances into dicts, so the JSON-column serialization path (which previously needed a manual loop with `c.model_dump()`) collapses to a single attribute set. Net –18 lines. Latent slice-1 partial-update bug exposed by slice 2's narrow PUT payload — the existing Add/Edit Event Source dialog sends the full form, so it never tripped the issue. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> * fix(event-sources): streamline model_dump call for JSON serialization --------- Co-authored-by: taylor_socfortress <taylor.walton@socfortress.co> Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

taylorcopilot committed May 9, 2026 at 12:38 UTC 0aca182c2deac753179ec63eb5934697cb926ae9
5 files changed +384 -71
backend/app/db/universal_models.py
+14 -12
@@ -532,18 +532,20 @@ class EventSources(SQLModel, table=True):
532 customer: Optional["Customers"] = Relationship()
533
534 def update_from_model(self, source_data):
535 - if hasattr(source_data, "name"):
536 - self.name = source_data.name
537 - if hasattr(source_data, "index_pattern"):
538 - self.index_pattern = source_data.index_pattern
539 - if hasattr(source_data, "event_type"):
540 - self.event_type = source_data.event_type
541 - if hasattr(source_data, "time_field"):
542 - self.time_field = source_data.time_field
543 - if hasattr(source_data, "enabled"):
544 - self.enabled = source_data.enabled
545 - if hasattr(source_data, "displayed_columns"):
546 - self.displayed_columns = source_data.displayed_columns
535 + """Apply only the fields the caller explicitly set on source_data.
536 +
537 + Pydantic 2's `model_dump(exclude_unset=True)` filters to fields the
538 + client actually sent, so partial updates (e.g. PUT with just
539 + `displayed_columns`) don't clobber unrelated columns to NULL. The
540 + same dump recursively flattens nested Pydantic models (DisplayColumn)
541 + into plain dicts, which is what the JSON column requires —
542 + SQLAlchemy's json_serializer otherwise raises "Object of type
543 + DisplayColumn is not JSON serializable" on commit.
544 + """
545 + data = source_data.model_dump(exclude_unset=True) if hasattr(source_data, "model_dump") else {}
546 + for field in ("name", "index_pattern", "event_type", "time_field", "enabled", "displayed_columns"):
547 + if field in data:
548 + setattr(self, field, data[field])
549 self.updated_at = datetime.utcnow()
550
551
frontend/src/api/endpoints/siem.ts
+3 -1
@@ -6,7 +6,7 @@ import type {
6 PanelDataResponse
7 } from "@/types/dashboards.d"
8 import type { EventSearchResult, FieldMapping } from "@/types/events.d"
9 -import type { EventSource } from "@/types/eventSources.d"
9 +import type { DisplayColumn, EventSource } from "@/types/eventSources.d"
10 import type { FlaskBaseResponse } from "@/types/flask.d"
11 import { HttpClient } from "../httpClient"
12
@@ -17,6 +17,7 @@ export interface EventSourceCreatePayload {
17 event_type: string
18 time_field: string
19 enabled: boolean
20 + displayed_columns?: DisplayColumn[] | null
21 }
22
23 export interface EventSourceUpdatePayload {
@@ -25,6 +26,7 @@ export interface EventSourceUpdatePayload {
26 event_type?: string
27 time_field?: string
28 enabled?: boolean
29 + displayed_columns?: DisplayColumn[] | null
30 }
31
32 export default {
frontend/src/components/events/ColumnConfigModal.vue new
+229
@@ -0,0 +1,229 @@
1 +<template>
2 + <n-modal
3 + v-model:show="showModel"
4 + preset="card"
5 + :title="`Configure columns: ${eventSource?.name ?? ''}`"
6 + style="width: 720px; max-width: 92vw"
7 + :bordered="false"
8 + :mask-closable="false"
9 + >
10 + <div class="flex flex-col gap-4">
11 + <!-- Active columns -->
12 + <div class="flex flex-col gap-2">
13 + <div class="flex items-center justify-between">
14 + <span class="text-sm font-medium">Active columns ({{ localColumns.length }})</span>
15 + <span class="text-xs opacity-60">Order matches table left-to-right</span>
16 + </div>
17 +
18 + <div
19 + v-if="!localColumns.length"
20 + class="rounded border border-dashed p-4 text-center text-sm opacity-60"
21 + >
22 + No columns configured. Defaults from the table will be shown.
23 + </div>
24 +
25 + <div v-else class="flex flex-col gap-1">
26 + <div
27 + v-for="(col, idx) in localColumns"
28 + :key="col.key"
29 + class="bg-default flex items-center gap-2 rounded border px-2 py-1.5"
30 + >
31 + <span class="text-xs opacity-50" style="width: 24px">{{ idx + 1 }}</span>
32 + <n-input
33 + v-model:value="col.label"
34 + size="small"
35 + placeholder="Column header"
36 + style="flex: 1; max-width: 200px"
37 + />
38 + <span
39 + class="font-mono text-xs opacity-60"
40 + style="
41 + flex: 1;
42 + min-width: 0;
43 + overflow: hidden;
44 + text-overflow: ellipsis;
45 + white-space: nowrap;
46 + "
47 + >
48 + {{ col.key }}
49 + </span>
50 + <n-button size="tiny" :disabled="idx === 0" @click="moveColumn(idx, -1)">
51 + <template #icon>
52 + <Icon :name="ArrowUpIcon" :size="14" />
53 + </template>
54 + </n-button>
55 + <n-button size="tiny" :disabled="idx === localColumns.length - 1" @click="moveColumn(idx, 1)">
56 + <template #icon>
57 + <Icon :name="ArrowDownIcon" :size="14" />
58 + </template>
59 + </n-button>
60 + <n-button size="tiny" type="error" quaternary @click="removeColumn(idx)">
61 + <template #icon>
62 + <Icon :name="CloseIcon" :size="14" />
63 + </template>
64 + </n-button>
65 + </div>
66 + </div>
67 + </div>
68 +
69 + <!-- Add column from available fields -->
70 + <div class="flex flex-col gap-2">
71 + <span class="text-sm font-medium">Add a column</span>
72 + <n-input v-model:value="fieldFilter" placeholder="Filter available fields..." clearable size="small">
73 + <template #prefix>
74 + <Icon :name="SearchIcon" :size="14" class="opacity-50" />
75 + </template>
76 + </n-input>
77 + <div class="max-h-60 overflow-y-auto rounded border">
78 + <div
79 + v-for="field in filteredFields"
80 + :key="field.field"
81 + class="hover:bg-hover-005 flex cursor-pointer items-center justify-between gap-2 px-2 py-1.5 text-sm"
82 + @click="addColumn(field.field)"
83 + >
84 + <span class="font-mono">{{ field.field }}</span>
85 + <span class="flex items-center gap-2">
86 + <span class="text-xs opacity-50">{{ field.type }}</span>
87 + <n-button size="tiny" quaternary>
88 + <template #icon>
89 + <Icon :name="AddIcon" :size="14" />
90 + </template>
91 + </n-button>
92 + </span>
93 + </div>
94 + <div v-if="!filteredFields.length" class="p-3 text-center text-sm opacity-60">
95 + {{ fieldMappings.length ? "No fields match your filter." : "Loading available fields..." }}
96 + </div>
97 + </div>
98 + </div>
99 + </div>
100 +
101 + <template #footer>
102 + <div class="flex items-center justify-between gap-2">
103 + <n-button quaternary type="warning" @click="resetToDefaults">Reset to defaults</n-button>
104 + <div class="flex gap-2">
105 + <n-button @click="showModel = false">Cancel</n-button>
106 + <n-button type="primary" :loading="saving" @click="onSave">Save</n-button>
107 + </div>
108 + </div>
109 + </template>
110 + </n-modal>
111 +</template>
112 +
113 +<script setup lang="ts">
114 +import type { FieldMapping } from "@/types/events.d"
115 +import type { DisplayColumn, EventSource } from "@/types/eventSources.d"
116 +import { NButton, NInput, NModal, useMessage } from "naive-ui"
117 +import { computed, ref, watch } from "vue"
118 +import Api from "@/api"
119 +import Icon from "@/components/common/Icon.vue"
120 +
121 +const props = defineProps<{
122 + show: boolean
123 + eventSource: EventSource | null
124 + fieldMappings: FieldMapping[]
125 +}>()
126 +
127 +const emit = defineEmits<{
128 + "update:show": [value: boolean]
129 + saved: [columns: DisplayColumn[] | null]
130 +}>()
131 +
132 +const SearchIcon = "carbon:search"
133 +const AddIcon = "carbon:add"
134 +const CloseIcon = "carbon:close"
135 +const ArrowUpIcon = "carbon:arrow-up"
136 +const ArrowDownIcon = "carbon:arrow-down"
137 +
138 +const message = useMessage()
139 +
140 +const showModel = computed({
141 + get: () => props.show,
142 + set: v => emit("update:show", v)
143 +})
144 +
145 +const localColumns = ref<DisplayColumn[]>([])
146 +const fieldFilter = ref("")
147 +const saving = ref(false)
148 +
149 +// Re-seed when the modal opens or the source changes
150 +watch(
151 + () => [props.show, props.eventSource?.id] as const,
152 + ([show]) => {
153 + if (show && props.eventSource) {
154 + localColumns.value = (props.eventSource.displayed_columns ?? []).map(c => ({
155 + key: c.key,
156 + label: c.label,
157 + width: c.width ?? null
158 + }))
159 + fieldFilter.value = ""
160 + }
161 + },
162 + { immediate: true }
163 +)
164 +
165 +const usedKeys = computed(() => new Set(localColumns.value.map(c => c.key)))
166 +
167 +const filteredFields = computed(() => {
168 + const q = fieldFilter.value.trim().toLowerCase()
169 + const all = props.fieldMappings.filter(f => !usedKeys.value.has(f.field))
170 + if (!q) return all.slice(0, 50)
171 + return all.filter(f => f.field.toLowerCase().includes(q)).slice(0, 50)
172 +})
173 +
174 +function defaultLabelFor(fieldPath: string): string {
175 + // Take the last dotted segment, replace _ with spaces, title-case
176 + const tail = fieldPath.split(".").pop() ?? fieldPath
177 + return tail.replace(/_/g, " ").replace(/\b\w/g, c => c.toUpperCase())
178 +}
179 +
180 +function addColumn(fieldPath: string) {
181 + if (usedKeys.value.has(fieldPath)) return
182 + localColumns.value.push({
183 + key: fieldPath,
184 + label: defaultLabelFor(fieldPath),
185 + width: null
186 + })
187 +}
188 +
189 +function removeColumn(idx: number) {
190 + localColumns.value.splice(idx, 1)
191 +}
192 +
193 +function moveColumn(idx: number, delta: number) {
194 + const target = idx + delta
195 + if (target < 0 || target >= localColumns.value.length) return
196 + const [item] = localColumns.value.splice(idx, 1)
197 + localColumns.value.splice(target, 0, item)
198 +}
199 +
200 +function resetToDefaults() {
201 + localColumns.value = []
202 +}
203 +
204 +function onSave() {
205 + if (!props.eventSource) return
206 + saving.value = true
207 +
208 + // Empty list means "use defaults" — send null so the backend stores NULL.
209 + const payload = localColumns.value.length ? localColumns.value : null
210 +
211 + Api.siem
212 + .updateEventSource(props.eventSource.id, { displayed_columns: payload })
213 + .then(res => {
214 + if (res.data.success) {
215 + message.success("Columns saved")
216 + emit("saved", payload)
217 + showModel.value = false
218 + } else {
219 + message.warning(res.data?.message || "Failed to save columns")
220 + }
221 + })
222 + .catch(err => {
223 + message.error(err.response?.data?.message || "Failed to save columns")
224 + })
225 + .finally(() => {
226 + saving.value = false
227 + })
228 +}
229 +</script>
frontend/src/components/events/EventSearch.vue
+128 -58
@@ -64,6 +64,17 @@
64 </template>
65 Search
66 </n-button>
67 + <n-button
68 + quaternary
69 + :disabled="!selectedEventSource"
70 + title="Configure which columns to display for this event source"
71 + @click="showColumnConfig = true"
72 + >
73 + <template #icon>
74 + <Icon :name="SettingsIcon" :size="16" />
75 + </template>
76 + Columns
77 + </n-button>
78 </div>
79
80 <!-- Query Bar with Autocomplete -->
@@ -138,6 +149,14 @@
149 @filter-add="addFilterFromDetail"
150 @filter-exclude="excludeFilterFromDetail"
151 />
152 +
153 + <!-- Column Config Modal -->
154 + <ColumnConfigModal
155 + v-model:show="showColumnConfig"
156 + :event-source="selectedEventSource"
157 + :field-mappings
158 + @saved="onColumnsSaved"
159 + />
160 </div>
161 </template>
162
@@ -145,18 +164,20 @@
164 import type { DataTableColumns } from "naive-ui"
165 import type { Customer } from "@/types/customers.d"
166 import type { EventSearchResult, FieldMapping } from "@/types/events.d"
148 -import type { EventSource } from "@/types/eventSources.d"
167 +import type { DisplayColumn, EventSource } from "@/types/eventSources.d"
168 import { NAlert, NButton, NCard, NDataTable, NDatePicker, NEmpty, NInput, NSelect, NSpin, useMessage } from "naive-ui"
169 import { computed, h, nextTick, onBeforeMount, ref } from "vue"
170 import { useRoute } from "vue-router"
171 import Api from "@/api"
172 import Icon from "@/components/common/Icon.vue"
173 +import ColumnConfigModal from "./ColumnConfigModal.vue"
174 import EventDetailDrawer from "./EventDetailDrawer.vue"
175
176 const route = useRoute()
177
178 const SearchIcon = "carbon:search"
179 const CodeIcon = "carbon:code"
180 +const SettingsIcon = "carbon:settings"
181 const FIELD_TOKEN_REGEX = /(?:^|[\s(])([a-z_][\w.]*)$/i
182
183 const message = useMessage()
@@ -202,6 +223,10 @@ const showNoSourcesWarning = computed(
223 () => selectedCustomerCode.value && !loadingEventSources.value && eventSourcesList.value.length === 0
224 )
225
226 +const selectedEventSource = computed<EventSource | null>(
227 + () => eventSourcesList.value.find(s => s.name === selectedSourceName.value) ?? null
228 +)
229 +
230 // -- Field mappings / autocomplete --
231 const fieldMappings = ref<FieldMapping[]>([])
232
@@ -397,68 +422,113 @@ function loadMoreEvents() {
422 }
423
424 // -- Table columns --
400 -const columns = computed<DataTableColumns<EventSearchResult>>(() => {
401 - const baseColumns: DataTableColumns<EventSearchResult> = [
402 - {
403 - title: "Timestamp",
404 - key: "timestamp",
405 - width: 180,
406 - sorter: (a, b) => {
407 - const timeA = a.timestamp || a["@timestamp"] || ""
408 - const timeB = b.timestamp || b["@timestamp"] || ""
409 - return new Date(timeA).getTime() - new Date(timeB).getTime()
410 - },
411 - render(row) {
412 - const ts = row.timestamp || row["@timestamp"]
413 - if (!ts) return "-"
414 - return new Date(ts).toLocaleString()
415 - }
416 - },
417 - {
418 - title: "Source",
419 - key: "agent_name",
420 - width: 140,
421 - ellipsis: { tooltip: true },
422 - render(row) {
423 - return row.agent_name || row.source || "-"
424 - }
425 - },
426 - {
427 - title: "Rule",
428 - key: "rule_description",
429 - ellipsis: { tooltip: true },
430 - render(row) {
431 - return row.rule_description || row.rule_id || "-"
432 - }
425 +// Default columns we fall back to when an event source has no displayed_columns
426 +// configured. These keep the prior behaviour for un-customised sources.
427 +const defaultColumns: DataTableColumns<EventSearchResult> = [
428 + {
429 + title: "Timestamp",
430 + key: "timestamp",
431 + width: 180,
432 + sorter: (a, b) => {
433 + const timeA = a.timestamp || a["@timestamp"] || ""
434 + const timeB = b.timestamp || b["@timestamp"] || ""
435 + return new Date(timeA).getTime() - new Date(timeB).getTime()
436 },
434 - {
435 - title: "Level",
436 - key: "rule_level",
437 - width: 80,
438 - sorter: (a, b) => (Number(a.rule_level) || 0) - (Number(b.rule_level) || 0),
439 - render(row) {
440 - if (row.rule_level === undefined || row.rule_level === null) return "-"
441 - const level = Number(row.rule_level)
442 - let type: "default" | "warning" | "error" | "success" | "info" = "default"
443 - if (level >= 12) type = "error"
444 - else if (level >= 8) type = "warning"
445 - else if (level >= 4) type = "info"
446 - return h("span", { class: `level-${type}` }, String(row.rule_level))
447 - }
448 - },
449 - {
450 - title: "Summary",
451 - key: "full_log",
452 - ellipsis: { tooltip: true },
453 - render(row) {
454 - return row.full_log || row.data || row.message || "-"
455 - }
437 + render(row) {
438 + const ts = row.timestamp || row["@timestamp"]
439 + if (!ts) return "-"
440 + return new Date(ts).toLocaleString()
441 + }
442 + },
443 + {
444 + title: "Source",
445 + key: "agent_name",
446 + width: 140,
447 + ellipsis: { tooltip: true },
448 + render(row) {
449 + return row.agent_name || row.source || "-"
450 + }
451 + },
452 + {
453 + title: "Rule",
454 + key: "rule_description",
455 + ellipsis: { tooltip: true },
456 + render(row) {
457 + return row.rule_description || row.rule_id || "-"
458 }
457 - ]
459 + },
460 + {
461 + title: "Level",
462 + key: "rule_level",
463 + width: 80,
464 + sorter: (a, b) => (Number(a.rule_level) || 0) - (Number(b.rule_level) || 0),
465 + render(row) {
466 + if (row.rule_level === undefined || row.rule_level === null) return "-"
467 + const level = Number(row.rule_level)
468 + let type: "default" | "warning" | "error" | "success" | "info" = "default"
469 + if (level >= 12) type = "error"
470 + else if (level >= 8) type = "warning"
471 + else if (level >= 4) type = "info"
472 + return h("span", { class: `level-${type}` }, String(row.rule_level))
473 + }
474 + },
475 + {
476 + title: "Summary",
477 + key: "full_log",
478 + ellipsis: { tooltip: true },
479 + render(row) {
480 + return row.full_log || row.data || row.message || "-"
481 + }
482 + }
483 +]
484 +
485 +/** Walk a dotted path (e.g. "agent.name") through a nested object. */
486 +function getNestedValue(obj: EventSearchResult, path: string): unknown {
487 + return path.split(".").reduce<unknown>((acc, segment) => {
488 + if (acc && typeof acc === "object") {
489 + return (acc as Record<string, unknown>)[segment]
490 + }
491 + return undefined
492 + }, obj)
493 +}
494 +
495 +function formatCellValue(val: unknown): string {
496 + if (val === undefined || val === null || val === "") return "-"
497 + if (Array.isArray(val)) return val.map(v => (v === null || v === undefined ? "" : String(v))).join(", ")
498 + if (typeof val === "object") return JSON.stringify(val)
499 + return String(val)
500 +}
501 +
502 +function buildColumnFromConfig(col: DisplayColumn): DataTableColumns<EventSearchResult>[number] {
503 + return {
504 + title: col.label || col.key,
505 + key: col.key,
506 + width: col.width || undefined,
507 + ellipsis: { tooltip: true },
508 + render(row: EventSearchResult) {
509 + return formatCellValue(getNestedValue(row, col.key))
510 + }
511 + }
512 +}
513
459 - return baseColumns
514 +const columns = computed<DataTableColumns<EventSearchResult>>(() => {
515 + const configured = selectedEventSource.value?.displayed_columns
516 + if (configured && configured.length > 0) {
517 + return configured.map(buildColumnFromConfig)
518 + }
519 + return defaultColumns
520 })
521
522 +// -- Configure columns modal --
523 +const showColumnConfig = ref(false)
524 +
525 +function onColumnsSaved(saved: DisplayColumn[] | null) {
526 + // Patch the local list so the table re-renders without a network round-trip.
527 + if (selectedEventSource.value) {
528 + selectedEventSource.value.displayed_columns = saved
529 + }
530 +}
531 +
532 // -- Event detail --
533 const showDetailDrawer = ref(false)
534 const selectedEvent = ref<EventSearchResult | null>(null)
frontend/src/types/eventSources.d.ts
+10
@@ -1,5 +1,14 @@
1 export type EventType = "EDR" | "EPP" | "Cloud Integration" | "Network Security"
2
3 +export interface DisplayColumn {
4 + /** Field path in the event _source object (dotted, e.g. "agent.name"). */
5 + key: string
6 + /** Human-readable column header. */
7 + label: string
8 + /** Optional pixel width hint. */
9 + width?: number | null
10 +}
11 +
12 export interface EventSource {
13 id: number
14 customer_code: string
@@ -8,6 +17,7 @@ export interface EventSource {
17 event_type: EventType
18 time_field: string
19 enabled: boolean
20 + displayed_columns?: DisplayColumn[] | null
21 created_at: string
22 updated_at: string
23 }