| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #include "ebpf-ipc.h" |
| 4 | |
| 5 | netdata_ebpf_pid_stats_t *integration_shm = NULL; |
| 6 | int shm_fd_ebpf_integration = -1; |
| 7 | sem_t *shm_mutex_ebpf_integration = SEM_FAILED; |
| 8 | |
| 9 | static Pvoid_t ebpf_ipc_JudyL = NULL; |
| 10 | ebpf_user_mem_stat_t ebpf_stat_values; |
| 11 | |
| 12 | // Judy stores index+1 directly in the pointer slot to avoid heap allocation. |
| 13 | // 0 means "not found" (Judy returns NULL for missing keys), |
| 14 | // so we offset by 1: stored = index+1, retrieved = stored-1. |
| 15 | #define IDX_TO_JVALUE(idx) ((Pvoid_t)((Word_t)(idx) + 1)) |
| 16 | #define JVALUE_TO_IDX(pv) ((uint32_t)((Word_t)(pv) - 1)) |
| 17 | #define JVALUE_IS_VALID(pv) ((pv) != NULL) |
| 18 | |
| 19 | static bool ebpf_shm_find_index_unsafe(uint32_t pid, uint32_t *result) |
| 20 | { |
| 21 | Pvoid_t *Pvalue = JudyLGet(ebpf_ipc_JudyL, (Word_t)pid, PJE0); |
| 22 | if (Pvalue && JVALUE_IS_VALID(*Pvalue)) { |
| 23 | *result = JVALUE_TO_IDX(*Pvalue); |
| 24 | return true; |
| 25 | } |
| 26 | return false; |
| 27 | } |
| 28 | |
| 29 | static bool ebpf_find_pid_shm_del_unsafe(uint32_t pid, enum ebpf_pids_index shm_idx) |
| 30 | { |
| 31 | uint32_t idx; |
| 32 | if (!ebpf_shm_find_index_unsafe(pid, &idx)) |
| 33 | return false; |
| 34 | |
| 35 | if (idx >= ebpf_stat_values.current) |
| 36 | return false; |
| 37 | |
| 38 | netdata_ebpf_pid_stats_t *ptr = &integration_shm[idx]; |
| 39 | if (!ptr->threads) |
| 40 | return false; |
| 41 | |
| 42 | ptr->threads &= ~(1UL << (shm_idx << 1)); |
| 43 | if (ptr->threads) |
| 44 | return true; |
| 45 | |
| 46 | (void)JudyLDel(&ebpf_ipc_JudyL, (Word_t)pid, PJE0); |
| 47 | ebpf_stat_values.current--; |
| 48 | |
| 49 | if (idx == ebpf_stat_values.current) |
| 50 | return false; |
| 51 | |
| 52 | // Compact: move last entry into the freed slot |
| 53 | uint32_t last_pid = integration_shm[ebpf_stat_values.current].pid; |
| 54 | Pvoid_t *Pvalue = JudyLGet(ebpf_ipc_JudyL, (Word_t)last_pid, PJE0); |
| 55 | if (Pvalue && JVALUE_IS_VALID(*Pvalue)) { |
| 56 | *Pvalue = IDX_TO_JVALUE(idx); |
| 57 | memcpy(ptr, &integration_shm[ebpf_stat_values.current], sizeof(*ptr)); |
| 58 | } |
| 59 | |
| 60 | return false; |
| 61 | } |
| 62 | |
| 63 | // Returns the slot index for pid, allocating a new slot if needed. A fresh |
| 64 | // slot is memset to zero so callers never inherit stale bits or counters from |
| 65 | // a prior PID that used the same index (prevents the compaction stale-tail |
| 66 | // and PID-reuse contamination paths). |
| 67 | static uint32_t ebpf_find_or_create_index_pid(uint32_t pid) |
| 68 | { |
| 69 | uint32_t idx; |
| 70 | if (ebpf_shm_find_index_unsafe(pid, &idx)) |
| 71 | return idx; |
| 72 | |
| 73 | if (ebpf_stat_values.current >= ebpf_stat_values.total) |
| 74 | return UINT32_MAX; |
| 75 | |
| 76 | Pvoid_t *Pvalue = JudyLIns(&ebpf_ipc_JudyL, (Word_t)pid, PJE0); |
| 77 | internal_fatal(!Pvalue || Pvalue == PJERR, "EBPF: pid judy index"); |
| 78 | |
| 79 | uint32_t new_idx = ebpf_stat_values.current++; |
| 80 | *Pvalue = IDX_TO_JVALUE(new_idx); |
| 81 | |
| 82 | memset(&integration_shm[new_idx], 0, sizeof(integration_shm[new_idx])); |
| 83 | |
| 84 | return new_idx; |
| 85 | } |
| 86 | |
| 87 | bool netdata_ebpf_reset_shm_pointer_unsafe(int fd, uint32_t pid, enum ebpf_pids_index idx) |
| 88 | { |
| 89 | if (idx != NETDATA_EBPF_PIDS_SOCKET_IDX) |
| 90 | bpf_map_delete_elem(fd, &pid); |
| 91 | |
| 92 | return ebpf_find_pid_shm_del_unsafe(pid, idx); |
| 93 | } |
| 94 | |
| 95 | netdata_ebpf_pid_stats_t *netdata_ebpf_get_shm_pointer_unsafe(uint32_t pid, enum ebpf_pids_index idx) |
| 96 | { |
| 97 | if (!integration_shm) |
| 98 | return NULL; |
| 99 | |
| 100 | // Do NOT short-circuit on a full pool here: an already-tracked PID |
| 101 | // must still be reachable so its module bits can be updated or |
| 102 | // cleared. ebpf_find_or_create_index_pid() returns the existing slot |
| 103 | // regardless of pool saturation and only rejects *new* allocations. |
| 104 | uint32_t shm_idx = ebpf_find_or_create_index_pid(pid); |
| 105 | if (shm_idx == UINT32_MAX || shm_idx >= ebpf_stat_values.total) |
| 106 | return NULL; |
| 107 | |
| 108 | netdata_ebpf_pid_stats_t *ptr = &integration_shm[shm_idx]; |
| 109 | ptr->pid = pid; |
| 110 | ptr->threads |= (1UL << (idx << 1)); |
| 111 | |
| 112 | return ptr; |
| 113 | } |
| 114 | |
| 115 | // Read-only lookup: returns the existing slot for pid or NULL. Does not |
| 116 | // allocate, does not set any bit. Aggregation paths that iterate PID lists |
| 117 | // from /proc or cgroup snapshots MUST use this variant, otherwise every live |
| 118 | // PID acquires module bits for modules that may never observe it in their own |
| 119 | // BPF map and the bits can never be cleared — the shm pool then fills |
| 120 | // monotonically. |
| 121 | netdata_ebpf_pid_stats_t *netdata_ebpf_lookup_shm_pointer_unsafe(uint32_t pid) |
| 122 | { |
| 123 | if (!integration_shm) |
| 124 | return NULL; |
| 125 | |
| 126 | uint32_t shm_idx; |
| 127 | if (!ebpf_shm_find_index_unsafe(pid, &shm_idx)) |
| 128 | return NULL; |
| 129 | |
| 130 | if (shm_idx >= ebpf_stat_values.current) |
| 131 | return NULL; |
| 132 | |
| 133 | return &integration_shm[shm_idx]; |
| 134 | } |
| 135 | |
| 136 | // Module teardown helper: clear this module's bit across every slot that has |
| 137 | // it set. For slots that become empty the existing del path compacts in place, |
| 138 | // which swaps the last slot into the freed index — so we do not advance i when |
| 139 | // the current counter drops. |
| 140 | void netdata_ebpf_sweep_shm_for_module_unsafe(enum ebpf_pids_index idx) |
| 141 | { |
| 142 | if (!integration_shm) |
| 143 | return; |
| 144 | |
| 145 | const uint32_t mask = (1U << (idx << 1)); |
| 146 | uint32_t i = 0; |
| 147 | while (i < ebpf_stat_values.current) { |
| 148 | netdata_ebpf_pid_stats_t *ptr = &integration_shm[i]; |
| 149 | if (!(ptr->threads & mask)) { |
| 150 | i++; |
| 151 | continue; |
| 152 | } |
| 153 | |
| 154 | uint32_t before = ebpf_stat_values.current; |
| 155 | (void)ebpf_find_pid_shm_del_unsafe(ptr->pid, idx); |
| 156 | if (ebpf_stat_values.current >= before) |
| 157 | i++; |
| 158 | } |
| 159 | } |
| 160 | |
| 161 | void netdata_integration_cleanup_shm() |
| 162 | { |
| 163 | if (shm_mutex_ebpf_integration != SEM_FAILED) { |
| 164 | sem_close(shm_mutex_ebpf_integration); |
| 165 | } |
| 166 | |
| 167 | if (integration_shm) { |
| 168 | size_t length = ebpf_stat_values.total * sizeof(netdata_ebpf_pid_stats_t); |
| 169 | nd_munmap(integration_shm, length); |
| 170 | integration_shm = NULL; |
| 171 | } |
| 172 | |
| 173 | // Values are stored inline (no heap allocation), just free the Judy array |
| 174 | (void)JudyLFreeArray(&ebpf_ipc_JudyL, PJE0); |
| 175 | ebpf_ipc_JudyL = NULL; |
| 176 | |
| 177 | if (shm_fd_ebpf_integration > 0) { |
| 178 | close(shm_fd_ebpf_integration); |
| 179 | shm_fd_ebpf_integration = -1; |
| 180 | } |
| 181 | |
| 182 | // Drop the POSIX shm object and the named semaphore so a subsequent |
| 183 | // plugin run starts from a fresh region and a freshly-initialised |
| 184 | // semaphore. Without the sem_unlink, a crashed previous instance can |
| 185 | // leave the semaphore at 0 and the next run will spin on |
| 186 | // sem_timedwait timeouts (sem_open(O_CREAT) ignores the initial value |
| 187 | // when the named semaphore already exists). |
| 188 | (void)shm_unlink(NETDATA_EBPF_INTEGRATION_NAME); |
| 189 | (void)sem_unlink(NETDATA_EBPF_SHM_INTEGRATION_NAME); |
| 190 | } |
| 191 | |
| 192 | int netdata_integration_initialize_shm(size_t pids) |
| 193 | { |
| 194 | if (!pids) |
| 195 | return -1; |
| 196 | |
| 197 | shm_fd_ebpf_integration = shm_open(NETDATA_EBPF_INTEGRATION_NAME, O_CREAT | O_RDWR, 0660); |
| 198 | if (shm_fd_ebpf_integration < 0) { |
| 199 | nd_log(NDLS_COLLECTORS, NDLP_ERR, "Cannot initialize shared memory. Integration won't happen."); |
| 200 | return -1; |
| 201 | } |
| 202 | |
| 203 | ebpf_stat_values.current = 0; |
| 204 | ebpf_stat_values.total = pids; |
| 205 | size_t length = pids * sizeof(netdata_ebpf_pid_stats_t); |
| 206 | if (ftruncate(shm_fd_ebpf_integration, (off_t)length)) { |
| 207 | nd_log(NDLS_COLLECTORS, NDLP_ERR, "Cannot set size for shared memory."); |
| 208 | goto end_shm; |
| 209 | } |
| 210 | |
| 211 | integration_shm = nd_mmap(NULL, length, PROT_READ | PROT_WRITE, MAP_SHARED, shm_fd_ebpf_integration, 0); |
| 212 | if (integration_shm == MAP_FAILED) { |
| 213 | nd_log( |
| 214 | NDLS_COLLECTORS, |
| 215 | NDLP_ERR, |
| 216 | "Cannot map shared memory used between cgroup and eBPF, integration won't happen"); |
| 217 | integration_shm = NULL; |
| 218 | goto end_shm; |
| 219 | } |
| 220 | |
| 221 | // Wipe any bytes left over from a prior plugin run — shm_open with |
| 222 | // O_CREAT on an existing object does not truncate, and ftruncate to the |
| 223 | // current size is a no-op. |
| 224 | memset(integration_shm, 0, length); |
| 225 | |
| 226 | // Drop any leftover named semaphore from a previous (possibly crashed) |
| 227 | // run so sem_open honours the initial value below instead of reusing |
| 228 | // whatever state the previous instance left it in. |
| 229 | (void)sem_unlink(NETDATA_EBPF_SHM_INTEGRATION_NAME); |
| 230 | shm_mutex_ebpf_integration = sem_open( |
| 231 | NETDATA_EBPF_SHM_INTEGRATION_NAME, O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH, 1); |
| 232 | if (shm_mutex_ebpf_integration != SEM_FAILED) { |
| 233 | return 0; |
| 234 | } |
| 235 | |
| 236 | nd_log(NDLS_COLLECTORS, NDLP_ERR, "Cannot create semaphore, integration between won't happen"); |
| 237 | |
| 238 | end_shm: |
| 239 | if (integration_shm) { |
| 240 | size_t unmap_len = ebpf_stat_values.total * sizeof(netdata_ebpf_pid_stats_t); |
| 241 | nd_munmap(integration_shm, unmap_len); |
| 242 | integration_shm = NULL; |
| 243 | } |
| 244 | if (shm_fd_ebpf_integration > 0) { |
| 245 | close(shm_fd_ebpf_integration); |
| 246 | shm_fd_ebpf_integration = -1; |
| 247 | } |
| 248 | return -1; |
| 249 | } |
| 250 | |
| 251 | void netdata_integration_current_ipc_data(ebpf_user_mem_stat_t *values) |
| 252 | { |
| 253 | memcpy(values, &ebpf_stat_values, sizeof(*values)); |
| 254 | } |