| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #include "../libnetdata.h" |
| 4 | |
| 5 | size_t nd_mmap_count = 0; |
| 6 | size_t nd_mmap_size = 0; |
| 7 | |
| 8 | #if !defined(MADV_DONTFORK) |
| 9 | #define MADV_DONTFORK 0 |
| 10 | #endif |
| 11 | |
| 12 | #if !defined(O_NOATIME) |
| 13 | #define O_NOATIME 0 |
| 14 | #endif |
| 15 | |
| 16 | #if defined(MADV_MERGEABLE) |
| 17 | int enable_ksm = CONFIG_BOOLEAN_AUTO; |
| 18 | #else |
| 19 | int enable_ksm = 0; |
| 20 | #endif |
| 21 | |
| 22 | static int memory_file_open(const char *filename, size_t size) { |
| 23 | int fd = open(filename, O_RDWR | O_CREAT | O_NOATIME | O_CLOEXEC, 0664); |
| 24 | if (fd == -1) { |
| 25 | netdata_log_error("Cannot create/open file '%s'.", filename); |
| 26 | return -1; |
| 27 | } |
| 28 | |
| 29 | if (ftruncate(fd, size) != 0) { |
| 30 | netdata_log_error("Cannot truncate file '%s' to size %zu.", filename, size); |
| 31 | close(fd); |
| 32 | return -1; |
| 33 | } |
| 34 | |
| 35 | struct stat st; |
| 36 | if (fstat(fd, &st) != 0 || (size_t)st.st_size < size) { |
| 37 | netdata_log_error("File '%s' is only %lld bytes, expected %zu!", filename, (long long)st.st_size, size); |
| 38 | close(fd); |
| 39 | return -1; |
| 40 | } |
| 41 | |
| 42 | return fd; |
| 43 | } |
| 44 | |
| 45 | inline int madvise_sequential(void *mem, size_t len) { |
| 46 | static int logger = 1; |
| 47 | int ret = madvise(mem, len, MADV_SEQUENTIAL); |
| 48 | |
| 49 | if (ret != 0 && logger-- > 0) |
| 50 | netdata_log_error("madvise(MADV_SEQUENTIAL) of size %zu, failed.", len); |
| 51 | return ret; |
| 52 | } |
| 53 | |
| 54 | inline int madvise_random(void *mem, size_t len) { |
| 55 | static int logger = 1; |
| 56 | int ret = madvise(mem, len, MADV_RANDOM); |
| 57 | |
| 58 | if (ret != 0 && logger-- > 0) |
| 59 | netdata_log_error("madvise(MADV_RANDOM) of size %zu, failed.", len); |
| 60 | return ret; |
| 61 | } |
| 62 | |
| 63 | inline int madvise_dontfork(void *mem, size_t len) { |
| 64 | static int logger = 1; |
| 65 | int ret = madvise(mem, len, MADV_DONTFORK); |
| 66 | |
| 67 | if (ret != 0 && logger-- > 0) |
| 68 | netdata_log_error("madvise(MADV_DONTFORK) of size %zu, failed.", len); |
| 69 | return ret; |
| 70 | } |
| 71 | |
| 72 | inline int madvise_willneed(void *mem, size_t len) { |
| 73 | static int logger = 1; |
| 74 | int ret = madvise(mem, len, MADV_WILLNEED); |
| 75 | |
| 76 | if (ret != 0 && logger-- > 0) |
| 77 | netdata_log_error("madvise(MADV_WILLNEED) of size %zu, failed.", len); |
| 78 | return ret; |
| 79 | } |
| 80 | |
| 81 | inline int madvise_dontneed(void *mem, size_t len) { |
| 82 | static int logger = 1; |
| 83 | int ret = madvise(mem, len, MADV_DONTNEED); |
| 84 | |
| 85 | if (ret != 0 && logger-- > 0) |
| 86 | netdata_log_error("madvise(MADV_DONTNEED) of size %zu, failed.", len); |
| 87 | return ret; |
| 88 | } |
| 89 | |
| 90 | inline int madvise_dontdump(void *mem __maybe_unused, size_t len __maybe_unused) { |
| 91 | #if __linux__ |
| 92 | static int logger = 1; |
| 93 | int ret = madvise(mem, len, MADV_DONTDUMP); |
| 94 | |
| 95 | if (ret != 0 && logger-- > 0) |
| 96 | netdata_log_error("madvise(MADV_DONTDUMP) of size %zu, failed.", len); |
| 97 | return ret; |
| 98 | #else |
| 99 | return 0; |
| 100 | #endif |
| 101 | } |
| 102 | |
| 103 | inline int madvise_mergeable(void *mem __maybe_unused, size_t len __maybe_unused) { |
| 104 | #ifdef MADV_MERGEABLE |
| 105 | static int logger = 1; |
| 106 | int ret = madvise(mem, len, MADV_MERGEABLE); |
| 107 | |
| 108 | if (ret != 0 && logger-- > 0) |
| 109 | netdata_log_error("madvise(MADV_MERGEABLE) of size %zu, failed.", len); |
| 110 | return ret; |
| 111 | #else |
| 112 | return 0; |
| 113 | #endif |
| 114 | } |
| 115 | |
| 116 | #define THP_SIZE (2 * 1024 * 1024) // 2 MiB THP size |
| 117 | #define THP_MASK (THP_SIZE - 1) // Mask for alignment check |
| 118 | |
| 119 | inline int madvise_thp(void *mem __maybe_unused, size_t len __maybe_unused) { |
| 120 | #ifdef MADV_HUGEPAGE |
| 121 | // Check if the size is at least THP size and aligned |
| 122 | if (len >= THP_SIZE && ((uintptr_t)mem & THP_MASK) == 0) { |
| 123 | return madvise(mem, len, MADV_HUGEPAGE); |
| 124 | } |
| 125 | #endif |
| 126 | return 0; // Do nothing if THP is not supported or size is too small |
| 127 | } |
| 128 | |
| 129 | int nd_munmap(void *ptr, size_t size) { |
| 130 | #ifdef NETDATA_TRACE_ALLOCATIONS |
| 131 | malloc_trace_munmap(size); |
| 132 | #endif |
| 133 | |
| 134 | workers_memory_call(WORKERS_MEMORY_CALL_MUNMAP); |
| 135 | int rc = munmap(ptr, size); |
| 136 | |
| 137 | if(rc == 0) { |
| 138 | __atomic_sub_fetch(&nd_mmap_count, 1, __ATOMIC_RELAXED); |
| 139 | __atomic_sub_fetch(&nd_mmap_size, size, __ATOMIC_RELAXED); |
| 140 | } |
| 141 | |
| 142 | return rc; |
| 143 | } |
| 144 | |
| 145 | void *nd_mmap(void *addr, size_t len, int prot, int flags, int fd, off_t offset) { |
| 146 | workers_memory_call(WORKERS_MEMORY_CALL_MMAP); |
| 147 | |
| 148 | void *rc = mmap(addr, len, prot, flags, fd, offset); |
| 149 | |
| 150 | if(rc != MAP_FAILED) { |
| 151 | __atomic_add_fetch(&nd_mmap_count, 1, __ATOMIC_RELAXED); |
| 152 | __atomic_add_fetch(&nd_mmap_size, len, __ATOMIC_RELAXED); |
| 153 | |
| 154 | #ifdef NETDATA_TRACE_ALLOCATIONS |
| 155 | malloc_trace_mmap(size); |
| 156 | #endif |
| 157 | } |
| 158 | |
| 159 | return rc; |
| 160 | } |
| 161 | |
| 162 | void *nd_mmap_advanced(const char *filename, size_t size, int flags, int ksm, bool read_only, bool dont_dump, int *open_fd) { |
| 163 | // netdata_log_info("netdata_mmap('%s', %zu", filename, size); |
| 164 | |
| 165 | // MAP_SHARED is used in memory mode map |
| 166 | // MAP_PRIVATE is used in memory mode ram and save |
| 167 | |
| 168 | if(unlikely(!(flags & MAP_SHARED) && !(flags & MAP_PRIVATE))) |
| 169 | fatal("Neither MAP_SHARED or MAP_PRIVATE were given to nd_mmap_advanced()"); |
| 170 | |
| 171 | if(unlikely((flags & MAP_SHARED) && (flags & MAP_PRIVATE))) |
| 172 | fatal("Both MAP_SHARED and MAP_PRIVATE were given to nd_mmap_advanced()"); |
| 173 | |
| 174 | if(unlikely((flags & MAP_SHARED) && (!filename || !*filename))) |
| 175 | fatal("MAP_SHARED requested, without a filename to nd_mmap_advanced()"); |
| 176 | |
| 177 | // don't enable ksm is the global setting is disabled |
| 178 | if(unlikely(!enable_ksm)) ksm = 0; |
| 179 | |
| 180 | // KSM only merges anonymous (private) pages, never pagecache (file) pages |
| 181 | // but MAP_PRIVATE without MAP_ANONYMOUS it fails too, so we need it always |
| 182 | if((flags & MAP_PRIVATE)) flags |= MAP_ANONYMOUS; |
| 183 | |
| 184 | int fd = -1; |
| 185 | void *mem = MAP_FAILED; |
| 186 | |
| 187 | errno_clear(); |
| 188 | |
| 189 | if(filename && *filename) { |
| 190 | // open/create the file to be used |
| 191 | fd = memory_file_open(filename, size); |
| 192 | if(fd == -1) goto cleanup; |
| 193 | } |
| 194 | |
| 195 | int fd_for_mmap = fd; |
| 196 | if(fd != -1 && (flags & MAP_PRIVATE)) { |
| 197 | // this is MAP_PRIVATE allocation |
| 198 | // no need for mmap() to use our fd |
| 199 | // we will copy the file into the memory allocated |
| 200 | fd_for_mmap = -1; |
| 201 | } |
| 202 | |
| 203 | mem = nd_mmap(NULL, size, read_only ? PROT_READ : PROT_READ | PROT_WRITE, flags, fd_for_mmap, 0); |
| 204 | if (mem != MAP_FAILED) { |
| 205 | // if we have a file open, but we didn't give it to mmap(), |
| 206 | // we have to read the file into the memory block we allocated |
| 207 | if(fd != -1 && fd_for_mmap == -1) { |
| 208 | if (lseek(fd, 0, SEEK_SET) == 0) { |
| 209 | if (read(fd, mem, size) != (ssize_t) size) |
| 210 | netdata_log_info("Cannot read from file '%s'", filename); |
| 211 | } |
| 212 | else netdata_log_info("Cannot seek to beginning of file '%s'.", filename); |
| 213 | } |
| 214 | |
| 215 | madvise_thp(mem, size); |
| 216 | // madvise_sequential(mem, size); |
| 217 | // madvise_dontfork(mem, size); // aral is initialized before we daemonize |
| 218 | if(dont_dump) madvise_dontdump(mem, size); |
| 219 | // if(flags & MAP_SHARED) madvise_willneed(mem, size); |
| 220 | if(ksm) madvise_mergeable(mem, size); |
| 221 | } |
| 222 | |
| 223 | cleanup: |
| 224 | if(fd != -1) { |
| 225 | if (open_fd) |
| 226 | *open_fd = fd; |
| 227 | else |
| 228 | close(fd); |
| 229 | } |
| 230 | |
| 231 | if(mem == MAP_FAILED) |
| 232 | return NULL; |
| 233 | |
| 234 | errno_clear(); |
| 235 | return mem; |
| 236 | } |