| 1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
| 2 | |
| 3 | #include "qemu/osdep.h" |
| 4 | #include "qemu/error-report.h" |
| 5 | #include "qemu/units.h" |
| 6 | #include "qemu/target-info.h" |
| 7 | #include "qemu/log.h" |
| 8 | #include "user/guest-base.h" |
| 9 | #include "user/mmap-min-addr.h" |
| 10 | #include "user/guest-base.h" |
| 11 | #include "user/guest-host.h" |
| 12 | #include "user/probe-guest-base.h" |
| 13 | #include "user/selfmap.h" |
| 14 | #include "exec/target_page.h" |
| 15 | #include <sys/shm.h> |
| 16 | |
| 17 | /* Linux and FreeBSD use different flags to express NOREPLACE. */ |
| 18 | #ifdef __FreeBSD__ |
| 19 | #define MAP_FIXED_NOREPLACE (MAP_FIXED | MAP_EXCL) |
| 20 | #endif |
| 21 | |
| 22 | uintptr_t guest_base; |
| 23 | bool have_guest_base; |
| 24 | |
| 25 | /** |
| 26 | * pgb_try_mmap: |
| 27 | * @addr: host start address |
| 28 | * @addr_last: host last address |
| 29 | * @keep: do not unmap the probe region |
| 30 | * |
| 31 | * Return 1 if [@addr, @addr_last] is not mapped in the host, |
| 32 | * return 0 if it is not available to map, and -1 on mmap error. |
| 33 | * If @keep, the region is left mapped on success, otherwise unmapped. |
| 34 | */ |
| 35 | static int pgb_try_mmap(uintptr_t addr, uintptr_t addr_last, bool keep) |
| 36 | { |
| 37 | size_t size = addr_last - addr + 1; |
| 38 | void *p = mmap((void *)addr, size, PROT_NONE, |
| 39 | MAP_ANONYMOUS | MAP_PRIVATE | |
| 40 | MAP_NORESERVE | MAP_FIXED_NOREPLACE, -1, 0); |
| 41 | int ret; |
| 42 | |
| 43 | if (p == MAP_FAILED) { |
| 44 | return errno == EEXIST ? 0 : -1; |
| 45 | } |
| 46 | ret = p == (void *)addr; |
| 47 | if (!keep || !ret) { |
| 48 | munmap(p, size); |
| 49 | } |
| 50 | return ret; |
| 51 | } |
| 52 | |
| 53 | /** |
| 54 | * pgb_try_mmap_skip_brk(uintptr_t addr, uintptr_t size, uintptr_t brk) |
| 55 | * @addr: host address |
| 56 | * @addr_last: host last address |
| 57 | * @brk: host brk |
| 58 | * |
| 59 | * Like pgb_try_mmap, but additionally reserve some memory following brk. |
| 60 | */ |
| 61 | static int pgb_try_mmap_skip_brk(uintptr_t addr, uintptr_t addr_last, |
| 62 | uintptr_t brk, bool keep) |
| 63 | { |
| 64 | uintptr_t brk_last = brk + 16 * MiB - 1; |
| 65 | |
| 66 | /* Do not map anything close to the host brk. */ |
| 67 | if (addr <= brk_last && brk <= addr_last) { |
| 68 | return 0; |
| 69 | } |
| 70 | return pgb_try_mmap(addr, addr_last, keep); |
| 71 | } |
| 72 | |
| 73 | /** |
| 74 | * pgb_try_mmap_set: |
| 75 | * @ga: set of guest addrs |
| 76 | * @base: guest_base |
| 77 | * @brk: host brk |
| 78 | * |
| 79 | * Return true if all @ga can be mapped by the host at @base. |
| 80 | * On success, retain the mapping at index 0 for reserved_va. |
| 81 | */ |
| 82 | |
| 83 | typedef struct PGBAddrs { |
| 84 | PGBRange bounds[3]; |
| 85 | int nbounds; |
| 86 | } PGBAddrs; |
| 87 | |
| 88 | static bool pgb_try_mmap_set(const PGBAddrs *ga, uintptr_t base, uintptr_t brk) |
| 89 | { |
| 90 | for (int i = ga->nbounds - 1; i >= 0; --i) { |
| 91 | if (pgb_try_mmap_skip_brk(ga->bounds[i].lo + base, |
| 92 | ga->bounds[i].hi + base, |
| 93 | brk, i == 0 && reserved_va) <= 0) { |
| 94 | return false; |
| 95 | } |
| 96 | } |
| 97 | return true; |
| 98 | } |
| 99 | |
| 100 | /** |
| 101 | * pgb_addr_set: |
| 102 | * @ga: output set of guest addrs |
| 103 | * @image_range: fixed guest image addresses |
| 104 | * @identity: create for identity mapping |
| 105 | * |
| 106 | * Fill in @ga with the image, COMMPAGE and NULL page. |
| 107 | */ |
| 108 | static bool pgb_addr_set(PGBAddrs *ga, const PGBRange *image_range, |
| 109 | const PGBRange *commpage_range, bool try_identity) |
| 110 | { |
| 111 | int n; |
| 112 | |
| 113 | /* |
| 114 | * With a low commpage, or a guest mapped very low, |
| 115 | * we may not be able to use the identity map. |
| 116 | */ |
| 117 | if (try_identity) { |
| 118 | if (commpage_range && commpage_range->lo < mmap_min_addr) { |
| 119 | return false; |
| 120 | } |
| 121 | if (image_range && image_range->lo < mmap_min_addr) { |
| 122 | return false; |
| 123 | } |
| 124 | } |
| 125 | |
| 126 | memset(ga, 0, sizeof(*ga)); |
| 127 | n = 0; |
| 128 | |
| 129 | if (reserved_va) { |
| 130 | ga->bounds[n].lo = try_identity ? mmap_min_addr : 0; |
| 131 | ga->bounds[n].hi = reserved_va; |
| 132 | n++; |
| 133 | /* Low COMMPAGE and NULL handled by reserving from 0. */ |
| 134 | } else { |
| 135 | /* Add any low COMMPAGE or NULL page. */ |
| 136 | if (!try_identity || (commpage_range && commpage_range->lo == 0)) { |
| 137 | ga->bounds[n].lo = 0; |
| 138 | ga->bounds[n].hi = TARGET_PAGE_SIZE - 1; |
| 139 | n++; |
| 140 | } |
| 141 | |
| 142 | /* Add the guest image for ET_EXEC. */ |
| 143 | if (image_range) { |
| 144 | ga->bounds[n++] = *image_range; |
| 145 | } |
| 146 | } |
| 147 | |
| 148 | /* Add any high COMMPAGE not covered by reserved_va. */ |
| 149 | if (commpage_range && reserved_va < commpage_range->hi) { |
| 150 | ga->bounds[n].lo = commpage_range->lo & qemu_real_host_page_mask(); |
| 151 | ga->bounds[n].hi = commpage_range->hi; |
| 152 | n++; |
| 153 | } |
| 154 | |
| 155 | ga->nbounds = n; |
| 156 | return true; |
| 157 | } |
| 158 | |
| 159 | static void pgb_fail_in_use(const char *image_name) |
| 160 | { |
| 161 | error_report("%s: requires virtual address space that is in use " |
| 162 | "(omit the -B option or choose a different value)", |
| 163 | image_name); |
| 164 | exit(EXIT_FAILURE); |
| 165 | } |
| 166 | |
| 167 | static void pgb_fixed(const char *image_name, const PGBRange *image_range, |
| 168 | const PGBRange *commpage_range, uintptr_t align) |
| 169 | { |
| 170 | PGBAddrs ga; |
| 171 | uintptr_t brk = (uintptr_t)sbrk(0); |
| 172 | |
| 173 | if (!QEMU_IS_ALIGNED(guest_base, align)) { |
| 174 | fprintf(stderr, "Requested guest base %p does not satisfy " |
| 175 | "host minimum alignment (0x%" PRIxPTR ")\n", |
| 176 | (void *)guest_base, align); |
| 177 | exit(EXIT_FAILURE); |
| 178 | } |
| 179 | |
| 180 | if (!pgb_addr_set(&ga, image_range, commpage_range, !guest_base) |
| 181 | || !pgb_try_mmap_set(&ga, guest_base, brk)) { |
| 182 | pgb_fail_in_use(image_name); |
| 183 | } |
| 184 | } |
| 185 | |
| 186 | /** |
| 187 | * pgb_find_fallback: |
| 188 | * |
| 189 | * This is a fallback method for finding holes in the host address space |
| 190 | * if we don't have the benefit of being able to access /proc/self/map. |
| 191 | * It can potentially take a very long time as we can only dumbly iterate |
| 192 | * up the host address space seeing if the allocation would work. |
| 193 | */ |
| 194 | static uintptr_t pgb_find_fallback(const PGBAddrs *ga, uintptr_t align, |
| 195 | uintptr_t brk) |
| 196 | { |
| 197 | /* TODO: come up with a better estimate of how much to skip. */ |
| 198 | uintptr_t skip = sizeof(uintptr_t) == 4 ? MiB : GiB; |
| 199 | |
| 200 | for (uintptr_t base = skip; ; base += skip) { |
| 201 | base = ROUND_UP(base, align); |
| 202 | if (pgb_try_mmap_set(ga, base, brk)) { |
| 203 | return base; |
| 204 | } |
| 205 | if (base >= -skip) { |
| 206 | return -1; |
| 207 | } |
| 208 | } |
| 209 | } |
| 210 | |
| 211 | static uintptr_t pgb_try_itree(const PGBAddrs *ga, uintptr_t base, |
| 212 | IntervalTreeRoot *root) |
| 213 | { |
| 214 | for (int i = ga->nbounds - 1; i >= 0; --i) { |
| 215 | uintptr_t s = base + ga->bounds[i].lo; |
| 216 | uintptr_t l = base + ga->bounds[i].hi; |
| 217 | IntervalTreeNode *n; |
| 218 | |
| 219 | if (l < s) { |
| 220 | /* Wraparound. Skip to advance S to mmap_min_addr. */ |
| 221 | return mmap_min_addr - s; |
| 222 | } |
| 223 | |
| 224 | n = interval_tree_iter_first(root, s, l); |
| 225 | if (n != NULL) { |
| 226 | /* Conflict. Skip to advance S to LAST + 1. */ |
| 227 | return n->last - s + 1; |
| 228 | } |
| 229 | } |
| 230 | return 0; /* success */ |
| 231 | } |
| 232 | |
| 233 | static uintptr_t pgb_find_itree(const PGBAddrs *ga, IntervalTreeRoot *root, |
| 234 | uintptr_t align, uintptr_t brk) |
| 235 | { |
| 236 | uintptr_t last = sizeof(uintptr_t) == 4 ? MiB : GiB; |
| 237 | uintptr_t base, skip; |
| 238 | |
| 239 | while (true) { |
| 240 | base = ROUND_UP(last, align); |
| 241 | if (base < last) { |
| 242 | return -1; |
| 243 | } |
| 244 | |
| 245 | skip = pgb_try_itree(ga, base, root); |
| 246 | if (skip == 0) { |
| 247 | break; |
| 248 | } |
| 249 | |
| 250 | last = base + skip; |
| 251 | if (last < base) { |
| 252 | return -1; |
| 253 | } |
| 254 | } |
| 255 | |
| 256 | /* |
| 257 | * We've chosen 'base' based on holes in the interval tree, |
| 258 | * but we don't yet know if it is a valid host address. |
| 259 | * Because it is the first matching hole, if the host addresses |
| 260 | * are invalid we know there are no further matches. |
| 261 | */ |
| 262 | return pgb_try_mmap_set(ga, base, brk) ? base : -1; |
| 263 | } |
| 264 | |
| 265 | static void pgb_dynamic(const char *image_name, const PGBRange *image_range, |
| 266 | const PGBRange *commpage_range, uintptr_t align) |
| 267 | { |
| 268 | IntervalTreeRoot *root; |
| 269 | uintptr_t brk, ret; |
| 270 | PGBAddrs ga; |
| 271 | |
| 272 | /* Try the identity map first. */ |
| 273 | if (pgb_addr_set(&ga, image_range, commpage_range, true)) { |
| 274 | brk = (uintptr_t)sbrk(0); |
| 275 | if (pgb_try_mmap_set(&ga, 0, brk)) { |
| 276 | guest_base = 0; |
| 277 | return; |
| 278 | } |
| 279 | } |
| 280 | |
| 281 | /* |
| 282 | * Rebuild the address set for non-identity map. |
| 283 | * This differs in the mapping of the guest NULL page. |
| 284 | */ |
| 285 | pgb_addr_set(&ga, image_range, commpage_range, false); |
| 286 | |
| 287 | root = read_self_maps(); |
| 288 | |
| 289 | /* Read brk after we've read the maps, which will malloc. */ |
| 290 | brk = (uintptr_t)sbrk(0); |
| 291 | |
| 292 | if (!root) { |
| 293 | ret = pgb_find_fallback(&ga, align, brk); |
| 294 | } else { |
| 295 | /* |
| 296 | * Reserve the area close to the host brk. |
| 297 | * This will be freed with the rest of the tree. |
| 298 | */ |
| 299 | IntervalTreeNode *b = g_new0(IntervalTreeNode, 1); |
| 300 | b->start = brk; |
| 301 | b->last = brk + 16 * MiB - 1; |
| 302 | interval_tree_insert(b, root); |
| 303 | |
| 304 | ret = pgb_find_itree(&ga, root, align, brk); |
| 305 | free_self_maps(root); |
| 306 | } |
| 307 | |
| 308 | if (ret == -1) { |
| 309 | int w = target_long_bits() / 4; |
| 310 | |
| 311 | error_report("%s: Unable to find a guest_base to satisfy all " |
| 312 | "guest address mapping requirements", image_name); |
| 313 | |
| 314 | for (int i = 0; i < ga.nbounds; ++i) { |
| 315 | error_printf(" %0*" VADDR_PRIx "-%0*" VADDR_PRIx "\n", |
| 316 | w, ga.bounds[i].lo, |
| 317 | w, ga.bounds[i].hi); |
| 318 | } |
| 319 | exit(EXIT_FAILURE); |
| 320 | } |
| 321 | guest_base = ret; |
| 322 | } |
| 323 | |
| 324 | void probe_guest_base(const char *image_name, const PGBRange *image_range, |
| 325 | const PGBRange *commpage_range) |
| 326 | { |
| 327 | /* In order to use host shmat, we must be able to honor SHMLBA. */ |
| 328 | uintptr_t align = MAX(SHMLBA, TARGET_PAGE_SIZE); |
| 329 | |
| 330 | /* Sanity check the guest binary. */ |
| 331 | if (reserved_va && image_range && image_range->hi > reserved_va) { |
| 332 | error_report("%s: requires more than reserved virtual " |
| 333 | "address space (0x%" VADDR_PRIx " > 0x%lx)", |
| 334 | image_name, image_range->hi, reserved_va); |
| 335 | exit(EXIT_FAILURE); |
| 336 | } |
| 337 | |
| 338 | if (have_guest_base) { |
| 339 | pgb_fixed(image_name, image_range, commpage_range, align); |
| 340 | } else { |
| 341 | pgb_dynamic(image_name, image_range, commpage_range, align); |
| 342 | } |
| 343 | |
| 344 | assert(QEMU_IS_ALIGNED(guest_base, align)); |
| 345 | qemu_log_mask(CPU_LOG_PAGE, "Locating guest address space " |
| 346 | "@ 0x%" PRIx64 "\n", (uint64_t)guest_base); |
| 347 | } |