| 1 | /* |
| 2 | * AArch64 specific helpers |
| 3 | * |
| 4 | * Copyright (c) 2013 Alexander Graf <agraf@suse.de> |
| 5 | * |
| 6 | * This library is free software; you can redistribute it and/or |
| 7 | * modify it under the terms of the GNU Lesser General Public |
| 8 | * License as published by the Free Software Foundation; either |
| 9 | * version 2.1 of the License, or (at your option) any later version. |
| 10 | * |
| 11 | * This library is distributed in the hope that it will be useful, |
| 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 14 | * Lesser General Public License for more details. |
| 15 | * |
| 16 | * You should have received a copy of the GNU Lesser General Public |
| 17 | * License along with this library; if not, see <http://www.gnu.org/licenses/>. |
| 18 | */ |
| 19 | |
| 20 | #include "qemu/osdep.h" |
| 21 | #include "qemu/units.h" |
| 22 | #include "cpu.h" |
| 23 | #include "gdbstub/helpers.h" |
| 24 | #include "helper.h" |
| 25 | #include "helper-a64.h" |
| 26 | #include "qemu/host-utils.h" |
| 27 | #include "qemu/log.h" |
| 28 | #include "qemu/main-loop.h" |
| 29 | #include "qemu/bitops.h" |
| 30 | #include "internals.h" |
| 31 | #include "qemu/crc32c.h" |
| 32 | #include "exec/cpu-common.h" |
| 33 | #include "accel/tcg/cpu-ldst.h" |
| 34 | #include "accel/tcg/cpu-loop.h" |
| 35 | #include "accel/tcg/helper-retaddr.h" |
| 36 | #include "accel/tcg/probe.h" |
| 37 | #include "exec/target_page.h" |
| 38 | #include "exec/tlb-flags.h" |
| 39 | #include "qemu/int128.h" |
| 40 | #include "qemu/atomic128.h" |
| 41 | #include "fpu/softfloat.h" |
| 42 | #include <zlib.h> /* for crc32 */ |
| 43 | #ifdef CONFIG_USER_ONLY |
| 44 | #include "user/page-protection.h" |
| 45 | #endif |
| 46 | #include "vec_internal.h" |
| 47 | |
| 48 | #define HELPER_H "tcg/helper-a64-defs.h" |
| 49 | #include "exec/helper-info.c.inc" |
| 50 | |
| 51 | /* C2.4.7 Multiply and divide */ |
| 52 | /* special cases for 0 and LLONG_MIN are mandated by the standard */ |
| 53 | uint64_t HELPER(udiv64)(uint64_t num, uint64_t den) |
| 54 | { |
| 55 | if (den == 0) { |
| 56 | return 0; |
| 57 | } |
| 58 | return num / den; |
| 59 | } |
| 60 | |
| 61 | int64_t HELPER(sdiv64)(int64_t num, int64_t den) |
| 62 | { |
| 63 | if (den == 0) { |
| 64 | return 0; |
| 65 | } |
| 66 | if (num == LLONG_MIN && den == -1) { |
| 67 | return LLONG_MIN; |
| 68 | } |
| 69 | return num / den; |
| 70 | } |
| 71 | |
| 72 | void HELPER(msr_i_spsel)(CPUARMState *env, uint32_t imm) |
| 73 | { |
| 74 | update_spsel(env, imm); |
| 75 | } |
| 76 | |
| 77 | void HELPER(msr_set_allint_el1)(CPUARMState *env) |
| 78 | { |
| 79 | /* ALLINT update to PSTATE. */ |
| 80 | if (arm_hcrx_el2_eff(env) & HCRX_TALLINT) { |
| 81 | raise_exception_ra(env, EXCP_UDEF, |
| 82 | syn_aa64_sysregtrap(0, 1, 0, 4, 1, 0x1f, 0), 2, |
| 83 | GETPC()); |
| 84 | } |
| 85 | |
| 86 | env->pstate |= PSTATE_ALLINT; |
| 87 | } |
| 88 | |
| 89 | static void daif_check(CPUARMState *env, uint32_t op, |
| 90 | uint32_t imm, uintptr_t ra) |
| 91 | { |
| 92 | /* DAIF update to PSTATE. This is OK from EL0 only if UMA is set. */ |
| 93 | if (arm_current_el(env) == 0 && !(arm_sctlr(env, 0) & SCTLR_UMA)) { |
| 94 | raise_exception_ra(env, EXCP_UDEF, |
| 95 | syn_aa64_sysregtrap(0, extract32(op, 0, 3), |
| 96 | extract32(op, 3, 3), 4, |
| 97 | imm, 0x1f, 0), |
| 98 | exception_target_el(env), ra); |
| 99 | } |
| 100 | } |
| 101 | |
| 102 | void HELPER(msr_i_daifset)(CPUARMState *env, uint32_t imm) |
| 103 | { |
| 104 | daif_check(env, 0x1e, imm, GETPC()); |
| 105 | env->daif |= (imm << 6) & PSTATE_DAIF; |
| 106 | arm_rebuild_hflags(env); |
| 107 | } |
| 108 | |
| 109 | void HELPER(msr_i_daifclear)(CPUARMState *env, uint32_t imm) |
| 110 | { |
| 111 | daif_check(env, 0x1f, imm, GETPC()); |
| 112 | env->daif &= ~((imm << 6) & PSTATE_DAIF); |
| 113 | arm_rebuild_hflags(env); |
| 114 | } |
| 115 | |
| 116 | /* Convert a softfloat float_relation_ (as returned by |
| 117 | * the float*_compare functions) to the correct ARM |
| 118 | * NZCV flag state. |
| 119 | */ |
| 120 | static inline uint32_t float_rel_to_flags(int res) |
| 121 | { |
| 122 | uint64_t flags; |
| 123 | switch (res) { |
| 124 | case float_relation_equal: |
| 125 | flags = PSTATE_Z | PSTATE_C; |
| 126 | break; |
| 127 | case float_relation_less: |
| 128 | flags = PSTATE_N; |
| 129 | break; |
| 130 | case float_relation_greater: |
| 131 | flags = PSTATE_C; |
| 132 | break; |
| 133 | case float_relation_unordered: |
| 134 | default: |
| 135 | flags = PSTATE_C | PSTATE_V; |
| 136 | break; |
| 137 | } |
| 138 | return flags; |
| 139 | } |
| 140 | |
| 141 | uint64_t HELPER(vfp_cmph_a64)(uint32_t x, uint32_t y, float_status *fp_status) |
| 142 | { |
| 143 | return float_rel_to_flags(float16_compare_quiet(x, y, fp_status)); |
| 144 | } |
| 145 | |
| 146 | uint64_t HELPER(vfp_cmpeh_a64)(uint32_t x, uint32_t y, float_status *fp_status) |
| 147 | { |
| 148 | return float_rel_to_flags(float16_compare(x, y, fp_status)); |
| 149 | } |
| 150 | |
| 151 | uint64_t HELPER(vfp_cmps_a64)(float32 x, float32 y, float_status *fp_status) |
| 152 | { |
| 153 | return float_rel_to_flags(float32_compare_quiet(x, y, fp_status)); |
| 154 | } |
| 155 | |
| 156 | uint64_t HELPER(vfp_cmpes_a64)(float32 x, float32 y, float_status *fp_status) |
| 157 | { |
| 158 | return float_rel_to_flags(float32_compare(x, y, fp_status)); |
| 159 | } |
| 160 | |
| 161 | uint64_t HELPER(vfp_cmpd_a64)(float64 x, float64 y, float_status *fp_status) |
| 162 | { |
| 163 | return float_rel_to_flags(float64_compare_quiet(x, y, fp_status)); |
| 164 | } |
| 165 | |
| 166 | uint64_t HELPER(vfp_cmped_a64)(float64 x, float64 y, float_status *fp_status) |
| 167 | { |
| 168 | return float_rel_to_flags(float64_compare(x, y, fp_status)); |
| 169 | } |
| 170 | |
| 171 | float32 HELPER(vfp_mulxs)(float32 a, float32 b, float_status *fpst) |
| 172 | { |
| 173 | a = float32_squash_input_denormal(a, fpst); |
| 174 | b = float32_squash_input_denormal(b, fpst); |
| 175 | |
| 176 | if ((float32_is_zero(a) && float32_is_infinity(b)) || |
| 177 | (float32_is_infinity(a) && float32_is_zero(b))) { |
| 178 | /* 2.0 with the sign bit set to sign(A) XOR sign(B) */ |
| 179 | return make_float32((1U << 30) | |
| 180 | ((float32_val(a) ^ float32_val(b)) & (1U << 31))); |
| 181 | } |
| 182 | return float32_mul(a, b, fpst); |
| 183 | } |
| 184 | |
| 185 | float64 HELPER(vfp_mulxd)(float64 a, float64 b, float_status *fpst) |
| 186 | { |
| 187 | a = float64_squash_input_denormal(a, fpst); |
| 188 | b = float64_squash_input_denormal(b, fpst); |
| 189 | |
| 190 | if ((float64_is_zero(a) && float64_is_infinity(b)) || |
| 191 | (float64_is_infinity(a) && float64_is_zero(b))) { |
| 192 | /* 2.0 with the sign bit set to sign(A) XOR sign(B) */ |
| 193 | return make_float64((1ULL << 62) | |
| 194 | ((float64_val(a) ^ float64_val(b)) & (1ULL << 63))); |
| 195 | } |
| 196 | return float64_mul(a, b, fpst); |
| 197 | } |
| 198 | |
| 199 | /* 64bit/double versions of the neon float compare functions */ |
| 200 | uint64_t HELPER(neon_ceq_f64)(float64 a, float64 b, float_status *fpst) |
| 201 | { |
| 202 | return -float64_eq_quiet(a, b, fpst); |
| 203 | } |
| 204 | |
| 205 | uint64_t HELPER(neon_cge_f64)(float64 a, float64 b, float_status *fpst) |
| 206 | { |
| 207 | return -float64_le(b, a, fpst); |
| 208 | } |
| 209 | |
| 210 | uint64_t HELPER(neon_cgt_f64)(float64 a, float64 b, float_status *fpst) |
| 211 | { |
| 212 | return -float64_lt(b, a, fpst); |
| 213 | } |
| 214 | |
| 215 | /* |
| 216 | * Reciprocal step and sqrt step. Note that unlike the A32/T32 |
| 217 | * versions, these do a fully fused multiply-add or |
| 218 | * multiply-add-and-halve. |
| 219 | * The FPCR.AH == 1 versions need to avoid flipping the sign of NaN. |
| 220 | */ |
| 221 | #define DO_RECPS(NAME, CTYPE, FLOATTYPE, CHSFN) \ |
| 222 | CTYPE HELPER(NAME)(CTYPE a, CTYPE b, float_status *fpst) \ |
| 223 | { \ |
| 224 | a = FLOATTYPE ## _squash_input_denormal(a, fpst); \ |
| 225 | b = FLOATTYPE ## _squash_input_denormal(b, fpst); \ |
| 226 | a = FLOATTYPE ## _ ## CHSFN(a); \ |
| 227 | if ((FLOATTYPE ## _is_infinity(a) && FLOATTYPE ## _is_zero(b)) || \ |
| 228 | (FLOATTYPE ## _is_infinity(b) && FLOATTYPE ## _is_zero(a))) { \ |
| 229 | return FLOATTYPE ## _two; \ |
| 230 | } \ |
| 231 | return FLOATTYPE ## _muladd(a, b, FLOATTYPE ## _two, 0, fpst); \ |
| 232 | } |
| 233 | |
| 234 | DO_RECPS(recpsf_f16, uint32_t, float16, chs) |
| 235 | DO_RECPS(recpsf_f32, float32, float32, chs) |
| 236 | DO_RECPS(recpsf_f64, float64, float64, chs) |
| 237 | DO_RECPS(recpsf_ah_f16, uint32_t, float16, ah_chs) |
| 238 | DO_RECPS(recpsf_ah_f32, float32, float32, ah_chs) |
| 239 | DO_RECPS(recpsf_ah_f64, float64, float64, ah_chs) |
| 240 | |
| 241 | #define DO_RSQRTSF(NAME, CTYPE, FLOATTYPE, CHSFN) \ |
| 242 | CTYPE HELPER(NAME)(CTYPE a, CTYPE b, float_status *fpst) \ |
| 243 | { \ |
| 244 | a = FLOATTYPE ## _squash_input_denormal(a, fpst); \ |
| 245 | b = FLOATTYPE ## _squash_input_denormal(b, fpst); \ |
| 246 | a = FLOATTYPE ## _ ## CHSFN(a); \ |
| 247 | if ((FLOATTYPE ## _is_infinity(a) && FLOATTYPE ## _is_zero(b)) || \ |
| 248 | (FLOATTYPE ## _is_infinity(b) && FLOATTYPE ## _is_zero(a))) { \ |
| 249 | return FLOATTYPE ## _one_point_five; \ |
| 250 | } \ |
| 251 | return FLOATTYPE ## _muladd_scalbn(a, b, FLOATTYPE ## _three, \ |
| 252 | -1, 0, fpst); \ |
| 253 | } \ |
| 254 | |
| 255 | DO_RSQRTSF(rsqrtsf_f16, uint32_t, float16, chs) |
| 256 | DO_RSQRTSF(rsqrtsf_f32, float32, float32, chs) |
| 257 | DO_RSQRTSF(rsqrtsf_f64, float64, float64, chs) |
| 258 | DO_RSQRTSF(rsqrtsf_ah_f16, uint32_t, float16, ah_chs) |
| 259 | DO_RSQRTSF(rsqrtsf_ah_f32, float32, float32, ah_chs) |
| 260 | DO_RSQRTSF(rsqrtsf_ah_f64, float64, float64, ah_chs) |
| 261 | |
| 262 | /* Floating-point reciprocal exponent - see FPRecpX in ARM ARM */ |
| 263 | uint32_t HELPER(frecpx_f16)(uint32_t a, float_status *fpst) |
| 264 | { |
| 265 | uint16_t val16, sbit; |
| 266 | int16_t exp; |
| 267 | |
| 268 | if (float16_is_any_nan(a)) { |
| 269 | float16 nan = a; |
| 270 | if (float16_is_signaling_nan(a, fpst)) { |
| 271 | float_raise(float_flag_invalid, fpst); |
| 272 | if (!get_default_nan_mode(fpst)) { |
| 273 | nan = float16_silence_nan(a, fpst); |
| 274 | } |
| 275 | } |
| 276 | if (get_default_nan_mode(fpst)) { |
| 277 | nan = float16_default_nan(fpst); |
| 278 | } |
| 279 | return nan; |
| 280 | } |
| 281 | |
| 282 | a = float16_squash_input_denormal(a, fpst); |
| 283 | |
| 284 | val16 = float16_val(a); |
| 285 | sbit = 0x8000 & val16; |
| 286 | exp = extract32(val16, 10, 5); |
| 287 | |
| 288 | if (exp == 0) { |
| 289 | return make_float16(deposit32(sbit, 10, 5, 0x1e)); |
| 290 | } else { |
| 291 | return make_float16(deposit32(sbit, 10, 5, ~exp)); |
| 292 | } |
| 293 | } |
| 294 | |
| 295 | float32 HELPER(frecpx_f32)(float32 a, float_status *fpst) |
| 296 | { |
| 297 | uint32_t val32, sbit; |
| 298 | int32_t exp; |
| 299 | |
| 300 | if (float32_is_any_nan(a)) { |
| 301 | float32 nan = a; |
| 302 | if (float32_is_signaling_nan(a, fpst)) { |
| 303 | float_raise(float_flag_invalid, fpst); |
| 304 | if (!get_default_nan_mode(fpst)) { |
| 305 | nan = float32_silence_nan(a, fpst); |
| 306 | } |
| 307 | } |
| 308 | if (get_default_nan_mode(fpst)) { |
| 309 | nan = float32_default_nan(fpst); |
| 310 | } |
| 311 | return nan; |
| 312 | } |
| 313 | |
| 314 | a = float32_squash_input_denormal(a, fpst); |
| 315 | |
| 316 | val32 = float32_val(a); |
| 317 | sbit = 0x80000000ULL & val32; |
| 318 | exp = extract32(val32, 23, 8); |
| 319 | |
| 320 | if (exp == 0) { |
| 321 | return make_float32(sbit | (0xfe << 23)); |
| 322 | } else { |
| 323 | return make_float32(sbit | (~exp & 0xff) << 23); |
| 324 | } |
| 325 | } |
| 326 | |
| 327 | float64 HELPER(frecpx_f64)(float64 a, float_status *fpst) |
| 328 | { |
| 329 | uint64_t val64, sbit; |
| 330 | int64_t exp; |
| 331 | |
| 332 | if (float64_is_any_nan(a)) { |
| 333 | float64 nan = a; |
| 334 | if (float64_is_signaling_nan(a, fpst)) { |
| 335 | float_raise(float_flag_invalid, fpst); |
| 336 | if (!get_default_nan_mode(fpst)) { |
| 337 | nan = float64_silence_nan(a, fpst); |
| 338 | } |
| 339 | } |
| 340 | if (get_default_nan_mode(fpst)) { |
| 341 | nan = float64_default_nan(fpst); |
| 342 | } |
| 343 | return nan; |
| 344 | } |
| 345 | |
| 346 | a = float64_squash_input_denormal(a, fpst); |
| 347 | |
| 348 | val64 = float64_val(a); |
| 349 | sbit = 0x8000000000000000ULL & val64; |
| 350 | exp = extract64(float64_val(a), 52, 11); |
| 351 | |
| 352 | if (exp == 0) { |
| 353 | return make_float64(sbit | (0x7feULL << 52)); |
| 354 | } else { |
| 355 | return make_float64(sbit | (~exp & 0x7ffULL) << 52); |
| 356 | } |
| 357 | } |
| 358 | |
| 359 | float32 HELPER(fcvtx_f64_to_f32)(float64 a, float_status *fpst) |
| 360 | { |
| 361 | float32 r; |
| 362 | int old = get_float_rounding_mode(fpst); |
| 363 | |
| 364 | set_float_rounding_mode(float_round_to_odd, fpst); |
| 365 | r = float64_to_float32(a, fpst); |
| 366 | set_float_rounding_mode(old, fpst); |
| 367 | return r; |
| 368 | } |
| 369 | |
| 370 | /* |
| 371 | * AH=1 min/max have some odd special cases: |
| 372 | * comparing two zeroes (regardless of sign), (NaN, anything), |
| 373 | * or (anything, NaN) should return the second argument (possibly |
| 374 | * squashed to zero). |
| 375 | * Also, denormal outputs are not squashed to zero regardless of FZ or FZ16. |
| 376 | */ |
| 377 | #define AH_MINMAX_HELPER(NAME, CTYPE, FLOATTYPE, MINMAX) \ |
| 378 | CTYPE HELPER(NAME)(CTYPE a, CTYPE b, float_status *fpst) \ |
| 379 | { \ |
| 380 | bool save; \ |
| 381 | CTYPE r; \ |
| 382 | a = FLOATTYPE ## _squash_input_denormal(a, fpst); \ |
| 383 | b = FLOATTYPE ## _squash_input_denormal(b, fpst); \ |
| 384 | if (FLOATTYPE ## _is_zero(a) && FLOATTYPE ## _is_zero(b)) { \ |
| 385 | return b; \ |
| 386 | } \ |
| 387 | if (FLOATTYPE ## _is_any_nan(a) || \ |
| 388 | FLOATTYPE ## _is_any_nan(b)) { \ |
| 389 | float_raise(float_flag_invalid, fpst); \ |
| 390 | return b; \ |
| 391 | } \ |
| 392 | save = get_flush_to_zero(fpst); \ |
| 393 | set_flush_to_zero(false, fpst); \ |
| 394 | r = FLOATTYPE ## _ ## MINMAX(a, b, fpst); \ |
| 395 | set_flush_to_zero(save, fpst); \ |
| 396 | return r; \ |
| 397 | } |
| 398 | |
| 399 | AH_MINMAX_HELPER(vfp_ah_minh, dh_ctype_f16, float16, min) |
| 400 | AH_MINMAX_HELPER(vfp_ah_mins, float32, float32, min) |
| 401 | AH_MINMAX_HELPER(vfp_ah_mind, float64, float64, min) |
| 402 | AH_MINMAX_HELPER(vfp_ah_maxh, dh_ctype_f16, float16, max) |
| 403 | AH_MINMAX_HELPER(vfp_ah_maxs, float32, float32, max) |
| 404 | AH_MINMAX_HELPER(vfp_ah_maxd, float64, float64, max) |
| 405 | AH_MINMAX_HELPER(sme2_ah_fmax_b16, bfloat16, bfloat16, max) |
| 406 | AH_MINMAX_HELPER(sme2_ah_fmin_b16, bfloat16, bfloat16, min) |
| 407 | |
| 408 | /* 64-bit versions of the CRC helpers. Note that although the operation |
| 409 | * (and the prototypes of crc32c() and crc32() mean that only the bottom |
| 410 | * 32 bits of the accumulator and result are used, we pass and return |
| 411 | * uint64_t for convenience of the generated code. Unlike the 32-bit |
| 412 | * instruction set versions, val may genuinely have 64 bits of data in it. |
| 413 | * The upper bytes of val (above the number specified by 'bytes') must have |
| 414 | * been zeroed out by the caller. |
| 415 | */ |
| 416 | uint64_t HELPER(crc32_64)(uint64_t acc, uint64_t val, uint32_t bytes) |
| 417 | { |
| 418 | uint8_t buf[8]; |
| 419 | |
| 420 | stq_le_p(buf, val); |
| 421 | |
| 422 | /* zlib crc32 converts the accumulator and output to one's complement. */ |
| 423 | return crc32(acc ^ 0xffffffff, buf, bytes) ^ 0xffffffff; |
| 424 | } |
| 425 | |
| 426 | uint64_t HELPER(crc32c_64)(uint64_t acc, uint64_t val, uint32_t bytes) |
| 427 | { |
| 428 | uint8_t buf[8]; |
| 429 | |
| 430 | stq_le_p(buf, val); |
| 431 | |
| 432 | /* Linux crc32c converts the output to one's complement. */ |
| 433 | return crc32c(acc, buf, bytes) ^ 0xffffffff; |
| 434 | } |
| 435 | |
| 436 | /* |
| 437 | * AdvSIMD half-precision |
| 438 | */ |
| 439 | |
| 440 | #define ADVSIMD_HELPER(name, suffix) HELPER(glue(glue(advsimd_, name), suffix)) |
| 441 | |
| 442 | #define ADVSIMD_HALFOP(name) \ |
| 443 | uint32_t ADVSIMD_HELPER(name, h)(uint32_t a, uint32_t b, float_status *fpst) \ |
| 444 | { \ |
| 445 | return float16_ ## name(a, b, fpst); \ |
| 446 | } |
| 447 | |
| 448 | #define ADVSIMD_TWOHALFOP(name) \ |
| 449 | uint32_t ADVSIMD_HELPER(name, 2h)(uint32_t two_a, uint32_t two_b, \ |
| 450 | float_status *fpst) \ |
| 451 | { \ |
| 452 | float16 a1, a2, b1, b2; \ |
| 453 | uint32_t r1, r2; \ |
| 454 | a1 = extract32(two_a, 0, 16); \ |
| 455 | a2 = extract32(two_a, 16, 16); \ |
| 456 | b1 = extract32(two_b, 0, 16); \ |
| 457 | b2 = extract32(two_b, 16, 16); \ |
| 458 | r1 = float16_ ## name(a1, b1, fpst); \ |
| 459 | r2 = float16_ ## name(a2, b2, fpst); \ |
| 460 | return deposit32(r1, 16, 16, r2); \ |
| 461 | } |
| 462 | |
| 463 | ADVSIMD_TWOHALFOP(add) |
| 464 | ADVSIMD_TWOHALFOP(sub) |
| 465 | ADVSIMD_TWOHALFOP(mul) |
| 466 | ADVSIMD_TWOHALFOP(div) |
| 467 | ADVSIMD_TWOHALFOP(min) |
| 468 | ADVSIMD_TWOHALFOP(max) |
| 469 | ADVSIMD_TWOHALFOP(minnum) |
| 470 | ADVSIMD_TWOHALFOP(maxnum) |
| 471 | |
| 472 | /* Data processing - scalar floating-point and advanced SIMD */ |
| 473 | static float16 float16_mulx(float16 a, float16 b, float_status *fpst) |
| 474 | { |
| 475 | a = float16_squash_input_denormal(a, fpst); |
| 476 | b = float16_squash_input_denormal(b, fpst); |
| 477 | |
| 478 | if ((float16_is_zero(a) && float16_is_infinity(b)) || |
| 479 | (float16_is_infinity(a) && float16_is_zero(b))) { |
| 480 | /* 2.0 with the sign bit set to sign(A) XOR sign(B) */ |
| 481 | return make_float16((1U << 14) | |
| 482 | ((float16_val(a) ^ float16_val(b)) & (1U << 15))); |
| 483 | } |
| 484 | return float16_mul(a, b, fpst); |
| 485 | } |
| 486 | |
| 487 | ADVSIMD_HALFOP(mulx) |
| 488 | ADVSIMD_TWOHALFOP(mulx) |
| 489 | |
| 490 | /* fused multiply-accumulate */ |
| 491 | uint32_t HELPER(advsimd_muladdh)(uint32_t a, uint32_t b, uint32_t c, |
| 492 | float_status *fpst) |
| 493 | { |
| 494 | return float16_muladd(a, b, c, 0, fpst); |
| 495 | } |
| 496 | |
| 497 | uint32_t HELPER(advsimd_muladd2h)(uint32_t two_a, uint32_t two_b, |
| 498 | uint32_t two_c, float_status *fpst) |
| 499 | { |
| 500 | float16 a1, a2, b1, b2, c1, c2; |
| 501 | uint32_t r1, r2; |
| 502 | a1 = extract32(two_a, 0, 16); |
| 503 | a2 = extract32(two_a, 16, 16); |
| 504 | b1 = extract32(two_b, 0, 16); |
| 505 | b2 = extract32(two_b, 16, 16); |
| 506 | c1 = extract32(two_c, 0, 16); |
| 507 | c2 = extract32(two_c, 16, 16); |
| 508 | r1 = float16_muladd(a1, b1, c1, 0, fpst); |
| 509 | r2 = float16_muladd(a2, b2, c2, 0, fpst); |
| 510 | return deposit32(r1, 16, 16, r2); |
| 511 | } |
| 512 | |
| 513 | /* |
| 514 | * Floating point comparisons produce an integer result. Softfloat |
| 515 | * routines return float_relation types which we convert to the 0/-1 |
| 516 | * Neon requires. |
| 517 | */ |
| 518 | |
| 519 | #define ADVSIMD_CMPRES(test) (test) ? 0xffff : 0 |
| 520 | |
| 521 | uint32_t HELPER(advsimd_ceq_f16)(uint32_t a, uint32_t b, float_status *fpst) |
| 522 | { |
| 523 | int compare = float16_compare_quiet(a, b, fpst); |
| 524 | return ADVSIMD_CMPRES(compare == float_relation_equal); |
| 525 | } |
| 526 | |
| 527 | uint32_t HELPER(advsimd_cge_f16)(uint32_t a, uint32_t b, float_status *fpst) |
| 528 | { |
| 529 | int compare = float16_compare(a, b, fpst); |
| 530 | return ADVSIMD_CMPRES(compare == float_relation_greater || |
| 531 | compare == float_relation_equal); |
| 532 | } |
| 533 | |
| 534 | uint32_t HELPER(advsimd_cgt_f16)(uint32_t a, uint32_t b, float_status *fpst) |
| 535 | { |
| 536 | int compare = float16_compare(a, b, fpst); |
| 537 | return ADVSIMD_CMPRES(compare == float_relation_greater); |
| 538 | } |
| 539 | |
| 540 | uint32_t HELPER(advsimd_acge_f16)(uint32_t a, uint32_t b, float_status *fpst) |
| 541 | { |
| 542 | float16 f0 = float16_abs(a); |
| 543 | float16 f1 = float16_abs(b); |
| 544 | int compare = float16_compare(f0, f1, fpst); |
| 545 | return ADVSIMD_CMPRES(compare == float_relation_greater || |
| 546 | compare == float_relation_equal); |
| 547 | } |
| 548 | |
| 549 | uint32_t HELPER(advsimd_acgt_f16)(uint32_t a, uint32_t b, float_status *fpst) |
| 550 | { |
| 551 | float16 f0 = float16_abs(a); |
| 552 | float16 f1 = float16_abs(b); |
| 553 | int compare = float16_compare(f0, f1, fpst); |
| 554 | return ADVSIMD_CMPRES(compare == float_relation_greater); |
| 555 | } |
| 556 | |
| 557 | /* round to integral */ |
| 558 | uint32_t HELPER(advsimd_rinth_exact)(uint32_t x, float_status *fp_status) |
| 559 | { |
| 560 | return float16_round_to_int(x, fp_status); |
| 561 | } |
| 562 | |
| 563 | uint32_t HELPER(advsimd_rinth)(uint32_t x, float_status *fp_status) |
| 564 | { |
| 565 | int old_flags = get_float_exception_flags(fp_status), new_flags; |
| 566 | float16 ret; |
| 567 | |
| 568 | ret = float16_round_to_int(x, fp_status); |
| 569 | |
| 570 | /* Suppress any inexact exceptions the conversion produced */ |
| 571 | if (!(old_flags & float_flag_inexact)) { |
| 572 | new_flags = get_float_exception_flags(fp_status); |
| 573 | set_float_exception_flags(new_flags & ~float_flag_inexact, fp_status); |
| 574 | } |
| 575 | |
| 576 | return ret; |
| 577 | } |
| 578 | |
| 579 | #ifndef CONFIG_USER_ONLY |
| 580 | static int el_from_spsr(uint32_t spsr) |
| 581 | { |
| 582 | /* Return the exception level that this SPSR is requesting a return to, |
| 583 | * or -1 if it is invalid (an illegal return) |
| 584 | */ |
| 585 | if (spsr & PSTATE_nRW) { |
| 586 | switch (spsr & CPSR_M) { |
| 587 | case ARM_CPU_MODE_USR: |
| 588 | return 0; |
| 589 | case ARM_CPU_MODE_HYP: |
| 590 | return 2; |
| 591 | case ARM_CPU_MODE_FIQ: |
| 592 | case ARM_CPU_MODE_IRQ: |
| 593 | case ARM_CPU_MODE_SVC: |
| 594 | case ARM_CPU_MODE_ABT: |
| 595 | case ARM_CPU_MODE_UND: |
| 596 | case ARM_CPU_MODE_SYS: |
| 597 | return 1; |
| 598 | case ARM_CPU_MODE_MON: |
| 599 | /* Returning to Mon from AArch64 is never possible, |
| 600 | * so this is an illegal return. |
| 601 | */ |
| 602 | default: |
| 603 | return -1; |
| 604 | } |
| 605 | } else { |
| 606 | if (extract32(spsr, 1, 1)) { |
| 607 | /* Return with reserved M[1] bit set */ |
| 608 | return -1; |
| 609 | } |
| 610 | if (extract32(spsr, 0, 4) == 1) { |
| 611 | /* return to EL0 with M[0] bit set */ |
| 612 | return -1; |
| 613 | } |
| 614 | return extract32(spsr, 2, 2); |
| 615 | } |
| 616 | } |
| 617 | |
| 618 | void HELPER(exception_return)(CPUARMState *env, uint64_t new_pc) |
| 619 | { |
| 620 | ARMCPU *cpu = env_archcpu(env); |
| 621 | int cur_el = arm_current_el(env); |
| 622 | unsigned int spsr_idx = aarch64_banked_spsr_index(cur_el); |
| 623 | uint64_t spsr = env->banked_spsr[spsr_idx]; |
| 624 | int new_el; |
| 625 | bool return_to_aa64 = (spsr & PSTATE_nRW) == 0; |
| 626 | |
| 627 | aarch64_save_sp(env, cur_el); |
| 628 | |
| 629 | arm_clear_exclusive(env); |
| 630 | |
| 631 | /* We must squash the PSTATE.SS bit to zero unless both of the |
| 632 | * following hold: |
| 633 | * 1. debug exceptions are currently disabled |
| 634 | * 2. singlestep will be active in the EL we return to |
| 635 | * We check 1 here and 2 after we've done the pstate/cpsr write() to |
| 636 | * transition to the EL we're going to. |
| 637 | */ |
| 638 | if (arm_generate_debug_exceptions(env)) { |
| 639 | spsr &= ~PSTATE_SS; |
| 640 | } |
| 641 | |
| 642 | new_el = el_from_spsr(spsr); |
| 643 | if (new_el == -1) { |
| 644 | goto illegal_return; |
| 645 | } |
| 646 | if (new_el > cur_el || (new_el == 2 && !arm_is_el2_enabled(env))) { |
| 647 | /* Disallow return to an EL which is unimplemented or higher |
| 648 | * than the current one. |
| 649 | */ |
| 650 | goto illegal_return; |
| 651 | } |
| 652 | |
| 653 | /* |
| 654 | * FEAT_RME forbids return from EL3 to a lower exception level |
| 655 | * with an invalid security state. |
| 656 | * We don't need an explicit check for FEAT_RME here because we enforce |
| 657 | * in scr_write() that you can't set the NSE bit without it. |
| 658 | */ |
| 659 | if (cur_el == 3 && new_el < 3 && |
| 660 | (env->cp15.scr_el3 & (SCR_NS | SCR_NSE)) == SCR_NSE) { |
| 661 | goto illegal_return; |
| 662 | } |
| 663 | |
| 664 | if (new_el != 0 && arm_el_is_aa64(env, new_el) != return_to_aa64) { |
| 665 | /* Return to an EL which is configured for a different register width */ |
| 666 | goto illegal_return; |
| 667 | } |
| 668 | |
| 669 | if (!return_to_aa64 && !cpu_isar_feature(aa64_aa32, cpu)) { |
| 670 | /* Return to AArch32 when CPU is AArch64-only */ |
| 671 | goto illegal_return; |
| 672 | } |
| 673 | |
| 674 | if (new_el == 1 && (arm_hcr_el2_eff(env) & HCR_TGE)) { |
| 675 | goto illegal_return; |
| 676 | } |
| 677 | |
| 678 | /* |
| 679 | * If GetCurrentEXLOCKEN, the exception return path must use GCSPOPCX, |
| 680 | * which will set PSTATE.EXLOCK. We need not explicitly check FEAT_GCS, |
| 681 | * because GCSCR_ELx cannot be set without it. |
| 682 | */ |
| 683 | if (new_el == cur_el && |
| 684 | (env->cp15.gcscr_el[cur_el] & GCSCR_EXLOCKEN) && |
| 685 | !(env->pstate & PSTATE_EXLOCK)) { |
| 686 | goto illegal_return; |
| 687 | } |
| 688 | |
| 689 | bql_lock(); |
| 690 | arm_call_pre_el_change_hook(cpu); |
| 691 | bql_unlock(); |
| 692 | |
| 693 | if (!return_to_aa64) { |
| 694 | env->aarch64 = false; |
| 695 | /* We do a raw CPSR write because aarch64_sync_64_to_32() |
| 696 | * will sort the register banks out for us, and we've already |
| 697 | * caught all the bad-mode cases in el_from_spsr(). |
| 698 | */ |
| 699 | cpsr_write_from_spsr_elx(env, spsr); |
| 700 | if (!arm_singlestep_active(env)) { |
| 701 | env->pstate &= ~PSTATE_SS; |
| 702 | } |
| 703 | aarch64_sync_64_to_32(env); |
| 704 | |
| 705 | if (spsr & CPSR_T) { |
| 706 | env->regs[15] = new_pc & ~0x1; |
| 707 | } else { |
| 708 | env->regs[15] = new_pc & ~0x3; |
| 709 | } |
| 710 | helper_rebuild_hflags_a32(env, new_el); |
| 711 | qemu_log_mask(CPU_LOG_INT, "Exception return from AArch64 EL%d to " |
| 712 | "AArch32 EL%d PC 0x%" PRIx32 "\n", |
| 713 | cur_el, new_el, env->regs[15]); |
| 714 | } else { |
| 715 | int tbii; |
| 716 | |
| 717 | env->aarch64 = true; |
| 718 | spsr &= aarch64_pstate_valid_mask(&cpu->isar); |
| 719 | pstate_write(env, spsr); |
| 720 | if (!arm_singlestep_active(env)) { |
| 721 | env->pstate &= ~PSTATE_SS; |
| 722 | } |
| 723 | aarch64_restore_sp(env, new_el); |
| 724 | helper_rebuild_hflags_a64(env, new_el); |
| 725 | |
| 726 | /* |
| 727 | * Apply TBI to the exception return address. We had to delay this |
| 728 | * until after we selected the new EL, so that we could select the |
| 729 | * correct TBI+TBID bits. This is made easier by waiting until after |
| 730 | * the hflags rebuild, since we can pull the composite TBII field |
| 731 | * from there. |
| 732 | */ |
| 733 | tbii = EX_TBFLAG_A64(env->hflags, TBII); |
| 734 | if ((tbii >> extract64(new_pc, 55, 1)) & 1) { |
| 735 | /* TBI is enabled. */ |
| 736 | int core_mmu_idx = arm_env_mmu_index(env); |
| 737 | if (regime_has_2_ranges(core_to_aa64_mmu_idx(core_mmu_idx))) { |
| 738 | new_pc = sextract64(new_pc, 0, 56); |
| 739 | } else { |
| 740 | new_pc = extract64(new_pc, 0, 56); |
| 741 | } |
| 742 | } |
| 743 | env->pc = new_pc; |
| 744 | |
| 745 | qemu_log_mask(CPU_LOG_INT, "Exception return from AArch64 EL%d to " |
| 746 | "AArch64 EL%d PC 0x%" PRIx64 "\n", |
| 747 | cur_el, new_el, env->pc); |
| 748 | } |
| 749 | |
| 750 | /* |
| 751 | * Note that cur_el can never be 0. If new_el is 0, then |
| 752 | * el0_a64 is return_to_aa64, else el0_a64 is ignored. |
| 753 | */ |
| 754 | aarch64_sve_change_el(env, cur_el, new_el, return_to_aa64); |
| 755 | |
| 756 | bql_lock(); |
| 757 | arm_call_el_change_hook(cpu); |
| 758 | bql_unlock(); |
| 759 | |
| 760 | return; |
| 761 | |
| 762 | illegal_return: |
| 763 | /* Illegal return events of various kinds have architecturally |
| 764 | * mandated behaviour: |
| 765 | * restore NZCV and DAIF from SPSR_ELx |
| 766 | * set PSTATE.IL |
| 767 | * restore PC from ELR_ELx |
| 768 | * no change to exception level, execution state or stack pointer |
| 769 | */ |
| 770 | env->pstate |= PSTATE_IL; |
| 771 | env->pc = new_pc; |
| 772 | spsr &= PSTATE_NZCV | PSTATE_DAIF | PSTATE_ALLINT; |
| 773 | spsr |= pstate_read(env) & ~(PSTATE_NZCV | PSTATE_DAIF | PSTATE_ALLINT); |
| 774 | pstate_write(env, spsr); |
| 775 | if (!arm_singlestep_active(env)) { |
| 776 | env->pstate &= ~PSTATE_SS; |
| 777 | } |
| 778 | helper_rebuild_hflags_a64(env, cur_el); |
| 779 | qemu_log_mask(LOG_GUEST_ERROR, "Illegal exception return at EL%d: " |
| 780 | "resuming execution at 0x%" PRIx64 "\n", cur_el, env->pc); |
| 781 | } |
| 782 | #endif /* !CONFIG_USER_ONLY */ |
| 783 | |
| 784 | void HELPER(dc_zva)(CPUARMState *env, uint64_t vaddr_in) |
| 785 | { |
| 786 | uintptr_t ra = GETPC(); |
| 787 | |
| 788 | /* |
| 789 | * Implement DC ZVA, which zeroes a fixed-length block of memory. |
| 790 | * Note that we do not implement the (architecturally mandated) |
| 791 | * alignment fault for attempts to use this on Device memory |
| 792 | * (which matches the usual QEMU behaviour of not implementing either |
| 793 | * alignment faults or any memory attribute handling). |
| 794 | */ |
| 795 | int blocklen = 4 << get_dczid_bs(env_archcpu(env)); |
| 796 | uint64_t vaddr = vaddr_in & ~(blocklen - 1); |
| 797 | int mmu_idx = arm_env_mmu_index(env); |
| 798 | void *mem; |
| 799 | |
| 800 | /* |
| 801 | * Trapless lookup. In addition to actual invalid page, may |
| 802 | * return NULL for I/O, watchpoints, clean pages, etc. |
| 803 | */ |
| 804 | mem = tlb_vaddr_to_host(env, vaddr, MMU_DATA_STORE, mmu_idx); |
| 805 | |
| 806 | #ifndef CONFIG_USER_ONLY |
| 807 | if (unlikely(!mem)) { |
| 808 | /* |
| 809 | * Trap if accessing an invalid page. DC_ZVA requires that we supply |
| 810 | * the original pointer for an invalid page. But watchpoints require |
| 811 | * that we probe the actual space. So do both. |
| 812 | */ |
| 813 | (void) probe_write(env, vaddr_in, 1, mmu_idx, ra); |
| 814 | mem = probe_write(env, vaddr, blocklen, mmu_idx, ra); |
| 815 | |
| 816 | if (unlikely(!mem)) { |
| 817 | /* |
| 818 | * The only remaining reason for mem == NULL is I/O. |
| 819 | * Just do a series of byte writes as the architecture demands. |
| 820 | */ |
| 821 | for (int i = 0; i < blocklen; i++) { |
| 822 | cpu_stb_mmuidx_ra(env, vaddr + i, 0, mmu_idx, ra); |
| 823 | } |
| 824 | return; |
| 825 | } |
| 826 | } |
| 827 | #endif |
| 828 | |
| 829 | set_helper_retaddr(ra); |
| 830 | memset(mem, 0, blocklen); |
| 831 | clear_helper_retaddr(); |
| 832 | } |
| 833 | |
| 834 | void HELPER(arm_unaligned_access)(CPUARMState *env, uint64_t addr, |
| 835 | uint32_t access_type, uint32_t mmu_idx) |
| 836 | { |
| 837 | arm_cpu_do_unaligned_access(env_cpu(env), addr, access_type, |
| 838 | mmu_idx, GETPC()); |
| 839 | } |
| 840 | |
| 841 | /* Memory operations (memset, memmove, memcpy) */ |
| 842 | |
| 843 | /* |
| 844 | * Return true if the CPY* and SET* insns can execute; compare |
| 845 | * pseudocode CheckMOPSEnabled(), though we refactor it a little. |
| 846 | */ |
| 847 | static bool mops_enabled(CPUARMState *env) |
| 848 | { |
| 849 | int el = arm_current_el(env); |
| 850 | |
| 851 | if (el < 2 && |
| 852 | (arm_hcr_el2_eff(env) & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE) && |
| 853 | !(arm_hcrx_el2_eff(env) & HCRX_MSCEN)) { |
| 854 | return false; |
| 855 | } |
| 856 | |
| 857 | if (el == 0) { |
| 858 | if (!el_is_in_host(env, 0)) { |
| 859 | return env->cp15.sctlr_el[1] & SCTLR_MSCEN; |
| 860 | } else { |
| 861 | return env->cp15.sctlr_el[2] & SCTLR_MSCEN; |
| 862 | } |
| 863 | } |
| 864 | return true; |
| 865 | } |
| 866 | |
| 867 | static void check_mops_enabled(CPUARMState *env, uintptr_t ra) |
| 868 | { |
| 869 | if (!mops_enabled(env)) { |
| 870 | raise_exception_ra(env, EXCP_UDEF, syn_uncategorized(), |
| 871 | exception_target_el(env), ra); |
| 872 | } |
| 873 | } |
| 874 | |
| 875 | /* |
| 876 | * Return the target exception level for an exception due |
| 877 | * to mismatched arguments in a FEAT_MOPS copy or set. |
| 878 | * Compare pseudocode MismatchedCpySetTargetEL() |
| 879 | */ |
| 880 | static int mops_mismatch_exception_target_el(CPUARMState *env) |
| 881 | { |
| 882 | int el = arm_current_el(env); |
| 883 | |
| 884 | if (el > 1) { |
| 885 | return el; |
| 886 | } |
| 887 | if (el == 0 && (arm_hcr_el2_eff(env) & HCR_TGE)) { |
| 888 | return 2; |
| 889 | } |
| 890 | if (el == 1 && (arm_hcrx_el2_eff(env) & HCRX_MCE2)) { |
| 891 | return 2; |
| 892 | } |
| 893 | return 1; |
| 894 | } |
| 895 | |
| 896 | /* |
| 897 | * Check whether an M or E instruction was executed with a CF value |
| 898 | * indicating the wrong option for this implementation. |
| 899 | * Assumes we are always Option A. |
| 900 | */ |
| 901 | static void check_mops_wrong_option(CPUARMState *env, uint32_t syndrome, |
| 902 | uintptr_t ra) |
| 903 | { |
| 904 | if (env->CF != 0) { |
| 905 | syndrome |= 1 << 17; /* Set the wrong-option bit */ |
| 906 | raise_exception_ra(env, EXCP_UDEF, syndrome, |
| 907 | mops_mismatch_exception_target_el(env), ra); |
| 908 | } |
| 909 | } |
| 910 | |
| 911 | /* |
| 912 | * Return the maximum number of bytes we can transfer starting at addr |
| 913 | * without crossing a page boundary. |
| 914 | */ |
| 915 | static uint64_t page_limit(uint64_t addr) |
| 916 | { |
| 917 | return TARGET_PAGE_ALIGN(addr + 1) - addr; |
| 918 | } |
| 919 | |
| 920 | /* |
| 921 | * Return the number of bytes we can copy starting from addr and working |
| 922 | * backwards without crossing a page boundary. |
| 923 | */ |
| 924 | static uint64_t page_limit_rev(uint64_t addr) |
| 925 | { |
| 926 | return (addr & ~TARGET_PAGE_MASK) + 1; |
| 927 | } |
| 928 | |
| 929 | /* |
| 930 | * Perform part of a memory set on an area of guest memory starting at |
| 931 | * toaddr (a dirty address) and extending for setsize bytes. |
| 932 | * |
| 933 | * Returns the number of bytes actually set, which might be less than |
| 934 | * setsize; the caller should loop until the whole set has been done. |
| 935 | * The caller should ensure that the guest registers are correct |
| 936 | * for the possibility that the first byte of the set encounters |
| 937 | * an exception or watchpoint. We guarantee not to take any faults |
| 938 | * for bytes other than the first. |
| 939 | */ |
| 940 | static uint64_t set_step(CPUARMState *env, uint64_t toaddr, |
| 941 | uint64_t setsize, uint32_t data, int memidx, |
| 942 | uint32_t *mtedesc, uintptr_t ra) |
| 943 | { |
| 944 | void *mem; |
| 945 | |
| 946 | setsize = MIN(setsize, page_limit(toaddr)); |
| 947 | if (*mtedesc) { |
| 948 | uint64_t mtesize = mte_mops_probe(env, toaddr, setsize, *mtedesc); |
| 949 | if (mtesize == 0) { |
| 950 | /* Trap, or not. All CPU state is up to date */ |
| 951 | mte_check_fail(env, *mtedesc, toaddr, ra); |
| 952 | /* Continue, with no further MTE checks required */ |
| 953 | *mtedesc = 0; |
| 954 | } else { |
| 955 | /* Advance to the end, or to the tag mismatch */ |
| 956 | setsize = MIN(setsize, mtesize); |
| 957 | } |
| 958 | } |
| 959 | |
| 960 | toaddr = useronly_clean_ptr(toaddr); |
| 961 | /* |
| 962 | * Trapless lookup: returns NULL for invalid page, I/O, |
| 963 | * watchpoints, clean pages, etc. |
| 964 | */ |
| 965 | mem = tlb_vaddr_to_host(env, toaddr, MMU_DATA_STORE, memidx); |
| 966 | |
| 967 | #ifndef CONFIG_USER_ONLY |
| 968 | if (unlikely(!mem)) { |
| 969 | /* |
| 970 | * Slow-path: just do one byte write. This will handle the |
| 971 | * watchpoint, invalid page, etc handling correctly. |
| 972 | * For clean code pages, the next iteration will see |
| 973 | * the page dirty and will use the fast path. |
| 974 | */ |
| 975 | cpu_stb_mmuidx_ra(env, toaddr, data, memidx, ra); |
| 976 | return 1; |
| 977 | } |
| 978 | #endif |
| 979 | /* Easy case: just memset the host memory */ |
| 980 | set_helper_retaddr(ra); |
| 981 | memset(mem, data, setsize); |
| 982 | clear_helper_retaddr(); |
| 983 | return setsize; |
| 984 | } |
| 985 | |
| 986 | /* |
| 987 | * Similar, but setting tags. The architecture requires us to do this |
| 988 | * in 16-byte chunks. SETP accesses are not tag checked; they set |
| 989 | * the tags. |
| 990 | */ |
| 991 | static uint64_t set_step_tags(CPUARMState *env, uint64_t toaddr, |
| 992 | uint64_t setsize, uint32_t data, int memidx, |
| 993 | uint32_t *mtedesc, uintptr_t ra) |
| 994 | { |
| 995 | void *mem; |
| 996 | uint64_t cleanaddr; |
| 997 | |
| 998 | setsize = MIN(setsize, page_limit(toaddr)); |
| 999 | |
| 1000 | cleanaddr = useronly_clean_ptr(toaddr); |
| 1001 | /* |
| 1002 | * Trapless lookup: returns NULL for invalid page, I/O, |
| 1003 | * watchpoints, clean pages, etc. |
| 1004 | */ |
| 1005 | mem = tlb_vaddr_to_host(env, cleanaddr, MMU_DATA_STORE, memidx); |
| 1006 | |
| 1007 | #ifndef CONFIG_USER_ONLY |
| 1008 | if (unlikely(!mem)) { |
| 1009 | /* |
| 1010 | * Slow-path: just do one write. This will handle the |
| 1011 | * watchpoint, invalid page, etc handling correctly. |
| 1012 | * The architecture requires that we do 16 bytes at a time, |
| 1013 | * and we know both ptr and size are 16 byte aligned. |
| 1014 | * For clean code pages, the next iteration will see |
| 1015 | * the page dirty and will use the fast path. |
| 1016 | */ |
| 1017 | uint64_t repldata = data * 0x0101010101010101ULL; |
| 1018 | MemOpIdx oi16 = make_memop_idx(MO_128, memidx); |
| 1019 | cpu_st16_mmu(env, toaddr, int128_make128(repldata, repldata), oi16, ra); |
| 1020 | mte_mops_set_tags(env, toaddr, 16, *mtedesc); |
| 1021 | return 16; |
| 1022 | } |
| 1023 | #endif |
| 1024 | /* Easy case: just memset the host memory */ |
| 1025 | set_helper_retaddr(ra); |
| 1026 | memset(mem, data, setsize); |
| 1027 | clear_helper_retaddr(); |
| 1028 | mte_mops_set_tags(env, toaddr, setsize, *mtedesc); |
| 1029 | return setsize; |
| 1030 | } |
| 1031 | |
| 1032 | typedef uint64_t StepFn(CPUARMState *env, uint64_t toaddr, |
| 1033 | uint64_t setsize, uint32_t data, |
| 1034 | int memidx, uint32_t *mtedesc, uintptr_t ra); |
| 1035 | |
| 1036 | /* Extract register numbers from a MOPS exception syndrome value */ |
| 1037 | static int mops_destreg(uint32_t syndrome) |
| 1038 | { |
| 1039 | return extract32(syndrome, 10, 5); |
| 1040 | } |
| 1041 | |
| 1042 | static int mops_srcreg(uint32_t syndrome) |
| 1043 | { |
| 1044 | return extract32(syndrome, 5, 5); |
| 1045 | } |
| 1046 | |
| 1047 | static int mops_sizereg(uint32_t syndrome) |
| 1048 | { |
| 1049 | return extract32(syndrome, 0, 5); |
| 1050 | } |
| 1051 | |
| 1052 | /* |
| 1053 | * Return true if the TCMA, TBI, and MTX bits mean we need to do MTE checks. |
| 1054 | * We only need to do this once per MOPS insn, not for every page. |
| 1055 | */ |
| 1056 | static bool mte_checks_needed(uint64_t ptr, uint32_t desc) |
| 1057 | { |
| 1058 | int bit55 = extract64(ptr, 55, 1); |
| 1059 | |
| 1060 | /* |
| 1061 | * Note that tbi_or_mtx_check() return true for "access checked", but |
| 1062 | * tcma_check() returns true for "access unchecked". |
| 1063 | */ |
| 1064 | if (!tbi_or_mtx_check(desc, bit55)) { |
| 1065 | return false; |
| 1066 | } |
| 1067 | |
| 1068 | return !tcma_check(desc, bit55, allocation_tag_from_addr(ptr)); |
| 1069 | } |
| 1070 | |
| 1071 | /* Take an exception if the SETG addr/size are not granule aligned */ |
| 1072 | static void check_setg_alignment(CPUARMState *env, uint64_t ptr, uint64_t size, |
| 1073 | uint32_t memidx, uintptr_t ra) |
| 1074 | { |
| 1075 | if ((size != 0 && !QEMU_IS_ALIGNED(ptr, TAG_GRANULE)) || |
| 1076 | !QEMU_IS_ALIGNED(size, TAG_GRANULE)) { |
| 1077 | arm_cpu_do_unaligned_access(env_cpu(env), ptr, MMU_DATA_STORE, |
| 1078 | memidx, ra); |
| 1079 | |
| 1080 | } |
| 1081 | } |
| 1082 | |
| 1083 | static uint64_t arm_reg_or_xzr(CPUARMState *env, int reg) |
| 1084 | { |
| 1085 | /* |
| 1086 | * Runtime equivalent of cpu_reg() -- return the CPU register value, |
| 1087 | * for contexts when index 31 means XZR (not SP). |
| 1088 | */ |
| 1089 | return reg == 31 ? 0 : env->xregs[reg]; |
| 1090 | } |
| 1091 | |
| 1092 | /* |
| 1093 | * For the Memory Set operation, our implementation chooses |
| 1094 | * always to use "option A", where we update Xd to the final |
| 1095 | * address in the SETP insn, and set Xn to be -(bytes remaining). |
| 1096 | * On SETM and SETE insns we only need update Xn. |
| 1097 | * |
| 1098 | * @env: CPU |
| 1099 | * @syndrome: syndrome value for mismatch exceptions |
| 1100 | * (also contains the register numbers we need to use) |
| 1101 | * @mtedesc: MTE descriptor word |
| 1102 | * @stepfn: function which does a single part of the set operation |
| 1103 | * @is_setg: true if this is the tag-setting SETG variant |
| 1104 | */ |
| 1105 | static void do_setp(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc, |
| 1106 | StepFn *stepfn, bool is_setg, uintptr_t ra) |
| 1107 | { |
| 1108 | /* Prologue: we choose to do up to the next page boundary */ |
| 1109 | int rd = mops_destreg(syndrome); |
| 1110 | int rs = mops_srcreg(syndrome); |
| 1111 | int rn = mops_sizereg(syndrome); |
| 1112 | uint8_t data = arm_reg_or_xzr(env, rs); |
| 1113 | uint32_t memidx = FIELD_EX32(mtedesc, MTEDESC, MIDX); |
| 1114 | uint64_t toaddr = env->xregs[rd]; |
| 1115 | uint64_t setsize = env->xregs[rn]; |
| 1116 | uint64_t stagesetsize, step; |
| 1117 | |
| 1118 | check_mops_enabled(env, ra); |
| 1119 | |
| 1120 | if (setsize > INT64_MAX) { |
| 1121 | setsize = INT64_MAX; |
| 1122 | if (is_setg) { |
| 1123 | setsize &= ~0xf; |
| 1124 | } |
| 1125 | } |
| 1126 | |
| 1127 | if (unlikely(is_setg)) { |
| 1128 | check_setg_alignment(env, toaddr, setsize, memidx, ra); |
| 1129 | } else if (!mte_checks_needed(toaddr, mtedesc)) { |
| 1130 | mtedesc = 0; |
| 1131 | } |
| 1132 | |
| 1133 | stagesetsize = MIN(setsize, page_limit(toaddr)); |
| 1134 | while (stagesetsize) { |
| 1135 | env->xregs[rd] = toaddr; |
| 1136 | env->xregs[rn] = setsize; |
| 1137 | step = stepfn(env, toaddr, stagesetsize, data, memidx, &mtedesc, ra); |
| 1138 | toaddr += step; |
| 1139 | setsize -= step; |
| 1140 | stagesetsize -= step; |
| 1141 | } |
| 1142 | /* Insn completed, so update registers to the Option A format */ |
| 1143 | env->xregs[rd] = toaddr + setsize; |
| 1144 | env->xregs[rn] = -setsize; |
| 1145 | |
| 1146 | /* Set NZCV = 0000 to indicate we are an Option A implementation */ |
| 1147 | env->NF = 0; |
| 1148 | env->ZF = 1; /* our env->ZF encoding is inverted */ |
| 1149 | env->CF = 0; |
| 1150 | env->VF = 0; |
| 1151 | } |
| 1152 | |
| 1153 | void HELPER(setp)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc) |
| 1154 | { |
| 1155 | do_setp(env, syndrome, mtedesc, set_step, false, GETPC()); |
| 1156 | } |
| 1157 | |
| 1158 | void HELPER(setgp)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc) |
| 1159 | { |
| 1160 | do_setp(env, syndrome, mtedesc, set_step_tags, true, GETPC()); |
| 1161 | } |
| 1162 | |
| 1163 | static void do_setm(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc, |
| 1164 | StepFn *stepfn, bool is_setg, uintptr_t ra) |
| 1165 | { |
| 1166 | /* Main: we choose to do all the full-page chunks */ |
| 1167 | CPUState *cs = env_cpu(env); |
| 1168 | int rd = mops_destreg(syndrome); |
| 1169 | int rs = mops_srcreg(syndrome); |
| 1170 | int rn = mops_sizereg(syndrome); |
| 1171 | uint8_t data = arm_reg_or_xzr(env, rs); |
| 1172 | uint64_t toaddr = env->xregs[rd] + env->xregs[rn]; |
| 1173 | uint64_t setsize = -env->xregs[rn]; |
| 1174 | uint32_t memidx = FIELD_EX32(mtedesc, MTEDESC, MIDX); |
| 1175 | uint64_t step, stagesetsize; |
| 1176 | |
| 1177 | check_mops_enabled(env, ra); |
| 1178 | |
| 1179 | /* |
| 1180 | * We're allowed to NOP out "no data to copy" before the consistency |
| 1181 | * checks; we choose to do so. |
| 1182 | */ |
| 1183 | if (env->xregs[rn] == 0) { |
| 1184 | return; |
| 1185 | } |
| 1186 | |
| 1187 | check_mops_wrong_option(env, syndrome, ra); |
| 1188 | |
| 1189 | /* |
| 1190 | * Our implementation will work fine even if we have an unaligned |
| 1191 | * destination address, and because we update Xn every time around |
| 1192 | * the loop below and the return value from stepfn() may be less |
| 1193 | * than requested, we might find toaddr is unaligned. So we don't |
| 1194 | * have an IMPDEF check for alignment here. |
| 1195 | */ |
| 1196 | |
| 1197 | if (unlikely(is_setg)) { |
| 1198 | check_setg_alignment(env, toaddr, setsize, memidx, ra); |
| 1199 | } else if (!mte_checks_needed(toaddr, mtedesc)) { |
| 1200 | mtedesc = 0; |
| 1201 | } |
| 1202 | |
| 1203 | /* Do the actual memset: we leave the last partial page to SETE */ |
| 1204 | stagesetsize = setsize & TARGET_PAGE_MASK; |
| 1205 | while (stagesetsize > 0) { |
| 1206 | step = stepfn(env, toaddr, stagesetsize, data, memidx, &mtedesc, ra); |
| 1207 | toaddr += step; |
| 1208 | setsize -= step; |
| 1209 | stagesetsize -= step; |
| 1210 | env->xregs[rn] = -setsize; |
| 1211 | if (stagesetsize > 0 && unlikely(cpu_loop_exit_requested(cs))) { |
| 1212 | cpu_loop_exit_restore(cs, ra); |
| 1213 | } |
| 1214 | } |
| 1215 | } |
| 1216 | |
| 1217 | void HELPER(setm)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc) |
| 1218 | { |
| 1219 | do_setm(env, syndrome, mtedesc, set_step, false, GETPC()); |
| 1220 | } |
| 1221 | |
| 1222 | void HELPER(setgm)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc) |
| 1223 | { |
| 1224 | do_setm(env, syndrome, mtedesc, set_step_tags, true, GETPC()); |
| 1225 | } |
| 1226 | |
| 1227 | static void do_sete(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc, |
| 1228 | StepFn *stepfn, bool is_setg, uintptr_t ra) |
| 1229 | { |
| 1230 | /* Epilogue: do the last partial page */ |
| 1231 | int rd = mops_destreg(syndrome); |
| 1232 | int rs = mops_srcreg(syndrome); |
| 1233 | int rn = mops_sizereg(syndrome); |
| 1234 | uint8_t data = arm_reg_or_xzr(env, rs); |
| 1235 | uint64_t toaddr = env->xregs[rd] + env->xregs[rn]; |
| 1236 | uint64_t setsize = -env->xregs[rn]; |
| 1237 | uint32_t memidx = FIELD_EX32(mtedesc, MTEDESC, MIDX); |
| 1238 | uint64_t step; |
| 1239 | |
| 1240 | check_mops_enabled(env, ra); |
| 1241 | |
| 1242 | /* |
| 1243 | * We're allowed to NOP out "no data to copy" before the consistency |
| 1244 | * checks; we choose to do so. |
| 1245 | */ |
| 1246 | if (setsize == 0) { |
| 1247 | return; |
| 1248 | } |
| 1249 | |
| 1250 | check_mops_wrong_option(env, syndrome, ra); |
| 1251 | |
| 1252 | /* |
| 1253 | * Our implementation has no address alignment requirements, but |
| 1254 | * we do want to enforce the "less than a page" size requirement, |
| 1255 | * so we don't need to have the "check for interrupts" here. |
| 1256 | */ |
| 1257 | if (setsize >= TARGET_PAGE_SIZE) { |
| 1258 | raise_exception_ra(env, EXCP_UDEF, syndrome, |
| 1259 | mops_mismatch_exception_target_el(env), ra); |
| 1260 | } |
| 1261 | |
| 1262 | if (unlikely(is_setg)) { |
| 1263 | check_setg_alignment(env, toaddr, setsize, memidx, ra); |
| 1264 | } else if (!mte_checks_needed(toaddr, mtedesc)) { |
| 1265 | mtedesc = 0; |
| 1266 | } |
| 1267 | |
| 1268 | /* Do the actual memset */ |
| 1269 | while (setsize > 0) { |
| 1270 | step = stepfn(env, toaddr, setsize, data, memidx, &mtedesc, ra); |
| 1271 | toaddr += step; |
| 1272 | setsize -= step; |
| 1273 | env->xregs[rn] = -setsize; |
| 1274 | } |
| 1275 | } |
| 1276 | |
| 1277 | void HELPER(sete)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc) |
| 1278 | { |
| 1279 | do_sete(env, syndrome, mtedesc, set_step, false, GETPC()); |
| 1280 | } |
| 1281 | |
| 1282 | void HELPER(setge)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc) |
| 1283 | { |
| 1284 | do_sete(env, syndrome, mtedesc, set_step_tags, true, GETPC()); |
| 1285 | } |
| 1286 | |
| 1287 | /* |
| 1288 | * Perform part of a memory copy from the guest memory at fromaddr |
| 1289 | * and extending for copysize bytes, to the guest memory at |
| 1290 | * toaddr. Both addresses are dirty. |
| 1291 | * |
| 1292 | * Returns the number of bytes actually set, which might be less than |
| 1293 | * copysize; the caller should loop until the whole copy has been done. |
| 1294 | * The caller should ensure that the guest registers are correct |
| 1295 | * for the possibility that the first byte of the copy encounters |
| 1296 | * an exception or watchpoint. We guarantee not to take any faults |
| 1297 | * for bytes other than the first. |
| 1298 | */ |
| 1299 | static uint64_t copy_step(CPUARMState *env, uint64_t toaddr, uint64_t fromaddr, |
| 1300 | uint64_t copysize, int wmemidx, int rmemidx, |
| 1301 | uint32_t *wdesc, uint32_t *rdesc, uintptr_t ra) |
| 1302 | { |
| 1303 | void *rmem; |
| 1304 | void *wmem; |
| 1305 | |
| 1306 | /* Don't cross a page boundary on either source or destination */ |
| 1307 | copysize = MIN(copysize, page_limit(toaddr)); |
| 1308 | copysize = MIN(copysize, page_limit(fromaddr)); |
| 1309 | /* |
| 1310 | * Handle MTE tag checks: either handle the tag mismatch for byte 0, |
| 1311 | * or else copy up to but not including the byte with the mismatch. |
| 1312 | */ |
| 1313 | if (*rdesc) { |
| 1314 | uint64_t mtesize = mte_mops_probe(env, fromaddr, copysize, *rdesc); |
| 1315 | if (mtesize == 0) { |
| 1316 | mte_check_fail(env, *rdesc, fromaddr, ra); |
| 1317 | *rdesc = 0; |
| 1318 | } else { |
| 1319 | copysize = MIN(copysize, mtesize); |
| 1320 | } |
| 1321 | } |
| 1322 | if (*wdesc) { |
| 1323 | uint64_t mtesize = mte_mops_probe(env, toaddr, copysize, *wdesc); |
| 1324 | if (mtesize == 0) { |
| 1325 | mte_check_fail(env, *wdesc, toaddr, ra); |
| 1326 | *wdesc = 0; |
| 1327 | } else { |
| 1328 | copysize = MIN(copysize, mtesize); |
| 1329 | } |
| 1330 | } |
| 1331 | |
| 1332 | toaddr = useronly_clean_ptr(toaddr); |
| 1333 | fromaddr = useronly_clean_ptr(fromaddr); |
| 1334 | /* Trapless lookup of whether we can get a host memory pointer */ |
| 1335 | wmem = tlb_vaddr_to_host(env, toaddr, MMU_DATA_STORE, wmemidx); |
| 1336 | rmem = tlb_vaddr_to_host(env, fromaddr, MMU_DATA_LOAD, rmemidx); |
| 1337 | |
| 1338 | #ifndef CONFIG_USER_ONLY |
| 1339 | /* |
| 1340 | * If we don't have host memory for both source and dest then just |
| 1341 | * do a single byte copy. This will handle watchpoints, invalid pages, |
| 1342 | * etc correctly. For clean code pages, the next iteration will see |
| 1343 | * the page dirty and will use the fast path. |
| 1344 | */ |
| 1345 | if (unlikely(!rmem || !wmem)) { |
| 1346 | uint8_t byte; |
| 1347 | if (rmem) { |
| 1348 | byte = *(uint8_t *)rmem; |
| 1349 | } else { |
| 1350 | byte = cpu_ldub_mmuidx_ra(env, fromaddr, rmemidx, ra); |
| 1351 | } |
| 1352 | if (wmem) { |
| 1353 | *(uint8_t *)wmem = byte; |
| 1354 | } else { |
| 1355 | cpu_stb_mmuidx_ra(env, toaddr, byte, wmemidx, ra); |
| 1356 | } |
| 1357 | return 1; |
| 1358 | } |
| 1359 | #endif |
| 1360 | /* Easy case: just memmove the host memory */ |
| 1361 | set_helper_retaddr(ra); |
| 1362 | memmove(wmem, rmem, copysize); |
| 1363 | clear_helper_retaddr(); |
| 1364 | return copysize; |
| 1365 | } |
| 1366 | |
| 1367 | /* |
| 1368 | * Do part of a backwards memory copy. Here toaddr and fromaddr point |
| 1369 | * to the *last* byte to be copied. |
| 1370 | */ |
| 1371 | static uint64_t copy_step_rev(CPUARMState *env, uint64_t toaddr, |
| 1372 | uint64_t fromaddr, |
| 1373 | uint64_t copysize, int wmemidx, int rmemidx, |
| 1374 | uint32_t *wdesc, uint32_t *rdesc, uintptr_t ra) |
| 1375 | { |
| 1376 | void *rmem; |
| 1377 | void *wmem; |
| 1378 | |
| 1379 | /* Don't cross a page boundary on either source or destination */ |
| 1380 | copysize = MIN(copysize, page_limit_rev(toaddr)); |
| 1381 | copysize = MIN(copysize, page_limit_rev(fromaddr)); |
| 1382 | |
| 1383 | /* |
| 1384 | * Handle MTE tag checks: either handle the tag mismatch for byte 0, |
| 1385 | * or else copy up to but not including the byte with the mismatch. |
| 1386 | */ |
| 1387 | if (*rdesc) { |
| 1388 | uint64_t mtesize = mte_mops_probe_rev(env, fromaddr, copysize, *rdesc); |
| 1389 | if (mtesize == 0) { |
| 1390 | mte_check_fail(env, *rdesc, fromaddr, ra); |
| 1391 | *rdesc = 0; |
| 1392 | } else { |
| 1393 | copysize = MIN(copysize, mtesize); |
| 1394 | } |
| 1395 | } |
| 1396 | if (*wdesc) { |
| 1397 | uint64_t mtesize = mte_mops_probe_rev(env, toaddr, copysize, *wdesc); |
| 1398 | if (mtesize == 0) { |
| 1399 | mte_check_fail(env, *wdesc, toaddr, ra); |
| 1400 | *wdesc = 0; |
| 1401 | } else { |
| 1402 | copysize = MIN(copysize, mtesize); |
| 1403 | } |
| 1404 | } |
| 1405 | |
| 1406 | toaddr = useronly_clean_ptr(toaddr); |
| 1407 | fromaddr = useronly_clean_ptr(fromaddr); |
| 1408 | /* Trapless lookup of whether we can get a host memory pointer */ |
| 1409 | wmem = tlb_vaddr_to_host(env, toaddr, MMU_DATA_STORE, wmemidx); |
| 1410 | rmem = tlb_vaddr_to_host(env, fromaddr, MMU_DATA_LOAD, rmemidx); |
| 1411 | |
| 1412 | #ifndef CONFIG_USER_ONLY |
| 1413 | /* |
| 1414 | * If we don't have host memory for both source and dest then just |
| 1415 | * do a single byte copy. This will handle watchpoints, invalid pages, |
| 1416 | * etc correctly. For clean code pages, the next iteration will see |
| 1417 | * the page dirty and will use the fast path. |
| 1418 | */ |
| 1419 | if (unlikely(!rmem || !wmem)) { |
| 1420 | uint8_t byte; |
| 1421 | if (rmem) { |
| 1422 | byte = *(uint8_t *)rmem; |
| 1423 | } else { |
| 1424 | byte = cpu_ldub_mmuidx_ra(env, fromaddr, rmemidx, ra); |
| 1425 | } |
| 1426 | if (wmem) { |
| 1427 | *(uint8_t *)wmem = byte; |
| 1428 | } else { |
| 1429 | cpu_stb_mmuidx_ra(env, toaddr, byte, wmemidx, ra); |
| 1430 | } |
| 1431 | return 1; |
| 1432 | } |
| 1433 | #endif |
| 1434 | /* |
| 1435 | * Easy case: just memmove the host memory. Note that wmem and |
| 1436 | * rmem here point to the *last* byte to copy. |
| 1437 | */ |
| 1438 | set_helper_retaddr(ra); |
| 1439 | memmove(wmem - (copysize - 1), rmem - (copysize - 1), copysize); |
| 1440 | clear_helper_retaddr(); |
| 1441 | return copysize; |
| 1442 | } |
| 1443 | |
| 1444 | /* |
| 1445 | * for the Memory Copy operation, our implementation chooses always |
| 1446 | * to use "option A", where we update Xd and Xs to the final addresses |
| 1447 | * in the CPYP insn, and then in CPYM and CPYE only need to update Xn. |
| 1448 | * |
| 1449 | * @env: CPU |
| 1450 | * @syndrome: syndrome value for mismatch exceptions |
| 1451 | * (also contains the register numbers we need to use) |
| 1452 | * @wdesc: MTE descriptor for the writes (destination) |
| 1453 | * @rdesc: MTE descriptor for the reads (source) |
| 1454 | * @move: true if this is CPY (memmove), false for CPYF (memcpy forwards) |
| 1455 | */ |
| 1456 | static void do_cpyp(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, |
| 1457 | uint32_t rdesc, uint32_t move, uintptr_t ra) |
| 1458 | { |
| 1459 | int rd = mops_destreg(syndrome); |
| 1460 | int rs = mops_srcreg(syndrome); |
| 1461 | int rn = mops_sizereg(syndrome); |
| 1462 | uint32_t rmemidx = FIELD_EX32(rdesc, MTEDESC, MIDX); |
| 1463 | uint32_t wmemidx = FIELD_EX32(wdesc, MTEDESC, MIDX); |
| 1464 | bool forwards = true; |
| 1465 | uint64_t toaddr = env->xregs[rd]; |
| 1466 | uint64_t fromaddr = env->xregs[rs]; |
| 1467 | uint64_t copysize = env->xregs[rn]; |
| 1468 | uint64_t stagecopysize, step; |
| 1469 | |
| 1470 | check_mops_enabled(env, ra); |
| 1471 | |
| 1472 | |
| 1473 | if (move) { |
| 1474 | /* |
| 1475 | * Copy backwards if necessary. The direction for a non-overlapping |
| 1476 | * copy is IMPDEF; we choose forwards. |
| 1477 | */ |
| 1478 | if (copysize > 0x007FFFFFFFFFFFFFULL) { |
| 1479 | copysize = 0x007FFFFFFFFFFFFFULL; |
| 1480 | } |
| 1481 | uint64_t fs = extract64(fromaddr, 0, 56); |
| 1482 | uint64_t ts = extract64(toaddr, 0, 56); |
| 1483 | uint64_t fe = extract64(fromaddr + copysize, 0, 56); |
| 1484 | |
| 1485 | if (fs < ts && fe > ts) { |
| 1486 | forwards = false; |
| 1487 | } |
| 1488 | } else { |
| 1489 | if (copysize > INT64_MAX) { |
| 1490 | copysize = INT64_MAX; |
| 1491 | } |
| 1492 | } |
| 1493 | |
| 1494 | if (!mte_checks_needed(fromaddr, rdesc)) { |
| 1495 | rdesc = 0; |
| 1496 | } |
| 1497 | if (!mte_checks_needed(toaddr, wdesc)) { |
| 1498 | wdesc = 0; |
| 1499 | } |
| 1500 | |
| 1501 | if (forwards) { |
| 1502 | stagecopysize = MIN(copysize, page_limit(toaddr)); |
| 1503 | stagecopysize = MIN(stagecopysize, page_limit(fromaddr)); |
| 1504 | while (stagecopysize) { |
| 1505 | env->xregs[rd] = toaddr; |
| 1506 | env->xregs[rs] = fromaddr; |
| 1507 | env->xregs[rn] = copysize; |
| 1508 | step = copy_step(env, toaddr, fromaddr, stagecopysize, |
| 1509 | wmemidx, rmemidx, &wdesc, &rdesc, ra); |
| 1510 | toaddr += step; |
| 1511 | fromaddr += step; |
| 1512 | copysize -= step; |
| 1513 | stagecopysize -= step; |
| 1514 | } |
| 1515 | /* Insn completed, so update registers to the Option A format */ |
| 1516 | env->xregs[rd] = toaddr + copysize; |
| 1517 | env->xregs[rs] = fromaddr + copysize; |
| 1518 | env->xregs[rn] = -copysize; |
| 1519 | } else { |
| 1520 | /* |
| 1521 | * In a reverse copy the to and from addrs in Xs and Xd are the start |
| 1522 | * of the range, but it's more convenient for us to work with pointers |
| 1523 | * to the last byte being copied. |
| 1524 | */ |
| 1525 | toaddr += copysize - 1; |
| 1526 | fromaddr += copysize - 1; |
| 1527 | stagecopysize = MIN(copysize, page_limit_rev(toaddr)); |
| 1528 | stagecopysize = MIN(stagecopysize, page_limit_rev(fromaddr)); |
| 1529 | while (stagecopysize) { |
| 1530 | env->xregs[rn] = copysize; |
| 1531 | step = copy_step_rev(env, toaddr, fromaddr, stagecopysize, |
| 1532 | wmemidx, rmemidx, &wdesc, &rdesc, ra); |
| 1533 | copysize -= step; |
| 1534 | stagecopysize -= step; |
| 1535 | toaddr -= step; |
| 1536 | fromaddr -= step; |
| 1537 | } |
| 1538 | /* |
| 1539 | * Insn completed, so update registers to the Option A format. |
| 1540 | * For a reverse copy this is no different to the CPYP input format. |
| 1541 | */ |
| 1542 | env->xregs[rn] = copysize; |
| 1543 | } |
| 1544 | |
| 1545 | /* Set NZCV = 0000 to indicate we are an Option A implementation */ |
| 1546 | env->NF = 0; |
| 1547 | env->ZF = 1; /* our env->ZF encoding is inverted */ |
| 1548 | env->CF = 0; |
| 1549 | env->VF = 0; |
| 1550 | } |
| 1551 | |
| 1552 | void HELPER(cpyp)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, |
| 1553 | uint32_t rdesc) |
| 1554 | { |
| 1555 | do_cpyp(env, syndrome, wdesc, rdesc, true, GETPC()); |
| 1556 | } |
| 1557 | |
| 1558 | void HELPER(cpyfp)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, |
| 1559 | uint32_t rdesc) |
| 1560 | { |
| 1561 | do_cpyp(env, syndrome, wdesc, rdesc, false, GETPC()); |
| 1562 | } |
| 1563 | |
| 1564 | static void do_cpym(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, |
| 1565 | uint32_t rdesc, uint32_t move, uintptr_t ra) |
| 1566 | { |
| 1567 | /* Main: we choose to copy until less than a page remaining */ |
| 1568 | CPUState *cs = env_cpu(env); |
| 1569 | int rd = mops_destreg(syndrome); |
| 1570 | int rs = mops_srcreg(syndrome); |
| 1571 | int rn = mops_sizereg(syndrome); |
| 1572 | uint32_t rmemidx = FIELD_EX32(rdesc, MTEDESC, MIDX); |
| 1573 | uint32_t wmemidx = FIELD_EX32(wdesc, MTEDESC, MIDX); |
| 1574 | bool forwards = true; |
| 1575 | uint64_t toaddr, fromaddr, copysize, step; |
| 1576 | |
| 1577 | check_mops_enabled(env, ra); |
| 1578 | |
| 1579 | /* We choose to NOP out "no data to copy" before consistency checks */ |
| 1580 | if (env->xregs[rn] == 0) { |
| 1581 | return; |
| 1582 | } |
| 1583 | |
| 1584 | check_mops_wrong_option(env, syndrome, ra); |
| 1585 | |
| 1586 | if (move) { |
| 1587 | forwards = (int64_t)env->xregs[rn] < 0; |
| 1588 | } |
| 1589 | |
| 1590 | if (forwards) { |
| 1591 | toaddr = env->xregs[rd] + env->xregs[rn]; |
| 1592 | fromaddr = env->xregs[rs] + env->xregs[rn]; |
| 1593 | copysize = -env->xregs[rn]; |
| 1594 | } else { |
| 1595 | copysize = env->xregs[rn]; |
| 1596 | /* This toaddr and fromaddr point to the *last* byte to copy */ |
| 1597 | toaddr = env->xregs[rd] + copysize - 1; |
| 1598 | fromaddr = env->xregs[rs] + copysize - 1; |
| 1599 | } |
| 1600 | |
| 1601 | if (!mte_checks_needed(fromaddr, rdesc)) { |
| 1602 | rdesc = 0; |
| 1603 | } |
| 1604 | if (!mte_checks_needed(toaddr, wdesc)) { |
| 1605 | wdesc = 0; |
| 1606 | } |
| 1607 | |
| 1608 | /* Our implementation has no particular parameter requirements for CPYM */ |
| 1609 | |
| 1610 | /* Do the actual memmove */ |
| 1611 | if (forwards) { |
| 1612 | while (copysize >= TARGET_PAGE_SIZE) { |
| 1613 | step = copy_step(env, toaddr, fromaddr, copysize, |
| 1614 | wmemidx, rmemidx, &wdesc, &rdesc, ra); |
| 1615 | toaddr += step; |
| 1616 | fromaddr += step; |
| 1617 | copysize -= step; |
| 1618 | env->xregs[rn] = -copysize; |
| 1619 | if (copysize >= TARGET_PAGE_SIZE && |
| 1620 | unlikely(cpu_loop_exit_requested(cs))) { |
| 1621 | cpu_loop_exit_restore(cs, ra); |
| 1622 | } |
| 1623 | } |
| 1624 | } else { |
| 1625 | while (copysize >= TARGET_PAGE_SIZE) { |
| 1626 | step = copy_step_rev(env, toaddr, fromaddr, copysize, |
| 1627 | wmemidx, rmemidx, &wdesc, &rdesc, ra); |
| 1628 | toaddr -= step; |
| 1629 | fromaddr -= step; |
| 1630 | copysize -= step; |
| 1631 | env->xregs[rn] = copysize; |
| 1632 | if (copysize >= TARGET_PAGE_SIZE && |
| 1633 | unlikely(cpu_loop_exit_requested(cs))) { |
| 1634 | cpu_loop_exit_restore(cs, ra); |
| 1635 | } |
| 1636 | } |
| 1637 | } |
| 1638 | } |
| 1639 | |
| 1640 | void HELPER(cpym)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, |
| 1641 | uint32_t rdesc) |
| 1642 | { |
| 1643 | do_cpym(env, syndrome, wdesc, rdesc, true, GETPC()); |
| 1644 | } |
| 1645 | |
| 1646 | void HELPER(cpyfm)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, |
| 1647 | uint32_t rdesc) |
| 1648 | { |
| 1649 | do_cpym(env, syndrome, wdesc, rdesc, false, GETPC()); |
| 1650 | } |
| 1651 | |
| 1652 | static void do_cpye(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, |
| 1653 | uint32_t rdesc, uint32_t move, uintptr_t ra) |
| 1654 | { |
| 1655 | /* Epilogue: do the last partial page */ |
| 1656 | int rd = mops_destreg(syndrome); |
| 1657 | int rs = mops_srcreg(syndrome); |
| 1658 | int rn = mops_sizereg(syndrome); |
| 1659 | uint32_t rmemidx = FIELD_EX32(rdesc, MTEDESC, MIDX); |
| 1660 | uint32_t wmemidx = FIELD_EX32(wdesc, MTEDESC, MIDX); |
| 1661 | bool forwards = true; |
| 1662 | uint64_t toaddr, fromaddr, copysize, step; |
| 1663 | |
| 1664 | check_mops_enabled(env, ra); |
| 1665 | |
| 1666 | /* We choose to NOP out "no data to copy" before consistency checks */ |
| 1667 | if (env->xregs[rn] == 0) { |
| 1668 | return; |
| 1669 | } |
| 1670 | |
| 1671 | check_mops_wrong_option(env, syndrome, ra); |
| 1672 | |
| 1673 | if (move) { |
| 1674 | forwards = (int64_t)env->xregs[rn] < 0; |
| 1675 | } |
| 1676 | |
| 1677 | if (forwards) { |
| 1678 | toaddr = env->xregs[rd] + env->xregs[rn]; |
| 1679 | fromaddr = env->xregs[rs] + env->xregs[rn]; |
| 1680 | copysize = -env->xregs[rn]; |
| 1681 | } else { |
| 1682 | copysize = env->xregs[rn]; |
| 1683 | /* This toaddr and fromaddr point to the *last* byte to copy */ |
| 1684 | toaddr = env->xregs[rd] + copysize - 1; |
| 1685 | fromaddr = env->xregs[rs] + copysize - 1; |
| 1686 | } |
| 1687 | |
| 1688 | if (!mte_checks_needed(fromaddr, rdesc)) { |
| 1689 | rdesc = 0; |
| 1690 | } |
| 1691 | if (!mte_checks_needed(toaddr, wdesc)) { |
| 1692 | wdesc = 0; |
| 1693 | } |
| 1694 | |
| 1695 | /* Check the size; we don't want to have do a check-for-interrupts */ |
| 1696 | if (copysize >= TARGET_PAGE_SIZE) { |
| 1697 | raise_exception_ra(env, EXCP_UDEF, syndrome, |
| 1698 | mops_mismatch_exception_target_el(env), ra); |
| 1699 | } |
| 1700 | |
| 1701 | /* Do the actual memmove */ |
| 1702 | if (forwards) { |
| 1703 | while (copysize > 0) { |
| 1704 | step = copy_step(env, toaddr, fromaddr, copysize, |
| 1705 | wmemidx, rmemidx, &wdesc, &rdesc, ra); |
| 1706 | toaddr += step; |
| 1707 | fromaddr += step; |
| 1708 | copysize -= step; |
| 1709 | env->xregs[rn] = -copysize; |
| 1710 | } |
| 1711 | } else { |
| 1712 | while (copysize > 0) { |
| 1713 | step = copy_step_rev(env, toaddr, fromaddr, copysize, |
| 1714 | wmemidx, rmemidx, &wdesc, &rdesc, ra); |
| 1715 | toaddr -= step; |
| 1716 | fromaddr -= step; |
| 1717 | copysize -= step; |
| 1718 | env->xregs[rn] = copysize; |
| 1719 | } |
| 1720 | } |
| 1721 | } |
| 1722 | |
| 1723 | void HELPER(cpye)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, |
| 1724 | uint32_t rdesc) |
| 1725 | { |
| 1726 | do_cpye(env, syndrome, wdesc, rdesc, true, GETPC()); |
| 1727 | } |
| 1728 | |
| 1729 | void HELPER(cpyfe)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, |
| 1730 | uint32_t rdesc) |
| 1731 | { |
| 1732 | do_cpye(env, syndrome, wdesc, rdesc, false, GETPC()); |
| 1733 | } |
| 1734 | |
| 1735 | static bool is_guarded_page(CPUARMState *env, vaddr addr, uintptr_t ra) |
| 1736 | { |
| 1737 | #ifdef CONFIG_USER_ONLY |
| 1738 | return page_get_flags(addr) & PAGE_BTI; |
| 1739 | #else |
| 1740 | CPUTLBEntryFull *full; |
| 1741 | void *host; |
| 1742 | int mmu_idx = cpu_mmu_index(env_cpu(env), true); |
| 1743 | int flags = probe_access_full(env, addr, 0, MMU_INST_FETCH, mmu_idx, |
| 1744 | false, &host, &full, ra); |
| 1745 | |
| 1746 | assert(!(flags & TLB_INVALID_MASK)); |
| 1747 | return full->extra.arm.guarded; |
| 1748 | #endif |
| 1749 | } |
| 1750 | |
| 1751 | void HELPER(guarded_page_check)(CPUARMState *env) |
| 1752 | { |
| 1753 | /* |
| 1754 | * We have already verified that bti is enabled, and that the |
| 1755 | * instruction at PC is not ok for BTYPE. This is always at |
| 1756 | * the beginning of a block, so PC is always up-to-date and |
| 1757 | * no unwind is required. |
| 1758 | */ |
| 1759 | if (is_guarded_page(env, env->pc, 0)) { |
| 1760 | raise_exception(env, EXCP_UDEF, syn_btitrap(env->btype), |
| 1761 | exception_target_el(env)); |
| 1762 | } |
| 1763 | } |
| 1764 | |
| 1765 | void HELPER(guarded_page_br)(CPUARMState *env, vaddr pc) |
| 1766 | { |
| 1767 | /* |
| 1768 | * We have already checked for branch via x16 and x17. |
| 1769 | * What remains for choosing BTYPE is checking for a guarded page. |
| 1770 | */ |
| 1771 | env->btype = is_guarded_page(env, pc, GETPC()) ? 3 : 1; |
| 1772 | } |