| 1 | /* |
| 2 | * ARM VFP floating-point operations |
| 3 | * |
| 4 | * Copyright (c) 2003 Fabrice Bellard |
| 5 | * |
| 6 | * This library is free software; you can redistribute it and/or |
| 7 | * modify it under the terms of the GNU Lesser General Public |
| 8 | * License as published by the Free Software Foundation; either |
| 9 | * version 2.1 of the License, or (at your option) any later version. |
| 10 | * |
| 11 | * This library is distributed in the hope that it will be useful, |
| 12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 14 | * Lesser General Public License for more details. |
| 15 | * |
| 16 | * You should have received a copy of the GNU Lesser General Public |
| 17 | * License along with this library; if not, see <http://www.gnu.org/licenses/>. |
| 18 | */ |
| 19 | |
| 20 | #include "qemu/osdep.h" |
| 21 | #include "cpu.h" |
| 22 | #include "helper.h" |
| 23 | #include "internals.h" |
| 24 | #include "cpu-features.h" |
| 25 | #include "fpu/softfloat.h" |
| 26 | #include "qemu/log.h" |
| 27 | |
| 28 | /* |
| 29 | * Set the float_status behaviour to match the Arm defaults: |
| 30 | * * tininess-before-rounding |
| 31 | * * 2-input NaN propagation prefers SNaN over QNaN, and then |
| 32 | * operand A over operand B (see FPProcessNaNs() pseudocode) |
| 33 | * * 3-input NaN propagation prefers SNaN over QNaN, and then |
| 34 | * operand C over A over B (see FPProcessNaNs3() pseudocode, |
| 35 | * but note that for QEMU muladd is a * b + c, whereas for |
| 36 | * the pseudocode function the arguments are in the order c, a, b. |
| 37 | * * 0 * Inf + NaN returns the default NaN if the input NaN is quiet, |
| 38 | * and the input NaN if it is signalling |
| 39 | * * Default NaN has sign bit clear, msb frac bit set |
| 40 | */ |
| 41 | void arm_set_default_fp_behaviours(float_status *s) |
| 42 | { |
| 43 | set_float_detect_tininess(float_tininess_before_rounding, s); |
| 44 | set_float_ftz_detection(float_ftz_before_rounding, s); |
| 45 | set_float_2nan_prop_rule(float_2nan_prop_s_ab, s); |
| 46 | set_float_3nan_prop_rule(float_3nan_prop_s_cab, s); |
| 47 | set_float_infzeronan_rule(float_infzeronan_dnan_if_qnan, s); |
| 48 | set_float_default_nan_pattern(0b01000000, s); |
| 49 | } |
| 50 | |
| 51 | /* |
| 52 | * Set the float_status behaviour to match the FEAT_AFP |
| 53 | * FPCR.AH=1 requirements: |
| 54 | * * tininess-after-rounding |
| 55 | * * 2-input NaN propagation prefers the first NaN |
| 56 | * * 3-input NaN propagation prefers a over b over c |
| 57 | * * 0 * Inf + NaN always returns the input NaN and doesn't |
| 58 | * set Invalid for a QNaN |
| 59 | * * default NaN has sign bit set, msb frac bit set |
| 60 | */ |
| 61 | void arm_set_ah_fp_behaviours(float_status *s) |
| 62 | { |
| 63 | set_float_detect_tininess(float_tininess_after_rounding, s); |
| 64 | set_float_ftz_detection(float_ftz_after_rounding, s); |
| 65 | set_float_2nan_prop_rule(float_2nan_prop_ab, s); |
| 66 | set_float_3nan_prop_rule(float_3nan_prop_abc, s); |
| 67 | set_float_infzeronan_rule(float_infzeronan_dnan_never | |
| 68 | float_infzeronan_suppress_invalid, s); |
| 69 | set_float_default_nan_pattern(0b11000000, s); |
| 70 | } |
| 71 | |
| 72 | /* Convert host exception flags to vfp form. */ |
| 73 | static inline uint32_t vfp_exceptbits_from_host(int host_bits, bool ah) |
| 74 | { |
| 75 | uint32_t target_bits = 0; |
| 76 | |
| 77 | if (host_bits & float_flag_invalid) { |
| 78 | target_bits |= FPSR_IOC; |
| 79 | } |
| 80 | if (host_bits & float_flag_divbyzero) { |
| 81 | target_bits |= FPSR_DZC; |
| 82 | } |
| 83 | if (host_bits & float_flag_overflow) { |
| 84 | target_bits |= FPSR_OFC; |
| 85 | } |
| 86 | if (host_bits & (float_flag_underflow | float_flag_output_denormal_flushed)) { |
| 87 | target_bits |= FPSR_UFC; |
| 88 | } |
| 89 | if (host_bits & float_flag_inexact) { |
| 90 | target_bits |= FPSR_IXC; |
| 91 | } |
| 92 | if (host_bits & float_flag_input_denormal_flushed) { |
| 93 | target_bits |= FPSR_IDC; |
| 94 | } |
| 95 | /* |
| 96 | * With FPCR.AH, IDC is set when an input denormal is used, |
| 97 | * and flushing an output denormal to zero sets both IXC and UFC. |
| 98 | */ |
| 99 | if (ah && (host_bits & float_flag_input_denormal_used)) { |
| 100 | target_bits |= FPSR_IDC; |
| 101 | } |
| 102 | if (ah && (host_bits & float_flag_output_denormal_flushed)) { |
| 103 | target_bits |= FPSR_IXC; |
| 104 | } |
| 105 | return target_bits; |
| 106 | } |
| 107 | |
| 108 | uint32_t vfp_get_fpsr_from_host(CPUARMState *env) |
| 109 | { |
| 110 | uint32_t a32_flags = 0, a64_flags = 0; |
| 111 | |
| 112 | a32_flags |= get_float_exception_flags(&env->vfp.fp_status[FPST_A32]); |
| 113 | a32_flags |= get_float_exception_flags(&env->vfp.fp_status[FPST_STD]); |
| 114 | /* FZ16 does not generate an input denormal exception. */ |
| 115 | a32_flags |= (get_float_exception_flags(&env->vfp.fp_status[FPST_A32_F16]) |
| 116 | & ~float_flag_input_denormal_flushed); |
| 117 | a32_flags |= (get_float_exception_flags(&env->vfp.fp_status[FPST_STD_F16]) |
| 118 | & ~float_flag_input_denormal_flushed); |
| 119 | |
| 120 | a64_flags |= get_float_exception_flags(&env->vfp.fp_status[FPST_A64]); |
| 121 | a64_flags |= (get_float_exception_flags(&env->vfp.fp_status[FPST_A64_F16]) |
| 122 | & ~(float_flag_input_denormal_flushed | float_flag_input_denormal_used)); |
| 123 | /* |
| 124 | * We do not merge in flags from FPST_{AH,ZA} or FPST_{AH,ZA}_F16, because |
| 125 | * they are used for insns that must not set the cumulative exception bits. |
| 126 | */ |
| 127 | |
| 128 | /* |
| 129 | * Flushing an input denormal *only* because FPCR.FIZ == 1 does |
| 130 | * not set FPSR.IDC; if FPCR.FZ is also set then this takes |
| 131 | * precedence and IDC is set (see the FPUnpackBase pseudocode). |
| 132 | * So squash it unless (FPCR.AH == 0 && FPCR.FZ == 1). |
| 133 | * We only do this for the a64 flags because FIZ has no effect |
| 134 | * on AArch32 even if it is set. |
| 135 | */ |
| 136 | if ((env->vfp.fpcr & (FPCR_FZ | FPCR_AH)) != FPCR_FZ) { |
| 137 | a64_flags &= ~float_flag_input_denormal_flushed; |
| 138 | } |
| 139 | return vfp_exceptbits_from_host(a64_flags, env->vfp.fpcr & FPCR_AH) | |
| 140 | vfp_exceptbits_from_host(a32_flags, false); |
| 141 | } |
| 142 | |
| 143 | void vfp_clear_float_status_exc_flags(CPUARMState *env) |
| 144 | { |
| 145 | /* |
| 146 | * Clear out all the exception-flag information in the float_status |
| 147 | * values. The caller should have arranged for env->vfp.fpsr to |
| 148 | * be the architecturally up-to-date exception flag information first. |
| 149 | */ |
| 150 | set_float_exception_flags(0, &env->vfp.fp_status[FPST_A32]); |
| 151 | set_float_exception_flags(0, &env->vfp.fp_status[FPST_A64]); |
| 152 | set_float_exception_flags(0, &env->vfp.fp_status[FPST_A32_F16]); |
| 153 | set_float_exception_flags(0, &env->vfp.fp_status[FPST_A64_F16]); |
| 154 | set_float_exception_flags(0, &env->vfp.fp_status[FPST_STD]); |
| 155 | set_float_exception_flags(0, &env->vfp.fp_status[FPST_STD_F16]); |
| 156 | set_float_exception_flags(0, &env->vfp.fp_status[FPST_AH]); |
| 157 | set_float_exception_flags(0, &env->vfp.fp_status[FPST_AH_F16]); |
| 158 | } |
| 159 | |
| 160 | static void vfp_sync_and_clear_float_status_exc_flags(CPUARMState *env) |
| 161 | { |
| 162 | /* |
| 163 | * Synchronize any pending exception-flag information in the |
| 164 | * float_status values into env->vfp.fpsr, and then clear out |
| 165 | * the float_status data. |
| 166 | */ |
| 167 | env->vfp.fpsr |= vfp_get_fpsr_from_host(env); |
| 168 | vfp_clear_float_status_exc_flags(env); |
| 169 | } |
| 170 | |
| 171 | void vfp_set_fpcr_to_host(CPUARMState *env, uint32_t val, uint32_t mask) |
| 172 | { |
| 173 | uint64_t changed = env->vfp.fpcr; |
| 174 | |
| 175 | changed ^= val; |
| 176 | changed &= mask; |
| 177 | if (changed & (3 << 22)) { |
| 178 | int i = (val >> 22) & 3; |
| 179 | switch (i) { |
| 180 | case FPROUNDING_TIEEVEN: |
| 181 | i = float_round_nearest_even; |
| 182 | break; |
| 183 | case FPROUNDING_POSINF: |
| 184 | i = float_round_up; |
| 185 | break; |
| 186 | case FPROUNDING_NEGINF: |
| 187 | i = float_round_down; |
| 188 | break; |
| 189 | case FPROUNDING_ZERO: |
| 190 | i = float_round_to_zero; |
| 191 | break; |
| 192 | } |
| 193 | set_float_rounding_mode(i, &env->vfp.fp_status[FPST_A32]); |
| 194 | set_float_rounding_mode(i, &env->vfp.fp_status[FPST_A64]); |
| 195 | set_float_rounding_mode(i, &env->vfp.fp_status[FPST_A32_F16]); |
| 196 | set_float_rounding_mode(i, &env->vfp.fp_status[FPST_A64_F16]); |
| 197 | set_float_rounding_mode(i, &env->vfp.fp_status[FPST_ZA]); |
| 198 | set_float_rounding_mode(i, &env->vfp.fp_status[FPST_ZA_F16]); |
| 199 | } |
| 200 | if (changed & FPCR_FZ16) { |
| 201 | bool ftz_enabled = val & FPCR_FZ16; |
| 202 | set_flush_to_zero(ftz_enabled, &env->vfp.fp_status[FPST_A32_F16]); |
| 203 | set_flush_to_zero(ftz_enabled, &env->vfp.fp_status[FPST_A64_F16]); |
| 204 | set_flush_to_zero(ftz_enabled, &env->vfp.fp_status[FPST_STD_F16]); |
| 205 | set_flush_to_zero(ftz_enabled, &env->vfp.fp_status[FPST_AH_F16]); |
| 206 | set_flush_to_zero(ftz_enabled, &env->vfp.fp_status[FPST_ZA_F16]); |
| 207 | set_flush_inputs_to_zero(ftz_enabled, &env->vfp.fp_status[FPST_A32_F16]); |
| 208 | set_flush_inputs_to_zero(ftz_enabled, &env->vfp.fp_status[FPST_A64_F16]); |
| 209 | set_flush_inputs_to_zero(ftz_enabled, &env->vfp.fp_status[FPST_STD_F16]); |
| 210 | set_flush_inputs_to_zero(ftz_enabled, &env->vfp.fp_status[FPST_AH_F16]); |
| 211 | set_flush_inputs_to_zero(ftz_enabled, &env->vfp.fp_status[FPST_ZA_F16]); |
| 212 | } |
| 213 | if (changed & FPCR_FZ) { |
| 214 | bool ftz_enabled = val & FPCR_FZ; |
| 215 | set_flush_to_zero(ftz_enabled, &env->vfp.fp_status[FPST_A32]); |
| 216 | set_flush_to_zero(ftz_enabled, &env->vfp.fp_status[FPST_A64]); |
| 217 | set_flush_to_zero(ftz_enabled, &env->vfp.fp_status[FPST_ZA]); |
| 218 | /* FIZ is A64 only so FZ always makes A32 code flush inputs to zero */ |
| 219 | set_flush_inputs_to_zero(ftz_enabled, &env->vfp.fp_status[FPST_A32]); |
| 220 | } |
| 221 | if (changed & (FPCR_FZ | FPCR_AH | FPCR_FIZ)) { |
| 222 | /* |
| 223 | * A64: Flush denormalized inputs to zero if FPCR.FIZ = 1, or |
| 224 | * both FPCR.AH = 0 and FPCR.FZ = 1. |
| 225 | */ |
| 226 | bool fitz_enabled = (val & FPCR_FIZ) || |
| 227 | (val & (FPCR_FZ | FPCR_AH)) == FPCR_FZ; |
| 228 | set_flush_inputs_to_zero(fitz_enabled, &env->vfp.fp_status[FPST_A64]); |
| 229 | set_flush_inputs_to_zero(fitz_enabled, &env->vfp.fp_status[FPST_ZA]); |
| 230 | } |
| 231 | if (changed & FPCR_DN) { |
| 232 | bool dnan_enabled = val & FPCR_DN; |
| 233 | set_default_nan_mode(dnan_enabled, &env->vfp.fp_status[FPST_A32]); |
| 234 | set_default_nan_mode(dnan_enabled, &env->vfp.fp_status[FPST_A64]); |
| 235 | set_default_nan_mode(dnan_enabled, &env->vfp.fp_status[FPST_A32_F16]); |
| 236 | set_default_nan_mode(dnan_enabled, &env->vfp.fp_status[FPST_A64_F16]); |
| 237 | set_default_nan_mode(dnan_enabled, &env->vfp.fp_status[FPST_AH]); |
| 238 | set_default_nan_mode(dnan_enabled, &env->vfp.fp_status[FPST_AH_F16]); |
| 239 | } |
| 240 | if (changed & FPCR_AH) { |
| 241 | bool ah_enabled = val & FPCR_AH; |
| 242 | |
| 243 | if (ah_enabled) { |
| 244 | /* Change behaviours for A64 FP operations */ |
| 245 | arm_set_ah_fp_behaviours(&env->vfp.fp_status[FPST_A64]); |
| 246 | arm_set_ah_fp_behaviours(&env->vfp.fp_status[FPST_A64_F16]); |
| 247 | arm_set_ah_fp_behaviours(&env->vfp.fp_status[FPST_ZA]); |
| 248 | arm_set_ah_fp_behaviours(&env->vfp.fp_status[FPST_ZA_F16]); |
| 249 | } else { |
| 250 | arm_set_default_fp_behaviours(&env->vfp.fp_status[FPST_A64]); |
| 251 | arm_set_default_fp_behaviours(&env->vfp.fp_status[FPST_A64_F16]); |
| 252 | arm_set_default_fp_behaviours(&env->vfp.fp_status[FPST_ZA]); |
| 253 | arm_set_default_fp_behaviours(&env->vfp.fp_status[FPST_ZA_F16]); |
| 254 | } |
| 255 | } |
| 256 | /* |
| 257 | * If any bits changed that we look at in vfp_get_fpsr_from_host(), |
| 258 | * we must sync the float_status flags into vfp.fpsr now (under the |
| 259 | * old regime) before we update vfp.fpcr. |
| 260 | */ |
| 261 | if (changed & (FPCR_FZ | FPCR_AH | FPCR_FIZ)) { |
| 262 | vfp_sync_and_clear_float_status_exc_flags(env); |
| 263 | } |
| 264 | } |
| 265 | |
| 266 | /* |
| 267 | * VFP support. We follow the convention used for VFP instructions: |
| 268 | * Single precision routines have a "s" suffix, double precision a |
| 269 | * "d" suffix. |
| 270 | */ |
| 271 | |
| 272 | #define VFP_HELPER(name, p) HELPER(glue(glue(vfp_,name),p)) |
| 273 | |
| 274 | #define VFP_BINOP(name) \ |
| 275 | dh_ctype_f16 VFP_HELPER(name, h)(dh_ctype_f16 a, dh_ctype_f16 b, float_status *fpst) \ |
| 276 | { \ |
| 277 | return float16_ ## name(a, b, fpst); \ |
| 278 | } \ |
| 279 | float32 VFP_HELPER(name, s)(float32 a, float32 b, float_status *fpst) \ |
| 280 | { \ |
| 281 | return float32_ ## name(a, b, fpst); \ |
| 282 | } \ |
| 283 | float64 VFP_HELPER(name, d)(float64 a, float64 b, float_status *fpst) \ |
| 284 | { \ |
| 285 | return float64_ ## name(a, b, fpst); \ |
| 286 | } |
| 287 | VFP_BINOP(add) |
| 288 | VFP_BINOP(sub) |
| 289 | VFP_BINOP(mul) |
| 290 | VFP_BINOP(div) |
| 291 | VFP_BINOP(min) |
| 292 | VFP_BINOP(max) |
| 293 | VFP_BINOP(minnum) |
| 294 | VFP_BINOP(maxnum) |
| 295 | #undef VFP_BINOP |
| 296 | |
| 297 | dh_ctype_f16 VFP_HELPER(sqrt, h)(dh_ctype_f16 a, float_status *fpst) |
| 298 | { |
| 299 | return float16_sqrt(a, fpst); |
| 300 | } |
| 301 | |
| 302 | float32 VFP_HELPER(sqrt, s)(float32 a, float_status *fpst) |
| 303 | { |
| 304 | return float32_sqrt(a, fpst); |
| 305 | } |
| 306 | |
| 307 | float64 VFP_HELPER(sqrt, d)(float64 a, float_status *fpst) |
| 308 | { |
| 309 | return float64_sqrt(a, fpst); |
| 310 | } |
| 311 | |
| 312 | static void softfloat_to_vfp_compare(CPUARMState *env, FloatRelation cmp) |
| 313 | { |
| 314 | uint32_t flags; |
| 315 | switch (cmp) { |
| 316 | case float_relation_equal: |
| 317 | flags = 0x6; |
| 318 | break; |
| 319 | case float_relation_less: |
| 320 | flags = 0x8; |
| 321 | break; |
| 322 | case float_relation_greater: |
| 323 | flags = 0x2; |
| 324 | break; |
| 325 | case float_relation_unordered: |
| 326 | flags = 0x3; |
| 327 | break; |
| 328 | default: |
| 329 | g_assert_not_reached(); |
| 330 | } |
| 331 | env->vfp.fpsr = deposit64(env->vfp.fpsr, 28, 4, flags); /* NZCV */ |
| 332 | } |
| 333 | |
| 334 | /* XXX: check quiet/signaling case */ |
| 335 | #define DO_VFP_cmp(P, FLOATTYPE, ARGTYPE, FPST) \ |
| 336 | void VFP_HELPER(cmp, P)(ARGTYPE a, ARGTYPE b, CPUARMState *env) \ |
| 337 | { \ |
| 338 | softfloat_to_vfp_compare(env, \ |
| 339 | FLOATTYPE ## _compare_quiet(a, b, &env->vfp.fp_status[FPST])); \ |
| 340 | } \ |
| 341 | void VFP_HELPER(cmpe, P)(ARGTYPE a, ARGTYPE b, CPUARMState *env) \ |
| 342 | { \ |
| 343 | softfloat_to_vfp_compare(env, \ |
| 344 | FLOATTYPE ## _compare(a, b, &env->vfp.fp_status[FPST])); \ |
| 345 | } |
| 346 | DO_VFP_cmp(h, float16, dh_ctype_f16, FPST_A32_F16) |
| 347 | DO_VFP_cmp(s, float32, float32, FPST_A32) |
| 348 | DO_VFP_cmp(d, float64, float64, FPST_A32) |
| 349 | #undef DO_VFP_cmp |
| 350 | |
| 351 | /* Integer to float and float to integer conversions */ |
| 352 | |
| 353 | #define CONV_ITOF(name, ftype, fsz, sign) \ |
| 354 | ftype HELPER(name)(uint32_t x, float_status *fpst) \ |
| 355 | { \ |
| 356 | return sign##int32_to_##float##fsz((sign##int32_t)x, fpst); \ |
| 357 | } |
| 358 | |
| 359 | #define CONV_FTOI(name, ftype, fsz, sign, round) \ |
| 360 | sign##int32_t HELPER(name)(ftype x, float_status *fpst) \ |
| 361 | { \ |
| 362 | if (float##fsz##_is_any_nan(x)) { \ |
| 363 | float_raise(float_flag_invalid, fpst); \ |
| 364 | return 0; \ |
| 365 | } \ |
| 366 | return float##fsz##_to_##sign##int32##round(x, fpst); \ |
| 367 | } |
| 368 | |
| 369 | #define FLOAT_CONVS(name, p, ftype, fsz, sign) \ |
| 370 | CONV_ITOF(vfp_##name##to##p, ftype, fsz, sign) \ |
| 371 | CONV_FTOI(vfp_to##name##p, ftype, fsz, sign, ) \ |
| 372 | CONV_FTOI(vfp_to##name##z##p, ftype, fsz, sign, _round_to_zero) |
| 373 | |
| 374 | FLOAT_CONVS(si, h, uint32_t, 16, ) |
| 375 | FLOAT_CONVS(si, s, float32, 32, ) |
| 376 | FLOAT_CONVS(si, d, float64, 64, ) |
| 377 | FLOAT_CONVS(ui, h, uint32_t, 16, u) |
| 378 | FLOAT_CONVS(ui, s, float32, 32, u) |
| 379 | FLOAT_CONVS(ui, d, float64, 64, u) |
| 380 | |
| 381 | #undef CONV_ITOF |
| 382 | #undef CONV_FTOI |
| 383 | #undef FLOAT_CONVS |
| 384 | |
| 385 | /* floating point conversion */ |
| 386 | float64 VFP_HELPER(fcvtd, s)(float32 x, float_status *status) |
| 387 | { |
| 388 | return float32_to_float64(x, status); |
| 389 | } |
| 390 | |
| 391 | float32 VFP_HELPER(fcvts, d)(float64 x, float_status *status) |
| 392 | { |
| 393 | return float64_to_float32(x, status); |
| 394 | } |
| 395 | |
| 396 | uint32_t HELPER(bfcvt)(float32 x, float_status *status) |
| 397 | { |
| 398 | return float32_to_bfloat16(x, status); |
| 399 | } |
| 400 | |
| 401 | uint32_t HELPER(bfcvt_pair)(uint64_t pair, float_status *status) |
| 402 | { |
| 403 | bfloat16 lo = float32_to_bfloat16(extract64(pair, 0, 32), status); |
| 404 | bfloat16 hi = float32_to_bfloat16(extract64(pair, 32, 32), status); |
| 405 | return deposit32(lo, 16, 16, hi); |
| 406 | } |
| 407 | |
| 408 | /* |
| 409 | * VFP3 fixed point conversion. The AArch32 versions of fix-to-float |
| 410 | * must always round-to-nearest; the AArch64 ones honour the FPSCR |
| 411 | * rounding mode. (For AArch32 Neon the standard-FPSCR is set to |
| 412 | * round-to-nearest so either helper will work.) AArch32 float-to-fix |
| 413 | * must round-to-zero. |
| 414 | */ |
| 415 | #define VFP_CONV_FIX_FLOAT(name, p, fsz, ftype, isz, itype) \ |
| 416 | ftype HELPER(vfp_##name##to##p)(uint##isz##_t x, uint32_t shift, \ |
| 417 | float_status *fpst) \ |
| 418 | { return itype##_to_##float##fsz##_scalbn(x, -shift, fpst); } |
| 419 | |
| 420 | #define VFP_CONV_FIX_FLOAT_ROUND(name, p, fsz, ftype, isz, itype) \ |
| 421 | ftype HELPER(vfp_##name##to##p##_round_to_nearest)(uint##isz##_t x, \ |
| 422 | uint32_t shift, \ |
| 423 | float_status *fpst) \ |
| 424 | { \ |
| 425 | ftype ret; \ |
| 426 | FloatRoundMode oldmode = get_float_rounding_mode(fpst); \ |
| 427 | set_float_rounding_mode(float_round_nearest_even, fpst); \ |
| 428 | ret = itype##_to_##float##fsz##_scalbn(x, -shift, fpst); \ |
| 429 | set_float_rounding_mode(oldmode, fpst); \ |
| 430 | return ret; \ |
| 431 | } |
| 432 | |
| 433 | #define VFP_CONV_FLOAT_FIX_ROUND(name, p, fsz, ftype, isz, itype, ROUND, suff) \ |
| 434 | uint##isz##_t HELPER(vfp_to##name##p##suff)(ftype x, uint32_t shift, \ |
| 435 | float_status *fpst) \ |
| 436 | { \ |
| 437 | if (unlikely(float##fsz##_is_any_nan(x))) { \ |
| 438 | float_raise(float_flag_invalid, fpst); \ |
| 439 | return 0; \ |
| 440 | } \ |
| 441 | return float##fsz##_to_##itype##_scalbn(x, ROUND, shift, fpst); \ |
| 442 | } |
| 443 | |
| 444 | #define VFP_CONV_FIX(name, p, fsz, ftype, isz, itype) \ |
| 445 | VFP_CONV_FIX_FLOAT(name, p, fsz, ftype, isz, itype) \ |
| 446 | VFP_CONV_FIX_FLOAT_ROUND(name, p, fsz, ftype, isz, itype) \ |
| 447 | VFP_CONV_FLOAT_FIX_ROUND(name, p, fsz, ftype, isz, itype, \ |
| 448 | float_round_to_zero, _round_to_zero) \ |
| 449 | VFP_CONV_FLOAT_FIX_ROUND(name, p, fsz, ftype, isz, itype, \ |
| 450 | get_float_rounding_mode(fpst), ) |
| 451 | |
| 452 | #define VFP_CONV_FIX_A64(name, p, fsz, ftype, isz, itype) \ |
| 453 | VFP_CONV_FIX_FLOAT(name, p, fsz, ftype, isz, itype) \ |
| 454 | VFP_CONV_FLOAT_FIX_ROUND(name, p, fsz, ftype, isz, itype, \ |
| 455 | get_float_rounding_mode(fpst), ) |
| 456 | |
| 457 | VFP_CONV_FIX(sh, d, 64, float64, 64, int16) |
| 458 | VFP_CONV_FIX(sl, d, 64, float64, 64, int32) |
| 459 | VFP_CONV_FIX_A64(sq, d, 64, float64, 64, int64) |
| 460 | VFP_CONV_FIX(uh, d, 64, float64, 64, uint16) |
| 461 | VFP_CONV_FIX(ul, d, 64, float64, 64, uint32) |
| 462 | VFP_CONV_FIX_A64(uq, d, 64, float64, 64, uint64) |
| 463 | VFP_CONV_FIX(sh, s, 32, float32, 32, int16) |
| 464 | VFP_CONV_FIX(sl, s, 32, float32, 32, int32) |
| 465 | VFP_CONV_FIX_A64(sq, s, 32, float32, 64, int64) |
| 466 | VFP_CONV_FIX(uh, s, 32, float32, 32, uint16) |
| 467 | VFP_CONV_FIX(ul, s, 32, float32, 32, uint32) |
| 468 | VFP_CONV_FIX_A64(uq, s, 32, float32, 64, uint64) |
| 469 | VFP_CONV_FIX(sh, h, 16, dh_ctype_f16, 32, int16) |
| 470 | VFP_CONV_FIX(sl, h, 16, dh_ctype_f16, 32, int32) |
| 471 | VFP_CONV_FIX_A64(sq, h, 16, dh_ctype_f16, 64, int64) |
| 472 | VFP_CONV_FIX(uh, h, 16, dh_ctype_f16, 32, uint16) |
| 473 | VFP_CONV_FIX(ul, h, 16, dh_ctype_f16, 32, uint32) |
| 474 | VFP_CONV_FIX_A64(uq, h, 16, dh_ctype_f16, 64, uint64) |
| 475 | VFP_CONV_FLOAT_FIX_ROUND(sq, d, 64, float64, 64, int64, |
| 476 | float_round_to_zero, _round_to_zero) |
| 477 | VFP_CONV_FLOAT_FIX_ROUND(uq, d, 64, float64, 64, uint64, |
| 478 | float_round_to_zero, _round_to_zero) |
| 479 | |
| 480 | #undef VFP_CONV_FIX |
| 481 | #undef VFP_CONV_FIX_FLOAT |
| 482 | #undef VFP_CONV_FLOAT_FIX_ROUND |
| 483 | #undef VFP_CONV_FIX_A64 |
| 484 | |
| 485 | /* Set the current fp rounding mode and return the old one. |
| 486 | * The argument is a softfloat float_round_ value. |
| 487 | */ |
| 488 | uint32_t HELPER(set_rmode)(uint32_t rmode, float_status *fp_status) |
| 489 | { |
| 490 | uint32_t prev_rmode = get_float_rounding_mode(fp_status); |
| 491 | set_float_rounding_mode(rmode, fp_status); |
| 492 | |
| 493 | return prev_rmode; |
| 494 | } |
| 495 | |
| 496 | /* Half precision conversions. */ |
| 497 | float32 HELPER(vfp_fcvt_f16_to_f32)(uint32_t a, float_status *fpst, |
| 498 | uint32_t ahp_mode) |
| 499 | { |
| 500 | /* Squash FZ16 to 0 for the duration of conversion. In this case, |
| 501 | * it would affect flushing input denormals. |
| 502 | */ |
| 503 | bool save = get_flush_inputs_to_zero(fpst); |
| 504 | set_flush_inputs_to_zero(false, fpst); |
| 505 | float32 r = float16_to_float32(a, !ahp_mode, fpst); |
| 506 | set_flush_inputs_to_zero(save, fpst); |
| 507 | return r; |
| 508 | } |
| 509 | |
| 510 | uint32_t HELPER(vfp_fcvt_f32_to_f16)(float32 a, float_status *fpst, |
| 511 | uint32_t ahp_mode) |
| 512 | { |
| 513 | /* Squash FZ16 to 0 for the duration of conversion. In this case, |
| 514 | * it would affect flushing output denormals. |
| 515 | */ |
| 516 | bool save = get_flush_to_zero(fpst); |
| 517 | set_flush_to_zero(false, fpst); |
| 518 | float16 r = float32_to_float16(a, !ahp_mode, fpst); |
| 519 | set_flush_to_zero(save, fpst); |
| 520 | return r; |
| 521 | } |
| 522 | |
| 523 | float64 HELPER(vfp_fcvt_f16_to_f64)(uint32_t a, float_status *fpst, |
| 524 | uint32_t ahp_mode) |
| 525 | { |
| 526 | /* Squash FZ16 to 0 for the duration of conversion. In this case, |
| 527 | * it would affect flushing input denormals. |
| 528 | */ |
| 529 | bool save = get_flush_inputs_to_zero(fpst); |
| 530 | set_flush_inputs_to_zero(false, fpst); |
| 531 | float64 r = float16_to_float64(a, !ahp_mode, fpst); |
| 532 | set_flush_inputs_to_zero(save, fpst); |
| 533 | return r; |
| 534 | } |
| 535 | |
| 536 | uint32_t HELPER(vfp_fcvt_f64_to_f16)(float64 a, float_status *fpst, |
| 537 | uint32_t ahp_mode) |
| 538 | { |
| 539 | /* Squash FZ16 to 0 for the duration of conversion. In this case, |
| 540 | * it would affect flushing output denormals. |
| 541 | */ |
| 542 | bool save = get_flush_to_zero(fpst); |
| 543 | set_flush_to_zero(false, fpst); |
| 544 | float16 r = float64_to_float16(a, !ahp_mode, fpst); |
| 545 | set_flush_to_zero(save, fpst); |
| 546 | return r; |
| 547 | } |
| 548 | |
| 549 | /* NEON helpers. */ |
| 550 | |
| 551 | /* Constants 256 and 512 are used in some helpers; we avoid relying on |
| 552 | * int->float conversions at run-time. */ |
| 553 | #define float64_256 make_float64(0x4070000000000000LL) |
| 554 | #define float64_512 make_float64(0x4080000000000000LL) |
| 555 | #define float16_maxnorm make_float16(0x7bff) |
| 556 | #define float32_maxnorm make_float32(0x7f7fffff) |
| 557 | #define float64_maxnorm make_float64(0x7fefffffffffffffLL) |
| 558 | |
| 559 | /* Reciprocal functions |
| 560 | * |
| 561 | * The algorithm that must be used to calculate the estimate |
| 562 | * is specified by the ARM ARM, see FPRecipEstimate()/RecipEstimate |
| 563 | */ |
| 564 | |
| 565 | /* See RecipEstimate() |
| 566 | * |
| 567 | * input is a 9 bit fixed point number |
| 568 | * input range 256 .. 511 for a number from 0.5 <= x < 1.0. |
| 569 | * result range 256 .. 511 for a number from 1.0 to 511/256. |
| 570 | */ |
| 571 | |
| 572 | static int recip_estimate(int input) |
| 573 | { |
| 574 | int a, b, r; |
| 575 | assert(256 <= input && input < 512); |
| 576 | a = (input * 2) + 1; |
| 577 | b = (1 << 19) / a; |
| 578 | r = (b + 1) >> 1; |
| 579 | assert(256 <= r && r < 512); |
| 580 | return r; |
| 581 | } |
| 582 | |
| 583 | /* |
| 584 | * Increased precision version: |
| 585 | * input is a 13 bit fixed point number |
| 586 | * input range 2048 .. 4095 for a number from 0.5 <= x < 1.0. |
| 587 | * result range 4096 .. 8191 for a number from 1.0 to 2.0 |
| 588 | */ |
| 589 | static int recip_estimate_incprec(int input) |
| 590 | { |
| 591 | int a, b, r; |
| 592 | assert(2048 <= input && input < 4096); |
| 593 | a = (input * 2) + 1; |
| 594 | /* |
| 595 | * The pseudocode expresses this as an operation on infinite |
| 596 | * precision reals where it calculates 2^25 / a and then looks |
| 597 | * at the error between that and the rounded-down-to-integer |
| 598 | * value to see if it should instead round up. We instead |
| 599 | * follow the same approach as the pseudocode for the 8-bit |
| 600 | * precision version, and calculate (2 * (2^25 / a)) as an |
| 601 | * integer so we can do the "add one and halve" to round it. |
| 602 | * So the 1 << 26 here is correct. |
| 603 | */ |
| 604 | b = (1 << 26) / a; |
| 605 | r = (b + 1) >> 1; |
| 606 | assert(4096 <= r && r < 8192); |
| 607 | return r; |
| 608 | } |
| 609 | |
| 610 | /* |
| 611 | * Common wrapper to call recip_estimate |
| 612 | * |
| 613 | * The parameters are exponent and 64 bit fraction (without implicit |
| 614 | * bit) where the binary point is nominally at bit 52. Returns a |
| 615 | * float64 which can then be rounded to the appropriate size by the |
| 616 | * callee. |
| 617 | */ |
| 618 | |
| 619 | static uint64_t call_recip_estimate(int *exp, int exp_off, uint64_t frac, |
| 620 | bool increasedprecision) |
| 621 | { |
| 622 | uint32_t scaled, estimate; |
| 623 | uint64_t result_frac; |
| 624 | int result_exp; |
| 625 | |
| 626 | /* Handle sub-normals */ |
| 627 | if (*exp == 0) { |
| 628 | if (extract64(frac, 51, 1) == 0) { |
| 629 | *exp = -1; |
| 630 | frac <<= 2; |
| 631 | } else { |
| 632 | frac <<= 1; |
| 633 | } |
| 634 | } |
| 635 | |
| 636 | if (increasedprecision) { |
| 637 | /* scaled = UInt('1':fraction<51:41>) */ |
| 638 | scaled = deposit32(1 << 11, 0, 11, extract64(frac, 41, 11)); |
| 639 | estimate = recip_estimate_incprec(scaled); |
| 640 | } else { |
| 641 | /* scaled = UInt('1':fraction<51:44>) */ |
| 642 | scaled = deposit32(1 << 8, 0, 8, extract64(frac, 44, 8)); |
| 643 | estimate = recip_estimate(scaled); |
| 644 | } |
| 645 | |
| 646 | result_exp = exp_off - *exp; |
| 647 | if (increasedprecision) { |
| 648 | result_frac = deposit64(0, 40, 12, estimate); |
| 649 | } else { |
| 650 | result_frac = deposit64(0, 44, 8, estimate); |
| 651 | } |
| 652 | if (result_exp == 0) { |
| 653 | result_frac = deposit64(result_frac >> 1, 51, 1, 1); |
| 654 | } else if (result_exp == -1) { |
| 655 | result_frac = deposit64(result_frac >> 2, 50, 2, 1); |
| 656 | result_exp = 0; |
| 657 | } |
| 658 | |
| 659 | *exp = result_exp; |
| 660 | |
| 661 | return result_frac; |
| 662 | } |
| 663 | |
| 664 | static bool round_to_inf(float_status *fpst, bool sign_bit) |
| 665 | { |
| 666 | switch (get_float_rounding_mode(fpst)) { |
| 667 | case float_round_nearest_even: /* Round to Nearest */ |
| 668 | return true; |
| 669 | case float_round_up: /* Round to +Inf */ |
| 670 | return !sign_bit; |
| 671 | case float_round_down: /* Round to -Inf */ |
| 672 | return sign_bit; |
| 673 | case float_round_to_zero: /* Round to Zero */ |
| 674 | return false; |
| 675 | default: |
| 676 | g_assert_not_reached(); |
| 677 | } |
| 678 | } |
| 679 | |
| 680 | uint32_t HELPER(recpe_f16)(uint32_t input, float_status *fpst) |
| 681 | { |
| 682 | float16 f16 = float16_squash_input_denormal(input, fpst); |
| 683 | uint32_t f16_val = float16_val(f16); |
| 684 | uint32_t f16_sign = float16_is_neg(f16); |
| 685 | int f16_exp = extract32(f16_val, 10, 5); |
| 686 | uint32_t f16_frac = extract32(f16_val, 0, 10); |
| 687 | uint64_t f64_frac; |
| 688 | |
| 689 | if (float16_is_any_nan(f16)) { |
| 690 | float16 nan = f16; |
| 691 | if (float16_is_signaling_nan(f16, fpst)) { |
| 692 | float_raise(float_flag_invalid, fpst); |
| 693 | if (!get_default_nan_mode(fpst)) { |
| 694 | nan = float16_silence_nan(f16, fpst); |
| 695 | } |
| 696 | } |
| 697 | if (get_default_nan_mode(fpst)) { |
| 698 | nan = float16_default_nan(fpst); |
| 699 | } |
| 700 | return nan; |
| 701 | } else if (float16_is_infinity(f16)) { |
| 702 | return float16_set_sign(float16_zero, float16_is_neg(f16)); |
| 703 | } else if (float16_is_zero(f16)) { |
| 704 | float_raise(float_flag_divbyzero, fpst); |
| 705 | return float16_set_sign(float16_infinity, float16_is_neg(f16)); |
| 706 | } else if (float16_abs(f16) < (1 << 8)) { |
| 707 | /* Abs(value) < 2.0^-16 */ |
| 708 | float_raise(float_flag_overflow | float_flag_inexact, fpst); |
| 709 | if (round_to_inf(fpst, f16_sign)) { |
| 710 | return float16_set_sign(float16_infinity, f16_sign); |
| 711 | } else { |
| 712 | return float16_set_sign(float16_maxnorm, f16_sign); |
| 713 | } |
| 714 | } else if (f16_exp >= 29 && get_flush_to_zero(fpst)) { |
| 715 | float_raise(float_flag_underflow, fpst); |
| 716 | return float16_set_sign(float16_zero, float16_is_neg(f16)); |
| 717 | } |
| 718 | |
| 719 | f64_frac = call_recip_estimate(&f16_exp, 29, |
| 720 | ((uint64_t) f16_frac) << (52 - 10), false); |
| 721 | |
| 722 | /* result = sign : result_exp<4:0> : fraction<51:42> */ |
| 723 | f16_val = deposit32(0, 15, 1, f16_sign); |
| 724 | f16_val = deposit32(f16_val, 10, 5, f16_exp); |
| 725 | f16_val = deposit32(f16_val, 0, 10, extract64(f64_frac, 52 - 10, 10)); |
| 726 | return make_float16(f16_val); |
| 727 | } |
| 728 | |
| 729 | /* |
| 730 | * FEAT_RPRES means the f32 FRECPE has an "increased precision" variant |
| 731 | * which is used when FPCR.AH == 1. |
| 732 | */ |
| 733 | static float32 do_recpe_f32(float32 input, float_status *fpst, bool rpres) |
| 734 | { |
| 735 | float32 f32 = float32_squash_input_denormal(input, fpst); |
| 736 | uint32_t f32_val = float32_val(f32); |
| 737 | bool f32_sign = float32_is_neg(f32); |
| 738 | int f32_exp = extract32(f32_val, 23, 8); |
| 739 | uint32_t f32_frac = extract32(f32_val, 0, 23); |
| 740 | uint64_t f64_frac; |
| 741 | |
| 742 | if (float32_is_any_nan(f32)) { |
| 743 | float32 nan = f32; |
| 744 | if (float32_is_signaling_nan(f32, fpst)) { |
| 745 | float_raise(float_flag_invalid, fpst); |
| 746 | if (!get_default_nan_mode(fpst)) { |
| 747 | nan = float32_silence_nan(f32, fpst); |
| 748 | } |
| 749 | } |
| 750 | if (get_default_nan_mode(fpst)) { |
| 751 | nan = float32_default_nan(fpst); |
| 752 | } |
| 753 | return nan; |
| 754 | } else if (float32_is_infinity(f32)) { |
| 755 | return float32_set_sign(float32_zero, float32_is_neg(f32)); |
| 756 | } else if (float32_is_zero(f32)) { |
| 757 | float_raise(float_flag_divbyzero, fpst); |
| 758 | return float32_set_sign(float32_infinity, float32_is_neg(f32)); |
| 759 | } else if (float32_abs(f32) < (1ULL << 21)) { |
| 760 | /* Abs(value) < 2.0^-128 */ |
| 761 | float_raise(float_flag_overflow | float_flag_inexact, fpst); |
| 762 | if (round_to_inf(fpst, f32_sign)) { |
| 763 | return float32_set_sign(float32_infinity, f32_sign); |
| 764 | } else { |
| 765 | return float32_set_sign(float32_maxnorm, f32_sign); |
| 766 | } |
| 767 | } else if (f32_exp >= 253 && get_flush_to_zero(fpst)) { |
| 768 | float_raise(float_flag_underflow, fpst); |
| 769 | return float32_set_sign(float32_zero, float32_is_neg(f32)); |
| 770 | } |
| 771 | |
| 772 | f64_frac = call_recip_estimate(&f32_exp, 253, |
| 773 | ((uint64_t) f32_frac) << (52 - 23), rpres); |
| 774 | |
| 775 | /* result = sign : result_exp<7:0> : fraction<51:29> */ |
| 776 | f32_val = deposit32(0, 31, 1, f32_sign); |
| 777 | f32_val = deposit32(f32_val, 23, 8, f32_exp); |
| 778 | f32_val = deposit32(f32_val, 0, 23, extract64(f64_frac, 52 - 23, 23)); |
| 779 | return make_float32(f32_val); |
| 780 | } |
| 781 | |
| 782 | float32 HELPER(recpe_f32)(float32 input, float_status *fpst) |
| 783 | { |
| 784 | return do_recpe_f32(input, fpst, false); |
| 785 | } |
| 786 | |
| 787 | float32 HELPER(recpe_rpres_f32)(float32 input, float_status *fpst) |
| 788 | { |
| 789 | return do_recpe_f32(input, fpst, true); |
| 790 | } |
| 791 | |
| 792 | float64 HELPER(recpe_f64)(float64 input, float_status *fpst) |
| 793 | { |
| 794 | float64 f64 = float64_squash_input_denormal(input, fpst); |
| 795 | uint64_t f64_val = float64_val(f64); |
| 796 | bool f64_sign = float64_is_neg(f64); |
| 797 | int f64_exp = extract64(f64_val, 52, 11); |
| 798 | uint64_t f64_frac = extract64(f64_val, 0, 52); |
| 799 | |
| 800 | /* Deal with any special cases */ |
| 801 | if (float64_is_any_nan(f64)) { |
| 802 | float64 nan = f64; |
| 803 | if (float64_is_signaling_nan(f64, fpst)) { |
| 804 | float_raise(float_flag_invalid, fpst); |
| 805 | if (!get_default_nan_mode(fpst)) { |
| 806 | nan = float64_silence_nan(f64, fpst); |
| 807 | } |
| 808 | } |
| 809 | if (get_default_nan_mode(fpst)) { |
| 810 | nan = float64_default_nan(fpst); |
| 811 | } |
| 812 | return nan; |
| 813 | } else if (float64_is_infinity(f64)) { |
| 814 | return float64_set_sign(float64_zero, float64_is_neg(f64)); |
| 815 | } else if (float64_is_zero(f64)) { |
| 816 | float_raise(float_flag_divbyzero, fpst); |
| 817 | return float64_set_sign(float64_infinity, float64_is_neg(f64)); |
| 818 | } else if ((f64_val & ~(1ULL << 63)) < (1ULL << 50)) { |
| 819 | /* Abs(value) < 2.0^-1024 */ |
| 820 | float_raise(float_flag_overflow | float_flag_inexact, fpst); |
| 821 | if (round_to_inf(fpst, f64_sign)) { |
| 822 | return float64_set_sign(float64_infinity, f64_sign); |
| 823 | } else { |
| 824 | return float64_set_sign(float64_maxnorm, f64_sign); |
| 825 | } |
| 826 | } else if (f64_exp >= 2045 && get_flush_to_zero(fpst)) { |
| 827 | float_raise(float_flag_underflow, fpst); |
| 828 | return float64_set_sign(float64_zero, float64_is_neg(f64)); |
| 829 | } |
| 830 | |
| 831 | f64_frac = call_recip_estimate(&f64_exp, 2045, f64_frac, false); |
| 832 | |
| 833 | /* result = sign : result_exp<10:0> : fraction<51:0>; */ |
| 834 | f64_val = deposit64(0, 63, 1, f64_sign); |
| 835 | f64_val = deposit64(f64_val, 52, 11, f64_exp); |
| 836 | f64_val = deposit64(f64_val, 0, 52, f64_frac); |
| 837 | return make_float64(f64_val); |
| 838 | } |
| 839 | |
| 840 | /* The algorithm that must be used to calculate the estimate |
| 841 | * is specified by the ARM ARM. |
| 842 | */ |
| 843 | |
| 844 | static int do_recip_sqrt_estimate(int a) |
| 845 | { |
| 846 | int b, estimate; |
| 847 | |
| 848 | assert(128 <= a && a < 512); |
| 849 | if (a < 256) { |
| 850 | a = a * 2 + 1; |
| 851 | } else { |
| 852 | a = (a >> 1) << 1; |
| 853 | a = (a + 1) * 2; |
| 854 | } |
| 855 | b = 512; |
| 856 | while (a * (b + 1) * (b + 1) < (1 << 28)) { |
| 857 | b += 1; |
| 858 | } |
| 859 | estimate = (b + 1) / 2; |
| 860 | assert(256 <= estimate && estimate < 512); |
| 861 | |
| 862 | return estimate; |
| 863 | } |
| 864 | |
| 865 | static int do_recip_sqrt_estimate_incprec(int a) |
| 866 | { |
| 867 | /* |
| 868 | * The Arm ARM describes the 12-bit precision version of RecipSqrtEstimate |
| 869 | * in terms of an infinite-precision floating point calculation of a |
| 870 | * square root. We implement this using the same kind of pure integer |
| 871 | * algorithm as the 8-bit mantissa, to get the same bit-for-bit result. |
| 872 | */ |
| 873 | int64_t b, estimate; |
| 874 | |
| 875 | assert(1024 <= a && a < 4096); |
| 876 | if (a < 2048) { |
| 877 | a = a * 2 + 1; |
| 878 | } else { |
| 879 | a = (a >> 1) << 1; |
| 880 | a = (a + 1) * 2; |
| 881 | } |
| 882 | b = 8192; |
| 883 | while (a * (b + 1) * (b + 1) < (1ULL << 39)) { |
| 884 | b += 1; |
| 885 | } |
| 886 | estimate = (b + 1) / 2; |
| 887 | |
| 888 | assert(4096 <= estimate && estimate < 8192); |
| 889 | |
| 890 | return estimate; |
| 891 | } |
| 892 | |
| 893 | static uint64_t recip_sqrt_estimate(int *exp , int exp_off, uint64_t frac, |
| 894 | bool increasedprecision) |
| 895 | { |
| 896 | int estimate; |
| 897 | uint32_t scaled; |
| 898 | |
| 899 | if (*exp == 0) { |
| 900 | while (extract64(frac, 51, 1) == 0) { |
| 901 | frac = frac << 1; |
| 902 | *exp -= 1; |
| 903 | } |
| 904 | frac = extract64(frac, 0, 51) << 1; |
| 905 | } |
| 906 | |
| 907 | if (increasedprecision) { |
| 908 | if (*exp & 1) { |
| 909 | /* scaled = UInt('01':fraction<51:42>) */ |
| 910 | scaled = deposit32(1 << 10, 0, 10, extract64(frac, 42, 10)); |
| 911 | } else { |
| 912 | /* scaled = UInt('1':fraction<51:41>) */ |
| 913 | scaled = deposit32(1 << 11, 0, 11, extract64(frac, 41, 11)); |
| 914 | } |
| 915 | estimate = do_recip_sqrt_estimate_incprec(scaled); |
| 916 | } else { |
| 917 | if (*exp & 1) { |
| 918 | /* scaled = UInt('01':fraction<51:45>) */ |
| 919 | scaled = deposit32(1 << 7, 0, 7, extract64(frac, 45, 7)); |
| 920 | } else { |
| 921 | /* scaled = UInt('1':fraction<51:44>) */ |
| 922 | scaled = deposit32(1 << 8, 0, 8, extract64(frac, 44, 8)); |
| 923 | } |
| 924 | estimate = do_recip_sqrt_estimate(scaled); |
| 925 | } |
| 926 | |
| 927 | *exp = (exp_off - *exp) / 2; |
| 928 | if (increasedprecision) { |
| 929 | return extract64(estimate, 0, 12) << 40; |
| 930 | } else { |
| 931 | return extract64(estimate, 0, 8) << 44; |
| 932 | } |
| 933 | } |
| 934 | |
| 935 | uint32_t HELPER(rsqrte_f16)(uint32_t input, float_status *s) |
| 936 | { |
| 937 | float16 f16 = float16_squash_input_denormal(input, s); |
| 938 | uint16_t val = float16_val(f16); |
| 939 | bool f16_sign = float16_is_neg(f16); |
| 940 | int f16_exp = extract32(val, 10, 5); |
| 941 | uint16_t f16_frac = extract32(val, 0, 10); |
| 942 | uint64_t f64_frac; |
| 943 | |
| 944 | if (float16_is_any_nan(f16)) { |
| 945 | float16 nan = f16; |
| 946 | if (float16_is_signaling_nan(f16, s)) { |
| 947 | float_raise(float_flag_invalid, s); |
| 948 | if (!get_default_nan_mode(s)) { |
| 949 | nan = float16_silence_nan(f16, s); |
| 950 | } |
| 951 | } |
| 952 | if (get_default_nan_mode(s)) { |
| 953 | nan = float16_default_nan(s); |
| 954 | } |
| 955 | return nan; |
| 956 | } else if (float16_is_zero(f16)) { |
| 957 | float_raise(float_flag_divbyzero, s); |
| 958 | return float16_set_sign(float16_infinity, f16_sign); |
| 959 | } else if (f16_sign) { |
| 960 | float_raise(float_flag_invalid, s); |
| 961 | return float16_default_nan(s); |
| 962 | } else if (float16_is_infinity(f16)) { |
| 963 | return float16_zero; |
| 964 | } |
| 965 | |
| 966 | /* Scale and normalize to a double-precision value between 0.25 and 1.0, |
| 967 | * preserving the parity of the exponent. */ |
| 968 | |
| 969 | f64_frac = ((uint64_t) f16_frac) << (52 - 10); |
| 970 | |
| 971 | f64_frac = recip_sqrt_estimate(&f16_exp, 44, f64_frac, false); |
| 972 | |
| 973 | /* result = sign : result_exp<4:0> : estimate<7:0> : Zeros(2) */ |
| 974 | val = deposit32(0, 15, 1, f16_sign); |
| 975 | val = deposit32(val, 10, 5, f16_exp); |
| 976 | val = deposit32(val, 2, 8, extract64(f64_frac, 52 - 8, 8)); |
| 977 | return make_float16(val); |
| 978 | } |
| 979 | |
| 980 | /* |
| 981 | * FEAT_RPRES means the f32 FRSQRTE has an "increased precision" variant |
| 982 | * which is used when FPCR.AH == 1. |
| 983 | */ |
| 984 | static float32 do_rsqrte_f32(float32 input, float_status *s, bool rpres) |
| 985 | { |
| 986 | float32 f32 = float32_squash_input_denormal(input, s); |
| 987 | uint32_t val = float32_val(f32); |
| 988 | uint32_t f32_sign = float32_is_neg(f32); |
| 989 | int f32_exp = extract32(val, 23, 8); |
| 990 | uint32_t f32_frac = extract32(val, 0, 23); |
| 991 | uint64_t f64_frac; |
| 992 | |
| 993 | if (float32_is_any_nan(f32)) { |
| 994 | float32 nan = f32; |
| 995 | if (float32_is_signaling_nan(f32, s)) { |
| 996 | float_raise(float_flag_invalid, s); |
| 997 | if (!get_default_nan_mode(s)) { |
| 998 | nan = float32_silence_nan(f32, s); |
| 999 | } |
| 1000 | } |
| 1001 | if (get_default_nan_mode(s)) { |
| 1002 | nan = float32_default_nan(s); |
| 1003 | } |
| 1004 | return nan; |
| 1005 | } else if (float32_is_zero(f32)) { |
| 1006 | float_raise(float_flag_divbyzero, s); |
| 1007 | return float32_set_sign(float32_infinity, float32_is_neg(f32)); |
| 1008 | } else if (float32_is_neg(f32)) { |
| 1009 | float_raise(float_flag_invalid, s); |
| 1010 | return float32_default_nan(s); |
| 1011 | } else if (float32_is_infinity(f32)) { |
| 1012 | return float32_zero; |
| 1013 | } |
| 1014 | |
| 1015 | /* Scale and normalize to a double-precision value between 0.25 and 1.0, |
| 1016 | * preserving the parity of the exponent. */ |
| 1017 | |
| 1018 | f64_frac = ((uint64_t) f32_frac) << 29; |
| 1019 | |
| 1020 | f64_frac = recip_sqrt_estimate(&f32_exp, 380, f64_frac, rpres); |
| 1021 | |
| 1022 | /* |
| 1023 | * result = sign : result_exp<7:0> : estimate<7:0> : Zeros(15) |
| 1024 | * or for increased precision |
| 1025 | * result = sign : result_exp<7:0> : estimate<11:0> : Zeros(11) |
| 1026 | */ |
| 1027 | val = deposit32(0, 31, 1, f32_sign); |
| 1028 | val = deposit32(val, 23, 8, f32_exp); |
| 1029 | if (rpres) { |
| 1030 | val = deposit32(val, 11, 12, extract64(f64_frac, 52 - 12, 12)); |
| 1031 | } else { |
| 1032 | val = deposit32(val, 15, 8, extract64(f64_frac, 52 - 8, 8)); |
| 1033 | } |
| 1034 | return make_float32(val); |
| 1035 | } |
| 1036 | |
| 1037 | float32 HELPER(rsqrte_f32)(float32 input, float_status *s) |
| 1038 | { |
| 1039 | return do_rsqrte_f32(input, s, false); |
| 1040 | } |
| 1041 | |
| 1042 | float32 HELPER(rsqrte_rpres_f32)(float32 input, float_status *s) |
| 1043 | { |
| 1044 | return do_rsqrte_f32(input, s, true); |
| 1045 | } |
| 1046 | |
| 1047 | float64 HELPER(rsqrte_f64)(float64 input, float_status *s) |
| 1048 | { |
| 1049 | float64 f64 = float64_squash_input_denormal(input, s); |
| 1050 | uint64_t val = float64_val(f64); |
| 1051 | bool f64_sign = float64_is_neg(f64); |
| 1052 | int f64_exp = extract64(val, 52, 11); |
| 1053 | uint64_t f64_frac = extract64(val, 0, 52); |
| 1054 | |
| 1055 | if (float64_is_any_nan(f64)) { |
| 1056 | float64 nan = f64; |
| 1057 | if (float64_is_signaling_nan(f64, s)) { |
| 1058 | float_raise(float_flag_invalid, s); |
| 1059 | if (!get_default_nan_mode(s)) { |
| 1060 | nan = float64_silence_nan(f64, s); |
| 1061 | } |
| 1062 | } |
| 1063 | if (get_default_nan_mode(s)) { |
| 1064 | nan = float64_default_nan(s); |
| 1065 | } |
| 1066 | return nan; |
| 1067 | } else if (float64_is_zero(f64)) { |
| 1068 | float_raise(float_flag_divbyzero, s); |
| 1069 | return float64_set_sign(float64_infinity, float64_is_neg(f64)); |
| 1070 | } else if (float64_is_neg(f64)) { |
| 1071 | float_raise(float_flag_invalid, s); |
| 1072 | return float64_default_nan(s); |
| 1073 | } else if (float64_is_infinity(f64)) { |
| 1074 | return float64_zero; |
| 1075 | } |
| 1076 | |
| 1077 | f64_frac = recip_sqrt_estimate(&f64_exp, 3068, f64_frac, false); |
| 1078 | |
| 1079 | /* result = sign : result_exp<4:0> : estimate<7:0> : Zeros(44) */ |
| 1080 | val = deposit64(0, 61, 1, f64_sign); |
| 1081 | val = deposit64(val, 52, 11, f64_exp); |
| 1082 | val = deposit64(val, 44, 8, extract64(f64_frac, 52 - 8, 8)); |
| 1083 | return make_float64(val); |
| 1084 | } |
| 1085 | |
| 1086 | uint32_t HELPER(recpe_u32)(uint32_t a) |
| 1087 | { |
| 1088 | int input, estimate; |
| 1089 | |
| 1090 | if ((a & 0x80000000) == 0) { |
| 1091 | return 0xffffffff; |
| 1092 | } |
| 1093 | |
| 1094 | input = extract32(a, 23, 9); |
| 1095 | estimate = recip_estimate(input); |
| 1096 | |
| 1097 | return deposit32(0, (32 - 9), 9, estimate); |
| 1098 | } |
| 1099 | |
| 1100 | uint32_t HELPER(rsqrte_u32)(uint32_t a) |
| 1101 | { |
| 1102 | int estimate; |
| 1103 | |
| 1104 | if ((a & 0xc0000000) == 0) { |
| 1105 | return 0xffffffff; |
| 1106 | } |
| 1107 | |
| 1108 | estimate = do_recip_sqrt_estimate(extract32(a, 23, 9)); |
| 1109 | |
| 1110 | return deposit32(0, 23, 9, estimate); |
| 1111 | } |
| 1112 | |
| 1113 | /* VFPv4 fused multiply-accumulate */ |
| 1114 | dh_ctype_f16 VFP_HELPER(muladd, h)(dh_ctype_f16 a, dh_ctype_f16 b, |
| 1115 | dh_ctype_f16 c, float_status *fpst) |
| 1116 | { |
| 1117 | return float16_muladd(a, b, c, 0, fpst); |
| 1118 | } |
| 1119 | |
| 1120 | float32 VFP_HELPER(muladd, s)(float32 a, float32 b, float32 c, |
| 1121 | float_status *fpst) |
| 1122 | { |
| 1123 | return float32_muladd(a, b, c, 0, fpst); |
| 1124 | } |
| 1125 | |
| 1126 | float64 VFP_HELPER(muladd, d)(float64 a, float64 b, float64 c, |
| 1127 | float_status *fpst) |
| 1128 | { |
| 1129 | return float64_muladd(a, b, c, 0, fpst); |
| 1130 | } |
| 1131 | |
| 1132 | /* ARMv8 round to integral */ |
| 1133 | dh_ctype_f16 HELPER(rinth_exact)(dh_ctype_f16 x, float_status *fp_status) |
| 1134 | { |
| 1135 | return float16_round_to_int(x, fp_status); |
| 1136 | } |
| 1137 | |
| 1138 | float32 HELPER(rints_exact)(float32 x, float_status *fp_status) |
| 1139 | { |
| 1140 | return float32_round_to_int(x, fp_status); |
| 1141 | } |
| 1142 | |
| 1143 | float64 HELPER(rintd_exact)(float64 x, float_status *fp_status) |
| 1144 | { |
| 1145 | return float64_round_to_int(x, fp_status); |
| 1146 | } |
| 1147 | |
| 1148 | dh_ctype_f16 HELPER(rinth)(dh_ctype_f16 x, float_status *fp_status) |
| 1149 | { |
| 1150 | int old_flags = get_float_exception_flags(fp_status), new_flags; |
| 1151 | float16 ret; |
| 1152 | |
| 1153 | ret = float16_round_to_int(x, fp_status); |
| 1154 | |
| 1155 | /* Suppress any inexact exceptions the conversion produced */ |
| 1156 | if (!(old_flags & float_flag_inexact)) { |
| 1157 | new_flags = get_float_exception_flags(fp_status); |
| 1158 | set_float_exception_flags(new_flags & ~float_flag_inexact, fp_status); |
| 1159 | } |
| 1160 | |
| 1161 | return ret; |
| 1162 | } |
| 1163 | |
| 1164 | float32 HELPER(rints)(float32 x, float_status *fp_status) |
| 1165 | { |
| 1166 | int old_flags = get_float_exception_flags(fp_status), new_flags; |
| 1167 | float32 ret; |
| 1168 | |
| 1169 | ret = float32_round_to_int(x, fp_status); |
| 1170 | |
| 1171 | /* Suppress any inexact exceptions the conversion produced */ |
| 1172 | if (!(old_flags & float_flag_inexact)) { |
| 1173 | new_flags = get_float_exception_flags(fp_status); |
| 1174 | set_float_exception_flags(new_flags & ~float_flag_inexact, fp_status); |
| 1175 | } |
| 1176 | |
| 1177 | return ret; |
| 1178 | } |
| 1179 | |
| 1180 | float64 HELPER(rintd)(float64 x, float_status *fp_status) |
| 1181 | { |
| 1182 | int old_flags = get_float_exception_flags(fp_status), new_flags; |
| 1183 | float64 ret; |
| 1184 | |
| 1185 | ret = float64_round_to_int(x, fp_status); |
| 1186 | |
| 1187 | /* Suppress any inexact exceptions the conversion produced */ |
| 1188 | if (!(old_flags & float_flag_inexact)) { |
| 1189 | new_flags = get_float_exception_flags(fp_status); |
| 1190 | set_float_exception_flags(new_flags & ~float_flag_inexact, fp_status); |
| 1191 | } |
| 1192 | |
| 1193 | return ret; |
| 1194 | } |
| 1195 | |
| 1196 | /* Convert ARM rounding mode to softfloat */ |
| 1197 | const FloatRoundMode arm_rmode_to_sf_map[] = { |
| 1198 | [FPROUNDING_TIEEVEN] = float_round_nearest_even, |
| 1199 | [FPROUNDING_POSINF] = float_round_up, |
| 1200 | [FPROUNDING_NEGINF] = float_round_down, |
| 1201 | [FPROUNDING_ZERO] = float_round_to_zero, |
| 1202 | [FPROUNDING_TIEAWAY] = float_round_ties_away, |
| 1203 | [FPROUNDING_ODD] = float_round_to_odd, |
| 1204 | }; |
| 1205 | |
| 1206 | /* |
| 1207 | * Implement float64 to int32_t conversion without saturation; |
| 1208 | * the result is supplied modulo 2^32. |
| 1209 | */ |
| 1210 | uint64_t HELPER(fjcvtzs)(float64 value, float_status *status) |
| 1211 | { |
| 1212 | uint32_t frac, e_old, e_new; |
| 1213 | bool inexact; |
| 1214 | |
| 1215 | e_old = get_float_exception_flags(status); |
| 1216 | set_float_exception_flags(0, status); |
| 1217 | frac = float64_to_int32_modulo(value, float_round_to_zero, status); |
| 1218 | e_new = get_float_exception_flags(status); |
| 1219 | set_float_exception_flags(e_old | e_new, status); |
| 1220 | |
| 1221 | /* Normal inexact, denormal with flush-to-zero, or overflow or NaN */ |
| 1222 | inexact = e_new & (float_flag_inexact | |
| 1223 | float_flag_input_denormal_flushed | |
| 1224 | float_flag_invalid); |
| 1225 | |
| 1226 | /* While not inexact for IEEE FP, -0.0 is inexact for JavaScript. */ |
| 1227 | inexact |= value == float64_chs(float64_zero); |
| 1228 | |
| 1229 | /* Pack the result and the env->ZF representation of Z together. */ |
| 1230 | return deposit64(frac, 32, 32, inexact); |
| 1231 | } |
| 1232 | |
| 1233 | uint32_t HELPER(vjcvt)(float64 value, CPUARMState *env) |
| 1234 | { |
| 1235 | uint64_t pair = HELPER(fjcvtzs)(value, &env->vfp.fp_status[FPST_A32]); |
| 1236 | uint32_t result = pair; |
| 1237 | uint32_t z = (pair >> 32) == 0; |
| 1238 | |
| 1239 | /* Store Z, clear NCV, in FPSCR.NZCV. */ |
| 1240 | env->vfp.fpsr = (env->vfp.fpsr & ~FPSR_NZCV_MASK) | (z * FPSR_Z); |
| 1241 | |
| 1242 | return result; |
| 1243 | } |
| 1244 | |
| 1245 | /* Round a float32 to an integer that fits in int32_t or int64_t. */ |
| 1246 | static float32 frint_s(float32 f, float_status *fpst, int intsize) |
| 1247 | { |
| 1248 | int old_flags = get_float_exception_flags(fpst); |
| 1249 | uint32_t exp = extract32(f, 23, 8); |
| 1250 | |
| 1251 | if (unlikely(exp == 0xff)) { |
| 1252 | /* NaN or Inf. */ |
| 1253 | goto overflow; |
| 1254 | } |
| 1255 | |
| 1256 | /* Round and re-extract the exponent. */ |
| 1257 | f = float32_round_to_int(f, fpst); |
| 1258 | exp = extract32(f, 23, 8); |
| 1259 | |
| 1260 | /* Validate the range of the result. */ |
| 1261 | if (exp < 126 + intsize) { |
| 1262 | /* abs(F) <= INT{N}_MAX */ |
| 1263 | return f; |
| 1264 | } |
| 1265 | if (exp == 126 + intsize) { |
| 1266 | uint32_t sign = extract32(f, 31, 1); |
| 1267 | uint32_t frac = extract32(f, 0, 23); |
| 1268 | if (sign && frac == 0) { |
| 1269 | /* F == INT{N}_MIN */ |
| 1270 | return f; |
| 1271 | } |
| 1272 | } |
| 1273 | |
| 1274 | overflow: |
| 1275 | /* |
| 1276 | * Raise Invalid and return INT{N}_MIN as a float. Revert any |
| 1277 | * inexact exception float32_round_to_int may have raised. |
| 1278 | */ |
| 1279 | set_float_exception_flags(old_flags | float_flag_invalid, fpst); |
| 1280 | return (0x100u + 126u + intsize) << 23; |
| 1281 | } |
| 1282 | |
| 1283 | float32 HELPER(frint32_s)(float32 f, float_status *fpst) |
| 1284 | { |
| 1285 | return frint_s(f, fpst, 32); |
| 1286 | } |
| 1287 | |
| 1288 | float32 HELPER(frint64_s)(float32 f, float_status *fpst) |
| 1289 | { |
| 1290 | return frint_s(f, fpst, 64); |
| 1291 | } |
| 1292 | |
| 1293 | /* Round a float64 to an integer that fits in int32_t or int64_t. */ |
| 1294 | static float64 frint_d(float64 f, float_status *fpst, int intsize) |
| 1295 | { |
| 1296 | int old_flags = get_float_exception_flags(fpst); |
| 1297 | uint32_t exp = extract64(f, 52, 11); |
| 1298 | |
| 1299 | if (unlikely(exp == 0x7ff)) { |
| 1300 | /* NaN or Inf. */ |
| 1301 | goto overflow; |
| 1302 | } |
| 1303 | |
| 1304 | /* Round and re-extract the exponent. */ |
| 1305 | f = float64_round_to_int(f, fpst); |
| 1306 | exp = extract64(f, 52, 11); |
| 1307 | |
| 1308 | /* Validate the range of the result. */ |
| 1309 | if (exp < 1022 + intsize) { |
| 1310 | /* abs(F) <= INT{N}_MAX */ |
| 1311 | return f; |
| 1312 | } |
| 1313 | if (exp == 1022 + intsize) { |
| 1314 | uint64_t sign = extract64(f, 63, 1); |
| 1315 | uint64_t frac = extract64(f, 0, 52); |
| 1316 | if (sign && frac == 0) { |
| 1317 | /* F == INT{N}_MIN */ |
| 1318 | return f; |
| 1319 | } |
| 1320 | } |
| 1321 | |
| 1322 | overflow: |
| 1323 | /* |
| 1324 | * Raise Invalid and return INT{N}_MIN as a float. Revert any |
| 1325 | * inexact exception float64_round_to_int may have raised. |
| 1326 | */ |
| 1327 | set_float_exception_flags(old_flags | float_flag_invalid, fpst); |
| 1328 | return (uint64_t)(0x800 + 1022 + intsize) << 52; |
| 1329 | } |
| 1330 | |
| 1331 | float64 HELPER(frint32_d)(float64 f, float_status *fpst) |
| 1332 | { |
| 1333 | return frint_d(f, fpst, 32); |
| 1334 | } |
| 1335 | |
| 1336 | float64 HELPER(frint64_d)(float64 f, float_status *fpst) |
| 1337 | { |
| 1338 | return frint_d(f, fpst, 64); |
| 1339 | } |
| 1340 | |
| 1341 | void HELPER(check_hcr_el2_trap)(CPUARMState *env, uint32_t rt, uint32_t reg) |
| 1342 | { |
| 1343 | uint32_t syndrome; |
| 1344 | |
| 1345 | switch (reg) { |
| 1346 | case ARM_VFP_MVFR0: |
| 1347 | case ARM_VFP_MVFR1: |
| 1348 | case ARM_VFP_MVFR2: |
| 1349 | if (!(arm_hcr_el2_eff(env) & HCR_TID3)) { |
| 1350 | return; |
| 1351 | } |
| 1352 | break; |
| 1353 | case ARM_VFP_FPSID: |
| 1354 | if (!(arm_hcr_el2_eff(env) & HCR_TID0)) { |
| 1355 | return; |
| 1356 | } |
| 1357 | break; |
| 1358 | default: |
| 1359 | g_assert_not_reached(); |
| 1360 | } |
| 1361 | |
| 1362 | syndrome = syn_cp10_rt_trap(1, 0xe, 7, reg, rt, 1); |
| 1363 | |
| 1364 | raise_exception(env, EXCP_HYP_TRAP, syndrome, 2); |
| 1365 | } |
| 1366 | |
| 1367 | uint32_t HELPER(vfp_get_fpscr)(CPUARMState *env) |
| 1368 | { |
| 1369 | return vfp_get_fpscr(env); |
| 1370 | } |
| 1371 | |
| 1372 | void HELPER(vfp_set_fpscr)(CPUARMState *env, uint32_t val) |
| 1373 | { |
| 1374 | vfp_set_fpscr(env, val); |
| 1375 | } |