target/i386: Fix pseudo-NaN handling in FPATAN/FYL2XP1/FYL2X helpers
According to Intel's x87 FPU specification (Table 8-10, Vol. 1), arithmetic operations on operands in unsupported formats (including pseudo-NaNs) must return the QNaN floating-point indefinite value. The helper functions for FPATAN, FYL2XP1, and FYL2X incorrectly check for signaling NaN before checking for invalid encodings. This causes pseudo-NaNs to be treated as valid signaling NaNs and silenced, rather than being rejected as unsupported formats. Reorder the checks to test floatx80_invalid_encoding before floatx80_is_signaling_nan, matching the correct behavior already implemented in helper_fscale. Signed-off-by: Max Chou <max.chou@sifive.com> Reviewed-by: Richard Henderson <richard.henderson@linaro.org> Reviewed-by: Chao Liu <chao.liu.zevorn@gmail.com> Signed-off-by: Richard Henderson <richard.henderson@linaro.org> Message-ID: <20260204051756.667397-5-max.chou@sifive.com>
Max Chou committed
Feb 4, 2026 at 13:17 UTC
cf10af6c703d88f2ec0b697da76530f51e9cf0a7
1 file changed
+15
-15
target/i386/tcg/fpu_helper.c
+15
-15
@@ -1377,16 +1377,16 @@ void helper_fpatan(CPUX86State *env)
1377
int32_t arg1_exp = extractFloatx80Exp(ST1);
1378
bool arg1_sign = extractFloatx80Sign(ST1);
1379
1380
- if (floatx80_is_signaling_nan(ST0, &env->fp_status)) {
1380
+ if (floatx80_invalid_encoding(ST0, &env->fp_status) ||
1381
+ floatx80_invalid_encoding(ST1, &env->fp_status)) {
1382
+ float_raise(float_flag_invalid, &env->fp_status);
1383
+ ST1 = floatx80_default_nan(&env->fp_status);
1384
+ } else if (floatx80_is_signaling_nan(ST0, &env->fp_status)) {
1385
float_raise(float_flag_invalid, &env->fp_status);
1386
ST1 = floatx80_silence_nan(ST0, &env->fp_status);
1387
} else if (floatx80_is_signaling_nan(ST1, &env->fp_status)) {
1388
float_raise(float_flag_invalid, &env->fp_status);
1389
ST1 = floatx80_silence_nan(ST1, &env->fp_status);
1386
- } else if (floatx80_invalid_encoding(ST0, &env->fp_status) ||
1387
- floatx80_invalid_encoding(ST1, &env->fp_status)) {
1388
- float_raise(float_flag_invalid, &env->fp_status);
1389
- ST1 = floatx80_default_nan(&env->fp_status);
1390
} else if (floatx80_is_any_nan(ST0)) {
1391
ST1 = ST0;
1392
} else if (floatx80_is_any_nan(ST1)) {
@@ -2061,16 +2061,16 @@ void helper_fyl2xp1(CPUX86State *env)
2061
int32_t arg1_exp = extractFloatx80Exp(ST1);
2062
bool arg1_sign = extractFloatx80Sign(ST1);
2063
2064
- if (floatx80_is_signaling_nan(ST0, &env->fp_status)) {
2064
+ if (floatx80_invalid_encoding(ST0, &env->fp_status) ||
2065
+ floatx80_invalid_encoding(ST1, &env->fp_status)) {
2066
+ float_raise(float_flag_invalid, &env->fp_status);
2067
+ ST1 = floatx80_default_nan(&env->fp_status);
2068
+ } else if (floatx80_is_signaling_nan(ST0, &env->fp_status)) {
2069
float_raise(float_flag_invalid, &env->fp_status);
2070
ST1 = floatx80_silence_nan(ST0, &env->fp_status);
2071
} else if (floatx80_is_signaling_nan(ST1, &env->fp_status)) {
2072
float_raise(float_flag_invalid, &env->fp_status);
2073
ST1 = floatx80_silence_nan(ST1, &env->fp_status);
2070
- } else if (floatx80_invalid_encoding(ST0, &env->fp_status) ||
2071
- floatx80_invalid_encoding(ST1, &env->fp_status)) {
2072
- float_raise(float_flag_invalid, &env->fp_status);
2073
- ST1 = floatx80_default_nan(&env->fp_status);
2074
} else if (floatx80_is_any_nan(ST0)) {
2075
ST1 = ST0;
2076
} else if (floatx80_is_any_nan(ST1)) {
@@ -2159,16 +2159,16 @@ void helper_fyl2x(CPUX86State *env)
2159
int32_t arg1_exp = extractFloatx80Exp(ST1);
2160
bool arg1_sign = extractFloatx80Sign(ST1);
2161
2162
- if (floatx80_is_signaling_nan(ST0, &env->fp_status)) {
2162
+ if (floatx80_invalid_encoding(ST0, &env->fp_status) ||
2163
+ floatx80_invalid_encoding(ST1, &env->fp_status)) {
2164
+ float_raise(float_flag_invalid, &env->fp_status);
2165
+ ST1 = floatx80_default_nan(&env->fp_status);
2166
+ } else if (floatx80_is_signaling_nan(ST0, &env->fp_status)) {
2167
float_raise(float_flag_invalid, &env->fp_status);
2168
ST1 = floatx80_silence_nan(ST0, &env->fp_status);
2169
} else if (floatx80_is_signaling_nan(ST1, &env->fp_status)) {
2170
float_raise(float_flag_invalid, &env->fp_status);
2171
ST1 = floatx80_silence_nan(ST1, &env->fp_status);
2168
- } else if (floatx80_invalid_encoding(ST0, &env->fp_status) ||
2169
- floatx80_invalid_encoding(ST1, &env->fp_status)) {
2170
- float_raise(float_flag_invalid, &env->fp_status);
2171
- ST1 = floatx80_default_nan(&env->fp_status);
2172
} else if (floatx80_is_any_nan(ST0)) {
2173
ST1 = ST0;
2174
} else if (floatx80_is_any_nan(ST1)) {