@samitouri / QOSamiQemu
Code Commits Issues Pulls Sessions CI/CD
master
master
c 2,729 lines 81.9 KB
Raw
1 /*
2 * ARM generic vector expansion
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 * Copyright (c) 2005-2007 CodeSourcery
6 * Copyright (c) 2007 OpenedHand, Ltd.
7 *
8 * This library is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
11 * version 2.1 of the License, or (at your option) any later version.
12 *
13 * This library is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 */
21
22 #include "qemu/osdep.h"
23 #include "translate.h"
24
25
26 static void gen_gvec_fn3_qc(uint32_t rd_ofs, uint32_t rn_ofs, uint32_t rm_ofs,
27 uint32_t opr_sz, uint32_t max_sz,
28 gen_helper_gvec_3_ptr *fn)
29 {
30 TCGv_ptr qc_ptr = tcg_temp_new_ptr();
31
32 tcg_debug_assert(opr_sz <= sizeof_field(CPUARMState, vfp.qc));
33 tcg_gen_addi_ptr(qc_ptr, tcg_env, offsetof(CPUARMState, vfp.qc));
34 tcg_gen_gvec_3_ptr(rd_ofs, rn_ofs, rm_ofs, qc_ptr,
35 opr_sz, max_sz, 0, fn);
36 }
37
38 void gen_gvec_sqdmulh_qc(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
39 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
40 {
41 static gen_helper_gvec_3_ptr * const fns[2] = {
42 gen_helper_neon_sqdmulh_h, gen_helper_neon_sqdmulh_s
43 };
44 tcg_debug_assert(vece >= 1 && vece <= 2);
45 gen_gvec_fn3_qc(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, fns[vece - 1]);
46 }
47
48 void gen_gvec_sqrdmulh_qc(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
49 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
50 {
51 static gen_helper_gvec_3_ptr * const fns[2] = {
52 gen_helper_neon_sqrdmulh_h, gen_helper_neon_sqrdmulh_s
53 };
54 tcg_debug_assert(vece >= 1 && vece <= 2);
55 gen_gvec_fn3_qc(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, fns[vece - 1]);
56 }
57
58 void gen_gvec_sqrdmlah_qc(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
59 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
60 {
61 static gen_helper_gvec_3_ptr * const fns[2] = {
62 gen_helper_gvec_qrdmlah_s16, gen_helper_gvec_qrdmlah_s32
63 };
64 tcg_debug_assert(vece >= 1 && vece <= 2);
65 gen_gvec_fn3_qc(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, fns[vece - 1]);
66 }
67
68 void gen_gvec_sqrdmlsh_qc(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
69 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
70 {
71 static gen_helper_gvec_3_ptr * const fns[2] = {
72 gen_helper_gvec_qrdmlsh_s16, gen_helper_gvec_qrdmlsh_s32
73 };
74 tcg_debug_assert(vece >= 1 && vece <= 2);
75 gen_gvec_fn3_qc(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, fns[vece - 1]);
76 }
77
78 #define GEN_CMP0(NAME, COND) \
79 void NAME(unsigned vece, uint32_t d, uint32_t m, \
80 uint32_t opr_sz, uint32_t max_sz) \
81 { tcg_gen_gvec_cmpi(COND, vece, d, m, 0, opr_sz, max_sz); }
82
83 GEN_CMP0(gen_gvec_ceq0, TCG_COND_EQ)
84 GEN_CMP0(gen_gvec_cle0, TCG_COND_LE)
85 GEN_CMP0(gen_gvec_cge0, TCG_COND_GE)
86 GEN_CMP0(gen_gvec_clt0, TCG_COND_LT)
87 GEN_CMP0(gen_gvec_cgt0, TCG_COND_GT)
88
89 #undef GEN_CMP0
90
91 void gen_gvec_sshr(unsigned vece, uint32_t rd_ofs, uint32_t rm_ofs,
92 int64_t shift, uint32_t opr_sz, uint32_t max_sz)
93 {
94 /* Signed shift out of range results in all-sign-bits */
95 shift = MIN(shift, (8 << vece) - 1);
96 tcg_gen_gvec_sari(vece, rd_ofs, rm_ofs, shift, opr_sz, max_sz);
97 }
98
99 void gen_gvec_ushr(unsigned vece, uint32_t rd_ofs, uint32_t rm_ofs,
100 int64_t shift, uint32_t opr_sz, uint32_t max_sz)
101 {
102 /* Unsigned shift out of range results in all-zero-bits */
103 if (shift >= (8 << vece)) {
104 tcg_gen_gvec_dup_imm(vece, rd_ofs, opr_sz, max_sz, 0);
105 } else {
106 tcg_gen_gvec_shri(vece, rd_ofs, rm_ofs, shift, opr_sz, max_sz);
107 }
108 }
109
110 static void gen_ssra8_i64(TCGv_i64 d, TCGv_i64 a, int64_t shift)
111 {
112 tcg_gen_vec_sar8i_i64(a, a, shift);
113 tcg_gen_vec_add8_i64(d, d, a);
114 }
115
116 static void gen_ssra16_i64(TCGv_i64 d, TCGv_i64 a, int64_t shift)
117 {
118 tcg_gen_vec_sar16i_i64(a, a, shift);
119 tcg_gen_vec_add16_i64(d, d, a);
120 }
121
122 static void gen_ssra32_i32(TCGv_i32 d, TCGv_i32 a, int32_t shift)
123 {
124 tcg_gen_sari_i32(a, a, shift);
125 tcg_gen_add_i32(d, d, a);
126 }
127
128 static void gen_ssra64_i64(TCGv_i64 d, TCGv_i64 a, int64_t shift)
129 {
130 tcg_gen_sari_i64(a, a, shift);
131 tcg_gen_add_i64(d, d, a);
132 }
133
134 static void gen_ssra_vec(unsigned vece, TCGv_vec d, TCGv_vec a, int64_t sh)
135 {
136 tcg_gen_sari_vec(vece, a, a, sh);
137 tcg_gen_add_vec(vece, d, d, a);
138 }
139
140 void gen_gvec_ssra(unsigned vece, uint32_t rd_ofs, uint32_t rm_ofs,
141 int64_t shift, uint32_t opr_sz, uint32_t max_sz)
142 {
143 static const TCGOpcode vecop_list[] = {
144 INDEX_op_sari_vec, INDEX_op_add_vec, 0
145 };
146 static const GVecGen2i ops[4] = {
147 { .fni8 = gen_ssra8_i64,
148 .fniv = gen_ssra_vec,
149 .fno = gen_helper_gvec_ssra_b,
150 .load_dest = true,
151 .opt_opc = vecop_list,
152 .vece = MO_8 },
153 { .fni8 = gen_ssra16_i64,
154 .fniv = gen_ssra_vec,
155 .fno = gen_helper_gvec_ssra_h,
156 .load_dest = true,
157 .opt_opc = vecop_list,
158 .vece = MO_16 },
159 { .fni4 = gen_ssra32_i32,
160 .fniv = gen_ssra_vec,
161 .fno = gen_helper_gvec_ssra_s,
162 .load_dest = true,
163 .opt_opc = vecop_list,
164 .vece = MO_32 },
165 { .fni8 = gen_ssra64_i64,
166 .fniv = gen_ssra_vec,
167 .fno = gen_helper_gvec_ssra_d,
168 .prefer_i64 = true,
169 .opt_opc = vecop_list,
170 .load_dest = true,
171 .vece = MO_64 },
172 };
173
174 /* tszimm encoding produces immediates in the range [1..esize]. */
175 tcg_debug_assert(shift > 0);
176 tcg_debug_assert(shift <= (8 << vece));
177
178 /*
179 * Shifts larger than the element size are architecturally valid.
180 * Signed results in all sign bits.
181 */
182 shift = MIN(shift, (8 << vece) - 1);
183 tcg_gen_gvec_2i(rd_ofs, rm_ofs, opr_sz, max_sz, shift, &ops[vece]);
184 }
185
186 static void gen_usra8_i64(TCGv_i64 d, TCGv_i64 a, int64_t shift)
187 {
188 tcg_gen_vec_shr8i_i64(a, a, shift);
189 tcg_gen_vec_add8_i64(d, d, a);
190 }
191
192 static void gen_usra16_i64(TCGv_i64 d, TCGv_i64 a, int64_t shift)
193 {
194 tcg_gen_vec_shr16i_i64(a, a, shift);
195 tcg_gen_vec_add16_i64(d, d, a);
196 }
197
198 static void gen_usra32_i32(TCGv_i32 d, TCGv_i32 a, int32_t shift)
199 {
200 tcg_gen_shri_i32(a, a, shift);
201 tcg_gen_add_i32(d, d, a);
202 }
203
204 static void gen_usra64_i64(TCGv_i64 d, TCGv_i64 a, int64_t shift)
205 {
206 tcg_gen_shri_i64(a, a, shift);
207 tcg_gen_add_i64(d, d, a);
208 }
209
210 static void gen_usra_vec(unsigned vece, TCGv_vec d, TCGv_vec a, int64_t sh)
211 {
212 tcg_gen_shri_vec(vece, a, a, sh);
213 tcg_gen_add_vec(vece, d, d, a);
214 }
215
216 void gen_gvec_usra(unsigned vece, uint32_t rd_ofs, uint32_t rm_ofs,
217 int64_t shift, uint32_t opr_sz, uint32_t max_sz)
218 {
219 static const TCGOpcode vecop_list[] = {
220 INDEX_op_shri_vec, INDEX_op_add_vec, 0
221 };
222 static const GVecGen2i ops[4] = {
223 { .fni8 = gen_usra8_i64,
224 .fniv = gen_usra_vec,
225 .fno = gen_helper_gvec_usra_b,
226 .load_dest = true,
227 .opt_opc = vecop_list,
228 .vece = MO_8, },
229 { .fni8 = gen_usra16_i64,
230 .fniv = gen_usra_vec,
231 .fno = gen_helper_gvec_usra_h,
232 .load_dest = true,
233 .opt_opc = vecop_list,
234 .vece = MO_16, },
235 { .fni4 = gen_usra32_i32,
236 .fniv = gen_usra_vec,
237 .fno = gen_helper_gvec_usra_s,
238 .load_dest = true,
239 .opt_opc = vecop_list,
240 .vece = MO_32, },
241 { .fni8 = gen_usra64_i64,
242 .fniv = gen_usra_vec,
243 .fno = gen_helper_gvec_usra_d,
244 .prefer_i64 = true,
245 .load_dest = true,
246 .opt_opc = vecop_list,
247 .vece = MO_64, },
248 };
249
250 /* tszimm encoding produces immediates in the range [1..esize]. */
251 tcg_debug_assert(shift > 0);
252 tcg_debug_assert(shift <= (8 << vece));
253
254 /*
255 * Shifts larger than the element size are architecturally valid.
256 * Unsigned results in all zeros as input to accumulate: nop.
257 */
258 if (shift < (8 << vece)) {
259 tcg_gen_gvec_2i(rd_ofs, rm_ofs, opr_sz, max_sz, shift, &ops[vece]);
260 } else {
261 /* Nop, but we do need to clear the tail. */
262 tcg_gen_gvec_mov(vece, rd_ofs, rd_ofs, opr_sz, max_sz);
263 }
264 }
265
266 /*
267 * Shift one less than the requested amount, and the low bit is
268 * the rounding bit. For the 8 and 16-bit operations, because we
269 * mask the low bit, we can perform a normal integer shift instead
270 * of a vector shift.
271 */
272 static void gen_srshr8_i64(TCGv_i64 d, TCGv_i64 a, int64_t sh)
273 {
274 TCGv_i64 t = tcg_temp_new_i64();
275
276 tcg_gen_shri_i64(t, a, sh - 1);
277 tcg_gen_andi_i64(t, t, dup_const(MO_8, 1));
278 tcg_gen_vec_sar8i_i64(d, a, sh);
279 tcg_gen_vec_add8_i64(d, d, t);
280 }
281
282 static void gen_srshr16_i64(TCGv_i64 d, TCGv_i64 a, int64_t sh)
283 {
284 TCGv_i64 t = tcg_temp_new_i64();
285
286 tcg_gen_shri_i64(t, a, sh - 1);
287 tcg_gen_andi_i64(t, t, dup_const(MO_16, 1));
288 tcg_gen_vec_sar16i_i64(d, a, sh);
289 tcg_gen_vec_add16_i64(d, d, t);
290 }
291
292 void gen_srshr32_i32(TCGv_i32 d, TCGv_i32 a, int32_t sh)
293 {
294 TCGv_i32 t;
295
296 /* Handle shift by the input size for the benefit of trans_SRSHR_ri */
297 if (sh == 32) {
298 tcg_gen_movi_i32(d, 0);
299 return;
300 }
301 t = tcg_temp_new_i32();
302 tcg_gen_extract_i32(t, a, sh - 1, 1);
303 tcg_gen_sari_i32(d, a, sh);
304 tcg_gen_add_i32(d, d, t);
305 }
306
307 void gen_srshr64_i64(TCGv_i64 d, TCGv_i64 a, int64_t sh)
308 {
309 TCGv_i64 t = tcg_temp_new_i64();
310
311 tcg_gen_extract_i64(t, a, sh - 1, 1);
312 tcg_gen_sari_i64(d, a, sh);
313 tcg_gen_add_i64(d, d, t);
314 }
315
316 static void gen_srshr_vec(unsigned vece, TCGv_vec d, TCGv_vec a, int64_t sh)
317 {
318 TCGv_vec t = tcg_temp_new_vec_matching(d);
319 TCGv_vec ones = tcg_constant_vec_matching(d, vece, 1);
320
321 tcg_gen_shri_vec(vece, t, a, sh - 1);
322 tcg_gen_and_vec(vece, t, t, ones);
323 tcg_gen_sari_vec(vece, d, a, sh);
324 tcg_gen_add_vec(vece, d, d, t);
325 }
326
327 void gen_gvec_srshr(unsigned vece, uint32_t rd_ofs, uint32_t rm_ofs,
328 int64_t shift, uint32_t opr_sz, uint32_t max_sz)
329 {
330 static const TCGOpcode vecop_list[] = {
331 INDEX_op_shri_vec, INDEX_op_sari_vec, INDEX_op_add_vec, 0
332 };
333 static const GVecGen2i ops[4] = {
334 { .fni8 = gen_srshr8_i64,
335 .fniv = gen_srshr_vec,
336 .fno = gen_helper_gvec_srshr_b,
337 .opt_opc = vecop_list,
338 .vece = MO_8 },
339 { .fni8 = gen_srshr16_i64,
340 .fniv = gen_srshr_vec,
341 .fno = gen_helper_gvec_srshr_h,
342 .opt_opc = vecop_list,
343 .vece = MO_16 },
344 { .fni4 = gen_srshr32_i32,
345 .fniv = gen_srshr_vec,
346 .fno = gen_helper_gvec_srshr_s,
347 .opt_opc = vecop_list,
348 .vece = MO_32 },
349 { .fni8 = gen_srshr64_i64,
350 .fniv = gen_srshr_vec,
351 .fno = gen_helper_gvec_srshr_d,
352 .prefer_i64 = true,
353 .opt_opc = vecop_list,
354 .vece = MO_64 },
355 };
356
357 /* tszimm encoding produces immediates in the range [1..esize] */
358 tcg_debug_assert(shift > 0);
359 tcg_debug_assert(shift <= (8 << vece));
360
361 if (shift == (8 << vece)) {
362 /*
363 * Shifts larger than the element size are architecturally valid.
364 * Signed results in all sign bits. With rounding, this produces
365 * (-1 + 1) >> 1 == 0, or (0 + 1) >> 1 == 0.
366 * I.e. always zero.
367 */
368 tcg_gen_gvec_dup_imm(vece, rd_ofs, opr_sz, max_sz, 0);
369 } else {
370 tcg_gen_gvec_2i(rd_ofs, rm_ofs, opr_sz, max_sz, shift, &ops[vece]);
371 }
372 }
373
374 static void gen_srsra8_i64(TCGv_i64 d, TCGv_i64 a, int64_t sh)
375 {
376 TCGv_i64 t = tcg_temp_new_i64();
377
378 gen_srshr8_i64(t, a, sh);
379 tcg_gen_vec_add8_i64(d, d, t);
380 }
381
382 static void gen_srsra16_i64(TCGv_i64 d, TCGv_i64 a, int64_t sh)
383 {
384 TCGv_i64 t = tcg_temp_new_i64();
385
386 gen_srshr16_i64(t, a, sh);
387 tcg_gen_vec_add16_i64(d, d, t);
388 }
389
390 static void gen_srsra32_i32(TCGv_i32 d, TCGv_i32 a, int32_t sh)
391 {
392 TCGv_i32 t = tcg_temp_new_i32();
393
394 gen_srshr32_i32(t, a, sh);
395 tcg_gen_add_i32(d, d, t);
396 }
397
398 static void gen_srsra64_i64(TCGv_i64 d, TCGv_i64 a, int64_t sh)
399 {
400 TCGv_i64 t = tcg_temp_new_i64();
401
402 gen_srshr64_i64(t, a, sh);
403 tcg_gen_add_i64(d, d, t);
404 }
405
406 static void gen_srsra_vec(unsigned vece, TCGv_vec d, TCGv_vec a, int64_t sh)
407 {
408 TCGv_vec t = tcg_temp_new_vec_matching(d);
409
410 gen_srshr_vec(vece, t, a, sh);
411 tcg_gen_add_vec(vece, d, d, t);
412 }
413
414 void gen_gvec_srsra(unsigned vece, uint32_t rd_ofs, uint32_t rm_ofs,
415 int64_t shift, uint32_t opr_sz, uint32_t max_sz)
416 {
417 static const TCGOpcode vecop_list[] = {
418 INDEX_op_shri_vec, INDEX_op_sari_vec, INDEX_op_add_vec, 0
419 };
420 static const GVecGen2i ops[4] = {
421 { .fni8 = gen_srsra8_i64,
422 .fniv = gen_srsra_vec,
423 .fno = gen_helper_gvec_srsra_b,
424 .opt_opc = vecop_list,
425 .load_dest = true,
426 .vece = MO_8 },
427 { .fni8 = gen_srsra16_i64,
428 .fniv = gen_srsra_vec,
429 .fno = gen_helper_gvec_srsra_h,
430 .opt_opc = vecop_list,
431 .load_dest = true,
432 .vece = MO_16 },
433 { .fni4 = gen_srsra32_i32,
434 .fniv = gen_srsra_vec,
435 .fno = gen_helper_gvec_srsra_s,
436 .opt_opc = vecop_list,
437 .load_dest = true,
438 .vece = MO_32 },
439 { .fni8 = gen_srsra64_i64,
440 .fniv = gen_srsra_vec,
441 .fno = gen_helper_gvec_srsra_d,
442 .prefer_i64 = true,
443 .opt_opc = vecop_list,
444 .load_dest = true,
445 .vece = MO_64 },
446 };
447
448 /* tszimm encoding produces immediates in the range [1..esize] */
449 tcg_debug_assert(shift > 0);
450 tcg_debug_assert(shift <= (8 << vece));
451
452 /*
453 * Shifts larger than the element size are architecturally valid.
454 * Signed results in all sign bits. With rounding, this produces
455 * (-1 + 1) >> 1 == 0, or (0 + 1) >> 1 == 0.
456 * I.e. always zero. With accumulation, this leaves D unchanged.
457 */
458 if (shift == (8 << vece)) {
459 /* Nop, but we do need to clear the tail. */
460 tcg_gen_gvec_mov(vece, rd_ofs, rd_ofs, opr_sz, max_sz);
461 } else {
462 tcg_gen_gvec_2i(rd_ofs, rm_ofs, opr_sz, max_sz, shift, &ops[vece]);
463 }
464 }
465
466 static void gen_urshr8_i64(TCGv_i64 d, TCGv_i64 a, int64_t sh)
467 {
468 TCGv_i64 t = tcg_temp_new_i64();
469
470 tcg_gen_shri_i64(t, a, sh - 1);
471 tcg_gen_andi_i64(t, t, dup_const(MO_8, 1));
472 tcg_gen_vec_shr8i_i64(d, a, sh);
473 tcg_gen_vec_add8_i64(d, d, t);
474 }
475
476 static void gen_urshr16_i64(TCGv_i64 d, TCGv_i64 a, int64_t sh)
477 {
478 TCGv_i64 t = tcg_temp_new_i64();
479
480 tcg_gen_shri_i64(t, a, sh - 1);
481 tcg_gen_andi_i64(t, t, dup_const(MO_16, 1));
482 tcg_gen_vec_shr16i_i64(d, a, sh);
483 tcg_gen_vec_add16_i64(d, d, t);
484 }
485
486 void gen_urshr32_i32(TCGv_i32 d, TCGv_i32 a, int32_t sh)
487 {
488 TCGv_i32 t;
489
490 /* Handle shift by the input size for the benefit of trans_URSHR_ri */
491 if (sh == 32) {
492 tcg_gen_extract_i32(d, a, sh - 1, 1);
493 return;
494 }
495 t = tcg_temp_new_i32();
496 tcg_gen_extract_i32(t, a, sh - 1, 1);
497 tcg_gen_shri_i32(d, a, sh);
498 tcg_gen_add_i32(d, d, t);
499 }
500
501 void gen_urshr64_i64(TCGv_i64 d, TCGv_i64 a, int64_t sh)
502 {
503 TCGv_i64 t = tcg_temp_new_i64();
504
505 tcg_gen_extract_i64(t, a, sh - 1, 1);
506 tcg_gen_shri_i64(d, a, sh);
507 tcg_gen_add_i64(d, d, t);
508 }
509
510 static void gen_urshr_vec(unsigned vece, TCGv_vec d, TCGv_vec a, int64_t shift)
511 {
512 TCGv_vec t = tcg_temp_new_vec_matching(d);
513 TCGv_vec ones = tcg_constant_vec_matching(d, vece, 1);
514
515 tcg_gen_shri_vec(vece, t, a, shift - 1);
516 tcg_gen_and_vec(vece, t, t, ones);
517 tcg_gen_shri_vec(vece, d, a, shift);
518 tcg_gen_add_vec(vece, d, d, t);
519 }
520
521 void gen_gvec_urshr(unsigned vece, uint32_t rd_ofs, uint32_t rm_ofs,
522 int64_t shift, uint32_t opr_sz, uint32_t max_sz)
523 {
524 static const TCGOpcode vecop_list[] = {
525 INDEX_op_shri_vec, INDEX_op_add_vec, 0
526 };
527 static const GVecGen2i ops[4] = {
528 { .fni8 = gen_urshr8_i64,
529 .fniv = gen_urshr_vec,
530 .fno = gen_helper_gvec_urshr_b,
531 .opt_opc = vecop_list,
532 .vece = MO_8 },
533 { .fni8 = gen_urshr16_i64,
534 .fniv = gen_urshr_vec,
535 .fno = gen_helper_gvec_urshr_h,
536 .opt_opc = vecop_list,
537 .vece = MO_16 },
538 { .fni4 = gen_urshr32_i32,
539 .fniv = gen_urshr_vec,
540 .fno = gen_helper_gvec_urshr_s,
541 .opt_opc = vecop_list,
542 .vece = MO_32 },
543 { .fni8 = gen_urshr64_i64,
544 .fniv = gen_urshr_vec,
545 .fno = gen_helper_gvec_urshr_d,
546 .prefer_i64 = true,
547 .opt_opc = vecop_list,
548 .vece = MO_64 },
549 };
550
551 /* tszimm encoding produces immediates in the range [1..esize] */
552 tcg_debug_assert(shift > 0);
553 tcg_debug_assert(shift <= (8 << vece));
554
555 if (shift == (8 << vece)) {
556 /*
557 * Shifts larger than the element size are architecturally valid.
558 * Unsigned results in zero. With rounding, this produces a
559 * copy of the most significant bit.
560 */
561 tcg_gen_gvec_shri(vece, rd_ofs, rm_ofs, shift - 1, opr_sz, max_sz);
562 } else {
563 tcg_gen_gvec_2i(rd_ofs, rm_ofs, opr_sz, max_sz, shift, &ops[vece]);
564 }
565 }
566
567 static void gen_ursra8_i64(TCGv_i64 d, TCGv_i64 a, int64_t sh)
568 {
569 TCGv_i64 t = tcg_temp_new_i64();
570
571 if (sh == 8) {
572 tcg_gen_vec_shr8i_i64(t, a, 7);
573 } else {
574 gen_urshr8_i64(t, a, sh);
575 }
576 tcg_gen_vec_add8_i64(d, d, t);
577 }
578
579 static void gen_ursra16_i64(TCGv_i64 d, TCGv_i64 a, int64_t sh)
580 {
581 TCGv_i64 t = tcg_temp_new_i64();
582
583 if (sh == 16) {
584 tcg_gen_vec_shr16i_i64(t, a, 15);
585 } else {
586 gen_urshr16_i64(t, a, sh);
587 }
588 tcg_gen_vec_add16_i64(d, d, t);
589 }
590
591 static void gen_ursra32_i32(TCGv_i32 d, TCGv_i32 a, int32_t sh)
592 {
593 TCGv_i32 t = tcg_temp_new_i32();
594
595 if (sh == 32) {
596 tcg_gen_shri_i32(t, a, 31);
597 } else {
598 gen_urshr32_i32(t, a, sh);
599 }
600 tcg_gen_add_i32(d, d, t);
601 }
602
603 static void gen_ursra64_i64(TCGv_i64 d, TCGv_i64 a, int64_t sh)
604 {
605 TCGv_i64 t = tcg_temp_new_i64();
606
607 if (sh == 64) {
608 tcg_gen_shri_i64(t, a, 63);
609 } else {
610 gen_urshr64_i64(t, a, sh);
611 }
612 tcg_gen_add_i64(d, d, t);
613 }
614
615 static void gen_ursra_vec(unsigned vece, TCGv_vec d, TCGv_vec a, int64_t sh)
616 {
617 TCGv_vec t = tcg_temp_new_vec_matching(d);
618
619 if (sh == (8 << vece)) {
620 tcg_gen_shri_vec(vece, t, a, sh - 1);
621 } else {
622 gen_urshr_vec(vece, t, a, sh);
623 }
624 tcg_gen_add_vec(vece, d, d, t);
625 }
626
627 void gen_gvec_ursra(unsigned vece, uint32_t rd_ofs, uint32_t rm_ofs,
628 int64_t shift, uint32_t opr_sz, uint32_t max_sz)
629 {
630 static const TCGOpcode vecop_list[] = {
631 INDEX_op_shri_vec, INDEX_op_add_vec, 0
632 };
633 static const GVecGen2i ops[4] = {
634 { .fni8 = gen_ursra8_i64,
635 .fniv = gen_ursra_vec,
636 .fno = gen_helper_gvec_ursra_b,
637 .opt_opc = vecop_list,
638 .load_dest = true,
639 .vece = MO_8 },
640 { .fni8 = gen_ursra16_i64,
641 .fniv = gen_ursra_vec,
642 .fno = gen_helper_gvec_ursra_h,
643 .opt_opc = vecop_list,
644 .load_dest = true,
645 .vece = MO_16 },
646 { .fni4 = gen_ursra32_i32,
647 .fniv = gen_ursra_vec,
648 .fno = gen_helper_gvec_ursra_s,
649 .opt_opc = vecop_list,
650 .load_dest = true,
651 .vece = MO_32 },
652 { .fni8 = gen_ursra64_i64,
653 .fniv = gen_ursra_vec,
654 .fno = gen_helper_gvec_ursra_d,
655 .prefer_i64 = true,
656 .opt_opc = vecop_list,
657 .load_dest = true,
658 .vece = MO_64 },
659 };
660
661 /* tszimm encoding produces immediates in the range [1..esize] */
662 tcg_debug_assert(shift > 0);
663 tcg_debug_assert(shift <= (8 << vece));
664
665 tcg_gen_gvec_2i(rd_ofs, rm_ofs, opr_sz, max_sz, shift, &ops[vece]);
666 }
667
668 static void gen_shr8_ins_i64(TCGv_i64 d, TCGv_i64 a, int64_t shift)
669 {
670 uint64_t mask = dup_const(MO_8, 0xff >> shift);
671 TCGv_i64 t = tcg_temp_new_i64();
672
673 tcg_gen_shri_i64(t, a, shift);
674 tcg_gen_andi_i64(t, t, mask);
675 tcg_gen_andi_i64(d, d, ~mask);
676 tcg_gen_or_i64(d, d, t);
677 }
678
679 static void gen_shr16_ins_i64(TCGv_i64 d, TCGv_i64 a, int64_t shift)
680 {
681 uint64_t mask = dup_const(MO_16, 0xffff >> shift);
682 TCGv_i64 t = tcg_temp_new_i64();
683
684 tcg_gen_shri_i64(t, a, shift);
685 tcg_gen_andi_i64(t, t, mask);
686 tcg_gen_andi_i64(d, d, ~mask);
687 tcg_gen_or_i64(d, d, t);
688 }
689
690 static void gen_shr32_ins_i32(TCGv_i32 d, TCGv_i32 a, int32_t shift)
691 {
692 tcg_gen_shri_i32(a, a, shift);
693 tcg_gen_deposit_i32(d, d, a, 0, 32 - shift);
694 }
695
696 static void gen_shr64_ins_i64(TCGv_i64 d, TCGv_i64 a, int64_t shift)
697 {
698 tcg_gen_shri_i64(a, a, shift);
699 tcg_gen_deposit_i64(d, d, a, 0, 64 - shift);
700 }
701
702 static void gen_shr_ins_vec(unsigned vece, TCGv_vec d, TCGv_vec a, int64_t sh)
703 {
704 TCGv_vec t = tcg_temp_new_vec_matching(d);
705 int64_t mi = MAKE_64BIT_MASK((8 << vece) - sh, sh);
706 TCGv_vec m = tcg_constant_vec_matching(d, vece, mi);
707
708 tcg_gen_shri_vec(vece, t, a, sh);
709 tcg_gen_and_vec(vece, d, d, m);
710 tcg_gen_or_vec(vece, d, d, t);
711 }
712
713 void gen_gvec_sri(unsigned vece, uint32_t rd_ofs, uint32_t rm_ofs,
714 int64_t shift, uint32_t opr_sz, uint32_t max_sz)
715 {
716 static const TCGOpcode vecop_list[] = { INDEX_op_shri_vec, 0 };
717 const GVecGen2i ops[4] = {
718 { .fni8 = gen_shr8_ins_i64,
719 .fniv = gen_shr_ins_vec,
720 .fno = gen_helper_gvec_sri_b,
721 .load_dest = true,
722 .opt_opc = vecop_list,
723 .vece = MO_8 },
724 { .fni8 = gen_shr16_ins_i64,
725 .fniv = gen_shr_ins_vec,
726 .fno = gen_helper_gvec_sri_h,
727 .load_dest = true,
728 .opt_opc = vecop_list,
729 .vece = MO_16 },
730 { .fni4 = gen_shr32_ins_i32,
731 .fniv = gen_shr_ins_vec,
732 .fno = gen_helper_gvec_sri_s,
733 .load_dest = true,
734 .opt_opc = vecop_list,
735 .vece = MO_32 },
736 { .fni8 = gen_shr64_ins_i64,
737 .fniv = gen_shr_ins_vec,
738 .fno = gen_helper_gvec_sri_d,
739 .prefer_i64 = true,
740 .load_dest = true,
741 .opt_opc = vecop_list,
742 .vece = MO_64 },
743 };
744
745 /* tszimm encoding produces immediates in the range [1..esize]. */
746 tcg_debug_assert(shift > 0);
747 tcg_debug_assert(shift <= (8 << vece));
748
749 /* Shift of esize leaves destination unchanged. */
750 if (shift < (8 << vece)) {
751 tcg_gen_gvec_2i(rd_ofs, rm_ofs, opr_sz, max_sz, shift, &ops[vece]);
752 } else {
753 /* Nop, but we do need to clear the tail. */
754 tcg_gen_gvec_mov(vece, rd_ofs, rd_ofs, opr_sz, max_sz);
755 }
756 }
757
758 static void gen_shl8_ins_i64(TCGv_i64 d, TCGv_i64 a, int64_t shift)
759 {
760 uint64_t mask = dup_const(MO_8, 0xff << shift);
761 TCGv_i64 t = tcg_temp_new_i64();
762
763 tcg_gen_shli_i64(t, a, shift);
764 tcg_gen_andi_i64(t, t, mask);
765 tcg_gen_andi_i64(d, d, ~mask);
766 tcg_gen_or_i64(d, d, t);
767 }
768
769 static void gen_shl16_ins_i64(TCGv_i64 d, TCGv_i64 a, int64_t shift)
770 {
771 uint64_t mask = dup_const(MO_16, 0xffff << shift);
772 TCGv_i64 t = tcg_temp_new_i64();
773
774 tcg_gen_shli_i64(t, a, shift);
775 tcg_gen_andi_i64(t, t, mask);
776 tcg_gen_andi_i64(d, d, ~mask);
777 tcg_gen_or_i64(d, d, t);
778 }
779
780 static void gen_shl32_ins_i32(TCGv_i32 d, TCGv_i32 a, int32_t shift)
781 {
782 tcg_gen_deposit_i32(d, d, a, shift, 32 - shift);
783 }
784
785 static void gen_shl64_ins_i64(TCGv_i64 d, TCGv_i64 a, int64_t shift)
786 {
787 tcg_gen_deposit_i64(d, d, a, shift, 64 - shift);
788 }
789
790 static void gen_shl_ins_vec(unsigned vece, TCGv_vec d, TCGv_vec a, int64_t sh)
791 {
792 TCGv_vec t = tcg_temp_new_vec_matching(d);
793 TCGv_vec m = tcg_constant_vec_matching(d, vece, MAKE_64BIT_MASK(0, sh));
794
795 tcg_gen_shli_vec(vece, t, a, sh);
796 tcg_gen_and_vec(vece, d, d, m);
797 tcg_gen_or_vec(vece, d, d, t);
798 }
799
800 void gen_gvec_sli(unsigned vece, uint32_t rd_ofs, uint32_t rm_ofs,
801 int64_t shift, uint32_t opr_sz, uint32_t max_sz)
802 {
803 static const TCGOpcode vecop_list[] = { INDEX_op_shli_vec, 0 };
804 const GVecGen2i ops[4] = {
805 { .fni8 = gen_shl8_ins_i64,
806 .fniv = gen_shl_ins_vec,
807 .fno = gen_helper_gvec_sli_b,
808 .load_dest = true,
809 .opt_opc = vecop_list,
810 .vece = MO_8 },
811 { .fni8 = gen_shl16_ins_i64,
812 .fniv = gen_shl_ins_vec,
813 .fno = gen_helper_gvec_sli_h,
814 .load_dest = true,
815 .opt_opc = vecop_list,
816 .vece = MO_16 },
817 { .fni4 = gen_shl32_ins_i32,
818 .fniv = gen_shl_ins_vec,
819 .fno = gen_helper_gvec_sli_s,
820 .load_dest = true,
821 .opt_opc = vecop_list,
822 .vece = MO_32 },
823 { .fni8 = gen_shl64_ins_i64,
824 .fniv = gen_shl_ins_vec,
825 .fno = gen_helper_gvec_sli_d,
826 .prefer_i64 = true,
827 .load_dest = true,
828 .opt_opc = vecop_list,
829 .vece = MO_64 },
830 };
831
832 /* tszimm encoding produces immediates in the range [0..esize-1]. */
833 tcg_debug_assert(shift >= 0);
834 tcg_debug_assert(shift < (8 << vece));
835
836 if (shift == 0) {
837 tcg_gen_gvec_mov(vece, rd_ofs, rm_ofs, opr_sz, max_sz);
838 } else {
839 tcg_gen_gvec_2i(rd_ofs, rm_ofs, opr_sz, max_sz, shift, &ops[vece]);
840 }
841 }
842
843 static void gen_mla8_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
844 {
845 gen_helper_neon_mul_u8(a, a, b);
846 gen_helper_neon_add_u8(d, d, a);
847 }
848
849 static void gen_mls8_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
850 {
851 gen_helper_neon_mul_u8(a, a, b);
852 gen_helper_neon_sub_u8(d, d, a);
853 }
854
855 static void gen_mla16_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
856 {
857 gen_helper_neon_mul_u16(a, a, b);
858 gen_helper_neon_add_u16(d, d, a);
859 }
860
861 static void gen_mls16_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
862 {
863 gen_helper_neon_mul_u16(a, a, b);
864 gen_helper_neon_sub_u16(d, d, a);
865 }
866
867 static void gen_mla32_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
868 {
869 tcg_gen_mul_i32(a, a, b);
870 tcg_gen_add_i32(d, d, a);
871 }
872
873 static void gen_mls32_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
874 {
875 tcg_gen_mul_i32(a, a, b);
876 tcg_gen_sub_i32(d, d, a);
877 }
878
879 static void gen_mla64_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
880 {
881 tcg_gen_mul_i64(a, a, b);
882 tcg_gen_add_i64(d, d, a);
883 }
884
885 static void gen_mls64_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
886 {
887 tcg_gen_mul_i64(a, a, b);
888 tcg_gen_sub_i64(d, d, a);
889 }
890
891 static void gen_mla_vec(unsigned vece, TCGv_vec d, TCGv_vec a, TCGv_vec b)
892 {
893 tcg_gen_mul_vec(vece, a, a, b);
894 tcg_gen_add_vec(vece, d, d, a);
895 }
896
897 static void gen_mls_vec(unsigned vece, TCGv_vec d, TCGv_vec a, TCGv_vec b)
898 {
899 tcg_gen_mul_vec(vece, a, a, b);
900 tcg_gen_sub_vec(vece, d, d, a);
901 }
902
903 /* Note that while NEON does not support VMLA and VMLS as 64-bit ops,
904 * these tables are shared with AArch64 which does support them.
905 */
906 void gen_gvec_mla(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
907 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
908 {
909 static const TCGOpcode vecop_list[] = {
910 INDEX_op_mul_vec, INDEX_op_add_vec, 0
911 };
912 static const GVecGen3 ops[4] = {
913 { .fni4 = gen_mla8_i32,
914 .fniv = gen_mla_vec,
915 .load_dest = true,
916 .opt_opc = vecop_list,
917 .vece = MO_8 },
918 { .fni4 = gen_mla16_i32,
919 .fniv = gen_mla_vec,
920 .load_dest = true,
921 .opt_opc = vecop_list,
922 .vece = MO_16 },
923 { .fni4 = gen_mla32_i32,
924 .fniv = gen_mla_vec,
925 .load_dest = true,
926 .opt_opc = vecop_list,
927 .vece = MO_32 },
928 { .fni8 = gen_mla64_i64,
929 .fniv = gen_mla_vec,
930 .prefer_i64 = true,
931 .load_dest = true,
932 .opt_opc = vecop_list,
933 .vece = MO_64 },
934 };
935 tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, &ops[vece]);
936 }
937
938 void gen_gvec_mls(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
939 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
940 {
941 static const TCGOpcode vecop_list[] = {
942 INDEX_op_mul_vec, INDEX_op_sub_vec, 0
943 };
944 static const GVecGen3 ops[4] = {
945 { .fni4 = gen_mls8_i32,
946 .fniv = gen_mls_vec,
947 .load_dest = true,
948 .opt_opc = vecop_list,
949 .vece = MO_8 },
950 { .fni4 = gen_mls16_i32,
951 .fniv = gen_mls_vec,
952 .load_dest = true,
953 .opt_opc = vecop_list,
954 .vece = MO_16 },
955 { .fni4 = gen_mls32_i32,
956 .fniv = gen_mls_vec,
957 .load_dest = true,
958 .opt_opc = vecop_list,
959 .vece = MO_32 },
960 { .fni8 = gen_mls64_i64,
961 .fniv = gen_mls_vec,
962 .prefer_i64 = true,
963 .load_dest = true,
964 .opt_opc = vecop_list,
965 .vece = MO_64 },
966 };
967 tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, &ops[vece]);
968 }
969
970 /* CMTST : test is "if (X & Y != 0)". */
971 static void gen_cmtst_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
972 {
973 tcg_gen_negsetcond_i32(TCG_COND_TSTNE, d, a, b);
974 }
975
976 void gen_cmtst_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
977 {
978 tcg_gen_negsetcond_i64(TCG_COND_TSTNE, d, a, b);
979 }
980
981 static void gen_cmtst_vec(unsigned vece, TCGv_vec d, TCGv_vec a, TCGv_vec b)
982 {
983 tcg_gen_cmp_vec(TCG_COND_TSTNE, vece, d, a, b);
984 }
985
986 void gen_gvec_cmtst(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
987 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
988 {
989 static const TCGOpcode vecop_list[] = { INDEX_op_cmp_vec, 0 };
990 static const GVecGen3 ops[4] = {
991 { .fni4 = gen_helper_neon_tst_u8,
992 .fniv = gen_cmtst_vec,
993 .opt_opc = vecop_list,
994 .vece = MO_8 },
995 { .fni4 = gen_helper_neon_tst_u16,
996 .fniv = gen_cmtst_vec,
997 .opt_opc = vecop_list,
998 .vece = MO_16 },
999 { .fni4 = gen_cmtst_i32,
1000 .fniv = gen_cmtst_vec,
1001 .opt_opc = vecop_list,
1002 .vece = MO_32 },
1003 { .fni8 = gen_cmtst_i64,
1004 .fniv = gen_cmtst_vec,
1005 .prefer_i64 = true,
1006 .opt_opc = vecop_list,
1007 .vece = MO_64 },
1008 };
1009 tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, &ops[vece]);
1010 }
1011
1012 void gen_ushl_i32(TCGv_i32 dst, TCGv_i32 src, TCGv_i32 shift)
1013 {
1014 TCGv_i32 lval = tcg_temp_new_i32();
1015 TCGv_i32 rval = tcg_temp_new_i32();
1016 TCGv_i32 lsh = tcg_temp_new_i32();
1017 TCGv_i32 rsh = tcg_temp_new_i32();
1018 TCGv_i32 zero = tcg_constant_i32(0);
1019 TCGv_i32 max = tcg_constant_i32(32);
1020
1021 /*
1022 * Rely on the TCG guarantee that out of range shifts produce
1023 * unspecified results, not undefined behaviour (i.e. no trap).
1024 * Discard out-of-range results after the fact.
1025 */
1026 tcg_gen_ext8s_i32(lsh, shift);
1027 tcg_gen_neg_i32(rsh, lsh);
1028 tcg_gen_shl_i32(lval, src, lsh);
1029 tcg_gen_shr_i32(rval, src, rsh);
1030 tcg_gen_movcond_i32(TCG_COND_LTU, dst, lsh, max, lval, zero);
1031 tcg_gen_movcond_i32(TCG_COND_LTU, dst, rsh, max, rval, dst);
1032 }
1033
1034 void gen_ushl_i64(TCGv_i64 dst, TCGv_i64 src, TCGv_i64 shift)
1035 {
1036 TCGv_i64 lval = tcg_temp_new_i64();
1037 TCGv_i64 rval = tcg_temp_new_i64();
1038 TCGv_i64 lsh = tcg_temp_new_i64();
1039 TCGv_i64 rsh = tcg_temp_new_i64();
1040 TCGv_i64 zero = tcg_constant_i64(0);
1041 TCGv_i64 max = tcg_constant_i64(64);
1042
1043 /*
1044 * Rely on the TCG guarantee that out of range shifts produce
1045 * unspecified results, not undefined behaviour (i.e. no trap).
1046 * Discard out-of-range results after the fact.
1047 */
1048 tcg_gen_ext8s_i64(lsh, shift);
1049 tcg_gen_neg_i64(rsh, lsh);
1050 tcg_gen_shl_i64(lval, src, lsh);
1051 tcg_gen_shr_i64(rval, src, rsh);
1052 tcg_gen_movcond_i64(TCG_COND_LTU, dst, lsh, max, lval, zero);
1053 tcg_gen_movcond_i64(TCG_COND_LTU, dst, rsh, max, rval, dst);
1054 }
1055
1056 static void gen_ushl_vec(unsigned vece, TCGv_vec dst,
1057 TCGv_vec src, TCGv_vec shift)
1058 {
1059 TCGv_vec lval = tcg_temp_new_vec_matching(dst);
1060 TCGv_vec rval = tcg_temp_new_vec_matching(dst);
1061 TCGv_vec lsh = tcg_temp_new_vec_matching(dst);
1062 TCGv_vec rsh = tcg_temp_new_vec_matching(dst);
1063 TCGv_vec max, zero;
1064
1065 tcg_gen_neg_vec(vece, rsh, shift);
1066 if (vece == MO_8) {
1067 tcg_gen_mov_vec(lsh, shift);
1068 } else {
1069 TCGv_vec msk = tcg_constant_vec_matching(dst, vece, 0xff);
1070 tcg_gen_and_vec(vece, lsh, shift, msk);
1071 tcg_gen_and_vec(vece, rsh, rsh, msk);
1072 }
1073
1074 /*
1075 * Rely on the TCG guarantee that out of range shifts produce
1076 * unspecified results, not undefined behaviour (i.e. no trap).
1077 * Discard out-of-range results after the fact.
1078 */
1079 tcg_gen_shlv_vec(vece, lval, src, lsh);
1080 tcg_gen_shrv_vec(vece, rval, src, rsh);
1081
1082 /*
1083 * The choice of GE (signed) and GEU (unsigned) are biased toward
1084 * the instructions of the x86_64 host. For MO_8, the whole byte
1085 * is significant so we must use an unsigned compare; otherwise we
1086 * have already masked to a byte and so a signed compare works.
1087 * Other tcg hosts have a full set of comparisons and do not care.
1088 */
1089 zero = tcg_constant_vec_matching(dst, vece, 0);
1090 max = tcg_constant_vec_matching(dst, vece, 8 << vece);
1091 if (vece == MO_8) {
1092 tcg_gen_cmpsel_vec(TCG_COND_GEU, vece, lval, lsh, max, zero, lval);
1093 tcg_gen_cmpsel_vec(TCG_COND_GEU, vece, rval, rsh, max, zero, rval);
1094 } else {
1095 tcg_gen_cmpsel_vec(TCG_COND_GE, vece, lval, lsh, max, zero, lval);
1096 tcg_gen_cmpsel_vec(TCG_COND_GE, vece, rval, rsh, max, zero, rval);
1097 }
1098 tcg_gen_or_vec(vece, dst, lval, rval);
1099 }
1100
1101 void gen_gvec_ushl(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
1102 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
1103 {
1104 static const TCGOpcode vecop_list[] = {
1105 INDEX_op_neg_vec, INDEX_op_shlv_vec,
1106 INDEX_op_shrv_vec, INDEX_op_cmpsel_vec, 0
1107 };
1108 static const GVecGen3 ops[4] = {
1109 { .fniv = gen_ushl_vec,
1110 .fno = gen_helper_gvec_ushl_b,
1111 .opt_opc = vecop_list,
1112 .vece = MO_8 },
1113 { .fniv = gen_ushl_vec,
1114 .fno = gen_helper_gvec_ushl_h,
1115 .opt_opc = vecop_list,
1116 .vece = MO_16 },
1117 { .fni4 = gen_ushl_i32,
1118 .fniv = gen_ushl_vec,
1119 .opt_opc = vecop_list,
1120 .vece = MO_32 },
1121 { .fni8 = gen_ushl_i64,
1122 .fniv = gen_ushl_vec,
1123 .opt_opc = vecop_list,
1124 .vece = MO_64 },
1125 };
1126 tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, &ops[vece]);
1127 }
1128
1129 void gen_sshl_i32(TCGv_i32 dst, TCGv_i32 src, TCGv_i32 shift)
1130 {
1131 TCGv_i32 lval = tcg_temp_new_i32();
1132 TCGv_i32 rval = tcg_temp_new_i32();
1133 TCGv_i32 lsh = tcg_temp_new_i32();
1134 TCGv_i32 rsh = tcg_temp_new_i32();
1135 TCGv_i32 zero = tcg_constant_i32(0);
1136 TCGv_i32 max = tcg_constant_i32(31);
1137
1138 /*
1139 * Rely on the TCG guarantee that out of range shifts produce
1140 * unspecified results, not undefined behaviour (i.e. no trap).
1141 * Discard out-of-range results after the fact.
1142 */
1143 tcg_gen_ext8s_i32(lsh, shift);
1144 tcg_gen_neg_i32(rsh, lsh);
1145 tcg_gen_shl_i32(lval, src, lsh);
1146 tcg_gen_umin_i32(rsh, rsh, max);
1147 tcg_gen_sar_i32(rval, src, rsh);
1148 tcg_gen_movcond_i32(TCG_COND_LEU, lval, lsh, max, lval, zero);
1149 tcg_gen_movcond_i32(TCG_COND_LT, dst, lsh, zero, rval, lval);
1150 }
1151
1152 void gen_sshl_i64(TCGv_i64 dst, TCGv_i64 src, TCGv_i64 shift)
1153 {
1154 TCGv_i64 lval = tcg_temp_new_i64();
1155 TCGv_i64 rval = tcg_temp_new_i64();
1156 TCGv_i64 lsh = tcg_temp_new_i64();
1157 TCGv_i64 rsh = tcg_temp_new_i64();
1158 TCGv_i64 zero = tcg_constant_i64(0);
1159 TCGv_i64 max = tcg_constant_i64(63);
1160
1161 /*
1162 * Rely on the TCG guarantee that out of range shifts produce
1163 * unspecified results, not undefined behaviour (i.e. no trap).
1164 * Discard out-of-range results after the fact.
1165 */
1166 tcg_gen_ext8s_i64(lsh, shift);
1167 tcg_gen_neg_i64(rsh, lsh);
1168 tcg_gen_shl_i64(lval, src, lsh);
1169 tcg_gen_umin_i64(rsh, rsh, max);
1170 tcg_gen_sar_i64(rval, src, rsh);
1171 tcg_gen_movcond_i64(TCG_COND_LEU, lval, lsh, max, lval, zero);
1172 tcg_gen_movcond_i64(TCG_COND_LT, dst, lsh, zero, rval, lval);
1173 }
1174
1175 static void gen_sshl_vec(unsigned vece, TCGv_vec dst,
1176 TCGv_vec src, TCGv_vec shift)
1177 {
1178 TCGv_vec lval = tcg_temp_new_vec_matching(dst);
1179 TCGv_vec rval = tcg_temp_new_vec_matching(dst);
1180 TCGv_vec lsh = tcg_temp_new_vec_matching(dst);
1181 TCGv_vec rsh = tcg_temp_new_vec_matching(dst);
1182 TCGv_vec max, zero;
1183
1184 /*
1185 * Rely on the TCG guarantee that out of range shifts produce
1186 * unspecified results, not undefined behaviour (i.e. no trap).
1187 * Discard out-of-range results after the fact.
1188 */
1189 tcg_gen_neg_vec(vece, rsh, shift);
1190 if (vece == MO_8) {
1191 tcg_gen_mov_vec(lsh, shift);
1192 } else {
1193 TCGv_vec msk = tcg_constant_vec_matching(dst, vece, 0xff);
1194 tcg_gen_and_vec(vece, lsh, shift, msk);
1195 tcg_gen_and_vec(vece, rsh, rsh, msk);
1196 }
1197
1198 /* Bound rsh so out of bound right shift gets -1. */
1199 max = tcg_constant_vec_matching(dst, vece, (8 << vece) - 1);
1200 tcg_gen_umin_vec(vece, rsh, rsh, max);
1201
1202 tcg_gen_shlv_vec(vece, lval, src, lsh);
1203 tcg_gen_sarv_vec(vece, rval, src, rsh);
1204
1205 /* Select in-bound left shift. */
1206 zero = tcg_constant_vec_matching(dst, vece, 0);
1207 tcg_gen_cmpsel_vec(TCG_COND_GT, vece, lval, lsh, max, zero, lval);
1208
1209 /* Select between left and right shift. */
1210 if (vece == MO_8) {
1211 tcg_gen_cmpsel_vec(TCG_COND_LT, vece, dst, lsh, zero, rval, lval);
1212 } else {
1213 TCGv_vec sgn = tcg_constant_vec_matching(dst, vece, 0x80);
1214 tcg_gen_cmpsel_vec(TCG_COND_LT, vece, dst, lsh, sgn, lval, rval);
1215 }
1216 }
1217
1218 void gen_gvec_sshl(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
1219 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
1220 {
1221 static const TCGOpcode vecop_list[] = {
1222 INDEX_op_neg_vec, INDEX_op_umin_vec, INDEX_op_shlv_vec,
1223 INDEX_op_sarv_vec, INDEX_op_cmpsel_vec, 0
1224 };
1225 static const GVecGen3 ops[4] = {
1226 { .fniv = gen_sshl_vec,
1227 .fno = gen_helper_gvec_sshl_b,
1228 .opt_opc = vecop_list,
1229 .vece = MO_8 },
1230 { .fniv = gen_sshl_vec,
1231 .fno = gen_helper_gvec_sshl_h,
1232 .opt_opc = vecop_list,
1233 .vece = MO_16 },
1234 { .fni4 = gen_sshl_i32,
1235 .fniv = gen_sshl_vec,
1236 .opt_opc = vecop_list,
1237 .vece = MO_32 },
1238 { .fni8 = gen_sshl_i64,
1239 .fniv = gen_sshl_vec,
1240 .opt_opc = vecop_list,
1241 .vece = MO_64 },
1242 };
1243 tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, &ops[vece]);
1244 }
1245
1246 void gen_gvec_srshl(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
1247 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
1248 {
1249 static gen_helper_gvec_3 * const fns[] = {
1250 gen_helper_gvec_srshl_b, gen_helper_gvec_srshl_h,
1251 gen_helper_gvec_srshl_s, gen_helper_gvec_srshl_d,
1252 };
1253 tcg_debug_assert(vece <= MO_64);
1254 tcg_gen_gvec_3_ool(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, 0, fns[vece]);
1255 }
1256
1257 void gen_gvec_urshl(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
1258 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
1259 {
1260 static gen_helper_gvec_3 * const fns[] = {
1261 gen_helper_gvec_urshl_b, gen_helper_gvec_urshl_h,
1262 gen_helper_gvec_urshl_s, gen_helper_gvec_urshl_d,
1263 };
1264 tcg_debug_assert(vece <= MO_64);
1265 tcg_gen_gvec_3_ool(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, 0, fns[vece]);
1266 }
1267
1268 void gen_neon_sqshl(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
1269 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
1270 {
1271 static gen_helper_gvec_3_ptr * const fns[] = {
1272 gen_helper_neon_sqshl_b, gen_helper_neon_sqshl_h,
1273 gen_helper_neon_sqshl_s, gen_helper_neon_sqshl_d,
1274 };
1275 tcg_debug_assert(vece <= MO_64);
1276 tcg_gen_gvec_3_ptr(rd_ofs, rn_ofs, rm_ofs, tcg_env,
1277 opr_sz, max_sz, 0, fns[vece]);
1278 }
1279
1280 void gen_neon_uqshl(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
1281 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
1282 {
1283 static gen_helper_gvec_3_ptr * const fns[] = {
1284 gen_helper_neon_uqshl_b, gen_helper_neon_uqshl_h,
1285 gen_helper_neon_uqshl_s, gen_helper_neon_uqshl_d,
1286 };
1287 tcg_debug_assert(vece <= MO_64);
1288 tcg_gen_gvec_3_ptr(rd_ofs, rn_ofs, rm_ofs, tcg_env,
1289 opr_sz, max_sz, 0, fns[vece]);
1290 }
1291
1292 void gen_neon_sqrshl(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
1293 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
1294 {
1295 static gen_helper_gvec_3_ptr * const fns[] = {
1296 gen_helper_neon_sqrshl_b, gen_helper_neon_sqrshl_h,
1297 gen_helper_neon_sqrshl_s, gen_helper_neon_sqrshl_d,
1298 };
1299 tcg_debug_assert(vece <= MO_64);
1300 tcg_gen_gvec_3_ptr(rd_ofs, rn_ofs, rm_ofs, tcg_env,
1301 opr_sz, max_sz, 0, fns[vece]);
1302 }
1303
1304 void gen_neon_uqrshl(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
1305 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
1306 {
1307 static gen_helper_gvec_3_ptr * const fns[] = {
1308 gen_helper_neon_uqrshl_b, gen_helper_neon_uqrshl_h,
1309 gen_helper_neon_uqrshl_s, gen_helper_neon_uqrshl_d,
1310 };
1311 tcg_debug_assert(vece <= MO_64);
1312 tcg_gen_gvec_3_ptr(rd_ofs, rn_ofs, rm_ofs, tcg_env,
1313 opr_sz, max_sz, 0, fns[vece]);
1314 }
1315
1316 void gen_neon_sqshli(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
1317 int64_t c, uint32_t opr_sz, uint32_t max_sz)
1318 {
1319 static gen_helper_gvec_2_ptr * const fns[] = {
1320 gen_helper_neon_sqshli_b, gen_helper_neon_sqshli_h,
1321 gen_helper_neon_sqshli_s, gen_helper_neon_sqshli_d,
1322 };
1323 tcg_debug_assert(vece <= MO_64);
1324 tcg_debug_assert(c >= 0 && c <= (8 << vece));
1325 tcg_gen_gvec_2_ptr(rd_ofs, rn_ofs, tcg_env, opr_sz, max_sz, c, fns[vece]);
1326 }
1327
1328 void gen_neon_uqshli(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
1329 int64_t c, uint32_t opr_sz, uint32_t max_sz)
1330 {
1331 static gen_helper_gvec_2_ptr * const fns[] = {
1332 gen_helper_neon_uqshli_b, gen_helper_neon_uqshli_h,
1333 gen_helper_neon_uqshli_s, gen_helper_neon_uqshli_d,
1334 };
1335 tcg_debug_assert(vece <= MO_64);
1336 tcg_debug_assert(c >= 0 && c <= (8 << vece));
1337 tcg_gen_gvec_2_ptr(rd_ofs, rn_ofs, tcg_env, opr_sz, max_sz, c, fns[vece]);
1338 }
1339
1340 void gen_neon_sqshlui(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
1341 int64_t c, uint32_t opr_sz, uint32_t max_sz)
1342 {
1343 static gen_helper_gvec_2_ptr * const fns[] = {
1344 gen_helper_neon_sqshlui_b, gen_helper_neon_sqshlui_h,
1345 gen_helper_neon_sqshlui_s, gen_helper_neon_sqshlui_d,
1346 };
1347 tcg_debug_assert(vece <= MO_64);
1348 tcg_debug_assert(c >= 0 && c <= (8 << vece));
1349 tcg_gen_gvec_2_ptr(rd_ofs, rn_ofs, tcg_env, opr_sz, max_sz, c, fns[vece]);
1350 }
1351
1352 void gen_uqadd_bhs(TCGv_i64 res, TCGv_i64 qc, TCGv_i64 a, TCGv_i64 b, MemOp esz)
1353 {
1354 uint64_t max = MAKE_64BIT_MASK(0, 8 << esz);
1355 TCGv_i64 tmp = tcg_temp_new_i64();
1356
1357 tcg_gen_add_i64(tmp, a, b);
1358 tcg_gen_umin_i64(res, tmp, tcg_constant_i64(max));
1359 tcg_gen_xor_i64(tmp, tmp, res);
1360 tcg_gen_or_i64(qc, qc, tmp);
1361 }
1362
1363 void gen_uqadd_d(TCGv_i64 res, TCGv_i64 qc, TCGv_i64 a, TCGv_i64 b)
1364 {
1365 TCGv_i64 t = tcg_temp_new_i64();
1366
1367 tcg_gen_add_i64(t, a, b);
1368 tcg_gen_movcond_i64(TCG_COND_LTU, res, t, a,
1369 tcg_constant_i64(UINT64_MAX), t);
1370 tcg_gen_xor_i64(t, t, res);
1371 tcg_gen_or_i64(qc, qc, t);
1372 }
1373
1374 static void gen_uqadd_vec(unsigned vece, TCGv_vec t, TCGv_vec qc,
1375 TCGv_vec a, TCGv_vec b)
1376 {
1377 TCGv_vec x = tcg_temp_new_vec_matching(t);
1378 tcg_gen_add_vec(vece, x, a, b);
1379 tcg_gen_usadd_vec(vece, t, a, b);
1380 tcg_gen_xor_vec(vece, x, x, t);
1381 tcg_gen_or_vec(vece, qc, qc, x);
1382 }
1383
1384 void gen_gvec_uqadd_qc(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
1385 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
1386 {
1387 static const TCGOpcode vecop_list[] = {
1388 INDEX_op_usadd_vec, INDEX_op_add_vec, 0
1389 };
1390 static const GVecGen4 ops[4] = {
1391 { .fniv = gen_uqadd_vec,
1392 .fno = gen_helper_gvec_uqadd_b,
1393 .write_aofs = true,
1394 .opt_opc = vecop_list,
1395 .vece = MO_8 },
1396 { .fniv = gen_uqadd_vec,
1397 .fno = gen_helper_gvec_uqadd_h,
1398 .write_aofs = true,
1399 .opt_opc = vecop_list,
1400 .vece = MO_16 },
1401 { .fniv = gen_uqadd_vec,
1402 .fno = gen_helper_gvec_uqadd_s,
1403 .write_aofs = true,
1404 .opt_opc = vecop_list,
1405 .vece = MO_32 },
1406 { .fniv = gen_uqadd_vec,
1407 .fni8 = gen_uqadd_d,
1408 .fno = gen_helper_gvec_uqadd_d,
1409 .write_aofs = true,
1410 .opt_opc = vecop_list,
1411 .vece = MO_64 },
1412 };
1413
1414 tcg_debug_assert(opr_sz <= sizeof_field(CPUARMState, vfp.qc));
1415 tcg_gen_gvec_4(rd_ofs, offsetof(CPUARMState, vfp.qc),
1416 rn_ofs, rm_ofs, opr_sz, max_sz, &ops[vece]);
1417 }
1418
1419 void gen_sqadd_bhs(TCGv_i64 res, TCGv_i64 qc, TCGv_i64 a, TCGv_i64 b, MemOp esz)
1420 {
1421 int64_t max = MAKE_64BIT_MASK(0, (8 << esz) - 1);
1422 int64_t min = -1ll - max;
1423 TCGv_i64 tmp = tcg_temp_new_i64();
1424
1425 tcg_gen_add_i64(tmp, a, b);
1426 tcg_gen_smin_i64(res, tmp, tcg_constant_i64(max));
1427 tcg_gen_smax_i64(res, res, tcg_constant_i64(min));
1428 tcg_gen_xor_i64(tmp, tmp, res);
1429 tcg_gen_or_i64(qc, qc, tmp);
1430 }
1431
1432 void gen_sqadd_d(TCGv_i64 res, TCGv_i64 qc, TCGv_i64 a, TCGv_i64 b)
1433 {
1434 TCGv_i64 t0 = tcg_temp_new_i64();
1435 TCGv_i64 t1 = tcg_temp_new_i64();
1436 TCGv_i64 t2 = tcg_temp_new_i64();
1437
1438 tcg_gen_add_i64(t0, a, b);
1439
1440 /* Compute signed overflow indication into T1 */
1441 tcg_gen_xor_i64(t1, a, b);
1442 tcg_gen_xor_i64(t2, t0, a);
1443 tcg_gen_andc_i64(t1, t2, t1);
1444
1445 /* Compute saturated value into T2 */
1446 tcg_gen_sari_i64(t2, a, 63);
1447 tcg_gen_xori_i64(t2, t2, INT64_MAX);
1448
1449 tcg_gen_movcond_i64(TCG_COND_LT, res, t1, tcg_constant_i64(0), t2, t0);
1450 tcg_gen_xor_i64(t0, t0, res);
1451 tcg_gen_or_i64(qc, qc, t0);
1452 }
1453
1454 static void gen_sqadd_vec(unsigned vece, TCGv_vec t, TCGv_vec qc,
1455 TCGv_vec a, TCGv_vec b)
1456 {
1457 TCGv_vec x = tcg_temp_new_vec_matching(t);
1458 tcg_gen_add_vec(vece, x, a, b);
1459 tcg_gen_ssadd_vec(vece, t, a, b);
1460 tcg_gen_xor_vec(vece, x, x, t);
1461 tcg_gen_or_vec(vece, qc, qc, x);
1462 }
1463
1464 void gen_gvec_sqadd_qc(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
1465 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
1466 {
1467 static const TCGOpcode vecop_list[] = {
1468 INDEX_op_ssadd_vec, INDEX_op_add_vec, 0
1469 };
1470 static const GVecGen4 ops[4] = {
1471 { .fniv = gen_sqadd_vec,
1472 .fno = gen_helper_gvec_sqadd_b,
1473 .opt_opc = vecop_list,
1474 .write_aofs = true,
1475 .vece = MO_8 },
1476 { .fniv = gen_sqadd_vec,
1477 .fno = gen_helper_gvec_sqadd_h,
1478 .opt_opc = vecop_list,
1479 .write_aofs = true,
1480 .vece = MO_16 },
1481 { .fniv = gen_sqadd_vec,
1482 .fno = gen_helper_gvec_sqadd_s,
1483 .opt_opc = vecop_list,
1484 .write_aofs = true,
1485 .vece = MO_32 },
1486 { .fniv = gen_sqadd_vec,
1487 .fni8 = gen_sqadd_d,
1488 .fno = gen_helper_gvec_sqadd_d,
1489 .opt_opc = vecop_list,
1490 .write_aofs = true,
1491 .vece = MO_64 },
1492 };
1493
1494 tcg_debug_assert(opr_sz <= sizeof_field(CPUARMState, vfp.qc));
1495 tcg_gen_gvec_4(rd_ofs, offsetof(CPUARMState, vfp.qc),
1496 rn_ofs, rm_ofs, opr_sz, max_sz, &ops[vece]);
1497 }
1498
1499 void gen_uqsub_bhs(TCGv_i64 res, TCGv_i64 qc, TCGv_i64 a, TCGv_i64 b, MemOp esz)
1500 {
1501 TCGv_i64 tmp = tcg_temp_new_i64();
1502
1503 tcg_gen_sub_i64(tmp, a, b);
1504 tcg_gen_smax_i64(res, tmp, tcg_constant_i64(0));
1505 tcg_gen_xor_i64(tmp, tmp, res);
1506 tcg_gen_or_i64(qc, qc, tmp);
1507 }
1508
1509 void gen_uqsub_d(TCGv_i64 res, TCGv_i64 qc, TCGv_i64 a, TCGv_i64 b)
1510 {
1511 TCGv_i64 t = tcg_temp_new_i64();
1512
1513 tcg_gen_sub_i64(t, a, b);
1514 tcg_gen_movcond_i64(TCG_COND_LTU, res, a, b, tcg_constant_i64(0), t);
1515 tcg_gen_xor_i64(t, t, res);
1516 tcg_gen_or_i64(qc, qc, t);
1517 }
1518
1519 static void gen_uqsub_vec(unsigned vece, TCGv_vec t, TCGv_vec qc,
1520 TCGv_vec a, TCGv_vec b)
1521 {
1522 TCGv_vec x = tcg_temp_new_vec_matching(t);
1523 tcg_gen_sub_vec(vece, x, a, b);
1524 tcg_gen_ussub_vec(vece, t, a, b);
1525 tcg_gen_xor_vec(vece, x, x, t);
1526 tcg_gen_or_vec(vece, qc, qc, x);
1527 }
1528
1529 void gen_gvec_uqsub_qc(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
1530 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
1531 {
1532 static const TCGOpcode vecop_list[] = {
1533 INDEX_op_ussub_vec, INDEX_op_sub_vec, 0
1534 };
1535 static const GVecGen4 ops[4] = {
1536 { .fniv = gen_uqsub_vec,
1537 .fno = gen_helper_gvec_uqsub_b,
1538 .opt_opc = vecop_list,
1539 .write_aofs = true,
1540 .vece = MO_8 },
1541 { .fniv = gen_uqsub_vec,
1542 .fno = gen_helper_gvec_uqsub_h,
1543 .opt_opc = vecop_list,
1544 .write_aofs = true,
1545 .vece = MO_16 },
1546 { .fniv = gen_uqsub_vec,
1547 .fno = gen_helper_gvec_uqsub_s,
1548 .opt_opc = vecop_list,
1549 .write_aofs = true,
1550 .vece = MO_32 },
1551 { .fniv = gen_uqsub_vec,
1552 .fni8 = gen_uqsub_d,
1553 .fno = gen_helper_gvec_uqsub_d,
1554 .opt_opc = vecop_list,
1555 .write_aofs = true,
1556 .vece = MO_64 },
1557 };
1558
1559 tcg_debug_assert(opr_sz <= sizeof_field(CPUARMState, vfp.qc));
1560 tcg_gen_gvec_4(rd_ofs, offsetof(CPUARMState, vfp.qc),
1561 rn_ofs, rm_ofs, opr_sz, max_sz, &ops[vece]);
1562 }
1563
1564 void gen_sqsub_bhs(TCGv_i64 res, TCGv_i64 qc, TCGv_i64 a, TCGv_i64 b, MemOp esz)
1565 {
1566 int64_t max = MAKE_64BIT_MASK(0, (8 << esz) - 1);
1567 int64_t min = -1ll - max;
1568 TCGv_i64 tmp = tcg_temp_new_i64();
1569
1570 tcg_gen_sub_i64(tmp, a, b);
1571 tcg_gen_smin_i64(res, tmp, tcg_constant_i64(max));
1572 tcg_gen_smax_i64(res, res, tcg_constant_i64(min));
1573 tcg_gen_xor_i64(tmp, tmp, res);
1574 tcg_gen_or_i64(qc, qc, tmp);
1575 }
1576
1577 void gen_sqsub_d(TCGv_i64 res, TCGv_i64 qc, TCGv_i64 a, TCGv_i64 b)
1578 {
1579 TCGv_i64 t0 = tcg_temp_new_i64();
1580 TCGv_i64 t1 = tcg_temp_new_i64();
1581 TCGv_i64 t2 = tcg_temp_new_i64();
1582
1583 tcg_gen_sub_i64(t0, a, b);
1584
1585 /* Compute signed overflow indication into T1 */
1586 tcg_gen_xor_i64(t1, a, b);
1587 tcg_gen_xor_i64(t2, t0, a);
1588 tcg_gen_and_i64(t1, t1, t2);
1589
1590 /* Compute saturated value into T2 */
1591 tcg_gen_sari_i64(t2, a, 63);
1592 tcg_gen_xori_i64(t2, t2, INT64_MAX);
1593
1594 tcg_gen_movcond_i64(TCG_COND_LT, res, t1, tcg_constant_i64(0), t2, t0);
1595 tcg_gen_xor_i64(t0, t0, res);
1596 tcg_gen_or_i64(qc, qc, t0);
1597 }
1598
1599 static void gen_sqsub_vec(unsigned vece, TCGv_vec t, TCGv_vec qc,
1600 TCGv_vec a, TCGv_vec b)
1601 {
1602 TCGv_vec x = tcg_temp_new_vec_matching(t);
1603 tcg_gen_sub_vec(vece, x, a, b);
1604 tcg_gen_sssub_vec(vece, t, a, b);
1605 tcg_gen_xor_vec(vece, x, x, t);
1606 tcg_gen_or_vec(vece, qc, qc, x);
1607 }
1608
1609 void gen_gvec_sqsub_qc(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
1610 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
1611 {
1612 static const TCGOpcode vecop_list[] = {
1613 INDEX_op_sssub_vec, INDEX_op_sub_vec, 0
1614 };
1615 static const GVecGen4 ops[4] = {
1616 { .fniv = gen_sqsub_vec,
1617 .fno = gen_helper_gvec_sqsub_b,
1618 .opt_opc = vecop_list,
1619 .write_aofs = true,
1620 .vece = MO_8 },
1621 { .fniv = gen_sqsub_vec,
1622 .fno = gen_helper_gvec_sqsub_h,
1623 .opt_opc = vecop_list,
1624 .write_aofs = true,
1625 .vece = MO_16 },
1626 { .fniv = gen_sqsub_vec,
1627 .fno = gen_helper_gvec_sqsub_s,
1628 .opt_opc = vecop_list,
1629 .write_aofs = true,
1630 .vece = MO_32 },
1631 { .fniv = gen_sqsub_vec,
1632 .fni8 = gen_sqsub_d,
1633 .fno = gen_helper_gvec_sqsub_d,
1634 .opt_opc = vecop_list,
1635 .write_aofs = true,
1636 .vece = MO_64 },
1637 };
1638
1639 tcg_debug_assert(opr_sz <= sizeof_field(CPUARMState, vfp.qc));
1640 tcg_gen_gvec_4(rd_ofs, offsetof(CPUARMState, vfp.qc),
1641 rn_ofs, rm_ofs, opr_sz, max_sz, &ops[vece]);
1642 }
1643
1644 static void gen_sabd_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1645 {
1646 TCGv_i32 t = tcg_temp_new_i32();
1647
1648 tcg_gen_sub_i32(t, a, b);
1649 tcg_gen_sub_i32(d, b, a);
1650 tcg_gen_movcond_i32(TCG_COND_LT, d, a, b, d, t);
1651 }
1652
1653 static void gen_sabd_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1654 {
1655 TCGv_i64 t = tcg_temp_new_i64();
1656
1657 tcg_gen_sub_i64(t, a, b);
1658 tcg_gen_sub_i64(d, b, a);
1659 tcg_gen_movcond_i64(TCG_COND_LT, d, a, b, d, t);
1660 }
1661
1662 static void gen_sabd_vec(unsigned vece, TCGv_vec d, TCGv_vec a, TCGv_vec b)
1663 {
1664 TCGv_vec t = tcg_temp_new_vec_matching(d);
1665
1666 tcg_gen_smin_vec(vece, t, a, b);
1667 tcg_gen_smax_vec(vece, d, a, b);
1668 tcg_gen_sub_vec(vece, d, d, t);
1669 }
1670
1671 void gen_gvec_sabd(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
1672 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
1673 {
1674 static const TCGOpcode vecop_list[] = {
1675 INDEX_op_sub_vec, INDEX_op_smin_vec, INDEX_op_smax_vec, 0
1676 };
1677 static const GVecGen3 ops[4] = {
1678 { .fniv = gen_sabd_vec,
1679 .fno = gen_helper_gvec_sabd_b,
1680 .opt_opc = vecop_list,
1681 .vece = MO_8 },
1682 { .fniv = gen_sabd_vec,
1683 .fno = gen_helper_gvec_sabd_h,
1684 .opt_opc = vecop_list,
1685 .vece = MO_16 },
1686 { .fni4 = gen_sabd_i32,
1687 .fniv = gen_sabd_vec,
1688 .fno = gen_helper_gvec_sabd_s,
1689 .opt_opc = vecop_list,
1690 .vece = MO_32 },
1691 { .fni8 = gen_sabd_i64,
1692 .fniv = gen_sabd_vec,
1693 .fno = gen_helper_gvec_sabd_d,
1694 .prefer_i64 = true,
1695 .opt_opc = vecop_list,
1696 .vece = MO_64 },
1697 };
1698 tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, &ops[vece]);
1699 }
1700
1701 static void gen_uabd_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1702 {
1703 TCGv_i32 t = tcg_temp_new_i32();
1704
1705 tcg_gen_sub_i32(t, a, b);
1706 tcg_gen_sub_i32(d, b, a);
1707 tcg_gen_movcond_i32(TCG_COND_LTU, d, a, b, d, t);
1708 }
1709
1710 static void gen_uabd_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1711 {
1712 TCGv_i64 t = tcg_temp_new_i64();
1713
1714 tcg_gen_sub_i64(t, a, b);
1715 tcg_gen_sub_i64(d, b, a);
1716 tcg_gen_movcond_i64(TCG_COND_LTU, d, a, b, d, t);
1717 }
1718
1719 static void gen_uabd_vec(unsigned vece, TCGv_vec d, TCGv_vec a, TCGv_vec b)
1720 {
1721 TCGv_vec t = tcg_temp_new_vec_matching(d);
1722
1723 tcg_gen_umin_vec(vece, t, a, b);
1724 tcg_gen_umax_vec(vece, d, a, b);
1725 tcg_gen_sub_vec(vece, d, d, t);
1726 }
1727
1728 void gen_gvec_uabd(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
1729 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
1730 {
1731 static const TCGOpcode vecop_list[] = {
1732 INDEX_op_sub_vec, INDEX_op_umin_vec, INDEX_op_umax_vec, 0
1733 };
1734 static const GVecGen3 ops[4] = {
1735 { .fniv = gen_uabd_vec,
1736 .fno = gen_helper_gvec_uabd_b,
1737 .opt_opc = vecop_list,
1738 .vece = MO_8 },
1739 { .fniv = gen_uabd_vec,
1740 .fno = gen_helper_gvec_uabd_h,
1741 .opt_opc = vecop_list,
1742 .vece = MO_16 },
1743 { .fni4 = gen_uabd_i32,
1744 .fniv = gen_uabd_vec,
1745 .fno = gen_helper_gvec_uabd_s,
1746 .opt_opc = vecop_list,
1747 .vece = MO_32 },
1748 { .fni8 = gen_uabd_i64,
1749 .fniv = gen_uabd_vec,
1750 .fno = gen_helper_gvec_uabd_d,
1751 .prefer_i64 = true,
1752 .opt_opc = vecop_list,
1753 .vece = MO_64 },
1754 };
1755 tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, &ops[vece]);
1756 }
1757
1758 static void gen_saba_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1759 {
1760 TCGv_i32 t = tcg_temp_new_i32();
1761 gen_sabd_i32(t, a, b);
1762 tcg_gen_add_i32(d, d, t);
1763 }
1764
1765 static void gen_saba_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1766 {
1767 TCGv_i64 t = tcg_temp_new_i64();
1768 gen_sabd_i64(t, a, b);
1769 tcg_gen_add_i64(d, d, t);
1770 }
1771
1772 static void gen_saba_vec(unsigned vece, TCGv_vec d, TCGv_vec a, TCGv_vec b)
1773 {
1774 TCGv_vec t = tcg_temp_new_vec_matching(d);
1775 gen_sabd_vec(vece, t, a, b);
1776 tcg_gen_add_vec(vece, d, d, t);
1777 }
1778
1779 void gen_gvec_saba(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
1780 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
1781 {
1782 static const TCGOpcode vecop_list[] = {
1783 INDEX_op_sub_vec, INDEX_op_add_vec,
1784 INDEX_op_smin_vec, INDEX_op_smax_vec, 0
1785 };
1786 static const GVecGen3 ops[4] = {
1787 { .fniv = gen_saba_vec,
1788 .fno = gen_helper_gvec_saba_b,
1789 .opt_opc = vecop_list,
1790 .load_dest = true,
1791 .vece = MO_8 },
1792 { .fniv = gen_saba_vec,
1793 .fno = gen_helper_gvec_saba_h,
1794 .opt_opc = vecop_list,
1795 .load_dest = true,
1796 .vece = MO_16 },
1797 { .fni4 = gen_saba_i32,
1798 .fniv = gen_saba_vec,
1799 .fno = gen_helper_gvec_saba_s,
1800 .opt_opc = vecop_list,
1801 .load_dest = true,
1802 .vece = MO_32 },
1803 { .fni8 = gen_saba_i64,
1804 .fniv = gen_saba_vec,
1805 .fno = gen_helper_gvec_saba_d,
1806 .prefer_i64 = true,
1807 .opt_opc = vecop_list,
1808 .load_dest = true,
1809 .vece = MO_64 },
1810 };
1811 tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, &ops[vece]);
1812 }
1813
1814 static void gen_uaba_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1815 {
1816 TCGv_i32 t = tcg_temp_new_i32();
1817 gen_uabd_i32(t, a, b);
1818 tcg_gen_add_i32(d, d, t);
1819 }
1820
1821 static void gen_uaba_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1822 {
1823 TCGv_i64 t = tcg_temp_new_i64();
1824 gen_uabd_i64(t, a, b);
1825 tcg_gen_add_i64(d, d, t);
1826 }
1827
1828 static void gen_uaba_vec(unsigned vece, TCGv_vec d, TCGv_vec a, TCGv_vec b)
1829 {
1830 TCGv_vec t = tcg_temp_new_vec_matching(d);
1831 gen_uabd_vec(vece, t, a, b);
1832 tcg_gen_add_vec(vece, d, d, t);
1833 }
1834
1835 void gen_gvec_uaba(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
1836 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
1837 {
1838 static const TCGOpcode vecop_list[] = {
1839 INDEX_op_sub_vec, INDEX_op_add_vec,
1840 INDEX_op_umin_vec, INDEX_op_umax_vec, 0
1841 };
1842 static const GVecGen3 ops[4] = {
1843 { .fniv = gen_uaba_vec,
1844 .fno = gen_helper_gvec_uaba_b,
1845 .opt_opc = vecop_list,
1846 .load_dest = true,
1847 .vece = MO_8 },
1848 { .fniv = gen_uaba_vec,
1849 .fno = gen_helper_gvec_uaba_h,
1850 .opt_opc = vecop_list,
1851 .load_dest = true,
1852 .vece = MO_16 },
1853 { .fni4 = gen_uaba_i32,
1854 .fniv = gen_uaba_vec,
1855 .fno = gen_helper_gvec_uaba_s,
1856 .opt_opc = vecop_list,
1857 .load_dest = true,
1858 .vece = MO_32 },
1859 { .fni8 = gen_uaba_i64,
1860 .fniv = gen_uaba_vec,
1861 .fno = gen_helper_gvec_uaba_d,
1862 .prefer_i64 = true,
1863 .opt_opc = vecop_list,
1864 .load_dest = true,
1865 .vece = MO_64 },
1866 };
1867 tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, &ops[vece]);
1868 }
1869
1870 void gen_gvec_addp(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
1871 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
1872 {
1873 static gen_helper_gvec_3 * const fns[4] = {
1874 gen_helper_gvec_addp_b,
1875 gen_helper_gvec_addp_h,
1876 gen_helper_gvec_addp_s,
1877 gen_helper_gvec_addp_d,
1878 };
1879 tcg_gen_gvec_3_ool(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, 0, fns[vece]);
1880 }
1881
1882 void gen_gvec_smaxp(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
1883 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
1884 {
1885 static gen_helper_gvec_3 * const fns[4] = {
1886 gen_helper_gvec_smaxp_b,
1887 gen_helper_gvec_smaxp_h,
1888 gen_helper_gvec_smaxp_s,
1889 };
1890 tcg_debug_assert(vece <= MO_32);
1891 tcg_gen_gvec_3_ool(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, 0, fns[vece]);
1892 }
1893
1894 void gen_gvec_sminp(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
1895 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
1896 {
1897 static gen_helper_gvec_3 * const fns[4] = {
1898 gen_helper_gvec_sminp_b,
1899 gen_helper_gvec_sminp_h,
1900 gen_helper_gvec_sminp_s,
1901 };
1902 tcg_debug_assert(vece <= MO_32);
1903 tcg_gen_gvec_3_ool(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, 0, fns[vece]);
1904 }
1905
1906 void gen_gvec_umaxp(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
1907 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
1908 {
1909 static gen_helper_gvec_3 * const fns[4] = {
1910 gen_helper_gvec_umaxp_b,
1911 gen_helper_gvec_umaxp_h,
1912 gen_helper_gvec_umaxp_s,
1913 };
1914 tcg_debug_assert(vece <= MO_32);
1915 tcg_gen_gvec_3_ool(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, 0, fns[vece]);
1916 }
1917
1918 void gen_gvec_uminp(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
1919 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
1920 {
1921 static gen_helper_gvec_3 * const fns[4] = {
1922 gen_helper_gvec_uminp_b,
1923 gen_helper_gvec_uminp_h,
1924 gen_helper_gvec_uminp_s,
1925 };
1926 tcg_debug_assert(vece <= MO_32);
1927 tcg_gen_gvec_3_ool(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, 0, fns[vece]);
1928 }
1929
1930 static void gen_shadd8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1931 {
1932 TCGv_i64 t = tcg_temp_new_i64();
1933
1934 tcg_gen_and_i64(t, a, b);
1935 tcg_gen_vec_sar8i_i64(a, a, 1);
1936 tcg_gen_vec_sar8i_i64(b, b, 1);
1937 tcg_gen_andi_i64(t, t, dup_const(MO_8, 1));
1938 tcg_gen_vec_add8_i64(d, a, b);
1939 tcg_gen_vec_add8_i64(d, d, t);
1940 }
1941
1942 static void gen_shadd16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1943 {
1944 TCGv_i64 t = tcg_temp_new_i64();
1945
1946 tcg_gen_and_i64(t, a, b);
1947 tcg_gen_vec_sar16i_i64(a, a, 1);
1948 tcg_gen_vec_sar16i_i64(b, b, 1);
1949 tcg_gen_andi_i64(t, t, dup_const(MO_16, 1));
1950 tcg_gen_vec_add16_i64(d, a, b);
1951 tcg_gen_vec_add16_i64(d, d, t);
1952 }
1953
1954 static void gen_shadd_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1955 {
1956 TCGv_i32 t = tcg_temp_new_i32();
1957
1958 tcg_gen_and_i32(t, a, b);
1959 tcg_gen_sari_i32(a, a, 1);
1960 tcg_gen_sari_i32(b, b, 1);
1961 tcg_gen_andi_i32(t, t, 1);
1962 tcg_gen_add_i32(d, a, b);
1963 tcg_gen_add_i32(d, d, t);
1964 }
1965
1966 static void gen_shadd_vec(unsigned vece, TCGv_vec d, TCGv_vec a, TCGv_vec b)
1967 {
1968 TCGv_vec t = tcg_temp_new_vec_matching(d);
1969
1970 tcg_gen_and_vec(vece, t, a, b);
1971 tcg_gen_sari_vec(vece, a, a, 1);
1972 tcg_gen_sari_vec(vece, b, b, 1);
1973 tcg_gen_and_vec(vece, t, t, tcg_constant_vec_matching(d, vece, 1));
1974 tcg_gen_add_vec(vece, d, a, b);
1975 tcg_gen_add_vec(vece, d, d, t);
1976 }
1977
1978 void gen_gvec_shadd(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
1979 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
1980 {
1981 static const TCGOpcode vecop_list[] = {
1982 INDEX_op_sari_vec, INDEX_op_add_vec, 0
1983 };
1984 static const GVecGen3 g[] = {
1985 { .fni8 = gen_shadd8_i64,
1986 .fniv = gen_shadd_vec,
1987 .opt_opc = vecop_list,
1988 .vece = MO_8 },
1989 { .fni8 = gen_shadd16_i64,
1990 .fniv = gen_shadd_vec,
1991 .opt_opc = vecop_list,
1992 .vece = MO_16 },
1993 { .fni4 = gen_shadd_i32,
1994 .fniv = gen_shadd_vec,
1995 .opt_opc = vecop_list,
1996 .vece = MO_32 },
1997 };
1998 tcg_debug_assert(vece <= MO_32);
1999 tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, &g[vece]);
2000 }
2001
2002 static void gen_uhadd8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2003 {
2004 TCGv_i64 t = tcg_temp_new_i64();
2005
2006 tcg_gen_and_i64(t, a, b);
2007 tcg_gen_vec_shr8i_i64(a, a, 1);
2008 tcg_gen_vec_shr8i_i64(b, b, 1);
2009 tcg_gen_andi_i64(t, t, dup_const(MO_8, 1));
2010 tcg_gen_vec_add8_i64(d, a, b);
2011 tcg_gen_vec_add8_i64(d, d, t);
2012 }
2013
2014 static void gen_uhadd16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2015 {
2016 TCGv_i64 t = tcg_temp_new_i64();
2017
2018 tcg_gen_and_i64(t, a, b);
2019 tcg_gen_vec_shr16i_i64(a, a, 1);
2020 tcg_gen_vec_shr16i_i64(b, b, 1);
2021 tcg_gen_andi_i64(t, t, dup_const(MO_16, 1));
2022 tcg_gen_vec_add16_i64(d, a, b);
2023 tcg_gen_vec_add16_i64(d, d, t);
2024 }
2025
2026 static void gen_uhadd_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
2027 {
2028 TCGv_i32 t = tcg_temp_new_i32();
2029
2030 tcg_gen_and_i32(t, a, b);
2031 tcg_gen_shri_i32(a, a, 1);
2032 tcg_gen_shri_i32(b, b, 1);
2033 tcg_gen_andi_i32(t, t, 1);
2034 tcg_gen_add_i32(d, a, b);
2035 tcg_gen_add_i32(d, d, t);
2036 }
2037
2038 static void gen_uhadd_vec(unsigned vece, TCGv_vec d, TCGv_vec a, TCGv_vec b)
2039 {
2040 TCGv_vec t = tcg_temp_new_vec_matching(d);
2041
2042 tcg_gen_and_vec(vece, t, a, b);
2043 tcg_gen_shri_vec(vece, a, a, 1);
2044 tcg_gen_shri_vec(vece, b, b, 1);
2045 tcg_gen_and_vec(vece, t, t, tcg_constant_vec_matching(d, vece, 1));
2046 tcg_gen_add_vec(vece, d, a, b);
2047 tcg_gen_add_vec(vece, d, d, t);
2048 }
2049
2050 void gen_gvec_uhadd(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
2051 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
2052 {
2053 static const TCGOpcode vecop_list[] = {
2054 INDEX_op_shri_vec, INDEX_op_add_vec, 0
2055 };
2056 static const GVecGen3 g[] = {
2057 { .fni8 = gen_uhadd8_i64,
2058 .fniv = gen_uhadd_vec,
2059 .opt_opc = vecop_list,
2060 .vece = MO_8 },
2061 { .fni8 = gen_uhadd16_i64,
2062 .fniv = gen_uhadd_vec,
2063 .opt_opc = vecop_list,
2064 .vece = MO_16 },
2065 { .fni4 = gen_uhadd_i32,
2066 .fniv = gen_uhadd_vec,
2067 .opt_opc = vecop_list,
2068 .vece = MO_32 },
2069 };
2070 tcg_debug_assert(vece <= MO_32);
2071 tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, &g[vece]);
2072 }
2073
2074 static void gen_shsub8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2075 {
2076 TCGv_i64 t = tcg_temp_new_i64();
2077
2078 tcg_gen_andc_i64(t, b, a);
2079 tcg_gen_vec_sar8i_i64(a, a, 1);
2080 tcg_gen_vec_sar8i_i64(b, b, 1);
2081 tcg_gen_andi_i64(t, t, dup_const(MO_8, 1));
2082 tcg_gen_vec_sub8_i64(d, a, b);
2083 tcg_gen_vec_sub8_i64(d, d, t);
2084 }
2085
2086 static void gen_shsub16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2087 {
2088 TCGv_i64 t = tcg_temp_new_i64();
2089
2090 tcg_gen_andc_i64(t, b, a);
2091 tcg_gen_vec_sar16i_i64(a, a, 1);
2092 tcg_gen_vec_sar16i_i64(b, b, 1);
2093 tcg_gen_andi_i64(t, t, dup_const(MO_16, 1));
2094 tcg_gen_vec_sub16_i64(d, a, b);
2095 tcg_gen_vec_sub16_i64(d, d, t);
2096 }
2097
2098 static void gen_shsub_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
2099 {
2100 TCGv_i32 t = tcg_temp_new_i32();
2101
2102 tcg_gen_andc_i32(t, b, a);
2103 tcg_gen_sari_i32(a, a, 1);
2104 tcg_gen_sari_i32(b, b, 1);
2105 tcg_gen_andi_i32(t, t, 1);
2106 tcg_gen_sub_i32(d, a, b);
2107 tcg_gen_sub_i32(d, d, t);
2108 }
2109
2110 static void gen_shsub_vec(unsigned vece, TCGv_vec d, TCGv_vec a, TCGv_vec b)
2111 {
2112 TCGv_vec t = tcg_temp_new_vec_matching(d);
2113
2114 tcg_gen_andc_vec(vece, t, b, a);
2115 tcg_gen_sari_vec(vece, a, a, 1);
2116 tcg_gen_sari_vec(vece, b, b, 1);
2117 tcg_gen_and_vec(vece, t, t, tcg_constant_vec_matching(d, vece, 1));
2118 tcg_gen_sub_vec(vece, d, a, b);
2119 tcg_gen_sub_vec(vece, d, d, t);
2120 }
2121
2122 void gen_gvec_shsub(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
2123 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
2124 {
2125 static const TCGOpcode vecop_list[] = {
2126 INDEX_op_sari_vec, INDEX_op_sub_vec, 0
2127 };
2128 static const GVecGen3 g[4] = {
2129 { .fni8 = gen_shsub8_i64,
2130 .fniv = gen_shsub_vec,
2131 .opt_opc = vecop_list,
2132 .vece = MO_8 },
2133 { .fni8 = gen_shsub16_i64,
2134 .fniv = gen_shsub_vec,
2135 .opt_opc = vecop_list,
2136 .vece = MO_16 },
2137 { .fni4 = gen_shsub_i32,
2138 .fniv = gen_shsub_vec,
2139 .opt_opc = vecop_list,
2140 .vece = MO_32 },
2141 };
2142 assert(vece <= MO_32);
2143 tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, &g[vece]);
2144 }
2145
2146 static void gen_uhsub8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2147 {
2148 TCGv_i64 t = tcg_temp_new_i64();
2149
2150 tcg_gen_andc_i64(t, b, a);
2151 tcg_gen_vec_shr8i_i64(a, a, 1);
2152 tcg_gen_vec_shr8i_i64(b, b, 1);
2153 tcg_gen_andi_i64(t, t, dup_const(MO_8, 1));
2154 tcg_gen_vec_sub8_i64(d, a, b);
2155 tcg_gen_vec_sub8_i64(d, d, t);
2156 }
2157
2158 static void gen_uhsub16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2159 {
2160 TCGv_i64 t = tcg_temp_new_i64();
2161
2162 tcg_gen_andc_i64(t, b, a);
2163 tcg_gen_vec_shr16i_i64(a, a, 1);
2164 tcg_gen_vec_shr16i_i64(b, b, 1);
2165 tcg_gen_andi_i64(t, t, dup_const(MO_16, 1));
2166 tcg_gen_vec_sub16_i64(d, a, b);
2167 tcg_gen_vec_sub16_i64(d, d, t);
2168 }
2169
2170 static void gen_uhsub_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
2171 {
2172 TCGv_i32 t = tcg_temp_new_i32();
2173
2174 tcg_gen_andc_i32(t, b, a);
2175 tcg_gen_shri_i32(a, a, 1);
2176 tcg_gen_shri_i32(b, b, 1);
2177 tcg_gen_andi_i32(t, t, 1);
2178 tcg_gen_sub_i32(d, a, b);
2179 tcg_gen_sub_i32(d, d, t);
2180 }
2181
2182 static void gen_uhsub_vec(unsigned vece, TCGv_vec d, TCGv_vec a, TCGv_vec b)
2183 {
2184 TCGv_vec t = tcg_temp_new_vec_matching(d);
2185
2186 tcg_gen_andc_vec(vece, t, b, a);
2187 tcg_gen_shri_vec(vece, a, a, 1);
2188 tcg_gen_shri_vec(vece, b, b, 1);
2189 tcg_gen_and_vec(vece, t, t, tcg_constant_vec_matching(d, vece, 1));
2190 tcg_gen_sub_vec(vece, d, a, b);
2191 tcg_gen_sub_vec(vece, d, d, t);
2192 }
2193
2194 void gen_gvec_uhsub(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
2195 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
2196 {
2197 static const TCGOpcode vecop_list[] = {
2198 INDEX_op_shri_vec, INDEX_op_sub_vec, 0
2199 };
2200 static const GVecGen3 g[4] = {
2201 { .fni8 = gen_uhsub8_i64,
2202 .fniv = gen_uhsub_vec,
2203 .opt_opc = vecop_list,
2204 .vece = MO_8 },
2205 { .fni8 = gen_uhsub16_i64,
2206 .fniv = gen_uhsub_vec,
2207 .opt_opc = vecop_list,
2208 .vece = MO_16 },
2209 { .fni4 = gen_uhsub_i32,
2210 .fniv = gen_uhsub_vec,
2211 .opt_opc = vecop_list,
2212 .vece = MO_32 },
2213 };
2214 assert(vece <= MO_32);
2215 tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, &g[vece]);
2216 }
2217
2218 static void gen_srhadd8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2219 {
2220 TCGv_i64 t = tcg_temp_new_i64();
2221
2222 tcg_gen_or_i64(t, a, b);
2223 tcg_gen_vec_sar8i_i64(a, a, 1);
2224 tcg_gen_vec_sar8i_i64(b, b, 1);
2225 tcg_gen_andi_i64(t, t, dup_const(MO_8, 1));
2226 tcg_gen_vec_add8_i64(d, a, b);
2227 tcg_gen_vec_add8_i64(d, d, t);
2228 }
2229
2230 static void gen_srhadd16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2231 {
2232 TCGv_i64 t = tcg_temp_new_i64();
2233
2234 tcg_gen_or_i64(t, a, b);
2235 tcg_gen_vec_sar16i_i64(a, a, 1);
2236 tcg_gen_vec_sar16i_i64(b, b, 1);
2237 tcg_gen_andi_i64(t, t, dup_const(MO_16, 1));
2238 tcg_gen_vec_add16_i64(d, a, b);
2239 tcg_gen_vec_add16_i64(d, d, t);
2240 }
2241
2242 static void gen_srhadd_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
2243 {
2244 TCGv_i32 t = tcg_temp_new_i32();
2245
2246 tcg_gen_or_i32(t, a, b);
2247 tcg_gen_sari_i32(a, a, 1);
2248 tcg_gen_sari_i32(b, b, 1);
2249 tcg_gen_andi_i32(t, t, 1);
2250 tcg_gen_add_i32(d, a, b);
2251 tcg_gen_add_i32(d, d, t);
2252 }
2253
2254 static void gen_srhadd_vec(unsigned vece, TCGv_vec d, TCGv_vec a, TCGv_vec b)
2255 {
2256 TCGv_vec t = tcg_temp_new_vec_matching(d);
2257
2258 tcg_gen_or_vec(vece, t, a, b);
2259 tcg_gen_sari_vec(vece, a, a, 1);
2260 tcg_gen_sari_vec(vece, b, b, 1);
2261 tcg_gen_and_vec(vece, t, t, tcg_constant_vec_matching(d, vece, 1));
2262 tcg_gen_add_vec(vece, d, a, b);
2263 tcg_gen_add_vec(vece, d, d, t);
2264 }
2265
2266 void gen_gvec_srhadd(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
2267 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
2268 {
2269 static const TCGOpcode vecop_list[] = {
2270 INDEX_op_sari_vec, INDEX_op_add_vec, 0
2271 };
2272 static const GVecGen3 g[] = {
2273 { .fni8 = gen_srhadd8_i64,
2274 .fniv = gen_srhadd_vec,
2275 .opt_opc = vecop_list,
2276 .vece = MO_8 },
2277 { .fni8 = gen_srhadd16_i64,
2278 .fniv = gen_srhadd_vec,
2279 .opt_opc = vecop_list,
2280 .vece = MO_16 },
2281 { .fni4 = gen_srhadd_i32,
2282 .fniv = gen_srhadd_vec,
2283 .opt_opc = vecop_list,
2284 .vece = MO_32 },
2285 };
2286 assert(vece <= MO_32);
2287 tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, &g[vece]);
2288 }
2289
2290 static void gen_urhadd8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2291 {
2292 TCGv_i64 t = tcg_temp_new_i64();
2293
2294 tcg_gen_or_i64(t, a, b);
2295 tcg_gen_vec_shr8i_i64(a, a, 1);
2296 tcg_gen_vec_shr8i_i64(b, b, 1);
2297 tcg_gen_andi_i64(t, t, dup_const(MO_8, 1));
2298 tcg_gen_vec_add8_i64(d, a, b);
2299 tcg_gen_vec_add8_i64(d, d, t);
2300 }
2301
2302 static void gen_urhadd16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2303 {
2304 TCGv_i64 t = tcg_temp_new_i64();
2305
2306 tcg_gen_or_i64(t, a, b);
2307 tcg_gen_vec_shr16i_i64(a, a, 1);
2308 tcg_gen_vec_shr16i_i64(b, b, 1);
2309 tcg_gen_andi_i64(t, t, dup_const(MO_16, 1));
2310 tcg_gen_vec_add16_i64(d, a, b);
2311 tcg_gen_vec_add16_i64(d, d, t);
2312 }
2313
2314 static void gen_urhadd_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
2315 {
2316 TCGv_i32 t = tcg_temp_new_i32();
2317
2318 tcg_gen_or_i32(t, a, b);
2319 tcg_gen_shri_i32(a, a, 1);
2320 tcg_gen_shri_i32(b, b, 1);
2321 tcg_gen_andi_i32(t, t, 1);
2322 tcg_gen_add_i32(d, a, b);
2323 tcg_gen_add_i32(d, d, t);
2324 }
2325
2326 static void gen_urhadd_vec(unsigned vece, TCGv_vec d, TCGv_vec a, TCGv_vec b)
2327 {
2328 TCGv_vec t = tcg_temp_new_vec_matching(d);
2329
2330 tcg_gen_or_vec(vece, t, a, b);
2331 tcg_gen_shri_vec(vece, a, a, 1);
2332 tcg_gen_shri_vec(vece, b, b, 1);
2333 tcg_gen_and_vec(vece, t, t, tcg_constant_vec_matching(d, vece, 1));
2334 tcg_gen_add_vec(vece, d, a, b);
2335 tcg_gen_add_vec(vece, d, d, t);
2336 }
2337
2338 void gen_gvec_urhadd(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
2339 uint32_t rm_ofs, uint32_t opr_sz, uint32_t max_sz)
2340 {
2341 static const TCGOpcode vecop_list[] = {
2342 INDEX_op_shri_vec, INDEX_op_add_vec, 0
2343 };
2344 static const GVecGen3 g[] = {
2345 { .fni8 = gen_urhadd8_i64,
2346 .fniv = gen_urhadd_vec,
2347 .opt_opc = vecop_list,
2348 .vece = MO_8 },
2349 { .fni8 = gen_urhadd16_i64,
2350 .fniv = gen_urhadd_vec,
2351 .opt_opc = vecop_list,
2352 .vece = MO_16 },
2353 { .fni4 = gen_urhadd_i32,
2354 .fniv = gen_urhadd_vec,
2355 .opt_opc = vecop_list,
2356 .vece = MO_32 },
2357 };
2358 assert(vece <= MO_32);
2359 tcg_gen_gvec_3(rd_ofs, rn_ofs, rm_ofs, opr_sz, max_sz, &g[vece]);
2360 }
2361
2362 void gen_gvec_cls(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
2363 uint32_t opr_sz, uint32_t max_sz)
2364 {
2365 static const GVecGen2 g[] = {
2366 { .fni4 = gen_helper_neon_cls_s8,
2367 .vece = MO_8 },
2368 { .fni4 = gen_helper_neon_cls_s16,
2369 .vece = MO_16 },
2370 { .fni4 = tcg_gen_clrsb_i32,
2371 .vece = MO_32 },
2372 };
2373 assert(vece <= MO_32);
2374 tcg_gen_gvec_2(rd_ofs, rn_ofs, opr_sz, max_sz, &g[vece]);
2375 }
2376
2377 static void gen_clz32_i32(TCGv_i32 d, TCGv_i32 n)
2378 {
2379 tcg_gen_clzi_i32(d, n, 32);
2380 }
2381
2382 void gen_gvec_clz(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
2383 uint32_t opr_sz, uint32_t max_sz)
2384 {
2385 static const GVecGen2 g[] = {
2386 { .fni4 = gen_helper_neon_clz_u8,
2387 .vece = MO_8 },
2388 { .fni4 = gen_helper_neon_clz_u16,
2389 .vece = MO_16 },
2390 { .fni4 = gen_clz32_i32,
2391 .vece = MO_32 },
2392 };
2393 assert(vece <= MO_32);
2394 tcg_gen_gvec_2(rd_ofs, rn_ofs, opr_sz, max_sz, &g[vece]);
2395 }
2396
2397 void gen_gvec_cnt(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
2398 uint32_t opr_sz, uint32_t max_sz)
2399 {
2400 assert(vece == MO_8);
2401 tcg_gen_gvec_2_ool(rd_ofs, rn_ofs, opr_sz, max_sz, 0,
2402 gen_helper_gvec_cnt_b);
2403 }
2404
2405 void gen_gvec_rbit(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
2406 uint32_t opr_sz, uint32_t max_sz)
2407 {
2408 assert(vece == MO_8);
2409 tcg_gen_gvec_2_ool(rd_ofs, rn_ofs, opr_sz, max_sz, 0,
2410 gen_helper_gvec_rbit_b);
2411 }
2412
2413 void gen_gvec_rev16(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
2414 uint32_t opr_sz, uint32_t max_sz)
2415 {
2416 assert(vece == MO_8);
2417 tcg_gen_gvec_rotli(MO_16, rd_ofs, rn_ofs, 8, opr_sz, max_sz);
2418 }
2419
2420 static void gen_bswap32_i64(TCGv_i64 d, TCGv_i64 n)
2421 {
2422 tcg_gen_bswap64_i64(d, n);
2423 tcg_gen_rotli_i64(d, d, 32);
2424 }
2425
2426 void gen_gvec_rev32(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
2427 uint32_t opr_sz, uint32_t max_sz)
2428 {
2429 static const GVecGen2 g = {
2430 .fni8 = gen_bswap32_i64,
2431 .fni4 = tcg_gen_bswap32_i32,
2432 .prefer_i64 = true,
2433 .vece = MO_32
2434 };
2435
2436 switch (vece) {
2437 case MO_16:
2438 tcg_gen_gvec_rotli(MO_32, rd_ofs, rn_ofs, 16, opr_sz, max_sz);
2439 break;
2440 case MO_8:
2441 tcg_gen_gvec_2(rd_ofs, rn_ofs, opr_sz, max_sz, &g);
2442 break;
2443 default:
2444 g_assert_not_reached();
2445 }
2446 }
2447
2448 void gen_gvec_rev64(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
2449 uint32_t opr_sz, uint32_t max_sz)
2450 {
2451 static const GVecGen2 g[] = {
2452 { .fni8 = tcg_gen_bswap64_i64,
2453 .vece = MO_64 },
2454 { .fni8 = tcg_gen_hswap_i64,
2455 .vece = MO_64 },
2456 };
2457
2458 switch (vece) {
2459 case MO_32:
2460 tcg_gen_gvec_rotli(MO_64, rd_ofs, rn_ofs, 32, opr_sz, max_sz);
2461 break;
2462 case MO_8:
2463 case MO_16:
2464 tcg_gen_gvec_2(rd_ofs, rn_ofs, opr_sz, max_sz, &g[vece]);
2465 break;
2466 default:
2467 g_assert_not_reached();
2468 }
2469 }
2470
2471 static void gen_saddlp_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
2472 {
2473 int half = 4 << vece;
2474 TCGv_vec t = tcg_temp_new_vec_matching(d);
2475
2476 tcg_gen_shli_vec(vece, t, n, half);
2477 tcg_gen_sari_vec(vece, d, n, half);
2478 tcg_gen_sari_vec(vece, t, t, half);
2479 tcg_gen_add_vec(vece, d, d, t);
2480 }
2481
2482 static void gen_saddlp_s_i64(TCGv_i64 d, TCGv_i64 n)
2483 {
2484 TCGv_i64 t = tcg_temp_new_i64();
2485
2486 tcg_gen_ext32s_i64(t, n);
2487 tcg_gen_sari_i64(d, n, 32);
2488 tcg_gen_add_i64(d, d, t);
2489 }
2490
2491 void gen_gvec_saddlp(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
2492 uint32_t opr_sz, uint32_t max_sz)
2493 {
2494 static const TCGOpcode vecop_list[] = {
2495 INDEX_op_sari_vec, INDEX_op_shli_vec, INDEX_op_add_vec, 0
2496 };
2497 static const GVecGen2 g[] = {
2498 { .fniv = gen_saddlp_vec,
2499 .fni8 = gen_helper_neon_addlp_s8,
2500 .opt_opc = vecop_list,
2501 .vece = MO_16 },
2502 { .fniv = gen_saddlp_vec,
2503 .fni8 = gen_helper_neon_addlp_s16,
2504 .opt_opc = vecop_list,
2505 .vece = MO_32 },
2506 { .fniv = gen_saddlp_vec,
2507 .fni8 = gen_saddlp_s_i64,
2508 .opt_opc = vecop_list,
2509 .vece = MO_64 },
2510 };
2511 assert(vece <= MO_32);
2512 tcg_gen_gvec_2(rd_ofs, rn_ofs, opr_sz, max_sz, &g[vece]);
2513 }
2514
2515 static void gen_sadalp_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
2516 {
2517 TCGv_vec t = tcg_temp_new_vec_matching(d);
2518
2519 gen_saddlp_vec(vece, t, n);
2520 tcg_gen_add_vec(vece, d, d, t);
2521 }
2522
2523 static void gen_sadalp_b_i64(TCGv_i64 d, TCGv_i64 n)
2524 {
2525 TCGv_i64 t = tcg_temp_new_i64();
2526
2527 gen_helper_neon_addlp_s8(t, n);
2528 tcg_gen_vec_add16_i64(d, d, t);
2529 }
2530
2531 static void gen_sadalp_h_i64(TCGv_i64 d, TCGv_i64 n)
2532 {
2533 TCGv_i64 t = tcg_temp_new_i64();
2534
2535 gen_helper_neon_addlp_s16(t, n);
2536 tcg_gen_vec_add32_i64(d, d, t);
2537 }
2538
2539 static void gen_sadalp_s_i64(TCGv_i64 d, TCGv_i64 n)
2540 {
2541 TCGv_i64 t = tcg_temp_new_i64();
2542
2543 gen_saddlp_s_i64(t, n);
2544 tcg_gen_add_i64(d, d, t);
2545 }
2546
2547 void gen_gvec_sadalp(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
2548 uint32_t opr_sz, uint32_t max_sz)
2549 {
2550 static const TCGOpcode vecop_list[] = {
2551 INDEX_op_sari_vec, INDEX_op_shli_vec, INDEX_op_add_vec, 0
2552 };
2553 static const GVecGen2 g[] = {
2554 { .fniv = gen_sadalp_vec,
2555 .fni8 = gen_sadalp_b_i64,
2556 .opt_opc = vecop_list,
2557 .load_dest = true,
2558 .vece = MO_16 },
2559 { .fniv = gen_sadalp_vec,
2560 .fni8 = gen_sadalp_h_i64,
2561 .opt_opc = vecop_list,
2562 .load_dest = true,
2563 .vece = MO_32 },
2564 { .fniv = gen_sadalp_vec,
2565 .fni8 = gen_sadalp_s_i64,
2566 .opt_opc = vecop_list,
2567 .load_dest = true,
2568 .vece = MO_64 },
2569 };
2570 assert(vece <= MO_32);
2571 tcg_gen_gvec_2(rd_ofs, rn_ofs, opr_sz, max_sz, &g[vece]);
2572 }
2573
2574 static void gen_uaddlp_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
2575 {
2576 int half = 4 << vece;
2577 TCGv_vec t = tcg_temp_new_vec_matching(d);
2578 TCGv_vec m = tcg_constant_vec_matching(d, vece, MAKE_64BIT_MASK(0, half));
2579
2580 tcg_gen_shri_vec(vece, t, n, half);
2581 tcg_gen_and_vec(vece, d, n, m);
2582 tcg_gen_add_vec(vece, d, d, t);
2583 }
2584
2585 static void gen_uaddlp_b_i64(TCGv_i64 d, TCGv_i64 n)
2586 {
2587 TCGv_i64 t = tcg_temp_new_i64();
2588 TCGv_i64 m = tcg_constant_i64(dup_const(MO_16, 0xff));
2589
2590 tcg_gen_shri_i64(t, n, 8);
2591 tcg_gen_and_i64(d, n, m);
2592 tcg_gen_and_i64(t, t, m);
2593 /* No carry between widened unsigned elements. */
2594 tcg_gen_add_i64(d, d, t);
2595 }
2596
2597 static void gen_uaddlp_h_i64(TCGv_i64 d, TCGv_i64 n)
2598 {
2599 TCGv_i64 t = tcg_temp_new_i64();
2600 TCGv_i64 m = tcg_constant_i64(dup_const(MO_32, 0xffff));
2601
2602 tcg_gen_shri_i64(t, n, 16);
2603 tcg_gen_and_i64(d, n, m);
2604 tcg_gen_and_i64(t, t, m);
2605 /* No carry between widened unsigned elements. */
2606 tcg_gen_add_i64(d, d, t);
2607 }
2608
2609 static void gen_uaddlp_s_i64(TCGv_i64 d, TCGv_i64 n)
2610 {
2611 TCGv_i64 t = tcg_temp_new_i64();
2612
2613 tcg_gen_ext32u_i64(t, n);
2614 tcg_gen_shri_i64(d, n, 32);
2615 tcg_gen_add_i64(d, d, t);
2616 }
2617
2618 void gen_gvec_uaddlp(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
2619 uint32_t opr_sz, uint32_t max_sz)
2620 {
2621 static const TCGOpcode vecop_list[] = {
2622 INDEX_op_shri_vec, INDEX_op_add_vec, 0
2623 };
2624 static const GVecGen2 g[] = {
2625 { .fniv = gen_uaddlp_vec,
2626 .fni8 = gen_uaddlp_b_i64,
2627 .opt_opc = vecop_list,
2628 .vece = MO_16 },
2629 { .fniv = gen_uaddlp_vec,
2630 .fni8 = gen_uaddlp_h_i64,
2631 .opt_opc = vecop_list,
2632 .vece = MO_32 },
2633 { .fniv = gen_uaddlp_vec,
2634 .fni8 = gen_uaddlp_s_i64,
2635 .opt_opc = vecop_list,
2636 .vece = MO_64 },
2637 };
2638 assert(vece <= MO_32);
2639 tcg_gen_gvec_2(rd_ofs, rn_ofs, opr_sz, max_sz, &g[vece]);
2640 }
2641
2642 static void gen_uadalp_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
2643 {
2644 TCGv_vec t = tcg_temp_new_vec_matching(d);
2645
2646 gen_uaddlp_vec(vece, t, n);
2647 tcg_gen_add_vec(vece, d, d, t);
2648 }
2649
2650 static void gen_uadalp_b_i64(TCGv_i64 d, TCGv_i64 n)
2651 {
2652 TCGv_i64 t = tcg_temp_new_i64();
2653
2654 gen_uaddlp_b_i64(t, n);
2655 tcg_gen_vec_add16_i64(d, d, t);
2656 }
2657
2658 static void gen_uadalp_h_i64(TCGv_i64 d, TCGv_i64 n)
2659 {
2660 TCGv_i64 t = tcg_temp_new_i64();
2661
2662 gen_uaddlp_h_i64(t, n);
2663 tcg_gen_vec_add32_i64(d, d, t);
2664 }
2665
2666 static void gen_uadalp_s_i64(TCGv_i64 d, TCGv_i64 n)
2667 {
2668 TCGv_i64 t = tcg_temp_new_i64();
2669
2670 gen_uaddlp_s_i64(t, n);
2671 tcg_gen_add_i64(d, d, t);
2672 }
2673
2674 void gen_gvec_uadalp(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
2675 uint32_t opr_sz, uint32_t max_sz)
2676 {
2677 static const TCGOpcode vecop_list[] = {
2678 INDEX_op_shri_vec, INDEX_op_add_vec, 0
2679 };
2680 static const GVecGen2 g[] = {
2681 { .fniv = gen_uadalp_vec,
2682 .fni8 = gen_uadalp_b_i64,
2683 .load_dest = true,
2684 .opt_opc = vecop_list,
2685 .vece = MO_16 },
2686 { .fniv = gen_uadalp_vec,
2687 .fni8 = gen_uadalp_h_i64,
2688 .load_dest = true,
2689 .opt_opc = vecop_list,
2690 .vece = MO_32 },
2691 { .fniv = gen_uadalp_vec,
2692 .fni8 = gen_uadalp_s_i64,
2693 .load_dest = true,
2694 .opt_opc = vecop_list,
2695 .vece = MO_64 },
2696 };
2697 assert(vece <= MO_32);
2698 tcg_gen_gvec_2(rd_ofs, rn_ofs, opr_sz, max_sz, &g[vece]);
2699 }
2700
2701 void gen_gvec_fabs(unsigned vece, uint32_t dofs, uint32_t aofs,
2702 uint32_t oprsz, uint32_t maxsz)
2703 {
2704 uint64_t s_bit = 1ull << ((8 << vece) - 1);
2705 tcg_gen_gvec_andi(vece, dofs, aofs, s_bit - 1, oprsz, maxsz);
2706 }
2707
2708 void gen_gvec_fneg(unsigned vece, uint32_t dofs, uint32_t aofs,
2709 uint32_t oprsz, uint32_t maxsz)
2710 {
2711 uint64_t s_bit = 1ull << ((8 << vece) - 1);
2712 tcg_gen_gvec_xori(vece, dofs, aofs, s_bit, oprsz, maxsz);
2713 }
2714
2715 void gen_gvec_urecpe(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
2716 uint32_t opr_sz, uint32_t max_sz)
2717 {
2718 assert(vece == MO_32);
2719 tcg_gen_gvec_2_ool(rd_ofs, rn_ofs, opr_sz, max_sz, 0,
2720 gen_helper_gvec_urecpe_s);
2721 }
2722
2723 void gen_gvec_ursqrte(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
2724 uint32_t opr_sz, uint32_t max_sz)
2725 {
2726 assert(vece == MO_32);
2727 tcg_gen_gvec_2_ool(rd_ofs, rn_ofs, opr_sz, max_sz, 0,
2728 gen_helper_gvec_ursqrte_s);
2729 }