tcg-op-gvec.c · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

c 4,090 lines 129 KB

Raw

1	/*
2	* Generic vector operation expansion
3	*
4	* Copyright (c) 2018 Linaro
5	*
6	* This library is free software; you can redistribute it and/or
7	* modify it under the terms of the GNU Lesser General Public
8	* License as published by the Free Software Foundation; either
9	* version 2.1 of the License, or (at your option) any later version.
10	*
11	* This library is distributed in the hope that it will be useful,
12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14	* Lesser General Public License for more details.
15	*
16	* You should have received a copy of the GNU Lesser General Public
17	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
18	*/
19
20	#include "qemu/osdep.h"
21	#include "tcg/tcg.h"
22	#include "tcg/tcg-temp-internal.h"
23	#include "tcg/tcg-op-common.h"
24	#include "tcg/tcg-op-gvec-common.h"
25	#include "tcg/tcg-gvec-desc.h"
26	#include "tcg-has.h"
27
28	#define MAX_UNROLL 4
29
30	#ifdef CONFIG_DEBUG_TCG
31	static const TCGOpcode vecop_list_empty[1] = { 0 };
32	#else
33	#define vecop_list_empty NULL
34	#endif
35
36
37	/* Verify vector size and alignment rules. OFS should be the OR of all
38	of the operand offsets so that we can check them all at once. */
39	static void check_size_align(uint32_t oprsz, uint32_t maxsz, uint32_t ofs)
40	{
41	uint32_t max_align;
42
43	switch (oprsz) {
44	case 8:
45	case 16:
46	case 32:
47	tcg_debug_assert(oprsz <= maxsz);
48	break;
49	default:
50	tcg_debug_assert(oprsz == maxsz);
51	break;
52	}
53	tcg_debug_assert(maxsz <= (8 << SIMD_MAXSZ_BITS));
54
55	max_align = maxsz >= 16 ? 15 : 7;
56	tcg_debug_assert((maxsz & max_align) == 0);
57	tcg_debug_assert((ofs & max_align) == 0);
58	}
59
60	/*
61	* Verify vector overlap rules for two operands.
62	* When dbase and abase are not the same pointer, we cannot check for
63	* overlap at compile-time, but the runtime restrictions remain.
64	*/
65	static void check_overlap_2(TCGv_ptr dbase, uint32_t d,
66	TCGv_ptr abase, uint32_t a, uint32_t s)
67	{
68	tcg_debug_assert(dbase != abase \|\| d == a \|\| d + s <= a \|\| a + s <= d);
69	}
70
71	/* Verify vector overlap rules for three operands. */
72	static void check_overlap_3(TCGv_ptr dbase, uint32_t d,
73	TCGv_ptr abase, uint32_t a,
74	TCGv_ptr bbase, uint32_t b, uint32_t s)
75	{
76	check_overlap_2(dbase, d, abase, a, s);
77	check_overlap_2(dbase, d, bbase, b, s);
78	check_overlap_2(abase, a, bbase, b, s);
79	}
80
81	/* Verify vector overlap rules for four operands. */
82	static void check_overlap_4(TCGv_ptr dbase, uint32_t d,
83	TCGv_ptr abase, uint32_t a,
84	TCGv_ptr bbase, uint32_t b,
85	TCGv_ptr cbase, uint32_t c, uint32_t s)
86	{
87	check_overlap_2(dbase, d, abase, a, s);
88	check_overlap_2(dbase, d, bbase, b, s);
89	check_overlap_2(dbase, d, cbase, c, s);
90	check_overlap_2(abase, a, bbase, b, s);
91	check_overlap_2(abase, a, cbase, c, s);
92	check_overlap_2(bbase, b, cbase, c, s);
93	}
94
95	/* Create a descriptor from components. */
96	uint32_t simd_desc(uint32_t oprsz, uint32_t maxsz, int32_t data)
97	{
98	uint32_t desc = 0;
99
100	check_size_align(oprsz, maxsz, 0);
101
102	/*
103	* We want to check that 'data' will fit into SIMD_DATA_BITS.
104	* However, some callers want to treat the data as a signed
105	* value (which they can later get back with simd_data())
106	* and some want to treat it as an unsigned value.
107	* So here we assert only that the data will fit into the
108	* field in at least one way. This means that some invalid
109	* values from the caller will not be detected, e.g. if the
110	* caller wants to handle the value as a signed integer but
111	* incorrectly passes us 1 << (SIMD_DATA_BITS - 1).
112	*/
113	tcg_debug_assert(data == sextract32(data, 0, SIMD_DATA_BITS) \|\|
114	data == extract32(data, 0, SIMD_DATA_BITS));
115
116	oprsz = (oprsz / 8) - 1;
117	maxsz = (maxsz / 8) - 1;
118
119	/*
120	* We have just asserted in check_size_align that either
121	* oprsz is {8,16,32} or matches maxsz. Encode the final
122	* case with '2', as that would otherwise map to 24.
123	*/
124	if (oprsz == maxsz) {
125	oprsz = 2;
126	}
127
128	desc = deposit32(desc, SIMD_OPRSZ_SHIFT, SIMD_OPRSZ_BITS, oprsz);
129	desc = deposit32(desc, SIMD_MAXSZ_SHIFT, SIMD_MAXSZ_BITS, maxsz);
130	desc = deposit32(desc, SIMD_DATA_SHIFT, SIMD_DATA_BITS, data);
131
132	return desc;
133	}
134
135	/* Generate a call to a gvec-style helper with two vector operands. */
136	static void expand_2_ool(TCGv_ptr dbase, uint32_t dofs,
137	TCGv_ptr abase, uint32_t aofs,
138	uint32_t oprsz, uint32_t maxsz,
139	int32_t data, gen_helper_gvec_2 *fn)
140	{
141	TCGv_ptr a0, a1;
142	TCGv_i32 desc = tcg_constant_i32(simd_desc(oprsz, maxsz, data));
143
144	a0 = tcg_temp_ebb_new_ptr();
145	a1 = tcg_temp_ebb_new_ptr();
146
147	tcg_gen_addi_ptr(a0, dbase, dofs);
148	tcg_gen_addi_ptr(a1, abase, aofs);
149
150	fn(a0, a1, desc);
151
152	tcg_temp_free_ptr(a0);
153	tcg_temp_free_ptr(a1);
154	}
155
156	void tcg_gen_gvec_2_ool(uint32_t dofs, uint32_t aofs,
157	uint32_t oprsz, uint32_t maxsz, int32_t data,
158	gen_helper_gvec_2 *fn)
159	{
160	expand_2_ool(tcg_env, dofs, tcg_env, aofs, oprsz, maxsz, data, fn);
161	}
162
163	/* Generate a call to a gvec-style helper with two vector operands
164	and one scalar operand. */
165	void tcg_gen_gvec_2i_ool(uint32_t dofs, uint32_t aofs, TCGv_i64 c,
166	uint32_t oprsz, uint32_t maxsz, int32_t data,
167	gen_helper_gvec_2i *fn)
168	{
169	TCGv_ptr a0, a1;
170	TCGv_i32 desc = tcg_constant_i32(simd_desc(oprsz, maxsz, data));
171
172	a0 = tcg_temp_ebb_new_ptr();
173	a1 = tcg_temp_ebb_new_ptr();
174
175	tcg_gen_addi_ptr(a0, tcg_env, dofs);
176	tcg_gen_addi_ptr(a1, tcg_env, aofs);
177
178	fn(a0, a1, c, desc);
179
180	tcg_temp_free_ptr(a0);
181	tcg_temp_free_ptr(a1);
182	}
183
184	/* Generate a call to a gvec-style helper with three vector operands. */
185	static void expand_3_ool(TCGv_ptr dbase, uint32_t dofs,
186	TCGv_ptr abase, uint32_t aofs,
187	TCGv_ptr bbase, uint32_t bofs,
188	uint32_t oprsz, uint32_t maxsz,
189	int32_t data, gen_helper_gvec_3 *fn)
190	{
191	TCGv_ptr a0, a1, a2;
192	TCGv_i32 desc = tcg_constant_i32(simd_desc(oprsz, maxsz, data));
193
194	a0 = tcg_temp_ebb_new_ptr();
195	a1 = tcg_temp_ebb_new_ptr();
196	a2 = tcg_temp_ebb_new_ptr();
197
198	tcg_gen_addi_ptr(a0, dbase, dofs);
199	tcg_gen_addi_ptr(a1, abase, aofs);
200	tcg_gen_addi_ptr(a2, bbase, bofs);
201
202	fn(a0, a1, a2, desc);
203
204	tcg_temp_free_ptr(a0);
205	tcg_temp_free_ptr(a1);
206	tcg_temp_free_ptr(a2);
207	}
208
209	void tcg_gen_gvec_3_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs,
210	uint32_t oprsz, uint32_t maxsz, int32_t data,
211	gen_helper_gvec_3 *fn)
212	{
213	expand_3_ool(tcg_env, dofs, tcg_env, aofs, tcg_env, bofs,
214	oprsz, maxsz, data, fn);
215	}
216
217	/* Generate a call to a gvec-style helper with four vector operands. */
218	void tcg_gen_gvec_4_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs,
219	uint32_t cofs, uint32_t oprsz, uint32_t maxsz,
220	int32_t data, gen_helper_gvec_4 *fn)
221	{
222	TCGv_ptr a0, a1, a2, a3;
223	TCGv_i32 desc = tcg_constant_i32(simd_desc(oprsz, maxsz, data));
224
225	a0 = tcg_temp_ebb_new_ptr();
226	a1 = tcg_temp_ebb_new_ptr();
227	a2 = tcg_temp_ebb_new_ptr();
228	a3 = tcg_temp_ebb_new_ptr();
229
230	tcg_gen_addi_ptr(a0, tcg_env, dofs);
231	tcg_gen_addi_ptr(a1, tcg_env, aofs);
232	tcg_gen_addi_ptr(a2, tcg_env, bofs);
233	tcg_gen_addi_ptr(a3, tcg_env, cofs);
234
235	fn(a0, a1, a2, a3, desc);
236
237	tcg_temp_free_ptr(a0);
238	tcg_temp_free_ptr(a1);
239	tcg_temp_free_ptr(a2);
240	tcg_temp_free_ptr(a3);
241	}
242
243	/* Generate a call to a gvec-style helper with five vector operands. */
244	void tcg_gen_gvec_5_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs,
245	uint32_t cofs, uint32_t xofs, uint32_t oprsz,
246	uint32_t maxsz, int32_t data, gen_helper_gvec_5 *fn)
247	{
248	TCGv_ptr a0, a1, a2, a3, a4;
249	TCGv_i32 desc = tcg_constant_i32(simd_desc(oprsz, maxsz, data));
250
251	a0 = tcg_temp_ebb_new_ptr();
252	a1 = tcg_temp_ebb_new_ptr();
253	a2 = tcg_temp_ebb_new_ptr();
254	a3 = tcg_temp_ebb_new_ptr();
255	a4 = tcg_temp_ebb_new_ptr();
256
257	tcg_gen_addi_ptr(a0, tcg_env, dofs);
258	tcg_gen_addi_ptr(a1, tcg_env, aofs);
259	tcg_gen_addi_ptr(a2, tcg_env, bofs);
260	tcg_gen_addi_ptr(a3, tcg_env, cofs);
261	tcg_gen_addi_ptr(a4, tcg_env, xofs);
262
263	fn(a0, a1, a2, a3, a4, desc);
264
265	tcg_temp_free_ptr(a0);
266	tcg_temp_free_ptr(a1);
267	tcg_temp_free_ptr(a2);
268	tcg_temp_free_ptr(a3);
269	tcg_temp_free_ptr(a4);
270	}
271
272	/* Generate a call to a gvec-style helper with three vector operands
273	and an extra pointer operand. */
274	void tcg_gen_gvec_2_ptr(uint32_t dofs, uint32_t aofs,
275	TCGv_ptr ptr, uint32_t oprsz, uint32_t maxsz,
276	int32_t data, gen_helper_gvec_2_ptr *fn)
277	{
278	TCGv_ptr a0, a1;
279	TCGv_i32 desc = tcg_constant_i32(simd_desc(oprsz, maxsz, data));
280
281	a0 = tcg_temp_ebb_new_ptr();
282	a1 = tcg_temp_ebb_new_ptr();
283
284	tcg_gen_addi_ptr(a0, tcg_env, dofs);
285	tcg_gen_addi_ptr(a1, tcg_env, aofs);
286
287	fn(a0, a1, ptr, desc);
288
289	tcg_temp_free_ptr(a0);
290	tcg_temp_free_ptr(a1);
291	}
292
293	/* Generate a call to a gvec-style helper with three vector operands
294	and an extra pointer operand. */
295	void tcg_gen_gvec_3_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs,
296	TCGv_ptr ptr, uint32_t oprsz, uint32_t maxsz,
297	int32_t data, gen_helper_gvec_3_ptr *fn)
298	{
299	TCGv_ptr a0, a1, a2;
300	TCGv_i32 desc = tcg_constant_i32(simd_desc(oprsz, maxsz, data));
301
302	a0 = tcg_temp_ebb_new_ptr();
303	a1 = tcg_temp_ebb_new_ptr();
304	a2 = tcg_temp_ebb_new_ptr();
305
306	tcg_gen_addi_ptr(a0, tcg_env, dofs);
307	tcg_gen_addi_ptr(a1, tcg_env, aofs);
308	tcg_gen_addi_ptr(a2, tcg_env, bofs);
309
310	fn(a0, a1, a2, ptr, desc);
311
312	tcg_temp_free_ptr(a0);
313	tcg_temp_free_ptr(a1);
314	tcg_temp_free_ptr(a2);
315	}
316
317	/* Generate a call to a gvec-style helper with four vector operands
318	and an extra pointer operand. */
319	void tcg_gen_gvec_4_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs,
320	uint32_t cofs, TCGv_ptr ptr, uint32_t oprsz,
321	uint32_t maxsz, int32_t data,
322	gen_helper_gvec_4_ptr *fn)
323	{
324	TCGv_ptr a0, a1, a2, a3;
325	TCGv_i32 desc = tcg_constant_i32(simd_desc(oprsz, maxsz, data));
326
327	a0 = tcg_temp_ebb_new_ptr();
328	a1 = tcg_temp_ebb_new_ptr();
329	a2 = tcg_temp_ebb_new_ptr();
330	a3 = tcg_temp_ebb_new_ptr();
331
332	tcg_gen_addi_ptr(a0, tcg_env, dofs);
333	tcg_gen_addi_ptr(a1, tcg_env, aofs);
334	tcg_gen_addi_ptr(a2, tcg_env, bofs);
335	tcg_gen_addi_ptr(a3, tcg_env, cofs);
336
337	fn(a0, a1, a2, a3, ptr, desc);
338
339	tcg_temp_free_ptr(a0);
340	tcg_temp_free_ptr(a1);
341	tcg_temp_free_ptr(a2);
342	tcg_temp_free_ptr(a3);
343	}
344
345	/* Generate a call to a gvec-style helper with five vector operands
346	and an extra pointer operand. */
347	void tcg_gen_gvec_5_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs,
348	uint32_t cofs, uint32_t eofs, TCGv_ptr ptr,
349	uint32_t oprsz, uint32_t maxsz, int32_t data,
350	gen_helper_gvec_5_ptr *fn)
351	{
352	TCGv_ptr a0, a1, a2, a3, a4;
353	TCGv_i32 desc = tcg_constant_i32(simd_desc(oprsz, maxsz, data));
354
355	a0 = tcg_temp_ebb_new_ptr();
356	a1 = tcg_temp_ebb_new_ptr();
357	a2 = tcg_temp_ebb_new_ptr();
358	a3 = tcg_temp_ebb_new_ptr();
359	a4 = tcg_temp_ebb_new_ptr();
360
361	tcg_gen_addi_ptr(a0, tcg_env, dofs);
362	tcg_gen_addi_ptr(a1, tcg_env, aofs);
363	tcg_gen_addi_ptr(a2, tcg_env, bofs);
364	tcg_gen_addi_ptr(a3, tcg_env, cofs);
365	tcg_gen_addi_ptr(a4, tcg_env, eofs);
366
367	fn(a0, a1, a2, a3, a4, ptr, desc);
368
369	tcg_temp_free_ptr(a0);
370	tcg_temp_free_ptr(a1);
371	tcg_temp_free_ptr(a2);
372	tcg_temp_free_ptr(a3);
373	tcg_temp_free_ptr(a4);
374	}
375
376	/* Return true if we want to implement something of OPRSZ bytes
377	in units of LNSZ. This limits the expansion of inline code. */
378	static inline bool check_size_impl(uint32_t oprsz, uint32_t lnsz)
379	{
380	uint32_t q, r;
381
382	if (oprsz < lnsz) {
383	return false;
384	}
385
386	q = oprsz / lnsz;
387	r = oprsz % lnsz;
388	tcg_debug_assert((r & 7) == 0);
389
390	if (lnsz < 16) {
391	/* For sizes below 16, accept no remainder. */
392	if (r != 0) {
393	return false;
394	}
395	} else {
396	/*
397	* Recall that ARM SVE allows vector sizes that are not a
398	* power of 2, but always a multiple of 16. The intent is
399	* that e.g. size == 80 would be expanded with 2x32 + 1x16.
400	* In addition, expand_clr needs to handle a multiple of 8.
401	* Thus we can handle the tail with one more operation per
402	* diminishing power of 2.
403	*/
404	q += ctpop32(r);
405	}
406
407	return q <= MAX_UNROLL;
408	}
409
410	static void expand_clr(TCGv_ptr dbase, uint32_t dofs, uint32_t maxsz);
411
412	/* Duplicate C as per VECE. */
413	uint64_t (dup_const)(unsigned vece, uint64_t c)
414	{
415	switch (vece) {
416	case MO_8:
417	return 0x0101010101010101ull * (uint8_t)c;
418	case MO_16:
419	return 0x0001000100010001ull * (uint16_t)c;
420	case MO_32:
421	return 0x0000000100000001ull * (uint32_t)c;
422	case MO_64:
423	return c;
424	default:
425	g_assert_not_reached();
426	}
427	}
428
429	/* Duplicate IN into OUT as per VECE. */
430	void tcg_gen_dup_i32(unsigned vece, TCGv_i32 out, TCGv_i32 in)
431	{
432	switch (vece) {
433	case MO_8:
434	tcg_gen_ext8u_i32(out, in);
435	tcg_gen_muli_i32(out, out, 0x01010101);
436	break;
437	case MO_16:
438	tcg_gen_deposit_i32(out, in, in, 16, 16);
439	break;
440	case MO_32:
441	tcg_gen_mov_i32(out, in);
442	break;
443	default:
444	g_assert_not_reached();
445	}
446	}
447
448	void tcg_gen_dup_i64(unsigned vece, TCGv_i64 out, TCGv_i64 in)
449	{
450	switch (vece) {
451	case MO_8:
452	tcg_gen_ext8u_i64(out, in);
453	tcg_gen_muli_i64(out, out, 0x0101010101010101ull);
454	break;
455	case MO_16:
456	tcg_gen_ext16u_i64(out, in);
457	tcg_gen_muli_i64(out, out, 0x0001000100010001ull);
458	break;
459	case MO_32:
460	tcg_gen_deposit_i64(out, in, in, 32, 32);
461	break;
462	case MO_64:
463	tcg_gen_mov_i64(out, in);
464	break;
465	default:
466	g_assert_not_reached();
467	}
468	}
469
470	/* Select a supported vector type for implementing an operation on SIZE
471	* bytes. If OP is 0, assume that the real operation to be performed is
472	* required by all backends. Otherwise, make sure than OP can be performed
473	* on elements of size VECE in the selected type. Do not select V64 if
474	* PREFER_I64 is true. Return 0 if no vector type is selected.
475	*/
476	static TCGType choose_vector_type(const TCGOpcode *list, unsigned vece,
477	uint32_t size, bool prefer_i64)
478	{
479	/*
480	* Recall that ARM SVE allows vector sizes that are not a
481	* power of 2, but always a multiple of 16. The intent is
482	* that e.g. size == 80 would be expanded with 2x32 + 1x16.
483	* It is hard to imagine a case in which v256 is supported
484	* but v128 is not, but check anyway.
485	* In addition, expand_clr needs to handle a multiple of 8.
486	*/
487	if (TCG_TARGET_HAS_v256 &&
488	check_size_impl(size, 32) &&
489	tcg_can_emit_vecop_list(list, TCG_TYPE_V256, vece) &&
490	(!(size & 16) \|\|
491	(TCG_TARGET_HAS_v128 &&
492	tcg_can_emit_vecop_list(list, TCG_TYPE_V128, vece))) &&
493	(!(size & 8) \|\|
494	(TCG_TARGET_HAS_v64 &&
495	tcg_can_emit_vecop_list(list, TCG_TYPE_V64, vece)))) {
496	return TCG_TYPE_V256;
497	}
498	if (TCG_TARGET_HAS_v128 &&
499	check_size_impl(size, 16) &&
500	tcg_can_emit_vecop_list(list, TCG_TYPE_V128, vece) &&
501	(!(size & 8) \|\|
502	(TCG_TARGET_HAS_v64 &&
503	tcg_can_emit_vecop_list(list, TCG_TYPE_V64, vece)))) {
504	return TCG_TYPE_V128;
505	}
506	if (TCG_TARGET_HAS_v64 && !prefer_i64 && check_size_impl(size, 8)
507	&& tcg_can_emit_vecop_list(list, TCG_TYPE_V64, vece)) {
508	return TCG_TYPE_V64;
509	}
510	return 0;
511	}
512
513	static void do_dup_store(TCGType type, TCGv_ptr dbase, uint32_t dofs,
514	uint32_t oprsz, uint32_t maxsz, TCGv_vec t_vec)
515	{
516	uint32_t i = 0;
517
518	tcg_debug_assert(oprsz >= 8);
519
520	/*
521	* This may be expand_clr for the tail of an operation, e.g.
522	* oprsz == 8 && maxsz == 64. The first 8 bytes of this store
523	* are misaligned wrt the maximum vector size, so do that first.
524	*/
525	if (dofs & 8) {
526	tcg_gen_stl_vec(t_vec, dbase, dofs + i, TCG_TYPE_V64);
527	i += 8;
528	}
529
530	switch (type) {
531	case TCG_TYPE_V256:
532	/*
533	* Recall that ARM SVE allows vector sizes that are not a
534	* power of 2, but always a multiple of 16. The intent is
535	* that e.g. size == 80 would be expanded with 2x32 + 1x16.
536	*/
537	for (; i + 32 <= oprsz; i += 32) {
538	tcg_gen_stl_vec(t_vec, dbase, dofs + i, TCG_TYPE_V256);
539	}
540	/* fallthru */
541	case TCG_TYPE_V128:
542	for (; i + 16 <= oprsz; i += 16) {
543	tcg_gen_stl_vec(t_vec, dbase, dofs + i, TCG_TYPE_V128);
544	}
545	break;
546	case TCG_TYPE_V64:
547	for (; i < oprsz; i += 8) {
548	tcg_gen_stl_vec(t_vec, dbase, dofs + i, TCG_TYPE_V64);
549	}
550	break;
551	default:
552	g_assert_not_reached();
553	}
554
555	if (oprsz < maxsz) {
556	expand_clr(dbase, dofs + oprsz, maxsz - oprsz);
557	}
558	}
559
560	/*
561	* Set OPRSZ bytes at DBASE + DOFS to replications of IN_32, IN_64 or IN_C.
562	* Only one of IN_32 or IN_64 may be set;
563	* IN_C is used if IN_32 and IN_64 are unset.
564	*/
565	static void do_dup(unsigned vece, TCGv_ptr dbase, uint32_t dofs,
566	uint32_t oprsz, uint32_t maxsz,
567	TCGv_i32 in_32, TCGv_i64 in_64, uint64_t in_c)
568	{
569	TCGType type;
570	TCGv_i64 t_64;
571	TCGv_i32 t_32, t_desc;
572	TCGv_ptr t_ptr;
573	uint32_t i;
574
575	assert(vece <= (in_32 ? MO_32 : MO_64));
576	assert(in_32 == NULL \|\| in_64 == NULL);
577
578	/* If we're storing 0, expand oprsz to maxsz. */
579	if (in_32 == NULL && in_64 == NULL) {
580	in_c = dup_const(vece, in_c);
581	if (in_c == 0) {
582	oprsz = maxsz;
583	vece = MO_8;
584	} else if (in_c == dup_const(MO_8, in_c)) {
585	vece = MO_8;
586	}
587	}
588
589	/*
590	* Implement inline with a vector type, if possible;
591	* prefer_i64 with 64-bit variable dup.
592	*/
593	type = choose_vector_type(NULL, vece, oprsz, vece == MO_64 && in_64);
594	if (type != 0) {
595	TCGv_vec t_vec = tcg_temp_new_vec(type);
596
597	if (in_32) {
598	tcg_gen_dup_i32_vec(vece, t_vec, in_32);
599	} else if (in_64) {
600	tcg_gen_dup_i64_vec(vece, t_vec, in_64);
601	} else {
602	tcg_gen_dupi_vec(vece, t_vec, in_c);
603	}
604	do_dup_store(type, dbase, dofs, oprsz, maxsz, t_vec);
605	return;
606	}
607
608	/* Otherwise, inline with an integer type, unless "large". */
609	if (check_size_impl(oprsz, sizeof(tcg_target_long))) {
610	t_64 = NULL;
611	t_32 = NULL;
612
613	if (in_32) {
614	/*
615	* We are given a 32-bit variable input. Use a 64-bit operation
616	* unless the 32-bit operation would be simple enough.
617	*/
618	if (vece != MO_32 \|\| !check_size_impl(oprsz, 4)) {
619	t_64 = tcg_temp_ebb_new_i64();
620	tcg_gen_extu_i32_i64(t_64, in_32);
621	tcg_gen_dup_i64(vece, t_64, t_64);
622	} else {
623	t_32 = tcg_temp_ebb_new_i32();
624	tcg_gen_dup_i32(vece, t_32, in_32);
625	}
626	} else if (in_64) {
627	/* We are given a 64-bit variable input. */
628	t_64 = tcg_temp_ebb_new_i64();
629	tcg_gen_dup_i64(vece, t_64, in_64);
630	} else {
631	/*
632	* We are given a constant input.
633	* Use 64-bit constants for "simple" constants or when we'd
634	* need too many 32-bit stores, or when a 64-bit constant
635	* is really required.
636	*/
637	if (vece == MO_64
638	\|\| in_c == 0
639	\|\| in_c == -1
640	\|\| !check_size_impl(oprsz, 4)) {
641	t_64 = tcg_constant_i64(in_c);
642	} else {
643	t_32 = tcg_constant_i32(in_c);
644	}
645	}
646
647	/* Implement inline if we picked an implementation size above. */
648	if (t_32) {
649	for (i = 0; i < oprsz; i += 4) {
650	tcg_gen_st_i32(t_32, dbase, dofs + i);
651	}
652	tcg_temp_free_i32(t_32);
653	goto done;
654	}
655	if (t_64) {
656	for (i = 0; i < oprsz; i += 8) {
657	tcg_gen_st_i64(t_64, dbase, dofs + i);
658	}
659	tcg_temp_free_i64(t_64);
660	goto done;
661	}
662	}
663
664	/* Otherwise implement out of line. */
665	t_ptr = tcg_temp_ebb_new_ptr();
666	tcg_gen_addi_ptr(t_ptr, dbase, dofs);
667
668	/*
669	* This may be expand_clr for the tail of an operation, e.g.
670	* oprsz == 8 && maxsz == 64. The size of the clear is misaligned
671	* wrt simd_desc and will assert. Simply pass all replicated byte
672	* stores through to memset.
673	*/
674	if (oprsz == maxsz && vece == MO_8) {
675	TCGv_ptr t_size = tcg_constant_ptr(oprsz);
676	TCGv_i32 t_val;
677
678	if (in_32) {
679	t_val = in_32;
680	} else if (in_64) {
681	t_val = tcg_temp_ebb_new_i32();
682	tcg_gen_extrl_i64_i32(t_val, in_64);
683	} else {
684	t_val = tcg_constant_i32(in_c);
685	}
686	gen_helper_memset(t_ptr, t_ptr, t_val, t_size);
687
688	if (in_64) {
689	tcg_temp_free_i32(t_val);
690	}
691	tcg_temp_free_ptr(t_ptr);
692	return;
693	}
694
695	t_desc = tcg_constant_i32(simd_desc(oprsz, maxsz, 0));
696
697	if (vece == MO_64) {
698	if (in_64) {
699	gen_helper_gvec_dup64(t_ptr, t_desc, in_64);
700	} else {
701	t_64 = tcg_constant_i64(in_c);
702	gen_helper_gvec_dup64(t_ptr, t_desc, t_64);
703	}
704	} else {
705	typedef void dup_fn(TCGv_ptr, TCGv_i32, TCGv_i32);
706	static dup_fn * const fns[3] = {
707	gen_helper_gvec_dup8,
708	gen_helper_gvec_dup16,
709	gen_helper_gvec_dup32
710	};
711
712	if (in_32) {
713	fns[vece](t_ptr, t_desc, in_32);
714	} else if (in_64) {
715	t_32 = tcg_temp_ebb_new_i32();
716	tcg_gen_extrl_i64_i32(t_32, in_64);
717	fns[vece](t_ptr, t_desc, t_32);
718	tcg_temp_free_i32(t_32);
719	} else {
720	if (vece == MO_8) {
721	in_c &= 0xff;
722	} else if (vece == MO_16) {
723	in_c &= 0xffff;
724	}
725	t_32 = tcg_constant_i32(in_c);
726	fns[vece](t_ptr, t_desc, t_32);
727	}
728	}
729
730	tcg_temp_free_ptr(t_ptr);
731	return;
732
733	done:
734	if (oprsz < maxsz) {
735	expand_clr(dbase, dofs + oprsz, maxsz - oprsz);
736	}
737	}
738
739	/* Likewise, but with zero. */
740	static void expand_clr(TCGv_ptr dbase, uint32_t dofs, uint32_t maxsz)
741	{
742	do_dup(MO_8, dbase, dofs, maxsz, maxsz, NULL, NULL, 0);
743	}
744
745	/* Expand OPSZ bytes worth of two-operand operations using i32 elements. */
746	static void expand_2_i32(TCGv_ptr dbase, uint32_t dofs, TCGv_ptr abase,
747	uint32_t aofs, uint32_t oprsz, bool load_dest,
748	void (*fni)(TCGv_i32, TCGv_i32))
749	{
750	TCGv_i32 t0 = tcg_temp_new_i32();
751	TCGv_i32 t1 = tcg_temp_new_i32();
752	uint32_t i;
753
754	for (i = 0; i < oprsz; i += 4) {
755	tcg_gen_ld_i32(t0, abase, aofs + i);
756	if (load_dest) {
757	tcg_gen_ld_i32(t1, dbase, dofs + i);
758	}
759	fni(t1, t0);
760	tcg_gen_st_i32(t1, dbase, dofs + i);
761	}
762	tcg_temp_free_i32(t0);
763	tcg_temp_free_i32(t1);
764	}
765
766	static void expand_2i_i32(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
767	int32_t c, bool load_dest,
768	void (*fni)(TCGv_i32, TCGv_i32, int32_t))
769	{
770	TCGv_i32 t0 = tcg_temp_new_i32();
771	TCGv_i32 t1 = tcg_temp_new_i32();
772	uint32_t i;
773
774	for (i = 0; i < oprsz; i += 4) {
775	tcg_gen_ld_i32(t0, tcg_env, aofs + i);
776	if (load_dest) {
777	tcg_gen_ld_i32(t1, tcg_env, dofs + i);
778	}
779	fni(t1, t0, c);
780	tcg_gen_st_i32(t1, tcg_env, dofs + i);
781	}
782	tcg_temp_free_i32(t0);
783	tcg_temp_free_i32(t1);
784	}
785
786	static void expand_2s_i32(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
787	TCGv_i32 c, bool scalar_first,
788	void (*fni)(TCGv_i32, TCGv_i32, TCGv_i32))
789	{
790	TCGv_i32 t0 = tcg_temp_new_i32();
791	TCGv_i32 t1 = tcg_temp_new_i32();
792	uint32_t i;
793
794	for (i = 0; i < oprsz; i += 4) {
795	tcg_gen_ld_i32(t0, tcg_env, aofs + i);
796	if (scalar_first) {
797	fni(t1, c, t0);
798	} else {
799	fni(t1, t0, c);
800	}
801	tcg_gen_st_i32(t1, tcg_env, dofs + i);
802	}
803	tcg_temp_free_i32(t0);
804	tcg_temp_free_i32(t1);
805	}
806
807	/* Expand OPSZ bytes worth of three-operand operations using i32 elements. */
808	static void expand_3_i32(TCGv_ptr dbase, uint32_t dofs,
809	TCGv_ptr abase, uint32_t aofs,
810	TCGv_ptr bbase, uint32_t bofs,
811	uint32_t oprsz, bool load_dest,
812	void (*fni)(TCGv_i32, TCGv_i32, TCGv_i32))
813	{
814	TCGv_i32 t0 = tcg_temp_new_i32();
815	TCGv_i32 t1 = tcg_temp_new_i32();
816	TCGv_i32 t2 = tcg_temp_new_i32();
817	uint32_t i;
818
819	for (i = 0; i < oprsz; i += 4) {
820	tcg_gen_ld_i32(t0, abase, aofs + i);
821	tcg_gen_ld_i32(t1, bbase, bofs + i);
822	if (load_dest) {
823	tcg_gen_ld_i32(t2, dbase, dofs + i);
824	}
825	fni(t2, t0, t1);
826	tcg_gen_st_i32(t2, dbase, dofs + i);
827	}
828	tcg_temp_free_i32(t2);
829	tcg_temp_free_i32(t1);
830	tcg_temp_free_i32(t0);
831	}
832
833	static void expand_3i_i32(uint32_t dofs, uint32_t aofs, uint32_t bofs,
834	uint32_t oprsz, int32_t c,
835	bool load_dest, bool write_aofs,
836	void (*fni)(TCGv_i32, TCGv_i32, TCGv_i32, int32_t))
837	{
838	TCGv_i32 t0 = tcg_temp_new_i32();
839	TCGv_i32 t1 = tcg_temp_new_i32();
840	TCGv_i32 t2 = tcg_temp_new_i32();
841	uint32_t i;
842
843	for (i = 0; i < oprsz; i += 4) {
844	tcg_gen_ld_i32(t0, tcg_env, aofs + i);
845	tcg_gen_ld_i32(t1, tcg_env, bofs + i);
846	if (load_dest) {
847	tcg_gen_ld_i32(t2, tcg_env, dofs + i);
848	}
849	fni(t2, t0, t1, c);
850	tcg_gen_st_i32(t2, tcg_env, dofs + i);
851	if (write_aofs) {
852	tcg_gen_st_i32(t0, tcg_env, aofs + i);
853	}
854	}
855	tcg_temp_free_i32(t0);
856	tcg_temp_free_i32(t1);
857	tcg_temp_free_i32(t2);
858	}
859
860	/* Expand OPSZ bytes worth of three-operand operations using i32 elements. */
861	static void expand_4_i32(uint32_t dofs, uint32_t aofs, uint32_t bofs,
862	uint32_t cofs, uint32_t oprsz, bool write_aofs,
863	void (*fni)(TCGv_i32, TCGv_i32, TCGv_i32, TCGv_i32))
864	{
865	TCGv_i32 t0 = tcg_temp_new_i32();
866	TCGv_i32 t1 = tcg_temp_new_i32();
867	TCGv_i32 t2 = tcg_temp_new_i32();
868	TCGv_i32 t3 = tcg_temp_new_i32();
869	uint32_t i;
870
871	for (i = 0; i < oprsz; i += 4) {
872	tcg_gen_ld_i32(t1, tcg_env, aofs + i);
873	tcg_gen_ld_i32(t2, tcg_env, bofs + i);
874	tcg_gen_ld_i32(t3, tcg_env, cofs + i);
875	fni(t0, t1, t2, t3);
876	tcg_gen_st_i32(t0, tcg_env, dofs + i);
877	if (write_aofs) {
878	tcg_gen_st_i32(t1, tcg_env, aofs + i);
879	}
880	}
881	tcg_temp_free_i32(t3);
882	tcg_temp_free_i32(t2);
883	tcg_temp_free_i32(t1);
884	tcg_temp_free_i32(t0);
885	}
886
887	static void expand_4i_i32(uint32_t dofs, uint32_t aofs, uint32_t bofs,
888	uint32_t cofs, uint32_t oprsz, int32_t c,
889	void (*fni)(TCGv_i32, TCGv_i32, TCGv_i32, TCGv_i32,
890	int32_t))
891	{
892	TCGv_i32 t0 = tcg_temp_new_i32();
893	TCGv_i32 t1 = tcg_temp_new_i32();
894	TCGv_i32 t2 = tcg_temp_new_i32();
895	TCGv_i32 t3 = tcg_temp_new_i32();
896	uint32_t i;
897
898	for (i = 0; i < oprsz; i += 4) {
899	tcg_gen_ld_i32(t1, tcg_env, aofs + i);
900	tcg_gen_ld_i32(t2, tcg_env, bofs + i);
901	tcg_gen_ld_i32(t3, tcg_env, cofs + i);
902	fni(t0, t1, t2, t3, c);
903	tcg_gen_st_i32(t0, tcg_env, dofs + i);
904	}
905	tcg_temp_free_i32(t3);
906	tcg_temp_free_i32(t2);
907	tcg_temp_free_i32(t1);
908	tcg_temp_free_i32(t0);
909	}
910
911	/* Expand OPSZ bytes worth of two-operand operations using i64 elements. */
912	static void expand_2_i64(TCGv_ptr dbase, uint32_t dofs, TCGv_ptr abase,
913	uint32_t aofs, uint32_t oprsz, bool load_dest,
914	void (*fni)(TCGv_i64, TCGv_i64))
915	{
916	TCGv_i64 t0 = tcg_temp_new_i64();
917	TCGv_i64 t1 = tcg_temp_new_i64();
918	uint32_t i;
919
920	for (i = 0; i < oprsz; i += 8) {
921	tcg_gen_ld_i64(t0, abase, aofs + i);
922	if (load_dest) {
923	tcg_gen_ld_i64(t1, dbase, dofs + i);
924	}
925	fni(t1, t0);
926	tcg_gen_st_i64(t1, dbase, dofs + i);
927	}
928	tcg_temp_free_i64(t0);
929	tcg_temp_free_i64(t1);
930	}
931
932	static void expand_2i_i64(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
933	int64_t c, bool load_dest,
934	void (*fni)(TCGv_i64, TCGv_i64, int64_t))
935	{
936	TCGv_i64 t0 = tcg_temp_new_i64();
937	TCGv_i64 t1 = tcg_temp_new_i64();
938	uint32_t i;
939
940	for (i = 0; i < oprsz; i += 8) {
941	tcg_gen_ld_i64(t0, tcg_env, aofs + i);
942	if (load_dest) {
943	tcg_gen_ld_i64(t1, tcg_env, dofs + i);
944	}
945	fni(t1, t0, c);
946	tcg_gen_st_i64(t1, tcg_env, dofs + i);
947	}
948	tcg_temp_free_i64(t0);
949	tcg_temp_free_i64(t1);
950	}
951
952	static void expand_2s_i64(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
953	TCGv_i64 c, bool scalar_first,
954	void (*fni)(TCGv_i64, TCGv_i64, TCGv_i64))
955	{
956	TCGv_i64 t0 = tcg_temp_new_i64();
957	TCGv_i64 t1 = tcg_temp_new_i64();
958	uint32_t i;
959
960	for (i = 0; i < oprsz; i += 8) {
961	tcg_gen_ld_i64(t0, tcg_env, aofs + i);
962	if (scalar_first) {
963	fni(t1, c, t0);
964	} else {
965	fni(t1, t0, c);
966	}
967	tcg_gen_st_i64(t1, tcg_env, dofs + i);
968	}
969	tcg_temp_free_i64(t0);
970	tcg_temp_free_i64(t1);
971	}
972
973	/* Expand OPSZ bytes worth of three-operand operations using i64 elements. */
974	static void expand_3_i64(TCGv_ptr dbase, uint32_t dofs,
975	TCGv_ptr abase, uint32_t aofs,
976	TCGv_ptr bbase, uint32_t bofs,
977	uint32_t oprsz, bool load_dest,
978	void (*fni)(TCGv_i64, TCGv_i64, TCGv_i64))
979	{
980	TCGv_i64 t0 = tcg_temp_new_i64();
981	TCGv_i64 t1 = tcg_temp_new_i64();
982	TCGv_i64 t2 = tcg_temp_new_i64();
983	uint32_t i;
984
985	for (i = 0; i < oprsz; i += 8) {
986	tcg_gen_ld_i64(t0, abase, aofs + i);
987	tcg_gen_ld_i64(t1, bbase, bofs + i);
988	if (load_dest) {
989	tcg_gen_ld_i64(t2, dbase, dofs + i);
990	}
991	fni(t2, t0, t1);
992	tcg_gen_st_i64(t2, dbase, dofs + i);
993	}
994	tcg_temp_free_i64(t2);
995	tcg_temp_free_i64(t1);
996	tcg_temp_free_i64(t0);
997	}
998
999	static void expand_3i_i64(uint32_t dofs, uint32_t aofs, uint32_t bofs,
1000	uint32_t oprsz, int64_t c,
1001	bool load_dest, bool write_aofs,
1002	void (*fni)(TCGv_i64, TCGv_i64, TCGv_i64, int64_t))
1003	{
1004	TCGv_i64 t0 = tcg_temp_new_i64();
1005	TCGv_i64 t1 = tcg_temp_new_i64();
1006	TCGv_i64 t2 = tcg_temp_new_i64();
1007	uint32_t i;
1008
1009	for (i = 0; i < oprsz; i += 8) {
1010	tcg_gen_ld_i64(t0, tcg_env, aofs + i);
1011	tcg_gen_ld_i64(t1, tcg_env, bofs + i);
1012	if (load_dest) {
1013	tcg_gen_ld_i64(t2, tcg_env, dofs + i);
1014	}
1015	fni(t2, t0, t1, c);
1016	tcg_gen_st_i64(t2, tcg_env, dofs + i);
1017	if (write_aofs) {
1018	tcg_gen_st_i64(t0, tcg_env, aofs + i);
1019	}
1020	}
1021	tcg_temp_free_i64(t0);
1022	tcg_temp_free_i64(t1);
1023	tcg_temp_free_i64(t2);
1024	}
1025
1026	/* Expand OPSZ bytes worth of three-operand operations using i64 elements. */
1027	static void expand_4_i64(uint32_t dofs, uint32_t aofs, uint32_t bofs,
1028	uint32_t cofs, uint32_t oprsz, bool write_aofs,
1029	void (*fni)(TCGv_i64, TCGv_i64, TCGv_i64, TCGv_i64))
1030	{
1031	TCGv_i64 t0 = tcg_temp_new_i64();
1032	TCGv_i64 t1 = tcg_temp_new_i64();
1033	TCGv_i64 t2 = tcg_temp_new_i64();
1034	TCGv_i64 t3 = tcg_temp_new_i64();
1035	uint32_t i;
1036
1037	for (i = 0; i < oprsz; i += 8) {
1038	tcg_gen_ld_i64(t1, tcg_env, aofs + i);
1039	tcg_gen_ld_i64(t2, tcg_env, bofs + i);
1040	tcg_gen_ld_i64(t3, tcg_env, cofs + i);
1041	fni(t0, t1, t2, t3);
1042	tcg_gen_st_i64(t0, tcg_env, dofs + i);
1043	if (write_aofs) {
1044	tcg_gen_st_i64(t1, tcg_env, aofs + i);
1045	}
1046	}
1047	tcg_temp_free_i64(t3);
1048	tcg_temp_free_i64(t2);
1049	tcg_temp_free_i64(t1);
1050	tcg_temp_free_i64(t0);
1051	}
1052
1053	static void expand_4i_i64(uint32_t dofs, uint32_t aofs, uint32_t bofs,
1054	uint32_t cofs, uint32_t oprsz, int64_t c,
1055	void (*fni)(TCGv_i64, TCGv_i64, TCGv_i64, TCGv_i64,
1056	int64_t))
1057	{
1058	TCGv_i64 t0 = tcg_temp_new_i64();
1059	TCGv_i64 t1 = tcg_temp_new_i64();
1060	TCGv_i64 t2 = tcg_temp_new_i64();
1061	TCGv_i64 t3 = tcg_temp_new_i64();
1062	uint32_t i;
1063
1064	for (i = 0; i < oprsz; i += 8) {
1065	tcg_gen_ld_i64(t1, tcg_env, aofs + i);
1066	tcg_gen_ld_i64(t2, tcg_env, bofs + i);
1067	tcg_gen_ld_i64(t3, tcg_env, cofs + i);
1068	fni(t0, t1, t2, t3, c);
1069	tcg_gen_st_i64(t0, tcg_env, dofs + i);
1070	}
1071	tcg_temp_free_i64(t3);
1072	tcg_temp_free_i64(t2);
1073	tcg_temp_free_i64(t1);
1074	tcg_temp_free_i64(t0);
1075	}
1076
1077	/* Expand OPSZ bytes worth of two-operand operations using host vectors. */
1078	static void expand_2_vec(unsigned vece, TCGv_ptr dbase, uint32_t dofs,
1079	TCGv_ptr abase, uint32_t aofs,
1080	uint32_t oprsz, uint32_t tysz, TCGType type,
1081	bool load_dest,
1082	void (*fni)(unsigned, TCGv_vec, TCGv_vec))
1083	{
1084	for (uint32_t i = 0; i < oprsz; i += tysz) {
1085	TCGv_vec t0 = tcg_temp_new_vec(type);
1086	TCGv_vec t1 = tcg_temp_new_vec(type);
1087
1088	tcg_gen_ld_vec(t0, abase, aofs + i);
1089	if (load_dest) {
1090	tcg_gen_ld_vec(t1, dbase, dofs + i);
1091	}
1092	fni(vece, t1, t0);
1093	tcg_gen_st_vec(t1, dbase, dofs + i);
1094	}
1095	}
1096
1097	/* Expand OPSZ bytes worth of two-vector operands and an immediate operand
1098	using host vectors. */
1099	static void expand_2i_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
1100	uint32_t oprsz, uint32_t tysz, TCGType type,
1101	int64_t c, bool load_dest,
1102	void (*fni)(unsigned, TCGv_vec, TCGv_vec, int64_t))
1103	{
1104	for (uint32_t i = 0; i < oprsz; i += tysz) {
1105	TCGv_vec t0 = tcg_temp_new_vec(type);
1106	TCGv_vec t1 = tcg_temp_new_vec(type);
1107
1108	tcg_gen_ld_vec(t0, tcg_env, aofs + i);
1109	if (load_dest) {
1110	tcg_gen_ld_vec(t1, tcg_env, dofs + i);
1111	}
1112	fni(vece, t1, t0, c);
1113	tcg_gen_st_vec(t1, tcg_env, dofs + i);
1114	}
1115	}
1116
1117	static void expand_2s_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
1118	uint32_t oprsz, uint32_t tysz, TCGType type,
1119	TCGv_vec c, bool scalar_first,
1120	void (*fni)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec))
1121	{
1122	for (uint32_t i = 0; i < oprsz; i += tysz) {
1123	TCGv_vec t0 = tcg_temp_new_vec(type);
1124	TCGv_vec t1 = tcg_temp_new_vec(type);
1125
1126	tcg_gen_ld_vec(t0, tcg_env, aofs + i);
1127	if (scalar_first) {
1128	fni(vece, t1, c, t0);
1129	} else {
1130	fni(vece, t1, t0, c);
1131	}
1132	tcg_gen_st_vec(t1, tcg_env, dofs + i);
1133	}
1134	}
1135
1136	/* Expand OPSZ bytes worth of three-operand operations using host vectors. */
1137	static void expand_3_vec(unsigned vece, TCGv_ptr dbase, uint32_t dofs,
1138	TCGv_ptr abase, uint32_t aofs,
1139	TCGv_ptr bbase, uint32_t bofs, uint32_t oprsz,
1140	uint32_t tysz, TCGType type, bool load_dest,
1141	void (*fni)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec))
1142	{
1143	for (uint32_t i = 0; i < oprsz; i += tysz) {
1144	TCGv_vec t0 = tcg_temp_new_vec(type);
1145	TCGv_vec t1 = tcg_temp_new_vec(type);
1146	TCGv_vec t2 = tcg_temp_new_vec(type);
1147
1148	tcg_gen_ld_vec(t0, abase, aofs + i);
1149	tcg_gen_ld_vec(t1, bbase, bofs + i);
1150	if (load_dest) {
1151	tcg_gen_ld_vec(t2, dbase, dofs + i);
1152	}
1153	fni(vece, t2, t0, t1);
1154	tcg_gen_st_vec(t2, dbase, dofs + i);
1155	}
1156	}
1157
1158	/*
1159	* Expand OPSZ bytes worth of three-vector operands and an immediate operand
1160	* using host vectors.
1161	*/
1162	static void expand_3i_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
1163	uint32_t bofs, uint32_t oprsz, uint32_t tysz,
1164	TCGType type, int64_t c,
1165	bool load_dest, bool write_aofs,
1166	void (*fni)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec,
1167	int64_t))
1168	{
1169	for (uint32_t i = 0; i < oprsz; i += tysz) {
1170	TCGv_vec t0 = tcg_temp_new_vec(type);
1171	TCGv_vec t1 = tcg_temp_new_vec(type);
1172	TCGv_vec t2 = tcg_temp_new_vec(type);
1173
1174	tcg_gen_ld_vec(t0, tcg_env, aofs + i);
1175	tcg_gen_ld_vec(t1, tcg_env, bofs + i);
1176	if (load_dest) {
1177	tcg_gen_ld_vec(t2, tcg_env, dofs + i);
1178	}
1179	fni(vece, t2, t0, t1, c);
1180	tcg_gen_st_vec(t2, tcg_env, dofs + i);
1181	if (write_aofs) {
1182	tcg_gen_st_vec(t0, tcg_env, aofs + i);
1183	}
1184	}
1185	}
1186
1187	/* Expand OPSZ bytes worth of four-operand operations using host vectors. */
1188	static void expand_4_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
1189	uint32_t bofs, uint32_t cofs, uint32_t oprsz,
1190	uint32_t tysz, TCGType type, bool write_aofs,
1191	void (*fni)(unsigned, TCGv_vec, TCGv_vec,
1192	TCGv_vec, TCGv_vec))
1193	{
1194	for (uint32_t i = 0; i < oprsz; i += tysz) {
1195	TCGv_vec t0 = tcg_temp_new_vec(type);
1196	TCGv_vec t1 = tcg_temp_new_vec(type);
1197	TCGv_vec t2 = tcg_temp_new_vec(type);
1198	TCGv_vec t3 = tcg_temp_new_vec(type);
1199
1200	tcg_gen_ld_vec(t1, tcg_env, aofs + i);
1201	tcg_gen_ld_vec(t2, tcg_env, bofs + i);
1202	tcg_gen_ld_vec(t3, tcg_env, cofs + i);
1203	fni(vece, t0, t1, t2, t3);
1204	tcg_gen_st_vec(t0, tcg_env, dofs + i);
1205	if (write_aofs) {
1206	tcg_gen_st_vec(t1, tcg_env, aofs + i);
1207	}
1208	}
1209	}
1210
1211	/*
1212	* Expand OPSZ bytes worth of four-vector operands and an immediate operand
1213	* using host vectors.
1214	*/
1215	static void expand_4i_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
1216	uint32_t bofs, uint32_t cofs, uint32_t oprsz,
1217	uint32_t tysz, TCGType type, int64_t c,
1218	void (*fni)(unsigned, TCGv_vec, TCGv_vec,
1219	TCGv_vec, TCGv_vec, int64_t))
1220	{
1221	for (uint32_t i = 0; i < oprsz; i += tysz) {
1222	TCGv_vec t0 = tcg_temp_new_vec(type);
1223	TCGv_vec t1 = tcg_temp_new_vec(type);
1224	TCGv_vec t2 = tcg_temp_new_vec(type);
1225	TCGv_vec t3 = tcg_temp_new_vec(type);
1226
1227	tcg_gen_ld_vec(t1, tcg_env, aofs + i);
1228	tcg_gen_ld_vec(t2, tcg_env, bofs + i);
1229	tcg_gen_ld_vec(t3, tcg_env, cofs + i);
1230	fni(vece, t0, t1, t2, t3, c);
1231	tcg_gen_st_vec(t0, tcg_env, dofs + i);
1232	}
1233	}
1234
1235	/* Expand a vector two-operand operation. */
1236	void tcg_gen_gvec_2_var(TCGv_ptr dbase, uint32_t dofs,
1237	TCGv_ptr abase, uint32_t aofs,
1238	uint32_t oprsz, uint32_t maxsz, const GVecGen2 *g)
1239	{
1240	const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1241	const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
1242	TCGType type;
1243	uint32_t some;
1244
1245	check_size_align(oprsz, maxsz, dofs \| aofs);
1246	check_overlap_2(dbase, dofs, abase, aofs, maxsz);
1247
1248	type = 0;
1249	if (g->fniv) {
1250	type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
1251	}
1252	switch (type) {
1253	case TCG_TYPE_V256:
1254	/* Recall that ARM SVE allows vector sizes that are not a
1255	* power of 2, but always a multiple of 16. The intent is
1256	* that e.g. size == 80 would be expanded with 2x32 + 1x16.
1257	*/
1258	some = QEMU_ALIGN_DOWN(oprsz, 32);
1259	expand_2_vec(g->vece, dbase, dofs, abase, aofs, some, 32,
1260	TCG_TYPE_V256, g->load_dest, g->fniv);
1261	if (some == oprsz) {
1262	break;
1263	}
1264	dofs += some;
1265	aofs += some;
1266	oprsz -= some;
1267	maxsz -= some;
1268	/* fallthru */
1269	case TCG_TYPE_V128:
1270	expand_2_vec(g->vece, dbase, dofs, abase, aofs, oprsz, 16,
1271	TCG_TYPE_V128, g->load_dest, g->fniv);
1272	break;
1273	case TCG_TYPE_V64:
1274	expand_2_vec(g->vece, dbase, dofs, abase, aofs, oprsz, 8,
1275	TCG_TYPE_V64, g->load_dest, g->fniv);
1276	break;
1277
1278	case 0:
1279	if (g->fni8 && check_size_impl(oprsz, 8)) {
1280	expand_2_i64(dbase, dofs, abase, aofs,
1281	oprsz, g->load_dest, g->fni8);
1282	} else if (g->fni4 && check_size_impl(oprsz, 4)) {
1283	expand_2_i32(dbase, dofs, abase, aofs,
1284	oprsz, g->load_dest, g->fni4);
1285	} else {
1286	assert(g->fno != NULL);
1287	expand_2_ool(dbase, dofs, abase, aofs,
1288	oprsz, maxsz, g->data, g->fno);
1289	oprsz = maxsz;
1290	}
1291	break;
1292
1293	default:
1294	g_assert_not_reached();
1295	}
1296	tcg_swap_vecop_list(hold_list);
1297
1298	if (oprsz < maxsz) {
1299	expand_clr(dbase, dofs + oprsz, maxsz - oprsz);
1300	}
1301	}
1302
1303	void tcg_gen_gvec_2(uint32_t dofs, uint32_t aofs,
1304	uint32_t oprsz, uint32_t maxsz, const GVecGen2 *g)
1305	{
1306	tcg_gen_gvec_2_var(tcg_env, dofs, tcg_env, aofs, oprsz, maxsz, g);
1307	}
1308
1309	/* Expand a vector operation with two vectors and an immediate. */
1310	void tcg_gen_gvec_2i(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
1311	uint32_t maxsz, int64_t c, const GVecGen2i *g)
1312	{
1313	const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1314	const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
1315	TCGType type;
1316	uint32_t some;
1317
1318	check_size_align(oprsz, maxsz, dofs \| aofs);
1319	check_overlap_2(tcg_env, dofs, tcg_env, aofs, maxsz);
1320
1321	type = 0;
1322	if (g->fniv) {
1323	type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
1324	}
1325	switch (type) {
1326	case TCG_TYPE_V256:
1327	/* Recall that ARM SVE allows vector sizes that are not a
1328	* power of 2, but always a multiple of 16. The intent is
1329	* that e.g. size == 80 would be expanded with 2x32 + 1x16.
1330	*/
1331	some = QEMU_ALIGN_DOWN(oprsz, 32);
1332	expand_2i_vec(g->vece, dofs, aofs, some, 32, TCG_TYPE_V256,
1333	c, g->load_dest, g->fniv);
1334	if (some == oprsz) {
1335	break;
1336	}
1337	dofs += some;
1338	aofs += some;
1339	oprsz -= some;
1340	maxsz -= some;
1341	/* fallthru */
1342	case TCG_TYPE_V128:
1343	expand_2i_vec(g->vece, dofs, aofs, oprsz, 16, TCG_TYPE_V128,
1344	c, g->load_dest, g->fniv);
1345	break;
1346	case TCG_TYPE_V64:
1347	expand_2i_vec(g->vece, dofs, aofs, oprsz, 8, TCG_TYPE_V64,
1348	c, g->load_dest, g->fniv);
1349	break;
1350
1351	case 0:
1352	if (g->fni8 && check_size_impl(oprsz, 8)) {
1353	expand_2i_i64(dofs, aofs, oprsz, c, g->load_dest, g->fni8);
1354	} else if (g->fni4 && check_size_impl(oprsz, 4)) {
1355	expand_2i_i32(dofs, aofs, oprsz, c, g->load_dest, g->fni4);
1356	} else {
1357	if (g->fno) {
1358	tcg_gen_gvec_2_ool(dofs, aofs, oprsz, maxsz, c, g->fno);
1359	} else {
1360	TCGv_i64 tcg_c = tcg_constant_i64(c);
1361	tcg_gen_gvec_2i_ool(dofs, aofs, tcg_c, oprsz,
1362	maxsz, c, g->fnoi);
1363	}
1364	oprsz = maxsz;
1365	}
1366	break;
1367
1368	default:
1369	g_assert_not_reached();
1370	}
1371	tcg_swap_vecop_list(hold_list);
1372
1373	if (oprsz < maxsz) {
1374	expand_clr(tcg_env, dofs + oprsz, maxsz - oprsz);
1375	}
1376	}
1377
1378	/* Expand a vector operation with two vectors and a scalar. */
1379	void tcg_gen_gvec_2s(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
1380	uint32_t maxsz, TCGv_i64 c, const GVecGen2s *g)
1381	{
1382	TCGType type;
1383
1384	check_size_align(oprsz, maxsz, dofs \| aofs);
1385	check_overlap_2(tcg_env, dofs, tcg_env, aofs, maxsz);
1386
1387	type = 0;
1388	if (g->fniv) {
1389	type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
1390	}
1391	if (type != 0) {
1392	const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1393	const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
1394	TCGv_vec t_vec = tcg_temp_new_vec(type);
1395	uint32_t some;
1396
1397	tcg_gen_dup_i64_vec(g->vece, t_vec, c);
1398
1399	switch (type) {
1400	case TCG_TYPE_V256:
1401	/* Recall that ARM SVE allows vector sizes that are not a
1402	* power of 2, but always a multiple of 16. The intent is
1403	* that e.g. size == 80 would be expanded with 2x32 + 1x16.
1404	*/
1405	some = QEMU_ALIGN_DOWN(oprsz, 32);
1406	expand_2s_vec(g->vece, dofs, aofs, some, 32, TCG_TYPE_V256,
1407	t_vec, g->scalar_first, g->fniv);
1408	if (some == oprsz) {
1409	break;
1410	}
1411	dofs += some;
1412	aofs += some;
1413	oprsz -= some;
1414	maxsz -= some;
1415	/* fallthru */
1416
1417	case TCG_TYPE_V128:
1418	expand_2s_vec(g->vece, dofs, aofs, oprsz, 16, TCG_TYPE_V128,
1419	t_vec, g->scalar_first, g->fniv);
1420	break;
1421
1422	case TCG_TYPE_V64:
1423	expand_2s_vec(g->vece, dofs, aofs, oprsz, 8, TCG_TYPE_V64,
1424	t_vec, g->scalar_first, g->fniv);
1425	break;
1426
1427	default:
1428	g_assert_not_reached();
1429	}
1430	tcg_temp_free_vec(t_vec);
1431	tcg_swap_vecop_list(hold_list);
1432	} else if (g->fni8 && check_size_impl(oprsz, 8)) {
1433	TCGv_i64 t64 = tcg_temp_new_i64();
1434
1435	tcg_gen_dup_i64(g->vece, t64, c);
1436	expand_2s_i64(dofs, aofs, oprsz, t64, g->scalar_first, g->fni8);
1437	tcg_temp_free_i64(t64);
1438	} else if (g->fni4 && check_size_impl(oprsz, 4)) {
1439	TCGv_i32 t32 = tcg_temp_new_i32();
1440
1441	tcg_gen_extrl_i64_i32(t32, c);
1442	tcg_gen_dup_i32(g->vece, t32, t32);
1443	expand_2s_i32(dofs, aofs, oprsz, t32, g->scalar_first, g->fni4);
1444	tcg_temp_free_i32(t32);
1445	} else {
1446	tcg_gen_gvec_2i_ool(dofs, aofs, c, oprsz, maxsz, 0, g->fno);
1447	return;
1448	}
1449
1450	if (oprsz < maxsz) {
1451	expand_clr(tcg_env, dofs + oprsz, maxsz - oprsz);
1452	}
1453	}
1454
1455	/* Expand a vector three-operand operation. */
1456	void tcg_gen_gvec_3_var(TCGv_ptr dbase, uint32_t dofs,
1457	TCGv_ptr abase, uint32_t aofs,
1458	TCGv_ptr bbase, uint32_t bofs,
1459	uint32_t oprsz, uint32_t maxsz, const GVecGen3 *g)
1460	{
1461	const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1462	const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
1463	TCGType type;
1464	uint32_t some;
1465
1466	check_size_align(oprsz, maxsz, dofs \| aofs \| bofs);
1467	check_overlap_3(dbase, dofs, abase, aofs, bbase, bofs, maxsz);
1468
1469	type = 0;
1470	if (g->fniv) {
1471	type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
1472	}
1473	switch (type) {
1474	case TCG_TYPE_V256:
1475	/* Recall that ARM SVE allows vector sizes that are not a
1476	* power of 2, but always a multiple of 16. The intent is
1477	* that e.g. size == 80 would be expanded with 2x32 + 1x16.
1478	*/
1479	some = QEMU_ALIGN_DOWN(oprsz, 32);
1480	expand_3_vec(g->vece, dbase, dofs, abase, aofs, bbase, bofs,
1481	some, 32, TCG_TYPE_V256, g->load_dest, g->fniv);
1482	if (some == oprsz) {
1483	break;
1484	}
1485	dofs += some;
1486	aofs += some;
1487	bofs += some;
1488	oprsz -= some;
1489	maxsz -= some;
1490	/* fallthru */
1491	case TCG_TYPE_V128:
1492	expand_3_vec(g->vece, dbase, dofs, abase, aofs, bbase, bofs,
1493	oprsz, 16, TCG_TYPE_V128, g->load_dest, g->fniv);
1494	break;
1495	case TCG_TYPE_V64:
1496	expand_3_vec(g->vece, dbase, dofs, abase, aofs, bbase, bofs,
1497	oprsz, 8, TCG_TYPE_V64, g->load_dest, g->fniv);
1498	break;
1499
1500	case 0:
1501	if (g->fni8 && check_size_impl(oprsz, 8)) {
1502	expand_3_i64(dbase, dofs, abase, aofs, bbase, bofs,
1503	oprsz, g->load_dest, g->fni8);
1504	} else if (g->fni4 && check_size_impl(oprsz, 4)) {
1505	expand_3_i32(dbase, dofs, abase, aofs, bbase, bofs,
1506	oprsz, g->load_dest, g->fni4);
1507	} else {
1508	assert(g->fno != NULL);
1509	expand_3_ool(dbase, dofs, abase, aofs, bbase, bofs,
1510	oprsz, maxsz, g->data, g->fno);
1511	oprsz = maxsz;
1512	}
1513	break;
1514
1515	default:
1516	g_assert_not_reached();
1517	}
1518	tcg_swap_vecop_list(hold_list);
1519
1520	if (oprsz < maxsz) {
1521	expand_clr(dbase, dofs + oprsz, maxsz - oprsz);
1522	}
1523	}
1524
1525	void tcg_gen_gvec_3(uint32_t dofs, uint32_t aofs, uint32_t bofs,
1526	uint32_t oprsz, uint32_t maxsz, const GVecGen3 *g)
1527	{
1528	tcg_gen_gvec_3_var(tcg_env, dofs, tcg_env, aofs, tcg_env, bofs,
1529	oprsz, maxsz, g);
1530	}
1531
1532	/* Expand a vector operation with three vectors and an immediate. */
1533	void tcg_gen_gvec_3i(uint32_t dofs, uint32_t aofs, uint32_t bofs,
1534	uint32_t oprsz, uint32_t maxsz, int64_t c,
1535	const GVecGen3i *g)
1536	{
1537	const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1538	const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
1539	TCGType type;
1540	uint32_t some;
1541
1542	check_size_align(oprsz, maxsz, dofs \| aofs \| bofs);
1543	check_overlap_3(tcg_env, dofs, tcg_env, aofs, tcg_env, bofs, maxsz);
1544
1545	type = 0;
1546	if (g->fniv) {
1547	type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
1548	}
1549	switch (type) {
1550	case TCG_TYPE_V256:
1551	/*
1552	* Recall that ARM SVE allows vector sizes that are not a
1553	* power of 2, but always a multiple of 16. The intent is
1554	* that e.g. size == 80 would be expanded with 2x32 + 1x16.
1555	*/
1556	some = QEMU_ALIGN_DOWN(oprsz, 32);
1557	expand_3i_vec(g->vece, dofs, aofs, bofs, some, 32, TCG_TYPE_V256,
1558	c, g->load_dest, g->write_aofs, g->fniv);
1559	if (some == oprsz) {
1560	break;
1561	}
1562	dofs += some;
1563	aofs += some;
1564	bofs += some;
1565	oprsz -= some;
1566	maxsz -= some;
1567	/* fallthru */
1568	case TCG_TYPE_V128:
1569	expand_3i_vec(g->vece, dofs, aofs, bofs, oprsz, 16, TCG_TYPE_V128,
1570	c, g->load_dest, g->write_aofs, g->fniv);
1571	break;
1572	case TCG_TYPE_V64:
1573	expand_3i_vec(g->vece, dofs, aofs, bofs, oprsz, 8, TCG_TYPE_V64,
1574	c, g->load_dest, g->write_aofs, g->fniv);
1575	break;
1576
1577	case 0:
1578	if (g->fni8 && check_size_impl(oprsz, 8)) {
1579	expand_3i_i64(dofs, aofs, bofs, oprsz, c,
1580	g->load_dest, g->write_aofs, g->fni8);
1581	} else if (g->fni4 && check_size_impl(oprsz, 4)) {
1582	expand_3i_i32(dofs, aofs, bofs, oprsz, c,
1583	g->load_dest, g->write_aofs, g->fni4);
1584	} else {
1585	assert(g->fno != NULL);
1586	tcg_gen_gvec_3_ool(dofs, aofs, bofs, oprsz, maxsz, c, g->fno);
1587	oprsz = maxsz;
1588	}
1589	break;
1590
1591	default:
1592	g_assert_not_reached();
1593	}
1594	tcg_swap_vecop_list(hold_list);
1595
1596	if (oprsz < maxsz) {
1597	expand_clr(tcg_env, dofs + oprsz, maxsz - oprsz);
1598	}
1599	}
1600
1601	/* Expand a vector four-operand operation. */
1602	void tcg_gen_gvec_4(uint32_t dofs, uint32_t aofs, uint32_t bofs, uint32_t cofs,
1603	uint32_t oprsz, uint32_t maxsz, const GVecGen4 *g)
1604	{
1605	const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1606	const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
1607	TCGType type;
1608	uint32_t some;
1609
1610	check_size_align(oprsz, maxsz, dofs \| aofs \| bofs \| cofs);
1611	check_overlap_4(tcg_env, dofs, tcg_env, aofs,
1612	tcg_env, bofs, tcg_env, cofs, maxsz);
1613
1614	type = 0;
1615	if (g->fniv) {
1616	type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
1617	}
1618	switch (type) {
1619	case TCG_TYPE_V256:
1620	/* Recall that ARM SVE allows vector sizes that are not a
1621	* power of 2, but always a multiple of 16. The intent is
1622	* that e.g. size == 80 would be expanded with 2x32 + 1x16.
1623	*/
1624	some = QEMU_ALIGN_DOWN(oprsz, 32);
1625	expand_4_vec(g->vece, dofs, aofs, bofs, cofs, some,
1626	32, TCG_TYPE_V256, g->write_aofs, g->fniv);
1627	if (some == oprsz) {
1628	break;
1629	}
1630	dofs += some;
1631	aofs += some;
1632	bofs += some;
1633	cofs += some;
1634	oprsz -= some;
1635	maxsz -= some;
1636	/* fallthru */
1637	case TCG_TYPE_V128:
1638	expand_4_vec(g->vece, dofs, aofs, bofs, cofs, oprsz,
1639	16, TCG_TYPE_V128, g->write_aofs, g->fniv);
1640	break;
1641	case TCG_TYPE_V64:
1642	expand_4_vec(g->vece, dofs, aofs, bofs, cofs, oprsz,
1643	8, TCG_TYPE_V64, g->write_aofs, g->fniv);
1644	break;
1645
1646	case 0:
1647	if (g->fni8 && check_size_impl(oprsz, 8)) {
1648	expand_4_i64(dofs, aofs, bofs, cofs, oprsz,
1649	g->write_aofs, g->fni8);
1650	} else if (g->fni4 && check_size_impl(oprsz, 4)) {
1651	expand_4_i32(dofs, aofs, bofs, cofs, oprsz,
1652	g->write_aofs, g->fni4);
1653	} else {
1654	assert(g->fno != NULL);
1655	tcg_gen_gvec_4_ool(dofs, aofs, bofs, cofs,
1656	oprsz, maxsz, g->data, g->fno);
1657	oprsz = maxsz;
1658	}
1659	break;
1660
1661	default:
1662	g_assert_not_reached();
1663	}
1664	tcg_swap_vecop_list(hold_list);
1665
1666	if (oprsz < maxsz) {
1667	expand_clr(tcg_env, dofs + oprsz, maxsz - oprsz);
1668	}
1669	}
1670
1671	/* Expand a vector four-operand operation. */
1672	void tcg_gen_gvec_4i(uint32_t dofs, uint32_t aofs, uint32_t bofs, uint32_t cofs,
1673	uint32_t oprsz, uint32_t maxsz, int64_t c,
1674	const GVecGen4i *g)
1675	{
1676	const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1677	const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
1678	TCGType type;
1679	uint32_t some;
1680
1681	check_size_align(oprsz, maxsz, dofs \| aofs \| bofs \| cofs);
1682	check_overlap_4(tcg_env, dofs, tcg_env, aofs,
1683	tcg_env, bofs, tcg_env, cofs, maxsz);
1684
1685	type = 0;
1686	if (g->fniv) {
1687	type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
1688	}
1689	switch (type) {
1690	case TCG_TYPE_V256:
1691	/*
1692	* Recall that ARM SVE allows vector sizes that are not a
1693	* power of 2, but always a multiple of 16. The intent is
1694	* that e.g. size == 80 would be expanded with 2x32 + 1x16.
1695	*/
1696	some = QEMU_ALIGN_DOWN(oprsz, 32);
1697	expand_4i_vec(g->vece, dofs, aofs, bofs, cofs, some,
1698	32, TCG_TYPE_V256, c, g->fniv);
1699	if (some == oprsz) {
1700	break;
1701	}
1702	dofs += some;
1703	aofs += some;
1704	bofs += some;
1705	cofs += some;
1706	oprsz -= some;
1707	maxsz -= some;
1708	/* fallthru */
1709	case TCG_TYPE_V128:
1710	expand_4i_vec(g->vece, dofs, aofs, bofs, cofs, oprsz,
1711	16, TCG_TYPE_V128, c, g->fniv);
1712	break;
1713	case TCG_TYPE_V64:
1714	expand_4i_vec(g->vece, dofs, aofs, bofs, cofs, oprsz,
1715	8, TCG_TYPE_V64, c, g->fniv);
1716	break;
1717
1718	case 0:
1719	if (g->fni8 && check_size_impl(oprsz, 8)) {
1720	expand_4i_i64(dofs, aofs, bofs, cofs, oprsz, c, g->fni8);
1721	} else if (g->fni4 && check_size_impl(oprsz, 4)) {
1722	expand_4i_i32(dofs, aofs, bofs, cofs, oprsz, c, g->fni4);
1723	} else {
1724	assert(g->fno != NULL);
1725	tcg_gen_gvec_4_ool(dofs, aofs, bofs, cofs,
1726	oprsz, maxsz, c, g->fno);
1727	oprsz = maxsz;
1728	}
1729	break;
1730
1731	default:
1732	g_assert_not_reached();
1733	}
1734	tcg_swap_vecop_list(hold_list);
1735
1736	if (oprsz < maxsz) {
1737	expand_clr(tcg_env, dofs + oprsz, maxsz - oprsz);
1738	}
1739	}
1740
1741	/*
1742	* Expand specific vector operations.
1743	*/
1744
1745	static void vec_mov2(unsigned vece, TCGv_vec a, TCGv_vec b)
1746	{
1747	tcg_gen_mov_vec(a, b);
1748	}
1749
1750	void tcg_gen_gvec_mov_var(unsigned vece, TCGv_ptr dbase, uint32_t dofs,
1751	TCGv_ptr abase, uint32_t aofs,
1752	uint32_t oprsz, uint32_t maxsz)
1753	{
1754	static const GVecGen2 g = {
1755	.fni8 = tcg_gen_mov_i64,
1756	.fniv = vec_mov2,
1757	.fno = gen_helper_gvec_mov,
1758	.prefer_i64 = true,
1759	};
1760
1761	if (dofs == aofs && dbase == abase) {
1762	check_size_align(oprsz, maxsz, dofs);
1763	if (oprsz < maxsz) {
1764	expand_clr(dbase, dofs + oprsz, maxsz - oprsz);
1765	}
1766	return;
1767	}
1768
1769	tcg_gen_gvec_2_var(dbase, dofs, abase, aofs, oprsz, maxsz, &g);
1770	}
1771
1772	void tcg_gen_gvec_mov(unsigned vece, uint32_t dofs, uint32_t aofs,
1773	uint32_t oprsz, uint32_t maxsz)
1774	{
1775	tcg_gen_gvec_mov_var(vece, tcg_env, dofs, tcg_env, aofs, oprsz, maxsz);
1776	}
1777
1778	void tcg_gen_gvec_dup_i32(unsigned vece, uint32_t dofs, uint32_t oprsz,
1779	uint32_t maxsz, TCGv_i32 in)
1780	{
1781	check_size_align(oprsz, maxsz, dofs);
1782	tcg_debug_assert(vece <= MO_32);
1783	do_dup(vece, tcg_env, dofs, oprsz, maxsz, in, NULL, 0);
1784	}
1785
1786	void tcg_gen_gvec_dup_i64(unsigned vece, uint32_t dofs, uint32_t oprsz,
1787	uint32_t maxsz, TCGv_i64 in)
1788	{
1789	check_size_align(oprsz, maxsz, dofs);
1790	tcg_debug_assert(vece <= MO_64);
1791	do_dup(vece, tcg_env, dofs, oprsz, maxsz, NULL, in, 0);
1792	}
1793
1794	void tcg_gen_gvec_dup_mem(unsigned vece, uint32_t dofs, uint32_t aofs,
1795	uint32_t oprsz, uint32_t maxsz)
1796	{
1797	check_size_align(oprsz, maxsz, dofs);
1798	if (vece <= MO_64) {
1799	TCGType type = choose_vector_type(NULL, vece, oprsz, 0);
1800	if (type != 0) {
1801	TCGv_vec t_vec = tcg_temp_new_vec(type);
1802	tcg_gen_dup_mem_vec(vece, t_vec, tcg_env, aofs);
1803	do_dup_store(type, tcg_env, dofs, oprsz, maxsz, t_vec);
1804	} else if (vece <= MO_32) {
1805	TCGv_i32 in = tcg_temp_ebb_new_i32();
1806	switch (vece) {
1807	case MO_8:
1808	tcg_gen_ld8u_i32(in, tcg_env, aofs);
1809	break;
1810	case MO_16:
1811	tcg_gen_ld16u_i32(in, tcg_env, aofs);
1812	break;
1813	default:
1814	tcg_gen_ld_i32(in, tcg_env, aofs);
1815	break;
1816	}
1817	do_dup(vece, tcg_env, dofs, oprsz, maxsz, in, NULL, 0);
1818	tcg_temp_free_i32(in);
1819	} else {
1820	TCGv_i64 in = tcg_temp_ebb_new_i64();
1821	tcg_gen_ld_i64(in, tcg_env, aofs);
1822	do_dup(vece, tcg_env, dofs, oprsz, maxsz, NULL, in, 0);
1823	tcg_temp_free_i64(in);
1824	}
1825	} else if (vece == 4) {
1826	/* 128-bit duplicate. */
1827	int i;
1828
1829	tcg_debug_assert(oprsz >= 16);
1830	if (TCG_TARGET_HAS_v128) {
1831	TCGv_vec in = tcg_temp_new_vec(TCG_TYPE_V128);
1832
1833	tcg_gen_ld_vec(in, tcg_env, aofs);
1834	for (i = (aofs == dofs) * 16; i < oprsz; i += 16) {
1835	tcg_gen_st_vec(in, tcg_env, dofs + i);
1836	}
1837	} else {
1838	TCGv_i64 in0 = tcg_temp_ebb_new_i64();
1839	TCGv_i64 in1 = tcg_temp_ebb_new_i64();
1840
1841	tcg_gen_ld_i64(in0, tcg_env, aofs);
1842	tcg_gen_ld_i64(in1, tcg_env, aofs + 8);
1843	for (i = (aofs == dofs) * 16; i < oprsz; i += 16) {
1844	tcg_gen_st_i64(in0, tcg_env, dofs + i);
1845	tcg_gen_st_i64(in1, tcg_env, dofs + i + 8);
1846	}
1847	tcg_temp_free_i64(in0);
1848	tcg_temp_free_i64(in1);
1849	}
1850	if (oprsz < maxsz) {
1851	expand_clr(tcg_env, dofs + oprsz, maxsz - oprsz);
1852	}
1853	} else if (vece == 5) {
1854	/* 256-bit duplicate. */
1855	int i;
1856
1857	tcg_debug_assert(oprsz >= 32);
1858	tcg_debug_assert(oprsz % 32 == 0);
1859	if (TCG_TARGET_HAS_v256) {
1860	TCGv_vec in = tcg_temp_new_vec(TCG_TYPE_V256);
1861
1862	tcg_gen_ld_vec(in, tcg_env, aofs);
1863	for (i = (aofs == dofs) * 32; i < oprsz; i += 32) {
1864	tcg_gen_st_vec(in, tcg_env, dofs + i);
1865	}
1866	} else if (TCG_TARGET_HAS_v128) {
1867	TCGv_vec in0 = tcg_temp_new_vec(TCG_TYPE_V128);
1868	TCGv_vec in1 = tcg_temp_new_vec(TCG_TYPE_V128);
1869
1870	tcg_gen_ld_vec(in0, tcg_env, aofs);
1871	tcg_gen_ld_vec(in1, tcg_env, aofs + 16);
1872	for (i = (aofs == dofs) * 32; i < oprsz; i += 32) {
1873	tcg_gen_st_vec(in0, tcg_env, dofs + i);
1874	tcg_gen_st_vec(in1, tcg_env, dofs + i + 16);
1875	}
1876	} else {
1877	TCGv_i64 in[4];
1878	int j;
1879
1880	for (j = 0; j < 4; ++j) {
1881	in[j] = tcg_temp_ebb_new_i64();
1882	tcg_gen_ld_i64(in[j], tcg_env, aofs + j * 8);
1883	}
1884	for (i = (aofs == dofs) * 32; i < oprsz; i += 32) {
1885	for (j = 0; j < 4; ++j) {
1886	tcg_gen_st_i64(in[j], tcg_env, dofs + i + j * 8);
1887	}
1888	}
1889	for (j = 0; j < 4; ++j) {
1890	tcg_temp_free_i64(in[j]);
1891	}
1892	}
1893	if (oprsz < maxsz) {
1894	expand_clr(tcg_env, dofs + oprsz, maxsz - oprsz);
1895	}
1896	} else {
1897	g_assert_not_reached();
1898	}
1899	}
1900
1901	void tcg_gen_gvec_dup_imm_var(unsigned vece, TCGv_ptr dbase, uint32_t dofs,
1902	uint32_t oprsz, uint32_t maxsz, uint64_t x)
1903	{
1904	check_size_align(oprsz, maxsz, dofs);
1905	do_dup(vece, dbase, dofs, oprsz, maxsz, NULL, NULL, x);
1906	}
1907
1908	void tcg_gen_gvec_dup_imm(unsigned vece, uint32_t dofs, uint32_t oprsz,
1909	uint32_t maxsz, uint64_t x)
1910	{
1911	tcg_gen_gvec_dup_imm_var(vece, tcg_env, dofs, oprsz, maxsz, x);
1912	}
1913
1914	void tcg_gen_gvec_not(unsigned vece, uint32_t dofs, uint32_t aofs,
1915	uint32_t oprsz, uint32_t maxsz)
1916	{
1917	static const GVecGen2 g = {
1918	.fni8 = tcg_gen_not_i64,
1919	.fniv = tcg_gen_not_vec,
1920	.fno = gen_helper_gvec_not,
1921	.prefer_i64 = true,
1922	};
1923	tcg_gen_gvec_2(dofs, aofs, oprsz, maxsz, &g);
1924	}
1925
1926	/* Perform a vector addition using normal addition and a mask. The mask
1927	should be the sign bit of each lane. This 6-operation form is more
1928	efficient than separate additions when there are 4 or more lanes in
1929	the 64-bit operation. */
1930	static void gen_addv_mask(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, TCGv_i64 m)
1931	{
1932	TCGv_i64 t1 = tcg_temp_ebb_new_i64();
1933	TCGv_i64 t2 = tcg_temp_ebb_new_i64();
1934	TCGv_i64 t3 = tcg_temp_ebb_new_i64();
1935
1936	tcg_gen_andc_i64(t1, a, m);
1937	tcg_gen_andc_i64(t2, b, m);
1938	tcg_gen_xor_i64(t3, a, b);
1939	tcg_gen_add_i64(d, t1, t2);
1940	tcg_gen_and_i64(t3, t3, m);
1941	tcg_gen_xor_i64(d, d, t3);
1942
1943	tcg_temp_free_i64(t1);
1944	tcg_temp_free_i64(t2);
1945	tcg_temp_free_i64(t3);
1946	}
1947
1948	void tcg_gen_vec_add8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1949	{
1950	TCGv_i64 m = tcg_constant_i64(dup_const(MO_8, 0x80));
1951	gen_addv_mask(d, a, b, m);
1952	}
1953
1954	void tcg_gen_vec_add8_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1955	{
1956	TCGv_i32 m = tcg_constant_i32((int32_t)dup_const(MO_8, 0x80));
1957	TCGv_i32 t1 = tcg_temp_ebb_new_i32();
1958	TCGv_i32 t2 = tcg_temp_ebb_new_i32();
1959	TCGv_i32 t3 = tcg_temp_ebb_new_i32();
1960
1961	tcg_gen_andc_i32(t1, a, m);
1962	tcg_gen_andc_i32(t2, b, m);
1963	tcg_gen_xor_i32(t3, a, b);
1964	tcg_gen_add_i32(d, t1, t2);
1965	tcg_gen_and_i32(t3, t3, m);
1966	tcg_gen_xor_i32(d, d, t3);
1967
1968	tcg_temp_free_i32(t1);
1969	tcg_temp_free_i32(t2);
1970	tcg_temp_free_i32(t3);
1971	}
1972
1973	void tcg_gen_vec_add16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1974	{
1975	TCGv_i64 m = tcg_constant_i64(dup_const(MO_16, 0x8000));
1976	gen_addv_mask(d, a, b, m);
1977	}
1978
1979	void tcg_gen_vec_add16_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
1980	{
1981	TCGv_i32 t1 = tcg_temp_ebb_new_i32();
1982	TCGv_i32 t2 = tcg_temp_ebb_new_i32();
1983
1984	tcg_gen_andi_i32(t1, a, ~0xffff);
1985	tcg_gen_add_i32(t2, a, b);
1986	tcg_gen_add_i32(t1, t1, b);
1987	tcg_gen_deposit_i32(d, t1, t2, 0, 16);
1988
1989	tcg_temp_free_i32(t1);
1990	tcg_temp_free_i32(t2);
1991	}
1992
1993	void tcg_gen_vec_add32_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1994	{
1995	TCGv_i64 t1 = tcg_temp_ebb_new_i64();
1996	TCGv_i64 t2 = tcg_temp_ebb_new_i64();
1997
1998	tcg_gen_andi_i64(t1, a, ~0xffffffffull);
1999	tcg_gen_add_i64(t2, a, b);
2000	tcg_gen_add_i64(t1, t1, b);
2001	tcg_gen_deposit_i64(d, t1, t2, 0, 32);
2002
2003	tcg_temp_free_i64(t1);
2004	tcg_temp_free_i64(t2);
2005	}
2006
2007	static const TCGOpcode vecop_list_add[] = { INDEX_op_add_vec, 0 };
2008
2009	void tcg_gen_gvec_add_var(unsigned vece, TCGv_ptr dbase, uint32_t dofs,
2010	TCGv_ptr abase, uint32_t aofs,
2011	TCGv_ptr bbase, uint32_t bofs,
2012	uint32_t oprsz, uint32_t maxsz)
2013	{
2014	static const GVecGen3 g[4] = {
2015	{ .fni8 = tcg_gen_vec_add8_i64,
2016	.fniv = tcg_gen_add_vec,
2017	.fno = gen_helper_gvec_add8,
2018	.opt_opc = vecop_list_add,
2019	.vece = MO_8 },
2020	{ .fni8 = tcg_gen_vec_add16_i64,
2021	.fniv = tcg_gen_add_vec,
2022	.fno = gen_helper_gvec_add16,
2023	.opt_opc = vecop_list_add,
2024	.vece = MO_16 },
2025	{ .fni4 = tcg_gen_add_i32,
2026	.fniv = tcg_gen_add_vec,
2027	.fno = gen_helper_gvec_add32,
2028	.opt_opc = vecop_list_add,
2029	.vece = MO_32 },
2030	{ .fni8 = tcg_gen_add_i64,
2031	.fniv = tcg_gen_add_vec,
2032	.fno = gen_helper_gvec_add64,
2033	.opt_opc = vecop_list_add,
2034	.prefer_i64 = true,
2035	.vece = MO_64 },
2036	};
2037
2038	tcg_debug_assert(vece <= MO_64);
2039	tcg_gen_gvec_3_var(dbase, dofs, abase, aofs, bbase, bofs,
2040	oprsz, maxsz, &g[vece]);
2041	}
2042
2043	void tcg_gen_gvec_add(unsigned vece, uint32_t dofs, uint32_t aofs,
2044	uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2045	{
2046	tcg_gen_gvec_add_var(vece, tcg_env, dofs, tcg_env, aofs, tcg_env, bofs,
2047	oprsz, maxsz);
2048	}
2049
2050	void tcg_gen_gvec_adds(unsigned vece, uint32_t dofs, uint32_t aofs,
2051	TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
2052	{
2053	static const GVecGen2s g[4] = {
2054	{ .fni8 = tcg_gen_vec_add8_i64,
2055	.fniv = tcg_gen_add_vec,
2056	.fno = gen_helper_gvec_adds8,
2057	.opt_opc = vecop_list_add,
2058	.vece = MO_8 },
2059	{ .fni8 = tcg_gen_vec_add16_i64,
2060	.fniv = tcg_gen_add_vec,
2061	.fno = gen_helper_gvec_adds16,
2062	.opt_opc = vecop_list_add,
2063	.vece = MO_16 },
2064	{ .fni4 = tcg_gen_add_i32,
2065	.fniv = tcg_gen_add_vec,
2066	.fno = gen_helper_gvec_adds32,
2067	.opt_opc = vecop_list_add,
2068	.vece = MO_32 },
2069	{ .fni8 = tcg_gen_add_i64,
2070	.fniv = tcg_gen_add_vec,
2071	.fno = gen_helper_gvec_adds64,
2072	.opt_opc = vecop_list_add,
2073	.prefer_i64 = true,
2074	.vece = MO_64 },
2075	};
2076
2077	tcg_debug_assert(vece <= MO_64);
2078	tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, c, &g[vece]);
2079	}
2080
2081	void tcg_gen_gvec_addi(unsigned vece, uint32_t dofs, uint32_t aofs,
2082	int64_t c, uint32_t oprsz, uint32_t maxsz)
2083	{
2084	TCGv_i64 tmp = tcg_constant_i64(c);
2085	tcg_gen_gvec_adds(vece, dofs, aofs, tmp, oprsz, maxsz);
2086	}
2087
2088	static const TCGOpcode vecop_list_sub[] = { INDEX_op_sub_vec, 0 };
2089
2090	void tcg_gen_gvec_subs(unsigned vece, uint32_t dofs, uint32_t aofs,
2091	TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
2092	{
2093	static const GVecGen2s g[4] = {
2094	{ .fni8 = tcg_gen_vec_sub8_i64,
2095	.fniv = tcg_gen_sub_vec,
2096	.fno = gen_helper_gvec_subs8,
2097	.opt_opc = vecop_list_sub,
2098	.vece = MO_8 },
2099	{ .fni8 = tcg_gen_vec_sub16_i64,
2100	.fniv = tcg_gen_sub_vec,
2101	.fno = gen_helper_gvec_subs16,
2102	.opt_opc = vecop_list_sub,
2103	.vece = MO_16 },
2104	{ .fni4 = tcg_gen_sub_i32,
2105	.fniv = tcg_gen_sub_vec,
2106	.fno = gen_helper_gvec_subs32,
2107	.opt_opc = vecop_list_sub,
2108	.vece = MO_32 },
2109	{ .fni8 = tcg_gen_sub_i64,
2110	.fniv = tcg_gen_sub_vec,
2111	.fno = gen_helper_gvec_subs64,
2112	.opt_opc = vecop_list_sub,
2113	.prefer_i64 = true,
2114	.vece = MO_64 },
2115	};
2116
2117	tcg_debug_assert(vece <= MO_64);
2118	tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, c, &g[vece]);
2119	}
2120
2121	/* Perform a vector subtraction using normal subtraction and a mask.
2122	Compare gen_addv_mask above. */
2123	static void gen_subv_mask(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, TCGv_i64 m)
2124	{
2125	TCGv_i64 t1 = tcg_temp_ebb_new_i64();
2126	TCGv_i64 t2 = tcg_temp_ebb_new_i64();
2127	TCGv_i64 t3 = tcg_temp_ebb_new_i64();
2128
2129	tcg_gen_or_i64(t1, a, m);
2130	tcg_gen_andc_i64(t2, b, m);
2131	tcg_gen_eqv_i64(t3, a, b);
2132	tcg_gen_sub_i64(d, t1, t2);
2133	tcg_gen_and_i64(t3, t3, m);
2134	tcg_gen_xor_i64(d, d, t3);
2135
2136	tcg_temp_free_i64(t1);
2137	tcg_temp_free_i64(t2);
2138	tcg_temp_free_i64(t3);
2139	}
2140
2141	void tcg_gen_vec_sub8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2142	{
2143	TCGv_i64 m = tcg_constant_i64(dup_const(MO_8, 0x80));
2144	gen_subv_mask(d, a, b, m);
2145	}
2146
2147	void tcg_gen_vec_sub8_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
2148	{
2149	TCGv_i32 m = tcg_constant_i32((int32_t)dup_const(MO_8, 0x80));
2150	TCGv_i32 t1 = tcg_temp_ebb_new_i32();
2151	TCGv_i32 t2 = tcg_temp_ebb_new_i32();
2152	TCGv_i32 t3 = tcg_temp_ebb_new_i32();
2153
2154	tcg_gen_or_i32(t1, a, m);
2155	tcg_gen_andc_i32(t2, b, m);
2156	tcg_gen_eqv_i32(t3, a, b);
2157	tcg_gen_sub_i32(d, t1, t2);
2158	tcg_gen_and_i32(t3, t3, m);
2159	tcg_gen_xor_i32(d, d, t3);
2160
2161	tcg_temp_free_i32(t1);
2162	tcg_temp_free_i32(t2);
2163	tcg_temp_free_i32(t3);
2164	}
2165
2166	void tcg_gen_vec_sub16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2167	{
2168	TCGv_i64 m = tcg_constant_i64(dup_const(MO_16, 0x8000));
2169	gen_subv_mask(d, a, b, m);
2170	}
2171
2172	void tcg_gen_vec_sub16_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
2173	{
2174	TCGv_i32 t1 = tcg_temp_ebb_new_i32();
2175	TCGv_i32 t2 = tcg_temp_ebb_new_i32();
2176
2177	tcg_gen_andi_i32(t1, b, ~0xffff);
2178	tcg_gen_sub_i32(t2, a, b);
2179	tcg_gen_sub_i32(t1, a, t1);
2180	tcg_gen_deposit_i32(d, t1, t2, 0, 16);
2181
2182	tcg_temp_free_i32(t1);
2183	tcg_temp_free_i32(t2);
2184	}
2185
2186	void tcg_gen_vec_sub32_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2187	{
2188	TCGv_i64 t1 = tcg_temp_ebb_new_i64();
2189	TCGv_i64 t2 = tcg_temp_ebb_new_i64();
2190
2191	tcg_gen_andi_i64(t1, b, ~0xffffffffull);
2192	tcg_gen_sub_i64(t2, a, b);
2193	tcg_gen_sub_i64(t1, a, t1);
2194	tcg_gen_deposit_i64(d, t1, t2, 0, 32);
2195
2196	tcg_temp_free_i64(t1);
2197	tcg_temp_free_i64(t2);
2198	}
2199
2200	void tcg_gen_gvec_sub_var(unsigned vece, TCGv_ptr dbase, uint32_t dofs,
2201	TCGv_ptr abase, uint32_t aofs,
2202	TCGv_ptr bbase, uint32_t bofs,
2203	uint32_t oprsz, uint32_t maxsz)
2204	{
2205	static const GVecGen3 g[4] = {
2206	{ .fni8 = tcg_gen_vec_sub8_i64,
2207	.fniv = tcg_gen_sub_vec,
2208	.fno = gen_helper_gvec_sub8,
2209	.opt_opc = vecop_list_sub,
2210	.vece = MO_8 },
2211	{ .fni8 = tcg_gen_vec_sub16_i64,
2212	.fniv = tcg_gen_sub_vec,
2213	.fno = gen_helper_gvec_sub16,
2214	.opt_opc = vecop_list_sub,
2215	.vece = MO_16 },
2216	{ .fni4 = tcg_gen_sub_i32,
2217	.fniv = tcg_gen_sub_vec,
2218	.fno = gen_helper_gvec_sub32,
2219	.opt_opc = vecop_list_sub,
2220	.vece = MO_32 },
2221	{ .fni8 = tcg_gen_sub_i64,
2222	.fniv = tcg_gen_sub_vec,
2223	.fno = gen_helper_gvec_sub64,
2224	.opt_opc = vecop_list_sub,
2225	.prefer_i64 = true,
2226	.vece = MO_64 },
2227	};
2228
2229	tcg_debug_assert(vece <= MO_64);
2230	tcg_gen_gvec_3_var(dbase, dofs, abase, aofs, bbase, bofs,
2231	oprsz, maxsz, &g[vece]);
2232	}
2233
2234	void tcg_gen_gvec_sub(unsigned vece, uint32_t dofs, uint32_t aofs,
2235	uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2236	{
2237	tcg_gen_gvec_sub_var(vece, tcg_env, dofs, tcg_env, aofs, tcg_env, bofs,
2238	oprsz, maxsz);
2239	}
2240
2241	static const TCGOpcode vecop_list_mul[] = { INDEX_op_mul_vec, 0 };
2242
2243	void tcg_gen_gvec_mul(unsigned vece, uint32_t dofs, uint32_t aofs,
2244	uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2245	{
2246	static const GVecGen3 g[4] = {
2247	{ .fniv = tcg_gen_mul_vec,
2248	.fno = gen_helper_gvec_mul8,
2249	.opt_opc = vecop_list_mul,
2250	.vece = MO_8 },
2251	{ .fniv = tcg_gen_mul_vec,
2252	.fno = gen_helper_gvec_mul16,
2253	.opt_opc = vecop_list_mul,
2254	.vece = MO_16 },
2255	{ .fni4 = tcg_gen_mul_i32,
2256	.fniv = tcg_gen_mul_vec,
2257	.fno = gen_helper_gvec_mul32,
2258	.opt_opc = vecop_list_mul,
2259	.vece = MO_32 },
2260	{ .fni8 = tcg_gen_mul_i64,
2261	.fniv = tcg_gen_mul_vec,
2262	.fno = gen_helper_gvec_mul64,
2263	.opt_opc = vecop_list_mul,
2264	.prefer_i64 = true,
2265	.vece = MO_64 },
2266	};
2267
2268	tcg_debug_assert(vece <= MO_64);
2269	tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2270	}
2271
2272	void tcg_gen_gvec_muls(unsigned vece, uint32_t dofs, uint32_t aofs,
2273	TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
2274	{
2275	static const GVecGen2s g[4] = {
2276	{ .fniv = tcg_gen_mul_vec,
2277	.fno = gen_helper_gvec_muls8,
2278	.opt_opc = vecop_list_mul,
2279	.vece = MO_8 },
2280	{ .fniv = tcg_gen_mul_vec,
2281	.fno = gen_helper_gvec_muls16,
2282	.opt_opc = vecop_list_mul,
2283	.vece = MO_16 },
2284	{ .fni4 = tcg_gen_mul_i32,
2285	.fniv = tcg_gen_mul_vec,
2286	.fno = gen_helper_gvec_muls32,
2287	.opt_opc = vecop_list_mul,
2288	.vece = MO_32 },
2289	{ .fni8 = tcg_gen_mul_i64,
2290	.fniv = tcg_gen_mul_vec,
2291	.fno = gen_helper_gvec_muls64,
2292	.opt_opc = vecop_list_mul,
2293	.prefer_i64 = true,
2294	.vece = MO_64 },
2295	};
2296
2297	tcg_debug_assert(vece <= MO_64);
2298	tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, c, &g[vece]);
2299	}
2300
2301	void tcg_gen_gvec_muli(unsigned vece, uint32_t dofs, uint32_t aofs,
2302	int64_t c, uint32_t oprsz, uint32_t maxsz)
2303	{
2304	TCGv_i64 tmp = tcg_constant_i64(c);
2305	tcg_gen_gvec_muls(vece, dofs, aofs, tmp, oprsz, maxsz);
2306	}
2307
2308	void tcg_gen_gvec_ssadd(unsigned vece, uint32_t dofs, uint32_t aofs,
2309	uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2310	{
2311	static const TCGOpcode vecop_list[] = { INDEX_op_ssadd_vec, 0 };
2312	static const GVecGen3 g[4] = {
2313	{ .fniv = tcg_gen_ssadd_vec,
2314	.fno = gen_helper_gvec_ssadd8,
2315	.opt_opc = vecop_list,
2316	.vece = MO_8 },
2317	{ .fniv = tcg_gen_ssadd_vec,
2318	.fno = gen_helper_gvec_ssadd16,
2319	.opt_opc = vecop_list,
2320	.vece = MO_16 },
2321	{ .fniv = tcg_gen_ssadd_vec,
2322	.fno = gen_helper_gvec_ssadd32,
2323	.opt_opc = vecop_list,
2324	.vece = MO_32 },
2325	{ .fniv = tcg_gen_ssadd_vec,
2326	.fno = gen_helper_gvec_ssadd64,
2327	.opt_opc = vecop_list,
2328	.vece = MO_64 },
2329	};
2330	tcg_debug_assert(vece <= MO_64);
2331	tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2332	}
2333
2334	void tcg_gen_gvec_sssub(unsigned vece, uint32_t dofs, uint32_t aofs,
2335	uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2336	{
2337	static const TCGOpcode vecop_list[] = { INDEX_op_sssub_vec, 0 };
2338	static const GVecGen3 g[4] = {
2339	{ .fniv = tcg_gen_sssub_vec,
2340	.fno = gen_helper_gvec_sssub8,
2341	.opt_opc = vecop_list,
2342	.vece = MO_8 },
2343	{ .fniv = tcg_gen_sssub_vec,
2344	.fno = gen_helper_gvec_sssub16,
2345	.opt_opc = vecop_list,
2346	.vece = MO_16 },
2347	{ .fniv = tcg_gen_sssub_vec,
2348	.fno = gen_helper_gvec_sssub32,
2349	.opt_opc = vecop_list,
2350	.vece = MO_32 },
2351	{ .fniv = tcg_gen_sssub_vec,
2352	.fno = gen_helper_gvec_sssub64,
2353	.opt_opc = vecop_list,
2354	.vece = MO_64 },
2355	};
2356	tcg_debug_assert(vece <= MO_64);
2357	tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2358	}
2359
2360	static void tcg_gen_usadd_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
2361	{
2362	TCGv_i32 max = tcg_constant_i32(-1);
2363	tcg_gen_add_i32(d, a, b);
2364	tcg_gen_movcond_i32(TCG_COND_LTU, d, d, a, max, d);
2365	}
2366
2367	static void tcg_gen_usadd_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2368	{
2369	TCGv_i64 max = tcg_constant_i64(-1);
2370	tcg_gen_add_i64(d, a, b);
2371	tcg_gen_movcond_i64(TCG_COND_LTU, d, d, a, max, d);
2372	}
2373
2374	void tcg_gen_gvec_usadd(unsigned vece, uint32_t dofs, uint32_t aofs,
2375	uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2376	{
2377	static const TCGOpcode vecop_list[] = { INDEX_op_usadd_vec, 0 };
2378	static const GVecGen3 g[4] = {
2379	{ .fniv = tcg_gen_usadd_vec,
2380	.fno = gen_helper_gvec_usadd8,
2381	.opt_opc = vecop_list,
2382	.vece = MO_8 },
2383	{ .fniv = tcg_gen_usadd_vec,
2384	.fno = gen_helper_gvec_usadd16,
2385	.opt_opc = vecop_list,
2386	.vece = MO_16 },
2387	{ .fni4 = tcg_gen_usadd_i32,
2388	.fniv = tcg_gen_usadd_vec,
2389	.fno = gen_helper_gvec_usadd32,
2390	.opt_opc = vecop_list,
2391	.vece = MO_32 },
2392	{ .fni8 = tcg_gen_usadd_i64,
2393	.fniv = tcg_gen_usadd_vec,
2394	.fno = gen_helper_gvec_usadd64,
2395	.opt_opc = vecop_list,
2396	.vece = MO_64 }
2397	};
2398	tcg_debug_assert(vece <= MO_64);
2399	tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2400	}
2401
2402	void tcg_gen_gvec_ussub(unsigned vece, uint32_t dofs, uint32_t aofs,
2403	uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2404	{
2405	static const TCGOpcode vecop_list[] = { INDEX_op_ussub_vec, 0 };
2406	static const GVecGen3 g[4] = {
2407	{ .fniv = tcg_gen_ussub_vec,
2408	.fno = gen_helper_gvec_ussub8,
2409	.opt_opc = vecop_list,
2410	.vece = MO_8 },
2411	{ .fniv = tcg_gen_ussub_vec,
2412	.fno = gen_helper_gvec_ussub16,
2413	.opt_opc = vecop_list,
2414	.vece = MO_16 },
2415	{ .fni4 = tcg_gen_ussub_i32,
2416	.fniv = tcg_gen_ussub_vec,
2417	.fno = gen_helper_gvec_ussub32,
2418	.opt_opc = vecop_list,
2419	.vece = MO_32 },
2420	{ .fni8 = tcg_gen_ussub_i64,
2421	.fniv = tcg_gen_ussub_vec,
2422	.fno = gen_helper_gvec_ussub64,
2423	.opt_opc = vecop_list,
2424	.vece = MO_64 }
2425	};
2426	tcg_debug_assert(vece <= MO_64);
2427	tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2428	}
2429
2430	void tcg_gen_gvec_smin(unsigned vece, uint32_t dofs, uint32_t aofs,
2431	uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2432	{
2433	static const TCGOpcode vecop_list[] = { INDEX_op_smin_vec, 0 };
2434	static const GVecGen3 g[4] = {
2435	{ .fniv = tcg_gen_smin_vec,
2436	.fno = gen_helper_gvec_smin8,
2437	.opt_opc = vecop_list,
2438	.vece = MO_8 },
2439	{ .fniv = tcg_gen_smin_vec,
2440	.fno = gen_helper_gvec_smin16,
2441	.opt_opc = vecop_list,
2442	.vece = MO_16 },
2443	{ .fni4 = tcg_gen_smin_i32,
2444	.fniv = tcg_gen_smin_vec,
2445	.fno = gen_helper_gvec_smin32,
2446	.opt_opc = vecop_list,
2447	.vece = MO_32 },
2448	{ .fni8 = tcg_gen_smin_i64,
2449	.fniv = tcg_gen_smin_vec,
2450	.fno = gen_helper_gvec_smin64,
2451	.opt_opc = vecop_list,
2452	.vece = MO_64 }
2453	};
2454	tcg_debug_assert(vece <= MO_64);
2455	tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2456	}
2457
2458	void tcg_gen_gvec_umin(unsigned vece, uint32_t dofs, uint32_t aofs,
2459	uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2460	{
2461	static const TCGOpcode vecop_list[] = { INDEX_op_umin_vec, 0 };
2462	static const GVecGen3 g[4] = {
2463	{ .fniv = tcg_gen_umin_vec,
2464	.fno = gen_helper_gvec_umin8,
2465	.opt_opc = vecop_list,
2466	.vece = MO_8 },
2467	{ .fniv = tcg_gen_umin_vec,
2468	.fno = gen_helper_gvec_umin16,
2469	.opt_opc = vecop_list,
2470	.vece = MO_16 },
2471	{ .fni4 = tcg_gen_umin_i32,
2472	.fniv = tcg_gen_umin_vec,
2473	.fno = gen_helper_gvec_umin32,
2474	.opt_opc = vecop_list,
2475	.vece = MO_32 },
2476	{ .fni8 = tcg_gen_umin_i64,
2477	.fniv = tcg_gen_umin_vec,
2478	.fno = gen_helper_gvec_umin64,
2479	.opt_opc = vecop_list,
2480	.vece = MO_64 }
2481	};
2482	tcg_debug_assert(vece <= MO_64);
2483	tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2484	}
2485
2486	void tcg_gen_gvec_smax(unsigned vece, uint32_t dofs, uint32_t aofs,
2487	uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2488	{
2489	static const TCGOpcode vecop_list[] = { INDEX_op_smax_vec, 0 };
2490	static const GVecGen3 g[4] = {
2491	{ .fniv = tcg_gen_smax_vec,
2492	.fno = gen_helper_gvec_smax8,
2493	.opt_opc = vecop_list,
2494	.vece = MO_8 },
2495	{ .fniv = tcg_gen_smax_vec,
2496	.fno = gen_helper_gvec_smax16,
2497	.opt_opc = vecop_list,
2498	.vece = MO_16 },
2499	{ .fni4 = tcg_gen_smax_i32,
2500	.fniv = tcg_gen_smax_vec,
2501	.fno = gen_helper_gvec_smax32,
2502	.opt_opc = vecop_list,
2503	.vece = MO_32 },
2504	{ .fni8 = tcg_gen_smax_i64,
2505	.fniv = tcg_gen_smax_vec,
2506	.fno = gen_helper_gvec_smax64,
2507	.opt_opc = vecop_list,
2508	.vece = MO_64 }
2509	};
2510	tcg_debug_assert(vece <= MO_64);
2511	tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2512	}
2513
2514	void tcg_gen_gvec_umax(unsigned vece, uint32_t dofs, uint32_t aofs,
2515	uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2516	{
2517	static const TCGOpcode vecop_list[] = { INDEX_op_umax_vec, 0 };
2518	static const GVecGen3 g[4] = {
2519	{ .fniv = tcg_gen_umax_vec,
2520	.fno = gen_helper_gvec_umax8,
2521	.opt_opc = vecop_list,
2522	.vece = MO_8 },
2523	{ .fniv = tcg_gen_umax_vec,
2524	.fno = gen_helper_gvec_umax16,
2525	.opt_opc = vecop_list,
2526	.vece = MO_16 },
2527	{ .fni4 = tcg_gen_umax_i32,
2528	.fniv = tcg_gen_umax_vec,
2529	.fno = gen_helper_gvec_umax32,
2530	.opt_opc = vecop_list,
2531	.vece = MO_32 },
2532	{ .fni8 = tcg_gen_umax_i64,
2533	.fniv = tcg_gen_umax_vec,
2534	.fno = gen_helper_gvec_umax64,
2535	.opt_opc = vecop_list,
2536	.vece = MO_64 }
2537	};
2538	tcg_debug_assert(vece <= MO_64);
2539	tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2540	}
2541
2542	/* Perform a vector negation using normal negation and a mask.
2543	Compare gen_subv_mask above. */
2544	static void gen_negv_mask(TCGv_i64 d, TCGv_i64 b, TCGv_i64 m)
2545	{
2546	TCGv_i64 t2 = tcg_temp_ebb_new_i64();
2547	TCGv_i64 t3 = tcg_temp_ebb_new_i64();
2548
2549	tcg_gen_andc_i64(t3, m, b);
2550	tcg_gen_andc_i64(t2, b, m);
2551	tcg_gen_sub_i64(d, m, t2);
2552	tcg_gen_xor_i64(d, d, t3);
2553
2554	tcg_temp_free_i64(t2);
2555	tcg_temp_free_i64(t3);
2556	}
2557
2558	void tcg_gen_vec_neg8_i64(TCGv_i64 d, TCGv_i64 b)
2559	{
2560	TCGv_i64 m = tcg_constant_i64(dup_const(MO_8, 0x80));
2561	gen_negv_mask(d, b, m);
2562	}
2563
2564	void tcg_gen_vec_neg16_i64(TCGv_i64 d, TCGv_i64 b)
2565	{
2566	TCGv_i64 m = tcg_constant_i64(dup_const(MO_16, 0x8000));
2567	gen_negv_mask(d, b, m);
2568	}
2569
2570	void tcg_gen_vec_neg32_i64(TCGv_i64 d, TCGv_i64 b)
2571	{
2572	TCGv_i64 t1 = tcg_temp_ebb_new_i64();
2573	TCGv_i64 t2 = tcg_temp_ebb_new_i64();
2574
2575	tcg_gen_andi_i64(t1, b, ~0xffffffffull);
2576	tcg_gen_neg_i64(t2, b);
2577	tcg_gen_neg_i64(t1, t1);
2578	tcg_gen_deposit_i64(d, t1, t2, 0, 32);
2579
2580	tcg_temp_free_i64(t1);
2581	tcg_temp_free_i64(t2);
2582	}
2583
2584	void tcg_gen_gvec_neg(unsigned vece, uint32_t dofs, uint32_t aofs,
2585	uint32_t oprsz, uint32_t maxsz)
2586	{
2587	static const TCGOpcode vecop_list[] = { INDEX_op_neg_vec, 0 };
2588	static const GVecGen2 g[4] = {
2589	{ .fni8 = tcg_gen_vec_neg8_i64,
2590	.fniv = tcg_gen_neg_vec,
2591	.fno = gen_helper_gvec_neg8,
2592	.opt_opc = vecop_list,
2593	.vece = MO_8 },
2594	{ .fni8 = tcg_gen_vec_neg16_i64,
2595	.fniv = tcg_gen_neg_vec,
2596	.fno = gen_helper_gvec_neg16,
2597	.opt_opc = vecop_list,
2598	.vece = MO_16 },
2599	{ .fni4 = tcg_gen_neg_i32,
2600	.fniv = tcg_gen_neg_vec,
2601	.fno = gen_helper_gvec_neg32,
2602	.opt_opc = vecop_list,
2603	.vece = MO_32 },
2604	{ .fni8 = tcg_gen_neg_i64,
2605	.fniv = tcg_gen_neg_vec,
2606	.fno = gen_helper_gvec_neg64,
2607	.opt_opc = vecop_list,
2608	.prefer_i64 = true,
2609	.vece = MO_64 },
2610	};
2611
2612	tcg_debug_assert(vece <= MO_64);
2613	tcg_gen_gvec_2(dofs, aofs, oprsz, maxsz, &g[vece]);
2614	}
2615
2616	static void gen_absv_mask(TCGv_i64 d, TCGv_i64 b, unsigned vece)
2617	{
2618	TCGv_i64 t = tcg_temp_ebb_new_i64();
2619	int nbit = 8 << vece;
2620
2621	/* Create -1 for each negative element. */
2622	tcg_gen_shri_i64(t, b, nbit - 1);
2623	tcg_gen_andi_i64(t, t, dup_const(vece, 1));
2624	tcg_gen_muli_i64(t, t, (1 << nbit) - 1);
2625
2626	/*
2627	* Invert (via xor -1) and add one.
2628	* Because of the ordering the msb is cleared,
2629	* so we never have carry into the next element.
2630	*/
2631	tcg_gen_xor_i64(d, b, t);
2632	tcg_gen_andi_i64(t, t, dup_const(vece, 1));
2633	tcg_gen_add_i64(d, d, t);
2634
2635	tcg_temp_free_i64(t);
2636	}
2637
2638	static void tcg_gen_vec_abs8_i64(TCGv_i64 d, TCGv_i64 b)
2639	{
2640	gen_absv_mask(d, b, MO_8);
2641	}
2642
2643	static void tcg_gen_vec_abs16_i64(TCGv_i64 d, TCGv_i64 b)
2644	{
2645	gen_absv_mask(d, b, MO_16);
2646	}
2647
2648	void tcg_gen_gvec_abs(unsigned vece, uint32_t dofs, uint32_t aofs,
2649	uint32_t oprsz, uint32_t maxsz)
2650	{
2651	static const TCGOpcode vecop_list[] = { INDEX_op_abs_vec, 0 };
2652	static const GVecGen2 g[4] = {
2653	{ .fni8 = tcg_gen_vec_abs8_i64,
2654	.fniv = tcg_gen_abs_vec,
2655	.fno = gen_helper_gvec_abs8,
2656	.opt_opc = vecop_list,
2657	.vece = MO_8 },
2658	{ .fni8 = tcg_gen_vec_abs16_i64,
2659	.fniv = tcg_gen_abs_vec,
2660	.fno = gen_helper_gvec_abs16,
2661	.opt_opc = vecop_list,
2662	.vece = MO_16 },
2663	{ .fni4 = tcg_gen_abs_i32,
2664	.fniv = tcg_gen_abs_vec,
2665	.fno = gen_helper_gvec_abs32,
2666	.opt_opc = vecop_list,
2667	.vece = MO_32 },
2668	{ .fni8 = tcg_gen_abs_i64,
2669	.fniv = tcg_gen_abs_vec,
2670	.fno = gen_helper_gvec_abs64,
2671	.opt_opc = vecop_list,
2672	.prefer_i64 = true,
2673	.vece = MO_64 },
2674	};
2675
2676	tcg_debug_assert(vece <= MO_64);
2677	tcg_gen_gvec_2(dofs, aofs, oprsz, maxsz, &g[vece]);
2678	}
2679
2680	void tcg_gen_gvec_and(unsigned vece, uint32_t dofs, uint32_t aofs,
2681	uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2682	{
2683	static const GVecGen3 g = {
2684	.fni8 = tcg_gen_and_i64,
2685	.fniv = tcg_gen_and_vec,
2686	.fno = gen_helper_gvec_and,
2687	.prefer_i64 = true,
2688	};
2689
2690	if (aofs == bofs) {
2691	tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
2692	} else {
2693	tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2694	}
2695	}
2696
2697	void tcg_gen_gvec_or(unsigned vece, uint32_t dofs, uint32_t aofs,
2698	uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2699	{
2700	static const GVecGen3 g = {
2701	.fni8 = tcg_gen_or_i64,
2702	.fniv = tcg_gen_or_vec,
2703	.fno = gen_helper_gvec_or,
2704	.prefer_i64 = true,
2705	};
2706
2707	if (aofs == bofs) {
2708	tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
2709	} else {
2710	tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2711	}
2712	}
2713
2714	void tcg_gen_gvec_xor(unsigned vece, uint32_t dofs, uint32_t aofs,
2715	uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2716	{
2717	static const GVecGen3 g = {
2718	.fni8 = tcg_gen_xor_i64,
2719	.fniv = tcg_gen_xor_vec,
2720	.fno = gen_helper_gvec_xor,
2721	.prefer_i64 = true,
2722	};
2723
2724	if (aofs == bofs) {
2725	tcg_gen_gvec_dup_imm(MO_64, dofs, oprsz, maxsz, 0);
2726	} else {
2727	tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2728	}
2729	}
2730
2731	void tcg_gen_gvec_andc(unsigned vece, uint32_t dofs, uint32_t aofs,
2732	uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2733	{
2734	static const GVecGen3 g = {
2735	.fni8 = tcg_gen_andc_i64,
2736	.fniv = tcg_gen_andc_vec,
2737	.fno = gen_helper_gvec_andc,
2738	.prefer_i64 = true,
2739	};
2740
2741	if (aofs == bofs) {
2742	tcg_gen_gvec_dup_imm(MO_64, dofs, oprsz, maxsz, 0);
2743	} else {
2744	tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2745	}
2746	}
2747
2748	void tcg_gen_gvec_orc(unsigned vece, uint32_t dofs, uint32_t aofs,
2749	uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2750	{
2751	static const GVecGen3 g = {
2752	.fni8 = tcg_gen_orc_i64,
2753	.fniv = tcg_gen_orc_vec,
2754	.fno = gen_helper_gvec_orc,
2755	.prefer_i64 = true,
2756	};
2757
2758	if (aofs == bofs) {
2759	tcg_gen_gvec_dup_imm(MO_64, dofs, oprsz, maxsz, -1);
2760	} else {
2761	tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2762	}
2763	}
2764
2765	void tcg_gen_gvec_nand(unsigned vece, uint32_t dofs, uint32_t aofs,
2766	uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2767	{
2768	static const GVecGen3 g = {
2769	.fni8 = tcg_gen_nand_i64,
2770	.fniv = tcg_gen_nand_vec,
2771	.fno = gen_helper_gvec_nand,
2772	.prefer_i64 = true,
2773	};
2774
2775	if (aofs == bofs) {
2776	tcg_gen_gvec_not(vece, dofs, aofs, oprsz, maxsz);
2777	} else {
2778	tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2779	}
2780	}
2781
2782	void tcg_gen_gvec_nor(unsigned vece, uint32_t dofs, uint32_t aofs,
2783	uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2784	{
2785	static const GVecGen3 g = {
2786	.fni8 = tcg_gen_nor_i64,
2787	.fniv = tcg_gen_nor_vec,
2788	.fno = gen_helper_gvec_nor,
2789	.prefer_i64 = true,
2790	};
2791
2792	if (aofs == bofs) {
2793	tcg_gen_gvec_not(vece, dofs, aofs, oprsz, maxsz);
2794	} else {
2795	tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2796	}
2797	}
2798
2799	void tcg_gen_gvec_eqv(unsigned vece, uint32_t dofs, uint32_t aofs,
2800	uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2801	{
2802	static const GVecGen3 g = {
2803	.fni8 = tcg_gen_eqv_i64,
2804	.fniv = tcg_gen_eqv_vec,
2805	.fno = gen_helper_gvec_eqv,
2806	.prefer_i64 = true,
2807	};
2808
2809	if (aofs == bofs) {
2810	tcg_gen_gvec_dup_imm(MO_64, dofs, oprsz, maxsz, -1);
2811	} else {
2812	tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2813	}
2814	}
2815
2816	static const GVecGen2s gop_ands = {
2817	.fni8 = tcg_gen_and_i64,
2818	.fniv = tcg_gen_and_vec,
2819	.fno = gen_helper_gvec_ands,
2820	.prefer_i64 = true,
2821	.vece = MO_64
2822	};
2823
2824	void tcg_gen_gvec_ands(unsigned vece, uint32_t dofs, uint32_t aofs,
2825	TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
2826	{
2827	TCGv_i64 tmp = tcg_temp_ebb_new_i64();
2828	tcg_gen_dup_i64(vece, tmp, c);
2829	tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_ands);
2830	tcg_temp_free_i64(tmp);
2831	}
2832
2833	void tcg_gen_gvec_andi(unsigned vece, uint32_t dofs, uint32_t aofs,
2834	int64_t c, uint32_t oprsz, uint32_t maxsz)
2835	{
2836	TCGv_i64 tmp = tcg_constant_i64(dup_const(vece, c));
2837	tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_ands);
2838	}
2839
2840	void tcg_gen_gvec_andcs(unsigned vece, uint32_t dofs, uint32_t aofs,
2841	TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
2842	{
2843	static GVecGen2s g = {
2844	.fni8 = tcg_gen_andc_i64,
2845	.fniv = tcg_gen_andc_vec,
2846	.fno = gen_helper_gvec_andcs,
2847	.prefer_i64 = true,
2848	.vece = MO_64
2849	};
2850
2851	TCGv_i64 tmp = tcg_temp_ebb_new_i64();
2852	tcg_gen_dup_i64(vece, tmp, c);
2853	tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &g);
2854	tcg_temp_free_i64(tmp);
2855	}
2856
2857	static const GVecGen2s gop_xors = {
2858	.fni8 = tcg_gen_xor_i64,
2859	.fniv = tcg_gen_xor_vec,
2860	.fno = gen_helper_gvec_xors,
2861	.prefer_i64 = true,
2862	.vece = MO_64
2863	};
2864
2865	void tcg_gen_gvec_xors(unsigned vece, uint32_t dofs, uint32_t aofs,
2866	TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
2867	{
2868	TCGv_i64 tmp = tcg_temp_ebb_new_i64();
2869	tcg_gen_dup_i64(vece, tmp, c);
2870	tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_xors);
2871	tcg_temp_free_i64(tmp);
2872	}
2873
2874	void tcg_gen_gvec_xori(unsigned vece, uint32_t dofs, uint32_t aofs,
2875	int64_t c, uint32_t oprsz, uint32_t maxsz)
2876	{
2877	TCGv_i64 tmp = tcg_constant_i64(dup_const(vece, c));
2878	tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_xors);
2879	}
2880
2881	static const GVecGen2s gop_ors = {
2882	.fni8 = tcg_gen_or_i64,
2883	.fniv = tcg_gen_or_vec,
2884	.fno = gen_helper_gvec_ors,
2885	.prefer_i64 = true,
2886	.vece = MO_64
2887	};
2888
2889	void tcg_gen_gvec_ors(unsigned vece, uint32_t dofs, uint32_t aofs,
2890	TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
2891	{
2892	TCGv_i64 tmp = tcg_temp_ebb_new_i64();
2893	tcg_gen_dup_i64(vece, tmp, c);
2894	tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_ors);
2895	tcg_temp_free_i64(tmp);
2896	}
2897
2898	void tcg_gen_gvec_ori(unsigned vece, uint32_t dofs, uint32_t aofs,
2899	int64_t c, uint32_t oprsz, uint32_t maxsz)
2900	{
2901	TCGv_i64 tmp = tcg_constant_i64(dup_const(vece, c));
2902	tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_ors);
2903	}
2904
2905	void tcg_gen_vec_shl8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2906	{
2907	uint64_t mask = dup_const(MO_8, 0xff << c);
2908	tcg_gen_shli_i64(d, a, c);
2909	tcg_gen_andi_i64(d, d, mask);
2910	}
2911
2912	void tcg_gen_vec_shl16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2913	{
2914	uint64_t mask = dup_const(MO_16, 0xffff << c);
2915	tcg_gen_shli_i64(d, a, c);
2916	tcg_gen_andi_i64(d, d, mask);
2917	}
2918
2919	void tcg_gen_vec_shl8i_i32(TCGv_i32 d, TCGv_i32 a, int32_t c)
2920	{
2921	uint32_t mask = dup_const(MO_8, 0xff << c);
2922	tcg_gen_shli_i32(d, a, c);
2923	tcg_gen_andi_i32(d, d, mask);
2924	}
2925
2926	void tcg_gen_vec_shl16i_i32(TCGv_i32 d, TCGv_i32 a, int32_t c)
2927	{
2928	uint32_t mask = dup_const(MO_16, 0xffff << c);
2929	tcg_gen_shli_i32(d, a, c);
2930	tcg_gen_andi_i32(d, d, mask);
2931	}
2932
2933	void tcg_gen_gvec_shli(unsigned vece, uint32_t dofs, uint32_t aofs,
2934	int64_t shift, uint32_t oprsz, uint32_t maxsz)
2935	{
2936	static const TCGOpcode vecop_list[] = { INDEX_op_shli_vec, 0 };
2937	static const GVecGen2i g[4] = {
2938	{ .fni8 = tcg_gen_vec_shl8i_i64,
2939	.fniv = tcg_gen_shli_vec,
2940	.fno = gen_helper_gvec_shl8i,
2941	.opt_opc = vecop_list,
2942	.vece = MO_8 },
2943	{ .fni8 = tcg_gen_vec_shl16i_i64,
2944	.fniv = tcg_gen_shli_vec,
2945	.fno = gen_helper_gvec_shl16i,
2946	.opt_opc = vecop_list,
2947	.vece = MO_16 },
2948	{ .fni4 = tcg_gen_shli_i32,
2949	.fniv = tcg_gen_shli_vec,
2950	.fno = gen_helper_gvec_shl32i,
2951	.opt_opc = vecop_list,
2952	.vece = MO_32 },
2953	{ .fni8 = tcg_gen_shli_i64,
2954	.fniv = tcg_gen_shli_vec,
2955	.fno = gen_helper_gvec_shl64i,
2956	.opt_opc = vecop_list,
2957	.prefer_i64 = true,
2958	.vece = MO_64 },
2959	};
2960
2961	tcg_debug_assert(vece <= MO_64);
2962	tcg_debug_assert(shift >= 0 && shift < (8 << vece));
2963	if (shift == 0) {
2964	tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
2965	} else {
2966	tcg_gen_gvec_2i(dofs, aofs, oprsz, maxsz, shift, &g[vece]);
2967	}
2968	}
2969
2970	void tcg_gen_vec_shr8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2971	{
2972	uint64_t mask = dup_const(MO_8, 0xff >> c);
2973	tcg_gen_shri_i64(d, a, c);
2974	tcg_gen_andi_i64(d, d, mask);
2975	}
2976
2977	void tcg_gen_vec_shr16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2978	{
2979	uint64_t mask = dup_const(MO_16, 0xffff >> c);
2980	tcg_gen_shri_i64(d, a, c);
2981	tcg_gen_andi_i64(d, d, mask);
2982	}
2983
2984	void tcg_gen_vec_shr8i_i32(TCGv_i32 d, TCGv_i32 a, int32_t c)
2985	{
2986	uint32_t mask = dup_const(MO_8, 0xff >> c);
2987	tcg_gen_shri_i32(d, a, c);
2988	tcg_gen_andi_i32(d, d, mask);
2989	}
2990
2991	void tcg_gen_vec_shr16i_i32(TCGv_i32 d, TCGv_i32 a, int32_t c)
2992	{
2993	uint32_t mask = dup_const(MO_16, 0xffff >> c);
2994	tcg_gen_shri_i32(d, a, c);
2995	tcg_gen_andi_i32(d, d, mask);
2996	}
2997
2998	void tcg_gen_gvec_shri(unsigned vece, uint32_t dofs, uint32_t aofs,
2999	int64_t shift, uint32_t oprsz, uint32_t maxsz)
3000	{
3001	static const TCGOpcode vecop_list[] = { INDEX_op_shri_vec, 0 };
3002	static const GVecGen2i g[4] = {
3003	{ .fni8 = tcg_gen_vec_shr8i_i64,
3004	.fniv = tcg_gen_shri_vec,
3005	.fno = gen_helper_gvec_shr8i,
3006	.opt_opc = vecop_list,
3007	.vece = MO_8 },
3008	{ .fni8 = tcg_gen_vec_shr16i_i64,
3009	.fniv = tcg_gen_shri_vec,
3010	.fno = gen_helper_gvec_shr16i,
3011	.opt_opc = vecop_list,
3012	.vece = MO_16 },
3013	{ .fni4 = tcg_gen_shri_i32,
3014	.fniv = tcg_gen_shri_vec,
3015	.fno = gen_helper_gvec_shr32i,
3016	.opt_opc = vecop_list,
3017	.vece = MO_32 },
3018	{ .fni8 = tcg_gen_shri_i64,
3019	.fniv = tcg_gen_shri_vec,
3020	.fno = gen_helper_gvec_shr64i,
3021	.opt_opc = vecop_list,
3022	.prefer_i64 = true,
3023	.vece = MO_64 },
3024	};
3025
3026	tcg_debug_assert(vece <= MO_64);
3027	tcg_debug_assert(shift >= 0 && shift < (8 << vece));
3028	if (shift == 0) {
3029	tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
3030	} else {
3031	tcg_gen_gvec_2i(dofs, aofs, oprsz, maxsz, shift, &g[vece]);
3032	}
3033	}
3034
3035	void tcg_gen_vec_sar8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
3036	{
3037	uint64_t s_mask = dup_const(MO_8, 0x80 >> c);
3038	uint64_t c_mask = dup_const(MO_8, 0xff >> c);
3039	TCGv_i64 s = tcg_temp_ebb_new_i64();
3040
3041	tcg_gen_shri_i64(d, a, c);
3042	tcg_gen_andi_i64(s, d, s_mask); /* isolate (shifted) sign bit */
3043	tcg_gen_muli_i64(s, s, (2 << c) - 2); /* replicate isolated signs */
3044	tcg_gen_andi_i64(d, d, c_mask); /* clear out bits above sign */
3045	tcg_gen_or_i64(d, d, s); /* include sign extension */
3046	tcg_temp_free_i64(s);
3047	}
3048
3049	void tcg_gen_vec_sar16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
3050	{
3051	uint64_t s_mask = dup_const(MO_16, 0x8000 >> c);
3052	uint64_t c_mask = dup_const(MO_16, 0xffff >> c);
3053	TCGv_i64 s = tcg_temp_ebb_new_i64();
3054
3055	tcg_gen_shri_i64(d, a, c);
3056	tcg_gen_andi_i64(s, d, s_mask); /* isolate (shifted) sign bit */
3057	tcg_gen_andi_i64(d, d, c_mask); /* clear out bits above sign */
3058	tcg_gen_muli_i64(s, s, (2 << c) - 2); /* replicate isolated signs */
3059	tcg_gen_or_i64(d, d, s); /* include sign extension */
3060	tcg_temp_free_i64(s);
3061	}
3062
3063	void tcg_gen_vec_sar8i_i32(TCGv_i32 d, TCGv_i32 a, int32_t c)
3064	{
3065	uint32_t s_mask = dup_const(MO_8, 0x80 >> c);
3066	uint32_t c_mask = dup_const(MO_8, 0xff >> c);
3067	TCGv_i32 s = tcg_temp_ebb_new_i32();
3068
3069	tcg_gen_shri_i32(d, a, c);
3070	tcg_gen_andi_i32(s, d, s_mask); /* isolate (shifted) sign bit */
3071	tcg_gen_muli_i32(s, s, (2 << c) - 2); /* replicate isolated signs */
3072	tcg_gen_andi_i32(d, d, c_mask); /* clear out bits above sign */
3073	tcg_gen_or_i32(d, d, s); /* include sign extension */
3074	tcg_temp_free_i32(s);
3075	}
3076
3077	void tcg_gen_vec_sar16i_i32(TCGv_i32 d, TCGv_i32 a, int32_t c)
3078	{
3079	uint32_t s_mask = dup_const(MO_16, 0x8000 >> c);
3080	uint32_t c_mask = dup_const(MO_16, 0xffff >> c);
3081	TCGv_i32 s = tcg_temp_ebb_new_i32();
3082
3083	tcg_gen_shri_i32(d, a, c);
3084	tcg_gen_andi_i32(s, d, s_mask); /* isolate (shifted) sign bit */
3085	tcg_gen_andi_i32(d, d, c_mask); /* clear out bits above sign */
3086	tcg_gen_muli_i32(s, s, (2 << c) - 2); /* replicate isolated signs */
3087	tcg_gen_or_i32(d, d, s); /* include sign extension */
3088	tcg_temp_free_i32(s);
3089	}
3090
3091	void tcg_gen_gvec_sari(unsigned vece, uint32_t dofs, uint32_t aofs,
3092	int64_t shift, uint32_t oprsz, uint32_t maxsz)
3093	{
3094	static const TCGOpcode vecop_list[] = { INDEX_op_sari_vec, 0 };
3095	static const GVecGen2i g[4] = {
3096	{ .fni8 = tcg_gen_vec_sar8i_i64,
3097	.fniv = tcg_gen_sari_vec,
3098	.fno = gen_helper_gvec_sar8i,
3099	.opt_opc = vecop_list,
3100	.vece = MO_8 },
3101	{ .fni8 = tcg_gen_vec_sar16i_i64,
3102	.fniv = tcg_gen_sari_vec,
3103	.fno = gen_helper_gvec_sar16i,
3104	.opt_opc = vecop_list,
3105	.vece = MO_16 },
3106	{ .fni4 = tcg_gen_sari_i32,
3107	.fniv = tcg_gen_sari_vec,
3108	.fno = gen_helper_gvec_sar32i,
3109	.opt_opc = vecop_list,
3110	.vece = MO_32 },
3111	{ .fni8 = tcg_gen_sari_i64,
3112	.fniv = tcg_gen_sari_vec,
3113	.fno = gen_helper_gvec_sar64i,
3114	.opt_opc = vecop_list,
3115	.prefer_i64 = true,
3116	.vece = MO_64 },
3117	};
3118
3119	tcg_debug_assert(vece <= MO_64);
3120	tcg_debug_assert(shift >= 0 && shift < (8 << vece));
3121	if (shift == 0) {
3122	tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
3123	} else {
3124	tcg_gen_gvec_2i(dofs, aofs, oprsz, maxsz, shift, &g[vece]);
3125	}
3126	}
3127
3128	void tcg_gen_vec_rotl8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
3129	{
3130	uint64_t mask = dup_const(MO_8, 0xff << c);
3131
3132	tcg_gen_shli_i64(d, a, c);
3133	tcg_gen_shri_i64(a, a, 8 - c);
3134	tcg_gen_andi_i64(d, d, mask);
3135	tcg_gen_andi_i64(a, a, ~mask);
3136	tcg_gen_or_i64(d, d, a);
3137	}
3138
3139	void tcg_gen_vec_rotl16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
3140	{
3141	uint64_t mask = dup_const(MO_16, 0xffff << c);
3142
3143	tcg_gen_shli_i64(d, a, c);
3144	tcg_gen_shri_i64(a, a, 16 - c);
3145	tcg_gen_andi_i64(d, d, mask);
3146	tcg_gen_andi_i64(a, a, ~mask);
3147	tcg_gen_or_i64(d, d, a);
3148	}
3149
3150	void tcg_gen_gvec_rotli(unsigned vece, uint32_t dofs, uint32_t aofs,
3151	int64_t shift, uint32_t oprsz, uint32_t maxsz)
3152	{
3153	static const TCGOpcode vecop_list[] = { INDEX_op_rotli_vec, 0 };
3154	static const GVecGen2i g[4] = {
3155	{ .fni8 = tcg_gen_vec_rotl8i_i64,
3156	.fniv = tcg_gen_rotli_vec,
3157	.fno = gen_helper_gvec_rotl8i,
3158	.opt_opc = vecop_list,
3159	.vece = MO_8 },
3160	{ .fni8 = tcg_gen_vec_rotl16i_i64,
3161	.fniv = tcg_gen_rotli_vec,
3162	.fno = gen_helper_gvec_rotl16i,
3163	.opt_opc = vecop_list,
3164	.vece = MO_16 },
3165	{ .fni4 = tcg_gen_rotli_i32,
3166	.fniv = tcg_gen_rotli_vec,
3167	.fno = gen_helper_gvec_rotl32i,
3168	.opt_opc = vecop_list,
3169	.vece = MO_32 },
3170	{ .fni8 = tcg_gen_rotli_i64,
3171	.fniv = tcg_gen_rotli_vec,
3172	.fno = gen_helper_gvec_rotl64i,
3173	.opt_opc = vecop_list,
3174	.prefer_i64 = true,
3175	.vece = MO_64 },
3176	};
3177
3178	tcg_debug_assert(vece <= MO_64);
3179	tcg_debug_assert(shift >= 0 && shift < (8 << vece));
3180	if (shift == 0) {
3181	tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
3182	} else {
3183	tcg_gen_gvec_2i(dofs, aofs, oprsz, maxsz, shift, &g[vece]);
3184	}
3185	}
3186
3187	void tcg_gen_gvec_rotri(unsigned vece, uint32_t dofs, uint32_t aofs,
3188	int64_t shift, uint32_t oprsz, uint32_t maxsz)
3189	{
3190	tcg_debug_assert(vece <= MO_64);
3191	tcg_debug_assert(shift >= 0 && shift < (8 << vece));
3192	tcg_gen_gvec_rotli(vece, dofs, aofs, -shift & ((8 << vece) - 1),
3193	oprsz, maxsz);
3194	}
3195
3196	/*
3197	* Specialized generation vector shifts by a non-constant scalar.
3198	*/
3199
3200	typedef struct {
3201	void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32);
3202	void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64);
3203	void (*fniv_s)(unsigned, TCGv_vec, TCGv_vec, TCGv_i32);
3204	void (*fniv_v)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec);
3205	gen_helper_gvec_2 *fno[4];
3206	TCGOpcode s_list[2];
3207	TCGOpcode v_list[2];
3208	} GVecGen2sh;
3209
3210	static void expand_2sh_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
3211	uint32_t oprsz, uint32_t tysz, TCGType type,
3212	TCGv_i32 shift,
3213	void (*fni)(unsigned, TCGv_vec, TCGv_vec, TCGv_i32))
3214	{
3215	for (uint32_t i = 0; i < oprsz; i += tysz) {
3216	TCGv_vec t0 = tcg_temp_new_vec(type);
3217	TCGv_vec t1 = tcg_temp_new_vec(type);
3218
3219	tcg_gen_ld_vec(t0, tcg_env, aofs + i);
3220	fni(vece, t1, t0, shift);
3221	tcg_gen_st_vec(t1, tcg_env, dofs + i);
3222	}
3223	}
3224
3225	static void
3226	do_gvec_shifts(unsigned vece, uint32_t dofs, uint32_t aofs, TCGv_i32 shift,
3227	uint32_t oprsz, uint32_t maxsz, const GVecGen2sh *g)
3228	{
3229	TCGType type;
3230	uint32_t some;
3231
3232	check_size_align(oprsz, maxsz, dofs \| aofs);
3233	check_overlap_2(tcg_env, dofs, tcg_env, aofs, maxsz);
3234
3235	/* If the backend has a scalar expansion, great. */
3236	type = choose_vector_type(g->s_list, vece, oprsz, vece == MO_64);
3237	if (type) {
3238	const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
3239	switch (type) {
3240	case TCG_TYPE_V256:
3241	some = QEMU_ALIGN_DOWN(oprsz, 32);
3242	expand_2sh_vec(vece, dofs, aofs, some, 32,
3243	TCG_TYPE_V256, shift, g->fniv_s);
3244	if (some == oprsz) {
3245	break;
3246	}
3247	dofs += some;
3248	aofs += some;
3249	oprsz -= some;
3250	maxsz -= some;
3251	/* fallthru */
3252	case TCG_TYPE_V128:
3253	expand_2sh_vec(vece, dofs, aofs, oprsz, 16,
3254	TCG_TYPE_V128, shift, g->fniv_s);
3255	break;
3256	case TCG_TYPE_V64:
3257	expand_2sh_vec(vece, dofs, aofs, oprsz, 8,
3258	TCG_TYPE_V64, shift, g->fniv_s);
3259	break;
3260	default:
3261	g_assert_not_reached();
3262	}
3263	tcg_swap_vecop_list(hold_list);
3264	goto clear_tail;
3265	}
3266
3267	/* If the backend supports variable vector shifts, also cool. */
3268	type = choose_vector_type(g->v_list, vece, oprsz, vece == MO_64);
3269	if (type) {
3270	const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
3271	TCGv_vec v_shift = tcg_temp_new_vec(type);
3272
3273	if (vece == MO_64) {
3274	TCGv_i64 sh64 = tcg_temp_ebb_new_i64();
3275	tcg_gen_extu_i32_i64(sh64, shift);
3276	tcg_gen_dup_i64_vec(MO_64, v_shift, sh64);
3277	tcg_temp_free_i64(sh64);
3278	} else {
3279	tcg_gen_dup_i32_vec(vece, v_shift, shift);
3280	}
3281
3282	switch (type) {
3283	case TCG_TYPE_V256:
3284	some = QEMU_ALIGN_DOWN(oprsz, 32);
3285	expand_2s_vec(vece, dofs, aofs, some, 32, TCG_TYPE_V256,
3286	v_shift, false, g->fniv_v);
3287	if (some == oprsz) {
3288	break;
3289	}
3290	dofs += some;
3291	aofs += some;
3292	oprsz -= some;
3293	maxsz -= some;
3294	/* fallthru */
3295	case TCG_TYPE_V128:
3296	expand_2s_vec(vece, dofs, aofs, oprsz, 16, TCG_TYPE_V128,
3297	v_shift, false, g->fniv_v);
3298	break;
3299	case TCG_TYPE_V64:
3300	expand_2s_vec(vece, dofs, aofs, oprsz, 8, TCG_TYPE_V64,
3301	v_shift, false, g->fniv_v);
3302	break;
3303	default:
3304	g_assert_not_reached();
3305	}
3306	tcg_temp_free_vec(v_shift);
3307	tcg_swap_vecop_list(hold_list);
3308	goto clear_tail;
3309	}
3310
3311	/* Otherwise fall back to integral... */
3312	if (vece == MO_32 && check_size_impl(oprsz, 4)) {
3313	expand_2s_i32(dofs, aofs, oprsz, shift, false, g->fni4);
3314	} else if (vece == MO_64 && check_size_impl(oprsz, 8)) {
3315	TCGv_i64 sh64 = tcg_temp_ebb_new_i64();
3316	tcg_gen_extu_i32_i64(sh64, shift);
3317	expand_2s_i64(dofs, aofs, oprsz, sh64, false, g->fni8);
3318	tcg_temp_free_i64(sh64);
3319	} else {
3320	TCGv_ptr a0 = tcg_temp_ebb_new_ptr();
3321	TCGv_ptr a1 = tcg_temp_ebb_new_ptr();
3322	TCGv_i32 desc = tcg_temp_ebb_new_i32();
3323
3324	tcg_gen_shli_i32(desc, shift, SIMD_DATA_SHIFT);
3325	tcg_gen_ori_i32(desc, desc, simd_desc(oprsz, maxsz, 0));
3326	tcg_gen_addi_ptr(a0, tcg_env, dofs);
3327	tcg_gen_addi_ptr(a1, tcg_env, aofs);
3328
3329	g->fno[vece](a0, a1, desc);
3330
3331	tcg_temp_free_ptr(a0);
3332	tcg_temp_free_ptr(a1);
3333	tcg_temp_free_i32(desc);
3334	return;
3335	}
3336
3337	clear_tail:
3338	if (oprsz < maxsz) {
3339	expand_clr(tcg_env, dofs + oprsz, maxsz - oprsz);
3340	}
3341	}
3342
3343	void tcg_gen_gvec_shls(unsigned vece, uint32_t dofs, uint32_t aofs,
3344	TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz)
3345	{
3346	static const GVecGen2sh g = {
3347	.fni4 = tcg_gen_shl_i32,
3348	.fni8 = tcg_gen_shl_i64,
3349	.fniv_s = tcg_gen_shls_vec,
3350	.fniv_v = tcg_gen_shlv_vec,
3351	.fno = {
3352	gen_helper_gvec_shl8i,
3353	gen_helper_gvec_shl16i,
3354	gen_helper_gvec_shl32i,
3355	gen_helper_gvec_shl64i,
3356	},
3357	.s_list = { INDEX_op_shls_vec, 0 },
3358	.v_list = { INDEX_op_shlv_vec, 0 },
3359	};
3360
3361	tcg_debug_assert(vece <= MO_64);
3362	do_gvec_shifts(vece, dofs, aofs, shift, oprsz, maxsz, &g);
3363	}
3364
3365	void tcg_gen_gvec_shrs(unsigned vece, uint32_t dofs, uint32_t aofs,
3366	TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz)
3367	{
3368	static const GVecGen2sh g = {
3369	.fni4 = tcg_gen_shr_i32,
3370	.fni8 = tcg_gen_shr_i64,
3371	.fniv_s = tcg_gen_shrs_vec,
3372	.fniv_v = tcg_gen_shrv_vec,
3373	.fno = {
3374	gen_helper_gvec_shr8i,
3375	gen_helper_gvec_shr16i,
3376	gen_helper_gvec_shr32i,
3377	gen_helper_gvec_shr64i,
3378	},
3379	.s_list = { INDEX_op_shrs_vec, 0 },
3380	.v_list = { INDEX_op_shrv_vec, 0 },
3381	};
3382
3383	tcg_debug_assert(vece <= MO_64);
3384	do_gvec_shifts(vece, dofs, aofs, shift, oprsz, maxsz, &g);
3385	}
3386
3387	void tcg_gen_gvec_sars(unsigned vece, uint32_t dofs, uint32_t aofs,
3388	TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz)
3389	{
3390	static const GVecGen2sh g = {
3391	.fni4 = tcg_gen_sar_i32,
3392	.fni8 = tcg_gen_sar_i64,
3393	.fniv_s = tcg_gen_sars_vec,
3394	.fniv_v = tcg_gen_sarv_vec,
3395	.fno = {
3396	gen_helper_gvec_sar8i,
3397	gen_helper_gvec_sar16i,
3398	gen_helper_gvec_sar32i,
3399	gen_helper_gvec_sar64i,
3400	},
3401	.s_list = { INDEX_op_sars_vec, 0 },
3402	.v_list = { INDEX_op_sarv_vec, 0 },
3403	};
3404
3405	tcg_debug_assert(vece <= MO_64);
3406	do_gvec_shifts(vece, dofs, aofs, shift, oprsz, maxsz, &g);
3407	}
3408
3409	void tcg_gen_gvec_rotls(unsigned vece, uint32_t dofs, uint32_t aofs,
3410	TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz)
3411	{
3412	static const GVecGen2sh g = {
3413	.fni4 = tcg_gen_rotl_i32,
3414	.fni8 = tcg_gen_rotl_i64,
3415	.fniv_s = tcg_gen_rotls_vec,
3416	.fniv_v = tcg_gen_rotlv_vec,
3417	.fno = {
3418	gen_helper_gvec_rotl8i,
3419	gen_helper_gvec_rotl16i,
3420	gen_helper_gvec_rotl32i,
3421	gen_helper_gvec_rotl64i,
3422	},
3423	.s_list = { INDEX_op_rotls_vec, 0 },
3424	.v_list = { INDEX_op_rotlv_vec, 0 },
3425	};
3426
3427	tcg_debug_assert(vece <= MO_64);
3428	do_gvec_shifts(vece, dofs, aofs, shift, oprsz, maxsz, &g);
3429	}
3430
3431	void tcg_gen_gvec_rotrs(unsigned vece, uint32_t dofs, uint32_t aofs,
3432	TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz)
3433	{
3434	TCGv_i32 tmp = tcg_temp_ebb_new_i32();
3435
3436	tcg_gen_neg_i32(tmp, shift);
3437	tcg_gen_andi_i32(tmp, tmp, (8 << vece) - 1);
3438	tcg_gen_gvec_rotls(vece, dofs, aofs, tmp, oprsz, maxsz);
3439	tcg_temp_free_i32(tmp);
3440	}
3441
3442	/*
3443	* Expand D = A << (B % element bits)
3444	*
3445	* Unlike scalar shifts, where it is easy for the target front end
3446	* to include the modulo as part of the expansion. If the target
3447	* naturally includes the modulo as part of the operation, great!
3448	* If the target has some other behaviour from out-of-range shifts,
3449	* then it could not use this function anyway, and would need to
3450	* do it's own expansion with custom functions.
3451	*/
3452	static void tcg_gen_shlv_mod_vec(unsigned vece, TCGv_vec d,
3453	TCGv_vec a, TCGv_vec b)
3454	{
3455	TCGv_vec t = tcg_temp_new_vec_matching(d);
3456	TCGv_vec m = tcg_constant_vec_matching(d, vece, (8 << vece) - 1);
3457
3458	tcg_gen_and_vec(vece, t, b, m);
3459	tcg_gen_shlv_vec(vece, d, a, t);
3460	tcg_temp_free_vec(t);
3461	}
3462
3463	static void tcg_gen_shl_mod_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
3464	{
3465	TCGv_i32 t = tcg_temp_ebb_new_i32();
3466
3467	tcg_gen_andi_i32(t, b, 31);
3468	tcg_gen_shl_i32(d, a, t);
3469	tcg_temp_free_i32(t);
3470	}
3471
3472	static void tcg_gen_shl_mod_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
3473	{
3474	TCGv_i64 t = tcg_temp_ebb_new_i64();
3475
3476	tcg_gen_andi_i64(t, b, 63);
3477	tcg_gen_shl_i64(d, a, t);
3478	tcg_temp_free_i64(t);
3479	}
3480
3481	void tcg_gen_gvec_shlv(unsigned vece, uint32_t dofs, uint32_t aofs,
3482	uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
3483	{
3484	static const TCGOpcode vecop_list[] = { INDEX_op_shlv_vec, 0 };
3485	static const GVecGen3 g[4] = {
3486	{ .fniv = tcg_gen_shlv_mod_vec,
3487	.fno = gen_helper_gvec_shl8v,
3488	.opt_opc = vecop_list,
3489	.vece = MO_8 },
3490	{ .fniv = tcg_gen_shlv_mod_vec,
3491	.fno = gen_helper_gvec_shl16v,
3492	.opt_opc = vecop_list,
3493	.vece = MO_16 },
3494	{ .fni4 = tcg_gen_shl_mod_i32,
3495	.fniv = tcg_gen_shlv_mod_vec,
3496	.fno = gen_helper_gvec_shl32v,
3497	.opt_opc = vecop_list,
3498	.vece = MO_32 },
3499	{ .fni8 = tcg_gen_shl_mod_i64,
3500	.fniv = tcg_gen_shlv_mod_vec,
3501	.fno = gen_helper_gvec_shl64v,
3502	.opt_opc = vecop_list,
3503	.prefer_i64 = true,
3504	.vece = MO_64 },
3505	};
3506
3507	tcg_debug_assert(vece <= MO_64);
3508	tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
3509	}
3510
3511	/*
3512	* Similarly for logical right shifts.
3513	*/
3514
3515	static void tcg_gen_shrv_mod_vec(unsigned vece, TCGv_vec d,
3516	TCGv_vec a, TCGv_vec b)
3517	{
3518	TCGv_vec t = tcg_temp_new_vec_matching(d);
3519	TCGv_vec m = tcg_constant_vec_matching(d, vece, (8 << vece) - 1);
3520
3521	tcg_gen_and_vec(vece, t, b, m);
3522	tcg_gen_shrv_vec(vece, d, a, t);
3523	tcg_temp_free_vec(t);
3524	}
3525
3526	static void tcg_gen_shr_mod_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
3527	{
3528	TCGv_i32 t = tcg_temp_ebb_new_i32();
3529
3530	tcg_gen_andi_i32(t, b, 31);
3531	tcg_gen_shr_i32(d, a, t);
3532	tcg_temp_free_i32(t);
3533	}
3534
3535	static void tcg_gen_shr_mod_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
3536	{
3537	TCGv_i64 t = tcg_temp_ebb_new_i64();
3538
3539	tcg_gen_andi_i64(t, b, 63);
3540	tcg_gen_shr_i64(d, a, t);
3541	tcg_temp_free_i64(t);
3542	}
3543
3544	void tcg_gen_gvec_shrv(unsigned vece, uint32_t dofs, uint32_t aofs,
3545	uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
3546	{
3547	static const TCGOpcode vecop_list[] = { INDEX_op_shrv_vec, 0 };
3548	static const GVecGen3 g[4] = {
3549	{ .fniv = tcg_gen_shrv_mod_vec,
3550	.fno = gen_helper_gvec_shr8v,
3551	.opt_opc = vecop_list,
3552	.vece = MO_8 },
3553	{ .fniv = tcg_gen_shrv_mod_vec,
3554	.fno = gen_helper_gvec_shr16v,
3555	.opt_opc = vecop_list,
3556	.vece = MO_16 },
3557	{ .fni4 = tcg_gen_shr_mod_i32,
3558	.fniv = tcg_gen_shrv_mod_vec,
3559	.fno = gen_helper_gvec_shr32v,
3560	.opt_opc = vecop_list,
3561	.vece = MO_32 },
3562	{ .fni8 = tcg_gen_shr_mod_i64,
3563	.fniv = tcg_gen_shrv_mod_vec,
3564	.fno = gen_helper_gvec_shr64v,
3565	.opt_opc = vecop_list,
3566	.prefer_i64 = true,
3567	.vece = MO_64 },
3568	};
3569
3570	tcg_debug_assert(vece <= MO_64);
3571	tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
3572	}
3573
3574	/*
3575	* Similarly for arithmetic right shifts.
3576	*/
3577
3578	static void tcg_gen_sarv_mod_vec(unsigned vece, TCGv_vec d,
3579	TCGv_vec a, TCGv_vec b)
3580	{
3581	TCGv_vec t = tcg_temp_new_vec_matching(d);
3582	TCGv_vec m = tcg_constant_vec_matching(d, vece, (8 << vece) - 1);
3583
3584	tcg_gen_and_vec(vece, t, b, m);
3585	tcg_gen_sarv_vec(vece, d, a, t);
3586	tcg_temp_free_vec(t);
3587	}
3588
3589	static void tcg_gen_sar_mod_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
3590	{
3591	TCGv_i32 t = tcg_temp_ebb_new_i32();
3592
3593	tcg_gen_andi_i32(t, b, 31);
3594	tcg_gen_sar_i32(d, a, t);
3595	tcg_temp_free_i32(t);
3596	}
3597
3598	static void tcg_gen_sar_mod_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
3599	{
3600	TCGv_i64 t = tcg_temp_ebb_new_i64();
3601
3602	tcg_gen_andi_i64(t, b, 63);
3603	tcg_gen_sar_i64(d, a, t);
3604	tcg_temp_free_i64(t);
3605	}
3606
3607	void tcg_gen_gvec_sarv(unsigned vece, uint32_t dofs, uint32_t aofs,
3608	uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
3609	{
3610	static const TCGOpcode vecop_list[] = { INDEX_op_sarv_vec, 0 };
3611	static const GVecGen3 g[4] = {
3612	{ .fniv = tcg_gen_sarv_mod_vec,
3613	.fno = gen_helper_gvec_sar8v,
3614	.opt_opc = vecop_list,
3615	.vece = MO_8 },
3616	{ .fniv = tcg_gen_sarv_mod_vec,
3617	.fno = gen_helper_gvec_sar16v,
3618	.opt_opc = vecop_list,
3619	.vece = MO_16 },
3620	{ .fni4 = tcg_gen_sar_mod_i32,
3621	.fniv = tcg_gen_sarv_mod_vec,
3622	.fno = gen_helper_gvec_sar32v,
3623	.opt_opc = vecop_list,
3624	.vece = MO_32 },
3625	{ .fni8 = tcg_gen_sar_mod_i64,
3626	.fniv = tcg_gen_sarv_mod_vec,
3627	.fno = gen_helper_gvec_sar64v,
3628	.opt_opc = vecop_list,
3629	.prefer_i64 = true,
3630	.vece = MO_64 },
3631	};
3632
3633	tcg_debug_assert(vece <= MO_64);
3634	tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
3635	}
3636
3637	/*
3638	* Similarly for rotates.
3639	*/
3640
3641	static void tcg_gen_rotlv_mod_vec(unsigned vece, TCGv_vec d,
3642	TCGv_vec a, TCGv_vec b)
3643	{
3644	TCGv_vec t = tcg_temp_new_vec_matching(d);
3645	TCGv_vec m = tcg_constant_vec_matching(d, vece, (8 << vece) - 1);
3646
3647	tcg_gen_and_vec(vece, t, b, m);
3648	tcg_gen_rotlv_vec(vece, d, a, t);
3649	tcg_temp_free_vec(t);
3650	}
3651
3652	static void tcg_gen_rotl_mod_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
3653	{
3654	TCGv_i32 t = tcg_temp_ebb_new_i32();
3655
3656	tcg_gen_andi_i32(t, b, 31);
3657	tcg_gen_rotl_i32(d, a, t);
3658	tcg_temp_free_i32(t);
3659	}
3660
3661	static void tcg_gen_rotl_mod_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
3662	{
3663	TCGv_i64 t = tcg_temp_ebb_new_i64();
3664
3665	tcg_gen_andi_i64(t, b, 63);
3666	tcg_gen_rotl_i64(d, a, t);
3667	tcg_temp_free_i64(t);
3668	}
3669
3670	void tcg_gen_gvec_rotlv(unsigned vece, uint32_t dofs, uint32_t aofs,
3671	uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
3672	{
3673	static const TCGOpcode vecop_list[] = { INDEX_op_rotlv_vec, 0 };
3674	static const GVecGen3 g[4] = {
3675	{ .fniv = tcg_gen_rotlv_mod_vec,
3676	.fno = gen_helper_gvec_rotl8v,
3677	.opt_opc = vecop_list,
3678	.vece = MO_8 },
3679	{ .fniv = tcg_gen_rotlv_mod_vec,
3680	.fno = gen_helper_gvec_rotl16v,
3681	.opt_opc = vecop_list,
3682	.vece = MO_16 },
3683	{ .fni4 = tcg_gen_rotl_mod_i32,
3684	.fniv = tcg_gen_rotlv_mod_vec,
3685	.fno = gen_helper_gvec_rotl32v,
3686	.opt_opc = vecop_list,
3687	.vece = MO_32 },
3688	{ .fni8 = tcg_gen_rotl_mod_i64,
3689	.fniv = tcg_gen_rotlv_mod_vec,
3690	.fno = gen_helper_gvec_rotl64v,
3691	.opt_opc = vecop_list,
3692	.prefer_i64 = true,
3693	.vece = MO_64 },
3694	};
3695
3696	tcg_debug_assert(vece <= MO_64);
3697	tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
3698	}
3699
3700	static void tcg_gen_rotrv_mod_vec(unsigned vece, TCGv_vec d,
3701	TCGv_vec a, TCGv_vec b)
3702	{
3703	TCGv_vec t = tcg_temp_new_vec_matching(d);
3704	TCGv_vec m = tcg_constant_vec_matching(d, vece, (8 << vece) - 1);
3705
3706	tcg_gen_and_vec(vece, t, b, m);
3707	tcg_gen_rotrv_vec(vece, d, a, t);
3708	tcg_temp_free_vec(t);
3709	}
3710
3711	static void tcg_gen_rotr_mod_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
3712	{
3713	TCGv_i32 t = tcg_temp_ebb_new_i32();
3714
3715	tcg_gen_andi_i32(t, b, 31);
3716	tcg_gen_rotr_i32(d, a, t);
3717	tcg_temp_free_i32(t);
3718	}
3719
3720	static void tcg_gen_rotr_mod_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
3721	{
3722	TCGv_i64 t = tcg_temp_ebb_new_i64();
3723
3724	tcg_gen_andi_i64(t, b, 63);
3725	tcg_gen_rotr_i64(d, a, t);
3726	tcg_temp_free_i64(t);
3727	}
3728
3729	void tcg_gen_gvec_rotrv(unsigned vece, uint32_t dofs, uint32_t aofs,
3730	uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
3731	{
3732	static const TCGOpcode vecop_list[] = { INDEX_op_rotrv_vec, 0 };
3733	static const GVecGen3 g[4] = {
3734	{ .fniv = tcg_gen_rotrv_mod_vec,
3735	.fno = gen_helper_gvec_rotr8v,
3736	.opt_opc = vecop_list,
3737	.vece = MO_8 },
3738	{ .fniv = tcg_gen_rotrv_mod_vec,
3739	.fno = gen_helper_gvec_rotr16v,
3740	.opt_opc = vecop_list,
3741	.vece = MO_16 },
3742	{ .fni4 = tcg_gen_rotr_mod_i32,
3743	.fniv = tcg_gen_rotrv_mod_vec,
3744	.fno = gen_helper_gvec_rotr32v,
3745	.opt_opc = vecop_list,
3746	.vece = MO_32 },
3747	{ .fni8 = tcg_gen_rotr_mod_i64,
3748	.fniv = tcg_gen_rotrv_mod_vec,
3749	.fno = gen_helper_gvec_rotr64v,
3750	.opt_opc = vecop_list,
3751	.prefer_i64 = true,
3752	.vece = MO_64 },
3753	};
3754
3755	tcg_debug_assert(vece <= MO_64);
3756	tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
3757	}
3758
3759	/* Expand OPSZ bytes worth of three-operand operations using i32 elements. */
3760	static void expand_cmp_i32(uint32_t dofs, uint32_t aofs, uint32_t bofs,
3761	uint32_t oprsz, TCGCond cond)
3762	{
3763	TCGv_i32 t0 = tcg_temp_ebb_new_i32();
3764	TCGv_i32 t1 = tcg_temp_ebb_new_i32();
3765	uint32_t i;
3766
3767	for (i = 0; i < oprsz; i += 4) {
3768	tcg_gen_ld_i32(t0, tcg_env, aofs + i);
3769	tcg_gen_ld_i32(t1, tcg_env, bofs + i);
3770	tcg_gen_negsetcond_i32(cond, t0, t0, t1);
3771	tcg_gen_st_i32(t0, tcg_env, dofs + i);
3772	}
3773	tcg_temp_free_i32(t1);
3774	tcg_temp_free_i32(t0);
3775	}
3776
3777	static void expand_cmp_i64(uint32_t dofs, uint32_t aofs, uint32_t bofs,
3778	uint32_t oprsz, TCGCond cond)
3779	{
3780	TCGv_i64 t0 = tcg_temp_ebb_new_i64();
3781	TCGv_i64 t1 = tcg_temp_ebb_new_i64();
3782	uint32_t i;
3783
3784	for (i = 0; i < oprsz; i += 8) {
3785	tcg_gen_ld_i64(t0, tcg_env, aofs + i);
3786	tcg_gen_ld_i64(t1, tcg_env, bofs + i);
3787	tcg_gen_negsetcond_i64(cond, t0, t0, t1);
3788	tcg_gen_st_i64(t0, tcg_env, dofs + i);
3789	}
3790	tcg_temp_free_i64(t1);
3791	tcg_temp_free_i64(t0);
3792	}
3793
3794	static void expand_cmp_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
3795	uint32_t bofs, uint32_t oprsz, uint32_t tysz,
3796	TCGType type, TCGCond cond)
3797	{
3798	for (uint32_t i = 0; i < oprsz; i += tysz) {
3799	TCGv_vec t0 = tcg_temp_new_vec(type);
3800	TCGv_vec t1 = tcg_temp_new_vec(type);
3801	TCGv_vec t2 = tcg_temp_new_vec(type);
3802
3803	tcg_gen_ld_vec(t0, tcg_env, aofs + i);
3804	tcg_gen_ld_vec(t1, tcg_env, bofs + i);
3805	tcg_gen_cmp_vec(cond, vece, t2, t0, t1);
3806	tcg_gen_st_vec(t2, tcg_env, dofs + i);
3807	}
3808	}
3809
3810	void tcg_gen_gvec_cmp(TCGCond cond, unsigned vece, uint32_t dofs,
3811	uint32_t aofs, uint32_t bofs,
3812	uint32_t oprsz, uint32_t maxsz)
3813	{
3814	static const TCGOpcode cmp_list[] = { INDEX_op_cmp_vec, 0 };
3815	static gen_helper_gvec_3 * const eq_fn[4] = {
3816	gen_helper_gvec_eq8, gen_helper_gvec_eq16,
3817	gen_helper_gvec_eq32, gen_helper_gvec_eq64
3818	};
3819	static gen_helper_gvec_3 * const ne_fn[4] = {
3820	gen_helper_gvec_ne8, gen_helper_gvec_ne16,
3821	gen_helper_gvec_ne32, gen_helper_gvec_ne64
3822	};
3823	static gen_helper_gvec_3 * const lt_fn[4] = {
3824	gen_helper_gvec_lt8, gen_helper_gvec_lt16,
3825	gen_helper_gvec_lt32, gen_helper_gvec_lt64
3826	};
3827	static gen_helper_gvec_3 * const le_fn[4] = {
3828	gen_helper_gvec_le8, gen_helper_gvec_le16,
3829	gen_helper_gvec_le32, gen_helper_gvec_le64
3830	};
3831	static gen_helper_gvec_3 * const ltu_fn[4] = {
3832	gen_helper_gvec_ltu8, gen_helper_gvec_ltu16,
3833	gen_helper_gvec_ltu32, gen_helper_gvec_ltu64
3834	};
3835	static gen_helper_gvec_3 * const leu_fn[4] = {
3836	gen_helper_gvec_leu8, gen_helper_gvec_leu16,
3837	gen_helper_gvec_leu32, gen_helper_gvec_leu64
3838	};
3839	static gen_helper_gvec_3 * const * const fns[16] = {
3840	[TCG_COND_EQ] = eq_fn,
3841	[TCG_COND_NE] = ne_fn,
3842	[TCG_COND_LT] = lt_fn,
3843	[TCG_COND_LE] = le_fn,
3844	[TCG_COND_LTU] = ltu_fn,
3845	[TCG_COND_LEU] = leu_fn,
3846	};
3847
3848	const TCGOpcode *hold_list;
3849	TCGType type;
3850	uint32_t some;
3851
3852	check_size_align(oprsz, maxsz, dofs \| aofs \| bofs);
3853	check_overlap_3(tcg_env, dofs, tcg_env, aofs, tcg_env, bofs, maxsz);
3854
3855	if (cond == TCG_COND_NEVER \|\| cond == TCG_COND_ALWAYS) {
3856	do_dup(MO_8, tcg_env, dofs, oprsz, maxsz,
3857	NULL, NULL, -(cond == TCG_COND_ALWAYS));
3858	return;
3859	}
3860
3861	/*
3862	* Implement inline with a vector type, if possible;
3863	* prefer_i64 for a 64-bit comparison.
3864	*/
3865	hold_list = tcg_swap_vecop_list(cmp_list);
3866	type = choose_vector_type(cmp_list, vece, oprsz, vece == MO_64);
3867	switch (type) {
3868	case TCG_TYPE_V256:
3869	/* Recall that ARM SVE allows vector sizes that are not a
3870	* power of 2, but always a multiple of 16. The intent is
3871	* that e.g. size == 80 would be expanded with 2x32 + 1x16.
3872	*/
3873	some = QEMU_ALIGN_DOWN(oprsz, 32);
3874	expand_cmp_vec(vece, dofs, aofs, bofs, some, 32, TCG_TYPE_V256, cond);
3875	if (some == oprsz) {
3876	break;
3877	}
3878	dofs += some;
3879	aofs += some;
3880	bofs += some;
3881	oprsz -= some;
3882	maxsz -= some;
3883	/* fallthru */
3884	case TCG_TYPE_V128:
3885	expand_cmp_vec(vece, dofs, aofs, bofs, oprsz, 16, TCG_TYPE_V128, cond);
3886	break;
3887	case TCG_TYPE_V64:
3888	expand_cmp_vec(vece, dofs, aofs, bofs, oprsz, 8, TCG_TYPE_V64, cond);
3889	break;
3890
3891	case 0:
3892	if (vece == MO_64 && check_size_impl(oprsz, 8)) {
3893	expand_cmp_i64(dofs, aofs, bofs, oprsz, cond);
3894	} else if (vece == MO_32 && check_size_impl(oprsz, 4)) {
3895	expand_cmp_i32(dofs, aofs, bofs, oprsz, cond);
3896	} else {
3897	gen_helper_gvec_3 * const *fn = fns[cond];
3898
3899	if (fn == NULL) {
3900	uint32_t tmp;
3901	tmp = aofs, aofs = bofs, bofs = tmp;
3902	cond = tcg_swap_cond(cond);
3903	fn = fns[cond];
3904	assert(fn != NULL);
3905	}
3906	tcg_gen_gvec_3_ool(dofs, aofs, bofs, oprsz, maxsz, 0, fn[vece]);
3907	oprsz = maxsz;
3908	}
3909	break;
3910
3911	default:
3912	g_assert_not_reached();
3913	}
3914	tcg_swap_vecop_list(hold_list);
3915
3916	if (oprsz < maxsz) {
3917	expand_clr(tcg_env, dofs + oprsz, maxsz - oprsz);
3918	}
3919	}
3920
3921	static void expand_cmps_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
3922	uint32_t oprsz, uint32_t tysz, TCGType type,
3923	TCGCond cond, TCGv_vec c)
3924	{
3925	TCGv_vec t0 = tcg_temp_new_vec(type);
3926	TCGv_vec t1 = tcg_temp_new_vec(type);
3927	uint32_t i;
3928
3929	for (i = 0; i < oprsz; i += tysz) {
3930	tcg_gen_ld_vec(t1, tcg_env, aofs + i);
3931	tcg_gen_cmp_vec(cond, vece, t0, t1, c);
3932	tcg_gen_st_vec(t0, tcg_env, dofs + i);
3933	}
3934	}
3935
3936	void tcg_gen_gvec_cmps(TCGCond cond, unsigned vece, uint32_t dofs,
3937	uint32_t aofs, TCGv_i64 c,
3938	uint32_t oprsz, uint32_t maxsz)
3939	{
3940	static const TCGOpcode cmp_list[] = { INDEX_op_cmp_vec, 0 };
3941	static gen_helper_gvec_2i * const eq_fn[4] = {
3942	gen_helper_gvec_eqs8, gen_helper_gvec_eqs16,
3943	gen_helper_gvec_eqs32, gen_helper_gvec_eqs64
3944	};
3945	static gen_helper_gvec_2i * const lt_fn[4] = {
3946	gen_helper_gvec_lts8, gen_helper_gvec_lts16,
3947	gen_helper_gvec_lts32, gen_helper_gvec_lts64
3948	};
3949	static gen_helper_gvec_2i * const le_fn[4] = {
3950	gen_helper_gvec_les8, gen_helper_gvec_les16,
3951	gen_helper_gvec_les32, gen_helper_gvec_les64
3952	};
3953	static gen_helper_gvec_2i * const ltu_fn[4] = {
3954	gen_helper_gvec_ltus8, gen_helper_gvec_ltus16,
3955	gen_helper_gvec_ltus32, gen_helper_gvec_ltus64
3956	};
3957	static gen_helper_gvec_2i * const leu_fn[4] = {
3958	gen_helper_gvec_leus8, gen_helper_gvec_leus16,
3959	gen_helper_gvec_leus32, gen_helper_gvec_leus64
3960	};
3961	static gen_helper_gvec_2i * const * const fns[16] = {
3962	[TCG_COND_EQ] = eq_fn,
3963	[TCG_COND_LT] = lt_fn,
3964	[TCG_COND_LE] = le_fn,
3965	[TCG_COND_LTU] = ltu_fn,
3966	[TCG_COND_LEU] = leu_fn,
3967	};
3968
3969	TCGType type;
3970
3971	check_size_align(oprsz, maxsz, dofs \| aofs);
3972	check_overlap_2(tcg_env, dofs, tcg_env, aofs, maxsz);
3973
3974	if (cond == TCG_COND_NEVER \|\| cond == TCG_COND_ALWAYS) {
3975	do_dup(MO_8, tcg_env, dofs, oprsz, maxsz,
3976	NULL, NULL, -(cond == TCG_COND_ALWAYS));
3977	return;
3978	}
3979
3980	/*
3981	* Implement inline with a vector type, if possible;
3982	* prefer_i64 for a 64-bit comparison.
3983	*/
3984	type = choose_vector_type(cmp_list, vece, oprsz, vece == MO_64);
3985	if (type != 0) {
3986	const TCGOpcode *hold_list = tcg_swap_vecop_list(cmp_list);
3987	TCGv_vec t_vec = tcg_temp_new_vec(type);
3988	uint32_t some;
3989
3990	tcg_gen_dup_i64_vec(vece, t_vec, c);
3991	switch (type) {
3992	case TCG_TYPE_V256:
3993	some = QEMU_ALIGN_DOWN(oprsz, 32);
3994	expand_cmps_vec(vece, dofs, aofs, some, 32,
3995	TCG_TYPE_V256, cond, t_vec);
3996	aofs += some;
3997	dofs += some;
3998	oprsz -= some;
3999	maxsz -= some;
4000	/* fallthru */
4001
4002	case TCG_TYPE_V128:
4003	some = QEMU_ALIGN_DOWN(oprsz, 16);
4004	expand_cmps_vec(vece, dofs, aofs, some, 16,
4005	TCG_TYPE_V128, cond, t_vec);
4006	break;
4007
4008	case TCG_TYPE_V64:
4009	some = QEMU_ALIGN_DOWN(oprsz, 8);
4010	expand_cmps_vec(vece, dofs, aofs, some, 8,
4011	TCG_TYPE_V64, cond, t_vec);
4012	break;
4013
4014	default:
4015	g_assert_not_reached();
4016	}
4017	tcg_temp_free_vec(t_vec);
4018	tcg_swap_vecop_list(hold_list);
4019	} else if (vece == MO_64 && check_size_impl(oprsz, 8)) {
4020	TCGv_i64 t0 = tcg_temp_ebb_new_i64();
4021	uint32_t i;
4022
4023	for (i = 0; i < oprsz; i += 8) {
4024	tcg_gen_ld_i64(t0, tcg_env, aofs + i);
4025	tcg_gen_negsetcond_i64(cond, t0, t0, c);
4026	tcg_gen_st_i64(t0, tcg_env, dofs + i);
4027	}
4028	tcg_temp_free_i64(t0);
4029	} else if (vece == MO_32 && check_size_impl(oprsz, 4)) {
4030	TCGv_i32 t0 = tcg_temp_ebb_new_i32();
4031	TCGv_i32 t1 = tcg_temp_ebb_new_i32();
4032	uint32_t i;
4033
4034	tcg_gen_extrl_i64_i32(t1, c);
4035	for (i = 0; i < oprsz; i += 4) {
4036	tcg_gen_ld_i32(t0, tcg_env, aofs + i);
4037	tcg_gen_negsetcond_i32(cond, t0, t0, t1);
4038	tcg_gen_st_i32(t0, tcg_env, dofs + i);
4039	}
4040	tcg_temp_free_i32(t0);
4041	tcg_temp_free_i32(t1);
4042	} else {
4043	gen_helper_gvec_2i * const *fn = fns[cond];
4044	bool inv = false;
4045
4046	if (fn == NULL) {
4047	cond = tcg_invert_cond(cond);
4048	fn = fns[cond];
4049	assert(fn != NULL);
4050	inv = true;
4051	}
4052	tcg_gen_gvec_2i_ool(dofs, aofs, c, oprsz, maxsz, inv, fn[vece]);
4053	return;
4054	}
4055
4056	if (oprsz < maxsz) {
4057	expand_clr(tcg_env, dofs + oprsz, maxsz - oprsz);
4058	}
4059	}
4060
4061	void tcg_gen_gvec_cmpi(TCGCond cond, unsigned vece, uint32_t dofs,
4062	uint32_t aofs, int64_t c,
4063	uint32_t oprsz, uint32_t maxsz)
4064	{
4065	TCGv_i64 tmp = tcg_constant_i64(c);
4066	tcg_gen_gvec_cmps(cond, vece, dofs, aofs, tmp, oprsz, maxsz);
4067	}
4068
4069	static void tcg_gen_bitsel_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, TCGv_i64 c)
4070	{
4071	TCGv_i64 t = tcg_temp_ebb_new_i64();
4072
4073	tcg_gen_and_i64(t, b, a);
4074	tcg_gen_andc_i64(d, c, a);
4075	tcg_gen_or_i64(d, d, t);
4076	tcg_temp_free_i64(t);
4077	}
4078
4079	void tcg_gen_gvec_bitsel(unsigned vece, uint32_t dofs, uint32_t aofs,
4080	uint32_t bofs, uint32_t cofs,
4081	uint32_t oprsz, uint32_t maxsz)
4082	{
4083	static const GVecGen4 g = {
4084	.fni8 = tcg_gen_bitsel_i64,
4085	.fniv = tcg_gen_bitsel_vec,
4086	.fno = gen_helper_gvec_bitsel,
4087	};
4088
4089	tcg_gen_gvec_4(dofs, aofs, bofs, cofs, oprsz, maxsz, &g);
4090	}