tcg-target.c.inc · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

inc 2,836 lines 83 KB

Raw

1	/*
2	* Tiny Code Generator for QEMU
3	*
4	* Copyright (c) 2021 WANG Xuerui <git@xen0n.name>
5	*
6	* Based on tcg/riscv/tcg-target.c.inc
7	*
8	* Copyright (c) 2018 SiFive, Inc
9	* Copyright (c) 2008-2009 Arnaud Patard <arnaud.patard@rtp-net.org>
10	* Copyright (c) 2009 Aurelien Jarno <aurelien@aurel32.net>
11	* Copyright (c) 2008 Fabrice Bellard
12	*
13	* Permission is hereby granted, free of charge, to any person obtaining a copy
14	* of this software and associated documentation files (the "Software"), to deal
15	* in the Software without restriction, including without limitation the rights
16	* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
17	* copies of the Software, and to permit persons to whom the Software is
18	* furnished to do so, subject to the following conditions:
19	*
20	* The above copyright notice and this permission notice shall be included in
21	* all copies or substantial portions of the Software.
22	*
23	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
24	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
25	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
26	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
27	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
28	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
29	* THE SOFTWARE.
30	*/
31
32	#include <asm/hwcap.h>
33
34	/* used for function call generation */
35	#define TCG_REG_CALL_STACK TCG_REG_SP
36	#define TCG_TARGET_STACK_ALIGN 16
37	#define TCG_TARGET_CALL_STACK_OFFSET 0
38	#define TCG_TARGET_CALL_ARG_I32 TCG_CALL_ARG_NORMAL
39	#define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL
40	#define TCG_TARGET_CALL_ARG_I128 TCG_CALL_ARG_NORMAL
41	#define TCG_TARGET_CALL_RET_I128 TCG_CALL_RET_NORMAL
42
43	#ifdef CONFIG_DEBUG_TCG
44	static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
45	"zero",
46	"ra",
47	"tp",
48	"sp",
49	"a0",
50	"a1",
51	"a2",
52	"a3",
53	"a4",
54	"a5",
55	"a6",
56	"a7",
57	"t0",
58	"t1",
59	"t2",
60	"t3",
61	"t4",
62	"t5",
63	"t6",
64	"t7",
65	"t8",
66	"r21", /* reserved in the LP64* ABI, hence no ABI name */
67	"s9",
68	"s0",
69	"s1",
70	"s2",
71	"s3",
72	"s4",
73	"s5",
74	"s6",
75	"s7",
76	"s8",
77	"vr0",
78	"vr1",
79	"vr2",
80	"vr3",
81	"vr4",
82	"vr5",
83	"vr6",
84	"vr7",
85	"vr8",
86	"vr9",
87	"vr10",
88	"vr11",
89	"vr12",
90	"vr13",
91	"vr14",
92	"vr15",
93	"vr16",
94	"vr17",
95	"vr18",
96	"vr19",
97	"vr20",
98	"vr21",
99	"vr22",
100	"vr23",
101	"vr24",
102	"vr25",
103	"vr26",
104	"vr27",
105	"vr28",
106	"vr29",
107	"vr30",
108	"vr31",
109	};
110	#endif
111
112	static const int tcg_target_reg_alloc_order[] = {
113	/* Registers preserved across calls */
114	/* TCG_REG_S0 reserved for TCG_AREG0 */
115	TCG_REG_S1,
116	TCG_REG_S2,
117	TCG_REG_S3,
118	TCG_REG_S4,
119	TCG_REG_S5,
120	TCG_REG_S6,
121	TCG_REG_S7,
122	TCG_REG_S8,
123	TCG_REG_S9,
124
125	/* Registers (potentially) clobbered across calls */
126	TCG_REG_T0,
127	TCG_REG_T1,
128	TCG_REG_T2,
129	TCG_REG_T3,
130	TCG_REG_T4,
131	TCG_REG_T5,
132	TCG_REG_T6,
133	TCG_REG_T7,
134	TCG_REG_T8,
135
136	/* Argument registers, opposite order of allocation. */
137	TCG_REG_A7,
138	TCG_REG_A6,
139	TCG_REG_A5,
140	TCG_REG_A4,
141	TCG_REG_A3,
142	TCG_REG_A2,
143	TCG_REG_A1,
144	TCG_REG_A0,
145
146	/* Vector registers */
147	TCG_REG_V0, TCG_REG_V1, TCG_REG_V2, TCG_REG_V3,
148	TCG_REG_V4, TCG_REG_V5, TCG_REG_V6, TCG_REG_V7,
149	TCG_REG_V8, TCG_REG_V9, TCG_REG_V10, TCG_REG_V11,
150	TCG_REG_V12, TCG_REG_V13, TCG_REG_V14, TCG_REG_V15,
151	TCG_REG_V16, TCG_REG_V17, TCG_REG_V18, TCG_REG_V19,
152	TCG_REG_V20, TCG_REG_V21, TCG_REG_V22, TCG_REG_V23,
153	/* V24 - V31 are caller-saved, and skipped. */
154	};
155
156	static const int tcg_target_call_iarg_regs[] = {
157	TCG_REG_A0,
158	TCG_REG_A1,
159	TCG_REG_A2,
160	TCG_REG_A3,
161	TCG_REG_A4,
162	TCG_REG_A5,
163	TCG_REG_A6,
164	TCG_REG_A7,
165	};
166
167	static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot)
168	{
169	tcg_debug_assert(kind == TCG_CALL_RET_NORMAL);
170	tcg_debug_assert(slot >= 0 && slot <= 1);
171	return TCG_REG_A0 + slot;
172	}
173
174	#define TCG_GUEST_BASE_REG TCG_REG_S1
175
176	#define TCG_CT_CONST_S12 0x100
177	#define TCG_CT_CONST_S32 0x200
178	#define TCG_CT_CONST_U12 0x400
179	#define TCG_CT_CONST_WSZ 0x800
180	#define TCG_CT_CONST_VCMP 0x1000
181	#define TCG_CT_CONST_VADD 0x2000
182
183	#define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 32)
184	#define ALL_VECTOR_REGS MAKE_64BIT_MASK(32, 32)
185
186	static inline tcg_target_long sextreg(tcg_target_long val, int pos, int len)
187	{
188	return sextract64(val, pos, len);
189	}
190
191	/* test if a constant matches the constraint */
192	static bool tcg_target_const_match(int64_t val, int ct,
193	TCGType type, TCGCond cond, int vece)
194	{
195	if (ct & TCG_CT_CONST) {
196	return true;
197	}
198	if ((ct & TCG_CT_CONST_S12) && val == sextreg(val, 0, 12)) {
199	return true;
200	}
201	if ((ct & TCG_CT_CONST_S32) && val == (int32_t)val) {
202	return true;
203	}
204	if ((ct & TCG_CT_CONST_U12) && val >= 0 && val <= 0xfff) {
205	return true;
206	}
207	if ((ct & TCG_CT_CONST_WSZ) && val == (type == TCG_TYPE_I32 ? 32 : 64)) {
208	return true;
209	}
210	if (ct & (TCG_CT_CONST_VCMP \| TCG_CT_CONST_VADD)) {
211	int64_t vec_val = sextract64(val, 0, 8 << vece);
212	if (ct & TCG_CT_CONST_VCMP) {
213	switch (cond) {
214	case TCG_COND_EQ:
215	case TCG_COND_LE:
216	case TCG_COND_LT:
217	return -0x10 <= vec_val && vec_val <= 0x0f;
218	case TCG_COND_LEU:
219	case TCG_COND_LTU:
220	return 0x00 <= vec_val && vec_val <= 0x1f;
221	default:
222	return false;
223	}
224	}
225	if ((ct & TCG_CT_CONST_VADD) && -0x1f <= vec_val && vec_val <= 0x1f) {
226	return true;
227	}
228	}
229	return false;
230	}
231
232	/*
233	* Relocations
234	*/
235
236	/*
237	* Relocation records defined in LoongArch ELF psABI v1.00 is way too
238	* complicated; a whopping stack machine is needed to stuff the fields, at
239	* the very least one SOP_PUSH and one SOP_POP (of the correct format) are
240	* needed.
241	*
242	* Hence, define our own simpler relocation types. Numbers are chosen as to
243	* not collide with potential future additions to the true ELF relocation
244	* type enum.
245	*/
246
247	/* Field Sk16, shifted right by 2; suitable for conditional jumps */
248	#define R_LOONGARCH_BR_SK16 256
249	/* Field Sd10k16, shifted right by 2; suitable for B and BL */
250	#define R_LOONGARCH_BR_SD10K16 257
251
252	static bool reloc_br_sk16(tcg_insn_unit src_rw, const tcg_insn_unit target)
253	{
254	const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
255	intptr_t offset = (intptr_t)target - (intptr_t)src_rx;
256
257	tcg_debug_assert((offset & 3) == 0);
258	offset >>= 2;
259	if (offset == sextreg(offset, 0, 16)) {
260	src_rw = deposit64(src_rw, 10, 16, offset);
261	return true;
262	}
263
264	return false;
265	}
266
267	static bool reloc_br_sd10k16(tcg_insn_unit *src_rw,
268	const tcg_insn_unit *target)
269	{
270	const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
271	intptr_t offset = (intptr_t)target - (intptr_t)src_rx;
272
273	tcg_debug_assert((offset & 3) == 0);
274	offset >>= 2;
275	if (offset == sextreg(offset, 0, 26)) {
276	src_rw = deposit64(src_rw, 0, 10, offset >> 16); /* slot d10 */
277	src_rw = deposit64(src_rw, 10, 16, offset); /* slot k16 */
278	return true;
279	}
280
281	return false;
282	}
283
284	static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
285	intptr_t value, intptr_t addend)
286	{
287	tcg_debug_assert(addend == 0);
288	switch (type) {
289	case R_LOONGARCH_BR_SK16:
290	return reloc_br_sk16(code_ptr, (tcg_insn_unit *)value);
291	case R_LOONGARCH_BR_SD10K16:
292	return reloc_br_sd10k16(code_ptr, (tcg_insn_unit *)value);
293	default:
294	g_assert_not_reached();
295	}
296	}
297
298	#include "tcg-insn-defs.c.inc"
299
300	/*
301	* TCG intrinsics
302	*/
303
304	static void tcg_out_mb(TCGContext *s, unsigned a0)
305	{
306	/* Baseline LoongArch only has the full barrier, unfortunately. */
307	tcg_out_opc_dbar(s, 0);
308	}
309
310	static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
311	{
312	if (ret == arg) {
313	return true;
314	}
315	switch (type) {
316	case TCG_TYPE_I32:
317	case TCG_TYPE_I64:
318	if (ret < TCG_REG_V0) {
319	if (arg < TCG_REG_V0) {
320	/*
321	* Conventional register-register move used in LoongArch is
322	* `or dst, src, zero`.
323	*/
324	tcg_out_opc_or(s, ret, arg, TCG_REG_ZERO);
325	} else {
326	tcg_out_opc_movfr2gr_d(s, ret, arg);
327	}
328	} else {
329	if (arg < TCG_REG_V0) {
330	tcg_out_opc_movgr2fr_d(s, ret, arg);
331	} else {
332	tcg_out_opc_fmov_d(s, ret, arg);
333	}
334	}
335	break;
336	case TCG_TYPE_V64:
337	case TCG_TYPE_V128:
338	tcg_out_opc_vori_b(s, ret, arg, 0);
339	break;
340	case TCG_TYPE_V256:
341	tcg_out_opc_xvori_b(s, ret, arg, 0);
342	break;
343	default:
344	g_assert_not_reached();
345	}
346	return true;
347	}
348
349	/* Loads a 32-bit immediate into rd, sign-extended. */
350	static void tcg_out_movi_i32(TCGContext *s, TCGReg rd, int32_t val)
351	{
352	tcg_target_long lo = sextreg(val, 0, 12);
353	tcg_target_long hi12 = sextreg(val, 12, 20);
354
355	/* Single-instruction cases. */
356	if (hi12 == 0) {
357	/* val fits in uimm12: ori rd, zero, val */
358	tcg_out_opc_ori(s, rd, TCG_REG_ZERO, val);
359	return;
360	}
361	if (hi12 == sextreg(lo, 12, 20)) {
362	/* val fits in simm12: addi.w rd, zero, val */
363	tcg_out_opc_addi_w(s, rd, TCG_REG_ZERO, val);
364	return;
365	}
366
367	/* High bits must be set; load with lu12i.w + optional ori. */
368	tcg_out_opc_lu12i_w(s, rd, hi12);
369	if (lo != 0) {
370	tcg_out_opc_ori(s, rd, rd, lo & 0xfff);
371	}
372	}
373
374	static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg rd,
375	tcg_target_long val)
376	{
377	/*
378	* LoongArch conventionally loads 64-bit immediates in at most 4 steps,
379	* with dedicated instructions for filling the respective bitfields
380	* below:
381	*
382	* 6 5 4 3
383	* 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2
384	* +-----------------------+---------------------------------------+...
385	* \| hi52 \| hi32 \|
386	* +-----------------------+---------------------------------------+...
387	* 3 2 1
388	* 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0
389	* ...+-------------------------------------+-------------------------+
390	* \| hi12 \| lo \|
391	* ...+-------------------------------------+-------------------------+
392	*
393	* Check if val belong to one of the several fast cases, before falling
394	* back to the slow path.
395	*/
396
397	intptr_t src_rx, pc_offset;
398	tcg_target_long hi12, hi32, hi52;
399
400	/* Value fits in signed i32. */
401	if (type == TCG_TYPE_I32 \|\| val == (int32_t)val) {
402	tcg_out_movi_i32(s, rd, val);
403	return;
404	}
405
406	/* PC-relative cases. */
407	src_rx = (intptr_t)tcg_splitwx_to_rx(s->code_ptr);
408	if ((val & 3) == 0) {
409	pc_offset = val - src_rx;
410	if (pc_offset == sextreg(pc_offset, 0, 22)) {
411	/* Single pcaddu2i. */
412	tcg_out_opc_pcaddu2i(s, rd, pc_offset >> 2);
413	return;
414	}
415	}
416
417	pc_offset = (val >> 12) - (src_rx >> 12);
418	if (pc_offset == sextreg(pc_offset, 0, 20)) {
419	/* Load with pcalau12i + ori. */
420	tcg_target_long val_lo = val & 0xfff;
421	tcg_out_opc_pcalau12i(s, rd, pc_offset);
422	if (val_lo != 0) {
423	tcg_out_opc_ori(s, rd, rd, val_lo);
424	}
425	return;
426	}
427
428	hi12 = sextreg(val, 12, 20);
429	hi32 = sextreg(val, 32, 20);
430	hi52 = sextreg(val, 52, 12);
431
432	/* Single cu52i.d case. */
433	if ((hi52 != 0) && (ctz64(val) >= 52)) {
434	tcg_out_opc_cu52i_d(s, rd, TCG_REG_ZERO, hi52);
435	return;
436	}
437
438	/* Slow path. Initialize the low 32 bits, then concat high bits. */
439	tcg_out_movi_i32(s, rd, val);
440
441	/* Load hi32 and hi52 explicitly when they are unexpected values. */
442	if (hi32 != sextreg(hi12, 20, 20)) {
443	tcg_out_opc_cu32i_d(s, rd, hi32);
444	}
445
446	if (hi52 != sextreg(hi32, 20, 12)) {
447	tcg_out_opc_cu52i_d(s, rd, rd, hi52);
448	}
449	}
450
451	static void tcg_out_addi(TCGContext *s, TCGType type, TCGReg rd,
452	TCGReg rs, tcg_target_long imm)
453	{
454	tcg_target_long lo12 = sextreg(imm, 0, 12);
455	tcg_target_long hi16 = sextreg(imm - lo12, 16, 16);
456
457	/*
458	* Note that there's a hole in between hi16 and lo12:
459	*
460	* 3 2 1 0
461	* 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0
462	* ...+-------------------------------+-------+-----------------------+
463	* \| hi16 \| \| lo12 \|
464	* ...+-------------------------------+-------+-----------------------+
465	*
466	* For bits within that hole, it's more efficient to use LU12I and ADD.
467	*/
468	if (imm == (hi16 << 16) + lo12) {
469	if (hi16) {
470	tcg_out_opc_addu16i_d(s, rd, rs, hi16);
471	rs = rd;
472	}
473	if (type == TCG_TYPE_I32) {
474	tcg_out_opc_addi_w(s, rd, rs, lo12);
475	} else if (lo12) {
476	tcg_out_opc_addi_d(s, rd, rs, lo12);
477	} else {
478	tcg_out_mov(s, type, rd, rs);
479	}
480	} else {
481	tcg_out_movi(s, type, TCG_REG_TMP0, imm);
482	if (type == TCG_TYPE_I32) {
483	tcg_out_opc_add_w(s, rd, rs, TCG_REG_TMP0);
484	} else {
485	tcg_out_opc_add_d(s, rd, rs, TCG_REG_TMP0);
486	}
487	}
488	}
489
490	static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
491	{
492	return false;
493	}
494
495	static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
496	tcg_target_long imm)
497	{
498	/* This function is only used for passing structs by reference. */
499	g_assert_not_reached();
500	}
501
502	static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg)
503	{
504	tcg_out_opc_andi(s, ret, arg, 0xff);
505	}
506
507	static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg)
508	{
509	tcg_out_opc_bstrpick_w(s, ret, arg, 0, 15);
510	}
511
512	static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg)
513	{
514	tcg_out_opc_bstrpick_d(s, ret, arg, 0, 31);
515	}
516
517	static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
518	{
519	tcg_out_opc_sext_b(s, ret, arg);
520	}
521
522	static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
523	{
524	tcg_out_opc_sext_h(s, ret, arg);
525	}
526
527	static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg)
528	{
529	tcg_out_opc_addi_w(s, ret, arg, 0);
530	}
531
532	static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg)
533	{
534	if (ret != arg) {
535	tcg_out_ext32s(s, ret, arg);
536	}
537	}
538
539	static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg)
540	{
541	tcg_out_ext32u(s, ret, arg);
542	}
543
544	static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg)
545	{
546	tcg_out_ext32s(s, ret, arg);
547	}
548
549	#define SETCOND_INV TCG_TARGET_NB_REGS
550	#define SETCOND_NEZ (SETCOND_INV << 1)
551	#define SETCOND_FLAGS (SETCOND_INV \| SETCOND_NEZ)
552
553	static int tcg_out_setcond_int(TCGContext *s, TCGCond cond, TCGReg ret,
554	TCGReg arg1, tcg_target_long arg2, bool c2)
555	{
556	int flags = 0;
557
558	switch (cond) {
559	case TCG_COND_EQ: /* -> NE */
560	case TCG_COND_GE: /* -> LT */
561	case TCG_COND_GEU: /* -> LTU */
562	case TCG_COND_GT: /* -> LE */
563	case TCG_COND_GTU: /* -> LEU */
564	cond = tcg_invert_cond(cond);
565	flags ^= SETCOND_INV;
566	break;
567	default:
568	break;
569	}
570
571	switch (cond) {
572	case TCG_COND_LE:
573	case TCG_COND_LEU:
574	/*
575	* If we have a constant input, the most efficient way to implement
576	* LE is by adding 1 and using LT. Watch out for wrap around for LEU.
577	* We don't need to care for this for LE because the constant input
578	* is still constrained to int32_t, and INT32_MAX+1 is representable
579	* in the 64-bit temporary register.
580	*/
581	if (c2) {
582	if (cond == TCG_COND_LEU) {
583	/* unsigned <= -1 is true */
584	if (arg2 == -1) {
585	tcg_out_movi(s, TCG_TYPE_REG, ret, !(flags & SETCOND_INV));
586	return ret;
587	}
588	cond = TCG_COND_LTU;
589	} else {
590	cond = TCG_COND_LT;
591	}
592	arg2 += 1;
593	} else {
594	TCGReg tmp = arg2;
595	arg2 = arg1;
596	arg1 = tmp;
597	cond = tcg_swap_cond(cond); /* LE -> GE */
598	cond = tcg_invert_cond(cond); /* GE -> LT */
599	flags ^= SETCOND_INV;
600	}
601	break;
602	default:
603	break;
604	}
605
606	switch (cond) {
607	case TCG_COND_NE:
608	flags \|= SETCOND_NEZ;
609	if (!c2) {
610	tcg_out_opc_xor(s, ret, arg1, arg2);
611	} else if (arg2 == 0) {
612	ret = arg1;
613	} else if (arg2 >= 0 && arg2 <= 0xfff) {
614	tcg_out_opc_xori(s, ret, arg1, arg2);
615	} else {
616	tcg_out_addi(s, TCG_TYPE_REG, ret, arg1, -arg2);
617	}
618	break;
619
620	case TCG_COND_LT:
621	case TCG_COND_LTU:
622	if (c2) {
623	if (arg2 >= -0x800 && arg2 <= 0x7ff) {
624	if (cond == TCG_COND_LT) {
625	tcg_out_opc_slti(s, ret, arg1, arg2);
626	} else {
627	tcg_out_opc_sltui(s, ret, arg1, arg2);
628	}
629	break;
630	}
631	tcg_out_movi(s, TCG_TYPE_REG, TCG_REG_TMP0, arg2);
632	arg2 = TCG_REG_TMP0;
633	}
634	if (cond == TCG_COND_LT) {
635	tcg_out_opc_slt(s, ret, arg1, arg2);
636	} else {
637	tcg_out_opc_sltu(s, ret, arg1, arg2);
638	}
639	break;
640
641	default:
642	g_assert_not_reached();
643	}
644
645	return ret \| flags;
646	}
647
648	static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret,
649	TCGReg arg1, tcg_target_long arg2,
650	bool c2, bool neg)
651	{
652	int tmpflags = tcg_out_setcond_int(s, cond, ret, arg1, arg2, c2);
653	TCGReg tmp = tmpflags & ~SETCOND_FLAGS;
654
655	if (neg) {
656	/* If intermediate result is zero/non-zero: test != 0. */
657	if (tmpflags & SETCOND_NEZ) {
658	tcg_out_opc_sltu(s, ret, TCG_REG_ZERO, tmp);
659	tmp = ret;
660	}
661	/* Produce the 0/-1 result. */
662	if (tmpflags & SETCOND_INV) {
663	tcg_out_opc_addi_d(s, ret, tmp, -1);
664	} else {
665	tcg_out_opc_sub_d(s, ret, TCG_REG_ZERO, tmp);
666	}
667	} else {
668	switch (tmpflags & SETCOND_FLAGS) {
669	case 0:
670	tcg_debug_assert(tmp == ret);
671	break;
672	case SETCOND_INV:
673	/* Intermediate result is boolean: simply invert. */
674	tcg_out_opc_xori(s, ret, tmp, 1);
675	break;
676	case SETCOND_NEZ:
677	/* Intermediate result is zero/non-zero: test != 0. */
678	tcg_out_opc_sltu(s, ret, TCG_REG_ZERO, tmp);
679	break;
680	case SETCOND_NEZ \| SETCOND_INV:
681	/* Intermediate result is zero/non-zero: test == 0. */
682	tcg_out_opc_sltui(s, ret, tmp, 1);
683	break;
684	default:
685	g_assert_not_reached();
686	}
687	}
688	}
689
690	static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond,
691	TCGReg dest, TCGReg arg1, TCGReg arg2)
692	{
693	tcg_out_setcond(s, cond, dest, arg1, arg2, false, false);
694	}
695
696	static void tgen_setcondi(TCGContext *s, TCGType type, TCGCond cond,
697	TCGReg dest, TCGReg arg1, tcg_target_long arg2)
698	{
699	tcg_out_setcond(s, cond, dest, arg1, arg2, true, false);
700	}
701
702	static const TCGOutOpSetcond outop_setcond = {
703	.base.static_constraint = C_O1_I2(r, r, rJ),
704	.out_rrr = tgen_setcond,
705	.out_rri = tgen_setcondi,
706	};
707
708	static void tgen_negsetcond(TCGContext *s, TCGType type, TCGCond cond,
709	TCGReg dest, TCGReg arg1, TCGReg arg2)
710	{
711	tcg_out_setcond(s, cond, dest, arg1, arg2, false, true);
712	}
713
714	static void tgen_negsetcondi(TCGContext *s, TCGType type, TCGCond cond,
715	TCGReg dest, TCGReg arg1, tcg_target_long arg2)
716	{
717	tcg_out_setcond(s, cond, dest, arg1, arg2, true, true);
718	}
719
720	static const TCGOutOpSetcond outop_negsetcond = {
721	.base.static_constraint = C_O1_I2(r, r, rJ),
722	.out_rrr = tgen_negsetcond,
723	.out_rri = tgen_negsetcondi,
724	};
725
726	static void tgen_movcond(TCGContext *s, TCGType type, TCGCond cond,
727	TCGReg ret, TCGReg c1, TCGArg c2, bool const_c2,
728	TCGArg v1, bool const_v1, TCGArg v2, bool const_v2)
729	{
730	int tmpflags = tcg_out_setcond_int(s, cond, TCG_REG_TMP0, c1, c2, const_c2);
731	TCGReg t;
732
733	/* Standardize the test below to t != 0. */
734	if (tmpflags & SETCOND_INV) {
735	t = v1, v1 = v2, v2 = t;
736	}
737
738	t = tmpflags & ~SETCOND_FLAGS;
739	if (v1 == TCG_REG_ZERO) {
740	tcg_out_opc_masknez(s, ret, v2, t);
741	} else if (v2 == TCG_REG_ZERO) {
742	tcg_out_opc_maskeqz(s, ret, v1, t);
743	} else {
744	tcg_out_opc_masknez(s, TCG_REG_TMP2, v2, t); /* t ? 0 : v2 */
745	tcg_out_opc_maskeqz(s, TCG_REG_TMP1, v1, t); /* t ? v1 : 0 */
746	tcg_out_opc_or(s, ret, TCG_REG_TMP1, TCG_REG_TMP2);
747	}
748	}
749
750	static const TCGOutOpMovcond outop_movcond = {
751	.base.static_constraint = C_O1_I4(r, r, rJ, rz, rz),
752	.out = tgen_movcond,
753	};
754
755	/*
756	* Branch helpers
757	*/
758
759	static void tcg_out_br(TCGContext s, TCGLabel l)
760	{
761	tcg_out_reloc(s, s->code_ptr, R_LOONGARCH_BR_SD10K16, l, 0);
762	tcg_out_opc_b(s, 0);
763	}
764
765	static const struct {
766	LoongArchInsn op;
767	bool swap;
768	} tcg_brcond_to_loongarch[] = {
769	[TCG_COND_EQ] = { OPC_BEQ, false },
770	[TCG_COND_NE] = { OPC_BNE, false },
771	[TCG_COND_LT] = { OPC_BGT, true },
772	[TCG_COND_GE] = { OPC_BLE, true },
773	[TCG_COND_LE] = { OPC_BLE, false },
774	[TCG_COND_GT] = { OPC_BGT, false },
775	[TCG_COND_LTU] = { OPC_BGTU, true },
776	[TCG_COND_GEU] = { OPC_BLEU, true },
777	[TCG_COND_LEU] = { OPC_BLEU, false },
778	[TCG_COND_GTU] = { OPC_BGTU, false }
779	};
780
781	static void tgen_brcond(TCGContext *s, TCGType type, TCGCond cond,
782	TCGReg arg1, TCGReg arg2, TCGLabel *l)
783	{
784	LoongArchInsn op = tcg_brcond_to_loongarch[cond].op;
785
786	tcg_debug_assert(op != 0);
787
788	if (tcg_brcond_to_loongarch[cond].swap) {
789	TCGReg t = arg1;
790	arg1 = arg2;
791	arg2 = t;
792	}
793
794	/* all conditional branch insns belong to DJSk16-format */
795	tcg_out_reloc(s, s->code_ptr, R_LOONGARCH_BR_SK16, l, 0);
796	tcg_out32(s, encode_djsk16_insn(op, arg1, arg2, 0));
797	}
798
799	static const TCGOutOpBrcond outop_brcond = {
800	.base.static_constraint = C_O0_I2(r, rz),
801	.out_rr = tgen_brcond,
802	};
803
804	static void tcg_out_call_int(TCGContext s, const tcg_insn_unit arg, bool tail)
805	{
806	TCGReg link = tail ? TCG_REG_ZERO : TCG_REG_RA;
807	ptrdiff_t offset = tcg_pcrel_diff(s, arg);
808
809	tcg_debug_assert((offset & 3) == 0);
810	if (offset == sextreg(offset, 0, 28)) {
811	/* short jump: +/- 256MiB */
812	if (tail) {
813	tcg_out_opc_b(s, offset >> 2);
814	} else {
815	tcg_out_opc_bl(s, offset >> 2);
816	}
817	} else if (offset == sextreg(offset, 0, 38)) {
818	/* long jump: +/- 256GiB */
819	tcg_target_long lo = sextreg(offset, 0, 18);
820	tcg_target_long hi = offset - lo;
821	tcg_out_opc_pcaddu18i(s, TCG_REG_TMP0, hi >> 18);
822	tcg_out_opc_jirl(s, link, TCG_REG_TMP0, lo >> 2);
823	} else {
824	/* far jump: 64-bit */
825	tcg_target_long lo = sextreg((tcg_target_long)arg, 0, 18);
826	tcg_target_long hi = (tcg_target_long)arg - lo;
827	tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP0, hi);
828	tcg_out_opc_jirl(s, link, TCG_REG_TMP0, lo >> 2);
829	}
830	}
831
832	static void tcg_out_call(TCGContext s, const tcg_insn_unit arg,
833	const TCGHelperInfo *info)
834	{
835	tcg_out_call_int(s, arg, false);
836	}
837
838	/*
839	* Load/store helpers
840	*/
841
842	static void tcg_out_ldst(TCGContext *s, LoongArchInsn opc, TCGReg data,
843	TCGReg addr, intptr_t offset)
844	{
845	intptr_t imm12 = sextreg(offset, 0, 12);
846
847	if (offset != imm12) {
848	intptr_t diff = tcg_pcrel_diff(s, (void *)offset);
849
850	if (addr == TCG_REG_ZERO && diff == (int32_t)diff) {
851	imm12 = sextreg(diff, 0, 12);
852	tcg_out_opc_pcaddu12i(s, TCG_REG_TMP2, (diff - imm12) >> 12);
853	} else {
854	tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP2, offset - imm12);
855	if (addr != TCG_REG_ZERO) {
856	tcg_out_opc_add_d(s, TCG_REG_TMP2, TCG_REG_TMP2, addr);
857	}
858	}
859	addr = TCG_REG_TMP2;
860	}
861
862	switch (opc) {
863	case OPC_LD_B:
864	case OPC_LD_BU:
865	case OPC_LD_H:
866	case OPC_LD_HU:
867	case OPC_LD_W:
868	case OPC_LD_WU:
869	case OPC_LD_D:
870	case OPC_ST_B:
871	case OPC_ST_H:
872	case OPC_ST_W:
873	case OPC_ST_D:
874	tcg_out32(s, encode_djsk12_insn(opc, data, addr, imm12));
875	break;
876	case OPC_FLD_S:
877	case OPC_FLD_D:
878	case OPC_FST_S:
879	case OPC_FST_D:
880	tcg_out32(s, encode_fdjsk12_insn(opc, data, addr, imm12));
881	break;
882	default:
883	g_assert_not_reached();
884	}
885	}
886
887	static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg dest,
888	TCGReg base, intptr_t offset)
889	{
890	switch (type) {
891	case TCG_TYPE_I32:
892	if (dest < TCG_REG_V0) {
893	tcg_out_ldst(s, OPC_LD_W, dest, base, offset);
894	} else {
895	tcg_out_ldst(s, OPC_FLD_S, dest, base, offset);
896	}
897	break;
898	case TCG_TYPE_I64:
899	case TCG_TYPE_V64:
900	if (dest < TCG_REG_V0) {
901	tcg_out_ldst(s, OPC_LD_D, dest, base, offset);
902	} else {
903	tcg_out_ldst(s, OPC_FLD_D, dest, base, offset);
904	}
905	break;
906	case TCG_TYPE_V128:
907	if (-0x800 <= offset && offset <= 0x7ff) {
908	tcg_out_opc_vld(s, dest, base, offset);
909	} else {
910	tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP0, offset);
911	tcg_out_opc_vldx(s, dest, base, TCG_REG_TMP0);
912	}
913	break;
914	case TCG_TYPE_V256:
915	if (-0x800 <= offset && offset <= 0x7ff) {
916	tcg_out_opc_xvld(s, dest, base, offset);
917	} else {
918	tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP0, offset);
919	tcg_out_opc_xvldx(s, dest, base, TCG_REG_TMP0);
920	}
921	break;
922	default:
923	g_assert_not_reached();
924	}
925	}
926
927	static void tcg_out_st(TCGContext *s, TCGType type, TCGReg src,
928	TCGReg base, intptr_t offset)
929	{
930	switch (type) {
931	case TCG_TYPE_I32:
932	if (src < TCG_REG_V0) {
933	tcg_out_ldst(s, OPC_ST_W, src, base, offset);
934	} else {
935	tcg_out_ldst(s, OPC_FST_S, src, base, offset);
936	}
937	break;
938	case TCG_TYPE_I64:
939	case TCG_TYPE_V64:
940	if (src < TCG_REG_V0) {
941	tcg_out_ldst(s, OPC_ST_D, src, base, offset);
942	} else {
943	tcg_out_ldst(s, OPC_FST_D, src, base, offset);
944	}
945	break;
946	case TCG_TYPE_V128:
947	if (-0x800 <= offset && offset <= 0x7ff) {
948	tcg_out_opc_vst(s, src, base, offset);
949	} else {
950	tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP0, offset);
951	tcg_out_opc_vstx(s, src, base, TCG_REG_TMP0);
952	}
953	break;
954	case TCG_TYPE_V256:
955	if (-0x800 <= offset && offset <= 0x7ff) {
956	tcg_out_opc_xvst(s, src, base, offset);
957	} else {
958	tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP0, offset);
959	tcg_out_opc_xvstx(s, src, base, TCG_REG_TMP0);
960	}
961	break;
962	default:
963	g_assert_not_reached();
964	}
965	}
966
967	static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
968	TCGReg base, intptr_t ofs)
969	{
970	if (val == 0) {
971	tcg_out_st(s, type, TCG_REG_ZERO, base, ofs);
972	return true;
973	}
974	return false;
975	}
976
977	/*
978	* Load/store helpers for SoftMMU, and qemu_ld/st implementations
979	*/
980
981	static bool tcg_out_goto(TCGContext s, const tcg_insn_unit target)
982	{
983	tcg_out_opc_b(s, 0);
984	return reloc_br_sd10k16(s->code_ptr - 1, target);
985	}
986
987	static const TCGLdstHelperParam ldst_helper_param = {
988	.ntmp = 1, .tmp = { TCG_REG_TMP0 }
989	};
990
991	static bool tcg_out_qemu_ld_slow_path(TCGContext s, TCGLabelQemuLdst l)
992	{
993	MemOp opc = get_memop(l->oi);
994
995	/* resolve label address */
996	if (!reloc_br_sk16(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
997	return false;
998	}
999
1000	tcg_out_ld_helper_args(s, l, &ldst_helper_param);
1001	tcg_out_call_int(s, qemu_ld_helpers[opc & MO_SIZE], false);
1002	tcg_out_ld_helper_ret(s, l, false, &ldst_helper_param);
1003	return tcg_out_goto(s, l->raddr);
1004	}
1005
1006	static bool tcg_out_qemu_st_slow_path(TCGContext s, TCGLabelQemuLdst l)
1007	{
1008	MemOp opc = get_memop(l->oi);
1009
1010	/* resolve label address */
1011	if (!reloc_br_sk16(l->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
1012	return false;
1013	}
1014
1015	tcg_out_st_helper_args(s, l, &ldst_helper_param);
1016	tcg_out_call_int(s, qemu_st_helpers[opc & MO_SIZE], false);
1017	return tcg_out_goto(s, l->raddr);
1018	}
1019
1020	typedef struct {
1021	TCGReg base;
1022	TCGReg index;
1023	TCGAtomAlign aa;
1024	} HostAddress;
1025
1026	bool tcg_target_has_memory_bswap(MemOp memop)
1027	{
1028	return false;
1029	}
1030
1031	/* We expect to use a 12-bit negative offset from ENV. */
1032	#define MIN_TLB_MASK_TABLE_OFS -(1 << 11)
1033
1034	/*
1035	* For system-mode, perform the TLB load and compare.
1036	* For user-mode, perform any required alignment tests.
1037	* In both cases, return a TCGLabelQemuLdst structure if the slow path
1038	* is required and fill in @h with the host address for the fast path.
1039	*/
1040	static TCGLabelQemuLdst prepare_host_addr(TCGContext s, HostAddress *h,
1041	TCGReg addr_reg, MemOpIdx oi,
1042	bool is_ld)
1043	{
1044	TCGType addr_type = s->addr_type;
1045	TCGLabelQemuLdst *ldst = NULL;
1046	MemOp opc = get_memop(oi);
1047	MemOp a_bits;
1048
1049	h->aa = atom_and_align_for_opc(s, opc, MO_ATOM_IFALIGN, false);
1050	a_bits = h->aa.align;
1051
1052	if (tcg_use_softmmu) {
1053	unsigned s_bits = opc & MO_SIZE;
1054	int mem_index = get_mmuidx(oi);
1055	int fast_ofs = tlb_mask_table_ofs(s, mem_index);
1056	int mask_ofs = fast_ofs + offsetof(CPUTLBDescFast, mask);
1057	int table_ofs = fast_ofs + offsetof(CPUTLBDescFast, table);
1058
1059	ldst = new_ldst_label(s);
1060	ldst->is_ld = is_ld;
1061	ldst->oi = oi;
1062	ldst->addr_reg = addr_reg;
1063
1064	tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_AREG0, mask_ofs);
1065	tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, TCG_AREG0, table_ofs);
1066
1067	tcg_out_opc_srli_d(s, TCG_REG_TMP2, addr_reg,
1068	TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
1069	tcg_out_opc_and(s, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP0);
1070	tcg_out_opc_add_d(s, TCG_REG_TMP2, TCG_REG_TMP2, TCG_REG_TMP1);
1071
1072	/* Load the tlb comparator and the addend. */
1073	QEMU_BUILD_BUG_ON(HOST_BIG_ENDIAN);
1074	tcg_out_ld(s, addr_type, TCG_REG_TMP0, TCG_REG_TMP2,
1075	is_ld ? offsetof(CPUTLBEntry, addr_read)
1076	: offsetof(CPUTLBEntry, addr_write));
1077	tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP2, TCG_REG_TMP2,
1078	offsetof(CPUTLBEntry, addend));
1079
1080	/*
1081	* For aligned accesses, we check the first byte and include the
1082	* alignment bits within the address. For unaligned access, we
1083	* check that we don't cross pages using the address of the last
1084	* byte of the access.
1085	*/
1086	if (a_bits < s_bits) {
1087	unsigned a_mask = (1u << a_bits) - 1;
1088	unsigned s_mask = (1u << s_bits) - 1;
1089	tcg_out_addi(s, addr_type, TCG_REG_TMP1, addr_reg, s_mask - a_mask);
1090	} else {
1091	tcg_out_mov(s, addr_type, TCG_REG_TMP1, addr_reg);
1092	}
1093	tcg_out_opc_bstrins_d(s, TCG_REG_TMP1, TCG_REG_ZERO,
1094	a_bits, TARGET_PAGE_BITS - 1);
1095
1096	/* Compare masked address with the TLB entry. */
1097	ldst->label_ptr[0] = s->code_ptr;
1098	tcg_out_opc_bne(s, TCG_REG_TMP0, TCG_REG_TMP1, 0);
1099
1100	h->index = TCG_REG_TMP2;
1101	} else {
1102	if (a_bits) {
1103	ldst = new_ldst_label(s);
1104
1105	ldst->is_ld = is_ld;
1106	ldst->oi = oi;
1107	ldst->addr_reg = addr_reg;
1108
1109	/*
1110	* Without micro-architecture details, we don't know which of
1111	* bstrpick or andi is faster, so use bstrpick as it's not
1112	* constrained by imm field width. Not to say alignments >= 2^12
1113	* are going to happen any time soon.
1114	*/
1115	tcg_out_opc_bstrpick_d(s, TCG_REG_TMP1, addr_reg, 0, a_bits - 1);
1116
1117	ldst->label_ptr[0] = s->code_ptr;
1118	tcg_out_opc_bne(s, TCG_REG_TMP1, TCG_REG_ZERO, 0);
1119	}
1120
1121	h->index = guest_base ? TCG_GUEST_BASE_REG : TCG_REG_ZERO;
1122	}
1123
1124	if (addr_type == TCG_TYPE_I32) {
1125	h->base = TCG_REG_TMP0;
1126	tcg_out_ext32u(s, h->base, addr_reg);
1127	} else {
1128	h->base = addr_reg;
1129	}
1130
1131	return ldst;
1132	}
1133
1134	static void tcg_out_qemu_ld_indexed(TCGContext *s, MemOp opc, TCGType type,
1135	TCGReg rd, HostAddress h)
1136	{
1137	/* Byte swapping is left to middle-end expansion. */
1138	tcg_debug_assert((opc & MO_BSWAP) == 0);
1139
1140	switch (opc & MO_SSIZE) {
1141	case MO_UB:
1142	tcg_out_opc_ldx_bu(s, rd, h.base, h.index);
1143	break;
1144	case MO_SB:
1145	tcg_out_opc_ldx_b(s, rd, h.base, h.index);
1146	break;
1147	case MO_UW:
1148	tcg_out_opc_ldx_hu(s, rd, h.base, h.index);
1149	break;
1150	case MO_SW:
1151	tcg_out_opc_ldx_h(s, rd, h.base, h.index);
1152	break;
1153	case MO_UL:
1154	if (type == TCG_TYPE_I64) {
1155	tcg_out_opc_ldx_wu(s, rd, h.base, h.index);
1156	break;
1157	}
1158	/* fallthrough */
1159	case MO_SL:
1160	tcg_out_opc_ldx_w(s, rd, h.base, h.index);
1161	break;
1162	case MO_UQ:
1163	tcg_out_opc_ldx_d(s, rd, h.base, h.index);
1164	break;
1165	default:
1166	g_assert_not_reached();
1167	}
1168	}
1169
1170	static void tgen_qemu_ld(TCGContext *s, TCGType type, TCGReg data_reg,
1171	TCGReg addr_reg, MemOpIdx oi)
1172	{
1173	TCGLabelQemuLdst *ldst;
1174	HostAddress h;
1175
1176	ldst = prepare_host_addr(s, &h, addr_reg, oi, true);
1177	tcg_out_qemu_ld_indexed(s, get_memop(oi), type, data_reg, h);
1178
1179	if (ldst) {
1180	ldst->type = type;
1181	ldst->datalo_reg = data_reg;
1182	ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
1183	}
1184	}
1185
1186	static const TCGOutOpQemuLdSt outop_qemu_ld = {
1187	.base.static_constraint = C_O1_I1(r, r),
1188	.out = tgen_qemu_ld,
1189	};
1190
1191	static void tcg_out_qemu_st_indexed(TCGContext *s, MemOp opc,
1192	TCGReg rd, HostAddress h)
1193	{
1194	/* Byte swapping is left to middle-end expansion. */
1195	tcg_debug_assert((opc & MO_BSWAP) == 0);
1196
1197	switch (opc & MO_SIZE) {
1198	case MO_8:
1199	tcg_out_opc_stx_b(s, rd, h.base, h.index);
1200	break;
1201	case MO_16:
1202	tcg_out_opc_stx_h(s, rd, h.base, h.index);
1203	break;
1204	case MO_32:
1205	tcg_out_opc_stx_w(s, rd, h.base, h.index);
1206	break;
1207	case MO_64:
1208	tcg_out_opc_stx_d(s, rd, h.base, h.index);
1209	break;
1210	default:
1211	g_assert_not_reached();
1212	}
1213	}
1214
1215	static void tgen_qemu_st(TCGContext *s, TCGType type, TCGReg data_reg,
1216	TCGReg addr_reg, MemOpIdx oi)
1217	{
1218	TCGLabelQemuLdst *ldst;
1219	HostAddress h;
1220
1221	ldst = prepare_host_addr(s, &h, addr_reg, oi, false);
1222	tcg_out_qemu_st_indexed(s, get_memop(oi), data_reg, h);
1223
1224	if (ldst) {
1225	ldst->type = type;
1226	ldst->datalo_reg = data_reg;
1227	ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
1228	}
1229	}
1230
1231	static const TCGOutOpQemuLdSt outop_qemu_st = {
1232	.base.static_constraint = C_O0_I2(rz, r),
1233	.out = tgen_qemu_st,
1234	};
1235
1236	static void tcg_out_qemu_ldst_i128(TCGContext *s, TCGReg data_lo, TCGReg data_hi,
1237	TCGReg addr_reg, MemOpIdx oi, bool is_ld)
1238	{
1239	TCGLabelQemuLdst *ldst;
1240	HostAddress h;
1241
1242	ldst = prepare_host_addr(s, &h, addr_reg, oi, is_ld);
1243
1244	if (h.aa.atom == MO_128) {
1245	/*
1246	* Use VLDX/VSTX when 128-bit atomicity is required.
1247	* If address is aligned to 16-bytes, the 128-bit load/store is atomic.
1248	*/
1249	if (is_ld) {
1250	tcg_out_opc_vldx(s, TCG_VEC_TMP0, h.base, h.index);
1251	tcg_out_opc_vpickve2gr_d(s, data_lo, TCG_VEC_TMP0, 0);
1252	tcg_out_opc_vpickve2gr_d(s, data_hi, TCG_VEC_TMP0, 1);
1253	} else {
1254	tcg_out_opc_vinsgr2vr_d(s, TCG_VEC_TMP0, data_lo, 0);
1255	tcg_out_opc_vinsgr2vr_d(s, TCG_VEC_TMP0, data_hi, 1);
1256	tcg_out_opc_vstx(s, TCG_VEC_TMP0, h.base, h.index);
1257	}
1258	} else {
1259	/* Otherwise use a pair of LD/ST. */
1260	TCGReg base = h.base;
1261	if (h.index != TCG_REG_ZERO) {
1262	base = TCG_REG_TMP0;
1263	tcg_out_opc_add_d(s, base, h.base, h.index);
1264	}
1265	if (is_ld) {
1266	tcg_debug_assert(base != data_lo);
1267	tcg_out_opc_ld_d(s, data_lo, base, 0);
1268	tcg_out_opc_ld_d(s, data_hi, base, 8);
1269	} else {
1270	tcg_out_opc_st_d(s, data_lo, base, 0);
1271	tcg_out_opc_st_d(s, data_hi, base, 8);
1272	}
1273	}
1274
1275	if (ldst) {
1276	ldst->type = TCG_TYPE_I128;
1277	ldst->datalo_reg = data_lo;
1278	ldst->datahi_reg = data_hi;
1279	ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
1280	}
1281	}
1282
1283	static void tgen_qemu_ld2(TCGContext *s, TCGType type, TCGReg datalo,
1284	TCGReg datahi, TCGReg addr_reg, MemOpIdx oi)
1285	{
1286	tcg_out_qemu_ldst_i128(s, datalo, datahi, addr_reg, oi, true);
1287	}
1288
1289	static const TCGOutOpQemuLdSt2 outop_qemu_ld2 = {
1290	.base.static_constraint = C_N2_I1(r, r, r),
1291	.out = tgen_qemu_ld2,
1292	};
1293
1294	static void tgen_qemu_st2(TCGContext *s, TCGType type, TCGReg datalo,
1295	TCGReg datahi, TCGReg addr_reg, MemOpIdx oi)
1296	{
1297	tcg_out_qemu_ldst_i128(s, datalo, datahi, addr_reg, oi, false);
1298	}
1299
1300	static const TCGOutOpQemuLdSt2 outop_qemu_st2 = {
1301	.base.static_constraint = C_O0_I3(r, r, r),
1302	.out = tgen_qemu_st2,
1303	};
1304
1305	/*
1306	* Entry-points
1307	*/
1308
1309	static const tcg_insn_unit *tb_ret_addr;
1310
1311	static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
1312	{
1313	/* Reuse the zeroing that exists for goto_ptr. */
1314	if (a0 == 0) {
1315	tcg_out_call_int(s, tcg_code_gen_epilogue, true);
1316	} else {
1317	tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, a0);
1318	tcg_out_call_int(s, tb_ret_addr, true);
1319	}
1320	}
1321
1322	static void tcg_out_goto_tb(TCGContext *s, int which)
1323	{
1324	/*
1325	* Direct branch, or load indirect address, to be patched
1326	* by tb_target_set_jmp_target. Check indirect load offset
1327	* in range early, regardless of direct branch distance,
1328	* via assert within tcg_out_opc_pcaddu2i.
1329	*/
1330	uintptr_t i_addr = get_jmp_target_addr(s, which);
1331	intptr_t i_disp = tcg_pcrel_diff(s, (void *)i_addr);
1332
1333	set_jmp_insn_offset(s, which);
1334	tcg_out_opc_pcaddu2i(s, TCG_REG_TMP0, i_disp >> 2);
1335
1336	/* Finish the load and indirect branch. */
1337	tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP0, TCG_REG_TMP0, 0);
1338	tcg_out_opc_jirl(s, TCG_REG_ZERO, TCG_REG_TMP0, 0);
1339	set_jmp_reset_offset(s, which);
1340	}
1341
1342	static void tcg_out_goto_ptr(TCGContext *s, TCGReg a0)
1343	{
1344	tcg_out_opc_jirl(s, TCG_REG_ZERO, a0, 0);
1345	}
1346
1347	void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
1348	uintptr_t jmp_rx, uintptr_t jmp_rw)
1349	{
1350	uintptr_t d_addr = tb->jmp_target_addr[n];
1351	ptrdiff_t d_disp = (ptrdiff_t)(d_addr - jmp_rx) >> 2;
1352	tcg_insn_unit insn;
1353
1354	/* Either directly branch, or load slot address for indirect branch. */
1355	if (d_disp == sextreg(d_disp, 0, 26)) {
1356	insn = encode_sd10k16_insn(OPC_B, d_disp);
1357	} else {
1358	uintptr_t i_addr = (uintptr_t)&tb->jmp_target_addr[n];
1359	intptr_t i_disp = i_addr - jmp_rx;
1360	insn = encode_dsj20_insn(OPC_PCADDU2I, TCG_REG_TMP0, i_disp >> 2);
1361	}
1362
1363	qatomic_set((tcg_insn_unit *)jmp_rw, insn);
1364	flush_idcache_range(jmp_rx, jmp_rw, 4);
1365	}
1366
1367
1368	static void tgen_add(TCGContext *s, TCGType type,
1369	TCGReg a0, TCGReg a1, TCGReg a2)
1370	{
1371	if (type == TCG_TYPE_I32) {
1372	tcg_out_opc_add_w(s, a0, a1, a2);
1373	} else {
1374	tcg_out_opc_add_d(s, a0, a1, a2);
1375	}
1376	}
1377
1378	static const TCGOutOpBinary outop_add = {
1379	.base.static_constraint = C_O1_I2(r, r, rJ),
1380	.out_rrr = tgen_add,
1381	.out_rri = tcg_out_addi,
1382	};
1383
1384	static const TCGOutOpBinary outop_addco = {
1385	.base.static_constraint = C_NotImplemented,
1386	};
1387
1388	static const TCGOutOpAddSubCarry outop_addci = {
1389	.base.static_constraint = C_NotImplemented,
1390	};
1391
1392	static const TCGOutOpBinary outop_addcio = {
1393	.base.static_constraint = C_NotImplemented,
1394	};
1395
1396	static void tcg_out_set_carry(TCGContext *s)
1397	{
1398	g_assert_not_reached();
1399	}
1400
1401	static void tgen_and(TCGContext *s, TCGType type,
1402	TCGReg a0, TCGReg a1, TCGReg a2)
1403	{
1404	tcg_out_opc_and(s, a0, a1, a2);
1405	}
1406
1407	static void tgen_andi(TCGContext *s, TCGType type,
1408	TCGReg a0, TCGReg a1, tcg_target_long a2)
1409	{
1410	tcg_out_opc_andi(s, a0, a1, a2);
1411	}
1412
1413	static const TCGOutOpBinary outop_and = {
1414	.base.static_constraint = C_O1_I2(r, r, rU),
1415	.out_rrr = tgen_and,
1416	.out_rri = tgen_andi,
1417	};
1418
1419	static void tgen_andc(TCGContext *s, TCGType type,
1420	TCGReg a0, TCGReg a1, TCGReg a2)
1421	{
1422	tcg_out_opc_andn(s, a0, a1, a2);
1423	}
1424
1425	static const TCGOutOpBinary outop_andc = {
1426	.base.static_constraint = C_O1_I2(r, r, r),
1427	.out_rrr = tgen_andc,
1428	};
1429
1430	static void tgen_clzi(TCGContext *s, TCGType type,
1431	TCGReg a0, TCGReg a1, tcg_target_long a2)
1432	{
1433	/* a2 is constrained to exactly the type width. */
1434	if (type == TCG_TYPE_I32) {
1435	tcg_out_opc_clz_w(s, a0, a1);
1436	} else {
1437	tcg_out_opc_clz_d(s, a0, a1);
1438	}
1439	}
1440
1441	static void tgen_clz(TCGContext *s, TCGType type,
1442	TCGReg a0, TCGReg a1, TCGReg a2)
1443	{
1444	tgen_clzi(s, type, TCG_REG_TMP0, a1, /* ignored */ 0);
1445	/* a0 = a1 ? REG_TMP0 : a2 */
1446	tcg_out_opc_maskeqz(s, TCG_REG_TMP0, TCG_REG_TMP0, a1);
1447	tcg_out_opc_masknez(s, a0, a2, a1);
1448	tcg_out_opc_or(s, a0, a0, TCG_REG_TMP0);
1449	}
1450
1451	static const TCGOutOpBinary outop_clz = {
1452	.base.static_constraint = C_O1_I2(r, r, rW),
1453	.out_rrr = tgen_clz,
1454	.out_rri = tgen_clzi,
1455	};
1456
1457	static const TCGOutOpUnary outop_ctpop = {
1458	.base.static_constraint = C_NotImplemented,
1459	};
1460
1461	static void tgen_ctzi(TCGContext *s, TCGType type,
1462	TCGReg a0, TCGReg a1, tcg_target_long a2)
1463	{
1464	/* a2 is constrained to exactly the type width. */
1465	if (type == TCG_TYPE_I32) {
1466	tcg_out_opc_ctz_w(s, a0, a1);
1467	} else {
1468	tcg_out_opc_ctz_d(s, a0, a1);
1469	}
1470	}
1471
1472	static void tgen_ctz(TCGContext *s, TCGType type,
1473	TCGReg a0, TCGReg a1, TCGReg a2)
1474	{
1475	tgen_ctzi(s, type, TCG_REG_TMP0, a1, /* ignored */ 0);
1476	/* a0 = a1 ? REG_TMP0 : a2 */
1477	tcg_out_opc_maskeqz(s, TCG_REG_TMP0, TCG_REG_TMP0, a1);
1478	tcg_out_opc_masknez(s, a0, a2, a1);
1479	tcg_out_opc_or(s, a0, a0, TCG_REG_TMP0);
1480	}
1481
1482	static const TCGOutOpBinary outop_ctz = {
1483	.base.static_constraint = C_O1_I2(r, r, rW),
1484	.out_rrr = tgen_ctz,
1485	.out_rri = tgen_ctzi,
1486	};
1487
1488	static void tgen_divs(TCGContext *s, TCGType type,
1489	TCGReg a0, TCGReg a1, TCGReg a2)
1490	{
1491	if (type == TCG_TYPE_I32) {
1492	tcg_out_opc_div_w(s, a0, a1, a2);
1493	} else {
1494	tcg_out_opc_div_d(s, a0, a1, a2);
1495	}
1496	}
1497
1498	static const TCGOutOpBinary outop_divs = {
1499	.base.static_constraint = C_O1_I2(r, r, r),
1500	.out_rrr = tgen_divs,
1501	};
1502
1503	static const TCGOutOpDivRem outop_divs2 = {
1504	.base.static_constraint = C_NotImplemented,
1505	};
1506
1507	static void tgen_divu(TCGContext *s, TCGType type,
1508	TCGReg a0, TCGReg a1, TCGReg a2)
1509	{
1510	if (type == TCG_TYPE_I32) {
1511	tcg_out_opc_div_wu(s, a0, a1, a2);
1512	} else {
1513	tcg_out_opc_div_du(s, a0, a1, a2);
1514	}
1515	}
1516
1517	static const TCGOutOpBinary outop_divu = {
1518	.base.static_constraint = C_O1_I2(r, r, r),
1519	.out_rrr = tgen_divu,
1520	};
1521
1522	static const TCGOutOpDivRem outop_divu2 = {
1523	.base.static_constraint = C_NotImplemented,
1524	};
1525
1526	static const TCGOutOpBinary outop_eqv = {
1527	.base.static_constraint = C_NotImplemented,
1528	};
1529
1530	static void tgen_extrh_i64_i32(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1)
1531	{
1532	tcg_out_opc_srai_d(s, a0, a1, 32);
1533	}
1534
1535	static const TCGOutOpUnary outop_extrh_i64_i32 = {
1536	.base.static_constraint = C_O1_I1(r, r),
1537	.out_rr = tgen_extrh_i64_i32,
1538	};
1539
1540	static void tgen_mul(TCGContext *s, TCGType type,
1541	TCGReg a0, TCGReg a1, TCGReg a2)
1542	{
1543	if (type == TCG_TYPE_I32) {
1544	tcg_out_opc_mul_w(s, a0, a1, a2);
1545	} else {
1546	tcg_out_opc_mul_d(s, a0, a1, a2);
1547	}
1548	}
1549
1550	static const TCGOutOpBinary outop_mul = {
1551	.base.static_constraint = C_O1_I2(r, r, r),
1552	.out_rrr = tgen_mul,
1553	};
1554
1555	static const TCGOutOpMul2 outop_muls2 = {
1556	.base.static_constraint = C_NotImplemented,
1557	};
1558
1559	static void tgen_mulsh(TCGContext *s, TCGType type,
1560	TCGReg a0, TCGReg a1, TCGReg a2)
1561	{
1562	if (type == TCG_TYPE_I32) {
1563	tcg_out_opc_mulh_w(s, a0, a1, a2);
1564	} else {
1565	tcg_out_opc_mulh_d(s, a0, a1, a2);
1566	}
1567	}
1568
1569	static const TCGOutOpBinary outop_mulsh = {
1570	.base.static_constraint = C_O1_I2(r, r, r),
1571	.out_rrr = tgen_mulsh,
1572	};
1573
1574	static const TCGOutOpMul2 outop_mulu2 = {
1575	.base.static_constraint = C_NotImplemented,
1576	};
1577
1578	static void tgen_muluh(TCGContext *s, TCGType type,
1579	TCGReg a0, TCGReg a1, TCGReg a2)
1580	{
1581	if (type == TCG_TYPE_I32) {
1582	tcg_out_opc_mulh_wu(s, a0, a1, a2);
1583	} else {
1584	tcg_out_opc_mulh_du(s, a0, a1, a2);
1585	}
1586	}
1587
1588	static const TCGOutOpBinary outop_muluh = {
1589	.base.static_constraint = C_O1_I2(r, r, r),
1590	.out_rrr = tgen_muluh,
1591	};
1592
1593	static const TCGOutOpBinary outop_nand = {
1594	.base.static_constraint = C_NotImplemented,
1595	};
1596
1597	static void tgen_nor(TCGContext *s, TCGType type,
1598	TCGReg a0, TCGReg a1, TCGReg a2)
1599	{
1600	tcg_out_opc_nor(s, a0, a1, a2);
1601	}
1602
1603	static const TCGOutOpBinary outop_nor = {
1604	.base.static_constraint = C_O1_I2(r, r, r),
1605	.out_rrr = tgen_nor,
1606	};
1607
1608	static void tgen_or(TCGContext *s, TCGType type,
1609	TCGReg a0, TCGReg a1, TCGReg a2)
1610	{
1611	tcg_out_opc_or(s, a0, a1, a2);
1612	}
1613
1614	static void tgen_ori(TCGContext *s, TCGType type,
1615	TCGReg a0, TCGReg a1, tcg_target_long a2)
1616	{
1617	tcg_out_opc_ori(s, a0, a1, a2);
1618	}
1619
1620	static const TCGOutOpBinary outop_or = {
1621	.base.static_constraint = C_O1_I2(r, r, rU),
1622	.out_rrr = tgen_or,
1623	.out_rri = tgen_ori,
1624	};
1625
1626	static void tgen_orc(TCGContext *s, TCGType type,
1627	TCGReg a0, TCGReg a1, TCGReg a2)
1628	{
1629	tcg_out_opc_orn(s, a0, a1, a2);
1630	}
1631
1632	static const TCGOutOpBinary outop_orc = {
1633	.base.static_constraint = C_O1_I2(r, r, r),
1634	.out_rrr = tgen_orc,
1635	};
1636
1637	static void tgen_rems(TCGContext *s, TCGType type,
1638	TCGReg a0, TCGReg a1, TCGReg a2)
1639	{
1640	if (type == TCG_TYPE_I32) {
1641	tcg_out_opc_mod_w(s, a0, a1, a2);
1642	} else {
1643	tcg_out_opc_mod_d(s, a0, a1, a2);
1644	}
1645	}
1646
1647	static const TCGOutOpBinary outop_rems = {
1648	.base.static_constraint = C_O1_I2(r, r, r),
1649	.out_rrr = tgen_rems,
1650	};
1651
1652	static void tgen_remu(TCGContext *s, TCGType type,
1653	TCGReg a0, TCGReg a1, TCGReg a2)
1654	{
1655	if (type == TCG_TYPE_I32) {
1656	tcg_out_opc_mod_wu(s, a0, a1, a2);
1657	} else {
1658	tcg_out_opc_mod_du(s, a0, a1, a2);
1659	}
1660	}
1661
1662	static const TCGOutOpBinary outop_remu = {
1663	.base.static_constraint = C_O1_I2(r, r, r),
1664	.out_rrr = tgen_remu,
1665	};
1666
1667	static const TCGOutOpBinary outop_rotl = {
1668	.base.static_constraint = C_NotImplemented,
1669	};
1670
1671	static void tgen_rotr(TCGContext *s, TCGType type,
1672	TCGReg a0, TCGReg a1, TCGReg a2)
1673	{
1674	if (type == TCG_TYPE_I32) {
1675	tcg_out_opc_rotr_w(s, a0, a1, a2);
1676	} else {
1677	tcg_out_opc_rotr_d(s, a0, a1, a2);
1678	}
1679	}
1680
1681	static void tgen_rotri(TCGContext *s, TCGType type,
1682	TCGReg a0, TCGReg a1, tcg_target_long a2)
1683	{
1684	if (type == TCG_TYPE_I32) {
1685	tcg_out_opc_rotri_w(s, a0, a1, a2 & 0x1f);
1686	} else {
1687	tcg_out_opc_rotri_d(s, a0, a1, a2 & 0x3f);
1688	}
1689	}
1690
1691	static const TCGOutOpBinary outop_rotr = {
1692	.base.static_constraint = C_O1_I2(r, r, ri),
1693	.out_rrr = tgen_rotr,
1694	.out_rri = tgen_rotri,
1695	};
1696
1697	static void tgen_sar(TCGContext *s, TCGType type,
1698	TCGReg a0, TCGReg a1, TCGReg a2)
1699	{
1700	if (type == TCG_TYPE_I32) {
1701	tcg_out_opc_sra_w(s, a0, a1, a2);
1702	} else {
1703	tcg_out_opc_sra_d(s, a0, a1, a2);
1704	}
1705	}
1706
1707	static void tgen_sari(TCGContext *s, TCGType type,
1708	TCGReg a0, TCGReg a1, tcg_target_long a2)
1709	{
1710	if (type == TCG_TYPE_I32) {
1711	tcg_out_opc_srai_w(s, a0, a1, a2 & 0x1f);
1712	} else {
1713	tcg_out_opc_srai_d(s, a0, a1, a2 & 0x3f);
1714	}
1715	}
1716
1717	static const TCGOutOpBinary outop_sar = {
1718	.base.static_constraint = C_O1_I2(r, r, ri),
1719	.out_rrr = tgen_sar,
1720	.out_rri = tgen_sari,
1721	};
1722
1723	static void tgen_shl(TCGContext *s, TCGType type,
1724	TCGReg a0, TCGReg a1, TCGReg a2)
1725	{
1726	if (type == TCG_TYPE_I32) {
1727	tcg_out_opc_sll_w(s, a0, a1, a2);
1728	} else {
1729	tcg_out_opc_sll_d(s, a0, a1, a2);
1730	}
1731	}
1732
1733	static void tgen_shli(TCGContext *s, TCGType type,
1734	TCGReg a0, TCGReg a1, tcg_target_long a2)
1735	{
1736	if (type == TCG_TYPE_I32) {
1737	tcg_out_opc_slli_w(s, a0, a1, a2 & 0x1f);
1738	} else {
1739	tcg_out_opc_slli_d(s, a0, a1, a2 & 0x3f);
1740	}
1741	}
1742
1743	static const TCGOutOpBinary outop_shl = {
1744	.base.static_constraint = C_O1_I2(r, r, ri),
1745	.out_rrr = tgen_shl,
1746	.out_rri = tgen_shli,
1747	};
1748
1749	static void tgen_shr(TCGContext *s, TCGType type,
1750	TCGReg a0, TCGReg a1, TCGReg a2)
1751	{
1752	if (type == TCG_TYPE_I32) {
1753	tcg_out_opc_srl_w(s, a0, a1, a2);
1754	} else {
1755	tcg_out_opc_srl_d(s, a0, a1, a2);
1756	}
1757	}
1758
1759	static void tgen_shri(TCGContext *s, TCGType type,
1760	TCGReg a0, TCGReg a1, tcg_target_long a2)
1761	{
1762	if (type == TCG_TYPE_I32) {
1763	tcg_out_opc_srli_w(s, a0, a1, a2 & 0x1f);
1764	} else {
1765	tcg_out_opc_srli_d(s, a0, a1, a2 & 0x3f);
1766	}
1767	}
1768
1769	static const TCGOutOpBinary outop_shr = {
1770	.base.static_constraint = C_O1_I2(r, r, ri),
1771	.out_rrr = tgen_shr,
1772	.out_rri = tgen_shri,
1773	};
1774
1775	static void tgen_sub(TCGContext *s, TCGType type,
1776	TCGReg a0, TCGReg a1, TCGReg a2)
1777	{
1778	if (type == TCG_TYPE_I32) {
1779	tcg_out_opc_sub_w(s, a0, a1, a2);
1780	} else {
1781	tcg_out_opc_sub_d(s, a0, a1, a2);
1782	}
1783	}
1784
1785	static const TCGOutOpSubtract outop_sub = {
1786	.base.static_constraint = C_O1_I2(r, r, r),
1787	.out_rrr = tgen_sub,
1788	};
1789
1790	static const TCGOutOpAddSubCarry outop_subbo = {
1791	.base.static_constraint = C_NotImplemented,
1792	};
1793
1794	static const TCGOutOpAddSubCarry outop_subbi = {
1795	.base.static_constraint = C_NotImplemented,
1796	};
1797
1798	static const TCGOutOpAddSubCarry outop_subbio = {
1799	.base.static_constraint = C_NotImplemented,
1800	};
1801
1802	static void tcg_out_set_borrow(TCGContext *s)
1803	{
1804	g_assert_not_reached();
1805	}
1806
1807	static const TCGOutOpBinary outop_smax = {
1808	.base.static_constraint = C_NotImplemented,
1809	};
1810
1811	static const TCGOutOpBinary outop_smin = {
1812	.base.static_constraint = C_NotImplemented,
1813	};
1814
1815	static const TCGOutOpBinary outop_umax = {
1816	.base.static_constraint = C_NotImplemented,
1817	};
1818
1819	static const TCGOutOpBinary outop_umin = {
1820	.base.static_constraint = C_NotImplemented,
1821	};
1822
1823	static void tgen_xor(TCGContext *s, TCGType type,
1824	TCGReg a0, TCGReg a1, TCGReg a2)
1825	{
1826	tcg_out_opc_xor(s, a0, a1, a2);
1827	}
1828
1829	static void tgen_xori(TCGContext *s, TCGType type,
1830	TCGReg a0, TCGReg a1, tcg_target_long a2)
1831	{
1832	tcg_out_opc_xori(s, a0, a1, a2);
1833	}
1834
1835	static const TCGOutOpBinary outop_xor = {
1836	.base.static_constraint = C_O1_I2(r, r, rU),
1837	.out_rrr = tgen_xor,
1838	.out_rri = tgen_xori,
1839	};
1840
1841	static void tgen_bswap16(TCGContext *s, TCGType type,
1842	TCGReg a0, TCGReg a1, unsigned flags)
1843	{
1844	tcg_out_opc_revb_2h(s, a0, a1);
1845	if (flags & TCG_BSWAP_OS) {
1846	tcg_out_ext16s(s, TCG_TYPE_REG, a0, a0);
1847	} else if ((flags & (TCG_BSWAP_IZ \| TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
1848	tcg_out_ext16u(s, a0, a0);
1849	}
1850	}
1851
1852	static const TCGOutOpBswap outop_bswap16 = {
1853	.base.static_constraint = C_O1_I1(r, r),
1854	.out_rr = tgen_bswap16,
1855	};
1856
1857	static void tgen_bswap32(TCGContext *s, TCGType type,
1858	TCGReg a0, TCGReg a1, unsigned flags)
1859	{
1860	tcg_out_opc_revb_2w(s, a0, a1);
1861
1862	/* All 32-bit values are computed sign-extended in the register. */
1863	if (type == TCG_TYPE_I32 \|\| (flags & TCG_BSWAP_OS)) {
1864	tcg_out_ext32s(s, a0, a0);
1865	} else if ((flags & (TCG_BSWAP_IZ \| TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
1866	tcg_out_ext32u(s, a0, a0);
1867	}
1868	}
1869
1870	static const TCGOutOpBswap outop_bswap32 = {
1871	.base.static_constraint = C_O1_I1(r, r),
1872	.out_rr = tgen_bswap32,
1873	};
1874
1875	static void tgen_bswap64(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
1876	{
1877	tcg_out_opc_revb_d(s, a0, a1);
1878	}
1879
1880	static const TCGOutOpUnary outop_bswap64 = {
1881	.base.static_constraint = C_O1_I1(r, r),
1882	.out_rr = tgen_bswap64,
1883	};
1884
1885	static void tgen_revbit8(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
1886	{
1887	if (type == TCG_TYPE_I32) {
1888	tcg_out_opc_revbit_4b(s, a0, a1);
1889	} else {
1890	tcg_out_opc_revbit_8b(s, a0, a1);
1891	}
1892	}
1893
1894	static const TCGOutOpUnary outop_revbit8 = {
1895	.base.static_constraint = C_O1_I1(r, r),
1896	.out_rr = tgen_revbit8,
1897	};
1898
1899	static void tgen_revbit32(TCGContext *s, TCGType type,
1900	TCGReg a0, TCGReg a1, unsigned flags)
1901	{
1902	tcg_out_opc_revbit_w(s, a0, a1);
1903
1904	/* All 32-bit values are computed sign-extended in the register. */
1905	if (type == TCG_TYPE_I64 && (flags & TCG_BSWAP_OZ)) {
1906	tcg_out_ext32u(s, a0, a0);
1907	}
1908	}
1909
1910	static const TCGOutOpBswap outop_revbit32 = {
1911	.base.static_constraint = C_O1_I1(r, r),
1912	.out_rr = tgen_revbit32,
1913	};
1914
1915	static void tgen_revbit64(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
1916	{
1917	tcg_out_opc_revbit_d(s, a0, a1);
1918	}
1919
1920	static const TCGOutOpUnary outop_revbit64 = {
1921	.base.static_constraint = C_O1_I1(r, r),
1922	.out_rr = tgen_revbit64,
1923	};
1924
1925	static void tgen_neg(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
1926	{
1927	tgen_sub(s, type, a0, TCG_REG_ZERO, a1);
1928	}
1929
1930	static const TCGOutOpUnary outop_neg = {
1931	.base.static_constraint = C_O1_I1(r, r),
1932	.out_rr = tgen_neg,
1933	};
1934
1935	static void tgen_not(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
1936	{
1937	tgen_nor(s, type, a0, a1, TCG_REG_ZERO);
1938	}
1939
1940	static const TCGOutOpUnary outop_not = {
1941	.base.static_constraint = C_O1_I1(r, r),
1942	.out_rr = tgen_not,
1943	};
1944
1945	static void tgen_deposit(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
1946	TCGReg a2, unsigned ofs, unsigned len)
1947	{
1948	if (type == TCG_TYPE_I32) {
1949	tcg_out_opc_bstrins_w(s, a0, a2, ofs, ofs + len - 1);
1950	} else {
1951	tcg_out_opc_bstrins_d(s, a0, a2, ofs, ofs + len - 1);
1952	}
1953	}
1954
1955	static const TCGOutOpDeposit outop_deposit = {
1956	.base.static_constraint = C_O1_I2(r, 0, rz),
1957	.out_rrr = tgen_deposit,
1958	};
1959
1960	static void tgen_extract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
1961	unsigned ofs, unsigned len)
1962	{
1963	if (ofs == 0 && len <= 12) {
1964	tcg_out_opc_andi(s, a0, a1, (1 << len) - 1);
1965	} else if (type == TCG_TYPE_I32) {
1966	tcg_out_opc_bstrpick_w(s, a0, a1, ofs, ofs + len - 1);
1967	} else {
1968	tcg_out_opc_bstrpick_d(s, a0, a1, ofs, ofs + len - 1);
1969	}
1970	}
1971
1972	static const TCGOutOpExtract outop_extract = {
1973	.base.static_constraint = C_O1_I1(r, r),
1974	.out_rr = tgen_extract,
1975	};
1976
1977	static void tgen_sextract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
1978	unsigned ofs, unsigned len)
1979	{
1980	if (ofs == 0) {
1981	switch (len) {
1982	case 8:
1983	tcg_out_ext8s(s, type, a0, a1);
1984	return;
1985	case 16:
1986	tcg_out_ext16s(s, type, a0, a1);
1987	return;
1988	case 32:
1989	tcg_out_ext32s(s, a0, a1);
1990	return;
1991	}
1992	} else if (ofs + len == 32) {
1993	tcg_out_opc_srai_w(s, a0, a1, ofs);
1994	return;
1995	}
1996	g_assert_not_reached();
1997	}
1998
1999	static const TCGOutOpExtract outop_sextract = {
2000	.base.static_constraint = C_O1_I1(r, r),
2001	.out_rr = tgen_sextract,
2002	};
2003
2004	static const TCGOutOpExtract2 outop_extract2 = {
2005	.base.static_constraint = C_NotImplemented,
2006	};
2007
2008	static void tgen_ld8u(TCGContext *s, TCGType type, TCGReg dest,
2009	TCGReg base, ptrdiff_t offset)
2010	{
2011	tcg_out_ldst(s, OPC_LD_BU, dest, base, offset);
2012	}
2013
2014	static const TCGOutOpLoad outop_ld8u = {
2015	.base.static_constraint = C_O1_I1(r, r),
2016	.out = tgen_ld8u,
2017	};
2018
2019	static void tgen_ld8s(TCGContext *s, TCGType type, TCGReg dest,
2020	TCGReg base, ptrdiff_t offset)
2021	{
2022	tcg_out_ldst(s, OPC_LD_B, dest, base, offset);
2023	}
2024
2025	static const TCGOutOpLoad outop_ld8s = {
2026	.base.static_constraint = C_O1_I1(r, r),
2027	.out = tgen_ld8s,
2028	};
2029
2030	static void tgen_ld16u(TCGContext *s, TCGType type, TCGReg dest,
2031	TCGReg base, ptrdiff_t offset)
2032	{
2033	tcg_out_ldst(s, OPC_LD_HU, dest, base, offset);
2034	}
2035
2036	static const TCGOutOpLoad outop_ld16u = {
2037	.base.static_constraint = C_O1_I1(r, r),
2038	.out = tgen_ld16u,
2039	};
2040
2041	static void tgen_ld16s(TCGContext *s, TCGType type, TCGReg dest,
2042	TCGReg base, ptrdiff_t offset)
2043	{
2044	tcg_out_ldst(s, OPC_LD_H, dest, base, offset);
2045	}
2046
2047	static const TCGOutOpLoad outop_ld16s = {
2048	.base.static_constraint = C_O1_I1(r, r),
2049	.out = tgen_ld16s,
2050	};
2051
2052	static void tgen_ld32u(TCGContext *s, TCGType type, TCGReg dest,
2053	TCGReg base, ptrdiff_t offset)
2054	{
2055	tcg_out_ldst(s, OPC_LD_WU, dest, base, offset);
2056	}
2057
2058	static const TCGOutOpLoad outop_ld32u = {
2059	.base.static_constraint = C_O1_I1(r, r),
2060	.out = tgen_ld32u,
2061	};
2062
2063	static void tgen_ld32s(TCGContext *s, TCGType type, TCGReg dest,
2064	TCGReg base, ptrdiff_t offset)
2065	{
2066	tcg_out_ldst(s, OPC_LD_W, dest, base, offset);
2067	}
2068
2069	static const TCGOutOpLoad outop_ld32s = {
2070	.base.static_constraint = C_O1_I1(r, r),
2071	.out = tgen_ld32s,
2072	};
2073
2074	static void tgen_st8_r(TCGContext *s, TCGType type, TCGReg data,
2075	TCGReg base, ptrdiff_t offset)
2076	{
2077	tcg_out_ldst(s, OPC_ST_B, data, base, offset);
2078	}
2079
2080	static const TCGOutOpStore outop_st8 = {
2081	.base.static_constraint = C_O0_I2(rz, r),
2082	.out_r = tgen_st8_r,
2083	};
2084
2085	static void tgen_st16_r(TCGContext *s, TCGType type, TCGReg data,
2086	TCGReg base, ptrdiff_t offset)
2087	{
2088	tcg_out_ldst(s, OPC_ST_H, data, base, offset);
2089	}
2090
2091	static const TCGOutOpStore outop_st16 = {
2092	.base.static_constraint = C_O0_I2(rz, r),
2093	.out_r = tgen_st16_r,
2094	};
2095
2096	static const TCGOutOpStore outop_st = {
2097	.base.static_constraint = C_O0_I2(rz, r),
2098	.out_r = tcg_out_st,
2099	};
2100
2101	static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
2102	TCGReg rd, TCGReg rs)
2103	{
2104	static const LoongArchInsn repl_insn[2][4] = {
2105	{ OPC_VREPLGR2VR_B, OPC_VREPLGR2VR_H,
2106	OPC_VREPLGR2VR_W, OPC_VREPLGR2VR_D },
2107	{ OPC_XVREPLGR2VR_B, OPC_XVREPLGR2VR_H,
2108	OPC_XVREPLGR2VR_W, OPC_XVREPLGR2VR_D },
2109	};
2110	bool lasx = type == TCG_TYPE_V256;
2111
2112	tcg_debug_assert(vece <= MO_64);
2113	tcg_out32(s, encode_vdj_insn(repl_insn[lasx][vece], rd, rs));
2114	return true;
2115	}
2116
2117	static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
2118	TCGReg r, TCGReg base, intptr_t offset)
2119	{
2120	bool lasx = type == TCG_TYPE_V256;
2121
2122	/* Handle imm overflow and division (vldrepl.d imm is divided by 8). */
2123	if (offset < -0x800 \|\| offset > 0x7ff \|\|
2124	(offset & ((1 << vece) - 1)) != 0) {
2125	tcg_out_addi(s, TCG_TYPE_I64, TCG_REG_TMP0, base, offset);
2126	base = TCG_REG_TMP0;
2127	offset = 0;
2128	}
2129	offset >>= vece;
2130
2131	switch (vece) {
2132	case MO_8:
2133	if (lasx) {
2134	tcg_out_opc_xvldrepl_b(s, r, base, offset);
2135	} else {
2136	tcg_out_opc_vldrepl_b(s, r, base, offset);
2137	}
2138	break;
2139	case MO_16:
2140	if (lasx) {
2141	tcg_out_opc_xvldrepl_h(s, r, base, offset);
2142	} else {
2143	tcg_out_opc_vldrepl_h(s, r, base, offset);
2144	}
2145	break;
2146	case MO_32:
2147	if (lasx) {
2148	tcg_out_opc_xvldrepl_w(s, r, base, offset);
2149	} else {
2150	tcg_out_opc_vldrepl_w(s, r, base, offset);
2151	}
2152	break;
2153	case MO_64:
2154	if (lasx) {
2155	tcg_out_opc_xvldrepl_d(s, r, base, offset);
2156	} else {
2157	tcg_out_opc_vldrepl_d(s, r, base, offset);
2158	}
2159	break;
2160	default:
2161	g_assert_not_reached();
2162	}
2163	return true;
2164	}
2165
2166	static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
2167	TCGReg rd, int64_t v64)
2168	{
2169	/* Try vldi if imm can fit */
2170	int64_t value = sextract64(v64, 0, 8 << vece);
2171	if (-0x200 <= value && value <= 0x1FF) {
2172	uint32_t imm = (vece << 10) \| ((uint32_t)v64 & 0x3FF);
2173
2174	if (type == TCG_TYPE_V256) {
2175	tcg_out_opc_xvldi(s, rd, imm);
2176	} else {
2177	tcg_out_opc_vldi(s, rd, imm);
2178	}
2179	return;
2180	}
2181
2182	/* TODO: vldi patterns when imm 12 is set */
2183
2184	tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP0, value);
2185	tcg_out_dup_vec(s, type, vece, rd, TCG_REG_TMP0);
2186	}
2187
2188	static void tcg_out_addsub_vec(TCGContext *s, bool lasx, unsigned vece,
2189	TCGArg a0, TCGArg a1, TCGArg a2,
2190	bool a2_is_const, bool is_add)
2191	{
2192	static const LoongArchInsn add_vec_insn[2][4] = {
2193	{ OPC_VADD_B, OPC_VADD_H, OPC_VADD_W, OPC_VADD_D },
2194	{ OPC_XVADD_B, OPC_XVADD_H, OPC_XVADD_W, OPC_XVADD_D },
2195	};
2196	static const LoongArchInsn add_vec_imm_insn[2][4] = {
2197	{ OPC_VADDI_BU, OPC_VADDI_HU, OPC_VADDI_WU, OPC_VADDI_DU },
2198	{ OPC_XVADDI_BU, OPC_XVADDI_HU, OPC_XVADDI_WU, OPC_XVADDI_DU },
2199	};
2200	static const LoongArchInsn sub_vec_insn[2][4] = {
2201	{ OPC_VSUB_B, OPC_VSUB_H, OPC_VSUB_W, OPC_VSUB_D },
2202	{ OPC_XVSUB_B, OPC_XVSUB_H, OPC_XVSUB_W, OPC_XVSUB_D },
2203	};
2204	static const LoongArchInsn sub_vec_imm_insn[2][4] = {
2205	{ OPC_VSUBI_BU, OPC_VSUBI_HU, OPC_VSUBI_WU, OPC_VSUBI_DU },
2206	{ OPC_XVSUBI_BU, OPC_XVSUBI_HU, OPC_XVSUBI_WU, OPC_XVSUBI_DU },
2207	};
2208	LoongArchInsn insn;
2209
2210	if (a2_is_const) {
2211	int64_t value = sextract64(a2, 0, 8 << vece);
2212
2213	if (!is_add) {
2214	value = -value;
2215	}
2216	if (value < 0) {
2217	insn = sub_vec_imm_insn[lasx][vece];
2218	value = -value;
2219	} else {
2220	insn = add_vec_imm_insn[lasx][vece];
2221	}
2222
2223	/* Constraint TCG_CT_CONST_VADD ensures validity. */
2224	tcg_debug_assert(0 <= value && value <= 0x1f);
2225
2226	tcg_out32(s, encode_vdvjuk5_insn(insn, a0, a1, value));
2227	return;
2228	}
2229
2230	if (is_add) {
2231	insn = add_vec_insn[lasx][vece];
2232	} else {
2233	insn = sub_vec_insn[lasx][vece];
2234	}
2235	tcg_out32(s, encode_vdvjvk_insn(insn, a0, a1, a2));
2236	}
2237
2238	static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
2239	unsigned vecl, unsigned vece,
2240	const TCGArg args[TCG_MAX_OP_ARGS],
2241	const int const_args[TCG_MAX_OP_ARGS])
2242	{
2243	TCGType type = vecl + TCG_TYPE_V64;
2244	bool lasx = type == TCG_TYPE_V256;
2245	TCGArg a0, a1, a2, a3;
2246	LoongArchInsn insn;
2247
2248	static const LoongArchInsn cmp_vec_insn[16][2][4] = {
2249	[TCG_COND_EQ] = {
2250	{ OPC_VSEQ_B, OPC_VSEQ_H, OPC_VSEQ_W, OPC_VSEQ_D },
2251	{ OPC_XVSEQ_B, OPC_XVSEQ_H, OPC_XVSEQ_W, OPC_XVSEQ_D },
2252	},
2253	[TCG_COND_LE] = {
2254	{ OPC_VSLE_B, OPC_VSLE_H, OPC_VSLE_W, OPC_VSLE_D },
2255	{ OPC_XVSLE_B, OPC_XVSLE_H, OPC_XVSLE_W, OPC_XVSLE_D },
2256	},
2257	[TCG_COND_LEU] = {
2258	{ OPC_VSLE_BU, OPC_VSLE_HU, OPC_VSLE_WU, OPC_VSLE_DU },
2259	{ OPC_XVSLE_BU, OPC_XVSLE_HU, OPC_XVSLE_WU, OPC_XVSLE_DU },
2260	},
2261	[TCG_COND_LT] = {
2262	{ OPC_VSLT_B, OPC_VSLT_H, OPC_VSLT_W, OPC_VSLT_D },
2263	{ OPC_XVSLT_B, OPC_XVSLT_H, OPC_XVSLT_W, OPC_XVSLT_D },
2264	},
2265	[TCG_COND_LTU] = {
2266	{ OPC_VSLT_BU, OPC_VSLT_HU, OPC_VSLT_WU, OPC_VSLT_DU },
2267	{ OPC_XVSLT_BU, OPC_XVSLT_HU, OPC_XVSLT_WU, OPC_XVSLT_DU },
2268	}
2269	};
2270	static const LoongArchInsn cmp_vec_imm_insn[16][2][4] = {
2271	[TCG_COND_EQ] = {
2272	{ OPC_VSEQI_B, OPC_VSEQI_H, OPC_VSEQI_W, OPC_VSEQI_D },
2273	{ OPC_XVSEQI_B, OPC_XVSEQI_H, OPC_XVSEQI_W, OPC_XVSEQI_D },
2274	},
2275	[TCG_COND_LE] = {
2276	{ OPC_VSLEI_B, OPC_VSLEI_H, OPC_VSLEI_W, OPC_VSLEI_D },
2277	{ OPC_XVSLEI_B, OPC_XVSLEI_H, OPC_XVSLEI_W, OPC_XVSLEI_D },
2278	},
2279	[TCG_COND_LEU] = {
2280	{ OPC_VSLEI_BU, OPC_VSLEI_HU, OPC_VSLEI_WU, OPC_VSLEI_DU },
2281	{ OPC_XVSLEI_BU, OPC_XVSLEI_HU, OPC_XVSLEI_WU, OPC_XVSLEI_DU },
2282	},
2283	[TCG_COND_LT] = {
2284	{ OPC_VSLTI_B, OPC_VSLTI_H, OPC_VSLTI_W, OPC_VSLTI_D },
2285	{ OPC_XVSLTI_B, OPC_XVSLTI_H, OPC_XVSLTI_W, OPC_XVSLTI_D },
2286	},
2287	[TCG_COND_LTU] = {
2288	{ OPC_VSLTI_BU, OPC_VSLTI_HU, OPC_VSLTI_WU, OPC_VSLTI_DU },
2289	{ OPC_XVSLTI_BU, OPC_XVSLTI_HU, OPC_XVSLTI_WU, OPC_XVSLTI_DU },
2290	}
2291	};
2292	static const LoongArchInsn neg_vec_insn[2][4] = {
2293	{ OPC_VNEG_B, OPC_VNEG_H, OPC_VNEG_W, OPC_VNEG_D },
2294	{ OPC_XVNEG_B, OPC_XVNEG_H, OPC_XVNEG_W, OPC_XVNEG_D },
2295	};
2296	static const LoongArchInsn mul_vec_insn[2][4] = {
2297	{ OPC_VMUL_B, OPC_VMUL_H, OPC_VMUL_W, OPC_VMUL_D },
2298	{ OPC_XVMUL_B, OPC_XVMUL_H, OPC_XVMUL_W, OPC_XVMUL_D },
2299	};
2300	static const LoongArchInsn smin_vec_insn[2][4] = {
2301	{ OPC_VMIN_B, OPC_VMIN_H, OPC_VMIN_W, OPC_VMIN_D },
2302	{ OPC_XVMIN_B, OPC_XVMIN_H, OPC_XVMIN_W, OPC_XVMIN_D },
2303	};
2304	static const LoongArchInsn umin_vec_insn[2][4] = {
2305	{ OPC_VMIN_BU, OPC_VMIN_HU, OPC_VMIN_WU, OPC_VMIN_DU },
2306	{ OPC_XVMIN_BU, OPC_XVMIN_HU, OPC_XVMIN_WU, OPC_XVMIN_DU },
2307	};
2308	static const LoongArchInsn smax_vec_insn[2][4] = {
2309	{ OPC_VMAX_B, OPC_VMAX_H, OPC_VMAX_W, OPC_VMAX_D },
2310	{ OPC_XVMAX_B, OPC_XVMAX_H, OPC_XVMAX_W, OPC_XVMAX_D },
2311	};
2312	static const LoongArchInsn umax_vec_insn[2][4] = {
2313	{ OPC_VMAX_BU, OPC_VMAX_HU, OPC_VMAX_WU, OPC_VMAX_DU },
2314	{ OPC_XVMAX_BU, OPC_XVMAX_HU, OPC_XVMAX_WU, OPC_XVMAX_DU },
2315	};
2316	static const LoongArchInsn ssadd_vec_insn[2][4] = {
2317	{ OPC_VSADD_B, OPC_VSADD_H, OPC_VSADD_W, OPC_VSADD_D },
2318	{ OPC_XVSADD_B, OPC_XVSADD_H, OPC_XVSADD_W, OPC_XVSADD_D },
2319	};
2320	static const LoongArchInsn usadd_vec_insn[2][4] = {
2321	{ OPC_VSADD_BU, OPC_VSADD_HU, OPC_VSADD_WU, OPC_VSADD_DU },
2322	{ OPC_XVSADD_BU, OPC_XVSADD_HU, OPC_XVSADD_WU, OPC_XVSADD_DU },
2323	};
2324	static const LoongArchInsn sssub_vec_insn[2][4] = {
2325	{ OPC_VSSUB_B, OPC_VSSUB_H, OPC_VSSUB_W, OPC_VSSUB_D },
2326	{ OPC_XVSSUB_B, OPC_XVSSUB_H, OPC_XVSSUB_W, OPC_XVSSUB_D },
2327	};
2328	static const LoongArchInsn ussub_vec_insn[2][4] = {
2329	{ OPC_VSSUB_BU, OPC_VSSUB_HU, OPC_VSSUB_WU, OPC_VSSUB_DU },
2330	{ OPC_XVSSUB_BU, OPC_XVSSUB_HU, OPC_XVSSUB_WU, OPC_XVSSUB_DU },
2331	};
2332	static const LoongArchInsn shlv_vec_insn[2][4] = {
2333	{ OPC_VSLL_B, OPC_VSLL_H, OPC_VSLL_W, OPC_VSLL_D },
2334	{ OPC_XVSLL_B, OPC_XVSLL_H, OPC_XVSLL_W, OPC_XVSLL_D },
2335	};
2336	static const LoongArchInsn shrv_vec_insn[2][4] = {
2337	{ OPC_VSRL_B, OPC_VSRL_H, OPC_VSRL_W, OPC_VSRL_D },
2338	{ OPC_XVSRL_B, OPC_XVSRL_H, OPC_XVSRL_W, OPC_XVSRL_D },
2339	};
2340	static const LoongArchInsn sarv_vec_insn[2][4] = {
2341	{ OPC_VSRA_B, OPC_VSRA_H, OPC_VSRA_W, OPC_VSRA_D },
2342	{ OPC_XVSRA_B, OPC_XVSRA_H, OPC_XVSRA_W, OPC_XVSRA_D },
2343	};
2344	static const LoongArchInsn shli_vec_insn[2][4] = {
2345	{ OPC_VSLLI_B, OPC_VSLLI_H, OPC_VSLLI_W, OPC_VSLLI_D },
2346	{ OPC_XVSLLI_B, OPC_XVSLLI_H, OPC_XVSLLI_W, OPC_XVSLLI_D },
2347	};
2348	static const LoongArchInsn shri_vec_insn[2][4] = {
2349	{ OPC_VSRLI_B, OPC_VSRLI_H, OPC_VSRLI_W, OPC_VSRLI_D },
2350	{ OPC_XVSRLI_B, OPC_XVSRLI_H, OPC_XVSRLI_W, OPC_XVSRLI_D },
2351	};
2352	static const LoongArchInsn sari_vec_insn[2][4] = {
2353	{ OPC_VSRAI_B, OPC_VSRAI_H, OPC_VSRAI_W, OPC_VSRAI_D },
2354	{ OPC_XVSRAI_B, OPC_XVSRAI_H, OPC_XVSRAI_W, OPC_XVSRAI_D },
2355	};
2356	static const LoongArchInsn rotrv_vec_insn[2][4] = {
2357	{ OPC_VROTR_B, OPC_VROTR_H, OPC_VROTR_W, OPC_VROTR_D },
2358	{ OPC_XVROTR_B, OPC_XVROTR_H, OPC_XVROTR_W, OPC_XVROTR_D },
2359	};
2360	static const LoongArchInsn rotri_vec_insn[2][4] = {
2361	{ OPC_VROTRI_B, OPC_VROTRI_H, OPC_VROTRI_W, OPC_VROTRI_D },
2362	{ OPC_XVROTRI_B, OPC_XVROTRI_H, OPC_XVROTRI_W, OPC_XVROTRI_D },
2363	};
2364
2365	a0 = args[0];
2366	a1 = args[1];
2367	a2 = args[2];
2368	a3 = args[3];
2369
2370	switch (opc) {
2371	case INDEX_op_st_vec:
2372	tcg_out_st(s, type, a0, a1, a2);
2373	break;
2374	case INDEX_op_ld_vec:
2375	tcg_out_ld(s, type, a0, a1, a2);
2376	break;
2377	case INDEX_op_and_vec:
2378	insn = lasx ? OPC_XVAND_V : OPC_VAND_V;
2379	goto vdvjvk;
2380	case INDEX_op_andc_vec:
2381	/*
2382	* vandn vd, vj, vk: vd = vk & ~vj
2383	* andc_vec vd, vj, vk: vd = vj & ~vk
2384	* vj and vk are swapped
2385	*/
2386	a1 = a2;
2387	a2 = args[1];
2388	insn = lasx ? OPC_XVANDN_V : OPC_VANDN_V;
2389	goto vdvjvk;
2390	case INDEX_op_or_vec:
2391	insn = lasx ? OPC_XVOR_V : OPC_VOR_V;
2392	goto vdvjvk;
2393	case INDEX_op_orc_vec:
2394	insn = lasx ? OPC_XVORN_V : OPC_VORN_V;
2395	goto vdvjvk;
2396	case INDEX_op_xor_vec:
2397	insn = lasx ? OPC_XVXOR_V : OPC_VXOR_V;
2398	goto vdvjvk;
2399	case INDEX_op_not_vec:
2400	a2 = a1;
2401	/* fall through */
2402	case INDEX_op_nor_vec:
2403	insn = lasx ? OPC_XVNOR_V : OPC_VNOR_V;
2404	goto vdvjvk;
2405	case INDEX_op_cmp_vec:
2406	{
2407	TCGCond cond = args[3];
2408
2409	if (const_args[2]) {
2410	/*
2411	* cmp_vec dest, src, value
2412	* Try vseqi/vslei/vslti
2413	*/
2414	int64_t value = sextract64(a2, 0, 8 << vece);
2415	switch (cond) {
2416	case TCG_COND_EQ:
2417	case TCG_COND_LE:
2418	case TCG_COND_LT:
2419	insn = cmp_vec_imm_insn[cond][lasx][vece];
2420	tcg_out32(s, encode_vdvjsk5_insn(insn, a0, a1, value));
2421	break;
2422	case TCG_COND_LEU:
2423	case TCG_COND_LTU:
2424	insn = cmp_vec_imm_insn[cond][lasx][vece];
2425	tcg_out32(s, encode_vdvjuk5_insn(insn, a0, a1, value));
2426	break;
2427	default:
2428	g_assert_not_reached();
2429	}
2430	} else {
2431	switch (cond) {
2432	case TCG_COND_EQ:
2433	case TCG_COND_LE:
2434	case TCG_COND_LEU:
2435	case TCG_COND_LT:
2436	case TCG_COND_LTU:
2437	insn = cmp_vec_insn[cond][lasx][vece];
2438	tcg_out32(s, encode_vdvjvk_insn(insn, a0, a1, a2));
2439	break;
2440	case TCG_COND_GE:
2441	case TCG_COND_GEU:
2442	case TCG_COND_GT:
2443	case TCG_COND_GTU:
2444	insn = cmp_vec_insn[tcg_swap_cond(cond)][lasx][vece];
2445	tcg_out32(s, encode_vdvjvk_insn(insn, a0, a2, a1));
2446	break;
2447	case TCG_COND_NE:
2448	/* ne -> not(eq) */
2449	insn = cmp_vec_insn[TCG_COND_EQ][lasx][vece];
2450	tcg_out32(s, encode_vdvjvk_insn(insn, a0, a1, a2));
2451	insn = lasx ? OPC_XVNOR_V : OPC_VNOR_V;
2452	tcg_out32(s, encode_vdvjvk_insn(insn, a0, a0, a0));
2453	break;
2454	default:
2455	g_assert_not_reached();
2456	}
2457	}
2458	}
2459	break;
2460	case INDEX_op_add_vec:
2461	tcg_out_addsub_vec(s, lasx, vece, a0, a1, a2, const_args[2], true);
2462	break;
2463	case INDEX_op_sub_vec:
2464	tcg_out_addsub_vec(s, lasx, vece, a0, a1, a2, const_args[2], false);
2465	break;
2466	case INDEX_op_neg_vec:
2467	tcg_out32(s, encode_vdvj_insn(neg_vec_insn[lasx][vece], a0, a1));
2468	break;
2469	case INDEX_op_mul_vec:
2470	insn = mul_vec_insn[lasx][vece];
2471	goto vdvjvk;
2472	case INDEX_op_smin_vec:
2473	insn = smin_vec_insn[lasx][vece];
2474	goto vdvjvk;
2475	case INDEX_op_smax_vec:
2476	insn = smax_vec_insn[lasx][vece];
2477	goto vdvjvk;
2478	case INDEX_op_umin_vec:
2479	insn = umin_vec_insn[lasx][vece];
2480	goto vdvjvk;
2481	case INDEX_op_umax_vec:
2482	insn = umax_vec_insn[lasx][vece];
2483	goto vdvjvk;
2484	case INDEX_op_ssadd_vec:
2485	insn = ssadd_vec_insn[lasx][vece];
2486	goto vdvjvk;
2487	case INDEX_op_usadd_vec:
2488	insn = usadd_vec_insn[lasx][vece];
2489	goto vdvjvk;
2490	case INDEX_op_sssub_vec:
2491	insn = sssub_vec_insn[lasx][vece];
2492	goto vdvjvk;
2493	case INDEX_op_ussub_vec:
2494	insn = ussub_vec_insn[lasx][vece];
2495	goto vdvjvk;
2496	case INDEX_op_shlv_vec:
2497	insn = shlv_vec_insn[lasx][vece];
2498	goto vdvjvk;
2499	case INDEX_op_shrv_vec:
2500	insn = shrv_vec_insn[lasx][vece];
2501	goto vdvjvk;
2502	case INDEX_op_sarv_vec:
2503	insn = sarv_vec_insn[lasx][vece];
2504	goto vdvjvk;
2505	case INDEX_op_rotlv_vec:
2506	/* rotlv_vec a1, a2 = rotrv_vec a1, -a2 */
2507	tcg_out32(s, encode_vdvj_insn(neg_vec_insn[lasx][vece],
2508	TCG_VEC_TMP0, a2));
2509	a2 = TCG_VEC_TMP0;
2510	/* fall through */
2511	case INDEX_op_rotrv_vec:
2512	insn = rotrv_vec_insn[lasx][vece];
2513	goto vdvjvk;
2514	case INDEX_op_shli_vec:
2515	insn = shli_vec_insn[lasx][vece];
2516	goto vdvjukN;
2517	case INDEX_op_shri_vec:
2518	insn = shri_vec_insn[lasx][vece];
2519	goto vdvjukN;
2520	case INDEX_op_sari_vec:
2521	insn = sari_vec_insn[lasx][vece];
2522	goto vdvjukN;
2523	case INDEX_op_rotli_vec:
2524	/* rotli_vec a1, a2 = rotri_vec a1, -a2 */
2525	a2 = extract32(-a2, 0, 3 + vece);
2526	insn = rotri_vec_insn[lasx][vece];
2527	goto vdvjukN;
2528	case INDEX_op_bitsel_vec:
2529	/* vbitsel vd, vj, vk, va = bitsel_vec vd, va, vk, vj */
2530	if (lasx) {
2531	tcg_out_opc_xvbitsel_v(s, a0, a3, a2, a1);
2532	} else {
2533	tcg_out_opc_vbitsel_v(s, a0, a3, a2, a1);
2534	}
2535	break;
2536	case INDEX_op_dupm_vec:
2537	tcg_out_dupm_vec(s, type, vece, a0, a1, a2);
2538	break;
2539	default:
2540	g_assert_not_reached();
2541	vdvjvk:
2542	tcg_out32(s, encode_vdvjvk_insn(insn, a0, a1, a2));
2543	break;
2544	vdvjukN:
2545	switch (vece) {
2546	case MO_8:
2547	tcg_out32(s, encode_vdvjuk3_insn(insn, a0, a1, a2));
2548	break;
2549	case MO_16:
2550	tcg_out32(s, encode_vdvjuk4_insn(insn, a0, a1, a2));
2551	break;
2552	case MO_32:
2553	tcg_out32(s, encode_vdvjuk5_insn(insn, a0, a1, a2));
2554	break;
2555	case MO_64:
2556	tcg_out32(s, encode_vdvjuk6_insn(insn, a0, a1, a2));
2557	break;
2558	default:
2559	g_assert_not_reached();
2560	}
2561	break;
2562	}
2563	}
2564
2565	int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece)
2566	{
2567	switch (opc) {
2568	case INDEX_op_ld_vec:
2569	case INDEX_op_st_vec:
2570	case INDEX_op_dup_vec:
2571	case INDEX_op_dupm_vec:
2572	case INDEX_op_cmp_vec:
2573	case INDEX_op_add_vec:
2574	case INDEX_op_sub_vec:
2575	case INDEX_op_and_vec:
2576	case INDEX_op_andc_vec:
2577	case INDEX_op_or_vec:
2578	case INDEX_op_orc_vec:
2579	case INDEX_op_xor_vec:
2580	case INDEX_op_nor_vec:
2581	case INDEX_op_not_vec:
2582	case INDEX_op_neg_vec:
2583	case INDEX_op_mul_vec:
2584	case INDEX_op_smin_vec:
2585	case INDEX_op_smax_vec:
2586	case INDEX_op_umin_vec:
2587	case INDEX_op_umax_vec:
2588	case INDEX_op_ssadd_vec:
2589	case INDEX_op_usadd_vec:
2590	case INDEX_op_sssub_vec:
2591	case INDEX_op_ussub_vec:
2592	case INDEX_op_shlv_vec:
2593	case INDEX_op_shrv_vec:
2594	case INDEX_op_sarv_vec:
2595	case INDEX_op_bitsel_vec:
2596	return 1;
2597	default:
2598	return 0;
2599	}
2600	}
2601
2602	void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece,
2603	TCGArg a0, ...)
2604	{
2605	g_assert_not_reached();
2606	}
2607
2608	static TCGConstraintSetIndex
2609	tcg_target_op_def(TCGOpcode op, TCGType type, unsigned flags)
2610	{
2611	switch (op) {
2612	case INDEX_op_ld_vec:
2613	case INDEX_op_dupm_vec:
2614	case INDEX_op_dup_vec:
2615	return C_O1_I1(w, r);
2616
2617	case INDEX_op_st_vec:
2618	return C_O0_I2(w, r);
2619
2620	case INDEX_op_cmp_vec:
2621	return C_O1_I2(w, w, wM);
2622
2623	case INDEX_op_add_vec:
2624	case INDEX_op_sub_vec:
2625	return C_O1_I2(w, w, wA);
2626
2627	case INDEX_op_and_vec:
2628	case INDEX_op_andc_vec:
2629	case INDEX_op_or_vec:
2630	case INDEX_op_orc_vec:
2631	case INDEX_op_xor_vec:
2632	case INDEX_op_nor_vec:
2633	case INDEX_op_mul_vec:
2634	case INDEX_op_smin_vec:
2635	case INDEX_op_smax_vec:
2636	case INDEX_op_umin_vec:
2637	case INDEX_op_umax_vec:
2638	case INDEX_op_ssadd_vec:
2639	case INDEX_op_usadd_vec:
2640	case INDEX_op_sssub_vec:
2641	case INDEX_op_ussub_vec:
2642	case INDEX_op_shlv_vec:
2643	case INDEX_op_shrv_vec:
2644	case INDEX_op_sarv_vec:
2645	case INDEX_op_rotrv_vec:
2646	case INDEX_op_rotlv_vec:
2647	return C_O1_I2(w, w, w);
2648
2649	case INDEX_op_not_vec:
2650	case INDEX_op_neg_vec:
2651	case INDEX_op_shli_vec:
2652	case INDEX_op_shri_vec:
2653	case INDEX_op_sari_vec:
2654	case INDEX_op_rotli_vec:
2655	return C_O1_I1(w, w);
2656
2657	case INDEX_op_bitsel_vec:
2658	return C_O1_I3(w, w, w, w);
2659
2660	default:
2661	return C_NotImplemented;
2662	}
2663	}
2664
2665	static const int tcg_target_callee_save_regs[] = {
2666	TCG_REG_S0, /* used for the global env (TCG_AREG0) */
2667	TCG_REG_S1,
2668	TCG_REG_S2,
2669	TCG_REG_S3,
2670	TCG_REG_S4,
2671	TCG_REG_S5,
2672	TCG_REG_S6,
2673	TCG_REG_S7,
2674	TCG_REG_S8,
2675	TCG_REG_S9,
2676	TCG_REG_RA, /* should be last for ABI compliance */
2677	};
2678
2679	/* Stack frame parameters. */
2680	#define REG_SIZE ((int)sizeof(tcg_target_long))
2681	#define SAVE_SIZE ((int)ARRAY_SIZE(tcg_target_callee_save_regs) * REG_SIZE)
2682	#define TEMP_SIZE (CPU_TEMP_BUF_NLONGS * (int)sizeof(long))
2683	#define FRAME_SIZE ((TCG_STATIC_CALL_ARGS_SIZE + TEMP_SIZE + SAVE_SIZE \
2684	+ TCG_TARGET_STACK_ALIGN - 1) \
2685	& -TCG_TARGET_STACK_ALIGN)
2686	#define SAVE_OFS (TCG_STATIC_CALL_ARGS_SIZE + TEMP_SIZE)
2687
2688	/* We're expecting to be able to use an immediate for frame allocation. */
2689	QEMU_BUILD_BUG_ON(FRAME_SIZE > 0x7ff);
2690
2691	/* Generate global QEMU prologue and epilogue code */
2692	static void tcg_target_qemu_prologue(TCGContext *s)
2693	{
2694	int i;
2695
2696	tcg_set_frame(s, TCG_REG_SP, TCG_STATIC_CALL_ARGS_SIZE, TEMP_SIZE);
2697
2698	/* TB prologue */
2699	tcg_out_opc_addi_d(s, TCG_REG_SP, TCG_REG_SP, -FRAME_SIZE);
2700	for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) {
2701	tcg_out_st(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i],
2702	TCG_REG_SP, SAVE_OFS + i * REG_SIZE);
2703	}
2704
2705	if (!tcg_use_softmmu && guest_base) {
2706	tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base);
2707	tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
2708	}
2709
2710	/* Call generated code */
2711	tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
2712	tcg_out_opc_jirl(s, TCG_REG_ZERO, tcg_target_call_iarg_regs[1], 0);
2713
2714	/* Return path for goto_ptr. Set return value to 0 */
2715	tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr);
2716	tcg_out_mov(s, TCG_TYPE_REG, TCG_REG_A0, TCG_REG_ZERO);
2717
2718	/* TB epilogue */
2719	tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr);
2720	for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); i++) {
2721	tcg_out_ld(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i],
2722	TCG_REG_SP, SAVE_OFS + i * REG_SIZE);
2723	}
2724
2725	tcg_out_opc_addi_d(s, TCG_REG_SP, TCG_REG_SP, FRAME_SIZE);
2726	tcg_out_opc_jirl(s, TCG_REG_ZERO, TCG_REG_RA, 0);
2727	}
2728
2729	static void tcg_out_tb_start(TCGContext *s)
2730	{
2731	/* nothing to do */
2732	}
2733
2734	static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
2735	{
2736	for (int i = 0; i < count; ++i) {
2737	/* Canonical nop is andi r0,r0,0 */
2738	p[i] = OPC_ANDI;
2739	}
2740	}
2741
2742	static void tcg_target_init(TCGContext *s)
2743	{
2744	unsigned long hwcap = qemu_getauxval(AT_HWCAP);
2745
2746	/* Server and desktop class cpus have UAL; embedded cpus do not. */
2747	if (!(hwcap & HWCAP_LOONGARCH_UAL)) {
2748	error_report("TCG: unaligned access support required; exiting");
2749	exit(EXIT_FAILURE);
2750	}
2751
2752	tcg_target_available_regs[TCG_TYPE_I32] = ALL_GENERAL_REGS;
2753	tcg_target_available_regs[TCG_TYPE_I64] = ALL_GENERAL_REGS;
2754
2755	tcg_target_call_clobber_regs = ALL_GENERAL_REGS \| ALL_VECTOR_REGS;
2756	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S0);
2757	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S1);
2758	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S2);
2759	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S3);
2760	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S4);
2761	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S5);
2762	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S6);
2763	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S7);
2764	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S8);
2765	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_S9);
2766
2767	if (cpuinfo & CPUINFO_LSX) {
2768	tcg_target_available_regs[TCG_TYPE_V64] = ALL_VECTOR_REGS;
2769	tcg_target_available_regs[TCG_TYPE_V128] = ALL_VECTOR_REGS;
2770	if (cpuinfo & CPUINFO_LASX) {
2771	tcg_target_available_regs[TCG_TYPE_V256] = ALL_VECTOR_REGS;
2772	}
2773	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V24);
2774	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V25);
2775	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V26);
2776	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V27);
2777	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V28);
2778	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V29);
2779	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V30);
2780	tcg_regset_reset_reg(tcg_target_call_clobber_regs, TCG_REG_V31);
2781	}
2782
2783	s->reserved_regs = 0;
2784	tcg_regset_set_reg(s->reserved_regs, TCG_REG_ZERO);
2785	tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP0);
2786	tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP1);
2787	tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP2);
2788	tcg_regset_set_reg(s->reserved_regs, TCG_REG_SP);
2789	tcg_regset_set_reg(s->reserved_regs, TCG_REG_TP);
2790	tcg_regset_set_reg(s->reserved_regs, TCG_REG_RESERVED);
2791	tcg_regset_set_reg(s->reserved_regs, TCG_VEC_TMP0);
2792	}
2793
2794	typedef struct {
2795	DebugFrameHeader h;
2796	uint8_t fde_def_cfa[4];
2797	uint8_t fde_reg_ofs[ARRAY_SIZE(tcg_target_callee_save_regs) * 2];
2798	} DebugFrame;
2799
2800	#define ELF_HOST_MACHINE EM_LOONGARCH
2801
2802	static const DebugFrame debug_frame = {
2803	.h.cie.len = sizeof(DebugFrameCIE) - 4, /* length after .len member */
2804	.h.cie.id = -1,
2805	.h.cie.version = 1,
2806	.h.cie.code_align = 1,
2807	.h.cie.data_align = -sizeof(tcg_target_long) & 0x7f, /* sleb128 */
2808	.h.cie.return_column = TCG_REG_RA,
2809
2810	/* Total FDE size does not include the "len" member. */
2811	.h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
2812
2813	.fde_def_cfa = {
2814	12, TCG_REG_SP, /* DW_CFA_def_cfa sp, ... */
2815	(FRAME_SIZE & 0x7f) \| 0x80, /* ... uleb128 FRAME_SIZE */
2816	(FRAME_SIZE >> 7)
2817	},
2818	.fde_reg_ofs = {
2819	0x80 + 23, 11, /* DW_CFA_offset, s0, -88 */
2820	0x80 + 24, 10, /* DW_CFA_offset, s1, -80 */
2821	0x80 + 25, 9, /* DW_CFA_offset, s2, -72 */
2822	0x80 + 26, 8, /* DW_CFA_offset, s3, -64 */
2823	0x80 + 27, 7, /* DW_CFA_offset, s4, -56 */
2824	0x80 + 28, 6, /* DW_CFA_offset, s5, -48 */
2825	0x80 + 29, 5, /* DW_CFA_offset, s6, -40 */
2826	0x80 + 30, 4, /* DW_CFA_offset, s7, -32 */
2827	0x80 + 31, 3, /* DW_CFA_offset, s8, -24 */
2828	0x80 + 22, 2, /* DW_CFA_offset, s9, -16 */
2829	0x80 + 1 , 1, /* DW_CFA_offset, ra, -8 */
2830	}
2831	};
2832
2833	void tcg_register_jit(const void *buf, size_t buf_size)
2834	{
2835	tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
2836	}