tcg-target.c.inc · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

inc 4,410 lines 126 KB

Raw

1	/*
2	* Tiny Code Generator for QEMU
3	*
4	* Copyright (c) 2008 Fabrice Bellard
5	*
6	* Permission is hereby granted, free of charge, to any person obtaining a copy
7	* of this software and associated documentation files (the "Software"), to deal
8	* in the Software without restriction, including without limitation the rights
9	* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10	* copies of the Software, and to permit persons to whom the Software is
11	* furnished to do so, subject to the following conditions:
12	*
13	* The above copyright notice and this permission notice shall be included in
14	* all copies or substantial portions of the Software.
15	*
16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22	* THE SOFTWARE.
23	*/
24
25	#include "elf.h"
26
27	/*
28	* Standardize on the _CALL_FOO symbols used by GCC:
29	* Apple XCode does not define _CALL_DARWIN.
30	* Clang defines _CALL_ELF (64-bit) but not _CALL_SYSV or _CALL_AIX.
31	*/
32	#ifdef _CALL_AIX
33	/* ok */
34	#elif defined(_CALL_ELF) && _CALL_ELF == 1
35	# define _CALL_AIX
36	#elif defined(_CALL_ELF) && _CALL_ELF == 2
37	/* ok */
38	#else
39	# error "Unknown ABI"
40	#endif
41
42	#define TCG_TARGET_CALL_ARG_I32 TCG_CALL_ARG_EXTEND
43	#define TCG_TARGET_CALL_RET_I128 TCG_CALL_RET_NORMAL
44	#ifdef _CALL_SYSV
45	# define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_EVEN
46	# define TCG_TARGET_CALL_ARG_I128 TCG_CALL_ARG_BY_REF
47	#else
48	# define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL
49	# define TCG_TARGET_CALL_ARG_I128 TCG_CALL_ARG_NORMAL
50	#endif
51
52	/* For some memory operations, we need a scratch that isn't R0. For the AIX
53	calling convention, we can re-use the TOC register since we'll be reloading
54	it at every call. Otherwise R12 will do nicely as neither a call-saved
55	register nor a parameter register. */
56	#ifdef _CALL_AIX
57	# define TCG_REG_TMP1 TCG_REG_R2
58	#else
59	# define TCG_REG_TMP1 TCG_REG_R12
60	#endif
61	#define TCG_REG_TMP2 TCG_REG_R11
62
63	#define TCG_VEC_TMP1 TCG_REG_V0
64	#define TCG_VEC_TMP2 TCG_REG_V1
65
66	#define TCG_REG_TB TCG_REG_R31
67	#define USE_REG_TB !have_isa_3_00
68
69	/* Shorthand for size of a pointer. Avoid promotion to unsigned. */
70	#define SZP ((int)sizeof(void *))
71
72	/* Shorthand for size of a register. */
73	#define SZR ((int)sizeof(tcg_target_long))
74
75	#define TCG_CT_CONST_S16 0x00100
76	#define TCG_CT_CONST_U16 0x00200
77	#define TCG_CT_CONST_N16 0x00400
78	#define TCG_CT_CONST_S32 0x00800
79	#define TCG_CT_CONST_U32 0x01000
80	#define TCG_CT_CONST_ZERO 0x02000
81	#define TCG_CT_CONST_MONE 0x04000
82	#define TCG_CT_CONST_WSZ 0x08000
83	#define TCG_CT_CONST_CMP 0x10000
84
85	#define ALL_GENERAL_REGS 0xffffffffu
86	#define ALL_VECTOR_REGS 0xffffffff00000000ull
87
88	#ifndef R_PPC64_PCREL34
89	#define R_PPC64_PCREL34 132
90	#endif
91
92	#define have_isel (cpuinfo & CPUINFO_ISEL)
93
94	#define TCG_GUEST_BASE_REG TCG_REG_R30
95
96	#ifdef CONFIG_DEBUG_TCG
97	static const char tcg_target_reg_names[TCG_TARGET_NB_REGS][4] = {
98	"r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7",
99	"r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15",
100	"r16", "r17", "r18", "r19", "r20", "r21", "r22", "r23",
101	"r24", "r25", "r26", "r27", "r28", "r29", "r30", "r31",
102	"v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
103	"v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15",
104	"v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
105	"v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31",
106	};
107	#endif
108
109	static const int tcg_target_reg_alloc_order[] = {
110	TCG_REG_R14, /* call saved registers */
111	TCG_REG_R15,
112	TCG_REG_R16,
113	TCG_REG_R17,
114	TCG_REG_R18,
115	TCG_REG_R19,
116	TCG_REG_R20,
117	TCG_REG_R21,
118	TCG_REG_R22,
119	TCG_REG_R23,
120	TCG_REG_R24,
121	TCG_REG_R25,
122	TCG_REG_R26,
123	TCG_REG_R27,
124	TCG_REG_R28,
125	TCG_REG_R29,
126	TCG_REG_R30,
127	TCG_REG_R31,
128	TCG_REG_R12, /* call clobbered, non-arguments */
129	TCG_REG_R11,
130	TCG_REG_R2,
131	TCG_REG_R13,
132	TCG_REG_R10, /* call clobbered, arguments */
133	TCG_REG_R9,
134	TCG_REG_R8,
135	TCG_REG_R7,
136	TCG_REG_R6,
137	TCG_REG_R5,
138	TCG_REG_R4,
139	TCG_REG_R3,
140
141	/* V0 and V1 reserved as temporaries; V20 - V31 are call-saved */
142	TCG_REG_V2, /* call clobbered, vectors */
143	TCG_REG_V3,
144	TCG_REG_V4,
145	TCG_REG_V5,
146	TCG_REG_V6,
147	TCG_REG_V7,
148	TCG_REG_V8,
149	TCG_REG_V9,
150	TCG_REG_V10,
151	TCG_REG_V11,
152	TCG_REG_V12,
153	TCG_REG_V13,
154	TCG_REG_V14,
155	TCG_REG_V15,
156	TCG_REG_V16,
157	TCG_REG_V17,
158	TCG_REG_V18,
159	TCG_REG_V19,
160	};
161
162	static const int tcg_target_call_iarg_regs[] = {
163	TCG_REG_R3,
164	TCG_REG_R4,
165	TCG_REG_R5,
166	TCG_REG_R6,
167	TCG_REG_R7,
168	TCG_REG_R8,
169	TCG_REG_R9,
170	TCG_REG_R10
171	};
172
173	static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot)
174	{
175	tcg_debug_assert(kind == TCG_CALL_RET_NORMAL);
176	tcg_debug_assert(slot >= 0 && slot <= 1);
177	return TCG_REG_R3 + slot;
178	}
179
180	static const int tcg_target_callee_save_regs[] = {
181	#ifdef _CALL_DARWIN
182	TCG_REG_R11,
183	#endif
184	TCG_REG_R14,
185	TCG_REG_R15,
186	TCG_REG_R16,
187	TCG_REG_R17,
188	TCG_REG_R18,
189	TCG_REG_R19,
190	TCG_REG_R20,
191	TCG_REG_R21,
192	TCG_REG_R22,
193	TCG_REG_R23,
194	TCG_REG_R24,
195	TCG_REG_R25,
196	TCG_REG_R26,
197	TCG_REG_R27, /* currently used for the global env */
198	TCG_REG_R28,
199	TCG_REG_R29,
200	TCG_REG_R30,
201	TCG_REG_R31
202	};
203
204	/* For PPC, we use TB+4 instead of TB as the base. */
205	static inline ptrdiff_t ppc_tbrel_diff(TCGContext s, const void target)
206	{
207	return tcg_tbrel_diff(s, target) - 4;
208	}
209
210	static inline bool in_range_b(tcg_target_long target)
211	{
212	return target == sextract64(target, 0, 26);
213	}
214
215	static uint32_t reloc_pc24_val(const tcg_insn_unit *pc,
216	const tcg_insn_unit *target)
217	{
218	ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
219	tcg_debug_assert(in_range_b(disp));
220	return disp & 0x3fffffc;
221	}
222
223	static bool reloc_pc24(tcg_insn_unit src_rw, const tcg_insn_unit target)
224	{
225	const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
226	ptrdiff_t disp = tcg_ptr_byte_diff(target, src_rx);
227
228	if (in_range_b(disp)) {
229	src_rw = (src_rw & ~0x3fffffc) \| (disp & 0x3fffffc);
230	return true;
231	}
232	return false;
233	}
234
235	static uint16_t reloc_pc14_val(const tcg_insn_unit *pc,
236	const tcg_insn_unit *target)
237	{
238	ptrdiff_t disp = tcg_ptr_byte_diff(target, pc);
239	tcg_debug_assert(disp == (int16_t) disp);
240	return disp & 0xfffc;
241	}
242
243	static bool reloc_pc14(tcg_insn_unit src_rw, const tcg_insn_unit target)
244	{
245	const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
246	ptrdiff_t disp = tcg_ptr_byte_diff(target, src_rx);
247
248	if (disp == (int16_t) disp) {
249	src_rw = (src_rw & ~0xfffc) \| (disp & 0xfffc);
250	return true;
251	}
252	return false;
253	}
254
255	static bool reloc_pc34(tcg_insn_unit src_rw, const tcg_insn_unit target)
256	{
257	const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
258	ptrdiff_t disp = tcg_ptr_byte_diff(target, src_rx);
259
260	if (disp == sextract64(disp, 0, 34)) {
261	src_rw[0] = (src_rw[0] & ~0x3ffff) \| ((disp >> 16) & 0x3ffff);
262	src_rw[1] = (src_rw[1] & ~0xffff) \| (disp & 0xffff);
263	return true;
264	}
265	return false;
266	}
267
268	static bool mask_operand(uint32_t c, int mb, int me);
269	static bool mask64_operand(uint64_t c, int mb, int me);
270
271	/* test if a constant matches the constraint */
272	static bool tcg_target_const_match(int64_t sval, int ct,
273	TCGType type, TCGCond cond, int vece)
274	{
275	uint64_t uval = sval;
276	int mb, me;
277
278	if (ct & TCG_CT_CONST) {
279	return 1;
280	}
281
282	if (type == TCG_TYPE_I32) {
283	uval = (uint32_t)sval;
284	sval = (int32_t)sval;
285	}
286
287	if (ct & TCG_CT_CONST_CMP) {
288	switch (cond) {
289	case TCG_COND_EQ:
290	case TCG_COND_NE:
291	ct \|= TCG_CT_CONST_S16 \| TCG_CT_CONST_U16;
292	break;
293	case TCG_COND_LT:
294	case TCG_COND_GE:
295	case TCG_COND_LE:
296	case TCG_COND_GT:
297	ct \|= TCG_CT_CONST_S16;
298	break;
299	case TCG_COND_LTU:
300	case TCG_COND_GEU:
301	case TCG_COND_LEU:
302	case TCG_COND_GTU:
303	ct \|= TCG_CT_CONST_U16;
304	break;
305	case TCG_COND_TSTEQ:
306	case TCG_COND_TSTNE:
307	if ((uval & ~0xffff) == 0 \|\| (uval & ~0xffff0000ull) == 0) {
308	return 1;
309	}
310	if (uval == (uint32_t)uval && mask_operand(uval, &mb, &me)) {
311	return 1;
312	}
313	if (mask64_operand(uval << clz64(uval), &mb, &me)) {
314	return 1;
315	}
316	return 0;
317	default:
318	g_assert_not_reached();
319	}
320	}
321
322	if ((ct & TCG_CT_CONST_S16) && sval == (int16_t)sval) {
323	return 1;
324	}
325	if ((ct & TCG_CT_CONST_U16) && uval == (uint16_t)uval) {
326	return 1;
327	}
328	if ((ct & TCG_CT_CONST_N16) && -sval == (int16_t)-sval) {
329	return 1;
330	}
331	if ((ct & TCG_CT_CONST_S32) && sval == (int32_t)sval) {
332	return 1;
333	}
334	if ((ct & TCG_CT_CONST_U32) && uval == (uint32_t)uval) {
335	return 1;
336	}
337	if ((ct & TCG_CT_CONST_ZERO) && sval == 0) {
338	return 1;
339	}
340	if ((ct & TCG_CT_CONST_MONE) && sval == -1) {
341	return 1;
342	}
343	if ((ct & TCG_CT_CONST_WSZ) && sval == (type == TCG_TYPE_I32 ? 32 : 64)) {
344	return 1;
345	}
346	return 0;
347	}
348
349	#define OPCD(opc) ((opc)<<26)
350	#define XO19(opc) (OPCD(19)\|((opc)<<1))
351	#define MD30(opc) (OPCD(30)\|((opc)<<2))
352	#define MDS30(opc) (OPCD(30)\|((opc)<<1))
353	#define XO31(opc) (OPCD(31)\|((opc)<<1))
354	#define XO58(opc) (OPCD(58)\|(opc))
355	#define XO62(opc) (OPCD(62)\|(opc))
356	#define VX4(opc) (OPCD(4)\|(opc))
357
358	#define B OPCD( 18)
359	#define BC OPCD( 16)
360
361	#define LBZ OPCD( 34)
362	#define LHZ OPCD( 40)
363	#define LHA OPCD( 42)
364	#define LWZ OPCD( 32)
365	#define LWZUX XO31( 55)
366	#define LD XO58( 0)
367	#define LDX XO31( 21)
368	#define LDU XO58( 1)
369	#define LDUX XO31( 53)
370	#define LWA XO58( 2)
371	#define LWAX XO31(341)
372	#define LQ OPCD( 56)
373
374	#define STB OPCD( 38)
375	#define STH OPCD( 44)
376	#define STW OPCD( 36)
377	#define STD XO62( 0)
378	#define STDU XO62( 1)
379	#define STDX XO31(149)
380	#define STQ XO62( 2)
381
382	#define PLWA OPCD( 41)
383	#define PLD OPCD( 57)
384	#define PLXSD OPCD( 42)
385	#define PLXV OPCD(25 * 2 + 1) /* force tx=1 */
386
387	#define PSTD OPCD( 61)
388	#define PSTXSD OPCD( 46)
389	#define PSTXV OPCD(27 * 2 + 1) /* force sx=1 */
390
391	#define ADDIC OPCD( 12)
392	#define ADDI OPCD( 14)
393	#define ADDIS OPCD( 15)
394	#define ORI OPCD( 24)
395	#define ORIS OPCD( 25)
396	#define XORI OPCD( 26)
397	#define XORIS OPCD( 27)
398	#define ANDI OPCD( 28)
399	#define ANDIS OPCD( 29)
400	#define MULLI OPCD( 7)
401	#define CMPLI OPCD( 10)
402	#define CMPI OPCD( 11)
403	#define SUBFIC OPCD( 8)
404
405	#define LWZU OPCD( 33)
406	#define STWU OPCD( 37)
407
408	#define RLWIMI OPCD( 20)
409	#define RLWINM OPCD( 21)
410	#define RLWNM OPCD( 23)
411
412	#define RLDICL MD30( 0)
413	#define RLDICR MD30( 1)
414	#define RLDIMI MD30( 3)
415	#define RLDCL MDS30( 8)
416
417	#define BCLR XO19( 16)
418	#define BCCTR XO19(528)
419	#define CRAND XO19(257)
420	#define CRANDC XO19(129)
421	#define CRNAND XO19(225)
422	#define CROR XO19(449)
423	#define CRNOR XO19( 33)
424	#define ADDPCIS XO19( 2)
425
426	#define EXTSB XO31(954)
427	#define EXTSH XO31(922)
428	#define EXTSW XO31(986)
429	#define ADD XO31(266)
430	#define ADDE XO31(138)
431	#define ADDME XO31(234)
432	#define ADDZE XO31(202)
433	#define ADDC XO31( 10)
434	#define AND XO31( 28)
435	#define SUBF XO31( 40)
436	#define SUBFC XO31( 8)
437	#define SUBFE XO31(136)
438	#define SUBFME XO31(232)
439	#define SUBFZE XO31(200)
440	#define OR XO31(444)
441	#define XOR XO31(316)
442	#define MULLW XO31(235)
443	#define MULHW XO31( 75)
444	#define MULHWU XO31( 11)
445	#define DIVW XO31(491)
446	#define DIVWU XO31(459)
447	#define MODSW XO31(779)
448	#define MODUW XO31(267)
449	#define CMP XO31( 0)
450	#define CMPL XO31( 32)
451	#define LHBRX XO31(790)
452	#define LWBRX XO31(534)
453	#define LDBRX XO31(532)
454	#define STHBRX XO31(918)
455	#define STWBRX XO31(662)
456	#define STDBRX XO31(660)
457	#define MFSPR XO31(339)
458	#define MTSPR XO31(467)
459	#define SRAWI XO31(824)
460	#define NEG XO31(104)
461	#define MFCR XO31( 19)
462	#define MFOCRF (MFCR \| (1u << 20))
463	#define NOR XO31(124)
464	#define CNTLZW XO31( 26)
465	#define CNTLZD XO31( 58)
466	#define CNTTZW XO31(538)
467	#define CNTTZD XO31(570)
468	#define CNTPOPW XO31(378)
469	#define CNTPOPD XO31(506)
470	#define ANDC XO31( 60)
471	#define ORC XO31(412)
472	#define EQV XO31(284)
473	#define NAND XO31(476)
474	#define ISEL XO31( 15)
475
476	#define MULLD XO31(233)
477	#define MULHD XO31( 73)
478	#define MULHDU XO31( 9)
479	#define DIVD XO31(489)
480	#define DIVDU XO31(457)
481	#define MODSD XO31(777)
482	#define MODUD XO31(265)
483
484	#define LBZX XO31( 87)
485	#define LHZX XO31(279)
486	#define LHAX XO31(343)
487	#define LWZX XO31( 23)
488	#define STBX XO31(215)
489	#define STHX XO31(407)
490	#define STWX XO31(151)
491
492	#define EIEIO XO31(854)
493	#define HWSYNC XO31(598)
494	#define LWSYNC (HWSYNC \| (1u << 21))
495
496	#define SPR(a, b) ((((a)<<5)\|(b))<<11)
497	#define LR SPR(8, 0)
498	#define CTR SPR(9, 0)
499
500	#define SLW XO31( 24)
501	#define SRW XO31(536)
502	#define SRAW XO31(792)
503
504	#define SLD XO31( 27)
505	#define SRD XO31(539)
506	#define SRAD XO31(794)
507	#define SRADI XO31(413<<1)
508
509	#define BRH XO31(219)
510	#define BRW XO31(155)
511	#define BRD XO31(187)
512
513	#define TW XO31( 4)
514	#define TRAP (TW \| TO(31))
515
516	#define SETBC XO31(384) /* v3.10 */
517	#define SETBCR XO31(416) /* v3.10 */
518	#define SETNBC XO31(448) /* v3.10 */
519	#define SETNBCR XO31(480) /* v3.10 */
520
521	#define NOP ORI /* ori 0,0,0 */
522
523	#define LVX XO31(103)
524	#define LVEBX XO31(7)
525	#define LVEHX XO31(39)
526	#define LVEWX XO31(71)
527	#define LXSDX (XO31(588) \| 1) /* v2.06, force tx=1 */
528	#define LXVDSX (XO31(332) \| 1) /* v2.06, force tx=1 */
529	#define LXSIWZX (XO31(12) \| 1) /* v2.07, force tx=1 */
530	#define LXV (OPCD(61) \| 8 \| 1) /* v3.00, force tx=1 */
531	#define LXSD (OPCD(57) \| 2) /* v3.00 */
532	#define LXVWSX (XO31(364) \| 1) /* v3.00, force tx=1 */
533
534	#define STVX XO31(231)
535	#define STVEWX XO31(199)
536	#define STXSDX (XO31(716) \| 1) /* v2.06, force sx=1 */
537	#define STXSIWX (XO31(140) \| 1) /* v2.07, force sx=1 */
538	#define STXV (OPCD(61) \| 8 \| 5) /* v3.00, force sx=1 */
539	#define STXSD (OPCD(61) \| 2) /* v3.00 */
540
541	#define VADDSBS VX4(768)
542	#define VADDUBS VX4(512)
543	#define VADDUBM VX4(0)
544	#define VADDSHS VX4(832)
545	#define VADDUHS VX4(576)
546	#define VADDUHM VX4(64)
547	#define VADDSWS VX4(896)
548	#define VADDUWS VX4(640)
549	#define VADDUWM VX4(128)
550	#define VADDUDM VX4(192) /* v2.07 */
551
552	#define VSUBSBS VX4(1792)
553	#define VSUBUBS VX4(1536)
554	#define VSUBUBM VX4(1024)
555	#define VSUBSHS VX4(1856)
556	#define VSUBUHS VX4(1600)
557	#define VSUBUHM VX4(1088)
558	#define VSUBSWS VX4(1920)
559	#define VSUBUWS VX4(1664)
560	#define VSUBUWM VX4(1152)
561	#define VSUBUDM VX4(1216) /* v2.07 */
562
563	#define VNEGW (VX4(1538) \| (6 << 16)) /* v3.00 */
564	#define VNEGD (VX4(1538) \| (7 << 16)) /* v3.00 */
565
566	#define VMAXSB VX4(258)
567	#define VMAXSH VX4(322)
568	#define VMAXSW VX4(386)
569	#define VMAXSD VX4(450) /* v2.07 */
570	#define VMAXUB VX4(2)
571	#define VMAXUH VX4(66)
572	#define VMAXUW VX4(130)
573	#define VMAXUD VX4(194) /* v2.07 */
574	#define VMINSB VX4(770)
575	#define VMINSH VX4(834)
576	#define VMINSW VX4(898)
577	#define VMINSD VX4(962) /* v2.07 */
578	#define VMINUB VX4(514)
579	#define VMINUH VX4(578)
580	#define VMINUW VX4(642)
581	#define VMINUD VX4(706) /* v2.07 */
582
583	#define VCMPEQUB VX4(6)
584	#define VCMPEQUH VX4(70)
585	#define VCMPEQUW VX4(134)
586	#define VCMPEQUD VX4(199) /* v2.07 */
587	#define VCMPGTSB VX4(774)
588	#define VCMPGTSH VX4(838)
589	#define VCMPGTSW VX4(902)
590	#define VCMPGTSD VX4(967) /* v2.07 */
591	#define VCMPGTUB VX4(518)
592	#define VCMPGTUH VX4(582)
593	#define VCMPGTUW VX4(646)
594	#define VCMPGTUD VX4(711) /* v2.07 */
595	#define VCMPNEB VX4(7) /* v3.00 */
596	#define VCMPNEH VX4(71) /* v3.00 */
597	#define VCMPNEW VX4(135) /* v3.00 */
598
599	#define VSLB VX4(260)
600	#define VSLH VX4(324)
601	#define VSLW VX4(388)
602	#define VSLD VX4(1476) /* v2.07 */
603	#define VSRB VX4(516)
604	#define VSRH VX4(580)
605	#define VSRW VX4(644)
606	#define VSRD VX4(1732) /* v2.07 */
607	#define VSRAB VX4(772)
608	#define VSRAH VX4(836)
609	#define VSRAW VX4(900)
610	#define VSRAD VX4(964) /* v2.07 */
611	#define VRLB VX4(4)
612	#define VRLH VX4(68)
613	#define VRLW VX4(132)
614	#define VRLD VX4(196) /* v2.07 */
615
616	#define VMULEUB VX4(520)
617	#define VMULEUH VX4(584)
618	#define VMULEUW VX4(648) /* v2.07 */
619	#define VMULOUB VX4(8)
620	#define VMULOUH VX4(72)
621	#define VMULOUW VX4(136) /* v2.07 */
622	#define VMULUWM VX4(137) /* v2.07 */
623	#define VMULLD VX4(457) /* v3.10 */
624	#define VMSUMUHM VX4(38)
625
626	#define VMRGHB VX4(12)
627	#define VMRGHH VX4(76)
628	#define VMRGHW VX4(140)
629	#define VMRGLB VX4(268)
630	#define VMRGLH VX4(332)
631	#define VMRGLW VX4(396)
632
633	#define VPKUHUM VX4(14)
634	#define VPKUWUM VX4(78)
635
636	#define VAND VX4(1028)
637	#define VANDC VX4(1092)
638	#define VNOR VX4(1284)
639	#define VOR VX4(1156)
640	#define VXOR VX4(1220)
641	#define VEQV VX4(1668) /* v2.07 */
642	#define VNAND VX4(1412) /* v2.07 */
643	#define VORC VX4(1348) /* v2.07 */
644
645	#define VSPLTB VX4(524)
646	#define VSPLTH VX4(588)
647	#define VSPLTW VX4(652)
648	#define VSPLTISB VX4(780)
649	#define VSPLTISH VX4(844)
650	#define VSPLTISW VX4(908)
651
652	#define VSLDOI VX4(44)
653
654	#define XXPERMDI (OPCD(60) \| (10 << 3) \| 7) /* v2.06, force ax=bx=tx=1 */
655	#define XXSEL (OPCD(60) \| (3 << 4) \| 0xf) /* v2.06, force ax=bx=cx=tx=1 */
656	#define XXSPLTIB (OPCD(60) \| (360 << 1) \| 1) /* v3.00, force tx=1 */
657
658	#define MFVSRD (XO31(51) \| 1) /* v2.07, force sx=1 */
659	#define MFVSRWZ (XO31(115) \| 1) /* v2.07, force sx=1 */
660	#define MTVSRD (XO31(179) \| 1) /* v2.07, force tx=1 */
661	#define MTVSRWZ (XO31(243) \| 1) /* v2.07, force tx=1 */
662	#define MTVSRDD (XO31(435) \| 1) /* v3.00, force tx=1 */
663	#define MTVSRWS (XO31(403) \| 1) /* v3.00, force tx=1 */
664
665	#define RT(r) ((r)<<21)
666	#define RS(r) ((r)<<21)
667	#define RA(r) ((r)<<16)
668	#define RB(r) ((r)<<11)
669	#define TO(t) ((t)<<21)
670	#define SH(s) ((s)<<11)
671	#define MB(b) ((b)<<6)
672	#define ME(e) ((e)<<1)
673	#define BO(o) ((o)<<21)
674	#define MB64(b) ((b)<<5)
675	#define FXM(b) (1 << (19 - (b)))
676
677	#define VRT(r) (((r) & 31) << 21)
678	#define VRA(r) (((r) & 31) << 16)
679	#define VRB(r) (((r) & 31) << 11)
680	#define VRC(r) (((r) & 31) << 6)
681
682	#define LK 1
683
684	#define TAB(t, a, b) (RT(t) \| RA(a) \| RB(b))
685	#define SAB(s, a, b) (RS(s) \| RA(a) \| RB(b))
686	#define TAI(s, a, i) (RT(s) \| RA(a) \| ((i) & 0xffff))
687	#define SAI(s, a, i) (RS(s) \| RA(a) \| ((i) & 0xffff))
688
689	#define BF(n) ((n)<<23)
690	#define BI(n, c) (((c)+((n)*4))<<16)
691	#define BT(n, c) (((c)+((n)*4))<<21)
692	#define BA(n, c) (((c)+((n)*4))<<16)
693	#define BB(n, c) (((c)+((n)*4))<<11)
694	#define BC_(n, c) (((c)+((n)*4))<<6)
695
696	#define BO_COND_TRUE BO(12)
697	#define BO_COND_FALSE BO( 4)
698	#define BO_ALWAYS BO(20)
699
700	enum {
701	CR_LT,
702	CR_GT,
703	CR_EQ,
704	CR_SO
705	};
706
707	static const uint32_t tcg_to_bc[16] = {
708	[TCG_COND_EQ] = BC \| BI(0, CR_EQ) \| BO_COND_TRUE,
709	[TCG_COND_NE] = BC \| BI(0, CR_EQ) \| BO_COND_FALSE,
710	[TCG_COND_TSTEQ] = BC \| BI(0, CR_EQ) \| BO_COND_TRUE,
711	[TCG_COND_TSTNE] = BC \| BI(0, CR_EQ) \| BO_COND_FALSE,
712	[TCG_COND_LT] = BC \| BI(0, CR_LT) \| BO_COND_TRUE,
713	[TCG_COND_GE] = BC \| BI(0, CR_LT) \| BO_COND_FALSE,
714	[TCG_COND_LE] = BC \| BI(0, CR_GT) \| BO_COND_FALSE,
715	[TCG_COND_GT] = BC \| BI(0, CR_GT) \| BO_COND_TRUE,
716	[TCG_COND_LTU] = BC \| BI(0, CR_LT) \| BO_COND_TRUE,
717	[TCG_COND_GEU] = BC \| BI(0, CR_LT) \| BO_COND_FALSE,
718	[TCG_COND_LEU] = BC \| BI(0, CR_GT) \| BO_COND_FALSE,
719	[TCG_COND_GTU] = BC \| BI(0, CR_GT) \| BO_COND_TRUE,
720	};
721
722	/* The low bit here is set if the RA and RB fields must be inverted. */
723	static const uint32_t tcg_to_isel[16] = {
724	[TCG_COND_EQ] = ISEL \| BC_(0, CR_EQ),
725	[TCG_COND_NE] = ISEL \| BC_(0, CR_EQ) \| 1,
726	[TCG_COND_TSTEQ] = ISEL \| BC_(0, CR_EQ),
727	[TCG_COND_TSTNE] = ISEL \| BC_(0, CR_EQ) \| 1,
728	[TCG_COND_LT] = ISEL \| BC_(0, CR_LT),
729	[TCG_COND_GE] = ISEL \| BC_(0, CR_LT) \| 1,
730	[TCG_COND_LE] = ISEL \| BC_(0, CR_GT) \| 1,
731	[TCG_COND_GT] = ISEL \| BC_(0, CR_GT),
732	[TCG_COND_LTU] = ISEL \| BC_(0, CR_LT),
733	[TCG_COND_GEU] = ISEL \| BC_(0, CR_LT) \| 1,
734	[TCG_COND_LEU] = ISEL \| BC_(0, CR_GT) \| 1,
735	[TCG_COND_GTU] = ISEL \| BC_(0, CR_GT),
736	};
737
738	static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
739	intptr_t value, intptr_t addend)
740	{
741	const tcg_insn_unit *target;
742	int16_t lo;
743	int32_t hi;
744
745	value += addend;
746	target = (const tcg_insn_unit *)value;
747
748	switch (type) {
749	case R_PPC_REL14:
750	return reloc_pc14(code_ptr, target);
751	case R_PPC_REL24:
752	return reloc_pc24(code_ptr, target);
753	case R_PPC64_PCREL34:
754	return reloc_pc34(code_ptr, target);
755	case R_PPC_ADDR16:
756	/*
757	* We are (slightly) abusing this relocation type. In particular,
758	* assert that the low 2 bits are zero, and do not modify them.
759	* That way we can use this with LD et al that have opcode bits
760	* in the low 2 bits of the insn.
761	*/
762	if ((value & 3) \|\| value != (int16_t)value) {
763	return false;
764	}
765	code_ptr = (code_ptr & ~0xfffc) \| (value & 0xfffc);
766	break;
767	case R_PPC_ADDR32:
768	/*
769	* We are abusing this relocation type. Again, this points to
770	* a pair of insns, lis + load. This is an absolute address
771	* relocation for PPC32 so the lis cannot be removed.
772	*/
773	lo = value;
774	hi = value - lo;
775	if (hi + lo != value) {
776	return false;
777	}
778	code_ptr[0] = deposit32(code_ptr[0], 0, 16, hi >> 16);
779	code_ptr[1] = deposit32(code_ptr[1], 0, 16, lo);
780	break;
781	default:
782	g_assert_not_reached();
783	}
784	return true;
785	}
786
787	/* Ensure that the prefixed instruction does not cross a 64-byte boundary. */
788	static bool tcg_out_need_prefix_align(TCGContext *s)
789	{
790	return ((uintptr_t)s->code_ptr & 0x3f) == 0x3c;
791	}
792
793	static void tcg_out_prefix_align(TCGContext *s)
794	{
795	if (tcg_out_need_prefix_align(s)) {
796	tcg_out32(s, NOP);
797	}
798	}
799
800	static ptrdiff_t tcg_pcrel_diff_for_prefix(TCGContext s, const void target)
801	{
802	return tcg_pcrel_diff(s, target) - (tcg_out_need_prefix_align(s) ? 4 : 0);
803	}
804
805	/* Output Type 00 Prefix - 8-Byte Load/Store Form (8LS:D) */
806	static void tcg_out_8ls_d(TCGContext *s, tcg_insn_unit opc, unsigned rt,
807	unsigned ra, tcg_target_long imm, bool r)
808	{
809	tcg_insn_unit p, i;
810
811	p = OPCD(1) \| (r << 20) \| ((imm >> 16) & 0x3ffff);
812	i = opc \| TAI(rt, ra, imm);
813
814	tcg_out_prefix_align(s);
815	tcg_out32(s, p);
816	tcg_out32(s, i);
817	}
818
819	/* Output Type 10 Prefix - Modified Load/Store Form (MLS:D) */
820	static void tcg_out_mls_d(TCGContext *s, tcg_insn_unit opc, unsigned rt,
821	unsigned ra, tcg_target_long imm, bool r)
822	{
823	tcg_insn_unit p, i;
824
825	p = OPCD(1) \| (2 << 24) \| (r << 20) \| ((imm >> 16) & 0x3ffff);
826	i = opc \| TAI(rt, ra, imm);
827
828	tcg_out_prefix_align(s);
829	tcg_out32(s, p);
830	tcg_out32(s, i);
831	}
832
833	static void tcg_out_mem_long(TCGContext *s, int opi, int opx, TCGReg rt,
834	TCGReg base, tcg_target_long offset);
835
836	static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
837	{
838	if (ret == arg) {
839	return true;
840	}
841	switch (type) {
842	case TCG_TYPE_I32:
843	case TCG_TYPE_I64:
844	if (ret < TCG_REG_V0) {
845	if (arg < TCG_REG_V0) {
846	tcg_out32(s, OR \| SAB(arg, ret, arg));
847	break;
848	} else if (have_isa_2_07) {
849	tcg_out32(s, (type == TCG_TYPE_I32 ? MFVSRWZ : MFVSRD)
850	\| VRT(arg) \| RA(ret));
851	break;
852	} else {
853	/* Altivec does not support vector->integer moves. */
854	return false;
855	}
856	} else if (arg < TCG_REG_V0) {
857	if (have_isa_2_07) {
858	tcg_out32(s, (type == TCG_TYPE_I32 ? MTVSRWZ : MTVSRD)
859	\| VRT(ret) \| RA(arg));
860	break;
861	} else {
862	/* Altivec does not support integer->vector moves. */
863	return false;
864	}
865	}
866	/* fallthru */
867	case TCG_TYPE_V64:
868	case TCG_TYPE_V128:
869	tcg_debug_assert(ret >= TCG_REG_V0 && arg >= TCG_REG_V0);
870	tcg_out32(s, VOR \| VRT(ret) \| VRA(arg) \| VRB(arg));
871	break;
872	default:
873	g_assert_not_reached();
874	}
875	return true;
876	}
877
878	static void tcg_out_rld_rc(TCGContext *s, int op, TCGReg ra, TCGReg rs,
879	int sh, int mb, bool rc)
880	{
881	sh = SH(sh & 0x1f) \| (((sh >> 5) & 1) << 1);
882	mb = MB64((mb >> 5) \| ((mb << 1) & 0x3f));
883	tcg_out32(s, op \| RA(ra) \| RS(rs) \| sh \| mb \| rc);
884	}
885
886	static void tcg_out_rld(TCGContext *s, int op, TCGReg ra, TCGReg rs,
887	int sh, int mb)
888	{
889	tcg_out_rld_rc(s, op, ra, rs, sh, mb, false);
890	}
891
892	static void tcg_out_rlw_rc(TCGContext *s, int op, TCGReg ra, TCGReg rs,
893	int sh, int mb, int me, bool rc)
894	{
895	tcg_debug_assert((mb & 0x1f) == mb);
896	tcg_debug_assert((me & 0x1f) == me);
897	tcg_out32(s, op \| RA(ra) \| RS(rs) \| SH(sh & 0x1f) \| MB(mb) \| ME(me) \| rc);
898	}
899
900	static void tcg_out_rlw(TCGContext *s, int op, TCGReg ra, TCGReg rs,
901	int sh, int mb, int me)
902	{
903	tcg_out_rlw_rc(s, op, ra, rs, sh, mb, me, false);
904	}
905
906	static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg dst, TCGReg src)
907	{
908	tcg_out32(s, EXTSB \| RA(dst) \| RS(src));
909	}
910
911	static void tcg_out_ext8u(TCGContext *s, TCGReg dst, TCGReg src)
912	{
913	tcg_out32(s, ANDI \| SAI(src, dst, 0xff));
914	}
915
916	static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg dst, TCGReg src)
917	{
918	tcg_out32(s, EXTSH \| RA(dst) \| RS(src));
919	}
920
921	static void tcg_out_ext16u(TCGContext *s, TCGReg dst, TCGReg src)
922	{
923	tcg_out32(s, ANDI \| SAI(src, dst, 0xffff));
924	}
925
926	static void tcg_out_ext32s(TCGContext *s, TCGReg dst, TCGReg src)
927	{
928	tcg_out32(s, EXTSW \| RA(dst) \| RS(src));
929	}
930
931	static void tcg_out_ext32u(TCGContext *s, TCGReg dst, TCGReg src)
932	{
933	tcg_out_rld(s, RLDICL, dst, src, 0, 32);
934	}
935
936	static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg dst, TCGReg src)
937	{
938	tcg_out_ext32s(s, dst, src);
939	}
940
941	static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg dst, TCGReg src)
942	{
943	tcg_out_ext32u(s, dst, src);
944	}
945
946	static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rn)
947	{
948	tcg_out_mov(s, TCG_TYPE_I32, rd, rn);
949	}
950
951	static inline void tcg_out_shli32(TCGContext *s, TCGReg dst, TCGReg src, int c)
952	{
953	tcg_out_rlw(s, RLWINM, dst, src, c, 0, 31 - c);
954	}
955
956	static inline void tcg_out_shli64(TCGContext *s, TCGReg dst, TCGReg src, int c)
957	{
958	tcg_out_rld(s, RLDICR, dst, src, c, 63 - c);
959	}
960
961	static inline void tcg_out_sari32(TCGContext *s, TCGReg dst, TCGReg src, int c)
962	{
963	/* Limit immediate shift count lest we create an illegal insn. */
964	tcg_out32(s, SRAWI \| RA(dst) \| RS(src) \| SH(c & 31));
965	}
966
967	static inline void tcg_out_shri32(TCGContext *s, TCGReg dst, TCGReg src, int c)
968	{
969	tcg_out_rlw(s, RLWINM, dst, src, 32 - c, c, 31);
970	}
971
972	static inline void tcg_out_shri64(TCGContext *s, TCGReg dst, TCGReg src, int c)
973	{
974	tcg_out_rld(s, RLDICL, dst, src, 64 - c, c);
975	}
976
977	static inline void tcg_out_sari64(TCGContext *s, TCGReg dst, TCGReg src, int c)
978	{
979	tcg_out32(s, SRADI \| RA(dst) \| RS(src) \| SH(c & 0x1f) \| ((c >> 4) & 2));
980	}
981
982	static void tcg_out_addpcis(TCGContext *s, TCGReg dst, intptr_t imm)
983	{
984	uint32_t d0, d1, d2;
985
986	tcg_debug_assert((imm & 0xffff) == 0);
987	tcg_debug_assert(imm == (int32_t)imm);
988
989	d2 = extract32(imm, 16, 1);
990	d1 = extract32(imm, 17, 5);
991	d0 = extract32(imm, 22, 10);
992	tcg_out32(s, ADDPCIS \| RT(dst) \| (d1 << 16) \| (d0 << 6) \| d2);
993	}
994
995	/* Emit a move into ret of arg, if it can be done in one insn. */
996	static bool tcg_out_movi_one(TCGContext *s, TCGReg ret, tcg_target_long arg)
997	{
998	if (arg == (int16_t)arg) {
999	tcg_out32(s, ADDI \| TAI(ret, 0, arg));
1000	return true;
1001	}
1002	if (arg == (int32_t)arg && (arg & 0xffff) == 0) {
1003	tcg_out32(s, ADDIS \| TAI(ret, 0, arg >> 16));
1004	return true;
1005	}
1006	return false;
1007	}
1008
1009	static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret,
1010	tcg_target_long arg, bool in_prologue)
1011	{
1012	intptr_t tb_diff;
1013	tcg_target_long tmp;
1014	int shift;
1015
1016	if (type == TCG_TYPE_I32) {
1017	arg = (int32_t)arg;
1018	}
1019
1020	/* Load 16-bit immediates with one insn. */
1021	if (tcg_out_movi_one(s, ret, arg)) {
1022	return;
1023	}
1024
1025	/* Load addresses within the TB with one insn. */
1026	tb_diff = ppc_tbrel_diff(s, (void *)arg);
1027	if (!in_prologue && USE_REG_TB && tb_diff == (int16_t)tb_diff) {
1028	tcg_out32(s, ADDI \| TAI(ret, TCG_REG_TB, tb_diff));
1029	return;
1030	}
1031
1032	/*
1033	* Load values up to 34 bits, and pc-relative addresses,
1034	* with one prefixed insn.
1035	*/
1036	if (have_isa_3_10) {
1037	if (arg == sextract64(arg, 0, 34)) {
1038	/* pli ret,value = paddi ret,0,value,0 */
1039	tcg_out_mls_d(s, ADDI, ret, 0, arg, 0);
1040	return;
1041	}
1042
1043	tmp = tcg_pcrel_diff_for_prefix(s, (void *)arg);
1044	if (tmp == sextract64(tmp, 0, 34)) {
1045	/* pla ret,value = paddi ret,0,value,1 */
1046	tcg_out_mls_d(s, ADDI, ret, 0, tmp, 1);
1047	return;
1048	}
1049	}
1050
1051	/* Load 32-bit immediates with two insns. Note that we've already
1052	eliminated bare ADDIS, so we know both insns are required. */
1053	if (arg == (int32_t)arg) {
1054	tcg_out32(s, ADDIS \| TAI(ret, 0, arg >> 16));
1055	tcg_out32(s, ORI \| SAI(ret, ret, arg));
1056	return;
1057	}
1058	if (arg == (uint32_t)arg && !(arg & 0x8000)) {
1059	tcg_out32(s, ADDI \| TAI(ret, 0, arg));
1060	tcg_out32(s, ORIS \| SAI(ret, ret, arg >> 16));
1061	return;
1062	}
1063
1064	/* Load masked 16-bit value. */
1065	if (arg > 0 && (arg & 0x8000)) {
1066	tmp = arg \| 0x7fff;
1067	if ((tmp & (tmp + 1)) == 0) {
1068	int mb = clz64(tmp + 1) + 1;
1069	tcg_out32(s, ADDI \| TAI(ret, 0, arg));
1070	tcg_out_rld(s, RLDICL, ret, ret, 0, mb);
1071	return;
1072	}
1073	}
1074
1075	/* Load common masks with 2 insns. */
1076	shift = ctz64(arg);
1077	tmp = arg >> shift;
1078	if (tmp == (int16_t)tmp) {
1079	tcg_out32(s, ADDI \| TAI(ret, 0, tmp));
1080	tcg_out_shli64(s, ret, ret, shift);
1081	return;
1082	}
1083	shift = clz64(arg);
1084	if (tcg_out_movi_one(s, ret, arg << shift)) {
1085	tcg_out_shri64(s, ret, ret, shift);
1086	return;
1087	}
1088
1089	/* Load addresses within 2GB with 2 insns. */
1090	if (have_isa_3_00) {
1091	intptr_t hi = tcg_pcrel_diff(s, (void *)arg) - 4;
1092	int16_t lo = hi;
1093
1094	hi -= lo;
1095	if (hi == (int32_t)hi) {
1096	tcg_out_addpcis(s, TCG_REG_TMP2, hi);
1097	tcg_out32(s, ADDI \| TAI(ret, TCG_REG_TMP2, lo));
1098	return;
1099	}
1100	}
1101
1102	/* Load addresses within 2GB of TB with 2 (or rarely 3) insns. */
1103	if (!in_prologue && USE_REG_TB && tb_diff == (int32_t)tb_diff) {
1104	tcg_out_mem_long(s, ADDI, ADD, ret, TCG_REG_TB, tb_diff);
1105	return;
1106	}
1107
1108	/* Use the constant pool, if possible. */
1109	if (!in_prologue && USE_REG_TB) {
1110	new_pool_label(s, arg, R_PPC_ADDR16, s->code_ptr,
1111	ppc_tbrel_diff(s, NULL));
1112	tcg_out32(s, LD \| TAI(ret, TCG_REG_TB, 0));
1113	return;
1114	}
1115	if (have_isa_3_10) {
1116	tcg_out_8ls_d(s, PLD, ret, 0, 0, 1);
1117	new_pool_label(s, arg, R_PPC64_PCREL34, s->code_ptr - 2, 0);
1118	return;
1119	}
1120	if (have_isa_3_00) {
1121	tcg_out_addpcis(s, TCG_REG_TMP2, 0);
1122	new_pool_label(s, arg, R_PPC_REL14, s->code_ptr, 0);
1123	tcg_out32(s, LD \| TAI(ret, TCG_REG_TMP2, 0));
1124	return;
1125	}
1126
1127	tmp = arg >> 31 >> 1;
1128	tcg_out_movi(s, TCG_TYPE_I32, ret, tmp);
1129	if (tmp) {
1130	tcg_out_shli64(s, ret, ret, 32);
1131	}
1132	if (arg & 0xffff0000) {
1133	tcg_out32(s, ORIS \| SAI(ret, ret, arg >> 16));
1134	}
1135	if (arg & 0xffff) {
1136	tcg_out32(s, ORI \| SAI(ret, ret, arg));
1137	}
1138	}
1139
1140	static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
1141	TCGReg ret, int64_t val)
1142	{
1143	uint32_t load_insn;
1144	int rel, low;
1145	intptr_t add;
1146
1147	switch (vece) {
1148	case MO_8:
1149	low = (int8_t)val;
1150	if (low >= -16 && low < 16) {
1151	tcg_out32(s, VSPLTISB \| VRT(ret) \| ((val & 31) << 16));
1152	return;
1153	}
1154	if (have_isa_3_00) {
1155	tcg_out32(s, XXSPLTIB \| VRT(ret) \| ((val & 0xff) << 11));
1156	return;
1157	}
1158	break;
1159
1160	case MO_16:
1161	low = (int16_t)val;
1162	if (low >= -16 && low < 16) {
1163	tcg_out32(s, VSPLTISH \| VRT(ret) \| ((val & 31) << 16));
1164	return;
1165	}
1166	break;
1167
1168	case MO_32:
1169	low = (int32_t)val;
1170	if (low >= -16 && low < 16) {
1171	tcg_out32(s, VSPLTISW \| VRT(ret) \| ((val & 31) << 16));
1172	return;
1173	}
1174	break;
1175	}
1176
1177	/*
1178	* Otherwise we must load the value from the constant pool.
1179	*/
1180	if (USE_REG_TB) {
1181	rel = R_PPC_ADDR16;
1182	add = ppc_tbrel_diff(s, NULL);
1183	} else if (have_isa_3_10) {
1184	if (type == TCG_TYPE_V64) {
1185	tcg_out_8ls_d(s, PLXSD, ret & 31, 0, 0, 1);
1186	new_pool_label(s, val, R_PPC64_PCREL34, s->code_ptr - 2, 0);
1187	} else {
1188	tcg_out_8ls_d(s, PLXV, ret & 31, 0, 0, 1);
1189	new_pool_l2(s, R_PPC64_PCREL34, s->code_ptr - 2, 0, val, val);
1190	}
1191	return;
1192	} else if (have_isa_3_00) {
1193	tcg_out_addpcis(s, TCG_REG_TMP1, 0);
1194	rel = R_PPC_REL14;
1195	add = 0;
1196	} else {
1197	rel = R_PPC_ADDR32;
1198	add = 0;
1199	}
1200
1201	if (have_vsx) {
1202	load_insn = type == TCG_TYPE_V64 ? LXSDX : LXVDSX;
1203	load_insn \|= VRT(ret) \| RB(TCG_REG_TMP1);
1204	new_pool_label(s, val, rel, s->code_ptr, add);
1205	} else {
1206	load_insn = LVX \| VRT(ret) \| RB(TCG_REG_TMP1);
1207	new_pool_l2(s, rel, s->code_ptr, add, val, val);
1208	}
1209
1210	if (USE_REG_TB) {
1211	tcg_out32(s, ADDI \| TAI(TCG_REG_TMP1, 0, 0));
1212	load_insn \|= RA(TCG_REG_TB);
1213	} else if (have_isa_3_00) {
1214	tcg_out32(s, ADDI \| TAI(TCG_REG_TMP1, TCG_REG_TMP1, 0));
1215	} else {
1216	tcg_out32(s, ADDIS \| TAI(TCG_REG_TMP1, 0, 0));
1217	tcg_out32(s, ADDI \| TAI(TCG_REG_TMP1, TCG_REG_TMP1, 0));
1218	}
1219	tcg_out32(s, load_insn);
1220	}
1221
1222	static void tcg_out_movi(TCGContext *s, TCGType type, TCGReg ret,
1223	tcg_target_long arg)
1224	{
1225	switch (type) {
1226	case TCG_TYPE_I32:
1227	case TCG_TYPE_I64:
1228	tcg_debug_assert(ret < TCG_REG_V0);
1229	tcg_out_movi_int(s, type, ret, arg, false);
1230	break;
1231
1232	default:
1233	g_assert_not_reached();
1234	}
1235	}
1236
1237	static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
1238	{
1239	return false;
1240	}
1241
1242	static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
1243	tcg_target_long imm)
1244	{
1245	/* This function is only used for passing structs by reference. */
1246	g_assert_not_reached();
1247	}
1248
1249	static bool mask_operand(uint32_t c, int mb, int me)
1250	{
1251	uint32_t lsb, test;
1252
1253	/* Accept a bit pattern like:
1254	0....01....1
1255	1....10....0
1256	0..01..10..0
1257	Keep track of the transitions. */
1258	if (c == 0 \|\| c == -1) {
1259	return false;
1260	}
1261	test = c;
1262	lsb = test & -test;
1263	test += lsb;
1264	if (test & (test - 1)) {
1265	return false;
1266	}
1267
1268	*me = clz32(lsb);
1269	*mb = test ? clz32(test & -test) + 1 : 0;
1270	return true;
1271	}
1272
1273	static bool mask64_operand(uint64_t c, int mb, int me)
1274	{
1275	uint64_t lsb;
1276
1277	if (c == 0) {
1278	return false;
1279	}
1280
1281	lsb = c & -c;
1282	/* Accept 1..10..0. */
1283	if (c == -lsb) {
1284	*mb = 0;
1285	*me = clz64(lsb);
1286	return true;
1287	}
1288	/* Accept 0..01..1. */
1289	if (lsb == 1 && (c & (c + 1)) == 0) {
1290	*mb = clz64(c + 1) + 1;
1291	*me = 63;
1292	return true;
1293	}
1294	return false;
1295	}
1296
1297	static void tcg_out_andi32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c)
1298	{
1299	int mb, me;
1300
1301	if (mask_operand(c, &mb, &me)) {
1302	tcg_out_rlw(s, RLWINM, dst, src, 0, mb, me);
1303	} else if ((c & 0xffff) == c) {
1304	tcg_out32(s, ANDI \| SAI(src, dst, c));
1305	return;
1306	} else if ((c & 0xffff0000) == c) {
1307	tcg_out32(s, ANDIS \| SAI(src, dst, c >> 16));
1308	return;
1309	} else {
1310	tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R0, c);
1311	tcg_out32(s, AND \| SAB(src, dst, TCG_REG_R0));
1312	}
1313	}
1314
1315	static void tcg_out_andi64(TCGContext *s, TCGReg dst, TCGReg src, uint64_t c)
1316	{
1317	int mb, me;
1318
1319	if (mask64_operand(c, &mb, &me)) {
1320	if (mb == 0) {
1321	tcg_out_rld(s, RLDICR, dst, src, 0, me);
1322	} else {
1323	tcg_out_rld(s, RLDICL, dst, src, 0, mb);
1324	}
1325	} else if ((c & 0xffff) == c) {
1326	tcg_out32(s, ANDI \| SAI(src, dst, c));
1327	return;
1328	} else if ((c & 0xffff0000) == c) {
1329	tcg_out32(s, ANDIS \| SAI(src, dst, c >> 16));
1330	return;
1331	} else {
1332	tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_R0, c);
1333	tcg_out32(s, AND \| SAB(src, dst, TCG_REG_R0));
1334	}
1335	}
1336
1337	static void tcg_out_zori32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c,
1338	int op_lo, int op_hi)
1339	{
1340	if (c >> 16) {
1341	tcg_out32(s, op_hi \| SAI(src, dst, c >> 16));
1342	src = dst;
1343	}
1344	if (c & 0xffff) {
1345	tcg_out32(s, op_lo \| SAI(src, dst, c));
1346	src = dst;
1347	}
1348	}
1349
1350	static void tcg_out_ori32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c)
1351	{
1352	tcg_out_zori32(s, dst, src, c, ORI, ORIS);
1353	}
1354
1355	static void tcg_out_xori32(TCGContext *s, TCGReg dst, TCGReg src, uint32_t c)
1356	{
1357	tcg_out_zori32(s, dst, src, c, XORI, XORIS);
1358	}
1359
1360	static void tcg_out_b(TCGContext s, int mask, const tcg_insn_unit target)
1361	{
1362	ptrdiff_t disp = tcg_pcrel_diff(s, target);
1363	if (in_range_b(disp)) {
1364	tcg_out32(s, B \| (disp & 0x3fffffc) \| mask);
1365	} else {
1366	tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R0, (uintptr_t)target);
1367	tcg_out32(s, MTSPR \| RS(TCG_REG_R0) \| CTR);
1368	tcg_out32(s, BCCTR \| BO_ALWAYS \| mask);
1369	}
1370	}
1371
1372	static void tcg_out_mem_long(TCGContext *s, int opi, int opx, TCGReg rt,
1373	TCGReg base, tcg_target_long offset)
1374	{
1375	tcg_target_long orig = offset, l0, l1, extra = 0, align = 0;
1376	bool is_int_store = false;
1377	TCGReg rs = TCG_REG_TMP1;
1378
1379	switch (opi) {
1380	case LD: case LWA:
1381	align = 3;
1382	/* FALLTHRU */
1383	default:
1384	if (rt > TCG_REG_R0 && rt < TCG_REG_V0) {
1385	rs = rt;
1386	break;
1387	}
1388	break;
1389	case LXSD:
1390	case STXSD:
1391	align = 3;
1392	break;
1393	case LXV:
1394	case STXV:
1395	align = 15;
1396	break;
1397	case STD:
1398	align = 3;
1399	/* FALLTHRU */
1400	case STB: case STH: case STW:
1401	is_int_store = true;
1402	break;
1403	}
1404
1405	/* For unaligned or large offsets, use the prefixed form. */
1406	if (have_isa_3_10
1407	&& (offset != (int16_t)offset \|\| (offset & align))
1408	&& offset == sextract64(offset, 0, 34)) {
1409	/*
1410	* Note that the MLS:D insns retain their un-prefixed opcode,
1411	* while the 8LS:D insns use a different opcode space.
1412	*/
1413	switch (opi) {
1414	case LBZ:
1415	case LHZ:
1416	case LHA:
1417	case LWZ:
1418	case STB:
1419	case STH:
1420	case STW:
1421	case ADDI:
1422	tcg_out_mls_d(s, opi, rt, base, offset, 0);
1423	return;
1424	case LWA:
1425	tcg_out_8ls_d(s, PLWA, rt, base, offset, 0);
1426	return;
1427	case LD:
1428	tcg_out_8ls_d(s, PLD, rt, base, offset, 0);
1429	return;
1430	case STD:
1431	tcg_out_8ls_d(s, PSTD, rt, base, offset, 0);
1432	return;
1433	case LXSD:
1434	tcg_out_8ls_d(s, PLXSD, rt & 31, base, offset, 0);
1435	return;
1436	case STXSD:
1437	tcg_out_8ls_d(s, PSTXSD, rt & 31, base, offset, 0);
1438	return;
1439	case LXV:
1440	tcg_out_8ls_d(s, PLXV, rt & 31, base, offset, 0);
1441	return;
1442	case STXV:
1443	tcg_out_8ls_d(s, PSTXV, rt & 31, base, offset, 0);
1444	return;
1445	}
1446	}
1447
1448	/* For unaligned, or very large offsets, use the indexed form. */
1449	if (offset & align \|\| offset != (int32_t)offset \|\| opi == 0) {
1450	if (rs == base) {
1451	rs = TCG_REG_R0;
1452	}
1453	tcg_debug_assert(!is_int_store \|\| rs != rt);
1454	tcg_out_movi(s, TCG_TYPE_PTR, rs, orig);
1455	tcg_out32(s, opx \| TAB(rt & 31, base, rs));
1456	return;
1457	}
1458
1459	l0 = (int16_t)offset;
1460	offset = (offset - l0) >> 16;
1461	l1 = (int16_t)offset;
1462
1463	if (l1 < 0 && orig >= 0) {
1464	extra = 0x4000;
1465	l1 = (int16_t)(offset - 0x4000);
1466	}
1467	if (l1) {
1468	tcg_out32(s, ADDIS \| TAI(rs, base, l1));
1469	base = rs;
1470	}
1471	if (extra) {
1472	tcg_out32(s, ADDIS \| TAI(rs, base, extra));
1473	base = rs;
1474	}
1475	if (opi != ADDI \|\| base != rt \|\| l0 != 0) {
1476	tcg_out32(s, opi \| TAI(rt & 31, base, l0));
1477	}
1478	}
1479
1480	static void tcg_out_vsldoi(TCGContext *s, TCGReg ret,
1481	TCGReg va, TCGReg vb, int shb)
1482	{
1483	tcg_out32(s, VSLDOI \| VRT(ret) \| VRA(va) \| VRB(vb) \| (shb << 6));
1484	}
1485
1486	static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
1487	TCGReg base, intptr_t offset)
1488	{
1489	int shift;
1490
1491	switch (type) {
1492	case TCG_TYPE_I32:
1493	if (ret < TCG_REG_V0) {
1494	tcg_out_mem_long(s, LWZ, LWZX, ret, base, offset);
1495	break;
1496	}
1497	if (have_isa_2_07 && have_vsx) {
1498	tcg_out_mem_long(s, 0, LXSIWZX, ret, base, offset);
1499	break;
1500	}
1501	tcg_debug_assert((offset & 3) == 0);
1502	tcg_out_mem_long(s, 0, LVEWX, ret, base, offset);
1503	shift = (offset - 4) & 0xc;
1504	if (shift) {
1505	tcg_out_vsldoi(s, ret, ret, ret, shift);
1506	}
1507	break;
1508	case TCG_TYPE_I64:
1509	if (ret < TCG_REG_V0) {
1510	tcg_out_mem_long(s, LD, LDX, ret, base, offset);
1511	break;
1512	}
1513	/* fallthru */
1514	case TCG_TYPE_V64:
1515	tcg_debug_assert(ret >= TCG_REG_V0);
1516	if (have_vsx) {
1517	tcg_out_mem_long(s, have_isa_3_00 ? LXSD : 0, LXSDX,
1518	ret, base, offset);
1519	break;
1520	}
1521	tcg_debug_assert((offset & 7) == 0);
1522	tcg_out_mem_long(s, 0, LVX, ret, base, offset & -16);
1523	if (offset & 8) {
1524	tcg_out_vsldoi(s, ret, ret, ret, 8);
1525	}
1526	break;
1527	case TCG_TYPE_V128:
1528	tcg_debug_assert(ret >= TCG_REG_V0);
1529	tcg_debug_assert((offset & 15) == 0);
1530	tcg_out_mem_long(s, have_isa_3_00 ? LXV : 0,
1531	LVX, ret, base, offset);
1532	break;
1533	default:
1534	g_assert_not_reached();
1535	}
1536	}
1537
1538	static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
1539	TCGReg base, intptr_t offset)
1540	{
1541	int shift;
1542
1543	switch (type) {
1544	case TCG_TYPE_I32:
1545	if (arg < TCG_REG_V0) {
1546	tcg_out_mem_long(s, STW, STWX, arg, base, offset);
1547	break;
1548	}
1549	if (have_isa_2_07 && have_vsx) {
1550	tcg_out_mem_long(s, 0, STXSIWX, arg, base, offset);
1551	break;
1552	}
1553	assert((offset & 3) == 0);
1554	tcg_debug_assert((offset & 3) == 0);
1555	shift = (offset - 4) & 0xc;
1556	if (shift) {
1557	tcg_out_vsldoi(s, TCG_VEC_TMP1, arg, arg, shift);
1558	arg = TCG_VEC_TMP1;
1559	}
1560	tcg_out_mem_long(s, 0, STVEWX, arg, base, offset);
1561	break;
1562	case TCG_TYPE_I64:
1563	if (arg < TCG_REG_V0) {
1564	tcg_out_mem_long(s, STD, STDX, arg, base, offset);
1565	break;
1566	}
1567	/* fallthru */
1568	case TCG_TYPE_V64:
1569	tcg_debug_assert(arg >= TCG_REG_V0);
1570	if (have_vsx) {
1571	tcg_out_mem_long(s, have_isa_3_00 ? STXSD : 0,
1572	STXSDX, arg, base, offset);
1573	break;
1574	}
1575	tcg_debug_assert((offset & 7) == 0);
1576	if (offset & 8) {
1577	tcg_out_vsldoi(s, TCG_VEC_TMP1, arg, arg, 8);
1578	arg = TCG_VEC_TMP1;
1579	}
1580	tcg_out_mem_long(s, 0, STVEWX, arg, base, offset);
1581	tcg_out_mem_long(s, 0, STVEWX, arg, base, offset + 4);
1582	break;
1583	case TCG_TYPE_V128:
1584	tcg_debug_assert(arg >= TCG_REG_V0);
1585	tcg_out_mem_long(s, have_isa_3_00 ? STXV : 0,
1586	STVX, arg, base, offset);
1587	break;
1588	default:
1589	g_assert_not_reached();
1590	}
1591	}
1592
1593	static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
1594	TCGReg base, intptr_t ofs)
1595	{
1596	return false;
1597	}
1598
1599	/*
1600	* Set dest non-zero if and only if (arg1 & arg2) is non-zero.
1601	* If RC, then also set RC0.
1602	*/
1603	static void tcg_out_test(TCGContext *s, TCGReg dest, TCGReg arg1, TCGArg arg2,
1604	bool const_arg2, TCGType type, bool rc)
1605	{
1606	int mb, me, sh;
1607
1608	if (!const_arg2) {
1609	tcg_out32(s, AND \| SAB(arg1, dest, arg2) \| rc);
1610	return;
1611	}
1612
1613	if (type == TCG_TYPE_I32) {
1614	arg2 = (uint32_t)arg2;
1615	}
1616
1617	if ((arg2 & ~0xffff) == 0) {
1618	tcg_out32(s, ANDI \| SAI(arg1, dest, arg2));
1619	return;
1620	}
1621	if ((arg2 & ~0xffff0000ull) == 0) {
1622	tcg_out32(s, ANDIS \| SAI(arg1, dest, arg2 >> 16));
1623	return;
1624	}
1625	if (arg2 == (uint32_t)arg2 && mask_operand(arg2, &mb, &me)) {
1626	tcg_out_rlw_rc(s, RLWINM, dest, arg1, 0, mb, me, rc);
1627	return;
1628	}
1629	sh = clz64(arg2);
1630	if (mask64_operand(arg2 << sh, &mb, &me)) {
1631	tcg_out_rld_rc(s, RLDICR, dest, arg1, sh, me, rc);
1632	return;
1633	}
1634	/* Constraints should satisfy this. */
1635	g_assert_not_reached();
1636	}
1637
1638	static void tcg_out_cmp(TCGContext *s, int cond, TCGArg arg1, TCGArg arg2,
1639	bool const_arg2, int cr, TCGType type)
1640	{
1641	uint32_t op;
1642
1643	/*
1644	* Simplify the comparisons below wrt CMPI.
1645	* All of the tests are 16-bit, so a 32-bit sign extend always works.
1646	*/
1647	if (type == TCG_TYPE_I32) {
1648	arg2 = (int32_t)arg2;
1649	}
1650
1651	switch (cond) {
1652	case TCG_COND_EQ:
1653	case TCG_COND_NE:
1654	if (const_arg2) {
1655	if ((int16_t)arg2 == arg2) {
1656	op = CMPI;
1657	break;
1658	}
1659	tcg_debug_assert((uint16_t)arg2 == arg2);
1660	op = CMPLI;
1661	break;
1662	}
1663	op = CMPL;
1664	break;
1665
1666	case TCG_COND_TSTEQ:
1667	case TCG_COND_TSTNE:
1668	tcg_debug_assert(cr == 0);
1669	tcg_out_test(s, TCG_REG_R0, arg1, arg2, const_arg2, type, true);
1670	return;
1671
1672	case TCG_COND_LT:
1673	case TCG_COND_GE:
1674	case TCG_COND_LE:
1675	case TCG_COND_GT:
1676	if (const_arg2) {
1677	tcg_debug_assert((int16_t)arg2 == arg2);
1678	op = CMPI;
1679	break;
1680	}
1681	op = CMP;
1682	break;
1683
1684	case TCG_COND_LTU:
1685	case TCG_COND_GEU:
1686	case TCG_COND_LEU:
1687	case TCG_COND_GTU:
1688	if (const_arg2) {
1689	tcg_debug_assert((uint16_t)arg2 == arg2);
1690	op = CMPLI;
1691	break;
1692	}
1693	op = CMPL;
1694	break;
1695
1696	default:
1697	g_assert_not_reached();
1698	}
1699	op \|= BF(cr) \| ((type == TCG_TYPE_I64) << 21);
1700	op \|= RA(arg1);
1701	op \|= const_arg2 ? arg2 & 0xffff : RB(arg2);
1702	tcg_out32(s, op);
1703	}
1704
1705	static void tcg_out_setcond_eq0(TCGContext *s, TCGType type,
1706	TCGReg dst, TCGReg src, bool neg)
1707	{
1708	if (neg && type == TCG_TYPE_I64) {
1709	/*
1710	* X != 0 implies X + -1 generates a carry.
1711	* RT = (~X + X) + CA
1712	* = -1 + CA
1713	* = CA ? 0 : -1
1714	*/
1715	tcg_out32(s, ADDIC \| TAI(TCG_REG_R0, src, -1));
1716	tcg_out32(s, SUBFE \| TAB(dst, src, src));
1717	return;
1718	}
1719
1720	if (type == TCG_TYPE_I32) {
1721	tcg_out32(s, CNTLZW \| RS(src) \| RA(dst));
1722	tcg_out_shri32(s, dst, dst, 5);
1723	} else {
1724	tcg_out32(s, CNTLZD \| RS(src) \| RA(dst));
1725	tcg_out_shri64(s, dst, dst, 6);
1726	}
1727	if (neg) {
1728	tcg_out32(s, NEG \| RT(dst) \| RA(dst));
1729	}
1730	}
1731
1732	static void tcg_out_setcond_ne0(TCGContext *s, TCGType type,
1733	TCGReg dst, TCGReg src, bool neg)
1734	{
1735	if (!neg && type == TCG_TYPE_I64) {
1736	/*
1737	* X != 0 implies X + -1 generates a carry. Extra addition
1738	* trickery means: R = X-1 + ~X + C = X-1 + (-X+1) + C = C.
1739	*/
1740	tcg_out32(s, ADDIC \| TAI(TCG_REG_R0, src, -1));
1741	tcg_out32(s, SUBFE \| TAB(dst, TCG_REG_R0, src));
1742	return;
1743	}
1744	tcg_out_setcond_eq0(s, type, dst, src, false);
1745	if (neg) {
1746	tcg_out32(s, ADDI \| TAI(dst, dst, -1));
1747	} else {
1748	tcg_out_xori32(s, dst, dst, 1);
1749	}
1750	}
1751
1752	static TCGReg tcg_gen_setcond_xor(TCGContext *s, TCGReg arg1, TCGArg arg2,
1753	bool const_arg2)
1754	{
1755	if (const_arg2) {
1756	if ((uint32_t)arg2 == arg2) {
1757	tcg_out_xori32(s, TCG_REG_R0, arg1, arg2);
1758	} else {
1759	tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_R0, arg2);
1760	tcg_out32(s, XOR \| SAB(arg1, TCG_REG_R0, TCG_REG_R0));
1761	}
1762	} else {
1763	tcg_out32(s, XOR \| SAB(arg1, TCG_REG_R0, arg2));
1764	}
1765	return TCG_REG_R0;
1766	}
1767
1768	static void tcg_out_setcond(TCGContext *s, TCGType type, TCGCond cond,
1769	TCGReg arg0, TCGReg arg1, TCGArg arg2,
1770	bool const_arg2, bool neg)
1771	{
1772	int sh;
1773	bool inv;
1774
1775	/* Ignore high bits of a potential constant arg2. */
1776	if (type == TCG_TYPE_I32) {
1777	arg2 = (uint32_t)arg2;
1778	}
1779
1780	/* With SETBC/SETBCR, we can always implement with 2 insns. */
1781	if (have_isa_3_10) {
1782	tcg_insn_unit bi, opc;
1783
1784	tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 0, type);
1785
1786	/* Re-use tcg_to_bc for BI and BO_COND_{TRUE,FALSE}. */
1787	bi = tcg_to_bc[cond] & (0x1f << 16);
1788	if (tcg_to_bc[cond] & BO(8)) {
1789	opc = neg ? SETNBC : SETBC;
1790	} else {
1791	opc = neg ? SETNBCR : SETBCR;
1792	}
1793	tcg_out32(s, opc \| RT(arg0) \| bi);
1794	return;
1795	}
1796
1797	/* Handle common and trivial cases before handling anything else. */
1798	if (arg2 == 0) {
1799	switch (cond) {
1800	case TCG_COND_EQ:
1801	tcg_out_setcond_eq0(s, type, arg0, arg1, neg);
1802	return;
1803	case TCG_COND_NE:
1804	tcg_out_setcond_ne0(s, type, arg0, arg1, neg);
1805	return;
1806	case TCG_COND_GE:
1807	tcg_out32(s, NOR \| SAB(arg1, arg0, arg1));
1808	arg1 = arg0;
1809	/* FALLTHRU */
1810	case TCG_COND_LT:
1811	/* Extract the sign bit. */
1812	if (type == TCG_TYPE_I32) {
1813	if (neg) {
1814	tcg_out_sari32(s, arg0, arg1, 31);
1815	} else {
1816	tcg_out_shri32(s, arg0, arg1, 31);
1817	}
1818	} else {
1819	if (neg) {
1820	tcg_out_sari64(s, arg0, arg1, 63);
1821	} else {
1822	tcg_out_shri64(s, arg0, arg1, 63);
1823	}
1824	}
1825	return;
1826	default:
1827	break;
1828	}
1829	}
1830
1831	/* If we have ISEL, we can implement everything with 3 or 4 insns.
1832	All other cases below are also at least 3 insns, so speed up the
1833	code generator by not considering them and always using ISEL. */
1834	if (have_isel) {
1835	int isel, tab;
1836
1837	tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 0, type);
1838
1839	isel = tcg_to_isel[cond];
1840
1841	tcg_out_movi(s, type, arg0, neg ? -1 : 1);
1842	if (isel & 1) {
1843	/* arg0 = (bc ? 0 : 1) */
1844	tab = TAB(arg0, 0, arg0);
1845	isel &= ~1;
1846	} else {
1847	/* arg0 = (bc ? 1 : 0) */
1848	tcg_out_movi(s, type, TCG_REG_R0, 0);
1849	tab = TAB(arg0, arg0, TCG_REG_R0);
1850	}
1851	tcg_out32(s, isel \| tab);
1852	return;
1853	}
1854
1855	inv = false;
1856	switch (cond) {
1857	case TCG_COND_EQ:
1858	arg1 = tcg_gen_setcond_xor(s, arg1, arg2, const_arg2);
1859	tcg_out_setcond_eq0(s, type, arg0, arg1, neg);
1860	break;
1861
1862	case TCG_COND_NE:
1863	arg1 = tcg_gen_setcond_xor(s, arg1, arg2, const_arg2);
1864	tcg_out_setcond_ne0(s, type, arg0, arg1, neg);
1865	break;
1866
1867	case TCG_COND_TSTEQ:
1868	tcg_out_test(s, TCG_REG_R0, arg1, arg2, const_arg2, type, false);
1869	tcg_out_setcond_eq0(s, type, arg0, TCG_REG_R0, neg);
1870	break;
1871
1872	case TCG_COND_TSTNE:
1873	tcg_out_test(s, TCG_REG_R0, arg1, arg2, const_arg2, type, false);
1874	tcg_out_setcond_ne0(s, type, arg0, TCG_REG_R0, neg);
1875	break;
1876
1877	case TCG_COND_LE:
1878	case TCG_COND_LEU:
1879	inv = true;
1880	/* fall through */
1881	case TCG_COND_GT:
1882	case TCG_COND_GTU:
1883	sh = 30; /* CR7 CR_GT */
1884	goto crtest;
1885
1886	case TCG_COND_GE:
1887	case TCG_COND_GEU:
1888	inv = true;
1889	/* fall through */
1890	case TCG_COND_LT:
1891	case TCG_COND_LTU:
1892	sh = 29; /* CR7 CR_LT */
1893	goto crtest;
1894
1895	crtest:
1896	tcg_out_cmp(s, cond, arg1, arg2, const_arg2, 7, type);
1897	tcg_out32(s, MFOCRF \| RT(TCG_REG_R0) \| FXM(7));
1898	tcg_out_rlw(s, RLWINM, arg0, TCG_REG_R0, sh, 31, 31);
1899	if (neg && inv) {
1900	tcg_out32(s, ADDI \| TAI(arg0, arg0, -1));
1901	} else if (neg) {
1902	tcg_out32(s, NEG \| RT(arg0) \| RA(arg0));
1903	} else if (inv) {
1904	tcg_out_xori32(s, arg0, arg0, 1);
1905	}
1906	break;
1907
1908	default:
1909	g_assert_not_reached();
1910	}
1911	}
1912
1913	static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond,
1914	TCGReg dest, TCGReg arg1, TCGReg arg2)
1915	{
1916	tcg_out_setcond(s, type, cond, dest, arg1, arg2, false, false);
1917	}
1918
1919	static void tgen_setcondi(TCGContext *s, TCGType type, TCGCond cond,
1920	TCGReg dest, TCGReg arg1, tcg_target_long arg2)
1921	{
1922	tcg_out_setcond(s, type, cond, dest, arg1, arg2, true, false);
1923	}
1924
1925	static const TCGOutOpSetcond outop_setcond = {
1926	.base.static_constraint = C_O1_I2(r, r, rC),
1927	.out_rrr = tgen_setcond,
1928	.out_rri = tgen_setcondi,
1929	};
1930
1931	static void tgen_negsetcond(TCGContext *s, TCGType type, TCGCond cond,
1932	TCGReg dest, TCGReg arg1, TCGReg arg2)
1933	{
1934	tcg_out_setcond(s, type, cond, dest, arg1, arg2, false, true);
1935	}
1936
1937	static void tgen_negsetcondi(TCGContext *s, TCGType type, TCGCond cond,
1938	TCGReg dest, TCGReg arg1, tcg_target_long arg2)
1939	{
1940	tcg_out_setcond(s, type, cond, dest, arg1, arg2, true, true);
1941	}
1942
1943	static const TCGOutOpSetcond outop_negsetcond = {
1944	.base.static_constraint = C_O1_I2(r, r, rC),
1945	.out_rrr = tgen_negsetcond,
1946	.out_rri = tgen_negsetcondi,
1947	};
1948
1949	void tcg_out_br(TCGContext s, TCGLabel l)
1950	{
1951	uint32_t insn = B;
1952
1953	if (l->has_value) {
1954	insn \|= reloc_pc24_val(tcg_splitwx_to_rx(s->code_ptr), l->u.value_ptr);
1955	} else {
1956	tcg_out_reloc(s, s->code_ptr, R_PPC_REL24, l, 0);
1957	}
1958	tcg_out32(s, insn);
1959	}
1960
1961	static void tcg_out_bc(TCGContext *s, TCGCond cond, int bd)
1962	{
1963	tcg_out32(s, tcg_to_bc[cond] \| bd);
1964	}
1965
1966	static void tcg_out_bc_lab(TCGContext s, TCGCond cond, TCGLabel l)
1967	{
1968	int bd = 0;
1969	if (l->has_value) {
1970	bd = reloc_pc14_val(tcg_splitwx_to_rx(s->code_ptr), l->u.value_ptr);
1971	} else {
1972	tcg_out_reloc(s, s->code_ptr, R_PPC_REL14, l, 0);
1973	}
1974	tcg_out_bc(s, cond, bd);
1975	}
1976
1977	static void tgen_brcond(TCGContext *s, TCGType type, TCGCond cond,
1978	TCGReg arg1, TCGReg arg2, TCGLabel *l)
1979	{
1980	tcg_out_cmp(s, cond, arg1, arg2, false, 0, type);
1981	tcg_out_bc_lab(s, cond, l);
1982	}
1983
1984	static void tgen_brcondi(TCGContext *s, TCGType type, TCGCond cond,
1985	TCGReg arg1, tcg_target_long arg2, TCGLabel *l)
1986	{
1987	tcg_out_cmp(s, cond, arg1, arg2, true, 0, type);
1988	tcg_out_bc_lab(s, cond, l);
1989	}
1990
1991	static const TCGOutOpBrcond outop_brcond = {
1992	.base.static_constraint = C_O0_I2(r, rC),
1993	.out_rr = tgen_brcond,
1994	.out_ri = tgen_brcondi,
1995	};
1996
1997	static void tgen_movcond(TCGContext *s, TCGType type, TCGCond cond,
1998	TCGReg dest, TCGReg c1, TCGArg c2, bool const_c2,
1999	TCGArg v1, bool const_v1, TCGArg v2, bool const_v2)
2000	{
2001	/* If for some reason both inputs are zero, don't produce bad code. */
2002	if (v1 == 0 && v2 == 0) {
2003	tcg_out_movi(s, type, dest, 0);
2004	return;
2005	}
2006
2007	tcg_out_cmp(s, cond, c1, c2, const_c2, 0, type);
2008
2009	if (have_isel) {
2010	int isel = tcg_to_isel[cond];
2011
2012	/* Swap the V operands if the operation indicates inversion. */
2013	if (isel & 1) {
2014	int t = v1;
2015	v1 = v2;
2016	v2 = t;
2017	isel &= ~1;
2018	}
2019	/* V1 == 0 is handled by isel; V2 == 0 must be handled by hand. */
2020	if (v2 == 0) {
2021	tcg_out_movi(s, type, TCG_REG_R0, 0);
2022	}
2023	tcg_out32(s, isel \| TAB(dest, v1, v2));
2024	} else {
2025	if (dest == v2) {
2026	cond = tcg_invert_cond(cond);
2027	v2 = v1;
2028	} else if (dest != v1) {
2029	if (v1 == 0) {
2030	tcg_out_movi(s, type, dest, 0);
2031	} else {
2032	tcg_out_mov(s, type, dest, v1);
2033	}
2034	}
2035	/* Branch forward over one insn */
2036	tcg_out_bc(s, cond, 8);
2037	if (v2 == 0) {
2038	tcg_out_movi(s, type, dest, 0);
2039	} else {
2040	tcg_out_mov(s, type, dest, v2);
2041	}
2042	}
2043	}
2044
2045	static const TCGOutOpMovcond outop_movcond = {
2046	.base.static_constraint = C_O1_I4(r, r, rC, rZ, rZ),
2047	.out = tgen_movcond,
2048	};
2049
2050	static void tcg_out_cntxz(TCGContext *s, TCGType type, uint32_t opc,
2051	TCGArg a0, TCGArg a1, TCGArg a2, bool const_a2)
2052	{
2053	if (const_a2 && a2 == (type == TCG_TYPE_I32 ? 32 : 64)) {
2054	tcg_out32(s, opc \| RA(a0) \| RS(a1));
2055	} else {
2056	tcg_out_cmp(s, TCG_COND_EQ, a1, 0, 1, 0, type);
2057	/* Note that the only other valid constant for a2 is 0. */
2058	if (have_isel) {
2059	tcg_out32(s, opc \| RA(TCG_REG_R0) \| RS(a1));
2060	tcg_out32(s, tcg_to_isel[TCG_COND_EQ] \| TAB(a0, a2, TCG_REG_R0));
2061	} else if (!const_a2 && a0 == a2) {
2062	tcg_out_bc(s, TCG_COND_EQ, 8);
2063	tcg_out32(s, opc \| RA(a0) \| RS(a1));
2064	} else {
2065	tcg_out32(s, opc \| RA(a0) \| RS(a1));
2066	tcg_out_bc(s, TCG_COND_NE, 8);
2067	if (const_a2) {
2068	tcg_out_movi(s, type, a0, 0);
2069	} else {
2070	tcg_out_mov(s, type, a0, a2);
2071	}
2072	}
2073	}
2074	}
2075
2076	static void tcg_out_mb(TCGContext *s, unsigned a0)
2077	{
2078	uint32_t insn;
2079
2080	if (a0 & TCG_MO_ST_LD) {
2081	insn = HWSYNC;
2082	} else {
2083	insn = LWSYNC;
2084	}
2085
2086	tcg_out32(s, insn);
2087	}
2088
2089	static void tcg_out_call_int(TCGContext *s, int lk,
2090	const tcg_insn_unit *target)
2091	{
2092	#ifdef _CALL_AIX
2093	/* Look through the descriptor. If the branch is in range, and we
2094	don't have to spend too much effort on building the toc. */
2095	const void tgt = ((const void const *)target)[0];
2096	uintptr_t toc = ((const uintptr_t *)target)[1];
2097	intptr_t diff = tcg_pcrel_diff(s, tgt);
2098
2099	if (in_range_b(diff) && toc == (uint32_t)toc) {
2100	tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP1, toc);
2101	tcg_out_b(s, lk, tgt);
2102	} else {
2103	/* Fold the low bits of the constant into the addresses below. */
2104	intptr_t arg = (intptr_t)target;
2105	int ofs = (int16_t)arg;
2106
2107	if (ofs + 8 < 0x8000) {
2108	arg -= ofs;
2109	} else {
2110	ofs = 0;
2111	}
2112	tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP1, arg);
2113	tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_TMP1, ofs);
2114	tcg_out32(s, MTSPR \| RA(TCG_REG_R0) \| CTR);
2115	tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R2, TCG_REG_TMP1, ofs + SZP);
2116	tcg_out32(s, BCCTR \| BO_ALWAYS \| lk);
2117	}
2118	#elif defined(_CALL_ELF) && _CALL_ELF == 2
2119	intptr_t diff;
2120
2121	/* In the ELFv2 ABI, we have to set up r12 to contain the destination
2122	address, which the callee uses to compute its TOC address. */
2123	/* FIXME: when the branch is in range, we could avoid r12 load if we
2124	knew that the destination uses the same TOC, and what its local
2125	entry point offset is. */
2126	tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R12, (intptr_t)target);
2127
2128	diff = tcg_pcrel_diff(s, target);
2129	if (in_range_b(diff)) {
2130	tcg_out_b(s, lk, target);
2131	} else {
2132	tcg_out32(s, MTSPR \| RS(TCG_REG_R12) \| CTR);
2133	tcg_out32(s, BCCTR \| BO_ALWAYS \| lk);
2134	}
2135	#else
2136	tcg_out_b(s, lk, target);
2137	#endif
2138	}
2139
2140	static void tcg_out_call(TCGContext s, const tcg_insn_unit target,
2141	const TCGHelperInfo *info)
2142	{
2143	tcg_out_call_int(s, LK, target);
2144	}
2145
2146	static const uint32_t qemu_ldx_opc[(MO_SSIZE + MO_BSWAP) + 1] = {
2147	[MO_UB] = LBZX,
2148	[MO_UW] = LHZX,
2149	[MO_UL] = LWZX,
2150	[MO_UQ] = LDX,
2151	[MO_SW] = LHAX,
2152	[MO_SL] = LWAX,
2153	[MO_BSWAP \| MO_UB] = LBZX,
2154	[MO_BSWAP \| MO_UW] = LHBRX,
2155	[MO_BSWAP \| MO_UL] = LWBRX,
2156	[MO_BSWAP \| MO_UQ] = LDBRX,
2157	};
2158
2159	static const uint32_t qemu_stx_opc[(MO_SIZE + MO_BSWAP) + 1] = {
2160	[MO_UB] = STBX,
2161	[MO_UW] = STHX,
2162	[MO_UL] = STWX,
2163	[MO_UQ] = STDX,
2164	[MO_BSWAP \| MO_UB] = STBX,
2165	[MO_BSWAP \| MO_UW] = STHBRX,
2166	[MO_BSWAP \| MO_UL] = STWBRX,
2167	[MO_BSWAP \| MO_UQ] = STDBRX,
2168	};
2169
2170	static TCGReg ldst_ra_gen(TCGContext s, const TCGLabelQemuLdst l, int arg)
2171	{
2172	if (arg < 0) {
2173	arg = TCG_REG_TMP1;
2174	}
2175	tcg_out32(s, MFSPR \| RT(arg) \| LR);
2176	return arg;
2177	}
2178
2179	/*
2180	* For the purposes of ppc32 sorting 4 input registers into 4 argument
2181	* registers, there is an outside chance we would require 3 temps.
2182	*/
2183	static const TCGLdstHelperParam ldst_helper_param = {
2184	.ra_gen = ldst_ra_gen,
2185	.ntmp = 3,
2186	.tmp = { TCG_REG_TMP1, TCG_REG_TMP2, TCG_REG_R0 }
2187	};
2188
2189	static bool tcg_out_qemu_ld_slow_path(TCGContext s, TCGLabelQemuLdst lb)
2190	{
2191	MemOp opc = get_memop(lb->oi);
2192
2193	if (!reloc_pc14(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
2194	return false;
2195	}
2196
2197	tcg_out_ld_helper_args(s, lb, &ldst_helper_param);
2198	tcg_out_call_int(s, LK, qemu_ld_helpers[opc & MO_SIZE]);
2199	tcg_out_ld_helper_ret(s, lb, false, &ldst_helper_param);
2200
2201	tcg_out_b(s, 0, lb->raddr);
2202	return true;
2203	}
2204
2205	static bool tcg_out_qemu_st_slow_path(TCGContext s, TCGLabelQemuLdst lb)
2206	{
2207	MemOp opc = get_memop(lb->oi);
2208
2209	if (!reloc_pc14(lb->label_ptr[0], tcg_splitwx_to_rx(s->code_ptr))) {
2210	return false;
2211	}
2212
2213	tcg_out_st_helper_args(s, lb, &ldst_helper_param);
2214	tcg_out_call_int(s, LK, qemu_st_helpers[opc & MO_SIZE]);
2215
2216	tcg_out_b(s, 0, lb->raddr);
2217	return true;
2218	}
2219
2220	typedef struct {
2221	TCGReg base;
2222	TCGReg index;
2223	TCGAtomAlign aa;
2224	} HostAddress;
2225
2226	bool tcg_target_has_memory_bswap(MemOp memop)
2227	{
2228	TCGAtomAlign aa;
2229
2230	if ((memop & MO_SIZE) <= MO_64) {
2231	return true;
2232	}
2233
2234	/*
2235	* Reject 16-byte memop with 16-byte atomicity,
2236	* but do allow a pair of 64-bit operations.
2237	*/
2238	aa = atom_and_align_for_opc(tcg_ctx, memop, MO_ATOM_IFALIGN, true);
2239	return aa.atom <= MO_64;
2240	}
2241
2242	/* We expect to use a 16-bit negative offset from ENV. */
2243	#define MIN_TLB_MASK_TABLE_OFS -32768
2244
2245	/*
2246	* For system-mode, perform the TLB load and compare.
2247	* For user-mode, perform any required alignment tests.
2248	* In both cases, return a TCGLabelQemuLdst structure if the slow path
2249	* is required and fill in @h with the host address for the fast path.
2250	*/
2251	static TCGLabelQemuLdst prepare_host_addr(TCGContext s, HostAddress *h,
2252	TCGReg addr, MemOpIdx oi, bool is_ld)
2253	{
2254	TCGType addr_type = s->addr_type;
2255	TCGLabelQemuLdst *ldst = NULL;
2256	MemOp opc = get_memop(oi);
2257	MemOp a_bits, s_bits;
2258
2259	/*
2260	* Book II, Section 1.4, Single-Copy Atomicity, specifies:
2261	*
2262	* Before 3.0, "An access that is not atomic is performed as a set of
2263	* smaller disjoint atomic accesses. In general, the number and alignment
2264	* of these accesses are implementation-dependent." Thus MO_ATOM_IFALIGN.
2265	*
2266	* As of 3.0, "the non-atomic access is performed as described in
2267	* the corresponding list", which matches MO_ATOM_SUBALIGN.
2268	*/
2269	s_bits = opc & MO_SIZE;
2270	h->aa = atom_and_align_for_opc(s, opc,
2271	have_isa_3_00 ? MO_ATOM_SUBALIGN
2272	: MO_ATOM_IFALIGN,
2273	s_bits == MO_128);
2274	a_bits = h->aa.align;
2275
2276	if (tcg_use_softmmu) {
2277	int mem_index = get_mmuidx(oi);
2278	int cmp_off = is_ld ? offsetof(CPUTLBEntry, addr_read)
2279	: offsetof(CPUTLBEntry, addr_write);
2280	int fast_off = tlb_mask_table_ofs(s, mem_index);
2281	int mask_off = fast_off + offsetof(CPUTLBDescFast, mask);
2282	int table_off = fast_off + offsetof(CPUTLBDescFast, table);
2283
2284	ldst = new_ldst_label(s);
2285	ldst->is_ld = is_ld;
2286	ldst->oi = oi;
2287	ldst->addr_reg = addr;
2288
2289	/* Load tlb_mask[mmu_idx] and tlb_table[mmu_idx]. */
2290	tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, TCG_AREG0, mask_off);
2291	tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP2, TCG_AREG0, table_off);
2292
2293	/* Extract the page index, shifted into place for tlb index. */
2294	tcg_out_shri64(s, TCG_REG_R0, addr,
2295	TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
2296	tcg_out32(s, AND \| SAB(TCG_REG_TMP1, TCG_REG_TMP1, TCG_REG_R0));
2297
2298	/*
2299	* Load the TLB comparator into TMP2.
2300	* For 64-bit host, always load the entire 64-bit slot for simplicity.
2301	* We will ignore the high bits with tcg_out_cmp(..., addr_type).
2302	*/
2303	if (cmp_off == 0) {
2304	tcg_out32(s, LDUX \| TAB(TCG_REG_TMP2, TCG_REG_TMP1, TCG_REG_TMP2));
2305	} else {
2306	tcg_out32(s, ADD \| TAB(TCG_REG_TMP1, TCG_REG_TMP1, TCG_REG_TMP2));
2307	tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP2, TCG_REG_TMP1, cmp_off);
2308	}
2309
2310	/*
2311	* Load the TLB addend for use on the fast path.
2312	* Do this asap to minimize any load use delay.
2313	*/
2314	tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, TCG_REG_TMP1,
2315	offsetof(CPUTLBEntry, addend));
2316
2317	/* Clear the non-page, non-alignment bits from the address in R0. */
2318	TCGReg t = addr;
2319
2320	/*
2321	* If the access is unaligned, we need to make sure we fail if we
2322	* cross a page boundary. The trick is to add the access size-1
2323	* to the address before masking the low bits. That will make the
2324	* address overflow to the next page if we cross a page boundary,
2325	* which will then force a mismatch of the TLB compare.
2326	*/
2327	if (a_bits < s_bits) {
2328	unsigned a_mask = (1 << a_bits) - 1;
2329	unsigned s_mask = (1 << s_bits) - 1;
2330	tcg_out32(s, ADDI \| TAI(TCG_REG_R0, t, s_mask - a_mask));
2331	t = TCG_REG_R0;
2332	}
2333
2334	/* Mask the address for the requested alignment. */
2335	if (addr_type == TCG_TYPE_I32) {
2336	tcg_out_rlw(s, RLWINM, TCG_REG_R0, t, 0,
2337	(32 - a_bits) & 31, 31 - TARGET_PAGE_BITS);
2338	} else if (a_bits == 0) {
2339	tcg_out_rld(s, RLDICR, TCG_REG_R0, t, 0, 63 - TARGET_PAGE_BITS);
2340	} else {
2341	tcg_out_rld(s, RLDICL, TCG_REG_R0, t,
2342	64 - TARGET_PAGE_BITS, TARGET_PAGE_BITS - a_bits);
2343	tcg_out_rld(s, RLDICL, TCG_REG_R0, TCG_REG_R0, TARGET_PAGE_BITS, 0);
2344	}
2345
2346	/* Full comparison into cr0. */
2347	tcg_out_cmp(s, TCG_COND_EQ, TCG_REG_R0, TCG_REG_TMP2, 0, 0, addr_type);
2348
2349	/* Load a pointer into the current opcode w/conditional branch-link. */
2350	ldst->label_ptr[0] = s->code_ptr;
2351	tcg_out_bc(s, TCG_COND_NE, LK);
2352
2353	h->base = TCG_REG_TMP1;
2354	} else {
2355	if (a_bits) {
2356	ldst = new_ldst_label(s);
2357	ldst->is_ld = is_ld;
2358	ldst->oi = oi;
2359	ldst->addr_reg = addr;
2360
2361	/* We are expecting a_bits to max out at 7, much lower than ANDI. */
2362	tcg_debug_assert(a_bits < 16);
2363	tcg_out32(s, ANDI \| SAI(addr, TCG_REG_R0, (1 << a_bits) - 1));
2364
2365	ldst->label_ptr[0] = s->code_ptr;
2366	tcg_out32(s, BC \| BI(0, CR_EQ) \| BO_COND_FALSE \| LK);
2367	}
2368
2369	h->base = guest_base ? TCG_GUEST_BASE_REG : 0;
2370	}
2371
2372	if (addr_type == TCG_TYPE_I32) {
2373	/* Zero-extend the guest address for use in the host address. */
2374	tcg_out_ext32u(s, TCG_REG_TMP2, addr);
2375	h->index = TCG_REG_TMP2;
2376	} else {
2377	h->index = addr;
2378	}
2379
2380	return ldst;
2381	}
2382
2383	static void tcg_out_qemu_ld(TCGContext *s, TCGReg datalo, TCGReg datahi,
2384	TCGReg addr, MemOpIdx oi, TCGType data_type)
2385	{
2386	MemOp opc = get_memop(oi);
2387	TCGLabelQemuLdst *ldst;
2388	HostAddress h;
2389	uint32_t insn = qemu_ldx_opc[opc & (MO_BSWAP \| MO_SSIZE)];
2390
2391	ldst = prepare_host_addr(s, &h, addr, oi, true);
2392
2393	if (!have_isa_2_06 && insn == LDBRX) {
2394	tcg_out32(s, ADDI \| TAI(TCG_REG_R0, h.index, 4));
2395	tcg_out32(s, LWBRX \| TAB(datalo, h.base, h.index));
2396	tcg_out32(s, LWBRX \| TAB(TCG_REG_R0, h.base, TCG_REG_R0));
2397	tcg_out_rld(s, RLDIMI, datalo, TCG_REG_R0, 32, 0);
2398	} else if (insn) {
2399	tcg_out32(s, insn \| TAB(datalo, h.base, h.index));
2400	} else {
2401	insn = qemu_ldx_opc[opc & (MO_SIZE \| MO_BSWAP)];
2402	tcg_out32(s, insn \| TAB(datalo, h.base, h.index));
2403	tcg_out_movext(s, TCG_TYPE_REG, datalo,
2404	TCG_TYPE_REG, opc & MO_SSIZE, datalo);
2405	}
2406
2407	if (ldst) {
2408	ldst->type = data_type;
2409	ldst->datalo_reg = datalo;
2410	ldst->datahi_reg = datahi;
2411	ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
2412	}
2413	}
2414
2415	static void tcg_out_qemu_st(TCGContext *s, TCGReg datalo, TCGReg datahi,
2416	TCGReg addr, MemOpIdx oi, TCGType data_type)
2417	{
2418	MemOp opc = get_memop(oi);
2419	TCGLabelQemuLdst *ldst;
2420	HostAddress h;
2421	uint32_t insn = qemu_stx_opc[opc & (MO_BSWAP \| MO_SIZE)];
2422
2423	ldst = prepare_host_addr(s, &h, addr, oi, false);
2424
2425	if (!have_isa_2_06 && insn == STDBRX) {
2426	tcg_out32(s, STWBRX \| SAB(datalo, h.base, h.index));
2427	tcg_out32(s, ADDI \| TAI(TCG_REG_TMP2, h.index, 4));
2428	tcg_out_shri64(s, TCG_REG_R0, datalo, 32);
2429	tcg_out32(s, STWBRX \| SAB(TCG_REG_R0, h.base, TCG_REG_TMP2));
2430	} else {
2431	tcg_out32(s, insn \| SAB(datalo, h.base, h.index));
2432	}
2433
2434	if (ldst) {
2435	ldst->type = data_type;
2436	ldst->datalo_reg = datalo;
2437	ldst->datahi_reg = datahi;
2438	ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
2439	}
2440	}
2441
2442	static void tcg_out_qemu_ldst_i128(TCGContext *s, TCGReg datalo, TCGReg datahi,
2443	TCGReg addr_reg, MemOpIdx oi, bool is_ld)
2444	{
2445	TCGLabelQemuLdst *ldst;
2446	HostAddress h;
2447	bool need_bswap;
2448	uint32_t insn;
2449	TCGReg index;
2450
2451	ldst = prepare_host_addr(s, &h, addr_reg, oi, is_ld);
2452
2453	/* Compose the final address, as LQ/STQ have no indexing. */
2454	index = h.index;
2455	if (h.base != 0) {
2456	index = TCG_REG_TMP1;
2457	tcg_out32(s, ADD \| TAB(index, h.base, h.index));
2458	}
2459	need_bswap = get_memop(oi) & MO_BSWAP;
2460
2461	if (h.aa.atom == MO_128) {
2462	tcg_debug_assert(!need_bswap);
2463	tcg_debug_assert(datalo & 1);
2464	tcg_debug_assert(datahi == datalo - 1);
2465	tcg_debug_assert(!is_ld \|\| datahi != index);
2466	insn = is_ld ? LQ : STQ;
2467	tcg_out32(s, insn \| TAI(datahi, index, 0));
2468	} else {
2469	TCGReg d1, d2;
2470
2471	if (HOST_BIG_ENDIAN ^ need_bswap) {
2472	d1 = datahi, d2 = datalo;
2473	} else {
2474	d1 = datalo, d2 = datahi;
2475	}
2476
2477	if (need_bswap) {
2478	tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R0, 8);
2479	insn = is_ld ? LDBRX : STDBRX;
2480	tcg_out32(s, insn \| TAB(d1, 0, index));
2481	tcg_out32(s, insn \| TAB(d2, index, TCG_REG_R0));
2482	} else {
2483	insn = is_ld ? LD : STD;
2484	tcg_out32(s, insn \| TAI(d1, index, 0));
2485	tcg_out32(s, insn \| TAI(d2, index, 8));
2486	}
2487	}
2488
2489	if (ldst) {
2490	ldst->type = TCG_TYPE_I128;
2491	ldst->datalo_reg = datalo;
2492	ldst->datahi_reg = datahi;
2493	ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
2494	}
2495	}
2496
2497	static void tgen_qemu_ld(TCGContext *s, TCGType type, TCGReg data,
2498	TCGReg addr, MemOpIdx oi)
2499	{
2500	tcg_out_qemu_ld(s, data, -1, addr, oi, type);
2501	}
2502
2503	static const TCGOutOpQemuLdSt outop_qemu_ld = {
2504	.base.static_constraint = C_O1_I1(r, r),
2505	.out = tgen_qemu_ld,
2506	};
2507
2508	static void tgen_qemu_ld2(TCGContext *s, TCGType type, TCGReg datalo,
2509	TCGReg datahi, TCGReg addr, MemOpIdx oi)
2510	{
2511	tcg_out_qemu_ldst_i128(s, datalo, datahi, addr, oi, true);
2512	}
2513
2514	static const TCGOutOpQemuLdSt2 outop_qemu_ld2 = {
2515	.base.static_constraint = C_N1O1_I1(o, m, r),
2516	.out = tgen_qemu_ld2,
2517	};
2518
2519	static void tgen_qemu_st(TCGContext *s, TCGType type, TCGReg data,
2520	TCGReg addr, MemOpIdx oi)
2521	{
2522	tcg_out_qemu_st(s, data, -1, addr, oi, type);
2523	}
2524
2525	static const TCGOutOpQemuLdSt outop_qemu_st = {
2526	.base.static_constraint = C_O0_I2(r, r),
2527	.out = tgen_qemu_st,
2528	};
2529
2530	static void tgen_qemu_st2(TCGContext *s, TCGType type, TCGReg datalo,
2531	TCGReg datahi, TCGReg addr, MemOpIdx oi)
2532	{
2533	tcg_out_qemu_ldst_i128(s, datalo, datahi, addr, oi, false);
2534	}
2535
2536	static const TCGOutOpQemuLdSt2 outop_qemu_st2 = {
2537	.base.static_constraint = C_O0_I3(o, m, r),
2538	.out = tgen_qemu_st2,
2539	};
2540
2541	static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
2542	{
2543	int i;
2544	for (i = 0; i < count; ++i) {
2545	p[i] = NOP;
2546	}
2547	}
2548
2549	/* Parameters for function call generation, used in tcg.c. */
2550	#define TCG_TARGET_STACK_ALIGN 16
2551
2552	#ifdef _CALL_AIX
2553	# define LINK_AREA_SIZE (6 * SZR)
2554	# define LR_OFFSET (1 * SZR)
2555	# define TCG_TARGET_CALL_STACK_OFFSET (LINK_AREA_SIZE + 8 * SZR)
2556	#elif defined(_CALL_DARWIN)
2557	# define LINK_AREA_SIZE (6 * SZR)
2558	# define LR_OFFSET (2 * SZR)
2559	#else
2560	# if defined(_CALL_ELF) && _CALL_ELF == 2
2561	# define LINK_AREA_SIZE (4 * SZR)
2562	# define LR_OFFSET (1 * SZR)
2563	# endif
2564	#endif
2565	#ifndef LR_OFFSET
2566	# error "Unhandled abi"
2567	#endif
2568	#ifndef TCG_TARGET_CALL_STACK_OFFSET
2569	# define TCG_TARGET_CALL_STACK_OFFSET LINK_AREA_SIZE
2570	#endif
2571
2572	#define CPU_TEMP_BUF_SIZE (CPU_TEMP_BUF_NLONGS * (int)sizeof(long))
2573	#define REG_SAVE_SIZE ((int)ARRAY_SIZE(tcg_target_callee_save_regs) * SZR)
2574
2575	#define FRAME_SIZE ((TCG_TARGET_CALL_STACK_OFFSET \
2576	+ TCG_STATIC_CALL_ARGS_SIZE \
2577	+ CPU_TEMP_BUF_SIZE \
2578	+ REG_SAVE_SIZE \
2579	+ TCG_TARGET_STACK_ALIGN - 1) \
2580	& -TCG_TARGET_STACK_ALIGN)
2581
2582	#define REG_SAVE_BOT (FRAME_SIZE - REG_SAVE_SIZE)
2583
2584	static void tcg_target_qemu_prologue(TCGContext *s)
2585	{
2586	int i;
2587
2588	#ifdef _CALL_AIX
2589	const void desc = (const void )s->code_ptr;
2590	desc[0] = tcg_splitwx_to_rx(desc + 2); /* entry point */
2591	desc[1] = 0; /* environment pointer */
2592	s->code_ptr = (void )(desc + 2); / skip over descriptor */
2593	#endif
2594
2595	tcg_set_frame(s, TCG_REG_CALL_STACK, REG_SAVE_BOT - CPU_TEMP_BUF_SIZE,
2596	CPU_TEMP_BUF_SIZE);
2597
2598	/* Prologue */
2599	tcg_out32(s, MFSPR \| RT(TCG_REG_R0) \| LR);
2600	tcg_out32(s, (SZR == 8 ? STDU : STWU)
2601	\| SAI(TCG_REG_R1, TCG_REG_R1, -FRAME_SIZE));
2602
2603	for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i) {
2604	tcg_out_st(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i],
2605	TCG_REG_R1, REG_SAVE_BOT + i * SZR);
2606	}
2607	tcg_out_st(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_R1, FRAME_SIZE+LR_OFFSET);
2608
2609	if (!tcg_use_softmmu && guest_base) {
2610	tcg_out_movi_int(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base, true);
2611	tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
2612	}
2613
2614	tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
2615	tcg_out32(s, MTSPR \| RS(tcg_target_call_iarg_regs[1]) \| CTR);
2616	tcg_out32(s, BCCTR \| BO_ALWAYS);
2617
2618	/* Epilogue */
2619	tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr);
2620
2621	tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R0, TCG_REG_R1, FRAME_SIZE+LR_OFFSET);
2622	for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i) {
2623	tcg_out_ld(s, TCG_TYPE_REG, tcg_target_callee_save_regs[i],
2624	TCG_REG_R1, REG_SAVE_BOT + i * SZR);
2625	}
2626	tcg_out32(s, MTSPR \| RS(TCG_REG_R0) \| LR);
2627	tcg_out32(s, ADDI \| TAI(TCG_REG_R1, TCG_REG_R1, FRAME_SIZE));
2628	tcg_out32(s, BCLR \| BO_ALWAYS);
2629	}
2630
2631	static void tcg_out_tb_start(TCGContext *s)
2632	{
2633	/* Load TCG_REG_TB. */
2634	if (USE_REG_TB) {
2635	if (have_isa_3_00) {
2636	/* lnia REG_TB */
2637	tcg_out_addpcis(s, TCG_REG_TB, 0);
2638	} else {
2639	/* bcl 20,31,$+4 (preferred form for getting nia) */
2640	tcg_out32(s, BC \| BO_ALWAYS \| BI(7, CR_SO) \| 0x4 \| LK);
2641	tcg_out32(s, MFSPR \| RT(TCG_REG_TB) \| LR);
2642	}
2643	}
2644	}
2645
2646	static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg)
2647	{
2648	tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R3, arg);
2649	tcg_out_b(s, 0, tcg_code_gen_epilogue);
2650	}
2651
2652	static void tcg_out_goto_tb(TCGContext *s, int which)
2653	{
2654	uintptr_t ptr = get_jmp_target_addr(s, which);
2655	int16_t lo;
2656
2657	/* Direct branch will be patched by tb_target_set_jmp_target. */
2658	set_jmp_insn_offset(s, which);
2659	tcg_out32(s, NOP);
2660
2661	/* When branch is out of range, fall through to indirect. */
2662	if (USE_REG_TB) {
2663	ptrdiff_t offset = ppc_tbrel_diff(s, (void *)ptr);
2664	tcg_out_mem_long(s, LD, LDX, TCG_REG_TMP1, TCG_REG_TB, offset);
2665	} else if (have_isa_3_10) {
2666	ptrdiff_t offset = tcg_pcrel_diff_for_prefix(s, (void *)ptr);
2667	tcg_out_8ls_d(s, PLD, TCG_REG_TMP1, 0, offset, 1);
2668	} else if (have_isa_3_00) {
2669	ptrdiff_t offset = tcg_pcrel_diff(s, (void *)ptr) - 4;
2670	lo = offset;
2671	tcg_out_addpcis(s, TCG_REG_TMP1, offset - lo);
2672	tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, TCG_REG_TMP1, lo);
2673	} else {
2674	lo = ptr;
2675	tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_TMP1, ptr - lo);
2676	tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TMP1, TCG_REG_TMP1, lo);
2677	}
2678
2679	tcg_out32(s, MTSPR \| RS(TCG_REG_TMP1) \| CTR);
2680	tcg_out32(s, BCCTR \| BO_ALWAYS);
2681	set_jmp_reset_offset(s, which);
2682	}
2683
2684	static void tcg_out_goto_ptr(TCGContext *s, TCGReg a0)
2685	{
2686	tcg_out32(s, MTSPR \| RS(a0) \| CTR);
2687	tcg_out32(s, ADDI \| TAI(TCG_REG_R3, 0, 0));
2688	tcg_out32(s, BCCTR \| BO_ALWAYS);
2689	}
2690
2691	void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
2692	uintptr_t jmp_rx, uintptr_t jmp_rw)
2693	{
2694	uintptr_t addr = tb->jmp_target_addr[n];
2695	intptr_t diff = addr - jmp_rx;
2696	tcg_insn_unit insn;
2697
2698	if (in_range_b(diff)) {
2699	insn = B \| (diff & 0x3fffffc);
2700	} else {
2701	insn = NOP;
2702	}
2703
2704	qatomic_set((uint32_t *)jmp_rw, insn);
2705	flush_idcache_range(jmp_rx, jmp_rw, 4);
2706	}
2707
2708
2709	static void tgen_add(TCGContext *s, TCGType type,
2710	TCGReg a0, TCGReg a1, TCGReg a2)
2711	{
2712	tcg_out32(s, ADD \| TAB(a0, a1, a2));
2713	}
2714
2715	static void tgen_addi(TCGContext *s, TCGType type,
2716	TCGReg a0, TCGReg a1, tcg_target_long a2)
2717	{
2718	tcg_out_mem_long(s, ADDI, ADD, a0, a1, a2);
2719	}
2720
2721	static const TCGOutOpBinary outop_add = {
2722	.base.static_constraint = C_O1_I2(r, r, rT),
2723	.out_rrr = tgen_add,
2724	.out_rri = tgen_addi,
2725	};
2726
2727	static void tgen_addco_rrr(TCGContext *s, TCGType type,
2728	TCGReg a0, TCGReg a1, TCGReg a2)
2729	{
2730	tcg_out32(s, ADDC \| TAB(a0, a1, a2));
2731	}
2732
2733	static void tgen_addco_rri(TCGContext *s, TCGType type,
2734	TCGReg a0, TCGReg a1, tcg_target_long a2)
2735	{
2736	tcg_out32(s, ADDIC \| TAI(a0, a1, a2));
2737	}
2738
2739	static TCGConstraintSetIndex cset_addco(TCGType type, unsigned flags)
2740	{
2741	/*
2742	* Note that the CA bit is defined based on the word size of the
2743	* environment. So in 64-bit mode it's always carry-out of bit 63.
2744	* The fallback code using deposit works just as well for TCG_TYPE_I32.
2745	*/
2746	return type == TCG_TYPE_REG ? C_O1_I2(r, r, rI) : C_NotImplemented;
2747	}
2748
2749	static const TCGOutOpBinary outop_addco = {
2750	.base.static_constraint = C_Dynamic,
2751	.base.dynamic_constraint = cset_addco,
2752	.out_rrr = tgen_addco_rrr,
2753	.out_rri = tgen_addco_rri,
2754	};
2755
2756	static void tgen_addcio_rrr(TCGContext *s, TCGType type,
2757	TCGReg a0, TCGReg a1, TCGReg a2)
2758	{
2759	tcg_out32(s, ADDE \| TAB(a0, a1, a2));
2760	}
2761
2762	static void tgen_addcio_rri(TCGContext *s, TCGType type,
2763	TCGReg a0, TCGReg a1, tcg_target_long a2)
2764	{
2765	tcg_out32(s, (a2 ? ADDME : ADDZE) \| RT(a0) \| RA(a1));
2766	}
2767
2768	static TCGConstraintSetIndex cset_addcio(TCGType type, unsigned flags)
2769	{
2770	return type == TCG_TYPE_REG ? C_O1_I2(r, r, rZM) : C_NotImplemented;
2771	}
2772
2773	static const TCGOutOpBinary outop_addcio = {
2774	.base.static_constraint = C_Dynamic,
2775	.base.dynamic_constraint = cset_addcio,
2776	.out_rrr = tgen_addcio_rrr,
2777	.out_rri = tgen_addcio_rri,
2778	};
2779
2780	static const TCGOutOpAddSubCarry outop_addci = {
2781	.base.static_constraint = C_Dynamic,
2782	.base.dynamic_constraint = cset_addcio,
2783	.out_rrr = tgen_addcio_rrr,
2784	.out_rri = tgen_addcio_rri,
2785	};
2786
2787	static void tcg_out_set_carry(TCGContext *s)
2788	{
2789	tcg_out32(s, SUBFC \| TAB(TCG_REG_R0, TCG_REG_R0, TCG_REG_R0));
2790	}
2791
2792	static void tgen_and(TCGContext *s, TCGType type,
2793	TCGReg a0, TCGReg a1, TCGReg a2)
2794	{
2795	tcg_out32(s, AND \| SAB(a1, a0, a2));
2796	}
2797
2798	static void tgen_andi(TCGContext *s, TCGType type,
2799	TCGReg a0, TCGReg a1, tcg_target_long a2)
2800	{
2801	if (type == TCG_TYPE_I32) {
2802	tcg_out_andi32(s, a0, a1, a2);
2803	} else {
2804	tcg_out_andi64(s, a0, a1, a2);
2805	}
2806	}
2807
2808	static const TCGOutOpBinary outop_and = {
2809	.base.static_constraint = C_O1_I2(r, r, ri),
2810	.out_rrr = tgen_and,
2811	.out_rri = tgen_andi,
2812	};
2813
2814	static void tgen_andc(TCGContext *s, TCGType type,
2815	TCGReg a0, TCGReg a1, TCGReg a2)
2816	{
2817	tcg_out32(s, ANDC \| SAB(a1, a0, a2));
2818	}
2819
2820	static const TCGOutOpBinary outop_andc = {
2821	.base.static_constraint = C_O1_I2(r, r, r),
2822	.out_rrr = tgen_andc,
2823	};
2824
2825	static void tgen_clz(TCGContext *s, TCGType type,
2826	TCGReg a0, TCGReg a1, TCGReg a2)
2827	{
2828	uint32_t insn = type == TCG_TYPE_I32 ? CNTLZW : CNTLZD;
2829	tcg_out_cntxz(s, type, insn, a0, a1, a2, false);
2830	}
2831
2832	static void tgen_clzi(TCGContext *s, TCGType type,
2833	TCGReg a0, TCGReg a1, tcg_target_long a2)
2834	{
2835	uint32_t insn = type == TCG_TYPE_I32 ? CNTLZW : CNTLZD;
2836	tcg_out_cntxz(s, type, insn, a0, a1, a2, true);
2837	}
2838
2839	static const TCGOutOpBinary outop_clz = {
2840	.base.static_constraint = C_O1_I2(r, r, rZW),
2841	.out_rrr = tgen_clz,
2842	.out_rri = tgen_clzi,
2843	};
2844
2845	static void tgen_ctpop(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
2846	{
2847	uint32_t insn = type == TCG_TYPE_I32 ? CNTPOPW : CNTPOPD;
2848	tcg_out32(s, insn \| SAB(a1, a0, 0));
2849	}
2850
2851	static TCGConstraintSetIndex cset_ctpop(TCGType type, unsigned flags)
2852	{
2853	return have_isa_2_06 ? C_O1_I1(r, r) : C_NotImplemented;
2854	}
2855
2856	static const TCGOutOpUnary outop_ctpop = {
2857	.base.static_constraint = C_Dynamic,
2858	.base.dynamic_constraint = cset_ctpop,
2859	.out_rr = tgen_ctpop,
2860	};
2861
2862	static void tgen_ctz(TCGContext *s, TCGType type,
2863	TCGReg a0, TCGReg a1, TCGReg a2)
2864	{
2865	uint32_t insn = type == TCG_TYPE_I32 ? CNTTZW : CNTTZD;
2866	tcg_out_cntxz(s, type, insn, a0, a1, a2, false);
2867	}
2868
2869	static void tgen_ctzi(TCGContext *s, TCGType type,
2870	TCGReg a0, TCGReg a1, tcg_target_long a2)
2871	{
2872	uint32_t insn = type == TCG_TYPE_I32 ? CNTTZW : CNTTZD;
2873	tcg_out_cntxz(s, type, insn, a0, a1, a2, true);
2874	}
2875
2876	static TCGConstraintSetIndex cset_ctz(TCGType type, unsigned flags)
2877	{
2878	return have_isa_3_00 ? C_O1_I2(r, r, rZW) : C_NotImplemented;
2879	}
2880
2881	static const TCGOutOpBinary outop_ctz = {
2882	.base.static_constraint = C_Dynamic,
2883	.base.dynamic_constraint = cset_ctz,
2884	.out_rrr = tgen_ctz,
2885	.out_rri = tgen_ctzi,
2886	};
2887
2888	static void tgen_eqv(TCGContext *s, TCGType type,
2889	TCGReg a0, TCGReg a1, TCGReg a2)
2890	{
2891	tcg_out32(s, EQV \| SAB(a1, a0, a2));
2892	}
2893
2894	static void tgen_extrh_i64_i32(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1)
2895	{
2896	tcg_out_shri64(s, a0, a1, 32);
2897	}
2898
2899	static const TCGOutOpUnary outop_extrh_i64_i32 = {
2900	.base.static_constraint = C_O1_I1(r, r),
2901	.out_rr = tgen_extrh_i64_i32,
2902	};
2903
2904	static void tgen_divs(TCGContext *s, TCGType type,
2905	TCGReg a0, TCGReg a1, TCGReg a2)
2906	{
2907	uint32_t insn = type == TCG_TYPE_I32 ? DIVW : DIVD;
2908	tcg_out32(s, insn \| TAB(a0, a1, a2));
2909	}
2910
2911	static const TCGOutOpBinary outop_divs = {
2912	.base.static_constraint = C_O1_I2(r, r, r),
2913	.out_rrr = tgen_divs,
2914	};
2915
2916	static const TCGOutOpDivRem outop_divs2 = {
2917	.base.static_constraint = C_NotImplemented,
2918	};
2919
2920	static void tgen_divu(TCGContext *s, TCGType type,
2921	TCGReg a0, TCGReg a1, TCGReg a2)
2922	{
2923	uint32_t insn = type == TCG_TYPE_I32 ? DIVWU : DIVDU;
2924	tcg_out32(s, insn \| TAB(a0, a1, a2));
2925	}
2926
2927	static const TCGOutOpBinary outop_divu = {
2928	.base.static_constraint = C_O1_I2(r, r, r),
2929	.out_rrr = tgen_divu,
2930	};
2931
2932	static const TCGOutOpDivRem outop_divu2 = {
2933	.base.static_constraint = C_NotImplemented,
2934	};
2935
2936	static const TCGOutOpBinary outop_eqv = {
2937	.base.static_constraint = C_O1_I2(r, r, r),
2938	.out_rrr = tgen_eqv,
2939	};
2940
2941	static void tgen_mul(TCGContext *s, TCGType type,
2942	TCGReg a0, TCGReg a1, TCGReg a2)
2943	{
2944	uint32_t insn = type == TCG_TYPE_I32 ? MULLW : MULLD;
2945	tcg_out32(s, insn \| TAB(a0, a1, a2));
2946	}
2947
2948	static void tgen_muli(TCGContext *s, TCGType type,
2949	TCGReg a0, TCGReg a1, tcg_target_long a2)
2950	{
2951	tcg_out32(s, MULLI \| TAI(a0, a1, a2));
2952	}
2953
2954	static const TCGOutOpBinary outop_mul = {
2955	.base.static_constraint = C_O1_I2(r, r, rI),
2956	.out_rrr = tgen_mul,
2957	.out_rri = tgen_muli,
2958	};
2959
2960	static const TCGOutOpMul2 outop_muls2 = {
2961	.base.static_constraint = C_NotImplemented,
2962	};
2963
2964	static void tgen_mulsh(TCGContext *s, TCGType type,
2965	TCGReg a0, TCGReg a1, TCGReg a2)
2966	{
2967	uint32_t insn = type == TCG_TYPE_I32 ? MULHW : MULHD;
2968	tcg_out32(s, insn \| TAB(a0, a1, a2));
2969	}
2970
2971	static const TCGOutOpBinary outop_mulsh = {
2972	.base.static_constraint = C_O1_I2(r, r, r),
2973	.out_rrr = tgen_mulsh,
2974	};
2975
2976	static const TCGOutOpMul2 outop_mulu2 = {
2977	.base.static_constraint = C_NotImplemented,
2978	};
2979
2980	static void tgen_muluh(TCGContext *s, TCGType type,
2981	TCGReg a0, TCGReg a1, TCGReg a2)
2982	{
2983	uint32_t insn = type == TCG_TYPE_I32 ? MULHWU : MULHDU;
2984	tcg_out32(s, insn \| TAB(a0, a1, a2));
2985	}
2986
2987	static const TCGOutOpBinary outop_muluh = {
2988	.base.static_constraint = C_O1_I2(r, r, r),
2989	.out_rrr = tgen_muluh,
2990	};
2991
2992	static void tgen_nand(TCGContext *s, TCGType type,
2993	TCGReg a0, TCGReg a1, TCGReg a2)
2994	{
2995	tcg_out32(s, NAND \| SAB(a1, a0, a2));
2996	}
2997
2998	static const TCGOutOpBinary outop_nand = {
2999	.base.static_constraint = C_O1_I2(r, r, r),
3000	.out_rrr = tgen_nand,
3001	};
3002
3003	static void tgen_nor(TCGContext *s, TCGType type,
3004	TCGReg a0, TCGReg a1, TCGReg a2)
3005	{
3006	tcg_out32(s, NOR \| SAB(a1, a0, a2));
3007	}
3008
3009	static const TCGOutOpBinary outop_nor = {
3010	.base.static_constraint = C_O1_I2(r, r, r),
3011	.out_rrr = tgen_nor,
3012	};
3013
3014	static void tgen_or(TCGContext *s, TCGType type,
3015	TCGReg a0, TCGReg a1, TCGReg a2)
3016	{
3017	tcg_out32(s, OR \| SAB(a1, a0, a2));
3018	}
3019
3020	static void tgen_ori(TCGContext *s, TCGType type,
3021	TCGReg a0, TCGReg a1, tcg_target_long a2)
3022	{
3023	tcg_out_ori32(s, a0, a1, a2);
3024	}
3025
3026	static const TCGOutOpBinary outop_or = {
3027	.base.static_constraint = C_O1_I2(r, r, rU),
3028	.out_rrr = tgen_or,
3029	.out_rri = tgen_ori,
3030	};
3031
3032	static void tgen_orc(TCGContext *s, TCGType type,
3033	TCGReg a0, TCGReg a1, TCGReg a2)
3034	{
3035	tcg_out32(s, ORC \| SAB(a1, a0, a2));
3036	}
3037
3038	static const TCGOutOpBinary outop_orc = {
3039	.base.static_constraint = C_O1_I2(r, r, r),
3040	.out_rrr = tgen_orc,
3041	};
3042
3043	static TCGConstraintSetIndex cset_mod(TCGType type, unsigned flags)
3044	{
3045	return have_isa_3_00 ? C_O1_I2(r, r, r) : C_NotImplemented;
3046	}
3047
3048	static void tgen_rems(TCGContext *s, TCGType type,
3049	TCGReg a0, TCGReg a1, TCGReg a2)
3050	{
3051	uint32_t insn = type == TCG_TYPE_I32 ? MODSW : MODSD;
3052	tcg_out32(s, insn \| TAB(a0, a1, a2));
3053	}
3054
3055	static const TCGOutOpBinary outop_rems = {
3056	.base.static_constraint = C_Dynamic,
3057	.base.dynamic_constraint = cset_mod,
3058	.out_rrr = tgen_rems,
3059	};
3060
3061	static void tgen_remu(TCGContext *s, TCGType type,
3062	TCGReg a0, TCGReg a1, TCGReg a2)
3063	{
3064	uint32_t insn = type == TCG_TYPE_I32 ? MODUW : MODUD;
3065	tcg_out32(s, insn \| TAB(a0, a1, a2));
3066	}
3067
3068	static const TCGOutOpBinary outop_remu = {
3069	.base.static_constraint = C_Dynamic,
3070	.base.dynamic_constraint = cset_mod,
3071	.out_rrr = tgen_remu,
3072	};
3073
3074	static void tgen_rotl(TCGContext *s, TCGType type,
3075	TCGReg a0, TCGReg a1, TCGReg a2)
3076	{
3077	if (type == TCG_TYPE_I32) {
3078	tcg_out32(s, RLWNM \| SAB(a1, a0, a2) \| MB(0) \| ME(31));
3079	} else {
3080	tcg_out32(s, RLDCL \| SAB(a1, a0, a2) \| MB64(0));
3081	}
3082	}
3083
3084	static void tgen_rotli(TCGContext *s, TCGType type,
3085	TCGReg a0, TCGReg a1, tcg_target_long a2)
3086	{
3087	if (type == TCG_TYPE_I32) {
3088	tcg_out_rlw(s, RLWINM, a0, a1, a2, 0, 31);
3089	} else {
3090	tcg_out_rld(s, RLDICL, a0, a1, a2, 0);
3091	}
3092	}
3093
3094	static const TCGOutOpBinary outop_rotl = {
3095	.base.static_constraint = C_O1_I2(r, r, ri),
3096	.out_rrr = tgen_rotl,
3097	.out_rri = tgen_rotli,
3098	};
3099
3100	static const TCGOutOpBinary outop_rotr = {
3101	.base.static_constraint = C_NotImplemented,
3102	};
3103
3104	static void tgen_sar(TCGContext *s, TCGType type,
3105	TCGReg a0, TCGReg a1, TCGReg a2)
3106	{
3107	uint32_t insn = type == TCG_TYPE_I32 ? SRAW : SRAD;
3108	tcg_out32(s, insn \| SAB(a1, a0, a2));
3109	}
3110
3111	static void tgen_sari(TCGContext *s, TCGType type,
3112	TCGReg a0, TCGReg a1, tcg_target_long a2)
3113	{
3114	/* Limit immediate shift count lest we create an illegal insn. */
3115	if (type == TCG_TYPE_I32) {
3116	tcg_out_sari32(s, a0, a1, a2 & 31);
3117	} else {
3118	tcg_out_sari64(s, a0, a1, a2 & 63);
3119	}
3120	}
3121
3122	static const TCGOutOpBinary outop_sar = {
3123	.base.static_constraint = C_O1_I2(r, r, ri),
3124	.out_rrr = tgen_sar,
3125	.out_rri = tgen_sari,
3126	};
3127
3128	static void tgen_shl(TCGContext *s, TCGType type,
3129	TCGReg a0, TCGReg a1, TCGReg a2)
3130	{
3131	uint32_t insn = type == TCG_TYPE_I32 ? SLW : SLD;
3132	tcg_out32(s, insn \| SAB(a1, a0, a2));
3133	}
3134
3135	static void tgen_shli(TCGContext *s, TCGType type,
3136	TCGReg a0, TCGReg a1, tcg_target_long a2)
3137	{
3138	/* Limit immediate shift count lest we create an illegal insn. */
3139	if (type == TCG_TYPE_I32) {
3140	tcg_out_shli32(s, a0, a1, a2 & 31);
3141	} else {
3142	tcg_out_shli64(s, a0, a1, a2 & 63);
3143	}
3144	}
3145
3146	static const TCGOutOpBinary outop_shl = {
3147	.base.static_constraint = C_O1_I2(r, r, ri),
3148	.out_rrr = tgen_shl,
3149	.out_rri = tgen_shli,
3150	};
3151
3152	static void tgen_shr(TCGContext *s, TCGType type,
3153	TCGReg a0, TCGReg a1, TCGReg a2)
3154	{
3155	uint32_t insn = type == TCG_TYPE_I32 ? SRW : SRD;
3156	tcg_out32(s, insn \| SAB(a1, a0, a2));
3157	}
3158
3159	static void tgen_shri(TCGContext *s, TCGType type,
3160	TCGReg a0, TCGReg a1, tcg_target_long a2)
3161	{
3162	/* Limit immediate shift count lest we create an illegal insn. */
3163	if (type == TCG_TYPE_I32) {
3164	tcg_out_shri32(s, a0, a1, a2 & 31);
3165	} else {
3166	tcg_out_shri64(s, a0, a1, a2 & 63);
3167	}
3168	}
3169
3170	static const TCGOutOpBinary outop_shr = {
3171	.base.static_constraint = C_O1_I2(r, r, ri),
3172	.out_rrr = tgen_shr,
3173	.out_rri = tgen_shri,
3174	};
3175
3176	static void tgen_sub(TCGContext *s, TCGType type,
3177	TCGReg a0, TCGReg a1, TCGReg a2)
3178	{
3179	tcg_out32(s, SUBF \| TAB(a0, a2, a1));
3180	}
3181
3182	static void tgen_subfi(TCGContext *s, TCGType type,
3183	TCGReg a0, tcg_target_long a1, TCGReg a2)
3184	{
3185	tcg_out32(s, SUBFIC \| TAI(a0, a2, a1));
3186	}
3187
3188	static const TCGOutOpSubtract outop_sub = {
3189	.base.static_constraint = C_O1_I2(r, rI, r),
3190	.out_rrr = tgen_sub,
3191	.out_rir = tgen_subfi,
3192	};
3193
3194	static void tgen_subbo_rrr(TCGContext *s, TCGType type,
3195	TCGReg a0, TCGReg a1, TCGReg a2)
3196	{
3197	tcg_out32(s, SUBFC \| TAB(a0, a2, a1));
3198	}
3199
3200	static void tgen_subbo_rri(TCGContext *s, TCGType type,
3201	TCGReg a0, TCGReg a1, tcg_target_long a2)
3202	{
3203	if (a2 == 0) {
3204	tcg_out_movi(s, type, TCG_REG_R0, 0);
3205	tgen_subbo_rrr(s, type, a0, a1, TCG_REG_R0);
3206	} else {
3207	tgen_addco_rri(s, type, a0, a1, -a2);
3208	}
3209	}
3210
3211	/* The underlying insn for subfi is subfic. */
3212	#define tgen_subbo_rir tgen_subfi
3213
3214	static void tgen_subbo_rii(TCGContext *s, TCGType type,
3215	TCGReg a0, tcg_target_long a1, tcg_target_long a2)
3216	{
3217	tcg_out_movi(s, type, TCG_REG_R0, a2);
3218	tgen_subbo_rir(s, type, a0, a1, TCG_REG_R0);
3219	}
3220
3221	static TCGConstraintSetIndex cset_subbo(TCGType type, unsigned flags)
3222	{
3223	/* Recall that the CA bit is defined based on the host word size. */
3224	return type == TCG_TYPE_REG ? C_O1_I2(r, rI, rN) : C_NotImplemented;
3225	}
3226
3227	static const TCGOutOpAddSubCarry outop_subbo = {
3228	.base.static_constraint = C_Dynamic,
3229	.base.dynamic_constraint = cset_subbo,
3230	.out_rrr = tgen_subbo_rrr,
3231	.out_rri = tgen_subbo_rri,
3232	.out_rir = tgen_subbo_rir,
3233	.out_rii = tgen_subbo_rii,
3234	};
3235
3236	static void tgen_subbio_rrr(TCGContext *s, TCGType type,
3237	TCGReg a0, TCGReg a1, TCGReg a2)
3238	{
3239	tcg_out32(s, SUBFE \| TAB(a0, a2, a1));
3240	}
3241
3242	static void tgen_subbio_rri(TCGContext *s, TCGType type,
3243	TCGReg a0, TCGReg a1, tcg_target_long a2)
3244	{
3245	tgen_addcio_rri(s, type, a0, a1, ~a2);
3246	}
3247
3248	static void tgen_subbio_rir(TCGContext *s, TCGType type,
3249	TCGReg a0, tcg_target_long a1, TCGReg a2)
3250	{
3251	tcg_debug_assert(a1 == 0 \|\| a1 == -1);
3252	tcg_out32(s, (a1 ? SUBFME : SUBFZE) \| RT(a0) \| RA(a2));
3253	}
3254
3255	static void tgen_subbio_rii(TCGContext *s, TCGType type,
3256	TCGReg a0, tcg_target_long a1, tcg_target_long a2)
3257	{
3258	tcg_out_movi(s, type, TCG_REG_R0, a2);
3259	tgen_subbio_rir(s, type, a0, a1, TCG_REG_R0);
3260	}
3261
3262	static TCGConstraintSetIndex cset_subbio(TCGType type, unsigned flags)
3263	{
3264	return type == TCG_TYPE_REG ? C_O1_I2(r, rZM, rZM) : C_NotImplemented;
3265	}
3266
3267	static const TCGOutOpAddSubCarry outop_subbio = {
3268	.base.static_constraint = C_Dynamic,
3269	.base.dynamic_constraint = cset_subbio,
3270	.out_rrr = tgen_subbio_rrr,
3271	.out_rri = tgen_subbio_rri,
3272	.out_rir = tgen_subbio_rir,
3273	.out_rii = tgen_subbio_rii,
3274	};
3275
3276	#define outop_subbi outop_subbio
3277
3278	static void tcg_out_set_borrow(TCGContext *s)
3279	{
3280	/* borrow = !carry */
3281	tcg_out32(s, ADDIC \| TAI(TCG_REG_R0, TCG_REG_R0, 0));
3282	}
3283
3284	static const TCGOutOpBinary outop_smax = {
3285	.base.static_constraint = C_NotImplemented,
3286	};
3287
3288	static const TCGOutOpBinary outop_smin = {
3289	.base.static_constraint = C_NotImplemented,
3290	};
3291
3292	static const TCGOutOpBinary outop_umax = {
3293	.base.static_constraint = C_NotImplemented,
3294	};
3295
3296	static const TCGOutOpBinary outop_umin = {
3297	.base.static_constraint = C_NotImplemented,
3298	};
3299
3300	static void tgen_xor(TCGContext *s, TCGType type,
3301	TCGReg a0, TCGReg a1, TCGReg a2)
3302	{
3303	tcg_out32(s, XOR \| SAB(a1, a0, a2));
3304	}
3305
3306	static void tgen_xori(TCGContext *s, TCGType type,
3307	TCGReg a0, TCGReg a1, tcg_target_long a2)
3308	{
3309	tcg_out_xori32(s, a0, a1, a2);
3310	}
3311
3312	static const TCGOutOpBinary outop_xor = {
3313	.base.static_constraint = C_O1_I2(r, r, rU),
3314	.out_rrr = tgen_xor,
3315	.out_rri = tgen_xori,
3316	};
3317
3318	static void tgen_bswap16(TCGContext *s, TCGType type,
3319	TCGReg dst, TCGReg src, unsigned flags)
3320	{
3321	TCGReg tmp = dst == src ? TCG_REG_R0 : dst;
3322
3323	if (have_isa_3_10) {
3324	tcg_out32(s, BRH \| RA(dst) \| RS(src));
3325	if (flags & TCG_BSWAP_OS) {
3326	tcg_out_ext16s(s, TCG_TYPE_REG, dst, dst);
3327	} else if ((flags & (TCG_BSWAP_IZ \| TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
3328	tcg_out_ext16u(s, dst, dst);
3329	}
3330	return;
3331	}
3332
3333	/*
3334	* In the following,
3335	* dep(a, b, m) -> (a & ~m) \| (b & m)
3336	*
3337	* Begin with: src = xxxxabcd
3338	*/
3339	/* tmp = rol32(src, 24) & 0x000000ff = 0000000c */
3340	tcg_out_rlw(s, RLWINM, tmp, src, 24, 24, 31);
3341	/* tmp = dep(tmp, rol32(src, 8), 0x0000ff00) = 000000dc */
3342	tcg_out_rlw(s, RLWIMI, tmp, src, 8, 16, 23);
3343
3344	if (flags & TCG_BSWAP_OS) {
3345	tcg_out_ext16s(s, TCG_TYPE_REG, dst, tmp);
3346	} else {
3347	tcg_out_mov(s, TCG_TYPE_REG, dst, tmp);
3348	}
3349	}
3350
3351	static const TCGOutOpBswap outop_bswap16 = {
3352	.base.static_constraint = C_O1_I1(r, r),
3353	.out_rr = tgen_bswap16,
3354	};
3355
3356	static void tgen_bswap32(TCGContext *s, TCGType type,
3357	TCGReg dst, TCGReg src, unsigned flags)
3358	{
3359	TCGReg tmp = dst == src ? TCG_REG_R0 : dst;
3360
3361	if (have_isa_3_10) {
3362	tcg_out32(s, BRW \| RA(dst) \| RS(src));
3363	if (flags & TCG_BSWAP_OS) {
3364	tcg_out_ext32s(s, dst, dst);
3365	} else if ((flags & (TCG_BSWAP_IZ \| TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
3366	tcg_out_ext32u(s, dst, dst);
3367	}
3368	return;
3369	}
3370
3371	/*
3372	* Stolen from gcc's builtin_bswap32.
3373	* In the following,
3374	* dep(a, b, m) -> (a & ~m) \| (b & m)
3375	*
3376	* Begin with: src = xxxxabcd
3377	*/
3378	/* tmp = rol32(src, 8) & 0xffffffff = 0000bcda */
3379	tcg_out_rlw(s, RLWINM, tmp, src, 8, 0, 31);
3380	/* tmp = dep(tmp, rol32(src, 24), 0xff000000) = 0000dcda */
3381	tcg_out_rlw(s, RLWIMI, tmp, src, 24, 0, 7);
3382	/* tmp = dep(tmp, rol32(src, 24), 0x0000ff00) = 0000dcba */
3383	tcg_out_rlw(s, RLWIMI, tmp, src, 24, 16, 23);
3384
3385	if (flags & TCG_BSWAP_OS) {
3386	tcg_out_ext32s(s, dst, tmp);
3387	} else {
3388	tcg_out_mov(s, TCG_TYPE_REG, dst, tmp);
3389	}
3390	}
3391
3392	static const TCGOutOpBswap outop_bswap32 = {
3393	.base.static_constraint = C_O1_I1(r, r),
3394	.out_rr = tgen_bswap32,
3395	};
3396
3397	static void tgen_bswap64(TCGContext *s, TCGType type, TCGReg dst, TCGReg src)
3398	{
3399	TCGReg t0 = dst == src ? TCG_REG_R0 : dst;
3400	TCGReg t1 = dst == src ? dst : TCG_REG_R0;
3401
3402	if (have_isa_3_10) {
3403	tcg_out32(s, BRD \| RA(dst) \| RS(src));
3404	return;
3405	}
3406
3407	/*
3408	* In the following,
3409	* dep(a, b, m) -> (a & ~m) \| (b & m)
3410	*
3411	* Begin with: src = abcdefgh
3412	*/
3413	/* t0 = rol32(src, 8) & 0xffffffff = 0000fghe */
3414	tcg_out_rlw(s, RLWINM, t0, src, 8, 0, 31);
3415	/* t0 = dep(t0, rol32(src, 24), 0xff000000) = 0000hghe */
3416	tcg_out_rlw(s, RLWIMI, t0, src, 24, 0, 7);
3417	/* t0 = dep(t0, rol32(src, 24), 0x0000ff00) = 0000hgfe */
3418	tcg_out_rlw(s, RLWIMI, t0, src, 24, 16, 23);
3419
3420	/* t0 = rol64(t0, 32) = hgfe0000 */
3421	tcg_out_rld(s, RLDICL, t0, t0, 32, 0);
3422	/* t1 = rol64(src, 32) = efghabcd */
3423	tcg_out_rld(s, RLDICL, t1, src, 32, 0);
3424
3425	/* t0 = dep(t0, rol32(t1, 24), 0xffffffff) = hgfebcda */
3426	tcg_out_rlw(s, RLWIMI, t0, t1, 8, 0, 31);
3427	/* t0 = dep(t0, rol32(t1, 24), 0xff000000) = hgfedcda */
3428	tcg_out_rlw(s, RLWIMI, t0, t1, 24, 0, 7);
3429	/* t0 = dep(t0, rol32(t1, 24), 0x0000ff00) = hgfedcba */
3430	tcg_out_rlw(s, RLWIMI, t0, t1, 24, 16, 23);
3431
3432	tcg_out_mov(s, TCG_TYPE_REG, dst, t0);
3433	}
3434
3435	static const TCGOutOpUnary outop_bswap64 = {
3436	.base.static_constraint = C_O1_I1(r, r),
3437	.out_rr = tgen_bswap64,
3438	};
3439
3440	static const TCGOutOpUnary outop_revbit8 = {
3441	.base.static_constraint = C_NotImplemented,
3442	};
3443
3444	static const TCGOutOpBswap outop_revbit32 = {
3445	.base.static_constraint = C_NotImplemented,
3446	};
3447
3448	static const TCGOutOpUnary outop_revbit64 = {
3449	.base.static_constraint = C_NotImplemented,
3450	};
3451
3452	static void tgen_neg(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
3453	{
3454	tcg_out32(s, NEG \| RT(a0) \| RA(a1));
3455	}
3456
3457	static const TCGOutOpUnary outop_neg = {
3458	.base.static_constraint = C_O1_I1(r, r),
3459	.out_rr = tgen_neg,
3460	};
3461
3462	static void tgen_not(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
3463	{
3464	tgen_nor(s, type, a0, a1, a1);
3465	}
3466
3467	static const TCGOutOpUnary outop_not = {
3468	.base.static_constraint = C_O1_I1(r, r),
3469	.out_rr = tgen_not,
3470	};
3471
3472	static void tgen_deposit(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
3473	TCGReg a2, unsigned ofs, unsigned len)
3474	{
3475	if (type == TCG_TYPE_I32) {
3476	tcg_out_rlw(s, RLWIMI, a0, a2, ofs, 32 - ofs - len, 31 - ofs);
3477	} else {
3478	tcg_out_rld(s, RLDIMI, a0, a2, ofs, 64 - ofs - len);
3479	}
3480	}
3481
3482	static void tgen_depositi(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
3483	tcg_target_long a2, unsigned ofs, unsigned len)
3484	{
3485	tgen_andi(s, type, a0, a1, ~MAKE_64BIT_MASK(ofs, len));
3486	}
3487
3488	static const TCGOutOpDeposit outop_deposit = {
3489	.base.static_constraint = C_O1_I2(r, 0, rZ),
3490	.out_rrr = tgen_deposit,
3491	.out_rri = tgen_depositi,
3492	};
3493
3494	static void tgen_extract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
3495	unsigned ofs, unsigned len)
3496	{
3497	if (ofs == 0 && len <= 16) {
3498	tgen_andi(s, TCG_TYPE_I32, a0, a1, (1 << len) - 1);
3499	} else if (type == TCG_TYPE_I32) {
3500	tcg_out_rlw(s, RLWINM, a0, a1, 32 - ofs, 32 - len, 31);
3501	} else {
3502	tcg_out_rld(s, RLDICL, a0, a1, 64 - ofs, 64 - len);
3503	}
3504	}
3505
3506	static const TCGOutOpExtract outop_extract = {
3507	.base.static_constraint = C_O1_I1(r, r),
3508	.out_rr = tgen_extract,
3509	};
3510
3511	static void tgen_sextract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
3512	unsigned ofs, unsigned len)
3513	{
3514	if (ofs == 0) {
3515	switch (len) {
3516	case 8:
3517	tcg_out_ext8s(s, type, a0, a1);
3518	return;
3519	case 16:
3520	tcg_out_ext16s(s, type, a0, a1);
3521	return;
3522	case 32:
3523	tcg_out_ext32s(s, a0, a1);
3524	return;
3525	}
3526	} else if (ofs + len == 32) {
3527	tcg_out_sari32(s, a0, a1, ofs);
3528	return;
3529	}
3530	g_assert_not_reached();
3531	}
3532
3533	static const TCGOutOpExtract outop_sextract = {
3534	.base.static_constraint = C_O1_I1(r, r),
3535	.out_rr = tgen_sextract,
3536	};
3537
3538	static const TCGOutOpExtract2 outop_extract2 = {
3539	.base.static_constraint = C_NotImplemented,
3540	};
3541
3542	static void tgen_ld8u(TCGContext *s, TCGType type, TCGReg dest,
3543	TCGReg base, ptrdiff_t offset)
3544	{
3545	tcg_out_mem_long(s, LBZ, LBZX, dest, base, offset);
3546	}
3547
3548	static const TCGOutOpLoad outop_ld8u = {
3549	.base.static_constraint = C_O1_I1(r, r),
3550	.out = tgen_ld8u,
3551	};
3552
3553	static void tgen_ld8s(TCGContext *s, TCGType type, TCGReg dest,
3554	TCGReg base, ptrdiff_t offset)
3555	{
3556	tgen_ld8u(s, type, dest, base, offset);
3557	tcg_out_ext8s(s, type, dest, dest);
3558	}
3559
3560	static const TCGOutOpLoad outop_ld8s = {
3561	.base.static_constraint = C_O1_I1(r, r),
3562	.out = tgen_ld8s,
3563	};
3564
3565	static void tgen_ld16u(TCGContext *s, TCGType type, TCGReg dest,
3566	TCGReg base, ptrdiff_t offset)
3567	{
3568	tcg_out_mem_long(s, LHZ, LHZX, dest, base, offset);
3569	}
3570
3571	static const TCGOutOpLoad outop_ld16u = {
3572	.base.static_constraint = C_O1_I1(r, r),
3573	.out = tgen_ld16u,
3574	};
3575
3576	static void tgen_ld16s(TCGContext *s, TCGType type, TCGReg dest,
3577	TCGReg base, ptrdiff_t offset)
3578	{
3579	tcg_out_mem_long(s, LHA, LHAX, dest, base, offset);
3580	}
3581
3582	static const TCGOutOpLoad outop_ld16s = {
3583	.base.static_constraint = C_O1_I1(r, r),
3584	.out = tgen_ld16s,
3585	};
3586
3587	static void tgen_ld32u(TCGContext *s, TCGType type, TCGReg dest,
3588	TCGReg base, ptrdiff_t offset)
3589	{
3590	tcg_out_mem_long(s, LWZ, LWZX, dest, base, offset);
3591	}
3592
3593	static const TCGOutOpLoad outop_ld32u = {
3594	.base.static_constraint = C_O1_I1(r, r),
3595	.out = tgen_ld32u,
3596	};
3597
3598	static void tgen_ld32s(TCGContext *s, TCGType type, TCGReg dest,
3599	TCGReg base, ptrdiff_t offset)
3600	{
3601	tcg_out_mem_long(s, LWA, LWAX, dest, base, offset);
3602	}
3603
3604	static const TCGOutOpLoad outop_ld32s = {
3605	.base.static_constraint = C_O1_I1(r, r),
3606	.out = tgen_ld32s,
3607	};
3608
3609	static void tgen_st8(TCGContext *s, TCGType type, TCGReg data,
3610	TCGReg base, ptrdiff_t offset)
3611	{
3612	tcg_out_mem_long(s, STB, STBX, data, base, offset);
3613	}
3614
3615	static const TCGOutOpStore outop_st8 = {
3616	.base.static_constraint = C_O0_I2(r, r),
3617	.out_r = tgen_st8,
3618	};
3619
3620	static void tgen_st16(TCGContext *s, TCGType type, TCGReg data,
3621	TCGReg base, ptrdiff_t offset)
3622	{
3623	tcg_out_mem_long(s, STH, STHX, data, base, offset);
3624	}
3625
3626	static const TCGOutOpStore outop_st16 = {
3627	.base.static_constraint = C_O0_I2(r, r),
3628	.out_r = tgen_st16,
3629	};
3630
3631	static const TCGOutOpStore outop_st = {
3632	.base.static_constraint = C_O0_I2(r, r),
3633	.out_r = tcg_out_st,
3634	};
3635
3636
3637	int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece)
3638	{
3639	switch (opc) {
3640	case INDEX_op_and_vec:
3641	case INDEX_op_or_vec:
3642	case INDEX_op_xor_vec:
3643	case INDEX_op_andc_vec:
3644	case INDEX_op_not_vec:
3645	case INDEX_op_nor_vec:
3646	case INDEX_op_eqv_vec:
3647	case INDEX_op_nand_vec:
3648	return 1;
3649	case INDEX_op_orc_vec:
3650	return have_isa_2_07;
3651	case INDEX_op_add_vec:
3652	case INDEX_op_sub_vec:
3653	case INDEX_op_smax_vec:
3654	case INDEX_op_smin_vec:
3655	case INDEX_op_umax_vec:
3656	case INDEX_op_umin_vec:
3657	case INDEX_op_shlv_vec:
3658	case INDEX_op_shrv_vec:
3659	case INDEX_op_sarv_vec:
3660	case INDEX_op_rotlv_vec:
3661	return vece <= MO_32 \|\| have_isa_2_07;
3662	case INDEX_op_ssadd_vec:
3663	case INDEX_op_sssub_vec:
3664	case INDEX_op_usadd_vec:
3665	case INDEX_op_ussub_vec:
3666	return vece <= MO_32;
3667	case INDEX_op_shli_vec:
3668	case INDEX_op_shri_vec:
3669	case INDEX_op_sari_vec:
3670	case INDEX_op_rotli_vec:
3671	return vece <= MO_32 \|\| have_isa_2_07 ? -1 : 0;
3672	case INDEX_op_cmp_vec:
3673	case INDEX_op_cmpsel_vec:
3674	return vece <= MO_32 \|\| have_isa_2_07 ? 1 : 0;
3675	case INDEX_op_neg_vec:
3676	return vece >= MO_32 && have_isa_3_00;
3677	case INDEX_op_mul_vec:
3678	switch (vece) {
3679	case MO_8:
3680	case MO_16:
3681	return -1;
3682	case MO_32:
3683	return have_isa_2_07 ? 1 : -1;
3684	case MO_64:
3685	return have_isa_3_10;
3686	}
3687	return 0;
3688	case INDEX_op_bitsel_vec:
3689	return have_vsx;
3690	case INDEX_op_rotrv_vec:
3691	return -1;
3692	default:
3693	return 0;
3694	}
3695	}
3696
3697	static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
3698	TCGReg dst, TCGReg src)
3699	{
3700	tcg_debug_assert(dst >= TCG_REG_V0);
3701
3702	/* Splat from integer reg allowed via constraints for v3.00. */
3703	if (src < TCG_REG_V0) {
3704	tcg_debug_assert(have_isa_3_00);
3705	switch (vece) {
3706	case MO_64:
3707	tcg_out32(s, MTVSRDD \| VRT(dst) \| RA(src) \| RB(src));
3708	return true;
3709	case MO_32:
3710	tcg_out32(s, MTVSRWS \| VRT(dst) \| RA(src));
3711	return true;
3712	default:
3713	/* Fail, so that we fall back on either dupm or mov+dup. */
3714	return false;
3715	}
3716	}
3717
3718	/*
3719	* Recall we use (or emulate) VSX integer loads, so the integer is
3720	* right justified within the left (zero-index) double-word.
3721	*/
3722	switch (vece) {
3723	case MO_8:
3724	tcg_out32(s, VSPLTB \| VRT(dst) \| VRB(src) \| (7 << 16));
3725	break;
3726	case MO_16:
3727	tcg_out32(s, VSPLTH \| VRT(dst) \| VRB(src) \| (3 << 16));
3728	break;
3729	case MO_32:
3730	tcg_out32(s, VSPLTW \| VRT(dst) \| VRB(src) \| (1 << 16));
3731	break;
3732	case MO_64:
3733	if (have_vsx) {
3734	tcg_out32(s, XXPERMDI \| VRT(dst) \| VRA(src) \| VRB(src));
3735	break;
3736	}
3737	tcg_out_vsldoi(s, TCG_VEC_TMP1, src, src, 8);
3738	tcg_out_vsldoi(s, dst, TCG_VEC_TMP1, src, 8);
3739	break;
3740	default:
3741	g_assert_not_reached();
3742	}
3743	return true;
3744	}
3745
3746	static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
3747	TCGReg out, TCGReg base, intptr_t offset)
3748	{
3749	int elt;
3750
3751	tcg_debug_assert(out >= TCG_REG_V0);
3752	switch (vece) {
3753	case MO_8:
3754	if (have_isa_3_00) {
3755	tcg_out_mem_long(s, LXV, LVX, out, base, offset & -16);
3756	} else {
3757	tcg_out_mem_long(s, 0, LVEBX, out, base, offset);
3758	}
3759	elt = extract32(offset, 0, 4);
3760	#if !HOST_BIG_ENDIAN
3761	elt ^= 15;
3762	#endif
3763	tcg_out32(s, VSPLTB \| VRT(out) \| VRB(out) \| (elt << 16));
3764	break;
3765	case MO_16:
3766	tcg_debug_assert((offset & 1) == 0);
3767	if (have_isa_3_00) {
3768	tcg_out_mem_long(s, LXV \| 8, LVX, out, base, offset & -16);
3769	} else {
3770	tcg_out_mem_long(s, 0, LVEHX, out, base, offset);
3771	}
3772	elt = extract32(offset, 1, 3);
3773	#if !HOST_BIG_ENDIAN
3774	elt ^= 7;
3775	#endif
3776	tcg_out32(s, VSPLTH \| VRT(out) \| VRB(out) \| (elt << 16));
3777	break;
3778	case MO_32:
3779	if (have_isa_3_00) {
3780	tcg_out_mem_long(s, 0, LXVWSX, out, base, offset);
3781	break;
3782	}
3783	tcg_debug_assert((offset & 3) == 0);
3784	tcg_out_mem_long(s, 0, LVEWX, out, base, offset);
3785	elt = extract32(offset, 2, 2);
3786	#if !HOST_BIG_ENDIAN
3787	elt ^= 3;
3788	#endif
3789	tcg_out32(s, VSPLTW \| VRT(out) \| VRB(out) \| (elt << 16));
3790	break;
3791	case MO_64:
3792	if (have_vsx) {
3793	tcg_out_mem_long(s, 0, LXVDSX, out, base, offset);
3794	break;
3795	}
3796	tcg_debug_assert((offset & 7) == 0);
3797	tcg_out_mem_long(s, 0, LVX, out, base, offset & -16);
3798	tcg_out_vsldoi(s, TCG_VEC_TMP1, out, out, 8);
3799	elt = extract32(offset, 3, 1);
3800	#if !HOST_BIG_ENDIAN
3801	elt = !elt;
3802	#endif
3803	if (elt) {
3804	tcg_out_vsldoi(s, out, out, TCG_VEC_TMP1, 8);
3805	} else {
3806	tcg_out_vsldoi(s, out, TCG_VEC_TMP1, out, 8);
3807	}
3808	break;
3809	default:
3810	g_assert_not_reached();
3811	}
3812	return true;
3813	}
3814
3815	static void tcg_out_not_vec(TCGContext *s, TCGReg a0, TCGReg a1)
3816	{
3817	tcg_out32(s, VNOR \| VRT(a0) \| VRA(a1) \| VRB(a1));
3818	}
3819
3820	static void tcg_out_or_vec(TCGContext *s, TCGReg a0, TCGReg a1, TCGReg a2)
3821	{
3822	tcg_out32(s, VOR \| VRT(a0) \| VRA(a1) \| VRB(a2));
3823	}
3824
3825	static void tcg_out_orc_vec(TCGContext *s, TCGReg a0, TCGReg a1, TCGReg a2)
3826	{
3827	tcg_out32(s, VORC \| VRT(a0) \| VRA(a1) \| VRB(a2));
3828	}
3829
3830	static void tcg_out_and_vec(TCGContext *s, TCGReg a0, TCGReg a1, TCGReg a2)
3831	{
3832	tcg_out32(s, VAND \| VRT(a0) \| VRA(a1) \| VRB(a2));
3833	}
3834
3835	static void tcg_out_andc_vec(TCGContext *s, TCGReg a0, TCGReg a1, TCGReg a2)
3836	{
3837	tcg_out32(s, VANDC \| VRT(a0) \| VRA(a1) \| VRB(a2));
3838	}
3839
3840	static void tcg_out_bitsel_vec(TCGContext *s, TCGReg d,
3841	TCGReg c, TCGReg t, TCGReg f)
3842	{
3843	if (TCG_TARGET_HAS_bitsel_vec) {
3844	tcg_out32(s, XXSEL \| VRT(d) \| VRC(c) \| VRB(t) \| VRA(f));
3845	} else {
3846	tcg_out_and_vec(s, TCG_VEC_TMP2, t, c);
3847	tcg_out_andc_vec(s, d, f, c);
3848	tcg_out_or_vec(s, d, d, TCG_VEC_TMP2);
3849	}
3850	}
3851
3852	static bool tcg_out_cmp_vec_noinv(TCGContext *s, unsigned vece, TCGReg a0,
3853	TCGReg a1, TCGReg a2, TCGCond cond)
3854	{
3855	static const uint32_t
3856	eq_op[4] = { VCMPEQUB, VCMPEQUH, VCMPEQUW, VCMPEQUD },
3857	ne_op[4] = { VCMPNEB, VCMPNEH, VCMPNEW, 0 },
3858	gts_op[4] = { VCMPGTSB, VCMPGTSH, VCMPGTSW, VCMPGTSD },
3859	gtu_op[4] = { VCMPGTUB, VCMPGTUH, VCMPGTUW, VCMPGTUD };
3860	uint32_t insn;
3861
3862	bool need_swap = false, need_inv = false;
3863
3864	tcg_debug_assert(vece <= MO_32 \|\| have_isa_2_07);
3865
3866	switch (cond) {
3867	case TCG_COND_EQ:
3868	case TCG_COND_GT:
3869	case TCG_COND_GTU:
3870	break;
3871	case TCG_COND_NE:
3872	if (have_isa_3_00 && vece <= MO_32) {
3873	break;
3874	}
3875	/* fall through */
3876	case TCG_COND_LE:
3877	case TCG_COND_LEU:
3878	need_inv = true;
3879	break;
3880	case TCG_COND_LT:
3881	case TCG_COND_LTU:
3882	need_swap = true;
3883	break;
3884	case TCG_COND_GE:
3885	case TCG_COND_GEU:
3886	need_swap = need_inv = true;
3887	break;
3888	default:
3889	g_assert_not_reached();
3890	}
3891
3892	if (need_inv) {
3893	cond = tcg_invert_cond(cond);
3894	}
3895	if (need_swap) {
3896	TCGReg swap = a1;
3897	a1 = a2;
3898	a2 = swap;
3899	cond = tcg_swap_cond(cond);
3900	}
3901
3902	switch (cond) {
3903	case TCG_COND_EQ:
3904	insn = eq_op[vece];
3905	break;
3906	case TCG_COND_NE:
3907	insn = ne_op[vece];
3908	break;
3909	case TCG_COND_GT:
3910	insn = gts_op[vece];
3911	break;
3912	case TCG_COND_GTU:
3913	insn = gtu_op[vece];
3914	break;
3915	default:
3916	g_assert_not_reached();
3917	}
3918	tcg_out32(s, insn \| VRT(a0) \| VRA(a1) \| VRB(a2));
3919
3920	return need_inv;
3921	}
3922
3923	static void tcg_out_cmp_vec(TCGContext *s, unsigned vece, TCGReg a0,
3924	TCGReg a1, TCGReg a2, TCGCond cond)
3925	{
3926	if (tcg_out_cmp_vec_noinv(s, vece, a0, a1, a2, cond)) {
3927	tcg_out_not_vec(s, a0, a0);
3928	}
3929	}
3930
3931	static void tcg_out_cmpsel_vec(TCGContext *s, unsigned vece, TCGReg a0,
3932	TCGReg c1, TCGReg c2, TCGArg v3, int const_v3,
3933	TCGReg v4, TCGCond cond)
3934	{
3935	bool inv = tcg_out_cmp_vec_noinv(s, vece, TCG_VEC_TMP1, c1, c2, cond);
3936
3937	if (!const_v3) {
3938	if (inv) {
3939	tcg_out_bitsel_vec(s, a0, TCG_VEC_TMP1, v4, v3);
3940	} else {
3941	tcg_out_bitsel_vec(s, a0, TCG_VEC_TMP1, v3, v4);
3942	}
3943	} else if (v3) {
3944	if (inv) {
3945	tcg_out_orc_vec(s, a0, v4, TCG_VEC_TMP1);
3946	} else {
3947	tcg_out_or_vec(s, a0, v4, TCG_VEC_TMP1);
3948	}
3949	} else {
3950	if (inv) {
3951	tcg_out_and_vec(s, a0, v4, TCG_VEC_TMP1);
3952	} else {
3953	tcg_out_andc_vec(s, a0, v4, TCG_VEC_TMP1);
3954	}
3955	}
3956	}
3957
3958	static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
3959	unsigned vecl, unsigned vece,
3960	const TCGArg args[TCG_MAX_OP_ARGS],
3961	const int const_args[TCG_MAX_OP_ARGS])
3962	{
3963	static const uint32_t
3964	add_op[4] = { VADDUBM, VADDUHM, VADDUWM, VADDUDM },
3965	sub_op[4] = { VSUBUBM, VSUBUHM, VSUBUWM, VSUBUDM },
3966	mul_op[4] = { 0, 0, VMULUWM, VMULLD },
3967	neg_op[4] = { 0, 0, VNEGW, VNEGD },
3968	ssadd_op[4] = { VADDSBS, VADDSHS, VADDSWS, 0 },
3969	usadd_op[4] = { VADDUBS, VADDUHS, VADDUWS, 0 },
3970	sssub_op[4] = { VSUBSBS, VSUBSHS, VSUBSWS, 0 },
3971	ussub_op[4] = { VSUBUBS, VSUBUHS, VSUBUWS, 0 },
3972	umin_op[4] = { VMINUB, VMINUH, VMINUW, VMINUD },
3973	smin_op[4] = { VMINSB, VMINSH, VMINSW, VMINSD },
3974	umax_op[4] = { VMAXUB, VMAXUH, VMAXUW, VMAXUD },
3975	smax_op[4] = { VMAXSB, VMAXSH, VMAXSW, VMAXSD },
3976	shlv_op[4] = { VSLB, VSLH, VSLW, VSLD },
3977	shrv_op[4] = { VSRB, VSRH, VSRW, VSRD },
3978	sarv_op[4] = { VSRAB, VSRAH, VSRAW, VSRAD },
3979	mrgh_op[4] = { VMRGHB, VMRGHH, VMRGHW, 0 },
3980	mrgl_op[4] = { VMRGLB, VMRGLH, VMRGLW, 0 },
3981	muleu_op[4] = { VMULEUB, VMULEUH, VMULEUW, 0 },
3982	mulou_op[4] = { VMULOUB, VMULOUH, VMULOUW, 0 },
3983	pkum_op[4] = { VPKUHUM, VPKUWUM, 0, 0 },
3984	rotl_op[4] = { VRLB, VRLH, VRLW, VRLD };
3985
3986	TCGType type = vecl + TCG_TYPE_V64;
3987	TCGArg a0 = args[0], a1 = args[1], a2 = args[2];
3988	uint32_t insn;
3989
3990	switch (opc) {
3991	case INDEX_op_ld_vec:
3992	tcg_out_ld(s, type, a0, a1, a2);
3993	return;
3994	case INDEX_op_st_vec:
3995	tcg_out_st(s, type, a0, a1, a2);
3996	return;
3997	case INDEX_op_dupm_vec:
3998	tcg_out_dupm_vec(s, type, vece, a0, a1, a2);
3999	return;
4000
4001	case INDEX_op_add_vec:
4002	insn = add_op[vece];
4003	break;
4004	case INDEX_op_sub_vec:
4005	insn = sub_op[vece];
4006	break;
4007	case INDEX_op_neg_vec:
4008	insn = neg_op[vece];
4009	a2 = a1;
4010	a1 = 0;
4011	break;
4012	case INDEX_op_mul_vec:
4013	insn = mul_op[vece];
4014	break;
4015	case INDEX_op_ssadd_vec:
4016	insn = ssadd_op[vece];
4017	break;
4018	case INDEX_op_sssub_vec:
4019	insn = sssub_op[vece];
4020	break;
4021	case INDEX_op_usadd_vec:
4022	insn = usadd_op[vece];
4023	break;
4024	case INDEX_op_ussub_vec:
4025	insn = ussub_op[vece];
4026	break;
4027	case INDEX_op_smin_vec:
4028	insn = smin_op[vece];
4029	break;
4030	case INDEX_op_umin_vec:
4031	insn = umin_op[vece];
4032	break;
4033	case INDEX_op_smax_vec:
4034	insn = smax_op[vece];
4035	break;
4036	case INDEX_op_umax_vec:
4037	insn = umax_op[vece];
4038	break;
4039	case INDEX_op_shlv_vec:
4040	insn = shlv_op[vece];
4041	break;
4042	case INDEX_op_shrv_vec:
4043	insn = shrv_op[vece];
4044	break;
4045	case INDEX_op_sarv_vec:
4046	insn = sarv_op[vece];
4047	break;
4048	case INDEX_op_and_vec:
4049	tcg_out_and_vec(s, a0, a1, a2);
4050	return;
4051	case INDEX_op_or_vec:
4052	tcg_out_or_vec(s, a0, a1, a2);
4053	return;
4054	case INDEX_op_xor_vec:
4055	insn = VXOR;
4056	break;
4057	case INDEX_op_andc_vec:
4058	tcg_out_andc_vec(s, a0, a1, a2);
4059	return;
4060	case INDEX_op_not_vec:
4061	tcg_out_not_vec(s, a0, a1);
4062	return;
4063	case INDEX_op_orc_vec:
4064	tcg_out_orc_vec(s, a0, a1, a2);
4065	return;
4066	case INDEX_op_nand_vec:
4067	insn = VNAND;
4068	break;
4069	case INDEX_op_nor_vec:
4070	insn = VNOR;
4071	break;
4072	case INDEX_op_eqv_vec:
4073	insn = VEQV;
4074	break;
4075
4076	case INDEX_op_cmp_vec:
4077	tcg_out_cmp_vec(s, vece, a0, a1, a2, args[3]);
4078	return;
4079	case INDEX_op_cmpsel_vec:
4080	tcg_out_cmpsel_vec(s, vece, a0, a1, a2,
4081	args[3], const_args[3], args[4], args[5]);
4082	return;
4083	case INDEX_op_bitsel_vec:
4084	tcg_out_bitsel_vec(s, a0, a1, a2, args[3]);
4085	return;
4086
4087	case INDEX_op_ppc_mrgh_vec:
4088	insn = mrgh_op[vece];
4089	break;
4090	case INDEX_op_ppc_mrgl_vec:
4091	insn = mrgl_op[vece];
4092	break;
4093	case INDEX_op_ppc_muleu_vec:
4094	insn = muleu_op[vece];
4095	break;
4096	case INDEX_op_ppc_mulou_vec:
4097	insn = mulou_op[vece];
4098	break;
4099	case INDEX_op_ppc_pkum_vec:
4100	insn = pkum_op[vece];
4101	break;
4102	case INDEX_op_rotlv_vec:
4103	insn = rotl_op[vece];
4104	break;
4105	case INDEX_op_ppc_msum_vec:
4106	tcg_debug_assert(vece == MO_16);
4107	tcg_out32(s, VMSUMUHM \| VRT(a0) \| VRA(a1) \| VRB(a2) \| VRC(args[3]));
4108	return;
4109
4110	case INDEX_op_mov_vec: /* Always emitted via tcg_out_mov. */
4111	case INDEX_op_dup_vec: /* Always emitted via tcg_out_dup_vec. */
4112	default:
4113	g_assert_not_reached();
4114	}
4115
4116	tcg_debug_assert(insn != 0);
4117	tcg_out32(s, insn \| VRT(a0) \| VRA(a1) \| VRB(a2));
4118	}
4119
4120	static void expand_vec_shi(TCGType type, unsigned vece, TCGv_vec v0,
4121	TCGv_vec v1, TCGArg imm, TCGOpcode opci)
4122	{
4123	TCGv_vec t1;
4124
4125	if (vece == MO_32) {
4126	/*
4127	* Only 5 bits are significant, and VSPLTISB can represent -16..15.
4128	* So using negative numbers gets us the 4th bit easily.
4129	*/
4130	imm = sextract32(imm, 0, 5);
4131	} else {
4132	imm &= (8 << vece) - 1;
4133	}
4134
4135	/* Splat w/bytes for xxspltib when 2.07 allows MO_64. */
4136	t1 = tcg_constant_vec(type, MO_8, imm);
4137	vec_gen_3(opci, type, vece, tcgv_vec_arg(v0),
4138	tcgv_vec_arg(v1), tcgv_vec_arg(t1));
4139	}
4140
4141	static void expand_vec_mul(TCGType type, unsigned vece, TCGv_vec v0,
4142	TCGv_vec v1, TCGv_vec v2)
4143	{
4144	TCGv_vec t1 = tcg_temp_new_vec(type);
4145	TCGv_vec t2 = tcg_temp_new_vec(type);
4146	TCGv_vec c0, c16;
4147
4148	switch (vece) {
4149	case MO_8:
4150	case MO_16:
4151	vec_gen_3(INDEX_op_ppc_muleu_vec, type, vece, tcgv_vec_arg(t1),
4152	tcgv_vec_arg(v1), tcgv_vec_arg(v2));
4153	vec_gen_3(INDEX_op_ppc_mulou_vec, type, vece, tcgv_vec_arg(t2),
4154	tcgv_vec_arg(v1), tcgv_vec_arg(v2));
4155	vec_gen_3(INDEX_op_ppc_mrgh_vec, type, vece + 1, tcgv_vec_arg(v0),
4156	tcgv_vec_arg(t1), tcgv_vec_arg(t2));
4157	vec_gen_3(INDEX_op_ppc_mrgl_vec, type, vece + 1, tcgv_vec_arg(t1),
4158	tcgv_vec_arg(t1), tcgv_vec_arg(t2));
4159	vec_gen_3(INDEX_op_ppc_pkum_vec, type, vece, tcgv_vec_arg(v0),
4160	tcgv_vec_arg(v0), tcgv_vec_arg(t1));
4161	break;
4162
4163	case MO_32:
4164	tcg_debug_assert(!have_isa_2_07);
4165	/*
4166	* Only 5 bits are significant, and VSPLTISB can represent -16..15.
4167	* So using -16 is a quick way to represent 16.
4168	*/
4169	c16 = tcg_constant_vec(type, MO_8, -16);
4170	c0 = tcg_constant_vec(type, MO_8, 0);
4171
4172	vec_gen_3(INDEX_op_rotlv_vec, type, MO_32, tcgv_vec_arg(t1),
4173	tcgv_vec_arg(v2), tcgv_vec_arg(c16));
4174	vec_gen_3(INDEX_op_ppc_mulou_vec, type, MO_16, tcgv_vec_arg(t2),
4175	tcgv_vec_arg(v1), tcgv_vec_arg(v2));
4176	vec_gen_4(INDEX_op_ppc_msum_vec, type, MO_16, tcgv_vec_arg(t1),
4177	tcgv_vec_arg(v1), tcgv_vec_arg(t1), tcgv_vec_arg(c0));
4178	vec_gen_3(INDEX_op_shlv_vec, type, MO_32, tcgv_vec_arg(t1),
4179	tcgv_vec_arg(t1), tcgv_vec_arg(c16));
4180	tcg_gen_add_vec(MO_32, v0, t1, t2);
4181	break;
4182
4183	default:
4184	g_assert_not_reached();
4185	}
4186	tcg_temp_free_vec(t1);
4187	tcg_temp_free_vec(t2);
4188	}
4189
4190	void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece,
4191	TCGArg a0, ...)
4192	{
4193	va_list va;
4194	TCGv_vec v0, v1, v2, t0;
4195	TCGArg a2;
4196
4197	va_start(va, a0);
4198	v0 = temp_tcgv_vec(arg_temp(a0));
4199	v1 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
4200	a2 = va_arg(va, TCGArg);
4201
4202	switch (opc) {
4203	case INDEX_op_shli_vec:
4204	expand_vec_shi(type, vece, v0, v1, a2, INDEX_op_shlv_vec);
4205	break;
4206	case INDEX_op_shri_vec:
4207	expand_vec_shi(type, vece, v0, v1, a2, INDEX_op_shrv_vec);
4208	break;
4209	case INDEX_op_sari_vec:
4210	expand_vec_shi(type, vece, v0, v1, a2, INDEX_op_sarv_vec);
4211	break;
4212	case INDEX_op_rotli_vec:
4213	expand_vec_shi(type, vece, v0, v1, a2, INDEX_op_rotlv_vec);
4214	break;
4215	case INDEX_op_mul_vec:
4216	v2 = temp_tcgv_vec(arg_temp(a2));
4217	expand_vec_mul(type, vece, v0, v1, v2);
4218	break;
4219	case INDEX_op_rotlv_vec:
4220	v2 = temp_tcgv_vec(arg_temp(a2));
4221	t0 = tcg_temp_new_vec(type);
4222	tcg_gen_neg_vec(vece, t0, v2);
4223	tcg_gen_rotlv_vec(vece, v0, v1, t0);
4224	tcg_temp_free_vec(t0);
4225	break;
4226	default:
4227	g_assert_not_reached();
4228	}
4229	va_end(va);
4230	}
4231
4232	static TCGConstraintSetIndex
4233	tcg_target_op_def(TCGOpcode op, TCGType type, unsigned flags)
4234	{
4235	switch (op) {
4236	case INDEX_op_add_vec:
4237	case INDEX_op_sub_vec:
4238	case INDEX_op_mul_vec:
4239	case INDEX_op_and_vec:
4240	case INDEX_op_or_vec:
4241	case INDEX_op_xor_vec:
4242	case INDEX_op_andc_vec:
4243	case INDEX_op_orc_vec:
4244	case INDEX_op_nor_vec:
4245	case INDEX_op_eqv_vec:
4246	case INDEX_op_nand_vec:
4247	case INDEX_op_cmp_vec:
4248	case INDEX_op_ssadd_vec:
4249	case INDEX_op_sssub_vec:
4250	case INDEX_op_usadd_vec:
4251	case INDEX_op_ussub_vec:
4252	case INDEX_op_smax_vec:
4253	case INDEX_op_smin_vec:
4254	case INDEX_op_umax_vec:
4255	case INDEX_op_umin_vec:
4256	case INDEX_op_shlv_vec:
4257	case INDEX_op_shrv_vec:
4258	case INDEX_op_sarv_vec:
4259	case INDEX_op_rotlv_vec:
4260	case INDEX_op_rotrv_vec:
4261	case INDEX_op_ppc_mrgh_vec:
4262	case INDEX_op_ppc_mrgl_vec:
4263	case INDEX_op_ppc_muleu_vec:
4264	case INDEX_op_ppc_mulou_vec:
4265	case INDEX_op_ppc_pkum_vec:
4266	return C_O1_I2(v, v, v);
4267
4268	case INDEX_op_not_vec:
4269	case INDEX_op_neg_vec:
4270	return C_O1_I1(v, v);
4271
4272	case INDEX_op_dup_vec:
4273	return have_isa_3_00 ? C_O1_I1(v, vr) : C_O1_I1(v, v);
4274
4275	case INDEX_op_ld_vec:
4276	case INDEX_op_dupm_vec:
4277	return C_O1_I1(v, r);
4278
4279	case INDEX_op_st_vec:
4280	return C_O0_I2(v, r);
4281
4282	case INDEX_op_bitsel_vec:
4283	case INDEX_op_ppc_msum_vec:
4284	return C_O1_I3(v, v, v, v);
4285	case INDEX_op_cmpsel_vec:
4286	return C_O1_I4(v, v, v, vZM, v);
4287
4288	default:
4289	return C_NotImplemented;
4290	}
4291	}
4292
4293	static void tcg_target_init(TCGContext *s)
4294	{
4295	tcg_target_available_regs[TCG_TYPE_I32] = 0xffffffff;
4296	tcg_target_available_regs[TCG_TYPE_I64] = 0xffffffff;
4297	if (have_altivec) {
4298	tcg_target_available_regs[TCG_TYPE_V64] = 0xffffffff00000000ull;
4299	tcg_target_available_regs[TCG_TYPE_V128] = 0xffffffff00000000ull;
4300	}
4301
4302	tcg_target_call_clobber_regs = 0;
4303	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R0);
4304	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R2);
4305	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R3);
4306	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R4);
4307	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R5);
4308	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R6);
4309	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R7);
4310	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R8);
4311	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R9);
4312	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R10);
4313	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R11);
4314	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R12);
4315
4316	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V0);
4317	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V1);
4318	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V2);
4319	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V3);
4320	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V4);
4321	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V5);
4322	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V6);
4323	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V7);
4324	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V8);
4325	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V9);
4326	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V10);
4327	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V11);
4328	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V12);
4329	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V13);
4330	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V14);
4331	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V15);
4332	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V16);
4333	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V17);
4334	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V18);
4335	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V19);
4336
4337	s->reserved_regs = 0;
4338	tcg_regset_set_reg(s->reserved_regs, TCG_REG_R0); /* tcg temp */
4339	tcg_regset_set_reg(s->reserved_regs, TCG_REG_R1); /* stack pointer */
4340	#if defined(_CALL_SYSV)
4341	tcg_regset_set_reg(s->reserved_regs, TCG_REG_R2); /* toc pointer */
4342	#endif
4343	tcg_regset_set_reg(s->reserved_regs, TCG_REG_R13); /* thread pointer */
4344	tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP1);
4345	tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP2);
4346	tcg_regset_set_reg(s->reserved_regs, TCG_VEC_TMP1);
4347	tcg_regset_set_reg(s->reserved_regs, TCG_VEC_TMP2);
4348	if (USE_REG_TB) {
4349	tcg_regset_set_reg(s->reserved_regs, TCG_REG_TB); /* tb->tc_ptr */
4350	}
4351	}
4352
4353	#ifdef __ELF__
4354	typedef struct {
4355	DebugFrameCIE cie;
4356	DebugFrameFDEHeader fde;
4357	uint8_t fde_def_cfa[4];
4358	uint8_t fde_reg_ofs[ARRAY_SIZE(tcg_target_callee_save_regs) * 2 + 3];
4359	} DebugFrame;
4360
4361	/* We're expecting a 2 byte uleb128 encoded value. */
4362	QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
4363
4364	#define ELF_HOST_MACHINE EM_PPC64
4365
4366	static DebugFrame debug_frame = {
4367	.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
4368	.cie.id = -1,
4369	.cie.version = 1,
4370	.cie.code_align = 1,
4371	.cie.data_align = (-SZR & 0x7f), /* sleb128 -SZR */
4372	.cie.return_column = 65,
4373
4374	/* Total FDE size does not include the "len" member. */
4375	.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, fde.cie_offset),
4376
4377	.fde_def_cfa = {
4378	12, TCG_REG_R1, /* DW_CFA_def_cfa r1, ... */
4379	(FRAME_SIZE & 0x7f) \| 0x80, /* ... uleb128 FRAME_SIZE */
4380	(FRAME_SIZE >> 7)
4381	},
4382	.fde_reg_ofs = {
4383	/* DW_CFA_offset_extended_sf, lr, LR_OFFSET */
4384	0x11, 65, (LR_OFFSET / -SZR) & 0x7f,
4385	}
4386	};
4387
4388	void tcg_register_jit(const void *buf, size_t buf_size)
4389	{
4390	uint8_t *p = &debug_frame.fde_reg_ofs[3];
4391	int i;
4392
4393	for (i = 0; i < ARRAY_SIZE(tcg_target_callee_save_regs); ++i, p += 2) {
4394	p[0] = 0x80 + tcg_target_callee_save_regs[i];
4395	p[1] = (FRAME_SIZE - (REG_SAVE_BOT + i * SZR)) / SZR;
4396	}
4397
4398	debug_frame.fde.func_start = (uintptr_t)buf;
4399	debug_frame.fde.func_len = buf_size;
4400
4401	tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
4402	}
4403	#endif /* __ELF__ */
4404	#undef VMULEUB
4405	#undef VMULEUH
4406	#undef VMULEUW
4407	#undef VMULOUB
4408	#undef VMULOUH
4409	#undef VMULOUW
4410	#undef VMSUMUHM