tcg-target.c.inc · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

inc 3,909 lines 111 KB

Raw

1	/*
2	* Tiny Code Generator for QEMU
3	*
4	* Copyright (c) 2009 Ulrich Hecht <uli@suse.de>
5	* Copyright (c) 2009 Alexander Graf <agraf@suse.de>
6	* Copyright (c) 2010 Richard Henderson <rth@twiddle.net>
7	*
8	* Permission is hereby granted, free of charge, to any person obtaining a copy
9	* of this software and associated documentation files (the "Software"), to deal
10	* in the Software without restriction, including without limitation the rights
11	* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12	* copies of the Software, and to permit persons to whom the Software is
13	* furnished to do so, subject to the following conditions:
14	*
15	* The above copyright notice and this permission notice shall be included in
16	* all copies or substantial portions of the Software.
17	*
18	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
24	* THE SOFTWARE.
25	*/
26
27	#include "elf.h"
28
29	/* Used for function call generation. */
30	#define TCG_TARGET_STACK_ALIGN 8
31	#define TCG_TARGET_CALL_STACK_OFFSET 160
32	#define TCG_TARGET_CALL_ARG_I32 TCG_CALL_ARG_EXTEND
33	#define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL
34	#define TCG_TARGET_CALL_ARG_I128 TCG_CALL_ARG_BY_REF
35	#define TCG_TARGET_CALL_RET_I128 TCG_CALL_RET_BY_REF
36
37	#define TCG_CT_CONST_S16 (1 << 8)
38	#define TCG_CT_CONST_S32 (1 << 9)
39	#define TCG_CT_CONST_U32 (1 << 10)
40	#define TCG_CT_CONST_ZERO (1 << 11)
41	#define TCG_CT_CONST_P32 (1 << 12)
42	#define TCG_CT_CONST_INV (1 << 13)
43	#define TCG_CT_CONST_INVRISBG (1 << 14)
44	#define TCG_CT_CONST_CMP (1 << 15)
45	#define TCG_CT_CONST_M1 (1 << 16)
46	#define TCG_CT_CONST_N32 (1 << 17)
47
48	#define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 16)
49	#define ALL_VECTOR_REGS MAKE_64BIT_MASK(32, 32)
50
51	/* Several places within the instruction set 0 means "no register"
52	rather than TCG_REG_R0. */
53	#define TCG_REG_NONE 0
54
55	/* A scratch register that may be be used throughout the backend. */
56	#define TCG_TMP0 TCG_REG_R1
57	#define TCG_VEC_TMP0 TCG_REG_V31
58
59	#define TCG_GUEST_BASE_REG TCG_REG_R13
60
61	/* All of the following instructions are prefixed with their instruction
62	format, and are defined as 8- or 16-bit quantities, even when the two
63	halves of the 16-bit quantity may appear 32 bits apart in the insn.
64	This makes it easy to copy the values from the tables in Appendix B. */
65	typedef enum S390Opcode {
66	RIL_AFI = 0xc209,
67	RIL_AGFI = 0xc208,
68	RIL_ALFI = 0xc20b,
69	RIL_ALGFI = 0xc20a,
70	RIL_BRASL = 0xc005,
71	RIL_BRCL = 0xc004,
72	RIL_CFI = 0xc20d,
73	RIL_CGFI = 0xc20c,
74	RIL_CLFI = 0xc20f,
75	RIL_CLGFI = 0xc20e,
76	RIL_CLRL = 0xc60f,
77	RIL_CLGRL = 0xc60a,
78	RIL_CRL = 0xc60d,
79	RIL_CGRL = 0xc608,
80	RIL_IIHF = 0xc008,
81	RIL_IILF = 0xc009,
82	RIL_LARL = 0xc000,
83	RIL_LGFI = 0xc001,
84	RIL_LGRL = 0xc408,
85	RIL_LLIHF = 0xc00e,
86	RIL_LLILF = 0xc00f,
87	RIL_LRL = 0xc40d,
88	RIL_MSFI = 0xc201,
89	RIL_MSGFI = 0xc200,
90	RIL_NIHF = 0xc00a,
91	RIL_NILF = 0xc00b,
92	RIL_OIHF = 0xc00c,
93	RIL_OILF = 0xc00d,
94	RIL_SLFI = 0xc205,
95	RIL_SLGFI = 0xc204,
96	RIL_XIHF = 0xc006,
97	RIL_XILF = 0xc007,
98
99	RI_AGHI = 0xa70b,
100	RI_AHI = 0xa70a,
101	RI_BRC = 0xa704,
102	RI_CHI = 0xa70e,
103	RI_CGHI = 0xa70f,
104	RI_IIHH = 0xa500,
105	RI_IIHL = 0xa501,
106	RI_IILH = 0xa502,
107	RI_IILL = 0xa503,
108	RI_LGHI = 0xa709,
109	RI_LLIHH = 0xa50c,
110	RI_LLIHL = 0xa50d,
111	RI_LLILH = 0xa50e,
112	RI_LLILL = 0xa50f,
113	RI_MGHI = 0xa70d,
114	RI_MHI = 0xa70c,
115	RI_NIHH = 0xa504,
116	RI_NIHL = 0xa505,
117	RI_NILH = 0xa506,
118	RI_NILL = 0xa507,
119	RI_OIHH = 0xa508,
120	RI_OIHL = 0xa509,
121	RI_OILH = 0xa50a,
122	RI_OILL = 0xa50b,
123	RI_TMLL = 0xa701,
124	RI_TMLH = 0xa700,
125	RI_TMHL = 0xa703,
126	RI_TMHH = 0xa702,
127
128	RIEb_CGRJ = 0xec64,
129	RIEb_CLGRJ = 0xec65,
130	RIEb_CLRJ = 0xec77,
131	RIEb_CRJ = 0xec76,
132
133	RIEc_CGIJ = 0xec7c,
134	RIEc_CIJ = 0xec7e,
135	RIEc_CLGIJ = 0xec7d,
136	RIEc_CLIJ = 0xec7f,
137
138	RIEd_ALHSIK = 0xecda,
139	RIEd_ALGHSIK = 0xecdb,
140
141	RIEf_RISBG = 0xec55,
142
143	RIEg_LOCGHI = 0xec46,
144
145	RRE_AGR = 0xb908,
146	RRE_ALGR = 0xb90a,
147	RRE_ALCR = 0xb998,
148	RRE_ALCGR = 0xb988,
149	RRE_ALGFR = 0xb91a,
150	RRE_CGR = 0xb920,
151	RRE_CLGR = 0xb921,
152	RRE_DLGR = 0xb987,
153	RRE_DLR = 0xb997,
154	RRE_DSGFR = 0xb91d,
155	RRE_DSGR = 0xb90d,
156	RRE_FLOGR = 0xb983,
157	RRE_LGBR = 0xb906,
158	RRE_LCGR = 0xb903,
159	RRE_LGFR = 0xb914,
160	RRE_LGHR = 0xb907,
161	RRE_LGR = 0xb904,
162	RRE_LLGCR = 0xb984,
163	RRE_LLGFR = 0xb916,
164	RRE_LLGHR = 0xb985,
165	RRE_LRVR = 0xb91f,
166	RRE_LRVGR = 0xb90f,
167	RRE_LTGR = 0xb902,
168	RRE_MLGR = 0xb986,
169	RRE_MSGR = 0xb90c,
170	RRE_MSR = 0xb252,
171	RRE_NGR = 0xb980,
172	RRE_OGR = 0xb981,
173	RRE_SGR = 0xb909,
174	RRE_SLGR = 0xb90b,
175	RRE_SLBR = 0xb999,
176	RRE_SLBGR = 0xb989,
177	RRE_XGR = 0xb982,
178
179	RRFa_ALRK = 0xb9fa,
180	RRFa_ALGRK = 0xb9ea,
181	RRFa_MGRK = 0xb9ec,
182	RRFa_MSRKC = 0xb9fd,
183	RRFa_MSGRKC = 0xb9ed,
184	RRFa_NCRK = 0xb9f5,
185	RRFa_NCGRK = 0xb9e5,
186	RRFa_NNRK = 0xb974,
187	RRFa_NNGRK = 0xb964,
188	RRFa_NORK = 0xb976,
189	RRFa_NOGRK = 0xb966,
190	RRFa_NRK = 0xb9f4,
191	RRFa_NGRK = 0xb9e4,
192	RRFa_NXRK = 0xb977,
193	RRFa_NXGRK = 0xb967,
194	RRFa_OCRK = 0xb975,
195	RRFa_OCGRK = 0xb965,
196	RRFa_ORK = 0xb9f6,
197	RRFa_OGRK = 0xb9e6,
198	RRFa_SRK = 0xb9f9,
199	RRFa_SGRK = 0xb9e9,
200	RRFa_SLRK = 0xb9fb,
201	RRFa_SLGRK = 0xb9eb,
202	RRFa_XRK = 0xb9f7,
203	RRFa_XGRK = 0xb9e7,
204
205	RRFam_SELGR = 0xb9e3,
206
207	RRFc_LOCR = 0xb9f2,
208	RRFc_LOCGR = 0xb9e2,
209	RRFc_POPCNT = 0xb9e1,
210
211	RR_AR = 0x1a,
212	RR_ALR = 0x1e,
213	RR_BASR = 0x0d,
214	RR_BCR = 0x07,
215	RR_CLR = 0x15,
216	RR_CR = 0x19,
217	RR_DR = 0x1d,
218	RR_LCR = 0x13,
219	RR_LR = 0x18,
220	RR_LTR = 0x12,
221	RR_NR = 0x14,
222	RR_OR = 0x16,
223	RR_SR = 0x1b,
224	RR_SLR = 0x1f,
225	RR_XR = 0x17,
226
227	RSY_RLL = 0xeb1d,
228	RSY_RLLG = 0xeb1c,
229	RSY_SLLG = 0xeb0d,
230	RSY_SLLK = 0xebdf,
231	RSY_SRAG = 0xeb0a,
232	RSY_SRAK = 0xebdc,
233	RSY_SRLG = 0xeb0c,
234	RSY_SRLK = 0xebde,
235
236	RS_SLL = 0x89,
237	RS_SRA = 0x8a,
238	RS_SRL = 0x88,
239
240	RXY_AG = 0xe308,
241	RXY_AY = 0xe35a,
242	RXY_CG = 0xe320,
243	RXY_CLG = 0xe321,
244	RXY_CLY = 0xe355,
245	RXY_CY = 0xe359,
246	RXY_LAY = 0xe371,
247	RXY_LB = 0xe376,
248	RXY_LG = 0xe304,
249	RXY_LGB = 0xe377,
250	RXY_LGF = 0xe314,
251	RXY_LGH = 0xe315,
252	RXY_LHY = 0xe378,
253	RXY_LLGC = 0xe390,
254	RXY_LLGF = 0xe316,
255	RXY_LLGH = 0xe391,
256	RXY_LMG = 0xeb04,
257	RXY_LPQ = 0xe38f,
258	RXY_LRV = 0xe31e,
259	RXY_LRVG = 0xe30f,
260	RXY_LRVH = 0xe31f,
261	RXY_LY = 0xe358,
262	RXY_NG = 0xe380,
263	RXY_OG = 0xe381,
264	RXY_STCY = 0xe372,
265	RXY_STG = 0xe324,
266	RXY_STHY = 0xe370,
267	RXY_STMG = 0xeb24,
268	RXY_STPQ = 0xe38e,
269	RXY_STRV = 0xe33e,
270	RXY_STRVG = 0xe32f,
271	RXY_STRVH = 0xe33f,
272	RXY_STY = 0xe350,
273	RXY_XG = 0xe382,
274
275	RX_A = 0x5a,
276	RX_C = 0x59,
277	RX_L = 0x58,
278	RX_LA = 0x41,
279	RX_LH = 0x48,
280	RX_ST = 0x50,
281	RX_STC = 0x42,
282	RX_STH = 0x40,
283
284	VRIa_VGBM = 0xe744,
285	VRIa_VREPI = 0xe745,
286	VRIb_VGM = 0xe746,
287	VRIc_VREP = 0xe74d,
288
289	VRRa_VLC = 0xe7de,
290	VRRa_VLP = 0xe7df,
291	VRRa_VLR = 0xe756,
292	VRRc_VA = 0xe7f3,
293	VRRc_VCEQ = 0xe7f8, /* we leave the m5 cs field 0 */
294	VRRc_VCH = 0xe7fb, /* " */
295	VRRc_VCHL = 0xe7f9, /* " */
296	VRRc_VERLLV = 0xe773,
297	VRRc_VESLV = 0xe770,
298	VRRc_VESRAV = 0xe77a,
299	VRRc_VESRLV = 0xe778,
300	VRRc_VML = 0xe7a2,
301	VRRc_VMN = 0xe7fe,
302	VRRc_VMNL = 0xe7fc,
303	VRRc_VMX = 0xe7ff,
304	VRRc_VMXL = 0xe7fd,
305	VRRc_VN = 0xe768,
306	VRRc_VNC = 0xe769,
307	VRRc_VNN = 0xe76e,
308	VRRc_VNO = 0xe76b,
309	VRRc_VNX = 0xe76c,
310	VRRc_VO = 0xe76a,
311	VRRc_VOC = 0xe76f,
312	VRRc_VPKS = 0xe797, /* we leave the m5 cs field 0 */
313	VRRc_VS = 0xe7f7,
314	VRRa_VUPH = 0xe7d7,
315	VRRa_VUPL = 0xe7d6,
316	VRRc_VX = 0xe76d,
317	VRRe_VSEL = 0xe78d,
318	VRRf_VLVGP = 0xe762,
319
320	VRSa_VERLL = 0xe733,
321	VRSa_VESL = 0xe730,
322	VRSa_VESRA = 0xe73a,
323	VRSa_VESRL = 0xe738,
324	VRSb_VLVG = 0xe722,
325	VRSc_VLGV = 0xe721,
326
327	VRX_VL = 0xe706,
328	VRX_VLLEZ = 0xe704,
329	VRX_VLREP = 0xe705,
330	VRX_VST = 0xe70e,
331	VRX_VSTEF = 0xe70b,
332	VRX_VSTEG = 0xe70a,
333
334	NOP = 0x0707,
335	} S390Opcode;
336
337	#ifdef CONFIG_DEBUG_TCG
338	static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
339	"%r0", "%r1", "%r2", "%r3", "%r4", "%r5", "%r6", "%r7",
340	"%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15",
341	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
342	"%v0", "%v1", "%v2", "%v3", "%v4", "%v5", "%v6", "%v7",
343	"%v8", "%v9", "%v10", "%v11", "%v12", "%v13", "%v14", "%v15",
344	"%v16", "%v17", "%v18", "%v19", "%v20", "%v21", "%v22", "%v23",
345	"%v24", "%v25", "%v26", "%v27", "%v28", "%v29", "%v30", "%v31",
346	};
347	#endif
348
349	/* Since R6 is a potential argument register, choose it last of the
350	call-saved registers. Likewise prefer the call-clobbered registers
351	in reverse order to maximize the chance of avoiding the arguments. */
352	static const int tcg_target_reg_alloc_order[] = {
353	/* Call saved registers. */
354	TCG_REG_R13,
355	TCG_REG_R12,
356	TCG_REG_R11,
357	TCG_REG_R10,
358	TCG_REG_R9,
359	TCG_REG_R8,
360	TCG_REG_R7,
361	TCG_REG_R6,
362	/* Call clobbered registers. */
363	TCG_REG_R14,
364	TCG_REG_R0,
365	TCG_REG_R1,
366	/* Argument registers, in reverse order of allocation. */
367	TCG_REG_R5,
368	TCG_REG_R4,
369	TCG_REG_R3,
370	TCG_REG_R2,
371
372	/* V8-V15 are call saved, and omitted. */
373	TCG_REG_V0,
374	TCG_REG_V1,
375	TCG_REG_V2,
376	TCG_REG_V3,
377	TCG_REG_V4,
378	TCG_REG_V5,
379	TCG_REG_V6,
380	TCG_REG_V7,
381	TCG_REG_V16,
382	TCG_REG_V17,
383	TCG_REG_V18,
384	TCG_REG_V19,
385	TCG_REG_V20,
386	TCG_REG_V21,
387	TCG_REG_V22,
388	TCG_REG_V23,
389	TCG_REG_V24,
390	TCG_REG_V25,
391	TCG_REG_V26,
392	TCG_REG_V27,
393	TCG_REG_V28,
394	TCG_REG_V29,
395	TCG_REG_V30,
396	TCG_REG_V31,
397	};
398
399	static const int tcg_target_call_iarg_regs[] = {
400	TCG_REG_R2,
401	TCG_REG_R3,
402	TCG_REG_R4,
403	TCG_REG_R5,
404	TCG_REG_R6,
405	};
406
407	static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot)
408	{
409	tcg_debug_assert(kind == TCG_CALL_RET_NORMAL);
410	tcg_debug_assert(slot == 0);
411	return TCG_REG_R2;
412	}
413
414	#define S390_CC_EQ 8
415	#define S390_CC_LT 4
416	#define S390_CC_GT 2
417	#define S390_CC_OV 1
418	#define S390_CC_NE (S390_CC_LT \| S390_CC_GT)
419	#define S390_CC_LE (S390_CC_LT \| S390_CC_EQ)
420	#define S390_CC_GE (S390_CC_GT \| S390_CC_EQ)
421	#define S390_CC_NEVER 0
422	#define S390_CC_ALWAYS 15
423
424	#define S390_TM_EQ 8 /* CC == 0 */
425	#define S390_TM_NE 7 /* CC in {1,2,3} */
426
427	/* Condition codes that result from a COMPARE and COMPARE LOGICAL. */
428	static const uint8_t tcg_cond_to_s390_cond[16] = {
429	[TCG_COND_EQ] = S390_CC_EQ,
430	[TCG_COND_NE] = S390_CC_NE,
431	[TCG_COND_TSTEQ] = S390_CC_EQ,
432	[TCG_COND_TSTNE] = S390_CC_NE,
433	[TCG_COND_LT] = S390_CC_LT,
434	[TCG_COND_LE] = S390_CC_LE,
435	[TCG_COND_GT] = S390_CC_GT,
436	[TCG_COND_GE] = S390_CC_GE,
437	[TCG_COND_LTU] = S390_CC_LT,
438	[TCG_COND_LEU] = S390_CC_LE,
439	[TCG_COND_GTU] = S390_CC_GT,
440	[TCG_COND_GEU] = S390_CC_GE,
441	};
442
443	/* Condition codes that result from a LOAD AND TEST. Here, we have no
444	unsigned instruction variation, however since the test is vs zero we
445	can re-map the outcomes appropriately. */
446	static const uint8_t tcg_cond_to_ltr_cond[16] = {
447	[TCG_COND_EQ] = S390_CC_EQ,
448	[TCG_COND_NE] = S390_CC_NE,
449	[TCG_COND_TSTEQ] = S390_CC_ALWAYS,
450	[TCG_COND_TSTNE] = S390_CC_NEVER,
451	[TCG_COND_LT] = S390_CC_LT,
452	[TCG_COND_LE] = S390_CC_LE,
453	[TCG_COND_GT] = S390_CC_GT,
454	[TCG_COND_GE] = S390_CC_GE,
455	[TCG_COND_LTU] = S390_CC_NEVER,
456	[TCG_COND_LEU] = S390_CC_EQ,
457	[TCG_COND_GTU] = S390_CC_NE,
458	[TCG_COND_GEU] = S390_CC_ALWAYS,
459	};
460
461	static const tcg_insn_unit *tb_ret_addr;
462	uint64_t s390_facilities[3];
463
464	static inline bool is_general_reg(TCGReg r)
465	{
466	return r <= TCG_REG_R15;
467	}
468
469	static inline bool is_vector_reg(TCGReg r)
470	{
471	return r >= TCG_REG_V0 && r <= TCG_REG_V31;
472	}
473
474	static bool patch_reloc(tcg_insn_unit *src_rw, int type,
475	intptr_t value, intptr_t addend)
476	{
477	const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
478	intptr_t pcrel2;
479	uint32_t old;
480
481	value += addend;
482	pcrel2 = (tcg_insn_unit *)value - src_rx;
483
484	switch (type) {
485	case R_390_PC16DBL:
486	if (pcrel2 == (int16_t)pcrel2) {
487	tcg_patch16(src_rw, pcrel2);
488	return true;
489	}
490	break;
491	case R_390_PC32DBL:
492	if (pcrel2 == (int32_t)pcrel2) {
493	tcg_patch32(src_rw, pcrel2);
494	return true;
495	}
496	break;
497	case R_390_20:
498	if (value == sextract64(value, 0, 20)) {
499	old = (uint32_t )src_rw & 0xf00000ff;
500	old \|= ((value & 0xfff) << 16) \| ((value & 0xff000) >> 4);
501	tcg_patch32(src_rw, old);
502	return true;
503	}
504	break;
505	default:
506	g_assert_not_reached();
507	}
508	return false;
509	}
510
511	static int is_const_p16(uint64_t val)
512	{
513	for (int i = 0; i < 4; ++i) {
514	uint64_t mask = 0xffffull << (i * 16);
515	if ((val & ~mask) == 0) {
516	return i;
517	}
518	}
519	return -1;
520	}
521
522	static int is_const_p32(uint64_t val)
523	{
524	if ((val & 0xffffffff00000000ull) == 0) {
525	return 0;
526	}
527	if ((val & 0x00000000ffffffffull) == 0) {
528	return 1;
529	}
530	return -1;
531	}
532
533	/*
534	* Accept bit patterns like these:
535	* 0....01....1
536	* 1....10....0
537	* 1..10..01..1
538	* 0..01..10..0
539	* Copied from gcc sources.
540	*/
541	static bool risbg_mask(uint64_t c)
542	{
543	uint64_t lsb;
544	/* We don't change the number of transitions by inverting,
545	so make sure we start with the LSB zero. */
546	if (c & 1) {
547	c = ~c;
548	}
549	/* Reject all zeros or all ones. */
550	if (c == 0) {
551	return false;
552	}
553	/* Find the first transition. */
554	lsb = c & -c;
555	/* Invert to look for a second transition. */
556	c = ~c;
557	/* Erase the first transition. */
558	c &= -lsb;
559	/* Find the second transition, if any. */
560	lsb = c & -c;
561	/* Match if all the bits are 1's, or if c is zero. */
562	return c == -lsb;
563	}
564
565	/* Test if a constant matches the constraint. */
566	static bool tcg_target_const_match(int64_t val, int ct,
567	TCGType type, TCGCond cond, int vece)
568	{
569	uint64_t uval = val;
570
571	if (ct & TCG_CT_CONST) {
572	return true;
573	}
574	if (type == TCG_TYPE_I32) {
575	uval = (uint32_t)val;
576	val = (int32_t)val;
577	}
578
579	if (ct & TCG_CT_CONST_CMP) {
580	if (is_tst_cond(cond)) {
581	if (is_const_p16(uval) >= 0) {
582	return true; /* TMxx */
583	}
584	if (risbg_mask(uval)) {
585	return true; /* RISBG */
586	}
587	return false;
588	}
589
590	if (type == TCG_TYPE_I32) {
591	return true;
592	}
593
594	switch (cond) {
595	case TCG_COND_EQ:
596	case TCG_COND_NE:
597	ct \|= TCG_CT_CONST_S32 \| TCG_CT_CONST_U32; /* CGFI or CLGFI */
598	break;
599	case TCG_COND_LT:
600	case TCG_COND_GE:
601	case TCG_COND_LE:
602	case TCG_COND_GT:
603	ct \|= TCG_CT_CONST_S32; /* CGFI */
604	break;
605	case TCG_COND_LTU:
606	case TCG_COND_GEU:
607	case TCG_COND_LEU:
608	case TCG_COND_GTU:
609	ct \|= TCG_CT_CONST_U32; /* CLGFI */
610	break;
611	case TCG_COND_TSTNE:
612	case TCG_COND_TSTEQ:
613	/* checked above, fallthru */
614	default:
615	g_assert_not_reached();
616	}
617	}
618
619	if ((ct & TCG_CT_CONST_S32) && val == (int32_t)val) {
620	return true;
621	}
622	if ((ct & TCG_CT_CONST_U32) && uval <= UINT32_MAX) {
623	return true;
624	}
625	if ((ct & TCG_CT_CONST_N32) && -uval <= UINT32_MAX) {
626	return true;
627	}
628	if ((ct & TCG_CT_CONST_S16) && val == (int16_t)val) {
629	return true;
630	}
631	if ((ct & TCG_CT_CONST_ZERO) && val == 0) {
632	return true;
633	}
634	if ((ct & TCG_CT_CONST_M1) && val == -1) {
635	return true;
636	}
637
638	if (ct & TCG_CT_CONST_INV) {
639	val = ~val;
640	}
641	if ((ct & TCG_CT_CONST_P32) && is_const_p32(val) >= 0) {
642	return true;
643	}
644	if ((ct & TCG_CT_CONST_INVRISBG) && risbg_mask(~val)) {
645	return true;
646	}
647	return false;
648	}
649
650	/* Emit instructions according to the given instruction format. */
651
652	static void tcg_out_insn_RR(TCGContext *s, S390Opcode op, TCGReg r1, TCGReg r2)
653	{
654	tcg_out16(s, (op << 8) \| (r1 << 4) \| r2);
655	}
656
657	static void tcg_out_insn_RRE(TCGContext *s, S390Opcode op,
658	TCGReg r1, TCGReg r2)
659	{
660	tcg_out32(s, (op << 16) \| (r1 << 4) \| r2);
661	}
662
663	/* RRF-a without the m4 field */
664	static void tcg_out_insn_RRFa(TCGContext *s, S390Opcode op,
665	TCGReg r1, TCGReg r2, TCGReg r3)
666	{
667	tcg_out32(s, (op << 16) \| (r3 << 12) \| (r1 << 4) \| r2);
668	}
669
670	/* RRF-a with the m4 field */
671	static void tcg_out_insn_RRFam(TCGContext *s, S390Opcode op,
672	TCGReg r1, TCGReg r2, TCGReg r3, int m4)
673	{
674	tcg_out32(s, (op << 16) \| (r3 << 12) \| (m4 << 8) \| (r1 << 4) \| r2);
675	}
676
677	static void tcg_out_insn_RRFc(TCGContext *s, S390Opcode op,
678	TCGReg r1, TCGReg r2, int m3)
679	{
680	tcg_out32(s, (op << 16) \| (m3 << 12) \| (r1 << 4) \| r2);
681	}
682
683	static void tcg_out_insn_RI(TCGContext *s, S390Opcode op, TCGReg r1, int i2)
684	{
685	tcg_out32(s, (op << 16) \| (r1 << 20) \| (i2 & 0xffff));
686	}
687
688	static void tcg_out_insn_RIEd(TCGContext *s, S390Opcode op,
689	TCGReg r1, TCGReg r3, int i2)
690	{
691	tcg_out16(s, (op & 0xff00) \| (r1 << 4) \| r3);
692	tcg_out16(s, i2);
693	tcg_out16(s, op & 0xff);
694	}
695
696	static void tcg_out_insn_RIEg(TCGContext *s, S390Opcode op, TCGReg r1,
697	int i2, int m3)
698	{
699	tcg_out16(s, (op & 0xff00) \| (r1 << 4) \| m3);
700	tcg_out32(s, (i2 << 16) \| (op & 0xff));
701	}
702
703	static void tcg_out_insn_RIL(TCGContext *s, S390Opcode op, TCGReg r1, int i2)
704	{
705	tcg_out16(s, op \| (r1 << 4));
706	tcg_out32(s, i2);
707	}
708
709	static void tcg_out_insn_RS(TCGContext *s, S390Opcode op, TCGReg r1,
710	TCGReg b2, TCGReg r3, int disp)
711	{
712	tcg_out32(s, (op << 24) \| (r1 << 20) \| (r3 << 16) \| (b2 << 12)
713	\| (disp & 0xfff));
714	}
715
716	static void tcg_out_insn_RSY(TCGContext *s, S390Opcode op, TCGReg r1,
717	TCGReg b2, TCGReg r3, int disp)
718	{
719	tcg_out16(s, (op & 0xff00) \| (r1 << 4) \| r3);
720	tcg_out32(s, (op & 0xff) \| (b2 << 28)
721	\| ((disp & 0xfff) << 16) \| ((disp & 0xff000) >> 4));
722	}
723
724	#define tcg_out_insn_RX tcg_out_insn_RS
725	#define tcg_out_insn_RXY tcg_out_insn_RSY
726
727	static int RXB(TCGReg v1, TCGReg v2, TCGReg v3, TCGReg v4)
728	{
729	/*
730	* Shift bit 4 of each regno to its corresponding bit of RXB.
731	* RXB itself begins at bit 8 of the instruction so 8 - 4 = 4
732	* is the left-shift of the 4th operand.
733	*/
734	return ((v1 & 0x10) << (4 + 3))
735	\| ((v2 & 0x10) << (4 + 2))
736	\| ((v3 & 0x10) << (4 + 1))
737	\| ((v4 & 0x10) << (4 + 0));
738	}
739
740	static void tcg_out_insn_VRIa(TCGContext *s, S390Opcode op,
741	TCGReg v1, uint16_t i2, int m3)
742	{
743	tcg_debug_assert(is_vector_reg(v1));
744	tcg_out16(s, (op & 0xff00) \| ((v1 & 0xf) << 4));
745	tcg_out16(s, i2);
746	tcg_out16(s, (op & 0x00ff) \| RXB(v1, 0, 0, 0) \| (m3 << 12));
747	}
748
749	static void tcg_out_insn_VRIb(TCGContext *s, S390Opcode op,
750	TCGReg v1, uint8_t i2, uint8_t i3, int m4)
751	{
752	tcg_debug_assert(is_vector_reg(v1));
753	tcg_out16(s, (op & 0xff00) \| ((v1 & 0xf) << 4));
754	tcg_out16(s, (i2 << 8) \| (i3 & 0xff));
755	tcg_out16(s, (op & 0x00ff) \| RXB(v1, 0, 0, 0) \| (m4 << 12));
756	}
757
758	static void tcg_out_insn_VRIc(TCGContext *s, S390Opcode op,
759	TCGReg v1, uint16_t i2, TCGReg v3, int m4)
760	{
761	tcg_debug_assert(is_vector_reg(v1));
762	tcg_debug_assert(is_vector_reg(v3));
763	tcg_out16(s, (op & 0xff00) \| ((v1 & 0xf) << 4) \| (v3 & 0xf));
764	tcg_out16(s, i2);
765	tcg_out16(s, (op & 0x00ff) \| RXB(v1, v3, 0, 0) \| (m4 << 12));
766	}
767
768	static void tcg_out_insn_VRRa(TCGContext *s, S390Opcode op,
769	TCGReg v1, TCGReg v2, int m3)
770	{
771	tcg_debug_assert(is_vector_reg(v1));
772	tcg_debug_assert(is_vector_reg(v2));
773	tcg_out16(s, (op & 0xff00) \| ((v1 & 0xf) << 4) \| (v2 & 0xf));
774	tcg_out32(s, (op & 0x00ff) \| RXB(v1, v2, 0, 0) \| (m3 << 12));
775	}
776
777	static void tcg_out_insn_VRRc(TCGContext *s, S390Opcode op,
778	TCGReg v1, TCGReg v2, TCGReg v3, int m4)
779	{
780	tcg_debug_assert(is_vector_reg(v1));
781	tcg_debug_assert(is_vector_reg(v2));
782	tcg_debug_assert(is_vector_reg(v3));
783	tcg_out16(s, (op & 0xff00) \| ((v1 & 0xf) << 4) \| (v2 & 0xf));
784	tcg_out16(s, v3 << 12);
785	tcg_out16(s, (op & 0x00ff) \| RXB(v1, v2, v3, 0) \| (m4 << 12));
786	}
787
788	static void tcg_out_insn_VRRe(TCGContext *s, S390Opcode op,
789	TCGReg v1, TCGReg v2, TCGReg v3, TCGReg v4)
790	{
791	tcg_debug_assert(is_vector_reg(v1));
792	tcg_debug_assert(is_vector_reg(v2));
793	tcg_debug_assert(is_vector_reg(v3));
794	tcg_debug_assert(is_vector_reg(v4));
795	tcg_out16(s, (op & 0xff00) \| ((v1 & 0xf) << 4) \| (v2 & 0xf));
796	tcg_out16(s, v3 << 12);
797	tcg_out16(s, (op & 0x00ff) \| RXB(v1, v2, v3, v4) \| (v4 << 12));
798	}
799
800	static void tcg_out_insn_VRRf(TCGContext *s, S390Opcode op,
801	TCGReg v1, TCGReg r2, TCGReg r3)
802	{
803	tcg_debug_assert(is_vector_reg(v1));
804	tcg_debug_assert(is_general_reg(r2));
805	tcg_debug_assert(is_general_reg(r3));
806	tcg_out16(s, (op & 0xff00) \| ((v1 & 0xf) << 4) \| r2);
807	tcg_out16(s, r3 << 12);
808	tcg_out16(s, (op & 0x00ff) \| RXB(v1, 0, 0, 0));
809	}
810
811	static void tcg_out_insn_VRSa(TCGContext *s, S390Opcode op, TCGReg v1,
812	intptr_t d2, TCGReg b2, TCGReg v3, int m4)
813	{
814	tcg_debug_assert(is_vector_reg(v1));
815	tcg_debug_assert(d2 >= 0 && d2 <= 0xfff);
816	tcg_debug_assert(is_general_reg(b2));
817	tcg_debug_assert(is_vector_reg(v3));
818	tcg_out16(s, (op & 0xff00) \| ((v1 & 0xf) << 4) \| (v3 & 0xf));
819	tcg_out16(s, b2 << 12 \| d2);
820	tcg_out16(s, (op & 0x00ff) \| RXB(v1, v3, 0, 0) \| (m4 << 12));
821	}
822
823	static void tcg_out_insn_VRSb(TCGContext *s, S390Opcode op, TCGReg v1,
824	intptr_t d2, TCGReg b2, TCGReg r3, int m4)
825	{
826	tcg_debug_assert(is_vector_reg(v1));
827	tcg_debug_assert(d2 >= 0 && d2 <= 0xfff);
828	tcg_debug_assert(is_general_reg(b2));
829	tcg_debug_assert(is_general_reg(r3));
830	tcg_out16(s, (op & 0xff00) \| ((v1 & 0xf) << 4) \| r3);
831	tcg_out16(s, b2 << 12 \| d2);
832	tcg_out16(s, (op & 0x00ff) \| RXB(v1, 0, 0, 0) \| (m4 << 12));
833	}
834
835	static void tcg_out_insn_VRSc(TCGContext *s, S390Opcode op, TCGReg r1,
836	intptr_t d2, TCGReg b2, TCGReg v3, int m4)
837	{
838	tcg_debug_assert(is_general_reg(r1));
839	tcg_debug_assert(d2 >= 0 && d2 <= 0xfff);
840	tcg_debug_assert(is_general_reg(b2));
841	tcg_debug_assert(is_vector_reg(v3));
842	tcg_out16(s, (op & 0xff00) \| (r1 << 4) \| (v3 & 0xf));
843	tcg_out16(s, b2 << 12 \| d2);
844	tcg_out16(s, (op & 0x00ff) \| RXB(0, v3, 0, 0) \| (m4 << 12));
845	}
846
847	static void tcg_out_insn_VRX(TCGContext *s, S390Opcode op, TCGReg v1,
848	TCGReg b2, TCGReg x2, intptr_t d2, int m3)
849	{
850	tcg_debug_assert(is_vector_reg(v1));
851	tcg_debug_assert(d2 >= 0 && d2 <= 0xfff);
852	tcg_debug_assert(is_general_reg(x2));
853	tcg_debug_assert(is_general_reg(b2));
854	tcg_out16(s, (op & 0xff00) \| ((v1 & 0xf) << 4) \| x2);
855	tcg_out16(s, (b2 << 12) \| d2);
856	tcg_out16(s, (op & 0x00ff) \| RXB(v1, 0, 0, 0) \| (m3 << 12));
857	}
858
859	/* Emit an opcode with "type-checking" of the format. */
860	#define tcg_out_insn(S, FMT, OP, ...) \
861	glue(tcg_out_insn_,FMT)(S, glue(glue(FMT,_),OP), ## __VA_ARGS__)
862
863
864	/* emit 64-bit shifts */
865	static void tcg_out_sh64(TCGContext* s, S390Opcode op, TCGReg dest,
866	TCGReg src, TCGReg sh_reg, int sh_imm)
867	{
868	tcg_out_insn_RSY(s, op, dest, sh_reg, src, sh_imm);
869	}
870
871	/* emit 32-bit shifts */
872	static void tcg_out_sh32(TCGContext* s, S390Opcode op, TCGReg dest,
873	TCGReg sh_reg, int sh_imm)
874	{
875	tcg_out_insn_RS(s, op, dest, sh_reg, 0, sh_imm);
876	}
877
878	static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg dst, TCGReg src)
879	{
880	if (src == dst) {
881	return true;
882	}
883	switch (type) {
884	case TCG_TYPE_I32:
885	if (likely(is_general_reg(dst) && is_general_reg(src))) {
886	tcg_out_insn(s, RR, LR, dst, src);
887	break;
888	}
889	/* fallthru */
890
891	case TCG_TYPE_I64:
892	if (likely(is_general_reg(dst))) {
893	if (likely(is_general_reg(src))) {
894	tcg_out_insn(s, RRE, LGR, dst, src);
895	} else {
896	tcg_out_insn(s, VRSc, VLGV, dst, 0, 0, src, 3);
897	}
898	break;
899	} else if (is_general_reg(src)) {
900	tcg_out_insn(s, VRSb, VLVG, dst, 0, 0, src, 3);
901	break;
902	}
903	/* fallthru */
904
905	case TCG_TYPE_V64:
906	case TCG_TYPE_V128:
907	tcg_out_insn(s, VRRa, VLR, dst, src, 0);
908	break;
909
910	default:
911	g_assert_not_reached();
912	}
913	return true;
914	}
915
916	static const S390Opcode li_insns[4] = {
917	RI_LLILL, RI_LLILH, RI_LLIHL, RI_LLIHH
918	};
919	static const S390Opcode oi_insns[4] = {
920	RI_OILL, RI_OILH, RI_OIHL, RI_OIHH
921	};
922	static const S390Opcode lif_insns[2] = {
923	RIL_LLILF, RIL_LLIHF,
924	};
925	static const S390Opcode tm_insns[4] = {
926	RI_TMLL, RI_TMLH, RI_TMHL, RI_TMHH
927	};
928
929	/* load a register with an immediate value */
930	static void tcg_out_movi(TCGContext *s, TCGType type,
931	TCGReg ret, tcg_target_long sval)
932	{
933	tcg_target_ulong uval = sval;
934	ptrdiff_t pc_off;
935	int i;
936
937	if (type == TCG_TYPE_I32) {
938	uval = (uint32_t)sval;
939	sval = (int32_t)sval;
940	}
941
942	/* Try all 32-bit insns that can load it in one go. */
943	if (sval >= -0x8000 && sval < 0x8000) {
944	tcg_out_insn(s, RI, LGHI, ret, sval);
945	return;
946	}
947
948	i = is_const_p16(uval);
949	if (i >= 0) {
950	tcg_out_insn_RI(s, li_insns[i], ret, uval >> (i * 16));
951	return;
952	}
953
954	/* Try all 48-bit insns that can load it in one go. */
955	if (sval == (int32_t)sval) {
956	tcg_out_insn(s, RIL, LGFI, ret, sval);
957	return;
958	}
959
960	i = is_const_p32(uval);
961	if (i >= 0) {
962	tcg_out_insn_RIL(s, lif_insns[i], ret, uval >> (i * 32));
963	return;
964	}
965
966	/* Try for PC-relative address load. For odd addresses, add one. */
967	pc_off = tcg_pcrel_diff(s, (void *)sval) >> 1;
968	if (pc_off == (int32_t)pc_off) {
969	tcg_out_insn(s, RIL, LARL, ret, pc_off);
970	if (sval & 1) {
971	tcg_out_insn(s, RX, LA, ret, ret, TCG_REG_NONE, 1);
972	}
973	return;
974	}
975
976	if (!s->carry_live) {
977	/* Load by parts, at most 2 instructions. */
978	i = is_const_p16((uint32_t)uval);
979	if (i >= 0) {
980	tcg_out_insn_RI(s, li_insns[i], ret, uval >> (i * 16));
981	} else {
982	tcg_out_insn(s, RIL, LLILF, ret, uval);
983	}
984	uval >>= 32;
985	i = is_const_p16(uval);
986	if (i >= 0) {
987	tcg_out_insn_RI(s, oi_insns[i + 2], ret, uval >> (i * 16));
988	} else {
989	tcg_out_insn(s, RIL, OIHF, ret, uval);
990	}
991	return;
992	}
993
994	/* Otherwise, stuff it in the constant pool. */
995	tcg_out_insn(s, RIL, LGRL, ret, 0);
996	new_pool_label(s, sval, R_390_PC32DBL, s->code_ptr - 2, 2);
997	}
998
999	/* Emit a load/store type instruction. Inputs are:
1000	DATA: The register to be loaded or stored.
1001	BASE+OFS: The effective address.
1002	OPC_RX: If the operation has an RX format opcode (e.g. STC), otherwise 0.
1003	OPC_RXY: The RXY format opcode for the operation (e.g. STCY). */
1004
1005	static void tcg_out_mem(TCGContext *s, S390Opcode opc_rx, S390Opcode opc_rxy,
1006	TCGReg data, TCGReg base, TCGReg index,
1007	tcg_target_long ofs)
1008	{
1009	if (ofs < -0x80000 \|\| ofs >= 0x80000) {
1010	/* Combine the low 20 bits of the offset with the actual load insn;
1011	the high 44 bits must come from an immediate load. */
1012	tcg_target_long low = ((ofs & 0xfffff) ^ 0x80000) - 0x80000;
1013	tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, ofs - low);
1014	ofs = low;
1015
1016	/* If we were already given an index register, add it in. */
1017	if (index != TCG_REG_NONE) {
1018	tcg_out_insn(s, RRE, AGR, TCG_TMP0, index);
1019	}
1020	index = TCG_TMP0;
1021	}
1022
1023	if (opc_rx && ofs >= 0 && ofs < 0x1000) {
1024	tcg_out_insn_RX(s, opc_rx, data, base, index, ofs);
1025	} else {
1026	tcg_out_insn_RXY(s, opc_rxy, data, base, index, ofs);
1027	}
1028	}
1029
1030	static void tcg_out_vrx_mem(TCGContext *s, S390Opcode opc_vrx,
1031	TCGReg data, TCGReg base, TCGReg index,
1032	tcg_target_long ofs, int m3)
1033	{
1034	if (ofs < 0 \|\| ofs >= 0x1000) {
1035	if (ofs >= -0x80000 && ofs < 0x80000) {
1036	tcg_out_insn(s, RXY, LAY, TCG_TMP0, base, index, ofs);
1037	base = TCG_TMP0;
1038	index = TCG_REG_NONE;
1039	ofs = 0;
1040	} else {
1041	tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, ofs);
1042	if (index != TCG_REG_NONE) {
1043	tcg_out_insn(s, RRE, AGR, TCG_TMP0, index);
1044	}
1045	index = TCG_TMP0;
1046	ofs = 0;
1047	}
1048	}
1049	tcg_out_insn_VRX(s, opc_vrx, data, base, index, ofs, m3);
1050	}
1051
1052	/* load data without address translation or endianness conversion */
1053	static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg data,
1054	TCGReg base, intptr_t ofs)
1055	{
1056	switch (type) {
1057	case TCG_TYPE_I32:
1058	if (likely(is_general_reg(data))) {
1059	tcg_out_mem(s, RX_L, RXY_LY, data, base, TCG_REG_NONE, ofs);
1060	break;
1061	}
1062	tcg_out_vrx_mem(s, VRX_VLLEZ, data, base, TCG_REG_NONE, ofs, MO_32);
1063	break;
1064
1065	case TCG_TYPE_I64:
1066	if (likely(is_general_reg(data))) {
1067	tcg_out_mem(s, 0, RXY_LG, data, base, TCG_REG_NONE, ofs);
1068	break;
1069	}
1070	/* fallthru */
1071
1072	case TCG_TYPE_V64:
1073	tcg_out_vrx_mem(s, VRX_VLLEZ, data, base, TCG_REG_NONE, ofs, MO_64);
1074	break;
1075
1076	case TCG_TYPE_V128:
1077	/* Hint quadword aligned. */
1078	tcg_out_vrx_mem(s, VRX_VL, data, base, TCG_REG_NONE, ofs, 4);
1079	break;
1080
1081	default:
1082	g_assert_not_reached();
1083	}
1084	}
1085
1086	static void tcg_out_st(TCGContext *s, TCGType type, TCGReg data,
1087	TCGReg base, intptr_t ofs)
1088	{
1089	switch (type) {
1090	case TCG_TYPE_I32:
1091	if (likely(is_general_reg(data))) {
1092	tcg_out_mem(s, RX_ST, RXY_STY, data, base, TCG_REG_NONE, ofs);
1093	} else {
1094	tcg_out_vrx_mem(s, VRX_VSTEF, data, base, TCG_REG_NONE, ofs, 1);
1095	}
1096	break;
1097
1098	case TCG_TYPE_I64:
1099	if (likely(is_general_reg(data))) {
1100	tcg_out_mem(s, 0, RXY_STG, data, base, TCG_REG_NONE, ofs);
1101	break;
1102	}
1103	/* fallthru */
1104
1105	case TCG_TYPE_V64:
1106	tcg_out_vrx_mem(s, VRX_VSTEG, data, base, TCG_REG_NONE, ofs, 0);
1107	break;
1108
1109	case TCG_TYPE_V128:
1110	/* Hint quadword aligned. */
1111	tcg_out_vrx_mem(s, VRX_VST, data, base, TCG_REG_NONE, ofs, 4);
1112	break;
1113
1114	default:
1115	g_assert_not_reached();
1116	}
1117	}
1118
1119	static inline bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
1120	TCGReg base, intptr_t ofs)
1121	{
1122	return false;
1123	}
1124
1125	static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
1126	{
1127	return false;
1128	}
1129
1130	static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
1131	tcg_target_long imm)
1132	{
1133	/* This function is only used for passing structs by reference. */
1134	tcg_out_mem(s, RX_LA, RXY_LAY, rd, rs, TCG_REG_NONE, imm);
1135	}
1136
1137	static inline void tcg_out_risbg(TCGContext *s, TCGReg dest, TCGReg src,
1138	int msb, int lsb, int ofs, int z)
1139	{
1140	/* Format RIE-f */
1141	tcg_out16(s, (RIEf_RISBG & 0xff00) \| (dest << 4) \| src);
1142	tcg_out16(s, (msb << 8) \| (z << 7) \| lsb);
1143	tcg_out16(s, (ofs << 8) \| (RIEf_RISBG & 0xff));
1144	}
1145
1146	static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
1147	{
1148	tcg_out_insn(s, RRE, LGBR, dest, src);
1149	}
1150
1151	static void tcg_out_ext8u(TCGContext *s, TCGReg dest, TCGReg src)
1152	{
1153	tcg_out_insn(s, RRE, LLGCR, dest, src);
1154	}
1155
1156	static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
1157	{
1158	tcg_out_insn(s, RRE, LGHR, dest, src);
1159	}
1160
1161	static void tcg_out_ext16u(TCGContext *s, TCGReg dest, TCGReg src)
1162	{
1163	tcg_out_insn(s, RRE, LLGHR, dest, src);
1164	}
1165
1166	static void tcg_out_ext32s(TCGContext *s, TCGReg dest, TCGReg src)
1167	{
1168	tcg_out_insn(s, RRE, LGFR, dest, src);
1169	}
1170
1171	static void tcg_out_ext32u(TCGContext *s, TCGReg dest, TCGReg src)
1172	{
1173	tcg_out_insn(s, RRE, LLGFR, dest, src);
1174	}
1175
1176	static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg dest, TCGReg src)
1177	{
1178	tcg_out_ext32s(s, dest, src);
1179	}
1180
1181	static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg dest, TCGReg src)
1182	{
1183	tcg_out_ext32u(s, dest, src);
1184	}
1185
1186	static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg dest, TCGReg src)
1187	{
1188	tcg_out_mov(s, TCG_TYPE_I32, dest, src);
1189	}
1190
1191	static void tgen_andi_risbg(TCGContext *s, TCGReg out, TCGReg in, uint64_t val)
1192	{
1193	int msb, lsb;
1194	if ((val & 0x8000000000000001ull) == 0x8000000000000001ull) {
1195	/* Achieve wraparound by swapping msb and lsb. */
1196	msb = 64 - ctz64(~val);
1197	lsb = clz64(~val) - 1;
1198	} else {
1199	msb = clz64(val);
1200	lsb = 63 - ctz64(val);
1201	}
1202	tcg_out_risbg(s, out, in, msb, lsb, 0, 1);
1203	}
1204
1205	static void tgen_andi(TCGContext *s, TCGType type, TCGReg dest, uint64_t val)
1206	{
1207	static const S390Opcode ni_insns[4] = {
1208	RI_NILL, RI_NILH, RI_NIHL, RI_NIHH
1209	};
1210	static const S390Opcode nif_insns[2] = {
1211	RIL_NILF, RIL_NIHF
1212	};
1213	uint64_t valid = (type == TCG_TYPE_I32 ? 0xffffffffull : -1ull);
1214	int i;
1215
1216	/* Look for the zero-extensions. */
1217	if ((val & valid) == 0xffffffff) {
1218	tcg_out_ext32u(s, dest, dest);
1219	return;
1220	}
1221	if ((val & valid) == 0xff) {
1222	tcg_out_ext8u(s, dest, dest);
1223	return;
1224	}
1225	if ((val & valid) == 0xffff) {
1226	tcg_out_ext16u(s, dest, dest);
1227	return;
1228	}
1229
1230	i = is_const_p16(~val & valid);
1231	if (i >= 0) {
1232	tcg_out_insn_RI(s, ni_insns[i], dest, val >> (i * 16));
1233	return;
1234	}
1235
1236	i = is_const_p32(~val & valid);
1237	tcg_debug_assert(i == 0 \|\| type != TCG_TYPE_I32);
1238	if (i >= 0) {
1239	tcg_out_insn_RIL(s, nif_insns[i], dest, val >> (i * 32));
1240	return;
1241	}
1242
1243	if (risbg_mask(val)) {
1244	tgen_andi_risbg(s, dest, dest, val);
1245	return;
1246	}
1247
1248	g_assert_not_reached();
1249	}
1250
1251	static void tgen_ori(TCGContext *s, TCGReg dest, uint64_t val)
1252	{
1253	static const S390Opcode oif_insns[2] = {
1254	RIL_OILF, RIL_OIHF
1255	};
1256
1257	int i;
1258
1259	i = is_const_p16(val);
1260	if (i >= 0) {
1261	tcg_out_insn_RI(s, oi_insns[i], dest, val >> (i * 16));
1262	return;
1263	}
1264
1265	i = is_const_p32(val);
1266	if (i >= 0) {
1267	tcg_out_insn_RIL(s, oif_insns[i], dest, val >> (i * 32));
1268	return;
1269	}
1270
1271	g_assert_not_reached();
1272	}
1273
1274	static void tgen_xori(TCGContext *s, TCGReg dest, uint64_t val)
1275	{
1276	switch (is_const_p32(val)) {
1277	case 0:
1278	tcg_out_insn(s, RIL, XILF, dest, val);
1279	break;
1280	case 1:
1281	tcg_out_insn(s, RIL, XIHF, dest, val >> 32);
1282	break;
1283	default:
1284	g_assert_not_reached();
1285	}
1286	}
1287
1288	static int tgen_cmp2(TCGContext *s, TCGType type, TCGCond c, TCGReg r1,
1289	TCGArg c2, bool c2const, bool need_carry, int *inv_cc)
1290	{
1291	bool is_unsigned = is_unsigned_cond(c);
1292	TCGCond inv_c = tcg_invert_cond(c);
1293	S390Opcode op;
1294
1295	if (is_tst_cond(c)) {
1296	tcg_debug_assert(!need_carry);
1297
1298	if (!c2const) {
1299	if (type == TCG_TYPE_I32) {
1300	tcg_out_insn(s, RRFa, NRK, TCG_REG_R0, r1, c2);
1301	} else {
1302	tcg_out_insn(s, RRFa, NGRK, TCG_REG_R0, r1, c2);
1303	}
1304	goto exit;
1305	}
1306
1307	if (type == TCG_TYPE_I32) {
1308	c2 = (uint32_t)c2;
1309	}
1310
1311	int i = is_const_p16(c2);
1312	if (i >= 0) {
1313	tcg_out_insn_RI(s, tm_insns[i], r1, c2 >> (i * 16));
1314	*inv_cc = c == TCG_COND_TSTEQ ? S390_TM_NE : S390_TM_EQ;
1315	return *inv_cc ^ 15;
1316	}
1317
1318	if (risbg_mask(c2)) {
1319	tgen_andi_risbg(s, TCG_REG_R0, r1, c2);
1320	goto exit;
1321	}
1322	g_assert_not_reached();
1323	}
1324
1325	if (c2const) {
1326	if (c2 == 0) {
1327	if (!(is_unsigned && need_carry)) {
1328	if (type == TCG_TYPE_I32) {
1329	tcg_out_insn(s, RR, LTR, r1, r1);
1330	} else {
1331	tcg_out_insn(s, RRE, LTGR, r1, r1);
1332	}
1333	*inv_cc = tcg_cond_to_ltr_cond[inv_c];
1334	return tcg_cond_to_ltr_cond[c];
1335	}
1336	}
1337
1338	if (!is_unsigned && c2 == (int16_t)c2) {
1339	op = (type == TCG_TYPE_I32 ? RI_CHI : RI_CGHI);
1340	tcg_out_insn_RI(s, op, r1, c2);
1341	goto exit;
1342	}
1343
1344	if (type == TCG_TYPE_I32) {
1345	op = (is_unsigned ? RIL_CLFI : RIL_CFI);
1346	tcg_out_insn_RIL(s, op, r1, c2);
1347	goto exit;
1348	}
1349
1350	/* Should match TCG_CT_CONST_CMP. */
1351	switch (c) {
1352	case TCG_COND_LT:
1353	case TCG_COND_GE:
1354	case TCG_COND_LE:
1355	case TCG_COND_GT:
1356	tcg_debug_assert(c2 == (int32_t)c2);
1357	op = RIL_CGFI;
1358	break;
1359	case TCG_COND_EQ:
1360	case TCG_COND_NE:
1361	if (c2 == (int32_t)c2) {
1362	op = RIL_CGFI;
1363	break;
1364	}
1365	/* fall through */
1366	case TCG_COND_LTU:
1367	case TCG_COND_GEU:
1368	case TCG_COND_LEU:
1369	case TCG_COND_GTU:
1370	tcg_debug_assert(c2 == (uint32_t)c2);
1371	op = RIL_CLGFI;
1372	break;
1373	default:
1374	g_assert_not_reached();
1375	}
1376	tcg_out_insn_RIL(s, op, r1, c2);
1377	} else if (type == TCG_TYPE_I32) {
1378	op = (is_unsigned ? RR_CLR : RR_CR);
1379	tcg_out_insn_RR(s, op, r1, c2);
1380	} else {
1381	op = (is_unsigned ? RRE_CLGR : RRE_CGR);
1382	tcg_out_insn_RRE(s, op, r1, c2);
1383	}
1384
1385	exit:
1386	*inv_cc = tcg_cond_to_s390_cond[inv_c];
1387	return tcg_cond_to_s390_cond[c];
1388	}
1389
1390	static int tgen_cmp(TCGContext *s, TCGType type, TCGCond c, TCGReg r1,
1391	TCGArg c2, bool c2const, bool need_carry)
1392	{
1393	int inv_cc;
1394	return tgen_cmp2(s, type, c, r1, c2, c2const, need_carry, &inv_cc);
1395	}
1396
1397	static void tgen_setcond_int(TCGContext *s, TCGType type, TCGCond cond,
1398	TCGReg dest, TCGReg c1, TCGArg c2,
1399	bool c2const, bool neg)
1400	{
1401	int cc;
1402
1403	/* With LOC2, we can always emit the minimum 3 insns. */
1404	if (HAVE_FACILITY(LOAD_ON_COND2)) {
1405	/* Emit: d = 0, d = (cc ? 1 : d). */
1406	cc = tgen_cmp(s, type, cond, c1, c2, c2const, false);
1407	tcg_out_movi(s, TCG_TYPE_I64, dest, 0);
1408	tcg_out_insn(s, RIEg, LOCGHI, dest, neg ? -1 : 1, cc);
1409	return;
1410	}
1411
1412	switch (cond) {
1413	case TCG_COND_GEU:
1414	case TCG_COND_LTU:
1415	case TCG_COND_LT:
1416	case TCG_COND_GE:
1417	/* Swap operands so that we can use LEU/GTU/GT/LE. */
1418	if (!c2const) {
1419	TCGReg t = c1;
1420	c1 = c2;
1421	c2 = t;
1422	cond = tcg_swap_cond(cond);
1423	}
1424	break;
1425	default:
1426	break;
1427	}
1428
1429	switch (cond) {
1430	case TCG_COND_NE:
1431	/* X != 0 is X > 0. */
1432	if (c2const && c2 == 0) {
1433	cond = TCG_COND_GTU;
1434	} else {
1435	break;
1436	}
1437	/* fallthru */
1438
1439	case TCG_COND_GTU:
1440	case TCG_COND_GT:
1441	/*
1442	* The result of a compare has CC=2 for GT and CC=3 unused.
1443	* ADD LOGICAL WITH CARRY considers (CC & 2) the carry bit.
1444	*/
1445	tgen_cmp(s, type, cond, c1, c2, c2const, true);
1446	tcg_out_movi(s, type, dest, 0);
1447	tcg_out_insn(s, RRE, ALCGR, dest, dest);
1448	if (neg) {
1449	if (type == TCG_TYPE_I32) {
1450	tcg_out_insn(s, RR, LCR, dest, dest);
1451	} else {
1452	tcg_out_insn(s, RRE, LCGR, dest, dest);
1453	}
1454	}
1455	return;
1456
1457	case TCG_COND_EQ:
1458	/* X == 0 is X <= 0. */
1459	if (c2const && c2 == 0) {
1460	cond = TCG_COND_LEU;
1461	} else {
1462	break;
1463	}
1464	/* fallthru */
1465
1466	case TCG_COND_LEU:
1467	case TCG_COND_LE:
1468	/*
1469	* As above, but we're looking for borrow, or !carry.
1470	* The second insn computes d - d - borrow, or -1 for true
1471	* and 0 for false. So we must mask to 1 bit afterward.
1472	*/
1473	tgen_cmp(s, type, cond, c1, c2, c2const, true);
1474	tcg_out_insn(s, RRE, SLBGR, dest, dest);
1475	if (!neg) {
1476	tgen_andi(s, type, dest, 1);
1477	}
1478	return;
1479
1480	default:
1481	g_assert_not_reached();
1482	}
1483
1484	cc = tgen_cmp(s, type, cond, c1, c2, c2const, false);
1485	/* Emit: d = 0, t = 1, d = (cc ? t : d). */
1486	tcg_out_movi(s, TCG_TYPE_I64, dest, 0);
1487	tcg_out_movi(s, TCG_TYPE_I64, TCG_TMP0, neg ? -1 : 1);
1488	tcg_out_insn(s, RRFc, LOCGR, dest, TCG_TMP0, cc);
1489	}
1490
1491	static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond,
1492	TCGReg dest, TCGReg arg1, TCGReg arg2)
1493	{
1494	tgen_setcond_int(s, type, cond, dest, arg1, arg2, false, false);
1495	}
1496
1497	static void tgen_setcondi(TCGContext *s, TCGType type, TCGCond cond,
1498	TCGReg dest, TCGReg arg1, tcg_target_long arg2)
1499	{
1500	tgen_setcond_int(s, type, cond, dest, arg1, arg2, true, false);
1501	}
1502
1503	static const TCGOutOpSetcond outop_setcond = {
1504	.base.static_constraint = C_O1_I2(r, r, rC),
1505	.out_rrr = tgen_setcond,
1506	.out_rri = tgen_setcondi,
1507	};
1508
1509	static void tgen_negsetcond(TCGContext *s, TCGType type, TCGCond cond,
1510	TCGReg dest, TCGReg arg1, TCGReg arg2)
1511	{
1512	tgen_setcond_int(s, type, cond, dest, arg1, arg2, false, true);
1513	}
1514
1515	static void tgen_negsetcondi(TCGContext *s, TCGType type, TCGCond cond,
1516	TCGReg dest, TCGReg arg1, tcg_target_long arg2)
1517	{
1518	tgen_setcond_int(s, type, cond, dest, arg1, arg2, true, true);
1519	}
1520
1521	static const TCGOutOpSetcond outop_negsetcond = {
1522	.base.static_constraint = C_O1_I2(r, r, rC),
1523	.out_rrr = tgen_negsetcond,
1524	.out_rri = tgen_negsetcondi,
1525	};
1526
1527	static void tgen_movcond_int(TCGContext *s, TCGType type, TCGReg dest,
1528	TCGArg v3, int v3const, TCGReg v4,
1529	int cc, int inv_cc)
1530	{
1531	TCGReg src;
1532
1533	if (v3const) {
1534	if (dest == v4) {
1535	if (HAVE_FACILITY(LOAD_ON_COND2)) {
1536	/* Emit: if (cc) dest = v3. */
1537	tcg_out_insn(s, RIEg, LOCGHI, dest, v3, cc);
1538	return;
1539	}
1540	tcg_out_insn(s, RI, LGHI, TCG_TMP0, v3);
1541	src = TCG_TMP0;
1542	} else {
1543	/* LGR+LOCGHI is larger than LGHI+LOCGR. */
1544	tcg_out_insn(s, RI, LGHI, dest, v3);
1545	cc = inv_cc;
1546	src = v4;
1547	}
1548	} else {
1549	if (HAVE_FACILITY(MISC_INSN_EXT3)) {
1550	/* Emit: dest = cc ? v3 : v4. */
1551	tcg_out_insn(s, RRFam, SELGR, dest, v3, v4, cc);
1552	return;
1553	}
1554	if (dest == v4) {
1555	src = v3;
1556	} else {
1557	tcg_out_mov(s, type, dest, v3);
1558	cc = inv_cc;
1559	src = v4;
1560	}
1561	}
1562
1563	/* Emit: if (cc) dest = src. */
1564	tcg_out_insn(s, RRFc, LOCGR, dest, src, cc);
1565	}
1566
1567	static void tgen_movcond(TCGContext *s, TCGType type, TCGCond c,
1568	TCGReg dest, TCGReg c1, TCGArg c2, bool c2const,
1569	TCGArg v3, bool v3const, TCGArg v4, bool v4const)
1570	{
1571	int cc, inv_cc;
1572
1573	cc = tgen_cmp2(s, type, c, c1, c2, c2const, false, &inv_cc);
1574	tgen_movcond_int(s, type, dest, v3, v3const, v4, cc, inv_cc);
1575	}
1576
1577	static const TCGOutOpMovcond outop_movcond = {
1578	.base.static_constraint = C_O1_I4(r, r, rC, rI, r),
1579	.out = tgen_movcond,
1580	};
1581
1582	static void tgen_deposit(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
1583	TCGReg a2, unsigned ofs, unsigned len)
1584	{
1585	unsigned lsb = (63 - ofs);
1586	unsigned msb = lsb - (len - 1);
1587
1588	/*
1589	* Since we can't support "0Z" as a constraint, we allow a1 in
1590	* any register. Fix things up as if a matching constraint.
1591	*/
1592	if (a0 != a1) {
1593	if (a0 == a2) {
1594	tcg_out_mov(s, type, TCG_TMP0, a2);
1595	a2 = TCG_TMP0;
1596	}
1597	tcg_out_mov(s, type, a0, a1);
1598	}
1599	tcg_out_risbg(s, a0, a2, msb, lsb, ofs, false);
1600	}
1601
1602	static void tgen_depositz(TCGContext *s, TCGType type, TCGReg a0, TCGReg a2,
1603	unsigned ofs, unsigned len)
1604	{
1605	unsigned lsb = (63 - ofs);
1606	unsigned msb = lsb - (len - 1);
1607	tcg_out_risbg(s, a0, a2, msb, lsb, ofs, true);
1608	}
1609
1610	static const TCGOutOpDeposit outop_deposit = {
1611	.base.static_constraint = C_O1_I2(r, rZ, r),
1612	.out_rrr = tgen_deposit,
1613	.out_rzr = tgen_depositz,
1614	};
1615
1616	static void tgen_extract(TCGContext *s, TCGType type, TCGReg dest,
1617	TCGReg src, unsigned ofs, unsigned len)
1618	{
1619	if (ofs == 0) {
1620	switch (len) {
1621	case 8:
1622	tcg_out_ext8u(s, dest, src);
1623	return;
1624	case 16:
1625	tcg_out_ext16u(s, dest, src);
1626	return;
1627	case 32:
1628	tcg_out_ext32u(s, dest, src);
1629	return;
1630	}
1631	}
1632	tcg_out_risbg(s, dest, src, 64 - len, 63, 64 - ofs, 1);
1633	}
1634
1635	static const TCGOutOpExtract outop_extract = {
1636	.base.static_constraint = C_O1_I1(r, r),
1637	.out_rr = tgen_extract,
1638	};
1639
1640	static void tgen_sextract(TCGContext *s, TCGType type, TCGReg dest,
1641	TCGReg src, unsigned ofs, unsigned len)
1642	{
1643	if (ofs == 0) {
1644	switch (len) {
1645	case 8:
1646	tcg_out_ext8s(s, TCG_TYPE_REG, dest, src);
1647	return;
1648	case 16:
1649	tcg_out_ext16s(s, TCG_TYPE_REG, dest, src);
1650	return;
1651	case 32:
1652	tcg_out_ext32s(s, dest, src);
1653	return;
1654	}
1655	}
1656	g_assert_not_reached();
1657	}
1658
1659	static const TCGOutOpExtract outop_sextract = {
1660	.base.static_constraint = C_O1_I1(r, r),
1661	.out_rr = tgen_sextract,
1662	};
1663
1664	static const TCGOutOpExtract2 outop_extract2 = {
1665	.base.static_constraint = C_NotImplemented,
1666	};
1667
1668	static void tgen_gotoi(TCGContext s, int cc, const tcg_insn_unit dest)
1669	{
1670	ptrdiff_t off = tcg_pcrel_diff(s, dest) >> 1;
1671	if (off == (int16_t)off) {
1672	tcg_out_insn(s, RI, BRC, cc, off);
1673	} else if (off == (int32_t)off) {
1674	tcg_out_insn(s, RIL, BRCL, cc, off);
1675	} else {
1676	tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, (uintptr_t)dest);
1677	tcg_out_insn(s, RR, BCR, cc, TCG_TMP0);
1678	}
1679	}
1680
1681	static void tgen_branch(TCGContext s, int cc, TCGLabel l)
1682	{
1683	if (l->has_value) {
1684	tgen_gotoi(s, cc, l->u.value_ptr);
1685	} else {
1686	tcg_out16(s, RI_BRC \| (cc << 4));
1687	tcg_out_reloc(s, s->code_ptr, R_390_PC16DBL, l, 2);
1688	s->code_ptr += 1;
1689	}
1690	}
1691
1692	static void tcg_out_br(TCGContext s, TCGLabel l)
1693	{
1694	tgen_branch(s, S390_CC_ALWAYS, l);
1695	}
1696
1697	static void tgen_compare_branch(TCGContext *s, S390Opcode opc, int cc,
1698	TCGReg r1, TCGReg r2, TCGLabel *l)
1699	{
1700	tcg_out_reloc(s, s->code_ptr + 1, R_390_PC16DBL, l, 2);
1701	/* Format RIE-b */
1702	tcg_out16(s, (opc & 0xff00) \| (r1 << 4) \| r2);
1703	tcg_out16(s, 0);
1704	tcg_out16(s, cc << 12 \| (opc & 0xff));
1705	}
1706
1707	static void tgen_compare_imm_branch(TCGContext *s, S390Opcode opc, int cc,
1708	TCGReg r1, int i2, TCGLabel *l)
1709	{
1710	tcg_out_reloc(s, s->code_ptr + 1, R_390_PC16DBL, l, 2);
1711	/* Format RIE-c */
1712	tcg_out16(s, (opc & 0xff00) \| (r1 << 4) \| cc);
1713	tcg_out16(s, 0);
1714	tcg_out16(s, (i2 << 8) \| (opc & 0xff));
1715	}
1716
1717	static void tgen_brcond(TCGContext *s, TCGType type, TCGCond c,
1718	TCGReg r1, TCGArg c2, int c2const, TCGLabel *l)
1719	{
1720	int cc;
1721
1722	if (!is_tst_cond(c)) {
1723	bool is_unsigned = is_unsigned_cond(c);
1724	bool in_range;
1725	S390Opcode opc;
1726
1727	cc = tcg_cond_to_s390_cond[c];
1728
1729	if (!c2const) {
1730	opc = (type == TCG_TYPE_I32
1731	? (is_unsigned ? RIEb_CLRJ : RIEb_CRJ)
1732	: (is_unsigned ? RIEb_CLGRJ : RIEb_CGRJ));
1733	tgen_compare_branch(s, opc, cc, r1, c2, l);
1734	return;
1735	}
1736
1737	/*
1738	* COMPARE IMMEDIATE AND BRANCH RELATIVE has an 8-bit immediate field.
1739	* If the immediate we've been given does not fit that range, we'll
1740	* fall back to separate compare and branch instructions using the
1741	* larger comparison range afforded by COMPARE IMMEDIATE.
1742	*/
1743	if (type == TCG_TYPE_I32) {
1744	if (is_unsigned) {
1745	opc = RIEc_CLIJ;
1746	in_range = (uint32_t)c2 == (uint8_t)c2;
1747	} else {
1748	opc = RIEc_CIJ;
1749	in_range = (int32_t)c2 == (int8_t)c2;
1750	}
1751	} else {
1752	if (is_unsigned) {
1753	opc = RIEc_CLGIJ;
1754	in_range = (uint64_t)c2 == (uint8_t)c2;
1755	} else {
1756	opc = RIEc_CGIJ;
1757	in_range = (int64_t)c2 == (int8_t)c2;
1758	}
1759	}
1760	if (in_range) {
1761	tgen_compare_imm_branch(s, opc, cc, r1, c2, l);
1762	return;
1763	}
1764	}
1765
1766	cc = tgen_cmp(s, type, c, r1, c2, c2const, false);
1767	tgen_branch(s, cc, l);
1768	}
1769
1770	static void tgen_brcondr(TCGContext *s, TCGType type, TCGCond c,
1771	TCGReg a0, TCGReg a1, TCGLabel *l)
1772	{
1773	tgen_brcond(s, type, c, a0, a1, false, l);
1774	}
1775
1776	static void tgen_brcondi(TCGContext *s, TCGType type, TCGCond c,
1777	TCGReg a0, tcg_target_long a1, TCGLabel *l)
1778	{
1779	tgen_brcond(s, type, c, a0, a1, true, l);
1780	}
1781
1782	static const TCGOutOpBrcond outop_brcond = {
1783	.base.static_constraint = C_O0_I2(r, rC),
1784	.out_rr = tgen_brcondr,
1785	.out_ri = tgen_brcondi,
1786	};
1787
1788	static void tcg_out_call_int(TCGContext s, const tcg_insn_unit dest)
1789	{
1790	ptrdiff_t off = tcg_pcrel_diff(s, dest) >> 1;
1791	if (off == (int32_t)off) {
1792	tcg_out_insn(s, RIL, BRASL, TCG_REG_R14, off);
1793	} else {
1794	tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, (uintptr_t)dest);
1795	tcg_out_insn(s, RR, BASR, TCG_REG_R14, TCG_TMP0);
1796	}
1797	}
1798
1799	static void tcg_out_call(TCGContext s, const tcg_insn_unit dest,
1800	const TCGHelperInfo *info)
1801	{
1802	tcg_out_call_int(s, dest);
1803	}
1804
1805	typedef struct {
1806	TCGReg base;
1807	TCGReg index;
1808	int disp;
1809	TCGAtomAlign aa;
1810	} HostAddress;
1811
1812	bool tcg_target_has_memory_bswap(MemOp memop)
1813	{
1814	TCGAtomAlign aa;
1815
1816	if ((memop & MO_SIZE) <= MO_64) {
1817	return true;
1818	}
1819
1820	/*
1821	* Reject 16-byte memop with 16-byte atomicity,
1822	* but do allow a pair of 64-bit operations.
1823	*/
1824	aa = atom_and_align_for_opc(tcg_ctx, memop, MO_ATOM_IFALIGN, true);
1825	return aa.atom <= MO_64;
1826	}
1827
1828	static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg data,
1829	HostAddress h)
1830	{
1831	switch (opc & (MO_SSIZE \| MO_BSWAP)) {
1832	case MO_UB:
1833	tcg_out_insn(s, RXY, LLGC, data, h.base, h.index, h.disp);
1834	break;
1835	case MO_SB:
1836	tcg_out_insn(s, RXY, LGB, data, h.base, h.index, h.disp);
1837	break;
1838
1839	case MO_UW \| MO_BSWAP:
1840	/* swapped unsigned halfword load with upper bits zeroed */
1841	tcg_out_insn(s, RXY, LRVH, data, h.base, h.index, h.disp);
1842	tcg_out_ext16u(s, data, data);
1843	break;
1844	case MO_UW:
1845	tcg_out_insn(s, RXY, LLGH, data, h.base, h.index, h.disp);
1846	break;
1847
1848	case MO_SW \| MO_BSWAP:
1849	/* swapped sign-extended halfword load */
1850	tcg_out_insn(s, RXY, LRVH, data, h.base, h.index, h.disp);
1851	tcg_out_ext16s(s, TCG_TYPE_REG, data, data);
1852	break;
1853	case MO_SW:
1854	tcg_out_insn(s, RXY, LGH, data, h.base, h.index, h.disp);
1855	break;
1856
1857	case MO_UL \| MO_BSWAP:
1858	/* swapped unsigned int load with upper bits zeroed */
1859	tcg_out_insn(s, RXY, LRV, data, h.base, h.index, h.disp);
1860	tcg_out_ext32u(s, data, data);
1861	break;
1862	case MO_UL:
1863	tcg_out_insn(s, RXY, LLGF, data, h.base, h.index, h.disp);
1864	break;
1865
1866	case MO_SL \| MO_BSWAP:
1867	/* swapped sign-extended int load */
1868	tcg_out_insn(s, RXY, LRV, data, h.base, h.index, h.disp);
1869	tcg_out_ext32s(s, data, data);
1870	break;
1871	case MO_SL:
1872	tcg_out_insn(s, RXY, LGF, data, h.base, h.index, h.disp);
1873	break;
1874
1875	case MO_UQ \| MO_BSWAP:
1876	tcg_out_insn(s, RXY, LRVG, data, h.base, h.index, h.disp);
1877	break;
1878	case MO_UQ:
1879	tcg_out_insn(s, RXY, LG, data, h.base, h.index, h.disp);
1880	break;
1881
1882	default:
1883	g_assert_not_reached();
1884	}
1885	}
1886
1887	static void tcg_out_qemu_st_direct(TCGContext *s, MemOp opc, TCGReg data,
1888	HostAddress h)
1889	{
1890	switch (opc & (MO_SIZE \| MO_BSWAP)) {
1891	case MO_UB:
1892	if (h.disp >= 0 && h.disp < 0x1000) {
1893	tcg_out_insn(s, RX, STC, data, h.base, h.index, h.disp);
1894	} else {
1895	tcg_out_insn(s, RXY, STCY, data, h.base, h.index, h.disp);
1896	}
1897	break;
1898
1899	case MO_UW \| MO_BSWAP:
1900	tcg_out_insn(s, RXY, STRVH, data, h.base, h.index, h.disp);
1901	break;
1902	case MO_UW:
1903	if (h.disp >= 0 && h.disp < 0x1000) {
1904	tcg_out_insn(s, RX, STH, data, h.base, h.index, h.disp);
1905	} else {
1906	tcg_out_insn(s, RXY, STHY, data, h.base, h.index, h.disp);
1907	}
1908	break;
1909
1910	case MO_UL \| MO_BSWAP:
1911	tcg_out_insn(s, RXY, STRV, data, h.base, h.index, h.disp);
1912	break;
1913	case MO_UL:
1914	if (h.disp >= 0 && h.disp < 0x1000) {
1915	tcg_out_insn(s, RX, ST, data, h.base, h.index, h.disp);
1916	} else {
1917	tcg_out_insn(s, RXY, STY, data, h.base, h.index, h.disp);
1918	}
1919	break;
1920
1921	case MO_UQ \| MO_BSWAP:
1922	tcg_out_insn(s, RXY, STRVG, data, h.base, h.index, h.disp);
1923	break;
1924	case MO_UQ:
1925	tcg_out_insn(s, RXY, STG, data, h.base, h.index, h.disp);
1926	break;
1927
1928	default:
1929	g_assert_not_reached();
1930	}
1931	}
1932
1933	static const TCGLdstHelperParam ldst_helper_param = {
1934	.ntmp = 1, .tmp = { TCG_TMP0 }
1935	};
1936
1937	static bool tcg_out_qemu_ld_slow_path(TCGContext s, TCGLabelQemuLdst lb)
1938	{
1939	MemOp opc = get_memop(lb->oi);
1940
1941	if (!patch_reloc(lb->label_ptr[0], R_390_PC16DBL,
1942	(intptr_t)tcg_splitwx_to_rx(s->code_ptr), 2)) {
1943	return false;
1944	}
1945
1946	tcg_out_ld_helper_args(s, lb, &ldst_helper_param);
1947	tcg_out_call_int(s, qemu_ld_helpers[opc & MO_SIZE]);
1948	tcg_out_ld_helper_ret(s, lb, false, &ldst_helper_param);
1949
1950	tgen_gotoi(s, S390_CC_ALWAYS, lb->raddr);
1951	return true;
1952	}
1953
1954	static bool tcg_out_qemu_st_slow_path(TCGContext s, TCGLabelQemuLdst lb)
1955	{
1956	MemOp opc = get_memop(lb->oi);
1957
1958	if (!patch_reloc(lb->label_ptr[0], R_390_PC16DBL,
1959	(intptr_t)tcg_splitwx_to_rx(s->code_ptr), 2)) {
1960	return false;
1961	}
1962
1963	tcg_out_st_helper_args(s, lb, &ldst_helper_param);
1964	tcg_out_call_int(s, qemu_st_helpers[opc & MO_SIZE]);
1965
1966	tgen_gotoi(s, S390_CC_ALWAYS, lb->raddr);
1967	return true;
1968	}
1969
1970	/* We're expecting to use a 20-bit negative offset on the tlb memory ops. */
1971	#define MIN_TLB_MASK_TABLE_OFS -(1 << 19)
1972
1973	/*
1974	* For system-mode, perform the TLB load and compare.
1975	* For user-mode, perform any required alignment tests.
1976	* In both cases, return a TCGLabelQemuLdst structure if the slow path
1977	* is required and fill in @h with the host address for the fast path.
1978	*/
1979	static TCGLabelQemuLdst prepare_host_addr(TCGContext s, HostAddress *h,
1980	TCGReg addr_reg, MemOpIdx oi,
1981	bool is_ld)
1982	{
1983	TCGType addr_type = s->addr_type;
1984	TCGLabelQemuLdst *ldst = NULL;
1985	MemOp opc = get_memop(oi);
1986	MemOp s_bits = opc & MO_SIZE;
1987	unsigned a_mask;
1988
1989	h->aa = atom_and_align_for_opc(s, opc, MO_ATOM_IFALIGN, s_bits == MO_128);
1990	a_mask = (1 << h->aa.align) - 1;
1991
1992	if (tcg_use_softmmu) {
1993	unsigned s_mask = (1 << s_bits) - 1;
1994	int mem_index = get_mmuidx(oi);
1995	int fast_off = tlb_mask_table_ofs(s, mem_index);
1996	int mask_off = fast_off + offsetof(CPUTLBDescFast, mask);
1997	int table_off = fast_off + offsetof(CPUTLBDescFast, table);
1998	int ofs, a_off;
1999	uint64_t tlb_mask;
2000
2001	ldst = new_ldst_label(s);
2002	ldst->is_ld = is_ld;
2003	ldst->oi = oi;
2004	ldst->addr_reg = addr_reg;
2005
2006	tcg_out_sh64(s, RSY_SRLG, TCG_TMP0, addr_reg, TCG_REG_NONE,
2007	TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
2008
2009	tcg_out_insn(s, RXY, NG, TCG_TMP0, TCG_AREG0, TCG_REG_NONE, mask_off);
2010	tcg_out_insn(s, RXY, AG, TCG_TMP0, TCG_AREG0, TCG_REG_NONE, table_off);
2011
2012	/*
2013	* For aligned accesses, we check the first byte and include the
2014	* alignment bits within the address. For unaligned access, we
2015	* check that we don't cross pages using the address of the last
2016	* byte of the access.
2017	*/
2018	a_off = (a_mask >= s_mask ? 0 : s_mask - a_mask);
2019	tlb_mask = (uint64_t)TARGET_PAGE_MASK \| a_mask;
2020	if (a_off == 0) {
2021	tgen_andi_risbg(s, TCG_REG_R0, addr_reg, tlb_mask);
2022	} else {
2023	tcg_out_insn(s, RX, LA, TCG_REG_R0, addr_reg, TCG_REG_NONE, a_off);
2024	tgen_andi(s, addr_type, TCG_REG_R0, tlb_mask);
2025	}
2026
2027	if (is_ld) {
2028	ofs = offsetof(CPUTLBEntry, addr_read);
2029	} else {
2030	ofs = offsetof(CPUTLBEntry, addr_write);
2031	}
2032	if (addr_type == TCG_TYPE_I32) {
2033	ofs += HOST_BIG_ENDIAN * 4;
2034	tcg_out_insn(s, RX, C, TCG_REG_R0, TCG_TMP0, TCG_REG_NONE, ofs);
2035	} else {
2036	tcg_out_insn(s, RXY, CG, TCG_REG_R0, TCG_TMP0, TCG_REG_NONE, ofs);
2037	}
2038
2039	tcg_out16(s, RI_BRC \| (S390_CC_NE << 4));
2040	ldst->label_ptr[0] = s->code_ptr++;
2041
2042	h->index = TCG_TMP0;
2043	tcg_out_insn(s, RXY, LG, h->index, TCG_TMP0, TCG_REG_NONE,
2044	offsetof(CPUTLBEntry, addend));
2045
2046	if (addr_type == TCG_TYPE_I32) {
2047	tcg_out_insn(s, RRE, ALGFR, h->index, addr_reg);
2048	h->base = TCG_REG_NONE;
2049	} else {
2050	h->base = addr_reg;
2051	}
2052	h->disp = 0;
2053	} else {
2054	if (a_mask) {
2055	ldst = new_ldst_label(s);
2056	ldst->is_ld = is_ld;
2057	ldst->oi = oi;
2058	ldst->addr_reg = addr_reg;
2059
2060	tcg_debug_assert(a_mask <= 0xffff);
2061	tcg_out_insn(s, RI, TMLL, addr_reg, a_mask);
2062
2063	tcg_out16(s, RI_BRC \| (S390_TM_NE << 4));
2064	ldst->label_ptr[0] = s->code_ptr++;
2065	}
2066
2067	h->base = addr_reg;
2068	if (addr_type == TCG_TYPE_I32) {
2069	tcg_out_ext32u(s, TCG_TMP0, addr_reg);
2070	h->base = TCG_TMP0;
2071	}
2072	if (guest_base < 0x80000) {
2073	h->index = TCG_REG_NONE;
2074	h->disp = guest_base;
2075	} else {
2076	h->index = TCG_GUEST_BASE_REG;
2077	h->disp = 0;
2078	}
2079	}
2080
2081	return ldst;
2082	}
2083
2084	static void tgen_qemu_ld(TCGContext *s, TCGType type, TCGReg data_reg,
2085	TCGReg addr_reg, MemOpIdx oi)
2086	{
2087	TCGLabelQemuLdst *ldst;
2088	HostAddress h;
2089
2090	ldst = prepare_host_addr(s, &h, addr_reg, oi, true);
2091	tcg_out_qemu_ld_direct(s, get_memop(oi), data_reg, h);
2092
2093	if (ldst) {
2094	ldst->type = type;
2095	ldst->datalo_reg = data_reg;
2096	ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
2097	}
2098	}
2099
2100	static const TCGOutOpQemuLdSt outop_qemu_ld = {
2101	.base.static_constraint = C_O1_I1(r, r),
2102	.out = tgen_qemu_ld,
2103	};
2104
2105	static void tgen_qemu_st(TCGContext *s, TCGType type, TCGReg data_reg,
2106	TCGReg addr_reg, MemOpIdx oi)
2107	{
2108	TCGLabelQemuLdst *ldst;
2109	HostAddress h;
2110
2111	ldst = prepare_host_addr(s, &h, addr_reg, oi, false);
2112	tcg_out_qemu_st_direct(s, get_memop(oi), data_reg, h);
2113
2114	if (ldst) {
2115	ldst->type = type;
2116	ldst->datalo_reg = data_reg;
2117	ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
2118	}
2119	}
2120
2121	static const TCGOutOpQemuLdSt outop_qemu_st = {
2122	.base.static_constraint = C_O0_I2(r, r),
2123	.out = tgen_qemu_st,
2124	};
2125
2126	static void tcg_out_qemu_ldst_i128(TCGContext *s, TCGReg datalo, TCGReg datahi,
2127	TCGReg addr_reg, MemOpIdx oi, bool is_ld)
2128	{
2129	TCGLabel l1 = NULL, l2 = NULL;
2130	TCGLabelQemuLdst *ldst;
2131	HostAddress h;
2132	bool need_bswap;
2133	bool use_pair;
2134	S390Opcode insn;
2135
2136	ldst = prepare_host_addr(s, &h, addr_reg, oi, is_ld);
2137
2138	use_pair = h.aa.atom < MO_128;
2139	need_bswap = get_memop(oi) & MO_BSWAP;
2140
2141	if (!use_pair) {
2142	/*
2143	* Atomicity requires we use LPQ. If we've already checked for
2144	* 16-byte alignment, that's all we need. If we arrive with
2145	* lesser alignment, we have determined that less than 16-byte
2146	* alignment can be satisfied with two 8-byte loads.
2147	*/
2148	if (h.aa.align < MO_128) {
2149	use_pair = true;
2150	l1 = gen_new_label();
2151	l2 = gen_new_label();
2152
2153	tcg_out_insn(s, RI, TMLL, addr_reg, 15);
2154	tgen_branch(s, S390_TM_NE, l1);
2155	}
2156
2157	tcg_debug_assert(!need_bswap);
2158	tcg_debug_assert(datalo & 1);
2159	tcg_debug_assert(datahi == datalo - 1);
2160	insn = is_ld ? RXY_LPQ : RXY_STPQ;
2161	tcg_out_insn_RXY(s, insn, datahi, h.base, h.index, h.disp);
2162
2163	if (use_pair) {
2164	tgen_branch(s, S390_CC_ALWAYS, l2);
2165	tcg_out_label(s, l1);
2166	}
2167	}
2168	if (use_pair) {
2169	TCGReg d1, d2;
2170
2171	if (need_bswap) {
2172	d1 = datalo, d2 = datahi;
2173	insn = is_ld ? RXY_LRVG : RXY_STRVG;
2174	} else {
2175	d1 = datahi, d2 = datalo;
2176	insn = is_ld ? RXY_LG : RXY_STG;
2177	}
2178
2179	if (h.base == d1 \|\| h.index == d1) {
2180	tcg_out_insn(s, RXY, LAY, TCG_TMP0, h.base, h.index, h.disp);
2181	h.base = TCG_TMP0;
2182	h.index = TCG_REG_NONE;
2183	h.disp = 0;
2184	}
2185	tcg_out_insn_RXY(s, insn, d1, h.base, h.index, h.disp);
2186	tcg_out_insn_RXY(s, insn, d2, h.base, h.index, h.disp + 8);
2187	}
2188	if (l2) {
2189	tcg_out_label(s, l2);
2190	}
2191
2192	if (ldst) {
2193	ldst->type = TCG_TYPE_I128;
2194	ldst->datalo_reg = datalo;
2195	ldst->datahi_reg = datahi;
2196	ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
2197	}
2198	}
2199
2200	static void tgen_qemu_ld2(TCGContext *s, TCGType type, TCGReg datalo,
2201	TCGReg datahi, TCGReg addr_reg, MemOpIdx oi)
2202	{
2203	tcg_out_qemu_ldst_i128(s, datalo, datahi, addr_reg, oi, true);
2204	}
2205
2206	static const TCGOutOpQemuLdSt2 outop_qemu_ld2 = {
2207	.base.static_constraint = C_O2_I1(o, m, r),
2208	.out = tgen_qemu_ld2,
2209	};
2210
2211	static void tgen_qemu_st2(TCGContext *s, TCGType type, TCGReg datalo,
2212	TCGReg datahi, TCGReg addr_reg, MemOpIdx oi)
2213	{
2214	tcg_out_qemu_ldst_i128(s, datalo, datahi, addr_reg, oi, false);
2215	}
2216
2217	static const TCGOutOpQemuLdSt2 outop_qemu_st2 = {
2218	.base.static_constraint = C_O0_I3(o, m, r),
2219	.out = tgen_qemu_st2,
2220	};
2221
2222	static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
2223	{
2224	/* Reuse the zeroing that exists for goto_ptr. */
2225	if (a0 == 0) {
2226	tgen_gotoi(s, S390_CC_ALWAYS, tcg_code_gen_epilogue);
2227	} else {
2228	tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, a0);
2229	tgen_gotoi(s, S390_CC_ALWAYS, tb_ret_addr);
2230	}
2231	}
2232
2233	static void tcg_out_goto_tb(TCGContext *s, int which)
2234	{
2235	/*
2236	* Branch displacement must be aligned for atomic patching;
2237	* see if we need to add extra nop before branch
2238	*/
2239	if (!QEMU_PTR_IS_ALIGNED(s->code_ptr + 1, 4)) {
2240	tcg_out16(s, NOP);
2241	}
2242	tcg_out16(s, RIL_BRCL \| (S390_CC_ALWAYS << 4));
2243	set_jmp_insn_offset(s, which);
2244	s->code_ptr += 2;
2245	set_jmp_reset_offset(s, which);
2246	}
2247
2248	static void tcg_out_goto_ptr(TCGContext *s, TCGReg a0)
2249	{
2250	tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, a0);
2251	}
2252
2253	void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
2254	uintptr_t jmp_rx, uintptr_t jmp_rw)
2255	{
2256	if (!HAVE_FACILITY(GEN_INST_EXT)) {
2257	return;
2258	}
2259	/* patch the branch destination */
2260	uintptr_t addr = tb->jmp_target_addr[n];
2261	intptr_t disp = addr - (jmp_rx - 2);
2262	qatomic_set((int32_t *)jmp_rw, disp / 2);
2263	/* no need to flush icache explicitly */
2264	}
2265
2266
2267	static void tgen_add(TCGContext *s, TCGType type,
2268	TCGReg a0, TCGReg a1, TCGReg a2)
2269	{
2270	if (a0 != a1) {
2271	tcg_out_insn(s, RX, LA, a0, a1, a2, 0);
2272	} else if (type == TCG_TYPE_I32) {
2273	tcg_out_insn(s, RR, AR, a0, a2);
2274	} else {
2275	tcg_out_insn(s, RRE, AGR, a0, a2);
2276	}
2277	}
2278
2279	static void tgen_addi(TCGContext *s, TCGType type,
2280	TCGReg a0, TCGReg a1, tcg_target_long a2)
2281	{
2282	if (a0 == a1) {
2283	if (type == TCG_TYPE_I32) {
2284	if (a2 == (int16_t)a2) {
2285	tcg_out_insn(s, RI, AHI, a0, a2);
2286	} else {
2287	tcg_out_insn(s, RIL, AFI, a0, a2);
2288	}
2289	return;
2290	}
2291	if (a2 == (int16_t)a2) {
2292	tcg_out_insn(s, RI, AGHI, a0, a2);
2293	return;
2294	}
2295	if (a2 == (int32_t)a2) {
2296	tcg_out_insn(s, RIL, AGFI, a0, a2);
2297	return;
2298	}
2299	if (a2 == (uint32_t)a2) {
2300	tcg_out_insn(s, RIL, ALGFI, a0, a2);
2301	return;
2302	}
2303	if (-a2 == (uint32_t)-a2) {
2304	tcg_out_insn(s, RIL, SLGFI, a0, -a2);
2305	return;
2306	}
2307	}
2308	tcg_out_mem(s, RX_LA, RXY_LAY, a0, a1, TCG_REG_NONE, a2);
2309	}
2310
2311	static const TCGOutOpBinary outop_add = {
2312	.base.static_constraint = C_O1_I2(r, r, ri),
2313	.out_rrr = tgen_add,
2314	.out_rri = tgen_addi,
2315	};
2316
2317	static void tgen_addco_rrr(TCGContext *s, TCGType type,
2318	TCGReg a0, TCGReg a1, TCGReg a2)
2319	{
2320	if (type != TCG_TYPE_I32) {
2321	tcg_out_insn(s, RRFa, ALGRK, a0, a1, a2);
2322	} else if (a0 == a1) {
2323	tcg_out_insn(s, RR, ALR, a0, a2);
2324	} else {
2325	tcg_out_insn(s, RRFa, ALRK, a0, a1, a2);
2326	}
2327	}
2328
2329	static void tgen_addco_rri(TCGContext *s, TCGType type,
2330	TCGReg a0, TCGReg a1, tcg_target_long a2)
2331	{
2332	if (a2 == (int16_t)a2) {
2333	if (type == TCG_TYPE_I32) {
2334	tcg_out_insn(s, RIEd, ALHSIK, a0, a1, a2);
2335	} else {
2336	tcg_out_insn(s, RIEd, ALGHSIK, a0, a1, a2);
2337	}
2338	return;
2339	}
2340
2341	tcg_out_mov(s, type, a0, a1);
2342	if (type == TCG_TYPE_I32) {
2343	tcg_out_insn(s, RIL, ALFI, a0, a2);
2344	} else if (a2 >= 0) {
2345	tcg_out_insn(s, RIL, ALGFI, a0, a2);
2346	} else {
2347	tcg_out_insn(s, RIL, SLGFI, a0, -a2);
2348	}
2349	}
2350
2351	static const TCGOutOpBinary outop_addco = {
2352	.base.static_constraint = C_O1_I2(r, r, rUV),
2353	.out_rrr = tgen_addco_rrr,
2354	.out_rri = tgen_addco_rri,
2355	};
2356
2357	static void tgen_addcio(TCGContext *s, TCGType type,
2358	TCGReg a0, TCGReg a1, TCGReg a2)
2359	{
2360	if (type == TCG_TYPE_I32) {
2361	tcg_out_insn(s, RRE, ALCR, a0, a2);
2362	} else {
2363	tcg_out_insn(s, RRE, ALCGR, a0, a2);
2364	}
2365	}
2366
2367	static const TCGOutOpBinary outop_addcio = {
2368	.base.static_constraint = C_O1_I2(r, 0, r),
2369	.out_rrr = tgen_addcio,
2370	};
2371
2372	static const TCGOutOpAddSubCarry outop_addci = {
2373	.base.static_constraint = C_O1_I2(r, 0, r),
2374	.out_rrr = tgen_addcio,
2375	};
2376
2377	static void tcg_out_set_carry(TCGContext *s)
2378	{
2379	tcg_out_insn(s, RR, SLR, TCG_REG_R0, TCG_REG_R0); /* cc = 2 */
2380	}
2381
2382	static void tgen_and(TCGContext *s, TCGType type,
2383	TCGReg a0, TCGReg a1, TCGReg a2)
2384	{
2385	if (type != TCG_TYPE_I32) {
2386	tcg_out_insn(s, RRFa, NGRK, a0, a1, a2);
2387	} else if (a0 == a1) {
2388	tcg_out_insn(s, RR, NR, a0, a2);
2389	} else {
2390	tcg_out_insn(s, RRFa, NRK, a0, a1, a2);
2391	}
2392	}
2393
2394	static void tgen_andi_3(TCGContext *s, TCGType type,
2395	TCGReg a0, TCGReg a1, tcg_target_long a2)
2396	{
2397	tcg_out_mov(s, type, a0, a1);
2398	tgen_andi(s, type, a0, a2);
2399	}
2400
2401	static const TCGOutOpBinary outop_and = {
2402	.base.static_constraint = C_O1_I2(r, r, rNKR),
2403	.out_rrr = tgen_and,
2404	.out_rri = tgen_andi_3,
2405	};
2406
2407	static void tgen_andc(TCGContext *s, TCGType type,
2408	TCGReg a0, TCGReg a1, TCGReg a2)
2409	{
2410	if (type == TCG_TYPE_I32) {
2411	tcg_out_insn(s, RRFa, NCRK, a0, a1, a2);
2412	} else {
2413	tcg_out_insn(s, RRFa, NCGRK, a0, a1, a2);
2414	}
2415	}
2416
2417	static TCGConstraintSetIndex cset_misc3_rrr(TCGType type, unsigned flags)
2418	{
2419	return HAVE_FACILITY(MISC_INSN_EXT3) ? C_O1_I2(r, r, r) : C_NotImplemented;
2420	}
2421
2422	static const TCGOutOpBinary outop_andc = {
2423	.base.static_constraint = C_Dynamic,
2424	.base.dynamic_constraint = cset_misc3_rrr,
2425	.out_rrr = tgen_andc,
2426	};
2427
2428	static void tgen_clz_int(TCGContext *s, TCGReg dest, TCGReg a1,
2429	TCGArg a2, int a2const)
2430	{
2431	/*
2432	* Since this sets both R and R+1, we have no choice but to store the
2433	* result into R0, allowing R1 == TCG_TMP0 to be clobbered as well.
2434	*/
2435	QEMU_BUILD_BUG_ON(TCG_TMP0 != TCG_REG_R1);
2436	tcg_out_insn(s, RRE, FLOGR, TCG_REG_R0, a1);
2437
2438	if (a2const && a2 == 64) {
2439	tcg_out_mov(s, TCG_TYPE_I64, dest, TCG_REG_R0);
2440	return;
2441	}
2442
2443	/*
2444	* Conditions from FLOGR are:
2445	* 2 -> one bit found
2446	* 8 -> no one bit found
2447	*/
2448	tgen_movcond_int(s, TCG_TYPE_I64, dest, a2, a2const, TCG_REG_R0, 8, 2);
2449	}
2450
2451	static void tgen_clz(TCGContext *s, TCGType type,
2452	TCGReg a0, TCGReg a1, TCGReg a2)
2453	{
2454	tgen_clz_int(s, a0, a1, a2, false);
2455	}
2456
2457	static void tgen_clzi(TCGContext *s, TCGType type,
2458	TCGReg a0, TCGReg a1, tcg_target_long a2)
2459	{
2460	tgen_clz_int(s, a0, a1, a2, true);
2461	}
2462
2463	static TCGConstraintSetIndex cset_clz(TCGType type, unsigned flags)
2464	{
2465	return type == TCG_TYPE_I64 ? C_O1_I2(r, r, rI) : C_NotImplemented;
2466	}
2467
2468	static const TCGOutOpBinary outop_clz = {
2469	.base.static_constraint = C_Dynamic,
2470	.base.dynamic_constraint = cset_clz,
2471	.out_rrr = tgen_clz,
2472	.out_rri = tgen_clzi,
2473	};
2474
2475	static void tgen_ctpop(TCGContext *s, TCGType type, TCGReg dest, TCGReg src)
2476	{
2477	/* With MIE3, and bit 0 of m4 set, we get the complete result. */
2478	if (HAVE_FACILITY(MISC_INSN_EXT3)) {
2479	if (type == TCG_TYPE_I32) {
2480	tcg_out_ext32u(s, dest, src);
2481	src = dest;
2482	}
2483	tcg_out_insn(s, RRFc, POPCNT, dest, src, 8);
2484	return;
2485	}
2486
2487	/* Without MIE3, each byte gets the count of bits for the byte. */
2488	tcg_out_insn(s, RRFc, POPCNT, dest, src, 0);
2489
2490	/* Multiply to sum each byte at the top of the word. */
2491	if (type == TCG_TYPE_I32) {
2492	tcg_out_insn(s, RIL, MSFI, dest, 0x01010101);
2493	tcg_out_sh32(s, RS_SRL, dest, TCG_REG_NONE, 24);
2494	} else {
2495	tcg_out_movi(s, TCG_TYPE_I64, TCG_TMP0, 0x0101010101010101ull);
2496	tcg_out_insn(s, RRE, MSGR, dest, TCG_TMP0);
2497	tcg_out_sh64(s, RSY_SRLG, dest, dest, TCG_REG_NONE, 56);
2498	}
2499	}
2500
2501	static const TCGOutOpUnary outop_ctpop = {
2502	.base.static_constraint = C_O1_I1(r, r),
2503	.out_rr = tgen_ctpop,
2504	};
2505
2506	static const TCGOutOpBinary outop_ctz = {
2507	.base.static_constraint = C_NotImplemented,
2508	};
2509
2510	static const TCGOutOpBinary outop_divs = {
2511	.base.static_constraint = C_NotImplemented,
2512	};
2513
2514	static void tgen_divs2(TCGContext *s, TCGType type,
2515	TCGReg a0, TCGReg a1, TCGReg a4)
2516	{
2517	tcg_debug_assert((a1 & 1) == 0);
2518	tcg_debug_assert(a0 == a1 + 1);
2519	if (type == TCG_TYPE_I32) {
2520	tcg_out_insn(s, RR, DR, a1, a4);
2521	} else {
2522	/*
2523	* TODO: Move the sign-extend of the numerator from a2 into a3
2524	* into the tcg backend, instead of in early expansion. It is
2525	* required for 32-bit DR, but not 64-bit DSGR.
2526	*/
2527	tcg_out_insn(s, RRE, DSGR, a1, a4);
2528	}
2529	}
2530
2531	static const TCGOutOpDivRem outop_divs2 = {
2532	.base.static_constraint = C_O2_I3(o, m, 0, 1, r),
2533	.out_rr01r = tgen_divs2,
2534	};
2535
2536	static const TCGOutOpBinary outop_divu = {
2537	.base.static_constraint = C_NotImplemented,
2538	};
2539
2540	static void tgen_divu2(TCGContext *s, TCGType type,
2541	TCGReg a0, TCGReg a1, TCGReg a4)
2542	{
2543	tcg_debug_assert((a1 & 1) == 0);
2544	tcg_debug_assert(a0 == a1 + 1);
2545	if (type == TCG_TYPE_I32) {
2546	tcg_out_insn(s, RRE, DLR, a1, a4);
2547	} else {
2548	tcg_out_insn(s, RRE, DLGR, a1, a4);
2549	}
2550	}
2551
2552	static const TCGOutOpDivRem outop_divu2 = {
2553	.base.static_constraint = C_O2_I3(o, m, 0, 1, r),
2554	.out_rr01r = tgen_divu2,
2555	};
2556
2557	static void tgen_eqv(TCGContext *s, TCGType type,
2558	TCGReg a0, TCGReg a1, TCGReg a2)
2559	{
2560	if (type == TCG_TYPE_I32) {
2561	tcg_out_insn(s, RRFa, NXRK, a0, a1, a2);
2562	} else {
2563	tcg_out_insn(s, RRFa, NXGRK, a0, a1, a2);
2564	}
2565	}
2566
2567	static const TCGOutOpBinary outop_eqv = {
2568	.base.static_constraint = C_Dynamic,
2569	.base.dynamic_constraint = cset_misc3_rrr,
2570	.out_rrr = tgen_eqv,
2571	};
2572
2573	static void tgen_extrh_i64_i32(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1)
2574	{
2575	tcg_out_sh64(s, RSY_SRLG, a0, a1, TCG_REG_NONE, 32);
2576	}
2577
2578	static const TCGOutOpUnary outop_extrh_i64_i32 = {
2579	.base.static_constraint = C_O1_I1(r, r),
2580	.out_rr = tgen_extrh_i64_i32,
2581	};
2582
2583	static void tgen_mul(TCGContext *s, TCGType type,
2584	TCGReg a0, TCGReg a1, TCGReg a2)
2585	{
2586	if (type == TCG_TYPE_I32) {
2587	if (a0 == a1) {
2588	tcg_out_insn(s, RRE, MSR, a0, a2);
2589	} else {
2590	tcg_out_insn(s, RRFa, MSRKC, a0, a1, a2);
2591	}
2592	} else {
2593	if (a0 == a1) {
2594	tcg_out_insn(s, RRE, MSGR, a0, a2);
2595	} else {
2596	tcg_out_insn(s, RRFa, MSGRKC, a0, a1, a2);
2597	}
2598	}
2599	}
2600
2601	static void tgen_muli(TCGContext *s, TCGType type,
2602	TCGReg a0, TCGReg a1, tcg_target_long a2)
2603	{
2604	tcg_out_mov(s, type, a0, a1);
2605	if (type == TCG_TYPE_I32) {
2606	if (a2 == (int16_t)a2) {
2607	tcg_out_insn(s, RI, MHI, a0, a2);
2608	} else {
2609	tcg_out_insn(s, RIL, MSFI, a0, a2);
2610	}
2611	} else {
2612	if (a2 == (int16_t)a2) {
2613	tcg_out_insn(s, RI, MGHI, a0, a2);
2614	} else {
2615	tcg_out_insn(s, RIL, MSGFI, a0, a2);
2616	}
2617	}
2618	}
2619
2620	static TCGConstraintSetIndex cset_mul(TCGType type, unsigned flags)
2621	{
2622	return (HAVE_FACILITY(MISC_INSN_EXT2)
2623	? C_O1_I2(r, r, rJ)
2624	: C_O1_I2(r, 0, rJ));
2625	}
2626
2627	static const TCGOutOpBinary outop_mul = {
2628	.base.static_constraint = C_Dynamic,
2629	.base.dynamic_constraint = cset_mul,
2630	.out_rrr = tgen_mul,
2631	.out_rri = tgen_muli,
2632	};
2633
2634	static void tgen_muls2(TCGContext *s, TCGType type,
2635	TCGReg a0, TCGReg a1, TCGReg a2, TCGReg a3)
2636	{
2637	tcg_debug_assert((a1 & 1) == 0);
2638	tcg_debug_assert(a0 == a1 + 1);
2639	tcg_out_insn(s, RRFa, MGRK, a1, a2, a3);
2640	}
2641
2642	static TCGConstraintSetIndex cset_muls2(TCGType type, unsigned flags)
2643	{
2644	return (type == TCG_TYPE_I64 && HAVE_FACILITY(MISC_INSN_EXT2)
2645	? C_O2_I2(o, m, r, r) : C_NotImplemented);
2646	}
2647
2648	static const TCGOutOpMul2 outop_muls2 = {
2649	.base.static_constraint = C_Dynamic,
2650	.base.dynamic_constraint = cset_muls2,
2651	.out_rrrr = tgen_muls2,
2652	};
2653
2654	static const TCGOutOpBinary outop_mulsh = {
2655	.base.static_constraint = C_NotImplemented,
2656	};
2657
2658	static void tgen_mulu2(TCGContext *s, TCGType type,
2659	TCGReg a0, TCGReg a1, TCGReg a2, TCGReg a3)
2660	{
2661	tcg_debug_assert(a0 == a2);
2662	tcg_debug_assert((a1 & 1) == 0);
2663	tcg_debug_assert(a0 == a1 + 1);
2664	tcg_out_insn(s, RRE, MLGR, a1, a3);
2665	}
2666
2667	static TCGConstraintSetIndex cset_mulu2(TCGType type, unsigned flags)
2668	{
2669	return (type == TCG_TYPE_I64 && HAVE_FACILITY(MISC_INSN_EXT2)
2670	? C_O2_I2(o, m, 0, r) : C_NotImplemented);
2671	}
2672
2673	static const TCGOutOpMul2 outop_mulu2 = {
2674	.base.static_constraint = C_Dynamic,
2675	.base.dynamic_constraint = cset_mulu2,
2676	.out_rrrr = tgen_mulu2,
2677	};
2678
2679	static const TCGOutOpBinary outop_muluh = {
2680	.base.static_constraint = C_NotImplemented,
2681	};
2682
2683	static void tgen_nand(TCGContext *s, TCGType type,
2684	TCGReg a0, TCGReg a1, TCGReg a2)
2685	{
2686	if (type == TCG_TYPE_I32) {
2687	tcg_out_insn(s, RRFa, NNRK, a0, a1, a2);
2688	} else {
2689	tcg_out_insn(s, RRFa, NNGRK, a0, a1, a2);
2690	}
2691	}
2692
2693	static const TCGOutOpBinary outop_nand = {
2694	.base.static_constraint = C_Dynamic,
2695	.base.dynamic_constraint = cset_misc3_rrr,
2696	.out_rrr = tgen_nand,
2697	};
2698
2699	static void tgen_nor(TCGContext *s, TCGType type,
2700	TCGReg a0, TCGReg a1, TCGReg a2)
2701	{
2702	if (type == TCG_TYPE_I32) {
2703	tcg_out_insn(s, RRFa, NORK, a0, a1, a2);
2704	} else {
2705	tcg_out_insn(s, RRFa, NOGRK, a0, a1, a2);
2706	}
2707	}
2708
2709	static const TCGOutOpBinary outop_nor = {
2710	.base.static_constraint = C_Dynamic,
2711	.base.dynamic_constraint = cset_misc3_rrr,
2712	.out_rrr = tgen_nor,
2713	};
2714
2715	static void tgen_or(TCGContext *s, TCGType type,
2716	TCGReg a0, TCGReg a1, TCGReg a2)
2717	{
2718	if (type != TCG_TYPE_I32) {
2719	tcg_out_insn(s, RRFa, OGRK, a0, a1, a2);
2720	} else if (a0 == a1) {
2721	tcg_out_insn(s, RR, OR, a0, a2);
2722	} else {
2723	tcg_out_insn(s, RRFa, ORK, a0, a1, a2);
2724	}
2725	}
2726
2727	static void tgen_ori_3(TCGContext *s, TCGType type,
2728	TCGReg a0, TCGReg a1, tcg_target_long a2)
2729	{
2730	tcg_out_mov(s, type, a0, a1);
2731	tgen_ori(s, a0, type == TCG_TYPE_I32 ? (uint32_t)a2 : a2);
2732	}
2733
2734	static const TCGOutOpBinary outop_or = {
2735	.base.static_constraint = C_O1_I2(r, r, rK),
2736	.out_rrr = tgen_or,
2737	.out_rri = tgen_ori_3,
2738	};
2739
2740	static void tgen_orc(TCGContext *s, TCGType type,
2741	TCGReg a0, TCGReg a1, TCGReg a2)
2742	{
2743	if (type == TCG_TYPE_I32) {
2744	tcg_out_insn(s, RRFa, OCRK, a0, a1, a2);
2745	} else {
2746	tcg_out_insn(s, RRFa, OCGRK, a0, a1, a2);
2747	}
2748	}
2749
2750	static const TCGOutOpBinary outop_orc = {
2751	.base.static_constraint = C_Dynamic,
2752	.base.dynamic_constraint = cset_misc3_rrr,
2753	.out_rrr = tgen_orc,
2754	};
2755
2756	static const TCGOutOpBinary outop_rems = {
2757	.base.static_constraint = C_NotImplemented,
2758	};
2759
2760	static const TCGOutOpBinary outop_remu = {
2761	.base.static_constraint = C_NotImplemented,
2762	};
2763
2764	static void tgen_rotl_int(TCGContext *s, TCGType type, TCGReg dst,
2765	TCGReg src, TCGReg v, tcg_target_long i)
2766	{
2767	S390Opcode insn = type == TCG_TYPE_I32 ? RSY_RLL : RSY_RLLG;
2768	tcg_out_sh64(s, insn, dst, src, v, i);
2769	}
2770
2771	static void tgen_rotl(TCGContext *s, TCGType type,
2772	TCGReg a0, TCGReg a1, TCGReg a2)
2773	{
2774	tgen_rotl_int(s, type, a0, a1, a2, 0);
2775	}
2776
2777	static void tgen_rotli(TCGContext *s, TCGType type,
2778	TCGReg a0, TCGReg a1, tcg_target_long a2)
2779	{
2780	tgen_rotl_int(s, type, a0, a1, TCG_REG_NONE, a2);
2781	}
2782
2783	static const TCGOutOpBinary outop_rotl = {
2784	.base.static_constraint = C_O1_I2(r, r, ri),
2785	.out_rrr = tgen_rotl,
2786	.out_rri = tgen_rotli,
2787	};
2788
2789	static const TCGOutOpBinary outop_rotr = {
2790	.base.static_constraint = C_NotImplemented,
2791	};
2792
2793	static void tgen_sar_int(TCGContext *s, TCGType type, TCGReg dst,
2794	TCGReg src, TCGReg v, tcg_target_long i)
2795	{
2796	if (type != TCG_TYPE_I32) {
2797	tcg_out_sh64(s, RSY_SRAG, dst, src, v, i);
2798	} else if (dst == src) {
2799	tcg_out_sh32(s, RS_SRA, dst, v, i);
2800	} else {
2801	tcg_out_sh64(s, RSY_SRAK, dst, src, v, i);
2802	}
2803	}
2804
2805	static void tgen_sar(TCGContext *s, TCGType type,
2806	TCGReg a0, TCGReg a1, TCGReg a2)
2807	{
2808	tgen_sar_int(s, type, a0, a1, a2, 0);
2809	}
2810
2811	static void tgen_sari(TCGContext *s, TCGType type,
2812	TCGReg a0, TCGReg a1, tcg_target_long a2)
2813	{
2814	tgen_sar_int(s, type, a0, a1, TCG_REG_NONE, a2);
2815	}
2816
2817	static const TCGOutOpBinary outop_sar = {
2818	.base.static_constraint = C_O1_I2(r, r, ri),
2819	.out_rrr = tgen_sar,
2820	.out_rri = tgen_sari,
2821	};
2822
2823	static void tgen_shl_int(TCGContext *s, TCGType type, TCGReg dst,
2824	TCGReg src, TCGReg v, tcg_target_long i)
2825	{
2826	if (type != TCG_TYPE_I32) {
2827	tcg_out_sh64(s, RSY_SLLG, dst, src, v, i);
2828	} else if (dst == src) {
2829	tcg_out_sh32(s, RS_SLL, dst, v, i);
2830	} else {
2831	tcg_out_sh64(s, RSY_SLLK, dst, src, v, i);
2832	}
2833	}
2834
2835	static void tgen_shl(TCGContext *s, TCGType type,
2836	TCGReg a0, TCGReg a1, TCGReg a2)
2837	{
2838	tgen_shl_int(s, type, a0, a1, a2, 0);
2839	}
2840
2841	static void tgen_shli(TCGContext *s, TCGType type,
2842	TCGReg a0, TCGReg a1, tcg_target_long a2)
2843	{
2844	tgen_shl_int(s, type, a0, a1, TCG_REG_NONE, a2);
2845	}
2846
2847	static const TCGOutOpBinary outop_shl = {
2848	.base.static_constraint = C_O1_I2(r, r, ri),
2849	.out_rrr = tgen_shl,
2850	.out_rri = tgen_shli,
2851	};
2852
2853	static void tgen_shr_int(TCGContext *s, TCGType type, TCGReg dst,
2854	TCGReg src, TCGReg v, tcg_target_long i)
2855	{
2856	if (type != TCG_TYPE_I32) {
2857	tcg_out_sh64(s, RSY_SRLG, dst, src, v, i);
2858	} else if (dst == src) {
2859	tcg_out_sh32(s, RS_SRL, dst, v, i);
2860	} else {
2861	tcg_out_sh64(s, RSY_SRLK, dst, src, v, i);
2862	}
2863	}
2864
2865	static void tgen_shr(TCGContext *s, TCGType type,
2866	TCGReg a0, TCGReg a1, TCGReg a2)
2867	{
2868	tgen_shr_int(s, type, a0, a1, a2, 0);
2869	}
2870
2871	static void tgen_shri(TCGContext *s, TCGType type,
2872	TCGReg a0, TCGReg a1, tcg_target_long a2)
2873	{
2874	tgen_shr_int(s, type, a0, a1, TCG_REG_NONE, a2);
2875	}
2876
2877	static const TCGOutOpBinary outop_shr = {
2878	.base.static_constraint = C_O1_I2(r, r, ri),
2879	.out_rrr = tgen_shr,
2880	.out_rri = tgen_shri,
2881	};
2882
2883	static void tgen_sub(TCGContext *s, TCGType type,
2884	TCGReg a0, TCGReg a1, TCGReg a2)
2885	{
2886	if (type != TCG_TYPE_I32) {
2887	tcg_out_insn(s, RRFa, SGRK, a0, a1, a2);
2888	} else if (a0 == a1) {
2889	tcg_out_insn(s, RR, SR, a0, a2);
2890	} else {
2891	tcg_out_insn(s, RRFa, SRK, a0, a1, a2);
2892	}
2893	}
2894
2895	static const TCGOutOpSubtract outop_sub = {
2896	.base.static_constraint = C_O1_I2(r, r, r),
2897	.out_rrr = tgen_sub,
2898	};
2899
2900	static void tgen_subbo_rrr(TCGContext *s, TCGType type,
2901	TCGReg a0, TCGReg a1, TCGReg a2)
2902	{
2903	if (type != TCG_TYPE_I32) {
2904	tcg_out_insn(s, RRFa, SLGRK, a0, a1, a2);
2905	} else if (a0 == a1) {
2906	tcg_out_insn(s, RR, SLR, a0, a2);
2907	} else {
2908	tcg_out_insn(s, RRFa, SLRK, a0, a1, a2);
2909	}
2910	}
2911
2912	static void tgen_subbo_rri(TCGContext *s, TCGType type,
2913	TCGReg a0, TCGReg a1, tcg_target_long a2)
2914	{
2915	tcg_out_mov(s, type, a0, a1);
2916	if (type == TCG_TYPE_I32) {
2917	tcg_out_insn(s, RIL, SLFI, a0, a2);
2918	} else if (a2 >= 0) {
2919	tcg_out_insn(s, RIL, SLGFI, a0, a2);
2920	} else {
2921	tcg_out_insn(s, RIL, ALGFI, a0, -a2);
2922	}
2923	}
2924
2925	static const TCGOutOpAddSubCarry outop_subbo = {
2926	.base.static_constraint = C_O1_I2(r, r, rUV),
2927	.out_rrr = tgen_subbo_rrr,
2928	.out_rri = tgen_subbo_rri,
2929	};
2930
2931	static void tgen_subbio(TCGContext *s, TCGType type,
2932	TCGReg a0, TCGReg a1, TCGReg a2)
2933	{
2934	if (type == TCG_TYPE_I32) {
2935	tcg_out_insn(s, RRE, SLBR, a0, a2);
2936	} else {
2937	tcg_out_insn(s, RRE, SLBGR, a0, a2);
2938	}
2939	}
2940
2941	static const TCGOutOpAddSubCarry outop_subbio = {
2942	.base.static_constraint = C_O1_I2(r, 0, r),
2943	.out_rrr = tgen_subbio,
2944	};
2945
2946	#define outop_subbi outop_subbio
2947
2948	static void tcg_out_set_borrow(TCGContext *s)
2949	{
2950	tcg_out_insn(s, RR, CLR, TCG_REG_R0, TCG_REG_R0); /* cc = 0 */
2951	}
2952
2953	static const TCGOutOpBinary outop_smax = {
2954	.base.static_constraint = C_NotImplemented,
2955	};
2956
2957	static const TCGOutOpBinary outop_smin = {
2958	.base.static_constraint = C_NotImplemented,
2959	};
2960
2961	static const TCGOutOpBinary outop_umax = {
2962	.base.static_constraint = C_NotImplemented,
2963	};
2964
2965	static const TCGOutOpBinary outop_umin = {
2966	.base.static_constraint = C_NotImplemented,
2967	};
2968
2969	static void tgen_xor(TCGContext *s, TCGType type,
2970	TCGReg a0, TCGReg a1, TCGReg a2)
2971	{
2972	if (type != TCG_TYPE_I32) {
2973	tcg_out_insn(s, RRFa, XGRK, a0, a1, a2);
2974	} else if (a0 == a1) {
2975	tcg_out_insn(s, RR, XR, a0, a2);
2976	} else {
2977	tcg_out_insn(s, RRFa, XRK, a0, a1, a2);
2978	}
2979	}
2980
2981	static void tgen_xori_3(TCGContext *s, TCGType type,
2982	TCGReg a0, TCGReg a1, tcg_target_long a2)
2983	{
2984	tcg_out_mov(s, type, a0, a1);
2985	tgen_xori(s, a0, type == TCG_TYPE_I32 ? (uint32_t)a2 : a2);
2986	}
2987
2988	static const TCGOutOpBinary outop_xor = {
2989	.base.static_constraint = C_O1_I2(r, r, rK),
2990	.out_rrr = tgen_xor,
2991	.out_rri = tgen_xori_3,
2992	};
2993
2994	static void tgen_bswap16(TCGContext *s, TCGType type,
2995	TCGReg a0, TCGReg a1, unsigned flags)
2996	{
2997	if (type == TCG_TYPE_I32) {
2998	tcg_out_insn(s, RRE, LRVR, a0, a1);
2999	tcg_out_sh32(s, (flags & TCG_BSWAP_OS ? RS_SRA : RS_SRL),
3000	a0, TCG_REG_NONE, 16);
3001	} else {
3002	tcg_out_insn(s, RRE, LRVGR, a0, a1);
3003	tcg_out_sh64(s, (flags & TCG_BSWAP_OS ? RSY_SRAG : RSY_SRLG),
3004	a0, a0, TCG_REG_NONE, 48);
3005	}
3006	}
3007
3008	static const TCGOutOpBswap outop_bswap16 = {
3009	.base.static_constraint = C_O1_I1(r, r),
3010	.out_rr = tgen_bswap16,
3011	};
3012
3013	static void tgen_bswap32(TCGContext *s, TCGType type,
3014	TCGReg a0, TCGReg a1, unsigned flags)
3015	{
3016	tcg_out_insn(s, RRE, LRVR, a0, a1);
3017	if (flags & TCG_BSWAP_OS) {
3018	tcg_out_ext32s(s, a0, a0);
3019	} else if ((flags & (TCG_BSWAP_IZ \| TCG_BSWAP_OZ)) == TCG_BSWAP_OZ) {
3020	tcg_out_ext32u(s, a0, a0);
3021	}
3022	}
3023
3024	static const TCGOutOpBswap outop_bswap32 = {
3025	.base.static_constraint = C_O1_I1(r, r),
3026	.out_rr = tgen_bswap32,
3027	};
3028
3029	static void tgen_bswap64(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
3030	{
3031	tcg_out_insn(s, RRE, LRVGR, a0, a1);
3032	}
3033
3034	static const TCGOutOpUnary outop_bswap64 = {
3035	.base.static_constraint = C_O1_I1(r, r),
3036	.out_rr = tgen_bswap64,
3037	};
3038
3039	static const TCGOutOpUnary outop_revbit8 = {
3040	.base.static_constraint = C_NotImplemented,
3041	};
3042
3043	static const TCGOutOpBswap outop_revbit32 = {
3044	.base.static_constraint = C_NotImplemented,
3045	};
3046
3047	static const TCGOutOpUnary outop_revbit64 = {
3048	.base.static_constraint = C_NotImplemented,
3049	};
3050
3051	static void tgen_neg(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
3052	{
3053	if (type == TCG_TYPE_I32) {
3054	tcg_out_insn(s, RR, LCR, a0, a1);
3055	} else {
3056	tcg_out_insn(s, RRE, LCGR, a0, a1);
3057	}
3058	}
3059
3060	static const TCGOutOpUnary outop_neg = {
3061	.base.static_constraint = C_O1_I1(r, r),
3062	.out_rr = tgen_neg,
3063	};
3064
3065	static void tgen_not(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
3066	{
3067	tgen_nor(s, type, a0, a1, a1);
3068	}
3069
3070	static TCGConstraintSetIndex cset_not(TCGType type, unsigned flags)
3071	{
3072	return HAVE_FACILITY(MISC_INSN_EXT3) ? C_O1_I1(r, r) : C_NotImplemented;
3073	}
3074
3075	static const TCGOutOpUnary outop_not = {
3076	.base.static_constraint = C_Dynamic,
3077	.base.dynamic_constraint = cset_not,
3078	.out_rr = tgen_not,
3079	};
3080
3081	static void tcg_out_mb(TCGContext *s, unsigned a0)
3082	{
3083	/*
3084	* The host memory model is quite strong, we simply need to
3085	* serialize the instruction stream.
3086	*/
3087	if (a0 & TCG_MO_ST_LD) {
3088	/* fast-bcr-serialization facility (45) is present */
3089	tcg_out_insn(s, RR, BCR, 14, 0);
3090	}
3091	}
3092
3093	static void tgen_ld8u(TCGContext *s, TCGType type, TCGReg dest,
3094	TCGReg base, ptrdiff_t offset)
3095	{
3096	tcg_out_mem(s, 0, RXY_LLGC, dest, base, TCG_REG_NONE, offset);
3097	}
3098
3099	static const TCGOutOpLoad outop_ld8u = {
3100	.base.static_constraint = C_O1_I1(r, r),
3101	.out = tgen_ld8u,
3102	};
3103
3104	static void tgen_ld8s(TCGContext *s, TCGType type, TCGReg dest,
3105	TCGReg base, ptrdiff_t offset)
3106	{
3107	tcg_out_mem(s, 0, RXY_LGB, dest, base, TCG_REG_NONE, offset);
3108	}
3109
3110	static const TCGOutOpLoad outop_ld8s = {
3111	.base.static_constraint = C_O1_I1(r, r),
3112	.out = tgen_ld8s,
3113	};
3114
3115	static void tgen_ld16u(TCGContext *s, TCGType type, TCGReg dest,
3116	TCGReg base, ptrdiff_t offset)
3117	{
3118	tcg_out_mem(s, 0, RXY_LLGH, dest, base, TCG_REG_NONE, offset);
3119	}
3120
3121	static const TCGOutOpLoad outop_ld16u = {
3122	.base.static_constraint = C_O1_I1(r, r),
3123	.out = tgen_ld16u,
3124	};
3125
3126	static void tgen_ld16s(TCGContext *s, TCGType type, TCGReg dest,
3127	TCGReg base, ptrdiff_t offset)
3128	{
3129	if (type == TCG_TYPE_I32) {
3130	tcg_out_mem(s, RX_LH, RXY_LHY, dest, base, TCG_REG_NONE, offset);
3131	} else {
3132	tcg_out_mem(s, 0, RXY_LGH, dest, base, TCG_REG_NONE, offset);
3133	}
3134	}
3135
3136	static const TCGOutOpLoad outop_ld16s = {
3137	.base.static_constraint = C_O1_I1(r, r),
3138	.out = tgen_ld16s,
3139	};
3140
3141	static void tgen_ld32u(TCGContext *s, TCGType type, TCGReg dest,
3142	TCGReg base, ptrdiff_t offset)
3143	{
3144	tcg_out_mem(s, 0, RXY_LLGF, dest, base, TCG_REG_NONE, offset);
3145	}
3146
3147	static const TCGOutOpLoad outop_ld32u = {
3148	.base.static_constraint = C_O1_I1(r, r),
3149	.out = tgen_ld32u,
3150	};
3151
3152	static void tgen_ld32s(TCGContext *s, TCGType type, TCGReg dest,
3153	TCGReg base, ptrdiff_t offset)
3154	{
3155	tcg_out_mem(s, 0, RXY_LGF, dest, base, TCG_REG_NONE, offset);
3156	}
3157
3158	static const TCGOutOpLoad outop_ld32s = {
3159	.base.static_constraint = C_O1_I1(r, r),
3160	.out = tgen_ld32s,
3161	};
3162
3163	static void tgen_st8(TCGContext *s, TCGType type, TCGReg data,
3164	TCGReg base, ptrdiff_t offset)
3165	{
3166	tcg_out_mem(s, RX_STC, RXY_STCY, data, base, TCG_REG_NONE, offset);
3167	}
3168
3169	static const TCGOutOpStore outop_st8 = {
3170	.base.static_constraint = C_O0_I2(r, r),
3171	.out_r = tgen_st8,
3172	};
3173
3174	static void tgen_st16(TCGContext *s, TCGType type, TCGReg data,
3175	TCGReg base, ptrdiff_t offset)
3176	{
3177	tcg_out_mem(s, RX_STH, RXY_STHY, data, base, TCG_REG_NONE, offset);
3178	}
3179
3180	static const TCGOutOpStore outop_st16 = {
3181	.base.static_constraint = C_O0_I2(r, r),
3182	.out_r = tgen_st16,
3183	};
3184
3185	static const TCGOutOpStore outop_st = {
3186	.base.static_constraint = C_O0_I2(r, r),
3187	.out_r = tcg_out_st,
3188	};
3189
3190
3191	static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
3192	TCGReg dst, TCGReg src)
3193	{
3194	if (is_general_reg(src)) {
3195	/* Replicate general register into two MO_64. */
3196	tcg_out_insn(s, VRRf, VLVGP, dst, src, src);
3197	if (vece == MO_64) {
3198	return true;
3199	}
3200	src = dst;
3201	}
3202
3203	/*
3204	* Recall that the "standard" integer, within a vector, is the
3205	* rightmost element of the leftmost doubleword, a-la VLLEZ.
3206	*/
3207	tcg_out_insn(s, VRIc, VREP, dst, (8 >> vece) - 1, src, vece);
3208	return true;
3209	}
3210
3211	static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
3212	TCGReg dst, TCGReg base, intptr_t offset)
3213	{
3214	tcg_out_vrx_mem(s, VRX_VLREP, dst, base, TCG_REG_NONE, offset, vece);
3215	return true;
3216	}
3217
3218	static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
3219	TCGReg dst, int64_t val)
3220	{
3221	int i, mask, msb, lsb;
3222
3223	/* Look for int16_t elements. */
3224	if (vece <= MO_16 \|\|
3225	(vece == MO_32 ? (int32_t)val : val) == (int16_t)val) {
3226	tcg_out_insn(s, VRIa, VREPI, dst, val, vece);
3227	return;
3228	}
3229
3230	/* Look for bit masks. */
3231	if (vece == MO_32) {
3232	if (risbg_mask((int32_t)val)) {
3233	/* Handle wraparound by swapping msb and lsb. */
3234	if ((val & 0x80000001u) == 0x80000001u) {
3235	msb = 32 - ctz32(~val);
3236	lsb = clz32(~val) - 1;
3237	} else {
3238	msb = clz32(val);
3239	lsb = 31 - ctz32(val);
3240	}
3241	tcg_out_insn(s, VRIb, VGM, dst, msb, lsb, MO_32);
3242	return;
3243	}
3244	} else {
3245	if (risbg_mask(val)) {
3246	/* Handle wraparound by swapping msb and lsb. */
3247	if ((val & 0x8000000000000001ull) == 0x8000000000000001ull) {
3248	/* Handle wraparound by swapping msb and lsb. */
3249	msb = 64 - ctz64(~val);
3250	lsb = clz64(~val) - 1;
3251	} else {
3252	msb = clz64(val);
3253	lsb = 63 - ctz64(val);
3254	}
3255	tcg_out_insn(s, VRIb, VGM, dst, msb, lsb, MO_64);
3256	return;
3257	}
3258	}
3259
3260	/* Look for all bytes 0x00 or 0xff. */
3261	for (i = mask = 0; i < 8; i++) {
3262	uint8_t byte = val >> (i * 8);
3263	if (byte == 0xff) {
3264	mask \|= 1 << i;
3265	} else if (byte != 0) {
3266	break;
3267	}
3268	}
3269	if (i == 8) {
3270	tcg_out_insn(s, VRIa, VGBM, dst, mask * 0x0101, 0);
3271	return;
3272	}
3273
3274	/* Otherwise, stuff it in the constant pool. */
3275	tcg_out_insn(s, RIL, LARL, TCG_TMP0, 0);
3276	new_pool_label(s, val, R_390_PC32DBL, s->code_ptr - 2, 2);
3277	tcg_out_insn(s, VRX, VLREP, dst, TCG_TMP0, TCG_REG_NONE, 0, MO_64);
3278	}
3279
3280	static bool tcg_out_cmp_vec_noinv(TCGContext *s, unsigned vece, TCGReg a0,
3281	TCGReg a1, TCGReg a2, TCGCond cond)
3282	{
3283	bool need_swap = false, need_inv = false;
3284
3285	switch (cond) {
3286	case TCG_COND_EQ:
3287	case TCG_COND_GT:
3288	case TCG_COND_GTU:
3289	break;
3290	case TCG_COND_NE:
3291	case TCG_COND_LE:
3292	case TCG_COND_LEU:
3293	need_inv = true;
3294	break;
3295	case TCG_COND_LT:
3296	case TCG_COND_LTU:
3297	need_swap = true;
3298	break;
3299	case TCG_COND_GE:
3300	case TCG_COND_GEU:
3301	need_swap = need_inv = true;
3302	break;
3303	default:
3304	g_assert_not_reached();
3305	}
3306
3307	if (need_inv) {
3308	cond = tcg_invert_cond(cond);
3309	}
3310	if (need_swap) {
3311	TCGReg swap = a1;
3312	a1 = a2;
3313	a2 = swap;
3314	cond = tcg_swap_cond(cond);
3315	}
3316
3317	switch (cond) {
3318	case TCG_COND_EQ:
3319	tcg_out_insn(s, VRRc, VCEQ, a0, a1, a2, vece);
3320	break;
3321	case TCG_COND_GT:
3322	tcg_out_insn(s, VRRc, VCH, a0, a1, a2, vece);
3323	break;
3324	case TCG_COND_GTU:
3325	tcg_out_insn(s, VRRc, VCHL, a0, a1, a2, vece);
3326	break;
3327	default:
3328	g_assert_not_reached();
3329	}
3330	return need_inv;
3331	}
3332
3333	static void tcg_out_cmp_vec(TCGContext *s, unsigned vece, TCGReg a0,
3334	TCGReg a1, TCGReg a2, TCGCond cond)
3335	{
3336	if (tcg_out_cmp_vec_noinv(s, vece, a0, a1, a2, cond)) {
3337	tcg_out_insn(s, VRRc, VNO, a0, a0, a0, 0);
3338	}
3339	}
3340
3341	static void tcg_out_cmpsel_vec(TCGContext *s, unsigned vece, TCGReg a0,
3342	TCGReg c1, TCGReg c2, TCGArg v3,
3343	int const_v3, TCGReg v4, TCGCond cond)
3344	{
3345	bool inv = tcg_out_cmp_vec_noinv(s, vece, TCG_VEC_TMP0, c1, c2, cond);
3346
3347	if (!const_v3) {
3348	if (inv) {
3349	tcg_out_insn(s, VRRe, VSEL, a0, v4, v3, TCG_VEC_TMP0);
3350	} else {
3351	tcg_out_insn(s, VRRe, VSEL, a0, v3, v4, TCG_VEC_TMP0);
3352	}
3353	} else if (v3) {
3354	if (inv) {
3355	tcg_out_insn(s, VRRc, VOC, a0, v4, TCG_VEC_TMP0, 0);
3356	} else {
3357	tcg_out_insn(s, VRRc, VO, a0, v4, TCG_VEC_TMP0, 0);
3358	}
3359	} else {
3360	if (inv) {
3361	tcg_out_insn(s, VRRc, VN, a0, v4, TCG_VEC_TMP0, 0);
3362	} else {
3363	tcg_out_insn(s, VRRc, VNC, a0, v4, TCG_VEC_TMP0, 0);
3364	}
3365	}
3366	}
3367
3368	static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
3369	unsigned vecl, unsigned vece,
3370	const TCGArg args[TCG_MAX_OP_ARGS],
3371	const int const_args[TCG_MAX_OP_ARGS])
3372	{
3373	TCGType type = vecl + TCG_TYPE_V64;
3374	TCGArg a0 = args[0], a1 = args[1], a2 = args[2];
3375
3376	switch (opc) {
3377	case INDEX_op_ld_vec:
3378	tcg_out_ld(s, type, a0, a1, a2);
3379	break;
3380	case INDEX_op_st_vec:
3381	tcg_out_st(s, type, a0, a1, a2);
3382	break;
3383	case INDEX_op_dupm_vec:
3384	tcg_out_dupm_vec(s, type, vece, a0, a1, a2);
3385	break;
3386
3387	case INDEX_op_abs_vec:
3388	tcg_out_insn(s, VRRa, VLP, a0, a1, vece);
3389	break;
3390	case INDEX_op_neg_vec:
3391	tcg_out_insn(s, VRRa, VLC, a0, a1, vece);
3392	break;
3393	case INDEX_op_not_vec:
3394	tcg_out_insn(s, VRRc, VNO, a0, a1, a1, 0);
3395	break;
3396
3397	case INDEX_op_add_vec:
3398	tcg_out_insn(s, VRRc, VA, a0, a1, a2, vece);
3399	break;
3400	case INDEX_op_sub_vec:
3401	tcg_out_insn(s, VRRc, VS, a0, a1, a2, vece);
3402	break;
3403	case INDEX_op_and_vec:
3404	tcg_out_insn(s, VRRc, VN, a0, a1, a2, 0);
3405	break;
3406	case INDEX_op_andc_vec:
3407	tcg_out_insn(s, VRRc, VNC, a0, a1, a2, 0);
3408	break;
3409	case INDEX_op_mul_vec:
3410	tcg_out_insn(s, VRRc, VML, a0, a1, a2, vece);
3411	break;
3412	case INDEX_op_or_vec:
3413	tcg_out_insn(s, VRRc, VO, a0, a1, a2, 0);
3414	break;
3415	case INDEX_op_orc_vec:
3416	tcg_out_insn(s, VRRc, VOC, a0, a1, a2, 0);
3417	break;
3418	case INDEX_op_xor_vec:
3419	tcg_out_insn(s, VRRc, VX, a0, a1, a2, 0);
3420	break;
3421	case INDEX_op_nand_vec:
3422	tcg_out_insn(s, VRRc, VNN, a0, a1, a2, 0);
3423	break;
3424	case INDEX_op_nor_vec:
3425	tcg_out_insn(s, VRRc, VNO, a0, a1, a2, 0);
3426	break;
3427	case INDEX_op_eqv_vec:
3428	tcg_out_insn(s, VRRc, VNX, a0, a1, a2, 0);
3429	break;
3430
3431	case INDEX_op_shli_vec:
3432	tcg_out_insn(s, VRSa, VESL, a0, a2, TCG_REG_NONE, a1, vece);
3433	break;
3434	case INDEX_op_shri_vec:
3435	tcg_out_insn(s, VRSa, VESRL, a0, a2, TCG_REG_NONE, a1, vece);
3436	break;
3437	case INDEX_op_sari_vec:
3438	tcg_out_insn(s, VRSa, VESRA, a0, a2, TCG_REG_NONE, a1, vece);
3439	break;
3440	case INDEX_op_rotli_vec:
3441	tcg_out_insn(s, VRSa, VERLL, a0, a2, TCG_REG_NONE, a1, vece);
3442	break;
3443	case INDEX_op_shls_vec:
3444	tcg_out_insn(s, VRSa, VESL, a0, 0, a2, a1, vece);
3445	break;
3446	case INDEX_op_shrs_vec:
3447	tcg_out_insn(s, VRSa, VESRL, a0, 0, a2, a1, vece);
3448	break;
3449	case INDEX_op_sars_vec:
3450	tcg_out_insn(s, VRSa, VESRA, a0, 0, a2, a1, vece);
3451	break;
3452	case INDEX_op_rotls_vec:
3453	tcg_out_insn(s, VRSa, VERLL, a0, 0, a2, a1, vece);
3454	break;
3455	case INDEX_op_shlv_vec:
3456	tcg_out_insn(s, VRRc, VESLV, a0, a1, a2, vece);
3457	break;
3458	case INDEX_op_shrv_vec:
3459	tcg_out_insn(s, VRRc, VESRLV, a0, a1, a2, vece);
3460	break;
3461	case INDEX_op_sarv_vec:
3462	tcg_out_insn(s, VRRc, VESRAV, a0, a1, a2, vece);
3463	break;
3464	case INDEX_op_rotlv_vec:
3465	tcg_out_insn(s, VRRc, VERLLV, a0, a1, a2, vece);
3466	break;
3467
3468	case INDEX_op_smin_vec:
3469	tcg_out_insn(s, VRRc, VMN, a0, a1, a2, vece);
3470	break;
3471	case INDEX_op_smax_vec:
3472	tcg_out_insn(s, VRRc, VMX, a0, a1, a2, vece);
3473	break;
3474	case INDEX_op_umin_vec:
3475	tcg_out_insn(s, VRRc, VMNL, a0, a1, a2, vece);
3476	break;
3477	case INDEX_op_umax_vec:
3478	tcg_out_insn(s, VRRc, VMXL, a0, a1, a2, vece);
3479	break;
3480
3481	case INDEX_op_bitsel_vec:
3482	tcg_out_insn(s, VRRe, VSEL, a0, a2, args[3], a1);
3483	break;
3484
3485	case INDEX_op_cmp_vec:
3486	tcg_out_cmp_vec(s, vece, a0, a1, a2, args[3]);
3487	break;
3488	case INDEX_op_cmpsel_vec:
3489	tcg_out_cmpsel_vec(s, vece, a0, a1, a2, args[3], const_args[3],
3490	args[4], args[5]);
3491	break;
3492
3493	case INDEX_op_s390_vuph_vec:
3494	tcg_out_insn(s, VRRa, VUPH, a0, a1, vece);
3495	break;
3496	case INDEX_op_s390_vupl_vec:
3497	tcg_out_insn(s, VRRa, VUPL, a0, a1, vece);
3498	break;
3499	case INDEX_op_s390_vpks_vec:
3500	tcg_out_insn(s, VRRc, VPKS, a0, a1, a2, vece);
3501	break;
3502
3503	case INDEX_op_mov_vec: /* Always emitted via tcg_out_mov. */
3504	case INDEX_op_dup_vec: /* Always emitted via tcg_out_dup_vec. */
3505	default:
3506	g_assert_not_reached();
3507	}
3508	}
3509
3510	int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece)
3511	{
3512	switch (opc) {
3513	case INDEX_op_abs_vec:
3514	case INDEX_op_add_vec:
3515	case INDEX_op_and_vec:
3516	case INDEX_op_andc_vec:
3517	case INDEX_op_bitsel_vec:
3518	case INDEX_op_eqv_vec:
3519	case INDEX_op_nand_vec:
3520	case INDEX_op_neg_vec:
3521	case INDEX_op_nor_vec:
3522	case INDEX_op_not_vec:
3523	case INDEX_op_or_vec:
3524	case INDEX_op_orc_vec:
3525	case INDEX_op_rotli_vec:
3526	case INDEX_op_rotls_vec:
3527	case INDEX_op_rotlv_vec:
3528	case INDEX_op_sari_vec:
3529	case INDEX_op_sars_vec:
3530	case INDEX_op_sarv_vec:
3531	case INDEX_op_shli_vec:
3532	case INDEX_op_shls_vec:
3533	case INDEX_op_shlv_vec:
3534	case INDEX_op_shri_vec:
3535	case INDEX_op_shrs_vec:
3536	case INDEX_op_shrv_vec:
3537	case INDEX_op_smax_vec:
3538	case INDEX_op_smin_vec:
3539	case INDEX_op_sub_vec:
3540	case INDEX_op_umax_vec:
3541	case INDEX_op_umin_vec:
3542	case INDEX_op_xor_vec:
3543	case INDEX_op_cmp_vec:
3544	case INDEX_op_cmpsel_vec:
3545	return 1;
3546	case INDEX_op_rotrv_vec:
3547	return -1;
3548	case INDEX_op_mul_vec:
3549	return vece < MO_64;
3550	case INDEX_op_ssadd_vec:
3551	case INDEX_op_sssub_vec:
3552	return vece < MO_64 ? -1 : 0;
3553	default:
3554	return 0;
3555	}
3556	}
3557
3558	static void expand_vec_sat(TCGType type, unsigned vece, TCGv_vec v0,
3559	TCGv_vec v1, TCGv_vec v2, TCGOpcode add_sub_opc)
3560	{
3561	TCGv_vec h1 = tcg_temp_new_vec(type);
3562	TCGv_vec h2 = tcg_temp_new_vec(type);
3563	TCGv_vec l1 = tcg_temp_new_vec(type);
3564	TCGv_vec l2 = tcg_temp_new_vec(type);
3565
3566	tcg_debug_assert (vece < MO_64);
3567
3568	/* Unpack with sign-extension. */
3569	vec_gen_2(INDEX_op_s390_vuph_vec, type, vece,
3570	tcgv_vec_arg(h1), tcgv_vec_arg(v1));
3571	vec_gen_2(INDEX_op_s390_vuph_vec, type, vece,
3572	tcgv_vec_arg(h2), tcgv_vec_arg(v2));
3573
3574	vec_gen_2(INDEX_op_s390_vupl_vec, type, vece,
3575	tcgv_vec_arg(l1), tcgv_vec_arg(v1));
3576	vec_gen_2(INDEX_op_s390_vupl_vec, type, vece,
3577	tcgv_vec_arg(l2), tcgv_vec_arg(v2));
3578
3579	/* Arithmetic on a wider element size. */
3580	vec_gen_3(add_sub_opc, type, vece + 1, tcgv_vec_arg(h1),
3581	tcgv_vec_arg(h1), tcgv_vec_arg(h2));
3582	vec_gen_3(add_sub_opc, type, vece + 1, tcgv_vec_arg(l1),
3583	tcgv_vec_arg(l1), tcgv_vec_arg(l2));
3584
3585	/* Pack with saturation. */
3586	vec_gen_3(INDEX_op_s390_vpks_vec, type, vece + 1,
3587	tcgv_vec_arg(v0), tcgv_vec_arg(h1), tcgv_vec_arg(l1));
3588
3589	tcg_temp_free_vec(h1);
3590	tcg_temp_free_vec(h2);
3591	tcg_temp_free_vec(l1);
3592	tcg_temp_free_vec(l2);
3593	}
3594
3595	void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece,
3596	TCGArg a0, ...)
3597	{
3598	va_list va;
3599	TCGv_vec v0, v1, v2, t0;
3600
3601	va_start(va, a0);
3602	v0 = temp_tcgv_vec(arg_temp(a0));
3603	v1 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
3604	v2 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg)));
3605
3606	switch (opc) {
3607	case INDEX_op_rotrv_vec:
3608	t0 = tcg_temp_new_vec(type);
3609	tcg_gen_neg_vec(vece, t0, v2);
3610	tcg_gen_rotlv_vec(vece, v0, v1, t0);
3611	tcg_temp_free_vec(t0);
3612	break;
3613
3614	case INDEX_op_ssadd_vec:
3615	expand_vec_sat(type, vece, v0, v1, v2, INDEX_op_add_vec);
3616	break;
3617	case INDEX_op_sssub_vec:
3618	expand_vec_sat(type, vece, v0, v1, v2, INDEX_op_sub_vec);
3619	break;
3620
3621	default:
3622	g_assert_not_reached();
3623	}
3624	va_end(va);
3625	}
3626
3627	static TCGConstraintSetIndex
3628	tcg_target_op_def(TCGOpcode op, TCGType type, unsigned flags)
3629	{
3630	switch (op) {
3631	case INDEX_op_st_vec:
3632	return C_O0_I2(v, r);
3633	case INDEX_op_ld_vec:
3634	case INDEX_op_dupm_vec:
3635	return C_O1_I1(v, r);
3636	case INDEX_op_dup_vec:
3637	return C_O1_I1(v, vr);
3638	case INDEX_op_abs_vec:
3639	case INDEX_op_neg_vec:
3640	case INDEX_op_not_vec:
3641	case INDEX_op_rotli_vec:
3642	case INDEX_op_sari_vec:
3643	case INDEX_op_shli_vec:
3644	case INDEX_op_shri_vec:
3645	case INDEX_op_s390_vuph_vec:
3646	case INDEX_op_s390_vupl_vec:
3647	return C_O1_I1(v, v);
3648	case INDEX_op_add_vec:
3649	case INDEX_op_sub_vec:
3650	case INDEX_op_and_vec:
3651	case INDEX_op_andc_vec:
3652	case INDEX_op_or_vec:
3653	case INDEX_op_orc_vec:
3654	case INDEX_op_xor_vec:
3655	case INDEX_op_nand_vec:
3656	case INDEX_op_nor_vec:
3657	case INDEX_op_eqv_vec:
3658	case INDEX_op_cmp_vec:
3659	case INDEX_op_mul_vec:
3660	case INDEX_op_rotlv_vec:
3661	case INDEX_op_rotrv_vec:
3662	case INDEX_op_shlv_vec:
3663	case INDEX_op_shrv_vec:
3664	case INDEX_op_sarv_vec:
3665	case INDEX_op_smax_vec:
3666	case INDEX_op_smin_vec:
3667	case INDEX_op_umax_vec:
3668	case INDEX_op_umin_vec:
3669	case INDEX_op_s390_vpks_vec:
3670	return C_O1_I2(v, v, v);
3671	case INDEX_op_rotls_vec:
3672	case INDEX_op_shls_vec:
3673	case INDEX_op_shrs_vec:
3674	case INDEX_op_sars_vec:
3675	return C_O1_I2(v, v, r);
3676	case INDEX_op_bitsel_vec:
3677	return C_O1_I3(v, v, v, v);
3678	case INDEX_op_cmpsel_vec:
3679	return (TCG_TARGET_HAS_orc_vec
3680	? C_O1_I4(v, v, v, vZM, v)
3681	: C_O1_I4(v, v, v, vZ, v));
3682
3683	default:
3684	return C_NotImplemented;
3685	}
3686	}
3687
3688	/*
3689	* Mainline glibc added HWCAP_S390_VX before it was kernel abi.
3690	* Some distros have fixed this up locally, others have not.
3691	*/
3692	#ifndef HWCAP_S390_VXRS
3693	#define HWCAP_S390_VXRS 2048
3694	#endif
3695
3696	static void query_s390_facilities(void)
3697	{
3698	unsigned long hwcap = qemu_getauxval(AT_HWCAP);
3699	const char *which;
3700
3701	/* Is STORE FACILITY LIST EXTENDED available? Honestly, I believe this
3702	is present on all 64-bit systems, but let's check for it anyway. */
3703	if (hwcap & HWCAP_S390_STFLE) {
3704	register int r0 __asm__("0") = ARRAY_SIZE(s390_facilities) - 1;
3705	register void *r1 __asm__("1") = s390_facilities;
3706
3707	/* stfle 0(%r1) */
3708	asm volatile(".word 0xb2b0,0x1000"
3709	: "=r"(r0) : "r"(r0), "r"(r1) : "memory", "cc");
3710	}
3711
3712	/*
3713	* Use of vector registers requires os support beyond the facility bit.
3714	* If the kernel does not advertise support, disable the facility bits.
3715	* There is nothing else we currently care about in the 3rd word, so
3716	* disable VECTOR with one store.
3717	*/
3718	if (!(hwcap & HWCAP_S390_VXRS)) {
3719	s390_facilities[2] = 0;
3720	}
3721
3722	/*
3723	* Minimum supported cpu revision is z196.
3724	* Check for all required facilities.
3725	* ZARCH_ACTIVE is done via preprocessor check for 64-bit.
3726	*/
3727	if (!HAVE_FACILITY(LONG_DISP)) {
3728	which = "long-displacement";
3729	goto fail;
3730	}
3731	if (!HAVE_FACILITY(EXT_IMM)) {
3732	which = "extended-immediate";
3733	goto fail;
3734	}
3735	if (!HAVE_FACILITY(GEN_INST_EXT)) {
3736	which = "general-instructions-extension";
3737	goto fail;
3738	}
3739	/*
3740	* Facility 45 is a big bin that contains: distinct-operands,
3741	* fast-BCR-serialization, high-word, population-count,
3742	* interlocked-access-1, and load/store-on-condition-1
3743	*/
3744	if (!HAVE_FACILITY(45)) {
3745	which = "45";
3746	goto fail;
3747	}
3748	return;
3749
3750	fail:
3751	error_report("%s: missing required facility %s", __func__, which);
3752	exit(EXIT_FAILURE);
3753	}
3754
3755	static void tcg_target_init(TCGContext *s)
3756	{
3757	query_s390_facilities();
3758
3759	tcg_target_available_regs[TCG_TYPE_I32] = 0xffff;
3760	tcg_target_available_regs[TCG_TYPE_I64] = 0xffff;
3761	if (HAVE_FACILITY(VECTOR)) {
3762	tcg_target_available_regs[TCG_TYPE_V64] = 0xffffffff00000000ull;
3763	tcg_target_available_regs[TCG_TYPE_V128] = 0xffffffff00000000ull;
3764	}
3765
3766	tcg_target_call_clobber_regs = 0;
3767	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R0);
3768	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R1);
3769	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R2);
3770	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R3);
3771	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R4);
3772	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R5);
3773	/* The r6 register is technically call-saved, but it's also a parameter
3774	register, so it can get killed by setup for the qemu_st helper. */
3775	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R6);
3776	/* The return register can be considered call-clobbered. */
3777	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_R14);
3778
3779	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V0);
3780	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V1);
3781	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V2);
3782	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V3);
3783	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V4);
3784	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V5);
3785	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V6);
3786	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V7);
3787	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V16);
3788	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V17);
3789	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V18);
3790	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V19);
3791	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V20);
3792	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V21);
3793	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V22);
3794	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V23);
3795	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V24);
3796	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V25);
3797	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V26);
3798	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V27);
3799	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V28);
3800	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V29);
3801	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V30);
3802	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_V31);
3803
3804	s->reserved_regs = 0;
3805	tcg_regset_set_reg(s->reserved_regs, TCG_TMP0);
3806	tcg_regset_set_reg(s->reserved_regs, TCG_VEC_TMP0);
3807	/* XXX many insns can't be used with R0, so we better avoid it for now */
3808	tcg_regset_set_reg(s->reserved_regs, TCG_REG_R0);
3809	tcg_regset_set_reg(s->reserved_regs, TCG_REG_CALL_STACK);
3810	}
3811
3812	#define FRAME_SIZE ((int)(TCG_TARGET_CALL_STACK_OFFSET \
3813	+ TCG_STATIC_CALL_ARGS_SIZE \
3814	+ CPU_TEMP_BUF_NLONGS * sizeof(long)))
3815
3816	static void tcg_target_qemu_prologue(TCGContext *s)
3817	{
3818	/* stmg %r6,%r15,48(%r15) (save registers) */
3819	tcg_out_insn(s, RXY, STMG, TCG_REG_R6, TCG_REG_R15, TCG_REG_R15, 48);
3820
3821	/* aghi %r15,-frame_size */
3822	tcg_out_insn(s, RI, AGHI, TCG_REG_R15, -FRAME_SIZE);
3823
3824	tcg_set_frame(s, TCG_REG_CALL_STACK,
3825	TCG_STATIC_CALL_ARGS_SIZE + TCG_TARGET_CALL_STACK_OFFSET,
3826	CPU_TEMP_BUF_NLONGS * sizeof(long));
3827
3828	if (!tcg_use_softmmu && guest_base >= 0x80000) {
3829	tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, guest_base);
3830	tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
3831	}
3832
3833	tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
3834
3835	/* br %r3 (go to TB) */
3836	tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, tcg_target_call_iarg_regs[1]);
3837
3838	/*
3839	* Return path for goto_ptr. Set return value to 0, a-la exit_tb,
3840	* and fall through to the rest of the epilogue.
3841	*/
3842	tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr);
3843	tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R2, 0);
3844
3845	/* TB epilogue */
3846	tb_ret_addr = tcg_splitwx_to_rx(s->code_ptr);
3847
3848	/* lmg %r6,%r15,fs+48(%r15) (restore registers) */
3849	tcg_out_insn(s, RXY, LMG, TCG_REG_R6, TCG_REG_R15, TCG_REG_R15,
3850	FRAME_SIZE + 48);
3851
3852	/* br %r14 (return) */
3853	tcg_out_insn(s, RR, BCR, S390_CC_ALWAYS, TCG_REG_R14);
3854	}
3855
3856	static void tcg_out_tb_start(TCGContext *s)
3857	{
3858	/* nothing to do */
3859	}
3860
3861	static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
3862	{
3863	memset(p, 0x07, count * sizeof(tcg_insn_unit));
3864	}
3865
3866	typedef struct {
3867	DebugFrameHeader h;
3868	uint8_t fde_def_cfa[4];
3869	uint8_t fde_reg_ofs[18];
3870	} DebugFrame;
3871
3872	/* We're expecting a 2 byte uleb128 encoded value. */
3873	QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
3874
3875	#define ELF_HOST_MACHINE EM_S390
3876
3877	static const DebugFrame debug_frame = {
3878	.h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
3879	.h.cie.id = -1,
3880	.h.cie.version = 1,
3881	.h.cie.code_align = 1,
3882	.h.cie.data_align = 8, /* sleb128 8 */
3883	.h.cie.return_column = TCG_REG_R14,
3884
3885	/* Total FDE size does not include the "len" member. */
3886	.h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
3887
3888	.fde_def_cfa = {
3889	12, TCG_REG_CALL_STACK, /* DW_CFA_def_cfa %r15, ... */
3890	(FRAME_SIZE & 0x7f) \| 0x80, /* ... uleb128 FRAME_SIZE */
3891	(FRAME_SIZE >> 7)
3892	},
3893	.fde_reg_ofs = {
3894	0x86, 6, /* DW_CFA_offset, %r6, 48 */
3895	0x87, 7, /* DW_CFA_offset, %r7, 56 */
3896	0x88, 8, /* DW_CFA_offset, %r8, 64 */
3897	0x89, 9, /* DW_CFA_offset, %r92, 72 */
3898	0x8a, 10, /* DW_CFA_offset, %r10, 80 */
3899	0x8b, 11, /* DW_CFA_offset, %r11, 88 */
3900	0x8c, 12, /* DW_CFA_offset, %r12, 96 */
3901	0x8d, 13, /* DW_CFA_offset, %r13, 104 */
3902	0x8e, 14, /* DW_CFA_offset, %r14, 112 */
3903	}
3904	};
3905
3906	void tcg_register_jit(const void *buf, size_t buf_size)
3907	{
3908	tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
3909	}