tcg-target.c.inc · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

inc 2,206 lines 64.9 KB

Raw

1	/*
2	* Tiny Code Generator for QEMU
3	*
4	* Copyright (c) 2008 Fabrice Bellard
5	*
6	* Permission is hereby granted, free of charge, to any person obtaining a copy
7	* of this software and associated documentation files (the "Software"), to deal
8	* in the Software without restriction, including without limitation the rights
9	* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10	* copies of the Software, and to permit persons to whom the Software is
11	* furnished to do so, subject to the following conditions:
12	*
13	* The above copyright notice and this permission notice shall be included in
14	* all copies or substantial portions of the Software.
15	*
16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22	* THE SOFTWARE.
23	*/
24
25	/* We only support generating code for 64-bit mode. */
26	#ifndef __arch64__
27	#error "unsupported code generation mode"
28	#endif
29
30	/* Used for function call generation. */
31	#define TCG_REG_CALL_STACK TCG_REG_O6
32	#define TCG_TARGET_STACK_BIAS 2047
33	#define TCG_TARGET_STACK_ALIGN 16
34	#define TCG_TARGET_CALL_STACK_OFFSET (128 + 6 * 8 + TCG_TARGET_STACK_BIAS)
35	#define TCG_TARGET_CALL_ARG_I32 TCG_CALL_ARG_EXTEND
36	#define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL
37	#define TCG_TARGET_CALL_ARG_I128 TCG_CALL_ARG_NORMAL
38	#define TCG_TARGET_CALL_RET_I128 TCG_CALL_RET_NORMAL
39
40	#ifdef CONFIG_DEBUG_TCG
41	static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
42	"%g0",
43	"%g1",
44	"%g2",
45	"%g3",
46	"%g4",
47	"%g5",
48	"%g6",
49	"%g7",
50	"%o0",
51	"%o1",
52	"%o2",
53	"%o3",
54	"%o4",
55	"%o5",
56	"%o6",
57	"%o7",
58	"%l0",
59	"%l1",
60	"%l2",
61	"%l3",
62	"%l4",
63	"%l5",
64	"%l6",
65	"%l7",
66	"%i0",
67	"%i1",
68	"%i2",
69	"%i3",
70	"%i4",
71	"%i5",
72	"%i6",
73	"%i7",
74	};
75	#endif
76
77	#define TCG_CT_CONST_S11 0x100
78	#define TCG_CT_CONST_S13 0x200
79
80	#define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 32)
81
82	/* Define some temporary registers. T3 is used for constant generation. */
83	#define TCG_REG_T1 TCG_REG_G1
84	#define TCG_REG_T2 TCG_REG_G2
85	#define TCG_REG_T3 TCG_REG_O7
86
87	#ifndef CONFIG_SOFTMMU
88	# define TCG_GUEST_BASE_REG TCG_REG_I5
89	#endif
90
91	#define TCG_REG_TB TCG_REG_I1
92
93	static const int tcg_target_reg_alloc_order[] = {
94	TCG_REG_L0,
95	TCG_REG_L1,
96	TCG_REG_L2,
97	TCG_REG_L3,
98	TCG_REG_L4,
99	TCG_REG_L5,
100	TCG_REG_L6,
101	TCG_REG_L7,
102
103	TCG_REG_I0,
104	TCG_REG_I1,
105	TCG_REG_I2,
106	TCG_REG_I3,
107	TCG_REG_I4,
108	TCG_REG_I5,
109
110	TCG_REG_G3,
111	TCG_REG_G4,
112	TCG_REG_G5,
113
114	TCG_REG_O0,
115	TCG_REG_O1,
116	TCG_REG_O2,
117	TCG_REG_O3,
118	TCG_REG_O4,
119	TCG_REG_O5,
120	};
121
122	static const int tcg_target_call_iarg_regs[6] = {
123	TCG_REG_O0,
124	TCG_REG_O1,
125	TCG_REG_O2,
126	TCG_REG_O3,
127	TCG_REG_O4,
128	TCG_REG_O5,
129	};
130
131	static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot)
132	{
133	tcg_debug_assert(kind == TCG_CALL_RET_NORMAL);
134	tcg_debug_assert(slot >= 0 && slot <= 3);
135	return TCG_REG_O0 + slot;
136	}
137
138	#define INSN_OP(x) ((x) << 30)
139	#define INSN_OP2(x) ((x) << 22)
140	#define INSN_OP3(x) ((x) << 19)
141	#define INSN_OPF(x) ((x) << 5)
142	#define INSN_RD(x) ((x) << 25)
143	#define INSN_RS1(x) ((x) << 14)
144	#define INSN_RS2(x) (x)
145	#define INSN_ASI(x) ((x) << 5)
146
147	#define INSN_IMM10(x) ((1 << 13) \| ((x) & 0x3ff))
148	#define INSN_IMM11(x) ((1 << 13) \| ((x) & 0x7ff))
149	#define INSN_IMM13(x) ((1 << 13) \| ((x) & 0x1fff))
150	#define INSN_OFF16(x) ((((x) >> 2) & 0x3fff) \| ((((x) >> 16) & 3) << 20))
151	#define INSN_OFF19(x) (((x) >> 2) & 0x07ffff)
152	#define INSN_COND(x) ((x) << 25)
153
154	#define COND_N 0x0
155	#define COND_E 0x1
156	#define COND_LE 0x2
157	#define COND_L 0x3
158	#define COND_LEU 0x4
159	#define COND_CS 0x5
160	#define COND_NEG 0x6
161	#define COND_VS 0x7
162	#define COND_A 0x8
163	#define COND_NE 0x9
164	#define COND_G 0xa
165	#define COND_GE 0xb
166	#define COND_GU 0xc
167	#define COND_CC 0xd
168	#define COND_POS 0xe
169	#define COND_VC 0xf
170	#define BA (INSN_OP(0) \| INSN_COND(COND_A) \| INSN_OP2(0x2))
171
172	#define RCOND_Z 1
173	#define RCOND_LEZ 2
174	#define RCOND_LZ 3
175	#define RCOND_NZ 5
176	#define RCOND_GZ 6
177	#define RCOND_GEZ 7
178
179	#define MOVCC_ICC (1 << 18)
180	#define MOVCC_XCC (1 << 18 \| 1 << 12)
181
182	#define BPCC_ICC 0
183	#define BPCC_XCC (2 << 20)
184	#define BPCC_PT (1 << 19)
185	#define BPCC_PN 0
186	#define BPCC_A (1 << 29)
187
188	#define BPR_PT BPCC_PT
189
190	#define ARITH_ADD (INSN_OP(2) \| INSN_OP3(0x00))
191	#define ARITH_ADDCC (INSN_OP(2) \| INSN_OP3(0x10))
192	#define ARITH_AND (INSN_OP(2) \| INSN_OP3(0x01))
193	#define ARITH_ANDCC (INSN_OP(2) \| INSN_OP3(0x11))
194	#define ARITH_ANDN (INSN_OP(2) \| INSN_OP3(0x05))
195	#define ARITH_OR (INSN_OP(2) \| INSN_OP3(0x02))
196	#define ARITH_ORCC (INSN_OP(2) \| INSN_OP3(0x12))
197	#define ARITH_ORN (INSN_OP(2) \| INSN_OP3(0x06))
198	#define ARITH_XOR (INSN_OP(2) \| INSN_OP3(0x03))
199	#define ARITH_SUB (INSN_OP(2) \| INSN_OP3(0x04))
200	#define ARITH_SUBCC (INSN_OP(2) \| INSN_OP3(0x14))
201	#define ARITH_ADDC (INSN_OP(2) \| INSN_OP3(0x08))
202	#define ARITH_ADDCCC (INSN_OP(2) \| INSN_OP3(0x18))
203	#define ARITH_SUBC (INSN_OP(2) \| INSN_OP3(0x0c))
204	#define ARITH_SUBCCC (INSN_OP(2) \| INSN_OP3(0x1c))
205	#define ARITH_UMUL (INSN_OP(2) \| INSN_OP3(0x0a))
206	#define ARITH_SMUL (INSN_OP(2) \| INSN_OP3(0x0b))
207	#define ARITH_UDIV (INSN_OP(2) \| INSN_OP3(0x0e))
208	#define ARITH_SDIV (INSN_OP(2) \| INSN_OP3(0x0f))
209	#define ARITH_MULX (INSN_OP(2) \| INSN_OP3(0x09))
210	#define ARITH_UDIVX (INSN_OP(2) \| INSN_OP3(0x0d))
211	#define ARITH_SDIVX (INSN_OP(2) \| INSN_OP3(0x2d))
212	#define ARITH_MOVCC (INSN_OP(2) \| INSN_OP3(0x2c))
213	#define ARITH_POPC (INSN_OP(2) \| INSN_OP3(0x2e))
214	#define ARITH_MOVR (INSN_OP(2) \| INSN_OP3(0x2f))
215
216	#define ARITH_ADDXC (INSN_OP(2) \| INSN_OP3(0x36) \| INSN_OPF(0x11))
217	#define ARITH_ADDXCCC (INSN_OP(2) \| INSN_OP3(0x36) \| INSN_OPF(0x13))
218	#define ARITH_UMULXHI (INSN_OP(2) \| INSN_OP3(0x36) \| INSN_OPF(0x16))
219
220	#define SHIFT_SLL (INSN_OP(2) \| INSN_OP3(0x25))
221	#define SHIFT_SRL (INSN_OP(2) \| INSN_OP3(0x26))
222	#define SHIFT_SRA (INSN_OP(2) \| INSN_OP3(0x27))
223
224	#define SHIFT_SLLX (INSN_OP(2) \| INSN_OP3(0x25) \| (1 << 12))
225	#define SHIFT_SRLX (INSN_OP(2) \| INSN_OP3(0x26) \| (1 << 12))
226	#define SHIFT_SRAX (INSN_OP(2) \| INSN_OP3(0x27) \| (1 << 12))
227
228	#define RDY (INSN_OP(2) \| INSN_OP3(0x28) \| INSN_RS1(0))
229	#define WRY (INSN_OP(2) \| INSN_OP3(0x30) \| INSN_RD(0))
230	#define WRCCR (INSN_OP(2) \| INSN_OP3(0x30) \| INSN_RD(2))
231	#define JMPL (INSN_OP(2) \| INSN_OP3(0x38))
232	#define RETURN (INSN_OP(2) \| INSN_OP3(0x39))
233	#define SAVE (INSN_OP(2) \| INSN_OP3(0x3c))
234	#define RESTORE (INSN_OP(2) \| INSN_OP3(0x3d))
235	#define SETHI (INSN_OP(0) \| INSN_OP2(0x4))
236	#define CALL INSN_OP(1)
237	#define LDUB (INSN_OP(3) \| INSN_OP3(0x01))
238	#define LDSB (INSN_OP(3) \| INSN_OP3(0x09))
239	#define LDUH (INSN_OP(3) \| INSN_OP3(0x02))
240	#define LDSH (INSN_OP(3) \| INSN_OP3(0x0a))
241	#define LDUW (INSN_OP(3) \| INSN_OP3(0x00))
242	#define LDSW (INSN_OP(3) \| INSN_OP3(0x08))
243	#define LDX (INSN_OP(3) \| INSN_OP3(0x0b))
244	#define STB (INSN_OP(3) \| INSN_OP3(0x05))
245	#define STH (INSN_OP(3) \| INSN_OP3(0x06))
246	#define STW (INSN_OP(3) \| INSN_OP3(0x04))
247	#define STX (INSN_OP(3) \| INSN_OP3(0x0e))
248	#define LDUBA (INSN_OP(3) \| INSN_OP3(0x11))
249	#define LDSBA (INSN_OP(3) \| INSN_OP3(0x19))
250	#define LDUHA (INSN_OP(3) \| INSN_OP3(0x12))
251	#define LDSHA (INSN_OP(3) \| INSN_OP3(0x1a))
252	#define LDUWA (INSN_OP(3) \| INSN_OP3(0x10))
253	#define LDSWA (INSN_OP(3) \| INSN_OP3(0x18))
254	#define LDXA (INSN_OP(3) \| INSN_OP3(0x1b))
255	#define STBA (INSN_OP(3) \| INSN_OP3(0x15))
256	#define STHA (INSN_OP(3) \| INSN_OP3(0x16))
257	#define STWA (INSN_OP(3) \| INSN_OP3(0x14))
258	#define STXA (INSN_OP(3) \| INSN_OP3(0x1e))
259
260	#define MEMBAR (INSN_OP(2) \| INSN_OP3(0x28) \| INSN_RS1(15) \| (1 << 13))
261
262	#define NOP (SETHI \| INSN_RD(TCG_REG_G0) \| 0)
263
264	#ifndef ASI_PRIMARY_LITTLE
265	#define ASI_PRIMARY_LITTLE 0x88
266	#endif
267
268	#define LDUH_LE (LDUHA \| INSN_ASI(ASI_PRIMARY_LITTLE))
269	#define LDSH_LE (LDSHA \| INSN_ASI(ASI_PRIMARY_LITTLE))
270	#define LDUW_LE (LDUWA \| INSN_ASI(ASI_PRIMARY_LITTLE))
271	#define LDSW_LE (LDSWA \| INSN_ASI(ASI_PRIMARY_LITTLE))
272	#define LDX_LE (LDXA \| INSN_ASI(ASI_PRIMARY_LITTLE))
273
274	#define STH_LE (STHA \| INSN_ASI(ASI_PRIMARY_LITTLE))
275	#define STW_LE (STWA \| INSN_ASI(ASI_PRIMARY_LITTLE))
276	#define STX_LE (STXA \| INSN_ASI(ASI_PRIMARY_LITTLE))
277
278	static bool use_popc_instructions;
279	#if defined(__VIS__) && __VIS__ >= 0x300
280	#define use_vis3_instructions 1
281	#else
282	static bool use_vis3_instructions;
283	#endif
284
285	static bool check_fit_i64(int64_t val, unsigned int bits)
286	{
287	return val == sextract64(val, 0, bits);
288	}
289
290	static bool check_fit_i32(int32_t val, unsigned int bits)
291	{
292	return val == sextract32(val, 0, bits);
293	}
294
295	#define check_fit_tl check_fit_i64
296	#define check_fit_ptr check_fit_i64
297
298	static bool patch_reloc(tcg_insn_unit *src_rw, int type,
299	intptr_t value, intptr_t addend)
300	{
301	const tcg_insn_unit *src_rx = tcg_splitwx_to_rx(src_rw);
302	uint32_t insn = *src_rw;
303	intptr_t pcrel;
304
305	value += addend;
306	pcrel = tcg_ptr_byte_diff((tcg_insn_unit *)value, src_rx);
307
308	switch (type) {
309	case R_SPARC_WDISP16:
310	if (!check_fit_ptr(pcrel >> 2, 16)) {
311	return false;
312	}
313	insn &= ~INSN_OFF16(-1);
314	insn \|= INSN_OFF16(pcrel);
315	break;
316	case R_SPARC_WDISP19:
317	if (!check_fit_ptr(pcrel >> 2, 19)) {
318	return false;
319	}
320	insn &= ~INSN_OFF19(-1);
321	insn \|= INSN_OFF19(pcrel);
322	break;
323	case R_SPARC_13:
324	if (!check_fit_ptr(value, 13)) {
325	return false;
326	}
327	insn &= ~INSN_IMM13(-1);
328	insn \|= INSN_IMM13(value);
329	break;
330	default:
331	g_assert_not_reached();
332	}
333
334	*src_rw = insn;
335	return true;
336	}
337
338	/* test if a constant matches the constraint */
339	static bool tcg_target_const_match(int64_t val, int ct,
340	TCGType type, TCGCond cond, int vece)
341	{
342	if (ct & TCG_CT_CONST) {
343	return 1;
344	}
345
346	if (type == TCG_TYPE_I32) {
347	val = (int32_t)val;
348	}
349
350	if ((ct & TCG_CT_CONST_S11) && check_fit_tl(val, 11)) {
351	return 1;
352	} else if ((ct & TCG_CT_CONST_S13) && check_fit_tl(val, 13)) {
353	return 1;
354	} else {
355	return 0;
356	}
357	}
358
359	static void tcg_out_nop(TCGContext *s)
360	{
361	tcg_out32(s, NOP);
362	}
363
364	static void tcg_out_arith(TCGContext *s, TCGReg rd, TCGReg rs1,
365	TCGReg rs2, int op)
366	{
367	tcg_out32(s, op \| INSN_RD(rd) \| INSN_RS1(rs1) \| INSN_RS2(rs2));
368	}
369
370	static void tcg_out_arithi(TCGContext *s, TCGReg rd, TCGReg rs1,
371	int32_t offset, int op)
372	{
373	tcg_out32(s, op \| INSN_RD(rd) \| INSN_RS1(rs1) \| INSN_IMM13(offset));
374	}
375
376	static void tcg_out_arithc(TCGContext *s, TCGReg rd, TCGReg rs1,
377	int32_t val2, int val2const, int op)
378	{
379	tcg_out32(s, op \| INSN_RD(rd) \| INSN_RS1(rs1)
380	\| (val2const ? INSN_IMM13(val2) : INSN_RS2(val2)));
381	}
382
383	static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg)
384	{
385	if (ret != arg) {
386	tcg_out_arith(s, ret, arg, TCG_REG_G0, ARITH_OR);
387	}
388	return true;
389	}
390
391	static void tcg_out_mov_delay(TCGContext *s, TCGReg ret, TCGReg arg)
392	{
393	if (ret != arg) {
394	tcg_out_arith(s, ret, arg, TCG_REG_G0, ARITH_OR);
395	} else {
396	tcg_out_nop(s);
397	}
398	}
399
400	static void tcg_out_sethi(TCGContext *s, TCGReg ret, uint32_t arg)
401	{
402	tcg_out32(s, SETHI \| INSN_RD(ret) \| ((arg & 0xfffffc00) >> 10));
403	}
404
405	/* A 13-bit constant sign-extended to 64 bits. */
406	static void tcg_out_movi_s13(TCGContext *s, TCGReg ret, int32_t arg)
407	{
408	tcg_out_arithi(s, ret, TCG_REG_G0, arg, ARITH_OR);
409	}
410
411	/* A 32-bit constant sign-extended to 64 bits. */
412	static void tcg_out_movi_s32(TCGContext *s, TCGReg ret, int32_t arg)
413	{
414	tcg_out_sethi(s, ret, ~arg);
415	tcg_out_arithi(s, ret, ret, (arg & 0x3ff) \| -0x400, ARITH_XOR);
416	}
417
418	/* A 32-bit constant zero-extended to 64 bits. */
419	static void tcg_out_movi_u32(TCGContext *s, TCGReg ret, uint32_t arg)
420	{
421	tcg_out_sethi(s, ret, arg);
422	if (arg & 0x3ff) {
423	tcg_out_arithi(s, ret, ret, arg & 0x3ff, ARITH_OR);
424	}
425	}
426
427	static void tcg_out_movi_int(TCGContext *s, TCGType type, TCGReg ret,
428	tcg_target_long arg, bool in_prologue,
429	TCGReg scratch)
430	{
431	tcg_target_long hi, lo = (int32_t)arg;
432	tcg_target_long test, lsb;
433
434	/* A 13-bit constant sign-extended to 64-bits. */
435	if (check_fit_tl(arg, 13)) {
436	tcg_out_movi_s13(s, ret, arg);
437	return;
438	}
439
440	/* A 32-bit constant, or 32-bit zero-extended to 64-bits. */
441	if (type == TCG_TYPE_I32 \|\| arg == (uint32_t)arg) {
442	tcg_out_movi_u32(s, ret, arg);
443	return;
444	}
445
446	/* A 13-bit constant relative to the TB. */
447	if (!in_prologue) {
448	test = tcg_tbrel_diff(s, (void *)arg);
449	if (check_fit_ptr(test, 13)) {
450	tcg_out_arithi(s, ret, TCG_REG_TB, test, ARITH_ADD);
451	return;
452	}
453	}
454
455	/* A 32-bit constant sign-extended to 64-bits. */
456	if (arg == lo) {
457	tcg_out_movi_s32(s, ret, arg);
458	return;
459	}
460
461	/* A 32-bit constant, shifted. */
462	lsb = ctz64(arg);
463	test = (tcg_target_long)arg >> lsb;
464	if (lsb > 10 && test == extract64(test, 0, 21)) {
465	tcg_out_sethi(s, ret, test << 10);
466	tcg_out_arithi(s, ret, ret, lsb - 10, SHIFT_SLLX);
467	return;
468	} else if (test == (uint32_t)test \|\| test == (int32_t)test) {
469	tcg_out_movi_int(s, TCG_TYPE_I64, ret, test, in_prologue, scratch);
470	tcg_out_arithi(s, ret, ret, lsb, SHIFT_SLLX);
471	return;
472	}
473
474	/* Use the constant pool, if possible. */
475	if (!in_prologue) {
476	new_pool_label(s, arg, R_SPARC_13, s->code_ptr,
477	tcg_tbrel_diff(s, NULL));
478	tcg_out32(s, LDX \| INSN_RD(ret) \| INSN_RS1(TCG_REG_TB));
479	return;
480	}
481
482	/* A 64-bit constant decomposed into 2 32-bit pieces. */
483	if (check_fit_i32(lo, 13)) {
484	hi = (arg - lo) >> 32;
485	tcg_out_movi_u32(s, ret, hi);
486	tcg_out_arithi(s, ret, ret, 32, SHIFT_SLLX);
487	tcg_out_arithi(s, ret, ret, lo, ARITH_ADD);
488	} else {
489	hi = arg >> 32;
490	tcg_out_movi_u32(s, ret, hi);
491	tcg_out_movi_u32(s, scratch, lo);
492	tcg_out_arithi(s, ret, ret, 32, SHIFT_SLLX);
493	tcg_out_arith(s, ret, ret, scratch, ARITH_OR);
494	}
495	}
496
497	static void tcg_out_movi(TCGContext *s, TCGType type,
498	TCGReg ret, tcg_target_long arg)
499	{
500	tcg_debug_assert(ret != TCG_REG_T3);
501	tcg_out_movi_int(s, type, ret, arg, false, TCG_REG_T3);
502	}
503
504	static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
505	{
506	g_assert_not_reached();
507	}
508
509	static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg rd, TCGReg rs)
510	{
511	g_assert_not_reached();
512	}
513
514	static void tcg_out_ext8u(TCGContext *s, TCGReg rd, TCGReg rs)
515	{
516	tcg_out_arithi(s, rd, rs, 0xff, ARITH_AND);
517	}
518
519	static void tcg_out_ext16u(TCGContext *s, TCGReg rd, TCGReg rs)
520	{
521	tcg_out_arithi(s, rd, rs, 16, SHIFT_SLL);
522	tcg_out_arithi(s, rd, rd, 16, SHIFT_SRL);
523	}
524
525	static void tcg_out_ext32s(TCGContext *s, TCGReg rd, TCGReg rs)
526	{
527	tcg_out_arithi(s, rd, rs, 0, SHIFT_SRA);
528	}
529
530	static void tcg_out_ext32u(TCGContext *s, TCGReg rd, TCGReg rs)
531	{
532	tcg_out_arithi(s, rd, rs, 0, SHIFT_SRL);
533	}
534
535	static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
536	{
537	tcg_out_ext32s(s, rd, rs);
538	}
539
540	static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg rd, TCGReg rs)
541	{
542	tcg_out_ext32u(s, rd, rs);
543	}
544
545	static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg rd, TCGReg rs)
546	{
547	tcg_out_ext32u(s, rd, rs);
548	}
549
550	static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2)
551	{
552	return false;
553	}
554
555	static void tcg_out_addi_ptr(TCGContext *s, TCGReg rd, TCGReg rs,
556	tcg_target_long imm)
557	{
558	/* This function is only used for passing structs by reference. */
559	g_assert_not_reached();
560	}
561
562	static void tcg_out_ldst_rr(TCGContext *s, TCGReg data, TCGReg a1,
563	TCGReg a2, int op)
564	{
565	tcg_out32(s, op \| INSN_RD(data) \| INSN_RS1(a1) \| INSN_RS2(a2));
566	}
567
568	static void tcg_out_ldst(TCGContext *s, TCGReg ret, TCGReg addr,
569	intptr_t offset, int op)
570	{
571	if (check_fit_ptr(offset, 13)) {
572	tcg_out32(s, op \| INSN_RD(ret) \| INSN_RS1(addr) \|
573	INSN_IMM13(offset));
574	} else {
575	tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, offset);
576	tcg_out_ldst_rr(s, ret, addr, TCG_REG_T1, op);
577	}
578	}
579
580	static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
581	TCGReg arg1, intptr_t arg2)
582	{
583	tcg_out_ldst(s, ret, arg1, arg2, (type == TCG_TYPE_I32 ? LDUW : LDX));
584	}
585
586	static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
587	TCGReg arg1, intptr_t arg2)
588	{
589	tcg_out_ldst(s, arg, arg1, arg2, (type == TCG_TYPE_I32 ? STW : STX));
590	}
591
592	static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
593	TCGReg base, intptr_t ofs)
594	{
595	if (val == 0) {
596	tcg_out_st(s, type, TCG_REG_G0, base, ofs);
597	return true;
598	}
599	return false;
600	}
601
602	static void tcg_out_sety(TCGContext *s, TCGReg rs)
603	{
604	tcg_out32(s, WRY \| INSN_RS1(TCG_REG_G0) \| INSN_RS2(rs));
605	}
606
607	static const uint8_t tcg_cond_to_bcond[16] = {
608	[TCG_COND_EQ] = COND_E,
609	[TCG_COND_NE] = COND_NE,
610	[TCG_COND_TSTEQ] = COND_E,
611	[TCG_COND_TSTNE] = COND_NE,
612	[TCG_COND_LT] = COND_L,
613	[TCG_COND_GE] = COND_GE,
614	[TCG_COND_LE] = COND_LE,
615	[TCG_COND_GT] = COND_G,
616	[TCG_COND_LTU] = COND_CS,
617	[TCG_COND_GEU] = COND_CC,
618	[TCG_COND_LEU] = COND_LEU,
619	[TCG_COND_GTU] = COND_GU,
620	};
621
622	static const uint8_t tcg_cond_to_rcond[16] = {
623	[TCG_COND_EQ] = RCOND_Z,
624	[TCG_COND_NE] = RCOND_NZ,
625	[TCG_COND_LT] = RCOND_LZ,
626	[TCG_COND_GT] = RCOND_GZ,
627	[TCG_COND_LE] = RCOND_LEZ,
628	[TCG_COND_GE] = RCOND_GEZ
629	};
630
631	static void tcg_out_bpcc0(TCGContext *s, int scond, int flags, int off19)
632	{
633	tcg_out32(s, INSN_OP(0) \| INSN_OP2(1) \| INSN_COND(scond) \| flags \| off19);
634	}
635
636	static void tcg_out_bpcc(TCGContext s, int scond, int flags, TCGLabel l)
637	{
638	int off19 = 0;
639
640	if (l->has_value) {
641	off19 = INSN_OFF19(tcg_pcrel_diff(s, l->u.value_ptr));
642	} else {
643	tcg_out_reloc(s, s->code_ptr, R_SPARC_WDISP19, l, 0);
644	}
645	tcg_out_bpcc0(s, scond, flags, off19);
646	}
647
648	static void tcg_out_br(TCGContext s, TCGLabel l)
649	{
650	tcg_out_bpcc(s, COND_A, BPCC_PT, l);
651	tcg_out_nop(s);
652	}
653
654	static void tcg_out_cmp(TCGContext *s, TCGCond cond,
655	TCGReg c1, int32_t c2, int c2const)
656	{
657	tcg_out_arithc(s, TCG_REG_G0, c1, c2, c2const,
658	is_tst_cond(cond) ? ARITH_ANDCC : ARITH_SUBCC);
659	}
660
661	static void tcg_out_brcond_i32(TCGContext *s, TCGCond cond, TCGReg arg1,
662	int32_t arg2, int const_arg2, TCGLabel *l)
663	{
664	tcg_out_cmp(s, cond, arg1, arg2, const_arg2);
665	tcg_out_bpcc(s, tcg_cond_to_bcond[cond], BPCC_ICC \| BPCC_PT, l);
666	tcg_out_nop(s);
667	}
668
669	static void tcg_out_movcc(TCGContext *s, int scond, int cc, TCGReg ret,
670	int32_t v1, int v1const)
671	{
672	tcg_out32(s, ARITH_MOVCC \| cc \| INSN_RD(ret) \| INSN_RS1(scond)
673	\| (v1const ? INSN_IMM11(v1) : INSN_RS2(v1)));
674	}
675
676	static void tcg_out_movcond_i32(TCGContext *s, TCGCond cond, TCGReg ret,
677	TCGReg c1, int32_t c2, int c2const,
678	int32_t v1, int v1const)
679	{
680	tcg_out_cmp(s, cond, c1, c2, c2const);
681	tcg_out_movcc(s, tcg_cond_to_bcond[cond], MOVCC_ICC, ret, v1, v1const);
682	}
683
684	static void tcg_out_brcond_i64(TCGContext *s, TCGCond cond, TCGReg arg1,
685	int32_t arg2, int const_arg2, TCGLabel *l)
686	{
687	/* For 64-bit signed comparisons vs zero, we can avoid the compare. */
688	int rcond = tcg_cond_to_rcond[cond];
689	if (arg2 == 0 && rcond) {
690	int off16 = 0;
691
692	if (l->has_value) {
693	off16 = INSN_OFF16(tcg_pcrel_diff(s, l->u.value_ptr));
694	} else {
695	tcg_out_reloc(s, s->code_ptr, R_SPARC_WDISP16, l, 0);
696	}
697	tcg_out32(s, INSN_OP(0) \| INSN_OP2(3) \| BPR_PT \| INSN_RS1(arg1)
698	\| INSN_COND(rcond) \| off16);
699	} else {
700	tcg_out_cmp(s, cond, arg1, arg2, const_arg2);
701	tcg_out_bpcc(s, tcg_cond_to_bcond[cond], BPCC_XCC \| BPCC_PT, l);
702	}
703	tcg_out_nop(s);
704	}
705
706	static void tcg_out_movr(TCGContext *s, int rcond, TCGReg ret, TCGReg c1,
707	int32_t v1, int v1const)
708	{
709	tcg_out32(s, ARITH_MOVR \| INSN_RD(ret) \| INSN_RS1(c1) \| (rcond << 10)
710	\| (v1const ? INSN_IMM10(v1) : INSN_RS2(v1)));
711	}
712
713	static void tcg_out_movcond_i64(TCGContext *s, TCGCond cond, TCGReg ret,
714	TCGReg c1, int32_t c2, int c2const,
715	int32_t v1, int v1const)
716	{
717	/* For 64-bit signed comparisons vs zero, we can avoid the compare.
718	Note that the immediate range is one bit smaller, so we must check
719	for that as well. */
720	int rcond = tcg_cond_to_rcond[cond];
721	if (c2 == 0 && rcond && (!v1const \|\| check_fit_i32(v1, 10))) {
722	tcg_out_movr(s, rcond, ret, c1, v1, v1const);
723	} else {
724	tcg_out_cmp(s, cond, c1, c2, c2const);
725	tcg_out_movcc(s, tcg_cond_to_bcond[cond], MOVCC_XCC, ret, v1, v1const);
726	}
727	}
728
729	static void tcg_out_setcond_i32(TCGContext *s, TCGCond cond, TCGReg ret,
730	TCGReg c1, int32_t c2, bool c2const, bool neg)
731	{
732	/* For 32-bit comparisons, we can play games with ADDC/SUBC. */
733	switch (cond) {
734	case TCG_COND_LTU:
735	case TCG_COND_GEU:
736	/* The result of the comparison is in the carry bit. */
737	break;
738
739	case TCG_COND_EQ:
740	case TCG_COND_NE:
741	/* For equality, we can transform to inequality vs zero. */
742	if (c2 != 0) {
743	tcg_out_arithc(s, TCG_REG_T1, c1, c2, c2const, ARITH_XOR);
744	c2 = TCG_REG_T1;
745	} else {
746	c2 = c1;
747	}
748	c1 = TCG_REG_G0, c2const = 0;
749	cond = (cond == TCG_COND_EQ ? TCG_COND_GEU : TCG_COND_LTU);
750	break;
751
752	case TCG_COND_TSTEQ:
753	case TCG_COND_TSTNE:
754	/* Transform to inequality vs zero. */
755	tcg_out_arithc(s, TCG_REG_T1, c1, c2, c2const, ARITH_AND);
756	c1 = TCG_REG_G0;
757	c2 = TCG_REG_T1, c2const = 0;
758	cond = (cond == TCG_COND_TSTEQ ? TCG_COND_GEU : TCG_COND_LTU);
759	break;
760
761	case TCG_COND_GTU:
762	case TCG_COND_LEU:
763	/* If we don't need to load a constant into a register, we can
764	swap the operands on GTU/LEU. There's no benefit to loading
765	the constant into a temporary register. */
766	if (!c2const \|\| c2 == 0) {
767	TCGReg t = c1;
768	c1 = c2;
769	c2 = t;
770	c2const = 0;
771	cond = tcg_swap_cond(cond);
772	break;
773	}
774	/* FALLTHRU */
775
776	default:
777	tcg_out_cmp(s, cond, c1, c2, c2const);
778	tcg_out_movi_s13(s, ret, 0);
779	tcg_out_movcc(s, tcg_cond_to_bcond[cond],
780	MOVCC_ICC, ret, neg ? -1 : 1, 1);
781	return;
782	}
783
784	tcg_out_cmp(s, cond, c1, c2, c2const);
785	if (cond == TCG_COND_LTU) {
786	if (neg) {
787	/* 0 - 0 - C = -C = (C ? -1 : 0) */
788	tcg_out_arithi(s, ret, TCG_REG_G0, 0, ARITH_SUBC);
789	} else {
790	/* 0 + 0 + C = C = (C ? 1 : 0) */
791	tcg_out_arithi(s, ret, TCG_REG_G0, 0, ARITH_ADDC);
792	}
793	} else {
794	if (neg) {
795	/* 0 + -1 + C = C - 1 = (C ? 0 : -1) */
796	tcg_out_arithi(s, ret, TCG_REG_G0, -1, ARITH_ADDC);
797	} else {
798	/* 0 - -1 - C = 1 - C = (C ? 0 : 1) */
799	tcg_out_arithi(s, ret, TCG_REG_G0, -1, ARITH_SUBC);
800	}
801	}
802	}
803
804	static void tcg_out_setcond_i64(TCGContext *s, TCGCond cond, TCGReg ret,
805	TCGReg c1, int32_t c2, bool c2const, bool neg)
806	{
807	int rcond;
808
809	if (use_vis3_instructions && !neg) {
810	switch (cond) {
811	case TCG_COND_NE:
812	if (c2 != 0) {
813	break;
814	}
815	c2 = c1, c2const = 0, c1 = TCG_REG_G0;
816	/* FALLTHRU */
817	case TCG_COND_LTU:
818	tcg_out_cmp(s, cond, c1, c2, c2const);
819	tcg_out_arith(s, ret, TCG_REG_G0, TCG_REG_G0, ARITH_ADDXC);
820	return;
821	default:
822	break;
823	}
824	}
825
826	/* For 64-bit signed comparisons vs zero, we can avoid the compare
827	if the input does not overlap the output. */
828	rcond = tcg_cond_to_rcond[cond];
829	if (c2 == 0 && rcond && c1 != ret) {
830	tcg_out_movi_s13(s, ret, 0);
831	tcg_out_movr(s, rcond, ret, c1, neg ? -1 : 1, 1);
832	} else {
833	tcg_out_cmp(s, cond, c1, c2, c2const);
834	tcg_out_movi_s13(s, ret, 0);
835	tcg_out_movcc(s, tcg_cond_to_bcond[cond],
836	MOVCC_XCC, ret, neg ? -1 : 1, 1);
837	}
838	}
839
840	static void tcg_out_brcond(TCGContext *s, TCGType type, TCGCond cond,
841	TCGReg arg1, TCGArg arg2, bool const_arg2,
842	TCGLabel *l)
843	{
844	if (type == TCG_TYPE_I32) {
845	tcg_out_brcond_i32(s, cond, arg1, arg2, const_arg2, l);
846	} else {
847	tcg_out_brcond_i64(s, cond, arg1, arg2, const_arg2, l);
848	}
849	}
850
851	static void tgen_brcond(TCGContext *s, TCGType type, TCGCond cond,
852	TCGReg arg1, TCGReg arg2, TCGLabel *l)
853	{
854	tcg_out_brcond(s, type, cond, arg1, arg2, false, l);
855	}
856
857	static void tgen_brcondi(TCGContext *s, TCGType type, TCGCond cond,
858	TCGReg arg1, tcg_target_long arg2, TCGLabel *l)
859	{
860	tcg_out_brcond(s, type, cond, arg1, arg2, true, l);
861	}
862
863	static const TCGOutOpBrcond outop_brcond = {
864	.base.static_constraint = C_O0_I2(r, rJ),
865	.out_rr = tgen_brcond,
866	.out_ri = tgen_brcondi,
867	};
868
869	static void tcg_out_setcond(TCGContext *s, TCGType type, TCGCond cond,
870	TCGReg ret, TCGReg c1,
871	TCGArg c2, bool c2const, bool neg)
872	{
873	if (type == TCG_TYPE_I32) {
874	tcg_out_setcond_i32(s, cond, ret, c1, c2, c2const, neg);
875	} else {
876	tcg_out_setcond_i64(s, cond, ret, c1, c2, c2const, neg);
877	}
878	}
879
880	static void tgen_setcond(TCGContext *s, TCGType type, TCGCond cond,
881	TCGReg dest, TCGReg arg1, TCGReg arg2)
882	{
883	tcg_out_setcond(s, type, cond, dest, arg1, arg2, false, false);
884	}
885
886	static void tgen_setcondi(TCGContext *s, TCGType type, TCGCond cond,
887	TCGReg dest, TCGReg arg1, tcg_target_long arg2)
888	{
889	tcg_out_setcond(s, type, cond, dest, arg1, arg2, true, false);
890	}
891
892	static const TCGOutOpSetcond outop_setcond = {
893	.base.static_constraint = C_O1_I2(r, r, rJ),
894	.out_rrr = tgen_setcond,
895	.out_rri = tgen_setcondi,
896	};
897
898	static void tgen_negsetcond(TCGContext *s, TCGType type, TCGCond cond,
899	TCGReg dest, TCGReg arg1, TCGReg arg2)
900	{
901	tcg_out_setcond(s, type, cond, dest, arg1, arg2, false, true);
902	}
903
904	static void tgen_negsetcondi(TCGContext *s, TCGType type, TCGCond cond,
905	TCGReg dest, TCGReg arg1, tcg_target_long arg2)
906	{
907	tcg_out_setcond(s, type, cond, dest, arg1, arg2, true, true);
908	}
909
910	static const TCGOutOpSetcond outop_negsetcond = {
911	.base.static_constraint = C_O1_I2(r, r, rJ),
912	.out_rrr = tgen_negsetcond,
913	.out_rri = tgen_negsetcondi,
914	};
915
916	static void tgen_movcond(TCGContext *s, TCGType type, TCGCond cond,
917	TCGReg ret, TCGReg c1, TCGArg c2, bool c2const,
918	TCGArg v1, bool v1const, TCGArg v2, bool v2consf)
919	{
920	if (type == TCG_TYPE_I32) {
921	tcg_out_movcond_i32(s, cond, ret, c1, c2, c2const, v1, v1const);
922	} else {
923	tcg_out_movcond_i64(s, cond, ret, c1, c2, c2const, v1, v1const);
924	}
925	}
926
927	static const TCGOutOpMovcond outop_movcond = {
928	.base.static_constraint = C_O1_I4(r, r, rJ, rI, 0),
929	.out = tgen_movcond,
930	};
931
932	static void tcg_out_jmpl_const(TCGContext s, const tcg_insn_unit dest,
933	bool in_prologue, bool tail_call)
934	{
935	uintptr_t desti = (uintptr_t)dest;
936
937	tcg_out_movi_int(s, TCG_TYPE_PTR, TCG_REG_T1,
938	desti & ~0xfff, in_prologue, TCG_REG_T2);
939	tcg_out_arithi(s, tail_call ? TCG_REG_G0 : TCG_REG_O7,
940	TCG_REG_T1, desti & 0xfff, JMPL);
941	}
942
943	static void tcg_out_call_nodelay(TCGContext s, const tcg_insn_unit dest,
944	bool in_prologue)
945	{
946	ptrdiff_t disp = tcg_pcrel_diff(s, dest);
947
948	if (disp == (int32_t)disp) {
949	tcg_out32(s, CALL \| (uint32_t)disp >> 2);
950	} else {
951	tcg_out_jmpl_const(s, dest, in_prologue, false);
952	}
953	}
954
955	static void tcg_out_call(TCGContext s, const tcg_insn_unit dest,
956	const TCGHelperInfo *info)
957	{
958	tcg_out_call_nodelay(s, dest, false);
959	tcg_out_nop(s);
960	}
961
962	static void tcg_out_mb(TCGContext *s, unsigned a0)
963	{
964	/* Note that the TCG memory order constants mirror the Sparc MEMBAR. */
965	tcg_out32(s, MEMBAR \| (a0 & TCG_MO_ALL));
966	}
967
968	/* Generate global QEMU prologue and epilogue code */
969	static void tcg_target_qemu_prologue(TCGContext *s)
970	{
971	int tmp_buf_size, frame_size;
972
973	/*
974	* The TCG temp buffer is at the top of the frame, immediately
975	* below the frame pointer. Use the logical (aligned) offset here;
976	* the stack bias is applied in temp_allocate_frame().
977	*/
978	tmp_buf_size = CPU_TEMP_BUF_NLONGS * (int)sizeof(long);
979	tcg_set_frame(s, TCG_REG_I6, -tmp_buf_size, tmp_buf_size);
980
981	/*
982	* TCG_TARGET_CALL_STACK_OFFSET includes the stack bias, but is
983	* otherwise the minimal frame usable by callees.
984	*/
985	frame_size = TCG_TARGET_CALL_STACK_OFFSET - TCG_TARGET_STACK_BIAS;
986	frame_size += TCG_STATIC_CALL_ARGS_SIZE + tmp_buf_size;
987	frame_size += TCG_TARGET_STACK_ALIGN - 1;
988	frame_size &= -TCG_TARGET_STACK_ALIGN;
989	tcg_out32(s, SAVE \| INSN_RD(TCG_REG_O6) \| INSN_RS1(TCG_REG_O6) \|
990	INSN_IMM13(-frame_size));
991
992	#ifndef CONFIG_SOFTMMU
993	if (guest_base != 0) {
994	tcg_out_movi_int(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG,
995	guest_base, true, TCG_REG_T1);
996	tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
997	}
998	#endif
999
1000	/* We choose TCG_REG_TB such that no move is required. */
1001	QEMU_BUILD_BUG_ON(TCG_REG_TB != TCG_REG_I1);
1002	tcg_regset_set_reg(s->reserved_regs, TCG_REG_TB);
1003
1004	tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I1, 0, JMPL);
1005	/* delay slot */
1006	tcg_out_nop(s);
1007
1008	/* Epilogue for goto_ptr. */
1009	tcg_code_gen_epilogue = tcg_splitwx_to_rx(s->code_ptr);
1010	tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
1011	/* delay slot */
1012	tcg_out_movi_s13(s, TCG_REG_O0, 0);
1013	}
1014
1015	static void tcg_out_tb_start(TCGContext *s)
1016	{
1017	/* nothing to do */
1018	}
1019
1020	static void tcg_out_nop_fill(tcg_insn_unit *p, int count)
1021	{
1022	int i;
1023	for (i = 0; i < count; ++i) {
1024	p[i] = NOP;
1025	}
1026	}
1027
1028	static const TCGLdstHelperParam ldst_helper_param = {
1029	.ntmp = 1, .tmp = { TCG_REG_T1 }
1030	};
1031
1032	static bool tcg_out_qemu_ld_slow_path(TCGContext s, TCGLabelQemuLdst lb)
1033	{
1034	MemOp opc = get_memop(lb->oi);
1035	MemOp sgn;
1036
1037	if (!patch_reloc(lb->label_ptr[0], R_SPARC_WDISP19,
1038	(intptr_t)tcg_splitwx_to_rx(s->code_ptr), 0)) {
1039	return false;
1040	}
1041
1042	/* Use inline tcg_out_ext32s; otherwise let the helper sign-extend. */
1043	sgn = (opc & MO_SIZE) < MO_32 ? MO_SIGN : 0;
1044
1045	tcg_out_ld_helper_args(s, lb, &ldst_helper_param);
1046	tcg_out_call(s, qemu_ld_helpers[opc & (MO_SIZE \| sgn)], NULL);
1047	tcg_out_ld_helper_ret(s, lb, sgn, &ldst_helper_param);
1048
1049	tcg_out_bpcc0(s, COND_A, BPCC_A \| BPCC_PT, 0);
1050	return patch_reloc(s->code_ptr - 1, R_SPARC_WDISP19,
1051	(intptr_t)lb->raddr, 0);
1052	}
1053
1054	static bool tcg_out_qemu_st_slow_path(TCGContext s, TCGLabelQemuLdst lb)
1055	{
1056	MemOp opc = get_memop(lb->oi);
1057
1058	if (!patch_reloc(lb->label_ptr[0], R_SPARC_WDISP19,
1059	(intptr_t)tcg_splitwx_to_rx(s->code_ptr), 0)) {
1060	return false;
1061	}
1062
1063	tcg_out_st_helper_args(s, lb, &ldst_helper_param);
1064	tcg_out_call(s, qemu_st_helpers[opc & MO_SIZE], NULL);
1065
1066	tcg_out_bpcc0(s, COND_A, BPCC_A \| BPCC_PT, 0);
1067	return patch_reloc(s->code_ptr - 1, R_SPARC_WDISP19,
1068	(intptr_t)lb->raddr, 0);
1069	}
1070
1071	typedef struct {
1072	TCGReg base;
1073	TCGReg index;
1074	TCGAtomAlign aa;
1075	} HostAddress;
1076
1077	bool tcg_target_has_memory_bswap(MemOp memop)
1078	{
1079	return true;
1080	}
1081
1082	/* We expect to use a 13-bit negative offset from ENV. */
1083	#define MIN_TLB_MASK_TABLE_OFS -(1 << 12)
1084
1085	/*
1086	* For system-mode, perform the TLB load and compare.
1087	* For user-mode, perform any required alignment tests.
1088	* In both cases, return a TCGLabelQemuLdst structure if the slow path
1089	* is required and fill in @h with the host address for the fast path.
1090	*/
1091	static TCGLabelQemuLdst prepare_host_addr(TCGContext s, HostAddress *h,
1092	TCGReg addr_reg, MemOpIdx oi,
1093	bool is_ld)
1094	{
1095	TCGType addr_type = s->addr_type;
1096	TCGLabelQemuLdst *ldst = NULL;
1097	MemOp opc = get_memop(oi);
1098	MemOp s_bits = opc & MO_SIZE;
1099	unsigned a_mask;
1100
1101	/* We don't support unaligned accesses. */
1102	h->aa = atom_and_align_for_opc(s, opc, MO_ATOM_IFALIGN, false);
1103	h->aa.align = MAX(h->aa.align, s_bits);
1104	a_mask = (1u << h->aa.align) - 1;
1105
1106	#ifdef CONFIG_SOFTMMU
1107	int mem_index = get_mmuidx(oi);
1108	int fast_off = tlb_mask_table_ofs(s, mem_index);
1109	int mask_off = fast_off + offsetof(CPUTLBDescFast, mask);
1110	int table_off = fast_off + offsetof(CPUTLBDescFast, table);
1111	int cmp_off = is_ld ? offsetof(CPUTLBEntry, addr_read)
1112	: offsetof(CPUTLBEntry, addr_write);
1113	int add_off = offsetof(CPUTLBEntry, addend);
1114	int compare_mask;
1115	int cc;
1116
1117	/* Load tlb_mask[mmu_idx] and tlb_table[mmu_idx]. */
1118	tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_T2, TCG_AREG0, mask_off);
1119	tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_T3, TCG_AREG0, table_off);
1120
1121	/* Extract the page index, shifted into place for tlb index. */
1122	tcg_out_arithi(s, TCG_REG_T1, addr_reg,
1123	TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS, SHIFT_SRL);
1124	tcg_out_arith(s, TCG_REG_T1, TCG_REG_T1, TCG_REG_T2, ARITH_AND);
1125
1126	/* Add the tlb_table pointer, creating the CPUTLBEntry address into R2. */
1127	tcg_out_arith(s, TCG_REG_T1, TCG_REG_T1, TCG_REG_T3, ARITH_ADD);
1128
1129	/*
1130	* Load the tlb comparator and the addend.
1131	* Always load the entire 64-bit comparator for simplicity.
1132	* We will ignore the high bits via BPCC_ICC below.
1133	*/
1134	tcg_out_ld(s, TCG_TYPE_I64, TCG_REG_T2, TCG_REG_T1, cmp_off);
1135	tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_T1, TCG_REG_T1, add_off);
1136	h->base = TCG_REG_T1;
1137
1138	/* Mask out the page offset, except for the required alignment. */
1139	compare_mask = TARGET_PAGE_MASK \| a_mask;
1140	if (check_fit_tl(compare_mask, 13)) {
1141	tcg_out_arithi(s, TCG_REG_T3, addr_reg, compare_mask, ARITH_AND);
1142	} else {
1143	tcg_out_movi_s32(s, TCG_REG_T3, compare_mask);
1144	tcg_out_arith(s, TCG_REG_T3, addr_reg, TCG_REG_T3, ARITH_AND);
1145	}
1146	tcg_out_cmp(s, TCG_COND_NE, TCG_REG_T2, TCG_REG_T3, 0);
1147
1148	ldst = new_ldst_label(s);
1149	ldst->is_ld = is_ld;
1150	ldst->oi = oi;
1151	ldst->addr_reg = addr_reg;
1152	ldst->label_ptr[0] = s->code_ptr;
1153
1154	/* bne,pn %[xi]cc, label0 */
1155	cc = addr_type == TCG_TYPE_I32 ? BPCC_ICC : BPCC_XCC;
1156	tcg_out_bpcc0(s, COND_NE, BPCC_PN \| cc, 0);
1157	#else
1158	/*
1159	* If the size equals the required alignment, we can skip the test
1160	* and allow host SIGBUS to deliver SIGBUS to the guest.
1161	* Otherwise, test for at least natural alignment and defer
1162	* everything else to the helper functions.
1163	*/
1164	if (s_bits != memop_alignment_bits(opc)) {
1165	tcg_debug_assert(check_fit_tl(a_mask, 13));
1166	tcg_out_arithi(s, TCG_REG_G0, addr_reg, a_mask, ARITH_ANDCC);
1167
1168	ldst = new_ldst_label(s);
1169	ldst->is_ld = is_ld;
1170	ldst->oi = oi;
1171	ldst->addr_reg = addr_reg;
1172	ldst->label_ptr[0] = s->code_ptr;
1173
1174	/* bne,pn %icc, label0 */
1175	tcg_out_bpcc0(s, COND_NE, BPCC_PN \| BPCC_ICC, 0);
1176	}
1177	h->base = guest_base ? TCG_GUEST_BASE_REG : TCG_REG_G0;
1178	#endif
1179
1180	/* If the guest address must be zero-extended, do in the delay slot. */
1181	if (addr_type == TCG_TYPE_I32) {
1182	tcg_out_ext32u(s, TCG_REG_T2, addr_reg);
1183	h->index = TCG_REG_T2;
1184	} else {
1185	if (ldst) {
1186	tcg_out_nop(s);
1187	}
1188	h->index = addr_reg;
1189	}
1190	return ldst;
1191	}
1192
1193	static void tgen_qemu_ld(TCGContext *s, TCGType type, TCGReg data,
1194	TCGReg addr, MemOpIdx oi)
1195	{
1196	static const int ld_opc[(MO_SSIZE \| MO_BSWAP) + 1] = {
1197	[MO_UB] = LDUB,
1198	[MO_SB] = LDSB,
1199	[MO_UB \| MO_LE] = LDUB,
1200	[MO_SB \| MO_LE] = LDSB,
1201
1202	[MO_BEUW] = LDUH,
1203	[MO_BESW] = LDSH,
1204	[MO_BEUL] = LDUW,
1205	[MO_BESL] = LDSW,
1206	[MO_BEUQ] = LDX,
1207	[MO_BESQ] = LDX,
1208
1209	[MO_LEUW] = LDUH_LE,
1210	[MO_LESW] = LDSH_LE,
1211	[MO_LEUL] = LDUW_LE,
1212	[MO_LESL] = LDSW_LE,
1213	[MO_LEUQ] = LDX_LE,
1214	[MO_LESQ] = LDX_LE,
1215	};
1216
1217	TCGLabelQemuLdst *ldst;
1218	HostAddress h;
1219
1220	ldst = prepare_host_addr(s, &h, addr, oi, true);
1221
1222	tcg_out_ldst_rr(s, data, h.base, h.index,
1223	ld_opc[get_memop(oi) & (MO_BSWAP \| MO_SSIZE)]);
1224
1225	if (ldst) {
1226	ldst->type = type;
1227	ldst->datalo_reg = data;
1228	ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
1229	}
1230	}
1231
1232	static const TCGOutOpQemuLdSt outop_qemu_ld = {
1233	.base.static_constraint = C_O1_I1(r, r),
1234	.out = tgen_qemu_ld,
1235	};
1236
1237	static const TCGOutOpQemuLdSt2 outop_qemu_ld2 = {
1238	.base.static_constraint = C_NotImplemented,
1239	};
1240
1241	static void tgen_qemu_st(TCGContext *s, TCGType type, TCGReg data,
1242	TCGReg addr, MemOpIdx oi)
1243	{
1244	static const int st_opc[(MO_SIZE \| MO_BSWAP) + 1] = {
1245	[MO_UB] = STB,
1246
1247	[MO_BEUW] = STH,
1248	[MO_BEUL] = STW,
1249	[MO_BEUQ] = STX,
1250
1251	[MO_LEUW] = STH_LE,
1252	[MO_LEUL] = STW_LE,
1253	[MO_LEUQ] = STX_LE,
1254	};
1255
1256	TCGLabelQemuLdst *ldst;
1257	HostAddress h;
1258
1259	ldst = prepare_host_addr(s, &h, addr, oi, false);
1260
1261	tcg_out_ldst_rr(s, data, h.base, h.index,
1262	st_opc[get_memop(oi) & (MO_BSWAP \| MO_SIZE)]);
1263
1264	if (ldst) {
1265	ldst->type = type;
1266	ldst->datalo_reg = data;
1267	ldst->raddr = tcg_splitwx_to_rx(s->code_ptr);
1268	}
1269	}
1270
1271	static const TCGOutOpQemuLdSt outop_qemu_st = {
1272	.base.static_constraint = C_O0_I2(rz, r),
1273	.out = tgen_qemu_st,
1274	};
1275
1276	static const TCGOutOpQemuLdSt2 outop_qemu_st2 = {
1277	.base.static_constraint = C_NotImplemented,
1278	};
1279
1280	static void tcg_out_exit_tb(TCGContext *s, uintptr_t a0)
1281	{
1282	if (check_fit_ptr(a0, 13)) {
1283	tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
1284	tcg_out_movi_s13(s, TCG_REG_O0, a0);
1285	return;
1286	} else {
1287	intptr_t tb_diff = tcg_tbrel_diff(s, (void *)a0);
1288	if (check_fit_ptr(tb_diff, 13)) {
1289	tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
1290	/* Note that TCG_REG_TB has been unwound to O1. */
1291	tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O1, tb_diff, ARITH_ADD);
1292	return;
1293	}
1294	}
1295	tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_I0, a0 & ~0x3ff);
1296	tcg_out_arithi(s, TCG_REG_G0, TCG_REG_I7, 8, RETURN);
1297	tcg_out_arithi(s, TCG_REG_O0, TCG_REG_O0, a0 & 0x3ff, ARITH_OR);
1298	}
1299
1300	static void tcg_out_goto_tb(TCGContext *s, int which)
1301	{
1302	ptrdiff_t off = tcg_tbrel_diff(s, (void *)get_jmp_target_addr(s, which));
1303
1304	/* Load link and indirect branch. */
1305	set_jmp_insn_offset(s, which);
1306	tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_TB, TCG_REG_TB, off);
1307	tcg_out_arithi(s, TCG_REG_G0, TCG_REG_TB, 0, JMPL);
1308	/* delay slot */
1309	tcg_out_nop(s);
1310	set_jmp_reset_offset(s, which);
1311
1312	/*
1313	* For the unlinked path of goto_tb, we need to reset TCG_REG_TB
1314	* to the beginning of this TB.
1315	*/
1316	off = -tcg_current_code_size(s);
1317	if (check_fit_i32(off, 13)) {
1318	tcg_out_arithi(s, TCG_REG_TB, TCG_REG_TB, off, ARITH_ADD);
1319	} else {
1320	tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, off);
1321	tcg_out_arith(s, TCG_REG_TB, TCG_REG_TB, TCG_REG_T1, ARITH_ADD);
1322	}
1323	}
1324
1325	static void tcg_out_goto_ptr(TCGContext *s, TCGReg a0)
1326	{
1327	tcg_out_arithi(s, TCG_REG_G0, a0, 0, JMPL);
1328	tcg_out_mov_delay(s, TCG_REG_TB, a0);
1329	}
1330
1331	void tb_target_set_jmp_target(const TranslationBlock *tb, int n,
1332	uintptr_t jmp_rx, uintptr_t jmp_rw)
1333	{
1334	}
1335
1336
1337	static void tgen_add(TCGContext *s, TCGType type,
1338	TCGReg a0, TCGReg a1, TCGReg a2)
1339	{
1340	tcg_out_arith(s, a0, a1, a2, ARITH_ADD);
1341	}
1342
1343	static void tgen_addi(TCGContext *s, TCGType type,
1344	TCGReg a0, TCGReg a1, tcg_target_long a2)
1345	{
1346	tcg_out_arithi(s, a0, a1, a2, ARITH_ADD);
1347	}
1348
1349	static const TCGOutOpBinary outop_add = {
1350	.base.static_constraint = C_O1_I2(r, r, rJ),
1351	.out_rrr = tgen_add,
1352	.out_rri = tgen_addi,
1353	};
1354
1355	static void tgen_addco_rrr(TCGContext *s, TCGType type,
1356	TCGReg a0, TCGReg a1, TCGReg a2)
1357	{
1358	tcg_out_arith(s, a0, a1, a2, ARITH_ADDCC);
1359	}
1360
1361	static void tgen_addco_rri(TCGContext *s, TCGType type,
1362	TCGReg a0, TCGReg a1, tcg_target_long a2)
1363	{
1364	tcg_out_arithi(s, a0, a1, a2, ARITH_ADDCC);
1365	}
1366
1367	static const TCGOutOpBinary outop_addco = {
1368	.base.static_constraint = C_O1_I2(r, r, rJ),
1369	.out_rrr = tgen_addco_rrr,
1370	.out_rri = tgen_addco_rri,
1371	};
1372
1373	static void tgen_addci_rrr(TCGContext *s, TCGType type,
1374	TCGReg a0, TCGReg a1, TCGReg a2)
1375	{
1376	if (type == TCG_TYPE_I32) {
1377	tcg_out_arith(s, a0, a1, a2, ARITH_ADDC);
1378	} else if (use_vis3_instructions) {
1379	tcg_out_arith(s, a0, a1, a2, ARITH_ADDXC);
1380	} else {
1381	tcg_out_arith(s, TCG_REG_T1, a1, a2, ARITH_ADD); /* for CC */
1382	tcg_out_arithi(s, a0, TCG_REG_T1, 1, ARITH_ADD); /* for CS */
1383	/* Select the correct result based on actual carry value. */
1384	tcg_out_movcc(s, COND_CC, MOVCC_XCC, a0, TCG_REG_T1, false);
1385	}
1386	}
1387
1388	static void tgen_addci_rri(TCGContext *s, TCGType type,
1389	TCGReg a0, TCGReg a1, tcg_target_long a2)
1390	{
1391	if (type == TCG_TYPE_I32) {
1392	tcg_out_arithi(s, a0, a1, a2, ARITH_ADDC);
1393	return;
1394	}
1395	/* !use_vis3_instructions */
1396	if (a2 != 0) {
1397	tcg_out_arithi(s, TCG_REG_T1, a1, a2, ARITH_ADD); /* for CC */
1398	tcg_out_arithi(s, a0, TCG_REG_T1, 1, ARITH_ADD); /* for CS */
1399	tcg_out_movcc(s, COND_CC, MOVCC_XCC, a0, TCG_REG_T1, false);
1400	} else if (a0 == a1) {
1401	tcg_out_arithi(s, TCG_REG_T1, a1, 1, ARITH_ADD);
1402	tcg_out_movcc(s, COND_CS, MOVCC_XCC, a0, TCG_REG_T1, false);
1403	} else {
1404	tcg_out_arithi(s, a0, a1, 1, ARITH_ADD);
1405	tcg_out_movcc(s, COND_CC, MOVCC_XCC, a0, a1, false);
1406	}
1407	}
1408
1409	static TCGConstraintSetIndex cset_addci(TCGType type, unsigned flags)
1410	{
1411	if (use_vis3_instructions && type == TCG_TYPE_I64) {
1412	/* Note that ADDXC doesn't accept immediates. */
1413	return C_O1_I2(r, rz, rz);
1414	}
1415	return C_O1_I2(r, rz, rJ);
1416	}
1417
1418	static const TCGOutOpAddSubCarry outop_addci = {
1419	.base.static_constraint = C_Dynamic,
1420	.base.dynamic_constraint = cset_addci,
1421	.out_rrr = tgen_addci_rrr,
1422	.out_rri = tgen_addci_rri,
1423	};
1424
1425	/* Copy %xcc.c to %icc.c */
1426	static void tcg_out_dup_xcc_c(TCGContext *s)
1427	{
1428	if (use_vis3_instructions) {
1429	tcg_out_arith(s, TCG_REG_T1, TCG_REG_G0, TCG_REG_G0, ARITH_ADDXC);
1430	} else {
1431	tcg_out_movi_s13(s, TCG_REG_T1, 0);
1432	tcg_out_movcc(s, COND_CS, MOVCC_XCC, TCG_REG_T1, 1, true);
1433	}
1434	/* Write carry-in into %icc via {0,1} + -1. */
1435	tcg_out_arithi(s, TCG_REG_G0, TCG_REG_T1, -1, ARITH_ADDCC);
1436	}
1437
1438	static void tgen_addcio_rrr(TCGContext *s, TCGType type,
1439	TCGReg a0, TCGReg a1, TCGReg a2)
1440	{
1441	if (type != TCG_TYPE_I32) {
1442	if (use_vis3_instructions) {
1443	tcg_out_arith(s, a0, a1, a2, ARITH_ADDXCCC);
1444	return;
1445	}
1446	tcg_out_dup_xcc_c(s);
1447	}
1448	tcg_out_arith(s, a0, a1, a2, ARITH_ADDCCC);
1449	}
1450
1451	static void tgen_addcio_rri(TCGContext *s, TCGType type,
1452	TCGReg a0, TCGReg a1, tcg_target_long a2)
1453	{
1454	if (type != TCG_TYPE_I32) {
1455	/* !use_vis3_instructions */
1456	tcg_out_dup_xcc_c(s);
1457	}
1458	tcg_out_arithi(s, a0, a1, a2, ARITH_ADDCCC);
1459	}
1460
1461	static TCGConstraintSetIndex cset_addcio(TCGType type, unsigned flags)
1462	{
1463	if (use_vis3_instructions && type == TCG_TYPE_I64) {
1464	/* Note that ADDXCCC doesn't accept immediates. */
1465	return C_O1_I2(r, rz, rz);
1466	}
1467	return C_O1_I2(r, rz, rJ);
1468	}
1469
1470	static const TCGOutOpBinary outop_addcio = {
1471	.base.static_constraint = C_Dynamic,
1472	.base.dynamic_constraint = cset_addcio,
1473	.out_rrr = tgen_addcio_rrr,
1474	.out_rri = tgen_addcio_rri,
1475	};
1476
1477	static void tcg_out_set_carry(TCGContext *s)
1478	{
1479	/* 0x11 -> xcc = nzvC, icc = nzvC */
1480	tcg_out_arithi(s, 0, TCG_REG_G0, 0x11, WRCCR);
1481	}
1482
1483	static void tgen_and(TCGContext *s, TCGType type,
1484	TCGReg a0, TCGReg a1, TCGReg a2)
1485	{
1486	tcg_out_arith(s, a0, a1, a2, ARITH_AND);
1487	}
1488
1489	static void tgen_andi(TCGContext *s, TCGType type,
1490	TCGReg a0, TCGReg a1, tcg_target_long a2)
1491	{
1492	tcg_out_arithi(s, a0, a1, a2, ARITH_AND);
1493	}
1494
1495	static const TCGOutOpBinary outop_and = {
1496	.base.static_constraint = C_O1_I2(r, r, rJ),
1497	.out_rrr = tgen_and,
1498	.out_rri = tgen_andi,
1499	};
1500
1501	static void tgen_andc(TCGContext *s, TCGType type,
1502	TCGReg a0, TCGReg a1, TCGReg a2)
1503	{
1504	tcg_out_arith(s, a0, a1, a2, ARITH_ANDN);
1505	}
1506
1507	static const TCGOutOpBinary outop_andc = {
1508	.base.static_constraint = C_O1_I2(r, r, r),
1509	.out_rrr = tgen_andc,
1510	};
1511
1512	static const TCGOutOpBinary outop_clz = {
1513	.base.static_constraint = C_NotImplemented,
1514	};
1515
1516	static void tgen_ctpop(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
1517	{
1518	tcg_out_arith(s, a0, TCG_REG_G0, a1, ARITH_POPC);
1519	}
1520
1521	static TCGConstraintSetIndex cset_ctpop(TCGType type, unsigned flags)
1522	{
1523	if (use_popc_instructions && type == TCG_TYPE_I64) {
1524	return C_O1_I1(r, r);
1525	}
1526	return C_NotImplemented;
1527	}
1528
1529	static const TCGOutOpUnary outop_ctpop = {
1530	.base.static_constraint = C_Dynamic,
1531	.base.dynamic_constraint = cset_ctpop,
1532	.out_rr = tgen_ctpop,
1533	};
1534
1535	static const TCGOutOpBinary outop_ctz = {
1536	.base.static_constraint = C_NotImplemented,
1537	};
1538
1539	static void tgen_divs_rJ(TCGContext *s, TCGType type,
1540	TCGReg a0, TCGReg a1, TCGArg a2, bool c2)
1541	{
1542	uint32_t insn;
1543
1544	if (type == TCG_TYPE_I32) {
1545	/* Load Y with the sign extension of a1 to 64-bits. */
1546	tcg_out_arithi(s, TCG_REG_T1, a1, 31, SHIFT_SRA);
1547	tcg_out_sety(s, TCG_REG_T1);
1548	insn = ARITH_SDIV;
1549	} else {
1550	insn = ARITH_SDIVX;
1551	}
1552	tcg_out_arithc(s, a0, a1, a2, c2, insn);
1553	}
1554
1555	static void tgen_divs(TCGContext *s, TCGType type,
1556	TCGReg a0, TCGReg a1, TCGReg a2)
1557	{
1558	tgen_divs_rJ(s, type, a0, a1, a2, false);
1559	}
1560
1561	static void tgen_divsi(TCGContext *s, TCGType type,
1562	TCGReg a0, TCGReg a1, tcg_target_long a2)
1563	{
1564	tgen_divs_rJ(s, type, a0, a1, a2, true);
1565	}
1566
1567	static const TCGOutOpBinary outop_divs = {
1568	.base.static_constraint = C_O1_I2(r, r, rJ),
1569	.out_rrr = tgen_divs,
1570	.out_rri = tgen_divsi,
1571	};
1572
1573	static const TCGOutOpDivRem outop_divs2 = {
1574	.base.static_constraint = C_NotImplemented,
1575	};
1576
1577	static void tgen_divu_rJ(TCGContext *s, TCGType type,
1578	TCGReg a0, TCGReg a1, TCGArg a2, bool c2)
1579	{
1580	uint32_t insn;
1581
1582	if (type == TCG_TYPE_I32) {
1583	/* Load Y with the zero extension to 64-bits. */
1584	tcg_out_sety(s, TCG_REG_G0);
1585	insn = ARITH_UDIV;
1586	} else {
1587	insn = ARITH_UDIVX;
1588	}
1589	tcg_out_arithc(s, a0, a1, a2, c2, insn);
1590	}
1591
1592	static void tgen_divu(TCGContext *s, TCGType type,
1593	TCGReg a0, TCGReg a1, TCGReg a2)
1594	{
1595	tgen_divu_rJ(s, type, a0, a1, a2, false);
1596	}
1597
1598	static void tgen_divui(TCGContext *s, TCGType type,
1599	TCGReg a0, TCGReg a1, tcg_target_long a2)
1600	{
1601	tgen_divu_rJ(s, type, a0, a1, a2, true);
1602	}
1603
1604	static const TCGOutOpBinary outop_divu = {
1605	.base.static_constraint = C_O1_I2(r, r, rJ),
1606	.out_rrr = tgen_divu,
1607	.out_rri = tgen_divui,
1608	};
1609
1610	static const TCGOutOpDivRem outop_divu2 = {
1611	.base.static_constraint = C_NotImplemented,
1612	};
1613
1614	static const TCGOutOpBinary outop_eqv = {
1615	.base.static_constraint = C_NotImplemented,
1616	};
1617
1618	static void tgen_extrh_i64_i32(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1)
1619	{
1620	tcg_out_arithi(s, a0, a1, 32, SHIFT_SRLX);
1621	}
1622
1623	static const TCGOutOpUnary outop_extrh_i64_i32 = {
1624	.base.static_constraint = C_O1_I1(r, r),
1625	.out_rr = tgen_extrh_i64_i32,
1626	};
1627
1628	static void tgen_mul(TCGContext *s, TCGType type,
1629	TCGReg a0, TCGReg a1, TCGReg a2)
1630	{
1631	uint32_t insn = type == TCG_TYPE_I32 ? ARITH_UMUL : ARITH_MULX;
1632	tcg_out_arith(s, a0, a1, a2, insn);
1633	}
1634
1635	static void tgen_muli(TCGContext *s, TCGType type,
1636	TCGReg a0, TCGReg a1, tcg_target_long a2)
1637	{
1638	uint32_t insn = type == TCG_TYPE_I32 ? ARITH_UMUL : ARITH_MULX;
1639	tcg_out_arithi(s, a0, a1, a2, insn);
1640	}
1641
1642	static const TCGOutOpBinary outop_mul = {
1643	.base.static_constraint = C_O1_I2(r, r, rJ),
1644	.out_rrr = tgen_mul,
1645	.out_rri = tgen_muli,
1646	};
1647
1648	/*
1649	* The 32-bit multiply insns produce a full 64-bit result.
1650	* Supporting 32-bit mul[us]2 opcodes avoids sign/zero-extensions
1651	* before the actual multiply; we only need extract the high part
1652	* into the separate operand.
1653	*/
1654	static TCGConstraintSetIndex cset_mul2(TCGType type, unsigned flags)
1655	{
1656	return type == TCG_TYPE_I32 ? C_O2_I2(r, r, r, r) : C_NotImplemented;
1657	}
1658
1659	static void tgen_muls2(TCGContext *s, TCGType type,
1660	TCGReg a0, TCGReg a1, TCGReg a2, TCGReg a3)
1661	{
1662	tcg_out_arith(s, a0, a2, a3, ARITH_SMUL);
1663	tcg_out_arithi(s, a1, a0, 32, SHIFT_SRLX);
1664	}
1665
1666	static const TCGOutOpMul2 outop_muls2 = {
1667	.base.static_constraint = C_Dynamic,
1668	.base.dynamic_constraint = cset_mul2,
1669	.out_rrrr = tgen_muls2,
1670	};
1671
1672	static const TCGOutOpBinary outop_mulsh = {
1673	.base.static_constraint = C_NotImplemented,
1674	};
1675
1676	static void tgen_mulu2(TCGContext *s, TCGType type,
1677	TCGReg a0, TCGReg a1, TCGReg a2, TCGReg a3)
1678	{
1679	tcg_out_arith(s, a0, a2, a3, ARITH_UMUL);
1680	tcg_out_arithi(s, a1, a0, 32, SHIFT_SRLX);
1681	}
1682
1683	static const TCGOutOpMul2 outop_mulu2 = {
1684	.base.static_constraint = C_Dynamic,
1685	.base.dynamic_constraint = cset_mul2,
1686	.out_rrrr = tgen_mulu2,
1687	};
1688
1689	static void tgen_muluh(TCGContext *s, TCGType type,
1690	TCGReg a0, TCGReg a1, TCGReg a2)
1691	{
1692	tcg_out_arith(s, a0, a1, a2, ARITH_UMULXHI);
1693	}
1694
1695	static TCGConstraintSetIndex cset_muluh(TCGType type, unsigned flags)
1696	{
1697	return (type == TCG_TYPE_I64 && use_vis3_instructions
1698	? C_O1_I2(r, r, r) : C_NotImplemented);
1699	}
1700
1701	static const TCGOutOpBinary outop_muluh = {
1702	.base.static_constraint = C_Dynamic,
1703	.base.dynamic_constraint = cset_muluh,
1704	.out_rrr = tgen_muluh,
1705	};
1706
1707	static const TCGOutOpBinary outop_nand = {
1708	.base.static_constraint = C_NotImplemented,
1709	};
1710
1711	static const TCGOutOpBinary outop_nor = {
1712	.base.static_constraint = C_NotImplemented,
1713	};
1714
1715	static void tgen_or(TCGContext *s, TCGType type,
1716	TCGReg a0, TCGReg a1, TCGReg a2)
1717	{
1718	tcg_out_arith(s, a0, a1, a2, ARITH_OR);
1719	}
1720
1721	static void tgen_ori(TCGContext *s, TCGType type,
1722	TCGReg a0, TCGReg a1, tcg_target_long a2)
1723	{
1724	tcg_out_arithi(s, a0, a1, a2, ARITH_OR);
1725	}
1726
1727	static const TCGOutOpBinary outop_or = {
1728	.base.static_constraint = C_O1_I2(r, r, rJ),
1729	.out_rrr = tgen_or,
1730	.out_rri = tgen_ori,
1731	};
1732
1733	static void tgen_orc(TCGContext *s, TCGType type,
1734	TCGReg a0, TCGReg a1, TCGReg a2)
1735	{
1736	tcg_out_arith(s, a0, a1, a2, ARITH_ORN);
1737	}
1738
1739	static const TCGOutOpBinary outop_orc = {
1740	.base.static_constraint = C_O1_I2(r, r, r),
1741	.out_rrr = tgen_orc,
1742	};
1743
1744	static const TCGOutOpBinary outop_rems = {
1745	.base.static_constraint = C_NotImplemented,
1746	};
1747
1748	static const TCGOutOpBinary outop_remu = {
1749	.base.static_constraint = C_NotImplemented,
1750	};
1751
1752	static const TCGOutOpBinary outop_rotl = {
1753	.base.static_constraint = C_NotImplemented,
1754	};
1755
1756	static const TCGOutOpBinary outop_rotr = {
1757	.base.static_constraint = C_NotImplemented,
1758	};
1759
1760	static void tgen_sar(TCGContext *s, TCGType type,
1761	TCGReg a0, TCGReg a1, TCGReg a2)
1762	{
1763	uint32_t insn = type == TCG_TYPE_I32 ? SHIFT_SRA : SHIFT_SRAX;
1764	tcg_out_arith(s, a0, a1, a2, insn);
1765	}
1766
1767	static void tgen_sari(TCGContext *s, TCGType type,
1768	TCGReg a0, TCGReg a1, tcg_target_long a2)
1769	{
1770	uint32_t insn = type == TCG_TYPE_I32 ? SHIFT_SRA : SHIFT_SRAX;
1771	uint32_t mask = type == TCG_TYPE_I32 ? 31 : 63;
1772	tcg_out_arithi(s, a0, a1, a2 & mask, insn);
1773	}
1774
1775	static const TCGOutOpBinary outop_sar = {
1776	.base.static_constraint = C_O1_I2(r, r, rJ),
1777	.out_rrr = tgen_sar,
1778	.out_rri = tgen_sari,
1779	};
1780
1781	static void tgen_shl(TCGContext *s, TCGType type,
1782	TCGReg a0, TCGReg a1, TCGReg a2)
1783	{
1784	uint32_t insn = type == TCG_TYPE_I32 ? SHIFT_SLL : SHIFT_SLLX;
1785	tcg_out_arith(s, a0, a1, a2, insn);
1786	}
1787
1788	static void tgen_shli(TCGContext *s, TCGType type,
1789	TCGReg a0, TCGReg a1, tcg_target_long a2)
1790	{
1791	uint32_t insn = type == TCG_TYPE_I32 ? SHIFT_SLL : SHIFT_SLLX;
1792	uint32_t mask = type == TCG_TYPE_I32 ? 31 : 63;
1793	tcg_out_arithi(s, a0, a1, a2 & mask, insn);
1794	}
1795
1796	static const TCGOutOpBinary outop_shl = {
1797	.base.static_constraint = C_O1_I2(r, r, rJ),
1798	.out_rrr = tgen_shl,
1799	.out_rri = tgen_shli,
1800	};
1801
1802	static void tgen_shr(TCGContext *s, TCGType type,
1803	TCGReg a0, TCGReg a1, TCGReg a2)
1804	{
1805	uint32_t insn = type == TCG_TYPE_I32 ? SHIFT_SRL : SHIFT_SRLX;
1806	tcg_out_arith(s, a0, a1, a2, insn);
1807	}
1808
1809	static void tgen_shri(TCGContext *s, TCGType type,
1810	TCGReg a0, TCGReg a1, tcg_target_long a2)
1811	{
1812	uint32_t insn = type == TCG_TYPE_I32 ? SHIFT_SRL : SHIFT_SRLX;
1813	uint32_t mask = type == TCG_TYPE_I32 ? 31 : 63;
1814	tcg_out_arithi(s, a0, a1, a2 & mask, insn);
1815	}
1816
1817	static const TCGOutOpBinary outop_shr = {
1818	.base.static_constraint = C_O1_I2(r, r, rJ),
1819	.out_rrr = tgen_shr,
1820	.out_rri = tgen_shri,
1821	};
1822
1823	static void tgen_sub(TCGContext *s, TCGType type,
1824	TCGReg a0, TCGReg a1, TCGReg a2)
1825	{
1826	tcg_out_arith(s, a0, a1, a2, ARITH_SUB);
1827	}
1828
1829	static const TCGOutOpSubtract outop_sub = {
1830	.base.static_constraint = C_O1_I2(r, r, r),
1831	.out_rrr = tgen_sub,
1832	};
1833
1834	static void tgen_subbo_rrr(TCGContext *s, TCGType type,
1835	TCGReg a0, TCGReg a1, TCGReg a2)
1836	{
1837	tcg_out_arith(s, a0, a1, a2, ARITH_SUBCC);
1838	}
1839
1840	static void tgen_subbo_rri(TCGContext *s, TCGType type,
1841	TCGReg a0, TCGReg a1, tcg_target_long a2)
1842	{
1843	tcg_out_arithi(s, a0, a1, a2, ARITH_SUBCC);
1844	}
1845
1846	static const TCGOutOpAddSubCarry outop_subbo = {
1847	.base.static_constraint = C_O1_I2(r, rz, rJ),
1848	.out_rrr = tgen_subbo_rrr,
1849	.out_rri = tgen_subbo_rri,
1850	};
1851
1852	static void tgen_subbi_rrr(TCGContext *s, TCGType type,
1853	TCGReg a0, TCGReg a1, TCGReg a2)
1854	{
1855	/* TODO: OSA 2015 added SUBXC */
1856	if (type == TCG_TYPE_I32) {
1857	tcg_out_arith(s, a0, a1, a2, ARITH_SUBC);
1858	} else {
1859	tcg_out_arith(s, TCG_REG_T1, a1, a2, ARITH_SUB); /* for CC */
1860	tcg_out_arithi(s, a0, TCG_REG_T1, 1, ARITH_SUB); /* for CS */
1861	/* Select the correct result based on actual borrow value. */
1862	tcg_out_movcc(s, COND_CC, MOVCC_XCC, a0, TCG_REG_T1, false);
1863	}
1864	}
1865
1866	static void tgen_subbi_rri(TCGContext *s, TCGType type,
1867	TCGReg a0, TCGReg a1, tcg_target_long a2)
1868	{
1869	if (type == TCG_TYPE_I32) {
1870	tcg_out_arithi(s, a0, a1, a2, ARITH_SUBC);
1871	} else if (a2 != 0) {
1872	tcg_out_arithi(s, TCG_REG_T1, a1, a2, ARITH_SUB); /* for CC */
1873	tcg_out_arithi(s, a0, TCG_REG_T1, 1, ARITH_SUB); /* for CS */
1874	tcg_out_movcc(s, COND_CC, MOVCC_XCC, a0, TCG_REG_T1, false);
1875	} else if (a0 == a1) {
1876	tcg_out_arithi(s, TCG_REG_T1, a1, 1, ARITH_SUB);
1877	tcg_out_movcc(s, COND_CS, MOVCC_XCC, a0, TCG_REG_T1, false);
1878	} else {
1879	tcg_out_arithi(s, a0, a1, 1, ARITH_SUB);
1880	tcg_out_movcc(s, COND_CC, MOVCC_XCC, a0, a1, false);
1881	}
1882	}
1883
1884	static const TCGOutOpAddSubCarry outop_subbi = {
1885	.base.static_constraint = C_O1_I2(r, rz, rJ),
1886	.out_rrr = tgen_subbi_rrr,
1887	.out_rri = tgen_subbi_rri,
1888	};
1889
1890	static void tgen_subbio_rrr(TCGContext *s, TCGType type,
1891	TCGReg a0, TCGReg a1, TCGReg a2)
1892	{
1893	if (type != TCG_TYPE_I32) {
1894	/* TODO: OSA 2015 added SUBXCCC */
1895	tcg_out_dup_xcc_c(s);
1896	}
1897	tcg_out_arith(s, a0, a1, a2, ARITH_SUBCCC);
1898	}
1899
1900	static void tgen_subbio_rri(TCGContext *s, TCGType type,
1901	TCGReg a0, TCGReg a1, tcg_target_long a2)
1902	{
1903	if (type != TCG_TYPE_I32) {
1904	tcg_out_dup_xcc_c(s);
1905	}
1906	tcg_out_arithi(s, a0, a1, a2, ARITH_SUBCCC);
1907	}
1908
1909	static const TCGOutOpAddSubCarry outop_subbio = {
1910	.base.static_constraint = C_O1_I2(r, rz, rJ),
1911	.out_rrr = tgen_subbio_rrr,
1912	.out_rri = tgen_subbio_rri,
1913	};
1914
1915	static void tcg_out_set_borrow(TCGContext *s)
1916	{
1917	tcg_out_set_carry(s); /* borrow == carry */
1918	}
1919
1920	static const TCGOutOpBinary outop_smax = {
1921	.base.static_constraint = C_NotImplemented,
1922	};
1923
1924	static const TCGOutOpBinary outop_smin = {
1925	.base.static_constraint = C_NotImplemented,
1926	};
1927
1928	static const TCGOutOpBinary outop_umax = {
1929	.base.static_constraint = C_NotImplemented,
1930	};
1931
1932	static const TCGOutOpBinary outop_umin = {
1933	.base.static_constraint = C_NotImplemented,
1934	};
1935
1936	static void tgen_xor(TCGContext *s, TCGType type,
1937	TCGReg a0, TCGReg a1, TCGReg a2)
1938	{
1939	tcg_out_arith(s, a0, a1, a2, ARITH_XOR);
1940	}
1941
1942	static void tgen_xori(TCGContext *s, TCGType type,
1943	TCGReg a0, TCGReg a1, tcg_target_long a2)
1944	{
1945	tcg_out_arithi(s, a0, a1, a2, ARITH_XOR);
1946	}
1947
1948	static const TCGOutOpBinary outop_xor = {
1949	.base.static_constraint = C_O1_I2(r, r, rJ),
1950	.out_rrr = tgen_xor,
1951	.out_rri = tgen_xori,
1952	};
1953
1954	static const TCGOutOpBswap outop_bswap16 = {
1955	.base.static_constraint = C_NotImplemented,
1956	};
1957
1958	static const TCGOutOpBswap outop_bswap32 = {
1959	.base.static_constraint = C_NotImplemented,
1960	};
1961
1962	static const TCGOutOpUnary outop_bswap64 = {
1963	.base.static_constraint = C_NotImplemented,
1964	};
1965
1966	static const TCGOutOpUnary outop_revbit8 = {
1967	.base.static_constraint = C_NotImplemented,
1968	};
1969
1970	static const TCGOutOpBswap outop_revbit32 = {
1971	.base.static_constraint = C_NotImplemented,
1972	};
1973
1974	static const TCGOutOpUnary outop_revbit64 = {
1975	.base.static_constraint = C_NotImplemented,
1976	};
1977
1978	static void tgen_neg(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
1979	{
1980	tgen_sub(s, type, a0, TCG_REG_G0, a1);
1981	}
1982
1983	static const TCGOutOpUnary outop_neg = {
1984	.base.static_constraint = C_O1_I1(r, r),
1985	.out_rr = tgen_neg,
1986	};
1987
1988	static void tgen_not(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1)
1989	{
1990	tgen_orc(s, type, a0, TCG_REG_G0, a1);
1991	}
1992
1993	static const TCGOutOpUnary outop_not = {
1994	.base.static_constraint = C_O1_I1(r, r),
1995	.out_rr = tgen_not,
1996	};
1997
1998	static const TCGOutOpDeposit outop_deposit = {
1999	.base.static_constraint = C_NotImplemented,
2000	};
2001
2002	static void tgen_extract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
2003	unsigned ofs, unsigned len)
2004	{
2005	tcg_debug_assert(ofs + len == 32);
2006	tcg_out_arithi(s, a0, a1, ofs, SHIFT_SRL);
2007	}
2008
2009	static const TCGOutOpExtract outop_extract = {
2010	.base.static_constraint = C_O1_I1(r, r),
2011	.out_rr = tgen_extract,
2012	};
2013
2014	static void tgen_sextract(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
2015	unsigned ofs, unsigned len)
2016	{
2017	tcg_debug_assert(ofs + len == 32);
2018	tcg_out_arithi(s, a0, a1, ofs, SHIFT_SRA);
2019	}
2020
2021	static const TCGOutOpExtract outop_sextract = {
2022	.base.static_constraint = C_O1_I1(r, r),
2023	.out_rr = tgen_sextract,
2024	};
2025
2026	static const TCGOutOpExtract2 outop_extract2 = {
2027	.base.static_constraint = C_NotImplemented,
2028	};
2029
2030	static void tgen_ld8u(TCGContext *s, TCGType type, TCGReg dest,
2031	TCGReg base, ptrdiff_t offset)
2032	{
2033	tcg_out_ldst(s, dest, base, offset, LDUB);
2034	}
2035
2036	static const TCGOutOpLoad outop_ld8u = {
2037	.base.static_constraint = C_O1_I1(r, r),
2038	.out = tgen_ld8u,
2039	};
2040
2041	static void tgen_ld8s(TCGContext *s, TCGType type, TCGReg dest,
2042	TCGReg base, ptrdiff_t offset)
2043	{
2044	tcg_out_ldst(s, dest, base, offset, LDSB);
2045	}
2046
2047	static const TCGOutOpLoad outop_ld8s = {
2048	.base.static_constraint = C_O1_I1(r, r),
2049	.out = tgen_ld8s,
2050	};
2051
2052	static void tgen_ld16u(TCGContext *s, TCGType type, TCGReg dest,
2053	TCGReg base, ptrdiff_t offset)
2054	{
2055	tcg_out_ldst(s, dest, base, offset, LDUH);
2056	}
2057
2058	static const TCGOutOpLoad outop_ld16u = {
2059	.base.static_constraint = C_O1_I1(r, r),
2060	.out = tgen_ld16u,
2061	};
2062
2063	static void tgen_ld16s(TCGContext *s, TCGType type, TCGReg dest,
2064	TCGReg base, ptrdiff_t offset)
2065	{
2066	tcg_out_ldst(s, dest, base, offset, LDSH);
2067	}
2068
2069	static const TCGOutOpLoad outop_ld16s = {
2070	.base.static_constraint = C_O1_I1(r, r),
2071	.out = tgen_ld16s,
2072	};
2073
2074	static void tgen_ld32u(TCGContext *s, TCGType type, TCGReg dest,
2075	TCGReg base, ptrdiff_t offset)
2076	{
2077	tcg_out_ldst(s, dest, base, offset, LDUW);
2078	}
2079
2080	static const TCGOutOpLoad outop_ld32u = {
2081	.base.static_constraint = C_O1_I1(r, r),
2082	.out = tgen_ld32u,
2083	};
2084
2085	static void tgen_ld32s(TCGContext *s, TCGType type, TCGReg dest,
2086	TCGReg base, ptrdiff_t offset)
2087	{
2088	tcg_out_ldst(s, dest, base, offset, LDSW);
2089	}
2090
2091	static const TCGOutOpLoad outop_ld32s = {
2092	.base.static_constraint = C_O1_I1(r, r),
2093	.out = tgen_ld32s,
2094	};
2095
2096	static void tgen_st8_r(TCGContext *s, TCGType type, TCGReg data,
2097	TCGReg base, ptrdiff_t offset)
2098	{
2099	tcg_out_ldst(s, data, base, offset, STB);
2100	}
2101
2102	static const TCGOutOpStore outop_st8 = {
2103	.base.static_constraint = C_O0_I2(rz, r),
2104	.out_r = tgen_st8_r,
2105	};
2106
2107	static void tgen_st16_r(TCGContext *s, TCGType type, TCGReg data,
2108	TCGReg base, ptrdiff_t offset)
2109	{
2110	tcg_out_ldst(s, data, base, offset, STH);
2111	}
2112
2113	static const TCGOutOpStore outop_st16 = {
2114	.base.static_constraint = C_O0_I2(rz, r),
2115	.out_r = tgen_st16_r,
2116	};
2117
2118	static const TCGOutOpStore outop_st = {
2119	.base.static_constraint = C_O0_I2(rz, r),
2120	.out_r = tcg_out_st,
2121	};
2122
2123
2124	static TCGConstraintSetIndex
2125	tcg_target_op_def(TCGOpcode op, TCGType type, unsigned flags)
2126	{
2127	return C_NotImplemented;
2128	}
2129
2130	static void tcg_target_init(TCGContext *s)
2131	{
2132	unsigned long hwcap = qemu_getauxval(AT_HWCAP);
2133
2134	/*
2135	* Only probe for the platform and capabilities if we haven't already
2136	* determined maximum values at compile time.
2137	*/
2138	use_popc_instructions = (hwcap & HWCAP_SPARC_POPC) != 0;
2139	#ifndef use_vis3_instructions
2140	use_vis3_instructions = (hwcap & HWCAP_SPARC_VIS3) != 0;
2141	#endif
2142
2143	tcg_target_available_regs[TCG_TYPE_I32] = ALL_GENERAL_REGS;
2144	tcg_target_available_regs[TCG_TYPE_I64] = ALL_GENERAL_REGS;
2145
2146	tcg_target_call_clobber_regs = 0;
2147	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_G1);
2148	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_G2);
2149	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_G3);
2150	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_G4);
2151	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_G5);
2152	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_G6);
2153	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_G7);
2154	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_O0);
2155	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_O1);
2156	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_O2);
2157	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_O3);
2158	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_O4);
2159	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_O5);
2160	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_O6);
2161	tcg_regset_set_reg(tcg_target_call_clobber_regs, TCG_REG_O7);
2162
2163	s->reserved_regs = 0;
2164	tcg_regset_set_reg(s->reserved_regs, TCG_REG_G0); /* zero */
2165	tcg_regset_set_reg(s->reserved_regs, TCG_REG_G6); /* reserved for os */
2166	tcg_regset_set_reg(s->reserved_regs, TCG_REG_G7); /* thread pointer */
2167	tcg_regset_set_reg(s->reserved_regs, TCG_REG_I6); /* frame pointer */
2168	tcg_regset_set_reg(s->reserved_regs, TCG_REG_I7); /* return address */
2169	tcg_regset_set_reg(s->reserved_regs, TCG_REG_O6); /* stack pointer */
2170	tcg_regset_set_reg(s->reserved_regs, TCG_REG_T1); /* for internal use */
2171	tcg_regset_set_reg(s->reserved_regs, TCG_REG_T2); /* for internal use */
2172	tcg_regset_set_reg(s->reserved_regs, TCG_REG_T3); /* for internal use */
2173	}
2174
2175	#define ELF_HOST_MACHINE EM_SPARCV9
2176
2177	typedef struct {
2178	DebugFrameHeader h;
2179	uint8_t fde_def_cfa[4];
2180	uint8_t fde_win_save;
2181	uint8_t fde_ret_save[3];
2182	} DebugFrame;
2183
2184	static const DebugFrame debug_frame = {
2185	.h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
2186	.h.cie.id = -1,
2187	.h.cie.version = 1,
2188	.h.cie.code_align = 1,
2189	.h.cie.data_align = -sizeof(void *) & 0x7f,
2190	.h.cie.return_column = 15, /* o7 */
2191
2192	/* Total FDE size does not include the "len" member. */
2193	.h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
2194
2195	.fde_def_cfa = {
2196	12, 30, /* DW_CFA_def_cfa i6, 2047 */
2197	(2047 & 0x7f) \| 0x80, (2047 >> 7)
2198	},
2199	.fde_win_save = 0x2d, /* DW_CFA_GNU_window_save */
2200	.fde_ret_save = { 9, 15, 31 }, /* DW_CFA_register o7, i7 */
2201	};
2202
2203	void tcg_register_jit(const void *buf, size_t buf_size)
2204	{
2205	tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
2206	}