tcg.c · samitouri/QOSamiQemu · siGit Code & Deploy

Code Commits Issues Pulls Sessions CI/CD

master

c 7,067 lines 222 KB

Raw

1	/*
2	* Tiny Code Generator for QEMU
3	*
4	* Copyright (c) 2008 Fabrice Bellard
5	*
6	* Permission is hereby granted, free of charge, to any person obtaining a copy
7	* of this software and associated documentation files (the "Software"), to deal
8	* in the Software without restriction, including without limitation the rights
9	* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10	* copies of the Software, and to permit persons to whom the Software is
11	* furnished to do so, subject to the following conditions:
12	*
13	* The above copyright notice and this permission notice shall be included in
14	* all copies or substantial portions of the Software.
15	*
16	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19	* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20	* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21	* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22	* THE SOFTWARE.
23	*/
24
25	#include "qemu/osdep.h"
26
27	/* Define to jump the ELF file used to communicate with GDB. */
28	#undef DEBUG_JIT
29
30	#include "qemu/error-report.h"
31	#include "qemu/cutils.h"
32	#include "qemu/host-utils.h"
33	#include "qemu/qemu-print.h"
34	#include "qemu/cacheflush.h"
35	#include "qemu/cacheinfo.h"
36	#include "qemu/timer.h"
37	#include "exec/target_page.h"
38	#include "exec/translation-block.h"
39	#include "exec/tlb-common.h"
40	#include "tcg/startup.h"
41	#include "tcg/tcg-op-common.h"
42
43	#if UINTPTR_MAX == UINT32_MAX
44	# define ELF_CLASS ELFCLASS32
45	#else
46	# define ELF_CLASS ELFCLASS64
47	#endif
48	#if HOST_BIG_ENDIAN
49	# define ELF_DATA ELFDATA2MSB
50	#else
51	# define ELF_DATA ELFDATA2LSB
52	#endif
53
54	#include "elf.h"
55	#include "exec/log.h"
56	#include "tcg/tcg-ldst.h"
57	#include "tcg/tcg-temp-internal.h"
58	#include "tcg-internal.h"
59	#include "tcg/perf.h"
60	#include "tcg-has.h"
61	#ifdef CONFIG_USER_ONLY
62	#include "user/guest-base.h"
63	#endif
64
65	/* Forward declarations for functions declared in tcg-target.c.inc and
66	used here. */
67	static void tcg_target_init(TCGContext *s);
68	static void tcg_target_qemu_prologue(TCGContext *s);
69	static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
70	intptr_t value, intptr_t addend);
71	static void tcg_out_nop_fill(tcg_insn_unit *p, int count);
72
73	typedef struct TCGLabelQemuLdst TCGLabelQemuLdst;
74	static bool tcg_out_qemu_ld_slow_path(TCGContext s, TCGLabelQemuLdst l);
75	static bool tcg_out_qemu_st_slow_path(TCGContext s, TCGLabelQemuLdst l);
76
77	/* The CIE and FDE header definitions will be common to all hosts. */
78	typedef struct {
79	uint32_t len __attribute__((aligned((sizeof(void *)))));
80	uint32_t id;
81	uint8_t version;
82	char augmentation[1];
83	uint8_t code_align;
84	uint8_t data_align;
85	uint8_t return_column;
86	} DebugFrameCIE;
87
88	typedef struct QEMU_PACKED {
89	uint32_t len __attribute__((aligned((sizeof(void *)))));
90	uint32_t cie_offset;
91	uintptr_t func_start;
92	uintptr_t func_len;
93	} DebugFrameFDEHeader;
94
95	typedef struct QEMU_PACKED {
96	DebugFrameCIE cie;
97	DebugFrameFDEHeader fde;
98	} DebugFrameHeader;
99
100	struct TCGLabelQemuLdst {
101	bool is_ld; /* qemu_ld: true, qemu_st: false */
102	MemOpIdx oi;
103	TCGType type; /* result type of a load */
104	TCGReg addr_reg; /* reg index for guest virtual addr */
105	TCGReg datalo_reg; /* reg index for low word to be loaded or stored */
106	TCGReg datahi_reg; /* reg index for high word to be loaded or stored */
107	const tcg_insn_unit raddr; / addr of the next IR of qemu_ld/st IR */
108	tcg_insn_unit label_ptr[2]; / label pointers to be updated */
109	QSIMPLEQ_ENTRY(TCGLabelQemuLdst) next;
110	};
111
112	static void tcg_register_jit_int(const void *buf, size_t size,
113	const void *debug_frame,
114	size_t debug_frame_size)
115	__attribute__((unused));
116
117	/* Forward declarations for functions declared and used in tcg-target.c.inc. */
118	static void tcg_out_tb_start(TCGContext *s);
119	static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
120	intptr_t arg2);
121	static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
122	static void tcg_out_movi(TCGContext *s, TCGType type,
123	TCGReg ret, tcg_target_long arg);
124	static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
125	static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
126	static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg);
127	static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg);
128	static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg);
129	static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg);
130	static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
131	static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
132	static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg);
133	static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
134	static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2);
135	static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
136	static void tcg_out_goto_tb(TCGContext *s, int which);
137	static void tcg_out_goto_ptr(TCGContext *s, TCGReg dest);
138	static void tcg_out_mb(TCGContext *s, unsigned bar);
139	static void tcg_out_br(TCGContext s, TCGLabel l);
140	static void tcg_out_set_carry(TCGContext *s);
141	static void tcg_out_set_borrow(TCGContext *s);
142	#if TCG_TARGET_MAYBE_vec
143	static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
144	TCGReg dst, TCGReg src);
145	static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
146	TCGReg dst, TCGReg base, intptr_t offset);
147	static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
148	TCGReg dst, int64_t arg);
149	static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
150	unsigned vecl, unsigned vece,
151	const TCGArg args[TCG_MAX_OP_ARGS],
152	const int const_args[TCG_MAX_OP_ARGS]);
153	#else
154	static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
155	TCGReg dst, TCGReg src)
156	{
157	g_assert_not_reached();
158	}
159	static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
160	TCGReg dst, TCGReg base, intptr_t offset)
161	{
162	g_assert_not_reached();
163	}
164	static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
165	TCGReg dst, int64_t arg)
166	{
167	g_assert_not_reached();
168	}
169	static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
170	unsigned vecl, unsigned vece,
171	const TCGArg args[TCG_MAX_OP_ARGS],
172	const int const_args[TCG_MAX_OP_ARGS])
173	{
174	g_assert_not_reached();
175	}
176	int tcg_can_emit_vec_op(TCGOpcode o, TCGType t, unsigned ve)
177	{
178	return 0;
179	}
180	#endif
181	static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
182	intptr_t arg2);
183	static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
184	TCGReg base, intptr_t ofs);
185	static void tcg_out_call(TCGContext s, const tcg_insn_unit target,
186	const TCGHelperInfo *info);
187	static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot);
188	static bool tcg_target_const_match(int64_t val, int ct,
189	TCGType type, TCGCond cond, int vece);
190
191	#ifndef CONFIG_USER_ONLY
192	#define guest_base ({ qemu_build_not_reached(); (uintptr_t)0; })
193	#endif
194
195	typedef struct TCGLdstHelperParam {
196	TCGReg (ra_gen)(TCGContext s, const TCGLabelQemuLdst *l, int arg_reg);
197	unsigned ntmp;
198	int tmp[3];
199	} TCGLdstHelperParam;
200
201	static void tcg_out_ld_helper_args(TCGContext s, const TCGLabelQemuLdst l,
202	const TCGLdstHelperParam *p)
203	__attribute__((unused));
204	static void tcg_out_ld_helper_ret(TCGContext s, const TCGLabelQemuLdst l,
205	bool load_sign, const TCGLdstHelperParam *p)
206	__attribute__((unused));
207	static void tcg_out_st_helper_args(TCGContext s, const TCGLabelQemuLdst l,
208	const TCGLdstHelperParam *p)
209	__attribute__((unused));
210
211	static void * const qemu_ld_helpers[MO_SSIZE + 1] __attribute__((unused)) = {
212	[MO_UB] = helper_ldub_mmu,
213	[MO_SB] = helper_ldsb_mmu,
214	[MO_UW] = helper_lduw_mmu,
215	[MO_SW] = helper_ldsw_mmu,
216	[MO_UL] = helper_ldul_mmu,
217	[MO_UQ] = helper_ldq_mmu,
218	[MO_SL] = helper_ldsl_mmu,
219	[MO_128] = helper_ld16_mmu,
220	};
221
222	static void * const qemu_st_helpers[MO_SIZE + 1] __attribute__((unused)) = {
223	[MO_8] = helper_stb_mmu,
224	[MO_16] = helper_stw_mmu,
225	[MO_32] = helper_stl_mmu,
226	[MO_64] = helper_stq_mmu,
227	[MO_128] = helper_st16_mmu,
228	};
229
230	typedef struct {
231	MemOp atom; /* lg2 bits of atomicity required */
232	MemOp align; /* lg2 bits of alignment to use */
233	} TCGAtomAlign;
234
235	static TCGAtomAlign atom_and_align_for_opc(TCGContext *s, MemOp opc,
236	MemOp host_atom, bool allow_two_ops)
237	__attribute__((unused));
238
239	TCGContext tcg_init_ctx;
240	__thread TCGContext *tcg_ctx;
241
242	TCGContext **tcg_ctxs;
243	unsigned int tcg_cur_ctxs;
244	unsigned int tcg_max_ctxs;
245	TCGv_env tcg_env;
246	const void *tcg_code_gen_epilogue;
247	ptrdiff_t tcg_splitwx_diff;
248
249	#ifndef CONFIG_TCG_INTERPRETER
250	tcg_prologue_fn *tcg_qemu_tb_exec;
251	#endif
252
253	static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT];
254	static TCGRegSet tcg_target_call_clobber_regs;
255
256	#if TCG_TARGET_INSN_UNIT_SIZE == 1
257	static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
258	{
259	*s->code_ptr++ = v;
260	}
261
262	static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
263	uint8_t v)
264	{
265	*p = v;
266	}
267	#endif
268
269	#if TCG_TARGET_INSN_UNIT_SIZE <= 2
270	static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
271	{
272	if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
273	*s->code_ptr++ = v;
274	} else {
275	tcg_insn_unit *p = s->code_ptr;
276	memcpy(p, &v, sizeof(v));
277	s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
278	}
279	}
280
281	static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
282	uint16_t v)
283	{
284	if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
285	*p = v;
286	} else {
287	memcpy(p, &v, sizeof(v));
288	}
289	}
290	#endif
291
292	#if TCG_TARGET_INSN_UNIT_SIZE <= 4
293	static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
294	{
295	if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
296	*s->code_ptr++ = v;
297	} else {
298	tcg_insn_unit *p = s->code_ptr;
299	memcpy(p, &v, sizeof(v));
300	s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
301	}
302	}
303
304	static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
305	uint32_t v)
306	{
307	if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
308	*p = v;
309	} else {
310	memcpy(p, &v, sizeof(v));
311	}
312	}
313	#endif
314
315	#if TCG_TARGET_INSN_UNIT_SIZE <= 8
316	static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
317	{
318	if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
319	*s->code_ptr++ = v;
320	} else {
321	tcg_insn_unit *p = s->code_ptr;
322	memcpy(p, &v, sizeof(v));
323	s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
324	}
325	}
326
327	static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
328	uint64_t v)
329	{
330	if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
331	*p = v;
332	} else {
333	memcpy(p, &v, sizeof(v));
334	}
335	}
336	#endif
337
338	/* label relocation processing */
339
340	static void tcg_out_reloc(TCGContext s, tcg_insn_unit code_ptr, int type,
341	TCGLabel *l, intptr_t addend)
342	{
343	TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation));
344
345	r->type = type;
346	r->ptr = code_ptr;
347	r->addend = addend;
348	QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
349	}
350
351	static void tcg_out_label(TCGContext s, TCGLabel l)
352	{
353	tcg_debug_assert(!l->has_value);
354	l->has_value = 1;
355	l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
356	}
357
358	TCGLabel *gen_new_label(void)
359	{
360	TCGContext *s = tcg_ctx;
361	TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
362
363	memset(l, 0, sizeof(TCGLabel));
364	l->id = s->nb_labels++;
365	QSIMPLEQ_INIT(&l->branches);
366	QSIMPLEQ_INIT(&l->relocs);
367
368	QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);
369
370	return l;
371	}
372
373	static bool tcg_resolve_relocs(TCGContext *s)
374	{
375	TCGLabel *l;
376
377	QSIMPLEQ_FOREACH(l, &s->labels, next) {
378	TCGRelocation *r;
379	uintptr_t value = l->u.value;
380
381	QSIMPLEQ_FOREACH(r, &l->relocs, next) {
382	if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
383	return false;
384	}
385	}
386	}
387	return true;
388	}
389
390	static void set_jmp_reset_offset(TCGContext *s, int which)
391	{
392	/*
393	* We will check for overflow at the end of the opcode loop in
394	* tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
395	*/
396	s->gen_tb->jmp_reset_offset[which] = tcg_current_code_size(s);
397	}
398
399	static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which)
400	{
401	/*
402	* We will check for overflow at the end of the opcode loop in
403	* tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
404	*/
405	s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s);
406	}
407
408	static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which)
409	{
410	/*
411	* Return the read-execute version of the pointer, for the benefit
412	* of any pc-relative addressing mode.
413	*/
414	return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]);
415	}
416
417	static int __attribute__((unused))
418	tlb_mask_table_ofs(TCGContext *s, int which)
419	{
420	int fi = mmuidx_to_fast_index(which);
421	return (offsetof(CPUNegativeOffsetState, tlb.f[fi]) -
422	sizeof(CPUNegativeOffsetState));
423	}
424
425	/* Signal overflow, starting over with fewer guest insns. */
426	static G_NORETURN
427	void tcg_raise_tb_overflow(TCGContext *s)
428	{
429	siglongjmp(s->jmp_trans, -2);
430	}
431
432	/*
433	* Used by tcg_out_movext{1,2} to hold the arguments for tcg_out_movext.
434	* By the time we arrive at tcg_out_movext1, @dst is always a TCGReg.
435	*
436	* However, tcg_out_helper_load_slots reuses this field to hold an
437	* argument slot number (which may designate a argument register or an
438	* argument stack slot), converting to TCGReg once all arguments that
439	* are destined for the stack are processed.
440	*/
441	typedef struct TCGMovExtend {
442	unsigned dst;
443	TCGReg src;
444	TCGType dst_type;
445	TCGType src_type;
446	MemOp src_ext;
447	} TCGMovExtend;
448
449	/**
450	* tcg_out_movext -- move and extend
451	* @s: tcg context
452	* @dst_type: integral type for destination
453	* @dst: destination register
454	* @src_type: integral type for source
455	* @src_ext: extension to apply to source
456	* @src: source register
457	*
458	* Move or extend @src into @dst, depending on @src_ext and the types.
459	*/
460	static void tcg_out_movext(TCGContext *s, TCGType dst_type, TCGReg dst,
461	TCGType src_type, MemOp src_ext, TCGReg src)
462	{
463	switch (src_ext) {
464	case MO_UB:
465	tcg_out_ext8u(s, dst, src);
466	break;
467	case MO_SB:
468	tcg_out_ext8s(s, dst_type, dst, src);
469	break;
470	case MO_UW:
471	tcg_out_ext16u(s, dst, src);
472	break;
473	case MO_SW:
474	tcg_out_ext16s(s, dst_type, dst, src);
475	break;
476	case MO_UL:
477	case MO_SL:
478	if (dst_type == TCG_TYPE_I32) {
479	if (src_type == TCG_TYPE_I32) {
480	tcg_out_mov(s, TCG_TYPE_I32, dst, src);
481	} else {
482	tcg_out_extrl_i64_i32(s, dst, src);
483	}
484	} else if (src_type == TCG_TYPE_I32) {
485	if (src_ext & MO_SIGN) {
486	tcg_out_exts_i32_i64(s, dst, src);
487	} else {
488	tcg_out_extu_i32_i64(s, dst, src);
489	}
490	} else {
491	if (src_ext & MO_SIGN) {
492	tcg_out_ext32s(s, dst, src);
493	} else {
494	tcg_out_ext32u(s, dst, src);
495	}
496	}
497	break;
498	case MO_UQ:
499	if (dst_type == TCG_TYPE_I32) {
500	tcg_out_extrl_i64_i32(s, dst, src);
501	} else {
502	tcg_out_mov(s, TCG_TYPE_I64, dst, src);
503	}
504	break;
505	default:
506	g_assert_not_reached();
507	}
508	}
509
510	/* Minor variations on a theme, using a structure. */
511	static void tcg_out_movext1_new_src(TCGContext s, const TCGMovExtend i,
512	TCGReg src)
513	{
514	tcg_out_movext(s, i->dst_type, i->dst, i->src_type, i->src_ext, src);
515	}
516
517	static void tcg_out_movext1(TCGContext s, const TCGMovExtend i)
518	{
519	tcg_out_movext1_new_src(s, i, i->src);
520	}
521
522	/**
523	* tcg_out_movext2 -- move and extend two pair
524	* @s: tcg context
525	* @i1: first move description
526	* @i2: second move description
527	* @scratch: temporary register, or -1 for none
528	*
529	* As tcg_out_movext, for both @i1 and @i2, caring for overlap
530	* between the sources and destinations.
531	*/
532
533	static void tcg_out_movext2(TCGContext s, const TCGMovExtend i1,
534	const TCGMovExtend *i2, int scratch)
535	{
536	TCGReg src1 = i1->src;
537	TCGReg src2 = i2->src;
538
539	if (i1->dst != src2) {
540	tcg_out_movext1(s, i1);
541	tcg_out_movext1(s, i2);
542	return;
543	}
544	if (i2->dst == src1) {
545	TCGType src1_type = i1->src_type;
546	TCGType src2_type = i2->src_type;
547
548	if (tcg_out_xchg(s, MAX(src1_type, src2_type), src1, src2)) {
549	/* The data is now in the correct registers, now extend. */
550	src1 = i2->src;
551	src2 = i1->src;
552	} else {
553	tcg_debug_assert(scratch >= 0);
554	tcg_out_mov(s, src1_type, scratch, src1);
555	src1 = scratch;
556	}
557	}
558	tcg_out_movext1_new_src(s, i2, src2);
559	tcg_out_movext1_new_src(s, i1, src1);
560	}
561
562	/**
563	* tcg_out_movext3 -- move and extend three pair
564	* @s: tcg context
565	* @i1: first move description
566	* @i2: second move description
567	* @i3: third move description
568	* @scratch: temporary register, or -1 for none
569	*
570	* As tcg_out_movext, for all of @i1, @i2 and @i3, caring for overlap
571	* between the sources and destinations.
572	*/
573
574	static void tcg_out_movext3(TCGContext s, const TCGMovExtend i1,
575	const TCGMovExtend i2, const TCGMovExtend i3,
576	int scratch)
577	{
578	TCGReg src1 = i1->src;
579	TCGReg src2 = i2->src;
580	TCGReg src3 = i3->src;
581
582	if (i1->dst != src2 && i1->dst != src3) {
583	tcg_out_movext1(s, i1);
584	tcg_out_movext2(s, i2, i3, scratch);
585	return;
586	}
587	if (i2->dst != src1 && i2->dst != src3) {
588	tcg_out_movext1(s, i2);
589	tcg_out_movext2(s, i1, i3, scratch);
590	return;
591	}
592	if (i3->dst != src1 && i3->dst != src2) {
593	tcg_out_movext1(s, i3);
594	tcg_out_movext2(s, i1, i2, scratch);
595	return;
596	}
597
598	/*
599	* There is a cycle. Since there are only 3 nodes, the cycle is
600	* either "clockwise" or "anti-clockwise", and can be solved with
601	* a single scratch or two xchg.
602	*/
603	if (i1->dst == src2 && i2->dst == src3 && i3->dst == src1) {
604	/* "Clockwise" */
605	if (tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2)) {
606	tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3);
607	/* The data is now in the correct registers, now extend. */
608	tcg_out_movext1_new_src(s, i1, i1->dst);
609	tcg_out_movext1_new_src(s, i2, i2->dst);
610	tcg_out_movext1_new_src(s, i3, i3->dst);
611	} else {
612	tcg_debug_assert(scratch >= 0);
613	tcg_out_mov(s, i1->src_type, scratch, src1);
614	tcg_out_movext1(s, i3);
615	tcg_out_movext1(s, i2);
616	tcg_out_movext1_new_src(s, i1, scratch);
617	}
618	} else if (i1->dst == src3 && i2->dst == src1 && i3->dst == src2) {
619	/* "Anti-clockwise" */
620	if (tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3)) {
621	tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2);
622	/* The data is now in the correct registers, now extend. */
623	tcg_out_movext1_new_src(s, i1, i1->dst);
624	tcg_out_movext1_new_src(s, i2, i2->dst);
625	tcg_out_movext1_new_src(s, i3, i3->dst);
626	} else {
627	tcg_debug_assert(scratch >= 0);
628	tcg_out_mov(s, i1->src_type, scratch, src1);
629	tcg_out_movext1(s, i2);
630	tcg_out_movext1(s, i3);
631	tcg_out_movext1_new_src(s, i1, scratch);
632	}
633	} else {
634	g_assert_not_reached();
635	}
636	}
637
638	/*
639	* Allocate a new TCGLabelQemuLdst entry.
640	*/
641
642	__attribute__((unused))
643	static TCGLabelQemuLdst new_ldst_label(TCGContext s)
644	{
645	TCGLabelQemuLdst l = tcg_malloc(sizeof(l));
646
647	memset(l, 0, sizeof(*l));
648	QSIMPLEQ_INSERT_TAIL(&s->ldst_labels, l, next);
649
650	return l;
651	}
652
653	/*
654	* Allocate new constant pool entries.
655	*/
656
657	typedef struct TCGLabelPoolData {
658	struct TCGLabelPoolData *next;
659	tcg_insn_unit *label;
660	intptr_t addend;
661	int rtype;
662	unsigned nlong;
663	tcg_target_ulong data[];
664	} TCGLabelPoolData;
665
666	static TCGLabelPoolData new_pool_alloc(TCGContext s, int nlong, int rtype,
667	tcg_insn_unit *label, intptr_t addend)
668	{
669	TCGLabelPoolData *n = tcg_malloc(sizeof(TCGLabelPoolData)
670	+ sizeof(tcg_target_ulong) * nlong);
671
672	n->label = label;
673	n->addend = addend;
674	n->rtype = rtype;
675	n->nlong = nlong;
676	return n;
677	}
678
679	static void new_pool_insert(TCGContext s, TCGLabelPoolData n)
680	{
681	TCGLabelPoolData i, *pp;
682	int nlong = n->nlong;
683
684	/* Insertion sort on the pool. */
685	for (pp = &s->pool_labels; (i = *pp) != NULL; pp = &i->next) {
686	if (nlong > i->nlong) {
687	break;
688	}
689	if (nlong < i->nlong) {
690	continue;
691	}
692	if (memcmp(n->data, i->data, sizeof(tcg_target_ulong) * nlong) >= 0) {
693	break;
694	}
695	}
696	n->next = *pp;
697	*pp = n;
698	}
699
700	/* The "usual" for generic integer code. */
701	__attribute__((unused))
702	static void new_pool_label(TCGContext *s, tcg_target_ulong d, int rtype,
703	tcg_insn_unit *label, intptr_t addend)
704	{
705	TCGLabelPoolData *n = new_pool_alloc(s, 1, rtype, label, addend);
706	n->data[0] = d;
707	new_pool_insert(s, n);
708	}
709
710	/* For v64 or v128, depending on the host. */
711	__attribute__((unused))
712	static void new_pool_l2(TCGContext s, int rtype, tcg_insn_unit label,
713	intptr_t addend, tcg_target_ulong d0,
714	tcg_target_ulong d1)
715	{
716	TCGLabelPoolData *n = new_pool_alloc(s, 2, rtype, label, addend);
717	n->data[0] = d0;
718	n->data[1] = d1;
719	new_pool_insert(s, n);
720	}
721
722	/* For v128 or v256, depending on the host. */
723	__attribute__((unused))
724	static void new_pool_l4(TCGContext s, int rtype, tcg_insn_unit label,
725	intptr_t addend, tcg_target_ulong d0,
726	tcg_target_ulong d1, tcg_target_ulong d2,
727	tcg_target_ulong d3)
728	{
729	TCGLabelPoolData *n = new_pool_alloc(s, 4, rtype, label, addend);
730	n->data[0] = d0;
731	n->data[1] = d1;
732	n->data[2] = d2;
733	n->data[3] = d3;
734	new_pool_insert(s, n);
735	}
736
737	/* For v256, for 32-bit host. */
738	__attribute__((unused))
739	static void new_pool_l8(TCGContext s, int rtype, tcg_insn_unit label,
740	intptr_t addend, tcg_target_ulong d0,
741	tcg_target_ulong d1, tcg_target_ulong d2,
742	tcg_target_ulong d3, tcg_target_ulong d4,
743	tcg_target_ulong d5, tcg_target_ulong d6,
744	tcg_target_ulong d7)
745	{
746	TCGLabelPoolData *n = new_pool_alloc(s, 8, rtype, label, addend);
747	n->data[0] = d0;
748	n->data[1] = d1;
749	n->data[2] = d2;
750	n->data[3] = d3;
751	n->data[4] = d4;
752	n->data[5] = d5;
753	n->data[6] = d6;
754	n->data[7] = d7;
755	new_pool_insert(s, n);
756	}
757
758	/*
759	* Generate TB finalization at the end of block
760	*/
761
762	static int tcg_out_ldst_finalize(TCGContext *s)
763	{
764	TCGLabelQemuLdst *lb;
765
766	/* qemu_ld/st slow paths */
767	QSIMPLEQ_FOREACH(lb, &s->ldst_labels, next) {
768	if (lb->is_ld
769	? !tcg_out_qemu_ld_slow_path(s, lb)
770	: !tcg_out_qemu_st_slow_path(s, lb)) {
771	return -2;
772	}
773
774	/*
775	* Test for (pending) buffer overflow. The assumption is that any
776	* one operation beginning below the high water mark cannot overrun
777	* the buffer completely. Thus we can test for overflow after
778	* generating code without having to check during generation.
779	*/
780	if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
781	return -1;
782	}
783	}
784	return 0;
785	}
786
787	static int tcg_out_pool_finalize(TCGContext *s)
788	{
789	TCGLabelPoolData *p = s->pool_labels;
790	TCGLabelPoolData *l = NULL;
791	void *a;
792
793	if (p == NULL) {
794	return 0;
795	}
796
797	/*
798	* ??? Round up to qemu_icache_linesize, but then do not round
799	* again when allocating the next TranslationBlock structure.
800	*/
801	a = (void *)ROUND_UP((uintptr_t)s->code_ptr,
802	sizeof(tcg_target_ulong) * p->nlong);
803	tcg_out_nop_fill(s->code_ptr, (tcg_insn_unit *)a - s->code_ptr);
804	s->data_gen_ptr = a;
805
806	for (; p != NULL; p = p->next) {
807	size_t size = sizeof(tcg_target_ulong) * p->nlong;
808	uintptr_t value;
809
810	if (!l \|\| l->nlong != p->nlong \|\| memcmp(l->data, p->data, size)) {
811	if (unlikely(a > s->code_gen_highwater)) {
812	return -1;
813	}
814	memcpy(a, p->data, size);
815	a += size;
816	l = p;
817	}
818
819	value = (uintptr_t)tcg_splitwx_to_rx(a) - size;
820	if (!patch_reloc(p->label, p->rtype, value, p->addend)) {
821	return -2;
822	}
823	}
824
825	s->code_ptr = a;
826	return 0;
827	}
828
829	#define C_PFX1(P, A) P##A
830	#define C_PFX2(P, A, B) P##A##_##B
831	#define C_PFX3(P, A, B, C) P##A##_##B##_##C
832	#define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D
833	#define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E
834	#define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F
835
836	/* Define an enumeration for the various combinations. */
837
838	#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1),
839	#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2),
840	#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3),
841	#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4),
842
843	#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1),
844	#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2),
845	#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3),
846	#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
847
848	#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
849	#define C_N1O1_I1(O1, O2, I1) C_PFX3(c_n1o1_i1_, O1, O2, I1),
850	#define C_N2_I1(O1, O2, I1) C_PFX3(c_n2_i1_, O1, O2, I1),
851
852	#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
853	#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
854	#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
855	#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
856	#define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4),
857
858	typedef enum {
859	C_Dynamic = -2,
860	C_NotImplemented = -1,
861	#include "tcg-target-con-set.h"
862	} TCGConstraintSetIndex;
863
864	static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode, TCGType, unsigned);
865
866	#undef C_O0_I1
867	#undef C_O0_I2
868	#undef C_O0_I3
869	#undef C_O0_I4
870	#undef C_O1_I1
871	#undef C_O1_I2
872	#undef C_O1_I3
873	#undef C_O1_I4
874	#undef C_N1_I2
875	#undef C_N1O1_I1
876	#undef C_N2_I1
877	#undef C_O2_I1
878	#undef C_O2_I2
879	#undef C_O2_I3
880	#undef C_O2_I4
881	#undef C_N1_O1_I4
882
883	/* Put all of the constraint sets into an array, indexed by the enum. */
884
885	typedef struct TCGConstraintSet {
886	uint8_t nb_oargs, nb_iargs;
887	const char *args_ct_str[TCG_MAX_OP_ARGS];
888	} TCGConstraintSet;
889
890	#define C_O0_I1(I1) { 0, 1, { #I1 } },
891	#define C_O0_I2(I1, I2) { 0, 2, { #I1, #I2 } },
892	#define C_O0_I3(I1, I2, I3) { 0, 3, { #I1, #I2, #I3 } },
893	#define C_O0_I4(I1, I2, I3, I4) { 0, 4, { #I1, #I2, #I3, #I4 } },
894
895	#define C_O1_I1(O1, I1) { 1, 1, { #O1, #I1 } },
896	#define C_O1_I2(O1, I1, I2) { 1, 2, { #O1, #I1, #I2 } },
897	#define C_O1_I3(O1, I1, I2, I3) { 1, 3, { #O1, #I1, #I2, #I3 } },
898	#define C_O1_I4(O1, I1, I2, I3, I4) { 1, 4, { #O1, #I1, #I2, #I3, #I4 } },
899
900	#define C_N1_I2(O1, I1, I2) { 1, 2, { "&" #O1, #I1, #I2 } },
901	#define C_N1O1_I1(O1, O2, I1) { 2, 1, { "&" #O1, #O2, #I1 } },
902	#define C_N2_I1(O1, O2, I1) { 2, 1, { "&" #O1, "&" #O2, #I1 } },
903
904	#define C_O2_I1(O1, O2, I1) { 2, 1, { #O1, #O2, #I1 } },
905	#define C_O2_I2(O1, O2, I1, I2) { 2, 2, { #O1, #O2, #I1, #I2 } },
906	#define C_O2_I3(O1, O2, I1, I2, I3) { 2, 3, { #O1, #O2, #I1, #I2, #I3 } },
907	#define C_O2_I4(O1, O2, I1, I2, I3, I4) { 2, 4, { #O1, #O2, #I1, #I2, #I3, #I4 } },
908	#define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) { 2, 4, { "&" #O1, #O2, #I1, #I2, #I3, #I4 } },
909
910	static const TCGConstraintSet constraint_sets[] = {
911	#include "tcg-target-con-set.h"
912	};
913
914	#undef C_O0_I1
915	#undef C_O0_I2
916	#undef C_O0_I3
917	#undef C_O0_I4
918	#undef C_O1_I1
919	#undef C_O1_I2
920	#undef C_O1_I3
921	#undef C_O1_I4
922	#undef C_N1_I2
923	#undef C_N1O1_I1
924	#undef C_N2_I1
925	#undef C_O2_I1
926	#undef C_O2_I2
927	#undef C_O2_I3
928	#undef C_O2_I4
929	#undef C_N1_O1_I4
930
931	/* Expand the enumerator to be returned from tcg_target_op_def(). */
932
933	#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1)
934	#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2)
935	#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3)
936	#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4)
937
938	#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1)
939	#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2)
940	#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3)
941	#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
942
943	#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
944	#define C_N1O1_I1(O1, O2, I1) C_PFX3(c_n1o1_i1_, O1, O2, I1)
945	#define C_N2_I1(O1, O2, I1) C_PFX3(c_n2_i1_, O1, O2, I1)
946
947	#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
948	#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
949	#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
950	#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
951	#define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4)
952
953	/*
954	* TCGOutOp is the base class for a set of structures that describe how
955	* to generate code for a given TCGOpcode.
956	*
957	* @static_constraint:
958	* C_NotImplemented: The TCGOpcode is not supported by the backend.
959	* C_Dynamic: Use @dynamic_constraint to select a constraint set
960	* based on any of @type, @flags, or host isa.
961	* Otherwise: The register allocation constrains for the TCGOpcode.
962	*
963	* Subclasses of TCGOutOp will define a set of output routines that may
964	* be used. Such routines will often be selected by the set of registers
965	* and constants that come out of register allocation. The set of
966	* routines that are provided will guide the set of constraints that are
967	* legal. In particular, assume that tcg_optimize() has done its job in
968	* swapping commutative operands and folding operations for which all
969	* operands are constant.
970	*/
971	typedef struct TCGOutOp {
972	TCGConstraintSetIndex static_constraint;
973	TCGConstraintSetIndex (*dynamic_constraint)(TCGType type, unsigned flags);
974	} TCGOutOp;
975
976	typedef struct TCGOutOpAddSubCarry {
977	TCGOutOp base;
978	void (out_rrr)(TCGContext s, TCGType type,
979	TCGReg a0, TCGReg a1, TCGReg a2);
980	void (out_rri)(TCGContext s, TCGType type,
981	TCGReg a0, TCGReg a1, tcg_target_long a2);
982	void (out_rir)(TCGContext s, TCGType type,
983	TCGReg a0, tcg_target_long a1, TCGReg a2);
984	void (out_rii)(TCGContext s, TCGType type,
985	TCGReg a0, tcg_target_long a1, tcg_target_long a2);
986	} TCGOutOpAddSubCarry;
987
988	typedef struct TCGOutOpBinary {
989	TCGOutOp base;
990	void (out_rrr)(TCGContext s, TCGType type,
991	TCGReg a0, TCGReg a1, TCGReg a2);
992	void (out_rri)(TCGContext s, TCGType type,
993	TCGReg a0, TCGReg a1, tcg_target_long a2);
994	} TCGOutOpBinary;
995
996	typedef struct TCGOutOpBrcond {
997	TCGOutOp base;
998	void (out_rr)(TCGContext s, TCGType type, TCGCond cond,
999	TCGReg a1, TCGReg a2, TCGLabel *label);
1000	void (out_ri)(TCGContext s, TCGType type, TCGCond cond,
1001	TCGReg a1, tcg_target_long a2, TCGLabel *label);
1002	} TCGOutOpBrcond;
1003
1004	typedef struct TCGOutOpBswap {
1005	TCGOutOp base;
1006	void (out_rr)(TCGContext s, TCGType type,
1007	TCGReg a0, TCGReg a1, unsigned flags);
1008	} TCGOutOpBswap;
1009
1010	typedef struct TCGOutOpDeposit {
1011	TCGOutOp base;
1012	void (out_rrr)(TCGContext s, TCGType type, TCGReg a0, TCGReg a1,
1013	TCGReg a2, unsigned ofs, unsigned len);
1014	void (out_rri)(TCGContext s, TCGType type, TCGReg a0, TCGReg a1,
1015	tcg_target_long a2, unsigned ofs, unsigned len);
1016	void (out_rzr)(TCGContext s, TCGType type, TCGReg a0,
1017	TCGReg a2, unsigned ofs, unsigned len);
1018	} TCGOutOpDeposit;
1019
1020	typedef struct TCGOutOpDivRem {
1021	TCGOutOp base;
1022	void (out_rr01r)(TCGContext s, TCGType type,
1023	TCGReg a0, TCGReg a1, TCGReg a4);
1024	} TCGOutOpDivRem;
1025
1026	typedef struct TCGOutOpExtract {
1027	TCGOutOp base;
1028	void (out_rr)(TCGContext s, TCGType type, TCGReg a0, TCGReg a1,
1029	unsigned ofs, unsigned len);
1030	} TCGOutOpExtract;
1031
1032	typedef struct TCGOutOpExtract2 {
1033	TCGOutOp base;
1034	void (out_rrr)(TCGContext s, TCGType type, TCGReg a0, TCGReg a1,
1035	TCGReg a2, unsigned shr);
1036	} TCGOutOpExtract2;
1037
1038	typedef struct TCGOutOpLoad {
1039	TCGOutOp base;
1040	void (out)(TCGContext s, TCGType type, TCGReg dest,
1041	TCGReg base, intptr_t offset);
1042	} TCGOutOpLoad;
1043
1044	typedef struct TCGOutOpMovcond {
1045	TCGOutOp base;
1046	void (out)(TCGContext s, TCGType type, TCGCond cond,
1047	TCGReg ret, TCGReg c1, TCGArg c2, bool const_c2,
1048	TCGArg vt, bool const_vt, TCGArg vf, bool consf_vf);
1049	} TCGOutOpMovcond;
1050
1051	typedef struct TCGOutOpMul2 {
1052	TCGOutOp base;
1053	void (out_rrrr)(TCGContext s, TCGType type,
1054	TCGReg a0, TCGReg a1, TCGReg a2, TCGReg a3);
1055	} TCGOutOpMul2;
1056
1057	typedef struct TCGOutOpQemuLdSt {
1058	TCGOutOp base;
1059	void (out)(TCGContext s, TCGType type, TCGReg dest,
1060	TCGReg addr, MemOpIdx oi);
1061	} TCGOutOpQemuLdSt;
1062
1063	typedef struct TCGOutOpQemuLdSt2 {
1064	TCGOutOp base;
1065	void (out)(TCGContext s, TCGType type, TCGReg dlo, TCGReg dhi,
1066	TCGReg addr, MemOpIdx oi);
1067	} TCGOutOpQemuLdSt2;
1068
1069	typedef struct TCGOutOpUnary {
1070	TCGOutOp base;
1071	void (out_rr)(TCGContext s, TCGType type, TCGReg a0, TCGReg a1);
1072	} TCGOutOpUnary;
1073
1074	typedef struct TCGOutOpSetcond {
1075	TCGOutOp base;
1076	void (out_rrr)(TCGContext s, TCGType type, TCGCond cond,
1077	TCGReg ret, TCGReg a1, TCGReg a2);
1078	void (out_rri)(TCGContext s, TCGType type, TCGCond cond,
1079	TCGReg ret, TCGReg a1, tcg_target_long a2);
1080	} TCGOutOpSetcond;
1081
1082	typedef struct TCGOutOpStore {
1083	TCGOutOp base;
1084	void (out_r)(TCGContext s, TCGType type, TCGReg data,
1085	TCGReg base, intptr_t offset);
1086	void (out_i)(TCGContext s, TCGType type, tcg_target_long data,
1087	TCGReg base, intptr_t offset);
1088	} TCGOutOpStore;
1089
1090	typedef struct TCGOutOpSubtract {
1091	TCGOutOp base;
1092	void (out_rrr)(TCGContext s, TCGType type,
1093	TCGReg a0, TCGReg a1, TCGReg a2);
1094	void (out_rir)(TCGContext s, TCGType type,
1095	TCGReg a0, tcg_target_long a1, TCGReg a2);
1096	} TCGOutOpSubtract;
1097
1098	#include "tcg-target.c.inc"
1099
1100	#ifndef CONFIG_TCG_INTERPRETER
1101	/* Validate CPUTLBDescFast placement. */
1102	QEMU_BUILD_BUG_ON((int)(offsetof(CPUNegativeOffsetState, tlb.f[0]) -
1103	sizeof(CPUNegativeOffsetState))
1104	< MIN_TLB_MASK_TABLE_OFS);
1105	#endif
1106
1107	/*
1108	* We require these functions for slow-path function calls.
1109	* Adapt them generically for opcode output.
1110	*/
1111
1112	static void tgen_exts_i32_i64(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1)
1113	{
1114	tcg_out_exts_i32_i64(s, a0, a1);
1115	}
1116
1117	static const TCGOutOpUnary outop_exts_i32_i64 = {
1118	.base.static_constraint = C_O1_I1(r, r),
1119	.out_rr = tgen_exts_i32_i64,
1120	};
1121
1122	static void tgen_extu_i32_i64(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1)
1123	{
1124	tcg_out_extu_i32_i64(s, a0, a1);
1125	}
1126
1127	static const TCGOutOpUnary outop_extu_i32_i64 = {
1128	.base.static_constraint = C_O1_I1(r, r),
1129	.out_rr = tgen_extu_i32_i64,
1130	};
1131
1132	static void tgen_extrl_i64_i32(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1)
1133	{
1134	tcg_out_extrl_i64_i32(s, a0, a1);
1135	}
1136
1137	static const TCGOutOpUnary outop_extrl_i64_i32 = {
1138	.base.static_constraint = C_O1_I1(r, r),
1139	.out_rr = TCG_TARGET_HAS_extr_i64_i32 ? tgen_extrl_i64_i32 : NULL,
1140	};
1141
1142	static const TCGOutOp outop_goto_ptr = {
1143	.static_constraint = C_O0_I1(r),
1144	};
1145
1146	static const TCGOutOpLoad outop_ld = {
1147	.base.static_constraint = C_O1_I1(r, r),
1148	.out = tcg_out_ld,
1149	};
1150
1151	/*
1152	* Register V as the TCGOutOp for O.
1153	* This verifies that V is of type T, otherwise give a nice compiler error.
1154	* This prevents trivial mistakes within each arch/tcg-target.c.inc.
1155	*/
1156	#define OUTOP(O, T, V) [O] = _Generic(V, T: &V.base)
1157
1158	/* Register allocation descriptions for every TCGOpcode. */
1159	static const TCGOutOp * const all_outop[NB_OPS] = {
1160	OUTOP(INDEX_op_add, TCGOutOpBinary, outop_add),
1161	OUTOP(INDEX_op_addci, TCGOutOpAddSubCarry, outop_addci),
1162	OUTOP(INDEX_op_addcio, TCGOutOpBinary, outop_addcio),
1163	OUTOP(INDEX_op_addco, TCGOutOpBinary, outop_addco),
1164	/* addc1o is implemented with set_carry + addcio */
1165	OUTOP(INDEX_op_addc1o, TCGOutOpBinary, outop_addcio),
1166	OUTOP(INDEX_op_and, TCGOutOpBinary, outop_and),
1167	OUTOP(INDEX_op_andc, TCGOutOpBinary, outop_andc),
1168	OUTOP(INDEX_op_brcond, TCGOutOpBrcond, outop_brcond),
1169	OUTOP(INDEX_op_bswap16, TCGOutOpBswap, outop_bswap16),
1170	OUTOP(INDEX_op_bswap32, TCGOutOpBswap, outop_bswap32),
1171	OUTOP(INDEX_op_clz, TCGOutOpBinary, outop_clz),
1172	OUTOP(INDEX_op_ctpop, TCGOutOpUnary, outop_ctpop),
1173	OUTOP(INDEX_op_ctz, TCGOutOpBinary, outop_ctz),
1174	OUTOP(INDEX_op_deposit, TCGOutOpDeposit, outop_deposit),
1175	OUTOP(INDEX_op_divs, TCGOutOpBinary, outop_divs),
1176	OUTOP(INDEX_op_divu, TCGOutOpBinary, outop_divu),
1177	OUTOP(INDEX_op_divs2, TCGOutOpDivRem, outop_divs2),
1178	OUTOP(INDEX_op_divu2, TCGOutOpDivRem, outop_divu2),
1179	OUTOP(INDEX_op_eqv, TCGOutOpBinary, outop_eqv),
1180	OUTOP(INDEX_op_extract, TCGOutOpExtract, outop_extract),
1181	OUTOP(INDEX_op_extract2, TCGOutOpExtract2, outop_extract2),
1182	OUTOP(INDEX_op_ld8u, TCGOutOpLoad, outop_ld8u),
1183	OUTOP(INDEX_op_ld8s, TCGOutOpLoad, outop_ld8s),
1184	OUTOP(INDEX_op_ld16u, TCGOutOpLoad, outop_ld16u),
1185	OUTOP(INDEX_op_ld16s, TCGOutOpLoad, outop_ld16s),
1186	OUTOP(INDEX_op_ld, TCGOutOpLoad, outop_ld),
1187	OUTOP(INDEX_op_movcond, TCGOutOpMovcond, outop_movcond),
1188	OUTOP(INDEX_op_mul, TCGOutOpBinary, outop_mul),
1189	OUTOP(INDEX_op_muls2, TCGOutOpMul2, outop_muls2),
1190	OUTOP(INDEX_op_mulsh, TCGOutOpBinary, outop_mulsh),
1191	OUTOP(INDEX_op_mulu2, TCGOutOpMul2, outop_mulu2),
1192	OUTOP(INDEX_op_muluh, TCGOutOpBinary, outop_muluh),
1193	OUTOP(INDEX_op_nand, TCGOutOpBinary, outop_nand),
1194	OUTOP(INDEX_op_neg, TCGOutOpUnary, outop_neg),
1195	OUTOP(INDEX_op_negsetcond, TCGOutOpSetcond, outop_negsetcond),
1196	OUTOP(INDEX_op_nor, TCGOutOpBinary, outop_nor),
1197	OUTOP(INDEX_op_not, TCGOutOpUnary, outop_not),
1198	OUTOP(INDEX_op_or, TCGOutOpBinary, outop_or),
1199	OUTOP(INDEX_op_orc, TCGOutOpBinary, outop_orc),
1200	OUTOP(INDEX_op_qemu_ld, TCGOutOpQemuLdSt, outop_qemu_ld),
1201	OUTOP(INDEX_op_qemu_ld2, TCGOutOpQemuLdSt2, outop_qemu_ld2),
1202	OUTOP(INDEX_op_qemu_st, TCGOutOpQemuLdSt, outop_qemu_st),
1203	OUTOP(INDEX_op_qemu_st2, TCGOutOpQemuLdSt2, outop_qemu_st2),
1204	OUTOP(INDEX_op_rems, TCGOutOpBinary, outop_rems),
1205	OUTOP(INDEX_op_remu, TCGOutOpBinary, outop_remu),
1206	OUTOP(INDEX_op_revbit32, TCGOutOpBswap, outop_revbit32),
1207	OUTOP(INDEX_op_rotl, TCGOutOpBinary, outop_rotl),
1208	OUTOP(INDEX_op_rotr, TCGOutOpBinary, outop_rotr),
1209	OUTOP(INDEX_op_sar, TCGOutOpBinary, outop_sar),
1210	OUTOP(INDEX_op_setcond, TCGOutOpSetcond, outop_setcond),
1211	OUTOP(INDEX_op_sextract, TCGOutOpExtract, outop_sextract),
1212	OUTOP(INDEX_op_shl, TCGOutOpBinary, outop_shl),
1213	OUTOP(INDEX_op_shr, TCGOutOpBinary, outop_shr),
1214	OUTOP(INDEX_op_smax, TCGOutOpBinary, outop_smax),
1215	OUTOP(INDEX_op_smin, TCGOutOpBinary, outop_smin),
1216	OUTOP(INDEX_op_st, TCGOutOpStore, outop_st),
1217	OUTOP(INDEX_op_st8, TCGOutOpStore, outop_st8),
1218	OUTOP(INDEX_op_st16, TCGOutOpStore, outop_st16),
1219	OUTOP(INDEX_op_sub, TCGOutOpSubtract, outop_sub),
1220	OUTOP(INDEX_op_subbi, TCGOutOpAddSubCarry, outop_subbi),
1221	OUTOP(INDEX_op_subbio, TCGOutOpAddSubCarry, outop_subbio),
1222	OUTOP(INDEX_op_subbo, TCGOutOpAddSubCarry, outop_subbo),
1223	/* subb1o is implemented with set_borrow + subbio */
1224	OUTOP(INDEX_op_subb1o, TCGOutOpAddSubCarry, outop_subbio),
1225	OUTOP(INDEX_op_umax, TCGOutOpBinary, outop_umax),
1226	OUTOP(INDEX_op_umin, TCGOutOpBinary, outop_umin),
1227	OUTOP(INDEX_op_xor, TCGOutOpBinary, outop_xor),
1228
1229	[INDEX_op_goto_ptr] = &outop_goto_ptr,
1230
1231	OUTOP(INDEX_op_bswap64, TCGOutOpUnary, outop_bswap64),
1232	OUTOP(INDEX_op_ext_i32_i64, TCGOutOpUnary, outop_exts_i32_i64),
1233	OUTOP(INDEX_op_extu_i32_i64, TCGOutOpUnary, outop_extu_i32_i64),
1234	OUTOP(INDEX_op_extrl_i64_i32, TCGOutOpUnary, outop_extrl_i64_i32),
1235	OUTOP(INDEX_op_extrh_i64_i32, TCGOutOpUnary, outop_extrh_i64_i32),
1236	OUTOP(INDEX_op_ld32u, TCGOutOpLoad, outop_ld32u),
1237	OUTOP(INDEX_op_ld32s, TCGOutOpLoad, outop_ld32s),
1238	OUTOP(INDEX_op_revbit8, TCGOutOpUnary, outop_revbit8),
1239	OUTOP(INDEX_op_revbit64, TCGOutOpUnary, outop_revbit64),
1240	OUTOP(INDEX_op_st32, TCGOutOpStore, outop_st),
1241	};
1242
1243	#undef OUTOP
1244
1245	/*
1246	* All TCG threads except the parent (i.e. the one that called tcg_context_init
1247	* and registered the target's TCG globals) must register with this function
1248	* before initiating translation.
1249	*
1250	* In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
1251	* of tcg_region_init() for the reasoning behind this.
1252	*
1253	* In system-mode each caller registers its context in tcg_ctxs[]. Note that in
1254	* system-mode tcg_ctxs[] does not track tcg_ctx_init, since the initial context
1255	* is not used anymore for translation once this function is called.
1256	*
1257	* Not tracking tcg_init_ctx in tcg_ctxs[] in system-mode keeps code that
1258	* iterates over the array (e.g. tcg_code_size() the same for both system/user
1259	* modes.
1260	*/
1261	#ifdef CONFIG_USER_ONLY
1262	void tcg_register_thread(void)
1263	{
1264	tcg_ctx = &tcg_init_ctx;
1265	}
1266	#else
1267	void tcg_register_thread(void)
1268	{
1269	TCGContext s = g_malloc(sizeof(s));
1270	unsigned int i, n;
1271
1272	*s = tcg_init_ctx;
1273
1274	/* Relink mem_base. */
1275	for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) {
1276	if (tcg_init_ctx.temps[i].mem_base) {
1277	ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
1278	tcg_debug_assert(b >= 0 && b < n);
1279	s->temps[i].mem_base = &s->temps[b];
1280	}
1281	}
1282
1283	/* Claim an entry in tcg_ctxs */
1284	n = qatomic_fetch_inc(&tcg_cur_ctxs);
1285	g_assert(n < tcg_max_ctxs);
1286	qatomic_set(&tcg_ctxs[n], s);
1287
1288	if (n > 0) {
1289	tcg_region_thread_initial_alloc(s);
1290	}
1291
1292	tcg_ctx = s;
1293	}
1294	#endif /* !CONFIG_USER_ONLY */
1295
1296	/* pool based memory allocation */
1297	void tcg_malloc_internal(TCGContext s, int size)
1298	{
1299	TCGPool *p;
1300	int pool_size;
1301
1302	if (size > TCG_POOL_CHUNK_SIZE) {
1303	/* big malloc: insert a new pool (XXX: could optimize) */
1304	p = g_malloc(sizeof(TCGPool) + size);
1305	p->size = size;
1306	p->next = s->pool_first_large;
1307	s->pool_first_large = p;
1308	return p->data;
1309	} else {
1310	p = s->pool_current;
1311	if (!p) {
1312	p = s->pool_first;
1313	if (!p) {
1314	goto new_pool;
1315	}
1316	} else {
1317	if (!p->next) {
1318	new_pool:
1319	pool_size = TCG_POOL_CHUNK_SIZE;
1320	p = g_malloc(sizeof(TCGPool) + pool_size);
1321	p->size = pool_size;
1322	p->next = NULL;
1323	if (s->pool_current) {
1324	s->pool_current->next = p;
1325	} else {
1326	s->pool_first = p;
1327	}
1328	} else {
1329	p = p->next;
1330	}
1331	}
1332	}
1333	s->pool_current = p;
1334	s->pool_cur = (uintptr_t)p->data + size;
1335	s->pool_end = (uintptr_t)p->data + p->size;
1336	return p->data;
1337	}
1338
1339	void tcg_pool_reset(TCGContext *s)
1340	{
1341	TCGPool p, t;
1342	for (p = s->pool_first_large; p; p = t) {
1343	t = p->next;
1344	g_free(p);
1345	}
1346	s->pool_first_large = NULL;
1347	s->pool_cur = s->pool_end = 0;
1348	s->pool_current = NULL;
1349	}
1350
1351	/*
1352	* Create TCGHelperInfo structures for "tcg/tcg-ldst.h" functions,
1353	* akin to what "exec/helper-tcg.h" does with DEF_HELPER_FLAGS_N.
1354	* We only use these for layout in tcg_out_ld_helper_ret and
1355	* tcg_out_st_helper_args, and share them between several of
1356	* the helpers, with the end result that it's easier to build manually.
1357	*/
1358
1359	#define dh_typecode_ttl dh_typecode_i64
1360
1361	static TCGHelperInfo info_helper_ld32_mmu = {
1362	.flags = TCG_CALL_NO_WG,
1363	.typemask = dh_typemask(ttl, 0) /* return tcg_target_ulong */
1364	\| dh_typemask(env, 1)
1365	\| dh_typemask(i64, 2) /* uint64_t addr */
1366	\| dh_typemask(i32, 3) /* unsigned oi */
1367	\| dh_typemask(ptr, 4) /* uintptr_t ra */
1368	};
1369
1370	static TCGHelperInfo info_helper_ld64_mmu = {
1371	.flags = TCG_CALL_NO_WG,
1372	.typemask = dh_typemask(i64, 0) /* return uint64_t */
1373	\| dh_typemask(env, 1)
1374	\| dh_typemask(i64, 2) /* uint64_t addr */
1375	\| dh_typemask(i32, 3) /* unsigned oi */
1376	\| dh_typemask(ptr, 4) /* uintptr_t ra */
1377	};
1378
1379	static TCGHelperInfo info_helper_ld128_mmu = {
1380	.flags = TCG_CALL_NO_WG,
1381	.typemask = dh_typemask(i128, 0) /* return Int128 */
1382	\| dh_typemask(env, 1)
1383	\| dh_typemask(i64, 2) /* uint64_t addr */
1384	\| dh_typemask(i32, 3) /* unsigned oi */
1385	\| dh_typemask(ptr, 4) /* uintptr_t ra */
1386	};
1387
1388	static TCGHelperInfo info_helper_st32_mmu = {
1389	.flags = TCG_CALL_NO_WG,
1390	.typemask = dh_typemask(void, 0)
1391	\| dh_typemask(env, 1)
1392	\| dh_typemask(i64, 2) /* uint64_t addr */
1393	\| dh_typemask(i32, 3) /* uint32_t data */
1394	\| dh_typemask(i32, 4) /* unsigned oi */
1395	\| dh_typemask(ptr, 5) /* uintptr_t ra */
1396	};
1397
1398	static TCGHelperInfo info_helper_st64_mmu = {
1399	.flags = TCG_CALL_NO_WG,
1400	.typemask = dh_typemask(void, 0)
1401	\| dh_typemask(env, 1)
1402	\| dh_typemask(i64, 2) /* uint64_t addr */
1403	\| dh_typemask(i64, 3) /* uint64_t data */
1404	\| dh_typemask(i32, 4) /* unsigned oi */
1405	\| dh_typemask(ptr, 5) /* uintptr_t ra */
1406	};
1407
1408	static TCGHelperInfo info_helper_st128_mmu = {
1409	.flags = TCG_CALL_NO_WG,
1410	.typemask = dh_typemask(void, 0)
1411	\| dh_typemask(env, 1)
1412	\| dh_typemask(i64, 2) /* uint64_t addr */
1413	\| dh_typemask(i128, 3) /* Int128 data */
1414	\| dh_typemask(i32, 4) /* unsigned oi */
1415	\| dh_typemask(ptr, 5) /* uintptr_t ra */
1416	};
1417
1418	#ifdef CONFIG_TCG_INTERPRETER
1419	static ffi_type *typecode_to_ffi(int argmask)
1420	{
1421	/*
1422	* libffi does not support __int128_t, so we have forced Int128
1423	* to use the structure definition instead of the builtin type.
1424	*/
1425	static ffi_type *ffi_type_i128_elements[3] = {
1426	&ffi_type_uint64,
1427	&ffi_type_uint64,
1428	NULL
1429	};
1430	static ffi_type ffi_type_i128 = {
1431	.size = 16,
1432	.alignment = __alignof__(Int128),
1433	.type = FFI_TYPE_STRUCT,
1434	.elements = ffi_type_i128_elements,
1435	};
1436
1437	switch (argmask) {
1438	case dh_typecode_void:
1439	return &ffi_type_void;
1440	case dh_typecode_i32:
1441	return &ffi_type_uint32;
1442	case dh_typecode_s32:
1443	return &ffi_type_sint32;
1444	case dh_typecode_i64:
1445	return &ffi_type_uint64;
1446	case dh_typecode_s64:
1447	return &ffi_type_sint64;
1448	case dh_typecode_ptr:
1449	return &ffi_type_pointer;
1450	case dh_typecode_i128:
1451	return &ffi_type_i128;
1452	}
1453	g_assert_not_reached();
1454	}
1455
1456	static ffi_cif init_ffi_layout(TCGHelperInfo info)
1457	{
1458	unsigned typemask = info->typemask;
1459	struct {
1460	ffi_cif cif;
1461	ffi_type *args[];
1462	} *ca;
1463	ffi_status status;
1464	int nargs;
1465
1466	/* Ignoring the return type, find the last non-zero field. */
1467	nargs = 32 - clz32(typemask >> 3);
1468	nargs = DIV_ROUND_UP(nargs, 3);
1469	assert(nargs <= MAX_CALL_IARGS);
1470
1471	ca = g_malloc0(sizeof(ca) + nargs sizeof(ffi_type *));
1472	ca->cif.rtype = typecode_to_ffi(typemask & 7);
1473	ca->cif.nargs = nargs;
1474
1475	if (nargs != 0) {
1476	ca->cif.arg_types = ca->args;
1477	for (int j = 0; j < nargs; ++j) {
1478	int typecode = extract32(typemask, (j + 1) * 3, 3);
1479	ca->args[j] = typecode_to_ffi(typecode);
1480	}
1481	}
1482
1483	status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
1484	ca->cif.rtype, ca->cif.arg_types);
1485	assert(status == FFI_OK);
1486
1487	return &ca->cif;
1488	}
1489
1490	#define HELPER_INFO_INIT(I) (&(I)->cif)
1491	#define HELPER_INFO_INIT_VAL(I) init_ffi_layout(I)
1492	#else
1493	#define HELPER_INFO_INIT(I) (&(I)->init)
1494	#define HELPER_INFO_INIT_VAL(I) 1
1495	#endif /* CONFIG_TCG_INTERPRETER */
1496
1497	static inline bool arg_slot_reg_p(unsigned arg_slot)
1498	{
1499	/*
1500	* Split the sizeof away from the comparison to avoid Werror from
1501	* "unsigned < 0 is always false", when iarg_regs is empty.
1502	*/
1503	unsigned nreg = ARRAY_SIZE(tcg_target_call_iarg_regs);
1504	return arg_slot < nreg;
1505	}
1506
1507	static inline int arg_slot_stk_ofs(unsigned arg_slot)
1508	{
1509	unsigned max = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long);
1510	unsigned stk_slot = arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs);
1511
1512	tcg_debug_assert(stk_slot < max);
1513	return TCG_TARGET_CALL_STACK_OFFSET + stk_slot * sizeof(tcg_target_long);
1514	}
1515
1516	typedef struct TCGCumulativeArgs {
1517	int arg_idx; /* tcg_gen_callN args[] */
1518	int info_in_idx; /* TCGHelperInfo in[] */
1519	int arg_slot; /* regs+stack slot */
1520	int ref_slot; /* stack slots for references */
1521	} TCGCumulativeArgs;
1522
1523	static void layout_arg_even(TCGCumulativeArgs *cum)
1524	{
1525	cum->arg_slot += cum->arg_slot & 1;
1526	}
1527
1528	static void layout_arg_1(TCGCumulativeArgs cum, TCGHelperInfo info,
1529	TCGCallArgumentKind kind)
1530	{
1531	TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
1532
1533	*loc = (TCGCallArgumentLoc){
1534	.kind = kind,
1535	.arg_idx = cum->arg_idx,
1536	.arg_slot = cum->arg_slot,
1537	};
1538	cum->info_in_idx++;
1539	cum->arg_slot++;
1540	}
1541
1542	static void layout_arg_normal_n(TCGCumulativeArgs *cum,
1543	TCGHelperInfo *info, int n)
1544	{
1545	TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
1546
1547	for (int i = 0; i < n; ++i) {
1548	/* Layout all using the same arg_idx, adjusting the subindex. */
1549	loc[i] = (TCGCallArgumentLoc){
1550	.kind = TCG_CALL_ARG_NORMAL,
1551	.arg_idx = cum->arg_idx,
1552	.tmp_subindex = i,
1553	.arg_slot = cum->arg_slot + i,
1554	};
1555	}
1556	cum->info_in_idx += n;
1557	cum->arg_slot += n;
1558	}
1559
1560	static void layout_arg_by_ref(TCGCumulativeArgs cum, TCGHelperInfo info)
1561	{
1562	TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
1563	int n = 128 / TCG_TARGET_REG_BITS;
1564
1565	/* The first subindex carries the pointer. */
1566	layout_arg_1(cum, info, TCG_CALL_ARG_BY_REF);
1567
1568	/*
1569	* The callee is allowed to clobber memory associated with
1570	* structure pass by-reference. Therefore we must make copies.
1571	* Allocate space from "ref_slot", which will be adjusted to
1572	* follow the parameters on the stack.
1573	*/
1574	loc[0].ref_slot = cum->ref_slot;
1575
1576	/*
1577	* Subsequent words also go into the reference slot, but
1578	* do not accumulate into the regular arguments.
1579	*/
1580	for (int i = 1; i < n; ++i) {
1581	loc[i] = (TCGCallArgumentLoc){
1582	.kind = TCG_CALL_ARG_BY_REF_N,
1583	.arg_idx = cum->arg_idx,
1584	.tmp_subindex = i,
1585	.ref_slot = cum->ref_slot + i,
1586	};
1587	}
1588	cum->info_in_idx += n - 1; /* i=0 accounted for in layout_arg_1 */
1589	cum->ref_slot += n;
1590	}
1591
1592	static void init_call_layout(TCGHelperInfo *info)
1593	{
1594	int max_reg_slots = ARRAY_SIZE(tcg_target_call_iarg_regs);
1595	int max_stk_slots = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long);
1596	unsigned typemask = info->typemask;
1597	unsigned typecode;
1598	TCGCumulativeArgs cum = { };
1599
1600	/*
1601	* Parse and place any function return value.
1602	*/
1603	typecode = typemask & 7;
1604	switch (typecode) {
1605	case dh_typecode_void:
1606	info->nr_out = 0;
1607	break;
1608	case dh_typecode_i32:
1609	case dh_typecode_s32:
1610	case dh_typecode_i64:
1611	case dh_typecode_s64:
1612	case dh_typecode_ptr:
1613	info->nr_out = 1;
1614	info->out_kind = TCG_CALL_RET_NORMAL;
1615	break;
1616	case dh_typecode_i128:
1617	info->nr_out = 128 / TCG_TARGET_REG_BITS;
1618	info->out_kind = TCG_TARGET_CALL_RET_I128;
1619	switch (TCG_TARGET_CALL_RET_I128) {
1620	case TCG_CALL_RET_NORMAL:
1621	/* Query the last register now to trigger any assert early. */
1622	tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1);
1623	break;
1624	case TCG_CALL_RET_BY_VEC:
1625	/* Query the single register now to trigger any assert early. */
1626	tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0);
1627	break;
1628	case TCG_CALL_RET_BY_REF:
1629	/*
1630	* Allocate the first argument to the output.
1631	* We don't need to store this anywhere, just make it
1632	* unavailable for use in the input loop below.
1633	*/
1634	cum.arg_slot = 1;
1635	break;
1636	default:
1637	qemu_build_not_reached();
1638	}
1639	break;
1640	default:
1641	g_assert_not_reached();
1642	}
1643
1644	/*
1645	* Parse and place function arguments.
1646	*/
1647	for (typemask >>= 3; typemask; typemask >>= 3, cum.arg_idx++) {
1648	TCGCallArgumentKind kind;
1649	TCGType type;
1650
1651	typecode = typemask & 7;
1652	switch (typecode) {
1653	case dh_typecode_i32:
1654	case dh_typecode_s32:
1655	type = TCG_TYPE_I32;
1656	break;
1657	case dh_typecode_i64:
1658	case dh_typecode_s64:
1659	type = TCG_TYPE_I64;
1660	break;
1661	case dh_typecode_ptr:
1662	type = TCG_TYPE_PTR;
1663	break;
1664	case dh_typecode_i128:
1665	type = TCG_TYPE_I128;
1666	break;
1667	default:
1668	g_assert_not_reached();
1669	}
1670
1671	switch (type) {
1672	case TCG_TYPE_I32:
1673	switch (TCG_TARGET_CALL_ARG_I32) {
1674	case TCG_CALL_ARG_EVEN:
1675	layout_arg_even(&cum);
1676	/* fall through */
1677	case TCG_CALL_ARG_NORMAL:
1678	layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
1679	break;
1680	case TCG_CALL_ARG_EXTEND:
1681	kind = TCG_CALL_ARG_EXTEND_U + (typecode & 1);
1682	layout_arg_1(&cum, info, kind);
1683	break;
1684	default:
1685	qemu_build_not_reached();
1686	}
1687	break;
1688
1689	case TCG_TYPE_I64:
1690	switch (TCG_TARGET_CALL_ARG_I64) {
1691	case TCG_CALL_ARG_EVEN:
1692	layout_arg_even(&cum);
1693	/* fall through */
1694	case TCG_CALL_ARG_NORMAL:
1695	layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
1696	break;
1697	default:
1698	qemu_build_not_reached();
1699	}
1700	break;
1701
1702	case TCG_TYPE_I128:
1703	switch (TCG_TARGET_CALL_ARG_I128) {
1704	case TCG_CALL_ARG_EVEN:
1705	layout_arg_even(&cum);
1706	/* fall through */
1707	case TCG_CALL_ARG_NORMAL:
1708	layout_arg_normal_n(&cum, info, 128 / TCG_TARGET_REG_BITS);
1709	break;
1710	case TCG_CALL_ARG_BY_REF:
1711	layout_arg_by_ref(&cum, info);
1712	break;
1713	default:
1714	qemu_build_not_reached();
1715	}
1716	break;
1717
1718	default:
1719	g_assert_not_reached();
1720	}
1721	}
1722	info->nr_in = cum.info_in_idx;
1723
1724	/* Validate that we didn't overrun the input array. */
1725	assert(cum.info_in_idx <= ARRAY_SIZE(info->in));
1726	/* Validate the backend has enough argument space. */
1727	assert(cum.arg_slot <= max_reg_slots + max_stk_slots);
1728
1729	/*
1730	* Relocate the "ref_slot" area to the end of the parameters.
1731	* Minimizing this stack offset helps code size for x86,
1732	* which has a signed 8-bit offset encoding.
1733	*/
1734	if (cum.ref_slot != 0) {
1735	int ref_base = 0;
1736
1737	if (cum.arg_slot > max_reg_slots) {
1738	int align = __alignof(Int128) / sizeof(tcg_target_long);
1739
1740	ref_base = cum.arg_slot - max_reg_slots;
1741	if (align > 1) {
1742	ref_base = ROUND_UP(ref_base, align);
1743	}
1744	}
1745	assert(ref_base + cum.ref_slot <= max_stk_slots);
1746	ref_base += max_reg_slots;
1747
1748	if (ref_base != 0) {
1749	for (int i = cum.info_in_idx - 1; i >= 0; --i) {
1750	TCGCallArgumentLoc *loc = &info->in[i];
1751	switch (loc->kind) {
1752	case TCG_CALL_ARG_BY_REF:
1753	case TCG_CALL_ARG_BY_REF_N:
1754	loc->ref_slot += ref_base;
1755	break;
1756	default:
1757	break;
1758	}
1759	}
1760	}
1761	}
1762	}
1763
1764	static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
1765	static void process_constraint_sets(void);
1766	static TCGTemp tcg_global_reg_new_internal(TCGContext s, TCGType type,
1767	TCGReg reg, const char *name);
1768
1769	static void tcg_context_init(unsigned max_threads)
1770	{
1771	TCGContext *s = &tcg_init_ctx;
1772	int n, i;
1773	TCGTemp *ts;
1774
1775	memset(s, 0, sizeof(*s));
1776	s->nb_globals = 0;
1777
1778	init_call_layout(&info_helper_ld32_mmu);
1779	init_call_layout(&info_helper_ld64_mmu);
1780	init_call_layout(&info_helper_ld128_mmu);
1781	init_call_layout(&info_helper_st32_mmu);
1782	init_call_layout(&info_helper_st64_mmu);
1783	init_call_layout(&info_helper_st128_mmu);
1784
1785	tcg_target_init(s);
1786	process_constraint_sets();
1787
1788	/* Reverse the order of the saved registers, assuming they're all at
1789	the start of tcg_target_reg_alloc_order. */
1790	for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
1791	int r = tcg_target_reg_alloc_order[n];
1792	if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
1793	break;
1794	}
1795	}
1796	for (i = 0; i < n; ++i) {
1797	indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
1798	}
1799	for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
1800	indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
1801	}
1802
1803	tcg_ctx = s;
1804	/*
1805	* In user-mode we simply share the init context among threads, since we
1806	* use a single region. See the documentation tcg_region_init() for the
1807	* reasoning behind this.
1808	* In system-mode we will have at most max_threads TCG threads.
1809	*/
1810	#ifdef CONFIG_USER_ONLY
1811	tcg_ctxs = &tcg_ctx;
1812	tcg_cur_ctxs = 1;
1813	tcg_max_ctxs = 1;
1814	#else
1815	tcg_max_ctxs = max_threads;
1816	tcg_ctxs = g_new0(TCGContext *, max_threads);
1817	#endif
1818
1819	tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
1820	ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
1821	tcg_env = temp_tcgv_ptr(ts);
1822	}
1823
1824	void tcg_init(size_t tb_size, int splitwx, unsigned max_threads)
1825	{
1826	tcg_context_init(max_threads);
1827	tcg_region_init(tb_size, splitwx, max_threads);
1828	}
1829
1830	/*
1831	* Allocate TBs right before their corresponding translated code, making
1832	* sure that TBs and code are on different cache lines.
1833	*/
1834	TranslationBlock tcg_tb_alloc(TCGContext s)
1835	{
1836	uintptr_t align = qemu_icache_linesize;
1837	TranslationBlock *tb;
1838	void *next;
1839
1840	while (1) {
1841	tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
1842
1843	/*
1844	* Note that code_gen_ptr can be NULL after vCPU hotplug.
1845	* See tcg_region_thread_initial_alloc.
1846	*/
1847	if (tb) {
1848	next = (void *)ROUND_UP((uintptr_t)(tb + 1), align);
1849	if (next <= s->code_gen_highwater) {
1850	qatomic_set(&s->code_gen_ptr, next);
1851	return tb;
1852	}
1853	}
1854	if (!tcg_region_alloc(s)) {
1855	return NULL;
1856	}
1857	}
1858	}
1859
1860	void tcg_prologue_init(void)
1861	{
1862	TCGContext *s = tcg_ctx;
1863	size_t prologue_size;
1864
1865	s->code_ptr = s->code_gen_ptr;
1866	s->code_buf = s->code_gen_ptr;
1867	s->data_gen_ptr = NULL;
1868
1869	#ifndef CONFIG_TCG_INTERPRETER
1870	tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr);
1871	#endif
1872
1873	s->pool_labels = NULL;
1874
1875	qemu_thread_jit_write();
1876	/* Generate the prologue. */
1877	tcg_target_qemu_prologue(s);
1878
1879	/* Allow the prologue to put e.g. guest_base into a pool entry. */
1880	{
1881	int result = tcg_out_pool_finalize(s);
1882	tcg_debug_assert(result == 0);
1883	}
1884
1885	prologue_size = tcg_current_code_size(s);
1886	perf_report_prologue(s->code_gen_ptr, prologue_size);
1887
1888	#ifndef CONFIG_TCG_INTERPRETER
1889	flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
1890	(uintptr_t)s->code_buf, prologue_size);
1891	#endif
1892
1893	if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
1894	FILE *logfile = qemu_log_trylock();
1895	if (logfile) {
1896	fprintf(logfile, "PROLOGUE: [size=%zu]\n", prologue_size);
1897	if (s->data_gen_ptr) {
1898	size_t code_size = s->data_gen_ptr - s->code_gen_ptr;
1899	size_t data_size = prologue_size - code_size;
1900	size_t i;
1901
1902	disas(logfile, s->code_gen_ptr, code_size);
1903
1904	for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
1905	if (sizeof(tcg_target_ulong) == 8) {
1906	fprintf(logfile,
1907	"0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n",
1908	(uintptr_t)s->data_gen_ptr + i,
1909	(uint64_t )(s->data_gen_ptr + i));
1910	} else {
1911	fprintf(logfile,
1912	"0x%08" PRIxPTR ": .long 0x%08x\n",
1913	(uintptr_t)s->data_gen_ptr + i,
1914	(uint32_t )(s->data_gen_ptr + i));
1915	}
1916	}
1917	} else {
1918	disas(logfile, s->code_gen_ptr, prologue_size);
1919	}
1920	fprintf(logfile, "\n");
1921	qemu_log_unlock(logfile);
1922	}
1923	}
1924
1925	#ifndef CONFIG_TCG_INTERPRETER
1926	/*
1927	* Assert that goto_ptr is implemented completely, setting an epilogue.
1928	* For tci, we use NULL as the signal to return from the interpreter,
1929	* so skip this check.
1930	*/
1931	tcg_debug_assert(tcg_code_gen_epilogue != NULL);
1932	#endif
1933
1934	tcg_region_prologue_set(s);
1935	}
1936
1937	void tcg_func_start(TCGContext *s)
1938	{
1939	tcg_pool_reset(s);
1940	s->nb_temps = s->nb_globals;
1941
1942	/* No temps have been previously allocated for size or locality. */
1943	tcg_temp_ebb_reset_freed(s);
1944
1945	/* No constant temps have been previously allocated. */
1946	for (int i = 0; i < TCG_TYPE_COUNT; ++i) {
1947	if (s->const_table[i]) {
1948	g_hash_table_remove_all(s->const_table[i]);
1949	}
1950	}
1951
1952	s->nb_ops = 0;
1953	s->nb_labels = 0;
1954	s->current_frame_offset = s->frame_start;
1955
1956	#ifdef CONFIG_DEBUG_TCG
1957	s->goto_tb_issue_mask = 0;
1958	#endif
1959
1960	QTAILQ_INIT(&s->ops);
1961	QTAILQ_INIT(&s->free_ops);
1962	s->emit_before_op = NULL;
1963	QSIMPLEQ_INIT(&s->labels);
1964
1965	tcg_debug_assert(s->addr_type <= TCG_TYPE_REG);
1966	}
1967
1968	static TCGTemp tcg_temp_alloc(TCGContext s)
1969	{
1970	int n = s->nb_temps++;
1971
1972	if (n >= TCG_MAX_TEMPS) {
1973	tcg_raise_tb_overflow(s);
1974	}
1975	return memset(&s->temps[n], 0, sizeof(TCGTemp));
1976	}
1977
1978	static TCGTemp tcg_global_alloc(TCGContext s)
1979	{
1980	TCGTemp *ts;
1981
1982	tcg_debug_assert(s->nb_globals == s->nb_temps);
1983	tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
1984	s->nb_globals++;
1985	ts = tcg_temp_alloc(s);
1986	ts->kind = TEMP_GLOBAL;
1987
1988	return ts;
1989	}
1990
1991	static TCGTemp tcg_global_reg_new_internal(TCGContext s, TCGType type,
1992	TCGReg reg, const char *name)
1993	{
1994	TCGTemp *ts = tcg_global_alloc(s);
1995
1996	ts->base_type = type;
1997	ts->type = type;
1998	ts->kind = TEMP_FIXED;
1999	ts->reg = reg;
2000	ts->name = name;
2001	tcg_regset_set_reg(s->reserved_regs, reg);
2002
2003	return ts;
2004	}
2005
2006	void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
2007	{
2008	s->frame_start = start;
2009	s->frame_end = start + size;
2010	s->frame_temp
2011	= tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
2012	}
2013
2014	static TCGTemp *tcg_global_mem_new_internal(TCGv_ptr base, intptr_t offset,
2015	const char *name, TCGType type)
2016	{
2017	TCGContext *s = tcg_ctx;
2018	TCGTemp *base_ts = tcgv_ptr_temp(base);
2019	TCGTemp *ts = tcg_global_alloc(s);
2020	int indirect_reg = 0;
2021
2022	switch (base_ts->kind) {
2023	case TEMP_FIXED:
2024	break;
2025	case TEMP_GLOBAL:
2026	/* We do not support double-indirect registers. */
2027	tcg_debug_assert(!base_ts->indirect_reg);
2028	base_ts->indirect_base = 1;
2029	s->nb_indirects += 1;
2030	indirect_reg = 1;
2031	break;
2032	default:
2033	g_assert_not_reached();
2034	}
2035
2036	ts->base_type = type;
2037	ts->type = type;
2038	ts->indirect_reg = indirect_reg;
2039	ts->mem_allocated = 1;
2040	ts->mem_base = base_ts;
2041	ts->mem_offset = offset;
2042	ts->name = name;
2043	return ts;
2044	}
2045
2046	TCGv_i32 tcg_global_mem_new_i32(TCGv_ptr reg, intptr_t off, const char *name)
2047	{
2048	TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_I32);
2049	return temp_tcgv_i32(ts);
2050	}
2051
2052	TCGv_i64 tcg_global_mem_new_i64(TCGv_ptr reg, intptr_t off, const char *name)
2053	{
2054	TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_I64);
2055	return temp_tcgv_i64(ts);
2056	}
2057
2058	TCGv_ptr tcg_global_mem_new_ptr(TCGv_ptr reg, intptr_t off, const char *name)
2059	{
2060	TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_PTR);
2061	return temp_tcgv_ptr(ts);
2062	}
2063
2064	TCGTemp *tcg_temp_new_internal(TCGType type, TCGTempKind kind)
2065	{
2066	TCGContext *s = tcg_ctx;
2067	TCGTemp *ts;
2068	int n;
2069
2070	if (kind == TEMP_EBB) {
2071	int idx = find_first_bit(s->free_temps[type].l, TCG_MAX_TEMPS);
2072
2073	if (idx < TCG_MAX_TEMPS) {
2074	/* There is already an available temp with the right type. */
2075	clear_bit(idx, s->free_temps[type].l);
2076
2077	ts = &s->temps[idx];
2078	ts->temp_allocated = 1;
2079	tcg_debug_assert(ts->base_type == type);
2080	tcg_debug_assert(ts->kind == kind);
2081	return ts;
2082	}
2083	} else {
2084	tcg_debug_assert(kind == TEMP_TB);
2085	}
2086
2087	switch (type) {
2088	case TCG_TYPE_I32:
2089	case TCG_TYPE_I64:
2090	case TCG_TYPE_V64:
2091	case TCG_TYPE_V128:
2092	case TCG_TYPE_V256:
2093	n = 1;
2094	break;
2095	case TCG_TYPE_I128:
2096	n = 128 / TCG_TARGET_REG_BITS;
2097	break;
2098	default:
2099	g_assert_not_reached();
2100	}
2101
2102	ts = tcg_temp_alloc(s);
2103	ts->base_type = type;
2104	ts->temp_allocated = 1;
2105	ts->kind = kind;
2106
2107	if (n == 1) {
2108	ts->type = type;
2109	} else {
2110	ts->type = TCG_TYPE_REG;
2111
2112	for (int i = 1; i < n; ++i) {
2113	TCGTemp *ts2 = tcg_temp_alloc(s);
2114
2115	tcg_debug_assert(ts2 == ts + i);
2116	ts2->base_type = type;
2117	ts2->type = TCG_TYPE_REG;
2118	ts2->temp_allocated = 1;
2119	ts2->temp_subindex = i;
2120	ts2->kind = kind;
2121	}
2122	}
2123	return ts;
2124	}
2125
2126	TCGv_i32 tcg_temp_new_i32(void)
2127	{
2128	return temp_tcgv_i32(tcg_temp_new_internal(TCG_TYPE_I32, TEMP_TB));
2129	}
2130
2131	TCGv_i32 tcg_temp_ebb_new_i32(void)
2132	{
2133	return temp_tcgv_i32(tcg_temp_new_internal(TCG_TYPE_I32, TEMP_EBB));
2134	}
2135
2136	TCGv_i64 tcg_temp_new_i64(void)
2137	{
2138	return temp_tcgv_i64(tcg_temp_new_internal(TCG_TYPE_I64, TEMP_TB));
2139	}
2140
2141	TCGv_i64 tcg_temp_ebb_new_i64(void)
2142	{
2143	return temp_tcgv_i64(tcg_temp_new_internal(TCG_TYPE_I64, TEMP_EBB));
2144	}
2145
2146	TCGv_ptr tcg_temp_new_ptr(void)
2147	{
2148	return temp_tcgv_ptr(tcg_temp_new_internal(TCG_TYPE_PTR, TEMP_TB));
2149	}
2150
2151	TCGv_ptr tcg_temp_ebb_new_ptr(void)
2152	{
2153	return temp_tcgv_ptr(tcg_temp_new_internal(TCG_TYPE_PTR, TEMP_EBB));
2154	}
2155
2156	TCGv_i128 tcg_temp_new_i128(void)
2157	{
2158	return temp_tcgv_i128(tcg_temp_new_internal(TCG_TYPE_I128, TEMP_TB));
2159	}
2160
2161	TCGv_i128 tcg_temp_ebb_new_i128(void)
2162	{
2163	return temp_tcgv_i128(tcg_temp_new_internal(TCG_TYPE_I128, TEMP_EBB));
2164	}
2165
2166	TCGv_vec tcg_temp_new_vec(TCGType type)
2167	{
2168	TCGTemp *t;
2169
2170	#ifdef CONFIG_DEBUG_TCG
2171	switch (type) {
2172	case TCG_TYPE_V64:
2173	assert(TCG_TARGET_HAS_v64);
2174	break;
2175	case TCG_TYPE_V128:
2176	assert(TCG_TARGET_HAS_v128);
2177	break;
2178	case TCG_TYPE_V256:
2179	assert(TCG_TARGET_HAS_v256);
2180	break;
2181	default:
2182	g_assert_not_reached();
2183	}
2184	#endif
2185
2186	t = tcg_temp_new_internal(type, TEMP_EBB);
2187	return temp_tcgv_vec(t);
2188	}
2189
2190	/* Create a new temp of the same type as an existing temp. */
2191	TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match)
2192	{
2193	TCGTemp *t = tcgv_vec_temp(match);
2194
2195	tcg_debug_assert(t->temp_allocated != 0);
2196
2197	t = tcg_temp_new_internal(t->base_type, TEMP_EBB);
2198	return temp_tcgv_vec(t);
2199	}
2200
2201	void tcg_temp_free_internal(TCGTemp *ts)
2202	{
2203	TCGContext *s = tcg_ctx;
2204
2205	switch (ts->kind) {
2206	case TEMP_CONST:
2207	case TEMP_TB:
2208	/* Silently ignore free. */
2209	break;
2210	case TEMP_EBB:
2211	tcg_debug_assert(ts->temp_allocated != 0);
2212	ts->temp_allocated = 0;
2213	set_bit(temp_idx(ts), s->free_temps[ts->base_type].l);
2214	break;
2215	default:
2216	/* It never made sense to free TEMP_FIXED or TEMP_GLOBAL. */
2217	g_assert_not_reached();
2218	}
2219	}
2220
2221	void tcg_temp_free_i32(TCGv_i32 arg)
2222	{
2223	tcg_temp_free_internal(tcgv_i32_temp(arg));
2224	}
2225
2226	void tcg_temp_free_i64(TCGv_i64 arg)
2227	{
2228	tcg_temp_free_internal(tcgv_i64_temp(arg));
2229	}
2230
2231	void tcg_temp_free_i128(TCGv_i128 arg)
2232	{
2233	tcg_temp_free_internal(tcgv_i128_temp(arg));
2234	}
2235
2236	void tcg_temp_free_ptr(TCGv_ptr arg)
2237	{
2238	tcg_temp_free_internal(tcgv_ptr_temp(arg));
2239	}
2240
2241	void tcg_temp_free_vec(TCGv_vec arg)
2242	{
2243	tcg_temp_free_internal(tcgv_vec_temp(arg));
2244	}
2245
2246	TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
2247	{
2248	TCGContext *s = tcg_ctx;
2249	GHashTable *h = s->const_table[type];
2250	TCGTemp *ts;
2251
2252	if (h == NULL) {
2253	h = g_hash_table_new(g_int64_hash, g_int64_equal);
2254	s->const_table[type] = h;
2255	}
2256
2257	ts = g_hash_table_lookup(h, &val);
2258	if (ts == NULL) {
2259	ts = tcg_temp_alloc(s);
2260	ts->base_type = type;
2261	ts->type = type;
2262	ts->kind = TEMP_CONST;
2263	ts->temp_allocated = 1;
2264	ts->val = val;
2265	g_hash_table_insert(h, &ts->val, ts);
2266	}
2267
2268	return ts;
2269	}
2270
2271	TCGv_i32 tcg_constant_i32(int32_t val)
2272	{
2273	return temp_tcgv_i32(tcg_constant_internal(TCG_TYPE_I32, val));
2274	}
2275
2276	TCGv_i64 tcg_constant_i64(int64_t val)
2277	{
2278	return temp_tcgv_i64(tcg_constant_internal(TCG_TYPE_I64, val));
2279	}
2280
2281	TCGv_vaddr tcg_constant_vaddr(uintptr_t val)
2282	{
2283	return temp_tcgv_vaddr(tcg_constant_internal(TCG_TYPE_PTR, val));
2284	}
2285
2286	TCGv_ptr tcg_constant_ptr_int(intptr_t val)
2287	{
2288	return temp_tcgv_ptr(tcg_constant_internal(TCG_TYPE_PTR, val));
2289	}
2290
2291	TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
2292	{
2293	val = dup_const(vece, val);
2294	return temp_tcgv_vec(tcg_constant_internal(type, val));
2295	}
2296
2297	TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val)
2298	{
2299	TCGTemp *t = tcgv_vec_temp(match);
2300
2301	tcg_debug_assert(t->temp_allocated != 0);
2302	return tcg_constant_vec(t->base_type, vece, val);
2303	}
2304
2305	#ifdef CONFIG_DEBUG_TCG
2306	size_t temp_idx(TCGTemp *ts)
2307	{
2308	ptrdiff_t n = ts - tcg_ctx->temps;
2309	assert(n >= 0 && n < tcg_ctx->nb_temps);
2310	return n;
2311	}
2312
2313	TCGTemp *tcgv_i32_temp(TCGv_i32 v)
2314	{
2315	uintptr_t o = (uintptr_t)v - offsetof(TCGContext, temps);
2316
2317	assert(o < sizeof(TCGTemp) * tcg_ctx->nb_temps);
2318	assert(o % sizeof(TCGTemp) == 0);
2319
2320	return (void *)tcg_ctx + (uintptr_t)v;
2321	}
2322	#endif /* CONFIG_DEBUG_TCG */
2323
2324	/*
2325	* Return true if OP may appear in the opcode stream with TYPE.
2326	* Test the runtime variable that controls each opcode.
2327	*/
2328	bool tcg_op_supported(TCGOpcode op, TCGType type, unsigned flags)
2329	{
2330	bool has_type;
2331
2332	switch (type) {
2333	case TCG_TYPE_I32:
2334	case TCG_TYPE_I64:
2335	has_type = true;
2336	break;
2337	case TCG_TYPE_V64:
2338	has_type = TCG_TARGET_HAS_v64;
2339	break;
2340	case TCG_TYPE_V128:
2341	has_type = TCG_TARGET_HAS_v128;
2342	break;
2343	case TCG_TYPE_V256:
2344	has_type = TCG_TARGET_HAS_v256;
2345	break;
2346	default:
2347	has_type = false;
2348	break;
2349	}
2350
2351	switch (op) {
2352	case INDEX_op_discard:
2353	case INDEX_op_set_label:
2354	case INDEX_op_call:
2355	case INDEX_op_br:
2356	case INDEX_op_mb:
2357	case INDEX_op_insn_start:
2358	case INDEX_op_exit_tb:
2359	case INDEX_op_goto_tb:
2360	case INDEX_op_goto_ptr:
2361	return true;
2362
2363	case INDEX_op_qemu_ld:
2364	case INDEX_op_qemu_st:
2365	tcg_debug_assert(type <= TCG_TYPE_REG);
2366	return true;
2367
2368	case INDEX_op_qemu_ld2:
2369	case INDEX_op_qemu_st2:
2370	tcg_debug_assert(type == TCG_TYPE_I128);
2371	goto do_lookup;
2372
2373	case INDEX_op_add:
2374	case INDEX_op_and:
2375	case INDEX_op_brcond:
2376	case INDEX_op_deposit:
2377	case INDEX_op_extract:
2378	case INDEX_op_ld8u:
2379	case INDEX_op_ld8s:
2380	case INDEX_op_ld16u:
2381	case INDEX_op_ld16s:
2382	case INDEX_op_ld:
2383	case INDEX_op_mov:
2384	case INDEX_op_movcond:
2385	case INDEX_op_negsetcond:
2386	case INDEX_op_or:
2387	case INDEX_op_setcond:
2388	case INDEX_op_sextract:
2389	case INDEX_op_st8:
2390	case INDEX_op_st16:
2391	case INDEX_op_st:
2392	case INDEX_op_xor:
2393	return has_type;
2394
2395	case INDEX_op_ld32u:
2396	case INDEX_op_ld32s:
2397	case INDEX_op_st32:
2398	case INDEX_op_ext_i32_i64:
2399	case INDEX_op_extu_i32_i64:
2400	case INDEX_op_extrl_i64_i32:
2401	case INDEX_op_extrh_i64_i32:
2402	return true;
2403
2404	case INDEX_op_mov_vec:
2405	case INDEX_op_dup_vec:
2406	case INDEX_op_dupm_vec:
2407	case INDEX_op_ld_vec:
2408	case INDEX_op_st_vec:
2409	case INDEX_op_add_vec:
2410	case INDEX_op_sub_vec:
2411	case INDEX_op_and_vec:
2412	case INDEX_op_or_vec:
2413	case INDEX_op_xor_vec:
2414	case INDEX_op_cmp_vec:
2415	return has_type;
2416	case INDEX_op_not_vec:
2417	return has_type && TCG_TARGET_HAS_not_vec;
2418	case INDEX_op_neg_vec:
2419	return has_type && TCG_TARGET_HAS_neg_vec;
2420	case INDEX_op_abs_vec:
2421	return has_type && TCG_TARGET_HAS_abs_vec;
2422	case INDEX_op_andc_vec:
2423	return has_type && TCG_TARGET_HAS_andc_vec;
2424	case INDEX_op_orc_vec:
2425	return has_type && TCG_TARGET_HAS_orc_vec;
2426	case INDEX_op_nand_vec:
2427	return has_type && TCG_TARGET_HAS_nand_vec;
2428	case INDEX_op_nor_vec:
2429	return has_type && TCG_TARGET_HAS_nor_vec;
2430	case INDEX_op_eqv_vec:
2431	return has_type && TCG_TARGET_HAS_eqv_vec;
2432	case INDEX_op_mul_vec:
2433	return has_type && TCG_TARGET_HAS_mul_vec;
2434	case INDEX_op_shli_vec:
2435	case INDEX_op_shri_vec:
2436	case INDEX_op_sari_vec:
2437	return has_type && TCG_TARGET_HAS_shi_vec;
2438	case INDEX_op_shls_vec:
2439	case INDEX_op_shrs_vec:
2440	case INDEX_op_sars_vec:
2441	return has_type && TCG_TARGET_HAS_shs_vec;
2442	case INDEX_op_shlv_vec:
2443	case INDEX_op_shrv_vec:
2444	case INDEX_op_sarv_vec:
2445	return has_type && TCG_TARGET_HAS_shv_vec;
2446	case INDEX_op_rotli_vec:
2447	return has_type && TCG_TARGET_HAS_roti_vec;
2448	case INDEX_op_rotls_vec:
2449	return has_type && TCG_TARGET_HAS_rots_vec;
2450	case INDEX_op_rotlv_vec:
2451	case INDEX_op_rotrv_vec:
2452	return has_type && TCG_TARGET_HAS_rotv_vec;
2453	case INDEX_op_ssadd_vec:
2454	case INDEX_op_usadd_vec:
2455	case INDEX_op_sssub_vec:
2456	case INDEX_op_ussub_vec:
2457	return has_type && TCG_TARGET_HAS_sat_vec;
2458	case INDEX_op_smin_vec:
2459	case INDEX_op_umin_vec:
2460	case INDEX_op_smax_vec:
2461	case INDEX_op_umax_vec:
2462	return has_type && TCG_TARGET_HAS_minmax_vec;
2463	case INDEX_op_bitsel_vec:
2464	return has_type && TCG_TARGET_HAS_bitsel_vec;
2465	case INDEX_op_cmpsel_vec:
2466	return has_type && TCG_TARGET_HAS_cmpsel_vec;
2467
2468	default:
2469	if (op < INDEX_op_last_generic) {
2470	const TCGOutOp *outop;
2471	TCGConstraintSetIndex con_set;
2472
2473	if (!has_type) {
2474	return false;
2475	}
2476
2477	do_lookup:
2478	outop = all_outop[op];
2479	tcg_debug_assert(outop != NULL);
2480
2481	con_set = outop->static_constraint;
2482	if (con_set == C_Dynamic) {
2483	con_set = outop->dynamic_constraint(type, flags);
2484	}
2485	if (con_set >= 0) {
2486	return true;
2487	}
2488	tcg_debug_assert(con_set == C_NotImplemented);
2489	return false;
2490	}
2491	tcg_debug_assert(op < NB_OPS);
2492	return true;
2493
2494	case INDEX_op_last_generic:
2495	g_assert_not_reached();
2496	}
2497	}
2498
2499	bool tcg_op_deposit_valid(TCGType type, unsigned ofs, unsigned len)
2500	{
2501	unsigned width;
2502
2503	tcg_debug_assert(type == TCG_TYPE_I32 \|\| type == TCG_TYPE_I64);
2504	width = (type == TCG_TYPE_I32 ? 32 : 64);
2505
2506	tcg_debug_assert(ofs < width);
2507	tcg_debug_assert(len > 0);
2508	tcg_debug_assert(len <= width - ofs);
2509
2510	return TCG_TARGET_deposit_valid(type, ofs, len);
2511	}
2512
2513	static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs);
2514
2515	static void tcg_gen_callN(void func, TCGHelperInfo info,
2516	TCGTemp ret, TCGTemp *args)
2517	{
2518	TCGv_i64 extend_free[MAX_CALL_IARGS];
2519	int n_extend = 0;
2520	TCGOp *op;
2521	int i, n, pi = 0, total_args;
2522
2523	if (unlikely(g_once_init_enter(HELPER_INFO_INIT(info)))) {
2524	init_call_layout(info);
2525	g_once_init_leave(HELPER_INFO_INIT(info), HELPER_INFO_INIT_VAL(info));
2526	}
2527
2528	total_args = info->nr_out + info->nr_in + 2;
2529	op = tcg_op_alloc(INDEX_op_call, total_args);
2530
2531	#ifdef CONFIG_PLUGIN
2532	/* Flag helpers that may affect guest state */
2533	if (tcg_ctx->plugin_insn && !(info->flags & TCG_CALL_NO_SIDE_EFFECTS)) {
2534	tcg_ctx->plugin_insn->calls_helpers = true;
2535	}
2536	#endif
2537
2538	TCGOP_CALLO(op) = n = info->nr_out;
2539	switch (n) {
2540	case 0:
2541	tcg_debug_assert(ret == NULL);
2542	break;
2543	case 1:
2544	tcg_debug_assert(ret != NULL);
2545	op->args[pi++] = temp_arg(ret);
2546	break;
2547	case 2:
2548	case 4:
2549	tcg_debug_assert(ret != NULL);
2550	tcg_debug_assert(ret->base_type == ret->type + ctz32(n));
2551	tcg_debug_assert(ret->temp_subindex == 0);
2552	for (i = 0; i < n; ++i) {
2553	op->args[pi++] = temp_arg(ret + i);
2554	}
2555	break;
2556	default:
2557	g_assert_not_reached();
2558	}
2559
2560	TCGOP_CALLI(op) = n = info->nr_in;
2561	for (i = 0; i < n; i++) {
2562	const TCGCallArgumentLoc *loc = &info->in[i];
2563	TCGTemp *ts = args[loc->arg_idx] + loc->tmp_subindex;
2564
2565	switch (loc->kind) {
2566	case TCG_CALL_ARG_NORMAL:
2567	case TCG_CALL_ARG_BY_REF:
2568	case TCG_CALL_ARG_BY_REF_N:
2569	op->args[pi++] = temp_arg(ts);
2570	break;
2571
2572	case TCG_CALL_ARG_EXTEND_U:
2573	case TCG_CALL_ARG_EXTEND_S:
2574	{
2575	TCGv_i64 temp = tcg_temp_ebb_new_i64();
2576	TCGv_i32 orig = temp_tcgv_i32(ts);
2577
2578	if (loc->kind == TCG_CALL_ARG_EXTEND_S) {
2579	tcg_gen_ext_i32_i64(temp, orig);
2580	} else {
2581	tcg_gen_extu_i32_i64(temp, orig);
2582	}
2583	op->args[pi++] = tcgv_i64_arg(temp);
2584	extend_free[n_extend++] = temp;
2585	}
2586	break;
2587
2588	default:
2589	g_assert_not_reached();
2590	}
2591	}
2592	op->args[pi++] = (uintptr_t)func;
2593	op->args[pi++] = (uintptr_t)info;
2594	tcg_debug_assert(pi == total_args);
2595
2596	if (tcg_ctx->emit_before_op) {
2597	QTAILQ_INSERT_BEFORE(tcg_ctx->emit_before_op, op, link);
2598	} else {
2599	QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
2600	}
2601
2602	tcg_debug_assert(n_extend < ARRAY_SIZE(extend_free));
2603	for (i = 0; i < n_extend; ++i) {
2604	tcg_temp_free_i64(extend_free[i]);
2605	}
2606	}
2607
2608	void tcg_gen_call0(void func, TCGHelperInfo info, TCGTemp *ret)
2609	{
2610	tcg_gen_callN(func, info, ret, NULL);
2611	}
2612
2613	void tcg_gen_call1(void func, TCGHelperInfo info, TCGTemp ret, TCGTemp t1)
2614	{
2615	tcg_gen_callN(func, info, ret, &t1);
2616	}
2617
2618	void tcg_gen_call2(void func, TCGHelperInfo info, TCGTemp *ret,
2619	TCGTemp t1, TCGTemp t2)
2620	{
2621	TCGTemp *args[2] = { t1, t2 };
2622	tcg_gen_callN(func, info, ret, args);
2623	}
2624
2625	void tcg_gen_call3(void func, TCGHelperInfo info, TCGTemp *ret,
2626	TCGTemp t1, TCGTemp t2, TCGTemp *t3)
2627	{
2628	TCGTemp *args[3] = { t1, t2, t3 };
2629	tcg_gen_callN(func, info, ret, args);
2630	}
2631
2632	void tcg_gen_call4(void func, TCGHelperInfo info, TCGTemp *ret,
2633	TCGTemp t1, TCGTemp t2, TCGTemp t3, TCGTemp t4)
2634	{
2635	TCGTemp *args[4] = { t1, t2, t3, t4 };
2636	tcg_gen_callN(func, info, ret, args);
2637	}
2638
2639	void tcg_gen_call5(void func, TCGHelperInfo info, TCGTemp ret, TCGTemp t1,
2640	TCGTemp t2, TCGTemp t3, TCGTemp t4, TCGTemp t5)
2641	{
2642	TCGTemp *args[5] = { t1, t2, t3, t4, t5 };
2643	tcg_gen_callN(func, info, ret, args);
2644	}
2645
2646	void tcg_gen_call6(void func, TCGHelperInfo info, TCGTemp *ret,
2647	TCGTemp t1, TCGTemp t2, TCGTemp *t3,
2648	TCGTemp t4, TCGTemp t5, TCGTemp *t6)
2649	{
2650	TCGTemp *args[6] = { t1, t2, t3, t4, t5, t6 };
2651	tcg_gen_callN(func, info, ret, args);
2652	}
2653
2654	void tcg_gen_call7(void func, TCGHelperInfo info, TCGTemp ret, TCGTemp t1,
2655	TCGTemp t2, TCGTemp t3, TCGTemp *t4,
2656	TCGTemp t5, TCGTemp t6, TCGTemp *t7)
2657	{
2658	TCGTemp *args[7] = { t1, t2, t3, t4, t5, t6, t7 };
2659	tcg_gen_callN(func, info, ret, args);
2660	}
2661
2662	static void tcg_reg_alloc_start(TCGContext *s)
2663	{
2664	int i, n;
2665
2666	for (i = 0, n = s->nb_temps; i < n; i++) {
2667	TCGTemp *ts = &s->temps[i];
2668	TCGTempVal val = TEMP_VAL_MEM;
2669
2670	switch (ts->kind) {
2671	case TEMP_CONST:
2672	val = TEMP_VAL_CONST;
2673	break;
2674	case TEMP_FIXED:
2675	val = TEMP_VAL_REG;
2676	break;
2677	case TEMP_GLOBAL:
2678	break;
2679	case TEMP_EBB:
2680	val = TEMP_VAL_DEAD;
2681	/* fall through */
2682	case TEMP_TB:
2683	ts->mem_allocated = 0;
2684	break;
2685	default:
2686	g_assert_not_reached();
2687	}
2688	ts->val_type = val;
2689	}
2690
2691	memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
2692	}
2693
2694	static char tcg_get_arg_str_ptr(TCGContext s, char *buf, int buf_size,
2695	TCGTemp *ts)
2696	{
2697	int idx = temp_idx(ts);
2698
2699	switch (ts->kind) {
2700	case TEMP_FIXED:
2701	case TEMP_GLOBAL:
2702	pstrcpy(buf, buf_size, ts->name);
2703	break;
2704	case TEMP_TB:
2705	snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
2706	break;
2707	case TEMP_EBB:
2708	snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
2709	break;
2710	case TEMP_CONST:
2711	switch (ts->type) {
2712	case TCG_TYPE_I32:
2713	snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
2714	break;
2715	case TCG_TYPE_I64:
2716	snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
2717	break;
2718	case TCG_TYPE_V64:
2719	case TCG_TYPE_V128:
2720	case TCG_TYPE_V256:
2721	snprintf(buf, buf_size, "v%d$0x%" PRIx64,
2722	64 << (ts->type - TCG_TYPE_V64), ts->val);
2723	break;
2724	default:
2725	g_assert_not_reached();
2726	}
2727	break;
2728	}
2729	return buf;
2730	}
2731
2732	static char tcg_get_arg_str(TCGContext s, char *buf,
2733	int buf_size, TCGArg arg)
2734	{
2735	return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
2736	}
2737
2738	static const char * const cond_name[] =
2739	{
2740	[TCG_COND_NEVER] = "never",
2741	[TCG_COND_ALWAYS] = "always",
2742	[TCG_COND_EQ] = "eq",
2743	[TCG_COND_NE] = "ne",
2744	[TCG_COND_LT] = "lt",
2745	[TCG_COND_GE] = "ge",
2746	[TCG_COND_LE] = "le",
2747	[TCG_COND_GT] = "gt",
2748	[TCG_COND_LTU] = "ltu",
2749	[TCG_COND_GEU] = "geu",
2750	[TCG_COND_LEU] = "leu",
2751	[TCG_COND_GTU] = "gtu",
2752	[TCG_COND_TSTEQ] = "tsteq",
2753	[TCG_COND_TSTNE] = "tstne",
2754	};
2755
2756	static const char * const ldst_name[(MO_BSWAP \| MO_SSIZE) + 1] =
2757	{
2758	[MO_UB] = "ub",
2759	[MO_SB] = "sb",
2760	[MO_LEUW] = "leuw",
2761	[MO_LESW] = "lesw",
2762	[MO_LEUL] = "leul",
2763	[MO_LESL] = "lesl",
2764	[MO_LEUQ] = "leq",
2765	[MO_BEUW] = "beuw",
2766	[MO_BESW] = "besw",
2767	[MO_BEUL] = "beul",
2768	[MO_BESL] = "besl",
2769	[MO_BEUQ] = "beq",
2770	[MO_128 + MO_BE] = "beo",
2771	[MO_128 + MO_LE] = "leo",
2772	};
2773
2774	static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = {
2775	[MO_UNALN >> MO_ASHIFT] = "un+",
2776	[MO_ALIGN >> MO_ASHIFT] = "al+",
2777	[MO_ALIGN_2 >> MO_ASHIFT] = "al2+",
2778	[MO_ALIGN_4 >> MO_ASHIFT] = "al4+",
2779	[MO_ALIGN_8 >> MO_ASHIFT] = "al8+",
2780	[MO_ALIGN_16 >> MO_ASHIFT] = "al16+",
2781	[MO_ALIGN_32 >> MO_ASHIFT] = "al32+",
2782	[MO_ALIGN_64 >> MO_ASHIFT] = "al64+",
2783	};
2784
2785	static const char * const atom_name[(MO_ATOM_MASK >> MO_ATOM_SHIFT) + 1] = {
2786	[MO_ATOM_IFALIGN >> MO_ATOM_SHIFT] = "",
2787	[MO_ATOM_IFALIGN_PAIR >> MO_ATOM_SHIFT] = "pair+",
2788	[MO_ATOM_WITHIN16 >> MO_ATOM_SHIFT] = "w16+",
2789	[MO_ATOM_WITHIN16_PAIR >> MO_ATOM_SHIFT] = "w16p+",
2790	[MO_ATOM_SUBALIGN >> MO_ATOM_SHIFT] = "sub+",
2791	[MO_ATOM_NONE >> MO_ATOM_SHIFT] = "noat+",
2792	};
2793
2794	static const char bswap_flag_name[][6] = {
2795	[TCG_BSWAP_IZ] = "iz",
2796	[TCG_BSWAP_OZ] = "oz",
2797	[TCG_BSWAP_OS] = "os",
2798	[TCG_BSWAP_IZ \| TCG_BSWAP_OZ] = "iz,oz",
2799	[TCG_BSWAP_IZ \| TCG_BSWAP_OS] = "iz,os",
2800	};
2801
2802	#ifdef CONFIG_PLUGIN
2803	static const char * const plugin_from_name[] = {
2804	"from-tb",
2805	"from-insn",
2806	"after-insn",
2807	"after-tb",
2808	};
2809	#endif
2810
2811	static inline bool tcg_regset_single(TCGRegSet d)
2812	{
2813	return (d & (d - 1)) == 0;
2814	}
2815
2816	static inline TCGReg tcg_regset_first(TCGRegSet d)
2817	{
2818	if (TCG_TARGET_NB_REGS <= 32) {
2819	return ctz32(d);
2820	} else {
2821	return ctz64(d);
2822	}
2823	}
2824
2825	/* Return only the number of characters output -- no error return. */
2826	#define ne_fprintf(...) \
2827	({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; })
2828
2829	void tcg_dump_ops(TCGContext s, FILE f, bool have_prefs)
2830	{
2831	char buf[128];
2832	TCGOp *op;
2833
2834	QTAILQ_FOREACH(op, &s->ops, link) {
2835	int i, k, nb_oargs, nb_iargs, nb_cargs;
2836	const TCGOpDef *def;
2837	TCGOpcode c;
2838	int col = 0;
2839
2840	c = op->opc;
2841	def = &tcg_op_defs[c];
2842
2843	if (c == INDEX_op_insn_start) {
2844	nb_oargs = 0;
2845	col += ne_fprintf(f, "\n ----");
2846
2847	for (i = 0, k = INSN_START_WORDS; i < k; ++i) {
2848	col += ne_fprintf(f, " %016" PRIx64,
2849	tcg_get_insn_start_param(op, i));
2850	}
2851	} else if (c == INDEX_op_call) {
2852	const TCGHelperInfo *info = tcg_call_info(op);
2853	void *func = tcg_call_func(op);
2854
2855	/* variable number of arguments */
2856	nb_oargs = TCGOP_CALLO(op);
2857	nb_iargs = TCGOP_CALLI(op);
2858	nb_cargs = def->nb_cargs;
2859
2860	col += ne_fprintf(f, " %s ", def->name);
2861
2862	/*
2863	* Print the function name from TCGHelperInfo, if available.
2864	* Note that plugins have a template function for the info,
2865	* but the actual function pointer comes from the plugin.
2866	*/
2867	if (func == info->func) {
2868	col += ne_fprintf(f, "%s", info->name);
2869	} else {
2870	col += ne_fprintf(f, "plugin(%p)", func);
2871	}
2872
2873	col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs);
2874	for (i = 0; i < nb_oargs; i++) {
2875	col += ne_fprintf(f, ",%s", tcg_get_arg_str(s, buf, sizeof(buf),
2876	op->args[i]));
2877	}
2878	for (i = 0; i < nb_iargs; i++) {
2879	TCGArg arg = op->args[nb_oargs + i];
2880	const char *t = tcg_get_arg_str(s, buf, sizeof(buf), arg);
2881	col += ne_fprintf(f, ",%s", t);
2882	}
2883	} else {
2884	if (def->flags & TCG_OPF_INT) {
2885	col += ne_fprintf(f, " %s_i%d ",
2886	def->name,
2887	8 * tcg_type_size(TCGOP_TYPE(op)));
2888	} else if (def->flags & TCG_OPF_VECTOR) {
2889	col += ne_fprintf(f, "%s v%d,e%d,",
2890	def->name,
2891	8 * tcg_type_size(TCGOP_TYPE(op)),
2892	8 << TCGOP_VECE(op));
2893	} else {
2894	col += ne_fprintf(f, " %s ", def->name);
2895	}
2896
2897	nb_oargs = def->nb_oargs;
2898	nb_iargs = def->nb_iargs;
2899	nb_cargs = def->nb_cargs;
2900
2901	k = 0;
2902	for (i = 0; i < nb_oargs; i++) {
2903	const char *sep = k ? "," : "";
2904	col += ne_fprintf(f, "%s%s", sep,
2905	tcg_get_arg_str(s, buf, sizeof(buf),
2906	op->args[k++]));
2907	}
2908	for (i = 0; i < nb_iargs; i++) {
2909	const char *sep = k ? "," : "";
2910	col += ne_fprintf(f, "%s%s", sep,
2911	tcg_get_arg_str(s, buf, sizeof(buf),
2912	op->args[k++]));
2913	}
2914	switch (c) {
2915	case INDEX_op_brcond:
2916	case INDEX_op_setcond:
2917	case INDEX_op_negsetcond:
2918	case INDEX_op_movcond:
2919	case INDEX_op_cmp_vec:
2920	case INDEX_op_cmpsel_vec:
2921	if (op->args[k] < ARRAY_SIZE(cond_name)
2922	&& cond_name[op->args[k]]) {
2923	col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]);
2924	} else {
2925	col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]);
2926	}
2927	i = 1;
2928	break;
2929	case INDEX_op_qemu_ld:
2930	case INDEX_op_qemu_st:
2931	case INDEX_op_qemu_ld2:
2932	case INDEX_op_qemu_st2:
2933	{
2934	const char s_al, s_tlb, s_op, s_at;
2935	MemOpIdx oi = op->args[k++];
2936	MemOp mop = get_memop(oi);
2937	unsigned ix = get_mmuidx(oi);
2938
2939	s_tlb = mop & MO_ALIGN_TLB_ONLY ? "tlb+" : "";
2940	s_al = alignment_name[(mop & MO_AMASK) >> MO_ASHIFT];
2941	s_op = ldst_name[mop & (MO_BSWAP \| MO_SSIZE)];
2942	s_at = atom_name[(mop & MO_ATOM_MASK) >> MO_ATOM_SHIFT];
2943	mop &= ~(MO_AMASK \| MO_BSWAP \| MO_SSIZE \|
2944	MO_ATOM_MASK \| MO_ALIGN_TLB_ONLY);
2945
2946	/* If all fields are accounted for, print symbolically. */
2947	if (!mop && s_al && s_op && s_at) {
2948	col += ne_fprintf(f, ",%s%s%s%s,%u",
2949	s_at, s_al, s_tlb, s_op, ix);
2950	} else {
2951	mop = get_memop(oi);
2952	col += ne_fprintf(f, ",$0x%x,%u", mop, ix);
2953	}
2954	i = 1;
2955	}
2956	break;
2957	case INDEX_op_bswap16:
2958	case INDEX_op_bswap32:
2959	case INDEX_op_bswap64:
2960	case INDEX_op_revbit32:
2961	{
2962	TCGArg flags = op->args[k];
2963	const char *name = NULL;
2964
2965	if (flags < ARRAY_SIZE(bswap_flag_name)) {
2966	name = bswap_flag_name[flags];
2967	}
2968	if (name && name[0]) {
2969	col += ne_fprintf(f, ",%s", name);
2970	} else {
2971	col += ne_fprintf(f, ",$0x%" TCG_PRIlx, flags);
2972	}
2973	i = k = 1;
2974	}
2975	break;
2976	#ifdef CONFIG_PLUGIN
2977	case INDEX_op_plugin_cb:
2978	{
2979	TCGArg from = op->args[k++];
2980	const char *name = NULL;
2981
2982	if (from < ARRAY_SIZE(plugin_from_name)) {
2983	name = plugin_from_name[from];
2984	}
2985	if (name) {
2986	col += ne_fprintf(f, "%s", name);
2987	} else {
2988	col += ne_fprintf(f, "$0x%" TCG_PRIlx, from);
2989	}
2990	i = 1;
2991	}
2992	break;
2993	#endif
2994	default:
2995	i = 0;
2996	break;
2997	}
2998	switch (c) {
2999	case INDEX_op_set_label:
3000	case INDEX_op_br:
3001	case INDEX_op_brcond:
3002	col += ne_fprintf(f, "%s$L%d", k ? "," : "",
3003	arg_label(op->args[k])->id);
3004	i++, k++;
3005	break;
3006	case INDEX_op_mb:
3007	{
3008	TCGBar membar = op->args[k];
3009	const char b_op, m_op;
3010
3011	switch (membar & TCG_BAR_SC) {
3012	case 0:
3013	b_op = "none";
3014	break;
3015	case TCG_BAR_LDAQ:
3016	b_op = "acq";
3017	break;
3018	case TCG_BAR_STRL:
3019	b_op = "rel";
3020	break;
3021	case TCG_BAR_SC:
3022	b_op = "seq";
3023	break;
3024	default:
3025	g_assert_not_reached();
3026	}
3027
3028	switch (membar & TCG_MO_ALL) {
3029	case 0:
3030	m_op = "none";
3031	break;
3032	case TCG_MO_LD_LD:
3033	m_op = "rr";
3034	break;
3035	case TCG_MO_LD_ST:
3036	m_op = "rw";
3037	break;
3038	case TCG_MO_ST_LD:
3039	m_op = "wr";
3040	break;
3041	case TCG_MO_ST_ST:
3042	m_op = "ww";
3043	break;
3044	case TCG_MO_LD_LD \| TCG_MO_LD_ST:
3045	m_op = "rr+rw";
3046	break;
3047	case TCG_MO_LD_LD \| TCG_MO_ST_LD:
3048	m_op = "rr+wr";
3049	break;
3050	case TCG_MO_LD_LD \| TCG_MO_ST_ST:
3051	m_op = "rr+ww";
3052	break;
3053	case TCG_MO_LD_ST \| TCG_MO_ST_LD:
3054	m_op = "rw+wr";
3055	break;
3056	case TCG_MO_LD_ST \| TCG_MO_ST_ST:
3057	m_op = "rw+ww";
3058	break;
3059	case TCG_MO_ST_LD \| TCG_MO_ST_ST:
3060	m_op = "wr+ww";
3061	break;
3062	case TCG_MO_LD_LD \| TCG_MO_LD_ST \| TCG_MO_ST_LD:
3063	m_op = "rr+rw+wr";
3064	break;
3065	case TCG_MO_LD_LD \| TCG_MO_LD_ST \| TCG_MO_ST_ST:
3066	m_op = "rr+rw+ww";
3067	break;
3068	case TCG_MO_LD_LD \| TCG_MO_ST_LD \| TCG_MO_ST_ST:
3069	m_op = "rr+wr+ww";
3070	break;
3071	case TCG_MO_LD_ST \| TCG_MO_ST_LD \| TCG_MO_ST_ST:
3072	m_op = "rw+wr+ww";
3073	break;
3074	case TCG_MO_ALL:
3075	m_op = "all";
3076	break;
3077	default:
3078	g_assert_not_reached();
3079	}
3080
3081	col += ne_fprintf(f, "%s%s:%s", (k ? "," : ""), b_op, m_op);
3082	i++, k++;
3083	}
3084	break;
3085	default:
3086	break;
3087	}
3088	for (; i < nb_cargs; i++, k++) {
3089	col += ne_fprintf(f, "%s$0x%" TCG_PRIlx, k ? "," : "",
3090	op->args[k]);
3091	}
3092	}
3093
3094	if (have_prefs \|\| op->life) {
3095	for (; col < 40; ++col) {
3096	putc(' ', f);
3097	}
3098	}
3099
3100	if (op->life) {
3101	unsigned life = op->life;
3102
3103	if (life & (SYNC_ARG * 3)) {
3104	ne_fprintf(f, " sync:");
3105	for (i = 0; i < 2; ++i) {
3106	if (life & (SYNC_ARG << i)) {
3107	ne_fprintf(f, " %d", i);
3108	}
3109	}
3110	}
3111	life /= DEAD_ARG;
3112	if (life) {
3113	ne_fprintf(f, " dead:");
3114	for (i = 0; life; ++i, life >>= 1) {
3115	if (life & 1) {
3116	ne_fprintf(f, " %d", i);
3117	}
3118	}
3119	}
3120	}
3121
3122	if (have_prefs) {
3123	for (i = 0; i < nb_oargs; ++i) {
3124	TCGRegSet set = output_pref(op, i);
3125
3126	if (i == 0) {
3127	ne_fprintf(f, " pref=");
3128	} else {
3129	ne_fprintf(f, ",");
3130	}
3131	if (set == 0) {
3132	ne_fprintf(f, "none");
3133	} else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) {
3134	ne_fprintf(f, "all");
3135	#ifdef CONFIG_DEBUG_TCG
3136	} else if (tcg_regset_single(set)) {
3137	TCGReg reg = tcg_regset_first(set);
3138	ne_fprintf(f, "%s", tcg_target_reg_names[reg]);
3139	#endif
3140	} else if (TCG_TARGET_NB_REGS <= 32) {
3141	ne_fprintf(f, "0x%x", (uint32_t)set);
3142	} else {
3143	ne_fprintf(f, "0x%" PRIx64, (uint64_t)set);
3144	}
3145	}
3146	}
3147
3148	putc('\n', f);
3149	}
3150	}
3151
3152	/* we give more priority to constraints with less registers */
3153	static int get_constraint_priority(const TCGArgConstraint *arg_ct, int k)
3154	{
3155	int n;
3156
3157	arg_ct += k;
3158	n = ctpop64(arg_ct->regs);
3159
3160	/*
3161	* Sort constraints of a single register first, which includes output
3162	* aliases (which must exactly match the input already allocated).
3163	*/
3164	if (n == 1 \|\| arg_ct->oalias) {
3165	return INT_MAX;
3166	}
3167
3168	/*
3169	* Sort register pairs next, first then second immediately after.
3170	* Arbitrarily sort multiple pairs by the index of the first reg;
3171	* there shouldn't be many pairs.
3172	*/
3173	switch (arg_ct->pair) {
3174	case 1:
3175	case 3:
3176	return (k + 1) * 2;
3177	case 2:
3178	return (arg_ct->pair_index + 1) * 2 - 1;
3179	}
3180
3181	/* Finally, sort by decreasing register count. */
3182	assert(n > 1);
3183	return -n;
3184	}
3185
3186	/* sort from highest priority to lowest */
3187	static void sort_constraints(TCGArgConstraint *a, int start, int n)
3188	{
3189	int i, j;
3190
3191	for (i = 0; i < n; i++) {
3192	a[start + i].sort_index = start + i;
3193	}
3194	if (n <= 1) {
3195	return;
3196	}
3197	for (i = 0; i < n - 1; i++) {
3198	for (j = i + 1; j < n; j++) {
3199	int p1 = get_constraint_priority(a, a[start + i].sort_index);
3200	int p2 = get_constraint_priority(a, a[start + j].sort_index);
3201	if (p1 < p2) {
3202	int tmp = a[start + i].sort_index;
3203	a[start + i].sort_index = a[start + j].sort_index;
3204	a[start + j].sort_index = tmp;
3205	}
3206	}
3207	}
3208	}
3209
3210	static const TCGArgConstraint empty_cts[TCG_MAX_OP_ARGS];
3211	static TCGArgConstraint all_cts[ARRAY_SIZE(constraint_sets)][TCG_MAX_OP_ARGS];
3212
3213	static void process_constraint_sets(void)
3214	{
3215	for (size_t c = 0; c < ARRAY_SIZE(constraint_sets); ++c) {
3216	const TCGConstraintSet *tdefs = &constraint_sets[c];
3217	TCGArgConstraint *args_ct = all_cts[c];
3218	int nb_oargs = tdefs->nb_oargs;
3219	int nb_iargs = tdefs->nb_iargs;
3220	int nb_args = nb_oargs + nb_iargs;
3221	bool saw_alias_pair = false;
3222
3223	for (int i = 0; i < nb_args; i++) {
3224	const char *ct_str = tdefs->args_ct_str[i];
3225	bool input_p = i >= nb_oargs;
3226	int o;
3227
3228	switch (*ct_str) {
3229	case '0' ... '9':
3230	o = *ct_str - '0';
3231	tcg_debug_assert(input_p);
3232	tcg_debug_assert(o < nb_oargs);
3233	tcg_debug_assert(args_ct[o].regs != 0);
3234	tcg_debug_assert(!args_ct[o].oalias);
3235	args_ct[i] = args_ct[o];
3236	/* The output sets oalias. */
3237	args_ct[o].oalias = 1;
3238	args_ct[o].alias_index = i;
3239	/* The input sets ialias. */
3240	args_ct[i].ialias = 1;
3241	args_ct[i].alias_index = o;
3242	if (args_ct[i].pair) {
3243	saw_alias_pair = true;
3244	}
3245	tcg_debug_assert(ct_str[1] == '\0');
3246	continue;
3247
3248	case '&':
3249	tcg_debug_assert(!input_p);
3250	args_ct[i].newreg = true;
3251	ct_str++;
3252	break;
3253
3254	case 'p': /* plus */
3255	/* Allocate to the register after the previous. */
3256	tcg_debug_assert(i > (input_p ? nb_oargs : 0));
3257	o = i - 1;
3258	tcg_debug_assert(!args_ct[o].pair);
3259	tcg_debug_assert(!args_ct[o].ct);
3260	args_ct[i] = (TCGArgConstraint){
3261	.pair = 2,
3262	.pair_index = o,
3263	.regs = args_ct[o].regs << 1,
3264	.newreg = args_ct[o].newreg,
3265	};
3266	args_ct[o].pair = 1;
3267	args_ct[o].pair_index = i;
3268	tcg_debug_assert(ct_str[1] == '\0');
3269	continue;
3270
3271	case 'm': /* minus */
3272	/* Allocate to the register before the previous. */
3273	tcg_debug_assert(i > (input_p ? nb_oargs : 0));
3274	o = i - 1;
3275	tcg_debug_assert(!args_ct[o].pair);
3276	tcg_debug_assert(!args_ct[o].ct);
3277	args_ct[i] = (TCGArgConstraint){
3278	.pair = 1,
3279	.pair_index = o,
3280	.regs = args_ct[o].regs >> 1,
3281	.newreg = args_ct[o].newreg,
3282	};
3283	args_ct[o].pair = 2;
3284	args_ct[o].pair_index = i;
3285	tcg_debug_assert(ct_str[1] == '\0');
3286	continue;
3287	}
3288
3289	do {
3290	switch (*ct_str) {
3291	case 'i':
3292	args_ct[i].ct \|= TCG_CT_CONST;
3293	break;
3294	#ifdef TCG_REG_ZERO
3295	case 'z':
3296	args_ct[i].ct \|= TCG_CT_REG_ZERO;
3297	break;
3298	#endif
3299
3300	/* Include all of the target-specific constraints. */
3301
3302	#undef CONST
3303	#define CONST(CASE, MASK) \
3304	case CASE: args_ct[i].ct \|= MASK; break;
3305	#define REGS(CASE, MASK) \
3306	case CASE: args_ct[i].regs \|= MASK; break;
3307
3308	#include "tcg-target-con-str.h"
3309
3310	#undef REGS
3311	#undef CONST
3312	default:
3313	case '0' ... '9':
3314	case '&':
3315	case 'p':
3316	case 'm':
3317	/* Typo in TCGConstraintSet constraint. */
3318	g_assert_not_reached();
3319	}
3320	} while (*++ct_str != '\0');
3321	}
3322
3323	/*
3324	* Fix up output pairs that are aliased with inputs.
3325	* When we created the alias, we copied pair from the output.
3326	* There are three cases:
3327	* (1a) Pairs of inputs alias pairs of outputs.
3328	* (1b) One input aliases the first of a pair of outputs.
3329	* (2) One input aliases the second of a pair of outputs.
3330	*
3331	* Case 1a is handled by making sure that the pair_index'es are
3332	* properly updated so that they appear the same as a pair of inputs.
3333	*
3334	* Case 1b is handled by setting the pair_index of the input to
3335	* itself, simply so it doesn't point to an unrelated argument.
3336	* Since we don't encounter the "second" during the input allocation
3337	* phase, nothing happens with the second half of the input pair.
3338	*
3339	* Case 2 is handled by setting the second input to pair=3, the
3340	* first output to pair=3, and the pair_index'es to match.
3341	*/
3342	if (saw_alias_pair) {
3343	for (int i = nb_oargs; i < nb_args; i++) {
3344	int o, o2, i2;
3345
3346	/*
3347	* Since [0-9pm] must be alone in the constraint string,
3348	* the only way they can both be set is if the pair comes
3349	* from the output alias.
3350	*/
3351	if (!args_ct[i].ialias) {
3352	continue;
3353	}
3354	switch (args_ct[i].pair) {
3355	case 0:
3356	break;
3357	case 1:
3358	o = args_ct[i].alias_index;
3359	o2 = args_ct[o].pair_index;
3360	tcg_debug_assert(args_ct[o].pair == 1);
3361	tcg_debug_assert(args_ct[o2].pair == 2);
3362	if (args_ct[o2].oalias) {
3363	/* Case 1a */
3364	i2 = args_ct[o2].alias_index;
3365	tcg_debug_assert(args_ct[i2].pair == 2);
3366	args_ct[i2].pair_index = i;
3367	args_ct[i].pair_index = i2;
3368	} else {
3369	/* Case 1b */
3370	args_ct[i].pair_index = i;
3371	}
3372	break;
3373	case 2:
3374	o = args_ct[i].alias_index;
3375	o2 = args_ct[o].pair_index;
3376	tcg_debug_assert(args_ct[o].pair == 2);
3377	tcg_debug_assert(args_ct[o2].pair == 1);
3378	if (args_ct[o2].oalias) {
3379	/* Case 1a */
3380	i2 = args_ct[o2].alias_index;
3381	tcg_debug_assert(args_ct[i2].pair == 1);
3382	args_ct[i2].pair_index = i;
3383	args_ct[i].pair_index = i2;
3384	} else {
3385	/* Case 2 */
3386	args_ct[i].pair = 3;
3387	args_ct[o2].pair = 3;
3388	args_ct[i].pair_index = o2;
3389	args_ct[o2].pair_index = i;
3390	}
3391	break;
3392	default:
3393	g_assert_not_reached();
3394	}
3395	}
3396	}
3397
3398	/* sort the constraints (XXX: this is just an heuristic) */
3399	sort_constraints(args_ct, 0, nb_oargs);
3400	sort_constraints(args_ct, nb_oargs, nb_iargs);
3401	}
3402	}
3403
3404	static const TCGArgConstraint *op_args_ct(TCGOpcode opc, TCGType type,
3405	unsigned flags)
3406	{
3407	const TCGOpDef *def = &tcg_op_defs[opc];
3408	const TCGOutOp *outop = all_outop[opc];
3409	TCGConstraintSetIndex con_set;
3410
3411	if (def->flags & TCG_OPF_NOT_PRESENT) {
3412	return empty_cts;
3413	}
3414
3415	if (outop) {
3416	con_set = outop->static_constraint;
3417	if (con_set == C_Dynamic) {
3418	con_set = outop->dynamic_constraint(type, flags);
3419	}
3420	} else {
3421	con_set = tcg_target_op_def(opc, type, flags);
3422	}
3423	tcg_debug_assert(con_set >= 0);
3424	tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets));
3425
3426	/* The constraint arguments must match TCGOpcode arguments. */
3427	tcg_debug_assert(constraint_sets[con_set].nb_oargs == def->nb_oargs);
3428	tcg_debug_assert(constraint_sets[con_set].nb_iargs == def->nb_iargs);
3429
3430	return all_cts[con_set];
3431	}
3432
3433	static const TCGArgConstraint opcode_args_ct(const TCGOp op)
3434	{
3435	return op_args_ct(op->opc, TCGOP_TYPE(op), TCGOP_FLAGS(op));
3436	}
3437
3438	bool tcg_op_imm_match(TCGOpcode opc, TCGType type, tcg_target_ulong imm)
3439	{
3440	const TCGArgConstraint *args_ct = op_args_ct(opc, type, 0);
3441	const TCGOpDef *def = &tcg_op_defs[opc];
3442
3443	tcg_debug_assert(def->nb_oargs == 1);
3444	tcg_debug_assert(def->nb_iargs == 2);
3445	return tcg_target_const_match(imm, args_ct[2].ct, type, 0, 0);
3446	}
3447
3448	static void remove_label_use(TCGOp *op, int idx)
3449	{
3450	TCGLabel *label = arg_label(op->args[idx]);
3451	TCGLabelUse *use;
3452
3453	QSIMPLEQ_FOREACH(use, &label->branches, next) {
3454	if (use->op == op) {
3455	QSIMPLEQ_REMOVE(&label->branches, use, TCGLabelUse, next);
3456	return;
3457	}
3458	}
3459	g_assert_not_reached();
3460	}
3461
3462	void tcg_op_remove(TCGContext s, TCGOp op)
3463	{
3464	switch (op->opc) {
3465	case INDEX_op_br:
3466	remove_label_use(op, 0);
3467	break;
3468	case INDEX_op_brcond:
3469	remove_label_use(op, 3);
3470	break;
3471	default:
3472	break;
3473	}
3474
3475	QTAILQ_REMOVE(&s->ops, op, link);
3476	QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
3477	s->nb_ops--;
3478	}
3479
3480	void tcg_remove_ops_after(TCGOp *op)
3481	{
3482	TCGContext *s = tcg_ctx;
3483
3484	while (true) {
3485	TCGOp *last = tcg_last_op();
3486	if (last == op) {
3487	return;
3488	}
3489	tcg_op_remove(s, last);
3490	}
3491	}
3492
3493	static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs)
3494	{
3495	TCGContext *s = tcg_ctx;
3496	TCGOp *op = NULL;
3497
3498	if (unlikely(!QTAILQ_EMPTY(&s->free_ops))) {
3499	QTAILQ_FOREACH(op, &s->free_ops, link) {
3500	if (nargs <= op->nargs) {
3501	QTAILQ_REMOVE(&s->free_ops, op, link);
3502	nargs = op->nargs;
3503	goto found;
3504	}
3505	}
3506	}
3507
3508	/* Most opcodes have 3 or 4 operands: reduce fragmentation. */
3509	nargs = MAX(4, nargs);
3510	op = tcg_malloc(sizeof(TCGOp) + sizeof(TCGArg) * nargs);
3511
3512	found:
3513	memset(op, 0, offsetof(TCGOp, link));
3514	op->opc = opc;
3515	op->nargs = nargs;
3516
3517	/* Check for bitfield overflow. */
3518	tcg_debug_assert(op->nargs == nargs);
3519
3520	s->nb_ops++;
3521	return op;
3522	}
3523
3524	TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs)
3525	{
3526	TCGOp *op = tcg_op_alloc(opc, nargs);
3527
3528	if (tcg_ctx->emit_before_op) {
3529	QTAILQ_INSERT_BEFORE(tcg_ctx->emit_before_op, op, link);
3530	} else {
3531	QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
3532	}
3533	return op;
3534	}
3535
3536	TCGOp tcg_op_insert_before(TCGContext s, TCGOp *old_op,
3537	TCGOpcode opc, TCGType type, unsigned nargs)
3538	{
3539	TCGOp *new_op = tcg_op_alloc(opc, nargs);
3540
3541	TCGOP_TYPE(new_op) = type;
3542	QTAILQ_INSERT_BEFORE(old_op, new_op, link);
3543	return new_op;
3544	}
3545
3546	TCGOp tcg_op_insert_after(TCGContext s, TCGOp *old_op,
3547	TCGOpcode opc, TCGType type, unsigned nargs)
3548	{
3549	TCGOp *new_op = tcg_op_alloc(opc, nargs);
3550
3551	TCGOP_TYPE(new_op) = type;
3552	QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link);
3553	return new_op;
3554	}
3555
3556	static void move_label_uses(TCGLabel to, TCGLabel from)
3557	{
3558	TCGLabelUse *u;
3559
3560	QSIMPLEQ_FOREACH(u, &from->branches, next) {
3561	TCGOp *op = u->op;
3562	switch (op->opc) {
3563	case INDEX_op_br:
3564	op->args[0] = label_arg(to);
3565	break;
3566	case INDEX_op_brcond:
3567	op->args[3] = label_arg(to);
3568	break;
3569	default:
3570	g_assert_not_reached();
3571	}
3572	}
3573
3574	QSIMPLEQ_CONCAT(&to->branches, &from->branches);
3575	}
3576
3577	/* Reachable analysis : remove unreachable code. */
3578	static void __attribute__((noinline))
3579	reachable_code_pass(TCGContext *s)
3580	{
3581	TCGOp op, op_next, *op_prev;
3582	bool dead = false;
3583
3584	QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
3585	bool remove = dead;
3586	TCGLabel *label;
3587
3588	switch (op->opc) {
3589	case INDEX_op_set_label:
3590	label = arg_label(op->args[0]);
3591
3592	/*
3593	* Note that the first op in the TB is always a load,
3594	* so there is always something before a label.
3595	*/
3596	op_prev = QTAILQ_PREV(op, link);
3597
3598	/*
3599	* If we find two sequential labels, move all branches to
3600	* reference the second label and remove the first label.
3601	* Do this before branch to next optimization, so that the
3602	* middle label is out of the way.
3603	*/
3604	if (op_prev->opc == INDEX_op_set_label) {
3605	move_label_uses(label, arg_label(op_prev->args[0]));
3606	tcg_op_remove(s, op_prev);
3607	op_prev = QTAILQ_PREV(op, link);
3608	}
3609
3610	/*
3611	* Optimization can fold conditional branches to unconditional.
3612	* If we find a label which is preceded by an unconditional
3613	* branch to next, remove the branch. We couldn't do this when
3614	* processing the branch because any dead code between the branch
3615	* and label had not yet been removed.
3616	*/
3617	if (op_prev->opc == INDEX_op_br &&
3618	label == arg_label(op_prev->args[0])) {
3619	tcg_op_remove(s, op_prev);
3620	/* Fall through means insns become live again. */
3621	dead = false;
3622	}
3623
3624	if (QSIMPLEQ_EMPTY(&label->branches)) {
3625	/*
3626	* While there is an occasional backward branch, virtually
3627	* all branches generated by the translators are forward.
3628	* Which means that generally we will have already removed
3629	* all references to the label that will be, and there is
3630	* little to be gained by iterating.
3631	*/
3632	remove = true;
3633	} else {
3634	/* Once we see a label, insns become live again. */
3635	dead = false;
3636	remove = false;
3637	}
3638	break;
3639
3640	case INDEX_op_br:
3641	case INDEX_op_exit_tb:
3642	case INDEX_op_goto_ptr:
3643	/* Unconditional branches; everything following is dead. */
3644	dead = true;
3645	break;
3646
3647	case INDEX_op_call:
3648	/* Notice noreturn helper calls, raising exceptions. */
3649	if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) {
3650	dead = true;
3651	}
3652	break;
3653
3654	case INDEX_op_insn_start:
3655	/* Never remove -- we need to keep these for unwind. */
3656	remove = false;
3657	break;
3658
3659	default:
3660	break;
3661	}
3662
3663	if (remove) {
3664	tcg_op_remove(s, op);
3665	}
3666	}
3667	}
3668
3669	#define TS_DEAD 1
3670	#define TS_MEM 2
3671
3672	#define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
3673	#define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
3674
3675	/* For liveness_pass_1, the register preferences for a given temp. */
3676	static inline TCGRegSet la_temp_pref(TCGTemp ts)
3677	{
3678	return ts->state_ptr;
3679	}
3680
3681	/* For liveness_pass_1, reset the preferences for a given temp to the
3682	* maximal regset for its type.
3683	*/
3684	static inline void la_reset_pref(TCGTemp *ts)
3685	{
3686	*la_temp_pref(ts)
3687	= (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]);
3688	}
3689
3690	/* liveness analysis: end of function: all temps are dead, and globals
3691	should be in memory. */
3692	static void la_func_end(TCGContext *s, int ng, int nt)
3693	{
3694	int i;
3695
3696	for (i = 0; i < ng; ++i) {
3697	s->temps[i].state = TS_DEAD \| TS_MEM;
3698	la_reset_pref(&s->temps[i]);
3699	}
3700	for (i = ng; i < nt; ++i) {
3701	s->temps[i].state = TS_DEAD;
3702	la_reset_pref(&s->temps[i]);
3703	}
3704	}
3705
3706	/* liveness analysis: end of basic block: all temps are dead, globals
3707	and local temps should be in memory. */
3708	static void la_bb_end(TCGContext *s, int ng, int nt)
3709	{
3710	int i;
3711
3712	for (i = 0; i < nt; ++i) {
3713	TCGTemp *ts = &s->temps[i];
3714	int state;
3715
3716	switch (ts->kind) {
3717	case TEMP_FIXED:
3718	case TEMP_GLOBAL:
3719	case TEMP_TB:
3720	state = TS_DEAD \| TS_MEM;
3721	break;
3722	case TEMP_EBB:
3723	case TEMP_CONST:
3724	state = TS_DEAD;
3725	break;
3726	default:
3727	g_assert_not_reached();
3728	}
3729	ts->state = state;
3730	la_reset_pref(ts);
3731	}
3732	}
3733
3734	/* liveness analysis: sync globals back to memory. */
3735	static void la_global_sync(TCGContext *s, int ng)
3736	{
3737	int i;
3738
3739	for (i = 0; i < ng; ++i) {
3740	int state = s->temps[i].state;
3741	s->temps[i].state = state \| TS_MEM;
3742	if (state == TS_DEAD) {
3743	/* If the global was previously dead, reset prefs. */
3744	la_reset_pref(&s->temps[i]);
3745	}
3746	}
3747	}
3748
3749	/*
3750	* liveness analysis: conditional branch: all temps are dead unless
3751	* explicitly live-across-conditional-branch, globals and local temps
3752	* should be synced.
3753	*/
3754	static void la_bb_sync(TCGContext *s, int ng, int nt)
3755	{
3756	la_global_sync(s, ng);
3757
3758	for (int i = ng; i < nt; ++i) {
3759	TCGTemp *ts = &s->temps[i];
3760	int state;
3761
3762	switch (ts->kind) {
3763	case TEMP_TB:
3764	state = ts->state;
3765	ts->state = state \| TS_MEM;
3766	if (state != TS_DEAD) {
3767	continue;
3768	}
3769	break;
3770	case TEMP_EBB:
3771	case TEMP_CONST:
3772	continue;
3773	default:
3774	g_assert_not_reached();
3775	}
3776	la_reset_pref(&s->temps[i]);
3777	}
3778	}
3779
3780	/* liveness analysis: sync globals back to memory and kill. */
3781	static void la_global_kill(TCGContext *s, int ng)
3782	{
3783	int i;
3784
3785	for (i = 0; i < ng; i++) {
3786	s->temps[i].state = TS_DEAD \| TS_MEM;
3787	la_reset_pref(&s->temps[i]);
3788	}
3789	}
3790
3791	/* liveness analysis: note live globals crossing calls. */
3792	static void la_cross_call(TCGContext *s, int nt)
3793	{
3794	TCGRegSet mask = ~tcg_target_call_clobber_regs;
3795	int i;
3796
3797	for (i = 0; i < nt; i++) {
3798	TCGTemp *ts = &s->temps[i];
3799	if (!(ts->state & TS_DEAD)) {
3800	TCGRegSet *pset = la_temp_pref(ts);
3801	TCGRegSet set = *pset;
3802
3803	set &= mask;
3804	/* If the combination is not possible, restart. */
3805	if (set == 0) {
3806	set = tcg_target_available_regs[ts->type] & mask;
3807	}
3808	*pset = set;
3809	}
3810	}
3811	}
3812
3813	/*
3814	* Liveness analysis: Verify the lifetime of TEMP_TB, and reduce
3815	* to TEMP_EBB, if possible.
3816	*/
3817	static void __attribute__((noinline))
3818	liveness_pass_0(TCGContext *s)
3819	{
3820	void * const multiple_ebb = (void *)(uintptr_t)-1;
3821	int nb_temps = s->nb_temps;
3822	TCGOp op, ebb;
3823
3824	for (int i = s->nb_globals; i < nb_temps; ++i) {
3825	s->temps[i].state_ptr = NULL;
3826	}
3827
3828	/*
3829	* Represent each EBB by the op at which it begins. In the case of
3830	* the first EBB, this is the first op, otherwise it is a label.
3831	* Collect the uses of each TEMP_TB: NULL for unused, EBB for use
3832	* within a single EBB, else MULTIPLE_EBB.
3833	*/
3834	ebb = QTAILQ_FIRST(&s->ops);
3835	QTAILQ_FOREACH(op, &s->ops, link) {
3836	const TCGOpDef *def;
3837	int nb_oargs, nb_iargs;
3838
3839	switch (op->opc) {
3840	case INDEX_op_set_label:
3841	ebb = op;
3842	continue;
3843	case INDEX_op_discard:
3844	continue;
3845	case INDEX_op_call:
3846	nb_oargs = TCGOP_CALLO(op);
3847	nb_iargs = TCGOP_CALLI(op);
3848	break;
3849	default:
3850	def = &tcg_op_defs[op->opc];
3851	nb_oargs = def->nb_oargs;
3852	nb_iargs = def->nb_iargs;
3853	break;
3854	}
3855
3856	for (int i = 0; i < nb_oargs + nb_iargs; ++i) {
3857	TCGTemp *ts = arg_temp(op->args[i]);
3858
3859	if (ts->kind != TEMP_TB) {
3860	continue;
3861	}
3862	if (ts->state_ptr == NULL) {
3863	ts->state_ptr = ebb;
3864	} else if (ts->state_ptr != ebb) {
3865	ts->state_ptr = multiple_ebb;
3866	}
3867	}
3868	}
3869
3870	/*
3871	* For TEMP_TB that turned out not to be used beyond one EBB,
3872	* reduce the liveness to TEMP_EBB.
3873	*/
3874	for (int i = s->nb_globals; i < nb_temps; ++i) {
3875	TCGTemp *ts = &s->temps[i];
3876	if (ts->kind == TEMP_TB && ts->state_ptr != multiple_ebb) {
3877	ts->kind = TEMP_EBB;
3878	}
3879	}
3880	}
3881
3882	static void assert_carry_dead(TCGContext *s)
3883	{
3884	/*
3885	* Carry operations can be separated by a few insns like mov,
3886	* load or store, but they should always be "close", and
3887	* carry-out operations should always be paired with carry-in.
3888	* At various boundaries, carry must have been consumed.
3889	*/
3890	tcg_debug_assert(!s->carry_live);
3891	}
3892
3893	/* Liveness analysis : update the opc_arg_life array to tell if a
3894	given input arguments is dead. Instructions updating dead
3895	temporaries are removed. */
3896	static void __attribute__((noinline))
3897	liveness_pass_1(TCGContext *s)
3898	{
3899	int nb_globals = s->nb_globals;
3900	int nb_temps = s->nb_temps;
3901	TCGOp op, op_prev;
3902	TCGRegSet *prefs;
3903
3904	prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps);
3905	for (int i = 0; i < nb_temps; ++i) {
3906	s->temps[i].state_ptr = prefs + i;
3907	}
3908
3909	/* ??? Should be redundant with the exit_tb that ends the TB. */
3910	la_func_end(s, nb_globals, nb_temps);
3911
3912	s->carry_live = false;
3913	QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) {
3914	int nb_iargs, nb_oargs;
3915	TCGOpcode opc_new, opc_new2;
3916	TCGLifeData arg_life = 0;
3917	TCGTemp *ts;
3918	TCGOpcode opc = op->opc;
3919	const TCGOpDef *def;
3920	const TCGArgConstraint *args_ct;
3921
3922	switch (opc) {
3923	case INDEX_op_call:
3924	assert_carry_dead(s);
3925	{
3926	const TCGHelperInfo *info = tcg_call_info(op);
3927	int call_flags = tcg_call_flags(op);
3928
3929	nb_oargs = TCGOP_CALLO(op);
3930	nb_iargs = TCGOP_CALLI(op);
3931
3932	/* pure functions can be removed if their result is unused */
3933	if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
3934	for (int i = 0; i < nb_oargs; i++) {
3935	ts = arg_temp(op->args[i]);
3936	if (ts->state != TS_DEAD) {
3937	goto do_not_remove_call;
3938	}
3939	}
3940	goto do_remove;
3941	}
3942	do_not_remove_call:
3943
3944	/* Output args are dead. */
3945	for (int i = 0; i < nb_oargs; i++) {
3946	ts = arg_temp(op->args[i]);
3947	if (ts->state & TS_DEAD) {
3948	arg_life \|= DEAD_ARG << i;
3949	}
3950	if (ts->state & TS_MEM) {
3951	arg_life \|= SYNC_ARG << i;
3952	}
3953	ts->state = TS_DEAD;
3954	la_reset_pref(ts);
3955	}
3956
3957	/* Not used -- it will be tcg_target_call_oarg_reg(). */
3958	memset(op->output_pref, 0, sizeof(op->output_pref));
3959
3960	if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS \|
3961	TCG_CALL_NO_READ_GLOBALS))) {
3962	la_global_kill(s, nb_globals);
3963	} else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
3964	la_global_sync(s, nb_globals);
3965	}
3966
3967	/* Record arguments that die in this helper. */
3968	for (int i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
3969	ts = arg_temp(op->args[i]);
3970	if (ts->state & TS_DEAD) {
3971	arg_life \|= DEAD_ARG << i;
3972	}
3973	}
3974
3975	/* For all live registers, remove call-clobbered prefs. */
3976	la_cross_call(s, nb_temps);
3977
3978	/*
3979	* Input arguments are live for preceding opcodes.
3980	*
3981	* For those arguments that die, and will be allocated in
3982	* registers, clear the register set for that arg, to be
3983	* filled in below. For args that will be on the stack,
3984	* reset to any available reg. Process arguments in reverse
3985	* order so that if a temp is used more than once, the stack
3986	* reset to max happens before the register reset to 0.
3987	*/
3988	for (int i = nb_iargs - 1; i >= 0; i--) {
3989	const TCGCallArgumentLoc *loc = &info->in[i];
3990	ts = arg_temp(op->args[nb_oargs + i]);
3991
3992	if (ts->state & TS_DEAD) {
3993	switch (loc->kind) {
3994	case TCG_CALL_ARG_NORMAL:
3995	case TCG_CALL_ARG_EXTEND_U:
3996	case TCG_CALL_ARG_EXTEND_S:
3997	if (arg_slot_reg_p(loc->arg_slot)) {
3998	*la_temp_pref(ts) = 0;
3999	break;
4000	}
4001	/* fall through */
4002	default:
4003	*la_temp_pref(ts) =
4004	tcg_target_available_regs[ts->type];
4005	break;
4006	}
4007	ts->state &= ~TS_DEAD;
4008	}
4009	}
4010
4011	/*
4012	* For each input argument, add its input register to prefs.
4013	* If a temp is used once, this produces a single set bit;
4014	* if a temp is used multiple times, this produces a set.
4015	*/
4016	for (int i = 0; i < nb_iargs; i++) {
4017	const TCGCallArgumentLoc *loc = &info->in[i];
4018	ts = arg_temp(op->args[nb_oargs + i]);
4019
4020	switch (loc->kind) {
4021	case TCG_CALL_ARG_NORMAL:
4022	case TCG_CALL_ARG_EXTEND_U:
4023	case TCG_CALL_ARG_EXTEND_S:
4024	if (arg_slot_reg_p(loc->arg_slot)) {
4025	tcg_regset_set_reg(*la_temp_pref(ts),
4026	tcg_target_call_iarg_regs[loc->arg_slot]);
4027	}
4028	break;
4029	default:
4030	break;
4031	}
4032	}
4033	}
4034	break;
4035	case INDEX_op_insn_start:
4036	assert_carry_dead(s);
4037	break;
4038	case INDEX_op_discard:
4039	/* mark the temporary as dead */
4040	ts = arg_temp(op->args[0]);
4041	ts->state = TS_DEAD;
4042	la_reset_pref(ts);
4043	break;
4044
4045	case INDEX_op_muls2:
4046	opc_new = INDEX_op_mul;
4047	opc_new2 = INDEX_op_mulsh;
4048	goto do_mul2;
4049	case INDEX_op_mulu2:
4050	opc_new = INDEX_op_mul;
4051	opc_new2 = INDEX_op_muluh;
4052	do_mul2:
4053	assert_carry_dead(s);
4054	if (arg_temp(op->args[1])->state == TS_DEAD) {
4055	if (arg_temp(op->args[0])->state == TS_DEAD) {
4056	/* Both parts of the operation are dead. */
4057	goto do_remove;
4058	}
4059	/* The high part of the operation is dead; generate the low. */
4060	op->opc = opc = opc_new;
4061	op->args[1] = op->args[2];
4062	op->args[2] = op->args[3];
4063	} else if (arg_temp(op->args[0])->state == TS_DEAD &&
4064	tcg_op_supported(opc_new2, TCGOP_TYPE(op), 0)) {
4065	/* The low part of the operation is dead; generate the high. */
4066	op->opc = opc = opc_new2;
4067	op->args[0] = op->args[1];
4068	op->args[1] = op->args[2];
4069	op->args[2] = op->args[3];
4070	} else {
4071	goto do_not_remove;
4072	}
4073	/* Mark the single-word operation live. */
4074	goto do_not_remove;
4075
4076	case INDEX_op_addco:
4077	if (s->carry_live) {
4078	goto do_not_remove;
4079	}
4080	op->opc = opc = INDEX_op_add;
4081	goto do_default;
4082
4083	case INDEX_op_addcio:
4084	if (s->carry_live) {
4085	goto do_not_remove;
4086	}
4087	op->opc = opc = INDEX_op_addci;
4088	goto do_default;
4089
4090	case INDEX_op_subbo:
4091	if (s->carry_live) {
4092	goto do_not_remove;
4093	}
4094	/* Lower to sub, but this may also require canonicalization. */
4095	op->opc = opc = INDEX_op_sub;
4096	ts = arg_temp(op->args[2]);
4097	if (ts->kind == TEMP_CONST) {
4098	ts = tcg_constant_internal(ts->type, -ts->val);
4099	if (ts->state_ptr == NULL) {
4100	tcg_debug_assert(temp_idx(ts) == nb_temps);
4101	nb_temps++;
4102	ts->state_ptr = tcg_malloc(sizeof(TCGRegSet));
4103	ts->state = TS_DEAD;
4104	la_reset_pref(ts);
4105	}
4106	op->args[2] = temp_arg(ts);
4107	op->opc = opc = INDEX_op_add;
4108	}
4109	goto do_default;
4110
4111	case INDEX_op_subbio:
4112	if (s->carry_live) {
4113	goto do_not_remove;
4114	}
4115	op->opc = opc = INDEX_op_subbi;
4116	goto do_default;
4117
4118	case INDEX_op_addc1o:
4119	if (s->carry_live) {
4120	goto do_not_remove;
4121	}
4122	/* Lower to add, add +1. */
4123	op_prev = tcg_op_insert_before(s, op, INDEX_op_add,
4124	TCGOP_TYPE(op), 3);
4125	op_prev->args[0] = op->args[0];
4126	op_prev->args[1] = op->args[1];
4127	op_prev->args[2] = op->args[2];
4128	op->opc = opc = INDEX_op_add;
4129	op->args[1] = op->args[0];
4130	ts = arg_temp(op->args[0]);
4131	ts = tcg_constant_internal(ts->type, 1);
4132	op->args[2] = temp_arg(ts);
4133	goto do_default;
4134
4135	case INDEX_op_subb1o:
4136	if (s->carry_live) {
4137	goto do_not_remove;
4138	}
4139	/* Lower to sub, add -1. */
4140	op_prev = tcg_op_insert_before(s, op, INDEX_op_sub,
4141	TCGOP_TYPE(op), 3);
4142	op_prev->args[0] = op->args[0];
4143	op_prev->args[1] = op->args[1];
4144	op_prev->args[2] = op->args[2];
4145	op->opc = opc = INDEX_op_add;
4146	op->args[1] = op->args[0];
4147	ts = arg_temp(op->args[0]);
4148	ts = tcg_constant_internal(ts->type, -1);
4149	op->args[2] = temp_arg(ts);
4150	goto do_default;
4151
4152	default:
4153	do_default:
4154	/*
4155	* Test if the operation can be removed because all
4156	* its outputs are dead. We assume that nb_oargs == 0
4157	* implies side effects.
4158	*/
4159	def = &tcg_op_defs[opc];
4160	if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && def->nb_oargs != 0) {
4161	for (int i = def->nb_oargs - 1; i >= 0; i--) {
4162	if (arg_temp(op->args[i])->state != TS_DEAD) {
4163	goto do_not_remove;
4164	}
4165	}
4166	goto do_remove;
4167	}
4168	goto do_not_remove;
4169
4170	do_remove:
4171	tcg_op_remove(s, op);
4172	break;
4173
4174	do_not_remove:
4175	def = &tcg_op_defs[opc];
4176	nb_iargs = def->nb_iargs;
4177	nb_oargs = def->nb_oargs;
4178
4179	for (int i = 0; i < nb_oargs; i++) {
4180	ts = arg_temp(op->args[i]);
4181
4182	/* Remember the preference of the uses that followed. */
4183	if (i < ARRAY_SIZE(op->output_pref)) {
4184	op->output_pref[i] = *la_temp_pref(ts);
4185	}
4186
4187	/* Output args are dead. */
4188	if (ts->state & TS_DEAD) {
4189	arg_life \|= DEAD_ARG << i;
4190	}
4191	if (ts->state & TS_MEM) {
4192	arg_life \|= SYNC_ARG << i;
4193	}
4194	ts->state = TS_DEAD;
4195	la_reset_pref(ts);
4196	}
4197
4198	/* If end of basic block, update. */
4199	if (def->flags & TCG_OPF_BB_EXIT) {
4200	assert_carry_dead(s);
4201	la_func_end(s, nb_globals, nb_temps);
4202	} else if (def->flags & TCG_OPF_COND_BRANCH) {
4203	assert_carry_dead(s);
4204	la_bb_sync(s, nb_globals, nb_temps);
4205	} else if (def->flags & TCG_OPF_BB_END) {
4206	assert_carry_dead(s);
4207	la_bb_end(s, nb_globals, nb_temps);
4208	} else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
4209	assert_carry_dead(s);
4210	la_global_sync(s, nb_globals);
4211	if (def->flags & TCG_OPF_CALL_CLOBBER) {
4212	la_cross_call(s, nb_temps);
4213	}
4214	}
4215
4216	/* Record arguments that die in this opcode. */
4217	for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
4218	ts = arg_temp(op->args[i]);
4219	if (ts->state & TS_DEAD) {
4220	arg_life \|= DEAD_ARG << i;
4221	}
4222	}
4223	if (def->flags & TCG_OPF_CARRY_OUT) {
4224	s->carry_live = false;
4225	}
4226
4227	/* Input arguments are live for preceding opcodes. */
4228	for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
4229	ts = arg_temp(op->args[i]);
4230	if (ts->state & TS_DEAD) {
4231	/* For operands that were dead, initially allow
4232	all regs for the type. */
4233	*la_temp_pref(ts) = tcg_target_available_regs[ts->type];
4234	ts->state &= ~TS_DEAD;
4235	}
4236	}
4237	if (def->flags & TCG_OPF_CARRY_IN) {
4238	s->carry_live = true;
4239	}
4240
4241	/* Incorporate constraints for this operand. */
4242	switch (opc) {
4243	case INDEX_op_mov:
4244	/* Note that these are TCG_OPF_NOT_PRESENT and do not
4245	have proper constraints. That said, special case
4246	moves to propagate preferences backward. */
4247	if (IS_DEAD_ARG(1)) {
4248	*la_temp_pref(arg_temp(op->args[0]))
4249	= *la_temp_pref(arg_temp(op->args[1]));
4250	}
4251	break;
4252
4253	default:
4254	args_ct = opcode_args_ct(op);
4255	for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
4256	const TCGArgConstraint *ct = &args_ct[i];
4257	TCGRegSet set, *pset;
4258
4259	ts = arg_temp(op->args[i]);
4260	pset = la_temp_pref(ts);
4261	set = *pset;
4262
4263	set &= ct->regs;
4264	if (ct->ialias) {
4265	set &= output_pref(op, ct->alias_index);
4266	}
4267	/* If the combination is not possible, restart. */
4268	if (set == 0) {
4269	set = ct->regs;
4270	}
4271	*pset = set;
4272	}
4273	break;
4274	}
4275	break;
4276	}
4277	op->life = arg_life;
4278	}
4279	assert_carry_dead(s);
4280	}
4281
4282	/* Liveness analysis: Convert indirect regs to direct temporaries. */
4283	static bool __attribute__((noinline))
4284	liveness_pass_2(TCGContext *s)
4285	{
4286	int nb_globals = s->nb_globals;
4287	int nb_temps, i;
4288	bool changes = false;
4289	TCGOp op, op_next;
4290
4291	/* Create a temporary for each indirect global. */
4292	for (i = 0; i < nb_globals; ++i) {
4293	TCGTemp *its = &s->temps[i];
4294	if (its->indirect_reg) {
4295	TCGTemp *dts = tcg_temp_alloc(s);
4296	dts->type = its->type;
4297	dts->base_type = its->base_type;
4298	dts->temp_subindex = its->temp_subindex;
4299	dts->kind = TEMP_EBB;
4300	its->state_ptr = dts;
4301	} else {
4302	its->state_ptr = NULL;
4303	}
4304	/* All globals begin dead. */
4305	its->state = TS_DEAD;
4306	}
4307	for (nb_temps = s->nb_temps; i < nb_temps; ++i) {
4308	TCGTemp *its = &s->temps[i];
4309	its->state_ptr = NULL;
4310	its->state = TS_DEAD;
4311	}
4312
4313	QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
4314	TCGOpcode opc = op->opc;
4315	const TCGOpDef *def = &tcg_op_defs[opc];
4316	TCGLifeData arg_life = op->life;
4317	int nb_iargs, nb_oargs, call_flags;
4318	TCGTemp arg_ts, dir_ts;
4319
4320	if (opc == INDEX_op_call) {
4321	nb_oargs = TCGOP_CALLO(op);
4322	nb_iargs = TCGOP_CALLI(op);
4323	call_flags = tcg_call_flags(op);
4324	} else {
4325	nb_iargs = def->nb_iargs;
4326	nb_oargs = def->nb_oargs;
4327
4328	/* Set flags similar to how calls require. */
4329	if (def->flags & TCG_OPF_COND_BRANCH) {
4330	/* Like reading globals: sync_globals */
4331	call_flags = TCG_CALL_NO_WRITE_GLOBALS;
4332	} else if (def->flags & TCG_OPF_BB_END) {
4333	/* Like writing globals: save_globals */
4334	call_flags = 0;
4335	} else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
4336	/* Like reading globals: sync_globals */
4337	call_flags = TCG_CALL_NO_WRITE_GLOBALS;
4338	} else {
4339	/* No effect on globals. */
4340	call_flags = (TCG_CALL_NO_READ_GLOBALS \|
4341	TCG_CALL_NO_WRITE_GLOBALS);
4342	}
4343	}
4344
4345	/* Make sure that input arguments are available. */
4346	for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
4347	arg_ts = arg_temp(op->args[i]);
4348	dir_ts = arg_ts->state_ptr;
4349	if (dir_ts && arg_ts->state == TS_DEAD) {
4350	TCGOp *lop = tcg_op_insert_before(s, op, INDEX_op_ld,
4351	arg_ts->type, 3);
4352
4353	lop->args[0] = temp_arg(dir_ts);
4354	lop->args[1] = temp_arg(arg_ts->mem_base);
4355	lop->args[2] = arg_ts->mem_offset;
4356
4357	/* Loaded, but synced with memory. */
4358	arg_ts->state = TS_MEM;
4359	}
4360	}
4361
4362	/* Perform input replacement, and mark inputs that became dead.
4363	No action is required except keeping temp_state up to date
4364	so that we reload when needed. */
4365	for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
4366	arg_ts = arg_temp(op->args[i]);
4367	dir_ts = arg_ts->state_ptr;
4368	if (dir_ts) {
4369	op->args[i] = temp_arg(dir_ts);
4370	changes = true;
4371	if (IS_DEAD_ARG(i)) {
4372	arg_ts->state = TS_DEAD;
4373	}
4374	}
4375	}
4376
4377	/* Liveness analysis should ensure that the following are
4378	all correct, for call sites and basic block end points. */
4379	if (call_flags & TCG_CALL_NO_READ_GLOBALS) {
4380	/* Nothing to do */
4381	} else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) {
4382	for (i = 0; i < nb_globals; ++i) {
4383	/* Liveness should see that globals are synced back,
4384	that is, either TS_DEAD or TS_MEM. */
4385	arg_ts = &s->temps[i];
4386	tcg_debug_assert(arg_ts->state_ptr == 0
4387	\|\| arg_ts->state != 0);
4388	}
4389	} else {
4390	for (i = 0; i < nb_globals; ++i) {
4391	/* Liveness should see that globals are saved back,
4392	that is, TS_DEAD, waiting to be reloaded. */
4393	arg_ts = &s->temps[i];
4394	tcg_debug_assert(arg_ts->state_ptr == 0
4395	\|\| arg_ts->state == TS_DEAD);
4396	}
4397	}
4398
4399	/* Outputs become available. */
4400	if (opc == INDEX_op_mov) {
4401	arg_ts = arg_temp(op->args[0]);
4402	dir_ts = arg_ts->state_ptr;
4403	if (dir_ts) {
4404	op->args[0] = temp_arg(dir_ts);
4405	changes = true;
4406
4407	/* The output is now live and modified. */
4408	arg_ts->state = 0;
4409
4410	if (NEED_SYNC_ARG(0)) {
4411	TCGOp *sop = tcg_op_insert_after(s, op, INDEX_op_st,
4412	arg_ts->type, 3);
4413	TCGTemp *out_ts = dir_ts;
4414
4415	if (IS_DEAD_ARG(0)) {
4416	out_ts = arg_temp(op->args[1]);
4417	arg_ts->state = TS_DEAD;
4418	tcg_op_remove(s, op);
4419	} else {
4420	arg_ts->state = TS_MEM;
4421	}
4422
4423	sop->args[0] = temp_arg(out_ts);
4424	sop->args[1] = temp_arg(arg_ts->mem_base);
4425	sop->args[2] = arg_ts->mem_offset;
4426	} else {
4427	tcg_debug_assert(!IS_DEAD_ARG(0));
4428	}
4429	}
4430	} else {
4431	for (i = 0; i < nb_oargs; i++) {
4432	arg_ts = arg_temp(op->args[i]);
4433	dir_ts = arg_ts->state_ptr;
4434	if (!dir_ts) {
4435	continue;
4436	}
4437	op->args[i] = temp_arg(dir_ts);
4438	changes = true;
4439
4440	/* The output is now live and modified. */
4441	arg_ts->state = 0;
4442
4443	/* Sync outputs upon their last write. */
4444	if (NEED_SYNC_ARG(i)) {
4445	TCGOp *sop = tcg_op_insert_after(s, op, INDEX_op_st,
4446	arg_ts->type, 3);
4447
4448	sop->args[0] = temp_arg(dir_ts);
4449	sop->args[1] = temp_arg(arg_ts->mem_base);
4450	sop->args[2] = arg_ts->mem_offset;
4451
4452	arg_ts->state = TS_MEM;
4453	}
4454	/* Drop outputs that are dead. */
4455	if (IS_DEAD_ARG(i)) {
4456	arg_ts->state = TS_DEAD;
4457	}
4458	}
4459	}
4460	}
4461
4462	return changes;
4463	}
4464
4465	static void temp_allocate_frame(TCGContext s, TCGTemp ts)
4466	{
4467	intptr_t off;
4468	int size, align;
4469
4470	/* When allocating an object, look at the full type. */
4471	size = tcg_type_size(ts->base_type);
4472	switch (ts->base_type) {
4473	case TCG_TYPE_I32:
4474	align = 4;
4475	break;
4476	case TCG_TYPE_I64:
4477	case TCG_TYPE_V64:
4478	align = 8;
4479	break;
4480	case TCG_TYPE_I128:
4481	case TCG_TYPE_V128:
4482	case TCG_TYPE_V256:
4483	/*
4484	* Note that we do not require aligned storage for V256,
4485	* and that we provide alignment for I128 to match V128,
4486	* even if that's above what the host ABI requires.
4487	*/
4488	align = 16;
4489	break;
4490	default:
4491	g_assert_not_reached();
4492	}
4493
4494	/*
4495	* Assume the stack is sufficiently aligned.
4496	* This affects e.g. ARM NEON, where we have 8 byte stack alignment
4497	* and do not require 16 byte vector alignment. This seems slightly
4498	* easier than fully parameterizing the above switch statement.
4499	*/
4500	align = MIN(TCG_TARGET_STACK_ALIGN, align);
4501	off = ROUND_UP(s->current_frame_offset, align);
4502
4503	/* If we've exhausted the stack frame, restart with a smaller TB. */
4504	if (off + size > s->frame_end) {
4505	tcg_raise_tb_overflow(s);
4506	}
4507	s->current_frame_offset = off + size;
4508	#if defined(__sparc__)
4509	off += TCG_TARGET_STACK_BIAS;
4510	#endif
4511
4512	/* If the object was subdivided, assign memory to all the parts. */
4513	if (ts->base_type != ts->type) {
4514	int part_size = tcg_type_size(ts->type);
4515	int part_count = size / part_size;
4516
4517	/*
4518	* Each part is allocated sequentially in tcg_temp_new_internal.
4519	* Jump back to the first part by subtracting the current index.
4520	*/
4521	ts -= ts->temp_subindex;
4522	for (int i = 0; i < part_count; ++i) {
4523	ts[i].mem_offset = off + i * part_size;
4524	ts[i].mem_base = s->frame_temp;
4525	ts[i].mem_allocated = 1;
4526	}
4527	} else {
4528	ts->mem_offset = off;
4529	ts->mem_base = s->frame_temp;
4530	ts->mem_allocated = 1;
4531	}
4532	}
4533
4534	/* Assign @reg to @ts, and update reg_to_temp[]. */
4535	static void set_temp_val_reg(TCGContext s, TCGTemp ts, TCGReg reg)
4536	{
4537	if (ts->val_type == TEMP_VAL_REG) {
4538	TCGReg old = ts->reg;
4539	tcg_debug_assert(s->reg_to_temp[old] == ts);
4540	if (old == reg) {
4541	return;
4542	}
4543	s->reg_to_temp[old] = NULL;
4544	}
4545	tcg_debug_assert(s->reg_to_temp[reg] == NULL);
4546	s->reg_to_temp[reg] = ts;
4547	ts->val_type = TEMP_VAL_REG;
4548	ts->reg = reg;
4549	}
4550
4551	/* Assign a non-register value type to @ts, and update reg_to_temp[]. */
4552	static void set_temp_val_nonreg(TCGContext s, TCGTemp ts, TCGTempVal type)
4553	{
4554	tcg_debug_assert(type != TEMP_VAL_REG);
4555	if (ts->val_type == TEMP_VAL_REG) {
4556	TCGReg reg = ts->reg;
4557	tcg_debug_assert(s->reg_to_temp[reg] == ts);
4558	s->reg_to_temp[reg] = NULL;
4559	}
4560	ts->val_type = type;
4561	}
4562
4563	static void temp_load(TCGContext , TCGTemp , TCGRegSet, TCGRegSet, TCGRegSet);
4564
4565	/* Mark a temporary as free or dead. If 'free_or_dead' is negative,
4566	mark it free; otherwise mark it dead. */
4567	static void temp_free_or_dead(TCGContext s, TCGTemp ts, int free_or_dead)
4568	{
4569	TCGTempVal new_type;
4570
4571	switch (ts->kind) {
4572	case TEMP_FIXED:
4573	return;
4574	case TEMP_GLOBAL:
4575	case TEMP_TB:
4576	new_type = TEMP_VAL_MEM;
4577	break;
4578	case TEMP_EBB:
4579	new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD;
4580	break;
4581	case TEMP_CONST:
4582	new_type = TEMP_VAL_CONST;
4583	break;
4584	default:
4585	g_assert_not_reached();
4586	}
4587	set_temp_val_nonreg(s, ts, new_type);
4588	}
4589
4590	/* Mark a temporary as dead. */
4591	static inline void temp_dead(TCGContext s, TCGTemp ts)
4592	{
4593	temp_free_or_dead(s, ts, 1);
4594	}
4595
4596	/* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
4597	registers needs to be allocated to store a constant. If 'free_or_dead'
4598	is non-zero, subsequently release the temporary; if it is positive, the
4599	temp is dead; if it is negative, the temp is free. */
4600	static void temp_sync(TCGContext s, TCGTemp ts, TCGRegSet allocated_regs,
4601	TCGRegSet preferred_regs, int free_or_dead)
4602	{
4603	if (!temp_readonly(ts) && !ts->mem_coherent) {
4604	if (!ts->mem_allocated) {
4605	temp_allocate_frame(s, ts);
4606	}
4607	switch (ts->val_type) {
4608	case TEMP_VAL_CONST:
4609	/* If we're going to free the temp immediately, then we won't
4610	require it later in a register, so attempt to store the
4611	constant to memory directly. */
4612	if (free_or_dead
4613	&& tcg_out_sti(s, ts->type, ts->val,
4614	ts->mem_base->reg, ts->mem_offset)) {
4615	break;
4616	}
4617	temp_load(s, ts, tcg_target_available_regs[ts->type],
4618	allocated_regs, preferred_regs);
4619	/* fallthrough */
4620
4621	case TEMP_VAL_REG:
4622	tcg_out_st(s, ts->type, ts->reg,
4623	ts->mem_base->reg, ts->mem_offset);
4624	break;
4625
4626	case TEMP_VAL_MEM:
4627	break;
4628
4629	case TEMP_VAL_DEAD:
4630	default:
4631	g_assert_not_reached();
4632	}
4633	ts->mem_coherent = 1;
4634	}
4635	if (free_or_dead) {
4636	temp_free_or_dead(s, ts, free_or_dead);
4637	}
4638	}
4639
4640	/* free register 'reg' by spilling the corresponding temporary if necessary */
4641	static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
4642	{
4643	TCGTemp *ts = s->reg_to_temp[reg];
4644	if (ts != NULL) {
4645	temp_sync(s, ts, allocated_regs, 0, -1);
4646	}
4647	}
4648
4649	/**
4650	* tcg_reg_alloc:
4651	* @required_regs: Set of registers in which we must allocate.
4652	* @allocated_regs: Set of registers which must be avoided.
4653	* @preferred_regs: Set of registers we should prefer.
4654	* @rev: True if we search the registers in "indirect" order.
4655	*
4656	* The allocated register must be in @required_regs & ~@allocated_regs,
4657	* but if we can put it in @preferred_regs we may save a move later.
4658	*/
4659	static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
4660	TCGRegSet allocated_regs,
4661	TCGRegSet preferred_regs, bool rev)
4662	{
4663	int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
4664	TCGRegSet reg_ct[2];
4665	const int *order;
4666
4667	reg_ct[1] = required_regs & ~allocated_regs;
4668	tcg_debug_assert(reg_ct[1] != 0);
4669	reg_ct[0] = reg_ct[1] & preferred_regs;
4670
4671	/* Skip the preferred_regs option if it cannot be satisfied,
4672	or if the preference made no difference. */
4673	f = reg_ct[0] == 0 \|\| reg_ct[0] == reg_ct[1];
4674
4675	order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
4676
4677	/* Try free registers, preferences first. */
4678	for (j = f; j < 2; j++) {
4679	TCGRegSet set = reg_ct[j];
4680
4681	if (tcg_regset_single(set)) {
4682	/* One register in the set. */
4683	TCGReg reg = tcg_regset_first(set);
4684	if (s->reg_to_temp[reg] == NULL) {
4685	return reg;
4686	}
4687	} else {
4688	for (i = 0; i < n; i++) {
4689	TCGReg reg = order[i];
4690	if (s->reg_to_temp[reg] == NULL &&
4691	tcg_regset_test_reg(set, reg)) {
4692	return reg;
4693	}
4694	}
4695	}
4696	}
4697
4698	/* We must spill something. */
4699	for (j = f; j < 2; j++) {
4700	TCGRegSet set = reg_ct[j];
4701
4702	if (tcg_regset_single(set)) {
4703	/* One register in the set. */
4704	TCGReg reg = tcg_regset_first(set);
4705	tcg_reg_free(s, reg, allocated_regs);
4706	return reg;
4707	} else {
4708	for (i = 0; i < n; i++) {
4709	TCGReg reg = order[i];
4710	if (tcg_regset_test_reg(set, reg)) {
4711	tcg_reg_free(s, reg, allocated_regs);
4712	return reg;
4713	}
4714	}
4715	}
4716	}
4717
4718	g_assert_not_reached();
4719	}
4720
4721	static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs,
4722	TCGRegSet allocated_regs,
4723	TCGRegSet preferred_regs, bool rev)
4724	{
4725	int i, j, k, fmin, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
4726	TCGRegSet reg_ct[2];
4727	const int *order;
4728
4729	/* Ensure that if I is not in allocated_regs, I+1 is not either. */
4730	reg_ct[1] = required_regs & ~(allocated_regs \| (allocated_regs >> 1));
4731	tcg_debug_assert(reg_ct[1] != 0);
4732	reg_ct[0] = reg_ct[1] & preferred_regs;
4733
4734	order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
4735
4736	/*
4737	* Skip the preferred_regs option if it cannot be satisfied,
4738	* or if the preference made no difference.
4739	*/
4740	k = reg_ct[0] == 0 \|\| reg_ct[0] == reg_ct[1];
4741
4742	/*
4743	* Minimize the number of flushes by looking for 2 free registers first,
4744	* then a single flush, then two flushes.
4745	*/
4746	for (fmin = 2; fmin >= 0; fmin--) {
4747	for (j = k; j < 2; j++) {
4748	TCGRegSet set = reg_ct[j];
4749
4750	for (i = 0; i < n; i++) {
4751	TCGReg reg = order[i];
4752
4753	if (tcg_regset_test_reg(set, reg)) {
4754	int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1];
4755	if (f >= fmin) {
4756	tcg_reg_free(s, reg, allocated_regs);
4757	tcg_reg_free(s, reg + 1, allocated_regs);
4758	return reg;
4759	}
4760	}
4761	}
4762	}
4763	}
4764	g_assert_not_reached();
4765	}
4766
4767	/* Make sure the temporary is in a register. If needed, allocate the register
4768	from DESIRED while avoiding ALLOCATED. */
4769	static void temp_load(TCGContext s, TCGTemp ts, TCGRegSet desired_regs,
4770	TCGRegSet allocated_regs, TCGRegSet preferred_regs)
4771	{
4772	TCGReg reg;
4773
4774	switch (ts->val_type) {
4775	case TEMP_VAL_REG:
4776	return;
4777	case TEMP_VAL_CONST:
4778	reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
4779	preferred_regs, ts->indirect_base);
4780	if (ts->type <= TCG_TYPE_I64) {
4781	tcg_out_movi(s, ts->type, reg, ts->val);
4782	} else {
4783	uint64_t val = ts->val;
4784	MemOp vece = MO_64;
4785
4786	/*
4787	* Find the minimal vector element that matches the constant.
4788	* The targets will, in general, have to do this search anyway,
4789	* do this generically.
4790	*/
4791	if (val == dup_const(MO_8, val)) {
4792	vece = MO_8;
4793	} else if (val == dup_const(MO_16, val)) {
4794	vece = MO_16;
4795	} else if (val == dup_const(MO_32, val)) {
4796	vece = MO_32;
4797	}
4798
4799	tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val);
4800	}
4801	ts->mem_coherent = 0;
4802	break;
4803	case TEMP_VAL_MEM:
4804	if (!ts->mem_allocated) {
4805	temp_allocate_frame(s, ts);
4806	}
4807	reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
4808	preferred_regs, ts->indirect_base);
4809	tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
4810	ts->mem_coherent = 1;
4811	break;
4812	case TEMP_VAL_DEAD:
4813	default:
4814	g_assert_not_reached();
4815	}
4816	set_temp_val_reg(s, ts, reg);
4817	}
4818
4819	/* Save a temporary to memory. 'allocated_regs' is used in case a
4820	temporary registers needs to be allocated to store a constant. */
4821	static void temp_save(TCGContext s, TCGTemp ts, TCGRegSet allocated_regs)
4822	{
4823	/* The liveness analysis already ensures that globals are back
4824	in memory. Keep an tcg_debug_assert for safety. */
4825	tcg_debug_assert(ts->val_type == TEMP_VAL_MEM \|\| temp_readonly(ts));
4826	}
4827
4828	/* save globals to their canonical location and assume they can be
4829	modified be the following code. 'allocated_regs' is used in case a
4830	temporary registers needs to be allocated to store a constant. */
4831	static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
4832	{
4833	int i, n;
4834
4835	for (i = 0, n = s->nb_globals; i < n; i++) {
4836	temp_save(s, &s->temps[i], allocated_regs);
4837	}
4838	}
4839
4840	/* sync globals to their canonical location and assume they can be
4841	read by the following code. 'allocated_regs' is used in case a
4842	temporary registers needs to be allocated to store a constant. */
4843	static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
4844	{
4845	int i, n;
4846
4847	for (i = 0, n = s->nb_globals; i < n; i++) {
4848	TCGTemp *ts = &s->temps[i];
4849	tcg_debug_assert(ts->val_type != TEMP_VAL_REG
4850	\|\| ts->kind == TEMP_FIXED
4851	\|\| ts->mem_coherent);
4852	}
4853	}
4854
4855	/* at the end of a basic block, we assume all temporaries are dead and
4856	all globals are stored at their canonical location. */
4857	static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
4858	{
4859	assert_carry_dead(s);
4860	for (int i = s->nb_globals; i < s->nb_temps; i++) {
4861	TCGTemp *ts = &s->temps[i];
4862
4863	switch (ts->kind) {
4864	case TEMP_TB:
4865	temp_save(s, ts, allocated_regs);
4866	break;
4867	case TEMP_EBB:
4868	/* The liveness analysis already ensures that temps are dead.
4869	Keep an tcg_debug_assert for safety. */
4870	tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
4871	break;
4872	case TEMP_CONST:
4873	/* Similarly, we should have freed any allocated register. */
4874	tcg_debug_assert(ts->val_type == TEMP_VAL_CONST);
4875	break;
4876	default:
4877	g_assert_not_reached();
4878	}
4879	}
4880
4881	save_globals(s, allocated_regs);
4882	}
4883
4884	/*
4885	* At a conditional branch, we assume all temporaries are dead unless
4886	* explicitly live-across-conditional-branch; all globals and local
4887	* temps are synced to their location.
4888	*/
4889	static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs)
4890	{
4891	assert_carry_dead(s);
4892	sync_globals(s, allocated_regs);
4893
4894	for (int i = s->nb_globals; i < s->nb_temps; i++) {
4895	TCGTemp *ts = &s->temps[i];
4896	/*
4897	* The liveness analysis already ensures that temps are dead.
4898	* Keep tcg_debug_asserts for safety.
4899	*/
4900	switch (ts->kind) {
4901	case TEMP_TB:
4902	tcg_debug_assert(ts->val_type != TEMP_VAL_REG \|\| ts->mem_coherent);
4903	break;
4904	case TEMP_EBB:
4905	case TEMP_CONST:
4906	break;
4907	default:
4908	g_assert_not_reached();
4909	}
4910	}
4911	}
4912
4913	/*
4914	* Specialized code generation for INDEX_op_mov_* with a constant.
4915	*/
4916	static void tcg_reg_alloc_do_movi(TCGContext s, TCGTemp ots,
4917	tcg_target_ulong val, TCGLifeData arg_life,
4918	TCGRegSet preferred_regs)
4919	{
4920	/* ENV should not be modified. */
4921	tcg_debug_assert(!temp_readonly(ots));
4922
4923	/* The movi is not explicitly generated here. */
4924	set_temp_val_nonreg(s, ots, TEMP_VAL_CONST);
4925	ots->val = val;
4926	ots->mem_coherent = 0;
4927	if (NEED_SYNC_ARG(0)) {
4928	temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0));
4929	} else if (IS_DEAD_ARG(0)) {
4930	temp_dead(s, ots);
4931	}
4932	}
4933
4934	/*
4935	* Specialized code generation for INDEX_op_mov_*.
4936	*/
4937	static void tcg_reg_alloc_mov(TCGContext s, const TCGOp op)
4938	{
4939	const TCGLifeData arg_life = op->life;
4940	TCGRegSet allocated_regs, preferred_regs;
4941	TCGTemp ts, ots;
4942	TCGType otype, itype;
4943	TCGReg oreg, ireg;
4944
4945	allocated_regs = s->reserved_regs;
4946	preferred_regs = output_pref(op, 0);
4947	ots = arg_temp(op->args[0]);
4948	ts = arg_temp(op->args[1]);
4949
4950	/* ENV should not be modified. */
4951	tcg_debug_assert(!temp_readonly(ots));
4952
4953	/* Note that otype != itype for no-op truncation. */
4954	otype = ots->type;
4955	itype = ts->type;
4956
4957	if (ts->val_type == TEMP_VAL_CONST) {
4958	/* propagate constant or generate sti */
4959	tcg_target_ulong val = ts->val;
4960	if (IS_DEAD_ARG(1)) {
4961	temp_dead(s, ts);
4962	}
4963	tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs);
4964	return;
4965	}
4966
4967	/* If the source value is in memory we're going to be forced
4968	to have it in a register in order to perform the copy. Copy
4969	the SOURCE value into its own register first, that way we
4970	don't have to reload SOURCE the next time it is used. */
4971	if (ts->val_type == TEMP_VAL_MEM) {
4972	temp_load(s, ts, tcg_target_available_regs[itype],
4973	allocated_regs, preferred_regs);
4974	}
4975	tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
4976	ireg = ts->reg;
4977
4978	if (IS_DEAD_ARG(0)) {
4979	/* mov to a non-saved dead register makes no sense (even with
4980	liveness analysis disabled). */
4981	tcg_debug_assert(NEED_SYNC_ARG(0));
4982	if (!ots->mem_allocated) {
4983	temp_allocate_frame(s, ots);
4984	}
4985	tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset);
4986	if (IS_DEAD_ARG(1)) {
4987	temp_dead(s, ts);
4988	}
4989	temp_dead(s, ots);
4990	return;
4991	}
4992
4993	if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) {
4994	/*
4995	* The mov can be suppressed. Kill input first, so that it
4996	* is unlinked from reg_to_temp, then set the output to the
4997	* reg that we saved from the input.
4998	*/
4999	temp_dead(s, ts);
5000	oreg = ireg;

Showing first 5,000 of 7,067 lines. View raw