translate-a64.c · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

c 11,314 lines 337 KB

Raw

1	/*
2	* AArch64 translation
3	*
4	* Copyright (c) 2013 Alexander Graf <agraf@suse.de>
5	*
6	* This library is free software; you can redistribute it and/or
7	* modify it under the terms of the GNU Lesser General Public
8	* License as published by the Free Software Foundation; either
9	* version 2.1 of the License, or (at your option) any later version.
10	*
11	* This library is distributed in the hope that it will be useful,
12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14	* Lesser General Public License for more details.
15	*
16	* You should have received a copy of the GNU Lesser General Public
17	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
18	*/
19	#include "qemu/osdep.h"
20	#include "exec/target_page.h"
21	#include "exec/translator.h"
22	#include "helper-a64.h"
23	#include "helper-sme.h"
24	#include "helper-sve.h"
25	#include "helper-fp8.h"
26	#include "translate.h"
27	#include "translate-a64.h"
28	#include "tcg/tcg-op.h"
29	#include "qemu/log.h"
30	#include "semihosting/semihost.h"
31	#include "cpregs.h"
32
33	static TCGv_i64 cpu_X[32];
34	static TCGv_i64 cpu_gcspr[4];
35	static TCGv_i64 cpu_pc;
36
37	/* Load/store exclusive handling */
38	static TCGv_i64 cpu_exclusive_high;
39
40	static const char *regnames[] = {
41	"x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7",
42	"x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15",
43	"x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23",
44	"x24", "x25", "x26", "x27", "x28", "x29", "lr", "sp"
45	};
46
47	enum a64_shift_type {
48	A64_SHIFT_TYPE_LSL = 0,
49	A64_SHIFT_TYPE_LSR = 1,
50	A64_SHIFT_TYPE_ASR = 2,
51	A64_SHIFT_TYPE_ROR = 3
52	};
53
54	/*
55	* Helpers for extracting complex instruction fields
56	*/
57
58	/*
59	* For load/store with an unsigned 12 bit immediate scaled by the element
60	* size. The input has the immediate field in bits [14:3] and the element
61	* size in [2:0].
62	*/
63	static int uimm_scaled(DisasContext *s, int x)
64	{
65	unsigned imm = x >> 3;
66	unsigned scale = extract32(x, 0, 3);
67	return imm << scale;
68	}
69
70	/* For load/store memory tags: scale offset by LOG2_TAG_GRANULE */
71	static int scale_by_log2_tag_granule(DisasContext *s, int x)
72	{
73	return x << LOG2_TAG_GRANULE;
74	}
75
76	/*
77	* Include the generated decoders.
78	*/
79
80	#include "decode-sme-fa64.c.inc"
81	#include "decode-a64.c.inc"
82
83	/* initialize TCG globals. */
84	void a64_translate_init(void)
85	{
86	static const char gcspr_names[4][12] = {
87	"gcspr_el0", "gcspr_el1", "gcspr_el2", "gcspr_el3"
88	};
89
90	int i;
91
92	cpu_pc = tcg_global_mem_new_i64(tcg_env,
93	offsetof(CPUARMState, pc),
94	"pc");
95	for (i = 0; i < 32; i++) {
96	cpu_X[i] = tcg_global_mem_new_i64(tcg_env,
97	offsetof(CPUARMState, xregs[i]),
98	regnames[i]);
99	}
100
101	cpu_exclusive_high = tcg_global_mem_new_i64(tcg_env,
102	offsetof(CPUARMState, exclusive_high), "exclusive_high");
103
104	for (i = 0; i < 4; i++) {
105	cpu_gcspr[i] =
106	tcg_global_mem_new_i64(tcg_env,
107	offsetof(CPUARMState, cp15.gcspr_el[i]),
108	gcspr_names[i]);
109	}
110	}
111
112	/*
113	* Return the full arm mmu_idx to use for A64 load/store insns which
114	* have a "unprivileged load/store" variant. Those insns access
115	* EL0 if executed from an EL which has control over EL0 (usually
116	* EL1) but behave like normal loads and stores if executed from
117	* elsewhere (eg EL3).
118	*
119	* @unpriv : true for the unprivileged encoding; false for the
120	* normal encoding (in which case we will return the same
121	* thing as get_mem_index().
122	*/
123	static ARMMMUIdx full_a64_user_mem_index(DisasContext *s, bool unpriv)
124	{
125	/*
126	* If AccType_UNPRIV is not used, the insn uses AccType_NORMAL,
127	* which is the usual mmu_idx for this cpu state.
128	*/
129	ARMMMUIdx useridx = s->mmu_idx;
130
131	if (unpriv && s->unpriv) {
132	/*
133	* We have pre-computed the condition for AccType_UNPRIV.
134	* Therefore we should never get here with a mmu_idx for
135	* which we do not know the corresponding user mmu_idx.
136	*/
137	switch (useridx) {
138	case ARMMMUIdx_E10_1:
139	case ARMMMUIdx_E10_1_PAN:
140	useridx = ARMMMUIdx_E10_0;
141	break;
142	case ARMMMUIdx_E20_2:
143	case ARMMMUIdx_E20_2_PAN:
144	useridx = ARMMMUIdx_E20_0;
145	break;
146	default:
147	g_assert_not_reached();
148	}
149	}
150	return useridx;
151	}
152
153	/* Return the core mmu_idx per above. */
154	static int core_a64_user_mem_index(DisasContext *s, bool unpriv)
155	{
156	return arm_to_core_mmu_idx(full_a64_user_mem_index(s, unpriv));
157	}
158
159	/* For a given translation regime, return the core mmu_idx for gcs access. */
160	static int core_gcs_mem_index(ARMMMUIdx armidx)
161	{
162	return arm_to_core_mmu_idx(regime_to_gcs(armidx));
163	}
164
165	static void set_btype_raw(int val)
166	{
167	tcg_gen_st_i32(tcg_constant_i32(val), tcg_env,
168	offsetof(CPUARMState, btype));
169	}
170
171	static void set_btype(DisasContext *s, int val)
172	{
173	/* BTYPE is a 2-bit field, and 0 should be done with reset_btype. */
174	tcg_debug_assert(val >= 1 && val <= 3);
175	set_btype_raw(val);
176	s->btype = -1;
177	}
178
179	static void reset_btype(DisasContext *s)
180	{
181	if (s->btype != 0) {
182	set_btype_raw(0);
183	s->btype = 0;
184	}
185	}
186
187	static void gen_pc_plus_diff(DisasContext *s, TCGv_i64 dest, target_long diff)
188	{
189	assert(s->pc_save != -1);
190	if (tb_cflags(s->base.tb) & CF_PCREL) {
191	tcg_gen_addi_i64(dest, cpu_pc, (s->pc_curr - s->pc_save) + diff);
192	} else {
193	tcg_gen_movi_i64(dest, s->pc_curr + diff);
194	}
195	}
196
197	void gen_a64_update_pc(DisasContext *s, target_long diff)
198	{
199	gen_pc_plus_diff(s, cpu_pc, diff);
200	s->pc_save = s->pc_curr + diff;
201	}
202
203	/*
204	* Handle Top Byte Ignore (TBI) bits.
205	*
206	* If address tagging is enabled via the TCR TBI bits:
207	* + for EL2 and EL3 there is only one TBI bit, and if it is set
208	* then the address is zero-extended, clearing bits [63:56]
209	* + for EL0 and EL1, TBI0 controls addresses with bit 55 == 0
210	* and TBI1 controls addresses with bit 55 == 1.
211	* If the appropriate TBI bit is set for the address then
212	* the address is sign-extended from bit 55 into bits [63:56]
213	*
214	* Here We have concatenated TBI{1,0} into tbi.
215	*/
216	static void gen_top_byte_ignore(DisasContext *s, TCGv_i64 dst,
217	TCGv_i64 src, int tbi)
218	{
219	if (tbi == 0) {
220	/* Load unmodified address */
221	tcg_gen_mov_i64(dst, src);
222	} else if (!regime_has_2_ranges(s->mmu_idx)) {
223	/* Force tag byte to all zero */
224	tcg_gen_extract_i64(dst, src, 0, 56);
225	} else {
226	/* Sign-extend from bit 55. */
227	tcg_gen_sextract_i64(dst, src, 0, 56);
228
229	switch (tbi) {
230	case 1:
231	/* tbi0 but !tbi1: only use the extension if positive */
232	tcg_gen_and_i64(dst, dst, src);
233	break;
234	case 2:
235	/* !tbi0 but tbi1: only use the extension if negative */
236	tcg_gen_or_i64(dst, dst, src);
237	break;
238	case 3:
239	/* tbi0 and tbi1: always use the extension */
240	break;
241	default:
242	g_assert_not_reached();
243	}
244	}
245	}
246
247	static void gen_a64_set_pc(DisasContext *s, TCGv_i64 src)
248	{
249	/*
250	* If address tagging is enabled for instructions via the TCR TBI bits,
251	* then loading an address into the PC will clear out any tag.
252	*/
253	gen_top_byte_ignore(s, cpu_pc, src, s->tbii);
254	s->pc_save = -1;
255	}
256
257	/*
258	* Handle MTE and/or TBI.
259	*
260	* For TBI, ideally, we would do nothing. Proper behaviour on fault is
261	* for the tag to be present in the FAR_ELx register. But for user-only
262	* mode we do not have a TLB with which to implement this, so we must
263	* remove the top byte now.
264	*
265	* Always return a fresh temporary that we can increment independently
266	* of the write-back address.
267	*/
268
269	TCGv_i64 clean_data_tbi(DisasContext *s, TCGv_i64 addr)
270	{
271	TCGv_i64 clean = tcg_temp_new_i64();
272	#ifdef CONFIG_USER_ONLY
273	gen_top_byte_ignore(s, clean, addr, s->tbid);
274	#else
275	tcg_gen_mov_i64(clean, addr);
276	#endif
277	return clean;
278	}
279
280	/* Insert a zero tag into src, with the result at dst. */
281	static void gen_address_with_allocation_tag0(TCGv_i64 dst, TCGv_i64 src)
282	{
283	tcg_gen_andi_i64(dst, src, ~MAKE_64BIT_MASK(56, 4));
284	}
285
286	static void gen_probe_access(DisasContext *s, TCGv_i64 ptr,
287	MMUAccessType acc, int log2_size)
288	{
289	gen_helper_probe_access(tcg_env, ptr,
290	tcg_constant_i32(acc),
291	tcg_constant_i32(get_mem_index(s)),
292	tcg_constant_i32(1 << log2_size));
293	}
294
295	/*
296	* For MTE, check a single logical or atomic access. This probes a single
297	* address, the exact one specified. The size and alignment of the access
298	* is not relevant to MTE, per se, but watchpoints do require the size,
299	* and we want to recognize those before making any other changes to state.
300	*/
301	static TCGv_i64 gen_mte_check1_mmuidx(DisasContext *s, TCGv_i64 addr,
302	bool is_write, bool tag_checked,
303	MemOp memop, bool is_unpriv,
304	int core_idx)
305	{
306	if (tag_checked && s->mte_active[is_unpriv] &&
307	(is_write \|\| !s->mte_store_only[is_unpriv])) {
308	TCGv_i64 ret;
309	int desc = 0;
310
311	desc = FIELD_DP32(desc, MTEDESC, MIDX, core_idx);
312	desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
313	desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
314	desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write);
315	desc = FIELD_DP32(desc, MTEDESC, ALIGN, memop_alignment_bits(memop));
316	desc = FIELD_DP32(desc, MTEDESC, MTX, s->mtx);
317	desc = FIELD_DP32(desc, MTEDESC, SIZEM1, memop_size(memop) - 1);
318
319	ret = tcg_temp_new_i64();
320	gen_helper_mte_check(ret, tcg_env, tcg_constant_i32(desc), addr);
321
322	return ret;
323	}
324	return clean_data_tbi(s, addr);
325	}
326
327	TCGv_i64 gen_mte_check1(DisasContext *s, TCGv_i64 addr, bool is_write,
328	bool tag_checked, MemOp memop)
329	{
330	return gen_mte_check1_mmuidx(s, addr, is_write, tag_checked, memop,
331	false, get_mem_index(s));
332	}
333
334	/*
335	* For MTE, check multiple logical sequential accesses.
336	*/
337	TCGv_i64 gen_mte_checkN(DisasContext *s, TCGv_i64 addr, bool is_write,
338	bool tag_checked, int total_size, MemOp single_mop)
339	{
340	if (tag_checked && s->mte_active[0] &&
341	(is_write \|\| !s->mte_store_only[0])) {
342	TCGv_i64 ret;
343	int desc = 0;
344
345	desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s));
346	desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
347	desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
348	desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write);
349	desc = FIELD_DP32(desc, MTEDESC, ALIGN, memop_alignment_bits(single_mop));
350	desc = FIELD_DP32(desc, MTEDESC, MTX, s->mtx);
351	desc = FIELD_DP32(desc, MTEDESC, SIZEM1, total_size - 1);
352
353	ret = tcg_temp_new_i64();
354	gen_helper_mte_check(ret, tcg_env, tcg_constant_i32(desc), addr);
355
356	return ret;
357	}
358	return clean_data_tbi(s, addr);
359	}
360
361	/*
362	* Generate the special alignment check that applies to AccType_ATOMIC
363	* and AccType_ORDERED insns under FEAT_LSE2: the access need not be
364	* naturally aligned, but it must not cross a 16-byte boundary.
365	* See AArch64.CheckAlignment().
366	*/
367	static void check_lse2_align(DisasContext *s, int rn, int imm,
368	bool is_write, MemOp mop)
369	{
370	TCGv_i32 tmp;
371	TCGv_i64 addr;
372	TCGLabel *over_label;
373	MMUAccessType type;
374	int mmu_idx;
375
376	tmp = tcg_temp_new_i32();
377	tcg_gen_extrl_i64_i32(tmp, cpu_reg_sp(s, rn));
378	tcg_gen_addi_i32(tmp, tmp, imm & 15);
379	tcg_gen_andi_i32(tmp, tmp, 15);
380	tcg_gen_addi_i32(tmp, tmp, memop_size(mop));
381
382	over_label = gen_new_label();
383	tcg_gen_brcondi_i32(TCG_COND_LEU, tmp, 16, over_label);
384
385	addr = tcg_temp_new_i64();
386	tcg_gen_addi_i64(addr, cpu_reg_sp(s, rn), imm);
387
388	type = is_write ? MMU_DATA_STORE : MMU_DATA_LOAD,
389	mmu_idx = get_mem_index(s);
390	gen_helper_arm_unaligned_access(tcg_env, addr, tcg_constant_i32(type),
391	tcg_constant_i32(mmu_idx));
392
393	gen_set_label(over_label);
394
395	}
396
397	/* Handle the alignment check for AccType_ATOMIC instructions. */
398	static MemOp check_atomic_align(DisasContext *s, int rn, MemOp mop)
399	{
400	MemOp size = mop & MO_SIZE;
401
402	if (size == MO_8) {
403	return mop;
404	}
405
406	/*
407	* If size == MO_128, this is a LDXP, and the operation is single-copy
408	* atomic for each doubleword, not the entire quadword; it still must
409	* be quadword aligned.
410	*/
411	if (size == MO_128) {
412	return finalize_memop_atom(s, MO_128 \| MO_ALIGN,
413	MO_ATOM_IFALIGN_PAIR);
414	}
415	if (dc_isar_feature(aa64_lse2, s)) {
416	check_lse2_align(s, rn, 0, true, mop);
417	} else {
418	mop \|= MO_ALIGN;
419	}
420	return finalize_memop(s, mop);
421	}
422
423	/* Handle the alignment check for AccType_ORDERED instructions. */
424	static MemOp check_ordered_align(DisasContext *s, int rn, int imm,
425	bool is_write, MemOp mop)
426	{
427	MemOp size = mop & MO_SIZE;
428
429	if (size == MO_8) {
430	return mop;
431	}
432	if (size == MO_128) {
433	return finalize_memop_atom(s, MO_128 \| MO_ALIGN,
434	MO_ATOM_IFALIGN_PAIR);
435	}
436	if (!dc_isar_feature(aa64_lse2, s)) {
437	mop \|= MO_ALIGN;
438	} else if (!s->naa) {
439	check_lse2_align(s, rn, imm, is_write, mop);
440	}
441	return finalize_memop(s, mop);
442	}
443
444	static void gen_add_gcs_record(DisasContext *s, TCGv_i64 value)
445	{
446	TCGv_i64 addr = tcg_temp_new_i64();
447	TCGv_i64 gcspr = cpu_gcspr[s->current_el];
448	int mmuidx = core_gcs_mem_index(s->mmu_idx);
449	MemOp mop = finalize_memop(s, MO_64 \| MO_ALIGN);
450
451	tcg_gen_addi_i64(addr, gcspr, -8);
452	tcg_gen_qemu_st_i64(value, clean_data_tbi(s, addr), mmuidx, mop);
453	tcg_gen_mov_i64(gcspr, addr);
454	}
455
456	static void gen_load_check_gcs_record(DisasContext *s, TCGv_i64 target,
457	GCSInstructionType it, int rt)
458	{
459	TCGv_i64 gcspr = cpu_gcspr[s->current_el];
460	int mmuidx = core_gcs_mem_index(s->mmu_idx);
461	MemOp mop = finalize_memop(s, MO_64 \| MO_ALIGN);
462	TCGv_i64 rec_va = tcg_temp_new_i64();
463
464	tcg_gen_qemu_ld_i64(rec_va, clean_data_tbi(s, gcspr), mmuidx, mop);
465
466	if (s->gcs_rvcen) {
467	TCGLabel *fail_label =
468	delay_exception(s, EXCP_UDEF, syn_gcs_data_check(it, rt));
469
470	tcg_gen_brcond_i64(TCG_COND_NE, rec_va, target, fail_label);
471	}
472
473	gen_a64_set_pc(s, rec_va);
474	tcg_gen_addi_i64(gcspr, gcspr, 8);
475	}
476
477	typedef struct DisasCompare64 {
478	TCGCond cond;
479	TCGv_i64 value;
480	} DisasCompare64;
481
482	static void a64_test_cc(DisasCompare64 *c64, int cc)
483	{
484	DisasCompare c32;
485
486	arm_test_cc(&c32, cc);
487
488	/*
489	* Sign-extend the 32-bit value so that the GE/LT comparisons work
490	* properly. The NE/EQ comparisons are also fine with this choice.
491	*/
492	c64->cond = c32.cond;
493	c64->value = tcg_temp_new_i64();
494	tcg_gen_ext_i32_i64(c64->value, c32.value);
495	}
496
497	static void gen_rebuild_hflags(DisasContext *s)
498	{
499	gen_helper_rebuild_hflags_a64(tcg_env, tcg_constant_i32(s->current_el));
500	}
501
502	static void gen_exception_internal_insn(DisasContext *s, int excp)
503	{
504	gen_a64_update_pc(s, 0);
505	gen_exception_internal(excp);
506	s->base.is_jmp = DISAS_NORETURN;
507	}
508
509	static void gen_exception_bkpt_insn(DisasContext *s, uint32_t syndrome)
510	{
511	gen_a64_update_pc(s, 0);
512	gen_helper_exception_bkpt_insn(tcg_env, tcg_constant_i32(syndrome));
513	s->base.is_jmp = DISAS_NORETURN;
514	}
515
516	static void gen_step_complete_exception(DisasContext *s)
517	{
518	/* We just completed step of an insn. Move from Active-not-pending
519	* to Active-pending, and then also take the swstep exception.
520	* This corresponds to making the (IMPDEF) choice to prioritize
521	* swstep exceptions over asynchronous exceptions taken to an exception
522	* level where debug is disabled. This choice has the advantage that
523	* we do not need to maintain internal state corresponding to the
524	* ISV/EX syndrome bits between completion of the step and generation
525	* of the exception, and our syndrome information is always correct.
526	*/
527	gen_ss_advance(s);
528	gen_swstep_exception(s, 1, s->is_ldex);
529	s->base.is_jmp = DISAS_NORETURN;
530	}
531
532	static inline bool use_goto_tb(DisasContext *s, uint64_t dest)
533	{
534	if (s->ss_active) {
535	return false;
536	}
537	return translator_use_goto_tb(&s->base, dest);
538	}
539
540	static void gen_goto_tb(DisasContext *s, unsigned tb_slot_idx, int64_t diff)
541	{
542	if (use_goto_tb(s, s->pc_curr + diff)) {
543	/*
544	* For pcrel, the pc must always be up-to-date on entry to
545	* the linked TB, so that it can use simple additions for all
546	* further adjustments. For !pcrel, the linked TB is compiled
547	* to know its full virtual address, so we can delay the
548	* update to pc to the unlinked path. A long chain of links
549	* can thus avoid many updates to the PC.
550	*/
551	if (tb_cflags(s->base.tb) & CF_PCREL) {
552	gen_a64_update_pc(s, diff);
553	tcg_gen_goto_tb(tb_slot_idx);
554	} else {
555	tcg_gen_goto_tb(tb_slot_idx);
556	gen_a64_update_pc(s, diff);
557	}
558	tcg_gen_exit_tb(s->base.tb, tb_slot_idx);
559	s->base.is_jmp = DISAS_NORETURN;
560	} else {
561	gen_a64_update_pc(s, diff);
562	if (s->ss_active) {
563	gen_step_complete_exception(s);
564	} else {
565	tcg_gen_lookup_and_goto_ptr();
566	s->base.is_jmp = DISAS_NORETURN;
567	}
568	}
569	}
570
571	/*
572	* Register access functions
573	*
574	* These functions are used for directly accessing a register in where
575	* changes to the final register value are likely to be made. If you
576	* need to use a register for temporary calculation (e.g. index type
577	* operations) use the read_* form.
578	*
579	* B1.2.1 Register mappings
580	*
581	* In instruction register encoding 31 can refer to ZR (zero register) or
582	* the SP (stack pointer) depending on context. In QEMU's case we map SP
583	* to cpu_X[31] and ZR accesses to a temporary which can be discarded.
584	* This is the point of the _sp forms.
585	*/
586	TCGv_i64 cpu_reg(DisasContext *s, int reg)
587	{
588	if (reg == 31) {
589	TCGv_i64 t = tcg_temp_new_i64();
590	tcg_gen_movi_i64(t, 0);
591	return t;
592	} else {
593	return cpu_X[reg];
594	}
595	}
596
597	/* register access for when 31 == SP */
598	TCGv_i64 cpu_reg_sp(DisasContext *s, int reg)
599	{
600	return cpu_X[reg];
601	}
602
603	/* read a cpu register in 32bit/64bit mode. Returns a TCGv_i64
604	* representing the register contents. This TCGv is an auto-freed
605	* temporary so it need not be explicitly freed, and may be modified.
606	*/
607	TCGv_i64 read_cpu_reg(DisasContext *s, int reg, int sf)
608	{
609	TCGv_i64 v = tcg_temp_new_i64();
610	if (reg != 31) {
611	if (sf) {
612	tcg_gen_mov_i64(v, cpu_X[reg]);
613	} else {
614	tcg_gen_ext32u_i64(v, cpu_X[reg]);
615	}
616	} else {
617	tcg_gen_movi_i64(v, 0);
618	}
619	return v;
620	}
621
622	TCGv_i64 read_cpu_reg_sp(DisasContext *s, int reg, int sf)
623	{
624	TCGv_i64 v = tcg_temp_new_i64();
625	if (sf) {
626	tcg_gen_mov_i64(v, cpu_X[reg]);
627	} else {
628	tcg_gen_ext32u_i64(v, cpu_X[reg]);
629	}
630	return v;
631	}
632
633	/* Return the offset into CPUARMState of a slice (from
634	* the least significant end) of FP register Qn (ie
635	* Dn, Sn, Hn or Bn).
636	* (Note that this is not the same mapping as for A32; see cpu.h)
637	*/
638	static inline int fp_reg_offset(DisasContext *s, int regno, MemOp size)
639	{
640	return vec_reg_offset(s, regno, 0, size);
641	}
642
643	/* Offset of the high half of the 128 bit vector Qn */
644	static inline int fp_reg_hi_offset(DisasContext *s, int regno)
645	{
646	return vec_reg_offset(s, regno, 1, MO_64);
647	}
648
649	/* Convenience accessors for reading and writing single and double
650	* FP registers. Writing clears the upper parts of the associated
651	* 128 bit vector register, as required by the architecture.
652	* Note that unlike the GP register accessors, the values returned
653	* by the read functions must be manually freed.
654	*/
655	static TCGv_i64 read_fp_dreg(DisasContext *s, int reg)
656	{
657	TCGv_i64 v = tcg_temp_new_i64();
658
659	tcg_gen_ld_i64(v, tcg_env, fp_reg_offset(s, reg, MO_64));
660	return v;
661	}
662
663	static TCGv_i32 read_fp_sreg(DisasContext *s, int reg)
664	{
665	TCGv_i32 v = tcg_temp_new_i32();
666
667	tcg_gen_ld_i32(v, tcg_env, fp_reg_offset(s, reg, MO_32));
668	return v;
669	}
670
671	static TCGv_i32 read_fp_hreg(DisasContext *s, int reg)
672	{
673	TCGv_i32 v = tcg_temp_new_i32();
674
675	tcg_gen_ld16u_i32(v, tcg_env, fp_reg_offset(s, reg, MO_16));
676	return v;
677	}
678
679	static void clear_vec(DisasContext *s, int rd)
680	{
681	unsigned ofs = fp_reg_offset(s, rd, MO_64);
682	unsigned vsz = vec_full_reg_size(s);
683
684	tcg_gen_gvec_dup_imm(MO_64, ofs, vsz, vsz, 0);
685	}
686
687	/*
688	* Clear the bits above an N-bit vector, for N = (is_q ? 128 : 64).
689	* If SVE is not enabled, then there are only 128 bits in the vector.
690	*/
691	static void clear_vec_high(DisasContext *s, bool is_q, int rd)
692	{
693	unsigned ofs = fp_reg_offset(s, rd, MO_64);
694	unsigned vsz = vec_full_reg_size(s);
695
696	/* Nop move, with side effect of clearing the tail. */
697	tcg_gen_gvec_mov(MO_64, ofs, ofs, is_q ? 16 : 8, vsz);
698	}
699
700	void write_fp_dreg(DisasContext *s, int reg, TCGv_i64 v)
701	{
702	unsigned ofs = fp_reg_offset(s, reg, MO_64);
703
704	tcg_gen_st_i64(v, tcg_env, ofs);
705	clear_vec_high(s, false, reg);
706	}
707
708	static void write_fp_sreg(DisasContext *s, int reg, TCGv_i32 v)
709	{
710	TCGv_i64 tmp = tcg_temp_new_i64();
711
712	tcg_gen_extu_i32_i64(tmp, v);
713	write_fp_dreg(s, reg, tmp);
714	}
715
716	/*
717	* Write a double result to 128 bit vector register reg, honouring FPCR.NEP:
718	* - if FPCR.NEP == 0, clear the high elements of reg
719	* - if FPCR.NEP == 1, set the high elements of reg from mergereg
720	* (i.e. merge the result with those high elements)
721	* In either case, SVE register bits above 128 are zeroed (per R_WKYLB).
722	*/
723	static void write_fp_dreg_merging(DisasContext *s, int reg, int mergereg,
724	TCGv_i64 v)
725	{
726	if (!s->fpcr_nep) {
727	write_fp_dreg(s, reg, v);
728	return;
729	}
730
731	/*
732	* Move from mergereg to reg; this sets the high elements and
733	* clears the bits above 128 as a side effect.
734	*/
735	tcg_gen_gvec_mov(MO_64, vec_full_reg_offset(s, reg),
736	vec_full_reg_offset(s, mergereg),
737	16, vec_full_reg_size(s));
738	tcg_gen_st_i64(v, tcg_env, vec_full_reg_offset(s, reg));
739	}
740
741	/*
742	* Write a single-prec result, but only clear the higher elements
743	* of the destination register if FPCR.NEP is 0; otherwise preserve them.
744	*/
745	static void write_fp_sreg_merging(DisasContext *s, int reg, int mergereg,
746	TCGv_i32 v)
747	{
748	if (!s->fpcr_nep) {
749	write_fp_sreg(s, reg, v);
750	return;
751	}
752
753	tcg_gen_gvec_mov(MO_64, vec_full_reg_offset(s, reg),
754	vec_full_reg_offset(s, mergereg),
755	16, vec_full_reg_size(s));
756	tcg_gen_st_i32(v, tcg_env, fp_reg_offset(s, reg, MO_32));
757	}
758
759	/*
760	* Write a half-prec result, but only clear the higher elements
761	* of the destination register if FPCR.NEP is 0; otherwise preserve them.
762	* The caller must ensure that the top 16 bits of v are zero.
763	*/
764	static void write_fp_hreg_merging(DisasContext *s, int reg, int mergereg,
765	TCGv_i32 v)
766	{
767	if (!s->fpcr_nep) {
768	write_fp_sreg(s, reg, v);
769	return;
770	}
771
772	tcg_gen_gvec_mov(MO_64, vec_full_reg_offset(s, reg),
773	vec_full_reg_offset(s, mergereg),
774	16, vec_full_reg_size(s));
775	tcg_gen_st16_i32(v, tcg_env, fp_reg_offset(s, reg, MO_16));
776	}
777
778	/* Expand a 2-operand AdvSIMD vector operation using an expander function. */
779	static void gen_gvec_fn2(DisasContext *s, bool is_q, int rd, int rn,
780	GVecGen2Fn *gvec_fn, int vece)
781	{
782	gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
783	is_q ? 16 : 8, vec_full_reg_size(s));
784	}
785
786	/* Expand a 2-operand + immediate AdvSIMD vector operation using
787	* an expander function.
788	*/
789	static void gen_gvec_fn2i(DisasContext *s, bool is_q, int rd, int rn,
790	int64_t imm, GVecGen2iFn *gvec_fn, int vece)
791	{
792	gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
793	imm, is_q ? 16 : 8, vec_full_reg_size(s));
794	}
795
796	/* Expand a 3-operand AdvSIMD vector operation using an expander function. */
797	static void gen_gvec_fn3(DisasContext *s, bool is_q, int rd, int rn, int rm,
798	GVecGen3Fn *gvec_fn, int vece)
799	{
800	gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
801	vec_full_reg_offset(s, rm), is_q ? 16 : 8, vec_full_reg_size(s));
802	}
803
804	/* Expand a 4-operand AdvSIMD vector operation using an expander function. */
805	static void gen_gvec_fn4(DisasContext *s, bool is_q, int rd, int rn, int rm,
806	int rx, GVecGen4Fn *gvec_fn, int vece)
807	{
808	gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
809	vec_full_reg_offset(s, rm), vec_full_reg_offset(s, rx),
810	is_q ? 16 : 8, vec_full_reg_size(s));
811	}
812
813	/* Expand a 2-operand operation using an out-of-line helper. */
814	static void gen_gvec_op2_ool(DisasContext *s, bool is_q, int rd,
815	int rn, int data, gen_helper_gvec_2 *fn)
816	{
817	tcg_gen_gvec_2_ool(vec_full_reg_offset(s, rd),
818	vec_full_reg_offset(s, rn),
819	is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
820	}
821
822	/* Expand a 3-operand operation using an out-of-line helper. */
823	static void gen_gvec_op3_ool(DisasContext *s, bool is_q, int rd,
824	int rn, int rm, int data, gen_helper_gvec_3 *fn)
825	{
826	tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd),
827	vec_full_reg_offset(s, rn),
828	vec_full_reg_offset(s, rm),
829	is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
830	}
831
832	/* Expand a 3-operand + fpstatus pointer + simd data value operation using
833	* an out-of-line helper.
834	*/
835	static void gen_gvec_op3_fpst(DisasContext *s, bool is_q, int rd, int rn,
836	int rm, ARMFPStatusFlavour fpsttype, int data,
837	gen_helper_gvec_3_ptr *fn)
838	{
839	TCGv_ptr fpst = fpstatus_ptr(fpsttype);
840	tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
841	vec_full_reg_offset(s, rn),
842	vec_full_reg_offset(s, rm), fpst,
843	is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
844	}
845
846	/* Expand a 4-operand operation using an out-of-line helper. */
847	static void gen_gvec_op4_ool(DisasContext *s, bool is_q, int rd, int rn,
848	int rm, int ra, int data, gen_helper_gvec_4 *fn)
849	{
850	tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd),
851	vec_full_reg_offset(s, rn),
852	vec_full_reg_offset(s, rm),
853	vec_full_reg_offset(s, ra),
854	is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
855	}
856
857	/*
858	* Expand a 4-operand operation using an out-of-line helper that takes
859	* a pointer to the CPU env.
860	*/
861	static void gen_gvec_op4_env(DisasContext *s, bool is_q, int rd, int rn,
862	int rm, int ra, int data,
863	gen_helper_gvec_4_ptr *fn)
864	{
865	tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd),
866	vec_full_reg_offset(s, rn),
867	vec_full_reg_offset(s, rm),
868	vec_full_reg_offset(s, ra),
869	tcg_env,
870	is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
871	}
872
873	/*
874	* Expand a 4-operand + fpstatus pointer + simd data value operation using
875	* an out-of-line helper.
876	*/
877	static void gen_gvec_op4_fpst(DisasContext *s, bool is_q, int rd, int rn,
878	int rm, int ra, ARMFPStatusFlavour fpsttype,
879	int data,
880	gen_helper_gvec_4_ptr *fn)
881	{
882	TCGv_ptr fpst = fpstatus_ptr(fpsttype);
883	tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd),
884	vec_full_reg_offset(s, rn),
885	vec_full_reg_offset(s, rm),
886	vec_full_reg_offset(s, ra), fpst,
887	is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
888	}
889
890	/*
891	* When FPCR.AH == 1, NEG and ABS do not flip the sign bit of a NaN.
892	* These functions implement
893	* d = floatN_is_any_nan(s) ? s : floatN_chs(s)
894	* which for float32 is
895	* d = (s & ~(1 << 31)) > 0x7f800000UL) ? s : (s ^ (1 << 31))
896	* and similarly for the other float sizes.
897	*/
898	static void gen_vfp_ah_negh(TCGv_i32 d, TCGv_i32 s)
899	{
900	TCGv_i32 abs_s = tcg_temp_new_i32(), chs_s = tcg_temp_new_i32();
901
902	gen_vfp_negh(chs_s, s);
903	gen_vfp_absh(abs_s, s);
904	tcg_gen_movcond_i32(TCG_COND_GTU, d,
905	abs_s, tcg_constant_i32(0x7c00),
906	s, chs_s);
907	}
908
909	static void gen_vfp_ah_negs(TCGv_i32 d, TCGv_i32 s)
910	{
911	TCGv_i32 abs_s = tcg_temp_new_i32(), chs_s = tcg_temp_new_i32();
912
913	gen_vfp_negs(chs_s, s);
914	gen_vfp_abss(abs_s, s);
915	tcg_gen_movcond_i32(TCG_COND_GTU, d,
916	abs_s, tcg_constant_i32(0x7f800000UL),
917	s, chs_s);
918	}
919
920	static void gen_vfp_ah_negd(TCGv_i64 d, TCGv_i64 s)
921	{
922	TCGv_i64 abs_s = tcg_temp_new_i64(), chs_s = tcg_temp_new_i64();
923
924	gen_vfp_negd(chs_s, s);
925	gen_vfp_absd(abs_s, s);
926	tcg_gen_movcond_i64(TCG_COND_GTU, d,
927	abs_s, tcg_constant_i64(0x7ff0000000000000ULL),
928	s, chs_s);
929	}
930
931	/*
932	* These functions implement
933	* d = floatN_is_any_nan(s) ? s : floatN_abs(s)
934	* which for float32 is
935	* d = (s & ~(1 << 31)) > 0x7f800000UL) ? s : (s & ~(1 << 31))
936	* and similarly for the other float sizes.
937	*/
938	static void gen_vfp_ah_absh(TCGv_i32 d, TCGv_i32 s)
939	{
940	TCGv_i32 abs_s = tcg_temp_new_i32();
941
942	gen_vfp_absh(abs_s, s);
943	tcg_gen_movcond_i32(TCG_COND_GTU, d,
944	abs_s, tcg_constant_i32(0x7c00),
945	s, abs_s);
946	}
947
948	static void gen_vfp_ah_abss(TCGv_i32 d, TCGv_i32 s)
949	{
950	TCGv_i32 abs_s = tcg_temp_new_i32();
951
952	gen_vfp_abss(abs_s, s);
953	tcg_gen_movcond_i32(TCG_COND_GTU, d,
954	abs_s, tcg_constant_i32(0x7f800000UL),
955	s, abs_s);
956	}
957
958	static void gen_vfp_ah_absd(TCGv_i64 d, TCGv_i64 s)
959	{
960	TCGv_i64 abs_s = tcg_temp_new_i64();
961
962	gen_vfp_absd(abs_s, s);
963	tcg_gen_movcond_i64(TCG_COND_GTU, d,
964	abs_s, tcg_constant_i64(0x7ff0000000000000ULL),
965	s, abs_s);
966	}
967
968	static void gen_vfp_maybe_ah_negh(DisasContext *dc, TCGv_i32 d, TCGv_i32 s)
969	{
970	if (dc->fpcr_ah) {
971	gen_vfp_ah_negh(d, s);
972	} else {
973	gen_vfp_negh(d, s);
974	}
975	}
976
977	static void gen_vfp_maybe_ah_negs(DisasContext *dc, TCGv_i32 d, TCGv_i32 s)
978	{
979	if (dc->fpcr_ah) {
980	gen_vfp_ah_negs(d, s);
981	} else {
982	gen_vfp_negs(d, s);
983	}
984	}
985
986	static void gen_vfp_maybe_ah_negd(DisasContext *dc, TCGv_i64 d, TCGv_i64 s)
987	{
988	if (dc->fpcr_ah) {
989	gen_vfp_ah_negd(d, s);
990	} else {
991	gen_vfp_negd(d, s);
992	}
993	}
994
995	/* Set ZF and NF based on a 64 bit result. This is alas fiddlier
996	* than the 32 bit equivalent.
997	*/
998	static inline void gen_set_NZ64(TCGv_i64 result)
999	{
1000	tcg_gen_extr_i64_i32(cpu_ZF, cpu_NF, result);
1001	tcg_gen_or_i32(cpu_ZF, cpu_ZF, cpu_NF);
1002	}
1003
1004	/* Set NZCV as for a logical operation: NZ as per result, CV cleared. */
1005	static inline void gen_logic_CC(int sf, TCGv_i64 result)
1006	{
1007	if (sf) {
1008	gen_set_NZ64(result);
1009	} else {
1010	tcg_gen_extrl_i64_i32(cpu_ZF, result);
1011	tcg_gen_mov_i32(cpu_NF, cpu_ZF);
1012	}
1013	tcg_gen_movi_i32(cpu_CF, 0);
1014	tcg_gen_movi_i32(cpu_VF, 0);
1015	}
1016
1017	/* dest = T0 + T1; compute C, N, V and Z flags */
1018	static void gen_add64_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
1019	{
1020	TCGv_i64 result, flag, tmp;
1021	result = tcg_temp_new_i64();
1022	flag = tcg_temp_new_i64();
1023	tmp = tcg_temp_new_i64();
1024
1025	tcg_gen_movi_i64(tmp, 0);
1026	tcg_gen_add2_i64(result, flag, t0, tmp, t1, tmp);
1027
1028	tcg_gen_extrl_i64_i32(cpu_CF, flag);
1029
1030	gen_set_NZ64(result);
1031
1032	tcg_gen_xor_i64(flag, result, t0);
1033	tcg_gen_xor_i64(tmp, t0, t1);
1034	tcg_gen_andc_i64(flag, flag, tmp);
1035	tcg_gen_extrh_i64_i32(cpu_VF, flag);
1036
1037	tcg_gen_mov_i64(dest, result);
1038	}
1039
1040	static void gen_add32_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
1041	{
1042	TCGv_i32 t0_32 = tcg_temp_new_i32();
1043	TCGv_i32 t1_32 = tcg_temp_new_i32();
1044	TCGv_i32 tmp = tcg_temp_new_i32();
1045
1046	tcg_gen_movi_i32(tmp, 0);
1047	tcg_gen_extrl_i64_i32(t0_32, t0);
1048	tcg_gen_extrl_i64_i32(t1_32, t1);
1049	tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, tmp, t1_32, tmp);
1050	tcg_gen_mov_i32(cpu_ZF, cpu_NF);
1051	tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);
1052	tcg_gen_xor_i32(tmp, t0_32, t1_32);
1053	tcg_gen_andc_i32(cpu_VF, cpu_VF, tmp);
1054	tcg_gen_extu_i32_i64(dest, cpu_NF);
1055	}
1056
1057	static void gen_add_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
1058	{
1059	if (sf) {
1060	gen_add64_CC(dest, t0, t1);
1061	} else {
1062	gen_add32_CC(dest, t0, t1);
1063	}
1064	}
1065
1066	/* dest = T0 - T1; compute C, N, V and Z flags */
1067	static void gen_sub64_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
1068	{
1069	/* 64 bit arithmetic */
1070	TCGv_i64 result, flag, tmp;
1071
1072	result = tcg_temp_new_i64();
1073	flag = tcg_temp_new_i64();
1074	tcg_gen_sub_i64(result, t0, t1);
1075
1076	gen_set_NZ64(result);
1077
1078	tcg_gen_setcond_i64(TCG_COND_GEU, flag, t0, t1);
1079	tcg_gen_extrl_i64_i32(cpu_CF, flag);
1080
1081	tcg_gen_xor_i64(flag, result, t0);
1082	tmp = tcg_temp_new_i64();
1083	tcg_gen_xor_i64(tmp, t0, t1);
1084	tcg_gen_and_i64(flag, flag, tmp);
1085	tcg_gen_extrh_i64_i32(cpu_VF, flag);
1086	tcg_gen_mov_i64(dest, result);
1087	}
1088
1089	static void gen_sub32_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
1090	{
1091	/* 32 bit arithmetic */
1092	TCGv_i32 t0_32 = tcg_temp_new_i32();
1093	TCGv_i32 t1_32 = tcg_temp_new_i32();
1094	TCGv_i32 tmp;
1095
1096	tcg_gen_extrl_i64_i32(t0_32, t0);
1097	tcg_gen_extrl_i64_i32(t1_32, t1);
1098	tcg_gen_sub_i32(cpu_NF, t0_32, t1_32);
1099	tcg_gen_mov_i32(cpu_ZF, cpu_NF);
1100	tcg_gen_setcond_i32(TCG_COND_GEU, cpu_CF, t0_32, t1_32);
1101	tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);
1102	tmp = tcg_temp_new_i32();
1103	tcg_gen_xor_i32(tmp, t0_32, t1_32);
1104	tcg_gen_and_i32(cpu_VF, cpu_VF, tmp);
1105	tcg_gen_extu_i32_i64(dest, cpu_NF);
1106	}
1107
1108	static void gen_sub_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
1109	{
1110	if (sf) {
1111	gen_sub64_CC(dest, t0, t1);
1112	} else {
1113	gen_sub32_CC(dest, t0, t1);
1114	}
1115	}
1116
1117	/* dest = T0 + T1 + CF; do not compute flags. */
1118	static void gen_adc(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
1119	{
1120	TCGv_i64 flag = tcg_temp_new_i64();
1121	tcg_gen_extu_i32_i64(flag, cpu_CF);
1122	tcg_gen_add_i64(dest, t0, t1);
1123	tcg_gen_add_i64(dest, dest, flag);
1124
1125	if (!sf) {
1126	tcg_gen_ext32u_i64(dest, dest);
1127	}
1128	}
1129
1130	/* dest = T0 + T1 + CF; compute C, N, V and Z flags. */
1131	static void gen_adc_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
1132	{
1133	if (sf) {
1134	TCGv_i64 result = tcg_temp_new_i64();
1135	TCGv_i64 cf_64 = tcg_temp_new_i64();
1136	TCGv_i64 vf_64 = tcg_temp_new_i64();
1137	TCGv_i64 tmp = tcg_temp_new_i64();
1138
1139	tcg_gen_extu_i32_i64(cf_64, cpu_CF);
1140	tcg_gen_addcio_i64(result, cf_64, t0, t1, cf_64);
1141	tcg_gen_extrl_i64_i32(cpu_CF, cf_64);
1142	gen_set_NZ64(result);
1143
1144	tcg_gen_xor_i64(vf_64, result, t0);
1145	tcg_gen_xor_i64(tmp, t0, t1);
1146	tcg_gen_andc_i64(vf_64, vf_64, tmp);
1147	tcg_gen_extrh_i64_i32(cpu_VF, vf_64);
1148
1149	tcg_gen_mov_i64(dest, result);
1150	} else {
1151	TCGv_i32 t0_32 = tcg_temp_new_i32();
1152	TCGv_i32 t1_32 = tcg_temp_new_i32();
1153	TCGv_i32 tmp = tcg_temp_new_i32();
1154
1155	tcg_gen_extrl_i64_i32(t0_32, t0);
1156	tcg_gen_extrl_i64_i32(t1_32, t1);
1157	tcg_gen_addcio_i32(cpu_NF, cpu_CF, t0_32, t1_32, cpu_CF);
1158
1159	tcg_gen_mov_i32(cpu_ZF, cpu_NF);
1160	tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);
1161	tcg_gen_xor_i32(tmp, t0_32, t1_32);
1162	tcg_gen_andc_i32(cpu_VF, cpu_VF, tmp);
1163	tcg_gen_extu_i32_i64(dest, cpu_NF);
1164	}
1165	}
1166
1167	/*
1168	* Load/Store generators
1169	*/
1170
1171	/*
1172	* Store from GPR register to memory.
1173	*/
1174	static void do_gpr_st_memidx(DisasContext *s, TCGv_i64 source,
1175	TCGv_i64 tcg_addr, MemOp memop, int memidx,
1176	bool iss_valid,
1177	unsigned int iss_srt,
1178	bool iss_sf, bool iss_ar)
1179	{
1180	tcg_gen_qemu_st_i64(source, tcg_addr, memidx, memop);
1181
1182	if (iss_valid) {
1183	uint32_t syn;
1184
1185	syn = syn_data_abort_with_iss(0,
1186	(memop & MO_SIZE),
1187	false,
1188	iss_srt,
1189	iss_sf,
1190	iss_ar,
1191	0, 0, 0, 0, 0, false);
1192	disas_set_insn_syndrome(s, syn);
1193	}
1194	}
1195
1196	static void do_gpr_st(DisasContext *s, TCGv_i64 source,
1197	TCGv_i64 tcg_addr, MemOp memop,
1198	bool iss_valid,
1199	unsigned int iss_srt,
1200	bool iss_sf, bool iss_ar)
1201	{
1202	do_gpr_st_memidx(s, source, tcg_addr, memop, get_mem_index(s),
1203	iss_valid, iss_srt, iss_sf, iss_ar);
1204	}
1205
1206	/*
1207	* Load from memory to GPR register
1208	*/
1209	static void do_gpr_ld_memidx(DisasContext *s, TCGv_i64 dest, TCGv_i64 tcg_addr,
1210	MemOp memop, bool extend, int memidx,
1211	bool iss_valid, unsigned int iss_srt,
1212	bool iss_sf, bool iss_ar)
1213	{
1214	tcg_gen_qemu_ld_i64(dest, tcg_addr, memidx, memop);
1215
1216	if (extend && (memop & MO_SIGN)) {
1217	g_assert((memop & MO_SIZE) <= MO_32);
1218	tcg_gen_ext32u_i64(dest, dest);
1219	}
1220
1221	if (iss_valid) {
1222	uint32_t syn;
1223
1224	syn = syn_data_abort_with_iss(0,
1225	(memop & MO_SIZE),
1226	(memop & MO_SIGN) != 0,
1227	iss_srt,
1228	iss_sf,
1229	iss_ar,
1230	0, 0, 0, 0, 0, false);
1231	disas_set_insn_syndrome(s, syn);
1232	}
1233	}
1234
1235	static void do_gpr_ld(DisasContext *s, TCGv_i64 dest, TCGv_i64 tcg_addr,
1236	MemOp memop, bool extend,
1237	bool iss_valid, unsigned int iss_srt,
1238	bool iss_sf, bool iss_ar)
1239	{
1240	do_gpr_ld_memidx(s, dest, tcg_addr, memop, extend, get_mem_index(s),
1241	iss_valid, iss_srt, iss_sf, iss_ar);
1242	}
1243
1244	/*
1245	* Store from FP register to memory
1246	*/
1247	static void do_fp_st(DisasContext *s, int srcidx, TCGv_i64 tcg_addr, MemOp mop)
1248	{
1249	/* This writes the bottom N bits of a 128 bit wide vector to memory */
1250	TCGv_i64 tmplo = tcg_temp_new_i64();
1251
1252	tcg_gen_ld_i64(tmplo, tcg_env, fp_reg_offset(s, srcidx, MO_64));
1253
1254	if ((mop & MO_SIZE) < MO_128) {
1255	tcg_gen_qemu_st_i64(tmplo, tcg_addr, get_mem_index(s), mop);
1256	} else {
1257	TCGv_i64 tmphi = tcg_temp_new_i64();
1258	TCGv_i128 t16 = tcg_temp_new_i128();
1259
1260	tcg_gen_ld_i64(tmphi, tcg_env, fp_reg_hi_offset(s, srcidx));
1261	tcg_gen_concat_i64_i128(t16, tmplo, tmphi);
1262
1263	tcg_gen_qemu_st_i128(t16, tcg_addr, get_mem_index(s), mop);
1264	}
1265	}
1266
1267	/*
1268	* Load from memory to FP register
1269	*/
1270	static void do_fp_ld(DisasContext *s, int destidx, TCGv_i64 tcg_addr, MemOp mop)
1271	{
1272	/* This always zero-extends and writes to a full 128 bit wide vector */
1273	TCGv_i64 tmplo = tcg_temp_new_i64();
1274	TCGv_i64 tmphi = NULL;
1275
1276	if ((mop & MO_SIZE) < MO_128) {
1277	tcg_gen_qemu_ld_i64(tmplo, tcg_addr, get_mem_index(s), mop);
1278	} else {
1279	TCGv_i128 t16 = tcg_temp_new_i128();
1280
1281	tcg_gen_qemu_ld_i128(t16, tcg_addr, get_mem_index(s), mop);
1282
1283	tmphi = tcg_temp_new_i64();
1284	tcg_gen_extr_i128_i64(tmplo, tmphi, t16);
1285	}
1286
1287	tcg_gen_st_i64(tmplo, tcg_env, fp_reg_offset(s, destidx, MO_64));
1288
1289	if (tmphi) {
1290	tcg_gen_st_i64(tmphi, tcg_env, fp_reg_hi_offset(s, destidx));
1291	}
1292	clear_vec_high(s, tmphi != NULL, destidx);
1293	}
1294
1295	/*
1296	* Vector load/store helpers.
1297	*
1298	* The principal difference between this and a FP load is that we don't
1299	* zero extend as we are filling a partial chunk of the vector register.
1300	* These functions don't support 128 bit loads/stores, which would be
1301	* normal load/store operations.
1302	*
1303	* The _i32 versions are useful when operating on 32 bit quantities
1304	* (eg for floating point single or using Neon helper functions).
1305	*/
1306
1307	/* Get value of an element within a vector register */
1308	static void read_vec_element(DisasContext *s, TCGv_i64 tcg_dest, int srcidx,
1309	int element, MemOp memop)
1310	{
1311	int vect_off = vec_reg_offset(s, srcidx, element, memop & MO_SIZE);
1312	switch ((unsigned)memop) {
1313	case MO_8:
1314	tcg_gen_ld8u_i64(tcg_dest, tcg_env, vect_off);
1315	break;
1316	case MO_16:
1317	tcg_gen_ld16u_i64(tcg_dest, tcg_env, vect_off);
1318	break;
1319	case MO_32:
1320	tcg_gen_ld32u_i64(tcg_dest, tcg_env, vect_off);
1321	break;
1322	case MO_8\|MO_SIGN:
1323	tcg_gen_ld8s_i64(tcg_dest, tcg_env, vect_off);
1324	break;
1325	case MO_16\|MO_SIGN:
1326	tcg_gen_ld16s_i64(tcg_dest, tcg_env, vect_off);
1327	break;
1328	case MO_32\|MO_SIGN:
1329	tcg_gen_ld32s_i64(tcg_dest, tcg_env, vect_off);
1330	break;
1331	case MO_64:
1332	case MO_64\|MO_SIGN:
1333	tcg_gen_ld_i64(tcg_dest, tcg_env, vect_off);
1334	break;
1335	default:
1336	g_assert_not_reached();
1337	}
1338	}
1339
1340	static void read_vec_element_i32(DisasContext *s, TCGv_i32 tcg_dest, int srcidx,
1341	int element, MemOp memop)
1342	{
1343	int vect_off = vec_reg_offset(s, srcidx, element, memop & MO_SIZE);
1344	switch (memop) {
1345	case MO_8:
1346	tcg_gen_ld8u_i32(tcg_dest, tcg_env, vect_off);
1347	break;
1348	case MO_16:
1349	tcg_gen_ld16u_i32(tcg_dest, tcg_env, vect_off);
1350	break;
1351	case MO_8\|MO_SIGN:
1352	tcg_gen_ld8s_i32(tcg_dest, tcg_env, vect_off);
1353	break;
1354	case MO_16\|MO_SIGN:
1355	tcg_gen_ld16s_i32(tcg_dest, tcg_env, vect_off);
1356	break;
1357	case MO_32:
1358	case MO_32\|MO_SIGN:
1359	tcg_gen_ld_i32(tcg_dest, tcg_env, vect_off);
1360	break;
1361	default:
1362	g_assert_not_reached();
1363	}
1364	}
1365
1366	/* Set value of an element within a vector register */
1367	static void write_vec_element(DisasContext *s, TCGv_i64 tcg_src, int destidx,
1368	int element, MemOp memop)
1369	{
1370	int vect_off = vec_reg_offset(s, destidx, element, memop & MO_SIZE);
1371	switch (memop) {
1372	case MO_8:
1373	tcg_gen_st8_i64(tcg_src, tcg_env, vect_off);
1374	break;
1375	case MO_16:
1376	tcg_gen_st16_i64(tcg_src, tcg_env, vect_off);
1377	break;
1378	case MO_32:
1379	tcg_gen_st32_i64(tcg_src, tcg_env, vect_off);
1380	break;
1381	case MO_64:
1382	tcg_gen_st_i64(tcg_src, tcg_env, vect_off);
1383	break;
1384	default:
1385	g_assert_not_reached();
1386	}
1387	}
1388
1389	static void write_vec_element_i32(DisasContext *s, TCGv_i32 tcg_src,
1390	int destidx, int element, MemOp memop)
1391	{
1392	int vect_off = vec_reg_offset(s, destidx, element, memop & MO_SIZE);
1393	switch (memop) {
1394	case MO_8:
1395	tcg_gen_st8_i32(tcg_src, tcg_env, vect_off);
1396	break;
1397	case MO_16:
1398	tcg_gen_st16_i32(tcg_src, tcg_env, vect_off);
1399	break;
1400	case MO_32:
1401	tcg_gen_st_i32(tcg_src, tcg_env, vect_off);
1402	break;
1403	default:
1404	g_assert_not_reached();
1405	}
1406	}
1407
1408	/* Store from vector register to memory */
1409	static void do_vec_st(DisasContext *s, int srcidx, int element,
1410	TCGv_i64 tcg_addr, MemOp mop)
1411	{
1412	TCGv_i64 tcg_tmp = tcg_temp_new_i64();
1413
1414	read_vec_element(s, tcg_tmp, srcidx, element, mop & MO_SIZE);
1415	tcg_gen_qemu_st_i64(tcg_tmp, tcg_addr, get_mem_index(s), mop);
1416	}
1417
1418	/* Load from memory to vector register */
1419	static void do_vec_ld(DisasContext *s, int destidx, int element,
1420	TCGv_i64 tcg_addr, MemOp mop)
1421	{
1422	TCGv_i64 tcg_tmp = tcg_temp_new_i64();
1423
1424	tcg_gen_qemu_ld_i64(tcg_tmp, tcg_addr, get_mem_index(s), mop);
1425	write_vec_element(s, tcg_tmp, destidx, element, mop & MO_SIZE);
1426	}
1427
1428	/* Check that FP/Neon access is enabled. If it is, return
1429	* true. If not, emit code to generate an appropriate exception,
1430	* and return false; the caller should not emit any code for
1431	* the instruction. Note that this check must happen after all
1432	* unallocated-encoding checks (otherwise the syndrome information
1433	* for the resulting exception will be incorrect).
1434	*/
1435	static bool fp_access_check_only(DisasContext *s)
1436	{
1437	if (s->fp_excp_el) {
1438	assert(!s->fp_access_checked);
1439	s->fp_access_checked = -1;
1440
1441	gen_exception_insn_el(s, 0, EXCP_UDEF,
1442	syn_a64_fp_access_trap(1, 0xe),
1443	s->fp_excp_el);
1444	return false;
1445	}
1446	s->fp_access_checked = 1;
1447	return true;
1448	}
1449
1450	static bool nonstreaming_check(DisasContext *s)
1451	{
1452	if (s->sme_trap_nonstreaming && s->is_nonstreaming) {
1453	gen_exception_insn(s, 0, EXCP_UDEF,
1454	syn_smetrap(SME_ET_Streaming, false));
1455	return false;
1456	}
1457	return true;
1458	}
1459
1460	static bool fp_access_check(DisasContext *s)
1461	{
1462	return fp_access_check_only(s) && nonstreaming_check(s);
1463	}
1464
1465	/*
1466	* Check that FPMR access is enabled, for an indirect reference by a
1467	* vector instruction. See CheckFPMREnabled().
1468	*/
1469	bool fpmr_access_check(DisasContext *s)
1470	{
1471	if (s->fpmr_el) {
1472	/*
1473	* While denied direct access to the FPMR raises SystemRegisterTrap
1474	* and targets a specific EL, denied indirect access to the FPMR
1475	* results in a simple UNDEFINED to the default exception level.
1476	*/
1477	unallocated_encoding(s);
1478	return false;
1479	}
1480	return true;
1481	}
1482
1483	/*
1484	* Return <0 for non-supported element sizes, with MO_16 controlled by
1485	* FEAT_FP16; return 0 for fp disabled; otherwise return >0 for success.
1486	*/
1487	static int fp_access_check_scalar_hsd(DisasContext *s, MemOp esz)
1488	{
1489	switch (esz) {
1490	case MO_64:
1491	case MO_32:
1492	break;
1493	case MO_16:
1494	if (!dc_isar_feature(aa64_fp16, s)) {
1495	return -1;
1496	}
1497	break;
1498	default:
1499	return -1;
1500	}
1501	return fp_access_check(s);
1502	}
1503
1504	/* Likewise, but vector MO_64 must have two elements. */
1505	static int fp_access_check_vector_hsd(DisasContext *s, bool is_q, MemOp esz)
1506	{
1507	switch (esz) {
1508	case MO_64:
1509	if (!is_q) {
1510	return -1;
1511	}
1512	break;
1513	case MO_32:
1514	break;
1515	case MO_16:
1516	if (!dc_isar_feature(aa64_fp16, s)) {
1517	return -1;
1518	}
1519	break;
1520	default:
1521	return -1;
1522	}
1523	return fp_access_check(s);
1524	}
1525
1526	/*
1527	* Check that SVE access is enabled. If it is, return true.
1528	* If not, emit code to generate an appropriate exception and return false.
1529	* This function corresponds to CheckSVEEnabled().
1530	*/
1531	bool sve_access_check(DisasContext *s)
1532	{
1533	if (dc_isar_feature(aa64_sme, s)) {
1534	bool ret;
1535
1536	if (s->pstate_sm) {
1537	ret = sme_enabled_check(s);
1538	} else if (dc_isar_feature(aa64_sve, s)) {
1539	goto continue_sve;
1540	} else {
1541	ret = sme_sm_enabled_check(s);
1542	}
1543	if (ret) {
1544	ret = nonstreaming_check(s);
1545	}
1546	s->sve_access_checked = (ret ? 1 : -1);
1547	return ret;
1548	}
1549
1550	continue_sve:
1551	if (s->sve_excp_el) {
1552	/* Assert that we only raise one exception per instruction. */
1553	assert(!s->sve_access_checked);
1554	gen_exception_insn_el(s, 0, EXCP_UDEF,
1555	syn_sve_access_trap(), s->sve_excp_el);
1556	s->sve_access_checked = -1;
1557	return false;
1558	}
1559	s->sve_access_checked = 1;
1560	return fp_access_check(s);
1561	}
1562
1563	/*
1564	* Check that SME access is enabled, raise an exception if not.
1565	* Note that this function corresponds to CheckSMEAccess and is
1566	* only used directly for cpregs.
1567	*/
1568	static bool sme_access_check(DisasContext *s)
1569	{
1570	if (s->sme_excp_el) {
1571	gen_exception_insn_el(s, 0, EXCP_UDEF,
1572	syn_smetrap(SME_ET_AccessTrap, false),
1573	s->sme_excp_el);
1574	return false;
1575	}
1576	return true;
1577	}
1578
1579	/* This function corresponds to CheckSMEEnabled. */
1580	bool sme_enabled_check(DisasContext *s)
1581	{
1582	/*
1583	* Note that unlike sve_excp_el, we have not constrained sme_excp_el
1584	* to be zero when fp_excp_el has priority. This is because we need
1585	* sme_excp_el by itself for cpregs access checks.
1586	*/
1587	if (s->sme_excp_el
1588	&& (!s->fp_excp_el \|\| s->sme_excp_el <= s->fp_excp_el)) {
1589	bool ret = sme_access_check(s);
1590	s->fp_access_checked = (ret ? 1 : -1);
1591	return ret;
1592	}
1593	return fp_access_check_only(s);
1594	}
1595
1596	/* Common subroutine for CheckSMEAndEnabled. /
1597	bool sme_enabled_check_with_svcr(DisasContext *s, unsigned req)
1598	{
1599	if (!sme_enabled_check(s)) {
1600	return false;
1601	}
1602	if (FIELD_EX64(req, SVCR, SM) && !s->pstate_sm) {
1603	gen_exception_insn(s, 0, EXCP_UDEF,
1604	syn_smetrap(SME_ET_NotStreaming, false));
1605	return false;
1606	}
1607	if (FIELD_EX64(req, SVCR, ZA) && !s->pstate_za) {
1608	gen_exception_insn(s, 0, EXCP_UDEF,
1609	syn_smetrap(SME_ET_InactiveZA, false));
1610	return false;
1611	}
1612	return true;
1613	}
1614
1615	/*
1616	* Expanders for AdvSIMD translation functions.
1617	*/
1618
1619	static bool do_gvec_op2_ool(DisasContext s, arg_qrr_e a, int data,
1620	gen_helper_gvec_2 *fn)
1621	{
1622	if (!a->q && a->esz == MO_64) {
1623	return false;
1624	}
1625	if (fp_access_check(s)) {
1626	gen_gvec_op2_ool(s, a->q, a->rd, a->rn, data, fn);
1627	}
1628	return true;
1629	}
1630
1631	static bool do_gvec_op3_ool(DisasContext s, arg_qrrr_e a, int data,
1632	gen_helper_gvec_3 *fn)
1633	{
1634	if (!a->q && a->esz == MO_64) {
1635	return false;
1636	}
1637	if (fp_access_check(s)) {
1638	gen_gvec_op3_ool(s, a->q, a->rd, a->rn, a->rm, data, fn);
1639	}
1640	return true;
1641	}
1642
1643	static bool do_gvec_fn3(DisasContext s, arg_qrrr_e a, GVecGen3Fn *fn)
1644	{
1645	if (!a->q && a->esz == MO_64) {
1646	return false;
1647	}
1648	if (fp_access_check(s)) {
1649	gen_gvec_fn3(s, a->q, a->rd, a->rn, a->rm, fn, a->esz);
1650	}
1651	return true;
1652	}
1653
1654	static bool do_gvec_fn3_no64(DisasContext s, arg_qrrr_e a, GVecGen3Fn *fn)
1655	{
1656	if (a->esz == MO_64) {
1657	return false;
1658	}
1659	if (fp_access_check(s)) {
1660	gen_gvec_fn3(s, a->q, a->rd, a->rn, a->rm, fn, a->esz);
1661	}
1662	return true;
1663	}
1664
1665	static bool do_gvec_fn3_no8_no64(DisasContext s, arg_qrrr_e a, GVecGen3Fn *fn)
1666	{
1667	if (a->esz == MO_8) {
1668	return false;
1669	}
1670	return do_gvec_fn3_no64(s, a, fn);
1671	}
1672
1673	static bool do_gvec_fn4(DisasContext s, arg_qrrrr_e a, GVecGen4Fn *fn)
1674	{
1675	if (!a->q && a->esz == MO_64) {
1676	return false;
1677	}
1678	if (fp_access_check(s)) {
1679	gen_gvec_fn4(s, a->q, a->rd, a->rn, a->rm, a->ra, fn, a->esz);
1680	}
1681	return true;
1682	}
1683
1684	/*
1685	* This utility function is for doing register extension with an
1686	* optional shift. You will likely want to pass a temporary for the
1687	* destination register. See DecodeRegExtend() in the ARM ARM.
1688	*/
1689	static void ext_and_shift_reg(TCGv_i64 tcg_out, TCGv_i64 tcg_in,
1690	int option, unsigned int shift)
1691	{
1692	int extsize = extract32(option, 0, 2);
1693	bool is_signed = extract32(option, 2, 1);
1694
1695	tcg_gen_ext_i64(tcg_out, tcg_in, extsize \| (is_signed ? MO_SIGN : 0));
1696	tcg_gen_shli_i64(tcg_out, tcg_out, shift);
1697	}
1698
1699	static inline void gen_check_sp_alignment(DisasContext *s)
1700	{
1701	/* The AArch64 architecture mandates that (if enabled via PSTATE
1702	* or SCTLR bits) there is a check that SP is 16-aligned on every
1703	* SP-relative load or store (with an exception generated if it is not).
1704	* In line with general QEMU practice regarding misaligned accesses,
1705	* we omit these checks for the sake of guest program performance.
1706	* This function is provided as a hook so we can more easily add these
1707	* checks in future (possibly as a "favour catching guest program bugs
1708	* over speed" user selectable option).
1709	*/
1710	}
1711
1712	/*
1713	* The instruction disassembly implemented here matches
1714	* the instruction encoding classifications in chapter C4
1715	* of the ARM Architecture Reference Manual (DDI0487B_a);
1716	* classification names and decode diagrams here should generally
1717	* match up with those in the manual.
1718	*/
1719
1720	static bool trans_B(DisasContext s, arg_i a)
1721	{
1722	reset_btype(s);
1723	gen_goto_tb(s, 0, a->imm);
1724	return true;
1725	}
1726
1727	static bool trans_BL(DisasContext s, arg_i a)
1728	{
1729	TCGv_i64 link = tcg_temp_new_i64();
1730
1731	gen_pc_plus_diff(s, link, 4);
1732	if (s->gcs_en) {
1733	gen_add_gcs_record(s, link);
1734	}
1735	tcg_gen_mov_i64(cpu_reg(s, 30), link);
1736
1737	reset_btype(s);
1738	gen_goto_tb(s, 0, a->imm);
1739	return true;
1740	}
1741
1742
1743	static bool trans_CBZ(DisasContext s, arg_cbz a)
1744	{
1745	DisasLabel match;
1746	TCGv_i64 tcg_cmp;
1747
1748	tcg_cmp = read_cpu_reg(s, a->rt, a->sf);
1749	reset_btype(s);
1750
1751	match = gen_disas_label(s);
1752	tcg_gen_brcondi_i64(a->nz ? TCG_COND_NE : TCG_COND_EQ,
1753	tcg_cmp, 0, match.label);
1754	gen_goto_tb(s, 0, 4);
1755	set_disas_label(s, match);
1756	gen_goto_tb(s, 1, a->imm);
1757	return true;
1758	}
1759
1760	static bool trans_TBZ(DisasContext s, arg_tbz a)
1761	{
1762	DisasLabel match;
1763	TCGv_i64 tcg_cmp;
1764
1765	tcg_cmp = tcg_temp_new_i64();
1766	tcg_gen_andi_i64(tcg_cmp, cpu_reg(s, a->rt), 1ULL << a->bitpos);
1767
1768	reset_btype(s);
1769
1770	match = gen_disas_label(s);
1771	tcg_gen_brcondi_i64(a->nz ? TCG_COND_NE : TCG_COND_EQ,
1772	tcg_cmp, 0, match.label);
1773	gen_goto_tb(s, 0, 4);
1774	set_disas_label(s, match);
1775	gen_goto_tb(s, 1, a->imm);
1776	return true;
1777	}
1778
1779	static bool trans_B_cond(DisasContext s, arg_B_cond a)
1780	{
1781	/* BC.cond is only present with FEAT_HBC */
1782	if (a->c && !dc_isar_feature(aa64_hbc, s)) {
1783	return false;
1784	}
1785	reset_btype(s);
1786	if (a->cond < 0x0e) {
1787	/* genuinely conditional branches */
1788	DisasLabel match = gen_disas_label(s);
1789	arm_gen_test_cc(a->cond, match.label);
1790	gen_goto_tb(s, 0, 4);
1791	set_disas_label(s, match);
1792	gen_goto_tb(s, 1, a->imm);
1793	} else {
1794	/* 0xe and 0xf are both "always" conditions */
1795	gen_goto_tb(s, 0, a->imm);
1796	}
1797	return true;
1798	}
1799
1800	static bool trans_CB_cond(DisasContext s, arg_CB_cond a)
1801	{
1802	static const TCGCond cb_cond[8] = {
1803	[0] = TCG_COND_GT,
1804	[1] = TCG_COND_GE,
1805	[2] = TCG_COND_GTU,
1806	[3] = TCG_COND_GEU,
1807	[4] = TCG_COND_NEVER, /* reserved */
1808	[5] = TCG_COND_NEVER, /* reserved */
1809	[6] = TCG_COND_EQ,
1810	[7] = TCG_COND_NE,
1811	};
1812	TCGCond cond = cb_cond[a->cc];
1813	TCGv_i64 t, m;
1814	DisasLabel match;
1815
1816	if (!dc_isar_feature(aa64_cmpbr, s) \|\| cond == TCG_COND_NEVER) {
1817	return false;
1818	}
1819
1820	t = cpu_reg(s, a->rt);
1821	m = cpu_reg(s, a->rm);
1822	if (a->esz != MO_64) {
1823	MemOp mop = a->esz \| (is_signed_cond(cond) ? MO_SIGN : 0);
1824	TCGv_i64 tt = tcg_temp_new_i64();
1825	TCGv_i64 tm = tcg_temp_new_i64();
1826
1827	tcg_gen_ext_i64(tt, t, mop);
1828	tcg_gen_ext_i64(tm, m, mop);
1829	t = tt;
1830	m = tm;
1831	}
1832
1833	reset_btype(s);
1834	match = gen_disas_label(s);
1835
1836	tcg_gen_brcond_i64(cond, t, m, match.label);
1837	gen_goto_tb(s, 0, 4);
1838	set_disas_label(s, match);
1839	gen_goto_tb(s, 1, a->imm);
1840	return true;
1841	}
1842
1843	static bool trans_CB_cond_imm(DisasContext s, arg_CB_cond_imm a)
1844	{
1845	/* Note that CB imm and CB encode the condition differently */
1846	static const TCGCond cb_cond[8] = {
1847	[0] = TCG_COND_GT,
1848	[1] = TCG_COND_LT,
1849	[2] = TCG_COND_GTU,
1850	[3] = TCG_COND_LTU,
1851	[4] = TCG_COND_NEVER, /* reserved */
1852	[5] = TCG_COND_NEVER, /* reserved */
1853	[6] = TCG_COND_EQ,
1854	[7] = TCG_COND_NE,
1855	};
1856	TCGCond cond = cb_cond[a->cc];
1857	TCGv_i64 t;
1858	DisasLabel match;
1859
1860	if (!dc_isar_feature(aa64_cmpbr, s) \|\| cond == TCG_COND_NEVER) {
1861	return false;
1862	}
1863
1864	t = cpu_reg(s, a->rt);
1865	if (!a->sf) {
1866	TCGv_i64 tt = tcg_temp_new_i64();
1867
1868	if (is_signed_cond(cond)) {
1869	tcg_gen_ext32s_i64(tt, t);
1870	} else {
1871	tcg_gen_ext32u_i64(tt, t);
1872	}
1873	t = tt;
1874	}
1875
1876	reset_btype(s);
1877	match = gen_disas_label(s);
1878
1879	tcg_gen_brcondi_i64(cond, t, a->imm6, match.label);
1880	gen_goto_tb(s, 0, 4);
1881	set_disas_label(s, match);
1882	gen_goto_tb(s, 1, a->imm9);
1883	return true;
1884	}
1885
1886	static void set_btype_for_br(DisasContext *s, int rn)
1887	{
1888	if (dc_isar_feature(aa64_bti, s)) {
1889	/* BR to {x16,x17} or !guard -> 1, else 3. */
1890	if (rn == 16 \|\| rn == 17) {
1891	set_btype(s, 1);
1892	} else {
1893	TCGv_i64 pc = tcg_temp_new_i64();
1894	gen_pc_plus_diff(s, pc, 0);
1895	gen_helper_guarded_page_br(tcg_env, pc);
1896	s->btype = -1;
1897	}
1898	}
1899	}
1900
1901	static void set_btype_for_blr(DisasContext *s)
1902	{
1903	if (dc_isar_feature(aa64_bti, s)) {
1904	/* BLR sets BTYPE to 2, regardless of source guarded page. */
1905	set_btype(s, 2);
1906	}
1907	}
1908
1909	static bool trans_BR(DisasContext s, arg_r a)
1910	{
1911	set_btype_for_br(s, a->rn);
1912	gen_a64_set_pc(s, cpu_reg(s, a->rn));
1913	s->base.is_jmp = DISAS_JUMP;
1914	return true;
1915	}
1916
1917	static bool trans_BLR(DisasContext s, arg_r a)
1918	{
1919	TCGv_i64 link = tcg_temp_new_i64();
1920
1921	gen_pc_plus_diff(s, link, 4);
1922	if (s->gcs_en) {
1923	gen_add_gcs_record(s, link);
1924	}
1925	gen_a64_set_pc(s, cpu_reg(s, a->rn));
1926	tcg_gen_mov_i64(cpu_reg(s, 30), link);
1927
1928	set_btype_for_blr(s);
1929	s->base.is_jmp = DISAS_JUMP;
1930	return true;
1931	}
1932
1933	static bool trans_RET(DisasContext s, arg_r a)
1934	{
1935	TCGv_i64 target = cpu_reg(s, a->rn);
1936
1937	if (s->gcs_en) {
1938	gen_load_check_gcs_record(s, target, GCS_IT_RET_nPauth, a->rn);
1939	} else {
1940	gen_a64_set_pc(s, target);
1941	}
1942	s->base.is_jmp = DISAS_JUMP;
1943	return true;
1944	}
1945
1946	static TCGv_i64 auth_branch_target(DisasContext *s, TCGv_i64 dst,
1947	TCGv_i64 modifier, bool use_key_a)
1948	{
1949	TCGv_i64 truedst;
1950	/*
1951	* Return the branch target for a BRAA/RETA/etc, which is either
1952	* just the destination dst, or that value with the pauth check
1953	* done and the code removed from the high bits.
1954	*/
1955	if (!s->pauth_active) {
1956	return dst;
1957	}
1958
1959	truedst = tcg_temp_new_i64();
1960	if (use_key_a) {
1961	gen_helper_autia_combined(truedst, tcg_env, dst, modifier);
1962	} else {
1963	gen_helper_autib_combined(truedst, tcg_env, dst, modifier);
1964	}
1965	return truedst;
1966	}
1967
1968	static bool trans_BRAZ(DisasContext s, arg_braz a)
1969	{
1970	TCGv_i64 dst;
1971
1972	if (!dc_isar_feature(aa64_pauth, s)) {
1973	return false;
1974	}
1975
1976	dst = auth_branch_target(s, cpu_reg(s, a->rn), tcg_constant_i64(0), !a->m);
1977	set_btype_for_br(s, a->rn);
1978	gen_a64_set_pc(s, dst);
1979	s->base.is_jmp = DISAS_JUMP;
1980	return true;
1981	}
1982
1983	static bool trans_BLRAZ(DisasContext s, arg_braz a)
1984	{
1985	TCGv_i64 dst, link;
1986
1987	if (!dc_isar_feature(aa64_pauth, s)) {
1988	return false;
1989	}
1990	dst = auth_branch_target(s, cpu_reg(s, a->rn), tcg_constant_i64(0), !a->m);
1991
1992	link = tcg_temp_new_i64();
1993	gen_pc_plus_diff(s, link, 4);
1994	if (s->gcs_en) {
1995	gen_add_gcs_record(s, link);
1996	}
1997	gen_a64_set_pc(s, dst);
1998	tcg_gen_mov_i64(cpu_reg(s, 30), link);
1999
2000	set_btype_for_blr(s);
2001	s->base.is_jmp = DISAS_JUMP;
2002	return true;
2003	}
2004
2005	static bool trans_RETA(DisasContext s, arg_reta a)
2006	{
2007	TCGv_i64 dst;
2008
2009	if (!dc_isar_feature(aa64_pauth, s)) {
2010	return false;
2011	}
2012
2013	dst = auth_branch_target(s, cpu_reg(s, 30), cpu_X[31], !a->m);
2014	if (s->gcs_en) {
2015	GCSInstructionType it = a->m ? GCS_IT_RET_PauthB : GCS_IT_RET_PauthA;
2016	gen_load_check_gcs_record(s, dst, it, 30);
2017	} else {
2018	gen_a64_set_pc(s, dst);
2019	}
2020	s->base.is_jmp = DISAS_JUMP;
2021	return true;
2022	}
2023
2024	static bool trans_BRA(DisasContext s, arg_bra a)
2025	{
2026	TCGv_i64 dst;
2027
2028	if (!dc_isar_feature(aa64_pauth, s)) {
2029	return false;
2030	}
2031	dst = auth_branch_target(s, cpu_reg(s,a->rn), cpu_reg_sp(s, a->rm), !a->m);
2032	set_btype_for_br(s, a->rn);
2033	gen_a64_set_pc(s, dst);
2034	s->base.is_jmp = DISAS_JUMP;
2035	return true;
2036	}
2037
2038	static bool trans_BLRA(DisasContext s, arg_bra a)
2039	{
2040	TCGv_i64 dst, link;
2041
2042	if (!dc_isar_feature(aa64_pauth, s)) {
2043	return false;
2044	}
2045	dst = auth_branch_target(s, cpu_reg(s, a->rn), cpu_reg_sp(s, a->rm), !a->m);
2046
2047	link = tcg_temp_new_i64();
2048	gen_pc_plus_diff(s, link, 4);
2049	if (s->gcs_en) {
2050	gen_add_gcs_record(s, link);
2051	}
2052	gen_a64_set_pc(s, dst);
2053	tcg_gen_mov_i64(cpu_reg(s, 30), link);
2054
2055	set_btype_for_blr(s);
2056	s->base.is_jmp = DISAS_JUMP;
2057	return true;
2058	}
2059
2060	static bool trans_ERET(DisasContext s, arg_ERET a)
2061	{
2062	#ifdef CONFIG_USER_ONLY
2063	return false;
2064	#else
2065	TCGv_i64 dst;
2066
2067	if (s->current_el == 0) {
2068	return false;
2069	}
2070	if (s->trap_eret) {
2071	gen_exception_insn_el(s, 0, EXCP_UDEF, syn_erettrap(0), 2);
2072	return true;
2073	}
2074	dst = tcg_temp_new_i64();
2075	tcg_gen_ld_i64(dst, tcg_env,
2076	offsetof(CPUARMState, elr_el[s->current_el]));
2077
2078	translator_io_start(&s->base);
2079
2080	gen_helper_exception_return(tcg_env, dst);
2081	/* Must exit loop to check un-masked IRQs */
2082	s->base.is_jmp = DISAS_EXIT;
2083	return true;
2084	#endif
2085	}
2086
2087	static bool trans_ERETA(DisasContext s, arg_reta a)
2088	{
2089	#ifdef CONFIG_USER_ONLY
2090	return false;
2091	#else
2092	TCGv_i64 dst;
2093
2094	if (!dc_isar_feature(aa64_pauth, s)) {
2095	return false;
2096	}
2097	if (s->current_el == 0) {
2098	return false;
2099	}
2100	/* The FGT trap takes precedence over an auth trap. */
2101	if (s->trap_eret) {
2102	gen_exception_insn_el(s, 0, EXCP_UDEF, syn_erettrap(a->m ? 3 : 2), 2);
2103	return true;
2104	}
2105	dst = tcg_temp_new_i64();
2106	tcg_gen_ld_i64(dst, tcg_env,
2107	offsetof(CPUARMState, elr_el[s->current_el]));
2108
2109	dst = auth_branch_target(s, dst, cpu_X[31], !a->m);
2110
2111	translator_io_start(&s->base);
2112
2113	gen_helper_exception_return(tcg_env, dst);
2114	/* Must exit loop to check un-masked IRQs */
2115	s->base.is_jmp = DISAS_EXIT;
2116	return true;
2117	#endif
2118	}
2119
2120	static bool trans_NOP(DisasContext s, arg_NOP a)
2121	{
2122	return true;
2123	}
2124
2125	static bool trans_YIELD(DisasContext s, arg_YIELD a)
2126	{
2127	/*
2128	* When running in MTTCG we don't generate jumps to the yield and
2129	* WFE helpers as it won't affect the scheduling of other vCPUs.
2130	* If we wanted to more completely model WFE/SEV so we don't busy
2131	* spin unnecessarily we would need to do something more involved.
2132	*/
2133	if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) {
2134	s->base.is_jmp = DISAS_YIELD;
2135	}
2136	return true;
2137	}
2138
2139	static bool trans_WFI(DisasContext s, arg_WFI a)
2140	{
2141	s->base.is_jmp = DISAS_WFI;
2142	return true;
2143	}
2144
2145	static bool trans_SEV(DisasContext s, arg_SEV a)
2146	{
2147	/*
2148	* SEV is a NOP for user-mode emulation.
2149	*/
2150	#ifndef CONFIG_USER_ONLY
2151	gen_helper_sev(tcg_env);
2152	#endif
2153	return true;
2154	}
2155
2156	static bool trans_SEVL(DisasContext s, arg_SEV a)
2157	{
2158	gen_event_reg();
2159	return true;
2160	}
2161
2162	static bool trans_WFE(DisasContext s, arg_WFI a)
2163	{
2164	s->base.is_jmp = DISAS_WFE;
2165	return true;
2166	}
2167
2168	static bool trans_WFIT(DisasContext s, arg_WFIT a)
2169	{
2170	if (!dc_isar_feature(aa64_wfxt, s)) {
2171	return false;
2172	}
2173
2174	/*
2175	* Because we need to pass the register value to the helper,
2176	* it's easier to emit the code now, unlike trans_WFI which
2177	* defers it to aarch64_tr_tb_stop(). That means we need to
2178	* check ss_active so that single-stepping a WFIT doesn't halt.
2179	*/
2180	if (s->ss_active) {
2181	/* Act like a NOP under architectural singlestep */
2182	return true;
2183	}
2184
2185	gen_a64_update_pc(s, 4);
2186	gen_helper_wfit(tcg_env, tcg_constant_i32(a->rd));
2187	/* Go back to the main loop to check for interrupts */
2188	s->base.is_jmp = DISAS_EXIT;
2189	return true;
2190	}
2191
2192	static bool trans_WFET(DisasContext s, arg_WFET a)
2193	{
2194	if (!dc_isar_feature(aa64_wfxt, s)) {
2195	return false;
2196	}
2197
2198	if (s->ss_active) {
2199	/* Act like a NOP under architectural singlestep */
2200	return true;
2201	}
2202
2203	gen_a64_update_pc(s, 4);
2204	gen_helper_wfet(tcg_env, tcg_constant_i32(a->rd));
2205	/* Go back to the main loop to check for interrupts */
2206	s->base.is_jmp = DISAS_EXIT;
2207	return true;
2208	}
2209
2210	static bool trans_XPACLRI(DisasContext s, arg_XPACLRI a)
2211	{
2212	if (s->pauth_active) {
2213	gen_helper_xpaci(cpu_X[30], tcg_env, cpu_X[30]);
2214	}
2215	return true;
2216	}
2217
2218	static bool trans_PACIA1716(DisasContext s, arg_PACIA1716 a)
2219	{
2220	if (s->pauth_active) {
2221	gen_helper_pacia(cpu_X[17], tcg_env, cpu_X[17], cpu_X[16]);
2222	}
2223	return true;
2224	}
2225
2226	static bool trans_PACIB1716(DisasContext s, arg_PACIB1716 a)
2227	{
2228	if (s->pauth_active) {
2229	gen_helper_pacib(cpu_X[17], tcg_env, cpu_X[17], cpu_X[16]);
2230	}
2231	return true;
2232	}
2233
2234	static bool trans_AUTIA1716(DisasContext s, arg_AUTIA1716 a)
2235	{
2236	if (s->pauth_active) {
2237	gen_helper_autia(cpu_X[17], tcg_env, cpu_X[17], cpu_X[16]);
2238	}
2239	return true;
2240	}
2241
2242	static bool trans_AUTIB1716(DisasContext s, arg_AUTIB1716 a)
2243	{
2244	if (s->pauth_active) {
2245	gen_helper_autib(cpu_X[17], tcg_env, cpu_X[17], cpu_X[16]);
2246	}
2247	return true;
2248	}
2249
2250	static bool trans_ESB(DisasContext s, arg_ESB a)
2251	{
2252	/* Without RAS, we must implement this as NOP. */
2253	if (dc_isar_feature(aa64_ras, s)) {
2254	/*
2255	* QEMU does not have a source of physical SErrors,
2256	* so we are only concerned with virtual SErrors.
2257	* The pseudocode in the ARM for this case is
2258	* if PSTATE.EL IN {EL0, EL1} && EL2Enabled() then
2259	* AArch64.vESBOperation();
2260	* Most of the condition can be evaluated at translation time.
2261	* Test for EL2 present, and defer test for SEL2 to runtime.
2262	*/
2263	if (s->current_el <= 1 && arm_dc_feature(s, ARM_FEATURE_EL2)) {
2264	gen_helper_vesb(tcg_env);
2265	}
2266	}
2267	return true;
2268	}
2269
2270	static bool trans_GCSB(DisasContext s, arg_GCSB a)
2271	{
2272	if (dc_isar_feature(aa64_gcs, s)) {
2273	tcg_gen_mb(TCG_BAR_SC \| TCG_MO_ALL);
2274	}
2275	return true;
2276	}
2277
2278	static bool trans_PACIAZ(DisasContext s, arg_PACIAZ a)
2279	{
2280	if (s->pauth_active) {
2281	gen_helper_pacia(cpu_X[30], tcg_env, cpu_X[30], tcg_constant_i64(0));
2282	}
2283	return true;
2284	}
2285
2286	static bool trans_PACIASP(DisasContext s, arg_PACIASP a)
2287	{
2288	if (s->pauth_active) {
2289	gen_helper_pacia(cpu_X[30], tcg_env, cpu_X[30], cpu_X[31]);
2290	}
2291	return true;
2292	}
2293
2294	static bool trans_PACIBZ(DisasContext s, arg_PACIBZ a)
2295	{
2296	if (s->pauth_active) {
2297	gen_helper_pacib(cpu_X[30], tcg_env, cpu_X[30], tcg_constant_i64(0));
2298	}
2299	return true;
2300	}
2301
2302	static bool trans_PACIBSP(DisasContext s, arg_PACIBSP a)
2303	{
2304	if (s->pauth_active) {
2305	gen_helper_pacib(cpu_X[30], tcg_env, cpu_X[30], cpu_X[31]);
2306	}
2307	return true;
2308	}
2309
2310	static bool trans_AUTIAZ(DisasContext s, arg_AUTIAZ a)
2311	{
2312	if (s->pauth_active) {
2313	gen_helper_autia(cpu_X[30], tcg_env, cpu_X[30], tcg_constant_i64(0));
2314	}
2315	return true;
2316	}
2317
2318	static bool trans_AUTIASP(DisasContext s, arg_AUTIASP a)
2319	{
2320	if (s->pauth_active) {
2321	gen_helper_autia(cpu_X[30], tcg_env, cpu_X[30], cpu_X[31]);
2322	}
2323	return true;
2324	}
2325
2326	static bool trans_AUTIBZ(DisasContext s, arg_AUTIBZ a)
2327	{
2328	if (s->pauth_active) {
2329	gen_helper_autib(cpu_X[30], tcg_env, cpu_X[30], tcg_constant_i64(0));
2330	}
2331	return true;
2332	}
2333
2334	static bool trans_AUTIBSP(DisasContext s, arg_AUTIBSP a)
2335	{
2336	if (s->pauth_active) {
2337	gen_helper_autib(cpu_X[30], tcg_env, cpu_X[30], cpu_X[31]);
2338	}
2339	return true;
2340	}
2341
2342	static bool trans_CHKFEAT(DisasContext s, arg_CHKFEAT a)
2343	{
2344	uint64_t feat_en = 0;
2345
2346	if (s->gcs_en) {
2347	feat_en \|= 1 << 0;
2348	}
2349	if (feat_en) {
2350	TCGv_i64 x16 = cpu_reg(s, 16);
2351	tcg_gen_andi_i64(x16, x16, ~feat_en);
2352	}
2353	return true;
2354	}
2355
2356	static bool trans_CLREX(DisasContext s, arg_CLREX a)
2357	{
2358	tcg_gen_movi_i64(cpu_exclusive_addr, -1);
2359	return true;
2360	}
2361
2362	static bool trans_DSB_DMB(DisasContext s, arg_DSB_DMB a)
2363	{
2364	/* We handle DSB and DMB the same way */
2365	TCGBar bar;
2366
2367	switch (a->types) {
2368	case 1: /* MBReqTypes_Reads */
2369	bar = TCG_BAR_SC \| TCG_MO_LD_LD \| TCG_MO_LD_ST;
2370	break;
2371	case 2: /* MBReqTypes_Writes */
2372	bar = TCG_BAR_SC \| TCG_MO_ST_ST;
2373	break;
2374	default: /* MBReqTypes_All */
2375	bar = TCG_BAR_SC \| TCG_MO_ALL;
2376	break;
2377	}
2378	tcg_gen_mb(bar);
2379	return true;
2380	}
2381
2382	static bool trans_DSB_nXS(DisasContext s, arg_DSB_nXS a)
2383	{
2384	if (!dc_isar_feature(aa64_xs, s)) {
2385	return false;
2386	}
2387	tcg_gen_mb(TCG_BAR_SC \| TCG_MO_ALL);
2388	return true;
2389	}
2390
2391	static bool trans_ISB(DisasContext s, arg_ISB a)
2392	{
2393	/*
2394	* We need to break the TB after this insn to execute
2395	* self-modifying code correctly and also to take
2396	* any pending interrupts immediately.
2397	*/
2398	reset_btype(s);
2399	gen_goto_tb(s, 0, 4);
2400	return true;
2401	}
2402
2403	static bool trans_SB(DisasContext s, arg_SB a)
2404	{
2405	if (!dc_isar_feature(aa64_sb, s)) {
2406	return false;
2407	}
2408	/*
2409	* TODO: There is no speculation barrier opcode for TCG;
2410	* MB and end the TB instead.
2411	*/
2412	tcg_gen_mb(TCG_MO_ALL \| TCG_BAR_SC);
2413	gen_goto_tb(s, 0, 4);
2414	return true;
2415	}
2416
2417	static bool trans_CFINV(DisasContext s, arg_CFINV a)
2418	{
2419	if (!dc_isar_feature(aa64_condm_4, s)) {
2420	return false;
2421	}
2422	tcg_gen_xori_i32(cpu_CF, cpu_CF, 1);
2423	return true;
2424	}
2425
2426	static bool trans_XAFLAG(DisasContext s, arg_XAFLAG a)
2427	{
2428	TCGv_i32 z;
2429
2430	if (!dc_isar_feature(aa64_condm_5, s)) {
2431	return false;
2432	}
2433
2434	z = tcg_temp_new_i32();
2435
2436	tcg_gen_setcondi_i32(TCG_COND_EQ, z, cpu_ZF, 0);
2437
2438	/*
2439	* (!C & !Z) << 31
2440	* (!(C \| Z)) << 31
2441	* ~((C \| Z) << 31)
2442	* ~-(C \| Z)
2443	* (C \| Z) - 1
2444	*/
2445	tcg_gen_or_i32(cpu_NF, cpu_CF, z);
2446	tcg_gen_subi_i32(cpu_NF, cpu_NF, 1);
2447
2448	/* !(Z & C) */
2449	tcg_gen_and_i32(cpu_ZF, z, cpu_CF);
2450	tcg_gen_xori_i32(cpu_ZF, cpu_ZF, 1);
2451
2452	/* (!C & Z) << 31 -> -(Z & ~C) */
2453	tcg_gen_andc_i32(cpu_VF, z, cpu_CF);
2454	tcg_gen_neg_i32(cpu_VF, cpu_VF);
2455
2456	/* C \| Z */
2457	tcg_gen_or_i32(cpu_CF, cpu_CF, z);
2458
2459	return true;
2460	}
2461
2462	static bool trans_AXFLAG(DisasContext s, arg_AXFLAG a)
2463	{
2464	if (!dc_isar_feature(aa64_condm_5, s)) {
2465	return false;
2466	}
2467
2468	tcg_gen_sari_i32(cpu_VF, cpu_VF, 31); /* V ? -1 : 0 */
2469	tcg_gen_andc_i32(cpu_CF, cpu_CF, cpu_VF); /* C & !V */
2470
2471	/* !(Z \| V) -> !(!ZF \| V) -> ZF & !V -> ZF & ~VF */
2472	tcg_gen_andc_i32(cpu_ZF, cpu_ZF, cpu_VF);
2473
2474	tcg_gen_movi_i32(cpu_NF, 0);
2475	tcg_gen_movi_i32(cpu_VF, 0);
2476
2477	return true;
2478	}
2479
2480	static bool trans_MSR_i_UAO(DisasContext s, arg_i a)
2481	{
2482	if (!dc_isar_feature(aa64_uao, s) \|\| s->current_el == 0) {
2483	return false;
2484	}
2485	if (a->imm & 1) {
2486	set_pstate_bits(PSTATE_UAO);
2487	} else {
2488	clear_pstate_bits(PSTATE_UAO);
2489	}
2490	gen_rebuild_hflags(s);
2491	s->base.is_jmp = DISAS_TOO_MANY;
2492	return true;
2493	}
2494
2495	static bool trans_MSR_i_PAN(DisasContext s, arg_i a)
2496	{
2497	if (!dc_isar_feature(aa64_pan, s) \|\| s->current_el == 0) {
2498	return false;
2499	}
2500	if (a->imm & 1) {
2501	set_pstate_bits(PSTATE_PAN);
2502	} else {
2503	clear_pstate_bits(PSTATE_PAN);
2504	}
2505	gen_rebuild_hflags(s);
2506	s->base.is_jmp = DISAS_TOO_MANY;
2507	return true;
2508	}
2509
2510	static bool trans_MSR_i_SPSEL(DisasContext s, arg_i a)
2511	{
2512	if (s->current_el == 0) {
2513	return false;
2514	}
2515	gen_helper_msr_i_spsel(tcg_env, tcg_constant_i32(a->imm & PSTATE_SP));
2516	s->base.is_jmp = DISAS_TOO_MANY;
2517	return true;
2518	}
2519
2520	static bool trans_MSR_i_SBSS(DisasContext s, arg_i a)
2521	{
2522	if (!dc_isar_feature(aa64_ssbs, s)) {
2523	return false;
2524	}
2525	if (a->imm & 1) {
2526	set_pstate_bits(PSTATE_SSBS);
2527	} else {
2528	clear_pstate_bits(PSTATE_SSBS);
2529	}
2530	/* Don't need to rebuild hflags since SSBS is a nop */
2531	s->base.is_jmp = DISAS_TOO_MANY;
2532	return true;
2533	}
2534
2535	static bool trans_MSR_i_DIT(DisasContext s, arg_i a)
2536	{
2537	if (!dc_isar_feature(aa64_dit, s)) {
2538	return false;
2539	}
2540	if (a->imm & 1) {
2541	set_pstate_bits(PSTATE_DIT);
2542	} else {
2543	clear_pstate_bits(PSTATE_DIT);
2544	}
2545	/* There's no need to rebuild hflags because DIT is a nop */
2546	s->base.is_jmp = DISAS_TOO_MANY;
2547	return true;
2548	}
2549
2550	static bool trans_MSR_i_TCO(DisasContext s, arg_i a)
2551	{
2552	if (dc_isar_feature(aa64_mte, s)) {
2553	/* Full MTE is enabled -- set the TCO bit as directed. */
2554	if (a->imm & 1) {
2555	set_pstate_bits(PSTATE_TCO);
2556	} else {
2557	clear_pstate_bits(PSTATE_TCO);
2558	}
2559	gen_rebuild_hflags(s);
2560	/* Many factors, including TCO, go into MTE_ACTIVE. */
2561	s->base.is_jmp = DISAS_UPDATE_NOCHAIN;
2562	return true;
2563	} else if (dc_isar_feature(aa64_mte_insn_reg, s)) {
2564	/* Only "instructions accessible at EL0" -- PSTATE.TCO is WI. */
2565	return true;
2566	} else {
2567	/* Insn not present */
2568	return false;
2569	}
2570	}
2571
2572	static bool trans_MSR_i_DAIFSET(DisasContext s, arg_i a)
2573	{
2574	gen_helper_msr_i_daifset(tcg_env, tcg_constant_i32(a->imm));
2575	s->base.is_jmp = DISAS_TOO_MANY;
2576	return true;
2577	}
2578
2579	static bool trans_MSR_i_DAIFCLEAR(DisasContext s, arg_i a)
2580	{
2581	gen_helper_msr_i_daifclear(tcg_env, tcg_constant_i32(a->imm));
2582	/* Exit the cpu loop to re-evaluate pending IRQs. */
2583	s->base.is_jmp = DISAS_UPDATE_EXIT;
2584	return true;
2585	}
2586
2587	static bool trans_MSR_i_ALLINT(DisasContext s, arg_i a)
2588	{
2589	if (!dc_isar_feature(aa64_nmi, s) \|\| s->current_el == 0) {
2590	return false;
2591	}
2592
2593	if (a->imm == 0) {
2594	clear_pstate_bits(PSTATE_ALLINT);
2595	} else if (s->current_el > 1) {
2596	set_pstate_bits(PSTATE_ALLINT);
2597	} else {
2598	gen_helper_msr_set_allint_el1(tcg_env);
2599	}
2600
2601	/* Exit the cpu loop to re-evaluate pending IRQs. */
2602	s->base.is_jmp = DISAS_UPDATE_EXIT;
2603	return true;
2604	}
2605
2606	static bool trans_MSR_i_SVCR(DisasContext s, arg_MSR_i_SVCR a)
2607	{
2608	if (!dc_isar_feature(aa64_sme, s) \|\| a->mask == 0) {
2609	return false;
2610	}
2611	if (sme_access_check(s)) {
2612	int old = s->pstate_sm \| (s->pstate_za << 1);
2613	int new = a->imm * 3;
2614
2615	if ((old ^ new) & a->mask) {
2616	/* At least one bit changes. */
2617	gen_helper_set_svcr(tcg_env, tcg_constant_i32(new),
2618	tcg_constant_i32(a->mask));
2619	s->base.is_jmp = DISAS_TOO_MANY;
2620	}
2621	}
2622	return true;
2623	}
2624
2625	static void gen_get_nzcv(TCGv_i64 tcg_rt)
2626	{
2627	TCGv_i32 tmp = tcg_temp_new_i32();
2628	TCGv_i32 nzcv = tcg_temp_new_i32();
2629
2630	/* build bit 31, N */
2631	tcg_gen_andi_i32(nzcv, cpu_NF, (1U << 31));
2632	/* build bit 30, Z */
2633	tcg_gen_setcondi_i32(TCG_COND_EQ, tmp, cpu_ZF, 0);
2634	tcg_gen_deposit_i32(nzcv, nzcv, tmp, 30, 1);
2635	/* build bit 29, C */
2636	tcg_gen_deposit_i32(nzcv, nzcv, cpu_CF, 29, 1);
2637	/* build bit 28, V */
2638	tcg_gen_shri_i32(tmp, cpu_VF, 31);
2639	tcg_gen_deposit_i32(nzcv, nzcv, tmp, 28, 1);
2640	/* generate result */
2641	tcg_gen_extu_i32_i64(tcg_rt, nzcv);
2642	}
2643
2644	static void gen_set_nzcv(TCGv_i64 tcg_rt)
2645	{
2646	TCGv_i32 nzcv = tcg_temp_new_i32();
2647
2648	/* take NZCV from R[t] */
2649	tcg_gen_extrl_i64_i32(nzcv, tcg_rt);
2650
2651	/* bit 31, N */
2652	tcg_gen_andi_i32(cpu_NF, nzcv, (1U << 31));
2653	/* bit 30, Z */
2654	tcg_gen_andi_i32(cpu_ZF, nzcv, (1 << 30));
2655	tcg_gen_setcondi_i32(TCG_COND_EQ, cpu_ZF, cpu_ZF, 0);
2656	/* bit 29, C */
2657	tcg_gen_andi_i32(cpu_CF, nzcv, (1 << 29));
2658	tcg_gen_shri_i32(cpu_CF, cpu_CF, 29);
2659	/* bit 28, V */
2660	tcg_gen_andi_i32(cpu_VF, nzcv, (1 << 28));
2661	tcg_gen_shli_i32(cpu_VF, cpu_VF, 3);
2662	}
2663
2664	static void gen_sysreg_undef(DisasContext *s, bool isread,
2665	uint8_t op0, uint8_t op1, uint8_t op2,
2666	uint8_t crn, uint8_t crm, uint8_t rt)
2667	{
2668	/*
2669	* Generate code to emit an UNDEF with correct syndrome
2670	* information for a failed system register access.
2671	* This is EC_UNCATEGORIZED (ie a standard UNDEF) in most cases,
2672	* but if FEAT_IDST is implemented then read accesses to registers
2673	* in the feature ID space are reported with the EC_SYSTEMREGISTERTRAP
2674	* syndrome.
2675	*/
2676	uint32_t syndrome;
2677
2678	if (isread && dc_isar_feature(aa64_ids, s) &&
2679	arm_cpreg_encoding_in_idspace(op0, op1, op2, crn, crm)) {
2680	syndrome = syn_aa64_sysregtrap(op0, op1, op2, crn, crm, rt, isread);
2681	} else {
2682	syndrome = syn_uncategorized();
2683	}
2684	gen_exception_insn(s, 0, EXCP_UDEF, syndrome);
2685	}
2686
2687	static void gen_gcspopm(DisasContext *s, int rt)
2688	{
2689	TCGv_i64 gcspr = cpu_gcspr[s->current_el];
2690	int mmuidx = core_gcs_mem_index(s->mmu_idx);
2691	MemOp mop = finalize_memop(s, MO_64 \| MO_ALIGN);
2692	TCGv_i64 value = tcg_temp_new_i64();
2693	TCGLabel *fail_label =
2694	delay_exception(s, EXCP_UDEF, syn_gcs_data_check(GCS_IT_GCSPOPM, rt));
2695
2696	/* The value at top-of-stack must have low 2 bits clear. */
2697	tcg_gen_qemu_ld_i64(value, clean_data_tbi(s, gcspr), mmuidx, mop);
2698	tcg_gen_brcondi_i64(TCG_COND_TSTNE, value, 3, fail_label);
2699
2700	/* Complete the pop and return the value. */
2701	tcg_gen_addi_i64(gcspr, gcspr, 8);
2702	tcg_gen_mov_i64(cpu_reg(s, rt), value);
2703	}
2704
2705	static void gen_gcspushx(DisasContext *s)
2706	{
2707	TCGv_i64 gcspr = cpu_gcspr[s->current_el];
2708	int spsr_idx = aarch64_banked_spsr_index(s->current_el);
2709	int spsr_off = offsetof(CPUARMState, banked_spsr[spsr_idx]);
2710	int elr_off = offsetof(CPUARMState, elr_el[s->current_el]);
2711	int mmuidx = core_gcs_mem_index(s->mmu_idx);
2712	MemOp mop = finalize_memop(s, MO_64 \| MO_ALIGN);
2713	TCGv_i64 addr = tcg_temp_new_i64();
2714	TCGv_i64 tmp = tcg_temp_new_i64();
2715
2716	tcg_gen_addi_i64(addr, gcspr, -8);
2717	tcg_gen_qemu_st_i64(cpu_reg(s, 30), addr, mmuidx, mop);
2718
2719	tcg_gen_ld_i64(tmp, tcg_env, spsr_off);
2720	tcg_gen_addi_i64(addr, addr, -8);
2721	tcg_gen_qemu_st_i64(tmp, addr, mmuidx, mop);
2722
2723	tcg_gen_ld_i64(tmp, tcg_env, elr_off);
2724	tcg_gen_addi_i64(addr, addr, -8);
2725	tcg_gen_qemu_st_i64(tmp, addr, mmuidx, mop);
2726
2727	tcg_gen_addi_i64(addr, addr, -8);
2728	tcg_gen_qemu_st_i64(tcg_constant_i64(0b1001), addr, mmuidx, mop);
2729
2730	tcg_gen_mov_i64(gcspr, addr);
2731	clear_pstate_bits(PSTATE_EXLOCK);
2732	}
2733
2734	static void gen_gcspopcx(DisasContext *s)
2735	{
2736	TCGv_i64 gcspr = cpu_gcspr[s->current_el];
2737	int spsr_idx = aarch64_banked_spsr_index(s->current_el);
2738	int spsr_off = offsetof(CPUARMState, banked_spsr[spsr_idx]);
2739	int elr_off = offsetof(CPUARMState, elr_el[s->current_el]);
2740	int gcscr_off = offsetof(CPUARMState, cp15.gcscr_el[s->current_el]);
2741	int pstate_off = offsetof(CPUARMState, pstate);
2742	int mmuidx = core_gcs_mem_index(s->mmu_idx);
2743	MemOp mop = finalize_memop(s, MO_64 \| MO_ALIGN);
2744	TCGv_i64 addr = tcg_temp_new_i64();
2745	TCGv_i64 tmp1 = tcg_temp_new_i64();
2746	TCGv_i64 tmp2 = tcg_temp_new_i64();
2747	TCGLabel *fail_label =
2748	delay_exception(s, EXCP_UDEF, syn_gcs_data_check(GCS_IT_GCSPOPCX, 31));
2749
2750	/* The value at top-of-stack must be an exception token. */
2751	tcg_gen_qemu_ld_i64(tmp1, gcspr, mmuidx, mop);
2752	tcg_gen_brcondi_i64(TCG_COND_NE, tmp1, 0b1001, fail_label);
2753
2754	/* Validate in turn, ELR ... */
2755	tcg_gen_addi_i64(addr, gcspr, 8);
2756	tcg_gen_qemu_ld_i64(tmp1, addr, mmuidx, mop);
2757	tcg_gen_ld_i64(tmp2, tcg_env, elr_off);
2758	tcg_gen_brcond_i64(TCG_COND_NE, tmp1, tmp2, fail_label);
2759
2760	/* ... SPSR ... */
2761	tcg_gen_addi_i64(addr, addr, 8);
2762	tcg_gen_qemu_ld_i64(tmp1, addr, mmuidx, mop);
2763	tcg_gen_ld_i64(tmp2, tcg_env, spsr_off);
2764	tcg_gen_brcond_i64(TCG_COND_NE, tmp1, tmp2, fail_label);
2765
2766	/* ... and LR. */
2767	tcg_gen_addi_i64(addr, addr, 8);
2768	tcg_gen_qemu_ld_i64(tmp1, addr, mmuidx, mop);
2769	tcg_gen_brcond_i64(TCG_COND_NE, tmp1, cpu_reg(s, 30), fail_label);
2770
2771	/* Writeback stack pointer after pop. */
2772	tcg_gen_addi_i64(gcspr, addr, 8);
2773
2774	/* PSTATE.EXLOCK = GetCurrentEXLOCKEN(). */
2775	tcg_gen_ld_i64(tmp1, tcg_env, gcscr_off);
2776	tcg_gen_ld_i64(tmp2, tcg_env, pstate_off);
2777	tcg_gen_shri_i64(tmp1, tmp1, ctz64(GCSCR_EXLOCKEN));
2778	tcg_gen_deposit_i64(tmp2, tmp2, tmp1, ctz64(PSTATE_EXLOCK), 1);
2779	tcg_gen_st_i64(tmp2, tcg_env, pstate_off);
2780	}
2781
2782	static void gen_gcspopx(DisasContext *s)
2783	{
2784	TCGv_i64 gcspr = cpu_gcspr[s->current_el];
2785	int mmuidx = core_gcs_mem_index(s->mmu_idx);
2786	MemOp mop = finalize_memop(s, MO_64 \| MO_ALIGN);
2787	TCGv_i64 addr = tcg_temp_new_i64();
2788	TCGv_i64 tmp = tcg_temp_new_i64();
2789	TCGLabel *fail_label =
2790	delay_exception(s, EXCP_UDEF, syn_gcs_data_check(GCS_IT_GCSPOPX, 31));
2791
2792	/* The value at top-of-stack must be an exception token. */
2793	tcg_gen_qemu_ld_i64(tmp, gcspr, mmuidx, mop);
2794	tcg_gen_brcondi_i64(TCG_COND_NE, tmp, 0b1001, fail_label);
2795
2796	/*
2797	* The other three values in the exception return record
2798	* are ignored, but are loaded anyway to raise faults.
2799	*/
2800	tcg_gen_addi_i64(addr, gcspr, 8);
2801	tcg_gen_qemu_ld_i64(tmp, addr, mmuidx, mop);
2802	tcg_gen_addi_i64(addr, addr, 8);
2803	tcg_gen_qemu_ld_i64(tmp, addr, mmuidx, mop);
2804	tcg_gen_addi_i64(addr, addr, 8);
2805	tcg_gen_qemu_ld_i64(tmp, addr, mmuidx, mop);
2806	tcg_gen_addi_i64(gcspr, addr, 8);
2807	}
2808
2809	static void gen_gcsss1(DisasContext *s, int rt)
2810	{
2811	TCGv_i64 gcspr = cpu_gcspr[s->current_el];
2812	int mmuidx = core_gcs_mem_index(s->mmu_idx);
2813	MemOp mop = finalize_memop(s, MO_64 \| MO_ALIGN);
2814	TCGv_i64 inptr = cpu_reg(s, rt);
2815	TCGv_i64 cmp = tcg_temp_new_i64();
2816	TCGv_i64 new = tcg_temp_new_i64();
2817	TCGv_i64 old = tcg_temp_new_i64();
2818	TCGLabel *fail_label =
2819	delay_exception(s, EXCP_UDEF, syn_gcs_data_check(GCS_IT_GCSSS1, rt));
2820
2821	/* Compute the valid cap entry that the new stack must have. */
2822	tcg_gen_deposit_i64(cmp, inptr, tcg_constant_i64(1), 0, 12);
2823	/* Compute the in-progress cap entry for the old stack. */
2824	tcg_gen_deposit_i64(new, gcspr, tcg_constant_i64(5), 0, 3);
2825
2826	/* Swap the valid cap the with the in-progress cap. */
2827	tcg_gen_atomic_cmpxchg_i64(old, inptr, cmp, new, mmuidx, mop);
2828	tcg_gen_brcond_i64(TCG_COND_NE, old, cmp, fail_label);
2829
2830	/* The new stack had a valid cap: change gcspr. */
2831	tcg_gen_andi_i64(gcspr, inptr, ~7);
2832	}
2833
2834	static void gen_gcsss2(DisasContext *s, int rt)
2835	{
2836	TCGv_i64 gcspr = cpu_gcspr[s->current_el];
2837	int mmuidx = core_gcs_mem_index(s->mmu_idx);
2838	MemOp mop = finalize_memop(s, MO_64 \| MO_ALIGN);
2839	TCGv_i64 outptr = tcg_temp_new_i64();
2840	TCGv_i64 tmp = tcg_temp_new_i64();
2841	TCGLabel *fail_label =
2842	delay_exception(s, EXCP_UDEF, syn_gcs_data_check(GCS_IT_GCSSS2, rt));
2843
2844	/* Validate that the new stack has an in-progress cap. */
2845	tcg_gen_qemu_ld_i64(outptr, gcspr, mmuidx, mop);
2846	tcg_gen_andi_i64(tmp, outptr, 7);
2847	tcg_gen_brcondi_i64(TCG_COND_NE, tmp, 5, fail_label);
2848
2849	/* Push a valid cap to the old stack. */
2850	tcg_gen_andi_i64(outptr, outptr, ~7);
2851	tcg_gen_addi_i64(outptr, outptr, -8);
2852	tcg_gen_deposit_i64(tmp, outptr, tcg_constant_i64(1), 0, 12);
2853	tcg_gen_qemu_st_i64(tmp, outptr, mmuidx, mop);
2854	tcg_gen_mb(TCG_BAR_SC \| TCG_MO_ALL);
2855
2856	/* Pop the in-progress cap from the new stack. */
2857	tcg_gen_addi_i64(gcspr, gcspr, 8);
2858
2859	/* Return a pointer to the old stack cap. */
2860	tcg_gen_mov_i64(cpu_reg(s, rt), outptr);
2861	}
2862
2863	/*
2864	* Look up @key, returning the cpreg, which must exist.
2865	* Additionally, the new cpreg must also be accessible.
2866	*/
2867	static const ARMCPRegInfo *
2868	redirect_cpreg(DisasContext *s, uint32_t key, bool isread)
2869	{
2870	const ARMCPRegInfo *ri = get_arm_cp_reginfo(s->cp_regs, key);
2871	assert(ri);
2872	assert(cp_access_ok(s->current_el, ri, isread));
2873	return ri;
2874	}
2875
2876	/* MRS - move from system register
2877	* MSR (register) - move to system register
2878	* SYS
2879	* SYSL
2880	* These are all essentially the same insn in 'read' and 'write'
2881	* versions, with varying op0 fields.
2882	*/
2883	static void handle_sys(DisasContext *s, bool isread,
2884	unsigned int op0, unsigned int op1, unsigned int op2,
2885	unsigned int crn, unsigned int crm, unsigned int rt)
2886	{
2887	uint32_t key = ENCODE_AA64_CP_REG(op0, op1, crn, crm, op2);
2888	const ARMCPRegInfo *ri = get_arm_cp_reginfo(s->cp_regs, key);
2889	bool need_helper = false;
2890	bool need_exit_tb = false;
2891	bool nv_trap_to_el2 = false;
2892	bool nv_redirect_reg = false;
2893	bool skip_fp_access_checks = false;
2894	bool nv2_mem_redirect = false;
2895	TCGv_ptr tcg_ri = NULL;
2896	TCGv_i64 tcg_rt;
2897	uint32_t syndrome = syn_aa64_sysregtrap(op0, op1, op2, crn, crm, rt, isread);
2898
2899	if (crn == 11 \|\| crn == 15) {
2900	/*
2901	* Check for TIDCP trap, which must take precedence over
2902	* the UNDEF for "no such register" etc.
2903	*/
2904	switch (s->current_el) {
2905	case 0:
2906	if (dc_isar_feature(aa64_tidcp1, s)) {
2907	gen_helper_tidcp_el0(tcg_env, tcg_constant_i32(syndrome));
2908	}
2909	break;
2910	case 1:
2911	gen_helper_tidcp_el1(tcg_env, tcg_constant_i32(syndrome));
2912	break;
2913	}
2914	}
2915
2916	if (!ri) {
2917	/* Unknown register; this might be a guest error or a QEMU
2918	* unimplemented feature.
2919	*/
2920	qemu_log_mask(LOG_UNIMP, "%s access to unsupported AArch64 "
2921	"system register op0:%d op1:%d crn:%d crm:%d op2:%d\n",
2922	isread ? "read" : "write", op0, op1, crn, crm, op2);
2923	gen_sysreg_undef(s, isread, op0, op1, op2, crn, crm, rt);
2924	return;
2925	}
2926
2927	if (s->nv2 && ri->nv2_redirect_offset) {
2928	/*
2929	* Some registers always redirect to memory; some only do so if
2930	* HCR_EL2.NV1 is 0, and some only if NV1 is 1 (these come in
2931	* pairs which share an offset; see the table in R_CSRPQ).
2932	*/
2933	if (ri->nv2_redirect_offset & NV2_REDIR_NV1) {
2934	nv2_mem_redirect = s->nv1;
2935	} else if (ri->nv2_redirect_offset & NV2_REDIR_NO_NV1) {
2936	nv2_mem_redirect = !s->nv1;
2937	} else {
2938	nv2_mem_redirect = true;
2939	}
2940	}
2941
2942	/* Check access permissions */
2943	if (!cp_access_ok(s->current_el, ri, isread)) {
2944	/*
2945	* FEAT_NV/NV2 handling does not do the usual FP access checks
2946	* for registers only accessible at EL2 (though it does do them
2947	* for registers accessible at EL1).
2948	*/
2949	skip_fp_access_checks = true;
2950	if (s->nv2 && (ri->type & ARM_CP_NV2_REDIRECT)) {
2951	/*
2952	* This is one of the few EL2 registers which should redirect
2953	* to the equivalent EL1 register. We do that after running
2954	* the EL2 register's accessfn.
2955	*/
2956	nv_redirect_reg = true;
2957	assert(!nv2_mem_redirect);
2958	} else if (nv2_mem_redirect) {
2959	/*
2960	* NV2 redirect-to-memory takes precedence over trap to EL2 or
2961	* UNDEF to EL1.
2962	*/
2963	} else if (s->nv && arm_cpreg_traps_in_nv(ri)) {
2964	/*
2965	* This register / instruction exists and is an EL2 register, so
2966	* we must trap to EL2 if accessed in nested virtualization EL1
2967	* instead of UNDEFing. We'll do that after the usual access checks.
2968	* (This makes a difference only for a couple of registers like
2969	* VSTTBR_EL2 where the "UNDEF if NonSecure" should take priority
2970	* over the trap-to-EL2. Most trapped-by-FEAT_NV registers have
2971	* an accessfn which does nothing when called from EL1, because
2972	* the trap-to-EL3 controls which would apply to that register
2973	* at EL2 don't take priority over the FEAT_NV trap-to-EL2.)
2974	*/
2975	nv_trap_to_el2 = true;
2976	} else {
2977	gen_sysreg_undef(s, isread, op0, op1, op2, crn, crm, rt);
2978	return;
2979	}
2980	}
2981
2982	if (ri->vhe_redir_to_el2 && s->current_el == 2 && s->e2h) {
2983	/*
2984	* This one of the FOO_EL1 registers which redirect to FOO_EL2
2985	* from EL2 when HCR_EL2.E2H is set.
2986	*/
2987	key = ri->vhe_redir_to_el2;
2988	ri = redirect_cpreg(s, key, isread);
2989	} else if (ri->vhe_redir_to_el01 && s->current_el >= 2) {
2990	/*
2991	* This is one of the FOO_EL12 or FOO_EL02 registers.
2992	* With !E2H, they all UNDEF.
2993	* With E2H, from EL2 or EL3, they redirect to FOO_EL1/FOO_EL0.
2994	*/
2995	if (!s->e2h) {
2996	gen_sysreg_undef(s, isread, op0, op1, op2, crn, crm, rt);
2997	return;
2998	}
2999	key = ri->vhe_redir_to_el01;
3000	ri = redirect_cpreg(s, key, isread);
3001	}
3002
3003	if (ri->accessfn) {
3004	need_helper = true;
3005	} else if (ri->fgt) {
3006	/*
3007	* EL3-only access means this must be an FGWTE3 trap (which are
3008	* always active); otherwise it's an FGT trap to EL2.
3009	*/
3010	if ((ri->access & ~PL3_RW) == 0) {
3011	need_helper = dc_isar_feature(aa64_fgwte3, s);
3012	} else {
3013	need_helper = s->fgt_active;
3014	}
3015	}
3016	if (need_helper) {
3017	/* Emit code to perform further access permissions checks at
3018	* runtime; this may result in an exception.
3019	*/
3020	gen_a64_update_pc(s, 0);
3021	tcg_ri = tcg_temp_new_ptr();
3022	gen_helper_access_check_cp_reg(tcg_ri, tcg_env,
3023	tcg_constant_i32(key),
3024	tcg_constant_i32(syndrome),
3025	tcg_constant_i32(isread));
3026	} else if (ri->type & ARM_CP_RAISES_EXC) {
3027	/*
3028	* The readfn or writefn might raise an exception;
3029	* synchronize the CPU state in case it does.
3030	*/
3031	gen_a64_update_pc(s, 0);
3032	}
3033
3034	if (!skip_fp_access_checks) {
3035	if ((ri->type & ARM_CP_FPMR) && s->fpmr_el != 0) {
3036	gen_exception_insn_el(s, 0, EXCP_UDEF, syndrome, s->fpmr_el);
3037	return;
3038	}
3039	if ((ri->type & ARM_CP_FPU) && !fp_access_check_only(s)) {
3040	return;
3041	} else if ((ri->type & ARM_CP_SVE) && !sve_access_check(s)) {
3042	return;
3043	} else if ((ri->type & ARM_CP_SME) && !sme_access_check(s)) {
3044	return;
3045	}
3046	}
3047
3048	if (nv_trap_to_el2) {
3049	gen_exception_insn_el(s, 0, EXCP_UDEF, syndrome, 2);
3050	return;
3051	}
3052
3053	if (nv_redirect_reg) {
3054	/*
3055	* FEAT_NV2 redirection of an EL2 register to an EL1 register.
3056	* Conveniently in all cases the encoding of the EL1 register is
3057	* identical to the EL2 register except that opc1 is 0.
3058	* Get the reginfo for the EL1 register to use for the actual access.
3059	* We don't use the EL1 register's access function, and
3060	* fine-grained-traps on EL1 also do not apply here.
3061	*/
3062	key = ENCODE_AA64_CP_REG(op0, 0, crn, crm, op2);
3063	ri = redirect_cpreg(s, key, isread);
3064	/*
3065	* We might not have done an update_pc earlier, so check we don't
3066	* need it. We could support this in future if necessary.
3067	*/
3068	assert(!(ri->type & ARM_CP_RAISES_EXC));
3069	}
3070
3071	if (nv2_mem_redirect) {
3072	/*
3073	* This system register is being redirected into an EL2 memory access.
3074	* This means it is not an IO operation, doesn't change hflags,
3075	* and need not end the TB, because it has no side effects.
3076	*
3077	* The access is 64-bit single copy atomic, guaranteed aligned because
3078	* of the definition of VCNR_EL2. Its endianness depends on
3079	* SCTLR_EL2.EE, not on the data endianness of EL1.
3080	* It is done under either the EL2 translation regime or the EL2&0
3081	* translation regime, depending on HCR_EL2.E2H. It behaves as if
3082	* PSTATE.PAN is 0.
3083	*/
3084	TCGv_i64 ptr = tcg_temp_new_i64();
3085	MemOp mop = MO_64 \| MO_ALIGN \| MO_ATOM_IFALIGN;
3086	ARMMMUIdx armmemidx = s->nv2_mem_e20 ? ARMMMUIdx_E20_2 : ARMMMUIdx_E2;
3087	int memidx = arm_to_core_mmu_idx(armmemidx);
3088	uint32_t syn;
3089
3090	mop \|= (s->nv2_mem_be ? MO_BE : MO_LE);
3091
3092	tcg_gen_ld_i64(ptr, tcg_env, offsetof(CPUARMState, cp15.vncr_el2));
3093	tcg_gen_addi_i64(ptr, ptr,
3094	(ri->nv2_redirect_offset & ~NV2_REDIR_FLAG_MASK));
3095	tcg_rt = cpu_reg(s, rt);
3096
3097	syn = syn_data_abort_vncr(0, !isread, 0);
3098	disas_set_insn_syndrome(s, syn);
3099	if (isread) {
3100	tcg_gen_qemu_ld_i64(tcg_rt, ptr, memidx, mop);
3101	} else {
3102	tcg_gen_qemu_st_i64(tcg_rt, ptr, memidx, mop);
3103	}
3104	return;
3105	}
3106
3107	/* Handle special cases first */
3108	switch (ri->type & ARM_CP_SPECIAL_MASK) {
3109	case 0:
3110	break;
3111	case ARM_CP_NOP:
3112	return;
3113	case ARM_CP_NZCV:
3114	tcg_rt = cpu_reg(s, rt);
3115	if (isread) {
3116	gen_get_nzcv(tcg_rt);
3117	} else {
3118	gen_set_nzcv(tcg_rt);
3119	}
3120	return;
3121	case ARM_CP_CURRENTEL:
3122	{
3123	/*
3124	* Reads as current EL value from pstate, which is
3125	* guaranteed to be constant by the tb flags.
3126	* For nested virt we should report EL2.
3127	*/
3128	int el = s->nv ? 2 : s->current_el;
3129	tcg_rt = cpu_reg(s, rt);
3130	tcg_gen_movi_i64(tcg_rt, el << 2);
3131	return;
3132	}
3133	case ARM_CP_DC_ZVA:
3134	/* Writes clear the aligned block of memory which rt points into. */
3135	if (s->mte_active[0]) {
3136	int desc = 0;
3137
3138	desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s));
3139	desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
3140	desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
3141	desc = FIELD_DP32(desc, MTEDESC, MTX, s->mtx);
3142
3143	tcg_rt = tcg_temp_new_i64();
3144	gen_helper_mte_check_zva(tcg_rt, tcg_env,
3145	tcg_constant_i32(desc), cpu_reg(s, rt));
3146	} else {
3147	tcg_rt = clean_data_tbi(s, cpu_reg(s, rt));
3148	}
3149	gen_helper_dc_zva(tcg_env, tcg_rt);
3150	return;
3151	case ARM_CP_DC_GVA:
3152	{
3153	TCGv_i64 clean_addr, tag;
3154
3155	/*
3156	* DC_GVA, like DC_ZVA, requires that we supply the original
3157	* pointer for an invalid page. Probe that address first.
3158	*/
3159	tcg_rt = cpu_reg(s, rt);
3160	clean_addr = clean_data_tbi(s, tcg_rt);
3161	gen_probe_access(s, clean_addr, MMU_DATA_STORE, MO_8);
3162
3163	if (s->ata[0]) {
3164	/* Extract the tag from the register to match STZGM. */
3165	tag = tcg_temp_new_i64();
3166	tcg_gen_shri_i64(tag, tcg_rt, 56);
3167	gen_helper_stzgm_tags(tcg_env, clean_addr, tag,
3168	tcg_constant_i32(s->mtx));
3169	}
3170	}
3171	return;
3172	case ARM_CP_DC_GZVA:
3173	{
3174	TCGv_i64 clean_addr, tag;
3175
3176	/* For DC_GZVA, we can rely on DC_ZVA for the proper fault. */
3177	tcg_rt = cpu_reg(s, rt);
3178	clean_addr = clean_data_tbi(s, tcg_rt);
3179	gen_helper_dc_zva(tcg_env, clean_addr);
3180
3181	if (s->ata[0]) {
3182	/* Extract the tag from the register to match STZGM. */
3183	tag = tcg_temp_new_i64();
3184	tcg_gen_shri_i64(tag, tcg_rt, 56);
3185	gen_helper_stzgm_tags(tcg_env, clean_addr, tag,
3186	tcg_constant_i32(s->mtx));
3187	}
3188	}
3189	return;
3190	case ARM_CP_GCSPUSHM:
3191	if (s->gcs_en) {
3192	gen_add_gcs_record(s, cpu_reg(s, rt));
3193	}
3194	return;
3195	case ARM_CP_GCSPOPM:
3196	/* Note that X[rt] is unchanged if !GCSEnabled. */
3197	if (s->gcs_en) {
3198	gen_gcspopm(s, rt);
3199	}
3200	return;
3201	case ARM_CP_GCSPUSHX:
3202	/* Choose the CONSTRAINED UNPREDICTABLE for UNDEF. */
3203	if (rt != 31) {
3204	unallocated_encoding(s);
3205	} else if (s->gcs_en) {
3206	gen_gcspushx(s);
3207	}
3208	return;
3209	case ARM_CP_GCSPOPCX:
3210	/* Choose the CONSTRAINED UNPREDICTABLE for UNDEF. */
3211	if (rt != 31) {
3212	unallocated_encoding(s);
3213	} else if (s->gcs_en) {
3214	gen_gcspopcx(s);
3215	}
3216	return;
3217	case ARM_CP_GCSPOPX:
3218	/* Choose the CONSTRAINED UNPREDICTABLE for UNDEF. */
3219	if (rt != 31) {
3220	unallocated_encoding(s);
3221	} else if (s->gcs_en) {
3222	gen_gcspopx(s);
3223	}
3224	return;
3225	case ARM_CP_GCSSS1:
3226	if (s->gcs_en) {
3227	gen_gcsss1(s, rt);
3228	}
3229	return;
3230	case ARM_CP_GCSSS2:
3231	if (s->gcs_en) {
3232	gen_gcsss2(s, rt);
3233	}
3234	return;
3235	default:
3236	g_assert_not_reached();
3237	}
3238
3239	if (ri->type & ARM_CP_IO) {
3240	/* I/O operations must end the TB here (whether read or write) */
3241	need_exit_tb = translator_io_start(&s->base);
3242	}
3243
3244	tcg_rt = cpu_reg(s, rt);
3245
3246	if (isread) {
3247	if (ri->type & ARM_CP_CONST) {
3248	tcg_gen_movi_i64(tcg_rt, ri->resetvalue);
3249	} else if (ri->readfn) {
3250	if (!tcg_ri) {
3251	tcg_ri = gen_lookup_cp_reg(key);
3252	}
3253	gen_helper_get_cp_reg64(tcg_rt, tcg_env, tcg_ri);
3254	} else {
3255	tcg_gen_ld_i64(tcg_rt, tcg_env, ri->fieldoffset);
3256	}
3257	} else {
3258	if (ri->type & ARM_CP_CONST) {
3259	/* If not forbidden by access permissions, treat as WI */
3260	return;
3261	} else if (ri->writefn) {
3262	if (!tcg_ri) {
3263	tcg_ri = gen_lookup_cp_reg(key);
3264	}
3265	gen_helper_set_cp_reg64(tcg_env, tcg_ri, tcg_rt);
3266	} else {
3267	tcg_gen_st_i64(tcg_rt, tcg_env, ri->fieldoffset);
3268	}
3269	}
3270
3271	if (!isread && !(ri->type & ARM_CP_SUPPRESS_TB_END)) {
3272	/*
3273	* A write to any coprocessor register that ends a TB
3274	* must rebuild the hflags for the next TB.
3275	*/
3276	gen_rebuild_hflags(s);
3277	/*
3278	* We default to ending the TB on a coprocessor register write,
3279	* but allow this to be suppressed by the register definition
3280	* (usually only necessary to work around guest bugs).
3281	*/
3282	need_exit_tb = true;
3283	}
3284	if (need_exit_tb) {
3285	s->base.is_jmp = DISAS_UPDATE_EXIT;
3286	}
3287	}
3288
3289	static bool trans_SYS(DisasContext s, arg_SYS a)
3290	{
3291	handle_sys(s, a->l, a->op0, a->op1, a->op2, a->crn, a->crm, a->rt);
3292	return true;
3293	}
3294
3295	static bool trans_SVC(DisasContext s, arg_i a)
3296	{
3297	/*
3298	* For SVC, HVC and SMC we advance the single-step state
3299	* machine before taking the exception. This is architecturally
3300	* mandated, to ensure that single-stepping a system call
3301	* instruction works properly.
3302	*/
3303	uint32_t syndrome = syn_aa64_svc(a->imm);
3304	if (s->fgt_svc) {
3305	gen_exception_insn_el(s, 0, EXCP_UDEF, syndrome, 2);
3306	return true;
3307	}
3308	gen_ss_advance(s);
3309	gen_exception_insn(s, 4, EXCP_SWI, syndrome);
3310	return true;
3311	}
3312
3313	static bool trans_HVC(DisasContext s, arg_i a)
3314	{
3315	int target_el = s->current_el == 3 ? 3 : 2;
3316
3317	if (s->current_el == 0) {
3318	unallocated_encoding(s);
3319	return true;
3320	}
3321	/*
3322	* The pre HVC helper handles cases when HVC gets trapped
3323	* as an undefined insn by runtime configuration.
3324	*/
3325	gen_a64_update_pc(s, 0);
3326	gen_helper_pre_hvc(tcg_env);
3327	/* Architecture requires ss advance before we do the actual work */
3328	gen_ss_advance(s);
3329	gen_exception_insn_el(s, 4, EXCP_HVC, syn_aa64_hvc(a->imm), target_el);
3330	return true;
3331	}
3332
3333	static bool trans_SMC(DisasContext s, arg_i a)
3334	{
3335	if (s->current_el == 0) {
3336	unallocated_encoding(s);
3337	return true;
3338	}
3339	gen_a64_update_pc(s, 0);
3340	gen_helper_pre_smc(tcg_env, tcg_constant_i32(syn_aa64_smc(a->imm)));
3341	/* Architecture requires ss advance before we do the actual work */
3342	gen_ss_advance(s);
3343	gen_exception_insn_el(s, 4, EXCP_SMC, syn_aa64_smc(a->imm), 3);
3344	return true;
3345	}
3346
3347	static bool trans_BRK(DisasContext s, arg_i a)
3348	{
3349	gen_exception_bkpt_insn(s, syn_aa64_bkpt(a->imm));
3350	return true;
3351	}
3352
3353	static bool trans_HLT(DisasContext s, arg_i a)
3354	{
3355	/*
3356	* HLT. This has two purposes.
3357	* Architecturally, it is an external halting debug instruction.
3358	* Since QEMU doesn't implement external debug, we treat this as
3359	* it is required for halting debug disabled: it will UNDEF.
3360	* Secondly, "HLT 0xf000" is the A64 semihosting syscall instruction.
3361	*/
3362	if (semihosting_enabled(s->current_el == 0) && a->imm == 0xf000) {
3363	gen_exception_internal_insn(s, EXCP_SEMIHOST);
3364	} else {
3365	unallocated_encoding(s);
3366	}
3367	return true;
3368	}
3369
3370	/*
3371	* Load/Store exclusive instructions are implemented by remembering
3372	* the value/address loaded, and seeing if these are the same
3373	* when the store is performed. This is not actually the architecturally
3374	* mandated semantics, but it works for typical guest code sequences
3375	* and avoids having to monitor regular stores.
3376	*
3377	* The store exclusive uses the atomic cmpxchg primitives to avoid
3378	* races in multi-threaded linux-user and when MTTCG softmmu is
3379	* enabled.
3380	*/
3381	static void gen_load_exclusive(DisasContext *s, int rt, int rt2, int rn,
3382	int size, bool is_pair)
3383	{
3384	int idx = get_mem_index(s);
3385	TCGv_i64 dirty_addr, clean_addr;
3386	MemOp memop = check_atomic_align(s, rn, size + is_pair);
3387
3388	s->is_ldex = true;
3389	dirty_addr = cpu_reg_sp(s, rn);
3390	clean_addr = gen_mte_check1(s, dirty_addr, false, rn != 31, memop);
3391
3392	g_assert(size <= 3);
3393	if (is_pair) {
3394	g_assert(size >= 2);
3395	if (size == 2) {
3396	tcg_gen_qemu_ld_i64(cpu_exclusive_val, clean_addr, idx, memop);
3397	if (s->be_data == MO_LE) {
3398	tcg_gen_extract_i64(cpu_reg(s, rt), cpu_exclusive_val, 0, 32);
3399	tcg_gen_extract_i64(cpu_reg(s, rt2), cpu_exclusive_val, 32, 32);
3400	} else {
3401	tcg_gen_extract_i64(cpu_reg(s, rt), cpu_exclusive_val, 32, 32);
3402	tcg_gen_extract_i64(cpu_reg(s, rt2), cpu_exclusive_val, 0, 32);
3403	}
3404	} else {
3405	TCGv_i128 t16 = tcg_temp_new_i128();
3406
3407	tcg_gen_qemu_ld_i128(t16, clean_addr, idx, memop);
3408
3409	if (s->be_data == MO_LE) {
3410	tcg_gen_extr_i128_i64(cpu_exclusive_val,
3411	cpu_exclusive_high, t16);
3412	} else {
3413	tcg_gen_extr_i128_i64(cpu_exclusive_high,
3414	cpu_exclusive_val, t16);
3415	}
3416	tcg_gen_mov_i64(cpu_reg(s, rt), cpu_exclusive_val);
3417	tcg_gen_mov_i64(cpu_reg(s, rt2), cpu_exclusive_high);
3418	}
3419	} else {
3420	tcg_gen_qemu_ld_i64(cpu_exclusive_val, clean_addr, idx, memop);
3421	tcg_gen_mov_i64(cpu_reg(s, rt), cpu_exclusive_val);
3422	}
3423	tcg_gen_mov_i64(cpu_exclusive_addr, clean_addr);
3424	}
3425
3426	static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2,
3427	int rn, int size, int is_pair)
3428	{
3429	/* if (env->exclusive_addr == addr && env->exclusive_val == [addr]
3430	* && (!is_pair \|\| env->exclusive_high == [addr + datasize])) {
3431	* [addr] = {Rt};
3432	* if (is_pair) {
3433	* [addr + datasize] = {Rt2};
3434	* }
3435	* {Rd} = 0;
3436	* } else {
3437	* {Rd} = 1;
3438	* }
3439	* env->exclusive_addr = -1;
3440	*/
3441	TCGLabel *fail_label = gen_new_label();
3442	TCGLabel *done_label = gen_new_label();
3443	TCGv_i64 tmp, clean_addr;
3444	MemOp memop;
3445
3446	/*
3447	* FIXME: We are out of spec here. We have recorded only the address
3448	* from load_exclusive, not the entire range, and we assume that the
3449	* size of the access on both sides match. The architecture allows the
3450	* store to be smaller than the load, so long as the stored bytes are
3451	* within the range recorded by the load.
3452	*/
3453
3454	/* See AArch64.ExclusiveMonitorsPass() and AArch64.IsExclusiveVA(). */
3455	clean_addr = clean_data_tbi(s, cpu_reg_sp(s, rn));
3456	tcg_gen_brcond_i64(TCG_COND_NE, clean_addr, cpu_exclusive_addr, fail_label);
3457
3458	/*
3459	* The write, and any associated faults, only happen if the virtual
3460	* and physical addresses pass the exclusive monitor check. These
3461	* faults are exceedingly unlikely, because normally the guest uses
3462	* the exact same address register for the load_exclusive, and we
3463	* would have recognized these faults there.
3464	*
3465	* It is possible to trigger an alignment fault pre-LSE2, e.g. with an
3466	* unaligned 4-byte write within the range of an aligned 8-byte load.
3467	* With LSE2, the store would need to cross a 16-byte boundary when the
3468	* load did not, which would mean the store is outside the range
3469	* recorded for the monitor, which would have failed a corrected monitor
3470	* check above. For now, we assume no size change and retain the
3471	* MO_ALIGN to let tcg know what we checked in the load_exclusive.
3472	*
3473	* It is possible to trigger an MTE fault, by performing the load with
3474	* a virtual address with a valid tag and performing the store with the
3475	* same virtual address and a different invalid tag.
3476	*/
3477	memop = size + is_pair;
3478	if (memop == MO_128 \|\| !dc_isar_feature(aa64_lse2, s)) {
3479	memop \|= MO_ALIGN;
3480	}
3481	memop = finalize_memop(s, memop);
3482	gen_mte_check1(s, cpu_reg_sp(s, rn), true, rn != 31, memop);
3483
3484	tmp = tcg_temp_new_i64();
3485	if (is_pair) {
3486	if (size == 2) {
3487	if (s->be_data == MO_LE) {
3488	tcg_gen_concat32_i64(tmp, cpu_reg(s, rt), cpu_reg(s, rt2));
3489	} else {
3490	tcg_gen_concat32_i64(tmp, cpu_reg(s, rt2), cpu_reg(s, rt));
3491	}
3492	tcg_gen_atomic_cmpxchg_i64(tmp, cpu_exclusive_addr,
3493	cpu_exclusive_val, tmp,
3494	get_mem_index(s), memop);
3495	tcg_gen_setcond_i64(TCG_COND_NE, tmp, tmp, cpu_exclusive_val);
3496	} else {
3497	TCGv_i128 t16 = tcg_temp_new_i128();
3498	TCGv_i128 c16 = tcg_temp_new_i128();
3499	TCGv_i64 a, b;
3500
3501	if (s->be_data == MO_LE) {
3502	tcg_gen_concat_i64_i128(t16, cpu_reg(s, rt), cpu_reg(s, rt2));
3503	tcg_gen_concat_i64_i128(c16, cpu_exclusive_val,
3504	cpu_exclusive_high);
3505	} else {
3506	tcg_gen_concat_i64_i128(t16, cpu_reg(s, rt2), cpu_reg(s, rt));
3507	tcg_gen_concat_i64_i128(c16, cpu_exclusive_high,
3508	cpu_exclusive_val);
3509	}
3510
3511	tcg_gen_atomic_cmpxchg_i128(t16, cpu_exclusive_addr, c16, t16,
3512	get_mem_index(s), memop);
3513
3514	a = tcg_temp_new_i64();
3515	b = tcg_temp_new_i64();
3516	if (s->be_data == MO_LE) {
3517	tcg_gen_extr_i128_i64(a, b, t16);
3518	} else {
3519	tcg_gen_extr_i128_i64(b, a, t16);
3520	}
3521
3522	tcg_gen_xor_i64(a, a, cpu_exclusive_val);
3523	tcg_gen_xor_i64(b, b, cpu_exclusive_high);
3524	tcg_gen_or_i64(tmp, a, b);
3525
3526	tcg_gen_setcondi_i64(TCG_COND_NE, tmp, tmp, 0);
3527	}
3528	} else {
3529	tcg_gen_atomic_cmpxchg_i64(tmp, cpu_exclusive_addr, cpu_exclusive_val,
3530	cpu_reg(s, rt), get_mem_index(s), memop);
3531	tcg_gen_setcond_i64(TCG_COND_NE, tmp, tmp, cpu_exclusive_val);
3532	}
3533	tcg_gen_mov_i64(cpu_reg(s, rd), tmp);
3534	tcg_gen_br(done_label);
3535
3536	gen_set_label(fail_label);
3537	tcg_gen_movi_i64(cpu_reg(s, rd), 1);
3538	gen_set_label(done_label);
3539	tcg_gen_movi_i64(cpu_exclusive_addr, -1);
3540	}
3541
3542	static void gen_compare_and_swap(DisasContext *s, int rs, int rt,
3543	int rn, int size)
3544	{
3545	TCGv_i64 tcg_rs = cpu_reg(s, rs);
3546	TCGv_i64 tcg_rt = cpu_reg(s, rt);
3547	int memidx = get_mem_index(s);
3548	TCGv_i64 clean_addr;
3549	MemOp memop;
3550
3551	if (rn == 31) {
3552	gen_check_sp_alignment(s);
3553	}
3554	memop = check_atomic_align(s, rn, size);
3555	clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), true, rn != 31, memop);
3556	tcg_gen_atomic_cmpxchg_i64(tcg_rs, clean_addr, tcg_rs, tcg_rt,
3557	memidx, memop);
3558	}
3559
3560	static void gen_compare_and_swap_pair(DisasContext *s, int rs, int rt,
3561	int rn, int size)
3562	{
3563	TCGv_i64 s1 = cpu_reg(s, rs);
3564	TCGv_i64 s2 = cpu_reg(s, rs + 1);
3565	TCGv_i64 t1 = cpu_reg(s, rt);
3566	TCGv_i64 t2 = cpu_reg(s, rt + 1);
3567	TCGv_i64 clean_addr;
3568	int memidx = get_mem_index(s);
3569	MemOp memop;
3570
3571	if (rn == 31) {
3572	gen_check_sp_alignment(s);
3573	}
3574
3575	/* This is a single atomic access, despite the "pair". */
3576	memop = check_atomic_align(s, rn, size + 1);
3577	clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), true, rn != 31, memop);
3578
3579	if (size == 2) {
3580	TCGv_i64 cmp = tcg_temp_new_i64();
3581	TCGv_i64 val = tcg_temp_new_i64();
3582
3583	if (s->be_data == MO_LE) {
3584	tcg_gen_concat32_i64(val, t1, t2);
3585	tcg_gen_concat32_i64(cmp, s1, s2);
3586	} else {
3587	tcg_gen_concat32_i64(val, t2, t1);
3588	tcg_gen_concat32_i64(cmp, s2, s1);
3589	}
3590
3591	tcg_gen_atomic_cmpxchg_i64(cmp, clean_addr, cmp, val, memidx, memop);
3592
3593	if (s->be_data == MO_LE) {
3594	tcg_gen_extr32_i64(s1, s2, cmp);
3595	} else {
3596	tcg_gen_extr32_i64(s2, s1, cmp);
3597	}
3598	} else {
3599	TCGv_i128 cmp = tcg_temp_new_i128();
3600	TCGv_i128 val = tcg_temp_new_i128();
3601
3602	if (s->be_data == MO_LE) {
3603	tcg_gen_concat_i64_i128(val, t1, t2);
3604	tcg_gen_concat_i64_i128(cmp, s1, s2);
3605	} else {
3606	tcg_gen_concat_i64_i128(val, t2, t1);
3607	tcg_gen_concat_i64_i128(cmp, s2, s1);
3608	}
3609
3610	tcg_gen_atomic_cmpxchg_i128(cmp, clean_addr, cmp, val, memidx, memop);
3611
3612	if (s->be_data == MO_LE) {
3613	tcg_gen_extr_i128_i64(s1, s2, cmp);
3614	} else {
3615	tcg_gen_extr_i128_i64(s2, s1, cmp);
3616	}
3617	}
3618	}
3619
3620	/*
3621	* Compute the ISS.SF bit for syndrome information if an exception
3622	* is taken on a load or store. This indicates whether the instruction
3623	* is accessing a 32-bit or 64-bit register. This logic is derived
3624	* from the ARMv8 specs for LDR (Shared decode for all encodings).
3625	*/
3626	static bool ldst_iss_sf(int size, bool sign, bool ext)
3627	{
3628
3629	if (sign) {
3630	/*
3631	* Signed loads are 64 bit results if we are not going to
3632	* do a zero-extend from 32 to 64 after the load.
3633	* (For a store, sign and ext are always false.)
3634	*/
3635	return !ext;
3636	} else {
3637	/* Unsigned loads/stores work at the specified size */
3638	return size == MO_64;
3639	}
3640	}
3641
3642	static bool trans_STXR(DisasContext s, arg_stxr a)
3643	{
3644	if (a->rn == 31) {
3645	gen_check_sp_alignment(s);
3646	}
3647	if (a->lasr) {
3648	tcg_gen_mb(TCG_MO_ALL \| TCG_BAR_STRL);
3649	}
3650	gen_store_exclusive(s, a->rs, a->rt, a->rt2, a->rn, a->sz, false);
3651	return true;
3652	}
3653
3654	static bool trans_LDXR(DisasContext s, arg_stxr a)
3655	{
3656	if (a->rn == 31) {
3657	gen_check_sp_alignment(s);
3658	}
3659	gen_load_exclusive(s, a->rt, a->rt2, a->rn, a->sz, false);
3660	if (a->lasr) {
3661	tcg_gen_mb(TCG_MO_ALL \| TCG_BAR_LDAQ);
3662	}
3663	return true;
3664	}
3665
3666	static bool trans_STLR(DisasContext s, arg_stlr a)
3667	{
3668	TCGv_i64 clean_addr;
3669	MemOp memop;
3670	bool iss_sf = ldst_iss_sf(a->sz, false, false);
3671
3672	/*
3673	* StoreLORelease is the same as Store-Release for QEMU, but
3674	* needs the feature-test.
3675	*/
3676	if (!a->lasr && !dc_isar_feature(aa64_lor, s)) {
3677	return false;
3678	}
3679	/* Generate ISS for non-exclusive accesses including LASR. */
3680	if (a->rn == 31) {
3681	gen_check_sp_alignment(s);
3682	}
3683	tcg_gen_mb(TCG_MO_ALL \| TCG_BAR_STRL);
3684	memop = check_ordered_align(s, a->rn, 0, true, a->sz);
3685	clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn),
3686	true, a->rn != 31, memop);
3687	do_gpr_st(s, cpu_reg(s, a->rt), clean_addr, memop, true, a->rt,
3688	iss_sf, a->lasr);
3689	return true;
3690	}
3691
3692	static bool trans_LDAR(DisasContext s, arg_stlr a)
3693	{
3694	TCGv_i64 clean_addr;
3695	MemOp memop;
3696	bool iss_sf = ldst_iss_sf(a->sz, false, false);
3697
3698	/* LoadLOAcquire is the same as Load-Acquire for QEMU. */
3699	if (!a->lasr && !dc_isar_feature(aa64_lor, s)) {
3700	return false;
3701	}
3702	/* Generate ISS for non-exclusive accesses including LASR. */
3703	if (a->rn == 31) {
3704	gen_check_sp_alignment(s);
3705	}
3706	memop = check_ordered_align(s, a->rn, 0, false, a->sz);
3707	clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn),
3708	false, a->rn != 31, memop);
3709	do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, memop, false, true,
3710	a->rt, iss_sf, a->lasr);
3711	tcg_gen_mb(TCG_MO_ALL \| TCG_BAR_LDAQ);
3712	return true;
3713	}
3714
3715	static bool trans_STXP(DisasContext s, arg_stxr a)
3716	{
3717	if (a->rn == 31) {
3718	gen_check_sp_alignment(s);
3719	}
3720	if (a->lasr) {
3721	tcg_gen_mb(TCG_MO_ALL \| TCG_BAR_STRL);
3722	}
3723	gen_store_exclusive(s, a->rs, a->rt, a->rt2, a->rn, a->sz, true);
3724	return true;
3725	}
3726
3727	static bool trans_LDXP(DisasContext s, arg_stxr a)
3728	{
3729	if (a->rn == 31) {
3730	gen_check_sp_alignment(s);
3731	}
3732	gen_load_exclusive(s, a->rt, a->rt2, a->rn, a->sz, true);
3733	if (a->lasr) {
3734	tcg_gen_mb(TCG_MO_ALL \| TCG_BAR_LDAQ);
3735	}
3736	return true;
3737	}
3738
3739	static bool trans_CASP(DisasContext s, arg_CASP a)
3740	{
3741	if (!dc_isar_feature(aa64_lse, s)) {
3742	return false;
3743	}
3744	if (((a->rt \| a->rs) & 1) != 0) {
3745	return false;
3746	}
3747
3748	gen_compare_and_swap_pair(s, a->rs, a->rt, a->rn, a->sz);
3749	return true;
3750	}
3751
3752	static bool trans_CAS(DisasContext s, arg_CAS a)
3753	{
3754	if (!dc_isar_feature(aa64_lse, s)) {
3755	return false;
3756	}
3757	gen_compare_and_swap(s, a->rs, a->rt, a->rn, a->sz);
3758	return true;
3759	}
3760
3761	static bool trans_LD_lit(DisasContext s, arg_ldlit a)
3762	{
3763	bool iss_sf = ldst_iss_sf(a->sz, a->sign, false);
3764	TCGv_i64 tcg_rt = cpu_reg(s, a->rt);
3765	TCGv_i64 clean_addr = tcg_temp_new_i64();
3766	MemOp memop = finalize_memop(s, a->sz + a->sign * MO_SIGN);
3767
3768	gen_pc_plus_diff(s, clean_addr, a->imm);
3769	do_gpr_ld(s, tcg_rt, clean_addr, memop,
3770	false, true, a->rt, iss_sf, false);
3771	return true;
3772	}
3773
3774	static bool trans_LD_lit_v(DisasContext s, arg_ldlit a)
3775	{
3776	/* Load register (literal), vector version */
3777	TCGv_i64 clean_addr;
3778	MemOp memop;
3779
3780	if (!fp_access_check(s)) {
3781	return true;
3782	}
3783	memop = finalize_memop_asimd(s, a->sz);
3784	clean_addr = tcg_temp_new_i64();
3785	gen_pc_plus_diff(s, clean_addr, a->imm);
3786	do_fp_ld(s, a->rt, clean_addr, memop);
3787	return true;
3788	}
3789
3790	static void op_addr_ldstpair_pre(DisasContext s, arg_ldstpair a,
3791	TCGv_i64 clean_addr, TCGv_i64 dirty_addr,
3792	uint64_t offset, bool is_store, MemOp mop)
3793	{
3794	if (a->rn == 31) {
3795	gen_check_sp_alignment(s);
3796	}
3797
3798	*dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3799	if (!a->p) {
3800	tcg_gen_addi_i64(dirty_addr, dirty_addr, offset);
3801	}
3802
3803	clean_addr = gen_mte_checkN(s, dirty_addr, is_store,
3804	(a->w \|\| a->rn != 31), 2 << a->sz, mop);
3805	}
3806
3807	static void op_addr_ldstpair_post(DisasContext s, arg_ldstpair a,
3808	TCGv_i64 dirty_addr, uint64_t offset)
3809	{
3810	if (a->w) {
3811	if (a->p) {
3812	tcg_gen_addi_i64(dirty_addr, dirty_addr, offset);
3813	}
3814	tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr);
3815	}
3816	}
3817
3818	static bool trans_STP(DisasContext s, arg_ldstpair a)
3819	{
3820	uint64_t offset = a->imm << a->sz;
3821	TCGv_i64 clean_addr, dirty_addr, tcg_rt, tcg_rt2;
3822	MemOp mop = finalize_memop(s, a->sz);
3823
3824	op_addr_ldstpair_pre(s, a, &clean_addr, &dirty_addr, offset, true, mop);
3825	tcg_rt = cpu_reg(s, a->rt);
3826	tcg_rt2 = cpu_reg(s, a->rt2);
3827	/*
3828	* We built mop above for the single logical access -- rebuild it
3829	* now for the paired operation.
3830	*
3831	* With LSE2, non-sign-extending pairs are treated atomically if
3832	* aligned, and if unaligned one of the pair will be completely
3833	* within a 16-byte block and that element will be atomic.
3834	* Otherwise each element is separately atomic.
3835	* In all cases, issue one operation with the correct atomicity.
3836	*/
3837	mop = a->sz + 1;
3838	mop \|= (a->sz == 2 ? MO_ALIGN_4 : MO_ALIGN_8);
3839	mop \|= (s->align_mem ? 0 : MO_ALIGN_TLB_ONLY);
3840	mop = finalize_memop_pair(s, mop);
3841	if (a->sz == 2) {
3842	TCGv_i64 tmp = tcg_temp_new_i64();
3843
3844	if (s->be_data == MO_LE) {
3845	tcg_gen_concat32_i64(tmp, tcg_rt, tcg_rt2);
3846	} else {
3847	tcg_gen_concat32_i64(tmp, tcg_rt2, tcg_rt);
3848	}
3849	tcg_gen_qemu_st_i64(tmp, clean_addr, get_mem_index(s), mop);
3850	} else {
3851	TCGv_i128 tmp = tcg_temp_new_i128();
3852
3853	if (s->be_data == MO_LE) {
3854	tcg_gen_concat_i64_i128(tmp, tcg_rt, tcg_rt2);
3855	} else {
3856	tcg_gen_concat_i64_i128(tmp, tcg_rt2, tcg_rt);
3857	}
3858	tcg_gen_qemu_st_i128(tmp, clean_addr, get_mem_index(s), mop);
3859	}
3860	op_addr_ldstpair_post(s, a, dirty_addr, offset);
3861	return true;
3862	}
3863
3864	static bool trans_LDP(DisasContext s, arg_ldstpair a)
3865	{
3866	uint64_t offset = a->imm << a->sz;
3867	TCGv_i64 clean_addr, dirty_addr, tcg_rt, tcg_rt2;
3868	MemOp mop = finalize_memop(s, a->sz);
3869
3870	op_addr_ldstpair_pre(s, a, &clean_addr, &dirty_addr, offset, false, mop);
3871	tcg_rt = cpu_reg(s, a->rt);
3872	tcg_rt2 = cpu_reg(s, a->rt2);
3873
3874	/*
3875	* We built mop above for the single logical access -- rebuild it
3876	* now for the paired operation.
3877	*
3878	* With LSE2, non-sign-extending pairs are treated atomically if
3879	* aligned, and if unaligned one of the pair will be completely
3880	* within a 16-byte block and that element will be atomic.
3881	* Otherwise each element is separately atomic.
3882	* In all cases, issue one operation with the correct atomicity.
3883	*
3884	* This treats sign-extending loads like zero-extending loads,
3885	* since that reuses the most code below.
3886	*/
3887	mop = a->sz + 1;
3888	mop \|= (a->sz == 2 ? MO_ALIGN_4 : MO_ALIGN_8);
3889	mop \|= (s->align_mem ? 0 : MO_ALIGN_TLB_ONLY);
3890	mop = finalize_memop_pair(s, mop);
3891	if (a->sz == 2) {
3892	int o2 = s->be_data == MO_LE ? 32 : 0;
3893	int o1 = o2 ^ 32;
3894
3895	tcg_gen_qemu_ld_i64(tcg_rt, clean_addr, get_mem_index(s), mop);
3896	if (a->sign) {
3897	tcg_gen_sextract_i64(tcg_rt2, tcg_rt, o2, 32);
3898	tcg_gen_sextract_i64(tcg_rt, tcg_rt, o1, 32);
3899	} else {
3900	tcg_gen_extract_i64(tcg_rt2, tcg_rt, o2, 32);
3901	tcg_gen_extract_i64(tcg_rt, tcg_rt, o1, 32);
3902	}
3903	} else {
3904	TCGv_i128 tmp = tcg_temp_new_i128();
3905
3906	tcg_gen_qemu_ld_i128(tmp, clean_addr, get_mem_index(s), mop);
3907	if (s->be_data == MO_LE) {
3908	tcg_gen_extr_i128_i64(tcg_rt, tcg_rt2, tmp);
3909	} else {
3910	tcg_gen_extr_i128_i64(tcg_rt2, tcg_rt, tmp);
3911	}
3912	}
3913	op_addr_ldstpair_post(s, a, dirty_addr, offset);
3914	return true;
3915	}
3916
3917	static bool trans_STP_v(DisasContext s, arg_ldstpair a)
3918	{
3919	uint64_t offset = a->imm << a->sz;
3920	TCGv_i64 clean_addr, dirty_addr;
3921	MemOp mop;
3922
3923	if (!fp_access_check(s)) {
3924	return true;
3925	}
3926
3927	/* LSE2 does not merge FP pairs; leave these as separate operations. */
3928	mop = finalize_memop_asimd(s, a->sz);
3929	op_addr_ldstpair_pre(s, a, &clean_addr, &dirty_addr, offset, true, mop);
3930	do_fp_st(s, a->rt, clean_addr, mop);
3931	tcg_gen_addi_i64(clean_addr, clean_addr, 1 << a->sz);
3932	do_fp_st(s, a->rt2, clean_addr, mop);
3933	op_addr_ldstpair_post(s, a, dirty_addr, offset);
3934	return true;
3935	}
3936
3937	static bool trans_LDP_v(DisasContext s, arg_ldstpair a)
3938	{
3939	uint64_t offset = a->imm << a->sz;
3940	TCGv_i64 clean_addr, dirty_addr;
3941	MemOp mop;
3942
3943	if (!fp_access_check(s)) {
3944	return true;
3945	}
3946
3947	/* LSE2 does not merge FP pairs; leave these as separate operations. */
3948	mop = finalize_memop_asimd(s, a->sz);
3949	op_addr_ldstpair_pre(s, a, &clean_addr, &dirty_addr, offset, false, mop);
3950	do_fp_ld(s, a->rt, clean_addr, mop);
3951	tcg_gen_addi_i64(clean_addr, clean_addr, 1 << a->sz);
3952	do_fp_ld(s, a->rt2, clean_addr, mop);
3953	op_addr_ldstpair_post(s, a, dirty_addr, offset);
3954	return true;
3955	}
3956
3957	static bool trans_STGP(DisasContext s, arg_ldstpair a)
3958	{
3959	TCGv_i64 clean_addr, dirty_addr, tcg_rt, tcg_rt2;
3960	uint64_t offset = a->imm << LOG2_TAG_GRANULE;
3961	MemOp mop;
3962	TCGv_i128 tmp;
3963
3964	/* STGP only comes in one size. */
3965	tcg_debug_assert(a->sz == MO_64);
3966
3967	if (!dc_isar_feature(aa64_mte_insn_reg, s)) {
3968	return false;
3969	}
3970
3971	if (a->rn == 31) {
3972	gen_check_sp_alignment(s);
3973	}
3974
3975	dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3976	if (!a->p) {
3977	tcg_gen_addi_i64(dirty_addr, dirty_addr, offset);
3978	}
3979
3980	clean_addr = clean_data_tbi(s, dirty_addr);
3981	tcg_rt = cpu_reg(s, a->rt);
3982	tcg_rt2 = cpu_reg(s, a->rt2);
3983
3984	/*
3985	* STGP is defined as two 8-byte memory operations, aligned to TAG_GRANULE,
3986	* and one tag operation. We implement it as one single aligned 16-byte
3987	* memory operation for convenience. Note that the alignment ensures
3988	* MO_ATOM_IFALIGN_PAIR produces 8-byte atomicity for the memory store.
3989	*/
3990	mop = finalize_memop_atom(s, MO_128 \| MO_ALIGN, MO_ATOM_IFALIGN_PAIR);
3991
3992	tmp = tcg_temp_new_i128();
3993	if (s->be_data == MO_LE) {
3994	tcg_gen_concat_i64_i128(tmp, tcg_rt, tcg_rt2);
3995	} else {
3996	tcg_gen_concat_i64_i128(tmp, tcg_rt2, tcg_rt);
3997	}
3998	tcg_gen_qemu_st_i128(tmp, clean_addr, get_mem_index(s), mop);
3999
4000	/* Perform the tag store, if tag access enabled. */
4001	if (s->ata[0]) {
4002	if (tb_cflags(s->base.tb) & CF_PARALLEL) {
4003	gen_helper_stg_parallel(tcg_env, dirty_addr, dirty_addr,
4004	tcg_constant_i32(s->mtx));
4005	} else {
4006	gen_helper_stg(tcg_env, dirty_addr, dirty_addr,
4007	tcg_constant_i32(s->mtx));
4008	}
4009	}
4010
4011	op_addr_ldstpair_post(s, a, dirty_addr, offset);
4012	return true;
4013	}
4014
4015	static void op_addr_ldst_imm_pre(DisasContext s, arg_ldst_imm a,
4016	TCGv_i64 clean_addr, TCGv_i64 dirty_addr,
4017	uint64_t offset, bool is_store, MemOp mop)
4018	{
4019	int memidx;
4020
4021	if (a->rn == 31) {
4022	gen_check_sp_alignment(s);
4023	}
4024
4025	*dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
4026	if (!a->p) {
4027	tcg_gen_addi_i64(dirty_addr, dirty_addr, offset);
4028	}
4029	memidx = core_a64_user_mem_index(s, a->unpriv);
4030	clean_addr = gen_mte_check1_mmuidx(s, dirty_addr, is_store,
4031	a->w \|\| a->rn != 31,
4032	mop, a->unpriv, memidx);
4033	}
4034
4035	static void op_addr_ldst_imm_post(DisasContext s, arg_ldst_imm a,
4036	TCGv_i64 dirty_addr, uint64_t offset)
4037	{
4038	if (a->w) {
4039	if (a->p) {
4040	tcg_gen_addi_i64(dirty_addr, dirty_addr, offset);
4041	}
4042	tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr);
4043	}
4044	}
4045
4046	static bool trans_STR_i(DisasContext s, arg_ldst_imm a)
4047	{
4048	bool iss_sf, iss_valid = !a->w;
4049	TCGv_i64 clean_addr, dirty_addr, tcg_rt;
4050	int memidx = core_a64_user_mem_index(s, a->unpriv);
4051	MemOp mop = finalize_memop(s, a->sz + a->sign * MO_SIGN);
4052
4053	op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, true, mop);
4054
4055	tcg_rt = cpu_reg(s, a->rt);
4056	iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
4057
4058	do_gpr_st_memidx(s, tcg_rt, clean_addr, mop, memidx,
4059	iss_valid, a->rt, iss_sf, false);
4060	op_addr_ldst_imm_post(s, a, dirty_addr, a->imm);
4061	return true;
4062	}
4063
4064	static bool trans_LDR_i(DisasContext s, arg_ldst_imm a)
4065	{
4066	bool iss_sf, iss_valid = !a->w;
4067	TCGv_i64 clean_addr, dirty_addr, tcg_rt;
4068	int memidx = core_a64_user_mem_index(s, a->unpriv);
4069	MemOp mop = finalize_memop(s, a->sz + a->sign * MO_SIGN);
4070
4071	op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, false, mop);
4072
4073	tcg_rt = cpu_reg(s, a->rt);
4074	iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
4075
4076	do_gpr_ld_memidx(s, tcg_rt, clean_addr, mop,
4077	a->ext, memidx, iss_valid, a->rt, iss_sf, false);
4078	op_addr_ldst_imm_post(s, a, dirty_addr, a->imm);
4079	return true;
4080	}
4081
4082	static bool trans_STR_v_i(DisasContext s, arg_ldst_imm a)
4083	{
4084	TCGv_i64 clean_addr, dirty_addr;
4085	MemOp mop;
4086
4087	if (!fp_access_check(s)) {
4088	return true;
4089	}
4090	mop = finalize_memop_asimd(s, a->sz);
4091	op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, true, mop);
4092	do_fp_st(s, a->rt, clean_addr, mop);
4093	op_addr_ldst_imm_post(s, a, dirty_addr, a->imm);
4094	return true;
4095	}
4096
4097	static bool trans_LDR_v_i(DisasContext s, arg_ldst_imm a)
4098	{
4099	TCGv_i64 clean_addr, dirty_addr;
4100	MemOp mop;
4101
4102	if (!fp_access_check(s)) {
4103	return true;
4104	}
4105	mop = finalize_memop_asimd(s, a->sz);
4106	op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, false, mop);
4107	do_fp_ld(s, a->rt, clean_addr, mop);
4108	op_addr_ldst_imm_post(s, a, dirty_addr, a->imm);
4109	return true;
4110	}
4111
4112	static void op_addr_ldst_pre(DisasContext s, arg_ldst a,
4113	TCGv_i64 clean_addr, TCGv_i64 dirty_addr,
4114	bool is_store, MemOp memop)
4115	{
4116	TCGv_i64 tcg_rm;
4117
4118	if (a->rn == 31) {
4119	gen_check_sp_alignment(s);
4120	}
4121	*dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
4122
4123	tcg_rm = read_cpu_reg(s, a->rm, 1);
4124	ext_and_shift_reg(tcg_rm, tcg_rm, a->opt, a->s ? a->sz : 0);
4125
4126	tcg_gen_add_i64(dirty_addr, dirty_addr, tcg_rm);
4127	clean_addr = gen_mte_check1(s, dirty_addr, is_store, true, memop);
4128	}
4129
4130	static bool trans_LDR(DisasContext s, arg_ldst a)
4131	{
4132	TCGv_i64 clean_addr, dirty_addr, tcg_rt;
4133	bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
4134	MemOp memop;
4135
4136	if (extract32(a->opt, 1, 1) == 0) {
4137	return false;
4138	}
4139
4140	memop = finalize_memop(s, a->sz + a->sign * MO_SIGN);
4141	op_addr_ldst_pre(s, a, &clean_addr, &dirty_addr, false, memop);
4142	tcg_rt = cpu_reg(s, a->rt);
4143	do_gpr_ld(s, tcg_rt, clean_addr, memop,
4144	a->ext, true, a->rt, iss_sf, false);
4145	return true;
4146	}
4147
4148	static bool trans_STR(DisasContext s, arg_ldst a)
4149	{
4150	TCGv_i64 clean_addr, dirty_addr, tcg_rt;
4151	bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
4152	MemOp memop;
4153
4154	if (extract32(a->opt, 1, 1) == 0) {
4155	return false;
4156	}
4157
4158	memop = finalize_memop(s, a->sz);
4159	op_addr_ldst_pre(s, a, &clean_addr, &dirty_addr, true, memop);
4160	tcg_rt = cpu_reg(s, a->rt);
4161	do_gpr_st(s, tcg_rt, clean_addr, memop, true, a->rt, iss_sf, false);
4162	return true;
4163	}
4164
4165	static bool trans_LDR_v(DisasContext s, arg_ldst a)
4166	{
4167	TCGv_i64 clean_addr, dirty_addr;
4168	MemOp memop;
4169
4170	if (extract32(a->opt, 1, 1) == 0) {
4171	return false;
4172	}
4173
4174	if (!fp_access_check(s)) {
4175	return true;
4176	}
4177
4178	memop = finalize_memop_asimd(s, a->sz);
4179	op_addr_ldst_pre(s, a, &clean_addr, &dirty_addr, false, memop);
4180	do_fp_ld(s, a->rt, clean_addr, memop);
4181	return true;
4182	}
4183
4184	static bool trans_STR_v(DisasContext s, arg_ldst a)
4185	{
4186	TCGv_i64 clean_addr, dirty_addr;
4187	MemOp memop;
4188
4189	if (extract32(a->opt, 1, 1) == 0) {
4190	return false;
4191	}
4192
4193	if (!fp_access_check(s)) {
4194	return true;
4195	}
4196
4197	memop = finalize_memop_asimd(s, a->sz);
4198	op_addr_ldst_pre(s, a, &clean_addr, &dirty_addr, true, memop);
4199	do_fp_st(s, a->rt, clean_addr, memop);
4200	return true;
4201	}
4202
4203
4204	static bool do_atomic_ld(DisasContext s, arg_atomic a, AtomicThreeOpFn *fn,
4205	int sign, bool invert)
4206	{
4207	MemOp mop = a->sz \| sign;
4208	TCGv_i64 clean_addr, tcg_rs, tcg_rt;
4209
4210	if (a->rn == 31) {
4211	gen_check_sp_alignment(s);
4212	}
4213	mop = check_atomic_align(s, a->rn, mop);
4214	clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), false,
4215	a->rn != 31, mop);
4216	tcg_rs = read_cpu_reg(s, a->rs, true);
4217	tcg_rt = cpu_reg(s, a->rt);
4218	if (invert) {
4219	tcg_gen_not_i64(tcg_rs, tcg_rs);
4220	}
4221	/*
4222	* The tcg atomic primitives are all full barriers. Therefore we
4223	* can ignore the Acquire and Release bits of this instruction.
4224	*/
4225	fn(tcg_rt, clean_addr, tcg_rs, get_mem_index(s), mop);
4226
4227	if (mop & MO_SIGN) {
4228	switch (a->sz) {
4229	case MO_8:
4230	tcg_gen_ext8u_i64(tcg_rt, tcg_rt);
4231	break;
4232	case MO_16:
4233	tcg_gen_ext16u_i64(tcg_rt, tcg_rt);
4234	break;
4235	case MO_32:
4236	tcg_gen_ext32u_i64(tcg_rt, tcg_rt);
4237	break;
4238	case MO_64:
4239	break;
4240	default:
4241	g_assert_not_reached();
4242	}
4243	}
4244	return true;
4245	}
4246
4247	TRANS_FEAT(LDADD, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_fetch_add_i64, 0, false)
4248	TRANS_FEAT(LDCLR, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_fetch_and_i64, 0, true)
4249	TRANS_FEAT(LDEOR, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_fetch_xor_i64, 0, false)
4250	TRANS_FEAT(LDSET, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_fetch_or_i64, 0, false)
4251	TRANS_FEAT(LDSMAX, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_fetch_smax_i64, MO_SIGN, false)
4252	TRANS_FEAT(LDSMIN, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_fetch_smin_i64, MO_SIGN, false)
4253	TRANS_FEAT(LDUMAX, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_fetch_umax_i64, 0, false)
4254	TRANS_FEAT(LDUMIN, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_fetch_umin_i64, 0, false)
4255	TRANS_FEAT(SWP, aa64_lse, do_atomic_ld, a, tcg_gen_atomic_xchg_i64, 0, false)
4256
4257	typedef void Atomic128ThreeOpFn(TCGv_i128, TCGv_i64, TCGv_i128, TCGArg, MemOp);
4258
4259	static bool do_atomic128_ld(DisasContext s, arg_atomic128 a,
4260	Atomic128ThreeOpFn *fn, bool invert)
4261	{
4262	MemOp mop;
4263	int rlo, rhi;
4264	TCGv_i64 clean_addr, tlo, thi;
4265	TCGv_i128 t16;
4266
4267	if (a->rt == 31 \|\| a->rt2 == 31 \|\| a->rt == a->rt2) {
4268	return false;
4269	}
4270	if (a->rn == 31) {
4271	gen_check_sp_alignment(s);
4272	}
4273	mop = check_atomic_align(s, a->rn, MO_128);
4274	clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), false,
4275	a->rn != 31, mop);
4276
4277	rlo = (s->be_data == MO_LE ? a->rt : a->rt2);
4278	rhi = (s->be_data == MO_LE ? a->rt2 : a->rt);
4279
4280	tlo = read_cpu_reg(s, rlo, true);
4281	thi = read_cpu_reg(s, rhi, true);
4282	if (invert) {
4283	tcg_gen_not_i64(tlo, tlo);
4284	tcg_gen_not_i64(thi, thi);
4285	}
4286	/*
4287	* The tcg atomic primitives are all full barriers. Therefore we
4288	* can ignore the Acquire and Release bits of this instruction.
4289	*/
4290	t16 = tcg_temp_new_i128();
4291	tcg_gen_concat_i64_i128(t16, tlo, thi);
4292
4293	fn(t16, clean_addr, t16, get_mem_index(s), mop);
4294
4295	tcg_gen_extr_i128_i64(cpu_reg(s, rlo), cpu_reg(s, rhi), t16);
4296	return true;
4297	}
4298
4299	TRANS_FEAT(LDCLRP, aa64_lse128, do_atomic128_ld,
4300	a, tcg_gen_atomic_fetch_and_i128, true)
4301	TRANS_FEAT(LDSETP, aa64_lse128, do_atomic128_ld,
4302	a, tcg_gen_atomic_fetch_or_i128, false)
4303	TRANS_FEAT(SWPP, aa64_lse128, do_atomic128_ld,
4304	a, tcg_gen_atomic_xchg_i128, false)
4305
4306	static bool trans_LDAPR(DisasContext s, arg_LDAPR a)
4307	{
4308	bool iss_sf = ldst_iss_sf(a->sz, false, false);
4309	TCGv_i64 clean_addr;
4310	MemOp mop;
4311
4312	if (!dc_isar_feature(aa64_lse, s) \|\|
4313	!dc_isar_feature(aa64_rcpc_8_3, s)) {
4314	return false;
4315	}
4316	if (a->rn == 31) {
4317	gen_check_sp_alignment(s);
4318	}
4319	mop = check_ordered_align(s, a->rn, 0, false, a->sz);
4320	clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), false,
4321	a->rn != 31, mop);
4322	/*
4323	* LDAPR* are a special case because they are a simple load, not a
4324	* fetch-and-do-something op.
4325	* The architectural consistency requirements here are weaker than
4326	* full load-acquire (we only need "load-acquire processor consistent"),
4327	* but we choose to implement them as full LDAQ.
4328	*/
4329	do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, mop, false,
4330	true, a->rt, iss_sf, true);
4331	tcg_gen_mb(TCG_MO_ALL \| TCG_BAR_LDAQ);
4332	return true;
4333	}
4334
4335	static bool trans_LDRA(DisasContext s, arg_LDRA a)
4336	{
4337	TCGv_i64 clean_addr, dirty_addr, tcg_rt;
4338	MemOp memop;
4339
4340	/* Load with pointer authentication */
4341	if (!dc_isar_feature(aa64_pauth, s)) {
4342	return false;
4343	}
4344
4345	if (a->rn == 31) {
4346	gen_check_sp_alignment(s);
4347	}
4348	dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
4349
4350	if (s->pauth_active) {
4351	if (!a->m) {
4352	gen_helper_autda_combined(dirty_addr, tcg_env, dirty_addr,
4353	tcg_constant_i64(0));
4354	} else {
4355	gen_helper_autdb_combined(dirty_addr, tcg_env, dirty_addr,
4356	tcg_constant_i64(0));
4357	}
4358	}
4359
4360	tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm);
4361
4362	memop = finalize_memop(s, MO_64);
4363
4364	/* Note that "clean" and "dirty" here refer to TBI not PAC. */
4365	clean_addr = gen_mte_check1(s, dirty_addr, false,
4366	a->w \|\| a->rn != 31, memop);
4367
4368	tcg_rt = cpu_reg(s, a->rt);
4369	do_gpr_ld(s, tcg_rt, clean_addr, memop,
4370	/* extend / false, / iss_valid */ !a->w,
4371	/* iss_srt / a->rt, / iss_sf / true, / iss_ar */ false);
4372
4373	if (a->w) {
4374	tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr);
4375	}
4376	return true;
4377	}
4378
4379	static bool trans_LDAPR_i(DisasContext s, arg_ldapr_stlr_i a)
4380	{
4381	TCGv_i64 clean_addr, dirty_addr;
4382	MemOp mop = a->sz \| (a->sign ? MO_SIGN : 0);
4383	bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
4384
4385	if (!dc_isar_feature(aa64_rcpc_8_4, s)) {
4386	return false;
4387	}
4388
4389	if (a->rn == 31) {
4390	gen_check_sp_alignment(s);
4391	}
4392
4393	mop = check_ordered_align(s, a->rn, a->imm, false, mop);
4394	dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
4395	tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm);
4396	clean_addr = clean_data_tbi(s, dirty_addr);
4397
4398	/*
4399	* Load-AcquirePC semantics; we implement as the slightly more
4400	* restrictive Load-Acquire.
4401	*/
4402	do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, mop, a->ext, true,
4403	a->rt, iss_sf, true);
4404	tcg_gen_mb(TCG_MO_ALL \| TCG_BAR_LDAQ);
4405	return true;
4406	}
4407
4408	static bool trans_STLR_i(DisasContext s, arg_ldapr_stlr_i a)
4409	{
4410	TCGv_i64 clean_addr, dirty_addr;
4411	MemOp mop = a->sz;
4412	bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
4413
4414	if (!dc_isar_feature(aa64_rcpc_8_4, s)) {
4415	return false;
4416	}
4417
4418	/* TODO: ARMv8.4-LSE SCTLR.nAA */
4419
4420	if (a->rn == 31) {
4421	gen_check_sp_alignment(s);
4422	}
4423
4424	mop = check_ordered_align(s, a->rn, a->imm, true, mop);
4425	dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
4426	tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm);
4427	clean_addr = clean_data_tbi(s, dirty_addr);
4428
4429	/* Store-Release semantics */
4430	tcg_gen_mb(TCG_MO_ALL \| TCG_BAR_STRL);
4431	do_gpr_st(s, cpu_reg(s, a->rt), clean_addr, mop, true, a->rt, iss_sf, true);
4432	return true;
4433	}
4434
4435	static bool trans_GCSSTR(DisasContext s, arg_GCSSTR a)
4436	{
4437	ARMMMUIdx armidx;
4438
4439	if (!dc_isar_feature(aa64_gcs, s)) {
4440	return false;
4441	}
4442
4443	/*
4444	* The pseudocode for GCSSTTR is
4445	*
4446	* effective_el = AArch64.IsUnprivAccessPriv() ? PSTATE.EL : EL0;
4447	* if (effective_el == PSTATE.EL) CheckGCSSTREnabled();
4448	*
4449	* We have cached the result of IsUnprivAccessPriv in DisasContext,
4450	* but since we need the result of full_a64_user_mem_index anyway,
4451	* use the mmu_idx test as a proxy for the effective_el test.
4452	*/
4453	armidx = full_a64_user_mem_index(s, a->unpriv);
4454	if (armidx == s->mmu_idx && s->gcsstr_el != 0) {
4455	gen_exception_insn_el(s, 0, EXCP_UDEF,
4456	syn_gcs_gcsstr(a->rn, a->rt),
4457	s->gcsstr_el);
4458	return true;
4459	}
4460
4461	if (a->rn == 31) {
4462	gen_check_sp_alignment(s);
4463	}
4464	tcg_gen_qemu_st_i64(cpu_reg(s, a->rt),
4465	clean_data_tbi(s, cpu_reg_sp(s, a->rn)),
4466	core_gcs_mem_index(armidx),
4467	finalize_memop(s, MO_64 \| MO_ALIGN));
4468	return true;
4469	}
4470
4471	static bool trans_LD_mult(DisasContext s, arg_ldst_mult a)
4472	{
4473	TCGv_i64 clean_addr, tcg_rn, tcg_ebytes;
4474	MemOp endian, align, mop;
4475
4476	int total; /* total bytes */
4477	int elements; /* elements per vector */
4478	int r;
4479	int size = a->sz;
4480
4481	if (!a->p && a->rm != 0) {
4482	/* For non-postindexed accesses the Rm field must be 0 */
4483	return false;
4484	}
4485	if (size == 3 && !a->q && a->selem != 1) {
4486	return false;
4487	}
4488	if (!fp_access_check(s)) {
4489	return true;
4490	}
4491
4492	if (a->rn == 31) {
4493	gen_check_sp_alignment(s);
4494	}
4495
4496	/* For our purposes, bytes are always little-endian. */
4497	endian = s->be_data;
4498	if (size == 0) {
4499	endian = MO_LE;
4500	}
4501
4502	total = a->rpt * a->selem * (a->q ? 16 : 8);
4503	tcg_rn = cpu_reg_sp(s, a->rn);
4504
4505	/*
4506	* Issue the MTE check vs the logical repeat count, before we
4507	* promote consecutive little-endian elements below.
4508	*/
4509	clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p \|\| a->rn != 31, total,
4510	finalize_memop_asimd(s, size));
4511
4512	/*
4513	* Consecutive little-endian elements from a single register
4514	* can be promoted to a larger little-endian operation.
4515	*/
4516	align = MO_ALIGN;
4517	if (a->selem == 1 && endian == MO_LE) {
4518	align = pow2_align(size);
4519	size = 3;
4520	}
4521	if (!s->align_mem) {
4522	align = 0;
4523	}
4524	mop = endian \| size \| align;
4525
4526	elements = (a->q ? 16 : 8) >> size;
4527	tcg_ebytes = tcg_constant_i64(1 << size);
4528	for (r = 0; r < a->rpt; r++) {
4529	int e;
4530	for (e = 0; e < elements; e++) {
4531	int xs;
4532	for (xs = 0; xs < a->selem; xs++) {
4533	int tt = (a->rt + r + xs) % 32;
4534	do_vec_ld(s, tt, e, clean_addr, mop);
4535	tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes);
4536	}
4537	}
4538	}
4539
4540	/*
4541	* For non-quad operations, setting a slice of the low 64 bits of
4542	* the register clears the high 64 bits (in the ARM ARM pseudocode
4543	* this is implicit in the fact that 'rval' is a 64 bit wide
4544	* variable). For quad operations, we might still need to zero
4545	* the high bits of SVE.
4546	*/
4547	for (r = 0; r < a->rpt * a->selem; r++) {
4548	int tt = (a->rt + r) % 32;
4549	clear_vec_high(s, a->q, tt);
4550	}
4551
4552	if (a->p) {
4553	if (a->rm == 31) {
4554	tcg_gen_addi_i64(tcg_rn, tcg_rn, total);
4555	} else {
4556	tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm));
4557	}
4558	}
4559	return true;
4560	}
4561
4562	static bool trans_ST_mult(DisasContext s, arg_ldst_mult a)
4563	{
4564	TCGv_i64 clean_addr, tcg_rn, tcg_ebytes;
4565	MemOp endian, align, mop;
4566
4567	int total; /* total bytes */
4568	int elements; /* elements per vector */
4569	int r;
4570	int size = a->sz;
4571
4572	if (!a->p && a->rm != 0) {
4573	/* For non-postindexed accesses the Rm field must be 0 */
4574	return false;
4575	}
4576	if (size == 3 && !a->q && a->selem != 1) {
4577	return false;
4578	}
4579	if (!fp_access_check(s)) {
4580	return true;
4581	}
4582
4583	if (a->rn == 31) {
4584	gen_check_sp_alignment(s);
4585	}
4586
4587	/* For our purposes, bytes are always little-endian. */
4588	endian = s->be_data;
4589	if (size == 0) {
4590	endian = MO_LE;
4591	}
4592
4593	total = a->rpt * a->selem * (a->q ? 16 : 8);
4594	tcg_rn = cpu_reg_sp(s, a->rn);
4595
4596	/*
4597	* Issue the MTE check vs the logical repeat count, before we
4598	* promote consecutive little-endian elements below.
4599	*/
4600	clean_addr = gen_mte_checkN(s, tcg_rn, true, a->p \|\| a->rn != 31, total,
4601	finalize_memop_asimd(s, size));
4602
4603	/*
4604	* Consecutive little-endian elements from a single register
4605	* can be promoted to a larger little-endian operation.
4606	*/
4607	align = MO_ALIGN;
4608	if (a->selem == 1 && endian == MO_LE) {
4609	align = pow2_align(size);
4610	size = 3;
4611	}
4612	if (!s->align_mem) {
4613	align = 0;
4614	}
4615	mop = endian \| size \| align;
4616
4617	elements = (a->q ? 16 : 8) >> size;
4618	tcg_ebytes = tcg_constant_i64(1 << size);
4619	for (r = 0; r < a->rpt; r++) {
4620	int e;
4621	for (e = 0; e < elements; e++) {
4622	int xs;
4623	for (xs = 0; xs < a->selem; xs++) {
4624	int tt = (a->rt + r + xs) % 32;
4625	do_vec_st(s, tt, e, clean_addr, mop);
4626	tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes);
4627	}
4628	}
4629	}
4630
4631	if (a->p) {
4632	if (a->rm == 31) {
4633	tcg_gen_addi_i64(tcg_rn, tcg_rn, total);
4634	} else {
4635	tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm));
4636	}
4637	}
4638	return true;
4639	}
4640
4641	static bool trans_ST_single(DisasContext s, arg_ldst_single a)
4642	{
4643	int xs, total, rt;
4644	TCGv_i64 clean_addr, tcg_rn, tcg_ebytes;
4645	MemOp mop;
4646
4647	if (!a->p && a->rm != 0) {
4648	return false;
4649	}
4650	if (!fp_access_check(s)) {
4651	return true;
4652	}
4653
4654	if (a->rn == 31) {
4655	gen_check_sp_alignment(s);
4656	}
4657
4658	total = a->selem << a->scale;
4659	tcg_rn = cpu_reg_sp(s, a->rn);
4660
4661	mop = finalize_memop_asimd(s, a->scale);
4662	clean_addr = gen_mte_checkN(s, tcg_rn, true, a->p \|\| a->rn != 31,
4663	total, mop);
4664
4665	tcg_ebytes = tcg_constant_i64(1 << a->scale);
4666	for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) {
4667	do_vec_st(s, rt, a->index, clean_addr, mop);
4668	tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes);
4669	}
4670
4671	if (a->p) {
4672	if (a->rm == 31) {
4673	tcg_gen_addi_i64(tcg_rn, tcg_rn, total);
4674	} else {
4675	tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm));
4676	}
4677	}
4678	return true;
4679	}
4680
4681	static bool trans_LD_single(DisasContext s, arg_ldst_single a)
4682	{
4683	int xs, total, rt;
4684	TCGv_i64 clean_addr, tcg_rn, tcg_ebytes;
4685	MemOp mop;
4686
4687	if (!a->p && a->rm != 0) {
4688	return false;
4689	}
4690	if (!fp_access_check(s)) {
4691	return true;
4692	}
4693
4694	if (a->rn == 31) {
4695	gen_check_sp_alignment(s);
4696	}
4697
4698	total = a->selem << a->scale;
4699	tcg_rn = cpu_reg_sp(s, a->rn);
4700
4701	mop = finalize_memop_asimd(s, a->scale);
4702	clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p \|\| a->rn != 31,
4703	total, mop);
4704
4705	tcg_ebytes = tcg_constant_i64(1 << a->scale);
4706	for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) {
4707	do_vec_ld(s, rt, a->index, clean_addr, mop);
4708	tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes);
4709	}
4710
4711	if (a->p) {
4712	if (a->rm == 31) {
4713	tcg_gen_addi_i64(tcg_rn, tcg_rn, total);
4714	} else {
4715	tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm));
4716	}
4717	}
4718	return true;
4719	}
4720
4721	static bool trans_LD_single_repl(DisasContext s, arg_LD_single_repl a)
4722	{
4723	int xs, total, rt;
4724	TCGv_i64 clean_addr, tcg_rn, tcg_ebytes;
4725	MemOp mop;
4726
4727	if (!a->p && a->rm != 0) {
4728	return false;
4729	}
4730	if (!fp_access_check(s)) {
4731	return true;
4732	}
4733
4734	if (a->rn == 31) {
4735	gen_check_sp_alignment(s);
4736	}
4737
4738	total = a->selem << a->scale;
4739	tcg_rn = cpu_reg_sp(s, a->rn);
4740
4741	mop = finalize_memop_asimd(s, a->scale);
4742	clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p \|\| a->rn != 31,
4743	total, mop);
4744
4745	tcg_ebytes = tcg_constant_i64(1 << a->scale);
4746	for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) {
4747	/* Load and replicate to all elements */
4748	TCGv_i64 tcg_tmp = tcg_temp_new_i64();
4749
4750	tcg_gen_qemu_ld_i64(tcg_tmp, clean_addr, get_mem_index(s), mop);
4751	tcg_gen_gvec_dup_i64(a->scale, vec_full_reg_offset(s, rt),
4752	(a->q + 1) * 8, vec_full_reg_size(s), tcg_tmp);
4753	tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes);
4754	}
4755
4756	if (a->p) {
4757	if (a->rm == 31) {
4758	tcg_gen_addi_i64(tcg_rn, tcg_rn, total);
4759	} else {
4760	tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm));
4761	}
4762	}
4763	return true;
4764	}
4765
4766	static bool trans_STZGM(DisasContext s, arg_ldst_tag a)
4767	{
4768	TCGv_i64 addr, clean_addr, tcg_rt;
4769	int size = 4 << s->dcz_blocksize;
4770
4771	if (!dc_isar_feature(aa64_mte, s)) {
4772	return false;
4773	}
4774	if (s->current_el == 0) {
4775	return false;
4776	}
4777
4778	if (a->rn == 31) {
4779	gen_check_sp_alignment(s);
4780	}
4781
4782	addr = read_cpu_reg_sp(s, a->rn, true);
4783	tcg_gen_addi_i64(addr, addr, a->imm);
4784	tcg_rt = cpu_reg(s, a->rt);
4785
4786	if (s->ata[0]) {
4787	gen_helper_stzgm_tags(tcg_env, addr, tcg_rt, tcg_constant_i32(s->mtx));
4788	}
4789	/*
4790	* The non-tags portion of STZGM is mostly like DC_ZVA,
4791	* except the alignment happens before the access.
4792	*/
4793	clean_addr = clean_data_tbi(s, addr);
4794	tcg_gen_andi_i64(clean_addr, clean_addr, -size);
4795	gen_helper_dc_zva(tcg_env, clean_addr);
4796	return true;
4797	}
4798
4799	static bool trans_STGM(DisasContext s, arg_ldst_tag a)
4800	{
4801	TCGv_i64 addr, clean_addr, tcg_rt;
4802
4803	if (!dc_isar_feature(aa64_mte, s)) {
4804	return false;
4805	}
4806	if (s->current_el == 0) {
4807	return false;
4808	}
4809
4810	if (a->rn == 31) {
4811	gen_check_sp_alignment(s);
4812	}
4813
4814	addr = read_cpu_reg_sp(s, a->rn, true);
4815	tcg_gen_addi_i64(addr, addr, a->imm);
4816	tcg_rt = cpu_reg(s, a->rt);
4817
4818	if (s->ata[0]) {
4819	gen_helper_stgm(tcg_env, addr, tcg_rt, tcg_constant_i32(s->mtx));
4820	} else {
4821	MMUAccessType acc = MMU_DATA_STORE;
4822	int size = 4 << s->gm_blocksize;
4823
4824	clean_addr = clean_data_tbi(s, addr);
4825	tcg_gen_andi_i64(clean_addr, clean_addr, -size);
4826	gen_probe_access(s, clean_addr, acc, size);
4827	}
4828	return true;
4829	}
4830
4831	static bool trans_LDGM(DisasContext s, arg_ldst_tag a)
4832	{
4833	TCGv_i64 addr, clean_addr, tcg_rt;
4834
4835	if (!dc_isar_feature(aa64_mte, s)) {
4836	return false;
4837	}
4838	if (s->current_el == 0) {
4839	return false;
4840	}
4841
4842	if (a->rn == 31) {
4843	gen_check_sp_alignment(s);
4844	}
4845
4846	addr = read_cpu_reg_sp(s, a->rn, true);
4847	tcg_gen_addi_i64(addr, addr, a->imm);
4848	tcg_rt = cpu_reg(s, a->rt);
4849
4850	if (s->ata[0]) {
4851	gen_helper_ldgm(tcg_rt, tcg_env, addr, tcg_constant_i32(s->mtx));
4852	} else {
4853	MMUAccessType acc = MMU_DATA_LOAD;
4854	int size = 4 << s->gm_blocksize;
4855
4856	clean_addr = clean_data_tbi(s, addr);
4857	tcg_gen_andi_i64(clean_addr, clean_addr, -size);
4858	gen_probe_access(s, clean_addr, acc, size);
4859	/* The result tags are zeros. */
4860	tcg_gen_movi_i64(tcg_rt, 0);
4861	}
4862	return true;
4863	}
4864
4865	static bool trans_LDG(DisasContext s, arg_ldst_tag a)
4866	{
4867	TCGv_i64 addr, clean_addr, tcg_rt;
4868
4869	if (!dc_isar_feature(aa64_mte_insn_reg, s)) {
4870	return false;
4871	}
4872
4873	if (a->rn == 31) {
4874	gen_check_sp_alignment(s);
4875	}
4876
4877	addr = read_cpu_reg_sp(s, a->rn, true);
4878	if (!a->p) {
4879	/* pre-index or signed offset */
4880	tcg_gen_addi_i64(addr, addr, a->imm);
4881	}
4882
4883	tcg_gen_andi_i64(addr, addr, -TAG_GRANULE);
4884	tcg_rt = cpu_reg(s, a->rt);
4885	if (s->ata[0]) {
4886	gen_helper_ldg(tcg_rt, tcg_env, addr, tcg_rt, tcg_constant_i32(s->mtx));
4887	} else {
4888	/*
4889	* Tag access disabled: we must check for aborts on the load
4890	* load from [rn+offset], and then insert a 0 tag into rt.
4891	*/
4892	clean_addr = clean_data_tbi(s, addr);
4893	gen_probe_access(s, clean_addr, MMU_DATA_LOAD, MO_8);
4894	gen_address_with_allocation_tag0(tcg_rt, tcg_rt);
4895	}
4896
4897	if (a->w) {
4898	/* pre-index or post-index */
4899	if (a->p) {
4900	/* post-index */
4901	tcg_gen_addi_i64(addr, addr, a->imm);
4902	}
4903	tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), addr);
4904	}
4905	return true;
4906	}
4907
4908	static bool do_STG(DisasContext s, arg_ldst_tag a, bool is_zero, bool is_pair)
4909	{
4910	TCGv_i64 addr, tcg_rt;
4911
4912	if (a->rn == 31) {
4913	gen_check_sp_alignment(s);
4914	}
4915
4916	addr = read_cpu_reg_sp(s, a->rn, true);
4917	if (!a->p) {
4918	/* pre-index or signed offset */
4919	tcg_gen_addi_i64(addr, addr, a->imm);
4920	}
4921	tcg_rt = cpu_reg_sp(s, a->rt);
4922	if (!s->ata[0]) {
4923	/*
4924	* For STG and ST2G, we need to check alignment and probe memory.
4925	* TODO: For STZG and STZ2G, we could rely on the stores below,
4926	* at least for system mode; user-only won't enforce alignment.
4927	*/
4928	if (is_pair) {
4929	gen_helper_st2g_stub(tcg_env, addr);
4930	} else {
4931	gen_helper_stg_stub(tcg_env, addr);
4932	}
4933	} else if (tb_cflags(s->base.tb) & CF_PARALLEL) {
4934	if (is_pair) {
4935	gen_helper_st2g_parallel(tcg_env, addr, tcg_rt,
4936	tcg_constant_i32(s->mtx));
4937	} else {
4938	gen_helper_stg_parallel(tcg_env, addr, tcg_rt,
4939	tcg_constant_i32(s->mtx));
4940	}
4941	} else {
4942	if (is_pair) {
4943	gen_helper_st2g(tcg_env, addr, tcg_rt, tcg_constant_i32(s->mtx));
4944	} else {
4945	gen_helper_stg(tcg_env, addr, tcg_rt, tcg_constant_i32(s->mtx));
4946	}
4947	}
4948
4949	if (is_zero) {
4950	TCGv_i64 clean_addr = clean_data_tbi(s, addr);
4951	TCGv_i128 zero128 = tcg_zero_i128();
4952	int mem_index = get_mem_index(s);
4953	MemOp mop = finalize_memop(s, MO_128 \| MO_ALIGN);
4954
4955	/* This is 1 or 2 atomic 16-byte operations. */
4956	tcg_gen_qemu_st_i128(zero128, clean_addr, mem_index, mop);
4957	if (is_pair) {
4958	tcg_gen_addi_i64(clean_addr, clean_addr, 16);
4959	tcg_gen_qemu_st_i128(zero128, clean_addr, mem_index, mop);
4960	}
4961	}
4962
4963	if (a->w) {
4964	/* pre-index or post-index */
4965	if (a->p) {
4966	/* post-index */
4967	tcg_gen_addi_i64(addr, addr, a->imm);
4968	}
4969	tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), addr);
4970	}
4971	return true;
4972	}
4973
4974	TRANS_FEAT(STG, aa64_mte_insn_reg, do_STG, a, false, false)
4975	TRANS_FEAT(STZG, aa64_mte_insn_reg, do_STG, a, true, false)
4976	TRANS_FEAT(ST2G, aa64_mte_insn_reg, do_STG, a, false, true)
4977	TRANS_FEAT(STZ2G, aa64_mte_insn_reg, do_STG, a, true, true)
4978
4979	typedef void SetFn(TCGv_env, TCGv_i32, TCGv_i32);
4980
4981	static bool do_SET(DisasContext s, arg_set a, bool is_epilogue,
4982	bool is_setg, SetFn fn)
4983	{
4984	int memidx;
4985	uint32_t syndrome, desc = 0;
4986
4987	if (is_setg && !dc_isar_feature(aa64_mte, s)) {
4988	return false;
4989	}
4990
4991	/*
4992	* UNPREDICTABLE cases: we choose to UNDEF, which allows
4993	* us to pull this check before the CheckMOPSEnabled() test
4994	* (which we do in the helper function)
4995	*/
4996	if (a->rs == a->rn \|\| a->rs == a->rd \|\| a->rn == a->rd \|\|
4997	a->rd == 31 \|\| a->rn == 31) {
4998	return false;
4999	}
5000

Showing first 5,000 of 11,314 lines. View raw