cputlb.c · samitouri/QOSamiQemu · siGit Code & Deploy

Code Commits Issues Pulls Sessions CI/CD

master

c 2,902 lines 91 KB

Raw

1	/*
2	* Common CPU TLB handling (system emulation)
3	*
4	* Copyright (c) 2003 Fabrice Bellard
5	*
6	* This library is free software; you can redistribute it and/or
7	* modify it under the terms of the GNU Lesser General Public
8	* License as published by the Free Software Foundation; either
9	* version 2.1 of the License, or (at your option) any later version.
10	*
11	* This library is distributed in the hope that it will be useful,
12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14	* Lesser General Public License for more details.
15	*
16	* You should have received a copy of the GNU Lesser General Public
17	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
18	*/
19
20	#include "qemu/osdep.h"
21	#include "qemu/main-loop.h"
22	#include "qemu/target-info.h"
23	#include "accel/tcg/cpu-loop.h"
24	#include "accel/tcg/cpu-ops.h"
25	#include "accel/tcg/iommu.h"
26	#include "accel/tcg/probe.h"
27	#include "exec/page-protection.h"
28	#include "system/memory.h"
29	#include "system/physmem.h"
30	#include "accel/tcg/cpu-ldst-common.h"
31	#include "accel/tcg/cpu-mmu-index.h"
32	#include "exec/cputlb.h"
33	#include "exec/tb-flush.h"
34	#include "system/ramblock.h"
35	#include "exec/mmu-access-type.h"
36	#include "exec/tlb-common.h"
37	#include "exec/vaddr.h"
38	#include "tcg/tcg.h"
39	#include "qemu/error-report.h"
40	#include "exec/log.h"
41	#include "exec/helper-proto-common.h"
42	#include "exec/tlb-flags.h"
43	#include "qemu/atomic.h"
44	#include "qemu/atomic128.h"
45	#include "tb-internal.h"
46	#include "trace.h"
47	#include "tb-hash.h"
48	#include "tlb-bounds.h"
49	#include "internal-common.h"
50	#include "system-page-protection.h"
51	#ifdef CONFIG_PLUGIN
52	#include "qemu/plugin-memory.h"
53	#endif
54	#include "tcg/tcg-ldst.h"
55	#include "backend-ldst.h"
56
57
58	/* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */
59	/* #define DEBUG_TLB */
60	/* #define DEBUG_TLB_LOG */
61
62	#ifdef DEBUG_TLB
63	# define DEBUG_TLB_GATE 1
64	# ifdef DEBUG_TLB_LOG
65	# define DEBUG_TLB_LOG_GATE 1
66	# else
67	# define DEBUG_TLB_LOG_GATE 0
68	# endif
69	#else
70	# define DEBUG_TLB_GATE 0
71	# define DEBUG_TLB_LOG_GATE 0
72	#endif
73
74	#define tlb_debug(fmt, ...) do { \
75	if (DEBUG_TLB_LOG_GATE) { \
76	qemu_log_mask(CPU_LOG_MMU, "%s: " fmt, __func__, \
77	## __VA_ARGS__); \
78	} else if (DEBUG_TLB_GATE) { \
79	fprintf(stderr, "%s: " fmt, __func__, ## __VA_ARGS__); \
80	} \
81	} while (0)
82
83	#define assert_cpu_is_self(cpu) do { \
84	if (DEBUG_TLB_GATE) { \
85	g_assert(!(cpu)->created \|\| qemu_cpu_is_self(cpu)); \
86	} \
87	} while (0)
88
89	/* run_on_cpu_data.target_ptr should always be big enough for a
90	* vaddr even on 32 bit builds
91	*/
92	QEMU_BUILD_BUG_ON(sizeof(vaddr) > sizeof(run_on_cpu_data));
93
94	#define ALL_MMUIDX_BITS ((1 << NB_MMU_MODES) - 1)
95
96	static inline size_t tlb_n_entries(CPUTLBDescFast *fast)
97	{
98	return (fast->mask >> CPU_TLB_ENTRY_BITS) + 1;
99	}
100
101	static inline size_t sizeof_tlb(CPUTLBDescFast *fast)
102	{
103	return fast->mask + (1 << CPU_TLB_ENTRY_BITS);
104	}
105
106	static inline uint64_t tlb_read_idx(const CPUTLBEntry *entry,
107	MMUAccessType access_type)
108	{
109	/* Do not rearrange the CPUTLBEntry structure members. */
110	QEMU_BUILD_BUG_ON(offsetof(CPUTLBEntry, addr_read) !=
111	MMU_DATA_LOAD * sizeof(uintptr_t));
112	QEMU_BUILD_BUG_ON(offsetof(CPUTLBEntry, addr_write) !=
113	MMU_DATA_STORE * sizeof(uintptr_t));
114	QEMU_BUILD_BUG_ON(offsetof(CPUTLBEntry, addr_code) !=
115	MMU_INST_FETCH * sizeof(uintptr_t));
116
117	const uintptr_t *ptr = &entry->addr_idx[access_type];
118	/* ofs might correspond to .addr_write, so use qatomic_read */
119	return qatomic_read(ptr);
120	}
121
122	static inline uint64_t tlb_addr_write(const CPUTLBEntry *entry)
123	{
124	return tlb_read_idx(entry, MMU_DATA_STORE);
125	}
126
127	/* Find the TLB index corresponding to the mmu_idx + address pair. */
128	static inline uintptr_t tlb_index(CPUState *cpu, uintptr_t mmu_idx,
129	vaddr addr)
130	{
131	uintptr_t size_mask = cpu_tlb_fast(cpu, mmu_idx)->mask >> CPU_TLB_ENTRY_BITS;
132
133	return (addr >> TARGET_PAGE_BITS) & size_mask;
134	}
135
136	/* Find the TLB entry corresponding to the mmu_idx + address pair. */
137	static inline CPUTLBEntry tlb_entry(CPUState cpu, uintptr_t mmu_idx,
138	vaddr addr)
139	{
140	return &cpu_tlb_fast(cpu, mmu_idx)->table[tlb_index(cpu, mmu_idx, addr)];
141	}
142
143	static void tlb_window_reset(CPUTLBDesc *desc, int64_t ns,
144	size_t max_entries)
145	{
146	desc->window_begin_ns = ns;
147	desc->window_max_entries = max_entries;
148	}
149
150	static void tb_jmp_cache_clear_page(CPUState *cpu, vaddr page_addr)
151	{
152	CPUJumpCache *jc = cpu->tb_jmp_cache;
153	int i, i0;
154
155	if (unlikely(!jc)) {
156	return;
157	}
158
159	i0 = tb_jmp_cache_hash_page(page_addr);
160	for (i = 0; i < TB_JMP_PAGE_SIZE; i++) {
161	qatomic_set(&jc->array[i0 + i].tb, NULL);
162	}
163	}
164
165	/**
166	* tlb_mmu_resize_locked() - perform TLB resize bookkeeping; resize if necessary
167	* @desc: The CPUTLBDesc portion of the TLB
168	* @fast: The CPUTLBDescFast portion of the same TLB
169	*
170	* Called with tlb_lock_held.
171	*
172	* We have two main constraints when resizing a TLB: (1) we only resize it
173	* on a TLB flush (otherwise we'd have to take a perf hit by either rehashing
174	* the array or unnecessarily flushing it), which means we do not control how
175	* frequently the resizing can occur; (2) we don't have access to the guest's
176	* future scheduling decisions, and therefore have to decide the magnitude of
177	* the resize based on past observations.
178	*
179	* In general, a memory-hungry process can benefit greatly from an appropriately
180	* sized TLB, since a guest TLB miss is very expensive. This doesn't mean that
181	* we just have to make the TLB as large as possible; while an oversized TLB
182	* results in minimal TLB miss rates, it also takes longer to be flushed
183	* (flushes can be _very_ frequent), and the reduced locality can also hurt
184	* performance.
185	*
186	* To achieve near-optimal performance for all kinds of workloads, we:
187	*
188	* 1. Aggressively increase the size of the TLB when the use rate of the
189	* TLB being flushed is high, since it is likely that in the near future this
190	* memory-hungry process will execute again, and its memory hungriness will
191	* probably be similar.
192	*
193	* 2. Slowly reduce the size of the TLB as the use rate declines over a
194	* reasonably large time window. The rationale is that if in such a time window
195	* we have not observed a high TLB use rate, it is likely that we won't observe
196	* it in the near future. In that case, once a time window expires we downsize
197	* the TLB to match the maximum use rate observed in the window.
198	*
199	* 3. Try to keep the maximum use rate in a time window in the 30-70% range,
200	* since in that range performance is likely near-optimal. Recall that the TLB
201	* is direct mapped, so we want the use rate to be low (or at least not too
202	* high), since otherwise we are likely to have a significant amount of
203	* conflict misses.
204	*/
205	static void tlb_mmu_resize_locked(CPUTLBDesc desc, CPUTLBDescFast fast,
206	int64_t now)
207	{
208	size_t old_size = tlb_n_entries(fast);
209	size_t rate;
210	size_t new_size = old_size;
211	int64_t window_len_ms = 100;
212	int64_t window_len_ns = window_len_ms * 1000 * 1000;
213	bool window_expired = now > desc->window_begin_ns + window_len_ns;
214
215	if (desc->n_used_entries > desc->window_max_entries) {
216	desc->window_max_entries = desc->n_used_entries;
217	}
218	rate = desc->window_max_entries * 100 / old_size;
219
220	if (rate > 70) {
221	new_size = MIN(old_size << 1, 1 << CPU_TLB_DYN_MAX_BITS);
222	} else if (rate < 30 && window_expired) {
223	size_t ceil = pow2ceil(desc->window_max_entries);
224	size_t expected_rate = desc->window_max_entries * 100 / ceil;
225
226	/*
227	* Avoid undersizing when the max number of entries seen is just below
228	* a pow2. For instance, if max_entries == 1025, the expected use rate
229	* would be 1025/2048==50%. However, if max_entries == 1023, we'd get
230	* 1023/1024==99.9% use rate, so we'd likely end up doubling the size
231	* later. Thus, make sure that the expected use rate remains below 70%.
232	* (and since we double the size, that means the lowest rate we'd
233	* expect to get is 35%, which is still in the 30-70% range where
234	* we consider that the size is appropriate.)
235	*/
236	if (expected_rate > 70) {
237	ceil *= 2;
238	}
239	new_size = MAX(ceil, 1 << CPU_TLB_DYN_MIN_BITS);
240	}
241
242	if (new_size == old_size) {
243	if (window_expired) {
244	tlb_window_reset(desc, now, desc->n_used_entries);
245	}
246	return;
247	}
248
249	g_free(fast->table);
250	g_free(desc->fulltlb);
251
252	tlb_window_reset(desc, now, 0);
253	/* desc->n_used_entries is cleared by the caller */
254	fast->mask = (new_size - 1) << CPU_TLB_ENTRY_BITS;
255	fast->table = g_try_new(CPUTLBEntry, new_size);
256	desc->fulltlb = g_try_new(CPUTLBEntryFull, new_size);
257
258	/*
259	* If the allocations fail, try smaller sizes. We just freed some
260	* memory, so going back to half of new_size has a good chance of working.
261	* Increased memory pressure elsewhere in the system might cause the
262	* allocations to fail though, so we progressively reduce the allocation
263	* size, aborting if we cannot even allocate the smallest TLB we support.
264	*/
265	while (fast->table == NULL \|\| desc->fulltlb == NULL) {
266	if (new_size == (1 << CPU_TLB_DYN_MIN_BITS)) {
267	error_report("%s: %s", __func__, strerror(errno));
268	abort();
269	}
270	new_size = MAX(new_size >> 1, 1 << CPU_TLB_DYN_MIN_BITS);
271	fast->mask = (new_size - 1) << CPU_TLB_ENTRY_BITS;
272
273	g_free(fast->table);
274	g_free(desc->fulltlb);
275	fast->table = g_try_new(CPUTLBEntry, new_size);
276	desc->fulltlb = g_try_new(CPUTLBEntryFull, new_size);
277	}
278	}
279
280	static void tlb_mmu_flush_locked(CPUTLBDesc desc, CPUTLBDescFast fast)
281	{
282	desc->n_used_entries = 0;
283	desc->large_page_addr = -1;
284	desc->large_page_mask = -1;
285	desc->vindex = 0;
286	memset(fast->table, -1, sizeof_tlb(fast));
287	memset(desc->vtable, -1, sizeof(desc->vtable));
288	}
289
290	static void tlb_flush_one_mmuidx_locked(CPUState *cpu, int mmu_idx,
291	int64_t now)
292	{
293	CPUTLBDesc *desc = &cpu->neg.tlb.d[mmu_idx];
294	CPUTLBDescFast *fast = cpu_tlb_fast(cpu, mmu_idx);
295
296	tlb_mmu_resize_locked(desc, fast, now);
297	tlb_mmu_flush_locked(desc, fast);
298	}
299
300	static void tlb_mmu_init(CPUTLBDesc desc, CPUTLBDescFast fast, int64_t now)
301	{
302	size_t n_entries = 1 << CPU_TLB_DYN_DEFAULT_BITS;
303
304	tlb_window_reset(desc, now, 0);
305	desc->n_used_entries = 0;
306	fast->mask = (n_entries - 1) << CPU_TLB_ENTRY_BITS;
307	fast->table = g_new(CPUTLBEntry, n_entries);
308	desc->fulltlb = g_new(CPUTLBEntryFull, n_entries);
309	tlb_mmu_flush_locked(desc, fast);
310	}
311
312	static inline void tlb_n_used_entries_inc(CPUState *cpu, uintptr_t mmu_idx)
313	{
314	cpu->neg.tlb.d[mmu_idx].n_used_entries++;
315	}
316
317	static inline void tlb_n_used_entries_dec(CPUState *cpu, uintptr_t mmu_idx)
318	{
319	cpu->neg.tlb.d[mmu_idx].n_used_entries--;
320	}
321
322	void tlb_init(CPUState *cpu)
323	{
324	int64_t now = get_clock_realtime();
325	int i;
326
327	qemu_spin_init(&cpu->neg.tlb.c.lock);
328
329	/* All tlbs are initialized flushed. */
330	cpu->neg.tlb.c.dirty = 0;
331
332	for (i = 0; i < NB_MMU_MODES; i++) {
333	tlb_mmu_init(&cpu->neg.tlb.d[i], cpu_tlb_fast(cpu, i), now);
334	}
335	}
336
337	void tlb_destroy(CPUState *cpu)
338	{
339	int i;
340
341	qemu_spin_destroy(&cpu->neg.tlb.c.lock);
342	for (i = 0; i < NB_MMU_MODES; i++) {
343	CPUTLBDesc *desc = &cpu->neg.tlb.d[i];
344	CPUTLBDescFast *fast = cpu_tlb_fast(cpu, i);
345
346	g_free(fast->table);
347	g_free(desc->fulltlb);
348	}
349	}
350
351	/* flush_all_helper: run fn across all cpus
352	*
353	* If the wait flag is set then the src cpu's helper will be queued as
354	* "safe" work and the loop exited creating a synchronisation point
355	* where all queued work will be finished before execution starts
356	* again.
357	*/
358	static void flush_all_helper(CPUState *src, run_on_cpu_func fn,
359	run_on_cpu_data d)
360	{
361	CPUState *cpu;
362
363	CPU_FOREACH(cpu) {
364	if (cpu != src) {
365	async_run_on_cpu(cpu, fn, d);
366	}
367	}
368	}
369
370	static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data)
371	{
372	MMUIdxMap asked = data.host_int;
373	MMUIdxMap all_dirty, work, to_clean;
374	int64_t now = get_clock_realtime();
375
376	assert_cpu_is_self(cpu);
377
378	tlb_debug("mmu_idx:0x%04" PRIx16 "\n", asked);
379
380	qemu_spin_lock(&cpu->neg.tlb.c.lock);
381
382	all_dirty = cpu->neg.tlb.c.dirty;
383	to_clean = asked & all_dirty;
384	all_dirty &= ~to_clean;
385	cpu->neg.tlb.c.dirty = all_dirty;
386
387	for (work = to_clean; work != 0; work &= work - 1) {
388	int mmu_idx = ctz32(work);
389	tlb_flush_one_mmuidx_locked(cpu, mmu_idx, now);
390	}
391
392	qemu_spin_unlock(&cpu->neg.tlb.c.lock);
393
394	tcg_flush_jmp_cache(cpu);
395
396	if (to_clean == ALL_MMUIDX_BITS) {
397	qatomic_set(&cpu->neg.tlb.c.full_flush_count,
398	cpu->neg.tlb.c.full_flush_count + 1);
399	} else {
400	qatomic_set(&cpu->neg.tlb.c.part_flush_count,
401	cpu->neg.tlb.c.part_flush_count + ctpop16(to_clean));
402	if (to_clean != asked) {
403	qatomic_set(&cpu->neg.tlb.c.elide_flush_count,
404	cpu->neg.tlb.c.elide_flush_count +
405	ctpop16(asked & ~to_clean));
406	}
407	}
408	}
409
410	void tlb_flush_by_mmuidx(CPUState *cpu, MMUIdxMap idxmap)
411	{
412	tlb_debug("mmu_idx: 0x%" PRIx16 "\n", idxmap);
413
414	assert_cpu_is_self(cpu);
415
416	tlb_flush_by_mmuidx_async_work(cpu, RUN_ON_CPU_HOST_INT(idxmap));
417	}
418
419	void tlb_flush(CPUState *cpu)
420	{
421	tlb_flush_by_mmuidx(cpu, ALL_MMUIDX_BITS);
422	}
423
424	void tlb_flush_by_mmuidx_all_cpus_synced(CPUState *src_cpu, MMUIdxMap idxmap)
425	{
426	const run_on_cpu_func fn = tlb_flush_by_mmuidx_async_work;
427
428	tlb_debug("mmu_idx: 0x%"PRIx16"\n", idxmap);
429
430	flush_all_helper(src_cpu, fn, RUN_ON_CPU_HOST_INT(idxmap));
431	async_safe_run_on_cpu(src_cpu, fn, RUN_ON_CPU_HOST_INT(idxmap));
432	}
433
434	void tlb_flush_all_cpus_synced(CPUState *src_cpu)
435	{
436	tlb_flush_by_mmuidx_all_cpus_synced(src_cpu, ALL_MMUIDX_BITS);
437	}
438
439	static bool tlb_hit_page_mask_anyprot(CPUTLBEntry *tlb_entry,
440	vaddr page, vaddr mask)
441	{
442	page &= mask;
443	mask &= TARGET_PAGE_MASK \| TLB_INVALID_MASK;
444
445	return (page == (tlb_entry->addr_read & mask) \|\|
446	page == (tlb_addr_write(tlb_entry) & mask) \|\|
447	page == (tlb_entry->addr_code & mask));
448	}
449
450	static inline bool tlb_hit_page_anyprot(CPUTLBEntry *tlb_entry, vaddr page)
451	{
452	return tlb_hit_page_mask_anyprot(tlb_entry, page, -1);
453	}
454
455	/**
456	* tlb_entry_is_empty - return true if the entry is not in use
457	* @te: pointer to CPUTLBEntry
458	*/
459	static inline bool tlb_entry_is_empty(const CPUTLBEntry *te)
460	{
461	return te->addr_read == -1 && te->addr_write == -1 && te->addr_code == -1;
462	}
463
464	/* Called with tlb_c.lock held */
465	static bool tlb_flush_entry_mask_locked(CPUTLBEntry *tlb_entry,
466	vaddr page,
467	vaddr mask)
468	{
469	if (tlb_hit_page_mask_anyprot(tlb_entry, page, mask)) {
470	memset(tlb_entry, -1, sizeof(*tlb_entry));
471	return true;
472	}
473	return false;
474	}
475
476	static inline bool tlb_flush_entry_locked(CPUTLBEntry *tlb_entry, vaddr page)
477	{
478	return tlb_flush_entry_mask_locked(tlb_entry, page, -1);
479	}
480
481	/* Called with tlb_c.lock held */
482	static void tlb_flush_vtlb_page_mask_locked(CPUState *cpu, int mmu_idx,
483	vaddr page,
484	vaddr mask)
485	{
486	CPUTLBDesc *d = &cpu->neg.tlb.d[mmu_idx];
487	int k;
488
489	assert_cpu_is_self(cpu);
490	for (k = 0; k < CPU_VTLB_SIZE; k++) {
491	if (tlb_flush_entry_mask_locked(&d->vtable[k], page, mask)) {
492	tlb_n_used_entries_dec(cpu, mmu_idx);
493	}
494	}
495	}
496
497	static inline void tlb_flush_vtlb_page_locked(CPUState *cpu, int mmu_idx,
498	vaddr page)
499	{
500	tlb_flush_vtlb_page_mask_locked(cpu, mmu_idx, page, -1);
501	}
502
503	static void tlb_flush_page_locked(CPUState *cpu, int midx, vaddr page)
504	{
505	vaddr lp_addr = cpu->neg.tlb.d[midx].large_page_addr;
506	vaddr lp_mask = cpu->neg.tlb.d[midx].large_page_mask;
507
508	/* Check if we need to flush due to large pages. */
509	if ((page & lp_mask) == lp_addr) {
510	tlb_debug("forcing full flush midx %d (%016"
511	VADDR_PRIx "/%016" VADDR_PRIx ")\n",
512	midx, lp_addr, lp_mask);
513	tlb_flush_one_mmuidx_locked(cpu, midx, get_clock_realtime());
514	} else {
515	if (tlb_flush_entry_locked(tlb_entry(cpu, midx, page), page)) {
516	tlb_n_used_entries_dec(cpu, midx);
517	}
518	tlb_flush_vtlb_page_locked(cpu, midx, page);
519	}
520	}
521
522	/**
523	* tlb_flush_page_by_mmuidx_async_0:
524	* @cpu: cpu on which to flush
525	* @addr: page of virtual address to flush
526	* @idxmap: set of mmu_idx to flush
527	*
528	* Helper for tlb_flush_page_by_mmuidx and friends, flush one page
529	* at @addr from the tlbs indicated by @idxmap from @cpu.
530	*/
531	static void tlb_flush_page_by_mmuidx_async_0(CPUState *cpu,
532	vaddr addr,
533	MMUIdxMap idxmap)
534	{
535	int mmu_idx;
536
537	assert_cpu_is_self(cpu);
538
539	tlb_debug("page addr: %016" VADDR_PRIx " mmu_map:0x%x\n", addr, idxmap);
540
541	qemu_spin_lock(&cpu->neg.tlb.c.lock);
542	for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
543	if ((idxmap >> mmu_idx) & 1) {
544	tlb_flush_page_locked(cpu, mmu_idx, addr);
545	}
546	}
547	qemu_spin_unlock(&cpu->neg.tlb.c.lock);
548
549	/*
550	* Discard jump cache entries for any tb which might potentially
551	* overlap the flushed page, which includes the previous.
552	*/
553	tb_jmp_cache_clear_page(cpu, addr - TARGET_PAGE_SIZE);
554	tb_jmp_cache_clear_page(cpu, addr);
555	}
556
557	/**
558	* tlb_flush_page_by_mmuidx_async_1:
559	* @cpu: cpu on which to flush
560	* @data: encoded addr + idxmap
561	*
562	* Helper for tlb_flush_page_by_mmuidx and friends, called through
563	* async_run_on_cpu. The idxmap parameter is encoded in the page
564	* offset of the target_ptr field. This limits the set of mmu_idx
565	* that can be passed via this method.
566	*/
567	static void tlb_flush_page_by_mmuidx_async_1(CPUState *cpu,
568	run_on_cpu_data data)
569	{
570	vaddr addr_and_idxmap = data.target_ptr;
571	vaddr addr = addr_and_idxmap & TARGET_PAGE_MASK;
572	MMUIdxMap idxmap = addr_and_idxmap & ~TARGET_PAGE_MASK;
573
574	tlb_flush_page_by_mmuidx_async_0(cpu, addr, idxmap);
575	}
576
577	typedef struct {
578	vaddr addr;
579	MMUIdxMap idxmap;
580	} TLBFlushPageByMMUIdxData;
581
582	/**
583	* tlb_flush_page_by_mmuidx_async_2:
584	* @cpu: cpu on which to flush
585	* @data: allocated addr + idxmap
586	*
587	* Helper for tlb_flush_page_by_mmuidx and friends, called through
588	* async_run_on_cpu. The addr+idxmap parameters are stored in a
589	* TLBFlushPageByMMUIdxData structure that has been allocated
590	* specifically for this helper. Free the structure when done.
591	*/
592	static void tlb_flush_page_by_mmuidx_async_2(CPUState *cpu,
593	run_on_cpu_data data)
594	{
595	TLBFlushPageByMMUIdxData *d = data.host_ptr;
596
597	tlb_flush_page_by_mmuidx_async_0(cpu, d->addr, d->idxmap);
598	g_free(d);
599	}
600
601	void tlb_flush_page_by_mmuidx(CPUState *cpu, vaddr addr, MMUIdxMap idxmap)
602	{
603	tlb_debug("addr: %016" VADDR_PRIx " mmu_idx:%" PRIx16 "\n", addr, idxmap);
604
605	assert_cpu_is_self(cpu);
606
607	/* This should already be page aligned */
608	addr &= TARGET_PAGE_MASK;
609
610	tlb_flush_page_by_mmuidx_async_0(cpu, addr, idxmap);
611	}
612
613	void tlb_flush_page(CPUState *cpu, vaddr addr)
614	{
615	tlb_flush_page_by_mmuidx(cpu, addr, ALL_MMUIDX_BITS);
616	}
617
618	void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
619	vaddr addr,
620	MMUIdxMap idxmap)
621	{
622	tlb_debug("addr: %016" VADDR_PRIx " mmu_idx:%"PRIx16"\n", addr, idxmap);
623
624	/* This should already be page aligned */
625	addr &= TARGET_PAGE_MASK;
626
627	/*
628	* Allocate memory to hold addr+idxmap only when needed.
629	* See tlb_flush_page_by_mmuidx for details.
630	*/
631	if (idxmap < TARGET_PAGE_SIZE) {
632	flush_all_helper(src_cpu, tlb_flush_page_by_mmuidx_async_1,
633	RUN_ON_CPU_TARGET_PTR(addr \| idxmap));
634	async_safe_run_on_cpu(src_cpu, tlb_flush_page_by_mmuidx_async_1,
635	RUN_ON_CPU_TARGET_PTR(addr \| idxmap));
636	} else {
637	CPUState *dst_cpu;
638	TLBFlushPageByMMUIdxData *d;
639
640	/* Allocate a separate data block for each destination cpu. */
641	CPU_FOREACH(dst_cpu) {
642	if (dst_cpu != src_cpu) {
643	d = g_new(TLBFlushPageByMMUIdxData, 1);
644	d->addr = addr;
645	d->idxmap = idxmap;
646	async_run_on_cpu(dst_cpu, tlb_flush_page_by_mmuidx_async_2,
647	RUN_ON_CPU_HOST_PTR(d));
648	}
649	}
650
651	d = g_new(TLBFlushPageByMMUIdxData, 1);
652	d->addr = addr;
653	d->idxmap = idxmap;
654	async_safe_run_on_cpu(src_cpu, tlb_flush_page_by_mmuidx_async_2,
655	RUN_ON_CPU_HOST_PTR(d));
656	}
657	}
658
659	void tlb_flush_page_all_cpus_synced(CPUState *src, vaddr addr)
660	{
661	tlb_flush_page_by_mmuidx_all_cpus_synced(src, addr, ALL_MMUIDX_BITS);
662	}
663
664	static void tlb_flush_range_locked(CPUState *cpu, int midx,
665	vaddr addr, vaddr len,
666	unsigned bits)
667	{
668	CPUTLBDesc *d = &cpu->neg.tlb.d[midx];
669	CPUTLBDescFast *f = cpu_tlb_fast(cpu, midx);
670	vaddr mask = MAKE_64BIT_MASK(0, bits);
671
672	/*
673	* If @bits is smaller than the tlb size, there may be multiple entries
674	* within the TLB; otherwise all addresses that match under @mask hit
675	* the same TLB entry.
676	* TODO: Perhaps allow bits to be a few bits less than the size.
677	* For now, just flush the entire TLB.
678	*
679	* If @len is larger than the tlb size, then it will take longer to
680	* test all of the entries in the TLB than it will to flush it all.
681	*/
682	if (mask < f->mask \|\| len > f->mask) {
683	tlb_debug("forcing full flush midx %d ("
684	"%016" VADDR_PRIx "/%016" VADDR_PRIx "+%016" VADDR_PRIx ")\n",
685	midx, addr, mask, len);
686	tlb_flush_one_mmuidx_locked(cpu, midx, get_clock_realtime());
687	return;
688	}
689
690	/*
691	* Check if we need to flush due to large pages.
692	* Because large_page_mask contains all 1's from the msb,
693	* we only need to test the end of the range.
694	*/
695	if (((addr + len - 1) & d->large_page_mask) == d->large_page_addr) {
696	tlb_debug("forcing full flush midx %d ("
697	"%016" VADDR_PRIx "/%016" VADDR_PRIx ")\n",
698	midx, d->large_page_addr, d->large_page_mask);
699	tlb_flush_one_mmuidx_locked(cpu, midx, get_clock_realtime());
700	return;
701	}
702
703	for (vaddr i = 0; i < len; i += TARGET_PAGE_SIZE) {
704	vaddr page = addr + i;
705	CPUTLBEntry *entry = tlb_entry(cpu, midx, page);
706
707	if (tlb_flush_entry_mask_locked(entry, page, mask)) {
708	tlb_n_used_entries_dec(cpu, midx);
709	}
710	tlb_flush_vtlb_page_mask_locked(cpu, midx, page, mask);
711	}
712	}
713
714	typedef struct {
715	vaddr addr;
716	vaddr len;
717	MMUIdxMap idxmap;
718	unsigned bits;
719	} TLBFlushRangeData;
720
721	static void tlb_flush_range_by_mmuidx_async_0(CPUState *cpu,
722	TLBFlushRangeData d)
723	{
724	int mmu_idx;
725
726	assert_cpu_is_self(cpu);
727
728	tlb_debug("range: %016" VADDR_PRIx "/%u+%016" VADDR_PRIx " mmu_map:0x%x\n",
729	d.addr, d.bits, d.len, d.idxmap);
730
731	qemu_spin_lock(&cpu->neg.tlb.c.lock);
732	for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
733	if ((d.idxmap >> mmu_idx) & 1) {
734	tlb_flush_range_locked(cpu, mmu_idx, d.addr, d.len, d.bits);
735	}
736	}
737	qemu_spin_unlock(&cpu->neg.tlb.c.lock);
738
739	/*
740	* If the length is larger than the jump cache size, then it will take
741	* longer to clear each entry individually than it will to clear it all.
742	*/
743	if (d.len >= (TARGET_PAGE_SIZE * TB_JMP_CACHE_SIZE)) {
744	tcg_flush_jmp_cache(cpu);
745	return;
746	}
747
748	/*
749	* Discard jump cache entries for any tb which might potentially
750	* overlap the flushed pages, which includes the previous.
751	*/
752	d.addr -= TARGET_PAGE_SIZE;
753	for (vaddr i = 0, n = d.len / TARGET_PAGE_SIZE + 1; i < n; i++) {
754	tb_jmp_cache_clear_page(cpu, d.addr);
755	d.addr += TARGET_PAGE_SIZE;
756	}
757	}
758
759	static void tlb_flush_range_by_mmuidx_async_1(CPUState *cpu,
760	run_on_cpu_data data)
761	{
762	TLBFlushRangeData *d = data.host_ptr;
763	tlb_flush_range_by_mmuidx_async_0(cpu, *d);
764	g_free(d);
765	}
766
767	void tlb_flush_range_by_mmuidx(CPUState *cpu, vaddr addr,
768	vaddr len, MMUIdxMap idxmap,
769	unsigned bits)
770	{
771	TLBFlushRangeData d;
772
773	assert_cpu_is_self(cpu);
774
775	/* If no page bits are significant, this devolves to tlb_flush. */
776	if (bits < TARGET_PAGE_BITS) {
777	tlb_flush_by_mmuidx(cpu, idxmap);
778	return;
779	}
780	/*
781	* If all bits are significant, and len is small,
782	* this devolves to tlb_flush_page.
783	*/
784	if (len <= TARGET_PAGE_SIZE && bits >= target_long_bits()) {
785	tlb_flush_page_by_mmuidx(cpu, addr, idxmap);
786	return;
787	}
788
789	/* This should already be page aligned */
790	d.addr = addr & TARGET_PAGE_MASK;
791	d.len = len;
792	d.idxmap = idxmap;
793	d.bits = bits;
794
795	tlb_flush_range_by_mmuidx_async_0(cpu, d);
796	}
797
798	void tlb_flush_page_bits_by_mmuidx(CPUState *cpu, vaddr addr,
799	MMUIdxMap idxmap, unsigned bits)
800	{
801	tlb_flush_range_by_mmuidx(cpu, addr, TARGET_PAGE_SIZE, idxmap, bits);
802	}
803
804	void tlb_flush_range_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
805	vaddr addr,
806	vaddr len,
807	MMUIdxMap idxmap,
808	unsigned bits)
809	{
810	TLBFlushRangeData d, *p;
811	CPUState *dst_cpu;
812
813	/* If no page bits are significant, this devolves to tlb_flush. */
814	if (bits < TARGET_PAGE_BITS) {
815	tlb_flush_by_mmuidx_all_cpus_synced(src_cpu, idxmap);
816	return;
817	}
818	/*
819	* If all bits are significant, and len is small,
820	* this devolves to tlb_flush_page.
821	*/
822	if (len <= TARGET_PAGE_SIZE && bits >= target_long_bits()) {
823	tlb_flush_page_by_mmuidx_all_cpus_synced(src_cpu, addr, idxmap);
824	return;
825	}
826
827	/* This should already be page aligned */
828	d.addr = addr & TARGET_PAGE_MASK;
829	d.len = len;
830	d.idxmap = idxmap;
831	d.bits = bits;
832
833	/* Allocate a separate data block for each destination cpu. */
834	CPU_FOREACH(dst_cpu) {
835	if (dst_cpu != src_cpu) {
836	p = g_memdup(&d, sizeof(d));
837	async_run_on_cpu(dst_cpu, tlb_flush_range_by_mmuidx_async_1,
838	RUN_ON_CPU_HOST_PTR(p));
839	}
840	}
841
842	p = g_memdup(&d, sizeof(d));
843	async_safe_run_on_cpu(src_cpu, tlb_flush_range_by_mmuidx_async_1,
844	RUN_ON_CPU_HOST_PTR(p));
845	}
846
847	void tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
848	vaddr addr,
849	MMUIdxMap idxmap,
850	unsigned bits)
851	{
852	tlb_flush_range_by_mmuidx_all_cpus_synced(src_cpu, addr, TARGET_PAGE_SIZE,
853	idxmap, bits);
854	}
855
856	/* update the TLBs so that writes to code in the virtual page 'addr'
857	can be detected */
858	void tlb_protect_code(ram_addr_t ram_addr)
859	{
860	physical_memory_test_and_clear_dirty(ram_addr & TARGET_PAGE_MASK,
861	TARGET_PAGE_SIZE,
862	DIRTY_MEMORY_CODE,
863	NULL);
864	}
865
866	/* update the TLB so that writes in physical page 'phys_addr' are no longer
867	tested for self modifying code */
868	void tlb_unprotect_code(ram_addr_t ram_addr)
869	{
870	physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_CODE);
871	}
872
873
874	/*
875	* Dirty write flag handling
876	*
877	* When the TCG code writes to a location it looks up the address in
878	* the TLB and uses that data to compute the final address. If any of
879	* the lower bits of the address are set then the slow path is forced.
880	* There are a number of reasons to do this but for normal RAM the
881	* most usual is detecting writes to code regions which may invalidate
882	* generated code.
883	*
884	* Other vCPUs might be reading their TLBs during guest execution, so we update
885	* te->addr_write with qatomic_set. We don't need to worry about this for
886	* oversized guests as MTTCG is disabled for them.
887	*
888	* Called with tlb_c.lock held.
889	*/
890	static void tlb_reset_dirty_range_locked(CPUTLBEntryFull full, CPUTLBEntry ent,
891	uintptr_t start, uintptr_t length)
892	{
893	const uintptr_t addr = ent->addr_write;
894	int flags = addr \| full->slow_flags[MMU_DATA_STORE];
895
896	flags &= TLB_INVALID_MASK \| TLB_MMIO \| TLB_DISCARD_WRITE \| TLB_NOTDIRTY;
897	if (flags == 0) {
898	uintptr_t host = (addr & TARGET_PAGE_MASK) + ent->addend;
899	if ((host - start) < length) {
900	qatomic_set(&ent->addr_write, addr \| TLB_NOTDIRTY);
901	}
902	}
903	}
904
905	/*
906	* Called with tlb_c.lock held.
907	* Called only from the vCPU context, i.e. the TLB's owner thread.
908	*/
909	static inline void copy_tlb_helper_locked(CPUTLBEntry d, const CPUTLBEntry s)
910	{
911	d = s;
912	}
913
914	/* This is a cross vCPU call (i.e. another vCPU resetting the flags of
915	* the target vCPU).
916	* We must take tlb_c.lock to avoid racing with another vCPU update. The only
917	* thing actually updated is the target TLB entry ->addr_write flags.
918	*/
919	void tlb_reset_dirty(CPUState *cpu, uintptr_t start, uintptr_t length)
920	{
921	int mmu_idx;
922
923	qemu_spin_lock(&cpu->neg.tlb.c.lock);
924	for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
925	CPUTLBDesc *desc = &cpu->neg.tlb.d[mmu_idx];
926	CPUTLBDescFast *fast = cpu_tlb_fast(cpu, mmu_idx);
927	unsigned int n = tlb_n_entries(fast);
928	unsigned int i;
929
930	for (i = 0; i < n; i++) {
931	tlb_reset_dirty_range_locked(&desc->fulltlb[i], &fast->table[i],
932	start, length);
933	}
934
935	for (i = 0; i < CPU_VTLB_SIZE; i++) {
936	tlb_reset_dirty_range_locked(&desc->vfulltlb[i], &desc->vtable[i],
937	start, length);
938	}
939	}
940	qemu_spin_unlock(&cpu->neg.tlb.c.lock);
941	}
942
943	/* Called with tlb_c.lock held */
944	static inline void tlb_set_dirty1_locked(CPUTLBEntry *tlb_entry,
945	vaddr addr)
946	{
947	if (tlb_entry->addr_write == (addr \| TLB_NOTDIRTY)) {
948	tlb_entry->addr_write = addr;
949	}
950	}
951
952	/* update the TLB corresponding to virtual page vaddr
953	so that it is no longer dirty */
954	static void tlb_set_dirty(CPUState *cpu, vaddr addr)
955	{
956	int mmu_idx;
957
958	assert_cpu_is_self(cpu);
959
960	addr &= TARGET_PAGE_MASK;
961	qemu_spin_lock(&cpu->neg.tlb.c.lock);
962	for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
963	tlb_set_dirty1_locked(tlb_entry(cpu, mmu_idx, addr), addr);
964	}
965
966	for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
967	int k;
968	for (k = 0; k < CPU_VTLB_SIZE; k++) {
969	tlb_set_dirty1_locked(&cpu->neg.tlb.d[mmu_idx].vtable[k], addr);
970	}
971	}
972	qemu_spin_unlock(&cpu->neg.tlb.c.lock);
973	}
974
975	/* Our TLB does not support large pages, so remember the area covered by
976	large pages and trigger a full TLB flush if these are invalidated. */
977	static void tlb_add_large_page(CPUState *cpu, int mmu_idx,
978	vaddr addr, uint64_t size)
979	{
980	vaddr lp_addr = cpu->neg.tlb.d[mmu_idx].large_page_addr;
981	vaddr lp_mask = ~(size - 1);
982
983	if (lp_addr == (vaddr)-1) {
984	/* No previous large page. */
985	lp_addr = addr;
986	} else {
987	/* Extend the existing region to include the new page.
988	This is a compromise between unnecessary flushes and
989	the cost of maintaining a full variable size TLB. */
990	lp_mask &= cpu->neg.tlb.d[mmu_idx].large_page_mask;
991	while (((lp_addr ^ addr) & lp_mask) != 0) {
992	lp_mask <<= 1;
993	}
994	}
995	cpu->neg.tlb.d[mmu_idx].large_page_addr = lp_addr & lp_mask;
996	cpu->neg.tlb.d[mmu_idx].large_page_mask = lp_mask;
997	}
998
999	static inline void tlb_set_compare(CPUTLBEntryFull full, CPUTLBEntry ent,
1000	vaddr address, int flags,
1001	MMUAccessType access_type, bool enable)
1002	{
1003	if (enable) {
1004	address \|= flags & TLB_FLAGS_MASK;
1005	flags &= TLB_SLOW_FLAGS_MASK;
1006	if (flags) {
1007	address \|= TLB_FORCE_SLOW;
1008	}
1009	} else {
1010	address = -1;
1011	flags = 0;
1012	}
1013	ent->addr_idx[access_type] = address;
1014	full->slow_flags[access_type] = flags;
1015	}
1016
1017	/*
1018	* Add a new TLB entry. At most one entry for a given virtual address
1019	* is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
1020	* supplied size is only used by tlb_flush_page.
1021	*
1022	* Called from TCG-generated code, which is under an RCU read-side
1023	* critical section.
1024	*/
1025	void tlb_set_page_full(CPUState *cpu, int mmu_idx,
1026	vaddr addr, CPUTLBEntryFull *full)
1027	{
1028	CPUTLB *tlb = &cpu->neg.tlb;
1029	CPUTLBDesc *desc = &tlb->d[mmu_idx];
1030	MemoryRegionSection *section;
1031	unsigned int index, read_flags, write_flags;
1032	uintptr_t addend;
1033	CPUTLBEntry *te, tn;
1034	hwaddr iotlb, xlat, sz, paddr_page;
1035	vaddr addr_page;
1036	int asidx, wp_flags, prot;
1037	bool is_ram, is_romd;
1038
1039	assert_cpu_is_self(cpu);
1040
1041	if (full->lg_page_size <= TARGET_PAGE_BITS) {
1042	sz = TARGET_PAGE_SIZE;
1043	} else {
1044	sz = (hwaddr)1 << full->lg_page_size;
1045	tlb_add_large_page(cpu, mmu_idx, addr, sz);
1046	}
1047	addr_page = addr & TARGET_PAGE_MASK;
1048	paddr_page = full->phys_addr & TARGET_PAGE_MASK;
1049
1050	prot = full->prot;
1051	asidx = cpu_asidx_from_attrs(cpu, full->attrs);
1052	section = address_space_translate_for_iotlb(cpu, asidx, paddr_page,
1053	&xlat, &sz, full->attrs, &prot);
1054	assert(sz >= TARGET_PAGE_SIZE);
1055
1056	tlb_debug("vaddr=%016" VADDR_PRIx " paddr=0x" HWADDR_FMT_plx
1057	" prot=%x idx=%d\n",
1058	addr, full->phys_addr, prot, mmu_idx);
1059
1060	read_flags = full->tlb_fill_flags;
1061	if (full->lg_page_size < TARGET_PAGE_BITS) {
1062	/* Repeat the MMU check and TLB fill on every access. */
1063	read_flags \|= TLB_INVALID_MASK;
1064	}
1065
1066	is_ram = memory_region_is_ram(section->mr);
1067	is_romd = memory_region_is_romd(section->mr);
1068
1069	if (is_ram \|\| is_romd) {
1070	/* RAM and ROMD both have associated host memory. */
1071	addend = (uintptr_t)memory_region_get_ram_ptr(section->mr) + xlat;
1072	} else {
1073	/* I/O does not; force the host address to NULL. */
1074	addend = 0;
1075	}
1076
1077	write_flags = read_flags;
1078	if (is_ram) {
1079	iotlb = memory_region_get_ram_addr(section->mr) + xlat;
1080	assert(!(iotlb & ~TARGET_PAGE_MASK));
1081	/*
1082	* Computing is_clean is expensive; avoid all that unless
1083	* the page is actually writable.
1084	*/
1085	if (prot & PAGE_WRITE) {
1086	if (section->readonly) {
1087	write_flags \|= TLB_DISCARD_WRITE;
1088	} else if (physical_memory_is_clean(iotlb)) {
1089	write_flags \|= TLB_NOTDIRTY;
1090	}
1091	}
1092	} else {
1093	/* I/O or ROMD */
1094	iotlb = xlat;
1095	/*
1096	* Writes to romd devices must go through MMIO to enable write.
1097	* Reads to romd devices go through the ram_ptr found above,
1098	* but of course reads to I/O must go through MMIO.
1099	*/
1100	write_flags \|= TLB_MMIO;
1101	if (!is_romd) {
1102	read_flags = write_flags;
1103	}
1104	}
1105
1106	wp_flags = cpu_watchpoint_address_matches(cpu, addr_page,
1107	TARGET_PAGE_SIZE);
1108
1109	index = tlb_index(cpu, mmu_idx, addr_page);
1110	te = tlb_entry(cpu, mmu_idx, addr_page);
1111
1112	/*
1113	* Hold the TLB lock for the rest of the function. We could acquire/release
1114	* the lock several times in the function, but it is faster to amortize the
1115	* acquisition cost by acquiring it just once. Note that this leads to
1116	* a longer critical section, but this is not a concern since the TLB lock
1117	* is unlikely to be contended.
1118	*/
1119	qemu_spin_lock(&tlb->c.lock);
1120
1121	/* Note that the tlb is no longer clean. */
1122	tlb->c.dirty \|= 1 << mmu_idx;
1123
1124	/* Make sure there's no cached translation for the new page. */
1125	tlb_flush_vtlb_page_locked(cpu, mmu_idx, addr_page);
1126
1127	/*
1128	* Only evict the old entry to the victim tlb if it's for a
1129	* different page; otherwise just overwrite the stale data.
1130	*/
1131	if (!tlb_hit_page_anyprot(te, addr_page) && !tlb_entry_is_empty(te)) {
1132	unsigned vidx = desc->vindex++ % CPU_VTLB_SIZE;
1133	CPUTLBEntry *tv = &desc->vtable[vidx];
1134
1135	/* Evict the old entry into the victim tlb. */
1136	copy_tlb_helper_locked(tv, te);
1137	desc->vfulltlb[vidx] = desc->fulltlb[index];
1138	tlb_n_used_entries_dec(cpu, mmu_idx);
1139	}
1140
1141	/* refill the tlb */
1142	/*
1143	* When memory region is ram, iotlb contains a TARGET_PAGE_BITS
1144	* aligned ram_addr_t of the page base of the target RAM.
1145	* Otherwise, iotlb contains a TARGET_PAGE_BITS aligned
1146	* offset within section->mr of the page base (I/O, ROMD)
1147	*
1148	* We subtract addr_page (which is page aligned and thus won't
1149	* disturb the low bits) to give an offset which can be added to the
1150	* (non-page-aligned) vaddr of the eventual memory access to get
1151	* the MemoryRegion offset for the access. Note that the vaddr we
1152	* subtract here is that of the page base, and not the same as the
1153	* vaddr we add back in io_prepare()/get_page_addr_code().
1154	*/
1155	desc->fulltlb[index] = *full;
1156	full = &desc->fulltlb[index];
1157	full->xlat_offset = iotlb - addr_page;
1158	full->section = section;
1159	full->phys_addr = paddr_page;
1160
1161	/* Now calculate the new entry */
1162	tn.addend = addend - addr_page;
1163
1164	tlb_set_compare(full, &tn, addr_page, read_flags,
1165	MMU_INST_FETCH, prot & PAGE_EXEC);
1166
1167	if (wp_flags & BP_MEM_READ) {
1168	read_flags \|= TLB_WATCHPOINT;
1169	}
1170	tlb_set_compare(full, &tn, addr_page, read_flags,
1171	MMU_DATA_LOAD, prot & PAGE_READ);
1172
1173	if (prot & PAGE_WRITE_INV) {
1174	write_flags \|= TLB_INVALID_MASK;
1175	}
1176	if (wp_flags & BP_MEM_WRITE) {
1177	write_flags \|= TLB_WATCHPOINT;
1178	}
1179	tlb_set_compare(full, &tn, addr_page, write_flags,
1180	MMU_DATA_STORE, prot & PAGE_WRITE);
1181
1182	copy_tlb_helper_locked(te, &tn);
1183	tlb_n_used_entries_inc(cpu, mmu_idx);
1184	qemu_spin_unlock(&tlb->c.lock);
1185	}
1186
1187	void tlb_set_page_with_attrs(CPUState *cpu, vaddr addr,
1188	hwaddr paddr, MemTxAttrs attrs, int prot,
1189	int mmu_idx, vaddr size)
1190	{
1191	CPUTLBEntryFull full = {
1192	.phys_addr = paddr,
1193	.attrs = attrs,
1194	.prot = prot,
1195	.lg_page_size = ctz64(size)
1196	};
1197
1198	assert(is_power_of_2(size));
1199	tlb_set_page_full(cpu, mmu_idx, addr, &full);
1200	}
1201
1202	void tlb_set_page(CPUState *cpu, vaddr addr,
1203	hwaddr paddr, int prot,
1204	int mmu_idx, vaddr size)
1205	{
1206	tlb_set_page_with_attrs(cpu, addr, paddr, MEMTXATTRS_UNSPECIFIED,
1207	prot, mmu_idx, size);
1208	}
1209
1210	/**
1211	* tlb_hit_page: return true if page aligned @addr is a hit against the
1212	* TLB entry @tlb_addr
1213	*
1214	* @addr: virtual address to test (must be page aligned)
1215	* @tlb_addr: TLB entry address (a CPUTLBEntry addr_read/write/code value)
1216	*/
1217	static inline bool tlb_hit_page(uint64_t tlb_addr, vaddr addr)
1218	{
1219	return addr == (tlb_addr & (TARGET_PAGE_MASK \| TLB_INVALID_MASK));
1220	}
1221
1222	/**
1223	* tlb_hit: return true if @addr is a hit against the TLB entry @tlb_addr
1224	*
1225	* @addr: virtual address to test (need not be page aligned)
1226	* @tlb_addr: TLB entry address (a CPUTLBEntry addr_read/write/code value)
1227	*/
1228	static inline bool tlb_hit(uint64_t tlb_addr, vaddr addr)
1229	{
1230	return tlb_hit_page(tlb_addr, addr & TARGET_PAGE_MASK);
1231	}
1232
1233	/*
1234	* Note: tlb_fill_align() can trigger a resize of the TLB.
1235	* This means that all of the caller's prior references to the TLB table
1236	* (e.g. CPUTLBEntry pointers) must be discarded and looked up again
1237	* (e.g. via tlb_entry()).
1238	*/
1239	static bool tlb_fill_align(CPUState *cpu, vaddr addr, MMUAccessType type,
1240	int mmu_idx, MemOp memop, int size,
1241	bool probe, uintptr_t ra)
1242	{
1243	const TCGCPUOps *ops = cpu->cc->tcg_ops;
1244	CPUTLBEntryFull full;
1245
1246	if (ops->tlb_fill_align) {
1247	if (ops->tlb_fill_align(cpu, &full, addr, type, mmu_idx,
1248	memop, size, probe, ra)) {
1249	tlb_set_page_full(cpu, mmu_idx, addr, &full);
1250	return true;
1251	}
1252	} else {
1253	/* Legacy behaviour is alignment before paging. */
1254	if (addr & ((1u << memop_alignment_bits(memop)) - 1)) {
1255	ops->do_unaligned_access(cpu, addr, type, mmu_idx, ra);
1256	}
1257	if (ops->tlb_fill(cpu, addr, size, type, mmu_idx, probe, ra)) {
1258	return true;
1259	}
1260	}
1261	assert(probe);
1262	return false;
1263	}
1264
1265	static inline void cpu_unaligned_access(CPUState *cpu, vaddr addr,
1266	MMUAccessType access_type,
1267	int mmu_idx, uintptr_t retaddr)
1268	{
1269	cpu->cc->tcg_ops->do_unaligned_access(cpu, addr, access_type,
1270	mmu_idx, retaddr);
1271	}
1272
1273	static MemoryRegionSection *
1274	io_prepare(hwaddr out_offset, CPUState cpu, CPUTLBEntryFull *full,
1275	vaddr addr, uintptr_t retaddr)
1276	{
1277	MemoryRegionSection *section;
1278	hwaddr mr_offset;
1279
1280	section = full->section;
1281	mr_offset = full->xlat_offset + addr;
1282	cpu->mem_io_pc = retaddr;
1283	if (!cpu->neg.can_do_io) {
1284	cpu_io_recompile(cpu, retaddr);
1285	}
1286
1287	*out_offset = mr_offset;
1288	return section;
1289	}
1290
1291	static void io_failed(CPUState cpu, CPUTLBEntryFull full, vaddr addr,
1292	unsigned size, MMUAccessType access_type, int mmu_idx,
1293	MemTxResult response, uintptr_t retaddr)
1294	{
1295	if (!cpu->ignore_memory_transaction_failures
1296	&& cpu->cc->tcg_ops->do_transaction_failed) {
1297	hwaddr physaddr = full->phys_addr \| (addr & ~TARGET_PAGE_MASK);
1298
1299	cpu->cc->tcg_ops->do_transaction_failed(cpu, physaddr, addr, size,
1300	access_type, mmu_idx,
1301	full->attrs, response, retaddr);
1302	}
1303	}
1304
1305	/* Return true if ADDR is present in the victim tlb, and has been copied
1306	back to the main tlb. */
1307	static bool victim_tlb_hit(CPUState *cpu, size_t mmu_idx, size_t index,
1308	MMUAccessType access_type, vaddr page)
1309	{
1310	size_t vidx;
1311
1312	assert_cpu_is_self(cpu);
1313	for (vidx = 0; vidx < CPU_VTLB_SIZE; ++vidx) {
1314	CPUTLBEntry *vtlb = &cpu->neg.tlb.d[mmu_idx].vtable[vidx];
1315	uint64_t cmp = tlb_read_idx(vtlb, access_type);
1316
1317	if (cmp == page) {
1318	/* Found entry in victim tlb, swap tlb and iotlb. */
1319	CPUTLBEntry tmptlb, *tlb = &cpu_tlb_fast(cpu, mmu_idx)->table[index];
1320
1321	qemu_spin_lock(&cpu->neg.tlb.c.lock);
1322	copy_tlb_helper_locked(&tmptlb, tlb);
1323	copy_tlb_helper_locked(tlb, vtlb);
1324	copy_tlb_helper_locked(vtlb, &tmptlb);
1325	qemu_spin_unlock(&cpu->neg.tlb.c.lock);
1326
1327	CPUTLBEntryFull *f1 = &cpu->neg.tlb.d[mmu_idx].fulltlb[index];
1328	CPUTLBEntryFull *f2 = &cpu->neg.tlb.d[mmu_idx].vfulltlb[vidx];
1329	CPUTLBEntryFull tmpf;
1330	tmpf = f1; f1 = f2; f2 = tmpf;
1331	return true;
1332	}
1333	}
1334	return false;
1335	}
1336
1337	static void notdirty_write(CPUState *cpu, vaddr mem_vaddr, unsigned size,
1338	CPUTLBEntryFull *full, uintptr_t retaddr)
1339	{
1340	ram_addr_t ram_addr = mem_vaddr + full->xlat_offset;
1341
1342	trace_memory_notdirty_write_access(mem_vaddr, ram_addr, size);
1343
1344	if (!physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
1345	tb_invalidate_phys_range_fast(cpu, ram_addr, size, retaddr);
1346	}
1347
1348	/*
1349	* Set both VGA and migration bits for simplicity and to remove
1350	* the notdirty callback faster.
1351	*/
1352	physical_memory_set_dirty_range(ram_addr, size, DIRTY_CLIENTS_NOCODE);
1353
1354	/* We remove the notdirty callback only if the code has been flushed. */
1355	if (!physical_memory_is_clean(ram_addr)) {
1356	trace_memory_notdirty_set_dirty(mem_vaddr);
1357	tlb_set_dirty(cpu, mem_vaddr);
1358	}
1359	}
1360
1361	static int probe_access_internal(CPUState *cpu, vaddr addr,
1362	int fault_size, MMUAccessType access_type,
1363	int mmu_idx, bool nonfault,
1364	void phost, CPUTLBEntryFull pfull,
1365	uintptr_t retaddr, bool check_mem_cbs)
1366	{
1367	uintptr_t index = tlb_index(cpu, mmu_idx, addr);
1368	CPUTLBEntry *entry = tlb_entry(cpu, mmu_idx, addr);
1369	uint64_t tlb_addr = tlb_read_idx(entry, access_type);
1370	vaddr page_addr = addr & TARGET_PAGE_MASK;
1371	int flags = TLB_FLAGS_MASK & ~TLB_FORCE_SLOW;
1372	bool force_mmio = check_mem_cbs && cpu_plugin_mem_cbs_enabled(cpu);
1373	CPUTLBEntryFull *full;
1374
1375	if (!tlb_hit_page(tlb_addr, page_addr)) {
1376	if (!victim_tlb_hit(cpu, mmu_idx, index, access_type, page_addr)) {
1377	if (!tlb_fill_align(cpu, addr, access_type, mmu_idx,
1378	0, fault_size, nonfault, retaddr)) {
1379	/* Non-faulting page table read failed. */
1380	*phost = NULL;
1381	*pfull = NULL;
1382	return TLB_INVALID_MASK;
1383	}
1384
1385	/* TLB resize via tlb_fill_align may have moved the entry. */
1386	index = tlb_index(cpu, mmu_idx, addr);
1387	entry = tlb_entry(cpu, mmu_idx, addr);
1388
1389	/*
1390	* With PAGE_WRITE_INV, we set TLB_INVALID_MASK immediately,
1391	* to force the next access through tlb_fill_align. We've just
1392	* called tlb_fill_align, so we know that this entry is valid.
1393	*/
1394	flags &= ~TLB_INVALID_MASK;
1395	}
1396	tlb_addr = tlb_read_idx(entry, access_type);
1397	}
1398	flags &= tlb_addr;
1399
1400	*pfull = full = &cpu->neg.tlb.d[mmu_idx].fulltlb[index];
1401	flags \|= full->slow_flags[access_type];
1402
1403	/* Fold all "mmio-like" bits into TLB_MMIO. This is not RAM. */
1404	if (unlikely(flags & ~(TLB_WATCHPOINT \| TLB_NOTDIRTY \| TLB_CHECK_ALIGNED))
1405	\|\| (access_type != MMU_INST_FETCH && force_mmio)) {
1406	*phost = NULL;
1407	return TLB_MMIO;
1408	}
1409
1410	/* Everything else is RAM. */
1411	phost = (void )((uintptr_t)addr + entry->addend);
1412	return flags;
1413	}
1414
1415	int probe_access_full(CPUArchState *env, vaddr addr, int size,
1416	MMUAccessType access_type, int mmu_idx,
1417	bool nonfault, void phost, CPUTLBEntryFull pfull,
1418	uintptr_t retaddr)
1419	{
1420	int flags = probe_access_internal(env_cpu(env), addr, size, access_type,
1421	mmu_idx, nonfault, phost, pfull, retaddr,
1422	true);
1423
1424	/* Handle clean RAM pages. */
1425	if (unlikely(flags & TLB_NOTDIRTY)) {
1426	int dirtysize = size == 0 ? 1 : size;
1427	notdirty_write(env_cpu(env), addr, dirtysize, *pfull, retaddr);
1428	flags &= ~TLB_NOTDIRTY;
1429	}
1430
1431	return flags;
1432	}
1433
1434	int probe_access_full_mmu(CPUArchState *env, vaddr addr, int size,
1435	MMUAccessType access_type, int mmu_idx,
1436	void phost, CPUTLBEntryFull pfull)
1437	{
1438	void *discard_phost;
1439	CPUTLBEntryFull *discard_tlb;
1440
1441	/* privately handle users that don't need full results */
1442	phost = phost ? phost : &discard_phost;
1443	pfull = pfull ? pfull : &discard_tlb;
1444
1445	int flags = probe_access_internal(env_cpu(env), addr, size, access_type,
1446	mmu_idx, true, phost, pfull, 0, false);
1447
1448	/* Handle clean RAM pages. */
1449	if (unlikely(flags & TLB_NOTDIRTY)) {
1450	int dirtysize = size == 0 ? 1 : size;
1451	notdirty_write(env_cpu(env), addr, dirtysize, *pfull, 0);
1452	flags &= ~TLB_NOTDIRTY;
1453	}
1454
1455	return flags;
1456	}
1457
1458	int probe_access_flags(CPUArchState *env, vaddr addr, int size,
1459	MMUAccessType access_type, int mmu_idx,
1460	bool nonfault, void **phost, uintptr_t retaddr)
1461	{
1462	CPUTLBEntryFull *full;
1463	int flags;
1464
1465	g_assert(-(addr \| TARGET_PAGE_MASK) >= size);
1466
1467	flags = probe_access_internal(env_cpu(env), addr, size, access_type,
1468	mmu_idx, nonfault, phost, &full, retaddr,
1469	true);
1470
1471	/* Handle clean RAM pages. */
1472	if (unlikely(flags & TLB_NOTDIRTY)) {
1473	int dirtysize = size == 0 ? 1 : size;
1474	notdirty_write(env_cpu(env), addr, dirtysize, full, retaddr);
1475	flags &= ~TLB_NOTDIRTY;
1476	}
1477
1478	return flags;
1479	}
1480
1481	void probe_access(CPUArchState env, vaddr addr, int size,
1482	MMUAccessType access_type, int mmu_idx, uintptr_t retaddr)
1483	{
1484	CPUTLBEntryFull *full;
1485	void *host;
1486	int flags;
1487
1488	g_assert(-(addr \| TARGET_PAGE_MASK) >= size);
1489
1490	flags = probe_access_internal(env_cpu(env), addr, size, access_type,
1491	mmu_idx, false, &host, &full, retaddr,
1492	true);
1493
1494	/* Per the interface, size == 0 merely faults the access. */
1495	if (size == 0) {
1496	return NULL;
1497	}
1498
1499	if (unlikely(flags & (TLB_NOTDIRTY \| TLB_WATCHPOINT))) {
1500	/* Handle watchpoints. */
1501	if (flags & TLB_WATCHPOINT) {
1502	int wp_access = (access_type == MMU_DATA_STORE
1503	? BP_MEM_WRITE : BP_MEM_READ);
1504	cpu_check_watchpoint(env_cpu(env), addr, size,
1505	full->attrs, wp_access, retaddr);
1506	}
1507
1508	/* Handle clean RAM pages. */
1509	if (flags & TLB_NOTDIRTY) {
1510	notdirty_write(env_cpu(env), addr, size, full, retaddr);
1511	}
1512	}
1513
1514	return host;
1515	}
1516
1517	void tlb_vaddr_to_host(CPUArchState env, vaddr addr,
1518	MMUAccessType access_type, int mmu_idx)
1519	{
1520	CPUTLBEntryFull *full;
1521	void *host;
1522	int flags;
1523
1524	flags = probe_access_internal(env_cpu(env), addr, 0, access_type,
1525	mmu_idx, true, &host, &full, 0, false);
1526
1527	/* No combination of flags are expected by the caller. */
1528	return flags ? NULL : host;
1529	}
1530
1531	/*
1532	* Return a ram_addr_t for the virtual address for execution.
1533	*
1534	* Return -1 if we can't translate and execute from an entire page
1535	* of RAM. This will force us to execute by loading and translating
1536	* one insn at a time, without caching.
1537	*
1538	* NOTE: This function will trigger an exception if the page is
1539	* not executable.
1540	*/
1541	tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, vaddr addr,
1542	void **hostp)
1543	{
1544	CPUTLBEntryFull *full;
1545	void *p;
1546
1547	(void)probe_access_internal(env_cpu(env), addr, 1, MMU_INST_FETCH,
1548	cpu_mmu_index(env_cpu(env), true), false,
1549	hostp, &full, 0, false);
1550
1551	p = *hostp;
1552	if (p == NULL) {
1553	return -1;
1554	}
1555
1556	if (full->lg_page_size < TARGET_PAGE_BITS) {
1557	*hostp = NULL;
1558	return -1;
1559	}
1560
1561	return qemu_ram_addr_from_host_nofail(p);
1562	}
1563
1564	/* Load/store with atomicity primitives. */
1565	#include "ldst_atomicity.c.inc"
1566
1567	#ifdef CONFIG_PLUGIN
1568	/*
1569	* Perform a TLB lookup and populate the qemu_plugin_hwaddr structure.
1570	* This should be a hot path as we will have just looked this path up
1571	* in the softmmu lookup code (or helper). We don't handle re-fills or
1572	* checking the victim table. This is purely informational.
1573	*
1574	* The one corner case is i/o write, which can cause changes to the
1575	* address space. Those changes, and the corresponding tlb flush,
1576	* should be delayed until the next TB, so even then this ought not fail.
1577	* But check, Just in Case.
1578	*/
1579	bool tlb_plugin_lookup(CPUState *cpu, vaddr addr, int mmu_idx,
1580	bool is_store, struct qemu_plugin_hwaddr *data)
1581	{
1582	CPUTLBEntry *tlbe = tlb_entry(cpu, mmu_idx, addr);
1583	uintptr_t index = tlb_index(cpu, mmu_idx, addr);
1584	MMUAccessType access_type = is_store ? MMU_DATA_STORE : MMU_DATA_LOAD;
1585	uint64_t tlb_addr = tlb_read_idx(tlbe, access_type);
1586	CPUTLBEntryFull *full;
1587
1588	if (unlikely(!tlb_hit(tlb_addr, addr))) {
1589	return false;
1590	}
1591
1592	full = &cpu->neg.tlb.d[mmu_idx].fulltlb[index];
1593	data->phys_addr = full->phys_addr \| (addr & ~TARGET_PAGE_MASK);
1594
1595	/* We must have an iotlb entry for MMIO */
1596	if (tlb_addr & TLB_MMIO) {
1597	MemoryRegionSection *section = full->section;
1598	data->is_io = true;
1599	data->mr = section->mr;
1600	} else {
1601	data->is_io = false;
1602	data->mr = NULL;
1603	}
1604	return true;
1605	}
1606	#endif
1607
1608	/*
1609	* Probe for a load/store operation.
1610	* Return the host address and into @flags.
1611	*/
1612
1613	typedef struct MMULookupPageData {
1614	CPUTLBEntryFull *full;
1615	void *haddr;
1616	vaddr addr;
1617	int flags;
1618	int size;
1619	} MMULookupPageData;
1620
1621	typedef struct MMULookupLocals {
1622	MMULookupPageData page[2];
1623	MemOp memop;
1624	int mmu_idx;
1625	} MMULookupLocals;
1626
1627	/**
1628	* mmu_lookup1: translate one page
1629	* @cpu: generic cpu state
1630	* @data: lookup parameters
1631	* @memop: memory operation for the access, or 0
1632	* @mmu_idx: virtual address context
1633	* @access_type: load/store/code
1634	* @ra: return address into tcg generated code, or 0
1635	*
1636	* Resolve the translation for the one page at @data.addr, filling in
1637	* the rest of @data with the results. If the translation fails,
1638	* tlb_fill_align will longjmp out. Return true if the softmmu tlb for
1639	* @mmu_idx may have resized.
1640	*/
1641	static bool mmu_lookup1(CPUState cpu, MMULookupPageData data, MemOp memop,
1642	int mmu_idx, MMUAccessType access_type, uintptr_t ra)
1643	{
1644	vaddr addr = data->addr;
1645	uintptr_t index = tlb_index(cpu, mmu_idx, addr);
1646	CPUTLBEntry *entry = tlb_entry(cpu, mmu_idx, addr);
1647	uint64_t tlb_addr = tlb_read_idx(entry, access_type);
1648	bool maybe_resized = false;
1649	CPUTLBEntryFull *full;
1650	int flags;
1651
1652	/* If the TLB entry is for a different page, reload and try again. */
1653	if (!tlb_hit(tlb_addr, addr)) {
1654	if (!victim_tlb_hit(cpu, mmu_idx, index, access_type,
1655	addr & TARGET_PAGE_MASK)) {
1656	tlb_fill_align(cpu, addr, access_type, mmu_idx,
1657	memop, data->size, false, ra);
1658	maybe_resized = true;
1659	index = tlb_index(cpu, mmu_idx, addr);
1660	entry = tlb_entry(cpu, mmu_idx, addr);
1661	}
1662	tlb_addr = tlb_read_idx(entry, access_type) & ~TLB_INVALID_MASK;
1663	}
1664
1665	full = &cpu->neg.tlb.d[mmu_idx].fulltlb[index];
1666	flags = tlb_addr & (TLB_FLAGS_MASK & ~TLB_FORCE_SLOW);
1667	flags \|= full->slow_flags[access_type];
1668
1669	if (likely(!maybe_resized)) {
1670	/* Alignment has not been checked by tlb_fill_align. */
1671	int a_bits = memop_tlb_alignment_bits(memop, flags & TLB_CHECK_ALIGNED);
1672	if (unlikely(addr & ((1 << a_bits) - 1))) {
1673	cpu_unaligned_access(cpu, addr, access_type, mmu_idx, ra);
1674	}
1675	}
1676
1677	data->full = full;
1678	data->flags = flags;
1679	/* Compute haddr speculatively; depending on flags it might be invalid. */
1680	data->haddr = (void *)((uintptr_t)addr + entry->addend);
1681
1682	return maybe_resized;
1683	}
1684
1685	/**
1686	* mmu_watch_or_dirty
1687	* @cpu: generic cpu state
1688	* @data: lookup parameters
1689	* @access_type: load/store/code
1690	* @ra: return address into tcg generated code, or 0
1691	*
1692	* Trigger watchpoints for @data.addr:@data.size;
1693	* record writes to protected clean pages.
1694	*/
1695	static void mmu_watch_or_dirty(CPUState cpu, MMULookupPageData data,
1696	MMUAccessType access_type, uintptr_t ra)
1697	{
1698	CPUTLBEntryFull *full = data->full;
1699	vaddr addr = data->addr;
1700	int flags = data->flags;
1701	int size = data->size;
1702
1703	/* On watchpoint hit, this will longjmp out. */
1704	if (flags & TLB_WATCHPOINT) {
1705	int wp = access_type == MMU_DATA_STORE ? BP_MEM_WRITE : BP_MEM_READ;
1706	cpu_check_watchpoint(cpu, addr, size, full->attrs, wp, ra);
1707	flags &= ~TLB_WATCHPOINT;
1708	}
1709
1710	/* Note that notdirty is only set for writes. */
1711	if (flags & TLB_NOTDIRTY) {
1712	notdirty_write(cpu, addr, size, full, ra);
1713	flags &= ~TLB_NOTDIRTY;
1714	}
1715	data->flags = flags;
1716	}
1717
1718	/**
1719	* mmu_lookup: translate page(s)
1720	* @cpu: generic cpu state
1721	* @addr: virtual address
1722	* @oi: combined mmu_idx and MemOp
1723	* @ra: return address into tcg generated code, or 0
1724	* @access_type: load/store/code
1725	* @l: output result
1726	*
1727	* Resolve the translation for the page(s) beginning at @addr, for MemOp.size
1728	* bytes. Return true if the lookup crosses a page boundary.
1729	*/
1730	static bool mmu_lookup(CPUState *cpu, vaddr addr, MemOpIdx oi,
1731	uintptr_t ra, MMUAccessType type, MMULookupLocals *l)
1732	{
1733	bool crosspage;
1734	vaddr last;
1735	int flags;
1736
1737	l->memop = get_memop(oi);
1738	l->mmu_idx = get_mmuidx(oi);
1739
1740	tcg_debug_assert(l->mmu_idx < NB_MMU_MODES);
1741
1742	l->page[0].addr = addr;
1743	l->page[0].size = memop_size(l->memop);
1744	l->page[1].addr = 0;
1745	l->page[1].size = 0;
1746
1747	/* Lookup and recognize exceptions from the first page. */
1748	mmu_lookup1(cpu, &l->page[0], l->memop, l->mmu_idx, type, ra);
1749
1750	last = addr + l->page[0].size - 1;
1751	crosspage = (addr ^ last) & TARGET_PAGE_MASK;
1752	if (likely(!crosspage)) {
1753	flags = l->page[0].flags;
1754	if (unlikely(flags & (TLB_WATCHPOINT \| TLB_NOTDIRTY))) {
1755	mmu_watch_or_dirty(cpu, &l->page[0], type, ra);
1756	}
1757	if (unlikely(flags & TLB_BSWAP)) {
1758	l->memop ^= MO_BSWAP;
1759	}
1760	} else {
1761	/* Finish compute of page crossing. */
1762	vaddr addr1 = last & TARGET_PAGE_MASK;
1763	int size0 = addr1 - addr;
1764	l->page[1].size = l->page[0].size - size0;
1765	l->page[0].size = size0;
1766	l->page[1].addr = cpu->cc->tcg_ops->pointer_wrap(cpu, l->mmu_idx,
1767	addr1, addr);
1768
1769	/*
1770	* Lookup and recognize exceptions from the second page.
1771	* If the lookup potentially resized the table, refresh the
1772	* first CPUTLBEntryFull pointer.
1773	*/
1774	if (mmu_lookup1(cpu, &l->page[1], 0, l->mmu_idx, type, ra)) {
1775	uintptr_t index = tlb_index(cpu, l->mmu_idx, addr);
1776	l->page[0].full = &cpu->neg.tlb.d[l->mmu_idx].fulltlb[index];
1777	}
1778
1779	flags = l->page[0].flags \| l->page[1].flags;
1780	if (unlikely(flags & (TLB_WATCHPOINT \| TLB_NOTDIRTY))) {
1781	mmu_watch_or_dirty(cpu, &l->page[0], type, ra);
1782	mmu_watch_or_dirty(cpu, &l->page[1], type, ra);
1783	}
1784
1785	/*
1786	* Since target/sparc is the only user of TLB_BSWAP, and all
1787	* Sparc accesses are aligned, any treatment across two pages
1788	* would be arbitrary. Refuse it until there's a use.
1789	*/
1790	tcg_debug_assert((flags & TLB_BSWAP) == 0);
1791	}
1792
1793	return crosspage;
1794	}
1795
1796	/*
1797	* Probe for an atomic operation. Do not allow unaligned operations,
1798	* or io operations to proceed. Return the host address.
1799	*/
1800	static void atomic_mmu_lookup(CPUState cpu, vaddr addr, MemOpIdx oi,
1801	int size, uintptr_t retaddr)
1802	{
1803	uintptr_t mmu_idx = get_mmuidx(oi);
1804	MemOp mop = get_memop(oi);
1805	uintptr_t index;
1806	CPUTLBEntry *tlbe;
1807	vaddr tlb_addr;
1808	void *hostaddr;
1809	CPUTLBEntryFull *full;
1810	bool did_tlb_fill = false;
1811
1812	tcg_debug_assert(mmu_idx < NB_MMU_MODES);
1813
1814	/* Adjust the given return address. */
1815	retaddr -= GETPC_ADJ;
1816
1817	index = tlb_index(cpu, mmu_idx, addr);
1818	tlbe = tlb_entry(cpu, mmu_idx, addr);
1819
1820	/* Check TLB entry and enforce page permissions. */
1821	tlb_addr = tlb_addr_write(tlbe);
1822	if (!tlb_hit(tlb_addr, addr)) {
1823	if (!victim_tlb_hit(cpu, mmu_idx, index, MMU_DATA_STORE,
1824	addr & TARGET_PAGE_MASK)) {
1825	tlb_fill_align(cpu, addr, MMU_DATA_STORE, mmu_idx,
1826	mop, size, false, retaddr);
1827	did_tlb_fill = true;
1828	index = tlb_index(cpu, mmu_idx, addr);
1829	tlbe = tlb_entry(cpu, mmu_idx, addr);
1830	}
1831	tlb_addr = tlb_addr_write(tlbe) & ~TLB_INVALID_MASK;
1832	}
1833
1834	/*
1835	* Let the guest notice RMW on a write-only page.
1836	* We have just verified that the page is writable.
1837	* Subpage lookups may have left TLB_INVALID_MASK set,
1838	* but addr_read will only be -1 if PAGE_READ was unset.
1839	*/
1840	if (unlikely(tlbe->addr_read == -1)) {
1841	tlb_fill_align(cpu, addr, MMU_DATA_LOAD, mmu_idx,
1842	0, size, false, retaddr);
1843	/*
1844	* Since we don't support reads and writes to different
1845	* addresses, and we do have the proper page loaded for
1846	* write, this shouldn't ever return.
1847	*/
1848	g_assert_not_reached();
1849	}
1850
1851	/* Enforce guest required alignment, if not handled by tlb_fill_align. */
1852	if (!did_tlb_fill && (addr & ((1 << memop_alignment_bits(mop)) - 1))) {
1853	cpu_unaligned_access(cpu, addr, MMU_DATA_STORE, mmu_idx, retaddr);
1854	}
1855
1856	/* Enforce qemu required alignment. */
1857	if (unlikely(addr & (size - 1))) {
1858	/*
1859	* We get here if guest alignment was not requested, or was not
1860	* enforced by cpu_unaligned_access or tlb_fill_align above.
1861	* We might widen the access and emulate, but for now
1862	* mark an exception and exit the cpu loop.
1863	*/
1864	goto stop_the_world;
1865	}
1866
1867	/* Finish collecting tlb flags for both read and write. */
1868	full = &cpu->neg.tlb.d[mmu_idx].fulltlb[index];
1869	tlb_addr \|= tlbe->addr_read;
1870	tlb_addr &= TLB_FLAGS_MASK & ~TLB_FORCE_SLOW;
1871	tlb_addr \|= full->slow_flags[MMU_DATA_STORE];
1872	tlb_addr \|= full->slow_flags[MMU_DATA_LOAD];
1873
1874	/* Notice an IO access or a needs-MMU-lookup access */
1875	if (unlikely(tlb_addr & (TLB_MMIO \| TLB_DISCARD_WRITE))) {
1876	/* There's really nothing that can be done to
1877	support this apart from stop-the-world. */
1878	goto stop_the_world;
1879	}
1880
1881	hostaddr = (void *)((uintptr_t)addr + tlbe->addend);
1882
1883	if (unlikely(tlb_addr & TLB_NOTDIRTY)) {
1884	notdirty_write(cpu, addr, size, full, retaddr);
1885	}
1886
1887	if (unlikely(tlb_addr & TLB_WATCHPOINT)) {
1888	int wp_flags = 0;
1889
1890	if (full->slow_flags[MMU_DATA_STORE] & TLB_WATCHPOINT) {
1891	wp_flags \|= BP_MEM_WRITE;
1892	}
1893	if (full->slow_flags[MMU_DATA_LOAD] & TLB_WATCHPOINT) {
1894	wp_flags \|= BP_MEM_READ;
1895	}
1896	cpu_check_watchpoint(cpu, addr, size,
1897	full->attrs, wp_flags, retaddr);
1898	}
1899
1900	return hostaddr;
1901
1902	stop_the_world:
1903	cpu_loop_exit_atomic(cpu, retaddr);
1904	}
1905
1906	/*
1907	* Load Helpers
1908	*
1909	* We support two different access types. SOFTMMU_CODE_ACCESS is
1910	* specifically for reading instructions from system memory. It is
1911	* called by the translation loop and in some helpers where the code
1912	* is disassembled. It shouldn't be called directly by guest code.
1913	*
1914	* For the benefit of TCG generated code, we want to avoid the
1915	* complication of ABI-specific return type promotion and always
1916	* return a value extended to the register size of the host. This is
1917	* tcg_target_long, except in the case of a 32-bit host and 64-bit
1918	* data, and for that we always have uint64_t.
1919	*
1920	* We don't bother with this widened value for SOFTMMU_CODE_ACCESS.
1921	*/
1922
1923	/**
1924	* do_ld_mmio_beN:
1925	* @cpu: generic cpu state
1926	* @full: page parameters
1927	* @ret_be: accumulated data
1928	* @addr: virtual address
1929	* @size: number of bytes
1930	* @mmu_idx: virtual address context
1931	* @ra: return address into tcg generated code, or 0
1932	* Context: BQL held
1933	*
1934	* Load @size bytes from @addr, which is memory-mapped i/o.
1935	* The bytes are concatenated in big-endian order with @ret_be.
1936	*/
1937	static uint64_t int_ld_mmio_beN(CPUState cpu, CPUTLBEntryFull full,
1938	uint64_t ret_be, vaddr addr, int size,
1939	int mmu_idx, MMUAccessType type, uintptr_t ra,
1940	MemoryRegion *mr, hwaddr mr_offset)
1941	{
1942	do {
1943	MemOp this_mop;
1944	unsigned this_size;
1945	uint64_t val;
1946	MemTxResult r;
1947
1948	/* Read aligned pieces up to 8 bytes. */
1949	this_mop = ctz32(size \| (int)addr \| 8);
1950	this_size = 1 << this_mop;
1951	this_mop \|= MO_BE;
1952
1953	r = memory_region_dispatch_read(mr, mr_offset, &val,
1954	this_mop, full->attrs);
1955	if (unlikely(r != MEMTX_OK)) {
1956	io_failed(cpu, full, addr, this_size, type, mmu_idx, r, ra);
1957	}
1958	if (this_size == 8) {
1959	return val;
1960	}
1961
1962	ret_be = (ret_be << (this_size * 8)) \| val;
1963	addr += this_size;
1964	mr_offset += this_size;
1965	size -= this_size;
1966	} while (size);
1967
1968	return ret_be;
1969	}
1970
1971	static uint64_t do_ld_mmio_beN(CPUState cpu, CPUTLBEntryFull full,
1972	uint64_t ret_be, vaddr addr, int size,
1973	int mmu_idx, MMUAccessType type, uintptr_t ra)
1974	{
1975	MemoryRegionSection *section;
1976	MemoryRegion *mr;
1977	hwaddr mr_offset;
1978
1979	tcg_debug_assert(size > 0 && size <= 8);
1980
1981	section = io_prepare(&mr_offset, cpu, full, addr, ra);
1982	mr = section->mr;
1983
1984	BQL_LOCK_GUARD();
1985	return int_ld_mmio_beN(cpu, full, ret_be, addr, size, mmu_idx,
1986	type, ra, mr, mr_offset);
1987	}
1988
1989	static Int128 do_ld16_mmio_beN(CPUState cpu, CPUTLBEntryFull full,
1990	uint64_t ret_be, vaddr addr, int size,
1991	int mmu_idx, uintptr_t ra)
1992	{
1993	MemoryRegionSection *section;
1994	MemoryRegion *mr;
1995	hwaddr mr_offset;
1996	uint64_t a, b;
1997
1998	tcg_debug_assert(size > 8 && size <= 16);
1999
2000	section = io_prepare(&mr_offset, cpu, full, addr, ra);
2001	mr = section->mr;
2002
2003	BQL_LOCK_GUARD();
2004	a = int_ld_mmio_beN(cpu, full, ret_be, addr, size - 8, mmu_idx,
2005	MMU_DATA_LOAD, ra, mr, mr_offset);
2006	b = int_ld_mmio_beN(cpu, full, ret_be, addr + size - 8, 8, mmu_idx,
2007	MMU_DATA_LOAD, ra, mr, mr_offset + size - 8);
2008	return int128_make128(b, a);
2009	}
2010
2011	/**
2012	* do_ld_bytes_beN
2013	* @p: translation parameters
2014	* @ret_be: accumulated data
2015	*
2016	* Load @p->size bytes from @p->haddr, which is RAM.
2017	* The bytes to concatenated in big-endian order with @ret_be.
2018	*/
2019	static uint64_t do_ld_bytes_beN(MMULookupPageData *p, uint64_t ret_be)
2020	{
2021	uint8_t *haddr = p->haddr;
2022	int i, size = p->size;
2023
2024	for (i = 0; i < size; i++) {
2025	ret_be = (ret_be << 8) \| haddr[i];
2026	}
2027	return ret_be;
2028	}
2029
2030	/**
2031	* do_ld_parts_beN
2032	* @p: translation parameters
2033	* @ret_be: accumulated data
2034	*
2035	* As do_ld_bytes_beN, but atomically on each aligned part.
2036	*/
2037	static uint64_t do_ld_parts_beN(MMULookupPageData *p, uint64_t ret_be)
2038	{
2039	void *haddr = p->haddr;
2040	int size = p->size;
2041
2042	do {
2043	uint64_t x;
2044	int n;
2045
2046	/*
2047	* Find minimum of alignment and size.
2048	* This is slightly stronger than required by MO_ATOM_SUBALIGN, which
2049	* would have only checked the low bits of addr\|size once at the start,
2050	* but is just as easy.
2051	*/
2052	switch (((uintptr_t)haddr \| size) & 7) {
2053	case 4:
2054	x = cpu_to_be32(load_atomic4(haddr));
2055	ret_be = (ret_be << 32) \| x;
2056	n = 4;
2057	break;
2058	case 2:
2059	case 6:
2060	x = cpu_to_be16(load_atomic2(haddr));
2061	ret_be = (ret_be << 16) \| x;
2062	n = 2;
2063	break;
2064	default:
2065	x = (uint8_t )haddr;
2066	ret_be = (ret_be << 8) \| x;
2067	n = 1;
2068	break;
2069	case 0:
2070	g_assert_not_reached();
2071	}
2072	haddr += n;
2073	size -= n;
2074	} while (size != 0);
2075	return ret_be;
2076	}
2077
2078	/**
2079	* do_ld_parts_be8
2080	* @p: translation parameters
2081	* @ret_be: accumulated data
2082	*
2083	* As do_ld_bytes_beN, but with one atomic load.
2084	* Eight aligned bytes are guaranteed to cover the load.
2085	*/
2086	static uint64_t do_ld_whole_be8(CPUState *cpu, uintptr_t ra,
2087	MMULookupPageData *p, uint64_t ret_be)
2088	{
2089	int o = p->addr & 7;
2090	uint64_t x = load_atomic8(p->haddr - o);
2091
2092	x = cpu_to_be64(x);
2093	x <<= o * 8;
2094	x >>= (8 - p->size) * 8;
2095	return (ret_be << (p->size * 8)) \| x;
2096	}
2097
2098	/**
2099	* do_ld_parts_be16
2100	* @p: translation parameters
2101	* @ret_be: accumulated data
2102	*
2103	* As do_ld_bytes_beN, but with one atomic load.
2104	* 16 aligned bytes are guaranteed to cover the load.
2105	*/
2106	static Int128 do_ld_whole_be16(CPUState *cpu, uintptr_t ra,
2107	MMULookupPageData *p, uint64_t ret_be)
2108	{
2109	int o = p->addr & 15;
2110	Int128 x, y = load_atomic16_or_exit(cpu, ra, p->haddr - o);
2111	int size = p->size;
2112
2113	if (!HOST_BIG_ENDIAN) {
2114	y = bswap128(y);
2115	}
2116	y = int128_lshift(y, o * 8);
2117	y = int128_urshift(y, (16 - size) * 8);
2118	x = int128_make64(ret_be);
2119	x = int128_lshift(x, size * 8);
2120	return int128_or(x, y);
2121	}
2122
2123	/*
2124	* Wrapper for the above.
2125	*/
2126	static uint64_t do_ld_beN(CPUState cpu, MMULookupPageData p,
2127	uint64_t ret_be, int mmu_idx, MMUAccessType type,
2128	MemOp mop, uintptr_t ra)
2129	{
2130	MemOp atom;
2131	unsigned tmp, half_size;
2132
2133	if (unlikely(p->flags & TLB_MMIO)) {
2134	return do_ld_mmio_beN(cpu, p->full, ret_be, p->addr, p->size,
2135	mmu_idx, type, ra);
2136	}
2137
2138	/*
2139	* It is a given that we cross a page and therefore there is no
2140	* atomicity for the load as a whole, but subobjects may need attention.
2141	*/
2142	atom = mop & MO_ATOM_MASK;
2143	switch (atom) {
2144	case MO_ATOM_SUBALIGN:
2145	return do_ld_parts_beN(p, ret_be);
2146
2147	case MO_ATOM_IFALIGN_PAIR:
2148	case MO_ATOM_WITHIN16_PAIR:
2149	tmp = mop & MO_SIZE;
2150	tmp = tmp ? tmp - 1 : 0;
2151	half_size = 1 << tmp;
2152	if (atom == MO_ATOM_IFALIGN_PAIR
2153	? p->size == half_size
2154	: p->size >= half_size) {
2155	return do_ld_whole_be8(cpu, ra, p, ret_be);
2156	}
2157	/* fall through */
2158
2159	case MO_ATOM_IFALIGN:
2160	case MO_ATOM_WITHIN16:
2161	case MO_ATOM_NONE:
2162	return do_ld_bytes_beN(p, ret_be);
2163
2164	default:
2165	g_assert_not_reached();
2166	}
2167	}
2168
2169	/*
2170	* Wrapper for the above, for 8 < size < 16.
2171	*/
2172	static Int128 do_ld16_beN(CPUState cpu, MMULookupPageData p,
2173	uint64_t a, int mmu_idx, MemOp mop, uintptr_t ra)
2174	{
2175	int size = p->size;
2176	uint64_t b;
2177	MemOp atom;
2178
2179	if (unlikely(p->flags & TLB_MMIO)) {
2180	return do_ld16_mmio_beN(cpu, p->full, a, p->addr, size, mmu_idx, ra);
2181	}
2182
2183	/*
2184	* It is a given that we cross a page and therefore there is no
2185	* atomicity for the load as a whole, but subobjects may need attention.
2186	*/
2187	atom = mop & MO_ATOM_MASK;
2188	switch (atom) {
2189	case MO_ATOM_SUBALIGN:
2190	p->size = size - 8;
2191	a = do_ld_parts_beN(p, a);
2192	p->haddr += size - 8;
2193	p->size = 8;
2194	b = do_ld_parts_beN(p, 0);
2195	break;
2196
2197	case MO_ATOM_WITHIN16_PAIR:
2198	/* Since size > 8, this is the half that must be atomic. */
2199	return do_ld_whole_be16(cpu, ra, p, a);
2200
2201	case MO_ATOM_IFALIGN_PAIR:
2202	/*
2203	* Since size > 8, both halves are misaligned,
2204	* and so neither is atomic.
2205	*/
2206	case MO_ATOM_IFALIGN:
2207	case MO_ATOM_WITHIN16:
2208	case MO_ATOM_NONE:
2209	p->size = size - 8;
2210	a = do_ld_bytes_beN(p, a);
2211	b = ldq_be_p(p->haddr + size - 8);
2212	break;
2213
2214	default:
2215	g_assert_not_reached();
2216	}
2217
2218	return int128_make128(b, a);
2219	}
2220
2221	static uint8_t do_ld_1(CPUState cpu, MMULookupPageData p, int mmu_idx,
2222	MMUAccessType type, uintptr_t ra)
2223	{
2224	if (unlikely(p->flags & TLB_MMIO)) {
2225	return do_ld_mmio_beN(cpu, p->full, 0, p->addr, 1, mmu_idx, type, ra);
2226	} else {
2227	return (uint8_t )p->haddr;
2228	}
2229	}
2230
2231	static uint16_t do_ld_2(CPUState cpu, MMULookupPageData p, int mmu_idx,
2232	MMUAccessType type, MemOp memop, uintptr_t ra)
2233	{
2234	uint16_t ret;
2235
2236	if (unlikely(p->flags & TLB_MMIO)) {
2237	ret = do_ld_mmio_beN(cpu, p->full, 0, p->addr, 2, mmu_idx, type, ra);
2238	if ((memop & MO_BSWAP) == MO_LE) {
2239	ret = bswap16(ret);
2240	}
2241	} else {
2242	/* Perform the load host endian, then swap if necessary. */
2243	ret = load_atom_2(cpu, ra, p->haddr, memop);
2244	if (memop & MO_BSWAP) {
2245	ret = bswap16(ret);
2246	}
2247	}
2248	return ret;
2249	}
2250
2251	static uint32_t do_ld_4(CPUState cpu, MMULookupPageData p, int mmu_idx,
2252	MMUAccessType type, MemOp memop, uintptr_t ra)
2253	{
2254	uint32_t ret;
2255
2256	if (unlikely(p->flags & TLB_MMIO)) {
2257	ret = do_ld_mmio_beN(cpu, p->full, 0, p->addr, 4, mmu_idx, type, ra);
2258	if ((memop & MO_BSWAP) == MO_LE) {
2259	ret = bswap32(ret);
2260	}
2261	} else {
2262	/* Perform the load host endian. */
2263	ret = load_atom_4(cpu, ra, p->haddr, memop);
2264	if (memop & MO_BSWAP) {
2265	ret = bswap32(ret);
2266	}
2267	}
2268	return ret;
2269	}
2270
2271	static uint64_t do_ld_8(CPUState cpu, MMULookupPageData p, int mmu_idx,
2272	MMUAccessType type, MemOp memop, uintptr_t ra)
2273	{
2274	uint64_t ret;
2275
2276	if (unlikely(p->flags & TLB_MMIO)) {
2277	ret = do_ld_mmio_beN(cpu, p->full, 0, p->addr, 8, mmu_idx, type, ra);
2278	if ((memop & MO_BSWAP) == MO_LE) {
2279	ret = bswap64(ret);
2280	}
2281	} else {
2282	/* Perform the load host endian. */
2283	ret = load_atom_8(cpu, ra, p->haddr, memop);
2284	if (memop & MO_BSWAP) {
2285	ret = bswap64(ret);
2286	}
2287	}
2288	return ret;
2289	}
2290
2291	static uint8_t do_ld1_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
2292	uintptr_t ra, MMUAccessType access_type)
2293	{
2294	MMULookupLocals l;
2295	bool crosspage;
2296
2297	cpu_req_mo(cpu, TCG_MO_LD_LD \| TCG_MO_ST_LD);
2298	crosspage = mmu_lookup(cpu, addr, oi, ra, access_type, &l);
2299	tcg_debug_assert(!crosspage);
2300
2301	return do_ld_1(cpu, &l.page[0], l.mmu_idx, access_type, ra);
2302	}
2303
2304	static uint16_t do_ld2_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
2305	uintptr_t ra, MMUAccessType access_type)
2306	{
2307	MMULookupLocals l;
2308	bool crosspage;
2309	uint16_t ret;
2310	uint8_t a, b;
2311
2312	cpu_req_mo(cpu, TCG_MO_LD_LD \| TCG_MO_ST_LD);
2313	crosspage = mmu_lookup(cpu, addr, oi, ra, access_type, &l);
2314	if (likely(!crosspage)) {
2315	return do_ld_2(cpu, &l.page[0], l.mmu_idx, access_type, l.memop, ra);
2316	}
2317
2318	a = do_ld_1(cpu, &l.page[0], l.mmu_idx, access_type, ra);
2319	b = do_ld_1(cpu, &l.page[1], l.mmu_idx, access_type, ra);
2320
2321	if ((l.memop & MO_BSWAP) == MO_LE) {
2322	ret = a \| (b << 8);
2323	} else {
2324	ret = b \| (a << 8);
2325	}
2326	return ret;
2327	}
2328
2329	static uint32_t do_ld4_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
2330	uintptr_t ra, MMUAccessType access_type)
2331	{
2332	MMULookupLocals l;
2333	bool crosspage;
2334	uint32_t ret;
2335
2336	cpu_req_mo(cpu, TCG_MO_LD_LD \| TCG_MO_ST_LD);
2337	crosspage = mmu_lookup(cpu, addr, oi, ra, access_type, &l);
2338	if (likely(!crosspage)) {
2339	return do_ld_4(cpu, &l.page[0], l.mmu_idx, access_type, l.memop, ra);
2340	}
2341
2342	ret = do_ld_beN(cpu, &l.page[0], 0, l.mmu_idx, access_type, l.memop, ra);
2343	ret = do_ld_beN(cpu, &l.page[1], ret, l.mmu_idx, access_type, l.memop, ra);
2344	if ((l.memop & MO_BSWAP) == MO_LE) {
2345	ret = bswap32(ret);
2346	}
2347	return ret;
2348	}
2349
2350	static uint64_t do_ld8_mmu(CPUState *cpu, vaddr addr, MemOpIdx oi,
2351	uintptr_t ra, MMUAccessType access_type)
2352	{
2353	MMULookupLocals l;
2354	bool crosspage;
2355	uint64_t ret;
2356
2357	cpu_req_mo(cpu, TCG_MO_LD_LD \| TCG_MO_ST_LD);
2358	crosspage = mmu_lookup(cpu, addr, oi, ra, access_type, &l);
2359	if (likely(!crosspage)) {
2360	return do_ld_8(cpu, &l.page[0], l.mmu_idx, access_type, l.memop, ra);
2361	}
2362
2363	ret = do_ld_beN(cpu, &l.page[0], 0, l.mmu_idx, access_type, l.memop, ra);
2364	ret = do_ld_beN(cpu, &l.page[1], ret, l.mmu_idx, access_type, l.memop, ra);
2365	if ((l.memop & MO_BSWAP) == MO_LE) {
2366	ret = bswap64(ret);
2367	}
2368	return ret;
2369	}
2370
2371	static Int128 do_ld16_mmu(CPUState *cpu, vaddr addr,
2372	MemOpIdx oi, uintptr_t ra)
2373	{
2374	MMULookupLocals l;
2375	bool crosspage;
2376	uint64_t a, b;
2377	Int128 ret;
2378	int first;
2379
2380	cpu_req_mo(cpu, TCG_MO_LD_LD \| TCG_MO_ST_LD);
2381	crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_LOAD, &l);
2382	if (likely(!crosspage)) {
2383	if (unlikely(l.page[0].flags & TLB_MMIO)) {
2384	ret = do_ld16_mmio_beN(cpu, l.page[0].full, 0, addr, 16,
2385	l.mmu_idx, ra);
2386	if ((l.memop & MO_BSWAP) == MO_LE) {
2387	ret = bswap128(ret);
2388	}
2389	} else {
2390	/* Perform the load host endian. */
2391	ret = load_atom_16(cpu, ra, l.page[0].haddr, l.memop);
2392	if (l.memop & MO_BSWAP) {
2393	ret = bswap128(ret);
2394	}
2395	}
2396	return ret;
2397	}
2398
2399	first = l.page[0].size;
2400	if (first == 8) {
2401	MemOp mop8 = (l.memop & ~MO_SIZE) \| MO_64;
2402
2403	a = do_ld_8(cpu, &l.page[0], l.mmu_idx, MMU_DATA_LOAD, mop8, ra);
2404	b = do_ld_8(cpu, &l.page[1], l.mmu_idx, MMU_DATA_LOAD, mop8, ra);
2405	if ((mop8 & MO_BSWAP) == MO_LE) {
2406	ret = int128_make128(a, b);
2407	} else {
2408	ret = int128_make128(b, a);
2409	}
2410	return ret;
2411	}
2412
2413	if (first < 8) {
2414	a = do_ld_beN(cpu, &l.page[0], 0, l.mmu_idx,
2415	MMU_DATA_LOAD, l.memop, ra);
2416	ret = do_ld16_beN(cpu, &l.page[1], a, l.mmu_idx, l.memop, ra);
2417	} else {
2418	ret = do_ld16_beN(cpu, &l.page[0], 0, l.mmu_idx, l.memop, ra);
2419	b = int128_getlo(ret);
2420	ret = int128_lshift(ret, l.page[1].size * 8);
2421	a = int128_gethi(ret);
2422	b = do_ld_beN(cpu, &l.page[1], b, l.mmu_idx,
2423	MMU_DATA_LOAD, l.memop, ra);
2424	ret = int128_make128(b, a);
2425	}
2426	if ((l.memop & MO_BSWAP) == MO_LE) {
2427	ret = bswap128(ret);
2428	}
2429	return ret;
2430	}
2431
2432	/*
2433	* Store Helpers
2434	*/
2435
2436	/**
2437	* do_st_mmio_leN:
2438	* @cpu: generic cpu state
2439	* @full: page parameters
2440	* @val_le: data to store
2441	* @addr: virtual address
2442	* @size: number of bytes
2443	* @mmu_idx: virtual address context
2444	* @ra: return address into tcg generated code, or 0
2445	* Context: BQL held
2446	*
2447	* Store @size bytes at @addr, which is memory-mapped i/o.
2448	* The bytes to store are extracted in little-endian order from @val_le;
2449	* return the bytes of @val_le beyond @p->size that have not been stored.
2450	*/
2451	static uint64_t int_st_mmio_leN(CPUState cpu, CPUTLBEntryFull full,
2452	uint64_t val_le, vaddr addr, int size,
2453	int mmu_idx, uintptr_t ra,
2454	MemoryRegion *mr, hwaddr mr_offset)
2455	{
2456	do {
2457	MemOp this_mop;
2458	unsigned this_size;
2459	MemTxResult r;
2460
2461	/* Store aligned pieces up to 8 bytes. */
2462	this_mop = ctz32(size \| (int)addr \| 8);
2463	this_size = 1 << this_mop;
2464	this_mop \|= MO_LE;
2465
2466	r = memory_region_dispatch_write(mr, mr_offset, val_le,
2467	this_mop, full->attrs);
2468	if (unlikely(r != MEMTX_OK)) {
2469	io_failed(cpu, full, addr, this_size, MMU_DATA_STORE,
2470	mmu_idx, r, ra);
2471	}
2472	if (this_size == 8) {
2473	return 0;
2474	}
2475
2476	val_le >>= this_size * 8;
2477	addr += this_size;
2478	mr_offset += this_size;
2479	size -= this_size;
2480	} while (size);
2481
2482	return val_le;
2483	}
2484
2485	static uint64_t do_st_mmio_leN(CPUState cpu, CPUTLBEntryFull full,
2486	uint64_t val_le, vaddr addr, int size,
2487	int mmu_idx, uintptr_t ra)
2488	{
2489	MemoryRegionSection *section;
2490	hwaddr mr_offset;
2491	MemoryRegion *mr;
2492
2493	tcg_debug_assert(size > 0 && size <= 8);
2494
2495	section = io_prepare(&mr_offset, cpu, full, addr, ra);
2496	mr = section->mr;
2497
2498	BQL_LOCK_GUARD();
2499	return int_st_mmio_leN(cpu, full, val_le, addr, size, mmu_idx,
2500	ra, mr, mr_offset);
2501	}
2502
2503	static uint64_t do_st16_mmio_leN(CPUState cpu, CPUTLBEntryFull full,
2504	Int128 val_le, vaddr addr, int size,
2505	int mmu_idx, uintptr_t ra)
2506	{
2507	MemoryRegionSection *section;
2508	MemoryRegion *mr;
2509	hwaddr mr_offset;
2510
2511	tcg_debug_assert(size > 8 && size <= 16);
2512
2513	section = io_prepare(&mr_offset, cpu, full, addr, ra);
2514	mr = section->mr;
2515
2516	BQL_LOCK_GUARD();
2517	int_st_mmio_leN(cpu, full, int128_getlo(val_le), addr, 8,
2518	mmu_idx, ra, mr, mr_offset);
2519	return int_st_mmio_leN(cpu, full, int128_gethi(val_le), addr + 8,
2520	size - 8, mmu_idx, ra, mr, mr_offset + 8);
2521	}
2522
2523	/*
2524	* Wrapper for the above.
2525	*/
2526	static uint64_t do_st_leN(CPUState cpu, MMULookupPageData p,
2527	uint64_t val_le, int mmu_idx,
2528	MemOp mop, uintptr_t ra)
2529	{
2530	MemOp atom;
2531	unsigned tmp, half_size;
2532
2533	if (unlikely(p->flags & TLB_MMIO)) {
2534	return do_st_mmio_leN(cpu, p->full, val_le, p->addr,
2535	p->size, mmu_idx, ra);
2536	} else if (unlikely(p->flags & TLB_DISCARD_WRITE)) {
2537	return val_le >> (p->size * 8);
2538	}
2539
2540	/*
2541	* It is a given that we cross a page and therefore there is no atomicity
2542	* for the store as a whole, but subobjects may need attention.
2543	*/
2544	atom = mop & MO_ATOM_MASK;
2545	switch (atom) {
2546	case MO_ATOM_SUBALIGN:
2547	return store_parts_leN(p->haddr, p->size, val_le);
2548
2549	case MO_ATOM_IFALIGN_PAIR:
2550	case MO_ATOM_WITHIN16_PAIR:
2551	tmp = mop & MO_SIZE;
2552	tmp = tmp ? tmp - 1 : 0;
2553	half_size = 1 << tmp;
2554	if (atom == MO_ATOM_IFALIGN_PAIR
2555	? p->size == half_size
2556	: p->size >= half_size) {
2557	return store_whole_le8(p->haddr, p->size, val_le);
2558	}
2559	/* fall through */
2560
2561	case MO_ATOM_IFALIGN:
2562	case MO_ATOM_WITHIN16:
2563	case MO_ATOM_NONE:
2564	return store_bytes_leN(p->haddr, p->size, val_le);
2565
2566	default:
2567	g_assert_not_reached();
2568	}
2569	}
2570
2571	/*
2572	* Wrapper for the above, for 8 < size < 16.
2573	*/
2574	static uint64_t do_st16_leN(CPUState cpu, MMULookupPageData p,
2575	Int128 val_le, int mmu_idx,
2576	MemOp mop, uintptr_t ra)
2577	{
2578	int size = p->size;
2579	MemOp atom;
2580
2581	if (unlikely(p->flags & TLB_MMIO)) {
2582	return do_st16_mmio_leN(cpu, p->full, val_le, p->addr,
2583	size, mmu_idx, ra);
2584	} else if (unlikely(p->flags & TLB_DISCARD_WRITE)) {
2585	return int128_gethi(val_le) >> ((size - 8) * 8);
2586	}
2587
2588	/*
2589	* It is a given that we cross a page and therefore there is no atomicity
2590	* for the store as a whole, but subobjects may need attention.
2591	*/
2592	atom = mop & MO_ATOM_MASK;
2593	switch (atom) {
2594	case MO_ATOM_SUBALIGN:
2595	store_parts_leN(p->haddr, 8, int128_getlo(val_le));
2596	return store_parts_leN(p->haddr + 8, p->size - 8,
2597	int128_gethi(val_le));
2598
2599	case MO_ATOM_WITHIN16_PAIR:
2600	/* Since size > 8, this is the half that must be atomic. */
2601	if (!HAVE_CMPXCHG128) {
2602	cpu_loop_exit_atomic(cpu, ra);
2603	}
2604	return store_whole_le16(p->haddr, p->size, val_le);
2605
2606	case MO_ATOM_IFALIGN_PAIR:
2607	/*
2608	* Since size > 8, both halves are misaligned,
2609	* and so neither is atomic.
2610	*/
2611	case MO_ATOM_IFALIGN:
2612	case MO_ATOM_WITHIN16:
2613	case MO_ATOM_NONE:
2614	stq_le_p(p->haddr, int128_getlo(val_le));
2615	return store_bytes_leN(p->haddr + 8, p->size - 8,
2616	int128_gethi(val_le));
2617
2618	default:
2619	g_assert_not_reached();
2620	}
2621	}
2622
2623	static void do_st_1(CPUState cpu, MMULookupPageData p, uint8_t val,
2624	int mmu_idx, uintptr_t ra)
2625	{
2626	if (unlikely(p->flags & TLB_MMIO)) {
2627	do_st_mmio_leN(cpu, p->full, val, p->addr, 1, mmu_idx, ra);
2628	} else if (unlikely(p->flags & TLB_DISCARD_WRITE)) {
2629	/* nothing */
2630	} else {
2631	(uint8_t )p->haddr = val;
2632	}
2633	}
2634
2635	static void do_st_2(CPUState cpu, MMULookupPageData p, uint16_t val,
2636	int mmu_idx, MemOp memop, uintptr_t ra)
2637	{
2638	if (unlikely(p->flags & TLB_MMIO)) {
2639	if ((memop & MO_BSWAP) != MO_LE) {
2640	val = bswap16(val);
2641	}
2642	do_st_mmio_leN(cpu, p->full, val, p->addr, 2, mmu_idx, ra);
2643	} else if (unlikely(p->flags & TLB_DISCARD_WRITE)) {
2644	/* nothing */
2645	} else {
2646	/* Swap to host endian if necessary, then store. */
2647	if (memop & MO_BSWAP) {
2648	val = bswap16(val);
2649	}
2650	store_atom_2(cpu, ra, p->haddr, memop, val);
2651	}
2652	}
2653
2654	static void do_st_4(CPUState cpu, MMULookupPageData p, uint32_t val,
2655	int mmu_idx, MemOp memop, uintptr_t ra)
2656	{
2657	if (unlikely(p->flags & TLB_MMIO)) {
2658	if ((memop & MO_BSWAP) != MO_LE) {
2659	val = bswap32(val);
2660	}
2661	do_st_mmio_leN(cpu, p->full, val, p->addr, 4, mmu_idx, ra);
2662	} else if (unlikely(p->flags & TLB_DISCARD_WRITE)) {
2663	/* nothing */
2664	} else {
2665	/* Swap to host endian if necessary, then store. */
2666	if (memop & MO_BSWAP) {
2667	val = bswap32(val);
2668	}
2669	store_atom_4(cpu, ra, p->haddr, memop, val);
2670	}
2671	}
2672
2673	static void do_st_8(CPUState cpu, MMULookupPageData p, uint64_t val,
2674	int mmu_idx, MemOp memop, uintptr_t ra)
2675	{
2676	if (unlikely(p->flags & TLB_MMIO)) {
2677	if ((memop & MO_BSWAP) != MO_LE) {
2678	val = bswap64(val);
2679	}
2680	do_st_mmio_leN(cpu, p->full, val, p->addr, 8, mmu_idx, ra);
2681	} else if (unlikely(p->flags & TLB_DISCARD_WRITE)) {
2682	/* nothing */
2683	} else {
2684	/* Swap to host endian if necessary, then store. */
2685	if (memop & MO_BSWAP) {
2686	val = bswap64(val);
2687	}
2688	store_atom_8(cpu, ra, p->haddr, memop, val);
2689	}
2690	}
2691
2692	static void do_st1_mmu(CPUState *cpu, vaddr addr, uint8_t val,
2693	MemOpIdx oi, uintptr_t ra)
2694	{
2695	MMULookupLocals l;
2696	bool crosspage;
2697
2698	cpu_req_mo(cpu, TCG_MO_LD_ST \| TCG_MO_ST_ST);
2699	crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l);
2700	tcg_debug_assert(!crosspage);
2701
2702	do_st_1(cpu, &l.page[0], val, l.mmu_idx, ra);
2703	}
2704
2705	static void do_st2_mmu(CPUState *cpu, vaddr addr, uint16_t val,
2706	MemOpIdx oi, uintptr_t ra)
2707	{
2708	MMULookupLocals l;
2709	bool crosspage;
2710	uint8_t a, b;
2711
2712	cpu_req_mo(cpu, TCG_MO_LD_ST \| TCG_MO_ST_ST);
2713	crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l);
2714	if (likely(!crosspage)) {
2715	do_st_2(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra);
2716	return;
2717	}
2718
2719	if ((l.memop & MO_BSWAP) == MO_LE) {
2720	a = val, b = val >> 8;
2721	} else {
2722	b = val, a = val >> 8;
2723	}
2724	do_st_1(cpu, &l.page[0], a, l.mmu_idx, ra);
2725	do_st_1(cpu, &l.page[1], b, l.mmu_idx, ra);
2726	}
2727
2728	static void do_st4_mmu(CPUState *cpu, vaddr addr, uint32_t val,
2729	MemOpIdx oi, uintptr_t ra)
2730	{
2731	MMULookupLocals l;
2732	bool crosspage;
2733
2734	cpu_req_mo(cpu, TCG_MO_LD_ST \| TCG_MO_ST_ST);
2735	crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l);
2736	if (likely(!crosspage)) {
2737	do_st_4(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra);
2738	return;
2739	}
2740
2741	/* Swap to little endian for simplicity, then store by bytes. */
2742	if ((l.memop & MO_BSWAP) != MO_LE) {
2743	val = bswap32(val);
2744	}
2745	val = do_st_leN(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra);
2746	(void) do_st_leN(cpu, &l.page[1], val, l.mmu_idx, l.memop, ra);
2747	}
2748
2749	static void do_st8_mmu(CPUState *cpu, vaddr addr, uint64_t val,
2750	MemOpIdx oi, uintptr_t ra)
2751	{
2752	MMULookupLocals l;
2753	bool crosspage;
2754
2755	cpu_req_mo(cpu, TCG_MO_LD_ST \| TCG_MO_ST_ST);
2756	crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l);
2757	if (likely(!crosspage)) {
2758	do_st_8(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra);
2759	return;
2760	}
2761
2762	/* Swap to little endian for simplicity, then store by bytes. */
2763	if ((l.memop & MO_BSWAP) != MO_LE) {
2764	val = bswap64(val);
2765	}
2766	val = do_st_leN(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra);
2767	(void) do_st_leN(cpu, &l.page[1], val, l.mmu_idx, l.memop, ra);
2768	}
2769
2770	static void do_st16_mmu(CPUState *cpu, vaddr addr, Int128 val,
2771	MemOpIdx oi, uintptr_t ra)
2772	{
2773	MMULookupLocals l;
2774	bool crosspage;
2775	uint64_t a, b;
2776	int first;
2777
2778	cpu_req_mo(cpu, TCG_MO_LD_ST \| TCG_MO_ST_ST);
2779	crosspage = mmu_lookup(cpu, addr, oi, ra, MMU_DATA_STORE, &l);
2780	if (likely(!crosspage)) {
2781	if (unlikely(l.page[0].flags & TLB_MMIO)) {
2782	if ((l.memop & MO_BSWAP) != MO_LE) {
2783	val = bswap128(val);
2784	}
2785	do_st16_mmio_leN(cpu, l.page[0].full, val, addr, 16, l.mmu_idx, ra);
2786	} else if (unlikely(l.page[0].flags & TLB_DISCARD_WRITE)) {
2787	/* nothing */
2788	} else {
2789	/* Swap to host endian if necessary, then store. */
2790	if (l.memop & MO_BSWAP) {
2791	val = bswap128(val);
2792	}
2793	store_atom_16(cpu, ra, l.page[0].haddr, l.memop, val);
2794	}
2795	return;
2796	}
2797
2798	first = l.page[0].size;
2799	if (first == 8) {
2800	MemOp mop8 = (l.memop & ~(MO_SIZE \| MO_BSWAP)) \| MO_64;
2801
2802	if (l.memop & MO_BSWAP) {
2803	val = bswap128(val);
2804	}
2805	if (HOST_BIG_ENDIAN) {
2806	b = int128_getlo(val), a = int128_gethi(val);
2807	} else {
2808	a = int128_getlo(val), b = int128_gethi(val);
2809	}
2810	do_st_8(cpu, &l.page[0], a, l.mmu_idx, mop8, ra);
2811	do_st_8(cpu, &l.page[1], b, l.mmu_idx, mop8, ra);
2812	return;
2813	}
2814
2815	if ((l.memop & MO_BSWAP) != MO_LE) {
2816	val = bswap128(val);
2817	}
2818	if (first < 8) {
2819	do_st_leN(cpu, &l.page[0], int128_getlo(val), l.mmu_idx, l.memop, ra);
2820	val = int128_urshift(val, first * 8);
2821	do_st16_leN(cpu, &l.page[1], val, l.mmu_idx, l.memop, ra);
2822	} else {
2823	b = do_st16_leN(cpu, &l.page[0], val, l.mmu_idx, l.memop, ra);
2824	do_st_leN(cpu, &l.page[1], b, l.mmu_idx, l.memop, ra);
2825	}
2826	}
2827
2828	#include "ldst_common.c.inc"
2829
2830	/*
2831	* First set of functions passes in OI and RETADDR.
2832	* This makes them callable from other helpers.
2833	*/
2834
2835	#define ATOMIC_NAME(X) \
2836	glue(glue(glue(cpu_atomic_ ## X, SUFFIX), END), _mmu)
2837
2838	#define ATOMIC_MMU_CLEANUP
2839
2840	#include "atomic_common.c.inc"
2841
2842	#define DATA_SIZE 1
2843	#include "atomic_template.h"
2844
2845	#define DATA_SIZE 2
2846	#include "atomic_template.h"
2847
2848	#define DATA_SIZE 4
2849	#include "atomic_template.h"
2850
2851	#define DATA_SIZE 8
2852	#include "atomic_template.h"
2853
2854	#if defined(CONFIG_ATOMIC128) \|\| HAVE_CMPXCHG128
2855	#define DATA_SIZE 16
2856	#include "atomic_template.h"
2857	#endif
2858
2859	/* Code access functions. */
2860
2861	uint8_t cpu_ldb_code_mmu(CPUArchState *env, vaddr addr,
2862	MemOpIdx oi, uintptr_t retaddr)
2863	{
2864	return do_ld1_mmu(env_cpu(env), addr, oi, retaddr, MMU_INST_FETCH);
2865	}
2866
2867	uint16_t cpu_ldw_code_mmu(CPUArchState *env, vaddr addr,
2868	MemOpIdx oi, uintptr_t retaddr)
2869	{
2870	return do_ld2_mmu(env_cpu(env), addr, oi, retaddr, MMU_INST_FETCH);
2871	}
2872
2873	uint32_t cpu_ldl_code_mmu(CPUArchState *env, vaddr addr,
2874	MemOpIdx oi, uintptr_t retaddr)
2875	{
2876	return do_ld4_mmu(env_cpu(env), addr, oi, retaddr, MMU_INST_FETCH);
2877	}
2878
2879	uint64_t cpu_ldq_code_mmu(CPUArchState *env, vaddr addr,
2880	MemOpIdx oi, uintptr_t retaddr)
2881	{
2882	return do_ld8_mmu(env_cpu(env), addr, oi, retaddr, MMU_INST_FETCH);
2883	}
2884
2885	/*
2886	* Common pointer_wrap implementations.
2887	*/
2888
2889	/*
2890	* To be used for strict alignment targets.
2891	* Because no accesses are unaligned, no accesses wrap either.
2892	*/
2893	vaddr cpu_pointer_wrap_notreached(CPUState *cs, int idx, vaddr res, vaddr base)
2894	{
2895	g_assert_not_reached();
2896	}
2897
2898	/* To be used for strict 32-bit targets. */
2899	vaddr cpu_pointer_wrap_uint32(CPUState *cs, int idx, vaddr res, vaddr base)
2900	{
2901	return (uint32_t)res;
2902	}