@samitouri / QOSamiQemu
Code Commits Issues Pulls Sessions CI/CD
master
master
h 311 lines 11.2 KB
Raw
1 /*
2 * ARM SVE Load/Store Helpers
3 *
4 * Copyright (c) 2018-2022 Linaro
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19
20 #ifndef TARGET_ARM_SVE_LDST_INTERNAL_H
21 #define TARGET_ARM_SVE_LDST_INTERNAL_H
22
23 #include "accel/tcg/cpu-ldst.h"
24
25 /*
26 * Load one element into @vd + @reg_off from @host.
27 * The controlling predicate is known to be true.
28 */
29 typedef void sve_ldst1_host_fn(void *vd, intptr_t reg_off, void *host);
30
31 /*
32 * Load one element into @vd + @reg_off from (@env, @vaddr, @ra).
33 * The controlling predicate is known to be true.
34 */
35 typedef void sve_ldst1_tlb_fn(CPUARMState *env, void *vd, intptr_t reg_off,
36 target_ulong vaddr, uintptr_t retaddr);
37
38 /*
39 * Generate the above primitives.
40 */
41
42 #define DO_LD_HOST(NAME, H, TYPEE, TYPEM, HOST) \
43 static inline void sve_##NAME##_host(void *vd, intptr_t reg_off, void *host) \
44 { TYPEM val = HOST(host); *(TYPEE *)(vd + H(reg_off)) = val; }
45
46 #define DO_ST_HOST(NAME, H, TYPEE, TYPEM, HOST) \
47 static inline void sve_##NAME##_host(void *vd, intptr_t reg_off, void *host) \
48 { TYPEM val = *(TYPEE *)(vd + H(reg_off)); HOST(host, val); }
49
50 #define DO_LD_TLB(NAME, H, TYPEE, TYPEM, TLB) \
51 static inline void sve_##NAME##_tlb(CPUARMState *env, void *vd, \
52 intptr_t reg_off, target_ulong addr, uintptr_t ra) \
53 { \
54 TYPEM val = TLB(env, useronly_clean_ptr(addr), ra); \
55 *(TYPEE *)(vd + H(reg_off)) = val; \
56 }
57
58 #define DO_ST_TLB(NAME, H, TYPEE, TYPEM, TLB) \
59 static inline void sve_##NAME##_tlb(CPUARMState *env, void *vd, \
60 intptr_t reg_off, target_ulong addr, uintptr_t ra) \
61 { \
62 TYPEM val = *(TYPEE *)(vd + H(reg_off)); \
63 TLB(env, useronly_clean_ptr(addr), val, ra); \
64 }
65
66 #define DO_LD_PRIM_1(NAME, H, TE, TM) \
67 DO_LD_HOST(NAME, H, TE, TM, ldub_p) \
68 DO_LD_TLB(NAME, H, TE, TM, cpu_ldub_data_ra)
69
70 DO_LD_PRIM_1(ld1bb, H1, uint8_t, uint8_t)
71 DO_LD_PRIM_1(ld1bhu, H1_2, uint16_t, uint8_t)
72 DO_LD_PRIM_1(ld1bhs, H1_2, uint16_t, int8_t)
73 DO_LD_PRIM_1(ld1bsu, H1_4, uint32_t, uint8_t)
74 DO_LD_PRIM_1(ld1bss, H1_4, uint32_t, int8_t)
75 DO_LD_PRIM_1(ld1bdu, H1_8, uint64_t, uint8_t)
76 DO_LD_PRIM_1(ld1bds, H1_8, uint64_t, int8_t)
77
78 #define DO_ST_PRIM_1(NAME, H, TE, TM) \
79 DO_ST_HOST(st1##NAME, H, TE, TM, stb_p) \
80 DO_ST_TLB(st1##NAME, H, TE, TM, cpu_stb_data_ra)
81
82 DO_ST_PRIM_1(bb, H1, uint8_t, uint8_t)
83 DO_ST_PRIM_1(bh, H1_2, uint16_t, uint8_t)
84 DO_ST_PRIM_1(bs, H1_4, uint32_t, uint8_t)
85 DO_ST_PRIM_1(bd, H1_8, uint64_t, uint8_t)
86
87 #define DO_LD_PRIM_2(NAME, H, TE, TM, LD) \
88 DO_LD_HOST(ld1##NAME##_be, H, TE, TM, LD##_be_p) \
89 DO_LD_HOST(ld1##NAME##_le, H, TE, TM, LD##_le_p) \
90 DO_LD_TLB(ld1##NAME##_be, H, TE, TM, cpu_##LD##_be_data_ra) \
91 DO_LD_TLB(ld1##NAME##_le, H, TE, TM, cpu_##LD##_le_data_ra)
92
93 #define DO_ST_PRIM_2(NAME, H, TE, TM, ST) \
94 DO_ST_HOST(st1##NAME##_be, H, TE, TM, ST##_be_p) \
95 DO_ST_HOST(st1##NAME##_le, H, TE, TM, ST##_le_p) \
96 DO_ST_TLB(st1##NAME##_be, H, TE, TM, cpu_##ST##_be_data_ra) \
97 DO_ST_TLB(st1##NAME##_le, H, TE, TM, cpu_##ST##_le_data_ra)
98
99 DO_LD_PRIM_2(hh, H1_2, uint16_t, uint16_t, lduw)
100 DO_LD_PRIM_2(hsu, H1_4, uint32_t, uint16_t, lduw)
101 DO_LD_PRIM_2(hss, H1_4, uint32_t, int16_t, lduw)
102 DO_LD_PRIM_2(hdu, H1_8, uint64_t, uint16_t, lduw)
103 DO_LD_PRIM_2(hds, H1_8, uint64_t, int16_t, lduw)
104
105 DO_ST_PRIM_2(hh, H1_2, uint16_t, uint16_t, stw)
106 DO_ST_PRIM_2(hs, H1_4, uint32_t, uint16_t, stw)
107 DO_ST_PRIM_2(hd, H1_8, uint64_t, uint16_t, stw)
108
109 DO_LD_PRIM_2(ss, H1_4, uint32_t, uint32_t, ldl)
110 DO_LD_PRIM_2(sdu, H1_8, uint64_t, uint32_t, ldl)
111 DO_LD_PRIM_2(sds, H1_8, uint64_t, int32_t, ldl)
112
113 DO_ST_PRIM_2(ss, H1_4, uint32_t, uint32_t, stl)
114 DO_ST_PRIM_2(sd, H1_8, uint64_t, uint32_t, stl)
115
116 DO_LD_PRIM_2(dd, H1_8, uint64_t, uint64_t, ldq)
117 DO_ST_PRIM_2(dd, H1_8, uint64_t, uint64_t, stq)
118
119 #define DO_LD_PRIM_3(NAME, FUNC) \
120 static inline void sve_##NAME##_host(void *vd, \
121 intptr_t reg_off, void *host) \
122 { sve_##FUNC##_host(vd, reg_off, host); \
123 *(uint64_t *)(vd + reg_off + 8) = 0; } \
124 static inline void sve_##NAME##_tlb(CPUARMState *env, void *vd, \
125 intptr_t reg_off, target_ulong addr, uintptr_t ra) \
126 { sve_##FUNC##_tlb(env, vd, reg_off, addr, ra); \
127 *(uint64_t *)(vd + reg_off + 8) = 0; }
128
129 DO_LD_PRIM_3(ld1squ_be, ld1sdu_be)
130 DO_LD_PRIM_3(ld1squ_le, ld1sdu_le)
131 DO_LD_PRIM_3(ld1dqu_be, ld1dd_be)
132 DO_LD_PRIM_3(ld1dqu_le, ld1dd_le)
133
134 #define sve_st1sq_be_host sve_st1sd_be_host
135 #define sve_st1sq_le_host sve_st1sd_le_host
136 #define sve_st1sq_be_tlb sve_st1sd_be_tlb
137 #define sve_st1sq_le_tlb sve_st1sd_le_tlb
138
139 #define sve_st1dq_be_host sve_st1dd_be_host
140 #define sve_st1dq_le_host sve_st1dd_le_host
141 #define sve_st1dq_be_tlb sve_st1dd_be_tlb
142 #define sve_st1dq_le_tlb sve_st1dd_le_tlb
143
144 /*
145 * The ARMVectorReg elements are stored in host-endian 64-bit units.
146 * For 128-bit quantities, the sequence defined by the Elem[] pseudocode
147 * corresponds to storing the two 64-bit pieces in little-endian order.
148 */
149 /* FIXME: Nothing in this file makes any effort at atomicity. */
150
151 static inline void sve_ld1qq_be_host(void *vd, intptr_t reg_off, void *host)
152 {
153 sve_ld1dd_be_host(vd, reg_off + 8, host);
154 sve_ld1dd_be_host(vd, reg_off, host + 8);
155 }
156
157 static inline void sve_ld1qq_le_host(void *vd, intptr_t reg_off, void *host)
158 {
159 sve_ld1dd_le_host(vd, reg_off, host);
160 sve_ld1dd_le_host(vd, reg_off + 8, host + 8);
161 }
162
163 static inline void
164 sve_ld1qq_be_tlb(CPUARMState *env, void *vd, intptr_t reg_off,
165 target_ulong addr, uintptr_t ra)
166 {
167 sve_ld1dd_be_tlb(env, vd, reg_off + 8, addr, ra);
168 sve_ld1dd_be_tlb(env, vd, reg_off, addr + 8, ra);
169 }
170
171 static inline void
172 sve_ld1qq_le_tlb(CPUARMState *env, void *vd, intptr_t reg_off,
173 target_ulong addr, uintptr_t ra)
174 {
175 sve_ld1dd_le_tlb(env, vd, reg_off, addr, ra);
176 sve_ld1dd_le_tlb(env, vd, reg_off + 8, addr + 8, ra);
177 }
178
179 static inline void sve_st1qq_be_host(void *vd, intptr_t reg_off, void *host)
180 {
181 sve_st1dd_be_host(vd, reg_off + 8, host);
182 sve_st1dd_be_host(vd, reg_off, host + 8);
183 }
184
185 static inline void sve_st1qq_le_host(void *vd, intptr_t reg_off, void *host)
186 {
187 sve_st1dd_le_host(vd, reg_off, host);
188 sve_st1dd_le_host(vd, reg_off + 8, host + 8);
189 }
190
191 static inline void
192 sve_st1qq_be_tlb(CPUARMState *env, void *vd, intptr_t reg_off,
193 target_ulong addr, uintptr_t ra)
194 {
195 sve_st1dd_be_tlb(env, vd, reg_off + 8, addr, ra);
196 sve_st1dd_be_tlb(env, vd, reg_off, addr + 8, ra);
197 }
198
199 static inline void
200 sve_st1qq_le_tlb(CPUARMState *env, void *vd, intptr_t reg_off,
201 target_ulong addr, uintptr_t ra)
202 {
203 sve_st1dd_le_tlb(env, vd, reg_off, addr, ra);
204 sve_st1dd_le_tlb(env, vd, reg_off + 8, addr + 8, ra);
205 }
206
207 #undef DO_LD_TLB
208 #undef DO_ST_TLB
209 #undef DO_LD_HOST
210 #undef DO_LD_PRIM_1
211 #undef DO_ST_PRIM_1
212 #undef DO_LD_PRIM_2
213 #undef DO_ST_PRIM_2
214 #undef DO_LD_PRIM_3
215
216 /*
217 * Resolve the guest virtual address to info->host and info->flags.
218 * If @nofault, return false if the page is invalid, otherwise
219 * exit via page fault exception.
220 */
221
222 typedef struct {
223 void *host;
224 int flags;
225 MemTxAttrs attrs;
226 bool tagged;
227 } SVEHostPage;
228
229 bool sve_probe_page(SVEHostPage *info, bool nofault, CPUARMState *env,
230 target_ulong addr, int mem_off, MMUAccessType access_type,
231 int mmu_idx, uintptr_t retaddr);
232
233 /*
234 * Analyse contiguous data, protected by a governing predicate.
235 */
236
237 typedef enum {
238 FAULT_NO,
239 FAULT_FIRST,
240 FAULT_ALL,
241 } SVEContFault;
242
243 typedef struct {
244 /*
245 * First and last element wholly contained within the two pages.
246 * mem_off_first[0] and reg_off_first[0] are always set >= 0.
247 * reg_off_last[0] may be < 0 if the first element crosses pages.
248 * All of mem_off_first[1], reg_off_first[1] and reg_off_last[1]
249 * are set >= 0 only if there are complete elements on a second page.
250 *
251 * The reg_off_* offsets are relative to the internal vector register.
252 * The mem_off_first offset is relative to the memory address; the
253 * two offsets are different when a load operation extends, a store
254 * operation truncates, or for multi-register operations.
255 */
256 int16_t mem_off_first[2];
257 int16_t reg_off_first[2];
258 int16_t reg_off_last[2];
259
260 /*
261 * One element that is misaligned and spans both pages,
262 * or -1 if there is no such active element.
263 */
264 int16_t mem_off_split;
265 int16_t reg_off_split;
266
267 /*
268 * The byte offset at which the entire operation crosses a page boundary.
269 * Set >= 0 if and only if the entire operation spans two pages.
270 */
271 int16_t page_split;
272
273 /* TLB data for the two pages. */
274 SVEHostPage page[2];
275 } SVEContLdSt;
276
277 /*
278 * Find first active element on each page, and a loose bound for the
279 * final element on each page. Identify any single element that spans
280 * the page boundary. Return true if there are any active elements.
281 */
282 bool sve_cont_ldst_elements(SVEContLdSt *info, target_ulong addr, uint64_t *vg,
283 intptr_t reg_max, int esz, int msize);
284
285 /*
286 * Resolve the guest virtual addresses to info->page[].
287 * Control the generation of page faults with @fault. Return false if
288 * there is no work to do, which can only happen with @fault == FAULT_NO.
289 */
290 bool sve_cont_ldst_pages(SVEContLdSt *info, SVEContFault fault,
291 CPUARMState *env, target_ulong addr,
292 MMUAccessType access_type, uintptr_t retaddr);
293
294 #ifdef CONFIG_USER_ONLY
295 static inline void
296 sve_cont_ldst_watchpoints(SVEContLdSt *info, CPUARMState *env, uint64_t *vg,
297 target_ulong addr, int esize, int msize,
298 int wp_access, uintptr_t retaddr)
299 { }
300 #else
301 void sve_cont_ldst_watchpoints(SVEContLdSt *info, CPUARMState *env,
302 uint64_t *vg, target_ulong addr,
303 int esize, int msize, int wp_access,
304 uintptr_t retaddr);
305 #endif
306
307 void sve_cont_ldst_mte_check(SVEContLdSt *info, CPUARMState *env, uint64_t *vg,
308 target_ulong addr, int esize, int msize,
309 uint32_t mtedesc, uintptr_t ra);
310
311 #endif /* TARGET_ARM_SVE_LDST_INTERNAL_H */