@samitouri / QOSamiQemu
Code Commits Issues Pulls Sessions CI/CD
master
master
c 2,918 lines 84.7 KB
Raw
1 /*
2 * S/390 memory access helper routines
3 *
4 * Copyright (c) 2009 Ulrich Hecht
5 * Copyright (c) 2009 Alexander Graf
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 2.1 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 */
20
21 #include "qemu/osdep.h"
22 #include "qemu/log.h"
23 #include "cpu.h"
24 #include "s390x-internal.h"
25 #include "tcg_s390x.h"
26 #include "exec/helper-proto.h"
27 #include "exec/cpu-common.h"
28 #include "exec/cputlb.h"
29 #include "exec/page-protection.h"
30 #include "accel/tcg/cpu-ldst.h"
31 #include "accel/tcg/cpu-loop.h"
32 #include "accel/tcg/probe.h"
33 #include "exec/target_page.h"
34 #include "exec/tlb-flags.h"
35 #include "accel/tcg/cpu-ops.h"
36 #include "accel/tcg/helper-retaddr.h"
37 #include "qemu/int128.h"
38 #include "qemu/atomic128.h"
39
40 #if defined(CONFIG_USER_ONLY)
41 #include "user/page-protection.h"
42 #else
43 #include "hw/s390x/storage-keys.h"
44 #include "hw/core/boards.h"
45 #include "system/memory.h"
46 #endif
47
48 #ifdef CONFIG_USER_ONLY
49 # define user_or_likely(X) true
50 #else
51 # define user_or_likely(X) likely(X)
52 #endif
53
54 /*****************************************************************************/
55 /* Softmmu support */
56
57 /* #define DEBUG_HELPER */
58 #ifdef DEBUG_HELPER
59 #define HELPER_LOG(x...) qemu_log(x)
60 #else
61 #define HELPER_LOG(x...)
62 #endif
63
64 static inline bool psw_key_valid(CPUS390XState *env, uint8_t psw_key)
65 {
66 uint16_t pkm = env->cregs[3] >> 16;
67
68 if (env->psw.mask & PSW_MASK_PSTATE) {
69 /* PSW key has range 0..15, it is valid if the bit is 1 in the PKM */
70 return pkm & (0x8000 >> psw_key);
71 }
72 return true;
73 }
74
75 static bool is_destructive_overlap(CPUS390XState *env, uint64_t dest,
76 uint64_t src, uint32_t len)
77 {
78 if (!len || src == dest) {
79 return false;
80 }
81 /* Take care of wrapping at the end of address space. */
82 if (unlikely(wrap_address(env, src + len - 1) < src)) {
83 return dest > src || dest <= wrap_address(env, src + len - 1);
84 }
85 return dest > src && dest <= src + len - 1;
86 }
87
88 /* Trigger a SPECIFICATION exception if an address or a length is not
89 naturally aligned. */
90 static inline void check_alignment(CPUS390XState *env, uint64_t v,
91 int wordsize, uintptr_t ra)
92 {
93 if (v % wordsize) {
94 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
95 }
96 }
97
98 /* Load a value from memory according to its size. */
99 static inline uint64_t cpu_ldusize_data_ra(CPUS390XState *env, uint64_t addr,
100 int wordsize, uintptr_t ra)
101 {
102 switch (wordsize) {
103 case 1:
104 return cpu_ldub_data_ra(env, addr, ra);
105 case 2:
106 return cpu_lduw_be_data_ra(env, addr, ra);
107 default:
108 abort();
109 }
110 }
111
112 /* Store a to memory according to its size. */
113 static inline void cpu_stsize_data_ra(CPUS390XState *env, uint64_t addr,
114 uint64_t value, int wordsize,
115 uintptr_t ra)
116 {
117 switch (wordsize) {
118 case 1:
119 cpu_stb_data_ra(env, addr, value, ra);
120 break;
121 case 2:
122 cpu_stw_be_data_ra(env, addr, value, ra);
123 break;
124 default:
125 abort();
126 }
127 }
128
129 /* An access covers at most 4096 bytes and therefore at most two pages. */
130 typedef struct S390Access {
131 vaddr vaddr1;
132 vaddr vaddr2;
133 void *haddr1;
134 void *haddr2;
135 uint16_t size1;
136 uint16_t size2;
137 /*
138 * If we can't access the host page directly, we'll have to do I/O access
139 * via ld/st helpers. These are internal details, so we store the
140 * mmu idx to do the access here instead of passing it around in the
141 * helpers.
142 */
143 int mmu_idx;
144 } S390Access;
145
146 /*
147 * With nonfault=1, return the PGM_ exception that would have been injected
148 * into the guest; return 0 if no exception was detected.
149 *
150 * For !CONFIG_USER_ONLY, the TEC is stored stored to env->tlb_fill_tec.
151 * For CONFIG_USER_ONLY, the faulting address is stored to env->__excp_addr.
152 */
153 static inline int s390_probe_access(CPUArchState *env, vaddr addr,
154 int size, MMUAccessType access_type,
155 int mmu_idx, bool nonfault,
156 void **phost, uintptr_t ra)
157 {
158 int flags = probe_access_flags(env, addr, size, access_type, mmu_idx,
159 nonfault, phost, ra);
160
161 if (unlikely(flags & TLB_INVALID_MASK)) {
162 #ifdef CONFIG_USER_ONLY
163 /* Address is in TEC in system mode; see s390_cpu_record_sigsegv. */
164 env->__excp_addr = addr & TARGET_PAGE_MASK;
165 return (page_get_flags(addr) & PAGE_VALID
166 ? PGM_PROTECTION : PGM_ADDRESSING);
167 #else
168 return env->tlb_fill_exc;
169 #endif
170 }
171
172 #ifndef CONFIG_USER_ONLY
173 if (unlikely(flags & TLB_WATCHPOINT)) {
174 /* S390 does not presently use transaction attributes. */
175 cpu_check_watchpoint(env_cpu(env), addr, size,
176 MEMTXATTRS_UNSPECIFIED,
177 (access_type == MMU_DATA_STORE
178 ? BP_MEM_WRITE : BP_MEM_READ), ra);
179 }
180 #endif
181
182 return 0;
183 }
184
185 static int access_prepare_nf(S390Access *access, CPUS390XState *env,
186 bool nonfault, vaddr vaddr1, int size,
187 MMUAccessType access_type,
188 int mmu_idx, uintptr_t ra)
189 {
190 int size1, size2, exc;
191
192 assert(size > 0 && size <= 4096);
193
194 size1 = MIN(size, -(vaddr1 | TARGET_PAGE_MASK)),
195 size2 = size - size1;
196
197 memset(access, 0, sizeof(*access));
198 access->vaddr1 = vaddr1;
199 access->size1 = size1;
200 access->size2 = size2;
201 access->mmu_idx = mmu_idx;
202
203 exc = s390_probe_access(env, vaddr1, size1, access_type, mmu_idx, nonfault,
204 &access->haddr1, ra);
205 if (unlikely(exc)) {
206 return exc;
207 }
208 if (unlikely(size2)) {
209 /* The access crosses page boundaries. */
210 vaddr vaddr2 = wrap_address(env, vaddr1 + size1);
211
212 access->vaddr2 = vaddr2;
213 exc = s390_probe_access(env, vaddr2, size2, access_type, mmu_idx,
214 nonfault, &access->haddr2, ra);
215 if (unlikely(exc)) {
216 return exc;
217 }
218 }
219 return 0;
220 }
221
222 static inline void access_prepare(S390Access *ret, CPUS390XState *env,
223 vaddr vaddr, int size,
224 MMUAccessType access_type, int mmu_idx,
225 uintptr_t ra)
226 {
227 int exc = access_prepare_nf(ret, env, false, vaddr, size,
228 access_type, mmu_idx, ra);
229 assert(!exc);
230 }
231
232 /* Helper to handle memset on a single page. */
233 static void do_access_memset(CPUS390XState *env, vaddr vaddr, char *haddr,
234 uint8_t byte, uint16_t size, int mmu_idx,
235 uintptr_t ra)
236 {
237 if (user_or_likely(haddr)) {
238 memset(haddr, byte, size);
239 } else {
240 MemOpIdx oi = make_memop_idx(MO_UB, mmu_idx);
241 for (int i = 0; i < size; i++) {
242 cpu_stb_mmu(env, vaddr + i, byte, oi, ra);
243 }
244 }
245 }
246
247 static void access_memset(CPUS390XState *env, S390Access *desta,
248 uint8_t byte, uintptr_t ra)
249 {
250 set_helper_retaddr(ra);
251 do_access_memset(env, desta->vaddr1, desta->haddr1, byte, desta->size1,
252 desta->mmu_idx, ra);
253 if (unlikely(desta->size2)) {
254 do_access_memset(env, desta->vaddr2, desta->haddr2, byte,
255 desta->size2, desta->mmu_idx, ra);
256 }
257 clear_helper_retaddr();
258 }
259
260 static uint8_t access_get_byte(CPUS390XState *env, S390Access *access,
261 int offset, uintptr_t ra)
262 {
263 vaddr vaddr = access->vaddr1;
264 void *haddr = access->haddr1;
265
266 if (unlikely(offset >= access->size1)) {
267 offset -= access->size1;
268 vaddr = access->vaddr2;
269 haddr = access->haddr2;
270 }
271
272 if (user_or_likely(haddr)) {
273 return ldub_p(haddr + offset);
274 } else {
275 MemOpIdx oi = make_memop_idx(MO_UB, access->mmu_idx);
276 return cpu_ldb_mmu(env, vaddr + offset, oi, ra);
277 }
278 }
279
280 static void access_set_byte(CPUS390XState *env, S390Access *access,
281 int offset, uint8_t byte, uintptr_t ra)
282 {
283 vaddr vaddr = access->vaddr1;
284 void *haddr = access->haddr1;
285
286 if (unlikely(offset >= access->size1)) {
287 offset -= access->size1;
288 vaddr = access->vaddr2;
289 haddr = access->haddr2;
290 }
291
292 if (user_or_likely(haddr)) {
293 stb_p(haddr + offset, byte);
294 } else {
295 MemOpIdx oi = make_memop_idx(MO_UB, access->mmu_idx);
296 cpu_stb_mmu(env, vaddr + offset, byte, oi, ra);
297 }
298 }
299
300 /*
301 * Move data with the same semantics as memmove() in case ranges don't overlap
302 * or src > dest. Undefined behavior on destructive overlaps.
303 */
304 static void access_memmove(CPUS390XState *env, S390Access *desta,
305 S390Access *srca, uintptr_t ra)
306 {
307 int len = desta->size1 + desta->size2;
308
309 assert(len == srca->size1 + srca->size2);
310
311 /* Fallback to slow access in case we don't have access to all host pages */
312 if (user_or_likely(desta->haddr1 &&
313 srca->haddr1 &&
314 (!desta->size2 || desta->haddr2) &&
315 (!srca->size2 || srca->haddr2))) {
316 int diff = desta->size1 - srca->size1;
317
318 if (likely(diff == 0)) {
319 memmove(desta->haddr1, srca->haddr1, srca->size1);
320 if (unlikely(srca->size2)) {
321 memmove(desta->haddr2, srca->haddr2, srca->size2);
322 }
323 } else if (diff > 0) {
324 memmove(desta->haddr1, srca->haddr1, srca->size1);
325 memmove(desta->haddr1 + srca->size1, srca->haddr2, diff);
326 if (likely(desta->size2)) {
327 memmove(desta->haddr2, srca->haddr2 + diff, desta->size2);
328 }
329 } else {
330 diff = -diff;
331 memmove(desta->haddr1, srca->haddr1, desta->size1);
332 memmove(desta->haddr2, srca->haddr1 + desta->size1, diff);
333 if (likely(srca->size2)) {
334 memmove(desta->haddr2 + diff, srca->haddr2, srca->size2);
335 }
336 }
337 } else {
338 for (int i = 0; i < len; i++) {
339 uint8_t byte = access_get_byte(env, srca, i, ra);
340 access_set_byte(env, desta, i, byte, ra);
341 }
342 }
343 }
344
345 static int mmu_idx_from_as(uint8_t as)
346 {
347 switch (as) {
348 case AS_PRIMARY:
349 return MMU_PRIMARY_IDX;
350 case AS_SECONDARY:
351 return MMU_SECONDARY_IDX;
352 case AS_HOME:
353 return MMU_HOME_IDX;
354 default:
355 /* FIXME AS_ACCREG */
356 g_assert_not_reached();
357 }
358 }
359
360 /* and on array */
361 static uint32_t do_helper_nc(CPUS390XState *env, uint32_t l, uint64_t dest,
362 uint64_t src, uintptr_t ra)
363 {
364 const int mmu_idx = s390x_env_mmu_index(env, false);
365 S390Access srca1, srca2, desta;
366 uint32_t i;
367 uint8_t c = 0;
368
369 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
370 __func__, l, dest, src);
371
372 /* NC always processes one more byte than specified - maximum is 256 */
373 l++;
374
375 access_prepare(&srca1, env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
376 access_prepare(&srca2, env, dest, l, MMU_DATA_LOAD, mmu_idx, ra);
377 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
378 set_helper_retaddr(ra);
379
380 for (i = 0; i < l; i++) {
381 const uint8_t x = access_get_byte(env, &srca1, i, ra) &
382 access_get_byte(env, &srca2, i, ra);
383
384 c |= x;
385 access_set_byte(env, &desta, i, x, ra);
386 }
387
388 clear_helper_retaddr();
389 return c != 0;
390 }
391
392 uint32_t HELPER(nc)(CPUS390XState *env, uint32_t l, uint64_t dest,
393 uint64_t src)
394 {
395 return do_helper_nc(env, l, dest, src, GETPC());
396 }
397
398 /* xor on array */
399 static uint32_t do_helper_xc(CPUS390XState *env, uint32_t l, uint64_t dest,
400 uint64_t src, uintptr_t ra)
401 {
402 const int mmu_idx = s390x_env_mmu_index(env, false);
403 S390Access srca1, srca2, desta;
404 uint32_t i;
405 uint8_t c = 0;
406
407 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
408 __func__, l, dest, src);
409
410 /* XC always processes one more byte than specified - maximum is 256 */
411 l++;
412
413 access_prepare(&srca1, env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
414 access_prepare(&srca2, env, dest, l, MMU_DATA_LOAD, mmu_idx, ra);
415 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
416
417 /* xor with itself is the same as memset(0) */
418 if (src == dest) {
419 access_memset(env, &desta, 0, ra);
420 return 0;
421 }
422
423 set_helper_retaddr(ra);
424 for (i = 0; i < l; i++) {
425 const uint8_t x = access_get_byte(env, &srca1, i, ra) ^
426 access_get_byte(env, &srca2, i, ra);
427
428 c |= x;
429 access_set_byte(env, &desta, i, x, ra);
430 }
431 clear_helper_retaddr();
432 return c != 0;
433 }
434
435 uint32_t HELPER(xc)(CPUS390XState *env, uint32_t l, uint64_t dest,
436 uint64_t src)
437 {
438 return do_helper_xc(env, l, dest, src, GETPC());
439 }
440
441 /* or on array */
442 static uint32_t do_helper_oc(CPUS390XState *env, uint32_t l, uint64_t dest,
443 uint64_t src, uintptr_t ra)
444 {
445 const int mmu_idx = s390x_env_mmu_index(env, false);
446 S390Access srca1, srca2, desta;
447 uint32_t i;
448 uint8_t c = 0;
449
450 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
451 __func__, l, dest, src);
452
453 /* OC always processes one more byte than specified - maximum is 256 */
454 l++;
455
456 access_prepare(&srca1, env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
457 access_prepare(&srca2, env, dest, l, MMU_DATA_LOAD, mmu_idx, ra);
458 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
459 set_helper_retaddr(ra);
460
461 for (i = 0; i < l; i++) {
462 const uint8_t x = access_get_byte(env, &srca1, i, ra) |
463 access_get_byte(env, &srca2, i, ra);
464
465 c |= x;
466 access_set_byte(env, &desta, i, x, ra);
467 }
468
469 clear_helper_retaddr();
470 return c != 0;
471 }
472
473 uint32_t HELPER(oc)(CPUS390XState *env, uint32_t l, uint64_t dest,
474 uint64_t src)
475 {
476 return do_helper_oc(env, l, dest, src, GETPC());
477 }
478
479 /* memmove */
480 static uint32_t do_helper_mvc(CPUS390XState *env, uint32_t l, uint64_t dest,
481 uint64_t src, uintptr_t ra)
482 {
483 const int mmu_idx = s390x_env_mmu_index(env, false);
484 S390Access srca, desta;
485 uint32_t i;
486
487 HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
488 __func__, l, dest, src);
489
490 /* MVC always copies one more byte than specified - maximum is 256 */
491 l++;
492
493 access_prepare(&srca, env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
494 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
495
496 /*
497 * "When the operands overlap, the result is obtained as if the operands
498 * were processed one byte at a time". Only non-destructive overlaps
499 * behave like memmove().
500 */
501 if (dest == src + 1) {
502 access_memset(env, &desta, access_get_byte(env, &srca, 0, ra), ra);
503 } else if (!is_destructive_overlap(env, dest, src, l)) {
504 access_memmove(env, &desta, &srca, ra);
505 } else {
506 set_helper_retaddr(ra);
507 for (i = 0; i < l; i++) {
508 uint8_t byte = access_get_byte(env, &srca, i, ra);
509
510 access_set_byte(env, &desta, i, byte, ra);
511 }
512 clear_helper_retaddr();
513 }
514
515 return env->cc_op;
516 }
517
518 void HELPER(mvc)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
519 {
520 do_helper_mvc(env, l, dest, src, GETPC());
521 }
522
523 /* move right to left */
524 void HELPER(mvcrl)(CPUS390XState *env, uint64_t l, uint64_t dest, uint64_t src)
525 {
526 const int mmu_idx = s390x_env_mmu_index(env, false);
527 const uint64_t ra = GETPC();
528 S390Access srca, desta;
529 int32_t i;
530
531 /* MVCRL always copies one more byte than specified - maximum is 256 */
532 l &= 0xff;
533 l++;
534
535 access_prepare(&srca, env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
536 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
537
538 set_helper_retaddr(ra);
539 for (i = l - 1; i >= 0; i--) {
540 uint8_t byte = access_get_byte(env, &srca, i, ra);
541 access_set_byte(env, &desta, i, byte, ra);
542 }
543 clear_helper_retaddr();
544 }
545
546 /* move inverse */
547 void HELPER(mvcin)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
548 {
549 const int mmu_idx = s390x_env_mmu_index(env, false);
550 S390Access srca, desta;
551 uintptr_t ra = GETPC();
552 int i;
553
554 /* MVCIN always copies one more byte than specified - maximum is 256 */
555 l++;
556
557 src = wrap_address(env, src - l + 1);
558 access_prepare(&srca, env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
559 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
560
561 set_helper_retaddr(ra);
562 for (i = 0; i < l; i++) {
563 const uint8_t x = access_get_byte(env, &srca, l - i - 1, ra);
564 access_set_byte(env, &desta, i, x, ra);
565 }
566 clear_helper_retaddr();
567 }
568
569 /* move numerics */
570 void HELPER(mvn)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
571 {
572 const int mmu_idx = s390x_env_mmu_index(env, false);
573 S390Access srca1, srca2, desta;
574 uintptr_t ra = GETPC();
575 int i;
576
577 /* MVN always copies one more byte than specified - maximum is 256 */
578 l++;
579
580 access_prepare(&srca1, env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
581 access_prepare(&srca2, env, dest, l, MMU_DATA_LOAD, mmu_idx, ra);
582 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
583
584 set_helper_retaddr(ra);
585 for (i = 0; i < l; i++) {
586 const uint8_t x = (access_get_byte(env, &srca1, i, ra) & 0x0f) |
587 (access_get_byte(env, &srca2, i, ra) & 0xf0);
588
589 access_set_byte(env, &desta, i, x, ra);
590 }
591 clear_helper_retaddr();
592 }
593
594 /* move with offset */
595 void HELPER(mvo)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
596 {
597 const int mmu_idx = s390x_env_mmu_index(env, false);
598 /* MVO always processes one more byte than specified - maximum is 16 */
599 const int len_dest = (l >> 4) + 1;
600 const int len_src = (l & 0xf) + 1;
601 uintptr_t ra = GETPC();
602 uint8_t byte_dest, byte_src;
603 S390Access srca, desta;
604 int i, j;
605
606 access_prepare(&srca, env, src, len_src, MMU_DATA_LOAD, mmu_idx, ra);
607 access_prepare(&desta, env, dest, len_dest, MMU_DATA_STORE, mmu_idx, ra);
608
609 /* Handle rightmost byte */
610 byte_dest = cpu_ldub_data_ra(env, dest + len_dest - 1, ra);
611
612 set_helper_retaddr(ra);
613 byte_src = access_get_byte(env, &srca, len_src - 1, ra);
614 byte_dest = (byte_dest & 0x0f) | (byte_src << 4);
615 access_set_byte(env, &desta, len_dest - 1, byte_dest, ra);
616
617 /* Process remaining bytes from right to left */
618 for (i = len_dest - 2, j = len_src - 2; i >= 0; i--, j--) {
619 byte_dest = byte_src >> 4;
620 if (j >= 0) {
621 byte_src = access_get_byte(env, &srca, j, ra);
622 } else {
623 byte_src = 0;
624 }
625 byte_dest |= byte_src << 4;
626 access_set_byte(env, &desta, i, byte_dest, ra);
627 }
628 clear_helper_retaddr();
629 }
630
631 /* move zones */
632 void HELPER(mvz)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
633 {
634 const int mmu_idx = s390x_env_mmu_index(env, false);
635 S390Access srca1, srca2, desta;
636 uintptr_t ra = GETPC();
637 int i;
638
639 /* MVZ always copies one more byte than specified - maximum is 256 */
640 l++;
641
642 access_prepare(&srca1, env, src, l, MMU_DATA_LOAD, mmu_idx, ra);
643 access_prepare(&srca2, env, dest, l, MMU_DATA_LOAD, mmu_idx, ra);
644 access_prepare(&desta, env, dest, l, MMU_DATA_STORE, mmu_idx, ra);
645
646 set_helper_retaddr(ra);
647 for (i = 0; i < l; i++) {
648 const uint8_t x = (access_get_byte(env, &srca1, i, ra) & 0xf0) |
649 (access_get_byte(env, &srca2, i, ra) & 0x0f);
650
651 access_set_byte(env, &desta, i, x, ra);
652 }
653 clear_helper_retaddr();
654 }
655
656 /* compare unsigned byte arrays */
657 static uint32_t do_helper_clc(CPUS390XState *env, uint32_t l, uint64_t s1,
658 uint64_t s2, uintptr_t ra)
659 {
660 uint32_t i;
661 uint32_t cc = 0;
662
663 HELPER_LOG("%s l %d s1 %" PRIx64 " s2 %" PRIx64 "\n",
664 __func__, l, s1, s2);
665
666 for (i = 0; i <= l; i++) {
667 uint8_t x = cpu_ldub_data_ra(env, s1 + i, ra);
668 uint8_t y = cpu_ldub_data_ra(env, s2 + i, ra);
669 HELPER_LOG("%02x (%c)/%02x (%c) ", x, x, y, y);
670 if (x < y) {
671 cc = 1;
672 break;
673 } else if (x > y) {
674 cc = 2;
675 break;
676 }
677 }
678
679 HELPER_LOG("\n");
680 return cc;
681 }
682
683 uint32_t HELPER(clc)(CPUS390XState *env, uint32_t l, uint64_t s1, uint64_t s2)
684 {
685 return do_helper_clc(env, l, s1, s2, GETPC());
686 }
687
688 /* compare logical under mask */
689 uint32_t HELPER(clm)(CPUS390XState *env, uint32_t r1, uint32_t mask,
690 uint64_t addr)
691 {
692 uintptr_t ra = GETPC();
693 uint32_t cc = 0;
694
695 HELPER_LOG("%s: r1 0x%x mask 0x%x addr 0x%" PRIx64 "\n", __func__, r1,
696 mask, addr);
697
698 if (!mask) {
699 /* Recognize access exceptions for the first byte */
700 probe_read(env, addr, 1, s390x_env_mmu_index(env, false), ra);
701 }
702
703 while (mask) {
704 if (mask & 8) {
705 uint8_t d = cpu_ldub_data_ra(env, addr, ra);
706 uint8_t r = extract32(r1, 24, 8);
707 HELPER_LOG("mask 0x%x %02x/%02x (0x%" PRIx64 ") ", mask, r, d,
708 addr);
709 if (r < d) {
710 cc = 1;
711 break;
712 } else if (r > d) {
713 cc = 2;
714 break;
715 }
716 addr++;
717 }
718 mask = (mask << 1) & 0xf;
719 r1 <<= 8;
720 }
721
722 HELPER_LOG("\n");
723 return cc;
724 }
725
726 static inline uint64_t get_address(CPUS390XState *env, int reg)
727 {
728 return wrap_address(env, env->regs[reg]);
729 }
730
731 /*
732 * Store the address to the given register, zeroing out unused leftmost
733 * bits in bit positions 32-63 (24-bit and 31-bit mode only).
734 */
735 static inline void set_address_zero(CPUS390XState *env, int reg,
736 uint64_t address)
737 {
738 if (env->psw.mask & PSW_MASK_64) {
739 env->regs[reg] = address;
740 } else {
741 if (!(env->psw.mask & PSW_MASK_32)) {
742 address &= 0x00ffffff;
743 } else {
744 address &= 0x7fffffff;
745 }
746 env->regs[reg] = deposit64(env->regs[reg], 0, 32, address);
747 }
748 }
749
750 static inline void set_address(CPUS390XState *env, int reg, uint64_t address)
751 {
752 if (env->psw.mask & PSW_MASK_64) {
753 /* 64-Bit mode */
754 env->regs[reg] = address;
755 } else {
756 if (!(env->psw.mask & PSW_MASK_32)) {
757 /* 24-Bit mode. According to the PoO it is implementation
758 dependent if bits 32-39 remain unchanged or are set to
759 zeros. Choose the former so that the function can also be
760 used for TRT. */
761 env->regs[reg] = deposit64(env->regs[reg], 0, 24, address);
762 } else {
763 /* 31-Bit mode. According to the PoO it is implementation
764 dependent if bit 32 remains unchanged or is set to zero.
765 Choose the latter so that the function can also be used for
766 TRT. */
767 address &= 0x7fffffff;
768 env->regs[reg] = deposit64(env->regs[reg], 0, 32, address);
769 }
770 }
771 }
772
773 static inline uint64_t wrap_length32(CPUS390XState *env, uint64_t length)
774 {
775 if (!(env->psw.mask & PSW_MASK_64)) {
776 return (uint32_t)length;
777 }
778 return length;
779 }
780
781 static inline uint64_t wrap_length31(CPUS390XState *env, uint64_t length)
782 {
783 if (!(env->psw.mask & PSW_MASK_64)) {
784 /* 24-Bit and 31-Bit mode */
785 length &= 0x7fffffff;
786 }
787 return length;
788 }
789
790 static inline uint64_t get_length(CPUS390XState *env, int reg)
791 {
792 return wrap_length31(env, env->regs[reg]);
793 }
794
795 static inline void set_length(CPUS390XState *env, int reg, uint64_t length)
796 {
797 if (env->psw.mask & PSW_MASK_64) {
798 /* 64-Bit mode */
799 env->regs[reg] = length;
800 } else {
801 /* 24-Bit and 31-Bit mode */
802 env->regs[reg] = deposit64(env->regs[reg], 0, 32, length);
803 }
804 }
805
806 /* search string (c is byte to search, r2 is string, r1 end of string) */
807 void HELPER(srst)(CPUS390XState *env, uint32_t r1, uint32_t r2)
808 {
809 uintptr_t ra = GETPC();
810 uint64_t end, str;
811 uint32_t len;
812 uint8_t v, c = env->regs[0];
813
814 /* Bits 32-55 must contain all 0. */
815 if (env->regs[0] & 0xffffff00u) {
816 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
817 }
818
819 str = get_address(env, r2);
820 end = get_address(env, r1);
821
822 /* Lest we fail to service interrupts in a timely manner, limit the
823 amount of work we're willing to do. For now, let's cap at 8k. */
824 for (len = 0; len < 0x2000; ++len) {
825 if (str + len == end) {
826 /* Character not found. R1 & R2 are unmodified. */
827 env->cc_op = 2;
828 return;
829 }
830 v = cpu_ldub_data_ra(env, str + len, ra);
831 if (v == c) {
832 /* Character found. Set R1 to the location; R2 is unmodified. */
833 env->cc_op = 1;
834 set_address(env, r1, str + len);
835 return;
836 }
837 }
838
839 /* CPU-determined bytes processed. Advance R2 to next byte to process. */
840 env->cc_op = 3;
841 set_address(env, r2, str + len);
842 }
843
844 void HELPER(srstu)(CPUS390XState *env, uint32_t r1, uint32_t r2)
845 {
846 uintptr_t ra = GETPC();
847 uint32_t len;
848 uint16_t v, c = env->regs[0];
849 uint64_t end, str, adj_end;
850
851 /* Bits 32-47 of R0 must be zero. */
852 if (env->regs[0] & 0xffff0000u) {
853 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
854 }
855
856 str = get_address(env, r2);
857 end = get_address(env, r1);
858
859 /* If the LSB of the two addresses differ, use one extra byte. */
860 adj_end = end + ((str ^ end) & 1);
861
862 /* Lest we fail to service interrupts in a timely manner, limit the
863 amount of work we're willing to do. For now, let's cap at 8k. */
864 for (len = 0; len < 0x2000; len += 2) {
865 if (str + len == adj_end) {
866 /* End of input found. */
867 env->cc_op = 2;
868 return;
869 }
870 v = cpu_lduw_be_data_ra(env, str + len, ra);
871 if (v == c) {
872 /* Character found. Set R1 to the location; R2 is unmodified. */
873 env->cc_op = 1;
874 set_address(env, r1, str + len);
875 return;
876 }
877 }
878
879 /* CPU-determined bytes processed. Advance R2 to next byte to process. */
880 env->cc_op = 3;
881 set_address(env, r2, str + len);
882 }
883
884 /* unsigned string compare (c is string terminator) */
885 Int128 HELPER(clst)(CPUS390XState *env, uint64_t c, uint64_t s1, uint64_t s2)
886 {
887 uintptr_t ra = GETPC();
888 uint32_t len;
889
890 c = c & 0xff;
891 s1 = wrap_address(env, s1);
892 s2 = wrap_address(env, s2);
893
894 /* Lest we fail to service interrupts in a timely manner, limit the
895 amount of work we're willing to do. For now, let's cap at 8k. */
896 for (len = 0; len < 0x2000; ++len) {
897 uint8_t v1 = cpu_ldub_data_ra(env, s1 + len, ra);
898 uint8_t v2 = cpu_ldub_data_ra(env, s2 + len, ra);
899 if (v1 == v2) {
900 if (v1 == c) {
901 /* Equal. CC=0, and don't advance the registers. */
902 env->cc_op = 0;
903 return int128_make128(s2, s1);
904 }
905 } else {
906 /* Unequal. CC={1,2}, and advance the registers. Note that
907 the terminator need not be zero, but the string that contains
908 the terminator is by definition "low". */
909 env->cc_op = (v1 == c ? 1 : v2 == c ? 2 : v1 < v2 ? 1 : 2);
910 return int128_make128(s2 + len, s1 + len);
911 }
912 }
913
914 /* CPU-determined bytes equal; advance the registers. */
915 env->cc_op = 3;
916 return int128_make128(s2 + len, s1 + len);
917 }
918
919 /* move page */
920 uint32_t HELPER(mvpg)(CPUS390XState *env, uint64_t r0, uint32_t r1, uint32_t r2)
921 {
922 const uint64_t src = get_address(env, r2) & TARGET_PAGE_MASK;
923 const uint64_t dst = get_address(env, r1) & TARGET_PAGE_MASK;
924 const int mmu_idx = s390x_env_mmu_index(env, false);
925 const bool f = extract64(r0, 11, 1);
926 const bool s = extract64(r0, 10, 1);
927 const bool cco = extract64(r0, 8, 1);
928 uintptr_t ra = GETPC();
929 S390Access srca, desta;
930 int exc;
931
932 if ((f && s) || extract64(r0, 12, 4)) {
933 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, GETPC());
934 }
935
936 /*
937 * We always manually handle exceptions such that we can properly store
938 * r1/r2 to the lowcore on page-translation exceptions.
939 *
940 * TODO: Access key handling
941 */
942 exc = access_prepare_nf(&srca, env, true, src, TARGET_PAGE_SIZE,
943 MMU_DATA_LOAD, mmu_idx, ra);
944 if (exc) {
945 if (cco) {
946 return 2;
947 }
948 goto inject_exc;
949 }
950 exc = access_prepare_nf(&desta, env, true, dst, TARGET_PAGE_SIZE,
951 MMU_DATA_STORE, mmu_idx, ra);
952 if (exc) {
953 if (cco && exc != PGM_PROTECTION) {
954 return 1;
955 }
956 goto inject_exc;
957 }
958 access_memmove(env, &desta, &srca, ra);
959 return 0; /* data moved */
960 inject_exc:
961 #if !defined(CONFIG_USER_ONLY)
962 if (exc != PGM_ADDRESSING) {
963 address_space_stq_be(env_cpu(env)->as,
964 env->psa + offsetof(LowCore, trans_exc_code),
965 env->tlb_fill_tec, MEMTXATTRS_UNSPECIFIED, NULL);
966 }
967 if (exc == PGM_PAGE_TRANS) {
968 address_space_stb(env_cpu(env)->as,
969 env->psa + offsetof(LowCore, op_access_id),
970 r1 << 4 | r2, MEMTXATTRS_UNSPECIFIED, NULL);
971 }
972 #endif
973 tcg_s390_program_interrupt(env, exc, ra);
974 }
975
976 /* string copy */
977 uint32_t HELPER(mvst)(CPUS390XState *env, uint32_t r1, uint32_t r2)
978 {
979 const int mmu_idx = s390x_env_mmu_index(env, false);
980 const uint64_t d = get_address(env, r1);
981 const uint64_t s = get_address(env, r2);
982 const uint8_t c = env->regs[0];
983 const int len = MIN(-(d | TARGET_PAGE_MASK), -(s | TARGET_PAGE_MASK));
984 S390Access srca, desta;
985 uintptr_t ra = GETPC();
986 int i;
987
988 if (env->regs[0] & 0xffffff00ull) {
989 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
990 }
991
992 /*
993 * Our access should not exceed single pages, as we must not report access
994 * exceptions exceeding the actually copied range (which we don't know at
995 * this point). We might over-indicate watchpoints within the pages
996 * (if we ever care, we have to limit processing to a single byte).
997 */
998 access_prepare(&srca, env, s, len, MMU_DATA_LOAD, mmu_idx, ra);
999 access_prepare(&desta, env, d, len, MMU_DATA_STORE, mmu_idx, ra);
1000
1001 set_helper_retaddr(ra);
1002 for (i = 0; i < len; i++) {
1003 const uint8_t v = access_get_byte(env, &srca, i, ra);
1004
1005 access_set_byte(env, &desta, i, v, ra);
1006 if (v == c) {
1007 clear_helper_retaddr();
1008 set_address_zero(env, r1, d + i);
1009 return 1;
1010 }
1011 }
1012 clear_helper_retaddr();
1013 set_address_zero(env, r1, d + len);
1014 set_address_zero(env, r2, s + len);
1015 return 3;
1016 }
1017
1018 /* load access registers r1 to r3 from memory at a2 */
1019 void HELPER(lam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
1020 {
1021 uintptr_t ra = GETPC();
1022 int i;
1023
1024 if (a2 & 0x3) {
1025 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
1026 }
1027
1028 for (i = r1;; i = (i + 1) % 16) {
1029 env->aregs[i] = cpu_ldl_be_data_ra(env, a2, ra);
1030 a2 += 4;
1031
1032 if (i == r3) {
1033 break;
1034 }
1035 }
1036 }
1037
1038 /* store access registers r1 to r3 in memory at a2 */
1039 void HELPER(stam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
1040 {
1041 uintptr_t ra = GETPC();
1042 int i;
1043
1044 if (a2 & 0x3) {
1045 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
1046 }
1047
1048 for (i = r1;; i = (i + 1) % 16) {
1049 cpu_stl_be_data_ra(env, a2, env->aregs[i], ra);
1050 a2 += 4;
1051
1052 if (i == r3) {
1053 break;
1054 }
1055 }
1056 }
1057
1058 /* move long helper */
1059 static inline uint32_t do_mvcl(CPUS390XState *env,
1060 uint64_t *dest, uint64_t *destlen,
1061 uint64_t *src, uint64_t *srclen,
1062 uint16_t pad, int wordsize, uintptr_t ra)
1063 {
1064 const int mmu_idx = s390x_env_mmu_index(env, false);
1065 int len = MIN(*destlen, -(*dest | TARGET_PAGE_MASK));
1066 S390Access srca, desta;
1067 int i, cc;
1068
1069 if (*destlen == *srclen) {
1070 cc = 0;
1071 } else if (*destlen < *srclen) {
1072 cc = 1;
1073 } else {
1074 cc = 2;
1075 }
1076
1077 if (!*destlen) {
1078 return cc;
1079 }
1080
1081 /*
1082 * Only perform one type of type of operation (move/pad) at a time.
1083 * Stay within single pages.
1084 */
1085 if (*srclen) {
1086 /* Copy the src array */
1087 len = MIN(MIN(*srclen, -(*src | TARGET_PAGE_MASK)), len);
1088 *destlen -= len;
1089 *srclen -= len;
1090 access_prepare(&srca, env, *src, len, MMU_DATA_LOAD, mmu_idx, ra);
1091 access_prepare(&desta, env, *dest, len, MMU_DATA_STORE, mmu_idx, ra);
1092 access_memmove(env, &desta, &srca, ra);
1093 *src = wrap_address(env, *src + len);
1094 *dest = wrap_address(env, *dest + len);
1095 } else if (wordsize == 1) {
1096 /* Pad the remaining area */
1097 *destlen -= len;
1098 access_prepare(&desta, env, *dest, len, MMU_DATA_STORE, mmu_idx, ra);
1099 access_memset(env, &desta, pad, ra);
1100 *dest = wrap_address(env, *dest + len);
1101 } else {
1102 access_prepare(&desta, env, *dest, len, MMU_DATA_STORE, mmu_idx, ra);
1103 set_helper_retaddr(ra);
1104
1105 /* The remaining length selects the padding byte. */
1106 for (i = 0; i < len; (*destlen)--, i++) {
1107 if (*destlen & 1) {
1108 access_set_byte(env, &desta, i, pad, ra);
1109 } else {
1110 access_set_byte(env, &desta, i, pad >> 8, ra);
1111 }
1112 }
1113 clear_helper_retaddr();
1114 *dest = wrap_address(env, *dest + len);
1115 }
1116
1117 return *destlen ? 3 : cc;
1118 }
1119
1120 /* move long */
1121 uint32_t HELPER(mvcl)(CPUS390XState *env, uint32_t r1, uint32_t r2)
1122 {
1123 const int mmu_idx = s390x_env_mmu_index(env, false);
1124 uintptr_t ra = GETPC();
1125 uint64_t destlen = env->regs[r1 + 1] & 0xffffff;
1126 uint64_t dest = get_address(env, r1);
1127 uint64_t srclen = env->regs[r2 + 1] & 0xffffff;
1128 uint64_t src = get_address(env, r2);
1129 uint8_t pad = env->regs[r2 + 1] >> 24;
1130 CPUState *cs = env_cpu(env);
1131 S390Access srca, desta;
1132 uint32_t cc, cur_len;
1133
1134 if (is_destructive_overlap(env, dest, src, MIN(srclen, destlen))) {
1135 cc = 3;
1136 } else if (srclen == destlen) {
1137 cc = 0;
1138 } else if (destlen < srclen) {
1139 cc = 1;
1140 } else {
1141 cc = 2;
1142 }
1143
1144 /* We might have to zero-out some bits even if there was no action. */
1145 if (unlikely(!destlen || cc == 3)) {
1146 set_address_zero(env, r2, src);
1147 set_address_zero(env, r1, dest);
1148 return cc;
1149 } else if (!srclen) {
1150 set_address_zero(env, r2, src);
1151 }
1152
1153 /*
1154 * Only perform one type of type of operation (move/pad) in one step.
1155 * Stay within single pages.
1156 */
1157 while (destlen) {
1158 cur_len = MIN(destlen, -(dest | TARGET_PAGE_MASK));
1159 if (!srclen) {
1160 access_prepare(&desta, env, dest, cur_len,
1161 MMU_DATA_STORE, mmu_idx, ra);
1162 access_memset(env, &desta, pad, ra);
1163 } else {
1164 cur_len = MIN(MIN(srclen, -(src | TARGET_PAGE_MASK)), cur_len);
1165
1166 access_prepare(&srca, env, src, cur_len,
1167 MMU_DATA_LOAD, mmu_idx, ra);
1168 access_prepare(&desta, env, dest, cur_len,
1169 MMU_DATA_STORE, mmu_idx, ra);
1170 access_memmove(env, &desta, &srca, ra);
1171 src = wrap_address(env, src + cur_len);
1172 srclen -= cur_len;
1173 env->regs[r2 + 1] = deposit64(env->regs[r2 + 1], 0, 24, srclen);
1174 set_address_zero(env, r2, src);
1175 }
1176 dest = wrap_address(env, dest + cur_len);
1177 destlen -= cur_len;
1178 env->regs[r1 + 1] = deposit64(env->regs[r1 + 1], 0, 24, destlen);
1179 set_address_zero(env, r1, dest);
1180
1181 /*
1182 * MVCL is interruptible. Return to the main loop if requested after
1183 * writing back all state to registers. If no interrupt will get
1184 * injected, we'll end up back in this handler and continue processing
1185 * the remaining parts.
1186 */
1187 if (destlen && unlikely(cpu_loop_exit_requested(cs))) {
1188 cpu_loop_exit_restore(cs, ra);
1189 }
1190 }
1191 return cc;
1192 }
1193
1194 /* move long extended */
1195 uint32_t HELPER(mvcle)(CPUS390XState *env, uint32_t r1, uint64_t a2,
1196 uint32_t r3)
1197 {
1198 uintptr_t ra = GETPC();
1199 uint64_t destlen = get_length(env, r1 + 1);
1200 uint64_t dest = get_address(env, r1);
1201 uint64_t srclen = get_length(env, r3 + 1);
1202 uint64_t src = get_address(env, r3);
1203 uint8_t pad = a2;
1204 uint32_t cc;
1205
1206 cc = do_mvcl(env, &dest, &destlen, &src, &srclen, pad, 1, ra);
1207
1208 set_length(env, r1 + 1, destlen);
1209 set_length(env, r3 + 1, srclen);
1210 set_address(env, r1, dest);
1211 set_address(env, r3, src);
1212
1213 return cc;
1214 }
1215
1216 /* move long unicode */
1217 uint32_t HELPER(mvclu)(CPUS390XState *env, uint32_t r1, uint64_t a2,
1218 uint32_t r3)
1219 {
1220 uintptr_t ra = GETPC();
1221 uint64_t destlen = get_length(env, r1 + 1);
1222 uint64_t dest = get_address(env, r1);
1223 uint64_t srclen = get_length(env, r3 + 1);
1224 uint64_t src = get_address(env, r3);
1225 uint16_t pad = a2;
1226 uint32_t cc;
1227
1228 cc = do_mvcl(env, &dest, &destlen, &src, &srclen, pad, 2, ra);
1229
1230 set_length(env, r1 + 1, destlen);
1231 set_length(env, r3 + 1, srclen);
1232 set_address(env, r1, dest);
1233 set_address(env, r3, src);
1234
1235 return cc;
1236 }
1237
1238 /* compare logical long helper */
1239 static inline uint32_t do_clcl(CPUS390XState *env,
1240 uint64_t *src1, uint64_t *src1len,
1241 uint64_t *src3, uint64_t *src3len,
1242 uint16_t pad, uint64_t limit,
1243 int wordsize, uintptr_t ra)
1244 {
1245 uint64_t len = MAX(*src1len, *src3len);
1246 uint32_t cc = 0;
1247
1248 check_alignment(env, *src1len | *src3len, wordsize, ra);
1249
1250 if (!len) {
1251 return cc;
1252 }
1253
1254 /* Lest we fail to service interrupts in a timely manner, limit the
1255 amount of work we're willing to do. */
1256 if (len > limit) {
1257 len = limit;
1258 cc = 3;
1259 }
1260
1261 for (; len; len -= wordsize) {
1262 uint16_t v1 = pad;
1263 uint16_t v3 = pad;
1264
1265 if (*src1len) {
1266 v1 = cpu_ldusize_data_ra(env, *src1, wordsize, ra);
1267 }
1268 if (*src3len) {
1269 v3 = cpu_ldusize_data_ra(env, *src3, wordsize, ra);
1270 }
1271
1272 if (v1 != v3) {
1273 cc = (v1 < v3) ? 1 : 2;
1274 break;
1275 }
1276
1277 if (*src1len) {
1278 *src1 += wordsize;
1279 *src1len -= wordsize;
1280 }
1281 if (*src3len) {
1282 *src3 += wordsize;
1283 *src3len -= wordsize;
1284 }
1285 }
1286
1287 return cc;
1288 }
1289
1290
1291 /* compare logical long */
1292 uint32_t HELPER(clcl)(CPUS390XState *env, uint32_t r1, uint32_t r2)
1293 {
1294 uintptr_t ra = GETPC();
1295 uint64_t src1len = extract64(env->regs[r1 + 1], 0, 24);
1296 uint64_t src1 = get_address(env, r1);
1297 uint64_t src3len = extract64(env->regs[r2 + 1], 0, 24);
1298 uint64_t src3 = get_address(env, r2);
1299 uint8_t pad = env->regs[r2 + 1] >> 24;
1300 uint32_t cc;
1301
1302 cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, -1, 1, ra);
1303
1304 env->regs[r1 + 1] = deposit64(env->regs[r1 + 1], 0, 24, src1len);
1305 env->regs[r2 + 1] = deposit64(env->regs[r2 + 1], 0, 24, src3len);
1306 set_address(env, r1, src1);
1307 set_address(env, r2, src3);
1308
1309 return cc;
1310 }
1311
1312 /* compare logical long extended memcompare insn with padding */
1313 uint32_t HELPER(clcle)(CPUS390XState *env, uint32_t r1, uint64_t a2,
1314 uint32_t r3)
1315 {
1316 uintptr_t ra = GETPC();
1317 uint64_t src1len = get_length(env, r1 + 1);
1318 uint64_t src1 = get_address(env, r1);
1319 uint64_t src3len = get_length(env, r3 + 1);
1320 uint64_t src3 = get_address(env, r3);
1321 uint8_t pad = a2;
1322 uint32_t cc;
1323
1324 cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, 0x2000, 1, ra);
1325
1326 set_length(env, r1 + 1, src1len);
1327 set_length(env, r3 + 1, src3len);
1328 set_address(env, r1, src1);
1329 set_address(env, r3, src3);
1330
1331 return cc;
1332 }
1333
1334 /* compare logical long unicode memcompare insn with padding */
1335 uint32_t HELPER(clclu)(CPUS390XState *env, uint32_t r1, uint64_t a2,
1336 uint32_t r3)
1337 {
1338 uintptr_t ra = GETPC();
1339 uint64_t src1len = get_length(env, r1 + 1);
1340 uint64_t src1 = get_address(env, r1);
1341 uint64_t src3len = get_length(env, r3 + 1);
1342 uint64_t src3 = get_address(env, r3);
1343 uint16_t pad = a2;
1344 uint32_t cc = 0;
1345
1346 cc = do_clcl(env, &src1, &src1len, &src3, &src3len, pad, 0x1000, 2, ra);
1347
1348 set_length(env, r1 + 1, src1len);
1349 set_length(env, r3 + 1, src3len);
1350 set_address(env, r1, src1);
1351 set_address(env, r3, src3);
1352
1353 return cc;
1354 }
1355
1356 /* checksum */
1357 Int128 HELPER(cksm)(CPUS390XState *env, uint64_t r1,
1358 uint64_t src, uint64_t src_len)
1359 {
1360 uintptr_t ra = GETPC();
1361 uint64_t max_len, len;
1362 uint64_t cksm = (uint32_t)r1;
1363
1364 /* Lest we fail to service interrupts in a timely manner, limit the
1365 amount of work we're willing to do. For now, let's cap at 8k. */
1366 max_len = (src_len > 0x2000 ? 0x2000 : src_len);
1367
1368 /* Process full words as available. */
1369 for (len = 0; len + 4 <= max_len; len += 4, src += 4) {
1370 cksm += (uint32_t)cpu_ldl_be_data_ra(env, src, ra);
1371 }
1372
1373 switch (max_len - len) {
1374 case 1:
1375 cksm += cpu_ldub_data_ra(env, src, ra) << 24;
1376 len += 1;
1377 break;
1378 case 2:
1379 cksm += cpu_lduw_be_data_ra(env, src, ra) << 16;
1380 len += 2;
1381 break;
1382 case 3:
1383 cksm += cpu_lduw_be_data_ra(env, src, ra) << 16;
1384 cksm += cpu_ldub_data_ra(env, src + 2, ra) << 8;
1385 len += 3;
1386 break;
1387 }
1388
1389 /* Fold the carry from the checksum. Note that we can see carry-out
1390 during folding more than once (but probably not more than twice). */
1391 while (cksm > 0xffffffffull) {
1392 cksm = (uint32_t)cksm + (cksm >> 32);
1393 }
1394
1395 /* Indicate whether or not we've processed everything. */
1396 env->cc_op = (len == src_len ? 0 : 3);
1397
1398 /* Return both cksm and processed length. */
1399 return int128_make128(cksm, len);
1400 }
1401
1402 void HELPER(pack)(CPUS390XState *env, uint32_t len, uint64_t dest, uint64_t src)
1403 {
1404 uintptr_t ra = GETPC();
1405 int len_dest = len >> 4;
1406 int len_src = len & 0xf;
1407 uint8_t b;
1408
1409 dest += len_dest;
1410 src += len_src;
1411
1412 /* last byte is special, it only flips the nibbles */
1413 b = cpu_ldub_data_ra(env, src, ra);
1414 cpu_stb_data_ra(env, dest, (b << 4) | (b >> 4), ra);
1415 src--;
1416 len_src--;
1417
1418 /* now pack every value */
1419 while (len_dest > 0) {
1420 b = 0;
1421
1422 if (len_src >= 0) {
1423 b = cpu_ldub_data_ra(env, src, ra) & 0x0f;
1424 src--;
1425 len_src--;
1426 }
1427 if (len_src >= 0) {
1428 b |= cpu_ldub_data_ra(env, src, ra) << 4;
1429 src--;
1430 len_src--;
1431 }
1432
1433 len_dest--;
1434 dest--;
1435 cpu_stb_data_ra(env, dest, b, ra);
1436 }
1437 }
1438
1439 static inline void do_pkau(CPUS390XState *env, uint64_t dest, uint64_t src,
1440 uint32_t srclen, int ssize, uintptr_t ra)
1441 {
1442 int i;
1443 /* The destination operand is always 16 bytes long. */
1444 const int destlen = 16;
1445
1446 /* The operands are processed from right to left. */
1447 src += srclen - 1;
1448 dest += destlen - 1;
1449
1450 for (i = 0; i < destlen; i++) {
1451 uint8_t b = 0;
1452
1453 /* Start with a positive sign */
1454 if (i == 0) {
1455 b = 0xc;
1456 } else if (srclen > ssize) {
1457 b = cpu_ldub_data_ra(env, src, ra) & 0x0f;
1458 src -= ssize;
1459 srclen -= ssize;
1460 }
1461
1462 if (srclen > ssize) {
1463 b |= cpu_ldub_data_ra(env, src, ra) << 4;
1464 src -= ssize;
1465 srclen -= ssize;
1466 }
1467
1468 cpu_stb_data_ra(env, dest, b, ra);
1469 dest--;
1470 }
1471 }
1472
1473
1474 void HELPER(pka)(CPUS390XState *env, uint64_t dest, uint64_t src,
1475 uint32_t srclen)
1476 {
1477 do_pkau(env, dest, src, srclen, 1, GETPC());
1478 }
1479
1480 void HELPER(pku)(CPUS390XState *env, uint64_t dest, uint64_t src,
1481 uint32_t srclen)
1482 {
1483 do_pkau(env, dest, src, srclen, 2, GETPC());
1484 }
1485
1486 void HELPER(unpk)(CPUS390XState *env, uint32_t len, uint64_t dest,
1487 uint64_t src)
1488 {
1489 uintptr_t ra = GETPC();
1490 int len_dest = len >> 4;
1491 int len_src = len & 0xf;
1492 uint8_t b;
1493 int second_nibble = 0;
1494
1495 dest += len_dest;
1496 src += len_src;
1497
1498 /* last byte is special, it only flips the nibbles */
1499 b = cpu_ldub_data_ra(env, src, ra);
1500 cpu_stb_data_ra(env, dest, (b << 4) | (b >> 4), ra);
1501 src--;
1502 len_src--;
1503
1504 /* now pad every nibble with 0xf0 */
1505
1506 while (len_dest > 0) {
1507 uint8_t cur_byte = 0;
1508
1509 if (len_src > 0) {
1510 cur_byte = cpu_ldub_data_ra(env, src, ra);
1511 }
1512
1513 len_dest--;
1514 dest--;
1515
1516 /* only advance one nibble at a time */
1517 if (second_nibble) {
1518 cur_byte >>= 4;
1519 len_src--;
1520 src--;
1521 }
1522 second_nibble = !second_nibble;
1523
1524 /* digit */
1525 cur_byte = (cur_byte & 0xf);
1526 /* zone bits */
1527 cur_byte |= 0xf0;
1528
1529 cpu_stb_data_ra(env, dest, cur_byte, ra);
1530 }
1531 }
1532
1533 static inline uint32_t do_unpkau(CPUS390XState *env, uint64_t dest,
1534 uint32_t destlen, int dsize, uint64_t src,
1535 uintptr_t ra)
1536 {
1537 int i;
1538 uint32_t cc;
1539 uint8_t b;
1540 /* The source operand is always 16 bytes long. */
1541 const int srclen = 16;
1542
1543 /* The operands are processed from right to left. */
1544 src += srclen - 1;
1545 dest += destlen - dsize;
1546
1547 /* Check for the sign. */
1548 b = cpu_ldub_data_ra(env, src, ra);
1549 src--;
1550 switch (b & 0xf) {
1551 case 0xa:
1552 case 0xc:
1553 case 0xe ... 0xf:
1554 cc = 0; /* plus */
1555 break;
1556 case 0xb:
1557 case 0xd:
1558 cc = 1; /* minus */
1559 break;
1560 default:
1561 case 0x0 ... 0x9:
1562 cc = 3; /* invalid */
1563 break;
1564 }
1565
1566 /* Now pad every nibble with 0x30, advancing one nibble at a time. */
1567 for (i = 0; i < destlen; i += dsize) {
1568 if (i == (31 * dsize)) {
1569 /* If length is 32/64 bytes, the leftmost byte is 0. */
1570 b = 0;
1571 } else if (i % (2 * dsize)) {
1572 b = cpu_ldub_data_ra(env, src, ra);
1573 src--;
1574 } else {
1575 b >>= 4;
1576 }
1577 cpu_stsize_data_ra(env, dest, 0x30 + (b & 0xf), dsize, ra);
1578 dest -= dsize;
1579 }
1580
1581 return cc;
1582 }
1583
1584 uint32_t HELPER(unpka)(CPUS390XState *env, uint64_t dest, uint32_t destlen,
1585 uint64_t src)
1586 {
1587 return do_unpkau(env, dest, destlen, 1, src, GETPC());
1588 }
1589
1590 uint32_t HELPER(unpku)(CPUS390XState *env, uint64_t dest, uint32_t destlen,
1591 uint64_t src)
1592 {
1593 return do_unpkau(env, dest, destlen, 2, src, GETPC());
1594 }
1595
1596 uint32_t HELPER(tp)(CPUS390XState *env, uint64_t dest, uint32_t destlen)
1597 {
1598 uintptr_t ra = GETPC();
1599 uint32_t cc = 0;
1600 int i;
1601
1602 for (i = 0; i < destlen; i++) {
1603 uint8_t b = cpu_ldub_data_ra(env, dest + i, ra);
1604 /* digit */
1605 cc |= (b & 0xf0) > 0x90 ? 2 : 0;
1606
1607 if (i == (destlen - 1)) {
1608 /* sign */
1609 cc |= (b & 0xf) < 0xa ? 1 : 0;
1610 } else {
1611 /* digit */
1612 cc |= (b & 0xf) > 0x9 ? 2 : 0;
1613 }
1614 }
1615
1616 return cc;
1617 }
1618
1619 static uint32_t do_helper_tr(CPUS390XState *env, uint32_t len, uint64_t array,
1620 uint64_t trans, uintptr_t ra)
1621 {
1622 uint32_t i;
1623
1624 for (i = 0; i <= len; i++) {
1625 uint8_t byte = cpu_ldub_data_ra(env, array + i, ra);
1626 uint8_t new_byte = cpu_ldub_data_ra(env, trans + byte, ra);
1627 cpu_stb_data_ra(env, array + i, new_byte, ra);
1628 }
1629
1630 return env->cc_op;
1631 }
1632
1633 void HELPER(tr)(CPUS390XState *env, uint32_t len, uint64_t array,
1634 uint64_t trans)
1635 {
1636 do_helper_tr(env, len, array, trans, GETPC());
1637 }
1638
1639 Int128 HELPER(tre)(CPUS390XState *env, uint64_t array,
1640 uint64_t len, uint64_t trans)
1641 {
1642 uintptr_t ra = GETPC();
1643 uint8_t end = env->regs[0] & 0xff;
1644 uint64_t l = len;
1645 uint64_t i;
1646 uint32_t cc = 0;
1647
1648 if (!(env->psw.mask & PSW_MASK_64)) {
1649 array &= 0x7fffffff;
1650 l = (uint32_t)l;
1651 }
1652
1653 /* Lest we fail to service interrupts in a timely manner, limit the
1654 amount of work we're willing to do. For now, let's cap at 8k. */
1655 if (l > 0x2000) {
1656 l = 0x2000;
1657 cc = 3;
1658 }
1659
1660 for (i = 0; i < l; i++) {
1661 uint8_t byte, new_byte;
1662
1663 byte = cpu_ldub_data_ra(env, array + i, ra);
1664
1665 if (byte == end) {
1666 cc = 1;
1667 break;
1668 }
1669
1670 new_byte = cpu_ldub_data_ra(env, trans + byte, ra);
1671 cpu_stb_data_ra(env, array + i, new_byte, ra);
1672 }
1673
1674 env->cc_op = cc;
1675 return int128_make128(len - i, array + i);
1676 }
1677
1678 static inline uint32_t do_helper_trt(CPUS390XState *env, int len,
1679 uint64_t array, uint64_t trans,
1680 int inc, uintptr_t ra)
1681 {
1682 int i;
1683
1684 for (i = 0; i <= len; i++) {
1685 uint8_t byte = cpu_ldub_data_ra(env, array + i * inc, ra);
1686 uint8_t sbyte = cpu_ldub_data_ra(env, trans + byte, ra);
1687
1688 if (sbyte != 0) {
1689 set_address(env, 1, array + i * inc);
1690 env->regs[2] = deposit64(env->regs[2], 0, 8, sbyte);
1691 return (i == len) ? 2 : 1;
1692 }
1693 }
1694
1695 return 0;
1696 }
1697
1698 static uint32_t do_helper_trt_fwd(CPUS390XState *env, uint32_t len,
1699 uint64_t array, uint64_t trans,
1700 uintptr_t ra)
1701 {
1702 return do_helper_trt(env, len, array, trans, 1, ra);
1703 }
1704
1705 uint32_t HELPER(trt)(CPUS390XState *env, uint32_t len, uint64_t array,
1706 uint64_t trans)
1707 {
1708 return do_helper_trt(env, len, array, trans, 1, GETPC());
1709 }
1710
1711 static uint32_t do_helper_trt_bkwd(CPUS390XState *env, uint32_t len,
1712 uint64_t array, uint64_t trans,
1713 uintptr_t ra)
1714 {
1715 return do_helper_trt(env, len, array, trans, -1, ra);
1716 }
1717
1718 uint32_t HELPER(trtr)(CPUS390XState *env, uint32_t len, uint64_t array,
1719 uint64_t trans)
1720 {
1721 return do_helper_trt(env, len, array, trans, -1, GETPC());
1722 }
1723
1724 /* Translate one/two to one/two */
1725 uint32_t HELPER(trXX)(CPUS390XState *env, uint32_t r1, uint32_t r2,
1726 uint32_t tst, uint32_t sizes)
1727 {
1728 uintptr_t ra = GETPC();
1729 int dsize = (sizes & 1) ? 1 : 2;
1730 int ssize = (sizes & 2) ? 1 : 2;
1731 uint64_t tbl = get_address(env, 1);
1732 uint64_t dst = get_address(env, r1);
1733 uint64_t len = get_length(env, r1 + 1);
1734 uint64_t src = get_address(env, r2);
1735 uint32_t cc = 3;
1736 int i;
1737
1738 /* The lower address bits of TBL are ignored. For TROO, TROT, it's
1739 the low 3 bits (double-word aligned). For TRTO, TRTT, it's either
1740 the low 12 bits (4K, without ETF2-ENH) or 3 bits (with ETF2-ENH). */
1741 if (ssize == 2 && !s390_has_feat(S390_FEAT_ETF2_ENH)) {
1742 tbl &= -4096;
1743 } else {
1744 tbl &= -8;
1745 }
1746
1747 check_alignment(env, len, ssize, ra);
1748
1749 /* Lest we fail to service interrupts in a timely manner, */
1750 /* limit the amount of work we're willing to do. */
1751 for (i = 0; i < 0x2000; i++) {
1752 uint16_t sval = cpu_ldusize_data_ra(env, src, ssize, ra);
1753 uint64_t tble = tbl + (sval * dsize);
1754 uint16_t dval = cpu_ldusize_data_ra(env, tble, dsize, ra);
1755 if (dval == tst) {
1756 cc = 1;
1757 break;
1758 }
1759 cpu_stsize_data_ra(env, dst, dval, dsize, ra);
1760
1761 len -= ssize;
1762 src += ssize;
1763 dst += dsize;
1764
1765 if (len == 0) {
1766 cc = 0;
1767 break;
1768 }
1769 }
1770
1771 set_address(env, r1, dst);
1772 set_length(env, r1 + 1, len);
1773 set_address(env, r2, src);
1774
1775 return cc;
1776 }
1777
1778 static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1,
1779 uint64_t a2, bool parallel)
1780 {
1781 uint32_t mem_idx = s390x_env_mmu_index(env, false);
1782 MemOpIdx oi16 = make_memop_idx(MO_BE | MO_128, mem_idx);
1783 MemOpIdx oi8 = make_memop_idx(MO_BE | MO_64, mem_idx);
1784 MemOpIdx oi4 = make_memop_idx(MO_BE | MO_32, mem_idx);
1785 MemOpIdx oi2 = make_memop_idx(MO_BE | MO_16, mem_idx);
1786 MemOpIdx oi1 = make_memop_idx(MO_8, mem_idx);
1787 uintptr_t ra = GETPC();
1788 uint32_t fc = extract32(env->regs[0], 0, 8);
1789 uint32_t sc = extract32(env->regs[0], 8, 8);
1790 uint64_t pl = get_address(env, 1) & -16;
1791 uint64_t svh, svl;
1792 uint32_t cc;
1793
1794 /* Sanity check the function code and storage characteristic. */
1795 if (fc > 1 || sc > 3) {
1796 if (!s390_has_feat(S390_FEAT_COMPARE_AND_SWAP_AND_STORE_2)) {
1797 goto spec_exception;
1798 }
1799 if (fc > 2 || sc > 4 || (fc == 2 && (r3 & 1))) {
1800 goto spec_exception;
1801 }
1802 }
1803
1804 /* Sanity check the alignments. */
1805 if (extract32(a1, 0, fc + 2) || extract32(a2, 0, sc)) {
1806 goto spec_exception;
1807 }
1808
1809 /* Sanity check writability of the store address. */
1810 probe_write(env, a2, 1 << sc, mem_idx, ra);
1811
1812 /*
1813 * Note that the compare-and-swap is atomic, and the store is atomic,
1814 * but the complete operation is not. Therefore we do not need to
1815 * assert serial context in order to implement this. That said,
1816 * restart early if we can't support either operation that is supposed
1817 * to be atomic.
1818 */
1819 if (parallel &&
1820 ((!HAVE_CMPXCHG128 && fc + 2 > MO_64) ||
1821 (!HAVE_ATOMIC128_RW && sc > MO_64))) {
1822 cpu_loop_exit_atomic(env_cpu(env), ra);
1823 }
1824
1825 /*
1826 * All loads happen before all stores. For simplicity, load the entire
1827 * store value area from the parameter list.
1828 */
1829 svh = cpu_ldq_mmu(env, pl + 16, oi8, ra);
1830 svl = cpu_ldq_mmu(env, pl + 24, oi8, ra);
1831
1832 switch (fc) {
1833 case 0:
1834 {
1835 uint32_t nv = cpu_ldl_mmu(env, pl, oi4, ra);
1836 uint32_t cv = env->regs[r3];
1837 uint32_t ov;
1838
1839 if (parallel) {
1840 ov = cpu_atomic_cmpxchgl_be_mmu(env, a1, cv, nv, oi4, ra);
1841 } else {
1842 ov = cpu_ldl_mmu(env, a1, oi4, ra);
1843 cpu_stl_mmu(env, a1, (ov == cv ? nv : ov), oi4, ra);
1844 }
1845 cc = (ov != cv);
1846 env->regs[r3] = deposit64(env->regs[r3], 32, 32, ov);
1847 }
1848 break;
1849
1850 case 1:
1851 {
1852 uint64_t nv = cpu_ldq_mmu(env, pl, oi8, ra);
1853 uint64_t cv = env->regs[r3];
1854 uint64_t ov;
1855
1856 if (parallel) {
1857 ov = cpu_atomic_cmpxchgq_be_mmu(env, a1, cv, nv, oi8, ra);
1858 } else {
1859 ov = cpu_ldq_mmu(env, a1, oi8, ra);
1860 cpu_stq_mmu(env, a1, (ov == cv ? nv : ov), oi8, ra);
1861 }
1862 cc = (ov != cv);
1863 env->regs[r3] = ov;
1864 }
1865 break;
1866
1867 case 2:
1868 {
1869 Int128 nv = cpu_ld16_mmu(env, pl, oi16, ra);
1870 Int128 cv = int128_make128(env->regs[r3 + 1], env->regs[r3]);
1871 Int128 ov;
1872
1873 if (!parallel) {
1874 ov = cpu_ld16_mmu(env, a1, oi16, ra);
1875 cc = !int128_eq(ov, cv);
1876 if (cc) {
1877 nv = ov;
1878 }
1879 cpu_st16_mmu(env, a1, nv, oi16, ra);
1880 } else if (HAVE_CMPXCHG128) {
1881 ov = cpu_atomic_cmpxchgo_be_mmu(env, a1, cv, nv, oi16, ra);
1882 cc = !int128_eq(ov, cv);
1883 } else {
1884 /* Note that we asserted !parallel above. */
1885 g_assert_not_reached();
1886 }
1887
1888 env->regs[r3 + 0] = int128_gethi(ov);
1889 env->regs[r3 + 1] = int128_getlo(ov);
1890 }
1891 break;
1892
1893 default:
1894 g_assert_not_reached();
1895 }
1896
1897 /* Store only if the comparison succeeded. Note that above we use a pair
1898 of 64-bit big-endian loads, so for sc < 3 we must extract the value
1899 from the most-significant bits of svh. */
1900 if (cc == 0) {
1901 switch (sc) {
1902 case 0:
1903 cpu_stb_mmu(env, a2, svh >> 56, oi1, ra);
1904 break;
1905 case 1:
1906 cpu_stw_mmu(env, a2, svh >> 48, oi2, ra);
1907 break;
1908 case 2:
1909 cpu_stl_mmu(env, a2, svh >> 32, oi4, ra);
1910 break;
1911 case 3:
1912 cpu_stq_mmu(env, a2, svh, oi8, ra);
1913 break;
1914 case 4:
1915 cpu_st16_mmu(env, a2, int128_make128(svl, svh), oi16, ra);
1916 break;
1917 default:
1918 g_assert_not_reached();
1919 }
1920 }
1921
1922 return cc;
1923
1924 spec_exception:
1925 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
1926 }
1927
1928 uint32_t HELPER(csst)(CPUS390XState *env, uint32_t r3, uint64_t a1, uint64_t a2)
1929 {
1930 return do_csst(env, r3, a1, a2, false);
1931 }
1932
1933 uint32_t HELPER(csst_parallel)(CPUS390XState *env, uint32_t r3, uint64_t a1,
1934 uint64_t a2)
1935 {
1936 return do_csst(env, r3, a1, a2, true);
1937 }
1938
1939 #if !defined(CONFIG_USER_ONLY)
1940 void HELPER(lctlg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
1941 {
1942 uintptr_t ra = GETPC();
1943 bool PERchanged = false;
1944 uint64_t src = a2;
1945 uint32_t i;
1946
1947 if (src & 0x7) {
1948 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
1949 }
1950
1951 for (i = r1;; i = (i + 1) % 16) {
1952 uint64_t val = cpu_ldq_be_data_ra(env, src, ra);
1953 if (env->cregs[i] != val && i >= 9 && i <= 11) {
1954 PERchanged = true;
1955 }
1956 if (i == 0 && !(env->cregs[i] & CR0_CKC_SC) && (val & CR0_CKC_SC)) {
1957 BQL_LOCK_GUARD();
1958 tcg_s390_tod_updated(env_cpu(env), RUN_ON_CPU_NULL);
1959 }
1960 env->cregs[i] = val;
1961 HELPER_LOG("load ctl %d from 0x%" PRIx64 " == 0x%" PRIx64 "\n",
1962 i, src, val);
1963 src += sizeof(uint64_t);
1964
1965 if (i == r3) {
1966 break;
1967 }
1968 }
1969
1970 if (PERchanged && env->psw.mask & PSW_MASK_PER) {
1971 s390_cpu_recompute_watchpoints(env_cpu(env));
1972 }
1973
1974 tlb_flush(env_cpu(env));
1975 }
1976
1977 void HELPER(lctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
1978 {
1979 uintptr_t ra = GETPC();
1980 bool PERchanged = false;
1981 uint64_t src = a2;
1982 uint32_t i;
1983
1984 if (src & 0x3) {
1985 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
1986 }
1987
1988 for (i = r1;; i = (i + 1) % 16) {
1989 uint32_t val = cpu_ldl_be_data_ra(env, src, ra);
1990 uint64_t val64 = deposit64(env->cregs[i], 0, 32, val);
1991 if ((uint32_t)env->cregs[i] != val && i >= 9 && i <= 11) {
1992 PERchanged = true;
1993 }
1994 if (i == 0 && !(env->cregs[i] & CR0_CKC_SC) && (val64 & CR0_CKC_SC)) {
1995 BQL_LOCK_GUARD();
1996 tcg_s390_tod_updated(env_cpu(env), RUN_ON_CPU_NULL);
1997 }
1998 env->cregs[i] = val64;
1999 HELPER_LOG("load ctl %d from 0x%" PRIx64 " == 0x%x\n", i, src, val);
2000 src += sizeof(uint32_t);
2001
2002 if (i == r3) {
2003 break;
2004 }
2005 }
2006
2007 if (PERchanged && env->psw.mask & PSW_MASK_PER) {
2008 s390_cpu_recompute_watchpoints(env_cpu(env));
2009 }
2010
2011 tlb_flush(env_cpu(env));
2012 }
2013
2014 void HELPER(stctg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
2015 {
2016 uintptr_t ra = GETPC();
2017 uint64_t dest = a2;
2018 uint32_t i;
2019
2020 if (dest & 0x7) {
2021 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
2022 }
2023
2024 for (i = r1;; i = (i + 1) % 16) {
2025 cpu_stq_be_data_ra(env, dest, env->cregs[i], ra);
2026 dest += sizeof(uint64_t);
2027
2028 if (i == r3) {
2029 break;
2030 }
2031 }
2032 }
2033
2034 void HELPER(stctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
2035 {
2036 uintptr_t ra = GETPC();
2037 uint64_t dest = a2;
2038 uint32_t i;
2039
2040 if (dest & 0x3) {
2041 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
2042 }
2043
2044 for (i = r1;; i = (i + 1) % 16) {
2045 cpu_stl_be_data_ra(env, dest, env->cregs[i], ra);
2046 dest += sizeof(uint32_t);
2047
2048 if (i == r3) {
2049 break;
2050 }
2051 }
2052 }
2053
2054 uint32_t HELPER(testblock)(CPUS390XState *env, uint64_t real_addr)
2055 {
2056 uintptr_t ra = GETPC();
2057 int i;
2058
2059 real_addr = wrap_address(env, real_addr) & TARGET_PAGE_MASK;
2060
2061 for (i = 0; i < TARGET_PAGE_SIZE; i += 8) {
2062 cpu_stq_be_mmuidx_ra(env, real_addr + i, 0, MMU_REAL_IDX, ra);
2063 }
2064
2065 return 0;
2066 }
2067
2068 uint32_t HELPER(tprot)(CPUS390XState *env, uint64_t a1, uint64_t a2)
2069 {
2070 S390CPU *cpu = env_archcpu(env);
2071 CPUState *cs = env_cpu(env);
2072
2073 /*
2074 * TODO: we currently don't handle all access protection types
2075 * (including access-list and key-controlled) as well as AR mode.
2076 */
2077 if (!s390_cpu_virt_mem_check_write(cpu, a1, 0, 1)) {
2078 /* Fetching permitted; storing permitted */
2079 return 0;
2080 }
2081
2082 if (env->int_pgm_code == PGM_PROTECTION) {
2083 /* retry if reading is possible */
2084 cs->exception_index = -1;
2085 if (!s390_cpu_virt_mem_check_read(cpu, a1, 0, 1)) {
2086 /* Fetching permitted; storing not permitted */
2087 return 1;
2088 }
2089 }
2090
2091 switch (env->int_pgm_code) {
2092 case PGM_PROTECTION:
2093 /* Fetching not permitted; storing not permitted */
2094 cs->exception_index = -1;
2095 return 2;
2096 case PGM_ADDRESSING:
2097 case PGM_TRANS_SPEC:
2098 /* exceptions forwarded to the guest */
2099 s390_cpu_virt_mem_handle_exc(cpu, GETPC());
2100 return 0;
2101 }
2102
2103 /* Translation not available */
2104 cs->exception_index = -1;
2105 return 3;
2106 }
2107
2108 /* insert storage key extended */
2109 uint64_t HELPER(iske)(CPUS390XState *env, uint64_t r2)
2110 {
2111 static S390SKeysState *ss;
2112 static S390SKeysClass *skeyclass;
2113 uint64_t addr = wrap_address(env, r2);
2114 uint8_t key;
2115 int rc;
2116
2117 addr = mmu_real2abs(env, addr);
2118 if (!mmu_absolute_addr_valid(addr, false)) {
2119 tcg_s390_program_interrupt(env, PGM_ADDRESSING, GETPC());
2120 }
2121
2122 if (unlikely(!ss)) {
2123 ss = s390_get_skeys_device();
2124 skeyclass = S390_SKEYS_GET_CLASS(ss);
2125 if (skeyclass->enable_skeys && !skeyclass->enable_skeys(ss)) {
2126 tlb_flush_all_cpus_synced(env_cpu(env));
2127 }
2128 }
2129
2130 rc = s390_skeys_get(ss, addr / TARGET_PAGE_SIZE, 1, &key);
2131 if (rc) {
2132 return 0;
2133 }
2134 return key;
2135 }
2136
2137 /* set storage key extended */
2138 void HELPER(sske)(CPUS390XState *env, uint64_t r1, uint64_t r2)
2139 {
2140 static S390SKeysState *ss;
2141 static S390SKeysClass *skeyclass;
2142 uint64_t addr = wrap_address(env, r2);
2143 uint8_t key;
2144
2145 addr = mmu_real2abs(env, addr);
2146 if (!mmu_absolute_addr_valid(addr, false)) {
2147 tcg_s390_program_interrupt(env, PGM_ADDRESSING, GETPC());
2148 }
2149
2150 if (unlikely(!ss)) {
2151 ss = s390_get_skeys_device();
2152 skeyclass = S390_SKEYS_GET_CLASS(ss);
2153 if (skeyclass->enable_skeys && !skeyclass->enable_skeys(ss)) {
2154 tlb_flush_all_cpus_synced(env_cpu(env));
2155 }
2156 }
2157
2158 key = r1 & 0xfe;
2159 s390_skeys_set(ss, addr / TARGET_PAGE_SIZE, 1, &key);
2160 /*
2161 * As we can only flush by virtual address and not all the entries
2162 * that point to a physical address we have to flush the whole TLB.
2163 */
2164 tlb_flush_all_cpus_synced(env_cpu(env));
2165 }
2166
2167 /* reset reference bit extended */
2168 uint32_t HELPER(rrbe)(CPUS390XState *env, uint64_t r2)
2169 {
2170 uint64_t addr = wrap_address(env, r2);
2171 static S390SKeysState *ss;
2172 static S390SKeysClass *skeyclass;
2173 uint8_t re, key;
2174 int rc;
2175
2176 addr = mmu_real2abs(env, addr);
2177 if (!mmu_absolute_addr_valid(addr, false)) {
2178 tcg_s390_program_interrupt(env, PGM_ADDRESSING, GETPC());
2179 }
2180
2181 if (unlikely(!ss)) {
2182 ss = s390_get_skeys_device();
2183 skeyclass = S390_SKEYS_GET_CLASS(ss);
2184 if (skeyclass->enable_skeys && !skeyclass->enable_skeys(ss)) {
2185 tlb_flush_all_cpus_synced(env_cpu(env));
2186 }
2187 }
2188
2189 rc = s390_skeys_get(ss, addr / TARGET_PAGE_SIZE, 1, &key);
2190 if (rc) {
2191 return 0;
2192 }
2193
2194 re = key & (SK_R | SK_C);
2195 key &= ~SK_R;
2196
2197 rc = s390_skeys_set(ss, addr / TARGET_PAGE_SIZE, 1, &key);
2198 if (rc) {
2199 return 0;
2200 }
2201 /*
2202 * As we can only flush by virtual address and not all the entries
2203 * that point to a physical address we have to flush the whole TLB.
2204 */
2205 tlb_flush_all_cpus_synced(env_cpu(env));
2206
2207 /*
2208 * cc
2209 *
2210 * 0 Reference bit zero; change bit zero
2211 * 1 Reference bit zero; change bit one
2212 * 2 Reference bit one; change bit zero
2213 * 3 Reference bit one; change bit one
2214 */
2215
2216 return re >> 1;
2217 }
2218
2219 uint32_t HELPER(mvcs)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2,
2220 uint64_t key)
2221 {
2222 const uint8_t psw_as = (env->psw.mask & PSW_MASK_ASC) >> PSW_SHIFT_ASC;
2223 S390Access srca, desta;
2224 uintptr_t ra = GETPC();
2225 int cc = 0;
2226
2227 HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n",
2228 __func__, l, a1, a2);
2229
2230 if (!(env->psw.mask & PSW_MASK_DAT) || !(env->cregs[0] & CR0_SECONDARY) ||
2231 psw_as == AS_HOME || psw_as == AS_ACCREG) {
2232 s390_program_interrupt(env, PGM_SPECIAL_OP, ra);
2233 }
2234
2235 if (!psw_key_valid(env, (key >> 4) & 0xf)) {
2236 s390_program_interrupt(env, PGM_PRIVILEGED, ra);
2237 }
2238
2239 l = wrap_length32(env, l);
2240 if (l > 256) {
2241 /* max 256 */
2242 l = 256;
2243 cc = 3;
2244 } else if (!l) {
2245 return cc;
2246 }
2247
2248 access_prepare(&srca, env, a2, l, MMU_DATA_LOAD, MMU_PRIMARY_IDX, ra);
2249 access_prepare(&desta, env, a1, l, MMU_DATA_STORE, MMU_SECONDARY_IDX, ra);
2250 access_memmove(env, &desta, &srca, ra);
2251 return cc;
2252 }
2253
2254 uint32_t HELPER(mvcp)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2,
2255 uint64_t key)
2256 {
2257 const uint8_t psw_as = (env->psw.mask & PSW_MASK_ASC) >> PSW_SHIFT_ASC;
2258 S390Access srca, desta;
2259 uintptr_t ra = GETPC();
2260 int cc = 0;
2261
2262 HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n",
2263 __func__, l, a1, a2);
2264
2265 if (!(env->psw.mask & PSW_MASK_DAT) || !(env->cregs[0] & CR0_SECONDARY) ||
2266 psw_as == AS_HOME || psw_as == AS_ACCREG) {
2267 s390_program_interrupt(env, PGM_SPECIAL_OP, ra);
2268 }
2269
2270 if (!psw_key_valid(env, (key >> 4) & 0xf)) {
2271 s390_program_interrupt(env, PGM_PRIVILEGED, ra);
2272 }
2273
2274 l = wrap_length32(env, l);
2275 if (l > 256) {
2276 /* max 256 */
2277 l = 256;
2278 cc = 3;
2279 } else if (!l) {
2280 return cc;
2281 }
2282 access_prepare(&srca, env, a2, l, MMU_DATA_LOAD, MMU_SECONDARY_IDX, ra);
2283 access_prepare(&desta, env, a1, l, MMU_DATA_STORE, MMU_PRIMARY_IDX, ra);
2284 access_memmove(env, &desta, &srca, ra);
2285 return cc;
2286 }
2287
2288 void HELPER(idte)(CPUS390XState *env, uint64_t r1, uint64_t r2, uint32_t m4)
2289 {
2290 CPUState *cs = env_cpu(env);
2291 const uintptr_t ra = GETPC();
2292 uint64_t table, entry, raddr;
2293 uint16_t entries, i, index = 0;
2294
2295 if (r2 & 0xff000) {
2296 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, ra);
2297 }
2298
2299 if (!(r2 & 0x800)) {
2300 /* invalidation-and-clearing operation */
2301 table = r1 & ASCE_ORIGIN;
2302 entries = (r2 & 0x7ff) + 1;
2303
2304 switch (r1 & ASCE_TYPE_MASK) {
2305 case ASCE_TYPE_REGION1:
2306 index = (r2 >> 53) & 0x7ff;
2307 break;
2308 case ASCE_TYPE_REGION2:
2309 index = (r2 >> 42) & 0x7ff;
2310 break;
2311 case ASCE_TYPE_REGION3:
2312 index = (r2 >> 31) & 0x7ff;
2313 break;
2314 case ASCE_TYPE_SEGMENT:
2315 index = (r2 >> 20) & 0x7ff;
2316 break;
2317 }
2318 for (i = 0; i < entries; i++) {
2319 /* addresses are not wrapped in 24/31bit mode but table index is */
2320 raddr = table + ((index + i) & 0x7ff) * sizeof(entry);
2321 entry = cpu_ldq_be_mmuidx_ra(env, raddr, MMU_REAL_IDX, ra);
2322 if (!(entry & REGION_ENTRY_I)) {
2323 /* we are allowed to not store if already invalid */
2324 entry |= REGION_ENTRY_I;
2325 cpu_stq_be_mmuidx_ra(env, raddr, entry, MMU_REAL_IDX, ra);
2326 }
2327 }
2328 }
2329
2330 /* We simply flush the complete tlb, therefore we can ignore r3. */
2331 if (m4 & 1) {
2332 tlb_flush(cs);
2333 } else {
2334 tlb_flush_all_cpus_synced(cs);
2335 }
2336 }
2337
2338 /* invalidate pte */
2339 void HELPER(ipte)(CPUS390XState *env, uint64_t pto, uint64_t vaddr,
2340 uint32_t m4)
2341 {
2342 CPUState *cs = env_cpu(env);
2343 const uintptr_t ra = GETPC();
2344 uint64_t page = vaddr & TARGET_PAGE_MASK;
2345 uint64_t pte_addr, pte;
2346
2347 /* Compute the page table entry address */
2348 pte_addr = (pto & SEGMENT_ENTRY_ORIGIN);
2349 pte_addr += VADDR_PAGE_TX(vaddr) * 8;
2350
2351 /* Mark the page table entry as invalid */
2352 pte = cpu_ldq_be_mmuidx_ra(env, pte_addr, MMU_REAL_IDX, ra);
2353 pte |= PAGE_ENTRY_I;
2354 cpu_stq_be_mmuidx_ra(env, pte_addr, pte, MMU_REAL_IDX, ra);
2355
2356 /* XXX we exploit the fact that Linux passes the exact virtual
2357 address here - it's not obliged to! */
2358 if (m4 & 1) {
2359 if (vaddr & ~VADDR_PAGE_TX_MASK) {
2360 tlb_flush_page(cs, page);
2361 /* XXX 31-bit hack */
2362 tlb_flush_page(cs, page ^ 0x80000000);
2363 } else {
2364 /* looks like we don't have a valid virtual address */
2365 tlb_flush(cs);
2366 }
2367 } else {
2368 if (vaddr & ~VADDR_PAGE_TX_MASK) {
2369 tlb_flush_page_all_cpus_synced(cs, page);
2370 /* XXX 31-bit hack */
2371 tlb_flush_page_all_cpus_synced(cs, page ^ 0x80000000);
2372 } else {
2373 /* looks like we don't have a valid virtual address */
2374 tlb_flush_all_cpus_synced(cs);
2375 }
2376 }
2377 }
2378
2379 /* flush local tlb */
2380 void HELPER(ptlb)(CPUS390XState *env)
2381 {
2382 tlb_flush(env_cpu(env));
2383 }
2384
2385 /* flush global tlb */
2386 void HELPER(purge)(CPUS390XState *env)
2387 {
2388 tlb_flush_all_cpus_synced(env_cpu(env));
2389 }
2390
2391 /* load real address */
2392 uint64_t HELPER(lra)(CPUS390XState *env, uint64_t r1, uint64_t addr)
2393 {
2394 uint64_t asc = env->psw.mask & PSW_MASK_ASC;
2395 uint64_t ret, tec;
2396 int flags, exc, cc;
2397
2398 /* XXX incomplete - has more corner cases */
2399 if (!(env->psw.mask & PSW_MASK_64) && (addr >> 32)) {
2400 tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, GETPC());
2401 }
2402
2403 exc = mmu_translate(env, addr, MMU_S390_LRA, asc, &ret, &flags, &tec);
2404 if (exc) {
2405 cc = 3;
2406 ret = (r1 & 0xFFFFFFFF00000000ULL) | exc | 0x80000000;
2407 } else {
2408 cc = 0;
2409 ret |= addr & ~TARGET_PAGE_MASK;
2410 }
2411
2412 env->cc_op = cc;
2413 return ret;
2414 }
2415 #endif
2416
2417 /* Execute instruction. This instruction executes an insn modified with
2418 the contents of r1. It does not change the executed instruction in memory;
2419 it does not change the program counter.
2420
2421 Perform this by recording the modified instruction in env->ex_value.
2422 This will be noticed by cpu_get_tb_cpu_state and thus tb translation.
2423 */
2424 void HELPER(ex)(CPUS390XState *env, uint32_t ilen, uint64_t r1, uint64_t addr)
2425 {
2426 CPUState *cs = env_cpu(env);
2427 uint64_t insn;
2428 uint8_t opc;
2429 MemOpIdx oi;
2430
2431 /* EXECUTE targets must be at even addresses. */
2432 if (addr & 1) {
2433 tcg_s390_program_interrupt(env, PGM_SPECIFICATION, GETPC());
2434 }
2435
2436 oi = make_memop_idx(MO_BEUW, cpu_mmu_index(cs, true));
2437 insn = cpu_ldw_code_mmu(env, addr, oi, 0);
2438 opc = insn >> 8;
2439
2440 /* Or in the contents of R1[56:63]. */
2441 insn |= r1 & 0xff;
2442
2443 /* Load the rest of the instruction. */
2444 insn <<= 48;
2445 switch (get_ilen(opc)) {
2446 case 2:
2447 break;
2448 case 4:
2449 insn |= (uint64_t)cpu_ldw_code_mmu(env, addr + 2, oi, 0) << 32;
2450 break;
2451 case 6:
2452 oi = make_memop_idx(MO_BEUL, cpu_mmu_index(cs, true));
2453 insn |= (uint64_t)(uint32_t)cpu_ldl_code_mmu(env, addr + 2, oi, 0) << 16;
2454 break;
2455 default:
2456 g_assert_not_reached();
2457 }
2458
2459 /* The very most common cases can be sped up by avoiding a new TB. */
2460 if ((opc & 0xf0) == 0xd0) {
2461 typedef uint32_t (*dx_helper)(CPUS390XState *, uint32_t, uint64_t,
2462 uint64_t, uintptr_t);
2463 static const dx_helper dx[16] = {
2464 [0x0] = do_helper_trt_bkwd,
2465 [0x2] = do_helper_mvc,
2466 [0x4] = do_helper_nc,
2467 [0x5] = do_helper_clc,
2468 [0x6] = do_helper_oc,
2469 [0x7] = do_helper_xc,
2470 [0xc] = do_helper_tr,
2471 [0xd] = do_helper_trt_fwd,
2472 };
2473 dx_helper helper = dx[opc & 0xf];
2474
2475 if (helper) {
2476 uint32_t l = extract64(insn, 48, 8);
2477 uint32_t b1 = extract64(insn, 44, 4);
2478 uint32_t d1 = extract64(insn, 32, 12);
2479 uint32_t b2 = extract64(insn, 28, 4);
2480 uint32_t d2 = extract64(insn, 16, 12);
2481 uint64_t a1 = wrap_address(env, (b1 ? env->regs[b1] : 0) + d1);
2482 uint64_t a2 = wrap_address(env, (b2 ? env->regs[b2] : 0) + d2);
2483
2484 env->cc_op = helper(env, l, a1, a2, 0);
2485 env->psw.addr += ilen;
2486 return;
2487 }
2488 } else if (opc == 0x0a) {
2489 env->int_svc_code = extract64(insn, 48, 8);
2490 env->int_svc_ilen = ilen;
2491 helper_exception(env, EXCP_SVC);
2492 g_assert_not_reached();
2493 }
2494
2495 /* Record the insn we want to execute as well as the ilen to use
2496 during the execution of the target insn. This will also ensure
2497 that ex_value is non-zero, which flags that we are in a state
2498 that requires such execution. */
2499 env->ex_value = insn | ilen;
2500 env->ex_target = addr;
2501 }
2502
2503 uint32_t HELPER(mvcos)(CPUS390XState *env, uint64_t dest, uint64_t src,
2504 uint64_t len)
2505 {
2506 const uint8_t psw_key = (env->psw.mask & PSW_MASK_KEY) >> PSW_SHIFT_KEY;
2507 const uint8_t psw_as = (env->psw.mask & PSW_MASK_ASC) >> PSW_SHIFT_ASC;
2508 const uint64_t r0 = env->regs[0];
2509 const uintptr_t ra = GETPC();
2510 uint8_t dest_key, dest_as, dest_k, dest_a;
2511 uint8_t src_key, src_as, src_k, src_a;
2512 uint64_t val;
2513 int cc = 0;
2514
2515 HELPER_LOG("%s dest %" PRIx64 ", src %" PRIx64 ", len %" PRIx64 "\n",
2516 __func__, dest, src, len);
2517
2518 if (!(env->psw.mask & PSW_MASK_DAT)) {
2519 tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, ra);
2520 }
2521
2522 /* OAC (operand access control) for the first operand -> dest */
2523 val = (r0 & 0xffff0000ULL) >> 16;
2524 dest_key = (val >> 12) & 0xf;
2525 dest_as = (val >> 6) & 0x3;
2526 dest_k = (val >> 1) & 0x1;
2527 dest_a = val & 0x1;
2528
2529 /* OAC (operand access control) for the second operand -> src */
2530 val = (r0 & 0x0000ffffULL);
2531 src_key = (val >> 12) & 0xf;
2532 src_as = (val >> 6) & 0x3;
2533 src_k = (val >> 1) & 0x1;
2534 src_a = val & 0x1;
2535
2536 if (!dest_k) {
2537 dest_key = psw_key;
2538 }
2539 if (!src_k) {
2540 src_key = psw_key;
2541 }
2542 if (!dest_a) {
2543 dest_as = psw_as;
2544 }
2545 if (!src_a) {
2546 src_as = psw_as;
2547 }
2548
2549 if (dest_a && dest_as == AS_HOME && (env->psw.mask & PSW_MASK_PSTATE)) {
2550 tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, ra);
2551 }
2552 if (!(env->cregs[0] & CR0_SECONDARY) &&
2553 (dest_as == AS_SECONDARY || src_as == AS_SECONDARY)) {
2554 tcg_s390_program_interrupt(env, PGM_SPECIAL_OP, ra);
2555 }
2556 if (!psw_key_valid(env, dest_key) || !psw_key_valid(env, src_key)) {
2557 tcg_s390_program_interrupt(env, PGM_PRIVILEGED, ra);
2558 }
2559
2560 len = wrap_length32(env, len);
2561 if (len > 4096) {
2562 cc = 3;
2563 len = 4096;
2564 }
2565
2566 /* FIXME: AR-mode and proper problem state mode (using PSW keys) missing */
2567 if (src_as == AS_ACCREG || dest_as == AS_ACCREG ||
2568 (env->psw.mask & PSW_MASK_PSTATE)) {
2569 qemu_log_mask(LOG_UNIMP, "%s: AR-mode and PSTATE support missing\n",
2570 __func__);
2571 tcg_s390_program_interrupt(env, PGM_ADDRESSING, ra);
2572 }
2573
2574 /* FIXME: Access using correct keys and AR-mode */
2575 if (len) {
2576 S390Access srca, desta;
2577
2578 access_prepare(&srca, env, src, len, MMU_DATA_LOAD,
2579 mmu_idx_from_as(src_as), ra);
2580 access_prepare(&desta, env, dest, len, MMU_DATA_STORE,
2581 mmu_idx_from_as(dest_as), ra);
2582
2583 access_memmove(env, &desta, &srca, ra);
2584 }
2585
2586 return cc;
2587 }
2588
2589 /* Decode a Unicode character. A return value < 0 indicates success, storing
2590 the UTF-32 result into OCHAR and the input length into OLEN. A return
2591 value >= 0 indicates failure, and the CC value to be returned. */
2592 typedef int (*decode_unicode_fn)(CPUS390XState *env, uint64_t addr,
2593 uint64_t ilen, bool enh_check, uintptr_t ra,
2594 uint32_t *ochar, uint32_t *olen);
2595
2596 /* Encode a Unicode character. A return value < 0 indicates success, storing
2597 the bytes into ADDR and the output length into OLEN. A return value >= 0
2598 indicates failure, and the CC value to be returned. */
2599 typedef int (*encode_unicode_fn)(CPUS390XState *env, uint64_t addr,
2600 uint64_t ilen, uintptr_t ra, uint32_t c,
2601 uint32_t *olen);
2602
2603 static int decode_utf8(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2604 bool enh_check, uintptr_t ra,
2605 uint32_t *ochar, uint32_t *olen)
2606 {
2607 uint8_t s0, s1, s2, s3;
2608 uint32_t c, l;
2609
2610 if (ilen < 1) {
2611 return 0;
2612 }
2613 s0 = cpu_ldub_data_ra(env, addr, ra);
2614 if (s0 <= 0x7f) {
2615 /* one byte character */
2616 l = 1;
2617 c = s0;
2618 } else if (s0 <= (enh_check ? 0xc1 : 0xbf)) {
2619 /* invalid character */
2620 return 2;
2621 } else if (s0 <= 0xdf) {
2622 /* two byte character */
2623 l = 2;
2624 if (ilen < 2) {
2625 return 0;
2626 }
2627 s1 = cpu_ldub_data_ra(env, addr + 1, ra);
2628 c = s0 & 0x1f;
2629 c = (c << 6) | (s1 & 0x3f);
2630 if (enh_check && (s1 & 0xc0) != 0x80) {
2631 return 2;
2632 }
2633 } else if (s0 <= 0xef) {
2634 /* three byte character */
2635 l = 3;
2636 if (ilen < 3) {
2637 return 0;
2638 }
2639 s1 = cpu_ldub_data_ra(env, addr + 1, ra);
2640 s2 = cpu_ldub_data_ra(env, addr + 2, ra);
2641 c = s0 & 0x0f;
2642 c = (c << 6) | (s1 & 0x3f);
2643 c = (c << 6) | (s2 & 0x3f);
2644 /* Fold the byte-by-byte range descriptions in the PoO into
2645 tests against the complete value. It disallows encodings
2646 that could be smaller, and the UTF-16 surrogates. */
2647 if (enh_check
2648 && ((s1 & 0xc0) != 0x80
2649 || (s2 & 0xc0) != 0x80
2650 || c < 0x1000
2651 || (c >= 0xd800 && c <= 0xdfff))) {
2652 return 2;
2653 }
2654 } else if (s0 <= (enh_check ? 0xf4 : 0xf7)) {
2655 /* four byte character */
2656 l = 4;
2657 if (ilen < 4) {
2658 return 0;
2659 }
2660 s1 = cpu_ldub_data_ra(env, addr + 1, ra);
2661 s2 = cpu_ldub_data_ra(env, addr + 2, ra);
2662 s3 = cpu_ldub_data_ra(env, addr + 3, ra);
2663 c = s0 & 0x07;
2664 c = (c << 6) | (s1 & 0x3f);
2665 c = (c << 6) | (s2 & 0x3f);
2666 c = (c << 6) | (s3 & 0x3f);
2667 /* See above. */
2668 if (enh_check
2669 && ((s1 & 0xc0) != 0x80
2670 || (s2 & 0xc0) != 0x80
2671 || (s3 & 0xc0) != 0x80
2672 || c < 0x010000
2673 || c > 0x10ffff)) {
2674 return 2;
2675 }
2676 } else {
2677 /* invalid character */
2678 return 2;
2679 }
2680
2681 *ochar = c;
2682 *olen = l;
2683 return -1;
2684 }
2685
2686 static int decode_utf16(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2687 bool enh_check, uintptr_t ra,
2688 uint32_t *ochar, uint32_t *olen)
2689 {
2690 uint16_t s0, s1;
2691 uint32_t c, l;
2692
2693 if (ilen < 2) {
2694 return 0;
2695 }
2696 s0 = cpu_lduw_be_data_ra(env, addr, ra);
2697 if ((s0 & 0xfc00) != 0xd800) {
2698 /* one word character */
2699 l = 2;
2700 c = s0;
2701 } else {
2702 /* two word character */
2703 l = 4;
2704 if (ilen < 4) {
2705 return 0;
2706 }
2707 s1 = cpu_lduw_be_data_ra(env, addr + 2, ra);
2708 c = extract32(s0, 6, 4) + 1;
2709 c = (c << 6) | (s0 & 0x3f);
2710 c = (c << 10) | (s1 & 0x3ff);
2711 if (enh_check && (s1 & 0xfc00) != 0xdc00) {
2712 /* invalid surrogate character */
2713 return 2;
2714 }
2715 }
2716
2717 *ochar = c;
2718 *olen = l;
2719 return -1;
2720 }
2721
2722 static int decode_utf32(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2723 bool enh_check, uintptr_t ra,
2724 uint32_t *ochar, uint32_t *olen)
2725 {
2726 uint32_t c;
2727
2728 if (ilen < 4) {
2729 return 0;
2730 }
2731 c = cpu_ldl_be_data_ra(env, addr, ra);
2732 if ((c >= 0xd800 && c <= 0xdbff) || c > 0x10ffff) {
2733 /* invalid unicode character */
2734 return 2;
2735 }
2736
2737 *ochar = c;
2738 *olen = 4;
2739 return -1;
2740 }
2741
2742 static int encode_utf8(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2743 uintptr_t ra, uint32_t c, uint32_t *olen)
2744 {
2745 uint8_t d[4];
2746 uint32_t l, i;
2747
2748 if (c <= 0x7f) {
2749 /* one byte character */
2750 l = 1;
2751 d[0] = c;
2752 } else if (c <= 0x7ff) {
2753 /* two byte character */
2754 l = 2;
2755 d[1] = 0x80 | extract32(c, 0, 6);
2756 d[0] = 0xc0 | extract32(c, 6, 5);
2757 } else if (c <= 0xffff) {
2758 /* three byte character */
2759 l = 3;
2760 d[2] = 0x80 | extract32(c, 0, 6);
2761 d[1] = 0x80 | extract32(c, 6, 6);
2762 d[0] = 0xe0 | extract32(c, 12, 4);
2763 } else {
2764 /* four byte character */
2765 l = 4;
2766 d[3] = 0x80 | extract32(c, 0, 6);
2767 d[2] = 0x80 | extract32(c, 6, 6);
2768 d[1] = 0x80 | extract32(c, 12, 6);
2769 d[0] = 0xf0 | extract32(c, 18, 3);
2770 }
2771
2772 if (ilen < l) {
2773 return 1;
2774 }
2775 for (i = 0; i < l; ++i) {
2776 cpu_stb_data_ra(env, addr + i, d[i], ra);
2777 }
2778
2779 *olen = l;
2780 return -1;
2781 }
2782
2783 static int encode_utf16(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2784 uintptr_t ra, uint32_t c, uint32_t *olen)
2785 {
2786 uint16_t d0, d1;
2787
2788 if (c <= 0xffff) {
2789 /* one word character */
2790 if (ilen < 2) {
2791 return 1;
2792 }
2793 cpu_stw_be_data_ra(env, addr, c, ra);
2794 *olen = 2;
2795 } else {
2796 /* two word character */
2797 if (ilen < 4) {
2798 return 1;
2799 }
2800 d1 = 0xdc00 | extract32(c, 0, 10);
2801 d0 = 0xd800 | extract32(c, 10, 6);
2802 d0 = deposit32(d0, 6, 4, extract32(c, 16, 5) - 1);
2803 cpu_stw_be_data_ra(env, addr + 0, d0, ra);
2804 cpu_stw_be_data_ra(env, addr + 2, d1, ra);
2805 *olen = 4;
2806 }
2807
2808 return -1;
2809 }
2810
2811 static int encode_utf32(CPUS390XState *env, uint64_t addr, uint64_t ilen,
2812 uintptr_t ra, uint32_t c, uint32_t *olen)
2813 {
2814 if (ilen < 4) {
2815 return 1;
2816 }
2817 cpu_stl_be_data_ra(env, addr, c, ra);
2818 *olen = 4;
2819 return -1;
2820 }
2821
2822 static inline uint32_t convert_unicode(CPUS390XState *env, uint32_t r1,
2823 uint32_t r2, uint32_t m3, uintptr_t ra,
2824 decode_unicode_fn decode,
2825 encode_unicode_fn encode)
2826 {
2827 uint64_t dst = get_address(env, r1);
2828 uint64_t dlen = get_length(env, r1 + 1);
2829 uint64_t src = get_address(env, r2);
2830 uint64_t slen = get_length(env, r2 + 1);
2831 bool enh_check = m3 & 1;
2832 int cc, i;
2833
2834 /* Lest we fail to service interrupts in a timely manner, limit the
2835 amount of work we're willing to do. For now, let's cap at 256. */
2836 for (i = 0; i < 256; ++i) {
2837 uint32_t c, ilen, olen;
2838
2839 cc = decode(env, src, slen, enh_check, ra, &c, &ilen);
2840 if (unlikely(cc >= 0)) {
2841 break;
2842 }
2843 cc = encode(env, dst, dlen, ra, c, &olen);
2844 if (unlikely(cc >= 0)) {
2845 break;
2846 }
2847
2848 src += ilen;
2849 slen -= ilen;
2850 dst += olen;
2851 dlen -= olen;
2852 cc = 3;
2853 }
2854
2855 set_address(env, r1, dst);
2856 set_length(env, r1 + 1, dlen);
2857 set_address(env, r2, src);
2858 set_length(env, r2 + 1, slen);
2859
2860 return cc;
2861 }
2862
2863 uint32_t HELPER(cu12)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2864 {
2865 return convert_unicode(env, r1, r2, m3, GETPC(),
2866 decode_utf8, encode_utf16);
2867 }
2868
2869 uint32_t HELPER(cu14)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2870 {
2871 return convert_unicode(env, r1, r2, m3, GETPC(),
2872 decode_utf8, encode_utf32);
2873 }
2874
2875 uint32_t HELPER(cu21)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2876 {
2877 return convert_unicode(env, r1, r2, m3, GETPC(),
2878 decode_utf16, encode_utf8);
2879 }
2880
2881 uint32_t HELPER(cu24)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2882 {
2883 return convert_unicode(env, r1, r2, m3, GETPC(),
2884 decode_utf16, encode_utf32);
2885 }
2886
2887 uint32_t HELPER(cu41)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2888 {
2889 return convert_unicode(env, r1, r2, m3, GETPC(),
2890 decode_utf32, encode_utf8);
2891 }
2892
2893 uint32_t HELPER(cu42)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3)
2894 {
2895 return convert_unicode(env, r1, r2, m3, GETPC(),
2896 decode_utf32, encode_utf16);
2897 }
2898
2899 void probe_write_access(CPUS390XState *env, uint64_t addr, uint64_t len,
2900 uintptr_t ra)
2901 {
2902 const int mmu_idx = s390x_env_mmu_index(env, false);
2903
2904 /* test the actual access, not just any access to the page due to LAP */
2905 while (len) {
2906 const uint64_t pagelen = -(addr | TARGET_PAGE_MASK);
2907 const uint64_t curlen = MIN(pagelen, len);
2908
2909 probe_write(env, addr, curlen, mmu_idx, ra);
2910 addr = wrap_address(env, addr + curlen);
2911 len -= curlen;
2912 }
2913 }
2914
2915 void HELPER(probe_write_access)(CPUS390XState *env, uint64_t addr, uint64_t len)
2916 {
2917 probe_write_access(env, addr, len, GETPC());
2918 }