| 1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
| 2 | /* |
| 3 | * Copyright (c) 2021 Loongson Technology Corporation Limited |
| 4 | */ |
| 5 | |
| 6 | static bool gen_ll(DisasContext *ctx, arg_rr_i *a, MemOp mop, bool acq) |
| 7 | { |
| 8 | TCGv t1 = tcg_temp_new(); |
| 9 | TCGv t2 = tcg_temp_new(); |
| 10 | TCGv src1 = gpr_src(ctx, a->rj, EXT_NONE); |
| 11 | TCGv t0 = make_address_i(ctx, src1, a->imm); |
| 12 | TCGv_i128 t16 = tcg_temp_new_i128(); |
| 13 | TCGv mask = tcg_constant_tl(0xf); |
| 14 | TCGv one = tcg_constant_tl(1); |
| 15 | TCGv zero = tcg_constant_tl(0); |
| 16 | TCGLabel *l1 = gen_new_label(); |
| 17 | TCGLabel *done = gen_new_label(); |
| 18 | |
| 19 | if (avail_SCQ(ctx) && mop == MO_LEUQ) { |
| 20 | /* |
| 21 | * The LL.D+LD.D may be paired with SC.Q, |
| 22 | * load 128-bit if aligned: (t0 & 0xf) == 0 |
| 23 | */ |
| 24 | tcg_gen_and_tl(t1, t0, mask); |
| 25 | tcg_gen_brcond_tl(TCG_COND_EQ, t1, zero, l1); |
| 26 | /* fallthrough if not aligned to 16B */ |
| 27 | } |
| 28 | |
| 29 | tcg_gen_qemu_ld_i64(t1, t0, ctx->mem_idx, mop | MO_ALIGN); |
| 30 | tcg_gen_st_tl(t0, tcg_env, offsetof(CPULoongArchState, lladdr)); |
| 31 | tcg_gen_st_tl(t1, tcg_env, offsetof(CPULoongArchState, llval)); |
| 32 | gen_set_gpr(a->rd, t1, EXT_NONE); |
| 33 | |
| 34 | if (avail_SCQ(ctx) && mop == MO_LEUQ) { |
| 35 | tcg_gen_br(done); |
| 36 | |
| 37 | gen_set_label(l1); |
| 38 | |
| 39 | /* Load 16B data and save into llval/llval_high */ |
| 40 | tcg_gen_qemu_ld_i128(t16, t0, ctx->mem_idx, MO_128 | MO_ALIGN); |
| 41 | tcg_gen_st_tl(t0, tcg_env, offsetof(CPULoongArchState, lladdr)); |
| 42 | tcg_gen_extr_i128_i64(t1, t2, t16); |
| 43 | tcg_gen_st_tl(t1, tcg_env, offsetof(CPULoongArchState, llval)); |
| 44 | tcg_gen_st_tl(t2, tcg_env, offsetof(CPULoongArchState, llval_high)); |
| 45 | tcg_gen_st_tl(one, tcg_env, offsetof(CPULoongArchState, llbit_scq)); |
| 46 | gen_set_gpr(a->rd, t1, EXT_NONE); |
| 47 | |
| 48 | gen_set_label(done); |
| 49 | } |
| 50 | |
| 51 | if (acq) { |
| 52 | tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ); |
| 53 | } |
| 54 | |
| 55 | return true; |
| 56 | } |
| 57 | |
| 58 | static bool gen_sc(DisasContext *ctx, arg_rr_i *a, MemOp mop, bool rel) |
| 59 | { |
| 60 | TCGv dest = gpr_dst(ctx, a->rd, EXT_NONE); |
| 61 | TCGv src1 = gpr_src(ctx, a->rj, EXT_NONE); |
| 62 | TCGv src2 = gpr_src(ctx, a->rd, EXT_NONE); |
| 63 | TCGv t0 = tcg_temp_new(); |
| 64 | TCGv val = tcg_temp_new(); |
| 65 | TCGv zero = tcg_constant_tl(0); |
| 66 | |
| 67 | TCGLabel *l1 = gen_new_label(); |
| 68 | TCGLabel *done = gen_new_label(); |
| 69 | |
| 70 | tcg_gen_addi_tl(t0, src1, a->imm); |
| 71 | |
| 72 | if (rel) { |
| 73 | tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL); |
| 74 | } |
| 75 | |
| 76 | if (avail_SCQ(ctx)) { |
| 77 | tcg_gen_st_tl(zero, tcg_env, offsetof(CPULoongArchState, llbit_scq)); |
| 78 | } |
| 79 | |
| 80 | tcg_gen_brcond_tl(TCG_COND_EQ, t0, cpu_lladdr, l1); |
| 81 | tcg_gen_movi_tl(dest, 0); |
| 82 | tcg_gen_br(done); |
| 83 | |
| 84 | gen_set_label(l1); |
| 85 | tcg_gen_mov_tl(val, src2); |
| 86 | /* generate cmpxchg */ |
| 87 | tcg_gen_atomic_cmpxchg_tl(t0, cpu_lladdr, cpu_llval, |
| 88 | val, ctx->mem_idx, mop | MO_ALIGN); |
| 89 | tcg_gen_setcond_tl(TCG_COND_EQ, dest, t0, cpu_llval); |
| 90 | gen_set_label(done); |
| 91 | gen_set_gpr(a->rd, dest, EXT_NONE); |
| 92 | |
| 93 | return true; |
| 94 | } |
| 95 | |
| 96 | static bool gen_sc_q(DisasContext *ctx, arg_rrr *a, MemOp mop) |
| 97 | { |
| 98 | TCGv dest = gpr_dst(ctx, a->rd, EXT_NONE); |
| 99 | TCGv src1 = gpr_src(ctx, a->rj, EXT_NONE); |
| 100 | TCGv src2 = gpr_src(ctx, a->rd, EXT_NONE); |
| 101 | TCGv src3 = gpr_src(ctx, a->rk, EXT_NONE); |
| 102 | TCGv_i128 t16 = tcg_temp_new_i128(); |
| 103 | TCGv_i128 c16 = tcg_temp_new_i128(); |
| 104 | TCGv t1 = tcg_temp_new(); |
| 105 | TCGv t2 = tcg_temp_new(); |
| 106 | TCGv zero = tcg_constant_tl(0); |
| 107 | |
| 108 | TCGLabel *l1 = gen_new_label(); |
| 109 | TCGLabel *done = gen_new_label(); |
| 110 | |
| 111 | tcg_gen_st_tl(zero, tcg_env, offsetof(CPULoongArchState, llbit_scq)); |
| 112 | tcg_gen_brcond_tl(TCG_COND_EQ, src1, cpu_lladdr, l1); |
| 113 | tcg_gen_movi_tl(dest, 0); |
| 114 | tcg_gen_br(done); |
| 115 | |
| 116 | gen_set_label(l1); |
| 117 | tcg_gen_concat_i64_i128(t16, src2, src3); |
| 118 | tcg_gen_concat_i64_i128(c16, cpu_llval, |
| 119 | cpu_llval_high); |
| 120 | |
| 121 | /* generate cmpxchg */ |
| 122 | tcg_gen_atomic_cmpxchg_i128(t16, cpu_lladdr, c16, |
| 123 | t16, ctx->mem_idx, mop | MO_ALIGN); |
| 124 | |
| 125 | /* check if success */ |
| 126 | tcg_gen_extr_i128_i64(t1, t2, t16); |
| 127 | tcg_gen_xor_i64(t1, t1, cpu_llval); |
| 128 | tcg_gen_xor_i64(t2, t2, cpu_llval_high); |
| 129 | tcg_gen_or_i64(t1, t1, t2); |
| 130 | tcg_gen_setcondi_i64(TCG_COND_EQ, dest, t1, 0); |
| 131 | gen_set_label(done); |
| 132 | gen_set_gpr(a->rd, dest, EXT_NONE); |
| 133 | |
| 134 | return true; |
| 135 | } |
| 136 | |
| 137 | static bool gen_cas(DisasContext *ctx, arg_rrr *a, |
| 138 | void (*func)(TCGv, TCGv, TCGv, TCGv, TCGArg, MemOp), |
| 139 | MemOp mop) |
| 140 | { |
| 141 | TCGv dest = gpr_dst(ctx, a->rd, EXT_NONE); |
| 142 | TCGv addr = gpr_src(ctx, a->rj, EXT_NONE); |
| 143 | TCGv val = gpr_src(ctx, a->rk, EXT_NONE); |
| 144 | TCGv old = gpr_src(ctx, a->rd, EXT_NONE); |
| 145 | |
| 146 | addr = make_address_i(ctx, addr, 0); |
| 147 | |
| 148 | func(dest, addr, old, val, ctx->mem_idx, mop | MO_ALIGN); |
| 149 | gen_set_gpr(a->rd, dest, EXT_NONE); |
| 150 | |
| 151 | return true; |
| 152 | } |
| 153 | |
| 154 | static bool gen_am(DisasContext *ctx, arg_rrr *a, |
| 155 | void (*func)(TCGv, TCGv, TCGv, TCGArg, MemOp), |
| 156 | MemOp mop) |
| 157 | { |
| 158 | TCGv dest = gpr_dst(ctx, a->rd, EXT_NONE); |
| 159 | TCGv addr = gpr_src(ctx, a->rj, EXT_NONE); |
| 160 | TCGv val = gpr_src(ctx, a->rk, EXT_NONE); |
| 161 | |
| 162 | if (a->rd != 0 && (a->rj == a->rd || a->rk == a->rd)) { |
| 163 | qemu_log_mask(LOG_GUEST_ERROR, |
| 164 | "Warning: source register overlaps destination register" |
| 165 | "in atomic insn at pc=0x%" VADDR_PRIx "\n", |
| 166 | ctx->base.pc_next - 4); |
| 167 | return false; |
| 168 | } |
| 169 | |
| 170 | addr = make_address_i(ctx, addr, 0); |
| 171 | |
| 172 | func(dest, addr, val, ctx->mem_idx, mop | MO_ALIGN); |
| 173 | gen_set_gpr(a->rd, dest, EXT_NONE); |
| 174 | |
| 175 | return true; |
| 176 | } |
| 177 | |
| 178 | TRANS(ll_w, ALL, gen_ll, MO_LESL, false) |
| 179 | TRANS(sc_w, ALL, gen_sc, MO_LESL, false) |
| 180 | TRANS(ll_d, 64, gen_ll, MO_LEUQ, false) |
| 181 | TRANS(sc_d, 64, gen_sc, MO_LEUQ, false) |
| 182 | TRANS(sc_q, 64, gen_sc_q, MO_128) |
| 183 | TRANS(llacq_w, LLACQ_SCREL, gen_ll, MO_LESL, true) |
| 184 | TRANS(screl_w, LLACQ_SCREL, gen_sc, MO_LESL, true) |
| 185 | TRANS(llacq_d, LLACQ_SCREL_64, gen_ll, MO_LEUQ, true) |
| 186 | TRANS(screl_d, LLACQ_SCREL_64, gen_sc, MO_LEUQ, true) |
| 187 | TRANS(amcas_b, LAMCAS, gen_cas, tcg_gen_atomic_cmpxchg_tl, MO_SB) |
| 188 | TRANS(amcas_h, LAMCAS, gen_cas, tcg_gen_atomic_cmpxchg_tl, MO_LESW) |
| 189 | TRANS(amcas_w, LAMCAS, gen_cas, tcg_gen_atomic_cmpxchg_tl, MO_LESL) |
| 190 | TRANS(amcas_d, LAMCAS, gen_cas, tcg_gen_atomic_cmpxchg_tl, MO_LEUQ) |
| 191 | TRANS(amcas_db_b, LAMCAS, gen_cas, tcg_gen_atomic_cmpxchg_tl, MO_SB) |
| 192 | TRANS(amcas_db_h, LAMCAS, gen_cas, tcg_gen_atomic_cmpxchg_tl, MO_LESW) |
| 193 | TRANS(amcas_db_w, LAMCAS, gen_cas, tcg_gen_atomic_cmpxchg_tl, MO_LESL) |
| 194 | TRANS(amcas_db_d, LAMCAS, gen_cas, tcg_gen_atomic_cmpxchg_tl, MO_LEUQ) |
| 195 | TRANS(amswap_b, LAM_BH, gen_am, tcg_gen_atomic_xchg_tl, MO_SB) |
| 196 | TRANS(amswap_h, LAM_BH, gen_am, tcg_gen_atomic_xchg_tl, MO_LESW) |
| 197 | TRANS(amadd_b, LAM_BH, gen_am, tcg_gen_atomic_fetch_add_tl, MO_SB) |
| 198 | TRANS(amadd_h, LAM_BH, gen_am, tcg_gen_atomic_fetch_add_tl, MO_LESW) |
| 199 | TRANS(amswap_db_b, LAM_BH, gen_am, tcg_gen_atomic_xchg_tl, MO_SB) |
| 200 | TRANS(amswap_db_h, LAM_BH, gen_am, tcg_gen_atomic_xchg_tl, MO_LESW) |
| 201 | TRANS(amadd_db_b, LAM_BH, gen_am, tcg_gen_atomic_fetch_add_tl, MO_SB) |
| 202 | TRANS(amadd_db_h, LAM_BH, gen_am, tcg_gen_atomic_fetch_add_tl, MO_LESW) |
| 203 | TRANS(amswap_w, LAM, gen_am, tcg_gen_atomic_xchg_tl, MO_LESL) |
| 204 | TRANS64(amswap_d, LAM, gen_am, tcg_gen_atomic_xchg_tl, MO_LEUQ) |
| 205 | TRANS(amadd_w, LAM, gen_am, tcg_gen_atomic_fetch_add_tl, MO_LESL) |
| 206 | TRANS64(amadd_d, LAM, gen_am, tcg_gen_atomic_fetch_add_tl, MO_LEUQ) |
| 207 | TRANS(amand_w, LAM, gen_am, tcg_gen_atomic_fetch_and_tl, MO_LESL) |
| 208 | TRANS64(amand_d, LAM, gen_am, tcg_gen_atomic_fetch_and_tl, MO_LEUQ) |
| 209 | TRANS(amor_w, LAM, gen_am, tcg_gen_atomic_fetch_or_tl, MO_LESL) |
| 210 | TRANS64(amor_d, LAM, gen_am, tcg_gen_atomic_fetch_or_tl, MO_LEUQ) |
| 211 | TRANS(amxor_w, LAM, gen_am, tcg_gen_atomic_fetch_xor_tl, MO_LESL) |
| 212 | TRANS64(amxor_d, LAM, gen_am, tcg_gen_atomic_fetch_xor_tl, MO_LEUQ) |
| 213 | TRANS(ammax_w, LAM, gen_am, tcg_gen_atomic_fetch_smax_tl, MO_LESL) |
| 214 | TRANS64(ammax_d, LAM, gen_am, tcg_gen_atomic_fetch_smax_tl, MO_LEUQ) |
| 215 | TRANS(ammin_w, LAM, gen_am, tcg_gen_atomic_fetch_smin_tl, MO_LESL) |
| 216 | TRANS64(ammin_d, LAM, gen_am, tcg_gen_atomic_fetch_smin_tl, MO_LEUQ) |
| 217 | TRANS(ammax_wu, LAM, gen_am, tcg_gen_atomic_fetch_umax_tl, MO_LESL) |
| 218 | TRANS64(ammax_du, LAM, gen_am, tcg_gen_atomic_fetch_umax_tl, MO_LEUQ) |
| 219 | TRANS(ammin_wu, LAM, gen_am, tcg_gen_atomic_fetch_umin_tl, MO_LESL) |
| 220 | TRANS64(ammin_du, LAM, gen_am, tcg_gen_atomic_fetch_umin_tl, MO_LEUQ) |
| 221 | TRANS(amswap_db_w, LAM, gen_am, tcg_gen_atomic_xchg_tl, MO_LESL) |
| 222 | TRANS64(amswap_db_d, LAM, gen_am, tcg_gen_atomic_xchg_tl, MO_LEUQ) |
| 223 | TRANS(amadd_db_w, LAM, gen_am, tcg_gen_atomic_fetch_add_tl, MO_LESL) |
| 224 | TRANS64(amadd_db_d, LAM, gen_am, tcg_gen_atomic_fetch_add_tl, MO_LEUQ) |
| 225 | TRANS(amand_db_w, LAM, gen_am, tcg_gen_atomic_fetch_and_tl, MO_LESL) |
| 226 | TRANS64(amand_db_d, LAM, gen_am, tcg_gen_atomic_fetch_and_tl, MO_LEUQ) |
| 227 | TRANS(amor_db_w, LAM, gen_am, tcg_gen_atomic_fetch_or_tl, MO_LESL) |
| 228 | TRANS64(amor_db_d, LAM, gen_am, tcg_gen_atomic_fetch_or_tl, MO_LEUQ) |
| 229 | TRANS(amxor_db_w, LAM, gen_am, tcg_gen_atomic_fetch_xor_tl, MO_LESL) |
| 230 | TRANS64(amxor_db_d, LAM, gen_am, tcg_gen_atomic_fetch_xor_tl, MO_LEUQ) |
| 231 | TRANS(ammax_db_w, LAM, gen_am, tcg_gen_atomic_fetch_smax_tl, MO_LESL) |
| 232 | TRANS64(ammax_db_d, LAM, gen_am, tcg_gen_atomic_fetch_smax_tl, MO_LEUQ) |
| 233 | TRANS(ammin_db_w, LAM, gen_am, tcg_gen_atomic_fetch_smin_tl, MO_LESL) |
| 234 | TRANS64(ammin_db_d, LAM, gen_am, tcg_gen_atomic_fetch_smin_tl, MO_LEUQ) |
| 235 | TRANS(ammax_db_wu, LAM, gen_am, tcg_gen_atomic_fetch_umax_tl, MO_LESL) |
| 236 | TRANS64(ammax_db_du, LAM, gen_am, tcg_gen_atomic_fetch_umax_tl, MO_LEUQ) |
| 237 | TRANS(ammin_db_wu, LAM, gen_am, tcg_gen_atomic_fetch_umin_tl, MO_LESL) |
| 238 | TRANS64(ammin_db_du, LAM, gen_am, tcg_gen_atomic_fetch_umin_tl, MO_LEUQ) |