@samitouri / QOSamiQemu
Code Commits Issues Pulls Sessions CI/CD
master
master
c 2,732 lines 81.7 KB
Raw
1 /*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 #include "qemu/osdep.h"
26 #include "tcg/tcg.h"
27 #include "tcg/tcg-temp-internal.h"
28 #include "tcg/tcg-op-common.h"
29 #include "exec/translation-block.h"
30 #include "exec/plugin-gen.h"
31 #include "tcg-internal.h"
32 #include "tcg-has.h"
33
34 /*
35 * Encourage the compiler to tail-call to a function, rather than inlining.
36 * Minimizes code size across 99 bottles of beer on the wall.
37 */
38 #define NI __attribute__((noinline))
39
40 TCGOp * NI tcg_gen_op1(TCGOpcode opc, TCGType type, TCGArg a1)
41 {
42 TCGOp *op = tcg_emit_op(opc, 1);
43 TCGOP_TYPE(op) = type;
44 op->args[0] = a1;
45 return op;
46 }
47
48 TCGOp * NI tcg_gen_op2(TCGOpcode opc, TCGType type, TCGArg a1, TCGArg a2)
49 {
50 TCGOp *op = tcg_emit_op(opc, 2);
51 TCGOP_TYPE(op) = type;
52 op->args[0] = a1;
53 op->args[1] = a2;
54 return op;
55 }
56
57 TCGOp * NI tcg_gen_op3(TCGOpcode opc, TCGType type, TCGArg a1,
58 TCGArg a2, TCGArg a3)
59 {
60 TCGOp *op = tcg_emit_op(opc, 3);
61 TCGOP_TYPE(op) = type;
62 op->args[0] = a1;
63 op->args[1] = a2;
64 op->args[2] = a3;
65 return op;
66 }
67
68 TCGOp * NI tcg_gen_op4(TCGOpcode opc, TCGType type, TCGArg a1, TCGArg a2,
69 TCGArg a3, TCGArg a4)
70 {
71 TCGOp *op = tcg_emit_op(opc, 4);
72 TCGOP_TYPE(op) = type;
73 op->args[0] = a1;
74 op->args[1] = a2;
75 op->args[2] = a3;
76 op->args[3] = a4;
77 return op;
78 }
79
80 TCGOp * NI tcg_gen_op5(TCGOpcode opc, TCGType type, TCGArg a1, TCGArg a2,
81 TCGArg a3, TCGArg a4, TCGArg a5)
82 {
83 TCGOp *op = tcg_emit_op(opc, 5);
84 TCGOP_TYPE(op) = type;
85 op->args[0] = a1;
86 op->args[1] = a2;
87 op->args[2] = a3;
88 op->args[3] = a4;
89 op->args[4] = a5;
90 return op;
91 }
92
93 TCGOp * NI tcg_gen_op6(TCGOpcode opc, TCGType type, TCGArg a1, TCGArg a2,
94 TCGArg a3, TCGArg a4, TCGArg a5, TCGArg a6)
95 {
96 TCGOp *op = tcg_emit_op(opc, 6);
97 TCGOP_TYPE(op) = type;
98 op->args[0] = a1;
99 op->args[1] = a2;
100 op->args[2] = a3;
101 op->args[3] = a4;
102 op->args[4] = a5;
103 op->args[5] = a6;
104 return op;
105 }
106
107 /*
108 * With CONFIG_DEBUG_TCG, tcgv_*_tmp via tcgv_*_arg, is an out-of-line
109 * assertion check. Force tail calls to avoid too much code expansion.
110 */
111 #ifdef CONFIG_DEBUG_TCG
112 # define DNI NI
113 #else
114 # define DNI
115 #endif
116
117 static void DNI tcg_gen_op1_i32(TCGOpcode opc, TCGType type, TCGv_i32 a1)
118 {
119 tcg_gen_op1(opc, type, tcgv_i32_arg(a1));
120 }
121
122 static void DNI tcg_gen_op1_i64(TCGOpcode opc, TCGType type, TCGv_i64 a1)
123 {
124 tcg_gen_op1(opc, type, tcgv_i64_arg(a1));
125 }
126
127 static TCGOp * DNI tcg_gen_op1i(TCGOpcode opc, TCGType type, TCGArg a1)
128 {
129 return tcg_gen_op1(opc, type, a1);
130 }
131
132 static void DNI tcg_gen_op2_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2)
133 {
134 tcg_gen_op2(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2));
135 }
136
137 static void DNI tcg_gen_op2_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2)
138 {
139 tcg_gen_op2(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2));
140 }
141
142 static void DNI tcg_gen_op3_i32(TCGOpcode opc, TCGv_i32 a1,
143 TCGv_i32 a2, TCGv_i32 a3)
144 {
145 tcg_gen_op3(opc, TCG_TYPE_I32, tcgv_i32_arg(a1),
146 tcgv_i32_arg(a2), tcgv_i32_arg(a3));
147 }
148
149 static void DNI tcg_gen_op3_i64(TCGOpcode opc, TCGv_i64 a1,
150 TCGv_i64 a2, TCGv_i64 a3)
151 {
152 tcg_gen_op3(opc, TCG_TYPE_I64, tcgv_i64_arg(a1),
153 tcgv_i64_arg(a2), tcgv_i64_arg(a3));
154 }
155
156 static void DNI tcg_gen_op3i_i32(TCGOpcode opc, TCGv_i32 a1,
157 TCGv_i32 a2, TCGArg a3)
158 {
159 tcg_gen_op3(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2), a3);
160 }
161
162 static void DNI tcg_gen_op3i_i64(TCGOpcode opc, TCGv_i64 a1,
163 TCGv_i64 a2, TCGArg a3)
164 {
165 tcg_gen_op3(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2), a3);
166 }
167
168 static void DNI tcg_gen_ldst_op_i32(TCGOpcode opc, TCGv_i32 val,
169 TCGv_ptr base, TCGArg offset)
170 {
171 tcg_gen_op3(opc, TCG_TYPE_I32, tcgv_i32_arg(val),
172 tcgv_ptr_arg(base), offset);
173 }
174
175 static void DNI tcg_gen_ldst_op_i64(TCGOpcode opc, TCGv_i64 val,
176 TCGv_ptr base, TCGArg offset)
177 {
178 tcg_gen_op3(opc, TCG_TYPE_I64, tcgv_i64_arg(val),
179 tcgv_ptr_arg(base), offset);
180 }
181
182 static void DNI tcg_gen_op4_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2,
183 TCGv_i32 a3, TCGv_i32 a4)
184 {
185 tcg_gen_op4(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2),
186 tcgv_i32_arg(a3), tcgv_i32_arg(a4));
187 }
188
189 static void DNI tcg_gen_op4_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2,
190 TCGv_i64 a3, TCGv_i64 a4)
191 {
192 tcg_gen_op4(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2),
193 tcgv_i64_arg(a3), tcgv_i64_arg(a4));
194 }
195
196 static void DNI tcg_gen_op4i_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2,
197 TCGv_i32 a3, TCGArg a4)
198 {
199 tcg_gen_op4(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2),
200 tcgv_i32_arg(a3), a4);
201 }
202
203 static void DNI tcg_gen_op4i_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2,
204 TCGv_i64 a3, TCGArg a4)
205 {
206 tcg_gen_op4(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2),
207 tcgv_i64_arg(a3), a4);
208 }
209
210 static TCGOp * DNI tcg_gen_op4ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2,
211 TCGArg a3, TCGArg a4)
212 {
213 return tcg_gen_op4(opc, TCG_TYPE_I32,
214 tcgv_i32_arg(a1), tcgv_i32_arg(a2), a3, a4);
215 }
216
217 static TCGOp * DNI tcg_gen_op4ii_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2,
218 TCGArg a3, TCGArg a4)
219 {
220 return tcg_gen_op4(opc, TCG_TYPE_I64,
221 tcgv_i64_arg(a1), tcgv_i64_arg(a2), a3, a4);
222 }
223
224 static void DNI tcg_gen_op5_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2,
225 TCGv_i32 a3, TCGv_i32 a4, TCGv_i32 a5)
226 {
227 tcg_gen_op5(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2),
228 tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5));
229 }
230
231 static void DNI tcg_gen_op5_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2,
232 TCGv_i64 a3, TCGv_i64 a4, TCGv_i64 a5)
233 {
234 tcg_gen_op5(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2),
235 tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5));
236 }
237
238 static void DNI tcg_gen_op5ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2,
239 TCGv_i32 a3, TCGArg a4, TCGArg a5)
240 {
241 tcg_gen_op5(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2),
242 tcgv_i32_arg(a3), a4, a5);
243 }
244
245 static void DNI tcg_gen_op5ii_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2,
246 TCGv_i64 a3, TCGArg a4, TCGArg a5)
247 {
248 tcg_gen_op5(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2),
249 tcgv_i64_arg(a3), a4, a5);
250 }
251
252 static void DNI tcg_gen_op6i_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2,
253 TCGv_i32 a3, TCGv_i32 a4,
254 TCGv_i32 a5, TCGArg a6)
255 {
256 tcg_gen_op6(opc, TCG_TYPE_I32, tcgv_i32_arg(a1), tcgv_i32_arg(a2),
257 tcgv_i32_arg(a3), tcgv_i32_arg(a4), tcgv_i32_arg(a5), a6);
258 }
259
260 static void DNI tcg_gen_op6i_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2,
261 TCGv_i64 a3, TCGv_i64 a4,
262 TCGv_i64 a5, TCGArg a6)
263 {
264 tcg_gen_op6(opc, TCG_TYPE_I64, tcgv_i64_arg(a1), tcgv_i64_arg(a2),
265 tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5), a6);
266 }
267
268 /* Generic ops. */
269
270 void gen_set_label(TCGLabel *l)
271 {
272 l->present = 1;
273 tcg_gen_op1(INDEX_op_set_label, 0, label_arg(l));
274 }
275
276 static void add_as_label_use(TCGLabel *l, TCGOp *op)
277 {
278 TCGLabelUse *u = tcg_malloc(sizeof(TCGLabelUse));
279
280 u->op = op;
281 QSIMPLEQ_INSERT_TAIL(&l->branches, u, next);
282 }
283
284 void tcg_gen_br(TCGLabel *l)
285 {
286 add_as_label_use(l, tcg_gen_op1(INDEX_op_br, 0, label_arg(l)));
287 }
288
289 void tcg_gen_mb(TCGBar mb_type)
290 {
291 #ifdef CONFIG_USER_ONLY
292 bool parallel = tcg_ctx->gen_tb->cflags & CF_PARALLEL;
293 #else
294 /*
295 * It is tempting to elide the barrier in a uniprocessor context.
296 * However, even with a single cpu we have i/o threads running in
297 * parallel, and lack of memory order can result in e.g. virtio
298 * queue entries being read incorrectly.
299 */
300 bool parallel = true;
301 #endif
302
303 if (parallel) {
304 tcg_gen_op1(INDEX_op_mb, 0, mb_type);
305 }
306 }
307
308 void tcg_gen_plugin_cb(unsigned from)
309 {
310 tcg_gen_op1(INDEX_op_plugin_cb, 0, from);
311 }
312
313 void tcg_gen_plugin_mem_cb(TCGv_i64 addr, unsigned meminfo)
314 {
315 tcg_gen_op2(INDEX_op_plugin_mem_cb, 0, tcgv_i64_arg(addr), meminfo);
316 }
317
318 /* 32 bit ops */
319
320 void tcg_gen_discard_i32(TCGv_i32 arg)
321 {
322 tcg_gen_op1_i32(INDEX_op_discard, TCG_TYPE_I32, arg);
323 }
324
325 void tcg_gen_mov_i32(TCGv_i32 ret, TCGv_i32 arg)
326 {
327 if (ret != arg) {
328 tcg_gen_op2_i32(INDEX_op_mov, ret, arg);
329 }
330 }
331
332 void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg)
333 {
334 tcg_gen_mov_i32(ret, tcg_constant_i32(arg));
335 }
336
337 void tcg_gen_add_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
338 {
339 tcg_gen_op3_i32(INDEX_op_add, ret, arg1, arg2);
340 }
341
342 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
343 {
344 /* some cases can be optimized here */
345 if (arg2 == 0) {
346 tcg_gen_mov_i32(ret, arg1);
347 } else {
348 tcg_gen_add_i32(ret, arg1, tcg_constant_i32(arg2));
349 }
350 }
351
352 void tcg_gen_sub_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
353 {
354 tcg_gen_op3_i32(INDEX_op_sub, ret, arg1, arg2);
355 }
356
357 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2)
358 {
359 if (arg1 == 0) {
360 tcg_gen_neg_i32(ret, arg2);
361 } else {
362 tcg_gen_sub_i32(ret, tcg_constant_i32(arg1), arg2);
363 }
364 }
365
366 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
367 {
368 tcg_gen_addi_i32(ret, arg1, -arg2);
369 }
370
371 void tcg_gen_neg_i32(TCGv_i32 ret, TCGv_i32 arg)
372 {
373 tcg_gen_op2_i32(INDEX_op_neg, ret, arg);
374 }
375
376 void tcg_gen_and_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
377 {
378 tcg_gen_op3_i32(INDEX_op_and, ret, arg1, arg2);
379 }
380
381 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
382 {
383 /* Some cases can be optimized here. */
384 switch (arg2) {
385 case 0:
386 tcg_gen_movi_i32(ret, 0);
387 return;
388 case -1:
389 tcg_gen_mov_i32(ret, arg1);
390 return;
391 default:
392 /*
393 * Canonicalize on extract, if valid. This aids x86 with its
394 * 2 operand MOVZBL and 2 operand AND, selecting the TCGOpcode
395 * which does not require matching operands. Other backends can
396 * trivially expand the extract to AND during code generation.
397 */
398 if (!(arg2 & (arg2 + 1))) {
399 unsigned len = ctz32(~arg2);
400 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, len)) {
401 tcg_gen_extract_i32(ret, arg1, 0, len);
402 return;
403 }
404 }
405 break;
406 }
407
408 tcg_gen_and_i32(ret, arg1, tcg_constant_i32(arg2));
409 }
410
411 void tcg_gen_or_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
412 {
413 tcg_gen_op3_i32(INDEX_op_or, ret, arg1, arg2);
414 }
415
416 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
417 {
418 /* Some cases can be optimized here. */
419 if (arg2 == -1) {
420 tcg_gen_movi_i32(ret, -1);
421 } else if (arg2 == 0) {
422 tcg_gen_mov_i32(ret, arg1);
423 } else {
424 tcg_gen_or_i32(ret, arg1, tcg_constant_i32(arg2));
425 }
426 }
427
428 void tcg_gen_xor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
429 {
430 tcg_gen_op3_i32(INDEX_op_xor, ret, arg1, arg2);
431 }
432
433 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
434 {
435 /* Some cases can be optimized here. */
436 if (arg2 == 0) {
437 tcg_gen_mov_i32(ret, arg1);
438 } else if (arg2 == -1 &&
439 tcg_op_supported(INDEX_op_not, TCG_TYPE_I32, 0)) {
440 /* Don't recurse with tcg_gen_not_i32. */
441 tcg_gen_op2_i32(INDEX_op_not, ret, arg1);
442 } else {
443 tcg_gen_xor_i32(ret, arg1, tcg_constant_i32(arg2));
444 }
445 }
446
447 void tcg_gen_not_i32(TCGv_i32 ret, TCGv_i32 arg)
448 {
449 if (tcg_op_supported(INDEX_op_not, TCG_TYPE_I32, 0)) {
450 tcg_gen_op2_i32(INDEX_op_not, ret, arg);
451 } else {
452 tcg_gen_xori_i32(ret, arg, -1);
453 }
454 }
455
456 void tcg_gen_shl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
457 {
458 tcg_gen_op3_i32(INDEX_op_shl, ret, arg1, arg2);
459 }
460
461 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
462 {
463 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
464 if (arg2 == 0) {
465 tcg_gen_mov_i32(ret, arg1);
466 } else {
467 tcg_gen_shl_i32(ret, arg1, tcg_constant_i32(arg2));
468 }
469 }
470
471 void tcg_gen_shr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
472 {
473 tcg_gen_op3_i32(INDEX_op_shr, ret, arg1, arg2);
474 }
475
476 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
477 {
478 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
479 if (arg2 == 0) {
480 tcg_gen_mov_i32(ret, arg1);
481 } else {
482 tcg_gen_shr_i32(ret, arg1, tcg_constant_i32(arg2));
483 }
484 }
485
486 void tcg_gen_sar_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
487 {
488 tcg_gen_op3_i32(INDEX_op_sar, ret, arg1, arg2);
489 }
490
491 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
492 {
493 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
494 if (arg2 == 0) {
495 tcg_gen_mov_i32(ret, arg1);
496 } else {
497 tcg_gen_sar_i32(ret, arg1, tcg_constant_i32(arg2));
498 }
499 }
500
501 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l)
502 {
503 if (cond == TCG_COND_ALWAYS) {
504 tcg_gen_br(l);
505 } else if (cond != TCG_COND_NEVER) {
506 TCGOp *op = tcg_gen_op4ii_i32(INDEX_op_brcond,
507 arg1, arg2, cond, label_arg(l));
508 add_as_label_use(l, op);
509 }
510 }
511
512 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l)
513 {
514 if (cond == TCG_COND_ALWAYS) {
515 tcg_gen_br(l);
516 } else if (cond != TCG_COND_NEVER) {
517 tcg_gen_brcond_i32(cond, arg1, tcg_constant_i32(arg2), l);
518 }
519 }
520
521 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
522 TCGv_i32 arg1, TCGv_i32 arg2)
523 {
524 if (cond == TCG_COND_ALWAYS) {
525 tcg_gen_movi_i32(ret, 1);
526 } else if (cond == TCG_COND_NEVER) {
527 tcg_gen_movi_i32(ret, 0);
528 } else {
529 tcg_gen_op4i_i32(INDEX_op_setcond, ret, arg1, arg2, cond);
530 }
531 }
532
533 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
534 TCGv_i32 arg1, int32_t arg2)
535 {
536 tcg_gen_setcond_i32(cond, ret, arg1, tcg_constant_i32(arg2));
537 }
538
539 void tcg_gen_negsetcond_i32(TCGCond cond, TCGv_i32 ret,
540 TCGv_i32 arg1, TCGv_i32 arg2)
541 {
542 if (cond == TCG_COND_ALWAYS) {
543 tcg_gen_movi_i32(ret, -1);
544 } else if (cond == TCG_COND_NEVER) {
545 tcg_gen_movi_i32(ret, 0);
546 } else {
547 tcg_gen_op4i_i32(INDEX_op_negsetcond, ret, arg1, arg2, cond);
548 }
549 }
550
551 void tcg_gen_negsetcondi_i32(TCGCond cond, TCGv_i32 ret,
552 TCGv_i32 arg1, int32_t arg2)
553 {
554 tcg_gen_negsetcond_i32(cond, ret, arg1, tcg_constant_i32(arg2));
555 }
556
557 void tcg_gen_mul_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
558 {
559 tcg_gen_op3_i32(INDEX_op_mul, ret, arg1, arg2);
560 }
561
562 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
563 {
564 if (arg2 == 0) {
565 tcg_gen_movi_i32(ret, 0);
566 } else if (is_power_of_2(arg2)) {
567 tcg_gen_shli_i32(ret, arg1, ctz32(arg2));
568 } else {
569 tcg_gen_mul_i32(ret, arg1, tcg_constant_i32(arg2));
570 }
571 }
572
573 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
574 {
575 if (tcg_op_supported(INDEX_op_divs, TCG_TYPE_I32, 0)) {
576 tcg_gen_op3_i32(INDEX_op_divs, ret, arg1, arg2);
577 } else if (tcg_op_supported(INDEX_op_divs2, TCG_TYPE_I32, 0)) {
578 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
579 tcg_gen_sari_i32(t0, arg1, 31);
580 tcg_gen_op5_i32(INDEX_op_divs2, ret, t0, arg1, t0, arg2);
581 tcg_temp_free_i32(t0);
582 } else {
583 gen_helper_div_i32(ret, arg1, arg2);
584 }
585 }
586
587 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
588 {
589 if (tcg_op_supported(INDEX_op_rems, TCG_TYPE_I32, 0)) {
590 tcg_gen_op3_i32(INDEX_op_rems, ret, arg1, arg2);
591 } else if (tcg_op_supported(INDEX_op_divs, TCG_TYPE_I32, 0)) {
592 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
593 tcg_gen_op3_i32(INDEX_op_divs, t0, arg1, arg2);
594 tcg_gen_mul_i32(t0, t0, arg2);
595 tcg_gen_sub_i32(ret, arg1, t0);
596 tcg_temp_free_i32(t0);
597 } else if (tcg_op_supported(INDEX_op_divs2, TCG_TYPE_I32, 0)) {
598 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
599 tcg_gen_sari_i32(t0, arg1, 31);
600 tcg_gen_op5_i32(INDEX_op_divs2, t0, ret, arg1, t0, arg2);
601 tcg_temp_free_i32(t0);
602 } else {
603 gen_helper_rem_i32(ret, arg1, arg2);
604 }
605 }
606
607 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
608 {
609 if (tcg_op_supported(INDEX_op_divu, TCG_TYPE_I32, 0)) {
610 tcg_gen_op3_i32(INDEX_op_divu, ret, arg1, arg2);
611 } else if (tcg_op_supported(INDEX_op_divu2, TCG_TYPE_I32, 0)) {
612 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
613 TCGv_i32 zero = tcg_constant_i32(0);
614 tcg_gen_op5_i32(INDEX_op_divu2, ret, t0, arg1, zero, arg2);
615 tcg_temp_free_i32(t0);
616 } else {
617 gen_helper_divu_i32(ret, arg1, arg2);
618 }
619 }
620
621 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
622 {
623 if (tcg_op_supported(INDEX_op_remu, TCG_TYPE_I32, 0)) {
624 tcg_gen_op3_i32(INDEX_op_remu, ret, arg1, arg2);
625 } else if (tcg_op_supported(INDEX_op_divu, TCG_TYPE_I32, 0)) {
626 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
627 tcg_gen_op3_i32(INDEX_op_divu, t0, arg1, arg2);
628 tcg_gen_mul_i32(t0, t0, arg2);
629 tcg_gen_sub_i32(ret, arg1, t0);
630 tcg_temp_free_i32(t0);
631 } else if (tcg_op_supported(INDEX_op_divu2, TCG_TYPE_I32, 0)) {
632 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
633 TCGv_i32 zero = tcg_constant_i32(0);
634 tcg_gen_op5_i32(INDEX_op_divu2, t0, ret, arg1, zero, arg2);
635 tcg_temp_free_i32(t0);
636 } else {
637 gen_helper_remu_i32(ret, arg1, arg2);
638 }
639 }
640
641 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
642 {
643 if (tcg_op_supported(INDEX_op_andc, TCG_TYPE_I32, 0)) {
644 tcg_gen_op3_i32(INDEX_op_andc, ret, arg1, arg2);
645 } else {
646 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
647 tcg_gen_not_i32(t0, arg2);
648 tcg_gen_and_i32(ret, arg1, t0);
649 tcg_temp_free_i32(t0);
650 }
651 }
652
653 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
654 {
655 if (tcg_op_supported(INDEX_op_eqv, TCG_TYPE_I32, 0)) {
656 tcg_gen_op3_i32(INDEX_op_eqv, ret, arg1, arg2);
657 } else {
658 tcg_gen_xor_i32(ret, arg1, arg2);
659 tcg_gen_not_i32(ret, ret);
660 }
661 }
662
663 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
664 {
665 if (tcg_op_supported(INDEX_op_nand, TCG_TYPE_I32, 0)) {
666 tcg_gen_op3_i32(INDEX_op_nand, ret, arg1, arg2);
667 } else {
668 tcg_gen_and_i32(ret, arg1, arg2);
669 tcg_gen_not_i32(ret, ret);
670 }
671 }
672
673 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
674 {
675 if (tcg_op_supported(INDEX_op_nor, TCG_TYPE_I32, 0)) {
676 tcg_gen_op3_i32(INDEX_op_nor, ret, arg1, arg2);
677 } else {
678 tcg_gen_or_i32(ret, arg1, arg2);
679 tcg_gen_not_i32(ret, ret);
680 }
681 }
682
683 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
684 {
685 if (tcg_op_supported(INDEX_op_orc, TCG_TYPE_I32, 0)) {
686 tcg_gen_op3_i32(INDEX_op_orc, ret, arg1, arg2);
687 } else {
688 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
689 tcg_gen_not_i32(t0, arg2);
690 tcg_gen_or_i32(ret, arg1, t0);
691 tcg_temp_free_i32(t0);
692 }
693 }
694
695 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
696 {
697 if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_I32, 0)) {
698 tcg_gen_op3_i32(INDEX_op_clz, ret, arg1, arg2);
699 } else if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_I64, 0)) {
700 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
701 TCGv_i64 t2 = tcg_temp_ebb_new_i64();
702 tcg_gen_extu_i32_i64(t1, arg1);
703 tcg_gen_extu_i32_i64(t2, arg2);
704 tcg_gen_addi_i64(t2, t2, 32);
705 tcg_gen_clz_i64(t1, t1, t2);
706 tcg_gen_extrl_i64_i32(ret, t1);
707 tcg_temp_free_i64(t1);
708 tcg_temp_free_i64(t2);
709 tcg_gen_subi_i32(ret, ret, 32);
710 } else {
711 gen_helper_clz_i32(ret, arg1, arg2);
712 }
713 }
714
715 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
716 {
717 tcg_gen_clz_i32(ret, arg1, tcg_constant_i32(arg2));
718 }
719
720 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
721 {
722 TCGv_i32 z, t;
723
724 if (tcg_op_supported(INDEX_op_ctz, TCG_TYPE_I32, 0)) {
725 tcg_gen_op3_i32(INDEX_op_ctz, ret, arg1, arg2);
726 return;
727 }
728 if (tcg_op_supported(INDEX_op_ctz, TCG_TYPE_I64, 0)) {
729 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
730 TCGv_i64 t2 = tcg_temp_ebb_new_i64();
731 tcg_gen_extu_i32_i64(t1, arg1);
732 tcg_gen_extu_i32_i64(t2, arg2);
733 tcg_gen_ctz_i64(t1, t1, t2);
734 tcg_gen_extrl_i64_i32(ret, t1);
735 tcg_temp_free_i64(t1);
736 tcg_temp_free_i64(t2);
737 return;
738 }
739 if (tcg_op_supported(INDEX_op_ctpop, TCG_TYPE_REG, 0)) {
740 t = tcg_temp_ebb_new_i32();
741 tcg_gen_subi_i32(t, arg1, 1);
742 tcg_gen_andc_i32(t, t, arg1);
743 tcg_gen_ctpop_i32(t, t);
744 } else if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_REG, 0)) {
745 t = tcg_temp_ebb_new_i32();
746 tcg_gen_neg_i32(t, arg1);
747 tcg_gen_and_i32(t, t, arg1);
748 tcg_gen_clzi_i32(t, t, 32);
749 tcg_gen_xori_i32(t, t, 31);
750 } else {
751 gen_helper_ctz_i32(ret, arg1, arg2);
752 return;
753 }
754
755 z = tcg_constant_i32(0);
756 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t);
757 tcg_temp_free_i32(t);
758 }
759
760 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
761 {
762 if (arg2 == 32
763 && !tcg_op_supported(INDEX_op_ctz, TCG_TYPE_I32, 0)
764 && tcg_op_supported(INDEX_op_ctpop, TCG_TYPE_REG, 0)) {
765 /* This equivalence has the advantage of not requiring a fixup. */
766 TCGv_i32 t = tcg_temp_ebb_new_i32();
767 tcg_gen_subi_i32(t, arg1, 1);
768 tcg_gen_andc_i32(t, t, arg1);
769 tcg_gen_ctpop_i32(ret, t);
770 tcg_temp_free_i32(t);
771 } else {
772 tcg_gen_ctz_i32(ret, arg1, tcg_constant_i32(arg2));
773 }
774 }
775
776 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg)
777 {
778 if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_REG, 0)) {
779 TCGv_i32 t = tcg_temp_ebb_new_i32();
780 tcg_gen_sari_i32(t, arg, 31);
781 tcg_gen_xor_i32(t, t, arg);
782 tcg_gen_clzi_i32(t, t, 32);
783 tcg_gen_subi_i32(ret, t, 1);
784 tcg_temp_free_i32(t);
785 } else {
786 gen_helper_clrsb_i32(ret, arg);
787 }
788 }
789
790 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1)
791 {
792 if (tcg_op_supported(INDEX_op_ctpop, TCG_TYPE_I32, 0)) {
793 tcg_gen_op2_i32(INDEX_op_ctpop, ret, arg1);
794 } else if (tcg_op_supported(INDEX_op_ctpop, TCG_TYPE_I64, 0)) {
795 TCGv_i64 t = tcg_temp_ebb_new_i64();
796 tcg_gen_extu_i32_i64(t, arg1);
797 tcg_gen_ctpop_i64(t, t);
798 tcg_gen_extrl_i64_i32(ret, t);
799 tcg_temp_free_i64(t);
800 } else {
801 gen_helper_ctpop_i32(ret, arg1);
802 }
803 }
804
805 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
806 {
807 if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I32, 0)) {
808 tcg_gen_op3_i32(INDEX_op_rotl, ret, arg1, arg2);
809 } else if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I32, 0)) {
810 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
811 tcg_gen_neg_i32(t0, arg2);
812 tcg_gen_op3_i32(INDEX_op_rotr, ret, arg1, t0);
813 tcg_temp_free_i32(t0);
814 } else {
815 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
816 TCGv_i32 t1 = tcg_temp_ebb_new_i32();
817 tcg_gen_shl_i32(t0, arg1, arg2);
818 tcg_gen_neg_i32(t1, arg2);
819 tcg_gen_shr_i32(t1, arg1, t1);
820 tcg_gen_or_i32(ret, t0, t1);
821 tcg_temp_free_i32(t0);
822 tcg_temp_free_i32(t1);
823 }
824 }
825
826 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
827 {
828 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
829 if (arg2 == 0) {
830 tcg_gen_mov_i32(ret, arg1);
831 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I32, 0)) {
832 tcg_gen_op3_i32(INDEX_op_rotl, ret, arg1, tcg_constant_i32(arg2));
833 } else {
834 tcg_gen_rotri_i32(ret, arg1, -arg2 & 31);
835 }
836 }
837
838 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
839 {
840 if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I32, 0)) {
841 tcg_gen_op3_i32(INDEX_op_rotr, ret, arg1, arg2);
842 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I32, 0)) {
843 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
844 tcg_gen_neg_i32(t0, arg2);
845 tcg_gen_op3_i32(INDEX_op_rotl, ret, arg1, t0);
846 tcg_temp_free_i32(t0);
847 } else {
848 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
849 TCGv_i32 t1 = tcg_temp_ebb_new_i32();
850 tcg_gen_shr_i32(t0, arg1, arg2);
851 tcg_gen_neg_i32(t1, arg2);
852 tcg_gen_shl_i32(t1, arg1, t1);
853 tcg_gen_or_i32(ret, t0, t1);
854 tcg_temp_free_i32(t0);
855 tcg_temp_free_i32(t1);
856 }
857 }
858
859 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
860 {
861 tcg_debug_assert(arg2 >= 0 && arg2 < 32);
862 if (arg2 == 0) {
863 tcg_gen_mov_i32(ret, arg1);
864 } else if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I32, 0)) {
865 tcg_gen_op3_i32(INDEX_op_rotr, ret, arg1, tcg_constant_i32(arg2));
866 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I32, 0)) {
867 tcg_gen_op3_i32(INDEX_op_rotl, ret, arg1, tcg_constant_i32(32 - arg2));
868 } else {
869 /* Do not recurse with the rotri simplification. */
870 tcg_gen_op4i_i32(INDEX_op_extract2, ret, arg1, arg1, arg2);
871 }
872 }
873
874 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2,
875 unsigned int ofs, unsigned int len)
876 {
877 tcg_debug_assert(ofs < 32);
878 tcg_debug_assert(len > 0);
879 tcg_debug_assert(len <= 32);
880 tcg_debug_assert(ofs + len <= 32);
881
882 if (len == 32) {
883 tcg_gen_mov_i32(ret, arg2);
884 } else {
885 tcg_gen_op5ii_i32(INDEX_op_deposit, ret, arg1, arg2, ofs, len);
886 }
887 }
888
889 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg,
890 unsigned int ofs, unsigned int len)
891 {
892 tcg_debug_assert(ofs < 32);
893 tcg_debug_assert(len > 0);
894 tcg_debug_assert(len <= 32);
895 tcg_debug_assert(ofs + len <= 32);
896
897 if (ofs + len == 32) {
898 tcg_gen_shli_i32(ret, arg, ofs);
899 } else if (ofs == 0) {
900 tcg_gen_extract_i32(ret, arg, 0, len);
901 } else {
902 TCGv_i32 zero = tcg_constant_i32(0);
903 tcg_gen_op5ii_i32(INDEX_op_deposit, ret, zero, arg, ofs, len);
904 }
905 }
906
907 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg,
908 unsigned int ofs, unsigned int len)
909 {
910 uint32_t mask;
911
912 tcg_debug_assert(ofs < 32);
913 tcg_debug_assert(len > 0);
914 tcg_debug_assert(len <= 32);
915 tcg_debug_assert(ofs + len <= 32);
916
917 /* Canonicalize certain special cases, even if extract is supported. */
918 if (ofs + len == 32) {
919 tcg_gen_shri_i32(ret, arg, 32 - len);
920 return;
921 }
922
923 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, ofs, len)) {
924 tcg_gen_op4ii_i32(INDEX_op_extract, ret, arg, ofs, len);
925 return;
926 }
927
928 mask = (1u << len) - 1;
929 if (ofs == 0) {
930 tcg_gen_andi_i32(ret, arg, mask);
931 return;
932 }
933
934 /* Assume that zero-extension, if available, is cheaper than a shift. */
935 if (TCG_TARGET_extract_valid(TCG_TYPE_I32, 0, ofs + len)) {
936 tcg_gen_op4ii_i32(INDEX_op_extract, ret, arg, 0, ofs + len);
937 tcg_gen_shri_i32(ret, ret, ofs);
938 return;
939 }
940
941 if (tcg_op_imm_match(INDEX_op_and, TCG_TYPE_I32, mask)) {
942 tcg_gen_shri_i32(ret, arg, ofs);
943 tcg_gen_andi_i32(ret, ret, mask);
944 } else {
945 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
946 tcg_gen_shri_i32(ret, ret, 32 - len);
947 }
948 }
949
950 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg,
951 unsigned int ofs, unsigned int len)
952 {
953 tcg_debug_assert(ofs < 32);
954 tcg_debug_assert(len > 0);
955 tcg_debug_assert(len <= 32);
956 tcg_debug_assert(ofs + len <= 32);
957
958 /* Canonicalize certain special cases, even if extract is supported. */
959 if (ofs + len == 32) {
960 tcg_gen_sari_i32(ret, arg, 32 - len);
961 return;
962 }
963
964 if (TCG_TARGET_sextract_valid(TCG_TYPE_I32, ofs, len)) {
965 tcg_gen_op4ii_i32(INDEX_op_sextract, ret, arg, ofs, len);
966 return;
967 }
968
969 /* Assume that sign-extension, if available, is cheaper than a shift. */
970 if (TCG_TARGET_sextract_valid(TCG_TYPE_I32, 0, ofs + len)) {
971 tcg_gen_op4ii_i32(INDEX_op_sextract, ret, arg, 0, ofs + len);
972 tcg_gen_sari_i32(ret, ret, ofs);
973 return;
974 }
975 if (TCG_TARGET_sextract_valid(TCG_TYPE_I32, 0, len)) {
976 tcg_gen_shri_i32(ret, arg, ofs);
977 tcg_gen_op4ii_i32(INDEX_op_sextract, ret, ret, 0, len);
978 return;
979 }
980
981 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
982 tcg_gen_sari_i32(ret, ret, 32 - len);
983 }
984
985 /*
986 * Extract 32-bits from a 64-bit input, ah:al, starting from ofs.
987 * Unlike tcg_gen_extract_i32 above, len is fixed at 32.
988 */
989 void tcg_gen_extract2_i32(TCGv_i32 ret, TCGv_i32 al, TCGv_i32 ah,
990 unsigned int ofs)
991 {
992 tcg_debug_assert(ofs <= 32);
993 if (ofs == 0) {
994 tcg_gen_mov_i32(ret, al);
995 } else if (ofs == 32) {
996 tcg_gen_mov_i32(ret, ah);
997 } else if (al == ah) {
998 tcg_gen_rotri_i32(ret, al, ofs);
999 } else {
1000 tcg_gen_op4i_i32(INDEX_op_extract2, ret, al, ah, ofs);
1001 }
1002 }
1003
1004 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1,
1005 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2)
1006 {
1007 if (cond == TCG_COND_ALWAYS) {
1008 tcg_gen_mov_i32(ret, v1);
1009 } else if (cond == TCG_COND_NEVER) {
1010 tcg_gen_mov_i32(ret, v2);
1011 } else {
1012 tcg_gen_op6i_i32(INDEX_op_movcond, ret, c1, c2, v1, v2, cond);
1013 }
1014 }
1015
1016 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
1017 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
1018 {
1019 if (tcg_op_supported(INDEX_op_addci, TCG_TYPE_I32, 0)) {
1020 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
1021 tcg_gen_op3_i32(INDEX_op_addco, t0, al, bl);
1022 tcg_gen_op3_i32(INDEX_op_addci, rh, ah, bh);
1023 tcg_gen_mov_i32(rl, t0);
1024 tcg_temp_free_i32(t0);
1025 } else {
1026 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
1027 TCGv_i32 t1 = tcg_temp_ebb_new_i32();
1028 tcg_gen_add_i32(t0, al, bl);
1029 tcg_gen_setcond_i32(TCG_COND_LTU, t1, t0, al);
1030 tcg_gen_add_i32(rh, ah, bh);
1031 tcg_gen_add_i32(rh, rh, t1);
1032 tcg_gen_mov_i32(rl, t0);
1033 tcg_temp_free_i32(t0);
1034 tcg_temp_free_i32(t1);
1035 }
1036 }
1037
1038 void tcg_gen_addcio_i32(TCGv_i32 r, TCGv_i32 co,
1039 TCGv_i32 a, TCGv_i32 b, TCGv_i32 ci)
1040 {
1041 if (tcg_op_supported(INDEX_op_addci, TCG_TYPE_I32, 0)) {
1042 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
1043 TCGv_i32 zero = tcg_constant_i32(0);
1044 TCGv_i32 mone = tcg_constant_i32(-1);
1045
1046 tcg_gen_op3_i32(INDEX_op_addco, t0, ci, mone);
1047 tcg_gen_op3_i32(INDEX_op_addcio, r, a, b);
1048 tcg_gen_op3_i32(INDEX_op_addci, co, zero, zero);
1049 tcg_temp_free_i32(t0);
1050 } else {
1051 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
1052 TCGv_i32 t1 = tcg_temp_ebb_new_i32();
1053
1054 tcg_gen_add_i32(t0, a, b);
1055 tcg_gen_setcond_i32(TCG_COND_LTU, t1, t0, a);
1056 tcg_gen_add_i32(r, t0, ci);
1057 tcg_gen_setcond_i32(TCG_COND_LTU, t0, r, t0);
1058 tcg_gen_or_i32(co, t0, t1);
1059
1060 tcg_temp_free_i32(t0);
1061 tcg_temp_free_i32(t1);
1062 }
1063 }
1064
1065 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
1066 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
1067 {
1068 if (tcg_op_supported(INDEX_op_subbi, TCG_TYPE_I32, 0)) {
1069 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
1070 tcg_gen_op3_i32(INDEX_op_subbo, t0, al, bl);
1071 tcg_gen_op3_i32(INDEX_op_subbi, rh, ah, bh);
1072 tcg_gen_mov_i32(rl, t0);
1073 tcg_temp_free_i32(t0);
1074 } else {
1075 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
1076 TCGv_i32 t1 = tcg_temp_ebb_new_i32();
1077 tcg_gen_sub_i32(t0, al, bl);
1078 tcg_gen_setcond_i32(TCG_COND_LTU, t1, al, bl);
1079 tcg_gen_sub_i32(rh, ah, bh);
1080 tcg_gen_sub_i32(rh, rh, t1);
1081 tcg_gen_mov_i32(rl, t0);
1082 tcg_temp_free_i32(t0);
1083 tcg_temp_free_i32(t1);
1084 }
1085 }
1086
1087 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
1088 {
1089 if (tcg_op_supported(INDEX_op_mulu2, TCG_TYPE_I32, 0)) {
1090 tcg_gen_op4_i32(INDEX_op_mulu2, rl, rh, arg1, arg2);
1091 } else if (tcg_op_supported(INDEX_op_muluh, TCG_TYPE_I32, 0)) {
1092 TCGv_i32 t = tcg_temp_ebb_new_i32();
1093 tcg_gen_op3_i32(INDEX_op_mul, t, arg1, arg2);
1094 tcg_gen_op3_i32(INDEX_op_muluh, rh, arg1, arg2);
1095 tcg_gen_mov_i32(rl, t);
1096 tcg_temp_free_i32(t);
1097 } else {
1098 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
1099 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
1100 tcg_gen_extu_i32_i64(t0, arg1);
1101 tcg_gen_extu_i32_i64(t1, arg2);
1102 tcg_gen_mul_i64(t0, t0, t1);
1103 tcg_gen_extr_i64_i32(rl, rh, t0);
1104 tcg_temp_free_i64(t0);
1105 tcg_temp_free_i64(t1);
1106 }
1107 }
1108
1109 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
1110 {
1111 if (tcg_op_supported(INDEX_op_muls2, TCG_TYPE_I32, 0)) {
1112 tcg_gen_op4_i32(INDEX_op_muls2, rl, rh, arg1, arg2);
1113 } else if (tcg_op_supported(INDEX_op_mulsh, TCG_TYPE_I32, 0)) {
1114 TCGv_i32 t = tcg_temp_ebb_new_i32();
1115 tcg_gen_op3_i32(INDEX_op_mul, t, arg1, arg2);
1116 tcg_gen_op3_i32(INDEX_op_mulsh, rh, arg1, arg2);
1117 tcg_gen_mov_i32(rl, t);
1118 tcg_temp_free_i32(t);
1119 } else {
1120 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
1121 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
1122 tcg_gen_ext_i32_i64(t0, arg1);
1123 tcg_gen_ext_i32_i64(t1, arg2);
1124 tcg_gen_mul_i64(t0, t0, t1);
1125 tcg_gen_extr_i64_i32(rl, rh, t0);
1126 tcg_temp_free_i64(t0);
1127 tcg_temp_free_i64(t1);
1128 }
1129 }
1130
1131 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
1132 {
1133 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
1134 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
1135 tcg_gen_ext_i32_i64(t0, arg1);
1136 tcg_gen_extu_i32_i64(t1, arg2);
1137 tcg_gen_mul_i64(t0, t0, t1);
1138 tcg_gen_extr_i64_i32(rl, rh, t0);
1139 tcg_temp_free_i64(t0);
1140 tcg_temp_free_i64(t1);
1141 }
1142
1143 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg)
1144 {
1145 tcg_gen_sextract_i32(ret, arg, 0, 8);
1146 }
1147
1148 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg)
1149 {
1150 tcg_gen_sextract_i32(ret, arg, 0, 16);
1151 }
1152
1153 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg)
1154 {
1155 tcg_gen_extract_i32(ret, arg, 0, 8);
1156 }
1157
1158 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg)
1159 {
1160 tcg_gen_extract_i32(ret, arg, 0, 16);
1161 }
1162
1163 /*
1164 * Internal helper for bit and byte reversal.
1165 * Given a repeating matched block of 1's and 0's, swap the bits within
1166 * those two blocks. E.g. mask=00ff00ff, shift the input bits left and
1167 * right 8 bits.
1168 */
1169 static void gen_bitswap_i32(TCGv_i32 ret, TCGv_i32 arg, uint32_t mask)
1170 {
1171 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
1172 TCGv_i32 t1 = tcg_temp_ebb_new_i32();
1173 int sh = cto32(mask);
1174
1175 tcg_gen_andi_i32(t0, arg, mask);
1176 tcg_gen_shri_i32(t1, arg, sh);
1177 tcg_gen_shli_i32(t0, t0, sh);
1178 tcg_gen_andi_i32(t1, t1, mask);
1179 tcg_gen_or_i32(ret, t0, t1);
1180
1181 tcg_temp_free_i32(t0);
1182 tcg_temp_free_i32(t1);
1183 }
1184
1185 /* Similarly for 64-bit operands. */
1186 static void gen_bitswap_i64(TCGv_i64 ret, TCGv_i64 arg, uint64_t mask)
1187 {
1188 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
1189 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
1190 int sh = cto64(mask);
1191
1192 tcg_gen_andi_i64(t0, arg, mask);
1193 tcg_gen_shri_i64(t1, arg, sh);
1194 tcg_gen_shli_i64(t0, t0, sh);
1195 tcg_gen_andi_i64(t1, t1, mask);
1196 tcg_gen_or_i64(ret, t0, t1);
1197
1198 tcg_temp_free_i64(t0);
1199 tcg_temp_free_i64(t1);
1200 }
1201
1202 /*
1203 * bswap16_i32: 16-bit byte swap on the low bits of a 32-bit value.
1204 *
1205 * Byte pattern: xxab -> yyba
1206 *
1207 * With TCG_BSWAP_IZ, x == zero, else undefined.
1208 * With TCG_BSWAP_OZ, y == zero, with TCG_BSWAP_OS y == sign, else undefined.
1209 */
1210 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg, int flags)
1211 {
1212 /* Only one extension flag may be present. */
1213 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
1214
1215 if (tcg_op_supported(INDEX_op_bswap16, TCG_TYPE_I32, 0)) {
1216 tcg_gen_op3i_i32(INDEX_op_bswap16, ret, arg, flags);
1217 } else {
1218 TCGv_i32 t0 = tcg_temp_ebb_new_i32();
1219 TCGv_i32 t1 = tcg_temp_ebb_new_i32();
1220
1221 /* arg = ..ab (IZ) xxab (!IZ) */
1222 tcg_gen_shri_i32(t0, arg, 8); /* t0 = ...a (IZ) .xxa (!IZ) */
1223 if (!(flags & TCG_BSWAP_IZ)) {
1224 tcg_gen_ext8u_i32(t0, t0); /* t0 = ...a */
1225 }
1226
1227 if (flags & TCG_BSWAP_OS) {
1228 tcg_gen_shli_i32(t1, arg, 24); /* t1 = b... */
1229 tcg_gen_sari_i32(t1, t1, 16); /* t1 = ssb. */
1230 } else if (flags & TCG_BSWAP_OZ) {
1231 tcg_gen_ext8u_i32(t1, arg); /* t1 = ...b */
1232 tcg_gen_shli_i32(t1, t1, 8); /* t1 = ..b. */
1233 } else {
1234 tcg_gen_shli_i32(t1, arg, 8); /* t1 = xab. */
1235 }
1236
1237 tcg_gen_or_i32(ret, t0, t1); /* ret = ..ba (OZ) */
1238 /* = ssba (OS) */
1239 /* = xaba (no flag) */
1240 tcg_temp_free_i32(t0);
1241 tcg_temp_free_i32(t1);
1242 }
1243 }
1244
1245 /*
1246 * bswap32_i32: 32-bit byte swap on a 32-bit value.
1247 *
1248 * Byte pattern: abcd -> dcba
1249 */
1250 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg)
1251 {
1252 if (tcg_op_supported(INDEX_op_bswap32, TCG_TYPE_I32, 0)) {
1253 tcg_gen_op3i_i32(INDEX_op_bswap32, ret, arg, 0);
1254 } else {
1255 gen_bitswap_i32(ret, arg, 0x00ff00ff);
1256 tcg_gen_hswap_i32(ret, ret);
1257 }
1258 }
1259
1260 /*
1261 * hswap_i32: Swap 16-bit halfwords within a 32-bit value.
1262 *
1263 * Byte pattern: abcd -> cdab
1264 */
1265 void tcg_gen_hswap_i32(TCGv_i32 ret, TCGv_i32 arg)
1266 {
1267 /* Swapping 2 16-bit elements is a rotate. */
1268 tcg_gen_rotli_i32(ret, arg, 16);
1269 }
1270
1271 void tcg_gen_revbit8_i32(TCGv_i32 ret, TCGv_i32 arg)
1272 {
1273 if (tcg_op_supported(INDEX_op_revbit8, TCG_TYPE_I32, 0)) {
1274 tcg_gen_op2_i32(INDEX_op_revbit8, ret, arg);
1275 } else if (tcg_op_supported(INDEX_op_revbit32, TCG_TYPE_I32, 0)) {
1276 tcg_gen_op2_i32(INDEX_op_revbit32, ret, arg);
1277 tcg_gen_bswap32_i32(ret, ret);
1278 } else {
1279 gen_bitswap_i32(ret, arg, 0x55555555u);
1280 gen_bitswap_i32(ret, ret, 0x33333333u);
1281 gen_bitswap_i32(ret, ret, 0x0f0f0f0fu);
1282 }
1283 }
1284
1285 void tcg_gen_revbit32_i32(TCGv_i32 ret, TCGv_i32 arg)
1286 {
1287 if (tcg_op_supported(INDEX_op_revbit32, TCG_TYPE_I32, 0)) {
1288 tcg_gen_op3i_i32(INDEX_op_revbit32, ret, arg, 0);
1289 } else {
1290 tcg_gen_revbit8_i32(ret, arg);
1291 tcg_gen_bswap32_i32(ret, ret);
1292 }
1293 }
1294
1295 void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1296 {
1297 if (tcg_op_supported(INDEX_op_smin, TCG_TYPE_I32, 0)) {
1298 tcg_gen_op3_i32(INDEX_op_smin, ret, a, b);
1299 } else {
1300 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b);
1301 }
1302 }
1303
1304 void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1305 {
1306 if (tcg_op_supported(INDEX_op_umin, TCG_TYPE_I32, 0)) {
1307 tcg_gen_op3_i32(INDEX_op_umin, ret, a, b);
1308 } else {
1309 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b);
1310 }
1311 }
1312
1313 void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1314 {
1315 if (tcg_op_supported(INDEX_op_smax, TCG_TYPE_I32, 0)) {
1316 tcg_gen_op3_i32(INDEX_op_smax, ret, a, b);
1317 } else {
1318 tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a);
1319 }
1320 }
1321
1322 void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1323 {
1324 if (tcg_op_supported(INDEX_op_umax, TCG_TYPE_I32, 0)) {
1325 tcg_gen_op3_i32(INDEX_op_umax, ret, a, b);
1326 } else {
1327 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a);
1328 }
1329 }
1330
1331 void tcg_gen_ussub_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1332 {
1333 TCGv_i32 t = tcg_temp_ebb_new_i32();
1334 TCGv_i32 z = tcg_constant_i32(0);
1335
1336 tcg_gen_sub_i32(t, a, b);
1337 tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, z, t);
1338 tcg_temp_free_i32(t);
1339 }
1340
1341 void tcg_gen_abs_i32(TCGv_i32 ret, TCGv_i32 a)
1342 {
1343 TCGv_i32 t = tcg_temp_ebb_new_i32();
1344
1345 tcg_gen_sari_i32(t, a, 31);
1346 tcg_gen_xor_i32(ret, a, t);
1347 tcg_gen_sub_i32(ret, ret, t);
1348 tcg_temp_free_i32(t);
1349 }
1350
1351 void tcg_gen_ld8u_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset)
1352 {
1353 tcg_gen_ldst_op_i32(INDEX_op_ld8u, ret, arg2, offset);
1354 }
1355
1356 void tcg_gen_ld8s_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset)
1357 {
1358 tcg_gen_ldst_op_i32(INDEX_op_ld8s, ret, arg2, offset);
1359 }
1360
1361 void tcg_gen_ld16u_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset)
1362 {
1363 tcg_gen_ldst_op_i32(INDEX_op_ld16u, ret, arg2, offset);
1364 }
1365
1366 void tcg_gen_ld16s_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset)
1367 {
1368 tcg_gen_ldst_op_i32(INDEX_op_ld16s, ret, arg2, offset);
1369 }
1370
1371 void tcg_gen_ld_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset)
1372 {
1373 tcg_gen_ldst_op_i32(INDEX_op_ld, ret, arg2, offset);
1374 }
1375
1376 void tcg_gen_st8_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset)
1377 {
1378 tcg_gen_ldst_op_i32(INDEX_op_st8, arg1, arg2, offset);
1379 }
1380
1381 void tcg_gen_st16_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset)
1382 {
1383 tcg_gen_ldst_op_i32(INDEX_op_st16, arg1, arg2, offset);
1384 }
1385
1386 void tcg_gen_st_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset)
1387 {
1388 tcg_gen_ldst_op_i32(INDEX_op_st, arg1, arg2, offset);
1389 }
1390
1391
1392 /* 64-bit ops */
1393
1394 void tcg_gen_discard_i64(TCGv_i64 arg)
1395 {
1396 tcg_gen_op1_i64(INDEX_op_discard, TCG_TYPE_I64, arg);
1397 }
1398
1399 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg)
1400 {
1401 if (ret != arg) {
1402 tcg_gen_op2_i64(INDEX_op_mov, ret, arg);
1403 }
1404 }
1405
1406 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1407 {
1408 tcg_gen_mov_i64(ret, tcg_constant_i64(arg));
1409 }
1410
1411 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1412 {
1413 tcg_gen_ldst_op_i64(INDEX_op_ld8u, ret, arg2, offset);
1414 }
1415
1416 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1417 {
1418 tcg_gen_ldst_op_i64(INDEX_op_ld8s, ret, arg2, offset);
1419 }
1420
1421 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1422 {
1423 tcg_gen_ldst_op_i64(INDEX_op_ld16u, ret, arg2, offset);
1424 }
1425
1426 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1427 {
1428 tcg_gen_ldst_op_i64(INDEX_op_ld16s, ret, arg2, offset);
1429 }
1430
1431 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1432 {
1433 tcg_gen_ldst_op_i64(INDEX_op_ld32u, ret, arg2, offset);
1434 }
1435
1436 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1437 {
1438 tcg_gen_ldst_op_i64(INDEX_op_ld32s, ret, arg2, offset);
1439 }
1440
1441 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1442 {
1443 tcg_gen_ldst_op_i64(INDEX_op_ld, ret, arg2, offset);
1444 }
1445
1446 void tcg_gen_st8_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1447 {
1448 tcg_gen_ldst_op_i64(INDEX_op_st8, arg1, arg2, offset);
1449 }
1450
1451 void tcg_gen_st16_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1452 {
1453 tcg_gen_ldst_op_i64(INDEX_op_st16, arg1, arg2, offset);
1454 }
1455
1456 void tcg_gen_st32_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1457 {
1458 tcg_gen_ldst_op_i64(INDEX_op_st32, arg1, arg2, offset);
1459 }
1460
1461 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1462 {
1463 tcg_gen_ldst_op_i64(INDEX_op_st, arg1, arg2, offset);
1464 }
1465
1466 void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1467 {
1468 tcg_gen_op3_i64(INDEX_op_add, ret, arg1, arg2);
1469 }
1470
1471 void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1472 {
1473 tcg_gen_op3_i64(INDEX_op_sub, ret, arg1, arg2);
1474 }
1475
1476 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1477 {
1478 tcg_gen_op3_i64(INDEX_op_and, ret, arg1, arg2);
1479 }
1480
1481 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1482 {
1483 tcg_gen_op3_i64(INDEX_op_or, ret, arg1, arg2);
1484 }
1485
1486 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1487 {
1488 tcg_gen_op3_i64(INDEX_op_xor, ret, arg1, arg2);
1489 }
1490
1491 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1492 {
1493 tcg_gen_op3_i64(INDEX_op_shl, ret, arg1, arg2);
1494 }
1495
1496 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1497 {
1498 tcg_gen_op3_i64(INDEX_op_shr, ret, arg1, arg2);
1499 }
1500
1501 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1502 {
1503 tcg_gen_op3_i64(INDEX_op_sar, ret, arg1, arg2);
1504 }
1505
1506 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1507 {
1508 tcg_gen_op3_i64(INDEX_op_mul, ret, arg1, arg2);
1509 }
1510
1511 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1512 {
1513 /* some cases can be optimized here */
1514 if (arg2 == 0) {
1515 tcg_gen_mov_i64(ret, arg1);
1516 } else {
1517 tcg_gen_add_i64(ret, arg1, tcg_constant_i64(arg2));
1518 }
1519 }
1520
1521 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2)
1522 {
1523 if (arg1 == 0) {
1524 tcg_gen_neg_i64(ret, arg2);
1525 } else {
1526 tcg_gen_sub_i64(ret, tcg_constant_i64(arg1), arg2);
1527 }
1528 }
1529
1530 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1531 {
1532 tcg_gen_addi_i64(ret, arg1, -arg2);
1533 }
1534
1535 void tcg_gen_neg_i64(TCGv_i64 ret, TCGv_i64 arg)
1536 {
1537 tcg_gen_op2_i64(INDEX_op_neg, ret, arg);
1538 }
1539
1540 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1541 {
1542 /* Some cases can be optimized here. */
1543 switch (arg2) {
1544 case 0:
1545 tcg_gen_movi_i64(ret, 0);
1546 return;
1547 case -1:
1548 tcg_gen_mov_i64(ret, arg1);
1549 return;
1550 default:
1551 /*
1552 * Canonicalize on extract, if valid. This aids x86 with its
1553 * 2 operand MOVZBL and 2 operand AND, selecting the TCGOpcode
1554 * which does not require matching operands. Other backends can
1555 * trivially expand the extract to AND during code generation.
1556 */
1557 if (!(arg2 & (arg2 + 1))) {
1558 unsigned len = ctz64(~arg2);
1559 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, len)) {
1560 tcg_gen_extract_i64(ret, arg1, 0, len);
1561 return;
1562 }
1563 }
1564 break;
1565 }
1566
1567 tcg_gen_and_i64(ret, arg1, tcg_constant_i64(arg2));
1568 }
1569
1570 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1571 {
1572 /* Some cases can be optimized here. */
1573 if (arg2 == -1) {
1574 tcg_gen_movi_i64(ret, -1);
1575 } else if (arg2 == 0) {
1576 tcg_gen_mov_i64(ret, arg1);
1577 } else {
1578 tcg_gen_or_i64(ret, arg1, tcg_constant_i64(arg2));
1579 }
1580 }
1581
1582 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1583 {
1584 /* Some cases can be optimized here. */
1585 if (arg2 == 0) {
1586 tcg_gen_mov_i64(ret, arg1);
1587 } else if (arg2 == -1 &&
1588 tcg_op_supported(INDEX_op_not, TCG_TYPE_I64, 0)) {
1589 /* Don't recurse with tcg_gen_not_i64. */
1590 tcg_gen_op2_i64(INDEX_op_not, ret, arg1);
1591 } else {
1592 tcg_gen_xor_i64(ret, arg1, tcg_constant_i64(arg2));
1593 }
1594 }
1595
1596 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1597 {
1598 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1599 if (arg2 == 0) {
1600 tcg_gen_mov_i64(ret, arg1);
1601 } else {
1602 tcg_gen_shl_i64(ret, arg1, tcg_constant_i64(arg2));
1603 }
1604 }
1605
1606 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1607 {
1608 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1609 if (arg2 == 0) {
1610 tcg_gen_mov_i64(ret, arg1);
1611 } else {
1612 tcg_gen_shr_i64(ret, arg1, tcg_constant_i64(arg2));
1613 }
1614 }
1615
1616 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1617 {
1618 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1619 if (arg2 == 0) {
1620 tcg_gen_mov_i64(ret, arg1);
1621 } else {
1622 tcg_gen_sar_i64(ret, arg1, tcg_constant_i64(arg2));
1623 }
1624 }
1625
1626 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l)
1627 {
1628 if (cond == TCG_COND_ALWAYS) {
1629 tcg_gen_br(l);
1630 } else if (cond != TCG_COND_NEVER) {
1631 TCGOp *op = tcg_gen_op4ii_i64(INDEX_op_brcond, arg1, arg2, cond,
1632 label_arg(l));
1633 add_as_label_use(l, op);
1634 }
1635 }
1636
1637 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l)
1638 {
1639 tcg_gen_brcond_i64(cond, arg1, tcg_constant_i64(arg2), l);
1640 }
1641
1642 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
1643 TCGv_i64 arg1, TCGv_i64 arg2)
1644 {
1645 if (cond == TCG_COND_ALWAYS) {
1646 tcg_gen_movi_i64(ret, 1);
1647 } else if (cond == TCG_COND_NEVER) {
1648 tcg_gen_movi_i64(ret, 0);
1649 } else {
1650 tcg_gen_op4i_i64(INDEX_op_setcond, ret, arg1, arg2, cond);
1651 }
1652 }
1653
1654 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
1655 TCGv_i64 arg1, int64_t arg2)
1656 {
1657 tcg_gen_setcond_i64(cond, ret, arg1, tcg_constant_i64(arg2));
1658 }
1659
1660 void tcg_gen_negsetcondi_i64(TCGCond cond, TCGv_i64 ret,
1661 TCGv_i64 arg1, int64_t arg2)
1662 {
1663 tcg_gen_negsetcond_i64(cond, ret, arg1, tcg_constant_i64(arg2));
1664 }
1665
1666 void tcg_gen_negsetcond_i64(TCGCond cond, TCGv_i64 ret,
1667 TCGv_i64 arg1, TCGv_i64 arg2)
1668 {
1669 if (cond == TCG_COND_ALWAYS) {
1670 tcg_gen_movi_i64(ret, -1);
1671 } else if (cond == TCG_COND_NEVER) {
1672 tcg_gen_movi_i64(ret, 0);
1673 } else {
1674 tcg_gen_op4i_i64(INDEX_op_negsetcond, ret, arg1, arg2, cond);
1675 }
1676 }
1677
1678 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1679 {
1680 if (arg2 == 0) {
1681 tcg_gen_movi_i64(ret, 0);
1682 } else if (is_power_of_2(arg2)) {
1683 tcg_gen_shli_i64(ret, arg1, ctz64(arg2));
1684 } else {
1685 tcg_gen_mul_i64(ret, arg1, tcg_constant_i64(arg2));
1686 }
1687 }
1688
1689 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1690 {
1691 if (tcg_op_supported(INDEX_op_divs, TCG_TYPE_I64, 0)) {
1692 tcg_gen_op3_i64(INDEX_op_divs, ret, arg1, arg2);
1693 } else if (tcg_op_supported(INDEX_op_divs2, TCG_TYPE_I64, 0)) {
1694 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
1695 tcg_gen_sari_i64(t0, arg1, 63);
1696 tcg_gen_op5_i64(INDEX_op_divs2, ret, t0, arg1, t0, arg2);
1697 tcg_temp_free_i64(t0);
1698 } else {
1699 gen_helper_div_i64(ret, arg1, arg2);
1700 }
1701 }
1702
1703 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1704 {
1705 if (tcg_op_supported(INDEX_op_rems, TCG_TYPE_I64, 0)) {
1706 tcg_gen_op3_i64(INDEX_op_rems, ret, arg1, arg2);
1707 } else if (tcg_op_supported(INDEX_op_divs, TCG_TYPE_I64, 0)) {
1708 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
1709 tcg_gen_op3_i64(INDEX_op_divs, t0, arg1, arg2);
1710 tcg_gen_mul_i64(t0, t0, arg2);
1711 tcg_gen_sub_i64(ret, arg1, t0);
1712 tcg_temp_free_i64(t0);
1713 } else if (tcg_op_supported(INDEX_op_divs2, TCG_TYPE_I64, 0)) {
1714 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
1715 tcg_gen_sari_i64(t0, arg1, 63);
1716 tcg_gen_op5_i64(INDEX_op_divs2, t0, ret, arg1, t0, arg2);
1717 tcg_temp_free_i64(t0);
1718 } else {
1719 gen_helper_rem_i64(ret, arg1, arg2);
1720 }
1721 }
1722
1723 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1724 {
1725 if (tcg_op_supported(INDEX_op_divu, TCG_TYPE_I64, 0)) {
1726 tcg_gen_op3_i64(INDEX_op_divu, ret, arg1, arg2);
1727 } else if (tcg_op_supported(INDEX_op_divu2, TCG_TYPE_I64, 0)) {
1728 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
1729 TCGv_i64 zero = tcg_constant_i64(0);
1730 tcg_gen_op5_i64(INDEX_op_divu2, ret, t0, arg1, zero, arg2);
1731 tcg_temp_free_i64(t0);
1732 } else {
1733 gen_helper_divu_i64(ret, arg1, arg2);
1734 }
1735 }
1736
1737 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1738 {
1739 if (tcg_op_supported(INDEX_op_remu, TCG_TYPE_I64, 0)) {
1740 tcg_gen_op3_i64(INDEX_op_remu, ret, arg1, arg2);
1741 } else if (tcg_op_supported(INDEX_op_divu, TCG_TYPE_I64, 0)) {
1742 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
1743 tcg_gen_op3_i64(INDEX_op_divu, t0, arg1, arg2);
1744 tcg_gen_mul_i64(t0, t0, arg2);
1745 tcg_gen_sub_i64(ret, arg1, t0);
1746 tcg_temp_free_i64(t0);
1747 } else if (tcg_op_supported(INDEX_op_divu2, TCG_TYPE_I64, 0)) {
1748 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
1749 TCGv_i64 zero = tcg_constant_i64(0);
1750 tcg_gen_op5_i64(INDEX_op_divu2, t0, ret, arg1, zero, arg2);
1751 tcg_temp_free_i64(t0);
1752 } else {
1753 gen_helper_remu_i64(ret, arg1, arg2);
1754 }
1755 }
1756
1757 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg)
1758 {
1759 tcg_gen_sextract_i64(ret, arg, 0, 8);
1760 }
1761
1762 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg)
1763 {
1764 tcg_gen_sextract_i64(ret, arg, 0, 16);
1765 }
1766
1767 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg)
1768 {
1769 tcg_gen_sextract_i64(ret, arg, 0, 32);
1770 }
1771
1772 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg)
1773 {
1774 tcg_gen_extract_i64(ret, arg, 0, 8);
1775 }
1776
1777 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg)
1778 {
1779 tcg_gen_extract_i64(ret, arg, 0, 16);
1780 }
1781
1782 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg)
1783 {
1784 tcg_gen_extract_i64(ret, arg, 0, 32);
1785 }
1786
1787 /*
1788 * bswap16_i64: 16-bit byte swap on the low bits of a 64-bit value.
1789 *
1790 * Byte pattern: xxxxxxxxab -> yyyyyyyyba
1791 *
1792 * With TCG_BSWAP_IZ, x == zero, else undefined.
1793 * With TCG_BSWAP_OZ, y == zero, with TCG_BSWAP_OS y == sign, else undefined.
1794 */
1795 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg, int flags)
1796 {
1797 /* Only one extension flag may be present. */
1798 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
1799
1800 if (tcg_op_supported(INDEX_op_bswap16, TCG_TYPE_I64, 0)) {
1801 tcg_gen_op3i_i64(INDEX_op_bswap16, ret, arg, flags);
1802 } else {
1803 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
1804 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
1805
1806 /* arg = ......ab or xxxxxxab */
1807 tcg_gen_shri_i64(t0, arg, 8); /* t0 = .......a or .xxxxxxa */
1808 if (!(flags & TCG_BSWAP_IZ)) {
1809 tcg_gen_ext8u_i64(t0, t0); /* t0 = .......a */
1810 }
1811
1812 if (flags & TCG_BSWAP_OS) {
1813 tcg_gen_shli_i64(t1, arg, 56); /* t1 = b....... */
1814 tcg_gen_sari_i64(t1, t1, 48); /* t1 = ssssssb. */
1815 } else if (flags & TCG_BSWAP_OZ) {
1816 tcg_gen_ext8u_i64(t1, arg); /* t1 = .......b */
1817 tcg_gen_shli_i64(t1, t1, 8); /* t1 = ......b. */
1818 } else {
1819 tcg_gen_shli_i64(t1, arg, 8); /* t1 = xxxxxab. */
1820 }
1821
1822 tcg_gen_or_i64(ret, t0, t1); /* ret = ......ba (OZ) */
1823 /* ssssssba (OS) */
1824 /* xxxxxaba (no flag) */
1825 tcg_temp_free_i64(t0);
1826 tcg_temp_free_i64(t1);
1827 }
1828 }
1829
1830 /*
1831 * bswap32_i64: 32-bit byte swap on the low bits of a 64-bit value.
1832 *
1833 * Byte pattern: xxxxabcd -> yyyydcba
1834 *
1835 * With TCG_BSWAP_IZ, x == zero, else undefined.
1836 * With TCG_BSWAP_OZ, y == zero, with TCG_BSWAP_OS y == sign, else undefined.
1837 */
1838 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg, int flags)
1839 {
1840 /* Only one extension flag may be present. */
1841 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
1842
1843 if (tcg_op_supported(INDEX_op_bswap32, TCG_TYPE_I64, 0)) {
1844 tcg_gen_op3i_i64(INDEX_op_bswap32, ret, arg, flags);
1845 } else {
1846 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
1847 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
1848
1849 /* arg = xxxxabcd */
1850 gen_bitswap_i64(ret, arg, 0x00ff00ff); /* ret = ....badc */
1851
1852 tcg_gen_shli_i64(t1, ret, 48); /* t1 = dc...... */
1853 tcg_gen_shri_i64(t0, ret, 16); /* t0 = ......ba */
1854 if (flags & TCG_BSWAP_OS) {
1855 tcg_gen_sari_i64(t1, t1, 32); /* t1 = ssssdc.. */
1856 } else {
1857 tcg_gen_shri_i64(t1, t1, 32); /* t1 = ....dc.. */
1858 }
1859 tcg_gen_or_i64(ret, t0, t1); /* ret = ssssdcba (OS) */
1860 /* ....dcba (else) */
1861
1862 tcg_temp_free_i64(t0);
1863 tcg_temp_free_i64(t1);
1864 }
1865 }
1866
1867 /*
1868 * bswap64_i64: 64-bit byte swap on a 64-bit value.
1869 *
1870 * Byte pattern: abcdefgh -> hgfedcba
1871 */
1872 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg)
1873 {
1874 if (tcg_op_supported(INDEX_op_bswap64, TCG_TYPE_I64, 0)) {
1875 tcg_gen_op3i_i64(INDEX_op_bswap64, ret, arg, 0);
1876 } else {
1877 gen_bitswap_i64(ret, arg, 0x00ff00ff00ff00ffull);
1878 tcg_gen_hswap_i64(ret, ret);
1879 }
1880 }
1881
1882 /*
1883 * hswap_i64: Swap 16-bit halfwords within a 64-bit value.
1884 * See also include/qemu/bitops.h, hswap64.
1885 *
1886 * Byte pattern: abcdefgh -> ghefcdab
1887 */
1888 void tcg_gen_hswap_i64(TCGv_i64 ret, TCGv_i64 arg)
1889 {
1890 gen_bitswap_i64(ret, arg, 0x0000ffff0000ffffull);
1891 tcg_gen_wswap_i64(ret, ret);
1892 }
1893
1894 /*
1895 * wswap_i64: Swap 32-bit words within a 64-bit value.
1896 *
1897 * Byte pattern: abcdefgh -> efghabcd
1898 */
1899 void tcg_gen_wswap_i64(TCGv_i64 ret, TCGv_i64 arg)
1900 {
1901 /* Swapping 2 32-bit elements is a rotate. */
1902 tcg_gen_rotli_i64(ret, arg, 32);
1903 }
1904
1905 void tcg_gen_revbit32_i64(TCGv_i64 ret, TCGv_i64 arg, int flags)
1906 {
1907 /* Only one extension flag may be present. */
1908 tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
1909
1910 if (tcg_op_supported(INDEX_op_revbit32, TCG_TYPE_I64, 0)) {
1911 tcg_gen_op3i_i64(INDEX_op_revbit32, ret, arg, flags);
1912 } else if (tcg_op_supported(INDEX_op_revbit64, TCG_TYPE_I64, 0)) {
1913 tcg_gen_op2_i64(INDEX_op_revbit64, ret, arg);
1914 if (flags & TCG_BSWAP_OS) {
1915 tcg_gen_sari_i64(ret, ret, 32);
1916 } else {
1917 tcg_gen_shri_i64(ret, ret, 32);
1918 }
1919 } else {
1920 if (tcg_op_supported(INDEX_op_revbit8, TCG_TYPE_I64, 0)) {
1921 tcg_gen_op2_i64(INDEX_op_revbit8, ret, arg);
1922 } else {
1923 gen_bitswap_i64(ret, arg, 0x55555555ull);
1924 gen_bitswap_i64(ret, ret, 0x33333333ull);
1925 gen_bitswap_i64(ret, ret, 0x0f0f0f0full);
1926 flags |= TCG_BSWAP_IZ;
1927 }
1928 tcg_gen_bswap32_i64(ret, ret, flags);
1929 }
1930 }
1931
1932 void tcg_gen_revbit8_i64(TCGv_i64 ret, TCGv_i64 arg)
1933 {
1934 if (tcg_op_supported(INDEX_op_revbit8, TCG_TYPE_I64, 0)) {
1935 tcg_gen_op2_i64(INDEX_op_revbit8, ret, arg);
1936 } else if (tcg_op_supported(INDEX_op_revbit64, TCG_TYPE_I64, 0)) {
1937 tcg_gen_op2_i64(INDEX_op_revbit64, ret, arg);
1938 tcg_gen_bswap64_i64(ret, ret);
1939 } else {
1940 gen_bitswap_i64(ret, arg, 0x5555555555555555ull);
1941 gen_bitswap_i64(ret, ret, 0x3333333333333333ull);
1942 gen_bitswap_i64(ret, ret, 0x0f0f0f0f0f0f0f0full);
1943 }
1944 }
1945
1946 void tcg_gen_revbit64_i64(TCGv_i64 ret, TCGv_i64 arg)
1947 {
1948 if (tcg_op_supported(INDEX_op_revbit64, TCG_TYPE_I64, 0)) {
1949 tcg_gen_op2_i64(INDEX_op_revbit64, ret, arg);
1950 } else {
1951 tcg_gen_revbit8_i64(ret, arg);
1952 tcg_gen_bswap64_i64(ret, ret);
1953 }
1954 }
1955
1956 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg)
1957 {
1958 if (tcg_op_supported(INDEX_op_not, TCG_TYPE_I64, 0)) {
1959 tcg_gen_op2_i64(INDEX_op_not, ret, arg);
1960 } else {
1961 tcg_gen_xori_i64(ret, arg, -1);
1962 }
1963 }
1964
1965 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1966 {
1967 if (tcg_op_supported(INDEX_op_andc, TCG_TYPE_I64, 0)) {
1968 tcg_gen_op3_i64(INDEX_op_andc, ret, arg1, arg2);
1969 } else {
1970 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
1971 tcg_gen_not_i64(t0, arg2);
1972 tcg_gen_and_i64(ret, arg1, t0);
1973 tcg_temp_free_i64(t0);
1974 }
1975 }
1976
1977 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1978 {
1979 if (tcg_op_supported(INDEX_op_eqv, TCG_TYPE_I64, 0)) {
1980 tcg_gen_op3_i64(INDEX_op_eqv, ret, arg1, arg2);
1981 } else {
1982 tcg_gen_xor_i64(ret, arg1, arg2);
1983 tcg_gen_not_i64(ret, ret);
1984 }
1985 }
1986
1987 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1988 {
1989 if (tcg_op_supported(INDEX_op_nand, TCG_TYPE_I64, 0)) {
1990 tcg_gen_op3_i64(INDEX_op_nand, ret, arg1, arg2);
1991 } else {
1992 tcg_gen_and_i64(ret, arg1, arg2);
1993 tcg_gen_not_i64(ret, ret);
1994 }
1995 }
1996
1997 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1998 {
1999 if (tcg_op_supported(INDEX_op_nor, TCG_TYPE_I64, 0)) {
2000 tcg_gen_op3_i64(INDEX_op_nor, ret, arg1, arg2);
2001 } else {
2002 tcg_gen_or_i64(ret, arg1, arg2);
2003 tcg_gen_not_i64(ret, ret);
2004 }
2005 }
2006
2007 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2008 {
2009 if (tcg_op_supported(INDEX_op_orc, TCG_TYPE_I64, 0)) {
2010 tcg_gen_op3_i64(INDEX_op_orc, ret, arg1, arg2);
2011 } else {
2012 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2013 tcg_gen_not_i64(t0, arg2);
2014 tcg_gen_or_i64(ret, arg1, t0);
2015 tcg_temp_free_i64(t0);
2016 }
2017 }
2018
2019 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2020 {
2021 if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_I64, 0)) {
2022 tcg_gen_op3_i64(INDEX_op_clz, ret, arg1, arg2);
2023 } else {
2024 gen_helper_clz_i64(ret, arg1, arg2);
2025 }
2026 }
2027
2028 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
2029 {
2030 tcg_gen_clz_i64(ret, arg1, tcg_constant_i64(arg2));
2031 }
2032
2033 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2034 {
2035 TCGv_i64 z, t;
2036
2037 if (tcg_op_supported(INDEX_op_ctz, TCG_TYPE_I64, 0)) {
2038 tcg_gen_op3_i64(INDEX_op_ctz, ret, arg1, arg2);
2039 return;
2040 }
2041 if (tcg_op_supported(INDEX_op_ctpop, TCG_TYPE_I64, 0)) {
2042 t = tcg_temp_ebb_new_i64();
2043 tcg_gen_subi_i64(t, arg1, 1);
2044 tcg_gen_andc_i64(t, t, arg1);
2045 tcg_gen_ctpop_i64(t, t);
2046 } else if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_I64, 0)) {
2047 t = tcg_temp_ebb_new_i64();
2048 tcg_gen_neg_i64(t, arg1);
2049 tcg_gen_and_i64(t, t, arg1);
2050 tcg_gen_clzi_i64(t, t, 64);
2051 tcg_gen_xori_i64(t, t, 63);
2052 } else {
2053 gen_helper_ctz_i64(ret, arg1, arg2);
2054 return;
2055 }
2056
2057 z = tcg_constant_i64(0);
2058 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t);
2059 tcg_temp_free_i64(t);
2060 }
2061
2062 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
2063 {
2064 if (arg2 == 64
2065 && !tcg_op_supported(INDEX_op_ctz, TCG_TYPE_I64, 0)
2066 && tcg_op_supported(INDEX_op_ctpop, TCG_TYPE_I64, 0)) {
2067 /* This equivalence has the advantage of not requiring a fixup. */
2068 TCGv_i64 t = tcg_temp_ebb_new_i64();
2069 tcg_gen_subi_i64(t, arg1, 1);
2070 tcg_gen_andc_i64(t, t, arg1);
2071 tcg_gen_ctpop_i64(ret, t);
2072 tcg_temp_free_i64(t);
2073 } else {
2074 tcg_gen_ctz_i64(ret, arg1, tcg_constant_i64(arg2));
2075 }
2076 }
2077
2078 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg)
2079 {
2080 if (tcg_op_supported(INDEX_op_clz, TCG_TYPE_I64, 0)) {
2081 TCGv_i64 t = tcg_temp_ebb_new_i64();
2082 tcg_gen_sari_i64(t, arg, 63);
2083 tcg_gen_xor_i64(t, t, arg);
2084 tcg_gen_clzi_i64(t, t, 64);
2085 tcg_gen_subi_i64(ret, t, 1);
2086 tcg_temp_free_i64(t);
2087 } else {
2088 gen_helper_clrsb_i64(ret, arg);
2089 }
2090 }
2091
2092 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1)
2093 {
2094 if (tcg_op_supported(INDEX_op_ctpop, TCG_TYPE_I64, 0)) {
2095 tcg_gen_op2_i64(INDEX_op_ctpop, ret, arg1);
2096 } else {
2097 gen_helper_ctpop_i64(ret, arg1);
2098 }
2099 }
2100
2101 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2102 {
2103 if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I64, 0)) {
2104 tcg_gen_op3_i64(INDEX_op_rotl, ret, arg1, arg2);
2105 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I64, 0)) {
2106 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2107 tcg_gen_neg_i64(t0, arg2);
2108 tcg_gen_op3_i64(INDEX_op_rotr, ret, arg1, t0);
2109 tcg_temp_free_i64(t0);
2110 } else {
2111 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2112 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
2113 tcg_gen_shl_i64(t0, arg1, arg2);
2114 tcg_gen_neg_i64(t1, arg2);
2115 tcg_gen_shr_i64(t1, arg1, t1);
2116 tcg_gen_or_i64(ret, t0, t1);
2117 tcg_temp_free_i64(t0);
2118 tcg_temp_free_i64(t1);
2119 }
2120 }
2121
2122 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
2123 {
2124 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
2125 if (arg2 == 0) {
2126 tcg_gen_mov_i64(ret, arg1);
2127 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I64, 0)) {
2128 tcg_gen_op3_i64(INDEX_op_rotl, ret, arg1, tcg_constant_i64(arg2));
2129 } else {
2130 tcg_gen_rotri_i64(ret, arg1, -arg2 & 63);
2131 }
2132 }
2133
2134 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2135 {
2136 if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I64, 0)) {
2137 tcg_gen_op3_i64(INDEX_op_rotr, ret, arg1, arg2);
2138 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I64, 0)) {
2139 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2140 tcg_gen_neg_i64(t0, arg2);
2141 tcg_gen_op3_i64(INDEX_op_rotl, ret, arg1, t0);
2142 tcg_temp_free_i64(t0);
2143 } else {
2144 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2145 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
2146 tcg_gen_shr_i64(t0, arg1, arg2);
2147 tcg_gen_neg_i64(t1, arg2);
2148 tcg_gen_shl_i64(t1, arg1, t1);
2149 tcg_gen_or_i64(ret, t0, t1);
2150 tcg_temp_free_i64(t0);
2151 tcg_temp_free_i64(t1);
2152 }
2153 }
2154
2155 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
2156 {
2157 tcg_debug_assert(arg2 >= 0 && arg2 < 64);
2158 if (arg2 == 0) {
2159 tcg_gen_mov_i64(ret, arg1);
2160 } else if (tcg_op_supported(INDEX_op_rotr, TCG_TYPE_I64, 0)) {
2161 tcg_gen_op3_i64(INDEX_op_rotr, ret, arg1, tcg_constant_i64(arg2));
2162 } else if (tcg_op_supported(INDEX_op_rotl, TCG_TYPE_I64, 0)) {
2163 tcg_gen_op3_i64(INDEX_op_rotl, ret, arg1, tcg_constant_i64(64 - arg2));
2164 } else {
2165 /* Do not recurse with the rotri simplification. */
2166 tcg_gen_op4i_i64(INDEX_op_extract2, ret, arg1, arg1, arg2);
2167 }
2168 }
2169
2170 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2,
2171 unsigned int ofs, unsigned int len)
2172 {
2173 tcg_debug_assert(ofs < 64);
2174 tcg_debug_assert(len > 0);
2175 tcg_debug_assert(len <= 64);
2176 tcg_debug_assert(ofs + len <= 64);
2177
2178 if (len == 64) {
2179 tcg_gen_mov_i64(ret, arg2);
2180 } else {
2181 tcg_gen_op5ii_i64(INDEX_op_deposit, ret, arg1, arg2, ofs, len);
2182 }
2183 }
2184
2185 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg,
2186 unsigned int ofs, unsigned int len)
2187 {
2188 tcg_debug_assert(ofs < 64);
2189 tcg_debug_assert(len > 0);
2190 tcg_debug_assert(len <= 64);
2191 tcg_debug_assert(ofs + len <= 64);
2192
2193 if (ofs + len == 64) {
2194 tcg_gen_shli_i64(ret, arg, ofs);
2195 } else if (ofs == 0) {
2196 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2197 } else {
2198 TCGv_i64 zero = tcg_constant_i64(0);
2199 tcg_gen_op5ii_i64(INDEX_op_deposit, ret, zero, arg, ofs, len);
2200 }
2201 }
2202
2203 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg,
2204 unsigned int ofs, unsigned int len)
2205 {
2206 uint64_t mask;
2207
2208 tcg_debug_assert(ofs < 64);
2209 tcg_debug_assert(len > 0);
2210 tcg_debug_assert(len <= 64);
2211 tcg_debug_assert(ofs + len <= 64);
2212
2213 /* Canonicalize certain special cases, even if extract is supported. */
2214 if (ofs + len == 64) {
2215 tcg_gen_shri_i64(ret, arg, 64 - len);
2216 return;
2217 }
2218
2219 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, ofs, len)) {
2220 tcg_gen_op4ii_i64(INDEX_op_extract, ret, arg, ofs, len);
2221 return;
2222 }
2223
2224 mask = (1ull << len) - 1;
2225 if (ofs == 0) {
2226 tcg_gen_andi_i64(ret, arg, mask);
2227 return;
2228 }
2229
2230 /* Assume that zero-extension, if available, is cheaper than a shift. */
2231 if (TCG_TARGET_extract_valid(TCG_TYPE_I64, 0, ofs + len)) {
2232 tcg_gen_op4ii_i64(INDEX_op_extract, ret, arg, 0, ofs + len);
2233 tcg_gen_shri_i64(ret, ret, ofs);
2234 return;
2235 }
2236
2237 if (tcg_op_imm_match(INDEX_op_and, TCG_TYPE_I64, mask)) {
2238 tcg_gen_shri_i64(ret, arg, ofs);
2239 tcg_gen_andi_i64(ret, ret, mask);
2240 } else {
2241 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2242 tcg_gen_shri_i64(ret, ret, 64 - len);
2243 }
2244 }
2245
2246 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg,
2247 unsigned int ofs, unsigned int len)
2248 {
2249 tcg_debug_assert(ofs < 64);
2250 tcg_debug_assert(len > 0);
2251 tcg_debug_assert(len <= 64);
2252 tcg_debug_assert(ofs + len <= 64);
2253
2254 /* Canonicalize certain special cases, even if sextract is supported. */
2255 if (ofs + len == 64) {
2256 tcg_gen_sari_i64(ret, arg, 64 - len);
2257 return;
2258 }
2259
2260 if (TCG_TARGET_sextract_valid(TCG_TYPE_I64, ofs, len)) {
2261 tcg_gen_op4ii_i64(INDEX_op_sextract, ret, arg, ofs, len);
2262 return;
2263 }
2264
2265 /* Assume that sign-extension, if available, is cheaper than a shift. */
2266 if (TCG_TARGET_sextract_valid(TCG_TYPE_I64, 0, ofs + len)) {
2267 tcg_gen_op4ii_i64(INDEX_op_sextract, ret, arg, 0, ofs + len);
2268 tcg_gen_sari_i64(ret, ret, ofs);
2269 return;
2270 }
2271 if (TCG_TARGET_sextract_valid(TCG_TYPE_I64, 0, len)) {
2272 tcg_gen_shri_i64(ret, arg, ofs);
2273 tcg_gen_op4ii_i64(INDEX_op_sextract, ret, ret, 0, len);
2274 return;
2275 }
2276
2277 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2278 tcg_gen_sari_i64(ret, ret, 64 - len);
2279 }
2280
2281 /*
2282 * Extract 64 bits from a 128-bit input, ah:al, starting from ofs.
2283 * Unlike tcg_gen_extract_i64 above, len is fixed at 64.
2284 */
2285 void tcg_gen_extract2_i64(TCGv_i64 ret, TCGv_i64 al, TCGv_i64 ah,
2286 unsigned int ofs)
2287 {
2288 tcg_debug_assert(ofs <= 64);
2289 if (ofs == 0) {
2290 tcg_gen_mov_i64(ret, al);
2291 } else if (ofs == 64) {
2292 tcg_gen_mov_i64(ret, ah);
2293 } else if (al == ah) {
2294 tcg_gen_rotri_i64(ret, al, ofs);
2295 } else {
2296 tcg_gen_op4i_i64(INDEX_op_extract2, ret, al, ah, ofs);
2297 }
2298 }
2299
2300 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1,
2301 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2)
2302 {
2303 if (cond == TCG_COND_ALWAYS) {
2304 tcg_gen_mov_i64(ret, v1);
2305 } else if (cond == TCG_COND_NEVER) {
2306 tcg_gen_mov_i64(ret, v2);
2307 } else {
2308 tcg_gen_op6i_i64(INDEX_op_movcond, ret, c1, c2, v1, v2, cond);
2309 }
2310 }
2311
2312 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2313 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2314 {
2315 if (tcg_op_supported(INDEX_op_addci, TCG_TYPE_REG, 0)) {
2316 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2317
2318 tcg_gen_op3_i64(INDEX_op_addco, t0, al, bl);
2319 tcg_gen_op3_i64(INDEX_op_addci, rh, ah, bh);
2320 tcg_gen_mov_i64(rl, t0);
2321 tcg_temp_free_i64(t0);
2322 } else {
2323 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2324 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
2325 tcg_gen_add_i64(t0, al, bl);
2326 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al);
2327 tcg_gen_add_i64(rh, ah, bh);
2328 tcg_gen_add_i64(rh, rh, t1);
2329 tcg_gen_mov_i64(rl, t0);
2330 tcg_temp_free_i64(t0);
2331 tcg_temp_free_i64(t1);
2332 }
2333 }
2334
2335 void tcg_gen_addcio_i64(TCGv_i64 r, TCGv_i64 co,
2336 TCGv_i64 a, TCGv_i64 b, TCGv_i64 ci)
2337 {
2338 if (tcg_op_supported(INDEX_op_addci, TCG_TYPE_I64, 0)) {
2339 TCGv_i64 discard = tcg_temp_ebb_new_i64();
2340 TCGv_i64 zero = tcg_constant_i64(0);
2341 TCGv_i64 mone = tcg_constant_i64(-1);
2342
2343 tcg_gen_op3_i64(INDEX_op_addco, discard, ci, mone);
2344 tcg_gen_op3_i64(INDEX_op_addcio, r, a, b);
2345 tcg_gen_op3_i64(INDEX_op_addci, co, zero, zero);
2346 tcg_temp_free_i64(discard);
2347 } else {
2348 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2349 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
2350
2351 tcg_gen_add_i64(t0, a, b);
2352 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, a);
2353 tcg_gen_add_i64(r, t0, ci);
2354 tcg_gen_setcond_i64(TCG_COND_LTU, t0, r, t0);
2355 tcg_gen_or_i64(co, t0, t1);
2356
2357 tcg_temp_free_i64(t0);
2358 tcg_temp_free_i64(t1);
2359 }
2360 }
2361
2362 void tcg_gen_addN_i64(int n, TCGv_i64 *r, TCGv_i64 *a, TCGv_i64 *b)
2363 {
2364 tcg_debug_assert(n > 2);
2365
2366 /* ??? Don't allow overlap for now. */
2367 for (int i = 0; i < n - 1; ++i) {
2368 for (int j = i + 1; j < n; ++j) {
2369 tcg_debug_assert(r[i] != a[j]);
2370 tcg_debug_assert(r[i] != b[j]);
2371 }
2372 }
2373
2374 if (tcg_op_supported(INDEX_op_addci, TCG_TYPE_I64, 0)) {
2375 tcg_gen_op3_i64(INDEX_op_addco, r[0], a[0], b[0]);
2376 for (int i = 1; i < n - 1; ++i) {
2377 tcg_gen_op3_i64(INDEX_op_addcio, r[i], a[i], b[i]);
2378 }
2379 tcg_gen_op3_i64(INDEX_op_addci, r[n - 1], a[n - 1], b[n - 1]);
2380 } else {
2381 TCGv_i64 t = tcg_temp_ebb_new_i64();
2382 TCGv_i64 c = tcg_temp_ebb_new_i64();
2383
2384 tcg_gen_add_i64(t, a[0], b[0]);
2385 tcg_gen_setcond_i64(TCG_COND_LTU, c, t, a[0]);
2386 tcg_gen_mov_i64(r[0], t);
2387
2388 for (int i = 1; i < n - 1; ++i) {
2389 tcg_gen_add_i64(t, a[i], c);
2390 tcg_gen_setcond_i64(TCG_COND_LTU, c, t, c);
2391 tcg_gen_add_i64(r[i], b[i], t);
2392 tcg_gen_setcond_i64(TCG_COND_LTU, t, r[i], t);
2393 tcg_gen_or_i64(c, c, t);
2394 }
2395
2396 tcg_gen_add_i64(r[n - 1], a[n - 1], b[n - 1]);
2397 tcg_gen_add_i64(r[n - 1], r[n - 1], c);
2398
2399 tcg_temp_free_i64(t);
2400 tcg_temp_free_i64(c);
2401 }
2402 }
2403
2404 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2405 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2406 {
2407 if (tcg_op_supported(INDEX_op_subbi, TCG_TYPE_REG, 0)) {
2408 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2409
2410 tcg_gen_op3_i64(INDEX_op_subbo, t0, al, bl);
2411 tcg_gen_op3_i64(INDEX_op_subbi, rh, ah, bh);
2412 tcg_gen_mov_i64(rl, t0);
2413 tcg_temp_free_i64(t0);
2414 } else {
2415 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2416 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
2417 tcg_gen_sub_i64(t0, al, bl);
2418 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl);
2419 tcg_gen_sub_i64(rh, ah, bh);
2420 tcg_gen_sub_i64(rh, rh, t1);
2421 tcg_gen_mov_i64(rl, t0);
2422 tcg_temp_free_i64(t0);
2423 tcg_temp_free_i64(t1);
2424 }
2425 }
2426
2427 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2428 {
2429 if (tcg_op_supported(INDEX_op_mulu2, TCG_TYPE_I64, 0)) {
2430 tcg_gen_op4_i64(INDEX_op_mulu2, rl, rh, arg1, arg2);
2431 } else if (tcg_op_supported(INDEX_op_muluh, TCG_TYPE_I64, 0)) {
2432 TCGv_i64 t = tcg_temp_ebb_new_i64();
2433 tcg_gen_op3_i64(INDEX_op_mul, t, arg1, arg2);
2434 tcg_gen_op3_i64(INDEX_op_muluh, rh, arg1, arg2);
2435 tcg_gen_mov_i64(rl, t);
2436 tcg_temp_free_i64(t);
2437 } else {
2438 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2439 tcg_gen_mul_i64(t0, arg1, arg2);
2440 gen_helper_muluh_i64(rh, arg1, arg2);
2441 tcg_gen_mov_i64(rl, t0);
2442 tcg_temp_free_i64(t0);
2443 }
2444 }
2445
2446 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2447 {
2448 if (tcg_op_supported(INDEX_op_muls2, TCG_TYPE_I64, 0)) {
2449 tcg_gen_op4_i64(INDEX_op_muls2, rl, rh, arg1, arg2);
2450 } else if (tcg_op_supported(INDEX_op_mulsh, TCG_TYPE_I64, 0)) {
2451 TCGv_i64 t = tcg_temp_ebb_new_i64();
2452 tcg_gen_op3_i64(INDEX_op_mul, t, arg1, arg2);
2453 tcg_gen_op3_i64(INDEX_op_mulsh, rh, arg1, arg2);
2454 tcg_gen_mov_i64(rl, t);
2455 tcg_temp_free_i64(t);
2456 } else if (tcg_op_supported(INDEX_op_mulu2, TCG_TYPE_I64, 0) ||
2457 tcg_op_supported(INDEX_op_muluh, TCG_TYPE_I64, 0)) {
2458 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2459 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
2460 TCGv_i64 t2 = tcg_temp_ebb_new_i64();
2461 TCGv_i64 t3 = tcg_temp_ebb_new_i64();
2462 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2463 /* Adjust for negative inputs. */
2464 tcg_gen_sari_i64(t2, arg1, 63);
2465 tcg_gen_sari_i64(t3, arg2, 63);
2466 tcg_gen_and_i64(t2, t2, arg2);
2467 tcg_gen_and_i64(t3, t3, arg1);
2468 tcg_gen_sub_i64(rh, t1, t2);
2469 tcg_gen_sub_i64(rh, rh, t3);
2470 tcg_gen_mov_i64(rl, t0);
2471 tcg_temp_free_i64(t0);
2472 tcg_temp_free_i64(t1);
2473 tcg_temp_free_i64(t2);
2474 tcg_temp_free_i64(t3);
2475 } else {
2476 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2477 tcg_gen_mul_i64(t0, arg1, arg2);
2478 gen_helper_mulsh_i64(rh, arg1, arg2);
2479 tcg_gen_mov_i64(rl, t0);
2480 tcg_temp_free_i64(t0);
2481 }
2482 }
2483
2484 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2485 {
2486 TCGv_i64 t0 = tcg_temp_ebb_new_i64();
2487 TCGv_i64 t1 = tcg_temp_ebb_new_i64();
2488 TCGv_i64 t2 = tcg_temp_ebb_new_i64();
2489 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2490 /* Adjust for negative input for the signed arg1. */
2491 tcg_gen_sari_i64(t2, arg1, 63);
2492 tcg_gen_and_i64(t2, t2, arg2);
2493 tcg_gen_sub_i64(rh, t1, t2);
2494 tcg_gen_mov_i64(rl, t0);
2495 tcg_temp_free_i64(t0);
2496 tcg_temp_free_i64(t1);
2497 tcg_temp_free_i64(t2);
2498 }
2499
2500 void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2501 {
2502 if (tcg_op_supported(INDEX_op_smin, TCG_TYPE_I64, 0)) {
2503 tcg_gen_op3_i64(INDEX_op_smin, ret, a, b);
2504 } else {
2505 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b);
2506 }
2507 }
2508
2509 void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2510 {
2511 if (tcg_op_supported(INDEX_op_umin, TCG_TYPE_I64, 0)) {
2512 tcg_gen_op3_i64(INDEX_op_umin, ret, a, b);
2513 } else {
2514 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b);
2515 }
2516 }
2517
2518 void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2519 {
2520 if (tcg_op_supported(INDEX_op_smax, TCG_TYPE_I64, 0)) {
2521 tcg_gen_op3_i64(INDEX_op_smax, ret, a, b);
2522 } else {
2523 tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a);
2524 }
2525 }
2526
2527 void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2528 {
2529 if (tcg_op_supported(INDEX_op_umax, TCG_TYPE_I64, 0)) {
2530 tcg_gen_op3_i64(INDEX_op_umax, ret, a, b);
2531 } else {
2532 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a);
2533 }
2534 }
2535
2536 void tcg_gen_ussub_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2537 {
2538 TCGv_i64 t = tcg_temp_ebb_new_i64();
2539 TCGv_i64 z = tcg_constant_i64(0);
2540
2541 tcg_gen_sub_i64(t, a, b);
2542 tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, z, t);
2543 tcg_temp_free_i64(t);
2544 }
2545
2546 void tcg_gen_abs_i64(TCGv_i64 ret, TCGv_i64 a)
2547 {
2548 TCGv_i64 t = tcg_temp_ebb_new_i64();
2549
2550 tcg_gen_sari_i64(t, a, 63);
2551 tcg_gen_xor_i64(ret, a, t);
2552 tcg_gen_sub_i64(ret, ret, t);
2553 tcg_temp_free_i64(t);
2554 }
2555
2556 /* Size changing operations. */
2557
2558 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2559 {
2560 tcg_gen_op2(INDEX_op_extrl_i64_i32, TCG_TYPE_I32,
2561 tcgv_i32_arg(ret), tcgv_i64_arg(arg));
2562 }
2563
2564 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2565 {
2566 tcg_gen_op2(INDEX_op_extrh_i64_i32, TCG_TYPE_I32,
2567 tcgv_i32_arg(ret), tcgv_i64_arg(arg));
2568 }
2569
2570 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2571 {
2572 tcg_gen_op2(INDEX_op_extu_i32_i64, TCG_TYPE_I64,
2573 tcgv_i64_arg(ret), tcgv_i32_arg(arg));
2574 }
2575
2576 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2577 {
2578 tcg_gen_op2(INDEX_op_ext_i32_i64, TCG_TYPE_I64,
2579 tcgv_i64_arg(ret), tcgv_i32_arg(arg));
2580 }
2581
2582 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high)
2583 {
2584 TCGv_i64 tmp = tcg_temp_ebb_new_i64();
2585
2586 /* These extensions are only needed for type correctness.
2587 We may be able to do better given target specific information. */
2588 tcg_gen_extu_i32_i64(tmp, high);
2589 tcg_gen_extu_i32_i64(dest, low);
2590 /* If deposit is available, use it. Otherwise use the extra
2591 knowledge that we have of the zero-extensions above. */
2592 if (TCG_TARGET_deposit_valid(TCG_TYPE_I64, 32, 32)) {
2593 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32);
2594 } else {
2595 tcg_gen_shli_i64(tmp, tmp, 32);
2596 tcg_gen_or_i64(dest, dest, tmp);
2597 }
2598 tcg_temp_free_i64(tmp);
2599 }
2600
2601 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg)
2602 {
2603 tcg_gen_extrl_i64_i32(lo, arg);
2604 tcg_gen_extrh_i64_i32(hi, arg);
2605 }
2606
2607 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg)
2608 {
2609 tcg_gen_ext32u_i64(lo, arg);
2610 tcg_gen_shri_i64(hi, arg, 32);
2611 }
2612
2613 void tcg_gen_concat32_i64(TCGv_i64 ret, TCGv_i64 lo, TCGv_i64 hi)
2614 {
2615 tcg_gen_deposit_i64(ret, lo, hi, 32, 32);
2616 }
2617
2618 void tcg_gen_extr_i128_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i128 arg)
2619 {
2620 tcg_gen_mov_i64(lo, TCGV128_LOW(arg));
2621 tcg_gen_mov_i64(hi, TCGV128_HIGH(arg));
2622 }
2623
2624 void tcg_gen_concat_i64_i128(TCGv_i128 ret, TCGv_i64 lo, TCGv_i64 hi)
2625 {
2626 tcg_gen_mov_i64(TCGV128_LOW(ret), lo);
2627 tcg_gen_mov_i64(TCGV128_HIGH(ret), hi);
2628 }
2629
2630 TCGv_i128 tcg_zero_i128(void)
2631 {
2632 TCGv_i64 zero64 = tcg_constant_i64(0);
2633 TCGv_i128 zero128 = tcg_temp_new_i128();
2634
2635 tcg_gen_concat_i64_i128(zero128, zero64, zero64);
2636
2637 return zero128;
2638 }
2639
2640 void tcg_gen_mov_i128(TCGv_i128 dst, TCGv_i128 src)
2641 {
2642 if (dst != src) {
2643 tcg_gen_mov_i64(TCGV128_LOW(dst), TCGV128_LOW(src));
2644 tcg_gen_mov_i64(TCGV128_HIGH(dst), TCGV128_HIGH(src));
2645 }
2646 }
2647
2648 void tcg_gen_ld_i128(TCGv_i128 ret, TCGv_ptr base, tcg_target_long offset)
2649 {
2650 if (HOST_BIG_ENDIAN) {
2651 tcg_gen_ld_i64(TCGV128_HIGH(ret), base, offset);
2652 tcg_gen_ld_i64(TCGV128_LOW(ret), base, offset + 8);
2653 } else {
2654 tcg_gen_ld_i64(TCGV128_LOW(ret), base, offset);
2655 tcg_gen_ld_i64(TCGV128_HIGH(ret), base, offset + 8);
2656 }
2657 }
2658
2659 void tcg_gen_st_i128(TCGv_i128 val, TCGv_ptr base, tcg_target_long offset)
2660 {
2661 if (HOST_BIG_ENDIAN) {
2662 tcg_gen_st_i64(TCGV128_HIGH(val), base, offset);
2663 tcg_gen_st_i64(TCGV128_LOW(val), base, offset + 8);
2664 } else {
2665 tcg_gen_st_i64(TCGV128_LOW(val), base, offset);
2666 tcg_gen_st_i64(TCGV128_HIGH(val), base, offset + 8);
2667 }
2668 }
2669
2670 /* QEMU specific operations. */
2671
2672 void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx)
2673 {
2674 /*
2675 * Let the jit code return the read-only version of the
2676 * TranslationBlock, so that we minimize the pc-relative
2677 * distance of the address of the exit_tb code to TB.
2678 * This will improve utilization of pc-relative address loads.
2679 *
2680 * TODO: Move this to translator_loop, so that all const
2681 * TranslationBlock pointers refer to read-only memory.
2682 * This requires coordination with targets that do not use
2683 * the translator_loop.
2684 */
2685 uintptr_t val = (uintptr_t)tcg_splitwx_to_rx((void *)tb) + idx;
2686
2687 if (tb == NULL) {
2688 tcg_debug_assert(idx == 0);
2689 } else if (idx <= TB_EXIT_IDXMAX) {
2690 #ifdef CONFIG_DEBUG_TCG
2691 /* This is an exit following a goto_tb. Verify that we have
2692 seen this numbered exit before, via tcg_gen_goto_tb. */
2693 tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx));
2694 #endif
2695 } else {
2696 /* This is an exit via the exitreq label. */
2697 tcg_debug_assert(idx == TB_EXIT_REQUESTED);
2698 }
2699
2700 tcg_gen_op1i(INDEX_op_exit_tb, 0, val);
2701 }
2702
2703 void tcg_gen_goto_tb(unsigned idx)
2704 {
2705 /* We tested CF_NO_GOTO_TB in translator_use_goto_tb. */
2706 tcg_debug_assert(!(tcg_ctx->gen_tb->cflags & CF_NO_GOTO_TB));
2707 /* We only support two chained exits. */
2708 tcg_debug_assert(idx <= TB_EXIT_IDXMAX);
2709 #ifdef CONFIG_DEBUG_TCG
2710 /* Verify that we haven't seen this numbered exit before. */
2711 tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0);
2712 tcg_ctx->goto_tb_issue_mask |= 1 << idx;
2713 #endif
2714 plugin_gen_disable_mem_helpers();
2715 tcg_gen_op1i(INDEX_op_goto_tb, 0, idx);
2716 }
2717
2718 void tcg_gen_lookup_and_goto_ptr(void)
2719 {
2720 TCGv_ptr ptr;
2721
2722 if (tcg_ctx->gen_tb->cflags & CF_NO_GOTO_PTR) {
2723 tcg_gen_exit_tb(NULL, 0);
2724 return;
2725 }
2726
2727 plugin_gen_disable_mem_helpers();
2728 ptr = tcg_temp_ebb_new_ptr();
2729 gen_helper_lookup_tb_ptr(ptr, tcg_env);
2730 tcg_gen_op1i(INDEX_op_goto_ptr, TCG_TYPE_PTR, tcgv_ptr_arg(ptr));
2731 tcg_temp_free_ptr(ptr);
2732 }