vec_helper.c · samitouri/QOSamiQemu

Code Commits Issues Pulls Sessions CI/CD

master

c 3,412 lines 118 KB

Raw

1	/*
2	* ARM AdvSIMD / SVE Vector Operations
3	*
4	* Copyright (c) 2018 Linaro
5	*
6	* This library is free software; you can redistribute it and/or
7	* modify it under the terms of the GNU Lesser General Public
8	* License as published by the Free Software Foundation; either
9	* version 2.1 of the License, or (at your option) any later version.
10	*
11	* This library is distributed in the hope that it will be useful,
12	* but WITHOUT ANY WARRANTY; without even the implied warranty of
13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14	* Lesser General Public License for more details.
15	*
16	* You should have received a copy of the GNU Lesser General Public
17	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
18	*/
19
20	#include "qemu/osdep.h"
21	#include "cpu.h"
22	#include "helper.h"
23	#include "tcg/tcg-gvec-desc.h"
24	#include "fpu/softfloat.h"
25	#include "fpu/softfloat-parts.h"
26	#include "qemu/int128.h"
27	#include "crypto/clmul.h"
28	#include "vec_internal.h"
29
30	/*
31	* Data for expanding active predicate bits to bytes, for byte elements.
32	*
33	* for (i = 0; i < 256; ++i) {
34	* unsigned long m = 0;
35	* for (j = 0; j < 8; j++) {
36	* if ((i >> j) & 1) {
37	* m \|= 0xfful << (j << 3);
38	* }
39	* }
40	* printf("0x%016lx,\n", m);
41	* }
42	*/
43	const uint64_t expand_pred_b_data[256] = {
44	0x0000000000000000, 0x00000000000000ff, 0x000000000000ff00,
45	0x000000000000ffff, 0x0000000000ff0000, 0x0000000000ff00ff,
46	0x0000000000ffff00, 0x0000000000ffffff, 0x00000000ff000000,
47	0x00000000ff0000ff, 0x00000000ff00ff00, 0x00000000ff00ffff,
48	0x00000000ffff0000, 0x00000000ffff00ff, 0x00000000ffffff00,
49	0x00000000ffffffff, 0x000000ff00000000, 0x000000ff000000ff,
50	0x000000ff0000ff00, 0x000000ff0000ffff, 0x000000ff00ff0000,
51	0x000000ff00ff00ff, 0x000000ff00ffff00, 0x000000ff00ffffff,
52	0x000000ffff000000, 0x000000ffff0000ff, 0x000000ffff00ff00,
53	0x000000ffff00ffff, 0x000000ffffff0000, 0x000000ffffff00ff,
54	0x000000ffffffff00, 0x000000ffffffffff, 0x0000ff0000000000,
55	0x0000ff00000000ff, 0x0000ff000000ff00, 0x0000ff000000ffff,
56	0x0000ff0000ff0000, 0x0000ff0000ff00ff, 0x0000ff0000ffff00,
57	0x0000ff0000ffffff, 0x0000ff00ff000000, 0x0000ff00ff0000ff,
58	0x0000ff00ff00ff00, 0x0000ff00ff00ffff, 0x0000ff00ffff0000,
59	0x0000ff00ffff00ff, 0x0000ff00ffffff00, 0x0000ff00ffffffff,
60	0x0000ffff00000000, 0x0000ffff000000ff, 0x0000ffff0000ff00,
61	0x0000ffff0000ffff, 0x0000ffff00ff0000, 0x0000ffff00ff00ff,
62	0x0000ffff00ffff00, 0x0000ffff00ffffff, 0x0000ffffff000000,
63	0x0000ffffff0000ff, 0x0000ffffff00ff00, 0x0000ffffff00ffff,
64	0x0000ffffffff0000, 0x0000ffffffff00ff, 0x0000ffffffffff00,
65	0x0000ffffffffffff, 0x00ff000000000000, 0x00ff0000000000ff,
66	0x00ff00000000ff00, 0x00ff00000000ffff, 0x00ff000000ff0000,
67	0x00ff000000ff00ff, 0x00ff000000ffff00, 0x00ff000000ffffff,
68	0x00ff0000ff000000, 0x00ff0000ff0000ff, 0x00ff0000ff00ff00,
69	0x00ff0000ff00ffff, 0x00ff0000ffff0000, 0x00ff0000ffff00ff,
70	0x00ff0000ffffff00, 0x00ff0000ffffffff, 0x00ff00ff00000000,
71	0x00ff00ff000000ff, 0x00ff00ff0000ff00, 0x00ff00ff0000ffff,
72	0x00ff00ff00ff0000, 0x00ff00ff00ff00ff, 0x00ff00ff00ffff00,
73	0x00ff00ff00ffffff, 0x00ff00ffff000000, 0x00ff00ffff0000ff,
74	0x00ff00ffff00ff00, 0x00ff00ffff00ffff, 0x00ff00ffffff0000,
75	0x00ff00ffffff00ff, 0x00ff00ffffffff00, 0x00ff00ffffffffff,
76	0x00ffff0000000000, 0x00ffff00000000ff, 0x00ffff000000ff00,
77	0x00ffff000000ffff, 0x00ffff0000ff0000, 0x00ffff0000ff00ff,
78	0x00ffff0000ffff00, 0x00ffff0000ffffff, 0x00ffff00ff000000,
79	0x00ffff00ff0000ff, 0x00ffff00ff00ff00, 0x00ffff00ff00ffff,
80	0x00ffff00ffff0000, 0x00ffff00ffff00ff, 0x00ffff00ffffff00,
81	0x00ffff00ffffffff, 0x00ffffff00000000, 0x00ffffff000000ff,
82	0x00ffffff0000ff00, 0x00ffffff0000ffff, 0x00ffffff00ff0000,
83	0x00ffffff00ff00ff, 0x00ffffff00ffff00, 0x00ffffff00ffffff,
84	0x00ffffffff000000, 0x00ffffffff0000ff, 0x00ffffffff00ff00,
85	0x00ffffffff00ffff, 0x00ffffffffff0000, 0x00ffffffffff00ff,
86	0x00ffffffffffff00, 0x00ffffffffffffff, 0xff00000000000000,
87	0xff000000000000ff, 0xff0000000000ff00, 0xff0000000000ffff,
88	0xff00000000ff0000, 0xff00000000ff00ff, 0xff00000000ffff00,
89	0xff00000000ffffff, 0xff000000ff000000, 0xff000000ff0000ff,
90	0xff000000ff00ff00, 0xff000000ff00ffff, 0xff000000ffff0000,
91	0xff000000ffff00ff, 0xff000000ffffff00, 0xff000000ffffffff,
92	0xff0000ff00000000, 0xff0000ff000000ff, 0xff0000ff0000ff00,
93	0xff0000ff0000ffff, 0xff0000ff00ff0000, 0xff0000ff00ff00ff,
94	0xff0000ff00ffff00, 0xff0000ff00ffffff, 0xff0000ffff000000,
95	0xff0000ffff0000ff, 0xff0000ffff00ff00, 0xff0000ffff00ffff,
96	0xff0000ffffff0000, 0xff0000ffffff00ff, 0xff0000ffffffff00,
97	0xff0000ffffffffff, 0xff00ff0000000000, 0xff00ff00000000ff,
98	0xff00ff000000ff00, 0xff00ff000000ffff, 0xff00ff0000ff0000,
99	0xff00ff0000ff00ff, 0xff00ff0000ffff00, 0xff00ff0000ffffff,
100	0xff00ff00ff000000, 0xff00ff00ff0000ff, 0xff00ff00ff00ff00,
101	0xff00ff00ff00ffff, 0xff00ff00ffff0000, 0xff00ff00ffff00ff,
102	0xff00ff00ffffff00, 0xff00ff00ffffffff, 0xff00ffff00000000,
103	0xff00ffff000000ff, 0xff00ffff0000ff00, 0xff00ffff0000ffff,
104	0xff00ffff00ff0000, 0xff00ffff00ff00ff, 0xff00ffff00ffff00,
105	0xff00ffff00ffffff, 0xff00ffffff000000, 0xff00ffffff0000ff,
106	0xff00ffffff00ff00, 0xff00ffffff00ffff, 0xff00ffffffff0000,
107	0xff00ffffffff00ff, 0xff00ffffffffff00, 0xff00ffffffffffff,
108	0xffff000000000000, 0xffff0000000000ff, 0xffff00000000ff00,
109	0xffff00000000ffff, 0xffff000000ff0000, 0xffff000000ff00ff,
110	0xffff000000ffff00, 0xffff000000ffffff, 0xffff0000ff000000,
111	0xffff0000ff0000ff, 0xffff0000ff00ff00, 0xffff0000ff00ffff,
112	0xffff0000ffff0000, 0xffff0000ffff00ff, 0xffff0000ffffff00,
113	0xffff0000ffffffff, 0xffff00ff00000000, 0xffff00ff000000ff,
114	0xffff00ff0000ff00, 0xffff00ff0000ffff, 0xffff00ff00ff0000,
115	0xffff00ff00ff00ff, 0xffff00ff00ffff00, 0xffff00ff00ffffff,
116	0xffff00ffff000000, 0xffff00ffff0000ff, 0xffff00ffff00ff00,
117	0xffff00ffff00ffff, 0xffff00ffffff0000, 0xffff00ffffff00ff,
118	0xffff00ffffffff00, 0xffff00ffffffffff, 0xffffff0000000000,
119	0xffffff00000000ff, 0xffffff000000ff00, 0xffffff000000ffff,
120	0xffffff0000ff0000, 0xffffff0000ff00ff, 0xffffff0000ffff00,
121	0xffffff0000ffffff, 0xffffff00ff000000, 0xffffff00ff0000ff,
122	0xffffff00ff00ff00, 0xffffff00ff00ffff, 0xffffff00ffff0000,
123	0xffffff00ffff00ff, 0xffffff00ffffff00, 0xffffff00ffffffff,
124	0xffffffff00000000, 0xffffffff000000ff, 0xffffffff0000ff00,
125	0xffffffff0000ffff, 0xffffffff00ff0000, 0xffffffff00ff00ff,
126	0xffffffff00ffff00, 0xffffffff00ffffff, 0xffffffffff000000,
127	0xffffffffff0000ff, 0xffffffffff00ff00, 0xffffffffff00ffff,
128	0xffffffffffff0000, 0xffffffffffff00ff, 0xffffffffffffff00,
129	0xffffffffffffffff,
130	};
131
132	/*
133	* Similarly for half-word elements.
134	* for (i = 0; i < 256; ++i) {
135	* unsigned long m = 0;
136	* if (i & 0xaa) {
137	* continue;
138	* }
139	* for (j = 0; j < 8; j += 2) {
140	* if ((i >> j) & 1) {
141	* m \|= 0xfffful << (j << 3);
142	* }
143	* }
144	* printf("[0x%x] = 0x%016lx,\n", i, m);
145	* }
146	*/
147	const uint64_t expand_pred_h_data[0x55 + 1] = {
148	[0x01] = 0x000000000000ffff, [0x04] = 0x00000000ffff0000,
149	[0x05] = 0x00000000ffffffff, [0x10] = 0x0000ffff00000000,
150	[0x11] = 0x0000ffff0000ffff, [0x14] = 0x0000ffffffff0000,
151	[0x15] = 0x0000ffffffffffff, [0x40] = 0xffff000000000000,
152	[0x41] = 0xffff00000000ffff, [0x44] = 0xffff0000ffff0000,
153	[0x45] = 0xffff0000ffffffff, [0x50] = 0xffffffff00000000,
154	[0x51] = 0xffffffff0000ffff, [0x54] = 0xffffffffffff0000,
155	[0x55] = 0xffffffffffffffff,
156	};
157
158	/* Signed saturating rounding doubling multiply-accumulate high half, 8-bit */
159	int8_t do_sqrdmlah_b(int8_t src1, int8_t src2, int8_t src3,
160	bool neg, bool round)
161	{
162	/*
163	* Simplify:
164	* = ((a3 << 8) + ((e1 * e2) << 1) + (round << 7)) >> 8
165	* = ((a3 << 7) + (e1 * e2) + (round << 6)) >> 7
166	*/
167	int32_t ret = (int32_t)src1 * src2;
168	if (neg) {
169	ret = -ret;
170	}
171	ret += ((int32_t)src3 << 7) + (round << 6);
172	ret >>= 7;
173
174	if (ret != (int8_t)ret) {
175	ret = (ret < 0 ? INT8_MIN : INT8_MAX);
176	}
177	return ret;
178	}
179
180	void HELPER(sve2_sqrdmlah_b)(void vd, void vn, void *vm,
181	void *va, uint32_t desc)
182	{
183	intptr_t i, opr_sz = simd_oprsz(desc);
184	int8_t d = vd, n = vn, m = vm, a = va;
185
186	for (i = 0; i < opr_sz; ++i) {
187	d[i] = do_sqrdmlah_b(n[i], m[i], a[i], false, true);
188	}
189	}
190
191	void HELPER(sve2_sqrdmlsh_b)(void vd, void vn, void *vm,
192	void *va, uint32_t desc)
193	{
194	intptr_t i, opr_sz = simd_oprsz(desc);
195	int8_t d = vd, n = vn, m = vm, a = va;
196
197	for (i = 0; i < opr_sz; ++i) {
198	d[i] = do_sqrdmlah_b(n[i], m[i], a[i], true, true);
199	}
200	}
201
202	void HELPER(sve2_sqdmulh_b)(void vd, void vn, void *vm, uint32_t desc)
203	{
204	intptr_t i, opr_sz = simd_oprsz(desc);
205	int8_t d = vd, n = vn, *m = vm;
206
207	for (i = 0; i < opr_sz; ++i) {
208	d[i] = do_sqrdmlah_b(n[i], m[i], 0, false, false);
209	}
210	}
211
212	void HELPER(sve2_sqrdmulh_b)(void vd, void vn, void *vm, uint32_t desc)
213	{
214	intptr_t i, opr_sz = simd_oprsz(desc);
215	int8_t d = vd, n = vn, *m = vm;
216
217	for (i = 0; i < opr_sz; ++i) {
218	d[i] = do_sqrdmlah_b(n[i], m[i], 0, false, true);
219	}
220	}
221
222	/* Signed saturating rounding doubling multiply-accumulate high half, 16-bit */
223	int16_t do_sqrdmlah_h(int16_t src1, int16_t src2, int16_t src3,
224	bool neg, bool round, uint32_t *sat)
225	{
226	/* Simplify similarly to do_sqrdmlah_b above. */
227	int32_t ret = (int32_t)src1 * src2;
228	if (neg) {
229	ret = -ret;
230	}
231	ret += ((int32_t)src3 << 15) + (round << 14);
232	ret >>= 15;
233
234	if (ret != (int16_t)ret) {
235	*sat = 1;
236	ret = (ret < 0 ? INT16_MIN : INT16_MAX);
237	}
238	return ret;
239	}
240
241	uint32_t HELPER(neon_qrdmlah_s16)(CPUARMState *env, uint32_t src1,
242	uint32_t src2, uint32_t src3)
243	{
244	uint32_t *sat = &env->vfp.qc[0];
245	uint16_t e1 = do_sqrdmlah_h(src1, src2, src3, false, true, sat);
246	uint16_t e2 = do_sqrdmlah_h(src1 >> 16, src2 >> 16, src3 >> 16,
247	false, true, sat);
248	return deposit32(e1, 16, 16, e2);
249	}
250
251	void HELPER(gvec_qrdmlah_s16)(void vd, void vn, void *vm,
252	void *vq, uint32_t desc)
253	{
254	uintptr_t opr_sz = simd_oprsz(desc);
255	int16_t *d = vd;
256	int16_t *n = vn;
257	int16_t *m = vm;
258	uintptr_t i;
259
260	for (i = 0; i < opr_sz / 2; ++i) {
261	d[i] = do_sqrdmlah_h(n[i], m[i], d[i], false, true, vq);
262	}
263	clear_tail(d, opr_sz, simd_maxsz(desc));
264	}
265
266	uint32_t HELPER(neon_qrdmlsh_s16)(CPUARMState *env, uint32_t src1,
267	uint32_t src2, uint32_t src3)
268	{
269	uint32_t *sat = &env->vfp.qc[0];
270	uint16_t e1 = do_sqrdmlah_h(src1, src2, src3, true, true, sat);
271	uint16_t e2 = do_sqrdmlah_h(src1 >> 16, src2 >> 16, src3 >> 16,
272	true, true, sat);
273	return deposit32(e1, 16, 16, e2);
274	}
275
276	void HELPER(gvec_qrdmlsh_s16)(void vd, void vn, void *vm,
277	void *vq, uint32_t desc)
278	{
279	uintptr_t opr_sz = simd_oprsz(desc);
280	int16_t *d = vd;
281	int16_t *n = vn;
282	int16_t *m = vm;
283	uintptr_t i;
284
285	for (i = 0; i < opr_sz / 2; ++i) {
286	d[i] = do_sqrdmlah_h(n[i], m[i], d[i], true, true, vq);
287	}
288	clear_tail(d, opr_sz, simd_maxsz(desc));
289	}
290
291	void HELPER(neon_sqdmulh_h)(void vd, void vn, void *vm,
292	void *vq, uint32_t desc)
293	{
294	intptr_t i, opr_sz = simd_oprsz(desc);
295	int16_t d = vd, n = vn, *m = vm;
296
297	for (i = 0; i < opr_sz / 2; ++i) {
298	d[i] = do_sqrdmlah_h(n[i], m[i], 0, false, false, vq);
299	}
300	clear_tail(d, opr_sz, simd_maxsz(desc));
301	}
302
303	void HELPER(neon_sqrdmulh_h)(void vd, void vn, void *vm,
304	void *vq, uint32_t desc)
305	{
306	intptr_t i, opr_sz = simd_oprsz(desc);
307	int16_t d = vd, n = vn, *m = vm;
308
309	for (i = 0; i < opr_sz / 2; ++i) {
310	d[i] = do_sqrdmlah_h(n[i], m[i], 0, false, true, vq);
311	}
312	clear_tail(d, opr_sz, simd_maxsz(desc));
313	}
314
315	void HELPER(neon_sqdmulh_idx_h)(void vd, void vn, void *vm,
316	void *vq, uint32_t desc)
317	{
318	intptr_t i, j, opr_sz = simd_oprsz(desc);
319	int idx = simd_data(desc);
320	int16_t d = vd, n = vn, m = (int16_t )vm + H2(idx);
321	intptr_t elements = opr_sz / 2;
322	intptr_t eltspersegment = MIN(16 / 2, elements);
323
324	for (i = 0; i < elements; i += 16 / 2) {
325	int16_t mm = m[i];
326	for (j = 0; j < eltspersegment; ++j) {
327	d[i + j] = do_sqrdmlah_h(n[i + j], mm, 0, false, false, vq);
328	}
329	}
330	clear_tail(d, opr_sz, simd_maxsz(desc));
331	}
332
333	void HELPER(neon_sqrdmulh_idx_h)(void vd, void vn, void *vm,
334	void *vq, uint32_t desc)
335	{
336	intptr_t i, j, opr_sz = simd_oprsz(desc);
337	int idx = simd_data(desc);
338	int16_t d = vd, n = vn, m = (int16_t )vm + H2(idx);
339	intptr_t elements = opr_sz / 2;
340	intptr_t eltspersegment = MIN(16 / 2, elements);
341
342	for (i = 0; i < elements; i += 16 / 2) {
343	int16_t mm = m[i];
344	for (j = 0; j < eltspersegment; ++j) {
345	d[i + j] = do_sqrdmlah_h(n[i + j], mm, 0, false, true, vq);
346	}
347	}
348	clear_tail(d, opr_sz, simd_maxsz(desc));
349	}
350
351	void HELPER(neon_sqrdmlah_idx_h)(void vd, void vn, void *vm,
352	void *vq, uint32_t desc)
353	{
354	intptr_t i, j, opr_sz = simd_oprsz(desc);
355	int idx = simd_data(desc);
356	int16_t d = vd, n = vn, m = (int16_t )vm + H2(idx);
357	intptr_t elements = opr_sz / 2;
358	intptr_t eltspersegment = MIN(16 / 2, elements);
359
360	for (i = 0; i < elements; i += 16 / 2) {
361	int16_t mm = m[i];
362	for (j = 0; j < eltspersegment; ++j) {
363	d[i + j] = do_sqrdmlah_h(n[i + j], mm, d[i + j], false, true, vq);
364	}
365	}
366	clear_tail(d, opr_sz, simd_maxsz(desc));
367	}
368
369	void HELPER(neon_sqrdmlsh_idx_h)(void vd, void vn, void *vm,
370	void *vq, uint32_t desc)
371	{
372	intptr_t i, j, opr_sz = simd_oprsz(desc);
373	int idx = simd_data(desc);
374	int16_t d = vd, n = vn, m = (int16_t )vm + H2(idx);
375	intptr_t elements = opr_sz / 2;
376	intptr_t eltspersegment = MIN(16 / 2, elements);
377
378	for (i = 0; i < elements; i += 16 / 2) {
379	int16_t mm = m[i];
380	for (j = 0; j < eltspersegment; ++j) {
381	d[i + j] = do_sqrdmlah_h(n[i + j], mm, d[i + j], true, true, vq);
382	}
383	}
384	clear_tail(d, opr_sz, simd_maxsz(desc));
385	}
386
387	void HELPER(sve2_sqrdmlah_h)(void vd, void vn, void *vm,
388	void *va, uint32_t desc)
389	{
390	intptr_t i, opr_sz = simd_oprsz(desc);
391	int16_t d = vd, n = vn, m = vm, a = va;
392	uint32_t discard;
393
394	for (i = 0; i < opr_sz / 2; ++i) {
395	d[i] = do_sqrdmlah_h(n[i], m[i], a[i], false, true, &discard);
396	}
397	}
398
399	void HELPER(sve2_sqrdmlsh_h)(void vd, void vn, void *vm,
400	void *va, uint32_t desc)
401	{
402	intptr_t i, opr_sz = simd_oprsz(desc);
403	int16_t d = vd, n = vn, m = vm, a = va;
404	uint32_t discard;
405
406	for (i = 0; i < opr_sz / 2; ++i) {
407	d[i] = do_sqrdmlah_h(n[i], m[i], a[i], true, true, &discard);
408	}
409	}
410
411	void HELPER(sve2_sqdmulh_h)(void vd, void vn, void *vm, uint32_t desc)
412	{
413	intptr_t i, opr_sz = simd_oprsz(desc);
414	int16_t d = vd, n = vn, *m = vm;
415	uint32_t discard;
416
417	for (i = 0; i < opr_sz / 2; ++i) {
418	d[i] = do_sqrdmlah_h(n[i], m[i], 0, false, false, &discard);
419	}
420	}
421
422	void HELPER(sve2_sqrdmulh_h)(void vd, void vn, void *vm, uint32_t desc)
423	{
424	intptr_t i, opr_sz = simd_oprsz(desc);
425	int16_t d = vd, n = vn, *m = vm;
426	uint32_t discard;
427
428	for (i = 0; i < opr_sz / 2; ++i) {
429	d[i] = do_sqrdmlah_h(n[i], m[i], 0, false, true, &discard);
430	}
431	}
432
433	void HELPER(sve2_sqdmulh_idx_h)(void vd, void vn, void *vm, uint32_t desc)
434	{
435	intptr_t i, j, opr_sz = simd_oprsz(desc);
436	int idx = simd_data(desc);
437	int16_t d = vd, n = vn, m = (int16_t )vm + H2(idx);
438	uint32_t discard;
439
440	for (i = 0; i < opr_sz / 2; i += 16 / 2) {
441	int16_t mm = m[i];
442	for (j = 0; j < 16 / 2; ++j) {
443	d[i + j] = do_sqrdmlah_h(n[i + j], mm, 0, false, false, &discard);
444	}
445	}
446	}
447
448	void HELPER(sve2_sqrdmulh_idx_h)(void vd, void vn, void *vm, uint32_t desc)
449	{
450	intptr_t i, j, opr_sz = simd_oprsz(desc);
451	int idx = simd_data(desc);
452	int16_t d = vd, n = vn, m = (int16_t )vm + H2(idx);
453	uint32_t discard;
454
455	for (i = 0; i < opr_sz / 2; i += 16 / 2) {
456	int16_t mm = m[i];
457	for (j = 0; j < 16 / 2; ++j) {
458	d[i + j] = do_sqrdmlah_h(n[i + j], mm, 0, false, true, &discard);
459	}
460	}
461	}
462
463	/* Signed saturating rounding doubling multiply-accumulate high half, 32-bit */
464	int32_t do_sqrdmlah_s(int32_t src1, int32_t src2, int32_t src3,
465	bool neg, bool round, uint32_t *sat)
466	{
467	/* Simplify similarly to do_sqrdmlah_b above. */
468	int64_t ret = (int64_t)src1 * src2;
469	if (neg) {
470	ret = -ret;
471	}
472	ret += ((int64_t)src3 << 31) + (round << 30);
473	ret >>= 31;
474
475	if (ret != (int32_t)ret) {
476	*sat = 1;
477	ret = (ret < 0 ? INT32_MIN : INT32_MAX);
478	}
479	return ret;
480	}
481
482	uint32_t HELPER(neon_qrdmlah_s32)(CPUARMState *env, int32_t src1,
483	int32_t src2, int32_t src3)
484	{
485	uint32_t *sat = &env->vfp.qc[0];
486	return do_sqrdmlah_s(src1, src2, src3, false, true, sat);
487	}
488
489	void HELPER(gvec_qrdmlah_s32)(void vd, void vn, void *vm,
490	void *vq, uint32_t desc)
491	{
492	uintptr_t opr_sz = simd_oprsz(desc);
493	int32_t *d = vd;
494	int32_t *n = vn;
495	int32_t *m = vm;
496	uintptr_t i;
497
498	for (i = 0; i < opr_sz / 4; ++i) {
499	d[i] = do_sqrdmlah_s(n[i], m[i], d[i], false, true, vq);
500	}
501	clear_tail(d, opr_sz, simd_maxsz(desc));
502	}
503
504	uint32_t HELPER(neon_qrdmlsh_s32)(CPUARMState *env, int32_t src1,
505	int32_t src2, int32_t src3)
506	{
507	uint32_t *sat = &env->vfp.qc[0];
508	return do_sqrdmlah_s(src1, src2, src3, true, true, sat);
509	}
510
511	void HELPER(gvec_qrdmlsh_s32)(void vd, void vn, void *vm,
512	void *vq, uint32_t desc)
513	{
514	uintptr_t opr_sz = simd_oprsz(desc);
515	int32_t *d = vd;
516	int32_t *n = vn;
517	int32_t *m = vm;
518	uintptr_t i;
519
520	for (i = 0; i < opr_sz / 4; ++i) {
521	d[i] = do_sqrdmlah_s(n[i], m[i], d[i], true, true, vq);
522	}
523	clear_tail(d, opr_sz, simd_maxsz(desc));
524	}
525
526	void HELPER(neon_sqdmulh_s)(void vd, void vn, void *vm,
527	void *vq, uint32_t desc)
528	{
529	intptr_t i, opr_sz = simd_oprsz(desc);
530	int32_t d = vd, n = vn, *m = vm;
531
532	for (i = 0; i < opr_sz / 4; ++i) {
533	d[i] = do_sqrdmlah_s(n[i], m[i], 0, false, false, vq);
534	}
535	clear_tail(d, opr_sz, simd_maxsz(desc));
536	}
537
538	void HELPER(neon_sqrdmulh_s)(void vd, void vn, void *vm,
539	void *vq, uint32_t desc)
540	{
541	intptr_t i, opr_sz = simd_oprsz(desc);
542	int32_t d = vd, n = vn, *m = vm;
543
544	for (i = 0; i < opr_sz / 4; ++i) {
545	d[i] = do_sqrdmlah_s(n[i], m[i], 0, false, true, vq);
546	}
547	clear_tail(d, opr_sz, simd_maxsz(desc));
548	}
549
550	void HELPER(neon_sqdmulh_idx_s)(void vd, void vn, void *vm,
551	void *vq, uint32_t desc)
552	{
553	intptr_t i, j, opr_sz = simd_oprsz(desc);
554	int idx = simd_data(desc);
555	int32_t d = vd, n = vn, m = (int32_t )vm + H4(idx);
556	intptr_t elements = opr_sz / 4;
557	intptr_t eltspersegment = MIN(16 / 4, elements);
558
559	for (i = 0; i < elements; i += 16 / 4) {
560	int32_t mm = m[i];
561	for (j = 0; j < eltspersegment; ++j) {
562	d[i + j] = do_sqrdmlah_s(n[i + j], mm, 0, false, false, vq);
563	}
564	}
565	clear_tail(d, opr_sz, simd_maxsz(desc));
566	}
567
568	void HELPER(neon_sqrdmulh_idx_s)(void vd, void vn, void *vm,
569	void *vq, uint32_t desc)
570	{
571	intptr_t i, j, opr_sz = simd_oprsz(desc);
572	int idx = simd_data(desc);
573	int32_t d = vd, n = vn, m = (int32_t )vm + H4(idx);
574	intptr_t elements = opr_sz / 4;
575	intptr_t eltspersegment = MIN(16 / 4, elements);
576
577	for (i = 0; i < elements; i += 16 / 4) {
578	int32_t mm = m[i];
579	for (j = 0; j < eltspersegment; ++j) {
580	d[i + j] = do_sqrdmlah_s(n[i + j], mm, 0, false, true, vq);
581	}
582	}
583	clear_tail(d, opr_sz, simd_maxsz(desc));
584	}
585
586	void HELPER(neon_sqrdmlah_idx_s)(void vd, void vn, void *vm,
587	void *vq, uint32_t desc)
588	{
589	intptr_t i, j, opr_sz = simd_oprsz(desc);
590	int idx = simd_data(desc);
591	int32_t d = vd, n = vn, m = (int32_t )vm + H4(idx);
592	intptr_t elements = opr_sz / 4;
593	intptr_t eltspersegment = MIN(16 / 4, elements);
594
595	for (i = 0; i < elements; i += 16 / 4) {
596	int32_t mm = m[i];
597	for (j = 0; j < eltspersegment; ++j) {
598	d[i + j] = do_sqrdmlah_s(n[i + j], mm, d[i + j], false, true, vq);
599	}
600	}
601	clear_tail(d, opr_sz, simd_maxsz(desc));
602	}
603
604	void HELPER(neon_sqrdmlsh_idx_s)(void vd, void vn, void *vm,
605	void *vq, uint32_t desc)
606	{
607	intptr_t i, j, opr_sz = simd_oprsz(desc);
608	int idx = simd_data(desc);
609	int32_t d = vd, n = vn, m = (int32_t )vm + H4(idx);
610	intptr_t elements = opr_sz / 4;
611	intptr_t eltspersegment = MIN(16 / 4, elements);
612
613	for (i = 0; i < elements; i += 16 / 4) {
614	int32_t mm = m[i];
615	for (j = 0; j < eltspersegment; ++j) {
616	d[i + j] = do_sqrdmlah_s(n[i + j], mm, d[i + j], true, true, vq);
617	}
618	}
619	clear_tail(d, opr_sz, simd_maxsz(desc));
620	}
621
622	void HELPER(sve2_sqrdmlah_s)(void vd, void vn, void *vm,
623	void *va, uint32_t desc)
624	{
625	intptr_t i, opr_sz = simd_oprsz(desc);
626	int32_t d = vd, n = vn, m = vm, a = va;
627	uint32_t discard;
628
629	for (i = 0; i < opr_sz / 4; ++i) {
630	d[i] = do_sqrdmlah_s(n[i], m[i], a[i], false, true, &discard);
631	}
632	}
633
634	void HELPER(sve2_sqrdmlsh_s)(void vd, void vn, void *vm,
635	void *va, uint32_t desc)
636	{
637	intptr_t i, opr_sz = simd_oprsz(desc);
638	int32_t d = vd, n = vn, m = vm, a = va;
639	uint32_t discard;
640
641	for (i = 0; i < opr_sz / 4; ++i) {
642	d[i] = do_sqrdmlah_s(n[i], m[i], a[i], true, true, &discard);
643	}
644	}
645
646	void HELPER(sve2_sqdmulh_s)(void vd, void vn, void *vm, uint32_t desc)
647	{
648	intptr_t i, opr_sz = simd_oprsz(desc);
649	int32_t d = vd, n = vn, *m = vm;
650	uint32_t discard;
651
652	for (i = 0; i < opr_sz / 4; ++i) {
653	d[i] = do_sqrdmlah_s(n[i], m[i], 0, false, false, &discard);
654	}
655	}
656
657	void HELPER(sve2_sqrdmulh_s)(void vd, void vn, void *vm, uint32_t desc)
658	{
659	intptr_t i, opr_sz = simd_oprsz(desc);
660	int32_t d = vd, n = vn, *m = vm;
661	uint32_t discard;
662
663	for (i = 0; i < opr_sz / 4; ++i) {
664	d[i] = do_sqrdmlah_s(n[i], m[i], 0, false, true, &discard);
665	}
666	}
667
668	void HELPER(sve2_sqdmulh_idx_s)(void vd, void vn, void *vm, uint32_t desc)
669	{
670	intptr_t i, j, opr_sz = simd_oprsz(desc);
671	int idx = simd_data(desc);
672	int32_t d = vd, n = vn, m = (int32_t )vm + H4(idx);
673	uint32_t discard;
674
675	for (i = 0; i < opr_sz / 4; i += 16 / 4) {
676	int32_t mm = m[i];
677	for (j = 0; j < 16 / 4; ++j) {
678	d[i + j] = do_sqrdmlah_s(n[i + j], mm, 0, false, false, &discard);
679	}
680	}
681	}
682
683	void HELPER(sve2_sqrdmulh_idx_s)(void vd, void vn, void *vm, uint32_t desc)
684	{
685	intptr_t i, j, opr_sz = simd_oprsz(desc);
686	int idx = simd_data(desc);
687	int32_t d = vd, n = vn, m = (int32_t )vm + H4(idx);
688	uint32_t discard;
689
690	for (i = 0; i < opr_sz / 4; i += 16 / 4) {
691	int32_t mm = m[i];
692	for (j = 0; j < 16 / 4; ++j) {
693	d[i + j] = do_sqrdmlah_s(n[i + j], mm, 0, false, true, &discard);
694	}
695	}
696	}
697
698	/* Signed saturating rounding doubling multiply-accumulate high half, 64-bit */
699	static int64_t do_sat128_d(Int128 r)
700	{
701	int64_t ls = int128_getlo(r);
702	int64_t hs = int128_gethi(r);
703
704	if (unlikely(hs != (ls >> 63))) {
705	return hs < 0 ? INT64_MIN : INT64_MAX;
706	}
707	return ls;
708	}
709
710	int64_t do_sqrdmlah_d(int64_t n, int64_t m, int64_t a, bool neg, bool round)
711	{
712	uint64_t l, h;
713	Int128 r, t;
714
715	/* As in do_sqrdmlah_b, but with 128-bit arithmetic. */
716	muls64(&l, &h, m, n);
717	r = int128_make128(l, h);
718	if (neg) {
719	r = int128_neg(r);
720	}
721	if (a) {
722	t = int128_exts64(a);
723	t = int128_lshift(t, 63);
724	r = int128_add(r, t);
725	}
726	if (round) {
727	t = int128_exts64(1ll << 62);
728	r = int128_add(r, t);
729	}
730	r = int128_rshift(r, 63);
731
732	return do_sat128_d(r);
733	}
734
735	void HELPER(sve2_sqrdmlah_d)(void vd, void vn, void *vm,
736	void *va, uint32_t desc)
737	{
738	intptr_t i, opr_sz = simd_oprsz(desc);
739	int64_t d = vd, n = vn, m = vm, a = va;
740
741	for (i = 0; i < opr_sz / 8; ++i) {
742	d[i] = do_sqrdmlah_d(n[i], m[i], a[i], false, true);
743	}
744	}
745
746	void HELPER(sve2_sqrdmlsh_d)(void vd, void vn, void *vm,
747	void *va, uint32_t desc)
748	{
749	intptr_t i, opr_sz = simd_oprsz(desc);
750	int64_t d = vd, n = vn, m = vm, a = va;
751
752	for (i = 0; i < opr_sz / 8; ++i) {
753	d[i] = do_sqrdmlah_d(n[i], m[i], a[i], true, true);
754	}
755	}
756
757	void HELPER(sve2_sqdmulh_d)(void vd, void vn, void *vm, uint32_t desc)
758	{
759	intptr_t i, opr_sz = simd_oprsz(desc);
760	int64_t d = vd, n = vn, *m = vm;
761
762	for (i = 0; i < opr_sz / 8; ++i) {
763	d[i] = do_sqrdmlah_d(n[i], m[i], 0, false, false);
764	}
765	}
766
767	void HELPER(sve2_sqrdmulh_d)(void vd, void vn, void *vm, uint32_t desc)
768	{
769	intptr_t i, opr_sz = simd_oprsz(desc);
770	int64_t d = vd, n = vn, *m = vm;
771
772	for (i = 0; i < opr_sz / 8; ++i) {
773	d[i] = do_sqrdmlah_d(n[i], m[i], 0, false, true);
774	}
775	}
776
777	void HELPER(sve2_sqdmulh_idx_d)(void vd, void vn, void *vm, uint32_t desc)
778	{
779	intptr_t i, j, opr_sz = simd_oprsz(desc);
780	int idx = simd_data(desc);
781	int64_t d = vd, n = vn, m = (int64_t )vm + idx;
782
783	for (i = 0; i < opr_sz / 8; i += 16 / 8) {
784	int64_t mm = m[i];
785	for (j = 0; j < 16 / 8; ++j) {
786	d[i + j] = do_sqrdmlah_d(n[i + j], mm, 0, false, false);
787	}
788	}
789	}
790
791	void HELPER(sve2_sqrdmulh_idx_d)(void vd, void vn, void *vm, uint32_t desc)
792	{
793	intptr_t i, j, opr_sz = simd_oprsz(desc);
794	int idx = simd_data(desc);
795	int64_t d = vd, n = vn, m = (int64_t )vm + idx;
796
797	for (i = 0; i < opr_sz / 8; i += 16 / 8) {
798	int64_t mm = m[i];
799	for (j = 0; j < 16 / 8; ++j) {
800	d[i + j] = do_sqrdmlah_d(n[i + j], mm, 0, false, true);
801	}
802	}
803	}
804
805	/* Integer 8 and 16-bit dot-product.
806	*
807	* Note that for the loops herein, host endianness does not matter
808	* with respect to the ordering of data within the quad-width lanes.
809	* All elements are treated equally, no matter where they are.
810	*/
811
812	#define DO_DOT(NAME, TYPED, TYPEN, TYPEM) \
813	void HELPER(NAME)(void vd, void vn, void vm, void va, uint32_t desc) \
814	{ \
815	intptr_t i, opr_sz = simd_oprsz(desc); \
816	TYPED d = vd, a = va; \
817	TYPEN *n = vn; \
818	TYPEM *m = vm; \
819	for (i = 0; i < opr_sz / sizeof(TYPED); ++i) { \
820	d[i] = (a[i] + \
821	(TYPED)n[i * 4 + 0] * m[i * 4 + 0] + \
822	(TYPED)n[i * 4 + 1] * m[i * 4 + 1] + \
823	(TYPED)n[i * 4 + 2] * m[i * 4 + 2] + \
824	(TYPED)n[i * 4 + 3] * m[i * 4 + 3]); \
825	} \
826	clear_tail(d, opr_sz, simd_maxsz(desc)); \
827	}
828
829	DO_DOT(gvec_sdot_4b, int32_t, int8_t, int8_t)
830	DO_DOT(gvec_udot_4b, uint32_t, uint8_t, uint8_t)
831	DO_DOT(gvec_usdot_4b, uint32_t, uint8_t, int8_t)
832	DO_DOT(gvec_sdot_4h, int64_t, int16_t, int16_t)
833	DO_DOT(gvec_udot_4h, uint64_t, uint16_t, uint16_t)
834
835	#define DO_DOT_IDX(NAME, TYPED, TYPEN, TYPEM, HD) \
836	void HELPER(NAME)(void vd, void vn, void vm, void va, uint32_t desc) \
837	{ \
838	intptr_t i = 0, opr_sz = simd_oprsz(desc); \
839	intptr_t opr_sz_n = opr_sz / sizeof(TYPED); \
840	/* \
841	* Special case: opr_sz == 8 from AA64/AA32 advsimd means the \
842	* first iteration might not be a full 16 byte segment. But \
843	* for vector lengths beyond that this must be SVE and we know \
844	* opr_sz is a multiple of 16, so we need not clamp segend \
845	* to opr_sz_n when we advance it at the end of the loop. \
846	*/ \
847	intptr_t segend = MIN(16 / sizeof(TYPED), opr_sz_n); \
848	intptr_t index = simd_data(desc); \
849	TYPED d = vd, a = va; \
850	TYPEN *n = vn; \
851	TYPEM m_indexed = (TYPEM )vm + HD(index) * 4; \
852	do { \
853	TYPED m0 = m_indexed[i * 4 + 0]; \
854	TYPED m1 = m_indexed[i * 4 + 1]; \
855	TYPED m2 = m_indexed[i * 4 + 2]; \
856	TYPED m3 = m_indexed[i * 4 + 3]; \
857	do { \
858	d[i] = (a[i] + \
859	n[i * 4 + 0] * m0 + \
860	n[i * 4 + 1] * m1 + \
861	n[i * 4 + 2] * m2 + \
862	n[i * 4 + 3] * m3); \
863	} while (++i < segend); \
864	segend = i + (16 / sizeof(TYPED)); \
865	} while (i < opr_sz_n); \
866	clear_tail(d, opr_sz, simd_maxsz(desc)); \
867	}
868
869	DO_DOT_IDX(gvec_sdot_idx_4b, int32_t, int8_t, int8_t, H4)
870	DO_DOT_IDX(gvec_udot_idx_4b, uint32_t, uint8_t, uint8_t, H4)
871	DO_DOT_IDX(gvec_sudot_idx_4b, int32_t, int8_t, uint8_t, H4)
872	DO_DOT_IDX(gvec_usdot_idx_4b, int32_t, uint8_t, int8_t, H4)
873	DO_DOT_IDX(gvec_sdot_idx_4h, int64_t, int16_t, int16_t, H8)
874	DO_DOT_IDX(gvec_udot_idx_4h, uint64_t, uint16_t, uint16_t, H8)
875
876	#undef DO_DOT
877	#undef DO_DOT_IDX
878
879	/* Similar for 2-way dot product */
880	#define DO_DOT(NAME, TYPED, TYPEN, TYPEM) \
881	void HELPER(NAME)(void vd, void vn, void vm, void va, uint32_t desc) \
882	{ \
883	intptr_t i, opr_sz = simd_oprsz(desc); \
884	TYPED d = vd, a = va; \
885	TYPEN *n = vn; \
886	TYPEM *m = vm; \
887	for (i = 0; i < opr_sz / sizeof(TYPED); ++i) { \
888	d[i] = (a[i] + \
889	(TYPED)n[i * 2 + 0] * m[i * 2 + 0] + \
890	(TYPED)n[i * 2 + 1] * m[i * 2 + 1]); \
891	} \
892	clear_tail(d, opr_sz, simd_maxsz(desc)); \
893	}
894
895	#define DO_DOT_IDX(NAME, TYPED, TYPEN, TYPEM, HD) \
896	void HELPER(NAME)(void vd, void vn, void vm, void va, uint32_t desc) \
897	{ \
898	intptr_t i = 0, opr_sz = simd_oprsz(desc); \
899	intptr_t opr_sz_n = opr_sz / sizeof(TYPED); \
900	intptr_t segend = MIN(16 / sizeof(TYPED), opr_sz_n); \
901	intptr_t index = simd_data(desc); \
902	TYPED d = vd, a = va; \
903	TYPEN *n = vn; \
904	TYPEM m_indexed = (TYPEM )vm + HD(index) * 2; \
905	do { \
906	TYPED m0 = m_indexed[i * 2 + 0]; \
907	TYPED m1 = m_indexed[i * 2 + 1]; \
908	do { \
909	d[i] = (a[i] + \
910	n[i * 2 + 0] * m0 + \
911	n[i * 2 + 1] * m1); \
912	} while (++i < segend); \
913	segend = i + (16 / sizeof(TYPED)); \
914	} while (i < opr_sz_n); \
915	clear_tail(d, opr_sz, simd_maxsz(desc)); \
916	}
917
918	DO_DOT(gvec_sdot_2h, int32_t, int16_t, int16_t)
919	DO_DOT(gvec_udot_2h, uint32_t, uint16_t, uint16_t)
920
921	DO_DOT_IDX(gvec_sdot_idx_2h, int32_t, int16_t, int16_t, H4)
922	DO_DOT_IDX(gvec_udot_idx_2h, uint32_t, uint16_t, uint16_t, H4)
923
924	#undef DO_DOT
925	#undef DO_DOT_IDX
926
927	void HELPER(gvec_fcaddh)(void vd, void vn, void *vm,
928	float_status *fpst, uint32_t desc)
929	{
930	uintptr_t opr_sz = simd_oprsz(desc);
931	float16 *d = vd;
932	float16 *n = vn;
933	float16 *m = vm;
934	bool rot = extract32(desc, SIMD_DATA_SHIFT, 1);
935	bool fpcr_ah = extract64(desc, SIMD_DATA_SHIFT + 1, 1);
936	uintptr_t i;
937
938	for (i = 0; i < opr_sz / 2; i += 2) {
939	float16 e0 = n[H2(i)];
940	float16 e1 = m[H2(i + 1)];
941	float16 e2 = n[H2(i + 1)];
942	float16 e3 = m[H2(i)];
943
944	if (rot) {
945	e3 = float16_maybe_ah_chs(e3, fpcr_ah);
946	} else {
947	e1 = float16_maybe_ah_chs(e1, fpcr_ah);
948	}
949
950	d[H2(i)] = float16_add(e0, e1, fpst);
951	d[H2(i + 1)] = float16_add(e2, e3, fpst);
952	}
953	clear_tail(d, opr_sz, simd_maxsz(desc));
954	}
955
956	void HELPER(gvec_fcadds)(void vd, void vn, void *vm,
957	float_status *fpst, uint32_t desc)
958	{
959	uintptr_t opr_sz = simd_oprsz(desc);
960	float32 *d = vd;
961	float32 *n = vn;
962	float32 *m = vm;
963	bool rot = extract32(desc, SIMD_DATA_SHIFT, 1);
964	bool fpcr_ah = extract64(desc, SIMD_DATA_SHIFT + 1, 1);
965	uintptr_t i;
966
967	for (i = 0; i < opr_sz / 4; i += 2) {
968	float32 e0 = n[H4(i)];
969	float32 e1 = m[H4(i + 1)];
970	float32 e2 = n[H4(i + 1)];
971	float32 e3 = m[H4(i)];
972
973	if (rot) {
974	e3 = float32_maybe_ah_chs(e3, fpcr_ah);
975	} else {
976	e1 = float32_maybe_ah_chs(e1, fpcr_ah);
977	}
978
979	d[H4(i)] = float32_add(e0, e1, fpst);
980	d[H4(i + 1)] = float32_add(e2, e3, fpst);
981	}
982	clear_tail(d, opr_sz, simd_maxsz(desc));
983	}
984
985	void HELPER(gvec_fcaddd)(void vd, void vn, void *vm,
986	float_status *fpst, uint32_t desc)
987	{
988	uintptr_t opr_sz = simd_oprsz(desc);
989	float64 *d = vd;
990	float64 *n = vn;
991	float64 *m = vm;
992	bool rot = extract32(desc, SIMD_DATA_SHIFT, 1);
993	bool fpcr_ah = extract64(desc, SIMD_DATA_SHIFT + 1, 1);
994	uintptr_t i;
995
996	for (i = 0; i < opr_sz / 8; i += 2) {
997	float64 e0 = n[i];
998	float64 e1 = m[i + 1];
999	float64 e2 = n[i + 1];
1000	float64 e3 = m[i];
1001
1002	if (rot) {
1003	e3 = float64_maybe_ah_chs(e3, fpcr_ah);
1004	} else {
1005	e1 = float64_maybe_ah_chs(e1, fpcr_ah);
1006	}
1007
1008	d[i] = float64_add(e0, e1, fpst);
1009	d[i + 1] = float64_add(e2, e3, fpst);
1010	}
1011	clear_tail(d, opr_sz, simd_maxsz(desc));
1012	}
1013
1014	void HELPER(gvec_fcmlah)(void vd, void vn, void vm, void va,
1015	float_status *fpst, uint32_t desc)
1016	{
1017	uintptr_t opr_sz = simd_oprsz(desc);
1018	float16 d = vd, n = vn, m = vm, a = va;
1019	intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1);
1020	uint32_t fpcr_ah = extract32(desc, SIMD_DATA_SHIFT + 2, 1);
1021	uint32_t negf_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1);
1022	uint32_t negf_real = flip ^ negf_imag;
1023	float16 negx_imag, negx_real;
1024	uintptr_t i;
1025
1026	/* With AH=0, use negx; with AH=1 use negf. */
1027	negx_real = (negf_real & ~fpcr_ah) << 15;
1028	negx_imag = (negf_imag & ~fpcr_ah) << 15;
1029	negf_real = (negf_real & fpcr_ah ? float_muladd_negate_product : 0);
1030	negf_imag = (negf_imag & fpcr_ah ? float_muladd_negate_product : 0);
1031
1032	for (i = 0; i < opr_sz / 2; i += 2) {
1033	float16 e2 = n[H2(i + flip)];
1034	float16 e1 = m[H2(i + flip)] ^ negx_real;
1035	float16 e4 = e2;
1036	float16 e3 = m[H2(i + 1 - flip)] ^ negx_imag;
1037
1038	d[H2(i)] = float16_muladd(e2, e1, a[H2(i)], negf_real, fpst);
1039	d[H2(i + 1)] = float16_muladd(e4, e3, a[H2(i + 1)], negf_imag, fpst);
1040	}
1041	clear_tail(d, opr_sz, simd_maxsz(desc));
1042	}
1043
1044	void HELPER(gvec_fcmlah_idx)(void vd, void vn, void vm, void va,
1045	float_status *fpst, uint32_t desc)
1046	{
1047	uintptr_t opr_sz = simd_oprsz(desc);
1048	float16 d = vd, n = vn, m = vm, a = va;
1049	intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1);
1050	uint32_t negf_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1);
1051	intptr_t index = extract32(desc, SIMD_DATA_SHIFT + 2, 2);
1052	uint32_t fpcr_ah = extract32(desc, SIMD_DATA_SHIFT + 4, 1);
1053	uint32_t negf_real = flip ^ negf_imag;
1054	intptr_t elements = opr_sz / sizeof(float16);
1055	intptr_t eltspersegment = MIN(16 / sizeof(float16), elements);
1056	float16 negx_imag, negx_real;
1057	intptr_t i, j;
1058
1059	/* With AH=0, use negx; with AH=1 use negf. */
1060	negx_real = (negf_real & ~fpcr_ah) << 15;
1061	negx_imag = (negf_imag & ~fpcr_ah) << 15;
1062	negf_real = (negf_real & fpcr_ah ? float_muladd_negate_product : 0);
1063	negf_imag = (negf_imag & fpcr_ah ? float_muladd_negate_product : 0);
1064
1065	for (i = 0; i < elements; i += eltspersegment) {
1066	float16 mr = m[H2(i + 2 * index + 0)];
1067	float16 mi = m[H2(i + 2 * index + 1)];
1068	float16 e1 = negx_real ^ (flip ? mi : mr);
1069	float16 e3 = negx_imag ^ (flip ? mr : mi);
1070
1071	for (j = i; j < i + eltspersegment; j += 2) {
1072	float16 e2 = n[H2(j + flip)];
1073	float16 e4 = e2;
1074
1075	d[H2(j)] = float16_muladd(e2, e1, a[H2(j)], negf_real, fpst);
1076	d[H2(j + 1)] = float16_muladd(e4, e3, a[H2(j + 1)], negf_imag, fpst);
1077	}
1078	}
1079	clear_tail(d, opr_sz, simd_maxsz(desc));
1080	}
1081
1082	void HELPER(gvec_fcmlas)(void vd, void vn, void vm, void va,
1083	float_status *fpst, uint32_t desc)
1084	{
1085	uintptr_t opr_sz = simd_oprsz(desc);
1086	float32 d = vd, n = vn, m = vm, a = va;
1087	intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1);
1088	uint32_t fpcr_ah = extract32(desc, SIMD_DATA_SHIFT + 2, 1);
1089	uint32_t negf_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1);
1090	uint32_t negf_real = flip ^ negf_imag;
1091	float32 negx_imag, negx_real;
1092	uintptr_t i;
1093
1094	/* With AH=0, use negx; with AH=1 use negf. */
1095	negx_real = (negf_real & ~fpcr_ah) << 31;
1096	negx_imag = (negf_imag & ~fpcr_ah) << 31;
1097	negf_real = (negf_real & fpcr_ah ? float_muladd_negate_product : 0);
1098	negf_imag = (negf_imag & fpcr_ah ? float_muladd_negate_product : 0);
1099
1100	for (i = 0; i < opr_sz / 4; i += 2) {
1101	float32 e2 = n[H4(i + flip)];
1102	float32 e1 = m[H4(i + flip)] ^ negx_real;
1103	float32 e4 = e2;
1104	float32 e3 = m[H4(i + 1 - flip)] ^ negx_imag;
1105
1106	d[H4(i)] = float32_muladd(e2, e1, a[H4(i)], negf_real, fpst);
1107	d[H4(i + 1)] = float32_muladd(e4, e3, a[H4(i + 1)], negf_imag, fpst);
1108	}
1109	clear_tail(d, opr_sz, simd_maxsz(desc));
1110	}
1111
1112	void HELPER(gvec_fcmlas_idx)(void vd, void vn, void vm, void va,
1113	float_status *fpst, uint32_t desc)
1114	{
1115	uintptr_t opr_sz = simd_oprsz(desc);
1116	float32 d = vd, n = vn, m = vm, a = va;
1117	intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1);
1118	uint32_t negf_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1);
1119	intptr_t index = extract32(desc, SIMD_DATA_SHIFT + 2, 2);
1120	uint32_t fpcr_ah = extract32(desc, SIMD_DATA_SHIFT + 4, 1);
1121	uint32_t negf_real = flip ^ negf_imag;
1122	intptr_t elements = opr_sz / sizeof(float32);
1123	intptr_t eltspersegment = MIN(16 / sizeof(float32), elements);
1124	float32 negx_imag, negx_real;
1125	intptr_t i, j;
1126
1127	/* With AH=0, use negx; with AH=1 use negf. */
1128	negx_real = (negf_real & ~fpcr_ah) << 31;
1129	negx_imag = (negf_imag & ~fpcr_ah) << 31;
1130	negf_real = (negf_real & fpcr_ah ? float_muladd_negate_product : 0);
1131	negf_imag = (negf_imag & fpcr_ah ? float_muladd_negate_product : 0);
1132
1133	for (i = 0; i < elements; i += eltspersegment) {
1134	float32 mr = m[H4(i + 2 * index + 0)];
1135	float32 mi = m[H4(i + 2 * index + 1)];
1136	float32 e1 = negx_real ^ (flip ? mi : mr);
1137	float32 e3 = negx_imag ^ (flip ? mr : mi);
1138
1139	for (j = i; j < i + eltspersegment; j += 2) {
1140	float32 e2 = n[H4(j + flip)];
1141	float32 e4 = e2;
1142
1143	d[H4(j)] = float32_muladd(e2, e1, a[H4(j)], negf_real, fpst);
1144	d[H4(j + 1)] = float32_muladd(e4, e3, a[H4(j + 1)], negf_imag, fpst);
1145	}
1146	}
1147	clear_tail(d, opr_sz, simd_maxsz(desc));
1148	}
1149
1150	void HELPER(gvec_fcmlad)(void vd, void vn, void vm, void va,
1151	float_status *fpst, uint32_t desc)
1152	{
1153	uintptr_t opr_sz = simd_oprsz(desc);
1154	float64 d = vd, n = vn, m = vm, a = va;
1155	intptr_t flip = extract32(desc, SIMD_DATA_SHIFT, 1);
1156	uint32_t fpcr_ah = extract32(desc, SIMD_DATA_SHIFT + 2, 1);
1157	uint32_t negf_imag = extract32(desc, SIMD_DATA_SHIFT + 1, 1);
1158	uint32_t negf_real = flip ^ negf_imag;
1159	float64 negx_real, negx_imag;
1160	uintptr_t i;
1161
1162	/* With AH=0, use negx; with AH=1 use negf. */
1163	negx_real = (uint64_t)(negf_real & ~fpcr_ah) << 63;
1164	negx_imag = (uint64_t)(negf_imag & ~fpcr_ah) << 63;
1165	negf_real = (negf_real & fpcr_ah ? float_muladd_negate_product : 0);
1166	negf_imag = (negf_imag & fpcr_ah ? float_muladd_negate_product : 0);
1167
1168	for (i = 0; i < opr_sz / 8; i += 2) {
1169	float64 e2 = n[i + flip];
1170	float64 e1 = m[i + flip] ^ negx_real;
1171	float64 e4 = e2;
1172	float64 e3 = m[i + 1 - flip] ^ negx_imag;
1173
1174	d[i] = float64_muladd(e2, e1, a[i], negf_real, fpst);
1175	d[i + 1] = float64_muladd(e4, e3, a[i + 1], negf_imag, fpst);
1176	}
1177	clear_tail(d, opr_sz, simd_maxsz(desc));
1178	}
1179
1180	/*
1181	* Floating point comparisons producing an integer result (all 1s or all 0s).
1182	* Note that EQ doesn't signal InvalidOp for QNaNs but GE and GT do.
1183	* Softfloat routines return 0/1, which we convert to the 0/-1 Neon requires.
1184	*/
1185	static uint16_t float16_ceq(float16 op1, float16 op2, float_status *stat)
1186	{
1187	return -float16_eq_quiet(op1, op2, stat);
1188	}
1189
1190	static uint32_t float32_ceq(float32 op1, float32 op2, float_status *stat)
1191	{
1192	return -float32_eq_quiet(op1, op2, stat);
1193	}
1194
1195	static uint64_t float64_ceq(float64 op1, float64 op2, float_status *stat)
1196	{
1197	return -float64_eq_quiet(op1, op2, stat);
1198	}
1199
1200	static uint16_t float16_cge(float16 op1, float16 op2, float_status *stat)
1201	{
1202	return -float16_le(op2, op1, stat);
1203	}
1204
1205	static uint32_t float32_cge(float32 op1, float32 op2, float_status *stat)
1206	{
1207	return -float32_le(op2, op1, stat);
1208	}
1209
1210	static uint64_t float64_cge(float64 op1, float64 op2, float_status *stat)
1211	{
1212	return -float64_le(op2, op1, stat);
1213	}
1214
1215	static uint16_t float16_cgt(float16 op1, float16 op2, float_status *stat)
1216	{
1217	return -float16_lt(op2, op1, stat);
1218	}
1219
1220	static uint32_t float32_cgt(float32 op1, float32 op2, float_status *stat)
1221	{
1222	return -float32_lt(op2, op1, stat);
1223	}
1224
1225	static uint64_t float64_cgt(float64 op1, float64 op2, float_status *stat)
1226	{
1227	return -float64_lt(op2, op1, stat);
1228	}
1229
1230	static uint16_t float16_acge(float16 op1, float16 op2, float_status *stat)
1231	{
1232	return -float16_le(float16_abs(op2), float16_abs(op1), stat);
1233	}
1234
1235	static uint32_t float32_acge(float32 op1, float32 op2, float_status *stat)
1236	{
1237	return -float32_le(float32_abs(op2), float32_abs(op1), stat);
1238	}
1239
1240	static uint64_t float64_acge(float64 op1, float64 op2, float_status *stat)
1241	{
1242	return -float64_le(float64_abs(op2), float64_abs(op1), stat);
1243	}
1244
1245	static uint16_t float16_acgt(float16 op1, float16 op2, float_status *stat)
1246	{
1247	return -float16_lt(float16_abs(op2), float16_abs(op1), stat);
1248	}
1249
1250	static uint32_t float32_acgt(float32 op1, float32 op2, float_status *stat)
1251	{
1252	return -float32_lt(float32_abs(op2), float32_abs(op1), stat);
1253	}
1254
1255	static uint64_t float64_acgt(float64 op1, float64 op2, float_status *stat)
1256	{
1257	return -float64_lt(float64_abs(op2), float64_abs(op1), stat);
1258	}
1259
1260	static int16_t vfp_tosszh(float16 x, float_status *fpst)
1261	{
1262	if (float16_is_any_nan(x)) {
1263	float_raise(float_flag_invalid, fpst);
1264	return 0;
1265	}
1266	return float16_to_int16_round_to_zero(x, fpst);
1267	}
1268
1269	static uint16_t vfp_touszh(float16 x, float_status *fpst)
1270	{
1271	if (float16_is_any_nan(x)) {
1272	float_raise(float_flag_invalid, fpst);
1273	return 0;
1274	}
1275	return float16_to_uint16_round_to_zero(x, fpst);
1276	}
1277
1278	#define DO_2OP(NAME, FUNC, TYPE) \
1279	void HELPER(NAME)(void vd, void vn, float_status *stat, uint32_t desc) \
1280	{ \
1281	intptr_t i, oprsz = simd_oprsz(desc); \
1282	TYPE d = vd, n = vn; \
1283	for (i = 0; i < oprsz / sizeof(TYPE); i++) { \
1284	d[i] = FUNC(n[i], stat); \
1285	} \
1286	clear_tail(d, oprsz, simd_maxsz(desc)); \
1287	}
1288
1289	DO_2OP(gvec_frecpe_h, helper_recpe_f16, float16)
1290	DO_2OP(gvec_frecpe_s, helper_recpe_f32, float32)
1291	DO_2OP(gvec_frecpe_rpres_s, helper_recpe_rpres_f32, float32)
1292	DO_2OP(gvec_frecpe_d, helper_recpe_f64, float64)
1293
1294	DO_2OP(gvec_frsqrte_h, helper_rsqrte_f16, float16)
1295	DO_2OP(gvec_frsqrte_s, helper_rsqrte_f32, float32)
1296	DO_2OP(gvec_frsqrte_rpres_s, helper_rsqrte_rpres_f32, float32)
1297	DO_2OP(gvec_frsqrte_d, helper_rsqrte_f64, float64)
1298
1299	DO_2OP(gvec_vrintx_h, float16_round_to_int, float16)
1300	DO_2OP(gvec_vrintx_s, float32_round_to_int, float32)
1301
1302	DO_2OP(gvec_sitos, helper_vfp_sitos, int32_t)
1303	DO_2OP(gvec_uitos, helper_vfp_uitos, uint32_t)
1304	DO_2OP(gvec_tosizs, helper_vfp_tosizs, float32)
1305	DO_2OP(gvec_touizs, helper_vfp_touizs, float32)
1306	DO_2OP(gvec_sstoh, int16_to_float16, int16_t)
1307	DO_2OP(gvec_ustoh, uint16_to_float16, uint16_t)
1308	DO_2OP(gvec_tosszh, vfp_tosszh, float16)
1309	DO_2OP(gvec_touszh, vfp_touszh, float16)
1310
1311	#define WRAP_CMP0_FWD(FN, CMPOP, TYPE) \
1312	static TYPE TYPE##_##FN##0(TYPE op, float_status *stat) \
1313	{ \
1314	return TYPE##_##CMPOP(op, TYPE##_zero, stat); \
1315	}
1316
1317	#define WRAP_CMP0_REV(FN, CMPOP, TYPE) \
1318	static TYPE TYPE##_##FN##0(TYPE op, float_status *stat) \
1319	{ \
1320	return TYPE##_##CMPOP(TYPE##_zero, op, stat); \
1321	}
1322
1323	#define DO_2OP_CMP0(FN, CMPOP, DIRN) \
1324	WRAP_CMP0_##DIRN(FN, CMPOP, float16) \
1325	WRAP_CMP0_##DIRN(FN, CMPOP, float32) \
1326	WRAP_CMP0_##DIRN(FN, CMPOP, float64) \
1327	DO_2OP(gvec_f##FN##0_h, float16_##FN##0, float16) \
1328	DO_2OP(gvec_f##FN##0_s, float32_##FN##0, float32) \
1329	DO_2OP(gvec_f##FN##0_d, float64_##FN##0, float64)
1330
1331	DO_2OP_CMP0(cgt, cgt, FWD)
1332	DO_2OP_CMP0(cge, cge, FWD)
1333	DO_2OP_CMP0(ceq, ceq, FWD)
1334	DO_2OP_CMP0(clt, cgt, REV)
1335	DO_2OP_CMP0(cle, cge, REV)
1336
1337	#undef DO_2OP
1338	#undef DO_2OP_CMP0
1339
1340	/* Floating-point trigonometric starting value.
1341	* See the ARM ARM pseudocode function FPTrigSMul.
1342	*/
1343	static float16 float16_ftsmul(float16 op1, uint16_t op2, float_status *stat)
1344	{
1345	float16 result = float16_mul(op1, op1, stat);
1346	if (!float16_is_any_nan(result)) {
1347	result = float16_set_sign(result, op2 & 1);
1348	}
1349	return result;
1350	}
1351
1352	static float32 float32_ftsmul(float32 op1, uint32_t op2, float_status *stat)
1353	{
1354	float32 result = float32_mul(op1, op1, stat);
1355	if (!float32_is_any_nan(result)) {
1356	result = float32_set_sign(result, op2 & 1);
1357	}
1358	return result;
1359	}
1360
1361	static float64 float64_ftsmul(float64 op1, uint64_t op2, float_status *stat)
1362	{
1363	float64 result = float64_mul(op1, op1, stat);
1364	if (!float64_is_any_nan(result)) {
1365	result = float64_set_sign(result, op2 & 1);
1366	}
1367	return result;
1368	}
1369
1370	static float16 float16_abd(float16 op1, float16 op2, float_status *stat)
1371	{
1372	return float16_abs(float16_sub(op1, op2, stat));
1373	}
1374
1375	static float32 float32_abd(float32 op1, float32 op2, float_status *stat)
1376	{
1377	return float32_abs(float32_sub(op1, op2, stat));
1378	}
1379
1380	static float64 float64_abd(float64 op1, float64 op2, float_status *stat)
1381	{
1382	return float64_abs(float64_sub(op1, op2, stat));
1383	}
1384
1385	/* ABD when FPCR.AH = 1: avoid flipping sign bit of a NaN result */
1386	static float16 float16_ah_abd(float16 op1, float16 op2, float_status *stat)
1387	{
1388	float16 r = float16_sub(op1, op2, stat);
1389	return float16_is_any_nan(r) ? r : float16_abs(r);
1390	}
1391
1392	static float32 float32_ah_abd(float32 op1, float32 op2, float_status *stat)
1393	{
1394	float32 r = float32_sub(op1, op2, stat);
1395	return float32_is_any_nan(r) ? r : float32_abs(r);
1396	}
1397
1398	static float64 float64_ah_abd(float64 op1, float64 op2, float_status *stat)
1399	{
1400	float64 r = float64_sub(op1, op2, stat);
1401	return float64_is_any_nan(r) ? r : float64_abs(r);
1402	}
1403
1404	/*
1405	* Reciprocal step. These are the AArch32 version which uses a
1406	* non-fused multiply-and-subtract.
1407	*/
1408	static float16 float16_recps_nf(float16 op1, float16 op2, float_status *stat)
1409	{
1410	op1 = float16_squash_input_denormal(op1, stat);
1411	op2 = float16_squash_input_denormal(op2, stat);
1412
1413	if ((float16_is_infinity(op1) && float16_is_zero(op2)) \|\|
1414	(float16_is_infinity(op2) && float16_is_zero(op1))) {
1415	return float16_two;
1416	}
1417	return float16_sub(float16_two, float16_mul(op1, op2, stat), stat);
1418	}
1419
1420	static float32 float32_recps_nf(float32 op1, float32 op2, float_status *stat)
1421	{
1422	op1 = float32_squash_input_denormal(op1, stat);
1423	op2 = float32_squash_input_denormal(op2, stat);
1424
1425	if ((float32_is_infinity(op1) && float32_is_zero(op2)) \|\|
1426	(float32_is_infinity(op2) && float32_is_zero(op1))) {
1427	return float32_two;
1428	}
1429	return float32_sub(float32_two, float32_mul(op1, op2, stat), stat);
1430	}
1431
1432	/* Reciprocal square-root step. AArch32 non-fused semantics. */
1433	static float16 float16_rsqrts_nf(float16 op1, float16 op2, float_status *stat)
1434	{
1435	op1 = float16_squash_input_denormal(op1, stat);
1436	op2 = float16_squash_input_denormal(op2, stat);
1437
1438	if ((float16_is_infinity(op1) && float16_is_zero(op2)) \|\|
1439	(float16_is_infinity(op2) && float16_is_zero(op1))) {
1440	return float16_one_point_five;
1441	}
1442	op1 = float16_sub(float16_three, float16_mul(op1, op2, stat), stat);
1443	return float16_div(op1, float16_two, stat);
1444	}
1445
1446	static float32 float32_rsqrts_nf(float32 op1, float32 op2, float_status *stat)
1447	{
1448	op1 = float32_squash_input_denormal(op1, stat);
1449	op2 = float32_squash_input_denormal(op2, stat);
1450
1451	if ((float32_is_infinity(op1) && float32_is_zero(op2)) \|\|
1452	(float32_is_infinity(op2) && float32_is_zero(op1))) {
1453	return float32_one_point_five;
1454	}
1455	op1 = float32_sub(float32_three, float32_mul(op1, op2, stat), stat);
1456	return float32_div(op1, float32_two, stat);
1457	}
1458
1459	DO_3OP(gvec_fadd_b16, bfloat16_add, float16)
1460	DO_3OP(gvec_fadd_h, float16_add, float16)
1461	DO_3OP(gvec_fadd_s, float32_add, float32)
1462	DO_3OP(gvec_fadd_d, float64_add, float64)
1463	DO_3OP(gvec_bfadd, bfloat16_add, bfloat16)
1464
1465	DO_3OP(gvec_fsub_b16, bfloat16_sub, float16)
1466	DO_3OP(gvec_fsub_h, float16_sub, float16)
1467	DO_3OP(gvec_fsub_s, float32_sub, float32)
1468	DO_3OP(gvec_fsub_d, float64_sub, float64)
1469	DO_3OP(gvec_bfsub, bfloat16_sub, bfloat16)
1470
1471	DO_3OP(gvec_fmul_b16, bfloat16_mul, float16)
1472	DO_3OP(gvec_fmul_h, float16_mul, float16)
1473	DO_3OP(gvec_fmul_s, float32_mul, float32)
1474	DO_3OP(gvec_fmul_d, float64_mul, float64)
1475
1476	DO_3OP(gvec_ftsmul_h, float16_ftsmul, float16)
1477	DO_3OP(gvec_ftsmul_s, float32_ftsmul, float32)
1478	DO_3OP(gvec_ftsmul_d, float64_ftsmul, float64)
1479
1480	DO_3OP(gvec_fabd_h, float16_abd, float16)
1481	DO_3OP(gvec_fabd_s, float32_abd, float32)
1482	DO_3OP(gvec_fabd_d, float64_abd, float64)
1483
1484	DO_3OP(gvec_ah_fabd_h, float16_ah_abd, float16)
1485	DO_3OP(gvec_ah_fabd_s, float32_ah_abd, float32)
1486	DO_3OP(gvec_ah_fabd_d, float64_ah_abd, float64)
1487
1488	DO_3OP(gvec_fceq_h, float16_ceq, float16)
1489	DO_3OP(gvec_fceq_s, float32_ceq, float32)
1490	DO_3OP(gvec_fceq_d, float64_ceq, float64)
1491
1492	DO_3OP(gvec_fcge_h, float16_cge, float16)
1493	DO_3OP(gvec_fcge_s, float32_cge, float32)
1494	DO_3OP(gvec_fcge_d, float64_cge, float64)
1495
1496	DO_3OP(gvec_fcgt_h, float16_cgt, float16)
1497	DO_3OP(gvec_fcgt_s, float32_cgt, float32)
1498	DO_3OP(gvec_fcgt_d, float64_cgt, float64)
1499
1500	DO_3OP(gvec_facge_h, float16_acge, float16)
1501	DO_3OP(gvec_facge_s, float32_acge, float32)
1502	DO_3OP(gvec_facge_d, float64_acge, float64)
1503
1504	DO_3OP(gvec_facgt_h, float16_acgt, float16)
1505	DO_3OP(gvec_facgt_s, float32_acgt, float32)
1506	DO_3OP(gvec_facgt_d, float64_acgt, float64)
1507
1508	DO_3OP(gvec_fmax_h, float16_max, float16)
1509	DO_3OP(gvec_fmax_s, float32_max, float32)
1510	DO_3OP(gvec_fmax_d, float64_max, float64)
1511
1512	DO_3OP(gvec_fmin_h, float16_min, float16)
1513	DO_3OP(gvec_fmin_s, float32_min, float32)
1514	DO_3OP(gvec_fmin_d, float64_min, float64)
1515
1516	DO_3OP(gvec_fmaxnum_h, float16_maxnum, float16)
1517	DO_3OP(gvec_fmaxnum_s, float32_maxnum, float32)
1518	DO_3OP(gvec_fmaxnum_d, float64_maxnum, float64)
1519
1520	DO_3OP(gvec_fminnum_h, float16_minnum, float16)
1521	DO_3OP(gvec_fminnum_s, float32_minnum, float32)
1522	DO_3OP(gvec_fminnum_d, float64_minnum, float64)
1523
1524	DO_3OP(gvec_recps_nf_h, float16_recps_nf, float16)
1525	DO_3OP(gvec_recps_nf_s, float32_recps_nf, float32)
1526
1527	DO_3OP(gvec_rsqrts_nf_h, float16_rsqrts_nf, float16)
1528	DO_3OP(gvec_rsqrts_nf_s, float32_rsqrts_nf, float32)
1529
1530	/* Non-fused multiply-add (unlike float16_muladd etc, which are fused) */
1531	static float16 float16_muladd_nf(float16 dest, float16 op1, float16 op2,
1532	float_status *stat)
1533	{
1534	return float16_add(dest, float16_mul(op1, op2, stat), stat);
1535	}
1536
1537	static float32 float32_muladd_nf(float32 dest, float32 op1, float32 op2,
1538	float_status *stat)
1539	{
1540	return float32_add(dest, float32_mul(op1, op2, stat), stat);
1541	}
1542
1543	static float16 float16_mulsub_nf(float16 dest, float16 op1, float16 op2,
1544	float_status *stat)
1545	{
1546	return float16_sub(dest, float16_mul(op1, op2, stat), stat);
1547	}
1548
1549	static float32 float32_mulsub_nf(float32 dest, float32 op1, float32 op2,
1550	float_status *stat)
1551	{
1552	return float32_sub(dest, float32_mul(op1, op2, stat), stat);
1553	}
1554
1555	/* Fused versions; these have the semantics Neon VFMA/VFMS want */
1556	static float16 float16_muladd_f(float16 dest, float16 op1, float16 op2,
1557	float_status *stat)
1558	{
1559	return float16_muladd(op1, op2, dest, 0, stat);
1560	}
1561
1562	static bfloat16 bfloat16_muladd_f(bfloat16 dest, bfloat16 op1, bfloat16 op2,
1563	float_status *stat)
1564	{
1565	return bfloat16_muladd(op1, op2, dest, 0, stat);
1566	}
1567
1568	static float32 float32_muladd_f(float32 dest, float32 op1, float32 op2,
1569	float_status *stat)
1570	{
1571	return float32_muladd(op1, op2, dest, 0, stat);
1572	}
1573
1574	static float64 float64_muladd_f(float64 dest, float64 op1, float64 op2,
1575	float_status *stat)
1576	{
1577	return float64_muladd(op1, op2, dest, 0, stat);
1578	}
1579
1580	static float16 float16_mulsub_f(float16 dest, float16 op1, float16 op2,
1581	float_status *stat)
1582	{
1583	return float16_muladd(float16_chs(op1), op2, dest, 0, stat);
1584	}
1585
1586	static bfloat16 bfloat16_mulsub_f(bfloat16 dest, bfloat16 op1, bfloat16 op2,
1587	float_status *stat)
1588	{
1589	return bfloat16_muladd(bfloat16_chs(op1), op2, dest, 0, stat);
1590	}
1591
1592	static float32 float32_mulsub_f(float32 dest, float32 op1, float32 op2,
1593	float_status *stat)
1594	{
1595	return float32_muladd(float32_chs(op1), op2, dest, 0, stat);
1596	}
1597
1598	static float64 float64_mulsub_f(float64 dest, float64 op1, float64 op2,
1599	float_status *stat)
1600	{
1601	return float64_muladd(float64_chs(op1), op2, dest, 0, stat);
1602	}
1603
1604	static float16 float16_ah_mulsub_f(float16 dest, float16 op1, float16 op2,
1605	float_status *stat)
1606	{
1607	return float16_muladd(op1, op2, dest, float_muladd_negate_product, stat);
1608	}
1609
1610	static bfloat16 bfloat16_ah_mulsub_f(bfloat16 dest, bfloat16 op1, bfloat16 op2,
1611	float_status *stat)
1612	{
1613	return bfloat16_muladd(op1, op2, dest, float_muladd_negate_product, stat);
1614	}
1615
1616	static float32 float32_ah_mulsub_f(float32 dest, float32 op1, float32 op2,
1617	float_status *stat)
1618	{
1619	return float32_muladd(op1, op2, dest, float_muladd_negate_product, stat);
1620	}
1621
1622	static float64 float64_ah_mulsub_f(float64 dest, float64 op1, float64 op2,
1623	float_status *stat)
1624	{
1625	return float64_muladd(op1, op2, dest, float_muladd_negate_product, stat);
1626	}
1627
1628	#define DO_MULADD(NAME, FUNC, TYPE) \
1629	void HELPER(NAME)(void vd, void vn, void *vm, \
1630	float_status *stat, uint32_t desc) \
1631	{ \
1632	intptr_t i, oprsz = simd_oprsz(desc); \
1633	TYPE d = vd, n = vn, *m = vm; \
1634	for (i = 0; i < oprsz / sizeof(TYPE); i++) { \
1635	d[i] = FUNC(d[i], n[i], m[i], stat); \
1636	} \
1637	clear_tail(d, oprsz, simd_maxsz(desc)); \
1638	}
1639
1640	DO_MULADD(gvec_fmla_nf_h, float16_muladd_nf, float16)
1641	DO_MULADD(gvec_fmla_nf_s, float32_muladd_nf, float32)
1642
1643	DO_MULADD(gvec_fmls_nf_h, float16_mulsub_nf, float16)
1644	DO_MULADD(gvec_fmls_nf_s, float32_mulsub_nf, float32)
1645
1646	DO_MULADD(gvec_vfma_h, float16_muladd_f, float16)
1647	DO_MULADD(gvec_vfma_s, float32_muladd_f, float32)
1648	DO_MULADD(gvec_vfma_d, float64_muladd_f, float64)
1649	DO_MULADD(gvec_bfmla, bfloat16_muladd_f, bfloat16)
1650
1651	DO_MULADD(gvec_vfms_h, float16_mulsub_f, float16)
1652	DO_MULADD(gvec_vfms_s, float32_mulsub_f, float32)
1653	DO_MULADD(gvec_vfms_d, float64_mulsub_f, float64)
1654	DO_MULADD(gvec_bfmls, bfloat16_mulsub_f, bfloat16)
1655
1656	DO_MULADD(gvec_ah_vfms_h, float16_ah_mulsub_f, float16)
1657	DO_MULADD(gvec_ah_vfms_s, float32_ah_mulsub_f, float32)
1658	DO_MULADD(gvec_ah_vfms_d, float64_ah_mulsub_f, float64)
1659	DO_MULADD(gvec_ah_bfmls, bfloat16_ah_mulsub_f, bfloat16)
1660
1661	#undef DO_MULADD
1662
1663	/* For the indexed ops, SVE applies the index per 128-bit vector segment.
1664	* For AdvSIMD, there is of course only one such vector segment.
1665	*/
1666
1667	#define DO_MUL_IDX(NAME, TYPE, H) \
1668	void HELPER(NAME)(void vd, void vn, void *vm, uint32_t desc) \
1669	{ \
1670	intptr_t i, j, oprsz = simd_oprsz(desc); \
1671	intptr_t segment = MIN(16, oprsz) / sizeof(TYPE); \
1672	intptr_t idx = simd_data(desc); \
1673	TYPE d = vd, n = vn, *m = vm; \
1674	for (i = 0; i < oprsz / sizeof(TYPE); i += segment) { \
1675	TYPE mm = m[H(i + idx)]; \
1676	for (j = 0; j < segment; j++) { \
1677	d[i + j] = n[i + j] * mm; \
1678	} \
1679	} \
1680	clear_tail(d, oprsz, simd_maxsz(desc)); \
1681	}
1682
1683	DO_MUL_IDX(gvec_mul_idx_h, uint16_t, H2)
1684	DO_MUL_IDX(gvec_mul_idx_s, uint32_t, H4)
1685	DO_MUL_IDX(gvec_mul_idx_d, uint64_t, H8)
1686
1687	#undef DO_MUL_IDX
1688
1689	#define DO_MLA_IDX(NAME, TYPE, OP, H) \
1690	void HELPER(NAME)(void vd, void vn, void vm, void va, uint32_t desc) \
1691	{ \
1692	intptr_t i, j, oprsz = simd_oprsz(desc); \
1693	intptr_t segment = MIN(16, oprsz) / sizeof(TYPE); \
1694	intptr_t idx = simd_data(desc); \
1695	TYPE d = vd, n = vn, m = vm, a = va; \
1696	for (i = 0; i < oprsz / sizeof(TYPE); i += segment) { \
1697	TYPE mm = m[H(i + idx)]; \
1698	for (j = 0; j < segment; j++) { \
1699	d[i + j] = a[i + j] OP n[i + j] * mm; \
1700	} \
1701	} \
1702	clear_tail(d, oprsz, simd_maxsz(desc)); \
1703	}
1704
1705	DO_MLA_IDX(gvec_mla_idx_h, uint16_t, +, H2)
1706	DO_MLA_IDX(gvec_mla_idx_s, uint32_t, +, H4)
1707	DO_MLA_IDX(gvec_mla_idx_d, uint64_t, +, H8)
1708
1709	DO_MLA_IDX(gvec_mls_idx_h, uint16_t, -, H2)
1710	DO_MLA_IDX(gvec_mls_idx_s, uint32_t, -, H4)
1711	DO_MLA_IDX(gvec_mls_idx_d, uint64_t, -, H8)
1712
1713	#undef DO_MLA_IDX
1714
1715	#define nop(N, M, S) (M)
1716
1717	DO_FMUL_IDX(gvec_fmul_idx_b16, nop, bfloat16_mul, float16, H2)
1718	DO_FMUL_IDX(gvec_fmul_idx_h, nop, float16_mul, float16, H2)
1719	DO_FMUL_IDX(gvec_fmul_idx_s, nop, float32_mul, float32, H4)
1720	DO_FMUL_IDX(gvec_fmul_idx_d, nop, float64_mul, float64, H8)
1721
1722	#undef nop
1723
1724	/*
1725	* Non-fused multiply-accumulate operations, for Neon. NB that unlike
1726	* the fused ops below they assume accumulate both from and into Vd.
1727	*/
1728	DO_FMUL_IDX(gvec_fmla_nf_idx_h, float16_add, float16_mul, float16, H2)
1729	DO_FMUL_IDX(gvec_fmla_nf_idx_s, float32_add, float32_mul, float32, H4)
1730	DO_FMUL_IDX(gvec_fmls_nf_idx_h, float16_sub, float16_mul, float16, H2)
1731	DO_FMUL_IDX(gvec_fmls_nf_idx_s, float32_sub, float32_mul, float32, H4)
1732
1733	#define DO_FMLA_IDX(NAME, TYPE, H, NEGX, NEGF) \
1734	void HELPER(NAME)(void vd, void vn, void vm, void va, \
1735	float_status *stat, uint32_t desc) \
1736	{ \
1737	intptr_t i, j, oprsz = simd_oprsz(desc); \
1738	intptr_t segment = MIN(16, oprsz) / sizeof(TYPE); \
1739	intptr_t idx = simd_data(desc); \
1740	TYPE d = vd, n = vn, m = vm, a = va; \
1741	for (i = 0; i < oprsz / sizeof(TYPE); i += segment) { \
1742	TYPE mm = m[H(i + idx)]; \
1743	for (j = 0; j < segment; j++) { \
1744	d[i + j] = TYPE##_muladd(n[i + j] ^ NEGX, mm, \
1745	a[i + j], NEGF, stat); \
1746	} \
1747	} \
1748	clear_tail(d, oprsz, simd_maxsz(desc)); \
1749	}
1750
1751	DO_FMLA_IDX(gvec_fmla_idx_h, float16, H2, 0, 0)
1752	DO_FMLA_IDX(gvec_fmla_idx_s, float32, H4, 0, 0)
1753	DO_FMLA_IDX(gvec_fmla_idx_d, float64, H8, 0, 0)
1754	DO_FMLA_IDX(gvec_bfmla_idx, bfloat16, H2, 0, 0)
1755
1756	DO_FMLA_IDX(gvec_fmls_idx_h, float16, H2, INT16_MIN, 0)
1757	DO_FMLA_IDX(gvec_fmls_idx_s, float32, H4, INT32_MIN, 0)
1758	DO_FMLA_IDX(gvec_fmls_idx_d, float64, H8, INT64_MIN, 0)
1759	DO_FMLA_IDX(gvec_bfmls_idx, bfloat16, H2, INT16_MIN, 0)
1760
1761	DO_FMLA_IDX(gvec_ah_fmls_idx_h, float16, H2, 0, float_muladd_negate_product)
1762	DO_FMLA_IDX(gvec_ah_fmls_idx_s, float32, H4, 0, float_muladd_negate_product)
1763	DO_FMLA_IDX(gvec_ah_fmls_idx_d, float64, H8, 0, float_muladd_negate_product)
1764	DO_FMLA_IDX(gvec_ah_bfmls_idx, bfloat16, H2, 0, float_muladd_negate_product)
1765
1766	#undef DO_FMLA_IDX
1767
1768	#define DO_SAT(NAME, WTYPE, TYPEN, TYPEM, OP, MIN, MAX) \
1769	void HELPER(NAME)(void vd, void vq, void vn, void vm, uint32_t desc) \
1770	{ \
1771	intptr_t i, oprsz = simd_oprsz(desc); \
1772	TYPEN d = vd, n = vn; TYPEM *m = vm; \
1773	bool q = false; \
1774	for (i = 0; i < oprsz / sizeof(TYPEN); i++) { \
1775	WTYPE dd = (WTYPE)n[i] OP m[i]; \
1776	if (dd < MIN) { \
1777	dd = MIN; \
1778	q = true; \
1779	} else if (dd > MAX) { \
1780	dd = MAX; \
1781	q = true; \
1782	} \
1783	d[i] = dd; \
1784	} \
1785	if (q) { \
1786	uint32_t *qc = vq; \
1787	qc[0] = 1; \
1788	} \
1789	clear_tail(d, oprsz, simd_maxsz(desc)); \
1790	}
1791
1792	DO_SAT(gvec_uqadd_b, int, uint8_t, uint8_t, +, 0, UINT8_MAX)
1793	DO_SAT(gvec_uqadd_h, int, uint16_t, uint16_t, +, 0, UINT16_MAX)
1794	DO_SAT(gvec_uqadd_s, int64_t, uint32_t, uint32_t, +, 0, UINT32_MAX)
1795
1796	DO_SAT(gvec_sqadd_b, int, int8_t, int8_t, +, INT8_MIN, INT8_MAX)
1797	DO_SAT(gvec_sqadd_h, int, int16_t, int16_t, +, INT16_MIN, INT16_MAX)
1798	DO_SAT(gvec_sqadd_s, int64_t, int32_t, int32_t, +, INT32_MIN, INT32_MAX)
1799
1800	DO_SAT(gvec_uqsub_b, int, uint8_t, uint8_t, -, 0, UINT8_MAX)
1801	DO_SAT(gvec_uqsub_h, int, uint16_t, uint16_t, -, 0, UINT16_MAX)
1802	DO_SAT(gvec_uqsub_s, int64_t, uint32_t, uint32_t, -, 0, UINT32_MAX)
1803
1804	DO_SAT(gvec_sqsub_b, int, int8_t, int8_t, -, INT8_MIN, INT8_MAX)
1805	DO_SAT(gvec_sqsub_h, int, int16_t, int16_t, -, INT16_MIN, INT16_MAX)
1806	DO_SAT(gvec_sqsub_s, int64_t, int32_t, int32_t, -, INT32_MIN, INT32_MAX)
1807
1808	DO_SAT(gvec_usqadd_b, int, uint8_t, int8_t, +, 0, UINT8_MAX)
1809	DO_SAT(gvec_usqadd_h, int, uint16_t, int16_t, +, 0, UINT16_MAX)
1810	DO_SAT(gvec_usqadd_s, int64_t, uint32_t, int32_t, +, 0, UINT32_MAX)
1811
1812	DO_SAT(gvec_suqadd_b, int, int8_t, uint8_t, +, INT8_MIN, INT8_MAX)
1813	DO_SAT(gvec_suqadd_h, int, int16_t, uint16_t, +, INT16_MIN, INT16_MAX)
1814	DO_SAT(gvec_suqadd_s, int64_t, int32_t, uint32_t, +, INT32_MIN, INT32_MAX)
1815
1816	#undef DO_SAT
1817
1818	void HELPER(gvec_uqadd_d)(void vd, void vq, void *vn,
1819	void *vm, uint32_t desc)
1820	{
1821	intptr_t i, oprsz = simd_oprsz(desc);
1822	uint64_t d = vd, n = vn, *m = vm;
1823	bool q = false;
1824
1825	for (i = 0; i < oprsz / 8; i++) {
1826	uint64_t nn = n[i], mm = m[i], dd = nn + mm;
1827	if (dd < nn) {
1828	dd = UINT64_MAX;
1829	q = true;
1830	}
1831	d[i] = dd;
1832	}
1833	if (q) {
1834	uint32_t *qc = vq;
1835	qc[0] = 1;
1836	}
1837	clear_tail(d, oprsz, simd_maxsz(desc));
1838	}
1839
1840	void HELPER(gvec_uqsub_d)(void vd, void vq, void *vn,
1841	void *vm, uint32_t desc)
1842	{
1843	intptr_t i, oprsz = simd_oprsz(desc);
1844	uint64_t d = vd, n = vn, *m = vm;
1845	bool q = false;
1846
1847	for (i = 0; i < oprsz / 8; i++) {
1848	uint64_t nn = n[i], mm = m[i], dd = nn - mm;
1849	if (nn < mm) {
1850	dd = 0;
1851	q = true;
1852	}
1853	d[i] = dd;
1854	}
1855	if (q) {
1856	uint32_t *qc = vq;
1857	qc[0] = 1;
1858	}
1859	clear_tail(d, oprsz, simd_maxsz(desc));
1860	}
1861
1862	void HELPER(gvec_sqadd_d)(void vd, void vq, void *vn,
1863	void *vm, uint32_t desc)
1864	{
1865	intptr_t i, oprsz = simd_oprsz(desc);
1866	int64_t d = vd, n = vn, *m = vm;
1867	bool q = false;
1868
1869	for (i = 0; i < oprsz / 8; i++) {
1870	int64_t nn = n[i], mm = m[i], dd = nn + mm;
1871	if (((dd ^ nn) & ~(nn ^ mm)) & INT64_MIN) {
1872	dd = (nn >> 63) ^ ~INT64_MIN;
1873	q = true;
1874	}
1875	d[i] = dd;
1876	}
1877	if (q) {
1878	uint32_t *qc = vq;
1879	qc[0] = 1;
1880	}
1881	clear_tail(d, oprsz, simd_maxsz(desc));
1882	}
1883
1884	void HELPER(gvec_sqsub_d)(void vd, void vq, void *vn,
1885	void *vm, uint32_t desc)
1886	{
1887	intptr_t i, oprsz = simd_oprsz(desc);
1888	int64_t d = vd, n = vn, *m = vm;
1889	bool q = false;
1890
1891	for (i = 0; i < oprsz / 8; i++) {
1892	int64_t nn = n[i], mm = m[i], dd = nn - mm;
1893	if (((dd ^ nn) & (nn ^ mm)) & INT64_MIN) {
1894	dd = (nn >> 63) ^ ~INT64_MIN;
1895	q = true;
1896	}
1897	d[i] = dd;
1898	}
1899	if (q) {
1900	uint32_t *qc = vq;
1901	qc[0] = 1;
1902	}
1903	clear_tail(d, oprsz, simd_maxsz(desc));
1904	}
1905
1906	void HELPER(gvec_usqadd_d)(void vd, void vq, void *vn,
1907	void *vm, uint32_t desc)
1908	{
1909	intptr_t i, oprsz = simd_oprsz(desc);
1910	uint64_t d = vd, n = vn, *m = vm;
1911	bool q = false;
1912
1913	for (i = 0; i < oprsz / 8; i++) {
1914	uint64_t nn = n[i];
1915	int64_t mm = m[i];
1916	uint64_t dd = nn + mm;
1917
1918	if (mm < 0) {
1919	if (nn < (uint64_t)-mm) {
1920	dd = 0;
1921	q = true;
1922	}
1923	} else {
1924	if (dd < nn) {
1925	dd = UINT64_MAX;
1926	q = true;
1927	}
1928	}
1929	d[i] = dd;
1930	}
1931	if (q) {
1932	uint32_t *qc = vq;
1933	qc[0] = 1;
1934	}
1935	clear_tail(d, oprsz, simd_maxsz(desc));
1936	}
1937
1938	void HELPER(gvec_suqadd_d)(void vd, void vq, void *vn,
1939	void *vm, uint32_t desc)
1940	{
1941	intptr_t i, oprsz = simd_oprsz(desc);
1942	uint64_t d = vd, n = vn, *m = vm;
1943	bool q = false;
1944
1945	for (i = 0; i < oprsz / 8; i++) {
1946	int64_t nn = n[i];
1947	uint64_t mm = m[i];
1948	int64_t dd = nn + mm;
1949
1950	if (mm > (uint64_t)(INT64_MAX - nn)) {
1951	dd = INT64_MAX;
1952	q = true;
1953	}
1954	d[i] = dd;
1955	}
1956	if (q) {
1957	uint32_t *qc = vq;
1958	qc[0] = 1;
1959	}
1960	clear_tail(d, oprsz, simd_maxsz(desc));
1961	}
1962
1963	#define DO_SRA(NAME, TYPE) \
1964	void HELPER(NAME)(void vd, void vn, uint32_t desc) \
1965	{ \
1966	intptr_t i, oprsz = simd_oprsz(desc); \
1967	int shift = simd_data(desc); \
1968	TYPE d = vd, n = vn; \
1969	for (i = 0; i < oprsz / sizeof(TYPE); i++) { \
1970	d[i] += n[i] >> shift; \
1971	} \
1972	clear_tail(d, oprsz, simd_maxsz(desc)); \
1973	}
1974
1975	DO_SRA(gvec_ssra_b, int8_t)
1976	DO_SRA(gvec_ssra_h, int16_t)
1977	DO_SRA(gvec_ssra_s, int32_t)
1978	DO_SRA(gvec_ssra_d, int64_t)
1979
1980	DO_SRA(gvec_usra_b, uint8_t)
1981	DO_SRA(gvec_usra_h, uint16_t)
1982	DO_SRA(gvec_usra_s, uint32_t)
1983	DO_SRA(gvec_usra_d, uint64_t)
1984
1985	#undef DO_SRA
1986
1987	#define DO_RSHR(NAME, TYPE) \
1988	void HELPER(NAME)(void vd, void vn, uint32_t desc) \
1989	{ \
1990	intptr_t i, oprsz = simd_oprsz(desc); \
1991	int shift = simd_data(desc); \
1992	TYPE d = vd, n = vn; \
1993	for (i = 0; i < oprsz / sizeof(TYPE); i++) { \
1994	TYPE tmp = n[i] >> (shift - 1); \
1995	d[i] = (tmp >> 1) + (tmp & 1); \
1996	} \
1997	clear_tail(d, oprsz, simd_maxsz(desc)); \
1998	}
1999
2000	DO_RSHR(gvec_srshr_b, int8_t)
2001	DO_RSHR(gvec_srshr_h, int16_t)
2002	DO_RSHR(gvec_srshr_s, int32_t)
2003	DO_RSHR(gvec_srshr_d, int64_t)
2004
2005	DO_RSHR(gvec_urshr_b, uint8_t)
2006	DO_RSHR(gvec_urshr_h, uint16_t)
2007	DO_RSHR(gvec_urshr_s, uint32_t)
2008	DO_RSHR(gvec_urshr_d, uint64_t)
2009
2010	#undef DO_RSHR
2011
2012	#define DO_RSRA(NAME, TYPE) \
2013	void HELPER(NAME)(void vd, void vn, uint32_t desc) \
2014	{ \
2015	intptr_t i, oprsz = simd_oprsz(desc); \
2016	int shift = simd_data(desc); \
2017	TYPE d = vd, n = vn; \
2018	for (i = 0; i < oprsz / sizeof(TYPE); i++) { \
2019	TYPE tmp = n[i] >> (shift - 1); \
2020	d[i] += (tmp >> 1) + (tmp & 1); \
2021	} \
2022	clear_tail(d, oprsz, simd_maxsz(desc)); \
2023	}
2024
2025	DO_RSRA(gvec_srsra_b, int8_t)
2026	DO_RSRA(gvec_srsra_h, int16_t)
2027	DO_RSRA(gvec_srsra_s, int32_t)
2028	DO_RSRA(gvec_srsra_d, int64_t)
2029
2030	DO_RSRA(gvec_ursra_b, uint8_t)
2031	DO_RSRA(gvec_ursra_h, uint16_t)
2032	DO_RSRA(gvec_ursra_s, uint32_t)
2033	DO_RSRA(gvec_ursra_d, uint64_t)
2034
2035	#undef DO_RSRA
2036
2037	#define DO_SRI(NAME, TYPE) \
2038	void HELPER(NAME)(void vd, void vn, uint32_t desc) \
2039	{ \
2040	intptr_t i, oprsz = simd_oprsz(desc); \
2041	int shift = simd_data(desc); \
2042	TYPE d = vd, n = vn; \
2043	for (i = 0; i < oprsz / sizeof(TYPE); i++) { \
2044	d[i] = deposit64(d[i], 0, sizeof(TYPE) * 8 - shift, n[i] >> shift); \
2045	} \
2046	clear_tail(d, oprsz, simd_maxsz(desc)); \
2047	}
2048
2049	DO_SRI(gvec_sri_b, uint8_t)
2050	DO_SRI(gvec_sri_h, uint16_t)
2051	DO_SRI(gvec_sri_s, uint32_t)
2052	DO_SRI(gvec_sri_d, uint64_t)
2053
2054	#undef DO_SRI
2055
2056	#define DO_SLI(NAME, TYPE) \
2057	void HELPER(NAME)(void vd, void vn, uint32_t desc) \
2058	{ \
2059	intptr_t i, oprsz = simd_oprsz(desc); \
2060	int shift = simd_data(desc); \
2061	TYPE d = vd, n = vn; \
2062	for (i = 0; i < oprsz / sizeof(TYPE); i++) { \
2063	d[i] = deposit64(d[i], shift, sizeof(TYPE) * 8 - shift, n[i]); \
2064	} \
2065	clear_tail(d, oprsz, simd_maxsz(desc)); \
2066	}
2067
2068	DO_SLI(gvec_sli_b, uint8_t)
2069	DO_SLI(gvec_sli_h, uint16_t)
2070	DO_SLI(gvec_sli_s, uint32_t)
2071	DO_SLI(gvec_sli_d, uint64_t)
2072
2073	#undef DO_SLI
2074
2075	/*
2076	* Convert float16 to float32, raising no exceptions and
2077	* preserving exceptional values, including SNaN.
2078	* This is effectively an unpack+repack operation.
2079	*/
2080	static float32 float16_to_float32_by_bits(uint32_t f16, bool fz16)
2081	{
2082	const int f16_bias = 15;
2083	const int f32_bias = 127;
2084	uint32_t sign = extract32(f16, 15, 1);
2085	uint32_t exp = extract32(f16, 10, 5);
2086	uint32_t frac = extract32(f16, 0, 10);
2087
2088	if (exp == 0x1f) {
2089	/* Inf or NaN */
2090	exp = 0xff;
2091	} else if (exp == 0) {
2092	/* Zero or denormal. */
2093	if (frac != 0) {
2094	if (fz16) {
2095	frac = 0;
2096	} else {
2097	/*
2098	* Denormal; these are all normal float32.
2099	* Shift the fraction so that the msb is at bit 11,
2100	* then remove bit 11 as the implicit bit of the
2101	* normalized float32. Note that we still go through
2102	* the shift for normal numbers below, to put the
2103	* float32 fraction at the right place.
2104	*/
2105	int shift = clz32(frac) - 21;
2106	frac = (frac << shift) & 0x3ff;
2107	exp = f32_bias - f16_bias - shift + 1;
2108	}
2109	}
2110	} else {
2111	/* Normal number; adjust the bias. */
2112	exp += f32_bias - f16_bias;
2113	}
2114	sign <<= 31;
2115	exp <<= 23;
2116	frac <<= 23 - 10;
2117
2118	return sign \| exp \| frac;
2119	}
2120
2121	static uint64_t load4_f16(uint64_t *ptr, int is_q, int is_2)
2122	{
2123	/*
2124	* Branchless load of u32[0], u64[0], u32[1], or u64[1].
2125	* Load the 2nd qword iff is_q & is_2.
2126	* Shift to the 2nd dword iff !is_q & is_2.
2127	* For !is_q & !is_2, the upper bits of the result are garbage.
2128	*/
2129	return ptr[is_q & is_2] >> ((is_2 & ~is_q) << 5);
2130	}
2131
2132	/*
2133	* Note that FMLAL requires oprsz == 8 or oprsz == 16,
2134	* as there is not yet SVE versions that might use blocking.
2135	*/
2136
2137	static void do_fmlal(float32 d, void vn, void *vm,
2138	CPUARMState *env, uint32_t desc,
2139	ARMFPStatusFlavour fpst_idx,
2140	uint64_t negx, int negf)
2141	{
2142	float_status *fpst = &env->vfp.fp_status[fpst_idx];
2143	bool fz16 = env->vfp.fpcr & FPCR_FZ16;
2144	intptr_t i, oprsz = simd_oprsz(desc);
2145	int is_2 = extract32(desc, SIMD_DATA_SHIFT + 1, 1);
2146	int is_q = oprsz == 16;
2147	uint64_t n_4, m_4;
2148
2149	/*
2150	* Pre-load all of the f16 data, avoiding overlap issues.
2151	* Negate all inputs for AH=0 FMLSL at once.
2152	*/
2153	n_4 = load4_f16(vn, is_q, is_2) ^ negx;
2154	m_4 = load4_f16(vm, is_q, is_2);
2155
2156	for (i = 0; i < oprsz / 4; i++) {
2157	float32 n_1 = float16_to_float32_by_bits(n_4 >> (i * 16), fz16);
2158	float32 m_1 = float16_to_float32_by_bits(m_4 >> (i * 16), fz16);
2159	d[H4(i)] = float32_muladd(n_1, m_1, d[H4(i)], negf, fpst);
2160	}
2161	clear_tail(d, oprsz, simd_maxsz(desc));
2162	}
2163
2164	void HELPER(gvec_fmlal_a32)(void vd, void vn, void *vm,
2165	CPUARMState *env, uint32_t desc)
2166	{
2167	bool is_s = extract32(desc, SIMD_DATA_SHIFT, 1);
2168	uint64_t negx = is_s ? 0x8000800080008000ull : 0;
2169
2170	do_fmlal(vd, vn, vm, env, desc, FPST_STD, negx, 0);
2171	}
2172
2173	void HELPER(gvec_fmlal_a64)(void vd, void vn, void *vm,
2174	CPUARMState *env, uint32_t desc)
2175	{
2176	bool is_s = extract32(desc, SIMD_DATA_SHIFT, 1);
2177	uint64_t negx = 0;
2178	int negf = 0;
2179
2180	if (is_s) {
2181	if (env->vfp.fpcr & FPCR_AH) {
2182	negf = float_muladd_negate_product;
2183	} else {
2184	negx = 0x8000800080008000ull;
2185	}
2186	}
2187	do_fmlal(vd, vn, vm, env, desc, FPST_A64, negx, negf);
2188	}
2189
2190	void HELPER(sve2_fmlal_zzzw_s)(void vd, void vn, void vm, void va,
2191	CPUARMState *env, uint32_t desc)
2192	{
2193	intptr_t i, oprsz = simd_oprsz(desc);
2194	bool is_s = extract32(desc, SIMD_DATA_SHIFT, 1);
2195	intptr_t sel = extract32(desc, SIMD_DATA_SHIFT + 1, 1) * sizeof(float16);
2196	bool za = extract32(desc, SIMD_DATA_SHIFT + 2, 1);
2197	float_status *status = &env->vfp.fp_status[za ? FPST_ZA : FPST_A64];
2198	bool fz16 = env->vfp.fpcr & FPCR_FZ16;
2199	int negx = 0, negf = 0;
2200
2201	if (is_s) {
2202	if (env->vfp.fpcr & FPCR_AH) {
2203	negf = float_muladd_negate_product;
2204	} else {
2205	negx = 0x8000;
2206	}
2207	}
2208
2209	for (i = 0; i < oprsz; i += sizeof(float32)) {
2210	float16 nn_16 = (float16 )(vn + H1_2(i + sel)) ^ negx;
2211	float16 mm_16 = (float16 )(vm + H1_2(i + sel));
2212	float32 nn = float16_to_float32_by_bits(nn_16, fz16);
2213	float32 mm = float16_to_float32_by_bits(mm_16, fz16);
2214	float32 aa = (float32 )(va + H1_4(i));
2215
2216	(float32 )(vd + H1_4(i)) = float32_muladd(nn, mm, aa, negf, status);
2217	}
2218	}
2219
2220	static void do_fmlal_idx(float32 d, void vn, void *vm,
2221	CPUARMState *env, uint32_t desc,
2222	ARMFPStatusFlavour fpst_idx,
2223	uint64_t negx, int negf)
2224	{
2225	float_status *fpst = &env->vfp.fp_status[fpst_idx];
2226	bool fz16 = env->vfp.fpcr & FPCR_FZ16;
2227	intptr_t i, oprsz = simd_oprsz(desc);
2228	int is_2 = extract32(desc, SIMD_DATA_SHIFT + 1, 1);
2229	int index = extract32(desc, SIMD_DATA_SHIFT + 2, 3);
2230	int is_q = oprsz == 16;
2231	uint64_t n_4;
2232	float32 m_1;
2233
2234	/*
2235	* Pre-load all of the f16 data, avoiding overlap issues.
2236	* Negate all inputs for AH=0 FMLSL at once.
2237	*/
2238	n_4 = load4_f16(vn, is_q, is_2) ^ negx;
2239	m_1 = float16_to_float32_by_bits(((float16 *)vm)[H2(index)], fz16);
2240
2241	for (i = 0; i < oprsz / 4; i++) {
2242	float32 n_1 = float16_to_float32_by_bits(n_4 >> (i * 16), fz16);
2243	d[H4(i)] = float32_muladd(n_1, m_1, d[H4(i)], negf, fpst);
2244	}
2245	clear_tail(d, oprsz, simd_maxsz(desc));
2246	}
2247
2248	void HELPER(gvec_fmlal_idx_a32)(void vd, void vn, void *vm,
2249	CPUARMState *env, uint32_t desc)
2250	{
2251	bool is_s = extract32(desc, SIMD_DATA_SHIFT, 1);
2252	uint64_t negx = is_s ? 0x8000800080008000ull : 0;
2253
2254	do_fmlal_idx(vd, vn, vm, env, desc, FPST_STD, negx, 0);
2255	}
2256
2257	void HELPER(gvec_fmlal_idx_a64)(void vd, void vn, void *vm,
2258	CPUARMState *env, uint32_t desc)
2259	{
2260	bool is_s = extract32(desc, SIMD_DATA_SHIFT, 1);
2261	uint64_t negx = 0;
2262	int negf = 0;
2263
2264	if (is_s) {
2265	if (env->vfp.fpcr & FPCR_AH) {
2266	negf = float_muladd_negate_product;
2267	} else {
2268	negx = 0x8000800080008000ull;
2269	}
2270	}
2271	do_fmlal_idx(vd, vn, vm, env, desc, FPST_A64, negx, negf);
2272	}
2273
2274	void HELPER(sve2_fmlal_zzxw_s)(void vd, void vn, void vm, void va,
2275	CPUARMState *env, uint32_t desc)
2276	{
2277	intptr_t i, j, oprsz = simd_oprsz(desc);
2278	bool is_s = extract32(desc, SIMD_DATA_SHIFT, 1);
2279	intptr_t sel = extract32(desc, SIMD_DATA_SHIFT + 1, 1) * sizeof(float16);
2280	bool za = extract32(desc, SIMD_DATA_SHIFT + 2, 1);
2281	intptr_t idx = extract32(desc, SIMD_DATA_SHIFT + 3, 3) * sizeof(float16);
2282	float_status *status = &env->vfp.fp_status[za ? FPST_ZA : FPST_A64];
2283	bool fz16 = env->vfp.fpcr & FPCR_FZ16;
2284	int negx = 0, negf = 0;
2285
2286	if (is_s) {
2287	if (env->vfp.fpcr & FPCR_AH) {
2288	negf = float_muladd_negate_product;
2289	} else {
2290	negx = 0x8000;
2291	}
2292	}
2293	for (i = 0; i < oprsz; i += 16) {
2294	float16 mm_16 = (float16 )(vm + i + idx);
2295	float32 mm = float16_to_float32_by_bits(mm_16, fz16);
2296
2297	for (j = 0; j < 16; j += sizeof(float32)) {
2298	float16 nn_16 = (float16 )(vn + H1_2(i + j + sel)) ^ negx;
2299	float32 nn = float16_to_float32_by_bits(nn_16, fz16);
2300	float32 aa = (float32 )(va + H1_4(i + j));
2301
2302	(float32 )(vd + H1_4(i + j)) =
2303	float32_muladd(nn, mm, aa, negf, status);
2304	}
2305	}
2306	}
2307
2308	void HELPER(gvec_sshl_b)(void vd, void vn, void *vm, uint32_t desc)
2309	{
2310	intptr_t i, opr_sz = simd_oprsz(desc);
2311	int8_t d = vd, n = vn, *m = vm;
2312
2313	for (i = 0; i < opr_sz; ++i) {
2314	int8_t mm = m[i];
2315	int8_t nn = n[i];
2316	int8_t res = 0;
2317	if (mm >= 0) {
2318	if (mm < 8) {
2319	res = nn << mm;
2320	}
2321	} else {
2322	res = nn >> (mm > -8 ? -mm : 7);
2323	}
2324	d[i] = res;
2325	}
2326	clear_tail(d, opr_sz, simd_maxsz(desc));
2327	}
2328
2329	void HELPER(gvec_sshl_h)(void vd, void vn, void *vm, uint32_t desc)
2330	{
2331	intptr_t i, opr_sz = simd_oprsz(desc);
2332	int16_t d = vd, n = vn, *m = vm;
2333
2334	for (i = 0; i < opr_sz / 2; ++i) {
2335	int8_t mm = m[i]; /* only 8 bits of shift are significant */
2336	int16_t nn = n[i];
2337	int16_t res = 0;
2338	if (mm >= 0) {
2339	if (mm < 16) {
2340	res = nn << mm;
2341	}
2342	} else {
2343	res = nn >> (mm > -16 ? -mm : 15);
2344	}
2345	d[i] = res;
2346	}
2347	clear_tail(d, opr_sz, simd_maxsz(desc));
2348	}
2349
2350	void HELPER(gvec_ushl_b)(void vd, void vn, void *vm, uint32_t desc)
2351	{
2352	intptr_t i, opr_sz = simd_oprsz(desc);
2353	uint8_t d = vd, n = vn, *m = vm;
2354
2355	for (i = 0; i < opr_sz; ++i) {
2356	int8_t mm = m[i];
2357	uint8_t nn = n[i];
2358	uint8_t res = 0;
2359	if (mm >= 0) {
2360	if (mm < 8) {
2361	res = nn << mm;
2362	}
2363	} else {
2364	if (mm > -8) {
2365	res = nn >> -mm;
2366	}
2367	}
2368	d[i] = res;
2369	}
2370	clear_tail(d, opr_sz, simd_maxsz(desc));
2371	}
2372
2373	void HELPER(gvec_ushl_h)(void vd, void vn, void *vm, uint32_t desc)
2374	{
2375	intptr_t i, opr_sz = simd_oprsz(desc);
2376	uint16_t d = vd, n = vn, *m = vm;
2377
2378	for (i = 0; i < opr_sz / 2; ++i) {
2379	int8_t mm = m[i]; /* only 8 bits of shift are significant */
2380	uint16_t nn = n[i];
2381	uint16_t res = 0;
2382	if (mm >= 0) {
2383	if (mm < 16) {
2384	res = nn << mm;
2385	}
2386	} else {
2387	if (mm > -16) {
2388	res = nn >> -mm;
2389	}
2390	}
2391	d[i] = res;
2392	}
2393	clear_tail(d, opr_sz, simd_maxsz(desc));
2394	}
2395
2396	/*
2397	* 8x8->8 polynomial multiply.
2398	*
2399	* Polynomial multiplication is like integer multiplication except the
2400	* partial products are XORed, not added.
2401	*
2402	* TODO: expose this as a generic vector operation, as it is a common
2403	* crypto building block.
2404	*/
2405	void HELPER(gvec_pmul_b)(void vd, void vn, void *vm, uint32_t desc)
2406	{
2407	intptr_t i, opr_sz = simd_oprsz(desc);
2408	uint64_t d = vd, n = vn, *m = vm;
2409
2410	for (i = 0; i < opr_sz / 8; ++i) {
2411	d[i] = clmul_8x8_low(n[i], m[i]);
2412	}
2413	clear_tail(d, opr_sz, simd_maxsz(desc));
2414	}
2415
2416	/*
2417	* 64x64->128 polynomial multiply.
2418	* Because of the lanes are not accessed in strict columns,
2419	* this probably cannot be turned into a generic helper.
2420	*/
2421	void HELPER(gvec_pmull_q)(void vd, void vn, void *vm, uint32_t desc)
2422	{
2423	intptr_t i, opr_sz = simd_oprsz(desc);
2424	intptr_t hi = simd_data(desc);
2425	uint64_t d = vd, n = vn, *m = vm;
2426
2427	for (i = 0; i < opr_sz / 8; i += 2) {
2428	Int128 r = clmul_64(n[i + hi], m[i + hi]);
2429	d[i] = int128_getlo(r);
2430	d[i + 1] = int128_gethi(r);
2431	}
2432	clear_tail(d, opr_sz, simd_maxsz(desc));
2433	}
2434
2435	void HELPER(neon_pmull_h)(void vd, void vn, void *vm, uint32_t desc)
2436	{
2437	int hi = simd_data(desc);
2438	uint64_t d = vd, n = vn, *m = vm;
2439	uint64_t nn = n[hi], mm = m[hi];
2440
2441	d[0] = clmul_8x4_packed(nn, mm);
2442	nn >>= 32;
2443	mm >>= 32;
2444	d[1] = clmul_8x4_packed(nn, mm);
2445
2446	clear_tail(d, 16, simd_maxsz(desc));
2447	}
2448
2449	#define DO_CMP0(NAME, TYPE, OP) \
2450	void HELPER(NAME)(void vd, void vn, uint32_t desc) \
2451	{ \
2452	intptr_t i, opr_sz = simd_oprsz(desc); \
2453	for (i = 0; i < opr_sz; i += sizeof(TYPE)) { \
2454	TYPE nn = (TYPE )(vn + i); \
2455	(TYPE )(vd + i) = -(nn OP 0); \
2456	} \
2457	clear_tail(vd, opr_sz, simd_maxsz(desc)); \
2458	}
2459
2460	DO_CMP0(gvec_ceq0_b, int8_t, ==)
2461	DO_CMP0(gvec_clt0_b, int8_t, <)
2462	DO_CMP0(gvec_cle0_b, int8_t, <=)
2463	DO_CMP0(gvec_cgt0_b, int8_t, >)
2464	DO_CMP0(gvec_cge0_b, int8_t, >=)
2465
2466	DO_CMP0(gvec_ceq0_h, int16_t, ==)
2467	DO_CMP0(gvec_clt0_h, int16_t, <)
2468	DO_CMP0(gvec_cle0_h, int16_t, <=)
2469	DO_CMP0(gvec_cgt0_h, int16_t, >)
2470	DO_CMP0(gvec_cge0_h, int16_t, >=)
2471
2472	#undef DO_CMP0
2473
2474	#define DO_ABD(NAME, TYPE) \
2475	void HELPER(NAME)(void vd, void vn, void *vm, uint32_t desc) \
2476	{ \
2477	intptr_t i, opr_sz = simd_oprsz(desc); \
2478	TYPE d = vd, n = vn, *m = vm; \
2479	\
2480	for (i = 0; i < opr_sz / sizeof(TYPE); ++i) { \
2481	d[i] = n[i] < m[i] ? m[i] - n[i] : n[i] - m[i]; \
2482	} \
2483	clear_tail(d, opr_sz, simd_maxsz(desc)); \
2484	}
2485
2486	DO_ABD(gvec_sabd_b, int8_t)
2487	DO_ABD(gvec_sabd_h, int16_t)
2488	DO_ABD(gvec_sabd_s, int32_t)
2489	DO_ABD(gvec_sabd_d, int64_t)
2490
2491	DO_ABD(gvec_uabd_b, uint8_t)
2492	DO_ABD(gvec_uabd_h, uint16_t)
2493	DO_ABD(gvec_uabd_s, uint32_t)
2494	DO_ABD(gvec_uabd_d, uint64_t)
2495
2496	#undef DO_ABD
2497
2498	#define DO_ABA(NAME, TYPE) \
2499	void HELPER(NAME)(void vd, void vn, void *vm, uint32_t desc) \
2500	{ \
2501	intptr_t i, opr_sz = simd_oprsz(desc); \
2502	TYPE d = vd, n = vn, *m = vm; \
2503	\
2504	for (i = 0; i < opr_sz / sizeof(TYPE); ++i) { \
2505	d[i] += n[i] < m[i] ? m[i] - n[i] : n[i] - m[i]; \
2506	} \
2507	clear_tail(d, opr_sz, simd_maxsz(desc)); \
2508	}
2509
2510	DO_ABA(gvec_saba_b, int8_t)
2511	DO_ABA(gvec_saba_h, int16_t)
2512	DO_ABA(gvec_saba_s, int32_t)
2513	DO_ABA(gvec_saba_d, int64_t)
2514
2515	DO_ABA(gvec_uaba_b, uint8_t)
2516	DO_ABA(gvec_uaba_h, uint16_t)
2517	DO_ABA(gvec_uaba_s, uint32_t)
2518	DO_ABA(gvec_uaba_d, uint64_t)
2519
2520	#undef DO_ABA
2521
2522	DO_3OP_PAIR(gvec_faddp_h, float16_add, float16, H2)
2523	DO_3OP_PAIR(gvec_faddp_s, float32_add, float32, H4)
2524	DO_3OP_PAIR(gvec_faddp_d, float64_add, float64, )
2525
2526	DO_3OP_PAIR(gvec_fmaxp_h, float16_max, float16, H2)
2527	DO_3OP_PAIR(gvec_fmaxp_s, float32_max, float32, H4)
2528	DO_3OP_PAIR(gvec_fmaxp_d, float64_max, float64, )
2529
2530	DO_3OP_PAIR(gvec_fminp_h, float16_min, float16, H2)
2531	DO_3OP_PAIR(gvec_fminp_s, float32_min, float32, H4)
2532	DO_3OP_PAIR(gvec_fminp_d, float64_min, float64, )
2533
2534	DO_3OP_PAIR(gvec_fmaxnump_h, float16_maxnum, float16, H2)
2535	DO_3OP_PAIR(gvec_fmaxnump_s, float32_maxnum, float32, H4)
2536	DO_3OP_PAIR(gvec_fmaxnump_d, float64_maxnum, float64, )
2537
2538	DO_3OP_PAIR(gvec_fminnump_h, float16_minnum, float16, H2)
2539	DO_3OP_PAIR(gvec_fminnump_s, float32_minnum, float32, H4)
2540	DO_3OP_PAIR(gvec_fminnump_d, float64_minnum, float64, )
2541
2542	#define DO_3OP_PAIR_NO_STATUS(NAME, FUNC, TYPE, H) \
2543	void HELPER(NAME)(void vd, void vn, void *vm, uint32_t desc) \
2544	{ \
2545	ARMVectorReg scratch; \
2546	intptr_t oprsz = simd_oprsz(desc); \
2547	intptr_t half = oprsz / sizeof(TYPE) / 2; \
2548	TYPE d = vd, n = vn, *m = vm; \
2549	if (unlikely(d == m)) { \
2550	m = memcpy(&scratch, m, oprsz); \
2551	} \
2552	for (intptr_t i = 0; i < half; ++i) { \
2553	d[H(i)] = FUNC(n[H(i * 2)], n[H(i * 2 + 1)]); \
2554	} \
2555	for (intptr_t i = 0; i < half; ++i) { \
2556	d[H(i + half)] = FUNC(m[H(i * 2)], m[H(i * 2 + 1)]); \
2557	} \
2558	clear_tail(d, oprsz, simd_maxsz(desc)); \
2559	}
2560
2561	#define ADD(A, B) (A + B)
2562	DO_3OP_PAIR_NO_STATUS(gvec_addp_b, ADD, uint8_t, H1)
2563	DO_3OP_PAIR_NO_STATUS(gvec_addp_h, ADD, uint16_t, H2)
2564	DO_3OP_PAIR_NO_STATUS(gvec_addp_s, ADD, uint32_t, H4)
2565	DO_3OP_PAIR_NO_STATUS(gvec_addp_d, ADD, uint64_t, /**/)
2566	#undef ADD
2567
2568	DO_3OP_PAIR_NO_STATUS(gvec_smaxp_b, MAX, int8_t, H1)
2569	DO_3OP_PAIR_NO_STATUS(gvec_smaxp_h, MAX, int16_t, H2)
2570	DO_3OP_PAIR_NO_STATUS(gvec_smaxp_s, MAX, int32_t, H4)
2571
2572	DO_3OP_PAIR_NO_STATUS(gvec_umaxp_b, MAX, uint8_t, H1)
2573	DO_3OP_PAIR_NO_STATUS(gvec_umaxp_h, MAX, uint16_t, H2)
2574	DO_3OP_PAIR_NO_STATUS(gvec_umaxp_s, MAX, uint32_t, H4)
2575
2576	DO_3OP_PAIR_NO_STATUS(gvec_sminp_b, MIN, int8_t, H1)
2577	DO_3OP_PAIR_NO_STATUS(gvec_sminp_h, MIN, int16_t, H2)
2578	DO_3OP_PAIR_NO_STATUS(gvec_sminp_s, MIN, int32_t, H4)
2579
2580	DO_3OP_PAIR_NO_STATUS(gvec_uminp_b, MIN, uint8_t, H1)
2581	DO_3OP_PAIR_NO_STATUS(gvec_uminp_h, MIN, uint16_t, H2)
2582	DO_3OP_PAIR_NO_STATUS(gvec_uminp_s, MIN, uint32_t, H4)
2583
2584	#undef DO_3OP_PAIR_NO_STATUS
2585
2586	#define DO_VCVT_FIXED(NAME, FUNC, TYPE) \
2587	void HELPER(NAME)(void vd, void vn, float_status *stat, uint32_t desc) \
2588	{ \
2589	intptr_t i, oprsz = simd_oprsz(desc); \
2590	int shift = simd_data(desc); \
2591	TYPE d = vd, n = vn; \
2592	float_status *fpst = stat; \
2593	for (i = 0; i < oprsz / sizeof(TYPE); i++) { \
2594	d[i] = FUNC(n[i], shift, fpst); \
2595	} \
2596	clear_tail(d, oprsz, simd_maxsz(desc)); \
2597	}
2598
2599	DO_VCVT_FIXED(gvec_vcvt_sd, helper_vfp_sqtod, uint64_t)
2600	DO_VCVT_FIXED(gvec_vcvt_ud, helper_vfp_uqtod, uint64_t)
2601	DO_VCVT_FIXED(gvec_vcvt_sf, helper_vfp_sltos, uint32_t)
2602	DO_VCVT_FIXED(gvec_vcvt_uf, helper_vfp_ultos, uint32_t)
2603	DO_VCVT_FIXED(gvec_vcvt_sh, helper_vfp_shtoh, uint16_t)
2604	DO_VCVT_FIXED(gvec_vcvt_uh, helper_vfp_uhtoh, uint16_t)
2605
2606	DO_VCVT_FIXED(gvec_vcvt_rz_ds, helper_vfp_tosqd_round_to_zero, uint64_t)
2607	DO_VCVT_FIXED(gvec_vcvt_rz_du, helper_vfp_touqd_round_to_zero, uint64_t)
2608	DO_VCVT_FIXED(gvec_vcvt_rz_fs, helper_vfp_tosls_round_to_zero, uint32_t)
2609	DO_VCVT_FIXED(gvec_vcvt_rz_fu, helper_vfp_touls_round_to_zero, uint32_t)
2610	DO_VCVT_FIXED(gvec_vcvt_rz_hs, helper_vfp_toshh_round_to_zero, uint16_t)
2611	DO_VCVT_FIXED(gvec_vcvt_rz_hu, helper_vfp_touhh_round_to_zero, uint16_t)
2612
2613	#undef DO_VCVT_FIXED
2614
2615	#define DO_VCVT_RMODE(NAME, FUNC, TYPE) \
2616	void HELPER(NAME)(void vd, void vn, float_status *fpst, uint32_t desc) \
2617	{ \
2618	intptr_t i, oprsz = simd_oprsz(desc); \
2619	uint32_t rmode = simd_data(desc); \
2620	uint32_t prev_rmode = get_float_rounding_mode(fpst); \
2621	TYPE d = vd, n = vn; \
2622	set_float_rounding_mode(rmode, fpst); \
2623	for (i = 0; i < oprsz / sizeof(TYPE); i++) { \
2624	d[i] = FUNC(n[i], 0, fpst); \
2625	} \
2626	set_float_rounding_mode(prev_rmode, fpst); \
2627	clear_tail(d, oprsz, simd_maxsz(desc)); \
2628	}
2629
2630	DO_VCVT_RMODE(gvec_vcvt_rm_sd, helper_vfp_tosqd, uint64_t)
2631	DO_VCVT_RMODE(gvec_vcvt_rm_ud, helper_vfp_touqd, uint64_t)
2632	DO_VCVT_RMODE(gvec_vcvt_rm_ss, helper_vfp_tosls, uint32_t)
2633	DO_VCVT_RMODE(gvec_vcvt_rm_us, helper_vfp_touls, uint32_t)
2634	DO_VCVT_RMODE(gvec_vcvt_rm_sh, helper_vfp_toshh, uint16_t)
2635	DO_VCVT_RMODE(gvec_vcvt_rm_uh, helper_vfp_touhh, uint16_t)
2636
2637	#undef DO_VCVT_RMODE
2638
2639	#define DO_VRINT_RMODE(NAME, FUNC, TYPE) \
2640	void HELPER(NAME)(void vd, void vn, float_status *fpst, uint32_t desc) \
2641	{ \
2642	intptr_t i, oprsz = simd_oprsz(desc); \
2643	uint32_t rmode = simd_data(desc); \
2644	uint32_t prev_rmode = get_float_rounding_mode(fpst); \
2645	TYPE d = vd, n = vn; \
2646	set_float_rounding_mode(rmode, fpst); \
2647	for (i = 0; i < oprsz / sizeof(TYPE); i++) { \
2648	d[i] = FUNC(n[i], fpst); \
2649	} \
2650	set_float_rounding_mode(prev_rmode, fpst); \
2651	clear_tail(d, oprsz, simd_maxsz(desc)); \
2652	}
2653
2654	DO_VRINT_RMODE(gvec_vrint_rm_h, helper_rinth, uint16_t)
2655	DO_VRINT_RMODE(gvec_vrint_rm_s, helper_rints, uint32_t)
2656
2657	#undef DO_VRINT_RMODE
2658
2659	/*
2660	* NxN -> N highpart multiply
2661	*
2662	* TODO: expose this as a generic vector operation.
2663	*/
2664
2665	void HELPER(gvec_smulh_b)(void vd, void vn, void *vm, uint32_t desc)
2666	{
2667	intptr_t i, opr_sz = simd_oprsz(desc);
2668	int8_t d = vd, n = vn, *m = vm;
2669
2670	for (i = 0; i < opr_sz; ++i) {
2671	d[i] = ((int32_t)n[i] * m[i]) >> 8;
2672	}
2673	clear_tail(d, opr_sz, simd_maxsz(desc));
2674	}
2675
2676	void HELPER(gvec_smulh_h)(void vd, void vn, void *vm, uint32_t desc)
2677	{
2678	intptr_t i, opr_sz = simd_oprsz(desc);
2679	int16_t d = vd, n = vn, *m = vm;
2680
2681	for (i = 0; i < opr_sz / 2; ++i) {
2682	d[i] = ((int32_t)n[i] * m[i]) >> 16;
2683	}
2684	clear_tail(d, opr_sz, simd_maxsz(desc));
2685	}
2686
2687	void HELPER(gvec_smulh_s)(void vd, void vn, void *vm, uint32_t desc)
2688	{
2689	intptr_t i, opr_sz = simd_oprsz(desc);
2690	int32_t d = vd, n = vn, *m = vm;
2691
2692	for (i = 0; i < opr_sz / 4; ++i) {
2693	d[i] = ((int64_t)n[i] * m[i]) >> 32;
2694	}
2695	clear_tail(d, opr_sz, simd_maxsz(desc));
2696	}
2697
2698	void HELPER(gvec_smulh_d)(void vd, void vn, void *vm, uint32_t desc)
2699	{
2700	intptr_t i, opr_sz = simd_oprsz(desc);
2701	uint64_t d = vd, n = vn, *m = vm;
2702	uint64_t discard;
2703
2704	for (i = 0; i < opr_sz / 8; ++i) {
2705	muls64(&discard, &d[i], n[i], m[i]);
2706	}
2707	clear_tail(d, opr_sz, simd_maxsz(desc));
2708	}
2709
2710	void HELPER(gvec_umulh_b)(void vd, void vn, void *vm, uint32_t desc)
2711	{
2712	intptr_t i, opr_sz = simd_oprsz(desc);
2713	uint8_t d = vd, n = vn, *m = vm;
2714
2715	for (i = 0; i < opr_sz; ++i) {
2716	d[i] = ((uint32_t)n[i] * m[i]) >> 8;
2717	}
2718	clear_tail(d, opr_sz, simd_maxsz(desc));
2719	}
2720
2721	void HELPER(gvec_umulh_h)(void vd, void vn, void *vm, uint32_t desc)
2722	{
2723	intptr_t i, opr_sz = simd_oprsz(desc);
2724	uint16_t d = vd, n = vn, *m = vm;
2725
2726	for (i = 0; i < opr_sz / 2; ++i) {
2727	d[i] = ((uint32_t)n[i] * m[i]) >> 16;
2728	}
2729	clear_tail(d, opr_sz, simd_maxsz(desc));
2730	}
2731
2732	void HELPER(gvec_umulh_s)(void vd, void vn, void *vm, uint32_t desc)
2733	{
2734	intptr_t i, opr_sz = simd_oprsz(desc);
2735	uint32_t d = vd, n = vn, *m = vm;
2736
2737	for (i = 0; i < opr_sz / 4; ++i) {
2738	d[i] = ((uint64_t)n[i] * m[i]) >> 32;
2739	}
2740	clear_tail(d, opr_sz, simd_maxsz(desc));
2741	}
2742
2743	void HELPER(gvec_umulh_d)(void vd, void vn, void *vm, uint32_t desc)
2744	{
2745	intptr_t i, opr_sz = simd_oprsz(desc);
2746	uint64_t d = vd, n = vn, *m = vm;
2747	uint64_t discard;
2748
2749	for (i = 0; i < opr_sz / 8; ++i) {
2750	mulu64(&discard, &d[i], n[i], m[i]);
2751	}
2752	clear_tail(d, opr_sz, simd_maxsz(desc));
2753	}
2754
2755	void HELPER(gvec_xar_d)(void vd, void vn, void *vm, uint32_t desc)
2756	{
2757	intptr_t i, opr_sz = simd_oprsz(desc) / 8;
2758	int shr = simd_data(desc);
2759	uint64_t d = vd, n = vn, *m = vm;
2760
2761	for (i = 0; i < opr_sz; ++i) {
2762	d[i] = ror64(n[i] ^ m[i], shr);
2763	}
2764	clear_tail(d, opr_sz * 8, simd_maxsz(desc));
2765	}
2766
2767	/*
2768	* Integer matrix-multiply accumulate
2769	*/
2770
2771	static uint32_t do_smmla_b(uint32_t sum, void vn, void vm)
2772	{
2773	int8_t n = vn, m = vm;
2774
2775	for (intptr_t k = 0; k < 8; ++k) {
2776	sum += n[H1(k)] * m[H1(k)];
2777	}
2778	return sum;
2779	}
2780
2781	static uint32_t do_ummla_b(uint32_t sum, void vn, void vm)
2782	{
2783	uint8_t n = vn, m = vm;
2784
2785	for (intptr_t k = 0; k < 8; ++k) {
2786	sum += n[H1(k)] * m[H1(k)];
2787	}
2788	return sum;
2789	}
2790
2791	static uint32_t do_usmmla_b(uint32_t sum, void vn, void vm)
2792	{
2793	uint8_t *n = vn;
2794	int8_t *m = vm;
2795
2796	for (intptr_t k = 0; k < 8; ++k) {
2797	sum += n[H1(k)] * m[H1(k)];
2798	}
2799	return sum;
2800	}
2801
2802	static void do_mmla_b(void vd, void vn, void vm, void va, uint32_t desc,
2803	uint32_t (inner_loop)(uint32_t, void , void *))
2804	{
2805	intptr_t seg, opr_sz = simd_oprsz(desc);
2806
2807	for (seg = 0; seg < opr_sz; seg += 16) {
2808	uint32_t *d = vd + seg;
2809	uint32_t *a = va + seg;
2810	uint32_t sum0, sum1, sum2, sum3;
2811
2812	/*
2813	* Process the entire segment at once, writing back the
2814	* results only after we've consumed all of the inputs.
2815	*
2816	* Key to indices by column:
2817	* i j i j
2818	*/
2819	sum0 = a[H4(0 + 0)];
2820	sum0 = inner_loop(sum0, vn + seg + 0, vm + seg + 0);
2821	sum1 = a[H4(0 + 1)];
2822	sum1 = inner_loop(sum1, vn + seg + 0, vm + seg + 8);
2823	sum2 = a[H4(2 + 0)];
2824	sum2 = inner_loop(sum2, vn + seg + 8, vm + seg + 0);
2825	sum3 = a[H4(2 + 1)];
2826	sum3 = inner_loop(sum3, vn + seg + 8, vm + seg + 8);
2827
2828	d[H4(0)] = sum0;
2829	d[H4(1)] = sum1;
2830	d[H4(2)] = sum2;
2831	d[H4(3)] = sum3;
2832	}
2833	clear_tail(vd, opr_sz, simd_maxsz(desc));
2834	}
2835
2836	#define DO_MMLA_B(NAME, INNER) \
2837	void HELPER(NAME)(void vd, void vn, void vm, void va, uint32_t desc) \
2838	{ do_mmla_b(vd, vn, vm, va, desc, INNER); }
2839
2840	DO_MMLA_B(gvec_smmla_b, do_smmla_b)
2841	DO_MMLA_B(gvec_ummla_b, do_ummla_b)
2842	DO_MMLA_B(gvec_usmmla_b, do_usmmla_b)
2843
2844	/*
2845	* BFloat16 Dot Product
2846	*/
2847
2848	bool is_ebf(CPUARMState env, float_status statusp)
2849	{
2850	/*
2851	* For BFDOT, BFMMLA, etc, the behaviour depends on FPCR.EBF.
2852	* For EBF = 0, we ignore the FPCR bits which determine rounding
2853	* mode and denormal-flushing, and we do unfused multiplies and
2854	* additions with intermediate rounding of all products and sums.
2855	* For EBF = 1, we honour FPCR rounding mode and denormal-flushing bits,
2856	* and we perform a fused two-way sum-of-products without intermediate
2857	* rounding of the products.
2858	* In either case, we don't set fp exception flags.
2859	*
2860	* EBF is AArch64 only, so even if it's set in the FPCR it has
2861	* no effect on AArch32 instructions.
2862	*/
2863	bool ebf = is_a64(env) && env->vfp.fpcr & FPCR_EBF;
2864
2865	*statusp = env->vfp.fp_status[is_a64(env) ? FPST_A64 : FPST_A32];
2866	set_default_nan_mode(true, statusp);
2867
2868	if (!ebf) {
2869	set_flush_to_zero(true, statusp);
2870	set_flush_inputs_to_zero(true, statusp);
2871	set_float_rounding_mode(float_round_to_odd_inf, statusp);
2872	}
2873	return ebf;
2874	}
2875
2876	float32 bfdotadd(float32 sum, uint32_t e1, uint32_t e2, float_status *fpst)
2877	{
2878	float32 t1, t2;
2879
2880	/*
2881	* Extract each BFloat16 from the element pair, and shift
2882	* them such that they become float32.
2883	*/
2884	t1 = float32_mul(e1 << 16, e2 << 16, fpst);
2885	t2 = float32_mul(e1 & 0xffff0000u, e2 & 0xffff0000u, fpst);
2886	t1 = float32_add(t1, t2, fpst);
2887	t1 = float32_add(sum, t1, fpst);
2888
2889	return t1;
2890	}
2891
2892	float32 bfdotadd_ebf(float32 sum, uint32_t e1, uint32_t e2, float_status *fpst)
2893	{
2894	/* Unpack two BFloat16 into two Float32, trivially. */
2895	float32 s1r = e1 << 16;
2896	float32 s1c = e1 & 0xffff0000u;
2897	float32 s2r = e2 << 16;
2898	float32 s2c = e2 & 0xffff0000u;
2899	float32 t32;
2900
2901	/*
2902	* Compare f16_dotadd() in sme_helper.c, but here we have
2903	* bfloat16 inputs. In particular that means that we do not
2904	* want the FPCR.FZ16 flush semantics, so we use the normal
2905	* float_status for the input handling here.
2906	*/
2907	FloatParts64 p1r = float32_unpack_canonical(s1r, fpst);
2908	FloatParts64 p1c = float32_unpack_canonical(s1c, fpst);
2909	FloatParts64 p2r = float32_unpack_canonical(s2r, fpst);
2910	FloatParts64 p2c = float32_unpack_canonical(s2c, fpst);
2911
2912	int all_mask = (float_cmask(p1r.cls) \| float_cmask(p1c.cls) \|
2913	float_cmask(p2r.cls) \| float_cmask(p2c.cls));
2914
2915	/* C.f. FPProcessNaNs4 */
2916	if (unlikely(all_mask & float_cmask_anynan)) {
2917	if (unlikely(all_mask & float_cmask_snan)) {
2918	if (p1r.cls == float_class_snan) {
2919	t32 = s1r;
2920	} else if (p1c.cls == float_class_snan) {
2921	t32 = s1c;
2922	} else if (p2r.cls == float_class_snan) {
2923	t32 = s2r;
2924	} else {
2925	t32 = s2c;
2926	}
2927	} else {
2928	if (p1r.cls == float_class_qnan) {
2929	t32 = s1r;
2930	} else if (p1c.cls == float_class_qnan) {
2931	t32 = s1c;
2932	} else if (p2r.cls == float_class_qnan) {
2933	t32 = s2r;
2934	} else {
2935	t32 = s2c;
2936	}
2937	}
2938	/*
2939	* FPConvertNaN(FPProcessNaN(t32)) will be done as part
2940	* of the final addition below.
2941	*/
2942	} else {
2943	/*
2944	* The ARM pseudocode function FPDot performs both multiplies
2945	* and the add with a single rounding operation.
2946	*/
2947	FloatParts64 tmp = parts64_mul(&p1r, &p2r, fpst);
2948	tmp = parts64_muladd(&p1c, &p2c, &tmp, 0, fpst);
2949	t32 = float32_round_pack_canonical(&tmp, fpst);
2950	}
2951
2952	/* The final accumulation step is not fused. */
2953	return float32_add(sum, t32, fpst);
2954	}
2955
2956	void HELPER(gvec_bfdot)(void vd, void vn, void vm, void va,
2957	CPUARMState *env, uint32_t desc)
2958	{
2959	intptr_t i, opr_sz = simd_oprsz(desc);
2960	float32 d = vd, a = va;
2961	uint32_t n = vn, m = vm;
2962	float_status fpst;
2963
2964	if (is_ebf(env, &fpst)) {
2965	for (i = 0; i < opr_sz / 4; ++i) {
2966	d[i] = bfdotadd_ebf(a[i], n[i], m[i], &fpst);
2967	}
2968	} else {
2969	for (i = 0; i < opr_sz / 4; ++i) {
2970	d[i] = bfdotadd(a[i], n[i], m[i], &fpst);
2971	}
2972	}
2973	clear_tail(d, opr_sz, simd_maxsz(desc));
2974	}
2975
2976	void HELPER(gvec_bfdot_idx)(void vd, void vn, void *vm,
2977	void va, CPUARMState env, uint32_t desc)
2978	{
2979	intptr_t i, j, opr_sz = simd_oprsz(desc);
2980	intptr_t index = simd_data(desc);
2981	intptr_t elements = opr_sz / 4;
2982	intptr_t eltspersegment = MIN(16 / 4, elements);
2983	float32 d = vd, a = va;
2984	uint32_t n = vn, m = vm;
2985	float_status fpst;
2986
2987	if (is_ebf(env, &fpst)) {
2988	for (i = 0; i < elements; i += eltspersegment) {
2989	uint32_t m_idx = m[i + H4(index)];
2990
2991	for (j = i; j < i + eltspersegment; j++) {
2992	d[j] = bfdotadd_ebf(a[j], n[j], m_idx, &fpst);
2993	}
2994	}
2995	} else {
2996	for (i = 0; i < elements; i += eltspersegment) {
2997	uint32_t m_idx = m[i + H4(index)];
2998
2999	for (j = i; j < i + eltspersegment; j++) {
3000	d[j] = bfdotadd(a[j], n[j], m_idx, &fpst);
3001	}
3002	}
3003	}
3004	clear_tail(d, opr_sz, simd_maxsz(desc));
3005	}
3006
3007	void HELPER(sme2_bfvdot_idx)(void vd, void vn, void *vm,
3008	void va, CPUARMState env, uint32_t desc)
3009	{
3010	intptr_t i, j, opr_sz = simd_oprsz(desc);
3011	intptr_t idx = extract32(desc, SIMD_DATA_SHIFT, 2);
3012	intptr_t sel = extract32(desc, SIMD_DATA_SHIFT + 2, 1);
3013	intptr_t elements = opr_sz / 4;
3014	intptr_t eltspersegment = MIN(16 / 4, elements);
3015	float32 d = vd, a = va;
3016	uint16_t *n0 = vn;
3017	uint16_t *n1 = vn + sizeof(ARMVectorReg);
3018	uint32_t *m = vm;
3019	float_status fpst;
3020
3021	if (is_ebf(env, &fpst)) {
3022	for (i = 0; i < elements; i += eltspersegment) {
3023	uint32_t m_idx = m[i + H4(idx)];
3024
3025	for (j = 0; j < eltspersegment; j++) {
3026	uint32_t nn = (n0[H2(2 * (i + j) + sel)])
3027	\| (n1[H2(2 * (i + j) + sel)] << 16);
3028	d[i + H4(j)] = bfdotadd_ebf(a[i + H4(j)], nn, m_idx, &fpst);
3029	}
3030	}
3031	} else {
3032	for (i = 0; i < elements; i += eltspersegment) {
3033	uint32_t m_idx = m[i + H4(idx)];
3034
3035	for (j = 0; j < eltspersegment; j++) {
3036	uint32_t nn = (n0[H2(2 * (i + j) + sel)])
3037	\| (n1[H2(2 * (i + j) + sel)] << 16);
3038	d[i + H4(j)] = bfdotadd(a[i + H4(j)], nn, m_idx, &fpst);
3039	}
3040	}
3041	}
3042	clear_tail(d, opr_sz, simd_maxsz(desc));
3043	}
3044
3045	void HELPER(gvec_bfmmla)(void vd, void vn, void vm, void va,
3046	CPUARMState *env, uint32_t desc)
3047	{
3048	intptr_t s, opr_sz = simd_oprsz(desc);
3049	float32 d = vd, a = va;
3050	uint32_t n = vn, m = vm;
3051	float_status fpst;
3052
3053	if (is_ebf(env, &fpst)) {
3054	for (s = 0; s < opr_sz / 4; s += 4) {
3055	float32 sum00, sum01, sum10, sum11;
3056
3057	/*
3058	* Process the entire segment at once, writing back the
3059	* results only after we've consumed all of the inputs.
3060	*
3061	* Key to indices by column:
3062	* i j i k j k
3063	*/
3064	sum00 = a[s + H4(0 + 0)];
3065	sum00 = bfdotadd_ebf(sum00, n[s + H4(0 + 0)], m[s + H4(0 + 0)], &fpst);
3066	sum00 = bfdotadd_ebf(sum00, n[s + H4(0 + 1)], m[s + H4(0 + 1)], &fpst);
3067
3068	sum01 = a[s + H4(0 + 1)];
3069	sum01 = bfdotadd_ebf(sum01, n[s + H4(0 + 0)], m[s + H4(2 + 0)], &fpst);
3070	sum01 = bfdotadd_ebf(sum01, n[s + H4(0 + 1)], m[s + H4(2 + 1)], &fpst);
3071
3072	sum10 = a[s + H4(2 + 0)];
3073	sum10 = bfdotadd_ebf(sum10, n[s + H4(2 + 0)], m[s + H4(0 + 0)], &fpst);
3074	sum10 = bfdotadd_ebf(sum10, n[s + H4(2 + 1)], m[s + H4(0 + 1)], &fpst);
3075
3076	sum11 = a[s + H4(2 + 1)];
3077	sum11 = bfdotadd_ebf(sum11, n[s + H4(2 + 0)], m[s + H4(2 + 0)], &fpst);
3078	sum11 = bfdotadd_ebf(sum11, n[s + H4(2 + 1)], m[s + H4(2 + 1)], &fpst);
3079
3080	d[s + H4(0 + 0)] = sum00;
3081	d[s + H4(0 + 1)] = sum01;
3082	d[s + H4(2 + 0)] = sum10;
3083	d[s + H4(2 + 1)] = sum11;
3084	}
3085	} else {
3086	for (s = 0; s < opr_sz / 4; s += 4) {
3087	float32 sum00, sum01, sum10, sum11;
3088
3089	/*
3090	* Process the entire segment at once, writing back the
3091	* results only after we've consumed all of the inputs.
3092	*
3093	* Key to indices by column:
3094	* i j i k j k
3095	*/
3096	sum00 = a[s + H4(0 + 0)];
3097	sum00 = bfdotadd(sum00, n[s + H4(0 + 0)], m[s + H4(0 + 0)], &fpst);
3098	sum00 = bfdotadd(sum00, n[s + H4(0 + 1)], m[s + H4(0 + 1)], &fpst);
3099
3100	sum01 = a[s + H4(0 + 1)];
3101	sum01 = bfdotadd(sum01, n[s + H4(0 + 0)], m[s + H4(2 + 0)], &fpst);
3102	sum01 = bfdotadd(sum01, n[s + H4(0 + 1)], m[s + H4(2 + 1)], &fpst);
3103
3104	sum10 = a[s + H4(2 + 0)];
3105	sum10 = bfdotadd(sum10, n[s + H4(2 + 0)], m[s + H4(0 + 0)], &fpst);
3106	sum10 = bfdotadd(sum10, n[s + H4(2 + 1)], m[s + H4(0 + 1)], &fpst);
3107
3108	sum11 = a[s + H4(2 + 1)];
3109	sum11 = bfdotadd(sum11, n[s + H4(2 + 0)], m[s + H4(2 + 0)], &fpst);
3110	sum11 = bfdotadd(sum11, n[s + H4(2 + 1)], m[s + H4(2 + 1)], &fpst);
3111
3112	d[s + H4(0 + 0)] = sum00;
3113	d[s + H4(0 + 1)] = sum01;
3114	d[s + H4(2 + 0)] = sum10;
3115	d[s + H4(2 + 1)] = sum11;
3116	}
3117	}
3118	clear_tail(d, opr_sz, simd_maxsz(desc));
3119	}
3120
3121	static void do_bfmlal(float32 d, bfloat16 n, bfloat16 m, float32 a,
3122	float_status *stat, uint32_t desc, int negx, int negf)
3123	{
3124	intptr_t i, opr_sz = simd_oprsz(desc);
3125	intptr_t sel = extract32(desc, SIMD_DATA_SHIFT, 1);
3126
3127	for (i = 0; i < opr_sz / 4; ++i) {
3128	float32 nn = (negx ^ n[H2(i * 2 + sel)]) << 16;
3129	float32 mm = m[H2(i * 2 + sel)] << 16;
3130	d[H4(i)] = float32_muladd(nn, mm, a[H4(i)], negf, stat);
3131	}
3132	clear_tail(d, opr_sz, simd_maxsz(desc));
3133	}
3134
3135	void HELPER(gvec_bfmlal)(void vd, void vn, void vm, void va,
3136	float_status *stat, uint32_t desc)
3137	{
3138	do_bfmlal(vd, vn, vm, va, stat, desc, 0, 0);
3139	}
3140
3141	void HELPER(gvec_bfmlsl)(void vd, void vn, void vm, void va,
3142	float_status *stat, uint32_t desc)
3143	{
3144	do_bfmlal(vd, vn, vm, va, stat, desc, 0x8000, 0);
3145	}
3146
3147	void HELPER(gvec_ah_bfmlsl)(void vd, void vn, void vm, void va,
3148	float_status *stat, uint32_t desc)
3149	{
3150	do_bfmlal(vd, vn, vm, va, stat, desc, 0, float_muladd_negate_product);
3151	}
3152
3153	static void do_bfmlal_idx(float32 d, bfloat16 n, bfloat16 m, float32 a,
3154	float_status *stat, uint32_t desc, int negx, int negf)
3155	{
3156	intptr_t i, j, opr_sz = simd_oprsz(desc);
3157	intptr_t sel = extract32(desc, SIMD_DATA_SHIFT, 1);
3158	intptr_t index = extract32(desc, SIMD_DATA_SHIFT + 1, 3);
3159	intptr_t elements = opr_sz / 4;
3160	intptr_t eltspersegment = MIN(16 / 4, elements);
3161
3162	for (i = 0; i < elements; i += eltspersegment) {
3163	float32 m_idx = m[H2(2 * i + index)] << 16;
3164
3165	for (j = i; j < i + eltspersegment; j++) {
3166	float32 n_j = (negx ^ n[H2(2 * j + sel)]) << 16;
3167	d[H4(j)] = float32_muladd(n_j, m_idx, a[H4(j)], negf, stat);
3168	}
3169	}
3170	clear_tail(d, opr_sz, simd_maxsz(desc));
3171	}
3172
3173	void HELPER(gvec_bfmlal_idx)(void vd, void vn, void vm, void va,
3174	float_status *stat, uint32_t desc)
3175	{
3176	do_bfmlal_idx(vd, vn, vm, va, stat, desc, 0, 0);
3177	}
3178
3179	void HELPER(gvec_bfmlsl_idx)(void vd, void vn, void vm, void va,
3180	float_status *stat, uint32_t desc)
3181	{
3182	do_bfmlal_idx(vd, vn, vm, va, stat, desc, 0x8000, 0);
3183	}
3184
3185	void HELPER(gvec_ah_bfmlsl_idx)(void vd, void vn, void vm, void va,
3186	float_status *stat, uint32_t desc)
3187	{
3188	do_bfmlal_idx(vd, vn, vm, va, stat, desc, 0, float_muladd_negate_product);
3189	}
3190
3191	#define DO_CLAMP(NAME, TYPE) \
3192	void HELPER(NAME)(void d, void n, void m, void a, uint32_t desc) \
3193	{ \
3194	intptr_t i, opr_sz = simd_oprsz(desc); \
3195	for (i = 0; i < opr_sz; i += sizeof(TYPE)) { \
3196	TYPE aa = (TYPE )(a + i); \
3197	TYPE nn = (TYPE )(n + i); \
3198	TYPE mm = (TYPE )(m + i); \
3199	TYPE dd = MIN(MAX(aa, nn), mm); \
3200	(TYPE )(d + i) = dd; \
3201	} \
3202	clear_tail(d, opr_sz, simd_maxsz(desc)); \
3203	}
3204
3205	DO_CLAMP(gvec_sclamp_b, int8_t)
3206	DO_CLAMP(gvec_sclamp_h, int16_t)
3207	DO_CLAMP(gvec_sclamp_s, int32_t)
3208	DO_CLAMP(gvec_sclamp_d, int64_t)
3209
3210	DO_CLAMP(gvec_uclamp_b, uint8_t)
3211	DO_CLAMP(gvec_uclamp_h, uint16_t)
3212	DO_CLAMP(gvec_uclamp_s, uint32_t)
3213	DO_CLAMP(gvec_uclamp_d, uint64_t)
3214
3215	/* Bit count in each 8-bit word. */
3216	void HELPER(gvec_cnt_b)(void vd, void vn, uint32_t desc)
3217	{
3218	intptr_t i, opr_sz = simd_oprsz(desc);
3219	uint8_t d = vd, n = vn;
3220
3221	for (i = 0; i < opr_sz; ++i) {
3222	d[i] = ctpop8(n[i]);
3223	}
3224	clear_tail(d, opr_sz, simd_maxsz(desc));
3225	}
3226
3227	/* Reverse bits in each 8 bit word */
3228	void HELPER(gvec_rbit_b)(void vd, void vn, uint32_t desc)
3229	{
3230	intptr_t i, opr_sz = simd_oprsz(desc);
3231	uint64_t d = vd, n = vn;
3232
3233	for (i = 0; i < opr_sz / 8; ++i) {
3234	d[i] = revbit64(bswap64(n[i]));
3235	}
3236	clear_tail(d, opr_sz, simd_maxsz(desc));
3237	}
3238
3239	void HELPER(gvec_urecpe_s)(void vd, void vn, uint32_t desc)
3240	{
3241	intptr_t i, opr_sz = simd_oprsz(desc);
3242	uint32_t d = vd, n = vn;
3243
3244	for (i = 0; i < opr_sz / 4; ++i) {
3245	d[i] = helper_recpe_u32(n[i]);
3246	}
3247	clear_tail(d, opr_sz, simd_maxsz(desc));
3248	}
3249
3250	void HELPER(gvec_ursqrte_s)(void vd, void vn, uint32_t desc)
3251	{
3252	intptr_t i, opr_sz = simd_oprsz(desc);
3253	uint32_t d = vd, n = vn;
3254
3255	for (i = 0; i < opr_sz / 4; ++i) {
3256	d[i] = helper_rsqrte_u32(n[i]);
3257	}
3258	clear_tail(d, opr_sz, simd_maxsz(desc));
3259	}
3260
3261	static inline void do_lut_b(void zd, uint64_t indexes, uint64_t *table,
3262	unsigned elements, unsigned segbase,
3263	unsigned dstride, unsigned isize,
3264	unsigned tsize, unsigned nreg)
3265	{
3266	for (unsigned r = 0; r < nreg; ++r) {
3267	uint8_t dst = zd + dstride r;
3268	unsigned base = segbase + r * elements;
3269
3270	for (unsigned e = 0; e < elements; ++e) {
3271	unsigned index = extractn(indexes, (base + e) * isize, isize);
3272	dst[H1(e)] = extractn(table, index * tsize, 8);
3273	}
3274	}
3275	}
3276
3277	static inline void do_lut_h(void zd, uint64_t indexes, uint64_t *table,
3278	unsigned elements, unsigned segbase,
3279	unsigned dstride, unsigned isize,
3280	unsigned tsize, unsigned nreg)
3281	{
3282	for (unsigned r = 0; r < nreg; ++r) {
3283	uint16_t dst = zd + dstride r;
3284	unsigned base = segbase + r * elements;
3285
3286	for (unsigned e = 0; e < elements; ++e) {
3287	unsigned index = extractn(indexes, (base + e) * isize, isize);
3288	dst[H2(e)] = extractn(table, index * tsize, 16);
3289	}
3290	}
3291	}
3292
3293	static inline void do_lut_s(void zd, uint64_t indexes, uint32_t *table,
3294	unsigned elements, unsigned segbase,
3295	unsigned dstride, unsigned isize,
3296	unsigned tsize, unsigned nreg)
3297	{
3298	for (unsigned r = 0; r < nreg; ++r) {
3299	uint32_t dst = zd + dstride r;
3300	unsigned base = segbase + r * elements;
3301
3302	for (unsigned e = 0; e < elements; ++e) {
3303	unsigned index = extractn(indexes, (base + e) * isize, isize);
3304	dst[H4(e)] = table[H4(index)];
3305	}
3306	}
3307	}
3308
3309	#define DO_SME2_LUT(ISIZE, NREG, SUFF, ESIZE) \
3310	void helper_sme2_luti##ISIZE##_##NREG##SUFF \
3311	(void zd, void zn, CPUARMState *env, uint32_t desc) \
3312	{ \
3313	unsigned vl = simd_oprsz(desc); \
3314	unsigned strided = extract32(desc, SIMD_DATA_SHIFT, 1); \
3315	unsigned idx = extract32(desc, SIMD_DATA_SHIFT + 1, 4); \
3316	unsigned elements = vl / ESIZE; \
3317	unsigned dstride = (!strided ? 1 : NREG == 4 ? 4 : 8); \
3318	unsigned segments = (ESIZE * 8) / (ISIZE * NREG); \
3319	unsigned segment = idx & (segments - 1); \
3320	ARMVectorReg indexes; \
3321	memcpy(&indexes, zn, vl); \
3322	do_lut_##SUFF(zd, indexes.d, (void *)env->za_state.zt0, elements, \
3323	segment * NREG * elements, \
3324	dstride * sizeof(ARMVectorReg), ISIZE, 32, NREG); \
3325	}
3326
3327	DO_SME2_LUT(2,1,b, 1)
3328	DO_SME2_LUT(2,1,h, 2)
3329	DO_SME2_LUT(2,1,s, 4)
3330	DO_SME2_LUT(2,2,b, 1)
3331	DO_SME2_LUT(2,2,h, 2)
3332	DO_SME2_LUT(2,2,s, 4)
3333	DO_SME2_LUT(2,4,b, 1)
3334	DO_SME2_LUT(2,4,h, 2)
3335	DO_SME2_LUT(2,4,s, 4)
3336
3337	DO_SME2_LUT(4,1,b, 1)
3338	DO_SME2_LUT(4,1,h, 2)
3339	DO_SME2_LUT(4,1,s, 4)
3340	DO_SME2_LUT(4,2,b, 1)
3341	DO_SME2_LUT(4,2,h, 2)
3342	DO_SME2_LUT(4,2,s, 4)
3343	DO_SME2_LUT(4,4,h, 2)
3344	DO_SME2_LUT(4,4,s, 4)
3345
3346	#undef DO_SME2_LUT
3347
3348	void helper_sme2_luti4_4b(void zd, void zn, CPUARMState *env, uint32_t desc)
3349	{
3350	unsigned vl = simd_oprsz(desc);
3351	unsigned strided = extract32(desc, SIMD_DATA_SHIFT, 1);
3352	unsigned dstride = !strided ? 1 : 4;
3353	uint64_t indexes[ARM_MAX_VQ * 4];
3354
3355	memcpy(&indexes, zn, vl);
3356	memcpy((void *)&indexes + vl, zn + sizeof(ARMVectorReg), vl);
3357
3358	do_lut_b(zd, indexes, (void *)env->za_state.zt0, vl, 0,
3359	dstride * sizeof(ARMVectorReg), 4, 32, 4);
3360	}
3361
3362	void HELPER(gvec_luti2_b)(void vd, void vn, void *vm, uint32_t desc)
3363	{
3364	unsigned part = simd_data(desc);
3365	unsigned vl = simd_oprsz(desc);
3366	unsigned elements = vl / 1;
3367	unsigned ibase = elements * part;
3368	ARMVectorReg scratch;
3369
3370	do_lut_b(&scratch, vm, vn, elements, ibase, 0, 2, 8, 1);
3371	memcpy(vd, &scratch, vl);
3372	clear_tail(vd, vl, simd_maxsz(desc));
3373	}
3374
3375	void HELPER(gvec_luti2_h)(void vd, void vn, void *vm, uint32_t desc)
3376	{
3377	unsigned part = simd_data(desc);
3378	unsigned vl = simd_oprsz(desc);
3379	unsigned elements = vl / 2;
3380	unsigned ibase = elements * part;
3381	ARMVectorReg scratch;
3382
3383	do_lut_h(&scratch, vm, vn, elements, ibase, 0, 2, 16, 1);
3384	memcpy(vd, &scratch, vl);
3385	clear_tail(vd, vl, simd_maxsz(desc));
3386	}
3387
3388	void HELPER(gvec_luti4_b)(void vd, void vn, void *vm, uint32_t desc)
3389	{
3390	unsigned part = simd_data(desc);
3391	unsigned vl = simd_oprsz(desc);
3392	unsigned elements = vl / 1;
3393	unsigned ibase = elements * part;
3394	ARMVectorReg scratch;
3395
3396	do_lut_b(&scratch, vm, vn, elements, ibase, 0, 4, 8, 1);
3397	memcpy(vd, &scratch, vl);
3398	clear_tail(vd, vl, simd_maxsz(desc));
3399	}
3400
3401	void HELPER(gvec_luti4_h)(void vd, void vn, void *vm, uint32_t desc)
3402	{
3403	unsigned part = simd_data(desc);
3404	unsigned vl = simd_oprsz(desc);
3405	unsigned elements = vl / 2;
3406	unsigned ibase = elements * part;
3407	ARMVectorReg scratch;
3408
3409	do_lut_h(&scratch, vm, vn, elements, ibase, 0, 4, 16, 1);
3410	memcpy(vd, &scratch, vl);
3411	clear_tail(vd, vl, simd_maxsz(desc));
3412	}