| 1 | /* |
| 2 | * Copyright(c) 2021-2023 Qualcomm Innovation Center, Inc. All Rights Reserved. |
| 3 | * |
| 4 | * This program is free software; you can redistribute it and/or modify |
| 5 | * it under the terms of the GNU General Public License as published by |
| 6 | * the Free Software Foundation; either version 2 of the License, or |
| 7 | * (at your option) any later version. |
| 8 | * |
| 9 | * This program is distributed in the hope that it will be useful, |
| 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 12 | * GNU General Public License for more details. |
| 13 | * |
| 14 | * You should have received a copy of the GNU General Public License |
| 15 | * along with this program; if not, see <http://www.gnu.org/licenses/>. |
| 16 | */ |
| 17 | |
| 18 | #ifndef HVX_MISC_H |
| 19 | #define HVX_MISC_H |
| 20 | |
| 21 | #include "hex_test.h" |
| 22 | |
| 23 | static inline void check(int line, int i, int j, |
| 24 | uint64_t result, uint64_t expect) |
| 25 | { |
| 26 | if (result != expect) { |
| 27 | printf("ERROR at line %d: [%d][%d] 0x%016llx != 0x%016llx\n", |
| 28 | line, i, j, result, expect); |
| 29 | err++; |
| 30 | } |
| 31 | } |
| 32 | |
| 33 | #define MAX_VEC_SIZE_BYTES 128 |
| 34 | |
| 35 | typedef union { |
| 36 | uint64_t ud[MAX_VEC_SIZE_BYTES / 8]; |
| 37 | int64_t d[MAX_VEC_SIZE_BYTES / 8]; |
| 38 | uint32_t uw[MAX_VEC_SIZE_BYTES / 4]; |
| 39 | uint32_t sf[MAX_VEC_SIZE_BYTES / 4]; /* convenience alias */ |
| 40 | int32_t w[MAX_VEC_SIZE_BYTES / 4]; |
| 41 | uint16_t uh[MAX_VEC_SIZE_BYTES / 2]; |
| 42 | uint16_t hf[MAX_VEC_SIZE_BYTES / 2]; /* convenience alias */ |
| 43 | int16_t h[MAX_VEC_SIZE_BYTES / 2]; |
| 44 | uint16_t bf[MAX_VEC_SIZE_BYTES / 2]; |
| 45 | uint8_t ub[MAX_VEC_SIZE_BYTES / 1]; |
| 46 | int8_t b[MAX_VEC_SIZE_BYTES / 1]; |
| 47 | } MMVector; |
| 48 | |
| 49 | #define BUFSIZE 16 |
| 50 | #define OUTSIZE 16 |
| 51 | #define MASKMOD 3 |
| 52 | |
| 53 | MMVector buffer0[BUFSIZE] __attribute__((aligned(MAX_VEC_SIZE_BYTES))); |
| 54 | MMVector buffer1[BUFSIZE] __attribute__((aligned(MAX_VEC_SIZE_BYTES))); |
| 55 | MMVector mask[BUFSIZE] __attribute__((aligned(MAX_VEC_SIZE_BYTES))); |
| 56 | MMVector output[OUTSIZE] __attribute__((aligned(MAX_VEC_SIZE_BYTES))); |
| 57 | MMVector expect[OUTSIZE] __attribute__((aligned(MAX_VEC_SIZE_BYTES))); |
| 58 | |
| 59 | #define CHECK_OUTPUT_FUNC(FIELD, FIELDSZ) \ |
| 60 | static inline void check_output_##FIELD(int line, size_t num_vectors) \ |
| 61 | { \ |
| 62 | for (int i = 0; i < num_vectors; i++) { \ |
| 63 | for (int j = 0; j < MAX_VEC_SIZE_BYTES / FIELDSZ; j++) { \ |
| 64 | check(line, i, j, output[i].FIELD[j], expect[i].FIELD[j]); \ |
| 65 | } \ |
| 66 | } \ |
| 67 | } |
| 68 | |
| 69 | CHECK_OUTPUT_FUNC(d, 8) |
| 70 | CHECK_OUTPUT_FUNC(w, 4) |
| 71 | CHECK_OUTPUT_FUNC(sf, 4) |
| 72 | CHECK_OUTPUT_FUNC(h, 2) |
| 73 | CHECK_OUTPUT_FUNC(uh, 2) |
| 74 | CHECK_OUTPUT_FUNC(hf, 2) |
| 75 | CHECK_OUTPUT_FUNC(ub, 1) |
| 76 | CHECK_OUTPUT_FUNC(b, 1) |
| 77 | CHECK_OUTPUT_FUNC(bf, 2) |
| 78 | |
| 79 | static inline void init_buffers(void) |
| 80 | { |
| 81 | int counter0 = 0; |
| 82 | int counter1 = 17; |
| 83 | for (int i = 0; i < BUFSIZE; i++) { |
| 84 | for (int j = 0; j < MAX_VEC_SIZE_BYTES; j++) { |
| 85 | buffer0[i].b[j] = counter0++; |
| 86 | buffer1[i].b[j] = counter1++; |
| 87 | } |
| 88 | for (int j = 0; j < MAX_VEC_SIZE_BYTES / 4; j++) { |
| 89 | mask[i].w[j] = (i + j % MASKMOD == 0) ? 0 : 1; |
| 90 | } |
| 91 | } |
| 92 | } |
| 93 | |
| 94 | static const uint32_t FP_VALUES[] = { |
| 95 | SF_INF, SF_INF_neg, SF_QNaN, SF_QNaN_special, SF_SNaN, SF_QNaN_neg, |
| 96 | SF_SNaN_neg, SF_HEX_NaN, SF_zero, SF_zero_neg, SF_one, SF_one_recip, |
| 97 | SF_one_invsqrta, SF_two, SF_four, SF_small_neg, SF_large_pos, SF_any, |
| 98 | SF_denorm, SF_random, SF_neg_two, |
| 99 | }; |
| 100 | #define FP_VALUES_MAX ARRAY_SIZE(FP_VALUES) |
| 101 | |
| 102 | static const uint16_t BF_VALUES[] = { |
| 103 | BF_INF, BF_INF_neg, BF_QNaN, BF_SNaN, BF_QNaN_neg, BF_SNaN_neg, |
| 104 | BF_HEX_NaN, BF_zero, BF_zero_neg, BF_one, BF_two, BF_four, |
| 105 | }; |
| 106 | #define BF_VALUES_MAX ARRAY_SIZE(BF_VALUES) |
| 107 | |
| 108 | static inline void init_buffers_fp(void) |
| 109 | { |
| 110 | _Static_assert(BUFSIZE * (MAX_VEC_SIZE_BYTES / 4) > |
| 111 | FP_VALUES_MAX * FP_VALUES_MAX, |
| 112 | "test arrays can't fit all FP_VALUES combinations"); |
| 113 | int counter1 = 0, counter2 = 0; |
| 114 | for (int i = 0; i < BUFSIZE; i++) { |
| 115 | for (int j = 0; j < MAX_VEC_SIZE_BYTES / 4; j++) { |
| 116 | buffer0[i].sf[j] = FP_VALUES[counter1]; |
| 117 | buffer1[i].sf[j] = FP_VALUES[counter2]; |
| 118 | counter2++; |
| 119 | if (counter2 == FP_VALUES_MAX) { |
| 120 | counter2 = 0; |
| 121 | counter1 = (counter1 + 1) % FP_VALUES_MAX; |
| 122 | } |
| 123 | } |
| 124 | } |
| 125 | } |
| 126 | |
| 127 | static inline void init_buffers_bf(void) |
| 128 | { |
| 129 | _Static_assert(BUFSIZE * (MAX_VEC_SIZE_BYTES / 2) > |
| 130 | BF_VALUES_MAX * BF_VALUES_MAX, |
| 131 | "test arrays can't fit all BF_VALUES combinations"); |
| 132 | int counter1 = 0, counter2 = 0; |
| 133 | for (int i = 0; i < BUFSIZE; i++) { |
| 134 | for (int j = 0; j < MAX_VEC_SIZE_BYTES / 2; j++) { |
| 135 | buffer0[i].bf[j] = BF_VALUES[counter1]; |
| 136 | buffer1[i].bf[j] = BF_VALUES[counter2]; |
| 137 | counter2++; |
| 138 | if (counter2 == BF_VALUES_MAX) { |
| 139 | counter2 = 0; |
| 140 | counter1 = (counter1 + 1) % BF_VALUES_MAX; |
| 141 | } |
| 142 | } |
| 143 | } |
| 144 | } |
| 145 | |
| 146 | #define VEC_OP1(ASM, EL, IN, OUT) \ |
| 147 | asm("v2 = vmem(%0 + #0)\n\t" \ |
| 148 | "v2" #EL " = " #ASM "(v2" #EL ")\n\t" \ |
| 149 | "vmem(%1 + #0) = v2\n\t" \ |
| 150 | : : "r"(IN), "r"(OUT) : "v2", "memory") |
| 151 | |
| 152 | #define VEC_OP2(ASM, EL, IN0, IN1, OUT) \ |
| 153 | asm("v2 = vmem(%0 + #0)\n\t" \ |
| 154 | "v3 = vmem(%1 + #0)\n\t" \ |
| 155 | "v2" #EL " = " #ASM "(v2" #EL ", v3" #EL ")\n\t" \ |
| 156 | "vmem(%2 + #0) = v2\n\t" \ |
| 157 | : : "r"(IN0), "r"(IN1), "r"(OUT) : "v2", "v3", "memory") |
| 158 | |
| 159 | #define TEST_VEC_OP1(NAME, ASM, EL, FIELD, FIELDSZ, OP) \ |
| 160 | static inline void test_##NAME(void) \ |
| 161 | { \ |
| 162 | void *pin = buffer0; \ |
| 163 | void *pout = output; \ |
| 164 | for (int i = 0; i < BUFSIZE; i++) { \ |
| 165 | VEC_OP1(ASM, EL, pin, pout); \ |
| 166 | pin += sizeof(MMVector); \ |
| 167 | pout += sizeof(MMVector); \ |
| 168 | } \ |
| 169 | for (int i = 0; i < BUFSIZE; i++) { \ |
| 170 | for (int j = 0; j < MAX_VEC_SIZE_BYTES / FIELDSZ; j++) { \ |
| 171 | expect[i].FIELD[j] = OP buffer0[i].FIELD[j]; \ |
| 172 | } \ |
| 173 | } \ |
| 174 | check_output_##FIELD(__LINE__, BUFSIZE); \ |
| 175 | } |
| 176 | |
| 177 | #define TEST_VEC_OP2(NAME, ASM, EL, FIELD, FIELDSZ, OP) \ |
| 178 | static inline void test_##NAME(void) \ |
| 179 | { \ |
| 180 | void *p0 = buffer0; \ |
| 181 | void *p1 = buffer1; \ |
| 182 | void *pout = output; \ |
| 183 | for (int i = 0; i < BUFSIZE; i++) { \ |
| 184 | VEC_OP2(ASM, EL, p0, p1, pout); \ |
| 185 | p0 += sizeof(MMVector); \ |
| 186 | p1 += sizeof(MMVector); \ |
| 187 | pout += sizeof(MMVector); \ |
| 188 | } \ |
| 189 | for (int i = 0; i < BUFSIZE; i++) { \ |
| 190 | for (int j = 0; j < MAX_VEC_SIZE_BYTES / FIELDSZ; j++) { \ |
| 191 | expect[i].FIELD[j] = buffer0[i].FIELD[j] OP buffer1[i].FIELD[j]; \ |
| 192 | } \ |
| 193 | } \ |
| 194 | check_output_##FIELD(__LINE__, BUFSIZE); \ |
| 195 | } |
| 196 | |
| 197 | #define THRESHOLD 31 |
| 198 | |
| 199 | #define PRED_OP2(ASM, IN0, IN1, OUT, INV) \ |
| 200 | asm("r4 = #%3\n\t" \ |
| 201 | "v1.b = vsplat(r4)\n\t" \ |
| 202 | "v2 = vmem(%0 + #0)\n\t" \ |
| 203 | "q0 = vcmp.gt(v2.b, v1.b)\n\t" \ |
| 204 | "v3 = vmem(%1 + #0)\n\t" \ |
| 205 | "q1 = vcmp.gt(v3.b, v1.b)\n\t" \ |
| 206 | "q2 = " #ASM "(q0, " INV "q1)\n\t" \ |
| 207 | "r4 = #0xff\n\t" \ |
| 208 | "v1.b = vsplat(r4)\n\t" \ |
| 209 | "if (q2) vmem(%2 + #0) = v1\n\t" \ |
| 210 | : : "r"(IN0), "r"(IN1), "r"(OUT), "i"(THRESHOLD) \ |
| 211 | : "r4", "v1", "v2", "v3", "q0", "q1", "q2", "memory") |
| 212 | |
| 213 | #define TEST_PRED_OP2(NAME, ASM, OP, INV) \ |
| 214 | static inline void test_##NAME(bool invert) \ |
| 215 | { \ |
| 216 | void *p0 = buffer0; \ |
| 217 | void *p1 = buffer1; \ |
| 218 | void *pout = output; \ |
| 219 | memset(output, 0, sizeof(expect)); \ |
| 220 | for (int i = 0; i < BUFSIZE; i++) { \ |
| 221 | PRED_OP2(ASM, p0, p1, pout, INV); \ |
| 222 | p0 += sizeof(MMVector); \ |
| 223 | p1 += sizeof(MMVector); \ |
| 224 | pout += sizeof(MMVector); \ |
| 225 | } \ |
| 226 | for (int i = 0; i < BUFSIZE; i++) { \ |
| 227 | for (int j = 0; j < MAX_VEC_SIZE_BYTES; j++) { \ |
| 228 | bool p0 = (buffer0[i].b[j] > THRESHOLD); \ |
| 229 | bool p1 = (buffer1[i].b[j] > THRESHOLD); \ |
| 230 | if (invert) { \ |
| 231 | expect[i].b[j] = (p0 OP !p1) ? 0xff : 0x00; \ |
| 232 | } else { \ |
| 233 | expect[i].b[j] = (p0 OP p1) ? 0xff : 0x00; \ |
| 234 | } \ |
| 235 | } \ |
| 236 | } \ |
| 237 | check_output_b(__LINE__, BUFSIZE); \ |
| 238 | } |
| 239 | |
| 240 | #define float_sf(x) ({ typeof(x) _x = (x); *((float *)&(_x)); }) |
| 241 | #define float_hf(x) ({ typeof(x) _x = (x); *((_Float16 *) &(_x)); }) |
| 242 | #define float_bf(x) ({ uint32_t _u = ((uint32_t)(x)) << 16; *((float *)&(_u)); }) |
| 243 | #define raw_sf(x) ({ typeof(x) _x = (x); *((uint32_t *)&(_x)); }) |
| 244 | #define raw_hf(x) ({ typeof(x) _x = (x); *((uint16_t *)&(_x)); }) |
| 245 | #define raw_bf(x) ({ typeof(x) _x = (x); (uint16_t)(*((uint32_t *)&(_x)) >> 16); }) |
| 246 | #define float_hf_to_sf(x) ((float)x) |
| 247 | #define bytes_hf 2 |
| 248 | #define bytes_sf 4 |
| 249 | #define bytes_bf 2 |
| 250 | |
| 251 | #endif |