@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1e9a77e037
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Hardware][RISC-V] Add riscv64 support for vLLM with scalar (#22112)

Signed-off-by: chenlang <chen.lang5@zte.com.cn> Co-authored-by: chenlang <10346245@zte.com.cn>

chenlang committed Sep 25, 2025 at 20:46 UTC 1e9a77e0371b160f3c49ee02e7e196eef30122c7
5 files changed +632 -2
cmake/cpu_extension.cmake
+8 -1
@@ -101,6 +101,7 @@ else()
101 find_isa(${CPUINFO} "asimd" ASIMD_FOUND) # Check for ARM NEON support
102 find_isa(${CPUINFO} "bf16" ARM_BF16_FOUND) # Check for ARM BF16 support
103 find_isa(${CPUINFO} "S390" S390_FOUND)
104 + find_isa(${CPUINFO} "v" RVV_FOUND) # Check for RISC-V RVV support
105 endif()
106
107 if (AVX512_FOUND AND NOT AVX512_DISABLED)
@@ -177,8 +178,14 @@ elseif (S390_FOUND)
178 "-mzvector"
179 "-march=native"
180 "-mtune=native")
181 +elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64")
182 + if(RVV_FOUND)
183 + message(FAIL_ERROR "Can't support rvv now.")
184 + else()
185 + list(APPEND CXX_COMPILE_FLAGS "-march=rv64gc")
186 + endif()
187 else()
181 - message(FATAL_ERROR "vLLM CPU backend requires AVX512, AVX2, Power9+ ISA, S390X ISA or ARMv8 support.")
188 + message(FATAL_ERROR "vLLM CPU backend requires AVX512, AVX2, Power9+ ISA, S390X ISA, ARMv8 or RISC-V support.")
189 endif()
190
191 #
csrc/cpu/cpu_types.hpp
+2 -1
@@ -14,7 +14,8 @@
14 // arm implementation
15 #include "cpu_types_arm.hpp"
16 #else
17 - #warning "unsupported vLLM cpu implementation"
17 + #warning "unsupported vLLM cpu implementation, vLLM will compile with scalar"
18 + #include "cpu_types_scalar.hpp"
19 #endif
20
21 #ifdef _OPENMP
csrc/cpu/cpu_types_scalar.hpp new
+513
@@ -0,0 +1,513 @@
1 +#include <cmath>
2 +#include <cstdint>
3 +#include <cstring>
4 +#include <torch/all.h>
5 +#include "float_convert.hpp"
6 +
7 +namespace vec_op {
8 +
9 +#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
10 + AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
11 + AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) \
12 + AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__)
13 +
14 +#define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
15 + AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
16 +
17 +#ifndef CPU_OP_GUARD
18 + #define CPU_KERNEL_GUARD_IN(NAME)
19 + #define CPU_KERNEL_GUARD_OUT(NAME)
20 +#else
21 + #define CPU_KERNEL_GUARD_IN(NAME) \
22 + std::cout << #NAME << " invoked." << std::endl;
23 + #define CPU_KERNEL_GUARD_OUT(NAME) \
24 + std::cout << #NAME << " exit." << std::endl;
25 +#endif
26 +
27 +#define FORCE_INLINE __attribute__((always_inline)) inline
28 +
29 +#define __max(a, b) ((a) > (b) ? (a) : (b))
30 +#define __min(a, b) ((a) < (b) ? (a) : (b))
31 +#define __abs(a) ((a) < (0) ? (0 - a) : (a))
32 +
33 +typedef struct f16x8_t {
34 + uint16_t val[8];
35 +} f16x8_t;
36 +
37 +typedef struct f16x16_t {
38 + uint16_t val[16];
39 +} f16x16_t;
40 +
41 +typedef struct f16x32_t {
42 + uint16_t val[32];
43 +} f16x32_t;
44 +
45 +typedef struct f32x4_t {
46 + float val[4];
47 +} f32x4_t;
48 +
49 +typedef struct f32x8_t {
50 + float val[8];
51 +} f32x8_t;
52 +
53 +typedef struct f32x16_t {
54 + float val[16];
55 +} f32x16_t;
56 +
57 +namespace {
58 +template <typename T, T... indexes, typename F>
59 +constexpr void unroll_loop_item(std::integer_sequence<T, indexes...>, F&& f) {
60 + (f(std::integral_constant<T, indexes>{}), ...);
61 +};
62 +}; // namespace
63 +
64 +template <typename T, T count, typename F,
65 + typename = std::enable_if_t<std::is_invocable_v<F, T> > >
66 +constexpr void unroll_loop(F&& f) {
67 + unroll_loop_item(std::make_integer_sequence<T, count>{}, std::forward<F>(f));
68 +}
69 +
70 +template <typename T>
71 +struct Vec {
72 + constexpr static int get_elem_num() { return T::VEC_ELEM_NUM; }
73 +};
74 +
75 +struct FP32Vec8;
76 +struct FP32Vec16;
77 +
78 +struct FP16Vec8 : public Vec<FP16Vec8> {
79 + constexpr static int VEC_ELEM_NUM = 8;
80 + f16x8_t reg;
81 +
82 + explicit FP16Vec8(const void* ptr)
83 + : reg(*reinterpret_cast<const f16x8_t*>(ptr)) {};
84 +
85 + explicit FP16Vec8(const FP32Vec8&);
86 +
87 + void save(void* ptr) const { *reinterpret_cast<f16x8_t*>(ptr) = reg; }
88 +};
89 +
90 +struct FP16Vec16 : public Vec<FP16Vec16> {
91 + constexpr static int VEC_ELEM_NUM = 16;
92 + f16x16_t reg;
93 +
94 + explicit FP16Vec16(const void* ptr)
95 + : reg(*reinterpret_cast<const f16x16_t*>(ptr)) {};
96 +
97 + explicit FP16Vec16(const FP32Vec16&);
98 +
99 + void save(void* ptr) const { *reinterpret_cast<f16x16_t*>(ptr) = reg; }
100 +
101 + void save(void* ptr, const int elem_num) const {
102 + int num = __min(elem_num, VEC_ELEM_NUM);
103 + std::memcpy(ptr, &(reg.val[0]), num * sizeof(uint16_t));
104 + }
105 +};
106 +
107 +struct BF16Vec8 : public Vec<BF16Vec8> {
108 + constexpr static int VEC_ELEM_NUM = 8;
109 + f16x8_t reg;
110 +
111 + explicit BF16Vec8(const void* ptr)
112 + : reg(*reinterpret_cast<const f16x8_t*>(ptr)) {};
113 +
114 + explicit BF16Vec8(const FP32Vec8&);
115 +
116 + void save(void* ptr) const { *reinterpret_cast<f16x8_t*>(ptr) = reg; }
117 +};
118 +
119 +struct BF16Vec16 : public Vec<BF16Vec16> {
120 + constexpr static int VEC_ELEM_NUM = 16;
121 + f16x16_t reg;
122 +
123 + explicit BF16Vec16(const void* ptr)
124 + : reg(*reinterpret_cast<const f16x16_t*>(ptr)) {};
125 +
126 + explicit BF16Vec16(const FP32Vec16&);
127 +
128 + void save(void* ptr) const { *reinterpret_cast<f16x16_t*>(ptr) = reg; }
129 +
130 + void save(void* ptr, const int elem_num) const {
131 + int num = __min(elem_num, VEC_ELEM_NUM);
132 + std::memcpy(ptr, &(reg.val[0]), num * sizeof(uint16_t));
133 + }
134 +};
135 +
136 +struct BF16Vec32 : public Vec<BF16Vec32> {
137 + constexpr static int VEC_ELEM_NUM = 32;
138 + f16x32_t reg;
139 +
140 + explicit BF16Vec32(const void* ptr)
141 + : reg(*reinterpret_cast<const f16x32_t*>(ptr)) {};
142 +
143 + explicit BF16Vec32(f16x32_t data) : reg(data) {};
144 +
145 + explicit BF16Vec32(BF16Vec8& vec8_data) {
146 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
147 + reg.val[i] = vec8_data.reg.val[i % BF16Vec8::VEC_ELEM_NUM];
148 + }
149 + }
150 +
151 + void save(void* ptr) const { *reinterpret_cast<f16x32_t*>(ptr) = reg; }
152 +};
153 +
154 +struct FP32Vec4 : public Vec<FP32Vec4> {
155 + constexpr static int VEC_ELEM_NUM = 4;
156 +
157 + f32x4_t reg;
158 +
159 + explicit FP32Vec4(float v) {
160 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
161 + reg.val[i] = v;
162 + }
163 + }
164 +
165 + explicit FP32Vec4() {
166 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
167 + reg.val[i] = 0.0f;
168 + }
169 + }
170 +
171 + explicit FP32Vec4(const float* ptr)
172 + : reg(*reinterpret_cast<const f32x4_t*>(ptr)) {};
173 +
174 + explicit FP32Vec4(f32x4_t data) : reg(data) {};
175 +
176 + explicit FP32Vec4(const FP32Vec4& data) : reg(data.reg) {};
177 +};
178 +
179 +struct FP32Vec8 : public Vec<FP32Vec8> {
180 + constexpr static int VEC_ELEM_NUM = 8;
181 +
182 + f32x8_t reg;
183 +
184 + explicit FP32Vec8(float v) {
185 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
186 + reg.val[i] = v;
187 + }
188 + }
189 +
190 + explicit FP32Vec8() {
191 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
192 + reg.val[i] = 0.0f;
193 + }
194 + }
195 +
196 + explicit FP32Vec8(const float* ptr)
197 + : reg(*reinterpret_cast<const f32x8_t*>(ptr)) {};
198 +
199 + explicit FP32Vec8(f32x8_t data) : reg(data) {};
200 +
201 + explicit FP32Vec8(const FP32Vec8& data) : reg(data.reg) {};
202 +
203 + explicit FP32Vec8(const FP16Vec8& v) {
204 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
205 + reg.val[i] = fp16_to_float(v.reg.val[i]);
206 + }
207 + }
208 +
209 + FP32Vec8(const BF16Vec8& v) {
210 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
211 + reg.val[i] = bf16_to_float(v.reg.val[i]);
212 + }
213 + }
214 +
215 + float reduce_sum() const {
216 + float result = 0;
217 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
218 + result += reg.val[i];
219 + }
220 + return result;
221 + }
222 +
223 + FP32Vec8 exp() const {
224 + f32x8_t ret;
225 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
226 + ret.val[i] = expf(reg.val[i]);
227 + }
228 + return FP32Vec8(ret);
229 + }
230 +
231 + FP32Vec8 tanh() const {
232 + f32x8_t ret;
233 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
234 + ret.val[i] = tanhf(reg.val[i]);
235 + }
236 + return FP32Vec8(ret);
237 + }
238 +
239 + FP32Vec8 er() const {
240 + f32x8_t ret;
241 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
242 + ret.val[i] = erf(reg.val[i]);
243 + }
244 + return FP32Vec8(ret);
245 + }
246 +
247 + FP32Vec8 operator*(const FP32Vec8& b) const {
248 + f32x8_t ret;
249 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
250 + ret.val[i] = reg.val[i] * b.reg.val[i];
251 + }
252 + return FP32Vec8(ret);
253 + }
254 +
255 + FP32Vec8 operator+(const FP32Vec8& b) const {
256 + f32x8_t ret;
257 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
258 + ret.val[i] = reg.val[i] + b.reg.val[i];
259 + }
260 + return FP32Vec8(ret);
261 + }
262 +
263 + FP32Vec8 operator-(const FP32Vec8& b) const {
264 + f32x8_t ret;
265 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
266 + ret.val[i] = reg.val[i] - b.reg.val[i];
267 + }
268 + return FP32Vec8(ret);
269 + }
270 +
271 + FP32Vec8 operator/(const FP32Vec8& b) const {
272 + f32x8_t ret;
273 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
274 + ret.val[i] = reg.val[i] / b.reg.val[i];
275 + }
276 + return FP32Vec8(ret);
277 + }
278 +
279 + void save(void* ptr) const { *reinterpret_cast<f32x8_t*>(ptr) = reg; }
280 +};
281 +
282 +struct FP32Vec16 : public Vec<FP32Vec16> {
283 + constexpr static int VEC_ELEM_NUM = 16;
284 + f32x16_t reg;
285 +
286 + explicit FP32Vec16(float v) {
287 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
288 + reg.val[i] = v;
289 + }
290 + }
291 +
292 + explicit FP32Vec16() {
293 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
294 + reg.val[i] = 0.0f;
295 + }
296 + }
297 +
298 + explicit FP32Vec16(const float* ptr)
299 + : reg(*reinterpret_cast<const f32x16_t*>(ptr)) {};
300 +
301 + explicit FP32Vec16(f32x16_t data) : reg(data) {};
302 +
303 + FP32Vec16(const FP32Vec4& data) {
304 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
305 + reg.val[i] = data.reg.val[i % FP32Vec4::VEC_ELEM_NUM];
306 + }
307 + }
308 +
309 + FP32Vec16(const FP32Vec8& data) {
310 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
311 + reg.val[i] = data.reg.val[i % FP32Vec8::VEC_ELEM_NUM];
312 + }
313 + }
314 +
315 + FP32Vec16(const FP32Vec16& data) : reg(data.reg) {};
316 +
317 + explicit FP32Vec16(const FP16Vec16& v) {
318 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
319 + reg.val[i] = fp16_to_float(v.reg.val[i]);
320 + }
321 + }
322 +
323 + explicit FP32Vec16(const BF16Vec16& v) {
324 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
325 + reg.val[i] = bf16_to_float(v.reg.val[i]);
326 + }
327 + }
328 +
329 + explicit FP32Vec16(const FP16Vec8& v) : FP32Vec16(FP32Vec8(v)) {};
330 +
331 + FP32Vec16(const BF16Vec8& v) : FP32Vec16(FP32Vec8(v)) {};
332 +
333 + FP32Vec16 operator*(const FP32Vec16& b) const {
334 + FP32Vec16 result(0.0f);
335 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
336 + result.reg.val[i] = reg.val[i] * b.reg.val[i];
337 + }
338 + return result;
339 + }
340 +
341 + FP32Vec16 operator+(const FP32Vec16& b) const {
342 + FP32Vec16 result(0.0f);
343 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
344 + result.reg.val[i] = reg.val[i] + b.reg.val[i];
345 + }
346 + return result;
347 + }
348 +
349 + FP32Vec16 operator-(const FP32Vec16& b) const {
350 + FP32Vec16 result(0.0f);
351 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
352 + result.reg.val[i] = reg.val[i] - b.reg.val[i];
353 + }
354 + return result;
355 + }
356 +
357 + FP32Vec16 operator/(const FP32Vec16& b) const {
358 + FP32Vec16 result(0.0f);
359 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
360 + result.reg.val[i] = reg.val[i] / b.reg.val[i];
361 + }
362 + return result;
363 + }
364 +
365 + FP32Vec16 max(const FP32Vec16& b) const {
366 + FP32Vec16 result(0.0f);
367 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
368 + result.reg.val[i] = __max(reg.val[i], b.reg.val[i]);
369 + }
370 + return result;
371 + }
372 +
373 + FP32Vec16 min(const FP32Vec16& b) const {
374 + FP32Vec16 result(0.0f);
375 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
376 + result.reg.val[i] = __min(reg.val[i], b.reg.val[i]);
377 + }
378 + return result;
379 + }
380 +
381 + FP32Vec16 abs() const {
382 + FP32Vec16 result(0.0f);
383 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
384 + result.reg.val[i] = __abs(reg.val[i]);
385 + }
386 + return result;
387 + }
388 +
389 + float reduce_sum() const {
390 + float result = 0.0f;
391 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
392 + result += reg.val[i];
393 + }
394 + return result;
395 + }
396 +
397 + float reduce_max() const {
398 + float result = reg.val[0];
399 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
400 + result = __max(reg.val[i], result);
401 + }
402 + return result;
403 + }
404 +
405 + float reduce_min() const {
406 + float result = reg.val[0];
407 + for (int i = 0; i < VEC_ELEM_NUM; ++i) {
408 + result = __min(reg.val[i], result);
409 + }
410 + return result;
411 + }
412 +
413 + template <int group_size>
414 + float reduce_sub_sum(int idx) {
415 + static_assert(VEC_ELEM_NUM % group_size == 0);
416 + float sum = 0.0;
417 + int start = idx * group_size;
418 + int end = (idx + 1) * group_size;
419 +
420 + for (; (start < VEC_ELEM_NUM) && (start < end); ++start) {
421 + sum += reg.val[start];
422 + }
423 +
424 + return sum;
425 + }
426 +
427 + void save(void* ptr) const { *reinterpret_cast<f32x16_t*>(ptr) = reg; }
428 +};
429 +
430 +template <typename T>
431 +struct VecType {
432 + using vec_type = void;
433 +};
434 +
435 +template <typename T>
436 +using vec_t = typename VecType<T>::vec_type;
437 +
438 +template <>
439 +struct VecType<float> {
440 + using vec_type = FP32Vec8;
441 +};
442 +
443 +template <>
444 +struct VecType<c10::Half> {
445 + using vec_type = FP16Vec8;
446 +};
447 +
448 +template <>
449 +struct VecType<c10::BFloat16> {
450 + using vec_type = BF16Vec8;
451 +};
452 +
453 +template <typename T>
454 +void storeFP32(float v, T* ptr) {
455 + *ptr = v;
456 +}
457 +
458 +/*
459 +template <> inline void storeFP32<c10::Half>(float v, c10::Half *ptr) {
460 + c10::Half __attribute__((__may_alias__)) *v_ptr =
461 + reinterpret_cast<c10::Half *>(&v);
462 + *ptr = *(v_ptr + 1);
463 +}
464 +*/
465 +
466 +template <>
467 +inline void storeFP32<c10::Half>(float v, c10::Half* ptr) {
468 + uint16_t fp16 = float_to_fp16(v);
469 + *reinterpret_cast<uint16_t*>(ptr) = fp16;
470 +}
471 +
472 +template <>
473 +inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16* ptr) {
474 + c10::BFloat16 __attribute__((__may_alias__))* v_ptr =
475 + reinterpret_cast<c10::BFloat16*>(&v);
476 + *ptr = *(v_ptr + 1);
477 +}
478 +
479 +inline FP16Vec16::FP16Vec16(const FP32Vec16& v) {
480 + int i = 0;
481 + for (i = 0; i < FP16Vec16::VEC_ELEM_NUM; ++i) {
482 + reg.val[i] = float_to_fp16(v.reg.val[i]);
483 + }
484 +}
485 +
486 +inline FP16Vec8 ::FP16Vec8(const FP32Vec8& v) {
487 + int i = 0;
488 + for (i = 0; i < FP16Vec8::VEC_ELEM_NUM; ++i) {
489 + reg.val[i] = float_to_fp16(v.reg.val[i]);
490 + }
491 +}
492 +
493 +inline void fma(FP32Vec16& acc, FP32Vec16& a, FP32Vec16& b) {
494 + acc = acc + a * b;
495 +}
496 +
497 +inline BF16Vec8::BF16Vec8(const FP32Vec8& v) {
498 + int i = 0;
499 + for (i = 0; i < BF16Vec8::VEC_ELEM_NUM; ++i) {
500 + reg.val[i] = float_to_bf16(v.reg.val[i]);
501 + }
502 +}
503 +
504 +inline BF16Vec16::BF16Vec16(const FP32Vec16& v) {
505 + int i = 0;
506 + for (i = 0; i < BF16Vec16::VEC_ELEM_NUM; ++i) {
507 + reg.val[i] = float_to_bf16(v.reg.val[i]);
508 + }
509 +}
510 +
511 +inline void prefetch(const void* addr) { __builtin_prefetch(addr, 0, 3); }
512 +
513 +}; // namespace vec_op
csrc/cpu/float_convert.hpp new
+106
@@ -0,0 +1,106 @@
1 +
2 +static float bf16_to_float(uint16_t bf16) {
3 + uint32_t bits = static_cast<uint32_t>(bf16) << 16;
4 + float fp32;
5 + std::memcpy(&fp32, &bits, sizeof(fp32));
6 + return fp32;
7 +}
8 +
9 +static uint16_t float_to_bf16(float fp32) {
10 + uint32_t bits;
11 + std::memcpy(&bits, &fp32, sizeof(fp32));
12 + return static_cast<uint16_t>(bits >> 16);
13 +}
14 +
15 +/************************************************
16 + * Copyright (c) 2015 Princeton Vision Group
17 + * Licensed under the MIT license.
18 + * Codes below copied from
19 + * https://github.com/PrincetonVision/marvin/tree/master/tools/tensorIO_matlab
20 + *************************************************/
21 +static uint16_t float_to_fp16(float fp32) {
22 + uint16_t fp16;
23 +
24 + unsigned x;
25 + unsigned u, remainder, shift, lsb, lsb_s1, lsb_m1;
26 + unsigned sign, exponent, mantissa;
27 +
28 + std::memcpy(&x, &fp32, sizeof(fp32));
29 + u = (x & 0x7fffffff);
30 +
31 + // Get rid of +NaN/-NaN case first.
32 + if (u > 0x7f800000) {
33 + fp16 = 0x7fffU;
34 + return fp16;
35 + }
36 +
37 + sign = ((x >> 16) & 0x8000);
38 +
39 + // Get rid of +Inf/-Inf, +0/-0.
40 + if (u > 0x477fefff) {
41 + fp16 = sign | 0x7c00U;
42 + return fp16;
43 + }
44 + if (u < 0x33000001) {
45 + fp16 = (sign | 0x0000);
46 + return fp16;
47 + }
48 +
49 + exponent = ((u >> 23) & 0xff);
50 + mantissa = (u & 0x7fffff);
51 +
52 + if (exponent > 0x70) {
53 + shift = 13;
54 + exponent -= 0x70;
55 + } else {
56 + shift = 0x7e - exponent;
57 + exponent = 0;
58 + mantissa |= 0x800000;
59 + }
60 + lsb = (1 << shift);
61 + lsb_s1 = (lsb >> 1);
62 + lsb_m1 = (lsb - 1);
63 +
64 + // Round to nearest even.
65 + remainder = (mantissa & lsb_m1);
66 + mantissa >>= shift;
67 + if (remainder > lsb_s1 || (remainder == lsb_s1 && (mantissa & 0x1))) {
68 + ++mantissa;
69 + if (!(mantissa & 0x3ff)) {
70 + ++exponent;
71 + mantissa = 0;
72 + }
73 + }
74 +
75 + fp16 = (sign | (exponent << 10) | mantissa);
76 +
77 + return fp16;
78 +}
79 +
80 +static float fp16_to_float(uint16_t fp16) {
81 + unsigned sign = ((fp16 >> 15) & 1);
82 + unsigned exponent = ((fp16 >> 10) & 0x1f);
83 + unsigned mantissa = ((fp16 & 0x3ff) << 13);
84 + int temp;
85 + float fp32;
86 + if (exponent == 0x1f) { /* NaN or Inf */
87 + mantissa = (mantissa ? (sign = 0, 0x7fffff) : 0);
88 + exponent = 0xff;
89 + } else if (!exponent) { /* Denorm or Zero */
90 + if (mantissa) {
91 + unsigned int msb;
92 + exponent = 0x71;
93 + do {
94 + msb = (mantissa & 0x400000);
95 + mantissa <<= 1; /* normalize */
96 + --exponent;
97 + } while (!msb);
98 + mantissa &= 0x7fffff; /* 1.mantissa is implicit */
99 + }
100 + } else {
101 + exponent += 0x70;
102 + }
103 + temp = ((sign << 31) | (exponent << 23) | mantissa);
104 + std::memcpy(&fp32, &temp, sizeof(temp));
105 + return fp32;
106 +}
vllm/platforms/interface.py
+3
@@ -85,6 +85,7 @@ class CpuArchEnum(enum.Enum):
85 ARM = enum.auto()
86 POWERPC = enum.auto()
87 S390X = enum.auto()
88 + RISCV = enum.auto()
89 OTHER = enum.auto()
90 UNKNOWN = enum.auto()
91
@@ -374,6 +375,8 @@ class Platform:
375 return CpuArchEnum.POWERPC
376 elif machine == "s390x":
377 return CpuArchEnum.S390X
378 + elif machine.startswith("riscv"):
379 + return CpuArchEnum.RISCV
380
381 return CpuArchEnum.OTHER if machine else CpuArchEnum.UNKNOWN
382