A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 06:15 UTC
[Fix] Add missing stubs from cpu fp8 attention changes (#41387)
Signed-off-by: Li, Tianmu <tianmu.li@intel.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
Tianmu Li committed
May 5, 2026 at 21:16 UTC
e47c98ef7a38792996e452ef53914e21e41928e9
4 files changed
+25
csrc/cpu/cpu_types_arm.hpp
+4
@@ -486,6 +486,10 @@ struct FP32Vec16 : public VectorizedRegWrapper<FP32Vec16, 4, float> {
486
487
explicit FP32Vec16(const BF16Vec8& v) : FP32Vec16(FP32Vec8(v)) {};
488
489
+ // FP8 stub: dead code on ARM (fp8 KV cache is x86-only), needed for
490
+ // load_b_pair_vec template to compile on all platforms.
491
+ explicit FP32Vec16(const BF16Vec32&, int) : Base() {}
492
+
493
explicit FP32Vec16(const FP16Vec16& v) {
494
reg.val[0] = Vectorized<float>(vcvt_f32_f16(vget_low_f16(v.reg.val[0])));
495
reg.val[1] = Vectorized<float>(vcvt_f32_f16(vget_high_f16(v.reg.val[0])));
csrc/cpu/cpu_types_scalar.hpp
+10
@@ -6,6 +6,9 @@
6
7
namespace vec_op {
8
9
+struct fp8_e4m3_tag {};
10
+struct fp8_e5m2_tag {};
11
+
12
#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \
13
AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
14
AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) \
@@ -145,6 +148,9 @@ struct BF16Vec32 : public Vec<BF16Vec32> {
148
}
149
150
void save(void* ptr) const { *reinterpret_cast<f16x32_t*>(ptr) = reg; }
151
+
152
+ explicit BF16Vec32(const uint8_t*, fp8_e4m3_tag) : reg{} {}
153
+ explicit BF16Vec32(const uint8_t*, fp8_e5m2_tag) : reg{} {}
154
};
155
156
struct FP32Vec4 : public Vec<FP32Vec4> {
@@ -302,6 +308,10 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
308
309
FP32Vec16(const BF16Vec8& v) : FP32Vec16(FP32Vec8(v)) {};
310
311
+ // FP8 stub: dead code on scalar path (fp8 KV cache is x86-only), needed for
312
+ // load_b_pair_vec template to compile on all platforms.
313
+ explicit FP32Vec16(const BF16Vec32&, int) : reg{} {}
314
+
315
FP32Vec16 operator*(const FP32Vec16& b) const {
316
f32x16_t ret;
317
unroll_loop<int, VEC_ELEM_NUM>(
csrc/cpu/cpu_types_vsx.hpp
+7
@@ -146,6 +146,9 @@ struct BF16Vec32 : public Vec<BF16Vec32> {
146
: reg({vec8_data.reg, vec8_data.reg, vec8_data.reg, vec8_data.reg}) {}
147
148
void save(void* ptr) const { *reinterpret_cast<ss16x8x4_t*>(ptr) = reg; }
149
+
150
+ explicit BF16Vec32(const uint8_t*, fp8_e4m3_tag) : reg{} {}
151
+ explicit BF16Vec32(const uint8_t*, fp8_e5m2_tag) : reg{} {}
152
};
153
154
struct FP32Vec4 : public Vec<FP32Vec4> {
@@ -408,6 +411,10 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
411
412
explicit FP32Vec16(const BF16Vec8& v) : FP32Vec16(FP32Vec8(v)) {}
413
414
+ // FP8 stub: dead code on PowerPC (fp8 KV cache is x86-only), needed for
415
+ // load_b_pair_vec template to compile on all platforms.
416
+ explicit FP32Vec16(const BF16Vec32&, int) : reg{} {}
417
+
418
explicit FP32Vec16(const INT32Vec16& v) {
419
reg.val[0] = vec_ctf(v.reg.val[0], 0);
420
reg.val[1] = vec_ctf(v.reg.val[1], 0);
csrc/cpu/cpu_types_vxe.hpp
+4
@@ -688,6 +688,10 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
688
689
explicit FP32Vec16(const BF16Vec8& v) : FP32Vec16(FP32Vec8(v)) {}
690
691
+ // FP8 stub: dead code on s390x (fp8 KV cache is x86-only), needed for
692
+ // load_b_pair_vec template to compile on all platforms.
693
+ explicit FP32Vec16(const BF16Vec32&, int) : reg{} {}
694
+
695
FP32Vec16 operator*(const FP32Vec16& b) const {
696
return FP32Vec16(f32x4x4_t({vec_mul(reg.val[0], b.reg.val[0]),
697
vec_mul(reg.val[1], b.reg.val[1]),