A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bug] Fix AttributeError: 'QKVParallelLinear' object has no attribute 'orig_dtype' (#25958)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Wentao Ye committed
Sep 30, 2025 at 14:13 UTC
e6a226efba6f11b0c0b32bc460f0f6fe315801d2
1 file changed
+1
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
+1
@@ -66,6 +66,7 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
66
output_size_per_partition = sum(output_partition_sizes)
67
layer.logical_widths = output_partition_sizes
68
layer.weight_block_size = None
69
+ layer.orig_dtype = params_dtype
70
71
if self.strategy == QuantizationStrategy.BLOCK:
72
assert self.weight_block_size is not None