A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[Bugfix] Fix chunked prefill for GGUF (#14666)
Signed-off-by: SzymonOzog <szymon.ozog@aleph-alpha.com>
Szymon Ożóg committed
Mar 13, 2025 at 08:19 UTC
55211b01e87a02bdd0045b455715dfe508580738
1 file changed
+7
vllm/model_executor/layers/quantization/gguf.py
+7
@@ -98,6 +98,13 @@ MMQ_QUANT_TYPES = STANDARD_QUANT_TYPES | KQUANT_TYPES
98
99
def _fuse_mul_mat(x: torch.Tensor, qweight: torch.Tensor,
100
qweight_type: int) -> torch.Tensor:
101
+ # HACK: when doing chunked prefill we don't generate output tokens
102
+ # so input to logits generator is empty which causes invalid parameter
103
+ if x.shape[0] == 0:
104
+ return torch.empty(x.shape[0],
105
+ qweight.shape[0],
106
+ dtype=x.dtype,
107
+ device=x.device)
108
# there is no need to call any kernel for fp16/bf16
109
if qweight_type in UNQUANTIZED_TYPES:
110
return x @ qweight.T