A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
[Bugfix] Fix Llama GGUF initialization (#18717)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
May 26, 2025 at 22:49 UTC
a869baca73eb90ae7bd18402915dc4bfc36cf06b
1 file changed
+1
-1
vllm/model_executor/models/llama.py
+1
-1
@@ -208,7 +208,7 @@ class LlamaAttention(nn.Module):
208
quant_config: Optional[QuantizationConfig]) -> None:
209
is_neox_style = True
210
is_gguf = quant_config and quant_config.get_name() == "gguf"
211
- if is_gguf and self.config.model_type == "llama":
211
+ if is_gguf and config.model_type == "llama":
212
is_neox_style = False
213
214
self.rotary_emb = get_rope(