A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bug] [Spec Dec]: Fix kv_cache dtype mismatch for Eagle3 drafter on FP8 target (#24505)
Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com>
vllmellm committed
Sep 16, 2025 at 12:45 UTC
8c546102658f97b10d13bcf25193b65edc6ea6ff
1 file changed
+9
-2
vllm/model_executor/models/llama_eagle3.py
+9
-2
@@ -9,7 +9,7 @@ import torch.nn as nn
9
from transformers import LlamaConfig
10
11
from vllm.compilation.decorators import support_torch_compile
12
-from vllm.config import VllmConfig
12
+from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config
13
from vllm.logger import init_logger
14
from vllm.model_executor.layers.layernorm import RMSNorm
15
from vllm.model_executor.layers.linear import QKVParallelLinear
@@ -33,10 +33,14 @@ class LlamaDecoderLayer(LlamaDecoderLayer):
33
def __init__(
34
self,
35
config: LlamaConfig,
36
+ cache_config: Optional[CacheConfig] = None,
37
quant_config: Optional[QuantizationConfig] = None,
38
prefix: str = "",
39
) -> None:
39
- super().__init__(config, quant_config=quant_config, prefix=prefix)
40
+ super().__init__(config,
41
+ cache_config=cache_config,
42
+ quant_config=quant_config,
43
+ prefix=prefix)
44
45
# override qkv
46
self.self_attn.qkv_proj = QKVParallelLinear(
@@ -114,6 +118,8 @@ class LlamaModel(nn.Module):
118
speculative_config.draft_model_config.hf_config
119
self.vocab_size = self.config.vocab_size
120
121
+ current_vllm_config = get_current_vllm_config()
122
+
123
self.embed_tokens = VocabParallelEmbedding(
124
self.config.vocab_size,
125
self.config.hidden_size,
@@ -123,6 +129,7 @@ class LlamaModel(nn.Module):
129
self.layers = nn.ModuleList([
130
LlamaDecoderLayer(
131
config=self.config,
132
+ cache_config=current_vllm_config.cache_config,
133
prefix=maybe_prefix(prefix, f"layers.{start_layer_id}"),
134
)
135
])