A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix] Bandaid fix for speculative decoding tests (#9327)
Tyler Michael Smith committed
Oct 13, 2024 at 19:02 UTC
16b24e7dcd8da5f2ac50f149daa77288fa8c14d7
1 file changed
+18
-3
vllm/worker/model_runner.py
+18
-3
@@ -17,6 +17,7 @@ import torch.nn as nn
17
import vllm.envs as envs
18
from vllm.attention import AttentionMetadata, get_attn_backend
19
from vllm.attention.backends.abstract import AttentionState
20
+from vllm.attention.backends.utils import CommonAttentionState
21
from vllm.compilation.compile_context import set_compile_context
22
from vllm.compilation.levels import CompilationLevel
23
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
@@ -1001,6 +1002,17 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1002
self.graph_block_tables = np.zeros(
1003
(self.max_batchsize_to_capture, self.get_max_block_per_batch()),
1004
dtype=np.int32)
1005
+
1006
+ # Attention-free but stateful models like Mamba need a placeholder attn
1007
+ # backend, as the attention metadata is needed to manage internal state.
1008
+ # However we must bypass attention selection altogether for some models
1009
+ # used for speculative decoding to avoid a divide-by-zero in
1010
+ # model_config.get_head_size()
1011
+ num_attn_heads = self.model_config.get_num_attention_heads(
1012
+ self.parallel_config)
1013
+ needs_attn_backend = (num_attn_heads != 0
1014
+ or self.model_config.is_attention_free)
1015
+
1016
self.attn_backend = get_attn_backend(
1017
self.model_config.get_head_size(),
1018
self.model_config.get_sliding_window(),
@@ -1008,9 +1020,12 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1020
self.kv_cache_dtype,
1021
self.block_size,
1022
self.model_config.is_attention_free,
1011
- )
1012
- self.attn_state = self.attn_backend.get_state_cls()(
1013
- weakref.proxy(self))
1023
+ ) if needs_attn_backend else None
1024
+ if self.attn_backend:
1025
+ self.attn_state = self.attn_backend.get_state_cls()(
1026
+ weakref.proxy(self))
1027
+ else:
1028
+ self.attn_state = CommonAttentionState(weakref.proxy(self))
1029
1030
# Multi-modal data support
1031
self.input_registry = input_registry