@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 16b24e7dcd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Bugfix] Bandaid fix for speculative decoding tests (#9327)

Tyler Michael Smith committed Oct 13, 2024 at 19:02 UTC 16b24e7dcd8da5f2ac50f149daa77288fa8c14d7
1 file changed +18 -3
vllm/worker/model_runner.py
+18 -3
@@ -17,6 +17,7 @@ import torch.nn as nn
17 import vllm.envs as envs
18 from vllm.attention import AttentionMetadata, get_attn_backend
19 from vllm.attention.backends.abstract import AttentionState
20 +from vllm.attention.backends.utils import CommonAttentionState
21 from vllm.compilation.compile_context import set_compile_context
22 from vllm.compilation.levels import CompilationLevel
23 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
@@ -1001,6 +1002,17 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1002 self.graph_block_tables = np.zeros(
1003 (self.max_batchsize_to_capture, self.get_max_block_per_batch()),
1004 dtype=np.int32)
1005 +
1006 + # Attention-free but stateful models like Mamba need a placeholder attn
1007 + # backend, as the attention metadata is needed to manage internal state.
1008 + # However we must bypass attention selection altogether for some models
1009 + # used for speculative decoding to avoid a divide-by-zero in
1010 + # model_config.get_head_size()
1011 + num_attn_heads = self.model_config.get_num_attention_heads(
1012 + self.parallel_config)
1013 + needs_attn_backend = (num_attn_heads != 0
1014 + or self.model_config.is_attention_free)
1015 +
1016 self.attn_backend = get_attn_backend(
1017 self.model_config.get_head_size(),
1018 self.model_config.get_sliding_window(),
@@ -1008,9 +1020,12 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1020 self.kv_cache_dtype,
1021 self.block_size,
1022 self.model_config.is_attention_free,
1011 - )
1012 - self.attn_state = self.attn_backend.get_state_cls()(
1013 - weakref.proxy(self))
1023 + ) if needs_attn_backend else None
1024 + if self.attn_backend:
1025 + self.attn_state = self.attn_backend.get_state_cls()(
1026 + weakref.proxy(self))
1027 + else:
1028 + self.attn_state = CommonAttentionState(weakref.proxy(self))
1029
1030 # Multi-modal data support
1031 self.input_registry = input_registry