A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix] Fix try-catch conditions to import correct Flash Attention Backend in Draft Model (#9101)
TJian committed
Oct 5, 2024 at 22:00 UTC
23fea8714a1e90f018163e0eee59d73bc5a500e7
1 file changed
+10
-5
vllm/spec_decode/draft_model_runner.py
+10
-5
@@ -6,11 +6,16 @@ from vllm.forward_context import set_forward_context
6
from vllm.model_executor.layers.sampler import SamplerOutput
7
8
try:
9
- from vllm.attention.backends.flash_attn import FlashAttentionMetadata
10
-except ModuleNotFoundError:
11
- # vllm_flash_attn is not installed, use the identical ROCm FA metadata
12
- from vllm.attention.backends.rocm_flash_attn import (
13
- ROCmFlashAttentionMetadata as FlashAttentionMetadata)
9
+ try:
10
+ from vllm.attention.backends.flash_attn import FlashAttentionMetadata
11
+ except (ModuleNotFoundError, ImportError):
12
+ # vllm_flash_attn is not installed, try the ROCm FA metadata
13
+ from vllm.attention.backends.rocm_flash_attn import (
14
+ ROCmFlashAttentionMetadata as FlashAttentionMetadata)
15
+except (ModuleNotFoundError, ImportError) as err:
16
+ raise RuntimeError(
17
+ "Draft model speculative decoding currently only supports"
18
+ "CUDA and ROCm flash attention backend.") from err
19
20
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
21
ModelConfig, ObservabilityConfig, ParallelConfig,