@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 23fea8714a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Bugfix] Fix try-catch conditions to import correct Flash Attention Backend in Draft Model (#9101)

TJian committed Oct 5, 2024 at 22:00 UTC 23fea8714a1e90f018163e0eee59d73bc5a500e7
1 file changed +10 -5
vllm/spec_decode/draft_model_runner.py
+10 -5
@@ -6,11 +6,16 @@ from vllm.forward_context import set_forward_context
6 from vllm.model_executor.layers.sampler import SamplerOutput
7
8 try:
9 - from vllm.attention.backends.flash_attn import FlashAttentionMetadata
10 -except ModuleNotFoundError:
11 - # vllm_flash_attn is not installed, use the identical ROCm FA metadata
12 - from vllm.attention.backends.rocm_flash_attn import (
13 - ROCmFlashAttentionMetadata as FlashAttentionMetadata)
9 + try:
10 + from vllm.attention.backends.flash_attn import FlashAttentionMetadata
11 + except (ModuleNotFoundError, ImportError):
12 + # vllm_flash_attn is not installed, try the ROCm FA metadata
13 + from vllm.attention.backends.rocm_flash_attn import (
14 + ROCmFlashAttentionMetadata as FlashAttentionMetadata)
15 +except (ModuleNotFoundError, ImportError) as err:
16 + raise RuntimeError(
17 + "Draft model speculative decoding currently only supports"
18 + "CUDA and ROCm flash attention backend.") from err
19
20 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
21 ModelConfig, ObservabilityConfig, ParallelConfig,