A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[BugFix] Fix MLA + V1 + TP==1 causing reinitialization of cuda context (#14910)
Lucas Wilkinson committed
Mar 16, 2025 at 23:35 UTC
1e799b7ec1b1c61952d2ae24c85ecf3fcb0f6de3
1 file changed
+1
-1
vllm/platforms/cuda.py
+1
-1
@@ -152,7 +152,7 @@ class CudaPlatformBase(Platform):
152
# here
153
use_flashmla = (envs.VLLM_ATTENTION_BACKEND is None \
154
or envs.VLLM_ATTENTION_BACKEND == "FLASHMLA")
155
- from vllm.attention.backends.flashmla import is_flashmla_supported
155
+ from vllm.attention.ops.flashmla import is_flashmla_supported
156
if use_flashmla and is_flashmla_supported()[0] \
157
and cache_config.block_size != 64:
158
cache_config.block_size = 64