A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 12:15 UTC
[MRV2] Fix hanging issue with DeepSeek V3.2 by setting `skip_attn=False` (#39098)
Signed-off-by: WoosukKwon <woosuk.kwon@berkeley.edu> Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Woosuk Kwon committed
Apr 6, 2026 at 12:55 UTC
f186cfe75e452aeb76f5233da7392d51ee34d3ef
1 file changed
+10
-1
vllm/v1/worker/gpu/model_runner.py
+10
-1
@@ -391,12 +391,17 @@ class GPUModelRunner(LoRAModelRunnerMixin):
391
self,
392
num_tokens: int,
393
*args,
394
- skip_attn: bool = True,
394
+ skip_attn: bool = False,
395
uniform_decode: bool = False,
396
skip_eplb: bool = False,
397
is_profile: bool = False,
398
**kwargs,
399
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
400
+ if skip_attn and not is_profile:
401
+ raise ValueError(
402
+ "skip_attn must only be True for initial memory profiling."
403
+ )
404
+
405
# Create a dummy scheduler output.
406
num_reqs = min(num_tokens, self.max_num_reqs)
407
if uniform_decode:
@@ -988,6 +993,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
993
if not skip_attn_for_dummy_run:
994
block_tables, slot_mappings = self.prepare_dummy_attn(input_batch)
995
else:
996
+ assert batch_desc.cg_mode != CUDAGraphMode.FULL, (
997
+ "Attention metadata must be prepared for dummy runs when using "
998
+ "FULL cudagraph mode."
999
+ )
1000
block_tables = None
1001
slot_mappings = None
1002
# FIXME(woosuk): Fix warmup for LoRA.