A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Misc] Fix seq_lens for graph capture (#23175)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Aug 19, 2025 at 03:58 UTC
40f26734b92ededc6e604fa3296c61248b8cc902
1 file changed
+2
-4
vllm/v1/worker/gpu_model_runner.py
+2
-4
@@ -2317,15 +2317,13 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
2317
2318
# If force_attention is True, we always capture attention. Otherwise,
2319
# it only happens for cudagraph_runtime_mode=FULL.
2320
- if force_attention or cudagraph_runtime_mode == \
2321
- CUDAGraphMode.FULL:
2320
+ if force_attention or cudagraph_runtime_mode == CUDAGraphMode.FULL:
2321
attn_metadata = {}
2322
2323
# Make sure max_model_len is used at the graph capture time.
2324
self.seq_lens_np[:num_reqs] = self.max_model_len
2325
self.seq_lens_np[num_reqs:] = 0
2327
- self.seq_lens[:num_reqs].copy_(self.seq_lens_cpu[:num_reqs],
2328
- non_blocking=True)
2326
+ self.seq_lens.copy_(self.seq_lens_cpu, non_blocking=True)
2327
2328
for kv_cache_group_id, kv_cache_group_spec in enumerate(
2329
self.kv_cache_config.kv_cache_groups):