A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 00:15 UTC
[Model Runner V2] Add model_state inputs to CUDA graph capture (#36544)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Woosuk Kwon committed
Mar 9, 2026 at 15:14 UTC
2a194ddd72a0cc5b6c404a694a64197d0c572f5b
1 file changed
+1
vllm/v1/worker/gpu/cudagraph_utils.py
+1
@@ -320,6 +320,7 @@ class ModelCudaGraphManager(CudaGraphManager):
320
model_inputs = {
321
"input_ids": input_buffers.input_ids[:num_tokens],
322
"positions": input_buffers.positions[:num_tokens],
323
+ **model_state.prepare_dummy_inputs(num_reqs, num_tokens),
324
}
325
model_output = model(**model_inputs)
326
if self.use_aux_hidden_state_outputs: