A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 02:30 UTC
[Model Runner V2] Remove unnecessary copies in PW CUDA graph capture (#34849)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Woosuk Kwon committed
Feb 18, 2026 at 15:52 UTC
c878b43b640bbd5a43e78593722b2fec361eaa05
1 file changed
+1
-3
vllm/v1/worker/gpu/cudagraph_utils.py
+1
-3
@@ -218,13 +218,11 @@ class CudaGraphManager:
218
batch_descriptor=batch_descriptor,
219
slot_mapping=slot_mappings,
220
):
221
- hidden_states = model(
221
+ model(
222
input_ids=input_ids,
223
positions=positions,
224
inputs_embeds=inputs_embeds,
225
)
226
- assert self.hidden_states is not None
227
- self.hidden_states[:num_tokens] = hidden_states
226
227
@torch.inference_mode()
228
def capture(