A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 05:00 UTC
[Model Runner V2] Fix unintended CPU-GPU sync in make_dummy (#34667)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Woosuk Kwon committed
Feb 16, 2026 at 19:00 UTC
d74278fb676cbafc835fab9e970f6bcc9fd5413d
1 file changed
+1
-1
vllm/v1/worker/gpu/input_batch.py
+1
-1
@@ -108,7 +108,7 @@ class InputBatch:
108
query_start_loc_np = np.empty(num_reqs + 1, dtype=np.int32)
109
query_start_loc_np[0] = 0
110
np.cumsum(num_scheduled_tokens, out=query_start_loc_np[1:])
111
- input_buffers.query_start_loc[0] = 0
111
+ input_buffers.query_start_loc[:1] = 0
112
torch.cumsum(
113
seq_lens, dim=0, out=input_buffers.query_start_loc[1 : num_reqs + 1]
114
)