A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 20:45 UTC
[BugFix] Fix incorrect preallocated sampled_token_ids tensor size (#28025)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Nov 4, 2025 at 07:38 UTC
5a0a6dfd55e1b9b2b518e0d2e91bd2c1241a7694
1 file changed
+1
-1
vllm/v1/worker/gpu_model_runner.py
+1
-1
@@ -524,7 +524,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
524
self._draft_token_ids: list[list[int]] | torch.Tensor | None = None
525
self.transfer_event = torch.cuda.Event()
526
self.sampled_token_ids_pinned_cpu = torch.empty(
527
- (self.max_model_len, 1),
527
+ (self.max_num_reqs, 1),
528
dtype=torch.int64,
529
device="cpu",
530
pin_memory=self.pin_memory,