A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Misc] Avoid redundant copy for encoder-only models (#24012)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Aug 31, 2025 at 21:02 UTC
8c742a66d1bdb92978a40f0420123bf0d07fd90d
1 file changed
+7
-7
vllm/v1/worker/gpu_model_runner.py
+7
-7
@@ -827,13 +827,13 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
827
blk_table_tensor = torch.zeros(
828
(num_reqs, 1),
829
dtype=torch.int32,
830
- pin_memory=self.pin_memory,
831
- device="cpu").to(self.device, non_blocking=True)
832
- slot_mapping = torch.zeros((total_num_scheduled_tokens, ),
833
- dtype=torch.int32,
834
- pin_memory=self.pin_memory,
835
- device="cpu").to(self.device,
836
- non_blocking=True)
830
+ device=self.device,
831
+ )
832
+ slot_mapping = torch.zeros(
833
+ (total_num_scheduled_tokens, ),
834
+ dtype=torch.int64,
835
+ device=self.device,
836
+ )
837
num_common_prefix_blocks = 0
838
else:
839
blk_table = self.input_batch.block_table[kv_cache_group_id]