A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Perf][Async Scheduling] Remove CPU->GPU sync in dummy_run (#27455)
Signed-off-by: Lehua Ding <lehuading@tencent.com>
Lehua Ding committed
Oct 25, 2025 at 04:45 UTC
040242820069b46aac9c7287fac2497b2d3c1124
1 file changed
+4
-1
vllm/v1/worker/gpu_model_runner.py
+4
-1
@@ -3492,7 +3492,10 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
3492
self.eplb_step(is_dummy=True, is_profile=is_profile)
3493
3494
logit_indices = np.cumsum(num_scheduled_tokens) - 1
3495
- return hidden_states, hidden_states[logit_indices]
3495
+ logit_indices_device = torch.from_numpy(logit_indices).to(
3496
+ self.device, non_blocking=True
3497
+ )
3498
+ return hidden_states, hidden_states[logit_indices_device]
3499
3500
@torch.inference_mode()
3501
def _dummy_sampler_run(