A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 09:00 UTC
[Perf] Optimize async scheduling placeholder using empty (#32056)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Wentao Ye committed
Jan 9, 2026 at 19:46 UTC
e18464a57d145872899463838ca07050ea15141b
1 file changed
+4
-1
vllm/v1/engine/output_processor.py
+4
-1
@@ -31,6 +31,9 @@ from vllm.v1.metrics.stats import (
31
SchedulerStats,
32
)
33
34
+# shared empty CPU tensor used as a placeholder pooling output
35
+EMPTY_CPU_TENSOR = torch.empty(0, device="cpu")
36
+
37
38
class RequestOutputCollector:
39
"""
@@ -426,7 +429,7 @@ class OutputProcessor:
429
new_token_ids=[],
430
# Set pooling_output is not None to
431
# correctly enter the abort pooling branch
429
- pooling_output=torch.randn(0, device="cpu")
432
+ pooling_output=EMPTY_CPU_TENSOR
433
if req_state.detokenizer is None
434
else None,
435
finish_reason=FinishReason.ABORT,