A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 04:45 UTC
[Misc] Configurable timeout for execute_model RPC calls via env var (#19544)
Signed-off-by: jinqinn <goodqinjin@163.com>
jinqinn committed
Jun 23, 2025 at 11:36 UTC
f39ab2d4bde85e169b85ea3555dc4b74224b3929
2 files changed
+12
-11
vllm/envs.py
+6
@@ -130,6 +130,7 @@ if TYPE_CHECKING:
130
VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS: int = 1
131
VLLM_SLEEP_WHEN_IDLE: bool = False
132
VLLM_MQ_MAX_CHUNK_BYTES_MB: int = 16
133
+ VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: int = 300
134
VLLM_KV_CACHE_LAYOUT: Optional[str] = None
135
VLLM_COMPUTE_NANS_IN_LOGITS: bool = False
136
@@ -897,6 +898,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
898
"VLLM_MQ_MAX_CHUNK_BYTES_MB":
899
lambda: int(os.getenv("VLLM_MQ_MAX_CHUNK_BYTES_MB", "16")),
900
901
+ # Timeout in seconds for execute_model RPC calls in multiprocessing
902
+ # executor (only applies when TP > 1).
903
+ "VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS":
904
+ lambda: int(os.getenv("VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS", "300")),
905
+
906
# KV Cache layout used throughout vllm.
907
# Some common values are:
908
# - NHD
vllm/v1/executor/multiproc_executor.py
+6
-11
@@ -37,11 +37,6 @@ from vllm.worker.worker_base import WorkerWrapperBase
37
38
logger = init_logger(__name__)
39
40
-POLLING_TIMEOUT_MS = 5000
41
-POLLING_TIMEOUT_S = POLLING_TIMEOUT_MS // 1000
42
-
43
-EXECUTE_MODEL_TIMEOUT_S = 300
44
-
40
41
class MultiprocExecutor(Executor):
42
@@ -160,12 +155,12 @@ class MultiprocExecutor(Executor):
155
self,
156
scheduler_output,
157
) -> Union[ModelRunnerOutput, Future[ModelRunnerOutput]]:
163
- (output, ) = self.collective_rpc("execute_model",
164
- args=(scheduler_output, ),
165
- unique_reply_rank=self.output_rank,
166
- non_block=self.max_concurrent_batches
167
- > 1,
168
- timeout=EXECUTE_MODEL_TIMEOUT_S)
158
+ (output, ) = self.collective_rpc(
159
+ "execute_model",
160
+ args=(scheduler_output, ),
161
+ unique_reply_rank=self.output_rank,
162
+ non_block=self.max_concurrent_batches > 1,
163
+ timeout=envs.VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS)
164
return output
165
166
def collective_rpc(self,