A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 06:00 UTC
[Multi Modal] Add an env var for message queue max chunk bytes (#19242)
Signed-off-by: yZhen <yZhen@fb.com> Co-authored-by: yZhen <yZhen@fb.com>
jennyyyyzhen committed
Jun 8, 2025 at 06:39 UTC
cda10fa3e2bb69ea276d663e5369ba16ec42cebb
2 files changed
+12
-1
vllm/envs.py
+7
@@ -123,6 +123,7 @@ if TYPE_CHECKING:
123
VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: int = 163840
124
VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS: int = 1
125
VLLM_SLEEP_WHEN_IDLE: bool = False
126
+ VLLM_MQ_MAX_CHUNK_BYTES_MB: int = 16
127
128
129
def get_default_cache_root():
@@ -847,6 +848,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
848
# latency penalty when a request eventually comes.
849
"VLLM_SLEEP_WHEN_IDLE":
850
lambda: bool(int(os.getenv("VLLM_SLEEP_WHEN_IDLE", "0"))),
851
+
852
+ # Control the max chunk bytes (in MB) for the rpc message queue.
853
+ # Object larger than this threshold will be broadcast to worker
854
+ # processes via zmq.
855
+ "VLLM_MQ_MAX_CHUNK_BYTES_MB":
856
+ lambda: int(os.getenv("VLLM_MQ_MAX_CHUNK_BYTES_MB", "16")),
857
}
858
859
# --8<-- [end:env-vars-definition]
vllm/v1/executor/multiproc_executor.py
+5
-1
@@ -20,6 +20,7 @@ from typing import Any, Callable, Optional, Union, cast
20
21
import cloudpickle
22
23
+import vllm.envs as envs
24
from vllm.config import VllmConfig
25
from vllm.distributed import (destroy_distributed_environment,
26
destroy_model_parallel)
@@ -72,7 +73,10 @@ class MultiprocExecutor(Executor):
73
74
# Initialize worker and set up message queues for SchedulerOutputs
75
# and ModelRunnerOutputs
75
- self.rpc_broadcast_mq = MessageQueue(self.world_size, self.world_size)
76
+ max_chunk_bytes = envs.VLLM_MQ_MAX_CHUNK_BYTES_MB * 1024 * 1024
77
+ self.rpc_broadcast_mq = MessageQueue(self.world_size,
78
+ self.world_size,
79
+ max_chunk_bytes=max_chunk_bytes)
80
scheduler_output_handle = self.rpc_broadcast_mq.export_handle()
81
82
# Create workers