A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[Bugfix] Register serializers for V0 MQ Engine (#15009)
Signed-off-by: simon-mo <simon.mo@hey.com>
Simon Mo committed
Mar 18, 2025 at 06:14 UTC
3b457143d2e91b9993fbb3dcdea31546bc32f704
2 files changed
+13
-3
vllm/engine/multiprocessing/engine.py
+8
-3
@@ -29,6 +29,8 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
29
# yapf: enable
30
from vllm.logger import init_logger
31
from vllm.outputs import RequestOutput
32
+from vllm.transformers_utils.config import (
33
+ maybe_register_config_serialize_by_value)
34
from vllm.usage.usage_lib import UsageContext
35
from vllm.worker.model_runner_base import InputProcessingError
36
@@ -42,12 +44,12 @@ class MQLLMEngine:
44
"""A multiprocessing wrapper for :class:`LLMEngine`.
45
46
This class is used to wrap the :class:`LLMEngine` class to enable use
45
- in concurrnet manner. It runs a background loop and uses zeromq to
47
+ in concurrnet manner. It runs a background loop and uses zeromq to
48
receive new requests and stream outputs incrementally via ipc.
47
-
49
+
50
The :class:`LLMEngine` generate or encode process is kicked off when a new
51
RPCProcessRequest is received by the input_socket.
50
-
52
+
53
The self.engine_loop checks the input_socket for new requests,
54
adds them to the LLMEngine if there are any, calls the internal
55
:class:`LLMEngine.step()`, and sends the RequestOutputs back over
@@ -428,6 +430,9 @@ def run_mp_engine(vllm_config: VllmConfig, usage_context: UsageContext,
430
ipc_path: str, disable_log_stats: bool,
431
disable_log_requests: bool, engine_alive):
432
try:
433
+ # Ensure we can serialize transformer config before spawning
434
+ maybe_register_config_serialize_by_value()
435
+
436
engine = MQLLMEngine.from_vllm_config(
437
vllm_config=vllm_config,
438
usage_context=usage_context,
vllm/entrypoints/openai/api_server.py
+5
@@ -82,6 +82,8 @@ from vllm.entrypoints.openai.serving_transcription import (
82
from vllm.entrypoints.openai.tool_parsers import ToolParserManager
83
from vllm.entrypoints.utils import load_aware_call, with_cancellation
84
from vllm.logger import init_logger
85
+from vllm.transformers_utils.config import (
86
+ maybe_register_config_serialize_by_value)
87
from vllm.usage.usage_lib import UsageContext
88
from vllm.utils import (FlexibleArgumentParser, get_open_zmq_ipc_path,
89
is_valid_ipv6_address, set_ulimit)
@@ -221,6 +223,9 @@ async def build_async_engine_client_from_engine_args(
223
# so we need to spawn a new process
224
context = multiprocessing.get_context("spawn")
225
226
+ # Ensure we can serialize transformer config before spawning
227
+ maybe_register_config_serialize_by_value()
228
+
229
# The Process can raise an exception during startup, which may
230
# not actually result in an exitcode being reported. As a result
231
# we use a shared variable to communicate the information.