A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[V1][Bugfix] Fix V1 TP trust-remote-code (#11182)
Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>
Tyler Michael Smith committed
Dec 14, 2024 at 03:21 UTC
ea7bd68d101884165ffd75c1fd6e94a97510f194
1 file changed
+5
vllm/v1/engine/core.py
+5
@@ -14,6 +14,8 @@ from msgspec import msgpack
14
from vllm.config import CacheConfig, VllmConfig
15
from vllm.executor.multiproc_worker_utils import get_mp_context
16
from vllm.logger import init_logger
17
+from vllm.transformers_utils.config import (
18
+ maybe_register_config_serialize_by_value)
19
from vllm.usage.usage_lib import UsageContext
20
from vllm.v1.core.scheduler import Scheduler
21
from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
@@ -241,6 +243,9 @@ class EngineCoreProc(EngineCore):
243
# processes to terminate without error
244
shutdown_requested = False
245
246
+ # Ensure we can serialize transformer config after spawning
247
+ maybe_register_config_serialize_by_value()
248
+
249
def signal_handler(signum, frame):
250
nonlocal shutdown_requested
251
if not shutdown_requested: