@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ea7bd68d10
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[V1][Bugfix] Fix V1 TP trust-remote-code (#11182)

Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>

Tyler Michael Smith committed Dec 14, 2024 at 03:21 UTC ea7bd68d101884165ffd75c1fd6e94a97510f194
1 file changed +5
vllm/v1/engine/core.py
+5
@@ -14,6 +14,8 @@ from msgspec import msgpack
14 from vllm.config import CacheConfig, VllmConfig
15 from vllm.executor.multiproc_worker_utils import get_mp_context
16 from vllm.logger import init_logger
17 +from vllm.transformers_utils.config import (
18 + maybe_register_config_serialize_by_value)
19 from vllm.usage.usage_lib import UsageContext
20 from vllm.v1.core.scheduler import Scheduler
21 from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
@@ -241,6 +243,9 @@ class EngineCoreProc(EngineCore):
243 # processes to terminate without error
244 shutdown_requested = False
245
246 + # Ensure we can serialize transformer config after spawning
247 + maybe_register_config_serialize_by_value()
248 +
249 def signal_handler(signum, frame):
250 nonlocal shutdown_requested
251 if not shutdown_requested: