A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
[V1] Fix _pickle.PicklingError: Can't pickle <class 'transformers_modules.deepseek-ai.DeepSeek-V2-Lite... (#18640)
Signed-off-by: Seiji Eicher <seiji@anyscale.com>
Seiji Eicher committed
May 24, 2025 at 20:07 UTC
7891fdf0c64777afba3c3a8dc93447600a0c5bba
1 file changed
+5
vllm/v1/engine/async_llm.py
+5
@@ -20,6 +20,8 @@ from vllm.outputs import RequestOutput
20
from vllm.pooling_params import PoolingParams
21
from vllm.prompt_adapter.request import PromptAdapterRequest
22
from vllm.sampling_params import SamplingParams
23
+from vllm.transformers_utils.config import (
24
+ maybe_register_config_serialize_by_value)
25
from vllm.transformers_utils.tokenizer import AnyTokenizer
26
from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
27
from vllm.usage.usage_lib import UsageContext
@@ -80,6 +82,9 @@ class AsyncLLM(EngineClient):
82
"AsyncLLMEngine.from_vllm_config(...) or explicitly set "
83
"VLLM_USE_V1=0 or 1 and report this issue on Github.")
84
85
+ # Ensure we can serialize custom transformer configs
86
+ maybe_register_config_serialize_by_value()
87
+
88
self.model_config = vllm_config.model_config
89
self.vllm_config = vllm_config
90
self.log_requests = log_requests