A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 06:00 UTC
[Fix] Convert kv_transfer_config from dict to KVTransferConfig (#19262)
maobaolong committed
Jun 15, 2025 at 03:32 UTC
08500011d3b796bf24843aaa9e9349b088ab9241
1 file changed
+18
vllm/entrypoints/llm.py
+18
@@ -10,6 +10,7 @@ from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Optional, Union,
10
11
import cloudpickle
12
import torch.nn as nn
13
+from pydantic import ValidationError
14
from tqdm.auto import tqdm
15
from typing_extensions import TypeVar, deprecated
16
@@ -195,6 +196,23 @@ class LLM:
196
if isinstance(worker_cls, type):
197
kwargs["worker_cls"] = cloudpickle.dumps(worker_cls)
198
199
+ if "kv_transfer_config" in kwargs and isinstance(
200
+ kwargs["kv_transfer_config"], dict):
201
+ from vllm.config import KVTransferConfig
202
+ raw_config_dict = kwargs["kv_transfer_config"]
203
+ try:
204
+ kwargs["kv_transfer_config"] = KVTransferConfig(
205
+ **raw_config_dict)
206
+ except ValidationError as e:
207
+ logger.error(
208
+ "Failed to convert 'kv_transfer_config' dict to "
209
+ "KVTransferConfig object. Dict: %s. Error: %s",
210
+ raw_config_dict, e)
211
+ # Consider re-raising a more specific vLLM error or ValueError
212
+ # to provide better context to the user.
213
+ raise ValueError(
214
+ f"Invalid 'kv_transfer_config' provided: {e}") from e
215
+
216
if hf_overrides is None:
217
hf_overrides = {}
218