A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
Skip device and quant Pydantic validation to make plugin device work (#18843)
Signed-off-by: Yikun Jiang <yikunkero@gmail.com>
Yikun Jiang committed
May 29, 2025 at 11:12 UTC
3c49dbdd03f33fb938bc67230dbc2e8f536ed490
1 file changed
+2
-2
vllm/config.py
+2
-2
@@ -304,7 +304,7 @@ class ModelConfig:
304
- 25.6k -> 25,600"""
305
spec_target_max_model_len: Optional[int] = None
306
"""Specify the maximum length for spec decoding draft models."""
307
- quantization: Optional[QuantizationMethods] = None
307
+ quantization: SkipValidation[Optional[QuantizationMethods]] = None
308
"""Method used to quantize the weights. If `None`, we first check the
309
`quantization_config` attribute in the model config file. If that is
310
`None`, we assume the model weights are not quantized and use `dtype` to
@@ -2231,7 +2231,7 @@ Device = Literal["auto", "cuda", "neuron", "cpu", "tpu", "xpu", "hpu"]
2231
class DeviceConfig:
2232
"""Configuration for the device to use for vLLM execution."""
2233
2234
- device: Union[Device, torch.device] = "auto"
2234
+ device: SkipValidation[Union[Device, torch.device]] = "auto"
2235
"""Device type for vLLM execution.
2236
This parameter is deprecated and will be
2237
removed in a future release.