@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 65ea2ddf17
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

feat(config): support parsing torch.dtype (#1641)

Signed-off-by: Aaron <29749331+aarnphm@users.noreply.github.com>

Aaron Pham committed Nov 16, 2023 at 04:31 UTC 65ea2ddf172a7234017f11d161ce87141deff3a2
1 file changed +18 -12
vllm/config.py
+18 -12
@@ -1,4 +1,4 @@
1 -from typing import Optional
1 +from typing import Optional, Union
2
3 import torch
4 from transformers import PretrainedConfig
@@ -58,7 +58,7 @@ class ModelConfig:
58 trust_remote_code: bool,
59 download_dir: Optional[str],
60 load_format: str,
61 - dtype: str,
61 + dtype: Union[str, torch.dtype],
62 seed: int,
63 revision: Optional[str] = None,
64 tokenizer_revision: Optional[str] = None,
@@ -331,7 +331,7 @@ _STR_DTYPE_TO_TORCH_DTYPE = {
331
332 def _get_and_verify_dtype(
333 config: PretrainedConfig,
334 - dtype: str,
334 + dtype: Union[str, torch.dtype],
335 ) -> torch.dtype:
336 # NOTE: getattr(config, "torch_dtype", torch.float32) is not correct
337 # because config.torch_dtype can be None.
@@ -339,17 +339,23 @@ def _get_and_verify_dtype(
339 if config_dtype is None:
340 config_dtype = torch.float32
341
342 - dtype = dtype.lower()
343 - if dtype == "auto":
344 - if config_dtype == torch.float32:
345 - # Following the common practice, we use float16 for float32 models.
346 - torch_dtype = torch.float16
342 + if isinstance(dtype, str):
343 + dtype = dtype.lower()
344 + if dtype == "auto":
345 + if config_dtype == torch.float32:
346 + # Following the common practice, we use float16 for float32
347 + # models.
348 + torch_dtype = torch.float16
349 + else:
350 + torch_dtype = config_dtype
351 else:
348 - torch_dtype = config_dtype
352 + if dtype not in _STR_DTYPE_TO_TORCH_DTYPE:
353 + raise ValueError(f"Unknown dtype: {dtype}")
354 + torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype]
355 + elif isinstance(dtype, torch.dtype):
356 + torch_dtype = dtype
357 else:
350 - if dtype not in _STR_DTYPE_TO_TORCH_DTYPE:
351 - raise ValueError(f"Unknown dtype: {dtype}")
352 - torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype]
358 + raise ValueError(f"Unknown dtype: {dtype}")
359
360 # Verify the dtype.
361 if torch_dtype != config_dtype: