A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
feat(config): support parsing torch.dtype (#1641)
Signed-off-by: Aaron <29749331+aarnphm@users.noreply.github.com>
Aaron Pham committed
Nov 16, 2023 at 04:31 UTC
65ea2ddf172a7234017f11d161ce87141deff3a2
1 file changed
+18
-12
vllm/config.py
+18
-12
@@ -1,4 +1,4 @@
1
-from typing import Optional
1
+from typing import Optional, Union
2
3
import torch
4
from transformers import PretrainedConfig
@@ -58,7 +58,7 @@ class ModelConfig:
58
trust_remote_code: bool,
59
download_dir: Optional[str],
60
load_format: str,
61
- dtype: str,
61
+ dtype: Union[str, torch.dtype],
62
seed: int,
63
revision: Optional[str] = None,
64
tokenizer_revision: Optional[str] = None,
@@ -331,7 +331,7 @@ _STR_DTYPE_TO_TORCH_DTYPE = {
331
332
def _get_and_verify_dtype(
333
config: PretrainedConfig,
334
- dtype: str,
334
+ dtype: Union[str, torch.dtype],
335
) -> torch.dtype:
336
# NOTE: getattr(config, "torch_dtype", torch.float32) is not correct
337
# because config.torch_dtype can be None.
@@ -339,17 +339,23 @@ def _get_and_verify_dtype(
339
if config_dtype is None:
340
config_dtype = torch.float32
341
342
- dtype = dtype.lower()
343
- if dtype == "auto":
344
- if config_dtype == torch.float32:
345
- # Following the common practice, we use float16 for float32 models.
346
- torch_dtype = torch.float16
342
+ if isinstance(dtype, str):
343
+ dtype = dtype.lower()
344
+ if dtype == "auto":
345
+ if config_dtype == torch.float32:
346
+ # Following the common practice, we use float16 for float32
347
+ # models.
348
+ torch_dtype = torch.float16
349
+ else:
350
+ torch_dtype = config_dtype
351
else:
348
- torch_dtype = config_dtype
352
+ if dtype not in _STR_DTYPE_TO_TORCH_DTYPE:
353
+ raise ValueError(f"Unknown dtype: {dtype}")
354
+ torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype]
355
+ elif isinstance(dtype, torch.dtype):
356
+ torch_dtype = dtype
357
else:
350
- if dtype not in _STR_DTYPE_TO_TORCH_DTYPE:
351
- raise ValueError(f"Unknown dtype: {dtype}")
352
- torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype]
358
+ raise ValueError(f"Unknown dtype: {dtype}")
359
360
# Verify the dtype.
361
if torch_dtype != config_dtype: