A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Use dtype from model config & Add Dolly V2 (#63)
Woosuk Kwon committed
May 4, 2023 at 03:05 UTC
189ae231336857bcc4c6f6157bf7868cdf56fb5f
2 files changed
+33
-7
cacheflow/master/server.py
+5
-1
@@ -214,7 +214,11 @@ def add_server_arguments(parser: argparse.ArgumentParser):
214
help='save a numpy copy of model weights for faster loading')
215
parser.add_argument('--use-dummy-weights', action='store_true', help='use dummy values for model weights')
216
# NOTE(woosuk): FlashAttention does not support float32.
217
- parser.add_argument('--dtype', type=str, default='half', choices=['half', 'bfloat16'], help='data type')
217
+ parser.add_argument('--dtype', type=str, default='default', choices=['default', 'half', 'bfloat16'],
218
+ help=('data type for model weights and activations. '
219
+ 'The "default" option will use FP16 precision '
220
+ 'for FP32 and FP16 models, and BF16 precision '
221
+ 'for BF16 models.'))
222
# Parallel arguments
223
parser.add_argument('--use-ray', action='store_true', help='use Ray for distributed serving, will be automatically set when using more than 1 GPU')
224
parser.add_argument('--pipeline-parallel-size', '-pp', type=int, default=1, help='number of pipeline stages')
cacheflow/models/model_utils.py
+28
-6
@@ -1,8 +1,9 @@
1
-from typing import Union, Optional
1
+from typing import Optional
2
3
import torch
4
import torch.nn as nn
5
from transformers import AutoConfig
6
+from transformers import PretrainedConfig
7
8
from cacheflow.models.memory_analyzer import CacheFlowMemoryAnalyzer
9
from cacheflow.models.memory_analyzer import GPT2MemoryAnalyzer
@@ -22,6 +23,7 @@ _MODELS = {
23
'opt': OPTForCausalLM,
24
'stablelm': GPTNeoXForCausalLM,
25
'pythia': GPTNeoXForCausalLM,
26
+ 'dolly-v2': GPTNeoXForCausalLM,
27
}
28
29
_MEMORY_ANALYZERS = {
@@ -30,19 +32,38 @@ _MEMORY_ANALYZERS = {
32
'opt': OPTMemoryAnalyzer,
33
'stablelm': GPTNeoXMemoryAnalyzer,
34
'pythia': GPTNeoXMemoryAnalyzer,
35
+ 'dolly-v2': GPTNeoXMemoryAnalyzer,
36
}
37
38
39
+def _get_dtype(config: PretrainedConfig, dtype: str) -> torch.dtype:
40
+ config_dtype: torch.dtype = getattr(config, 'torch_dtype', torch.float32)
41
+ if dtype == 'default':
42
+ if config_dtype == torch.float32:
43
+ # Following the common practice, we use float16 for float32 models.
44
+ torch_dtype = torch.float16
45
+ else:
46
+ torch_dtype = config_dtype
47
+ else:
48
+ torch_dtype = get_torch_dtype(dtype)
49
+ if torch_dtype != config_dtype and config_dtype != torch.float32:
50
+ # TODO(woosuk): Allow using float16 for bfloat16 models and
51
+ # vice versa. Print a warning message and continue.
52
+ raise ValueError(
53
+ f'Cannot use {torch_dtype} for {config_dtype} model.')
54
+ return torch_dtype
55
+
56
+
57
def get_model(
58
model_name: str,
38
- dtype: Union[torch.dtype, str],
59
+ dtype: str,
60
cache_dir: Optional[str],
61
use_dummy_weights: bool,
62
use_np_cache: bool,
63
) -> nn.Module:
43
- torch_dtype = get_torch_dtype(dtype)
44
- torch.set_default_dtype(torch_dtype)
64
config = AutoConfig.from_pretrained(model_name)
65
+ torch_dtype = _get_dtype(config, dtype)
66
+ torch.set_default_dtype(torch_dtype)
67
for model_class_name, model_class in _MODELS.items():
68
if model_class_name in model_name:
69
if use_dummy_weights:
@@ -66,12 +87,13 @@ def get_model(
87
def get_memory_analyzer(
88
model_name: str,
89
block_size: int,
69
- dtype: Union[torch.dtype, str],
90
+ dtype: str,
91
gpu_memory: int,
92
cpu_memory: int,
93
tensor_parallel_size: int = 1,
94
) -> CacheFlowMemoryAnalyzer:
74
- torch_dtype = get_torch_dtype(dtype)
95
+ config = AutoConfig.from_pretrained(model_name)
96
+ torch_dtype = _get_dtype(config, dtype)
97
for model_class, memory_analyzer in _MEMORY_ANALYZERS.items():
98
if model_class in model_name:
99
return memory_analyzer(