@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 189ae23133
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Use dtype from model config & Add Dolly V2 (#63)

Woosuk Kwon committed May 4, 2023 at 03:05 UTC 189ae231336857bcc4c6f6157bf7868cdf56fb5f
2 files changed +33 -7
cacheflow/master/server.py
+5 -1
@@ -214,7 +214,11 @@ def add_server_arguments(parser: argparse.ArgumentParser):
214 help='save a numpy copy of model weights for faster loading')
215 parser.add_argument('--use-dummy-weights', action='store_true', help='use dummy values for model weights')
216 # NOTE(woosuk): FlashAttention does not support float32.
217 - parser.add_argument('--dtype', type=str, default='half', choices=['half', 'bfloat16'], help='data type')
217 + parser.add_argument('--dtype', type=str, default='default', choices=['default', 'half', 'bfloat16'],
218 + help=('data type for model weights and activations. '
219 + 'The "default" option will use FP16 precision '
220 + 'for FP32 and FP16 models, and BF16 precision '
221 + 'for BF16 models.'))
222 # Parallel arguments
223 parser.add_argument('--use-ray', action='store_true', help='use Ray for distributed serving, will be automatically set when using more than 1 GPU')
224 parser.add_argument('--pipeline-parallel-size', '-pp', type=int, default=1, help='number of pipeline stages')
cacheflow/models/model_utils.py
+28 -6
@@ -1,8 +1,9 @@
1 -from typing import Union, Optional
1 +from typing import Optional
2
3 import torch
4 import torch.nn as nn
5 from transformers import AutoConfig
6 +from transformers import PretrainedConfig
7
8 from cacheflow.models.memory_analyzer import CacheFlowMemoryAnalyzer
9 from cacheflow.models.memory_analyzer import GPT2MemoryAnalyzer
@@ -22,6 +23,7 @@ _MODELS = {
23 'opt': OPTForCausalLM,
24 'stablelm': GPTNeoXForCausalLM,
25 'pythia': GPTNeoXForCausalLM,
26 + 'dolly-v2': GPTNeoXForCausalLM,
27 }
28
29 _MEMORY_ANALYZERS = {
@@ -30,19 +32,38 @@ _MEMORY_ANALYZERS = {
32 'opt': OPTMemoryAnalyzer,
33 'stablelm': GPTNeoXMemoryAnalyzer,
34 'pythia': GPTNeoXMemoryAnalyzer,
35 + 'dolly-v2': GPTNeoXMemoryAnalyzer,
36 }
37
38
39 +def _get_dtype(config: PretrainedConfig, dtype: str) -> torch.dtype:
40 + config_dtype: torch.dtype = getattr(config, 'torch_dtype', torch.float32)
41 + if dtype == 'default':
42 + if config_dtype == torch.float32:
43 + # Following the common practice, we use float16 for float32 models.
44 + torch_dtype = torch.float16
45 + else:
46 + torch_dtype = config_dtype
47 + else:
48 + torch_dtype = get_torch_dtype(dtype)
49 + if torch_dtype != config_dtype and config_dtype != torch.float32:
50 + # TODO(woosuk): Allow using float16 for bfloat16 models and
51 + # vice versa. Print a warning message and continue.
52 + raise ValueError(
53 + f'Cannot use {torch_dtype} for {config_dtype} model.')
54 + return torch_dtype
55 +
56 +
57 def get_model(
58 model_name: str,
38 - dtype: Union[torch.dtype, str],
59 + dtype: str,
60 cache_dir: Optional[str],
61 use_dummy_weights: bool,
62 use_np_cache: bool,
63 ) -> nn.Module:
43 - torch_dtype = get_torch_dtype(dtype)
44 - torch.set_default_dtype(torch_dtype)
64 config = AutoConfig.from_pretrained(model_name)
65 + torch_dtype = _get_dtype(config, dtype)
66 + torch.set_default_dtype(torch_dtype)
67 for model_class_name, model_class in _MODELS.items():
68 if model_class_name in model_name:
69 if use_dummy_weights:
@@ -66,12 +87,13 @@ def get_model(
87 def get_memory_analyzer(
88 model_name: str,
89 block_size: int,
69 - dtype: Union[torch.dtype, str],
90 + dtype: str,
91 gpu_memory: int,
92 cpu_memory: int,
93 tensor_parallel_size: int = 1,
94 ) -> CacheFlowMemoryAnalyzer:
74 - torch_dtype = get_torch_dtype(dtype)
95 + config = AutoConfig.from_pretrained(model_name)
96 + torch_dtype = _get_dtype(config, dtype)
97 for model_class, memory_analyzer in _MEMORY_ANALYZERS.items():
98 if model_class in model_name:
99 return memory_analyzer(