@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / add055e151
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Enhance model loader (#83)

Woosuk Kwon committed May 9, 2023 at 15:46 UTC add055e151f32f89dab5932d25e5285b2fc823f1
2 files changed +56 -42
cacheflow/core/server.py
+1 -1
@@ -12,8 +12,8 @@ from cacheflow.core.scheduler import Scheduler
12 from cacheflow.frontend.simple_frontend import SimpleFrontend
13 from cacheflow.logger import init_logger
14 from cacheflow.model_executor import get_memory_analyzer
15 -from cacheflow.sequence import SequenceGroup
15 from cacheflow.sampling_params import SamplingParams
16 +from cacheflow.sequence import SequenceGroup
17 from cacheflow.utils import get_gpu_memory, get_cpu_memory
18 from cacheflow.worker.controller import Controller, DeviceID
19
cacheflow/model_executor/model_loader.py
+55 -41
@@ -14,32 +14,51 @@ from cacheflow.model_executor.utils import get_torch_dtype
14 from cacheflow.model_executor.weight_utils import initialize_dummy_weights
15
16
17 -_MODELS = {
18 - 'gpt2': GPT2LMHeadModel,
19 - 'llama': LlamaForCausalLM,
20 - 'opt': OPTForCausalLM,
21 - 'stablelm': GPTNeoXForCausalLM,
22 - 'pythia': GPTNeoXForCausalLM,
23 - 'dolly-v2': GPTNeoXForCausalLM,
17 +# TODO(woosuk): Lazy-load the model classes.
18 +_MODEL_REGISTRY = {
19 + "GPT2LMHeadModel": GPT2LMHeadModel,
20 + "GPTNeoXForCausalLM": GPTNeoXForCausalLM,
21 + "LlamaForCausalLM": LlamaForCausalLM,
22 + "OPTForCausalLM": OPTForCausalLM,
23 }
24
25 _MEMORY_ANALYZERS = {
27 - 'gpt2': GPT2MemoryAnalyzer,
28 - 'llama': LlamaMemoryAnalyzer,
29 - 'opt': OPTMemoryAnalyzer,
30 - 'stablelm': GPTNeoXMemoryAnalyzer,
31 - 'pythia': GPTNeoXMemoryAnalyzer,
32 - 'dolly-v2': GPTNeoXMemoryAnalyzer,
26 + "GPT2LMHeadModel": GPT2MemoryAnalyzer,
27 + "GPTNeoXForCausalLM": GPTNeoXMemoryAnalyzer,
28 + "LlamaForCausalLM": LlamaMemoryAnalyzer,
29 + "OPTForCausalLM": OPTMemoryAnalyzer,
30 }
31
32
33 +def _get_model_architecture(config: PretrainedConfig) -> nn.Module:
34 + architectures = getattr(config, "architectures", [])
35 + for arch in architectures:
36 + if arch in _MODEL_REGISTRY:
37 + return _MODEL_REGISTRY[arch]
38 + raise ValueError(
39 + f"Model architectures {architectures} are not supported for now. "
40 + f"Supported architectures: {list(_MODEL_REGISTRY.keys())}"
41 + )
42 +
43 +
44 +def _get_memory_analyzer(config: PretrainedConfig) -> CacheFlowMemoryAnalyzer:
45 + architectures = getattr(config, "architectures", [])
46 + for arch in architectures:
47 + if arch in _MEMORY_ANALYZERS:
48 + return _MEMORY_ANALYZERS[arch]
49 + raise ValueError(
50 + f"Model architectures {architectures} are not supported for now. "
51 + f"Supported architectures: {list(_MEMORY_ANALYZERS.keys())}"
52 + )
53 +
54 +
55 def _get_dtype(config: PretrainedConfig, dtype: str) -> torch.dtype:
37 - # NOTE: getattr(config, 'torch_dtype', torch.float32) is not correct
56 + # NOTE: getattr(config, "torch_dtype", torch.float32) is not correct
57 # because config.torch_dtype can be None.
39 - config_dtype = getattr(config, 'torch_dtype', None)
58 + config_dtype = getattr(config, "torch_dtype", None)
59 if config_dtype is None:
60 config_dtype = torch.float32
42 - if dtype == 'default':
61 + if dtype == "default":
62 if config_dtype == torch.float32:
63 # Following the common practice, we use float16 for float32 models.
64 torch_dtype = torch.float16
@@ -51,7 +70,7 @@ def _get_dtype(config: PretrainedConfig, dtype: str) -> torch.dtype:
70 # TODO(woosuk): Allow using float16 for bfloat16 models and
71 # vice versa. Print a warning message and continue.
72 raise ValueError(
54 - f'Cannot use {torch_dtype} for {config_dtype} model.')
73 + f"Cannot use {torch_dtype} for {config_dtype} model.")
74 return torch_dtype
75
76
@@ -65,24 +84,21 @@ def get_model(
84 config = AutoConfig.from_pretrained(model_name)
85 torch_dtype = _get_dtype(config, dtype)
86 torch.set_default_dtype(torch_dtype)
68 - for model_class_name, model_class in _MODELS.items():
69 - if model_class_name in model_name:
70 - if use_dummy_weights:
71 - # Create a model instance.
72 - # The weights will be initialized as empty tensors.
73 - model = model_class(config)
74 - model = model.cuda()
75 - # NOTE(woosuk): For precise performance evaluation, we assign
76 - # random values to the weights.
77 - initialize_dummy_weights(model)
78 - else:
79 - # Create a model instance.
80 - model = model_class(config)
81 - # Load the weights from the cached or downloaded files.
82 - model.load_weights(model_name, cache_dir, use_np_cache)
83 - model = model.cuda()
84 - return model.eval(), torch_dtype
85 - raise ValueError(f'Unsupported model name: {model_name}')
87 + model_class = _get_model_architecture(config)
88 +
89 + # Create a model instance.
90 + # The weights will be initialized as empty tensors.
91 + model = model_class(config)
92 + if use_dummy_weights:
93 + model = model.cuda()
94 + # NOTE(woosuk): For accurate performance evaluation, we assign
95 + # random values to the weights.
96 + initialize_dummy_weights(model)
97 + else:
98 + # Load the weights from the cached or downloaded files.
99 + model.load_weights(model_name, cache_dir, use_np_cache)
100 + model = model.cuda()
101 + return model.eval(), torch_dtype
102
103
104 def get_memory_analyzer(
@@ -95,9 +111,7 @@ def get_memory_analyzer(
111 ) -> CacheFlowMemoryAnalyzer:
112 config = AutoConfig.from_pretrained(model_name)
113 torch_dtype = _get_dtype(config, dtype)
98 - for model_class, memory_analyzer in _MEMORY_ANALYZERS.items():
99 - if model_class in model_name:
100 - return memory_analyzer(
101 - model_name, block_size, torch_dtype, gpu_memory, cpu_memory,
102 - tensor_parallel_size)
103 - raise ValueError(f'Unsupported model name: {model_name}')
114 + memory_analyzer = _get_memory_analyzer(config)
115 + return memory_analyzer(
116 + model_name, block_size, torch_dtype, gpu_memory, cpu_memory,
117 + tensor_parallel_size)