A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Add Model Revision Support (#1014)
Co-authored-by: Jasmond Loh <Jasmond.Loh@hotmail.com> Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
Jasmond L committed
Sep 14, 2023 at 06:20 UTC
ab019eea7513eb1e26ead79cf162863f3f19e971
20 files changed
+75
-35
vllm/config.py
+6
-1
@@ -38,6 +38,9 @@ class ModelConfig:
38
will use FP16 precision for FP32 and FP16 models, and BF16 precision
39
for BF16 models.
40
seed: Random seed for reproducibility.
41
+ revision: The specific model version to use. It can be a branch name,
42
+ a tag name, or a commit id. If unspecified, will use the default
43
+ version.
44
max_model_len: Maximum length of a sequence (including prompt and
45
output). If None, will be derived from the model.
46
"""
@@ -52,6 +55,7 @@ class ModelConfig:
55
load_format: str,
56
dtype: str,
57
seed: int,
58
+ revision: Optional[str],
59
max_model_len: Optional[int] = None,
60
) -> None:
61
self.model = model
@@ -61,8 +65,9 @@ class ModelConfig:
65
self.download_dir = download_dir
66
self.load_format = load_format
67
self.seed = seed
68
+ self.revision = revision
69
65
- self.hf_config = get_config(model, trust_remote_code)
70
+ self.hf_config = get_config(model, trust_remote_code, revision)
71
self.dtype = _get_and_verify_dtype(self.hf_config, dtype)
72
self._verify_load_format()
73
self._verify_tokenizer_mode()
vllm/engine/arg_utils.py
+10
-1
@@ -28,6 +28,7 @@ class EngineArgs:
28
max_num_batched_tokens: int = 2560
29
max_num_seqs: int = 256
30
disable_log_stats: bool = False
31
+ revision: Optional[str] = None
32
33
def __post_init__(self):
34
if self.tokenizer is None:
@@ -49,6 +50,13 @@ class EngineArgs:
50
type=str,
51
default=EngineArgs.tokenizer,
52
help='name or path of the huggingface tokenizer to use')
53
+ parser.add_argument(
54
+ '--revision',
55
+ type=str,
56
+ default=None,
57
+ help='the specific model version to use. It can be a branch '
58
+ 'name, a tag name, or a commit id. If unspecified, will use '
59
+ 'the default version.')
60
parser.add_argument('--tokenizer-mode',
61
type=str,
62
default=EngineArgs.tokenizer_mode,
@@ -159,7 +167,8 @@ class EngineArgs:
167
model_config = ModelConfig(self.model, self.tokenizer,
168
self.tokenizer_mode, self.trust_remote_code,
169
self.download_dir, self.load_format,
162
- self.dtype, self.seed, self.max_model_len)
170
+ self.dtype, self.seed, self.revision,
171
+ self.max_model_len)
172
cache_config = CacheConfig(self.block_size,
173
self.gpu_memory_utilization,
174
self.swap_space)
vllm/engine/llm_engine.py
+3
-1
@@ -74,6 +74,7 @@ class LLMEngine:
74
f"model={model_config.model!r}, "
75
f"tokenizer={model_config.tokenizer!r}, "
76
f"tokenizer_mode={model_config.tokenizer_mode}, "
77
+ f"revision={model_config.revision}, "
78
f"trust_remote_code={model_config.trust_remote_code}, "
79
f"dtype={model_config.dtype}, "
80
f"download_dir={model_config.download_dir!r}, "
@@ -92,7 +93,8 @@ class LLMEngine:
93
self.tokenizer = get_tokenizer(
94
model_config.tokenizer,
95
tokenizer_mode=model_config.tokenizer_mode,
95
- trust_remote_code=model_config.trust_remote_code)
96
+ trust_remote_code=model_config.trust_remote_code,
97
+ revision=model_config.revision)
98
self.seq_counter = Counter()
99
100
# Create the parallel GPU workers.
vllm/entrypoints/llm.py
+2
@@ -38,6 +38,8 @@ class LLM:
38
However, if the `torch_dtype` in the config is `float32`, we will
39
use `float16` instead.
40
seed: The seed to initialize the random number generator for sampling.
41
+ revision: The specific model version to use. It can be a branch name,
42
+ a tag name, or a commit id.
43
"""
44
45
def __init__(
vllm/model_executor/model_loader.py
+1
-1
@@ -64,6 +64,6 @@ def get_model(model_config: ModelConfig) -> nn.Module:
64
else:
65
# Load the weights from the cached or downloaded files.
66
model.load_weights(model_config.model, model_config.download_dir,
67
- model_config.load_format)
67
+ model_config.load_format, model_config.revision)
68
model = model.cuda()
69
return model.eval()
vllm/model_executor/models/aquila.py
+3
-2
@@ -288,7 +288,8 @@ class AquilaForCausalLM(nn.Module):
288
def load_weights(self,
289
model_name_or_path: str,
290
cache_dir: Optional[str] = None,
291
- load_format: str = "auto"):
291
+ load_format: str = "auto",
292
+ revision: Optional[str] = None):
293
tp_size = get_tensor_model_parallel_world_size()
294
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
295
q_proj_shard_size = (self.config.hidden_size // tp_size)
@@ -305,7 +306,7 @@ class AquilaForCausalLM(nn.Module):
306
state_dict = self.state_dict()
307
308
for name, loaded_weight in hf_model_weights_iterator(
308
- model_name_or_path, cache_dir, load_format):
309
+ model_name_or_path, cache_dir, load_format, revision):
310
if "rotary_emb.inv_freq" in name:
311
continue
312
vllm/model_executor/models/baichuan.py
+3
-2
@@ -303,13 +303,14 @@ class BaiChuanBaseForCausalLM(nn.Module):
303
def load_weights(self,
304
model_name_or_path: str,
305
cache_dir: Optional[str] = None,
306
- load_format: str = "auto"):
306
+ load_format: str = "auto",
307
+ revision: Optional[str] = None):
308
tp_world_size = get_tensor_model_parallel_world_size()
309
tp_rank = get_tensor_model_parallel_rank()
310
state_dict = self.state_dict()
311
312
for name, loaded_weight in hf_model_weights_iterator(
312
- model_name_or_path, cache_dir, load_format):
313
+ model_name_or_path, cache_dir, load_format, revision):
314
if "rotary_emb.inv_freq" in name:
315
continue
316
vllm/model_executor/models/bloom.py
+3
-2
@@ -279,11 +279,12 @@ class BloomForCausalLM(nn.Module):
279
def load_weights(self,
280
model_name_or_path: str,
281
cache_dir: Optional[str] = None,
282
- load_format: str = "auto"):
282
+ load_format: str = "auto",
283
+ revision: Optional[str] = None):
284
tp_rank = get_tensor_model_parallel_rank()
285
state_dict = self.state_dict()
286
for name, loaded_weight in hf_model_weights_iterator(
286
- model_name_or_path, cache_dir, load_format):
287
+ model_name_or_path, cache_dir, load_format, revision):
288
if name == "lm_head.weight":
289
# Since hidden_states are parallelized, we need to
290
# load lm_head.weight in parallel.
vllm/model_executor/models/falcon.py
+3
-2
@@ -420,7 +420,8 @@ class FalconForCausalLM(nn.Module):
420
def load_weights(self,
421
model_name_or_path: str,
422
cache_dir: Optional[str] = None,
423
- load_format: str = "auto"):
423
+ load_format: str = "auto",
424
+ revision: Optional[str] = None):
425
tp_size = (get_tensor_model_parallel_world_size())
426
tp_rank = get_tensor_model_parallel_rank()
427
@@ -452,7 +453,7 @@ class FalconForCausalLM(nn.Module):
453
state_dict = self.state_dict()
454
455
for name, loaded_weight in hf_model_weights_iterator(
455
- model_name_or_path, cache_dir, load_format):
456
+ model_name_or_path, cache_dir, load_format, revision):
457
if "query_key_value" in name:
458
loaded_weight = convert_pyslice_to_tensor(loaded_weight)
459
loaded_weight_size = loaded_weight.size()
vllm/model_executor/models/gpt2.py
+3
-2
@@ -231,14 +231,15 @@ class GPT2LMHeadModel(nn.Module):
231
def load_weights(self,
232
model_name_or_path: str,
233
cache_dir: Optional[str] = None,
234
- load_format: str = "auto"):
234
+ load_format: str = "auto",
235
+ revision: Optional[str] = None):
236
tensor_model_parallel_world_size = (
237
get_tensor_model_parallel_world_size())
238
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
239
state_dict = self.state_dict()
240
241
for name, loaded_weight in hf_model_weights_iterator(
241
- model_name_or_path, cache_dir, load_format):
242
+ model_name_or_path, cache_dir, load_format, revision):
243
if "lm_head.weight" in name:
244
# GPT-2 ties the weights of the embedding layer and the final
245
# linear layer.
vllm/model_executor/models/gpt_bigcode.py
+3
-2
@@ -259,14 +259,15 @@ class GPTBigCodeForCausalLM(nn.Module):
259
def load_weights(self,
260
model_name_or_path: str,
261
cache_dir: Optional[str] = None,
262
- load_format: str = "auto"):
262
+ load_format: str = "auto",
263
+ revision: Optional[str] = None):
264
tensor_model_parallel_world_size = (
265
get_tensor_model_parallel_world_size())
266
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
267
state_dict = self.state_dict()
268
269
for name, loaded_weight in hf_model_weights_iterator(
269
- model_name_or_path, cache_dir, load_format):
270
+ model_name_or_path, cache_dir, load_format, revision):
271
if "lm_head.weight" in name:
272
# GPT-2 ties the weights of the embedding layer and the final
273
# linear layer.
vllm/model_executor/models/gpt_j.py
+3
-2
@@ -222,11 +222,12 @@ class GPTJForCausalLM(nn.Module):
222
def load_weights(self,
223
model_name_or_path: str,
224
cache_dir: Optional[str] = None,
225
- load_format: str = "auto"):
225
+ load_format: str = "auto",
226
+ revision: Optional[str] = None):
227
tp_rank = get_tensor_model_parallel_rank()
228
state_dict = self.state_dict()
229
for name, loaded_weight in hf_model_weights_iterator(
229
- model_name_or_path, cache_dir, load_format):
230
+ model_name_or_path, cache_dir, load_format, revision):
231
if "attn.bias" in name or "attn.masked_bias" in name:
232
continue
233
vllm/model_executor/models/gpt_neox.py
+3
-2
@@ -231,11 +231,12 @@ class GPTNeoXForCausalLM(nn.Module):
231
def load_weights(self,
232
model_name_or_path: str,
233
cache_dir: Optional[str] = None,
234
- load_format: str = "auto"):
234
+ load_format: str = "auto",
235
+ revision: Optional[str] = None):
236
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
237
state_dict = self.state_dict()
238
for name, loaded_weight in hf_model_weights_iterator(
238
- model_name_or_path, cache_dir, load_format):
239
+ model_name_or_path, cache_dir, load_format, revision):
240
if ("attention.bias" in name or "attention.masked_bias" in name
241
or "rotary_emb.inv_freq" in name):
242
continue
vllm/model_executor/models/internlm.py
+3
-2
@@ -233,12 +233,13 @@ class InternLMForCausalLM(nn.Module):
233
def load_weights(self,
234
model_name_or_path: str,
235
cache_dir: Optional[str] = None,
236
- load_format: str = "auto"):
236
+ load_format: str = "auto",
237
+ revision: Optional[str] = None):
238
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
239
state_dict = self.state_dict()
240
241
for name, loaded_weight in hf_model_weights_iterator(
241
- model_name_or_path, cache_dir, load_format):
242
+ model_name_or_path, cache_dir, load_format, revision):
243
if "rotary_emb.inv_freq" in name:
244
continue
245
vllm/model_executor/models/llama.py
+3
-2
@@ -271,7 +271,8 @@ class LlamaForCausalLM(nn.Module):
271
def load_weights(self,
272
model_name_or_path: str,
273
cache_dir: Optional[str] = None,
274
- load_format: str = "auto"):
274
+ load_format: str = "auto",
275
+ revision: Optional[str] = None):
276
tp_size = get_tensor_model_parallel_world_size()
277
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
278
q_proj_shard_size = (self.config.hidden_size // tp_size)
@@ -288,7 +289,7 @@ class LlamaForCausalLM(nn.Module):
289
state_dict = self.state_dict()
290
291
for name, loaded_weight in hf_model_weights_iterator(
291
- model_name_or_path, cache_dir, load_format):
292
+ model_name_or_path, cache_dir, load_format, revision):
293
if "rotary_emb.inv_freq" in name:
294
continue
295
vllm/model_executor/models/mpt.py
+3
-2
@@ -244,12 +244,13 @@ class MPTForCausalLM(nn.Module):
244
def load_weights(self,
245
model_name_or_path: str,
246
cache_dir: Optional[str] = None,
247
- load_format: str = "auto"):
247
+ load_format: str = "auto",
248
+ revision: Optional[str] = None):
249
tp_world_size = get_tensor_model_parallel_world_size()
250
tp_rank = get_tensor_model_parallel_rank()
251
state_dict = self.state_dict()
252
for name, loaded_weight in hf_model_weights_iterator(
252
- model_name_or_path, cache_dir, load_format):
253
+ model_name_or_path, cache_dir, load_format, revision):
254
if "Wqkv" in name:
255
# NOTE(woosuk): MPT's fused QKV has the shape of
256
# [3 * num_heads * head_size, hidden_size].
vllm/model_executor/models/opt.py
+3
-2
@@ -297,12 +297,13 @@ class OPTForCausalLM(nn.Module):
297
def load_weights(self,
298
model_name_or_path: str,
299
cache_dir: Optional[str] = None,
300
- load_format: str = "auto"):
300
+ load_format: str = "auto",
301
+ revision: Optional[str] = None):
302
tensor_model_parallel_rank = get_tensor_model_parallel_rank()
303
state_dict = self.state_dict()
304
305
for name, loaded_weight in hf_model_weights_iterator(
305
- model_name_or_path, cache_dir, load_format):
306
+ model_name_or_path, cache_dir, load_format, revision):
307
if "lm_head.weight" in name:
308
continue
309
vllm/model_executor/models/qwen.py
+2
-1
@@ -251,13 +251,14 @@ class QWenLMHeadModel(nn.Module):
251
model_name_or_path: str,
252
cache_dir: Optional[str] = None,
253
load_format: str = "auto",
254
+ revision: Optional[str] = None,
255
):
256
tp_world_size = get_tensor_model_parallel_world_size()
257
tp_rank = get_tensor_model_parallel_rank()
258
state_dict = self.state_dict()
259
260
for name, loaded_weight in hf_model_weights_iterator(
260
- model_name_or_path, cache_dir, load_format):
261
+ model_name_or_path, cache_dir, load_format, revision):
262
if "rotary_emb.inv_freq" in name:
263
continue
264
vllm/model_executor/weight_utils.py
+8
-3
@@ -83,6 +83,7 @@ def prepare_hf_model_weights(
83
cache_dir: Optional[str] = None,
84
use_safetensors: bool = False,
85
fall_back_to_pt: bool = True,
86
+ revision: Optional[str] = None,
87
):
88
# Download model weights from huggingface.
89
is_local = os.path.isdir(model_name_or_path)
@@ -94,7 +95,8 @@ def prepare_hf_model_weights(
95
hf_folder = snapshot_download(model_name_or_path,
96
allow_patterns=allow_patterns,
97
cache_dir=cache_dir,
97
- tqdm_class=Disabledtqdm)
98
+ tqdm_class=Disabledtqdm,
99
+ revision=revision)
100
else:
101
hf_folder = model_name_or_path
102
hf_weights_files = glob.glob(os.path.join(hf_folder, allow_patterns))
@@ -107,7 +109,8 @@ def prepare_hf_model_weights(
109
return prepare_hf_model_weights(model_name_or_path,
110
cache_dir=cache_dir,
111
use_safetensors=False,
110
- fall_back_to_pt=False)
112
+ fall_back_to_pt=False,
113
+ revision=revision)
114
115
if len(hf_weights_files) == 0:
116
raise RuntimeError(
@@ -120,6 +123,7 @@ def hf_model_weights_iterator(
123
model_name_or_path: str,
124
cache_dir: Optional[str] = None,
125
load_format: str = "auto",
126
+ revision: Optional[str] = None,
127
) -> Iterator[Tuple[str, torch.Tensor]]:
128
use_safetensors = False
129
use_np_cache = False
@@ -140,7 +144,8 @@ def hf_model_weights_iterator(
144
model_name_or_path,
145
cache_dir=cache_dir,
146
use_safetensors=use_safetensors,
143
- fall_back_to_pt=fall_back_to_pt)
147
+ fall_back_to_pt=fall_back_to_pt,
148
+ revision=revision)
149
150
if use_np_cache:
151
# Currently np_cache only support *.bin checkpoints
vllm/transformers_utils/config.py
+7
-3
@@ -1,3 +1,5 @@
1
+from typing import Optional
2
+
3
from transformers import AutoConfig, PretrainedConfig
4
5
from vllm.transformers_utils.configs import * # pylint: disable=wildcard-import
@@ -12,10 +14,12 @@ _CONFIG_REGISTRY = {
14
}
15
16
15
-def get_config(model: str, trust_remote_code: bool) -> PretrainedConfig:
17
+def get_config(model: str,
18
+ trust_remote_code: bool,
19
+ revision: Optional[str] = None) -> PretrainedConfig:
20
try:
21
config = AutoConfig.from_pretrained(
18
- model, trust_remote_code=trust_remote_code)
22
+ model, trust_remote_code=trust_remote_code, revision=revision)
23
except ValueError as e:
24
if (not trust_remote_code and
25
"requires you to execute the configuration file" in str(e)):
@@ -29,5 +33,5 @@ def get_config(model: str, trust_remote_code: bool) -> PretrainedConfig:
33
raise e
34
if config.model_type in _CONFIG_REGISTRY:
35
config_class = _CONFIG_REGISTRY[config.model_type]
32
- config = config_class.from_pretrained(model)
36
+ config = config_class.from_pretrained(model, revision=revision)
37
return config