@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ab019eea75
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Add Model Revision Support (#1014)

Co-authored-by: Jasmond Loh <Jasmond.Loh@hotmail.com> Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>

Jasmond L committed Sep 14, 2023 at 06:20 UTC ab019eea7513eb1e26ead79cf162863f3f19e971
20 files changed +75 -35
vllm/config.py
+6 -1
@@ -38,6 +38,9 @@ class ModelConfig:
38 will use FP16 precision for FP32 and FP16 models, and BF16 precision
39 for BF16 models.
40 seed: Random seed for reproducibility.
41 + revision: The specific model version to use. It can be a branch name,
42 + a tag name, or a commit id. If unspecified, will use the default
43 + version.
44 max_model_len: Maximum length of a sequence (including prompt and
45 output). If None, will be derived from the model.
46 """
@@ -52,6 +55,7 @@ class ModelConfig:
55 load_format: str,
56 dtype: str,
57 seed: int,
58 + revision: Optional[str],
59 max_model_len: Optional[int] = None,
60 ) -> None:
61 self.model = model
@@ -61,8 +65,9 @@ class ModelConfig:
65 self.download_dir = download_dir
66 self.load_format = load_format
67 self.seed = seed
68 + self.revision = revision
69
65 - self.hf_config = get_config(model, trust_remote_code)
70 + self.hf_config = get_config(model, trust_remote_code, revision)
71 self.dtype = _get_and_verify_dtype(self.hf_config, dtype)
72 self._verify_load_format()
73 self._verify_tokenizer_mode()
vllm/engine/arg_utils.py
+10 -1
@@ -28,6 +28,7 @@ class EngineArgs:
28 max_num_batched_tokens: int = 2560
29 max_num_seqs: int = 256
30 disable_log_stats: bool = False
31 + revision: Optional[str] = None
32
33 def __post_init__(self):
34 if self.tokenizer is None:
@@ -49,6 +50,13 @@ class EngineArgs:
50 type=str,
51 default=EngineArgs.tokenizer,
52 help='name or path of the huggingface tokenizer to use')
53 + parser.add_argument(
54 + '--revision',
55 + type=str,
56 + default=None,
57 + help='the specific model version to use. It can be a branch '
58 + 'name, a tag name, or a commit id. If unspecified, will use '
59 + 'the default version.')
60 parser.add_argument('--tokenizer-mode',
61 type=str,
62 default=EngineArgs.tokenizer_mode,
@@ -159,7 +167,8 @@ class EngineArgs:
167 model_config = ModelConfig(self.model, self.tokenizer,
168 self.tokenizer_mode, self.trust_remote_code,
169 self.download_dir, self.load_format,
162 - self.dtype, self.seed, self.max_model_len)
170 + self.dtype, self.seed, self.revision,
171 + self.max_model_len)
172 cache_config = CacheConfig(self.block_size,
173 self.gpu_memory_utilization,
174 self.swap_space)
vllm/engine/llm_engine.py
+3 -1
@@ -74,6 +74,7 @@ class LLMEngine:
74 f"model={model_config.model!r}, "
75 f"tokenizer={model_config.tokenizer!r}, "
76 f"tokenizer_mode={model_config.tokenizer_mode}, "
77 + f"revision={model_config.revision}, "
78 f"trust_remote_code={model_config.trust_remote_code}, "
79 f"dtype={model_config.dtype}, "
80 f"download_dir={model_config.download_dir!r}, "
@@ -92,7 +93,8 @@ class LLMEngine:
93 self.tokenizer = get_tokenizer(
94 model_config.tokenizer,
95 tokenizer_mode=model_config.tokenizer_mode,
95 - trust_remote_code=model_config.trust_remote_code)
96 + trust_remote_code=model_config.trust_remote_code,
97 + revision=model_config.revision)
98 self.seq_counter = Counter()
99
100 # Create the parallel GPU workers.
vllm/entrypoints/llm.py
+2
@@ -38,6 +38,8 @@ class LLM:
38 However, if the `torch_dtype` in the config is `float32`, we will
39 use `float16` instead.
40 seed: The seed to initialize the random number generator for sampling.
41 + revision: The specific model version to use. It can be a branch name,
42 + a tag name, or a commit id.
43 """
44
45 def __init__(
vllm/model_executor/model_loader.py
+1 -1
@@ -64,6 +64,6 @@ def get_model(model_config: ModelConfig) -> nn.Module:
64 else:
65 # Load the weights from the cached or downloaded files.
66 model.load_weights(model_config.model, model_config.download_dir,
67 - model_config.load_format)
67 + model_config.load_format, model_config.revision)
68 model = model.cuda()
69 return model.eval()
vllm/model_executor/models/aquila.py
+3 -2
@@ -288,7 +288,8 @@ class AquilaForCausalLM(nn.Module):
288 def load_weights(self,
289 model_name_or_path: str,
290 cache_dir: Optional[str] = None,
291 - load_format: str = "auto"):
291 + load_format: str = "auto",
292 + revision: Optional[str] = None):
293 tp_size = get_tensor_model_parallel_world_size()
294 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
295 q_proj_shard_size = (self.config.hidden_size // tp_size)
@@ -305,7 +306,7 @@ class AquilaForCausalLM(nn.Module):
306 state_dict = self.state_dict()
307
308 for name, loaded_weight in hf_model_weights_iterator(
308 - model_name_or_path, cache_dir, load_format):
309 + model_name_or_path, cache_dir, load_format, revision):
310 if "rotary_emb.inv_freq" in name:
311 continue
312
vllm/model_executor/models/baichuan.py
+3 -2
@@ -303,13 +303,14 @@ class BaiChuanBaseForCausalLM(nn.Module):
303 def load_weights(self,
304 model_name_or_path: str,
305 cache_dir: Optional[str] = None,
306 - load_format: str = "auto"):
306 + load_format: str = "auto",
307 + revision: Optional[str] = None):
308 tp_world_size = get_tensor_model_parallel_world_size()
309 tp_rank = get_tensor_model_parallel_rank()
310 state_dict = self.state_dict()
311
312 for name, loaded_weight in hf_model_weights_iterator(
312 - model_name_or_path, cache_dir, load_format):
313 + model_name_or_path, cache_dir, load_format, revision):
314 if "rotary_emb.inv_freq" in name:
315 continue
316
vllm/model_executor/models/bloom.py
+3 -2
@@ -279,11 +279,12 @@ class BloomForCausalLM(nn.Module):
279 def load_weights(self,
280 model_name_or_path: str,
281 cache_dir: Optional[str] = None,
282 - load_format: str = "auto"):
282 + load_format: str = "auto",
283 + revision: Optional[str] = None):
284 tp_rank = get_tensor_model_parallel_rank()
285 state_dict = self.state_dict()
286 for name, loaded_weight in hf_model_weights_iterator(
286 - model_name_or_path, cache_dir, load_format):
287 + model_name_or_path, cache_dir, load_format, revision):
288 if name == "lm_head.weight":
289 # Since hidden_states are parallelized, we need to
290 # load lm_head.weight in parallel.
vllm/model_executor/models/falcon.py
+3 -2
@@ -420,7 +420,8 @@ class FalconForCausalLM(nn.Module):
420 def load_weights(self,
421 model_name_or_path: str,
422 cache_dir: Optional[str] = None,
423 - load_format: str = "auto"):
423 + load_format: str = "auto",
424 + revision: Optional[str] = None):
425 tp_size = (get_tensor_model_parallel_world_size())
426 tp_rank = get_tensor_model_parallel_rank()
427
@@ -452,7 +453,7 @@ class FalconForCausalLM(nn.Module):
453 state_dict = self.state_dict()
454
455 for name, loaded_weight in hf_model_weights_iterator(
455 - model_name_or_path, cache_dir, load_format):
456 + model_name_or_path, cache_dir, load_format, revision):
457 if "query_key_value" in name:
458 loaded_weight = convert_pyslice_to_tensor(loaded_weight)
459 loaded_weight_size = loaded_weight.size()
vllm/model_executor/models/gpt2.py
+3 -2
@@ -231,14 +231,15 @@ class GPT2LMHeadModel(nn.Module):
231 def load_weights(self,
232 model_name_or_path: str,
233 cache_dir: Optional[str] = None,
234 - load_format: str = "auto"):
234 + load_format: str = "auto",
235 + revision: Optional[str] = None):
236 tensor_model_parallel_world_size = (
237 get_tensor_model_parallel_world_size())
238 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
239 state_dict = self.state_dict()
240
241 for name, loaded_weight in hf_model_weights_iterator(
241 - model_name_or_path, cache_dir, load_format):
242 + model_name_or_path, cache_dir, load_format, revision):
243 if "lm_head.weight" in name:
244 # GPT-2 ties the weights of the embedding layer and the final
245 # linear layer.
vllm/model_executor/models/gpt_bigcode.py
+3 -2
@@ -259,14 +259,15 @@ class GPTBigCodeForCausalLM(nn.Module):
259 def load_weights(self,
260 model_name_or_path: str,
261 cache_dir: Optional[str] = None,
262 - load_format: str = "auto"):
262 + load_format: str = "auto",
263 + revision: Optional[str] = None):
264 tensor_model_parallel_world_size = (
265 get_tensor_model_parallel_world_size())
266 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
267 state_dict = self.state_dict()
268
269 for name, loaded_weight in hf_model_weights_iterator(
269 - model_name_or_path, cache_dir, load_format):
270 + model_name_or_path, cache_dir, load_format, revision):
271 if "lm_head.weight" in name:
272 # GPT-2 ties the weights of the embedding layer and the final
273 # linear layer.
vllm/model_executor/models/gpt_j.py
+3 -2
@@ -222,11 +222,12 @@ class GPTJForCausalLM(nn.Module):
222 def load_weights(self,
223 model_name_or_path: str,
224 cache_dir: Optional[str] = None,
225 - load_format: str = "auto"):
225 + load_format: str = "auto",
226 + revision: Optional[str] = None):
227 tp_rank = get_tensor_model_parallel_rank()
228 state_dict = self.state_dict()
229 for name, loaded_weight in hf_model_weights_iterator(
229 - model_name_or_path, cache_dir, load_format):
230 + model_name_or_path, cache_dir, load_format, revision):
231 if "attn.bias" in name or "attn.masked_bias" in name:
232 continue
233
vllm/model_executor/models/gpt_neox.py
+3 -2
@@ -231,11 +231,12 @@ class GPTNeoXForCausalLM(nn.Module):
231 def load_weights(self,
232 model_name_or_path: str,
233 cache_dir: Optional[str] = None,
234 - load_format: str = "auto"):
234 + load_format: str = "auto",
235 + revision: Optional[str] = None):
236 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
237 state_dict = self.state_dict()
238 for name, loaded_weight in hf_model_weights_iterator(
238 - model_name_or_path, cache_dir, load_format):
239 + model_name_or_path, cache_dir, load_format, revision):
240 if ("attention.bias" in name or "attention.masked_bias" in name
241 or "rotary_emb.inv_freq" in name):
242 continue
vllm/model_executor/models/internlm.py
+3 -2
@@ -233,12 +233,13 @@ class InternLMForCausalLM(nn.Module):
233 def load_weights(self,
234 model_name_or_path: str,
235 cache_dir: Optional[str] = None,
236 - load_format: str = "auto"):
236 + load_format: str = "auto",
237 + revision: Optional[str] = None):
238 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
239 state_dict = self.state_dict()
240
241 for name, loaded_weight in hf_model_weights_iterator(
241 - model_name_or_path, cache_dir, load_format):
242 + model_name_or_path, cache_dir, load_format, revision):
243 if "rotary_emb.inv_freq" in name:
244 continue
245
vllm/model_executor/models/llama.py
+3 -2
@@ -271,7 +271,8 @@ class LlamaForCausalLM(nn.Module):
271 def load_weights(self,
272 model_name_or_path: str,
273 cache_dir: Optional[str] = None,
274 - load_format: str = "auto"):
274 + load_format: str = "auto",
275 + revision: Optional[str] = None):
276 tp_size = get_tensor_model_parallel_world_size()
277 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
278 q_proj_shard_size = (self.config.hidden_size // tp_size)
@@ -288,7 +289,7 @@ class LlamaForCausalLM(nn.Module):
289 state_dict = self.state_dict()
290
291 for name, loaded_weight in hf_model_weights_iterator(
291 - model_name_or_path, cache_dir, load_format):
292 + model_name_or_path, cache_dir, load_format, revision):
293 if "rotary_emb.inv_freq" in name:
294 continue
295
vllm/model_executor/models/mpt.py
+3 -2
@@ -244,12 +244,13 @@ class MPTForCausalLM(nn.Module):
244 def load_weights(self,
245 model_name_or_path: str,
246 cache_dir: Optional[str] = None,
247 - load_format: str = "auto"):
247 + load_format: str = "auto",
248 + revision: Optional[str] = None):
249 tp_world_size = get_tensor_model_parallel_world_size()
250 tp_rank = get_tensor_model_parallel_rank()
251 state_dict = self.state_dict()
252 for name, loaded_weight in hf_model_weights_iterator(
252 - model_name_or_path, cache_dir, load_format):
253 + model_name_or_path, cache_dir, load_format, revision):
254 if "Wqkv" in name:
255 # NOTE(woosuk): MPT's fused QKV has the shape of
256 # [3 * num_heads * head_size, hidden_size].
vllm/model_executor/models/opt.py
+3 -2
@@ -297,12 +297,13 @@ class OPTForCausalLM(nn.Module):
297 def load_weights(self,
298 model_name_or_path: str,
299 cache_dir: Optional[str] = None,
300 - load_format: str = "auto"):
300 + load_format: str = "auto",
301 + revision: Optional[str] = None):
302 tensor_model_parallel_rank = get_tensor_model_parallel_rank()
303 state_dict = self.state_dict()
304
305 for name, loaded_weight in hf_model_weights_iterator(
305 - model_name_or_path, cache_dir, load_format):
306 + model_name_or_path, cache_dir, load_format, revision):
307 if "lm_head.weight" in name:
308 continue
309
vllm/model_executor/models/qwen.py
+2 -1
@@ -251,13 +251,14 @@ class QWenLMHeadModel(nn.Module):
251 model_name_or_path: str,
252 cache_dir: Optional[str] = None,
253 load_format: str = "auto",
254 + revision: Optional[str] = None,
255 ):
256 tp_world_size = get_tensor_model_parallel_world_size()
257 tp_rank = get_tensor_model_parallel_rank()
258 state_dict = self.state_dict()
259
260 for name, loaded_weight in hf_model_weights_iterator(
260 - model_name_or_path, cache_dir, load_format):
261 + model_name_or_path, cache_dir, load_format, revision):
262 if "rotary_emb.inv_freq" in name:
263 continue
264
vllm/model_executor/weight_utils.py
+8 -3
@@ -83,6 +83,7 @@ def prepare_hf_model_weights(
83 cache_dir: Optional[str] = None,
84 use_safetensors: bool = False,
85 fall_back_to_pt: bool = True,
86 + revision: Optional[str] = None,
87 ):
88 # Download model weights from huggingface.
89 is_local = os.path.isdir(model_name_or_path)
@@ -94,7 +95,8 @@ def prepare_hf_model_weights(
95 hf_folder = snapshot_download(model_name_or_path,
96 allow_patterns=allow_patterns,
97 cache_dir=cache_dir,
97 - tqdm_class=Disabledtqdm)
98 + tqdm_class=Disabledtqdm,
99 + revision=revision)
100 else:
101 hf_folder = model_name_or_path
102 hf_weights_files = glob.glob(os.path.join(hf_folder, allow_patterns))
@@ -107,7 +109,8 @@ def prepare_hf_model_weights(
109 return prepare_hf_model_weights(model_name_or_path,
110 cache_dir=cache_dir,
111 use_safetensors=False,
110 - fall_back_to_pt=False)
112 + fall_back_to_pt=False,
113 + revision=revision)
114
115 if len(hf_weights_files) == 0:
116 raise RuntimeError(
@@ -120,6 +123,7 @@ def hf_model_weights_iterator(
123 model_name_or_path: str,
124 cache_dir: Optional[str] = None,
125 load_format: str = "auto",
126 + revision: Optional[str] = None,
127 ) -> Iterator[Tuple[str, torch.Tensor]]:
128 use_safetensors = False
129 use_np_cache = False
@@ -140,7 +144,8 @@ def hf_model_weights_iterator(
144 model_name_or_path,
145 cache_dir=cache_dir,
146 use_safetensors=use_safetensors,
143 - fall_back_to_pt=fall_back_to_pt)
147 + fall_back_to_pt=fall_back_to_pt,
148 + revision=revision)
149
150 if use_np_cache:
151 # Currently np_cache only support *.bin checkpoints
vllm/transformers_utils/config.py
+7 -3
@@ -1,3 +1,5 @@
1 +from typing import Optional
2 +
3 from transformers import AutoConfig, PretrainedConfig
4
5 from vllm.transformers_utils.configs import * # pylint: disable=wildcard-import
@@ -12,10 +14,12 @@ _CONFIG_REGISTRY = {
14 }
15
16
15 -def get_config(model: str, trust_remote_code: bool) -> PretrainedConfig:
17 +def get_config(model: str,
18 + trust_remote_code: bool,
19 + revision: Optional[str] = None) -> PretrainedConfig:
20 try:
21 config = AutoConfig.from_pretrained(
18 - model, trust_remote_code=trust_remote_code)
22 + model, trust_remote_code=trust_remote_code, revision=revision)
23 except ValueError as e:
24 if (not trust_remote_code and
25 "requires you to execute the configuration file" in str(e)):
@@ -29,5 +33,5 @@ def get_config(model: str, trust_remote_code: bool) -> PretrainedConfig:
33 raise e
34 if config.model_type in _CONFIG_REGISTRY:
35 config_class = _CONFIG_REGISTRY[config.model_type]
32 - config = config_class.from_pretrained(model)
36 + config = config_class.from_pretrained(model, revision=revision)
37 return config