A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Add code-revision config argument for Hugging Face Hub (#2892)
Mark Mozolewski committed
Feb 17, 2024 at 22:36 UTC
786b7f18a541a0460a9ee56154558ac7121601ac
3 files changed
+30
-11
vllm/config.py
+7
-1
@@ -44,6 +44,9 @@ class ModelConfig:
44
revision: The specific model version to use. It can be a branch name,
45
a tag name, or a commit id. If unspecified, will use the default
46
version.
47
+ code_revision: The specific revision to use for the model code on
48
+ Hugging Face Hub. It can be a branch name, a tag name, or a
49
+ commit id. If unspecified, will use the default version.
50
tokenizer_revision: The specific tokenizer version to use. It can be a
51
branch name, a tag name, or a commit id. If unspecified, will use
52
the default version.
@@ -70,6 +73,7 @@ class ModelConfig:
73
dtype: Union[str, torch.dtype],
74
seed: int,
75
revision: Optional[str] = None,
76
+ code_revision: Optional[str] = None,
77
tokenizer_revision: Optional[str] = None,
78
max_model_len: Optional[int] = None,
79
quantization: Optional[str] = None,
@@ -84,6 +88,7 @@ class ModelConfig:
88
self.load_format = load_format
89
self.seed = seed
90
self.revision = revision
91
+ self.code_revision = code_revision
92
self.tokenizer_revision = tokenizer_revision
93
self.quantization = quantization
94
self.enforce_eager = enforce_eager
@@ -103,7 +108,8 @@ class ModelConfig:
108
self.download_dir = model_path
109
self.tokenizer = model_path
110
106
- self.hf_config = get_config(self.model, trust_remote_code, revision)
111
+ self.hf_config = get_config(self.model, trust_remote_code, revision,
112
+ code_revision)
113
self.dtype = _get_and_verify_dtype(self.hf_config, dtype)
114
self.max_model_len = _get_and_verify_max_len(self.hf_config,
115
max_model_len)
vllm/engine/arg_utils.py
+14
-7
@@ -32,6 +32,7 @@ class EngineArgs:
32
max_paddings: int = 256
33
disable_log_stats: bool = False
34
revision: Optional[str] = None
35
+ code_revision: Optional[str] = None
36
tokenizer_revision: Optional[str] = None
37
quantization: Optional[str] = None
38
enforce_eager: bool = False
@@ -75,6 +76,13 @@ class EngineArgs:
76
help='the specific model version to use. It can be a branch '
77
'name, a tag name, or a commit id. If unspecified, will use '
78
'the default version.')
79
+ parser.add_argument(
80
+ '--code-revision',
81
+ type=str,
82
+ default=None,
83
+ help='the specific revision to use for the model code on '
84
+ 'Hugging Face Hub. It can be a branch name, a tag name, or a '
85
+ 'commit id. If unspecified, will use the default version.')
86
parser.add_argument(
87
'--tokenizer-revision',
88
type=str,
@@ -279,13 +287,12 @@ class EngineArgs:
287
) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig,
288
DeviceConfig, Optional[LoRAConfig]]:
289
device_config = DeviceConfig(self.device)
282
- model_config = ModelConfig(self.model, self.tokenizer,
283
- self.tokenizer_mode, self.trust_remote_code,
284
- self.download_dir, self.load_format,
285
- self.dtype, self.seed, self.revision,
286
- self.tokenizer_revision, self.max_model_len,
287
- self.quantization, self.enforce_eager,
288
- self.max_context_len_to_capture)
290
+ model_config = ModelConfig(
291
+ self.model, self.tokenizer, self.tokenizer_mode,
292
+ self.trust_remote_code, self.download_dir, self.load_format,
293
+ self.dtype, self.seed, self.revision, self.code_revision,
294
+ self.tokenizer_revision, self.max_model_len, self.quantization,
295
+ self.enforce_eager, self.max_context_len_to_capture)
296
cache_config = CacheConfig(self.block_size,
297
self.gpu_memory_utilization,
298
self.swap_space, self.kv_cache_dtype,
vllm/transformers_utils/config.py
+9
-3
@@ -16,10 +16,14 @@ _CONFIG_REGISTRY = {
16
17
def get_config(model: str,
18
trust_remote_code: bool,
19
- revision: Optional[str] = None) -> PretrainedConfig:
19
+ revision: Optional[str] = None,
20
+ code_revision: Optional[str] = None) -> PretrainedConfig:
21
try:
22
config = AutoConfig.from_pretrained(
22
- model, trust_remote_code=trust_remote_code, revision=revision)
23
+ model,
24
+ trust_remote_code=trust_remote_code,
25
+ revision=revision,
26
+ code_revision=code_revision)
27
except ValueError as e:
28
if (not trust_remote_code and
29
"requires you to execute the configuration file" in str(e)):
@@ -33,5 +37,7 @@ def get_config(model: str,
37
raise e
38
if config.model_type in _CONFIG_REGISTRY:
39
config_class = _CONFIG_REGISTRY[config.model_type]
36
- config = config_class.from_pretrained(model, revision=revision)
40
+ config = config_class.from_pretrained(model,
41
+ revision=revision,
42
+ code_revision=code_revision)
43
return config