@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 786b7f18a5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Add code-revision config argument for Hugging Face Hub (#2892)

Mark Mozolewski committed Feb 17, 2024 at 22:36 UTC 786b7f18a541a0460a9ee56154558ac7121601ac
3 files changed +30 -11
vllm/config.py
+7 -1
@@ -44,6 +44,9 @@ class ModelConfig:
44 revision: The specific model version to use. It can be a branch name,
45 a tag name, or a commit id. If unspecified, will use the default
46 version.
47 + code_revision: The specific revision to use for the model code on
48 + Hugging Face Hub. It can be a branch name, a tag name, or a
49 + commit id. If unspecified, will use the default version.
50 tokenizer_revision: The specific tokenizer version to use. It can be a
51 branch name, a tag name, or a commit id. If unspecified, will use
52 the default version.
@@ -70,6 +73,7 @@ class ModelConfig:
73 dtype: Union[str, torch.dtype],
74 seed: int,
75 revision: Optional[str] = None,
76 + code_revision: Optional[str] = None,
77 tokenizer_revision: Optional[str] = None,
78 max_model_len: Optional[int] = None,
79 quantization: Optional[str] = None,
@@ -84,6 +88,7 @@ class ModelConfig:
88 self.load_format = load_format
89 self.seed = seed
90 self.revision = revision
91 + self.code_revision = code_revision
92 self.tokenizer_revision = tokenizer_revision
93 self.quantization = quantization
94 self.enforce_eager = enforce_eager
@@ -103,7 +108,8 @@ class ModelConfig:
108 self.download_dir = model_path
109 self.tokenizer = model_path
110
106 - self.hf_config = get_config(self.model, trust_remote_code, revision)
111 + self.hf_config = get_config(self.model, trust_remote_code, revision,
112 + code_revision)
113 self.dtype = _get_and_verify_dtype(self.hf_config, dtype)
114 self.max_model_len = _get_and_verify_max_len(self.hf_config,
115 max_model_len)
vllm/engine/arg_utils.py
+14 -7
@@ -32,6 +32,7 @@ class EngineArgs:
32 max_paddings: int = 256
33 disable_log_stats: bool = False
34 revision: Optional[str] = None
35 + code_revision: Optional[str] = None
36 tokenizer_revision: Optional[str] = None
37 quantization: Optional[str] = None
38 enforce_eager: bool = False
@@ -75,6 +76,13 @@ class EngineArgs:
76 help='the specific model version to use. It can be a branch '
77 'name, a tag name, or a commit id. If unspecified, will use '
78 'the default version.')
79 + parser.add_argument(
80 + '--code-revision',
81 + type=str,
82 + default=None,
83 + help='the specific revision to use for the model code on '
84 + 'Hugging Face Hub. It can be a branch name, a tag name, or a '
85 + 'commit id. If unspecified, will use the default version.')
86 parser.add_argument(
87 '--tokenizer-revision',
88 type=str,
@@ -279,13 +287,12 @@ class EngineArgs:
287 ) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig,
288 DeviceConfig, Optional[LoRAConfig]]:
289 device_config = DeviceConfig(self.device)
282 - model_config = ModelConfig(self.model, self.tokenizer,
283 - self.tokenizer_mode, self.trust_remote_code,
284 - self.download_dir, self.load_format,
285 - self.dtype, self.seed, self.revision,
286 - self.tokenizer_revision, self.max_model_len,
287 - self.quantization, self.enforce_eager,
288 - self.max_context_len_to_capture)
290 + model_config = ModelConfig(
291 + self.model, self.tokenizer, self.tokenizer_mode,
292 + self.trust_remote_code, self.download_dir, self.load_format,
293 + self.dtype, self.seed, self.revision, self.code_revision,
294 + self.tokenizer_revision, self.max_model_len, self.quantization,
295 + self.enforce_eager, self.max_context_len_to_capture)
296 cache_config = CacheConfig(self.block_size,
297 self.gpu_memory_utilization,
298 self.swap_space, self.kv_cache_dtype,
vllm/transformers_utils/config.py
+9 -3
@@ -16,10 +16,14 @@ _CONFIG_REGISTRY = {
16
17 def get_config(model: str,
18 trust_remote_code: bool,
19 - revision: Optional[str] = None) -> PretrainedConfig:
19 + revision: Optional[str] = None,
20 + code_revision: Optional[str] = None) -> PretrainedConfig:
21 try:
22 config = AutoConfig.from_pretrained(
22 - model, trust_remote_code=trust_remote_code, revision=revision)
23 + model,
24 + trust_remote_code=trust_remote_code,
25 + revision=revision,
26 + code_revision=code_revision)
27 except ValueError as e:
28 if (not trust_remote_code and
29 "requires you to execute the configuration file" in str(e)):
@@ -33,5 +37,7 @@ def get_config(model: str,
37 raise e
38 if config.model_type in _CONFIG_REGISTRY:
39 config_class = _CONFIG_REGISTRY[config.model_type]
36 - config = config_class.from_pretrained(model, revision=revision)
40 + config = config_class.from_pretrained(model,
41 + revision=revision,
42 + code_revision=code_revision)
43 return config