A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
support ignore patterns in model loader (#6673)
Simon Mo committed
Jul 22, 2024 at 23:59 UTC
3eda4ec780d572120ed02e9e94bcef383c3e0399
4 files changed
+51
-10
vllm/config.py
+14
-1
@@ -599,12 +599,16 @@ class LoadConfig:
599
mainly for profiling.
600
"tensorizer" will use CoreWeave's tensorizer library for
601
fast weight loading.
602
+ ignore_patterns: The list of patterns to ignore when loading the model.
603
+ Default to "original/**/*" to avoid repeated loading of llama's
604
+ checkpoints.
605
"""
606
607
load_format: Union[str, LoadFormat, "BaseModelLoader"] = LoadFormat.AUTO
608
download_dir: Optional[str] = None
609
model_loader_extra_config: Optional[Union[str, dict]] = field(
610
default_factory=dict)
611
+ ignore_patterns: Optional[Union[List[str], str]] = None
612
613
def __post_init__(self):
614
model_loader_extra_config = self.model_loader_extra_config or {}
@@ -613,6 +617,13 @@ class LoadConfig:
617
model_loader_extra_config)
618
self._verify_load_format()
619
620
+ if self.ignore_patterns is not None and len(self.ignore_patterns) > 0:
621
+ logger.info(
622
+ "Ignoring the following patterns when downloading weights: %s",
623
+ self.ignore_patterns)
624
+ else:
625
+ self.ignore_patterns = ["original/**/*"]
626
+
627
def _verify_load_format(self) -> None:
628
if not isinstance(self.load_format, str):
629
return
@@ -801,7 +812,9 @@ class SchedulerConfig:
812
# for higher throughput.
813
self.max_num_batched_tokens = max(max_model_len, 2048)
814
if enable_chunked_prefill:
804
- logger.info("Chunked prefill is enabled (EXPERIMENTAL).")
815
+ logger.info(
816
+ "Chunked prefill is enabled with max_num_batched_tokens=%d.",
817
+ max_num_batched_tokens)
818
819
self.max_num_seqs = max_num_seqs
820
self.max_model_len = max_model_len
vllm/engine/arg_utils.py
+10
@@ -95,6 +95,7 @@ class EngineArgs:
95
num_gpu_blocks_override: Optional[int] = None
96
num_lookahead_slots: int = 0
97
model_loader_extra_config: Optional[dict] = None
98
+ ignore_patterns: Optional[Union[str, List[str]]] = None
99
preemption_mode: Optional[str] = None
100
101
scheduler_delay_factor: float = 0.0
@@ -619,6 +620,14 @@ class EngineArgs:
620
'corresponding to the chosen load_format. '
621
'This should be a JSON string that will be '
622
'parsed into a dictionary.')
623
+ parser.add_argument(
624
+ '--ignore-patterns',
625
+ action="append",
626
+ type=str,
627
+ default=[],
628
+ help="The pattern(s) to ignore when loading the model."
629
+ "Default to 'original/**/*' to avoid repeated loading of llama's "
630
+ "checkpoints.")
631
parser.add_argument(
632
'--preemption-mode',
633
type=str,
@@ -824,6 +833,7 @@ class EngineArgs:
833
load_format=self.load_format,
834
download_dir=self.download_dir,
835
model_loader_extra_config=self.model_loader_extra_config,
836
+ ignore_patterns=self.ignore_patterns,
837
)
838
839
prompt_adapter_config = PromptAdapterConfig(
vllm/model_executor/model_loader/loader.py
+21
-8
@@ -161,6 +161,7 @@ class DefaultModelLoader(BaseModelLoader):
161
cache_dir=self.load_config.download_dir,
162
local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
163
revision=revision,
164
+ ignore_patterns=self.load_config.ignore_patterns,
165
)
166
else:
167
model_path = model
@@ -196,9 +197,13 @@ class DefaultModelLoader(BaseModelLoader):
197
allow_patterns += ["*.pt"]
198
199
if not is_local:
199
- hf_folder = download_weights_from_hf(model_name_or_path,
200
- self.load_config.download_dir,
201
- allow_patterns, revision)
200
+ hf_folder = download_weights_from_hf(
201
+ model_name_or_path,
202
+ self.load_config.download_dir,
203
+ allow_patterns,
204
+ revision,
205
+ ignore_patterns=self.load_config.ignore_patterns,
206
+ )
207
else:
208
hf_folder = model_name_or_path
209
@@ -489,9 +494,13 @@ class ShardedStateLoader(BaseModelLoader):
494
return model_name_or_path
495
else:
496
allow_patterns = ["*.safetensors"]
492
- return download_weights_from_hf(model_name_or_path,
493
- self.load_config.download_dir,
494
- allow_patterns, revision)
497
+ return download_weights_from_hf(
498
+ model_name_or_path,
499
+ self.load_config.download_dir,
500
+ allow_patterns,
501
+ revision,
502
+ ignore_patterns=self.load_config.ignore_patterns,
503
+ )
504
505
def load_model(self, *, model_config: ModelConfig,
506
device_config: DeviceConfig,
@@ -663,8 +672,12 @@ class BitsAndBytesModelLoader(BaseModelLoader):
672
matching_files = fnmatch.filter(repo_files, pattern)
673
if matching_files:
674
hf_folder = download_weights_from_hf(
666
- model_name_or_path, self.load_config.download_dir,
667
- [pattern], revision)
675
+ model_name_or_path,
676
+ self.load_config.download_dir,
677
+ [pattern],
678
+ revision,
679
+ ignore_patterns=self.load_config.ignore_patterns,
680
+ )
681
return glob.glob(os.path.join(hf_folder, pattern)), pattern
682
683
raise RuntimeError(
vllm/model_executor/model_loader/weight_utils.py
+6
-1
@@ -6,7 +6,7 @@ import json
6
import os
7
import tempfile
8
from collections import defaultdict
9
-from typing import Any, Generator, Iterable, List, Optional, Tuple
9
+from typing import Any, Generator, Iterable, List, Optional, Tuple, Union
10
11
import filelock
12
import huggingface_hub.constants
@@ -189,6 +189,7 @@ def download_weights_from_hf(
189
cache_dir: Optional[str],
190
allow_patterns: List[str],
191
revision: Optional[str] = None,
192
+ ignore_patterns: Optional[Union[str, List[str]]] = None,
193
) -> str:
194
"""Download model weights from Hugging Face Hub.
195
@@ -200,6 +201,9 @@ def download_weights_from_hf(
201
weight files. Files matched by any of the patterns will be
202
downloaded.
203
revision (Optional[str]): The revision of the model.
204
+ ignore_patterns (Optional[Union[str, List[str]]]): The patterns to
205
+ filter out the weight files. Files matched by any of the patterns
206
+ will be ignored.
207
208
Returns:
209
str: The path to the downloaded model weights.
@@ -223,6 +227,7 @@ def download_weights_from_hf(
227
hf_folder = snapshot_download(
228
model_name_or_path,
229
allow_patterns=allow_patterns,
230
+ ignore_patterns=ignore_patterns,
231
cache_dir=cache_dir,
232
tqdm_class=DisabledTqdm,
233
revision=revision,