@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3eda4ec780
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

support ignore patterns in model loader (#6673)

Simon Mo committed Jul 22, 2024 at 23:59 UTC 3eda4ec780d572120ed02e9e94bcef383c3e0399
4 files changed +51 -10
vllm/config.py
+14 -1
@@ -599,12 +599,16 @@ class LoadConfig:
599 mainly for profiling.
600 "tensorizer" will use CoreWeave's tensorizer library for
601 fast weight loading.
602 + ignore_patterns: The list of patterns to ignore when loading the model.
603 + Default to "original/**/*" to avoid repeated loading of llama's
604 + checkpoints.
605 """
606
607 load_format: Union[str, LoadFormat, "BaseModelLoader"] = LoadFormat.AUTO
608 download_dir: Optional[str] = None
609 model_loader_extra_config: Optional[Union[str, dict]] = field(
610 default_factory=dict)
611 + ignore_patterns: Optional[Union[List[str], str]] = None
612
613 def __post_init__(self):
614 model_loader_extra_config = self.model_loader_extra_config or {}
@@ -613,6 +617,13 @@ class LoadConfig:
617 model_loader_extra_config)
618 self._verify_load_format()
619
620 + if self.ignore_patterns is not None and len(self.ignore_patterns) > 0:
621 + logger.info(
622 + "Ignoring the following patterns when downloading weights: %s",
623 + self.ignore_patterns)
624 + else:
625 + self.ignore_patterns = ["original/**/*"]
626 +
627 def _verify_load_format(self) -> None:
628 if not isinstance(self.load_format, str):
629 return
@@ -801,7 +812,9 @@ class SchedulerConfig:
812 # for higher throughput.
813 self.max_num_batched_tokens = max(max_model_len, 2048)
814 if enable_chunked_prefill:
804 - logger.info("Chunked prefill is enabled (EXPERIMENTAL).")
815 + logger.info(
816 + "Chunked prefill is enabled with max_num_batched_tokens=%d.",
817 + max_num_batched_tokens)
818
819 self.max_num_seqs = max_num_seqs
820 self.max_model_len = max_model_len
vllm/engine/arg_utils.py
+10
@@ -95,6 +95,7 @@ class EngineArgs:
95 num_gpu_blocks_override: Optional[int] = None
96 num_lookahead_slots: int = 0
97 model_loader_extra_config: Optional[dict] = None
98 + ignore_patterns: Optional[Union[str, List[str]]] = None
99 preemption_mode: Optional[str] = None
100
101 scheduler_delay_factor: float = 0.0
@@ -619,6 +620,14 @@ class EngineArgs:
620 'corresponding to the chosen load_format. '
621 'This should be a JSON string that will be '
622 'parsed into a dictionary.')
623 + parser.add_argument(
624 + '--ignore-patterns',
625 + action="append",
626 + type=str,
627 + default=[],
628 + help="The pattern(s) to ignore when loading the model."
629 + "Default to 'original/**/*' to avoid repeated loading of llama's "
630 + "checkpoints.")
631 parser.add_argument(
632 '--preemption-mode',
633 type=str,
@@ -824,6 +833,7 @@ class EngineArgs:
833 load_format=self.load_format,
834 download_dir=self.download_dir,
835 model_loader_extra_config=self.model_loader_extra_config,
836 + ignore_patterns=self.ignore_patterns,
837 )
838
839 prompt_adapter_config = PromptAdapterConfig(
vllm/model_executor/model_loader/loader.py
+21 -8
@@ -161,6 +161,7 @@ class DefaultModelLoader(BaseModelLoader):
161 cache_dir=self.load_config.download_dir,
162 local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
163 revision=revision,
164 + ignore_patterns=self.load_config.ignore_patterns,
165 )
166 else:
167 model_path = model
@@ -196,9 +197,13 @@ class DefaultModelLoader(BaseModelLoader):
197 allow_patterns += ["*.pt"]
198
199 if not is_local:
199 - hf_folder = download_weights_from_hf(model_name_or_path,
200 - self.load_config.download_dir,
201 - allow_patterns, revision)
200 + hf_folder = download_weights_from_hf(
201 + model_name_or_path,
202 + self.load_config.download_dir,
203 + allow_patterns,
204 + revision,
205 + ignore_patterns=self.load_config.ignore_patterns,
206 + )
207 else:
208 hf_folder = model_name_or_path
209
@@ -489,9 +494,13 @@ class ShardedStateLoader(BaseModelLoader):
494 return model_name_or_path
495 else:
496 allow_patterns = ["*.safetensors"]
492 - return download_weights_from_hf(model_name_or_path,
493 - self.load_config.download_dir,
494 - allow_patterns, revision)
497 + return download_weights_from_hf(
498 + model_name_or_path,
499 + self.load_config.download_dir,
500 + allow_patterns,
501 + revision,
502 + ignore_patterns=self.load_config.ignore_patterns,
503 + )
504
505 def load_model(self, *, model_config: ModelConfig,
506 device_config: DeviceConfig,
@@ -663,8 +672,12 @@ class BitsAndBytesModelLoader(BaseModelLoader):
672 matching_files = fnmatch.filter(repo_files, pattern)
673 if matching_files:
674 hf_folder = download_weights_from_hf(
666 - model_name_or_path, self.load_config.download_dir,
667 - [pattern], revision)
675 + model_name_or_path,
676 + self.load_config.download_dir,
677 + [pattern],
678 + revision,
679 + ignore_patterns=self.load_config.ignore_patterns,
680 + )
681 return glob.glob(os.path.join(hf_folder, pattern)), pattern
682
683 raise RuntimeError(
vllm/model_executor/model_loader/weight_utils.py
+6 -1
@@ -6,7 +6,7 @@ import json
6 import os
7 import tempfile
8 from collections import defaultdict
9 -from typing import Any, Generator, Iterable, List, Optional, Tuple
9 +from typing import Any, Generator, Iterable, List, Optional, Tuple, Union
10
11 import filelock
12 import huggingface_hub.constants
@@ -189,6 +189,7 @@ def download_weights_from_hf(
189 cache_dir: Optional[str],
190 allow_patterns: List[str],
191 revision: Optional[str] = None,
192 + ignore_patterns: Optional[Union[str, List[str]]] = None,
193 ) -> str:
194 """Download model weights from Hugging Face Hub.
195
@@ -200,6 +201,9 @@ def download_weights_from_hf(
201 weight files. Files matched by any of the patterns will be
202 downloaded.
203 revision (Optional[str]): The revision of the model.
204 + ignore_patterns (Optional[Union[str, List[str]]]): The patterns to
205 + filter out the weight files. Files matched by any of the patterns
206 + will be ignored.
207
208 Returns:
209 str: The path to the downloaded model weights.
@@ -223,6 +227,7 @@ def download_weights_from_hf(
227 hf_folder = snapshot_download(
228 model_name_or_path,
229 allow_patterns=allow_patterns,
230 + ignore_patterns=ignore_patterns,
231 cache_dir=cache_dir,
232 tqdm_class=DisabledTqdm,
233 revision=revision,