A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Bugfix] Fix broken OpenAI tensorizer test (#8258)
Cyrus Leung committed
Sep 7, 2024 at 16:02 UTC
9f68e00d27b0f8252549be3adbb47c5b735a8103
4 files changed
+81
-40
tests/utils.py
+6
-6
@@ -20,7 +20,7 @@ from vllm.distributed import (ensure_model_parallel_initialized,
20
init_distributed_environment)
21
from vllm.engine.arg_utils import AsyncEngineArgs
22
from vllm.entrypoints.openai.cli_args import make_arg_parser
23
-from vllm.model_executor.model_loader.loader import DefaultModelLoader
23
+from vllm.model_executor.model_loader.loader import get_model_loader
24
from vllm.platforms import current_platform
25
from vllm.utils import FlexibleArgumentParser, get_open_port, is_hip
26
@@ -89,11 +89,11 @@ class RemoteOpenAIServer:
89
is_local = os.path.isdir(model)
90
if not is_local:
91
engine_args = AsyncEngineArgs.from_cli_args(args)
92
- engine_config = engine_args.create_engine_config()
93
- dummy_loader = DefaultModelLoader(engine_config.load_config)
94
- dummy_loader._prepare_weights(engine_config.model_config.model,
95
- engine_config.model_config.revision,
96
- fall_back_to_pt=True)
92
+ model_config = engine_args.create_model_config()
93
+ load_config = engine_args.create_load_config()
94
+
95
+ model_loader = get_model_loader(load_config)
96
+ model_loader.download_model(model_config)
97
98
env = os.environ.copy()
99
# the current process might initialize cuda,
vllm/engine/arg_utils.py
+39
-33
@@ -771,33 +771,8 @@ class EngineArgs:
771
engine_args = cls(**{attr: getattr(args, attr) for attr in attrs})
772
return engine_args
773
774
- def create_engine_config(self) -> EngineConfig:
775
- # gguf file needs a specific model loader and doesn't use hf_repo
776
- if check_gguf_file(self.model):
777
- self.quantization = self.load_format = "gguf"
778
-
779
- # bitsandbytes quantization needs a specific model loader
780
- # so we make sure the quant method and the load format are consistent
781
- if (self.quantization == "bitsandbytes" or
782
- self.qlora_adapter_name_or_path is not None) and \
783
- self.load_format != "bitsandbytes":
784
- raise ValueError(
785
- "BitsAndBytes quantization and QLoRA adapter only support "
786
- f"'bitsandbytes' load format, but got {self.load_format}")
787
-
788
- if (self.load_format == "bitsandbytes" or
789
- self.qlora_adapter_name_or_path is not None) and \
790
- self.quantization != "bitsandbytes":
791
- raise ValueError(
792
- "BitsAndBytes load format and QLoRA adapter only support "
793
- f"'bitsandbytes' quantization, but got {self.quantization}")
794
-
795
- assert self.cpu_offload_gb >= 0, (
796
- "CPU offload space must be non-negative"
797
- f", but got {self.cpu_offload_gb}")
798
-
799
- device_config = DeviceConfig(device=self.device)
800
- model_config = ModelConfig(
774
+ def create_model_config(self) -> ModelConfig:
775
+ return ModelConfig(
776
model=self.model,
777
tokenizer=self.tokenizer,
778
tokenizer_mode=self.tokenizer_mode,
@@ -825,6 +800,42 @@ class EngineArgs:
800
config_format=self.config_format,
801
)
802
803
+ def create_load_config(self) -> LoadConfig:
804
+ return LoadConfig(
805
+ load_format=self.load_format,
806
+ download_dir=self.download_dir,
807
+ model_loader_extra_config=self.model_loader_extra_config,
808
+ ignore_patterns=self.ignore_patterns,
809
+ )
810
+
811
+ def create_engine_config(self) -> EngineConfig:
812
+ # gguf file needs a specific model loader and doesn't use hf_repo
813
+ if check_gguf_file(self.model):
814
+ self.quantization = self.load_format = "gguf"
815
+
816
+ # bitsandbytes quantization needs a specific model loader
817
+ # so we make sure the quant method and the load format are consistent
818
+ if (self.quantization == "bitsandbytes" or
819
+ self.qlora_adapter_name_or_path is not None) and \
820
+ self.load_format != "bitsandbytes":
821
+ raise ValueError(
822
+ "BitsAndBytes quantization and QLoRA adapter only support "
823
+ f"'bitsandbytes' load format, but got {self.load_format}")
824
+
825
+ if (self.load_format == "bitsandbytes" or
826
+ self.qlora_adapter_name_or_path is not None) and \
827
+ self.quantization != "bitsandbytes":
828
+ raise ValueError(
829
+ "BitsAndBytes load format and QLoRA adapter only support "
830
+ f"'bitsandbytes' quantization, but got {self.quantization}")
831
+
832
+ assert self.cpu_offload_gb >= 0, (
833
+ "CPU offload space must be non-negative"
834
+ f", but got {self.cpu_offload_gb}")
835
+
836
+ device_config = DeviceConfig(device=self.device)
837
+ model_config = self.create_model_config()
838
+
839
cache_config = CacheConfig(
840
block_size=self.block_size if self.device != "neuron" else
841
self.max_model_len, # neuron needs block_size = max_model_len
@@ -967,12 +978,7 @@ class EngineArgs:
978
self.model_loader_extra_config[
979
"qlora_adapter_name_or_path"] = self.qlora_adapter_name_or_path
980
970
- load_config = LoadConfig(
971
- load_format=self.load_format,
972
- download_dir=self.download_dir,
973
- model_loader_extra_config=self.model_loader_extra_config,
974
- ignore_patterns=self.ignore_patterns,
975
- )
981
+ load_config = self.create_load_config()
982
983
prompt_adapter_config = PromptAdapterConfig(
984
max_prompt_adapters=self.max_prompt_adapters,
vllm/model_executor/model_loader/loader.py
+29
-1
@@ -185,6 +185,11 @@ class BaseModelLoader(ABC):
185
def __init__(self, load_config: LoadConfig):
186
self.load_config = load_config
187
188
+ @abstractmethod
189
+ def download_model(self, model_config: ModelConfig) -> None:
190
+ """Download a model so that it can be immediately loaded."""
191
+ raise NotImplementedError
192
+
193
@abstractmethod
194
def load_model(self, *, model_config: ModelConfig,
195
device_config: DeviceConfig,
@@ -193,7 +198,7 @@ class BaseModelLoader(ABC):
198
scheduler_config: SchedulerConfig,
199
cache_config: CacheConfig) -> nn.Module:
200
"""Load a model with the given configurations."""
196
- ...
201
+ raise NotImplementedError
202
203
204
class DefaultModelLoader(BaseModelLoader):
@@ -335,6 +340,11 @@ class DefaultModelLoader(BaseModelLoader):
340
weights_iterator = _xla_weights_iterator(weights_iterator)
341
return weights_iterator
342
343
+ def download_model(self, model_config: ModelConfig) -> None:
344
+ self._prepare_weights(model_config.model,
345
+ model_config.revision,
346
+ fall_back_to_pt=True)
347
+
348
def load_model(self, *, model_config: ModelConfig,
349
device_config: DeviceConfig,
350
lora_config: Optional[LoRAConfig],
@@ -377,6 +387,9 @@ class DummyModelLoader(BaseModelLoader):
387
raise ValueError(f"Model loader extra config is not supported for "
388
f"load format {load_config.load_format}")
389
390
+ def download_model(self, model_config: ModelConfig) -> None:
391
+ pass # Nothing to download
392
+
393
def load_model(self, *, model_config: ModelConfig,
394
device_config: DeviceConfig,
395
lora_config: Optional[LoRAConfig],
@@ -467,6 +480,12 @@ class TensorizerLoader(BaseModelLoader):
480
model = load_with_tensorizer(tensorizer_config, **extra_kwargs)
481
return model.eval()
482
483
+ def download_model(self, model_config: ModelConfig) -> None:
484
+ self.tensorizer_config.verify_with_model_config(model_config)
485
+
486
+ with self.tensorizer_config.open_stream():
487
+ pass
488
+
489
def load_model(self, *, model_config: ModelConfig,
490
device_config: DeviceConfig,
491
lora_config: Optional[LoRAConfig],
@@ -568,6 +587,9 @@ class ShardedStateLoader(BaseModelLoader):
587
ignore_patterns=self.load_config.ignore_patterns,
588
)
589
590
+ def download_model(self, model_config: ModelConfig) -> None:
591
+ self._prepare_weights(model_config.model, model_config.revision)
592
+
593
def load_model(self, *, model_config: ModelConfig,
594
device_config: DeviceConfig,
595
lora_config: Optional[LoRAConfig],
@@ -995,6 +1017,9 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1017
set_weight_attrs(
1018
param, {"matmul_state": [None] * len(quant_states)})
1019
1020
+ def download_model(self, model_config: ModelConfig) -> None:
1021
+ self._prepare_weights(model_config.model, model_config.revision)
1022
+
1023
def load_model(self, *, model_config: ModelConfig,
1024
device_config: DeviceConfig,
1025
lora_config: Optional[LoRAConfig],
@@ -1070,6 +1095,9 @@ class GGUFModelLoader(BaseModelLoader):
1095
return gguf_quant_weights_iterator(model_name_or_path,
1096
gguf_to_hf_name_map)
1097
1098
+ def download_model(self, model_config: ModelConfig) -> None:
1099
+ self._prepare_weights(model_config.model)
1100
+
1101
def load_model(self, *, model_config: ModelConfig,
1102
device_config: DeviceConfig,
1103
lora_config: Optional[LoRAConfig],
vllm/model_executor/model_loader/tensorizer.py
+7
@@ -99,6 +99,13 @@ class TensorizerConfig:
99
"Loading a model using Tensorizer with quantization on vLLM"
100
" is unstable and may lead to errors.")
101
102
+ def open_stream(self, tensorizer_args: Optional["TensorizerArgs"] = None):
103
+ if tensorizer_args is None:
104
+ tensorizer_args = self._construct_tensorizer_args()
105
+
106
+ return open_stream(self.tensorizer_uri,
107
+ **tensorizer_args.stream_params)
108
+
109
110
def load_with_tensorizer(tensorizer_config: TensorizerConfig,
111
**extra_kwargs) -> nn.Module: