@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cea808f325
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[3/N] model runner pass the whole config to model (#9958)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 2, 2024 at 12:08 UTC cea808f32549973cc19204355c950ad005eeed87
9 files changed +87 -140
tests/lora/conftest.py
+4 -5
@@ -248,11 +248,10 @@ def llama_2_7b_engine_extra_embeddings():
248 cleanup_dist_env_and_memory(shutdown_ray=True)
249 get_model_old = get_model
250
251 - def get_model_patched(*, model_config, device_config, **kwargs):
252 - kwargs["lora_config"] = LoRAConfig(max_loras=4, max_lora_rank=8)
253 - return get_model_old(model_config=model_config,
254 - device_config=device_config,
255 - **kwargs)
251 + def get_model_patched(**kwargs):
252 + kwargs["vllm_config"].lora_config = LoRAConfig(max_loras=4,
253 + max_lora_rank=8)
254 + return get_model_old(**kwargs)
255
256 with patch("vllm.worker.model_runner.get_model", get_model_patched):
257 engine = vllm.LLM("meta-llama/Llama-2-7b-hf", enable_lora=False)
vllm/model_executor/model_loader/__init__.py
+4 -16
@@ -1,27 +1,15 @@
1 -from typing import Optional
2 -
1 from torch import nn
2
5 -from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6 - ModelConfig, ParallelConfig, SchedulerConfig)
3 +from vllm.config import VllmConfig
4 from vllm.model_executor.model_loader.loader import (BaseModelLoader,
5 get_model_loader)
6 from vllm.model_executor.model_loader.utils import (
7 get_architecture_class_name, get_model_architecture)
8
9
13 -def get_model(*, model_config: ModelConfig, load_config: LoadConfig,
14 - device_config: DeviceConfig, parallel_config: ParallelConfig,
15 - scheduler_config: SchedulerConfig,
16 - lora_config: Optional[LoRAConfig],
17 - cache_config: CacheConfig) -> nn.Module:
18 - loader = get_model_loader(load_config)
19 - return loader.load_model(model_config=model_config,
20 - device_config=device_config,
21 - lora_config=lora_config,
22 - parallel_config=parallel_config,
23 - scheduler_config=scheduler_config,
24 - cache_config=cache_config)
10 +def get_model(*, vllm_config: VllmConfig) -> nn.Module:
11 + loader = get_model_loader(vllm_config.load_config)
12 + return loader.load_model(vllm_config=vllm_config)
13
14
15 __all__ = [
vllm/model_executor/model_loader/loader.py
+54 -78
@@ -21,9 +21,9 @@ from torch import nn
21 from transformers import AutoModelForCausalLM, PretrainedConfig
22 from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
23
24 -from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoadFormat,
25 - LoRAConfig, ModelConfig, MultiModalConfig,
26 - ParallelConfig, PoolerConfig, SchedulerConfig)
24 +from vllm.config import (CacheConfig, LoadConfig, LoadFormat, LoRAConfig,
25 + ModelConfig, MultiModalConfig, ParallelConfig,
26 + PoolerConfig, SchedulerConfig, VllmConfig)
27 from vllm.distributed import (get_tensor_model_parallel_rank,
28 get_tensor_model_parallel_world_size)
29 from vllm.envs import VLLM_USE_MODELSCOPE
@@ -150,6 +150,7 @@ def _get_model_initialization_kwargs(
150
151
152 def build_model(model_class: Type[nn.Module],
153 + vllm_config: VllmConfig,
154 hf_config: PretrainedConfig,
155 cache_config: Optional[CacheConfig],
156 quant_config: Optional[QuantizationConfig],
@@ -166,23 +167,29 @@ def build_model(model_class: Type[nn.Module],
167 if prefix:
168 extra_kwargs["prefix"] = prefix
169
170 + # TODO: unify all the module initialization code
171 + # to only take the `VllmConfig` object as input
172 + from vllm.plugins import set_vllm_config
173 + set_vllm_config(vllm_config)
174 +
175 return model_class(config=hf_config,
176 cache_config=cache_config,
177 quant_config=quant_config,
178 **extra_kwargs)
179
180
175 -def _initialize_model(
176 - model_config: ModelConfig,
177 - load_config: LoadConfig,
178 - lora_config: Optional[LoRAConfig],
179 - cache_config: CacheConfig,
180 - scheduler_config: Optional[SchedulerConfig] = None) -> nn.Module:
181 +def _initialize_model(vllm_config: VllmConfig) -> nn.Module:
182 """Initialize a model with the given configurations."""
183 + model_config = vllm_config.model_config
184 + lora_config = vllm_config.lora_config
185 + scheduler_config = vllm_config.scheduler_config
186 + cache_config = vllm_config.cache_config
187 + load_config = vllm_config.load_config
188 model_class, _ = get_model_architecture(model_config)
189
190 return build_model(
191 model_class,
192 + vllm_config,
193 model_config.hf_config,
194 cache_config=cache_config,
195 quant_config=_get_quantization_config(model_config, load_config),
@@ -205,12 +212,7 @@ class BaseModelLoader(ABC):
212 raise NotImplementedError
213
214 @abstractmethod
208 - def load_model(self, *, model_config: ModelConfig,
209 - device_config: DeviceConfig,
210 - lora_config: Optional[LoRAConfig],
211 - parallel_config: ParallelConfig,
212 - scheduler_config: SchedulerConfig,
213 - cache_config: CacheConfig) -> nn.Module:
215 + def load_model(self, *, vllm_config: VllmConfig) -> nn.Module:
216 """Load a model with the given configurations."""
217 raise NotImplementedError
218
@@ -396,18 +398,14 @@ class DefaultModelLoader(BaseModelLoader):
398 model_config.revision,
399 fall_back_to_pt=True)
400
399 - def load_model(self, *, model_config: ModelConfig,
400 - device_config: DeviceConfig,
401 - lora_config: Optional[LoRAConfig],
402 - parallel_config: ParallelConfig,
403 - scheduler_config: SchedulerConfig,
404 - cache_config: CacheConfig) -> nn.Module:
401 + def load_model(self, vllm_config: VllmConfig) -> nn.Module:
402 + device_config = vllm_config.device_config
403 + model_config = vllm_config.model_config
404 +
405 target_device = torch.device(device_config.device)
406 with set_default_torch_dtype(model_config.dtype):
407 with target_device:
408 - model = _initialize_model(model_config, self.load_config,
409 - lora_config, cache_config,
410 - scheduler_config)
408 + model = _initialize_model(vllm_config=vllm_config)
409
410 model.load_weights(self._get_all_weights(model_config, model))
411
@@ -436,17 +434,12 @@ class DummyModelLoader(BaseModelLoader):
434 def download_model(self, model_config: ModelConfig) -> None:
435 pass # Nothing to download
436
439 - def load_model(self, *, model_config: ModelConfig,
440 - device_config: DeviceConfig,
441 - lora_config: Optional[LoRAConfig],
442 - parallel_config: ParallelConfig,
443 - scheduler_config: SchedulerConfig,
444 - cache_config: CacheConfig) -> nn.Module:
437 + def load_model(self, vllm_config: VllmConfig) -> nn.Module:
438 + device_config = vllm_config.device_config
439 + model_config = vllm_config.model_config
440 with set_default_torch_dtype(model_config.dtype):
441 with torch.device(device_config.device):
447 - model = _initialize_model(model_config, self.load_config,
448 - lora_config, cache_config,
449 - scheduler_config)
442 + model = _initialize_model(vllm_config=vllm_config)
443 # NOTE(woosuk): For accurate performance evaluation, we assign
444 # random values to the weights.
445 initialize_dummy_weights(model)
@@ -488,10 +481,7 @@ class TensorizerLoader(BaseModelLoader):
481
482 def _load_model_serialized_cpu(
483 self,
491 - model_config: ModelConfig,
492 - device_config: DeviceConfig,
493 - lora_config: Optional[LoRAConfig],
494 - cache_config: CacheConfig,
484 + vllm_config: VllmConfig,
485 ) -> nn.Module:
486 """Load a serialized model with tensorizer to the CPU.
487
@@ -500,26 +490,30 @@ class TensorizerLoader(BaseModelLoader):
490 default HuggingFace loading, but will be slower than loading a
491 vLLM-tensorized model.
492 """
493 + device_config = vllm_config.device_config
494 + model_config = vllm_config.model_config
495 with set_default_torch_dtype(model_config.dtype):
496 with torch.device(device_config.device):
505 - model = _initialize_model(model_config, self.load_config,
506 - lora_config, cache_config)
497 + model = _initialize_model(vllm_config=vllm_config)
498
499 model.load_weights(self._get_weights_iterator())
500 return model.eval()
501
502 def _load_model_serialized(
503 self,
513 - model_config: ModelConfig,
514 - device_config: DeviceConfig,
515 - lora_config: Optional[LoRAConfig],
516 - cache_config: CacheConfig,
504 + vllm_config: VllmConfig,
505 ) -> nn.Module:
506 """Load a serialized model with tensorizer.
507
508 Expects a vLLM-tensorized model. See the
509 examples/tensorize_vllm_model.py example script
510 for serializing vLLM models."""
511 +
512 + device_config = vllm_config.device_config
513 + model_config = vllm_config.model_config
514 + lora_config = vllm_config.lora_config
515 + cache_config = vllm_config.cache_config
516 +
517 with set_default_torch_dtype(model_config.dtype):
518 with torch.device(device_config.device):
519 model_class = get_model_architecture(model_config)[0]
@@ -544,12 +538,9 @@ class TensorizerLoader(BaseModelLoader):
538 with self.tensorizer_config.open_stream():
539 pass
540
547 - def load_model(self, *, model_config: ModelConfig,
548 - device_config: DeviceConfig,
549 - lora_config: Optional[LoRAConfig],
550 - parallel_config: ParallelConfig,
551 - scheduler_config: SchedulerConfig,
552 - cache_config: CacheConfig) -> nn.Module:
541 + def load_model(self, vllm_config: VllmConfig) -> nn.Module:
542 + model_config = vllm_config.model_config
543 + parallel_config = vllm_config.parallel_config
544 self._verify_config(model_config, parallel_config)
545
546 if parallel_config.tensor_parallel_size > 1:
@@ -559,10 +550,8 @@ class TensorizerLoader(BaseModelLoader):
550 % get_tensor_model_parallel_rank()
551
552 if is_vllm_tensorized(self.tensorizer_config):
562 - return self._load_model_serialized(model_config, device_config,
563 - lora_config, cache_config)
564 - return self._load_model_serialized_cpu(model_config, device_config,
565 - lora_config, cache_config)
553 + return self._load_model_serialized(vllm_config=vllm_config)
554 + return self._load_model_serialized_cpu(vllm_config=vllm_config)
555
556 @staticmethod
557 def save_model(
@@ -648,12 +637,9 @@ class ShardedStateLoader(BaseModelLoader):
637 def download_model(self, model_config: ModelConfig) -> None:
638 self._prepare_weights(model_config.model, model_config.revision)
639
651 - def load_model(self, *, model_config: ModelConfig,
652 - device_config: DeviceConfig,
653 - lora_config: Optional[LoRAConfig],
654 - parallel_config: ParallelConfig,
655 - scheduler_config: SchedulerConfig,
656 - cache_config: CacheConfig) -> nn.Module:
640 + def load_model(self, vllm_config: VllmConfig) -> nn.Module:
641 + device_config = vllm_config.device_config
642 + model_config = vllm_config.model_config
643 from safetensors.torch import safe_open
644
645 from vllm.distributed import get_tensor_model_parallel_rank
@@ -663,8 +649,7 @@ class ShardedStateLoader(BaseModelLoader):
649
650 with set_default_torch_dtype(model_config.dtype):
651 with torch.device(device_config.device):
666 - model = _initialize_model(model_config, self.load_config,
667 - lora_config, cache_config)
652 + model = _initialize_model(vllm_config=vllm_config)
653 for _, module in model.named_modules():
654 quant_method = getattr(module, "quant_method", None)
655 if quant_method is not None:
@@ -1157,16 +1142,12 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1142 def download_model(self, model_config: ModelConfig) -> None:
1143 self._prepare_weights(model_config.model, model_config.revision)
1144
1160 - def load_model(self, *, model_config: ModelConfig,
1161 - device_config: DeviceConfig,
1162 - lora_config: Optional[LoRAConfig],
1163 - parallel_config: ParallelConfig,
1164 - scheduler_config: SchedulerConfig,
1165 - cache_config: CacheConfig) -> nn.Module:
1145 + def load_model(self, vllm_config: VllmConfig) -> nn.Module:
1146 + device_config = vllm_config.device_config
1147 + model_config = vllm_config.model_config
1148 with set_default_torch_dtype(model_config.dtype):
1149 with torch.device(device_config.device):
1168 - model = _initialize_model(model_config, self.load_config,
1169 - lora_config, cache_config)
1150 + model = _initialize_model(vllm_config=vllm_config)
1151
1152 self._load_weights(model_config, model)
1153
@@ -1235,13 +1216,9 @@ class GGUFModelLoader(BaseModelLoader):
1216 def download_model(self, model_config: ModelConfig) -> None:
1217 self._prepare_weights(model_config.model)
1218
1238 - def load_model(self, *, model_config: ModelConfig,
1239 - device_config: DeviceConfig,
1240 - lora_config: Optional[LoRAConfig],
1241 - parallel_config: ParallelConfig,
1242 - scheduler_config: SchedulerConfig,
1243 - cache_config: CacheConfig) -> nn.Module:
1244 -
1219 + def load_model(self, vllm_config: VllmConfig) -> nn.Module:
1220 + device_config = vllm_config.device_config
1221 + model_config = vllm_config.model_config
1222 local_model_path = self._prepare_weights(model_config.model)
1223 gguf_weights_map = self._get_gguf_weights_map(model_config)
1224 # we can only know if tie word embeddings after mapping weights
@@ -1251,8 +1228,7 @@ class GGUFModelLoader(BaseModelLoader):
1228
1229 with set_default_torch_dtype(model_config.dtype):
1230 with torch.device(device_config.device):
1254 - model = _initialize_model(model_config, self.load_config,
1255 - lora_config, cache_config)
1231 + model = _initialize_model(vllm_config=vllm_config)
1232 model.load_weights(
1233 self._get_weights_iterator(local_model_path, gguf_weights_map))
1234 return model
vllm/plugins/__init__.py
+20 -2
@@ -1,8 +1,14 @@
1 import logging
2 -from typing import Callable, Optional, Union
2 +from typing import TYPE_CHECKING, Callable, Optional, Union
3
4 import vllm.envs as envs
5 -from vllm.compilation.config import CompilationConfig
5 +
6 +if TYPE_CHECKING:
7 + from vllm.compilation.config import CompilationConfig
8 + from vllm.config import VllmConfig
9 +else:
10 + CompilationConfig = None
11 + VllmConfig = None
12
13 logger = logging.getLogger(__name__)
14
@@ -55,3 +61,15 @@ def set_compilation_config(config: Optional[CompilationConfig]):
61
62 def get_compilation_config() -> Optional[CompilationConfig]:
63 return _compilation_config
64 +
65 +
66 +_vllm_config: Optional[VllmConfig] = None
67 +
68 +
69 +def set_vllm_config(config: Optional[VllmConfig]):
70 + global _vllm_config
71 + _vllm_config = config
72 +
73 +
74 +def get_vllm_config() -> Optional[VllmConfig]:
75 + return _vllm_config
vllm/v1/worker/gpu_model_runner.py
+1 -7
@@ -369,13 +369,7 @@ class GPUModelRunner:
369 logger.info("Starting to load model %s...", self.model_config.model)
370 with DeviceMemoryProfiler() as m: # noqa: SIM117
371 with patch("vllm.model_executor.layers.sampler.Sampler", Sampler):
372 - self.model = get_model(model_config=self.model_config,
373 - device_config=self.device_config,
374 - load_config=self.load_config,
375 - lora_config=self.lora_config,
376 - parallel_config=self.parallel_config,
377 - scheduler_config=self.scheduler_config,
378 - cache_config=self.cache_config)
372 + self.model = get_model(vllm_config=self.vllm_config)
373
374 self.model_memory_usage = m.consumed_memory
375 logger.info("Loading model weights took %.4f GB",
vllm/worker/cpu_model_runner.py
+1 -7
@@ -453,13 +453,7 @@ class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
453 return uses_mrope(self.model_config.hf_config)
454
455 def load_model(self) -> None:
456 - self.model = get_model(model_config=self.model_config,
457 - load_config=self.load_config,
458 - device_config=self.device_config,
459 - lora_config=self.lora_config,
460 - parallel_config=self.parallel_config,
461 - scheduler_config=self.scheduler_config,
462 - cache_config=self.cache_config)
456 + self.model = get_model(vllm_config=self.vllm_config)
457
458 def make_model_input_from_broadcasted_tensor_dict(
459 self,
vllm/worker/model_runner.py
+1 -7
@@ -1051,13 +1051,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1051 def load_model(self) -> None:
1052 logger.info("Starting to load model %s...", self.model_config.model)
1053 with DeviceMemoryProfiler() as m:
1054 - self.model = get_model(model_config=self.model_config,
1055 - device_config=self.device_config,
1056 - load_config=self.load_config,
1057 - lora_config=self.lora_config,
1058 - parallel_config=self.parallel_config,
1059 - scheduler_config=self.scheduler_config,
1060 - cache_config=self.cache_config)
1054 + self.model = get_model(vllm_config=self.vllm_config)
1055
1056 self.model_memory_usage = m.consumed_memory
1057 logger.info("Loading model weights took %.4f GB",
vllm/worker/tpu_model_runner.py
+1 -9
@@ -137,15 +137,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
137 "vllm.model_executor.layers.vocab_parallel_embedding."
138 "get_tensor_model_parallel_rank",
139 return_value=xm_tp_rank):
140 - model = get_model(
141 - model_config=self.model_config,
142 - load_config=self.load_config,
143 - device_config=self.device_config,
144 - parallel_config=self.parallel_config,
145 - cache_config=self.cache_config,
146 - scheduler_config=self.scheduler_config,
147 - lora_config=None,
148 - )
140 + model = get_model(vllm_config=self.vllm_config)
141 model = model.eval()
142 xm.wait_device_ops()
143 self.model = ModelWrapper(model)
vllm/worker/xpu_model_runner.py
+1 -9
@@ -405,15 +405,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
405
406 def load_model(self) -> None:
407 with DeviceMemoryProfiler() as m:
408 - self.model = get_model(
409 - model_config=self.model_config,
410 - device_config=self.device_config,
411 - load_config=self.load_config,
412 - lora_config=self.lora_config,
413 - parallel_config=self.parallel_config,
414 - scheduler_config=self.scheduler_config,
415 - cache_config=self.cache_config,
416 - )
408 + self.model = get_model(vllm_config=self.vllm_config)
409
410 self.model_memory_usage = m.consumed_memory
411 logger.info("Loading model weights took %.4f GB",