A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[3/N] model runner pass the whole config to model (#9958)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 2, 2024 at 12:08 UTC
cea808f32549973cc19204355c950ad005eeed87
9 files changed
+87
-140
tests/lora/conftest.py
+4
-5
@@ -248,11 +248,10 @@ def llama_2_7b_engine_extra_embeddings():
248
cleanup_dist_env_and_memory(shutdown_ray=True)
249
get_model_old = get_model
250
251
- def get_model_patched(*, model_config, device_config, **kwargs):
252
- kwargs["lora_config"] = LoRAConfig(max_loras=4, max_lora_rank=8)
253
- return get_model_old(model_config=model_config,
254
- device_config=device_config,
255
- **kwargs)
251
+ def get_model_patched(**kwargs):
252
+ kwargs["vllm_config"].lora_config = LoRAConfig(max_loras=4,
253
+ max_lora_rank=8)
254
+ return get_model_old(**kwargs)
255
256
with patch("vllm.worker.model_runner.get_model", get_model_patched):
257
engine = vllm.LLM("meta-llama/Llama-2-7b-hf", enable_lora=False)
vllm/model_executor/model_loader/__init__.py
+4
-16
@@ -1,27 +1,15 @@
1
-from typing import Optional
2
-
1
from torch import nn
2
5
-from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6
- ModelConfig, ParallelConfig, SchedulerConfig)
3
+from vllm.config import VllmConfig
4
from vllm.model_executor.model_loader.loader import (BaseModelLoader,
5
get_model_loader)
6
from vllm.model_executor.model_loader.utils import (
7
get_architecture_class_name, get_model_architecture)
8
9
13
-def get_model(*, model_config: ModelConfig, load_config: LoadConfig,
14
- device_config: DeviceConfig, parallel_config: ParallelConfig,
15
- scheduler_config: SchedulerConfig,
16
- lora_config: Optional[LoRAConfig],
17
- cache_config: CacheConfig) -> nn.Module:
18
- loader = get_model_loader(load_config)
19
- return loader.load_model(model_config=model_config,
20
- device_config=device_config,
21
- lora_config=lora_config,
22
- parallel_config=parallel_config,
23
- scheduler_config=scheduler_config,
24
- cache_config=cache_config)
10
+def get_model(*, vllm_config: VllmConfig) -> nn.Module:
11
+ loader = get_model_loader(vllm_config.load_config)
12
+ return loader.load_model(vllm_config=vllm_config)
13
14
15
__all__ = [
vllm/model_executor/model_loader/loader.py
+54
-78
@@ -21,9 +21,9 @@ from torch import nn
21
from transformers import AutoModelForCausalLM, PretrainedConfig
22
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
23
24
-from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoadFormat,
25
- LoRAConfig, ModelConfig, MultiModalConfig,
26
- ParallelConfig, PoolerConfig, SchedulerConfig)
24
+from vllm.config import (CacheConfig, LoadConfig, LoadFormat, LoRAConfig,
25
+ ModelConfig, MultiModalConfig, ParallelConfig,
26
+ PoolerConfig, SchedulerConfig, VllmConfig)
27
from vllm.distributed import (get_tensor_model_parallel_rank,
28
get_tensor_model_parallel_world_size)
29
from vllm.envs import VLLM_USE_MODELSCOPE
@@ -150,6 +150,7 @@ def _get_model_initialization_kwargs(
150
151
152
def build_model(model_class: Type[nn.Module],
153
+ vllm_config: VllmConfig,
154
hf_config: PretrainedConfig,
155
cache_config: Optional[CacheConfig],
156
quant_config: Optional[QuantizationConfig],
@@ -166,23 +167,29 @@ def build_model(model_class: Type[nn.Module],
167
if prefix:
168
extra_kwargs["prefix"] = prefix
169
170
+ # TODO: unify all the module initialization code
171
+ # to only take the `VllmConfig` object as input
172
+ from vllm.plugins import set_vllm_config
173
+ set_vllm_config(vllm_config)
174
+
175
return model_class(config=hf_config,
176
cache_config=cache_config,
177
quant_config=quant_config,
178
**extra_kwargs)
179
180
175
-def _initialize_model(
176
- model_config: ModelConfig,
177
- load_config: LoadConfig,
178
- lora_config: Optional[LoRAConfig],
179
- cache_config: CacheConfig,
180
- scheduler_config: Optional[SchedulerConfig] = None) -> nn.Module:
181
+def _initialize_model(vllm_config: VllmConfig) -> nn.Module:
182
"""Initialize a model with the given configurations."""
183
+ model_config = vllm_config.model_config
184
+ lora_config = vllm_config.lora_config
185
+ scheduler_config = vllm_config.scheduler_config
186
+ cache_config = vllm_config.cache_config
187
+ load_config = vllm_config.load_config
188
model_class, _ = get_model_architecture(model_config)
189
190
return build_model(
191
model_class,
192
+ vllm_config,
193
model_config.hf_config,
194
cache_config=cache_config,
195
quant_config=_get_quantization_config(model_config, load_config),
@@ -205,12 +212,7 @@ class BaseModelLoader(ABC):
212
raise NotImplementedError
213
214
@abstractmethod
208
- def load_model(self, *, model_config: ModelConfig,
209
- device_config: DeviceConfig,
210
- lora_config: Optional[LoRAConfig],
211
- parallel_config: ParallelConfig,
212
- scheduler_config: SchedulerConfig,
213
- cache_config: CacheConfig) -> nn.Module:
215
+ def load_model(self, *, vllm_config: VllmConfig) -> nn.Module:
216
"""Load a model with the given configurations."""
217
raise NotImplementedError
218
@@ -396,18 +398,14 @@ class DefaultModelLoader(BaseModelLoader):
398
model_config.revision,
399
fall_back_to_pt=True)
400
399
- def load_model(self, *, model_config: ModelConfig,
400
- device_config: DeviceConfig,
401
- lora_config: Optional[LoRAConfig],
402
- parallel_config: ParallelConfig,
403
- scheduler_config: SchedulerConfig,
404
- cache_config: CacheConfig) -> nn.Module:
401
+ def load_model(self, vllm_config: VllmConfig) -> nn.Module:
402
+ device_config = vllm_config.device_config
403
+ model_config = vllm_config.model_config
404
+
405
target_device = torch.device(device_config.device)
406
with set_default_torch_dtype(model_config.dtype):
407
with target_device:
408
- model = _initialize_model(model_config, self.load_config,
409
- lora_config, cache_config,
410
- scheduler_config)
408
+ model = _initialize_model(vllm_config=vllm_config)
409
410
model.load_weights(self._get_all_weights(model_config, model))
411
@@ -436,17 +434,12 @@ class DummyModelLoader(BaseModelLoader):
434
def download_model(self, model_config: ModelConfig) -> None:
435
pass # Nothing to download
436
439
- def load_model(self, *, model_config: ModelConfig,
440
- device_config: DeviceConfig,
441
- lora_config: Optional[LoRAConfig],
442
- parallel_config: ParallelConfig,
443
- scheduler_config: SchedulerConfig,
444
- cache_config: CacheConfig) -> nn.Module:
437
+ def load_model(self, vllm_config: VllmConfig) -> nn.Module:
438
+ device_config = vllm_config.device_config
439
+ model_config = vllm_config.model_config
440
with set_default_torch_dtype(model_config.dtype):
441
with torch.device(device_config.device):
447
- model = _initialize_model(model_config, self.load_config,
448
- lora_config, cache_config,
449
- scheduler_config)
442
+ model = _initialize_model(vllm_config=vllm_config)
443
# NOTE(woosuk): For accurate performance evaluation, we assign
444
# random values to the weights.
445
initialize_dummy_weights(model)
@@ -488,10 +481,7 @@ class TensorizerLoader(BaseModelLoader):
481
482
def _load_model_serialized_cpu(
483
self,
491
- model_config: ModelConfig,
492
- device_config: DeviceConfig,
493
- lora_config: Optional[LoRAConfig],
494
- cache_config: CacheConfig,
484
+ vllm_config: VllmConfig,
485
) -> nn.Module:
486
"""Load a serialized model with tensorizer to the CPU.
487
@@ -500,26 +490,30 @@ class TensorizerLoader(BaseModelLoader):
490
default HuggingFace loading, but will be slower than loading a
491
vLLM-tensorized model.
492
"""
493
+ device_config = vllm_config.device_config
494
+ model_config = vllm_config.model_config
495
with set_default_torch_dtype(model_config.dtype):
496
with torch.device(device_config.device):
505
- model = _initialize_model(model_config, self.load_config,
506
- lora_config, cache_config)
497
+ model = _initialize_model(vllm_config=vllm_config)
498
499
model.load_weights(self._get_weights_iterator())
500
return model.eval()
501
502
def _load_model_serialized(
503
self,
513
- model_config: ModelConfig,
514
- device_config: DeviceConfig,
515
- lora_config: Optional[LoRAConfig],
516
- cache_config: CacheConfig,
504
+ vllm_config: VllmConfig,
505
) -> nn.Module:
506
"""Load a serialized model with tensorizer.
507
508
Expects a vLLM-tensorized model. See the
509
examples/tensorize_vllm_model.py example script
510
for serializing vLLM models."""
511
+
512
+ device_config = vllm_config.device_config
513
+ model_config = vllm_config.model_config
514
+ lora_config = vllm_config.lora_config
515
+ cache_config = vllm_config.cache_config
516
+
517
with set_default_torch_dtype(model_config.dtype):
518
with torch.device(device_config.device):
519
model_class = get_model_architecture(model_config)[0]
@@ -544,12 +538,9 @@ class TensorizerLoader(BaseModelLoader):
538
with self.tensorizer_config.open_stream():
539
pass
540
547
- def load_model(self, *, model_config: ModelConfig,
548
- device_config: DeviceConfig,
549
- lora_config: Optional[LoRAConfig],
550
- parallel_config: ParallelConfig,
551
- scheduler_config: SchedulerConfig,
552
- cache_config: CacheConfig) -> nn.Module:
541
+ def load_model(self, vllm_config: VllmConfig) -> nn.Module:
542
+ model_config = vllm_config.model_config
543
+ parallel_config = vllm_config.parallel_config
544
self._verify_config(model_config, parallel_config)
545
546
if parallel_config.tensor_parallel_size > 1:
@@ -559,10 +550,8 @@ class TensorizerLoader(BaseModelLoader):
550
% get_tensor_model_parallel_rank()
551
552
if is_vllm_tensorized(self.tensorizer_config):
562
- return self._load_model_serialized(model_config, device_config,
563
- lora_config, cache_config)
564
- return self._load_model_serialized_cpu(model_config, device_config,
565
- lora_config, cache_config)
553
+ return self._load_model_serialized(vllm_config=vllm_config)
554
+ return self._load_model_serialized_cpu(vllm_config=vllm_config)
555
556
@staticmethod
557
def save_model(
@@ -648,12 +637,9 @@ class ShardedStateLoader(BaseModelLoader):
637
def download_model(self, model_config: ModelConfig) -> None:
638
self._prepare_weights(model_config.model, model_config.revision)
639
651
- def load_model(self, *, model_config: ModelConfig,
652
- device_config: DeviceConfig,
653
- lora_config: Optional[LoRAConfig],
654
- parallel_config: ParallelConfig,
655
- scheduler_config: SchedulerConfig,
656
- cache_config: CacheConfig) -> nn.Module:
640
+ def load_model(self, vllm_config: VllmConfig) -> nn.Module:
641
+ device_config = vllm_config.device_config
642
+ model_config = vllm_config.model_config
643
from safetensors.torch import safe_open
644
645
from vllm.distributed import get_tensor_model_parallel_rank
@@ -663,8 +649,7 @@ class ShardedStateLoader(BaseModelLoader):
649
650
with set_default_torch_dtype(model_config.dtype):
651
with torch.device(device_config.device):
666
- model = _initialize_model(model_config, self.load_config,
667
- lora_config, cache_config)
652
+ model = _initialize_model(vllm_config=vllm_config)
653
for _, module in model.named_modules():
654
quant_method = getattr(module, "quant_method", None)
655
if quant_method is not None:
@@ -1157,16 +1142,12 @@ class BitsAndBytesModelLoader(BaseModelLoader):
1142
def download_model(self, model_config: ModelConfig) -> None:
1143
self._prepare_weights(model_config.model, model_config.revision)
1144
1160
- def load_model(self, *, model_config: ModelConfig,
1161
- device_config: DeviceConfig,
1162
- lora_config: Optional[LoRAConfig],
1163
- parallel_config: ParallelConfig,
1164
- scheduler_config: SchedulerConfig,
1165
- cache_config: CacheConfig) -> nn.Module:
1145
+ def load_model(self, vllm_config: VllmConfig) -> nn.Module:
1146
+ device_config = vllm_config.device_config
1147
+ model_config = vllm_config.model_config
1148
with set_default_torch_dtype(model_config.dtype):
1149
with torch.device(device_config.device):
1168
- model = _initialize_model(model_config, self.load_config,
1169
- lora_config, cache_config)
1150
+ model = _initialize_model(vllm_config=vllm_config)
1151
1152
self._load_weights(model_config, model)
1153
@@ -1235,13 +1216,9 @@ class GGUFModelLoader(BaseModelLoader):
1216
def download_model(self, model_config: ModelConfig) -> None:
1217
self._prepare_weights(model_config.model)
1218
1238
- def load_model(self, *, model_config: ModelConfig,
1239
- device_config: DeviceConfig,
1240
- lora_config: Optional[LoRAConfig],
1241
- parallel_config: ParallelConfig,
1242
- scheduler_config: SchedulerConfig,
1243
- cache_config: CacheConfig) -> nn.Module:
1244
-
1219
+ def load_model(self, vllm_config: VllmConfig) -> nn.Module:
1220
+ device_config = vllm_config.device_config
1221
+ model_config = vllm_config.model_config
1222
local_model_path = self._prepare_weights(model_config.model)
1223
gguf_weights_map = self._get_gguf_weights_map(model_config)
1224
# we can only know if tie word embeddings after mapping weights
@@ -1251,8 +1228,7 @@ class GGUFModelLoader(BaseModelLoader):
1228
1229
with set_default_torch_dtype(model_config.dtype):
1230
with torch.device(device_config.device):
1254
- model = _initialize_model(model_config, self.load_config,
1255
- lora_config, cache_config)
1231
+ model = _initialize_model(vllm_config=vllm_config)
1232
model.load_weights(
1233
self._get_weights_iterator(local_model_path, gguf_weights_map))
1234
return model
vllm/plugins/__init__.py
+20
-2
@@ -1,8 +1,14 @@
1
import logging
2
-from typing import Callable, Optional, Union
2
+from typing import TYPE_CHECKING, Callable, Optional, Union
3
4
import vllm.envs as envs
5
-from vllm.compilation.config import CompilationConfig
5
+
6
+if TYPE_CHECKING:
7
+ from vllm.compilation.config import CompilationConfig
8
+ from vllm.config import VllmConfig
9
+else:
10
+ CompilationConfig = None
11
+ VllmConfig = None
12
13
logger = logging.getLogger(__name__)
14
@@ -55,3 +61,15 @@ def set_compilation_config(config: Optional[CompilationConfig]):
61
62
def get_compilation_config() -> Optional[CompilationConfig]:
63
return _compilation_config
64
+
65
+
66
+_vllm_config: Optional[VllmConfig] = None
67
+
68
+
69
+def set_vllm_config(config: Optional[VllmConfig]):
70
+ global _vllm_config
71
+ _vllm_config = config
72
+
73
+
74
+def get_vllm_config() -> Optional[VllmConfig]:
75
+ return _vllm_config
vllm/v1/worker/gpu_model_runner.py
+1
-7
@@ -369,13 +369,7 @@ class GPUModelRunner:
369
logger.info("Starting to load model %s...", self.model_config.model)
370
with DeviceMemoryProfiler() as m: # noqa: SIM117
371
with patch("vllm.model_executor.layers.sampler.Sampler", Sampler):
372
- self.model = get_model(model_config=self.model_config,
373
- device_config=self.device_config,
374
- load_config=self.load_config,
375
- lora_config=self.lora_config,
376
- parallel_config=self.parallel_config,
377
- scheduler_config=self.scheduler_config,
378
- cache_config=self.cache_config)
372
+ self.model = get_model(vllm_config=self.vllm_config)
373
374
self.model_memory_usage = m.consumed_memory
375
logger.info("Loading model weights took %.4f GB",
vllm/worker/cpu_model_runner.py
+1
-7
@@ -453,13 +453,7 @@ class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
453
return uses_mrope(self.model_config.hf_config)
454
455
def load_model(self) -> None:
456
- self.model = get_model(model_config=self.model_config,
457
- load_config=self.load_config,
458
- device_config=self.device_config,
459
- lora_config=self.lora_config,
460
- parallel_config=self.parallel_config,
461
- scheduler_config=self.scheduler_config,
462
- cache_config=self.cache_config)
456
+ self.model = get_model(vllm_config=self.vllm_config)
457
458
def make_model_input_from_broadcasted_tensor_dict(
459
self,
vllm/worker/model_runner.py
+1
-7
@@ -1051,13 +1051,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1051
def load_model(self) -> None:
1052
logger.info("Starting to load model %s...", self.model_config.model)
1053
with DeviceMemoryProfiler() as m:
1054
- self.model = get_model(model_config=self.model_config,
1055
- device_config=self.device_config,
1056
- load_config=self.load_config,
1057
- lora_config=self.lora_config,
1058
- parallel_config=self.parallel_config,
1059
- scheduler_config=self.scheduler_config,
1060
- cache_config=self.cache_config)
1054
+ self.model = get_model(vllm_config=self.vllm_config)
1055
1056
self.model_memory_usage = m.consumed_memory
1057
logger.info("Loading model weights took %.4f GB",
vllm/worker/tpu_model_runner.py
+1
-9
@@ -137,15 +137,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
137
"vllm.model_executor.layers.vocab_parallel_embedding."
138
"get_tensor_model_parallel_rank",
139
return_value=xm_tp_rank):
140
- model = get_model(
141
- model_config=self.model_config,
142
- load_config=self.load_config,
143
- device_config=self.device_config,
144
- parallel_config=self.parallel_config,
145
- cache_config=self.cache_config,
146
- scheduler_config=self.scheduler_config,
147
- lora_config=None,
148
- )
140
+ model = get_model(vllm_config=self.vllm_config)
141
model = model.eval()
142
xm.wait_device_ops()
143
self.model = ModelWrapper(model)
vllm/worker/xpu_model_runner.py
+1
-9
@@ -405,15 +405,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
405
406
def load_model(self) -> None:
407
with DeviceMemoryProfiler() as m:
408
- self.model = get_model(
409
- model_config=self.model_config,
410
- device_config=self.device_config,
411
- load_config=self.load_config,
412
- lora_config=self.lora_config,
413
- parallel_config=self.parallel_config,
414
- scheduler_config=self.scheduler_config,
415
- cache_config=self.cache_config,
416
- )
408
+ self.model = get_model(vllm_config=self.vllm_config)
409
410
self.model_memory_usage = m.consumed_memory
411
logger.info("Loading model weights took %.4f GB",