A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Core][Model] Support loading weights by ID within models (#7931)
Peter Salas committed
Sep 24, 2024 at 00:14 UTC
3f06bae9079ee495a34cfadcd9c1ef2a23636084
2 files changed
+73
-17
vllm/model_executor/model_loader/loader.py
+47
-13
@@ -1,6 +1,7 @@
1
# ruff: noqa: SIM117
2
import collections
3
import copy
4
+import dataclasses
5
import fnmatch
6
import glob
7
import json
@@ -8,7 +9,8 @@ import math
9
import os
10
from abc import ABC, abstractmethod
11
from contextlib import contextmanager
11
-from typing import Any, Dict, Generator, List, Optional, Tuple, Type
12
+from typing import (Any, Dict, Generator, Iterable, List, Optional, Tuple,
13
+ Type, cast)
14
15
import gguf
16
import huggingface_hub
@@ -207,6 +209,22 @@ class BaseModelLoader(ABC):
209
class DefaultModelLoader(BaseModelLoader):
210
"""Model loader that can load different file types from disk."""
211
212
+ @dataclasses.dataclass
213
+ class Source:
214
+ """A source for weights."""
215
+
216
+ model_or_path: str
217
+ """The model ID or path."""
218
+
219
+ revision: Optional[str]
220
+ """The optional model revision."""
221
+
222
+ prefix: str = ""
223
+ """A prefix to prepend to all weights."""
224
+
225
+ fall_back_to_pt: bool = True
226
+ """Whether .pt weights can be used."""
227
+
228
def __init__(self, load_config: LoadConfig):
229
super().__init__(load_config)
230
if load_config.model_loader_extra_config:
@@ -313,17 +331,16 @@ class DefaultModelLoader(BaseModelLoader):
331
return hf_folder, hf_weights_files, use_safetensors
332
333
def _get_weights_iterator(
316
- self, model_name_or_path: str, revision: Optional[str],
317
- fall_back_to_pt: bool
334
+ self, source: "Source"
335
) -> Generator[Tuple[str, torch.Tensor], None, None]:
336
"""Get an iterator for the model weights based on the load format."""
337
hf_folder, hf_weights_files, use_safetensors = self._prepare_weights(
321
- model_name_or_path, revision, fall_back_to_pt)
338
+ source.model_or_path, source.revision, source.fall_back_to_pt)
339
if self.load_config.load_format == LoadFormat.NPCACHE:
340
# Currently np_cache only support *.bin checkpoints
341
assert use_safetensors is False
342
weights_iterator = np_cache_weights_iterator(
326
- model_name_or_path, self.load_config.download_dir, hf_folder,
343
+ source.model_or_path, self.load_config.download_dir, hf_folder,
344
hf_weights_files)
345
elif use_safetensors:
346
weights_iterator = safetensors_weights_iterator(hf_weights_files)
@@ -341,7 +358,29 @@ class DefaultModelLoader(BaseModelLoader):
358
xm.mark_step()
359
360
weights_iterator = _xla_weights_iterator(weights_iterator)
344
- return weights_iterator
361
+
362
+ # Apply the prefix.
363
+ return ((source.prefix + name, tensor)
364
+ for (name, tensor) in weights_iterator)
365
+
366
+ def _get_all_weights(
367
+ self,
368
+ model_config: ModelConfig,
369
+ model: nn.Module,
370
+ ) -> Generator[Tuple[str, torch.Tensor], None, None]:
371
+
372
+ primary_weights = DefaultModelLoader.Source(
373
+ model_config.model,
374
+ model_config.revision,
375
+ prefix="",
376
+ fall_back_to_pt=getattr(model, "fall_back_to_pt_during_load",
377
+ True))
378
+ yield from self._get_weights_iterator(primary_weights)
379
+
380
+ secondary_weights = cast(Iterable[DefaultModelLoader.Source],
381
+ getattr(model, "secondary_weights", ()))
382
+ for source in secondary_weights:
383
+ yield from self._get_weights_iterator(source)
384
385
def download_model(self, model_config: ModelConfig) -> None:
386
self._prepare_weights(model_config.model,
@@ -360,13 +399,8 @@ class DefaultModelLoader(BaseModelLoader):
399
model = _initialize_model(model_config, self.load_config,
400
lora_config, cache_config,
401
scheduler_config)
363
- model.load_weights(
364
- self._get_weights_iterator(model_config.model,
365
- model_config.revision,
366
- fall_back_to_pt=getattr(
367
- model,
368
- "fall_back_to_pt_during_load",
369
- True)), )
402
+
403
+ model.load_weights(self._get_all_weights(model_config, model))
404
405
for _, module in model.named_modules():
406
quant_method = getattr(module, "quant_method", None)
vllm/model_executor/models/ultravox.py
+26
-4
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.layernorm import RMSNorm
25
from vllm.model_executor.layers.quantization.base_config import (
26
QuantizationConfig)
27
from vllm.model_executor.layers.sampler import SamplerOutput
28
+from vllm.model_executor.model_loader.loader import DefaultModelLoader
29
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
30
from vllm.model_executor.models.interfaces import SupportsMultiModal
31
from vllm.model_executor.models.utils import (flatten_bn,
@@ -334,14 +335,23 @@ class UltravoxModel(nn.Module, SupportsMultiModal):
335
self.multi_modal_config = multimodal_config
336
assert self.multi_modal_config
337
338
+ self.secondary_weights = []
339
+ self.audio_tower = ModifiedWhisperEncoder(config.audio_config)
340
if config.audio_model_id is not None:
338
- self.audio_tower = ModifiedWhisperEncoder.from_pretrained(
339
- config.audio_model_id)
340
- else:
341
- self.audio_tower = ModifiedWhisperEncoder(config.audio_config)
341
+ self.secondary_weights.append(
342
+ DefaultModelLoader.Source(
343
+ model_or_path=config.audio_model_id,
344
+ revision=None,
345
+ prefix="audio_tower.",
346
+ ))
347
self.multi_modal_projector = UltravoxProjector(config)
348
self.language_model = init_vllm_registered_model(
349
config.text_config, cache_config, quant_config)
350
+ if config.text_model_id is not None:
351
+ self.secondary_weights.append(
352
+ DefaultModelLoader.Source(model_or_path=config.text_model_id,
353
+ revision=None,
354
+ prefix="language_model."))
355
356
def _audio_features_to_embeddings(
357
self, input_features: torch.Tensor) -> torch.Tensor:
@@ -466,6 +476,18 @@ class UltravoxModel(nn.Module, SupportsMultiModal):
476
# prepare weight iterators for components
477
weights_group = group_weights_with_prefix(weights)
478
479
+ # load audio tower weights
480
+ audio_tower_weights = weights_group["audio_tower"]
481
+ audio_tower_params_dict = dict(
482
+ self.audio_tower.named_parameters(
483
+ prefix=self.audio_tower.base_model_prefix))
484
+ for name, loaded_weight in audio_tower_weights:
485
+ if name in audio_tower_params_dict:
486
+ param = audio_tower_params_dict[name]
487
+ weight_loader = getattr(param, "weight_loader",
488
+ default_weight_loader)
489
+ weight_loader(param, loaded_weight)
490
+
491
# load projector weights
492
projector_weights = weights_group["multi_modal_projector"]
493
projector_params_dict = dict(