@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
py 120 lines 4.83 KB
Raw
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 # Adapted from https://github.com/fixie-ai/ultravox/blob/ecd58c4041030bae2ad15aa6bcf04ab43199ea02/ultravox/model/ultravox_config.py
5 from typing import Any
6
7 import transformers
8
9
10 class UltravoxConfig(transformers.PreTrainedConfig):
11 r"""This is the configuration class to store the configuration of a
12 [`UltravoxForConditionalGeneration`]. It is used to instantiate an
13 Ultravox model according to the specified arguments, defining the model
14 architecture.
15
16 Configuration objects inherit from [`PreTrainedConfig`] and can be used to
17 control the model outputs. Read the documentation from [`PreTrainedConfig`]
18 for more information.
19
20 Args:
21 audio_config (`Union[AutoConfig, dict]`, *optional*):
22 Custom audio config or dict.
23 text_config (`Union[AutoConfig, dict]`, *optional*):
24 The config object of the text backbone.
25 audio_model_id (`str`, *optional*):
26 The model ID of the audio backbone.
27 text_model_id (`str`, *optional*):
28 The model ID of the text backbone.
29 ignore_index (`int`, *optional*, defaults to -100):
30 The ignore index for the loss function.
31 audio_token_index (`int`, *optional*, defaults to 32000):
32 The audio token index to encode the audio prompt.
33 stack_factor (`int`, *optional*, defaults to 8):
34 Audio downsampling factor for the multimodal projector.
35 norm_init (`float`, *optional*, defaults to 0.4):
36 The initialization value for the layer normalization.
37 projector_act (`str`, *optional*, defaults to `"swiglu"`):
38 The activation function used by the multimodal projector.
39 projector_ln_mid (`bool`, *optional*, defaults to `False`):
40 Whether to apply layer normalization at the middle of the
41 projector or at the end. Versions v0.4.1 and below
42 use `False`, but v0.5 and above use `True`.
43
44 """
45
46 model_type = "ultravox"
47 audio_token = "<|audio|>"
48 is_composition = False
49
50 def __init__(
51 self,
52 audio_config: dict[str, Any] | None = None,
53 text_config: dict[str, Any] | None = None,
54 audio_model_id: str | None = None,
55 text_model_id: str | None = None,
56 ignore_index: int = -100,
57 audio_token_index: int = 32000,
58 hidden_size: int = 4096,
59 stack_factor: int = 8,
60 norm_init: float = 0.4,
61 projector_act: str = "swiglu",
62 projector_ln_mid: bool = False,
63 num_projector_layers: int = 0,
64 **kwargs,
65 ):
66 self.ignore_index = ignore_index
67 self.audio_token_index = audio_token_index
68
69 self.hidden_size = hidden_size
70 self.stack_factor = stack_factor
71 self.norm_init = norm_init
72 self.projector_act = projector_act
73 self.projector_ln_mid = projector_ln_mid
74 self.num_projector_layers = num_projector_layers
75
76 # N.B. May set the wrapped_model_config below.
77 self.wrapped_model_config: transformers.PreTrainedConfig
78 self.text_model_id = text_model_id
79 if text_model_id is None:
80 text_config = text_config or {}
81 self.wrapped_model_config = transformers.CONFIG_MAPPING[
82 text_config.get("model_type", "llama")
83 ](**text_config)
84
85 # N.B. May set the audio_config below.
86 self.audio_model_id = audio_model_id
87 if audio_model_id is None:
88 self.audio_model_id = None
89 audio_config = audio_config or {}
90 self.audio_config = transformers.CONFIG_MAPPING[
91 audio_config.get("model_type", "whisper")
92 ](**audio_config)
93
94 super().__init__(**kwargs)
95
96 def __setattr__(self, key, value):
97 # Since --hf-overrides are applied _after_ the UltravoxConfig is
98 # instantiated, load the configs implicitly when assigning text_model_id
99 # or audio_model_id. This allows:
100 #
101 # --hf-overrides.text_model_id=<quantized variant>
102 #
103 # to behave as intended.
104 if key == "text_model_id" and value is not None:
105 from vllm.transformers_utils.config import get_config
106
107 self.wrapped_model_config = get_config(value, trust_remote_code=False)
108 elif key == "audio_model_id" and value is not None:
109 from vllm.transformers_utils.config import get_config
110
111 self.audio_config = get_config(value, trust_remote_code=False)
112
113 return super().__setattr__(key, value)
114
115 @property
116 def text_config(self) -> transformers.PreTrainedConfig:
117 # When Ultravox wraps a multi-modal model (e.g. Gemma), we instantiate
118 # the full model, but the text config is the text config of the inner
119 # model.
120 return self.wrapped_model_config.get_text_config()