A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 05:15 UTC
refactor funasr model. (#36108)
Signed-off-by: zixiao <shunli.dsl@alibaba-inc.com> Co-authored-by: zixiao <shunli.dsl@alibaba-inc.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
AllenDou committed
Mar 6, 2026 at 00:07 UTC
3ee68590c7fafe05f1db1f1bee019c7b3a83ec96
3 files changed
+24
-57
vllm/model_executor/models/funasr.py
+18
-53
@@ -51,7 +51,6 @@ from vllm.multimodal.processing import (
51
)
52
from vllm.transformers_utils.processor import cached_processor_from_config
53
from vllm.transformers_utils.processors.funasr_processor import FunASRFeatureExtractor
54
-from vllm.utils.jsontree import json_map_leaves
54
from vllm.utils.tensor_schema import TensorSchema, TensorShape
55
56
from .interfaces import (
@@ -611,6 +610,10 @@ class FunASRAudioInputs(TensorSchema):
610
list[torch.Tensor] | None,
611
TensorShape("b"),
612
]
613
+ fake_token_lengths: Annotated[
614
+ list[torch.Tensor] | None,
615
+ TensorShape("b"),
616
+ ]
617
618
619
class FunASREncoder(nn.Module):
@@ -732,9 +735,6 @@ class FunASRProcessingInfo(BaseProcessingInfo):
735
def get_target_channels(self) -> int:
736
return 1
737
735
- def get_num_audio_tokens(self) -> int:
736
- return self.get_hf_config().max_source_positions
737
-
738
739
class FunASRDummyInputsBuilder(BaseDummyInputsBuilder[FunASRProcessingInfo]):
740
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
@@ -798,7 +798,7 @@ class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]):
798
return dict(
799
input_features=MultiModalFieldConfig.batched("audio"),
800
speech_lengths=MultiModalFieldConfig.batched("audio"),
801
- fake_token_len=MultiModalFieldConfig.batched("audio"),
801
+ fake_token_lengths=MultiModalFieldConfig.batched("audio"),
802
)
803
804
def _get_prompt_updates(
@@ -812,22 +812,16 @@ class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]):
812
813
out_mm_data = out_mm_kwargs.get_data()
814
815
- fake_token_len = out_mm_data.get("fake_token_len")
816
- if fake_token_len is None:
815
+ fake_token_lengths = out_mm_data.get("fake_token_lengths")
816
+ if fake_token_lengths is None:
817
audio_output_lengths = []
818
else:
819
- assert isinstance(fake_token_len, torch.Tensor)
819
+ assert isinstance(fake_token_lengths, torch.Tensor)
820
821
- audio_output_lengths = fake_token_len.tolist()
821
+ audio_output_lengths = fake_token_lengths.tolist()
822
823
def get_replacement_qwen2_audio(item_idx: int):
824
- if audio_output_lengths:
825
- num_features = audio_output_lengths[item_idx]
826
- else:
827
- audio_embeds = out_mm_data["audio_embeds"][item_idx]
828
- assert len(audio_embeds.shape) == 2, "audio_embeds must be a 2D tensor"
829
- num_features = audio_embeds.shape[0]
830
-
824
+ num_features = audio_output_lengths[item_idx]
825
return [audio_token_id] * num_features
826
827
return [
@@ -847,21 +841,16 @@ class FunASRMultiModalProcessor(BaseMultiModalProcessor[FunASRProcessingInfo]):
841
class FunASRForConditionalGeneration(
842
nn.Module, SupportsTranscription, SupportsMultiModal
843
):
850
- packed_modules_mapping = {
851
- "self_attn.qkv_proj": [
852
- "self_attn.q_proj",
853
- "self_attn.k_proj",
854
- "self_attn.v_proj",
855
- ],
856
- "encoder_attn.kv_proj": ["encoder_attn.k_proj", "encoder_attn.v_proj"],
857
- }
858
-
844
hf_to_vllm_mapper = WeightsMapper(
845
orig_to_new_substr={
846
"linear_q.": "q_proj.",
847
"linear_k.": "k_proj.",
848
"linear_v.": "v_proj.",
849
"linear_out.": "out_proj.",
850
+ "audio_adaptor.": "model.encoder.audio_adaptor.",
851
+ "audio_encoder.": "model.encoder.audio_encoder.",
852
+ "llm.model.": "model.decoder.",
853
+ "llm.lm_head": "lm_head",
854
}
855
)
856
@@ -969,9 +958,6 @@ class FunASRForConditionalGeneration(
958
)
959
return decoder_outputs
960
972
- def get_language_model(self) -> torch.nn.Module:
973
- return self.model.decoder
974
-
961
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
962
audio_input = self._parse_and_validate_audio_input(**kwargs)
963
@@ -1002,15 +988,12 @@ class FunASRForConditionalGeneration(
988
def _parse_and_validate_audio_input(self, **kwargs: object) -> FunASRAudioInputs:
989
input_features = kwargs.pop("input_features", None)
990
speech_lengths = kwargs.pop("speech_lengths", None)
1005
-
1006
- if input_features is not None:
1007
- input_features = json_map_leaves(lambda x: x.to(self.dtype), input_features)
1008
-
1009
- if speech_lengths is not None:
1010
- speech_lengths = json_map_leaves(lambda x: x.to(self.dtype), speech_lengths)
991
+ fake_token_lengths = kwargs.pop("fake_token_lengths", None)
992
993
return FunASRAudioInputs(
1013
- input_features=input_features, speech_lengths=speech_lengths
994
+ input_features=input_features,
995
+ speech_lengths=speech_lengths,
996
+ fake_token_lengths=fake_token_lengths,
997
)
998
999
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -1022,22 +1005,4 @@ class FunASRForConditionalGeneration(
1005
self,
1006
)
1007
1025
- # add fake zeros bias for k_proj to state_dict
1026
- weights = _create_fake_bias_for_k_proj(weights)
1008
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1028
-
1029
-
1030
-def _create_fake_bias_for_k_proj(
1031
- weights: Iterable[tuple[str, torch.Tensor]],
1032
-) -> Iterable[tuple[str, torch.Tensor]]:
1033
- """
1034
- Create full zeros bias for k_proj weight in self-attn and x-attn layers.
1035
- So that the bias for k_proj in qkv_proj can be initialized with zeros.
1036
- """
1037
- for name, weight in weights:
1038
- if name.endswith(".k_proj.weight"):
1039
- bias = torch.zeros(weight.size(0))
1040
- bias_name = name.replace("weight", "bias")
1041
- yield from [(name, weight), (bias_name, bias)]
1042
- else:
1043
- yield name, weight
vllm/model_executor/models/qwen3_omni_moe_thinker.py
+1
-1
@@ -1794,7 +1794,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
1794
return []
1795
1796
# The result multimodal_embeddings is tuple of tensors, with each
1797
- # tensor correspoending to a multimodal data item (image or video).
1797
+ # tensor corresponding to a multimodal data item (image or video).
1798
multimodal_embeddings: tuple[torch.Tensor, ...] = ()
1799
1800
# NOTE: It is important to iterate over the keys in this dictionary
vllm/transformers_utils/processors/funasr_processor.py
+5
-3
@@ -370,7 +370,7 @@ class FunASRFeatureExtractor(SequenceFeatureExtractor):
370
)
371
olens = 1 + (speech_lengths - 3 + 2 * 1) // 2
372
olens = 1 + (olens - 3 + 2 * 1) // 2
373
- fake_token_len = (olens - 1) // 2 + 1
373
+ fake_token_lengths = (olens - 1) // 2 + 1
374
if isinstance(input_features[0], list):
375
padded_inputs["input_features"] = [
376
np.asarray(feature, dtype=np.float32) for feature in input_features
@@ -382,8 +382,10 @@ class FunASRFeatureExtractor(SequenceFeatureExtractor):
382
if return_tensors is not None:
383
padded_inputs = padded_inputs.convert_to_tensors(return_tensors)
384
385
+ fake_token_lengths = torch.clamp(fake_token_lengths, min=1)
386
+
387
padded_inputs["speech_lengths"] = speech_lengths
386
- padded_inputs["fake_token_len"] = fake_token_len
388
+ padded_inputs["fake_token_lengths"] = fake_token_lengths
389
390
return padded_inputs
391
@@ -471,7 +473,7 @@ class FunASRProcessor(ProcessorMixin):
473
for sample in text:
474
replace_str = []
475
while self.audio_token in sample:
474
- num_audio_tokens = inputs["fake_token_len"].item()
476
+ num_audio_tokens = inputs["fake_token_lengths"].item()
477
478
expanded_audio_token = self.audio_token * num_audio_tokens
479