A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[VLM] Add TP support for Phi-4-MM (#14453)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Mar 8, 2025 at 21:57 UTC
03fe18ae0fb03cd2269a54b2dddca58bb56bd6b8
4 files changed
+50
-295
examples/offline_inference/audio_language.py
+1
@@ -77,6 +77,7 @@ def run_phi4mm(questions: str, audio_count: int):
77
enable_lora=True,
78
max_lora_rank=320,
79
lora_extra_vocab_size=0,
80
+ limit_mm_per_prompt={"audio": audio_count},
81
)
82
lora_request = LoRARequest("speech", 1, speech_lora_path)
83
# To maintain code compatibility in this script, we add LoRA here.
vllm/model_executor/models/phi4mm.py
+24
-49
@@ -15,7 +15,7 @@ from transformers import PretrainedConfig
15
from transformers.utils import logging
16
17
from vllm.config import VllmConfig
18
-from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
18
+from vllm.distributed import get_pp_group
19
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
20
InputContext)
21
from vllm.inputs.data import TokenInputs, token_inputs
@@ -34,7 +34,7 @@ from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
34
35
from .interfaces import SupportsLoRA, SupportsMultiModal
36
from .phi4mm_audio import AudioEmbedding
37
-from .utils import maybe_prefix
37
+from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
38
from .vision_siglip_navit import get_siglip_vision_model
39
40
# <|endoftext10|> (see vocab.json in hf model)
@@ -352,12 +352,6 @@ class Phi4MMImageEncoder(nn.Module):
352
# n_embed or hidden_size
353
hidden_size = config.n_embd if hasattr(
354
config, 'n_embd') else config.hidden_size
355
- if hasattr(config, 'embd_pdrop') or hasattr(config, 'embed_pdrop'):
356
- embd_drop = config.embd_pdrop if hasattr(
357
- config, 'embd_pdrop') else config.embed_pdrop
358
- self.drop = nn.Dropout(embd_drop)
359
- else:
360
- self.drop = None
355
356
# layer_idx to output the img features
357
if isinstance(config.img_processor, dict):
@@ -1431,6 +1425,20 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
1425
],
1426
}
1427
1428
+ hf_to_vllm_mapper = WeightsMapper(
1429
+ orig_to_new_substr={
1430
+ "base_layer.": "",
1431
+ },
1432
+ orig_to_new_prefix={
1433
+ "model.embed_tokens_extend.audio_embed.audio_projection.vision.":
1434
+ "embed_tokens_extend.audio_projection_for_vision.",
1435
+ "model.embed_tokens_extend.audio_embed.audio_projection.speech.":
1436
+ "embed_tokens_extend.audio_projection.",
1437
+ "model.embed_tokens_extend.audio_embed.": "embed_tokens_extend.",
1438
+ "model.embed_tokens_extend.image_embed.": "vision_encoder.",
1439
+ },
1440
+ )
1441
+
1442
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1443
super().__init__()
1444
config = vllm_config.model_config.hf_config
@@ -1445,8 +1453,6 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
1453
self.lora_config = lora_config
1454
1455
# Tensor/Pipeline parallel not supported for now.
1448
- assert get_tensor_model_parallel_world_size(
1449
- ) == 1, "tensor parallel is not supported"
1456
assert get_pp_group(
1457
).world_size == 1, "pipeline parallel is not supported"
1458
@@ -1686,44 +1692,6 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
1692
)
1693
return merged_embeds
1694
1689
- def load_weights(self, weights: Iterable[Tuple[str,
1690
- torch.Tensor]]) -> None:
1691
- weights = {name: weight for name, weight in weights}
1692
- adjusted_weights = {}
1693
-
1694
- for name, weight in weights.items():
1695
- # NOTE vision-speech tasks use a separate projection layer
1696
- audio_proj_4v = \
1697
- "model.embed_tokens_extend.audio_embed.audio_projection.vision"
1698
- if name.startswith(audio_proj_4v):
1699
- name = name.replace(
1700
- audio_proj_4v,
1701
- "embed_tokens_extend.audio_projection_for_vision")
1702
-
1703
- name = (name.replace(
1704
- "model.embed_tokens_extend.audio_embed."\
1705
- "audio_projection.speech.",
1706
- "embed_tokens_extend.audio_projection.",
1707
- ).replace(
1708
- "model.embed_tokens_extend.audio_embed.",
1709
- "embed_tokens_extend.",
1710
- ).replace("model.embed_tokens_extend.image_embed.",
1711
- "vision_encoder."))
1712
- # NOTE: this is deal with LoRA injection, where `base_layer`
1713
- # remains as the original layer in the model
1714
- if name.endswith(".base_layer.weight"):
1715
- name = name.replace(".base_layer.weight", ".weight")
1716
- adjusted_weights[name] = weight
1717
-
1718
- missing_keys, unexpected_keys = self.load_state_dict(adjusted_weights,
1719
- strict=False)
1720
- logger.debug("*** missing keys:")
1721
- for key in missing_keys:
1722
- logger.debug(key)
1723
- logger.debug("**** unexpected keys:")
1724
- for key in unexpected_keys:
1725
- logger.debug(key)
1726
-
1695
def forward(
1696
self,
1697
input_ids: torch.Tensor,
@@ -1796,6 +1764,13 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
1764
next_tokens = self.sampler(logits, sampling_metadata)
1765
return next_tokens
1766
1767
+ def load_weights(self, weights: Iterable[Tuple[str,
1768
+ torch.Tensor]]) -> None:
1769
+ weights = ((name, data) for name, data in weights
1770
+ if "lora" not in name)
1771
+ loader = AutoWeightsLoader(self)
1772
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1773
+
1774
def get_mm_mapping(self) -> MultiModelKeys:
1775
"""
1776
Get the module prefix in multimodal models
@@ -1804,4 +1779,4 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
1779
language_model="model.",
1780
connector=["audio_projection_for_vision", "audio_projection"],
1781
tower_model=["vision_encoder", "embed_tokens_extend"],
1807
- )
\ No newline at end of file
1782
+ )
vllm/model_executor/models/phi4mm_audio.py
+18
-150
@@ -6,69 +6,26 @@
6
#!/usr/bin/env python3
7
import abc
8
import math
9
-from functools import partial
10
-from typing import Callable, Dict, List, Literal, Optional, Union
9
+from typing import List, Literal, Optional
10
11
import numpy as np
12
import torch
13
import torch.nn.functional as F
14
from torch import Tensor, nn
15
from torch.distributed.algorithms._checkpoint.checkpoint_wrapper import (
17
- CheckpointImpl, CheckpointWrapper, checkpoint_wrapper, offload_wrapper)
16
+ CheckpointWrapper)
17
from torch.distributed.fsdp.fully_sharded_data_parallel import (
18
FullyShardedDataParallel)
20
-from torch.utils.checkpoint import checkpoint
19
from transformers import PretrainedConfig
20
21
from vllm.model_executor.models.phi4mm_utils import (
22
AbsolutePositionalEncoding, ConvModule, FeedForward, MeanVarianceNormLayer,
25
- MultiHeadedAttention, NemoConvSubsampling, T5RelativeAttentionLogitBias,
26
- adaptive_enc_mask, attn_checkpointing, embedding_checkpoint_wrapper,
27
- get_offset, repeat, unfold_tensor, validate_checkpointing_config)
23
+ MultiHeadedAttention, MultiSequential, NemoConvSubsampling,
24
+ T5RelativeAttentionLogitBias, adaptive_enc_mask, get_offset, unfold_tensor)
25
26
_AUDIO_PLACEHOLDER_TOKEN_ID = 200011 # <|endoftext11|>
27
28
32
-def encoder_checkpoint_wrapper(
33
- activation_checkpointing: Union[str, Dict],
34
- layer_cls: type,
35
- idx: int = 0,
36
-) -> Callable:
37
- """return encoder activation checkpoint wrapper"""
38
- validate_checkpointing_config(activation_checkpointing)
39
-
40
- if isinstance(activation_checkpointing, str):
41
- if activation_checkpointing:
42
- if activation_checkpointing == "offload":
43
- return offload_wrapper
44
- return partial(checkpoint_wrapper)
45
- return lambda x: x
46
-
47
- if isinstance(activation_checkpointing, dict):
48
- target_layer_cls = activation_checkpointing.get(
49
- "module", "transformer")
50
- if target_layer_cls.lower() == "transformer":
51
- target_layer_cls = (
52
- "EncoderLayer",
53
- "ConformerEncoderLayer",
54
- )
55
- elif target_layer_cls.lower() == "attention":
56
- target_layer_cls = ("MultiHeadedAttention", "MultiHeadAttention")
57
- checkpointing_interval = activation_checkpointing.get("interval", 1)
58
- offloading = activation_checkpointing.get("offload", False)
59
- impl = (CheckpointImpl.REENTRANT if activation_checkpointing.get(
60
- "reentrant", True) else CheckpointImpl.NO_REENTRANT)
61
-
62
- if (idx % checkpointing_interval == 0
63
- and layer_cls.__name__ in target_layer_cls):
64
- if offloading:
65
- return offload_wrapper
66
- return partial(checkpoint_wrapper, checkpoint_impl=impl)
67
- return lambda x: x
68
-
69
- raise ValueError("Invalid activation_checkpointing config")
70
-
71
-
29
class ConformerEncoderLayer(nn.Module):
30
"""ConformerEncoder Layer module.
31
for more details see conformer paper:
@@ -208,10 +165,7 @@ class ConformerEncoderLayer(nn.Module):
165
bias_in_glu=bias_in_glu,
166
)
167
211
- self.self_attn = encoder_checkpoint_wrapper(
212
- activation_checkpointing,
213
- MultiHeadedAttention,
214
- )(MultiHeadedAttention(
168
+ self.self_attn = MultiHeadedAttention(
169
n_head,
170
d_model,
171
dropout_rate,
@@ -221,7 +175,7 @@ class ConformerEncoderLayer(nn.Module):
175
use_pt_scaled_dot_product_attention=
176
use_pt_scaled_dot_product_attention,
177
group_size=attn_group_sizes,
224
- ))
178
+ )
179
self.conv = ConvModule(
180
d_model,
181
ext_pw_out_channel,
@@ -441,26 +395,8 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
395
else:
396
raise NotImplementedError
397
444
- def post_init(self, init_model_config):
445
-
446
- pretrained_speech_encoder_path = init_model_config.get(
447
- "pretrained_speech_encoder_path", None)
448
- if pretrained_speech_encoder_path:
449
- model_state = torch.load(pretrained_speech_encoder_path,
450
- map_location="cpu")
451
- encoder_state_dict = {}
452
- for k, v in model_state.items():
453
- if "encoder." in k:
454
- tmp_k = k.replace("encoder.", "")
455
- encoder_state_dict[tmp_k] = v
456
-
457
- if hasattr(self, "encoder_embedding"):
458
- del self.encoder_embedding
459
- self.load_state_dict(encoder_state_dict)
460
-
461
- if not hasattr(self, "encoder_embedding"):
462
- self.encoder_embedding = MeanVarianceNormLayer(
463
- self.encoder_embedding_config["input_size"])
398
+ self.encoder_embedding = MeanVarianceNormLayer(
399
+ self.encoder_embedding_config["input_size"])
400
401
def compute_lens_change(self, feature_lens):
402
"""feature_lens: int
@@ -558,14 +494,6 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
494
# Create mask matrix for streaming
495
# S stores start index. if chunksize is 18, s is [0,18,36,....]
496
chunk_start_idx = np.arange(0, seq_len, chunk_size_train_eff)
561
- # avoid randomness when run evaluation or decoding
562
- if self.training and np.random.rand() > 0.5:
563
- # Either first or last chunk is not complete.
564
- # If only the last one is not complete, EOS is not effective
565
- chunk_start_idx = seq_len - chunk_start_idx
566
- chunk_start_idx = chunk_start_idx[::-1]
567
- chunk_start_idx = chunk_start_idx[:-1]
568
- chunk_start_idx = np.insert(chunk_start_idx, 0, 0)
497
498
enc_streaming_mask = (adaptive_enc_mask(
499
seq_len, chunk_start_idx,
@@ -883,23 +811,17 @@ class ConformerEncoder(TransformerEncoderBase):
811
self.num_blocks = num_blocks
812
self.num_lang = num_lang
813
self.kernel_size = kernel_size
886
- self.embed = embedding_checkpoint_wrapper(activation_checkpointing)(
887
- self.embed)
814
self.replication_pad_for_subsample_embedding: bool = (
815
replication_pad_for_subsample_embedding)
816
assert (self.num_heads % attention_group_size == 0
817
), "attention_group_size must divide n_head"
818
self.num_heads_k = self.num_heads // attention_group_size
819
894
- self.encoders = repeat(
895
- num_blocks,
896
- lambda i: encoder_checkpoint_wrapper(activation_checkpointing,
897
- ConformerEncoderLayer, i)
898
- (ConformerEncoderLayer(
820
+ self.encoders = MultiSequential(*[
821
+ ConformerEncoderLayer(
822
d_model=attention_dim,
823
ext_pw_out_channel=ext_pw_out_channel,
901
- depthwise_seperable_out_channel=
902
- depthwise_seperable_out_channel,
824
+ depthwise_seperable_out_channel=depthwise_seperable_out_channel,
825
depthwise_multiplier=depthwise_multiplier,
826
n_head=attention_heads,
827
d_ffn=linear_units,
@@ -916,14 +838,13 @@ class ConformerEncoder(TransformerEncoderBase):
838
bias_in_glu=bias_in_glu,
839
linear_glu_in_convm=linear_glu_in_convm,
840
attention_glu_type=attention_glu_type,
919
- activation_checkpointing=attn_checkpointing(
920
- activation_checkpointing, i),
841
+ activation_checkpointing=activation_checkpointing,
842
export=export,
843
use_pt_scaled_dot_product_attention=
844
use_pt_scaled_dot_product_attention,
845
attn_group_sizes=attention_group_size,
925
- )),
926
- )
846
+ ) for _ in range(num_blocks)
847
+ ])
848
self.extra_layer_output_idx = extra_layer_output_idx
849
self.extra_multi_layer_output_idxs = extra_multi_layer_output_idxs
850
# Make a zeros scalar we can use in get_initial_state to determine
@@ -1041,9 +962,6 @@ class ConformerEncoder(TransformerEncoderBase):
962
963
return input_tensor, masks # , layer_emb
964
1044
- def gradient_checkpointing_enable(self):
1045
- pass
1046
-
965
966
class WindowQformer(nn.Module):
967
"""Window-level Qformer"""
@@ -1077,13 +995,6 @@ class WindowQformer(nn.Module):
995
self.after_norm = (nn.LayerNorm(attention_dim, eps=1e-12)
996
if normalize_before else None)
997
self.window_size = window_size
1080
- self.gradient_checkpointing_enable = False
1081
-
1082
- def enable_gradient_checkpointing(self):
1083
- self.gradient_checkpointing_enable = True
1084
-
1085
- def disable_gradient_checkpointing(self):
1086
- self.gradient_checkpointing_enable = False
998
999
def forward(self, audio_embed, mask, embed_len=None):
1000
"""forward decoder"""
@@ -1111,20 +1022,10 @@ class WindowQformer(nn.Module):
1022
# NT' x 1 x D
1023
q = self.queries.expand(bsz * slen, -1, -1)
1024
for layer in self.decoders:
1114
- if self.gradient_checkpointing_enable and self.training:
1115
- q = checkpoint(
1116
- layer.__call__,
1117
- q,
1118
- embed_chunk,
1119
- None,
1120
- mask,
1121
- use_reentrant=True,
1122
- )
1123
- else:
1124
- q = layer(tgt=q,
1125
- memory=embed_chunk,
1126
- tgt_mask=None,
1127
- memory_mask=mask)
1025
+ q = layer(tgt=q,
1026
+ memory=embed_chunk,
1027
+ tgt_mask=None,
1028
+ memory_mask=mask)
1029
1030
if self.after_norm is not None:
1031
q = self.after_norm(q)
@@ -1147,13 +1048,6 @@ class AudioEmbedding(nn.Module):
1048
hidden_size = (config.n_embd
1049
if hasattr(config, "n_embd") else config.hidden_size)
1050
1150
- if hasattr(config, "embd_pdrop") or hasattr(config, "embed_pdrop"):
1151
- embd_drop = (config.embd_pdrop if hasattr(config, "embd_pdrop")
1152
- else config.embed_pdrop)
1153
- self.drop = nn.Dropout(embd_drop)
1154
- else:
1155
- self.drop = None
1156
-
1051
# self.wte = nn.Embedding(config.vocab_size, hidden_size)
1052
1053
audio_dim_out = (
@@ -1167,12 +1061,6 @@ class AudioEmbedding(nn.Module):
1061
assert encoder_config is not None
1062
self.encoder = ConformerEncoder(**encoder_config)
1063
1170
- # fake initialization, create encoder_embedding layer only so that
1171
- # in decoding, all parameters can be loaded in
1172
- # from_pretrained_function in training, we do post init after
1173
- # from_pretrained function to make sure the correct initialization
1174
- self.encoder.post_init({})
1175
-
1064
audio_dim_out = encoder_config["attention_dim"]
1065
n_mels = encoder_config["input_size"]
1066
else:
@@ -1221,14 +1109,6 @@ class AudioEmbedding(nn.Module):
1109
else:
1110
self.conv_ds = None
1111
1224
- enable_gradient_checkpointing = kwargs.get(
1225
- "enable_gradient_checkpointing", False)
1226
- if enable_gradient_checkpointing:
1227
- self.encoder.gradient_checkpointing_enable()
1228
-
1229
- if self.qformer:
1230
- self.qformer.enable_gradient_checkpointing()
1231
-
1112
projection_cls = kwargs.get("projection_cls", "linear")
1113
if projection_cls == "linear":
1114
self.audio_projection = nn.Linear(audio_dim_out, hidden_size)
@@ -1388,16 +1268,4 @@ class AudioEmbedding(nn.Module):
1268
hidden_states.dtype).to(hidden_states.device))
1269
idx += cnt
1270
1391
- else:
1392
- if self.training:
1393
- # hidden_states[:, 0:img_set_tensor.shape[0]] =
1394
- # hidden_states[:, 0:img_set_tensor.shape[0]] +
1395
- # 0 * img_set_tensor.to(hidden_states.dtype)
1396
- # .to(hidden_states.device)
1397
- hidden_states[:, 0:1] = hidden_states[:, 0:1] + \
1398
- 0 * audio_set_tensor[:, 0:1].to(hidden_states.dtype)\
1399
- .to(hidden_states.device)
1400
-
1401
- if self.drop is not None:
1402
- hidden_states = self.drop(hidden_states)
1271
return hidden_states
vllm/model_executor/models/phi4mm_utils.py
+7
-96
@@ -5,14 +5,11 @@
5
# but implemented by the Phi-Speech team
6
#!/usr/bin/env python3
7
import math
8
-from functools import partial
9
-from typing import Callable, Dict, Optional, Tuple, Union
8
+from typing import Optional, Tuple, Union
9
10
import torch
11
import torch.nn.functional as F
12
from torch import Tensor, nn
14
-from torch.distributed.algorithms._checkpoint.checkpoint_wrapper import (
15
- CheckpointImpl, checkpoint_wrapper, offload_wrapper)
13
14
15
class Block(nn.Module):
@@ -873,10 +870,8 @@ class MeanVarianceNormLayer(nn.Module):
870
def __init__(self, input_size):
871
super().__init__()
872
self.input_size = input_size
876
- self.register_buffer("global_mean", torch.zeros(input_size))
877
- self.register_buffer("global_invstd", torch.ones(input_size))
878
- self.global_mean: Optional[Tensor]
879
- self.global_invstd: Optional[Tensor]
873
+ self.global_mean = nn.Parameter(torch.zeros(input_size))
874
+ self.global_invstd = nn.Parameter(torch.ones(input_size))
875
876
def forward(self, input_: Tensor) -> Tensor:
877
"""MeanVarianceNormLayer Forward
@@ -1023,21 +1018,10 @@ class CausalConv2D(nn.Conv2d):
1018
self,
1019
x,
1020
):
1026
- if self.training:
1027
- x = F.pad(
1028
- x,
1029
- pad=(
1030
- self._left_padding,
1031
- self._right_padding,
1032
- self._left_padding,
1033
- self._right_padding,
1034
- ),
1035
- )
1036
- else:
1037
- x = F.pad(
1038
- x,
1039
- pad=(self._left_padding, self._right_padding, 0, 0),
1040
- )
1021
+ x = F.pad(
1022
+ x,
1023
+ pad=(self._left_padding, self._right_padding, 0, 0),
1024
+ )
1025
x = super().forward(x)
1026
return x
1027
@@ -1840,68 +1824,6 @@ class MultiHeadedAttention(nn.Module):
1824
return self.linear_out(x) # (batch, time1, d_model)
1825
1826
1843
-def validate_checkpointing_config(activation_checkpointing):
1844
- """validate activation checkpointing configuration"""
1845
- if isinstance(activation_checkpointing, str):
1846
- assert activation_checkpointing in (
1847
- "",
1848
- "checkpoint",
1849
- "offload",
1850
- ), "activation_checkpointing has to be a dict or a str in "\
1851
- "('', 'checkpoint', 'offload')."
1852
- elif isinstance(activation_checkpointing, dict):
1853
- assert activation_checkpointing.get("module", "transformer") in (
1854
- "transformer",
1855
- "attention",
1856
- ), "module in activation_checkpointing has to be in "\
1857
- "('transformer', 'attention')."
1858
- else:
1859
- raise ValueError("activation_checkpointing has to be a str"\
1860
- " or dict.")
1861
-
1862
-
1863
-def embedding_checkpoint_wrapper(
1864
- activation_checkpointing: Union[str, Dict], ) -> Callable:
1865
- """return encoder embedding activation checkpoint wrapper"""
1866
- validate_checkpointing_config(activation_checkpointing)
1867
-
1868
- if isinstance(activation_checkpointing, str):
1869
- if activation_checkpointing:
1870
- if activation_checkpointing == "offload":
1871
- return offload_wrapper
1872
- return partial(checkpoint_wrapper)
1873
- return lambda x: x
1874
-
1875
- if isinstance(activation_checkpointing, dict):
1876
- enabled = activation_checkpointing.get("embed", False)
1877
- if enabled:
1878
- offloading = activation_checkpointing.get("offload", False)
1879
- if offloading:
1880
- return offload_wrapper
1881
- impl = (CheckpointImpl.REENTRANT if activation_checkpointing.get(
1882
- "reentrant", False) else CheckpointImpl.NO_REENTRANT)
1883
- return partial(checkpoint_wrapper, checkpoint_impl=impl)
1884
- return lambda x: x
1885
- raise ValueError("Invalid activation_checkpointing config")
1886
-
1887
-
1888
-def attn_checkpointing(activation_checkpointing: Union[str, Dict],
1889
- i) -> Union[str, Dict]:
1890
- """return activation checkpointing config for attention layer"""
1891
- if isinstance(activation_checkpointing, str):
1892
- return ""
1893
-
1894
- if isinstance(activation_checkpointing, dict):
1895
- target_layer_cls = activation_checkpointing.get(
1896
- "module", "transformer")
1897
- checkpointing_interval = activation_checkpointing.get("interval", 1)
1898
- if target_layer_cls == "attention" and i % checkpointing_interval == 0:
1899
- return activation_checkpointing
1900
- return ""
1901
-
1902
- raise ValueError("Invalid activation_checkpointing config")
1903
-
1904
-
1827
class MultiSequential(torch.nn.Sequential):
1828
"""Multi-input multi-output torch.nn.Sequential"""
1829
@@ -1913,17 +1835,6 @@ class MultiSequential(torch.nn.Sequential):
1835
return args
1836
1837
1916
-def repeat(repeat_num, module_gen_fn):
1917
- """repeat module N times
1918
-
1919
- :param int repeat_num: repeat time
1920
- :param function module_gen_fn: function to generate module
1921
- :return: repeated modules
1922
- :rtype: MultiSequential
1923
- """
1924
- return MultiSequential(*[module_gen_fn(i) for i in range(repeat_num)])
1925
-
1926
-
1838
def get_offset(input_layer: str, time_reduction: int):
1839
"""Get an offset. We will use the offset for determining #frames of a
1840
subsampled feature.