[Refactor] Remove dead code multiple places (#58002)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Wentao Ye committed
Sep 22, 2026 at 16:36 UTC
fae5cbfc8b7e957c891ad88881969f3a41de065e
12 files changed
+3
-585
vllm/entrypoints/generate/base/protocol.py
-50
@@ -6,10 +6,8 @@
6
import json
7
from typing import Annotated, Any, Literal, TypeAlias
8
9
-import regex as re
9
from pydantic import (
10
BaseModel,
12
- ConfigDict,
11
Field,
12
model_serializer,
13
)
@@ -21,7 +19,6 @@ from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
19
from vllm.exceptions import VLLMValidationError
20
from vllm.logger import init_logger
21
from vllm.sampling_params import StructuredOutputsParams
24
-from vllm.utils.import_utils import resolve_obj_by_qualname
22
23
logger = init_logger(__name__)
24
@@ -261,53 +258,6 @@ class FunctionDefinition(OpenAIBaseModel):
258
return data
259
260
264
-# extra="forbid" is a workaround to have kwargs as a field,
265
-# see https://github.com/pydantic/pydantic/issues/3125
266
-class LogitsProcessorConstructor(BaseModel):
267
- qualname: str
268
- args: list[Any] | None = None
269
- kwargs: dict[str, Any] | None = None
270
-
271
- model_config = ConfigDict(extra="forbid")
272
-
273
-
274
-LogitsProcessors = list[str | LogitsProcessorConstructor]
275
-
276
-
277
-def get_logits_processors(
278
- processors: LogitsProcessors | None, pattern: str | None
279
-) -> list[Any] | None:
280
- if processors and pattern:
281
- logits_processors = []
282
- for processor in processors:
283
- qualname = processor if isinstance(processor, str) else processor.qualname
284
- if not re.match(pattern, qualname):
285
- raise ValueError(
286
- f"Logits processor '{qualname}' is not allowed by this "
287
- "server. See --logits-processor-pattern engine argument "
288
- "for more information."
289
- )
290
- try:
291
- logits_processor = resolve_obj_by_qualname(qualname)
292
- except Exception as e:
293
- raise ValueError(
294
- f"Logits processor '{qualname}' could not be resolved: {e}"
295
- ) from e
296
- if isinstance(processor, LogitsProcessorConstructor):
297
- logits_processor = logits_processor(
298
- *processor.args or [], **processor.kwargs or {}
299
- )
300
- logits_processors.append(logits_processor)
301
- return logits_processors
302
- elif processors:
303
- raise ValueError(
304
- "The `logits_processors` argument is not supported by this "
305
- "server. See --logits-processor-pattern engine argument "
306
- "for more information."
307
- )
308
- return None
309
-
310
-
261
class FunctionCall(OpenAIBaseModel):
262
# Internal field to preserve native tool call ID from tool parser.
263
# Excluded from serialization to maintain OpenAI API compatibility
vllm/logits_process.py
+1
-106
@@ -1,12 +1,10 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-from collections.abc import Callable, Sequence
3
+from collections.abc import Callable
4
from typing import TypeAlias
5
6
import torch
7
8
-from vllm.tokenizers import TokenizerLike
9
-
8
LogitsProcessor: TypeAlias = (
9
Callable[[list[int], torch.Tensor], torch.Tensor]
10
| Callable[[list[int], list[int], torch.Tensor], torch.Tensor]
@@ -16,106 +14,3 @@ of previously generated tokens, the logits tensor
14
for the next token and, optionally, prompt tokens as a
15
first argument, and returns a modified tensor of logits
16
to sample from."""
19
-
20
-
21
-def get_bad_words_logits_processors(
22
- bad_words: list[str], tokenizer: TokenizerLike
23
-) -> list[LogitsProcessor]:
24
- bad_words_ids: list[list[int]] = list()
25
-
26
- for bad_word in bad_words:
27
- # To prohibit words both at the beginning
28
- # and in the middle of text
29
- # (related to add_prefix_space tokenizer parameter)
30
- for add_prefix_space in [False, True]:
31
- prefix = " " if add_prefix_space else ""
32
- prompt = prefix + bad_word.lstrip()
33
-
34
- prompt_token_ids = tokenizer.encode(text=prompt, add_special_tokens=False)
35
-
36
- # If no space at the beginning
37
- # or if prefix space produces a new word token
38
- if (not add_prefix_space) or (
39
- add_prefix_space
40
- and prompt_token_ids[0] != bad_words_ids[-1][0]
41
- and len(prompt_token_ids) == len(bad_words_ids[-1])
42
- ):
43
- bad_words_ids.append(prompt_token_ids)
44
-
45
- return [NoBadWordsLogitsProcessor(bad_words_ids=bad_words_ids)]
46
-
47
-
48
-class NoBadWordsLogitsProcessor:
49
- _SMALLEST_LOGIT = float("-inf")
50
- _NEUTRAL_LOGIT = 0.0
51
-
52
- def __init__(self, bad_words_ids: list[list[int]]):
53
- self.bad_words_ids = bad_words_ids
54
- self.word_bias: torch.FloatTensor = None
55
-
56
- def __call__(
57
- self,
58
- past_tokens_ids: Sequence[int],
59
- logits: torch.FloatTensor,
60
- ) -> torch.Tensor:
61
- if self.word_bias is None:
62
- self._init_word_bias(logits=logits)
63
-
64
- last_token_bias = torch.zeros_like(logits)
65
-
66
- for bad_word_ids in self.bad_words_ids:
67
- if len(bad_word_ids) == 1: # 1-token words already processed
68
- continue
69
-
70
- if len(bad_word_ids) > len(past_tokens_ids) + 1:
71
- continue
72
-
73
- prefix_length = len(bad_word_ids) - 1
74
- last_token_id = bad_word_ids[-1]
75
- actual_prefix = past_tokens_ids[-prefix_length:]
76
- expected_prefix = bad_word_ids[:prefix_length]
77
-
78
- assert len(actual_prefix) == len(expected_prefix)
79
-
80
- is_match = tuple(actual_prefix) == tuple(expected_prefix)
81
- last_token_bias[last_token_id] += (
82
- self._SMALLEST_LOGIT if is_match else self._NEUTRAL_LOGIT
83
- )
84
-
85
- logits = logits + self.word_bias + last_token_bias
86
-
87
- return logits
88
-
89
- def _init_word_bias(self, logits: torch.FloatTensor) -> None:
90
- # Code based on NoBadWordsLogitsProcessor and SequenceBiasLogitsProcessor # noqa: E501
91
- # from https://github.com/huggingface/transformers/blob/main/src/transformers/generation/logits_process.py
92
-
93
- vocab_size = logits.shape[-1]
94
-
95
- self._check_token_ids_bounds(vocab_size=vocab_size)
96
-
97
- self.word_bias = torch.zeros(
98
- (vocab_size,), dtype=torch.float, device=logits.device
99
- )
100
-
101
- for bad_word_ids in self.bad_words_ids:
102
- if len(bad_word_ids) == 1:
103
- bad_word_id = bad_word_ids[-1]
104
- self.word_bias[bad_word_id] = self._SMALLEST_LOGIT
105
-
106
- def _check_token_ids_bounds(self, vocab_size: int) -> None:
107
- invalid_token_ids = []
108
-
109
- for bad_word_ids in self.bad_words_ids:
110
- for token_id in bad_word_ids:
111
- if token_id < 0 or token_id >= vocab_size:
112
- invalid_token_ids.append(token_id)
113
-
114
- if len(invalid_token_ids) > 0:
115
- raise ValueError(
116
- f"The model vocabulary size is {vocab_size},"
117
- f" but the following tokens"
118
- f" were specified as bad: {invalid_token_ids}."
119
- f" All token id values should be integers satisfying:"
120
- f" 0 <= token_id < {vocab_size}."
121
- )
vllm/model_executor/layers/fused_moe/utils.py
-29
@@ -453,35 +453,6 @@ def normalize_batched_scales_shape(
453
return scales
454
455
456
-@triton.jit
457
-def _pack_topk_ids_weights_kernel(
458
- topk_ids_ptr,
459
- topk_weights_ptr,
460
- output_ptr,
461
- n_elements,
462
- BLOCK_SIZE: tl.constexpr,
463
- USE_GDC: tl.constexpr,
464
- launch_pdl: tl.constexpr, # triton metadata
465
-):
466
- pid = tl.program_id(axis=0)
467
- offsets = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
468
- mask = offsets < n_elements
469
- if USE_GDC:
470
- tl.extra.cuda.gdc_launch_dependents()
471
- tl.extra.cuda.gdc_wait()
472
- expert_id = tl.load(topk_ids_ptr + offsets, mask=mask, other=0).to(tl.int32)
473
- expert_id_shifted = expert_id << 16
474
-
475
- weight = tl.load(topk_weights_ptr + offsets, mask=mask, other=0.0)
476
- weight_bf16 = weight.to(tl.bfloat16)
477
- weight_int16 = weight_bf16.to(tl.int16, bitcast=True)
478
-
479
- weight_int32 = weight_int16.to(tl.int32) & 0xFFFF
480
-
481
- packed = expert_id_shifted | weight_int32
482
- tl.store(output_ptr + offsets, packed, mask=mask)
483
-
484
-
456
def fi_moe_largest_bucket(moe_config: "FusedMoEConfig") -> int:
457
"""Estimate FlashInfer's MoE autotuning maximum token count.
458
vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py
-72
@@ -59,7 +59,6 @@ from vllm.third_party.flash_linear_attention.ops import (
59
from vllm.third_party.flash_linear_attention.ops.chunk import l2norm_fwd
60
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
61
from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
62
-from vllm.triton_utils import tl, triton
62
from vllm.utils.torch_utils import (
63
LayerNameType,
64
_encode_layer_name,
@@ -1961,74 +1960,3 @@ direct_register_custom_op(
1960
op_func=qwen_gdn_attention_core_fused_norm_packed,
1961
mutates_args=["core_attn_out"],
1962
)
1964
-
1965
-
1966
-@triton.jit
1967
-def fused_gdn_gating_kernel(
1968
- g,
1969
- beta_output,
1970
- A_log,
1971
- a,
1972
- b,
1973
- dt_bias,
1974
- seq_len,
1975
- NUM_HEADS: tl.constexpr,
1976
- beta: tl.constexpr,
1977
- threshold: tl.constexpr,
1978
- BLK_HEADS: tl.constexpr,
1979
-):
1980
- i_b, i_s, i_d = tl.program_id(0), tl.program_id(1), tl.program_id(2)
1981
- head_off = i_d * BLK_HEADS + tl.arange(0, BLK_HEADS)
1982
- off = i_b * seq_len * NUM_HEADS + i_s * NUM_HEADS + head_off
1983
- mask = head_off < NUM_HEADS
1984
- blk_A_log = tl.load(A_log + head_off, mask=mask)
1985
- blk_a = tl.load(a + off, mask=mask)
1986
- blk_b = tl.load(b + off, mask=mask)
1987
- blk_bias = tl.load(dt_bias + head_off, mask=mask)
1988
- # If the model is loaded in fp16, without the .float() here, A might be -inf
1989
- x = blk_a.to(tl.float32) + blk_bias.to(tl.float32)
1990
- softplus_x = tl.where(
1991
- beta * x <= threshold, (1 / beta) * tl.log(1 + tl.exp(beta * x)), x
1992
- )
1993
- blk_g = -tl.exp(blk_A_log.to(tl.float32)) * softplus_x
1994
- tl.store(g + off, blk_g.to(g.dtype.element_ty), mask=mask)
1995
- # compute beta_output = sigmoid(b)
1996
- blk_beta_output = tl.sigmoid(blk_b.to(tl.float32))
1997
- tl.store(
1998
- beta_output + off, blk_beta_output.to(beta_output.dtype.element_ty), mask=mask
1999
- )
2000
-
2001
-
2002
-def fused_gdn_gating(
2003
- A_log: torch.Tensor,
2004
- a: torch.Tensor,
2005
- b: torch.Tensor,
2006
- dt_bias: torch.Tensor,
2007
- beta: float = 1.0,
2008
- threshold: float = 20.0,
2009
-) -> tuple[torch.Tensor, torch.Tensor]:
2010
- """Fused computation of g and beta for Gated Delta Net.
2011
- g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias)
2012
- beta_output = b.sigmoid()
2013
- TODO maybe use torch.compile to replace this triton kernel
2014
- """
2015
- batch, num_heads = a.shape
2016
- seq_len = 1
2017
- grid = (batch, seq_len, triton.cdiv(num_heads, 8))
2018
- g = torch.empty(1, batch, num_heads, dtype=torch.float32, device=a.device)
2019
- beta_output = torch.empty(1, batch, num_heads, dtype=b.dtype, device=b.device)
2020
- fused_gdn_gating_kernel[grid](
2021
- g,
2022
- beta_output,
2023
- A_log,
2024
- a,
2025
- b,
2026
- dt_bias,
2027
- seq_len,
2028
- num_heads,
2029
- beta,
2030
- threshold,
2031
- 8,
2032
- num_warps=1,
2033
- )
2034
- return g, beta_output
vllm/model_executor/model_loader/weight_utils.py
+1
-75
@@ -12,7 +12,6 @@ import os
12
import tempfile
13
import threading
14
import time
15
-from collections import defaultdict
15
from collections.abc import Callable, Generator, Iterable
16
from contextlib import contextmanager
17
from pathlib import Path
@@ -23,7 +22,7 @@ import huggingface_hub.constants
22
import numpy as np
23
import regex as re
24
import torch
26
-from safetensors.torch import load, load_file, safe_open, save_file
25
+from safetensors.torch import load, load_file, safe_open
26
from tqdm.auto import tqdm
27
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
28
@@ -186,56 +185,6 @@ def maybe_download_from_modelscope(
185
return None
186
187
189
-def _shared_pointers(tensors):
190
- ptrs = defaultdict(list)
191
- for k, v in tensors.items():
192
- ptrs[v.data_ptr()].append(k)
193
- failing = []
194
- for _, names in ptrs.items():
195
- if len(names) > 1:
196
- failing.append(names)
197
- return failing
198
-
199
-
200
-def convert_bin_to_safetensor_file(
201
- pt_filename: str,
202
- sf_filename: str,
203
-) -> None:
204
- loaded = torch.load(pt_filename, map_location="cpu", weights_only=True)
205
- if "state_dict" in loaded:
206
- loaded = loaded["state_dict"]
207
- shared = _shared_pointers(loaded)
208
- for shared_weights in shared:
209
- for name in shared_weights[1:]:
210
- loaded.pop(name)
211
-
212
- # For tensors to be contiguous
213
- loaded = {k: v.contiguous() for k, v in loaded.items()}
214
-
215
- dirname = os.path.dirname(sf_filename)
216
- os.makedirs(dirname, exist_ok=True)
217
- save_file(loaded, sf_filename, metadata={"format": "pt"})
218
-
219
- # check file size
220
- sf_size = os.stat(sf_filename).st_size
221
- pt_size = os.stat(pt_filename).st_size
222
- if (sf_size - pt_size) / pt_size > 0.01:
223
- raise RuntimeError(
224
- f"""The file size different is more than 1%:
225
- - {sf_filename}: {sf_size}
226
- - {pt_filename}: {pt_size}
227
- """
228
- )
229
-
230
- # check if the tensors are the same
231
- reloaded = load_file(sf_filename)
232
- for k in loaded:
233
- pt_tensor = loaded[k]
234
- sf_tensor = reloaded[k]
235
- if not torch.equal(pt_tensor, sf_tensor):
236
- raise RuntimeError(f"The output tensors do not match for key {k}")
237
-
238
-
188
# TODO(woosuk): Move this to other place.
189
def get_quant_config(
190
model_config: ModelConfig, load_config: LoadConfig
@@ -1313,29 +1262,6 @@ def composed_weight_loader(
1262
return composed_loader
1263
1264
1316
-def initialize_dummy_weights(
1317
- model: torch.nn.Module,
1318
- model_config: ModelConfig,
1319
- low: float = -1e-3,
1320
- high: float = 1e-3,
1321
- seed: int = 1234,
1322
-) -> None:
1323
- """Initialize model weights with random values.
1324
-
1325
- The model weights must be randomly initialized for accurate performance
1326
- measurements. Additionally, the model weights should not cause NaNs in the
1327
- forward pass. We empirically found that initializing the weights with
1328
- values between -1e-3 and 1e-3 works well for most models.
1329
-
1330
- We use per-parameter random seed, so that dummy weights are consistent,
1331
- even if the model is partitioned across multiple devices. When the seed
1332
- is fixed, the random values generated by this function only depends on
1333
- the parameter's number of elements and its data type.
1334
- """
1335
- for param in model.state_dict().values():
1336
- initialize_single_dummy_weight(param, low, high, seed)
1337
-
1338
-
1265
@torch.no_grad()
1266
def initialize_single_dummy_weight(
1267
param: torch.Tensor,
vllm/model_executor/models/intern_vit.py
-28
@@ -114,34 +114,6 @@ class InternVisionEmbeddings(nn.Module):
114
return embeddings
115
116
117
-class InternVisionPatchModel(nn.Module):
118
- def __init__(self, config: PretrainedConfig):
119
- super().__init__()
120
- self.config = config
121
- self.embeddings = InternVisionEmbeddings(config)
122
-
123
- def get_input_embeddings(self):
124
- return self.embeddings
125
-
126
- def forward(
127
- self,
128
- pixel_values: torch.Tensor | None = None,
129
- pixel_embeds: torch.Tensor | None = None,
130
- ) -> torch.FloatTensor:
131
- if pixel_values is None and pixel_embeds is None:
132
- raise ValueError("You have to specify pixel_values or pixel_embeds")
133
-
134
- if pixel_embeds is not None:
135
- hidden_states = pixel_embeds
136
- elif pixel_values is not None:
137
- if pixel_values.ndim == 4:
138
- hidden_states = self.embeddings(pixel_values)
139
- else:
140
- raise ValueError(f"wrong pixel_values size: {pixel_values.shape}")
141
-
142
- return hidden_states
143
-
144
-
117
class InternParallelAttention(nn.Module):
118
"""Multi-headed attention from 'Attention Is All You Need' paper."""
119
vllm/model_executor/models/mllama4.py
-20
@@ -18,7 +18,6 @@
18
# limitations under the License.
19
import math
20
from collections.abc import Hashable, Iterable, Mapping
21
-from itertools import tee
21
from typing import Annotated, Any, Literal
22
23
import torch
@@ -1041,25 +1040,6 @@ class Llama4ForConditionalGeneration(
1040
) -> torch.Tensor | None:
1041
return self.language_model.compute_logits(hidden_states)
1042
1044
- def separate_weights(
1045
- self,
1046
- weights: Iterable[tuple[str, torch.Tensor]],
1047
- prefix: str,
1048
- ) -> tuple[Iterable[tuple[str, torch.Tensor]], Iterable[tuple[str, torch.Tensor]]]:
1049
- weights1, weights2 = tee(weights, 2)
1050
-
1051
- def get_prefix_weights() -> Iterable[tuple[str, torch.Tensor]]:
1052
- for name, data in weights1:
1053
- if name.startswith(prefix):
1054
- yield (name, data)
1055
-
1056
- def get_other_weights() -> Iterable[tuple[str, torch.Tensor]]:
1057
- for name, data in weights2:
1058
- if not name.startswith(prefix):
1059
- yield (name, data)
1060
-
1061
- return get_prefix_weights(), get_other_weights()
1062
-
1043
def _consolidate_qkv_weights(
1044
self, weights: Iterable[tuple[str, torch.Tensor]]
1045
) -> Iterable[tuple[str, torch.Tensor]]:
vllm/model_executor/models/teleflm.py
-33
@@ -23,47 +23,14 @@
23
# See the License for the specific language governing permissions and
24
# limitations under the License.
25
26
-import torch
27
-import torch.nn as nn
26
27
from vllm.config import VllmConfig
28
from vllm.model_executor.layers.logits_processor import LogitsProcessor
29
from vllm.model_executor.models.llama import (
32
- LlamaDecoderLayer,
30
LlamaForCausalLM,
34
- LlamaModel,
31
)
32
33
38
-class TeleFLMModel(LlamaModel):
39
- def __init__(
40
- self,
41
- *,
42
- vllm_config: VllmConfig,
43
- prefix: str = "",
44
- layer_type: type[nn.Module] = LlamaDecoderLayer,
45
- ):
46
- super().__init__(vllm_config=vllm_config, prefix=prefix, layer_type=layer_type)
47
- """
48
- This implementation is based on the µScaling paper presented at
49
- the ICLR 2025 Workshop:
50
- NanoLM: An Affordable LLM Study Benchmark \
51
- via Accurate Loss Prediction across Scales
52
- by Yiqun Yao et al.
53
- Available at: https://openreview.net/forum?id=IwaPYg1SCA
54
- arXiv preprint: https://arxiv.org/abs/2304.06875
55
- """
56
- self.use_mup = self.config.use_mup
57
- if self.use_mup:
58
- self.input_mult = self.config.input_mult
59
-
60
- def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
61
- embedding = self.embed_tokens(input_ids)
62
- if self.use_mup:
63
- embedding = embedding * self.input_mult
64
- return embedding
65
-
66
-
34
class TeleFLMForCausalLM(LlamaForCausalLM):
35
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
36
super().__init__(vllm_config=vllm_config, prefix=prefix)
vllm/transformers_utils/processors/funasr.py
-50
@@ -164,56 +164,6 @@ class WavFrontend(nn.Module):
164
feats_pad = pad_sequence(feats, batch_first=True, padding_value=0.0)
165
return feats_pad, feats_lens
166
167
- def forward_fbank(
168
- self, input: torch.Tensor, input_lengths: torch.Tensor
169
- ) -> tuple[torch.Tensor, torch.Tensor]:
170
- batch_size = input.size(0)
171
- feats = []
172
- feats_lens = []
173
- for i in range(batch_size):
174
- waveform_length = input_lengths[i]
175
- waveform = input[i][:waveform_length]
176
- waveform = waveform * (1 << 15)
177
- waveform = waveform.unsqueeze(0)
178
- mat = kaldi.fbank(
179
- waveform,
180
- num_mel_bins=self.n_mels,
181
- frame_length=self.frame_length,
182
- frame_shift=self.frame_shift,
183
- dither=self.dither,
184
- energy_floor=0.0,
185
- window_type=self.window,
186
- sample_frequency=self.fs,
187
- )
188
-
189
- feat_length = mat.size(0)
190
- feats.append(mat)
191
- feats_lens.append(feat_length)
192
-
193
- feats_lens = torch.as_tensor(feats_lens)
194
- feats_pad = pad_sequence(feats, batch_first=True, padding_value=0.0)
195
- return feats_pad, feats_lens
196
-
197
- def forward_lfr_cmvn(
198
- self, input: torch.Tensor, input_lengths: torch.Tensor
199
- ) -> tuple[torch.Tensor, torch.Tensor]:
200
- batch_size = input.size(0)
201
- feats = []
202
- feats_lens = []
203
- for i in range(batch_size):
204
- mat = input[i, : input_lengths[i], :]
205
- if self.lfr_m != 1 or self.lfr_n != 1:
206
- mat = apply_lfr(mat, self.lfr_m, self.lfr_n)
207
- if self.cmvn is not None:
208
- mat = apply_cmvn(mat, self.cmvn)
209
- feat_length = mat.size(0)
210
- feats.append(mat)
211
- feats_lens.append(feat_length)
212
-
213
- feats_lens = torch.as_tensor(feats_lens)
214
- feats_pad = pad_sequence(feats, batch_first=True, padding_value=0.0)
215
- return feats_pad, feats_lens
216
-
167
168
class FunASRFeatureExtractor(SequenceFeatureExtractor):
169
r"""Constructs a FunASR feature extractor.
vllm/transformers_utils/processors/minicpmo.py
+1
-65
@@ -17,12 +17,11 @@
17
"""Processor class for MiniCPMO."""
18
19
import math
20
-from typing import Literal, TypeAlias
20
+from typing import TypeAlias
21
22
import numpy as np
23
import regex
24
import torch
25
-import torchaudio
25
from transformers.image_processing_utils import BatchFeature
26
from transformers.image_utils import ImageInput
27
from transformers.processing_utils import ProcessorMixin
@@ -522,66 +521,3 @@ class MiniCPMOProcessor(ProcessorMixin):
521
padding_length.append(tensor.shape[-1] - len(item))
522
523
return tensor, padding_length
525
-
526
-
527
-class MelSpectrogramFeatures(torch.nn.Module):
528
- def __init__(
529
- self,
530
- sample_rate=24000,
531
- n_fft=1024,
532
- hop_length=256,
533
- n_mels=100,
534
- padding: Literal["center", "same"] = "center",
535
- ):
536
- super().__init__()
537
- if padding not in ["center", "same"]:
538
- raise ValueError("Padding must be 'center' or 'same'.")
539
- self.padding = padding
540
- self.mel_spec = torchaudio.transforms.MelSpectrogram(
541
- sample_rate=sample_rate,
542
- n_fft=n_fft,
543
- hop_length=hop_length,
544
- n_mels=n_mels,
545
- center=padding == "center",
546
- power=1,
547
- )
548
-
549
- def __call__(self, audio: torch.Tensor) -> torch.Tensor:
550
- """audio: Tensor([num_channels, num_samples])."""
551
- return super().__call__(audio)
552
-
553
- def forward(self, audio: torch.Tensor) -> torch.Tensor:
554
- """audio: Tensor([num_channels, num_samples])."""
555
- mel: torch.Tensor = self.mel_spec(audio)
556
- features = torch.log(torch.clip(mel, min=1e-5))
557
- return features
558
-
559
-
560
-class ChatTTSProcessor:
561
- def __init__(self, text_tokenizer):
562
- self.audio_processor = MelSpectrogramFeatures()
563
- self.text_tokenizer = text_tokenizer
564
-
565
- def __call__(self, text_list, audio_list):
566
- assert len(text_list) == len(audio_list)
567
- input_ids_varlen = []
568
- for text in text_list:
569
- input_ids_ = self.text_tokenizer.encode(
570
- text, return_tensors="pt", add_special_tokens=False
571
- ) # [1, seq_len]
572
- input_ids_ = input_ids_.squeeze(0) # [seq_len]
573
- input_ids_varlen.append(input_ids_)
574
-
575
- audio_features_varlen = []
576
- for audio in audio_list:
577
- assert audio.shape.__len__() == 1 # [seq_len]
578
- try:
579
- mel = self.audio_processor(audio) # [100(num_mel_bins), seq_len_mel]
580
- except Exception as e:
581
- raise e
582
- audio_features_varlen.append(mel)
583
-
584
- return {
585
- "tts_input_ids_varlen": input_ids_varlen, # return List[Tensor]
586
- "tts_input_features_varlen": audio_features_varlen, # return List[Tensor]
587
- }
vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py
-42
@@ -276,48 +276,6 @@ def generate_sparse_seqlen_triton(
276
return out
277
278
279
-@triton.jit
280
-def fetch_id_to_ragged_kernel(
281
- in_tensor_ptr, # [num_seq, topk]
282
- cumsum_ptr, # [num_seq + 1]
283
- out_tensor_ptr, # [max_num_seq * topk]
284
- in_tensor_ptr_stride,
285
- TOPK: tl.constexpr,
286
- TOKEN_NUM: tl.constexpr,
287
- BLOCK_SIZE: tl.constexpr,
288
-):
289
- seq_id = tl.program_id(0)
290
- block_id = tl.program_id(1)
291
- offset = tl.arange(0, BLOCK_SIZE)
292
- token_start = tl.load(cumsum_ptr + seq_id)
293
- token_end = tl.load(cumsum_ptr + seq_id + 1)
294
- token_num = token_end - token_start
295
- row_offset = block_id * BLOCK_SIZE
296
- if row_offset >= token_num:
297
- return
298
- in_tensor_offset = seq_id * in_tensor_ptr_stride + row_offset + offset
299
- in_tensor_mask = (row_offset + offset) < TOPK
300
- in_tensor_val = tl.load(in_tensor_ptr + in_tensor_offset, mask=in_tensor_mask)
301
- out_tensor_offset = token_start + row_offset + offset
302
- out_tensor_mask = (out_tensor_offset < token_end) & in_tensor_mask
303
- tl.store(out_tensor_ptr + out_tensor_offset, in_tensor_val, mask=out_tensor_mask)
304
-
305
-
306
-def fetch_id_to_ragged_triton(
307
- in_tensor: torch.Tensor, cumsum: torch.Tensor, out_tensor: torch.Tensor, topk
308
-):
309
- num_tokens = in_tensor.size(0)
310
- block_size = 64
311
- num_block_per_row = triton.cdiv(topk, block_size)
312
- grid = (
313
- num_tokens,
314
- num_block_per_row,
315
- )
316
- fetch_id_to_ragged_kernel[grid](
317
- in_tensor, cumsum, out_tensor, in_tensor.stride(0), topk, num_tokens, block_size
318
- )
319
-
320
-
279
class ROCMAiterMLASparseBackend(AttentionBackend):
280
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.float16, torch.bfloat16]
281
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
vllm/v1/worker/gpu/sample/gumbel.py
-15
@@ -62,21 +62,6 @@ def apply_temperature(
62
)
63
64
65
-@triton.jit
66
-def tl_rand64(seed, offset, includes_zero: tl.constexpr):
67
- lo, hi, _, _ = tl.randint4x(seed, offset)
68
- lo = lo.to(tl.uint32, bitcast=True).to(tl.uint64)
69
- hi = hi.to(tl.uint32, bitcast=True).to(tl.uint64)
70
- r = (hi << 32) | lo
71
-
72
- # 1 / 2**64
73
- scale = 5.421010862427522170037e-20
74
- u = r.to(tl.float64) * scale
75
- if not includes_zero:
76
- u = tl.maximum(u, 2.2250738585072014e-308) # float64 tiny
77
- return u
78
-
79
-
65
@triton.jit
66
def tl_rand32(seed, offset, includes_zero: tl.constexpr):
67
u = tl.rand(seed, offset)