b5bcb3ce88 · tduncan/Nvidia-Optimization-Stack-VLLM

vllm/model_executor/kernels/linear/cute_dsl/_ll_bf16_dotprod.py

-4

@@ -307,7 +307,3 @@ class LLBf16Dotprod:

307 )

308 if const_expr(self.use_pdl):

309 cute.arch.griddepcontrol_launch_dependents()

310 -

311 -

312 -def make_host_bf16(k_val: int, bs: int = 128):

313 - return LLBf16Dotprod(k=k_val, bs=bs)

vllm/model_executor/layers/fused_moe/prepare_finalize/nixl_ep.py

-5

@@ -160,11 +160,6 @@ class NixlEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):

160 return topk_ids

161 return self.global_to_physical[topk_ids]

162

163 - def _map_local_to_global_ids(self, expert_topk_ids: torch.Tensor) -> torch.Tensor:

164 - if self.local_expert_global_ids is None:

165 - return expert_topk_ids

166 - return self.local_expert_global_ids[expert_topk_ids]

167 -

163 def _do_quant(

164 self,

165 x: torch.Tensor | tuple[torch.Tensor, torch.Tensor],

vllm/model_executor/layers/quantization/fp8.py

-10

@@ -254,16 +254,6 @@ class CopyNumelCounter(TorchDispatchMode):

254 return out

255

256

257 -def _copy_missing_attrs(old: torch.Tensor, new: torch.Tensor) -> None:

258 - """Copies any attrs present in `old` but not in `new` to `new`"""

259 - new_attrs = set(dir(new))

260 - attrs_to_set = {}

261 - for attr in dir(old):

262 - if attr not in new_attrs:

263 - attrs_to_set[attr] = getattr(old, attr)

264 - set_weight_attrs(new, attrs_to_set)

265 -

266 -

257 class Fp8LinearMethod(LinearMethodBase):

258 """Linear method for FP8.

259 Supports loading FP8 checkpoints with static weight scale and

vllm/model_executor/layers/quantization/modelopt.py

-87

@@ -58,9 +58,6 @@ from vllm.model_executor.layers.quantization.base_config import (

58 QuantizeMethodBase,

59 )

60 from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod

61 -from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (

62 - swap_w13_to_w31,

63 -)

61 from vllm.model_executor.layers.quantization.utils.fp8_utils import (

62 process_fp8_input_tensor_strategy_moe,

63 process_fp8_weight_channel_strategy,

@@ -2014,90 +2011,6 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):

2011 f"Expected {name} dtype {expected_dtype}, got {actual}."

2012 )

2013

2017 - def _shuffle_weights_for_trtllm(self, layer: torch.nn.Module) -> None:

2018 - """Shuffle weights and scales into FlashInfer TRTLLM MXFP8 layout."""

2019 - from flashinfer import (

2020 - reorder_rows_for_gated_act_gemm,

2021 - shuffle_matrix_a,

2022 - shuffle_matrix_sf_a,

2023 - )

2024 -

2025 - epilogue_tile_m = 128

2026 - num_experts = layer.w13_weight.shape[0]

2027 - is_gated = self.moe.is_act_and_mul

2028 - intermediate_size_factor = 2 if is_gated else 1

2029 -

2030 - w13_weight = layer.w13_weight.data

2031 - w13_scale = layer.w13_weight_scale.data

2032 - if is_gated:

2033 - # FI TRTLLM gated kernels use W31 ordering. Model checkpoints store

2034 - # gated projection as W13, so convert once before shuffling.

2035 - w13_weight = swap_w13_to_w31(w13_weight)

2036 - w13_scale = swap_w13_to_w31(w13_scale)

2037 -

2038 - w13_weight_shuffled = []

2039 - w2_weight_shuffled = []

2040 - w13_scale_shuffled = []

2041 - w2_scale_shuffled = []

2042 - for i in range(num_experts):

2043 - w13_i = w13_weight[i].reshape(

2044 - intermediate_size_factor * layer.intermediate_size_per_partition, -1

2045 - )

2046 - w13_sf_i = w13_scale[i].reshape(

2047 - intermediate_size_factor * layer.intermediate_size_per_partition, -1

2048 - )

2049 - if is_gated:

2050 - # Reorder rows for gated activation layout expected by TRTLLM.

2051 - w13_i = reorder_rows_for_gated_act_gemm(w13_i.clone())

2052 - w13_sf_i = reorder_rows_for_gated_act_gemm(w13_sf_i.clone())

2053 -

2054 - w13_shuffled_i = shuffle_matrix_a(w13_i.view(torch.uint8), epilogue_tile_m)

2055 - w2_shuffled_i = shuffle_matrix_a(

2056 - layer.w2_weight.data[i].view(torch.uint8), epilogue_tile_m

2057 - )

2058 - w13_weight_shuffled.append(

2059 - w13_shuffled_i.contiguous().view(MXFP8_VALUE_DTYPE)

2060 - )

2061 - w2_weight_shuffled.append(

2062 - w2_shuffled_i.contiguous().view(MXFP8_VALUE_DTYPE)

2063 - )

2064 - w13_sf_shuffled_i = shuffle_matrix_sf_a(

2065 - w13_sf_i.view(torch.uint8).reshape(

2066 - intermediate_size_factor * layer.intermediate_size_per_partition,

2067 - -1,

2068 - ),

2069 - epilogue_tile_m,

2070 - )

2071 - w2_sf_shuffled_i = shuffle_matrix_sf_a(

2072 - layer.w2_weight_scale.data[i]

2073 - .view(torch.uint8)

2074 - .reshape(layer.hidden_size, -1),

2075 - epilogue_tile_m,

2076 - )

2077 - w13_scale_shuffled.append(

2078 - w13_sf_shuffled_i.contiguous().view(MXFP8_SCALE_DTYPE)

2079 - )

2080 - w2_scale_shuffled.append(

2081 - w2_sf_shuffled_i.contiguous().view(MXFP8_SCALE_DTYPE)

2082 - )

2083 -

2084 - replace_parameter(

2085 - layer, "w13_weight", torch.stack(w13_weight_shuffled).contiguous()

2086 - )

2087 - replace_parameter(

2088 - layer, "w2_weight", torch.stack(w2_weight_shuffled).contiguous()

2089 - )

2090 - replace_parameter(

2091 - layer,

2092 - "w13_weight_scale",

2093 - torch.stack(w13_scale_shuffled).contiguous(),

2094 - )

2095 - replace_parameter(

2096 - layer,

2097 - "w2_weight_scale",

2098 - torch.stack(w2_scale_shuffled).contiguous(),

2099 - )

2100 -

2014 def _dequant_mxfp8_weights_to_bf16(self, layer: RoutedExperts) -> None:

2015 """One-time MXFP8->BF16 weight dequant for the emulation path.

2016

vllm/model_executor/layers/quantization/utils/nvfp4_emulation_utils.py

-13

@@ -116,19 +116,6 @@ def _dequantize_nvfp4_kernel(

116 tl.store(output_ptr + out_indices, result, mask=block_mask[:, None])

117

118

119 -@triton.jit

120 -def _e2m1_lookup(magnitude):

121 - """Lookup E2M1 float value from 3-bit magnitude."""

122 - result = tl.where(magnitude == 1, 0.5, 0.0)

123 - result = tl.where(magnitude == 2, 1.0, result)

124 - result = tl.where(magnitude == 3, 1.5, result)

125 - result = tl.where(magnitude == 4, 2.0, result)

126 - result = tl.where(magnitude == 5, 3.0, result)

127 - result = tl.where(magnitude == 6, 4.0, result)

128 - result = tl.where(magnitude == 7, 6.0, result)

129 - return result

130 -

131 -

119 @triton.jit

120 def _round_to_fp4(x):

121 """Round float values to the nearest E2M1 representable value.

vllm/model_executor/models/funaudiochat.py

-19

@@ -257,25 +257,6 @@ class FunAudioChatAudioEncoder(nn.Module):

257 def dtype(self) -> torch.dtype:

258 return self.conv1.weight.dtype

259

260 - def _prepare_attention_mask(

261 - self, inputs_tensor: torch.Tensor, cu_seqlens: torch.Tensor

262 - ) -> torch.Tensor | None:

263 - if getattr(self.config, "_attn_implementation", "eager") == "flash_attention_2":

264 - return None

265 -

266 - seq_length = inputs_tensor.shape[0]

267 - attention_mask = torch.full(

268 - (1, 1, seq_length, seq_length),

269 - torch.finfo(inputs_tensor.dtype).min,

270 - device=inputs_tensor.device,

271 - dtype=inputs_tensor.dtype,

272 - )

273 - for i in range(1, len(cu_seqlens)):

274 - start = int(cu_seqlens[i - 1].item())

275 - end = int(cu_seqlens[i].item())

276 - attention_mask[..., start:end, start:end] = 0

277 - return attention_mask

278 -

260 def forward(

261 self,

262 input_features: torch.Tensor,

vllm/model_executor/models/hyperclovax_vision.py

-32

@@ -2,7 +2,6 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 # copied from : https://github.com/huggingface/transformers

4 import ast

5 -from collections import defaultdict

5 from collections.abc import Iterable, Mapping, Sequence

6 from functools import partial

7 from itertools import accumulate

@@ -891,37 +890,6 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

890 for i in range(len(feats_per_video))

891 )

892

894 - def _prepare_multimodal_kwargs(self, **kwargs: object):

895 - output = defaultdict(list)

896 - for k, v in kwargs.items():

897 - if len(v) < 1 or len(v[0]) < 1:

898 - continue # if empty batch of empty sample

899 -

900 - new_k, is_video = k, False

901 - if not k.endswith("_images") and not k.endswith("_videos"):

902 - pass

903 - else:

904 - new_k, is_video = k.split("_")[:-1], k.split("_")[-1]

905 - new_k = "_".join(new_k)

906 - is_video = is_video == "videos"

907 -

908 - for _sample_idx, _v in enumerate(v): # batch -> sample

909 - if new_k not in ["pixel_values"]:

910 - if len(output[new_k]) < _sample_idx + 1:

911 - output[new_k].append(list())

912 - _v = _v.detach().cpu().numpy().tolist()

913 - output[new_k][_sample_idx] += _v

914 - elif isinstance(_v, torch.Tensor):

915 - if len(output[new_k]) < _sample_idx + 1:

916 - output[new_k].append(list())

917 - output["is_videos"].append(list())

918 - _v = list(torch.unbind(_v, dim=0))

919 - output[new_k][_sample_idx] += _v

920 - output["is_videos"][_sample_idx] += [

921 - is_video,

922 - ] * len(_v)

923 - return dict(output)

924 -

893 def compute_logits(

894 self,

895 hidden_states: torch.Tensor,

vllm/model_executor/models/idefics3.py

-24

@@ -138,30 +138,6 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):

138

139 return height, width

140

141 - def _get_resize_output_image_size(

142 - self,

143 - *,

144 - image_width: int,

145 - image_height: int,

146 - resolution_max_side: int,

147 - ) -> tuple[int, int]:

148 - hf_processor = self.get_hf_processor()

149 - image_processor: Idefics3ImageProcessor = hf_processor.image_processor

150 - max_image_size = image_processor.size["longest_edge"]

151 - if resolution_max_side > max_image_size:

152 - raise ValueError(

153 - "`resolution_max_side` cannot be larger than `max_image_size`"

154 - )

155 -

156 - height, width = image_height, image_width

157 -

158 - # Find the output size, when rescaling the longest edge to max_len and

159 - # preserving the aspect ratio

160 - height, width = self._resize_output_size(

161 - height=height, width=width, max_len=resolution_max_side

162 - )

163 - return height, width

164 -

141 def _get_image_feature_grid_size(

142 self,

143 *,

vllm/model_executor/models/llava_onevision2.py

-4

@@ -603,10 +603,6 @@ _OV2_FRAME_FACTOR = 2

603 _OV2_FPS_MIN_FRAMES = 4

604

605

606 -def _round_by_factor(n: float, factor: int) -> int:

607 - return round(n / factor) * factor

608 -

609 -

606 def _ceil_by_factor(n: float, factor: int) -> int:

607 import math as _math

608

vllm/model_executor/models/qwen3_omni_moe_thinker.py

-46

@@ -1530,52 +1530,6 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(

1530 result_placeholders["audio"] = audio_placeholders

1531 return result_placeholders

1532

1533 - def _get_raw_input_ids(

1534 - self,

1535 - token_ids: list[int],

1536 - use_audio_in_video: bool = False,

1537 - ) -> list[int]:

1538 - tokenizer = self.info.get_tokenizer()

1539 - vision_bos_token = tokenizer.encode(tokenizer.vision_bos_token)[0]

1540 - vision_eos_token = tokenizer.encode(tokenizer.vision_eos_token)[0]

1541 - audio_bos_token = tokenizer.encode(tokenizer.audio_bos_token)[0]

1542 - audio_eos_token = tokenizer.encode(tokenizer.audio_eos_token)[0]

1543 - audio_token = tokenizer.encode("<|audio_pad|>")[0]

1544 - image_token = tokenizer.encode("<|image_pad|>")[0]

1545 - video_token = tokenizer.encode("<|video_pad|>")[0]

1546 -

1547 - result = token_ids[:]

1548 - if use_audio_in_video:

1549 - while True:

1550 - start = None

1551 - for i in range(len(result) - 1):

1552 - if result[i : i + 2] == [vision_bos_token, audio_bos_token]:

1553 - start = i

1554 - break

1555 - if start is not None:

1556 - end = None

1557 - for i in range(start + 2, len(result) - 1):

1558 - if result[i : i + 2] == [audio_eos_token, vision_eos_token]:

1559 - end = i

1560 - break

1561 - if end is not None:

1562 - result = (

1563 - result[:start]

1564 - + [vision_bos_token, video_token, vision_eos_token]

1565 - + result[end + 2 :]

1566 - )

1567 - else:

1568 - break

1569 -

1570 - for mm_token in [audio_token, image_token, video_token]:

1571 - compressed = []

1572 - for x in result:

1573 - if x != mm_token or (not compressed or compressed[-1] != mm_token):

1574 - compressed.append(x)

1575 - result = compressed

1576 -

1577 - return result

1578 -

1533

1534 class Qwen3OmniMoeConditionalGenerationMixin(Qwen2_5OmniConditionalGenerationMixin):

1535 def _process_audio_input(

vllm/model_executor/models/radio.py

-48

@@ -331,54 +331,6 @@ class ViTPatchGenerator(nn.Module):

331 def num_skip(self):

332 return self.num_cls_tokens + self.num_registers

333

334 - def _load_embed(self, src_embed: torch.Tensor, targ_embed: nn.Parameter):

335 - if src_embed.shape != targ_embed.shape:

336 - src_size = int(math.sqrt(src_embed.shape[1]))

337 -

338 - assert src_size**2 == src_embed.shape[1], (

339 - "Unable to interpolate non-square embedding"

340 - )

341 -

342 - src_embed = rearrange(

343 - src_embed, "b (h w) c -> b c h w", h=src_size, w=src_size

344 - )

345 - src_embed = F.interpolate(

346 - src_embed,

347 - size=(self.num_rows, self.num_cols),

348 - mode="bicubic",

349 - align_corners=True,

350 - antialias=False,

351 - )

352 - src_embed = rearrange(src_embed, "b c h w -> b (h w) c")

353 - targ_embed.data.copy_(src_embed)

354 -

355 - def _load_projection(

356 - self, src_proj_weight: torch.Tensor, targ_proj_weight: torch.Tensor

357 - ):

358 - if src_proj_weight.shape != targ_proj_weight.shape:

359 - src_patch_size = int(math.sqrt(src_proj_weight.shape[1] // 3))

360 -

361 - assert (src_patch_size**2) * 3 == src_proj_weight.shape[1], (

362 - "Unable to interpolate non-square patch size"

363 - )

364 -

365 - src_proj_weight = rearrange(

366 - src_proj_weight,

367 - "b (c h w) -> b c h w",

368 - c=3,

369 - h=src_patch_size,

370 - w=src_patch_size,

371 - )

372 - src_proj_weight = F.interpolate(

373 - src_proj_weight,

374 - size=(self.patch_size, self.patch_size),

375 - mode="bicubic",

376 - align_corners=True,

377 - antialias=False,

378 - )

379 - src_proj_weight = rearrange(src_proj_weight, "b c h w -> b (c h w)")

380 - targ_proj_weight.data.copy_(src_proj_weight)

381 -

334 def embed_patches(self, x: torch.Tensor) -> torch.Tensor:

335 patches = self.im_to_patches(x)

336 patches = self.embedder(patches)

vllm/reasoning/hunyuan_a13b_reasoning_parser.py

-12

@@ -130,18 +130,6 @@ class HunyuanA13BReasoningParser(ReasoningParser):

130

131 return None, model_output

132

133 - def _is_strict_increasing_subsequence(

134 - self, subsequence: Sequence[int], sequence: Sequence[int]

135 - ) -> bool:

136 - if not subsequence:

137 - return False

138 -

139 - sub_idx = 0

140 - for num in sequence:

141 - if sub_idx < len(subsequence) and num == subsequence[sub_idx]:

142 - sub_idx += 1

143 - return sub_idx == len(subsequence)

144 -

133 def extract_reasoning_streaming(

134 self,

135 previous_text: str,

vllm/tokenizers/mistral.py

-14

@@ -45,20 +45,6 @@ if TYPE_CHECKING:

45 logger = init_logger(__name__)

46

47

48 -def _pop_unallowed_keys_and_warn(

49 - dictionary: dict[str, Any], allowed_keys: set[str], err_dict_name: str

50 -):

51 - keys = list(dictionary.keys())

52 - for key in keys:

53 - if key not in allowed_keys:

54 - dictionary.pop(key)

55 - logger.warning_once(

56 - f"'{key=}' is not supported by mistral-common "

57 - f"for {err_dict_name}. It has been popped from the "

58 - "object."

59 - )

60 -

61 -

48 def maybe_serialize_tool_calls(request: "MistralChatCompletionRequest"):

49 # SEE: https://github.com/vllm-project/vllm/pull/9951

50 # Credits go to: @gcalmettes

vllm/tool_parsers/gemma4_utils.py

-7

@@ -37,15 +37,8 @@ do not need a transformers dependency for output parsing.

37

38 import regex as re

39

40 -# Tool call delimiter tokens as they appear in decoded text.

41 -# Standard format: <|tool_call>call:name{args}<tool_call|>

42 -_TOOL_CALL_START_TAG = "<|tool_call>"

43 -_TOOL_CALL_END_TAG = "<tool_call|>"

40 _TOOL_RESPONSE_START_TAG = "<|tool_response>"

41

46 -# Gemma4 escape token as it appears in decoded text.

47 -_ESCAPE_TOKEN = '<|"|>'

48 -

42

43 def _parse_tool_arguments(args_str: str) -> dict[str, str]:

44 """Parse tool call arguments from the Gemma4 compact format.

vllm/tool_parsers/streaming.py

-6

@@ -47,12 +47,6 @@ def _bracket_level_state(

47 return level, in_string, escaped

48

49

50 -def _bracket_level(s: str, opening: str = "{", closing: str = "}") -> int:

51 - """Calculate the current level of nested brackets in a string."""

52 - level, _, _ = _bracket_level_state(s, opening, closing)

53 - return level

54 -

55 -

50 def filter_delta_text(

51 delta_text: str,

52 previous_text: str,

vllm/transformers_utils/configs/hunyuan_vl.py

-41

@@ -194,7 +194,6 @@ class HunYuanVLTextConfig(PretrainedConfig):

194 self.use_cache = use_cache

195 self.rope_theta = rope_theta

196 self.rope_scaling = rope_scaling

197 - # self._rope_scaling_validation() # TODO: Need validation?

197 self.attention_bias = attention_bias

198 self.attention_dropout = attention_dropout

199

@@ -206,46 +205,6 @@ class HunYuanVLTextConfig(PretrainedConfig):

205 **kwargs,

206 )

207

209 - def _rope_scaling_validation(self):

210 - """

211 - Validate the `rope_scaling` configuration.

212 - """

213 - if self.rope_scaling is None:

214 - return

215 -

216 - if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2:

217 - raise ValueError(

218 - "`rope_scaling` must be a dictionary with with two fields, `type` and "

219 - f"`factor` or `type` and `alpha`, got {self.rope_scaling}"

220 - )

221 - rope_scaling_type = self.rope_scaling.get("type", None)

222 - rope_scaling_factor = self.rope_scaling.get("factor", None)

223 - rope_scaling_alpha = self.rope_scaling.get("alpha", None)

224 - if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:

225 - raise ValueError(

226 - "`rope_scaling`'s type field must be one of ['linear', 'dynamic'], "

227 - f"got {rope_scaling_type}"

228 - )

229 - if rope_scaling_factor is None and rope_scaling_alpha is None:

230 - raise ValueError(

231 - "`rope_scaling`'s factor or alpha field must be have one, "

232 - "got both of none"

233 - )

234 - if rope_scaling_factor is not None and (

235 - not isinstance(rope_scaling_factor, float) or rope_scaling_factor <= 1.0

236 - ):

237 - raise ValueError(

238 - "`rope_scaling`'s factor field must be a float > 1.0, "

239 - f"got {rope_scaling_factor}"

240 - )

241 - if rope_scaling_alpha is not None and (

242 - not isinstance(rope_scaling_alpha, float) or rope_scaling_alpha <= 1.0

243 - ):

244 - raise ValueError(

245 - "`rope_scaling`'s alpha field must be a float > 1.0, "

246 - f"got {rope_scaling_alpha}"

247 - )

248 -

208

209 class HunYuanVLConfig(PretrainedConfig):

210 model_type = "hunyuan_vl"

vllm/transformers_utils/processors/minicpmo.py

-13

@@ -72,19 +72,6 @@ class MiniCPMOProcessor(ProcessorMixin):

72 self.version = getattr(image_processor, "version", None)

73 self.pool_step = pool_step

74

75 - def _safe_get_token_id(self, attr_name, default_token_str):

76 - """Get token ID safely, with fallback to default."""

77 - val = getattr(self.tokenizer, attr_name, None)

78 - if val is None:

79 - val = self.tokenizer.convert_tokens_to_ids(default_token_str)

80 - if val is None:

81 - return -1

82 - return val

83 -

84 - def _safe_get_token_str(self, attr_name, default_token_str):

85 - """Get token string safely, with fallback to default."""

86 - return getattr(self.tokenizer, attr_name, default_token_str)

87 -

75 def __call__(

76 self,

77 text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput],

vllm/transformers_utils/processors/minimax_m3.py

-80

@@ -15,7 +15,6 @@ feeds decoded frames to the processor.

15

16 import math

17

18 -import regex as re

18 import torch

19 from torchvision.transforms import InterpolationMode

20 from transformers import AutoTokenizer, BatchFeature

@@ -554,85 +553,6 @@ class MiniMaxVLProcessor(ProcessorMixin):

553 self.VISION_END_TOKEN

554 )

555

557 - def _prune_video_tokens(

558 - self,

559 - input_text: str,

560 - video_segments: list[int],

561 - video_token: str,

562 - ) -> str:

563 - """Prune video tokens by temporal_patch_size (e.g., 2:1).

564 -

565 - Expects the prompt to carry exactly sum(video_segments) video tokens

566 - — i.e. one token per *sampled* frame — then drops tokens.

567 - """

568 - # If no videos or temporal_patch_size <= 1, no pruning needed

569 - if not video_segments or self.video_processor.temporal_patch_size <= 1:

570 - return input_text

571 -

572 - # Split while keeping delimiters

573 - special_tokens = [video_token]

574 - pattern = "|".join(map(re.escape, special_tokens))

575 - parts = re.split(f"({pattern})", input_text)

576 -

577 - def is_timestamp(text: str) -> bool:

578 - """Check if text ends with timestamp format like ']<]0.0 seconds[>['"""

579 - return (

580 - text.endswith("seconds[>[")

581 - or text.endswith("seconds[>[ ")

582 - or text.endswith("seconds [>[")

583 - or text.endswith("seconds [>[ ")

584 - )

585 -

586 - def extract_timestamp(text: str) -> str:

587 - """Extract timestamp text from the end, starting from ']<]'"""

588 - start_index = text.rfind("]<]")

589 - if start_index == -1:

590 - raise ValueError(f"Failed to extract timestamp: {text}")

591 - return text[start_index:]

592 -

593 - # Build new text with pruned video tokens

594 - final_parts = []

595 - current_seg_idx = 0 # Which video segment we're in

596 - frame_in_seg = 0 # Frame index within current segment

597 - last_timestamp_len = 0 # Length of timestamp to potentially remove

598 -

599 - for part in parts:

600 - if part == video_token:

601 - if current_seg_idx < len(video_segments):

602 - if frame_in_seg % self.video_processor.temporal_patch_size == 0:

603 - # Keep this video token

604 - final_parts.append(part)

605 - frame_in_seg += 1

606 - if frame_in_seg >= video_segments[current_seg_idx]:

607 - current_seg_idx += 1

608 - frame_in_seg = 0

609 - last_timestamp_len = 0

610 - else:

611 - # Skip this video token

612 - frame_in_seg += 1

613 - if frame_in_seg >= video_segments[current_seg_idx]:

614 - current_seg_idx += 1

615 - frame_in_seg = 0

616 - # Remove the timestamp that was already appended

617 - if last_timestamp_len > 0:

618 - assert len(final_parts) > 0

619 - final_parts[-1] = final_parts[-1][:-last_timestamp_len]

620 - last_timestamp_len = 0

621 - else:

622 - # No more video segments, keep as is

623 - final_parts.append(part)

624 - last_timestamp_len = 0

625 - else:

626 - # Text part

627 - final_parts.append(part)

628 - # Check if this text ends with a timestamp

629 - if is_timestamp(part):

630 - last_timestamp_len = len(extract_timestamp(part))

631 - else:

632 - last_timestamp_len = 0

633 -

634 - return "".join(final_parts)

635 -

556 def __call__(

557 self,

558 images=None,

vllm/utils/nvtx_pytorch_hooks.py

-3

@@ -231,9 +231,6 @@ class PytHooks:

231 super().__init__()

232 self.module_to_name_map = {}

233

234 - def _process_layer_params(self, module_obj):

235 - return process_layer_params(module_obj)

236 -

234 def module_fwd_hook(self, module_obj, in_tensor, out_tensor):

235 """Callback function that ends the NVTX marker.

236 Records the module name and tensor information.

vllm/v1/attention/ops/rocm_aiter_mla_sparse.py

-31

@@ -592,37 +592,6 @@ def rocm_fp8_mqa_logits(

592 return fp8_mqa_logits_torch(q, kv, weights, cu_seqlen_ks, cu_seqlen_ke)

593

594

595 -def _topk_indices_torch(

596 - logits: torch.Tensor,

597 - topk_tokens: int,

598 - row_starts: torch.Tensor | None = None,

599 -) -> torch.Tensor:

600 - k = min(topk_tokens, logits.shape[-1])

601 - values, indices = torch.topk(logits, k=k, dim=-1)

602 - indices = indices.to(torch.int32)

603 - indices = torch.where(

604 - values == float("-inf"),

605 - torch.full_like(indices, -1, dtype=torch.int32),

606 - indices,

607 - )

608 - if row_starts is not None:

609 - # Match the CUDA top_k_per_row_prefill contract: indices are local to

610 - # each row's valid [row_start, row_end) range, not columns in the

611 - # concatenated chunk logits matrix.

612 - starts = row_starts.to(dtype=torch.int32).view(-1, 1)

613 - indices = torch.where(indices < 0, indices, indices - starts)

614 - if k == topk_tokens:

615 - return indices

616 - padded = torch.full(

617 - (logits.shape[0], topk_tokens),

618 - -1,

619 - dtype=torch.int32,

620 - device=logits.device,

621 - )

622 - padded[:, :k] = indices

623 - return padded

624 -

625 -

595 def rocm_aiter_sparse_attn_indexer_fake(

596 hidden_states: torch.Tensor,

597 k_cache_prefix: LayerNameType,

vllm/v1/worker/gpu/model_runner.py

-4

@@ -731,10 +731,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):

731 if self.encoder_cache is not None:

732 self.encoder_cache.reset_encoder_cache()

733

734 - def _get_num_input_tokens(self, num_scheduled_tokens: int) -> int:

735 - # SP is not supported yet.

736 - return num_scheduled_tokens

737 -

734 def profile_cudagraph_memory(self) -> int:

735 # NOTE(woosuk): It is TBD whether we keep this API or not.

736 return 0

vllm/v1/worker/tpu_input_batch.py

-16

@@ -495,22 +495,6 @@ class InputBatch:

495 del self._req_ids[self.num_reqs :]

496 del self.req_output_token_ids[self.num_reqs :]

497

498 - def _make_prompt_token_ids_tensor(self) -> torch.Tensor:

499 - max_prompt_len = self.num_prompt_tokens[: self.num_reqs].max()

500 - prompt_token_ids_cpu_tensor = torch.empty(

501 - (self.num_reqs, max_prompt_len),

502 - device="cpu",

503 - dtype=torch.int64,

504 - pin_memory=self.pin_memory,

505 - )

506 - prompt_token_ids = prompt_token_ids_cpu_tensor.numpy()

507 - prompt_token_ids[:] = self.token_ids_cpu[: self.num_reqs, :max_prompt_len]

508 - # Use the value of vocab_size as a pad since we don't have a

509 - # token_id of this value.

510 - for i in range(self.num_reqs):

511 - prompt_token_ids[i, self.num_prompt_tokens[i] :] = self.vocab_size

512 - return prompt_token_ids_cpu_tensor.to(device=self.device, non_blocking=True)

513 -

498 def make_lora_inputs(

499 self, num_scheduled_tokens: np.ndarray, num_sampled_tokens: np.ndarray

500 ) -> tuple[tuple[int, ...], tuple[int, ...], set[LoRARequest]]:

[Refactor] Remove dead code in multiple files (#49745)