0cf49a5d15 · tduncan/Nvidia-Optimization-Stack-VLLM

vllm/compilation/passes/fx_utils.py

-19

@@ -14,10 +14,6 @@ def is_func(node: fx.Node, target: Target) -> bool:

14 return bool(node.op == "call_function" and node.target == target)

15

16

17 -def is_auto_func(node: fx.Node, op: OpOverload) -> bool:

18 - return is_func(node, auto_functionalized) and node.args[0] == op

19 -

20 -

17 # Returns the first auto_functionalized node with the given op (if it exists)

18 def find_auto_fn_maybe(nodes: Iterable[fx.Node], op: OpOverload) -> fx.Node | None:

19 for node in nodes:

@@ -42,13 +38,6 @@ def find_getitem_maybe(node: fx.Node, idx: int) -> fx.Node | None:

38 return None

39

40

45 -# Returns the getitem node that extracts the idx-th element from node

46 -def find_getitem(node: fx.Node, idx: int) -> fx.Node:

47 - ret = find_getitem_maybe(node, idx)

48 - assert ret is not None, f"Could not find getitem {idx} in node {node}"

49 - return ret

50 -

51 -

41 # An auto-functionalization-aware utility for finding nodes with a specific op

42 # Also handles op overload packets and finds all overloads

43 def find_op_nodes(

@@ -67,11 +56,3 @@ def find_op_nodes(

56 for n in graph.find_nodes(op="call_function", target=auto_functionalized):

57 if n.args[0] == op:

58 yield n

70 -

71 -

72 -# Asserts that the node only has one user and returns it

73 -# Even if a node has only 1 user, it might share storage with another node,

74 -# which might need to be taken into account.

75 -def get_only_user(node: fx.Node) -> fx.Node:

76 - assert len(node.users) == 1

77 - return next(iter(node.users))

vllm/compilation/passes/vllm_inductor_pass.py

-9

@@ -333,12 +333,3 @@ class VllmFusionPatternMatcherPass(VllmPatternMatcherPass):

333 def __call__(self, graph: torch.fx.Graph) -> None:

334 self.matched_count = self.pm_pass.apply(graph)

335 VllmPatternMatcherPass.match_table[self.pass_name] += self.matched_count

336 -

337 -

338 -class PrinterInductorPass(VllmInductorPass):

339 - def __init__(self, name: str, config: VllmConfig) -> None:

340 - super().__init__(config)

341 - self.name = name

342 -

343 - def __call__(self, graph: torch.fx.Graph) -> None:

344 - self.dump_graph(graph, self.name)

vllm/distributed/parallel_state.py

-3

@@ -2081,9 +2081,6 @@ def model_parallel_is_initialized():

2081 return _TP is not None and _PP is not None

2082

2083

2084 -_TP_STATE_PATCHED = False

2085 -

2086 -

2084 def get_tensor_model_parallel_world_size() -> int:

2085 """Return world size for the tensor model parallel group."""

2086 return get_tp_group().world_size

vllm/entrypoints/chat_utils.py

-9

@@ -2029,15 +2029,6 @@ async def parse_chat_messages_async(

2029 return conversation, mm_data, mm_uuids

2030

2031

2032 -def get_history_tool_calls_cnt(conversation: list[ConversationMessage]):

2033 - idx = 0

2034 - for msg in conversation:

2035 - if msg["role"] == "assistant":

2036 - tool_calls = msg.get("tool_calls")

2037 - idx += len(list(tool_calls)) if tool_calls is not None else 0 # noqa

2038 - return idx

2039 -

2040 -

2032 _KIMI_MODEL_TYPES = ("kimi_k2", "kimi_k25", "kimi_k3")

2033

2034

vllm/model_executor/layers/activation.py

-48

@@ -8,14 +8,8 @@ import torch

8 import torch.nn as nn

9 import torch.nn.functional as F

10

11 -from vllm.distributed import (

12 - divide,

13 - get_tensor_model_parallel_rank,

14 - get_tensor_model_parallel_world_size,

15 -)

11 from vllm.logger import init_logger

12 from vllm.model_executor.custom_op import CustomOp

18 -from vllm.model_executor.utils import set_weight_attrs

13 from vllm.platforms import CpuArchEnum, current_platform

14 from vllm.triton_utils import tl, triton

15 from vllm.utils.collection_utils import LazyDict

@@ -788,48 +782,6 @@ class XIELU(CustomOp):

782 return self.forward_native(input)

783

784

791 -class ScaledActivation(nn.Module):

792 - """An activation function with post-scale parameters.

793 -

794 - This is used for some quantization methods like AWQ.

795 - """

796 -

797 - def __init__(

798 - self,

799 - act_module: nn.Module,

800 - intermediate_size: int,

801 - input_is_parallel: bool = True,

802 - params_dtype: torch.dtype | None = None,

803 - ):

804 - super().__init__()

805 - self.act = act_module

806 - self.input_is_parallel = input_is_parallel

807 - if input_is_parallel:

808 - tp_size = get_tensor_model_parallel_world_size()

809 - intermediate_size_per_partition = divide(intermediate_size, tp_size)

810 - else:

811 - intermediate_size_per_partition = intermediate_size

812 - if params_dtype is None:

813 - params_dtype = torch.get_default_dtype()

814 - self.scales = nn.Parameter(

815 - torch.empty(intermediate_size_per_partition, dtype=params_dtype)

816 - )

817 - set_weight_attrs(self.scales, {"weight_loader": self.weight_loader})

818 -

819 - def forward(self, x: torch.Tensor) -> torch.Tensor:

820 - return self.act(x) / self.scales

821 -

822 - def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor):

823 - param_data = param.data

824 - if self.input_is_parallel:

825 - tp_rank = get_tensor_model_parallel_rank()

826 - shard_size = param_data.shape[0]

827 - start_idx = tp_rank * shard_size

828 - loaded_weight = loaded_weight.narrow(0, start_idx, shard_size)

829 - assert param_data.shape == loaded_weight.shape

830 - param_data.copy_(loaded_weight)

831 -

832 -

785 _ACTIVATION_REGISTRY = LazyDict(

786 {

787 "gelu": lambda: GELU(),

vllm/model_executor/layers/conv.py

-52

@@ -155,58 +155,6 @@ class Conv2dLayer(ConvLayerBase):

155 return self._forward_conv(x)

156

157

158 -class CausalConv2dLayer(Conv2dLayer):

159 - """

160 - A causal version of nn.Conv2d where each location in the 2D matrix would

161 - have no access to locations on its right or down

162 - All arguments are the same as nn.Conv2d except padding which should be

163 - set as None

164 - """

165 -

166 - def __init__(

167 - self,

168 - in_channels: int,

169 - out_channels: int,

170 - kernel_size: int,

171 - stride: int,

172 - padding: int = 0,

173 - dilation: int = 1,

174 - groups: int = 1,

175 - bias: bool = True,

176 - padding_mode: str = "zeros",

177 - *,

178 - params_dtype: torch.dtype | None = None,

179 - ) -> None:

180 - if padding is not None:

181 - raise ValueError(

182 - "Argument padding should be set to None for CausalConv2dLayer."

183 - )

184 - self._left_padding: int = kernel_size - 1

185 - self._right_padding: int = stride - 1

186 - padding = 0

187 -

188 - super().__init__(

189 - in_channels,

190 - out_channels,

191 - kernel_size,

192 - stride,

193 - padding,

194 - dilation,

195 - groups,

196 - bias,

197 - padding_mode,

198 - params_dtype=params_dtype,

199 - )

200 -

201 - def forward(

202 - self,

203 - x: torch.Tensor,

204 - ) -> torch.Tensor:

205 - x = F.pad(x, pad=(self._left_padding, self._right_padding, 0, 0))

206 - x = super().forward(x)

207 - return x

208 -

209 -

158 # --8<-- [start:conv3d]

159 @CustomOp.register("conv3d")

160 class Conv3dLayer(ConvLayerBase):

vllm/model_executor/layers/quantization/fp8.py

-21

@@ -4,7 +4,6 @@

4 from typing import TYPE_CHECKING, Any

5

6 import torch

7 -from torch.utils._python_dispatch import TorchDispatchMode

7

8 import vllm.envs as envs

9 import vllm.model_executor.layers.fused_moe.modular_kernel as mk

@@ -234,26 +233,6 @@ class Fp8Config(QuantizationConfig):

233 return cache_scale_mapper | QuantizationConfig.get_cache_scale_mapper()

234

235

237 -class CopyNumelCounter(TorchDispatchMode):

238 - """

239 - Tracks total number of elements modified with `copy_`. Useful for keeping

240 - track of weight loading where underlying weights can be arbitrarily

241 - transformed (such as with `narrow`) before calling copy.

242 - """

243 -

244 - def __init__(self):

245 - super().__init__()

246 - self.copied_numel = 0

247 -

248 - def __torch_dispatch__(self, func, types, args=(), kwargs=None):

249 - if kwargs is None:

250 - kwargs = {}

251 - out = func(*args, **kwargs)

252 - if func == torch.ops.aten.copy_.default:

253 - self.copied_numel += args[0].numel()

254 - return out

255 -

256 -

236 class Fp8LinearMethod(LinearMethodBase):

237 """Linear method for FP8.

238 Supports loading FP8 checkpoints with static weight scale and

vllm/model_executor/layers/utils.py

-6

@@ -25,12 +25,6 @@ MOE_LAYER_ROUTER_GATE_SUFFIXES = {

25 }

26

27

28 -def is_layer_moe_router_gate(prefix: str) -> bool:

29 - if not prefix:

30 - return False

31 - return prefix.rsplit(".", 1)[-1] in MOE_LAYER_ROUTER_GATE_SUFFIXES

32 -

33 -

28 def get_token_bin_counts_and_mask(

29 tokens: torch.Tensor,

30 vocab_size: int,

vllm/model_executor/models/glm_ocr.py

-4

@@ -242,10 +242,6 @@ class GlmOcrVisionBlock(Glm4vVisionBlock):

242 )

243

244

245 -class GlmOcrVisionPatchEmbed(Glm4vVisionPatchEmbed):

246 - pass

247 -

248 -

245 class GlmOcrPatchMerger(Glm4vPatchMerger):

246 pass

247

vllm/model_executor/models/idefics3.py

-35

@@ -103,41 +103,6 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):

103 def get_supported_mm_limits(self) -> Mapping[str, int | None]:

104 return {"image": None}

105

106 - def _resize_output_size(

107 - self,

108 - *,

109 - height: int,

110 - width: int,

111 - max_len: int | None = None,

112 - min_len: int = 1,

113 - max_size: int | None = None,

114 - ) -> tuple[int, int]:

115 - # Set default value for max_len if not provided

116 - max_len = max(height, width) if max_len is None else max_len

117 - aspect_ratio = width / height

118 -

119 - # Handle the maximum size constraint

120 - if max_size is not None:

121 - max_len = min(max_len, max_size)

122 -

123 - # Adjust dimensions according to the aspect ratio

124 - if width >= height:

125 - width = max_len

126 - height = int(width / aspect_ratio)

127 - else:

128 - height = max_len

129 - width = int(height * aspect_ratio)

130 -

131 - # Ensure both width and height are even (if needed)

132 - height += height % 2

133 - width += width % 2

134 -

135 - # Ensure dimensions are not smaller than the minimum length

136 - height = max(height, min_len)

137 - width = max(width, min_len)

138 -

139 - return height, width

140 -

106 def _get_image_feature_grid_size(

107 self,

108 *,

vllm/model_executor/models/llava_onevision2.py

-7

@@ -1531,13 +1531,6 @@ class LlavaOnevision2MultiModalDataParser(MultiModalDataParser):

1531 class LlavaOnevision2MultiModalProcessor(

1532 BaseMultiModalProcessor[LlavaOnevision2ProcessingInfo]

1533 ):

1534 - def _get_data_parser(self) -> MultiModalDataParser:

1535 - # Retained for symmetry; vLLM actually fetches the parser via

1536 - # info.get_data_parser() (see ProcessingInfo override above).

1537 - return LlavaOnevision2MultiModalDataParser(

1538 - self.info.get_hf_config().vision_config.spatial_merge_size

1539 - )

1540 -

1534 def _call_hf_processor(

1535 self,

1536 prompt: str,

vllm/model_executor/models/nemotron_parse.py

-18

@@ -77,24 +77,6 @@ class BartScaledWordEmbedding(VocabParallelEmbedding):

77 return super().forward(input_ids) * self.embed_scale

78

79

80 -class BartParallelLMHead(ParallelLMHead):

81 - """

82 - This module overrides ParallelLMHead's

83 - forward by dividing by embeddings scale,

84 - yielding effectively the inverse of

85 - BartScaledWordEmbedding

86 - """

87 -

88 - def __init__(

89 - self, num_embeddings: int, embedding_dim: int, embed_scale: float = 1.0

90 - ):

91 - super().__init__(num_embeddings, embedding_dim)

92 - self.embed_scale = embed_scale

93 -

94 - def forward(self, input_ids: torch.Tensor) -> torch.Tensor:

95 - return super().forward(input_ids) / self.embed_scale

96 -

97 -

80 class BartDecoderLayer(nn.Module):

81 def __init__(

82 self,

vllm/model_executor/models/phi4mm.py

-26

@@ -521,32 +521,6 @@ class Phi4MMAudioEmbeddingInputs(TensorSchema):

521 Phi4MMAudioInputs: TypeAlias = Phi4MMAudioFeatureInputs | Phi4MMAudioEmbeddingInputs

522

523

524 -def cat_with_pad(tensors, dim, padding_value=0):

525 - """

526 - cat along dim, while pad to max for all other dims

527 - """

528 - ndim = tensors[0].dim()

529 - assert all(t.dim() == ndim for t in tensors[1:]), (

530 - "All tensors must have the same number of dimensions"

531 - )

532 -

533 - out_size = [max(t.shape[i] for t in tensors) for i in range(ndim)]

534 - out_size[dim] = sum(t.shape[dim] for t in tensors)

535 - output = tensors[0].new_full(out_size, padding_value)

536 -

537 - index = 0

538 - for t in tensors:

539 - # Create a slice list where every dimension except dim is full slice

540 - slices = [slice(0, t.shape[d]) for d in range(ndim)]

541 - # Update only the concat dimension slice

542 - slices[dim] = slice(index, index + t.shape[dim])

543 -

544 - output[slices] = t

545 - index += t.shape[dim]

546 -

547 - return output

548 -

549 -

524 def stack_with_pad(

525 tensors: torch.Tensor | list[torch.Tensor],

526 padding_value: int | float = 0,

vllm/model_executor/models/phi4mm_utils.py

-8

@@ -1595,14 +1595,6 @@ class AttModule(nn.Module):

1595 return x, memory, pos_emb, att_mask

1596

1597

1598 -class AttBlock(BlockBase, AttModule):

1599 - """Attention Block module to support both Attention and Block module."""

1600 -

1601 - def memory_dims(self, max_len: bool = False) -> tuple[int, int]:

1602 - """memory dimensions"""

1603 - return (1, self.input_size)

1604 -

1605 -

1598 def masked_softmax(

1599 scores: Tensor,

1600 mask: Tensor | None,

vllm/model_executor/models/skyworkr1v.py

+1 -31

@@ -7,7 +7,7 @@

8 # Licensed under The MIT License [see LICENSE for details]

9 # --------------------------------------------------------

10 -from collections.abc import Iterable, Mapping

10 +from collections.abc import Iterable

11 from typing import Annotated, Literal, TypeAlias

12

13 import torch

@@ -15,8 +15,6 @@ import torch.nn as nn

15 from transformers import PretrainedConfig

16

17 from vllm.config import VllmConfig

18 -from vllm.config.multimodal import BaseDummyOptions

19 -from vllm.inputs import MultiModalDataDict

18 from vllm.model_executor.layers.linear import ReplicatedLinear

19 from vllm.model_executor.layers.quantization import QuantizationConfig

20 from vllm.model_executor.layers.quantization.auto_awq import AutoAWQConfig

@@ -24,7 +22,6 @@ from vllm.model_executor.models.intern_vit import (

22 InternVisionModel,

23 )

24 from vllm.multimodal import MULTIMODAL_REGISTRY

27 -from vllm.multimodal.processing import BaseDummyInputsBuilder

25 from vllm.sequence import IntermediateTensors

26 from vllm.transformers_utils.processors.internvl import (

27 InternVLImageProcessor,

@@ -117,33 +114,6 @@ class SkyworkR1VProcessingInfo(BaseInternVLProcessingInfo):

114 )

115

116

120 -class SkyworkR1VDummyInputsBuilder(BaseDummyInputsBuilder[SkyworkR1VProcessingInfo]):

121 - def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:

122 - num_images = mm_counts.get("image", 0)

123 -

124 - return "<image>" * num_images

125 -

126 - def get_dummy_mm_data(

127 - self,

128 - seq_len: int,

129 - mm_counts: Mapping[str, int],

130 - mm_options: Mapping[str, BaseDummyOptions],

131 - ) -> MultiModalDataDict:

132 - target_width, target_height = self.info.get_image_size_with_most_features()

133 - num_images = mm_counts.get("image", 0)

134 -

135 - image_overrides = mm_options.get("image")

136 -

137 - return {

138 - "image": self._get_dummy_images(

139 - width=target_width,

140 - height=target_height,

141 - num_images=num_images,

142 - overrides=image_overrides,

143 - )

144 - }

145 -

146 -

117 @MULTIMODAL_REGISTRY.register_processor(

118 BaseInternVLMultiModalProcessor,

119 info=SkyworkR1VProcessingInfo,

vllm/model_executor/models/utils.py

-12

@@ -27,7 +27,6 @@ from vllm.multimodal import NestedTensors

27 from vllm.sequence import IntermediateTensors

28 from vllm.utils.math_utils import cdiv

29 from vllm.utils.torch_utils import (

30 - async_tensor_h2d,

30 direct_register_custom_op,

31 )

32

@@ -673,17 +672,6 @@ def _merge_multimodal_embeddings(

672 return inputs_embeds

673

674

676 -def isin_list(

677 - elements: torch.Tensor,

678 - test_elements_list: list[int],

679 -) -> torch.Tensor:

680 - test_elements = async_tensor_h2d(

681 - test_elements_list, dtype=torch.int64, device=elements.device

682 - )

683 -

684 - return torch.isin(elements, test_elements)

685 -

686 -

675 class StageMissingLayer(nn.Module):

676 def __init__(self, stage_name: str, module: nn.Module | None = None) -> None:

677 super().__init__()

vllm/model_executor/models/vision.py

-37

@@ -580,40 +580,3 @@ def run_dp_sharded_mrope_vision_model(

580 "Found unassigned embeddings"

581 )

582 return out_embeddings

583 -

584 -

585 -def get_llm_pos_ids_for_vision(

586 - start_idx: int,

587 - vision_idx: int,

588 - spatial_merge_size: int,

589 - t_index: list[int],

590 - grid_hs: torch.Tensor,

591 - grid_ws: torch.Tensor,

592 -) -> torch.Tensor:

593 - llm_pos_ids_list = []

594 - llm_grid_h = grid_hs[vision_idx] // spatial_merge_size

595 - llm_grid_w = grid_ws[vision_idx] // spatial_merge_size

596 - h_index = (

597 - torch.arange(llm_grid_h)

598 - .view(1, -1, 1)

599 - .expand(len(t_index), -1, llm_grid_w)

600 - .flatten()

601 - )

602 - w_index = (

603 - torch.arange(llm_grid_w)

604 - .view(1, 1, -1)

605 - .expand(len(t_index), llm_grid_h, -1)

606 - .flatten()

607 - )

608 - t_index_tensor = (

609 - torch.Tensor(t_index)

610 - .to(llm_grid_h.device)

611 - .view(-1, 1)

612 - .expand(-1, llm_grid_h * llm_grid_w)

613 - .long()

614 - .flatten()

615 - )

616 - _llm_pos_ids = torch.stack([t_index_tensor, h_index, w_index])

617 - llm_pos_ids_list.append(_llm_pos_ids + start_idx)

618 - llm_pos_ids = torch.cat(llm_pos_ids_list, dim=1)

619 - return llm_pos_ids

vllm/models/minimax_m3/common/mm_preprocess.py

-9

@@ -23,7 +23,6 @@ from vllm.multimodal.inputs import (

23 from vllm.multimodal.parse import (

24 ImageSize,

25 MultiModalDataItems,

26 - MultiModalDataParser,

26 )

27 from vllm.multimodal.processing import (

28 BaseDummyInputsBuilder,

@@ -307,14 +306,6 @@ class MiniMaxM3VLDummyInputsBuilder(BaseDummyInputsBuilder[MiniMaxM3VLProcessing

306 class MiniMaxM3VLMultiModalProcessor(

307 BaseMultiModalProcessor[MiniMaxM3VLProcessingInfo]

308 ):

310 - def _get_data_parser(self) -> MultiModalDataParser:

311 - # Request video metadata (fps + sampled frame indices) so the HF

312 - # processor can emit per-frame ``]<]X.X seconds[>[`` timestamp markers,

313 - # matching MiniMax's reference video token stream. ``_get_prompt_updates``

314 - # reconstructs the same markers from the metadata to keep the prompt

315 - # replacement aligned with the processor output.

316 - return MultiModalDataParser(video_needs_metadata=True)

317 -

309 def _call_hf_processor(

310 self,

311 prompt: str,

vllm/profiler/utils.py

-9

@@ -12,15 +12,6 @@ from torch._C._profiler import _EventType, _ProfilerEvent, _TensorMetadata

12 #

13

14

15 -def trim_string_front(string: str, width: int) -> str:

16 - if len(string) > width:

17 - offset = len(string) - width + 3

18 - string = string[offset:]

19 - if len(string) > 3:

20 - string = "..." + string[3:]

21 - return string

22 -

23 -

15 def trim_string_back(string: str, width: int) -> str:

16 if len(string) > width:

17 offset = len(string) - width + 3

vllm/utils/cache.py

-4

@@ -182,10 +182,6 @@ class LRUCache(cachetools.LRUCache[_K, _V]):

182

183 self.popitem(remove_pinned=remove_pinned)

184

185 - def _remove_old_if_needed(self) -> None:

186 - while self.currsize > self.capacity:

187 - self.remove_oldest()

188 -

185 def popitem(self, remove_pinned: bool = False):

186 """Remove and return the `(key, value)` pair least recently used."""

187 if not remove_pinned:

vllm/utils/multi_stream_utils.py

-4

@@ -8,10 +8,6 @@ from typing import Any

8 import torch

9

10

11 -class AuxStreamType(Enum):

12 - Attention = 1

13 -

14 -

11 class EventType(Enum):

12 Main = 0

13 Attention = 1

vllm/v1/core/kv_cache_utils.py

-34

@@ -427,26 +427,6 @@ class FreeKVCacheBlockQueue:

427 curr_block = curr_block.next_free_block

428

429

430 -def need_extra_keys(request: Request) -> bool:

431 - """Check whether the blocks allocated to this request need extra hash keys.

432 -

433 - Args:

434 - request (Request): The request.

435 -

436 - Returns:

437 - bool: Whether blocks allocated to this request need extra hash keys.

438 - """

439 -

440 - # Multimodal requests need to include the MM hash.

441 - # LoRA requests need to include the LoRA name.

442 - # Request with provided cache salt need to include the salt.

443 - return (

444 - bool(request.mm_features)

445 - or (request.lora_request is not None)

446 - or (request.cache_salt is not None)

447 - )

448 -

449 -

430 def _gen_mm_extra_hash_keys(

431 request: Request, start_token_idx: int, end_token_idx: int, start_mm_idx: int

432 ) -> tuple[list[Any], int]:

@@ -1053,20 +1033,6 @@ def _get_kv_cache_groups_uniform_type(

1033 return [KVCacheGroupSpec(list(spec.kv_cache_specs.keys()), spec)]

1034

1035

1056 -def is_kv_cache_page_size_uniform(kv_cache_spec: dict[str, KVCacheSpec]) -> bool:

1057 - """

1058 - Whether all layers in the given KVCacheSpec have the same page size.

1059 - Args:

1060 - kv_cache_spec: The KVCacheSpec of each attention layer in the model

1061 -

1062 - Returns:

1063 - True if all layers have the same page size, False otherwise.

1064 - """

1065 -

1066 - page_sizes = {layer.page_size_bytes for layer in kv_cache_spec.values()}

1067 - return len(page_sizes) == 1

1068 -

1069 -

1036 def unify_kv_cache_spec_page_size(

1037 kv_cache_spec: dict[str, KVCacheSpec],

1038 ) -> dict[str, KVCacheSpec]:

[Refactor] Remove multiple dead codes (#50285)