40b8f55358 · tduncan/Nvidia-Optimization-Stack-VLLM

mkdocs.yaml

+2 -2

@@ -63,8 +63,9 @@ plugins:

63 - git-revision-date-localized:

64 # exclude autogenerated files

65 exclude:

66 - - argparse/*

66 + - api/*

67 - examples/*

68 + - generated/*

69 - minify:

70 minify_html: true

71 minify_js: true

@@ -92,7 +93,6 @@ plugins:

93 - "!.*_pb2_grpc" # Exclude auto-generated gRPC stubs

94 summary:

95 modules: true

95 - show_if_no_docstring: true

96 show_signature_annotations: true

97 separate_signature: true

98 show_overloads: true

vllm/config/model.py

+1

@@ -1557,6 +1557,7 @@ class ModelConfig:

1557

1558 @property

1559 def attn_type(self) -> AttnTypeStr:

1560 + """Determine the attention type based on model configuration."""

1561 if self.pooler_config is not None:

1562 seq_pooling_type = self._model_info.default_seq_pooling_type

1563 if seq_pooling_type == "CLS":

vllm/engine/async_llm_engine.py

+1

@@ -4,3 +4,4 @@

4 from vllm.v1.engine.async_llm import AsyncLLM

5

6 AsyncLLMEngine = AsyncLLM # type: ignore

7 +"""The `AsyncLLMEngine` class is an alias of [vllm.v1.engine.async_llm.AsyncLLM][]."""

vllm/engine/llm_engine.py

+1

@@ -4,3 +4,4 @@

4 from vllm.v1.engine.llm_engine import LLMEngine as V1LLMEngine

5

6 LLMEngine = V1LLMEngine # type: ignore

7 +"""The `LLMEngine` class is an alias of [vllm.v1.engine.llm_engine.LLMEngine][]."""

vllm/inputs/data.py

+1

@@ -298,6 +298,7 @@ which can be passed to

298

299

300 SingletonInputs: TypeAlias = DecoderOnlyInputs | MultiModalEncDecInputs

301 +"""The inputs for a single encoder/decoder prompt."""

302

303

304 @dataclass

vllm/model_executor/layers/fused_moe/cpu_fused_moe.py

+2

@@ -206,6 +206,8 @@ class SGLFusedMOE:

206

207

208 class CPUFusedMOE:

209 + """CPU-based fused MoE implementation."""

210 +

211 def __init__(self, layer: torch.nn.Module) -> None:

212 use_grouped_gemm, isa = self.check_grouped_gemm(layer)

213 self.isa = isa

vllm/model_executor/layers/fused_moe/cutlass_moe.py

+6

@@ -376,6 +376,8 @@ class CutlassExpertsFp8Base(mk.FusedMoEPermuteExpertsUnpermute):

376

377

378 class CutlassExpertsFp8(CutlassExpertsFp8Base):

379 + """CUTLASS FP8 fused MoE expert implementation."""

380 +

381 @staticmethod

382 def activation_format() -> mk.FusedMoEActivationFormat:

383 return mk.FusedMoEActivationFormat.Standard

@@ -423,6 +425,8 @@ class CutlassExpertsFp8(CutlassExpertsFp8Base):

425

426

427 class CutlassBatchedExpertsFp8(CutlassExpertsFp8Base):

428 + """Batched CUTLASS FP8 fused MoE expert implementation."""

429 +

430 @staticmethod

431 def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:

432 # BATCHED activation format works with EP because

@@ -651,6 +655,8 @@ def run_cutlass_moe_fp4(

655

656

657 class CutlassExpertsFp4(mk.FusedMoEPermuteExpertsUnpermute):

658 + """CUTLASS FP4 fused MoE expert implementation."""

659 +

660 @property

661 def expects_unquantized_inputs(self) -> bool:

662 return True

vllm/model_executor/layers/fused_moe/deep_gemm_moe.py

+2

@@ -113,6 +113,8 @@ def _valid_deep_gemm(

113

114

115 class DeepGemmExperts(mk.FusedMoEPermuteExpertsUnpermute):

116 + """DeepGemm-based fused MoE expert implementation."""

117 +

118 def __init__(self, moe_config: FusedMoEConfig, quant_config: FusedMoEQuantConfig):

119 super().__init__(moe_config=moe_config, quant_config=quant_config)

120 assert quant_config.block_shape == get_mk_alignment_for_contiguous_layout()

vllm/model_executor/layers/fused_moe/fused_marlin_moe.py

+4

@@ -637,6 +637,8 @@ class MarlinExpertsBase(mk.FusedMoEPermuteExpertsUnpermute):

637

638

639 class MarlinExperts(MarlinExpertsBase):

640 + """Marlin-based fused MoE expert implementation."""

641 +

642 def supports_expert_map(self) -> bool:

643 return True

644

@@ -738,6 +740,8 @@ class MarlinExperts(MarlinExpertsBase):

740

741

742 class BatchedMarlinExperts(MarlinExpertsBase):

743 + """Batched Marlin-based fused MoE expert implementation."""

744 +

745 def __init__(

746 self,

747 moe_config: FusedMoEConfig,

vllm/model_executor/layers/fused_moe/fused_moe.py

+3

@@ -1527,6 +1527,7 @@ def fused_experts(

1527 expert_map: torch.Tensor | None = None,

1528 quant_config: FusedMoEQuantConfig | None = None,

1529 ) -> torch.Tensor:

1530 + """Run fused MoE expert computation using Triton kernels."""

1531 if quant_config is None:

1532 quant_config = FUSED_MOE_UNQUANTIZED_CONFIG

1533

@@ -1879,6 +1880,8 @@ def fused_experts_impl(

1880

1881

1882 class TritonExperts(mk.FusedMoEPermuteExpertsUnpermute):

1883 + """Triton-based fused MoE expert implementation."""

1884 +

1885 def __init__(

1886 self,

1887 moe_config: FusedMoEConfig,

vllm/model_executor/layers/fused_moe/gpt_oss_triton_kernels_moe.py

+3

@@ -221,6 +221,7 @@ def triton_kernel_fused_experts(

221 intermediate_cache: torch.Tensor | None = None,

222 a1q_scale: torch.Tensor | None = None,

223 ) -> torch.Tensor:

224 + """Triton implementation of fused expert computation using OAI kernels."""

225 if quant_config is None:

226 quant_config = FUSED_MOE_UNQUANTIZED_CONFIG

227

@@ -444,6 +445,8 @@ class BaseOAITritonExperts(mk.FusedMoEPermuteExpertsUnpermute):

445

446

447 class OAITritonExperts(BaseOAITritonExperts):

448 + """OAI Triton-based fused MoE expert implementation."""

449 +

450 @staticmethod

451 def activation_format() -> mk.FusedMoEActivationFormat:

452 return mk.FusedMoEActivationFormat.Standard

vllm/model_executor/layers/fused_moe/pplx_prepare_finalize.py

+2

@@ -63,6 +63,8 @@ def pplx_hidden_dim_scale_bytes(

63

64

65 class PplxPrepareAndFinalize(mk.FusedMoEPrepareAndFinalize):

66 + """PPLX-based prepare and finalize for expert parallelism."""

67 +

68 def __init__(

69 self,

70 a2a: pplx.AllToAll,

vllm/model_executor/layers/fused_moe/prepare_finalize.py

+2

@@ -131,6 +131,8 @@ class MoEPrepareAndFinalizeNaiveEP(mk.FusedMoEPrepareAndFinalize):

131

132

133 class MoEPrepareAndFinalizeNoEP(mk.FusedMoEPrepareAndFinalize):

134 + """MoE prepare and finalize without expert parallelism."""

135 +

136 @property

137 def activation_format(self) -> mk.FusedMoEActivationFormat:

138 return mk.FusedMoEActivationFormat.Standard

vllm/model_executor/layers/fused_moe/rocm_aiter_fused_moe.py

+1

@@ -192,6 +192,7 @@ def rocm_aiter_fused_experts(

192 num_local_tokens: torch.Tensor | None = None,

193 output_dtype: torch.dtype | None = None,

194 ) -> torch.Tensor:

195 + """ROCm AITER fused MoE expert computation."""

196 if quant_config is None:

197 quant_config = FUSED_MOE_UNQUANTIZED_CONFIG

198

vllm/model_executor/layers/fused_moe/trtllm_moe.py

+2

@@ -18,6 +18,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (

18

19

20 class TrtLlmGenExperts(mk.FusedMoEPermuteExpertsUnpermute):

21 + """TensorRT-LLM-based fused MoE expert implementation."""

22 +

23 def __init__(

24 self,

25 moe_config: FusedMoEConfig,

vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py

+2

@@ -680,6 +680,8 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):

680

681

682 class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):

683 + """W8A8 FP8 MoE quantization using compressed tensors."""

684 +

685 def __init__(

686 self,

687 weight_quant: QuantizationArgs,

vllm/model_executor/layers/quantization/mxfp4.py

+2

@@ -235,6 +235,8 @@ class Mxfp4Config(QuantizationConfig):

235

236

237 class Mxfp4MoEMethod(FusedMoEMethodBase):

238 + """MXFP4 MoE quantization method."""

239 +

240 def __init__(self, moe: FusedMoEConfig):

241 super().__init__(moe)

242 self.weight_dtype = "mxfp4"

vllm/model_executor/models/blip2.py

+1

@@ -73,6 +73,7 @@ class Blip2ImageEmbeddingInputs(TensorSchema):

73

74

75 Blip2ImageInputs: TypeAlias = Blip2ImagePixelInputs | Blip2ImageEmbeddingInputs

76 +"""Alias for supported BLIP-2 image input types."""

77

78

79 class Blip2QFormerMultiHeadAttention(nn.Module):

vllm/model_executor/models/llava.py

+1

@@ -121,6 +121,7 @@ class LlavaImageEmbeddingInputs(TensorSchema):

121 LlavaImageInputs: TypeAlias = (

122 LlavaImagePixelInputs | PixtralHFImagePixelInputs | LlavaImageEmbeddingInputs

123 )

124 +"""Alias for supported LLaVA image input types."""

125

126

127 class LlavaMultiModalProjector(nn.Module):

vllm/model_executor/models/llava_next.py

+2

@@ -78,6 +78,7 @@ class LlavaNextImageEmbeddingInputs(TensorSchema):

78 LlavaNextImageInputs: TypeAlias = (

79 LlavaNextImagePixelInputs | LlavaNextImageEmbeddingInputs

80 )

81 +"""Alias for supported LLaVA-NeXT image input types."""

82

83

84 class LlavaNextLikeConfig(LlavaLikeConfig, Protocol):

@@ -106,6 +107,7 @@ class LlavaNextProcessingInfo(BaseLlavaProcessingInfo):

107 image_width: int,

108 image_height: int,

109 ) -> int:

110 + """Get the number of image tokens for the given image dimensions."""

111 hf_config = self.get_hf_config()

112 vision_encoder_info = self.get_vision_encoder_info()

113

vllm/multimodal/processing/processor.py

+2

@@ -1110,6 +1110,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1110 self,

1111 mm_items: MultiModalDataItems,

1112 ) -> tuple[Mapping[str, object], Mapping[str, object]]:

1113 + """Extract processor and passthrough data from multi-modal items."""

1114 processor_data = dict[str, object]()

1115 passthrough_data = dict[str, object]()

1116

@@ -1616,6 +1617,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1617 token_ids: list[int],

1618 mm_prompt_updates: MultiModalPromptUpdates,

1619 ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:

1620 + """Apply multi-modal prompt updates to token IDs."""

1621 tokenizer = self.info.get_tokenizer()

1622

1623 new_token_ids, match_result = self._apply_token_matches(

vllm/platforms/interface.py

+2

@@ -35,6 +35,8 @@ def in_wsl() -> bool:

35

36

37 class PlatformEnum(enum.Enum):

38 + """Enumeration of supported hardware platforms."""

39 +

40 CUDA = enum.auto()

41 ROCM = enum.auto()

42 TPU = enum.auto()

vllm/plugins/__init__.py

+1

@@ -26,6 +26,7 @@ plugins_loaded = False

26

27

28 def load_plugins_by_group(group: str) -> dict[str, Callable[[], Any]]:

29 + """Load plugins registered under the given entry point group."""

30 from importlib.metadata import entry_points

31

32 allowed_plugins = envs.VLLM_PLUGINS

vllm/plugins/io_processors/interface.py

+2

@@ -16,6 +16,8 @@ IOProcessorOutput = TypeVar("IOProcessorOutput")

16

17

18 class IOProcessor(ABC, Generic[IOProcessorInput, IOProcessorOutput]):

19 + """Abstract interface for pre/post-processing of engine I/O."""

20 +

21 def __init__(self, vllm_config: VllmConfig):

22 super().__init__()

23

vllm/v1/engine/async_llm.py

+2

@@ -69,6 +69,8 @@ class InputStreamError(Exception):

69

70

71 class AsyncLLM(EngineClient):

72 + """An asynchronous wrapper for the vLLM engine."""

73 +

74 def __init__(

75 self,

76 vllm_config: VllmConfig,

[Docs] Reduce time spent generating API docs (#34255)