6c85da3a18 · tduncan/Nvidia-Optimization-Stack-VLLM

vllm/config.py

+5

@@ -1039,6 +1039,11 @@ class ModelConfig:

1039 def runner_type(self) -> RunnerType:

1040 return _TASK_RUNNER[self.task]

1041

1042 + @property

1043 + def is_v1_compatible(self) -> bool:

1044 + architectures = getattr(self.hf_config, "architectures", [])

1045 + return ModelRegistry.is_v1_compatible(architectures)

1046 +

1047

1048 class CacheConfig:

1049 """Configuration for the KV cache.

vllm/model_executor/models/__init__.py

+5 -2

@@ -1,8 +1,9 @@

1 # SPDX-License-Identifier: Apache-2.0

2

3 from .interfaces import (HasInnerState, SupportsLoRA, SupportsMultiModal,

4 - SupportsPP, has_inner_state, supports_lora,

5 - supports_multimodal, supports_pp)

4 + SupportsPP, SupportsV0Only, has_inner_state,

5 + supports_lora, supports_multimodal, supports_pp,

6 + supports_v0_only)

7 from .interfaces_base import (VllmModelForPooling, VllmModelForTextGeneration,

8 is_pooling_model, is_text_generation_model)

9 from .registry import ModelRegistry

@@ -21,4 +22,6 @@ __all__ = [

22 "supports_multimodal",

23 "SupportsPP",

24 "supports_pp",

25 + "SupportsV0Only",

26 + "supports_v0_only",

27 ]

vllm/model_executor/models/bamba.py

+3 -2

@@ -32,7 +32,8 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata

32 from vllm.sequence import IntermediateTensors

33 from vllm.utils import LayerBlockType

34

35 -from .interfaces import HasInnerState, IsHybrid, SupportsLoRA, SupportsPP

35 +from .interfaces import (HasInnerState, IsHybrid, SupportsLoRA, SupportsPP,

36 + SupportsV0Only)

37 from .utils import (is_pp_missing_parameter,

38 make_empty_intermediate_tensors_factory, make_layers,

39 maybe_prefix)

@@ -366,7 +367,7 @@ class BambaModel(nn.Module):

367

368

369 class BambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP,

369 - IsHybrid):

370 + IsHybrid, SupportsV0Only):

371 packed_modules_mapping = {

372 "qkv_proj": [

373 "q_proj",

vllm/model_executor/models/bart.py

+2 -1

@@ -43,6 +43,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader

43 from vllm.model_executor.sampling_metadata import SamplingMetadata

44 from vllm.sequence import IntermediateTensors

45

46 +from .interfaces import SupportsV0Only

47 from .utils import maybe_prefix

48

49 logger = logging.get_logger(__name__)

@@ -776,7 +777,7 @@ class BartModel(nn.Module):

777 return decoder_outputs

778

779

779 -class BartForConditionalGeneration(nn.Module):

780 +class BartForConditionalGeneration(nn.Module, SupportsV0Only):

781 base_model_prefix = "model"

782

783 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

vllm/model_executor/models/bert.py

+2 -2

@@ -26,7 +26,7 @@ from vllm.sequence import IntermediateTensors, PoolerOutput

26 from vllm.transformers_utils.config import (

27 get_cross_encoder_activation_function)

28

29 -from .interfaces import SupportsCrossEncoding

29 +from .interfaces import SupportsCrossEncoding, SupportsV0Only

30 from .utils import WeightsMapper, maybe_prefix

31

32

@@ -385,7 +385,7 @@ class BertModel(nn.Module):

385 return loaded_params

386

387

388 -class BertEmbeddingModel(nn.Module):

388 +class BertEmbeddingModel(nn.Module, SupportsV0Only):

389 """A model that uses Bert to provide embedding functionalities.

390

391 This class encapsulates the BertModel and provides an interface for

vllm/model_executor/models/florence2.py

+2 -2

@@ -29,7 +29,7 @@ from vllm.multimodal.processing import (BaseProcessingInfo,

29 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

30 from vllm.sequence import IntermediateTensors

31

32 -from .interfaces import SupportsMultiModal

32 +from .interfaces import SupportsMultiModal, SupportsV0Only

33 from .utils import AutoWeightsLoader, flatten_bn, merge_multimodal_embeddings

34

35

@@ -651,7 +651,7 @@ class Florence2LanguageModel(nn.Module):

651 return decoder_outputs

652

653

654 -class Florence2LanguageForConditionalGeneration(nn.Module):

654 +class Florence2LanguageForConditionalGeneration(nn.Module, SupportsV0Only):

655

656 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

657 super().__init__()

vllm/model_executor/models/gritlm.py

+3 -1

@@ -19,6 +19,8 @@ from vllm.sequence import (IntermediateTensors, PoolerOutput,

19 PoolingSequenceGroupOutput)

20 from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config

21

22 +from .interfaces import SupportsV0Only

23 +

24 logger = init_logger(__name__)

25

26

@@ -177,7 +179,7 @@ class GritLMPooler(nn.Module):

179 return PoolerOutput(outputs=pooled_outputs)

180

181

180 -class GritLM(LlamaForCausalLM):

182 +class GritLM(LlamaForCausalLM, SupportsV0Only):

183 """This class implements the embedding model for parasail-ai/GritLM-7B-vllm.

184

185 The class inherits from LlamaForCausalLM and provides a custom pooling

vllm/model_executor/models/interfaces.py

+26

@@ -498,3 +498,29 @@ def supports_transcription(

498 return isinstance(model, SupportsTranscription)

499

500 return isinstance(model, SupportsTranscription)

501 +

502 +

503 +@runtime_checkable

504 +class SupportsV0Only(Protocol):

505 + """Models with this interface are not compatible with V1 vLLM."""

506 +

507 + supports_v0_only: ClassVar[Literal[True]] = True

508 +

509 +

510 +@overload

511 +def supports_v0_only(model: Type[object]) -> TypeIs[Type[SupportsV0Only]]:

512 + ...

513 +

514 +

515 +@overload

516 +def supports_v0_only(model: object) -> TypeIs[SupportsV0Only]:

517 + ...

518 +

519 +

520 +def supports_v0_only(

521 + model: Union[Type[object], object],

522 +) -> Union[TypeIs[Type[SupportsV0Only]], TypeIs[SupportsV0Only]]:

523 + if isinstance(model, type):

524 + return isinstance(model, SupportsV0Only)

525 +

526 + return isinstance(model, SupportsV0Only)

vllm/model_executor/models/jamba.py

+3 -2

@@ -30,7 +30,8 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata

30 from vllm.sequence import IntermediateTensors, PoolerOutput

31 from vllm.utils import LayerBlockType

32

33 -from .interfaces import HasInnerState, IsHybrid, SupportsLoRA, SupportsPP

33 +from .interfaces import (HasInnerState, IsHybrid, SupportsLoRA, SupportsPP,

34 + SupportsV0Only)

35 from .utils import (is_pp_missing_parameter,

36 make_empty_intermediate_tensors_factory, make_layers,

37 maybe_prefix)

@@ -353,7 +354,7 @@ class JambaModel(nn.Module):

354

355

356 class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP,

356 - IsHybrid):

357 + IsHybrid, SupportsV0Only):

358 packed_modules_mapping = {

359 "qkv_proj": [

360 "q_proj",

vllm/model_executor/models/mamba.py

+4 -2

@@ -19,7 +19,8 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (

19 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

20 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

21 from vllm.model_executor.models.interfaces import (HasInnerState,

22 - IsAttentionFree, SupportsPP)

22 + IsAttentionFree, SupportsPP,

23 + SupportsV0Only)

24 from vllm.model_executor.models.mamba_cache import (MambaCacheManager,

25 MambaCacheParams)

26 from vllm.model_executor.sampling_metadata import SamplingMetadata

@@ -155,7 +156,8 @@ class MambaModel(nn.Module):

156 return hidden_states

157

158

158 -class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree, SupportsPP):

159 +class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree, SupportsPP,

160 + SupportsV0Only):

161

162 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

163 config = vllm_config.model_config.hf_config

vllm/model_executor/models/mamba2.py

+4 -2

@@ -22,7 +22,8 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (

22 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)

23 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

24 from vllm.model_executor.models.interfaces import (HasInnerState,

25 - IsAttentionFree)

25 + IsAttentionFree,

26 + SupportsV0Only)

27 from vllm.model_executor.models.mamba_cache import (MambaCacheManager,

28 MambaCacheParams)

29 from vllm.model_executor.sampling_metadata import SamplingMetadata

@@ -174,7 +175,8 @@ class Mamba2Model(nn.Module):

175 return hidden_states

176

177

177 -class Mamba2ForCausalLM(nn.Module, HasInnerState, IsAttentionFree):

178 +class Mamba2ForCausalLM(nn.Module, HasInnerState, IsAttentionFree,

179 + SupportsV0Only):

180

181 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):

182 config = vllm_config.model_config.hf_config

vllm/model_executor/models/minicpmv.py

+4 -2

@@ -63,7 +63,8 @@ from vllm.platforms import current_platform

63 from vllm.sequence import IntermediateTensors

64

65 from .idefics2_vision_model import Idefics2VisionTransformer

66 -from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP

66 +from .interfaces import (SupportsLoRA, SupportsMultiModal, SupportsPP,

67 + SupportsV0Only)

68 from .utils import AutoWeightsLoader, maybe_prefix

69

70 CPU_DEVICE = torch.device("cpu")

@@ -804,7 +805,8 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

805 return result

806

807

807 -class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):

808 +class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP,

809 + SupportsV0Only):

810 """

811 The abstract class of MiniCPMV can only be inherited, but cannot be

812 instantiated.

vllm/model_executor/models/mllama.py

+3 -2

@@ -63,7 +63,7 @@ from vllm.multimodal.processing import (BaseProcessingInfo,

63 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

64

65 from .clip import CLIPMLP

66 -from .interfaces import SupportsMultiModal

66 +from .interfaces import SupportsMultiModal, SupportsV0Only

67 from .llama import LlamaDecoderLayer, LlamaMLP

68 from .utils import maybe_prefix

69

@@ -1128,7 +1128,8 @@ class MllamaForCausalLM(nn.Module):

1128 @MULTIMODAL_REGISTRY.register_processor(MllamaMultiModalProcessor,

1129 info=MllamaProcessingInfo,

1130 dummy_inputs=MllamaDummyInputsBuilder)

1131 -class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):

1131 +class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal,

1132 + SupportsV0Only):

1133 packed_modules_mapping = {

1134 "qkv_proj": ["q_proj", "k_proj", "v_proj"],

1135 "gate_up_proj": ["gate_proj", "up_proj"]

vllm/model_executor/models/paligemma.py

+2 -2

@@ -18,7 +18,7 @@ from vllm.multimodal.inputs import NestedTensors

18 from vllm.sequence import IntermediateTensors

19 from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config

20

21 -from .interfaces import SupportsMultiModal, SupportsPP

21 +from .interfaces import SupportsMultiModal, SupportsPP, SupportsV0Only

22 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,

23 dummy_seq_data_for_siglip, get_max_siglip_image_tokens)

24 from .utils import (AutoWeightsLoader, init_vllm_registered_model,

@@ -136,7 +136,7 @@ class PaliGemmaMultiModalProjector(nn.Module):

136 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_paligemma)

137 @INPUT_REGISTRY.register_input_processor(input_processor_for_paligemma)

138 class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,

139 - SupportsPP):

139 + SupportsPP, SupportsV0Only):

140 packed_modules_mapping = {

141 "qkv_proj": [

142 "q_proj",

vllm/model_executor/models/prithvi_geospatial_mae.py

+4 -2

@@ -25,7 +25,8 @@ from transformers import BatchFeature

25 from vllm.config import VllmConfig

26 from vllm.model_executor.model_loader.weight_utils import default_weight_loader

27 from vllm.model_executor.models.interfaces import (IsAttentionFree,

28 - SupportsMultiModal)

28 + SupportsMultiModal,

29 + SupportsV0Only)

30 from vllm.model_executor.models.utils import AutoWeightsLoader

31 from vllm.model_executor.pooling_metadata import PoolingMetadata

32 from vllm.multimodal import MULTIMODAL_REGISTRY

@@ -111,7 +112,8 @@ class PrithviGeoSpatialMAEMultiModalProcessor(BaseMultiModalProcessor):

112 PrithviGeoSpatialMAEMultiModalProcessor,

113 info=PrithviGeoSpatialMAEProcessingInfo,

114 dummy_inputs=PrithviGeoSpatialMAEInputBuilder)

114 -class PrithviGeoSpatialMAE(nn.Module, IsAttentionFree, SupportsMultiModal):

115 +class PrithviGeoSpatialMAE(nn.Module, IsAttentionFree, SupportsMultiModal,

116 + SupportsV0Only):

117 """ Prithvi Masked Autoencoder"""

118

119 def _instantiate_model(self, config: dict) -> Optional[nn.Module]:

vllm/model_executor/models/qwen2_rm.py

+3 -2

@@ -17,7 +17,7 @@ from vllm.model_executor.layers.pooler import Pooler, PoolingType, SimplePooler

17 from vllm.model_executor.pooling_metadata import PoolingMetadata

18 from vllm.sequence import IntermediateTensors, PoolerOutput

19

20 -from .interfaces import SupportsLoRA, SupportsPP

20 +from .interfaces import SupportsLoRA, SupportsPP, SupportsV0Only

21 from .qwen2 import Qwen2Model

22 from .utils import AutoWeightsLoader, maybe_prefix

23

@@ -33,7 +33,8 @@ class ReLU(nn.Module):

33 return self.activation(input)

34

35

36 -class Qwen2RewardBaseModel(nn.Module, SupportsLoRA, SupportsPP):

36 +class Qwen2RewardBaseModel(nn.Module, SupportsLoRA, SupportsPP,

37 + SupportsV0Only):

38 packed_modules_mapping = {

39 "qkv_proj": [

40 "q_proj",

vllm/model_executor/models/registry.py

+12 -2

@@ -22,7 +22,7 @@ from vllm.logger import init_logger

22

23 from .interfaces import (has_inner_state, is_attention_free, is_hybrid,

24 supports_cross_encoding, supports_multimodal,

25 - supports_pp, supports_transcription)

25 + supports_pp, supports_transcription, supports_v0_only)

26 from .interfaces_base import is_text_generation_model

27

28 logger = init_logger(__name__)

@@ -228,6 +228,7 @@ class _ModelInfo:

228 is_attention_free: bool

229 is_hybrid: bool

230 supports_transcription: bool

231 + supports_v0_only: bool

232

233 @staticmethod

234 def from_model_cls(model: Type[nn.Module]) -> "_ModelInfo":

@@ -241,7 +242,9 @@ class _ModelInfo:

242 has_inner_state=has_inner_state(model),

243 is_attention_free=is_attention_free(model),

244 is_hybrid=is_hybrid(model),

244 - supports_transcription=supports_transcription(model))

245 + supports_transcription=supports_transcription(model),

246 + supports_v0_only=supports_v0_only(model),

247 + )

248

249

250 class _BaseRegisteredModel(ABC):

@@ -504,6 +507,13 @@ class _ModelRegistry:

507 model_cls, _ = self.inspect_model_cls(architectures)

508 return model_cls.supports_transcription

509

510 + def is_v1_compatible(

511 + self,

512 + architectures: Union[str, List[str]],

513 + ) -> bool:

514 + model_cls, _ = self.inspect_model_cls(architectures)

515 + return not model_cls.supports_v0_only

516 +

517

518 ModelRegistry = _ModelRegistry({

519 model_arch:

vllm/model_executor/models/roberta.py

+3 -2

@@ -19,7 +19,7 @@ from vllm.sequence import IntermediateTensors, PoolerOutput

19 from vllm.transformers_utils.config import (

20 get_cross_encoder_activation_function)

21

22 -from .interfaces import SupportsCrossEncoding

22 +from .interfaces import SupportsCrossEncoding, SupportsV0Only

23

24

25 def roberta_task_weights_filter(

@@ -191,7 +191,8 @@ class RobertaEmbeddingModel(BertEmbeddingModel):

191 assert len(loaded), "Unable to load RobertaEmbeddingModel"

192

193

194 -class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding):

194 +class RobertaForSequenceClassification(nn.Module, SupportsCrossEncoding,

195 + SupportsV0Only):

196 """A model that uses Roberta to provide embedding functionalities.

197

198 This class encapsulates the BertModel and provides an interface for

vllm/model_executor/models/whisper.py

+3 -2

@@ -34,7 +34,8 @@ from vllm.multimodal.processing import (BaseProcessingInfo,

34 PromptReplacement, PromptUpdate)

35 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

36

37 -from .interfaces import SupportsMultiModal, SupportsTranscription

37 +from .interfaces import (SupportsMultiModal, SupportsTranscription,

38 + SupportsV0Only)

39 from .utils import (AutoWeightsLoader, WeightsMapper, cast_overflow_tensors,

40 make_layers)

41

@@ -643,7 +644,7 @@ class WhisperMultiModalProcessor(

644 info=WhisperProcessingInfo,

645 dummy_inputs=WhisperDummyInputsBuilder)

646 class WhisperForConditionalGeneration(nn.Module, SupportsTranscription,

646 - SupportsMultiModal):

647 + SupportsMultiModal, SupportsV0Only):

648 packed_modules_mapping = {

649 "self_attn.qkv_proj": [

650 "self_attn.q_proj",

[V1]`SupportsV0Only` protocol for model definitions (#13959)