@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6c7f881541
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Model] Add JambaForSequenceClassification model (#10860)

Signed-off-by: Yehoshua Cohen <yehoshuaco@ai21.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: Yehoshua Cohen <yehoshuaco@ai21.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>

Yehoshua Cohen committed Dec 19, 2024 at 16:48 UTC 6c7f8815416f1968a8c1578f52a7e5b63f9310ed
5 files changed +48 -2
docs/source/models/supported_models.rst
+5
@@ -476,6 +476,11 @@ Classification (``--task classify``)
476 - Example HF Models
477 - :ref:`LoRA <lora>`
478 - :ref:`PP <distributed_serving>`
479 + * - :code:`JambaForSequenceClassification`
480 + - Jamba
481 + - :code:`ai21labs/Jamba-tiny-reward-dev`, etc.
482 + - ✅︎
483 + - ✅︎
484 * - :code:`Qwen2ForSequenceClassification`
485 - Qwen2-based
486 - :code:`jason9693/Qwen2.5-1.5B-apeach`, etc.
tests/models/registry.py
+1
@@ -138,6 +138,7 @@ _EMBEDDING_EXAMPLE_MODELS = {
138 "BertModel": _HfExamplesInfo("BAAI/bge-base-en-v1.5"),
139 "Gemma2Model": _HfExamplesInfo("BAAI/bge-multilingual-gemma2"),
140 "GritLM": _HfExamplesInfo("parasail-ai/GritLM-7B-vllm"),
141 + "JambaForSequenceClassification": _HfExamplesInfo("ai21labs/Jamba-tiny-reward-dev"), # noqa: E501
142 "LlamaModel": _HfExamplesInfo("llama", is_available_online=False),
143 "MistralModel": _HfExamplesInfo("intfloat/e5-mistral-7b-instruct"),
144 "Qwen2Model": _HfExamplesInfo("ssmits/Qwen2-7B-Instruct-embed-base"),
vllm/model_executor/models/jamba.py
+35 -1
@@ -17,6 +17,7 @@ from vllm.model_executor.layers.linear import (QKVParallelLinear,
17 RowParallelLinear)
18 from vllm.model_executor.layers.logits_processor import LogitsProcessor
19 from vllm.model_executor.layers.mamba.mamba_mixer import MambaMixer
20 +from vllm.model_executor.layers.pooler import Pooler, PoolingType
21 from vllm.model_executor.layers.quantization import QuantizationConfig
22 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
23 from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -24,8 +25,9 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
25 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
26 from vllm.model_executor.models.mamba_cache import (MambaCacheManager,
27 MambaCacheParams)
28 +from vllm.model_executor.pooling_metadata import PoolingMetadata
29 from vllm.model_executor.sampling_metadata import SamplingMetadata
28 -from vllm.sequence import IntermediateTensors
30 +from vllm.sequence import IntermediateTensors, PoolerOutput
31 from vllm.utils import LayerBlockType
32
33 from .interfaces import HasInnerState, IsHybrid, SupportsLoRA, SupportsPP
@@ -593,3 +595,35 @@ def _is_moe_layer(name: str):
595 "experts",
596 "router",
597 ]])
598 +
599 +
600 +class JambaForSequenceClassification(JambaForCausalLM):
601 +
602 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
603 + super().__init__(vllm_config=vllm_config, prefix=prefix)
604 + config = vllm_config.model_config.hf_config
605 + num_labels: int = config.num_labels
606 + score_bias: bool = getattr(config, 'score_bias', False)
607 + self.score = nn.Linear(config.hidden_size, num_labels, bias=score_bias)
608 +
609 + pooler_config = vllm_config.model_config.pooler_config
610 + self._pooler = Pooler.from_config_with_defaults(
611 + pooler_config,
612 + pooling_type=PoolingType.LAST,
613 + normalize=False,
614 + softmax=False)
615 +
616 + def pooler(
617 + self,
618 + hidden_states: torch.Tensor,
619 + pooling_metadata: PoolingMetadata,
620 + ) -> Optional[PoolerOutput]:
621 + hidden_states = hidden_states.float()
622 + logits = self.score(hidden_states)
623 + return self._pooler(logits, pooling_metadata)
624 +
625 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
626 + # TODO: The reward weights themselves have float32 accuracy data, we
627 + # would like to load them in fp32 to get that extra precision.
628 + super().load_weights(weights)
629 + self.score = self.score.float()
vllm/model_executor/models/registry.py
+1
@@ -113,6 +113,7 @@ _EMBEDDING_MODELS = {
113 "Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
114 "GlmForCausalLM": ("glm", "GlmForCausalLM"),
115 "GritLM": ("gritlm", "GritLM"),
116 + "JambaForSequenceClassification": ("jamba", "JambaForSequenceClassification"), # noqa: E501
117 "LlamaModel": ("llama", "LlamaForCausalLM"),
118 **{
119 # Multiple models share the same architecture, so we include them all
vllm/worker/pooling_model_runner.py
+6 -1
@@ -91,6 +91,10 @@ class PoolingModelRunner(
91 ]
92
93 multi_modal_kwargs = model_input.multi_modal_kwargs or {}
94 + seqlen_agnostic_kwargs = {
95 + "finished_requests_ids": model_input.finished_requests_ids,
96 + "request_ids_to_seq_ids": model_input.request_ids_to_seq_ids,
97 + } if self.has_inner_state else {}
98 if (self.observability_config is not None
99 and self.observability_config.collect_model_forward_time):
100 model_forward_start = torch.cuda.Event(enable_timing=True)
@@ -110,7 +114,8 @@ class PoolingModelRunner(
114 intermediate_tensors=intermediate_tensors,
115 **MultiModalKwargs.as_kwargs(multi_modal_kwargs,
116 device=self.device),
113 - **cross_enc_kwargs)
117 + **cross_enc_kwargs,
118 + **seqlen_agnostic_kwargs)
119
120 if (self.observability_config is not None
121 and self.observability_config.collect_model_forward_time):