A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Model] Add classification Task with Qwen2ForSequenceClassification (#9704)
Signed-off-by: Kevin-Yang <ykcha9@gmail.com> Co-authored-by: Kevin-Yang <ykcha9@gmail.com>
kakao-kevin-us committed
Oct 27, 2024 at 02:53 UTC
6650e6a930dbdf1cd4def9b58e952376400ccfcf
6 files changed
+211
-1
docs/source/models/supported_models.rst
+22
@@ -361,6 +361,28 @@ Reward Modeling
361
.. note::
362
As an interim measure, these models are supported via Embeddings API. See `this RFC <https://github.com/vllm-project/vllm/issues/8967>`_ for upcoming changes.
363
364
+Classification
365
+---------------
366
+
367
+.. list-table::
368
+ :widths: 25 25 50 5 5
369
+ :header-rows: 1
370
+
371
+ * - Architecture
372
+ - Models
373
+ - Example HF Models
374
+ - :ref:`LoRA <lora>`
375
+ - :ref:`PP <distributed_serving>`
376
+ * - :code:`Qwen2ForSequenceClassification`
377
+ - Qwen2-based
378
+ - :code:`jason9693/Qwen2.5-1.5B-apeach`, etc.
379
+ -
380
+ - ✅︎
381
+
382
+.. note::
383
+ As an interim measure, these models are supported via Embeddings API. It will be supported via Classification API in the future (no reference APIs exist now).
384
+
385
+
386
Multimodal Language Models
387
^^^^^^^^^^^^^^^^^^^^^^^^^^
388
tests/conftest.py
+19
@@ -343,6 +343,17 @@ class HfRunner:
343
344
return all_inputs
345
346
+ def classify(self, prompts: List[str]) -> List[str]:
347
+ # output is final logits
348
+ all_inputs = self.get_inputs(prompts)
349
+ outputs = []
350
+ for inputs in all_inputs:
351
+ output = self.model(**self.wrap_device(inputs))
352
+ logits = output.logits.softmax(dim=-1)[0].tolist()
353
+ outputs.append(logits)
354
+
355
+ return outputs
356
+
357
def generate(
358
self,
359
prompts: List[str],
@@ -688,6 +699,14 @@ class VllmRunner:
699
700
return inputs
701
702
+ def classify(self, prompts: List[str]) -> List[str]:
703
+ req_outputs = self.model.encode(prompts)
704
+ outputs = []
705
+ for req_output in req_outputs:
706
+ embedding = req_output.outputs.embedding
707
+ outputs.append(embedding)
708
+ return outputs
709
+
710
def generate(
711
self,
712
prompts: List[str],
tests/models/embedding/language/test_cls_models.py
new
+53
@@ -0,0 +1,53 @@
1
+"""Compare the outputs of HF and vLLM when using greedy sampling.
2
+
3
+This test only tests small models. Big models such as 7B should be tested from
4
+test_big_models.py because it could use a larger instance to run tests.
5
+
6
+Run `pytest tests/models/test_cls_models.py`.
7
+"""
8
+import pytest
9
+import torch
10
+from transformers import AutoModelForSequenceClassification
11
+
12
+CLASSIFICATION_MODELS = ["jason9693/Qwen2.5-1.5B-apeach"]
13
+
14
+
15
+@pytest.mark.parametrize("model", CLASSIFICATION_MODELS)
16
+@pytest.mark.parametrize("dtype", ["float"])
17
+def test_classification_models(
18
+ hf_runner,
19
+ vllm_runner,
20
+ example_prompts,
21
+ model: str,
22
+ dtype: str,
23
+) -> None:
24
+ with hf_runner(model,
25
+ dtype=dtype,
26
+ auto_cls=AutoModelForSequenceClassification) as hf_model:
27
+ hf_outputs = hf_model.classify(example_prompts)
28
+
29
+ with vllm_runner(model, dtype=dtype) as vllm_model:
30
+ vllm_outputs = vllm_model.classify(example_prompts)
31
+
32
+ print(hf_outputs, vllm_outputs)
33
+
34
+ # check logits difference
35
+ for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
36
+ hf_output = torch.tensor(hf_output)
37
+ vllm_output = torch.tensor(vllm_output)
38
+
39
+ assert torch.allclose(hf_output, vllm_output, 1e-3)
40
+
41
+
42
+@pytest.mark.parametrize("model", CLASSIFICATION_MODELS)
43
+@pytest.mark.parametrize("dtype", ["float"])
44
+def test_classification_model_print(
45
+ vllm_runner,
46
+ model: str,
47
+ dtype: str,
48
+) -> None:
49
+ with vllm_runner(model, dtype=dtype) as vllm_model:
50
+ # This test is for verifying whether the model's extra_repr
51
+ # can be printed correctly.
52
+ print(vllm_model.model.llm_engine.model_executor.driver_worker.
53
+ model_runner.model)
vllm/model_executor/layers/pooler.py
+8
-1
@@ -28,11 +28,15 @@ class Pooler(nn.Module):
28
normalize: Whether to normalize the pooled data.
29
"""
30
31
- def __init__(self, pooling_type: PoolingType, normalize: bool):
31
+ def __init__(self,
32
+ pooling_type: PoolingType,
33
+ normalize: bool,
34
+ softmax: bool = False):
35
super().__init__()
36
37
self.pooling_type = pooling_type
38
self.normalize = normalize
39
+ self.softmax = softmax
40
41
def forward(
42
self,
@@ -64,6 +68,9 @@ class Pooler(nn.Module):
68
if self.normalize:
69
pooled_data = nn.functional.normalize(pooled_data, p=2, dim=1)
70
71
+ if self.softmax:
72
+ pooled_data = nn.functional.softmax(pooled_data, dim=-1)
73
+
74
pooled_outputs = [
75
EmbeddingSequenceGroupOutput(data.tolist()) for data in pooled_data
76
]
vllm/model_executor/models/qwen2_cls.py
new
+107
@@ -0,0 +1,107 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://huggingface.co/Qwen/Qwen2.5-Math-RM-72B/blob/main/modeling_qwen2_rm.py
4
+# Copyright 2024 Kakao Corp. (Kanana-X Team)
5
+# Copyright 2024 The Qwen team.
6
+# Copyright 2023 The vLLM team.
7
+"""Inference-only Qwen2-Classification model compatible with HF weights."""
8
+from typing import Iterable, List, Optional, Tuple
9
+
10
+import torch
11
+from torch import nn
12
+from transformers import Qwen2Config
13
+
14
+from vllm.attention import AttentionMetadata
15
+from vllm.config import CacheConfig, LoRAConfig
16
+from vllm.model_executor.layers.linear import RowParallelLinear
17
+from vllm.model_executor.layers.pooler import Pooler, PoolingType
18
+from vllm.model_executor.layers.quantization.base_config import (
19
+ QuantizationConfig)
20
+from vllm.model_executor.models.qwen2 import Qwen2Model
21
+from vllm.model_executor.pooling_metadata import PoolingMetadata
22
+from vllm.sequence import IntermediateTensors, PoolerOutput
23
+
24
+from .utils import AutoWeightsLoader
25
+
26
+
27
+class Qwen2ForSequenceClassification(nn.Module):
28
+ packed_modules_mapping = {
29
+ "qkv_proj": [
30
+ "q_proj",
31
+ "k_proj",
32
+ "v_proj",
33
+ ],
34
+ "gate_up_proj": [
35
+ "gate_proj",
36
+ "up_proj",
37
+ ],
38
+ }
39
+
40
+ # LoRA specific attributes
41
+ supported_lora_modules = [
42
+ "qkv_proj",
43
+ "o_proj",
44
+ "gate_up_proj",
45
+ "down_proj",
46
+ ]
47
+ embedding_modules = {}
48
+ embedding_padding_modules = []
49
+
50
+ def __init__(
51
+ self,
52
+ config: Qwen2Config,
53
+ cache_config: Optional[CacheConfig] = None,
54
+ quant_config: Optional[QuantizationConfig] = None,
55
+ lora_config: Optional[LoRAConfig] = None,
56
+ ) -> None:
57
+ # TODO (@robertgshaw2): see if this can be moved out
58
+ if (cache_config.sliding_window is not None
59
+ and hasattr(config, "max_window_layers")):
60
+ raise ValueError("Sliding window for some but all layers is not "
61
+ "supported. This model uses sliding window "
62
+ "but `max_window_layers` = %s is less than "
63
+ "`num_hidden_layers` = %s. Please open an issue "
64
+ "to discuss this feature." % (
65
+ config.max_window_layers,
66
+ config.num_hidden_layers,
67
+ ))
68
+
69
+ super().__init__()
70
+
71
+ self.config = config
72
+ self.lora_config = lora_config
73
+
74
+ self.quant_config = quant_config
75
+ self.model = Qwen2Model(config, cache_config, quant_config)
76
+
77
+ self.score = RowParallelLinear(config.hidden_size,
78
+ config.num_labels,
79
+ quant_config=quant_config)
80
+ self._pooler = Pooler(pooling_type=PoolingType.LAST,
81
+ normalize=False,
82
+ softmax=True)
83
+
84
+ def forward(
85
+ self,
86
+ input_ids: torch.Tensor,
87
+ positions: torch.Tensor,
88
+ kv_caches: List[torch.Tensor],
89
+ attn_metadata: AttentionMetadata,
90
+ intermediate_tensors: Optional[IntermediateTensors] = None,
91
+ ) -> torch.Tensor:
92
+ hidden_states = self.model(input_ids, positions, kv_caches,
93
+ attn_metadata, intermediate_tensors)
94
+ logits, _ = self.score(hidden_states)
95
+ return logits
96
+
97
+ def pooler(
98
+ self,
99
+ hidden_states: torch.Tensor,
100
+ pooling_metadata: PoolingMetadata,
101
+ ) -> Optional[PoolerOutput]:
102
+ return self._pooler(hidden_states, pooling_metadata)
103
+
104
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
105
+ loader = AutoWeightsLoader(self,
106
+ ignore_unexpected_prefixes=["lm_head."])
107
+ loader.load_weights(weights)
vllm/model_executor/models/registry.py
+2
@@ -96,6 +96,8 @@ _EMBEDDING_MODELS = {
96
"Gemma2Model": ("gemma2", "Gemma2EmbeddingModel"),
97
"MistralModel": ("llama", "LlamaEmbeddingModel"),
98
"Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
99
+ "Qwen2ForSequenceClassification": (
100
+ "qwen2_cls", "Qwen2ForSequenceClassification"),
101
# [Multimodal]
102
"LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501
103
"Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),