@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6f1229f91d

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 03:30 UTC

[Model][2/N] Automatic conversion of CrossEncoding model (#19978)

Signed-off-by: wang.yuqi <noooop@126.com>

wang.yuqi committed Jul 3, 2025 at 21:59 UTC 6f1229f91ddbc8c12b77929b4eae4006d3a46ca5

16 files changed +200 -93

docs/models/supported_models.md

+1 -1

@@ -471,7 +471,7 @@ Specified using `--task classify`.

471 | `JambaForSequenceClassification` | Jamba | `ai21labs/Jamba-tiny-reward-dev`, etc. | ✅︎ | ✅︎ | |

472 | `GPT2ForSequenceClassification` | GPT2 | `nie3e/sentiment-polish-gpt2-small` | | | ✅︎ |

473 If your model is not in the above list, we will try to automatically convert the model using

474 -[as_classification_model][vllm.model_executor.models.adapters.as_classification_model]. By default, the class probabilities are extracted from the softmaxed hidden state corresponding to the last token.

474 +[as_seq_cls_model][vllm.model_executor.models.adapters.as_seq_cls_model]. By default, the class probabilities are extracted from the softmaxed hidden state corresponding to the last token.

475

476 #### Sentence Pair Scoring

477

docs/serving/openai_compatible_server.md

+1 -1

@@ -426,7 +426,7 @@ Code example: <gh-file:examples/online_serving/openai_pooling_client.py>

426

427 Our Classification API directly supports Hugging Face sequence-classification models such as [ai21labs/Jamba-tiny-reward-dev](https://huggingface.co/ai21labs/Jamba-tiny-reward-dev) and [jason9693/Qwen2.5-1.5B-apeach](https://huggingface.co/jason9693/Qwen2.5-1.5B-apeach).

428

429 -We automatically wrap any other transformer via `as_classification_model()`, which pools on the last token, attaches a `RowParallelLinear` head, and applies a softmax to produce per-class probabilities.

429 +We automatically wrap any other transformer via `as_seq_cls_model()`, which pools on the last token, attaches a `RowParallelLinear` head, and applies a softmax to produce per-class probabilities.

430

431 Code example: <gh-file:examples/online_serving/openai_classification_client.py>

432

tests/entrypoints/openai/correctness/test_mteb_score.py

+14 -11

@@ -6,19 +6,16 @@ import pytest

7 # yapf conflicts with isort for this block

8 # yapf: disable

9 -from tests.models.language.pooling.mteb_utils import (MTEB_RERANK_LANGS,

10 - MTEB_RERANK_TASKS,

11 - MTEB_RERANK_TOL,

12 - RerankClientMtebEncoder,

13 - ScoreClientMtebEncoder,

14 - run_mteb_rerank)

9 +from tests.models.language.pooling.mteb_utils import (

10 + MTEB_RERANK_LANGS, MTEB_RERANK_TASKS, MTEB_RERANK_TOL,

11 + RerankClientMtebEncoder, ScoreClientMtebEncoder,

12 + mteb_test_rerank_models_hf, run_mteb_rerank)

13 # yapf: enable

14 from tests.utils import RemoteOpenAIServer

16 os.environ["VLLM_LOGGING_LEVEL"] = "WARNING"

18 MODEL_NAME = "cross-encoder/ms-marco-MiniLM-L-6-v2"

21 -MAIN_SCORE = 0.33437

21 @pytest.fixture(scope="module")

@@ -31,12 +28,19 @@ def server():

28 yield remote_server

34 -def test_mteb_score(server):

31 +@pytest.fixture(scope="module")

32 +def st_main_score(hf_runner):

33 + # The main score related to the version of the dependency.

34 + # So we need to recalculate every time.

35 + main_score, st_dtype = mteb_test_rerank_models_hf(hf_runner, MODEL_NAME)

36 + return main_score

37 +

38 +

39 +def test_mteb_score(server, st_main_score):

40 url = server.url_for("score")

41 encoder = ScoreClientMtebEncoder(MODEL_NAME, url)

42 vllm_main_score = run_mteb_rerank(encoder, MTEB_RERANK_TASKS,

43 MTEB_RERANK_LANGS)

39 - st_main_score = MAIN_SCORE

45 print("VLLM main score: ", vllm_main_score)

46 print("SentenceTransformer main score: ", st_main_score)

@@ -45,12 +49,11 @@ def test_mteb_score(server):

49 assert st_main_score == pytest.approx(vllm_main_score, abs=MTEB_RERANK_TOL)

48 -def test_mteb_rerank(server):

52 +def test_mteb_rerank(server, st_main_score):

53 url = server.url_for("rerank")

54 encoder = RerankClientMtebEncoder(MODEL_NAME, url)

55 vllm_main_score = run_mteb_rerank(encoder, MTEB_RERANK_TASKS,

56 MTEB_RERANK_LANGS)

53 - st_main_score = MAIN_SCORE

58 print("VLLM main score: ", vllm_main_score)

59 print("SentenceTransformer main score: ", st_main_score)

tests/models/language/pooling/mteb_utils.py

+31 -25

@@ -234,6 +234,35 @@ def run_mteb_rerank(cross_encoder, tasks, languages):

234 return main_score

235

236

237 +def mteb_test_rerank_models_hf(hf_runner, model_name, hf_model_callback=None):

238 + with hf_runner(model_name, is_cross_encoder=True,

239 + dtype="float32") as hf_model:

240 +

241 + original_predict = hf_model.predict

242 +

243 + def _predict(

244 + sentences: list[tuple[str, str,

245 + Optional[str]]], # query, corpus, prompt

246 + *args,

247 + **kwargs,

248 + ):

249 + # vllm and st both remove the prompt, fair comparison.

250 + prompts = [(s[0], s[1]) for s in sentences]

251 + return original_predict(prompts, *args, **kwargs, batch_size=8)

252 +

253 + hf_model.predict = _predict

254 + hf_model.original_predict = original_predict

255 +

256 + if hf_model_callback is not None:

257 + hf_model_callback(hf_model)

258 +

259 + st_main_score = run_mteb_rerank(hf_model,

260 + tasks=MTEB_RERANK_TASKS,

261 + languages=MTEB_RERANK_LANGS)

262 + st_dtype = next(hf_model.model.model.parameters()).dtype

263 + return st_main_score, st_dtype

264 +

265 +

266 def mteb_test_rerank_models(hf_runner,

267 vllm_runner,

268 model_info: RerankModelInfo,

@@ -264,31 +293,8 @@ def mteb_test_rerank_models(hf_runner,

293 languages=MTEB_RERANK_LANGS)

294 vllm_dtype = model_config.dtype

295

267 - with hf_runner(model_info.name, is_cross_encoder=True,

268 - dtype="float32") as hf_model:

269 -

270 - original_predict = hf_model.predict

271 -

272 - def _predict(

273 - sentences: list[tuple[str, str,

274 - Optional[str]]], # query, corpus, prompt

275 - *args,

276 - **kwargs,

277 - ):

278 - # vllm and st both remove the prompt, fair comparison.

279 - prompts = [(s[0], s[1]) for s in sentences]

280 - return original_predict(prompts, *args, **kwargs, batch_size=8)

281 -

282 - hf_model.predict = _predict

283 - hf_model.original_predict = original_predict

284 -

285 - if hf_model_callback is not None:

286 - hf_model_callback(hf_model)

287 -

288 - st_main_score = run_mteb_rerank(hf_model,

289 - tasks=MTEB_RERANK_TASKS,

290 - languages=MTEB_RERANK_LANGS)

291 - st_dtype = next(hf_model.model.model.parameters()).dtype

296 + st_main_score, st_dtype = mteb_test_rerank_models_hf(

297 + hf_runner, model_info.name, hf_model_callback)

298

299 print("VLLM:", vllm_dtype, vllm_main_score)

300 print("SentenceTransformers:", st_dtype, st_main_score)

tests/models/test_registry.py

+4 -4

@@ -9,9 +9,9 @@ import torch.cuda

9 from vllm.model_executor.models import (is_pooling_model,

10 is_text_generation_model,

11 supports_multimodal)

12 -from vllm.model_executor.models.adapters import (as_classification_model,

13 - as_embedding_model,

14 - as_reward_model)

12 +from vllm.model_executor.models.adapters import (as_embedding_model,

13 + as_reward_model,

14 + as_seq_cls_model)

15 from vllm.model_executor.models.registry import (_MULTIMODAL_MODELS,

16 _SPECULATIVE_DECODING_MODELS,

17 _TEXT_GENERATION_MODELS,

@@ -38,7 +38,7 @@ def test_registry_imports(model_arch):

38 assert is_text_generation_model(model_cls)

40 # All vLLM models should be convertible to a pooling model

41 - assert is_pooling_model(as_classification_model(model_cls))

41 + assert is_pooling_model(as_seq_cls_model(model_cls))

42 assert is_pooling_model(as_embedding_model(model_cls))

43 assert is_pooling_model(as_reward_model(model_cls))

tests/test_config.py

+27 -1

@@ -52,7 +52,7 @@ def test_get_field():

52 ("distilbert/distilgpt2", "generate", "generate"),

53 ("intfloat/multilingual-e5-small", "pooling", "embed"),

54 ("jason9693/Qwen2.5-1.5B-apeach", "pooling", "classify"),

55 - ("cross-encoder/ms-marco-MiniLM-L-6-v2", "pooling", "score"),

55 + ("cross-encoder/ms-marco-MiniLM-L-6-v2", "pooling", "classify"),

56 ("Qwen/Qwen2.5-Math-RM-72B", "pooling", "reward"),

57 ("openai/whisper-small", "transcription", "transcription"),

58 ],

@@ -72,6 +72,32 @@ def test_auto_task(model_id, expected_runner_type, expected_task):

72 assert config.task == expected_task

75 +@pytest.mark.parametrize(

76 + ("model_id", "expected_runner_type", "expected_task"),

77 + [

78 + ("distilbert/distilgpt2", "pooling", "embed"),

79 + ("intfloat/multilingual-e5-small", "pooling", "embed"),

80 + ("jason9693/Qwen2.5-1.5B-apeach", "pooling", "classify"),

81 + ("cross-encoder/ms-marco-MiniLM-L-6-v2", "pooling", "classify"),

82 + ("Qwen/Qwen2.5-Math-RM-72B", "pooling", "embed"),

83 + ("openai/whisper-small", "pooling", "embed"),

84 + ],

85 +)

86 +def test_score_task(model_id, expected_runner_type, expected_task):

87 + config = ModelConfig(

88 + model_id,

89 + task="score",

90 + tokenizer=model_id,

91 + tokenizer_mode="auto",

92 + trust_remote_code=False,

93 + seed=0,

94 + dtype="float16",

95 + )

96 +

97 + assert config.runner_type == expected_runner_type

98 + assert config.task == expected_task

99 +

100 +

101 @pytest.mark.parametrize(("model_id", "bad_task"), [

102 ("Qwen/Qwen2.5-Math-RM-72B", "generate"),

103 ])

vllm/config.py

+22 -12

@@ -93,14 +93,14 @@ ConfigT = TypeVar("ConfigT", bound=ConfigType)

93 TaskOption = Literal["auto", "generate", "embedding", "embed", "classify",

94 "score", "reward", "transcription"]

96 -_ResolvedTask = Literal["generate", "embed", "classify", "score", "reward",

97 - "draft", "transcription"]

96 +_ResolvedTask = Literal["generate", "embed", "classify", "reward", "draft",

97 + "transcription"]

99 RunnerType = Literal["generate", "pooling", "draft", "transcription"]

100

101 _RUNNER_TASKS: dict[RunnerType, list[_ResolvedTask]] = {

102 "generate": ["generate"],

103 - "pooling": ["embed", "classify", "score", "reward"],

103 + "pooling": ["embed", "classify", "reward"],

104 "draft": ["draft"],

105 "transcription": ["transcription"],

106 }

@@ -777,7 +777,7 @@ class ModelConfig:

777 if get_pooling_config(model_id, self.revision):

778 return "embed"

779 if self.registry.is_cross_encoder_model(architectures):

780 - return "score"

780 + return "classify"

781 if self.registry.is_transcription_model(architectures):

782 return "transcription"

783

@@ -841,14 +841,24 @@ class ModelConfig:

841 "This model supports multiple tasks: %s. "

842 "Defaulting to '%s'.", supported_tasks, selected_task)

843 else:

844 - # Aliases

845 - if task_option == "embedding":

846 - msg = ("The 'embedding' task has been renamed to "

847 - "'embed', please use the new name. The old name "

848 - "will be removed in v1.0.")

849 - warnings.warn(msg, DeprecationWarning, stacklevel=2)

850 -

851 - task_option = "embed"

844 + if task_option == "score":

845 + if not runner_support["pooling"]:

846 + msg = (f"This model does not support the '{task_option}' "

847 + f"task. Supported tasks: {supported_tasks}")

848 + raise ValueError(msg)

849 + if self.registry.is_cross_encoder_model(architectures):

850 + task_option = "classify"

851 + else:

852 + task_option = "embed"

853 + else:

854 + # Aliases

855 + if task_option == "embedding":

856 + msg = ("The 'embedding' task has been renamed to "

857 + "'embed', please use the new name. The old name "

858 + "will be removed in v1.0.")

859 + warnings.warn(msg, DeprecationWarning, stacklevel=2)

860 +

861 + task_option = "embed"

862

863 if task_option not in supported_tasks:

864 msg = (

vllm/entrypoints/llm.py

+7 -3

@@ -1289,9 +1289,13 @@ class LLM:

1289

1290 raise ValueError(" ".join(messages))

1291

1292 - if self.llm_engine.model_config.task not in ("embed", "score"):

1293 - raise ValueError(

1294 - "Score API is only enabled for `--task embed or --task score`")

1292 + if self.llm_engine.model_config.task not in ("embed", "classify"):

1293 + raise ValueError("Score API is only enabled for "

1294 + "`--task embed or --task classify`.")

1295 +

1296 + if (self.llm_engine.model_config.task == "classify"

1297 + and self.llm_engine.model_config.hf_config.num_labels != 1):

1298 + raise ValueError("Score API is only enabled for num_labels == 1.")

1299

1300 # the tokenizer for models such as

1301 # "cross-encoder/ms-marco-MiniLM-L-6-v2" doesn't support passing

vllm/entrypoints/openai/api_server.py

+11 -8

@@ -1311,24 +1311,27 @@ async def init_app_state(

1311 chat_template=resolved_chat_template,

1312 chat_template_content_format=args.chat_template_content_format,

1313 ) if model_config.task == "embed" else None

1314 - state.openai_serving_scores = ServingScores(

1315 - engine_client,

1316 - model_config,

1317 - state.openai_serving_models,

1318 - request_logger=request_logger) if model_config.task in (

1319 - "score", "embed", "pooling") else None

1314 state.openai_serving_classification = ServingClassification(

1315 engine_client,

1316 model_config,

1317 state.openai_serving_models,

1318 request_logger=request_logger,

1319 ) if model_config.task == "classify" else None

1320 +

1321 + enable_serving_reranking = (model_config.task == "classify" and getattr(

1322 + model_config.hf_config, "num_labels", 0) == 1)

1323 state.jinaai_serving_reranking = ServingScores(

1324 engine_client,

1325 model_config,

1326 state.openai_serving_models,

1330 - request_logger=request_logger

1331 - ) if model_config.task == "score" else None

1327 + request_logger=request_logger) if enable_serving_reranking else None

1328 + state.openai_serving_scores = ServingScores(

1329 + engine_client,

1330 + model_config,

1331 + state.openai_serving_models,

1332 + request_logger=request_logger) if (

1333 + model_config.task == "embed" or enable_serving_reranking) else None

1334 +

1335 state.openai_serving_tokenization = OpenAIServingTokenization(

1336 engine_client,

1337 model_config,

vllm/entrypoints/openai/run_batch.py

+5 -1

@@ -357,12 +357,16 @@ async def main(args):

357 chat_template=None,

358 chat_template_content_format="auto",

359 ) if model_config.task == "embed" else None

360 +

361 + enable_serving_reranking = (model_config.task == "classify" and getattr(

362 + model_config.hf_config, "num_labels", 0) == 1)

363 +

364 openai_serving_scores = (ServingScores(

365 engine,

366 model_config,

367 openai_serving_models,

368 request_logger=request_logger,

365 - ) if model_config.task == "score" else None)

369 + ) if (model_config.task == "embed" or enable_serving_reranking) else None)

370

371 tracker = BatchProgressTracker()

372 logger.info("Reading batch from %s...", args.input_file)

vllm/model_executor/layers/pooler.py

+17 -6

@@ -285,6 +285,7 @@ class PoolerHead(nn.Module):

285 else:

286 pooled_data = pooled_data.to(torch.float32)

287

288 + # for matryoshka representation

289 if isinstance(pooling_metadata, V0PoolingMetadata):

290 dimensions_list = [

291 pooling_param.dimensions

@@ -299,10 +300,16 @@ class PoolerHead(nn.Module):

300 if any(d is not None for d in dimensions_list):

301 # change the output dimension

302 assert len(pooled_data) == len(dimensions_list)

302 - pooled_data = [

303 - vecs if d is None else vecs[..., :d]

304 - for vecs, d in zip(pooled_data, dimensions_list)

305 - ]

303 + if len(set(dimensions_list)) == 1 and not isinstance(

304 + pooled_data, list):

305 + # if all dimensions are the same

306 + d = dimensions_list[0]

307 + pooled_data = pooled_data[..., :d]

308 + else:

309 + pooled_data = [

310 + vecs if d is None else vecs[..., :d]

311 + for vecs, d in zip(pooled_data, dimensions_list)

312 + ]

313

314 if self.normalize:

315 if isinstance(pooled_data, list):

@@ -325,6 +332,10 @@ class PoolerHead(nn.Module):

332 else:

333 pooled_data = F.sigmoid(pooled_data)

334

335 + # shape:

336 + # classify (& score) -> (batch_size, num_classes)

337 + # embed -> (batch_size, embedding_dim) or list(embedding_dim)

338 + # (batch_size, dimensions) or list(dimensions) if using MRL

339 return pooled_data

340

341

@@ -419,7 +430,6 @@ class ClassifierPooler(nn.Module):

430 offset += prompt_len

431 pooled_data.append(pooled_data_i)

432

422 - offset = 0

433 pooled_data_lst = []

434 for pooled_data_i in pooled_data:

435

@@ -436,7 +446,8 @@ class ClassifierPooler(nn.Module):

446 # apply classifier once on the full batch if possible

447 pooled_output = self.classifier(pooled_output)

448

439 - scores = self.default_activation_function(pooled_output).squeeze(-1)

449 + # shape: (batch_size, num_labels)

450 + scores = self.default_activation_function(pooled_output)

451

452 pooled_outputs = [PoolingSequenceGroupOutput(data) for data in scores]

453 return PoolerOutput(outputs=pooled_outputs)

vllm/model_executor/model_loader/utils.py

+4 -3

@@ -21,8 +21,7 @@ from vllm.model_executor.layers.linear import QKVCrossParallelLinear

21 from vllm.model_executor.layers.quantization.base_config import (

22 QuantizationConfig, QuantizeMethodBase)

23 from vllm.model_executor.models import ModelRegistry

24 -from vllm.model_executor.models.adapters import (as_classification_model,

25 - as_embedding_model,

24 +from vllm.model_executor.models.adapters import (as_embedding_model,

25 as_reward_model)

26 from vllm.model_executor.models.interfaces import SupportsQuant

27 from vllm.utils import is_pin_memory_available

@@ -246,7 +245,9 @@ def get_model_architecture(

245 if model_config.task == "embed":

246 model_cls = as_embedding_model(model_cls)

247 elif model_config.task == "classify":

249 - model_cls = as_classification_model(model_cls)

248 + # Cannot automatically run as_seq_cls_model,

249 + # otherwise it will cause a circular reference on is_cross_encoder_model

250 + pass

251 elif model_config.task == "reward":

252 model_cls = as_reward_model(model_cls)

253

vllm/model_executor/models/adapters.py

+44 -13

@@ -2,7 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 from collections.abc import Iterable

5 -from typing import TYPE_CHECKING, Any, Optional, TypeVar

5 +from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union

7 import torch

8 import torch.nn as nn

@@ -145,9 +145,9 @@ def as_embedding_model(cls: _T) -> _T:

145 return ModelForEmbedding # type: ignore

146

147

148 -def as_classification_model(cls: _T) -> _T:

148 +def as_seq_cls_model(cls: _T) -> _T:

149 """

150 - Subclass an existing vLLM model to support classification.

150 + Subclass an existing vLLM model to support classify and score tasks.

151

152 By default, the class probabilities are extracted from the softmaxed

153 hidden state corresponding to the last token.

@@ -164,7 +164,9 @@ def as_classification_model(cls: _T) -> _T:

164 # Lazy import

165 from vllm.config import VllmConfig

166 from vllm.model_executor.layers.linear import RowParallelLinear

167 - from vllm.model_executor.layers.pooler import PoolingType

167 + from vllm.model_executor.layers.pooler import PoolerOutput, PoolingType

168 + from vllm.model_executor.models.interfaces import SupportsCrossEncoding

169 + from vllm.model_executor.pooling_metadata import PoolingMetadata

170 from vllm.sequence import IntermediateTensors

171

172 from .utils import maybe_prefix

@@ -176,7 +178,8 @@ def as_classification_model(cls: _T) -> _T:

178 default_softmax=True,

179 )

180

179 - class ModelForClassification(ModelForPooling):

181 + class ModelForSequenceClassification(ModelForPooling,

182 + SupportsCrossEncoding):

183

184 def __init__(

185 self,

@@ -190,6 +193,10 @@ def as_classification_model(cls: _T) -> _T:

193 config = vllm_config.model_config.hf_config

194 quant_config = vllm_config.quant_config

195

196 + self.task = vllm_config.model_config.task

197 + self.pooling_type = (

198 + vllm_config.model_config.pooler_config.pooling_type)

199 +

200 self.score = RowParallelLinear(config.hidden_size,

201 config.num_labels,

202 quant_config=quant_config,

@@ -205,17 +212,41 @@ def as_classification_model(cls: _T) -> _T:

212 intermediate_tensors: Optional[IntermediateTensors] = None,

213 inputs_embeds: Optional[torch.Tensor] = None,

214 ) -> torch.Tensor:

208 - hidden_states = super().forward(input_ids, positions,

209 - intermediate_tensors,

210 - inputs_embeds)

211 - logits, _ = self.score(hidden_states)

212 - return logits

215 + return super().forward(input_ids, positions, intermediate_tensors,

216 + inputs_embeds)

217 +

218 + def pooler(

219 + self,

220 + hidden_states: Union[torch.Tensor, list[torch.Tensor]],

221 + pooling_metadata: PoolingMetadata,

222 + ) -> PoolerOutput:

223 +

224 + def get_logits(hidden_states):

225 + if isinstance(hidden_states, list):

226 + logits = [self.score(state)[0] for state in hidden_states]

227 + else:

228 + logits, _ = self.score(hidden_states)

229 + return logits

230 +

231 + if self.pooling_type == PoolingType.ALL:

232 + logits = get_logits(hidden_states)

233 + return self._pooler(logits, pooling_metadata)

234 + else:

235 + hidden_states = self._pooler.extract_states(

236 + hidden_states, pooling_metadata)

237 + logits = get_logits(hidden_states)

238 + pooled_data = self._pooler.head(logits, pooling_metadata)

239 +

240 + pooled_outputs = [

241 + self._pooler.build_output(data) for data in pooled_data

242 + ]

243 + return PoolerOutput(outputs=pooled_outputs)

244

245

215 - ModelForClassification.__name__ = \

216 - _get_pooling_model_name(cls.__name__, "ForClassification")

246 + ModelForSequenceClassification.__name__ = \

247 + _get_pooling_model_name(cls.__name__, "ForSequenceClassification")

248

218 - return ModelForClassification # type: ignore

249 + return ModelForSequenceClassification # type: ignore

250

251

252 def as_reward_model(cls: _T) -> _T:

vllm/model_executor/models/qwen2.py

@@ -50,6 +50,7 @@ from vllm.model_executor.model_loader.weight_utils import (

50 from vllm.model_executor.sampling_metadata import SamplingMetadata

51 from vllm.sequence import IntermediateTensors

53 +from .adapters import as_seq_cls_model

54 from .interfaces import SupportsLoRA, SupportsPP

55 from .utils import (AutoWeightsLoader, PPMissingLayer, extract_layer_index,

56 is_pp_missing_parameter,

@@ -495,3 +496,6 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):

496 if self.config.tie_word_embeddings else None),

497 )

498 return loader.load_weights(weights)

499 +

500 +

501 +Qwen2ForSequenceClassification = as_seq_cls_model(Qwen2ForCausalLM)

vllm/model_executor/models/registry.py

+3 -3

@@ -164,8 +164,6 @@ _EMBEDDING_MODELS = {

164 "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501

165 "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),

166 "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501

167 - # [Auto-converted (see adapters.py)]

168 - "Qwen2ForSequenceClassification": ("qwen2", "Qwen2ForCausalLM"),

167 # Technically PrithviGeoSpatialMAE is a model that works on images, both in

168 # input and output. I am adding it here because it piggy-backs on embedding

169 # models for the time being.

@@ -180,7 +178,9 @@ _CROSS_ENCODER_MODELS = {

178 "RobertaForSequenceClassification"),

179 "ModernBertForSequenceClassification": ("modernbert",

180 "ModernBertForSequenceClassification"),

183 - "Qwen3ForSequenceClassification": ("qwen3", "Qwen3ForSequenceClassification"), # noqa: E501

181 + # [Auto-converted (see adapters.py)]

182 + "Qwen2ForSequenceClassification": ("qwen2", "Qwen2ForSequenceClassification"), # noqa: E501

183 + "Qwen3ForSequenceClassification": ("qwen3", "Qwen3ForSequenceClassification"), # noqa: E501

184 }

185

186 _MULTIMODAL_MODELS = {

vllm/outputs.py

+5 -1

@@ -453,6 +453,7 @@ class ClassificationOutput:

453

454 @staticmethod

455 def from_base(pooling_output: PoolingOutput):

456 + # pooling_output shape: (num_classes)

457 pooled_data = pooling_output.data

458 if pooled_data.ndim != 1:

459 raise ValueError("pooled_data should be a 1-D probability vector")

@@ -490,7 +491,10 @@ class ScoringOutput:

491

492 @staticmethod

493 def from_base(pooling_output: PoolingOutput):

493 - pooled_data = pooling_output.data

494 + # pooling_output shape:

495 + # classify task: (num_classes) num_classes == 1

496 + # embed task: a scalar value

497 + pooled_data = pooling_output.data.squeeze()

498 if pooled_data.ndim != 0:

499 raise ValueError("pooled_data should be a scalar score")

500