@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 360bd8762f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 13, 05:45 UTC

[Frontend] Added chat-style multimodal support to /classify. (#27516)

[Frontend] Added chat-style multimodal support to /classify. (#27516) Signed-off-by: WorldExplored <srreyansh.sethi@gmail.com> Signed-off-by: Srreyansh Sethi <107075589+WorldExplored@users.noreply.github.com> Signed-off-by: vnadathur <glvikramn@gmail.com> Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Co-authored-by: vnadathur <236933696+vnadathur@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: vnadathur <glvikramn@gmail.com> Co-authored-by: wang.yuqi <noooop@126.com> Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>

Srreyansh Sethi committed Nov 14, 2025 at 03:03 UTC 360bd8762f053c59ee19e2fd72cb1e5a28423958
6 files changed +318 -27
tests/entrypoints/pooling/openai/test_classification.py
+10
@@ -46,6 +46,16 @@ def test_single_input_classification(server: RemoteOpenAIServer, model_name: str
46 assert hasattr(output.data[0], "probs")
47
48
49 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
50 +def test_add_special_tokens_false(server: RemoteOpenAIServer, model_name: str):
51 + response = requests.post(
52 + server.url_for("classify"),
53 + json={"model": model_name, "input": "hello", "add_special_tokens": False},
54 + )
55 + response.raise_for_status()
56 + ClassificationResponse.model_validate(response.json())
57 +
58 +
59 @pytest.mark.parametrize("model_name", [MODEL_NAME])
60 def test_multiple_inputs_classification(server: RemoteOpenAIServer, model_name: str):
61 input_texts = [
tests/entrypoints/pooling/openai/test_vision_classification.py new
+95
@@ -0,0 +1,95 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import json
5 +
6 +import pytest
7 +import requests
8 +
9 +from tests.utils import RemoteOpenAIServer
10 +from vllm.entrypoints.openai.protocol import ClassificationResponse
11 +
12 +VLM_MODEL_NAME = "muziyongshixin/Qwen2.5-VL-7B-for-VideoCls"
13 +MAXIMUM_VIDEOS = 1
14 +TEST_VIDEO_URL = "https://www.bogotobogo.com/python/OpenCV_Python/images/mean_shift_tracking/slow_traffic_small.mp4"
15 +
16 +HF_OVERRIDES = {
17 + "text_config": {
18 + "architectures": ["Qwen2_5_VLForSequenceClassification"],
19 + },
20 +}
21 +
22 +
23 +@pytest.fixture(scope="module")
24 +def server_vlm_classify():
25 + args = [
26 + "--runner",
27 + "pooling",
28 + "--max-model-len",
29 + "5000",
30 + "--enforce-eager",
31 + "--limit-mm-per-prompt",
32 + json.dumps({"video": MAXIMUM_VIDEOS}),
33 + ]
34 +
35 + with RemoteOpenAIServer(
36 + VLM_MODEL_NAME, args, override_hf_configs=HF_OVERRIDES
37 + ) as remote_server:
38 + yield remote_server
39 +
40 +
41 +@pytest.mark.parametrize("model_name", [VLM_MODEL_NAME])
42 +def test_classify_accepts_chat_text_only(
43 + server_vlm_classify: RemoteOpenAIServer, model_name: str
44 +) -> None:
45 + messages = [
46 + {
47 + "role": "user",
48 + "content": [
49 + {"type": "text", "text": "Please classify this text request."},
50 + ],
51 + }
52 + ]
53 +
54 + response = requests.post(
55 + server_vlm_classify.url_for("classify"),
56 + json={"model": model_name, "messages": messages},
57 + )
58 + response.raise_for_status()
59 +
60 + output = ClassificationResponse.model_validate(response.json())
61 +
62 + assert output.object == "list"
63 + assert output.model == model_name
64 + assert len(output.data) == 1
65 + assert len(output.data[0].probs) == 2
66 + assert output.usage.prompt_tokens == 22
67 +
68 +
69 +@pytest.mark.parametrize("model_name", [VLM_MODEL_NAME])
70 +def test_classify_accepts_chat_video_url(
71 + server_vlm_classify: RemoteOpenAIServer, model_name: str
72 +) -> None:
73 + messages = [
74 + {
75 + "role": "user",
76 + "content": [
77 + {"type": "text", "text": "Please classify this video."},
78 + {"type": "video_url", "video_url": {"url": TEST_VIDEO_URL}},
79 + ],
80 + }
81 + ]
82 +
83 + response = requests.post(
84 + server_vlm_classify.url_for("classify"),
85 + json={"model": model_name, "messages": messages},
86 + )
87 + response.raise_for_status()
88 +
89 + output = ClassificationResponse.model_validate(response.json())
90 +
91 + assert output.object == "list"
92 + assert output.model == model_name
93 + assert len(output.data) == 1
94 + assert len(output.data[0].probs) == 2
95 + assert output.usage.prompt_tokens == 4807
vllm/entrypoints/openai/api_server.py
+3
@@ -1784,6 +1784,9 @@ async def init_app_state(
1784 engine_client,
1785 state.openai_serving_models,
1786 request_logger=request_logger,
1787 + chat_template=resolved_chat_template,
1788 + chat_template_content_format=args.chat_template_content_format,
1789 + trust_request_chat_template=args.trust_request_chat_template,
1790 log_error_stack=args.log_error_stack,
1791 )
1792 if "classify" in supported_tasks
vllm/entrypoints/openai/protocol.py
+113 -3
@@ -2000,10 +2000,10 @@ class ScoreResponse(OpenAIBaseModel):
2000 usage: UsageInfo
2001
2002
2003 -class ClassificationRequest(OpenAIBaseModel):
2003 +class ClassificationCompletionRequest(OpenAIBaseModel):
2004 model: str | None = None
2005 input: list[str] | str
2006 - truncate_prompt_tokens: int | None = None
2006 + truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
2007 user: str | None = None
2008
2009 # --8<-- [start:classification-extra-params]
@@ -2015,7 +2015,21 @@ class ClassificationRequest(OpenAIBaseModel):
2015 "if the served model does not use priority scheduling."
2016 ),
2017 )
2018 -
2018 + add_special_tokens: bool = Field(
2019 + default=True,
2020 + description=(
2021 + "If true (the default), special tokens (e.g. BOS) will be added to "
2022 + "the prompt."
2023 + ),
2024 + )
2025 + request_id: str = Field(
2026 + default_factory=lambda: f"{random_uuid()}",
2027 + description=(
2028 + "The request_id related to this request. If the caller does "
2029 + "not set it, a random_uuid will be generated. This id is used "
2030 + "through out the inference process and return in response."
2031 + ),
2032 + )
2033 softmax: bool | None = Field(
2034 default=None,
2035 description="softmax will be deprecated, please use use_activation instead.",
@@ -2040,6 +2054,102 @@ class ClassificationRequest(OpenAIBaseModel):
2054 )
2055
2056
2057 +class ClassificationChatRequest(OpenAIBaseModel):
2058 + model: str | None = None
2059 + messages: list[ChatCompletionMessageParam]
2060 + truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
2061 + user: str | None = None
2062 +
2063 + # --8<-- [start:chat-classification-extra-params]
2064 + add_generation_prompt: bool = Field(
2065 + default=False,
2066 + description=(
2067 + "If true, the generation prompt will be added to the chat template. "
2068 + "This is a parameter used by chat template in tokenizer config of the "
2069 + "model."
2070 + ),
2071 + )
2072 +
2073 + add_special_tokens: bool = Field(
2074 + default=False,
2075 + description=(
2076 + "If true, special tokens (e.g. BOS) will be added to the prompt "
2077 + "on top of what is added by the chat template. "
2078 + "For most models, the chat template takes care of adding the "
2079 + "special tokens so this should be set to false (as is the "
2080 + "default)."
2081 + ),
2082 + )
2083 +
2084 + chat_template: str | None = Field(
2085 + default=None,
2086 + description=(
2087 + "A Jinja template to use for this conversion. "
2088 + "As of transformers v4.44, default chat template is no longer "
2089 + "allowed, so you must provide a chat template if the tokenizer "
2090 + "does not define one."
2091 + ),
2092 + )
2093 +
2094 + chat_template_kwargs: dict[str, Any] | None = Field(
2095 + default=None,
2096 + description=(
2097 + "Additional keyword args to pass to the template renderer. "
2098 + "Will be accessible by the chat template."
2099 + ),
2100 + )
2101 +
2102 + mm_processor_kwargs: dict[str, Any] | None = Field(
2103 + default=None,
2104 + description=("Additional kwargs to pass to the HF processor."),
2105 + )
2106 +
2107 + priority: int = Field(
2108 + default=0,
2109 + description=(
2110 + "The priority of the request (lower means earlier handling; "
2111 + "default: 0). Any priority other than 0 will raise an error "
2112 + "if the served model does not use priority scheduling."
2113 + ),
2114 + )
2115 +
2116 + request_id: str = Field(
2117 + default_factory=lambda: f"{random_uuid()}",
2118 + description=(
2119 + "The request_id related to this request. If the caller does "
2120 + "not set it, a random_uuid will be generated. This id is used "
2121 + "through out the inference process and return in response."
2122 + ),
2123 + )
2124 + softmax: bool | None = Field(
2125 + default=None,
2126 + description="softmax will be deprecated, please use use_activation instead.",
2127 + )
2128 +
2129 + activation: bool | None = Field(
2130 + default=None,
2131 + description="activation will be deprecated, please use use_activation instead.",
2132 + )
2133 +
2134 + use_activation: bool | None = Field(
2135 + default=None,
2136 + description="Whether to use activation for classification outputs. "
2137 + "Default is True.",
2138 + )
2139 + # --8<-- [end:chat-classification-extra-params]
2140 +
2141 + def to_pooling_params(self):
2142 + return PoolingParams(
2143 + truncate_prompt_tokens=self.truncate_prompt_tokens,
2144 + use_activation=get_use_activation(self),
2145 + )
2146 +
2147 +
2148 +ClassificationRequest: TypeAlias = (
2149 + ClassificationCompletionRequest | ClassificationChatRequest
2150 +)
2151 +
2152 +
2153 class ClassificationData(OpenAIBaseModel):
2154 index: int
2155 label: str | None
vllm/entrypoints/openai/serving_classification.py
+81 -19
@@ -4,13 +4,17 @@
4 from http import HTTPStatus
5 from typing import cast
6
7 +import jinja2
8 import numpy as np
9 from fastapi import Request
9 -from typing_extensions import override
10
11 from vllm.engine.protocol import EngineClient
12 +from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
13 from vllm.entrypoints.logger import RequestLogger
14 from vllm.entrypoints.openai.protocol import (
15 + ChatCompletionRequest,
16 + ClassificationChatRequest,
17 + ClassificationCompletionRequest,
18 ClassificationData,
19 ClassificationRequest,
20 ClassificationResponse,
@@ -32,7 +36,10 @@ logger = init_logger(__name__)
36
37
38 class ClassificationMixin(OpenAIServing):
35 - @override
39 + chat_template: str | None
40 + chat_template_content_format: ChatTemplateContentFormatOption
41 + trust_request_chat_template: bool
42 +
43 async def _preprocess(
44 self,
45 ctx: ServeContext,
@@ -42,31 +49,79 @@ class ClassificationMixin(OpenAIServing):
49 and prepare model-specific inputs.
50 """
51 ctx = cast(ClassificationServeContext, ctx)
45 - if isinstance(ctx.request.input, str) and not ctx.request.input:
46 - return self.create_error_response(
47 - "Input cannot be empty for classification",
48 - status_code=HTTPStatus.BAD_REQUEST,
49 - )
50 -
51 - if isinstance(ctx.request.input, list) and len(ctx.request.input) == 0:
52 - return None
53 -
52 try:
53 ctx.tokenizer = await self.engine_client.get_tokenizer()
54
57 - renderer = self._get_renderer(ctx.tokenizer)
58 - ctx.engine_prompts = await renderer.render_prompt(
59 - prompt_or_prompts=ctx.request.input,
60 - config=self._build_render_config(ctx.request),
61 - )
55 + request_obj = ctx.request
56 +
57 + if isinstance(request_obj, ClassificationChatRequest):
58 + chat_request = request_obj
59 + messages = chat_request.messages
60 + trust_request_chat_template = getattr(
61 + self,
62 + "trust_request_chat_template",
63 + False,
64 + )
65 + ret = self._validate_chat_template(
66 + request_chat_template=chat_request.chat_template,
67 + chat_template_kwargs=chat_request.chat_template_kwargs,
68 + trust_request_chat_template=trust_request_chat_template,
69 + )
70 + if ret:
71 + return ret
72 +
73 + (
74 + _,
75 + _,
76 + engine_prompts,
77 + ) = await self._preprocess_chat(
78 + cast(ChatCompletionRequest, chat_request),
79 + ctx.tokenizer,
80 + messages,
81 + chat_template=(
82 + chat_request.chat_template
83 + or getattr(self, "chat_template", None)
84 + ),
85 + chat_template_content_format=cast(
86 + ChatTemplateContentFormatOption,
87 + getattr(self, "chat_template_content_format", "auto"),
88 + ),
89 + add_generation_prompt=False,
90 + continue_final_message=False,
91 + add_special_tokens=chat_request.add_special_tokens,
92 + )
93 + ctx.engine_prompts = engine_prompts
94 +
95 + elif isinstance(request_obj, ClassificationCompletionRequest):
96 + completion_request = request_obj
97 + input_data = completion_request.input
98 + if input_data in (None, ""):
99 + return self.create_error_response(
100 + "Input or messages must be provided",
101 + status_code=HTTPStatus.BAD_REQUEST,
102 + )
103 + if isinstance(input_data, list) and not input_data:
104 + ctx.engine_prompts = []
105 + return None
106 +
107 + renderer = self._get_renderer(ctx.tokenizer)
108 + prompt_input = cast(str | list[str], input_data)
109 + ctx.engine_prompts = await renderer.render_prompt(
110 + prompt_or_prompts=prompt_input,
111 + config=self._build_render_config(completion_request),
112 + )
113 + else:
114 + return self.create_error_response(
115 + "Invalid classification request type",
116 + status_code=HTTPStatus.BAD_REQUEST,
117 + )
118
119 return None
120
65 - except (ValueError, TypeError) as e:
121 + except (ValueError, TypeError, jinja2.TemplateError) as e:
122 logger.exception("Error in preprocessing prompt inputs")
123 return self.create_error_response(str(e))
124
69 - @override
125 def _build_response(
126 self,
127 ctx: ServeContext,
@@ -118,6 +173,7 @@ class ClassificationMixin(OpenAIServing):
173 return RenderConfig(
174 max_length=self.max_model_len,
175 truncate_prompt_tokens=request.truncate_prompt_tokens,
176 + add_special_tokens=request.add_special_tokens,
177 )
178
179
@@ -130,6 +186,9 @@ class ServingClassification(ClassificationMixin):
186 models: OpenAIServingModels,
187 *,
188 request_logger: RequestLogger | None,
189 + chat_template: str | None = None,
190 + chat_template_content_format: ChatTemplateContentFormatOption = "auto",
191 + trust_request_chat_template: bool = False,
192 log_error_stack: bool = False,
193 ) -> None:
194 super().__init__(
@@ -139,6 +198,10 @@ class ServingClassification(ClassificationMixin):
198 log_error_stack=log_error_stack,
199 )
200
201 + self.chat_template = chat_template
202 + self.chat_template_content_format = chat_template_content_format
203 + self.trust_request_chat_template = trust_request_chat_template
204 +
205 async def create_classify(
206 self,
207 request: ClassificationRequest,
@@ -156,7 +219,6 @@ class ServingClassification(ClassificationMixin):
219
220 return await super().handle(ctx) # type: ignore
221
159 - @override
222 def _create_pooling_params(
223 self,
224 ctx: ClassificationServeContext,
vllm/entrypoints/openai/serving_engine.py
+16 -5
@@ -43,6 +43,8 @@ from vllm.entrypoints.openai.protocol import (
43 ChatCompletionNamedToolChoiceParam,
44 ChatCompletionRequest,
45 ChatCompletionResponse,
46 + ClassificationChatRequest,
47 + ClassificationCompletionRequest,
48 ClassificationRequest,
49 ClassificationResponse,
50 CompletionRequest,
@@ -114,13 +116,16 @@ CompletionLikeRequest: TypeAlias = (
116 | DetokenizeRequest
117 | EmbeddingCompletionRequest
118 | RerankRequest
117 - | ClassificationRequest
119 + | ClassificationCompletionRequest
120 | ScoreRequest
121 | TokenizeCompletionRequest
122 )
123
124 ChatLikeRequest: TypeAlias = (
123 - ChatCompletionRequest | EmbeddingChatRequest | TokenizeChatRequest
125 + ChatCompletionRequest
126 + | EmbeddingChatRequest
127 + | TokenizeChatRequest
128 + | ClassificationChatRequest
129 )
130 SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest
131 AnyRequest: TypeAlias = (
@@ -814,7 +819,11 @@ class OpenAIServing:
819 if not hasattr(request, "messages"):
820 return message_types
821
817 - for message in request.messages:
822 + messages = request.messages
823 + if messages is None or isinstance(messages, (str, bytes)):
824 + return message_types
825 +
826 + for message in messages:
827 if (
828 isinstance(message, dict)
829 and "content" in message
@@ -907,7 +916,8 @@ class OpenAIServing:
916 EmbeddingCompletionRequest,
917 ScoreRequest,
918 RerankRequest,
910 - ClassificationRequest,
919 + ClassificationCompletionRequest,
920 + ClassificationChatRequest,
921 ),
922 ):
923 # Note: input length can be up to the entire model context length
@@ -915,7 +925,8 @@ class OpenAIServing:
925 if token_num > self.max_model_len:
926 operations: dict[type[AnyRequest], str] = {
927 ScoreRequest: "score",
918 - ClassificationRequest: "classification",
928 + ClassificationCompletionRequest: "classification",
929 + ClassificationChatRequest: "classification",
930 }
931 operation = operations.get(type(request), "embedding generation")
932 raise ValueError(