@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c9461e05a4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

Support Anthropic API /v1/messages Endpoint (#22627)

Signed-off-by: liuli <ll407707@alibaba-inc.com> Co-authored-by: liuli <ll407707@alibaba-inc.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk> Co-authored-by: Michael Goin <mgoin64@gmail.com>

RED committed Oct 23, 2025 at 00:13 UTC c9461e05a4ed3557cfbf4b15ded1e26761cc39ca
10 files changed +1262 -46
requirements/common.txt
+1
@@ -48,3 +48,4 @@ pybase64 # fast base64 implementation
48 cbor2 # Required for cross-language serialization of hashable objects
49 setproctitle # Used to set process names for better debugging and monitoring
50 openai-harmony >= 0.0.3 # Required for gpt-oss
51 +anthropic == 0.71.0
tests/entrypoints/anthropic/__init__.py
tests/entrypoints/anthropic/test_messages.py new
+141
@@ -0,0 +1,141 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import anthropic
5 +import pytest
6 +import pytest_asyncio
7 +
8 +from ...utils import RemoteAnthropicServer
9 +
10 +MODEL_NAME = "Qwen/Qwen3-0.6B"
11 +
12 +
13 +@pytest.fixture(scope="module")
14 +def server(): # noqa: F811
15 + args = [
16 + "--max-model-len",
17 + "2048",
18 + "--enforce-eager",
19 + "--enable-auto-tool-choice",
20 + "--tool-call-parser",
21 + "hermes",
22 + "--served-model-name",
23 + "claude-3-7-sonnet-latest",
24 + ]
25 +
26 + with RemoteAnthropicServer(MODEL_NAME, args) as remote_server:
27 + yield remote_server
28 +
29 +
30 +@pytest_asyncio.fixture
31 +async def client(server):
32 + async with server.get_async_client() as async_client:
33 + yield async_client
34 +
35 +
36 +@pytest.mark.asyncio
37 +async def test_simple_messages(client: anthropic.AsyncAnthropic):
38 + resp = await client.messages.create(
39 + model="claude-3-7-sonnet-latest",
40 + max_tokens=1024,
41 + messages=[{"role": "user", "content": "how are you!"}],
42 + )
43 + assert resp.stop_reason == "end_turn"
44 + assert resp.role == "assistant"
45 +
46 + print(f"Anthropic response: {resp.model_dump_json()}")
47 +
48 +
49 +@pytest.mark.asyncio
50 +async def test_system_message(client: anthropic.AsyncAnthropic):
51 + resp = await client.messages.create(
52 + model="claude-3-7-sonnet-latest",
53 + max_tokens=1024,
54 + system="you are a helpful assistant",
55 + messages=[{"role": "user", "content": "how are you!"}],
56 + )
57 + assert resp.stop_reason == "end_turn"
58 + assert resp.role == "assistant"
59 +
60 + print(f"Anthropic response: {resp.model_dump_json()}")
61 +
62 +
63 +@pytest.mark.asyncio
64 +async def test_anthropic_streaming(client: anthropic.AsyncAnthropic):
65 + resp = await client.messages.create(
66 + model="claude-3-7-sonnet-latest",
67 + max_tokens=1024,
68 + messages=[{"role": "user", "content": "how are you!"}],
69 + stream=True,
70 + )
71 +
72 + async for chunk in resp:
73 + print(chunk.model_dump_json())
74 +
75 +
76 +@pytest.mark.asyncio
77 +async def test_anthropic_tool_call(client: anthropic.AsyncAnthropic):
78 + resp = await client.messages.create(
79 + model="claude-3-7-sonnet-latest",
80 + max_tokens=1024,
81 + messages=[
82 + {"role": "user", "content": "What's the weather like in New York today?"}
83 + ],
84 + tools=[
85 + {
86 + "name": "get_current_weather",
87 + "description": "Useful for querying the weather in a specified city.",
88 + "input_schema": {
89 + "type": "object",
90 + "properties": {
91 + "location": {
92 + "type": "string",
93 + "description": "City or region, for example: "
94 + "New York, London, Tokyo, etc.",
95 + }
96 + },
97 + "required": ["location"],
98 + },
99 + }
100 + ],
101 + stream=False,
102 + )
103 + assert resp.stop_reason == "tool_use"
104 + assert resp.role == "assistant"
105 +
106 + print(f"Anthropic response: {resp.model_dump_json()}")
107 +
108 + @pytest.mark.asyncio
109 + async def test_anthropic_tool_call_streaming(client: anthropic.AsyncAnthropic):
110 + resp = await client.messages.create(
111 + model="claude-3-7-sonnet-latest",
112 + max_tokens=1024,
113 + messages=[
114 + {
115 + "role": "user",
116 + "content": "What's the weather like in New York today?",
117 + }
118 + ],
119 + tools=[
120 + {
121 + "name": "get_current_weather",
122 + "description": "Useful for querying the weather "
123 + "in a specified city.",
124 + "input_schema": {
125 + "type": "object",
126 + "properties": {
127 + "location": {
128 + "type": "string",
129 + "description": "City or region, for example: "
130 + "New York, London, Tokyo, etc.",
131 + }
132 + },
133 + "required": ["location"],
134 + },
135 + }
136 + ],
137 + stream=True,
138 + )
139 +
140 + async for chunk in resp:
141 + print(chunk.model_dump_json())
tests/utils.py
+126
@@ -23,6 +23,7 @@ from pathlib import Path
23 from typing import Any, Literal
24 from unittest.mock import patch
25
26 +import anthropic
27 import cloudpickle
28 import httpx
29 import openai
@@ -294,6 +295,131 @@ class RemoteOpenAIServerCustom(RemoteOpenAIServer):
295 self.proc.kill()
296
297
298 +class RemoteAnthropicServer:
299 + DUMMY_API_KEY = "token-abc123" # vLLM's Anthropic server does not need API key
300 +
301 + def __init__(
302 + self,
303 + model: str,
304 + vllm_serve_args: list[str],
305 + *,
306 + env_dict: dict[str, str] | None = None,
307 + seed: int | None = 0,
308 + auto_port: bool = True,
309 + max_wait_seconds: float | None = None,
310 + ) -> None:
311 + if auto_port:
312 + if "-p" in vllm_serve_args or "--port" in vllm_serve_args:
313 + raise ValueError(
314 + "You have manually specified the port when `auto_port=True`."
315 + )
316 +
317 + # Don't mutate the input args
318 + vllm_serve_args = vllm_serve_args + ["--port", str(get_open_port())]
319 + if seed is not None:
320 + if "--seed" in vllm_serve_args:
321 + raise ValueError(
322 + f"You have manually specified the seed when `seed={seed}`."
323 + )
324 +
325 + vllm_serve_args = vllm_serve_args + ["--seed", str(seed)]
326 +
327 + parser = FlexibleArgumentParser(description="vLLM's remote Anthropic server.")
328 + subparsers = parser.add_subparsers(required=False, dest="subparser")
329 + parser = ServeSubcommand().subparser_init(subparsers)
330 + args = parser.parse_args(["--model", model, *vllm_serve_args])
331 + self.host = str(args.host or "localhost")
332 + self.port = int(args.port)
333 +
334 + self.show_hidden_metrics = args.show_hidden_metrics_for_version is not None
335 +
336 + # download the model before starting the server to avoid timeout
337 + is_local = os.path.isdir(model)
338 + if not is_local:
339 + engine_args = AsyncEngineArgs.from_cli_args(args)
340 + model_config = engine_args.create_model_config()
341 + load_config = engine_args.create_load_config()
342 +
343 + model_loader = get_model_loader(load_config)
344 + model_loader.download_model(model_config)
345 +
346 + env = os.environ.copy()
347 + # the current process might initialize cuda,
348 + # to be safe, we should use spawn method
349 + env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
350 + if env_dict is not None:
351 + env.update(env_dict)
352 + self.proc = subprocess.Popen(
353 + [
354 + sys.executable,
355 + "-m",
356 + "vllm.entrypoints.anthropic.api_server",
357 + model,
358 + *vllm_serve_args,
359 + ],
360 + env=env,
361 + stdout=sys.stdout,
362 + stderr=sys.stderr,
363 + )
364 + max_wait_seconds = max_wait_seconds or 240
365 + self._wait_for_server(url=self.url_for("health"), timeout=max_wait_seconds)
366 +
367 + def __enter__(self):
368 + return self
369 +
370 + def __exit__(self, exc_type, exc_value, traceback):
371 + self.proc.terminate()
372 + try:
373 + self.proc.wait(8)
374 + except subprocess.TimeoutExpired:
375 + # force kill if needed
376 + self.proc.kill()
377 +
378 + def _wait_for_server(self, *, url: str, timeout: float):
379 + # run health check
380 + start = time.time()
381 + while True:
382 + try:
383 + if requests.get(url).status_code == 200:
384 + break
385 + except Exception:
386 + # this exception can only be raised by requests.get,
387 + # which means the server is not ready yet.
388 + # the stack trace is not useful, so we suppress it
389 + # by using `raise from None`.
390 + result = self.proc.poll()
391 + if result is not None and result != 0:
392 + raise RuntimeError("Server exited unexpectedly.") from None
393 +
394 + time.sleep(0.5)
395 + if time.time() - start > timeout:
396 + raise RuntimeError("Server failed to start in time.") from None
397 +
398 + @property
399 + def url_root(self) -> str:
400 + return f"http://{self.host}:{self.port}"
401 +
402 + def url_for(self, *parts: str) -> str:
403 + return self.url_root + "/" + "/".join(parts)
404 +
405 + def get_client(self, **kwargs):
406 + if "timeout" not in kwargs:
407 + kwargs["timeout"] = 600
408 + return anthropic.Anthropic(
409 + base_url=self.url_for(),
410 + api_key=self.DUMMY_API_KEY,
411 + max_retries=0,
412 + **kwargs,
413 + )
414 +
415 + def get_async_client(self, **kwargs):
416 + if "timeout" not in kwargs:
417 + kwargs["timeout"] = 600
418 + return anthropic.AsyncAnthropic(
419 + base_url=self.url_for(), api_key=self.DUMMY_API_KEY, max_retries=0, **kwargs
420 + )
421 +
422 +
423 def _test_completion(
424 client: openai.OpenAI,
425 model: str,
vllm/entrypoints/anthropic/__init__.py
vllm/entrypoints/anthropic/api_server.py new
+300
@@ -0,0 +1,300 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +# Adapted from:
4 +# https://github.com/vllm/vllm/entrypoints/openai/api_server.py
5 +
6 +import asyncio
7 +import signal
8 +import tempfile
9 +from argparse import Namespace
10 +from http import HTTPStatus
11 +
12 +import uvloop
13 +from fastapi import APIRouter, Depends, FastAPI, Request
14 +from fastapi.middleware.cors import CORSMiddleware
15 +from fastapi.responses import JSONResponse, Response, StreamingResponse
16 +from starlette.datastructures import State
17 +
18 +import vllm.envs as envs
19 +from vllm.engine.protocol import EngineClient
20 +from vllm.entrypoints.anthropic.protocol import (
21 + AnthropicErrorResponse,
22 + AnthropicMessagesRequest,
23 + AnthropicMessagesResponse,
24 +)
25 +from vllm.entrypoints.anthropic.serving_messages import AnthropicServingMessages
26 +from vllm.entrypoints.launcher import serve_http
27 +from vllm.entrypoints.logger import RequestLogger
28 +from vllm.entrypoints.openai.api_server import (
29 + build_async_engine_client,
30 + create_server_socket,
31 + lifespan,
32 + load_log_config,
33 + validate_api_server_args,
34 + validate_json_request,
35 +)
36 +from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
37 +from vllm.entrypoints.openai.protocol import ErrorResponse
38 +from vllm.entrypoints.openai.serving_models import (
39 + BaseModelPath,
40 + OpenAIServingModels,
41 +)
42 +
43 +#
44 +# yapf: enable
45 +from vllm.entrypoints.openai.tool_parsers import ToolParserManager
46 +from vllm.entrypoints.utils import (
47 + cli_env_setup,
48 + load_aware_call,
49 + process_chat_template,
50 + process_lora_modules,
51 + with_cancellation,
52 +)
53 +from vllm.logger import init_logger
54 +from vllm.utils import FlexibleArgumentParser, set_ulimit
55 +from vllm.utils.network_utils import is_valid_ipv6_address
56 +from vllm.version import __version__ as VLLM_VERSION
57 +
58 +prometheus_multiproc_dir: tempfile.TemporaryDirectory
59 +
60 +# Cannot use __name__ (https://github.com/vllm-project/vllm/pull/4765)
61 +logger = init_logger("vllm.entrypoints.anthropic.api_server")
62 +
63 +_running_tasks: set[asyncio.Task] = set()
64 +
65 +router = APIRouter()
66 +
67 +
68 +def messages(request: Request) -> AnthropicServingMessages:
69 + return request.app.state.anthropic_serving_messages
70 +
71 +
72 +def engine_client(request: Request) -> EngineClient:
73 + return request.app.state.engine_client
74 +
75 +
76 +@router.get("/health", response_class=Response)
77 +async def health(raw_request: Request) -> Response:
78 + """Health check."""
79 + await engine_client(raw_request).check_health()
80 + return Response(status_code=200)
81 +
82 +
83 +@router.get("/ping", response_class=Response)
84 +@router.post("/ping", response_class=Response)
85 +async def ping(raw_request: Request) -> Response:
86 + """Ping check. Endpoint required for SageMaker"""
87 + return await health(raw_request)
88 +
89 +
90 +@router.post(
91 + "/v1/messages",
92 + dependencies=[Depends(validate_json_request)],
93 + responses={
94 + HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
95 + HTTPStatus.BAD_REQUEST.value: {"model": AnthropicErrorResponse},
96 + HTTPStatus.NOT_FOUND.value: {"model": AnthropicErrorResponse},
97 + HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": AnthropicErrorResponse},
98 + },
99 +)
100 +@with_cancellation
101 +@load_aware_call
102 +async def create_messages(request: AnthropicMessagesRequest, raw_request: Request):
103 + handler = messages(raw_request)
104 + if handler is None:
105 + return messages(raw_request).create_error_response(
106 + message="The model does not support Messages API"
107 + )
108 +
109 + generator = await handler.create_messages(request, raw_request)
110 +
111 + if isinstance(generator, ErrorResponse):
112 + return JSONResponse(content=generator.model_dump())
113 +
114 + elif isinstance(generator, AnthropicMessagesResponse):
115 + logger.debug(
116 + "Anthropic Messages Response: %s", generator.model_dump(exclude_none=True)
117 + )
118 + return JSONResponse(content=generator.model_dump(exclude_none=True))
119 +
120 + return StreamingResponse(content=generator, media_type="text/event-stream")
121 +
122 +
123 +async def init_app_state(
124 + engine_client: EngineClient,
125 + state: State,
126 + args: Namespace,
127 +) -> None:
128 + vllm_config = engine_client.vllm_config
129 +
130 + if args.served_model_name is not None:
131 + served_model_names = args.served_model_name
132 + else:
133 + served_model_names = [args.model]
134 +
135 + if args.disable_log_requests:
136 + request_logger = None
137 + else:
138 + request_logger = RequestLogger(max_log_len=args.max_log_len)
139 +
140 + base_model_paths = [
141 + BaseModelPath(name=name, model_path=args.model) for name in served_model_names
142 + ]
143 +
144 + state.engine_client = engine_client
145 + state.log_stats = not args.disable_log_stats
146 + state.vllm_config = vllm_config
147 + model_config = vllm_config.model_config
148 +
149 + default_mm_loras = (
150 + vllm_config.lora_config.default_mm_loras
151 + if vllm_config.lora_config is not None
152 + else {}
153 + )
154 + lora_modules = process_lora_modules(args.lora_modules, default_mm_loras)
155 +
156 + resolved_chat_template = await process_chat_template(
157 + args.chat_template, engine_client, model_config
158 + )
159 +
160 + state.openai_serving_models = OpenAIServingModels(
161 + engine_client=engine_client,
162 + base_model_paths=base_model_paths,
163 + lora_modules=lora_modules,
164 + )
165 + await state.openai_serving_models.init_static_loras()
166 + state.anthropic_serving_messages = AnthropicServingMessages(
167 + engine_client,
168 + state.openai_serving_models,
169 + args.response_role,
170 + request_logger=request_logger,
171 + chat_template=resolved_chat_template,
172 + chat_template_content_format=args.chat_template_content_format,
173 + return_tokens_as_token_ids=args.return_tokens_as_token_ids,
174 + enable_auto_tools=args.enable_auto_tool_choice,
175 + tool_parser=args.tool_call_parser,
176 + reasoning_parser=args.reasoning_parser,
177 + enable_prompt_tokens_details=args.enable_prompt_tokens_details,
178 + enable_force_include_usage=args.enable_force_include_usage,
179 + )
180 +
181 +
182 +def setup_server(args):
183 + """Validate API server args, set up signal handler, create socket
184 + ready to serve."""
185 +
186 + logger.info("vLLM API server version %s", VLLM_VERSION)
187 +
188 + if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
189 + ToolParserManager.import_tool_parser(args.tool_parser_plugin)
190 +
191 + validate_api_server_args(args)
192 +
193 + # workaround to make sure that we bind the port before the engine is set up.
194 + # This avoids race conditions with ray.
195 + # see https://github.com/vllm-project/vllm/issues/8204
196 + sock_addr = (args.host or "", args.port)
197 + sock = create_server_socket(sock_addr)
198 +
199 + # workaround to avoid footguns where uvicorn drops requests with too
200 + # many concurrent requests active
201 + set_ulimit()
202 +
203 + def signal_handler(*_) -> None:
204 + # Interrupt server on sigterm while initializing
205 + raise KeyboardInterrupt("terminated")
206 +
207 + signal.signal(signal.SIGTERM, signal_handler)
208 +
209 + addr, port = sock_addr
210 + is_ssl = args.ssl_keyfile and args.ssl_certfile
211 + host_part = f"[{addr}]" if is_valid_ipv6_address(addr) else addr or "0.0.0.0"
212 + listen_address = f"http{'s' if is_ssl else ''}://{host_part}:{port}"
213 +
214 + return listen_address, sock
215 +
216 +
217 +async def run_server(args, **uvicorn_kwargs) -> None:
218 + """Run a single-worker API server."""
219 + listen_address, sock = setup_server(args)
220 + await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
221 +
222 +
223 +def build_app(args: Namespace) -> FastAPI:
224 + app = FastAPI(lifespan=lifespan)
225 + app.include_router(router)
226 + app.root_path = args.root_path
227 +
228 + app.add_middleware(
229 + CORSMiddleware,
230 + allow_origins=args.allowed_origins,
231 + allow_credentials=args.allow_credentials,
232 + allow_methods=args.allowed_methods,
233 + allow_headers=args.allowed_headers,
234 + )
235 +
236 + return app
237 +
238 +
239 +async def run_server_worker(
240 + listen_address, sock, args, client_config=None, **uvicorn_kwargs
241 +) -> None:
242 + """Run a single API server worker."""
243 +
244 + if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
245 + ToolParserManager.import_tool_parser(args.tool_parser_plugin)
246 +
247 + server_index = client_config.get("client_index", 0) if client_config else 0
248 +
249 + # Load logging config for uvicorn if specified
250 + log_config = load_log_config(args.log_config_file)
251 + if log_config is not None:
252 + uvicorn_kwargs["log_config"] = log_config
253 +
254 + async with build_async_engine_client(
255 + args,
256 + client_config=client_config,
257 + ) as engine_client:
258 + app = build_app(args)
259 +
260 + await init_app_state(engine_client, app.state, args)
261 +
262 + logger.info("Starting vLLM API server %d on %s", server_index, listen_address)
263 + shutdown_task = await serve_http(
264 + app,
265 + sock=sock,
266 + enable_ssl_refresh=args.enable_ssl_refresh,
267 + host=args.host,
268 + port=args.port,
269 + log_level=args.uvicorn_log_level,
270 + # NOTE: When the 'disable_uvicorn_access_log' value is True,
271 + # no access log will be output.
272 + access_log=not args.disable_uvicorn_access_log,
273 + timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
274 + ssl_keyfile=args.ssl_keyfile,
275 + ssl_certfile=args.ssl_certfile,
276 + ssl_ca_certs=args.ssl_ca_certs,
277 + ssl_cert_reqs=args.ssl_cert_reqs,
278 + **uvicorn_kwargs,
279 + )
280 +
281 + # NB: Await server shutdown only after the backend context is exited
282 + try:
283 + await shutdown_task
284 + finally:
285 + sock.close()
286 +
287 +
288 +if __name__ == "__main__":
289 + # NOTE(simon):
290 + # This section should be in sync with vllm/entrypoints/cli/main.py for CLI
291 + # entrypoints.
292 + cli_env_setup()
293 + parser = FlexibleArgumentParser(
294 + description="vLLM Anthropic-Compatible RESTful API server."
295 + )
296 + parser = make_arg_parser(parser)
297 + args = parser.parse_args()
298 + validate_parsed_serve_args(args)
299 +
300 + uvloop.run(run_server(args))
vllm/entrypoints/anthropic/protocol.py new
+162
@@ -0,0 +1,162 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Pydantic models for Anthropic API protocol"""
4 +
5 +import time
6 +from typing import Any, Literal, Optional
7 +
8 +from pydantic import BaseModel, field_validator
9 +
10 +
11 +class AnthropicError(BaseModel):
12 + """Error structure for Anthropic API"""
13 +
14 + type: str
15 + message: str
16 +
17 +
18 +class AnthropicErrorResponse(BaseModel):
19 + """Error response structure for Anthropic API"""
20 +
21 + type: Literal["error"] = "error"
22 + error: AnthropicError
23 +
24 +
25 +class AnthropicUsage(BaseModel):
26 + """Token usage information"""
27 +
28 + input_tokens: int
29 + output_tokens: int
30 + cache_creation_input_tokens: int | None = None
31 + cache_read_input_tokens: int | None = None
32 +
33 +
34 +class AnthropicContentBlock(BaseModel):
35 + """Content block in message"""
36 +
37 + type: Literal["text", "image", "tool_use", "tool_result"]
38 + text: str | None = None
39 + # For image content
40 + source: dict[str, Any] | None = None
41 + # For tool use/result
42 + id: str | None = None
43 + name: str | None = None
44 + input: dict[str, Any] | None = None
45 + content: str | list[dict[str, Any]] | None = None
46 + is_error: bool | None = None
47 +
48 +
49 +class AnthropicMessage(BaseModel):
50 + """Message structure"""
51 +
52 + role: Literal["user", "assistant"]
53 + content: str | list[AnthropicContentBlock]
54 +
55 +
56 +class AnthropicTool(BaseModel):
57 + """Tool definition"""
58 +
59 + name: str
60 + description: str | None = None
61 + input_schema: dict[str, Any]
62 +
63 + @field_validator("input_schema")
64 + @classmethod
65 + def validate_input_schema(cls, v):
66 + if not isinstance(v, dict):
67 + raise ValueError("input_schema must be a dictionary")
68 + if "type" not in v:
69 + v["type"] = "object" # Default to object type
70 + return v
71 +
72 +
73 +class AnthropicToolChoice(BaseModel):
74 + """Tool Choice definition"""
75 +
76 + type: Literal["auto", "any", "tool"]
77 + name: str | None = None
78 +
79 +
80 +class AnthropicMessagesRequest(BaseModel):
81 + """Anthropic Messages API request"""
82 +
83 + model: str
84 + messages: list[AnthropicMessage]
85 + max_tokens: int
86 + metadata: dict[str, Any] | None = None
87 + stop_sequences: list[str] | None = None
88 + stream: bool | None = False
89 + system: str | list[AnthropicContentBlock] | None = None
90 + temperature: float | None = None
91 + tool_choice: AnthropicToolChoice | None = None
92 + tools: list[AnthropicTool] | None = None
93 + top_k: int | None = None
94 + top_p: float | None = None
95 +
96 + @field_validator("model")
97 + @classmethod
98 + def validate_model(cls, v):
99 + if not v:
100 + raise ValueError("Model is required")
101 + return v
102 +
103 + @field_validator("max_tokens")
104 + @classmethod
105 + def validate_max_tokens(cls, v):
106 + if v <= 0:
107 + raise ValueError("max_tokens must be positive")
108 + return v
109 +
110 +
111 +class AnthropicDelta(BaseModel):
112 + """Delta for streaming responses"""
113 +
114 + type: Literal["text_delta", "input_json_delta"] | None = None
115 + text: str | None = None
116 + partial_json: str | None = None
117 +
118 + # Message delta
119 + stop_reason: (
120 + Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"] | None
121 + ) = None
122 + stop_sequence: str | None = None
123 +
124 +
125 +class AnthropicStreamEvent(BaseModel):
126 + """Streaming event"""
127 +
128 + type: Literal[
129 + "message_start",
130 + "message_delta",
131 + "message_stop",
132 + "content_block_start",
133 + "content_block_delta",
134 + "content_block_stop",
135 + "ping",
136 + "error",
137 + ]
138 + message: Optional["AnthropicMessagesResponse"] = None
139 + delta: AnthropicDelta | None = None
140 + content_block: AnthropicContentBlock | None = None
141 + index: int | None = None
142 + error: AnthropicError | None = None
143 + usage: AnthropicUsage | None = None
144 +
145 +
146 +class AnthropicMessagesResponse(BaseModel):
147 + """Anthropic Messages API response"""
148 +
149 + id: str
150 + type: Literal["message"] = "message"
151 + role: Literal["assistant"] = "assistant"
152 + content: list[AnthropicContentBlock]
153 + model: str
154 + stop_reason: (
155 + Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"] | None
156 + ) = None
157 + stop_sequence: str | None = None
158 + usage: AnthropicUsage | None = None
159 +
160 + def model_post_init(self, __context):
161 + if not self.id:
162 + self.id = f"msg_{int(time.time() * 1000)}"
vllm/entrypoints/anthropic/serving_messages.py new
+458
@@ -0,0 +1,458 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +# Adapted from
4 +# https://github.com/vllm/vllm/entrypoints/openai/serving_chat.py
5 +
6 +"""Anthropic Messages API serving handler"""
7 +
8 +import json
9 +import logging
10 +import time
11 +from collections.abc import AsyncGenerator
12 +from typing import Any
13 +
14 +from fastapi import Request
15 +
16 +from vllm.engine.protocol import EngineClient
17 +from vllm.entrypoints.anthropic.protocol import (
18 + AnthropicContentBlock,
19 + AnthropicDelta,
20 + AnthropicError,
21 + AnthropicMessagesRequest,
22 + AnthropicMessagesResponse,
23 + AnthropicStreamEvent,
24 + AnthropicUsage,
25 +)
26 +from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
27 +from vllm.entrypoints.logger import RequestLogger
28 +from vllm.entrypoints.openai.protocol import (
29 + ChatCompletionNamedToolChoiceParam,
30 + ChatCompletionRequest,
31 + ChatCompletionResponse,
32 + ChatCompletionStreamResponse,
33 + ChatCompletionToolsParam,
34 + ErrorResponse,
35 + StreamOptions,
36 +)
37 +from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
38 +from vllm.entrypoints.openai.serving_models import OpenAIServingModels
39 +
40 +logger = logging.getLogger(__name__)
41 +
42 +
43 +def wrap_data_with_event(data: str, event: str):
44 + return f"event: {event}\ndata: {data}\n\n"
45 +
46 +
47 +class AnthropicServingMessages(OpenAIServingChat):
48 + """Handler for Anthropic Messages API requests"""
49 +
50 + def __init__(
51 + self,
52 + engine_client: EngineClient,
53 + models: OpenAIServingModels,
54 + response_role: str,
55 + *,
56 + request_logger: RequestLogger | None,
57 + chat_template: str | None,
58 + chat_template_content_format: ChatTemplateContentFormatOption,
59 + return_tokens_as_token_ids: bool = False,
60 + reasoning_parser: str = "",
61 + enable_auto_tools: bool = False,
62 + tool_parser: str | None = None,
63 + enable_prompt_tokens_details: bool = False,
64 + enable_force_include_usage: bool = False,
65 + ):
66 + super().__init__(
67 + engine_client=engine_client,
68 + models=models,
69 + response_role=response_role,
70 + request_logger=request_logger,
71 + chat_template=chat_template,
72 + chat_template_content_format=chat_template_content_format,
73 + return_tokens_as_token_ids=return_tokens_as_token_ids,
74 + reasoning_parser=reasoning_parser,
75 + enable_auto_tools=enable_auto_tools,
76 + tool_parser=tool_parser,
77 + enable_prompt_tokens_details=enable_prompt_tokens_details,
78 + enable_force_include_usage=enable_force_include_usage,
79 + )
80 + self.stop_reason_map = {
81 + "stop": "end_turn",
82 + "length": "max_tokens",
83 + "tool_calls": "tool_use",
84 + }
85 +
86 + def _convert_anthropic_to_openai_request(
87 + self, anthropic_request: AnthropicMessagesRequest
88 + ) -> ChatCompletionRequest:
89 + """Convert Anthropic message format to OpenAI format"""
90 + openai_messages = []
91 +
92 + # Add system message if provided
93 + if anthropic_request.system:
94 + if isinstance(anthropic_request.system, str):
95 + openai_messages.append(
96 + {"role": "system", "content": anthropic_request.system}
97 + )
98 + else:
99 + system_prompt = ""
100 + for block in anthropic_request.system:
101 + if block.type == "text" and block.text:
102 + system_prompt += block.text
103 + openai_messages.append({"role": "system", "content": system_prompt})
104 +
105 + for msg in anthropic_request.messages:
106 + openai_msg: dict[str, Any] = {"role": msg.role} # type: ignore
107 + if isinstance(msg.content, str):
108 + openai_msg["content"] = msg.content
109 + else:
110 + # Handle complex content blocks
111 + content_parts: list[dict[str, Any]] = []
112 + tool_calls: list[dict[str, Any]] = []
113 +
114 + for block in msg.content:
115 + if block.type == "text" and block.text:
116 + content_parts.append({"type": "text", "text": block.text})
117 + elif block.type == "image" and block.source:
118 + content_parts.append(
119 + {
120 + "type": "image_url",
121 + "image_url": {"url": block.source.get("data", "")},
122 + }
123 + )
124 + elif block.type == "tool_use":
125 + # Convert tool use to function call format
126 + tool_call = {
127 + "id": block.id or f"call_{int(time.time())}",
128 + "type": "function",
129 + "function": {
130 + "name": block.name or "",
131 + "arguments": json.dumps(block.input or {}),
132 + },
133 + }
134 + tool_calls.append(tool_call)
135 + elif block.type == "tool_result":
136 + if msg.role == "user":
137 + openai_messages.append(
138 + {
139 + "role": "tool",
140 + "tool_call_id": block.id or "",
141 + "content": str(block.content)
142 + if block.content
143 + else "",
144 + }
145 + )
146 + else:
147 + # Assistant tool result becomes regular text
148 + tool_result_text = (
149 + str(block.content) if block.content else ""
150 + )
151 + content_parts.append(
152 + {
153 + "type": "text",
154 + "text": f"Tool result: {tool_result_text}",
155 + }
156 + )
157 +
158 + # Add tool calls to the message if any
159 + if tool_calls:
160 + openai_msg["tool_calls"] = tool_calls # type: ignore
161 +
162 + # Add content parts if any
163 + if content_parts:
164 + if len(content_parts) == 1 and content_parts[0]["type"] == "text":
165 + openai_msg["content"] = content_parts[0]["text"]
166 + else:
167 + openai_msg["content"] = content_parts # type: ignore
168 + elif not tool_calls:
169 + continue
170 +
171 + openai_messages.append(openai_msg)
172 +
173 + req = ChatCompletionRequest(
174 + model=anthropic_request.model,
175 + messages=openai_messages,
176 + max_tokens=anthropic_request.max_tokens,
177 + max_completion_tokens=anthropic_request.max_tokens,
178 + stop=anthropic_request.stop_sequences,
179 + temperature=anthropic_request.temperature,
180 + top_p=anthropic_request.top_p,
181 + top_k=anthropic_request.top_k,
182 + )
183 +
184 + if anthropic_request.stream:
185 + req.stream = anthropic_request.stream
186 + req.stream_options = StreamOptions.validate({"include_usage": True})
187 +
188 + if anthropic_request.tool_choice is None:
189 + req.tool_choice = None
190 + elif anthropic_request.tool_choice.type == "auto":
191 + req.tool_choice = "auto"
192 + elif anthropic_request.tool_choice.type == "any":
193 + req.tool_choice = "required"
194 + elif anthropic_request.tool_choice.type == "tool":
195 + req.tool_choice = ChatCompletionNamedToolChoiceParam.model_validate(
196 + {
197 + "type": "function",
198 + "function": {"name": anthropic_request.tool_choice.name},
199 + }
200 + )
201 +
202 + tools = []
203 + if anthropic_request.tools is None:
204 + return req
205 + for tool in anthropic_request.tools:
206 + tools.append(
207 + ChatCompletionToolsParam.model_validate(
208 + {
209 + "type": "function",
210 + "function": {
211 + "name": tool.name,
212 + "description": tool.description,
213 + "parameters": tool.input_schema,
214 + },
215 + }
216 + )
217 + )
218 + if req.tool_choice is None:
219 + req.tool_choice = "auto"
220 + req.tools = tools
221 + return req
222 +
223 + async def create_messages(
224 + self,
225 + request: AnthropicMessagesRequest,
226 + raw_request: Request | None = None,
227 + ) -> AsyncGenerator[str, None] | AnthropicMessagesResponse | ErrorResponse:
228 + """
229 + Messages API similar to Anthropic's API.
230 +
231 + See https://docs.anthropic.com/en/api/messages
232 + for the API specification. This API mimics the Anthropic messages API.
233 + """
234 + logger.debug("Received messages request %s", request.model_dump_json())
235 + chat_req = self._convert_anthropic_to_openai_request(request)
236 + logger.debug("Convert to OpenAI request %s", request.model_dump_json())
237 + generator = await self.create_chat_completion(chat_req, raw_request)
238 +
239 + if isinstance(generator, ErrorResponse):
240 + return generator
241 +
242 + elif isinstance(generator, ChatCompletionResponse):
243 + return self.messages_full_converter(generator)
244 +
245 + return self.message_stream_converter(generator)
246 +
247 + def messages_full_converter(
248 + self,
249 + generator: ChatCompletionResponse,
250 + ) -> AnthropicMessagesResponse:
251 + result = AnthropicMessagesResponse(
252 + id=generator.id,
253 + content=[],
254 + model=generator.model,
255 + usage=AnthropicUsage(
256 + input_tokens=generator.usage.prompt_tokens,
257 + output_tokens=generator.usage.completion_tokens,
258 + ),
259 + )
260 + if generator.choices[0].finish_reason == "stop":
261 + result.stop_reason = "end_turn"
262 + elif generator.choices[0].finish_reason == "length":
263 + result.stop_reason = "max_tokens"
264 + elif generator.choices[0].finish_reason == "tool_calls":
265 + result.stop_reason = "tool_use"
266 +
267 + content: list[AnthropicContentBlock] = [
268 + AnthropicContentBlock(
269 + type="text",
270 + text=generator.choices[0].message.content
271 + if generator.choices[0].message.content
272 + else "",
273 + )
274 + ]
275 +
276 + for tool_call in generator.choices[0].message.tool_calls:
277 + anthropic_tool_call = AnthropicContentBlock(
278 + type="tool_use",
279 + id=tool_call.id,
280 + name=tool_call.function.name,
281 + input=json.loads(tool_call.function.arguments),
282 + )
283 + content += [anthropic_tool_call]
284 +
285 + result.content = content
286 +
287 + return result
288 +
289 + async def message_stream_converter(
290 + self,
291 + generator: AsyncGenerator[str, None],
292 + ) -> AsyncGenerator[str, None]:
293 + try:
294 + first_item = True
295 + finish_reason = None
296 + content_block_index = 0
297 + content_block_started = False
298 +
299 + async for item in generator:
300 + if item.startswith("data:"):
301 + data_str = item[5:].strip().rstrip("\n")
302 + if data_str == "[DONE]":
303 + stop_message = AnthropicStreamEvent(
304 + type="message_stop",
305 + )
306 + data = stop_message.model_dump_json(
307 + exclude_unset=True, exclude_none=True
308 + )
309 + yield wrap_data_with_event(data, "message_stop")
310 + yield "data: [DONE]\n\n"
311 + else:
312 + origin_chunk = ChatCompletionStreamResponse.model_validate_json(
313 + data_str
314 + )
315 +
316 + if first_item:
317 + chunk = AnthropicStreamEvent(
318 + type="message_start",
319 + message=AnthropicMessagesResponse(
320 + id=origin_chunk.id,
321 + content=[],
322 + model=origin_chunk.model,
323 + ),
324 + )
325 + first_item = False
326 + data = chunk.model_dump_json(exclude_unset=True)
327 + yield wrap_data_with_event(data, "message_start")
328 + continue
329 +
330 + # last chunk including usage info
331 + if len(origin_chunk.choices) == 0:
332 + if content_block_started:
333 + stop_chunk = AnthropicStreamEvent(
334 + index=content_block_index,
335 + type="content_block_stop",
336 + )
337 + data = stop_chunk.model_dump_json(exclude_unset=True)
338 + yield wrap_data_with_event(data, "content_block_stop")
339 + stop_reason = self.stop_reason_map.get(
340 + finish_reason or "stop"
341 + )
342 + chunk = AnthropicStreamEvent(
343 + type="message_delta",
344 + delta=AnthropicDelta(stop_reason=stop_reason),
345 + usage=AnthropicUsage(
346 + input_tokens=origin_chunk.usage.prompt_tokens
347 + if origin_chunk.usage
348 + else 0,
349 + output_tokens=origin_chunk.usage.completion_tokens
350 + if origin_chunk.usage
351 + else 0,
352 + ),
353 + )
354 + data = chunk.model_dump_json(exclude_unset=True)
355 + yield wrap_data_with_event(data, "message_delta")
356 + continue
357 +
358 + if origin_chunk.choices[0].finish_reason is not None:
359 + finish_reason = origin_chunk.choices[0].finish_reason
360 + continue
361 +
362 + # content
363 + if origin_chunk.choices[0].delta.content is not None:
364 + if not content_block_started:
365 + chunk = AnthropicStreamEvent(
366 + index=content_block_index,
367 + type="content_block_start",
368 + content_block=AnthropicContentBlock(
369 + type="text", text=""
370 + ),
371 + )
372 + data = chunk.model_dump_json(exclude_unset=True)
373 + yield wrap_data_with_event(data, "content_block_start")
374 + content_block_started = True
375 +
376 + if origin_chunk.choices[0].delta.content == "":
377 + continue
378 + chunk = AnthropicStreamEvent(
379 + index=content_block_index,
380 + type="content_block_delta",
381 + delta=AnthropicDelta(
382 + type="text_delta",
383 + text=origin_chunk.choices[0].delta.content,
384 + ),
385 + )
386 + data = chunk.model_dump_json(exclude_unset=True)
387 + yield wrap_data_with_event(data, "content_block_delta")
388 + continue
389 +
390 + # tool calls
391 + elif len(origin_chunk.choices[0].delta.tool_calls) > 0:
392 + tool_call = origin_chunk.choices[0].delta.tool_calls[0]
393 + if tool_call.id is not None:
394 + if content_block_started:
395 + stop_chunk = AnthropicStreamEvent(
396 + index=content_block_index,
397 + type="content_block_stop",
398 + )
399 + data = stop_chunk.model_dump_json(
400 + exclude_unset=True
401 + )
402 + yield wrap_data_with_event(
403 + data, "content_block_stop"
404 + )
405 + content_block_started = False
406 + content_block_index += 1
407 +
408 + chunk = AnthropicStreamEvent(
409 + index=content_block_index,
410 + type="content_block_start",
411 + content_block=AnthropicContentBlock(
412 + type="tool_use",
413 + id=tool_call.id,
414 + name=tool_call.function.name
415 + if tool_call.function
416 + else None,
417 + input={},
418 + ),
419 + )
420 + data = chunk.model_dump_json(exclude_unset=True)
421 + yield wrap_data_with_event(data, "content_block_start")
422 + content_block_started = True
423 +
424 + else:
425 + chunk = AnthropicStreamEvent(
426 + index=content_block_index,
427 + type="content_block_delta",
428 + delta=AnthropicDelta(
429 + type="input_json_delta",
430 + partial_json=tool_call.function.arguments
431 + if tool_call.function
432 + else None,
433 + ),
434 + )
435 + data = chunk.model_dump_json(exclude_unset=True)
436 + yield wrap_data_with_event(data, "content_block_delta")
437 + continue
438 + else:
439 + error_response = AnthropicStreamEvent(
440 + type="error",
441 + error=AnthropicError(
442 + type="internal_error",
443 + message="Invalid data format received",
444 + ),
445 + )
446 + data = error_response.model_dump_json(exclude_unset=True)
447 + yield wrap_data_with_event(data, "error")
448 + yield "data: [DONE]\n\n"
449 +
450 + except Exception as e:
451 + logger.exception("Error in message stream converter.")
452 + error_response = AnthropicStreamEvent(
453 + type="error",
454 + error=AnthropicError(type="internal_error", message=str(e)),
455 + )
456 + data = error_response.model_dump_json(exclude_unset=True)
457 + yield wrap_data_with_event(data, "error")
458 + yield "data: [DONE]\n\n"
vllm/entrypoints/openai/api_server.py
+11 -46
@@ -41,11 +41,6 @@ import vllm.envs as envs
41 from vllm.config import VllmConfig
42 from vllm.engine.arg_utils import AsyncEngineArgs
43 from vllm.engine.protocol import EngineClient
44 -from vllm.entrypoints.chat_utils import (
45 - load_chat_template,
46 - resolve_hf_chat_template,
47 - resolve_mistral_chat_template,
48 -)
44 from vllm.entrypoints.launcher import serve_http
45 from vllm.entrypoints.logger import RequestLogger
46 from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
@@ -90,7 +85,6 @@ from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
85 from vllm.entrypoints.openai.serving_engine import OpenAIServing
86 from vllm.entrypoints.openai.serving_models import (
87 BaseModelPath,
93 - LoRAModulePath,
88 OpenAIServingModels,
89 )
90 from vllm.entrypoints.openai.serving_pooling import OpenAIServingPooling
@@ -107,11 +101,12 @@ from vllm.entrypoints.utils import (
101 cli_env_setup,
102 load_aware_call,
103 log_non_default_args,
104 + process_chat_template,
105 + process_lora_modules,
106 with_cancellation,
107 )
108 from vllm.logger import init_logger
109 from vllm.reasoning import ReasoningParserManager
114 -from vllm.transformers_utils.tokenizer import MistralTokenizer
110 from vllm.usage.usage_lib import UsageContext
111 from vllm.utils import (
112 Device,
@@ -1655,32 +1650,9 @@ async def init_app_state(
1650 supported_tasks = await engine_client.get_supported_tasks()
1651 logger.info("Supported tasks: %s", supported_tasks)
1652
1658 - resolved_chat_template = load_chat_template(args.chat_template)
1659 - if resolved_chat_template is not None:
1660 - # Get the tokenizer to check official template
1661 - tokenizer = await engine_client.get_tokenizer()
1662 -
1663 - if isinstance(tokenizer, MistralTokenizer):
1664 - # The warning is logged in resolve_mistral_chat_template.
1665 - resolved_chat_template = resolve_mistral_chat_template(
1666 - chat_template=resolved_chat_template
1667 - )
1668 - else:
1669 - hf_chat_template = resolve_hf_chat_template(
1670 - tokenizer=tokenizer,
1671 - chat_template=None,
1672 - tools=None,
1673 - model_config=vllm_config.model_config,
1674 - )
1675 -
1676 - if hf_chat_template != resolved_chat_template:
1677 - logger.warning(
1678 - "Using supplied chat template: %s\n"
1679 - "It is different from official chat template '%s'. "
1680 - "This discrepancy may lead to performance degradation.",
1681 - resolved_chat_template,
1682 - args.model,
1683 - )
1653 + resolved_chat_template = await process_chat_template(
1654 + args.chat_template, engine_client, vllm_config.model_config
1655 + )
1656
1657 if args.tool_server == "demo":
1658 tool_server: ToolServer | None = DemoToolServer()
@@ -1699,19 +1671,12 @@ async def init_app_state(
1671 else {}
1672 )
1673
1702 - lora_modules = args.lora_modules
1703 - if default_mm_loras:
1704 - default_mm_lora_paths = [
1705 - LoRAModulePath(
1706 - name=modality,
1707 - path=lora_path,
1708 - )
1709 - for modality, lora_path in default_mm_loras.items()
1710 - ]
1711 - if args.lora_modules is None:
1712 - lora_modules = default_mm_lora_paths
1713 - else:
1714 - lora_modules += default_mm_lora_paths
1674 + default_mm_loras = (
1675 + vllm_config.lora_config.default_mm_loras
1676 + if vllm_config.lora_config is not None
1677 + else {}
1678 + )
1679 + lora_modules = process_lora_modules(args.lora_modules, default_mm_loras)
1680
1681 state.openai_serving_models = OpenAIServingModels(
1682 engine_client=engine_client,
vllm/entrypoints/utils.py
+63
@@ -6,21 +6,31 @@ import dataclasses
6 import functools
7 import os
8 from argparse import Namespace
9 +from pathlib import Path
10 from typing import Any
11
12 from fastapi import Request
13 from fastapi.responses import JSONResponse, StreamingResponse
14 from starlette.background import BackgroundTask, BackgroundTasks
15
16 +from vllm.config import ModelConfig
17 from vllm.engine.arg_utils import EngineArgs
18 +from vllm.engine.protocol import EngineClient
19 +from vllm.entrypoints.chat_utils import (
20 + load_chat_template,
21 + resolve_hf_chat_template,
22 + resolve_mistral_chat_template,
23 +)
24 from vllm.entrypoints.openai.cli_args import make_arg_parser
25 from vllm.entrypoints.openai.protocol import (
26 ChatCompletionRequest,
27 CompletionRequest,
28 StreamOptions,
29 )
30 +from vllm.entrypoints.openai.serving_models import LoRAModulePath
31 from vllm.logger import init_logger
32 from vllm.platforms import current_platform
33 +from vllm.transformers_utils.tokenizers import MistralTokenizer
34 from vllm.utils import FlexibleArgumentParser
35
36 logger = init_logger(__name__)
@@ -254,3 +264,56 @@ def should_include_usage(
264 else:
265 include_usage, include_continuous_usage = enable_force_include_usage, False
266 return include_usage, include_continuous_usage
267 +
268 +
269 +def process_lora_modules(
270 + args_lora_modules: list[LoRAModulePath], default_mm_loras: dict[str, str] | None
271 +) -> list[LoRAModulePath]:
272 + lora_modules = args_lora_modules
273 + if default_mm_loras:
274 + default_mm_lora_paths = [
275 + LoRAModulePath(
276 + name=modality,
277 + path=lora_path,
278 + )
279 + for modality, lora_path in default_mm_loras.items()
280 + ]
281 + if args_lora_modules is None:
282 + lora_modules = default_mm_lora_paths
283 + else:
284 + lora_modules += default_mm_lora_paths
285 + return lora_modules
286 +
287 +
288 +async def process_chat_template(
289 + args_chat_template: Path | str | None,
290 + engine_client: EngineClient,
291 + model_config: ModelConfig,
292 +) -> str | None:
293 + resolved_chat_template = load_chat_template(args_chat_template)
294 + if resolved_chat_template is not None:
295 + # Get the tokenizer to check official template
296 + tokenizer = await engine_client.get_tokenizer()
297 +
298 + if isinstance(tokenizer, MistralTokenizer):
299 + # The warning is logged in resolve_mistral_chat_template.
300 + resolved_chat_template = resolve_mistral_chat_template(
301 + chat_template=resolved_chat_template
302 + )
303 + else:
304 + hf_chat_template = resolve_hf_chat_template(
305 + tokenizer=tokenizer,
306 + chat_template=None,
307 + tools=None,
308 + model_config=model_config,
309 + )
310 +
311 + if hf_chat_template != resolved_chat_template:
312 + logger.warning(
313 + "Using supplied chat template: %s\n"
314 + "It is different from official chat template '%s'. "
315 + "This discrepancy may lead to performance degradation.",
316 + resolved_chat_template,
317 + model_config.model,
318 + )
319 + return resolved_chat_template