@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1e88fb751b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 13, 22:00 UTC

Adds anthropic /v1/messages endpoint to openai api_server (#27882)

Signed-off-by: bbartels <benjamin@bartels.dev> Signed-off-by: Benjamin Bartels <benjamin@bartels.dev>

Benjamin Bartels committed Nov 1, 2025 at 19:45 UTC 1e88fb751bce13c74355d177fd06035858ce77c4
5 files changed +138 -461
tests/entrypoints/anthropic/__init__.py
tests/entrypoints/openai/test_messages.py renamed
+35 -35
@@ -5,7 +5,7 @@ import anthropic
5 import pytest
6 import pytest_asyncio
7
8 -from ...utils import RemoteAnthropicServer
8 +from ...utils import RemoteOpenAIServer
9
10 MODEL_NAME = "Qwen/Qwen3-0.6B"
11
@@ -23,13 +23,13 @@ def server(): # noqa: F811
23 "claude-3-7-sonnet-latest",
24 ]
25
26 - with RemoteAnthropicServer(MODEL_NAME, args) as remote_server:
26 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
27 yield remote_server
28
29
30 @pytest_asyncio.fixture
31 async def client(server):
32 - async with server.get_async_client() as async_client:
32 + async with server.get_async_client_anthropic() as async_client:
33 yield async_client
34
35
@@ -105,37 +105,37 @@ async def test_anthropic_tool_call(client: anthropic.AsyncAnthropic):
105
106 print(f"Anthropic response: {resp.model_dump_json()}")
107
108 - @pytest.mark.asyncio
109 - async def test_anthropic_tool_call_streaming(client: anthropic.AsyncAnthropic):
110 - resp = await client.messages.create(
111 - model="claude-3-7-sonnet-latest",
112 - max_tokens=1024,
113 - messages=[
114 - {
115 - "role": "user",
116 - "content": "What's the weather like in New York today?",
117 - }
118 - ],
119 - tools=[
120 - {
121 - "name": "get_current_weather",
122 - "description": "Useful for querying the weather "
123 - "in a specified city.",
124 - "input_schema": {
125 - "type": "object",
126 - "properties": {
127 - "location": {
128 - "type": "string",
129 - "description": "City or region, for example: "
130 - "New York, London, Tokyo, etc.",
131 - }
132 - },
133 - "required": ["location"],
108 +
109 +@pytest.mark.asyncio
110 +async def test_anthropic_tool_call_streaming(client: anthropic.AsyncAnthropic):
111 + resp = await client.messages.create(
112 + model="claude-3-7-sonnet-latest",
113 + max_tokens=1024,
114 + messages=[
115 + {
116 + "role": "user",
117 + "content": "What's the weather like in New York today?",
118 + }
119 + ],
120 + tools=[
121 + {
122 + "name": "get_current_weather",
123 + "description": "Useful for querying the weather in a specified city.",
124 + "input_schema": {
125 + "type": "object",
126 + "properties": {
127 + "location": {
128 + "type": "string",
129 + "description": "City or region, for example: "
130 + "New York, London, Tokyo, etc.",
131 + }
132 },
135 - }
136 - ],
137 - stream=True,
138 - )
133 + "required": ["location"],
134 + },
135 + }
136 + ],
137 + stream=True,
138 + )
139
140 - async for chunk in resp:
141 - print(chunk.model_dump_json())
140 + async for chunk in resp:
141 + print(chunk.model_dump_json())
tests/utils.py
+17 -125
@@ -247,6 +247,23 @@ class RemoteOpenAIServer:
247 **kwargs,
248 )
249
250 + def get_client_anthropic(self, **kwargs):
251 + if "timeout" not in kwargs:
252 + kwargs["timeout"] = 600
253 + return anthropic.Anthropic(
254 + base_url=self.url_for(),
255 + api_key=self.DUMMY_API_KEY,
256 + max_retries=0,
257 + **kwargs,
258 + )
259 +
260 + def get_async_client_anthropic(self, **kwargs):
261 + if "timeout" not in kwargs:
262 + kwargs["timeout"] = 600
263 + return anthropic.AsyncAnthropic(
264 + base_url=self.url_for(), api_key=self.DUMMY_API_KEY, max_retries=0, **kwargs
265 + )
266 +
267
268 class RemoteOpenAIServerCustom(RemoteOpenAIServer):
269 """Launch test server with custom child process"""
@@ -293,131 +310,6 @@ class RemoteOpenAIServerCustom(RemoteOpenAIServer):
310 self.proc.kill()
311
312
296 -class RemoteAnthropicServer:
297 - DUMMY_API_KEY = "token-abc123" # vLLM's Anthropic server does not need API key
298 -
299 - def __init__(
300 - self,
301 - model: str,
302 - vllm_serve_args: list[str],
303 - *,
304 - env_dict: dict[str, str] | None = None,
305 - seed: int | None = 0,
306 - auto_port: bool = True,
307 - max_wait_seconds: float | None = None,
308 - ) -> None:
309 - if auto_port:
310 - if "-p" in vllm_serve_args or "--port" in vllm_serve_args:
311 - raise ValueError(
312 - "You have manually specified the port when `auto_port=True`."
313 - )
314 -
315 - # Don't mutate the input args
316 - vllm_serve_args = vllm_serve_args + ["--port", str(get_open_port())]
317 - if seed is not None:
318 - if "--seed" in vllm_serve_args:
319 - raise ValueError(
320 - f"You have manually specified the seed when `seed={seed}`."
321 - )
322 -
323 - vllm_serve_args = vllm_serve_args + ["--seed", str(seed)]
324 -
325 - parser = FlexibleArgumentParser(description="vLLM's remote Anthropic server.")
326 - subparsers = parser.add_subparsers(required=False, dest="subparser")
327 - parser = ServeSubcommand().subparser_init(subparsers)
328 - args = parser.parse_args(["--model", model, *vllm_serve_args])
329 - self.host = str(args.host or "localhost")
330 - self.port = int(args.port)
331 -
332 - self.show_hidden_metrics = args.show_hidden_metrics_for_version is not None
333 -
334 - # download the model before starting the server to avoid timeout
335 - is_local = os.path.isdir(model)
336 - if not is_local:
337 - engine_args = AsyncEngineArgs.from_cli_args(args)
338 - model_config = engine_args.create_model_config()
339 - load_config = engine_args.create_load_config()
340 -
341 - model_loader = get_model_loader(load_config)
342 - model_loader.download_model(model_config)
343 -
344 - env = os.environ.copy()
345 - # the current process might initialize cuda,
346 - # to be safe, we should use spawn method
347 - env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
348 - if env_dict is not None:
349 - env.update(env_dict)
350 - self.proc = subprocess.Popen(
351 - [
352 - sys.executable,
353 - "-m",
354 - "vllm.entrypoints.anthropic.api_server",
355 - model,
356 - *vllm_serve_args,
357 - ],
358 - env=env,
359 - stdout=sys.stdout,
360 - stderr=sys.stderr,
361 - )
362 - max_wait_seconds = max_wait_seconds or 240
363 - self._wait_for_server(url=self.url_for("health"), timeout=max_wait_seconds)
364 -
365 - def __enter__(self):
366 - return self
367 -
368 - def __exit__(self, exc_type, exc_value, traceback):
369 - self.proc.terminate()
370 - try:
371 - self.proc.wait(8)
372 - except subprocess.TimeoutExpired:
373 - # force kill if needed
374 - self.proc.kill()
375 -
376 - def _wait_for_server(self, *, url: str, timeout: float):
377 - # run health check
378 - start = time.time()
379 - while True:
380 - try:
381 - if requests.get(url).status_code == 200:
382 - break
383 - except Exception:
384 - # this exception can only be raised by requests.get,
385 - # which means the server is not ready yet.
386 - # the stack trace is not useful, so we suppress it
387 - # by using `raise from None`.
388 - result = self.proc.poll()
389 - if result is not None and result != 0:
390 - raise RuntimeError("Server exited unexpectedly.") from None
391 -
392 - time.sleep(0.5)
393 - if time.time() - start > timeout:
394 - raise RuntimeError("Server failed to start in time.") from None
395 -
396 - @property
397 - def url_root(self) -> str:
398 - return f"http://{self.host}:{self.port}"
399 -
400 - def url_for(self, *parts: str) -> str:
401 - return self.url_root + "/" + "/".join(parts)
402 -
403 - def get_client(self, **kwargs):
404 - if "timeout" not in kwargs:
405 - kwargs["timeout"] = 600
406 - return anthropic.Anthropic(
407 - base_url=self.url_for(),
408 - api_key=self.DUMMY_API_KEY,
409 - max_retries=0,
410 - **kwargs,
411 - )
412 -
413 - def get_async_client(self, **kwargs):
414 - if "timeout" not in kwargs:
415 - kwargs["timeout"] = 600
416 - return anthropic.AsyncAnthropic(
417 - base_url=self.url_for(), api_key=self.DUMMY_API_KEY, max_retries=0, **kwargs
418 - )
419 -
420 -
313 def _test_completion(
314 client: openai.OpenAI,
315 model: str,
vllm/entrypoints/anthropic/api_server.py deleted
-301
@@ -1,301 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -# Adapted from:
4 -# https://github.com/vllm/vllm/entrypoints/openai/api_server.py
5 -
6 -import asyncio
7 -import signal
8 -import tempfile
9 -from argparse import Namespace
10 -from http import HTTPStatus
11 -
12 -import uvloop
13 -from fastapi import APIRouter, Depends, FastAPI, Request
14 -from fastapi.middleware.cors import CORSMiddleware
15 -from fastapi.responses import JSONResponse, Response, StreamingResponse
16 -from starlette.datastructures import State
17 -
18 -import vllm.envs as envs
19 -from vllm.engine.protocol import EngineClient
20 -from vllm.entrypoints.anthropic.protocol import (
21 - AnthropicErrorResponse,
22 - AnthropicMessagesRequest,
23 - AnthropicMessagesResponse,
24 -)
25 -from vllm.entrypoints.anthropic.serving_messages import AnthropicServingMessages
26 -from vllm.entrypoints.launcher import serve_http
27 -from vllm.entrypoints.logger import RequestLogger
28 -from vllm.entrypoints.openai.api_server import (
29 - build_async_engine_client,
30 - create_server_socket,
31 - lifespan,
32 - load_log_config,
33 - validate_api_server_args,
34 - validate_json_request,
35 -)
36 -from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
37 -from vllm.entrypoints.openai.protocol import ErrorResponse
38 -from vllm.entrypoints.openai.serving_models import (
39 - BaseModelPath,
40 - OpenAIServingModels,
41 -)
42 -
43 -#
44 -# yapf: enable
45 -from vllm.entrypoints.openai.tool_parsers import ToolParserManager
46 -from vllm.entrypoints.utils import (
47 - cli_env_setup,
48 - load_aware_call,
49 - process_chat_template,
50 - process_lora_modules,
51 - with_cancellation,
52 -)
53 -from vllm.logger import init_logger
54 -from vllm.utils.argparse_utils import FlexibleArgumentParser
55 -from vllm.utils.network_utils import is_valid_ipv6_address
56 -from vllm.utils.system_utils import set_ulimit
57 -from vllm.version import __version__ as VLLM_VERSION
58 -
59 -prometheus_multiproc_dir: tempfile.TemporaryDirectory
60 -
61 -# Cannot use __name__ (https://github.com/vllm-project/vllm/pull/4765)
62 -logger = init_logger("vllm.entrypoints.anthropic.api_server")
63 -
64 -_running_tasks: set[asyncio.Task] = set()
65 -
66 -router = APIRouter()
67 -
68 -
69 -def messages(request: Request) -> AnthropicServingMessages:
70 - return request.app.state.anthropic_serving_messages
71 -
72 -
73 -def engine_client(request: Request) -> EngineClient:
74 - return request.app.state.engine_client
75 -
76 -
77 -@router.get("/health", response_class=Response)
78 -async def health(raw_request: Request) -> Response:
79 - """Health check."""
80 - await engine_client(raw_request).check_health()
81 - return Response(status_code=200)
82 -
83 -
84 -@router.get("/ping", response_class=Response)
85 -@router.post("/ping", response_class=Response)
86 -async def ping(raw_request: Request) -> Response:
87 - """Ping check. Endpoint required for SageMaker"""
88 - return await health(raw_request)
89 -
90 -
91 -@router.post(
92 - "/v1/messages",
93 - dependencies=[Depends(validate_json_request)],
94 - responses={
95 - HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
96 - HTTPStatus.BAD_REQUEST.value: {"model": AnthropicErrorResponse},
97 - HTTPStatus.NOT_FOUND.value: {"model": AnthropicErrorResponse},
98 - HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": AnthropicErrorResponse},
99 - },
100 -)
101 -@with_cancellation
102 -@load_aware_call
103 -async def create_messages(request: AnthropicMessagesRequest, raw_request: Request):
104 - handler = messages(raw_request)
105 - if handler is None:
106 - return messages(raw_request).create_error_response(
107 - message="The model does not support Messages API"
108 - )
109 -
110 - generator = await handler.create_messages(request, raw_request)
111 -
112 - if isinstance(generator, ErrorResponse):
113 - return JSONResponse(content=generator.model_dump())
114 -
115 - elif isinstance(generator, AnthropicMessagesResponse):
116 - logger.debug(
117 - "Anthropic Messages Response: %s", generator.model_dump(exclude_none=True)
118 - )
119 - return JSONResponse(content=generator.model_dump(exclude_none=True))
120 -
121 - return StreamingResponse(content=generator, media_type="text/event-stream")
122 -
123 -
124 -async def init_app_state(
125 - engine_client: EngineClient,
126 - state: State,
127 - args: Namespace,
128 -) -> None:
129 - vllm_config = engine_client.vllm_config
130 -
131 - if args.served_model_name is not None:
132 - served_model_names = args.served_model_name
133 - else:
134 - served_model_names = [args.model]
135 -
136 - if args.disable_log_requests:
137 - request_logger = None
138 - else:
139 - request_logger = RequestLogger(max_log_len=args.max_log_len)
140 -
141 - base_model_paths = [
142 - BaseModelPath(name=name, model_path=args.model) for name in served_model_names
143 - ]
144 -
145 - state.engine_client = engine_client
146 - state.log_stats = not args.disable_log_stats
147 - state.vllm_config = vllm_config
148 - model_config = vllm_config.model_config
149 -
150 - default_mm_loras = (
151 - vllm_config.lora_config.default_mm_loras
152 - if vllm_config.lora_config is not None
153 - else {}
154 - )
155 - lora_modules = process_lora_modules(args.lora_modules, default_mm_loras)
156 -
157 - resolved_chat_template = await process_chat_template(
158 - args.chat_template, engine_client, model_config
159 - )
160 -
161 - state.openai_serving_models = OpenAIServingModels(
162 - engine_client=engine_client,
163 - base_model_paths=base_model_paths,
164 - lora_modules=lora_modules,
165 - )
166 - await state.openai_serving_models.init_static_loras()
167 - state.anthropic_serving_messages = AnthropicServingMessages(
168 - engine_client,
169 - state.openai_serving_models,
170 - args.response_role,
171 - request_logger=request_logger,
172 - chat_template=resolved_chat_template,
173 - chat_template_content_format=args.chat_template_content_format,
174 - return_tokens_as_token_ids=args.return_tokens_as_token_ids,
175 - enable_auto_tools=args.enable_auto_tool_choice,
176 - tool_parser=args.tool_call_parser,
177 - reasoning_parser=args.reasoning_parser,
178 - enable_prompt_tokens_details=args.enable_prompt_tokens_details,
179 - enable_force_include_usage=args.enable_force_include_usage,
180 - )
181 -
182 -
183 -def setup_server(args):
184 - """Validate API server args, set up signal handler, create socket
185 - ready to serve."""
186 -
187 - logger.info("vLLM API server version %s", VLLM_VERSION)
188 -
189 - if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
190 - ToolParserManager.import_tool_parser(args.tool_parser_plugin)
191 -
192 - validate_api_server_args(args)
193 -
194 - # workaround to make sure that we bind the port before the engine is set up.
195 - # This avoids race conditions with ray.
196 - # see https://github.com/vllm-project/vllm/issues/8204
197 - sock_addr = (args.host or "", args.port)
198 - sock = create_server_socket(sock_addr)
199 -
200 - # workaround to avoid footguns where uvicorn drops requests with too
201 - # many concurrent requests active
202 - set_ulimit()
203 -
204 - def signal_handler(*_) -> None:
205 - # Interrupt server on sigterm while initializing
206 - raise KeyboardInterrupt("terminated")
207 -
208 - signal.signal(signal.SIGTERM, signal_handler)
209 -
210 - addr, port = sock_addr
211 - is_ssl = args.ssl_keyfile and args.ssl_certfile
212 - host_part = f"[{addr}]" if is_valid_ipv6_address(addr) else addr or "0.0.0.0"
213 - listen_address = f"http{'s' if is_ssl else ''}://{host_part}:{port}"
214 -
215 - return listen_address, sock
216 -
217 -
218 -async def run_server(args, **uvicorn_kwargs) -> None:
219 - """Run a single-worker API server."""
220 - listen_address, sock = setup_server(args)
221 - await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
222 -
223 -
224 -def build_app(args: Namespace) -> FastAPI:
225 - app = FastAPI(lifespan=lifespan)
226 - app.include_router(router)
227 - app.root_path = args.root_path
228 -
229 - app.add_middleware(
230 - CORSMiddleware,
231 - allow_origins=args.allowed_origins,
232 - allow_credentials=args.allow_credentials,
233 - allow_methods=args.allowed_methods,
234 - allow_headers=args.allowed_headers,
235 - )
236 -
237 - return app
238 -
239 -
240 -async def run_server_worker(
241 - listen_address, sock, args, client_config=None, **uvicorn_kwargs
242 -) -> None:
243 - """Run a single API server worker."""
244 -
245 - if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
246 - ToolParserManager.import_tool_parser(args.tool_parser_plugin)
247 -
248 - server_index = client_config.get("client_index", 0) if client_config else 0
249 -
250 - # Load logging config for uvicorn if specified
251 - log_config = load_log_config(args.log_config_file)
252 - if log_config is not None:
253 - uvicorn_kwargs["log_config"] = log_config
254 -
255 - async with build_async_engine_client(
256 - args,
257 - client_config=client_config,
258 - ) as engine_client:
259 - app = build_app(args)
260 -
261 - await init_app_state(engine_client, app.state, args)
262 -
263 - logger.info("Starting vLLM API server %d on %s", server_index, listen_address)
264 - shutdown_task = await serve_http(
265 - app,
266 - sock=sock,
267 - enable_ssl_refresh=args.enable_ssl_refresh,
268 - host=args.host,
269 - port=args.port,
270 - log_level=args.uvicorn_log_level,
271 - # NOTE: When the 'disable_uvicorn_access_log' value is True,
272 - # no access log will be output.
273 - access_log=not args.disable_uvicorn_access_log,
274 - timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
275 - ssl_keyfile=args.ssl_keyfile,
276 - ssl_certfile=args.ssl_certfile,
277 - ssl_ca_certs=args.ssl_ca_certs,
278 - ssl_cert_reqs=args.ssl_cert_reqs,
279 - **uvicorn_kwargs,
280 - )
281 -
282 - # NB: Await server shutdown only after the backend context is exited
283 - try:
284 - await shutdown_task
285 - finally:
286 - sock.close()
287 -
288 -
289 -if __name__ == "__main__":
290 - # NOTE(simon):
291 - # This section should be in sync with vllm/entrypoints/cli/main.py for CLI
292 - # entrypoints.
293 - cli_env_setup()
294 - parser = FlexibleArgumentParser(
295 - description="vLLM Anthropic-Compatible RESTful API server."
296 - )
297 - parser = make_arg_parser(parser)
298 - args = parser.parse_args()
299 - validate_parsed_serve_args(args)
300 -
301 - uvloop.run(run_server(args))
vllm/entrypoints/openai/api_server.py
+86
@@ -41,6 +41,13 @@ import vllm.envs as envs
41 from vllm.config import VllmConfig
42 from vllm.engine.arg_utils import AsyncEngineArgs
43 from vllm.engine.protocol import Device, EngineClient
44 +from vllm.entrypoints.anthropic.protocol import (
45 + AnthropicError,
46 + AnthropicErrorResponse,
47 + AnthropicMessagesRequest,
48 + AnthropicMessagesResponse,
49 +)
50 +from vllm.entrypoints.anthropic.serving_messages import AnthropicServingMessages
51 from vllm.entrypoints.launcher import serve_http
52 from vllm.entrypoints.logger import RequestLogger
53 from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
@@ -308,6 +315,10 @@ def responses(request: Request) -> OpenAIServingResponses | None:
315 return request.app.state.openai_serving_responses
316
317
318 +def messages(request: Request) -> AnthropicServingMessages:
319 + return request.app.state.anthropic_serving_messages
320 +
321 +
322 def chat(request: Request) -> OpenAIServingChat | None:
323 return request.app.state.openai_serving_chat
324
@@ -591,6 +602,63 @@ async def cancel_responses(response_id: str, raw_request: Request):
602 return JSONResponse(content=response.model_dump())
603
604
605 +@router.post(
606 + "/v1/messages",
607 + dependencies=[Depends(validate_json_request)],
608 + responses={
609 + HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
610 + HTTPStatus.BAD_REQUEST.value: {"model": AnthropicErrorResponse},
611 + HTTPStatus.NOT_FOUND.value: {"model": AnthropicErrorResponse},
612 + HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": AnthropicErrorResponse},
613 + },
614 +)
615 +@with_cancellation
616 +@load_aware_call
617 +async def create_messages(request: AnthropicMessagesRequest, raw_request: Request):
618 + def translate_error_response(response: ErrorResponse) -> JSONResponse:
619 + anthropic_error = AnthropicErrorResponse(
620 + error=AnthropicError(
621 + type=response.error.type,
622 + message=response.error.message,
623 + )
624 + )
625 + return JSONResponse(
626 + status_code=response.error.code, content=anthropic_error.model_dump()
627 + )
628 +
629 + handler = messages(raw_request)
630 + if handler is None:
631 + error = base(raw_request).create_error_response(
632 + message="The model does not support Messages API"
633 + )
634 + return translate_error_response(error)
635 +
636 + try:
637 + generator = await handler.create_messages(request, raw_request)
638 + except Exception as e:
639 + logger.exception("Error in create_messages: %s", e)
640 + return JSONResponse(
641 + status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
642 + content=AnthropicErrorResponse(
643 + error=AnthropicError(
644 + type="internal_error",
645 + message=str(e),
646 + )
647 + ).model_dump(),
648 + )
649 +
650 + if isinstance(generator, ErrorResponse):
651 + return translate_error_response(generator)
652 +
653 + elif isinstance(generator, AnthropicMessagesResponse):
654 + logger.debug(
655 + "Anthropic Messages Response: %s", generator.model_dump(exclude_none=True)
656 + )
657 + return JSONResponse(content=generator.model_dump(exclude_none=True))
658 +
659 + return StreamingResponse(content=generator, media_type="text/event-stream")
660 +
661 +
662 @router.post(
663 "/v1/chat/completions",
664 dependencies=[Depends(validate_json_request)],
@@ -1817,6 +1885,24 @@ async def init_app_state(
1885 if "transcription" in supported_tasks
1886 else None
1887 )
1888 + state.anthropic_serving_messages = (
1889 + AnthropicServingMessages(
1890 + engine_client,
1891 + state.openai_serving_models,
1892 + args.response_role,
1893 + request_logger=request_logger,
1894 + chat_template=resolved_chat_template,
1895 + chat_template_content_format=args.chat_template_content_format,
1896 + return_tokens_as_token_ids=args.return_tokens_as_token_ids,
1897 + enable_auto_tools=args.enable_auto_tool_choice,
1898 + tool_parser=args.tool_call_parser,
1899 + reasoning_parser=args.structured_outputs_config.reasoning_parser,
1900 + enable_prompt_tokens_details=args.enable_prompt_tokens_details,
1901 + enable_force_include_usage=args.enable_force_include_usage,
1902 + )
1903 + if "generate" in supported_tasks
1904 + else None
1905 + )
1906
1907 state.enable_server_load_tracking = args.enable_server_load_tracking
1908 state.server_load_metrics = 0