@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e3b2324ec4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 7, 19:00 UTC

[Frontend] Use init_app_state and FrontendArgs in run_batch (#32967)

Signed-off-by: Pooya Davoodi <pooya.davoodi@parasail.io> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>

Pooya Davoodi committed Feb 24, 2026 at 19:40 UTC e3b2324ec4c74df79c7c114f9361e336eeb40a2a
4 files changed +632 -330
tests/entrypoints/instrumentator/test_metrics.py
+1 -1
@@ -447,7 +447,7 @@ def test_metrics_exist_run_batch():
447 "--model",
448 "intfloat/multilingual-e5-small",
449 "--enable-metrics",
450 - "--url",
450 + "--host",
451 base_url,
452 "--port",
453 port,
tests/entrypoints/openai/test_run_batch.py
+419 -51
@@ -10,59 +10,361 @@ import pytest
10 from vllm.assets.audio import AudioAsset
11 from vllm.entrypoints.openai.run_batch import BatchRequestOutput
12
13 -MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
14 -
15 -# ruff: noqa: E501
16 -INPUT_BATCH = (
17 - '{{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {{"model": "{0}", "messages": [{{"role": "system", "content": "You are a helpful assistant."}},{{"role": "user", "content": "Hello world!"}}],"max_tokens": 1000}}}}\n'
18 - '{{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {{"model": "{0}", "messages": [{{"role": "system", "content": "You are an unhelpful assistant."}},{{"role": "user", "content": "Hello world!"}}],"max_tokens": 1000}}}}\n'
19 - '{{"custom_id": "request-3", "method": "POST", "url": "/v1/chat/completions", "body": {{"model": "NonExistModel", "messages": [{{"role": "system", "content": "You are an unhelpful assistant."}},{{"role": "user", "content": "Hello world!"}}],"max_tokens": 1000}}}}\n'
20 - '{{"custom_id": "request-4", "method": "POST", "url": "/bad_url", "body": {{"model": "{0}", "messages": [{{"role": "system", "content": "You are an unhelpful assistant."}},{{"role": "user", "content": "Hello world!"}}],"max_tokens": 1000}}}}\n'
21 - '{{"custom_id": "request-5", "method": "POST", "url": "/v1/chat/completions", "body": {{"stream": "True", "model": "{0}", "messages": [{{"role": "system", "content": "You are an unhelpful assistant."}},{{"role": "user", "content": "Hello world!"}}],"max_tokens": 1000}}}}'
22 -).format(MODEL_NAME)
23 -
24 -INVALID_INPUT_BATCH = (
25 - '{{"invalid_field": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {{"model": "{0}", "messages": [{{"role": "system", "content": "You are a helpful assistant."}},{{"role": "user", "content": "Hello world!"}}],"max_tokens": 1000}}}}\n'
26 - '{{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {{"model": "{0}", "messages": [{{"role": "system", "content": "You are an unhelpful assistant."}},{{"role": "user", "content": "Hello world!"}}],"max_tokens": 1000}}}}'
27 -).format(MODEL_NAME)
28 -
29 -INPUT_EMBEDDING_BATCH = (
30 - '{"custom_id": "request-1", "method": "POST", "url": "/v1/embeddings", "body": {"model": "intfloat/multilingual-e5-small", "input": "You are a helpful assistant."}}\n'
31 - '{"custom_id": "request-2", "method": "POST", "url": "/v1/embeddings", "body": {"model": "intfloat/multilingual-e5-small", "input": "You are an unhelpful assistant."}}\n'
32 - '{"custom_id": "request-3", "method": "POST", "url": "/v1/embeddings", "body": {"model": "intfloat/multilingual-e5-small", "input": "Hello world!"}}\n'
33 - '{"custom_id": "request-4", "method": "POST", "url": "/v1/embeddings", "body": {"model": "NonExistModel", "input": "Hello world!"}}'
13 +CHAT_MODEL_NAME = "hmellor/tiny-random-LlamaForCausalLM"
14 +EMBEDDING_MODEL_NAME = "intfloat/multilingual-e5-small"
15 +RERANKER_MODEL_NAME = "BAAI/bge-reranker-v2-m3"
16 +REASONING_MODEL_NAME = "Qwen/Qwen3-0.6B"
17 +SPEECH_LARGE_MODEL_NAME = "openai/whisper-large-v3"
18 +SPEECH_SMALL_MODEL_NAME = "openai/whisper-small"
19 +
20 +INPUT_BATCH = "\n".join(
21 + json.dumps(req)
22 + for req in [
23 + {
24 + "custom_id": "request-1",
25 + "method": "POST",
26 + "url": "/v1/chat/completions",
27 + "body": {
28 + "model": CHAT_MODEL_NAME,
29 + "messages": [
30 + {
31 + "role": "system",
32 + "content": "You are a helpful assistant.",
33 + },
34 + {"role": "user", "content": "Hello world!"},
35 + ],
36 + "max_tokens": 1000,
37 + },
38 + },
39 + {
40 + "custom_id": "request-2",
41 + "method": "POST",
42 + "url": "/v1/chat/completions",
43 + "body": {
44 + "model": CHAT_MODEL_NAME,
45 + "messages": [
46 + {
47 + "role": "system",
48 + "content": "You are an unhelpful assistant.",
49 + },
50 + {"role": "user", "content": "Hello world!"},
51 + ],
52 + "max_tokens": 1000,
53 + },
54 + },
55 + {
56 + "custom_id": "request-3",
57 + "method": "POST",
58 + "url": "/v1/chat/completions",
59 + "body": {
60 + "model": "NonExistModel",
61 + "messages": [
62 + {
63 + "role": "system",
64 + "content": "You are an unhelpful assistant.",
65 + },
66 + {"role": "user", "content": "Hello world!"},
67 + ],
68 + "max_tokens": 1000,
69 + },
70 + },
71 + {
72 + "custom_id": "request-4",
73 + "method": "POST",
74 + "url": "/bad_url",
75 + "body": {
76 + "model": CHAT_MODEL_NAME,
77 + "messages": [
78 + {
79 + "role": "system",
80 + "content": "You are an unhelpful assistant.",
81 + },
82 + {"role": "user", "content": "Hello world!"},
83 + ],
84 + "max_tokens": 1000,
85 + },
86 + },
87 + {
88 + "custom_id": "request-5",
89 + "method": "POST",
90 + "url": "/v1/chat/completions",
91 + "body": {
92 + "stream": "True",
93 + "model": CHAT_MODEL_NAME,
94 + "messages": [
95 + {
96 + "role": "system",
97 + "content": "You are an unhelpful assistant.",
98 + },
99 + {"role": "user", "content": "Hello world!"},
100 + ],
101 + "max_tokens": 1000,
102 + },
103 + },
104 + ]
105 )
106
36 -INPUT_SCORE_BATCH = """{"custom_id": "request-1", "method": "POST", "url": "/score", "body": {"model": "BAAI/bge-reranker-v2-m3", "queries": "What is the capital of France?", "documents": ["The capital of Brazil is Brasilia.", "The capital of France is Paris."]}}
37 -{"custom_id": "request-2", "method": "POST", "url": "/v1/score", "body": {"model": "BAAI/bge-reranker-v2-m3", "queries": "What is the capital of France?", "documents": ["The capital of Brazil is Brasilia.", "The capital of France is Paris."]}}"""
107 +INVALID_INPUT_BATCH = "\n".join(
108 + json.dumps(req)
109 + for req in [
110 + {
111 + "invalid_field": "request-1",
112 + "method": "POST",
113 + "url": "/v1/chat/completions",
114 + "body": {
115 + "model": CHAT_MODEL_NAME,
116 + "messages": [
117 + {"role": "system", "content": "You are a helpful assistant."},
118 + {"role": "user", "content": "Hello world!"},
119 + ],
120 + "max_tokens": 1000,
121 + },
122 + },
123 + {
124 + "custom_id": "request-2",
125 + "method": "POST",
126 + "url": "/v1/chat/completions",
127 + "body": {
128 + "model": CHAT_MODEL_NAME,
129 + "messages": [
130 + {"role": "system", "content": "You are an unhelpful assistant."},
131 + {"role": "user", "content": "Hello world!"},
132 + ],
133 + "max_tokens": 1000,
134 + },
135 + },
136 + ]
137 +)
138 +
139 +INPUT_EMBEDDING_BATCH = "\n".join(
140 + json.dumps(req)
141 + for req in [
142 + {
143 + "custom_id": "request-1",
144 + "method": "POST",
145 + "url": "/v1/embeddings",
146 + "body": {
147 + "model": EMBEDDING_MODEL_NAME,
148 + "input": "You are a helpful assistant.",
149 + },
150 + },
151 + {
152 + "custom_id": "request-2",
153 + "method": "POST",
154 + "url": "/v1/embeddings",
155 + "body": {
156 + "model": EMBEDDING_MODEL_NAME,
157 + "input": "You are an unhelpful assistant.",
158 + },
159 + },
160 + {
161 + "custom_id": "request-3",
162 + "method": "POST",
163 + "url": "/v1/embeddings",
164 + "body": {
165 + "model": EMBEDDING_MODEL_NAME,
166 + "input": "Hello world!",
167 + },
168 + },
169 + {
170 + "custom_id": "request-4",
171 + "method": "POST",
172 + "url": "/v1/embeddings",
173 + "body": {
174 + "model": "NonExistModel",
175 + "input": "Hello world!",
176 + },
177 + },
178 + ]
179 +)
180
39 -INPUT_RERANK_BATCH = """{"custom_id": "request-1", "method": "POST", "url": "/rerank", "body": {"model": "BAAI/bge-reranker-v2-m3", "query": "What is the capital of France?", "documents": ["The capital of Brazil is Brasilia.", "The capital of France is Paris."]}}
40 -{"custom_id": "request-2", "method": "POST", "url": "/v1/rerank", "body": {"model": "BAAI/bge-reranker-v2-m3", "query": "What is the capital of France?", "documents": ["The capital of Brazil is Brasilia.", "The capital of France is Paris."]}}
41 -{"custom_id": "request-2", "method": "POST", "url": "/v2/rerank", "body": {"model": "BAAI/bge-reranker-v2-m3", "query": "What is the capital of France?", "documents": ["The capital of Brazil is Brasilia.", "The capital of France is Paris."]}}"""
181 +_SCORE_RERANK_DOCUMENTS = [
182 + "The capital of Brazil is Brasilia.",
183 + "The capital of France is Paris.",
184 +]
185 +
186 +INPUT_SCORE_BATCH = "\n".join(
187 + json.dumps(req)
188 + for req in [
189 + {
190 + "custom_id": "request-1",
191 + "method": "POST",
192 + "url": "/score",
193 + "body": {
194 + "model": RERANKER_MODEL_NAME,
195 + "queries": "What is the capital of France?",
196 + "documents": _SCORE_RERANK_DOCUMENTS,
197 + },
198 + },
199 + {
200 + "custom_id": "request-2",
201 + "method": "POST",
202 + "url": "/v1/score",
203 + "body": {
204 + "model": RERANKER_MODEL_NAME,
205 + "queries": "What is the capital of France?",
206 + "documents": _SCORE_RERANK_DOCUMENTS,
207 + },
208 + },
209 + ]
210 +)
211
43 -INPUT_REASONING_BATCH = """{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "Qwen/Qwen3-0.6B", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Solve this math problem: 2+2=?"}]}}
44 -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "Qwen/Qwen3-0.6B", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "What is the capital of France?"}]}}"""
212 +INPUT_RERANK_BATCH = "\n".join(
213 + json.dumps(req)
214 + for req in [
215 + {
216 + "custom_id": "request-1",
217 + "method": "POST",
218 + "url": "/rerank",
219 + "body": {
220 + "model": RERANKER_MODEL_NAME,
221 + "query": "What is the capital of France?",
222 + "documents": _SCORE_RERANK_DOCUMENTS,
223 + },
224 + },
225 + {
226 + "custom_id": "request-2",
227 + "method": "POST",
228 + "url": "/v1/rerank",
229 + "body": {
230 + "model": RERANKER_MODEL_NAME,
231 + "query": "What is the capital of France?",
232 + "documents": _SCORE_RERANK_DOCUMENTS,
233 + },
234 + },
235 + {
236 + "custom_id": "request-2",
237 + "method": "POST",
238 + "url": "/v2/rerank",
239 + "body": {
240 + "model": RERANKER_MODEL_NAME,
241 + "query": "What is the capital of France?",
242 + "documents": _SCORE_RERANK_DOCUMENTS,
243 + },
244 + },
245 + ]
246 +)
247 +
248 +INPUT_REASONING_BATCH = "\n".join(
249 + json.dumps(req)
250 + for req in [
251 + {
252 + "custom_id": "request-1",
253 + "method": "POST",
254 + "url": "/v1/chat/completions",
255 + "body": {
256 + "model": REASONING_MODEL_NAME,
257 + "messages": [
258 + {"role": "system", "content": "You are a helpful assistant."},
259 + {"role": "user", "content": "Solve this math problem: 2+2=?"},
260 + ],
261 + },
262 + },
263 + {
264 + "custom_id": "request-2",
265 + "method": "POST",
266 + "url": "/v1/chat/completions",
267 + "body": {
268 + "model": REASONING_MODEL_NAME,
269 + "messages": [
270 + {"role": "system", "content": "You are a helpful assistant."},
271 + {"role": "user", "content": "What is the capital of France?"},
272 + ],
273 + },
274 + },
275 + ]
276 +)
277
46 -# This is a valid but minimal audio file for testing
278 MINIMAL_WAV_BASE64 = "UklGRiQAAABXQVZFZm10IBAAAAABAAEAQB8AAEAfAAABAAgAZGF0YQAAAAA="
279 INPUT_TRANSCRIPTION_BATCH = (
49 - '{{"custom_id": "request-1", "method": "POST", "url": "/v1/audio/transcriptions", '
50 - '"body": {{"model": "openai/whisper-large-v3", "file_url": "data:audio/wav;base64,{}", '
51 - '"response_format": "json"}}}}\n'
52 -).format(MINIMAL_WAV_BASE64)
280 + json.dumps(
281 + {
282 + "custom_id": "request-1",
283 + "method": "POST",
284 + "url": "/v1/audio/transcriptions",
285 + "body": {
286 + "model": SPEECH_LARGE_MODEL_NAME,
287 + "file_url": f"data:audio/wav;base64,{MINIMAL_WAV_BASE64}",
288 + "response_format": "json",
289 + },
290 + }
291 + )
292 + + "\n"
293 +)
294
295 INPUT_TRANSCRIPTION_HTTP_BATCH = (
55 - '{{"custom_id": "request-1", "method": "POST", "url": "/v1/audio/transcriptions", '
56 - '"body": {{"model": "openai/whisper-large-v3", "file_url": "{}", '
57 - '"response_format": "json"}}}}\n'
58 -).format(AudioAsset("mary_had_lamb").url)
296 + json.dumps(
297 + {
298 + "custom_id": "request-1",
299 + "method": "POST",
300 + "url": "/v1/audio/transcriptions",
301 + "body": {
302 + "model": SPEECH_LARGE_MODEL_NAME,
303 + "file_url": AudioAsset("mary_had_lamb").url,
304 + "response_format": "json",
305 + },
306 + }
307 + )
308 + + "\n"
309 +)
310
311 INPUT_TRANSLATION_BATCH = (
61 - '{{"custom_id": "request-1", "method": "POST", "url": "/v1/audio/translations", '
62 - '"body": {{"model": "openai/whisper-small", "file_url": "{}", '
63 - '"response_format": "text", "language": "it", "to_language": "en", '
64 - '"temperature": 0.0}}}}\n'
65 -).format(AudioAsset("mary_had_lamb").url)
312 + json.dumps(
313 + {
314 + "custom_id": "request-1",
315 + "method": "POST",
316 + "url": "/v1/audio/translations",
317 + "body": {
318 + "model": SPEECH_SMALL_MODEL_NAME,
319 + "file_url": AudioAsset("mary_had_lamb").url,
320 + "response_format": "text",
321 + "language": "it",
322 + "to_language": "en",
323 + "temperature": 0.0,
324 + },
325 + }
326 + )
327 + + "\n"
328 +)
329 +
330 +WEATHER_TOOL = {
331 + "type": "function",
332 + "function": {
333 + "name": "get_current_weather",
334 + "description": "Get the current weather in a given location",
335 + "parameters": {
336 + "type": "object",
337 + "properties": {
338 + "location": {
339 + "type": "string",
340 + "description": "The city and state, e.g. San Francisco, CA",
341 + },
342 + "unit": {
343 + "type": "string",
344 + "enum": ["celsius", "fahrenheit"],
345 + },
346 + },
347 + "required": ["location"],
348 + },
349 + },
350 +}
351 +
352 +INPUT_TOOL_CALLING_BATCH = json.dumps(
353 + {
354 + "custom_id": "request-1",
355 + "method": "POST",
356 + "url": "/v1/chat/completions",
357 + "body": {
358 + "model": REASONING_MODEL_NAME,
359 + "messages": [
360 + {"role": "user", "content": "What is the weather in San Francisco?"},
361 + ],
362 + "tools": [WEATHER_TOOL],
363 + "tool_choice": "required",
364 + "max_tokens": 1000,
365 + },
366 + }
367 +)
368
369
370 def test_empty_file():
@@ -81,7 +383,7 @@ def test_empty_file():
383 "-o",
384 output_file.name,
385 "--model",
84 - "intfloat/multilingual-e5-small",
386 + EMBEDDING_MODEL_NAME,
387 ],
388 )
389 proc.communicate()
@@ -108,7 +410,7 @@ def test_completions():
410 "-o",
411 output_file.name,
412 "--model",
111 - MODEL_NAME,
413 + CHAT_MODEL_NAME,
414 ],
415 )
416 proc.communicate()
@@ -141,7 +443,7 @@ def test_completions_invalid_input():
443 "-o",
444 output_file.name,
445 "--model",
144 - MODEL_NAME,
446 + CHAT_MODEL_NAME,
447 ],
448 )
449 proc.communicate()
@@ -165,7 +467,7 @@ def test_embeddings():
467 "-o",
468 output_file.name,
469 "--model",
168 - "intfloat/multilingual-e5-small",
470 + EMBEDDING_MODEL_NAME,
471 ],
472 )
473 proc.communicate()
@@ -196,7 +498,7 @@ def test_score(input_batch):
498 "-o",
499 output_file.name,
500 "--model",
199 - "BAAI/bge-reranker-v2-m3",
501 + RERANKER_MODEL_NAME,
502 ],
503 )
504 proc.communicate()
@@ -234,7 +536,7 @@ def test_reasoning_parser():
536 "-o",
537 output_file.name,
538 "--model",
237 - "Qwen/Qwen3-0.6B",
539 + REASONING_MODEL_NAME,
540 "--reasoning-parser",
541 "qwen3",
542 ],
@@ -278,7 +580,7 @@ def test_transcription():
580 "-o",
581 output_file.name,
582 "--model",
281 - "openai/whisper-large-v3",
583 + SPEECH_LARGE_MODEL_NAME,
584 ],
585 )
586 proc.communicate()
@@ -316,7 +618,7 @@ def test_transcription_http_url():
618 "-o",
619 output_file.name,
620 "--model",
319 - "openai/whisper-large-v3",
621 + SPEECH_LARGE_MODEL_NAME,
622 ],
623 )
624 proc.communicate()
@@ -356,7 +658,7 @@ def test_translation():
658 "-o",
659 output_file.name,
660 "--model",
359 - "openai/whisper-small",
661 + SPEECH_SMALL_MODEL_NAME,
662 ],
663 )
664 proc.communicate()
@@ -378,3 +680,69 @@ def test_translation():
680 translation_text = response_body["text"]
681 translation_text_lower = str(translation_text).strip().lower()
682 assert "mary" in translation_text_lower or "lamb" in translation_text_lower
683 +
684 +
685 +def test_tool_calling():
686 + """
687 + Test that tool calling works correctly in run_batch.
688 + Verifies that requests with tools return tool_calls in the response.
689 + """
690 + with (
691 + tempfile.NamedTemporaryFile("w") as input_file,
692 + tempfile.NamedTemporaryFile("r") as output_file,
693 + ):
694 + input_file.write(INPUT_TOOL_CALLING_BATCH)
695 + input_file.flush()
696 + proc = subprocess.Popen(
697 + [
698 + "vllm",
699 + "run-batch",
700 + "-i",
701 + input_file.name,
702 + "-o",
703 + output_file.name,
704 + "--model",
705 + REASONING_MODEL_NAME,
706 + "--enable-auto-tool-choice",
707 + "--tool-call-parser",
708 + "hermes",
709 + ],
710 + )
711 + proc.communicate()
712 + proc.wait()
713 + assert proc.returncode == 0, f"{proc=}"
714 +
715 + contents = output_file.read()
716 + for line in contents.strip().split("\n"):
717 + if not line.strip(): # Skip empty lines
718 + continue
719 + # Ensure that the output format conforms to the openai api.
720 + # Validation should throw if the schema is wrong.
721 + BatchRequestOutput.model_validate_json(line)
722 +
723 + # Ensure that there is no error in the response.
724 + line_dict = json.loads(line)
725 + assert isinstance(line_dict, dict)
726 + assert line_dict["error"] is None
727 +
728 + # Check that tool_calls are present in the response
729 + # With tool_choice="required", the model must call a tool
730 + response_body = line_dict["response"]["body"]
731 + assert response_body is not None
732 + message = response_body["choices"][0]["message"]
733 + assert "tool_calls" in message
734 + tool_calls = message.get("tool_calls")
735 + # With tool_choice="required", tool_calls must be present and non-empty
736 + assert tool_calls is not None
737 + assert isinstance(tool_calls, list)
738 + assert len(tool_calls) > 0
739 + # Verify tool_calls have the expected structure
740 + for tool_call in tool_calls:
741 + assert "id" in tool_call
742 + assert "type" in tool_call
743 + assert tool_call["type"] == "function"
744 + assert "function" in tool_call
745 + assert "name" in tool_call["function"]
746 + assert "arguments" in tool_call["function"]
747 + # Verify the tool name matches our tool definition
748 + assert tool_call["function"]["name"] == "get_current_weather"
vllm/entrypoints/openai/cli_args.py
+131 -91
@@ -67,38 +67,14 @@ class LoRAParserAction(argparse.Action):
67
68
69 @config
70 -class FrontendArgs:
71 - """Arguments for the OpenAI-compatible frontend server."""
70 +class BaseFrontendArgs:
71 + """Base arguments for the OpenAI-compatible frontend server.
72 +
73 + This base class does not include host, port, and server-specific arguments
74 + like SSL, CORS, and HTTP server settings. Those arguments are added by
75 + the subclasses.
76 + """
77
73 - host: str | None = None
74 - """Host name."""
75 - port: int = 8000
76 - """Port number."""
77 - uds: str | None = None
78 - """Unix domain socket path. If set, host and port arguments are ignored."""
79 - uvicorn_log_level: Literal[
80 - "critical", "error", "warning", "info", "debug", "trace"
81 - ] = "info"
82 - """Log level for uvicorn."""
83 - disable_uvicorn_access_log: bool = False
84 - """Disable uvicorn access log."""
85 - disable_access_log_for_endpoints: str | None = None
86 - """Comma-separated list of endpoint paths to exclude from uvicorn access
87 - logs. This is useful to reduce log noise from high-frequency endpoints
88 - like health checks. Example: "/health,/metrics,/ping".
89 - When set, access logs for requests to these paths will be suppressed
90 - while keeping logs for other endpoints."""
91 - allow_credentials: bool = False
92 - """Allow credentials."""
93 - allowed_origins: list[str] = field(default_factory=lambda: ["*"])
94 - """Allowed origins."""
95 - allowed_methods: list[str] = field(default_factory=lambda: ["*"])
96 - """Allowed methods."""
97 - allowed_headers: list[str] = field(default_factory=lambda: ["*"])
98 - """Allowed headers."""
99 - api_key: list[str] | None = None
100 - """If provided, the server will require one of these keys to be presented in
101 - the header."""
78 lora_modules: list[LoRAModulePath] | None = None
79 """LoRA modules configurations in either 'name=path' format or JSON format
80 or JSON list format. Example (old format): `'name=path'` Example (new
@@ -125,27 +101,6 @@ class FrontendArgs:
101 to disable thinking mode by default for Qwen3/DeepSeek models."""
102 response_role: str = "assistant"
103 """The role name to return if `request.add_generation_prompt=true`."""
128 - ssl_keyfile: str | None = None
129 - """The file path to the SSL key file."""
130 - ssl_certfile: str | None = None
131 - """The file path to the SSL cert file."""
132 - ssl_ca_certs: str | None = None
133 - """The CA certificates file."""
134 - enable_ssl_refresh: bool = False
135 - """Refresh SSL Context when SSL certificate files change"""
136 - ssl_cert_reqs: int = int(ssl.CERT_NONE)
137 - """Whether client certificate is required (see stdlib ssl module's)."""
138 - ssl_ciphers: str | None = None
139 - """SSL cipher suites for HTTPS (TLS 1.2 and below only).
140 - Example: 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305'"""
141 - root_path: str | None = None
142 - """FastAPI root_path when app is behind a path based routing proxy."""
143 - middleware: list[str] = field(default_factory=lambda: [])
144 - """Additional ASGI middleware to apply to the app. We accept multiple
145 - --middleware arguments. The value should be an import path. If a function
146 - is provided, vLLM will add it to the server using
147 - `@app.middleware('http')`. If a class is provided, vLLM will
148 - add it to the server using `app.add_middleware()`."""
104 return_tokens_as_token_ids: bool = False
105 """When `--max-logprobs` is specified, represents single tokens as
106 strings of the form 'token_id:{token_id}' so that tokens that are not
@@ -153,8 +108,6 @@ class FrontendArgs:
108 disable_frontend_multiprocessing: bool = False
109 """If specified, will run the OpenAI frontend server in the same process as
110 the model serving engine."""
156 - enable_request_id_headers: bool = False
157 - """If specified, API server will add X-Request-Id header to responses."""
111 enable_auto_tool_choice: bool = False
112 """Enable auto tool choice for supported models. Use `--tool-call-parser`
113 to specify which parser to use."""
@@ -179,8 +132,6 @@ class FrontendArgs:
132 max_log_len: int | None = None
133 """Max number of prompt characters or prompt ID numbers being printed in
134 log. The default of None means unlimited."""
182 - disable_fastapi_docs: bool = False
183 - """Disable FastAPI's OpenAPI schema, Swagger UI, and ReDoc endpoint."""
135 enable_prompt_tokens_details: bool = False
136 """If set to True, enable prompt_tokens_details in usage."""
137 enable_server_load_tracking: bool = False
@@ -197,12 +148,6 @@ class FrontendArgs:
148 """If set to False, output deltas will not be logged. Relevant only if
149 --enable-log-outputs is set.
150 """
200 - h11_max_incomplete_event_size: int = H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT
201 - """Maximum size (bytes) of an incomplete HTTP event (header or body) for
202 - h11 parser. Helps mitigate header abuse. Default: 4194304 (4 MB)."""
203 - h11_max_header_count: int = H11_MAX_HEADER_COUNT_DEFAULT
204 - """Maximum number of HTTP headers allowed in a request for h11 parser.
205 - Helps mitigate header abuse. Default: 256."""
151 log_error_stack: bool = envs.VLLM_SERVER_DEV_MODE
152 """If set to True, log the stack trace of error responses"""
153 tokens_only: bool = False
@@ -210,17 +155,135 @@ class FrontendArgs:
155 If set to True, only enable the Tokens In<>Out endpoint.
156 This is intended for use in a Disaggregated Everything setup.
157 """
158 +
159 + @classmethod
160 + def _customize_cli_kwargs(
161 + cls,
162 + frontend_kwargs: dict[str, Any],
163 + ) -> dict[str, Any]:
164 + """Customize argparse kwargs before arguments are registered.
165 +
166 + Subclasses should override this and call
167 + ``super()._customize_cli_kwargs(frontend_kwargs)`` first.
168 + """
169 + # Special case: default_chat_template_kwargs needs json.loads type
170 + frontend_kwargs["default_chat_template_kwargs"]["type"] = json.loads
171 +
172 + # Special case: LoRA modules need custom parser action and
173 + # optional_type(str)
174 + frontend_kwargs["lora_modules"]["type"] = optional_type(str)
175 + frontend_kwargs["lora_modules"]["action"] = LoRAParserAction
176 +
177 + # Special case: Tool call parser shows built-in options.
178 + valid_tool_parsers = list(ToolParserManager.list_registered())
179 + parsers_str = ",".join(valid_tool_parsers)
180 + frontend_kwargs["tool_call_parser"]["metavar"] = (
181 + f"{{{parsers_str}}} or name registered in --tool-parser-plugin"
182 + )
183 + return frontend_kwargs
184 +
185 + @classmethod
186 + def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
187 + """Register CLI arguments for this frontend class.
188 +
189 + Subclasses should override ``_customize_cli_kwargs`` instead of
190 + this method so that base-class postprocessing is always applied.
191 + """
192 + from vllm.engine.arg_utils import get_kwargs
193 +
194 + frontend_kwargs = get_kwargs(cls)
195 + frontend_kwargs = cls._customize_cli_kwargs(frontend_kwargs)
196 +
197 + group_name = cls.__name__.replace("Args", "")
198 + frontend_group = parser.add_argument_group(
199 + title=group_name,
200 + description=cls.__doc__,
201 + )
202 + for key, value in frontend_kwargs.items():
203 + extra_flags = value.pop("flags", [])
204 + frontend_group.add_argument(
205 + *extra_flags, f"--{key.replace('_', '-')}", **value
206 + )
207 +
208 + return parser
209 +
210 +
211 +@config
212 +class FrontendArgs(BaseFrontendArgs):
213 + """Arguments for the OpenAI-compatible frontend server."""
214 +
215 + host: str | None = None
216 + """Host name."""
217 + port: int = 8000
218 + """Port number."""
219 + uds: str | None = None
220 + """Unix domain socket path. If set, host and port arguments are ignored."""
221 + uvicorn_log_level: Literal[
222 + "critical", "error", "warning", "info", "debug", "trace"
223 + ] = "info"
224 + """Log level for uvicorn."""
225 + disable_uvicorn_access_log: bool = False
226 + """Disable uvicorn access log."""
227 + disable_access_log_for_endpoints: str | None = None
228 + """Comma-separated list of endpoint paths to exclude from uvicorn access
229 + logs. This is useful to reduce log noise from high-frequency endpoints
230 + like health checks. Example: "/health,/metrics,/ping".
231 + When set, access logs for requests to these paths will be suppressed
232 + while keeping logs for other endpoints."""
233 + allow_credentials: bool = False
234 + """Allow credentials."""
235 + allowed_origins: list[str] = field(default_factory=lambda: ["*"])
236 + """Allowed origins."""
237 + allowed_methods: list[str] = field(default_factory=lambda: ["*"])
238 + """Allowed methods."""
239 + allowed_headers: list[str] = field(default_factory=lambda: ["*"])
240 + """Allowed headers."""
241 + api_key: list[str] | None = None
242 + """If provided, the server will require one of these keys to be presented in
243 + the header."""
244 + ssl_keyfile: str | None = None
245 + """The file path to the SSL key file."""
246 + ssl_certfile: str | None = None
247 + """The file path to the SSL cert file."""
248 + ssl_ca_certs: str | None = None
249 + """The CA certificates file."""
250 + enable_ssl_refresh: bool = False
251 + """Refresh SSL Context when SSL certificate files change"""
252 + ssl_cert_reqs: int = int(ssl.CERT_NONE)
253 + """Whether client certificate is required (see stdlib ssl module's)."""
254 + ssl_ciphers: str | None = None
255 + """SSL cipher suites for HTTPS (TLS 1.2 and below only).
256 + Example: 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305'"""
257 + root_path: str | None = None
258 + """FastAPI root_path when app is behind a path based routing proxy."""
259 + middleware: list[str] = field(default_factory=lambda: [])
260 + """Additional ASGI middleware to apply to the app. We accept multiple
261 + --middleware arguments. The value should be an import path. If a function
262 + is provided, vLLM will add it to the server using
263 + `@app.middleware('http')`. If a class is provided, vLLM will
264 + add it to the server using `app.add_middleware()`."""
265 + enable_request_id_headers: bool = False
266 + """If specified, API server will add X-Request-Id header to responses."""
267 + disable_fastapi_docs: bool = False
268 + """Disable FastAPI's OpenAPI schema, Swagger UI, and ReDoc endpoint."""
269 + h11_max_incomplete_event_size: int = H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT
270 + """Maximum size (bytes) of an incomplete HTTP event (header or body) for
271 + h11 parser. Helps mitigate header abuse. Default: 4194304 (4 MB)."""
272 + h11_max_header_count: int = H11_MAX_HEADER_COUNT_DEFAULT
273 + """Maximum number of HTTP headers allowed in a request for h11 parser.
274 + Helps mitigate header abuse. Default: 256."""
275 enable_offline_docs: bool = False
276 """
277 Enable offline FastAPI documentation for air-gapped environments.
278 Uses vendored static assets bundled with vLLM.
279 """
280
219 - @staticmethod
220 - def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
221 - from vllm.engine.arg_utils import get_kwargs
222 -
223 - frontend_kwargs = get_kwargs(FrontendArgs)
281 + @classmethod
282 + def _customize_cli_kwargs(
283 + cls,
284 + frontend_kwargs: dict[str, Any],
285 + ) -> dict[str, Any]:
286 + frontend_kwargs = super()._customize_cli_kwargs(frontend_kwargs)
287
288 # Special case: allowed_origins, allowed_methods, allowed_headers all
289 # need json.loads type
@@ -232,14 +295,6 @@ class FrontendArgs:
295 del frontend_kwargs["allowed_methods"]["nargs"]
296 del frontend_kwargs["allowed_headers"]["nargs"]
297
235 - # Special case: default_chat_template_kwargs needs json.loads type
236 - frontend_kwargs["default_chat_template_kwargs"]["type"] = json.loads
237 -
238 - # Special case: LoRA modules need custom parser action and
239 - # optional_type(str)
240 - frontend_kwargs["lora_modules"]["type"] = optional_type(str)
241 - frontend_kwargs["lora_modules"]["action"] = LoRAParserAction
242 -
298 # Special case: Middleware needs to append action
299 frontend_kwargs["middleware"]["action"] = "append"
300 frontend_kwargs["middleware"]["type"] = str
@@ -252,22 +307,7 @@ class FrontendArgs:
307 if "nargs" in frontend_kwargs["disable_access_log_for_endpoints"]:
308 del frontend_kwargs["disable_access_log_for_endpoints"]["nargs"]
309
255 - # Special case: Tool call parser shows built-in options.
256 - valid_tool_parsers = list(ToolParserManager.list_registered())
257 - parsers_str = ",".join(valid_tool_parsers)
258 - frontend_kwargs["tool_call_parser"]["metavar"] = (
259 - f"{{{parsers_str}}} or name registered in --tool-parser-plugin"
260 - )
261 -
262 - frontend_group = parser.add_argument_group(
263 - title="Frontend",
264 - description=FrontendArgs.__doc__,
265 - )
266 -
267 - for key, value in frontend_kwargs.items():
268 - frontend_group.add_argument(f"--{key.replace('_', '-')}", **value)
269 -
270 - return parser
310 + return frontend_kwargs
311
312
313 def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
vllm/entrypoints/openai/run_batch.py
+81 -187
@@ -3,6 +3,7 @@
3
4 import asyncio
5 import base64
6 +import sys
7 import tempfile
8 from argparse import Namespace
9 from collections.abc import Awaitable, Callable
@@ -17,23 +18,23 @@ from fastapi import UploadFile
18 from prometheus_client import start_http_server
19 from pydantic import Field, TypeAdapter, field_validator, model_validator
20 from pydantic_core.core_schema import ValidationInfo
21 +from starlette.datastructures import State
22 from tqdm import tqdm
23
22 -from vllm.engine.arg_utils import AsyncEngineArgs, optional_type
24 +from vllm.config import config
25 +from vllm.engine.arg_utils import AsyncEngineArgs
26 from vllm.engine.protocol import EngineClient
24 -from vllm.entrypoints.logger import RequestLogger
27 +from vllm.entrypoints.openai.api_server import init_app_state
28 from vllm.entrypoints.openai.chat_completion.protocol import (
29 ChatCompletionRequest,
30 ChatCompletionResponse,
31 )
29 -from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
32 +from vllm.entrypoints.openai.cli_args import BaseFrontendArgs
33 from vllm.entrypoints.openai.engine.protocol import (
34 ErrorInfo,
35 ErrorResponse,
36 OpenAIBaseModel,
37 )
35 -from vllm.entrypoints.openai.models.protocol import BaseModelPath
36 -from vllm.entrypoints.openai.models.serving import OpenAIServingModels
38 from vllm.entrypoints.openai.speech_to_text.protocol import (
39 TranscriptionRequest,
40 TranscriptionResponse,
@@ -42,25 +43,18 @@ from vllm.entrypoints.openai.speech_to_text.protocol import (
43 TranslationResponse,
44 TranslationResponseVerbose,
45 )
45 -from vllm.entrypoints.openai.speech_to_text.serving import (
46 - OpenAIServingTranscription,
47 - OpenAIServingTranslation,
48 -)
46 from vllm.entrypoints.pooling.embed.protocol import (
47 EmbeddingRequest,
48 EmbeddingResponse,
49 )
53 -from vllm.entrypoints.pooling.embed.serving import OpenAIServingEmbedding
50 from vllm.entrypoints.pooling.score.protocol import (
51 RerankRequest,
52 RerankResponse,
53 ScoreRequest,
54 ScoreResponse,
55 )
60 -from vllm.entrypoints.pooling.score.serving import ServingScores
56 from vllm.logger import init_logger
57 from vllm.reasoning import ReasoningParserManager
63 -from vllm.tasks import SupportedTask
58 from vllm.utils import random_uuid
59 from vllm.utils.argparse_utils import FlexibleArgumentParser
60 from vllm.version import __version__ as VLLM_VERSION
@@ -219,87 +213,73 @@ class BatchRequestOutput(OpenAIBaseModel):
213 error: Any | None
214
215
222 -def make_arg_parser(parser: FlexibleArgumentParser):
223 - parser.add_argument(
224 - "-i",
225 - "--input-file",
226 - required=True,
227 - type=str,
228 - help="The path or url to a single input file. Currently supports local file "
229 - "paths, or the http protocol (http or https). If a URL is specified, "
230 - "the file should be available via HTTP GET.",
231 - )
232 - parser.add_argument(
233 - "-o",
234 - "--output-file",
235 - required=True,
236 - type=str,
237 - help="The path or url to a single output file. Currently supports "
238 - "local file paths, or web (http or https) urls. If a URL is specified,"
239 - " the file should be available via HTTP PUT.",
240 - )
241 - parser.add_argument(
242 - "--output-tmp-dir",
243 - type=str,
244 - default=None,
245 - help="The directory to store the output file before uploading it "
246 - "to the output URL.",
247 - )
248 - parser.add_argument(
249 - "--response-role",
250 - type=optional_type(str),
251 - default="assistant",
252 - help="The role name to return if `request.add_generation_prompt=True`.",
253 - )
216 +@config
217 +class BatchFrontendArgs(BaseFrontendArgs):
218 + """Arguments for the batch runner frontend."""
219 +
220 + input_file: str | None = None
221 + """The path or url to a single input file. Currently supports local file
222 + paths, or the http protocol (http or https). If a URL is specified,
223 + the file should be available via HTTP GET."""
224 + output_file: str | None = None
225 + """The path or url to a single output file. Currently supports
226 + local file paths, or web (http or https) urls. If a URL is specified,
227 + the file should be available via HTTP PUT."""
228 + output_tmp_dir: str | None = None
229 + """The directory to store the output file before uploading it
230 + to the output URL."""
231 + enable_metrics: bool = False
232 + """Enable Prometheus metrics"""
233 + host: str | None = None
234 + """Host name for the Prometheus metrics server
235 + (only needed if enable-metrics is set)."""
236 + port: int = 8000
237 + """Port number for the Prometheus metrics server
238 + (only needed if enable-metrics is set)."""
239 + url: str = "0.0.0.0"
240 + """[DEPRECATED] Host name for the Prometheus metrics server
241 + (only needed if enable-metrics is set). Use --host instead."""
242
255 - parser = AsyncEngineArgs.add_cli_args(parser)
243 + @classmethod
244 + def _customize_cli_kwargs(
245 + cls,
246 + frontend_kwargs: dict[str, Any],
247 + ) -> dict[str, Any]:
248 + frontend_kwargs = super()._customize_cli_kwargs(frontend_kwargs)
249
257 - parser.add_argument(
258 - "--max-log-len",
259 - type=int,
260 - default=None,
261 - help="Max number of prompt characters or prompt "
262 - "ID numbers being printed in log."
263 - "\n\nDefault: Unlimited",
264 - )
250 + frontend_kwargs["input_file"]["flags"] = ["-i"]
251 + frontend_kwargs["input_file"]["required"] = True
252 + frontend_kwargs["output_file"]["flags"] = ["-o"]
253 + frontend_kwargs["output_file"]["required"] = True
254 +
255 + frontend_kwargs["enable_metrics"]["action"] = "store_true"
256 +
257 + frontend_kwargs["url"]["deprecated"] = True
258 + return frontend_kwargs
259
266 - parser.add_argument(
267 - "--enable-metrics", action="store_true", help="Enable Prometheus metrics"
268 - )
269 - parser.add_argument(
270 - "--url",
271 - type=str,
272 - default="0.0.0.0",
273 - help="URL to the Prometheus metrics server "
274 - "(only needed if enable-metrics is set).",
275 - )
276 - parser.add_argument(
277 - "--port",
278 - type=int,
279 - default=8000,
280 - help="Port number for the Prometheus metrics server "
281 - "(only needed if enable-metrics is set).",
282 - )
283 - parser.add_argument(
284 - "--enable-prompt-tokens-details",
285 - action="store_true",
286 - default=False,
287 - help="If set to True, enable prompt_tokens_details in usage.",
288 - )
289 - parser.add_argument(
290 - "--enable-force-include-usage",
291 - action="store_true",
292 - default=False,
293 - help="If set to True, include usage on every request "
294 - "(even when stream_options is not specified)",
295 - )
260
261 +def make_arg_parser(parser: FlexibleArgumentParser):
262 + parser = BatchFrontendArgs.add_cli_args(parser)
263 + parser = AsyncEngineArgs.add_cli_args(parser)
264 return parser
265
266
267 def parse_args():
268 parser = FlexibleArgumentParser(description="vLLM OpenAI-Compatible batch runner.")
302 - return make_arg_parser(parser).parse_args()
269 + args = make_arg_parser(parser).parse_args()
270 +
271 + # Backward compatibility: If --url is set, use it for host
272 + url_explicit = any(arg == "--url" or arg.startswith("--url=") for arg in sys.argv)
273 + host_explicit = any(
274 + arg == "--host" or arg.startswith("--host=") for arg in sys.argv
275 + )
276 + if url_explicit and hasattr(args, "url") and not host_explicit:
277 + args.host = args.url
278 + logger.warning_once(
279 + "Using --url for metrics is deprecated. Please use --host instead."
280 + )
281 +
282 + return args
283
284
285 # explicitly use pure text format, with a newline at the end
@@ -671,12 +651,9 @@ def make_transcription_wrapper(is_translation: bool) -> WrapperFn:
651 return wrapper
652
653
674 -def build_endpoint_registry(
654 +async def build_endpoint_registry(
655 engine_client: EngineClient,
656 args: Namespace,
677 - base_model_paths: list[BaseModelPath],
678 - request_logger: RequestLogger | None,
679 - supported_tasks: tuple[SupportedTask, ...],
657 ) -> dict[str, dict[str, Any]]:
658 """
659 Build the endpoint registry with all serving objects and handler configurations.
@@ -684,90 +661,27 @@ def build_endpoint_registry(
661 Args:
662 engine_client: The engine client
663 args: Command line arguments
687 - base_model_paths: List of base model paths
688 - request_logger: Optional request logger
689 - supported_tasks: Tuple of supported tasks
664
665 Returns:
666 Dictionary mapping endpoint keys to their configurations
667 """
694 - model_config = engine_client.model_config
695 -
696 - # Create the openai serving objects.
697 - openai_serving_models = OpenAIServingModels(
698 - engine_client=engine_client,
699 - base_model_paths=base_model_paths,
700 - lora_modules=None,
701 - )
668 + supported_tasks = await engine_client.get_supported_tasks()
669 + logger.info("Supported tasks: %s", supported_tasks)
670
703 - openai_serving_chat = (
704 - OpenAIServingChat(
705 - engine_client,
706 - openai_serving_models,
707 - args.response_role,
708 - request_logger=request_logger,
709 - chat_template=None,
710 - chat_template_content_format="auto",
711 - reasoning_parser=args.structured_outputs_config.reasoning_parser,
712 - enable_prompt_tokens_details=args.enable_prompt_tokens_details,
713 - enable_force_include_usage=args.enable_force_include_usage,
714 - default_chat_template_kwargs=getattr(
715 - args, "default_chat_template_kwargs", None
716 - ),
717 - )
718 - if "generate" in supported_tasks
719 - else None
720 - )
671 + # Create a state object to hold serving objects
672 + state = State()
673
722 - openai_serving_embedding = (
723 - OpenAIServingEmbedding(
724 - engine_client,
725 - openai_serving_models,
726 - request_logger=request_logger,
727 - chat_template=None,
728 - chat_template_content_format="auto",
729 - )
730 - if "embed" in supported_tasks
731 - else None
732 - )
674 + # Initialize all serving objects using init_app_state
675 + # This provides full functionality including chat template processing,
676 + # LoRA support, tool servers, etc.
677 + await init_app_state(engine_client, state, args, supported_tasks)
678
734 - enable_serving_reranking = (
735 - "classify" in supported_tasks
736 - and getattr(model_config.hf_config, "num_labels", 0) == 1
737 - )
738 -
739 - openai_serving_scores = (
740 - ServingScores(
741 - engine_client,
742 - openai_serving_models,
743 - request_logger=request_logger,
744 - score_template=None,
745 - )
746 - if ("embed" in supported_tasks or enable_serving_reranking)
747 - else None
748 - )
749 -
750 - openai_serving_transcription = (
751 - OpenAIServingTranscription(
752 - engine_client,
753 - openai_serving_models,
754 - request_logger=request_logger,
755 - enable_force_include_usage=args.enable_force_include_usage,
756 - )
757 - if "transcription" in supported_tasks
758 - else None
759 - )
760 -
761 - openai_serving_translation = (
762 - OpenAIServingTranslation(
763 - engine_client,
764 - openai_serving_models,
765 - request_logger=request_logger,
766 - enable_force_include_usage=args.enable_force_include_usage,
767 - )
768 - if "transcription" in supported_tasks
769 - else None
770 - )
679 + # Get serving objects from state (defaulting to None if not set)
680 + openai_serving_chat = getattr(state, "openai_serving_chat", None)
681 + openai_serving_embedding = getattr(state, "openai_serving_embedding", None)
682 + openai_serving_scores = getattr(state, "openai_serving_scores", None)
683 + openai_serving_transcription = getattr(state, "openai_serving_transcription", None)
684 + openai_serving_translation = getattr(state, "openai_serving_translation", None)
685
686 # Registry of endpoint configurations
687 endpoint_registry: dict[str, dict[str, Any]] = {
@@ -845,29 +759,9 @@ async def run_batch(
759 engine_client: EngineClient,
760 args: Namespace,
761 ) -> None:
848 - if args.served_model_name is not None:
849 - served_model_names = args.served_model_name
850 - else:
851 - served_model_names = [args.model]
852 -
853 - if args.enable_log_requests:
854 - request_logger = RequestLogger(max_log_len=args.max_log_len)
855 - else:
856 - request_logger = None
857 -
858 - base_model_paths = [
859 - BaseModelPath(name=name, model_path=args.model) for name in served_model_names
860 - ]
861 -
862 - supported_tasks = await engine_client.get_supported_tasks()
863 - logger.info("Supported tasks: %s", supported_tasks)
864 -
865 - endpoint_registry = build_endpoint_registry(
762 + endpoint_registry = await build_endpoint_registry(
763 engine_client=engine_client,
764 args=args,
868 - base_model_paths=base_model_paths,
869 - request_logger=request_logger,
870 - supported_tasks=supported_tasks,
765 )
766
767 tracker = BatchProgressTracker()
@@ -942,7 +836,7 @@ if __name__ == "__main__":
836 # to publish metrics at the /metrics endpoint.
837 if args.enable_metrics:
838 logger.info("Prometheus metrics enabled")
945 - start_http_server(port=args.port, addr=args.url)
839 + start_http_server(port=args.port, addr=args.host)
840 else:
841 logger.info("Prometheus metrics disabled")
842