A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
[Frontend] `run-batch` supports V1 (#21541)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jul 25, 2025 at 11:05 UTC
34ddcf9ff433a9a80657ec507a57c049d7ef8183
5 files changed
+54
-23
benchmarks/benchmark_throughput.py
+2
-1
@@ -167,7 +167,8 @@ async def run_vllm_async(
167
from vllm import SamplingParams
168
169
async with build_async_engine_client_from_engine_args(
170
- engine_args, disable_frontend_multiprocessing
170
+ engine_args,
171
+ disable_frontend_multiprocessing=disable_frontend_multiprocessing,
172
) as llm:
173
model_config = await llm.get_model_config()
174
assert all(
tests/entrypoints/openai/test_metrics.py
+2
-3
@@ -295,8 +295,6 @@ async def test_metrics_exist(server: RemoteOpenAIServer,
295
296
297
def test_metrics_exist_run_batch(use_v1: bool):
298
- if use_v1:
299
- pytest.skip("Skipping test on vllm V1")
298
input_batch = """{"custom_id": "request-0", "method": "POST", "url": "/v1/embeddings", "body": {"model": "intfloat/multilingual-e5-small", "input": "You are a helpful assistant."}}""" # noqa: E501
299
300
base_url = "0.0.0.0"
@@ -323,7 +321,8 @@ def test_metrics_exist_run_batch(use_v1: bool):
321
base_url,
322
"--port",
323
port,
326
- ], )
324
+ ],
325
+ env={"VLLM_USE_V1": "1" if use_v1 else "0"})
326
327
def is_server_up(url):
328
try:
vllm/benchmarks/throughput.py
+3
-1
@@ -148,7 +148,9 @@ async def run_vllm_async(
148
from vllm import SamplingParams
149
150
async with build_async_engine_client_from_engine_args(
151
- engine_args, disable_frontend_multiprocessing) as llm:
151
+ engine_args,
152
+ disable_frontend_multiprocessing=disable_frontend_multiprocessing,
153
+ ) as llm:
154
model_config = await llm.get_model_config()
155
assert all(
156
model_config.max_model_len >= (request.prompt_len +
vllm/entrypoints/openai/api_server.py
+18
-4
@@ -149,6 +149,9 @@ async def lifespan(app: FastAPI):
149
@asynccontextmanager
150
async def build_async_engine_client(
151
args: Namespace,
152
+ *,
153
+ usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
154
+ disable_frontend_multiprocessing: Optional[bool] = None,
155
client_config: Optional[dict[str, Any]] = None,
156
) -> AsyncIterator[EngineClient]:
157
@@ -156,15 +159,24 @@ async def build_async_engine_client(
159
# Ensures everything is shutdown and cleaned up on error/exit
160
engine_args = AsyncEngineArgs.from_cli_args(args)
161
162
+ if disable_frontend_multiprocessing is None:
163
+ disable_frontend_multiprocessing = bool(
164
+ args.disable_frontend_multiprocessing)
165
+
166
async with build_async_engine_client_from_engine_args(
160
- engine_args, args.disable_frontend_multiprocessing,
161
- client_config) as engine:
167
+ engine_args,
168
+ usage_context=usage_context,
169
+ disable_frontend_multiprocessing=disable_frontend_multiprocessing,
170
+ client_config=client_config,
171
+ ) as engine:
172
yield engine
173
174
175
@asynccontextmanager
176
async def build_async_engine_client_from_engine_args(
177
engine_args: AsyncEngineArgs,
178
+ *,
179
+ usage_context: UsageContext = UsageContext.OPENAI_API_SERVER,
180
disable_frontend_multiprocessing: bool = False,
181
client_config: Optional[dict[str, Any]] = None,
182
) -> AsyncIterator[EngineClient]:
@@ -177,7 +189,6 @@ async def build_async_engine_client_from_engine_args(
189
"""
190
191
# Create the EngineConfig (determines if we can use V1).
180
- usage_context = UsageContext.OPENAI_API_SERVER
192
vllm_config = engine_args.create_engine_config(usage_context=usage_context)
193
194
# V1 AsyncLLM.
@@ -1811,7 +1822,10 @@ async def run_server_worker(listen_address,
1822
if log_config is not None:
1823
uvicorn_kwargs['log_config'] = log_config
1824
1814
- async with build_async_engine_client(args, client_config) as engine_client:
1825
+ async with build_async_engine_client(
1826
+ args,
1827
+ client_config=client_config,
1828
+ ) as engine_client:
1829
maybe_register_tokenizer_info_endpoint(args)
1830
app = build_app(args)
1831
vllm/entrypoints/openai/run_batch.py
+29
-14
@@ -3,6 +3,7 @@
3
4
import asyncio
5
import tempfile
6
+from argparse import Namespace
7
from collections.abc import Awaitable
8
from http import HTTPStatus
9
from io import StringIO
@@ -13,10 +14,12 @@ import torch
14
from prometheus_client import start_http_server
15
from tqdm import tqdm
16
17
+from vllm.config import VllmConfig
18
from vllm.engine.arg_utils import AsyncEngineArgs, optional_type
17
-from vllm.engine.async_llm_engine import AsyncLLMEngine
19
+from vllm.engine.protocol import EngineClient
20
from vllm.entrypoints.logger import RequestLogger
21
# yapf: disable
22
+from vllm.entrypoints.openai.api_server import build_async_engine_client
23
from vllm.entrypoints.openai.protocol import (BatchRequestInput,
24
BatchRequestOutput,
25
BatchResponseData,
@@ -310,36 +313,37 @@ async def run_request(serving_engine_func: Callable,
313
return batch_output
314
315
313
-async def main(args):
316
+async def run_batch(
317
+ engine_client: EngineClient,
318
+ vllm_config: VllmConfig,
319
+ args: Namespace,
320
+) -> None:
321
if args.served_model_name is not None:
322
served_model_names = args.served_model_name
323
else:
324
served_model_names = [args.model]
325
319
- engine_args = AsyncEngineArgs.from_cli_args(args)
320
- engine = AsyncLLMEngine.from_engine_args(
321
- engine_args, usage_context=UsageContext.OPENAI_BATCH_RUNNER)
326
+ if args.disable_log_requests:
327
+ request_logger = None
328
+ else:
329
+ request_logger = RequestLogger(max_log_len=args.max_log_len)
330
323
- model_config = await engine.get_model_config()
331
base_model_paths = [
332
BaseModelPath(name=name, model_path=args.model)
333
for name in served_model_names
334
]
335
329
- if args.disable_log_requests:
330
- request_logger = None
331
- else:
332
- request_logger = RequestLogger(max_log_len=args.max_log_len)
336
+ model_config = vllm_config.model_config
337
338
# Create the openai serving objects.
339
openai_serving_models = OpenAIServingModels(
336
- engine_client=engine,
340
+ engine_client=engine_client,
341
model_config=model_config,
342
base_model_paths=base_model_paths,
343
lora_modules=None,
344
)
345
openai_serving_chat = OpenAIServingChat(
342
- engine,
346
+ engine_client,
347
model_config,
348
openai_serving_models,
349
args.response_role,
@@ -349,7 +353,7 @@ async def main(args):
353
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
354
) if "generate" in model_config.supported_tasks else None
355
openai_serving_embedding = OpenAIServingEmbedding(
352
- engine,
356
+ engine_client,
357
model_config,
358
openai_serving_models,
359
request_logger=request_logger,
@@ -362,7 +366,7 @@ async def main(args):
366
"num_labels", 0) == 1)
367
368
openai_serving_scores = ServingScores(
365
- engine,
369
+ engine_client,
370
model_config,
371
openai_serving_models,
372
request_logger=request_logger,
@@ -457,6 +461,17 @@ async def main(args):
461
await write_file(args.output_file, responses, args.output_tmp_dir)
462
463
464
+async def main(args: Namespace):
465
+ async with build_async_engine_client(
466
+ args,
467
+ usage_context=UsageContext.OPENAI_BATCH_RUNNER,
468
+ disable_frontend_multiprocessing=False,
469
+ ) as engine_client:
470
+ vllm_config = await engine_client.get_vllm_config()
471
+
472
+ await run_batch(engine_client, vllm_config, args)
473
+
474
+
475
if __name__ == "__main__":
476
args = parse_args()
477