| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import argparse |
| 5 | import asyncio |
| 6 | import importlib.metadata |
| 7 | import typing |
| 8 | |
| 9 | from vllm.entrypoints.cli.types import CLISubcommand |
| 10 | from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG |
| 11 | from vllm.logger import init_logger |
| 12 | |
| 13 | if typing.TYPE_CHECKING: |
| 14 | from vllm.utils.argparse_utils import FlexibleArgumentParser |
| 15 | else: |
| 16 | FlexibleArgumentParser = argparse.ArgumentParser |
| 17 | |
| 18 | logger = init_logger(__name__) |
| 19 | |
| 20 | |
| 21 | class RunBatchSubcommand(CLISubcommand): |
| 22 | """The `run-batch` subcommand for vLLM CLI.""" |
| 23 | |
| 24 | name = "run-batch" |
| 25 | |
| 26 | @staticmethod |
| 27 | def cmd(args: argparse.Namespace) -> None: |
| 28 | from vllm.entrypoints.launchers.run_batch import main as run_batch_main |
| 29 | |
| 30 | logger.info( |
| 31 | "vLLM batch processing API version %s", importlib.metadata.version("vllm") |
| 32 | ) |
| 33 | logger.info("args: %s", args) |
| 34 | |
| 35 | # Start the Prometheus metrics server. |
| 36 | # LLMEngine uses the Prometheus client |
| 37 | # to publish metrics at the /metrics endpoint. |
| 38 | if args.enable_metrics: |
| 39 | from prometheus_client import start_http_server |
| 40 | |
| 41 | logger.info("Prometheus metrics enabled") |
| 42 | start_http_server(port=args.port, addr=args.url) |
| 43 | else: |
| 44 | logger.info("Prometheus metrics disabled") |
| 45 | |
| 46 | asyncio.run(run_batch_main(args)) |
| 47 | |
| 48 | def subparser_init( |
| 49 | self, subparsers: argparse._SubParsersAction |
| 50 | ) -> FlexibleArgumentParser: |
| 51 | from vllm.entrypoints.launchers.run_batch import make_arg_parser |
| 52 | |
| 53 | run_batch_parser = subparsers.add_parser( |
| 54 | self.name, |
| 55 | help="Run batch prompts and write results to file.", |
| 56 | description=( |
| 57 | "Run batch prompts using vLLM's OpenAI-compatible API.\n" |
| 58 | "Supports local or HTTP input/output files." |
| 59 | ), |
| 60 | usage="vllm run-batch -i INPUT.jsonl -o OUTPUT.jsonl --model <model>", |
| 61 | ) |
| 62 | run_batch_parser = make_arg_parser(run_batch_parser) |
| 63 | run_batch_parser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(subcmd=self.name) |
| 64 | return run_batch_parser |
| 65 | |
| 66 | |
| 67 | def cmd_init() -> list[CLISubcommand]: |
| 68 | return [RunBatchSubcommand()] |