@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 24154f8618
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Frontend] Disallow passing `model` as both argument and option (#7347)

Cyrus Leung committed Aug 12, 2024 at 20:58 UTC 24154f8618aa5da695b18cbbd71704f59803ec6b
2 files changed +9 -2
vllm/engine/arg_utils.py
+2 -2
@@ -32,7 +32,7 @@ def nullable_str(val: str):
32 @dataclass
33 class EngineArgs:
34 """Arguments for vLLM engine."""
35 - model: str
35 + model: str = 'facebook/opt-125m'
36 served_model_name: Optional[Union[List[str]]] = None
37 tokenizer: Optional[str] = None
38 skip_tokenizer_init: bool = False
@@ -133,7 +133,7 @@ class EngineArgs:
133 parser.add_argument(
134 '--model',
135 type=str,
136 - default='facebook/opt-125m',
136 + default=EngineArgs.model,
137 help='Name or path of the huggingface model to use.')
138 parser.add_argument(
139 '--tokenizer',
vllm/scripts.py
+7
@@ -9,6 +9,7 @@ from typing import List, Optional
9 from openai import OpenAI
10 from openai.types.chat import ChatCompletionMessageParam
11
12 +from vllm.engine.arg_utils import EngineArgs
13 from vllm.entrypoints.openai.api_server import run_server
14 from vllm.entrypoints.openai.cli_args import make_arg_parser
15 from vllm.utils import FlexibleArgumentParser
@@ -24,6 +25,12 @@ def register_signal_handlers():
25
26
27 def serve(args: argparse.Namespace) -> None:
28 + # The default value of `--model`
29 + if args.model != EngineArgs.model:
30 + raise ValueError(
31 + "With `vllm serve`, you should provide the model as a "
32 + "positional argument instead of via the `--model` option.")
33 +
34 # EngineArgs expects the model name to be passed as --model.
35 args.model = args.model_tag
36