A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Frontend] Disallow passing `model` as both argument and option (#7347)
Cyrus Leung committed
Aug 12, 2024 at 20:58 UTC
24154f8618aa5da695b18cbbd71704f59803ec6b
2 files changed
+9
-2
vllm/engine/arg_utils.py
+2
-2
@@ -32,7 +32,7 @@ def nullable_str(val: str):
32
@dataclass
33
class EngineArgs:
34
"""Arguments for vLLM engine."""
35
- model: str
35
+ model: str = 'facebook/opt-125m'
36
served_model_name: Optional[Union[List[str]]] = None
37
tokenizer: Optional[str] = None
38
skip_tokenizer_init: bool = False
@@ -133,7 +133,7 @@ class EngineArgs:
133
parser.add_argument(
134
'--model',
135
type=str,
136
- default='facebook/opt-125m',
136
+ default=EngineArgs.model,
137
help='Name or path of the huggingface model to use.')
138
parser.add_argument(
139
'--tokenizer',
vllm/scripts.py
+7
@@ -9,6 +9,7 @@ from typing import List, Optional
9
from openai import OpenAI
10
from openai.types.chat import ChatCompletionMessageParam
11
12
+from vllm.engine.arg_utils import EngineArgs
13
from vllm.entrypoints.openai.api_server import run_server
14
from vllm.entrypoints.openai.cli_args import make_arg_parser
15
from vllm.utils import FlexibleArgumentParser
@@ -24,6 +25,12 @@ def register_signal_handlers():
25
26
27
def serve(args: argparse.Namespace) -> None:
28
+ # The default value of `--model`
29
+ if args.model != EngineArgs.model:
30
+ raise ValueError(
31
+ "With `vllm serve`, you should provide the model as a "
32
+ "positional argument instead of via the `--model` option.")
33
+
34
# EngineArgs expects the model name to be passed as --model.
35
args.model = args.model_tag
36