A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Bugfix] Allow "None" or "" to be passed to CLI for string args that default to None (#4586)
Michael Goin committed
May 3, 2024 at 13:32 UTC
7e65477e5e737927c2f07c913ede0763134504a3
2 files changed
+34
-25
vllm/engine/arg_utils.py
+19
-13
@@ -11,6 +11,12 @@ from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
11
from vllm.utils import str_to_int_tuple
12
13
14
+def nullable_str(val: str):
15
+ if not val or val == "None":
16
+ return None
17
+ return val
18
+
19
+
20
@dataclass
21
class EngineArgs:
22
"""Arguments for vLLM engine."""
@@ -96,7 +102,7 @@ class EngineArgs:
102
help='Name or path of the huggingface model to use.')
103
parser.add_argument(
104
'--tokenizer',
99
- type=str,
105
+ type=nullable_str,
106
default=EngineArgs.tokenizer,
107
help='Name or path of the huggingface tokenizer to use.')
108
parser.add_argument(
@@ -105,21 +111,21 @@ class EngineArgs:
111
help='Skip initialization of tokenizer and detokenizer')
112
parser.add_argument(
113
'--revision',
108
- type=str,
114
+ type=nullable_str,
115
default=None,
116
help='The specific model version to use. It can be a branch '
117
'name, a tag name, or a commit id. If unspecified, will use '
118
'the default version.')
119
parser.add_argument(
120
'--code-revision',
115
- type=str,
121
+ type=nullable_str,
122
default=None,
123
help='The specific revision to use for the model code on '
124
'Hugging Face Hub. It can be a branch name, a tag name, or a '
125
'commit id. If unspecified, will use the default version.')
126
parser.add_argument(
127
'--tokenizer-revision',
122
- type=str,
128
+ type=nullable_str,
129
default=None,
130
help='The specific tokenizer version to use. It can be a branch '
131
'name, a tag name, or a commit id. If unspecified, will use '
@@ -136,7 +142,7 @@ class EngineArgs:
142
action='store_true',
143
help='Trust remote code from huggingface.')
144
parser.add_argument('--download-dir',
139
- type=str,
145
+ type=nullable_str,
146
default=EngineArgs.download_dir,
147
help='Directory to download and load the weights, '
148
'default to the default cache dir of '
@@ -187,7 +193,7 @@ class EngineArgs:
193
'supported for common inference criteria.')
194
parser.add_argument(
195
'--quantization-param-path',
190
- type=str,
196
+ type=nullable_str,
197
default=None,
198
help='Path to the JSON file containing the KV cache '
199
'scaling factors. This should generally be supplied, when '
@@ -304,7 +310,7 @@ class EngineArgs:
310
# Quantization settings.
311
parser.add_argument('--quantization',
312
'-q',
307
- type=str,
313
+ type=nullable_str,
314
choices=[*QUANTIZATION_METHODS, None],
315
default=EngineArgs.quantization,
316
help='Method used to quantize the weights. If '
@@ -349,7 +355,7 @@ class EngineArgs:
355
'asynchronous tokenization. Ignored '
356
'if tokenizer_pool_size is 0.')
357
parser.add_argument('--tokenizer-pool-extra-config',
352
- type=str,
358
+ type=nullable_str,
359
default=EngineArgs.tokenizer_pool_extra_config,
360
help='Extra config for tokenizer pool. '
361
'This should be a JSON string that will be '
@@ -404,7 +410,7 @@ class EngineArgs:
410
# Related to Vision-language models such as llava
411
parser.add_argument(
412
'--image-input-type',
407
- type=str,
413
+ type=nullable_str,
414
default=None,
415
choices=[
416
t.name.lower() for t in VisionLanguageConfig.ImageInputType
@@ -417,7 +423,7 @@ class EngineArgs:
423
help=('Input id for image token.'))
424
parser.add_argument(
425
'--image-input-shape',
420
- type=str,
426
+ type=nullable_str,
427
default=None,
428
help=('The biggest image input shape (worst for memory footprint) '
429
'given an input type. Only used for vLLM\'s profile_run.'))
@@ -440,7 +446,7 @@ class EngineArgs:
446
447
parser.add_argument(
448
'--speculative-model',
443
- type=str,
449
+ type=nullable_str,
450
default=EngineArgs.speculative_model,
451
help=
452
'The name of the draft model to be used in speculative decoding.')
@@ -454,7 +460,7 @@ class EngineArgs:
460
461
parser.add_argument(
462
'--speculative-max-model-len',
457
- type=str,
463
+ type=int,
464
default=EngineArgs.speculative_max_model_len,
465
help='The maximum sequence length supported by the '
466
'draft model. Sequences over this length will skip '
@@ -475,7 +481,7 @@ class EngineArgs:
481
'decoding.')
482
483
parser.add_argument('--model-loader-extra-config',
478
- type=str,
484
+ type=nullable_str,
485
default=EngineArgs.model_loader_extra_config,
486
help='Extra config for model loader. '
487
'This will be passed to the model loader '
vllm/entrypoints/openai/cli_args.py
+15
-12
@@ -8,7 +8,7 @@ import argparse
8
import json
9
import ssl
10
11
-from vllm.engine.arg_utils import AsyncEngineArgs
11
+from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str
12
from vllm.entrypoints.openai.serving_engine import LoRAModulePath
13
14
@@ -25,7 +25,10 @@ class LoRAParserAction(argparse.Action):
25
def make_arg_parser():
26
parser = argparse.ArgumentParser(
27
description="vLLM OpenAI-Compatible RESTful API server.")
28
- parser.add_argument("--host", type=str, default=None, help="host name")
28
+ parser.add_argument("--host",
29
+ type=nullable_str,
30
+ default=None,
31
+ help="host name")
32
parser.add_argument("--port", type=int, default=8000, help="port number")
33
parser.add_argument(
34
"--uvicorn-log-level",
@@ -49,13 +52,13 @@ def make_arg_parser():
52
default=["*"],
53
help="allowed headers")
54
parser.add_argument("--api-key",
52
- type=str,
55
+ type=nullable_str,
56
default=None,
57
help="If provided, the server will require this key "
58
"to be presented in the header.")
59
parser.add_argument("--served-model-name",
60
nargs="+",
58
- type=str,
61
+ type=nullable_str,
62
default=None,
63
help="The model name(s) used in the API. If multiple "
64
"names are provided, the server will respond to any "
@@ -65,33 +68,33 @@ def make_arg_parser():
68
"same as the `--model` argument.")
69
parser.add_argument(
70
"--lora-modules",
68
- type=str,
71
+ type=nullable_str,
72
default=None,
73
nargs='+',
74
action=LoRAParserAction,
75
help="LoRA module configurations in the format name=path. "
76
"Multiple modules can be specified.")
77
parser.add_argument("--chat-template",
75
- type=str,
78
+ type=nullable_str,
79
default=None,
80
help="The file path to the chat template, "
81
"or the template in single-line form "
82
"for the specified model")
83
parser.add_argument("--response-role",
81
- type=str,
84
+ type=nullable_str,
85
default="assistant",
86
help="The role name to return if "
87
"`request.add_generation_prompt=true`.")
88
parser.add_argument("--ssl-keyfile",
86
- type=str,
89
+ type=nullable_str,
90
default=None,
91
help="The file path to the SSL key file")
92
parser.add_argument("--ssl-certfile",
90
- type=str,
93
+ type=nullable_str,
94
default=None,
95
help="The file path to the SSL cert file")
96
parser.add_argument("--ssl-ca-certs",
94
- type=str,
97
+ type=nullable_str,
98
default=None,
99
help="The CA certificates file")
100
parser.add_argument(
@@ -102,12 +105,12 @@ def make_arg_parser():
105
)
106
parser.add_argument(
107
"--root-path",
105
- type=str,
108
+ type=nullable_str,
109
default=None,
110
help="FastAPI root_path when app is behind a path based routing proxy")
111
parser.add_argument(
112
"--middleware",
110
- type=str,
113
+ type=nullable_str,
114
action="append",
115
default=[],
116
help="Additional ASGI middleware to apply to the app. "