@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7e65477e5e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Bugfix] Allow "None" or "" to be passed to CLI for string args that default to None (#4586)

Michael Goin committed May 3, 2024 at 13:32 UTC 7e65477e5e737927c2f07c913ede0763134504a3
2 files changed +34 -25
vllm/engine/arg_utils.py
+19 -13
@@ -11,6 +11,12 @@ from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
11 from vllm.utils import str_to_int_tuple
12
13
14 +def nullable_str(val: str):
15 + if not val or val == "None":
16 + return None
17 + return val
18 +
19 +
20 @dataclass
21 class EngineArgs:
22 """Arguments for vLLM engine."""
@@ -96,7 +102,7 @@ class EngineArgs:
102 help='Name or path of the huggingface model to use.')
103 parser.add_argument(
104 '--tokenizer',
99 - type=str,
105 + type=nullable_str,
106 default=EngineArgs.tokenizer,
107 help='Name or path of the huggingface tokenizer to use.')
108 parser.add_argument(
@@ -105,21 +111,21 @@ class EngineArgs:
111 help='Skip initialization of tokenizer and detokenizer')
112 parser.add_argument(
113 '--revision',
108 - type=str,
114 + type=nullable_str,
115 default=None,
116 help='The specific model version to use. It can be a branch '
117 'name, a tag name, or a commit id. If unspecified, will use '
118 'the default version.')
119 parser.add_argument(
120 '--code-revision',
115 - type=str,
121 + type=nullable_str,
122 default=None,
123 help='The specific revision to use for the model code on '
124 'Hugging Face Hub. It can be a branch name, a tag name, or a '
125 'commit id. If unspecified, will use the default version.')
126 parser.add_argument(
127 '--tokenizer-revision',
122 - type=str,
128 + type=nullable_str,
129 default=None,
130 help='The specific tokenizer version to use. It can be a branch '
131 'name, a tag name, or a commit id. If unspecified, will use '
@@ -136,7 +142,7 @@ class EngineArgs:
142 action='store_true',
143 help='Trust remote code from huggingface.')
144 parser.add_argument('--download-dir',
139 - type=str,
145 + type=nullable_str,
146 default=EngineArgs.download_dir,
147 help='Directory to download and load the weights, '
148 'default to the default cache dir of '
@@ -187,7 +193,7 @@ class EngineArgs:
193 'supported for common inference criteria.')
194 parser.add_argument(
195 '--quantization-param-path',
190 - type=str,
196 + type=nullable_str,
197 default=None,
198 help='Path to the JSON file containing the KV cache '
199 'scaling factors. This should generally be supplied, when '
@@ -304,7 +310,7 @@ class EngineArgs:
310 # Quantization settings.
311 parser.add_argument('--quantization',
312 '-q',
307 - type=str,
313 + type=nullable_str,
314 choices=[*QUANTIZATION_METHODS, None],
315 default=EngineArgs.quantization,
316 help='Method used to quantize the weights. If '
@@ -349,7 +355,7 @@ class EngineArgs:
355 'asynchronous tokenization. Ignored '
356 'if tokenizer_pool_size is 0.')
357 parser.add_argument('--tokenizer-pool-extra-config',
352 - type=str,
358 + type=nullable_str,
359 default=EngineArgs.tokenizer_pool_extra_config,
360 help='Extra config for tokenizer pool. '
361 'This should be a JSON string that will be '
@@ -404,7 +410,7 @@ class EngineArgs:
410 # Related to Vision-language models such as llava
411 parser.add_argument(
412 '--image-input-type',
407 - type=str,
413 + type=nullable_str,
414 default=None,
415 choices=[
416 t.name.lower() for t in VisionLanguageConfig.ImageInputType
@@ -417,7 +423,7 @@ class EngineArgs:
423 help=('Input id for image token.'))
424 parser.add_argument(
425 '--image-input-shape',
420 - type=str,
426 + type=nullable_str,
427 default=None,
428 help=('The biggest image input shape (worst for memory footprint) '
429 'given an input type. Only used for vLLM\'s profile_run.'))
@@ -440,7 +446,7 @@ class EngineArgs:
446
447 parser.add_argument(
448 '--speculative-model',
443 - type=str,
449 + type=nullable_str,
450 default=EngineArgs.speculative_model,
451 help=
452 'The name of the draft model to be used in speculative decoding.')
@@ -454,7 +460,7 @@ class EngineArgs:
460
461 parser.add_argument(
462 '--speculative-max-model-len',
457 - type=str,
463 + type=int,
464 default=EngineArgs.speculative_max_model_len,
465 help='The maximum sequence length supported by the '
466 'draft model. Sequences over this length will skip '
@@ -475,7 +481,7 @@ class EngineArgs:
481 'decoding.')
482
483 parser.add_argument('--model-loader-extra-config',
478 - type=str,
484 + type=nullable_str,
485 default=EngineArgs.model_loader_extra_config,
486 help='Extra config for model loader. '
487 'This will be passed to the model loader '
vllm/entrypoints/openai/cli_args.py
+15 -12
@@ -8,7 +8,7 @@ import argparse
8 import json
9 import ssl
10
11 -from vllm.engine.arg_utils import AsyncEngineArgs
11 +from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str
12 from vllm.entrypoints.openai.serving_engine import LoRAModulePath
13
14
@@ -25,7 +25,10 @@ class LoRAParserAction(argparse.Action):
25 def make_arg_parser():
26 parser = argparse.ArgumentParser(
27 description="vLLM OpenAI-Compatible RESTful API server.")
28 - parser.add_argument("--host", type=str, default=None, help="host name")
28 + parser.add_argument("--host",
29 + type=nullable_str,
30 + default=None,
31 + help="host name")
32 parser.add_argument("--port", type=int, default=8000, help="port number")
33 parser.add_argument(
34 "--uvicorn-log-level",
@@ -49,13 +52,13 @@ def make_arg_parser():
52 default=["*"],
53 help="allowed headers")
54 parser.add_argument("--api-key",
52 - type=str,
55 + type=nullable_str,
56 default=None,
57 help="If provided, the server will require this key "
58 "to be presented in the header.")
59 parser.add_argument("--served-model-name",
60 nargs="+",
58 - type=str,
61 + type=nullable_str,
62 default=None,
63 help="The model name(s) used in the API. If multiple "
64 "names are provided, the server will respond to any "
@@ -65,33 +68,33 @@ def make_arg_parser():
68 "same as the `--model` argument.")
69 parser.add_argument(
70 "--lora-modules",
68 - type=str,
71 + type=nullable_str,
72 default=None,
73 nargs='+',
74 action=LoRAParserAction,
75 help="LoRA module configurations in the format name=path. "
76 "Multiple modules can be specified.")
77 parser.add_argument("--chat-template",
75 - type=str,
78 + type=nullable_str,
79 default=None,
80 help="The file path to the chat template, "
81 "or the template in single-line form "
82 "for the specified model")
83 parser.add_argument("--response-role",
81 - type=str,
84 + type=nullable_str,
85 default="assistant",
86 help="The role name to return if "
87 "`request.add_generation_prompt=true`.")
88 parser.add_argument("--ssl-keyfile",
86 - type=str,
89 + type=nullable_str,
90 default=None,
91 help="The file path to the SSL key file")
92 parser.add_argument("--ssl-certfile",
90 - type=str,
93 + type=nullable_str,
94 default=None,
95 help="The file path to the SSL cert file")
96 parser.add_argument("--ssl-ca-certs",
94 - type=str,
97 + type=nullable_str,
98 default=None,
99 help="The CA certificates file")
100 parser.add_argument(
@@ -102,12 +105,12 @@ def make_arg_parser():
105 )
106 parser.add_argument(
107 "--root-path",
105 - type=str,
108 + type=nullable_str,
109 default=None,
110 help="FastAPI root_path when app is behind a path based routing proxy")
111 parser.add_argument(
112 "--middleware",
110 - type=str,
113 + type=nullable_str,
114 action="append",
115 default=[],
116 help="Additional ASGI middleware to apply to the app. "