A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Frontend] Add FlexibleArgumentParser to support both underscore and dash in names (#5718)
Michael Goin committed
Jun 20, 2024 at 19:00 UTC
8065a7e220cca1dd53107da85b6f3932ac9e25e8
22 files changed
+72
-45
benchmarks/benchmark_latency.py
+2
-1
@@ -13,6 +13,7 @@ from vllm import LLM, SamplingParams
13
from vllm.engine.arg_utils import EngineArgs
14
from vllm.inputs import PromptStrictInputs
15
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
16
+from vllm.utils import FlexibleArgumentParser
17
18
19
def main(args: argparse.Namespace):
@@ -120,7 +121,7 @@ def main(args: argparse.Namespace):
121
122
123
if __name__ == '__main__':
123
- parser = argparse.ArgumentParser(
124
+ parser = FlexibleArgumentParser(
125
description='Benchmark the latency of processing a single batch of '
126
'requests till completion.')
127
parser.add_argument('--model', type=str, default='facebook/opt-125m')
benchmarks/benchmark_prefix_caching.py
+2
-2
@@ -1,7 +1,7 @@
1
-import argparse
1
import time
2
3
from vllm import LLM, SamplingParams
4
+from vllm.utils import FlexibleArgumentParser
5
6
PROMPT = "You are a helpful assistant in recognizes the content of tables in markdown format. Here is a table as fellows. You need to answer my question about the table.\n# Table\n|Opening|Opening|Sl. No.|Film|Cast|Director|Music Director|Notes|\n|----|----|----|----|----|----|----|----|\n|J A N|9|1|Agni Pushpam|Jayabharathi, Kamalahasan|Jeassy|M. K. Arjunan||\n|J A N|16|2|Priyamvada|Mohan Sharma, Lakshmi, KPAC Lalitha|K. S. Sethumadhavan|V. Dakshinamoorthy||\n|J A N|23|3|Yakshagaanam|Madhu, Sheela|Sheela|M. S. Viswanathan||\n|J A N|30|4|Paalkkadal|Sheela, Sharada|T. K. Prasad|A. T. Ummer||\n|F E B|5|5|Amma|Madhu, Srividya|M. Krishnan Nair|M. K. Arjunan||\n|F E B|13|6|Appooppan|Thikkurissi Sukumaran Nair, Kamal Haasan|P. Bhaskaran|M. S. Baburaj||\n|F E B|20|7|Srishti|Chowalloor Krishnankutty, Ravi Alummoodu|K. T. Muhammad|M. S. Baburaj||\n|F E B|20|8|Vanadevatha|Prem Nazir, Madhubala|Yusufali Kechery|G. Devarajan||\n|F E B|27|9|Samasya|Madhu, Kamalahaasan|K. Thankappan|Shyam||\n|F E B|27|10|Yudhabhoomi|K. P. Ummer, Vidhubala|Crossbelt Mani|R. K. Shekhar||\n|M A R|5|11|Seemantha Puthran|Prem Nazir, Jayabharathi|A. B. Raj|M. K. Arjunan||\n|M A R|12|12|Swapnadanam|Rani Chandra, Dr. Mohandas|K. G. George|Bhaskar Chandavarkar||\n|M A R|19|13|Thulavarsham|Prem Nazir, sreedevi, Sudheer|N. Sankaran Nair|V. Dakshinamoorthy||\n|M A R|20|14|Aruthu|Kaviyoor Ponnamma, Kamalahasan|Ravi|G. Devarajan||\n|M A R|26|15|Swimming Pool|Kamal Haasan, M. G. Soman|J. Sasikumar|M. K. Arjunan||\n\n# Question\nWhat' s the content in the (1,1) cells\n" # noqa: E501
7
@@ -44,7 +44,7 @@ def main(args):
44
45
46
if __name__ == "__main__":
47
- parser = argparse.ArgumentParser(
47
+ parser = FlexibleArgumentParser(
48
description='Benchmark the performance with or without automatic '
49
'prefix caching.')
50
parser.add_argument('--model',
benchmarks/benchmark_serving.py
+6
-1
@@ -44,6 +44,11 @@ try:
44
except ImportError:
45
from backend_request_func import get_tokenizer
46
47
+try:
48
+ from vllm.utils import FlexibleArgumentParser
49
+except ImportError:
50
+ from argparse import ArgumentParser as FlexibleArgumentParser
51
+
52
53
@dataclass
54
class BenchmarkMetrics:
@@ -511,7 +516,7 @@ def main(args: argparse.Namespace):
516
517
518
if __name__ == "__main__":
514
- parser = argparse.ArgumentParser(
519
+ parser = FlexibleArgumentParser(
520
description="Benchmark the online serving throughput.")
521
parser.add_argument(
522
"--backend",
benchmarks/benchmark_throughput.py
+2
-1
@@ -12,6 +12,7 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer,
12
13
from vllm.engine.arg_utils import EngineArgs
14
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
15
+from vllm.utils import FlexibleArgumentParser
16
17
18
def sample_requests(
@@ -261,7 +262,7 @@ def main(args: argparse.Namespace):
262
263
264
if __name__ == "__main__":
264
- parser = argparse.ArgumentParser(description="Benchmark the throughput.")
265
+ parser = FlexibleArgumentParser(description="Benchmark the throughput.")
266
parser.add_argument("--backend",
267
type=str,
268
choices=["vllm", "hf", "mii"],
benchmarks/cutlass_benchmarks/w8a8_benchmarks.py
+2
-1
@@ -11,6 +11,7 @@ from torch.utils.benchmark import Measurement as TMeasurement
11
from weight_shapes import WEIGHT_SHAPES
12
13
from vllm import _custom_ops as ops
14
+from vllm.utils import FlexibleArgumentParser
15
16
DEFAULT_MODELS = list(WEIGHT_SHAPES.keys())[1:]
17
DEFAULT_BATCH_SIZES = [1, 16, 32, 64, 128, 256, 512]
@@ -293,7 +294,7 @@ if __name__ == '__main__':
294
return torch.float8_e4m3fn
295
raise ValueError("unsupported dtype")
296
296
- parser = argparse.ArgumentParser(
297
+ parser = FlexibleArgumentParser(
298
description="""
299
Benchmark Cutlass GEMM.
300
benchmarks/kernels/benchmark_aqlm.py
+2
-2
@@ -1,4 +1,3 @@
1
-import argparse
1
import os
2
import sys
3
from typing import Optional
@@ -10,6 +9,7 @@ from vllm import _custom_ops as ops
9
from vllm.model_executor.layers.quantization.aqlm import (
10
dequantize_weight, generic_dequantize_gemm, get_int_dtype,
11
optimized_dequantize_gemm)
12
+from vllm.utils import FlexibleArgumentParser
13
14
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
15
@@ -137,7 +137,7 @@ def dequant_test(k: int, parts: torch.Tensor, nbooks: int, bits: int) -> None:
137
138
def main():
139
140
- parser = argparse.ArgumentParser(description="Benchmark aqlm performance.")
140
+ parser = FlexibleArgumentParser(description="Benchmark aqlm performance.")
141
142
# Add arguments
143
parser.add_argument("--nbooks",
benchmarks/kernels/benchmark_marlin.py
+2
-2
@@ -1,4 +1,3 @@
1
-import argparse
1
from typing import List
2
3
import torch
@@ -16,6 +15,7 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
15
MarlinWorkspace, marlin_24_quantize, marlin_quantize)
16
from vllm.model_executor.layers.quantization.utils.quant_utils import (
17
gptq_pack, quantize_weights, sort_weights)
18
+from vllm.utils import FlexibleArgumentParser
19
20
DEFAULT_MODELS = ["meta-llama/Llama-2-7b-hf/TP1"]
21
DEFAULT_BATCH_SIZES = [1, 16, 32, 64, 128, 256, 512]
@@ -211,7 +211,7 @@ def main(args):
211
# python benchmark_marlin.py --batch-sizes 1 16 32 --limit-k 4096 --limit-n 4096 --limit-group-size 128 --limit-num-bits 4 --limit-act-order 0 --limit-k-full 1 # noqa E501
212
#
213
if __name__ == "__main__":
214
- parser = argparse.ArgumentParser(
214
+ parser = FlexibleArgumentParser(
215
description="Benchmark Marlin across specified models/shapes/batches")
216
parser.add_argument(
217
"--models",
benchmarks/kernels/benchmark_moe.py
+2
-1
@@ -10,6 +10,7 @@ from ray.experimental.tqdm_ray import tqdm
10
from transformers import AutoConfig
11
12
from vllm.model_executor.layers.fused_moe.fused_moe import *
13
+from vllm.utils import FlexibleArgumentParser
14
15
16
class BenchmarkConfig(TypedDict):
@@ -315,7 +316,7 @@ def main(args: argparse.Namespace):
316
317
318
if __name__ == "__main__":
318
- parser = argparse.ArgumentParser()
319
+ parser = FlexibleArgumentParser()
320
parser.add_argument("--model",
321
type=str,
322
default="mistralai/Mixtral-8x7B-Instruct-v0.1")
benchmarks/kernels/benchmark_paged_attention.py
+3
-3
@@ -1,4 +1,3 @@
1
-import argparse
1
import random
2
import time
3
from typing import List, Optional
@@ -6,7 +5,8 @@ from typing import List, Optional
5
import torch
6
7
from vllm import _custom_ops as ops
9
-from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, create_kv_caches_with_random
8
+from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, FlexibleArgumentParser,
9
+ create_kv_caches_with_random)
10
11
NUM_BLOCKS = 1024
12
PARTITION_SIZE = 512
@@ -161,7 +161,7 @@ def main(
161
162
163
if __name__ == '__main__':
164
- parser = argparse.ArgumentParser(
164
+ parser = FlexibleArgumentParser(
165
description="Benchmark the paged attention kernel.")
166
parser.add_argument("--version",
167
type=str,
benchmarks/kernels/benchmark_rope.py
+2
-2
@@ -1,4 +1,3 @@
1
-import argparse
1
from itertools import accumulate
2
from typing import List, Optional
3
@@ -7,6 +6,7 @@ import torch
6
7
from vllm.model_executor.layers.rotary_embedding import (RotaryEmbedding,
8
get_rope)
9
+from vllm.utils import FlexibleArgumentParser
10
11
12
def benchmark_rope_kernels_multi_lora(
@@ -86,7 +86,7 @@ def benchmark_rope_kernels_multi_lora(
86
87
88
if __name__ == '__main__':
89
- parser = argparse.ArgumentParser(
89
+ parser = FlexibleArgumentParser(
90
description="Benchmark the rotary embedding kernels.")
91
parser.add_argument("--is-neox-style", type=bool, default=True)
92
parser.add_argument("--batch-size", type=int, default=16)
benchmarks/overheads/benchmark_hashing.py
+2
-2
@@ -1,8 +1,8 @@
1
-import argparse
1
import cProfile
2
import pstats
3
4
from vllm import LLM, SamplingParams
5
+from vllm.utils import FlexibleArgumentParser
6
7
# A very long prompt, total number of tokens is about 15k.
8
LONG_PROMPT = ["You are an expert in large language models, aren't you?"
@@ -47,7 +47,7 @@ def main(args):
47
48
49
if __name__ == "__main__":
50
- parser = argparse.ArgumentParser(
50
+ parser = FlexibleArgumentParser(
51
description='Benchmark the performance of hashing function in'
52
'automatic prefix caching.')
53
parser.add_argument('--model', type=str, default='lmsys/longchat-7b-16k')
examples/aqlm_example.py
+2
-3
@@ -1,11 +1,10 @@
1
-import argparse
2
-
1
from vllm import LLM, SamplingParams
2
+from vllm.utils import FlexibleArgumentParser
3
4
5
def main():
6
8
- parser = argparse.ArgumentParser(description='AQLM examples')
7
+ parser = FlexibleArgumentParser(description='AQLM examples')
8
9
parser.add_argument('--model',
10
'-m',
examples/llm_engine_example.py
+2
-1
@@ -2,6 +2,7 @@ import argparse
2
from typing import List, Tuple
3
4
from vllm import EngineArgs, LLMEngine, RequestOutput, SamplingParams
5
+from vllm.utils import FlexibleArgumentParser
6
7
8
def create_test_prompts() -> List[Tuple[str, SamplingParams]]:
@@ -55,7 +56,7 @@ def main(args: argparse.Namespace):
56
57
58
if __name__ == '__main__':
58
- parser = argparse.ArgumentParser(
59
+ parser = FlexibleArgumentParser(
60
description='Demo on using the LLMEngine class directly')
61
parser = EngineArgs.add_cli_args(parser)
62
args = parser.parse_args()
examples/save_sharded_state.py
+2
-2
@@ -20,15 +20,15 @@ llm = LLM(
20
tensor_parallel_size=8,
21
)
22
"""
23
-import argparse
23
import dataclasses
24
import os
25
import shutil
26
from pathlib import Path
27
28
from vllm import LLM, EngineArgs
29
+from vllm.utils import FlexibleArgumentParser
30
31
-parser = argparse.ArgumentParser()
31
+parser = FlexibleArgumentParser()
32
EngineArgs.add_cli_args(parser)
33
parser.add_argument("--output",
34
"-o",
examples/tensorize_vllm_model.py
+2
-1
@@ -9,6 +9,7 @@ from vllm.engine.arg_utils import EngineArgs
9
from vllm.model_executor.model_loader.tensorizer import (TensorizerArgs,
10
TensorizerConfig,
11
tensorize_vllm_model)
12
+from vllm.utils import FlexibleArgumentParser
13
14
# yapf conflicts with isort for this docstring
15
# yapf: disable
@@ -96,7 +97,7 @@ deserialization in this example script, although `--tensorizer-uri` and
97
98
99
def parse_args():
99
- parser = argparse.ArgumentParser(
100
+ parser = FlexibleArgumentParser(
101
description="An example script that can be used to serialize and "
102
"deserialize vLLM models. These models "
103
"can be loaded using tensorizer directly to the GPU "
tests/async_engine/api_server_async_engine.py
+2
-2
@@ -1,5 +1,4 @@
1
"""vllm.entrypoints.api_server with some extra logging for testing."""
2
-import argparse
2
from typing import Any, Dict
3
4
import uvicorn
@@ -8,6 +7,7 @@ from fastapi.responses import JSONResponse, Response
7
import vllm.entrypoints.api_server
8
from vllm.engine.arg_utils import AsyncEngineArgs
9
from vllm.engine.async_llm_engine import AsyncLLMEngine
10
+from vllm.utils import FlexibleArgumentParser
11
12
app = vllm.entrypoints.api_server.app
13
@@ -33,7 +33,7 @@ def stats() -> Response:
33
34
35
if __name__ == "__main__":
36
- parser = argparse.ArgumentParser()
36
+ parser = FlexibleArgumentParser()
37
parser.add_argument("--host", type=str, default="localhost")
38
parser.add_argument("--port", type=int, default=8000)
39
parser = AsyncEngineArgs.add_cli_args(parser)
vllm/engine/arg_utils.py
+8
-9
@@ -11,7 +11,7 @@ from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
11
SpeculativeConfig, TokenizerPoolConfig,
12
VisionLanguageConfig)
13
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
14
-from vllm.utils import str_to_int_tuple
14
+from vllm.utils import FlexibleArgumentParser, str_to_int_tuple
15
16
17
def nullable_str(val: str):
@@ -110,7 +110,7 @@ class EngineArgs:
110
111
@staticmethod
112
def add_cli_args_for_vlm(
113
- parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
113
+ parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
114
parser.add_argument('--image-input-type',
115
type=nullable_str,
116
default=None,
@@ -156,8 +156,7 @@ class EngineArgs:
156
return parser
157
158
@staticmethod
159
- def add_cli_args(
160
- parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
159
+ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
160
"""Shared CLI arguments for vLLM engine."""
161
162
# Model arguments
@@ -800,8 +799,8 @@ class AsyncEngineArgs(EngineArgs):
799
max_log_len: Optional[int] = None
800
801
@staticmethod
803
- def add_cli_args(parser: argparse.ArgumentParser,
804
- async_args_only: bool = False) -> argparse.ArgumentParser:
802
+ def add_cli_args(parser: FlexibleArgumentParser,
803
+ async_args_only: bool = False) -> FlexibleArgumentParser:
804
if not async_args_only:
805
parser = EngineArgs.add_cli_args(parser)
806
parser.add_argument('--engine-use-ray',
@@ -822,13 +821,13 @@ class AsyncEngineArgs(EngineArgs):
821
822
# These functions are used by sphinx to build the documentation
823
def _engine_args_parser():
825
- return EngineArgs.add_cli_args(argparse.ArgumentParser())
824
+ return EngineArgs.add_cli_args(FlexibleArgumentParser())
825
826
827
def _async_engine_args_parser():
829
- return AsyncEngineArgs.add_cli_args(argparse.ArgumentParser(),
828
+ return AsyncEngineArgs.add_cli_args(FlexibleArgumentParser(),
829
async_args_only=True)
830
831
832
def _vlm_engine_args_parser():
834
- return EngineArgs.add_cli_args_for_vlm(argparse.ArgumentParser())
833
+ return EngineArgs.add_cli_args_for_vlm(FlexibleArgumentParser())
vllm/entrypoints/api_server.py
+2
-3
@@ -6,7 +6,6 @@ We are also not going to accept PRs modifying this file, please
6
change `vllm/entrypoints/openai/api_server.py` instead.
7
"""
8
9
-import argparse
9
import json
10
import ssl
11
from typing import AsyncGenerator
@@ -19,7 +18,7 @@ from vllm.engine.arg_utils import AsyncEngineArgs
18
from vllm.engine.async_llm_engine import AsyncLLMEngine
19
from vllm.sampling_params import SamplingParams
20
from vllm.usage.usage_lib import UsageContext
22
-from vllm.utils import random_uuid
21
+from vllm.utils import FlexibleArgumentParser, random_uuid
22
23
TIMEOUT_KEEP_ALIVE = 5 # seconds.
24
app = FastAPI()
@@ -80,7 +79,7 @@ async def generate(request: Request) -> Response:
79
80
81
if __name__ == "__main__":
83
- parser = argparse.ArgumentParser()
82
+ parser = FlexibleArgumentParser()
83
parser.add_argument("--host", type=str, default=None)
84
parser.add_argument("--port", type=int, default=8000)
85
parser.add_argument("--ssl-keyfile", type=str, default=None)
vllm/entrypoints/openai/cli_args.py
+2
-1
@@ -10,6 +10,7 @@ import ssl
10
11
from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str
12
from vllm.entrypoints.openai.serving_engine import LoRAModulePath
13
+from vllm.utils import FlexibleArgumentParser
14
15
16
class LoRAParserAction(argparse.Action):
@@ -23,7 +24,7 @@ class LoRAParserAction(argparse.Action):
24
25
26
def make_arg_parser():
26
- parser = argparse.ArgumentParser(
27
+ parser = FlexibleArgumentParser(
28
description="vLLM OpenAI-Compatible RESTful API server.")
29
parser.add_argument("--host",
30
type=nullable_str,
vllm/entrypoints/openai/run_batch.py
+2
-3
@@ -1,4 +1,3 @@
1
-import argparse
1
import asyncio
2
import sys
3
from io import StringIO
@@ -16,14 +15,14 @@ from vllm.entrypoints.openai.protocol import (BatchRequestInput,
15
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
16
from vllm.logger import init_logger
17
from vllm.usage.usage_lib import UsageContext
19
-from vllm.utils import random_uuid
18
+from vllm.utils import FlexibleArgumentParser, random_uuid
19
from vllm.version import __version__ as VLLM_VERSION
20
21
logger = init_logger(__name__)
22
23
24
def parse_args():
26
- parser = argparse.ArgumentParser(
25
+ parser = FlexibleArgumentParser(
26
description="vLLM OpenAI-Compatible batch runner.")
27
parser.add_argument(
28
"-i",
vllm/model_executor/model_loader/tensorizer.py
+2
-2
@@ -21,6 +21,7 @@ from vllm.model_executor.layers.quantization.base_config import (
21
QuantizationConfig)
22
from vllm.model_executor.layers.vocab_parallel_embedding import (
23
VocabParallelEmbedding)
24
+from vllm.utils import FlexibleArgumentParser
25
26
tensorizer_error_msg = None
27
@@ -177,8 +178,7 @@ class TensorizerArgs:
178
self.deserializer_params['encryption'] = decryption_params
179
180
@staticmethod
180
- def add_cli_args(
181
- parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
181
+ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
182
"""Tensorizer CLI arguments"""
183
184
# Tensorizer options arg group
vllm/utils.py
+19
@@ -1,3 +1,4 @@
1
+import argparse
2
import asyncio
3
import datetime
4
import enum
@@ -775,3 +776,21 @@ def run_once(f):
776
777
wrapper.has_run = False # type: ignore[attr-defined]
778
return wrapper
779
+
780
+
781
+class FlexibleArgumentParser(argparse.ArgumentParser):
782
+ """ArgumentParser that allows both underscore and dash in names."""
783
+
784
+ def parse_args(self, args=None, namespace=None):
785
+ if args is None:
786
+ args = sys.argv[1:]
787
+
788
+ # Convert underscores to dashes and vice versa in argument names
789
+ processed_args = []
790
+ for arg in args:
791
+ if arg.startswith('--'):
792
+ processed_args.append('--' + arg[len('--'):].replace('_', '-'))
793
+ else:
794
+ processed_args.append(arg)
795
+
796
+ return super().parse_args(processed_args, namespace)