@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8065a7e220
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Frontend] Add FlexibleArgumentParser to support both underscore and dash in names (#5718)

Michael Goin committed Jun 20, 2024 at 19:00 UTC 8065a7e220cca1dd53107da85b6f3932ac9e25e8
22 files changed +72 -45
benchmarks/benchmark_latency.py
+2 -1
@@ -13,6 +13,7 @@ from vllm import LLM, SamplingParams
13 from vllm.engine.arg_utils import EngineArgs
14 from vllm.inputs import PromptStrictInputs
15 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
16 +from vllm.utils import FlexibleArgumentParser
17
18
19 def main(args: argparse.Namespace):
@@ -120,7 +121,7 @@ def main(args: argparse.Namespace):
121
122
123 if __name__ == '__main__':
123 - parser = argparse.ArgumentParser(
124 + parser = FlexibleArgumentParser(
125 description='Benchmark the latency of processing a single batch of '
126 'requests till completion.')
127 parser.add_argument('--model', type=str, default='facebook/opt-125m')
benchmarks/benchmark_prefix_caching.py
+2 -2
@@ -1,7 +1,7 @@
1 -import argparse
1 import time
2
3 from vllm import LLM, SamplingParams
4 +from vllm.utils import FlexibleArgumentParser
5
6 PROMPT = "You are a helpful assistant in recognizes the content of tables in markdown format. Here is a table as fellows. You need to answer my question about the table.\n# Table\n|Opening|Opening|Sl. No.|Film|Cast|Director|Music Director|Notes|\n|----|----|----|----|----|----|----|----|\n|J A N|9|1|Agni Pushpam|Jayabharathi, Kamalahasan|Jeassy|M. K. Arjunan||\n|J A N|16|2|Priyamvada|Mohan Sharma, Lakshmi, KPAC Lalitha|K. S. Sethumadhavan|V. Dakshinamoorthy||\n|J A N|23|3|Yakshagaanam|Madhu, Sheela|Sheela|M. S. Viswanathan||\n|J A N|30|4|Paalkkadal|Sheela, Sharada|T. K. Prasad|A. T. Ummer||\n|F E B|5|5|Amma|Madhu, Srividya|M. Krishnan Nair|M. K. Arjunan||\n|F E B|13|6|Appooppan|Thikkurissi Sukumaran Nair, Kamal Haasan|P. Bhaskaran|M. S. Baburaj||\n|F E B|20|7|Srishti|Chowalloor Krishnankutty, Ravi Alummoodu|K. T. Muhammad|M. S. Baburaj||\n|F E B|20|8|Vanadevatha|Prem Nazir, Madhubala|Yusufali Kechery|G. Devarajan||\n|F E B|27|9|Samasya|Madhu, Kamalahaasan|K. Thankappan|Shyam||\n|F E B|27|10|Yudhabhoomi|K. P. Ummer, Vidhubala|Crossbelt Mani|R. K. Shekhar||\n|M A R|5|11|Seemantha Puthran|Prem Nazir, Jayabharathi|A. B. Raj|M. K. Arjunan||\n|M A R|12|12|Swapnadanam|Rani Chandra, Dr. Mohandas|K. G. George|Bhaskar Chandavarkar||\n|M A R|19|13|Thulavarsham|Prem Nazir, sreedevi, Sudheer|N. Sankaran Nair|V. Dakshinamoorthy||\n|M A R|20|14|Aruthu|Kaviyoor Ponnamma, Kamalahasan|Ravi|G. Devarajan||\n|M A R|26|15|Swimming Pool|Kamal Haasan, M. G. Soman|J. Sasikumar|M. K. Arjunan||\n\n# Question\nWhat' s the content in the (1,1) cells\n" # noqa: E501
7
@@ -44,7 +44,7 @@ def main(args):
44
45
46 if __name__ == "__main__":
47 - parser = argparse.ArgumentParser(
47 + parser = FlexibleArgumentParser(
48 description='Benchmark the performance with or without automatic '
49 'prefix caching.')
50 parser.add_argument('--model',
benchmarks/benchmark_serving.py
+6 -1
@@ -44,6 +44,11 @@ try:
44 except ImportError:
45 from backend_request_func import get_tokenizer
46
47 +try:
48 + from vllm.utils import FlexibleArgumentParser
49 +except ImportError:
50 + from argparse import ArgumentParser as FlexibleArgumentParser
51 +
52
53 @dataclass
54 class BenchmarkMetrics:
@@ -511,7 +516,7 @@ def main(args: argparse.Namespace):
516
517
518 if __name__ == "__main__":
514 - parser = argparse.ArgumentParser(
519 + parser = FlexibleArgumentParser(
520 description="Benchmark the online serving throughput.")
521 parser.add_argument(
522 "--backend",
benchmarks/benchmark_throughput.py
+2 -1
@@ -12,6 +12,7 @@ from transformers import (AutoModelForCausalLM, AutoTokenizer,
12
13 from vllm.engine.arg_utils import EngineArgs
14 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
15 +from vllm.utils import FlexibleArgumentParser
16
17
18 def sample_requests(
@@ -261,7 +262,7 @@ def main(args: argparse.Namespace):
262
263
264 if __name__ == "__main__":
264 - parser = argparse.ArgumentParser(description="Benchmark the throughput.")
265 + parser = FlexibleArgumentParser(description="Benchmark the throughput.")
266 parser.add_argument("--backend",
267 type=str,
268 choices=["vllm", "hf", "mii"],
benchmarks/cutlass_benchmarks/w8a8_benchmarks.py
+2 -1
@@ -11,6 +11,7 @@ from torch.utils.benchmark import Measurement as TMeasurement
11 from weight_shapes import WEIGHT_SHAPES
12
13 from vllm import _custom_ops as ops
14 +from vllm.utils import FlexibleArgumentParser
15
16 DEFAULT_MODELS = list(WEIGHT_SHAPES.keys())[1:]
17 DEFAULT_BATCH_SIZES = [1, 16, 32, 64, 128, 256, 512]
@@ -293,7 +294,7 @@ if __name__ == '__main__':
294 return torch.float8_e4m3fn
295 raise ValueError("unsupported dtype")
296
296 - parser = argparse.ArgumentParser(
297 + parser = FlexibleArgumentParser(
298 description="""
299 Benchmark Cutlass GEMM.
300
benchmarks/kernels/benchmark_aqlm.py
+2 -2
@@ -1,4 +1,3 @@
1 -import argparse
1 import os
2 import sys
3 from typing import Optional
@@ -10,6 +9,7 @@ from vllm import _custom_ops as ops
9 from vllm.model_executor.layers.quantization.aqlm import (
10 dequantize_weight, generic_dequantize_gemm, get_int_dtype,
11 optimized_dequantize_gemm)
12 +from vllm.utils import FlexibleArgumentParser
13
14 os.environ['CUDA_VISIBLE_DEVICES'] = '0'
15
@@ -137,7 +137,7 @@ def dequant_test(k: int, parts: torch.Tensor, nbooks: int, bits: int) -> None:
137
138 def main():
139
140 - parser = argparse.ArgumentParser(description="Benchmark aqlm performance.")
140 + parser = FlexibleArgumentParser(description="Benchmark aqlm performance.")
141
142 # Add arguments
143 parser.add_argument("--nbooks",
benchmarks/kernels/benchmark_marlin.py
+2 -2
@@ -1,4 +1,3 @@
1 -import argparse
1 from typing import List
2
3 import torch
@@ -16,6 +15,7 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
15 MarlinWorkspace, marlin_24_quantize, marlin_quantize)
16 from vllm.model_executor.layers.quantization.utils.quant_utils import (
17 gptq_pack, quantize_weights, sort_weights)
18 +from vllm.utils import FlexibleArgumentParser
19
20 DEFAULT_MODELS = ["meta-llama/Llama-2-7b-hf/TP1"]
21 DEFAULT_BATCH_SIZES = [1, 16, 32, 64, 128, 256, 512]
@@ -211,7 +211,7 @@ def main(args):
211 # python benchmark_marlin.py --batch-sizes 1 16 32 --limit-k 4096 --limit-n 4096 --limit-group-size 128 --limit-num-bits 4 --limit-act-order 0 --limit-k-full 1 # noqa E501
212 #
213 if __name__ == "__main__":
214 - parser = argparse.ArgumentParser(
214 + parser = FlexibleArgumentParser(
215 description="Benchmark Marlin across specified models/shapes/batches")
216 parser.add_argument(
217 "--models",
benchmarks/kernels/benchmark_moe.py
+2 -1
@@ -10,6 +10,7 @@ from ray.experimental.tqdm_ray import tqdm
10 from transformers import AutoConfig
11
12 from vllm.model_executor.layers.fused_moe.fused_moe import *
13 +from vllm.utils import FlexibleArgumentParser
14
15
16 class BenchmarkConfig(TypedDict):
@@ -315,7 +316,7 @@ def main(args: argparse.Namespace):
316
317
318 if __name__ == "__main__":
318 - parser = argparse.ArgumentParser()
319 + parser = FlexibleArgumentParser()
320 parser.add_argument("--model",
321 type=str,
322 default="mistralai/Mixtral-8x7B-Instruct-v0.1")
benchmarks/kernels/benchmark_paged_attention.py
+3 -3
@@ -1,4 +1,3 @@
1 -import argparse
1 import random
2 import time
3 from typing import List, Optional
@@ -6,7 +5,8 @@ from typing import List, Optional
5 import torch
6
7 from vllm import _custom_ops as ops
9 -from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, create_kv_caches_with_random
8 +from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, FlexibleArgumentParser,
9 + create_kv_caches_with_random)
10
11 NUM_BLOCKS = 1024
12 PARTITION_SIZE = 512
@@ -161,7 +161,7 @@ def main(
161
162
163 if __name__ == '__main__':
164 - parser = argparse.ArgumentParser(
164 + parser = FlexibleArgumentParser(
165 description="Benchmark the paged attention kernel.")
166 parser.add_argument("--version",
167 type=str,
benchmarks/kernels/benchmark_rope.py
+2 -2
@@ -1,4 +1,3 @@
1 -import argparse
1 from itertools import accumulate
2 from typing import List, Optional
3
@@ -7,6 +6,7 @@ import torch
6
7 from vllm.model_executor.layers.rotary_embedding import (RotaryEmbedding,
8 get_rope)
9 +from vllm.utils import FlexibleArgumentParser
10
11
12 def benchmark_rope_kernels_multi_lora(
@@ -86,7 +86,7 @@ def benchmark_rope_kernels_multi_lora(
86
87
88 if __name__ == '__main__':
89 - parser = argparse.ArgumentParser(
89 + parser = FlexibleArgumentParser(
90 description="Benchmark the rotary embedding kernels.")
91 parser.add_argument("--is-neox-style", type=bool, default=True)
92 parser.add_argument("--batch-size", type=int, default=16)
benchmarks/overheads/benchmark_hashing.py
+2 -2
@@ -1,8 +1,8 @@
1 -import argparse
1 import cProfile
2 import pstats
3
4 from vllm import LLM, SamplingParams
5 +from vllm.utils import FlexibleArgumentParser
6
7 # A very long prompt, total number of tokens is about 15k.
8 LONG_PROMPT = ["You are an expert in large language models, aren't you?"
@@ -47,7 +47,7 @@ def main(args):
47
48
49 if __name__ == "__main__":
50 - parser = argparse.ArgumentParser(
50 + parser = FlexibleArgumentParser(
51 description='Benchmark the performance of hashing function in'
52 'automatic prefix caching.')
53 parser.add_argument('--model', type=str, default='lmsys/longchat-7b-16k')
examples/aqlm_example.py
+2 -3
@@ -1,11 +1,10 @@
1 -import argparse
2 -
1 from vllm import LLM, SamplingParams
2 +from vllm.utils import FlexibleArgumentParser
3
4
5 def main():
6
8 - parser = argparse.ArgumentParser(description='AQLM examples')
7 + parser = FlexibleArgumentParser(description='AQLM examples')
8
9 parser.add_argument('--model',
10 '-m',
examples/llm_engine_example.py
+2 -1
@@ -2,6 +2,7 @@ import argparse
2 from typing import List, Tuple
3
4 from vllm import EngineArgs, LLMEngine, RequestOutput, SamplingParams
5 +from vllm.utils import FlexibleArgumentParser
6
7
8 def create_test_prompts() -> List[Tuple[str, SamplingParams]]:
@@ -55,7 +56,7 @@ def main(args: argparse.Namespace):
56
57
58 if __name__ == '__main__':
58 - parser = argparse.ArgumentParser(
59 + parser = FlexibleArgumentParser(
60 description='Demo on using the LLMEngine class directly')
61 parser = EngineArgs.add_cli_args(parser)
62 args = parser.parse_args()
examples/save_sharded_state.py
+2 -2
@@ -20,15 +20,15 @@ llm = LLM(
20 tensor_parallel_size=8,
21 )
22 """
23 -import argparse
23 import dataclasses
24 import os
25 import shutil
26 from pathlib import Path
27
28 from vllm import LLM, EngineArgs
29 +from vllm.utils import FlexibleArgumentParser
30
31 -parser = argparse.ArgumentParser()
31 +parser = FlexibleArgumentParser()
32 EngineArgs.add_cli_args(parser)
33 parser.add_argument("--output",
34 "-o",
examples/tensorize_vllm_model.py
+2 -1
@@ -9,6 +9,7 @@ from vllm.engine.arg_utils import EngineArgs
9 from vllm.model_executor.model_loader.tensorizer import (TensorizerArgs,
10 TensorizerConfig,
11 tensorize_vllm_model)
12 +from vllm.utils import FlexibleArgumentParser
13
14 # yapf conflicts with isort for this docstring
15 # yapf: disable
@@ -96,7 +97,7 @@ deserialization in this example script, although `--tensorizer-uri` and
97
98
99 def parse_args():
99 - parser = argparse.ArgumentParser(
100 + parser = FlexibleArgumentParser(
101 description="An example script that can be used to serialize and "
102 "deserialize vLLM models. These models "
103 "can be loaded using tensorizer directly to the GPU "
tests/async_engine/api_server_async_engine.py
+2 -2
@@ -1,5 +1,4 @@
1 """vllm.entrypoints.api_server with some extra logging for testing."""
2 -import argparse
2 from typing import Any, Dict
3
4 import uvicorn
@@ -8,6 +7,7 @@ from fastapi.responses import JSONResponse, Response
7 import vllm.entrypoints.api_server
8 from vllm.engine.arg_utils import AsyncEngineArgs
9 from vllm.engine.async_llm_engine import AsyncLLMEngine
10 +from vllm.utils import FlexibleArgumentParser
11
12 app = vllm.entrypoints.api_server.app
13
@@ -33,7 +33,7 @@ def stats() -> Response:
33
34
35 if __name__ == "__main__":
36 - parser = argparse.ArgumentParser()
36 + parser = FlexibleArgumentParser()
37 parser.add_argument("--host", type=str, default="localhost")
38 parser.add_argument("--port", type=int, default=8000)
39 parser = AsyncEngineArgs.add_cli_args(parser)
vllm/engine/arg_utils.py
+8 -9
@@ -11,7 +11,7 @@ from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
11 SpeculativeConfig, TokenizerPoolConfig,
12 VisionLanguageConfig)
13 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
14 -from vllm.utils import str_to_int_tuple
14 +from vllm.utils import FlexibleArgumentParser, str_to_int_tuple
15
16
17 def nullable_str(val: str):
@@ -110,7 +110,7 @@ class EngineArgs:
110
111 @staticmethod
112 def add_cli_args_for_vlm(
113 - parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
113 + parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
114 parser.add_argument('--image-input-type',
115 type=nullable_str,
116 default=None,
@@ -156,8 +156,7 @@ class EngineArgs:
156 return parser
157
158 @staticmethod
159 - def add_cli_args(
160 - parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
159 + def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
160 """Shared CLI arguments for vLLM engine."""
161
162 # Model arguments
@@ -800,8 +799,8 @@ class AsyncEngineArgs(EngineArgs):
799 max_log_len: Optional[int] = None
800
801 @staticmethod
803 - def add_cli_args(parser: argparse.ArgumentParser,
804 - async_args_only: bool = False) -> argparse.ArgumentParser:
802 + def add_cli_args(parser: FlexibleArgumentParser,
803 + async_args_only: bool = False) -> FlexibleArgumentParser:
804 if not async_args_only:
805 parser = EngineArgs.add_cli_args(parser)
806 parser.add_argument('--engine-use-ray',
@@ -822,13 +821,13 @@ class AsyncEngineArgs(EngineArgs):
821
822 # These functions are used by sphinx to build the documentation
823 def _engine_args_parser():
825 - return EngineArgs.add_cli_args(argparse.ArgumentParser())
824 + return EngineArgs.add_cli_args(FlexibleArgumentParser())
825
826
827 def _async_engine_args_parser():
829 - return AsyncEngineArgs.add_cli_args(argparse.ArgumentParser(),
828 + return AsyncEngineArgs.add_cli_args(FlexibleArgumentParser(),
829 async_args_only=True)
830
831
832 def _vlm_engine_args_parser():
834 - return EngineArgs.add_cli_args_for_vlm(argparse.ArgumentParser())
833 + return EngineArgs.add_cli_args_for_vlm(FlexibleArgumentParser())
vllm/entrypoints/api_server.py
+2 -3
@@ -6,7 +6,6 @@ We are also not going to accept PRs modifying this file, please
6 change `vllm/entrypoints/openai/api_server.py` instead.
7 """
8
9 -import argparse
9 import json
10 import ssl
11 from typing import AsyncGenerator
@@ -19,7 +18,7 @@ from vllm.engine.arg_utils import AsyncEngineArgs
18 from vllm.engine.async_llm_engine import AsyncLLMEngine
19 from vllm.sampling_params import SamplingParams
20 from vllm.usage.usage_lib import UsageContext
22 -from vllm.utils import random_uuid
21 +from vllm.utils import FlexibleArgumentParser, random_uuid
22
23 TIMEOUT_KEEP_ALIVE = 5 # seconds.
24 app = FastAPI()
@@ -80,7 +79,7 @@ async def generate(request: Request) -> Response:
79
80
81 if __name__ == "__main__":
83 - parser = argparse.ArgumentParser()
82 + parser = FlexibleArgumentParser()
83 parser.add_argument("--host", type=str, default=None)
84 parser.add_argument("--port", type=int, default=8000)
85 parser.add_argument("--ssl-keyfile", type=str, default=None)
vllm/entrypoints/openai/cli_args.py
+2 -1
@@ -10,6 +10,7 @@ import ssl
10
11 from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str
12 from vllm.entrypoints.openai.serving_engine import LoRAModulePath
13 +from vllm.utils import FlexibleArgumentParser
14
15
16 class LoRAParserAction(argparse.Action):
@@ -23,7 +24,7 @@ class LoRAParserAction(argparse.Action):
24
25
26 def make_arg_parser():
26 - parser = argparse.ArgumentParser(
27 + parser = FlexibleArgumentParser(
28 description="vLLM OpenAI-Compatible RESTful API server.")
29 parser.add_argument("--host",
30 type=nullable_str,
vllm/entrypoints/openai/run_batch.py
+2 -3
@@ -1,4 +1,3 @@
1 -import argparse
1 import asyncio
2 import sys
3 from io import StringIO
@@ -16,14 +15,14 @@ from vllm.entrypoints.openai.protocol import (BatchRequestInput,
15 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
16 from vllm.logger import init_logger
17 from vllm.usage.usage_lib import UsageContext
19 -from vllm.utils import random_uuid
18 +from vllm.utils import FlexibleArgumentParser, random_uuid
19 from vllm.version import __version__ as VLLM_VERSION
20
21 logger = init_logger(__name__)
22
23
24 def parse_args():
26 - parser = argparse.ArgumentParser(
25 + parser = FlexibleArgumentParser(
26 description="vLLM OpenAI-Compatible batch runner.")
27 parser.add_argument(
28 "-i",
vllm/model_executor/model_loader/tensorizer.py
+2 -2
@@ -21,6 +21,7 @@ from vllm.model_executor.layers.quantization.base_config import (
21 QuantizationConfig)
22 from vllm.model_executor.layers.vocab_parallel_embedding import (
23 VocabParallelEmbedding)
24 +from vllm.utils import FlexibleArgumentParser
25
26 tensorizer_error_msg = None
27
@@ -177,8 +178,7 @@ class TensorizerArgs:
178 self.deserializer_params['encryption'] = decryption_params
179
180 @staticmethod
180 - def add_cli_args(
181 - parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
181 + def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
182 """Tensorizer CLI arguments"""
183
184 # Tensorizer options arg group
vllm/utils.py
+19
@@ -1,3 +1,4 @@
1 +import argparse
2 import asyncio
3 import datetime
4 import enum
@@ -775,3 +776,21 @@ def run_once(f):
776
777 wrapper.has_run = False # type: ignore[attr-defined]
778 return wrapper
779 +
780 +
781 +class FlexibleArgumentParser(argparse.ArgumentParser):
782 + """ArgumentParser that allows both underscore and dash in names."""
783 +
784 + def parse_args(self, args=None, namespace=None):
785 + if args is None:
786 + args = sys.argv[1:]
787 +
788 + # Convert underscores to dashes and vice versa in argument names
789 + processed_args = []
790 + for arg in args:
791 + if arg.startswith('--'):
792 + processed_args.append('--' + arg[len('--'):].replace('_', '-'))
793 + else:
794 + processed_args.append(arg)
795 +
796 + return super().parse_args(processed_args, namespace)