A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Misc] Move device options to a single place (#8322)
Aarni Koskela committed
Sep 11, 2024 at 23:25 UTC
8baa454937be70c2b6f283b3bf8538848531b769
3 files changed
+23
-20
benchmarks/benchmark_latency.py
+6
-8
@@ -10,7 +10,7 @@ import torch
10
from tqdm import tqdm
11
12
from vllm import LLM, SamplingParams
13
-from vllm.engine.arg_utils import EngineArgs
13
+from vllm.engine.arg_utils import DEVICE_OPTIONS, EngineArgs
14
from vllm.inputs import PromptInputs
15
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
16
from vllm.utils import FlexibleArgumentParser
@@ -205,13 +205,11 @@ if __name__ == '__main__':
205
default=None,
206
help=('path to save the pytorch profiler output. Can be visualized '
207
'with ui.perfetto.dev or Tensorboard.'))
208
- parser.add_argument(
209
- "--device",
210
- type=str,
211
- default="auto",
212
- choices=["auto", "cuda", "cpu", "openvino", "tpu", "xpu"],
213
- help='device type for vLLM execution, supporting CUDA, OpenVINO and '
214
- 'CPU.')
208
+ parser.add_argument("--device",
209
+ type=str,
210
+ default="auto",
211
+ choices=DEVICE_OPTIONS,
212
+ help='device type for vLLM execution')
213
parser.add_argument('--block-size',
214
type=int,
215
default=16,
benchmarks/benchmark_throughput.py
+6
-8
@@ -11,7 +11,7 @@ from tqdm import tqdm
11
from transformers import (AutoModelForCausalLM, AutoTokenizer,
12
PreTrainedTokenizerBase)
13
14
-from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
14
+from vllm.engine.arg_utils import DEVICE_OPTIONS, AsyncEngineArgs, EngineArgs
15
from vllm.entrypoints.openai.api_server import (
16
build_async_engine_client_from_engine_args)
17
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
@@ -451,13 +451,11 @@ if __name__ == "__main__":
451
'accuracy issues. FP8_E5M2 (without scaling) is only supported on '
452
'cuda version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is '
453
'instead supported for common inference criteria.')
454
- parser.add_argument(
455
- "--device",
456
- type=str,
457
- default="auto",
458
- choices=["auto", "cuda", "cpu", "openvino", "tpu", "xpu"],
459
- help='device type for vLLM execution, supporting CUDA, OpenVINO and '
460
- 'CPU.')
454
+ parser.add_argument("--device",
455
+ type=str,
456
+ default="auto",
457
+ choices=DEVICE_OPTIONS,
458
+ help='device type for vLLM execution')
459
parser.add_argument(
460
"--num-scheduler-steps",
461
type=int,
vllm/engine/arg_utils.py
+11
-4
@@ -26,6 +26,16 @@ logger = init_logger(__name__)
26
27
ALLOWED_DETAILED_TRACE_MODULES = ["model", "worker", "all"]
28
29
+DEVICE_OPTIONS = [
30
+ "auto",
31
+ "cuda",
32
+ "neuron",
33
+ "cpu",
34
+ "openvino",
35
+ "tpu",
36
+ "xpu",
37
+]
38
+
39
40
def nullable_str(val: str):
41
if not val or val == "None":
@@ -553,10 +563,7 @@ class EngineArgs:
563
parser.add_argument("--device",
564
type=str,
565
default=EngineArgs.device,
556
- choices=[
557
- "auto", "cuda", "neuron", "cpu", "openvino",
558
- "tpu", "xpu"
559
- ],
566
+ choices=DEVICE_OPTIONS,
567
help='Device type for vLLM execution.')
568
parser.add_argument('--num-scheduler-steps',
569
type=int,