@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8baa454937
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Misc] Move device options to a single place (#8322)

Aarni Koskela committed Sep 11, 2024 at 23:25 UTC 8baa454937be70c2b6f283b3bf8538848531b769
3 files changed +23 -20
benchmarks/benchmark_latency.py
+6 -8
@@ -10,7 +10,7 @@ import torch
10 from tqdm import tqdm
11
12 from vllm import LLM, SamplingParams
13 -from vllm.engine.arg_utils import EngineArgs
13 +from vllm.engine.arg_utils import DEVICE_OPTIONS, EngineArgs
14 from vllm.inputs import PromptInputs
15 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
16 from vllm.utils import FlexibleArgumentParser
@@ -205,13 +205,11 @@ if __name__ == '__main__':
205 default=None,
206 help=('path to save the pytorch profiler output. Can be visualized '
207 'with ui.perfetto.dev or Tensorboard.'))
208 - parser.add_argument(
209 - "--device",
210 - type=str,
211 - default="auto",
212 - choices=["auto", "cuda", "cpu", "openvino", "tpu", "xpu"],
213 - help='device type for vLLM execution, supporting CUDA, OpenVINO and '
214 - 'CPU.')
208 + parser.add_argument("--device",
209 + type=str,
210 + default="auto",
211 + choices=DEVICE_OPTIONS,
212 + help='device type for vLLM execution')
213 parser.add_argument('--block-size',
214 type=int,
215 default=16,
benchmarks/benchmark_throughput.py
+6 -8
@@ -11,7 +11,7 @@ from tqdm import tqdm
11 from transformers import (AutoModelForCausalLM, AutoTokenizer,
12 PreTrainedTokenizerBase)
13
14 -from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
14 +from vllm.engine.arg_utils import DEVICE_OPTIONS, AsyncEngineArgs, EngineArgs
15 from vllm.entrypoints.openai.api_server import (
16 build_async_engine_client_from_engine_args)
17 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
@@ -451,13 +451,11 @@ if __name__ == "__main__":
451 'accuracy issues. FP8_E5M2 (without scaling) is only supported on '
452 'cuda version greater than 11.8. On ROCm (AMD GPU), FP8_E4M3 is '
453 'instead supported for common inference criteria.')
454 - parser.add_argument(
455 - "--device",
456 - type=str,
457 - default="auto",
458 - choices=["auto", "cuda", "cpu", "openvino", "tpu", "xpu"],
459 - help='device type for vLLM execution, supporting CUDA, OpenVINO and '
460 - 'CPU.')
454 + parser.add_argument("--device",
455 + type=str,
456 + default="auto",
457 + choices=DEVICE_OPTIONS,
458 + help='device type for vLLM execution')
459 parser.add_argument(
460 "--num-scheduler-steps",
461 type=int,
vllm/engine/arg_utils.py
+11 -4
@@ -26,6 +26,16 @@ logger = init_logger(__name__)
26
27 ALLOWED_DETAILED_TRACE_MODULES = ["model", "worker", "all"]
28
29 +DEVICE_OPTIONS = [
30 + "auto",
31 + "cuda",
32 + "neuron",
33 + "cpu",
34 + "openvino",
35 + "tpu",
36 + "xpu",
37 +]
38 +
39
40 def nullable_str(val: str):
41 if not val or val == "None":
@@ -553,10 +563,7 @@ class EngineArgs:
563 parser.add_argument("--device",
564 type=str,
565 default=EngineArgs.device,
556 - choices=[
557 - "auto", "cuda", "neuron", "cpu", "openvino",
558 - "tpu", "xpu"
559 - ],
566 + choices=DEVICE_OPTIONS,
567 help='Device type for vLLM execution.')
568 parser.add_argument('--num-scheduler-steps',
569 type=int,