A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Benchmark] Add cpu options to bench scripts (#3915)
Zedong Peng committed
Apr 10, 2024 at 12:30 UTC
c013d32c758699fbe5804af1b9d9408acd6cb8b7
2 files changed
+4
-4
benchmarks/benchmark_latency.py
+2
-2
@@ -169,8 +169,8 @@ if __name__ == '__main__':
169
"--device",
170
type=str,
171
default="cuda",
172
- choices=["cuda"],
173
- help='device type for vLLM execution, supporting CUDA only currently.')
172
+ choices=["cuda", "cpu"],
173
+ help='device type for vLLM execution, supporting CUDA and CPU.')
174
parser.add_argument('--block-size',
175
type=int,
176
default=16,
benchmarks/benchmark_throughput.py
+2
-2
@@ -329,8 +329,8 @@ if __name__ == "__main__":
329
"--device",
330
type=str,
331
default="cuda",
332
- choices=["cuda"],
333
- help='device type for vLLM execution, supporting CUDA only currently.')
332
+ choices=["cuda", "cpu"],
333
+ help='device type for vLLM execution, supporting CUDA and CPU.')
334
parser.add_argument(
335
"--enable-prefix-caching",
336
action='store_true',