@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c013d32c75
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Benchmark] Add cpu options to bench scripts (#3915)

Zedong Peng committed Apr 10, 2024 at 12:30 UTC c013d32c758699fbe5804af1b9d9408acd6cb8b7
2 files changed +4 -4
benchmarks/benchmark_latency.py
+2 -2
@@ -169,8 +169,8 @@ if __name__ == '__main__':
169 "--device",
170 type=str,
171 default="cuda",
172 - choices=["cuda"],
173 - help='device type for vLLM execution, supporting CUDA only currently.')
172 + choices=["cuda", "cpu"],
173 + help='device type for vLLM execution, supporting CUDA and CPU.')
174 parser.add_argument('--block-size',
175 type=int,
176 default=16,
benchmarks/benchmark_throughput.py
+2 -2
@@ -329,8 +329,8 @@ if __name__ == "__main__":
329 "--device",
330 type=str,
331 default="cuda",
332 - choices=["cuda"],
333 - help='device type for vLLM execution, supporting CUDA only currently.')
332 + choices=["cuda", "cpu"],
333 + help='device type for vLLM execution, supporting CUDA and CPU.')
334 parser.add_argument(
335 "--enable-prefix-caching",
336 action='store_true',