A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Benchmark] Change mii to use persistent deployment and support tensor parallel (#3628)
Yile (Michael) Gu committed
Mar 28, 2024 at 17:33 UTC
98a42e70782ec38c1f0f20d0226d0f71147d1ac2
1 file changed
+5
-3
benchmarks/benchmark_throughput.py
+5
-3
@@ -183,13 +183,15 @@ def run_mii(
183
tensor_parallel_size: int,
184
output_len: int,
185
) -> float:
186
- from mii import pipeline
187
- llm = pipeline(model, tensor_parallel=tensor_parallel_size)
186
+ from mii import client, serve
187
+ llm = serve(model, tensor_parallel=tensor_parallel_size)
188
prompts = [prompt for prompt, _, _ in requests]
189
190
start = time.perf_counter()
191
- llm(prompts, max_new_tokens=output_len)
191
+ llm.generate(prompts, max_new_tokens=output_len)
192
end = time.perf_counter()
193
+ client = client(model)
194
+ client.terminate_server()
195
return end - start
196
197