@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 98a42e7078
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Benchmark] Change mii to use persistent deployment and support tensor parallel (#3628)

Yile (Michael) Gu committed Mar 28, 2024 at 17:33 UTC 98a42e70782ec38c1f0f20d0226d0f71147d1ac2
1 file changed +5 -3
benchmarks/benchmark_throughput.py
+5 -3
@@ -183,13 +183,15 @@ def run_mii(
183 tensor_parallel_size: int,
184 output_len: int,
185 ) -> float:
186 - from mii import pipeline
187 - llm = pipeline(model, tensor_parallel=tensor_parallel_size)
186 + from mii import client, serve
187 + llm = serve(model, tensor_parallel=tensor_parallel_size)
188 prompts = [prompt for prompt, _, _ in requests]
189
190 start = time.perf_counter()
191 - llm(prompts, max_new_tokens=output_len)
191 + llm.generate(prompts, max_new_tokens=output_len)
192 end = time.perf_counter()
193 + client = client(model)
194 + client.terminate_server()
195 return end - start
196
197