@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a395a638c2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Misc] Use public API in benchmark_throughput (#4300)

zifeitong committed Apr 24, 2024 at 14:10 UTC a395a638c2f18d549e7d01655cf7a6dbee566f91
1 file changed +13 -16
benchmarks/benchmark_throughput.py
+13 -16
@@ -103,25 +103,22 @@ def run_vllm(
103 )
104
105 # Add the requests to the engine.
106 + prompts = []
107 + sampling_params = []
108 for prompt, _, output_len in requests:
107 - sampling_params = SamplingParams(
108 - n=n,
109 - temperature=0.0 if use_beam_search else 1.0,
110 - top_p=1.0,
111 - use_beam_search=use_beam_search,
112 - ignore_eos=True,
113 - max_tokens=output_len,
114 - )
115 - # FIXME(woosuk): Do not use internal method.
116 - llm._add_request(
117 - prompt=prompt,
118 - prompt_token_ids=None,
119 - sampling_params=sampling_params,
120 - )
109 + prompts.append(prompt)
110 + sampling_params.append(
111 + SamplingParams(
112 + n=n,
113 + temperature=0.0 if use_beam_search else 1.0,
114 + top_p=1.0,
115 + use_beam_search=use_beam_search,
116 + ignore_eos=True,
117 + max_tokens=output_len,
118 + ))
119
120 start = time.perf_counter()
123 - # FIXME(woosuk): Do not use internal method.
124 - llm._run_engine(use_tqdm=True)
121 + llm.generate(prompts, sampling_params, use_tqdm=True)
122 end = time.perf_counter()
123 return end - start
124