A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Misc] Use public API in benchmark_throughput (#4300)
zifeitong committed
Apr 24, 2024 at 14:10 UTC
a395a638c2f18d549e7d01655cf7a6dbee566f91
1 file changed
+13
-16
benchmarks/benchmark_throughput.py
+13
-16
@@ -103,25 +103,22 @@ def run_vllm(
103
)
104
105
# Add the requests to the engine.
106
+ prompts = []
107
+ sampling_params = []
108
for prompt, _, output_len in requests:
107
- sampling_params = SamplingParams(
108
- n=n,
109
- temperature=0.0 if use_beam_search else 1.0,
110
- top_p=1.0,
111
- use_beam_search=use_beam_search,
112
- ignore_eos=True,
113
- max_tokens=output_len,
114
- )
115
- # FIXME(woosuk): Do not use internal method.
116
- llm._add_request(
117
- prompt=prompt,
118
- prompt_token_ids=None,
119
- sampling_params=sampling_params,
120
- )
109
+ prompts.append(prompt)
110
+ sampling_params.append(
111
+ SamplingParams(
112
+ n=n,
113
+ temperature=0.0 if use_beam_search else 1.0,
114
+ top_p=1.0,
115
+ use_beam_search=use_beam_search,
116
+ ignore_eos=True,
117
+ max_tokens=output_len,
118
+ ))
119
120
start = time.perf_counter()
123
- # FIXME(woosuk): Do not use internal method.
124
- llm._run_engine(use_tqdm=True)
121
+ llm.generate(prompts, sampling_params, use_tqdm=True)
122
end = time.perf_counter()
123
return end - start
124