@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 02cc3b51a7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[misc] benchmark_serving.py -- add ITL results and tweak TPOT results (#5263)

Tyler Michael Smith committed Jun 5, 2024 at 13:17 UTC 02cc3b51a7f2af012a8f17f0d836529d57012eee
2 files changed +23 -2
.buildkite/run-benchmarks.sh
+1 -1
@@ -50,7 +50,7 @@ echo "### Serving Benchmarks" >> benchmark_results.md
50 sed -n '1p' benchmark_serving.txt >> benchmark_results.md # first line
51 echo "" >> benchmark_results.md
52 echo '```' >> benchmark_results.md
53 -tail -n 20 benchmark_serving.txt >> benchmark_results.md # last 20 lines
53 +tail -n 24 benchmark_serving.txt >> benchmark_results.md # last 24 lines
54 echo '```' >> benchmark_results.md
55
56 # if the agent binary is not found, skip uploading the results, exit 0
benchmarks/benchmark_serving.py
+22 -1
@@ -56,6 +56,9 @@ class BenchmarkMetrics:
56 mean_tpot_ms: float
57 median_tpot_ms: float
58 p99_tpot_ms: float
59 + mean_itl_ms: float
60 + median_itl_ms: float
61 + p99_itl_ms: float
62
63
64 def sample_sharegpt_requests(
@@ -200,16 +203,24 @@ def calculate_metrics(
203 actual_output_lens = []
204 total_input = 0
205 completed = 0
206 + itls = []
207 tpots = []
208 ttfts = []
209 for i in range(len(outputs)):
210 if outputs[i].success:
207 - output_len = len(tokenizer(outputs[i].generated_text).input_ids)
211 + # We use the tokenizer to count the number of output tokens for all
212 + # serving backends instead of looking at len(outputs[i].itl) since
213 + # multiple output tokens may be bundled together
214 + # Note: this may inflate the output token count slightly
215 + output_len = len(
216 + tokenizer(outputs[i].generated_text,
217 + add_special_tokens=False).input_ids)
218 actual_output_lens.append(output_len)
219 total_input += input_requests[i][1]
220 if output_len > 1:
221 tpots.append(
222 (outputs[i].latency - outputs[i].ttft) / (output_len - 1))
223 + itls += outputs[i].itl
224 ttfts.append(outputs[i].ttft)
225 completed += 1
226 else:
@@ -234,6 +245,9 @@ def calculate_metrics(
245 mean_tpot_ms=np.mean(tpots or 0) * 1000,
246 median_tpot_ms=np.median(tpots or 0) * 1000,
247 p99_tpot_ms=np.percentile(tpots or 0, 99) * 1000,
248 + mean_itl_ms=np.mean(itls or 0) * 1000,
249 + median_itl_ms=np.median(itls or 0) * 1000,
250 + p99_itl_ms=np.percentile(itls or 0, 99) * 1000,
251 )
252
253 return metrics, actual_output_lens
@@ -333,6 +347,10 @@ async def benchmark(
347 print("{:<40} {:<10.2f}".format("Median TPOT (ms):",
348 metrics.median_tpot_ms))
349 print("{:<40} {:<10.2f}".format("P99 TPOT (ms):", metrics.p99_tpot_ms))
350 + print("{s:{c}^{n}}".format(s='Inter-token Latency', n=50, c='-'))
351 + print("{:<40} {:<10.2f}".format("Mean ITL (ms):", metrics.mean_itl_ms))
352 + print("{:<40} {:<10.2f}".format("Median ITL (ms):", metrics.median_itl_ms))
353 + print("{:<40} {:<10.2f}".format("P99 ITL (ms):", metrics.p99_itl_ms))
354 print("=" * 50)
355
356 result = {
@@ -349,6 +367,9 @@ async def benchmark(
367 "mean_tpot_ms": metrics.mean_tpot_ms,
368 "median_tpot_ms": metrics.median_tpot_ms,
369 "p99_tpot_ms": metrics.p99_tpot_ms,
370 + "mean_itl_ms": metrics.mean_itl_ms,
371 + "median_itl_ms": metrics.median_itl_ms,
372 + "p99_itl_ms": metrics.p99_itl_ms,
373 "input_lens": [output.prompt_len for output in outputs],
374 "output_lens": actual_output_lens,
375 "ttfts": [output.ttft for output in outputs],