A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[misc] benchmark_serving.py -- add ITL results and tweak TPOT results (#5263)
Tyler Michael Smith committed
Jun 5, 2024 at 13:17 UTC
02cc3b51a7f2af012a8f17f0d836529d57012eee
2 files changed
+23
-2
.buildkite/run-benchmarks.sh
+1
-1
@@ -50,7 +50,7 @@ echo "### Serving Benchmarks" >> benchmark_results.md
50
sed -n '1p' benchmark_serving.txt >> benchmark_results.md # first line
51
echo "" >> benchmark_results.md
52
echo '```' >> benchmark_results.md
53
-tail -n 20 benchmark_serving.txt >> benchmark_results.md # last 20 lines
53
+tail -n 24 benchmark_serving.txt >> benchmark_results.md # last 24 lines
54
echo '```' >> benchmark_results.md
55
56
# if the agent binary is not found, skip uploading the results, exit 0
benchmarks/benchmark_serving.py
+22
-1
@@ -56,6 +56,9 @@ class BenchmarkMetrics:
56
mean_tpot_ms: float
57
median_tpot_ms: float
58
p99_tpot_ms: float
59
+ mean_itl_ms: float
60
+ median_itl_ms: float
61
+ p99_itl_ms: float
62
63
64
def sample_sharegpt_requests(
@@ -200,16 +203,24 @@ def calculate_metrics(
203
actual_output_lens = []
204
total_input = 0
205
completed = 0
206
+ itls = []
207
tpots = []
208
ttfts = []
209
for i in range(len(outputs)):
210
if outputs[i].success:
207
- output_len = len(tokenizer(outputs[i].generated_text).input_ids)
211
+ # We use the tokenizer to count the number of output tokens for all
212
+ # serving backends instead of looking at len(outputs[i].itl) since
213
+ # multiple output tokens may be bundled together
214
+ # Note: this may inflate the output token count slightly
215
+ output_len = len(
216
+ tokenizer(outputs[i].generated_text,
217
+ add_special_tokens=False).input_ids)
218
actual_output_lens.append(output_len)
219
total_input += input_requests[i][1]
220
if output_len > 1:
221
tpots.append(
222
(outputs[i].latency - outputs[i].ttft) / (output_len - 1))
223
+ itls += outputs[i].itl
224
ttfts.append(outputs[i].ttft)
225
completed += 1
226
else:
@@ -234,6 +245,9 @@ def calculate_metrics(
245
mean_tpot_ms=np.mean(tpots or 0) * 1000,
246
median_tpot_ms=np.median(tpots or 0) * 1000,
247
p99_tpot_ms=np.percentile(tpots or 0, 99) * 1000,
248
+ mean_itl_ms=np.mean(itls or 0) * 1000,
249
+ median_itl_ms=np.median(itls or 0) * 1000,
250
+ p99_itl_ms=np.percentile(itls or 0, 99) * 1000,
251
)
252
253
return metrics, actual_output_lens
@@ -333,6 +347,10 @@ async def benchmark(
347
print("{:<40} {:<10.2f}".format("Median TPOT (ms):",
348
metrics.median_tpot_ms))
349
print("{:<40} {:<10.2f}".format("P99 TPOT (ms):", metrics.p99_tpot_ms))
350
+ print("{s:{c}^{n}}".format(s='Inter-token Latency', n=50, c='-'))
351
+ print("{:<40} {:<10.2f}".format("Mean ITL (ms):", metrics.mean_itl_ms))
352
+ print("{:<40} {:<10.2f}".format("Median ITL (ms):", metrics.median_itl_ms))
353
+ print("{:<40} {:<10.2f}".format("P99 ITL (ms):", metrics.p99_itl_ms))
354
print("=" * 50)
355
356
result = {
@@ -349,6 +367,9 @@ async def benchmark(
367
"mean_tpot_ms": metrics.mean_tpot_ms,
368
"median_tpot_ms": metrics.median_tpot_ms,
369
"p99_tpot_ms": metrics.p99_tpot_ms,
370
+ "mean_itl_ms": metrics.mean_itl_ms,
371
+ "median_itl_ms": metrics.median_itl_ms,
372
+ "p99_itl_ms": metrics.p99_itl_ms,
373
"input_lens": [output.prompt_len for output in outputs],
374
"output_lens": actual_output_lens,
375
"ttfts": [output.ttft for output in outputs],