A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[CI]add genai-perf benchmark in nightly benchmark (#10704)
Signed-off-by: Kunshang Ji <kunshang.ji@intel.com>
Kunshang Ji committed
Jan 17, 2025 at 12:15 UTC
fead53ba78dbcdd4da616308f1ef1b4a312f8897
4 files changed
+196
-4
.buildkite/nightly-benchmarks/scripts/run-nightly-benchmarks.sh
+107
@@ -301,6 +301,104 @@ run_serving_tests() {
301
kill_gpu_processes
302
}
303
304
+run_genai_perf_tests() {
305
+ # run genai-perf tests
306
+
307
+ # $1: a json file specifying genai-perf test cases
308
+ local genai_perf_test_file
309
+ genai_perf_test_file=$1
310
+
311
+ # Iterate over genai-perf tests
312
+ jq -c '.[]' "$genai_perf_test_file" | while read -r params; do
313
+ # get the test name, and append the GPU type back to it.
314
+ test_name=$(echo "$params" | jq -r '.test_name')
315
+
316
+ # if TEST_SELECTOR is set, only run the test cases that match the selector
317
+ if [[ -n "$TEST_SELECTOR" ]] && [[ ! "$test_name" =~ $TEST_SELECTOR ]]; then
318
+ echo "Skip test case $test_name."
319
+ continue
320
+ fi
321
+
322
+ # prepend the current serving engine to the test name
323
+ test_name=${CURRENT_LLM_SERVING_ENGINE}_${test_name}
324
+
325
+ # get common parameters
326
+ common_params=$(echo "$params" | jq -r '.common_parameters')
327
+ model=$(echo "$common_params" | jq -r '.model')
328
+ tp=$(echo "$common_params" | jq -r '.tp')
329
+ dataset_name=$(echo "$common_params" | jq -r '.dataset_name')
330
+ dataset_path=$(echo "$common_params" | jq -r '.dataset_path')
331
+ port=$(echo "$common_params" | jq -r '.port')
332
+ num_prompts=$(echo "$common_params" | jq -r '.num_prompts')
333
+ reuse_server=$(echo "$common_params" | jq -r '.reuse_server')
334
+
335
+ # get client and server arguments
336
+ server_params=$(echo "$params" | jq -r ".${CURRENT_LLM_SERVING_ENGINE}_server_parameters")
337
+ qps_list=$(echo "$params" | jq -r '.qps_list')
338
+ qps_list=$(echo "$qps_list" | jq -r '.[] | @sh')
339
+ echo "Running over qps list $qps_list"
340
+
341
+ # check if there is enough GPU to run the test
342
+ if [[ $gpu_count -lt $tp ]]; then
343
+ echo "Required num-shard $tp but only $gpu_count GPU found. Skip testcase $test_name."
344
+ continue
345
+ fi
346
+
347
+ if [[ $reuse_server == "true" ]]; then
348
+ echo "Reuse previous server for test case $test_name"
349
+ else
350
+ kill_gpu_processes
351
+ bash "$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/scripts/launch-server.sh" \
352
+ "$server_params" "$common_params"
353
+ fi
354
+
355
+ if wait_for_server; then
356
+ echo ""
357
+ echo "$CURRENT_LLM_SERVING_ENGINE server is up and running."
358
+ else
359
+ echo ""
360
+ echo "$CURRENT_LLM_SERVING_ENGINE failed to start within the timeout period."
361
+ break
362
+ fi
363
+
364
+ # iterate over different QPS
365
+ for qps in $qps_list; do
366
+ # remove the surrounding single quote from qps
367
+ if [[ "$qps" == *"inf"* ]]; then
368
+ echo "qps was $qps"
369
+ qps=$num_prompts
370
+ echo "now qps is $qps"
371
+ fi
372
+
373
+ new_test_name=$test_name"_qps_"$qps
374
+ backend=$CURRENT_LLM_SERVING_ENGINE
375
+
376
+ if [[ "$backend" == *"vllm"* ]]; then
377
+ backend="vllm"
378
+ fi
379
+ #TODO: add output dir.
380
+ client_command="genai-perf profile \
381
+ -m $model \
382
+ --service-kind openai \
383
+ --backend vllm \
384
+ --endpoint-type chat \
385
+ --streaming \
386
+ --url localhost:$port \
387
+ --request-rate $qps \
388
+ --num-prompts $num_prompts \
389
+ "
390
+
391
+ echo "Client command: $client_command"
392
+
393
+ eval "$client_command"
394
+
395
+ #TODO: process/record outputs
396
+ done
397
+ done
398
+
399
+ kill_gpu_processes
400
+
401
+}
402
403
prepare_dataset() {
404
@@ -328,12 +426,17 @@ main() {
426
427
pip install -U transformers
428
429
+ pip install -r requirements-dev.txt
430
+ which genai-perf
431
+
432
# check storage
433
df -h
434
435
ensure_installed wget
436
ensure_installed curl
437
ensure_installed jq
438
+ # genai-perf dependency
439
+ ensure_installed libb64-0d
440
441
prepare_dataset
442
@@ -345,6 +448,10 @@ main() {
448
# run the test
449
run_serving_tests "$BENCHMARK_ROOT/tests/nightly-tests.json"
450
451
+ # run genai-perf tests
452
+ run_genai_perf_tests "$BENCHMARK_ROOT/tests/genai-perf-tests.json"
453
+ mv artifacts/ $RESULTS_FOLDER/
454
+
455
# upload benchmark results to buildkite
456
python3 -m pip install tabulate pandas
457
python3 "$BENCHMARK_ROOT/scripts/summary-nightly-results.py"
.buildkite/nightly-benchmarks/tests/genai-perf-tests.json
new
+23
@@ -0,0 +1,23 @@
1
+[
2
+ {
3
+ "test_name": "llama8B_tp1_genai_perf",
4
+ "qps_list": [4,8,16,32],
5
+ "common_parameters": {
6
+ "model": "meta-llama/Meta-Llama-3-8B-Instruct",
7
+ "tp": 1,
8
+ "port": 8000,
9
+ "num_prompts": 500,
10
+ "reuse_server": false
11
+ },
12
+ "vllm_server_parameters": {
13
+ "disable_log_stats": "",
14
+ "disable_log_requests": "",
15
+ "gpu_memory_utilization": 0.9,
16
+ "num_scheduler_steps": 10,
17
+ "max_num_seqs": 512,
18
+ "dtype": "bfloat16"
19
+ },
20
+ "genai_perf_input_parameters": {
21
+ }
22
+ }
23
+]
\ No newline at end of file
requirements-test.in
+3
@@ -29,4 +29,7 @@ lm-eval[api]==0.4.4 # required for model evaluation test
29
bitsandbytes>=0.45.0
30
buildkite-test-collector==0.1.9
31
32
+genai_perf==0.0.8
33
+tritonclient==2.51.0
34
+
35
numpy < 2.0.0
requirements-test.txt
+63
-4
@@ -37,7 +37,7 @@ audioread==3.0.1
37
# via librosa
38
awscli==1.35.23
39
# via -r requirements-test.in
40
-bitsandbytes>=0.45.0
40
+bitsandbytes==0.45.0
41
# via -r requirements-test.in
42
black==24.10.0
43
# via datamodel-code-generator
@@ -75,6 +75,8 @@ colorama==0.4.6
75
# tqdm-multiprocess
76
contourpy==1.3.0
77
# via matplotlib
78
+cramjam==2.9.0
79
+ # via fastparquet
80
cupy-cuda12x==13.3.0
81
# via ray
82
cycler==0.12.1
@@ -109,6 +111,8 @@ email-validator==2.2.0
111
# via pydantic
112
evaluate==0.4.3
113
# via lm-eval
114
+fastparquet==2024.11.0
115
+ # via genai-perf
116
fastrlock==0.8.2
117
# via cupy-cuda12x
118
filelock==3.16.1
@@ -130,8 +134,11 @@ fsspec[http]==2024.9.0
134
# via
135
# datasets
136
# evaluate
137
+ # fastparquet
138
# huggingface-hub
139
# torch
140
+genai-perf==0.0.8
141
+ # via -r requirements-test.in
142
genson==1.3.0
143
# via datamodel-code-generator
144
h11==0.14.0
@@ -186,6 +193,8 @@ jsonschema==4.23.0
193
# ray
194
jsonschema-specifications==2024.10.1
195
# via jsonschema
196
+kaleido==0.2.1
197
+ # via genai-perf
198
kiwisolver==1.4.7
199
# via matplotlib
200
lazy-loader==0.4
@@ -200,6 +209,8 @@ lm-eval[api]==0.4.4
209
# via -r requirements-test.in
210
lxml==5.3.0
211
# via sacrebleu
212
+markdown-it-py==3.0.0
213
+ # via rich
214
markupsafe==3.0.2
215
# via jinja2
216
matplotlib==3.9.2
@@ -209,6 +220,8 @@ mbstrdecoder==1.1.3
220
# dataproperty
221
# pytablewriter
222
# typepy
223
+mdurl==0.1.2
224
+ # via markdown-it-py
225
mistral-common[opencv]==1.5.1
226
# via
227
# -r requirements-test.in
@@ -249,6 +262,8 @@ numpy==1.26.4
262
# datasets
263
# decord
264
# evaluate
265
+ # fastparquet
266
+ # genai-perf
267
# librosa
268
# matplotlib
269
# mistral-common
@@ -256,15 +271,18 @@ numpy==1.26.4
271
# numexpr
272
# opencv-python-headless
273
# pandas
274
+ # patsy
275
# peft
276
# rouge-score
277
# sacrebleu
278
# scikit-learn
279
# scipy
280
# soxr
281
+ # statsmodels
282
# tensorizer
283
# torchvision
284
# transformers
285
+ # tritonclient
286
nvidia-cublas-cu12==12.4.5.8
287
# via
288
# nvidia-cudnn-cu12
@@ -306,30 +324,39 @@ packaging==24.1
324
# datamodel-code-generator
325
# datasets
326
# evaluate
327
+ # fastparquet
328
# huggingface-hub
329
# lazy-loader
330
# matplotlib
331
# peft
332
+ # plotly
333
# pooch
334
# pytest
335
# pytest-rerunfailures
336
# ray
337
+ # statsmodels
338
# transformers
339
# typepy
340
pandas==2.2.3
341
# via
342
# datasets
343
# evaluate
344
+ # fastparquet
345
+ # genai-perf
346
+ # statsmodels
347
pathspec==0.12.1
348
# via black
349
pathvalidate==3.2.1
350
# via pytablewriter
351
+patsy==1.0.1
352
+ # via statsmodels
353
peft==0.13.2
354
# via
355
# -r requirements-test.in
356
# lm-eval
357
pillow==10.4.0
358
# via
359
+ # genai-perf
360
# matplotlib
361
# mistral-common
362
# sentence-transformers
@@ -338,6 +365,8 @@ platformdirs==4.3.6
365
# via
366
# black
367
# pooch
368
+plotly==5.24.1
369
+ # via genai-perf
370
pluggy==1.5.0
371
# via pytest
372
pooch==1.8.2
@@ -360,7 +389,9 @@ psutil==6.1.0
389
py==1.11.0
390
# via pytest-forked
391
pyarrow==18.0.0
363
- # via datasets
392
+ # via
393
+ # datasets
394
+ # genai-perf
395
pyasn1==0.6.1
396
# via rsa
397
pybind11==2.13.6
@@ -373,6 +404,8 @@ pydantic[email]==2.9.2
404
# mistral-common
405
pydantic-core==2.23.4
406
# via pydantic
407
+pygments==2.18.0
408
+ # via rich
409
pyparsing==3.2.0
410
# via matplotlib
411
pytablewriter==1.2.0
@@ -381,14 +414,18 @@ pytest==8.3.3
414
# via
415
# -r requirements-test.in
416
# buildkite-test-collector
417
+ # genai-perf
418
# pytest-asyncio
419
# pytest-forked
420
+ # pytest-mock
421
# pytest-rerunfailures
422
# pytest-shard
423
pytest-asyncio==0.24.0
424
# via -r requirements-test.in
425
pytest-forked==1.6.0
426
# via -r requirements-test.in
427
+pytest-mock==3.14.0
428
+ # via genai-perf
429
pytest-rerunfailures==14.0
430
# via -r requirements-test.in
431
pytest-shard==0.1.2
@@ -399,6 +436,8 @@ python-dateutil==2.9.0.post0
436
# matplotlib
437
# pandas
438
# typepy
439
+python-rapidjson==1.20
440
+ # via tritonclient
441
pytz==2024.2
442
# via
443
# pandas
@@ -409,9 +448,11 @@ pyyaml==6.0.2
448
# awscli
449
# datamodel-code-generator
450
# datasets
451
+ # genai-perf
452
# huggingface-hub
453
# peft
454
# ray
455
+ # responses
456
# timm
457
# transformers
458
ray[adag]==2.40.0
@@ -438,8 +479,13 @@ requests==2.32.3
479
# mistral-common
480
# pooch
481
# ray
482
+ # responses
483
# tiktoken
484
# transformers
485
+responses==0.25.3
486
+ # via genai-perf
487
+rich==13.9.4
488
+ # via genai-perf
489
rouge-score==0.1.2
490
# via lm-eval
491
rpds-py==0.20.1
@@ -470,6 +516,7 @@ scipy==1.13.1
516
# librosa
517
# scikit-learn
518
# sentence-transformers
519
+ # statsmodels
520
sentence-transformers==3.2.1
521
# via -r requirements-test.in
522
sentencepiece==0.2.0
@@ -490,6 +537,8 @@ soxr==0.5.0.post1
537
# via librosa
538
sqlitedict==2.1.0
539
# via lm-eval
540
+statsmodels==0.14.4
541
+ # via genai-perf
542
sympy==1.13.1
543
# via torch
544
tabledata==1.3.3
@@ -499,7 +548,9 @@ tabulate==0.9.0
548
tcolorpy==0.1.6
549
# via pytablewriter
550
tenacity==9.0.0
502
- # via lm-eval
551
+ # via
552
+ # lm-eval
553
+ # plotly
554
tensorizer==2.9.0
555
# via -r requirements-test.in
556
threadpoolctl==3.5.0
@@ -540,6 +591,7 @@ tqdm-multiprocess==0.0.11
591
# via lm-eval
592
transformers==4.47.0
593
# via
594
+ # genai-perf
595
# lm-eval
596
# peft
597
# sentence-transformers
@@ -548,6 +600,10 @@ transformers-stream-generator==0.0.5
600
# via -r requirements-test.in
601
triton==3.1.0
602
# via torch
603
+tritonclient==2.51.0
604
+ # via
605
+ # -r requirements-test.in
606
+ # genai-perf
607
typepy[datetime]==1.3.2
608
# via
609
# dataproperty
@@ -555,6 +611,7 @@ typepy[datetime]==1.3.2
611
# tabledata
612
typing-extensions==4.12.2
613
# via
614
+ # bitsandbytes
615
# huggingface-hub
616
# librosa
617
# mistral-common
@@ -563,10 +620,12 @@ typing-extensions==4.12.2
620
# torch
621
tzdata==2024.2
622
# via pandas
566
-urllib3==1.26.20
623
+urllib3==2.2.3
624
# via
625
# botocore
626
# requests
627
+ # responses
628
+ # tritonclient
629
word2number==1.1
630
# via lm-eval
631
xxhash==3.5.0