@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fead53ba78
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[CI]add genai-perf benchmark in nightly benchmark (#10704)

Signed-off-by: Kunshang Ji <kunshang.ji@intel.com>

Kunshang Ji committed Jan 17, 2025 at 12:15 UTC fead53ba78dbcdd4da616308f1ef1b4a312f8897
4 files changed +196 -4
.buildkite/nightly-benchmarks/scripts/run-nightly-benchmarks.sh
+107
@@ -301,6 +301,104 @@ run_serving_tests() {
301 kill_gpu_processes
302 }
303
304 +run_genai_perf_tests() {
305 + # run genai-perf tests
306 +
307 + # $1: a json file specifying genai-perf test cases
308 + local genai_perf_test_file
309 + genai_perf_test_file=$1
310 +
311 + # Iterate over genai-perf tests
312 + jq -c '.[]' "$genai_perf_test_file" | while read -r params; do
313 + # get the test name, and append the GPU type back to it.
314 + test_name=$(echo "$params" | jq -r '.test_name')
315 +
316 + # if TEST_SELECTOR is set, only run the test cases that match the selector
317 + if [[ -n "$TEST_SELECTOR" ]] && [[ ! "$test_name" =~ $TEST_SELECTOR ]]; then
318 + echo "Skip test case $test_name."
319 + continue
320 + fi
321 +
322 + # prepend the current serving engine to the test name
323 + test_name=${CURRENT_LLM_SERVING_ENGINE}_${test_name}
324 +
325 + # get common parameters
326 + common_params=$(echo "$params" | jq -r '.common_parameters')
327 + model=$(echo "$common_params" | jq -r '.model')
328 + tp=$(echo "$common_params" | jq -r '.tp')
329 + dataset_name=$(echo "$common_params" | jq -r '.dataset_name')
330 + dataset_path=$(echo "$common_params" | jq -r '.dataset_path')
331 + port=$(echo "$common_params" | jq -r '.port')
332 + num_prompts=$(echo "$common_params" | jq -r '.num_prompts')
333 + reuse_server=$(echo "$common_params" | jq -r '.reuse_server')
334 +
335 + # get client and server arguments
336 + server_params=$(echo "$params" | jq -r ".${CURRENT_LLM_SERVING_ENGINE}_server_parameters")
337 + qps_list=$(echo "$params" | jq -r '.qps_list')
338 + qps_list=$(echo "$qps_list" | jq -r '.[] | @sh')
339 + echo "Running over qps list $qps_list"
340 +
341 + # check if there is enough GPU to run the test
342 + if [[ $gpu_count -lt $tp ]]; then
343 + echo "Required num-shard $tp but only $gpu_count GPU found. Skip testcase $test_name."
344 + continue
345 + fi
346 +
347 + if [[ $reuse_server == "true" ]]; then
348 + echo "Reuse previous server for test case $test_name"
349 + else
350 + kill_gpu_processes
351 + bash "$VLLM_SOURCE_CODE_LOC/.buildkite/nightly-benchmarks/scripts/launch-server.sh" \
352 + "$server_params" "$common_params"
353 + fi
354 +
355 + if wait_for_server; then
356 + echo ""
357 + echo "$CURRENT_LLM_SERVING_ENGINE server is up and running."
358 + else
359 + echo ""
360 + echo "$CURRENT_LLM_SERVING_ENGINE failed to start within the timeout period."
361 + break
362 + fi
363 +
364 + # iterate over different QPS
365 + for qps in $qps_list; do
366 + # remove the surrounding single quote from qps
367 + if [[ "$qps" == *"inf"* ]]; then
368 + echo "qps was $qps"
369 + qps=$num_prompts
370 + echo "now qps is $qps"
371 + fi
372 +
373 + new_test_name=$test_name"_qps_"$qps
374 + backend=$CURRENT_LLM_SERVING_ENGINE
375 +
376 + if [[ "$backend" == *"vllm"* ]]; then
377 + backend="vllm"
378 + fi
379 + #TODO: add output dir.
380 + client_command="genai-perf profile \
381 + -m $model \
382 + --service-kind openai \
383 + --backend vllm \
384 + --endpoint-type chat \
385 + --streaming \
386 + --url localhost:$port \
387 + --request-rate $qps \
388 + --num-prompts $num_prompts \
389 + "
390 +
391 + echo "Client command: $client_command"
392 +
393 + eval "$client_command"
394 +
395 + #TODO: process/record outputs
396 + done
397 + done
398 +
399 + kill_gpu_processes
400 +
401 +}
402
403 prepare_dataset() {
404
@@ -328,12 +426,17 @@ main() {
426
427 pip install -U transformers
428
429 + pip install -r requirements-dev.txt
430 + which genai-perf
431 +
432 # check storage
433 df -h
434
435 ensure_installed wget
436 ensure_installed curl
437 ensure_installed jq
438 + # genai-perf dependency
439 + ensure_installed libb64-0d
440
441 prepare_dataset
442
@@ -345,6 +448,10 @@ main() {
448 # run the test
449 run_serving_tests "$BENCHMARK_ROOT/tests/nightly-tests.json"
450
451 + # run genai-perf tests
452 + run_genai_perf_tests "$BENCHMARK_ROOT/tests/genai-perf-tests.json"
453 + mv artifacts/ $RESULTS_FOLDER/
454 +
455 # upload benchmark results to buildkite
456 python3 -m pip install tabulate pandas
457 python3 "$BENCHMARK_ROOT/scripts/summary-nightly-results.py"
.buildkite/nightly-benchmarks/tests/genai-perf-tests.json new
+23
@@ -0,0 +1,23 @@
1 +[
2 + {
3 + "test_name": "llama8B_tp1_genai_perf",
4 + "qps_list": [4,8,16,32],
5 + "common_parameters": {
6 + "model": "meta-llama/Meta-Llama-3-8B-Instruct",
7 + "tp": 1,
8 + "port": 8000,
9 + "num_prompts": 500,
10 + "reuse_server": false
11 + },
12 + "vllm_server_parameters": {
13 + "disable_log_stats": "",
14 + "disable_log_requests": "",
15 + "gpu_memory_utilization": 0.9,
16 + "num_scheduler_steps": 10,
17 + "max_num_seqs": 512,
18 + "dtype": "bfloat16"
19 + },
20 + "genai_perf_input_parameters": {
21 + }
22 + }
23 +]
\ No newline at end of file
requirements-test.in
+3
@@ -29,4 +29,7 @@ lm-eval[api]==0.4.4 # required for model evaluation test
29 bitsandbytes>=0.45.0
30 buildkite-test-collector==0.1.9
31
32 +genai_perf==0.0.8
33 +tritonclient==2.51.0
34 +
35 numpy < 2.0.0
requirements-test.txt
+63 -4
@@ -37,7 +37,7 @@ audioread==3.0.1
37 # via librosa
38 awscli==1.35.23
39 # via -r requirements-test.in
40 -bitsandbytes>=0.45.0
40 +bitsandbytes==0.45.0
41 # via -r requirements-test.in
42 black==24.10.0
43 # via datamodel-code-generator
@@ -75,6 +75,8 @@ colorama==0.4.6
75 # tqdm-multiprocess
76 contourpy==1.3.0
77 # via matplotlib
78 +cramjam==2.9.0
79 + # via fastparquet
80 cupy-cuda12x==13.3.0
81 # via ray
82 cycler==0.12.1
@@ -109,6 +111,8 @@ email-validator==2.2.0
111 # via pydantic
112 evaluate==0.4.3
113 # via lm-eval
114 +fastparquet==2024.11.0
115 + # via genai-perf
116 fastrlock==0.8.2
117 # via cupy-cuda12x
118 filelock==3.16.1
@@ -130,8 +134,11 @@ fsspec[http]==2024.9.0
134 # via
135 # datasets
136 # evaluate
137 + # fastparquet
138 # huggingface-hub
139 # torch
140 +genai-perf==0.0.8
141 + # via -r requirements-test.in
142 genson==1.3.0
143 # via datamodel-code-generator
144 h11==0.14.0
@@ -186,6 +193,8 @@ jsonschema==4.23.0
193 # ray
194 jsonschema-specifications==2024.10.1
195 # via jsonschema
196 +kaleido==0.2.1
197 + # via genai-perf
198 kiwisolver==1.4.7
199 # via matplotlib
200 lazy-loader==0.4
@@ -200,6 +209,8 @@ lm-eval[api]==0.4.4
209 # via -r requirements-test.in
210 lxml==5.3.0
211 # via sacrebleu
212 +markdown-it-py==3.0.0
213 + # via rich
214 markupsafe==3.0.2
215 # via jinja2
216 matplotlib==3.9.2
@@ -209,6 +220,8 @@ mbstrdecoder==1.1.3
220 # dataproperty
221 # pytablewriter
222 # typepy
223 +mdurl==0.1.2
224 + # via markdown-it-py
225 mistral-common[opencv]==1.5.1
226 # via
227 # -r requirements-test.in
@@ -249,6 +262,8 @@ numpy==1.26.4
262 # datasets
263 # decord
264 # evaluate
265 + # fastparquet
266 + # genai-perf
267 # librosa
268 # matplotlib
269 # mistral-common
@@ -256,15 +271,18 @@ numpy==1.26.4
271 # numexpr
272 # opencv-python-headless
273 # pandas
274 + # patsy
275 # peft
276 # rouge-score
277 # sacrebleu
278 # scikit-learn
279 # scipy
280 # soxr
281 + # statsmodels
282 # tensorizer
283 # torchvision
284 # transformers
285 + # tritonclient
286 nvidia-cublas-cu12==12.4.5.8
287 # via
288 # nvidia-cudnn-cu12
@@ -306,30 +324,39 @@ packaging==24.1
324 # datamodel-code-generator
325 # datasets
326 # evaluate
327 + # fastparquet
328 # huggingface-hub
329 # lazy-loader
330 # matplotlib
331 # peft
332 + # plotly
333 # pooch
334 # pytest
335 # pytest-rerunfailures
336 # ray
337 + # statsmodels
338 # transformers
339 # typepy
340 pandas==2.2.3
341 # via
342 # datasets
343 # evaluate
344 + # fastparquet
345 + # genai-perf
346 + # statsmodels
347 pathspec==0.12.1
348 # via black
349 pathvalidate==3.2.1
350 # via pytablewriter
351 +patsy==1.0.1
352 + # via statsmodels
353 peft==0.13.2
354 # via
355 # -r requirements-test.in
356 # lm-eval
357 pillow==10.4.0
358 # via
359 + # genai-perf
360 # matplotlib
361 # mistral-common
362 # sentence-transformers
@@ -338,6 +365,8 @@ platformdirs==4.3.6
365 # via
366 # black
367 # pooch
368 +plotly==5.24.1
369 + # via genai-perf
370 pluggy==1.5.0
371 # via pytest
372 pooch==1.8.2
@@ -360,7 +389,9 @@ psutil==6.1.0
389 py==1.11.0
390 # via pytest-forked
391 pyarrow==18.0.0
363 - # via datasets
392 + # via
393 + # datasets
394 + # genai-perf
395 pyasn1==0.6.1
396 # via rsa
397 pybind11==2.13.6
@@ -373,6 +404,8 @@ pydantic[email]==2.9.2
404 # mistral-common
405 pydantic-core==2.23.4
406 # via pydantic
407 +pygments==2.18.0
408 + # via rich
409 pyparsing==3.2.0
410 # via matplotlib
411 pytablewriter==1.2.0
@@ -381,14 +414,18 @@ pytest==8.3.3
414 # via
415 # -r requirements-test.in
416 # buildkite-test-collector
417 + # genai-perf
418 # pytest-asyncio
419 # pytest-forked
420 + # pytest-mock
421 # pytest-rerunfailures
422 # pytest-shard
423 pytest-asyncio==0.24.0
424 # via -r requirements-test.in
425 pytest-forked==1.6.0
426 # via -r requirements-test.in
427 +pytest-mock==3.14.0
428 + # via genai-perf
429 pytest-rerunfailures==14.0
430 # via -r requirements-test.in
431 pytest-shard==0.1.2
@@ -399,6 +436,8 @@ python-dateutil==2.9.0.post0
436 # matplotlib
437 # pandas
438 # typepy
439 +python-rapidjson==1.20
440 + # via tritonclient
441 pytz==2024.2
442 # via
443 # pandas
@@ -409,9 +448,11 @@ pyyaml==6.0.2
448 # awscli
449 # datamodel-code-generator
450 # datasets
451 + # genai-perf
452 # huggingface-hub
453 # peft
454 # ray
455 + # responses
456 # timm
457 # transformers
458 ray[adag]==2.40.0
@@ -438,8 +479,13 @@ requests==2.32.3
479 # mistral-common
480 # pooch
481 # ray
482 + # responses
483 # tiktoken
484 # transformers
485 +responses==0.25.3
486 + # via genai-perf
487 +rich==13.9.4
488 + # via genai-perf
489 rouge-score==0.1.2
490 # via lm-eval
491 rpds-py==0.20.1
@@ -470,6 +516,7 @@ scipy==1.13.1
516 # librosa
517 # scikit-learn
518 # sentence-transformers
519 + # statsmodels
520 sentence-transformers==3.2.1
521 # via -r requirements-test.in
522 sentencepiece==0.2.0
@@ -490,6 +537,8 @@ soxr==0.5.0.post1
537 # via librosa
538 sqlitedict==2.1.0
539 # via lm-eval
540 +statsmodels==0.14.4
541 + # via genai-perf
542 sympy==1.13.1
543 # via torch
544 tabledata==1.3.3
@@ -499,7 +548,9 @@ tabulate==0.9.0
548 tcolorpy==0.1.6
549 # via pytablewriter
550 tenacity==9.0.0
502 - # via lm-eval
551 + # via
552 + # lm-eval
553 + # plotly
554 tensorizer==2.9.0
555 # via -r requirements-test.in
556 threadpoolctl==3.5.0
@@ -540,6 +591,7 @@ tqdm-multiprocess==0.0.11
591 # via lm-eval
592 transformers==4.47.0
593 # via
594 + # genai-perf
595 # lm-eval
596 # peft
597 # sentence-transformers
@@ -548,6 +600,10 @@ transformers-stream-generator==0.0.5
600 # via -r requirements-test.in
601 triton==3.1.0
602 # via torch
603 +tritonclient==2.51.0
604 + # via
605 + # -r requirements-test.in
606 + # genai-perf
607 typepy[datetime]==1.3.2
608 # via
609 # dataproperty
@@ -555,6 +611,7 @@ typepy[datetime]==1.3.2
611 # tabledata
612 typing-extensions==4.12.2
613 # via
614 + # bitsandbytes
615 # huggingface-hub
616 # librosa
617 # mistral-common
@@ -563,10 +620,12 @@ typing-extensions==4.12.2
620 # torch
621 tzdata==2024.2
622 # via pandas
566 -urllib3==1.26.20
623 +urllib3==2.2.3
624 # via
625 # botocore
626 # requests
627 + # responses
628 + # tritonclient
629 word2number==1.1
630 # via lm-eval
631 xxhash==3.5.0