@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d1dec64243
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[CI/Build][ROCm] Enabling LoRA tests on ROCm (#7369)

Co-authored-by: Simon Mo <simon.mo@hey.com>

alexeykondrat committed Sep 4, 2024 at 14:57 UTC d1dec6424307a6070bf3ab1700633996f20ef248
4 files changed +64 -14
.buildkite/run-amd-test.sh
+41 -6
@@ -1,5 +1,5 @@
1 # This script runs test inside the corresponding ROCm docker container.
2 -set -ex
2 +set -o pipefail
3
4 # Print ROCm version
5 echo "--- Confirming Clean Initial State"
@@ -70,16 +70,51 @@ HF_CACHE="$(realpath ~)/huggingface"
70 mkdir -p ${HF_CACHE}
71 HF_MOUNT="/root/.cache/huggingface"
72
73 -docker run \
73 +commands=$@
74 +PARALLEL_JOB_COUNT=8
75 +# check if the command contains shard flag, we will run all shards in parallel because the host have 8 GPUs.
76 +if [[ $commands == *"--shard-id="* ]]; then
77 + for GPU in $(seq 0 $(($PARALLEL_JOB_COUNT-1))); do
78 + #replace shard arguments
79 + commands=${@//"--shard-id= "/"--shard-id=${GPU} "}
80 + commands=${commands//"--num-shards= "/"--num-shards=${PARALLEL_JOB_COUNT} "}
81 + docker run \
82 --device /dev/kfd --device /dev/dri \
83 --network host \
84 --shm-size=16gb \
85 --rm \
78 - -e HIP_VISIBLE_DEVICES=0 \
86 + -e HIP_VISIBLE_DEVICES=${GPU} \
87 -e HF_TOKEN \
88 -v ${HF_CACHE}:${HF_MOUNT} \
89 -e HF_HOME=${HF_MOUNT} \
82 - --name ${container_name} \
90 + --name ${container_name}_${GPU} \
91 ${image_name} \
84 - /bin/bash -c "${@}"
85 -
92 + /bin/bash -c "${commands}" \
93 + |& while read -r line; do echo ">>Shard $GPU: $line"; done &
94 + PIDS+=($!)
95 + done
96 + #wait for all processes to finish and collect exit codes
97 + for pid in ${PIDS[@]}; do
98 + wait ${pid}
99 + STATUS+=($?)
100 + done
101 + for st in ${STATUS[@]}; do
102 + if [[ ${st} -ne 0 ]]; then
103 + echo "One of the processes failed with $st"
104 + exit ${st}
105 + fi
106 + done
107 +else
108 + docker run \
109 + --device /dev/kfd --device /dev/dri \
110 + --network host \
111 + --shm-size=16gb \
112 + --rm \
113 + -e HIP_VISIBLE_DEVICES=0 \
114 + -e HF_TOKEN \
115 + -v ${HF_CACHE}:${HF_MOUNT} \
116 + -e HF_HOME=${HF_MOUNT} \
117 + --name ${container_name} \
118 + ${image_name} \
119 + /bin/bash -c "${commands}"
120 +fi
.buildkite/test-pipeline.yaml
+1 -2
@@ -218,9 +218,9 @@ steps:
218 - pytest -v -s spec_decode
219
220 - label: LoRA Test %N # 30min each
221 + mirror_hardwares: [amd]
222 source_file_dependencies:
223 - vllm/lora
223 - - csrc/punica
224 - tests/lora
225 command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_long_context.py
226 parallelism: 4
@@ -360,7 +360,6 @@ steps:
360 num_gpus: 4
361 source_file_dependencies:
362 - vllm/lora
363 - - csrc/punica
363 - tests/lora/test_long_context
364 commands:
365 # FIXIT: find out which code initialize cuda before running the test
tests/lora/test_gemma.py
+4
@@ -1,7 +1,10 @@
1 from typing import List
2
3 +import pytest
4 +
5 import vllm
6 from vllm.lora.request import LoRARequest
7 +from vllm.utils import is_hip
8
9 MODEL_PATH = "google/gemma-7b"
10
@@ -28,6 +31,7 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
31 return generated_texts
32
33
34 +@pytest.mark.xfail(is_hip(), reason="There can be output mismatch on ROCm")
35 def test_gemma_lora(gemma_lora_files):
36 llm = vllm.LLM(MODEL_PATH,
37 max_model_len=1024,
tests/lora/test_quant_model.py
+18 -6
@@ -7,6 +7,7 @@ import pytest
7
8 import vllm
9 from vllm.lora.request import LoRARequest
10 +from vllm.utils import is_hip
11
12 from .conftest import cleanup
13
@@ -17,12 +18,23 @@ class ModelWithQuantization:
18 quantization: str
19
20
20 -MODELS: List[ModelWithQuantization] = [
21 - ModelWithQuantization(model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ",
22 - quantization="AWQ"),
23 - ModelWithQuantization(model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ",
24 - quantization="GPTQ"),
25 -]
21 +MODELS: List[ModelWithQuantization]
22 +#AWQ quantization is currently not supported in ROCm.
23 +if is_hip():
24 + MODELS = [
25 + ModelWithQuantization(
26 + model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ",
27 + quantization="GPTQ"),
28 + ]
29 +else:
30 + MODELS = [
31 + ModelWithQuantization(
32 + model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ",
33 + quantization="AWQ"),
34 + ModelWithQuantization(
35 + model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ",
36 + quantization="GPTQ"),
37 + ]
38
39
40 def do_sample(llm: vllm.LLM,