A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[CI/Build][ROCm] Enabling LoRA tests on ROCm (#7369)
Co-authored-by: Simon Mo <simon.mo@hey.com>
alexeykondrat committed
Sep 4, 2024 at 14:57 UTC
d1dec6424307a6070bf3ab1700633996f20ef248
4 files changed
+64
-14
.buildkite/run-amd-test.sh
+41
-6
@@ -1,5 +1,5 @@
1
# This script runs test inside the corresponding ROCm docker container.
2
-set -ex
2
+set -o pipefail
3
4
# Print ROCm version
5
echo "--- Confirming Clean Initial State"
@@ -70,16 +70,51 @@ HF_CACHE="$(realpath ~)/huggingface"
70
mkdir -p ${HF_CACHE}
71
HF_MOUNT="/root/.cache/huggingface"
72
73
-docker run \
73
+commands=$@
74
+PARALLEL_JOB_COUNT=8
75
+# check if the command contains shard flag, we will run all shards in parallel because the host have 8 GPUs.
76
+if [[ $commands == *"--shard-id="* ]]; then
77
+ for GPU in $(seq 0 $(($PARALLEL_JOB_COUNT-1))); do
78
+ #replace shard arguments
79
+ commands=${@//"--shard-id= "/"--shard-id=${GPU} "}
80
+ commands=${commands//"--num-shards= "/"--num-shards=${PARALLEL_JOB_COUNT} "}
81
+ docker run \
82
--device /dev/kfd --device /dev/dri \
83
--network host \
84
--shm-size=16gb \
85
--rm \
78
- -e HIP_VISIBLE_DEVICES=0 \
86
+ -e HIP_VISIBLE_DEVICES=${GPU} \
87
-e HF_TOKEN \
88
-v ${HF_CACHE}:${HF_MOUNT} \
89
-e HF_HOME=${HF_MOUNT} \
82
- --name ${container_name} \
90
+ --name ${container_name}_${GPU} \
91
${image_name} \
84
- /bin/bash -c "${@}"
85
-
92
+ /bin/bash -c "${commands}" \
93
+ |& while read -r line; do echo ">>Shard $GPU: $line"; done &
94
+ PIDS+=($!)
95
+ done
96
+ #wait for all processes to finish and collect exit codes
97
+ for pid in ${PIDS[@]}; do
98
+ wait ${pid}
99
+ STATUS+=($?)
100
+ done
101
+ for st in ${STATUS[@]}; do
102
+ if [[ ${st} -ne 0 ]]; then
103
+ echo "One of the processes failed with $st"
104
+ exit ${st}
105
+ fi
106
+ done
107
+else
108
+ docker run \
109
+ --device /dev/kfd --device /dev/dri \
110
+ --network host \
111
+ --shm-size=16gb \
112
+ --rm \
113
+ -e HIP_VISIBLE_DEVICES=0 \
114
+ -e HF_TOKEN \
115
+ -v ${HF_CACHE}:${HF_MOUNT} \
116
+ -e HF_HOME=${HF_MOUNT} \
117
+ --name ${container_name} \
118
+ ${image_name} \
119
+ /bin/bash -c "${commands}"
120
+fi
.buildkite/test-pipeline.yaml
+1
-2
@@ -218,9 +218,9 @@ steps:
218
- pytest -v -s spec_decode
219
220
- label: LoRA Test %N # 30min each
221
+ mirror_hardwares: [amd]
222
source_file_dependencies:
223
- vllm/lora
223
- - csrc/punica
224
- tests/lora
225
command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_long_context.py
226
parallelism: 4
@@ -360,7 +360,6 @@ steps:
360
num_gpus: 4
361
source_file_dependencies:
362
- vllm/lora
363
- - csrc/punica
363
- tests/lora/test_long_context
364
commands:
365
# FIXIT: find out which code initialize cuda before running the test
tests/lora/test_gemma.py
+4
@@ -1,7 +1,10 @@
1
from typing import List
2
3
+import pytest
4
+
5
import vllm
6
from vllm.lora.request import LoRARequest
7
+from vllm.utils import is_hip
8
9
MODEL_PATH = "google/gemma-7b"
10
@@ -28,6 +31,7 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
31
return generated_texts
32
33
34
+@pytest.mark.xfail(is_hip(), reason="There can be output mismatch on ROCm")
35
def test_gemma_lora(gemma_lora_files):
36
llm = vllm.LLM(MODEL_PATH,
37
max_model_len=1024,
tests/lora/test_quant_model.py
+18
-6
@@ -7,6 +7,7 @@ import pytest
7
8
import vllm
9
from vllm.lora.request import LoRARequest
10
+from vllm.utils import is_hip
11
12
from .conftest import cleanup
13
@@ -17,12 +18,23 @@ class ModelWithQuantization:
18
quantization: str
19
20
20
-MODELS: List[ModelWithQuantization] = [
21
- ModelWithQuantization(model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ",
22
- quantization="AWQ"),
23
- ModelWithQuantization(model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ",
24
- quantization="GPTQ"),
25
-]
21
+MODELS: List[ModelWithQuantization]
22
+#AWQ quantization is currently not supported in ROCm.
23
+if is_hip():
24
+ MODELS = [
25
+ ModelWithQuantization(
26
+ model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ",
27
+ quantization="GPTQ"),
28
+ ]
29
+else:
30
+ MODELS = [
31
+ ModelWithQuantization(
32
+ model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ",
33
+ quantization="AWQ"),
34
+ ModelWithQuantization(
35
+ model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ",
36
+ quantization="GPTQ"),
37
+ ]
38
39
40
def do_sample(llm: vllm.LLM,