@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5208dc7a20
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Bugfix][CI/Build][Hardware][AMD] Shard ID parameters in AMD tests running parallel jobs (#9279)

Signed-off-by: Hissu Hyvarinen <hissu.hyvarinen@amd.com>

hissu-hyvarinen committed Nov 4, 2024 at 21:37 UTC 5208dc7a203b210fa4462332a56c0012ab8b7a89
2 files changed +12 -6
.buildkite/run-amd-test.sh
+6 -5
@@ -107,11 +107,12 @@ fi
107 PARALLEL_JOB_COUNT=8
108 # check if the command contains shard flag, we will run all shards in parallel because the host have 8 GPUs.
109 if [[ $commands == *"--shard-id="* ]]; then
110 + # assign job count as the number of shards used
111 + commands=${commands//"--num-shards= "/"--num-shards=${PARALLEL_JOB_COUNT} "}
112 for GPU in $(seq 0 $(($PARALLEL_JOB_COUNT-1))); do
111 - #replace shard arguments
112 - commands=${commands//"--shard-id= "/"--shard-id=${GPU} "}
113 - commands=${commands//"--num-shards= "/"--num-shards=${PARALLEL_JOB_COUNT} "}
114 - echo "Shard ${GPU} commands:$commands"
113 + # assign shard-id for each shard
114 + commands_gpu=${commands//"--shard-id= "/"--shard-id=${GPU} "}
115 + echo "Shard ${GPU} commands:$commands_gpu"
116 docker run \
117 --device /dev/kfd --device /dev/dri \
118 --network host \
@@ -123,7 +124,7 @@ if [[ $commands == *"--shard-id="* ]]; then
124 -e HF_HOME=${HF_MOUNT} \
125 --name ${container_name}_${GPU} \
126 ${image_name} \
126 - /bin/bash -c "${commands}" \
127 + /bin/bash -c "${commands_gpu}" \
128 |& while read -r line; do echo ">>Shard $GPU: $line"; done &
129 PIDS+=($!)
130 done
tests/lora/test_minicpmv.py
+6 -1
@@ -1,8 +1,11 @@
1 from typing import List
2
3 +import pytest
4 +
5 import vllm
6 from vllm.assets.image import ImageAsset
7 from vllm.lora.request import LoRARequest
8 +from vllm.platforms import current_platform
9
10 MODEL_PATH = "openbmb/MiniCPM-Llama3-V-2_5"
11
@@ -53,6 +56,9 @@ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> List[str]:
56 return generated_texts
57
58
59 +@pytest.mark.xfail(
60 + current_platform.is_rocm(),
61 + reason="MiniCPM-V dependency xformers incompatible with ROCm")
62 def test_minicpmv_lora(minicpmv_lora_files):
63 llm = vllm.LLM(
64 MODEL_PATH,
@@ -63,7 +69,6 @@ def test_minicpmv_lora(minicpmv_lora_files):
69 trust_remote_code=True,
70 gpu_memory_utilization=0.97 # This model is pretty big for CI gpus
71 )
66 -
72 output1 = do_sample(llm, minicpmv_lora_files, lora_id=1)
73 for i in range(len(EXPECTED_OUTPUT)):
74 assert EXPECTED_OUTPUT[i].startswith(output1[i])