A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Build/CI] Extending the set of AMD tests with Regression, Basic Correctness, Distributed, Engine, Llava Tests (#4797)
Alexei-V-Ivanov-AMD committed
May 16, 2024 at 22:58 UTC
26148120b3c05704409a425d017f0a51fca3b7cc
5 files changed
+28
-20
.buildkite/run-amd-test.sh
+6
-5
@@ -1,4 +1,4 @@
1
-# This script build the ROCm docker image and runs test inside it.
1
+# This script runs test inside the corresponding ROCm docker container.
2
set -ex
3
4
# Print ROCm version
@@ -19,15 +19,16 @@ done
19
20
echo "--- Building container"
21
sha=$(git rev-parse --short HEAD)
22
-container_name=rocm_${sha}
22
+image_name=rocm_${sha}
23
+container_name=rocm_${sha}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)
24
docker build \
24
- -t ${container_name} \
25
+ -t ${image_name} \
26
-f Dockerfile.rocm \
27
--progress plain \
28
.
29
30
remove_docker_container() {
30
- docker rm -f ${container_name} || docker image rm -f ${container_name} || true
31
+ docker rm -f ${container_name} || docker image rm -f ${image_name} || true
32
}
33
trap remove_docker_container EXIT
34
@@ -39,6 +40,6 @@ docker run \
40
--rm \
41
-e HF_TOKEN \
42
--name ${container_name} \
42
- ${container_name} \
43
+ ${image_name} \
44
/bin/bash -c "${@}"
45
.buildkite/test-pipeline.yaml
+15
-3
@@ -5,13 +5,16 @@
5
6
steps:
7
- label: Regression Test
8
+ mirror_hardwares: [amd]
9
command: pytest -v -s test_regression.py
10
working_dir: "/vllm-workspace/tests" # optional
11
12
- label: AsyncEngine Test
13
+ #mirror_hardwares: [amd]
14
command: pytest -v -s async_engine
15
16
- label: Basic Correctness Test
17
+ mirror_hardwares: [amd]
18
commands:
19
- VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_basic_correctness.py
20
- VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_basic_correctness.py
@@ -24,14 +27,15 @@ steps:
27
command: pytest -v -s core
28
29
- label: Distributed Comm Ops Test
30
+ #mirror_hardwares: [amd]
31
command: pytest -v -s distributed/test_comm_ops.py
32
working_dir: "/vllm-workspace/tests"
33
num_gpus: 2
34
35
- label: Distributed Tests
36
+ mirror_hardwares: [amd]
37
working_dir: "/vllm-workspace/tests"
38
num_gpus: 2
34
- mirror_hardwares: [amd]
39
commands:
40
- pytest -v -s distributed/test_pynccl_library.py
41
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
@@ -45,16 +49,18 @@ steps:
49
- pytest -v -s spec_decode/e2e/test_integration_dist.py
50
51
- label: Distributed Tests (Multiple Groups)
52
+ #mirror_hardwares: [amd]
53
working_dir: "/vllm-workspace/tests"
54
num_gpus: 4
55
commands:
56
- pytest -v -s distributed/test_pynccl.py
57
58
- label: Engine Test
54
- #mirror_hardwares: [amd]
59
+ mirror_hardwares: [amd]
60
command: pytest -v -s engine tokenization test_sequence.py test_config.py test_logger.py
61
62
- label: Entrypoints Test
63
+ #mirror_hardwares: [amd]
64
commands:
65
# these tests have to be separated, because each one will allocate all posible GPU memory
66
- pytest -v -s entrypoints --ignore=entrypoints/test_server_oot_registration.py
@@ -74,6 +80,7 @@ steps:
80
- python3 tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
81
82
- label: Kernels Test %N
83
+ #mirror_hardwares: [amd]
84
command: pytest -v -s kernels --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
85
parallelism: 4
86
@@ -84,7 +91,7 @@ steps:
91
- pytest -v -s models --ignore=models/test_llava.py
92
93
- label: Llava Test
87
- #mirror_hardwares: [amd]
94
+ mirror_hardwares: [amd]
95
commands:
96
- bash ../.buildkite/download-images.sh
97
- pytest -v -s models/test_llava.py
@@ -95,6 +102,7 @@ steps:
102
- pytest -v -s prefix_caching
103
104
- label: Samplers Test
105
+ #mirror_hardwares: [amd]
106
command: pytest -v -s samplers
107
108
- label: LogitsProcessor Test
@@ -110,16 +118,20 @@ steps:
118
command: pytest -v -s spec_decode
119
120
- label: LoRA Test %N
121
+ #mirror_hardwares: [amd]
122
command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
123
parallelism: 4
124
125
- label: Tensorizer Test
126
+ #mirror_hardwares: [amd]
127
command: apt-get install curl libsodium23 && pytest -v -s tensorizer_loader
128
129
- label: Metrics Test
130
+ mirror_hardwares: [amd]
131
command: pytest -v -s metrics
132
133
- label: Quantization Test
134
+ #mirror_hardwares: [amd]
135
command: pytest -v -s quantization
136
137
- label: Benchmarks
.buildkite/test-template.j2
+1
-2
@@ -3,9 +3,8 @@
3
{% set default_working_dir = "/vllm-workspace/tests" %}
4
5
steps:
6
-
6
- label: ":docker: build image"
8
- commands:
7
+ commands:
8
- "docker build --build-arg max_jobs=16 --tag {{ docker_image }} --target test --progress plain ."
9
- "docker push {{ docker_image }}"
10
env:
tests/engine/test_stop_reason.py
+5
-1
@@ -32,6 +32,7 @@ def test_stop_reason(vllm_model, example_prompts):
32
# test stop token
33
outputs = llm.generate(example_prompts,
34
sampling_params=SamplingParams(
35
+ ignore_eos=True,
36
seed=SEED,
37
max_tokens=MAX_TOKENS,
38
stop_token_ids=[stop_token_id]))
@@ -43,7 +44,10 @@ def test_stop_reason(vllm_model, example_prompts):
44
# test stop string
45
outputs = llm.generate(example_prompts,
46
sampling_params=SamplingParams(
46
- seed=SEED, max_tokens=MAX_TOKENS, stop="."))
47
+ ignore_eos=True,
48
+ seed=SEED,
49
+ max_tokens=MAX_TOKENS,
50
+ stop="."))
51
for output in outputs:
52
output = output.outputs[0]
53
assert output.finish_reason == "stop"
vllm/config.py
+1
-9
@@ -1060,7 +1060,7 @@ _STR_DTYPE_TO_TORCH_DTYPE = {
1060
"bfloat16": torch.bfloat16,
1061
}
1062
1063
-_ROCM_NOT_SUPPORTED_DTYPE = ["float", "float32"]
1063
+_ROCM_NOT_SUPPORTED_DTYPE: List[str] = [] #
1064
1065
1066
def _get_and_verify_dtype(
@@ -1092,14 +1092,6 @@ def _get_and_verify_dtype(
1092
else:
1093
raise ValueError(f"Unknown dtype: {dtype}")
1094
1095
- if is_hip() and torch_dtype == torch.float32:
1096
- rocm_supported_dtypes = [
1097
- k for k, v in _STR_DTYPE_TO_TORCH_DTYPE.items()
1098
- if (k not in _ROCM_NOT_SUPPORTED_DTYPE)
1099
- ]
1100
- raise ValueError(f"dtype '{dtype}' is not supported in ROCm. "
1101
- f"Supported dtypes are {rocm_supported_dtypes}")
1102
-
1095
# Verify the dtype.
1096
if torch_dtype != config_dtype:
1097
if torch_dtype == torch.float32: