@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 26148120b3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Build/CI] Extending the set of AMD tests with Regression, Basic Correctness, Distributed, Engine, Llava Tests (#4797)

Alexei-V-Ivanov-AMD committed May 16, 2024 at 22:58 UTC 26148120b3c05704409a425d017f0a51fca3b7cc
5 files changed +28 -20
.buildkite/run-amd-test.sh
+6 -5
@@ -1,4 +1,4 @@
1 -# This script build the ROCm docker image and runs test inside it.
1 +# This script runs test inside the corresponding ROCm docker container.
2 set -ex
3
4 # Print ROCm version
@@ -19,15 +19,16 @@ done
19
20 echo "--- Building container"
21 sha=$(git rev-parse --short HEAD)
22 -container_name=rocm_${sha}
22 +image_name=rocm_${sha}
23 +container_name=rocm_${sha}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)
24 docker build \
24 - -t ${container_name} \
25 + -t ${image_name} \
26 -f Dockerfile.rocm \
27 --progress plain \
28 .
29
30 remove_docker_container() {
30 - docker rm -f ${container_name} || docker image rm -f ${container_name} || true
31 + docker rm -f ${container_name} || docker image rm -f ${image_name} || true
32 }
33 trap remove_docker_container EXIT
34
@@ -39,6 +40,6 @@ docker run \
40 --rm \
41 -e HF_TOKEN \
42 --name ${container_name} \
42 - ${container_name} \
43 + ${image_name} \
44 /bin/bash -c "${@}"
45
.buildkite/test-pipeline.yaml
+15 -3
@@ -5,13 +5,16 @@
5
6 steps:
7 - label: Regression Test
8 + mirror_hardwares: [amd]
9 command: pytest -v -s test_regression.py
10 working_dir: "/vllm-workspace/tests" # optional
11
12 - label: AsyncEngine Test
13 + #mirror_hardwares: [amd]
14 command: pytest -v -s async_engine
15
16 - label: Basic Correctness Test
17 + mirror_hardwares: [amd]
18 commands:
19 - VLLM_ATTENTION_BACKEND=XFORMERS pytest -v -s basic_correctness/test_basic_correctness.py
20 - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s basic_correctness/test_basic_correctness.py
@@ -24,14 +27,15 @@ steps:
27 command: pytest -v -s core
28
29 - label: Distributed Comm Ops Test
30 + #mirror_hardwares: [amd]
31 command: pytest -v -s distributed/test_comm_ops.py
32 working_dir: "/vllm-workspace/tests"
33 num_gpus: 2
34
35 - label: Distributed Tests
36 + mirror_hardwares: [amd]
37 working_dir: "/vllm-workspace/tests"
38 num_gpus: 2
34 - mirror_hardwares: [amd]
39 commands:
40 - pytest -v -s distributed/test_pynccl_library.py
41 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
@@ -45,16 +49,18 @@ steps:
49 - pytest -v -s spec_decode/e2e/test_integration_dist.py
50
51 - label: Distributed Tests (Multiple Groups)
52 + #mirror_hardwares: [amd]
53 working_dir: "/vllm-workspace/tests"
54 num_gpus: 4
55 commands:
56 - pytest -v -s distributed/test_pynccl.py
57
58 - label: Engine Test
54 - #mirror_hardwares: [amd]
59 + mirror_hardwares: [amd]
60 command: pytest -v -s engine tokenization test_sequence.py test_config.py test_logger.py
61
62 - label: Entrypoints Test
63 + #mirror_hardwares: [amd]
64 commands:
65 # these tests have to be separated, because each one will allocate all posible GPU memory
66 - pytest -v -s entrypoints --ignore=entrypoints/test_server_oot_registration.py
@@ -74,6 +80,7 @@ steps:
80 - python3 tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
81
82 - label: Kernels Test %N
83 + #mirror_hardwares: [amd]
84 command: pytest -v -s kernels --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
85 parallelism: 4
86
@@ -84,7 +91,7 @@ steps:
91 - pytest -v -s models --ignore=models/test_llava.py
92
93 - label: Llava Test
87 - #mirror_hardwares: [amd]
94 + mirror_hardwares: [amd]
95 commands:
96 - bash ../.buildkite/download-images.sh
97 - pytest -v -s models/test_llava.py
@@ -95,6 +102,7 @@ steps:
102 - pytest -v -s prefix_caching
103
104 - label: Samplers Test
105 + #mirror_hardwares: [amd]
106 command: pytest -v -s samplers
107
108 - label: LogitsProcessor Test
@@ -110,16 +118,20 @@ steps:
118 command: pytest -v -s spec_decode
119
120 - label: LoRA Test %N
121 + #mirror_hardwares: [amd]
122 command: pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
123 parallelism: 4
124
125 - label: Tensorizer Test
126 + #mirror_hardwares: [amd]
127 command: apt-get install curl libsodium23 && pytest -v -s tensorizer_loader
128
129 - label: Metrics Test
130 + mirror_hardwares: [amd]
131 command: pytest -v -s metrics
132
133 - label: Quantization Test
134 + #mirror_hardwares: [amd]
135 command: pytest -v -s quantization
136
137 - label: Benchmarks
.buildkite/test-template.j2
+1 -2
@@ -3,9 +3,8 @@
3 {% set default_working_dir = "/vllm-workspace/tests" %}
4
5 steps:
6 -
6 - label: ":docker: build image"
8 - commands:
7 + commands:
8 - "docker build --build-arg max_jobs=16 --tag {{ docker_image }} --target test --progress plain ."
9 - "docker push {{ docker_image }}"
10 env:
tests/engine/test_stop_reason.py
+5 -1
@@ -32,6 +32,7 @@ def test_stop_reason(vllm_model, example_prompts):
32 # test stop token
33 outputs = llm.generate(example_prompts,
34 sampling_params=SamplingParams(
35 + ignore_eos=True,
36 seed=SEED,
37 max_tokens=MAX_TOKENS,
38 stop_token_ids=[stop_token_id]))
@@ -43,7 +44,10 @@ def test_stop_reason(vllm_model, example_prompts):
44 # test stop string
45 outputs = llm.generate(example_prompts,
46 sampling_params=SamplingParams(
46 - seed=SEED, max_tokens=MAX_TOKENS, stop="."))
47 + ignore_eos=True,
48 + seed=SEED,
49 + max_tokens=MAX_TOKENS,
50 + stop="."))
51 for output in outputs:
52 output = output.outputs[0]
53 assert output.finish_reason == "stop"
vllm/config.py
+1 -9
@@ -1060,7 +1060,7 @@ _STR_DTYPE_TO_TORCH_DTYPE = {
1060 "bfloat16": torch.bfloat16,
1061 }
1062
1063 -_ROCM_NOT_SUPPORTED_DTYPE = ["float", "float32"]
1063 +_ROCM_NOT_SUPPORTED_DTYPE: List[str] = [] #
1064
1065
1066 def _get_and_verify_dtype(
@@ -1092,14 +1092,6 @@ def _get_and_verify_dtype(
1092 else:
1093 raise ValueError(f"Unknown dtype: {dtype}")
1094
1095 - if is_hip() and torch_dtype == torch.float32:
1096 - rocm_supported_dtypes = [
1097 - k for k, v in _STR_DTYPE_TO_TORCH_DTYPE.items()
1098 - if (k not in _ROCM_NOT_SUPPORTED_DTYPE)
1099 - ]
1100 - raise ValueError(f"dtype '{dtype}' is not supported in ROCm. "
1101 - f"Supported dtypes are {rocm_supported_dtypes}")
1102 -
1095 # Verify the dtype.
1096 if torch_dtype != config_dtype:
1097 if torch_dtype == torch.float32: