A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[CI/Build] Update CPU tests to include all "standard" tests (#5481)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Nov 8, 2024 at 23:30 UTC
b489fc3c91778d8815243f89132d36b2c6eefd5a
14 files changed
+63
-48
.buildkite/run-cpu-test-ppc64le.sh
+13
-8
@@ -19,17 +19,22 @@ docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/hugg
19
20
# Run basic model test
21
docker exec cpu-test bash -c "
22
- pip install pytest matplotlib einops transformers_stream_generator
23
- pytest -v -s tests/models -m \"not vlm\" \
24
- --ignore=tests/models/test_embedding.py \
25
- --ignore=tests/models/test_oot_registration.py \
26
- --ignore=tests/models/test_registry.py \
27
- --ignore=tests/models/test_jamba.py \
28
- --ignore=tests/models/test_mamba.py \
29
- --ignore=tests/models/test_danube3_4b.py" # Mamba kernels and Danube3-4B on CPU is not supported
22
+ set -e
23
+ pip install pytest pytest-asyncio \
24
+ decord einops librosa peft Pillow sentence-transformers soundfile \
25
+ transformers_stream_generator matplotlib datamodel_code_generator
26
+ pip install torchvision --index-url https://download.pytorch.org/whl/cpu
27
+ # Embedding models are not supported for CPU yet
28
+ # pytest -v -s tests/models/embedding/language
29
+ pytest -v -s tests/models/encoder_decoder/language
30
+ pytest -v -s tests/models/decoder_only/language/test_models.py
31
+ # Chunked prefill not supported for CPU yet
32
+ # pytest -v -s tests/models/decoder_only/audio_language -m cpu_model
33
+ pytest -v -s tests/models/decoder_only/vision_language -m cpu_model"
34
35
# online inference
36
docker exec cpu-test bash -c "
37
+ set -e
38
python3 -m vllm.entrypoints.openai.api_server --model facebook/opt-125m &
39
timeout 600 bash -c 'until curl localhost:8000/v1/models; do sleep 1; done' || exit 1
40
python3 benchmarks/benchmark_serving.py \
.buildkite/run-cpu-test.sh
+17
-8
@@ -20,32 +20,41 @@ docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/hugg
20
--cpuset-mems=1 --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test-avx2 cpu-test-avx2
21
22
# offline inference
23
-docker exec cpu-test-avx2 bash -c "python3 examples/offline_inference.py"
23
+docker exec cpu-test-avx2 bash -c "
24
+ set -e
25
+ python3 examples/offline_inference.py"
26
27
# Run basic model test
28
docker exec cpu-test bash -c "
27
- pip install pytest matplotlib einops transformers_stream_generator datamodel_code_generator
29
+ set -e
30
+ pip install pytest pytest-asyncio \
31
+ decord einops librosa peft Pillow sentence-transformers soundfile \
32
+ transformers_stream_generator matplotlib datamodel_code_generator
33
+ pip install torchvision --index-url https://download.pytorch.org/whl/cpu
34
+ # Embedding models are not supported for CPU yet
35
+ # pytest -v -s tests/models/embedding/language
36
pytest -v -s tests/models/encoder_decoder/language
29
- pytest -v -s tests/models/decoder_only/language \
30
- --ignore=tests/models/test_fp8.py \
31
- --ignore=tests/models/decoder_only/language/test_jamba.py \
32
- --ignore=tests/models/decoder_only/language/test_mamba.py \
33
- --ignore=tests/models/decoder_only/language/test_granitemoe.py \
34
- --ignore=tests/models/decoder_only/language/test_danube3_4b.py" # Mamba and Danube3-4B on CPU is not supported
37
+ pytest -v -s tests/models/decoder_only/language/test_models.py
38
+ # Chunked prefill not supported for CPU yet
39
+ # pytest -v -s tests/models/decoder_only/audio_language -m cpu_model
40
+ pytest -v -s tests/models/decoder_only/vision_language -m cpu_model"
41
42
# Run compressed-tensor test
43
docker exec cpu-test bash -c "
44
+ set -e
45
pytest -s -v \
46
tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_static_setup \
47
tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_dynamic_per_token"
48
49
# Run AWQ test
50
docker exec cpu-test bash -c "
51
+ set -e
52
pytest -s -v \
53
tests/quantization/test_ipex_quant.py"
54
55
# online inference
56
docker exec cpu-test bash -c "
57
+ set -e
58
export VLLM_CPU_KVCACHE_SPACE=10
59
export VLLM_CPU_OMP_THREADS_BIND=48-92
60
python3 -m vllm.entrypoints.openai.api_server --model facebook/opt-125m --dtype half &
.buildkite/test-pipeline.yaml
+1
-2
@@ -269,7 +269,6 @@ steps:
269
source_file_dependencies:
270
- benchmarks/
271
commands:
272
- - pip install aiohttp
272
- bash run-benchmarks.sh
273
274
- label: Quantization Test # 33min
@@ -331,7 +330,7 @@ steps:
330
commands:
331
- pytest -v -s models/decoder_only/language --ignore=models/decoder_only/language/test_models.py
332
334
-- label: Decoder-only Multi-Modal Models Test (Standard)
333
+- label: Decoder-only Multi-Modal Models Test (Standard) # 26min
334
#mirror_hardwares: [amd]
335
source_file_dependencies:
336
- vllm/
pyproject.toml
+2
-1
@@ -93,7 +93,8 @@ skip_gitignore = true
93
[tool.pytest.ini_options]
94
markers = [
95
"skip_global_cleanup",
96
- "core_model: run this model test in each PR instead of just daily",
96
+ "core_model: enable this model test in each PR instead of only nightly",
97
+ "cpu_model: enable this model test in CPU tests",
98
"distributed_2_gpus: run this test only in distributed tests for 2 GPUs",
99
"skip_v1: do not run this test with v1",
100
]
requirements-test.in
-5
@@ -12,9 +12,7 @@ decord # required for video tests
12
einops # required for MPT, qwen-vl and Mamba
13
httpx
14
librosa # required for audio tests
15
-opencv-python # required for video tests
15
peft
17
-requests
16
ray[adag]==2.35
17
sentence-transformers # required for embedding tests
18
soundfile # required for audio tests
@@ -29,9 +27,6 @@ lm-eval[api]==0.4.4 # required for model evaluation test
27
# TODO: Add this after fully implementing llava(mantis)
28
# git+https://github.com/TIGER-AI-Lab/Mantis.git # required for llava(mantis) test
29
32
-# Benchmarking
33
-aiohttp
34
-
30
# quantization
31
bitsandbytes>=0.44.0
32
buildkite-test-collector==0.1.9
tests/models/decoder_only/audio_language/test_ultravox.py
+13
-4
@@ -5,11 +5,11 @@ import pytest
5
import pytest_asyncio
6
from transformers import AutoModel, AutoTokenizer, BatchEncoding
7
8
-from tests.utils import RemoteOpenAIServer
8
from vllm.sequence import SampleLogprobs
9
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
10
11
from ....conftest import HfRunner, VllmRunner
12
+from ....utils import RemoteOpenAIServer
13
from ...utils import check_logprobs_close
14
15
MODEL_NAME = "fixie-ai/ultravox-v0_3"
@@ -39,7 +39,10 @@ def audio(request):
39
return AudioAsset(request.param)
40
41
42
-@pytest.fixture(params=({}, CHUNKED_PREFILL_KWARGS))
42
+@pytest.fixture(params=[
43
+ pytest.param({}, marks=pytest.mark.cpu_model),
44
+ pytest.param(CHUNKED_PREFILL_KWARGS),
45
+])
46
def server(request, audio_assets):
47
args = [
48
"--dtype=bfloat16", "--max-model-len=4096", "--enforce-eager",
@@ -185,7 +188,10 @@ def run_multi_audio_test(
188
@pytest.mark.parametrize("dtype", ["half"])
189
@pytest.mark.parametrize("max_tokens", [128])
190
@pytest.mark.parametrize("num_logprobs", [5])
188
-@pytest.mark.parametrize("vllm_kwargs", [{}, CHUNKED_PREFILL_KWARGS])
191
+@pytest.mark.parametrize("vllm_kwargs", [
192
+ pytest.param({}, marks=pytest.mark.cpu_model),
193
+ pytest.param(CHUNKED_PREFILL_KWARGS),
194
+])
195
def test_models(hf_runner, vllm_runner, audio, dtype: str, max_tokens: int,
196
num_logprobs: int, vllm_kwargs: dict) -> None:
197
@@ -207,7 +213,10 @@ def test_models(hf_runner, vllm_runner, audio, dtype: str, max_tokens: int,
213
@pytest.mark.parametrize("dtype", ["half"])
214
@pytest.mark.parametrize("max_tokens", [128])
215
@pytest.mark.parametrize("num_logprobs", [5])
210
-@pytest.mark.parametrize("vllm_kwargs", [{}, CHUNKED_PREFILL_KWARGS])
216
+@pytest.mark.parametrize("vllm_kwargs", [
217
+ pytest.param({}, marks=pytest.mark.cpu_model),
218
+ pytest.param(CHUNKED_PREFILL_KWARGS),
219
+])
220
def test_models_with_multiple_audios(vllm_runner, audio_assets, dtype: str,
221
max_tokens: int, num_logprobs: int,
222
vllm_kwargs: dict) -> None:
tests/models/decoder_only/vision_language/test_h2ovl.py
-1
@@ -14,7 +14,6 @@ models = [
14
"h2oai/h2ovl-mississippi-800m", # Replace with your actual model names
15
"h2oai/h2ovl-mississippi-2b",
16
]
17
-target_dtype = "bfloat16"
17
18
19
def run_preprocessing_test(
tests/models/decoder_only/vision_language/test_models.py
+4
-7
@@ -94,7 +94,7 @@ VLM_TEST_SETTINGS = {
94
),
95
limit_mm_per_prompt={"image": 4},
96
)],
97
- marks=[pytest.mark.core_model],
97
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model],
98
),
99
"paligemma": VLMTestInfo(
100
models=["google/paligemma-3b-mix-224"],
@@ -111,7 +111,8 @@ VLM_TEST_SETTINGS = {
111
"pixel_values"
112
),
113
vllm_output_post_proc=model_utils.paligemma_vllm_to_hf_output,
114
- dtype="half" if current_platform.is_rocm() else ("half", "float"),
114
+ dtype=("half" if current_platform.is_cpu() or current_platform.is_rocm()
115
+ else ("half", "float")),
116
marks=[pytest.mark.core_model],
117
),
118
"qwen2_vl": VLMTestInfo(
@@ -128,7 +129,7 @@ VLM_TEST_SETTINGS = {
129
max_num_seqs=2,
130
auto_cls=AutoModelForVision2Seq,
131
vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
131
- marks=[pytest.mark.core_model],
132
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model],
133
image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
134
),
135
#### Extended model tests
@@ -172,7 +173,6 @@ VLM_TEST_SETTINGS = {
173
use_tokenizer_eos=True,
174
vllm_output_post_proc=model_utils.fuyu_vllm_to_hf_output,
175
num_logprobs=10,
175
- dtype="bfloat16" if current_platform.is_cpu() else "half",
176
image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
177
),
178
"glm4": VLMTestInfo(
@@ -245,7 +245,6 @@ VLM_TEST_SETTINGS = {
245
models=["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"],
246
test_type=VLMTestType.CUSTOM_INPUTS,
247
prompt_formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
248
- dtype="half",
248
num_video_frames=16,
249
max_model_len=16384,
250
postprocess_inputs=model_utils.get_key_type_post_processor(
@@ -404,7 +403,6 @@ VLM_TEST_SETTINGS = {
403
prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
404
test_type=VLMTestType.CUSTOM_INPUTS,
405
max_model_len=4096,
407
- dtype="bfloat16" if current_platform.is_cpu() else "half",
406
use_tokenizer_eos=True,
407
patch_hf_runner=model_utils.internvl_patch_hf_runner,
408
custom_test_opts=[
@@ -419,7 +417,6 @@ VLM_TEST_SETTINGS = {
417
test_type=VLMTestType.CUSTOM_INPUTS,
418
max_model_len=16384,
419
max_num_seqs=2,
422
- dtype="half",
420
postprocess_inputs=model_utils.get_key_type_post_processor(
421
"pixel_values"
422
),
tests/models/decoder_only/vision_language/test_phi3v.py
-2
@@ -44,8 +44,6 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
44
45
46
target_dtype = "half"
47
-if current_platform.is_cpu():
48
- target_dtype = "bfloat16"
47
48
# ROCm Triton FA can run into shared memory issues with these models,
49
# use other backends in the meantime
tests/models/utils.py
+1
-2
@@ -5,7 +5,6 @@ import torch
5
6
from vllm.config import ModelConfig, TaskOption
7
from vllm.inputs import InputContext
8
-from vllm.platforms import current_platform
8
from vllm.sequence import Logprob, PromptLogprobs, SampleLogprobs
9
10
TokensText = Tuple[List[int], str]
@@ -270,7 +269,7 @@ def build_model_context(model_name: str,
269
if tokenizer_name is None:
270
tokenizer_name = model_name
271
if dtype is None:
273
- dtype = "bfloat16" if current_platform.is_cpu() else "half"
272
+ dtype = "half"
273
274
model_config = ModelConfig(
275
model_name,
vllm/assets/image.py
+1
-1
@@ -27,4 +27,4 @@ class ImageAsset:
27
"""
28
image_path = get_vllm_public_assets(filename=f"{self.name}.pt",
29
s3_prefix=VLM_IMAGES_DIR)
30
- return torch.load(image_path)
30
+ return torch.load(image_path, map_location="cpu")
vllm/model_executor/models/ultravox.py
+2
-2
@@ -134,9 +134,9 @@ def input_mapper_for_ultravox(ctx: InputContext, data: object):
134
if sr != feature_extractor.sampling_rate:
135
try:
136
import librosa
137
- except ImportError:
137
+ except ImportError as exc:
138
raise ImportError(
139
- "Please install vllm[audio] for audio support.") from None
139
+ "Please install vllm[audio] for audio support.") from exc
140
audio = librosa.resample(audio,
141
orig_sr=sr,
142
target_sr=feature_extractor.sampling_rate)
vllm/multimodal/utils.py
+4
-4
@@ -206,9 +206,9 @@ def try_import_audio_packages() -> Tuple[Any, Any]:
206
try:
207
import librosa
208
import soundfile
209
- except ImportError:
209
+ except ImportError as exc:
210
raise ImportError(
211
- "Please install vllm[audio] for audio support.") from None
211
+ "Please install vllm[audio] for audio support.") from exc
212
return librosa, soundfile
213
214
@@ -344,9 +344,9 @@ def try_import_video_packages() -> Any:
344
try:
345
import cv2
346
import decord
347
- except ImportError:
347
+ except ImportError as exc:
348
raise ImportError(
349
- "Please install vllm[video] for video support.") from None
349
+ "Please install vllm[video] for video support.") from exc
350
return cv2, decord
351
352
vllm/worker/cpu_worker.py
+5
-1
@@ -151,7 +151,11 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
151
self.local_omp_cpuid = omp_cpuids.split("|")[rank]
152
153
ModelRunnerClass: Type[CPUModelRunner] = CPUModelRunner
154
- if self.model_config.is_encoder_decoder:
154
+ if self.model_config.task == "embedding":
155
+ raise NotImplementedError(
156
+ "Embedding models are not supported for CPU backend")
157
+ # ModelRunnerClass = CPUEmbeddingModelRunner
158
+ elif self.model_config.is_encoder_decoder:
159
ModelRunnerClass = CPUEncoderDecoderModelRunner
160
self.model_runner: CPUModelRunner = ModelRunnerClass(
161
vllm_config=vllm_config,