@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b489fc3c91
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[CI/Build] Update CPU tests to include all "standard" tests (#5481)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Nov 8, 2024 at 23:30 UTC b489fc3c91778d8815243f89132d36b2c6eefd5a
14 files changed +63 -48
.buildkite/run-cpu-test-ppc64le.sh
+13 -8
@@ -19,17 +19,22 @@ docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/hugg
19
20 # Run basic model test
21 docker exec cpu-test bash -c "
22 - pip install pytest matplotlib einops transformers_stream_generator
23 - pytest -v -s tests/models -m \"not vlm\" \
24 - --ignore=tests/models/test_embedding.py \
25 - --ignore=tests/models/test_oot_registration.py \
26 - --ignore=tests/models/test_registry.py \
27 - --ignore=tests/models/test_jamba.py \
28 - --ignore=tests/models/test_mamba.py \
29 - --ignore=tests/models/test_danube3_4b.py" # Mamba kernels and Danube3-4B on CPU is not supported
22 + set -e
23 + pip install pytest pytest-asyncio \
24 + decord einops librosa peft Pillow sentence-transformers soundfile \
25 + transformers_stream_generator matplotlib datamodel_code_generator
26 + pip install torchvision --index-url https://download.pytorch.org/whl/cpu
27 + # Embedding models are not supported for CPU yet
28 + # pytest -v -s tests/models/embedding/language
29 + pytest -v -s tests/models/encoder_decoder/language
30 + pytest -v -s tests/models/decoder_only/language/test_models.py
31 + # Chunked prefill not supported for CPU yet
32 + # pytest -v -s tests/models/decoder_only/audio_language -m cpu_model
33 + pytest -v -s tests/models/decoder_only/vision_language -m cpu_model"
34
35 # online inference
36 docker exec cpu-test bash -c "
37 + set -e
38 python3 -m vllm.entrypoints.openai.api_server --model facebook/opt-125m &
39 timeout 600 bash -c 'until curl localhost:8000/v1/models; do sleep 1; done' || exit 1
40 python3 benchmarks/benchmark_serving.py \
.buildkite/run-cpu-test.sh
+17 -8
@@ -20,32 +20,41 @@ docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/hugg
20 --cpuset-mems=1 --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test-avx2 cpu-test-avx2
21
22 # offline inference
23 -docker exec cpu-test-avx2 bash -c "python3 examples/offline_inference.py"
23 +docker exec cpu-test-avx2 bash -c "
24 + set -e
25 + python3 examples/offline_inference.py"
26
27 # Run basic model test
28 docker exec cpu-test bash -c "
27 - pip install pytest matplotlib einops transformers_stream_generator datamodel_code_generator
29 + set -e
30 + pip install pytest pytest-asyncio \
31 + decord einops librosa peft Pillow sentence-transformers soundfile \
32 + transformers_stream_generator matplotlib datamodel_code_generator
33 + pip install torchvision --index-url https://download.pytorch.org/whl/cpu
34 + # Embedding models are not supported for CPU yet
35 + # pytest -v -s tests/models/embedding/language
36 pytest -v -s tests/models/encoder_decoder/language
29 - pytest -v -s tests/models/decoder_only/language \
30 - --ignore=tests/models/test_fp8.py \
31 - --ignore=tests/models/decoder_only/language/test_jamba.py \
32 - --ignore=tests/models/decoder_only/language/test_mamba.py \
33 - --ignore=tests/models/decoder_only/language/test_granitemoe.py \
34 - --ignore=tests/models/decoder_only/language/test_danube3_4b.py" # Mamba and Danube3-4B on CPU is not supported
37 + pytest -v -s tests/models/decoder_only/language/test_models.py
38 + # Chunked prefill not supported for CPU yet
39 + # pytest -v -s tests/models/decoder_only/audio_language -m cpu_model
40 + pytest -v -s tests/models/decoder_only/vision_language -m cpu_model"
41
42 # Run compressed-tensor test
43 docker exec cpu-test bash -c "
44 + set -e
45 pytest -s -v \
46 tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_static_setup \
47 tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_dynamic_per_token"
48
49 # Run AWQ test
50 docker exec cpu-test bash -c "
51 + set -e
52 pytest -s -v \
53 tests/quantization/test_ipex_quant.py"
54
55 # online inference
56 docker exec cpu-test bash -c "
57 + set -e
58 export VLLM_CPU_KVCACHE_SPACE=10
59 export VLLM_CPU_OMP_THREADS_BIND=48-92
60 python3 -m vllm.entrypoints.openai.api_server --model facebook/opt-125m --dtype half &
.buildkite/test-pipeline.yaml
+1 -2
@@ -269,7 +269,6 @@ steps:
269 source_file_dependencies:
270 - benchmarks/
271 commands:
272 - - pip install aiohttp
272 - bash run-benchmarks.sh
273
274 - label: Quantization Test # 33min
@@ -331,7 +330,7 @@ steps:
330 commands:
331 - pytest -v -s models/decoder_only/language --ignore=models/decoder_only/language/test_models.py
332
334 -- label: Decoder-only Multi-Modal Models Test (Standard)
333 +- label: Decoder-only Multi-Modal Models Test (Standard) # 26min
334 #mirror_hardwares: [amd]
335 source_file_dependencies:
336 - vllm/
pyproject.toml
+2 -1
@@ -93,7 +93,8 @@ skip_gitignore = true
93 [tool.pytest.ini_options]
94 markers = [
95 "skip_global_cleanup",
96 - "core_model: run this model test in each PR instead of just daily",
96 + "core_model: enable this model test in each PR instead of only nightly",
97 + "cpu_model: enable this model test in CPU tests",
98 "distributed_2_gpus: run this test only in distributed tests for 2 GPUs",
99 "skip_v1: do not run this test with v1",
100 ]
requirements-test.in
-5
@@ -12,9 +12,7 @@ decord # required for video tests
12 einops # required for MPT, qwen-vl and Mamba
13 httpx
14 librosa # required for audio tests
15 -opencv-python # required for video tests
15 peft
17 -requests
16 ray[adag]==2.35
17 sentence-transformers # required for embedding tests
18 soundfile # required for audio tests
@@ -29,9 +27,6 @@ lm-eval[api]==0.4.4 # required for model evaluation test
27 # TODO: Add this after fully implementing llava(mantis)
28 # git+https://github.com/TIGER-AI-Lab/Mantis.git # required for llava(mantis) test
29
32 -# Benchmarking
33 -aiohttp
34 -
30 # quantization
31 bitsandbytes>=0.44.0
32 buildkite-test-collector==0.1.9
tests/models/decoder_only/audio_language/test_ultravox.py
+13 -4
@@ -5,11 +5,11 @@ import pytest
5 import pytest_asyncio
6 from transformers import AutoModel, AutoTokenizer, BatchEncoding
7
8 -from tests.utils import RemoteOpenAIServer
8 from vllm.sequence import SampleLogprobs
9 from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
10
11 from ....conftest import HfRunner, VllmRunner
12 +from ....utils import RemoteOpenAIServer
13 from ...utils import check_logprobs_close
14
15 MODEL_NAME = "fixie-ai/ultravox-v0_3"
@@ -39,7 +39,10 @@ def audio(request):
39 return AudioAsset(request.param)
40
41
42 -@pytest.fixture(params=({}, CHUNKED_PREFILL_KWARGS))
42 +@pytest.fixture(params=[
43 + pytest.param({}, marks=pytest.mark.cpu_model),
44 + pytest.param(CHUNKED_PREFILL_KWARGS),
45 +])
46 def server(request, audio_assets):
47 args = [
48 "--dtype=bfloat16", "--max-model-len=4096", "--enforce-eager",
@@ -185,7 +188,10 @@ def run_multi_audio_test(
188 @pytest.mark.parametrize("dtype", ["half"])
189 @pytest.mark.parametrize("max_tokens", [128])
190 @pytest.mark.parametrize("num_logprobs", [5])
188 -@pytest.mark.parametrize("vllm_kwargs", [{}, CHUNKED_PREFILL_KWARGS])
191 +@pytest.mark.parametrize("vllm_kwargs", [
192 + pytest.param({}, marks=pytest.mark.cpu_model),
193 + pytest.param(CHUNKED_PREFILL_KWARGS),
194 +])
195 def test_models(hf_runner, vllm_runner, audio, dtype: str, max_tokens: int,
196 num_logprobs: int, vllm_kwargs: dict) -> None:
197
@@ -207,7 +213,10 @@ def test_models(hf_runner, vllm_runner, audio, dtype: str, max_tokens: int,
213 @pytest.mark.parametrize("dtype", ["half"])
214 @pytest.mark.parametrize("max_tokens", [128])
215 @pytest.mark.parametrize("num_logprobs", [5])
210 -@pytest.mark.parametrize("vllm_kwargs", [{}, CHUNKED_PREFILL_KWARGS])
216 +@pytest.mark.parametrize("vllm_kwargs", [
217 + pytest.param({}, marks=pytest.mark.cpu_model),
218 + pytest.param(CHUNKED_PREFILL_KWARGS),
219 +])
220 def test_models_with_multiple_audios(vllm_runner, audio_assets, dtype: str,
221 max_tokens: int, num_logprobs: int,
222 vllm_kwargs: dict) -> None:
tests/models/decoder_only/vision_language/test_h2ovl.py
-1
@@ -14,7 +14,6 @@ models = [
14 "h2oai/h2ovl-mississippi-800m", # Replace with your actual model names
15 "h2oai/h2ovl-mississippi-2b",
16 ]
17 -target_dtype = "bfloat16"
17
18
19 def run_preprocessing_test(
tests/models/decoder_only/vision_language/test_models.py
+4 -7
@@ -94,7 +94,7 @@ VLM_TEST_SETTINGS = {
94 ),
95 limit_mm_per_prompt={"image": 4},
96 )],
97 - marks=[pytest.mark.core_model],
97 + marks=[pytest.mark.core_model, pytest.mark.cpu_model],
98 ),
99 "paligemma": VLMTestInfo(
100 models=["google/paligemma-3b-mix-224"],
@@ -111,7 +111,8 @@ VLM_TEST_SETTINGS = {
111 "pixel_values"
112 ),
113 vllm_output_post_proc=model_utils.paligemma_vllm_to_hf_output,
114 - dtype="half" if current_platform.is_rocm() else ("half", "float"),
114 + dtype=("half" if current_platform.is_cpu() or current_platform.is_rocm()
115 + else ("half", "float")),
116 marks=[pytest.mark.core_model],
117 ),
118 "qwen2_vl": VLMTestInfo(
@@ -128,7 +129,7 @@ VLM_TEST_SETTINGS = {
129 max_num_seqs=2,
130 auto_cls=AutoModelForVision2Seq,
131 vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
131 - marks=[pytest.mark.core_model],
132 + marks=[pytest.mark.core_model, pytest.mark.cpu_model],
133 image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
134 ),
135 #### Extended model tests
@@ -172,7 +173,6 @@ VLM_TEST_SETTINGS = {
173 use_tokenizer_eos=True,
174 vllm_output_post_proc=model_utils.fuyu_vllm_to_hf_output,
175 num_logprobs=10,
175 - dtype="bfloat16" if current_platform.is_cpu() else "half",
176 image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
177 ),
178 "glm4": VLMTestInfo(
@@ -245,7 +245,6 @@ VLM_TEST_SETTINGS = {
245 models=["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"],
246 test_type=VLMTestType.CUSTOM_INPUTS,
247 prompt_formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
248 - dtype="half",
248 num_video_frames=16,
249 max_model_len=16384,
250 postprocess_inputs=model_utils.get_key_type_post_processor(
@@ -404,7 +403,6 @@ VLM_TEST_SETTINGS = {
403 prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
404 test_type=VLMTestType.CUSTOM_INPUTS,
405 max_model_len=4096,
407 - dtype="bfloat16" if current_platform.is_cpu() else "half",
406 use_tokenizer_eos=True,
407 patch_hf_runner=model_utils.internvl_patch_hf_runner,
408 custom_test_opts=[
@@ -419,7 +417,6 @@ VLM_TEST_SETTINGS = {
417 test_type=VLMTestType.CUSTOM_INPUTS,
418 max_model_len=16384,
419 max_num_seqs=2,
422 - dtype="half",
420 postprocess_inputs=model_utils.get_key_type_post_processor(
421 "pixel_values"
422 ),
tests/models/decoder_only/vision_language/test_phi3v.py
-2
@@ -44,8 +44,6 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
44
45
46 target_dtype = "half"
47 -if current_platform.is_cpu():
48 - target_dtype = "bfloat16"
47
48 # ROCm Triton FA can run into shared memory issues with these models,
49 # use other backends in the meantime
tests/models/utils.py
+1 -2
@@ -5,7 +5,6 @@ import torch
5
6 from vllm.config import ModelConfig, TaskOption
7 from vllm.inputs import InputContext
8 -from vllm.platforms import current_platform
8 from vllm.sequence import Logprob, PromptLogprobs, SampleLogprobs
9
10 TokensText = Tuple[List[int], str]
@@ -270,7 +269,7 @@ def build_model_context(model_name: str,
269 if tokenizer_name is None:
270 tokenizer_name = model_name
271 if dtype is None:
273 - dtype = "bfloat16" if current_platform.is_cpu() else "half"
272 + dtype = "half"
273
274 model_config = ModelConfig(
275 model_name,
vllm/assets/image.py
+1 -1
@@ -27,4 +27,4 @@ class ImageAsset:
27 """
28 image_path = get_vllm_public_assets(filename=f"{self.name}.pt",
29 s3_prefix=VLM_IMAGES_DIR)
30 - return torch.load(image_path)
30 + return torch.load(image_path, map_location="cpu")
vllm/model_executor/models/ultravox.py
+2 -2
@@ -134,9 +134,9 @@ def input_mapper_for_ultravox(ctx: InputContext, data: object):
134 if sr != feature_extractor.sampling_rate:
135 try:
136 import librosa
137 - except ImportError:
137 + except ImportError as exc:
138 raise ImportError(
139 - "Please install vllm[audio] for audio support.") from None
139 + "Please install vllm[audio] for audio support.") from exc
140 audio = librosa.resample(audio,
141 orig_sr=sr,
142 target_sr=feature_extractor.sampling_rate)
vllm/multimodal/utils.py
+4 -4
@@ -206,9 +206,9 @@ def try_import_audio_packages() -> Tuple[Any, Any]:
206 try:
207 import librosa
208 import soundfile
209 - except ImportError:
209 + except ImportError as exc:
210 raise ImportError(
211 - "Please install vllm[audio] for audio support.") from None
211 + "Please install vllm[audio] for audio support.") from exc
212 return librosa, soundfile
213
214
@@ -344,9 +344,9 @@ def try_import_video_packages() -> Any:
344 try:
345 import cv2
346 import decord
347 - except ImportError:
347 + except ImportError as exc:
348 raise ImportError(
349 - "Please install vllm[video] for video support.") from None
349 + "Please install vllm[video] for video support.") from exc
350 return cv2, decord
351
352
vllm/worker/cpu_worker.py
+5 -1
@@ -151,7 +151,11 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
151 self.local_omp_cpuid = omp_cpuids.split("|")[rank]
152
153 ModelRunnerClass: Type[CPUModelRunner] = CPUModelRunner
154 - if self.model_config.is_encoder_decoder:
154 + if self.model_config.task == "embedding":
155 + raise NotImplementedError(
156 + "Embedding models are not supported for CPU backend")
157 + # ModelRunnerClass = CPUEmbeddingModelRunner
158 + elif self.model_config.is_encoder_decoder:
159 ModelRunnerClass = CPUEncoderDecoderModelRunner
160 self.model_runner: CPUModelRunner = ModelRunnerClass(
161 vllm_config=vllm_config,