@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ea49e6a3c8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[misc][ci] fix cpu test with plugins (#7489)

youkaichao committed Aug 13, 2024 at 19:27 UTC ea49e6a3c82bdd48f377efc79bfe44f37f13b3ad
4 files changed +15 -8
.buildkite/run-cpu-test.sh
+1 -1
@@ -23,7 +23,7 @@ docker exec cpu-test-avx2 bash -c "python3 examples/offline_inference.py"
23 # Run basic model test
24 docker exec cpu-test bash -c "
25 pip install pytest Pillow protobuf
26 - pytest -v -s tests/models -m \"not vlm\" --ignore=tests/models/test_embedding.py --ignore=tests/models/test_registry.py --ignore=tests/models/test_jamba.py --ignore=tests/models/test_danube3_4b.py" # Mamba and Danube3-4B on CPU is not supported
26 + pytest -v -s tests/models -m \"not vlm\" --ignore=tests/models/test_embedding.py --ignore=tests/models/test_oot_registration.py --ignore=tests/models/test_registry.py --ignore=tests/models/test_jamba.py --ignore=tests/models/test_danube3_4b.py" # Mamba and Danube3-4B on CPU is not supported
27
28 # online inference
29 docker exec cpu-test bash -c "
.buildkite/test-pipeline.yaml
+1
@@ -292,6 +292,7 @@ steps:
292 - pytest -v -s distributed/test_chunked_prefill_distributed.py
293 - pytest -v -s distributed/test_multimodal_broadcast.py
294 - pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
295 + - pip install -e ./plugins/vllm_add_dummy_model
296 - pytest -v -s distributed/test_distributed_oot.py
297 - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
298 - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s distributed/test_utils.py
tests/models/test_oot_registration.py
+3 -4
@@ -4,12 +4,10 @@ import pytest
4
5 from vllm import LLM, SamplingParams
6
7 -# NOTE: the order of the tests is important
8 -# the first test does not load any plugins
9 -# the second test loads the plugin
10 -# they share the same process, so the plugin is loaded for the second test
7 +from ..utils import fork_new_process_for_each_test
8
9
10 +@fork_new_process_for_each_test
11 def test_plugin(dummy_opt_path):
12 os.environ["VLLM_PLUGINS"] = ""
13 with pytest.raises(Exception) as excinfo:
@@ -17,6 +15,7 @@ def test_plugin(dummy_opt_path):
15 assert "are not supported for now" in str(excinfo.value)
16
17
18 +@fork_new_process_for_each_test
19 def test_oot_registration(dummy_opt_path):
20 os.environ["VLLM_PLUGINS"] = "register_dummy_model"
21 prompts = ["Hello, my name is", "The text does not matter"]
tests/utils.py
+10 -3
@@ -10,7 +10,6 @@ from pathlib import Path
10 from typing import Any, Callable, Dict, List, Optional
11
12 import openai
13 -import ray
13 import requests
14 from transformers import AutoTokenizer
15 from typing_extensions import ParamSpec
@@ -18,9 +17,10 @@ from typing_extensions import ParamSpec
17 from vllm.distributed import (ensure_model_parallel_initialized,
18 init_distributed_environment)
19 from vllm.entrypoints.openai.cli_args import make_arg_parser
20 +from vllm.platforms import current_platform
21 from vllm.utils import FlexibleArgumentParser, get_open_port, is_hip
22
23 -if is_hip():
23 +if current_platform.is_rocm():
24 from amdsmi import (amdsmi_get_gpu_vram_usage,
25 amdsmi_get_processor_handles, amdsmi_init,
26 amdsmi_shut_down)
@@ -32,7 +32,7 @@ if is_hip():
32 yield
33 finally:
34 amdsmi_shut_down()
35 -else:
35 +elif current_platform.is_cuda():
36 from pynvml import (nvmlDeviceGetHandleByIndex, nvmlDeviceGetMemoryInfo,
37 nvmlInit, nvmlShutdown)
38
@@ -43,6 +43,11 @@ else:
43 yield
44 finally:
45 nvmlShutdown()
46 +else:
47 +
48 + @contextmanager
49 + def _nvml():
50 + yield
51
52
53 VLLM_PATH = Path(__file__).parent.parent
@@ -293,6 +298,8 @@ def multi_process_parallel(
298 pp_size: int,
299 test_target: Any,
300 ) -> None:
301 + import ray
302 +
303 # Using ray helps debugging the error when it failed
304 # as compared to multiprocessing.
305 # NOTE: We need to set working_dir for distributed tests,