A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[misc][ci] fix cpu test with plugins (#7489)
youkaichao committed
Aug 13, 2024 at 19:27 UTC
ea49e6a3c82bdd48f377efc79bfe44f37f13b3ad
4 files changed
+15
-8
.buildkite/run-cpu-test.sh
+1
-1
@@ -23,7 +23,7 @@ docker exec cpu-test-avx2 bash -c "python3 examples/offline_inference.py"
23
# Run basic model test
24
docker exec cpu-test bash -c "
25
pip install pytest Pillow protobuf
26
- pytest -v -s tests/models -m \"not vlm\" --ignore=tests/models/test_embedding.py --ignore=tests/models/test_registry.py --ignore=tests/models/test_jamba.py --ignore=tests/models/test_danube3_4b.py" # Mamba and Danube3-4B on CPU is not supported
26
+ pytest -v -s tests/models -m \"not vlm\" --ignore=tests/models/test_embedding.py --ignore=tests/models/test_oot_registration.py --ignore=tests/models/test_registry.py --ignore=tests/models/test_jamba.py --ignore=tests/models/test_danube3_4b.py" # Mamba and Danube3-4B on CPU is not supported
27
28
# online inference
29
docker exec cpu-test bash -c "
.buildkite/test-pipeline.yaml
+1
@@ -292,6 +292,7 @@ steps:
292
- pytest -v -s distributed/test_chunked_prefill_distributed.py
293
- pytest -v -s distributed/test_multimodal_broadcast.py
294
- pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
295
+ - pip install -e ./plugins/vllm_add_dummy_model
296
- pytest -v -s distributed/test_distributed_oot.py
297
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s test_sharded_state_loader.py
298
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s distributed/test_utils.py
tests/models/test_oot_registration.py
+3
-4
@@ -4,12 +4,10 @@ import pytest
4
5
from vllm import LLM, SamplingParams
6
7
-# NOTE: the order of the tests is important
8
-# the first test does not load any plugins
9
-# the second test loads the plugin
10
-# they share the same process, so the plugin is loaded for the second test
7
+from ..utils import fork_new_process_for_each_test
8
9
10
+@fork_new_process_for_each_test
11
def test_plugin(dummy_opt_path):
12
os.environ["VLLM_PLUGINS"] = ""
13
with pytest.raises(Exception) as excinfo:
@@ -17,6 +15,7 @@ def test_plugin(dummy_opt_path):
15
assert "are not supported for now" in str(excinfo.value)
16
17
18
+@fork_new_process_for_each_test
19
def test_oot_registration(dummy_opt_path):
20
os.environ["VLLM_PLUGINS"] = "register_dummy_model"
21
prompts = ["Hello, my name is", "The text does not matter"]
tests/utils.py
+10
-3
@@ -10,7 +10,6 @@ from pathlib import Path
10
from typing import Any, Callable, Dict, List, Optional
11
12
import openai
13
-import ray
13
import requests
14
from transformers import AutoTokenizer
15
from typing_extensions import ParamSpec
@@ -18,9 +17,10 @@ from typing_extensions import ParamSpec
17
from vllm.distributed import (ensure_model_parallel_initialized,
18
init_distributed_environment)
19
from vllm.entrypoints.openai.cli_args import make_arg_parser
20
+from vllm.platforms import current_platform
21
from vllm.utils import FlexibleArgumentParser, get_open_port, is_hip
22
23
-if is_hip():
23
+if current_platform.is_rocm():
24
from amdsmi import (amdsmi_get_gpu_vram_usage,
25
amdsmi_get_processor_handles, amdsmi_init,
26
amdsmi_shut_down)
@@ -32,7 +32,7 @@ if is_hip():
32
yield
33
finally:
34
amdsmi_shut_down()
35
-else:
35
+elif current_platform.is_cuda():
36
from pynvml import (nvmlDeviceGetHandleByIndex, nvmlDeviceGetMemoryInfo,
37
nvmlInit, nvmlShutdown)
38
@@ -43,6 +43,11 @@ else:
43
yield
44
finally:
45
nvmlShutdown()
46
+else:
47
+
48
+ @contextmanager
49
+ def _nvml():
50
+ yield
51
52
53
VLLM_PATH = Path(__file__).parent.parent
@@ -293,6 +298,8 @@ def multi_process_parallel(
298
pp_size: int,
299
test_target: Any,
300
) -> None:
301
+ import ray
302
+
303
# Using ray helps debugging the error when it failed
304
# as compared to multiprocessing.
305
# NOTE: We need to set working_dir for distributed tests,