A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[CI/Build] Add text-only test for Qwen models (#7475)
Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com>
Alex Brooks committed
Aug 18, 2024 at 17:43 UTC
40e1360bb6e9b299d604863b3be5ccbe9a3bee48
3 files changed
+52
-2
.buildkite/run-cpu-test.sh
+1
-1
@@ -22,7 +22,7 @@ docker exec cpu-test-avx2 bash -c "python3 examples/offline_inference.py"
22
23
# Run basic model test
24
docker exec cpu-test bash -c "
25
- pip install pytest
25
+ pip install pytest matplotlib einops transformers_stream_generator
26
pytest -v -s tests/models -m \"not vlm\" --ignore=tests/models/test_embedding.py --ignore=tests/models/test_oot_registration.py --ignore=tests/models/test_registry.py --ignore=tests/models/test_jamba.py --ignore=tests/models/test_danube3_4b.py" # Mamba and Danube3-4B on CPU is not supported
27
28
# online inference
requirements-test.txt
+3
-1
@@ -11,7 +11,7 @@ pytest-shard
11
12
# testing utils
13
awscli
14
-einops # required for MPT
14
+einops # required for MPT and qwen-vl
15
httpx
16
peft
17
requests
@@ -19,6 +19,8 @@ ray
19
sentence-transformers # required for embedding
20
compressed-tensors==0.4.0 # required for compressed-tensors
21
timm # required for internvl test
22
+transformers_stream_generator # required for qwen-vl test
23
+matplotlib # required for qwen-vl test
24
25
# TODO: Add this after fully implementing llava(mantis)
26
# git+https://github.com/TIGER-AI-Lab/Mantis.git # required for llava(mantis) test
tests/models/test_qwen.py
new
+48
@@ -0,0 +1,48 @@
1
+from typing import Type
2
+
3
+import pytest
4
+
5
+from ..conftest import HfRunner, VllmRunner
6
+from .utils import check_logprobs_close
7
+
8
+models = ["qwen/qwen-vl"]
9
+
10
+
11
+@pytest.mark.parametrize("dtype", ["half"])
12
+@pytest.mark.parametrize("max_tokens", [32])
13
+@pytest.mark.parametrize("num_logprobs", [5])
14
+@pytest.mark.parametrize("model", models)
15
+def test_text_only_qwen_model(
16
+ hf_runner: Type[HfRunner],
17
+ vllm_runner: Type[VllmRunner],
18
+ example_prompts,
19
+ model: str,
20
+ *,
21
+ dtype: str,
22
+ max_tokens: int,
23
+ num_logprobs: int,
24
+):
25
+ # This test checks language inputs only, since the visual component
26
+ # for qwen-vl is still unsupported in VLLM. In the near-future, the
27
+ # implementation and this test will be extended to consider
28
+ # visual inputs as well.
29
+ with hf_runner(model, dtype=dtype, is_vision_model=False) as hf_model:
30
+ hf_outputs = hf_model.generate_greedy_logprobs_limit(
31
+ example_prompts,
32
+ max_tokens,
33
+ num_logprobs=num_logprobs,
34
+ )
35
+
36
+ with vllm_runner(model, dtype=dtype) as vllm_model:
37
+ vllm_outputs = vllm_model.generate_greedy_logprobs(
38
+ example_prompts,
39
+ max_tokens,
40
+ num_logprobs=num_logprobs,
41
+ )
42
+
43
+ check_logprobs_close(
44
+ outputs_0_lst=hf_outputs,
45
+ outputs_1_lst=vllm_outputs,
46
+ name_0="hf",
47
+ name_1="vllm",
48
+ )