@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 40e1360bb6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[CI/Build] Add text-only test for Qwen models (#7475)

Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com>

Alex Brooks committed Aug 18, 2024 at 17:43 UTC 40e1360bb6e9b299d604863b3be5ccbe9a3bee48
3 files changed +52 -2
.buildkite/run-cpu-test.sh
+1 -1
@@ -22,7 +22,7 @@ docker exec cpu-test-avx2 bash -c "python3 examples/offline_inference.py"
22
23 # Run basic model test
24 docker exec cpu-test bash -c "
25 - pip install pytest
25 + pip install pytest matplotlib einops transformers_stream_generator
26 pytest -v -s tests/models -m \"not vlm\" --ignore=tests/models/test_embedding.py --ignore=tests/models/test_oot_registration.py --ignore=tests/models/test_registry.py --ignore=tests/models/test_jamba.py --ignore=tests/models/test_danube3_4b.py" # Mamba and Danube3-4B on CPU is not supported
27
28 # online inference
requirements-test.txt
+3 -1
@@ -11,7 +11,7 @@ pytest-shard
11
12 # testing utils
13 awscli
14 -einops # required for MPT
14 +einops # required for MPT and qwen-vl
15 httpx
16 peft
17 requests
@@ -19,6 +19,8 @@ ray
19 sentence-transformers # required for embedding
20 compressed-tensors==0.4.0 # required for compressed-tensors
21 timm # required for internvl test
22 +transformers_stream_generator # required for qwen-vl test
23 +matplotlib # required for qwen-vl test
24
25 # TODO: Add this after fully implementing llava(mantis)
26 # git+https://github.com/TIGER-AI-Lab/Mantis.git # required for llava(mantis) test
tests/models/test_qwen.py new
+48
@@ -0,0 +1,48 @@
1 +from typing import Type
2 +
3 +import pytest
4 +
5 +from ..conftest import HfRunner, VllmRunner
6 +from .utils import check_logprobs_close
7 +
8 +models = ["qwen/qwen-vl"]
9 +
10 +
11 +@pytest.mark.parametrize("dtype", ["half"])
12 +@pytest.mark.parametrize("max_tokens", [32])
13 +@pytest.mark.parametrize("num_logprobs", [5])
14 +@pytest.mark.parametrize("model", models)
15 +def test_text_only_qwen_model(
16 + hf_runner: Type[HfRunner],
17 + vllm_runner: Type[VllmRunner],
18 + example_prompts,
19 + model: str,
20 + *,
21 + dtype: str,
22 + max_tokens: int,
23 + num_logprobs: int,
24 +):
25 + # This test checks language inputs only, since the visual component
26 + # for qwen-vl is still unsupported in VLLM. In the near-future, the
27 + # implementation and this test will be extended to consider
28 + # visual inputs as well.
29 + with hf_runner(model, dtype=dtype, is_vision_model=False) as hf_model:
30 + hf_outputs = hf_model.generate_greedy_logprobs_limit(
31 + example_prompts,
32 + max_tokens,
33 + num_logprobs=num_logprobs,
34 + )
35 +
36 + with vllm_runner(model, dtype=dtype) as vllm_model:
37 + vllm_outputs = vllm_model.generate_greedy_logprobs(
38 + example_prompts,
39 + max_tokens,
40 + num_logprobs=num_logprobs,
41 + )
42 +
43 + check_logprobs_close(
44 + outputs_0_lst=hf_outputs,
45 + outputs_1_lst=vllm_outputs,
46 + name_0="hf",
47 + name_1="vllm",
48 + )