A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[V1][Tests] Adding additional testing for multimodal models to V1 (#13308)
Signed-off-by: andoorve <37849411+andoorve@users.noreply.github.com>
Murali Andoorveedu committed
Feb 18, 2025 at 09:53 UTC
a4d577b37944cbfa1bc62e4869667d1e2739d62a
1 file changed
+48
-12
tests/v1/engine/test_async_llm.py
+48
-12
@@ -8,7 +8,9 @@ import pytest
8
9
from tests.v1.engine.utils import PLP_APC_UNSUPPORTED_MSG
10
from vllm import SamplingParams
11
+from vllm.assets.image import ImageAsset
12
from vllm.engine.arg_utils import AsyncEngineArgs
13
+from vllm.inputs import PromptType
14
from vllm.platforms import current_platform
15
from vllm.sampling_params import RequestOutputKind
16
from vllm.v1.engine.async_llm import AsyncLLM
@@ -17,13 +19,32 @@ if not current_platform.is_cuda():
19
pytest.skip(reason="V1 currently only supported on CUDA.",
20
allow_module_level=True)
21
20
-ENGINE_ARGS = AsyncEngineArgs(model="meta-llama/Llama-3.2-1B-Instruct",
21
- enforce_eager=True,
22
- disable_log_requests=True)
22
+TEXT_ENGINE_ARGS = AsyncEngineArgs(model="meta-llama/Llama-3.2-1B-Instruct",
23
+ enforce_eager=True,
24
+ disable_log_requests=True)
25
+
26
+VISION_ENGINE_ARGS = AsyncEngineArgs(model="Qwen/Qwen2-VL-2B-Instruct",
27
+ enforce_eager=True,
28
+ disable_log_requests=True)
29
+
30
+TEXT_PROMPT = "Hello my name is Robert and"
31
+
32
+VISION_PROMPT_TEMPLATE = (
33
+ "<|im_start|>system\nYou are a helpful assistant.<|im_end|>"
34
+ "\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
35
+ "What is in the image?<|im_end|>\n"
36
+ "<|im_start|>assistant\n")
37
+VISION_PROMPT = {
38
+ "prompt": VISION_PROMPT_TEMPLATE,
39
+ "multi_modal_data": {
40
+ "image": ImageAsset("stop_sign").pil_image
41
+ }
42
+}
43
44
45
async def generate(engine: AsyncLLM,
46
request_id: str,
47
+ prompt: PromptType,
48
output_kind: RequestOutputKind,
49
max_tokens: int,
50
prompt_logprobs: Optional[int] = None) -> Tuple[int, str]:
@@ -32,11 +53,12 @@ async def generate(engine: AsyncLLM,
53
54
count = 0
55
sampling_params = SamplingParams(max_tokens=max_tokens,
56
+ ignore_eos=True,
57
output_kind=output_kind,
58
temperature=0,
59
prompt_logprobs=prompt_logprobs)
60
async for out in engine.generate(request_id=request_id,
39
- prompt="Hello my name is Robert and",
61
+ prompt=prompt,
62
sampling_params=sampling_params):
63
64
num_tokens = len(out.outputs[0].token_ids)
@@ -74,6 +96,7 @@ async def test_async_llm_refuses_prompt_logprobs_with_apc(
96
await asyncio.create_task(
97
generate(engine,
98
"request-0",
99
+ TEXT_PROMPT,
100
output_kind,
101
10,
102
prompt_logprobs=5))
@@ -86,18 +109,24 @@ async def test_async_llm_refuses_prompt_logprobs_with_apc(
109
110
@pytest.mark.parametrize(
111
"output_kind", [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY])
112
+@pytest.mark.parametrize("engine_args_and_prompt",
113
+ [(TEXT_ENGINE_ARGS, TEXT_PROMPT),
114
+ (VISION_ENGINE_ARGS, VISION_PROMPT)])
115
@pytest.mark.asyncio
90
-async def test_load(monkeypatch, output_kind: RequestOutputKind):
116
+async def test_load(monkeypatch, output_kind: RequestOutputKind,
117
+ engine_args_and_prompt: Tuple[AsyncEngineArgs,
118
+ PromptType]):
119
# TODO(rickyx): Remove monkeypatch once we have a better way to test V1
120
# so that in the future when we switch, we don't have to change all the
121
# tests.
122
with monkeypatch.context() as m, ExitStack() as after:
123
m.setenv("VLLM_USE_V1", "1")
124
+ engine_args, prompt = engine_args_and_prompt
125
97
- engine = AsyncLLM.from_engine_args(ENGINE_ARGS)
126
+ engine = AsyncLLM.from_engine_args(engine_args)
127
after.callback(engine.shutdown)
128
100
- NUM_REQUESTS = 10000
129
+ NUM_REQUESTS = 100
130
NUM_EXPECTED_TOKENS = 10
131
132
request_ids = [f"request-{i}" for i in range(NUM_REQUESTS)]
@@ -107,7 +136,7 @@ async def test_load(monkeypatch, output_kind: RequestOutputKind):
136
for request_id in request_ids:
137
tasks.append(
138
asyncio.create_task(
110
- generate(engine, request_id, output_kind,
139
+ generate(engine, request_id, prompt, output_kind,
140
NUM_EXPECTED_TOKENS)))
141
142
# Confirm that we got all the EXPECTED tokens from the requests.
@@ -126,13 +155,19 @@ async def test_load(monkeypatch, output_kind: RequestOutputKind):
155
156
@pytest.mark.parametrize(
157
"output_kind", [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY])
158
+@pytest.mark.parametrize("engine_args_and_prompt",
159
+ [(TEXT_ENGINE_ARGS, TEXT_PROMPT),
160
+ (VISION_ENGINE_ARGS, VISION_PROMPT)])
161
@pytest.mark.asyncio
130
-async def test_abort(monkeypatch, output_kind: RequestOutputKind):
162
+async def test_abort(monkeypatch, output_kind: RequestOutputKind,
163
+ engine_args_and_prompt: Tuple[AsyncEngineArgs,
164
+ PromptType]):
165
166
with monkeypatch.context() as m, ExitStack() as after:
167
m.setenv("VLLM_USE_V1", "1")
168
+ engine_args, prompt = engine_args_and_prompt
169
135
- engine = AsyncLLM.from_engine_args(ENGINE_ARGS)
170
+ engine = AsyncLLM.from_engine_args(engine_args)
171
after.callback(engine.shutdown)
172
173
NUM_REQUESTS = 100
@@ -146,7 +181,7 @@ async def test_abort(monkeypatch, output_kind: RequestOutputKind):
181
for request_id in request_ids:
182
tasks.append(
183
asyncio.create_task(
149
- generate(engine, request_id, output_kind,
184
+ generate(engine, request_id, prompt, output_kind,
185
NUM_EXPECTED_TOKENS)))
186
187
# API server cancels requests when they disconnect.
@@ -172,7 +207,8 @@ async def test_abort(monkeypatch, output_kind: RequestOutputKind):
207
# Confirm we can do another generation.
208
request_id = f"request-{REQUEST_IDS_TO_ABORT[0]}"
209
task = asyncio.create_task(
175
- generate(engine, request_id, output_kind, NUM_EXPECTED_TOKENS))
210
+ generate(engine, request_id, prompt, output_kind,
211
+ NUM_EXPECTED_TOKENS))
212
num_generated_tokens, request_id = await task
213
assert num_generated_tokens == NUM_EXPECTED_TOKENS
214
assert not engine.output_processor.has_unfinished_requests()