A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[V1][Frontend] Coalesce bunched `RequestOutput`s (#12298)
Signed-off-by: Nick Hill <nhill@redhat.com> Co-authored-by: Robert Shaw <rshaw@neuralmagic.com>
Nick Hill committed
Jan 23, 2025 at 17:17 UTC
24b0205f58c28e05ac060ec6f0e4defe8e9c32eb
3 files changed
+65
-18
tests/v1/engine/test_async_llm.py
+35
-16
@@ -1,4 +1,5 @@
1
import asyncio
2
+from contextlib import ExitStack
3
from typing import List, Tuple
4
5
import pytest
@@ -6,6 +7,7 @@ import pytest
7
from vllm import SamplingParams
8
from vllm.engine.arg_utils import AsyncEngineArgs
9
from vllm.platforms import current_platform
10
+from vllm.sampling_params import RequestOutputKind
11
from vllm.v1.engine.async_llm import AsyncLLM
12
13
if not current_platform.is_cuda():
@@ -18,28 +20,39 @@ ENGINE_ARGS = AsyncEngineArgs(model="meta-llama/Llama-3.2-1B",
20
21
22
async def generate(engine: AsyncLLM, request_id: str,
23
+ output_kind: RequestOutputKind,
24
max_tokens: int) -> Tuple[int, str]:
25
count = 0
23
- async for _ in engine.generate(request_id=request_id,
24
- prompt="Hello my name is Robert and",
25
- sampling_params=SamplingParams(
26
- max_tokens=max_tokens, temperature=0)):
26
+ sampling_params = SamplingParams(max_tokens=max_tokens,
27
+ output_kind=output_kind,
28
+ temperature=0)
29
+ async for out in engine.generate(request_id=request_id,
30
+ prompt="Hello my name is Robert and",
31
+ sampling_params=sampling_params):
32
+
33
+ num_tokens = len(out.outputs[0].token_ids)
34
+ if output_kind == RequestOutputKind.DELTA:
35
+ count += num_tokens
36
+ else:
37
+ count = num_tokens
38
28
- count += 1
39
await asyncio.sleep(0.)
40
41
return count, request_id
42
43
44
+@pytest.mark.parametrize(
45
+ "output_kind", [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY])
46
@pytest.mark.asyncio
35
-async def test_load(monkeypatch):
47
+async def test_load(monkeypatch, output_kind: RequestOutputKind):
48
# TODO(rickyx): Remove monkeypatch once we have a better way to test V1
49
# so that in the future when we switch, we don't have to change all the
50
# tests.
39
- with monkeypatch.context() as m:
51
+ with monkeypatch.context() as m, ExitStack() as after:
52
m.setenv("VLLM_USE_V1", "1")
53
54
engine = AsyncLLM.from_engine_args(ENGINE_ARGS)
55
+ after.callback(engine.shutdown)
56
57
NUM_REQUESTS = 10000
58
NUM_EXPECTED_TOKENS = 10
@@ -51,26 +64,33 @@ async def test_load(monkeypatch):
64
for request_id in request_ids:
65
tasks.append(
66
asyncio.create_task(
54
- generate(engine, request_id, NUM_EXPECTED_TOKENS)))
67
+ generate(engine, request_id, output_kind,
68
+ NUM_EXPECTED_TOKENS)))
69
70
# Confirm that we got all the EXPECTED tokens from the requests.
57
- for task in tasks:
71
+ done, pending = await asyncio.wait(tasks,
72
+ return_when=asyncio.FIRST_EXCEPTION)
73
+ for task in pending:
74
+ task.cancel()
75
+ for task in done:
76
num_generated_tokens, request_id = await task
77
assert num_generated_tokens == NUM_EXPECTED_TOKENS, (
78
f"{request_id} generated {num_generated_tokens} but "
79
f"expected {NUM_EXPECTED_TOKENS}")
80
81
assert not engine.output_processor.has_unfinished_requests()
64
- engine.shutdown()
82
83
84
+@pytest.mark.parametrize(
85
+ "output_kind", [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY])
86
@pytest.mark.asyncio
68
-async def test_abort(monkeypatch):
87
+async def test_abort(monkeypatch, output_kind: RequestOutputKind):
88
70
- with monkeypatch.context() as m:
89
+ with monkeypatch.context() as m, ExitStack() as after:
90
m.setenv("VLLM_USE_V1", "1")
91
92
engine = AsyncLLM.from_engine_args(ENGINE_ARGS)
93
+ after.callback(engine.shutdown)
94
95
NUM_REQUESTS = 100
96
NUM_EXPECTED_TOKENS = 100
@@ -83,7 +103,8 @@ async def test_abort(monkeypatch):
103
for request_id in request_ids:
104
tasks.append(
105
asyncio.create_task(
86
- generate(engine, request_id, NUM_EXPECTED_TOKENS)))
106
+ generate(engine, request_id, output_kind,
107
+ NUM_EXPECTED_TOKENS)))
108
109
# API server cancels requests when they disconnect.
110
for idx in REQUEST_IDS_TO_ABORT:
@@ -108,9 +129,7 @@ async def test_abort(monkeypatch):
129
# Confirm we can do another generation.
130
request_id = f"request-{REQUEST_IDS_TO_ABORT[0]}"
131
task = asyncio.create_task(
111
- generate(engine, request_id, NUM_EXPECTED_TOKENS))
132
+ generate(engine, request_id, output_kind, NUM_EXPECTED_TOKENS))
133
num_generated_tokens, request_id = await task
134
assert num_generated_tokens == NUM_EXPECTED_TOKENS
135
assert not engine.output_processor.has_unfinished_requests()
115
-
116
- engine.shutdown()
vllm/outputs.py
+21
-1
@@ -1,6 +1,6 @@
1
import time
2
from dataclasses import dataclass
3
-from typing import Dict, Generic, List, Optional
3
+from typing import Dict, Generic, List, MutableSequence, Optional
4
from typing import Sequence as GenericSequence
5
from typing import Union
6
@@ -162,6 +162,26 @@ class RequestOutput:
162
finished=finished,
163
)
164
165
+ def add(self, next_output: "RequestOutput") -> None:
166
+ """Merge subsequent RequestOutput into this one"""
167
+
168
+ self.prompt = next_output.prompt
169
+ self.prompt_token_ids = next_output.prompt_token_ids
170
+ self.prompt_logprobs = next_output.prompt_logprobs
171
+ self.finished |= next_output.finished
172
+
173
+ #TODO assuming n == 1 for now
174
+ completion = self.outputs[0]
175
+ next_completion = next_output.outputs[0]
176
+ completion.text += next_completion.text
177
+ if not isinstance(completion.token_ids, MutableSequence):
178
+ completion.token_ids = list(completion.token_ids)
179
+ completion.token_ids.extend(next_completion.token_ids)
180
+ if next_completion.logprobs:
181
+ assert completion.logprobs is not None
182
+ completion.logprobs.extend(next_completion.logprobs)
183
+ completion.cumulative_logprob = next_completion.cumulative_logprob
184
+
185
@classmethod
186
def from_seq_group(
187
cls, seq_group: SequenceGroup, use_cache: bool,
vllm/v1/engine/async_llm.py
+9
-1
@@ -15,7 +15,7 @@ from vllm.lora.request import LoRARequest
15
from vllm.outputs import RequestOutput
16
from vllm.pooling_params import PoolingParams
17
from vllm.prompt_adapter.request import PromptAdapterRequest
18
-from vllm.sampling_params import SamplingParams
18
+from vllm.sampling_params import RequestOutputKind, SamplingParams
19
from vllm.transformers_utils.tokenizer import AnyTokenizer
20
from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
21
from vllm.usage.usage_lib import UsageContext
@@ -214,6 +214,14 @@ class AsyncLLM(EngineClient):
214
# task switching under load which helps performance).
215
out = q.get_nowait() if not q.empty() else await q.get()
216
217
+ # Coalesce any additional queued outputs
218
+ while not q.empty():
219
+ next_out = q.get_nowait()
220
+ if sampling_params.output_kind == RequestOutputKind.DELTA:
221
+ out.add(next_out)
222
+ else:
223
+ out = next_out
224
+
225
# Note: both OutputProcessor and EngineCore handle their
226
# own request cleanup based on finished.
227
finished = out.finished