@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 24b0205f58
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[V1][Frontend] Coalesce bunched `RequestOutput`s (#12298)

Signed-off-by: Nick Hill <nhill@redhat.com> Co-authored-by: Robert Shaw <rshaw@neuralmagic.com>

Nick Hill committed Jan 23, 2025 at 17:17 UTC 24b0205f58c28e05ac060ec6f0e4defe8e9c32eb
3 files changed +65 -18
tests/v1/engine/test_async_llm.py
+35 -16
@@ -1,4 +1,5 @@
1 import asyncio
2 +from contextlib import ExitStack
3 from typing import List, Tuple
4
5 import pytest
@@ -6,6 +7,7 @@ import pytest
7 from vllm import SamplingParams
8 from vllm.engine.arg_utils import AsyncEngineArgs
9 from vllm.platforms import current_platform
10 +from vllm.sampling_params import RequestOutputKind
11 from vllm.v1.engine.async_llm import AsyncLLM
12
13 if not current_platform.is_cuda():
@@ -18,28 +20,39 @@ ENGINE_ARGS = AsyncEngineArgs(model="meta-llama/Llama-3.2-1B",
20
21
22 async def generate(engine: AsyncLLM, request_id: str,
23 + output_kind: RequestOutputKind,
24 max_tokens: int) -> Tuple[int, str]:
25 count = 0
23 - async for _ in engine.generate(request_id=request_id,
24 - prompt="Hello my name is Robert and",
25 - sampling_params=SamplingParams(
26 - max_tokens=max_tokens, temperature=0)):
26 + sampling_params = SamplingParams(max_tokens=max_tokens,
27 + output_kind=output_kind,
28 + temperature=0)
29 + async for out in engine.generate(request_id=request_id,
30 + prompt="Hello my name is Robert and",
31 + sampling_params=sampling_params):
32 +
33 + num_tokens = len(out.outputs[0].token_ids)
34 + if output_kind == RequestOutputKind.DELTA:
35 + count += num_tokens
36 + else:
37 + count = num_tokens
38
28 - count += 1
39 await asyncio.sleep(0.)
40
41 return count, request_id
42
43
44 +@pytest.mark.parametrize(
45 + "output_kind", [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY])
46 @pytest.mark.asyncio
35 -async def test_load(monkeypatch):
47 +async def test_load(monkeypatch, output_kind: RequestOutputKind):
48 # TODO(rickyx): Remove monkeypatch once we have a better way to test V1
49 # so that in the future when we switch, we don't have to change all the
50 # tests.
39 - with monkeypatch.context() as m:
51 + with monkeypatch.context() as m, ExitStack() as after:
52 m.setenv("VLLM_USE_V1", "1")
53
54 engine = AsyncLLM.from_engine_args(ENGINE_ARGS)
55 + after.callback(engine.shutdown)
56
57 NUM_REQUESTS = 10000
58 NUM_EXPECTED_TOKENS = 10
@@ -51,26 +64,33 @@ async def test_load(monkeypatch):
64 for request_id in request_ids:
65 tasks.append(
66 asyncio.create_task(
54 - generate(engine, request_id, NUM_EXPECTED_TOKENS)))
67 + generate(engine, request_id, output_kind,
68 + NUM_EXPECTED_TOKENS)))
69
70 # Confirm that we got all the EXPECTED tokens from the requests.
57 - for task in tasks:
71 + done, pending = await asyncio.wait(tasks,
72 + return_when=asyncio.FIRST_EXCEPTION)
73 + for task in pending:
74 + task.cancel()
75 + for task in done:
76 num_generated_tokens, request_id = await task
77 assert num_generated_tokens == NUM_EXPECTED_TOKENS, (
78 f"{request_id} generated {num_generated_tokens} but "
79 f"expected {NUM_EXPECTED_TOKENS}")
80
81 assert not engine.output_processor.has_unfinished_requests()
64 - engine.shutdown()
82
83
84 +@pytest.mark.parametrize(
85 + "output_kind", [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY])
86 @pytest.mark.asyncio
68 -async def test_abort(monkeypatch):
87 +async def test_abort(monkeypatch, output_kind: RequestOutputKind):
88
70 - with monkeypatch.context() as m:
89 + with monkeypatch.context() as m, ExitStack() as after:
90 m.setenv("VLLM_USE_V1", "1")
91
92 engine = AsyncLLM.from_engine_args(ENGINE_ARGS)
93 + after.callback(engine.shutdown)
94
95 NUM_REQUESTS = 100
96 NUM_EXPECTED_TOKENS = 100
@@ -83,7 +103,8 @@ async def test_abort(monkeypatch):
103 for request_id in request_ids:
104 tasks.append(
105 asyncio.create_task(
86 - generate(engine, request_id, NUM_EXPECTED_TOKENS)))
106 + generate(engine, request_id, output_kind,
107 + NUM_EXPECTED_TOKENS)))
108
109 # API server cancels requests when they disconnect.
110 for idx in REQUEST_IDS_TO_ABORT:
@@ -108,9 +129,7 @@ async def test_abort(monkeypatch):
129 # Confirm we can do another generation.
130 request_id = f"request-{REQUEST_IDS_TO_ABORT[0]}"
131 task = asyncio.create_task(
111 - generate(engine, request_id, NUM_EXPECTED_TOKENS))
132 + generate(engine, request_id, output_kind, NUM_EXPECTED_TOKENS))
133 num_generated_tokens, request_id = await task
134 assert num_generated_tokens == NUM_EXPECTED_TOKENS
135 assert not engine.output_processor.has_unfinished_requests()
115 -
116 - engine.shutdown()
vllm/outputs.py
+21 -1
@@ -1,6 +1,6 @@
1 import time
2 from dataclasses import dataclass
3 -from typing import Dict, Generic, List, Optional
3 +from typing import Dict, Generic, List, MutableSequence, Optional
4 from typing import Sequence as GenericSequence
5 from typing import Union
6
@@ -162,6 +162,26 @@ class RequestOutput:
162 finished=finished,
163 )
164
165 + def add(self, next_output: "RequestOutput") -> None:
166 + """Merge subsequent RequestOutput into this one"""
167 +
168 + self.prompt = next_output.prompt
169 + self.prompt_token_ids = next_output.prompt_token_ids
170 + self.prompt_logprobs = next_output.prompt_logprobs
171 + self.finished |= next_output.finished
172 +
173 + #TODO assuming n == 1 for now
174 + completion = self.outputs[0]
175 + next_completion = next_output.outputs[0]
176 + completion.text += next_completion.text
177 + if not isinstance(completion.token_ids, MutableSequence):
178 + completion.token_ids = list(completion.token_ids)
179 + completion.token_ids.extend(next_completion.token_ids)
180 + if next_completion.logprobs:
181 + assert completion.logprobs is not None
182 + completion.logprobs.extend(next_completion.logprobs)
183 + completion.cumulative_logprob = next_completion.cumulative_logprob
184 +
185 @classmethod
186 def from_seq_group(
187 cls, seq_group: SequenceGroup, use_cache: bool,
vllm/v1/engine/async_llm.py
+9 -1
@@ -15,7 +15,7 @@ from vllm.lora.request import LoRARequest
15 from vllm.outputs import RequestOutput
16 from vllm.pooling_params import PoolingParams
17 from vllm.prompt_adapter.request import PromptAdapterRequest
18 -from vllm.sampling_params import SamplingParams
18 +from vllm.sampling_params import RequestOutputKind, SamplingParams
19 from vllm.transformers_utils.tokenizer import AnyTokenizer
20 from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
21 from vllm.usage.usage_lib import UsageContext
@@ -214,6 +214,14 @@ class AsyncLLM(EngineClient):
214 # task switching under load which helps performance).
215 out = q.get_nowait() if not q.empty() else await q.get()
216
217 + # Coalesce any additional queued outputs
218 + while not q.empty():
219 + next_out = q.get_nowait()
220 + if sampling_params.output_kind == RequestOutputKind.DELTA:
221 + out.add(next_out)
222 + else:
223 + out = next_out
224 +
225 # Note: both OutputProcessor and EngineCore handle their
226 # own request cleanup based on finished.
227 finished = out.finished