A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[V1] [2/n] Logging and Metrics - `OutputProcessor` Abstraction (#11973)
Signed-off-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com>
Robert Shaw committed
Jan 12, 2025 at 23:54 UTC
619ae268c3dc848a9b2f04579ea78ac5655f190f
8 files changed
+450
-210
tests/v1/engine/test_async_llm.py
+58
-11
@@ -1,5 +1,5 @@
1
import asyncio
2
-from typing import Tuple
2
+from typing import List, Tuple
3
4
import pytest
5
@@ -13,6 +13,7 @@ if not current_platform.is_cuda():
13
allow_module_level=True)
14
15
ENGINE_ARGS = AsyncEngineArgs(model="meta-llama/Llama-3.2-1B",
16
+ enforce_eager=True,
17
disable_log_requests=True)
18
19
@@ -53,17 +54,63 @@ async def test_load(monkeypatch):
54
generate(engine, request_id, NUM_EXPECTED_TOKENS)))
55
56
# Confirm that we got all the EXPECTED tokens from the requests.
56
- failed_request_id = None
57
- tokens = None
57
for task in tasks:
58
num_generated_tokens, request_id = await task
60
- if (num_generated_tokens != NUM_EXPECTED_TOKENS
61
- and failed_request_id is None):
62
- failed_request_id = request_id
63
- tokens = num_generated_tokens
64
-
65
- assert failed_request_id is None, (
66
- f"{failed_request_id} generated {tokens} but "
67
- f"expected {NUM_EXPECTED_TOKENS}")
59
+ assert num_generated_tokens == NUM_EXPECTED_TOKENS, (
60
+ f"{request_id} generated {num_generated_tokens} but "
61
+ f"expected {NUM_EXPECTED_TOKENS}")
62
+
63
+ assert not engine.output_processor.has_unfinished_requests()
64
+ engine.shutdown()
65
+
66
+
67
+@pytest.mark.asyncio
68
+async def test_abort(monkeypatch):
69
+
70
+ with monkeypatch.context() as m:
71
+ m.setenv("VLLM_USE_V1", "1")
72
+
73
+ engine = AsyncLLM.from_engine_args(ENGINE_ARGS)
74
+
75
+ NUM_REQUESTS = 100
76
+ NUM_EXPECTED_TOKENS = 100
77
+ REQUEST_IDS_TO_ABORT = range(1, 100, 10)
78
+
79
+ request_ids = [f"request-{i}" for i in range(NUM_REQUESTS)]
80
+
81
+ # Create concurrent requests.
82
+ tasks: List[asyncio.Task] = []
83
+ for request_id in request_ids:
84
+ tasks.append(
85
+ asyncio.create_task(
86
+ generate(engine, request_id, NUM_EXPECTED_TOKENS)))
87
+
88
+ # API server cancels requests when they disconnect.
89
+ for idx in REQUEST_IDS_TO_ABORT:
90
+ tasks[idx].cancel()
91
+ await asyncio.sleep(0.1)
92
+
93
+ # Confirm the other requests are okay.
94
+ for idx, task in enumerate(tasks):
95
+ # Confirm that it was actually canceled.
96
+ if idx in REQUEST_IDS_TO_ABORT:
97
+ with pytest.raises(asyncio.CancelledError):
98
+ await task
99
+ else:
100
+ # Otherwise, make sure the request was not impacted.
101
+ num_generated_tokens, request_id = await task
102
+ assert num_generated_tokens == NUM_EXPECTED_TOKENS, (
103
+ f"{request_id} generated {num_generated_tokens} but "
104
+ f"expected {NUM_EXPECTED_TOKENS}")
105
+
106
+ assert not engine.output_processor.has_unfinished_requests()
107
+
108
+ # Confirm we can do another generation.
109
+ request_id = f"request-{REQUEST_IDS_TO_ABORT[0]}"
110
+ task = asyncio.create_task(
111
+ generate(engine, request_id, NUM_EXPECTED_TOKENS))
112
+ num_generated_tokens, request_id = await task
113
+ assert num_generated_tokens == NUM_EXPECTED_TOKENS
114
+ assert not engine.output_processor.has_unfinished_requests()
115
116
engine.shutdown()
tests/v1/engine/test_output_processor.py
renamed
+88
-11
@@ -3,11 +3,18 @@ from typing import List
3
import pytest
4
from transformers import AutoTokenizer
5
6
+from vllm.engine.arg_utils import EngineArgs
7
from vllm.sampling_params import RequestOutputKind, SamplingParams
8
+from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
9
from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
8
-from vllm.v1.engine.detokenizer import Detokenizer
10
+from vllm.v1.engine.output_processor import OutputProcessor
11
12
TOKENIZER_NAME = "mistralai/Mistral-7B-Instruct-v0.3"
13
+VLLM_CONFIG = EngineArgs(model=TOKENIZER_NAME).create_engine_config()
14
+TOKENIZER_GROUP = init_tokenizer_from_configs(VLLM_CONFIG.model_config,
15
+ VLLM_CONFIG.scheduler_config,
16
+ VLLM_CONFIG.parallel_config,
17
+ VLLM_CONFIG.lora_config)
18
tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_NAME)
19
20
FULL_STRINGS = [
@@ -66,7 +73,7 @@ class MockEngineCore:
73
"request_output_kind",
74
[RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY])
75
def test_incremental_detokenization(request_output_kind: RequestOutputKind):
69
- detokenizer = Detokenizer(TOKENIZER_NAME)
76
+ output_processor = OutputProcessor(TOKENIZER_GROUP, log_stats=False)
77
engine_core = MockEngineCore(GENERATION_TOKENS)
78
79
# Make N requests.
@@ -93,7 +100,7 @@ def test_incremental_detokenization(request_output_kind: RequestOutputKind):
100
101
# Add requests to the detokenizer.
102
for request in requests:
96
- detokenizer.add_request(request)
103
+ output_processor.add_request(request)
104
105
gen_strings = {}
106
gen_tokens = {}
@@ -104,7 +111,9 @@ def test_incremental_detokenization(request_output_kind: RequestOutputKind):
111
break
112
113
# Step the Detokenizer.
107
- request_outputs, requests_to_abort = detokenizer.step(outputs)
114
+ processed_outputs = output_processor.process_outputs(outputs, )
115
+ request_outputs = processed_outputs.request_outputs
116
+ requests_to_abort = processed_outputs.reqs_to_abort
117
assert len(requests_to_abort) == 0
118
119
# Update tracking.
@@ -128,13 +137,13 @@ def test_incremental_detokenization(request_output_kind: RequestOutputKind):
137
assert gen_str == ref_gen_str, f"{gen_str=}, {ref_gen_str=}"
138
assert gen_toks == ref_gen_toks, f"{gen_toks=}, {ref_gen_toks=}"
139
131
- assert detokenizer.get_num_unfinished_requests() == 0
132
- assert not detokenizer.has_unfinished_requests()
140
+ assert output_processor.get_num_unfinished_requests() == 0
141
+ assert not output_processor.has_unfinished_requests()
142
143
144
@pytest.mark.parametrize("include_stop_str_in_output", [True, False])
145
def test_stop_string(include_stop_str_in_output: bool):
137
- detokenizer = Detokenizer(TOKENIZER_NAME)
146
+ output_processor = OutputProcessor(TOKENIZER_GROUP, log_stats=False)
147
engine_core = MockEngineCore(GENERATION_TOKENS)
148
149
# Make N requests.
@@ -162,7 +171,7 @@ def test_stop_string(include_stop_str_in_output: bool):
171
172
# Add requests to the detokenizer.
173
for request in requests:
165
- detokenizer.add_request(request)
174
+ output_processor.add_request(request)
175
176
gen_strings = {}
177
aborted = []
@@ -173,7 +182,9 @@ def test_stop_string(include_stop_str_in_output: bool):
182
break
183
184
# Step the Detokenizer.
176
- request_outputs, requests_to_abort = detokenizer.step(outputs)
185
+ processed_outputs = output_processor.process_outputs(outputs)
186
+ request_outputs = processed_outputs.request_outputs
187
+ requests_to_abort = processed_outputs.reqs_to_abort
188
for request_output in request_outputs:
189
# If aborted, we should not get a request output.
190
assert request_output.request_id not in aborted
@@ -214,5 +225,71 @@ def test_stop_string(include_stop_str_in_output: bool):
225
assert gen_str == ref_str_exc_stop, (
226
f"{gen_str=}, {ref_str_exc_stop=}")
227
217
- assert detokenizer.get_num_unfinished_requests() == 0
218
- assert not detokenizer.has_unfinished_requests()
228
+ assert output_processor.get_num_unfinished_requests() == 0
229
+ assert not output_processor.has_unfinished_requests()
230
+
231
+
232
+def test_iteration_stats():
233
+ output_processor = OutputProcessor(TOKENIZER_GROUP, log_stats=True)
234
+ engine_core = MockEngineCore(GENERATION_TOKENS)
235
+
236
+ # Make N requests.
237
+ requests = [
238
+ EngineCoreRequest(
239
+ request_id=f"request-{idx}",
240
+ prompt=prompt,
241
+ prompt_token_ids=prompt_tokens,
242
+ arrival_time=0,
243
+ mm_inputs=None,
244
+ mm_hashes=None,
245
+ mm_placeholders=None,
246
+ eos_token_id=None,
247
+ lora_request=None,
248
+ sampling_params=SamplingParams(),
249
+ ) for idx, (
250
+ prompt,
251
+ prompt_tokens) in enumerate(zip(PROMPT_STRINGS, PROMPT_TOKENS))
252
+ ]
253
+
254
+ # Add all requests except one to the OutputProcessor.
255
+ num_active = len(GENERATION_TOKENS) - 1
256
+ for request in requests[:num_active]:
257
+ output_processor.add_request(request)
258
+ inactive_request = requests[num_active]
259
+
260
+ # First iteration has 2 prefills.
261
+ outputs = engine_core.get_outputs()[:num_active]
262
+ processed_outputs = output_processor.process_outputs(outputs)
263
+ iteration_stats = processed_outputs.iteration_stats
264
+ total_prompt_tokens = sum(
265
+ [len(prompt_tokens) for prompt_tokens in PROMPT_TOKENS[:num_active]])
266
+
267
+ assert iteration_stats.num_prompt_tokens == total_prompt_tokens
268
+ assert iteration_stats.num_generation_tokens == num_active
269
+
270
+ # Just decodes in this step.
271
+ outputs = engine_core.get_outputs()[:num_active]
272
+ processed_outputs = output_processor.process_outputs(outputs)
273
+ iteration_stats = processed_outputs.iteration_stats
274
+
275
+ assert iteration_stats.num_prompt_tokens == 0
276
+ assert iteration_stats.num_generation_tokens == num_active
277
+
278
+ # Add a new request - prefill and 2 decodes in this step.
279
+ output_processor.add_request(inactive_request)
280
+ num_active += 1
281
+ outputs = engine_core.get_outputs()[:num_active]
282
+ processed_outputs = output_processor.process_outputs(outputs)
283
+ iteration_stats = processed_outputs.iteration_stats
284
+ total_prompt_tokens = len(PROMPT_TOKENS[num_active - 1])
285
+
286
+ assert iteration_stats.num_prompt_tokens == total_prompt_tokens
287
+ assert iteration_stats.num_generation_tokens == num_active
288
+
289
+ # Just decodes in this step.
290
+ outputs = engine_core.get_outputs()[:num_active]
291
+ processed_outputs = output_processor.process_outputs(outputs)
292
+ iteration_stats = processed_outputs.iteration_stats
293
+
294
+ assert iteration_stats.num_prompt_tokens == 0
295
+ assert iteration_stats.num_generation_tokens == num_active
vllm/v1/engine/async_llm.py
+37
-52
@@ -1,6 +1,6 @@
1
import asyncio
2
import os
3
-from typing import AsyncGenerator, Dict, List, Mapping, Optional, Type, Union
3
+from typing import AsyncGenerator, List, Mapping, Optional, Type, Union
4
5
from vllm.config import ModelConfig, VllmConfig
6
from vllm.engine.arg_utils import AsyncEngineArgs
@@ -18,11 +18,11 @@ from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
18
from vllm.usage.usage_lib import UsageContext
19
from vllm.utils import kill_process_tree
20
from vllm.v1.engine.core_client import EngineCoreClient
21
-from vllm.v1.engine.detokenizer import Detokenizer
21
+from vllm.v1.engine.output_processor import OutputProcessor
22
from vllm.v1.engine.processor import Processor
23
from vllm.v1.executor.abstract import Executor
24
from vllm.v1.metrics.loggers import LoggingStatLogger, StatLoggerBase
25
-from vllm.v1.metrics.stats import SchedulerStats
25
+from vllm.v1.metrics.stats import IterationStats, SchedulerStats
26
27
logger = init_logger(__name__)
28
@@ -59,9 +59,6 @@ class AsyncLLM(EngineClient):
59
lora_config=vllm_config.lora_config)
60
self.tokenizer.ping()
61
62
- # Request streams (map of request_id -> queue).
63
- self.rid_to_queue: Dict[str, asyncio.Queue] = {}
64
-
62
# Processor (converts Inputs --> EngineCoreRequests).
63
self.processor = Processor(
64
model_config=vllm_config.model_config,
@@ -71,13 +68,9 @@ class AsyncLLM(EngineClient):
68
input_registry=input_registry,
69
)
70
74
- # Detokenizer (converts EngineCoreOutputs --> RequestOutput).
75
- self.detokenizer = Detokenizer(
76
- tokenizer_name=vllm_config.model_config.tokenizer,
77
- tokenizer_mode=vllm_config.model_config.tokenizer_mode,
78
- trust_remote_code=vllm_config.model_config.trust_remote_code,
79
- revision=vllm_config.model_config.tokenizer_revision,
80
- )
71
+ # OutputProcessor (converts EngineCoreOutputs --> RequestOutput).
72
+ self.output_processor = OutputProcessor(self.tokenizer,
73
+ log_stats=self.log_stats)
74
75
# EngineCore (starts the engine in background process).
76
self.engine_core = EngineCoreClient.make_client(
@@ -140,9 +133,9 @@ class AsyncLLM(EngineClient):
133
"""Add new request to the AsyncLLM."""
134
135
# 1) Create a new output queue for the request.
143
- if request_id in self.rid_to_queue:
136
+ if self.output_processor.is_request_active(request_id):
137
raise ValueError(f"Request id {request_id} already running.")
145
- self.rid_to_queue[request_id] = asyncio.Queue()
138
+ queue: asyncio.Queue[RequestOutput] = asyncio.Queue()
139
140
# 2) Convert Input --> Request.
141
request = self.processor.process_inputs(request_id, prompt, params,
@@ -151,8 +144,8 @@ class AsyncLLM(EngineClient):
144
prompt_adapter_request,
145
priority)
146
154
- # 3) Add the request to Detokenizer (this process).
155
- self.detokenizer.add_request(request)
147
+ # 3) Add the request to OutputProcessor (this process).
148
+ self.output_processor.add_request(request, queue)
149
150
# 4) Add the EngineCoreRequest to EngineCore (separate process).
151
await self.engine_core.add_request_async(request)
@@ -160,7 +153,7 @@ class AsyncLLM(EngineClient):
153
if self.log_requests:
154
logger.info("Added request %s.", request_id)
155
163
- return self.rid_to_queue[request_id]
156
+ return queue
157
158
# TODO: we should support multiple prompts in one call, as you
159
# can do with LLM.generate. So that for multi-prompt completion
@@ -217,10 +210,9 @@ class AsyncLLM(EngineClient):
210
# task switching under load which helps performance).
211
out = q.get_nowait() if q.qsize() > 0 else await q.get()
212
220
- # Note: both Detokenizer and EngineCore handle their
213
+ # Note: both OutputProcessor and EngineCore handle their
214
# own request cleanup based on finished.
215
if out.finished:
223
- del self.rid_to_queue[request_id]
216
yield out
217
break
218
@@ -233,57 +225,51 @@ class AsyncLLM(EngineClient):
225
await self.abort(request_id)
226
raise
227
236
- def _process_request_outputs(self, request_outputs: List[RequestOutput]):
237
- """Process outputs by putting them into per-request queues."""
238
-
239
- for request_output in request_outputs:
240
- request_id = request_output.request_id
241
-
242
- # Note: it is possible a request was aborted and removed from
243
- # the state due to client cancellations, so if we encounter a
244
- # request id not in the state, we skip.
245
- if request_id in self.rid_to_queue:
246
- self.rid_to_queue[request_id].put_nowait(request_output)
247
-
228
async def _run_output_handler(self):
229
"""Background loop: pulls from EngineCore and pushes to AsyncStreams."""
230
231
try:
232
while True:
253
- # 1) Pull EngineCoreOutput from the EngineCore.
233
+ # 1) Pull EngineCoreOutputs from the EngineCore.
234
outputs = await self.engine_core.get_output_async()
235
256
- # 2) Detokenize based on the output.
257
- request_outputs, reqs_to_abort = self.detokenizer.step(
236
+ # 2) Process EngineCoreOutputs.
237
+ processed_outputs = self.output_processor.process_outputs(
238
outputs.outputs)
239
+ # NOTE: RequestOutputs are pushed to their queues.
240
+ assert len(processed_outputs.request_outputs) == 0
241
260
- # 3) Put the RequestOutputs into the per-request queues.
261
- self._process_request_outputs(request_outputs)
242
+ # 3) Abort any reqs that finished due to stop strings.
243
+ await self.engine_core.abort_requests_async(
244
+ processed_outputs.reqs_to_abort)
245
263
- # 4) Abort any requests that finished due to stop strings.
264
- await self.engine_core.abort_requests_async(reqs_to_abort)
265
-
266
- # 5) Log any stats.
267
- await self._log_stats(scheduler_stats=outputs.scheduler_stats)
246
+ # 4) Logging.
247
+ # TODO(rob): make into a coroutine and launch it in
248
+ # background thread once we add Prometheus.
249
+ self._log_stats(
250
+ scheduler_stats=outputs.scheduler_stats,
251
+ iteration_stats=processed_outputs.iteration_stats,
252
+ )
253
254
except Exception as e:
255
logger.exception("EngineCore output handler hit an error: %s", e)
256
kill_process_tree(os.getpid())
257
258
async def abort(self, request_id: str) -> None:
274
- """Abort RequestId in self, detokenizer, and engine core."""
259
+ """Abort RequestId in OutputProcessor and EngineCore."""
260
261
request_ids = [request_id]
262
await self.engine_core.abort_requests_async(request_ids)
278
- self.detokenizer.abort_requests(request_ids)
263
+ self.output_processor.abort_requests(request_ids)
264
280
- # If a request finishes while we await then the request_id
281
- # will be removed from the tracked queues before we get here.
282
- if request_id in self.rid_to_queue:
283
- del self.rid_to_queue[request_id]
265
+ if self.log_requests:
266
+ logger.info("Aborted request %s.", request_id)
267
285
- async def _log_stats(self, scheduler_stats: SchedulerStats):
286
- """Log stats to the stat loggers."""
268
+ def _log_stats(
269
+ self,
270
+ scheduler_stats: SchedulerStats,
271
+ iteration_stats: IterationStats,
272
+ ):
273
if not self.log_stats:
274
return
275
@@ -314,8 +300,7 @@ class AsyncLLM(EngineClient):
300
self,
301
lora_request: Optional[LoRARequest] = None,
302
) -> AnyTokenizer:
317
- assert lora_request is None
318
- return self.detokenizer.tokenizer
303
+ return self.tokenizer.get_lora_tokenizer(lora_request)
304
305
async def is_tracing_enabled(self) -> bool:
306
return False
vllm/v1/engine/core_client.py
+4
-2
@@ -105,7 +105,8 @@ class InprocClient(EngineCoreClient):
105
self.engine_core.add_request(request)
106
107
def abort_requests(self, request_ids: List[str]) -> None:
108
- self.engine_core.abort_requests(request_ids)
108
+ if len(request_ids) > 0:
109
+ self.engine_core.abort_requests(request_ids)
110
111
def shutdown(self):
112
self.engine_core.shutdown()
@@ -221,7 +222,8 @@ class SyncMPClient(MPClient):
222
self._send_input(EngineCoreRequestType.ADD, request)
223
224
def abort_requests(self, request_ids: List[str]) -> None:
224
- self._send_input(EngineCoreRequestType.ABORT, request_ids)
225
+ if len(request_ids) > 0:
226
+ self._send_input(EngineCoreRequestType.ABORT, request_ids)
227
228
def profile(self, is_start: bool = True) -> None:
229
self._send_input(EngineCoreRequestType.PROFILE,
vllm/v1/engine/detokenizer.py
+22
-115
@@ -1,18 +1,25 @@
1
from dataclasses import dataclass
2
-from typing import Dict, Iterable, List, Optional, Tuple, Union
2
+from typing import List, Optional, Union
3
4
from vllm.engine.output_processor.stop_checker import StopChecker
5
from vllm.logger import init_logger
6
-from vllm.outputs import RequestOutput
6
from vllm.sampling_params import RequestOutputKind
7
from vllm.transformers_utils.detokenizer_utils import (
8
AnyTokenizer, convert_prompt_ids_to_tokens, detokenize_incrementally)
10
-from vllm.transformers_utils.tokenizer import get_tokenizer
9
from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
10
11
logger = init_logger(__name__)
12
13
14
+@dataclass
15
+class DetokenizerOutput:
16
+ output_text: str
17
+ token_ids: List[int]
18
+ finished: bool
19
+ finish_reason: Optional[str] = None
20
+ stop_reason: Union[int, str, None] = None
21
+
22
+
23
@dataclass
24
class IncrementalDetokenizer:
25
@@ -20,6 +27,7 @@ class IncrementalDetokenizer:
27
output_text: str
28
tokens: List[str]
29
token_ids: List[int]
30
+ prompt_len: int
31
32
# Stop strings
33
stop: List[str]
@@ -34,11 +42,6 @@ class IncrementalDetokenizer:
42
spaces_between_special_tokens: bool
43
output_kind: RequestOutputKind
44
37
- # TODO: Probably decouple these
38
- request_id: str
39
- prompt: Optional[str]
40
- prompt_token_ids: List[int]
41
-
45
# Tokenizer for this request
46
tokenizer: AnyTokenizer
47
@@ -48,8 +51,7 @@ class IncrementalDetokenizer:
51
52
@property
53
def output_token_ids(self) -> List[int]:
51
- assert len(self.token_ids) >= len(self.prompt_token_ids)
52
- return self.token_ids[len(self.prompt_token_ids):]
54
+ return self.token_ids[self.prompt_len:]
55
56
@classmethod
57
def from_new_request(
@@ -87,25 +89,25 @@ class IncrementalDetokenizer:
89
spaces_between_special_tokens=request.sampling_params.
90
spaces_between_special_tokens,
91
output_kind=request.sampling_params.output_kind,
90
- request_id=request.request_id,
91
- prompt=request.prompt,
92
- prompt_token_ids=request.prompt_token_ids,
92
+ prompt_len=len(request.prompt_token_ids),
93
tokenizer=tokenizer,
94
stop_buffer_length=stop_buffer_length,
95
)
96
97
- def add_tokens(
97
+ def update_from_output(
98
self,
99
- new_token_ids: List[int],
100
- finish_reason: Optional[str],
101
- stop_reason: Optional[Union[int, str, None]],
102
- ) -> Optional[RequestOutput]:
99
+ output: EngineCoreOutput,
100
+ ) -> Optional[DetokenizerOutput]:
101
"""
102
Update RequestState for the request_id by:
103
1) Detokenize the new token ids incrementally.
104
2) Update the RequestOutput with the new text.
105
"""
106
107
+ new_token_ids = output.new_token_ids
108
+ finish_reason = output.finish_reason
109
+ stop_reason = output.stop_reason
110
+
111
# 1) Detokenize the new token ids incrementally.
112
# TODO(woosuk): This method becomes very inefficient when the number of
113
# new_token_ids is more than 1. We need to optimize this.
@@ -158,21 +160,8 @@ class IncrementalDetokenizer:
160
output_text = self._get_next_output_text(finished, delta)
161
token_ids = new_token_ids if delta else self.output_token_ids
162
161
- request_output = RequestOutput.new(
162
- self.request_id,
163
- self.prompt,
164
- self.prompt_token_ids,
165
- output_text,
166
- token_ids,
167
- finished,
168
- )
169
-
170
- if finished:
171
- completion_output = request_output.outputs[0]
172
- completion_output.finish_reason = finish_reason
173
- completion_output.stop_reason = stop_reason
174
-
175
- return request_output
163
+ return DetokenizerOutput(output_text, token_ids, finished,
164
+ finish_reason, stop_reason)
165
166
def _get_next_output_text(self, finished: bool, delta: bool) -> str:
167
"""If delta is True, only new text since the last call to
@@ -189,85 +178,3 @@ class IncrementalDetokenizer:
178
self._last_output_text_offset = length
179
return self.output_text[last_offset:length]
180
return ""
192
-
193
-
194
-class Detokenizer:
195
-
196
- def __init__(self,
197
- tokenizer_name: str,
198
- tokenizer_mode: str = "auto",
199
- trust_remote_code: bool = False,
200
- revision: Optional[str] = None):
201
- # TODO: once we support LoRA, we should should pass the tokenizer
202
- # here. We currently have two copies (this + in the LLMEngine).
203
- self.tokenizer = get_tokenizer(tokenizer_name=tokenizer_name,
204
- tokenizer_mode=tokenizer_mode,
205
- trust_remote_code=trust_remote_code,
206
- revision=revision)
207
-
208
- # Request id -> IncrementalDetokenizer
209
- self.request_states: Dict[str, IncrementalDetokenizer] = {}
210
-
211
- def is_request_active(self, request_id: str):
212
- return request_id in self.request_states
213
-
214
- def get_num_unfinished_requests(self):
215
- return len(self.request_states)
216
-
217
- def has_unfinished_requests(self) -> bool:
218
- return len(self.request_states) > 0
219
-
220
- def abort_requests(
221
- self,
222
- request_ids: Iterable[str],
223
- ) -> None:
224
- """Remove the request_ids from the Detokenizer."""
225
-
226
- for request_id in request_ids:
227
- self.request_states.pop(request_id, None)
228
-
229
- def add_request(
230
- self,
231
- request: EngineCoreRequest,
232
- ):
233
- """Add new request to the Detokenizer."""
234
-
235
- assert (request.request_id not in self.request_states)
236
-
237
- request_state = IncrementalDetokenizer.from_new_request(
238
- self.tokenizer, request)
239
- self.request_states[request.request_id] = request_state
240
-
241
- def step(
242
- self, encore_core_outputs: List[EngineCoreOutput]
243
- ) -> Tuple[List[RequestOutput], List[str]]:
244
- """Update state and request the RequestOutputs to the LLMEngine."""
245
-
246
- request_outputs: List[RequestOutput] = []
247
- requests_to_abort: List[str] = []
248
- for engine_core_output in encore_core_outputs:
249
- request_id = engine_core_output.request_id
250
- detokenizer = self.request_states.get(request_id)
251
- if detokenizer is None:
252
- # Ignore output for already-aborted request.
253
- continue
254
-
255
- # Detokenize and update state.
256
- request_output = detokenizer.add_tokens(
257
- new_token_ids=engine_core_output.new_token_ids,
258
- finish_reason=engine_core_output.finish_reason,
259
- stop_reason=engine_core_output.stop_reason,
260
- )
261
-
262
- if request_output is not None:
263
- # Add to RequestOutputs list.
264
- request_outputs.append(request_output)
265
-
266
- # Free completed requests.
267
- if request_output.finished:
268
- self.request_states.pop(request_id)
269
- if not engine_core_output.finished:
270
- requests_to_abort.append(request_id)
271
-
272
- # Return to EngineClient.
273
- return request_outputs, requests_to_abort
vllm/v1/engine/llm_engine.py
+14
-19
@@ -18,7 +18,7 @@ from vllm.transformers_utils.tokenizer_group import (
18
BaseTokenizerGroup, init_tokenizer_from_configs)
19
from vllm.usage.usage_lib import UsageContext
20
from vllm.v1.engine.core_client import EngineCoreClient
21
-from vllm.v1.engine.detokenizer import Detokenizer
21
+from vllm.v1.engine.output_processor import OutputProcessor
22
from vllm.v1.engine.processor import Processor
23
from vllm.v1.executor.abstract import Executor
24
@@ -60,13 +60,9 @@ class LLMEngine:
60
input_registry=input_registry,
61
mm_registry=mm_registry)
62
63
- # Detokenizer (converts EngineCoreOutputs --> RequestOutput)
64
- self.detokenizer = Detokenizer(
65
- tokenizer_name=vllm_config.model_config.tokenizer,
66
- tokenizer_mode=vllm_config.model_config.tokenizer_mode,
67
- trust_remote_code=vllm_config.model_config.trust_remote_code,
68
- revision=vllm_config.model_config.tokenizer_revision,
69
- )
63
+ # OutputProcessor (convert EngineCoreOutputs --> RequestOutput).
64
+ self.output_processor = OutputProcessor(self.tokenizer,
65
+ log_stats=False)
66
67
# EngineCore (gets EngineCoreRequests and gives EngineCoreOutputs)
68
self.engine_core = EngineCoreClient.make_client(
@@ -103,10 +99,10 @@ class LLMEngine:
99
multiprocess_mode=enable_multiprocessing)
100
101
def get_num_unfinished_requests(self) -> int:
106
- return self.detokenizer.get_num_unfinished_requests()
102
+ return self.output_processor.get_num_unfinished_requests()
103
104
def has_unfinished_requests(self) -> bool:
109
- return self.detokenizer.has_unfinished_requests()
105
+ return self.output_processor.has_unfinished_requests()
106
107
@classmethod
108
def validate_outputs(cls, outputs, output_type):
@@ -116,7 +112,7 @@ class LLMEngine:
112
"""Remove request_ids from EngineCore and Detokenizer."""
113
114
self.engine_core.abort_requests(request_ids)
119
- self.detokenizer.abort_requests(request_ids)
115
+ self.output_processor.abort_requests(request_ids)
116
117
def add_request(
118
self,
@@ -137,8 +133,8 @@ class LLMEngine:
133
prompt_adapter_request,
134
priority)
135
140
- # 2) Add the request to Detokenizer.
141
- self.detokenizer.add_request(request)
136
+ # 2) Make a new RequestState and queue.
137
+ self.output_processor.add_request(request)
138
139
# 3) Add the request to EngineCore.
140
self.engine_core.add_request(request)
@@ -148,15 +144,14 @@ class LLMEngine:
144
# 1) Get EngineCoreOutput from the EngineCore.
145
outputs = self.engine_core.get_output()
146
151
- # 2) Detokenizer the EngineCoreOutput.
152
- request_outputs, requests_to_abort = self.detokenizer.step(
147
+ # 2) Process EngineCoreOutputs.
148
+ processed_outputs = self.output_processor.process_outputs(
149
outputs.outputs)
150
155
- # 3) Abort requests that finished due to stopping criteria.
156
- if requests_to_abort:
157
- self.abort_request(requests_to_abort)
151
+ # 3) Abort any reqs that finished due to stop strings.
152
+ self.engine_core.abort_requests(processed_outputs.reqs_to_abort)
153
159
- return request_outputs
154
+ return processed_outputs.request_outputs
155
156
def get_model_config(self):
157
return self.model_config
vllm/v1/engine/output_processor.py
new
+200
@@ -0,0 +1,200 @@
1
+import asyncio
2
+from dataclasses import dataclass
3
+from typing import Dict, List, Optional
4
+
5
+from vllm.outputs import RequestOutput
6
+from vllm.transformers_utils.detokenizer_utils import AnyTokenizer
7
+from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
8
+from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
9
+from vllm.v1.engine.detokenizer import (DetokenizerOutput,
10
+ IncrementalDetokenizer)
11
+from vllm.v1.metrics.stats import IterationStats
12
+
13
+
14
+@dataclass
15
+class OutputProcessorOutput:
16
+
17
+ request_outputs: List[RequestOutput]
18
+ reqs_to_abort: List[str]
19
+ iteration_stats: IterationStats
20
+
21
+
22
+class RequestState:
23
+
24
+ def __init__(
25
+ self,
26
+ request_id: str,
27
+ prompt: Optional[str],
28
+ prompt_token_ids: List[int],
29
+ detokenizer: IncrementalDetokenizer,
30
+ queue: Optional[asyncio.Queue[RequestOutput]],
31
+ ):
32
+ self.request_id = request_id
33
+ self.prompt = prompt
34
+ self.prompt_token_ids = prompt_token_ids
35
+ self.prompt_len = len(prompt_token_ids)
36
+ self.detokenizer = detokenizer
37
+ self.is_prefilling = True
38
+ self.queue = queue
39
+
40
+ @classmethod
41
+ def from_new_request(
42
+ cls,
43
+ tokenizer: AnyTokenizer,
44
+ request: EngineCoreRequest,
45
+ queue: Optional[asyncio.Queue[RequestOutput]] = None,
46
+ ) -> "RequestState":
47
+ return cls(
48
+ request_id=request.request_id,
49
+ prompt=request.prompt,
50
+ prompt_token_ids=request.prompt_token_ids,
51
+ detokenizer=IncrementalDetokenizer.from_new_request(
52
+ tokenizer=tokenizer,
53
+ request=request,
54
+ ),
55
+ queue=queue,
56
+ )
57
+
58
+
59
+class OutputProcessor:
60
+ """Process EngineCoreOutputs into RequestOutputs."""
61
+
62
+ def __init__(
63
+ self,
64
+ tokenizer: BaseTokenizerGroup,
65
+ log_stats: bool,
66
+ ):
67
+ self.log_stats = log_stats
68
+ self.tokenizer = tokenizer
69
+ self.request_states: Dict[str, RequestState] = {}
70
+
71
+ def is_request_active(self, request_id: str) -> bool:
72
+ return request_id in self.request_states
73
+
74
+ def get_num_unfinished_requests(self):
75
+ return len(self.request_states)
76
+
77
+ def has_unfinished_requests(self) -> bool:
78
+ return len(self.request_states) > 0
79
+
80
+ def abort_requests(
81
+ self,
82
+ request_ids: List[str],
83
+ ) -> None:
84
+ for request_id in request_ids:
85
+ self.request_states.pop(request_id, None)
86
+
87
+ def add_request(
88
+ self,
89
+ request: EngineCoreRequest,
90
+ queue: Optional[asyncio.Queue[RequestOutput]] = None,
91
+ ) -> None:
92
+ request_id = request.request_id
93
+ if request_id in self.request_states:
94
+ raise ValueError(f"Request id {request_id} already running.")
95
+
96
+ self.request_states[request_id] = RequestState.from_new_request(
97
+ tokenizer=self.tokenizer.get_lora_tokenizer(request.lora_request),
98
+ request=request,
99
+ queue=queue)
100
+
101
+ def process_outputs(
102
+ self,
103
+ engine_core_outputs: List[EngineCoreOutput],
104
+ ) -> OutputProcessorOutput:
105
+ """
106
+ Process the EngineCoreOutputs:
107
+ 1) Compute stats for logging
108
+ 2) Detokenize
109
+ 3) Create and handle RequestOutput objects:
110
+ * If there is a queue (for usage with AsyncLLM),
111
+ put the RequestOutput objects into the queue for
112
+ handling by the per-request generate() tasks.
113
+
114
+ * If there is no queue (for usage with LLMEngine),
115
+ return a list of RequestOutput objects.
116
+
117
+ ****************** NOTE FOR DEVELOPERS ******************
118
+
119
+ VLLM V1 minimizes the number of python loops over the full
120
+ batch to ensure system overheads are minimized. This is the
121
+ only function that should loop over EngineCoreOutputs.
122
+
123
+ If you need to touch every element of the batch, implement a
124
+ method called XXXClass.update_from_output() to be called
125
+ within the loop below. For examples, see:
126
+ * IterationStats.update_from_output()
127
+ * Detokenizer.update_from_output()
128
+
129
+ TODO(rob): add Protocol makes update_from_output explicit.
130
+
131
+ **********************************************************
132
+ """
133
+
134
+ request_outputs: List[RequestOutput] = []
135
+ reqs_to_abort: List[str] = []
136
+ iteration_stats = IterationStats(self.log_stats)
137
+ for engine_core_output in engine_core_outputs:
138
+ req_id = engine_core_output.request_id
139
+ req_state = self.request_states.get(req_id)
140
+ if req_state is None:
141
+ # Ignore output for already-aborted request.
142
+ continue
143
+
144
+ # 1) Compute stats for this iteration.
145
+ iteration_stats.update_from_output(engine_core_output,
146
+ req_state.is_prefilling,
147
+ req_state.prompt_len)
148
+ req_state.is_prefilling = False
149
+
150
+ # 2) Detokenize the token ids into text.
151
+ detokenizer_output = req_state.detokenizer.update_from_output(
152
+ engine_core_output)
153
+
154
+ # 3) Create and handle RequestOutput objects.
155
+ if request_output := self._make_request_output(
156
+ req_state, detokenizer_output):
157
+ if req_state.queue is not None:
158
+ # AsyncLLM: put into queue for handling by generate().
159
+ req_state.queue.put_nowait(request_output)
160
+ else:
161
+ # LLMEngine: return list of RequestOutputs.
162
+ request_outputs.append(request_output)
163
+
164
+ # Free completed requests.
165
+ if request_output.finished:
166
+ self.request_states.pop(req_id)
167
+ if not engine_core_output.finished:
168
+ # If req not finished in EngineCore, but Detokenizer
169
+ # detected stop string, abort needed in EngineCore.
170
+ reqs_to_abort.append(req_id)
171
+
172
+ return OutputProcessorOutput(
173
+ request_outputs=request_outputs,
174
+ reqs_to_abort=reqs_to_abort,
175
+ iteration_stats=iteration_stats,
176
+ )
177
+
178
+ def _make_request_output(
179
+ self,
180
+ request_state: RequestState,
181
+ detokenizer_output: Optional[DetokenizerOutput],
182
+ ) -> Optional[RequestOutput]:
183
+
184
+ if detokenizer_output is None:
185
+ return None
186
+
187
+ request_output = RequestOutput.new(
188
+ request_state.request_id,
189
+ request_state.prompt,
190
+ request_state.prompt_token_ids,
191
+ detokenizer_output.output_text,
192
+ detokenizer_output.token_ids,
193
+ detokenizer_output.finished,
194
+ )
195
+ if detokenizer_output.finished:
196
+ completion_output = request_output.outputs[0]
197
+ completion_output.finish_reason = detokenizer_output.finish_reason
198
+ completion_output.stop_reason = detokenizer_output.stop_reason
199
+
200
+ return request_output
vllm/v1/metrics/stats.py
+27
@@ -1,4 +1,8 @@
1
from dataclasses import dataclass
2
+from typing import TYPE_CHECKING
3
+
4
+if TYPE_CHECKING:
5
+ from vllm.v1.engine import EngineCoreOutput
6
7
8
@dataclass
@@ -10,3 +14,26 @@ class SchedulerStats:
14
15
# gpu_cache_usage: float = 0.0
16
# gpu_prefix_cache_hit_rate: float = 0.0
17
+
18
+
19
+class IterationStats:
20
+ """Stats associated with a single set of EngineCoreOutputs."""
21
+
22
+ def __init__(self, log_stats: bool):
23
+ self.log_stats = log_stats
24
+ self.num_generation_tokens = 0
25
+ self.num_prompt_tokens = 0
26
+
27
+ def update_from_output(self, output: "EngineCoreOutput",
28
+ is_prefilling: bool, prompt_len: int):
29
+ if not self.log_stats:
30
+ return
31
+
32
+ self.num_generation_tokens += len(output.new_token_ids)
33
+ if is_prefilling:
34
+ # This relies on the invariant that EngineCore does
35
+ # not stream outputs for partially completed prefills
36
+ # (scheduler.update_from_output makes EngineCoreOutput
37
+ # iff num_computed_tokens == num_tokens).
38
+ assert (len(output.new_token_ids) > 0)
39
+ self.num_prompt_tokens += prompt_len