@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 619ae268c3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[V1] [2/n] Logging and Metrics - `OutputProcessor` Abstraction (#11973)

Signed-off-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com>

Robert Shaw committed Jan 12, 2025 at 23:54 UTC 619ae268c3dc848a9b2f04579ea78ac5655f190f
8 files changed +450 -210
tests/v1/engine/test_async_llm.py
+58 -11
@@ -1,5 +1,5 @@
1 import asyncio
2 -from typing import Tuple
2 +from typing import List, Tuple
3
4 import pytest
5
@@ -13,6 +13,7 @@ if not current_platform.is_cuda():
13 allow_module_level=True)
14
15 ENGINE_ARGS = AsyncEngineArgs(model="meta-llama/Llama-3.2-1B",
16 + enforce_eager=True,
17 disable_log_requests=True)
18
19
@@ -53,17 +54,63 @@ async def test_load(monkeypatch):
54 generate(engine, request_id, NUM_EXPECTED_TOKENS)))
55
56 # Confirm that we got all the EXPECTED tokens from the requests.
56 - failed_request_id = None
57 - tokens = None
57 for task in tasks:
58 num_generated_tokens, request_id = await task
60 - if (num_generated_tokens != NUM_EXPECTED_TOKENS
61 - and failed_request_id is None):
62 - failed_request_id = request_id
63 - tokens = num_generated_tokens
64 -
65 - assert failed_request_id is None, (
66 - f"{failed_request_id} generated {tokens} but "
67 - f"expected {NUM_EXPECTED_TOKENS}")
59 + assert num_generated_tokens == NUM_EXPECTED_TOKENS, (
60 + f"{request_id} generated {num_generated_tokens} but "
61 + f"expected {NUM_EXPECTED_TOKENS}")
62 +
63 + assert not engine.output_processor.has_unfinished_requests()
64 + engine.shutdown()
65 +
66 +
67 +@pytest.mark.asyncio
68 +async def test_abort(monkeypatch):
69 +
70 + with monkeypatch.context() as m:
71 + m.setenv("VLLM_USE_V1", "1")
72 +
73 + engine = AsyncLLM.from_engine_args(ENGINE_ARGS)
74 +
75 + NUM_REQUESTS = 100
76 + NUM_EXPECTED_TOKENS = 100
77 + REQUEST_IDS_TO_ABORT = range(1, 100, 10)
78 +
79 + request_ids = [f"request-{i}" for i in range(NUM_REQUESTS)]
80 +
81 + # Create concurrent requests.
82 + tasks: List[asyncio.Task] = []
83 + for request_id in request_ids:
84 + tasks.append(
85 + asyncio.create_task(
86 + generate(engine, request_id, NUM_EXPECTED_TOKENS)))
87 +
88 + # API server cancels requests when they disconnect.
89 + for idx in REQUEST_IDS_TO_ABORT:
90 + tasks[idx].cancel()
91 + await asyncio.sleep(0.1)
92 +
93 + # Confirm the other requests are okay.
94 + for idx, task in enumerate(tasks):
95 + # Confirm that it was actually canceled.
96 + if idx in REQUEST_IDS_TO_ABORT:
97 + with pytest.raises(asyncio.CancelledError):
98 + await task
99 + else:
100 + # Otherwise, make sure the request was not impacted.
101 + num_generated_tokens, request_id = await task
102 + assert num_generated_tokens == NUM_EXPECTED_TOKENS, (
103 + f"{request_id} generated {num_generated_tokens} but "
104 + f"expected {NUM_EXPECTED_TOKENS}")
105 +
106 + assert not engine.output_processor.has_unfinished_requests()
107 +
108 + # Confirm we can do another generation.
109 + request_id = f"request-{REQUEST_IDS_TO_ABORT[0]}"
110 + task = asyncio.create_task(
111 + generate(engine, request_id, NUM_EXPECTED_TOKENS))
112 + num_generated_tokens, request_id = await task
113 + assert num_generated_tokens == NUM_EXPECTED_TOKENS
114 + assert not engine.output_processor.has_unfinished_requests()
115
116 engine.shutdown()
tests/v1/engine/test_output_processor.py renamed
+88 -11
@@ -3,11 +3,18 @@ from typing import List
3 import pytest
4 from transformers import AutoTokenizer
5
6 +from vllm.engine.arg_utils import EngineArgs
7 from vllm.sampling_params import RequestOutputKind, SamplingParams
8 +from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
9 from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
8 -from vllm.v1.engine.detokenizer import Detokenizer
10 +from vllm.v1.engine.output_processor import OutputProcessor
11
12 TOKENIZER_NAME = "mistralai/Mistral-7B-Instruct-v0.3"
13 +VLLM_CONFIG = EngineArgs(model=TOKENIZER_NAME).create_engine_config()
14 +TOKENIZER_GROUP = init_tokenizer_from_configs(VLLM_CONFIG.model_config,
15 + VLLM_CONFIG.scheduler_config,
16 + VLLM_CONFIG.parallel_config,
17 + VLLM_CONFIG.lora_config)
18 tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_NAME)
19
20 FULL_STRINGS = [
@@ -66,7 +73,7 @@ class MockEngineCore:
73 "request_output_kind",
74 [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY])
75 def test_incremental_detokenization(request_output_kind: RequestOutputKind):
69 - detokenizer = Detokenizer(TOKENIZER_NAME)
76 + output_processor = OutputProcessor(TOKENIZER_GROUP, log_stats=False)
77 engine_core = MockEngineCore(GENERATION_TOKENS)
78
79 # Make N requests.
@@ -93,7 +100,7 @@ def test_incremental_detokenization(request_output_kind: RequestOutputKind):
100
101 # Add requests to the detokenizer.
102 for request in requests:
96 - detokenizer.add_request(request)
103 + output_processor.add_request(request)
104
105 gen_strings = {}
106 gen_tokens = {}
@@ -104,7 +111,9 @@ def test_incremental_detokenization(request_output_kind: RequestOutputKind):
111 break
112
113 # Step the Detokenizer.
107 - request_outputs, requests_to_abort = detokenizer.step(outputs)
114 + processed_outputs = output_processor.process_outputs(outputs, )
115 + request_outputs = processed_outputs.request_outputs
116 + requests_to_abort = processed_outputs.reqs_to_abort
117 assert len(requests_to_abort) == 0
118
119 # Update tracking.
@@ -128,13 +137,13 @@ def test_incremental_detokenization(request_output_kind: RequestOutputKind):
137 assert gen_str == ref_gen_str, f"{gen_str=}, {ref_gen_str=}"
138 assert gen_toks == ref_gen_toks, f"{gen_toks=}, {ref_gen_toks=}"
139
131 - assert detokenizer.get_num_unfinished_requests() == 0
132 - assert not detokenizer.has_unfinished_requests()
140 + assert output_processor.get_num_unfinished_requests() == 0
141 + assert not output_processor.has_unfinished_requests()
142
143
144 @pytest.mark.parametrize("include_stop_str_in_output", [True, False])
145 def test_stop_string(include_stop_str_in_output: bool):
137 - detokenizer = Detokenizer(TOKENIZER_NAME)
146 + output_processor = OutputProcessor(TOKENIZER_GROUP, log_stats=False)
147 engine_core = MockEngineCore(GENERATION_TOKENS)
148
149 # Make N requests.
@@ -162,7 +171,7 @@ def test_stop_string(include_stop_str_in_output: bool):
171
172 # Add requests to the detokenizer.
173 for request in requests:
165 - detokenizer.add_request(request)
174 + output_processor.add_request(request)
175
176 gen_strings = {}
177 aborted = []
@@ -173,7 +182,9 @@ def test_stop_string(include_stop_str_in_output: bool):
182 break
183
184 # Step the Detokenizer.
176 - request_outputs, requests_to_abort = detokenizer.step(outputs)
185 + processed_outputs = output_processor.process_outputs(outputs)
186 + request_outputs = processed_outputs.request_outputs
187 + requests_to_abort = processed_outputs.reqs_to_abort
188 for request_output in request_outputs:
189 # If aborted, we should not get a request output.
190 assert request_output.request_id not in aborted
@@ -214,5 +225,71 @@ def test_stop_string(include_stop_str_in_output: bool):
225 assert gen_str == ref_str_exc_stop, (
226 f"{gen_str=}, {ref_str_exc_stop=}")
227
217 - assert detokenizer.get_num_unfinished_requests() == 0
218 - assert not detokenizer.has_unfinished_requests()
228 + assert output_processor.get_num_unfinished_requests() == 0
229 + assert not output_processor.has_unfinished_requests()
230 +
231 +
232 +def test_iteration_stats():
233 + output_processor = OutputProcessor(TOKENIZER_GROUP, log_stats=True)
234 + engine_core = MockEngineCore(GENERATION_TOKENS)
235 +
236 + # Make N requests.
237 + requests = [
238 + EngineCoreRequest(
239 + request_id=f"request-{idx}",
240 + prompt=prompt,
241 + prompt_token_ids=prompt_tokens,
242 + arrival_time=0,
243 + mm_inputs=None,
244 + mm_hashes=None,
245 + mm_placeholders=None,
246 + eos_token_id=None,
247 + lora_request=None,
248 + sampling_params=SamplingParams(),
249 + ) for idx, (
250 + prompt,
251 + prompt_tokens) in enumerate(zip(PROMPT_STRINGS, PROMPT_TOKENS))
252 + ]
253 +
254 + # Add all requests except one to the OutputProcessor.
255 + num_active = len(GENERATION_TOKENS) - 1
256 + for request in requests[:num_active]:
257 + output_processor.add_request(request)
258 + inactive_request = requests[num_active]
259 +
260 + # First iteration has 2 prefills.
261 + outputs = engine_core.get_outputs()[:num_active]
262 + processed_outputs = output_processor.process_outputs(outputs)
263 + iteration_stats = processed_outputs.iteration_stats
264 + total_prompt_tokens = sum(
265 + [len(prompt_tokens) for prompt_tokens in PROMPT_TOKENS[:num_active]])
266 +
267 + assert iteration_stats.num_prompt_tokens == total_prompt_tokens
268 + assert iteration_stats.num_generation_tokens == num_active
269 +
270 + # Just decodes in this step.
271 + outputs = engine_core.get_outputs()[:num_active]
272 + processed_outputs = output_processor.process_outputs(outputs)
273 + iteration_stats = processed_outputs.iteration_stats
274 +
275 + assert iteration_stats.num_prompt_tokens == 0
276 + assert iteration_stats.num_generation_tokens == num_active
277 +
278 + # Add a new request - prefill and 2 decodes in this step.
279 + output_processor.add_request(inactive_request)
280 + num_active += 1
281 + outputs = engine_core.get_outputs()[:num_active]
282 + processed_outputs = output_processor.process_outputs(outputs)
283 + iteration_stats = processed_outputs.iteration_stats
284 + total_prompt_tokens = len(PROMPT_TOKENS[num_active - 1])
285 +
286 + assert iteration_stats.num_prompt_tokens == total_prompt_tokens
287 + assert iteration_stats.num_generation_tokens == num_active
288 +
289 + # Just decodes in this step.
290 + outputs = engine_core.get_outputs()[:num_active]
291 + processed_outputs = output_processor.process_outputs(outputs)
292 + iteration_stats = processed_outputs.iteration_stats
293 +
294 + assert iteration_stats.num_prompt_tokens == 0
295 + assert iteration_stats.num_generation_tokens == num_active
vllm/v1/engine/async_llm.py
+37 -52
@@ -1,6 +1,6 @@
1 import asyncio
2 import os
3 -from typing import AsyncGenerator, Dict, List, Mapping, Optional, Type, Union
3 +from typing import AsyncGenerator, List, Mapping, Optional, Type, Union
4
5 from vllm.config import ModelConfig, VllmConfig
6 from vllm.engine.arg_utils import AsyncEngineArgs
@@ -18,11 +18,11 @@ from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
18 from vllm.usage.usage_lib import UsageContext
19 from vllm.utils import kill_process_tree
20 from vllm.v1.engine.core_client import EngineCoreClient
21 -from vllm.v1.engine.detokenizer import Detokenizer
21 +from vllm.v1.engine.output_processor import OutputProcessor
22 from vllm.v1.engine.processor import Processor
23 from vllm.v1.executor.abstract import Executor
24 from vllm.v1.metrics.loggers import LoggingStatLogger, StatLoggerBase
25 -from vllm.v1.metrics.stats import SchedulerStats
25 +from vllm.v1.metrics.stats import IterationStats, SchedulerStats
26
27 logger = init_logger(__name__)
28
@@ -59,9 +59,6 @@ class AsyncLLM(EngineClient):
59 lora_config=vllm_config.lora_config)
60 self.tokenizer.ping()
61
62 - # Request streams (map of request_id -> queue).
63 - self.rid_to_queue: Dict[str, asyncio.Queue] = {}
64 -
62 # Processor (converts Inputs --> EngineCoreRequests).
63 self.processor = Processor(
64 model_config=vllm_config.model_config,
@@ -71,13 +68,9 @@ class AsyncLLM(EngineClient):
68 input_registry=input_registry,
69 )
70
74 - # Detokenizer (converts EngineCoreOutputs --> RequestOutput).
75 - self.detokenizer = Detokenizer(
76 - tokenizer_name=vllm_config.model_config.tokenizer,
77 - tokenizer_mode=vllm_config.model_config.tokenizer_mode,
78 - trust_remote_code=vllm_config.model_config.trust_remote_code,
79 - revision=vllm_config.model_config.tokenizer_revision,
80 - )
71 + # OutputProcessor (converts EngineCoreOutputs --> RequestOutput).
72 + self.output_processor = OutputProcessor(self.tokenizer,
73 + log_stats=self.log_stats)
74
75 # EngineCore (starts the engine in background process).
76 self.engine_core = EngineCoreClient.make_client(
@@ -140,9 +133,9 @@ class AsyncLLM(EngineClient):
133 """Add new request to the AsyncLLM."""
134
135 # 1) Create a new output queue for the request.
143 - if request_id in self.rid_to_queue:
136 + if self.output_processor.is_request_active(request_id):
137 raise ValueError(f"Request id {request_id} already running.")
145 - self.rid_to_queue[request_id] = asyncio.Queue()
138 + queue: asyncio.Queue[RequestOutput] = asyncio.Queue()
139
140 # 2) Convert Input --> Request.
141 request = self.processor.process_inputs(request_id, prompt, params,
@@ -151,8 +144,8 @@ class AsyncLLM(EngineClient):
144 prompt_adapter_request,
145 priority)
146
154 - # 3) Add the request to Detokenizer (this process).
155 - self.detokenizer.add_request(request)
147 + # 3) Add the request to OutputProcessor (this process).
148 + self.output_processor.add_request(request, queue)
149
150 # 4) Add the EngineCoreRequest to EngineCore (separate process).
151 await self.engine_core.add_request_async(request)
@@ -160,7 +153,7 @@ class AsyncLLM(EngineClient):
153 if self.log_requests:
154 logger.info("Added request %s.", request_id)
155
163 - return self.rid_to_queue[request_id]
156 + return queue
157
158 # TODO: we should support multiple prompts in one call, as you
159 # can do with LLM.generate. So that for multi-prompt completion
@@ -217,10 +210,9 @@ class AsyncLLM(EngineClient):
210 # task switching under load which helps performance).
211 out = q.get_nowait() if q.qsize() > 0 else await q.get()
212
220 - # Note: both Detokenizer and EngineCore handle their
213 + # Note: both OutputProcessor and EngineCore handle their
214 # own request cleanup based on finished.
215 if out.finished:
223 - del self.rid_to_queue[request_id]
216 yield out
217 break
218
@@ -233,57 +225,51 @@ class AsyncLLM(EngineClient):
225 await self.abort(request_id)
226 raise
227
236 - def _process_request_outputs(self, request_outputs: List[RequestOutput]):
237 - """Process outputs by putting them into per-request queues."""
238 -
239 - for request_output in request_outputs:
240 - request_id = request_output.request_id
241 -
242 - # Note: it is possible a request was aborted and removed from
243 - # the state due to client cancellations, so if we encounter a
244 - # request id not in the state, we skip.
245 - if request_id in self.rid_to_queue:
246 - self.rid_to_queue[request_id].put_nowait(request_output)
247 -
228 async def _run_output_handler(self):
229 """Background loop: pulls from EngineCore and pushes to AsyncStreams."""
230
231 try:
232 while True:
253 - # 1) Pull EngineCoreOutput from the EngineCore.
233 + # 1) Pull EngineCoreOutputs from the EngineCore.
234 outputs = await self.engine_core.get_output_async()
235
256 - # 2) Detokenize based on the output.
257 - request_outputs, reqs_to_abort = self.detokenizer.step(
236 + # 2) Process EngineCoreOutputs.
237 + processed_outputs = self.output_processor.process_outputs(
238 outputs.outputs)
239 + # NOTE: RequestOutputs are pushed to their queues.
240 + assert len(processed_outputs.request_outputs) == 0
241
260 - # 3) Put the RequestOutputs into the per-request queues.
261 - self._process_request_outputs(request_outputs)
242 + # 3) Abort any reqs that finished due to stop strings.
243 + await self.engine_core.abort_requests_async(
244 + processed_outputs.reqs_to_abort)
245
263 - # 4) Abort any requests that finished due to stop strings.
264 - await self.engine_core.abort_requests_async(reqs_to_abort)
265 -
266 - # 5) Log any stats.
267 - await self._log_stats(scheduler_stats=outputs.scheduler_stats)
246 + # 4) Logging.
247 + # TODO(rob): make into a coroutine and launch it in
248 + # background thread once we add Prometheus.
249 + self._log_stats(
250 + scheduler_stats=outputs.scheduler_stats,
251 + iteration_stats=processed_outputs.iteration_stats,
252 + )
253
254 except Exception as e:
255 logger.exception("EngineCore output handler hit an error: %s", e)
256 kill_process_tree(os.getpid())
257
258 async def abort(self, request_id: str) -> None:
274 - """Abort RequestId in self, detokenizer, and engine core."""
259 + """Abort RequestId in OutputProcessor and EngineCore."""
260
261 request_ids = [request_id]
262 await self.engine_core.abort_requests_async(request_ids)
278 - self.detokenizer.abort_requests(request_ids)
263 + self.output_processor.abort_requests(request_ids)
264
280 - # If a request finishes while we await then the request_id
281 - # will be removed from the tracked queues before we get here.
282 - if request_id in self.rid_to_queue:
283 - del self.rid_to_queue[request_id]
265 + if self.log_requests:
266 + logger.info("Aborted request %s.", request_id)
267
285 - async def _log_stats(self, scheduler_stats: SchedulerStats):
286 - """Log stats to the stat loggers."""
268 + def _log_stats(
269 + self,
270 + scheduler_stats: SchedulerStats,
271 + iteration_stats: IterationStats,
272 + ):
273 if not self.log_stats:
274 return
275
@@ -314,8 +300,7 @@ class AsyncLLM(EngineClient):
300 self,
301 lora_request: Optional[LoRARequest] = None,
302 ) -> AnyTokenizer:
317 - assert lora_request is None
318 - return self.detokenizer.tokenizer
303 + return self.tokenizer.get_lora_tokenizer(lora_request)
304
305 async def is_tracing_enabled(self) -> bool:
306 return False
vllm/v1/engine/core_client.py
+4 -2
@@ -105,7 +105,8 @@ class InprocClient(EngineCoreClient):
105 self.engine_core.add_request(request)
106
107 def abort_requests(self, request_ids: List[str]) -> None:
108 - self.engine_core.abort_requests(request_ids)
108 + if len(request_ids) > 0:
109 + self.engine_core.abort_requests(request_ids)
110
111 def shutdown(self):
112 self.engine_core.shutdown()
@@ -221,7 +222,8 @@ class SyncMPClient(MPClient):
222 self._send_input(EngineCoreRequestType.ADD, request)
223
224 def abort_requests(self, request_ids: List[str]) -> None:
224 - self._send_input(EngineCoreRequestType.ABORT, request_ids)
225 + if len(request_ids) > 0:
226 + self._send_input(EngineCoreRequestType.ABORT, request_ids)
227
228 def profile(self, is_start: bool = True) -> None:
229 self._send_input(EngineCoreRequestType.PROFILE,
vllm/v1/engine/detokenizer.py
+22 -115
@@ -1,18 +1,25 @@
1 from dataclasses import dataclass
2 -from typing import Dict, Iterable, List, Optional, Tuple, Union
2 +from typing import List, Optional, Union
3
4 from vllm.engine.output_processor.stop_checker import StopChecker
5 from vllm.logger import init_logger
6 -from vllm.outputs import RequestOutput
6 from vllm.sampling_params import RequestOutputKind
7 from vllm.transformers_utils.detokenizer_utils import (
8 AnyTokenizer, convert_prompt_ids_to_tokens, detokenize_incrementally)
10 -from vllm.transformers_utils.tokenizer import get_tokenizer
9 from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
10
11 logger = init_logger(__name__)
12
13
14 +@dataclass
15 +class DetokenizerOutput:
16 + output_text: str
17 + token_ids: List[int]
18 + finished: bool
19 + finish_reason: Optional[str] = None
20 + stop_reason: Union[int, str, None] = None
21 +
22 +
23 @dataclass
24 class IncrementalDetokenizer:
25
@@ -20,6 +27,7 @@ class IncrementalDetokenizer:
27 output_text: str
28 tokens: List[str]
29 token_ids: List[int]
30 + prompt_len: int
31
32 # Stop strings
33 stop: List[str]
@@ -34,11 +42,6 @@ class IncrementalDetokenizer:
42 spaces_between_special_tokens: bool
43 output_kind: RequestOutputKind
44
37 - # TODO: Probably decouple these
38 - request_id: str
39 - prompt: Optional[str]
40 - prompt_token_ids: List[int]
41 -
45 # Tokenizer for this request
46 tokenizer: AnyTokenizer
47
@@ -48,8 +51,7 @@ class IncrementalDetokenizer:
51
52 @property
53 def output_token_ids(self) -> List[int]:
51 - assert len(self.token_ids) >= len(self.prompt_token_ids)
52 - return self.token_ids[len(self.prompt_token_ids):]
54 + return self.token_ids[self.prompt_len:]
55
56 @classmethod
57 def from_new_request(
@@ -87,25 +89,25 @@ class IncrementalDetokenizer:
89 spaces_between_special_tokens=request.sampling_params.
90 spaces_between_special_tokens,
91 output_kind=request.sampling_params.output_kind,
90 - request_id=request.request_id,
91 - prompt=request.prompt,
92 - prompt_token_ids=request.prompt_token_ids,
92 + prompt_len=len(request.prompt_token_ids),
93 tokenizer=tokenizer,
94 stop_buffer_length=stop_buffer_length,
95 )
96
97 - def add_tokens(
97 + def update_from_output(
98 self,
99 - new_token_ids: List[int],
100 - finish_reason: Optional[str],
101 - stop_reason: Optional[Union[int, str, None]],
102 - ) -> Optional[RequestOutput]:
99 + output: EngineCoreOutput,
100 + ) -> Optional[DetokenizerOutput]:
101 """
102 Update RequestState for the request_id by:
103 1) Detokenize the new token ids incrementally.
104 2) Update the RequestOutput with the new text.
105 """
106
107 + new_token_ids = output.new_token_ids
108 + finish_reason = output.finish_reason
109 + stop_reason = output.stop_reason
110 +
111 # 1) Detokenize the new token ids incrementally.
112 # TODO(woosuk): This method becomes very inefficient when the number of
113 # new_token_ids is more than 1. We need to optimize this.
@@ -158,21 +160,8 @@ class IncrementalDetokenizer:
160 output_text = self._get_next_output_text(finished, delta)
161 token_ids = new_token_ids if delta else self.output_token_ids
162
161 - request_output = RequestOutput.new(
162 - self.request_id,
163 - self.prompt,
164 - self.prompt_token_ids,
165 - output_text,
166 - token_ids,
167 - finished,
168 - )
169 -
170 - if finished:
171 - completion_output = request_output.outputs[0]
172 - completion_output.finish_reason = finish_reason
173 - completion_output.stop_reason = stop_reason
174 -
175 - return request_output
163 + return DetokenizerOutput(output_text, token_ids, finished,
164 + finish_reason, stop_reason)
165
166 def _get_next_output_text(self, finished: bool, delta: bool) -> str:
167 """If delta is True, only new text since the last call to
@@ -189,85 +178,3 @@ class IncrementalDetokenizer:
178 self._last_output_text_offset = length
179 return self.output_text[last_offset:length]
180 return ""
192 -
193 -
194 -class Detokenizer:
195 -
196 - def __init__(self,
197 - tokenizer_name: str,
198 - tokenizer_mode: str = "auto",
199 - trust_remote_code: bool = False,
200 - revision: Optional[str] = None):
201 - # TODO: once we support LoRA, we should should pass the tokenizer
202 - # here. We currently have two copies (this + in the LLMEngine).
203 - self.tokenizer = get_tokenizer(tokenizer_name=tokenizer_name,
204 - tokenizer_mode=tokenizer_mode,
205 - trust_remote_code=trust_remote_code,
206 - revision=revision)
207 -
208 - # Request id -> IncrementalDetokenizer
209 - self.request_states: Dict[str, IncrementalDetokenizer] = {}
210 -
211 - def is_request_active(self, request_id: str):
212 - return request_id in self.request_states
213 -
214 - def get_num_unfinished_requests(self):
215 - return len(self.request_states)
216 -
217 - def has_unfinished_requests(self) -> bool:
218 - return len(self.request_states) > 0
219 -
220 - def abort_requests(
221 - self,
222 - request_ids: Iterable[str],
223 - ) -> None:
224 - """Remove the request_ids from the Detokenizer."""
225 -
226 - for request_id in request_ids:
227 - self.request_states.pop(request_id, None)
228 -
229 - def add_request(
230 - self,
231 - request: EngineCoreRequest,
232 - ):
233 - """Add new request to the Detokenizer."""
234 -
235 - assert (request.request_id not in self.request_states)
236 -
237 - request_state = IncrementalDetokenizer.from_new_request(
238 - self.tokenizer, request)
239 - self.request_states[request.request_id] = request_state
240 -
241 - def step(
242 - self, encore_core_outputs: List[EngineCoreOutput]
243 - ) -> Tuple[List[RequestOutput], List[str]]:
244 - """Update state and request the RequestOutputs to the LLMEngine."""
245 -
246 - request_outputs: List[RequestOutput] = []
247 - requests_to_abort: List[str] = []
248 - for engine_core_output in encore_core_outputs:
249 - request_id = engine_core_output.request_id
250 - detokenizer = self.request_states.get(request_id)
251 - if detokenizer is None:
252 - # Ignore output for already-aborted request.
253 - continue
254 -
255 - # Detokenize and update state.
256 - request_output = detokenizer.add_tokens(
257 - new_token_ids=engine_core_output.new_token_ids,
258 - finish_reason=engine_core_output.finish_reason,
259 - stop_reason=engine_core_output.stop_reason,
260 - )
261 -
262 - if request_output is not None:
263 - # Add to RequestOutputs list.
264 - request_outputs.append(request_output)
265 -
266 - # Free completed requests.
267 - if request_output.finished:
268 - self.request_states.pop(request_id)
269 - if not engine_core_output.finished:
270 - requests_to_abort.append(request_id)
271 -
272 - # Return to EngineClient.
273 - return request_outputs, requests_to_abort
vllm/v1/engine/llm_engine.py
+14 -19
@@ -18,7 +18,7 @@ from vllm.transformers_utils.tokenizer_group import (
18 BaseTokenizerGroup, init_tokenizer_from_configs)
19 from vllm.usage.usage_lib import UsageContext
20 from vllm.v1.engine.core_client import EngineCoreClient
21 -from vllm.v1.engine.detokenizer import Detokenizer
21 +from vllm.v1.engine.output_processor import OutputProcessor
22 from vllm.v1.engine.processor import Processor
23 from vllm.v1.executor.abstract import Executor
24
@@ -60,13 +60,9 @@ class LLMEngine:
60 input_registry=input_registry,
61 mm_registry=mm_registry)
62
63 - # Detokenizer (converts EngineCoreOutputs --> RequestOutput)
64 - self.detokenizer = Detokenizer(
65 - tokenizer_name=vllm_config.model_config.tokenizer,
66 - tokenizer_mode=vllm_config.model_config.tokenizer_mode,
67 - trust_remote_code=vllm_config.model_config.trust_remote_code,
68 - revision=vllm_config.model_config.tokenizer_revision,
69 - )
63 + # OutputProcessor (convert EngineCoreOutputs --> RequestOutput).
64 + self.output_processor = OutputProcessor(self.tokenizer,
65 + log_stats=False)
66
67 # EngineCore (gets EngineCoreRequests and gives EngineCoreOutputs)
68 self.engine_core = EngineCoreClient.make_client(
@@ -103,10 +99,10 @@ class LLMEngine:
99 multiprocess_mode=enable_multiprocessing)
100
101 def get_num_unfinished_requests(self) -> int:
106 - return self.detokenizer.get_num_unfinished_requests()
102 + return self.output_processor.get_num_unfinished_requests()
103
104 def has_unfinished_requests(self) -> bool:
109 - return self.detokenizer.has_unfinished_requests()
105 + return self.output_processor.has_unfinished_requests()
106
107 @classmethod
108 def validate_outputs(cls, outputs, output_type):
@@ -116,7 +112,7 @@ class LLMEngine:
112 """Remove request_ids from EngineCore and Detokenizer."""
113
114 self.engine_core.abort_requests(request_ids)
119 - self.detokenizer.abort_requests(request_ids)
115 + self.output_processor.abort_requests(request_ids)
116
117 def add_request(
118 self,
@@ -137,8 +133,8 @@ class LLMEngine:
133 prompt_adapter_request,
134 priority)
135
140 - # 2) Add the request to Detokenizer.
141 - self.detokenizer.add_request(request)
136 + # 2) Make a new RequestState and queue.
137 + self.output_processor.add_request(request)
138
139 # 3) Add the request to EngineCore.
140 self.engine_core.add_request(request)
@@ -148,15 +144,14 @@ class LLMEngine:
144 # 1) Get EngineCoreOutput from the EngineCore.
145 outputs = self.engine_core.get_output()
146
151 - # 2) Detokenizer the EngineCoreOutput.
152 - request_outputs, requests_to_abort = self.detokenizer.step(
147 + # 2) Process EngineCoreOutputs.
148 + processed_outputs = self.output_processor.process_outputs(
149 outputs.outputs)
150
155 - # 3) Abort requests that finished due to stopping criteria.
156 - if requests_to_abort:
157 - self.abort_request(requests_to_abort)
151 + # 3) Abort any reqs that finished due to stop strings.
152 + self.engine_core.abort_requests(processed_outputs.reqs_to_abort)
153
159 - return request_outputs
154 + return processed_outputs.request_outputs
155
156 def get_model_config(self):
157 return self.model_config
vllm/v1/engine/output_processor.py new
+200
@@ -0,0 +1,200 @@
1 +import asyncio
2 +from dataclasses import dataclass
3 +from typing import Dict, List, Optional
4 +
5 +from vllm.outputs import RequestOutput
6 +from vllm.transformers_utils.detokenizer_utils import AnyTokenizer
7 +from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
8 +from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
9 +from vllm.v1.engine.detokenizer import (DetokenizerOutput,
10 + IncrementalDetokenizer)
11 +from vllm.v1.metrics.stats import IterationStats
12 +
13 +
14 +@dataclass
15 +class OutputProcessorOutput:
16 +
17 + request_outputs: List[RequestOutput]
18 + reqs_to_abort: List[str]
19 + iteration_stats: IterationStats
20 +
21 +
22 +class RequestState:
23 +
24 + def __init__(
25 + self,
26 + request_id: str,
27 + prompt: Optional[str],
28 + prompt_token_ids: List[int],
29 + detokenizer: IncrementalDetokenizer,
30 + queue: Optional[asyncio.Queue[RequestOutput]],
31 + ):
32 + self.request_id = request_id
33 + self.prompt = prompt
34 + self.prompt_token_ids = prompt_token_ids
35 + self.prompt_len = len(prompt_token_ids)
36 + self.detokenizer = detokenizer
37 + self.is_prefilling = True
38 + self.queue = queue
39 +
40 + @classmethod
41 + def from_new_request(
42 + cls,
43 + tokenizer: AnyTokenizer,
44 + request: EngineCoreRequest,
45 + queue: Optional[asyncio.Queue[RequestOutput]] = None,
46 + ) -> "RequestState":
47 + return cls(
48 + request_id=request.request_id,
49 + prompt=request.prompt,
50 + prompt_token_ids=request.prompt_token_ids,
51 + detokenizer=IncrementalDetokenizer.from_new_request(
52 + tokenizer=tokenizer,
53 + request=request,
54 + ),
55 + queue=queue,
56 + )
57 +
58 +
59 +class OutputProcessor:
60 + """Process EngineCoreOutputs into RequestOutputs."""
61 +
62 + def __init__(
63 + self,
64 + tokenizer: BaseTokenizerGroup,
65 + log_stats: bool,
66 + ):
67 + self.log_stats = log_stats
68 + self.tokenizer = tokenizer
69 + self.request_states: Dict[str, RequestState] = {}
70 +
71 + def is_request_active(self, request_id: str) -> bool:
72 + return request_id in self.request_states
73 +
74 + def get_num_unfinished_requests(self):
75 + return len(self.request_states)
76 +
77 + def has_unfinished_requests(self) -> bool:
78 + return len(self.request_states) > 0
79 +
80 + def abort_requests(
81 + self,
82 + request_ids: List[str],
83 + ) -> None:
84 + for request_id in request_ids:
85 + self.request_states.pop(request_id, None)
86 +
87 + def add_request(
88 + self,
89 + request: EngineCoreRequest,
90 + queue: Optional[asyncio.Queue[RequestOutput]] = None,
91 + ) -> None:
92 + request_id = request.request_id
93 + if request_id in self.request_states:
94 + raise ValueError(f"Request id {request_id} already running.")
95 +
96 + self.request_states[request_id] = RequestState.from_new_request(
97 + tokenizer=self.tokenizer.get_lora_tokenizer(request.lora_request),
98 + request=request,
99 + queue=queue)
100 +
101 + def process_outputs(
102 + self,
103 + engine_core_outputs: List[EngineCoreOutput],
104 + ) -> OutputProcessorOutput:
105 + """
106 + Process the EngineCoreOutputs:
107 + 1) Compute stats for logging
108 + 2) Detokenize
109 + 3) Create and handle RequestOutput objects:
110 + * If there is a queue (for usage with AsyncLLM),
111 + put the RequestOutput objects into the queue for
112 + handling by the per-request generate() tasks.
113 +
114 + * If there is no queue (for usage with LLMEngine),
115 + return a list of RequestOutput objects.
116 +
117 + ****************** NOTE FOR DEVELOPERS ******************
118 +
119 + VLLM V1 minimizes the number of python loops over the full
120 + batch to ensure system overheads are minimized. This is the
121 + only function that should loop over EngineCoreOutputs.
122 +
123 + If you need to touch every element of the batch, implement a
124 + method called XXXClass.update_from_output() to be called
125 + within the loop below. For examples, see:
126 + * IterationStats.update_from_output()
127 + * Detokenizer.update_from_output()
128 +
129 + TODO(rob): add Protocol makes update_from_output explicit.
130 +
131 + **********************************************************
132 + """
133 +
134 + request_outputs: List[RequestOutput] = []
135 + reqs_to_abort: List[str] = []
136 + iteration_stats = IterationStats(self.log_stats)
137 + for engine_core_output in engine_core_outputs:
138 + req_id = engine_core_output.request_id
139 + req_state = self.request_states.get(req_id)
140 + if req_state is None:
141 + # Ignore output for already-aborted request.
142 + continue
143 +
144 + # 1) Compute stats for this iteration.
145 + iteration_stats.update_from_output(engine_core_output,
146 + req_state.is_prefilling,
147 + req_state.prompt_len)
148 + req_state.is_prefilling = False
149 +
150 + # 2) Detokenize the token ids into text.
151 + detokenizer_output = req_state.detokenizer.update_from_output(
152 + engine_core_output)
153 +
154 + # 3) Create and handle RequestOutput objects.
155 + if request_output := self._make_request_output(
156 + req_state, detokenizer_output):
157 + if req_state.queue is not None:
158 + # AsyncLLM: put into queue for handling by generate().
159 + req_state.queue.put_nowait(request_output)
160 + else:
161 + # LLMEngine: return list of RequestOutputs.
162 + request_outputs.append(request_output)
163 +
164 + # Free completed requests.
165 + if request_output.finished:
166 + self.request_states.pop(req_id)
167 + if not engine_core_output.finished:
168 + # If req not finished in EngineCore, but Detokenizer
169 + # detected stop string, abort needed in EngineCore.
170 + reqs_to_abort.append(req_id)
171 +
172 + return OutputProcessorOutput(
173 + request_outputs=request_outputs,
174 + reqs_to_abort=reqs_to_abort,
175 + iteration_stats=iteration_stats,
176 + )
177 +
178 + def _make_request_output(
179 + self,
180 + request_state: RequestState,
181 + detokenizer_output: Optional[DetokenizerOutput],
182 + ) -> Optional[RequestOutput]:
183 +
184 + if detokenizer_output is None:
185 + return None
186 +
187 + request_output = RequestOutput.new(
188 + request_state.request_id,
189 + request_state.prompt,
190 + request_state.prompt_token_ids,
191 + detokenizer_output.output_text,
192 + detokenizer_output.token_ids,
193 + detokenizer_output.finished,
194 + )
195 + if detokenizer_output.finished:
196 + completion_output = request_output.outputs[0]
197 + completion_output.finish_reason = detokenizer_output.finish_reason
198 + completion_output.stop_reason = detokenizer_output.stop_reason
199 +
200 + return request_output
vllm/v1/metrics/stats.py
+27
@@ -1,4 +1,8 @@
1 from dataclasses import dataclass
2 +from typing import TYPE_CHECKING
3 +
4 +if TYPE_CHECKING:
5 + from vllm.v1.engine import EngineCoreOutput
6
7
8 @dataclass
@@ -10,3 +14,26 @@ class SchedulerStats:
14
15 # gpu_cache_usage: float = 0.0
16 # gpu_prefix_cache_hit_rate: float = 0.0
17 +
18 +
19 +class IterationStats:
20 + """Stats associated with a single set of EngineCoreOutputs."""
21 +
22 + def __init__(self, log_stats: bool):
23 + self.log_stats = log_stats
24 + self.num_generation_tokens = 0
25 + self.num_prompt_tokens = 0
26 +
27 + def update_from_output(self, output: "EngineCoreOutput",
28 + is_prefilling: bool, prompt_len: int):
29 + if not self.log_stats:
30 + return
31 +
32 + self.num_generation_tokens += len(output.new_token_ids)
33 + if is_prefilling:
34 + # This relies on the invariant that EngineCore does
35 + # not stream outputs for partially completed prefills
36 + # (scheduler.update_from_output makes EngineCoreOutput
37 + # iff num_computed_tokens == num_tokens).
38 + assert (len(output.new_token_ids) > 0)
39 + self.num_prompt_tokens += prompt_len