@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 551ce01078
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Core] Add engine option to return only deltas or final output (#7381)

Nick Hill committed Sep 12, 2024 at 20:02 UTC 551ce01078a655068e5ec3764d0a55ac744ea425
10 files changed +371 -137
.buildkite/test-pipeline.yaml
+1
@@ -50,6 +50,7 @@ steps:
50 - tests/worker
51 commands:
52 - pytest -v -s async_engine # Async Engine
53 + - NUM_SCHEDULER_STEPS=4 pytest -v -s async_engine/test_async_llm_engine.py
54 - pytest -v -s test_inputs.py
55 - pytest -v -s multimodal
56 - pytest -v -s test_utils.py # Utils
tests/async_engine/test_async_llm_engine.py
+147 -14
@@ -1,7 +1,10 @@
1 import asyncio
2 +import os
3 +import uuid
4 from asyncio import CancelledError
5 +from copy import copy
6 from dataclasses import dataclass
4 -from typing import Optional
7 +from typing import List, Optional
8
9 import pytest
10 import pytest_asyncio
@@ -11,6 +14,7 @@ from vllm import SamplingParams
14 from vllm.config import ParallelConfig
15 from vllm.engine.async_llm_engine import AsyncEngineArgs, AsyncLLMEngine
16 from vllm.outputs import RequestOutput as RealRequestOutput
17 +from vllm.sampling_params import RequestOutputKind
18
19 from ..conftest import cleanup
20 from ..utils import wait_for_gpu_memory_to_clear
@@ -122,8 +126,17 @@ def start_engine():
126 timeout_s=60,
127 )
128
129 + num_scheduler_steps = int(os.getenv("NUM_SCHEDULER_STEPS", "1"))
130 + print(f"Starting engine with num_scheduler_steps={num_scheduler_steps}")
131 +
132 return AsyncLLMEngine.from_engine_args(
126 - AsyncEngineArgs(model="facebook/opt-125m", enforce_eager=True))
133 + AsyncEngineArgs(model="facebook/opt-125m",
134 + enforce_eager=True,
135 + num_scheduler_steps=num_scheduler_steps))
136 +
137 +
138 +def uid() -> str:
139 + return str(uuid.uuid4())
140
141
142 @pytest_asyncio.fixture(scope="module")
@@ -148,57 +161,177 @@ def should_do_global_cleanup_after_test(request) -> bool:
161 @pytest.mark.asyncio(scope="module")
162 async def test_asyncio_run(async_engine):
163
164 + scheduler_config = await async_engine.get_scheduler_config()
165 + num_scheduler_steps = scheduler_config.num_scheduler_steps
166 +
167 async def run(prompt: str):
168 sampling_params = SamplingParams(
169 temperature=0,
170 max_tokens=32,
171 + min_tokens=32,
172 )
173
174 + output_count = 0
175 + final_output = None
176 async for output in async_engine.generate(prompt,
177 sampling_params,
159 - request_id=prompt):
178 + request_id=uid()):
179 + output_count += 1
180 final_output = output
161 - return final_output
181 + return final_output, output_count
182
183 results = await asyncio.gather(
184 run("test0"),
165 - run("test1"),
185 + run("test0"),
186 )
187 assert len(results) == 2
188 + first, second = results
189 +
190 + # remove nondeterministic fields for comparison
191 + first[0].metrics = None
192 + second[0].metrics = None
193 + first[0].request_id = None
194 + second[0].request_id = None
195 +
196 + assert str(first) == str(second)
197 +
198 + output_count = results[0][1]
199 + if num_scheduler_steps == 1:
200 + assert output_count == 32
201 + else:
202 + assert 1 < output_count < 32
203 +
204 +
205 +@pytest.mark.asyncio(scope="module")
206 +async def test_output_kinds(async_engine):
207 + """Test that output_kind works as expected and that
208 + results are equivalent across different kinds."""
209 +
210 + scheduler_config = await async_engine.get_scheduler_config()
211 + num_scheduler_steps = scheduler_config.num_scheduler_steps
212 +
213 + sampling_params = SamplingParams(
214 + temperature=0,
215 + max_tokens=32,
216 + min_tokens=32,
217 + )
218 +
219 + async def run(prompt: str, kind: RequestOutputKind):
220 + params = copy(sampling_params)
221 + params.output_kind = kind
222 +
223 + output_count = 0
224 + final_output = None
225 + async for output in async_engine.generate(prompt,
226 + params,
227 + request_id=uid()):
228 + output_count += 1
229 + final_output = output
230 +
231 + assert final_output is not None
232 + return (final_output.prompt_token_ids,
233 + final_output.outputs[0].token_ids,
234 + final_output.outputs[0].text, output_count)
235 +
236 + async def run_deltas(prompt: str):
237 + params = copy(sampling_params)
238 + params.output_kind = RequestOutputKind.DELTA
239 +
240 + prompt_tokens = None
241 + output_tokens: List[int] = []
242 + output_text = ""
243 + output_count = 0
244 + async for output in async_engine.generate(prompt,
245 + params,
246 + request_id=uid()):
247 + token_ids = output.outputs[0].token_ids
248 + text = output.outputs[0].text
249 +
250 + # Ensure we get prompt ids iff we haven't yet received output tokens
251 + if output_tokens:
252 + assert 1 <= len(token_ids) <= num_scheduler_steps
253 + assert text
254 + assert not output.prompt_token_ids
255 + else:
256 + assert output.prompt_token_ids
257 + prompt_tokens = output.prompt_token_ids
258 +
259 + output_tokens.extend(token_ids)
260 + output_text += text
261 +
262 + output_count += 1
263 + return prompt_tokens, output_tokens, output_text, output_count
264 +
265 + results = await asyncio.gather(
266 + run("common input prompt", RequestOutputKind.CUMULATIVE),
267 + run("common input prompt", RequestOutputKind.FINAL_ONLY),
268 + run_deltas("common input prompt"))
269 +
270 + # Make sure outputs are the same
271 + prompt_set = set(tuple(prompt_ids) for prompt_ids, _, _, _ in results)
272 + assert len(prompt_set) == 1
273 +
274 + text_set = set(text for _, _, text, _ in results)
275 + assert len(text_set) == 1
276 +
277 + tokens_set = set(tuple(ids) for _, ids, _, _ in results)
278 + assert len(tokens_set) == 1
279 +
280 + cumulative, final, deltas = results
281 +
282 + # output message counts
283 + assert cumulative[3] == deltas[3]
284 +
285 + if num_scheduler_steps == 1:
286 + assert cumulative[3] == 32
287 + else:
288 + assert 1 < cumulative[3] < 32
289 +
290 + assert final[3] == 1
291
292
293 @pytest.mark.asyncio(scope="module")
294 async def test_cancellation(async_engine):
295 + scheduler_config = await async_engine.get_scheduler_config()
296 + num_scheduler_steps = scheduler_config.num_scheduler_steps
297 +
298 sampling_params = SamplingParams(
299 temperature=0,
174 - min_tokens=10,
175 - max_tokens=10,
300 + min_tokens=13,
301 + max_tokens=13,
302 )
303
304 + stop_at = 5 if num_scheduler_steps == 1 else 1
305 +
306 + request_id = uid()
307 +
308 i = 0
309 with pytest.raises(CancelledError):
310 async for output in async_engine.generate("test2",
311 sampling_params,
182 - request_id="test2"):
312 + request_id=request_id):
313 assert not output.finished
314 i += 1
185 - if i == 5:
186 - await async_engine.abort("test2")
315 + if i == stop_at:
316 + await async_engine.abort(request_id)
317
188 - assert i == 5
318 + assert i == stop_at
319
320
321 @pytest.mark.asyncio(scope="module")
322 async def test_delayed_generator(async_engine):
323 + scheduler_config = await async_engine.get_scheduler_config()
324 +
325 + if scheduler_config.num_scheduler_steps != 1:
326 + pytest.skip("no need to test this one with multistep")
327 +
328 sampling_params = SamplingParams(
329 temperature=0,
330 min_tokens=10,
331 max_tokens=10,
332 )
333
199 - stream = async_engine.generate("test3",
200 - sampling_params,
201 - request_id="test3")
334 + stream = async_engine.generate("test3", sampling_params, request_id=uid())
335 i = 0
336 final_output: Optional[RealRequestOutput] = None
337 async for output in stream:
vllm/engine/llm_engine.py
+13 -11
@@ -39,7 +39,7 @@ from vllm.outputs import (EmbeddingRequestOutput, RequestOutput,
39 RequestOutputFactory)
40 from vllm.pooling_params import PoolingParams
41 from vllm.prompt_adapter.request import PromptAdapterRequest
42 -from vllm.sampling_params import SamplingParams
42 +from vllm.sampling_params import RequestOutputKind, SamplingParams
43 from vllm.sequence import (EmbeddingSequenceGroupOutput, ExecuteModelRequest,
44 Sequence, SequenceGroup, SequenceGroupMetadata,
45 SequenceStatus)
@@ -225,9 +225,6 @@ class LLMEngine:
225 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
226 stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
227 input_registry: InputRegistry = INPUT_REGISTRY,
228 - # To improve performance, only final requests outputs may be required.
229 - # If this set to true, then no intermediate outputs will be returned.
230 - step_return_finished_only: bool = False,
228 ) -> None:
229 logger.info(
230 "Initializing an LLM engine (v%s) with config: "
@@ -295,7 +292,6 @@ class LLMEngine:
292 self.observability_config = observability_config or ObservabilityConfig(
293 )
294 self.log_stats = log_stats
298 - self.step_return_finished_only = step_return_finished_only
295
296 if not self.model_config.skip_tokenizer_init:
297 self.tokenizer = self._init_tokenizer()
@@ -1273,7 +1269,7 @@ class LLMEngine:
1269
1270 ctx: The virtual engine context to work on
1271 request_id: If provided, then only this request is going to be processed
1276 -
1272 +
1273 """
1274 now = time.time()
1275
@@ -1378,7 +1374,8 @@ class LLMEngine:
1374 seq_group = scheduled_seq_group.seq_group
1375 seq_group.maybe_set_first_token_time(now)
1376 request_output = RequestOutputFactory.create(seq_group)
1381 - ctx.request_outputs.append(request_output)
1377 + if request_output:
1378 + ctx.request_outputs.append(request_output)
1379
1380 # When we process a single request, we skip it for the next time,
1381 # and invoke the request output callback (if there was final output)
@@ -1415,14 +1412,19 @@ class LLMEngine:
1412
1413 seq_group = scheduled_seq_group.seq_group
1414 seq_group.maybe_set_first_token_time(now)
1418 - if (seq_group.is_finished()
1419 - if self.step_return_finished_only else True):
1420 - request_output = RequestOutputFactory.create(seq_group)
1415 + request_output = RequestOutputFactory.create(seq_group)
1416 + if request_output:
1417 ctx.request_outputs.append(request_output)
1418
1419 for seq_group in scheduler_outputs.ignored_seq_groups:
1420 + params = seq_group.sampling_params
1421 + if params is not None and params.output_kind == (
1422 + RequestOutputKind.DELTA) and not seq_group.is_finished():
1423 + continue
1424 +
1425 request_output = RequestOutputFactory.create(seq_group)
1425 - ctx.request_outputs.append(request_output)
1426 + if request_output:
1427 + ctx.request_outputs.append(request_output)
1428
1429 # Immediately process request outputs here (if callback is given)
1430 if (ctx.request_outputs
vllm/entrypoints/llm.py
+8 -15
@@ -19,7 +19,7 @@ from vllm.model_executor.guided_decoding.guided_fields import LLMGuidedOptions
19 from vllm.outputs import EmbeddingRequestOutput, RequestOutput
20 from vllm.pooling_params import PoolingParams
21 from vllm.prompt_adapter.request import PromptAdapterRequest
22 -from vllm.sampling_params import SamplingParams
22 +from vllm.sampling_params import RequestOutputKind, SamplingParams
23 from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
24 get_cached_tokenizer)
25 from vllm.transformers_utils.tokenizer_group import TokenizerGroup
@@ -642,14 +642,12 @@ class LLM:
642 raise ValueError("The lengths of prompts and lora_request "
643 "must be the same.")
644
645 - if isinstance(params, list):
646 - params = [
647 - self._add_guided_processor(param, guided_options)
648 - if isinstance(param, SamplingParams) else param
649 - for param in params
650 - ]
651 - elif isinstance(params, SamplingParams):
652 - params = self._add_guided_processor(params, guided_options)
645 + for sp in params if isinstance(params, list) else (params, ):
646 + if isinstance(sp, SamplingParams):
647 + self._add_guided_processor(sp, guided_options)
648 +
649 + # We only care about the final output
650 + sp.output_kind = RequestOutputKind.FINAL_ONLY
651
652 # Add requests to the engine.
653 for i, request_inputs in enumerate(inputs):
@@ -709,9 +707,6 @@ class LLM:
707 f"output: {0:.2f} toks/s"),
708 )
709
712 - # In the loop below, only finished outputs are used
713 - self.llm_engine.step_return_finished_only = True
714 -
710 # Run the engine.
711 outputs: List[Union[RequestOutput, EmbeddingRequestOutput]] = []
712 total_in_toks = 0
@@ -724,6 +719,7 @@ class LLM:
719 if use_tqdm:
720 if isinstance(output, RequestOutput):
721 # Calculate tokens only for RequestOutput
722 + assert output.prompt_token_ids is not None
723 total_in_toks += len(output.prompt_token_ids)
724 in_spd = total_in_toks / pbar.format_dict["elapsed"]
725 total_out_toks += sum(
@@ -735,9 +731,6 @@ class LLM:
731 f"output: {out_spd:.2f} toks/s")
732 pbar.update(1)
733
738 - # Restore original behavior
739 - self.llm_engine.step_return_finished_only = False
740 -
734 if use_tqdm:
735 pbar.close()
736 # Sort the outputs by request ID.
vllm/entrypoints/openai/protocol.py
+6 -1
@@ -12,7 +12,8 @@ from typing_extensions import Annotated, Required, TypedDict
12 from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
13 from vllm.entrypoints.openai.logits_processors import get_logits_processors
14 from vllm.pooling_params import PoolingParams
15 -from vllm.sampling_params import LogitsProcessor, SamplingParams
15 +from vllm.sampling_params import (LogitsProcessor, RequestOutputKind,
16 + SamplingParams)
17 from vllm.sequence import Logprob
18 from vllm.transformers_utils.tokenizer import AnyTokenizer
19 from vllm.utils import random_uuid
@@ -316,6 +317,8 @@ class ChatCompletionRequest(OpenAIBaseModel):
317 length_penalty=self.length_penalty,
318 logits_processors=logits_processors,
319 truncate_prompt_tokens=self.truncate_prompt_tokens,
320 + output_kind=RequestOutputKind.DELTA if self.stream \
321 + else RequestOutputKind.FINAL_ONLY,
322 )
323
324 @model_validator(mode="before")
@@ -559,6 +562,8 @@ class CompletionRequest(OpenAIBaseModel):
562 length_penalty=self.length_penalty,
563 logits_processors=logits_processors,
564 truncate_prompt_tokens=self.truncate_prompt_tokens,
565 + output_kind=RequestOutputKind.DELTA if self.stream \
566 + else RequestOutputKind.FINAL_ONLY,
567 )
568
569 @model_validator(mode="before")
vllm/entrypoints/openai/serving_chat.py
+72 -53
@@ -246,8 +246,7 @@ class OpenAIServingChat(OpenAIServing):
246 def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
247 if request.add_generation_prompt:
248 return self.response_role
249 - else:
250 - return request.messages[-1]["role"]
249 + return request.messages[-1]["role"]
250
251 async def chat_completion_stream_generator(
252 self,
@@ -264,15 +263,37 @@ class OpenAIServingChat(OpenAIServing):
263
264 # Send response for each token for each request.n (index)
265 num_choices = 1 if request.n is None else request.n
267 - previous_texts = [""] * num_choices
266 previous_num_tokens = [0] * num_choices
267 finish_reason_sent = [False] * num_choices
268
269 + num_prompt_tokens = 0
270 +
271 tool_parser: Optional[ToolParser] = self.tool_parser(
272 tokenizer) if self.tool_parser else None
273
274 + if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
275 + tool_choice_function_name = request.tool_choice.function.name
276 + else:
277 + tool_choice_function_name = None
278 +
279 + # Determine whether tools are in use with "auto" tool choice
280 + tool_choice_auto = (
281 + not tool_choice_function_name
282 + and self._should_stream_with_auto_tool_parsing(request))
283 +
284 + all_previous_token_ids: Optional[List[List[int]]]
285 + if tool_choice_auto:
286 + # These are only required in "auto" tool choice case
287 + previous_texts = [""] * num_choices
288 + all_previous_token_ids = [[]] * num_choices
289 + else:
290 + previous_texts, all_previous_token_ids = None, None
291 +
292 try:
293 async for res in result_generator:
294 + if res.prompt_token_ids is not None:
295 + num_prompt_tokens = len(res.prompt_token_ids)
296 +
297 # We need to do it here, because if there are exceptions in
298 # the result_generator, it needs to be sent as the FIRST
299 # response (by the try...catch).
@@ -305,10 +326,10 @@ class OpenAIServingChat(OpenAIServing):
326 and request.stream_options.include_usage):
327 # if continuous usage stats are requested, add it
328 if request.stream_options.continuous_usage_stats:
308 - prompt_tokens = len(res.prompt_token_ids)
309 - usage = UsageInfo(prompt_tokens=prompt_tokens,
310 - completion_tokens=0,
311 - total_tokens=prompt_tokens)
329 + usage = UsageInfo(
330 + prompt_tokens=num_prompt_tokens,
331 + completion_tokens=0,
332 + total_tokens=num_prompt_tokens)
333 chunk.usage = usage
334 # otherwise don't
335 else:
@@ -344,12 +365,10 @@ class OpenAIServingChat(OpenAIServing):
365 request.stream_options.include_usage):
366 if (request.stream_options.
367 continuous_usage_stats):
347 - prompt_tokens = len(
348 - res.prompt_token_ids)
368 usage = UsageInfo(
350 - prompt_tokens=prompt_tokens,
369 + prompt_tokens=num_prompt_tokens,
370 completion_tokens=0,
352 - total_tokens=prompt_tokens)
371 + total_tokens=num_prompt_tokens)
372 chunk.usage = usage
373 else:
374 chunk.usage = None
@@ -360,65 +379,66 @@ class OpenAIServingChat(OpenAIServing):
379 first_iteration = False
380
381 for output in res.outputs:
363 -
382 i = output.index
383
384 if finish_reason_sent[i]:
385 continue
386
369 - delta_token_ids = output.token_ids[previous_num_tokens[i]:]
370 - out_logprobs = output.logprobs[
371 - previous_num_tokens[i]:] if output.logprobs else None
372 -
387 if request.logprobs and request.top_logprobs is not None:
374 - assert out_logprobs is not None, (
388 + assert output.logprobs is not None, (
389 "Did not output logprobs")
390 logprobs = self._create_chat_logprobs(
377 - token_ids=delta_token_ids,
378 - top_logprobs=out_logprobs,
391 + token_ids=output.token_ids,
392 + top_logprobs=output.logprobs,
393 tokenizer=tokenizer,
394 num_output_top_logprobs=request.top_logprobs,
395 )
396 else:
397 logprobs = None
398
385 - delta_text = output.text[len(previous_texts[i]):]
386 - delta_message: Optional[DeltaMessage] = None
399 + delta_text = output.text
400 + delta_message: Optional[DeltaMessage]
401
402 # handle streaming deltas for tools with named tool_choice
389 - if (request.tool_choice and type(request.tool_choice) is
390 - ChatCompletionNamedToolChoiceParam):
403 + if tool_choice_function_name:
404 delta_message = DeltaMessage(tool_calls=[
405 DeltaToolCall(function=DeltaFunctionCall(
393 - name=request.tool_choice.function.name,
406 + name=tool_choice_function_name,
407 arguments=delta_text),
408 index=i)
409 ])
410
411 # handle streaming deltas for tools with "auto" tool choice
399 - elif (self._should_stream_with_auto_tool_parsing(request)
400 - and tool_parser):
412 + elif tool_choice_auto:
413 + assert previous_texts is not None
414 + assert all_previous_token_ids is not None
415 + assert tool_parser is not None
416 + #TODO optimize manipulation of these lists
417 + previous_text = previous_texts[i]
418 + previous_token_ids = all_previous_token_ids[i]
419 + current_text = previous_text + delta_text
420 + current_token_ids = previous_token_ids + list(
421 + output.token_ids)
422 +
423 delta_message = (
424 tool_parser.extract_tool_calls_streaming(
403 - previous_text=previous_texts[i],
404 - current_text=output.text,
425 + previous_text=previous_text,
426 + current_text=current_text,
427 delta_text=delta_text,
406 - previous_token_ids= \
407 - output.token_ids[
408 - :-1 * len(delta_token_ids)
409 - ],
410 - current_token_ids=output.token_ids,
411 - delta_token_ids=delta_token_ids
412 - )
413 - )
428 + previous_token_ids=previous_token_ids,
429 + current_token_ids=current_token_ids,
430 + delta_token_ids=output.token_ids))
431 +
432 + # update the previous values for the next iteration
433 + previous_texts[i] = current_text
434 + all_previous_token_ids[i] = current_token_ids
435
436 # handle streaming just a content delta
437 else:
438 delta_message = DeltaMessage(content=delta_text)
439
440 # set the previous values for the next iteration
420 - previous_texts[i] = output.text
421 - previous_num_tokens[i] = len(output.token_ids)
441 + previous_num_tokens[i] += len(output.token_ids)
442
443 # if the message delta is None (e.g. because it was a
444 # "control token" for tool calls or the parser otherwise
@@ -445,13 +465,12 @@ class OpenAIServingChat(OpenAIServing):
465 # handle usage stats if requested & if continuous
466 if (request.stream_options
467 and request.stream_options.include_usage):
448 - if (request.stream_options.continuous_usage_stats):
449 - prompt_tokens = len(res.prompt_token_ids)
468 + if request.stream_options.continuous_usage_stats:
469 completion_tokens = len(output.token_ids)
470 usage = UsageInfo(
452 - prompt_tokens=prompt_tokens,
471 + prompt_tokens=num_prompt_tokens,
472 completion_tokens=completion_tokens,
454 - total_tokens=prompt_tokens +
473 + total_tokens=num_prompt_tokens +
474 completion_tokens,
475 )
476 chunk.usage = usage
@@ -482,7 +501,7 @@ class OpenAIServingChat(OpenAIServing):
501 tool_parser.prev_tool_call_arr[index].get(
502 "arguments", {}))
503
485 - # get what we've streamed so for for arguments
504 + # get what we've streamed so far for arguments
505 # for the current tool
506 actual_call = tool_parser.streamed_args_for_tool[
507 index]
@@ -500,7 +519,6 @@ class OpenAIServingChat(OpenAIServing):
519 ])
520
521 # Send the finish response for each request.n only once
503 - prompt_tokens = len(res.prompt_token_ids)
522 choice_data = ChatCompletionResponseStreamChoice(
523 index=i,
524 delta=delta_message,
@@ -518,13 +536,12 @@ class OpenAIServingChat(OpenAIServing):
536 model=model_name)
537 if (request.stream_options
538 and request.stream_options.include_usage):
521 - if (request.stream_options.continuous_usage_stats):
522 - prompt_tokens = len(res.prompt_token_ids)
539 + if request.stream_options.continuous_usage_stats:
540 completion_tokens = len(output.token_ids)
541 usage = UsageInfo(
525 - prompt_tokens=prompt_tokens,
542 + prompt_tokens=num_prompt_tokens,
543 completion_tokens=completion_tokens,
527 - total_tokens=prompt_tokens +
544 + total_tokens=num_prompt_tokens +
545 completion_tokens,
546 )
547 chunk.usage = usage
@@ -538,10 +555,11 @@ class OpenAIServingChat(OpenAIServing):
555 # is sent, send the usage
556 if (request.stream_options
557 and request.stream_options.include_usage):
558 + completion_tokens = previous_num_tokens[i]
559 final_usage = UsageInfo(
542 - prompt_tokens=prompt_tokens,
543 - completion_tokens=previous_num_tokens[i],
544 - total_tokens=prompt_tokens + previous_num_tokens[i],
560 + prompt_tokens=num_prompt_tokens,
561 + completion_tokens=completion_tokens,
562 + total_tokens=num_prompt_tokens + completion_tokens,
563 )
564
565 final_usage_chunk = ChatCompletionStreamResponse(
@@ -680,6 +698,7 @@ class OpenAIServingChat(OpenAIServing):
698 or "")
699 choice.message.content = full_message
700
701 + assert final_res.prompt_token_ids is not None
702 num_prompt_tokens = len(final_res.prompt_token_ids)
703 num_generated_tokens = sum(
704 len(output.token_ids) for output in final_res.outputs)
@@ -789,9 +808,9 @@ class OpenAIServingChat(OpenAIServing):
808 return bool(
809 # if there is a delta message that includes tool calls which
810 # include a function that has arguments
792 - self.enable_auto_tools and self.tool_parser and delta_message
811 + output.finish_reason is not None
812 + and self.enable_auto_tools and self.tool_parser and delta_message
813 and delta_message.tool_calls and delta_message.tool_calls[0]
814 and delta_message.tool_calls[0].function
815 and delta_message.tool_calls[0].function.arguments is not None
796 - and output.finish_reason is not None
816 )
vllm/entrypoints/openai/serving_completion.py
+19 -13
@@ -223,9 +223,10 @@ class OpenAIServingCompletion(OpenAIServing):
223 tokenizer: AnyTokenizer,
224 ) -> AsyncGenerator[str, None]:
225 num_choices = 1 if request.n is None else request.n
226 - previous_texts = [""] * num_choices * num_prompts
226 + previous_text_lens = [0] * num_choices * num_prompts
227 previous_num_tokens = [0] * num_choices * num_prompts
228 has_echoed = [False] * num_choices * num_prompts
229 + num_prompt_tokens = [0] * num_prompts
230
231 try:
232 async for prompt_idx, res in result_generator:
@@ -233,6 +234,10 @@ class OpenAIServingCompletion(OpenAIServing):
234 prompt_logprobs = res.prompt_logprobs
235 prompt_text = res.prompt
236
237 + # Prompt details are excluded from later streamed outputs
238 + if res.prompt_token_ids is not None:
239 + num_prompt_tokens[prompt_idx] = len(res.prompt_token_ids)
240 +
241 delta_token_ids: GenericSequence[int]
242 out_logprobs: Optional[GenericSequence[Optional[Dict[
243 int, Logprob]]]]
@@ -244,6 +249,7 @@ class OpenAIServingCompletion(OpenAIServing):
249
250 assert request.max_tokens is not None
251 if request.echo and request.max_tokens == 0:
252 + assert prompt_token_ids is not None
253 assert prompt_text is not None
254 # only return the prompt
255 delta_text = prompt_text
@@ -252,6 +258,7 @@ class OpenAIServingCompletion(OpenAIServing):
258 has_echoed[i] = True
259 elif (request.echo and request.max_tokens > 0
260 and not has_echoed[i]):
261 + assert prompt_token_ids is not None
262 assert prompt_text is not None
263 assert prompt_logprobs is not None
264 # echo the prompt and first token
@@ -266,11 +273,9 @@ class OpenAIServingCompletion(OpenAIServing):
273 has_echoed[i] = True
274 else:
275 # return just the delta
269 - delta_text = output.text[len(previous_texts[i]):]
270 - delta_token_ids = output.token_ids[
271 - previous_num_tokens[i]:]
272 - out_logprobs = output.logprobs[previous_num_tokens[
273 - i]:] if output.logprobs else None
276 + delta_text = output.text
277 + delta_token_ids = output.token_ids
278 + out_logprobs = output.logprobs
279
280 if request.logprobs is not None:
281 assert out_logprobs is not None, (
@@ -280,13 +285,13 @@ class OpenAIServingCompletion(OpenAIServing):
285 top_logprobs=out_logprobs,
286 num_output_top_logprobs=request.logprobs,
287 tokenizer=tokenizer,
283 - initial_text_offset=len(previous_texts[i]),
288 + initial_text_offset=previous_text_lens[i],
289 )
290 else:
291 logprobs = None
292
288 - previous_texts[i] = output.text
289 - previous_num_tokens[i] = len(output.token_ids)
293 + previous_text_lens[i] += len(output.text)
294 + previous_num_tokens[i] += len(output.token_ids)
295 finish_reason = output.finish_reason
296 stop_reason = output.stop_reason
297
@@ -307,8 +312,8 @@ class OpenAIServingCompletion(OpenAIServing):
312 and request.stream_options.include_usage):
313 if (request.stream_options.continuous_usage_stats
314 or output.finish_reason is not None):
310 - prompt_tokens = len(prompt_token_ids)
311 - completion_tokens = len(output.token_ids)
315 + prompt_tokens = num_prompt_tokens[prompt_idx]
316 + completion_tokens = previous_num_tokens[i]
317 usage = UsageInfo(
318 prompt_tokens=prompt_tokens,
319 completion_tokens=completion_tokens,
@@ -356,6 +361,7 @@ class OpenAIServingCompletion(OpenAIServing):
361
362 for final_res in final_res_batch:
363 prompt_token_ids = final_res.prompt_token_ids
364 + assert prompt_token_ids is not None
365 prompt_logprobs = final_res.prompt_logprobs
366 prompt_text = final_res.prompt
367
@@ -411,9 +417,9 @@ class OpenAIServingCompletion(OpenAIServing):
417 )
418 choices.append(choice_data)
419
420 + num_generated_tokens += len(output.token_ids)
421 +
422 num_prompt_tokens += len(prompt_token_ids)
415 - num_generated_tokens += sum(
416 - len(output.token_ids) for output in final_res.outputs)
423
424 usage = UsageInfo(
425 prompt_tokens=num_prompt_tokens,
vllm/outputs.py
+55 -24
@@ -5,6 +5,7 @@ from typing import Sequence as GenericSequence
5 from typing import Union
6
7 from vllm.lora.request import LoRARequest
8 +from vllm.sampling_params import RequestOutputKind
9 from vllm.sequence import (PromptLogprobs, RequestMetrics, SampleLogprobs,
10 SequenceGroup, SequenceStatus)
11
@@ -92,7 +93,7 @@ class RequestOutput:
93 self,
94 request_id: str,
95 prompt: Optional[str],
95 - prompt_token_ids: List[int],
96 + prompt_token_ids: Optional[List[int]],
97 prompt_logprobs: Optional[PromptLogprobs],
98 outputs: List[CompletionOutput],
99 finished: bool,
@@ -113,19 +114,26 @@ class RequestOutput:
114 self.encoder_prompt_token_ids = encoder_prompt_token_ids
115
116 @classmethod
116 - def from_seq_group(cls, seq_group: SequenceGroup) -> "RequestOutput":
117 - if seq_group.sampling_params is None:
117 + def from_seq_group(cls,
118 + seq_group: SequenceGroup) -> Optional["RequestOutput"]:
119 + sampling_params = seq_group.sampling_params
120 + if sampling_params is None:
121 raise ValueError(
122 "Sampling parameters are missing for a CompletionRequest.")
123 + finished = seq_group.is_finished()
124 + if sampling_params.output_kind == RequestOutputKind.FINAL_ONLY and (
125 + not finished):
126 + return None
127 +
128 seqs = seq_group.get_seqs()
129 if len(seqs) == 1:
130 top_n_seqs = seqs
131 else:
132 # Get the top-n sequences.
125 - n = seq_group.sampling_params.n
126 - if seq_group.sampling_params.use_beam_search:
133 + n = sampling_params.n
134 + if sampling_params.use_beam_search:
135 sorting_key = lambda seq: seq.get_beam_search_score(
128 - seq_group.sampling_params.length_penalty)
136 + sampling_params.length_penalty)
137 else:
138 sorting_key = lambda seq: seq.get_cumulative_logprob()
139 sorted_seqs = sorted(seqs, key=sorting_key, reverse=True)
@@ -135,26 +143,49 @@ class RequestOutput:
143 # NOTE: We need omit logprobs here explicitly because the sequence
144 # always has the logprobs of the sampled tokens even if the
145 # logprobs are not requested.
138 - include_logprobs = seq_group.sampling_params.logprobs is not None
139 - text_buffer_length = seq_group.sampling_params.output_text_buffer_length
140 - outputs = [
141 - CompletionOutput(
142 - seqs.index(seq),
143 - seq.get_output_text_to_return(text_buffer_length),
144 - seq.data._output_token_ids,
145 - seq.get_cumulative_logprob() if include_logprobs else None,
146 - seq.output_logprobs if include_logprobs else None,
147 - SequenceStatus.get_finished_reason(seq.status),
148 - seq.stop_reason) for seq in top_n_seqs
149 - ]
146 + include_logprobs = sampling_params.logprobs is not None
147 + text_buffer_length = sampling_params.output_text_buffer_length
148 + delta = sampling_params.output_kind == RequestOutputKind.DELTA
149 +
150 + outputs = []
151 + include_prompt = True
152 + for seq in top_n_seqs:
153 + output_text = seq.get_output_text_to_return(
154 + text_buffer_length, delta)
155 + output_token_ids = seq.get_output_token_ids_to_return(delta)
156 + output_logprobs = seq.output_logprobs if include_logprobs else None
157 +
158 + if delta:
159 + # Slice logprobs delta if applicable
160 + if output_logprobs:
161 + output_logprobs = output_logprobs[-len(output_token_ids):]
162 + # Don't include prompt if this is after the first output
163 + # containing decode token ids
164 + if include_prompt and seq.get_output_len() > len(
165 + output_token_ids):
166 + include_prompt = False
167 +
168 + outputs.append(
169 + CompletionOutput(
170 + seqs.index(seq), output_text, output_token_ids,
171 + seq.get_cumulative_logprob() if include_logprobs else None,
172 + output_logprobs,
173 + SequenceStatus.get_finished_reason(seq.status),
174 + seq.stop_reason))
175
176 # Every sequence in the sequence group should have the same prompt.
152 - prompt = seq_group.prompt
153 - prompt_token_ids = seq_group.prompt_token_ids
154 - encoder_prompt = seq_group.encoder_prompt
155 - encoder_prompt_token_ids = seq_group.encoder_prompt_token_ids
156 - prompt_logprobs = seq_group.prompt_logprobs
157 - finished = seq_group.is_finished()
177 + if include_prompt:
178 + prompt = seq_group.prompt
179 + prompt_token_ids = seq_group.prompt_token_ids
180 + encoder_prompt = seq_group.encoder_prompt
181 + encoder_prompt_token_ids = seq_group.encoder_prompt_token_ids
182 + prompt_logprobs = seq_group.prompt_logprobs
183 + else:
184 + prompt = None
185 + prompt_token_ids = None
186 + encoder_prompt = None
187 + encoder_prompt_token_ids = None
188 + prompt_logprobs = None
189 finished_time = time.time() if finished else None
190 seq_group.set_finished_time(finished_time)
191 return cls(seq_group.request_id,
vllm/sampling_params.py
+16 -1
@@ -1,6 +1,6 @@
1 """Sampling parameters for text generation."""
2 import copy
3 -from enum import IntEnum
3 +from enum import Enum, IntEnum
4 from functools import cached_property
5 from typing import Any, Callable, Dict, List, Optional, Set, Union
6
@@ -33,6 +33,15 @@ first argument, and returns a modified tensor of logits
33 to sample from."""
34
35
36 +class RequestOutputKind(Enum):
37 + # Return entire output so far in every RequestOutput
38 + CUMULATIVE = 0
39 + # Return only deltas in each RequestOutput
40 + DELTA = 1
41 + # Do not return intermediate RequestOuputs
42 + FINAL_ONLY = 2
43 +
44 +
45 class SamplingParams(
46 msgspec.Struct,
47 omit_defaults=True, # type: ignore[call-arg]
@@ -147,6 +156,7 @@ class SamplingParams(
156 logits_processors: Optional[Any] = None
157 include_stop_str_in_output: bool = False
158 truncate_prompt_tokens: Optional[Annotated[int, msgspec.Meta(ge=1)]] = None
159 + output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE
160
161 # The below fields are not supposed to be used as an input.
162 # They are set in post_init.
@@ -182,6 +192,7 @@ class SamplingParams(
192 logits_processors: Optional[List[LogitsProcessor]] = None,
193 truncate_prompt_tokens: Optional[Annotated[int,
194 msgspec.Meta(ge=1)]] = None,
195 + output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE,
196 ) -> "SamplingParams":
197 return SamplingParams(
198 n=1 if n is None else n,
@@ -213,6 +224,7 @@ class SamplingParams(
224 spaces_between_special_tokens=spaces_between_special_tokens,
225 logits_processors=logits_processors,
226 truncate_prompt_tokens=truncate_prompt_tokens,
227 + output_kind=output_kind,
228 )
229
230 def __post_init__(self) -> None:
@@ -317,6 +329,9 @@ class SamplingParams(
329 raise ValueError(
330 "stop strings are only supported when detokenize is True. "
331 "Set detokenize=True to use stop.")
332 + if self.best_of != self.n and self.output_kind == (
333 + RequestOutputKind.DELTA):
334 + raise ValueError("best_of must equal n to use output_kind=DELTA")
335
336 def _verify_beam_search(self) -> None:
337 if self.best_of == 1:
vllm/sequence.py
+34 -5
@@ -5,8 +5,9 @@ from abc import ABC, abstractmethod
5 from array import array
6 from collections import defaultdict
7 from dataclasses import dataclass
8 -from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Mapping,
9 - Optional, Set, Tuple, Union, cast)
8 +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Mapping, Optional
9 +from typing import Sequence as GenericSequence
10 +from typing import Set, Tuple, Union, cast
11
12 import msgspec
13 import torch
@@ -407,6 +408,10 @@ class Sequence:
408 self.status = SequenceStatus.WAITING
409 self.stop_reason: Union[int, str, None] = None
410
411 + # These are used to keep track of delta outputs
412 + self._last_token_ids_offset: int = 0
413 + self._last_output_text_offset: int = 0
414 +
415 # Used for incremental detokenization
416 self.prefix_offset = 0
417 self.read_offset = 0
@@ -462,11 +467,35 @@ class Sequence:
467 return self.prompt_adapter_request.prompt_adapter_id \
468 if self.prompt_adapter_request else 0
469
465 - def get_output_text_to_return(self, buffer_length: int):
470 + def get_output_text_to_return(self, buffer_length: int,
471 + delta: bool) -> str:
472 + """If delta is True, only new text since the last call to
473 + this method is returned"""
474 +
475 # We return the full output text if the sequence is finished.
476 truncate = buffer_length and not self.is_finished()
468 - return self.output_text[:-buffer_length] if truncate else (
469 - self.output_text)
477 + if not delta:
478 + return self.output_text[:-buffer_length] if truncate else (
479 + self.output_text)
480 + length = len(self.output_text) - buffer_length
481 + last_offset = self._last_output_text_offset
482 + if last_offset < length:
483 + self._last_output_text_offset = length
484 + return self.output_text[last_offset:length]
485 + return ""
486 +
487 + def get_output_token_ids_to_return(self,
488 + delta: bool) -> GenericSequence[int]:
489 + """If delta is True, only new tokens since the last call to
490 + this method are returned"""
491 + if not delta:
492 + return self.get_output_token_ids()
493 + length = self.get_output_len()
494 + last_offset = self._last_token_ids_offset
495 + if last_offset < length:
496 + self._last_token_ids_offset = length
497 + return self.data._output_token_ids[last_offset:]
498 + return ()
499
500 def hash_of_block(self, logical_idx: int) -> int:
501 # TODO This can produce incorrect hash when block size > prompt size