A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Core] Add engine option to return only deltas or final output (#7381)
Nick Hill committed
Sep 12, 2024 at 20:02 UTC
551ce01078a655068e5ec3764d0a55ac744ea425
10 files changed
+371
-137
.buildkite/test-pipeline.yaml
+1
@@ -50,6 +50,7 @@ steps:
50
- tests/worker
51
commands:
52
- pytest -v -s async_engine # Async Engine
53
+ - NUM_SCHEDULER_STEPS=4 pytest -v -s async_engine/test_async_llm_engine.py
54
- pytest -v -s test_inputs.py
55
- pytest -v -s multimodal
56
- pytest -v -s test_utils.py # Utils
tests/async_engine/test_async_llm_engine.py
+147
-14
@@ -1,7 +1,10 @@
1
import asyncio
2
+import os
3
+import uuid
4
from asyncio import CancelledError
5
+from copy import copy
6
from dataclasses import dataclass
4
-from typing import Optional
7
+from typing import List, Optional
8
9
import pytest
10
import pytest_asyncio
@@ -11,6 +14,7 @@ from vllm import SamplingParams
14
from vllm.config import ParallelConfig
15
from vllm.engine.async_llm_engine import AsyncEngineArgs, AsyncLLMEngine
16
from vllm.outputs import RequestOutput as RealRequestOutput
17
+from vllm.sampling_params import RequestOutputKind
18
19
from ..conftest import cleanup
20
from ..utils import wait_for_gpu_memory_to_clear
@@ -122,8 +126,17 @@ def start_engine():
126
timeout_s=60,
127
)
128
129
+ num_scheduler_steps = int(os.getenv("NUM_SCHEDULER_STEPS", "1"))
130
+ print(f"Starting engine with num_scheduler_steps={num_scheduler_steps}")
131
+
132
return AsyncLLMEngine.from_engine_args(
126
- AsyncEngineArgs(model="facebook/opt-125m", enforce_eager=True))
133
+ AsyncEngineArgs(model="facebook/opt-125m",
134
+ enforce_eager=True,
135
+ num_scheduler_steps=num_scheduler_steps))
136
+
137
+
138
+def uid() -> str:
139
+ return str(uuid.uuid4())
140
141
142
@pytest_asyncio.fixture(scope="module")
@@ -148,57 +161,177 @@ def should_do_global_cleanup_after_test(request) -> bool:
161
@pytest.mark.asyncio(scope="module")
162
async def test_asyncio_run(async_engine):
163
164
+ scheduler_config = await async_engine.get_scheduler_config()
165
+ num_scheduler_steps = scheduler_config.num_scheduler_steps
166
+
167
async def run(prompt: str):
168
sampling_params = SamplingParams(
169
temperature=0,
170
max_tokens=32,
171
+ min_tokens=32,
172
)
173
174
+ output_count = 0
175
+ final_output = None
176
async for output in async_engine.generate(prompt,
177
sampling_params,
159
- request_id=prompt):
178
+ request_id=uid()):
179
+ output_count += 1
180
final_output = output
161
- return final_output
181
+ return final_output, output_count
182
183
results = await asyncio.gather(
184
run("test0"),
165
- run("test1"),
185
+ run("test0"),
186
)
187
assert len(results) == 2
188
+ first, second = results
189
+
190
+ # remove nondeterministic fields for comparison
191
+ first[0].metrics = None
192
+ second[0].metrics = None
193
+ first[0].request_id = None
194
+ second[0].request_id = None
195
+
196
+ assert str(first) == str(second)
197
+
198
+ output_count = results[0][1]
199
+ if num_scheduler_steps == 1:
200
+ assert output_count == 32
201
+ else:
202
+ assert 1 < output_count < 32
203
+
204
+
205
+@pytest.mark.asyncio(scope="module")
206
+async def test_output_kinds(async_engine):
207
+ """Test that output_kind works as expected and that
208
+ results are equivalent across different kinds."""
209
+
210
+ scheduler_config = await async_engine.get_scheduler_config()
211
+ num_scheduler_steps = scheduler_config.num_scheduler_steps
212
+
213
+ sampling_params = SamplingParams(
214
+ temperature=0,
215
+ max_tokens=32,
216
+ min_tokens=32,
217
+ )
218
+
219
+ async def run(prompt: str, kind: RequestOutputKind):
220
+ params = copy(sampling_params)
221
+ params.output_kind = kind
222
+
223
+ output_count = 0
224
+ final_output = None
225
+ async for output in async_engine.generate(prompt,
226
+ params,
227
+ request_id=uid()):
228
+ output_count += 1
229
+ final_output = output
230
+
231
+ assert final_output is not None
232
+ return (final_output.prompt_token_ids,
233
+ final_output.outputs[0].token_ids,
234
+ final_output.outputs[0].text, output_count)
235
+
236
+ async def run_deltas(prompt: str):
237
+ params = copy(sampling_params)
238
+ params.output_kind = RequestOutputKind.DELTA
239
+
240
+ prompt_tokens = None
241
+ output_tokens: List[int] = []
242
+ output_text = ""
243
+ output_count = 0
244
+ async for output in async_engine.generate(prompt,
245
+ params,
246
+ request_id=uid()):
247
+ token_ids = output.outputs[0].token_ids
248
+ text = output.outputs[0].text
249
+
250
+ # Ensure we get prompt ids iff we haven't yet received output tokens
251
+ if output_tokens:
252
+ assert 1 <= len(token_ids) <= num_scheduler_steps
253
+ assert text
254
+ assert not output.prompt_token_ids
255
+ else:
256
+ assert output.prompt_token_ids
257
+ prompt_tokens = output.prompt_token_ids
258
+
259
+ output_tokens.extend(token_ids)
260
+ output_text += text
261
+
262
+ output_count += 1
263
+ return prompt_tokens, output_tokens, output_text, output_count
264
+
265
+ results = await asyncio.gather(
266
+ run("common input prompt", RequestOutputKind.CUMULATIVE),
267
+ run("common input prompt", RequestOutputKind.FINAL_ONLY),
268
+ run_deltas("common input prompt"))
269
+
270
+ # Make sure outputs are the same
271
+ prompt_set = set(tuple(prompt_ids) for prompt_ids, _, _, _ in results)
272
+ assert len(prompt_set) == 1
273
+
274
+ text_set = set(text for _, _, text, _ in results)
275
+ assert len(text_set) == 1
276
+
277
+ tokens_set = set(tuple(ids) for _, ids, _, _ in results)
278
+ assert len(tokens_set) == 1
279
+
280
+ cumulative, final, deltas = results
281
+
282
+ # output message counts
283
+ assert cumulative[3] == deltas[3]
284
+
285
+ if num_scheduler_steps == 1:
286
+ assert cumulative[3] == 32
287
+ else:
288
+ assert 1 < cumulative[3] < 32
289
+
290
+ assert final[3] == 1
291
292
293
@pytest.mark.asyncio(scope="module")
294
async def test_cancellation(async_engine):
295
+ scheduler_config = await async_engine.get_scheduler_config()
296
+ num_scheduler_steps = scheduler_config.num_scheduler_steps
297
+
298
sampling_params = SamplingParams(
299
temperature=0,
174
- min_tokens=10,
175
- max_tokens=10,
300
+ min_tokens=13,
301
+ max_tokens=13,
302
)
303
304
+ stop_at = 5 if num_scheduler_steps == 1 else 1
305
+
306
+ request_id = uid()
307
+
308
i = 0
309
with pytest.raises(CancelledError):
310
async for output in async_engine.generate("test2",
311
sampling_params,
182
- request_id="test2"):
312
+ request_id=request_id):
313
assert not output.finished
314
i += 1
185
- if i == 5:
186
- await async_engine.abort("test2")
315
+ if i == stop_at:
316
+ await async_engine.abort(request_id)
317
188
- assert i == 5
318
+ assert i == stop_at
319
320
321
@pytest.mark.asyncio(scope="module")
322
async def test_delayed_generator(async_engine):
323
+ scheduler_config = await async_engine.get_scheduler_config()
324
+
325
+ if scheduler_config.num_scheduler_steps != 1:
326
+ pytest.skip("no need to test this one with multistep")
327
+
328
sampling_params = SamplingParams(
329
temperature=0,
330
min_tokens=10,
331
max_tokens=10,
332
)
333
199
- stream = async_engine.generate("test3",
200
- sampling_params,
201
- request_id="test3")
334
+ stream = async_engine.generate("test3", sampling_params, request_id=uid())
335
i = 0
336
final_output: Optional[RealRequestOutput] = None
337
async for output in stream:
vllm/engine/llm_engine.py
+13
-11
@@ -39,7 +39,7 @@ from vllm.outputs import (EmbeddingRequestOutput, RequestOutput,
39
RequestOutputFactory)
40
from vllm.pooling_params import PoolingParams
41
from vllm.prompt_adapter.request import PromptAdapterRequest
42
-from vllm.sampling_params import SamplingParams
42
+from vllm.sampling_params import RequestOutputKind, SamplingParams
43
from vllm.sequence import (EmbeddingSequenceGroupOutput, ExecuteModelRequest,
44
Sequence, SequenceGroup, SequenceGroupMetadata,
45
SequenceStatus)
@@ -225,9 +225,6 @@ class LLMEngine:
225
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
226
stat_loggers: Optional[Dict[str, StatLoggerBase]] = None,
227
input_registry: InputRegistry = INPUT_REGISTRY,
228
- # To improve performance, only final requests outputs may be required.
229
- # If this set to true, then no intermediate outputs will be returned.
230
- step_return_finished_only: bool = False,
228
) -> None:
229
logger.info(
230
"Initializing an LLM engine (v%s) with config: "
@@ -295,7 +292,6 @@ class LLMEngine:
292
self.observability_config = observability_config or ObservabilityConfig(
293
)
294
self.log_stats = log_stats
298
- self.step_return_finished_only = step_return_finished_only
295
296
if not self.model_config.skip_tokenizer_init:
297
self.tokenizer = self._init_tokenizer()
@@ -1273,7 +1269,7 @@ class LLMEngine:
1269
1270
ctx: The virtual engine context to work on
1271
request_id: If provided, then only this request is going to be processed
1276
-
1272
+
1273
"""
1274
now = time.time()
1275
@@ -1378,7 +1374,8 @@ class LLMEngine:
1374
seq_group = scheduled_seq_group.seq_group
1375
seq_group.maybe_set_first_token_time(now)
1376
request_output = RequestOutputFactory.create(seq_group)
1381
- ctx.request_outputs.append(request_output)
1377
+ if request_output:
1378
+ ctx.request_outputs.append(request_output)
1379
1380
# When we process a single request, we skip it for the next time,
1381
# and invoke the request output callback (if there was final output)
@@ -1415,14 +1412,19 @@ class LLMEngine:
1412
1413
seq_group = scheduled_seq_group.seq_group
1414
seq_group.maybe_set_first_token_time(now)
1418
- if (seq_group.is_finished()
1419
- if self.step_return_finished_only else True):
1420
- request_output = RequestOutputFactory.create(seq_group)
1415
+ request_output = RequestOutputFactory.create(seq_group)
1416
+ if request_output:
1417
ctx.request_outputs.append(request_output)
1418
1419
for seq_group in scheduler_outputs.ignored_seq_groups:
1420
+ params = seq_group.sampling_params
1421
+ if params is not None and params.output_kind == (
1422
+ RequestOutputKind.DELTA) and not seq_group.is_finished():
1423
+ continue
1424
+
1425
request_output = RequestOutputFactory.create(seq_group)
1425
- ctx.request_outputs.append(request_output)
1426
+ if request_output:
1427
+ ctx.request_outputs.append(request_output)
1428
1429
# Immediately process request outputs here (if callback is given)
1430
if (ctx.request_outputs
vllm/entrypoints/llm.py
+8
-15
@@ -19,7 +19,7 @@ from vllm.model_executor.guided_decoding.guided_fields import LLMGuidedOptions
19
from vllm.outputs import EmbeddingRequestOutput, RequestOutput
20
from vllm.pooling_params import PoolingParams
21
from vllm.prompt_adapter.request import PromptAdapterRequest
22
-from vllm.sampling_params import SamplingParams
22
+from vllm.sampling_params import RequestOutputKind, SamplingParams
23
from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
24
get_cached_tokenizer)
25
from vllm.transformers_utils.tokenizer_group import TokenizerGroup
@@ -642,14 +642,12 @@ class LLM:
642
raise ValueError("The lengths of prompts and lora_request "
643
"must be the same.")
644
645
- if isinstance(params, list):
646
- params = [
647
- self._add_guided_processor(param, guided_options)
648
- if isinstance(param, SamplingParams) else param
649
- for param in params
650
- ]
651
- elif isinstance(params, SamplingParams):
652
- params = self._add_guided_processor(params, guided_options)
645
+ for sp in params if isinstance(params, list) else (params, ):
646
+ if isinstance(sp, SamplingParams):
647
+ self._add_guided_processor(sp, guided_options)
648
+
649
+ # We only care about the final output
650
+ sp.output_kind = RequestOutputKind.FINAL_ONLY
651
652
# Add requests to the engine.
653
for i, request_inputs in enumerate(inputs):
@@ -709,9 +707,6 @@ class LLM:
707
f"output: {0:.2f} toks/s"),
708
)
709
712
- # In the loop below, only finished outputs are used
713
- self.llm_engine.step_return_finished_only = True
714
-
710
# Run the engine.
711
outputs: List[Union[RequestOutput, EmbeddingRequestOutput]] = []
712
total_in_toks = 0
@@ -724,6 +719,7 @@ class LLM:
719
if use_tqdm:
720
if isinstance(output, RequestOutput):
721
# Calculate tokens only for RequestOutput
722
+ assert output.prompt_token_ids is not None
723
total_in_toks += len(output.prompt_token_ids)
724
in_spd = total_in_toks / pbar.format_dict["elapsed"]
725
total_out_toks += sum(
@@ -735,9 +731,6 @@ class LLM:
731
f"output: {out_spd:.2f} toks/s")
732
pbar.update(1)
733
738
- # Restore original behavior
739
- self.llm_engine.step_return_finished_only = False
740
-
734
if use_tqdm:
735
pbar.close()
736
# Sort the outputs by request ID.
vllm/entrypoints/openai/protocol.py
+6
-1
@@ -12,7 +12,8 @@ from typing_extensions import Annotated, Required, TypedDict
12
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
13
from vllm.entrypoints.openai.logits_processors import get_logits_processors
14
from vllm.pooling_params import PoolingParams
15
-from vllm.sampling_params import LogitsProcessor, SamplingParams
15
+from vllm.sampling_params import (LogitsProcessor, RequestOutputKind,
16
+ SamplingParams)
17
from vllm.sequence import Logprob
18
from vllm.transformers_utils.tokenizer import AnyTokenizer
19
from vllm.utils import random_uuid
@@ -316,6 +317,8 @@ class ChatCompletionRequest(OpenAIBaseModel):
317
length_penalty=self.length_penalty,
318
logits_processors=logits_processors,
319
truncate_prompt_tokens=self.truncate_prompt_tokens,
320
+ output_kind=RequestOutputKind.DELTA if self.stream \
321
+ else RequestOutputKind.FINAL_ONLY,
322
)
323
324
@model_validator(mode="before")
@@ -559,6 +562,8 @@ class CompletionRequest(OpenAIBaseModel):
562
length_penalty=self.length_penalty,
563
logits_processors=logits_processors,
564
truncate_prompt_tokens=self.truncate_prompt_tokens,
565
+ output_kind=RequestOutputKind.DELTA if self.stream \
566
+ else RequestOutputKind.FINAL_ONLY,
567
)
568
569
@model_validator(mode="before")
vllm/entrypoints/openai/serving_chat.py
+72
-53
@@ -246,8 +246,7 @@ class OpenAIServingChat(OpenAIServing):
246
def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
247
if request.add_generation_prompt:
248
return self.response_role
249
- else:
250
- return request.messages[-1]["role"]
249
+ return request.messages[-1]["role"]
250
251
async def chat_completion_stream_generator(
252
self,
@@ -264,15 +263,37 @@ class OpenAIServingChat(OpenAIServing):
263
264
# Send response for each token for each request.n (index)
265
num_choices = 1 if request.n is None else request.n
267
- previous_texts = [""] * num_choices
266
previous_num_tokens = [0] * num_choices
267
finish_reason_sent = [False] * num_choices
268
269
+ num_prompt_tokens = 0
270
+
271
tool_parser: Optional[ToolParser] = self.tool_parser(
272
tokenizer) if self.tool_parser else None
273
274
+ if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
275
+ tool_choice_function_name = request.tool_choice.function.name
276
+ else:
277
+ tool_choice_function_name = None
278
+
279
+ # Determine whether tools are in use with "auto" tool choice
280
+ tool_choice_auto = (
281
+ not tool_choice_function_name
282
+ and self._should_stream_with_auto_tool_parsing(request))
283
+
284
+ all_previous_token_ids: Optional[List[List[int]]]
285
+ if tool_choice_auto:
286
+ # These are only required in "auto" tool choice case
287
+ previous_texts = [""] * num_choices
288
+ all_previous_token_ids = [[]] * num_choices
289
+ else:
290
+ previous_texts, all_previous_token_ids = None, None
291
+
292
try:
293
async for res in result_generator:
294
+ if res.prompt_token_ids is not None:
295
+ num_prompt_tokens = len(res.prompt_token_ids)
296
+
297
# We need to do it here, because if there are exceptions in
298
# the result_generator, it needs to be sent as the FIRST
299
# response (by the try...catch).
@@ -305,10 +326,10 @@ class OpenAIServingChat(OpenAIServing):
326
and request.stream_options.include_usage):
327
# if continuous usage stats are requested, add it
328
if request.stream_options.continuous_usage_stats:
308
- prompt_tokens = len(res.prompt_token_ids)
309
- usage = UsageInfo(prompt_tokens=prompt_tokens,
310
- completion_tokens=0,
311
- total_tokens=prompt_tokens)
329
+ usage = UsageInfo(
330
+ prompt_tokens=num_prompt_tokens,
331
+ completion_tokens=0,
332
+ total_tokens=num_prompt_tokens)
333
chunk.usage = usage
334
# otherwise don't
335
else:
@@ -344,12 +365,10 @@ class OpenAIServingChat(OpenAIServing):
365
request.stream_options.include_usage):
366
if (request.stream_options.
367
continuous_usage_stats):
347
- prompt_tokens = len(
348
- res.prompt_token_ids)
368
usage = UsageInfo(
350
- prompt_tokens=prompt_tokens,
369
+ prompt_tokens=num_prompt_tokens,
370
completion_tokens=0,
352
- total_tokens=prompt_tokens)
371
+ total_tokens=num_prompt_tokens)
372
chunk.usage = usage
373
else:
374
chunk.usage = None
@@ -360,65 +379,66 @@ class OpenAIServingChat(OpenAIServing):
379
first_iteration = False
380
381
for output in res.outputs:
363
-
382
i = output.index
383
384
if finish_reason_sent[i]:
385
continue
386
369
- delta_token_ids = output.token_ids[previous_num_tokens[i]:]
370
- out_logprobs = output.logprobs[
371
- previous_num_tokens[i]:] if output.logprobs else None
372
-
387
if request.logprobs and request.top_logprobs is not None:
374
- assert out_logprobs is not None, (
388
+ assert output.logprobs is not None, (
389
"Did not output logprobs")
390
logprobs = self._create_chat_logprobs(
377
- token_ids=delta_token_ids,
378
- top_logprobs=out_logprobs,
391
+ token_ids=output.token_ids,
392
+ top_logprobs=output.logprobs,
393
tokenizer=tokenizer,
394
num_output_top_logprobs=request.top_logprobs,
395
)
396
else:
397
logprobs = None
398
385
- delta_text = output.text[len(previous_texts[i]):]
386
- delta_message: Optional[DeltaMessage] = None
399
+ delta_text = output.text
400
+ delta_message: Optional[DeltaMessage]
401
402
# handle streaming deltas for tools with named tool_choice
389
- if (request.tool_choice and type(request.tool_choice) is
390
- ChatCompletionNamedToolChoiceParam):
403
+ if tool_choice_function_name:
404
delta_message = DeltaMessage(tool_calls=[
405
DeltaToolCall(function=DeltaFunctionCall(
393
- name=request.tool_choice.function.name,
406
+ name=tool_choice_function_name,
407
arguments=delta_text),
408
index=i)
409
])
410
411
# handle streaming deltas for tools with "auto" tool choice
399
- elif (self._should_stream_with_auto_tool_parsing(request)
400
- and tool_parser):
412
+ elif tool_choice_auto:
413
+ assert previous_texts is not None
414
+ assert all_previous_token_ids is not None
415
+ assert tool_parser is not None
416
+ #TODO optimize manipulation of these lists
417
+ previous_text = previous_texts[i]
418
+ previous_token_ids = all_previous_token_ids[i]
419
+ current_text = previous_text + delta_text
420
+ current_token_ids = previous_token_ids + list(
421
+ output.token_ids)
422
+
423
delta_message = (
424
tool_parser.extract_tool_calls_streaming(
403
- previous_text=previous_texts[i],
404
- current_text=output.text,
425
+ previous_text=previous_text,
426
+ current_text=current_text,
427
delta_text=delta_text,
406
- previous_token_ids= \
407
- output.token_ids[
408
- :-1 * len(delta_token_ids)
409
- ],
410
- current_token_ids=output.token_ids,
411
- delta_token_ids=delta_token_ids
412
- )
413
- )
428
+ previous_token_ids=previous_token_ids,
429
+ current_token_ids=current_token_ids,
430
+ delta_token_ids=output.token_ids))
431
+
432
+ # update the previous values for the next iteration
433
+ previous_texts[i] = current_text
434
+ all_previous_token_ids[i] = current_token_ids
435
436
# handle streaming just a content delta
437
else:
438
delta_message = DeltaMessage(content=delta_text)
439
440
# set the previous values for the next iteration
420
- previous_texts[i] = output.text
421
- previous_num_tokens[i] = len(output.token_ids)
441
+ previous_num_tokens[i] += len(output.token_ids)
442
443
# if the message delta is None (e.g. because it was a
444
# "control token" for tool calls or the parser otherwise
@@ -445,13 +465,12 @@ class OpenAIServingChat(OpenAIServing):
465
# handle usage stats if requested & if continuous
466
if (request.stream_options
467
and request.stream_options.include_usage):
448
- if (request.stream_options.continuous_usage_stats):
449
- prompt_tokens = len(res.prompt_token_ids)
468
+ if request.stream_options.continuous_usage_stats:
469
completion_tokens = len(output.token_ids)
470
usage = UsageInfo(
452
- prompt_tokens=prompt_tokens,
471
+ prompt_tokens=num_prompt_tokens,
472
completion_tokens=completion_tokens,
454
- total_tokens=prompt_tokens +
473
+ total_tokens=num_prompt_tokens +
474
completion_tokens,
475
)
476
chunk.usage = usage
@@ -482,7 +501,7 @@ class OpenAIServingChat(OpenAIServing):
501
tool_parser.prev_tool_call_arr[index].get(
502
"arguments", {}))
503
485
- # get what we've streamed so for for arguments
504
+ # get what we've streamed so far for arguments
505
# for the current tool
506
actual_call = tool_parser.streamed_args_for_tool[
507
index]
@@ -500,7 +519,6 @@ class OpenAIServingChat(OpenAIServing):
519
])
520
521
# Send the finish response for each request.n only once
503
- prompt_tokens = len(res.prompt_token_ids)
522
choice_data = ChatCompletionResponseStreamChoice(
523
index=i,
524
delta=delta_message,
@@ -518,13 +536,12 @@ class OpenAIServingChat(OpenAIServing):
536
model=model_name)
537
if (request.stream_options
538
and request.stream_options.include_usage):
521
- if (request.stream_options.continuous_usage_stats):
522
- prompt_tokens = len(res.prompt_token_ids)
539
+ if request.stream_options.continuous_usage_stats:
540
completion_tokens = len(output.token_ids)
541
usage = UsageInfo(
525
- prompt_tokens=prompt_tokens,
542
+ prompt_tokens=num_prompt_tokens,
543
completion_tokens=completion_tokens,
527
- total_tokens=prompt_tokens +
544
+ total_tokens=num_prompt_tokens +
545
completion_tokens,
546
)
547
chunk.usage = usage
@@ -538,10 +555,11 @@ class OpenAIServingChat(OpenAIServing):
555
# is sent, send the usage
556
if (request.stream_options
557
and request.stream_options.include_usage):
558
+ completion_tokens = previous_num_tokens[i]
559
final_usage = UsageInfo(
542
- prompt_tokens=prompt_tokens,
543
- completion_tokens=previous_num_tokens[i],
544
- total_tokens=prompt_tokens + previous_num_tokens[i],
560
+ prompt_tokens=num_prompt_tokens,
561
+ completion_tokens=completion_tokens,
562
+ total_tokens=num_prompt_tokens + completion_tokens,
563
)
564
565
final_usage_chunk = ChatCompletionStreamResponse(
@@ -680,6 +698,7 @@ class OpenAIServingChat(OpenAIServing):
698
or "")
699
choice.message.content = full_message
700
701
+ assert final_res.prompt_token_ids is not None
702
num_prompt_tokens = len(final_res.prompt_token_ids)
703
num_generated_tokens = sum(
704
len(output.token_ids) for output in final_res.outputs)
@@ -789,9 +808,9 @@ class OpenAIServingChat(OpenAIServing):
808
return bool(
809
# if there is a delta message that includes tool calls which
810
# include a function that has arguments
792
- self.enable_auto_tools and self.tool_parser and delta_message
811
+ output.finish_reason is not None
812
+ and self.enable_auto_tools and self.tool_parser and delta_message
813
and delta_message.tool_calls and delta_message.tool_calls[0]
814
and delta_message.tool_calls[0].function
815
and delta_message.tool_calls[0].function.arguments is not None
796
- and output.finish_reason is not None
816
)
vllm/entrypoints/openai/serving_completion.py
+19
-13
@@ -223,9 +223,10 @@ class OpenAIServingCompletion(OpenAIServing):
223
tokenizer: AnyTokenizer,
224
) -> AsyncGenerator[str, None]:
225
num_choices = 1 if request.n is None else request.n
226
- previous_texts = [""] * num_choices * num_prompts
226
+ previous_text_lens = [0] * num_choices * num_prompts
227
previous_num_tokens = [0] * num_choices * num_prompts
228
has_echoed = [False] * num_choices * num_prompts
229
+ num_prompt_tokens = [0] * num_prompts
230
231
try:
232
async for prompt_idx, res in result_generator:
@@ -233,6 +234,10 @@ class OpenAIServingCompletion(OpenAIServing):
234
prompt_logprobs = res.prompt_logprobs
235
prompt_text = res.prompt
236
237
+ # Prompt details are excluded from later streamed outputs
238
+ if res.prompt_token_ids is not None:
239
+ num_prompt_tokens[prompt_idx] = len(res.prompt_token_ids)
240
+
241
delta_token_ids: GenericSequence[int]
242
out_logprobs: Optional[GenericSequence[Optional[Dict[
243
int, Logprob]]]]
@@ -244,6 +249,7 @@ class OpenAIServingCompletion(OpenAIServing):
249
250
assert request.max_tokens is not None
251
if request.echo and request.max_tokens == 0:
252
+ assert prompt_token_ids is not None
253
assert prompt_text is not None
254
# only return the prompt
255
delta_text = prompt_text
@@ -252,6 +258,7 @@ class OpenAIServingCompletion(OpenAIServing):
258
has_echoed[i] = True
259
elif (request.echo and request.max_tokens > 0
260
and not has_echoed[i]):
261
+ assert prompt_token_ids is not None
262
assert prompt_text is not None
263
assert prompt_logprobs is not None
264
# echo the prompt and first token
@@ -266,11 +273,9 @@ class OpenAIServingCompletion(OpenAIServing):
273
has_echoed[i] = True
274
else:
275
# return just the delta
269
- delta_text = output.text[len(previous_texts[i]):]
270
- delta_token_ids = output.token_ids[
271
- previous_num_tokens[i]:]
272
- out_logprobs = output.logprobs[previous_num_tokens[
273
- i]:] if output.logprobs else None
276
+ delta_text = output.text
277
+ delta_token_ids = output.token_ids
278
+ out_logprobs = output.logprobs
279
280
if request.logprobs is not None:
281
assert out_logprobs is not None, (
@@ -280,13 +285,13 @@ class OpenAIServingCompletion(OpenAIServing):
285
top_logprobs=out_logprobs,
286
num_output_top_logprobs=request.logprobs,
287
tokenizer=tokenizer,
283
- initial_text_offset=len(previous_texts[i]),
288
+ initial_text_offset=previous_text_lens[i],
289
)
290
else:
291
logprobs = None
292
288
- previous_texts[i] = output.text
289
- previous_num_tokens[i] = len(output.token_ids)
293
+ previous_text_lens[i] += len(output.text)
294
+ previous_num_tokens[i] += len(output.token_ids)
295
finish_reason = output.finish_reason
296
stop_reason = output.stop_reason
297
@@ -307,8 +312,8 @@ class OpenAIServingCompletion(OpenAIServing):
312
and request.stream_options.include_usage):
313
if (request.stream_options.continuous_usage_stats
314
or output.finish_reason is not None):
310
- prompt_tokens = len(prompt_token_ids)
311
- completion_tokens = len(output.token_ids)
315
+ prompt_tokens = num_prompt_tokens[prompt_idx]
316
+ completion_tokens = previous_num_tokens[i]
317
usage = UsageInfo(
318
prompt_tokens=prompt_tokens,
319
completion_tokens=completion_tokens,
@@ -356,6 +361,7 @@ class OpenAIServingCompletion(OpenAIServing):
361
362
for final_res in final_res_batch:
363
prompt_token_ids = final_res.prompt_token_ids
364
+ assert prompt_token_ids is not None
365
prompt_logprobs = final_res.prompt_logprobs
366
prompt_text = final_res.prompt
367
@@ -411,9 +417,9 @@ class OpenAIServingCompletion(OpenAIServing):
417
)
418
choices.append(choice_data)
419
420
+ num_generated_tokens += len(output.token_ids)
421
+
422
num_prompt_tokens += len(prompt_token_ids)
415
- num_generated_tokens += sum(
416
- len(output.token_ids) for output in final_res.outputs)
423
424
usage = UsageInfo(
425
prompt_tokens=num_prompt_tokens,
vllm/outputs.py
+55
-24
@@ -5,6 +5,7 @@ from typing import Sequence as GenericSequence
5
from typing import Union
6
7
from vllm.lora.request import LoRARequest
8
+from vllm.sampling_params import RequestOutputKind
9
from vllm.sequence import (PromptLogprobs, RequestMetrics, SampleLogprobs,
10
SequenceGroup, SequenceStatus)
11
@@ -92,7 +93,7 @@ class RequestOutput:
93
self,
94
request_id: str,
95
prompt: Optional[str],
95
- prompt_token_ids: List[int],
96
+ prompt_token_ids: Optional[List[int]],
97
prompt_logprobs: Optional[PromptLogprobs],
98
outputs: List[CompletionOutput],
99
finished: bool,
@@ -113,19 +114,26 @@ class RequestOutput:
114
self.encoder_prompt_token_ids = encoder_prompt_token_ids
115
116
@classmethod
116
- def from_seq_group(cls, seq_group: SequenceGroup) -> "RequestOutput":
117
- if seq_group.sampling_params is None:
117
+ def from_seq_group(cls,
118
+ seq_group: SequenceGroup) -> Optional["RequestOutput"]:
119
+ sampling_params = seq_group.sampling_params
120
+ if sampling_params is None:
121
raise ValueError(
122
"Sampling parameters are missing for a CompletionRequest.")
123
+ finished = seq_group.is_finished()
124
+ if sampling_params.output_kind == RequestOutputKind.FINAL_ONLY and (
125
+ not finished):
126
+ return None
127
+
128
seqs = seq_group.get_seqs()
129
if len(seqs) == 1:
130
top_n_seqs = seqs
131
else:
132
# Get the top-n sequences.
125
- n = seq_group.sampling_params.n
126
- if seq_group.sampling_params.use_beam_search:
133
+ n = sampling_params.n
134
+ if sampling_params.use_beam_search:
135
sorting_key = lambda seq: seq.get_beam_search_score(
128
- seq_group.sampling_params.length_penalty)
136
+ sampling_params.length_penalty)
137
else:
138
sorting_key = lambda seq: seq.get_cumulative_logprob()
139
sorted_seqs = sorted(seqs, key=sorting_key, reverse=True)
@@ -135,26 +143,49 @@ class RequestOutput:
143
# NOTE: We need omit logprobs here explicitly because the sequence
144
# always has the logprobs of the sampled tokens even if the
145
# logprobs are not requested.
138
- include_logprobs = seq_group.sampling_params.logprobs is not None
139
- text_buffer_length = seq_group.sampling_params.output_text_buffer_length
140
- outputs = [
141
- CompletionOutput(
142
- seqs.index(seq),
143
- seq.get_output_text_to_return(text_buffer_length),
144
- seq.data._output_token_ids,
145
- seq.get_cumulative_logprob() if include_logprobs else None,
146
- seq.output_logprobs if include_logprobs else None,
147
- SequenceStatus.get_finished_reason(seq.status),
148
- seq.stop_reason) for seq in top_n_seqs
149
- ]
146
+ include_logprobs = sampling_params.logprobs is not None
147
+ text_buffer_length = sampling_params.output_text_buffer_length
148
+ delta = sampling_params.output_kind == RequestOutputKind.DELTA
149
+
150
+ outputs = []
151
+ include_prompt = True
152
+ for seq in top_n_seqs:
153
+ output_text = seq.get_output_text_to_return(
154
+ text_buffer_length, delta)
155
+ output_token_ids = seq.get_output_token_ids_to_return(delta)
156
+ output_logprobs = seq.output_logprobs if include_logprobs else None
157
+
158
+ if delta:
159
+ # Slice logprobs delta if applicable
160
+ if output_logprobs:
161
+ output_logprobs = output_logprobs[-len(output_token_ids):]
162
+ # Don't include prompt if this is after the first output
163
+ # containing decode token ids
164
+ if include_prompt and seq.get_output_len() > len(
165
+ output_token_ids):
166
+ include_prompt = False
167
+
168
+ outputs.append(
169
+ CompletionOutput(
170
+ seqs.index(seq), output_text, output_token_ids,
171
+ seq.get_cumulative_logprob() if include_logprobs else None,
172
+ output_logprobs,
173
+ SequenceStatus.get_finished_reason(seq.status),
174
+ seq.stop_reason))
175
176
# Every sequence in the sequence group should have the same prompt.
152
- prompt = seq_group.prompt
153
- prompt_token_ids = seq_group.prompt_token_ids
154
- encoder_prompt = seq_group.encoder_prompt
155
- encoder_prompt_token_ids = seq_group.encoder_prompt_token_ids
156
- prompt_logprobs = seq_group.prompt_logprobs
157
- finished = seq_group.is_finished()
177
+ if include_prompt:
178
+ prompt = seq_group.prompt
179
+ prompt_token_ids = seq_group.prompt_token_ids
180
+ encoder_prompt = seq_group.encoder_prompt
181
+ encoder_prompt_token_ids = seq_group.encoder_prompt_token_ids
182
+ prompt_logprobs = seq_group.prompt_logprobs
183
+ else:
184
+ prompt = None
185
+ prompt_token_ids = None
186
+ encoder_prompt = None
187
+ encoder_prompt_token_ids = None
188
+ prompt_logprobs = None
189
finished_time = time.time() if finished else None
190
seq_group.set_finished_time(finished_time)
191
return cls(seq_group.request_id,
vllm/sampling_params.py
+16
-1
@@ -1,6 +1,6 @@
1
"""Sampling parameters for text generation."""
2
import copy
3
-from enum import IntEnum
3
+from enum import Enum, IntEnum
4
from functools import cached_property
5
from typing import Any, Callable, Dict, List, Optional, Set, Union
6
@@ -33,6 +33,15 @@ first argument, and returns a modified tensor of logits
33
to sample from."""
34
35
36
+class RequestOutputKind(Enum):
37
+ # Return entire output so far in every RequestOutput
38
+ CUMULATIVE = 0
39
+ # Return only deltas in each RequestOutput
40
+ DELTA = 1
41
+ # Do not return intermediate RequestOuputs
42
+ FINAL_ONLY = 2
43
+
44
+
45
class SamplingParams(
46
msgspec.Struct,
47
omit_defaults=True, # type: ignore[call-arg]
@@ -147,6 +156,7 @@ class SamplingParams(
156
logits_processors: Optional[Any] = None
157
include_stop_str_in_output: bool = False
158
truncate_prompt_tokens: Optional[Annotated[int, msgspec.Meta(ge=1)]] = None
159
+ output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE
160
161
# The below fields are not supposed to be used as an input.
162
# They are set in post_init.
@@ -182,6 +192,7 @@ class SamplingParams(
192
logits_processors: Optional[List[LogitsProcessor]] = None,
193
truncate_prompt_tokens: Optional[Annotated[int,
194
msgspec.Meta(ge=1)]] = None,
195
+ output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE,
196
) -> "SamplingParams":
197
return SamplingParams(
198
n=1 if n is None else n,
@@ -213,6 +224,7 @@ class SamplingParams(
224
spaces_between_special_tokens=spaces_between_special_tokens,
225
logits_processors=logits_processors,
226
truncate_prompt_tokens=truncate_prompt_tokens,
227
+ output_kind=output_kind,
228
)
229
230
def __post_init__(self) -> None:
@@ -317,6 +329,9 @@ class SamplingParams(
329
raise ValueError(
330
"stop strings are only supported when detokenize is True. "
331
"Set detokenize=True to use stop.")
332
+ if self.best_of != self.n and self.output_kind == (
333
+ RequestOutputKind.DELTA):
334
+ raise ValueError("best_of must equal n to use output_kind=DELTA")
335
336
def _verify_beam_search(self) -> None:
337
if self.best_of == 1:
vllm/sequence.py
+34
-5
@@ -5,8 +5,9 @@ from abc import ABC, abstractmethod
5
from array import array
6
from collections import defaultdict
7
from dataclasses import dataclass
8
-from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Mapping,
9
- Optional, Set, Tuple, Union, cast)
8
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Mapping, Optional
9
+from typing import Sequence as GenericSequence
10
+from typing import Set, Tuple, Union, cast
11
12
import msgspec
13
import torch
@@ -407,6 +408,10 @@ class Sequence:
408
self.status = SequenceStatus.WAITING
409
self.stop_reason: Union[int, str, None] = None
410
411
+ # These are used to keep track of delta outputs
412
+ self._last_token_ids_offset: int = 0
413
+ self._last_output_text_offset: int = 0
414
+
415
# Used for incremental detokenization
416
self.prefix_offset = 0
417
self.read_offset = 0
@@ -462,11 +467,35 @@ class Sequence:
467
return self.prompt_adapter_request.prompt_adapter_id \
468
if self.prompt_adapter_request else 0
469
465
- def get_output_text_to_return(self, buffer_length: int):
470
+ def get_output_text_to_return(self, buffer_length: int,
471
+ delta: bool) -> str:
472
+ """If delta is True, only new text since the last call to
473
+ this method is returned"""
474
+
475
# We return the full output text if the sequence is finished.
476
truncate = buffer_length and not self.is_finished()
468
- return self.output_text[:-buffer_length] if truncate else (
469
- self.output_text)
477
+ if not delta:
478
+ return self.output_text[:-buffer_length] if truncate else (
479
+ self.output_text)
480
+ length = len(self.output_text) - buffer_length
481
+ last_offset = self._last_output_text_offset
482
+ if last_offset < length:
483
+ self._last_output_text_offset = length
484
+ return self.output_text[last_offset:length]
485
+ return ""
486
+
487
+ def get_output_token_ids_to_return(self,
488
+ delta: bool) -> GenericSequence[int]:
489
+ """If delta is True, only new tokens since the last call to
490
+ this method are returned"""
491
+ if not delta:
492
+ return self.get_output_token_ids()
493
+ length = self.get_output_len()
494
+ last_offset = self._last_token_ids_offset
495
+ if last_offset < length:
496
+ self._last_token_ids_offset = length
497
+ return self.data._output_token_ids[last_offset:]
498
+ return ()
499
500
def hash_of_block(self, logical_idx: int) -> int:
501
# TODO This can produce incorrect hash when block size > prompt size