A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Frontend] Add per-request number of cached token stats (#10174)
zifeitong committed
Nov 12, 2024 at 08:42 UTC
47db6ec8310129699a62567b61d8ed380636b053
9 files changed
+89
-23
tests/prefix_caching/test_prefix_caching.py
+22
-2
@@ -27,6 +27,7 @@ UNSTABLE_PROMPT_SEQUENCE = [
27
@pytest.mark.parametrize("dtype", ["half"])
28
@pytest.mark.parametrize("max_tokens", [5])
29
@pytest.mark.parametrize("cached_position", [0, 1])
30
+@pytest.mark.parametrize("block_size", [16])
31
def test_mixed_requests(
32
hf_runner,
33
vllm_runner,
@@ -36,11 +37,12 @@ def test_mixed_requests(
37
dtype: str,
38
max_tokens: int,
39
cached_position: int,
40
+ block_size: int,
41
monkeypatch,
42
) -> None:
43
"""
44
Test the case when some sequences have the prefix cache hit
43
- and the others don't. The cached position determines where
45
+ and the others don't. The cached position determines where
46
the sequence is at among the batch of prefills.
47
"""
48
override_backend_env_variable(monkeypatch, backend)
@@ -53,12 +55,30 @@ def test_mixed_requests(
55
model,
56
dtype=dtype,
57
enable_prefix_caching=True,
58
+ block_size=block_size,
59
) as vllm_model:
60
# Run the first prompt so the cache is populated
61
vllm_outputs = vllm_model.generate_greedy([cached_prompt], max_tokens)
62
63
# Run all the promopts
61
- vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
64
+ greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
65
+ req_outputs = vllm_model.model.generate(example_prompts, greedy_params)
66
+
67
+ # Verify number of cached tokens
68
+ for i in range(len(req_outputs)):
69
+ if i == cached_position:
70
+ expected_num_cached_tokens = (
71
+ len(req_outputs[i].prompt_token_ids) //
72
+ block_size) * block_size
73
+ else:
74
+ expected_num_cached_tokens = 0
75
+ assert req_outputs[
76
+ i].num_cached_tokens == expected_num_cached_tokens
77
+
78
+ vllm_outputs = [
79
+ (output.prompt_token_ids + list(output.outputs[0].token_ids),
80
+ output.prompt + output.outputs[0].text) for output in req_outputs
81
+ ]
82
83
check_outputs_equal(
84
outputs_0_lst=hf_outputs,
vllm/entrypoints/openai/api_server.py
+1
@@ -540,6 +540,7 @@ def init_app_state(
540
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
541
enable_auto_tools=args.enable_auto_tool_choice,
542
tool_parser=args.tool_call_parser,
543
+ enable_prompt_tokens_details=args.enable_prompt_tokens_details,
544
) if model_config.task == "generate" else None
545
state.openai_serving_completion = OpenAIServingCompletion(
546
engine_client,
vllm/entrypoints/openai/cli_args.py
+5
@@ -228,6 +228,11 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
228
default=False,
229
help="Disable FastAPI's OpenAPI schema, Swagger UI, and ReDoc endpoint"
230
)
231
+ parser.add_argument(
232
+ "--enable-prompt-tokens-details",
233
+ action='store_true',
234
+ default=False,
235
+ help="If set to True, enable prompt_tokens_details in usage.")
236
237
return parser
238
vllm/entrypoints/openai/protocol.py
+5
@@ -99,10 +99,15 @@ class ModelList(OpenAIBaseModel):
99
data: List[ModelCard] = Field(default_factory=list)
100
101
102
+class PromptTokenUsageInfo(OpenAIBaseModel):
103
+ cached_tokens: Optional[int] = None
104
+
105
+
106
class UsageInfo(OpenAIBaseModel):
107
prompt_tokens: int = 0
108
total_tokens: int = 0
109
completion_tokens: Optional[int] = 0
110
+ prompt_tokens_details: Optional[PromptTokenUsageInfo] = None
111
112
113
class RequestResponseMetadata(BaseModel):
vllm/entrypoints/openai/run_batch.py
+6
@@ -78,6 +78,11 @@ def parse_args():
78
help="Port number for the Prometheus metrics server "
79
"(only needed if enable-metrics is set).",
80
)
81
+ parser.add_argument(
82
+ "--enable-prompt-tokens-details",
83
+ action='store_true',
84
+ default=False,
85
+ help="If set to True, enable prompt_tokens_details in usage.")
86
87
return parser.parse_args()
88
@@ -217,6 +222,7 @@ async def main(args):
222
prompt_adapters=None,
223
request_logger=request_logger,
224
chat_template=None,
225
+ enable_prompt_tokens_details=args.enable_prompt_tokens_details,
226
) if model_config.task == "generate" else None
227
openai_serving_embedding = OpenAIServingEmbedding(
228
engine,
vllm/entrypoints/openai/serving_chat.py
+22
-13
@@ -18,8 +18,8 @@ from vllm.entrypoints.openai.protocol import (
18
ChatCompletionRequest, ChatCompletionResponse,
19
ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
20
ChatCompletionStreamResponse, ChatMessage, DeltaFunctionCall, DeltaMessage,
21
- DeltaToolCall, ErrorResponse, FunctionCall, RequestResponseMetadata,
22
- ToolCall, UsageInfo)
21
+ DeltaToolCall, ErrorResponse, FunctionCall, PromptTokenUsageInfo,
22
+ RequestResponseMetadata, ToolCall, UsageInfo)
23
from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
24
LoRAModulePath,
25
OpenAIServing,
@@ -49,7 +49,8 @@ class OpenAIServingChat(OpenAIServing):
49
chat_template: Optional[str],
50
return_tokens_as_token_ids: bool = False,
51
enable_auto_tools: bool = False,
52
- tool_parser: Optional[str] = None):
52
+ tool_parser: Optional[str] = None,
53
+ enable_prompt_tokens_details: bool = False):
54
super().__init__(engine_client=engine_client,
55
model_config=model_config,
56
base_model_paths=base_model_paths,
@@ -80,6 +81,8 @@ class OpenAIServingChat(OpenAIServing):
81
f"tool_parser:'{tool_parser}' which has not "
82
"been registered") from e
83
84
+ self.enable_prompt_tokens_details = enable_prompt_tokens_details
85
+
86
async def create_chat_completion(
87
self,
88
request: ChatCompletionRequest,
@@ -252,6 +255,7 @@ class OpenAIServingChat(OpenAIServing):
255
previous_num_tokens = [0] * num_choices
256
finish_reason_sent = [False] * num_choices
257
num_prompt_tokens = 0
258
+ num_cached_tokens = None
259
260
if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
261
tool_choice_function_name = request.tool_choice.function.name
@@ -305,6 +309,7 @@ class OpenAIServingChat(OpenAIServing):
309
# the result_generator, it needs to be sent as the FIRST
310
# response (by the try...catch).
311
if first_iteration:
312
+ num_cached_tokens = res.num_cached_tokens
313
# Send first response for each request.n (index) with
314
# the role
315
role = self.get_chat_request_role(request)
@@ -530,11 +535,13 @@ class OpenAIServingChat(OpenAIServing):
535
# is sent, send the usage
536
if include_usage:
537
completion_tokens = sum(previous_num_tokens)
533
- final_usage = UsageInfo(
534
- prompt_tokens=num_prompt_tokens,
535
- completion_tokens=completion_tokens,
536
- total_tokens=num_prompt_tokens + completion_tokens,
537
- )
538
+ final_usage = UsageInfo(prompt_tokens=num_prompt_tokens,
539
+ completion_tokens=completion_tokens,
540
+ total_tokens=num_prompt_tokens +
541
+ completion_tokens)
542
+ if self.enable_prompt_tokens_details and num_cached_tokens:
543
+ final_usage.prompt_tokens_details = PromptTokenUsageInfo(
544
+ cached_tokens=num_cached_tokens)
545
546
final_usage_chunk = ChatCompletionStreamResponse(
547
id=request_id,
@@ -702,11 +709,13 @@ class OpenAIServingChat(OpenAIServing):
709
num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
710
num_generated_tokens = sum(
711
len(output.token_ids) for output in final_res.outputs)
705
- usage = UsageInfo(
706
- prompt_tokens=num_prompt_tokens,
707
- completion_tokens=num_generated_tokens,
708
- total_tokens=num_prompt_tokens + num_generated_tokens,
709
- )
712
+ usage = UsageInfo(prompt_tokens=num_prompt_tokens,
713
+ completion_tokens=num_generated_tokens,
714
+ total_tokens=num_prompt_tokens +
715
+ num_generated_tokens)
716
+ if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
717
+ usage.prompt_tokens_details = PromptTokenUsageInfo(
718
+ cached_tokens=final_res.num_cached_tokens)
719
720
request_metadata.final_usage_info = usage
721
vllm/outputs.py
+13
-6
@@ -83,10 +83,11 @@ class RequestOutput:
83
finished: Whether the whole request is finished.
84
metrics: Metrics associated with the request.
85
lora_request: The LoRA request that was used to generate the output.
86
- encoder_prompt: The encoder prompt string of the request;
87
- None if decoder-only
88
- encoder_prompt_token_ids: The token IDs of the encoder prompt;
89
- None if decoder-only
86
+ encoder_prompt: The encoder prompt string of the request.
87
+ None if decoder-only.
88
+ encoder_prompt_token_ids: The token IDs of the encoder prompt.
89
+ None if decoder-only.
90
+ num_cached_tokens: The number of tokens with prefix cache hit.
91
"""
92
93
def __init__(
@@ -101,6 +102,7 @@ class RequestOutput:
102
lora_request: Optional[LoRARequest] = None,
103
encoder_prompt: Optional[str] = None,
104
encoder_prompt_token_ids: Optional[List[int]] = None,
105
+ num_cached_tokens: Optional[int] = None,
106
) -> None:
107
self.request_id = request_id
108
self.prompt = prompt
@@ -112,6 +114,7 @@ class RequestOutput:
114
self.lora_request = lora_request
115
self.encoder_prompt = encoder_prompt
116
self.encoder_prompt_token_ids = encoder_prompt_token_ids
117
+ self.num_cached_tokens = num_cached_tokens
118
119
@classmethod
120
def new(
@@ -192,6 +195,8 @@ class RequestOutput:
195
196
outputs = []
197
include_prompt = True
198
+ # num_cached_tokens should be the same for all the sequences
199
+ num_cached_tokens = None
200
for i, seq in enumerate(top_n_seqs):
201
output_text = seq.get_output_text_to_return(
202
text_buffer_length, delta)
@@ -199,6 +204,7 @@ class RequestOutput:
204
output_token_ids = seq.get_output_token_ids_to_return(delta)
205
num_output_tokens = 1 if isinstance(output_token_ids,
206
int) else len(output_token_ids)
207
+ num_cached_tokens = seq.data.get_num_cached_tokens()
208
209
output_logprobs = seq.output_logprobs if include_logprobs else None
210
@@ -272,7 +278,7 @@ class RequestOutput:
278
init_args = (seq_group.request_id, prompt, prompt_token_ids,
279
prompt_logprobs, outputs, finished, seq_group.metrics,
280
seq_group.lora_request, encoder_prompt,
275
- encoder_prompt_token_ids)
281
+ encoder_prompt_token_ids, num_cached_tokens)
282
283
if use_cache:
284
request_output = seq_group.cached_request_output
@@ -293,7 +299,8 @@ class RequestOutput:
299
f"outputs={self.outputs}, "
300
f"finished={self.finished}, "
301
f"metrics={self.metrics}, "
296
- f"lora_request={self.lora_request})")
302
+ f"lora_request={self.lora_request}, "
303
+ f"num_cached_tokens={self.num_cached_tokens})")
304
305
306
class EmbeddingRequestOutput:
vllm/sequence.py
+12
-2
@@ -167,6 +167,8 @@ class SequenceData(msgspec.Struct,
167
...] = msgspec.field(default_factory=tuple)
168
# The number of tokens that are computed (that run against the model).
169
_num_computed_tokens: int = 0
170
+ # The number of tokens with prefix cache hit.
171
+ _num_cached_tokens: int = 0
172
_stage: SequenceStage = SequenceStage.PREFILL
173
_cached_all_token_ids: List[int] = msgspec.field(default_factory=list)
174
@@ -323,6 +325,14 @@ class SequenceData(msgspec.Struct,
325
if self.get_num_uncomputed_tokens() == 0:
326
self._stage = SequenceStage.DECODE
327
328
+ def get_num_cached_tokens(self) -> int:
329
+ """Return the number of tokens with prefix cache hit."""
330
+ return self._num_cached_tokens
331
+
332
+ def update_num_cached_tokens(self, num_cached_tokens: int):
333
+ """Update the number of tokens with prefix cache hit."""
334
+ self._num_cached_tokens = num_cached_tokens
335
+
336
def reset_state_for_recompute(self) -> None:
337
"""Reset the number of computed tokens from this sequence. It is
338
supposed to be called when a sequence needs to be started from
@@ -379,7 +389,7 @@ class SequenceData(msgspec.Struct,
389
390
class Sequence:
391
"""Stores the data, status, and block information of a sequence.
382
-
392
+
393
The sequence is constructed from the :data:`DecoderOnlyInputs`
394
(for decoder-only) or :data:`EncoderDecoderInputs` (for encoder-decoder)
395
instance passed in through the :code:`inputs` constructor argument.
@@ -906,7 +916,7 @@ class SequenceGroupMetadata(
916
multi_modal_data: Multi modal data.
917
mm_processor_kwargs: Multimodal input processor / mapper overrides.
918
encoder_seq_data: Optional sequence data for encoder prompt
909
- (SequenceGroup.encoder_seq). Should be None
919
+ (SequenceGroup.encoder_seq). Should be None
920
unless you are working with an encoder/decoder
921
model.
922
cross_block_table: Optional cross-attention block table associated
vllm/worker/model_runner.py
+3
@@ -542,6 +542,9 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
542
# this may be larger than the sequence length if chunked
543
# prefill is enabled.
544
prefix_cache_len = len(computed_block_nums) * self.block_size
545
+ seq_group_metadata.seq_data[inter_data.seq_ids[
546
+ seq_idx]].update_num_cached_tokens(prefix_cache_len)
547
+
548
# The number of so far computed prompt tokens in this sequence.
549
context_len = inter_data.context_lens[seq_idx]
550
# The total number of prompt tokens in this sequence.