@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 47db6ec831
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Frontend] Add per-request number of cached token stats (#10174)

zifeitong committed Nov 12, 2024 at 08:42 UTC 47db6ec8310129699a62567b61d8ed380636b053
9 files changed +89 -23
tests/prefix_caching/test_prefix_caching.py
+22 -2
@@ -27,6 +27,7 @@ UNSTABLE_PROMPT_SEQUENCE = [
27 @pytest.mark.parametrize("dtype", ["half"])
28 @pytest.mark.parametrize("max_tokens", [5])
29 @pytest.mark.parametrize("cached_position", [0, 1])
30 +@pytest.mark.parametrize("block_size", [16])
31 def test_mixed_requests(
32 hf_runner,
33 vllm_runner,
@@ -36,11 +37,12 @@ def test_mixed_requests(
37 dtype: str,
38 max_tokens: int,
39 cached_position: int,
40 + block_size: int,
41 monkeypatch,
42 ) -> None:
43 """
44 Test the case when some sequences have the prefix cache hit
43 - and the others don't. The cached position determines where
45 + and the others don't. The cached position determines where
46 the sequence is at among the batch of prefills.
47 """
48 override_backend_env_variable(monkeypatch, backend)
@@ -53,12 +55,30 @@ def test_mixed_requests(
55 model,
56 dtype=dtype,
57 enable_prefix_caching=True,
58 + block_size=block_size,
59 ) as vllm_model:
60 # Run the first prompt so the cache is populated
61 vllm_outputs = vllm_model.generate_greedy([cached_prompt], max_tokens)
62
63 # Run all the promopts
61 - vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
64 + greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
65 + req_outputs = vllm_model.model.generate(example_prompts, greedy_params)
66 +
67 + # Verify number of cached tokens
68 + for i in range(len(req_outputs)):
69 + if i == cached_position:
70 + expected_num_cached_tokens = (
71 + len(req_outputs[i].prompt_token_ids) //
72 + block_size) * block_size
73 + else:
74 + expected_num_cached_tokens = 0
75 + assert req_outputs[
76 + i].num_cached_tokens == expected_num_cached_tokens
77 +
78 + vllm_outputs = [
79 + (output.prompt_token_ids + list(output.outputs[0].token_ids),
80 + output.prompt + output.outputs[0].text) for output in req_outputs
81 + ]
82
83 check_outputs_equal(
84 outputs_0_lst=hf_outputs,
vllm/entrypoints/openai/api_server.py
+1
@@ -540,6 +540,7 @@ def init_app_state(
540 return_tokens_as_token_ids=args.return_tokens_as_token_ids,
541 enable_auto_tools=args.enable_auto_tool_choice,
542 tool_parser=args.tool_call_parser,
543 + enable_prompt_tokens_details=args.enable_prompt_tokens_details,
544 ) if model_config.task == "generate" else None
545 state.openai_serving_completion = OpenAIServingCompletion(
546 engine_client,
vllm/entrypoints/openai/cli_args.py
+5
@@ -228,6 +228,11 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
228 default=False,
229 help="Disable FastAPI's OpenAPI schema, Swagger UI, and ReDoc endpoint"
230 )
231 + parser.add_argument(
232 + "--enable-prompt-tokens-details",
233 + action='store_true',
234 + default=False,
235 + help="If set to True, enable prompt_tokens_details in usage.")
236
237 return parser
238
vllm/entrypoints/openai/protocol.py
+5
@@ -99,10 +99,15 @@ class ModelList(OpenAIBaseModel):
99 data: List[ModelCard] = Field(default_factory=list)
100
101
102 +class PromptTokenUsageInfo(OpenAIBaseModel):
103 + cached_tokens: Optional[int] = None
104 +
105 +
106 class UsageInfo(OpenAIBaseModel):
107 prompt_tokens: int = 0
108 total_tokens: int = 0
109 completion_tokens: Optional[int] = 0
110 + prompt_tokens_details: Optional[PromptTokenUsageInfo] = None
111
112
113 class RequestResponseMetadata(BaseModel):
vllm/entrypoints/openai/run_batch.py
+6
@@ -78,6 +78,11 @@ def parse_args():
78 help="Port number for the Prometheus metrics server "
79 "(only needed if enable-metrics is set).",
80 )
81 + parser.add_argument(
82 + "--enable-prompt-tokens-details",
83 + action='store_true',
84 + default=False,
85 + help="If set to True, enable prompt_tokens_details in usage.")
86
87 return parser.parse_args()
88
@@ -217,6 +222,7 @@ async def main(args):
222 prompt_adapters=None,
223 request_logger=request_logger,
224 chat_template=None,
225 + enable_prompt_tokens_details=args.enable_prompt_tokens_details,
226 ) if model_config.task == "generate" else None
227 openai_serving_embedding = OpenAIServingEmbedding(
228 engine,
vllm/entrypoints/openai/serving_chat.py
+22 -13
@@ -18,8 +18,8 @@ from vllm.entrypoints.openai.protocol import (
18 ChatCompletionRequest, ChatCompletionResponse,
19 ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
20 ChatCompletionStreamResponse, ChatMessage, DeltaFunctionCall, DeltaMessage,
21 - DeltaToolCall, ErrorResponse, FunctionCall, RequestResponseMetadata,
22 - ToolCall, UsageInfo)
21 + DeltaToolCall, ErrorResponse, FunctionCall, PromptTokenUsageInfo,
22 + RequestResponseMetadata, ToolCall, UsageInfo)
23 from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
24 LoRAModulePath,
25 OpenAIServing,
@@ -49,7 +49,8 @@ class OpenAIServingChat(OpenAIServing):
49 chat_template: Optional[str],
50 return_tokens_as_token_ids: bool = False,
51 enable_auto_tools: bool = False,
52 - tool_parser: Optional[str] = None):
52 + tool_parser: Optional[str] = None,
53 + enable_prompt_tokens_details: bool = False):
54 super().__init__(engine_client=engine_client,
55 model_config=model_config,
56 base_model_paths=base_model_paths,
@@ -80,6 +81,8 @@ class OpenAIServingChat(OpenAIServing):
81 f"tool_parser:'{tool_parser}' which has not "
82 "been registered") from e
83
84 + self.enable_prompt_tokens_details = enable_prompt_tokens_details
85 +
86 async def create_chat_completion(
87 self,
88 request: ChatCompletionRequest,
@@ -252,6 +255,7 @@ class OpenAIServingChat(OpenAIServing):
255 previous_num_tokens = [0] * num_choices
256 finish_reason_sent = [False] * num_choices
257 num_prompt_tokens = 0
258 + num_cached_tokens = None
259
260 if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
261 tool_choice_function_name = request.tool_choice.function.name
@@ -305,6 +309,7 @@ class OpenAIServingChat(OpenAIServing):
309 # the result_generator, it needs to be sent as the FIRST
310 # response (by the try...catch).
311 if first_iteration:
312 + num_cached_tokens = res.num_cached_tokens
313 # Send first response for each request.n (index) with
314 # the role
315 role = self.get_chat_request_role(request)
@@ -530,11 +535,13 @@ class OpenAIServingChat(OpenAIServing):
535 # is sent, send the usage
536 if include_usage:
537 completion_tokens = sum(previous_num_tokens)
533 - final_usage = UsageInfo(
534 - prompt_tokens=num_prompt_tokens,
535 - completion_tokens=completion_tokens,
536 - total_tokens=num_prompt_tokens + completion_tokens,
537 - )
538 + final_usage = UsageInfo(prompt_tokens=num_prompt_tokens,
539 + completion_tokens=completion_tokens,
540 + total_tokens=num_prompt_tokens +
541 + completion_tokens)
542 + if self.enable_prompt_tokens_details and num_cached_tokens:
543 + final_usage.prompt_tokens_details = PromptTokenUsageInfo(
544 + cached_tokens=num_cached_tokens)
545
546 final_usage_chunk = ChatCompletionStreamResponse(
547 id=request_id,
@@ -702,11 +709,13 @@ class OpenAIServingChat(OpenAIServing):
709 num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
710 num_generated_tokens = sum(
711 len(output.token_ids) for output in final_res.outputs)
705 - usage = UsageInfo(
706 - prompt_tokens=num_prompt_tokens,
707 - completion_tokens=num_generated_tokens,
708 - total_tokens=num_prompt_tokens + num_generated_tokens,
709 - )
712 + usage = UsageInfo(prompt_tokens=num_prompt_tokens,
713 + completion_tokens=num_generated_tokens,
714 + total_tokens=num_prompt_tokens +
715 + num_generated_tokens)
716 + if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
717 + usage.prompt_tokens_details = PromptTokenUsageInfo(
718 + cached_tokens=final_res.num_cached_tokens)
719
720 request_metadata.final_usage_info = usage
721
vllm/outputs.py
+13 -6
@@ -83,10 +83,11 @@ class RequestOutput:
83 finished: Whether the whole request is finished.
84 metrics: Metrics associated with the request.
85 lora_request: The LoRA request that was used to generate the output.
86 - encoder_prompt: The encoder prompt string of the request;
87 - None if decoder-only
88 - encoder_prompt_token_ids: The token IDs of the encoder prompt;
89 - None if decoder-only
86 + encoder_prompt: The encoder prompt string of the request.
87 + None if decoder-only.
88 + encoder_prompt_token_ids: The token IDs of the encoder prompt.
89 + None if decoder-only.
90 + num_cached_tokens: The number of tokens with prefix cache hit.
91 """
92
93 def __init__(
@@ -101,6 +102,7 @@ class RequestOutput:
102 lora_request: Optional[LoRARequest] = None,
103 encoder_prompt: Optional[str] = None,
104 encoder_prompt_token_ids: Optional[List[int]] = None,
105 + num_cached_tokens: Optional[int] = None,
106 ) -> None:
107 self.request_id = request_id
108 self.prompt = prompt
@@ -112,6 +114,7 @@ class RequestOutput:
114 self.lora_request = lora_request
115 self.encoder_prompt = encoder_prompt
116 self.encoder_prompt_token_ids = encoder_prompt_token_ids
117 + self.num_cached_tokens = num_cached_tokens
118
119 @classmethod
120 def new(
@@ -192,6 +195,8 @@ class RequestOutput:
195
196 outputs = []
197 include_prompt = True
198 + # num_cached_tokens should be the same for all the sequences
199 + num_cached_tokens = None
200 for i, seq in enumerate(top_n_seqs):
201 output_text = seq.get_output_text_to_return(
202 text_buffer_length, delta)
@@ -199,6 +204,7 @@ class RequestOutput:
204 output_token_ids = seq.get_output_token_ids_to_return(delta)
205 num_output_tokens = 1 if isinstance(output_token_ids,
206 int) else len(output_token_ids)
207 + num_cached_tokens = seq.data.get_num_cached_tokens()
208
209 output_logprobs = seq.output_logprobs if include_logprobs else None
210
@@ -272,7 +278,7 @@ class RequestOutput:
278 init_args = (seq_group.request_id, prompt, prompt_token_ids,
279 prompt_logprobs, outputs, finished, seq_group.metrics,
280 seq_group.lora_request, encoder_prompt,
275 - encoder_prompt_token_ids)
281 + encoder_prompt_token_ids, num_cached_tokens)
282
283 if use_cache:
284 request_output = seq_group.cached_request_output
@@ -293,7 +299,8 @@ class RequestOutput:
299 f"outputs={self.outputs}, "
300 f"finished={self.finished}, "
301 f"metrics={self.metrics}, "
296 - f"lora_request={self.lora_request})")
302 + f"lora_request={self.lora_request}, "
303 + f"num_cached_tokens={self.num_cached_tokens})")
304
305
306 class EmbeddingRequestOutput:
vllm/sequence.py
+12 -2
@@ -167,6 +167,8 @@ class SequenceData(msgspec.Struct,
167 ...] = msgspec.field(default_factory=tuple)
168 # The number of tokens that are computed (that run against the model).
169 _num_computed_tokens: int = 0
170 + # The number of tokens with prefix cache hit.
171 + _num_cached_tokens: int = 0
172 _stage: SequenceStage = SequenceStage.PREFILL
173 _cached_all_token_ids: List[int] = msgspec.field(default_factory=list)
174
@@ -323,6 +325,14 @@ class SequenceData(msgspec.Struct,
325 if self.get_num_uncomputed_tokens() == 0:
326 self._stage = SequenceStage.DECODE
327
328 + def get_num_cached_tokens(self) -> int:
329 + """Return the number of tokens with prefix cache hit."""
330 + return self._num_cached_tokens
331 +
332 + def update_num_cached_tokens(self, num_cached_tokens: int):
333 + """Update the number of tokens with prefix cache hit."""
334 + self._num_cached_tokens = num_cached_tokens
335 +
336 def reset_state_for_recompute(self) -> None:
337 """Reset the number of computed tokens from this sequence. It is
338 supposed to be called when a sequence needs to be started from
@@ -379,7 +389,7 @@ class SequenceData(msgspec.Struct,
389
390 class Sequence:
391 """Stores the data, status, and block information of a sequence.
382 -
392 +
393 The sequence is constructed from the :data:`DecoderOnlyInputs`
394 (for decoder-only) or :data:`EncoderDecoderInputs` (for encoder-decoder)
395 instance passed in through the :code:`inputs` constructor argument.
@@ -906,7 +916,7 @@ class SequenceGroupMetadata(
916 multi_modal_data: Multi modal data.
917 mm_processor_kwargs: Multimodal input processor / mapper overrides.
918 encoder_seq_data: Optional sequence data for encoder prompt
909 - (SequenceGroup.encoder_seq). Should be None
919 + (SequenceGroup.encoder_seq). Should be None
920 unless you are working with an encoder/decoder
921 model.
922 cross_block_table: Optional cross-attention block table associated
vllm/worker/model_runner.py
+3
@@ -542,6 +542,9 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
542 # this may be larger than the sequence length if chunked
543 # prefill is enabled.
544 prefix_cache_len = len(computed_block_nums) * self.block_size
545 + seq_group_metadata.seq_data[inter_data.seq_ids[
546 + seq_idx]].update_num_cached_tokens(prefix_cache_len)
547 +
548 # The number of so far computed prompt tokens in this sequence.
549 context_len = inter_data.context_lens[seq_idx]
550 # The total number of prompt tokens in this sequence.