A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
add(v1): RequestStatesStats to RequestOutput (#24947)
Signed-off-by: huijjj <huijong.jeong@squeezebits.com>
HUIJONG JEONG committed
Oct 3, 2025 at 17:56 UTC
3e70e3d4d5616b854959ac16163924f9ecde53d4
3 files changed
+24
-11
tests/entrypoints/llm/test_generate.py
+13
@@ -86,3 +86,16 @@ def test_max_model_len():
86
# It can be less if generation finishes due to other reasons (e.g., EOS)
87
# before reaching the absolute model length limit.
88
assert num_total_tokens <= max_model_len
89
+
90
+
91
+def test_log_stats():
92
+ llm = LLM(
93
+ model=MODEL_NAME,
94
+ disable_log_stats=False,
95
+ gpu_memory_utilization=0.10,
96
+ enforce_eager=True, # reduce test time
97
+ )
98
+ outputs = llm.generate(PROMPTS, sampling_params=None)
99
+
100
+ # disable_log_stats is False, every output should have metrics
101
+ assert all(output.metrics is not None for output in outputs)
vllm/outputs.py
+2
-1
@@ -14,6 +14,7 @@ from vllm.logprobs import PromptLogprobs, SampleLogprobs
14
from vllm.lora.request import LoRARequest
15
from vllm.multimodal.inputs import MultiModalPlaceholderDict
16
from vllm.sequence import RequestMetrics
17
+from vllm.v1.metrics.stats import RequestStateStats
18
19
logger = init_logger(__name__)
20
@@ -108,7 +109,7 @@ class RequestOutput:
109
prompt_logprobs: Optional[PromptLogprobs],
110
outputs: list[CompletionOutput],
111
finished: bool,
111
- metrics: Optional[RequestMetrics] = None,
112
+ metrics: Optional[Union[RequestMetrics, RequestStateStats]] = None,
113
lora_request: Optional[LoRARequest] = None,
114
encoder_prompt: Optional[str] = None,
115
encoder_prompt_token_ids: Optional[list[int]] = None,
vllm/v1/engine/output_processor.py
+9
-10
@@ -248,16 +248,15 @@ class RequestState:
248
if prompt_token_ids is None and self.prompt_embeds is not None:
249
prompt_token_ids = [0] * len(self.prompt_embeds)
250
251
- return RequestOutput(
252
- request_id=request_id,
253
- prompt=self.prompt,
254
- prompt_token_ids=prompt_token_ids,
255
- prompt_logprobs=prompt_logprobs,
256
- outputs=cast(list[CompletionOutput], outputs),
257
- finished=finished,
258
- kv_transfer_params=kv_transfer_params,
259
- num_cached_tokens=self.num_cached_tokens,
260
- )
251
+ return RequestOutput(request_id=request_id,
252
+ prompt=self.prompt,
253
+ prompt_token_ids=prompt_token_ids,
254
+ prompt_logprobs=prompt_logprobs,
255
+ outputs=cast(list[CompletionOutput], outputs),
256
+ finished=finished,
257
+ kv_transfer_params=kv_transfer_params,
258
+ num_cached_tokens=self.num_cached_tokens,
259
+ metrics=self.stats)
260
261
def _new_completion_output(
262
self,