A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 18:15 UTC
[Frontend] add logprob, compression_rate to 'verbose_json' features (#31059)
Signed-off-by: sangbumlikeagod <oironese@naver.com> Signed-off-by: sangbumlikeagod <98077576+sangbumlikeagod@users.noreply.github.com>
sangbumlikeagod committed
Jan 24, 2026 at 01:35 UTC
9b77bb790dd6d833a9b31814ba00f48b5fa47afb
4 files changed
+32
-14
docs/serving/openai_compatible_server.md
+1
-1
@@ -521,7 +521,7 @@ For `verbose_json` response format:
521
]
522
}
523
```
524
-Currently “verbose_json” response format doesn’t support avg_logprob, compression_ratio, no_speech_prob.
524
+Currently “verbose_json” response format doesn’t support no_speech_prob.
525
526
#### Extra Parameters
527
tests/entrypoints/openai/test_transcription_validation_whisper.py
+2
@@ -244,6 +244,8 @@ async def test_audio_with_timestamp(mary_had_lamb, whisper_client):
244
)
245
assert transcription.segments is not None
246
assert len(transcription.segments) > 0
247
+ assert transcription.segments[0].avg_logprob is not None
248
+ assert transcription.segments[0].compression_ratio is not None
249
250
251
@pytest.mark.asyncio
vllm/entrypoints/openai/translations/protocol.py
+4
-4
@@ -271,13 +271,13 @@ class TranscriptionSegment(OpenAIBaseModel):
271
id: int
272
"""Unique identifier of the segment."""
273
274
- avg_logprob: float | None = None
274
+ avg_logprob: float
275
"""Average logprob of the segment.
276
277
If the value is lower than -1, consider the logprobs failed.
278
"""
279
280
- compression_ratio: float | None = None
280
+ compression_ratio: float
281
"""Compression ratio of the segment.
282
283
If the value is greater than 2.4, consider the compression failed.
@@ -487,13 +487,13 @@ class TranslationSegment(OpenAIBaseModel):
487
id: int
488
"""Unique identifier of the segment."""
489
490
- avg_logprob: float | None = None
490
+ avg_logprob: float
491
"""Average logprob of the segment.
492
493
If the value is lower than -1, consider the logprobs failed.
494
"""
495
496
- compression_ratio: float | None = None
496
+ compression_ratio: float
497
"""Compression ratio of the segment.
498
499
If the value is greater than 2.4, consider the compression failed.
vllm/entrypoints/openai/translations/speech_to_text.py
+25
-9
@@ -4,6 +4,7 @@ import asyncio
4
import io
5
import math
6
import time
7
+import zlib
8
from collections.abc import AsyncGenerator, Callable
9
from functools import cached_property
10
from typing import Literal, TypeAlias, TypeVar, cast
@@ -38,6 +39,7 @@ from vllm.entrypoints.openai.translations.protocol import (
39
from vllm.exceptions import VLLMValidationError
40
from vllm.inputs.data import PromptType
41
from vllm.logger import init_logger
42
+from vllm.logprobs import FlatLogprobs, Logprob
43
from vllm.model_executor.models import SupportsTranscription, supports_transcription
44
from vllm.outputs import RequestOutput
45
from vllm.tokenizers import get_tokenizer
@@ -317,6 +319,7 @@ class OpenAISpeechToText(OpenAIServing):
319
def _get_verbose_segments(
320
self,
321
tokens: tuple,
322
+ log_probs: FlatLogprobs | list[dict[int, Logprob]],
323
request: SpeechToTextRequest,
324
segment_class: type[SpeechToTextSegment],
325
start_time: float = 0,
@@ -329,8 +332,7 @@ class OpenAISpeechToText(OpenAIServing):
332
If the tokens do not include timestamp information,
333
the segments may not be generated correctly.
334
332
- Note: Fields like avg_logprob, compression_ratio,
333
- and no_speech_prob are not supported
335
+ Note: No_speech_prob field is not supported
336
in this implementation and will be None. See docs for details.
337
"""
338
BASE_OFFSET = 0.02
@@ -344,17 +346,17 @@ class OpenAISpeechToText(OpenAIServing):
346
347
if tokens_with_start[-2] < init_token and tokens_with_start[-1] >= init_token:
348
tokens_with_start = tokens_with_start + (tokens_with_start[-1],)
347
- for idx, token in enumerate(tokens_with_start):
349
+ avg_logprob = 0.0
350
+ for idx in range(1, len(tokens_with_start)):
351
# Timestamp tokens (e.g., <|0.00|>) are assumed to be sorted.
352
# If the ordering is violated, this slicing may produce incorrect results.
350
- if (
351
- token >= init_token
352
- and idx != 0
353
- and tokens_with_start[idx - 1] >= init_token
354
- ):
353
+ token = tokens_with_start[idx]
354
+ if token >= init_token and tokens_with_start[idx - 1] >= init_token:
355
sliced_timestamp_tokens = tokens_with_start[last_timestamp_start:idx]
356
start_timestamp = sliced_timestamp_tokens[0] - init_token
357
end_timestamp = sliced_timestamp_tokens[-1] - init_token
358
+ text = self.tokenizer.decode(sliced_timestamp_tokens[1:-1])
359
+ text_bytes = text.encode("utf-8")
360
361
casting_segment = cast(
362
SpeechToTextSegment,
@@ -364,12 +366,22 @@ class OpenAISpeechToText(OpenAIServing):
366
start=start_time + BASE_OFFSET * start_timestamp,
367
end=start_time + BASE_OFFSET * end_timestamp,
368
temperature=request.temperature,
367
- text=self.tokenizer.decode(sliced_timestamp_tokens[1:-1]),
369
+ text=text,
370
+ # The compression ratio measures
371
+ # how compressible the generated text is.
372
+ # A higher ratio indicates more repetitive content,
373
+ # which is a strong sign of hallucination in outputs.
374
+ compression_ratio=len(text_bytes)
375
+ / len(zlib.compress(text_bytes)),
376
tokens=sliced_timestamp_tokens[1:-1],
377
+ avg_logprob=avg_logprob / (idx - last_timestamp_start),
378
),
379
)
380
segments.append(casting_segment)
381
last_timestamp_start = idx
382
+ avg_logprob = 0
383
+ else:
384
+ avg_logprob += log_probs[idx - 1][token].logprob
385
return segments
386
387
async def _create_speech_to_text(
@@ -443,6 +455,8 @@ class OpenAISpeechToText(OpenAIServing):
455
sampling_params = request.to_sampling_params(
456
default_max_tokens, self.default_sampling_params
457
)
458
+ if request.response_format == "verbose_json":
459
+ sampling_params.logprobs = 1
460
461
self._log_inputs(
462
request_id,
@@ -490,12 +504,14 @@ class OpenAISpeechToText(OpenAIServing):
504
)
505
async for op in result_generator:
506
if request.response_format == "verbose_json":
507
+ assert op.outputs[0].logprobs
508
segments: list[SpeechToTextSegment] = (
509
self._get_verbose_segments(
510
tokens=tuple(op.outputs[0].token_ids),
511
segment_class=segment_class,
512
request=request,
513
start_time=start_time,
514
+ log_probs=op.outputs[0].logprobs,
515
)
516
)
517