A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Catch and log invalid token ids in detokenizer (#24351)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Sep 5, 2025 at 22:30 UTC
6432739ef1b35be382733c6c081dab76696b1f96
1 file changed
+5
vllm/v1/engine/detokenizer.py
+5
@@ -233,6 +233,11 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
233
def _protected_step(self, next_token_id: int) -> Optional[str]:
234
try:
235
token = self.stream.step(self.tokenizer, next_token_id)
236
+ except OverflowError:
237
+ # Handle rare observed overflow, still to be diagnosed.
238
+ # See https://github.com/vllm-project/vllm/issues/21951.
239
+ logger.exception("Encountered invalid token id: %d", next_token_id)
240
+ token = None
241
except Exception as e:
242
if not str(e).startswith(INVALID_PREFIX_ERR_MSG):
243
raise e