@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2f4e6548ef
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 10, 17:45 UTC

[Bugfix] vLLM produces invalid UTF-8 tokens and “�” (#28874)

Signed-off-by: John Calderon <jcalderon@nvidia.com> Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>

John Calderon committed Jan 5, 2026 at 19:23 UTC 2f4e6548efec402b913ffddc8726230d9311948d
3 files changed +535 -29
tests/v1/engine/test_output_processor.py
+44 -12
@@ -274,12 +274,28 @@ def _validate_logprobs(
274 # the logprob token id at this sequence position
275 decoded_token = pos_logprob_dict[lp_tok].decoded_token
276 ref_decoded_token = _ref_convert_id_to_token(dtv.tokenizer, lp_tok)
277 - assert decoded_token == ref_decoded_token, (
278 - f"Sampled logprob token id {lp_tok} decodes to"
279 - f" {ref_decoded_token} but Logprob decoded"
280 - f" token is {decoded_token} instead"
281 - f" (at position {idx})"
282 - )
277 +
278 + # With UTF-8 correction logic, tokens ending with "�"
279 + # (incomplete byte sequences) are corrected to either
280 + # empty string or proper UTF-8 characters
281 + if ref_decoded_token.endswith("�"):
282 + # Token needs UTF-8 correction
283 + assert not decoded_token.endswith("�"), (
284 + f"Sampled logprob token id {lp_tok} decodes to"
285 + f" '{ref_decoded_token}' (ends with replacement char)"
286 + f" but corrected decoded token '{decoded_token}'"
287 + f" still ends with replacement char"
288 + f" (at position {idx}). UTF-8 correction should"
289 + f" have removed it."
290 + )
291 + else:
292 + # No correction needed, should match exactly
293 + assert decoded_token == ref_decoded_token, (
294 + f"Sampled logprob token id {lp_tok} decodes to"
295 + f" {ref_decoded_token} but Logprob decoded"
296 + f" token is {decoded_token} instead"
297 + f" (at position {idx})"
298 + )
299
300 ref_cumulative_logprob += pos_logprob_dict[sampled_token].logprob
301 # Assert that cumulative logprobs are correct
@@ -420,12 +436,28 @@ def _validate_logprobs(
436 # the logprob token id at this sequence position
437 decoded_token = pos_logprob_dict[plp_tok].decoded_token
438 ref_decoded_token = _ref_convert_id_to_token(dtv.tokenizer, plp_tok)
423 - assert decoded_token == ref_decoded_token, (
424 - f"Prompt logprob token id {plp_tok} decodes to"
425 - f" {ref_decoded_token} but Logprob decoded"
426 - f" token is {decoded_token} instead"
427 - f" (at position {idx})"
428 - )
439 +
440 + # With UTF-8 correction logic, tokens ending with "�"
441 + # (incomplete byte sequences) are corrected to either
442 + # empty string or proper UTF-8 characters
443 + if ref_decoded_token.endswith("�"):
444 + # Token needs UTF-8 correction
445 + assert not decoded_token.endswith("�"), (
446 + f"Prompt logprob token id {plp_tok} decodes to"
447 + f" '{ref_decoded_token}' (ends with replacement char)"
448 + f" but corrected decoded token '{decoded_token}'"
449 + f" still ends with replacement char"
450 + f" (at position {idx}). UTF-8 correction should"
451 + f" have removed it."
452 + )
453 + else:
454 + # No correction needed, should match exactly
455 + assert decoded_token == ref_decoded_token, (
456 + f"Prompt logprob token id {plp_tok} decodes to"
457 + f" {ref_decoded_token} but Logprob decoded"
458 + f" token is {decoded_token} instead"
459 + f" (at position {idx})"
460 + )
461 else:
462 # Prompt logprobs disabled for this request
463 assert prompt_logprobs is None
tests/v1/sample/test_logprobs.py
+418
@@ -514,6 +514,424 @@ def test_logprobs_mode(logprobs_mode: LogprobsMode):
514 del llm
515
516
517 +class TestCorrectDecodedToken:
518 + """Unit tests for _correct_decoded_token method in LogprobsProcessor.
519 +
520 + This method handles UTF-8 decoding issues where incomplete byte sequences
521 + result in the Unicode replacement character "�" (U+FFFD). This commonly
522 + happens with byte-fallback tokenization when multi-byte UTF-8 characters
523 + are split across tokens.
524 + """
525 +
526 + @pytest.fixture
527 + def mock_tokenizer(self):
528 + """Create a mock tokenizer for testing."""
529 + from unittest.mock import Mock
530 +
531 + tokenizer = Mock()
532 + return tokenizer
533 +
534 + @pytest.fixture
535 + def processor_with_empty_logprobs(self, mock_tokenizer):
536 + """Create a LogprobsProcessor with empty logprobs."""
537 + from vllm.v1.engine.logprobs import LogprobsProcessor
538 +
539 + processor = LogprobsProcessor(
540 + tokenizer=mock_tokenizer,
541 + logprobs=[],
542 + prompt_logprobs=None,
543 + cumulative_logprob=0.0,
544 + num_logprobs=1,
545 + num_prompt_logprobs=None,
546 + )
547 + return processor
548 +
549 + @pytest.fixture
550 + def processor_with_previous_logprobs(self, mock_tokenizer):
551 + """Create a LogprobsProcessor with previous logprobs."""
552 + from vllm.v1.engine.logprobs import LogprobsProcessor
553 +
554 + processor = LogprobsProcessor(
555 + tokenizer=mock_tokenizer,
556 + logprobs=[{123: None}], # Previous token ID is 123
557 + prompt_logprobs=None,
558 + cumulative_logprob=0.0,
559 + num_logprobs=1,
560 + num_prompt_logprobs=None,
561 + )
562 + return processor
563 +
564 + def test_correction_with_previous_token_in_list(
565 + self, processor_with_empty_logprobs
566 + ):
567 + """Test correction using previous token in the same list.
568 +
569 + Scenario: Token at idx=1 ends with "�", but when decoded with
570 + the previous token (idx=0), it forms a valid UTF-8 sequence.
571 + Example: token[0]="�", token[1]="�" -> together form "polarized"
572 + """
573 + processor = processor_with_empty_logprobs
574 + tokens = [100, 101, 102] # token IDs
575 +
576 + # Mock tokenizer behavior:
577 + # - decode([102]) returns "�" (ends with replacement char)
578 + # - decode([101, 102]) returns "valid" (no replacement char)
579 + processor.tokenizer.decode.side_effect = lambda ids: (
580 + "valid" if ids == [101, 102] else "�"
581 + )
582 +
583 + result = processor._correct_decoded_token(2, tokens)
584 + assert result == "valid"
585 + processor.tokenizer.decode.assert_called_with([101, 102])
586 +
587 + def test_correction_with_previous_logprob_token(
588 + self, processor_with_previous_logprobs
589 + ):
590 + """Test correction using previous logprob token.
591 +
592 + Scenario: Cannot correct with previous token in list (idx=0),
593 + but can correct with previous logprob token.
594 + """
595 + processor = processor_with_previous_logprobs
596 + tokens = [100] # single token
597 +
598 + # Mock tokenizer behavior:
599 + # - decode([100]) returns "�" (ends with replacement char)
600 + # - decode([123, 100]) returns " "polarized" (no replacement char)
601 + # Token 123 is from previous logprobs
602 + def mock_decode(ids):
603 + if ids == [123, 100]:
604 + return ' "polarized"'
605 + return "�"
606 +
607 + processor.tokenizer.decode.side_effect = mock_decode
608 +
609 + result = processor._correct_decoded_token(0, tokens)
610 + assert result == ' "polarized"'
611 +
612 + def test_correction_at_idx_zero_no_previous_logprobs(
613 + self, processor_with_empty_logprobs
614 + ):
615 + """Test correction at idx=0 with no previous logprobs.
616 +
617 + Scenario: First token in list, no previous logprobs available.
618 + Should return empty string as fallback.
619 + """
620 + processor = processor_with_empty_logprobs
621 + tokens = [100]
622 +
623 + # Mock tokenizer always returns "�"
624 + processor.tokenizer.decode.return_value = "�"
625 +
626 + result = processor._correct_decoded_token(0, tokens)
627 + assert result == ""
628 +
629 + def test_correction_at_idx_zero_with_previous_logprobs(
630 + self, processor_with_previous_logprobs
631 + ):
632 + """Test correction at idx=0 with previous logprobs available.
633 +
634 + Scenario: First token in list, but previous logprobs exist.
635 + Should try correction with previous logprob token.
636 + """
637 + processor = processor_with_previous_logprobs
638 + tokens = [200]
639 +
640 + # Mock tokenizer behavior
641 + def mock_decode(ids):
642 + if ids == [123, 200]:
643 + return "corrected"
644 + return "�"
645 +
646 + processor.tokenizer.decode.side_effect = mock_decode
647 +
648 + result = processor._correct_decoded_token(0, tokens)
649 + assert result == "corrected"
650 +
651 + def test_no_correction_needed_returns_fallback(
652 + self, processor_with_previous_logprobs
653 + ):
654 + """Test fallback to empty string when no correction works.
655 +
656 + Scenario: All correction attempts still end with "�".
657 + Should return empty string as final fallback.
658 + """
659 + processor = processor_with_previous_logprobs
660 + tokens = [100, 101, 102]
661 +
662 + # Mock tokenizer always returns text ending with "�"
663 + processor.tokenizer.decode.return_value = "still�"
664 +
665 + result = processor._correct_decoded_token(2, tokens)
666 + assert result == ""
667 +
668 + def test_middle_token_correction(self, processor_with_previous_logprobs):
669 + """Test correction for a token in the middle of the list.
670 +
671 + Scenario: Token at idx=5 in a longer list needs correction.
672 + """
673 + processor = processor_with_previous_logprobs
674 + tokens = [10, 20, 30, 40, 50, 60, 70, 80]
675 +
676 + # Mock tokenizer behavior for middle token
677 + def mock_decode(ids):
678 + if ids == [50, 60]:
679 + return "olar"
680 + return "�"
681 +
682 + processor.tokenizer.decode.side_effect = mock_decode
683 +
684 + result = processor._correct_decoded_token(5, tokens)
685 + assert result == "olar"
686 +
687 + def test_multiple_consecutive_replacement_chars(
688 + self, processor_with_previous_logprobs
689 + ):
690 + """Test handling of multiple consecutive replacement characters.
691 +
692 + Scenario: Sequence like ["�", "�", "p"] where first two should
693 + become empty strings.
694 + """
695 + processor = processor_with_previous_logprobs
696 +
697 + # Test first replacement char
698 + tokens = [100, 101, 102]
699 + processor.tokenizer.decode.return_value = "still�"
700 + result1 = processor._correct_decoded_token(0, tokens)
701 + assert result1 == ""
702 +
703 + # Test second replacement char
704 + result2 = processor._correct_decoded_token(1, tokens)
705 + assert result2 == ""
706 +
707 + def test_correction_with_multibyte_utf8(self, processor_with_previous_logprobs):
708 + """Test correction involving multi-byte UTF-8 characters.
709 +
710 + Scenario: Byte-fallback tokenization splits multi-byte UTF-8
711 + characters (e.g., curly quotes, Chinese characters, emojis).
712 + Example from user: "�", "�" -> "", "\""
713 + """
714 + processor = processor_with_previous_logprobs
715 + tokens = [200, 201]
716 +
717 + # Mock tokenizer behavior for multi-byte UTF-8 correction
718 + def mock_decode(ids):
719 + # When decoding first token (idx=0) with previous logprob token
720 + if ids == [123, 200]:
721 + return ' "' # Space + left curly quote
722 + # When decoding second token (idx=1) with previous token in list
723 + elif ids == [200, 201]:
724 + return '"' # Right curly quote
725 + # When decoding second token (idx=1) with previous logprob + prev token
726 + elif ids == [123, 200, 201]:
727 + return ' ""' # Full sequence
728 + return "�"
729 +
730 + processor.tokenizer.decode.side_effect = mock_decode
731 +
732 + # First token correction (idx=0)
733 + # Will call decode([123, 200]) since idx=0 uses previous logprob token
734 + result1 = processor._correct_decoded_token(0, tokens)
735 + assert result1 == ' "'
736 +
737 + # Second token correction (idx=1)
738 + # Will call decode([200, 201]) since idx>0 uses previous token in list
739 + result2 = processor._correct_decoded_token(1, tokens)
740 + assert result2 == '"'
741 +
742 + def test_real_world_opt125m_scenario(self, mock_tokenizer):
743 + """Test the real-world scenario from user's example.
744 +
745 + User's example with facebook/opt-125m:
746 + Before: [" the", " term", " �", "�", "p", "olar", "ized", "�", "�", ...]
747 + After: [" the", " term", "", " "", "p", "olar", "ized", "", "\"", ...]
748 + """
749 + from vllm.v1.engine.logprobs import LogprobsProcessor
750 +
751 + # Simulate the sequence of tokens
752 + processor = LogprobsProcessor(
753 + tokenizer=mock_tokenizer,
754 + logprobs=[],
755 + prompt_logprobs=None,
756 + cumulative_logprob=0.0,
757 + num_logprobs=1,
758 + num_prompt_logprobs=None,
759 + )
760 +
761 + # Token IDs representing the problematic sequence
762 + tokens = [1, 2, 3, 4, 5, 6, 7, 8, 9] # placeholder IDs
763 +
764 + # Mock decode behavior simulating the real scenario
765 + def mock_decode(ids):
766 + # Simulate cases where individual tokens decode to "�"
767 + # but combinations decode correctly
768 + if len(ids) == 1:
769 + if ids[0] == 3 or ids[0] == 4 or ids[0] == 8 or ids[0] == 9:
770 + return "�"
771 + elif len(ids) == 2:
772 + if ids == [2, 3]:
773 + return " term�" # Still ends with �, need more context
774 + elif ids == [3, 4]:
775 + return ' "' # Corrected to space + left curly quote
776 + elif ids == [7, 8]:
777 + return "ized�" # Still ends with �
778 + elif ids == [8, 9]:
779 + return '"' # Corrected to right curly quote
780 + elif len(ids) == 3:
781 + if ids == [1, 2, 3]:
782 + return " the term�" # Still ends with issue
783 + elif ids == [2, 3, 4]:
784 + return ' term "' # With all context
785 + return "normal_text"
786 +
787 + mock_tokenizer.decode.side_effect = mock_decode
788 +
789 + # Test token at index 2 (should fail to correct, return "")
790 + # Token 3 individually is "�"
791 + # decode([2, 3]) = " term�" (still ends with �)
792 + # No previous logprobs, so fallback to ""
793 + result = processor._correct_decoded_token(2, tokens)
794 + assert result == ""
795 +
796 + # Test token at index 3 (should correct to " "")
797 + # Token 4 individually is "�"
798 + # decode([3, 4]) = " "" (corrected!)
799 + processor.logprobs = [{2: None}] # Add previous logprob
800 + result = processor._correct_decoded_token(3, tokens)
801 + assert result == ' "'
802 +
803 +
804 +def test_verify_tokens_integration():
805 + """Integration test for _verify_tokens with real model.
806 +
807 + This test validates that _verify_tokens correctly identifies and
808 + corrects tokens ending with the replacement character "�".
809 + Uses facebook/opt-125m which is known to produce these issues.
810 + """
811 + runner = VllmRunner(
812 + "facebook/opt-125m",
813 + max_logprobs=0,
814 + enable_prefix_caching=False,
815 + gpu_memory_utilization=0.15,
816 + max_model_len=256,
817 + )
818 +
819 + # Use a prompt that triggers multi-byte UTF-8 issues
820 + # Based on user's example: "In this example,"
821 + test_prompts = ["In this example,"]
822 +
823 + sampling_params = SamplingParams(
824 + max_tokens=16,
825 + temperature=0,
826 + logprobs=0,
827 + )
828 +
829 + results = runner.llm.generate(test_prompts, sampling_params=sampling_params)
830 +
831 + # Verify that decoded tokens don't contain replacement characters
832 + for result in results:
833 + assert result.outputs[0].logprobs is not None
834 + for logprob_dict in result.outputs[0].logprobs:
835 + for token_id, logprob_info in logprob_dict.items():
836 + decoded_token = logprob_info.decoded_token
837 + # Decoded tokens should not end with replacement character
838 + # They should either be corrected or empty string
839 + assert not decoded_token.endswith("�"), (
840 + f"Token {token_id} decoded to '{decoded_token}' which "
841 + f"ends with replacement character"
842 + )
843 + # Decoded tokens should not contain lone replacement characters
844 + assert decoded_token != "�", (
845 + f"Token {token_id} is a lone replacement character"
846 + )
847 +
848 +
849 +def test_utf8_edge_cases_with_real_model():
850 + """Test various UTF-8 edge cases with a real model.
851 +
852 + Tests prompts that are likely to trigger byte-fallback tokenization
853 + and multi-byte UTF-8 splitting.
854 + """
855 + runner = VllmRunner(
856 + "facebook/opt-125m",
857 + max_logprobs=1,
858 + enable_prefix_caching=False,
859 + gpu_memory_utilization=0.15,
860 + max_model_len=256,
861 + )
862 +
863 + # Prompts with various multi-byte UTF-8 characters
864 + test_prompts = [
865 + 'Smart quotes: "Hello"', # Curly quotes
866 + "Em dash — test", # Em dash
867 + "Ellipsis… continues", # Ellipsis
868 + "Chinese: 你好", # Chinese characters
869 + "Emoji: 😀 🎉", # Emojis
870 + 'Mixed: "quoted" — with symbols', # Mixed
871 + ]
872 +
873 + sampling_params = SamplingParams(
874 + max_tokens=10,
875 + temperature=0,
876 + logprobs=1,
877 + )
878 +
879 + results = runner.llm.generate(test_prompts, sampling_params=sampling_params)
880 +
881 + for i, result in enumerate(results):
882 + prompt = test_prompts[i]
883 + assert result.outputs[0].logprobs is not None
884 +
885 + # Check that no decoded tokens end with replacement character
886 + for logprob_dict in result.outputs[0].logprobs:
887 + for token_id, logprob_info in logprob_dict.items():
888 + decoded_token = logprob_info.decoded_token
889 + assert not decoded_token.endswith("�"), (
890 + f"Prompt: '{prompt}'\n"
891 + f"Token {token_id} decoded to '{decoded_token}' which "
892 + f"ends with replacement character"
893 + )
894 +
895 +
896 +def test_correct_decoded_token_preserves_valid_tokens():
897 + """Test that valid tokens (not ending with �) are not modified.
898 +
899 + The _correct_decoded_token method should only be called for tokens
900 + ending with "�", but this test verifies the broader _verify_tokens
901 + logic doesn't affect valid tokens.
902 + """
903 + runner = VllmRunner(
904 + "facebook/opt-125m",
905 + max_logprobs=2,
906 + enable_prefix_caching=False,
907 + gpu_memory_utilization=0.15,
908 + max_model_len=256,
909 + )
910 +
911 + # Simple prompt with standard ASCII characters
912 + test_prompts = ["Hello world, this is a test."]
913 +
914 + sampling_params = SamplingParams(
915 + max_tokens=10,
916 + temperature=0,
917 + logprobs=2,
918 + )
919 +
920 + results = runner.llm.generate(test_prompts, sampling_params=sampling_params)
921 +
922 + for result in results:
923 + assert result.outputs[0].logprobs is not None
924 +
925 + # All decoded tokens should be valid strings
926 + for logprob_dict in result.outputs[0].logprobs:
927 + for token_id, logprob_info in logprob_dict.items():
928 + decoded_token = logprob_info.decoded_token
929 + # Valid tokens should be non-empty strings (or empty if corrected)
930 + assert isinstance(decoded_token, str)
931 + # Should not contain replacement character
932 + assert "�" not in decoded_token
933 +
934 +
935 @pytest.mark.parametrize("logprobs_mode", get_args(LogprobsMode))
936 @pytest.mark.parametrize(
937 "model_setup",
vllm/v1/engine/logprobs.py
+73 -17
@@ -2,6 +2,7 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 import itertools
5 +from collections.abc import Iterable
6 from dataclasses import dataclass
7
8 from vllm.logger import init_logger
@@ -88,11 +89,16 @@ class LogprobsProcessor:
89 logprobs = logprobs_np.tolist()
90 token_ids = token_ids_np.tolist()
91 # Detokenize (non-incrementally).
91 - decoded_tokens = (
92 - NONES
93 - if self.tokenizer is None
94 - else (convert_ids_list_to_tokens(self.tokenizer, token_ids))
95 - )
92 + decoded_tokens: list[str] | Iterable[None]
93 + if self.tokenizer is None:
94 + decoded_tokens = NONES
95 + else:
96 + decoded_tokens_list = convert_ids_list_to_tokens(
97 + self.tokenizer, token_ids
98 + )
99 + decoded_tokens = self._verify_tokens(
100 + decoded_tokens_list=decoded_tokens_list, tokens=token_ids
101 + )
102
103 # Sampler puts the sampled logprob in first.
104 sampled_token_logprob = logprobs[0]
@@ -126,37 +132,45 @@ class LogprobsProcessor:
132
133 token_ids, logprobs, ranks = prompt_logprobs_tensors
134
135 + # Recover shapes.
136 + num_prompt_tokens, num_logprobs = logprobs.shape
137 +
138 # Detokenize non-incrementally.
139 # Output is flat: [num_tok, num_lps] -> [num_tok * num_lps]
131 - decoded_tokens = (
140 + all_decoded_tokens: list[str] | None = (
141 None
142 if self.tokenizer is None
134 - else (
135 - convert_ids_list_to_tokens(self.tokenizer, token_ids.flatten().tolist())
143 + else convert_ids_list_to_tokens(
144 + self.tokenizer, token_ids.flatten().tolist()
145 )
146 )
147
139 - # Recover shapes.
140 - num_prompt_tokens, num_logprobs = logprobs.shape
141 -
148 # Pythonize the torch tensors.
149 prompt_token_ranks = ranks.tolist()
150 prompt_logprobs = logprobs.tolist()
145 - token_ids = token_ids.tolist()
151 + token_ids_list = token_ids.tolist()
152
153 # Make Logprob for each position.
154 for pos in range(num_prompt_tokens):
149 - # Handle flattening.
155 + # Handle flattening and UTF-8 correction per position
156 offset = pos * num_logprobs
157 offset_end = offset + num_logprobs
152 - decoded_tokens_for_pos = (
153 - NONES if decoded_tokens is None else decoded_tokens[offset:offset_end]
154 - )
158 +
159 + decoded_tokens_for_pos: list[str] | Iterable[None]
160 + if all_decoded_tokens is None:
161 + decoded_tokens_for_pos = NONES
162 + else:
163 + # Extract decoded tokens for this position
164 + decoded_tokens_slice = all_decoded_tokens[offset:offset_end]
165 + # Apply UTF-8 correction within this position's token boundaries
166 + decoded_tokens_for_pos = self._verify_tokens(
167 + decoded_tokens_list=decoded_tokens_slice, tokens=token_ids_list[pos]
168 + )
169
170 # Update with the Logprob container for this pos.
171 append_logprobs_for_next_position(
172 self.prompt_logprobs,
159 - token_ids[pos],
173 + token_ids_list[pos],
174 prompt_logprobs[pos],
175 decoded_tokens_for_pos,
176 prompt_token_ranks[pos],
@@ -182,6 +196,48 @@ class LogprobsProcessor:
196 self.prompt_logprobs = []
197 return plp
198
199 + def _correct_decoded_token(self, idx: int, tokens: list[int]) -> str:
200 + assert self.tokenizer is not None, "self.tokenizer should not be None"
201 +
202 + # try with prev token id in same list
203 + if idx > 0:
204 + possible_decoded_token = self.tokenizer.decode(tokens[idx - 1 : idx + 1])
205 + if not possible_decoded_token.endswith("�"):
206 + return possible_decoded_token
207 + # try with previous logprob token id
208 + if self.logprobs:
209 + latest_token_id = next(iter(self.logprobs[-1]))
210 +
211 + decode_ids = [latest_token_id]
212 + if idx > 0:
213 + decode_ids.extend(tokens[idx - 1 : idx + 1])
214 + else:
215 + decode_ids.extend(tokens[idx : idx + 1])
216 +
217 + possible_decoded_token = self.tokenizer.decode(decode_ids)
218 + if not possible_decoded_token.endswith("�"):
219 + return possible_decoded_token
220 +
221 + # by default return empty string
222 + return ""
223 +
224 + def _verify_tokens(
225 + self, decoded_tokens_list: list[str], tokens: list[int]
226 + ) -> list[str]:
227 + corrected_decoded_token_map = dict()
228 + for idx, text in enumerate(decoded_tokens_list):
229 + if text.endswith("�"):
230 + # utf-8 char at the end means it's a potential unfinished byte sequence
231 + # from byte fallback tokenization.
232 + corrected_decoded_token_map[idx] = self._correct_decoded_token(
233 + idx, tokens
234 + )
235 +
236 + for idx, text in corrected_decoded_token_map.items():
237 + decoded_tokens_list[idx] = text
238 +
239 + return decoded_tokens_list
240 +
241 def update_from_output(self, output: EngineCoreOutput) -> None:
242 if output.new_logprobs is not None:
243 self._update_sample_logprobs(output.new_logprobs)