@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8904fc4d19
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 5, 11:00 UTC

[Bugfix] Fix V1 logprobs empty strings for multi-byte UTF-8 tokens when logprobs > 0 (#34875)

Signed-off-by: haosdent <haosdent@gmail.com>

haosdent committed Apr 8, 2026 at 23:30 UTC 8904fc4d1942ee0771c094b2b084cd62c55de89d
2 files changed +348 -196
tests/v1/sample/test_logprobs.py
+215 -170
@@ -539,6 +539,10 @@ class TestCorrectDecodedToken:
539 result in the Unicode replacement character "�" (U+FFFD). This commonly
540 happens with byte-fallback tokenization when multi-byte UTF-8 characters
541 are split across tokens.
542 +
543 + The method signature is _correct_decoded_token(token_id, context_token_ids)
544 + where token_id is the single token to correct and context_token_ids are
545 + the preceding sampled tokens in sequential order.
546 """
547
548 @pytest.fixture
@@ -550,8 +554,8 @@ class TestCorrectDecodedToken:
554 return tokenizer
555
556 @pytest.fixture
553 - def processor_with_empty_logprobs(self, mock_tokenizer):
554 - """Create a LogprobsProcessor with empty logprobs."""
557 + def processor(self, mock_tokenizer):
558 + """Create a LogprobsProcessor."""
559 from vllm.v1.engine.logprobs import LogprobsProcessor
560
561 processor = LogprobsProcessor(
@@ -564,209 +568,191 @@ class TestCorrectDecodedToken:
568 )
569 return processor
570
567 - @pytest.fixture
568 - def processor_with_previous_logprobs(self, mock_tokenizer):
569 - """Create a LogprobsProcessor with previous logprobs."""
570 - from vllm.v1.engine.logprobs import LogprobsProcessor
571 + def test_correction_with_context(self, processor):
572 + """Test correction using context from preceding sampled tokens.
573
572 - processor = LogprobsProcessor(
573 - tokenizer=mock_tokenizer,
574 - logprobs=[{123: None}], # Previous token ID is 123
575 - prompt_logprobs=None,
576 - cumulative_logprob=0.0,
577 - num_logprobs=1,
578 - num_prompt_logprobs=None,
579 - )
580 - return processor
574 + Scenario: A byte-fallback token that completes a multi-byte
575 + UTF-8 sequence when decoded with context.
576 + """
577
582 - def test_correction_with_previous_token_in_list(
583 - self, processor_with_empty_logprobs
584 - ):
585 - """Test correction using previous token in the same list.
578 + # Context is [101] (a preceding sampled token)
579 + # Token 102 individually decodes to "�"
580 + # decode([101, 102]) returns "valid" (complete sequence)
581 + def mock_decode(ids):
582 + if ids == [101, 102]:
583 + return "hello valid"
584 + if ids == [101]:
585 + return "hello "
586 + return "�"
587
587 - Scenario: Token at idx=1 ends with "�", but when decoded with
588 - the previous token (idx=0), it forms a valid UTF-8 sequence.
589 - Example: token[0]="�", token[1]="�" -> together form "polarized"
590 - """
591 - processor = processor_with_empty_logprobs
592 - tokens = [100, 101, 102] # token IDs
593 -
594 - # Mock tokenizer behavior:
595 - # - decode([102]) returns "�" (ends with replacement char)
596 - # - decode([101, 102]) returns "valid" (no replacement char)
597 - processor.tokenizer.decode.side_effect = lambda ids: (
598 - "valid" if ids == [101, 102] else "�"
599 - )
588 + processor.tokenizer.decode.side_effect = mock_decode
589
601 - result = processor._correct_decoded_token(2, tokens)
590 + result = processor._correct_decoded_token(102, [101])
591 assert result == "valid"
603 - processor.tokenizer.decode.assert_called_with([101, 102])
592
605 - def test_correction_with_previous_logprob_token(
606 - self, processor_with_previous_logprobs
607 - ):
608 - """Test correction using previous logprob token.
593 + def test_correction_with_context_from_logprobs(self, processor):
594 + """Test correction using context from previous logprob entries.
595
610 - Scenario: Cannot correct with previous token in list (idx=0),
611 - but can correct with previous logprob token.
596 + Scenario: Token decoded with context from previously sampled
597 + tokens completes a UTF-8 sequence.
598 """
613 - processor = processor_with_previous_logprobs
614 - tokens = [100] # single token
599
616 - # Mock tokenizer behavior:
617 - # - decode([100]) returns "�" (ends with replacement char)
618 - # - decode([123, 100]) returns " "polarized" (no replacement char)
619 - # Token 123 is from previous logprobs
600 + # Token 123 was previously sampled (in context)
601 def mock_decode(ids):
602 if ids == [123, 100]:
622 - return ' "polarized"'
603 + return 'hello "polarized"'
604 + if ids == [123]:
605 + return "hello "
606 return "�"
607
608 processor.tokenizer.decode.side_effect = mock_decode
609
627 - result = processor._correct_decoded_token(0, tokens)
628 - assert result == ' "polarized"'
610 + result = processor._correct_decoded_token(100, [123])
611 + assert result == '"polarized"'
612
630 - def test_correction_at_idx_zero_no_previous_logprobs(
631 - self, processor_with_empty_logprobs
632 - ):
633 - """Test correction at idx=0 with no previous logprobs.
613 + def test_correction_no_context(self, processor):
614 + """Test correction with no context available.
615
635 - Scenario: First token in list, no previous logprobs available.
616 Should return empty string as fallback.
617 """
638 - processor = processor_with_empty_logprobs
639 - tokens = [100]
640 -
641 - # Mock tokenizer always returns "�"
618 processor.tokenizer.decode.return_value = "�"
619
644 - result = processor._correct_decoded_token(0, tokens)
620 + result = processor._correct_decoded_token(100, [])
621 assert result == ""
622
647 - def test_correction_at_idx_zero_with_previous_logprobs(
648 - self, processor_with_previous_logprobs
649 - ):
650 - """Test correction at idx=0 with previous logprobs available.
651 -
652 - Scenario: First token in list, but previous logprobs exist.
653 - Should try correction with previous logprob token.
654 - """
655 - processor = processor_with_previous_logprobs
656 - tokens = [200]
623 + def test_correction_with_context_succeeds(self, processor):
624 + """Test correction with context from previously sampled tokens."""
625
658 - # Mock tokenizer behavior
626 def mock_decode(ids):
627 if ids == [123, 200]:
661 - return "corrected"
628 + return "hello corrected"
629 + if ids == [123]:
630 + return "hello "
631 return "�"
632
633 processor.tokenizer.decode.side_effect = mock_decode
634
666 - result = processor._correct_decoded_token(0, tokens)
635 + result = processor._correct_decoded_token(200, [123])
636 assert result == "corrected"
637
669 - def test_no_correction_needed_returns_fallback(
670 - self, processor_with_previous_logprobs
671 - ):
672 - """Test fallback to empty string when no correction works.
673 -
674 - Scenario: All correction attempts still end with "�".
675 - Should return empty string as final fallback.
676 - """
677 - processor = processor_with_previous_logprobs
678 - tokens = [100, 101, 102]
679 -
680 - # Mock tokenizer always returns text ending with "�"
638 + def test_fallback_when_all_attempts_fail(self, processor):
639 + """Test fallback to empty string when no correction works."""
640 processor.tokenizer.decode.return_value = "still�"
641
683 - result = processor._correct_decoded_token(2, tokens)
642 + result = processor._correct_decoded_token(102, [100, 101])
643 assert result == ""
644
686 - def test_middle_token_correction(self, processor_with_previous_logprobs):
687 - """Test correction for a token in the middle of the list.
645 + def test_increasing_context_window(self, processor):
646 + """Test that increasing context window finds the correction.
647
689 - Scenario: Token at idx=5 in a longer list needs correction.
648 + Scenario: 3-byte UTF-8 char. With 1 context token, still
649 + incomplete. With 2 context tokens, completes the sequence.
650 """
691 - processor = processor_with_previous_logprobs
692 - tokens = [10, 20, 30, 40, 50, 60, 70, 80]
651
694 - # Mock tokenizer behavior for middle token
652 def mock_decode(ids):
696 - if ids == [50, 60]:
697 - return "olar"
653 + # 1 context token: still incomplete
654 + if ids == [81, 82]:
655 + return "�"
656 + # 2 context tokens: complete
657 + if ids == [80, 81, 82]:
658 + return "\u201c"
659 + # Context-only decodes
660 + if ids == [81]:
661 + return "�"
662 + if ids == [80, 81]:
663 + return "�"
664 return "�"
665
666 processor.tokenizer.decode.side_effect = mock_decode
667
702 - result = processor._correct_decoded_token(5, tokens)
703 - assert result == "olar"
668 + # Context has 2 preceding tokens [80, 81]
669 + result = processor._correct_decoded_token(82, [80, 81])
670 + assert result == "\u201c"
671
705 - def test_multiple_consecutive_replacement_chars(
706 - self, processor_with_previous_logprobs
707 - ):
672 + def test_multiple_consecutive_replacement_chars(self, processor):
673 """Test handling of multiple consecutive replacement characters.
674
710 - Scenario: Sequence like ["�", "�", "p"] where first two should
711 - become empty strings.
675 + Scenario: Multi-byte sequence where intermediate bytes return
676 + empty string and the final byte returns the complete character.
677 """
713 - processor = processor_with_previous_logprobs
714 -
715 - # Test first replacement char
716 - tokens = [100, 101, 102]
678 processor.tokenizer.decode.return_value = "still�"
718 - result1 = processor._correct_decoded_token(0, tokens)
679 +
680 + # First byte with no useful context: returns ""
681 + result1 = processor._correct_decoded_token(100, [50])
682 assert result1 == ""
683
721 - # Test second replacement char
722 - result2 = processor._correct_decoded_token(1, tokens)
684 + # Second byte with same context: still returns ""
685 + result2 = processor._correct_decoded_token(101, [50])
686 assert result2 == ""
687
725 - def test_correction_with_multibyte_utf8(self, processor_with_previous_logprobs):
688 + def test_correction_with_multibyte_utf8(self, processor):
689 """Test correction involving multi-byte UTF-8 characters.
690
728 - Scenario: Byte-fallback tokenization splits multi-byte UTF-8
729 - characters (e.g., curly quotes, Chinese characters, emojis).
730 - Example from user: "�", "�" -> "", "\""
691 + Scenario: Byte-fallback tokenization splits curly quotes.
692 + The last byte token should produce the complete character.
693 """
732 - processor = processor_with_previous_logprobs
733 - tokens = [200, 201]
694
735 - # Mock tokenizer behavior for multi-byte UTF-8 correction
695 def mock_decode(ids):
737 - # When decoding first token (idx=0) with previous logprob token
696 + # Context [123] + first byte: completes to left curly quote
697 if ids == [123, 200]:
739 - return ' "' # Space + left curly quote
740 - # When decoding second token (idx=1) with previous token in list
741 - elif ids == [200, 201]:
742 - return '"' # Right curly quote
743 - # When decoding second token (idx=1) with previous logprob + prev token
744 - elif ids == [123, 200, 201]:
745 - return ' ""' # Full sequence
746 - return "�"
698 + return "hello \u201c"
699 + if ids == [123]:
700 + return "hello "
701 + # Context [123] + second byte: completes to right curly quote
702 + if ids == [123, 201]:
703 + return "hello \u201d"
704 + return "\ufffd"
705
706 processor.tokenizer.decode.side_effect = mock_decode
707
750 - # First token correction (idx=0)
751 - # Will call decode([123, 200]) since idx=0 uses previous logprob token
752 - result1 = processor._correct_decoded_token(0, tokens)
753 - assert result1 == ' "'
708 + # Each top-k token is corrected independently with same context
709 + result1 = processor._correct_decoded_token(200, [123])
710 + assert result1 == "\u201c"
711 +
712 + result2 = processor._correct_decoded_token(201, [123])
713 + assert result2 == "\u201d"
714
755 - # Second token correction (idx=1)
756 - # Will call decode([200, 201]) since idx>0 uses previous token in list
757 - result2 = processor._correct_decoded_token(1, tokens)
758 - assert result2 == '"'
715 + def test_topk_tokens_corrected_independently(self, processor):
716 + """Test that top-k alternatives at the same position are each
717 + corrected independently using only sequential context, not
718 + each other.
719 +
720 + This is the core fix for issue #27300: when logprobs > 0,
721 + alternative tokens must not be combined with each other.
722 + """
723 + # Context: previously sampled token 50
724 + context = [50]
725 +
726 + def mock_decode(ids):
727 + # Token 100 (sampled) with context
728 + if ids == [50, 100]:
729 + return "prefix \u201c"
730 + # Token 200 (top-k alternative) with context
731 + if ids == [50, 200]:
732 + return "prefix \u2014"
733 + # Context alone
734 + if ids == [50]:
735 + return "prefix "
736 + return "\ufffd"
737 +
738 + processor.tokenizer.decode.side_effect = mock_decode
739 +
740 + # Both tokens at the same position use the SAME context [50]
741 + result_sampled = processor._correct_decoded_token(100, context)
742 + assert result_sampled == "\u201c"
743 +
744 + result_alt = processor._correct_decoded_token(200, context)
745 + assert result_alt == "\u2014"
746
747 def test_real_world_opt125m_scenario(self, mock_tokenizer):
761 - """Test the real-world scenario from user's example.
748 + """Test the real-world scenario from the bug report.
749
763 - User's example with facebook/opt-125m:
764 - Before: [" the", " term", " �", "�", "p", "olar", "ized", "�", "�", ...]
765 - After: [" the", " term", "", " "", "p", "olar", "ized", "", "\"", ...]
750 + Simulates the OPT-125m sequence where curly quotes are split
751 + into byte-fallback tokens. Each token is corrected using only
752 + the preceding sampled tokens as context.
753 """
754 from vllm.v1.engine.logprobs import LogprobsProcessor
755
769 - # Simulate the sequence of tokens
756 processor = LogprobsProcessor(
757 tokenizer=mock_tokenizer,
758 logprobs=[],
@@ -776,47 +762,106 @@ class TestCorrectDecodedToken:
762 num_prompt_logprobs=None,
763 )
764
779 - # Token IDs representing the problematic sequence
780 - tokens = [1, 2, 3, 4, 5, 6, 7, 8, 9] # placeholder IDs
781 -
782 - # Mock decode behavior simulating the real scenario
765 + # Simulating: byte tokens 3, 4 form left curly quote "\u201c"
766 + # byte tokens 8, 9 form right curly quote "\u201d"
767 def mock_decode(ids):
784 - # Simulate cases where individual tokens decode to "�"
785 - # but combinations decode correctly
786 - if len(ids) == 1:
787 - if ids[0] in (3, 4, 8, 9):
788 - return "�"
789 - elif len(ids) == 2:
790 - if ids == [2, 3]:
791 - return " term�" # Still ends with �, need more context
792 - elif ids == [3, 4]:
793 - return ' "' # Corrected to space + left curly quote
794 - elif ids == [7, 8]:
795 - return "ized�" # Still ends with �
796 - elif ids == [8, 9]:
797 - return '"' # Corrected to right curly quote
798 - elif len(ids) == 3:
799 - if ids == [1, 2, 3]:
800 - return " the term�" # Still ends with issue
801 - elif ids == [2, 3, 4]:
802 - return ' term "' # With all context
768 + # Context decodes
769 + if ids == [2]:
770 + return " term"
771 + if ids == [1, 2]:
772 + return " the term"
773 + if ids == [3]:
774 + return "\ufffd"
775 + if ids == [2, 3]:
776 + return " term\ufffd"
777 + if ids == [1, 2, 3]:
778 + return " the term\ufffd"
779 + # Token 4 with context [2, 3] -> completes left curly quote
780 + if ids == [3, 4]:
781 + return "\u201c"
782 + if ids == [2, 3, 4]:
783 + return " term\u201c"
784 + # Context for right curly quote
785 + if ids == [7]:
786 + return "ized"
787 + if ids == [7, 8]:
788 + return "ized\ufffd"
789 + if ids == [8, 9]:
790 + return "\u201d"
791 + if ids == [7, 8, 9]:
792 + return "ized\u201d"
793 return "normal_text"
794
795 mock_tokenizer.decode.side_effect = mock_decode
796
807 - # Test token at index 2 (should fail to correct, return "")
808 - # Token 3 individually is "�"
809 - # decode([2, 3]) = " term�" (still ends with �)
810 - # No previous logprobs, so fallback to ""
811 - result = processor._correct_decoded_token(2, tokens)
797 + # First byte (token 3) of left curly quote with no context
798 + result = processor._correct_decoded_token(3, [])
799 + assert result == ""
800 +
801 + # First byte (token 3) with context [2] -> still incomplete
802 + result = processor._correct_decoded_token(3, [2])
803 + assert result == ""
804 +
805 + # Second byte (token 4) of left curly quote with context [2, 3]
806 + # Token 3 is byte-fallback, so clean context is [2] only.
807 + # decode([2, 3, 4]) = " term\u201c", decode([2]) = " term"
808 + # result = "\u201c"
809 + result = processor._correct_decoded_token(4, [2, 3])
810 + assert result == "\u201c"
811 +
812 + # Second byte (token 9) of right curly quote with context [7, 8]
813 + result = processor._correct_decoded_token(9, [7, 8])
814 + assert result == "\u201d"
815 +
816 + def test_byte_fallback_context_preserves_space(self, mock_tokenizer):
817 + """Test that text from byte-fallback context tokens is preserved.
818 +
819 + In OPT-125m, token 44 = space + 2 bytes of curly quote.
820 + When token 44 returns "" (incomplete), the space it carried
821 + must be attributed to the completing token (48).
822 + """
823 + from vllm.v1.engine.logprobs import LogprobsProcessor
824 +
825 + processor = LogprobsProcessor(
826 + tokenizer=mock_tokenizer,
827 + logprobs=[],
828 + prompt_logprobs=None,
829 + cumulative_logprob=0.0,
830 + num_logprobs=1,
831 + num_prompt_logprobs=None,
832 + )
833 +
834 + def mock_decode(ids):
835 + # Token 44 = space + 2 bytes (like OPT-125m's \u0120\u00e2\u0080)
836 + if ids == [44]:
837 + return " \ufffd"
838 + if ids == [48]:
839 + return "\ufffd"
840 + # Together they form: space + left curly quote
841 + if ids == [44, 48]:
842 + return " \u201c"
843 + # With preceding clean context
844 + if ids == [1385]:
845 + return " term"
846 + if ids == [1385, 44]:
847 + return " term \ufffd"
848 + if ids == [1385, 44, 48]:
849 + return " term \u201c"
850 + return "\ufffd"
851 +
852 + mock_tokenizer.decode.side_effect = mock_decode
853 +
854 + # Token 44 with context [1385] -> still ends with replacement
855 + result = processor._correct_decoded_token(44, [1385])
856 assert result == ""
857
814 - # Test token at index 3 (should correct to " "")
815 - # Token 4 individually is "�"
816 - # decode([3, 4]) = " "" (corrected!)
817 - processor.logprobs = [{2: None}] # Add previous logprob
818 - result = processor._correct_decoded_token(3, tokens)
819 - assert result == ' "'
858 + # Token 48 with context [1385, 44]:
859 + # Token 44 is byte-fallback, so clean context is [1385].
860 + # decode([1385, 44, 48]) = " term \u201c"
861 + # decode([1385]) = " term"
862 + # result = " \u201c" (space preserved from token 44!)
863 + result = processor._correct_decoded_token(48, [1385, 44])
864 + assert result == " \u201c"
865
866
867 def test_verify_tokens_integration():
vllm/v1/engine/logprobs.py
+133 -26
@@ -7,6 +7,7 @@ from dataclasses import dataclass
7
8 from vllm.logger import init_logger
9 from vllm.logprobs import (
10 + FlatLogprobs,
11 PromptLogprobs,
12 SampleLogprobs,
13 append_logprobs_for_next_position,
@@ -96,8 +97,11 @@ class LogprobsProcessor:
97 decoded_tokens_list = convert_ids_list_to_tokens(
98 self.tokenizer, token_ids
99 )
100 + context_token_ids = self._get_sampled_context_ids(self.logprobs)
101 decoded_tokens = self._verify_tokens(
100 - decoded_tokens_list=decoded_tokens_list, tokens=token_ids
102 + decoded_tokens_list=decoded_tokens_list,
103 + tokens=token_ids,
104 + context_token_ids=context_token_ids,
105 )
106
107 # Sampler puts the sampled logprob in first.
@@ -162,9 +166,14 @@ class LogprobsProcessor:
166 else:
167 # Extract decoded tokens for this position
168 decoded_tokens_slice = all_decoded_tokens[offset:offset_end]
169 + # Context: preceding prompt tokens accumulated in
170 + # self.prompt_logprobs from previous loop iterations.
171 + context_token_ids = self._get_sampled_context_ids(self.prompt_logprobs)
172 # Apply UTF-8 correction within this position's token boundaries
173 decoded_tokens_for_pos = self._verify_tokens(
167 - decoded_tokens_list=decoded_tokens_slice, tokens=token_ids_list[pos]
174 + decoded_tokens_list=decoded_tokens_slice,
175 + tokens=token_ids_list[pos],
176 + context_token_ids=context_token_ids,
177 )
178
179 # Update with the Logprob container for this pos.
@@ -196,41 +205,139 @@ class LogprobsProcessor:
205 self.prompt_logprobs = []
206 return plp
207
199 - def _correct_decoded_token(self, idx: int, tokens: list[int]) -> str:
200 - assert self.tokenizer is not None, "self.tokenizer should not be None"
201 -
202 - # try with prev token id in same list
203 - if idx > 0:
204 - possible_decoded_token = self.tokenizer.decode(tokens[idx - 1 : idx + 1])
205 - if not possible_decoded_token.endswith("�"):
206 - return possible_decoded_token
207 - # try with previous logprob token id
208 - if self.logprobs:
209 - latest_token_id = next(iter(self.logprobs[-1]))
210 -
211 - decode_ids = [latest_token_id]
212 - if idx > 0:
213 - decode_ids.extend(tokens[idx - 1 : idx + 1])
208 + @staticmethod
209 + def _get_sampled_context_ids(
210 + logprobs_source: SampleLogprobs | PromptLogprobs | None,
211 + max_context: int = 4,
212 + ) -> list[int]:
213 + """Extract recent sampled token IDs from a logprobs source.
214 +
215 + The sampled (or prompt) token at each position is the first
216 + entry, since it is always inserted first by
217 + append_logprobs_for_next_position.
218 +
219 + Args:
220 + logprobs_source: The logprobs container to extract from.
221 + max_context: Maximum number of preceding tokens to return.
222 + 4 is sufficient for any UTF-8 multi-byte sequence.
223 +
224 + Returns:
225 + List of sampled token IDs, oldest first, most recent last.
226 + """
227 + if not logprobs_source:
228 + return []
229 +
230 + n = len(logprobs_source)
231 + start = max(0, n - max_context)
232 +
233 + # Efficient path for FlatLogprobs: access token_ids directly.
234 + if isinstance(logprobs_source, FlatLogprobs):
235 + return [
236 + logprobs_source.token_ids[logprobs_source.start_indices[i]]
237 + for i in range(start, n)
238 + if logprobs_source.start_indices[i] < logprobs_source.end_indices[i]
239 + ]
240 +
241 + # list[dict] path
242 + result: list[int] = []
243 + for i in range(start, n):
244 + entry = logprobs_source[i]
245 + if entry is not None:
246 + result.append(next(iter(entry)))
247 + return result
248 +
249 + def _correct_decoded_token(
250 + self, token_id: int, context_token_ids: list[int]
251 + ) -> str:
252 + """Correct a decoded token that contains the replacement character.
253 +
254 + When byte-fallback tokenization splits multi-byte UTF-8
255 + characters across tokens, individual token decoding produces
256 + the replacement character U+FFFD. This method uses preceding
257 + sampled tokens as context to reconstruct the correct text.
258 +
259 + Args:
260 + token_id: The single token ID to correct.
261 + context_token_ids: Preceding sampled token IDs in sequential
262 + order (oldest first). These are the actual tokens in
263 + the generated sequence, NOT top-k alternatives.
264 +
265 + Returns:
266 + The corrected decoded string, or empty string if the byte
267 + sequence is genuinely incomplete at this point.
268 + """
269 + assert self.tokenizer is not None
270 +
271 + max_ctx = min(len(context_token_ids), 4)
272 +
273 + for num_ctx in range(1, max_ctx + 1):
274 + context = context_token_ids[-num_ctx:]
275 + full_decoded = self.tokenizer.decode(context + [token_id])
276 +
277 + if full_decoded.endswith("�"):
278 + continue
279 +
280 + # Find the boundary between "clean" context tokens and
281 + # byte-fallback tokens that are part of the same incomplete
282 + # sequence. Byte-fallback context tokens returned "" when
283 + # they were processed, so their text must be attributed to
284 + # this completing token.
285 + clean_end = len(context)
286 + for j in range(len(context) - 1, -1, -1):
287 + if self.tokenizer.decode([context[j]]).endswith("�"):
288 + clean_end = j
289 + else:
290 + break
291 +
292 + # Decode only the clean (non-byte-fallback) prefix.
293 + if clean_end > 0:
294 + clean_prefix = self.tokenizer.decode(context[:clean_end])
295 else:
215 - decode_ids.extend(tokens[idx : idx + 1])
296 + clean_prefix = ""
297
217 - possible_decoded_token = self.tokenizer.decode(decode_ids)
218 - if not possible_decoded_token.endswith("�"):
219 - return possible_decoded_token
298 + if full_decoded.startswith(clean_prefix):
299 + return full_decoded[len(clean_prefix) :]
300 +
301 + # Tokenizer normalization may cause prefix mismatch.
302 + # Find the longest common prefix between them.
303 + common_len = 0
304 + for a, b in zip(clean_prefix, full_decoded):
305 + if a != b:
306 + break
307 + common_len += 1
308 + return full_decoded[common_len:]
309
221 - # by default return empty string
310 return ""
311
312 def _verify_tokens(
225 - self, decoded_tokens_list: list[str], tokens: list[int]
313 + self,
314 + decoded_tokens_list: list[str],
315 + tokens: list[int],
316 + context_token_ids: list[int] | None = None,
317 ) -> list[str]:
318 + """Verify and correct decoded tokens with replacement characters.
319 +
320 + Args:
321 + decoded_tokens_list: Decoded token strings to verify.
322 + tokens: Token IDs corresponding to decoded_tokens_list.
323 + These are alternatives at the SAME position (e.g.
324 + [sampled, top1, top2]), NOT sequential tokens.
325 + context_token_ids: Preceding sampled token IDs providing
326 + sequential context. If None, extracted from
327 + self.logprobs.
328 + """
329 + if context_token_ids is None:
330 + context_token_ids = self._get_sampled_context_ids(self.logprobs)
331 +
332 corrected_decoded_token_map = dict()
333 for idx, text in enumerate(decoded_tokens_list):
334 if text.endswith("�"):
230 - # utf-8 char at the end means it's a potential unfinished byte sequence
231 - # from byte fallback tokenization.
335 + # Replacement char at the end means a potential
336 + # unfinished byte sequence from byte-fallback
337 + # tokenization. Correct each token independently
338 + # using only the sequential context.
339 corrected_decoded_token_map[idx] = self._correct_decoded_token(
233 - idx, tokens
340 + tokens[idx], context_token_ids
341 )
342
343 for idx, text in corrected_decoded_token_map.items():