ea5ff3a1f6 · tduncan/Nvidia-Optimization-Stack-VLLM

tests/detokenizer/test_min_tokens.py

-1

@@ -39,7 +39,6 @@ def test_min_tokens_with_stop(min_tokens: int, stop: str, truth: str):

39 mm_features=None,

40 sampling_params=params,

41 pooling_params=None,

42 - eos_token_id=None,

42 arrival_time=0.0,

43 lora_request=None,

44 cache_salt=None,

tests/detokenizer/test_stop_string_while_stop_model_terminates.py

-1

@@ -35,7 +35,6 @@ def _make_request(stop, include_stop_str_in_output: bool, min_tokens: int = 0):

35 mm_features=None,

36 sampling_params=params,

37 pooling_params=None,

38 - eos_token_id=None,

38 arrival_time=0.0,

39 lora_request=None,

40 cache_salt=None,

tests/tokenizers_/test_detokenize.py

-1

@@ -67,7 +67,6 @@ def _run_incremental_decode(

67 mm_features=None,

68 sampling_params=params,

69 pooling_params=None,

70 - eos_token_id=None,

70 arrival_time=0.0,

71 lora_request=None,

72 cache_salt=None,

tests/tool_parsers/test_step3p5_tool_parser.py

+1 -1

@@ -1123,7 +1123,7 @@ rectangle

1123

1124 # Encode all content tokens at once

1125 all_token_ids = step3p5_tokenizer.encode(model_output, add_special_tokens=False)

1126 - eos_token_id = getattr(step3p5_tokenizer, "eos_token_id", None)

1126 + eos_token_id = step3p5_tokenizer.eos_token_id

1127

1128 # Include EOS token in delta_token_ids if available

1129 if eos_token_id is not None:

tests/v1/core/test_kv_cache_utils.py

+4 -2

@@ -84,13 +84,15 @@ def make_request(

84 )

85 mm_features.append(mm_feature)

86

87 + sampling_params = SamplingParams(max_tokens=17)

88 + sampling_params.update_from_generation_config({}, eos_token_id=100)

89 +

90 return Request(

91 request_id=request_id,

92 prompt_token_ids=prompt_token_ids,

93 mm_features=mm_features if mm_features else None,

91 - sampling_params=SamplingParams(max_tokens=17),

94 + sampling_params=sampling_params,

95 pooling_params=None,

93 - eos_token_id=100,

96 lora_request=None,

97 cache_salt=cache_salt,

98 block_hasher=get_request_block_hasher(block_size, hash_fn),

tests/v1/core/test_prefix_caching.py

+4 -2

@@ -75,13 +75,15 @@ def make_request(

75 )

76 mm_features.append(mm_feature)

77

78 + sampling_params = SamplingParams(max_tokens=17, prompt_logprobs=prompt_logprobs)

79 + sampling_params.update_from_generation_config({}, eos_token_id=100)

80 +

81 return Request(

82 request_id=request_id,

83 prompt_token_ids=prompt_token_ids,

84 mm_features=mm_features if mm_features else None,

82 - sampling_params=SamplingParams(max_tokens=17, prompt_logprobs=prompt_logprobs),

85 + sampling_params=sampling_params,

86 pooling_params=None,

84 - eos_token_id=100,

87 lora_request=lora_request,

88 cache_salt=cache_salt,

89 block_hasher=get_request_block_hasher(block_size, hash_fn),

tests/v1/core/test_priority_scheduler_random.py

+3 -5

@@ -48,10 +48,9 @@ def _create_random_request(

48

49 request_id = uuid.uuid4().hex

50

51 - sampling_params = SamplingParams(

52 - ignore_eos=False,

53 - max_tokens=max_tokens,

54 - )

51 + sampling_params = SamplingParams(ignore_eos=False, max_tokens=max_tokens)

52 + sampling_params.update_from_generation_config({}, EOS_TOKEN_ID)

53 +

54 mm_features = []

55 for j, position in enumerate(mm_positions):

56 identifier = f"{request_id}_hash_{j}"

@@ -79,7 +78,6 @@ def _create_random_request(

78 sampling_params=sampling_params,

79 pooling_params=None,

80 mm_features=mm_features if mm_features else None,

82 - eos_token_id=EOS_TOKEN_ID,

81 arrival_time=arrival_time,

82 priority=priority,

83 block_hasher=block_hasher,

tests/v1/core/test_scheduler.py

+6 -7

@@ -469,8 +469,7 @@ def test_stop_via_update_from_output():

469

470 # Test case 4: Ignore EOS flag

471 scheduler = create_scheduler(num_speculative_tokens=2)

472 - requests = create_requests(num_requests=1, max_tokens=10)

473 - requests[0].sampling_params.ignore_eos = True

472 + requests = create_requests(num_requests=1, max_tokens=10, ignore_eos=True)

473 requests[0].num_computed_tokens = requests[0].num_tokens

474 scheduler.requests[requests[0].request_id] = requests[0]

475 scheduler.running.append(requests[0])

@@ -515,12 +514,12 @@ def test_check_stop_min_tokens():

514 max_tokens=20,

515 min_tokens=5,

516 )

517 + sampling_params.update_from_generation_config({}, EOS_TOKEN_ID)

518 request = Request(

519 request_id="0",

520 prompt_token_ids=[0, 1, 2],

521 sampling_params=sampling_params,

522 pooling_params=None,

523 - eos_token_id=EOS_TOKEN_ID,

523 )

524 # Simulate having generated 3 output tokens (less than min_tokens=5)

525 request.append_output_token_ids([10, 11, EOS_TOKEN_ID]) # EOS token present

@@ -551,12 +550,12 @@ def test_check_stop_min_tokens():

550 max_tokens=20,

551 min_tokens=0,

552 )

553 + sampling_params_no_min.update_from_generation_config({}, EOS_TOKEN_ID)

554 request_no_min = Request(

555 request_id="1",

556 prompt_token_ids=[0, 1, 2],

557 sampling_params=sampling_params_no_min,

558 pooling_params=None,

559 - eos_token_id=EOS_TOKEN_ID,

559 )

560 request_no_min.append_output_token_ids([10, EOS_TOKEN_ID])

561

@@ -571,12 +570,12 @@ def test_check_stop_min_tokens():

570 min_tokens=5,

571 stop_token_ids=[42],

572 )

573 + sampling_params_stop.update_from_generation_config({}, EOS_TOKEN_ID)

574 request_stop = Request(

575 request_id="2",

576 prompt_token_ids=[0, 1, 2],

577 sampling_params=sampling_params_stop,

578 pooling_params=None,

579 - eos_token_id=EOS_TOKEN_ID,

579 )

580 # Only 3 output tokens, less than min_tokens=5, but has stop token

581 request_stop.append_output_token_ids([10, 11, 42])

@@ -1877,6 +1876,7 @@ def create_requests_with_priority(

1876 stop_token_ids=stop_token_ids,

1877 prompt_logprobs=prompt_logprobs,

1878 )

1879 + sampling_params.update_from_generation_config({}, EOS_TOKEN_ID)

1880 requests = []

1881

1882 if mm_hashes_list is not None:

@@ -1938,7 +1938,6 @@ def create_requests_with_priority(

1938 sampling_params=sampling_params,

1939 pooling_params=None,

1940 mm_features=mm_features if mm_features else None,

1941 - eos_token_id=EOS_TOKEN_ID,

1941 arrival_time=arrival_times[i],

1942 priority=priorities[i],

1943 block_hasher=block_hasher,

@@ -2429,13 +2428,13 @@ def test_schedule_skip_tokenizer_init_structured_output_request():

2428 max_tokens=16,

2429 structured_outputs=structured_outputs_params,

2430 )

2431 + sampling_params.update_from_generation_config({}, EOS_TOKEN_ID)

2432 request = Request(

2433 request_id="0",

2434 prompt_token_ids=[0, 1],

2435 mm_features=None,

2436 sampling_params=sampling_params,

2437 pooling_params=None,

2438 - eos_token_id=EOS_TOKEN_ID,

2438 )

2439 scheduler.add_request(request)

2440 output = scheduler.schedule()

tests/v1/core/utils.py

+3 -2

@@ -174,6 +174,7 @@ def create_requests(

174 num_tokens: int = 10,

175 mm_hashes_list: list[list[str]] | None = None,

176 mm_positions: list[list[PlaceholderRange]] | None = None,

177 + ignore_eos: bool = False,

178 max_tokens: int = 16,

179 stop_token_ids: list[int] | None = None,

180 prompt_logprobs: int | None = None,

@@ -188,11 +189,12 @@ def create_requests(

189

190 block_hasher = get_request_block_hasher(block_size, sha256)

191 sampling_params = SamplingParams(

191 - ignore_eos=False,

192 + ignore_eos=ignore_eos,

193 max_tokens=max_tokens,

194 stop_token_ids=stop_token_ids,

195 prompt_logprobs=prompt_logprobs,

196 )

197 + sampling_params.update_from_generation_config({}, EOS_TOKEN_ID)

198 requests = []

199

200 if mm_hashes_list is not None:

@@ -250,7 +252,6 @@ def create_requests(

252 sampling_params=sampling_params,

253 pooling_params=None,

254 mm_features=mm_features if mm_features else None,

253 - eos_token_id=EOS_TOKEN_ID,

255 block_hasher=block_hasher,

256 )

257 requests.append(request)

tests/v1/engine/test_engine_core.py

-1

@@ -54,7 +54,6 @@ def make_request() -> EngineCoreRequest:

54 mm_features=None,

55 sampling_params=SamplingParams(),

56 pooling_params=None,

57 - eos_token_id=None,

57 arrival_time=time.time(),

58 lora_request=None,

59 cache_salt=None,

tests/v1/engine/test_engine_core_client.py

-1

@@ -69,7 +69,6 @@ def make_request(

69 mm_features=None,

70 sampling_params=params,

71 pooling_params=None,

72 - eos_token_id=None,

72 arrival_time=time.time(),

73 lora_request=None,

74 cache_salt=None,

tests/v1/engine/test_fast_incdec_prefix_err.py

-1

@@ -32,7 +32,6 @@ def test_fast_inc_detok_invalid_utf8_err_case():

32 mm_features=None,

33 sampling_params=params,

34 pooling_params=None,

35 - eos_token_id=None,

35 arrival_time=0.0,

36 lora_request=None,

37 cache_salt=None,

tests/v1/engine/test_output_processor.py

+16 -21

@@ -66,7 +66,6 @@ def test_incremental_detokenization(

66 external_req_id=f"request-{idx}",

67 prompt_token_ids=prompt_tokens,

68 mm_features=None,

69 - eos_token_id=None,

69 arrival_time=0,

70 lora_request=None,

71 cache_salt=None,

@@ -487,7 +486,6 @@ def test_logprobs_processor(

486 external_req_id=request_id_list[idx],

487 prompt_token_ids=prompt_tokens,

488 mm_features=None,

490 - eos_token_id=None,

489 arrival_time=0,

490 lora_request=None,

491 cache_salt=None,

@@ -663,6 +661,19 @@ def test_stop_token(

661 prompt_string = dummy_test_vectors.prompt_strings[0]

662 prompt_tokens = dummy_test_vectors.prompt_tokens[0]

663

664 + sampling_params = SamplingParams(

665 + skip_special_tokens=False,

666 + spaces_between_special_tokens=False,

667 + output_kind=RequestOutputKind.DELTA,

668 + stop=[],

669 + stop_token_ids=stop_token_ids,

670 + include_stop_str_in_output=include_stop_str_in_output,

671 + logprobs=num_sample_logprobs,

672 + prompt_logprobs=None,

673 + ignore_eos=ignore_eos,

674 + )

675 + sampling_params.update_from_generation_config({}, eos_token_id)

676 +

677 # Make request.

678 request_id = "request-0"

679 request = EngineCoreRequest(

@@ -670,22 +681,11 @@ def test_stop_token(

681 external_req_id=request_id + "-ext",

682 prompt_token_ids=prompt_tokens,

683 mm_features=None,

673 - eos_token_id=eos_token_id,

684 arrival_time=0,

685 lora_request=None,

686 cache_salt=None,

687 data_parallel_rank=None,

678 - sampling_params=SamplingParams(

679 - skip_special_tokens=False,

680 - spaces_between_special_tokens=False,

681 - output_kind=RequestOutputKind.DELTA,

682 - stop=[],

683 - stop_token_ids=stop_token_ids,

684 - include_stop_str_in_output=include_stop_str_in_output,

685 - logprobs=num_sample_logprobs,

686 - prompt_logprobs=None,

687 - ignore_eos=ignore_eos,

688 - ),

688 + sampling_params=sampling_params,

689 pooling_params=None,

690 )

691

@@ -693,9 +693,8 @@ def test_stop_token(

693 tokens_list=[generation_tokens],

694 generated_logprobs_raw=[generation_logprobs] if do_logprobs else None,

695 prompt_logprobs_raw=None,

696 - eos_token_id=eos_token_id,

697 - stop_token_ids=stop_token_ids,

698 - ignore_eos=ignore_eos,

696 + eos_token_id=sampling_params.eos_token_id,

697 + stop_token_ids=sampling_params.stop_token_ids,

698 request_ids=[request.request_id],

699 )

700

@@ -775,7 +774,6 @@ def test_stop_string(

774 external_req_id=request_id_list[idx],

775 prompt_token_ids=prompt_tokens,

776 mm_features=None,

778 - eos_token_id=None,

777 arrival_time=0,

778 lora_request=None,

779 cache_salt=None,

@@ -907,7 +905,6 @@ def test_iteration_stats(dummy_test_vectors):

905 external_req_id=f"request-{idx}-ext",

906 prompt_token_ids=prompt_tokens,

907 mm_features=None,

910 - eos_token_id=None,

908 arrival_time=0,

909 lora_request=None,

910 cache_salt=None,

@@ -994,7 +991,6 @@ def test_lora_request_tracking(log_stats: bool, dummy_test_vectors):

991 external_req_id=f"request-{idx}",

992 prompt_token_ids=prompt_tokens,

993 mm_features=None,

997 - eos_token_id=None,

994 arrival_time=0,

995 lora_request=lora_assignments[idx],

996 cache_salt=None,

@@ -1315,7 +1311,6 @@ def test_abort_requests(runner: str, abort_by: str, dummy_test_vectors):

1311 external_req_id=f"external-{idx}",

1312 prompt_token_ids=prompt_tokens,

1313 mm_features=None,

1318 - eos_token_id=None,

1314 arrival_time=0,

1315 lora_request=None,

1316 cache_salt=None,

tests/v1/engine/test_parallel_sampling.py

-1

@@ -76,7 +76,6 @@ def make_request(sampling_params: SamplingParams) -> EngineCoreRequest:

76 mm_features=None,

77 sampling_params=sampling_params,

78 pooling_params=None,

79 - eos_token_id=None,

79 arrival_time=0.0,

80 lora_request=None,

81 cache_salt=None,

tests/v1/engine/utils.py

+1 -3

@@ -342,7 +342,6 @@ class MockEngineCore:

342 prompt_logprobs_raw: list[LogprobsTensors] | None = None,

343 eos_token_id: int | None = None,

344 stop_token_ids: list[int] | None = None,

345 - ignore_eos: bool = False,

345 request_ids: list[str] | None = None,

346 ) -> None:

347 self.num_requests = len(tokens_list)

@@ -355,7 +354,6 @@ class MockEngineCore:

354 self.request_finished = [False for _ in range(self.num_requests)]

355 self.eos_token_id = eos_token_id

356 self.stop_token_ids = stop_token_ids

358 - self.ignore_eos = ignore_eos

357 self.request_ids = (

358 request_ids

359 if request_ids is not None

@@ -400,7 +398,7 @@ class MockEngineCore:

398 if token_idx == len(token_ids) - 1:

399 output.finish_reason = FinishReason.LENGTH

400 self.request_finished[req_idx] = True

403 - if not self.ignore_eos and new_token_id == self.eos_token_id:

401 + if new_token_id == self.eos_token_id:

402 output.finish_reason = FinishReason.STOP

403 self.request_finished[req_idx] = True

404 if new_token_id in (self.stop_token_ids or ()):

tests/v1/kv_connector/unit/test_decode_bench_connector.py

+4 -2

@@ -93,12 +93,14 @@ class DecodeBenchTestRunner:

93 """Create a new request with given token IDs."""

94 self.req_id += 1

95

96 + sampling_params = SamplingParams(max_tokens=100)

97 + sampling_params.update_from_generation_config({}, EOS_TOKEN_ID)

98 +

99 req = Request(

100 request_id=str(self.req_id),

101 prompt_token_ids=token_ids,

99 - sampling_params=SamplingParams(max_tokens=100),

102 + sampling_params=sampling_params,

103 pooling_params=None,

101 - eos_token_id=EOS_TOKEN_ID,

104 block_hasher=self._block_hasher,

105 )

106

tests/v1/kv_connector/unit/test_lmcache_integration.py

+4 -2

@@ -142,12 +142,14 @@ def test_request_interface():

142 from vllm.sampling_params import SamplingParams

143 from vllm.v1.request import Request

144

145 + sampling_params = SamplingParams(max_tokens=10)

146 + sampling_params.update_from_generation_config({}, eos_token_id=100)

147 +

148 req = Request(

149 request_id="test_request",

150 prompt_token_ids=[1, 2, 3],

148 - sampling_params=SamplingParams(max_tokens=10),

151 + sampling_params=sampling_params,

152 pooling_params=None,

150 - eos_token_id=100,

153 lora_request=None,

154 )

155 assumes(req, "mm_features", is_instance_of=(list, NoneType))

tests/v1/kv_connector/unit/test_offloading_connector.py

+4 -2

@@ -226,12 +226,14 @@ class RequestRunner:

226 def new_request(self, token_ids: list[int]):

227 self.req_id += 1

228

229 + sampling_params = SamplingParams(max_tokens=1000)

230 + sampling_params.update_from_generation_config({}, EOS_TOKEN_ID)

231 +

232 req = Request(

233 request_id=str(self.req_id),

234 prompt_token_ids=token_ids,

232 - sampling_params=SamplingParams(max_tokens=1000),

235 + sampling_params=sampling_params,

236 pooling_params=None,

234 - eos_token_id=EOS_TOKEN_ID,

237 block_hasher=self._block_hasher,

238 )

239

tests/v1/kv_connector/unit/utils.py

+1 -1

@@ -212,6 +212,7 @@ def create_request(

212

213 max_tokens = 1 if do_remote_decode else max_tokens

214 sampling_params = SamplingParams(max_tokens=max_tokens)

215 + sampling_params.update_from_generation_config({}, EOS_TOKEN_ID)

216

217 common_prefix = [1] * common_prefix_len if common_prefix_len > 0 else []

218 suffix = [i * request_id for i in range(num_tokens - common_prefix_len)]

@@ -223,7 +224,6 @@ def create_request(

224 sampling_params=sampling_params,

225 pooling_params=None,

226 mm_features=None,

226 - eos_token_id=EOS_TOKEN_ID,

227 block_hasher=get_request_block_hasher(block_size, hash_fn),

228 )

229 req.kv_transfer_params = kv_transfer_params

tests/v1/streaming_input/test_scheduler_streaming.py

-1

@@ -43,7 +43,6 @@ class DummyRequest(Request):

43 stop_token_ids=[STOP_TOKEN], max_tokens=max_tokens

44 ),

45 pooling_params=None,

46 - eos_token_id=None,

46 mm_features=mm_features,

47 resumable=resumable,

48 )

tests/v1/structured_output/test_backend_guidance.py

+2 -2

@@ -83,6 +83,7 @@ def test_grammar_bitmask_with_specdec():

83 ),

84 )

85 sampling_params.structured_outputs._backend = "guidance"

86 + sampling_params.update_from_generation_config({}, tokenizer.eos_token_id)

87

88 my_req_id = f"my_req_id_{i}"

89 request = Request(

@@ -90,7 +91,6 @@ def test_grammar_bitmask_with_specdec():

91 prompt_token_ids=prompt[:i],

92 sampling_params=sampling_params,

93 pooling_params=None,

93 - eos_token_id=tokenizer.eos_token_id,

94 )

95

96 structured_output_manager.grammar_init(request)

@@ -147,13 +147,13 @@ def test_grammar_init_async_and_sync(async_grammar):

147 ),

148 )

149 sampling_params.structured_outputs._backend = "guidance"

150 + sampling_params.update_from_generation_config({}, tokenizer.eos_token_id)

151

152 request = Request(

153 "test_request",

154 prompt_token_ids=prompt,

155 sampling_params=sampling_params,

156 pooling_params=None,

156 - eos_token_id=tokenizer.eos_token_id,

157 )

158

159 structured_output_manager.grammar_init(request)

vllm/inputs/preprocess.py

+3 -22

@@ -77,24 +77,6 @@ class InputPreprocessor:

77 def get_tokenizer(self) -> TokenizerLike:

78 return self.renderer.get_tokenizer()

79

80 - def get_bos_token_id(self) -> int | None:

81 - if self.tokenizer is None:

82 - logger.warning_once(

83 - "Using None for BOS token id because tokenizer is not initialized"

84 - )

85 - return None

86 -

87 - return self.tokenizer.bos_token_id

88 -

89 - def get_eos_token_id(self) -> int | None:

90 - if self.tokenizer is None:

91 - logger.warning_once(

92 - "Using None for EOS token id because tokenizer is not initialized"

93 - )

94 - return None

95 -

96 - return self.tokenizer.eos_token_id

97 -

80 def get_decoder_start_token_id(self) -> int:

81 """

82 Obtain the decoder start token id employed by an encoder/decoder

@@ -106,11 +88,10 @@ class InputPreprocessor:

88

89 if dec_start_token_id is None:

90 logger.warning_once(

109 - "Falling back on <BOS> for decoder start token "

110 - "id because decoder start token id is not "

111 - "available."

91 + "Falling back on <BOS> for decoder start token id "

92 + "because decoder start token id is not available."

93 )

113 - dec_start_token_id = self.get_bos_token_id()

94 + dec_start_token_id = self.renderer.get_bos_token_id()

95

96 if dec_start_token_id is None:

97 raise RuntimeError("Cannot find decoder start token id or <BOS>")

vllm/renderers/base.py

+21

@@ -6,6 +6,7 @@ from collections.abc import Sequence

6 from typing import TYPE_CHECKING, Any, overload

7

8 from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt

9 +from vllm.logger import init_logger

10 from vllm.tokenizers import TokenizerLike

11 from vllm.utils.async_utils import AsyncMicrobatchTokenizer

12

@@ -26,6 +27,8 @@ if TYPE_CHECKING:

27 ConversationMessage,

28 )

29

30 +logger = init_logger(__name__)

31 +

32

33 class BaseRenderer(ABC):

34 @classmethod

@@ -63,6 +66,24 @@ class BaseRenderer(ABC):

66

67 return self._async_tokenizer

68

69 + def get_bos_token_id(self) -> int | None:

70 + if self.tokenizer is None:

71 + logger.warning_once(

72 + "Using None for BOS token id because tokenizer is not initialized"

73 + )

74 + return None

75 +

76 + return self.tokenizer.bos_token_id

77 +

78 + def get_eos_token_id(self) -> int | None:

79 + if self.tokenizer is None:

80 + logger.warning_once(

81 + "Using None for EOS token id because tokenizer is not initialized"

82 + )

83 + return None

84 +

85 + return self.tokenizer.eos_token_id

86 +

87 # Step 1: Convert raw inputs to prompts

88 def render_prompt(

89 self,

vllm/sampling_params.py

+12 -5

@@ -223,6 +223,7 @@ class SamplingParams(

223 # The below fields are not supposed to be used as an input.

224 # They are set in post_init.

225 output_text_buffer_length: int = 0

226 + _eos_token_id: int | None = None

227 _all_stop_token_ids: set[int] = msgspec.field(default_factory=set)

228

229 # Fields used to construct logits processors

@@ -477,24 +478,26 @@ class SamplingParams(

478 def update_from_generation_config(

479 self,

480 generation_config: dict[str, Any],

480 - model_eos_token_id: int | None = None,

481 + eos_token_id: int | None = None,

482 ) -> None:

483 """Update if there are non-default values from generation_config"""

484 + if not self.ignore_eos:

485 + self._eos_token_id = eos_token_id

486

484 - if model_eos_token_id is not None:

487 + if eos_token_id is not None:

488 # Add the eos token id into the sampling_params to support

489 # min_tokens processing.

487 - self._all_stop_token_ids.add(model_eos_token_id)

490 + self._all_stop_token_ids.add(eos_token_id)

491

492 # Update eos_token_id for generation

493 if (eos_ids := generation_config.get("eos_token_id")) is not None:

494 # it can be either int or list of int

495 eos_ids = {eos_ids} if isinstance(eos_ids, int) else set(eos_ids)

493 - if model_eos_token_id is not None:

496 + if eos_token_id is not None:

497 # We don't need to include the primary eos_token_id in

498 # stop_token_ids since it's handled separately for stopping

499 # purposes.

497 - eos_ids.discard(model_eos_token_id)

500 + eos_ids.discard(eos_token_id)

501 if eos_ids:

502 self._all_stop_token_ids.update(eos_ids)

503 if not self.ignore_eos:

@@ -550,6 +553,10 @@ class SamplingParams(

553 return SamplingType.RANDOM_SEED

554 return SamplingType.RANDOM

555

556 + @property

557 + def eos_token_id(self) -> int | None:

558 + return self._eos_token_id

559 +

560 @property

561 def all_stop_token_ids(self) -> set[int]:

562 return self._all_stop_token_ids

vllm/v1/core/sched/utils.py

+1 -1

@@ -47,7 +47,7 @@ def check_stop(request: Request, max_model_len: int) -> bool:

47 return False

48

49 last_token_id = request.output_token_ids[-1]

50 - if not sampling_params.ignore_eos and last_token_id == request.eos_token_id:

50 + if last_token_id == sampling_params.eos_token_id:

51 request.status = RequestStatus.FINISHED_STOPPED

52 return True

53

vllm/v1/engine/__init__.py

+12 -1

@@ -9,6 +9,7 @@ from typing import Any, Literal

9 import msgspec

10 import numpy as np

11 import torch

12 +from typing_extensions import deprecated

13

14 from vllm.lora.request import LoRARequest

15 from vllm.multimodal.inputs import MultiModalFeatureSpec

@@ -63,7 +64,6 @@ class EngineCoreRequest(

64 mm_features: list[MultiModalFeatureSpec] | None

65 sampling_params: SamplingParams | None

66 pooling_params: PoolingParams | None

66 - eos_token_id: int | None

67 arrival_time: float

68 lora_request: LoRARequest | None

69 cache_salt: str | None

@@ -99,6 +99,17 @@ class EngineCoreRequest(

99 assert self.pooling_params is not None

100 return self.pooling_params

101

102 + @property

103 + @deprecated(

104 + "EngineCoreRequest.eos_token_id will be removed in v0.18. "

105 + "Please use EngineCoreRequest.sampling_params.eos_token_id instead."

106 + )

107 + def eos_token_id(self) -> int | None:

108 + if self.sampling_params is None:

109 + return None

110 +

111 + return self.sampling_params.eos_token_id

112 +

113

114 class EngineCoreEventType(enum.IntEnum):

115 """The type of engine core request event."""

vllm/v1/engine/input_processor.py

+1 -4

@@ -376,8 +376,6 @@ class InputProcessor:

376 processed_inputs=processed_inputs,

377 )

378

379 - eos_token_id = self.input_preprocessor.get_eos_token_id()

380 -

379 encoder_inputs, decoder_inputs = split_enc_dec_inputs(processed_inputs)

380 self._validate_model_inputs(encoder_inputs, decoder_inputs)

381

@@ -403,7 +401,7 @@ class InputProcessor:

401

402 sampling_params.update_from_generation_config(

403 self.generation_config_fields,

406 - None if self.tokenizer is None else self.tokenizer.eos_token_id,

404 + self.renderer.get_eos_token_id(),

405 )

406 if self.tokenizer is not None:

407 sampling_params.update_from_tokenizer(self.tokenizer)

@@ -446,7 +444,6 @@ class InputProcessor:

444 mm_features=mm_features,

445 sampling_params=sampling_params,

446 pooling_params=pooling_params,

449 - eos_token_id=eos_token_id,

447 arrival_time=arrival_time,

448 lora_request=lora_request,

449 cache_salt=decoder_inputs.get("cache_salt"),

vllm/v1/request.py

+12 -4

@@ -9,6 +9,7 @@ from dataclasses import dataclass

9 from typing import TYPE_CHECKING, Any

10

11 import torch

12 +from typing_extensions import deprecated

13

14 from vllm.multimodal.inputs import MultiModalFeatureSpec

15 from vllm.pooling_params import PoolingParams

@@ -62,7 +63,6 @@ class Request:

63 prompt_token_ids: list[int] | None,

64 sampling_params: SamplingParams | None,

65 pooling_params: PoolingParams | None,

65 - eos_token_id: int | None,

66 client_index: int = 0,

67 arrival_time: float | None = None,

68 prompt_embeds: torch.Tensor | None = None,

@@ -80,8 +80,6 @@ class Request:

80 self.priority = priority

81 self.sampling_params = sampling_params

82 self.pooling_params = pooling_params

83 - # Because of LoRA, the eos token id can be different for each request.

84 - self.eos_token_id = eos_token_id

83 self.lora_request = lora_request

84 self.structured_output_request = StructuredOutputRequest.from_sampling_params(

85 sampling_params

@@ -176,6 +174,17 @@ class Request:

174 # None entry in the queue means finished.

175 self.streaming_queue: deque[StreamingUpdate | None] | None = None

176

177 + @property

178 + @deprecated(

179 + "Request.eos_token_id will be removed in v0.18. "

180 + "Please use Request.sampling_params.eos_token_id instead."

181 + )

182 + def eos_token_id(self) -> int | None:

183 + if self.sampling_params is None:

184 + return None

185 +

186 + return self.sampling_params.eos_token_id

187 +

188 @classmethod

189 def from_engine_core_request(

190 cls,

@@ -190,7 +199,6 @@ class Request:

199 mm_features=request.mm_features,

200 sampling_params=request.sampling_params,

201 pooling_params=request.pooling_params,

193 - eos_token_id=request.eos_token_id,

202 arrival_time=request.arrival_time,

203 lora_request=request.lora_request,

204 cache_salt=request.cache_salt,

vllm/v1/structured_output/utils.py

+8 -26

@@ -185,14 +185,13 @@ re_llama_byte_token = re.compile(r"^<0x[0-9A-F]{2}>$")

185 re_replacement_seq = re.compile(r"^.{0,6}�+.{0,6}$")

186

187

188 -def _reduced_vocabulary(

189 - tokenizer: TokenizerLike, eos_token_id: int

190 -) -> dict[bytes, list[int]]:

188 +def _reduced_vocabulary(tokenizer: TokenizerLike) -> dict[bytes, list[int]]:

189 """Create a map from vocabulary tokens to lists of equivalent token ids.

190

191 Returns:

192 A Dict of token string -> equivalent token ids

193 """

194 + eos_token_id = tokenizer.eos_token_id

195

196 unicode_to_bytes = {

197 v: k for k, v in convert_slow_tokenizer.bytes_to_unicode().items()

@@ -260,30 +259,13 @@ def get_outlines_vocabulary(tokenizer: TokenizerLike) -> oc.Vocabulary:

259 if hasattr(tokenizer, "_outlines_vocabulary"):

260 return tokenizer._outlines_vocabulary # type: ignore

261

263 - try:

264 - if hasattr(tokenizer, "eos_token_id") and tokenizer.eos_token_id is not None:

265 - eos_token_id = tokenizer.eos_token_id

266 - else:

267 - raise ValueError(

268 - "Error during structured outputs setup for outlines: Tokenizer "

269 - f"({type(tokenizer)}) has no `eos_token_id` property, but "

270 - "`eos_token_id` is required for structured outputs to work properly."

271 - )

272 -

273 - reduced_vocab = _reduced_vocabulary(

274 - tokenizer,

275 - eos_token_id, # type: ignore

276 - )

277 - vocabulary = OutlinesVocabulary(oc.Vocabulary(eos_token_id, reduced_vocab))

278 - tokenizer._outlines_vocabulary = vocabulary # type: ignore

262 + reduced_vocab = _reduced_vocabulary(tokenizer)

263 + vocabulary = OutlinesVocabulary(

264 + oc.Vocabulary(tokenizer.eos_token_id, reduced_vocab)

265 + )

266 + tokenizer._outlines_vocabulary = vocabulary # type: ignore

267

280 - return vocabulary

281 - except AttributeError as e:

282 - raise ValueError(

283 - "Cannot get the vocabulary of the tokenizer "

284 - f"({type(tokenizer)}). The tokenizer should have a "

285 - "get_vocab method."

286 - ) from e

268 + return vocabulary

269

270

271 def grammar_is_likely_lark(grammar_str: str) -> bool:

[Refactor] Simplify BOS/EOS token handling (#34435)