@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6c47f6bfa4

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Core] Remove tokenizer group in vLLM (#24078)

Signed-off-by: Zhuohan Li <zhuohan123@gmail.com>

Zhuohan Li committed Sep 17, 2025 at 01:42 UTC 6c47f6bfa4794178035d9d941d7d40c1d71473b7

49 files changed +276 -934

tests/detokenizer/test_stop_checker.py

+1 -7

@@ -1,10 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 -from unittest.mock import MagicMock

5 -

4 import pytest

7 -from transformers import PreTrainedTokenizer

6 from vllm.engine.output_processor.stop_checker import StopChecker

7 from vllm.inputs import token_inputs

@@ -54,10 +51,7 @@ def test_stop_on_eos_token(text_wo_eos: str, eos_token: str, eos_token_id: int,

51 - When the EOS token should be ignored, and the sequence continues

52 """

57 - tokenizer = MagicMock(spec=PreTrainedTokenizer)

58 - get_tokenizer_for_seq = MagicMock(return_value=tokenizer)

59 - stop_checker = StopChecker(max_model_len=1024,

60 - get_tokenizer_for_seq=get_tokenizer_for_seq)

54 + stop_checker = StopChecker(max_model_len=1024)

56 seq = sequence_with_eos(

57 text=text_wo_eos,

tests/engine/test_stop_checker.py

+2 -5

@@ -58,16 +58,13 @@ def deepseek_r1_qwen_tokenizer():

59 @pytest.fixture

60 def stop_checker():

61 - return StopChecker(max_model_len=10,

62 - get_tokenizer_for_seq=deepseek_r1_qwen_tokenizer)

61 + return StopChecker(max_model_len=10)

64 @pytest.fixture

65 def stop_checker_with_reasoner():

66 reasoner = MockReasoningParser(deepseek_r1_qwen_tokenizer)

68 - return StopChecker(max_model_len=10,

69 - get_tokenizer_for_seq=deepseek_r1_qwen_tokenizer,

70 - reasoner=reasoner)

67 + return StopChecker(max_model_len=10, reasoner=reasoner)

70 def test_eos_token_stopping(stop_checker):

tests/entrypoints/conftest.py

-22

@@ -208,25 +208,3 @@ def zephyr_lora_files():

208 """Download zephyr LoRA files once per test session."""

209 from huggingface_hub import snapshot_download

210 return snapshot_download(repo_id="typeof/zephyr-7b-beta-lora")

211 -

212 -

213 -@pytest.fixture(scope="session")

214 -def zephyr_lora_added_tokens_files(zephyr_lora_files):

215 - """Create zephyr LoRA files with added tokens once per test session."""

216 - import shutil

217 - from tempfile import TemporaryDirectory

218 -

219 - from transformers import AutoTokenizer

220 -

221 - tmp_dir = TemporaryDirectory()

222 - tmp_model_dir = f"{tmp_dir.name}/zephyr"

223 - shutil.copytree(zephyr_lora_files, tmp_model_dir)

224 - tokenizer = AutoTokenizer.from_pretrained("HuggingFaceH4/zephyr-7b-beta")

225 - # Copy tokenizer to adapter and add some unique tokens

226 - # 32000, 32001, 32002

227 - added = tokenizer.add_tokens(["vllm1", "vllm2", "vllm3"],

228 - special_tokens=True)

229 - assert added == 3

230 - tokenizer.save_pretrained(tmp_model_dir)

231 - yield tmp_model_dir

232 - tmp_dir.cleanup()

tests/entrypoints/openai/test_chat.py

+2 -7

@@ -29,11 +29,7 @@ def monkeypatch_module():

31 @pytest.fixture(scope="module", params=[False, True])

32 -def server(

33 - request,

34 - monkeypatch_module,

35 - zephyr_lora_files, #noqa: F811

36 - zephyr_lora_added_tokens_files): # noqa: F811

32 +def server(request, monkeypatch_module, zephyr_lora_files): #noqa: F811

34 use_v1 = request.param

35 monkeypatch_module.setenv('VLLM_USE_V1', '1' if use_v1 else '0')

@@ -49,7 +45,6 @@ def server(

45 "--enable-lora",

46 "--lora-modules",

47 f"zephyr-lora={zephyr_lora_files}",

52 - f"zephyr-lora2={zephyr_lora_added_tokens_files}",

48 "--max-lora-rank",

49 "64",

50 "--max-cpu-loras",

@@ -79,7 +74,7 @@ async def client(server):

74 @pytest.mark.parametrize(

75 # first test base model, then test loras

76 "model_name",

82 - [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],

77 + [MODEL_NAME, "zephyr-lora"],

78 )

79 async def test_no_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):

80 messages = [{

tests/entrypoints/openai/test_completion.py

+4 -19

@@ -27,7 +27,7 @@ GUIDED_DECODING_BACKENDS = ["outlines", "xgrammar", "guidance"]

29 @pytest.fixture(scope="module")

30 -def default_server_args(zephyr_lora_files, zephyr_lora_added_tokens_files):

30 +def default_server_args(zephyr_lora_files):

31 return [

32 # use half precision for speed and memory savings in CI environment

33 "--dtype",

@@ -41,7 +41,6 @@ def default_server_args(zephyr_lora_files, zephyr_lora_added_tokens_files):

41 "--enable-lora",

42 "--lora-modules",

43 f"zephyr-lora={zephyr_lora_files}",

44 - f"zephyr-lora2={zephyr_lora_added_tokens_files}",

44 "--max-lora-rank",

45 "64",

46 "--max-cpu-loras",

@@ -87,7 +86,7 @@ async def client(server):

86 @pytest.mark.parametrize(

87 # first test base model, then test loras

88 "model_name",

90 - [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],

89 + [MODEL_NAME, "zephyr-lora"],

90 )

91 async def test_single_completion(client: openai.AsyncOpenAI, model_name: str):

92 completion = await client.completions.create(model=model_name,

@@ -115,20 +114,6 @@ async def test_single_completion(client: openai.AsyncOpenAI, model_name: str):

114 assert completion.choices[0].prompt_logprobs is None

115

116

118 -@pytest.mark.asyncio

119 -async def test_added_lora_tokens(client: openai.AsyncOpenAI):

120 - # test using token IDs

121 - completion = await client.completions.create(

122 - model="zephyr-lora2",

123 - prompt=[0, 0, 32000, 32001, 32002],

124 - echo=True,

125 - max_tokens=5,

126 - temperature=0.0,

127 - )

128 - # Added tokens should appear in tokenized prompt

129 - assert completion.choices[0].text.startswith("<unk><unk>vllm1vllm2vllm3")

130 -

131 -

117 @pytest.mark.asyncio

118 async def test_added_lora_tokens_base_model(client: openai.AsyncOpenAI):

119 # test using token IDs

@@ -147,7 +132,7 @@ async def test_added_lora_tokens_base_model(client: openai.AsyncOpenAI):

132 @pytest.mark.parametrize(

133 # first test base model, then test loras

134 "model_name",

150 - [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],

135 + [MODEL_NAME, "zephyr-lora"],

136 )

137 async def test_no_logprobs(client: openai.AsyncOpenAI, model_name: str):

138 # test using token IDs

@@ -713,7 +698,7 @@ async def test_guided_grammar(client: openai.AsyncOpenAI,

698 @pytest.mark.parametrize(

699 # first test base model, then test loras

700 "model_name",

716 - [MODEL_NAME, "zephyr-lora", "zephyr-lora2"],

701 + [MODEL_NAME, "zephyr-lora"],

702 )

703 @pytest.mark.parametrize("logprobs_arg", [1, 0])

704 async def test_echo_logprob_completion(client: openai.AsyncOpenAI,

tests/entrypoints/openai/test_completion_with_prompt_embeds.py

+1 -4

@@ -21,10 +21,7 @@ CONFIG = AutoConfig.from_pretrained(MODEL_NAME)

23 @pytest.fixture(scope="module")

24 -def default_server_args(

25 - zephyr_lora_files,

26 - zephyr_lora_added_tokens_files,

27 -) -> list[str]:

24 +def default_server_args() -> list[str]:

25 return [

26 # use half precision for speed and memory savings in CI environment

27 "--dtype",

tests/entrypoints/openai/test_lora_adapters.py

+1 -9

@@ -67,12 +67,6 @@ def server_with_lora_modules_json(request, monkeypatch_module,

67 "base_model_name": MODEL_NAME

68 }

70 - lora_module_2 = {

71 - "name": "zephyr-lora2",

72 - "path": zephyr_lora_files,

73 - "base_model_name": MODEL_NAME

74 - }

75 -

70 args = [

71 # use half precision for speed and memory savings in CI environment

72 "--dtype",

@@ -84,7 +78,6 @@ def server_with_lora_modules_json(request, monkeypatch_module,

78 "--enable-lora",

79 "--lora-modules",

80 json.dumps(lora_module_1),

87 - json.dumps(lora_module_2),

81 "--max-lora-rank",

82 "64",

83 "--max-cpu-loras",

@@ -121,7 +114,6 @@ async def test_static_lora_lineage(client: openai.AsyncOpenAI,

114 for lora_model in lora_models)

115 assert all(lora_model.parent == MODEL_NAME for lora_model in lora_models)

116 assert lora_models[0].id == "zephyr-lora"

124 - assert lora_models[1].id == "zephyr-lora2"

117

118

119 @pytest.mark.asyncio

@@ -209,7 +201,7 @@ async def test_dynamic_lora_badrequests(client: openai.AsyncOpenAI, tmp_path,

201 @pytest.mark.asyncio

202 async def test_multiple_lora_adapters(client: openai.AsyncOpenAI, tmp_path,

203 zephyr_lora_files):

212 - """Validate that many loras can be dynamically registered and inferenced

204 + """Validate that many loras can be dynamically registered and inferenced

205 with concurrently"""

206

207 # This test file configures the server with --max-cpu-loras=2 and this test

tests/entrypoints/openai/test_models.py

-2

@@ -26,7 +26,6 @@ def server(zephyr_lora_files):

26 "--enable-lora",

27 "--lora-modules",

28 f"zephyr-lora={zephyr_lora_files}",

29 - f"zephyr-lora2={zephyr_lora_files}",

29 "--max-lora-rank",

30 "64",

31 "--max-cpu-loras",

@@ -56,4 +55,3 @@ async def test_check_models(client: openai.AsyncOpenAI, zephyr_lora_files):

55 assert all(lora_model.root == zephyr_lora_files

56 for lora_model in lora_models)

57 assert lora_models[0].id == "zephyr-lora"

59 - assert lora_models[1].id == "zephyr-lora2"

tests/entrypoints/openai/test_tokenization.py

+10 -18

@@ -14,7 +14,7 @@ MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"

16 @pytest.fixture(scope="module")

17 -def server(zephyr_lora_added_tokens_files: str): # noqa: F811

17 +def server():

18 args = [

19 # use half precision for speed and memory savings in CI environment

20 "--dtype",

@@ -24,12 +24,6 @@ def server(zephyr_lora_added_tokens_files: str): # noqa: F811

24 "--enforce-eager",

25 "--max-num-seqs",

26 "128",

27 - # lora config

28 - "--enable-lora",

29 - "--lora-modules",

30 - f"zephyr-lora2={zephyr_lora_added_tokens_files}",

31 - "--max-lora-rank",

32 - "64",

27 "--enable-tokenizer-info-endpoint",

28 ]

@@ -38,10 +32,8 @@ def server(zephyr_lora_added_tokens_files: str): # noqa: F811

34 @pytest.fixture(scope="module")

41 -def tokenizer_name(model_name: str,

42 - zephyr_lora_added_tokens_files: str): # noqa: F811

43 - return zephyr_lora_added_tokens_files if (

44 - model_name == "zephyr-lora2") else model_name

35 +def tokenizer_name(model_name: str):

36 + return model_name

39 @pytest_asyncio.fixture

@@ -53,7 +45,7 @@ async def client(server):

45 @pytest.mark.asyncio

46 @pytest.mark.parametrize(

47 "model_name,tokenizer_name",

56 - [(MODEL_NAME, MODEL_NAME), ("zephyr-lora2", "zephyr-lora2")],

48 + [(MODEL_NAME, MODEL_NAME)],

49 indirect=["tokenizer_name"],

50 )

51 async def test_tokenize_completions(

@@ -86,7 +78,7 @@ async def test_tokenize_completions(

78 @pytest.mark.asyncio

79 @pytest.mark.parametrize(

80 "model_name,tokenizer_name",

89 - [(MODEL_NAME, MODEL_NAME), ("zephyr-lora2", "zephyr-lora2")],

81 + [(MODEL_NAME, MODEL_NAME)],

82 indirect=["tokenizer_name"],

83 )

84 async def test_tokenize_chat(

@@ -148,7 +140,7 @@ async def test_tokenize_chat(

140 @pytest.mark.asyncio

141 @pytest.mark.parametrize(

142 "model_name,tokenizer_name",

151 - [(MODEL_NAME, MODEL_NAME), ("zephyr-lora2", "zephyr-lora2")],

143 + [(MODEL_NAME, MODEL_NAME)],

144 indirect=["tokenizer_name"],

145 )

146 async def test_tokenize_chat_with_tools(

@@ -225,7 +217,7 @@ async def test_tokenize_chat_with_tools(

217 @pytest.mark.asyncio

218 @pytest.mark.parametrize(

219 "model_name, tokenizer_name",

228 - [(MODEL_NAME, MODEL_NAME), ("zephyr-lora2", "zephyr-lora2")],

220 + [(MODEL_NAME, MODEL_NAME)],

221 indirect=["tokenizer_name"],

222 )

223 async def test_tokenize_with_return_token_strs(

@@ -260,7 +252,7 @@ async def test_tokenize_with_return_token_strs(

252 @pytest.mark.asyncio

253 @pytest.mark.parametrize(

254 "model_name,tokenizer_name",

263 - [(MODEL_NAME, MODEL_NAME), ("zephyr-lora2", "zephyr-lora2")],

255 + [(MODEL_NAME, MODEL_NAME)],

256 indirect=["tokenizer_name"],

257 )

258 async def test_detokenize(

@@ -287,7 +279,7 @@ async def test_detokenize(

279 @pytest.mark.asyncio

280 @pytest.mark.parametrize(

281 "model_name,tokenizer_name",

290 - [(MODEL_NAME, MODEL_NAME), ("zephyr-lora2", "zephyr-lora2")],

282 + [(MODEL_NAME, MODEL_NAME)],

283 indirect=["tokenizer_name"],

284 )

285 async def test_tokenizer_info_basic(

@@ -384,4 +376,4 @@ async def test_tokenizer_info_chat_template(server: RemoteOpenAIServer):

376 if chat_template:

377 assert isinstance(chat_template,

378 str), ("Chat template should be a string")

387 - assert chat_template.strip(), "Chat template should not be empty"

\ No newline at end of file

379 + assert chat_template.strip(), "Chat template should not be empty"

tests/entrypoints/openai/tool_parsers/test_hermes_tool_parser.py

@@ -18,6 +18,8 @@ SERVER_ARGS = [

18 "--enable-lora",

19 "--lora-modules",

20 f"{LORA_MODEL}={LORA_MODEL}",

21 + "--tokenizer",

22 + f"{LORA_MODEL}",

23 ]

25 TOOLS = [{

tests/entrypoints/test_chat_utils.py

+10 -46

@@ -23,7 +23,7 @@ from vllm.entrypoints.chat_utils import (_try_extract_ast, load_chat_template,

23 from vllm.multimodal import MultiModalDataDict, MultiModalUUIDDict

24 from vllm.multimodal.utils import (encode_audio_base64, encode_image_base64,

25 encode_video_base64)

26 -from vllm.transformers_utils.tokenizer_group import TokenizerGroup

26 +from vllm.transformers_utils.tokenizer import get_tokenizer

27 from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer

29 from ..models.registry import HF_EXAMPLE_MODELS

@@ -69,12 +69,7 @@ def phi3v_model_config_mm_interleaved():

70 @pytest.fixture(scope="module")

71 def phi3v_tokenizer():

72 - return TokenizerGroup(

73 - tokenizer_id=PHI3V_MODEL_ID,

74 - enable_lora=False,

75 - max_num_seqs=5,

76 - max_input_length=None,

77 - )

72 + return get_tokenizer(PHI3V_MODEL_ID)

75 @pytest.fixture(scope="function")

@@ -91,12 +86,7 @@ def qwen2_audio_model_config():

87 @pytest.fixture(scope="module")

88 def qwen2_audio_tokenizer():

94 - return TokenizerGroup(

95 - tokenizer_id=QWEN2AUDIO_MODEL_ID,

96 - enable_lora=False,

97 - max_num_seqs=5,

98 - max_input_length=None,

99 - )

89 + return get_tokenizer(QWEN2AUDIO_MODEL_ID)

92 @pytest.fixture(scope="function")

@@ -115,12 +105,7 @@ def qwen25omni_model_config_mm_interleaved():

105

106 @pytest.fixture(scope="module")

107 def qwen25omni_tokenizer():

118 - return TokenizerGroup(

119 - tokenizer_id=QWEN25OMNI_MODEL_ID,

120 - enable_lora=False,

121 - max_num_seqs=5,

122 - max_input_length=None,

123 - )

108 + return get_tokenizer(QWEN25OMNI_MODEL_ID)

109

110

111 @pytest.fixture(scope="function")

@@ -136,12 +121,7 @@ def mistral_model_config():

121

122 @pytest.fixture(scope="module")

123 def mistral_tokenizer():

139 - return TokenizerGroup(

140 - tokenizer_id=MISTRAL_MODEL_ID,

141 - enable_lora=False,

142 - max_num_seqs=5,

143 - max_input_length=None,

144 - )

124 + return get_tokenizer(MISTRAL_MODEL_ID)

125

126

127 @pytest.fixture(scope="module")

@@ -2250,15 +2230,11 @@ def test_resolve_hf_chat_template(sample_json_schema, model, use_tools):

2230 enforce_eager=model_info.enforce_eager,

2231 dtype=model_info.dtype)

2232

2253 - # Build the tokenizer group and grab the underlying tokenizer

2254 - tokenizer_group = TokenizerGroup(

2233 + # Build the tokenizer

2234 + tokenizer = get_tokenizer(

2235 model,

2256 - enable_lora=False,

2257 - max_num_seqs=5,

2258 - max_input_length=None,

2236 trust_remote_code=model_config.trust_remote_code,

2237 )

2261 - tokenizer = tokenizer_group.tokenizer

2238

2239 tools = ([{

2240 "type": "function",

@@ -2307,14 +2283,10 @@ def test_resolve_content_format_hf_defined(model, expected_format):

2283 enforce_eager=model_info.enforce_eager,

2284 dtype=model_info.dtype)

2285

2310 - tokenizer_group = TokenizerGroup(

2286 + tokenizer = get_tokenizer(

2287 model,

2312 - enable_lora=False,

2313 - max_num_seqs=5,

2314 - max_input_length=None,

2288 trust_remote_code=model_config.trust_remote_code,

2289 )

2317 - tokenizer = tokenizer_group.tokenizer

2290

2291 # Test detecting the tokenizer's chat_template

2292 chat_template = resolve_hf_chat_template(

@@ -2368,14 +2340,10 @@ def test_resolve_content_format_fallbacks(model, expected_format):

2340 enforce_eager=model_info.enforce_eager,

2341 dtype=model_info.dtype)

2342

2371 - tokenizer_group = TokenizerGroup(

2343 + tokenizer = get_tokenizer(

2344 model_config.tokenizer,

2373 - enable_lora=False,

2374 - max_num_seqs=5,

2375 - max_input_length=None,

2345 trust_remote_code=model_config.trust_remote_code,

2346 )

2378 - tokenizer = tokenizer_group.tokenizer

2347

2348 # Test detecting the tokenizer's chat_template

2349 chat_template = resolve_hf_chat_template(

@@ -2432,14 +2400,10 @@ def test_resolve_content_format_examples(template_path, expected_format):

2400 trust_remote_code=True,

2401 )

2402

2435 - tokenizer_group = TokenizerGroup(

2403 + dummy_tokenizer = get_tokenizer(

2404 PHI3V_MODEL_ID, # Dummy

2437 - enable_lora=False,

2438 - max_num_seqs=5,

2439 - max_input_length=None,

2405 trust_remote_code=model_config.trust_remote_code,

2406 )

2442 - dummy_tokenizer = tokenizer_group.tokenizer

2407 dummy_tokenizer.chat_template = None

2408

2409 chat_template = load_chat_template(EXAMPLES_DIR / template_path)

tests/lora/test_llama_tp.py

+4 -24

@@ -13,14 +13,6 @@ from ..utils import VLLM_PATH, create_new_process_for_each_test, multi_gpu_test

14 MODEL_PATH = "meta-llama/Llama-2-7b-hf"

16 -EXPECTED_NO_LORA_OUTPUT = [

17 - "\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_75 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_76 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_77 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_78 (icao VARCHAR, airport VARCHAR)\n\n question: Name the ICAO for lilongwe international airport [/user]", # noqa: E501

18 - " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_11 (nationality VARCHAR, elector VARCHAR)\n\n question: When Anchero Pantaleone was the elector what is under nationality? ", # noqa: E501

19 - "\n\n answer: 1\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_96 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_97 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one mora for a high tone mora with a gloss of /˧kot/ [kòt]? [/user] [assistant]\n\n answer: 2\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_98 (one_mora VARCHAR, gloss VARCHAR, accented_mora VARCHAR)\n\n question: What is the one m", # noqa: E501

20 - " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE candidate (people_id VARCHAR, unsure_rate INTEGER); CREATE TABLE people (sex VARCHAR, people_id VARCHAR)\n\n question: which gender got the highest average uncertain ratio. ", # noqa: E501

21 - " Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_name_60 (pick INTEGER, former_wnba_team VARCHAR)\n\n question: What pick was a player that previously played for the Minnesota Lynx? ", # noqa: E501

22 - "\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE table_28138035_4 (womens_doubles VARCHAR, mens_singles VARCHAR)\n\n question: Name the women's doubles for werner schlager [/user] [assistant]\n\n [user] Write a SQL query to answer the question based on the table schema.\n\n context: CREATE TABLE", # noqa: E501

23 -]

16 EXPECTED_LORA_OUTPUT = [

17 " SELECT icao FROM table_name_74 WHERE airport = 'lilongwe international airport' ", # noqa: E501

18 " SELECT nationality FROM table_name_11 WHERE elector = 'anchero pantaleone' ", # noqa: E501

@@ -79,23 +71,12 @@ def generate_and_test(llm,

71 sql_lora_files,

72 tensorizer_config_dict: Union[dict, None] = None):

73 print("lora adapter created")

82 - assert do_sample(llm,

83 - sql_lora_files,

84 - tensorizer_config_dict=tensorizer_config_dict,

85 - lora_id=0) == EXPECTED_NO_LORA_OUTPUT

86 -

74 print("lora 1")

75 assert do_sample(llm,

76 sql_lora_files,

77 tensorizer_config_dict=tensorizer_config_dict,

78 lora_id=1) == EXPECTED_LORA_OUTPUT

93 - print("no lora")

94 - assert do_sample(llm,

95 - sql_lora_files,

96 - tensorizer_config_dict=tensorizer_config_dict,

97 - lora_id=0) == EXPECTED_NO_LORA_OUTPUT

98 -

80 print("lora 2")

81 assert do_sample(llm,

82 sql_lora_files,

@@ -110,6 +91,7 @@ def test_llama_lora(sql_lora_files):

92 llm = vllm.LLM(

93 MODEL_PATH,

94 + tokenizer=sql_lora_files,

95 enable_lora=True,

96 # also test odd max_num_seqs

97 max_num_seqs=13,

@@ -123,6 +105,7 @@ def test_llama_lora_tp4(sql_lora_files):

105

106 llm = vllm.LLM(

107 MODEL_PATH,

108 + tokenizer=sql_lora_files,

109 enable_lora=True,

110 max_num_seqs=16,

111 max_loras=4,

@@ -137,6 +120,7 @@ def test_llama_lora_tp4_fully_sharded_loras(sql_lora_files):

120

121 llm = vllm.LLM(

122 MODEL_PATH,

123 + tokenizer=sql_lora_files,

124 enable_lora=True,

125 max_num_seqs=16,

126 max_loras=4,

@@ -184,6 +168,7 @@ def test_tp2_serialize_and_deserialize_lora(tmp_path, sql_lora_files,

168 tensorizer_config = TensorizerConfig(tensorizer_uri=str(model_uri))

169

170 loaded_llm = LLM(model=model_ref,

171 + tokenizer=sql_lora_files,

172 load_format="tensorizer",

173 enable_lora=True,

174 enforce_eager=True,

@@ -195,11 +180,6 @@ def test_tp2_serialize_and_deserialize_lora(tmp_path, sql_lora_files,

180 tc_as_dict = tensorizer_config.to_serializable()

181

182 print("lora adapter created")

198 - assert do_sample(loaded_llm,

199 - sql_lora_files,

200 - tensorizer_config_dict=tc_as_dict,

201 - lora_id=0) == EXPECTED_NO_LORA_OUTPUT

202 -

183 print("lora 1")

184 assert do_sample(loaded_llm,

185 sql_lora_files,

tests/lora/test_lora_allowed_token_ids.py deleted

-135

@@ -1,135 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -import pytest

5 -

6 -from vllm.config import CacheConfig, DeviceConfig, ModelConfig, VllmConfig

7 -from vllm.config.lora import LoRAConfig

8 -from vllm.lora.request import LoRARequest

9 -from vllm.sampling_params import SamplingParams

10 -from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs

11 -from vllm.v1.engine.processor import Processor

12 -

13 -

14 -def test_allowed_token_ids_with_lora_vocab(llama_2_7b_base_huggingface_id,

15 - sql_lora_files):

16 - """

17 - Test that we properly resolve the range of allowed token ids for lora

18 - adapters that define additional tokens.

19 - """

20 -

21 - # Set up a base model compatible with the sql_lora_files adapter and

22 - # a known number of tokens in the base model.

23 - model_config = ModelConfig(

24 - model=llama_2_7b_base_huggingface_id,

25 - tokenizer=llama_2_7b_base_huggingface_id,

26 - tokenizer_mode="auto",

27 - )

28 -

29 - vllm_config = VllmConfig(

30 - model_config=model_config,

31 - cache_config=CacheConfig(),

32 - device_config=DeviceConfig(),

33 - lora_config=LoRAConfig(),

34 - )

35 -

36 - tokenizer = init_tokenizer_from_configs(

37 - model_config=vllm_config.model_config,

38 - scheduler_config=vllm_config.scheduler_config,

39 - lora_config=vllm_config.lora_config)

40 - processor = Processor(vllm_config, tokenizer)

41 -

42 - lora_request = LoRARequest("1", 1, str(sql_lora_files))

43 - request_id = "1"

44 - prompt = "a prompt"

45 -

46 - # tokens added in the lora adapter should not raise an error

47 - lora_token_ids = [32000, 32001, 32002, 32003]

48 - processor.process_inputs(

49 - request_id,

50 - prompt,

51 - params=SamplingParams(allowed_token_ids=lora_token_ids),

52 - lora_request=lora_request)

53 -

54 - # tokens in the base model should not raise an error

55 - base_token_ids = [1000, 1001, 1002, 1003]

56 - processor.process_inputs(

57 - request_id,

58 - prompt,

59 - params=SamplingParams(allowed_token_ids=base_token_ids),

60 - lora_request=lora_request)

61 -

62 - # tokens not in the lora adapter should raise an error

63 - invalid_token_ids = [35000, 35001, 35002, 35003]

64 - with pytest.raises(ValueError):

65 - processor.process_inputs(

66 - request_id,

67 - prompt,

68 - params=SamplingParams(allowed_token_ids=invalid_token_ids),

69 - lora_request=lora_request)

70 -

71 - # tokens in the lora adapter with no lora request should raise an error

72 - with pytest.raises(ValueError):

73 - processor.process_inputs(

74 - request_id,

75 - prompt,

76 - params=SamplingParams(allowed_token_ids=lora_token_ids),

77 - )

78 -

79 -

80 -def test_allowed_token_ids_with_lora_adapter_no_vocab(

81 - qwen25vl_base_huggingface_id, qwen25vl_lora_files):

82 - """

83 - Test that we properly resolve the range of allowed token ids for lora

84 - adapters that do not define additional tokens.

85 - """

86 -

87 - # Set up a base model compatible with the qwen25vl_lora_files adapter and

88 - # a known number of tokens in the base model.

89 - model_config = ModelConfig(

90 - model=qwen25vl_base_huggingface_id,

91 - tokenizer=qwen25vl_base_huggingface_id,

92 - tokenizer_mode="auto",

93 - )

94 -

95 - vllm_config = VllmConfig(

96 - model_config=model_config,

97 - cache_config=CacheConfig(),

98 - device_config=DeviceConfig(),

99 - lora_config=LoRAConfig(),

100 - )

101 -

102 - tokenizer = init_tokenizer_from_configs(

103 - model_config=vllm_config.model_config,

104 - scheduler_config=vllm_config.scheduler_config,

105 - lora_config=vllm_config.lora_config)

106 - processor = Processor(vllm_config, tokenizer)

107 -

108 - lora_request = LoRARequest("1", 1, str(qwen25vl_lora_files))

109 - request_id = "1"

110 - prompt = "a prompt"

111 -

112 - # tokens in the base model should not raise an error

113 - base_token_ids = [1000, 1001, 1002, 1003]

114 - processor.process_inputs(

115 - request_id,

116 - prompt,

117 - params=SamplingParams(allowed_token_ids=base_token_ids),

118 - lora_request=lora_request)

119 -

120 - # tokens in the base model with no lora request should not raise an error

121 - base_token_ids = [1000, 1001, 1002, 1003]

122 - processor.process_inputs(

123 - request_id,

124 - prompt,

125 - params=SamplingParams(allowed_token_ids=base_token_ids),

126 - )

127 -

128 - # tokens not in the base model should raise an error

129 - invalid_token_ids = [200000, 200001, 200002, 200003]

130 - with pytest.raises(ValueError):

131 - processor.process_inputs(

132 - request_id,

133 - prompt,

134 - params=SamplingParams(allowed_token_ids=invalid_token_ids),

135 - lora_request=lora_request)

tests/lora/test_quant_model.py

+2 -27

@@ -82,31 +82,20 @@ def test_quant_model_lora(tinyllama_lora_files, model):

82 gpu_memory_utilization=0.2, #avoid OOM

83 quantization=model.quantization,

84 trust_remote_code=True,

85 - enable_chunked_prefill=True)

85 + enable_chunked_prefill=True,

86 + tokenizer=tinyllama_lora_files)

88 if model.quantization is None:

88 - expected_no_lora_output = [

89 - "Here are some examples of orange-brown colors",

90 - "I'm sorry, I don't have"

91 - ]

89 expected_lora_output = [

90 "#ff8050",

91 "#ff8080",

92 ]

93 elif model.quantization == "awq":

97 - expected_no_lora_output = [

98 - "I'm sorry, I don't understand",

99 - "I'm sorry, I don't understand",

100 - ]

94 expected_lora_output = [

95 "#f07700: A v",

96 "#f00000: A v",

97 ]

98 elif model.quantization == "gptq":

106 - expected_no_lora_output = [

107 - "I'm sorry, I don't have",

108 - "I'm sorry, I don't have",

109 - ]

99 expected_lora_output = [

100 "#f08800: This is",

101 "#f07788 \n#",

@@ -117,7 +106,6 @@ def test_quant_model_lora(tinyllama_lora_files, model):

106 # Assert that the outputs changed.

107 if (model.quantization == "gptq"

108 and expected_output is expected_lora_output):

120 - assert output != expected_no_lora_output

109 for i, o in enumerate(output):

110 assert o.startswith(

111 '#'), f"Expected example {i} to start with # but got {o}"

@@ -127,12 +115,6 @@ def test_quant_model_lora(tinyllama_lora_files, model):

115 max_tokens = 10

116

117 print("lora adapter created")

130 - output = do_sample(llm,

131 - tinyllama_lora_files,

132 - lora_id=0,

133 - max_tokens=max_tokens)

134 - expect_match(output, expected_no_lora_output)

135 -

118 print("lora 1")

119 output = do_sample(llm,

120 tinyllama_lora_files,

@@ -140,13 +122,6 @@ def test_quant_model_lora(tinyllama_lora_files, model):

122 max_tokens=max_tokens)

123 expect_match(output, expected_lora_output)

124

143 - print("no lora")

144 - output = do_sample(llm,

145 - tinyllama_lora_files,

146 - lora_id=0,

147 - max_tokens=max_tokens)

148 - expect_match(output, expected_no_lora_output)

149 -

125 print("lora 2")

126 output = do_sample(llm,

127 tinyllama_lora_files,

tests/lora/test_tokenizer_group.py deleted

-72

@@ -1,72 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -import pytest

5 -from transformers import AutoTokenizer, PreTrainedTokenizerBase

6 -

7 -from vllm.lora.request import LoRARequest

8 -from vllm.transformers_utils.tokenizer import get_lora_tokenizer

9 -from vllm.transformers_utils.tokenizer_group import TokenizerGroup

10 -

11 -

12 -@pytest.mark.asyncio

13 -@pytest.mark.parametrize("tokenizer_group_type", [None, "ray"])

14 -async def test_tokenizer_group_lora(sql_lora_files, tokenizer_group_type):

15 - reference_tokenizer = AutoTokenizer.from_pretrained(sql_lora_files)

16 - tokenizer_group = TokenizerGroup(

17 - tokenizer_id="gpt2",

18 - enable_lora=True,

19 - max_num_seqs=1,

20 - max_loras=1,

21 - max_input_length=None,

22 - )

23 - lora_request = LoRARequest("1", 1, sql_lora_files)

24 - assert reference_tokenizer.encode("prompt") == tokenizer_group.encode(

25 - prompt="prompt", lora_request=lora_request)

26 - assert reference_tokenizer.encode(

27 - "prompt") == await tokenizer_group.encode_async(

28 - prompt="prompt", lora_request=lora_request)

29 - assert isinstance(tokenizer_group.get_lora_tokenizer(None),

30 - PreTrainedTokenizerBase)

31 - assert tokenizer_group.get_lora_tokenizer(

32 - None) == await tokenizer_group.get_lora_tokenizer_async(None)

33 -

34 - assert isinstance(tokenizer_group.get_lora_tokenizer(lora_request),

35 - PreTrainedTokenizerBase)

36 - assert tokenizer_group.get_lora_tokenizer(

37 - lora_request) != tokenizer_group.get_lora_tokenizer(None)

38 - assert tokenizer_group.get_lora_tokenizer(

39 - lora_request) == await tokenizer_group.get_lora_tokenizer_async(

40 - lora_request)

41 -

42 -

43 -def test_get_lora_tokenizer(sql_lora_files, tmp_path):

44 - lora_request = None

45 - tokenizer = get_lora_tokenizer(lora_request)

46 - assert not tokenizer

47 -

48 - lora_request = LoRARequest("1", 1, sql_lora_files)

49 - tokenizer = get_lora_tokenizer(lora_request)

50 - assert tokenizer.get_added_vocab()

51 -

52 - lora_request = LoRARequest("1", 1, str(tmp_path))

53 - tokenizer = get_lora_tokenizer(lora_request)

54 - assert not tokenizer

55 -

56 -

57 -@pytest.mark.parametrize("enable_lora", [True, False])

58 -@pytest.mark.parametrize("max_num_seqs", [1, 2])

59 -@pytest.mark.parametrize("max_loras", [1, 2])

60 -def test_lora_tokenizers(enable_lora, max_num_seqs, max_loras):

61 - tokenizer_group = TokenizerGroup(

62 - tokenizer_id="gpt2",

63 - enable_lora=enable_lora,

64 - max_num_seqs=max_num_seqs,

65 - max_loras=max_loras,

66 - max_input_length=None,

67 - )

68 - if enable_lora:

69 - assert tokenizer_group.lora_tokenizers.capacity == max(

70 - max_num_seqs, max_loras)

71 - else:

72 - assert tokenizer_group.lora_tokenizers.capacity == 0

tests/test_cache_block_hashing.py

+3 -8

@@ -11,7 +11,7 @@ import pytest

11 from vllm.inputs import token_inputs

12 from vllm.lora.request import LoRARequest

13 from vllm.sequence import Sequence

14 -from vllm.transformers_utils.tokenizer_group import TokenizerGroup

14 +from vllm.transformers_utils.tokenizer import get_tokenizer

16 # Make two prefixes with different first blocks.

17 prefix_start = [("You are an expert"), ("You are a")]

@@ -47,12 +47,7 @@ def flatten_2d(li):

47 def test_auto_prefix_caching(model: str, block_size: int, max_num_seqs: int,

48 concurrent_lora_int_ids: list[Optional[int]]):

50 - tokenizer = TokenizerGroup(

51 - tokenizer_id="facebook/opt-125m",

52 - enable_lora=False,

53 - max_num_seqs=max_num_seqs,

54 - max_input_length=None,

55 - )

50 + tokenizer = get_tokenizer("facebook/opt-125m")

52 hashes: list[list[list[int]]] = []

@@ -76,7 +71,7 @@ def test_auto_prefix_caching(model: str, block_size: int, max_num_seqs: int,

71 inputs=token_inputs(prompt_token_ids,

72 prompt=prompt),

73 block_size=block_size,

79 - eos_token_id=tokenizer.tokenizer.eos_token_id,

74 + eos_token_id=tokenizer.eos_token_id,

75 lora_request=lora_request)

77 num_blocks = len(prompt_token_ids) // block_size

tests/tokenization/test_detokenize.py

+6 -10

@@ -11,7 +11,7 @@ from transformers import (AutoTokenizer, PreTrainedTokenizer,

11 from vllm.inputs import token_inputs

12 from vllm.sequence import Logprob, SamplingParams, Sequence, SequenceGroup

13 from vllm.transformers_utils.detokenizer import Detokenizer

14 -from vllm.transformers_utils.tokenizer_group import TokenizerGroup

14 +from vllm.transformers_utils.tokenizer import get_tokenizer

15 from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer

16 from vllm.v1.engine import EngineCoreRequest

17 from vllm.v1.engine.detokenizer import (FastIncrementalDetokenizer,

@@ -221,17 +221,14 @@ def test_oov_decode(tokenizer, fast):

221

222 @pytest.fixture

223 def detokenizer(tokenizer_name: str) -> Detokenizer:

224 - tokenizer_group = TokenizerGroup(

225 - tokenizer_id=tokenizer_name,

226 - enable_lora=False,

227 - max_num_seqs=100,

228 - max_input_length=None,

224 + tokenizer = get_tokenizer(

225 + tokenizer_name,

226 tokenizer_mode="mistral" if "mistral" in tokenizer_name else "auto",

227 trust_remote_code=False,

228 revision=None,

229 )

230

234 - return Detokenizer(tokenizer_group)

231 + return Detokenizer(tokenizer)

232

233

234 @pytest.fixture(name="complete_sequence_token_ids")

@@ -312,8 +309,7 @@ def test_decode_prompt_logprobs(complete_sequence: str,

309 # don't support that.

310 if complete_sequence not in SPECIAL_TOKS_TRUTH:

311 skip_special_tokens = True

315 - elif not isinstance(detokenizer.tokenizer_group.get_lora_tokenizer(None),

316 - MistralTokenizer):

312 + elif not isinstance(detokenizer.tokenizer, MistralTokenizer):

313 skip_special_tokens = False

314 else:

315 pytest.skip("MistralTokenizers don't support "

@@ -339,7 +335,7 @@ def test_decode_prompt_logprobs(complete_sequence: str,

335

336 # decoded_prompt_logprobs doesn't contain the first token.

337 token_ids = complete_sequence_token_ids

342 - tokenizer = detokenizer.get_tokenizer_for_seq(seq)

338 + tokenizer = detokenizer.tokenizer

339 text_full = tokenizer.decode(token_ids,

340 skip_special_tokens=skip_special_tokens)

341 text_first = tokenizer.decode(token_ids[0],

tests/tokenization/test_tokenizer_group.py deleted

-27

@@ -1,27 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -import pytest

5 -from transformers import AutoTokenizer, PreTrainedTokenizerBase

6 -

7 -from vllm.transformers_utils.tokenizer_group import TokenizerGroup

8 -

9 -

10 -@pytest.mark.asyncio

11 -async def test_tokenizer_group():

12 - reference_tokenizer = AutoTokenizer.from_pretrained("gpt2")

13 - tokenizer_group = TokenizerGroup(

14 - tokenizer_id="gpt2",

15 - enable_lora=False,

16 - max_num_seqs=1,

17 - max_input_length=None,

18 - )

19 - assert reference_tokenizer.encode("prompt") == tokenizer_group.encode(

20 - prompt="prompt", lora_request=None)

21 - assert reference_tokenizer.encode(

22 - "prompt") == await tokenizer_group.encode_async(prompt="prompt",

23 - lora_request=None)

24 - assert isinstance(tokenizer_group.get_lora_tokenizer(None),

25 - PreTrainedTokenizerBase)

26 - assert tokenizer_group.get_lora_tokenizer(

27 - None) == await tokenizer_group.get_lora_tokenizer_async(None)

tests/tokenization/test_tokenizer_registry.py

@@ -57,6 +57,10 @@ class TestTokenizer(TokenizerBase):

57 def max_token_id(self) -> int:

58 raise NotImplementedError()

60 + @property

61 + def truncation_side(self) -> str:

62 + raise NotImplementedError()

63 +

64 def __call__(

65 self,

66 text: Union[str, list[str], list[int]],

tests/v1/engine/conftest.py

+2 -6

@@ -12,7 +12,6 @@ from tests.v1.engine.utils import (NUM_PROMPT_LOGPROBS_UNDER_TEST,

12 generate_dummy_prompt_logprobs_tensors,

13 generate_dummy_sample_logprobs)

14 from vllm.engine.arg_utils import EngineArgs

15 -from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs

16 from ...distributed.conftest import publisher_config, random_port # noqa: F401

@@ -24,7 +23,7 @@ EngineCorePromptLogprobsType = tuple[torch.Tensor, torch.Tensor]

24 def _build_test_vectors_no_logprobs() -> DummyOutputProcessorTestVectors:

25 """Generate output processor dummy test vectors, without logprobs

27 -

26 +

27 Returns:

28 DummyOutputProcessorTestVectors instance with no logprobs

29 """

@@ -48,9 +47,6 @@ def _build_test_vectors_no_logprobs() -> DummyOutputProcessorTestVectors:

47 ]

48 return DummyOutputProcessorTestVectors(

49 tokenizer=tokenizer,

51 - tokenizer_group=init_tokenizer_from_configs(

52 - vllm_config.model_config, vllm_config.scheduler_config,

53 - vllm_config.lora_config),

50 vllm_config=vllm_config,

51 full_tokens=[tokenizer(text).input_ids for text in FULL_STRINGS],

52 prompt_tokens=prompt_tokens,

@@ -68,7 +64,7 @@ def _build_test_vectors_no_logprobs() -> DummyOutputProcessorTestVectors:

64 @pytest.fixture

65 def dummy_test_vectors() -> DummyOutputProcessorTestVectors:

66 """Generate output processor dummy test vectors, with logprobs

71 -

67 +

68 Returns:

69 DummyOutputProcessorTestVectors instance with logprobs

70 """

tests/v1/engine/test_output_processor.py

+5 -5

@@ -43,7 +43,7 @@ def _ref_convert_id_to_token(

43 [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY])

44 def test_incremental_detokenization(request_output_kind: RequestOutputKind,

45 dummy_test_vectors):

46 - output_processor = OutputProcessor(dummy_test_vectors.tokenizer_group,

46 + output_processor = OutputProcessor(dummy_test_vectors.tokenizer,

47 log_stats=False)

48 engine_core = MockEngineCore(

49 tokens_list=dummy_test_vectors.generation_tokens)

@@ -382,7 +382,7 @@ def test_logprobs_processor(request_output_kind: RequestOutputKind,

382 num_sample_logprobs: Optional[int],

383 num_prompt_logprobs: Optional[int],

384 dummy_test_vectors):

385 - output_processor = OutputProcessor(dummy_test_vectors.tokenizer_group,

385 + output_processor = OutputProcessor(dummy_test_vectors.tokenizer,

386 log_stats=False)

387 engine_core = MockEngineCore(

388 tokens_list=dummy_test_vectors.generation_tokens,

@@ -535,7 +535,7 @@ def test_stop_token(include_stop_str_in_output: bool,

535 ) # '<|end_of_text|>'

536 stop_token_ids = [128009] if not is_eos_test else None # '<|eot_id|>'

537

538 - output_processor = OutputProcessor(dummy_test_vectors.tokenizer_group,

538 + output_processor = OutputProcessor(dummy_test_vectors.tokenizer,

539 log_stats=False)

540 # Dummy engine core outputs, with control tokens suffixed to test stops

541 suffix_token = ([eos_token_id] if is_eos_test else stop_token_ids)

@@ -642,7 +642,7 @@ def test_stop_token(include_stop_str_in_output: bool,

642 [None, NUM_SAMPLE_LOGPROBS_UNDER_TEST])

643 def test_stop_string(include_stop_str_in_output: bool,

644 num_sample_logprobs: Optional[int], dummy_test_vectors):

645 - output_processor = OutputProcessor(dummy_test_vectors.tokenizer_group,

645 + output_processor = OutputProcessor(dummy_test_vectors.tokenizer,

646 log_stats=False)

647 engine_core = MockEngineCore(

648 tokens_list=dummy_test_vectors.generation_tokens,

@@ -763,7 +763,7 @@ def test_stop_string(include_stop_str_in_output: bool,

763

764

765 def test_iteration_stats(dummy_test_vectors):

766 - output_processor = OutputProcessor(dummy_test_vectors.tokenizer_group,

766 + output_processor = OutputProcessor(dummy_test_vectors.tokenizer,

767 log_stats=True)

768 engine_core = MockEngineCore(dummy_test_vectors.generation_tokens)

769 engine_core_timestamp = time.monotonic()

tests/v1/engine/utils.py

+2 -4

@@ -9,7 +9,6 @@ import torch

9 from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast

11 from vllm.engine.arg_utils import EngineArgs

12 -from vllm.transformers_utils.tokenizer_group import TokenizerGroup

12 from vllm.v1.engine import EngineCoreOutput, FinishReason

13 from vllm.v1.outputs import LogprobsLists, LogprobsTensors

@@ -39,7 +38,7 @@ def _create_random_top_logprob_test_vector(

38 upper: float,

39 ) -> torch.Tensor:

40 """Create a random vector of top logprob float values.

42 -

41 +

42 Use to create fake sample logprobs for testing.

44 Note that a real production scenario would require

@@ -63,7 +62,7 @@ def _create_random_top_logprob_test_matrix(

62 upper: float,

63 ) -> torch.Tensor:

64 """Create a random matrix of top logprob float values.

66 -

65 +

66 Use to create fake prompt logprobs for testing.

68 Note that a real production scenario would require

@@ -296,7 +295,6 @@ def generate_dummy_prompt_logprobs_tensors(

295 class DummyOutputProcessorTestVectors:

296 """Dummy test vectors for output processor tests"""

297 tokenizer: GeneralTokenizerType

299 - tokenizer_group: TokenizerGroup

298 vllm_config: EngineArgs

299 full_tokens: list[list[int]] # Prompt + generated tokens

300 prompt_tokens: list[list[int]]

tests/v1/entrypoints/llm/test_struct_output_generate.py

+1 -1

@@ -582,7 +582,7 @@ def test_structured_output_with_reasoning_matrices(

582 reasoning_parser=reasoning_parser,

583 speculative_config=speculative_config,

584 )

585 - tokenizer = llm.get_tokenizer(None)

585 + tokenizer = llm.get_tokenizer()

586 reasoner = ReasoningParserManager.get_reasoning_parser(reasoning_parser)(

587 tokenizer=tokenizer)

588

vllm/benchmarks/datasets.py

+80 -93

@@ -37,7 +37,7 @@ from vllm.lora.request import LoRARequest

37 from vllm.lora.utils import get_adapter_absolute_path

38 from vllm.multimodal import MultiModalDataDict

39 from vllm.multimodal.image import convert_image_mode

40 -from vllm.transformers_utils.tokenizer import AnyTokenizer, get_lora_tokenizer

40 +from vllm.transformers_utils.tokenizer import AnyTokenizer

41 from vllm.utils import PlaceholderModule

43 try:

@@ -100,8 +100,8 @@ class BenchmarkDataset(ABC):

100 ) -> None:

101 """

102 Initialize the BenchmarkDataset with an optional dataset path and random

103 - seed.

104 -

103 + seed.

104 +

105 Args:

106 dataset_path (Optional[str]): Path to the dataset. If None, it

107 indicates that a default or random dataset might be used.

@@ -133,10 +133,10 @@ class BenchmarkDataset(ABC):

133 elif isinstance(mm_content, dict):

134 content.append(mm_content)

135 else:

136 - raise TypeError(

136 + raise TypeError(

137 "Could not process multimodal content of type: " +

138 - f"{type(mm_content)}"

139 - )

138 + f"{type(mm_content)}"

139 + )

140 return [{"role": "user", "content": content}]

141

142 def load_data(self) -> None:

@@ -155,34 +155,26 @@ class BenchmarkDataset(ABC):

155

156 def get_random_lora_request(

157 self,

158 - tokenizer: PreTrainedTokenizerBase,

158 max_loras: Optional[int] = None,

159 lora_path: Optional[str] = None,

161 - ) -> tuple[Optional[LoRARequest], AnyTokenizer]:

160 + ) -> Optional[LoRARequest]:

161 """

163 - Optionally select a random LoRA request and return its associated

164 - tokenizer.

162 + Optionally select a random LoRA request.

163

164 This method is used when LoRA parameters are provided. It randomly

167 - selects a LoRA based on max_loras and retrieves a cached tokenizer for

168 - that LoRA if available. Otherwise, it returns the base tokenizer.

165 + selects a LoRA based on max_loras.

166

167 Args:

171 - tokenizer (PreTrainedTokenizerBase): The base tokenizer to use if no

172 - LoRA is selected.

168 max_loras (Optional[int]): The maximum number of LoRAs available.

169 If `None`, LoRA is not used.

170 lora_path (Optional[str]): Path to the LoRA parameters on disk.

171 If `None`, LoRA is not used.

172

173 Returns:

179 - A tuple with the following elements:

180 - - A new [LoRARequest][] (or `None` if not applicable).

181 - - The tokenizer associated with the LoRA request

182 - (or the base tokenizer).

174 + A new [LoRARequest][] (or `None` if not applicable).

175 """

176 if max_loras is None or lora_path is None:

185 - return None, tokenizer

177 + return None

178

179 # Generate a random LoRA ID in the range [1, max_loras].

180 lora_id = random.randint(1, max_loras)

@@ -191,11 +183,7 @@ class BenchmarkDataset(ABC):

183 lora_int_id=lora_id,

184 lora_path=lora_path_on_disk(lora_path),

185 )

194 - if lora_id not in lora_tokenizer_cache:

195 - lora_tokenizer_cache[lora_id] = get_lora_tokenizer(lora_request)

196 - # Return lora_request and the cached tokenizer if available; otherwise,

197 - # return the base tokenizer

198 - return lora_request, lora_tokenizer_cache[lora_id] or tokenizer

186 + return lora_request

187

188 @abstractmethod

189 def sample(self, tokenizer: PreTrainedTokenizerBase,

@@ -213,7 +201,7 @@ class BenchmarkDataset(ABC):

201 for processing the dataset's text.

202 num_requests (int): The number of sample requests to generate.

203 request_id_prefix (str) The prefix of request_id.

216 -

204 +

205

206 Returns:

207 list[SampleRequest]: A list of sample requests generated from the

@@ -527,7 +515,7 @@ class RandomDataset(BenchmarkDataset):

515 size=num_requests)

516 output_lens = self._rng.integers(output_low, output_high + 1,

517 size=num_requests)

530 - offsets = self._rng.integers(0, tokenizer.vocab_size,

518 + offsets = self._rng.integers(0, tokenizer.vocab_size,

519 size=num_requests)

520 return input_lens, output_lens, offsets

521

@@ -555,7 +543,7 @@ class RandomDataset(BenchmarkDataset):

543 the encoded sequence is truncated before being decoded again.

544 """

545 # Build the inner sequence by sampling sequentially from the vocab

558 - inner_seq = ((offset + index + np.arange(input_len))

546 + inner_seq = ((offset + index + np.arange(input_len))

547 % vocab_size).tolist()

548 token_sequence = prefix_token_ids + inner_seq

549

@@ -590,9 +578,9 @@ class RandomMultiModalDataset(RandomDataset):

578 `num_mm_items_range_ratio` in [0, 1]. r=0 keeps it fixed; r=1 allows 0.

579 The maximum is further clamped to the sum of per-modality limits.

580 2) Each item’s modality and shape is sampled from `bucket_config`, a dict

593 - mapping (height, width, num_frames) → probability. We treat

594 - `num_frames`=1 as image and and `num_frames` > 1 as video.

595 - Entries with zero probability are removed and the rest are renormalized

581 + mapping (height, width, num_frames) → probability. We treat

582 + `num_frames`=1 as image and and `num_frames` > 1 as video.

583 + Entries with zero probability are removed and the rest are renormalized

584 to sum to 1.

585 3) Per-modality hard caps are enforced via `limit_mm_per_prompt`.

586 When a modality reaches its cap, all of its buckets are excluded and the

@@ -600,8 +588,8 @@ class RandomMultiModalDataset(RandomDataset):

588

589 Example bucket configuration:

590 {(256, 256, 1): 0.5, (720, 1280, 1): 0.4, (720, 1280, 16): 0.1}

603 - - Two image buckets (`num_frames`=1) and one video bucket

604 - (`num_frames`=16).

591 + - Two image buckets (`num_frames`=1) and one video bucket

592 + (`num_frames`=16).

593 OBS.: Only image sampling is supported for now.

594 """

595

@@ -624,9 +612,9 @@ class RandomMultiModalDataset(RandomDataset):

612

613 def generate_synthetic_image(self, width: int, height: int) -> Image.Image:

614 """Generate synthetic PIL image with random RGB values.

627 -

628 - NOTE: iid pixel sampling results in worst-case compression

629 - (good for stressing I/O), but very unlike real photos.

615 +

616 + NOTE: iid pixel sampling results in worst-case compression

617 + (good for stressing I/O), but very unlike real photos.

618 We could consider a “low-freq” mode (e.g., noise blur)

619 to emulate network realism instead of max stress.

620 """

@@ -638,11 +626,11 @@ class RandomMultiModalDataset(RandomDataset):

626 )

627 return Image.fromarray(random_pixels)

628

641 - def generate_synthetic_video(self, width: int,

642 - height: int,

629 + def generate_synthetic_video(self, width: int,

630 + height: int,

631 num_frames: int) -> Any:

632 """Generate synthetic video with random values.

645 -

633 +

634 TODO: Finish this method.

635 """

636 raise NotImplementedError("Video sampling is WIP.")

@@ -656,7 +644,7 @@ class RandomMultiModalDataset(RandomDataset):

644 else:

645 raise ValueError(f"Invalid multimodal item configuration: {config}")

646

659 - def normalize_bucket_config(self, bucket_config: dict[tuple[int, int, int],

647 + def normalize_bucket_config(self, bucket_config: dict[tuple[int, int, int],

648 float]) -> dict[tuple[int, int, int], float]:

649 """

650 Remove zero probability entries

@@ -676,24 +664,24 @@ class RandomMultiModalDataset(RandomDataset):

664 return {k: v / total for k, v in bucket_config.items()}

665

666

679 - def generate_mm_item(self,

667 + def generate_mm_item(self,

668 mm_item_config: tuple[int, int, int],

669 ) -> Mapping[str, Any]:

670 """

683 - Create synthetic images and videos and

671 + Create synthetic images and videos and

672 apply process_image/process_video respectively.

673 This follows the OpenAI API chat completions

674 https://github.com/openai/openai-python

675 """

688 -

676 +

677 if self.map_config_to_modality(mm_item_config) == "image":

678 return process_image(self.generate_synthetic_image(

679 mm_item_config[1],

680 mm_item_config[0]))

681 elif self.map_config_to_modality(mm_item_config) == "video":

682 return process_video(self.generate_synthetic_video(

695 - mm_item_config[1],

696 - mm_item_config[0],

683 + mm_item_config[1],

684 + mm_item_config[0],

685 mm_item_config[2]))

686 else:

687 raise ValueError(f"Invalid multimodal item configuration: "

@@ -723,17 +711,17 @@ class RandomMultiModalDataset(RandomDataset):

711 f"limit_mm_per_prompt: "

712 f"{limit_mm_per_prompt.keys()}")

713

726 - # Remove zero probability entries

714 + # Remove zero probability entries

715 # and normalize bucket config to sum to 1

716 bucket_config = self.normalize_bucket_config(bucket_config)

717 logger.info(

718 "Normalized bucket config: %s", bucket_config,

719 )

720 # Only consider limit per prompt for modalities in bucket config

733 - allowed_modalities = {self.map_config_to_modality(cfg)

721 + allowed_modalities = {self.map_config_to_modality(cfg)

722 for cfg in bucket_config}

723 limit_mm_per_prompt = {

736 - k: v for k, v in limit_mm_per_prompt.items()

724 + k: v for k, v in limit_mm_per_prompt.items()

725 if k in allowed_modalities}

726 if not limit_mm_per_prompt:

727 raise ValueError("No valid limits for modalities present in "

@@ -746,19 +734,19 @@ class RandomMultiModalDataset(RandomDataset):

734 # Get max and min num mm items and ensure

735 # it is at most the sum of limit_mm_per_prompt for all modalities

736 max_num_mm_items = min(

749 - sum(limit_mm_per_prompt.values()),

737 + sum(limit_mm_per_prompt.values()),

738 math.ceil(base_items_per_request * (1 + num_mm_items_range_ratio))

739 )

740 # Ensure min num mm items is at least 0

741 min_num_mm_items = max(

754 - 0,

742 + 0,

743 math.floor(base_items_per_request * (1 - num_mm_items_range_ratio))

744 )

745 # Raise error if min num mm items is greater than max num mm items

746 if min_num_mm_items > max_num_mm_items:

747 raise ValueError(f"Min num mm items is greater than max mm items: "

748 f"{min_num_mm_items} > {max_num_mm_items}")

761 -

749 +

750 logger.info(

751 "Sampling number of multimodal items from [%s, %s]",

752 min_num_mm_items, max_num_mm_items,

@@ -783,8 +771,8 @@ class RandomMultiModalDataset(RandomDataset):

771 whose size is between min_num_mm_items and max_num_mm_items.

772

773 Loop over the bucket config and sample a multimodal item.

786 - Loop until the number of multimodal items sampled is equal to

787 - request_num_mm_items or limit of multimodal items per prompt

774 + Loop until the number of multimodal items sampled is equal to

775 + request_num_mm_items or limit of multimodal items per prompt

776 for all modalities is reached.

777

778 Note:

@@ -796,19 +784,19 @@ class RandomMultiModalDataset(RandomDataset):

784 # Get the number of multimodal items to sample

785 request_num_mm_items = int(

786 self._rng.integers(min_num_mm_items, max_num_mm_items + 1)

799 - )

787 + )

788 # If request_num_mm_items is 0, yield an empty iterator

789 if request_num_mm_items == 0:

790 return

791 # Initialize modality counters

804 - modality_counter = {self.map_config_to_modality(k): 0

792 + modality_counter = {self.map_config_to_modality(k): 0

793 for k in bucket_config}

794 # Copy the bucket config to avoid modifying the original

795 bucket_config_copy = bucket_config.copy()

796 # Loop over the number of multimodal items to sample

797 while sum(modality_counter.values()) < request_num_mm_items:

798 # Sample a multimodal item config

811 - mm_item_config = self._rng.choice(list(bucket_config_copy.keys()),

799 + mm_item_config = self._rng.choice(list(bucket_config_copy.keys()),

800 p=list(bucket_config_copy.values()))

801 modality = self.map_config_to_modality(mm_item_config)

802 # Check that modality count is less than limit per prompt

@@ -849,7 +837,7 @@ class RandomMultiModalDataset(RandomDataset):

837 limit_mm_per_prompt: dict[str, int] = DEFAULT_LIMIT_MM_PER_PROMPT,

838 base_items_per_request: int = DEFAULT_BASE_ITEMS_PER_REQUEST,

839 num_mm_items_range_ratio: float = DEFAULT_NUM_MM_ITEMS_RANGE_RATIO,

852 - bucket_config: dict[tuple[int, int, int], float] =

840 + bucket_config: dict[tuple[int, int, int], float] =

841 DEFAULT_MM_ITEM_BUCKET_CONFIG,

842 enable_multimodal_chat: bool = DEFAULT_ENABLE_MULTIMODAL_CHAT,

843 **kwargs,

@@ -857,7 +845,7 @@ class RandomMultiModalDataset(RandomDataset):

845

846 # NOTE: Video sampling is WIP. Raise error if video is in bucket config

847 # and probability is non-zero.

860 - if any(self.map_config_to_modality(cfg) == "video" and p > 0

848 + if any(self.map_config_to_modality(cfg) == "video" and p > 0

849 for cfg, p in bucket_config.items()):

850 raise NotImplementedError("Video sampling not implemented; "

851 "set its probability to 0.")

@@ -908,7 +896,7 @@ class RandomMultiModalDataset(RandomDataset):

896 ])

897

898 if enable_multimodal_chat:

911 - # NOTE: For now this option is only provided for completeness

899 + # NOTE: For now this option is only provided for completeness

900 # given that the serve.py benchmark currently does not use it.

901 mm_chat_prompt: Any = prompt

902 mm_chat_prompt = self.apply_multimodal_chat_transformation(

@@ -982,8 +970,8 @@ class ShareGPTDataset(BenchmarkDataset):

970 entry["conversations"][1]["value"],

971 )

972

985 - lora_request, tokenizer = self.get_random_lora_request(

986 - tokenizer=tokenizer, max_loras=max_loras, lora_path=lora_path)

973 + lora_request = self.get_random_lora_request(

974 + max_loras=max_loras, lora_path=lora_path)

975 prompt_ids = tokenizer(prompt).input_ids

976 completion_ids = tokenizer(completion).input_ids

977 prompt_len = len(prompt_ids)

@@ -994,11 +982,11 @@ class ShareGPTDataset(BenchmarkDataset):

982 skip_min_output_len_check=output_len

983 is not None):

984 continue

997 - if image_path := entry.get("image"):

998 - mm_content = process_image(image_path)

999 - elif video_path := entry.get("video"):

985 + if image_path := entry.get("image"):

986 + mm_content = process_image(image_path)

987 + elif video_path := entry.get("video"):

988 mm_content = process_video(video_path)

1001 - else:

989 + else:

990 mm_content = None

991 if enable_multimodal_chat:

992 prompt = self.apply_multimodal_chat_transformation(

@@ -1013,9 +1001,9 @@ class ShareGPTDataset(BenchmarkDataset):

1001 request_id=request_id_prefix + str(ind),

1002 ))

1003 ind += 1

1016 - self.maybe_oversample_requests(samples,

1017 - num_requests,

1018 - request_id_prefix,

1004 + self.maybe_oversample_requests(samples,

1005 + num_requests,

1006 + request_id_prefix,

1007 no_oversample)

1008 return samples

1009

@@ -1024,11 +1012,11 @@ class _ValidateDatasetArgs(argparse.Action):

1012 """Argparse action to validate dataset name and path compatibility."""

1013 def __call__(self, parser, namespace, values, option_string=None):

1014 setattr(namespace, self.dest, values)

1027 -

1015 +

1016 # Get current values of both dataset_name and dataset_path

1017 dataset_name = getattr(namespace, 'dataset_name', 'random')

1018 dataset_path = getattr(namespace, 'dataset_path', None)

1031 -

1019 +

1020 # Validate the combination

1021 if dataset_name == "random" and dataset_path is not None:

1022 parser.error(

@@ -1053,7 +1041,7 @@ def add_dataset_parser(parser: FlexibleArgumentParser):

1041 default="random",

1042 action=_ValidateDatasetArgs,

1043 choices=[

1056 - "sharegpt", "burstgpt", "sonnet", "random", "random-mm", "hf",

1044 + "sharegpt", "burstgpt", "sonnet", "random", "random-mm", "hf",

1045 "custom", "prefix_repetition", "spec_bench"

1046 ],

1047 help="Name of the dataset to benchmark on.",

@@ -1502,7 +1490,7 @@ def get_samples(args, tokenizer) -> list[SampleRequest]:

1490 # For datasets that follow a similar structure, use a mapping.

1491 dataset_mapping = {

1492 "spec_bench":

1505 - lambda: SpecBench(dataset_path=args.dataset_path,

1493 + lambda: SpecBench(dataset_path=args.dataset_path,

1494 category=args.spec_bench_category).sample(

1495 num_requests=args.num_prompts,

1496 tokenizer=tokenizer,

@@ -1660,7 +1648,7 @@ class CustomDataset(BenchmarkDataset):

1648 logger.info("num_requests is set to 0 or negative, "

1649 "so using all available samples: %d",

1650 num_requests)

1663 -

1651 +

1652 sampled_requests = []

1653 for i, item in enumerate(self.data):

1654 if len(sampled_requests) >= num_requests:

@@ -1686,7 +1674,7 @@ class CustomDataset(BenchmarkDataset):

1674 expected_output_len=output_len,

1675 request_id=request_id_prefix + str(i),

1676 ))

1689 - self.maybe_oversample_requests(sampled_requests, num_requests,

1677 + self.maybe_oversample_requests(sampled_requests, num_requests,

1678 request_id_prefix, no_oversample)

1679

1680 return sampled_requests

@@ -1700,7 +1688,7 @@ class CustomDataset(BenchmarkDataset):

1688 class SpecBench(CustomDataset):

1689 """

1690 Implements the SpecBench dataset: https://github.com/hemingkx/Spec-Bench

1703 - Download the dataset using:

1691 + Download the dataset using:

1692 wget https://raw.githubusercontent.com/hemingkx/Spec-Bench/refs/heads/main/data/spec_bench/question.jsonl

1693 """ # noqa: E501

1694

@@ -1736,8 +1724,8 @@ class SpecBench(CustomDataset):

1724 # leverage CustomDataset sample

1725 kwargs["skip_chat_template"] = False

1726 return super().sample(**kwargs)

1739 -

1740 -

1727 +

1728 +

1729 # -----------------------------------------------------------------------------

1730 # Sonnet Dataset Implementation

1731 # -----------------------------------------------------------------------------

@@ -1882,8 +1870,8 @@ class BurstGPTDataset(BenchmarkDataset):

1870 for i in range(num_requests):

1871 input_len = int(data[i][2])

1872 output_len = int(data[i][3])

1885 - lora_req, tokenizer = self.get_random_lora_request(

1886 - tokenizer=tokenizer, max_loras=max_loras, lora_path=lora_path)

1873 + lora_req = self.get_random_lora_request(

1874 + max_loras=max_loras, lora_path=lora_path)

1875 vocab_size = tokenizer.vocab_size

1876 # Generate a synthetic prompt: a list of token IDs computed as (i +

1877 # j) modulo vocab_size.

@@ -1995,7 +1983,7 @@ class ConversationDataset(HuggingFaceDataset):

1983 request_id=request_id_prefix + str(ind),

1984 ))

1985 ind += 1

1998 - self.maybe_oversample_requests(sampled_requests, num_requests,

1986 + self.maybe_oversample_requests(sampled_requests, num_requests,

1987 request_id_prefix, no_oversample)

1988 return sampled_requests

1989

@@ -2055,7 +2043,7 @@ class VisionArenaDataset(HuggingFaceDataset):

2043 multi_modal_data=mm_content,

2044 request_id=request_id_prefix + str(i),

2045 ))

2058 - self.maybe_oversample_requests(sampled_requests, num_requests,

2046 + self.maybe_oversample_requests(sampled_requests, num_requests,

2047 request_id_prefix, no_oversample)

2048 return sampled_requests

2049

@@ -2172,7 +2160,7 @@ class InstructCoderDataset(HuggingFaceDataset):

2160 expected_output_len=output_len,

2161 request_id=request_id_prefix + str(i),

2162 ))

2175 - self.maybe_oversample_requests(sampled_requests, num_requests,

2163 + self.maybe_oversample_requests(sampled_requests, num_requests,

2164 request_id_prefix, no_oversample)

2165 return sampled_requests

2166

@@ -2234,7 +2222,7 @@ class MTBenchDataset(HuggingFaceDataset):

2222 expected_output_len=output_len,

2223 request_id=request_id_prefix + str(i),

2224 ))

2237 - self.maybe_oversample_requests(sampled_requests, num_requests,

2225 + self.maybe_oversample_requests(sampled_requests, num_requests,

2226 request_id_prefix, no_oversample)

2227 return sampled_requests

2228

@@ -2288,8 +2276,8 @@ class BlazeditDataset(HuggingFaceDataset):

2276 # compare the levenshtein distance normalized by code length

2277 if norm_distance < min_distance or norm_distance > max_distance:

2278 continue

2291 -

2292 - # template copied from

2279 +

2280 + # template copied from

2281 # https://github.com/ise-uiuc/blazedit/blob/7765137e656fd62de877422d2e4cf8de51228054/dataset/create_refined_dataset.py#L94-L105 # noqa: E501

2282 instruction = f"""Given a code file, please apply the change requests and generate the new file.

2283

@@ -2322,9 +2310,9 @@ Please generate the new code file in the "New file" section below.""" # noqa: E5

2310 expected_output_len=output_len,

2311 request_id=request_id_prefix + str(i),

2312 ))

2325 - self.maybe_oversample_requests(sampled_requests, num_requests,

2313 + self.maybe_oversample_requests(sampled_requests, num_requests,

2314 request_id_prefix, no_oversample)

2327 -

2315 +

2316 return sampled_requests

2317

2318

@@ -2376,7 +2364,6 @@ class AIMODataset(HuggingFaceDataset):

2364 expected_output_len=output_len,

2365 multi_modal_data=None,

2366 request_id=request_id_prefix + str(ind),

2379 -

2367 ))

2368 ind += 1

2369 self.maybe_oversample_requests(sampled_requests, num_requests,

@@ -2470,9 +2457,9 @@ class NextEditPredictionDataset(HuggingFaceDataset):

2457 ))

2458 if len(samples) >= num_requests:

2459 break

2473 - self.maybe_oversample_requests(samples,

2474 - num_requests,

2475 - request_id_prefix,

2460 + self.maybe_oversample_requests(samples,

2461 + num_requests,

2462 + request_id_prefix,

2463 no_oversample)

2464 return samples

2465

@@ -2562,7 +2549,7 @@ class ASRDataset(HuggingFaceDataset):

2549 " what Whisper supports.",

2550 skipped,

2551 )

2565 - self.maybe_oversample_requests(sampled_requests, num_requests,

2552 + self.maybe_oversample_requests(sampled_requests, num_requests,

2553 request_id_prefix, no_oversample)

2554 return sampled_requests

2555

@@ -2647,7 +2634,7 @@ class MLPerfDataset(HuggingFaceDataset):

2634 )

2635 ind += 1

2636

2650 - self.maybe_oversample_requests(sampled_requests, num_requests,

2637 + self.maybe_oversample_requests(sampled_requests, num_requests,

2638 request_id_prefix, no_oversample)

2639 return sampled_requests

2640

@@ -2658,7 +2645,7 @@ class MLPerfDataset(HuggingFaceDataset):

2645

2646

2647 class PrefixRepetitionRandomDataset(BenchmarkDataset):

2661 - # Default values copied from benchmark_serving.py for the repeated prefix

2648 + # Default values copied from benchmark_serving.py for the repeated prefix

2649 # dataset.

2650 DEFAULT_PREFIX_LEN = 256

2651 DEFAULT_SUFFIX_LEN = 256

vllm/engine/async_llm_engine.py

+4 -11

@@ -390,11 +390,8 @@ class _AsyncLLMEngine(LLMEngine):

390 """Stop the remote worker execution loop."""

391 await self.model_executor.stop_remote_worker_execution_loop_async()

392

393 - async def get_tokenizer_async(self,

394 - lora_request: Optional[LoRARequest] = None

395 - ) -> AnyTokenizer:

396 - return await (

397 - self.get_tokenizer_group().get_lora_tokenizer_async(lora_request))

393 + async def get_tokenizer_async(self) -> AnyTokenizer:

394 + return self.get_tokenizer()

395

396 async def add_request_async(

397 self,

@@ -435,7 +432,6 @@ class _AsyncLLMEngine(LLMEngine):

432

433 processed_inputs = await self.input_preprocessor.preprocess_async(

434 prompt,

438 - lora_request=lora_request,

435 tokenization_kwargs=tokenization_kwargs,

436 )

437

@@ -614,11 +610,8 @@ class AsyncLLMEngine(EngineClient):

610 async def get_input_preprocessor(self) -> InputPreprocessor:

611 return self.engine.input_preprocessor

612

617 - async def get_tokenizer(

618 - self,

619 - lora_request: Optional[LoRARequest] = None,

620 - ) -> AnyTokenizer:

621 - return await self.engine.get_tokenizer_async(lora_request)

613 + async def get_tokenizer(self) -> AnyTokenizer:

614 + return self.engine.get_tokenizer()

615

616 def start_background_loop(self) -> None:

617 """Start the background loop."""

vllm/engine/llm_engine.py

+14 -43

@@ -49,9 +49,8 @@ from vllm.sequence import (ExecuteModelRequest, ParallelSampleSequenceGroup,

49 from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context,

50 init_tracer)

51 from vllm.transformers_utils.detokenizer import Detokenizer

52 -from vllm.transformers_utils.tokenizer import AnyTokenizer

53 -from vllm.transformers_utils.tokenizer_group import (

54 - TokenizerGroup, init_tokenizer_from_configs)

52 +from vllm.transformers_utils.tokenizer import (AnyTokenizer,

53 + init_tokenizer_from_configs)

54 from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled,

55 usage_message)

56 from vllm.utils import Counter, Device, resolve_obj_by_qualname, weak_bind

@@ -186,7 +185,7 @@ class LLMEngine:

185

186 return outputs_

187

189 - tokenizer: Optional[TokenizerGroup]

188 + tokenizer: Optional[AnyTokenizer]

189

190 def __init__(

191 self,

@@ -233,18 +232,9 @@ class LLMEngine:

232 if self.model_config.skip_tokenizer_init:

233 self.tokenizer = None

234 self.detokenizer = None

236 - tokenizer_group = None

235 else:

236 self.tokenizer = self._init_tokenizer()

237 self.detokenizer = Detokenizer(self.tokenizer)

240 - tokenizer_group = self.get_tokenizer_group()

241 -

242 - # Ensure that the function doesn't contain a reference to self,

243 - # to avoid engine GC issues

244 - def get_tokenizer_for_seq(sequence: Sequence) -> AnyTokenizer:

245 - assert tokenizer_group, ("tokenizer_group cannot be None, "

246 - "make sure skip_tokenizer_init is False")

247 - return tokenizer_group.get_lora_tokenizer(sequence.lora_request)

238

239 self.seq_counter = Counter()

240 self.generation_config_fields = (

@@ -389,10 +379,8 @@ class LLMEngine:

379 self.detokenizer,

380 self.scheduler,

381 self.seq_counter,

392 - get_tokenizer_for_seq,

382 stop_checker=StopChecker(

383 self.scheduler_config.max_model_len,

395 - get_tokenizer_for_seq,

384 self.reasoner if self.decoding_config.reasoning_backend

385 and self.tokenizer else None,

386 ),

@@ -521,24 +509,15 @@ class LLMEngine:

509 if model_executor := getattr(self, "model_executor", None):

510 model_executor.shutdown()

511

524 - def get_tokenizer_group(self) -> TokenizerGroup:

512 + def get_tokenizer(self) -> AnyTokenizer:

513 if self.tokenizer is None:

514 raise ValueError("Unable to get tokenizer because "

515 "skip_tokenizer_init is True")

516

517 return self.tokenizer

518

531 - def get_tokenizer(

532 - self,

533 - lora_request: Optional[LoRARequest] = None,

534 - ) -> AnyTokenizer:

535 - return self.get_tokenizer_group().get_lora_tokenizer(lora_request)

536 -

537 - def _init_tokenizer(self) -> TokenizerGroup:

538 - return init_tokenizer_from_configs(

539 - model_config=self.model_config,

540 - scheduler_config=self.scheduler_config,

541 - lora_config=self.lora_config)

519 + def _init_tokenizer(self) -> AnyTokenizer:

520 + return init_tokenizer_from_configs(model_config=self.model_config)

521

522 def _verify_args(self) -> None:

523 self.model_config.verify_with_parallel_config(self.parallel_config)

@@ -574,11 +553,11 @@ class LLMEngine:

553 )

554 return None

555

577 - self._validate_model_inputs(processed_inputs, lora_request)

556 + self._validate_model_inputs(processed_inputs)

557 # Create the sequences.

558 block_size = self.cache_config.block_size

559 seq_id = next(self.seq_counter)

581 - eos_token_id = self.input_preprocessor.get_eos_token_id(lora_request)

560 + eos_token_id = self.input_preprocessor.get_eos_token_id()

561

562 encoder_inputs, decoder_inputs = split_enc_dec_inputs(processed_inputs)

563

@@ -700,7 +679,6 @@ class LLMEngine:

679 processed_inputs = self.input_preprocessor.preprocess(

680 prompt,

681 tokenization_kwargs=tokenization_kwargs,

703 - lora_request=lora_request,

682 )

683

684 self._add_processed_request(

@@ -1739,29 +1717,22 @@ class LLMEngine:

1717 SpanAttributes.GEN_AI_LATENCY_TIME_IN_MODEL_EXECUTE,

1718 metrics.model_execute_time)

1719

1742 - def _validate_model_inputs(self, inputs: ProcessorInputs,

1743 - lora_request: Optional[LoRARequest]):

1720 + def _validate_model_inputs(self, inputs: ProcessorInputs):

1721 encoder_inputs, decoder_inputs = split_enc_dec_inputs(inputs)

1722

1723 if encoder_inputs is not None:

1747 - self._validate_model_input(encoder_inputs,

1748 - lora_request,

1749 - prompt_type="encoder")

1724 + self._validate_model_input(encoder_inputs, prompt_type="encoder")

1725

1751 - self._validate_model_input(decoder_inputs,

1752 - lora_request,

1753 - prompt_type="decoder")

1726 + self._validate_model_input(decoder_inputs, prompt_type="decoder")

1727

1728 def _validate_model_input(

1729 self,

1730 prompt_inputs: SingletonInputs,

1758 - lora_request: Optional[LoRARequest],

1731 *,

1732 prompt_type: Literal["encoder", "decoder"],

1733 ):

1734 model_config = self.model_config

1763 - tokenizer = (None if self.tokenizer is None else

1764 - self.tokenizer.get_lora_tokenizer(lora_request))

1735 + tokenizer = self.tokenizer

1736

1737 prompt_ids = prompt_inputs.get("prompt_token_ids", [])

1738 if not prompt_ids:

@@ -1822,7 +1793,7 @@ class LLMEngine:

1793 logits_processors = []

1794

1795 if (sampling_params.logit_bias or sampling_params.allowed_token_ids):

1825 - tokenizer = self.get_tokenizer(lora_request=lora_request)

1796 + tokenizer = self.get_tokenizer()

1797

1798 processors = get_openai_logits_processors(

1799 logit_bias=sampling_params.logit_bias,

@@ -1835,7 +1806,7 @@ class LLMEngine:

1806 sampling_params.allowed_token_ids = None

1807

1808 if len(sampling_params.bad_words) > 0:

1838 - tokenizer = self.get_tokenizer(lora_request)

1809 + tokenizer = self.get_tokenizer()

1810 processors = get_bad_words_logits_processors(

1811 bad_words=sampling_params.bad_words, tokenizer=tokenizer)

1812 logits_processors.extend(processors)

vllm/engine/output_processor/interfaces.py

+2 -4

@@ -2,14 +2,13 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 from abc import ABC, abstractmethod

5 -from typing import Callable, List

5 +from typing import List

7 from vllm.config import SchedulerConfig

8 from vllm.core.scheduler import Scheduler

9 from vllm.engine.output_processor.stop_checker import StopChecker

10 -from vllm.sequence import Sequence, SequenceGroup, SequenceGroupOutput

10 +from vllm.sequence import SequenceGroup, SequenceGroupOutput

11 from vllm.transformers_utils.detokenizer import Detokenizer

12 -from vllm.transformers_utils.tokenizer import AnyTokenizer

12 from vllm.utils import Counter

@@ -31,7 +30,6 @@ class SequenceGroupOutputProcessor(ABC):

30 detokenizer: Detokenizer,

31 scheduler: List[Scheduler],

32 seq_counter: Counter,

34 - get_tokenizer_for_seq: Callable[[Sequence], AnyTokenizer],

33 stop_checker: "StopChecker",

34 ):

35 """Create an output processor.

vllm/engine/output_processor/stop_checker.py

+1 -4

@@ -1,13 +1,12 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 -from typing import Callable, List, Optional, Tuple

4 +from typing import List, Optional, Tuple

6 from vllm.lora.request import LoRARequest

7 from vllm.reasoning import ReasoningParser

8 from vllm.sampling_params import SamplingParams

9 from vllm.sequence import Sequence, SequenceStatus

10 -from vllm.transformers_utils.tokenizer import AnyTokenizer

12 class StopChecker:

@@ -20,12 +19,10 @@ class StopChecker:

19 def __init__(

20 self,

21 max_model_len: int,

23 - get_tokenizer_for_seq: Callable[[Sequence], AnyTokenizer],

22 reasoner: Optional[ReasoningParser] = None,

23 ):

24 # Do not use it directly, but use `self._get_max_model_len`.

25 self._max_model_len = max_model_len

28 - self.get_tokenizer_for_seq = get_tokenizer_for_seq

26 self.reasoner = reasoner

28 def _get_max_model_len(self, lora_req: Optional[LoRARequest]):

vllm/engine/protocol.py

+3 -7

@@ -76,8 +76,7 @@ class EngineClient(ABC):

76 include_stop_str_in_output = params.include_stop_str_in_output

78 preprocessor = await self.get_input_preprocessor()

79 - tokenizer_group = preprocessor.get_tokenizer_group()

80 - tokenizer = await tokenizer_group.get_lora_tokenizer_async()

79 + tokenizer = preprocessor.get_tokenizer()

80 eos_token_id = tokenizer.eos_token_id

82 if is_explicit_encoder_decoder_prompt(prompt):

@@ -260,11 +259,8 @@ class EngineClient(ABC):

259 ...

260

261 @abstractmethod

263 - async def get_tokenizer(

264 - self,

265 - lora_request: Optional[LoRARequest] = None,

266 - ) -> AnyTokenizer:

267 - """Get the appropriate tokenizer for the request"""

262 + async def get_tokenizer(self) -> AnyTokenizer:

263 + """Get the tokenizer"""

264 ...

265

266 async def get_io_processor(self) -> IOProcessor:

vllm/entrypoints/llm.py

+6 -14

@@ -301,23 +301,17 @@ class LLM:

301 self.io_processor = get_io_processor(self.llm_engine.vllm_config,

302 io_processor_plugin)

303

304 - def get_tokenizer(

305 - self,

306 - lora_request: Optional[LoRARequest] = None,

307 - ) -> AnyTokenizer:

308 - return self.llm_engine.get_tokenizer_group().get_lora_tokenizer(

309 - lora_request)

304 + def get_tokenizer(self) -> AnyTokenizer:

305 + return self.llm_engine.get_tokenizer()

306

307 def set_tokenizer(self, tokenizer: AnyTokenizer) -> None:

312 - tokenizer_group = self.llm_engine.get_tokenizer_group()

313 -

308 # While CachedTokenizer is dynamic, have no choice but

309 # compare class name. Misjudgment will arise from

310 # user-defined tokenizer started with 'Cached'

311 if tokenizer.__class__.__name__.startswith("Cached"):

318 - tokenizer_group.tokenizer = tokenizer

312 + self.llm_engine.tokenizer = tokenizer

313 else:

320 - tokenizer_group.tokenizer = get_cached_tokenizer(tokenizer)

314 + self.llm_engine.tokenizer = get_cached_tokenizer(tokenizer)

315

316 def get_default_sampling_params(self) -> SamplingParams:

317 if self.default_sampling_params is None:

@@ -707,7 +701,6 @@ class LLM:

701 self,

702 messages: Union[list[ChatCompletionMessageParam],

703 list[list[ChatCompletionMessageParam]]],

710 - lora_request: Optional[LoRARequest] = None,

704 chat_template: Optional[str] = None,

705 chat_template_content_format: ChatTemplateContentFormatOption = "auto",

706 add_generation_prompt: bool = True,

@@ -739,7 +732,7 @@ class LLM:

732 cast(list[ChatCompletionMessageParam], messages)

733 ]

734

742 - tokenizer = self.get_tokenizer(lora_request)

735 + tokenizer = self.get_tokenizer()

736 model_config = self.llm_engine.get_model_config()

737 resolved_content_format = resolve_chat_template_content_format(

738 chat_template,

@@ -872,7 +865,6 @@ class LLM:

865

866 prompts = self.preprocess_chat(

867 messages=messages,

875 - lora_request=lora_request,

868 chat_template=chat_template,

869 chat_template_content_format=chat_template_content_format,

870 add_generation_prompt=add_generation_prompt,

@@ -1519,7 +1511,7 @@ class LLM:

1511 ):

1512 """

1513 Validate that if any multi-modal data is skipped (i.e. None),

1522 - then its corresponding UUID must be set.

1514 + then its corresponding UUID must be set.

1515 """

1516 if multi_modal_data is None:

1517 return

vllm/entrypoints/openai/serving_chat.py

+1 -1

@@ -188,7 +188,7 @@ class OpenAIServingChat(OpenAIServing):

188

189 model_name = self.models.model_name(lora_request)

190

191 - tokenizer = await self.engine_client.get_tokenizer(lora_request)

191 + tokenizer = await self.engine_client.get_tokenizer()

192

193 tool_parser = self.tool_parser

194

vllm/entrypoints/openai/serving_classification.py

+1 -4

@@ -50,10 +50,7 @@ class ClassificationMixin(OpenAIServing):

50 return None

52 try:

53 - ctx.lora_request = self._maybe_get_adapters(ctx.request)

54 -

55 - ctx.tokenizer = await self.engine_client.get_tokenizer(

56 - ctx.lora_request)

53 + ctx.tokenizer = await self.engine_client.get_tokenizer()

55 renderer = self._get_renderer(ctx.tokenizer)

56 ctx.engine_prompts = await renderer.render_prompt(

vllm/entrypoints/openai/serving_completion.py

+1 -2

@@ -127,8 +127,7 @@ class OpenAIServingCompletion(OpenAIServing):

127 if self.model_config.skip_tokenizer_init:

128 tokenizer = None

129 else:

130 - tokenizer = await self.engine_client.get_tokenizer(lora_request

131 - )

130 + tokenizer = await self.engine_client.get_tokenizer()

131 renderer = self._get_renderer(tokenizer)

132

133 engine_prompts = await renderer.render_prompt_and_embeds(

vllm/entrypoints/openai/serving_embedding.py

+3 -4

@@ -76,8 +76,7 @@ class EmbeddingMixin(OpenAIServing):

76 try:

77 ctx.lora_request = self._maybe_get_adapters(ctx.request)

79 - tokenizer = await self.engine_client.get_tokenizer(ctx.lora_request

80 - )

79 + tokenizer = await self.engine_client.get_tokenizer()

80 renderer = self._get_renderer(tokenizer)

82 if isinstance(ctx.request, EmbeddingChatRequest):

@@ -394,8 +393,8 @@ class EmbeddingMixin(OpenAIServing):

393 ) -> Optional[ErrorResponse]:

394 """Collect and aggregate batch results

395 with support for chunked processing.

397 -

398 - For chunked requests, performs online aggregation to

396 +

397 + For chunked requests, performs online aggregation to

398 minimize memory usage.

399 For regular requests, collects results normally.

400 """

vllm/entrypoints/openai/serving_pooling.py

+1 -2

@@ -103,8 +103,7 @@ class OpenAIServingPooling(OpenAIServing):

103 if self.model_config.skip_tokenizer_init:

104 tokenizer = None

105 else:

106 - tokenizer = await self.engine_client.get_tokenizer(lora_request

107 - )

106 + tokenizer = await self.engine_client.get_tokenizer()

107 renderer = self._get_renderer(tokenizer)

108

109 if getattr(request, "dimensions", None) is not None:

vllm/entrypoints/openai/serving_responses.py

+1 -1

@@ -240,7 +240,7 @@ class OpenAIServingResponses(OpenAIServing):

240 try:

241 lora_request = self._maybe_get_adapters(request)

242 model_name = self.models.model_name(lora_request)

243 - tokenizer = await self.engine_client.get_tokenizer(lora_request)

243 + tokenizer = await self.engine_client.get_tokenizer()

244

245 if self.use_harmony:

246 messages, request_prompts, engine_prompts = (

vllm/entrypoints/openai/serving_score.py

+1 -1

@@ -269,7 +269,7 @@ class ServingScores(OpenAIServing):

269 ) -> Union[list[PoolingRequestOutput], ErrorResponse]:

270 lora_request = self._maybe_get_adapters(request)

271

272 - tokenizer = await self.engine_client.get_tokenizer(lora_request)

272 + tokenizer = await self.engine_client.get_tokenizer()

273

274 truncate_prompt_tokens = getattr(request, "truncate_prompt_tokens",

275 None)

vllm/entrypoints/openai/serving_tokenization.py

+2 -2

@@ -65,7 +65,7 @@ class OpenAIServingTokenization(OpenAIServing):

65 try:

66 lora_request = self._maybe_get_adapters(request)

68 - tokenizer = await self.engine_client.get_tokenizer(lora_request)

68 + tokenizer = await self.engine_client.get_tokenizer()

69 renderer = self._get_renderer(tokenizer)

71 if isinstance(request, TokenizeChatRequest):

@@ -130,7 +130,7 @@ class OpenAIServingTokenization(OpenAIServing):

130

131 lora_request = self._maybe_get_adapters(request)

132

133 - tokenizer = await self.engine_client.get_tokenizer(lora_request)

133 + tokenizer = await self.engine_client.get_tokenizer()

134

135 self._log_inputs(request_id,

136 request.tokens,

vllm/inputs/preprocess.py

+18 -66

@@ -9,13 +9,11 @@ from typing_extensions import assert_never

10 from vllm.config import ModelConfig

11 from vllm.logger import init_logger

12 -from vllm.lora.request import LoRARequest

12 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry

13 from vllm.multimodal.cache import BaseMultiModalProcessorCache

14 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalEncDecInputs,

15 MultiModalInputs, MultiModalUUIDDict)

16 from vllm.transformers_utils.tokenizer import AnyTokenizer

18 -from vllm.transformers_utils.tokenizer_group import TokenizerGroup

18 from .data import (DecoderOnlyInputs, EmbedsInputs, EmbedsPrompt,

19 EncoderDecoderInputs, ProcessorInputs, PromptType,

@@ -31,7 +29,7 @@ class InputPreprocessor:

29 def __init__(

30 self,

31 model_config: ModelConfig,

34 - tokenizer: Optional[TokenizerGroup],

32 + tokenizer: Optional[AnyTokenizer],

33 mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,

34 mm_processor_cache: Optional[BaseMultiModalProcessorCache] = None,

35 ) -> None:

@@ -42,32 +40,28 @@ class InputPreprocessor:

40 self.mm_registry = mm_registry

41 self.mm_processor_cache = mm_processor_cache

45 - def get_tokenizer_group(self) -> TokenizerGroup:

43 + def get_tokenizer(self) -> AnyTokenizer:

44 if self.tokenizer is None:

45 raise ValueError("You cannot pass text prompts when "

46 "`skip_tokenizer_init` is True")

48 return self.tokenizer

52 - def get_bos_token_id(self,

53 - lora_request: Optional[LoRARequest] = None

54 - ) -> Optional[int]:

50 + def get_bos_token_id(self) -> Optional[int]:

51 if self.tokenizer is None:

52 logger.warning("Using None for BOS token id because tokenizer "

53 "is not initialized")

54 return None

60 - return self.tokenizer.get_lora_tokenizer(lora_request).bos_token_id

56 + return self.tokenizer.bos_token_id

62 - def get_eos_token_id(self,

63 - lora_request: Optional[LoRARequest] = None

64 - ) -> Optional[int]:

58 + def get_eos_token_id(self) -> Optional[int]:

59 if self.tokenizer is None:

60 logger.warning("Using None for EOS token id because tokenizer "

61 "is not initialized")

62 return None

70 - return self.tokenizer.get_lora_tokenizer(lora_request).eos_token_id

64 + return self.tokenizer.eos_token_id

66 def get_decoder_start_token_id(self) -> Optional[int]:

67 """

@@ -190,14 +184,13 @@ class InputPreprocessor:

184 def _tokenize_prompt(

185 self,

186 prompt: str,

193 - lora_request: Optional[LoRARequest],

187 tokenization_kwargs: Optional[dict[str, Any]] = None,

188 ) -> list[int]:

189 """

190 Apply the model's tokenizer to a text prompt, returning the

191 corresponding token IDs.

192 """

200 - tokenizer = self.get_tokenizer_group()

193 + tokenizer = self.get_tokenizer()

194 tokenization_kwargs = self._get_tokenization_kw(tokenization_kwargs)

195

196 encoder_config = self.model_config.encoder_config

@@ -205,50 +198,39 @@ class InputPreprocessor:

198 if encoder_config and encoder_config.get("do_lower_case", False):

199 prompt = prompt.lower()

200

208 - return tokenizer.encode(prompt=prompt,

209 - lora_request=lora_request,

210 - **tokenization_kwargs)

201 + return tokenizer.encode(prompt, **tokenization_kwargs)

202

203 async def _tokenize_prompt_async(

204 self,

205 prompt: str,

215 - lora_request: Optional[LoRARequest],

206 tokenization_kwargs: Optional[dict[str, Any]] = None,

207 ) -> list[int]:

208 """

209 Async version of

210 [`_tokenize_prompt`][vllm.inputs.preprocess.InputPreprocessor._tokenize_prompt].

211 """

222 - tokenizer = self.get_tokenizer_group()

212 + tokenizer = self.get_tokenizer()

213 tokenization_kwargs = self._get_tokenization_kw(tokenization_kwargs)

214

225 - return await tokenizer.encode_async(prompt=prompt,

226 - lora_request=lora_request,

227 - **tokenization_kwargs)

215 + return tokenizer.encode(prompt, **tokenization_kwargs)

216

229 - def _get_mm_tokenizer(

230 - self,

231 - lora_request: Optional[LoRARequest],

232 - ) -> AnyTokenizer:

217 + def _get_mm_tokenizer(self) -> AnyTokenizer:

218 # PrithviGeoSpatialMAE needs to be initialized without a tokenizer

219 # while using also multi-modal input

220 if not self.tokenizer:

221 return cast(AnyTokenizer, object()) # Dummy

222

238 - tokenizer_group = self.get_tokenizer_group()

239 - return tokenizer_group.get_lora_tokenizer(lora_request)

223 + tokenizer = self.get_tokenizer()

224 + return tokenizer

225

241 - async def _get_mm_tokenizer_async(

242 - self,

243 - lora_request: Optional[LoRARequest],

244 - ) -> AnyTokenizer:

226 + async def _get_mm_tokenizer_async(self) -> AnyTokenizer:

227 # PrithviGeoSpatialMAE needs to be initialized without a tokenizer

228 # while using also multi-modal input

229 if not self.tokenizer:

230 return cast(AnyTokenizer, object()) # Dummy

231

250 - tokenizer_group = self.get_tokenizer_group()

251 - return await tokenizer_group.get_lora_tokenizer_async(lora_request)

232 + tokenizer = self.get_tokenizer()

233 + return tokenizer

234

235 def _process_multimodal(

236 self,

@@ -256,7 +238,6 @@ class InputPreprocessor:

238 mm_data: MultiModalDataDict,

239 mm_processor_kwargs: Optional[Mapping[str, object]],

240 tokenization_kwargs: Optional[dict[str, Any]] = None,

259 - lora_request: Optional[LoRARequest] = None,

241 *,

242 mm_uuids: Optional[MultiModalUUIDDict] = None,

243 ) -> MultiModalInputs:

@@ -264,7 +245,7 @@ class InputPreprocessor:

245 Apply the model's multi-modal processor to a multi-modal prompt,

246 returning the corresponding token IDs and metadata.

247 """

267 - tokenizer = self._get_mm_tokenizer(lora_request)

248 + tokenizer = self._get_mm_tokenizer()

249

250 mm_processor = self.mm_registry.create_processor(

251 self.model_config,

@@ -299,7 +280,6 @@ class InputPreprocessor:

280 mm_data: MultiModalDataDict,

281 mm_processor_kwargs: Optional[Mapping[str, object]],

282 tokenization_kwargs: Optional[dict[str, Any]] = None,

302 - lora_request: Optional[LoRARequest] = None,

283 *,

284 mm_uuids: Optional[MultiModalUUIDDict] = None,

285 ) -> MultiModalInputs:

@@ -307,7 +287,7 @@ class InputPreprocessor:

287 Async version of

288 [`_process_multimodal`][vllm.inputs.preprocess.InputPreprocessor._process_multimodal].

289 """

310 - tokenizer = await self._get_mm_tokenizer_async(lora_request)

290 + tokenizer = await self._get_mm_tokenizer_async()

291

292 mm_processor = self.mm_registry.create_processor(

293 self.model_config,

@@ -386,7 +366,6 @@ class InputPreprocessor:

366 self,

367 parsed_content: TokensPrompt,

368 tokenization_kwargs: Optional[dict[str, Any]] = None,

389 - lora_request: Optional[LoRARequest] = None,

369 *,

370 mm_uuids: Optional[MultiModalUUIDDict] = None,

371 ) -> Union[TokenInputs, MultiModalInputs]:

@@ -400,7 +379,6 @@ class InputPreprocessor:

379 multi_modal_data,

380 parsed_content.get("mm_processor_kwargs"),

381 tokenization_kwargs=tokenization_kwargs,

403 - lora_request=lora_request,

382 mm_uuids=mm_uuids,

383 )

384 else:

@@ -415,7 +393,6 @@ class InputPreprocessor:

393 self,

394 parsed_content: TokensPrompt,

395 tokenization_kwargs: Optional[dict[str, Any]] = None,

418 - lora_request: Optional[LoRARequest] = None,

396 *,

397 mm_uuids: Optional[MultiModalUUIDDict] = None,

398 ) -> Union[TokenInputs, MultiModalInputs]:

@@ -429,7 +406,6 @@ class InputPreprocessor:

406 multi_modal_data,

407 parsed_content.get("mm_processor_kwargs"),

408 tokenization_kwargs=tokenization_kwargs,

432 - lora_request=lora_request,

409 mm_uuids=mm_uuids,

410 )

411 else:

@@ -444,7 +420,6 @@ class InputPreprocessor:

420 self,

421 parsed_content: TextPrompt,

422 tokenization_kwargs: Optional[dict[str, Any]] = None,

447 - lora_request: Optional[LoRARequest] = None,

423 *,

424 mm_uuids: Optional[MultiModalUUIDDict] = None,

425 ) -> Union[TokenInputs, MultiModalInputs]:

@@ -457,13 +432,11 @@ class InputPreprocessor:

432 multi_modal_data,

433 parsed_content.get("mm_processor_kwargs"),

434 tokenization_kwargs=tokenization_kwargs,

460 - lora_request=lora_request,

435 mm_uuids=mm_uuids,

436 )

437 else:

438 prompt_token_ids = self._tokenize_prompt(

439 prompt_text,

466 - lora_request=lora_request,

440 tokenization_kwargs=tokenization_kwargs,

441 )

442 inputs = token_inputs(

@@ -480,7 +453,6 @@ class InputPreprocessor:

453 self,

454 parsed_content: TextPrompt,

455 tokenization_kwargs: Optional[dict[str, Any]] = None,

483 - lora_request: Optional[LoRARequest] = None,

456 *,

457 mm_uuids: Optional[MultiModalUUIDDict] = None,

458 ) -> Union[TokenInputs, MultiModalInputs]:

@@ -493,13 +465,11 @@ class InputPreprocessor:

465 multi_modal_data,

466 parsed_content.get("mm_processor_kwargs"),

467 tokenization_kwargs=tokenization_kwargs,

496 - lora_request=lora_request,

468 mm_uuids=mm_uuids,

469 )

470 else:

471 prompt_token_ids = await self._tokenize_prompt_async(

472 prompt_text,

502 - lora_request=lora_request,

473 tokenization_kwargs=tokenization_kwargs,

474 )

475 inputs = token_inputs(

@@ -516,7 +486,6 @@ class InputPreprocessor:

486 self,

487 prompt: SingletonPrompt,

488 tokenization_kwargs: Optional[dict[str, Any]] = None,

519 - lora_request: Optional[LoRARequest] = None,

489 *,

490 mm_uuids: Optional[MultiModalUUIDDict] = None,

491 ) -> SingletonInputs:

@@ -526,7 +495,6 @@ class InputPreprocessor:

495 Arguments:

496

497 * prompt: single encoder or decoder input prompt

529 - * lora_request: this is only valid for decoder prompts

498

499 Returns:

500

@@ -539,21 +507,18 @@ class InputPreprocessor:

507 if parsed["type"] == "tokens":

508 return self._process_tokens(

509 parsed["content"],

542 - lora_request=lora_request,

510 mm_uuids=mm_uuids,

511 )

512 if parsed["type"] == "text":

513 return self._process_text(

514 parsed["content"],

515 tokenization_kwargs=tokenization_kwargs,

549 - lora_request=lora_request,

516 mm_uuids=mm_uuids,

517 )

518 if parsed["type"] == "str":

519 return self._process_text(

520 TextPrompt(prompt=parsed["content"]),

521 tokenization_kwargs=tokenization_kwargs,

556 - lora_request=lora_request,

522 mm_uuids=mm_uuids,

523 )

524

@@ -563,7 +528,6 @@ class InputPreprocessor:

528 self,

529 prompt: SingletonPrompt,

530 tokenization_kwargs: Optional[dict[str, Any]] = None,

566 - lora_request: Optional[LoRARequest] = None,

531 *,

532 mm_uuids: Optional[MultiModalUUIDDict] = None,

533 ) -> SingletonInputs:

@@ -578,21 +542,18 @@ class InputPreprocessor:

542 if parsed["type"] == "tokens":

543 return await self._process_tokens_async(

544 parsed["content"],

581 - lora_request=lora_request,

545 mm_uuids=mm_uuids,

546 )

547 if parsed["type"] == "text":

548 return await self._process_text_async(

549 parsed["content"],

550 tokenization_kwargs=tokenization_kwargs,

588 - lora_request=lora_request,

551 mm_uuids=mm_uuids,

552 )

553 if parsed["type"] == "str":

554 return await self._process_text_async(

555 TextPrompt(prompt=parsed["content"]),

556 tokenization_kwargs=tokenization_kwargs,

595 - lora_request=lora_request,

557 mm_uuids=mm_uuids,

558 )

559

@@ -844,7 +805,6 @@ class InputPreprocessor:

805 self,

806 prompt: SingletonPrompt,

807 tokenization_kwargs: Optional[dict[str, Any]] = None,

847 - lora_request: Optional[LoRARequest] = None,

808 *,

809 mm_uuids: Optional[MultiModalUUIDDict] = None,

810 ) -> DecoderOnlyInputs:

@@ -856,7 +816,6 @@ class InputPreprocessor:

816 Arguments:

817

818 * prompt: input prompt

859 - * lora_request

819

820 Returns:

821

@@ -866,7 +825,6 @@ class InputPreprocessor:

825 prompt_comps = self._prompt_to_llm_inputs(

826 prompt,

827 tokenization_kwargs=tokenization_kwargs,

869 - lora_request=lora_request,

828 mm_uuids=mm_uuids,

829 )

830

@@ -876,7 +834,6 @@ class InputPreprocessor:

834 self,

835 prompt: SingletonPrompt,

836 tokenization_kwargs: Optional[dict[str, Any]] = None,

879 - lora_request: Optional[LoRARequest] = None,

837 *,

838 mm_uuids: Optional[MultiModalUUIDDict] = None,

839 ) -> DecoderOnlyInputs:

@@ -887,7 +844,6 @@ class InputPreprocessor:

844 prompt_comps = await self._prompt_to_llm_inputs_async(

845 prompt,

846 tokenization_kwargs=tokenization_kwargs,

890 - lora_request=lora_request,

847 mm_uuids=mm_uuids,

848 )

849

@@ -897,7 +853,6 @@ class InputPreprocessor:

853 self,

854 prompt: PromptType,

855 tokenization_kwargs: Optional[dict[str, Any]] = None,

900 - lora_request: Optional[LoRARequest] = None,

856 *,

857 mm_uuids: Optional[MultiModalUUIDDict] = None,

858 ) -> ProcessorInputs:

@@ -919,7 +874,6 @@ class InputPreprocessor:

874 return self._process_decoder_only_prompt(

875 prompt,

876 tokenization_kwargs=tokenization_kwargs,

922 - lora_request=lora_request,

877 mm_uuids=mm_uuids,

878 )

879

@@ -927,7 +881,6 @@ class InputPreprocessor:

881 self,

882 prompt: PromptType,

883 tokenization_kwargs: Optional[dict[str, Any]] = None,

930 - lora_request: Optional[LoRARequest] = None,

884 *,

885 mm_uuids: Optional[MultiModalUUIDDict] = None,

886 ) -> ProcessorInputs:

@@ -952,7 +905,6 @@ class InputPreprocessor:

905 return await self._process_decoder_only_prompt_async(

906 prompt,

907 tokenization_kwargs=tokenization_kwargs,

955 - lora_request=lora_request,

908 mm_uuids=mm_uuids,

909 )

910

vllm/transformers_utils/detokenizer.py

+8 -15

@@ -10,18 +10,13 @@ from vllm.sequence import (VLLM_INVALID_TOKEN_ID, SamplingParams, Sequence,

10 from .detokenizer_utils import (convert_prompt_ids_to_tokens,

11 detokenize_incrementally)

12 from .tokenizer import AnyTokenizer

13 -from .tokenizer_group import TokenizerGroup

15 class Detokenizer:

16 """Provides methods to decode the output of a model into text."""

19 - def __init__(self, tokenizer_group: TokenizerGroup):

20 - self.tokenizer_group = tokenizer_group

21 -

22 - def get_tokenizer_for_seq(self, sequence: Sequence) -> AnyTokenizer:

23 - """Returns the HF tokenizer to use for a given sequence."""

24 - return self.tokenizer_group.get_lora_tokenizer(sequence.lora_request)

18 + def __init__(self, tokenizer: AnyTokenizer):

19 + self.tokenizer = tokenizer

21 def decode_prompt_logprobs_inplace(self, seq_group: SequenceGroup,

22 prompt_logprobs: list[Optional[dict[

@@ -32,9 +27,9 @@ class Detokenizer:

27 Args:

28 seq_group: The sequence group to decode.

29 prompt_logprobs: The logprobs to decode.

35 - position_offset: Offset of the first index of the logprobs

30 + position_offset: Offset of the first index of the logprobs

31 relative to the start of the sequence (for chunked prefill).

37 -

32 +

33 Returns:

34 The prompt logprobs with the decoded tokens.

35 """

@@ -46,7 +41,6 @@ class Detokenizer:

41 # Only prompt, without the generated token.

42 all_token_ids = seq.get_token_ids()

43 prompt_token_ids = all_token_ids[:-1]

49 - tokenizer = self.get_tokenizer_for_seq(seq)

44 prefix_offset = 0

45 read_offset = 0

46 next_iter_prefix_offset = 0

@@ -70,7 +64,7 @@ class Detokenizer:

64 prompt_token_ids[:token_position] + [token_id])

65 (new_tokens, new_text, new_prefix_offset,

66 new_read_offset) = detokenize_incrementally(

73 - tokenizer=tokenizer,

67 + tokenizer=self.tokenizer,

68 all_input_ids=prompt_token_ids_with_token,

69 prev_tokens=prev_tokens,

70 prefix_offset=prefix_offset,

@@ -111,7 +105,6 @@ class Detokenizer:

105 """

106 all_input_ids = seq.get_token_ids()

107 token_id_generated_this_iteration = all_input_ids[-1]

114 - tokenizer = self.get_tokenizer_for_seq(seq)

108

109 # Convert prompt token IDs to tokens if necessary.

110 # Do it here so that we don't have to repeat this

@@ -119,14 +112,14 @@ class Detokenizer:

112 if seq.tokens is None:

113 (seq.tokens, seq.prefix_offset,

114 seq.read_offset) = convert_prompt_ids_to_tokens(

122 - tokenizer=tokenizer,

115 + tokenizer=self.tokenizer,

116 prompt_ids=all_input_ids[:-1],

117 skip_special_tokens=prms.skip_special_tokens,

118 )

119

120 (new_tokens, new_decoded_token_text, prefix_offset,

121 read_offset) = detokenize_incrementally(

129 - tokenizer=tokenizer,

122 + tokenizer=self.tokenizer,

123 all_input_ids=all_input_ids,

124 prev_tokens=seq.tokens,

125 prefix_offset=seq.prefix_offset,

@@ -150,7 +143,7 @@ class Detokenizer:

143 and token_id != VLLM_INVALID_TOKEN_ID):

144 all_input_ids_with_logprob = previous_tokens + [token_id]

145 (_, new_text, _, _) = detokenize_incrementally(

153 - tokenizer=tokenizer,

146 + tokenizer=self.tokenizer,

147 all_input_ids=all_input_ids_with_logprob,

148 prev_tokens=seq.tokens,

149 prefix_offset=seq.prefix_offset,

vllm/transformers_utils/tokenizer.py

+16 -17

@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any, Optional, Union

12 import huggingface_hub

13 from transformers import (AutoTokenizer, PreTrainedTokenizer,

14 PreTrainedTokenizerFast)

15 +from typing_extensions import assert_never

17 from vllm import envs

18 from vllm.logger import init_logger

@@ -19,7 +20,6 @@ from vllm.transformers_utils.config import (

20 get_sentence_transformer_tokenizer_config)

21 from vllm.transformers_utils.tokenizers import MistralTokenizer

22 from vllm.transformers_utils.utils import check_gguf_file

22 -from vllm.utils import make_async

24 if TYPE_CHECKING:

25 from vllm.config import ModelConfig

@@ -274,20 +274,19 @@ def cached_tokenizer_from_config(

274 )

275

276

277 -def get_lora_tokenizer(lora_request: LoRARequest, *args,

278 - **kwargs) -> Optional[AnyTokenizer]:

279 - if lora_request is None:

280 - return None

281 - try:

282 - tokenizer = get_tokenizer(lora_request.lora_path, *args, **kwargs)

283 - except Exception as e:

284 - # No tokenizer was found in the LoRA folder,

285 - # use base model tokenizer

286 - logger.warning(

287 - "No tokenizer found in %s, using base model tokenizer instead. "

288 - "(Exception: %s)", lora_request.lora_path, e)

289 - tokenizer = None

290 - return tokenizer

291 -

277 +def init_tokenizer_from_configs(model_config: ModelConfig):

278 + runner_type = model_config.runner_type

279 + if runner_type == "generate" or runner_type == "draft":

280 + truncation_side = "left"

281 + elif runner_type == "pooling":

282 + truncation_side = "right"

283 + else:

284 + assert_never(runner_type)

285

293 -get_lora_tokenizer_async = make_async(get_lora_tokenizer)

286 + return get_tokenizer(

287 + model_config.tokenizer,

288 + tokenizer_mode=model_config.tokenizer_mode,

289 + trust_remote_code=model_config.trust_remote_code,

290 + revision=model_config.tokenizer_revision,

291 + truncation_side=truncation_side,

292 + )

vllm/transformers_utils/tokenizer_base.py

@@ -61,6 +61,11 @@ class TokenizerBase(ABC):

61 def max_token_id(self) -> int:

62 raise NotImplementedError()

64 + @property

65 + @abstractmethod

66 + def truncation_side(self) -> str:

67 + raise NotImplementedError()

68 +

69 def __len__(self) -> int:

70 return self.vocab_size

vllm/transformers_utils/tokenizer_group.py deleted

-132

@@ -1,132 +0,0 @@

1 -# SPDX-License-Identifier: Apache-2.0

2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 -

4 -from typing import Optional

5 -

6 -from typing_extensions import assert_never

7 -

8 -from vllm.config import ModelConfig, SchedulerConfig

9 -from vllm.config.lora import LoRAConfig

10 -from vllm.lora.request import LoRARequest

11 -from vllm.transformers_utils.tokenizer import (AnyTokenizer, encode_tokens,

12 - get_lora_tokenizer,

13 - get_lora_tokenizer_async,

14 - get_tokenizer)

15 -from vllm.utils import LRUCache

16 -

17 -

18 -class TokenizerGroup:

19 - """A group of tokenizers that can be used for LoRA adapters."""

20 -

21 - def __init__(self, tokenizer_id: str, enable_lora: bool, max_num_seqs: int,

22 - max_input_length: Optional[int], **tokenizer_config):

23 - self.tokenizer_id = tokenizer_id

24 - self.tokenizer_config = tokenizer_config

25 - self.enable_lora = enable_lora

26 - self.max_input_length = max_input_length

27 - self.truncation_side = tokenizer_config.get("truncation_side", "left")

28 - self.tokenizer = get_tokenizer(self.tokenizer_id, **tokenizer_config)

29 - max_loras = tokenizer_config.get("max_loras", 0)

30 - self.lora_tokenizers = LRUCache[int, AnyTokenizer](

31 - capacity=max(max_loras, max_num_seqs) if enable_lora else 0)

32 -

33 - def get_max_input_len(self,

34 - lora_request: Optional[LoRARequest] = None

35 - ) -> Optional[int]:

36 - """Get the maximum input length for the LoRA request."""

37 - return self.max_input_length

38 -

39 - def _raise_if_input_too_long(self,

40 - encoded_tokens: list[int],

41 - lora_request: Optional[LoRARequest] = None):

42 - input_length = len(encoded_tokens)

43 - if lora_request:

44 - max_input_length = (lora_request.long_lora_max_len

45 - or self.max_input_length)

46 - else:

47 - max_input_length = self.max_input_length

48 - if max_input_length is not None and input_length > max_input_length:

49 - raise ValueError("Input too long.", input_length, max_input_length)

50 -

51 - def encode(self,

52 - prompt: str,

53 - max_length: Optional[int] = None,

54 - truncation: Optional[bool] = None,

55 - lora_request: Optional[LoRARequest] = None,

56 - add_special_tokens: Optional[bool] = None) -> list[int]:

57 -

58 - tokenizer = self.get_lora_tokenizer(lora_request)

59 - ret = encode_tokens(tokenizer,

60 - prompt,

61 - max_length=max_length,

62 - truncation=truncation,

63 - add_special_tokens=add_special_tokens)

64 - self._raise_if_input_too_long(ret, lora_request)

65 - return ret

66 -

67 - async def encode_async(

68 - self,

69 - prompt: str,

70 - max_length: Optional[int] = None,

71 - truncation: Optional[bool] = None,

72 - lora_request: Optional[LoRARequest] = None,

73 - add_special_tokens: Optional[bool] = None) -> list[int]:

74 - tokenizer = await self.get_lora_tokenizer_async(lora_request)

75 - ret = encode_tokens(tokenizer,

76 - prompt,

77 - max_length=max_length,

78 - truncation=truncation,

79 - add_special_tokens=add_special_tokens)

80 - self._raise_if_input_too_long(ret, lora_request)

81 - return ret

82 -

83 - def get_lora_tokenizer(

84 - self,

85 - lora_request: Optional[LoRARequest] = None,

86 - ) -> AnyTokenizer:

87 - if not lora_request or not self.enable_lora:

88 - return self.tokenizer

89 - if lora_request.lora_int_id not in self.lora_tokenizers:

90 - tokenizer = (get_lora_tokenizer(

91 - lora_request, **self.tokenizer_config) or self.tokenizer)

92 - self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer)

93 - return tokenizer

94 - else:

95 - return self.lora_tokenizers[lora_request.lora_int_id]

96 -

97 - async def get_lora_tokenizer_async(

98 - self,

99 - lora_request: Optional[LoRARequest] = None,

100 - ) -> AnyTokenizer:

101 - if not lora_request or not self.enable_lora:

102 - return self.tokenizer

103 - if lora_request.lora_int_id not in self.lora_tokenizers:

104 - tokenizer = (await get_lora_tokenizer_async(

105 - lora_request, **self.tokenizer_config) or self.tokenizer)

106 - self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer)

107 - return tokenizer

108 - else:

109 - return self.lora_tokenizers[lora_request.lora_int_id]

110 -

111 -

112 -def init_tokenizer_from_configs(model_config: ModelConfig,

113 - scheduler_config: SchedulerConfig,

114 - lora_config: Optional[LoRAConfig]):

115 - runner_type = model_config.runner_type

116 - if runner_type == "generate" or runner_type == "draft":

117 - truncation_side = "left"

118 - elif runner_type == "pooling":

119 - truncation_side = "right"

120 - else:

121 - assert_never(runner_type)

122 -

123 - return TokenizerGroup(

124 - tokenizer_id=model_config.tokenizer,

125 - enable_lora=bool(lora_config),

126 - max_num_seqs=scheduler_config.max_num_seqs,

127 - max_loras=lora_config.max_loras if lora_config else 0,

128 - max_input_length=None,

129 - tokenizer_mode=model_config.tokenizer_mode,

130 - trust_remote_code=model_config.trust_remote_code,

131 - revision=model_config.tokenizer_revision,

132 - truncation_side=truncation_side)

vllm/transformers_utils/tokenizers/mistral.py

@@ -327,6 +327,10 @@ class MistralTokenizer(TokenizerBase):

327 def max_token_id(self) -> int:

328 return self._max_token_id

329

330 + @property

331 + def truncation_side(self) -> str:

332 + raise NotImplementedError()

333 +

334 def __len__(self) -> int:

335 return self.vocab_size

336

vllm/v1/engine/async_llm.py

+5 -10

@@ -29,8 +29,8 @@ from vllm.tasks import SupportedTask

29 from vllm.tracing import init_tracer

30 from vllm.transformers_utils.config import (

31 maybe_register_config_serialize_by_value)

32 -from vllm.transformers_utils.tokenizer import AnyTokenizer

33 -from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs

32 +from vllm.transformers_utils.tokenizer import (AnyTokenizer,

33 + init_tokenizer_from_configs)

34 from vllm.usage.usage_lib import UsageContext

35 from vllm.utils import (Device, as_list, cancel_task_threadsafe, cdiv,

36 deprecate_kwargs)

@@ -112,9 +112,7 @@ class AsyncLLM(EngineClient):

112 else:

113 # Tokenizer (+ ensure liveness if running in another process).

114 self.tokenizer = init_tokenizer_from_configs(

115 - model_config=vllm_config.model_config,

116 - scheduler_config=vllm_config.scheduler_config,

117 - lora_config=vllm_config.lora_config)

115 + model_config=vllm_config.model_config)

116

117 # Processor (converts Inputs --> EngineCoreRequests).

118 self.processor = Processor(

@@ -596,15 +594,12 @@ class AsyncLLM(EngineClient):

594 async def get_input_preprocessor(self) -> InputPreprocessor:

595 return self.processor.input_preprocessor

596

599 - async def get_tokenizer(

600 - self,

601 - lora_request: Optional[LoRARequest] = None,

602 - ) -> AnyTokenizer:

597 + async def get_tokenizer(self) -> AnyTokenizer:

598 if self.tokenizer is None:

599 raise ValueError("Unable to get tokenizer because "

600 "skip_tokenizer_init is True")

601

607 - return self.tokenizer.get_lora_tokenizer(lora_request)

602 + return self.tokenizer

603

604 async def is_tracing_enabled(self) -> bool:

605 return self.observability_config.otlp_traces_endpoint is not None

vllm/v1/engine/llm_engine.py

+4 -6

@@ -20,8 +20,8 @@ from vllm.pooling_params import PoolingParams

20 from vllm.sampling_params import SamplingParams

21 from vllm.tasks import SupportedTask

22 from vllm.tracing import init_tracer

23 -from vllm.transformers_utils.tokenizer_group import (

24 - TokenizerGroup, init_tokenizer_from_configs)

23 +from vllm.transformers_utils.tokenizer import (AnyTokenizer,

24 + init_tokenizer_from_configs)

25 from vllm.usage.usage_lib import UsageContext

26 from vllm.utils import Device

27 from vllm.v1.engine.core_client import EngineCoreClient

@@ -89,9 +89,7 @@ class LLMEngine:

89 else:

90 # Tokenizer (+ ensure liveness if running in another process).

91 self.tokenizer = init_tokenizer_from_configs(

92 - model_config=vllm_config.model_config,

93 - scheduler_config=vllm_config.scheduler_config,

94 - lora_config=vllm_config.lora_config)

92 + model_config=vllm_config.model_config)

94 # Processor (convert Inputs --> EngineCoreRequests)

95 self.processor = Processor(vllm_config=vllm_config,

@@ -297,7 +295,7 @@ class LLMEngine:

295 assert self.log_stats, "Stat logging disabled"

296 return get_metrics_snapshot()

297

300 - def get_tokenizer_group(self) -> TokenizerGroup:

298 + def get_tokenizer(self) -> AnyTokenizer:

299 if self.tokenizer is None:

300 raise ValueError("Unable to get tokenizer because "

301 "skip_tokenizer_init is True")

vllm/v1/engine/output_processor.py

+2 -6

@@ -14,7 +14,6 @@ from vllm.sampling_params import RequestOutputKind

14 from vllm.tracing import (SpanAttributes, SpanKind, Tracer,

15 extract_trace_context)

16 from vllm.transformers_utils.tokenizer import AnyTokenizer

17 -from vllm.transformers_utils.tokenizer_group import TokenizerGroup

17 from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason

18 from vllm.v1.engine.detokenizer import IncrementalDetokenizer

19 from vllm.v1.engine.logprobs import LogprobsProcessor

@@ -290,7 +289,7 @@ class RequestState:

289 class OutputProcessor:

290 """Process EngineCoreOutputs into RequestOutputs."""

291

293 - def __init__(self, tokenizer: TokenizerGroup, log_stats: bool):

292 + def __init__(self, tokenizer: AnyTokenizer, log_stats: bool):

293 self.log_stats = log_stats

294 self.tokenizer = tokenizer

295 self.request_states: dict[str, RequestState] = {}

@@ -347,10 +346,7 @@ class OutputProcessor:

346 if request_id in self.request_states:

347 raise ValueError(f"Request id {request_id} already running.")

348

350 - tokenizer = None if not self.tokenizer else \

351 - self.tokenizer.get_lora_tokenizer(request.lora_request)

352 -

353 - req_state = RequestState.from_new_request(tokenizer=tokenizer,

349 + req_state = RequestState.from_new_request(tokenizer=self.tokenizer,

350 request=request,

351 prompt=prompt,

352 parent_req=parent_req,

vllm/v1/engine/processor.py

+29 -26

@@ -9,6 +9,7 @@ from vllm.config import VllmConfig

9 from vllm.inputs import ProcessorInputs, PromptType, SingletonInputs

10 from vllm.inputs.parse import split_enc_dec_inputs

11 from vllm.inputs.preprocess import InputPreprocessor

12 +from vllm.logger import init_logger

13 from vllm.lora.request import LoRARequest

14 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry

15 from vllm.multimodal.cache import processor_cache_from_config

@@ -17,7 +18,7 @@ from vllm.multimodal.processing import EncDecMultiModalProcessor

18 from vllm.multimodal.utils import argsort_mm_positions

19 from vllm.pooling_params import PoolingParams

20 from vllm.sampling_params import SamplingParams

20 -from vllm.transformers_utils.tokenizer_group import TokenizerGroup

21 +from vllm.transformers_utils.tokenizer import AnyTokenizer

22 from vllm.v1.engine import EngineCoreRequest

23 from vllm.v1.structured_output.backend_guidance import (

24 validate_guidance_grammar)

@@ -28,13 +29,15 @@ from vllm.v1.structured_output.backend_outlines import (

29 from vllm.v1.structured_output.backend_xgrammar import (

30 validate_xgrammar_grammar)

32 +logger = init_logger(__name__)

33 +

35 class Processor:

37 def __init__(

38 self,

39 vllm_config: VllmConfig,

37 - tokenizer: TokenizerGroup,

40 + tokenizer: AnyTokenizer,

41 mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,

42 ):

@@ -90,7 +93,6 @@ class Processor:

93 def _validate_sampling_params(

94 self,

95 params: SamplingParams,

93 - lora_request: Optional[LoRARequest],

96 ) -> None:

97 self._validate_structured_output(params)

98 self._validate_logit_bias(params)

@@ -103,8 +105,7 @@ class Processor:

105 # When skip_tokenizer_init=True, we can't validate token IDs

106 # Skip validation and let the model handle invalid tokens

107 return

106 - tokenizer = self.tokenizer.get_lora_tokenizer(lora_request)

107 - vocab_size = len(tokenizer)

108 + vocab_size = len(self.tokenizer)

109 if not all(0 <= tid < vocab_size for tid in params.allowed_token_ids):

110 raise ValueError(

111 "allowed_token_ids contains out-of-vocab token id!")

@@ -144,7 +145,6 @@ class Processor:

145 def _validate_params(

146 self,

147 params: Union[SamplingParams, PoolingParams],

147 - lora_request: Optional[LoRARequest],

148 ):

149 """

150 Validate supported SamplingParam.

@@ -155,14 +155,14 @@ class Processor:

155 return

156

157 self._validate_logprobs(params)

158 - self._validate_sampling_params(params, lora_request)

158 + self._validate_sampling_params(params)

159 self._validate_supported_sampling_params(params)

160

161 def _validate_multi_modal_uuids(self, prompt: PromptType) -> None:

162 """

163 Validate that user-provided multi_modal_uuids align with

164 multi_modal_data in the incoming request prompt(s).

165 - Only checks lengths; `None` entries are allowed and will be

165 + Only checks lengths; `None` entries are allowed and will be

166 auto-hashed downstream.

167 """

168

@@ -202,10 +202,22 @@ class Processor:

202 _validate_single_prompt(prompt) # type: ignore[arg-type]

203

204 def _validate_lora(self, lora_request: Optional[LoRARequest]) -> None:

205 - if lora_request is not None and not self.lora_config:

205 + if lora_request is None:

206 + return

207 +

208 + # LoRA request passed in while LoRA is not enabled

209 + if not self.lora_config:

210 raise ValueError(f"Got lora_request {lora_request} but LoRA is "

211 "not enabled!")

212

213 + if self.tokenizer is not None:

214 + logger.warning_once(

215 + "vLLM has deprecated support for supporting different "

216 + "tokenizers for different LoRAs. By default, vLLM uses base "

217 + "model's tokenizer. If you are using a LoRA "

218 + "with its own tokenizer, consider specifying `--tokenizer "

219 + "[lora_path]` to use the LoRA tokenizer.")

220 +

221 def _validate_structured_output(self, params: SamplingParams) -> None:

222 if not params.guided_decoding or not self.decoding_config:

223 return

@@ -326,7 +338,7 @@ class Processor:

338

339 # TODO(woosuk): Support pooling models.

340 self._validate_lora(lora_request)

329 - self._validate_params(params, lora_request)

341 + self._validate_params(params)

342

343 data_parallel_size = self.vllm_config.parallel_config.data_parallel_size

344 if data_parallel_rank is not None and not (0 <= data_parallel_rank <

@@ -365,7 +377,6 @@ class Processor:

377 processed_inputs: ProcessorInputs = self.input_preprocessor.preprocess(

378 prompt,

379 tokenization_kwargs=tokenization_kwargs,

368 - lora_request=lora_request,

380 mm_uuids=mm_uuids,

381 )

382 from vllm.platforms import current_platform

@@ -375,9 +386,9 @@ class Processor:

386 processed_inputs=processed_inputs,

387 )

388

378 - eos_token_id = self.input_preprocessor.get_eos_token_id(lora_request)

389 + eos_token_id = self.input_preprocessor.get_eos_token_id()

390

380 - self._validate_model_inputs(processed_inputs, lora_request)

391 + self._validate_model_inputs(processed_inputs)

392

393 encoder_inputs, decoder_inputs = split_enc_dec_inputs(processed_inputs)

394

@@ -394,8 +405,7 @@ class Processor:

405 sampling_params.update_from_generation_config(

406 self.generation_config_fields, eos_token_id)

407 if self.tokenizer is not None:

397 - sampling_params.update_from_tokenizer(

398 - self.tokenizer.get_lora_tokenizer(lora_request))

408 + sampling_params.update_from_tokenizer(self.tokenizer)

409 else:

410 pooling_params = params.clone()

411

@@ -436,24 +446,17 @@ class Processor:

446 trace_headers=trace_headers,

447 )

448

439 - def _validate_model_inputs(self,

440 - inputs: ProcessorInputs,

441 - lora_request: Optional[LoRARequest] = None):

449 + def _validate_model_inputs(self, inputs: ProcessorInputs):

450 encoder_inputs, decoder_inputs = split_enc_dec_inputs(inputs)

451

452 if encoder_inputs is not None:

445 - self._validate_model_input(encoder_inputs,

446 - lora_request,

447 - prompt_type="encoder")

453 + self._validate_model_input(encoder_inputs, prompt_type="encoder")

454

449 - self._validate_model_input(decoder_inputs,

450 - lora_request,

451 - prompt_type="decoder")

455 + self._validate_model_input(decoder_inputs, prompt_type="decoder")

456

457 def _validate_model_input(

458 self,

459 prompt_inputs: SingletonInputs,

456 - lora_request: Optional[LoRARequest],

460 *,

461 prompt_type: Literal["encoder", "decoder"],

462 ):

@@ -469,7 +472,7 @@ class Processor:

472 if self.model_config.skip_tokenizer_init:

473 tokenizer = None

474 else:

472 - tokenizer = self.tokenizer.get_lora_tokenizer(lora_request)

475 + tokenizer = self.tokenizer

476 max_input_id = max(prompt_ids, default=0)

477

478 # NOTE: tokenizer.max_token_id is the tokenizer’s vocab size while

vllm/v1/structured_output/__init__.py

+2 -5

@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Optional

9 from vllm.config import VllmConfig

10 from vllm.logger import init_logger

11 from vllm.reasoning import ReasoningParserManager

12 -from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs

12 +from vllm.transformers_utils.tokenizer import init_tokenizer_from_configs

13 from vllm.utils import LazyLoader

14 from vllm.v1.structured_output.backend_guidance import GuidanceBackend

15 from vllm.v1.structured_output.backend_types import (StructuredOutputBackend,

@@ -60,10 +60,7 @@ class StructuredOutputManager:

60 max_workers = max(1, (multiprocessing.cpu_count() + 1) // 2)

61 self.executor = ThreadPoolExecutor(max_workers=max_workers)

62 self.tokenizer = init_tokenizer_from_configs(

63 - model_config=self.vllm_config.model_config,

64 - scheduler_config=self.vllm_config.scheduler_config,

65 - lora_config=self.vllm_config.lora_config,

66 - ).get_lora_tokenizer(None)

63 + model_config=self.vllm_config.model_config)

64 reasoning_backend = \

65 self.vllm_config.decoding_config.reasoning_backend

66 if reasoning_backend: