@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0a05ed57e6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 13:30 UTC

Simplify `TokenizerGroup` (#16790)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed Apr 24, 2025 at 12:43 UTC 0a05ed57e6865cea4c00acf060dd2e6c456c0a49
24 files changed +80 -752
tests/conftest.py
+1 -15
@@ -23,7 +23,7 @@ from tests.models.utils import (TokensTextLogprobs,
23 from vllm import LLM, SamplingParams
24 from vllm.assets.image import ImageAsset
25 from vllm.assets.video import VideoAsset
26 -from vllm.config import TaskOption, TokenizerPoolConfig, _get_and_verify_dtype
26 +from vllm.config import TaskOption, _get_and_verify_dtype
27 from vllm.connections import global_http_connection
28 from vllm.distributed import (cleanup_dist_env_and_memory,
29 init_distributed_environment,
@@ -1010,20 +1010,6 @@ def vllm_runner():
1010 return VllmRunner
1011
1012
1013 -def get_tokenizer_pool_config(tokenizer_group_type):
1014 - if tokenizer_group_type is None:
1015 - return None
1016 - if tokenizer_group_type == "ray":
1017 - return TokenizerPoolConfig(pool_size=1,
1018 - pool_type="ray",
1019 - extra_config={})
1020 - if isinstance(tokenizer_group_type, type):
1021 - return TokenizerPoolConfig(pool_size=1,
1022 - pool_type=tokenizer_group_type,
1023 - extra_config={})
1024 - raise ValueError(f"Unknown tokenizer_group_type: {tokenizer_group_type}")
1025 -
1026 -
1013 @pytest.fixture()
1014 def temporary_enable_log_propagate():
1015 import logging
tests/lora/test_tokenizer_group.py
+3 -7
@@ -5,17 +5,14 @@ from transformers import AutoTokenizer, PreTrainedTokenizerBase
5
6 from vllm.lora.request import LoRARequest
7 from vllm.transformers_utils.tokenizer import get_lora_tokenizer
8 -from vllm.transformers_utils.tokenizer_group import get_tokenizer_group
9 -
10 -from ..conftest import get_tokenizer_pool_config
8 +from vllm.transformers_utils.tokenizer_group import TokenizerGroup
9
10
11 @pytest.mark.asyncio
12 @pytest.mark.parametrize("tokenizer_group_type", [None, "ray"])
13 async def test_tokenizer_group_lora(sql_lora_files, tokenizer_group_type):
14 reference_tokenizer = AutoTokenizer.from_pretrained(sql_lora_files)
17 - tokenizer_group = get_tokenizer_group(
18 - get_tokenizer_pool_config(tokenizer_group_type),
15 + tokenizer_group = TokenizerGroup(
16 tokenizer_id="gpt2",
17 enable_lora=True,
18 max_num_seqs=1,
@@ -60,8 +57,7 @@ def test_get_lora_tokenizer(sql_lora_files, tmp_path):
57 @pytest.mark.parametrize("max_num_seqs", [1, 2])
58 @pytest.mark.parametrize("max_loras", [1, 2])
59 def test_lora_tokenizers(enable_lora, max_num_seqs, max_loras):
63 - tokenizer_group = get_tokenizer_group(
64 - get_tokenizer_pool_config(None),
60 + tokenizer_group = TokenizerGroup(
61 tokenizer_id="gpt2",
62 enable_lora=enable_lora,
63 max_num_seqs=max_num_seqs,
tests/tokenization/test_detokenize.py
+2 -7
@@ -10,7 +10,7 @@ from transformers import (AutoTokenizer, PreTrainedTokenizer,
10 from vllm.inputs import token_inputs
11 from vllm.sequence import Logprob, SamplingParams, Sequence, SequenceGroup
12 from vllm.transformers_utils.detokenizer import Detokenizer
13 -from vllm.transformers_utils.tokenizer_group import get_tokenizer_group
13 +from vllm.transformers_utils.tokenizer_group import TokenizerGroup
14 from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
15 from vllm.v1.engine import EngineCoreRequest
16 from vllm.v1.engine.detokenizer import (FastIncrementalDetokenizer,
@@ -212,7 +212,7 @@ def test_oov_decode(tokenizer, fast):
212
213 @pytest.fixture
214 def detokenizer(tokenizer_name: str) -> Detokenizer:
215 - init_kwargs = dict(
215 + tokenizer_group = TokenizerGroup(
216 tokenizer_id=tokenizer_name,
217 enable_lora=False,
218 max_num_seqs=100,
@@ -222,11 +222,6 @@ def detokenizer(tokenizer_name: str) -> Detokenizer:
222 revision=None,
223 )
224
225 - tokenizer_group = get_tokenizer_group(
226 - None,
227 - **init_kwargs,
228 - )
229 -
225 return Detokenizer(tokenizer_group)
226
227
tests/tokenization/test_tokenizer_group.py
+3 -184
@@ -1,40 +1,15 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 -import asyncio
4 -import os
5 -import sys
6 -from typing import Optional
7 -from unittest.mock import patch
8 -
3 import pytest
4 from transformers import AutoTokenizer, PreTrainedTokenizerBase
5
12 -from vllm.transformers_utils.tokenizer_group import (TokenizerGroup,
13 - get_tokenizer_group)
14 -from vllm.transformers_utils.tokenizer_group.ray_tokenizer_group import (
15 - RayTokenizerGroupPool)
16 -
17 -from ..conftest import get_tokenizer_pool_config
18 -
19 -
20 -class CustomTokenizerGroup(TokenizerGroup):
21 -
22 - def __init__(self, *args, **kwargs):
23 - super().__init__(*args, **kwargs)
24 - self._i = 0
25 -
26 - def encode(self, *args, **kwargs):
27 - self._i += 1
28 - return super().encode(*args, **kwargs)
6 +from vllm.transformers_utils.tokenizer_group import TokenizerGroup
7
8
9 @pytest.mark.asyncio
32 -@pytest.mark.parametrize("tokenizer_group_type",
33 - [None, "ray", CustomTokenizerGroup])
34 -async def test_tokenizer_group(tokenizer_group_type):
10 +async def test_tokenizer_group():
11 reference_tokenizer = AutoTokenizer.from_pretrained("gpt2")
36 - tokenizer_group = get_tokenizer_group(
37 - get_tokenizer_pool_config(tokenizer_group_type),
12 + tokenizer_group = TokenizerGroup(
13 tokenizer_id="gpt2",
14 enable_lora=False,
15 max_num_seqs=1,
@@ -49,159 +24,3 @@ async def test_tokenizer_group(tokenizer_group_type):
24 PreTrainedTokenizerBase)
25 assert tokenizer_group.get_lora_tokenizer(
26 None) == await tokenizer_group.get_lora_tokenizer_async(None)
52 - if tokenizer_group_type is CustomTokenizerGroup:
53 - assert tokenizer_group._i > 0
54 -
55 -
56 -@pytest.mark.asyncio
57 -@pytest.mark.parametrize("tokenizer_group_type", ["ray"])
58 -async def test_tokenizer_group_pool(tokenizer_group_type):
59 - reference_tokenizer = AutoTokenizer.from_pretrained("gpt2")
60 - tokenizer_group_pool = get_tokenizer_group(
61 - get_tokenizer_pool_config(tokenizer_group_type),
62 - tokenizer_id="gpt2",
63 - enable_lora=False,
64 - max_num_seqs=1,
65 - max_input_length=None,
66 - )
67 - # Send multiple requests to the tokenizer group pool
68 - # (more than the pool size)
69 - # and check that all requests are processed correctly.
70 - num_requests = tokenizer_group_pool.pool_size * 5
71 - requests = [
72 - tokenizer_group_pool.encode_async(prompt=f"prompt {i}",
73 - lora_request=None)
74 - for i in range(num_requests)
75 - ]
76 - results = await asyncio.gather(*requests)
77 - expected_results = [
78 - reference_tokenizer.encode(f"prompt {i}") for i in range(num_requests)
79 - ]
80 - assert results == expected_results
81 -
82 -
83 -@pytest.mark.asyncio
84 -@pytest.mark.parametrize("tokenizer_group_type", ["ray"])
85 -async def test_tokenizer_group_ray_pool_env_var_propagation(
86 - tokenizer_group_type):
87 - """Test that env vars from caller process are propagated to
88 - tokenizer Ray actors."""
89 - env_var = "MY_ENV_VAR"
90 -
91 - class EnvVarCheckerTokenizerGroup(TokenizerGroup):
92 -
93 - def ping(self):
94 - assert os.environ.get(env_var) == "1"
95 - return super().ping()
96 -
97 - class EnvVarCheckerRayTokenizerGroupPool(RayTokenizerGroupPool):
98 - _worker_cls = EnvVarCheckerTokenizerGroup
99 -
100 - tokenizer_pool_config = get_tokenizer_pool_config(tokenizer_group_type)
101 - tokenizer_pool = EnvVarCheckerRayTokenizerGroupPool.from_config(
102 - tokenizer_pool_config,
103 - tokenizer_id="gpt2",
104 - enable_lora=False,
105 - max_num_seqs=1,
106 - max_input_length=None)
107 - with pytest.raises(AssertionError):
108 - tokenizer_pool.ping()
109 -
110 - with patch.dict(os.environ, {env_var: "1"}):
111 - tokenizer_pool_config = get_tokenizer_pool_config(tokenizer_group_type)
112 - tokenizer_pool = EnvVarCheckerRayTokenizerGroupPool.from_config(
113 - tokenizer_pool_config,
114 - tokenizer_id="gpt2",
115 - enable_lora=False,
116 - max_num_seqs=1,
117 - max_input_length=None)
118 - tokenizer_pool.ping()
119 -
120 -
121 -@pytest.mark.asyncio
122 -@pytest.mark.parametrize("tokenizer_group_type", ["ray"])
123 -async def test_tokenizer_group_ray_pool_fault_tolerance(tokenizer_group_type):
124 - """Test that Ray tokenizer pool group can recover from failures and
125 - if that's not possible, mark itself as unhealthy."""
126 -
127 - class FailingTokenizerGroup(TokenizerGroup):
128 -
129 - def __init__(self,
130 - *args,
131 - fail_at: Optional[list[int]] = None,
132 - **kwargs):
133 - super().__init__(*args, **kwargs)
134 - self.i = 0
135 - self.fail_at = fail_at or []
136 -
137 - def encode(self, *args, **kwargs):
138 - self.i += 1
139 - if self.i in self.fail_at:
140 - sys.exit(1)
141 - return super().encode(*args, **kwargs)
142 -
143 - class FailingRayTokenizerGroupPool(RayTokenizerGroupPool):
144 - _worker_cls = FailingTokenizerGroup
145 -
146 - # Fail at first iteration
147 - fail_at = [1]
148 - tokenizer_pool_config = get_tokenizer_pool_config(tokenizer_group_type)
149 - tokenizer_group_pool = FailingRayTokenizerGroupPool.from_config(
150 - tokenizer_pool_config,
151 - tokenizer_id="gpt2",
152 - enable_lora=False,
153 - max_num_seqs=1,
154 - max_input_length=None,
155 - fail_at=fail_at)
156 - tokenizer_actors = tokenizer_group_pool.tokenizer_actors.copy()
157 -
158 - # Modify fail at to not fail at all (will be re-read when actor is
159 - # re-initialized).
160 - fail_at[0] = 1000
161 -
162 - # We should recover successfully.
163 - await tokenizer_group_pool.encode_async(prompt="prompt", lora_request=None)
164 - await tokenizer_group_pool.encode_async(prompt="prompt", lora_request=None)
165 -
166 - # Check that we have a new actor
167 - assert len(tokenizer_group_pool.tokenizer_actors) == len(tokenizer_actors)
168 - assert tokenizer_group_pool.tokenizer_actors != tokenizer_actors
169 -
170 - # Fail at first iteration
171 - fail_at = [1]
172 - tokenizer_group_pool = FailingRayTokenizerGroupPool.from_config(
173 - tokenizer_pool_config,
174 - tokenizer_id="gpt2",
175 - enable_lora=False,
176 - max_num_seqs=1,
177 - max_input_length=None,
178 - fail_at=fail_at)
179 -
180 - # We should fail after re-initialization.
181 - with pytest.raises(RuntimeError):
182 - await tokenizer_group_pool.encode_async(prompt="prompt",
183 - lora_request=None)
184 -
185 - # check_health should raise the same thing
186 - with pytest.raises(RuntimeError):
187 - tokenizer_group_pool.check_health()
188 -
189 - # Ensure that non-ActorDiedErrors are still propagated correctly and do not
190 - # cause a re-initialization.
191 - fail_at = []
192 - tokenizer_group_pool = FailingRayTokenizerGroupPool.from_config(
193 - tokenizer_pool_config,
194 - tokenizer_id="gpt2",
195 - enable_lora=False,
196 - max_num_seqs=1,
197 - max_input_length=2,
198 - fail_at=fail_at)
199 - tokenizer_actors = tokenizer_group_pool.tokenizer_actors.copy()
200 -
201 - # Prompt too long error
202 - with pytest.raises(ValueError):
203 - await tokenizer_group_pool.encode_async(prompt="prompt" * 100,
204 - lora_request=None)
205 - await tokenizer_group_pool.encode_async(prompt="prompt", lora_request=None)
206 - # Actors should stay the same.
207 - assert tokenizer_group_pool.tokenizer_actors == tokenizer_actors
tests/v1/engine/conftest.py
+1 -1
@@ -47,7 +47,7 @@ def _build_test_vectors_no_logprobs() -> DummyOutputProcessorTestVectors:
47 tokenizer=tokenizer,
48 tokenizer_group=init_tokenizer_from_configs(
49 vllm_config.model_config, vllm_config.scheduler_config,
50 - vllm_config.parallel_config, vllm_config.lora_config),
50 + vllm_config.lora_config),
51 vllm_config=vllm_config,
52 full_tokens=[tokenizer(text).input_ids for text in FULL_STRINGS],
53 prompt_tokens=prompt_tokens,
tests/v1/engine/utils.py
+2 -3
@@ -8,8 +8,7 @@ import torch
8 from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
9
10 from vllm.engine.arg_utils import EngineArgs
11 -from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
12 - BaseTokenizerGroup)
11 +from vllm.transformers_utils.tokenizer_group import TokenizerGroup
12 from vllm.v1.engine import EngineCoreOutput, FinishReason
13 from vllm.v1.outputs import LogprobsLists, LogprobsTensors
14
@@ -296,7 +295,7 @@ def generate_dummy_prompt_logprobs_tensors(
295 class DummyOutputProcessorTestVectors:
296 """Dummy test vectors for output processor tests"""
297 tokenizer: GeneralTokenizerType
299 - tokenizer_group: BaseTokenizerGroup
298 + tokenizer_group: TokenizerGroup
299 vllm_config: EngineArgs
300 full_tokens: list[list[int]] # Prompt + generated tokens
301 prompt_tokens: list[list[int]]
vllm/config.py
+22 -75
@@ -52,8 +52,6 @@ if TYPE_CHECKING:
52 from vllm.model_executor.layers.quantization.base_config import (
53 QuantizationConfig)
54 from vllm.model_executor.model_loader.loader import BaseModelLoader
55 - from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
56 - BaseTokenizerGroup)
55
56 ConfigType = type[DataclassInstance]
57 else:
@@ -1407,83 +1405,33 @@ class CacheConfig:
1405 logger.warning("Possibly too large swap space. %s", msg)
1406
1407
1410 -PoolType = Literal["ray"]
1411 -
1412 -
1408 @config
1409 @dataclass
1410 class TokenizerPoolConfig:
1416 - """Configuration for the tokenizer pool."""
1411 + """This config is deprecated and will be removed in a future release.
1412
1418 - pool_size: int = 0
1419 - """Number of tokenizer workers in the pool to use for asynchronous
1420 - tokenization. If 0, will use synchronous tokenization."""
1421 -
1422 - pool_type: Union[PoolType, type["BaseTokenizerGroup"]] = "ray"
1423 - """Type of tokenizer pool to use for asynchronous tokenization. Ignored if
1424 - tokenizer_pool_size is 0."""
1413 + Passing these parameters will have no effect. Please remove them from your
1414 + configurations.
1415 + """
1416
1417 + pool_size: int = 0
1418 + """This parameter is deprecated and will be removed in a future release.
1419 + Passing this parameter will have no effect. Please remove it from your
1420 + configurations."""
1421 + pool_type: str = "ray"
1422 + """This parameter is deprecated and will be removed in a future release.
1423 + Passing this parameter will have no effect. Please remove it from your
1424 + configurations."""
1425 extra_config: dict = field(default_factory=dict)
1427 - """Additional config for the pool. The way the config will be used depends
1428 - on the pool type. This should be a JSON string that will be parsed into a
1429 - dictionary. Ignored if tokenizer_pool_size is 0."""
1430 -
1431 - def compute_hash(self) -> str:
1432 - """
1433 - WARNING: Whenever a new field is added to this config,
1434 - ensure that it is included in the factors list if
1435 - it affects the computation graph.
1426 + """This parameter is deprecated and will be removed in a future release.
1427 + Passing this parameter will have no effect. Please remove it from your
1428 + configurations."""
1429
1437 - Provide a hash that uniquely identifies all the configs
1438 - that affect the structure of the computation
1439 - graph from input ids/embeddings to the final hidden states,
1440 - excluding anything before input ids/embeddings and after
1441 - the final hidden states.
1442 - """
1443 - # no factors to consider.
1444 - # this config will not affect the computation graph.
1445 - factors: list[Any] = []
1446 - hash_str = hashlib.md5(str(factors).encode(),
1447 - usedforsecurity=False).hexdigest()
1448 - return hash_str
1449 -
1450 - def __post_init__(self):
1451 - if self.pool_type not in ("ray", ) and not isinstance(
1452 - self.pool_type, type):
1453 - raise ValueError(f"Unknown pool type: {self.pool_type}")
1454 - if not isinstance(self.extra_config, dict):
1455 - raise ValueError("extra_config must be a dictionary.")
1456 -
1457 - @classmethod
1458 - def create_config(
1459 - cls, tokenizer_pool_size: int,
1460 - tokenizer_pool_type: Union[PoolType, type["BaseTokenizerGroup"]],
1461 - tokenizer_pool_extra_config: Optional[Union[str, dict]]
1462 - ) -> Optional["TokenizerPoolConfig"]:
1463 - """Create a TokenizerPoolConfig from the given parameters.
1464 -
1465 - If tokenizer_pool_size is 0, return None.
1466 -
1467 - Args:
1468 - tokenizer_pool_size: Number of tokenizer workers in the pool.
1469 - tokenizer_pool_type: Type of the pool.
1470 - tokenizer_pool_extra_config: Additional config for the pool.
1471 - The way the config will be used depends on the
1472 - pool type. This can be a JSON string (will be parsed).
1473 - """
1474 - if tokenizer_pool_size:
1475 - if isinstance(tokenizer_pool_extra_config, str):
1476 - tokenizer_pool_extra_config_parsed = json.loads(
1477 - tokenizer_pool_extra_config)
1478 - else:
1479 - tokenizer_pool_extra_config_parsed = (
1480 - tokenizer_pool_extra_config or {})
1481 - tokenizer_pool_config = cls(tokenizer_pool_size,
1482 - tokenizer_pool_type,
1483 - tokenizer_pool_extra_config_parsed)
1484 - else:
1485 - tokenizer_pool_config = None
1486 - return tokenizer_pool_config
1430 + def __post_init__(self) -> None:
1431 + logger.warning_once(
1432 + "TokenizerPoolConfig is deprecated and will be removed in a "
1433 + "future release. Passing this parameter will have no effect. "
1434 + "Please remove it from your configurations.")
1435
1436
1437 class LoadFormat(str, enum.Enum):
@@ -1624,8 +1572,8 @@ class ParallelConfig:
1572 """Disable the custom all-reduce kernel and fall back to NCCL."""
1573
1574 tokenizer_pool_config: Optional[TokenizerPoolConfig] = None
1627 - """Config for the tokenizer pool. If None, will use synchronous
1628 - tokenization."""
1575 + """This parameter is deprecated and will be removed in a future release.
1576 + Please remove it from your configs"""
1577
1578 ray_workers_use_nsight: bool = False
1579 """Whether to profile Ray workers with nsight, see https://docs.ray.io/en/latest/ray-observability/user-guides/profiling.html#profiling-nsight-profiler."""
@@ -2544,7 +2492,6 @@ class SpeculativeConfig:
2492 max_parallel_loading_workers,
2493 disable_custom_all_reduce=target_parallel_config.
2494 disable_custom_all_reduce,
2547 - tokenizer_pool_config=target_parallel_config.tokenizer_pool_config,
2495 ray_workers_use_nsight=target_parallel_config.
2496 ray_workers_use_nsight,
2497 placement_group=target_parallel_config.placement_group,
vllm/engine/arg_utils.py
+8 -18
@@ -7,9 +7,8 @@ import json
7 import re
8 import threading
9 from dataclasses import MISSING, dataclass, fields
10 -from typing import (TYPE_CHECKING, Any, Callable, Dict, List, Literal,
11 - Optional, Tuple, Type, TypeVar, Union, cast, get_args,
12 - get_origin)
10 +from typing import (Any, Callable, Dict, List, Literal, Optional, Tuple, Type,
11 + TypeVar, Union, cast, get_args, get_origin)
12
13 import torch
14 from typing_extensions import TypeIs
@@ -23,7 +22,7 @@ from vllm.config import (BlockSize, CacheConfig, CacheDType, CompilationConfig,
22 KVTransferConfig, LoadConfig, LoadFormat, LoRAConfig,
23 ModelConfig, ModelImpl, MultiModalConfig,
24 ObservabilityConfig, ParallelConfig, PoolerConfig,
26 - PoolType, PrefixCachingHashAlgo, PromptAdapterConfig,
25 + PrefixCachingHashAlgo, PromptAdapterConfig,
26 SchedulerConfig, SchedulerPolicy, SpeculativeConfig,
27 TaskOption, TokenizerPoolConfig, VllmConfig,
28 get_attr_docs, get_field)
@@ -39,9 +38,6 @@ from vllm.utils import FlexibleArgumentParser, GiB_bytes, is_in_ray_actor
38
39 # yapf: enable
40
42 -if TYPE_CHECKING:
43 - from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
44 -
41 logger = init_logger(__name__)
42
43 ALLOWED_DETAILED_TRACE_MODULES = ["model", "worker", "all"]
@@ -185,13 +181,12 @@ class EngineArgs:
181 enforce_eager: Optional[bool] = None
182 max_seq_len_to_capture: int = 8192
183 disable_custom_all_reduce: bool = ParallelConfig.disable_custom_all_reduce
184 + # The following three fields are deprecated and will be removed in a future
185 + # release. Setting them will have no effect. Please remove them from your
186 + # configurations.
187 tokenizer_pool_size: int = TokenizerPoolConfig.pool_size
189 - # Note: Specifying a tokenizer pool by passing a class
190 - # is intended for expert use only. The API may change without
191 - # notice.
192 - tokenizer_pool_type: Union[PoolType, Type["BaseTokenizerGroup"]] = \
193 - TokenizerPoolConfig.pool_type
194 - tokenizer_pool_extra_config: dict[str, Any] = \
188 + tokenizer_pool_type: str = TokenizerPoolConfig.pool_type
189 + tokenizer_pool_extra_config: dict = \
190 get_field(TokenizerPoolConfig, "extra_config")
191 limit_mm_per_prompt: dict[str, int] = \
192 get_field(MultiModalConfig, "limit_per_prompt")
@@ -1187,11 +1182,6 @@ class EngineArgs:
1182 enable_expert_parallel=self.enable_expert_parallel,
1183 max_parallel_loading_workers=self.max_parallel_loading_workers,
1184 disable_custom_all_reduce=self.disable_custom_all_reduce,
1190 - tokenizer_pool_config=TokenizerPoolConfig.create_config(
1191 - self.tokenizer_pool_size,
1192 - self.tokenizer_pool_type,
1193 - self.tokenizer_pool_extra_config,
1194 - ),
1185 ray_workers_use_nsight=self.ray_workers_use_nsight,
1186 placement_group=placement_group,
1187 distributed_executor_backend=self.distributed_executor_backend,
vllm/engine/async_llm_engine.py
-2
@@ -526,8 +526,6 @@ class _AsyncLLMEngine(LLMEngine):
526 )
527
528 async def check_health_async(self) -> None:
529 - if self.tokenizer:
530 - self.tokenizer.check_health()
529 self.model_executor.check_health()
530
531
vllm/engine/llm_engine.py
+6 -24
@@ -55,7 +55,7 @@ from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context,
55 from vllm.transformers_utils.detokenizer import Detokenizer
56 from vllm.transformers_utils.tokenizer import AnyTokenizer
57 from vllm.transformers_utils.tokenizer_group import (
58 - BaseTokenizerGroup, init_tokenizer_from_configs)
58 + TokenizerGroup, init_tokenizer_from_configs)
59 from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled,
60 usage_message)
61 from vllm.utils import (Counter, Device, deprecate_kwargs,
@@ -66,7 +66,6 @@ from vllm.worker.model_runner_base import InputProcessingError
66 logger = init_logger(__name__)
67 _LOCAL_LOGGING_INTERVAL_SEC = 5
68
69 -_G = TypeVar("_G", bound=BaseTokenizerGroup, default=BaseTokenizerGroup)
69 _O = TypeVar("_O", RequestOutput, PoolingRequestOutput)
70 _R = TypeVar("_R", default=Any)
71
@@ -205,7 +204,7 @@ class LLMEngine:
204
205 return outputs_
206
208 - tokenizer: Optional[BaseTokenizerGroup]
207 + tokenizer: Optional[TokenizerGroup]
208
209 def __init__(
210 self,
@@ -321,11 +320,6 @@ class LLMEngine:
320 self.parallel_config.disable_custom_all_reduce,
321 })
322
324 - if self.tokenizer:
325 - # Ping the tokenizer to ensure liveness if it runs in a
326 - # different process.
327 - self.tokenizer.ping()
328 -
323 self.cached_scheduler_outputs = [
324 SchedulerOutputState()
325 for _ in range(self.parallel_config.pipeline_parallel_size)
@@ -537,21 +531,12 @@ class LLMEngine:
531 if model_executor := getattr(self, "model_executor", None):
532 model_executor.shutdown()
533
540 - def get_tokenizer_group(
541 - self,
542 - group_type: Type[_G] = BaseTokenizerGroup,
543 - ) -> _G:
544 - tokenizer_group = self.tokenizer
545 -
546 - if tokenizer_group is None:
534 + def get_tokenizer_group(self) -> TokenizerGroup:
535 + if self.tokenizer is None:
536 raise ValueError("Unable to get tokenizer because "
537 "skip_tokenizer_init is True")
549 - if not isinstance(tokenizer_group, group_type):
550 - raise TypeError("Invalid type of tokenizer group. "
551 - f"Expected type: {group_type}, but "
552 - f"found type: {type(tokenizer_group)}")
538
554 - return tokenizer_group
539 + return self.tokenizer
540
541 def get_tokenizer(
542 self,
@@ -559,11 +544,10 @@ class LLMEngine:
544 ) -> AnyTokenizer:
545 return self.get_tokenizer_group().get_lora_tokenizer(lora_request)
546
562 - def _init_tokenizer(self) -> BaseTokenizerGroup:
547 + def _init_tokenizer(self) -> TokenizerGroup:
548 return init_tokenizer_from_configs(
549 model_config=self.model_config,
550 scheduler_config=self.scheduler_config,
566 - parallel_config=self.parallel_config,
551 lora_config=self.lora_config)
552
553 def _verify_args(self) -> None:
@@ -1952,8 +1936,6 @@ class LLMEngine:
1936 return self.model_executor.is_sleeping
1937
1938 def check_health(self) -> None:
1955 - if self.tokenizer:
1956 - self.tokenizer.check_health()
1939 self.model_executor.check_health()
1940
1941 def is_tracing_enabled(self) -> bool:
vllm/engine/multiprocessing/client.py
-1
@@ -101,7 +101,6 @@ class MQLLMEngineClient(EngineClient):
101 self.tokenizer = init_tokenizer_from_configs(
102 model_config=self.model_config,
103 scheduler_config=engine_config.scheduler_config,
104 - parallel_config=engine_config.parallel_config,
104 lora_config=engine_config.lora_config)
105 self.input_preprocessor = InputPreprocessor(self.model_config,
106 self.tokenizer)
vllm/entrypoints/llm.py
+2 -3
@@ -40,7 +40,6 @@ from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
40 RequestOutputKind, SamplingParams)
41 from vllm.transformers_utils.tokenizer import (AnyTokenizer, MistralTokenizer,
42 get_cached_tokenizer)
43 -from vllm.transformers_utils.tokenizer_group import TokenizerGroup
43 from vllm.usage.usage_lib import UsageContext
44 from vllm.utils import (Counter, Device, deprecate_args, deprecate_kwargs,
45 is_list_of)
@@ -253,10 +252,10 @@ class LLM:
252 self.default_sampling_params: Union[dict[str, Any], None] = None
253
254 def get_tokenizer(self) -> AnyTokenizer:
256 - return self.llm_engine.get_tokenizer_group(TokenizerGroup).tokenizer
255 + return self.llm_engine.get_tokenizer_group().tokenizer
256
257 def set_tokenizer(self, tokenizer: AnyTokenizer) -> None:
259 - tokenizer_group = self.llm_engine.get_tokenizer_group(TokenizerGroup)
258 + tokenizer_group = self.llm_engine.get_tokenizer_group()
259
260 # While CachedTokenizer is dynamic, have no choice but
261 # compare class name. Misjudgment will arise from
vllm/inputs/preprocess.py
+3 -3
@@ -13,7 +13,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
13 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalEncDecInputs,
14 MultiModalInputs)
15 from vllm.prompt_adapter.request import PromptAdapterRequest
16 -from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
16 +from vllm.transformers_utils.tokenizer_group import TokenizerGroup
17
18 from .data import (DecoderOnlyInputs, EncoderDecoderInputs, ProcessorInputs,
19 PromptType, SingletonInputs, SingletonPrompt, token_inputs)
@@ -27,7 +27,7 @@ class InputPreprocessor:
27 def __init__(
28 self,
29 model_config: ModelConfig,
30 - tokenizer: Optional[BaseTokenizerGroup],
30 + tokenizer: Optional[TokenizerGroup],
31 mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
32 ) -> None:
33 super().__init__()
@@ -36,7 +36,7 @@ class InputPreprocessor:
36 self.tokenizer = tokenizer
37 self.mm_registry = mm_registry
38
39 - def get_tokenizer_group(self) -> BaseTokenizerGroup:
39 + def get_tokenizer_group(self) -> TokenizerGroup:
40 if self.tokenizer is None:
41 raise ValueError("You cannot pass text prompts when "
42 "`skip_tokenizer_init` is True")
vllm/transformers_utils/detokenizer.py
+2 -2
@@ -8,13 +8,13 @@ from vllm.sequence import (VLLM_INVALID_TOKEN_ID, Logprob, SamplingParams,
8 from .detokenizer_utils import (convert_prompt_ids_to_tokens,
9 detokenize_incrementally)
10 from .tokenizer import AnyTokenizer
11 -from .tokenizer_group import BaseTokenizerGroup
11 +from .tokenizer_group import TokenizerGroup
12
13
14 class Detokenizer:
15 """Provides methods to decode the output of a model into text."""
16
17 - def __init__(self, tokenizer_group: BaseTokenizerGroup):
17 + def __init__(self, tokenizer_group: TokenizerGroup):
18 self.tokenizer_group = tokenizer_group
19
20 def get_tokenizer_for_seq(self, sequence: Sequence) -> AnyTokenizer:
vllm/transformers_utils/tokenizer_group.py renamed
+17 -13
@@ -2,7 +2,7 @@
2
3 from typing import List, Optional
4
5 -from vllm.config import TokenizerPoolConfig
5 +from vllm.config import LoRAConfig, ModelConfig, SchedulerConfig
6 from vllm.lora.request import LoRARequest
7 from vllm.transformers_utils.tokenizer import (AnyTokenizer, encode_tokens,
8 get_lora_tokenizer,
@@ -10,10 +10,8 @@ from vllm.transformers_utils.tokenizer import (AnyTokenizer, encode_tokens,
10 get_tokenizer)
11 from vllm.utils import LRUCache
12
13 -from .base_tokenizer_group import BaseTokenizerGroup
13
15 -
16 -class TokenizerGroup(BaseTokenizerGroup):
14 +class TokenizerGroup:
15 """A group of tokenizers that can be used for LoRA adapters."""
16
17 def __init__(self, tokenizer_id: str, enable_lora: bool, max_num_seqs: int,
@@ -27,15 +25,6 @@ class TokenizerGroup(BaseTokenizerGroup):
25 self.lora_tokenizers = LRUCache[int, AnyTokenizer](
26 capacity=max(max_loras, max_num_seqs) if enable_lora else 0)
27
30 - @classmethod
31 - def from_config(cls, tokenizer_pool_config: Optional[TokenizerPoolConfig],
32 - **init_kwargs) -> "TokenizerGroup":
33 - return cls(**init_kwargs)
34 -
35 - def ping(self) -> bool:
36 - """Check if the tokenizer group is alive."""
37 - return True
38 -
28 def get_max_input_len(self,
29 lora_request: Optional[LoRARequest] = None
30 ) -> Optional[int]:
@@ -104,3 +93,18 @@ class TokenizerGroup(BaseTokenizerGroup):
93 return tokenizer
94 else:
95 return self.lora_tokenizers[lora_request.lora_int_id]
96 +
97 +
98 +def init_tokenizer_from_configs(model_config: ModelConfig,
99 + scheduler_config: SchedulerConfig,
100 + lora_config: Optional[LoRAConfig]):
101 + return TokenizerGroup(
102 + tokenizer_id=model_config.tokenizer,
103 + enable_lora=bool(lora_config),
104 + max_num_seqs=scheduler_config.max_num_seqs,
105 + max_loras=lora_config.max_loras if lora_config else 0,
106 + max_input_length=None,
107 + tokenizer_mode=model_config.tokenizer_mode,
108 + trust_remote_code=model_config.trust_remote_code,
109 + revision=model_config.tokenizer_revision,
110 + truncation_side=model_config.truncation_side)
vllm/transformers_utils/tokenizer_group/__init__.py deleted
-56
@@ -1,56 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -
3 -from typing import Optional, Type
4 -
5 -from vllm.config import (LoRAConfig, ModelConfig, ParallelConfig,
6 - SchedulerConfig, TokenizerPoolConfig)
7 -from vllm.executor.ray_utils import ray
8 -
9 -from .base_tokenizer_group import AnyTokenizer, BaseTokenizerGroup
10 -from .tokenizer_group import TokenizerGroup
11 -
12 -if ray:
13 - from .ray_tokenizer_group import RayTokenizerGroupPool
14 -else:
15 - RayTokenizerGroupPool = None # type: ignore
16 -
17 -
18 -def init_tokenizer_from_configs(model_config: ModelConfig,
19 - scheduler_config: SchedulerConfig,
20 - parallel_config: ParallelConfig,
21 - lora_config: Optional[LoRAConfig]):
22 - init_kwargs = dict(tokenizer_id=model_config.tokenizer,
23 - enable_lora=bool(lora_config),
24 - max_num_seqs=scheduler_config.max_num_seqs,
25 - max_loras=lora_config.max_loras if lora_config else 0,
26 - max_input_length=None,
27 - tokenizer_mode=model_config.tokenizer_mode,
28 - trust_remote_code=model_config.trust_remote_code,
29 - revision=model_config.tokenizer_revision,
30 - truncation_side=model_config.truncation_side)
31 -
32 - return get_tokenizer_group(parallel_config.tokenizer_pool_config,
33 - **init_kwargs)
34 -
35 -
36 -def get_tokenizer_group(tokenizer_pool_config: Optional[TokenizerPoolConfig],
37 - **init_kwargs) -> BaseTokenizerGroup:
38 - tokenizer_cls: Type[BaseTokenizerGroup]
39 - if tokenizer_pool_config is None:
40 - tokenizer_cls = TokenizerGroup
41 - elif isinstance(tokenizer_pool_config.pool_type, type) and issubclass(
42 - tokenizer_pool_config.pool_type, BaseTokenizerGroup):
43 - tokenizer_cls = tokenizer_pool_config.pool_type
44 - elif tokenizer_pool_config.pool_type == "ray":
45 - if RayTokenizerGroupPool is None:
46 - raise ImportError(
47 - "RayTokenizerGroupPool is not available. Please install "
48 - "the ray package to use the Ray tokenizer group pool.")
49 - tokenizer_cls = RayTokenizerGroupPool
50 - else:
51 - raise ValueError(
52 - f"Unknown pool type: {tokenizer_pool_config.pool_type}")
53 - return tokenizer_cls.from_config(tokenizer_pool_config, **init_kwargs)
54 -
55 -
56 -__all__ = ["AnyTokenizer", "get_tokenizer_group", "BaseTokenizerGroup"]
vllm/transformers_utils/tokenizer_group/base_tokenizer_group.py deleted
-68
@@ -1,68 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -
3 -from abc import ABC, abstractmethod
4 -from typing import List, Optional
5 -
6 -from vllm.config import TokenizerPoolConfig
7 -from vllm.lora.request import LoRARequest
8 -from vllm.transformers_utils.tokenizer import AnyTokenizer
9 -
10 -
11 -class BaseTokenizerGroup(ABC):
12 - """A group of tokenizers that can be used for LoRA adapters."""
13 -
14 - @classmethod
15 - @abstractmethod
16 - def from_config(cls, tokenizer_pool_config: Optional[TokenizerPoolConfig],
17 - **init_kwargs) -> "BaseTokenizerGroup":
18 - pass
19 -
20 - @abstractmethod
21 - def ping(self) -> bool:
22 - """Check if the tokenizer group is alive."""
23 - pass
24 -
25 - @abstractmethod
26 - def get_max_input_len(
27 - self,
28 - lora_request: Optional[LoRARequest] = None,
29 - ) -> Optional[int]:
30 - """Get the maximum input length for the LoRA request."""
31 - pass
32 -
33 - @abstractmethod
34 - def encode(self,
35 - prompt: str,
36 - lora_request: Optional[LoRARequest] = None,
37 - add_special_tokens: Optional[bool] = None) -> List[int]:
38 - """Encode a prompt using the tokenizer group."""
39 - pass
40 -
41 - @abstractmethod
42 - async def encode_async(
43 - self,
44 - prompt: str,
45 - lora_request: Optional[LoRARequest] = None,
46 - add_special_tokens: Optional[bool] = None) -> List[int]:
47 - """Encode a prompt using the tokenizer group."""
48 - pass
49 -
50 - @abstractmethod
51 - def get_lora_tokenizer(
52 - self,
53 - lora_request: Optional[LoRARequest] = None,
54 - ) -> AnyTokenizer:
55 - """Get a tokenizer for a LoRA request."""
56 - pass
57 -
58 - @abstractmethod
59 - async def get_lora_tokenizer_async(
60 - self,
61 - lora_request: Optional[LoRARequest] = None,
62 - ) -> AnyTokenizer:
63 - """Get a tokenizer for a LoRA request."""
64 - pass
65 -
66 - def check_health(self):
67 - """Raise exception if the tokenizer group is unhealthy."""
68 - return
vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py deleted
-244
@@ -1,244 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -
3 -import asyncio
4 -import os
5 -from typing import List, Optional
6 -
7 -try:
8 - from ray.exceptions import ActorDiedError # type: ignore
9 -except ImportError:
10 - # For older versions of Ray
11 - from ray.exceptions import RayActorError as ActorDiedError # type: ignore
12 -from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy
13 -
14 -from vllm.config import TokenizerPoolConfig
15 -from vllm.executor.ray_utils import ray
16 -from vllm.logger import init_logger
17 -from vllm.lora.request import LoRARequest
18 -from vllm.transformers_utils.tokenizer import AnyTokenizer
19 -
20 -from .base_tokenizer_group import BaseTokenizerGroup
21 -from .tokenizer_group import TokenizerGroup
22 -
23 -logger = init_logger(__name__)
24 -
25 -
26 -class RayTokenizerGroupPool(BaseTokenizerGroup):
27 - """A Ray-based pool of TokenizerGroups for async tokenization."""
28 -
29 - # Class to use for workers making up the pool.
30 - _worker_cls = TokenizerGroup
31 -
32 - @classmethod
33 - def from_config(cls, tokenizer_pool_config: Optional[TokenizerPoolConfig],
34 - **init_kwargs) -> "RayTokenizerGroupPool":
35 - if not tokenizer_pool_config:
36 - raise ValueError("tokenizer_pool_config must not be None.")
37 - ray_actor_options = (tokenizer_pool_config.extra_config or {
38 - "num_cpus": 0
39 - })
40 - ray_actor_options.setdefault(
41 - "scheduling_strategy",
42 - NodeAffinitySchedulingStrategy(
43 - node_id=ray.get_runtime_context().get_node_id(), soft=True))
44 -
45 - # Carry over the env vars to the actors.
46 - # This is necessary for API keys and such.
47 - ray_actor_options.setdefault("runtime_env", {})
48 - _carry_over_env_vars_to_runtime_env(ray_actor_options["runtime_env"])
49 -
50 - init_kwargs["num_actors"] = tokenizer_pool_config.pool_size
51 - init_kwargs["ray_actor_options"] = ray_actor_options
52 -
53 - return cls(**init_kwargs)
54 -
55 - def __init__(self, tokenizer_id: str, enable_lora: bool, max_num_seqs: int,
56 - max_input_length: Optional[int], num_actors: int,
57 - ray_actor_options: dict, **tokenizer_config):
58 - # Store a local copy of the TokenizerGroup for quick access
59 - # to underlying HF tokenizers.
60 - self._tokenizer_config = {
61 - "tokenizer_id": tokenizer_id,
62 - "enable_lora": enable_lora,
63 - "max_num_seqs": max_num_seqs,
64 - "max_input_length": max_input_length,
65 - **tokenizer_config
66 - }
67 - self._local_tokenizer_group = self._worker_cls(
68 - **self._tokenizer_config, )
69 -
70 - self._ray_tokenizer_group_cls = ray.remote(
71 - self._worker_cls).options(**ray_actor_options) # type: ignore
72 - self.tokenizer_actors = [self._init_actor() for _ in range(num_actors)]
73 - self._idle_actors: Optional[asyncio.Queue] = None
74 -
75 - # If set, actor is unhealthy. Will reraise on the next
76 - # check_health call.
77 - self._exception: Optional[ActorDiedError] = None
78 -
79 - def _init_actor(self) -> ray.ObjectRef:
80 - return self._ray_tokenizer_group_cls.remote(**self._tokenizer_config)
81 -
82 - @property
83 - def pool_size(self) -> int:
84 - return len(self.tokenizer_actors)
85 -
86 - def ping(self):
87 - return ray.get([
88 - actor.ping.remote() # type: ignore
89 - for actor in self.tokenizer_actors
90 - ])
91 -
92 - def _ensure_queue_initialized(self):
93 - if self._idle_actors is None:
94 - self._idle_actors = asyncio.Queue()
95 - for actor in self.tokenizer_actors:
96 - self._idle_actors.put_nowait(actor)
97 -
98 - def _finalize_encode(self, actor: ray.ObjectRef,
99 - original_actor: ray.ObjectRef, actor_is_alive: bool):
100 - assert self._idle_actors is not None
101 - # Cleanup the dead actor.
102 - if not actor_is_alive or original_actor is not actor:
103 - self.tokenizer_actors.remove(original_actor)
104 - if actor_is_alive:
105 - # Put the actor back in the queue.
106 - # This is done in a finally block to ensure that the actor is
107 - # always put back in the queue, even if an exception/cancellation
108 - # is raised.
109 - self._idle_actors.put_nowait(actor)
110 - # Add back the new actor.
111 - if original_actor is not actor:
112 - self.tokenizer_actors.append(actor)
113 -
114 - def encode(self,
115 - prompt: str,
116 - lora_request: Optional[LoRARequest] = None,
117 - add_special_tokens: Optional[bool] = None) -> List[int]:
118 - """Encode a prompt using the tokenizer group.
119 -
120 - We pick an idle actor and use it to encode the prompt.
121 - The actor is then put back in the queue for future use.
122 - This is blocking.
123 - """
124 - self.check_health()
125 - self._ensure_queue_initialized()
126 - assert self._idle_actors is not None
127 -
128 - if self._idle_actors.empty():
129 - raise RuntimeError("No idle actors available.")
130 - actor = self._idle_actors.get_nowait()
131 - actor_is_alive = True
132 - original_actor = actor
133 - try:
134 - ret = ray.get(
135 - actor.encode.remote(prompt=prompt,
136 - lora_request=lora_request,
137 - add_special_tokens=add_special_tokens))
138 - except ActorDiedError as e:
139 - # If the actor is dead, we first try to reinitialize it.
140 - logger.warning("%s died with ActorDiedError, reinitializing.",
141 - actor,
142 - exc_info=e)
143 - actor = self._init_actor()
144 - try:
145 - ret = ray.get(
146 - actor.encode.remote(prompt=prompt,
147 - lora_request=lora_request,
148 - add_special_tokens=add_special_tokens))
149 - except ActorDiedError as e:
150 - logger.error(
151 - "%s died for second time in a row, marking "
152 - "RayTokenizerGroupPool as unhealthy.", actor)
153 - actor_is_alive = False
154 - if not self._exception:
155 - self._exception = e
156 - self.check_health()
157 - finally:
158 - self._finalize_encode(actor, original_actor, actor_is_alive)
159 - return ret
160 -
161 - async def encode_async(
162 - self,
163 - prompt: str,
164 - lora_request: Optional[LoRARequest] = None,
165 - add_special_tokens: Optional[bool] = None) -> List[int]:
166 - """Encode a prompt using the tokenizer group.
167 -
168 - We pick an idle actor and use it to encode the prompt.
169 - If there are no idle actors, we wait until one becomes
170 - available.
171 - The actor is then put back in the queue for future use.
172 - This is non-blocking.
173 - """
174 - self.check_health()
175 - self._ensure_queue_initialized()
176 - assert self._idle_actors is not None
177 -
178 - actor = await self._idle_actors.get()
179 - actor_is_alive = True
180 - original_actor = actor
181 - try:
182 - ret = await actor.encode.remote(
183 - prompt=prompt,
184 - lora_request=lora_request,
185 - add_special_tokens=add_special_tokens)
186 - except ActorDiedError as e:
187 - # If the actor is dead, we first try to reinitialize it.
188 - logger.warning("%s died with ActorDiedError, reinitializing.",
189 - actor,
190 - exc_info=e)
191 - actor = self._init_actor()
192 - try:
193 - ret = await actor.encode.remote(
194 - prompt=prompt,
195 - lora_request=lora_request,
196 - add_special_tokens=add_special_tokens)
197 - except ActorDiedError as e:
198 - logger.error(
199 - "%s died for second time in a row, marking "
200 - "RayTokenizerGroupPool as unhealthy.", actor)
201 - actor_is_alive = False
202 - if not self._exception:
203 - self._exception = e
204 - self.check_health()
205 - finally:
206 - self._finalize_encode(actor, original_actor, actor_is_alive)
207 - return ret
208 -
209 - def get_max_input_len(self,
210 - lora_request: Optional[LoRARequest] = None
211 - ) -> Optional[int]:
212 - """Get the maximum input length for the LoRA request."""
213 - return self._local_tokenizer_group.get_max_input_len(lora_request)
214 -
215 - def get_lora_tokenizer(
216 - self,
217 - lora_request: Optional[LoRARequest] = None,
218 - ) -> AnyTokenizer:
219 - return self._local_tokenizer_group.get_lora_tokenizer(lora_request)
220 -
221 - async def get_lora_tokenizer_async(
222 - self,
223 - lora_request: Optional[LoRARequest] = None,
224 - ) -> AnyTokenizer:
225 - return await self._local_tokenizer_group.get_lora_tokenizer_async(
226 - lora_request)
227 -
228 - def check_health(self):
229 - if self._exception:
230 - raise RuntimeError(
231 - "TokenizerGroupPool is unhealthy.") from self._exception
232 -
233 -
234 -def _carry_over_env_vars_to_runtime_env(runtime_env: dict) -> None:
235 - """Copy over all current process environment variables to the runtime_env.
236 -
237 - The variables in runtime_env will take precedence over the current process
238 - environment variables.
239 -
240 - runtime_env will be modified in place."""
241 - env_vars = os.environ.copy()
242 - runtime_env.setdefault("env_vars", {})
243 - env_vars.update(runtime_env["env_vars"])
244 - runtime_env["env_vars"] = env_vars
vllm/v1/engine/async_llm.py
-2
@@ -81,9 +81,7 @@ class AsyncLLM(EngineClient):
81 self.tokenizer = init_tokenizer_from_configs(
82 model_config=vllm_config.model_config,
83 scheduler_config=vllm_config.scheduler_config,
84 - parallel_config=vllm_config.parallel_config,
84 lora_config=vllm_config.lora_config)
86 - self.tokenizer.ping()
85
86 # Processor (converts Inputs --> EngineCoreRequests).
87 self.processor = Processor(
vllm/v1/engine/llm_engine.py
+4 -16
@@ -20,7 +20,7 @@ from vllm.pooling_params import PoolingParams
20 from vllm.prompt_adapter.request import PromptAdapterRequest
21 from vllm.sampling_params import SamplingParams
22 from vllm.transformers_utils.tokenizer_group import (
23 - BaseTokenizerGroup, init_tokenizer_from_configs)
23 + TokenizerGroup, init_tokenizer_from_configs)
24 from vllm.usage.usage_lib import UsageContext
25 from vllm.utils import Device
26 from vllm.v1.engine.core_client import EngineCoreClient
@@ -32,7 +32,6 @@ from vllm.v1.utils import report_usage_stats
32
33 logger = init_logger(__name__)
34
35 -_G = TypeVar("_G", bound=BaseTokenizerGroup, default=BaseTokenizerGroup)
35 _R = TypeVar("_R", default=Any)
36
37
@@ -74,9 +73,7 @@ class LLMEngine:
73 self.tokenizer = init_tokenizer_from_configs(
74 model_config=vllm_config.model_config,
75 scheduler_config=vllm_config.scheduler_config,
77 - parallel_config=vllm_config.parallel_config,
76 lora_config=vllm_config.lora_config)
79 - self.tokenizer.ping()
77
78 # Processor (convert Inputs --> EngineCoreRequests)
79 self.processor = Processor(vllm_config=vllm_config,
@@ -258,21 +255,12 @@ class LLMEngine:
255 def is_sleeping(self) -> bool:
256 return self.engine_core.is_sleeping()
257
261 - def get_tokenizer_group(
262 - self,
263 - group_type: type[_G] = BaseTokenizerGroup,
264 - ) -> _G:
265 - tokenizer_group = self.tokenizer
266 -
267 - if tokenizer_group is None:
258 + def get_tokenizer_group(self) -> TokenizerGroup:
259 + if self.tokenizer is None:
260 raise ValueError("Unable to get tokenizer because "
261 "skip_tokenizer_init is True")
270 - if not isinstance(tokenizer_group, group_type):
271 - raise TypeError("Invalid type of tokenizer group. "
272 - f"Expected type: {group_type}, but "
273 - f"found type: {type(tokenizer_group)}")
262
275 - return tokenizer_group
263 + return self.tokenizer
264
265 def add_lora(self, lora_request: LoRARequest) -> bool:
266 """Load a new LoRA adapter into the engine for future requests."""
vllm/v1/engine/output_processor.py
+2 -2
@@ -8,7 +8,7 @@ from typing import Optional, Union
8 from vllm.outputs import CompletionOutput, RequestOutput
9 from vllm.sampling_params import RequestOutputKind
10 from vllm.transformers_utils.tokenizer import AnyTokenizer
11 -from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
11 +from vllm.transformers_utils.tokenizer_group import TokenizerGroup
12 from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason
13 from vllm.v1.engine.detokenizer import IncrementalDetokenizer
14 from vllm.v1.engine.logprobs import LogprobsProcessor
@@ -225,7 +225,7 @@ class OutputProcessor:
225
226 def __init__(
227 self,
228 - tokenizer: BaseTokenizerGroup,
228 + tokenizer: TokenizerGroup,
229 log_stats: bool,
230 ):
231 self.log_stats = log_stats
vllm/v1/engine/processor.py
+2 -2
@@ -17,7 +17,7 @@ from vllm.multimodal.utils import merge_and_sort_multimodal_metadata
17 from vllm.pooling_params import PoolingParams
18 from vllm.prompt_adapter.request import PromptAdapterRequest
19 from vllm.sampling_params import SamplingParams
20 -from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
20 +from vllm.transformers_utils.tokenizer_group import TokenizerGroup
21 from vllm.v1.engine import EngineCoreRequest
22 from vllm.v1.engine.mm_input_cache import MirroredProcessingCache
23 from vllm.v1.structured_output.backend_guidance import (
@@ -31,7 +31,7 @@ class Processor:
31 def __init__(
32 self,
33 vllm_config: VllmConfig,
34 - tokenizer: BaseTokenizerGroup,
34 + tokenizer: TokenizerGroup,
35 mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
36 ):
37
vllm/v1/structured_output/backend_guidance.py
-2
@@ -61,9 +61,7 @@ class GuidanceBackend(StructuredOutputBackend):
61 tokenizer_group = init_tokenizer_from_configs(
62 model_config=vllm_config.model_config,
63 scheduler_config=vllm_config.scheduler_config,
64 - parallel_config=vllm_config.parallel_config,
64 lora_config=vllm_config.lora_config) # type: ignore[arg-type]
66 - tokenizer_group.ping()
65 self.vllm_config = vllm_config
66 self.vocab_size = vllm_config.model_config.get_vocab_size()
67
vllm/v1/structured_output/backend_xgrammar.py
-2
@@ -35,9 +35,7 @@ class XgrammarBackend(StructuredOutputBackend):
35 tokenizer_group = init_tokenizer_from_configs(
36 model_config=vllm_config.model_config,
37 scheduler_config=vllm_config.scheduler_config,
38 - parallel_config=vllm_config.parallel_config,
38 lora_config=vllm_config.lora_config) # type: ignore[arg-type]
40 - tokenizer_group.ping()
39
40 self.disable_any_whitespace = False
41 backend_options = GuidedDecodingParams(