A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
Update deprecated type hinting in `vllm/transformers_utils` (#18058)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
May 13, 2025 at 12:34 UTC
8c946cecca72ac9c05ab17dd4ffb51ecd2094074
17 files changed
+98
-102
vllm/transformers_utils/config.py
+5
-5
@@ -6,7 +6,7 @@ import os
6
import time
7
from functools import cache
8
from pathlib import Path
9
-from typing import Any, Callable, Dict, Literal, Optional, Type, Union
9
+from typing import Any, Callable, Literal, Optional, Union
10
11
import huggingface_hub
12
from huggingface_hub import hf_hub_download
@@ -55,11 +55,11 @@ HF_TOKEN = os.getenv('HF_TOKEN', None)
55
56
logger = init_logger(__name__)
57
58
-_CONFIG_REGISTRY_OVERRIDE_HF: Dict[str, Type[PretrainedConfig]] = {
58
+_CONFIG_REGISTRY_OVERRIDE_HF: dict[str, type[PretrainedConfig]] = {
59
"mllama": MllamaConfig
60
}
61
62
-_CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
62
+_CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = {
63
"chatglm": ChatGLMConfig,
64
"cohere2": Cohere2Config,
65
"dbrx": DbrxConfig,
@@ -199,7 +199,7 @@ def patch_rope_scaling(config: PretrainedConfig) -> None:
199
patch_rope_scaling_dict(rope_scaling)
200
201
202
-def patch_rope_scaling_dict(rope_scaling: Dict[str, Any]) -> None:
202
+def patch_rope_scaling_dict(rope_scaling: dict[str, Any]) -> None:
203
if "rope_type" in rope_scaling and "type" in rope_scaling:
204
rope_type = rope_scaling["rope_type"]
205
rope_type_legacy = rope_scaling["type"]
@@ -748,7 +748,7 @@ def get_hf_image_processor_config(
748
hf_token: Optional[Union[bool, str]] = None,
749
revision: Optional[str] = None,
750
**kwargs,
751
-) -> Dict[str, Any]:
751
+) -> dict[str, Any]:
752
# ModelScope does not provide an interface for image_processor
753
if VLLM_USE_MODELSCOPE:
754
return dict()
vllm/transformers_utils/configs/arctic.py
+3
-3
@@ -8,7 +8,7 @@
8
""" Arctic model configuration"""
9
10
from dataclasses import asdict, dataclass
11
-from typing import Any, Dict
11
+from typing import Any
12
13
from transformers.configuration_utils import PretrainedConfig
14
from transformers.utils import logging
@@ -192,14 +192,14 @@ class ArcticConfig(PretrainedConfig):
192
)
193
194
@classmethod
195
- def from_dict(cls, config_dict: Dict[str, Any], **kwargs) -> "ArcticConfig":
195
+ def from_dict(cls, config_dict: dict[str, Any], **kwargs) -> "ArcticConfig":
196
result = super().from_dict(config_dict, **kwargs)
197
config = result[0] if isinstance(result, tuple) else result
198
if isinstance(config.quantization, dict):
199
config.quantization = ArcticQuantizationConfig(**config.quantization)
200
return result
201
202
- def to_dict(self) -> Dict[str, Any]:
202
+ def to_dict(self) -> dict[str, Any]:
203
ret = super().to_dict()
204
if isinstance(ret["quantization"], ArcticQuantizationConfig):
205
ret["quantization"] = asdict(ret["quantization"])
vllm/transformers_utils/configs/cohere2.py
+3
-3
@@ -61,7 +61,7 @@ class Cohere2Config(PretrainedConfig):
61
Whether to tie weight embeddings
62
rope_theta (`float`, *optional*, defaults to 10000.0):
63
The base period of the RoPE embeddings.
64
- rope_scaling (`Dict`, *optional*):
64
+ rope_scaling (`dict`, *optional*):
65
Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
66
and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
67
accordingly.
@@ -86,11 +86,11 @@ class Cohere2Config(PretrainedConfig):
86
`beta_slow` (`float`, *optional*):
87
Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
88
ramp function. If unspecified, it defaults to 1.
89
- `short_factor` (`List[float]`, *optional*):
89
+ `short_factor` (`list[float]`, *optional*):
90
Only used with 'longrope'. The scaling factor to be applied to short contexts (<
91
`original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
92
size divided by the number of attention heads divided by 2
93
- `long_factor` (`List[float]`, *optional*):
93
+ `long_factor` (`list[float]`, *optional*):
94
Only used with 'longrope'. The scaling factor to be applied to long contexts (<
95
`original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
96
size divided by the number of attention heads divided by 2
vllm/transformers_utils/configs/deepseek_vl2.py
+2
-3
@@ -1,7 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
# adapted from https://github.com/deepseek-ai/DeepSeek-VL2/blob/faf18023f24b962b32d9f0a2d89e402a8d383a78/deepseek_vl2/models/modeling_deepseek_vl_v2.py#L115-L268
4
-from typing import Tuple
4
5
from transformers.configuration_utils import PretrainedConfig
6
@@ -191,12 +190,12 @@ class DeepseekVLV2Config(PretrainedConfig):
190
191
tile_tag: str = "2D"
192
global_view_pos: str = "head"
194
- candidate_resolutions: Tuple[Tuple[int, int]] = ((384, 384), )
193
+ candidate_resolutions: tuple[tuple[int, int]] = ((384, 384), )
194
195
def __init__(self,
196
tile_tag: str = "tile_tag",
197
global_view_pos: str = "head",
199
- candidate_resolutions: Tuple[Tuple[int,
198
+ candidate_resolutions: tuple[tuple[int,
199
int]] = ((384, 384), ),
200
**kwargs):
201
super().__init__(**kwargs)
vllm/transformers_utils/configs/exaone.py
+1
-3
@@ -17,14 +17,12 @@
17
# limitations under the License.
18
"""Exaone model configuration"""
19
20
-from typing import Dict
21
-
20
from transformers.configuration_utils import PretrainedConfig
21
from transformers.utils import logging
22
23
logger = logging.get_logger(__name__)
24
27
-EXAONE_PRETRAINED_CONFIG_ARCHIVE_MAP: Dict[str, str] = {}
25
+EXAONE_PRETRAINED_CONFIG_ARCHIVE_MAP: dict[str, str] = {}
26
27
28
class ExaoneConfig(PretrainedConfig):
vllm/transformers_utils/configs/jais.py
+2
-2
@@ -98,7 +98,7 @@ class JAISConfig(PretrainedConfig):
98
Scale attention weights by dividing by hidden_size instead of
99
sqrt(hidden_size). Need to set scale_attn_weights to `True` as
100
well.
101
- alibi_scaling (`Dict`, *optional*):
101
+ alibi_scaling (`dict`, *optional*):
102
Dictionary containing the scaling configuration for ALiBi
103
embeddings. Currently only supports linear
104
scaling strategy. Can specify either the scaling `factor` (must be
@@ -108,7 +108,7 @@ class JAISConfig(PretrainedConfig):
108
formats are `{"type": strategy name, "factor": scaling factor}` or
109
`{"type": strategy name,
110
"train_seq_len": training sequence length}`.
111
- architectures (`List`, *optional*, defaults to ['JAISLMHeadModel']):
111
+ architectures (`list`, *optional*, defaults to ['JAISLMHeadModel']):
112
architecture names for Jais.
113
114
Example:
vllm/transformers_utils/configs/mlp_speculator.py
+3
-3
@@ -1,6 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-from typing import List, Optional
3
+from typing import Optional
4
5
from transformers import PretrainedConfig
6
@@ -17,7 +17,7 @@ class MLPSpeculatorConfig(PretrainedConfig):
17
emb_dim: int = 4096,
18
inner_dim: int = 0,
19
n_predict: int = 3,
20
- top_k_tokens_per_head: Optional[List[int]] = None,
20
+ top_k_tokens_per_head: Optional[list[int]] = None,
21
n_candidates: int = 5,
22
tie_weights: bool = False,
23
scale_input: bool = False,
@@ -34,7 +34,7 @@ class MLPSpeculatorConfig(PretrainedConfig):
34
the inner dimension of the model. If 0, will be the emb_dim.
35
n_predict: int
36
the number of lookaheads for the speculator
37
- top_k_tokens_per_head: List[int]
37
+ top_k_tokens_per_head: list[int]
38
Number of tokens to consider from each head when forming the
39
candidate tree.
40
For each candidate branch in the tree, head n produces topk[n]
vllm/transformers_utils/configs/mpt.py
+9
-9
@@ -4,11 +4,11 @@
4
# https://huggingface.co/mosaicml/mpt-7b/blob/main/configuration_mpt.py
5
"""A HuggingFace-style model configuration."""
6
import warnings
7
-from typing import Any, Dict, Optional, Union
7
+from typing import Any, Optional, Union
8
9
from transformers import PretrainedConfig
10
11
-attn_config_defaults: Dict = {
11
+attn_config_defaults: dict = {
12
'attn_type': 'multihead_attention',
13
'attn_pdrop': 0.0,
14
'attn_impl': 'triton',
@@ -20,8 +20,8 @@ attn_config_defaults: Dict = {
20
'alibi': False,
21
'alibi_bias_max': 8
22
}
23
-ffn_config_defaults: Dict = {'ffn_type': 'mptmlp'}
24
-init_config_defaults: Dict = {
23
+ffn_config_defaults: dict = {'ffn_type': 'mptmlp'}
24
+init_config_defaults: dict = {
25
'name': 'kaiming_normal_',
26
'fan_mode': 'fan_in',
27
'init_nonlinearity': 'relu',
@@ -52,15 +52,15 @@ class MPTConfig(PretrainedConfig):
52
resid_pdrop: float = 0.0,
53
emb_pdrop: float = 0.0,
54
learned_pos_emb: bool = True,
55
- attn_config: Dict = attn_config_defaults,
56
- ffn_config: Dict = ffn_config_defaults,
55
+ attn_config: dict = attn_config_defaults,
56
+ ffn_config: dict = ffn_config_defaults,
57
init_device: str = 'cpu',
58
logit_scale: Optional[Union[float, str]] = None,
59
no_bias: bool = False,
60
embedding_fraction: float = 1.0,
61
norm_type: str = 'low_precision_layernorm',
62
use_cache: bool = False,
63
- init_config: Dict = init_config_defaults,
63
+ init_config: dict = init_config_defaults,
64
fc_type: str = 'torch',
65
verbose: Optional[int] = None,
66
**kwargs: Any):
@@ -102,8 +102,8 @@ class MPTConfig(PretrainedConfig):
102
self._validate_config()
103
104
def _set_config_defaults(
105
- self, config: Dict[str, Any],
106
- config_defaults: Dict[str, Any]) -> Dict[str, Any]:
105
+ self, config: dict[str, Any],
106
+ config_defaults: dict[str, Any]) -> dict[str, Any]:
107
for (k, v) in config_defaults.items():
108
if k not in config:
109
config[k] = v
vllm/transformers_utils/configs/solar.py
+1
-1
@@ -108,7 +108,7 @@ class SolarConfig(PretrainedConfig):
108
Whether to tie weight embeddings
109
rope_theta (`float`, *optional*, defaults to 10000.0):
110
The base period of the RoPE embeddings.
111
- rope_scaling (`Dict`, *optional*):
111
+ rope_scaling (`dict`, *optional*):
112
Dictionary containing the scaling configuration for
113
the RoPE embeddings.
114
Currently supports two scaling
vllm/transformers_utils/configs/ultravox.py
+5
-5
@@ -1,7 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
# Adapted from https://github.com/fixie-ai/ultravox/blob/ecd58c4041030bae2ad15aa6bcf04ab43199ea02/ultravox/model/ultravox_config.py
4
-from typing import Any, Dict, Optional
4
+from typing import Any, Optional
5
6
import transformers
7
@@ -48,8 +48,8 @@ class UltravoxConfig(transformers.PretrainedConfig):
48
49
def __init__(
50
self,
51
- audio_config: Optional[Dict[str, Any]] = None,
52
- text_config: Optional[Dict[str, Any]] = None,
51
+ audio_config: Optional[dict[str, Any]] = None,
52
+ text_config: Optional[dict[str, Any]] = None,
53
audio_model_id: Optional[str] = None,
54
text_model_id: Optional[str] = None,
55
ignore_index: int = -100,
@@ -58,8 +58,8 @@ class UltravoxConfig(transformers.PretrainedConfig):
58
stack_factor: int = 8,
59
norm_init: float = 0.4,
60
projector_act: str = "swiglu",
61
- text_model_lora_config: Optional[Dict[str, Any]] = None,
62
- audio_model_lora_config: Optional[Dict[str, Any]] = None,
61
+ text_model_lora_config: Optional[dict[str, Any]] = None,
62
+ audio_model_lora_config: Optional[dict[str, Any]] = None,
63
projector_ln_mid: bool = False,
64
**kwargs,
65
):
vllm/transformers_utils/detokenizer.py
+3
-3
@@ -1,6 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-from typing import Dict, List, Optional
3
+from typing import Optional
4
5
from vllm.sequence import (VLLM_INVALID_TOKEN_ID, Logprob, SamplingParams,
6
Sequence, SequenceGroup)
@@ -22,7 +22,7 @@ class Detokenizer:
22
return self.tokenizer_group.get_lora_tokenizer(sequence.lora_request)
23
24
def decode_prompt_logprobs_inplace(self, seq_group: SequenceGroup,
25
- prompt_logprobs: List[Optional[Dict[
25
+ prompt_logprobs: list[Optional[dict[
26
int, Logprob]]],
27
position_offset: int) -> None:
28
"""Decodes the logprobs for the prompt of a sequence group.
@@ -49,7 +49,7 @@ class Detokenizer:
49
read_offset = 0
50
next_iter_prefix_offset = 0
51
next_iter_read_offset = 0
52
- next_iter_tokens: List[str] = []
52
+ next_iter_tokens: list[str] = []
53
prev_tokens = None
54
55
for token_position_in_logprob, prompt_logprobs_for_token in enumerate(
vllm/transformers_utils/detokenizer_utils.py
+12
-12
@@ -1,11 +1,11 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-from typing import List, Optional, Tuple
3
+from typing import Optional
4
5
from .tokenizer import AnyTokenizer
6
7
8
-def _replace_none_with_empty(tokens: List[Optional[str]]):
8
+def _replace_none_with_empty(tokens: list[Optional[str]]):
9
for i, token in enumerate(tokens):
10
if token is None:
11
tokens[i] = ""
@@ -13,7 +13,7 @@ def _replace_none_with_empty(tokens: List[Optional[str]]):
13
14
def _convert_tokens_to_string_with_added_encoders(
15
tokenizer: AnyTokenizer,
16
- output_tokens: List[str],
16
+ output_tokens: list[str],
17
skip_special_tokens: bool,
18
spaces_between_special_tokens: bool,
19
) -> str:
@@ -22,8 +22,8 @@ def _convert_tokens_to_string_with_added_encoders(
22
# NOTE(woosuk): The following code is slow because it runs a for loop over
23
# the output_tokens. In Python, running a for loop over a list can be slow
24
# even when the loop body is very simple.
25
- sub_texts: List[str] = []
26
- current_sub_text: List[str] = []
25
+ sub_texts: list[str] = []
26
+ current_sub_text: list[str] = []
27
all_special_tokens = set(tokenizer.all_special_tokens)
28
for token in output_tokens:
29
if skip_special_tokens and token in all_special_tokens:
@@ -52,9 +52,9 @@ INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET = 5
52
53
def convert_prompt_ids_to_tokens(
54
tokenizer: AnyTokenizer,
55
- prompt_ids: List[int],
55
+ prompt_ids: list[int],
56
skip_special_tokens: bool = False,
57
-) -> Tuple[List[str], int, int]:
57
+) -> tuple[list[str], int, int]:
58
"""Converts the prompt ids to tokens and returns the tokens and offsets
59
for incremental detokenization.
60
@@ -76,8 +76,8 @@ def convert_prompt_ids_to_tokens(
76
77
def convert_ids_list_to_tokens(
78
tokenizer: AnyTokenizer,
79
- token_ids: List[int],
80
-) -> List[str]:
79
+ token_ids: list[int],
80
+) -> list[str]:
81
"""Detokenize the input ids individually.
82
83
Args:
@@ -98,13 +98,13 @@ def convert_ids_list_to_tokens(
98
# under Apache 2.0 license
99
def detokenize_incrementally(
100
tokenizer: AnyTokenizer,
101
- all_input_ids: List[int],
102
- prev_tokens: Optional[List[str]],
101
+ all_input_ids: list[int],
102
+ prev_tokens: Optional[list[str]],
103
prefix_offset: int,
104
read_offset: int,
105
skip_special_tokens: bool = False,
106
spaces_between_special_tokens: bool = True,
107
-) -> Tuple[List[str], str, int, int]:
107
+) -> tuple[list[str], str, int, int]:
108
"""Detokenizes the input ids incrementally and returns the new tokens
109
and the new text.
110
vllm/transformers_utils/processors/deepseek_vl2.py
+14
-15
@@ -24,7 +24,6 @@
24
# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25
26
import math
27
-from typing import List, Tuple
27
28
import torch
29
import torchvision.transforms as T
@@ -36,8 +35,8 @@ from transformers.processing_utils import ProcessorMixin
35
class ImageTransform:
36
37
def __init__(self,
39
- mean: Tuple[float, float, float] = (0.5, 0.5, 0.5),
40
- std: Tuple[float, float, float] = (0.5, 0.5, 0.5),
38
+ mean: tuple[float, float, float] = (0.5, 0.5, 0.5),
39
+ std: tuple[float, float, float] = (0.5, 0.5, 0.5),
40
normalize: bool = True):
41
self.mean = mean
42
self.std = std
@@ -62,11 +61,11 @@ class DeepseekVLV2Processor(ProcessorMixin):
61
def __init__(
62
self,
63
tokenizer: LlamaTokenizerFast,
65
- candidate_resolutions: Tuple[Tuple[int, int]],
64
+ candidate_resolutions: tuple[tuple[int, int]],
65
patch_size: int,
66
downsample_ratio: int,
68
- image_mean: Tuple[float, float, float] = (0.5, 0.5, 0.5),
69
- image_std: Tuple[float, float, float] = (0.5, 0.5, 0.5),
67
+ image_mean: tuple[float, float, float] = (0.5, 0.5, 0.5),
68
+ image_std: tuple[float, float, float] = (0.5, 0.5, 0.5),
69
normalize: bool = True,
70
image_token: str = "<image>",
71
pad_token: str = "<｜▁pad▁｜>",
@@ -170,13 +169,13 @@ class DeepseekVLV2Processor(ProcessorMixin):
169
170
return t
171
173
- def decode(self, t: List[int], **kwargs) -> str:
172
+ def decode(self, t: list[int], **kwargs) -> str:
173
return self.tokenizer.decode(t, **kwargs)
174
175
def process_one(
176
self,
177
prompt: str,
179
- images: List[Image.Image],
178
+ images: list[Image.Image],
179
inference_mode: bool = True,
180
**kwargs,
181
):
@@ -184,8 +183,8 @@ class DeepseekVLV2Processor(ProcessorMixin):
183
184
Args:
185
prompt (str): the formatted prompt;
187
- conversations (List[Dict]): conversations with a list of messages;
188
- images (List[ImageType]): the list of images;
186
+ conversations (list[dict]): conversations with a list of messages;
187
+ images (list[ImageType]): the list of images;
188
inference_mode (bool): if True, then remove the last eos token;
189
system_prompt (str): the system prompt;
190
**kwargs:
@@ -196,7 +195,7 @@ class DeepseekVLV2Processor(ProcessorMixin):
195
- target_ids (torch.LongTensor): [N + image tokens]
196
- pixel_values (torch.FloatTensor): [n_patches, 3, H, W]
197
- image_id (int): the id of the image token
199
- - num_image_tokens (List[int]): the number of image tokens
198
+ - num_image_tokens (list[int]): the number of image tokens
199
"""
200
201
assert (prompt is not None and images is not None
@@ -257,7 +256,7 @@ class DeepseekVLV2Processor(ProcessorMixin):
256
self,
257
*,
258
prompt: str,
260
- images: List[Image.Image],
259
+ images: list[Image.Image],
260
inference_mode: bool = True,
261
**kwargs,
262
):
@@ -265,7 +264,7 @@ class DeepseekVLV2Processor(ProcessorMixin):
264
265
Args:
266
prompt (str): the formatted prompt;
268
- images (List[ImageType]): the list of images;
267
+ images (list[ImageType]): the list of images;
268
inference_mode (bool): if True, then remove the last eos token;
269
**kwargs:
270
@@ -274,7 +273,7 @@ class DeepseekVLV2Processor(ProcessorMixin):
273
- input_ids (torch.LongTensor): [N + image tokens]
274
- images (torch.FloatTensor): [n_images, 3, H, W]
275
- image_id (int): the id of the image token
277
- - num_image_tokens (List[int]): the number of image tokens
276
+ - num_image_tokens (list[int]): the number of image tokens
277
"""
278
279
prepare = self.process_one(
@@ -288,7 +287,7 @@ class DeepseekVLV2Processor(ProcessorMixin):
287
def tokenize_with_images(
288
self,
289
conversation: str,
291
- images: List[Image.Image],
290
+ images: list[Image.Image],
291
bos: bool = True,
292
eos: bool = True,
293
cropping: bool = True,
vllm/transformers_utils/processors/ovis.py
+6
-6
@@ -23,7 +23,7 @@
23
# See the License for the specific language governing permissions and
24
# limitations under the License.
25
from functools import cached_property
26
-from typing import List, Union
26
+from typing import Union
27
28
import PIL
29
import torch
@@ -102,7 +102,7 @@ class OvisProcessor(ProcessorMixin):
102
def __call__(
103
self,
104
images: ImageInput = None,
105
- text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,
105
+ text: Union[TextInput, PreTokenizedInput, list[TextInput], list[PreTokenizedInput]] = None,
106
**kwargs: Unpack[OvisProcessorKwargs],
107
) -> BatchFeature:
108
"""
@@ -111,14 +111,14 @@ class OvisProcessor(ProcessorMixin):
111
the text. To prepare the vision inputs, this method forwards the `vision_infos` and `kwrags` arguments to
112
Qwen2VLImageProcessor's [`~Qwen2VLImageProcessor.__call__`] if `vision_infos` is not `None`.
113
Args:
114
- images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`, `List[torch.Tensor]`):
114
+ images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `list[PIL.Image.Image]`, `list[np.ndarray]`, `list[torch.Tensor]`):
115
The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch
116
tensor. Both channels-first and channels-last formats are supported.
117
- text (`str`, `List[str]`, `List[List[str]]`):
117
+ text (`str`, `list[str]`, `list[list[str]]`):
118
The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
119
(pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
120
`is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
121
- videos (`np.ndarray`, `torch.Tensor`, `List[np.ndarray]`, `List[torch.Tensor]`):
121
+ videos (`np.ndarray`, `torch.Tensor`, `list[np.ndarray]`, `list[torch.Tensor]`):
122
The image or batch of videos to be prepared. Each video can be a 4D NumPy array or PyTorch
123
tensor, or a nested list of 3D frames. Both channels-first and channels-last formats are supported.
124
return_tensors (`str` or [`~utils.TensorType`], *optional*):
@@ -400,7 +400,7 @@ class OvisProcessor(ProcessorMixin):
400
The output of the model `generate` function. The output is expected to be a tensor of shape `(batch_size, sequence_length)`
401
or `(sequence_length,)`.
402
Returns:
403
- `List[str]`: The decoded text.
403
+ `list[str]`: The decoded text.
404
"""
405
return self.tokenizer.batch_decode(
406
generated_outputs, skip_special_tokens=True, clean_up_tokenization_spaces=False
vllm/transformers_utils/tokenizer_group.py
+4
-4
@@ -1,6 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-from typing import List, Optional
3
+from typing import Optional
4
5
from vllm.config import LoRAConfig, ModelConfig, SchedulerConfig
6
from vllm.lora.request import LoRARequest
@@ -32,7 +32,7 @@ class TokenizerGroup:
32
return self.max_input_length
33
34
def _raise_if_input_too_long(self,
35
- encoded_tokens: List[int],
35
+ encoded_tokens: list[int],
36
lora_request: Optional[LoRARequest] = None):
37
input_length = len(encoded_tokens)
38
if lora_request:
@@ -48,7 +48,7 @@ class TokenizerGroup:
48
max_length: Optional[int] = None,
49
truncation: Optional[bool] = None,
50
lora_request: Optional[LoRARequest] = None,
51
- add_special_tokens: Optional[bool] = None) -> List[int]:
51
+ add_special_tokens: Optional[bool] = None) -> list[int]:
52
53
tokenizer = self.get_lora_tokenizer(lora_request)
54
ret = encode_tokens(tokenizer,
@@ -65,7 +65,7 @@ class TokenizerGroup:
65
max_length: Optional[int] = None,
66
truncation: Optional[bool] = None,
67
lora_request: Optional[LoRARequest] = None,
68
- add_special_tokens: Optional[bool] = None) -> List[int]:
68
+ add_special_tokens: Optional[bool] = None) -> list[int]:
69
tokenizer = await self.get_lora_tokenizer_async(lora_request)
70
ret = encode_tokens(tokenizer,
71
prompt,
vllm/transformers_utils/tokenizers/mistral.py
+23
-23
@@ -4,7 +4,7 @@ import os
4
import re
5
from dataclasses import dataclass
6
from pathlib import Path
7
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast
7
+from typing import TYPE_CHECKING, Any, Optional, Union, cast
8
9
import huggingface_hub
10
from huggingface_hub import HfApi, hf_hub_download
@@ -28,7 +28,7 @@ logger = init_logger(__name__)
28
29
@dataclass
30
class Encoding:
31
- input_ids: Union[List[int], List[List[int]]]
31
+ input_ids: Union[list[int], list[list[int]]]
32
33
34
def maybe_serialize_tool_calls(request: "ChatCompletionRequest"):
@@ -105,7 +105,7 @@ def validate_request_params(request: "ChatCompletionRequest"):
105
"for Mistral tokenizers.")
106
107
108
-def list_local_repo_files(repo_id: str, revision: Optional[str]) -> List[str]:
108
+def list_local_repo_files(repo_id: str, revision: Optional[str]) -> list[str]:
109
repo_cache = os.path.join(
110
huggingface_hub.constants.HF_HUB_CACHE,
111
huggingface_hub.constants.REPO_ID_SEPARATOR.join(
@@ -125,7 +125,7 @@ def list_local_repo_files(repo_id: str, revision: Optional[str]) -> List[str]:
125
return []
126
127
128
-def find_tokenizer_file(files: List[str]):
128
+def find_tokenizer_file(files: list[str]):
129
file_pattern = re.compile(
130
r"^tokenizer\.model\.v.*$|^tekken\.json$|^tokenizer\.mm\.model\.v.*$")
131
@@ -145,10 +145,10 @@ def find_tokenizer_file(files: List[str]):
145
146
147
def make_mistral_chat_completion_request(
148
- messages: List["ChatCompletionMessageParam"],
149
- tools: Optional[List[Dict[str,
148
+ messages: list["ChatCompletionMessageParam"],
149
+ tools: Optional[list[dict[str,
150
Any]]] = None) -> "ChatCompletionRequest":
151
- last_message = cast(Dict[str, Any], messages[-1])
151
+ last_message = cast(dict[str, Any], messages[-1])
152
if last_message["role"] == "assistant":
153
last_message["prefix"] = True
154
@@ -199,7 +199,7 @@ class MistralTokenizer(TokenizerBase):
199
raise TypeError(f"Unsupported tokenizer: {type(tokenizer_)}")
200
201
self._vocab = tokenizer_.vocab()
202
- # Convert to a Dict[str, int] to match protocol, but this is a lossy
202
+ # Convert to a dict[str, int] to match protocol, but this is a lossy
203
# conversion. There may be multiple token ids that decode to the same
204
# string due to partial UTF-8 byte sequences being converted to �
205
self._vocab_dict = {
@@ -314,21 +314,21 @@ class MistralTokenizer(TokenizerBase):
314
315
def __call__(
316
self,
317
- text: Union[str, List[str], List[int]],
317
+ text: Union[str, list[str], list[int]],
318
text_pair: Optional[str] = None,
319
add_special_tokens: bool = False,
320
truncation: bool = False,
321
max_length: Optional[int] = None,
322
):
323
- input_ids: Union[List[int], List[List[int]]]
324
- # For List[str], original prompt text
323
+ input_ids: Union[list[int], list[list[int]]]
324
+ # For list[str], original prompt text
325
if is_list_of(text, str):
326
- input_ids_: List[List[int]] = []
326
+ input_ids_: list[list[int]] = []
327
for p in text:
328
each_input_ids = self.encode_one(p, truncation, max_length)
329
input_ids_.append(each_input_ids)
330
input_ids = input_ids_
331
- # For List[int], apply chat template output, already tokens.
331
+ # For list[int], apply chat template output, already tokens.
332
elif is_list_of(text, int):
333
input_ids = text
334
# For str, single prompt text
@@ -350,7 +350,7 @@ class MistralTokenizer(TokenizerBase):
350
text: str,
351
truncation: bool = False,
352
max_length: Optional[int] = None,
353
- ) -> List[int]:
353
+ ) -> list[int]:
354
# Mistral Tokenizers should not add special tokens
355
input_ids = self.encode(text)
356
@@ -362,7 +362,7 @@ class MistralTokenizer(TokenizerBase):
362
text: str,
363
truncation: Optional[bool] = None,
364
max_length: Optional[int] = None,
365
- add_special_tokens: Optional[bool] = None) -> List[int]:
365
+ add_special_tokens: Optional[bool] = None) -> list[int]:
366
# `encode` should only be used for prompt completion
367
# it should never be used for chat_completion.
368
# For chat completion use `apply_chat_template`
@@ -374,9 +374,9 @@ class MistralTokenizer(TokenizerBase):
374
return self.tokenizer.encode(text, bos=True, eos=False)
375
376
def apply_chat_template(self,
377
- messages: List["ChatCompletionMessageParam"],
378
- tools: Optional[List[Dict[str, Any]]] = None,
379
- **kwargs) -> List[int]:
377
+ messages: list["ChatCompletionMessageParam"],
378
+ tools: Optional[list[dict[str, Any]]] = None,
379
+ **kwargs) -> list[int]:
380
381
request = make_mistral_chat_completion_request(messages, tools)
382
encoded = self.mistral.encode_chat_completion(request)
@@ -384,7 +384,7 @@ class MistralTokenizer(TokenizerBase):
384
# encode-decode to get clean prompt
385
return encoded.tokens
386
387
- def convert_tokens_to_string(self, tokens: List[str]) -> str:
387
+ def convert_tokens_to_string(self, tokens: list[str]) -> str:
388
from mistral_common.tokens.tokenizers.base import SpecialTokens
389
if self.is_tekken:
390
tokens = [
@@ -417,7 +417,7 @@ class MistralTokenizer(TokenizerBase):
417
# make sure certain special tokens like Tool calls are
418
# not decoded
419
special_tokens = {SpecialTokens.tool_calls}
420
- regular_tokens: List[str] = []
420
+ regular_tokens: list[str] = []
421
decoded_list = []
422
423
for token in tokens:
@@ -442,7 +442,7 @@ class MistralTokenizer(TokenizerBase):
442
# See: guided_decoding/outlines_logits_processors.py::_adapt_tokenizer
443
# for more.
444
def decode(self,
445
- ids: Union[List[int], int],
445
+ ids: Union[list[int], int],
446
skip_special_tokens: bool = True) -> str:
447
assert (
448
skip_special_tokens
@@ -454,9 +454,9 @@ class MistralTokenizer(TokenizerBase):
454
455
def convert_ids_to_tokens(
456
self,
457
- ids: List[int],
457
+ ids: list[int],
458
skip_special_tokens: bool = True,
459
- ) -> List[str]:
459
+ ) -> list[str]:
460
from mistral_common.tokens.tokenizers.base import SpecialTokens
461
462
# TODO(Patrick) - potentially allow special tokens to not be skipped
vllm/transformers_utils/utils.py
+2
-2
@@ -4,7 +4,7 @@ import json
4
from functools import cache
5
from os import PathLike
6
from pathlib import Path
7
-from typing import List, Optional, Union
7
+from typing import Optional, Union
8
9
from vllm.envs import VLLM_MODEL_REDIRECT_PATH
10
from vllm.logger import init_logger
@@ -38,7 +38,7 @@ def modelscope_list_repo_files(
38
repo_id: str,
39
revision: Optional[str] = None,
40
token: Union[str, bool, None] = None,
41
-) -> List[str]:
41
+) -> list[str]:
42
"""List files in a modelscope repo."""
43
from modelscope.hub.api import HubApi
44
api = HubApi()