A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[mypy] Enable following imports for some directories (#6681)
Cyrus Leung committed
Jul 31, 2024 at 10:38 UTC
da1f7cc12a12ea4a744d26122e9a13ea4b3f4c7b
18 files changed
+185
-143
.github/workflows/mypy.yaml
+13
-18
@@ -32,22 +32,17 @@ jobs:
32
pip install types-setuptools
33
- name: Mypy
34
run: |
35
- mypy tests --config-file pyproject.toml
36
- mypy vllm/*.py --config-file pyproject.toml
37
- mypy vllm/attention --config-file pyproject.toml
38
- mypy vllm/core --config-file pyproject.toml
39
- mypy vllm/distributed --config-file pyproject.toml
40
- mypy vllm/engine --config-file pyproject.toml
41
- mypy vllm/entrypoints --config-file pyproject.toml
42
- mypy vllm/executor --config-file pyproject.toml
43
- mypy vllm/inputs --config-file pyproject.toml
44
- mypy vllm/logging --config-file pyproject.toml
45
- mypy vllm/lora --config-file pyproject.toml
46
- mypy vllm/model_executor --config-file pyproject.toml
47
- mypy vllm/multimodal --config-file pyproject.toml
48
- mypy vllm/platforms --config-file pyproject.toml
49
- mypy vllm/spec_decode --config-file pyproject.toml
50
- mypy vllm/transformers_utils --config-file pyproject.toml
51
- mypy vllm/usage --config-file pyproject.toml
52
- mypy vllm/worker --config-file pyproject.toml
35
+ mypy tests --follow-imports skip
36
+ mypy vllm/attention --follow-imports skip
37
+ mypy vllm/core --follow-imports skip
38
+ mypy vllm/distributed --follow-imports skip
39
+ mypy vllm/engine --follow-imports skip
40
+ mypy vllm/entrypoints --follow-imports skip
41
+ mypy vllm/executor --follow-imports skip
42
+ mypy vllm/lora --follow-imports skip
43
+ mypy vllm/model_executor --follow-imports skip
44
+ mypy vllm/prompt_adapter --follow-imports skip
45
+ mypy vllm/spec_decode --follow-imports skip
46
+ mypy vllm/worker --follow-imports skip
47
+ mypy
48
format.sh
+13
-17
@@ -96,23 +96,19 @@ echo 'vLLM yapf: Done'
96
97
# Run mypy
98
echo 'vLLM mypy:'
99
-mypy tests --config-file pyproject.toml
100
-mypy vllm/*.py --config-file pyproject.toml
101
-mypy vllm/attention --config-file pyproject.toml
102
-mypy vllm/core --config-file pyproject.toml
103
-mypy vllm/distributed --config-file pyproject.toml
104
-mypy vllm/engine --config-file pyproject.toml
105
-mypy vllm/entrypoints --config-file pyproject.toml
106
-mypy vllm/executor --config-file pyproject.toml
107
-mypy vllm/logging --config-file pyproject.toml
108
-mypy vllm/lora --config-file pyproject.toml
109
-mypy vllm/model_executor --config-file pyproject.toml
110
-mypy vllm/multimodal --config-file pyproject.toml
111
-mypy vllm/prompt_adapter --config-file pyproject.toml
112
-mypy vllm/spec_decode --config-file pyproject.toml
113
-mypy vllm/transformers_utils --config-file pyproject.toml
114
-mypy vllm/usage --config-file pyproject.toml
115
-mypy vllm/worker --config-file pyproject.toml
99
+mypy tests --follow-imports skip
100
+mypy vllm/attention --follow-imports skip
101
+mypy vllm/core --follow-imports skip
102
+mypy vllm/distributed --follow-imports skip
103
+mypy vllm/engine --follow-imports skip
104
+mypy vllm/entrypoints --follow-imports skip
105
+mypy vllm/executor --follow-imports skip
106
+mypy vllm/lora --follow-imports skip
107
+mypy vllm/model_executor --follow-imports skip
108
+mypy vllm/prompt_adapter --follow-imports skip
109
+mypy vllm/spec_decode --follow-imports skip
110
+mypy vllm/worker --follow-imports skip
111
+mypy
112
113
114
# If git diff returns a file that is in the skip list, the file may be checked anyway:
pyproject.toml
+16
-2
@@ -48,9 +48,23 @@ python_version = "3.8"
48
49
ignore_missing_imports = true
50
check_untyped_defs = true
51
-follow_imports = "skip"
51
+follow_imports = "silent"
52
53
-files = "vllm"
53
+# After fixing type errors resulting from follow_imports: "skip" -> "silent",
54
+# move the directory here and remove it from format.sh and mypy.yaml
55
+files = [
56
+ "vllm/*.py",
57
+ "vllm/adapter_commons",
58
+ "vllm/assets",
59
+ "vllm/inputs",
60
+ "vllm/logging",
61
+ "vllm/multimodal",
62
+ "vllm/platforms",
63
+ "vllm/server",
64
+ "vllm/transformers_utils",
65
+ "vllm/triton_utils",
66
+ "vllm/usage",
67
+]
68
# TODO(woosuk): Include the code from Megatron and HuggingFace.
69
exclude = [
70
"vllm/model_executor/parallel_utils/|vllm/model_executor/models/",
vllm/_custom_ops.py
+1
-1
@@ -239,7 +239,7 @@ def cutlass_scaled_mm(a: torch.Tensor,
239
b: torch.Tensor,
240
scale_a: torch.Tensor,
241
scale_b: torch.Tensor,
242
- out_dtype: Type[torch.dtype],
242
+ out_dtype: torch.dtype,
243
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
244
assert (b.shape[0] % 16 == 0 and b.shape[1] % 16 == 0)
245
assert (out_dtype is torch.bfloat16 or out_dtype is torch.float16)
vllm/_ipex_ops.py
+49
-13
@@ -25,27 +25,33 @@ class ipex_ops:
25
x2 = x2.reshape(num, d)
26
return x1, x2
27
28
+ @staticmethod
29
def silu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
30
x1, x2 = ipex_ops._reshape_activation_tensor(x)
31
ipex.llm.functional.silu_mul(x1, x2, out)
32
33
+ @staticmethod
34
def gelu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
35
x1, x2 = ipex_ops._reshape_activation_tensor(x)
36
ipex.llm.functional.gelu_mul(x1, x2, out, "none")
37
38
+ @staticmethod
39
def gelu_tanh_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
40
x1, x2 = ipex_ops._reshape_activation_tensor(x)
41
ipex.llm.functional.gelu_mul(x1, x2, out, "tanh")
42
43
+ @staticmethod
44
def gelu_fast(out: torch.Tensor, x: torch.Tensor) -> None:
45
out.copy_(torch.nn.functional.gelu(x))
46
47
+ @staticmethod
48
def gelu_new(out: torch.Tensor, x: torch.Tensor) -> None:
49
out.copy_(torch.nn.functional.gelu(x))
50
51
# TODO add implementation of gelu_quick here
52
# def gelu_quick(out: torch.Tensor, x: torch.Tensor) -> None:
53
54
+ @staticmethod
55
def paged_attention_v1(
56
out: torch.Tensor,
57
query: torch.Tensor,
@@ -78,12 +84,21 @@ class ipex_ops:
84
).view(num_kv_heads,
85
1).repeat_interleave(num_queries_per_tokens).flatten()
86
# todo: ipex will refactor namespace
81
- torch.xpu.paged_attention_v1(out, query.contiguous(),
82
- key_cache.view_as(value_cache),
83
- value_cache, head_mapping, scale,
84
- block_tables, context_lens, block_size,
85
- max_context_len, alibi_slopes)
87
+ torch.xpu.paged_attention_v1( # type: ignore
88
+ out,
89
+ query.contiguous(),
90
+ key_cache.view_as(value_cache),
91
+ value_cache,
92
+ head_mapping,
93
+ scale,
94
+ block_tables,
95
+ context_lens,
96
+ block_size,
97
+ max_context_len,
98
+ alibi_slopes,
99
+ )
100
101
+ @staticmethod
102
def paged_attention_v2(
103
out: torch.Tensor,
104
exp_sum: torch.Tensor,
@@ -119,13 +134,24 @@ class ipex_ops:
134
).view(num_kv_heads,
135
1).repeat_interleave(num_queries_per_tokens).flatten()
136
# todo: ipex will refactor namespace
122
- torch.xpu.paged_attention_v2(out, exp_sum, max_logits, tmp_out,
123
- query.contiguous(),
124
- key_cache.view_as(value_cache),
125
- value_cache, head_mapping, block_tables,
126
- context_lens, scale, block_size,
127
- max_context_len, alibi_slopes)
137
+ torch.xpu.paged_attention_v2( # type: ignore
138
+ out,
139
+ exp_sum,
140
+ max_logits,
141
+ tmp_out,
142
+ query.contiguous(),
143
+ key_cache.view_as(value_cache),
144
+ value_cache,
145
+ head_mapping,
146
+ block_tables,
147
+ context_lens,
148
+ scale,
149
+ block_size,
150
+ max_context_len,
151
+ alibi_slopes,
152
+ )
153
154
+ @staticmethod
155
def rotary_embedding(
156
positions: torch.Tensor, # [batch_size, seq_len]
157
query: torch.Tensor, # [batch_size, seq_len, num_heads*head_size]
@@ -158,6 +184,7 @@ class ipex_ops:
184
ipex.llm.functional.rotary_embedding(query_rot, key_rot, sin, cos,
185
rotary_dim, is_neox, positions)
186
187
+ @staticmethod
188
def batched_rotary_embedding(positions: torch.Tensor, query: torch.Tensor,
189
key: torch.Tensor, head_size: int,
190
cos_sin_cache: torch.Tensor, is_neox: bool,
@@ -189,17 +216,20 @@ class ipex_ops:
216
ipex.llm.functional.rotary_embedding(query_rot, key_rot, sin, cos,
217
rotary_dim, is_neox, positions)
218
219
+ @staticmethod
220
def rms_norm(out: torch.Tensor, input: torch.Tensor, weight: torch.Tensor,
221
epsilon: float) -> None:
222
tmp = ipex.llm.functional.rms_norm(input, weight, epsilon)
223
out.copy_(tmp)
224
225
+ @staticmethod
226
def fused_add_rms_norm(input: torch.Tensor, residual: torch.Tensor,
227
weight: torch.Tensor, epsilon: float) -> None:
228
tmp = ipex.llm.functional.add_rms_norm(residual, input, weight, None,
229
epsilon, True)
230
input.copy_(tmp)
231
232
+ @staticmethod
233
def varlen_attention(
234
query: torch.Tensor,
235
key: torch.Tensor,
@@ -222,6 +252,7 @@ class ipex_ops:
252
softmax_scale, zero_tensors,
253
is_causal, return_softmax, gen_)
254
255
+ @staticmethod
256
def reshape_and_cache(
257
key: torch.Tensor,
258
value: torch.Tensor,
@@ -240,8 +271,13 @@ class ipex_ops:
271
def copy_blocks(key_caches: List[torch.Tensor],
272
value_caches: List[torch.Tensor],
273
block_mapping: torch.Tensor) -> None:
243
- torch.xpu.copy_blocks(key_caches, value_caches, block_mapping)
274
+ torch.xpu.copy_blocks( # type: ignore
275
+ key_caches,
276
+ value_caches,
277
+ block_mapping,
278
+ )
279
280
+ @staticmethod
281
def swap_blocks(src: torch.Tensor, dst: torch.Tensor,
282
block_mapping: torch.Tensor) -> None:
247
- torch.xpu.swap_blocks(src, dst, block_mapping)
283
+ torch.xpu.swap_blocks(src, dst, block_mapping) # type: ignore
vllm/adapter_commons/models.py
+15
-15
@@ -31,7 +31,7 @@ class AdapterLRUCache(LRUCache[T]):
31
super().__init__(capacity)
32
self.deactivate_fn = deactivate_fn
33
34
- def _on_remove(self, key: Hashable, value: T):
34
+ def _on_remove(self, key: Hashable, value: Optional[T]):
35
logger.debug("Removing adapter int id: %d", key)
36
self.deactivate_fn(key)
37
return super()._on_remove(key, value)
@@ -59,46 +59,46 @@ class AdapterModelManager(ABC):
59
60
@property
61
@abstractmethod
62
- def adapter_slots(self):
63
- ...
62
+ def adapter_slots(self) -> int:
63
+ raise NotImplementedError
64
65
@property
66
@abstractmethod
67
- def capacity(self):
68
- ...
67
+ def capacity(self) -> int:
68
+ raise NotImplementedError
69
70
@abstractmethod
71
def activate_adapter(self, adapter_id: int) -> bool:
72
- ...
72
+ raise NotImplementedError
73
74
@abstractmethod
75
def deactivate_adapter(self, adapter_id: int) -> bool:
76
- ...
76
+ raise NotImplementedError
77
78
@abstractmethod
79
def add_adapter(self, adapter: Any) -> bool:
80
- ...
80
+ raise NotImplementedError
81
82
@abstractmethod
83
def set_adapter_mapping(self, mapping: Any) -> None:
84
- ...
84
+ raise NotImplementedError
85
86
@abstractmethod
87
def remove_adapter(self, adapter_id: int) -> bool:
88
- ...
88
+ raise NotImplementedError
89
90
@abstractmethod
91
- def remove_all_adapters(self):
92
- ...
91
+ def remove_all_adapters(self) -> None:
92
+ raise NotImplementedError
93
94
@abstractmethod
95
def get_adapter(self, adapter_id: int) -> Optional[Any]:
96
- ...
96
+ raise NotImplementedError
97
98
@abstractmethod
99
def list_adapters(self) -> Dict[int, Any]:
100
- ...
100
+ raise NotImplementedError
101
102
@abstractmethod
103
def pin_adapter(self, adapter_id: int) -> bool:
104
- ...
104
+ raise NotImplementedError
vllm/adapter_commons/request.py
+5
-5
@@ -1,19 +1,19 @@
1
-from abc import abstractmethod
1
+from abc import ABC, abstractmethod
2
from dataclasses import dataclass
3
4
5
@dataclass
6
-class AdapterRequest:
6
+class AdapterRequest(ABC):
7
"""
8
Base class for adapter requests.
9
"""
10
11
@property
12
@abstractmethod
13
- def adapter_id(self):
14
- ...
13
+ def adapter_id(self) -> int:
14
+ raise NotImplementedError
15
16
- def __post_init__(self):
16
+ def __post_init__(self) -> None:
17
if self.adapter_id < 1:
18
raise ValueError(f"id must be > 0, got {self.adapter_id}")
19
vllm/adapter_commons/worker_manager.py
+7
-7
@@ -12,25 +12,25 @@ class AbstractWorkerManager(ABC):
12
@property
13
@abstractmethod
14
def is_enabled(self) -> bool:
15
- ...
15
+ raise NotImplementedError
16
17
@abstractmethod
18
def set_active_adapters(self, requests: Set[Any],
19
mapping: Optional[Any]) -> None:
20
- ...
20
+ raise NotImplementedError
21
22
@abstractmethod
23
def add_adapter(self, adapter_request: Any) -> bool:
24
- ...
24
+ raise NotImplementedError
25
26
@abstractmethod
27
def remove_adapter(self, adapter_id: int) -> bool:
28
- ...
28
+ raise NotImplementedError
29
30
@abstractmethod
31
- def remove_all_adapters(self):
32
- ...
31
+ def remove_all_adapters(self) -> None:
32
+ raise NotImplementedError
33
34
@abstractmethod
35
def list_adapters(self) -> Set[int]:
36
- ...
36
+ raise NotImplementedError
vllm/config.py
+2
-1
@@ -724,7 +724,7 @@ class ParallelConfig:
724
backend)
725
726
self._verify_args()
727
- self.rank = 0
727
+ self.rank: int = 0
728
729
@property
730
def use_ray(self) -> bool:
@@ -850,6 +850,7 @@ class SchedulerConfig:
850
851
852
class DeviceConfig:
853
+ device: Optional[torch.device]
854
855
def __init__(self, device: str = "auto") -> None:
856
if device == "auto":
vllm/engine/llm_engine.py
+6
-8
@@ -5,8 +5,6 @@ from typing import (TYPE_CHECKING, Any, ClassVar, Dict, Iterable, List,
5
from typing import Sequence as GenericSequence
6
from typing import Set, Type, TypeVar, Union
7
8
-from transformers import PreTrainedTokenizer
9
-
8
import vllm.envs as envs
9
from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
10
EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
@@ -40,7 +38,8 @@ from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context,
38
init_tracer)
39
from vllm.transformers_utils.config import try_get_generation_config
40
from vllm.transformers_utils.detokenizer import Detokenizer
43
-from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
41
+from vllm.transformers_utils.tokenizer_group import (AnyTokenizer,
42
+ BaseTokenizerGroup,
43
get_tokenizer_group)
44
from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled,
45
usage_message)
@@ -477,13 +476,12 @@ class LLMEngine:
476
return self.tokenizer
477
478
def get_tokenizer(
480
- self,
481
- lora_request: Optional[LoRARequest] = None
482
- ) -> "PreTrainedTokenizer":
479
+ self,
480
+ lora_request: Optional[LoRARequest] = None,
481
+ ) -> AnyTokenizer:
482
return self.get_tokenizer_group().get_lora_tokenizer(lora_request)
483
485
- def get_tokenizer_for_seq(self,
486
- sequence: Sequence) -> "PreTrainedTokenizer":
484
+ def get_tokenizer_for_seq(self, sequence: Sequence) -> AnyTokenizer:
485
return self.get_tokenizer_group().get_lora_tokenizer(
486
sequence.lora_request)
487
vllm/entrypoints/openai/serving_engine.py
+1
-3
@@ -5,7 +5,6 @@ from http import HTTPStatus
5
from typing import Iterable, Iterator, List, Optional, Tuple, TypedDict, Union
6
7
from pydantic import Field
8
-from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
8
from typing_extensions import Annotated
9
10
from vllm.config import ModelConfig
@@ -30,6 +29,7 @@ from vllm.pooling_params import PoolingParams
29
from vllm.prompt_adapter.request import PromptAdapterRequest
30
from vllm.sampling_params import SamplingParams
31
from vllm.sequence import Logprob
32
+from vllm.transformers_utils.tokenizer_group import AnyTokenizer
33
34
logger = init_logger(__name__)
35
@@ -49,8 +49,6 @@ class LoRAModulePath:
49
AnyRequest = Union[ChatCompletionRequest, CompletionRequest, DetokenizeRequest,
50
EmbeddingRequest, TokenizeRequest]
51
52
-AnyTokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
53
-
52
53
class TextTokensPrompt(TypedDict):
54
prompt: str
vllm/scripts.py
+5
-5
@@ -4,9 +4,10 @@ import asyncio
4
import os
5
import signal
6
import sys
7
-from typing import Optional
7
+from typing import List, Optional
8
9
from openai import OpenAI
10
+from openai.types.chat import ChatCompletionMessageParam
11
12
from vllm.entrypoints.openai.api_server import run_server
13
from vllm.entrypoints.openai.cli_args import make_arg_parser
@@ -63,15 +64,14 @@ def complete(model_name: str, client: OpenAI) -> None:
64
65
def chat(system_prompt: Optional[str], model_name: str,
66
client: OpenAI) -> None:
66
- conversation = []
67
+ conversation: List[ChatCompletionMessageParam] = []
68
if system_prompt is not None:
69
conversation.append({"role": "system", "content": system_prompt})
70
71
print("Please enter a message for the chat model:")
72
while True:
73
input_message = input("> ")
73
- message = {"role": "user", "content": input_message}
74
- conversation.append(message)
74
+ conversation.append({"role": "user", "content": input_message})
75
76
chat_completion = client.chat.completions.create(model=model_name,
77
messages=conversation)
@@ -79,7 +79,7 @@ def chat(system_prompt: Optional[str], model_name: str,
79
response_message = chat_completion.choices[0].message
80
output = response_message.content
81
82
- conversation.append(response_message)
82
+ conversation.append(response_message) # type: ignore
83
print(output)
84
85
vllm/transformers_utils/detokenizer.py
+2
@@ -37,6 +37,8 @@ class Detokenizer:
37
The prompt logprobs with the decoded tokens.
38
"""
39
prms = seq_group.sampling_params
40
+ assert prms is not None
41
+
42
# We can pick any sequence for the prompt.
43
seq = next(iter(seq_group.seqs_dict.values()))
44
# Only prompt, without the generated token.
vllm/transformers_utils/tokenizer_group/__init__.py
+4
-5
@@ -2,10 +2,9 @@ from typing import Optional, Type
2
3
from vllm.config import TokenizerPoolConfig
4
from vllm.executor.ray_utils import ray
5
-from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
6
- BaseTokenizerGroup)
7
-from vllm.transformers_utils.tokenizer_group.tokenizer_group import (
8
- TokenizerGroup)
5
+
6
+from .base_tokenizer_group import AnyTokenizer, BaseTokenizerGroup
7
+from .tokenizer_group import TokenizerGroup
8
9
if ray:
10
from vllm.transformers_utils.tokenizer_group.ray_tokenizer_group import (
@@ -34,4 +33,4 @@ def get_tokenizer_group(tokenizer_pool_config: Optional[TokenizerPoolConfig],
33
return tokenizer_cls.from_config(tokenizer_pool_config, **init_kwargs)
34
35
37
-__all__ = ["get_tokenizer_group", "BaseTokenizerGroup"]
36
+__all__ = ["AnyTokenizer", "get_tokenizer_group", "BaseTokenizerGroup"]
vllm/transformers_utils/tokenizer_group/base_tokenizer_group.py
+10
-8
@@ -1,11 +1,13 @@
1
from abc import ABC, abstractmethod
2
-from typing import List, Optional
2
+from typing import List, Optional, Union
3
4
-from transformers import PreTrainedTokenizer
4
+from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
5
6
from vllm.config import TokenizerPoolConfig
7
from vllm.lora.request import LoRARequest
8
9
+AnyTokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
10
+
11
12
class BaseTokenizerGroup(ABC):
13
"""A group of tokenizers that can be used for LoRA adapters."""
@@ -47,17 +49,17 @@ class BaseTokenizerGroup(ABC):
49
50
@abstractmethod
51
def get_lora_tokenizer(
50
- self,
51
- lora_request: Optional[LoRARequest] = None
52
- ) -> "PreTrainedTokenizer":
52
+ self,
53
+ lora_request: Optional[LoRARequest] = None,
54
+ ) -> AnyTokenizer:
55
"""Get a tokenizer for a LoRA request."""
56
pass
57
58
@abstractmethod
59
async def get_lora_tokenizer_async(
58
- self,
59
- lora_request: Optional[LoRARequest] = None
60
- ) -> "PreTrainedTokenizer":
60
+ self,
61
+ lora_request: Optional[LoRARequest] = None,
62
+ ) -> AnyTokenizer:
63
"""Get a tokenizer for a LoRA request."""
64
pass
65
vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py
+15
-15
@@ -6,18 +6,16 @@ try:
6
from ray.exceptions import ActorDiedError
7
except ImportError:
8
# For older versions of Ray
9
- from ray.exceptions import RayActorError as ActorDiedError
9
+ from ray.exceptions import RayActorError as ActorDiedError # type: ignore
10
from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy
11
-from transformers import PreTrainedTokenizer
11
12
from vllm.config import TokenizerPoolConfig
13
from vllm.executor.ray_utils import ray
14
from vllm.logger import init_logger
15
from vllm.lora.request import LoRARequest
17
-from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
18
- BaseTokenizerGroup)
19
-from vllm.transformers_utils.tokenizer_group.tokenizer_group import (
20
- TokenizerGroup)
16
+
17
+from .base_tokenizer_group import AnyTokenizer, BaseTokenizerGroup
18
+from .tokenizer_group import TokenizerGroup
19
20
logger = init_logger(__name__)
21
@@ -67,7 +65,7 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
65
**self._tokenizer_config, )
66
67
self._ray_tokenizer_group_cls = ray.remote(
70
- self._worker_cls).options(**ray_actor_options)
68
+ self._worker_cls).options(**ray_actor_options) # type: ignore
69
self.tokenizer_actors = [self._init_actor() for _ in range(num_actors)]
70
self._idle_actors: Optional[asyncio.Queue] = None
71
@@ -83,8 +81,10 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
81
return len(self.tokenizer_actors)
82
83
def ping(self):
86
- return ray.get(
87
- [actor.ping.remote() for actor in self.tokenizer_actors])
84
+ return ray.get([
85
+ actor.ping.remote() # type: ignore
86
+ for actor in self.tokenizer_actors
87
+ ])
88
89
def _ensure_queue_initialized(self):
90
if self._idle_actors is None:
@@ -208,15 +208,15 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
208
return self._local_tokenizer_group.get_max_input_len(lora_request)
209
210
def get_lora_tokenizer(
211
- self,
212
- lora_request: Optional[LoRARequest] = None
213
- ) -> "PreTrainedTokenizer":
211
+ self,
212
+ lora_request: Optional[LoRARequest] = None,
213
+ ) -> AnyTokenizer:
214
return self._local_tokenizer_group.get_lora_tokenizer(lora_request)
215
216
async def get_lora_tokenizer_async(
217
- self,
218
- lora_request: Optional[LoRARequest] = None
219
- ) -> "PreTrainedTokenizer":
217
+ self,
218
+ lora_request: Optional[LoRARequest] = None,
219
+ ) -> AnyTokenizer:
220
return await self._local_tokenizer_group.get_lora_tokenizer_async(
221
lora_request)
222
vllm/transformers_utils/tokenizer_group/tokenizer_group.py
+13
-15
@@ -1,16 +1,14 @@
1
from typing import List, Optional
2
3
-from transformers import PreTrainedTokenizer
4
-
3
from vllm.config import TokenizerPoolConfig
4
from vllm.lora.request import LoRARequest
5
from vllm.transformers_utils.tokenizer import (get_lora_tokenizer,
6
get_lora_tokenizer_async,
7
get_tokenizer)
10
-from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
11
- BaseTokenizerGroup)
8
from vllm.utils import LRUCache
9
10
+from .base_tokenizer_group import AnyTokenizer, BaseTokenizerGroup
11
+
12
13
class TokenizerGroup(BaseTokenizerGroup):
14
"""A group of tokenizers that can be used for LoRA adapters."""
@@ -22,8 +20,8 @@ class TokenizerGroup(BaseTokenizerGroup):
20
self.enable_lora = enable_lora
21
self.max_input_length = max_input_length
22
self.tokenizer = get_tokenizer(self.tokenizer_id, **tokenizer_config)
25
- self.lora_tokenizers = LRUCache[PreTrainedTokenizer](
26
- capacity=max_num_seqs) if enable_lora else None
23
+ self.lora_tokenizers = LRUCache[AnyTokenizer](
24
+ capacity=max_num_seqs if enable_lora else 0)
25
26
@classmethod
27
def from_config(cls, tokenizer_pool_config: Optional[TokenizerPoolConfig],
@@ -41,7 +39,7 @@ class TokenizerGroup(BaseTokenizerGroup):
39
return self.max_input_length
40
41
def _raise_if_input_too_long(self,
44
- encoded_tokens: List[str],
42
+ encoded_tokens: List[int],
43
lora_request: Optional[LoRARequest] = None):
44
input_length = len(encoded_tokens)
45
if lora_request:
@@ -72,9 +70,9 @@ class TokenizerGroup(BaseTokenizerGroup):
70
return ret
71
72
def get_lora_tokenizer(
75
- self,
76
- lora_request: Optional[LoRARequest] = None
77
- ) -> "PreTrainedTokenizer":
73
+ self,
74
+ lora_request: Optional[LoRARequest] = None,
75
+ ) -> AnyTokenizer:
76
if not lora_request or not self.enable_lora:
77
return self.tokenizer
78
if lora_request.lora_int_id not in self.lora_tokenizers:
@@ -83,12 +81,12 @@ class TokenizerGroup(BaseTokenizerGroup):
81
self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer)
82
return tokenizer
83
else:
86
- return self.lora_tokenizers.get(lora_request.lora_int_id)
84
+ return self.lora_tokenizers[lora_request.lora_int_id]
85
86
async def get_lora_tokenizer_async(
89
- self,
90
- lora_request: Optional[LoRARequest] = None
91
- ) -> "PreTrainedTokenizer":
87
+ self,
88
+ lora_request: Optional[LoRARequest] = None,
89
+ ) -> AnyTokenizer:
90
if not lora_request or not self.enable_lora:
91
return self.tokenizer
92
if lora_request.lora_int_id not in self.lora_tokenizers:
@@ -97,4 +95,4 @@ class TokenizerGroup(BaseTokenizerGroup):
95
self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer)
96
return tokenizer
97
else:
100
- return self.lora_tokenizers.get(lora_request.lora_int_id)
98
+ return self.lora_tokenizers[lora_request.lora_int_id]
vllm/utils.py
+8
-5
@@ -94,8 +94,10 @@ class LRUCache(Generic[T]):
94
def __len__(self) -> int:
95
return len(self.cache)
96
97
- def __getitem__(self, key: Hashable) -> Optional[T]:
98
- return self.get(key)
97
+ def __getitem__(self, key: Hashable) -> T:
98
+ value = self.cache[key] # Raise KeyError if not exists
99
+ self.cache.move_to_end(key)
100
+ return value
101
102
def __setitem__(self, key: Hashable, value: T) -> None:
103
self.put(key, value)
@@ -109,8 +111,9 @@ class LRUCache(Generic[T]):
111
def get(self,
112
key: Hashable,
113
default_value: Optional[T] = None) -> Optional[T]:
114
+ value: Optional[T]
115
if key in self.cache:
113
- value: Optional[T] = self.cache[key]
116
+ value = self.cache[key]
117
self.cache.move_to_end(key)
118
else:
119
value = default_value
@@ -590,8 +593,8 @@ class CudaMemoryProfiler:
593
torch.cuda.reset_peak_memory_stats(self.device)
594
mem = torch.cuda.max_memory_allocated(self.device)
595
elif is_xpu():
593
- torch.xpu.reset_peak_memory_stats(self.device)
594
- mem = torch.xpu.max_memory_allocated(self.device)
596
+ torch.xpu.reset_peak_memory_stats(self.device) # type: ignore
597
+ mem = torch.xpu.max_memory_allocated(self.device) # type: ignore
598
return mem
599
600
def __enter__(self):