@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / da1f7cc12a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[mypy] Enable following imports for some directories (#6681)

Cyrus Leung committed Jul 31, 2024 at 10:38 UTC da1f7cc12a12ea4a744d26122e9a13ea4b3f4c7b
18 files changed +185 -143
.github/workflows/mypy.yaml
+13 -18
@@ -32,22 +32,17 @@ jobs:
32 pip install types-setuptools
33 - name: Mypy
34 run: |
35 - mypy tests --config-file pyproject.toml
36 - mypy vllm/*.py --config-file pyproject.toml
37 - mypy vllm/attention --config-file pyproject.toml
38 - mypy vllm/core --config-file pyproject.toml
39 - mypy vllm/distributed --config-file pyproject.toml
40 - mypy vllm/engine --config-file pyproject.toml
41 - mypy vllm/entrypoints --config-file pyproject.toml
42 - mypy vllm/executor --config-file pyproject.toml
43 - mypy vllm/inputs --config-file pyproject.toml
44 - mypy vllm/logging --config-file pyproject.toml
45 - mypy vllm/lora --config-file pyproject.toml
46 - mypy vllm/model_executor --config-file pyproject.toml
47 - mypy vllm/multimodal --config-file pyproject.toml
48 - mypy vllm/platforms --config-file pyproject.toml
49 - mypy vllm/spec_decode --config-file pyproject.toml
50 - mypy vllm/transformers_utils --config-file pyproject.toml
51 - mypy vllm/usage --config-file pyproject.toml
52 - mypy vllm/worker --config-file pyproject.toml
35 + mypy tests --follow-imports skip
36 + mypy vllm/attention --follow-imports skip
37 + mypy vllm/core --follow-imports skip
38 + mypy vllm/distributed --follow-imports skip
39 + mypy vllm/engine --follow-imports skip
40 + mypy vllm/entrypoints --follow-imports skip
41 + mypy vllm/executor --follow-imports skip
42 + mypy vllm/lora --follow-imports skip
43 + mypy vllm/model_executor --follow-imports skip
44 + mypy vllm/prompt_adapter --follow-imports skip
45 + mypy vllm/spec_decode --follow-imports skip
46 + mypy vllm/worker --follow-imports skip
47 + mypy
48
format.sh
+13 -17
@@ -96,23 +96,19 @@ echo 'vLLM yapf: Done'
96
97 # Run mypy
98 echo 'vLLM mypy:'
99 -mypy tests --config-file pyproject.toml
100 -mypy vllm/*.py --config-file pyproject.toml
101 -mypy vllm/attention --config-file pyproject.toml
102 -mypy vllm/core --config-file pyproject.toml
103 -mypy vllm/distributed --config-file pyproject.toml
104 -mypy vllm/engine --config-file pyproject.toml
105 -mypy vllm/entrypoints --config-file pyproject.toml
106 -mypy vllm/executor --config-file pyproject.toml
107 -mypy vllm/logging --config-file pyproject.toml
108 -mypy vllm/lora --config-file pyproject.toml
109 -mypy vllm/model_executor --config-file pyproject.toml
110 -mypy vllm/multimodal --config-file pyproject.toml
111 -mypy vllm/prompt_adapter --config-file pyproject.toml
112 -mypy vllm/spec_decode --config-file pyproject.toml
113 -mypy vllm/transformers_utils --config-file pyproject.toml
114 -mypy vllm/usage --config-file pyproject.toml
115 -mypy vllm/worker --config-file pyproject.toml
99 +mypy tests --follow-imports skip
100 +mypy vllm/attention --follow-imports skip
101 +mypy vllm/core --follow-imports skip
102 +mypy vllm/distributed --follow-imports skip
103 +mypy vllm/engine --follow-imports skip
104 +mypy vllm/entrypoints --follow-imports skip
105 +mypy vllm/executor --follow-imports skip
106 +mypy vllm/lora --follow-imports skip
107 +mypy vllm/model_executor --follow-imports skip
108 +mypy vllm/prompt_adapter --follow-imports skip
109 +mypy vllm/spec_decode --follow-imports skip
110 +mypy vllm/worker --follow-imports skip
111 +mypy
112
113
114 # If git diff returns a file that is in the skip list, the file may be checked anyway:
pyproject.toml
+16 -2
@@ -48,9 +48,23 @@ python_version = "3.8"
48
49 ignore_missing_imports = true
50 check_untyped_defs = true
51 -follow_imports = "skip"
51 +follow_imports = "silent"
52
53 -files = "vllm"
53 +# After fixing type errors resulting from follow_imports: "skip" -> "silent",
54 +# move the directory here and remove it from format.sh and mypy.yaml
55 +files = [
56 + "vllm/*.py",
57 + "vllm/adapter_commons",
58 + "vllm/assets",
59 + "vllm/inputs",
60 + "vllm/logging",
61 + "vllm/multimodal",
62 + "vllm/platforms",
63 + "vllm/server",
64 + "vllm/transformers_utils",
65 + "vllm/triton_utils",
66 + "vllm/usage",
67 +]
68 # TODO(woosuk): Include the code from Megatron and HuggingFace.
69 exclude = [
70 "vllm/model_executor/parallel_utils/|vllm/model_executor/models/",
vllm/_custom_ops.py
+1 -1
@@ -239,7 +239,7 @@ def cutlass_scaled_mm(a: torch.Tensor,
239 b: torch.Tensor,
240 scale_a: torch.Tensor,
241 scale_b: torch.Tensor,
242 - out_dtype: Type[torch.dtype],
242 + out_dtype: torch.dtype,
243 bias: Optional[torch.Tensor] = None) -> torch.Tensor:
244 assert (b.shape[0] % 16 == 0 and b.shape[1] % 16 == 0)
245 assert (out_dtype is torch.bfloat16 or out_dtype is torch.float16)
vllm/_ipex_ops.py
+49 -13
@@ -25,27 +25,33 @@ class ipex_ops:
25 x2 = x2.reshape(num, d)
26 return x1, x2
27
28 + @staticmethod
29 def silu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
30 x1, x2 = ipex_ops._reshape_activation_tensor(x)
31 ipex.llm.functional.silu_mul(x1, x2, out)
32
33 + @staticmethod
34 def gelu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
35 x1, x2 = ipex_ops._reshape_activation_tensor(x)
36 ipex.llm.functional.gelu_mul(x1, x2, out, "none")
37
38 + @staticmethod
39 def gelu_tanh_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
40 x1, x2 = ipex_ops._reshape_activation_tensor(x)
41 ipex.llm.functional.gelu_mul(x1, x2, out, "tanh")
42
43 + @staticmethod
44 def gelu_fast(out: torch.Tensor, x: torch.Tensor) -> None:
45 out.copy_(torch.nn.functional.gelu(x))
46
47 + @staticmethod
48 def gelu_new(out: torch.Tensor, x: torch.Tensor) -> None:
49 out.copy_(torch.nn.functional.gelu(x))
50
51 # TODO add implementation of gelu_quick here
52 # def gelu_quick(out: torch.Tensor, x: torch.Tensor) -> None:
53
54 + @staticmethod
55 def paged_attention_v1(
56 out: torch.Tensor,
57 query: torch.Tensor,
@@ -78,12 +84,21 @@ class ipex_ops:
84 ).view(num_kv_heads,
85 1).repeat_interleave(num_queries_per_tokens).flatten()
86 # todo: ipex will refactor namespace
81 - torch.xpu.paged_attention_v1(out, query.contiguous(),
82 - key_cache.view_as(value_cache),
83 - value_cache, head_mapping, scale,
84 - block_tables, context_lens, block_size,
85 - max_context_len, alibi_slopes)
87 + torch.xpu.paged_attention_v1( # type: ignore
88 + out,
89 + query.contiguous(),
90 + key_cache.view_as(value_cache),
91 + value_cache,
92 + head_mapping,
93 + scale,
94 + block_tables,
95 + context_lens,
96 + block_size,
97 + max_context_len,
98 + alibi_slopes,
99 + )
100
101 + @staticmethod
102 def paged_attention_v2(
103 out: torch.Tensor,
104 exp_sum: torch.Tensor,
@@ -119,13 +134,24 @@ class ipex_ops:
134 ).view(num_kv_heads,
135 1).repeat_interleave(num_queries_per_tokens).flatten()
136 # todo: ipex will refactor namespace
122 - torch.xpu.paged_attention_v2(out, exp_sum, max_logits, tmp_out,
123 - query.contiguous(),
124 - key_cache.view_as(value_cache),
125 - value_cache, head_mapping, block_tables,
126 - context_lens, scale, block_size,
127 - max_context_len, alibi_slopes)
137 + torch.xpu.paged_attention_v2( # type: ignore
138 + out,
139 + exp_sum,
140 + max_logits,
141 + tmp_out,
142 + query.contiguous(),
143 + key_cache.view_as(value_cache),
144 + value_cache,
145 + head_mapping,
146 + block_tables,
147 + context_lens,
148 + scale,
149 + block_size,
150 + max_context_len,
151 + alibi_slopes,
152 + )
153
154 + @staticmethod
155 def rotary_embedding(
156 positions: torch.Tensor, # [batch_size, seq_len]
157 query: torch.Tensor, # [batch_size, seq_len, num_heads*head_size]
@@ -158,6 +184,7 @@ class ipex_ops:
184 ipex.llm.functional.rotary_embedding(query_rot, key_rot, sin, cos,
185 rotary_dim, is_neox, positions)
186
187 + @staticmethod
188 def batched_rotary_embedding(positions: torch.Tensor, query: torch.Tensor,
189 key: torch.Tensor, head_size: int,
190 cos_sin_cache: torch.Tensor, is_neox: bool,
@@ -189,17 +216,20 @@ class ipex_ops:
216 ipex.llm.functional.rotary_embedding(query_rot, key_rot, sin, cos,
217 rotary_dim, is_neox, positions)
218
219 + @staticmethod
220 def rms_norm(out: torch.Tensor, input: torch.Tensor, weight: torch.Tensor,
221 epsilon: float) -> None:
222 tmp = ipex.llm.functional.rms_norm(input, weight, epsilon)
223 out.copy_(tmp)
224
225 + @staticmethod
226 def fused_add_rms_norm(input: torch.Tensor, residual: torch.Tensor,
227 weight: torch.Tensor, epsilon: float) -> None:
228 tmp = ipex.llm.functional.add_rms_norm(residual, input, weight, None,
229 epsilon, True)
230 input.copy_(tmp)
231
232 + @staticmethod
233 def varlen_attention(
234 query: torch.Tensor,
235 key: torch.Tensor,
@@ -222,6 +252,7 @@ class ipex_ops:
252 softmax_scale, zero_tensors,
253 is_causal, return_softmax, gen_)
254
255 + @staticmethod
256 def reshape_and_cache(
257 key: torch.Tensor,
258 value: torch.Tensor,
@@ -240,8 +271,13 @@ class ipex_ops:
271 def copy_blocks(key_caches: List[torch.Tensor],
272 value_caches: List[torch.Tensor],
273 block_mapping: torch.Tensor) -> None:
243 - torch.xpu.copy_blocks(key_caches, value_caches, block_mapping)
274 + torch.xpu.copy_blocks( # type: ignore
275 + key_caches,
276 + value_caches,
277 + block_mapping,
278 + )
279
280 + @staticmethod
281 def swap_blocks(src: torch.Tensor, dst: torch.Tensor,
282 block_mapping: torch.Tensor) -> None:
247 - torch.xpu.swap_blocks(src, dst, block_mapping)
283 + torch.xpu.swap_blocks(src, dst, block_mapping) # type: ignore
vllm/adapter_commons/models.py
+15 -15
@@ -31,7 +31,7 @@ class AdapterLRUCache(LRUCache[T]):
31 super().__init__(capacity)
32 self.deactivate_fn = deactivate_fn
33
34 - def _on_remove(self, key: Hashable, value: T):
34 + def _on_remove(self, key: Hashable, value: Optional[T]):
35 logger.debug("Removing adapter int id: %d", key)
36 self.deactivate_fn(key)
37 return super()._on_remove(key, value)
@@ -59,46 +59,46 @@ class AdapterModelManager(ABC):
59
60 @property
61 @abstractmethod
62 - def adapter_slots(self):
63 - ...
62 + def adapter_slots(self) -> int:
63 + raise NotImplementedError
64
65 @property
66 @abstractmethod
67 - def capacity(self):
68 - ...
67 + def capacity(self) -> int:
68 + raise NotImplementedError
69
70 @abstractmethod
71 def activate_adapter(self, adapter_id: int) -> bool:
72 - ...
72 + raise NotImplementedError
73
74 @abstractmethod
75 def deactivate_adapter(self, adapter_id: int) -> bool:
76 - ...
76 + raise NotImplementedError
77
78 @abstractmethod
79 def add_adapter(self, adapter: Any) -> bool:
80 - ...
80 + raise NotImplementedError
81
82 @abstractmethod
83 def set_adapter_mapping(self, mapping: Any) -> None:
84 - ...
84 + raise NotImplementedError
85
86 @abstractmethod
87 def remove_adapter(self, adapter_id: int) -> bool:
88 - ...
88 + raise NotImplementedError
89
90 @abstractmethod
91 - def remove_all_adapters(self):
92 - ...
91 + def remove_all_adapters(self) -> None:
92 + raise NotImplementedError
93
94 @abstractmethod
95 def get_adapter(self, adapter_id: int) -> Optional[Any]:
96 - ...
96 + raise NotImplementedError
97
98 @abstractmethod
99 def list_adapters(self) -> Dict[int, Any]:
100 - ...
100 + raise NotImplementedError
101
102 @abstractmethod
103 def pin_adapter(self, adapter_id: int) -> bool:
104 - ...
104 + raise NotImplementedError
vllm/adapter_commons/request.py
+5 -5
@@ -1,19 +1,19 @@
1 -from abc import abstractmethod
1 +from abc import ABC, abstractmethod
2 from dataclasses import dataclass
3
4
5 @dataclass
6 -class AdapterRequest:
6 +class AdapterRequest(ABC):
7 """
8 Base class for adapter requests.
9 """
10
11 @property
12 @abstractmethod
13 - def adapter_id(self):
14 - ...
13 + def adapter_id(self) -> int:
14 + raise NotImplementedError
15
16 - def __post_init__(self):
16 + def __post_init__(self) -> None:
17 if self.adapter_id < 1:
18 raise ValueError(f"id must be > 0, got {self.adapter_id}")
19
vllm/adapter_commons/worker_manager.py
+7 -7
@@ -12,25 +12,25 @@ class AbstractWorkerManager(ABC):
12 @property
13 @abstractmethod
14 def is_enabled(self) -> bool:
15 - ...
15 + raise NotImplementedError
16
17 @abstractmethod
18 def set_active_adapters(self, requests: Set[Any],
19 mapping: Optional[Any]) -> None:
20 - ...
20 + raise NotImplementedError
21
22 @abstractmethod
23 def add_adapter(self, adapter_request: Any) -> bool:
24 - ...
24 + raise NotImplementedError
25
26 @abstractmethod
27 def remove_adapter(self, adapter_id: int) -> bool:
28 - ...
28 + raise NotImplementedError
29
30 @abstractmethod
31 - def remove_all_adapters(self):
32 - ...
31 + def remove_all_adapters(self) -> None:
32 + raise NotImplementedError
33
34 @abstractmethod
35 def list_adapters(self) -> Set[int]:
36 - ...
36 + raise NotImplementedError
vllm/config.py
+2 -1
@@ -724,7 +724,7 @@ class ParallelConfig:
724 backend)
725
726 self._verify_args()
727 - self.rank = 0
727 + self.rank: int = 0
728
729 @property
730 def use_ray(self) -> bool:
@@ -850,6 +850,7 @@ class SchedulerConfig:
850
851
852 class DeviceConfig:
853 + device: Optional[torch.device]
854
855 def __init__(self, device: str = "auto") -> None:
856 if device == "auto":
vllm/engine/llm_engine.py
+6 -8
@@ -5,8 +5,6 @@ from typing import (TYPE_CHECKING, Any, ClassVar, Dict, Iterable, List,
5 from typing import Sequence as GenericSequence
6 from typing import Set, Type, TypeVar, Union
7
8 -from transformers import PreTrainedTokenizer
9 -
8 import vllm.envs as envs
9 from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
10 EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
@@ -40,7 +38,8 @@ from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context,
38 init_tracer)
39 from vllm.transformers_utils.config import try_get_generation_config
40 from vllm.transformers_utils.detokenizer import Detokenizer
43 -from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
41 +from vllm.transformers_utils.tokenizer_group import (AnyTokenizer,
42 + BaseTokenizerGroup,
43 get_tokenizer_group)
44 from vllm.usage.usage_lib import (UsageContext, is_usage_stats_enabled,
45 usage_message)
@@ -477,13 +476,12 @@ class LLMEngine:
476 return self.tokenizer
477
478 def get_tokenizer(
480 - self,
481 - lora_request: Optional[LoRARequest] = None
482 - ) -> "PreTrainedTokenizer":
479 + self,
480 + lora_request: Optional[LoRARequest] = None,
481 + ) -> AnyTokenizer:
482 return self.get_tokenizer_group().get_lora_tokenizer(lora_request)
483
485 - def get_tokenizer_for_seq(self,
486 - sequence: Sequence) -> "PreTrainedTokenizer":
484 + def get_tokenizer_for_seq(self, sequence: Sequence) -> AnyTokenizer:
485 return self.get_tokenizer_group().get_lora_tokenizer(
486 sequence.lora_request)
487
vllm/entrypoints/openai/serving_engine.py
+1 -3
@@ -5,7 +5,6 @@ from http import HTTPStatus
5 from typing import Iterable, Iterator, List, Optional, Tuple, TypedDict, Union
6
7 from pydantic import Field
8 -from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
8 from typing_extensions import Annotated
9
10 from vllm.config import ModelConfig
@@ -30,6 +29,7 @@ from vllm.pooling_params import PoolingParams
29 from vllm.prompt_adapter.request import PromptAdapterRequest
30 from vllm.sampling_params import SamplingParams
31 from vllm.sequence import Logprob
32 +from vllm.transformers_utils.tokenizer_group import AnyTokenizer
33
34 logger = init_logger(__name__)
35
@@ -49,8 +49,6 @@ class LoRAModulePath:
49 AnyRequest = Union[ChatCompletionRequest, CompletionRequest, DetokenizeRequest,
50 EmbeddingRequest, TokenizeRequest]
51
52 -AnyTokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
53 -
52
53 class TextTokensPrompt(TypedDict):
54 prompt: str
vllm/scripts.py
+5 -5
@@ -4,9 +4,10 @@ import asyncio
4 import os
5 import signal
6 import sys
7 -from typing import Optional
7 +from typing import List, Optional
8
9 from openai import OpenAI
10 +from openai.types.chat import ChatCompletionMessageParam
11
12 from vllm.entrypoints.openai.api_server import run_server
13 from vllm.entrypoints.openai.cli_args import make_arg_parser
@@ -63,15 +64,14 @@ def complete(model_name: str, client: OpenAI) -> None:
64
65 def chat(system_prompt: Optional[str], model_name: str,
66 client: OpenAI) -> None:
66 - conversation = []
67 + conversation: List[ChatCompletionMessageParam] = []
68 if system_prompt is not None:
69 conversation.append({"role": "system", "content": system_prompt})
70
71 print("Please enter a message for the chat model:")
72 while True:
73 input_message = input("> ")
73 - message = {"role": "user", "content": input_message}
74 - conversation.append(message)
74 + conversation.append({"role": "user", "content": input_message})
75
76 chat_completion = client.chat.completions.create(model=model_name,
77 messages=conversation)
@@ -79,7 +79,7 @@ def chat(system_prompt: Optional[str], model_name: str,
79 response_message = chat_completion.choices[0].message
80 output = response_message.content
81
82 - conversation.append(response_message)
82 + conversation.append(response_message) # type: ignore
83 print(output)
84
85
vllm/transformers_utils/detokenizer.py
+2
@@ -37,6 +37,8 @@ class Detokenizer:
37 The prompt logprobs with the decoded tokens.
38 """
39 prms = seq_group.sampling_params
40 + assert prms is not None
41 +
42 # We can pick any sequence for the prompt.
43 seq = next(iter(seq_group.seqs_dict.values()))
44 # Only prompt, without the generated token.
vllm/transformers_utils/tokenizer_group/__init__.py
+4 -5
@@ -2,10 +2,9 @@ from typing import Optional, Type
2
3 from vllm.config import TokenizerPoolConfig
4 from vllm.executor.ray_utils import ray
5 -from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
6 - BaseTokenizerGroup)
7 -from vllm.transformers_utils.tokenizer_group.tokenizer_group import (
8 - TokenizerGroup)
5 +
6 +from .base_tokenizer_group import AnyTokenizer, BaseTokenizerGroup
7 +from .tokenizer_group import TokenizerGroup
8
9 if ray:
10 from vllm.transformers_utils.tokenizer_group.ray_tokenizer_group import (
@@ -34,4 +33,4 @@ def get_tokenizer_group(tokenizer_pool_config: Optional[TokenizerPoolConfig],
33 return tokenizer_cls.from_config(tokenizer_pool_config, **init_kwargs)
34
35
37 -__all__ = ["get_tokenizer_group", "BaseTokenizerGroup"]
36 +__all__ = ["AnyTokenizer", "get_tokenizer_group", "BaseTokenizerGroup"]
vllm/transformers_utils/tokenizer_group/base_tokenizer_group.py
+10 -8
@@ -1,11 +1,13 @@
1 from abc import ABC, abstractmethod
2 -from typing import List, Optional
2 +from typing import List, Optional, Union
3
4 -from transformers import PreTrainedTokenizer
4 +from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
5
6 from vllm.config import TokenizerPoolConfig
7 from vllm.lora.request import LoRARequest
8
9 +AnyTokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
10 +
11
12 class BaseTokenizerGroup(ABC):
13 """A group of tokenizers that can be used for LoRA adapters."""
@@ -47,17 +49,17 @@ class BaseTokenizerGroup(ABC):
49
50 @abstractmethod
51 def get_lora_tokenizer(
50 - self,
51 - lora_request: Optional[LoRARequest] = None
52 - ) -> "PreTrainedTokenizer":
52 + self,
53 + lora_request: Optional[LoRARequest] = None,
54 + ) -> AnyTokenizer:
55 """Get a tokenizer for a LoRA request."""
56 pass
57
58 @abstractmethod
59 async def get_lora_tokenizer_async(
58 - self,
59 - lora_request: Optional[LoRARequest] = None
60 - ) -> "PreTrainedTokenizer":
60 + self,
61 + lora_request: Optional[LoRARequest] = None,
62 + ) -> AnyTokenizer:
63 """Get a tokenizer for a LoRA request."""
64 pass
65
vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py
+15 -15
@@ -6,18 +6,16 @@ try:
6 from ray.exceptions import ActorDiedError
7 except ImportError:
8 # For older versions of Ray
9 - from ray.exceptions import RayActorError as ActorDiedError
9 + from ray.exceptions import RayActorError as ActorDiedError # type: ignore
10 from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy
11 -from transformers import PreTrainedTokenizer
11
12 from vllm.config import TokenizerPoolConfig
13 from vllm.executor.ray_utils import ray
14 from vllm.logger import init_logger
15 from vllm.lora.request import LoRARequest
17 -from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
18 - BaseTokenizerGroup)
19 -from vllm.transformers_utils.tokenizer_group.tokenizer_group import (
20 - TokenizerGroup)
16 +
17 +from .base_tokenizer_group import AnyTokenizer, BaseTokenizerGroup
18 +from .tokenizer_group import TokenizerGroup
19
20 logger = init_logger(__name__)
21
@@ -67,7 +65,7 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
65 **self._tokenizer_config, )
66
67 self._ray_tokenizer_group_cls = ray.remote(
70 - self._worker_cls).options(**ray_actor_options)
68 + self._worker_cls).options(**ray_actor_options) # type: ignore
69 self.tokenizer_actors = [self._init_actor() for _ in range(num_actors)]
70 self._idle_actors: Optional[asyncio.Queue] = None
71
@@ -83,8 +81,10 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
81 return len(self.tokenizer_actors)
82
83 def ping(self):
86 - return ray.get(
87 - [actor.ping.remote() for actor in self.tokenizer_actors])
84 + return ray.get([
85 + actor.ping.remote() # type: ignore
86 + for actor in self.tokenizer_actors
87 + ])
88
89 def _ensure_queue_initialized(self):
90 if self._idle_actors is None:
@@ -208,15 +208,15 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
208 return self._local_tokenizer_group.get_max_input_len(lora_request)
209
210 def get_lora_tokenizer(
211 - self,
212 - lora_request: Optional[LoRARequest] = None
213 - ) -> "PreTrainedTokenizer":
211 + self,
212 + lora_request: Optional[LoRARequest] = None,
213 + ) -> AnyTokenizer:
214 return self._local_tokenizer_group.get_lora_tokenizer(lora_request)
215
216 async def get_lora_tokenizer_async(
217 - self,
218 - lora_request: Optional[LoRARequest] = None
219 - ) -> "PreTrainedTokenizer":
217 + self,
218 + lora_request: Optional[LoRARequest] = None,
219 + ) -> AnyTokenizer:
220 return await self._local_tokenizer_group.get_lora_tokenizer_async(
221 lora_request)
222
vllm/transformers_utils/tokenizer_group/tokenizer_group.py
+13 -15
@@ -1,16 +1,14 @@
1 from typing import List, Optional
2
3 -from transformers import PreTrainedTokenizer
4 -
3 from vllm.config import TokenizerPoolConfig
4 from vllm.lora.request import LoRARequest
5 from vllm.transformers_utils.tokenizer import (get_lora_tokenizer,
6 get_lora_tokenizer_async,
7 get_tokenizer)
10 -from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
11 - BaseTokenizerGroup)
8 from vllm.utils import LRUCache
9
10 +from .base_tokenizer_group import AnyTokenizer, BaseTokenizerGroup
11 +
12
13 class TokenizerGroup(BaseTokenizerGroup):
14 """A group of tokenizers that can be used for LoRA adapters."""
@@ -22,8 +20,8 @@ class TokenizerGroup(BaseTokenizerGroup):
20 self.enable_lora = enable_lora
21 self.max_input_length = max_input_length
22 self.tokenizer = get_tokenizer(self.tokenizer_id, **tokenizer_config)
25 - self.lora_tokenizers = LRUCache[PreTrainedTokenizer](
26 - capacity=max_num_seqs) if enable_lora else None
23 + self.lora_tokenizers = LRUCache[AnyTokenizer](
24 + capacity=max_num_seqs if enable_lora else 0)
25
26 @classmethod
27 def from_config(cls, tokenizer_pool_config: Optional[TokenizerPoolConfig],
@@ -41,7 +39,7 @@ class TokenizerGroup(BaseTokenizerGroup):
39 return self.max_input_length
40
41 def _raise_if_input_too_long(self,
44 - encoded_tokens: List[str],
42 + encoded_tokens: List[int],
43 lora_request: Optional[LoRARequest] = None):
44 input_length = len(encoded_tokens)
45 if lora_request:
@@ -72,9 +70,9 @@ class TokenizerGroup(BaseTokenizerGroup):
70 return ret
71
72 def get_lora_tokenizer(
75 - self,
76 - lora_request: Optional[LoRARequest] = None
77 - ) -> "PreTrainedTokenizer":
73 + self,
74 + lora_request: Optional[LoRARequest] = None,
75 + ) -> AnyTokenizer:
76 if not lora_request or not self.enable_lora:
77 return self.tokenizer
78 if lora_request.lora_int_id not in self.lora_tokenizers:
@@ -83,12 +81,12 @@ class TokenizerGroup(BaseTokenizerGroup):
81 self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer)
82 return tokenizer
83 else:
86 - return self.lora_tokenizers.get(lora_request.lora_int_id)
84 + return self.lora_tokenizers[lora_request.lora_int_id]
85
86 async def get_lora_tokenizer_async(
89 - self,
90 - lora_request: Optional[LoRARequest] = None
91 - ) -> "PreTrainedTokenizer":
87 + self,
88 + lora_request: Optional[LoRARequest] = None,
89 + ) -> AnyTokenizer:
90 if not lora_request or not self.enable_lora:
91 return self.tokenizer
92 if lora_request.lora_int_id not in self.lora_tokenizers:
@@ -97,4 +95,4 @@ class TokenizerGroup(BaseTokenizerGroup):
95 self.lora_tokenizers.put(lora_request.lora_int_id, tokenizer)
96 return tokenizer
97 else:
100 - return self.lora_tokenizers.get(lora_request.lora_int_id)
98 + return self.lora_tokenizers[lora_request.lora_int_id]
vllm/utils.py
+8 -5
@@ -94,8 +94,10 @@ class LRUCache(Generic[T]):
94 def __len__(self) -> int:
95 return len(self.cache)
96
97 - def __getitem__(self, key: Hashable) -> Optional[T]:
98 - return self.get(key)
97 + def __getitem__(self, key: Hashable) -> T:
98 + value = self.cache[key] # Raise KeyError if not exists
99 + self.cache.move_to_end(key)
100 + return value
101
102 def __setitem__(self, key: Hashable, value: T) -> None:
103 self.put(key, value)
@@ -109,8 +111,9 @@ class LRUCache(Generic[T]):
111 def get(self,
112 key: Hashable,
113 default_value: Optional[T] = None) -> Optional[T]:
114 + value: Optional[T]
115 if key in self.cache:
113 - value: Optional[T] = self.cache[key]
116 + value = self.cache[key]
117 self.cache.move_to_end(key)
118 else:
119 value = default_value
@@ -590,8 +593,8 @@ class CudaMemoryProfiler:
593 torch.cuda.reset_peak_memory_stats(self.device)
594 mem = torch.cuda.max_memory_allocated(self.device)
595 elif is_xpu():
593 - torch.xpu.reset_peak_memory_stats(self.device)
594 - mem = torch.xpu.max_memory_allocated(self.device)
596 + torch.xpu.reset_peak_memory_stats(self.device) # type: ignore
597 + mem = torch.xpu.max_memory_allocated(self.device) # type: ignore
598 return mem
599
600 def __enter__(self):