A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[mypy] Misc. typing improvements (#7417)
Cyrus Leung committed
Aug 13, 2024 at 09:20 UTC
9ba85bc1527928f30ec56961520d7e07ee385167
16 files changed
+74
-75
tests/tensorizer_loader/conftest.py
+12
-4
@@ -1,10 +1,12 @@
1
import contextlib
2
import functools
3
import gc
4
+from typing import Callable, TypeVar
5
6
import pytest
7
import ray
8
import torch
9
+from typing_extensions import ParamSpec
10
11
from vllm.distributed import (destroy_distributed_environment,
12
destroy_model_parallel)
@@ -22,12 +24,16 @@ def cleanup():
24
torch.cuda.empty_cache()
25
26
25
-def retry_until_skip(n):
27
+_P = ParamSpec("_P")
28
+_R = TypeVar("_R")
29
27
- def decorator_retry(func):
30
+
31
+def retry_until_skip(n: int):
32
+
33
+ def decorator_retry(func: Callable[_P, _R]) -> Callable[_P, _R]:
34
35
@functools.wraps(func)
30
- def wrapper_retry(*args, **kwargs):
36
+ def wrapper_retry(*args: _P.args, **kwargs: _P.kwargs) -> _R:
37
for i in range(n):
38
try:
39
return func(*args, **kwargs)
@@ -35,7 +41,9 @@ def retry_until_skip(n):
41
gc.collect()
42
torch.cuda.empty_cache()
43
if i == n - 1:
38
- pytest.skip("Skipping test after attempts..")
44
+ pytest.skip(f"Skipping test after {n} attempts.")
45
+
46
+ raise AssertionError("Code should not be reached")
47
48
return wrapper_retry
49
tests/test_utils.py
+8
-22
@@ -1,10 +1,8 @@
1
import asyncio
2
import os
3
import socket
4
-import sys
4
from functools import partial
6
-from typing import (TYPE_CHECKING, Any, AsyncIterator, Awaitable, Protocol,
7
- Tuple, TypeVar)
5
+from typing import AsyncIterator, Tuple
6
7
import pytest
8
@@ -13,26 +11,11 @@ from vllm.utils import (FlexibleArgumentParser, deprecate_kwargs,
11
12
from .utils import error_on_warning
13
16
-if sys.version_info < (3, 10):
17
- if TYPE_CHECKING:
18
- _AwaitableT = TypeVar("_AwaitableT", bound=Awaitable[Any])
19
- _AwaitableT_co = TypeVar("_AwaitableT_co",
20
- bound=Awaitable[Any],
21
- covariant=True)
22
-
23
- class _SupportsSynchronousAnext(Protocol[_AwaitableT_co]):
24
-
25
- def __anext__(self) -> _AwaitableT_co:
26
- ...
27
-
28
- def anext(i: "_SupportsSynchronousAnext[_AwaitableT]", /) -> "_AwaitableT":
29
- return i.__anext__()
30
-
14
15
@pytest.mark.asyncio
16
async def test_merge_async_iterators():
17
35
- async def mock_async_iterator(idx: int) -> AsyncIterator[str]:
18
+ async def mock_async_iterator(idx: int):
19
try:
20
while True:
21
yield f"item from iterator {idx}"
@@ -41,8 +24,10 @@ async def test_merge_async_iterators():
24
print(f"iterator {idx} cancelled")
25
26
iterators = [mock_async_iterator(i) for i in range(3)]
44
- merged_iterator: AsyncIterator[Tuple[int, str]] = merge_async_iterators(
45
- *iterators, is_cancelled=partial(asyncio.sleep, 0, result=False))
27
+ merged_iterator = merge_async_iterators(*iterators,
28
+ is_cancelled=partial(asyncio.sleep,
29
+ 0,
30
+ result=False))
31
32
async def stream_output(generator: AsyncIterator[Tuple[int, str]]):
33
async for idx, output in generator:
@@ -56,7 +41,8 @@ async def test_merge_async_iterators():
41
42
for iterator in iterators:
43
try:
59
- await asyncio.wait_for(anext(iterator), 1)
44
+ # Can use anext() in python >= 3.10
45
+ await asyncio.wait_for(iterator.__anext__(), 1)
46
except StopAsyncIteration:
47
# All iterators should be cancelled and print this message.
48
print("Iterator was cancelled normally")
tests/utils.py
+8
-3
@@ -7,12 +7,13 @@ import time
7
import warnings
8
from contextlib import contextmanager
9
from pathlib import Path
10
-from typing import Any, Dict, List, Optional
10
+from typing import Any, Callable, Dict, List, Optional
11
12
import openai
13
import ray
14
import requests
15
from transformers import AutoTokenizer
16
+from typing_extensions import ParamSpec
17
18
from vllm.distributed import (ensure_model_parallel_initialized,
19
init_distributed_environment)
@@ -360,13 +361,17 @@ def wait_for_gpu_memory_to_clear(devices: List[int],
361
time.sleep(5)
362
363
363
-def fork_new_process_for_each_test(f):
364
+_P = ParamSpec("_P")
365
+
366
+
367
+def fork_new_process_for_each_test(
368
+ f: Callable[_P, None]) -> Callable[_P, None]:
369
"""Decorator to fork a new process for each test function.
370
See https://github.com/vllm-project/vllm/issues/7053 for more details.
371
"""
372
373
@functools.wraps(f)
369
- def wrapper(*args, **kwargs):
374
+ def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None:
375
# Make the process the leader of its own process group
376
# to avoid sending SIGTERM to the parent process
377
os.setpgrp()
vllm/inputs/registry.py
+4
-4
@@ -1,10 +1,10 @@
1
import functools
2
from dataclasses import dataclass
3
-from typing import (TYPE_CHECKING, Callable, Dict, Optional, Tuple, Type,
4
- TypeVar)
3
+from typing import TYPE_CHECKING, Callable, Dict, Optional, Tuple, Type
4
5
from torch import nn
6
from transformers import PretrainedConfig
7
+from typing_extensions import TypeVar
8
9
from vllm.logger import init_logger
10
@@ -17,7 +17,7 @@ if TYPE_CHECKING:
17
18
logger = init_logger(__name__)
19
20
-C = TypeVar("C", bound=PretrainedConfig)
20
+C = TypeVar("C", bound=PretrainedConfig, default=PretrainedConfig)
21
22
23
@dataclass(frozen=True)
@@ -44,7 +44,7 @@ class InputContext:
44
45
return multimodal_config
46
47
- def get_hf_config(self, hf_config_type: Type[C]) -> C:
47
+ def get_hf_config(self, hf_config_type: Type[C] = PretrainedConfig) -> C:
48
"""
49
Get the HuggingFace configuration
50
(:class:`transformers.PretrainedConfig`) of the model,
vllm/model_executor/models/internvl.py
+3
-3
@@ -165,7 +165,7 @@ def get_internvl_num_patches(image_size: int, patch_size: int,
165
166
167
def get_max_internvl_image_tokens(ctx: InputContext):
168
- hf_config = ctx.get_hf_config(PretrainedConfig)
168
+ hf_config = ctx.get_hf_config()
169
vision_config = hf_config.vision_config
170
171
use_thumbnail = hf_config.use_thumbnail
@@ -187,7 +187,7 @@ def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
187
return llm_inputs
188
189
model_config = ctx.model_config
190
- hf_config = ctx.get_hf_config(PretrainedConfig)
190
+ hf_config = ctx.get_hf_config()
191
vision_config = hf_config.vision_config
192
193
image_size = vision_config.image_size
@@ -260,7 +260,7 @@ def dummy_data_for_internvl(ctx: InputContext, seq_len: int):
260
261
image_feature_size = get_max_internvl_image_tokens(ctx)
262
model_config = ctx.model_config
263
- hf_config = ctx.get_hf_config(PretrainedConfig)
263
+ hf_config = ctx.get_hf_config()
264
vision_config = hf_config.vision_config
265
tokenizer = cached_get_tokenizer(model_config.tokenizer,
266
trust_remote_code=True)
vllm/model_executor/models/minicpmv.py
+3
-3
@@ -34,7 +34,7 @@ import torch.types
34
from PIL import Image
35
from torch import nn
36
from torch.nn.init import trunc_normal_
37
-from transformers.configuration_utils import PretrainedConfig
37
+from transformers import PretrainedConfig
38
39
from vllm.attention import AttentionMetadata
40
from vllm.config import CacheConfig, MultiModalConfig
@@ -404,7 +404,7 @@ def get_version_by_config(config: PretrainedConfig) -> Tuple[int, ...]:
404
405
406
def get_max_minicpmv_image_tokens(ctx: InputContext):
407
- hf_config = ctx.get_hf_config(PretrainedConfig)
407
+ hf_config = ctx.get_hf_config()
408
return getattr(hf_config, "query_num", 64)
409
410
@@ -420,7 +420,7 @@ def dummy_image_for_minicpmv(hf_config: PretrainedConfig):
420
421
422
def dummy_data_for_minicpmv(ctx: InputContext, seq_len: int):
423
- hf_config = ctx.get_hf_config(PretrainedConfig)
423
+ hf_config = ctx.get_hf_config()
424
425
seq_data = dummy_seq_data_for_minicpmv(seq_len)
426
mm_data = dummy_image_for_minicpmv(hf_config)
vllm/model_executor/models/phi3v.py
+2
-2
@@ -341,7 +341,7 @@ def get_phi3v_image_feature_size(
341
def get_max_phi3v_image_tokens(ctx: InputContext):
342
343
return get_phi3v_image_feature_size(
344
- ctx.get_hf_config(PretrainedConfig),
344
+ ctx.get_hf_config(),
345
input_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
346
input_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
347
)
@@ -391,7 +391,7 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):
391
return llm_inputs
392
393
model_config = ctx.model_config
394
- hf_config = ctx.get_hf_config(PretrainedConfig)
394
+ hf_config = ctx.get_hf_config()
395
396
image_data = multi_modal_data["image"]
397
if isinstance(image_data, Image.Image):
vllm/multimodal/image.py
+2
-3
@@ -3,13 +3,12 @@ from typing import List, Optional, Tuple, TypeVar
3
4
import torch
5
from PIL import Image
6
-from transformers import PreTrainedTokenizerBase
6
7
from vllm.config import ModelConfig
8
from vllm.inputs.registry import InputContext
9
from vllm.logger import init_logger
10
from vllm.transformers_utils.image_processor import get_image_processor
12
-from vllm.transformers_utils.tokenizer import get_tokenizer
11
+from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
12
from vllm.utils import is_list_of
13
14
from .base import MultiModalInputs, MultiModalPlugin
@@ -40,7 +39,7 @@ def repeat_and_pad_token(
39
40
41
def repeat_and_pad_image_tokens(
43
- tokenizer: PreTrainedTokenizerBase,
42
+ tokenizer: AnyTokenizer,
43
prompt: Optional[str],
44
prompt_token_ids: List[int],
45
*,
vllm/platforms/cuda.py
+7
-3
@@ -4,9 +4,10 @@ pynvml. However, it should not initialize cuda context.
4
5
import os
6
from functools import lru_cache, wraps
7
-from typing import List, Tuple
7
+from typing import Callable, List, Tuple, TypeVar
8
9
import pynvml
10
+from typing_extensions import ParamSpec
11
12
from vllm.logger import init_logger
13
@@ -14,16 +15,19 @@ from .interface import Platform, PlatformEnum
15
16
logger = init_logger(__name__)
17
18
+_P = ParamSpec("_P")
19
+_R = TypeVar("_R")
20
+
21
# NVML utils
22
# Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
23
# all the related functions work on real physical device ids.
24
# the major benefit of using NVML is that it will not initialize CUDA
25
26
23
-def with_nvml_context(fn):
27
+def with_nvml_context(fn: Callable[_P, _R]) -> Callable[_P, _R]:
28
29
@wraps(fn)
26
- def wrapper(*args, **kwargs):
30
+ def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R:
31
pynvml.nvmlInit()
32
try:
33
return fn(*args, **kwargs)
vllm/transformers_utils/detokenizer.py
+8
-10
@@ -1,10 +1,9 @@
1
-from typing import Dict, List, Optional, Tuple, Union
2
-
3
-from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
1
+from typing import Dict, List, Optional, Tuple
2
3
from vllm.sequence import Logprob, SamplingParams, Sequence, SequenceGroup
6
-from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
7
- BaseTokenizerGroup)
4
+
5
+from .tokenizer import AnyTokenizer
6
+from .tokenizer_group import BaseTokenizerGroup
7
8
# Used eg. for marking rejected tokens in spec decoding.
9
INVALID_TOKEN_ID = -1
@@ -16,8 +15,7 @@ class Detokenizer:
15
def __init__(self, tokenizer_group: BaseTokenizerGroup):
16
self.tokenizer_group = tokenizer_group
17
19
- def get_tokenizer_for_seq(self,
20
- sequence: Sequence) -> "PreTrainedTokenizer":
18
+ def get_tokenizer_for_seq(self, sequence: Sequence) -> AnyTokenizer:
19
"""Returns the HF tokenizer to use for a given sequence."""
20
return self.tokenizer_group.get_lora_tokenizer(sequence.lora_request)
21
@@ -174,7 +172,7 @@ def _replace_none_with_empty(tokens: List[Optional[str]]):
172
173
174
def _convert_tokens_to_string_with_added_encoders(
177
- tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
175
+ tokenizer: AnyTokenizer,
176
output_tokens: List[str],
177
skip_special_tokens: bool,
178
spaces_between_special_tokens: bool,
@@ -213,7 +211,7 @@ INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET = 5
211
212
213
def convert_prompt_ids_to_tokens(
216
- tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
214
+ tokenizer: AnyTokenizer,
215
prompt_ids: List[int],
216
skip_special_tokens: bool = False,
217
) -> Tuple[List[str], int, int]:
@@ -240,7 +238,7 @@ def convert_prompt_ids_to_tokens(
238
# https://github.com/huggingface/text-generation-inference/blob/v0.9.4/server/text_generation_server/models/model.py#L62C9-L62C15
239
# under Apache 2.0 license
240
def detokenize_incrementally(
243
- tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
241
+ tokenizer: AnyTokenizer,
242
all_input_ids: List[int],
243
prev_tokens: Optional[List[str]],
244
prefix_offset: int,
vllm/transformers_utils/tokenizer.py
+3
-3
@@ -12,10 +12,10 @@ from vllm.lora.request import LoRARequest
12
from vllm.transformers_utils.tokenizers import BaichuanTokenizer
13
from vllm.utils import make_async
14
15
-from .tokenizer_group import AnyTokenizer
16
-
15
logger = init_logger(__name__)
16
17
+AnyTokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
18
+
19
20
def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
21
"""Get tokenizer with cached properties.
@@ -141,7 +141,7 @@ def get_tokenizer(
141
142
143
def get_lora_tokenizer(lora_request: LoRARequest, *args,
144
- **kwargs) -> Optional[PreTrainedTokenizer]:
144
+ **kwargs) -> Optional[AnyTokenizer]:
145
if lora_request is None:
146
return None
147
try:
vllm/transformers_utils/tokenizer_group/__init__.py
+1
-2
@@ -8,8 +8,7 @@ from .base_tokenizer_group import AnyTokenizer, BaseTokenizerGroup
8
from .tokenizer_group import TokenizerGroup
9
10
if ray:
11
- from vllm.transformers_utils.tokenizer_group.ray_tokenizer_group import (
12
- RayTokenizerGroupPool)
11
+ from .ray_tokenizer_group import RayTokenizerGroupPool
12
else:
13
RayTokenizerGroupPool = None # type: ignore
14
vllm/transformers_utils/tokenizer_group/base_tokenizer_group.py
+6
-8
@@ -1,12 +1,9 @@
1
from abc import ABC, abstractmethod
2
-from typing import List, Optional, Union
3
-
4
-from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
2
+from typing import List, Optional
3
4
from vllm.config import TokenizerPoolConfig
5
from vllm.lora.request import LoRARequest
8
-
9
-AnyTokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
6
+from vllm.transformers_utils.tokenizer import AnyTokenizer
7
8
9
class BaseTokenizerGroup(ABC):
@@ -24,9 +21,10 @@ class BaseTokenizerGroup(ABC):
21
pass
22
23
@abstractmethod
27
- def get_max_input_len(self,
28
- lora_request: Optional[LoRARequest] = None
29
- ) -> Optional[int]:
24
+ def get_max_input_len(
25
+ self,
26
+ lora_request: Optional[LoRARequest] = None,
27
+ ) -> Optional[int]:
28
"""Get the maximum input length for the LoRA request."""
29
pass
30
vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py
+2
-1
@@ -13,8 +13,9 @@ from vllm.config import TokenizerPoolConfig
13
from vllm.executor.ray_utils import ray
14
from vllm.logger import init_logger
15
from vllm.lora.request import LoRARequest
16
+from vllm.transformers_utils.tokenizer import AnyTokenizer
17
17
-from .base_tokenizer_group import AnyTokenizer, BaseTokenizerGroup
18
+from .base_tokenizer_group import BaseTokenizerGroup
19
from .tokenizer_group import TokenizerGroup
20
21
logger = init_logger(__name__)
vllm/transformers_utils/tokenizer_group/tokenizer_group.py
+3
-2
@@ -2,12 +2,13 @@ from typing import List, Optional
2
3
from vllm.config import TokenizerPoolConfig
4
from vllm.lora.request import LoRARequest
5
-from vllm.transformers_utils.tokenizer import (get_lora_tokenizer,
5
+from vllm.transformers_utils.tokenizer import (AnyTokenizer,
6
+ get_lora_tokenizer,
7
get_lora_tokenizer_async,
8
get_tokenizer)
9
from vllm.utils import LRUCache
10
10
-from .base_tokenizer_group import AnyTokenizer, BaseTokenizerGroup
11
+from .base_tokenizer_group import BaseTokenizerGroup
12
13
14
class TokenizerGroup(BaseTokenizerGroup):
vllm/utils.py
+2
-2
@@ -1101,9 +1101,9 @@ def cuda_device_count_stateless() -> int:
1101
1102
1103
#From: https://stackoverflow.com/a/4104188/2749989
1104
-def run_once(f):
1104
+def run_once(f: Callable[P, None]) -> Callable[P, None]:
1105
1106
- def wrapper(*args, **kwargs) -> Any:
1106
+ def wrapper(*args: P.args, **kwargs: P.kwargs) -> None:
1107
if not wrapper.has_run: # type: ignore[attr-defined]
1108
wrapper.has_run = True # type: ignore[attr-defined]
1109
return f(*args, **kwargs)