@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9ba85bc152
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[mypy] Misc. typing improvements (#7417)

Cyrus Leung committed Aug 13, 2024 at 09:20 UTC 9ba85bc1527928f30ec56961520d7e07ee385167
16 files changed +74 -75
tests/tensorizer_loader/conftest.py
+12 -4
@@ -1,10 +1,12 @@
1 import contextlib
2 import functools
3 import gc
4 +from typing import Callable, TypeVar
5
6 import pytest
7 import ray
8 import torch
9 +from typing_extensions import ParamSpec
10
11 from vllm.distributed import (destroy_distributed_environment,
12 destroy_model_parallel)
@@ -22,12 +24,16 @@ def cleanup():
24 torch.cuda.empty_cache()
25
26
25 -def retry_until_skip(n):
27 +_P = ParamSpec("_P")
28 +_R = TypeVar("_R")
29
27 - def decorator_retry(func):
30 +
31 +def retry_until_skip(n: int):
32 +
33 + def decorator_retry(func: Callable[_P, _R]) -> Callable[_P, _R]:
34
35 @functools.wraps(func)
30 - def wrapper_retry(*args, **kwargs):
36 + def wrapper_retry(*args: _P.args, **kwargs: _P.kwargs) -> _R:
37 for i in range(n):
38 try:
39 return func(*args, **kwargs)
@@ -35,7 +41,9 @@ def retry_until_skip(n):
41 gc.collect()
42 torch.cuda.empty_cache()
43 if i == n - 1:
38 - pytest.skip("Skipping test after attempts..")
44 + pytest.skip(f"Skipping test after {n} attempts.")
45 +
46 + raise AssertionError("Code should not be reached")
47
48 return wrapper_retry
49
tests/test_utils.py
+8 -22
@@ -1,10 +1,8 @@
1 import asyncio
2 import os
3 import socket
4 -import sys
4 from functools import partial
6 -from typing import (TYPE_CHECKING, Any, AsyncIterator, Awaitable, Protocol,
7 - Tuple, TypeVar)
5 +from typing import AsyncIterator, Tuple
6
7 import pytest
8
@@ -13,26 +11,11 @@ from vllm.utils import (FlexibleArgumentParser, deprecate_kwargs,
11
12 from .utils import error_on_warning
13
16 -if sys.version_info < (3, 10):
17 - if TYPE_CHECKING:
18 - _AwaitableT = TypeVar("_AwaitableT", bound=Awaitable[Any])
19 - _AwaitableT_co = TypeVar("_AwaitableT_co",
20 - bound=Awaitable[Any],
21 - covariant=True)
22 -
23 - class _SupportsSynchronousAnext(Protocol[_AwaitableT_co]):
24 -
25 - def __anext__(self) -> _AwaitableT_co:
26 - ...
27 -
28 - def anext(i: "_SupportsSynchronousAnext[_AwaitableT]", /) -> "_AwaitableT":
29 - return i.__anext__()
30 -
14
15 @pytest.mark.asyncio
16 async def test_merge_async_iterators():
17
35 - async def mock_async_iterator(idx: int) -> AsyncIterator[str]:
18 + async def mock_async_iterator(idx: int):
19 try:
20 while True:
21 yield f"item from iterator {idx}"
@@ -41,8 +24,10 @@ async def test_merge_async_iterators():
24 print(f"iterator {idx} cancelled")
25
26 iterators = [mock_async_iterator(i) for i in range(3)]
44 - merged_iterator: AsyncIterator[Tuple[int, str]] = merge_async_iterators(
45 - *iterators, is_cancelled=partial(asyncio.sleep, 0, result=False))
27 + merged_iterator = merge_async_iterators(*iterators,
28 + is_cancelled=partial(asyncio.sleep,
29 + 0,
30 + result=False))
31
32 async def stream_output(generator: AsyncIterator[Tuple[int, str]]):
33 async for idx, output in generator:
@@ -56,7 +41,8 @@ async def test_merge_async_iterators():
41
42 for iterator in iterators:
43 try:
59 - await asyncio.wait_for(anext(iterator), 1)
44 + # Can use anext() in python >= 3.10
45 + await asyncio.wait_for(iterator.__anext__(), 1)
46 except StopAsyncIteration:
47 # All iterators should be cancelled and print this message.
48 print("Iterator was cancelled normally")
tests/utils.py
+8 -3
@@ -7,12 +7,13 @@ import time
7 import warnings
8 from contextlib import contextmanager
9 from pathlib import Path
10 -from typing import Any, Dict, List, Optional
10 +from typing import Any, Callable, Dict, List, Optional
11
12 import openai
13 import ray
14 import requests
15 from transformers import AutoTokenizer
16 +from typing_extensions import ParamSpec
17
18 from vllm.distributed import (ensure_model_parallel_initialized,
19 init_distributed_environment)
@@ -360,13 +361,17 @@ def wait_for_gpu_memory_to_clear(devices: List[int],
361 time.sleep(5)
362
363
363 -def fork_new_process_for_each_test(f):
364 +_P = ParamSpec("_P")
365 +
366 +
367 +def fork_new_process_for_each_test(
368 + f: Callable[_P, None]) -> Callable[_P, None]:
369 """Decorator to fork a new process for each test function.
370 See https://github.com/vllm-project/vllm/issues/7053 for more details.
371 """
372
373 @functools.wraps(f)
369 - def wrapper(*args, **kwargs):
374 + def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None:
375 # Make the process the leader of its own process group
376 # to avoid sending SIGTERM to the parent process
377 os.setpgrp()
vllm/inputs/registry.py
+4 -4
@@ -1,10 +1,10 @@
1 import functools
2 from dataclasses import dataclass
3 -from typing import (TYPE_CHECKING, Callable, Dict, Optional, Tuple, Type,
4 - TypeVar)
3 +from typing import TYPE_CHECKING, Callable, Dict, Optional, Tuple, Type
4
5 from torch import nn
6 from transformers import PretrainedConfig
7 +from typing_extensions import TypeVar
8
9 from vllm.logger import init_logger
10
@@ -17,7 +17,7 @@ if TYPE_CHECKING:
17
18 logger = init_logger(__name__)
19
20 -C = TypeVar("C", bound=PretrainedConfig)
20 +C = TypeVar("C", bound=PretrainedConfig, default=PretrainedConfig)
21
22
23 @dataclass(frozen=True)
@@ -44,7 +44,7 @@ class InputContext:
44
45 return multimodal_config
46
47 - def get_hf_config(self, hf_config_type: Type[C]) -> C:
47 + def get_hf_config(self, hf_config_type: Type[C] = PretrainedConfig) -> C:
48 """
49 Get the HuggingFace configuration
50 (:class:`transformers.PretrainedConfig`) of the model,
vllm/model_executor/models/internvl.py
+3 -3
@@ -165,7 +165,7 @@ def get_internvl_num_patches(image_size: int, patch_size: int,
165
166
167 def get_max_internvl_image_tokens(ctx: InputContext):
168 - hf_config = ctx.get_hf_config(PretrainedConfig)
168 + hf_config = ctx.get_hf_config()
169 vision_config = hf_config.vision_config
170
171 use_thumbnail = hf_config.use_thumbnail
@@ -187,7 +187,7 @@ def input_processor_for_internvl(ctx: InputContext, llm_inputs: LLMInputs):
187 return llm_inputs
188
189 model_config = ctx.model_config
190 - hf_config = ctx.get_hf_config(PretrainedConfig)
190 + hf_config = ctx.get_hf_config()
191 vision_config = hf_config.vision_config
192
193 image_size = vision_config.image_size
@@ -260,7 +260,7 @@ def dummy_data_for_internvl(ctx: InputContext, seq_len: int):
260
261 image_feature_size = get_max_internvl_image_tokens(ctx)
262 model_config = ctx.model_config
263 - hf_config = ctx.get_hf_config(PretrainedConfig)
263 + hf_config = ctx.get_hf_config()
264 vision_config = hf_config.vision_config
265 tokenizer = cached_get_tokenizer(model_config.tokenizer,
266 trust_remote_code=True)
vllm/model_executor/models/minicpmv.py
+3 -3
@@ -34,7 +34,7 @@ import torch.types
34 from PIL import Image
35 from torch import nn
36 from torch.nn.init import trunc_normal_
37 -from transformers.configuration_utils import PretrainedConfig
37 +from transformers import PretrainedConfig
38
39 from vllm.attention import AttentionMetadata
40 from vllm.config import CacheConfig, MultiModalConfig
@@ -404,7 +404,7 @@ def get_version_by_config(config: PretrainedConfig) -> Tuple[int, ...]:
404
405
406 def get_max_minicpmv_image_tokens(ctx: InputContext):
407 - hf_config = ctx.get_hf_config(PretrainedConfig)
407 + hf_config = ctx.get_hf_config()
408 return getattr(hf_config, "query_num", 64)
409
410
@@ -420,7 +420,7 @@ def dummy_image_for_minicpmv(hf_config: PretrainedConfig):
420
421
422 def dummy_data_for_minicpmv(ctx: InputContext, seq_len: int):
423 - hf_config = ctx.get_hf_config(PretrainedConfig)
423 + hf_config = ctx.get_hf_config()
424
425 seq_data = dummy_seq_data_for_minicpmv(seq_len)
426 mm_data = dummy_image_for_minicpmv(hf_config)
vllm/model_executor/models/phi3v.py
+2 -2
@@ -341,7 +341,7 @@ def get_phi3v_image_feature_size(
341 def get_max_phi3v_image_tokens(ctx: InputContext):
342
343 return get_phi3v_image_feature_size(
344 - ctx.get_hf_config(PretrainedConfig),
344 + ctx.get_hf_config(),
345 input_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
346 input_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
347 )
@@ -391,7 +391,7 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):
391 return llm_inputs
392
393 model_config = ctx.model_config
394 - hf_config = ctx.get_hf_config(PretrainedConfig)
394 + hf_config = ctx.get_hf_config()
395
396 image_data = multi_modal_data["image"]
397 if isinstance(image_data, Image.Image):
vllm/multimodal/image.py
+2 -3
@@ -3,13 +3,12 @@ from typing import List, Optional, Tuple, TypeVar
3
4 import torch
5 from PIL import Image
6 -from transformers import PreTrainedTokenizerBase
6
7 from vllm.config import ModelConfig
8 from vllm.inputs.registry import InputContext
9 from vllm.logger import init_logger
10 from vllm.transformers_utils.image_processor import get_image_processor
12 -from vllm.transformers_utils.tokenizer import get_tokenizer
11 +from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
12 from vllm.utils import is_list_of
13
14 from .base import MultiModalInputs, MultiModalPlugin
@@ -40,7 +39,7 @@ def repeat_and_pad_token(
39
40
41 def repeat_and_pad_image_tokens(
43 - tokenizer: PreTrainedTokenizerBase,
42 + tokenizer: AnyTokenizer,
43 prompt: Optional[str],
44 prompt_token_ids: List[int],
45 *,
vllm/platforms/cuda.py
+7 -3
@@ -4,9 +4,10 @@ pynvml. However, it should not initialize cuda context.
4
5 import os
6 from functools import lru_cache, wraps
7 -from typing import List, Tuple
7 +from typing import Callable, List, Tuple, TypeVar
8
9 import pynvml
10 +from typing_extensions import ParamSpec
11
12 from vllm.logger import init_logger
13
@@ -14,16 +15,19 @@ from .interface import Platform, PlatformEnum
15
16 logger = init_logger(__name__)
17
18 +_P = ParamSpec("_P")
19 +_R = TypeVar("_R")
20 +
21 # NVML utils
22 # Note that NVML is not affected by `CUDA_VISIBLE_DEVICES`,
23 # all the related functions work on real physical device ids.
24 # the major benefit of using NVML is that it will not initialize CUDA
25
26
23 -def with_nvml_context(fn):
27 +def with_nvml_context(fn: Callable[_P, _R]) -> Callable[_P, _R]:
28
29 @wraps(fn)
26 - def wrapper(*args, **kwargs):
30 + def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R:
31 pynvml.nvmlInit()
32 try:
33 return fn(*args, **kwargs)
vllm/transformers_utils/detokenizer.py
+8 -10
@@ -1,10 +1,9 @@
1 -from typing import Dict, List, Optional, Tuple, Union
2 -
3 -from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
1 +from typing import Dict, List, Optional, Tuple
2
3 from vllm.sequence import Logprob, SamplingParams, Sequence, SequenceGroup
6 -from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
7 - BaseTokenizerGroup)
4 +
5 +from .tokenizer import AnyTokenizer
6 +from .tokenizer_group import BaseTokenizerGroup
7
8 # Used eg. for marking rejected tokens in spec decoding.
9 INVALID_TOKEN_ID = -1
@@ -16,8 +15,7 @@ class Detokenizer:
15 def __init__(self, tokenizer_group: BaseTokenizerGroup):
16 self.tokenizer_group = tokenizer_group
17
19 - def get_tokenizer_for_seq(self,
20 - sequence: Sequence) -> "PreTrainedTokenizer":
18 + def get_tokenizer_for_seq(self, sequence: Sequence) -> AnyTokenizer:
19 """Returns the HF tokenizer to use for a given sequence."""
20 return self.tokenizer_group.get_lora_tokenizer(sequence.lora_request)
21
@@ -174,7 +172,7 @@ def _replace_none_with_empty(tokens: List[Optional[str]]):
172
173
174 def _convert_tokens_to_string_with_added_encoders(
177 - tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
175 + tokenizer: AnyTokenizer,
176 output_tokens: List[str],
177 skip_special_tokens: bool,
178 spaces_between_special_tokens: bool,
@@ -213,7 +211,7 @@ INITIAL_INCREMENTAL_DETOKENIZATION_OFFSET = 5
211
212
213 def convert_prompt_ids_to_tokens(
216 - tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
214 + tokenizer: AnyTokenizer,
215 prompt_ids: List[int],
216 skip_special_tokens: bool = False,
217 ) -> Tuple[List[str], int, int]:
@@ -240,7 +238,7 @@ def convert_prompt_ids_to_tokens(
238 # https://github.com/huggingface/text-generation-inference/blob/v0.9.4/server/text_generation_server/models/model.py#L62C9-L62C15
239 # under Apache 2.0 license
240 def detokenize_incrementally(
243 - tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
241 + tokenizer: AnyTokenizer,
242 all_input_ids: List[int],
243 prev_tokens: Optional[List[str]],
244 prefix_offset: int,
vllm/transformers_utils/tokenizer.py
+3 -3
@@ -12,10 +12,10 @@ from vllm.lora.request import LoRARequest
12 from vllm.transformers_utils.tokenizers import BaichuanTokenizer
13 from vllm.utils import make_async
14
15 -from .tokenizer_group import AnyTokenizer
16 -
15 logger = init_logger(__name__)
16
17 +AnyTokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
18 +
19
20 def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
21 """Get tokenizer with cached properties.
@@ -141,7 +141,7 @@ def get_tokenizer(
141
142
143 def get_lora_tokenizer(lora_request: LoRARequest, *args,
144 - **kwargs) -> Optional[PreTrainedTokenizer]:
144 + **kwargs) -> Optional[AnyTokenizer]:
145 if lora_request is None:
146 return None
147 try:
vllm/transformers_utils/tokenizer_group/__init__.py
+1 -2
@@ -8,8 +8,7 @@ from .base_tokenizer_group import AnyTokenizer, BaseTokenizerGroup
8 from .tokenizer_group import TokenizerGroup
9
10 if ray:
11 - from vllm.transformers_utils.tokenizer_group.ray_tokenizer_group import (
12 - RayTokenizerGroupPool)
11 + from .ray_tokenizer_group import RayTokenizerGroupPool
12 else:
13 RayTokenizerGroupPool = None # type: ignore
14
vllm/transformers_utils/tokenizer_group/base_tokenizer_group.py
+6 -8
@@ -1,12 +1,9 @@
1 from abc import ABC, abstractmethod
2 -from typing import List, Optional, Union
3 -
4 -from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
2 +from typing import List, Optional
3
4 from vllm.config import TokenizerPoolConfig
5 from vllm.lora.request import LoRARequest
8 -
9 -AnyTokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
6 +from vllm.transformers_utils.tokenizer import AnyTokenizer
7
8
9 class BaseTokenizerGroup(ABC):
@@ -24,9 +21,10 @@ class BaseTokenizerGroup(ABC):
21 pass
22
23 @abstractmethod
27 - def get_max_input_len(self,
28 - lora_request: Optional[LoRARequest] = None
29 - ) -> Optional[int]:
24 + def get_max_input_len(
25 + self,
26 + lora_request: Optional[LoRARequest] = None,
27 + ) -> Optional[int]:
28 """Get the maximum input length for the LoRA request."""
29 pass
30
vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py
+2 -1
@@ -13,8 +13,9 @@ from vllm.config import TokenizerPoolConfig
13 from vllm.executor.ray_utils import ray
14 from vllm.logger import init_logger
15 from vllm.lora.request import LoRARequest
16 +from vllm.transformers_utils.tokenizer import AnyTokenizer
17
17 -from .base_tokenizer_group import AnyTokenizer, BaseTokenizerGroup
18 +from .base_tokenizer_group import BaseTokenizerGroup
19 from .tokenizer_group import TokenizerGroup
20
21 logger = init_logger(__name__)
vllm/transformers_utils/tokenizer_group/tokenizer_group.py
+3 -2
@@ -2,12 +2,13 @@ from typing import List, Optional
2
3 from vllm.config import TokenizerPoolConfig
4 from vllm.lora.request import LoRARequest
5 -from vllm.transformers_utils.tokenizer import (get_lora_tokenizer,
5 +from vllm.transformers_utils.tokenizer import (AnyTokenizer,
6 + get_lora_tokenizer,
7 get_lora_tokenizer_async,
8 get_tokenizer)
9 from vllm.utils import LRUCache
10
10 -from .base_tokenizer_group import AnyTokenizer, BaseTokenizerGroup
11 +from .base_tokenizer_group import BaseTokenizerGroup
12
13
14 class TokenizerGroup(BaseTokenizerGroup):
vllm/utils.py
+2 -2
@@ -1101,9 +1101,9 @@ def cuda_device_count_stateless() -> int:
1101
1102
1103 #From: https://stackoverflow.com/a/4104188/2749989
1104 -def run_once(f):
1104 +def run_once(f: Callable[P, None]) -> Callable[P, None]:
1105
1106 - def wrapper(*args, **kwargs) -> Any:
1106 + def wrapper(*args: P.args, **kwargs: P.kwargs) -> None:
1107 if not wrapper.has_run: # type: ignore[attr-defined]
1108 wrapper.has_run = True # type: ignore[attr-defined]
1109 return f(*args, **kwargs)