@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 09473ee41c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[mypy] Add mypy type annotation part 1 (#4006)

SangBin Cho committed Apr 13, 2024 at 06:35 UTC 09473ee41c0a22c4d18936ea7eb2328071c19308
25 files changed +171 -72
.github/workflows/mypy.yaml new
+50
@@ -0,0 +1,50 @@
1 +name: mypy
2 +
3 +on:
4 + # Trigger the workflow on push or pull request,
5 + # but only for the main branch
6 + push:
7 + branches:
8 + - main
9 + pull_request:
10 + branches:
11 + - main
12 +
13 +jobs:
14 + ruff:
15 + runs-on: ubuntu-latest
16 + strategy:
17 + matrix:
18 + python-version: ["3.8"]
19 + steps:
20 + - uses: actions/checkout@v2
21 + - name: Set up Python ${{ matrix.python-version }}
22 + uses: actions/setup-python@v2
23 + with:
24 + python-version: ${{ matrix.python-version }}
25 + - name: Install dependencies
26 + run: |
27 + python -m pip install --upgrade pip
28 + pip install mypy==1.9.0
29 + pip install types-setuptools
30 + pip install types-PyYAML
31 + pip install types-requests
32 + pip install types-setuptools
33 + - name: Mypy
34 + run: |
35 + mypy vllm/attention/*.py --follow-imports=skip --config-file pyproject.toml
36 + mypy vllm/core/*.py --follow-imports=skip --config-file pyproject.toml
37 + mypy vllm/distributed/*.py --follow-imports=skip --config-file pyproject.toml
38 + mypy vllm/entrypoints/*.py --follow-imports=skip --config-file pyproject.toml
39 + mypy vllm/executor/*.py --follow-imports=skip --config-file pyproject.toml
40 + mypy vllm/usage/*.py --follow-imports=skip --config-file pyproject.toml
41 + mypy vllm/*.py --follow-imports=skip --config-file pyproject.toml
42 + mypy vllm/transformers_utils/*.py --follow-imports=skip --config-file pyproject.toml
43 +
44 + # TODO(sang): Follow up
45 + # mypy vllm/engine/*.py --follow-imports=skip --config-file pyproject.toml
46 + # mypy vllm/worker/*.py --follow-imports=skip --config-file pyproject.toml
47 + # mypy vllm/spec_decoding/*.py --follow-imports=skip --config-file pyproject.toml
48 + # mypy vllm/model_executor/*.py --follow-imports=skip --config-file pyproject.toml
49 + # mypy vllm/lora/*.py --follow-imports=skip --config-file pyproject.toml
50 +
format.sh
+17 -5
@@ -93,9 +93,23 @@ fi
93 echo 'vLLM yapf: Done'
94
95 # Run mypy
96 -# TODO(zhuohan): Enable mypy
97 -# echo 'vLLM mypy:'
98 -# mypy
96 +echo 'vLLM mypy:'
97 +mypy vllm/attention/*.py --follow-imports=skip --config-file pyproject.toml
98 +mypy vllm/core/*.py --follow-imports=skip --config-file pyproject.toml
99 +mypy vllm/distributed/*.py --follow-imports=skip --config-file pyproject.toml
100 +mypy vllm/entrypoints/*.py --follow-imports=skip --config-file pyproject.toml
101 +mypy vllm/executor/*.py --follow-imports=skip --config-file pyproject.toml
102 +mypy vllm/usage/*.py --follow-imports=skip --config-file pyproject.toml
103 +mypy vllm/*.py --follow-imports=skip --config-file pyproject.toml
104 +mypy vllm/transformers_utils/*.py --follow-imports=skip --config-file pyproject.toml
105 +
106 +# TODO(sang): Follow up
107 +# mypy vllm/engine/*.py --follow-imports=skip --config-file pyproject.toml
108 +# mypy vllm/worker/*.py --follow-imports=skip --config-file pyproject.toml
109 +# mypy vllm/spec_decoding/*.py --follow-imports=skip --config-file pyproject.toml
110 +# mypy vllm/model_executor/*.py --follow-imports=skip --config-file pyproject.toml
111 +# mypy vllm/lora/*.py --follow-imports=skip --config-file pyproject.toml
112 +
113
114 CODESPELL_EXCLUDES=(
115 '--skip' '*docs/source/_build/**'
@@ -228,5 +242,3 @@ if ! git diff --quiet &>/dev/null; then
242
243 exit 1
244 fi
231 -
232 -
pyproject.toml
+4 -1
@@ -46,10 +46,13 @@ ignore = [
46 python_version = "3.8"
47
48 ignore_missing_imports = true
49 + check_untyped_defs = true
50
51 files = "vllm"
52 # TODO(woosuk): Include the code from Megatron and HuggingFace.
52 -exclude = "vllm/model_executor/parallel_utils/|vllm/model_executor/models/"
53 +exclude = [
54 + "vllm/model_executor/parallel_utils/|vllm/model_executor/models/",
55 +]
56
57
58 [tool.codespell]
requirements-common.txt
+2 -1
@@ -11,4 +11,5 @@ uvicorn[standard]
11 pydantic >= 2.0 # Required for OpenAI server.
12 prometheus_client >= 0.18.0
13 tiktoken == 0.6.0 # Required for DBRX tokenizer
14 -outlines == 0.0.34 # Requires torch >= 2.1.0
\ No newline at end of file
14 +outlines == 0.0.34 # Requires torch >= 2.1.0
15 +typing_extensions
\ No newline at end of file
requirements-dev.txt
+1 -1
@@ -7,7 +7,7 @@ codespell==2.2.6
7 isort==5.13.2
8
9 # type checking
10 -mypy==0.991
10 +mypy==1.9.0
11 types-PyYAML
12 types-requests
13 types-setuptools
vllm/config.py
+6 -3
@@ -2,7 +2,7 @@ import enum
2 import json
3 import os
4 from dataclasses import dataclass, fields
5 -from typing import TYPE_CHECKING, ClassVar, Optional, Union
5 +from typing import TYPE_CHECKING, ClassVar, List, Optional, Union
6
7 import torch
8 from packaging.version import Version
@@ -141,7 +141,7 @@ class ModelConfig:
141 supported_load_format = [
142 "auto", "pt", "safetensors", "npcache", "dummy"
143 ]
144 - rocm_not_supported_load_format = []
144 + rocm_not_supported_load_format: List[str] = []
145 if load_format not in supported_load_format:
146 raise ValueError(
147 f"Unknown load format: {self.load_format}. Must be one of "
@@ -679,6 +679,9 @@ class SpeculativeConfig:
679 "num_speculative_tokens to be provided, but found "
680 f"{speculative_model=} and {num_speculative_tokens=}.")
681
682 + assert (speculative_model is not None
683 + and num_speculative_tokens is not None)
684 +
685 # TODO: The user should be able to specify revision/quantization/max
686 # model len for the draft model. It is not currently supported.
687 draft_revision = None
@@ -993,7 +996,7 @@ def _get_and_verify_max_len(
996 derived_max_model_len *= scaling_factor
997
998 if max_model_len is None:
996 - max_model_len = derived_max_model_len
999 + max_model_len = int(derived_max_model_len)
1000 elif max_model_len > derived_max_model_len:
1001 # Some models might have a separate key for specifying model_max_length
1002 # that will be bigger than derived_max_model_len. We compare user input
vllm/core/block_manager_v1.py
+7 -5
@@ -1,5 +1,6 @@
1 """A block manager that manages token blocks."""
2 from abc import ABC, abstractmethod
3 +from collections.abc import Sequence as GenericSequence
4 from itertools import count, takewhile
5 from os.path import commonprefix
6 from typing import Dict, List, Optional, Set
@@ -231,10 +232,10 @@ class BlockSpaceManagerV1(BlockSpaceManager):
232
233 if self.enable_caching:
234 logger.info("Automatic prefix caching is enabled.")
234 - self.gpu_allocator = CachedBlockAllocator(Device.GPU, block_size,
235 - num_gpu_blocks)
236 - self.cpu_allocator = CachedBlockAllocator(Device.CPU, block_size,
237 - num_cpu_blocks)
235 + self.gpu_allocator: BlockAllocatorBase = CachedBlockAllocator(
236 + Device.GPU, block_size, num_gpu_blocks)
237 + self.cpu_allocator: BlockAllocatorBase = CachedBlockAllocator(
238 + Device.CPU, block_size, num_cpu_blocks)
239 else:
240 self.gpu_allocator = UncachedBlockAllocator(
241 Device.GPU, block_size, num_gpu_blocks)
@@ -588,7 +589,8 @@ class BlockSpaceManagerV1(BlockSpaceManager):
589 for b in takewhile(lambda b: b.computed, block_table[:-1])
590 ]
591
591 - def get_common_computed_block_ids(self, seqs: List[Sequence]) -> List[int]:
592 + def get_common_computed_block_ids(
593 + self, seqs: List[Sequence]) -> GenericSequence[int]:
594 """Return the block ids that are common for a given sequence group.
595
596 Used in prefill (can skip prefill of some blocks).
vllm/core/block_manager_v2.py
+3 -1
@@ -1,4 +1,5 @@
1 """A block manager that manages token blocks."""
2 +from collections.abc import Sequence as GenericSequence
3 from typing import Dict, List, Optional
4
5 from vllm.core.block.block_table import BlockTable
@@ -205,7 +206,8 @@ class BlockSpaceManagerV2(BlockSpaceManager):
206 # as computed.
207 self.block_allocator.mark_blocks_as_computed()
208
208 - def get_common_computed_block_ids(self, seqs: List[Sequence]) -> List[int]:
209 + def get_common_computed_block_ids(
210 + self, seqs: List[Sequence]) -> GenericSequence[int]:
211 """Determine which blocks for which we skip prefill.
212
213 With prefix caching we can skip prefill for previously-generated blocks.
vllm/core/interfaces.py
+3 -1
@@ -1,5 +1,6 @@
1 import enum
2 from abc import ABC, abstractmethod
3 +from collections.abc import Sequence as GenericSequence
4 from typing import Dict, List
5
6 from vllm.sequence import Sequence, SequenceGroup
@@ -103,7 +104,8 @@ class BlockSpaceManager(ABC):
104 pass
105
106 @abstractmethod
106 - def get_common_computed_block_ids(self, seqs: List[Sequence]) -> List[int]:
107 + def get_common_computed_block_ids(
108 + self, seqs: List[Sequence]) -> GenericSequence[int]:
109 pass
110
111 @abstractmethod
vllm/core/scheduler.py
+15 -10
@@ -42,8 +42,8 @@ class SchedulingBudget:
42 """
43 token_budget: int
44 max_num_seqs: int
45 - _requeset_ids_num_batched_tokens: Set[int] = field(default_factory=set)
46 - _requeset_ids_num_curr_seqs: Set[int] = field(default_factory=set)
45 + _requeset_ids_num_batched_tokens: Set[str] = field(default_factory=set)
46 + _requeset_ids_num_curr_seqs: Set[str] = field(default_factory=set)
47 _num_batched_tokens: int = 0
48 _num_curr_seqs: int = 0
49
@@ -133,7 +133,7 @@ class SchedulerOutputs:
133 return (not self.scheduled_seq_groups and not self.blocks_to_swap_in
134 and not self.blocks_to_swap_out and not self.blocks_to_copy)
135
136 - def _sort_by_lora_ids(self) -> bool:
136 + def _sort_by_lora_ids(self):
137 self.scheduled_seq_groups = sorted(
138 self.scheduled_seq_groups,
139 key=lambda g: (g.seq_group.lora_int_id, g.seq_group.request_id))
@@ -337,7 +337,8 @@ class Scheduler:
337 self.free_seq(seq)
338
339 def has_unfinished_seqs(self) -> bool:
340 - return self.waiting or self.running or self.swapped
340 + return len(self.waiting) != 0 or len(self.running) != 0 or len(
341 + self.swapped) != 0
342
343 def get_num_unfinished_seq_groups(self) -> int:
344 return len(self.waiting) + len(self.running) + len(self.swapped)
@@ -404,7 +405,7 @@ class Scheduler:
405 budget.subtract_num_seqs(seq_group.request_id,
406 num_running_seqs)
407 if curr_loras is not None and seq_group.lora_int_id > 0:
407 - curr_loras.pop(seq_group.lora_int_id)
408 + curr_loras.remove(seq_group.lora_int_id)
409
410 if running_queue:
411 # Preempt the lowest-priority sequence groups.
@@ -496,7 +497,7 @@ class Scheduler:
497 now = time.time()
498 swapped_queue = policy.sort_by_priority(now, swapped_queue)
499
499 - leftover_swapped = deque()
500 + leftover_swapped: Deque[SequenceGroup] = deque()
501 while swapped_queue:
502 seq_group = swapped_queue[0]
503
@@ -507,7 +508,9 @@ class Scheduler:
508 lora_int_id = 0
509 if self.lora_enabled:
510 lora_int_id = seq_group.lora_int_id
510 - if (lora_int_id > 0 and lora_int_id not in curr_loras
511 + assert curr_loras is not None
512 + assert self.lora_config is not None
513 + if (lora_int_id > 0 and (lora_int_id not in curr_loras)
514 and len(curr_loras) >= self.lora_config.max_loras):
515 # We don't have a space for another LoRA, so
516 # we ignore this request for now.
@@ -593,7 +596,7 @@ class Scheduler:
596 # Copy the queue so that the input queue is not modified.
597 waiting_queue = deque([s for s in waiting_queue])
598
596 - leftover_waiting_sequences = deque()
599 + leftover_waiting_sequences: Deque[SequenceGroup] = deque()
600 while self._passed_delay(time.time()) and waiting_queue:
601 seq_group = waiting_queue[0]
602
@@ -635,6 +638,8 @@ class Scheduler:
638 lora_int_id = 0
639 if self.lora_enabled:
640 lora_int_id = seq_group.lora_int_id
641 + assert curr_loras is not None
642 + assert self.lora_config is not None
643 if (self.lora_enabled and lora_int_id > 0
644 and lora_int_id not in curr_loras
645 and len(curr_loras) >= self.lora_config.max_loras):
@@ -780,7 +785,7 @@ class Scheduler:
785 token_budget=self.scheduler_config.max_num_batched_tokens,
786 max_num_seqs=self.scheduler_config.max_num_seqs,
787 )
783 - curr_loras = set()
788 + curr_loras: Set[int] = set()
789
790 remaining_waiting, prefills = (self.waiting,
791 SchedulerPrefillOutputs.create_empty())
@@ -1087,7 +1092,7 @@ class Scheduler:
1092
1093 def _get_num_new_tokens(self, seq_group: SequenceGroup,
1094 status: SequenceStatus, enable_chunking: bool,
1090 - budget: SchedulingBudget) -> Tuple[int, bool]:
1095 + budget: SchedulingBudget) -> int:
1096 """Get the next new tokens to compute for a given sequence group
1097 that's in a given `status`.
1098
vllm/distributed/communication_op.py
+5 -5
@@ -1,5 +1,5 @@
1 from collections import namedtuple
2 -from typing import Any, Dict, List, Optional, Union
2 +from typing import Any, Dict, List, Optional, Tuple, Union
3
4 import torch
5 from torch.distributed import ProcessGroup
@@ -144,7 +144,7 @@ def broadcast_tensor_dict(
144 tensor_dict: Optional[Dict[Any, Union[torch.Tensor, Any]]] = None,
145 src: int = 0,
146 group: Optional[ProcessGroup] = None,
147 -) -> Dict[Any, Union[torch.Tensor, Any]]:
147 +) -> Optional[Dict[Any, Union[torch.Tensor, Any]]]:
148 """Broadcast the input tensor dictionary."""
149 group = group or torch.distributed.group.WORLD
150 ranks = torch.distributed.get_process_group_ranks(group)
@@ -157,10 +157,10 @@ def broadcast_tensor_dict(
157
158 rank = torch.distributed.get_rank()
159 if rank == src:
160 + metadata_list: List[Tuple[Any, Any]] = []
161 assert isinstance(
162 tensor_dict,
163 dict), (f"Expecting a dictionary, got {type(tensor_dict)}")
163 - metadata_list = []
164 for key, value in tensor_dict.items():
165 if isinstance(value, torch.Tensor):
166 assert value.is_cuda, (
@@ -190,10 +190,10 @@ def broadcast_tensor_dict(
190 torch.distributed.broadcast_object_list(recv_metadata_list,
191 src=src,
192 group=group)
193 - metadata_list = recv_metadata_list[0]
193 + assert recv_metadata_list[0] is not None
194 tensor_dict = {}
195 async_handles = []
196 - for key, value in metadata_list:
196 + for key, value in recv_metadata_list[0]:
197 if isinstance(value, TensorMetadata):
198 tensor = torch.empty(value.size,
199 dtype=value.dtype,
vllm/engine/ray_utils.py
+12 -6
@@ -1,9 +1,10 @@
1 import pickle
2 -from typing import List, Optional, Tuple
2 +from typing import Callable, List, Optional, Tuple
3
4 from vllm.config import ParallelConfig
5 from vllm.logger import init_logger
6 from vllm.utils import get_ip, is_hip, set_cuda_visible_devices
7 +from vllm.worker.worker import Worker
8
9 logger = init_logger(__name__)
10
@@ -18,15 +19,20 @@ try:
19 if init_cached_hf_modules:
20 from transformers.dynamic_module_utils import init_hf_modules
21 init_hf_modules()
21 - self.worker = None
22 + self._worker: Optional[Worker] = None
23 # Since the compiled DAG runs a main execution
24 # in a different thread that calls cuda.set_device.
25 # The flag indicates is set_device is called on
26 # that thread.
27 self.compiled_dag_cuda_device_set = False
28
28 - def init_worker(self, worker_init_fn):
29 - self.worker = worker_init_fn()
29 + def init_worker(self, worker_init_fn: Callable[[], Worker]):
30 + self._worker = worker_init_fn()
31 +
32 + @property
33 + def worker(self) -> Worker:
34 + assert self._worker is not None
35 + return self._worker
36
37 def __getattr__(self, name):
38 return getattr(self.worker, name)
@@ -70,8 +76,8 @@ except ImportError as e:
76 logger.warning(f"Failed to import Ray with {e!r}. "
77 "For distributed inference, please install Ray with "
78 "`pip install ray`.")
73 - ray = None
74 - RayWorkerVllm = None
79 + ray = None # type: ignore
80 + RayWorkerVllm = None # type: ignore
81
82
83 def initialize_ray_cluster(
vllm/entrypoints/api_server.py
+1
@@ -47,6 +47,7 @@ async def generate(request: Request) -> Response:
47 sampling_params = SamplingParams(**request_dict)
48 request_id = random_uuid()
49
50 + assert engine is not None
51 results_generator = engine.generate(prompt, sampling_params, request_id)
52
53 # Streaming case
vllm/entrypoints/llm.py
+6 -2
@@ -170,8 +170,12 @@ class LLM:
170 multi_modal_data.data = multi_modal_data.data.to(torch.float16)
171
172 # Add requests to the engine.
173 - num_requests = len(prompts) if prompts is not None else len(
174 - prompt_token_ids)
173 + if prompts is not None:
174 + num_requests = len(prompts)
175 + else:
176 + assert prompt_token_ids is not None
177 + num_requests = len(prompt_token_ids)
178 +
179 for i in range(num_requests):
180 prompt = prompts[i] if prompts is not None else None
181 token_ids = None if prompt_token_ids is None else prompt_token_ids[
vllm/executor/cpu_executor.py
+2 -2
@@ -1,5 +1,5 @@
1 import os
2 -from typing import Dict, List, Optional
2 +from typing import Dict, List, Optional, Tuple
3
4 import torch
5
@@ -61,7 +61,7 @@ class CPUExecutor(ExecutorBase):
61 self.driver_worker.init_device()
62 self.driver_worker.load_model()
63
64 - def determine_num_available_blocks(self) -> tuple[int, int]:
64 + def determine_num_available_blocks(self) -> Tuple[int, int]:
65 """Determine the number of available KV blocks by invoking the
66 underlying worker.
67 """
vllm/executor/gpu_executor.py
+2 -2
@@ -1,4 +1,4 @@
1 -from typing import Dict, List, Optional
1 +from typing import Dict, List, Optional, Tuple
2
3 from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
4 ParallelConfig, SchedulerConfig, SpeculativeConfig,
@@ -66,7 +66,7 @@ class GPUExecutor(ExecutorBase):
66 self.driver_worker.init_device()
67 self.driver_worker.load_model()
68
69 - def determine_num_available_blocks(self) -> tuple[int, int]:
69 + def determine_num_available_blocks(self) -> Tuple[int, int]:
70 """Determine the number of available KV blocks by invoking the
71 underlying worker.
72 """
vllm/executor/neuron_executor.py
+2 -2
@@ -1,4 +1,4 @@
1 -from typing import Dict, List, Optional
1 +from typing import Dict, List, Optional, Tuple
2
3 from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
4 ParallelConfig, SchedulerConfig, SpeculativeConfig,
@@ -47,7 +47,7 @@ class NeuronExecutor(ExecutorBase):
47 self.driver_worker.init_device()
48 self.driver_worker.load_model()
49
50 - def determine_num_available_blocks(self) -> tuple[int, int]:
50 + def determine_num_available_blocks(self) -> Tuple[int, int]:
51 """Determine the number of available KV blocks by invoking the
52 underlying worker.
53 """
vllm/executor/ray_gpu_executor.py
+6 -5
@@ -3,7 +3,7 @@ import copy
3 import os
4 import pickle
5 from collections import defaultdict
6 -from typing import TYPE_CHECKING, Any, Dict, List, Optional
6 +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
7
8 from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
9 ParallelConfig, SchedulerConfig, SpeculativeConfig,
@@ -197,7 +197,7 @@ class RayGPUExecutor(ExecutorBase):
197 max_parallel_loading_workers,
198 )
199
200 - def determine_num_available_blocks(self) -> tuple[int, int]:
200 + def determine_num_available_blocks(self) -> Tuple[int, int]:
201 """Determine the number of available KV blocks.
202
203 This invokes `determine_num_available_blocks` on each worker and takes
@@ -205,7 +205,7 @@ class RayGPUExecutor(ExecutorBase):
205 compatible with all workers.
206
207 Returns:
208 - - tuple[num_gpu_blocks, num_cpu_blocks]
208 + - Tuple[num_gpu_blocks, num_cpu_blocks]
209 """
210 # Get the maximum number of blocks that can be allocated on GPU and CPU.
211 num_blocks = self._run_workers("determine_num_available_blocks", )
@@ -276,7 +276,7 @@ class RayGPUExecutor(ExecutorBase):
276 self,
277 method: str,
278 *args,
279 - driver_args: Optional[List[Any]] = None,
279 + driver_args: Optional[Tuple[Any, ...]] = None,
280 driver_kwargs: Optional[Dict[str, Any]] = None,
281 max_concurrent_workers: Optional[int] = None,
282 use_ray_compiled_dag: bool = False,
@@ -291,6 +291,7 @@ class RayGPUExecutor(ExecutorBase):
291 if use_ray_compiled_dag:
292 # Right now, compiled DAG can only accept a single
293 # input. TODO(sang): Fix it.
294 + assert self.forward_dag is not None
295 output_channels = self.forward_dag.execute(1)
296 else:
297 # Start the ray workers first.
@@ -369,7 +370,7 @@ class RayGPUExecutorAsync(RayGPUExecutor, ExecutorAsyncBase):
370 self,
371 method: str,
372 *args,
372 - driver_args: Optional[List[Any]] = None,
373 + driver_args: Optional[Tuple[Any, ...]] = None,
374 driver_kwargs: Optional[Dict[str, Any]] = None,
375 **kwargs,
376 ) -> Any:
vllm/sampling_params.py
+3 -2
@@ -5,7 +5,8 @@ from functools import cached_property
5 from typing import Callable, List, Optional, Union
6
7 import torch
8 -from pydantic import conint
8 +from pydantic import Field
9 +from typing_extensions import Annotated
10
11 _SAMPLING_EPS = 1e-5
12
@@ -127,7 +128,7 @@ class SamplingParams:
128 skip_special_tokens: bool = True,
129 spaces_between_special_tokens: bool = True,
130 logits_processors: Optional[List[LogitsProcessor]] = None,
130 - truncate_prompt_tokens: Optional[conint(ge=1)] = None,
131 + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,
132 ) -> None:
133 self.n = n
134 self.best_of = best_of if best_of is not None else n
vllm/sequence.py
+4 -4
@@ -171,10 +171,10 @@ class SequenceData:
171 return self.prompt_token_ids[-1]
172 return self.output_token_ids[-1]
173
174 - def get_prompt_token_ids(self) -> int:
174 + def get_prompt_token_ids(self) -> List[int]:
175 return self.prompt_token_ids
176
177 - def get_output_token_ids(self) -> int:
177 + def get_output_token_ids(self) -> List[int]:
178 return self.output_token_ids
179
180 @property
@@ -370,7 +370,7 @@ class SequenceGroupState:
370 """Mutable state tied to a specific sequence group"""
371
372 # torch.Generator used in seeded sampling
373 - generator: Optional = None
373 + generator: Optional = None # type: ignore
374
375
376 class MultiModalData:
@@ -599,7 +599,7 @@ class SequenceGroupMetadata:
599 return self.lora_request.lora_int_id if self.lora_request else 0
600
601 @property
602 - def token_chunk_size(self) -> int:
602 + def token_chunk_size(self) -> Optional[int]:
603 """Return the number of tokens to be processed (chunk size)."""
604 return self._token_chunk_size
605
vllm/transformers_utils/config.py
+2 -1
@@ -2,7 +2,8 @@ from typing import Dict, Optional
2
3 from transformers import AutoConfig, PretrainedConfig
4
5 -from vllm.transformers_utils.configs import *
5 +from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
6 + JAISConfig, MPTConfig, RWConfig)
7
8 _CONFIG_REGISTRY: Dict[str, PretrainedConfig] = {
9 "chatglm": ChatGLMConfig,
vllm/transformers_utils/detokenizer.py
+5 -2
@@ -168,8 +168,8 @@ def _convert_tokens_to_string_with_added_encoders(
168 # NOTE(woosuk): The following code is slow because it runs a for loop over
169 # the output_tokens. In Python, running a for loop over a list can be slow
170 # even when the loop body is very simple.
171 - sub_texts = []
172 - current_sub_text = []
171 + sub_texts: List[str] = []
172 + current_sub_text: List[str] = []
173 all_special_tokens = set(tokenizer.all_special_tokens)
174 for token in output_tokens:
175 if skip_special_tokens and token in all_special_tokens:
@@ -263,6 +263,7 @@ def detokenize_incrementally(
263 tokenizer,
264 all_input_ids[:-1],
265 skip_special_tokens=skip_special_tokens)
266 + assert prev_tokens is not None
267
268 # If the new token id is out of bounds, return an empty string.
269 if new_token_id >= len(tokenizer):
@@ -271,6 +272,8 @@ def detokenize_incrementally(
272 # Put new_token_id in a list so skip_special_tokens is respected
273 new_tokens = tokenizer.convert_ids_to_tokens(
274 [new_token_id], skip_special_tokens=skip_special_tokens)
275 + if isinstance(new_tokens, str):
276 + new_tokens = [new_tokens]
277 output_tokens = prev_tokens + new_tokens
278
279 # If this is the first iteration, return all tokens.
vllm/transformers_utils/tokenizer.py
+2 -2
@@ -5,7 +5,7 @@ from transformers import (AutoTokenizer, PreTrainedTokenizer,
5
6 from vllm.logger import init_logger
7 from vllm.lora.request import LoRARequest
8 -from vllm.transformers_utils.tokenizers import *
8 +from vllm.transformers_utils.tokenizers import BaichuanTokenizer
9 from vllm.utils import make_async
10
11 logger = init_logger(__name__)
@@ -28,7 +28,7 @@ def get_cached_tokenizer(
28 tokenizer_all_special_tokens = set(tokenizer.all_special_tokens)
29 tokenizer_len = len(tokenizer)
30
31 - class CachedTokenizer(tokenizer.__class__):
31 + class CachedTokenizer(tokenizer.__class__): # type: ignore
32
33 @property
34 def all_special_ids(self):
vllm/usage/usage_lib.py
+4 -4
@@ -7,7 +7,7 @@ import time
7 from enum import Enum
8 from pathlib import Path
9 from threading import Thread
10 -from typing import Dict, Optional
10 +from typing import Any, Dict, Optional
11 from uuid import uuid4
12
13 import cpuinfo
@@ -124,7 +124,7 @@ class UsageMessage:
124 def report_usage(self,
125 model_architecture: str,
126 usage_context: UsageContext,
127 - extra_kvs: Dict[str, any] = None) -> None:
127 + extra_kvs: Optional[Dict[str, Any]] = None) -> None:
128 t = Thread(target=self._report_usage_worker,
129 args=(model_architecture, usage_context, extra_kvs or {}),
130 daemon=True)
@@ -132,13 +132,13 @@ class UsageMessage:
132
133 def _report_usage_worker(self, model_architecture: str,
134 usage_context: UsageContext,
135 - extra_kvs: Dict[str, any]) -> None:
135 + extra_kvs: Dict[str, Any]) -> None:
136 self._report_usage_once(model_architecture, usage_context, extra_kvs)
137 self._report_continous_usage()
138
139 def _report_usage_once(self, model_architecture: str,
140 usage_context: UsageContext,
141 - extra_kvs: Dict[str, any]) -> None:
141 + extra_kvs: Dict[str, Any]) -> None:
142 # Platform information
143 if torch.cuda.is_available():
144 device_property = torch.cuda.get_device_properties(0)
vllm/utils.py
+7 -5
@@ -60,7 +60,7 @@ class LRUCache(Generic[T]):
60 def __len__(self) -> int:
61 return len(self.cache)
62
63 - def __getitem__(self, key: Hashable) -> T:
63 + def __getitem__(self, key: Hashable) -> Optional[T]:
64 return self.get(key)
65
66 def __setitem__(self, key: Hashable, value: T) -> None:
@@ -76,7 +76,7 @@ class LRUCache(Generic[T]):
76 key: Hashable,
77 default_value: Optional[T] = None) -> Optional[T]:
78 if key in self.cache:
79 - value = self.cache[key]
79 + value: Optional[T] = self.cache[key]
80 self.cache.move_to_end(key)
81 else:
82 value = default_value
@@ -87,7 +87,7 @@ class LRUCache(Generic[T]):
87 self.cache.move_to_end(key)
88 self._remove_old_if_needed()
89
90 - def _on_remove(self, key: Hashable, value: T):
90 + def _on_remove(self, key: Hashable, value: Optional[T]):
91 pass
92
93 def remove_oldest(self):
@@ -100,9 +100,11 @@ class LRUCache(Generic[T]):
100 while len(self.cache) > self.capacity:
101 self.remove_oldest()
102
103 - def pop(self, key: Hashable, default_value: Optional[Any] = None) -> T:
103 + def pop(self,
104 + key: Hashable,
105 + default_value: Optional[T] = None) -> Optional[T]:
106 run_on_remove = key in self.cache
105 - value = self.cache.pop(key, default_value)
107 + value: Optional[T] = self.cache.pop(key, default_value)
108 if run_on_remove:
109 self._on_remove(key, value)
110 return value