A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[mypy] Add mypy type annotation part 1 (#4006)
SangBin Cho committed
Apr 13, 2024 at 06:35 UTC
09473ee41c0a22c4d18936ea7eb2328071c19308
25 files changed
+171
-72
.github/workflows/mypy.yaml
new
+50
@@ -0,0 +1,50 @@
1
+name: mypy
2
+
3
+on:
4
+ # Trigger the workflow on push or pull request,
5
+ # but only for the main branch
6
+ push:
7
+ branches:
8
+ - main
9
+ pull_request:
10
+ branches:
11
+ - main
12
+
13
+jobs:
14
+ ruff:
15
+ runs-on: ubuntu-latest
16
+ strategy:
17
+ matrix:
18
+ python-version: ["3.8"]
19
+ steps:
20
+ - uses: actions/checkout@v2
21
+ - name: Set up Python ${{ matrix.python-version }}
22
+ uses: actions/setup-python@v2
23
+ with:
24
+ python-version: ${{ matrix.python-version }}
25
+ - name: Install dependencies
26
+ run: |
27
+ python -m pip install --upgrade pip
28
+ pip install mypy==1.9.0
29
+ pip install types-setuptools
30
+ pip install types-PyYAML
31
+ pip install types-requests
32
+ pip install types-setuptools
33
+ - name: Mypy
34
+ run: |
35
+ mypy vllm/attention/*.py --follow-imports=skip --config-file pyproject.toml
36
+ mypy vllm/core/*.py --follow-imports=skip --config-file pyproject.toml
37
+ mypy vllm/distributed/*.py --follow-imports=skip --config-file pyproject.toml
38
+ mypy vllm/entrypoints/*.py --follow-imports=skip --config-file pyproject.toml
39
+ mypy vllm/executor/*.py --follow-imports=skip --config-file pyproject.toml
40
+ mypy vllm/usage/*.py --follow-imports=skip --config-file pyproject.toml
41
+ mypy vllm/*.py --follow-imports=skip --config-file pyproject.toml
42
+ mypy vllm/transformers_utils/*.py --follow-imports=skip --config-file pyproject.toml
43
+
44
+ # TODO(sang): Follow up
45
+ # mypy vllm/engine/*.py --follow-imports=skip --config-file pyproject.toml
46
+ # mypy vllm/worker/*.py --follow-imports=skip --config-file pyproject.toml
47
+ # mypy vllm/spec_decoding/*.py --follow-imports=skip --config-file pyproject.toml
48
+ # mypy vllm/model_executor/*.py --follow-imports=skip --config-file pyproject.toml
49
+ # mypy vllm/lora/*.py --follow-imports=skip --config-file pyproject.toml
50
+
format.sh
+17
-5
@@ -93,9 +93,23 @@ fi
93
echo 'vLLM yapf: Done'
94
95
# Run mypy
96
-# TODO(zhuohan): Enable mypy
97
-# echo 'vLLM mypy:'
98
-# mypy
96
+echo 'vLLM mypy:'
97
+mypy vllm/attention/*.py --follow-imports=skip --config-file pyproject.toml
98
+mypy vllm/core/*.py --follow-imports=skip --config-file pyproject.toml
99
+mypy vllm/distributed/*.py --follow-imports=skip --config-file pyproject.toml
100
+mypy vllm/entrypoints/*.py --follow-imports=skip --config-file pyproject.toml
101
+mypy vllm/executor/*.py --follow-imports=skip --config-file pyproject.toml
102
+mypy vllm/usage/*.py --follow-imports=skip --config-file pyproject.toml
103
+mypy vllm/*.py --follow-imports=skip --config-file pyproject.toml
104
+mypy vllm/transformers_utils/*.py --follow-imports=skip --config-file pyproject.toml
105
+
106
+# TODO(sang): Follow up
107
+# mypy vllm/engine/*.py --follow-imports=skip --config-file pyproject.toml
108
+# mypy vllm/worker/*.py --follow-imports=skip --config-file pyproject.toml
109
+# mypy vllm/spec_decoding/*.py --follow-imports=skip --config-file pyproject.toml
110
+# mypy vllm/model_executor/*.py --follow-imports=skip --config-file pyproject.toml
111
+# mypy vllm/lora/*.py --follow-imports=skip --config-file pyproject.toml
112
+
113
114
CODESPELL_EXCLUDES=(
115
'--skip' '*docs/source/_build/**'
@@ -228,5 +242,3 @@ if ! git diff --quiet &>/dev/null; then
242
243
exit 1
244
fi
231
-
232
-
pyproject.toml
+4
-1
@@ -46,10 +46,13 @@ ignore = [
46
python_version = "3.8"
47
48
ignore_missing_imports = true
49
+ check_untyped_defs = true
50
51
files = "vllm"
52
# TODO(woosuk): Include the code from Megatron and HuggingFace.
52
-exclude = "vllm/model_executor/parallel_utils/|vllm/model_executor/models/"
53
+exclude = [
54
+ "vllm/model_executor/parallel_utils/|vllm/model_executor/models/",
55
+]
56
57
58
[tool.codespell]
requirements-common.txt
+2
-1
@@ -11,4 +11,5 @@ uvicorn[standard]
11
pydantic >= 2.0 # Required for OpenAI server.
12
prometheus_client >= 0.18.0
13
tiktoken == 0.6.0 # Required for DBRX tokenizer
14
-outlines == 0.0.34 # Requires torch >= 2.1.0
\ No newline at end of file
14
+outlines == 0.0.34 # Requires torch >= 2.1.0
15
+typing_extensions
\ No newline at end of file
requirements-dev.txt
+1
-1
@@ -7,7 +7,7 @@ codespell==2.2.6
7
isort==5.13.2
8
9
# type checking
10
-mypy==0.991
10
+mypy==1.9.0
11
types-PyYAML
12
types-requests
13
types-setuptools
vllm/config.py
+6
-3
@@ -2,7 +2,7 @@ import enum
2
import json
3
import os
4
from dataclasses import dataclass, fields
5
-from typing import TYPE_CHECKING, ClassVar, Optional, Union
5
+from typing import TYPE_CHECKING, ClassVar, List, Optional, Union
6
7
import torch
8
from packaging.version import Version
@@ -141,7 +141,7 @@ class ModelConfig:
141
supported_load_format = [
142
"auto", "pt", "safetensors", "npcache", "dummy"
143
]
144
- rocm_not_supported_load_format = []
144
+ rocm_not_supported_load_format: List[str] = []
145
if load_format not in supported_load_format:
146
raise ValueError(
147
f"Unknown load format: {self.load_format}. Must be one of "
@@ -679,6 +679,9 @@ class SpeculativeConfig:
679
"num_speculative_tokens to be provided, but found "
680
f"{speculative_model=} and {num_speculative_tokens=}.")
681
682
+ assert (speculative_model is not None
683
+ and num_speculative_tokens is not None)
684
+
685
# TODO: The user should be able to specify revision/quantization/max
686
# model len for the draft model. It is not currently supported.
687
draft_revision = None
@@ -993,7 +996,7 @@ def _get_and_verify_max_len(
996
derived_max_model_len *= scaling_factor
997
998
if max_model_len is None:
996
- max_model_len = derived_max_model_len
999
+ max_model_len = int(derived_max_model_len)
1000
elif max_model_len > derived_max_model_len:
1001
# Some models might have a separate key for specifying model_max_length
1002
# that will be bigger than derived_max_model_len. We compare user input
vllm/core/block_manager_v1.py
+7
-5
@@ -1,5 +1,6 @@
1
"""A block manager that manages token blocks."""
2
from abc import ABC, abstractmethod
3
+from collections.abc import Sequence as GenericSequence
4
from itertools import count, takewhile
5
from os.path import commonprefix
6
from typing import Dict, List, Optional, Set
@@ -231,10 +232,10 @@ class BlockSpaceManagerV1(BlockSpaceManager):
232
233
if self.enable_caching:
234
logger.info("Automatic prefix caching is enabled.")
234
- self.gpu_allocator = CachedBlockAllocator(Device.GPU, block_size,
235
- num_gpu_blocks)
236
- self.cpu_allocator = CachedBlockAllocator(Device.CPU, block_size,
237
- num_cpu_blocks)
235
+ self.gpu_allocator: BlockAllocatorBase = CachedBlockAllocator(
236
+ Device.GPU, block_size, num_gpu_blocks)
237
+ self.cpu_allocator: BlockAllocatorBase = CachedBlockAllocator(
238
+ Device.CPU, block_size, num_cpu_blocks)
239
else:
240
self.gpu_allocator = UncachedBlockAllocator(
241
Device.GPU, block_size, num_gpu_blocks)
@@ -588,7 +589,8 @@ class BlockSpaceManagerV1(BlockSpaceManager):
589
for b in takewhile(lambda b: b.computed, block_table[:-1])
590
]
591
591
- def get_common_computed_block_ids(self, seqs: List[Sequence]) -> List[int]:
592
+ def get_common_computed_block_ids(
593
+ self, seqs: List[Sequence]) -> GenericSequence[int]:
594
"""Return the block ids that are common for a given sequence group.
595
596
Used in prefill (can skip prefill of some blocks).
vllm/core/block_manager_v2.py
+3
-1
@@ -1,4 +1,5 @@
1
"""A block manager that manages token blocks."""
2
+from collections.abc import Sequence as GenericSequence
3
from typing import Dict, List, Optional
4
5
from vllm.core.block.block_table import BlockTable
@@ -205,7 +206,8 @@ class BlockSpaceManagerV2(BlockSpaceManager):
206
# as computed.
207
self.block_allocator.mark_blocks_as_computed()
208
208
- def get_common_computed_block_ids(self, seqs: List[Sequence]) -> List[int]:
209
+ def get_common_computed_block_ids(
210
+ self, seqs: List[Sequence]) -> GenericSequence[int]:
211
"""Determine which blocks for which we skip prefill.
212
213
With prefix caching we can skip prefill for previously-generated blocks.
vllm/core/interfaces.py
+3
-1
@@ -1,5 +1,6 @@
1
import enum
2
from abc import ABC, abstractmethod
3
+from collections.abc import Sequence as GenericSequence
4
from typing import Dict, List
5
6
from vllm.sequence import Sequence, SequenceGroup
@@ -103,7 +104,8 @@ class BlockSpaceManager(ABC):
104
pass
105
106
@abstractmethod
106
- def get_common_computed_block_ids(self, seqs: List[Sequence]) -> List[int]:
107
+ def get_common_computed_block_ids(
108
+ self, seqs: List[Sequence]) -> GenericSequence[int]:
109
pass
110
111
@abstractmethod
vllm/core/scheduler.py
+15
-10
@@ -42,8 +42,8 @@ class SchedulingBudget:
42
"""
43
token_budget: int
44
max_num_seqs: int
45
- _requeset_ids_num_batched_tokens: Set[int] = field(default_factory=set)
46
- _requeset_ids_num_curr_seqs: Set[int] = field(default_factory=set)
45
+ _requeset_ids_num_batched_tokens: Set[str] = field(default_factory=set)
46
+ _requeset_ids_num_curr_seqs: Set[str] = field(default_factory=set)
47
_num_batched_tokens: int = 0
48
_num_curr_seqs: int = 0
49
@@ -133,7 +133,7 @@ class SchedulerOutputs:
133
return (not self.scheduled_seq_groups and not self.blocks_to_swap_in
134
and not self.blocks_to_swap_out and not self.blocks_to_copy)
135
136
- def _sort_by_lora_ids(self) -> bool:
136
+ def _sort_by_lora_ids(self):
137
self.scheduled_seq_groups = sorted(
138
self.scheduled_seq_groups,
139
key=lambda g: (g.seq_group.lora_int_id, g.seq_group.request_id))
@@ -337,7 +337,8 @@ class Scheduler:
337
self.free_seq(seq)
338
339
def has_unfinished_seqs(self) -> bool:
340
- return self.waiting or self.running or self.swapped
340
+ return len(self.waiting) != 0 or len(self.running) != 0 or len(
341
+ self.swapped) != 0
342
343
def get_num_unfinished_seq_groups(self) -> int:
344
return len(self.waiting) + len(self.running) + len(self.swapped)
@@ -404,7 +405,7 @@ class Scheduler:
405
budget.subtract_num_seqs(seq_group.request_id,
406
num_running_seqs)
407
if curr_loras is not None and seq_group.lora_int_id > 0:
407
- curr_loras.pop(seq_group.lora_int_id)
408
+ curr_loras.remove(seq_group.lora_int_id)
409
410
if running_queue:
411
# Preempt the lowest-priority sequence groups.
@@ -496,7 +497,7 @@ class Scheduler:
497
now = time.time()
498
swapped_queue = policy.sort_by_priority(now, swapped_queue)
499
499
- leftover_swapped = deque()
500
+ leftover_swapped: Deque[SequenceGroup] = deque()
501
while swapped_queue:
502
seq_group = swapped_queue[0]
503
@@ -507,7 +508,9 @@ class Scheduler:
508
lora_int_id = 0
509
if self.lora_enabled:
510
lora_int_id = seq_group.lora_int_id
510
- if (lora_int_id > 0 and lora_int_id not in curr_loras
511
+ assert curr_loras is not None
512
+ assert self.lora_config is not None
513
+ if (lora_int_id > 0 and (lora_int_id not in curr_loras)
514
and len(curr_loras) >= self.lora_config.max_loras):
515
# We don't have a space for another LoRA, so
516
# we ignore this request for now.
@@ -593,7 +596,7 @@ class Scheduler:
596
# Copy the queue so that the input queue is not modified.
597
waiting_queue = deque([s for s in waiting_queue])
598
596
- leftover_waiting_sequences = deque()
599
+ leftover_waiting_sequences: Deque[SequenceGroup] = deque()
600
while self._passed_delay(time.time()) and waiting_queue:
601
seq_group = waiting_queue[0]
602
@@ -635,6 +638,8 @@ class Scheduler:
638
lora_int_id = 0
639
if self.lora_enabled:
640
lora_int_id = seq_group.lora_int_id
641
+ assert curr_loras is not None
642
+ assert self.lora_config is not None
643
if (self.lora_enabled and lora_int_id > 0
644
and lora_int_id not in curr_loras
645
and len(curr_loras) >= self.lora_config.max_loras):
@@ -780,7 +785,7 @@ class Scheduler:
785
token_budget=self.scheduler_config.max_num_batched_tokens,
786
max_num_seqs=self.scheduler_config.max_num_seqs,
787
)
783
- curr_loras = set()
788
+ curr_loras: Set[int] = set()
789
790
remaining_waiting, prefills = (self.waiting,
791
SchedulerPrefillOutputs.create_empty())
@@ -1087,7 +1092,7 @@ class Scheduler:
1092
1093
def _get_num_new_tokens(self, seq_group: SequenceGroup,
1094
status: SequenceStatus, enable_chunking: bool,
1090
- budget: SchedulingBudget) -> Tuple[int, bool]:
1095
+ budget: SchedulingBudget) -> int:
1096
"""Get the next new tokens to compute for a given sequence group
1097
that's in a given `status`.
1098
vllm/distributed/communication_op.py
+5
-5
@@ -1,5 +1,5 @@
1
from collections import namedtuple
2
-from typing import Any, Dict, List, Optional, Union
2
+from typing import Any, Dict, List, Optional, Tuple, Union
3
4
import torch
5
from torch.distributed import ProcessGroup
@@ -144,7 +144,7 @@ def broadcast_tensor_dict(
144
tensor_dict: Optional[Dict[Any, Union[torch.Tensor, Any]]] = None,
145
src: int = 0,
146
group: Optional[ProcessGroup] = None,
147
-) -> Dict[Any, Union[torch.Tensor, Any]]:
147
+) -> Optional[Dict[Any, Union[torch.Tensor, Any]]]:
148
"""Broadcast the input tensor dictionary."""
149
group = group or torch.distributed.group.WORLD
150
ranks = torch.distributed.get_process_group_ranks(group)
@@ -157,10 +157,10 @@ def broadcast_tensor_dict(
157
158
rank = torch.distributed.get_rank()
159
if rank == src:
160
+ metadata_list: List[Tuple[Any, Any]] = []
161
assert isinstance(
162
tensor_dict,
163
dict), (f"Expecting a dictionary, got {type(tensor_dict)}")
163
- metadata_list = []
164
for key, value in tensor_dict.items():
165
if isinstance(value, torch.Tensor):
166
assert value.is_cuda, (
@@ -190,10 +190,10 @@ def broadcast_tensor_dict(
190
torch.distributed.broadcast_object_list(recv_metadata_list,
191
src=src,
192
group=group)
193
- metadata_list = recv_metadata_list[0]
193
+ assert recv_metadata_list[0] is not None
194
tensor_dict = {}
195
async_handles = []
196
- for key, value in metadata_list:
196
+ for key, value in recv_metadata_list[0]:
197
if isinstance(value, TensorMetadata):
198
tensor = torch.empty(value.size,
199
dtype=value.dtype,
vllm/engine/ray_utils.py
+12
-6
@@ -1,9 +1,10 @@
1
import pickle
2
-from typing import List, Optional, Tuple
2
+from typing import Callable, List, Optional, Tuple
3
4
from vllm.config import ParallelConfig
5
from vllm.logger import init_logger
6
from vllm.utils import get_ip, is_hip, set_cuda_visible_devices
7
+from vllm.worker.worker import Worker
8
9
logger = init_logger(__name__)
10
@@ -18,15 +19,20 @@ try:
19
if init_cached_hf_modules:
20
from transformers.dynamic_module_utils import init_hf_modules
21
init_hf_modules()
21
- self.worker = None
22
+ self._worker: Optional[Worker] = None
23
# Since the compiled DAG runs a main execution
24
# in a different thread that calls cuda.set_device.
25
# The flag indicates is set_device is called on
26
# that thread.
27
self.compiled_dag_cuda_device_set = False
28
28
- def init_worker(self, worker_init_fn):
29
- self.worker = worker_init_fn()
29
+ def init_worker(self, worker_init_fn: Callable[[], Worker]):
30
+ self._worker = worker_init_fn()
31
+
32
+ @property
33
+ def worker(self) -> Worker:
34
+ assert self._worker is not None
35
+ return self._worker
36
37
def __getattr__(self, name):
38
return getattr(self.worker, name)
@@ -70,8 +76,8 @@ except ImportError as e:
76
logger.warning(f"Failed to import Ray with {e!r}. "
77
"For distributed inference, please install Ray with "
78
"`pip install ray`.")
73
- ray = None
74
- RayWorkerVllm = None
79
+ ray = None # type: ignore
80
+ RayWorkerVllm = None # type: ignore
81
82
83
def initialize_ray_cluster(
vllm/entrypoints/api_server.py
+1
@@ -47,6 +47,7 @@ async def generate(request: Request) -> Response:
47
sampling_params = SamplingParams(**request_dict)
48
request_id = random_uuid()
49
50
+ assert engine is not None
51
results_generator = engine.generate(prompt, sampling_params, request_id)
52
53
# Streaming case
vllm/entrypoints/llm.py
+6
-2
@@ -170,8 +170,12 @@ class LLM:
170
multi_modal_data.data = multi_modal_data.data.to(torch.float16)
171
172
# Add requests to the engine.
173
- num_requests = len(prompts) if prompts is not None else len(
174
- prompt_token_ids)
173
+ if prompts is not None:
174
+ num_requests = len(prompts)
175
+ else:
176
+ assert prompt_token_ids is not None
177
+ num_requests = len(prompt_token_ids)
178
+
179
for i in range(num_requests):
180
prompt = prompts[i] if prompts is not None else None
181
token_ids = None if prompt_token_ids is None else prompt_token_ids[
vllm/executor/cpu_executor.py
+2
-2
@@ -1,5 +1,5 @@
1
import os
2
-from typing import Dict, List, Optional
2
+from typing import Dict, List, Optional, Tuple
3
4
import torch
5
@@ -61,7 +61,7 @@ class CPUExecutor(ExecutorBase):
61
self.driver_worker.init_device()
62
self.driver_worker.load_model()
63
64
- def determine_num_available_blocks(self) -> tuple[int, int]:
64
+ def determine_num_available_blocks(self) -> Tuple[int, int]:
65
"""Determine the number of available KV blocks by invoking the
66
underlying worker.
67
"""
vllm/executor/gpu_executor.py
+2
-2
@@ -1,4 +1,4 @@
1
-from typing import Dict, List, Optional
1
+from typing import Dict, List, Optional, Tuple
2
3
from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
4
ParallelConfig, SchedulerConfig, SpeculativeConfig,
@@ -66,7 +66,7 @@ class GPUExecutor(ExecutorBase):
66
self.driver_worker.init_device()
67
self.driver_worker.load_model()
68
69
- def determine_num_available_blocks(self) -> tuple[int, int]:
69
+ def determine_num_available_blocks(self) -> Tuple[int, int]:
70
"""Determine the number of available KV blocks by invoking the
71
underlying worker.
72
"""
vllm/executor/neuron_executor.py
+2
-2
@@ -1,4 +1,4 @@
1
-from typing import Dict, List, Optional
1
+from typing import Dict, List, Optional, Tuple
2
3
from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
4
ParallelConfig, SchedulerConfig, SpeculativeConfig,
@@ -47,7 +47,7 @@ class NeuronExecutor(ExecutorBase):
47
self.driver_worker.init_device()
48
self.driver_worker.load_model()
49
50
- def determine_num_available_blocks(self) -> tuple[int, int]:
50
+ def determine_num_available_blocks(self) -> Tuple[int, int]:
51
"""Determine the number of available KV blocks by invoking the
52
underlying worker.
53
"""
vllm/executor/ray_gpu_executor.py
+6
-5
@@ -3,7 +3,7 @@ import copy
3
import os
4
import pickle
5
from collections import defaultdict
6
-from typing import TYPE_CHECKING, Any, Dict, List, Optional
6
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
7
8
from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
9
ParallelConfig, SchedulerConfig, SpeculativeConfig,
@@ -197,7 +197,7 @@ class RayGPUExecutor(ExecutorBase):
197
max_parallel_loading_workers,
198
)
199
200
- def determine_num_available_blocks(self) -> tuple[int, int]:
200
+ def determine_num_available_blocks(self) -> Tuple[int, int]:
201
"""Determine the number of available KV blocks.
202
203
This invokes `determine_num_available_blocks` on each worker and takes
@@ -205,7 +205,7 @@ class RayGPUExecutor(ExecutorBase):
205
compatible with all workers.
206
207
Returns:
208
- - tuple[num_gpu_blocks, num_cpu_blocks]
208
+ - Tuple[num_gpu_blocks, num_cpu_blocks]
209
"""
210
# Get the maximum number of blocks that can be allocated on GPU and CPU.
211
num_blocks = self._run_workers("determine_num_available_blocks", )
@@ -276,7 +276,7 @@ class RayGPUExecutor(ExecutorBase):
276
self,
277
method: str,
278
*args,
279
- driver_args: Optional[List[Any]] = None,
279
+ driver_args: Optional[Tuple[Any, ...]] = None,
280
driver_kwargs: Optional[Dict[str, Any]] = None,
281
max_concurrent_workers: Optional[int] = None,
282
use_ray_compiled_dag: bool = False,
@@ -291,6 +291,7 @@ class RayGPUExecutor(ExecutorBase):
291
if use_ray_compiled_dag:
292
# Right now, compiled DAG can only accept a single
293
# input. TODO(sang): Fix it.
294
+ assert self.forward_dag is not None
295
output_channels = self.forward_dag.execute(1)
296
else:
297
# Start the ray workers first.
@@ -369,7 +370,7 @@ class RayGPUExecutorAsync(RayGPUExecutor, ExecutorAsyncBase):
370
self,
371
method: str,
372
*args,
372
- driver_args: Optional[List[Any]] = None,
373
+ driver_args: Optional[Tuple[Any, ...]] = None,
374
driver_kwargs: Optional[Dict[str, Any]] = None,
375
**kwargs,
376
) -> Any:
vllm/sampling_params.py
+3
-2
@@ -5,7 +5,8 @@ from functools import cached_property
5
from typing import Callable, List, Optional, Union
6
7
import torch
8
-from pydantic import conint
8
+from pydantic import Field
9
+from typing_extensions import Annotated
10
11
_SAMPLING_EPS = 1e-5
12
@@ -127,7 +128,7 @@ class SamplingParams:
128
skip_special_tokens: bool = True,
129
spaces_between_special_tokens: bool = True,
130
logits_processors: Optional[List[LogitsProcessor]] = None,
130
- truncate_prompt_tokens: Optional[conint(ge=1)] = None,
131
+ truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,
132
) -> None:
133
self.n = n
134
self.best_of = best_of if best_of is not None else n
vllm/sequence.py
+4
-4
@@ -171,10 +171,10 @@ class SequenceData:
171
return self.prompt_token_ids[-1]
172
return self.output_token_ids[-1]
173
174
- def get_prompt_token_ids(self) -> int:
174
+ def get_prompt_token_ids(self) -> List[int]:
175
return self.prompt_token_ids
176
177
- def get_output_token_ids(self) -> int:
177
+ def get_output_token_ids(self) -> List[int]:
178
return self.output_token_ids
179
180
@property
@@ -370,7 +370,7 @@ class SequenceGroupState:
370
"""Mutable state tied to a specific sequence group"""
371
372
# torch.Generator used in seeded sampling
373
- generator: Optional = None
373
+ generator: Optional = None # type: ignore
374
375
376
class MultiModalData:
@@ -599,7 +599,7 @@ class SequenceGroupMetadata:
599
return self.lora_request.lora_int_id if self.lora_request else 0
600
601
@property
602
- def token_chunk_size(self) -> int:
602
+ def token_chunk_size(self) -> Optional[int]:
603
"""Return the number of tokens to be processed (chunk size)."""
604
return self._token_chunk_size
605
vllm/transformers_utils/config.py
+2
-1
@@ -2,7 +2,8 @@ from typing import Dict, Optional
2
3
from transformers import AutoConfig, PretrainedConfig
4
5
-from vllm.transformers_utils.configs import *
5
+from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
6
+ JAISConfig, MPTConfig, RWConfig)
7
8
_CONFIG_REGISTRY: Dict[str, PretrainedConfig] = {
9
"chatglm": ChatGLMConfig,
vllm/transformers_utils/detokenizer.py
+5
-2
@@ -168,8 +168,8 @@ def _convert_tokens_to_string_with_added_encoders(
168
# NOTE(woosuk): The following code is slow because it runs a for loop over
169
# the output_tokens. In Python, running a for loop over a list can be slow
170
# even when the loop body is very simple.
171
- sub_texts = []
172
- current_sub_text = []
171
+ sub_texts: List[str] = []
172
+ current_sub_text: List[str] = []
173
all_special_tokens = set(tokenizer.all_special_tokens)
174
for token in output_tokens:
175
if skip_special_tokens and token in all_special_tokens:
@@ -263,6 +263,7 @@ def detokenize_incrementally(
263
tokenizer,
264
all_input_ids[:-1],
265
skip_special_tokens=skip_special_tokens)
266
+ assert prev_tokens is not None
267
268
# If the new token id is out of bounds, return an empty string.
269
if new_token_id >= len(tokenizer):
@@ -271,6 +272,8 @@ def detokenize_incrementally(
272
# Put new_token_id in a list so skip_special_tokens is respected
273
new_tokens = tokenizer.convert_ids_to_tokens(
274
[new_token_id], skip_special_tokens=skip_special_tokens)
275
+ if isinstance(new_tokens, str):
276
+ new_tokens = [new_tokens]
277
output_tokens = prev_tokens + new_tokens
278
279
# If this is the first iteration, return all tokens.
vllm/transformers_utils/tokenizer.py
+2
-2
@@ -5,7 +5,7 @@ from transformers import (AutoTokenizer, PreTrainedTokenizer,
5
6
from vllm.logger import init_logger
7
from vllm.lora.request import LoRARequest
8
-from vllm.transformers_utils.tokenizers import *
8
+from vllm.transformers_utils.tokenizers import BaichuanTokenizer
9
from vllm.utils import make_async
10
11
logger = init_logger(__name__)
@@ -28,7 +28,7 @@ def get_cached_tokenizer(
28
tokenizer_all_special_tokens = set(tokenizer.all_special_tokens)
29
tokenizer_len = len(tokenizer)
30
31
- class CachedTokenizer(tokenizer.__class__):
31
+ class CachedTokenizer(tokenizer.__class__): # type: ignore
32
33
@property
34
def all_special_ids(self):
vllm/usage/usage_lib.py
+4
-4
@@ -7,7 +7,7 @@ import time
7
from enum import Enum
8
from pathlib import Path
9
from threading import Thread
10
-from typing import Dict, Optional
10
+from typing import Any, Dict, Optional
11
from uuid import uuid4
12
13
import cpuinfo
@@ -124,7 +124,7 @@ class UsageMessage:
124
def report_usage(self,
125
model_architecture: str,
126
usage_context: UsageContext,
127
- extra_kvs: Dict[str, any] = None) -> None:
127
+ extra_kvs: Optional[Dict[str, Any]] = None) -> None:
128
t = Thread(target=self._report_usage_worker,
129
args=(model_architecture, usage_context, extra_kvs or {}),
130
daemon=True)
@@ -132,13 +132,13 @@ class UsageMessage:
132
133
def _report_usage_worker(self, model_architecture: str,
134
usage_context: UsageContext,
135
- extra_kvs: Dict[str, any]) -> None:
135
+ extra_kvs: Dict[str, Any]) -> None:
136
self._report_usage_once(model_architecture, usage_context, extra_kvs)
137
self._report_continous_usage()
138
139
def _report_usage_once(self, model_architecture: str,
140
usage_context: UsageContext,
141
- extra_kvs: Dict[str, any]) -> None:
141
+ extra_kvs: Dict[str, Any]) -> None:
142
# Platform information
143
if torch.cuda.is_available():
144
device_property = torch.cuda.get_device_properties(0)
vllm/utils.py
+7
-5
@@ -60,7 +60,7 @@ class LRUCache(Generic[T]):
60
def __len__(self) -> int:
61
return len(self.cache)
62
63
- def __getitem__(self, key: Hashable) -> T:
63
+ def __getitem__(self, key: Hashable) -> Optional[T]:
64
return self.get(key)
65
66
def __setitem__(self, key: Hashable, value: T) -> None:
@@ -76,7 +76,7 @@ class LRUCache(Generic[T]):
76
key: Hashable,
77
default_value: Optional[T] = None) -> Optional[T]:
78
if key in self.cache:
79
- value = self.cache[key]
79
+ value: Optional[T] = self.cache[key]
80
self.cache.move_to_end(key)
81
else:
82
value = default_value
@@ -87,7 +87,7 @@ class LRUCache(Generic[T]):
87
self.cache.move_to_end(key)
88
self._remove_old_if_needed()
89
90
- def _on_remove(self, key: Hashable, value: T):
90
+ def _on_remove(self, key: Hashable, value: Optional[T]):
91
pass
92
93
def remove_oldest(self):
@@ -100,9 +100,11 @@ class LRUCache(Generic[T]):
100
while len(self.cache) > self.capacity:
101
self.remove_oldest()
102
103
- def pop(self, key: Hashable, default_value: Optional[Any] = None) -> T:
103
+ def pop(self,
104
+ key: Hashable,
105
+ default_value: Optional[T] = None) -> Optional[T]:
106
run_on_remove = key in self.cache
105
- value = self.cache.pop(key, default_value)
107
+ value: Optional[T] = self.cache.pop(key, default_value)
108
if run_on_remove:
109
self._on_remove(key, value)
110
return value