A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Mypy] Part 3 fix typing for nested directories for most of directory (#4161)
SangBin Cho committed
Apr 23, 2024 at 13:32 UTC
0ae11f78ab89556d5d867abb98f8a132f7507269
29 files changed
+126
-88
.github/workflows/mypy.yaml
+15
-14
@@ -32,19 +32,20 @@ jobs:
32
pip install types-setuptools
33
- name: Mypy
34
run: |
35
- mypy vllm/attention/*.py --follow-imports=skip --config-file pyproject.toml
35
+ mypy vllm/attention --config-file pyproject.toml
36
+ # TODO(sang): Fix nested dir
37
mypy vllm/core/*.py --follow-imports=skip --config-file pyproject.toml
37
- mypy vllm/distributed/*.py --follow-imports=skip --config-file pyproject.toml
38
- mypy vllm/entrypoints/*.py --follow-imports=skip --config-file pyproject.toml
39
- mypy vllm/executor/*.py --follow-imports=skip --config-file pyproject.toml
40
- mypy vllm/usage/*.py --follow-imports=skip --config-file pyproject.toml
41
- mypy vllm/*.py --follow-imports=skip --config-file pyproject.toml
42
- mypy vllm/transformers_utils/*.py --follow-imports=skip --config-file pyproject.toml
43
-
44
- mypy vllm/engine/*.py --follow-imports=skip --config-file pyproject.toml
45
- mypy vllm/worker/*.py --follow-imports=skip --config-file pyproject.toml
46
- mypy vllm/spec_decode/*.py --follow-imports=skip --config-file pyproject.toml
47
- mypy vllm/model_executor/*.py --follow-imports=skip --config-file pyproject.toml
48
- # TODO(sang): Follow up
49
- # mypy vllm/lora/*.py --follow-imports=skip --config-file pyproject.toml
38
+ mypy vllm/distributed --config-file pyproject.toml
39
+ mypy vllm/entrypoints --config-file pyproject.toml
40
+ mypy vllm/executor --config-file pyproject.toml
41
+ mypy vllm/usage --config-file pyproject.toml
42
+ mypy vllm/*.py --config-file pyproject.toml
43
+ mypy vllm/transformers_utils --config-file pyproject.toml
44
+ mypy vllm/engine --config-file pyproject.toml
45
+ mypy vllm/worker --config-file pyproject.toml
46
+ mypy vllm/spec_decode --config-file pyproject.toml
47
+ # TODO(sang): Fix nested dir
48
+ mypy vllm/model_executor/*.py --config-file pyproject.toml
49
+ # TODO(sang): Fix nested dir
50
+ # mypy vllm/lora/*.py --config-file pyproject.toml
51
format.sh
+12
-14
@@ -94,21 +94,19 @@ echo 'vLLM yapf: Done'
94
95
# Run mypy
96
echo 'vLLM mypy:'
97
-mypy vllm/attention/*.py --follow-imports=skip --config-file pyproject.toml
97
+mypy vllm/attention --config-file pyproject.toml
98
mypy vllm/core/*.py --follow-imports=skip --config-file pyproject.toml
99
-mypy vllm/distributed/*.py --follow-imports=skip --config-file pyproject.toml
100
-mypy vllm/entrypoints/*.py --follow-imports=skip --config-file pyproject.toml
101
-mypy vllm/executor/*.py --follow-imports=skip --config-file pyproject.toml
102
-mypy vllm/usage/*.py --follow-imports=skip --config-file pyproject.toml
103
-mypy vllm/*.py --follow-imports=skip --config-file pyproject.toml
104
-mypy vllm/transformers_utils/*.py --follow-imports=skip --config-file pyproject.toml
105
-
106
-# TODO(sang): Follow up
107
-mypy vllm/engine/*.py --follow-imports=skip --config-file pyproject.toml
108
-mypy vllm/worker/*.py --follow-imports=skip --config-file pyproject.toml
109
-mypy vllm/spec_decode/*.py --follow-imports=skip --config-file pyproject.toml
110
-mypy vllm/model_executor/*.py --follow-imports=skip --config-file pyproject.toml
111
-# mypy vllm/lora/*.py --follow-imports=skip --config-file pyproject.toml
99
+mypy vllm/distributed --config-file pyproject.toml
100
+mypy vllm/entrypoints --config-file pyproject.toml
101
+mypy vllm/executor --config-file pyproject.toml
102
+mypy vllm/usage --config-file pyproject.toml
103
+mypy vllm/*.py --config-file pyproject.toml
104
+mypy vllm/transformers_utils --config-file pyproject.toml
105
+mypy vllm/engine --config-file pyproject.toml
106
+mypy vllm/worker --config-file pyproject.toml
107
+mypy vllm/spec_decode --config-file pyproject.toml
108
+mypy vllm/model_executor/*.py --config-file pyproject.toml
109
+# mypy vllm/lora/*.py --config-file pyproject.toml
110
111
112
CODESPELL_EXCLUDES=(
pyproject.toml
+4
-2
@@ -46,15 +46,17 @@ ignore = [
46
python_version = "3.8"
47
48
ignore_missing_imports = true
49
- check_untyped_defs = true
49
+check_untyped_defs = true
50
+follow_imports = "skip"
51
52
files = "vllm"
53
# TODO(woosuk): Include the code from Megatron and HuggingFace.
54
exclude = [
55
"vllm/model_executor/parallel_utils/|vllm/model_executor/models/",
56
+ # Ignore triton kernels in ops.
57
+ 'vllm/attention/ops/.*\.py$'
58
]
59
57
-
60
[tool.codespell]
61
ignore-words-list = "dout, te, indicies"
62
skip = "./tests/prompts,./benchmarks/sonnet.txt"
vllm/attention/backends/abstract.py
+1
-1
@@ -116,7 +116,7 @@ class AttentionImpl(ABC):
116
key: torch.Tensor,
117
value: torch.Tensor,
118
kv_cache: torch.Tensor,
119
- attn_metadata: AttentionMetadata[AttentionMetadataPerStage],
119
+ attn_metadata: AttentionMetadata,
120
kv_scale: float,
121
) -> torch.Tensor:
122
raise NotImplementedError
vllm/attention/backends/rocm_flash_attn.py
+1
@@ -248,6 +248,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
248
249
if prefill_meta := attn_metadata.prefill_metadata:
250
# Prompt run.
251
+ assert prefill_meta.prompt_lens is not None
252
if kv_cache is None or prefill_meta.block_tables.numel() == 0:
253
# triton attention
254
# When block_tables are not filled, it means q and k are the
vllm/attention/backends/torch_sdpa.py
+2
-1
@@ -106,7 +106,7 @@ class TorchSDPABackendImpl(AttentionImpl):
106
key: torch.Tensor,
107
value: torch.Tensor,
108
kv_cache: Optional[torch.Tensor],
109
- attn_metadata: TorchSDPAMetadata,
109
+ attn_metadata: TorchSDPAMetadata, # type: ignore
110
kv_scale: float,
111
) -> torch.Tensor:
112
"""Forward pass with torch SDPA and PagedAttention.
@@ -136,6 +136,7 @@ class TorchSDPABackendImpl(AttentionImpl):
136
kv_scale)
137
138
if attn_metadata.is_prompt:
139
+ assert attn_metadata.prompt_lens is not None
140
if (kv_cache is None or attn_metadata.block_tables.numel() == 0):
141
if self.num_kv_heads != self.num_heads:
142
key = key.repeat_interleave(self.num_queries_per_kv, dim=1)
vllm/attention/backends/xformers.py
+1
@@ -288,6 +288,7 @@ class XFormersImpl(AttentionImpl):
288
value: shape = [num_prefill_tokens, num_kv_heads, head_size]
289
attn_metadata: Metadata for attention.
290
"""
291
+ assert attn_metadata.prompt_lens is not None
292
original_query = query
293
if self.num_kv_heads != self.num_heads:
294
# GQA/MQA requires the shape [B, M, G, H, K].
vllm/core/block/block_table.py
+1
@@ -104,6 +104,7 @@ class BlockTable:
104
token_ids (List[int]): The sequence of token IDs to be appended.
105
"""
106
assert self._is_allocated
107
+ assert self._blocks is not None
108
109
self.ensure_num_empty_slots(num_empty_slots=len(token_ids) +
110
num_lookahead_slots)
vllm/core/block/common.py
+4
-2
@@ -99,7 +99,7 @@ class CopyOnWriteTracker:
99
refcounter: RefCounter,
100
allocator: BlockAllocator,
101
):
102
- self._copy_on_writes = defaultdict(list)
102
+ self._copy_on_writes: Dict[BlockId, List[BlockId]] = defaultdict(list)
103
self._refcounter = refcounter
104
self._allocator = allocator
105
@@ -138,6 +138,8 @@ class CopyOnWriteTracker:
138
prev_block=block.prev_block).block_id
139
140
# Track src/dst copy.
141
+ assert src_block_id is not None
142
+ assert block_id is not None
143
self._copy_on_writes[src_block_id].append(block_id)
144
145
return block_id
@@ -180,6 +182,6 @@ def get_all_blocks_recursively(last_block: Block) -> List[Block]:
182
recurse(block.prev_block, lst)
183
lst.append(block)
184
183
- all_blocks = []
185
+ all_blocks: List[Block] = []
186
recurse(last_block, all_blocks)
187
return all_blocks
vllm/core/block/interfaces.py
+2
-4
@@ -52,8 +52,7 @@ class Block(ABC):
52
class BlockAllocator(ABC):
53
54
@abstractmethod
55
- def allocate_mutable(self, prev_block: Optional[Block],
56
- device: Device) -> Block:
55
+ def allocate_mutable(self, prev_block: Optional[Block]) -> Block:
56
pass
57
58
@abstractmethod
@@ -98,8 +97,7 @@ class BlockAllocator(ABC):
97
class DeviceAwareBlockAllocator(BlockAllocator):
98
99
@abstractmethod
101
- def allocate_mutable(self, prev_block: Optional[Block],
102
- device: Device) -> Block:
100
+ def allocate_mutable(self, prev_block: Optional[Block]) -> Block:
101
pass
102
103
@abstractmethod
vllm/distributed/device_communicators/custom_all_reduce.py
+9
-7
@@ -1,6 +1,6 @@
1
import os
2
from contextlib import contextmanager
3
-from typing import List, Optional
3
+from typing import Any, List, Optional
4
5
import torch
6
import torch.distributed as dist
@@ -18,7 +18,7 @@ except ImportError:
18
19
logger = init_logger(__name__)
20
21
-_CA_HANDLE = None
21
+_CA_HANDLE: Optional["CustomAllreduce"] = None
22
_IS_CAPTURING = False
23
_SUPPORTED_WORLD_SIZES = [2, 4, 6, 8]
24
@@ -51,7 +51,7 @@ def init_custom_ar() -> None:
51
"Cannot test GPU P2P because not all GPUs are visible to the "
52
"current process. This might be the case if 'CUDA_VISIBLE_DEVICES'"
53
" is set.")
54
- return False
54
+ return
55
# test nvlink first, this will filter out most of the cases
56
# where custom allreduce is not supported
57
if "CUDA_VISIBLE_DEVICES" in os.environ:
@@ -117,7 +117,7 @@ def custom_all_reduce(input: torch.Tensor) -> Optional[torch.Tensor]:
117
ca_handle = get_handle()
118
# when custom allreduce is disabled, this will be None
119
if ca_handle is None:
120
- return
120
+ return None
121
if is_capturing():
122
if torch.cuda.is_current_stream_capturing():
123
if ca_handle.should_custom_ar(input):
@@ -135,6 +135,8 @@ def custom_all_reduce(input: torch.Tensor) -> Optional[torch.Tensor]:
135
if ca_handle.should_custom_ar(input):
136
return ca_handle.all_reduce_unreg(input)
137
138
+ return None
139
+
140
141
@contextmanager
142
def _nvml():
@@ -224,14 +226,14 @@ class CustomAllreduce:
226
return self._gather_ipc_meta(shard_data)
227
228
def _gather_ipc_meta(self, shard_data):
227
- all_data = [None] * self.world_size
229
+ all_data: List[Optional[Any]] = [None] * self.world_size
230
dist.all_gather_object(all_data, shard_data)
231
232
handles = []
233
offsets = []
234
for i in range(len(all_data)):
233
- handles.append(all_data[i][0])
234
- offsets.append(all_data[i][1])
235
+ handles.append(all_data[i][0]) # type: ignore
236
+ offsets.append(all_data[i][1]) # type: ignore
237
return handles, offsets
238
239
def register_buffer(self, inp: torch.Tensor):
vllm/distributed/device_communicators/pynccl.py
+13
-9
@@ -107,9 +107,10 @@ _c_ncclCommInitRank.argtypes = [
107
ctypes.POINTER(ctypes.c_void_p), ctypes.c_int, NcclUniqueId, ctypes.c_int
108
]
109
110
+ncclDataType_t = ctypes.c_int
111
111
-# enums
112
-class ncclDataType_t(ctypes.c_int):
112
+
113
+class ncclDataTypeEnum:
114
ncclInt8 = 0
115
ncclChar = 0
116
ncclUint8 = 1
@@ -128,7 +129,7 @@ class ncclDataType_t(ctypes.c_int):
129
ncclNumTypes = 10
130
131
@classmethod
131
- def from_torch(cls, dtype: torch.dtype) -> 'ncclDataType_t':
132
+ def from_torch(cls, dtype: torch.dtype) -> int:
133
if dtype == torch.int8:
134
return cls.ncclInt8
135
if dtype == torch.uint8:
@@ -148,7 +149,10 @@ class ncclDataType_t(ctypes.c_int):
149
raise ValueError(f"Unsupported dtype: {dtype}")
150
151
151
-class ncclRedOp_t(ctypes.c_int):
152
+ncclRedOp_t = ctypes.c_int
153
+
154
+
155
+class ncclRedOpTypeEnum:
156
ncclSum = 0
157
ncclProd = 1
158
ncclMax = 2
@@ -157,7 +161,7 @@ class ncclRedOp_t(ctypes.c_int):
161
ncclNumOps = 5
162
163
@classmethod
160
- def from_torch(cls, op: ReduceOp) -> 'ncclRedOp_t':
164
+ def from_torch(cls, op: ReduceOp) -> int:
165
if op == ReduceOp.SUM:
166
return cls.ncclSum
167
if op == ReduceOp.PRODUCT:
@@ -180,8 +184,8 @@ class ncclRedOp_t(ctypes.c_int):
184
_c_ncclAllReduce = nccl.ncclAllReduce
185
_c_ncclAllReduce.restype = ctypes.c_int
186
_c_ncclAllReduce.argtypes = [
183
- ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t, ncclDataType_t,
184
- ncclRedOp_t, ctypes.c_void_p, ctypes.c_void_p
187
+ ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t, ncclRedOp_t,
188
+ ncclDataType_t, ctypes.c_void_p, ctypes.c_void_p
189
]
190
191
# equivalent to c declaration:
@@ -251,8 +255,8 @@ class NCCLCommunicator:
255
result = _c_ncclAllReduce(ctypes.c_void_p(tensor.data_ptr()),
256
ctypes.c_void_p(tensor.data_ptr()),
257
tensor.numel(),
254
- ncclDataType_t.from_torch(tensor.dtype),
255
- ncclRedOp_t.from_torch(op), self.comm,
258
+ ncclDataTypeEnum.from_torch(tensor.dtype),
259
+ ncclRedOpTypeEnum.from_torch(op), self.comm,
260
ctypes.c_void_p(stream.cuda_stream))
261
assert result == 0
262
vllm/distributed/device_communicators/pynccl_utils.py
+4
-1
@@ -30,6 +30,7 @@ def is_initialized() -> bool:
30
def set_pynccl_stream(stream: torch.cuda.Stream):
31
"""Set the cuda stream for communication"""
32
try:
33
+ assert comm is not None
34
comm.stream = stream
35
yield
36
finally:
@@ -52,6 +53,7 @@ def init_process_group(world_size: int,
53
def all_reduce(input_: torch.Tensor, op=ReduceOp.SUM) -> None:
54
"""All-reduces the input tensor across the process group."""
55
assert input_.is_cuda, f"{input_} should be a cuda tensor"
56
+ assert comm is not None
57
comm.all_reduce(input_, op)
58
59
@@ -62,8 +64,9 @@ def destroy_process_group() -> None:
64
65
def get_world_size() -> int:
66
"""Returns the world size."""
67
+ assert comm is not None
68
return comm.world_size
69
70
68
-def get_nccl_backend():
71
+def get_nccl_backend() -> Optional["NCCLCommunicator"]:
72
return comm
vllm/engine/output_processor/interfaces.py
+3
-2
@@ -1,5 +1,5 @@
1
from abc import ABC, abstractmethod
2
-from typing import Callable, Iterable, List
2
+from typing import Callable, List
3
4
from transformers import PreTrainedTokenizer
5
@@ -8,6 +8,7 @@ from vllm.core.scheduler import Scheduler
8
from vllm.engine.output_processor.stop_checker import StopChecker
9
from vllm.sequence import Sequence, SequenceGroup, SequenceGroupOutput
10
from vllm.transformers_utils.detokenizer import Detokenizer
11
+from vllm.utils import Counter
12
13
14
class SequenceGroupOutputProcessor(ABC):
@@ -27,7 +28,7 @@ class SequenceGroupOutputProcessor(ABC):
28
scheduler_config: SchedulerConfig,
29
detokenizer: Detokenizer,
30
scheduler: Scheduler,
30
- seq_counter: Iterable[int],
31
+ seq_counter: Counter,
32
get_tokenizer_for_seq: Callable[[Sequence], PreTrainedTokenizer],
33
stop_checker: "StopChecker",
34
):
vllm/engine/output_processor/multi_step.py
+3
-2
@@ -1,4 +1,4 @@
1
-from typing import Callable, Iterable, List
1
+from typing import Callable, List
2
3
from transformers import PreTrainedTokenizer
4
@@ -11,6 +11,7 @@ from vllm.sampling_params import SamplingParams
11
from vllm.sequence import (Logprob, Sequence, SequenceGroup,
12
SequenceGroupOutput, SequenceOutput, SequenceStatus)
13
from vllm.transformers_utils.detokenizer import Detokenizer
14
+from vllm.utils import Counter
15
16
logger = init_logger(__name__)
17
@@ -33,7 +34,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
34
self,
35
detokenizer: Detokenizer,
36
scheduler: Scheduler,
36
- seq_counter: Iterable[int],
37
+ seq_counter: Counter,
38
get_tokenizer_for_seq: Callable[[Sequence], PreTrainedTokenizer],
39
stop_checker: StopChecker,
40
):
vllm/engine/output_processor/single_step.py
+5
-4
@@ -1,4 +1,4 @@
1
-from typing import Iterable, List, Tuple, Union
1
+from typing import Dict, List, Tuple, Union
2
3
from vllm.config import SchedulerConfig
4
from vllm.core.scheduler import Scheduler
@@ -10,6 +10,7 @@ from vllm.sampling_params import SamplingParams
10
from vllm.sequence import (Sequence, SequenceGroup, SequenceGroupOutput,
11
SequenceOutput, SequenceStatus)
12
from vllm.transformers_utils.detokenizer import Detokenizer
13
+from vllm.utils import Counter
14
15
logger = init_logger(__name__)
16
@@ -33,7 +34,7 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
34
scheduler_config: SchedulerConfig,
35
detokenizer: Detokenizer,
36
scheduler: Scheduler,
36
- seq_counter: Iterable[int],
37
+ seq_counter: Counter,
38
stop_checker: StopChecker,
39
):
40
self.scheduler_config = scheduler_config
@@ -69,7 +70,7 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
70
samples = outputs.samples
71
parent_seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
72
existing_finished_seqs = seq_group.get_finished_seqs()
72
- parent_child_dict = {
73
+ parent_child_dict: Dict[int, List[SequenceOutput]] = {
74
parent_seq.seq_id: []
75
for parent_seq in parent_seqs
76
}
@@ -92,7 +93,7 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
93
continue
94
# Fork the parent sequence if there are multiple child samples.
95
for child_sample in child_samples[:-1]:
95
- new_child_seq_id = next(self.seq_counter)
96
+ new_child_seq_id: int = next(self.seq_counter)
97
child = parent.fork(new_child_seq_id)
98
child.append_token_id(child_sample.output_token,
99
child_sample.logprobs)
vllm/engine/output_processor/util.py
+3
-1
@@ -8,7 +8,9 @@ def create_output_by_sequence_group(sampler_outputs: List[SamplerOutput],
8
"""Helper method which transforms a 2d list organized by
9
[step][sequence group] into [sequence group][step].
10
"""
11
- output_by_sequence_group = [[] for _ in range(num_seq_groups)]
11
+ output_by_sequence_group: List[List[SamplerOutput]] = [
12
+ [] for _ in range(num_seq_groups)
13
+ ]
14
for step in sampler_outputs:
15
for i, sequence_group_output in enumerate(step):
16
output_by_sequence_group[i].append(sequence_group_output)
vllm/entrypoints/openai/api_server.py
+4
-2
@@ -18,6 +18,7 @@ from vllm.engine.arg_utils import AsyncEngineArgs
18
from vllm.engine.async_llm_engine import AsyncLLMEngine
19
from vllm.entrypoints.openai.cli_args import make_arg_parser
20
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
21
+ ChatCompletionResponse,
22
CompletionRequest, ErrorResponse)
23
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
24
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
@@ -26,8 +27,8 @@ from vllm.usage.usage_lib import UsageContext
27
28
TIMEOUT_KEEP_ALIVE = 5 # seconds
29
29
-openai_serving_chat: OpenAIServingChat = None
30
-openai_serving_completion: OpenAIServingCompletion = None
30
+openai_serving_chat: OpenAIServingChat
31
+openai_serving_completion: OpenAIServingCompletion
32
logger = init_logger(__name__)
33
34
@@ -95,6 +96,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
96
return StreamingResponse(content=generator,
97
media_type="text/event-stream")
98
else:
99
+ assert isinstance(generator, ChatCompletionResponse)
100
return JSONResponse(content=generator.model_dump())
101
102
vllm/entrypoints/openai/protocol.py
+8
-5
@@ -4,7 +4,8 @@ import time
4
from typing import Dict, List, Literal, Optional, Union
5
6
import torch
7
-from pydantic import BaseModel, Field, conint, model_validator
7
+from pydantic import BaseModel, Field, model_validator
8
+from typing_extensions import Annotated
9
10
from vllm.sampling_params import SamplingParams
11
from vllm.utils import random_uuid
@@ -30,7 +31,7 @@ class ModelPermission(BaseModel):
31
allow_fine_tuning: bool = False
32
organization: str = "*"
33
group: Optional[str] = None
33
- is_blocking: str = False
34
+ is_blocking: bool = False
35
36
37
class ModelCard(BaseModel):
@@ -56,7 +57,7 @@ class UsageInfo(BaseModel):
57
58
class ResponseFormat(BaseModel):
59
# type must be "json_object" or "text"
59
- type: str = Literal["text", "json_object"]
60
+ type: Literal["text", "json_object"]
61
62
63
class ChatCompletionRequest(BaseModel):
@@ -152,6 +153,7 @@ class ChatCompletionRequest(BaseModel):
153
def logit_bias_logits_processor(
154
token_ids: List[int],
155
logits: torch.Tensor) -> torch.Tensor:
156
+ assert self.logit_bias is not None
157
for token_id, bias in self.logit_bias.items():
158
# Clamp the bias between -100 and 100 per OpenAI API spec
159
bias = min(100, max(-100, bias))
@@ -213,7 +215,7 @@ class CompletionRequest(BaseModel):
215
logit_bias: Optional[Dict[str, float]] = None
216
logprobs: Optional[int] = None
217
max_tokens: Optional[int] = 16
216
- n: Optional[int] = 1
218
+ n: int = 1
219
presence_penalty: Optional[float] = 0.0
220
seed: Optional[int] = None
221
stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
@@ -235,7 +237,7 @@ class CompletionRequest(BaseModel):
237
min_tokens: Optional[int] = 0
238
skip_special_tokens: Optional[bool] = True
239
spaces_between_special_tokens: Optional[bool] = True
238
- truncate_prompt_tokens: Optional[conint(ge=1)] = None
240
+ truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
241
# doc: end-completion-sampling-params
242
243
# doc: begin-completion-extra-params
@@ -289,6 +291,7 @@ class CompletionRequest(BaseModel):
291
def logit_bias_logits_processor(
292
token_ids: List[int],
293
logits: torch.Tensor) -> torch.Tensor:
294
+ assert self.logit_bias is not None
295
for token_id, bias in self.logit_bias.items():
296
# Clamp the bias between -100 and 100 per OpenAI API spec
297
bias = min(100, max(-100, bias))
vllm/entrypoints/openai/serving_chat.py
+1
-1
@@ -115,12 +115,12 @@ class OpenAIServingChat(OpenAIServing):
115
first_iteration = True
116
117
# Send response for each token for each request.n (index)
118
+ assert request.n is not None
119
previous_texts = [""] * request.n
120
previous_num_tokens = [0] * request.n
121
finish_reason_sent = [False] * request.n
122
try:
123
async for res in result_generator:
123
- res: RequestOutput
124
# We need to do it here, because if there are exceptions in
125
# the result_generator, it needs to be sent as the FIRST
126
# response (by the try...catch).
vllm/entrypoints/openai/serving_completion.py
+4
-1
@@ -185,6 +185,7 @@ class OpenAIServingCompletion(OpenAIServing):
185
model_name: str,
186
num_prompts: int,
187
) -> AsyncGenerator[str, None]:
188
+ assert request.n is not None
189
previous_texts = [""] * request.n * num_prompts
190
previous_num_tokens = [0] * request.n * num_prompts
191
has_echoed = [False] * request.n * num_prompts
@@ -202,6 +203,7 @@ class OpenAIServingCompletion(OpenAIServing):
203
# TODO(simon): optimize the performance by avoiding full
204
# text O(n^2) sending.
205
206
+ assert request.max_tokens is not None
207
if request.echo and request.max_tokens == 0:
208
# only return the prompt
209
delta_text = res.prompt
@@ -279,7 +281,7 @@ class OpenAIServingCompletion(OpenAIServing):
281
created_time: int,
282
model_name: str,
283
) -> CompletionResponse:
282
- choices = []
284
+ choices: List[CompletionResponseChoice] = []
285
num_prompt_tokens = 0
286
num_generated_tokens = 0
287
for final_res in final_res_batch:
@@ -289,6 +291,7 @@ class OpenAIServingCompletion(OpenAIServing):
291
prompt_text = final_res.prompt
292
293
for output in final_res.outputs:
294
+ assert request.max_tokens is not None
295
if request.echo and request.max_tokens == 0:
296
token_ids = prompt_token_ids
297
top_logprobs = prompt_logprobs
vllm/entrypoints/openai/serving_engine.py
+12
-7
@@ -4,7 +4,9 @@ from dataclasses import dataclass
4
from http import HTTPStatus
5
from typing import Dict, List, Optional, Tuple, Union
6
7
-from pydantic import conint
7
+from pydantic import Field
8
+from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
9
+from typing_extensions import Annotated
10
11
from vllm.engine.async_llm_engine import AsyncLLMEngine
12
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
@@ -45,7 +47,8 @@ class OpenAIServing:
47
]
48
49
self.max_model_len = 0
48
- self.tokenizer = None
50
+ # Lazy initialized
51
+ self.tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
52
53
try:
54
event_loop = asyncio.get_running_loop()
@@ -92,7 +95,7 @@ class OpenAIServing:
95
def _create_logprobs(
96
self,
97
token_ids: List[int],
95
- top_logprobs: Optional[List[Optional[Dict[int, Logprob]]]] = None,
98
+ top_logprobs: List[Optional[Dict[int, Logprob]]],
99
num_output_top_logprobs: Optional[int] = None,
100
initial_text_offset: int = 0,
101
) -> LogProbs:
@@ -108,6 +111,7 @@ class OpenAIServing:
111
token = self.tokenizer.decode(token_id)
112
logprobs.tokens.append(token)
113
logprobs.token_logprobs.append(None)
114
+ assert logprobs.top_logprobs is not None
115
logprobs.top_logprobs.append(None)
116
else:
117
token_logprob = step_top_logprobs[token_id].logprob
@@ -116,6 +120,7 @@ class OpenAIServing:
120
logprobs.token_logprobs.append(token_logprob)
121
122
if num_output_top_logprobs:
123
+ assert logprobs.top_logprobs is not None
124
logprobs.top_logprobs.append({
125
# Convert float("-inf") to the
126
# JSON-serializable float that OpenAI uses
@@ -155,9 +160,9 @@ class OpenAIServing:
160
161
async def _check_model(self, request) -> Optional[ErrorResponse]:
162
if request.model in self.served_model_names:
158
- return
163
+ return None
164
if request.model in [lora.lora_name for lora in self.lora_requests]:
160
- return
165
+ return None
166
return self.create_error_response(
167
message=f"The model `{request.model}` does not exist.",
168
err_type="NotFoundError",
@@ -165,7 +170,7 @@ class OpenAIServing:
170
171
def _maybe_get_lora(self, request) -> Optional[LoRARequest]:
172
if request.model in self.served_model_names:
168
- return
173
+ return None
174
for lora in self.lora_requests:
175
if request.model == lora.lora_name:
176
return lora
@@ -177,7 +182,7 @@ class OpenAIServing:
182
request: Union[ChatCompletionRequest, CompletionRequest],
183
prompt: Optional[str] = None,
184
prompt_ids: Optional[List[int]] = None,
180
- truncate_prompt_tokens: Optional[conint(ge=1)] = None
185
+ truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
186
) -> Tuple[List[int], str]:
187
if not (prompt or prompt_ids):
188
raise ValueError("Either prompt or prompt_ids should be provided.")
vllm/lora/lora.py
+1
-1
@@ -33,7 +33,7 @@ class LoRALayerWeights:
33
def optimize(self) -> "LoRALayerWeights":
34
"""Optimize the LoRA by merging the scaling into lora_b."""
35
if self.scaling == 1:
36
- return
36
+ return self
37
self.lora_b *= self.scaling
38
self.scaling = 1
39
return self
vllm/model_executor/layers/ops/sample.py
+2
-1
@@ -29,8 +29,8 @@ def _multi_split_sample(
29
sampled_tokens_size: Tuple[int, int],
30
sampled_logprobs_size: Tuple[int, int],
31
sample_indices: torch.Tensor,
32
+ logprobs: torch.Tensor,
33
*,
33
- logprobs: Optional[torch.Tensor] = None,
34
modify_greedy_probs: bool = False,
35
save_logprobs: bool = False,
36
):
@@ -167,6 +167,7 @@ def sample(
167
sampled_logprobs_size = (0, 0)
168
logprobs = probs
169
170
+ assert logprobs is not None
171
if _save_modified_probs:
172
sampled_modified_probs_size = sampled_tokens_size
173
else:
vllm/model_executor/layers/rotary_embedding.py
+2
-1
@@ -108,7 +108,8 @@ class RotaryEmbedding(nn.Module):
108
query_pass = query[..., self.rotary_dim:]
109
key_pass = key[..., self.rotary_dim:]
110
111
- self.cos_sin_cache = self.cos_sin_cache.to(positions.device)
111
+ self.cos_sin_cache: torch.Tensor = self.cos_sin_cache.to(
112
+ positions.device)
113
cos_sin = self.cos_sin_cache[torch.add(positions, offsets)
114
if offsets is not None else positions]
115
cos, sin = cos_sin.chunk(2, dim=-1)
vllm/transformers_utils/configs/jais.py
+4
-2
@@ -222,13 +222,15 @@ class JAISConfig(PretrainedConfig):
222
f"got {alibi_scaling_type}")
223
if (alibi_scaling_factor is not None
224
and not isinstance(alibi_scaling_factor, float)
225
- or alibi_scaling_factor <= 1.0):
225
+ or (alibi_scaling_factor is not None
226
+ and alibi_scaling_factor <= 1.0)):
227
raise ValueError(
228
f"`alibi_scaling`'s factor field must be a float > 1.0,"
229
f"got {alibi_scaling_factor}")
230
if (alibi_dynamic_scaling is not None
231
and not isinstance(alibi_dynamic_scaling, int)
231
- or alibi_dynamic_scaling <= 1):
232
+ or (alibi_dynamic_scaling is not None
233
+ and alibi_dynamic_scaling <= 1)):
234
raise ValueError(
235
f"`alibi_scaling`'s `train_seq_len` field must be an"
236
f"integer > 1, got {alibi_dynamic_scaling}")
vllm/transformers_utils/tokenizer_group/__init__.py
+1
-1
@@ -11,7 +11,7 @@ if ray:
11
from vllm.transformers_utils.tokenizer_group.ray_tokenizer_group import (
12
RayTokenizerGroupPool)
13
else:
14
- RayTokenizerGroupPool = None
14
+ RayTokenizerGroupPool = None # type: ignore
15
16
17
def get_tokenizer_group(tokenizer_pool_config: Optional[TokenizerPoolConfig],
vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py
+2
@@ -89,6 +89,7 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
89
This is blocking.
90
"""
91
self._ensure_queue_initialized()
92
+ assert self._idle_actors is not None
93
94
if self._idle_actors.empty():
95
raise RuntimeError("No idle actors available.")
@@ -120,6 +121,7 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
121
This is non-blocking.
122
"""
123
self._ensure_queue_initialized()
124
+ assert self._idle_actors is not None
125
126
actor = await self._idle_actors.get()
127
try:
vllm/transformers_utils/tokenizers/baichuan.py
+2
-2
@@ -114,9 +114,9 @@ class BaichuanTokenizer(PreTrainedTokenizer):
114
token = self.sp_model.IdToPiece(index)
115
return token
116
117
- def convert_tokens_to_string(self, tokens):
117
+ def convert_tokens_to_string(self, tokens: List[str]):
118
"""Converts a sequence of tokens (string) in a single string."""
119
- current_sub_tokens = []
119
+ current_sub_tokens: List[str] = []
120
out_string = ""
121
prev_is_special = False
122
for i, token in enumerate(tokens):