@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0ae11f78ab
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Mypy] Part 3 fix typing for nested directories for most of directory (#4161)

SangBin Cho committed Apr 23, 2024 at 13:32 UTC 0ae11f78ab89556d5d867abb98f8a132f7507269
29 files changed +126 -88
.github/workflows/mypy.yaml
+15 -14
@@ -32,19 +32,20 @@ jobs:
32 pip install types-setuptools
33 - name: Mypy
34 run: |
35 - mypy vllm/attention/*.py --follow-imports=skip --config-file pyproject.toml
35 + mypy vllm/attention --config-file pyproject.toml
36 + # TODO(sang): Fix nested dir
37 mypy vllm/core/*.py --follow-imports=skip --config-file pyproject.toml
37 - mypy vllm/distributed/*.py --follow-imports=skip --config-file pyproject.toml
38 - mypy vllm/entrypoints/*.py --follow-imports=skip --config-file pyproject.toml
39 - mypy vllm/executor/*.py --follow-imports=skip --config-file pyproject.toml
40 - mypy vllm/usage/*.py --follow-imports=skip --config-file pyproject.toml
41 - mypy vllm/*.py --follow-imports=skip --config-file pyproject.toml
42 - mypy vllm/transformers_utils/*.py --follow-imports=skip --config-file pyproject.toml
43 -
44 - mypy vllm/engine/*.py --follow-imports=skip --config-file pyproject.toml
45 - mypy vllm/worker/*.py --follow-imports=skip --config-file pyproject.toml
46 - mypy vllm/spec_decode/*.py --follow-imports=skip --config-file pyproject.toml
47 - mypy vllm/model_executor/*.py --follow-imports=skip --config-file pyproject.toml
48 - # TODO(sang): Follow up
49 - # mypy vllm/lora/*.py --follow-imports=skip --config-file pyproject.toml
38 + mypy vllm/distributed --config-file pyproject.toml
39 + mypy vllm/entrypoints --config-file pyproject.toml
40 + mypy vllm/executor --config-file pyproject.toml
41 + mypy vllm/usage --config-file pyproject.toml
42 + mypy vllm/*.py --config-file pyproject.toml
43 + mypy vllm/transformers_utils --config-file pyproject.toml
44 + mypy vllm/engine --config-file pyproject.toml
45 + mypy vllm/worker --config-file pyproject.toml
46 + mypy vllm/spec_decode --config-file pyproject.toml
47 + # TODO(sang): Fix nested dir
48 + mypy vllm/model_executor/*.py --config-file pyproject.toml
49 + # TODO(sang): Fix nested dir
50 + # mypy vllm/lora/*.py --config-file pyproject.toml
51
format.sh
+12 -14
@@ -94,21 +94,19 @@ echo 'vLLM yapf: Done'
94
95 # Run mypy
96 echo 'vLLM mypy:'
97 -mypy vllm/attention/*.py --follow-imports=skip --config-file pyproject.toml
97 +mypy vllm/attention --config-file pyproject.toml
98 mypy vllm/core/*.py --follow-imports=skip --config-file pyproject.toml
99 -mypy vllm/distributed/*.py --follow-imports=skip --config-file pyproject.toml
100 -mypy vllm/entrypoints/*.py --follow-imports=skip --config-file pyproject.toml
101 -mypy vllm/executor/*.py --follow-imports=skip --config-file pyproject.toml
102 -mypy vllm/usage/*.py --follow-imports=skip --config-file pyproject.toml
103 -mypy vllm/*.py --follow-imports=skip --config-file pyproject.toml
104 -mypy vllm/transformers_utils/*.py --follow-imports=skip --config-file pyproject.toml
105 -
106 -# TODO(sang): Follow up
107 -mypy vllm/engine/*.py --follow-imports=skip --config-file pyproject.toml
108 -mypy vllm/worker/*.py --follow-imports=skip --config-file pyproject.toml
109 -mypy vllm/spec_decode/*.py --follow-imports=skip --config-file pyproject.toml
110 -mypy vllm/model_executor/*.py --follow-imports=skip --config-file pyproject.toml
111 -# mypy vllm/lora/*.py --follow-imports=skip --config-file pyproject.toml
99 +mypy vllm/distributed --config-file pyproject.toml
100 +mypy vllm/entrypoints --config-file pyproject.toml
101 +mypy vllm/executor --config-file pyproject.toml
102 +mypy vllm/usage --config-file pyproject.toml
103 +mypy vllm/*.py --config-file pyproject.toml
104 +mypy vllm/transformers_utils --config-file pyproject.toml
105 +mypy vllm/engine --config-file pyproject.toml
106 +mypy vllm/worker --config-file pyproject.toml
107 +mypy vllm/spec_decode --config-file pyproject.toml
108 +mypy vllm/model_executor/*.py --config-file pyproject.toml
109 +# mypy vllm/lora/*.py --config-file pyproject.toml
110
111
112 CODESPELL_EXCLUDES=(
pyproject.toml
+4 -2
@@ -46,15 +46,17 @@ ignore = [
46 python_version = "3.8"
47
48 ignore_missing_imports = true
49 - check_untyped_defs = true
49 +check_untyped_defs = true
50 +follow_imports = "skip"
51
52 files = "vllm"
53 # TODO(woosuk): Include the code from Megatron and HuggingFace.
54 exclude = [
55 "vllm/model_executor/parallel_utils/|vllm/model_executor/models/",
56 + # Ignore triton kernels in ops.
57 + 'vllm/attention/ops/.*\.py$'
58 ]
59
57 -
60 [tool.codespell]
61 ignore-words-list = "dout, te, indicies"
62 skip = "./tests/prompts,./benchmarks/sonnet.txt"
vllm/attention/backends/abstract.py
+1 -1
@@ -116,7 +116,7 @@ class AttentionImpl(ABC):
116 key: torch.Tensor,
117 value: torch.Tensor,
118 kv_cache: torch.Tensor,
119 - attn_metadata: AttentionMetadata[AttentionMetadataPerStage],
119 + attn_metadata: AttentionMetadata,
120 kv_scale: float,
121 ) -> torch.Tensor:
122 raise NotImplementedError
vllm/attention/backends/rocm_flash_attn.py
+1
@@ -248,6 +248,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
248
249 if prefill_meta := attn_metadata.prefill_metadata:
250 # Prompt run.
251 + assert prefill_meta.prompt_lens is not None
252 if kv_cache is None or prefill_meta.block_tables.numel() == 0:
253 # triton attention
254 # When block_tables are not filled, it means q and k are the
vllm/attention/backends/torch_sdpa.py
+2 -1
@@ -106,7 +106,7 @@ class TorchSDPABackendImpl(AttentionImpl):
106 key: torch.Tensor,
107 value: torch.Tensor,
108 kv_cache: Optional[torch.Tensor],
109 - attn_metadata: TorchSDPAMetadata,
109 + attn_metadata: TorchSDPAMetadata, # type: ignore
110 kv_scale: float,
111 ) -> torch.Tensor:
112 """Forward pass with torch SDPA and PagedAttention.
@@ -136,6 +136,7 @@ class TorchSDPABackendImpl(AttentionImpl):
136 kv_scale)
137
138 if attn_metadata.is_prompt:
139 + assert attn_metadata.prompt_lens is not None
140 if (kv_cache is None or attn_metadata.block_tables.numel() == 0):
141 if self.num_kv_heads != self.num_heads:
142 key = key.repeat_interleave(self.num_queries_per_kv, dim=1)
vllm/attention/backends/xformers.py
+1
@@ -288,6 +288,7 @@ class XFormersImpl(AttentionImpl):
288 value: shape = [num_prefill_tokens, num_kv_heads, head_size]
289 attn_metadata: Metadata for attention.
290 """
291 + assert attn_metadata.prompt_lens is not None
292 original_query = query
293 if self.num_kv_heads != self.num_heads:
294 # GQA/MQA requires the shape [B, M, G, H, K].
vllm/core/block/block_table.py
+1
@@ -104,6 +104,7 @@ class BlockTable:
104 token_ids (List[int]): The sequence of token IDs to be appended.
105 """
106 assert self._is_allocated
107 + assert self._blocks is not None
108
109 self.ensure_num_empty_slots(num_empty_slots=len(token_ids) +
110 num_lookahead_slots)
vllm/core/block/common.py
+4 -2
@@ -99,7 +99,7 @@ class CopyOnWriteTracker:
99 refcounter: RefCounter,
100 allocator: BlockAllocator,
101 ):
102 - self._copy_on_writes = defaultdict(list)
102 + self._copy_on_writes: Dict[BlockId, List[BlockId]] = defaultdict(list)
103 self._refcounter = refcounter
104 self._allocator = allocator
105
@@ -138,6 +138,8 @@ class CopyOnWriteTracker:
138 prev_block=block.prev_block).block_id
139
140 # Track src/dst copy.
141 + assert src_block_id is not None
142 + assert block_id is not None
143 self._copy_on_writes[src_block_id].append(block_id)
144
145 return block_id
@@ -180,6 +182,6 @@ def get_all_blocks_recursively(last_block: Block) -> List[Block]:
182 recurse(block.prev_block, lst)
183 lst.append(block)
184
183 - all_blocks = []
185 + all_blocks: List[Block] = []
186 recurse(last_block, all_blocks)
187 return all_blocks
vllm/core/block/interfaces.py
+2 -4
@@ -52,8 +52,7 @@ class Block(ABC):
52 class BlockAllocator(ABC):
53
54 @abstractmethod
55 - def allocate_mutable(self, prev_block: Optional[Block],
56 - device: Device) -> Block:
55 + def allocate_mutable(self, prev_block: Optional[Block]) -> Block:
56 pass
57
58 @abstractmethod
@@ -98,8 +97,7 @@ class BlockAllocator(ABC):
97 class DeviceAwareBlockAllocator(BlockAllocator):
98
99 @abstractmethod
101 - def allocate_mutable(self, prev_block: Optional[Block],
102 - device: Device) -> Block:
100 + def allocate_mutable(self, prev_block: Optional[Block]) -> Block:
101 pass
102
103 @abstractmethod
vllm/distributed/device_communicators/custom_all_reduce.py
+9 -7
@@ -1,6 +1,6 @@
1 import os
2 from contextlib import contextmanager
3 -from typing import List, Optional
3 +from typing import Any, List, Optional
4
5 import torch
6 import torch.distributed as dist
@@ -18,7 +18,7 @@ except ImportError:
18
19 logger = init_logger(__name__)
20
21 -_CA_HANDLE = None
21 +_CA_HANDLE: Optional["CustomAllreduce"] = None
22 _IS_CAPTURING = False
23 _SUPPORTED_WORLD_SIZES = [2, 4, 6, 8]
24
@@ -51,7 +51,7 @@ def init_custom_ar() -> None:
51 "Cannot test GPU P2P because not all GPUs are visible to the "
52 "current process. This might be the case if 'CUDA_VISIBLE_DEVICES'"
53 " is set.")
54 - return False
54 + return
55 # test nvlink first, this will filter out most of the cases
56 # where custom allreduce is not supported
57 if "CUDA_VISIBLE_DEVICES" in os.environ:
@@ -117,7 +117,7 @@ def custom_all_reduce(input: torch.Tensor) -> Optional[torch.Tensor]:
117 ca_handle = get_handle()
118 # when custom allreduce is disabled, this will be None
119 if ca_handle is None:
120 - return
120 + return None
121 if is_capturing():
122 if torch.cuda.is_current_stream_capturing():
123 if ca_handle.should_custom_ar(input):
@@ -135,6 +135,8 @@ def custom_all_reduce(input: torch.Tensor) -> Optional[torch.Tensor]:
135 if ca_handle.should_custom_ar(input):
136 return ca_handle.all_reduce_unreg(input)
137
138 + return None
139 +
140
141 @contextmanager
142 def _nvml():
@@ -224,14 +226,14 @@ class CustomAllreduce:
226 return self._gather_ipc_meta(shard_data)
227
228 def _gather_ipc_meta(self, shard_data):
227 - all_data = [None] * self.world_size
229 + all_data: List[Optional[Any]] = [None] * self.world_size
230 dist.all_gather_object(all_data, shard_data)
231
232 handles = []
233 offsets = []
234 for i in range(len(all_data)):
233 - handles.append(all_data[i][0])
234 - offsets.append(all_data[i][1])
235 + handles.append(all_data[i][0]) # type: ignore
236 + offsets.append(all_data[i][1]) # type: ignore
237 return handles, offsets
238
239 def register_buffer(self, inp: torch.Tensor):
vllm/distributed/device_communicators/pynccl.py
+13 -9
@@ -107,9 +107,10 @@ _c_ncclCommInitRank.argtypes = [
107 ctypes.POINTER(ctypes.c_void_p), ctypes.c_int, NcclUniqueId, ctypes.c_int
108 ]
109
110 +ncclDataType_t = ctypes.c_int
111
111 -# enums
112 -class ncclDataType_t(ctypes.c_int):
112 +
113 +class ncclDataTypeEnum:
114 ncclInt8 = 0
115 ncclChar = 0
116 ncclUint8 = 1
@@ -128,7 +129,7 @@ class ncclDataType_t(ctypes.c_int):
129 ncclNumTypes = 10
130
131 @classmethod
131 - def from_torch(cls, dtype: torch.dtype) -> 'ncclDataType_t':
132 + def from_torch(cls, dtype: torch.dtype) -> int:
133 if dtype == torch.int8:
134 return cls.ncclInt8
135 if dtype == torch.uint8:
@@ -148,7 +149,10 @@ class ncclDataType_t(ctypes.c_int):
149 raise ValueError(f"Unsupported dtype: {dtype}")
150
151
151 -class ncclRedOp_t(ctypes.c_int):
152 +ncclRedOp_t = ctypes.c_int
153 +
154 +
155 +class ncclRedOpTypeEnum:
156 ncclSum = 0
157 ncclProd = 1
158 ncclMax = 2
@@ -157,7 +161,7 @@ class ncclRedOp_t(ctypes.c_int):
161 ncclNumOps = 5
162
163 @classmethod
160 - def from_torch(cls, op: ReduceOp) -> 'ncclRedOp_t':
164 + def from_torch(cls, op: ReduceOp) -> int:
165 if op == ReduceOp.SUM:
166 return cls.ncclSum
167 if op == ReduceOp.PRODUCT:
@@ -180,8 +184,8 @@ class ncclRedOp_t(ctypes.c_int):
184 _c_ncclAllReduce = nccl.ncclAllReduce
185 _c_ncclAllReduce.restype = ctypes.c_int
186 _c_ncclAllReduce.argtypes = [
183 - ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t, ncclDataType_t,
184 - ncclRedOp_t, ctypes.c_void_p, ctypes.c_void_p
187 + ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t, ncclRedOp_t,
188 + ncclDataType_t, ctypes.c_void_p, ctypes.c_void_p
189 ]
190
191 # equivalent to c declaration:
@@ -251,8 +255,8 @@ class NCCLCommunicator:
255 result = _c_ncclAllReduce(ctypes.c_void_p(tensor.data_ptr()),
256 ctypes.c_void_p(tensor.data_ptr()),
257 tensor.numel(),
254 - ncclDataType_t.from_torch(tensor.dtype),
255 - ncclRedOp_t.from_torch(op), self.comm,
258 + ncclDataTypeEnum.from_torch(tensor.dtype),
259 + ncclRedOpTypeEnum.from_torch(op), self.comm,
260 ctypes.c_void_p(stream.cuda_stream))
261 assert result == 0
262
vllm/distributed/device_communicators/pynccl_utils.py
+4 -1
@@ -30,6 +30,7 @@ def is_initialized() -> bool:
30 def set_pynccl_stream(stream: torch.cuda.Stream):
31 """Set the cuda stream for communication"""
32 try:
33 + assert comm is not None
34 comm.stream = stream
35 yield
36 finally:
@@ -52,6 +53,7 @@ def init_process_group(world_size: int,
53 def all_reduce(input_: torch.Tensor, op=ReduceOp.SUM) -> None:
54 """All-reduces the input tensor across the process group."""
55 assert input_.is_cuda, f"{input_} should be a cuda tensor"
56 + assert comm is not None
57 comm.all_reduce(input_, op)
58
59
@@ -62,8 +64,9 @@ def destroy_process_group() -> None:
64
65 def get_world_size() -> int:
66 """Returns the world size."""
67 + assert comm is not None
68 return comm.world_size
69
70
68 -def get_nccl_backend():
71 +def get_nccl_backend() -> Optional["NCCLCommunicator"]:
72 return comm
vllm/engine/output_processor/interfaces.py
+3 -2
@@ -1,5 +1,5 @@
1 from abc import ABC, abstractmethod
2 -from typing import Callable, Iterable, List
2 +from typing import Callable, List
3
4 from transformers import PreTrainedTokenizer
5
@@ -8,6 +8,7 @@ from vllm.core.scheduler import Scheduler
8 from vllm.engine.output_processor.stop_checker import StopChecker
9 from vllm.sequence import Sequence, SequenceGroup, SequenceGroupOutput
10 from vllm.transformers_utils.detokenizer import Detokenizer
11 +from vllm.utils import Counter
12
13
14 class SequenceGroupOutputProcessor(ABC):
@@ -27,7 +28,7 @@ class SequenceGroupOutputProcessor(ABC):
28 scheduler_config: SchedulerConfig,
29 detokenizer: Detokenizer,
30 scheduler: Scheduler,
30 - seq_counter: Iterable[int],
31 + seq_counter: Counter,
32 get_tokenizer_for_seq: Callable[[Sequence], PreTrainedTokenizer],
33 stop_checker: "StopChecker",
34 ):
vllm/engine/output_processor/multi_step.py
+3 -2
@@ -1,4 +1,4 @@
1 -from typing import Callable, Iterable, List
1 +from typing import Callable, List
2
3 from transformers import PreTrainedTokenizer
4
@@ -11,6 +11,7 @@ from vllm.sampling_params import SamplingParams
11 from vllm.sequence import (Logprob, Sequence, SequenceGroup,
12 SequenceGroupOutput, SequenceOutput, SequenceStatus)
13 from vllm.transformers_utils.detokenizer import Detokenizer
14 +from vllm.utils import Counter
15
16 logger = init_logger(__name__)
17
@@ -33,7 +34,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
34 self,
35 detokenizer: Detokenizer,
36 scheduler: Scheduler,
36 - seq_counter: Iterable[int],
37 + seq_counter: Counter,
38 get_tokenizer_for_seq: Callable[[Sequence], PreTrainedTokenizer],
39 stop_checker: StopChecker,
40 ):
vllm/engine/output_processor/single_step.py
+5 -4
@@ -1,4 +1,4 @@
1 -from typing import Iterable, List, Tuple, Union
1 +from typing import Dict, List, Tuple, Union
2
3 from vllm.config import SchedulerConfig
4 from vllm.core.scheduler import Scheduler
@@ -10,6 +10,7 @@ from vllm.sampling_params import SamplingParams
10 from vllm.sequence import (Sequence, SequenceGroup, SequenceGroupOutput,
11 SequenceOutput, SequenceStatus)
12 from vllm.transformers_utils.detokenizer import Detokenizer
13 +from vllm.utils import Counter
14
15 logger = init_logger(__name__)
16
@@ -33,7 +34,7 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
34 scheduler_config: SchedulerConfig,
35 detokenizer: Detokenizer,
36 scheduler: Scheduler,
36 - seq_counter: Iterable[int],
37 + seq_counter: Counter,
38 stop_checker: StopChecker,
39 ):
40 self.scheduler_config = scheduler_config
@@ -69,7 +70,7 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
70 samples = outputs.samples
71 parent_seqs = seq_group.get_seqs(status=SequenceStatus.RUNNING)
72 existing_finished_seqs = seq_group.get_finished_seqs()
72 - parent_child_dict = {
73 + parent_child_dict: Dict[int, List[SequenceOutput]] = {
74 parent_seq.seq_id: []
75 for parent_seq in parent_seqs
76 }
@@ -92,7 +93,7 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
93 continue
94 # Fork the parent sequence if there are multiple child samples.
95 for child_sample in child_samples[:-1]:
95 - new_child_seq_id = next(self.seq_counter)
96 + new_child_seq_id: int = next(self.seq_counter)
97 child = parent.fork(new_child_seq_id)
98 child.append_token_id(child_sample.output_token,
99 child_sample.logprobs)
vllm/engine/output_processor/util.py
+3 -1
@@ -8,7 +8,9 @@ def create_output_by_sequence_group(sampler_outputs: List[SamplerOutput],
8 """Helper method which transforms a 2d list organized by
9 [step][sequence group] into [sequence group][step].
10 """
11 - output_by_sequence_group = [[] for _ in range(num_seq_groups)]
11 + output_by_sequence_group: List[List[SamplerOutput]] = [
12 + [] for _ in range(num_seq_groups)
13 + ]
14 for step in sampler_outputs:
15 for i, sequence_group_output in enumerate(step):
16 output_by_sequence_group[i].append(sequence_group_output)
vllm/entrypoints/openai/api_server.py
+4 -2
@@ -18,6 +18,7 @@ from vllm.engine.arg_utils import AsyncEngineArgs
18 from vllm.engine.async_llm_engine import AsyncLLMEngine
19 from vllm.entrypoints.openai.cli_args import make_arg_parser
20 from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
21 + ChatCompletionResponse,
22 CompletionRequest, ErrorResponse)
23 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
24 from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
@@ -26,8 +27,8 @@ from vllm.usage.usage_lib import UsageContext
27
28 TIMEOUT_KEEP_ALIVE = 5 # seconds
29
29 -openai_serving_chat: OpenAIServingChat = None
30 -openai_serving_completion: OpenAIServingCompletion = None
30 +openai_serving_chat: OpenAIServingChat
31 +openai_serving_completion: OpenAIServingCompletion
32 logger = init_logger(__name__)
33
34
@@ -95,6 +96,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
96 return StreamingResponse(content=generator,
97 media_type="text/event-stream")
98 else:
99 + assert isinstance(generator, ChatCompletionResponse)
100 return JSONResponse(content=generator.model_dump())
101
102
vllm/entrypoints/openai/protocol.py
+8 -5
@@ -4,7 +4,8 @@ import time
4 from typing import Dict, List, Literal, Optional, Union
5
6 import torch
7 -from pydantic import BaseModel, Field, conint, model_validator
7 +from pydantic import BaseModel, Field, model_validator
8 +from typing_extensions import Annotated
9
10 from vllm.sampling_params import SamplingParams
11 from vllm.utils import random_uuid
@@ -30,7 +31,7 @@ class ModelPermission(BaseModel):
31 allow_fine_tuning: bool = False
32 organization: str = "*"
33 group: Optional[str] = None
33 - is_blocking: str = False
34 + is_blocking: bool = False
35
36
37 class ModelCard(BaseModel):
@@ -56,7 +57,7 @@ class UsageInfo(BaseModel):
57
58 class ResponseFormat(BaseModel):
59 # type must be "json_object" or "text"
59 - type: str = Literal["text", "json_object"]
60 + type: Literal["text", "json_object"]
61
62
63 class ChatCompletionRequest(BaseModel):
@@ -152,6 +153,7 @@ class ChatCompletionRequest(BaseModel):
153 def logit_bias_logits_processor(
154 token_ids: List[int],
155 logits: torch.Tensor) -> torch.Tensor:
156 + assert self.logit_bias is not None
157 for token_id, bias in self.logit_bias.items():
158 # Clamp the bias between -100 and 100 per OpenAI API spec
159 bias = min(100, max(-100, bias))
@@ -213,7 +215,7 @@ class CompletionRequest(BaseModel):
215 logit_bias: Optional[Dict[str, float]] = None
216 logprobs: Optional[int] = None
217 max_tokens: Optional[int] = 16
216 - n: Optional[int] = 1
218 + n: int = 1
219 presence_penalty: Optional[float] = 0.0
220 seed: Optional[int] = None
221 stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
@@ -235,7 +237,7 @@ class CompletionRequest(BaseModel):
237 min_tokens: Optional[int] = 0
238 skip_special_tokens: Optional[bool] = True
239 spaces_between_special_tokens: Optional[bool] = True
238 - truncate_prompt_tokens: Optional[conint(ge=1)] = None
240 + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
241 # doc: end-completion-sampling-params
242
243 # doc: begin-completion-extra-params
@@ -289,6 +291,7 @@ class CompletionRequest(BaseModel):
291 def logit_bias_logits_processor(
292 token_ids: List[int],
293 logits: torch.Tensor) -> torch.Tensor:
294 + assert self.logit_bias is not None
295 for token_id, bias in self.logit_bias.items():
296 # Clamp the bias between -100 and 100 per OpenAI API spec
297 bias = min(100, max(-100, bias))
vllm/entrypoints/openai/serving_chat.py
+1 -1
@@ -115,12 +115,12 @@ class OpenAIServingChat(OpenAIServing):
115 first_iteration = True
116
117 # Send response for each token for each request.n (index)
118 + assert request.n is not None
119 previous_texts = [""] * request.n
120 previous_num_tokens = [0] * request.n
121 finish_reason_sent = [False] * request.n
122 try:
123 async for res in result_generator:
123 - res: RequestOutput
124 # We need to do it here, because if there are exceptions in
125 # the result_generator, it needs to be sent as the FIRST
126 # response (by the try...catch).
vllm/entrypoints/openai/serving_completion.py
+4 -1
@@ -185,6 +185,7 @@ class OpenAIServingCompletion(OpenAIServing):
185 model_name: str,
186 num_prompts: int,
187 ) -> AsyncGenerator[str, None]:
188 + assert request.n is not None
189 previous_texts = [""] * request.n * num_prompts
190 previous_num_tokens = [0] * request.n * num_prompts
191 has_echoed = [False] * request.n * num_prompts
@@ -202,6 +203,7 @@ class OpenAIServingCompletion(OpenAIServing):
203 # TODO(simon): optimize the performance by avoiding full
204 # text O(n^2) sending.
205
206 + assert request.max_tokens is not None
207 if request.echo and request.max_tokens == 0:
208 # only return the prompt
209 delta_text = res.prompt
@@ -279,7 +281,7 @@ class OpenAIServingCompletion(OpenAIServing):
281 created_time: int,
282 model_name: str,
283 ) -> CompletionResponse:
282 - choices = []
284 + choices: List[CompletionResponseChoice] = []
285 num_prompt_tokens = 0
286 num_generated_tokens = 0
287 for final_res in final_res_batch:
@@ -289,6 +291,7 @@ class OpenAIServingCompletion(OpenAIServing):
291 prompt_text = final_res.prompt
292
293 for output in final_res.outputs:
294 + assert request.max_tokens is not None
295 if request.echo and request.max_tokens == 0:
296 token_ids = prompt_token_ids
297 top_logprobs = prompt_logprobs
vllm/entrypoints/openai/serving_engine.py
+12 -7
@@ -4,7 +4,9 @@ from dataclasses import dataclass
4 from http import HTTPStatus
5 from typing import Dict, List, Optional, Tuple, Union
6
7 -from pydantic import conint
7 +from pydantic import Field
8 +from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
9 +from typing_extensions import Annotated
10
11 from vllm.engine.async_llm_engine import AsyncLLMEngine
12 from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
@@ -45,7 +47,8 @@ class OpenAIServing:
47 ]
48
49 self.max_model_len = 0
48 - self.tokenizer = None
50 + # Lazy initialized
51 + self.tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
52
53 try:
54 event_loop = asyncio.get_running_loop()
@@ -92,7 +95,7 @@ class OpenAIServing:
95 def _create_logprobs(
96 self,
97 token_ids: List[int],
95 - top_logprobs: Optional[List[Optional[Dict[int, Logprob]]]] = None,
98 + top_logprobs: List[Optional[Dict[int, Logprob]]],
99 num_output_top_logprobs: Optional[int] = None,
100 initial_text_offset: int = 0,
101 ) -> LogProbs:
@@ -108,6 +111,7 @@ class OpenAIServing:
111 token = self.tokenizer.decode(token_id)
112 logprobs.tokens.append(token)
113 logprobs.token_logprobs.append(None)
114 + assert logprobs.top_logprobs is not None
115 logprobs.top_logprobs.append(None)
116 else:
117 token_logprob = step_top_logprobs[token_id].logprob
@@ -116,6 +120,7 @@ class OpenAIServing:
120 logprobs.token_logprobs.append(token_logprob)
121
122 if num_output_top_logprobs:
123 + assert logprobs.top_logprobs is not None
124 logprobs.top_logprobs.append({
125 # Convert float("-inf") to the
126 # JSON-serializable float that OpenAI uses
@@ -155,9 +160,9 @@ class OpenAIServing:
160
161 async def _check_model(self, request) -> Optional[ErrorResponse]:
162 if request.model in self.served_model_names:
158 - return
163 + return None
164 if request.model in [lora.lora_name for lora in self.lora_requests]:
160 - return
165 + return None
166 return self.create_error_response(
167 message=f"The model `{request.model}` does not exist.",
168 err_type="NotFoundError",
@@ -165,7 +170,7 @@ class OpenAIServing:
170
171 def _maybe_get_lora(self, request) -> Optional[LoRARequest]:
172 if request.model in self.served_model_names:
168 - return
173 + return None
174 for lora in self.lora_requests:
175 if request.model == lora.lora_name:
176 return lora
@@ -177,7 +182,7 @@ class OpenAIServing:
182 request: Union[ChatCompletionRequest, CompletionRequest],
183 prompt: Optional[str] = None,
184 prompt_ids: Optional[List[int]] = None,
180 - truncate_prompt_tokens: Optional[conint(ge=1)] = None
185 + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
186 ) -> Tuple[List[int], str]:
187 if not (prompt or prompt_ids):
188 raise ValueError("Either prompt or prompt_ids should be provided.")
vllm/lora/lora.py
+1 -1
@@ -33,7 +33,7 @@ class LoRALayerWeights:
33 def optimize(self) -> "LoRALayerWeights":
34 """Optimize the LoRA by merging the scaling into lora_b."""
35 if self.scaling == 1:
36 - return
36 + return self
37 self.lora_b *= self.scaling
38 self.scaling = 1
39 return self
vllm/model_executor/layers/ops/sample.py
+2 -1
@@ -29,8 +29,8 @@ def _multi_split_sample(
29 sampled_tokens_size: Tuple[int, int],
30 sampled_logprobs_size: Tuple[int, int],
31 sample_indices: torch.Tensor,
32 + logprobs: torch.Tensor,
33 *,
33 - logprobs: Optional[torch.Tensor] = None,
34 modify_greedy_probs: bool = False,
35 save_logprobs: bool = False,
36 ):
@@ -167,6 +167,7 @@ def sample(
167 sampled_logprobs_size = (0, 0)
168 logprobs = probs
169
170 + assert logprobs is not None
171 if _save_modified_probs:
172 sampled_modified_probs_size = sampled_tokens_size
173 else:
vllm/model_executor/layers/rotary_embedding.py
+2 -1
@@ -108,7 +108,8 @@ class RotaryEmbedding(nn.Module):
108 query_pass = query[..., self.rotary_dim:]
109 key_pass = key[..., self.rotary_dim:]
110
111 - self.cos_sin_cache = self.cos_sin_cache.to(positions.device)
111 + self.cos_sin_cache: torch.Tensor = self.cos_sin_cache.to(
112 + positions.device)
113 cos_sin = self.cos_sin_cache[torch.add(positions, offsets)
114 if offsets is not None else positions]
115 cos, sin = cos_sin.chunk(2, dim=-1)
vllm/transformers_utils/configs/jais.py
+4 -2
@@ -222,13 +222,15 @@ class JAISConfig(PretrainedConfig):
222 f"got {alibi_scaling_type}")
223 if (alibi_scaling_factor is not None
224 and not isinstance(alibi_scaling_factor, float)
225 - or alibi_scaling_factor <= 1.0):
225 + or (alibi_scaling_factor is not None
226 + and alibi_scaling_factor <= 1.0)):
227 raise ValueError(
228 f"`alibi_scaling`'s factor field must be a float > 1.0,"
229 f"got {alibi_scaling_factor}")
230 if (alibi_dynamic_scaling is not None
231 and not isinstance(alibi_dynamic_scaling, int)
231 - or alibi_dynamic_scaling <= 1):
232 + or (alibi_dynamic_scaling is not None
233 + and alibi_dynamic_scaling <= 1)):
234 raise ValueError(
235 f"`alibi_scaling`'s `train_seq_len` field must be an"
236 f"integer > 1, got {alibi_dynamic_scaling}")
vllm/transformers_utils/tokenizer_group/__init__.py
+1 -1
@@ -11,7 +11,7 @@ if ray:
11 from vllm.transformers_utils.tokenizer_group.ray_tokenizer_group import (
12 RayTokenizerGroupPool)
13 else:
14 - RayTokenizerGroupPool = None
14 + RayTokenizerGroupPool = None # type: ignore
15
16
17 def get_tokenizer_group(tokenizer_pool_config: Optional[TokenizerPoolConfig],
vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py
+2
@@ -89,6 +89,7 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
89 This is blocking.
90 """
91 self._ensure_queue_initialized()
92 + assert self._idle_actors is not None
93
94 if self._idle_actors.empty():
95 raise RuntimeError("No idle actors available.")
@@ -120,6 +121,7 @@ class RayTokenizerGroupPool(BaseTokenizerGroup):
121 This is non-blocking.
122 """
123 self._ensure_queue_initialized()
124 + assert self._idle_actors is not None
125
126 actor = await self._idle_actors.get()
127 try:
vllm/transformers_utils/tokenizers/baichuan.py
+2 -2
@@ -114,9 +114,9 @@ class BaichuanTokenizer(PreTrainedTokenizer):
114 token = self.sp_model.IdToPiece(index)
115 return token
116
117 - def convert_tokens_to_string(self, tokens):
117 + def convert_tokens_to_string(self, tokens: List[str]):
118 """Converts a sequence of tokens (string) in a single string."""
119 - current_sub_tokens = []
119 + current_sub_tokens: List[str] = []
120 out_string = ""
121 prev_is_special = False
122 for i, token in enumerate(tokens):