@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6d917d0eeb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

Enable mypy checking on V1 code (#11105)

Signed-off-by: Mark McLoughlin <markmc@redhat.com>

Mark McLoughlin committed Dec 14, 2024 at 17:54 UTC 6d917d0eebd03990edf2443780a5f2506026ea78
21 files changed +160 -121
tools/mypy.sh
+1
@@ -29,3 +29,4 @@ run_mypy vllm/plugins
29 run_mypy vllm/prompt_adapter
30 run_mypy vllm/spec_decode
31 run_mypy vllm/worker
32 +run_mypy vllm/v1
vllm/v1/attention/backends/flash_attn.py
+2
@@ -135,6 +135,8 @@ class FlashAttentionImpl(AttentionImpl):
135 assert k_scale == 1.0 and v_scale == 1.0, (
136 "key/v_scale is not supported in FlashAttention.")
137
138 + assert output is not None, "Output tensor must be provided."
139 +
140 if attn_metadata is None:
141 # Profiling run.
142 return output
vllm/v1/core/kv_cache_manager.py
+5 -5
@@ -1,5 +1,5 @@
1 from collections import defaultdict
2 -from typing import Dict, List, Optional
2 +from typing import Dict, Iterable, List, Optional
3
4 from vllm.logger import init_logger
5 from vllm.utils import cdiv
@@ -263,12 +263,13 @@ class KVCacheManager:
263 """
264 # Default to [] in case a request is freed (aborted) before alloc.
265 blocks = self.req_to_blocks.pop(request.request_id, [])
266 + ordered_blocks: Iterable[KVCacheBlock] = blocks
267 if self.enable_caching:
268 # Free blocks in reverse order so that the tail blocks are
269 # freed first.
269 - blocks = reversed(blocks)
270 + ordered_blocks = reversed(blocks)
271
271 - for block in blocks:
272 + for block in ordered_blocks:
273 block.decr_ref()
274 if block.ref_cnt == 0:
275 self.free_block_queue.append(block)
@@ -396,8 +397,7 @@ class KVCacheManager:
397 f"{request.request_id}({request})")
398
399 # Compute the hash of the current block.
399 - block_hash = hash_block_tokens(prev_block_hash_value,
400 - tuple(block_tokens))
400 + block_hash = hash_block_tokens(prev_block_hash_value, block_tokens)
401
402 # Update and added the full block to the cache.
403 blk.block_hash = block_hash
vllm/v1/core/kv_cache_utils.py
+9 -8
@@ -1,4 +1,5 @@
1 """KV-Cache Utilities."""
2 +from collections.abc import Sequence
3 from dataclasses import dataclass
4 from typing import List, NamedTuple, Optional, Tuple
5
@@ -13,7 +14,7 @@ class BlockHashType(NamedTuple):
14 collision happens when the hash value is the same.
15 """
16 hash_value: int
16 - token_ids: Tuple[int]
17 + token_ids: Tuple[int, ...]
18
19
20 @dataclass
@@ -79,8 +80,8 @@ class FreeKVCacheBlockQueue:
80 self.num_free_blocks = len(blocks)
81
82 # Initialize the doubly linked list of free blocks.
82 - self.free_list_head = blocks[0]
83 - self.free_list_tail = blocks[-1]
83 + self.free_list_head: Optional[KVCacheBlock] = blocks[0]
84 + self.free_list_tail: Optional[KVCacheBlock] = blocks[-1]
85 for i in range(self.num_free_blocks):
86 if i > 0:
87 blocks[i].prev_free_block = blocks[i - 1]
@@ -159,7 +160,7 @@ class FreeKVCacheBlockQueue:
160
161
162 def hash_block_tokens(parent_block_hash: Optional[int],
162 - curr_block_token_ids: Tuple[int]) -> BlockHashType:
163 + curr_block_token_ids: Sequence[int]) -> BlockHashType:
164 """Computes a hash value corresponding to the contents of a block and
165 the contents of the preceding block(s). The hash value is used for
166 prefix caching. We use LRU cache for this function to avoid recomputing
@@ -171,7 +172,7 @@ def hash_block_tokens(parent_block_hash: Optional[int],
172 Args:
173 parent_block_hash: The hash of the parent block. None
174 if this is the first block.
174 - curr_block_token_ids: A tuple of token ids in the current
175 + curr_block_token_ids: A list of token ids in the current
176 block. The current block is assumed to be full.
177
178 Returns:
@@ -179,11 +180,11 @@ def hash_block_tokens(parent_block_hash: Optional[int],
180 The entire tuple is used as the hash key of the block.
181 """
182 return BlockHashType(hash((parent_block_hash, *curr_block_token_ids)),
182 - curr_block_token_ids)
183 + tuple(curr_block_token_ids))
184
185
186 def hash_request_tokens(block_size: int,
186 - token_ids: List[int]) -> List[BlockHashType]:
187 + token_ids: Sequence[int]) -> List[BlockHashType]:
188 """Computes hash values of a chain of blocks given a sequence of
189 token IDs. The hash value is used for prefix caching.
190
@@ -198,7 +199,7 @@ def hash_request_tokens(block_size: int,
199 parent_block_hash_value = None
200 for start in range(0, len(token_ids), block_size):
201 end = start + block_size
201 - block_token_ids = tuple(token_ids[start:end])
202 + block_token_ids = token_ids[start:end]
203 # Do not hash the block if it is not full.
204 if len(block_token_ids) < block_size:
205 break
vllm/v1/core/scheduler.py
+1
@@ -152,6 +152,7 @@ class Scheduler:
152 break
153 if not can_schedule:
154 break
155 + assert new_blocks is not None
156
157 # Schedule the request.
158 scheduled_running_reqs.append(request)
vllm/v1/engine/__init__.py
+14 -9
@@ -36,7 +36,7 @@ class EngineCoreRequest:
36 prompt: Optional[str]
37 prompt_token_ids: List[int]
38 mm_inputs: Optional[List[Optional[MultiModalKwargs]]]
39 - mm_hashes: Optional[List[Optional[str]]]
39 + mm_hashes: Optional[List[str]]
40 mm_placeholders: Optional[MultiModalPlaceholderDict]
41 sampling_params: SamplingParams
42 eos_token_id: Optional[int]
@@ -44,10 +44,11 @@ class EngineCoreRequest:
44 lora_request: Optional[LoRARequest]
45
46
47 -class EngineCoreOutput(msgspec.Struct,
48 - array_like=True,
49 - omit_defaults=True,
50 - gc=False):
47 +class EngineCoreOutput(
48 + msgspec.Struct,
49 + array_like=True, # type: ignore[call-arg]
50 + omit_defaults=True, # type: ignore[call-arg]
51 + gc=False): # type: ignore[call-arg]
52
53 request_id: str
54 new_token_ids: List[int]
@@ -56,10 +57,11 @@ class EngineCoreOutput(msgspec.Struct,
57 stop_reason: Union[int, str, None] = None
58
59
59 -class EngineCoreOutputs(msgspec.Struct,
60 - array_like=True,
61 - omit_defaults=True,
62 - gc=False):
60 +class EngineCoreOutputs(
61 + msgspec.Struct,
62 + array_like=True, # type: ignore[call-arg]
63 + omit_defaults=True, # type: ignore[call-arg]
64 + gc=False): # type: ignore[call-arg]
65
66 #NOTE(Nick): We could consider ways to make this more compact,
67 # e.g. columnwise layout and using an int enum for finish/stop reason
@@ -81,3 +83,6 @@ class EngineCoreRequestType(enum.Enum):
83 ADD = b'\x00'
84 ABORT = b'\x01'
85 PROFILE = b'\x02'
86 +
87 +
88 +EngineCoreRequestUnion = Union[EngineCoreRequest, EngineCoreProfile, List[str]]
vllm/v1/engine/async_llm.py
+6 -5
@@ -81,7 +81,7 @@ class AsyncLLM(EngineClient):
81 asyncio_mode=True,
82 )
83
84 - self.output_handler = None
84 + self.output_handler: Optional[asyncio.Task] = None
85
86 def __del__(self):
87 self.shutdown()
@@ -126,7 +126,8 @@ class AsyncLLM(EngineClient):
126 handler.cancel()
127
128 @classmethod
129 - def _get_executor_cls(cls, vllm_config: VllmConfig):
129 + def _get_executor_cls(cls, vllm_config: VllmConfig) -> Type[Executor]:
130 + executor_class: Type[Executor]
131 distributed_executor_backend = (
132 vllm_config.parallel_config.distributed_executor_backend)
133 if distributed_executor_backend == "mp":
@@ -361,10 +362,10 @@ class AsyncLLM(EngineClient):
362 logger.debug("Called check_health.")
363
364 async def start_profile(self) -> None:
364 - await self.engine_core.profile(True)
365 + await self.engine_core.profile_async(True)
366
367 async def stop_profile(self) -> None:
367 - await self.engine_core.profile(False)
368 + await self.engine_core.profile_async(False)
369
370 @property
371 def is_running(self) -> bool:
@@ -380,7 +381,7 @@ class AsyncLLM(EngineClient):
381
382 @property
383 def dead_error(self) -> BaseException:
383 - return Exception
384 + return Exception() # TODO: implement
385
386
387 # Retain V0 name for backwards compatibility.
vllm/v1/engine/core.py
+10 -10
@@ -5,7 +5,7 @@ import threading
5 import time
6 from dataclasses import dataclass
7 from multiprocessing.process import BaseProcess
8 -from typing import List, Tuple, Type, Union
8 +from typing import List, Tuple, Type
9
10 import zmq
11 import zmq.asyncio
@@ -20,7 +20,7 @@ from vllm.usage.usage_lib import UsageContext
20 from vllm.v1.core.scheduler import Scheduler
21 from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
22 EngineCoreProfile, EngineCoreRequest,
23 - EngineCoreRequestType)
23 + EngineCoreRequestType, EngineCoreRequestUnion)
24 from vllm.v1.engine.mm_input_mapper import MMInputMapperServer
25 from vllm.v1.executor.abstract import Executor
26 from vllm.v1.request import Request, RequestStatus
@@ -97,8 +97,10 @@ class EngineCore:
97 # Note that the cache here is mirrored with the client side of the
98 # MM mapper, so anything that has a hash must have a HIT cache
99 # entry here as well.
100 - request.mm_inputs = self.mm_input_mapper_server.process_inputs(
101 - request.mm_inputs, request.mm_hashes)
100 + assert request.mm_inputs is not None
101 + request.mm_inputs, request.mm_hashes = (
102 + self.mm_input_mapper_server.process_inputs(
103 + request.mm_inputs, request.mm_hashes))
104
105 req = Request.from_engine_core_request(request)
106
@@ -128,7 +130,7 @@ class EngineCore:
130 def shutdown(self):
131 self.model_executor.shutdown()
132
131 - def profile(self, is_start=True):
133 + def profile(self, is_start: bool = True):
134 self.model_executor.profile(is_start)
135
136
@@ -161,8 +163,8 @@ class EngineCoreProc(EngineCore):
163 # and to overlap some serialization/deserialization with the
164 # model forward pass.
165 # Threads handle Socket <-> Queues and core_busy_loop uses Queue.
164 - self.input_queue = queue.Queue()
165 - self.output_queue = queue.Queue()
166 + self.input_queue: queue.Queue[EngineCoreRequestUnion] = queue.Queue()
167 + self.output_queue: queue.Queue[List[EngineCoreOutput]] = queue.Queue()
168 threading.Thread(target=self.process_input_socket,
169 args=(input_path, ),
170 daemon=True).start()
@@ -318,9 +320,7 @@ class EngineCoreProc(EngineCore):
320
321 self._last_logging_time = now
322
321 - def _handle_client_request(
322 - self, request: Union[EngineCoreRequest, EngineCoreProfile,
323 - List[str]]) -> None:
323 + def _handle_client_request(self, request: EngineCoreRequestUnion) -> None:
324 """Handle EngineCoreRequest or EngineCoreABORT from Client."""
325
326 if isinstance(request, EngineCoreRequest):
vllm/v1/engine/core_client.py
+24 -19
@@ -1,6 +1,6 @@
1 import atexit
2 import os
3 -from typing import List, Union
3 +from typing import List, Optional
4
5 import msgspec
6 import zmq
@@ -10,8 +10,9 @@ from vllm.logger import init_logger
10 from vllm.utils import get_open_zmq_ipc_path, kill_process_tree
11 from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
12 EngineCoreProfile, EngineCoreRequest,
13 - EngineCoreRequestType)
14 -from vllm.v1.engine.core import EngineCore, EngineCoreProc
13 + EngineCoreRequestType, EngineCoreRequestUnion)
14 +from vllm.v1.engine.core import (EngineCore, EngineCoreProc,
15 + EngineCoreProcHandle)
16 from vllm.v1.serial_utils import PickleEncoder
17
18 logger = init_logger(__name__)
@@ -59,7 +60,7 @@ class EngineCoreClient:
60 def add_request(self, request: EngineCoreRequest) -> None:
61 raise NotImplementedError
62
62 - async def profile(self, is_start=True) -> None:
63 + def profile(self, is_start: bool = True) -> None:
64 raise NotImplementedError
65
66 def abort_requests(self, request_ids: List[str]) -> None:
@@ -71,6 +72,9 @@ class EngineCoreClient:
72 async def add_request_async(self, request: EngineCoreRequest) -> None:
73 raise NotImplementedError
74
75 + async def profile_async(self, is_start: bool = True) -> None:
76 + raise NotImplementedError
77 +
78 async def abort_requests_async(self, request_ids: List[str]) -> None:
79 raise NotImplementedError
80
@@ -105,7 +109,7 @@ class InprocClient(EngineCoreClient):
109 def __del__(self):
110 self.shutdown()
111
108 - def profile(self, is_start=True) -> None:
112 + def profile(self, is_start: bool = True) -> None:
113 self.engine_core.profile(is_start)
114
115
@@ -133,7 +137,10 @@ class MPClient(EngineCoreClient):
137 self.decoder = msgspec.msgpack.Decoder(EngineCoreOutputs)
138
139 # ZMQ setup.
136 - self.ctx = (zmq.asyncio.Context() if asyncio_mode else zmq.Context())
140 + if asyncio_mode:
141 + self.ctx = zmq.asyncio.Context()
142 + else:
143 + self.ctx = zmq.Context() # type: ignore[attr-defined]
144
145 # Path for IPC.
146 ready_path = get_open_zmq_ipc_path()
@@ -149,11 +156,13 @@ class MPClient(EngineCoreClient):
156 self.input_socket.bind(input_path)
157
158 # Start EngineCore in background process.
159 + self.proc_handle: Optional[EngineCoreProcHandle]
160 self.proc_handle = EngineCoreProc.make_engine_core_process(
161 *args,
154 - input_path=input_path,
155 - output_path=output_path,
156 - ready_path=ready_path,
162 + input_path=
163 + input_path, # type: ignore[misc] # MyPy incorrectly flags duplicate keywords
164 + output_path=output_path, # type: ignore[misc]
165 + ready_path=ready_path, # type: ignore[misc]
166 **kwargs,
167 )
168 atexit.register(self.shutdown)
@@ -204,10 +213,8 @@ class SyncMPClient(MPClient):
213 engine_core_outputs = self.decoder.decode(frame.buffer).outputs
214 return engine_core_outputs
215
207 - def _send_input(
208 - self, request_type: EngineCoreRequestType,
209 - request: Union[EngineCoreRequest, EngineCoreProfile,
210 - List[str]]) -> None:
216 + def _send_input(self, request_type: EngineCoreRequestType,
217 + request: EngineCoreRequestUnion) -> None:
218
219 # (RequestType, SerializedRequest)
220 msg = (request_type.value, self.encoder.encode(request))
@@ -219,7 +226,7 @@ class SyncMPClient(MPClient):
226 def abort_requests(self, request_ids: List[str]) -> None:
227 self._send_input(EngineCoreRequestType.ABORT, request_ids)
228
222 - def profile(self, is_start=True) -> None:
229 + def profile(self, is_start: bool = True) -> None:
230 self._send_input(EngineCoreRequestType.PROFILE,
231 EngineCoreProfile(is_start))
232
@@ -237,10 +244,8 @@ class AsyncMPClient(MPClient):
244
245 return engine_core_outputs
246
240 - async def _send_input(
241 - self, request_type: EngineCoreRequestType,
242 - request: Union[EngineCoreRequest, EngineCoreProfile,
243 - List[str]]) -> None:
247 + async def _send_input(self, request_type: EngineCoreRequestType,
248 + request: EngineCoreRequestUnion) -> None:
249
250 msg = (request_type.value, self.encoder.encode(request))
251 await self.input_socket.send_multipart(msg, copy=False)
@@ -252,6 +257,6 @@ class AsyncMPClient(MPClient):
257 if len(request_ids) > 0:
258 await self._send_input(EngineCoreRequestType.ABORT, request_ids)
259
255 - async def profile(self, is_start=True) -> None:
260 + async def profile_async(self, is_start: bool = True) -> None:
261 await self._send_input(EngineCoreRequestType.PROFILE,
262 EngineCoreProfile(is_start))
vllm/v1/engine/detokenizer.py
+2 -2
@@ -1,5 +1,5 @@
1 from dataclasses import dataclass
2 -from typing import Dict, Iterable, List, Optional, Tuple
2 +from typing import Dict, Iterable, List, Optional, Tuple, Union
3
4 from vllm.engine.output_processor.stop_checker import StopChecker
5 from vllm.logger import init_logger
@@ -97,7 +97,7 @@ class IncrementalDetokenizer:
97 self,
98 new_token_ids: List[int],
99 finish_reason: Optional[str],
100 - stop_reason: Optional[str],
100 + stop_reason: Optional[Union[int, str, None]],
101 ) -> Optional[RequestOutput]:
102 """
103 Update RequestState for the request_id by:
vllm/v1/engine/llm_engine.py
+2 -1
@@ -103,7 +103,8 @@ class LLMEngine:
103 multiprocess_mode=enable_multiprocessing)
104
105 @classmethod
106 - def _get_executor_cls(cls, vllm_config: VllmConfig):
106 + def _get_executor_cls(cls, vllm_config: VllmConfig) -> Type[Executor]:
107 + executor_class: Type[Executor]
108 distributed_executor_backend = (
109 vllm_config.parallel_config.distributed_executor_backend)
110 if distributed_executor_backend == "mp":
vllm/v1/engine/mm_input_mapper.py
+13 -7
@@ -1,4 +1,4 @@
1 -from typing import Any, Dict, List, Optional
1 +from typing import Any, Dict, List, Optional, Tuple
2
3 import PIL
4 from blake3 import blake3
@@ -42,14 +42,14 @@ class MMInputMapperClient:
42 model_config)
43 self.mm_registry.init_mm_limits_per_prompt(model_config)
44
45 - self.mm_cache = LRUDictCache(MM_CACHE_SIZE)
45 + self.mm_cache = LRUDictCache[str, MultiModalKwargs](MM_CACHE_SIZE)
46
47 # DEBUG: Set to None to disable
48 self.mm_debug_cache_hit_ratio_steps = None
49 self.mm_cache_hits = 0
50 self.mm_cache_total = 0
51
52 - def cache_hit_ratio(self, steps) -> float:
52 + def cache_hit_ratio(self, steps):
53 if self.mm_cache_total > 0 and self.mm_cache_total % steps == 0:
54 logger.debug("MMInputMapper: cache_hit_ratio = %.2f ",
55 self.mm_cache_hits / self.mm_cache_total)
@@ -60,7 +60,7 @@ class MMInputMapperClient:
60 mm_hashes: Optional[List[str]],
61 mm_processor_kwargs: Optional[Dict[str, Any]],
62 precomputed_mm_inputs: Optional[List[MultiModalKwargs]],
63 - ) -> List[MultiModalKwargs]:
63 + ) -> Tuple[List[MultiModalKwargs], Optional[List[str]]]:
64 if precomputed_mm_inputs is None:
65 image_inputs = mm_data["image"]
66 if not isinstance(image_inputs, list):
@@ -72,6 +72,7 @@ class MMInputMapperClient:
72 # Check if hash is enabled
73 use_hash = mm_hashes is not None
74 if use_hash:
75 + assert mm_hashes is not None
76 assert num_inputs == len(
77 mm_hashes), "num_inputs = {} len(mm_hashes) = {}".format(
78 num_inputs, len(mm_hashes))
@@ -79,7 +80,7 @@ class MMInputMapperClient:
80 # Process each image input separately, so that later we can schedule
81 # them in a fine-grained manner.
82 # Apply caching (if enabled) and reuse precomputed inputs (if provided)
82 - ret_hashes = [] if use_hash else None
83 + ret_hashes: Optional[List[str]] = [] if use_hash else None
84 ret_inputs: List[MultiModalKwargs] = []
85 for input_id in range(num_inputs):
86 if self.mm_debug_cache_hit_ratio_steps is not None:
@@ -88,6 +89,7 @@ class MMInputMapperClient:
89 mm_hash = None
90 mm_input = None
91 if use_hash:
92 + assert mm_hashes is not None
93 mm_hash = mm_hashes[input_id]
94 mm_input = self.mm_cache.get(mm_hash)
95
@@ -105,12 +107,15 @@ class MMInputMapperClient:
107
108 if use_hash:
109 # Add to cache
110 + assert mm_hash is not None
111 self.mm_cache.put(mm_hash, mm_input)
112 else:
113 self.mm_cache_hits += 1
114 mm_input = None # Avoids sending mm_input to Server
115
116 if use_hash:
117 + assert mm_hash is not None
118 + assert ret_hashes is not None
119 ret_hashes.append(mm_hash)
120 ret_inputs.append(mm_input)
121
@@ -120,17 +125,18 @@ class MMInputMapperClient:
125 class MMInputMapperServer:
126
127 def __init__(self, ):
123 - self.mm_cache = LRUDictCache(MM_CACHE_SIZE)
128 + self.mm_cache = LRUDictCache[str, MultiModalKwargs](MM_CACHE_SIZE)
129
130 def process_inputs(
131 self,
132 mm_inputs: List[Optional[MultiModalKwargs]],
128 - mm_hashes: List[Optional[str]],
133 + mm_hashes: List[str],
134 ) -> List[MultiModalKwargs]:
135 assert len(mm_inputs) == len(mm_hashes)
136
137 full_mm_inputs = []
138 for mm_input, mm_hash in zip(mm_inputs, mm_hashes):
139 + assert mm_hash is not None
140 if mm_input is None:
141 mm_input = self.mm_cache.get(mm_hash)
142 assert mm_input is not None
vllm/v1/engine/processor.py
+1 -1
@@ -56,7 +56,7 @@ class Processor:
56 request_id: str,
57 prompt: PromptType,
58 params: Union[SamplingParams, PoolingParams],
59 - arrival_time: float,
59 + arrival_time: Optional[float] = None,
60 lora_request: Optional[LoRARequest] = None,
61 trace_headers: Optional[Mapping[str, str]] = None,
62 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
vllm/v1/executor/abstract.py
+2 -10
@@ -1,5 +1,5 @@
1 from abc import ABC, abstractmethod
2 -from typing import Dict, Optional, Tuple
2 +from typing import Tuple
3
4 from vllm.config import VllmConfig
5 from vllm.v1.outputs import ModelRunnerOutput
@@ -28,7 +28,7 @@ class Executor(ABC):
28 raise NotImplementedError
29
30 @abstractmethod
31 - def profile(self, is_start=True):
31 + def profile(self, is_start: bool = True):
32 raise NotImplementedError
33
34 @abstractmethod
@@ -38,11 +38,3 @@ class Executor(ABC):
38 @abstractmethod
39 def check_health(self) -> None:
40 raise NotImplementedError
41 -
42 - @abstractmethod
43 - def collective_rpc(self,
44 - method: str,
45 - timeout: Optional[float] = None,
46 - args: Tuple = (),
47 - kwargs: Optional[Dict] = None) -> []:
48 - raise NotImplementedError
vllm/v1/executor/multiproc_executor.py
+8 -7
@@ -7,7 +7,7 @@ import time
7 from dataclasses import dataclass
8 from enum import Enum, auto
9 from multiprocessing.process import BaseProcess
10 -from typing import Dict, List, Optional, Tuple
10 +from typing import Any, Dict, List, Optional, Tuple
11
12 import zmq
13
@@ -21,6 +21,7 @@ from vllm.executor.multiproc_worker_utils import (
21 from vllm.logger import init_logger
22 from vllm.utils import (get_distributed_init_method, get_open_port,
23 get_open_zmq_ipc_path)
24 +from vllm.v1.executor.abstract import Executor
25 from vllm.v1.outputs import ModelRunnerOutput
26 from vllm.v1.utils import make_zmq_socket
27 from vllm.worker.worker_base import WorkerWrapperBase
@@ -31,7 +32,7 @@ POLLING_TIMEOUT_MS = 5000
32 POLLING_TIMEOUT_S = POLLING_TIMEOUT_MS // 1000
33
34
34 -class MultiprocExecutor:
35 +class MultiprocExecutor(Executor):
36
37 def __init__(self, vllm_config: VllmConfig) -> None:
38 # Call self.shutdown at exit to clean up
@@ -103,7 +104,7 @@ class MultiprocExecutor:
104 method: str,
105 timeout: Optional[float] = None,
106 args: Tuple = (),
106 - kwargs: Optional[Dict] = None) -> []:
107 + kwargs: Optional[Dict] = None) -> List[Any]:
108 """
109 Execute an RPC call on workers.
110
@@ -125,7 +126,7 @@ class MultiprocExecutor:
126
127 responses = [None] * self.world_size
128 for w in self.workers:
128 - dequeue_timeout = timeout - (time.monotonic() - start_time()
129 + dequeue_timeout = timeout - (time.monotonic() - start_time
130 ) if timeout is not None else None
131 status, result = w.worker_response_mq.dequeue(
132 timeout=dequeue_timeout)
@@ -153,7 +154,7 @@ class MultiprocExecutor:
154 args=(scheduler_output, ))[0]
155 return model_output
156
156 - def profile(self, is_start=True):
157 + def profile(self, is_start: bool = True):
158 self.collective_rpc("profile", args=(is_start, ))
159 return
160
@@ -185,7 +186,6 @@ class MultiprocExecutor:
186 p.kill()
187
188 self._cleanup_sockets()
188 - self.workers = None
189
190 def _cleanup_sockets(self):
191 for w in self.workers:
@@ -200,7 +200,8 @@ class MultiprocExecutor:
200 # again
201 atexit.unregister(self.shutdown)
202 """Properly shut down the executor and its workers"""
203 - if (hasattr(self, 'workers') and self.workers is not None):
203 + if getattr(self, 'shutting_down', False):
204 + self.shutting_down = True
205 for w in self.workers: #TODO: not sure if needed
206 w.worker_response_mq = None
207 self._ensure_worker_termination()
vllm/v1/executor/uniproc_executor.py
+4 -3
@@ -4,13 +4,14 @@ from typing import Optional, Tuple
4 from vllm.config import VllmConfig
5 from vllm.logger import init_logger
6 from vllm.utils import get_distributed_init_method, get_ip, get_open_port
7 +from vllm.v1.executor.abstract import Executor
8 from vllm.v1.outputs import ModelRunnerOutput
9 from vllm.v1.worker.gpu_worker import Worker
10
11 logger = init_logger(__name__)
12
13
13 -class UniprocExecutor:
14 +class UniprocExecutor(Executor):
15
16 def __init__(self, vllm_config: VllmConfig) -> None:
17 self.vllm_config = vllm_config
@@ -25,7 +26,7 @@ class UniprocExecutor:
26 self.prompt_adapter_config = vllm_config.prompt_adapter_config
27 self.observability_config = vllm_config.observability_config
28
28 - self.worker = self._create_worker()
29 + self.worker: Worker = self._create_worker()
30 self.worker.initialize()
31 self.worker.load_model()
32
@@ -75,7 +76,7 @@ class UniprocExecutor:
76 self.worker.profile(is_start)
77
78 def shutdown(self):
78 - self.worker = None
79 + pass
80
81 def check_health(self) -> None:
82 # UniprocExecutor will always be healthy as long as
vllm/v1/request.py
+1 -2
@@ -52,10 +52,9 @@ class Request:
52 else:
53 self.mm_positions = []
54 # Output of the mm input mapper (e.g., image tensors).
55 + self.mm_inputs: List[MultiModalKwargs] = []
56 if self.inputs.multi_modal_inputs:
57 self.mm_inputs = self.inputs.multi_modal_inputs
57 - else:
58 - self.mm_inputs: List[MultiModalKwargs] = []
58
59 @classmethod
60 def from_engine_core_request(cls, request: EngineCoreRequest) -> "Request":
vllm/v1/utils.py
+27 -15
@@ -1,6 +1,8 @@
1 from collections import OrderedDict
2 +from collections.abc import Sequence
3 from contextlib import contextmanager
3 -from typing import Any, Generic, Iterator, List, TypeVar, overload
4 +from typing import (Any, Generic, Iterator, List, Optional, TypeVar, Union,
5 + overload)
6
7 import zmq
8
@@ -11,7 +13,7 @@ logger = init_logger(__name__)
13 T = TypeVar("T")
14
15
14 -class ConstantList(Generic[T]):
16 +class ConstantList(Generic[T], Sequence):
17
18 def __init__(self, x: List[T]) -> None:
19 self._x = x
@@ -34,29 +36,33 @@ class ConstantList(Generic[T]):
36 def clear(self):
37 raise Exception("Cannot clear a constant list")
38
37 - def index(self, item):
38 - return self._x.index(item)
39 + def index(self,
40 + item: T,
41 + start: int = 0,
42 + stop: Optional[int] = None) -> int:
43 + return self._x.index(item, start,
44 + stop if stop is not None else len(self._x))
45
46 @overload
41 - def __getitem__(self, item) -> T:
47 + def __getitem__(self, item: int) -> T:
48 ...
49
50 @overload
51 def __getitem__(self, s: slice, /) -> List[T]:
52 ...
53
48 - def __getitem__(self, item):
54 + def __getitem__(self, item: Union[int, slice]) -> Union[T, List[T]]:
55 return self._x[item]
56
57 @overload
52 - def __setitem__(self, item, value):
58 + def __setitem__(self, item: int, value: T):
59 ...
60
61 @overload
56 - def __setitem__(self, s: slice, value, /):
62 + def __setitem__(self, s: slice, value: T, /):
63 ...
64
59 - def __setitem__(self, item, value):
65 + def __setitem__(self, item: Union[int, slice], value: Union[T, List[T]]):
66 raise Exception("Cannot set item in a constant list")
67
68 def __delitem__(self, item):
@@ -73,10 +79,12 @@ class ConstantList(Generic[T]):
79
80
81 @contextmanager
76 -def make_zmq_socket(path: str, type: Any) -> Iterator[zmq.Socket]:
82 +def make_zmq_socket(
83 + path: str,
84 + type: Any) -> Iterator[zmq.Socket]: # type: ignore[name-defined]
85 """Context manager for a ZMQ socket"""
86
79 - ctx = zmq.Context()
87 + ctx = zmq.Context() # type: ignore[attr-defined]
88 try:
89 socket = ctx.socket(type)
90
@@ -96,20 +104,24 @@ def make_zmq_socket(path: str, type: Any) -> Iterator[zmq.Socket]:
104 ctx.destroy(linger=0)
105
106
99 -class LRUDictCache:
107 +K = TypeVar('K')
108 +V = TypeVar('V')
109 +
110 +
111 +class LRUDictCache(Generic[K, V]):
112
113 def __init__(self, size: int):
102 - self.cache = OrderedDict()
114 + self.cache: OrderedDict[K, V] = OrderedDict()
115 self.size = size
116
105 - def get(self, key, default=None):
117 + def get(self, key: K, default=None) -> V:
118 if key not in self.cache:
119 return default
120
121 self.cache.move_to_end(key)
122 return self.cache[key]
123
112 - def put(self, key, value):
124 + def put(self, key: K, value: V):
125 self.cache[key] = value
126 self.cache.move_to_end(key)
127 if len(self.cache) > self.size:
vllm/v1/worker/gpu_input_batch.py
+1
@@ -215,6 +215,7 @@ class InputBatch:
215
216 # Swap the states.
217 req_id = self.req_ids[last_req_index]
218 + assert req_id is not None
219 self.req_ids[empty_index] = req_id
220 self.req_ids[last_req_index] = None
221 self.req_id_to_index[req_id] = empty_index
vllm/v1/worker/gpu_model_runner.py
+26 -16
@@ -1,6 +1,6 @@
1 import gc
2 import time
3 -from typing import TYPE_CHECKING, Dict, List, Tuple
3 +from typing import TYPE_CHECKING, Dict, List, Tuple, cast
4
5 import numpy as np
6 import torch
@@ -193,9 +193,9 @@ class GPUModelRunner:
193
194 req_ids_to_add: List[str] = []
195 # Add new requests to the cached states.
196 - for req_data in scheduler_output.scheduled_new_reqs:
197 - req_id = req_data.req_id
198 - sampling_params = req_data.sampling_params
196 + for new_req_data in scheduler_output.scheduled_new_reqs:
197 + req_id = new_req_data.req_id
198 + sampling_params = new_req_data.sampling_params
199 if sampling_params.sampling_type == SamplingType.RANDOM_SEED:
200 generator = torch.Generator(device=self.device)
201 generator.manual_seed(sampling_params.seed)
@@ -204,25 +204,25 @@ class GPUModelRunner:
204
205 self.requests[req_id] = CachedRequestState(
206 req_id=req_id,
207 - prompt_token_ids=req_data.prompt_token_ids,
208 - prompt=req_data.prompt,
209 - mm_inputs=req_data.mm_inputs,
210 - mm_positions=req_data.mm_positions,
207 + prompt_token_ids=new_req_data.prompt_token_ids,
208 + prompt=new_req_data.prompt,
209 + mm_inputs=new_req_data.mm_inputs,
210 + mm_positions=new_req_data.mm_positions,
211 sampling_params=sampling_params,
212 generator=generator,
213 - block_ids=req_data.block_ids,
214 - num_computed_tokens=req_data.num_computed_tokens,
213 + block_ids=new_req_data.block_ids,
214 + num_computed_tokens=new_req_data.num_computed_tokens,
215 output_token_ids=[],
216 )
217 req_ids_to_add.append(req_id)
218
219 # Update the cached states of the resumed requests.
220 - for req_data in scheduler_output.scheduled_resumed_reqs:
221 - req_id = req_data.req_id
220 + for res_req_data in scheduler_output.scheduled_resumed_reqs:
221 + req_id = res_req_data.req_id
222 req_state = self.requests[req_id]
223
224 - req_state.block_ids = req_data.block_ids
225 - req_state.num_computed_tokens = req_data.num_computed_tokens
224 + req_state.block_ids = res_req_data.block_ids
225 + req_state.num_computed_tokens = res_req_data.num_computed_tokens
226 req_ids_to_add.append(req_id)
227
228 # Add the new or resumed requests to the persistent batch.
@@ -259,6 +259,7 @@ class GPUModelRunner:
259 num_scheduled_tokens = []
260 max_num_scheduled_tokens = 0
261 for req_id in self.input_batch.req_ids[:num_reqs]:
262 + assert req_id is not None
263 num_tokens = scheduler_output.num_scheduled_tokens[req_id]
264 num_scheduled_tokens.append(num_tokens)
265 max_num_scheduled_tokens = max(max_num_scheduled_tokens,
@@ -373,7 +374,7 @@ class GPUModelRunner:
374
375 # Batch the multi-modal inputs.
376 mm_inputs: List[MultiModalKwargs] = []
376 - req_input_ids: List[Tuple[int, int]] = []
377 + req_input_ids: List[Tuple[str, int]] = []
378 for req_id, encoder_input_ids in scheduled_encoder_inputs.items():
379 req_state = self.requests[req_id]
380 for input_id in encoder_input_ids:
@@ -406,6 +407,7 @@ class GPUModelRunner:
407 encoder_outputs: List[torch.Tensor] = []
408 num_reqs = self.input_batch.num_reqs
409 for req_id in self.input_batch.req_ids[:num_reqs]:
410 + assert req_id is not None
411 num_scheduled_tokens = scheduler_output.num_scheduled_tokens[
412 req_id]
413 req_state = self.requests[req_id]
@@ -514,6 +516,7 @@ class GPUModelRunner:
516 # the requests one by one. Optimize.
517 num_reqs = self.input_batch.num_reqs
518 for i, req_id in enumerate(self.input_batch.req_ids[:num_reqs]):
519 + assert req_id is not None
520 req_state = self.requests[req_id]
521 seq_len = (req_state.num_computed_tokens +
522 scheduler_output.num_scheduled_tokens[req_id])
@@ -539,8 +542,15 @@ class GPUModelRunner:
542 logprobs = None
543 else:
544 logprobs = sampler_output.logprobs.cpu()
545 +
546 + # num_reqs entries should be non-None
547 + assert all(
548 + req_id is not None for req_id in
549 + self.input_batch.req_ids[:num_reqs]), "req_ids contains None"
550 + req_ids = cast(List[str], self.input_batch.req_ids[:num_reqs])
551 +
552 model_runner_output = ModelRunnerOutput(
543 - req_ids=self.input_batch.req_ids[:num_reqs],
553 + req_ids=req_ids,
554 req_id_to_index=self.input_batch.req_id_to_index,
555 sampled_token_ids=sampled_token_ids,
556 logprob_token_ids_cpu=logprob_token_ids,
vllm/v1/worker/gpu_worker.py
+1 -1
@@ -204,7 +204,7 @@ class Worker:
204 return output if self.rank == 0 else None
205 return output
206
207 - def profile(self, is_start=True):
207 + def profile(self, is_start: bool = True):
208 if self.profiler is None:
209 raise RuntimeError("Profiler is not enabled.")
210 if is_start: