A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
Enable mypy checking on V1 code (#11105)
Signed-off-by: Mark McLoughlin <markmc@redhat.com>
Mark McLoughlin committed
Dec 14, 2024 at 17:54 UTC
6d917d0eebd03990edf2443780a5f2506026ea78
21 files changed
+160
-121
tools/mypy.sh
+1
@@ -29,3 +29,4 @@ run_mypy vllm/plugins
29
run_mypy vllm/prompt_adapter
30
run_mypy vllm/spec_decode
31
run_mypy vllm/worker
32
+run_mypy vllm/v1
vllm/v1/attention/backends/flash_attn.py
+2
@@ -135,6 +135,8 @@ class FlashAttentionImpl(AttentionImpl):
135
assert k_scale == 1.0 and v_scale == 1.0, (
136
"key/v_scale is not supported in FlashAttention.")
137
138
+ assert output is not None, "Output tensor must be provided."
139
+
140
if attn_metadata is None:
141
# Profiling run.
142
return output
vllm/v1/core/kv_cache_manager.py
+5
-5
@@ -1,5 +1,5 @@
1
from collections import defaultdict
2
-from typing import Dict, List, Optional
2
+from typing import Dict, Iterable, List, Optional
3
4
from vllm.logger import init_logger
5
from vllm.utils import cdiv
@@ -263,12 +263,13 @@ class KVCacheManager:
263
"""
264
# Default to [] in case a request is freed (aborted) before alloc.
265
blocks = self.req_to_blocks.pop(request.request_id, [])
266
+ ordered_blocks: Iterable[KVCacheBlock] = blocks
267
if self.enable_caching:
268
# Free blocks in reverse order so that the tail blocks are
269
# freed first.
269
- blocks = reversed(blocks)
270
+ ordered_blocks = reversed(blocks)
271
271
- for block in blocks:
272
+ for block in ordered_blocks:
273
block.decr_ref()
274
if block.ref_cnt == 0:
275
self.free_block_queue.append(block)
@@ -396,8 +397,7 @@ class KVCacheManager:
397
f"{request.request_id}({request})")
398
399
# Compute the hash of the current block.
399
- block_hash = hash_block_tokens(prev_block_hash_value,
400
- tuple(block_tokens))
400
+ block_hash = hash_block_tokens(prev_block_hash_value, block_tokens)
401
402
# Update and added the full block to the cache.
403
blk.block_hash = block_hash
vllm/v1/core/kv_cache_utils.py
+9
-8
@@ -1,4 +1,5 @@
1
"""KV-Cache Utilities."""
2
+from collections.abc import Sequence
3
from dataclasses import dataclass
4
from typing import List, NamedTuple, Optional, Tuple
5
@@ -13,7 +14,7 @@ class BlockHashType(NamedTuple):
14
collision happens when the hash value is the same.
15
"""
16
hash_value: int
16
- token_ids: Tuple[int]
17
+ token_ids: Tuple[int, ...]
18
19
20
@dataclass
@@ -79,8 +80,8 @@ class FreeKVCacheBlockQueue:
80
self.num_free_blocks = len(blocks)
81
82
# Initialize the doubly linked list of free blocks.
82
- self.free_list_head = blocks[0]
83
- self.free_list_tail = blocks[-1]
83
+ self.free_list_head: Optional[KVCacheBlock] = blocks[0]
84
+ self.free_list_tail: Optional[KVCacheBlock] = blocks[-1]
85
for i in range(self.num_free_blocks):
86
if i > 0:
87
blocks[i].prev_free_block = blocks[i - 1]
@@ -159,7 +160,7 @@ class FreeKVCacheBlockQueue:
160
161
162
def hash_block_tokens(parent_block_hash: Optional[int],
162
- curr_block_token_ids: Tuple[int]) -> BlockHashType:
163
+ curr_block_token_ids: Sequence[int]) -> BlockHashType:
164
"""Computes a hash value corresponding to the contents of a block and
165
the contents of the preceding block(s). The hash value is used for
166
prefix caching. We use LRU cache for this function to avoid recomputing
@@ -171,7 +172,7 @@ def hash_block_tokens(parent_block_hash: Optional[int],
172
Args:
173
parent_block_hash: The hash of the parent block. None
174
if this is the first block.
174
- curr_block_token_ids: A tuple of token ids in the current
175
+ curr_block_token_ids: A list of token ids in the current
176
block. The current block is assumed to be full.
177
178
Returns:
@@ -179,11 +180,11 @@ def hash_block_tokens(parent_block_hash: Optional[int],
180
The entire tuple is used as the hash key of the block.
181
"""
182
return BlockHashType(hash((parent_block_hash, *curr_block_token_ids)),
182
- curr_block_token_ids)
183
+ tuple(curr_block_token_ids))
184
185
186
def hash_request_tokens(block_size: int,
186
- token_ids: List[int]) -> List[BlockHashType]:
187
+ token_ids: Sequence[int]) -> List[BlockHashType]:
188
"""Computes hash values of a chain of blocks given a sequence of
189
token IDs. The hash value is used for prefix caching.
190
@@ -198,7 +199,7 @@ def hash_request_tokens(block_size: int,
199
parent_block_hash_value = None
200
for start in range(0, len(token_ids), block_size):
201
end = start + block_size
201
- block_token_ids = tuple(token_ids[start:end])
202
+ block_token_ids = token_ids[start:end]
203
# Do not hash the block if it is not full.
204
if len(block_token_ids) < block_size:
205
break
vllm/v1/core/scheduler.py
+1
@@ -152,6 +152,7 @@ class Scheduler:
152
break
153
if not can_schedule:
154
break
155
+ assert new_blocks is not None
156
157
# Schedule the request.
158
scheduled_running_reqs.append(request)
vllm/v1/engine/__init__.py
+14
-9
@@ -36,7 +36,7 @@ class EngineCoreRequest:
36
prompt: Optional[str]
37
prompt_token_ids: List[int]
38
mm_inputs: Optional[List[Optional[MultiModalKwargs]]]
39
- mm_hashes: Optional[List[Optional[str]]]
39
+ mm_hashes: Optional[List[str]]
40
mm_placeholders: Optional[MultiModalPlaceholderDict]
41
sampling_params: SamplingParams
42
eos_token_id: Optional[int]
@@ -44,10 +44,11 @@ class EngineCoreRequest:
44
lora_request: Optional[LoRARequest]
45
46
47
-class EngineCoreOutput(msgspec.Struct,
48
- array_like=True,
49
- omit_defaults=True,
50
- gc=False):
47
+class EngineCoreOutput(
48
+ msgspec.Struct,
49
+ array_like=True, # type: ignore[call-arg]
50
+ omit_defaults=True, # type: ignore[call-arg]
51
+ gc=False): # type: ignore[call-arg]
52
53
request_id: str
54
new_token_ids: List[int]
@@ -56,10 +57,11 @@ class EngineCoreOutput(msgspec.Struct,
57
stop_reason: Union[int, str, None] = None
58
59
59
-class EngineCoreOutputs(msgspec.Struct,
60
- array_like=True,
61
- omit_defaults=True,
62
- gc=False):
60
+class EngineCoreOutputs(
61
+ msgspec.Struct,
62
+ array_like=True, # type: ignore[call-arg]
63
+ omit_defaults=True, # type: ignore[call-arg]
64
+ gc=False): # type: ignore[call-arg]
65
66
#NOTE(Nick): We could consider ways to make this more compact,
67
# e.g. columnwise layout and using an int enum for finish/stop reason
@@ -81,3 +83,6 @@ class EngineCoreRequestType(enum.Enum):
83
ADD = b'\x00'
84
ABORT = b'\x01'
85
PROFILE = b'\x02'
86
+
87
+
88
+EngineCoreRequestUnion = Union[EngineCoreRequest, EngineCoreProfile, List[str]]
vllm/v1/engine/async_llm.py
+6
-5
@@ -81,7 +81,7 @@ class AsyncLLM(EngineClient):
81
asyncio_mode=True,
82
)
83
84
- self.output_handler = None
84
+ self.output_handler: Optional[asyncio.Task] = None
85
86
def __del__(self):
87
self.shutdown()
@@ -126,7 +126,8 @@ class AsyncLLM(EngineClient):
126
handler.cancel()
127
128
@classmethod
129
- def _get_executor_cls(cls, vllm_config: VllmConfig):
129
+ def _get_executor_cls(cls, vllm_config: VllmConfig) -> Type[Executor]:
130
+ executor_class: Type[Executor]
131
distributed_executor_backend = (
132
vllm_config.parallel_config.distributed_executor_backend)
133
if distributed_executor_backend == "mp":
@@ -361,10 +362,10 @@ class AsyncLLM(EngineClient):
362
logger.debug("Called check_health.")
363
364
async def start_profile(self) -> None:
364
- await self.engine_core.profile(True)
365
+ await self.engine_core.profile_async(True)
366
367
async def stop_profile(self) -> None:
367
- await self.engine_core.profile(False)
368
+ await self.engine_core.profile_async(False)
369
370
@property
371
def is_running(self) -> bool:
@@ -380,7 +381,7 @@ class AsyncLLM(EngineClient):
381
382
@property
383
def dead_error(self) -> BaseException:
383
- return Exception
384
+ return Exception() # TODO: implement
385
386
387
# Retain V0 name for backwards compatibility.
vllm/v1/engine/core.py
+10
-10
@@ -5,7 +5,7 @@ import threading
5
import time
6
from dataclasses import dataclass
7
from multiprocessing.process import BaseProcess
8
-from typing import List, Tuple, Type, Union
8
+from typing import List, Tuple, Type
9
10
import zmq
11
import zmq.asyncio
@@ -20,7 +20,7 @@ from vllm.usage.usage_lib import UsageContext
20
from vllm.v1.core.scheduler import Scheduler
21
from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
22
EngineCoreProfile, EngineCoreRequest,
23
- EngineCoreRequestType)
23
+ EngineCoreRequestType, EngineCoreRequestUnion)
24
from vllm.v1.engine.mm_input_mapper import MMInputMapperServer
25
from vllm.v1.executor.abstract import Executor
26
from vllm.v1.request import Request, RequestStatus
@@ -97,8 +97,10 @@ class EngineCore:
97
# Note that the cache here is mirrored with the client side of the
98
# MM mapper, so anything that has a hash must have a HIT cache
99
# entry here as well.
100
- request.mm_inputs = self.mm_input_mapper_server.process_inputs(
101
- request.mm_inputs, request.mm_hashes)
100
+ assert request.mm_inputs is not None
101
+ request.mm_inputs, request.mm_hashes = (
102
+ self.mm_input_mapper_server.process_inputs(
103
+ request.mm_inputs, request.mm_hashes))
104
105
req = Request.from_engine_core_request(request)
106
@@ -128,7 +130,7 @@ class EngineCore:
130
def shutdown(self):
131
self.model_executor.shutdown()
132
131
- def profile(self, is_start=True):
133
+ def profile(self, is_start: bool = True):
134
self.model_executor.profile(is_start)
135
136
@@ -161,8 +163,8 @@ class EngineCoreProc(EngineCore):
163
# and to overlap some serialization/deserialization with the
164
# model forward pass.
165
# Threads handle Socket <-> Queues and core_busy_loop uses Queue.
164
- self.input_queue = queue.Queue()
165
- self.output_queue = queue.Queue()
166
+ self.input_queue: queue.Queue[EngineCoreRequestUnion] = queue.Queue()
167
+ self.output_queue: queue.Queue[List[EngineCoreOutput]] = queue.Queue()
168
threading.Thread(target=self.process_input_socket,
169
args=(input_path, ),
170
daemon=True).start()
@@ -318,9 +320,7 @@ class EngineCoreProc(EngineCore):
320
321
self._last_logging_time = now
322
321
- def _handle_client_request(
322
- self, request: Union[EngineCoreRequest, EngineCoreProfile,
323
- List[str]]) -> None:
323
+ def _handle_client_request(self, request: EngineCoreRequestUnion) -> None:
324
"""Handle EngineCoreRequest or EngineCoreABORT from Client."""
325
326
if isinstance(request, EngineCoreRequest):
vllm/v1/engine/core_client.py
+24
-19
@@ -1,6 +1,6 @@
1
import atexit
2
import os
3
-from typing import List, Union
3
+from typing import List, Optional
4
5
import msgspec
6
import zmq
@@ -10,8 +10,9 @@ from vllm.logger import init_logger
10
from vllm.utils import get_open_zmq_ipc_path, kill_process_tree
11
from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
12
EngineCoreProfile, EngineCoreRequest,
13
- EngineCoreRequestType)
14
-from vllm.v1.engine.core import EngineCore, EngineCoreProc
13
+ EngineCoreRequestType, EngineCoreRequestUnion)
14
+from vllm.v1.engine.core import (EngineCore, EngineCoreProc,
15
+ EngineCoreProcHandle)
16
from vllm.v1.serial_utils import PickleEncoder
17
18
logger = init_logger(__name__)
@@ -59,7 +60,7 @@ class EngineCoreClient:
60
def add_request(self, request: EngineCoreRequest) -> None:
61
raise NotImplementedError
62
62
- async def profile(self, is_start=True) -> None:
63
+ def profile(self, is_start: bool = True) -> None:
64
raise NotImplementedError
65
66
def abort_requests(self, request_ids: List[str]) -> None:
@@ -71,6 +72,9 @@ class EngineCoreClient:
72
async def add_request_async(self, request: EngineCoreRequest) -> None:
73
raise NotImplementedError
74
75
+ async def profile_async(self, is_start: bool = True) -> None:
76
+ raise NotImplementedError
77
+
78
async def abort_requests_async(self, request_ids: List[str]) -> None:
79
raise NotImplementedError
80
@@ -105,7 +109,7 @@ class InprocClient(EngineCoreClient):
109
def __del__(self):
110
self.shutdown()
111
108
- def profile(self, is_start=True) -> None:
112
+ def profile(self, is_start: bool = True) -> None:
113
self.engine_core.profile(is_start)
114
115
@@ -133,7 +137,10 @@ class MPClient(EngineCoreClient):
137
self.decoder = msgspec.msgpack.Decoder(EngineCoreOutputs)
138
139
# ZMQ setup.
136
- self.ctx = (zmq.asyncio.Context() if asyncio_mode else zmq.Context())
140
+ if asyncio_mode:
141
+ self.ctx = zmq.asyncio.Context()
142
+ else:
143
+ self.ctx = zmq.Context() # type: ignore[attr-defined]
144
145
# Path for IPC.
146
ready_path = get_open_zmq_ipc_path()
@@ -149,11 +156,13 @@ class MPClient(EngineCoreClient):
156
self.input_socket.bind(input_path)
157
158
# Start EngineCore in background process.
159
+ self.proc_handle: Optional[EngineCoreProcHandle]
160
self.proc_handle = EngineCoreProc.make_engine_core_process(
161
*args,
154
- input_path=input_path,
155
- output_path=output_path,
156
- ready_path=ready_path,
162
+ input_path=
163
+ input_path, # type: ignore[misc] # MyPy incorrectly flags duplicate keywords
164
+ output_path=output_path, # type: ignore[misc]
165
+ ready_path=ready_path, # type: ignore[misc]
166
**kwargs,
167
)
168
atexit.register(self.shutdown)
@@ -204,10 +213,8 @@ class SyncMPClient(MPClient):
213
engine_core_outputs = self.decoder.decode(frame.buffer).outputs
214
return engine_core_outputs
215
207
- def _send_input(
208
- self, request_type: EngineCoreRequestType,
209
- request: Union[EngineCoreRequest, EngineCoreProfile,
210
- List[str]]) -> None:
216
+ def _send_input(self, request_type: EngineCoreRequestType,
217
+ request: EngineCoreRequestUnion) -> None:
218
219
# (RequestType, SerializedRequest)
220
msg = (request_type.value, self.encoder.encode(request))
@@ -219,7 +226,7 @@ class SyncMPClient(MPClient):
226
def abort_requests(self, request_ids: List[str]) -> None:
227
self._send_input(EngineCoreRequestType.ABORT, request_ids)
228
222
- def profile(self, is_start=True) -> None:
229
+ def profile(self, is_start: bool = True) -> None:
230
self._send_input(EngineCoreRequestType.PROFILE,
231
EngineCoreProfile(is_start))
232
@@ -237,10 +244,8 @@ class AsyncMPClient(MPClient):
244
245
return engine_core_outputs
246
240
- async def _send_input(
241
- self, request_type: EngineCoreRequestType,
242
- request: Union[EngineCoreRequest, EngineCoreProfile,
243
- List[str]]) -> None:
247
+ async def _send_input(self, request_type: EngineCoreRequestType,
248
+ request: EngineCoreRequestUnion) -> None:
249
250
msg = (request_type.value, self.encoder.encode(request))
251
await self.input_socket.send_multipart(msg, copy=False)
@@ -252,6 +257,6 @@ class AsyncMPClient(MPClient):
257
if len(request_ids) > 0:
258
await self._send_input(EngineCoreRequestType.ABORT, request_ids)
259
255
- async def profile(self, is_start=True) -> None:
260
+ async def profile_async(self, is_start: bool = True) -> None:
261
await self._send_input(EngineCoreRequestType.PROFILE,
262
EngineCoreProfile(is_start))
vllm/v1/engine/detokenizer.py
+2
-2
@@ -1,5 +1,5 @@
1
from dataclasses import dataclass
2
-from typing import Dict, Iterable, List, Optional, Tuple
2
+from typing import Dict, Iterable, List, Optional, Tuple, Union
3
4
from vllm.engine.output_processor.stop_checker import StopChecker
5
from vllm.logger import init_logger
@@ -97,7 +97,7 @@ class IncrementalDetokenizer:
97
self,
98
new_token_ids: List[int],
99
finish_reason: Optional[str],
100
- stop_reason: Optional[str],
100
+ stop_reason: Optional[Union[int, str, None]],
101
) -> Optional[RequestOutput]:
102
"""
103
Update RequestState for the request_id by:
vllm/v1/engine/llm_engine.py
+2
-1
@@ -103,7 +103,8 @@ class LLMEngine:
103
multiprocess_mode=enable_multiprocessing)
104
105
@classmethod
106
- def _get_executor_cls(cls, vllm_config: VllmConfig):
106
+ def _get_executor_cls(cls, vllm_config: VllmConfig) -> Type[Executor]:
107
+ executor_class: Type[Executor]
108
distributed_executor_backend = (
109
vllm_config.parallel_config.distributed_executor_backend)
110
if distributed_executor_backend == "mp":
vllm/v1/engine/mm_input_mapper.py
+13
-7
@@ -1,4 +1,4 @@
1
-from typing import Any, Dict, List, Optional
1
+from typing import Any, Dict, List, Optional, Tuple
2
3
import PIL
4
from blake3 import blake3
@@ -42,14 +42,14 @@ class MMInputMapperClient:
42
model_config)
43
self.mm_registry.init_mm_limits_per_prompt(model_config)
44
45
- self.mm_cache = LRUDictCache(MM_CACHE_SIZE)
45
+ self.mm_cache = LRUDictCache[str, MultiModalKwargs](MM_CACHE_SIZE)
46
47
# DEBUG: Set to None to disable
48
self.mm_debug_cache_hit_ratio_steps = None
49
self.mm_cache_hits = 0
50
self.mm_cache_total = 0
51
52
- def cache_hit_ratio(self, steps) -> float:
52
+ def cache_hit_ratio(self, steps):
53
if self.mm_cache_total > 0 and self.mm_cache_total % steps == 0:
54
logger.debug("MMInputMapper: cache_hit_ratio = %.2f ",
55
self.mm_cache_hits / self.mm_cache_total)
@@ -60,7 +60,7 @@ class MMInputMapperClient:
60
mm_hashes: Optional[List[str]],
61
mm_processor_kwargs: Optional[Dict[str, Any]],
62
precomputed_mm_inputs: Optional[List[MultiModalKwargs]],
63
- ) -> List[MultiModalKwargs]:
63
+ ) -> Tuple[List[MultiModalKwargs], Optional[List[str]]]:
64
if precomputed_mm_inputs is None:
65
image_inputs = mm_data["image"]
66
if not isinstance(image_inputs, list):
@@ -72,6 +72,7 @@ class MMInputMapperClient:
72
# Check if hash is enabled
73
use_hash = mm_hashes is not None
74
if use_hash:
75
+ assert mm_hashes is not None
76
assert num_inputs == len(
77
mm_hashes), "num_inputs = {} len(mm_hashes) = {}".format(
78
num_inputs, len(mm_hashes))
@@ -79,7 +80,7 @@ class MMInputMapperClient:
80
# Process each image input separately, so that later we can schedule
81
# them in a fine-grained manner.
82
# Apply caching (if enabled) and reuse precomputed inputs (if provided)
82
- ret_hashes = [] if use_hash else None
83
+ ret_hashes: Optional[List[str]] = [] if use_hash else None
84
ret_inputs: List[MultiModalKwargs] = []
85
for input_id in range(num_inputs):
86
if self.mm_debug_cache_hit_ratio_steps is not None:
@@ -88,6 +89,7 @@ class MMInputMapperClient:
89
mm_hash = None
90
mm_input = None
91
if use_hash:
92
+ assert mm_hashes is not None
93
mm_hash = mm_hashes[input_id]
94
mm_input = self.mm_cache.get(mm_hash)
95
@@ -105,12 +107,15 @@ class MMInputMapperClient:
107
108
if use_hash:
109
# Add to cache
110
+ assert mm_hash is not None
111
self.mm_cache.put(mm_hash, mm_input)
112
else:
113
self.mm_cache_hits += 1
114
mm_input = None # Avoids sending mm_input to Server
115
116
if use_hash:
117
+ assert mm_hash is not None
118
+ assert ret_hashes is not None
119
ret_hashes.append(mm_hash)
120
ret_inputs.append(mm_input)
121
@@ -120,17 +125,18 @@ class MMInputMapperClient:
125
class MMInputMapperServer:
126
127
def __init__(self, ):
123
- self.mm_cache = LRUDictCache(MM_CACHE_SIZE)
128
+ self.mm_cache = LRUDictCache[str, MultiModalKwargs](MM_CACHE_SIZE)
129
130
def process_inputs(
131
self,
132
mm_inputs: List[Optional[MultiModalKwargs]],
128
- mm_hashes: List[Optional[str]],
133
+ mm_hashes: List[str],
134
) -> List[MultiModalKwargs]:
135
assert len(mm_inputs) == len(mm_hashes)
136
137
full_mm_inputs = []
138
for mm_input, mm_hash in zip(mm_inputs, mm_hashes):
139
+ assert mm_hash is not None
140
if mm_input is None:
141
mm_input = self.mm_cache.get(mm_hash)
142
assert mm_input is not None
vllm/v1/engine/processor.py
+1
-1
@@ -56,7 +56,7 @@ class Processor:
56
request_id: str,
57
prompt: PromptType,
58
params: Union[SamplingParams, PoolingParams],
59
- arrival_time: float,
59
+ arrival_time: Optional[float] = None,
60
lora_request: Optional[LoRARequest] = None,
61
trace_headers: Optional[Mapping[str, str]] = None,
62
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
vllm/v1/executor/abstract.py
+2
-10
@@ -1,5 +1,5 @@
1
from abc import ABC, abstractmethod
2
-from typing import Dict, Optional, Tuple
2
+from typing import Tuple
3
4
from vllm.config import VllmConfig
5
from vllm.v1.outputs import ModelRunnerOutput
@@ -28,7 +28,7 @@ class Executor(ABC):
28
raise NotImplementedError
29
30
@abstractmethod
31
- def profile(self, is_start=True):
31
+ def profile(self, is_start: bool = True):
32
raise NotImplementedError
33
34
@abstractmethod
@@ -38,11 +38,3 @@ class Executor(ABC):
38
@abstractmethod
39
def check_health(self) -> None:
40
raise NotImplementedError
41
-
42
- @abstractmethod
43
- def collective_rpc(self,
44
- method: str,
45
- timeout: Optional[float] = None,
46
- args: Tuple = (),
47
- kwargs: Optional[Dict] = None) -> []:
48
- raise NotImplementedError
vllm/v1/executor/multiproc_executor.py
+8
-7
@@ -7,7 +7,7 @@ import time
7
from dataclasses import dataclass
8
from enum import Enum, auto
9
from multiprocessing.process import BaseProcess
10
-from typing import Dict, List, Optional, Tuple
10
+from typing import Any, Dict, List, Optional, Tuple
11
12
import zmq
13
@@ -21,6 +21,7 @@ from vllm.executor.multiproc_worker_utils import (
21
from vllm.logger import init_logger
22
from vllm.utils import (get_distributed_init_method, get_open_port,
23
get_open_zmq_ipc_path)
24
+from vllm.v1.executor.abstract import Executor
25
from vllm.v1.outputs import ModelRunnerOutput
26
from vllm.v1.utils import make_zmq_socket
27
from vllm.worker.worker_base import WorkerWrapperBase
@@ -31,7 +32,7 @@ POLLING_TIMEOUT_MS = 5000
32
POLLING_TIMEOUT_S = POLLING_TIMEOUT_MS // 1000
33
34
34
-class MultiprocExecutor:
35
+class MultiprocExecutor(Executor):
36
37
def __init__(self, vllm_config: VllmConfig) -> None:
38
# Call self.shutdown at exit to clean up
@@ -103,7 +104,7 @@ class MultiprocExecutor:
104
method: str,
105
timeout: Optional[float] = None,
106
args: Tuple = (),
106
- kwargs: Optional[Dict] = None) -> []:
107
+ kwargs: Optional[Dict] = None) -> List[Any]:
108
"""
109
Execute an RPC call on workers.
110
@@ -125,7 +126,7 @@ class MultiprocExecutor:
126
127
responses = [None] * self.world_size
128
for w in self.workers:
128
- dequeue_timeout = timeout - (time.monotonic() - start_time()
129
+ dequeue_timeout = timeout - (time.monotonic() - start_time
130
) if timeout is not None else None
131
status, result = w.worker_response_mq.dequeue(
132
timeout=dequeue_timeout)
@@ -153,7 +154,7 @@ class MultiprocExecutor:
154
args=(scheduler_output, ))[0]
155
return model_output
156
156
- def profile(self, is_start=True):
157
+ def profile(self, is_start: bool = True):
158
self.collective_rpc("profile", args=(is_start, ))
159
return
160
@@ -185,7 +186,6 @@ class MultiprocExecutor:
186
p.kill()
187
188
self._cleanup_sockets()
188
- self.workers = None
189
190
def _cleanup_sockets(self):
191
for w in self.workers:
@@ -200,7 +200,8 @@ class MultiprocExecutor:
200
# again
201
atexit.unregister(self.shutdown)
202
"""Properly shut down the executor and its workers"""
203
- if (hasattr(self, 'workers') and self.workers is not None):
203
+ if getattr(self, 'shutting_down', False):
204
+ self.shutting_down = True
205
for w in self.workers: #TODO: not sure if needed
206
w.worker_response_mq = None
207
self._ensure_worker_termination()
vllm/v1/executor/uniproc_executor.py
+4
-3
@@ -4,13 +4,14 @@ from typing import Optional, Tuple
4
from vllm.config import VllmConfig
5
from vllm.logger import init_logger
6
from vllm.utils import get_distributed_init_method, get_ip, get_open_port
7
+from vllm.v1.executor.abstract import Executor
8
from vllm.v1.outputs import ModelRunnerOutput
9
from vllm.v1.worker.gpu_worker import Worker
10
11
logger = init_logger(__name__)
12
13
13
-class UniprocExecutor:
14
+class UniprocExecutor(Executor):
15
16
def __init__(self, vllm_config: VllmConfig) -> None:
17
self.vllm_config = vllm_config
@@ -25,7 +26,7 @@ class UniprocExecutor:
26
self.prompt_adapter_config = vllm_config.prompt_adapter_config
27
self.observability_config = vllm_config.observability_config
28
28
- self.worker = self._create_worker()
29
+ self.worker: Worker = self._create_worker()
30
self.worker.initialize()
31
self.worker.load_model()
32
@@ -75,7 +76,7 @@ class UniprocExecutor:
76
self.worker.profile(is_start)
77
78
def shutdown(self):
78
- self.worker = None
79
+ pass
80
81
def check_health(self) -> None:
82
# UniprocExecutor will always be healthy as long as
vllm/v1/request.py
+1
-2
@@ -52,10 +52,9 @@ class Request:
52
else:
53
self.mm_positions = []
54
# Output of the mm input mapper (e.g., image tensors).
55
+ self.mm_inputs: List[MultiModalKwargs] = []
56
if self.inputs.multi_modal_inputs:
57
self.mm_inputs = self.inputs.multi_modal_inputs
57
- else:
58
- self.mm_inputs: List[MultiModalKwargs] = []
58
59
@classmethod
60
def from_engine_core_request(cls, request: EngineCoreRequest) -> "Request":
vllm/v1/utils.py
+27
-15
@@ -1,6 +1,8 @@
1
from collections import OrderedDict
2
+from collections.abc import Sequence
3
from contextlib import contextmanager
3
-from typing import Any, Generic, Iterator, List, TypeVar, overload
4
+from typing import (Any, Generic, Iterator, List, Optional, TypeVar, Union,
5
+ overload)
6
7
import zmq
8
@@ -11,7 +13,7 @@ logger = init_logger(__name__)
13
T = TypeVar("T")
14
15
14
-class ConstantList(Generic[T]):
16
+class ConstantList(Generic[T], Sequence):
17
18
def __init__(self, x: List[T]) -> None:
19
self._x = x
@@ -34,29 +36,33 @@ class ConstantList(Generic[T]):
36
def clear(self):
37
raise Exception("Cannot clear a constant list")
38
37
- def index(self, item):
38
- return self._x.index(item)
39
+ def index(self,
40
+ item: T,
41
+ start: int = 0,
42
+ stop: Optional[int] = None) -> int:
43
+ return self._x.index(item, start,
44
+ stop if stop is not None else len(self._x))
45
46
@overload
41
- def __getitem__(self, item) -> T:
47
+ def __getitem__(self, item: int) -> T:
48
...
49
50
@overload
51
def __getitem__(self, s: slice, /) -> List[T]:
52
...
53
48
- def __getitem__(self, item):
54
+ def __getitem__(self, item: Union[int, slice]) -> Union[T, List[T]]:
55
return self._x[item]
56
57
@overload
52
- def __setitem__(self, item, value):
58
+ def __setitem__(self, item: int, value: T):
59
...
60
61
@overload
56
- def __setitem__(self, s: slice, value, /):
62
+ def __setitem__(self, s: slice, value: T, /):
63
...
64
59
- def __setitem__(self, item, value):
65
+ def __setitem__(self, item: Union[int, slice], value: Union[T, List[T]]):
66
raise Exception("Cannot set item in a constant list")
67
68
def __delitem__(self, item):
@@ -73,10 +79,12 @@ class ConstantList(Generic[T]):
79
80
81
@contextmanager
76
-def make_zmq_socket(path: str, type: Any) -> Iterator[zmq.Socket]:
82
+def make_zmq_socket(
83
+ path: str,
84
+ type: Any) -> Iterator[zmq.Socket]: # type: ignore[name-defined]
85
"""Context manager for a ZMQ socket"""
86
79
- ctx = zmq.Context()
87
+ ctx = zmq.Context() # type: ignore[attr-defined]
88
try:
89
socket = ctx.socket(type)
90
@@ -96,20 +104,24 @@ def make_zmq_socket(path: str, type: Any) -> Iterator[zmq.Socket]:
104
ctx.destroy(linger=0)
105
106
99
-class LRUDictCache:
107
+K = TypeVar('K')
108
+V = TypeVar('V')
109
+
110
+
111
+class LRUDictCache(Generic[K, V]):
112
113
def __init__(self, size: int):
102
- self.cache = OrderedDict()
114
+ self.cache: OrderedDict[K, V] = OrderedDict()
115
self.size = size
116
105
- def get(self, key, default=None):
117
+ def get(self, key: K, default=None) -> V:
118
if key not in self.cache:
119
return default
120
121
self.cache.move_to_end(key)
122
return self.cache[key]
123
112
- def put(self, key, value):
124
+ def put(self, key: K, value: V):
125
self.cache[key] = value
126
self.cache.move_to_end(key)
127
if len(self.cache) > self.size:
vllm/v1/worker/gpu_input_batch.py
+1
@@ -215,6 +215,7 @@ class InputBatch:
215
216
# Swap the states.
217
req_id = self.req_ids[last_req_index]
218
+ assert req_id is not None
219
self.req_ids[empty_index] = req_id
220
self.req_ids[last_req_index] = None
221
self.req_id_to_index[req_id] = empty_index
vllm/v1/worker/gpu_model_runner.py
+26
-16
@@ -1,6 +1,6 @@
1
import gc
2
import time
3
-from typing import TYPE_CHECKING, Dict, List, Tuple
3
+from typing import TYPE_CHECKING, Dict, List, Tuple, cast
4
5
import numpy as np
6
import torch
@@ -193,9 +193,9 @@ class GPUModelRunner:
193
194
req_ids_to_add: List[str] = []
195
# Add new requests to the cached states.
196
- for req_data in scheduler_output.scheduled_new_reqs:
197
- req_id = req_data.req_id
198
- sampling_params = req_data.sampling_params
196
+ for new_req_data in scheduler_output.scheduled_new_reqs:
197
+ req_id = new_req_data.req_id
198
+ sampling_params = new_req_data.sampling_params
199
if sampling_params.sampling_type == SamplingType.RANDOM_SEED:
200
generator = torch.Generator(device=self.device)
201
generator.manual_seed(sampling_params.seed)
@@ -204,25 +204,25 @@ class GPUModelRunner:
204
205
self.requests[req_id] = CachedRequestState(
206
req_id=req_id,
207
- prompt_token_ids=req_data.prompt_token_ids,
208
- prompt=req_data.prompt,
209
- mm_inputs=req_data.mm_inputs,
210
- mm_positions=req_data.mm_positions,
207
+ prompt_token_ids=new_req_data.prompt_token_ids,
208
+ prompt=new_req_data.prompt,
209
+ mm_inputs=new_req_data.mm_inputs,
210
+ mm_positions=new_req_data.mm_positions,
211
sampling_params=sampling_params,
212
generator=generator,
213
- block_ids=req_data.block_ids,
214
- num_computed_tokens=req_data.num_computed_tokens,
213
+ block_ids=new_req_data.block_ids,
214
+ num_computed_tokens=new_req_data.num_computed_tokens,
215
output_token_ids=[],
216
)
217
req_ids_to_add.append(req_id)
218
219
# Update the cached states of the resumed requests.
220
- for req_data in scheduler_output.scheduled_resumed_reqs:
221
- req_id = req_data.req_id
220
+ for res_req_data in scheduler_output.scheduled_resumed_reqs:
221
+ req_id = res_req_data.req_id
222
req_state = self.requests[req_id]
223
224
- req_state.block_ids = req_data.block_ids
225
- req_state.num_computed_tokens = req_data.num_computed_tokens
224
+ req_state.block_ids = res_req_data.block_ids
225
+ req_state.num_computed_tokens = res_req_data.num_computed_tokens
226
req_ids_to_add.append(req_id)
227
228
# Add the new or resumed requests to the persistent batch.
@@ -259,6 +259,7 @@ class GPUModelRunner:
259
num_scheduled_tokens = []
260
max_num_scheduled_tokens = 0
261
for req_id in self.input_batch.req_ids[:num_reqs]:
262
+ assert req_id is not None
263
num_tokens = scheduler_output.num_scheduled_tokens[req_id]
264
num_scheduled_tokens.append(num_tokens)
265
max_num_scheduled_tokens = max(max_num_scheduled_tokens,
@@ -373,7 +374,7 @@ class GPUModelRunner:
374
375
# Batch the multi-modal inputs.
376
mm_inputs: List[MultiModalKwargs] = []
376
- req_input_ids: List[Tuple[int, int]] = []
377
+ req_input_ids: List[Tuple[str, int]] = []
378
for req_id, encoder_input_ids in scheduled_encoder_inputs.items():
379
req_state = self.requests[req_id]
380
for input_id in encoder_input_ids:
@@ -406,6 +407,7 @@ class GPUModelRunner:
407
encoder_outputs: List[torch.Tensor] = []
408
num_reqs = self.input_batch.num_reqs
409
for req_id in self.input_batch.req_ids[:num_reqs]:
410
+ assert req_id is not None
411
num_scheduled_tokens = scheduler_output.num_scheduled_tokens[
412
req_id]
413
req_state = self.requests[req_id]
@@ -514,6 +516,7 @@ class GPUModelRunner:
516
# the requests one by one. Optimize.
517
num_reqs = self.input_batch.num_reqs
518
for i, req_id in enumerate(self.input_batch.req_ids[:num_reqs]):
519
+ assert req_id is not None
520
req_state = self.requests[req_id]
521
seq_len = (req_state.num_computed_tokens +
522
scheduler_output.num_scheduled_tokens[req_id])
@@ -539,8 +542,15 @@ class GPUModelRunner:
542
logprobs = None
543
else:
544
logprobs = sampler_output.logprobs.cpu()
545
+
546
+ # num_reqs entries should be non-None
547
+ assert all(
548
+ req_id is not None for req_id in
549
+ self.input_batch.req_ids[:num_reqs]), "req_ids contains None"
550
+ req_ids = cast(List[str], self.input_batch.req_ids[:num_reqs])
551
+
552
model_runner_output = ModelRunnerOutput(
543
- req_ids=self.input_batch.req_ids[:num_reqs],
553
+ req_ids=req_ids,
554
req_id_to_index=self.input_batch.req_id_to_index,
555
sampled_token_ids=sampled_token_ids,
556
logprob_token_ids_cpu=logprob_token_ids,
vllm/v1/worker/gpu_worker.py
+1
-1
@@ -204,7 +204,7 @@ class Worker:
204
return output if self.rank == 0 else None
205
return output
206
207
- def profile(self, is_start=True):
207
+ def profile(self, is_start: bool = True):
208
if self.profiler is None:
209
raise RuntimeError("Profiler is not enabled.")
210
if is_start: