A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[V1] Use msgpack for core request serialization (#12918)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Feb 9, 2025 at 19:35 UTC
67c4637ccfd1f1b4e4aa2b645a5635096cf6d1fe
4 files changed
+62
-95
vllm/v1/engine/__init__.py
+14
-28
@@ -1,20 +1,17 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import enum
4
-from dataclasses import dataclass
5
-from typing import TYPE_CHECKING, List, Optional, Union
4
+from typing import List, Optional, Union
5
6
import msgspec
7
8
+from vllm.lora.request import LoRARequest
9
+from vllm.multimodal import MultiModalKwargs
10
+from vllm.multimodal.inputs import PlaceholderRange
11
+from vllm.sampling_params import SamplingParams
12
from vllm.v1.metrics.stats import SchedulerStats
13
from vllm.v1.outputs import LogprobsLists, LogprobsTensors
14
12
-if TYPE_CHECKING:
13
- from vllm.lora.request import LoRARequest
14
- from vllm.multimodal import MultiModalKwargs
15
- from vllm.multimodal.inputs import PlaceholderRange
16
- from vllm.sampling_params import SamplingParams
17
-
15
# These are possible values of RequestOutput.finish_reason,
16
# so form part of the external API.
17
FINISH_REASON_STRINGS = ("stop", "length", "abort")
@@ -39,8 +36,11 @@ class FinishReason(enum.IntEnum):
36
return FINISH_REASON_STRINGS[self.value]
37
38
42
-@dataclass
43
-class EngineCoreRequest:
39
+class EngineCoreRequest(
40
+ msgspec.Struct,
41
+ array_like=True, # type: ignore[call-arg]
42
+ omit_defaults=True, # type: ignore[call-arg]
43
+ gc=False): # type: ignore[call-arg]
44
45
# NOTE: prompt and prompt_token_ids should be DecoderOnlyInput,
46
# but this object is currently not playing well with msgspec
@@ -51,13 +51,13 @@ class EngineCoreRequest:
51
# Detokenizer, but set to None when it is added to EngineCoreClient.
52
prompt: Optional[str]
53
prompt_token_ids: List[int]
54
- mm_inputs: Optional[List[Optional["MultiModalKwargs"]]]
54
+ mm_inputs: Optional[List[Optional[MultiModalKwargs]]]
55
mm_hashes: Optional[List[str]]
56
- mm_placeholders: Optional[List["PlaceholderRange"]]
57
- sampling_params: "SamplingParams"
56
+ mm_placeholders: Optional[List[PlaceholderRange]]
57
+ sampling_params: SamplingParams
58
eos_token_id: Optional[int]
59
arrival_time: float
60
- lora_request: Optional["LoRARequest"]
60
+ lora_request: Optional[LoRARequest]
61
62
63
class EngineCoreOutput(
@@ -94,16 +94,6 @@ class EngineCoreOutputs(
94
scheduler_stats: SchedulerStats
95
96
97
-@dataclass
98
-class EngineCoreProfile:
99
- is_start: bool
100
-
101
-
102
-@dataclass
103
-class EngineCoreResetPrefixCache:
104
- pass
105
-
106
-
97
class EngineCoreRequestType(enum.Enum):
98
"""
99
Request types defined as hex byte strings, so it can be sent over sockets
@@ -113,7 +103,3 @@ class EngineCoreRequestType(enum.Enum):
103
ABORT = b'\x01'
104
PROFILE = b'\x02'
105
RESET_PREFIX_CACHE = b'\x03'
116
-
117
-
118
-EngineCoreRequestUnion = Union[EngineCoreRequest, EngineCoreProfile,
119
- EngineCoreResetPrefixCache, List[str]]
vllm/v1/engine/core.py
+26
-35
@@ -1,12 +1,11 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-import pickle
3
import queue
4
import signal
5
import threading
6
import time
7
from multiprocessing.connection import Connection
9
-from typing import List, Tuple, Type
8
+from typing import Any, List, Tuple, Type
9
10
import psutil
11
import zmq
@@ -19,13 +18,12 @@ from vllm.transformers_utils.config import (
18
from vllm.utils import get_exception_traceback, zmq_socket_ctx
19
from vllm.v1.core.kv_cache_utils import get_kv_cache_config
20
from vllm.v1.core.scheduler import Scheduler
22
-from vllm.v1.engine import (EngineCoreOutputs, EngineCoreProfile,
23
- EngineCoreRequest, EngineCoreRequestType,
24
- EngineCoreRequestUnion, EngineCoreResetPrefixCache)
21
+from vllm.v1.engine import (EngineCoreOutputs, EngineCoreRequest,
22
+ EngineCoreRequestType)
23
from vllm.v1.engine.mm_input_mapper import MMInputMapperServer
24
from vllm.v1.executor.abstract import Executor
25
from vllm.v1.request import Request, RequestStatus
28
-from vllm.v1.serial_utils import MsgpackEncoder, PickleEncoder
26
+from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
27
from vllm.version import __version__ as VLLM_VERSION
28
29
logger = init_logger(__name__)
@@ -161,7 +159,8 @@ class EngineCoreProc(EngineCore):
159
# and to overlap some serialization/deserialization with the
160
# model forward pass.
161
# Threads handle Socket <-> Queues and core_busy_loop uses Queue.
164
- self.input_queue: queue.Queue[EngineCoreRequestUnion] = queue.Queue()
162
+ self.input_queue: queue.Queue[Tuple[EngineCoreRequestType,
163
+ Any]] = queue.Queue()
164
self.output_queue: queue.Queue[EngineCoreOutputs] = queue.Queue()
165
threading.Thread(target=self.process_input_socket,
166
args=(input_path, ),
@@ -223,7 +222,7 @@ class EngineCoreProc(EngineCore):
222
while True:
223
try:
224
req = self.input_queue.get(timeout=POLLING_TIMEOUT_S)
226
- self._handle_client_request(req)
225
+ self._handle_client_request(*req)
226
break
227
except queue.Empty:
228
logger.debug("EngineCore busy loop waiting.")
@@ -233,10 +232,10 @@ class EngineCoreProc(EngineCore):
232
except BaseException:
233
raise
234
236
- # 2) Handle any new client requests (Abort or Add).
235
+ # 2) Handle any new client requests.
236
while not self.input_queue.empty():
237
req = self.input_queue.get_nowait()
239
- self._handle_client_request(req)
238
+ self._handle_client_request(*req)
239
240
# 3) Step the engine core.
241
outputs = self.step()
@@ -244,48 +243,40 @@ class EngineCoreProc(EngineCore):
243
# 5) Put EngineCoreOutputs into the output queue.
244
self.output_queue.put_nowait(outputs)
245
247
- def _handle_client_request(self, request: EngineCoreRequestUnion) -> None:
248
- """Handle EngineCoreRequest or EngineCoreABORT from Client."""
246
+ def _handle_client_request(self, request_type: EngineCoreRequestType,
247
+ request: Any) -> None:
248
+ """Dispatch request from client."""
249
250
- if isinstance(request, EngineCoreRequest):
250
+ if request_type == EngineCoreRequestType.ADD:
251
self.add_request(request)
252
- elif isinstance(request, EngineCoreProfile):
253
- self.model_executor.profile(request.is_start)
254
- elif isinstance(request, EngineCoreResetPrefixCache):
255
- self.reset_prefix_cache()
256
- else:
257
- # TODO: make an EngineCoreAbort wrapper
258
- assert isinstance(request, list)
252
+ elif request_type == EngineCoreRequestType.ABORT:
253
self.abort_requests(request)
254
+ elif request_type == EngineCoreRequestType.RESET_PREFIX_CACHE:
255
+ self.reset_prefix_cache()
256
+ elif request_type == EngineCoreRequestType.PROFILE:
257
+ self.model_executor.profile(request)
258
259
def process_input_socket(self, input_path: str):
260
"""Input socket IO thread."""
261
262
# Msgpack serialization decoding.
265
- decoder_add_req = PickleEncoder()
266
- decoder_abort_req = PickleEncoder()
263
+ add_request_decoder = MsgpackDecoder(EngineCoreRequest)
264
+ generic_decoder = MsgpackDecoder()
265
266
with zmq_socket_ctx(input_path, zmq.constants.PULL) as socket:
267
while True:
268
# (RequestType, RequestData)
269
type_frame, data_frame = socket.recv_multipart(copy=False)
272
- request_type = type_frame.buffer
273
- request_data = data_frame.buffer
270
+ request_type = EngineCoreRequestType(bytes(type_frame.buffer))
271
272
# Deserialize the request data.
276
- if request_type == EngineCoreRequestType.ADD.value:
277
- request = decoder_add_req.decode(request_data)
278
- elif request_type == EngineCoreRequestType.ABORT.value:
279
- request = decoder_abort_req.decode(request_data)
280
- elif request_type in (
281
- EngineCoreRequestType.PROFILE.value,
282
- EngineCoreRequestType.RESET_PREFIX_CACHE.value):
283
- request = pickle.loads(request_data)
284
- else:
285
- raise ValueError(f"Unknown RequestType: {request_type}")
273
+ decoder = add_request_decoder if (
274
+ request_type
275
+ == EngineCoreRequestType.ADD) else generic_decoder
276
+ request = decoder.decode(data_frame.buffer)
277
278
# Push to input queue for core busy loop.
288
- self.input_queue.put_nowait(request)
279
+ self.input_queue.put_nowait((request_type, request))
280
281
def process_output_socket(self, output_path: str):
282
"""Output socket IO thread."""
vllm/v1/engine/core_client.py
+11
-16
@@ -5,7 +5,7 @@ import os
5
import signal
6
import weakref
7
from abc import ABC, abstractmethod
8
-from typing import List, Optional, Type
8
+from typing import Any, List, Optional, Type
9
10
import zmq
11
import zmq.asyncio
@@ -14,12 +14,11 @@ from vllm.config import VllmConfig
14
from vllm.logger import init_logger
15
from vllm.utils import (get_open_zmq_ipc_path, kill_process_tree,
16
make_zmq_socket)
17
-from vllm.v1.engine import (EngineCoreOutputs, EngineCoreProfile,
18
- EngineCoreRequest, EngineCoreRequestType,
19
- EngineCoreRequestUnion, EngineCoreResetPrefixCache)
17
+from vllm.v1.engine import (EngineCoreOutputs, EngineCoreRequest,
18
+ EngineCoreRequestType)
19
from vllm.v1.engine.core import EngineCore, EngineCoreProc
20
from vllm.v1.executor.abstract import Executor
22
-from vllm.v1.serial_utils import MsgpackDecoder, PickleEncoder
21
+from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
22
from vllm.v1.utils import BackgroundProcHandle
23
24
logger = init_logger(__name__)
@@ -161,7 +160,7 @@ class MPClient(EngineCoreClient):
160
signal.signal(signal.SIGUSR1, sigusr1_handler)
161
162
# Serialization setup.
164
- self.encoder = PickleEncoder()
163
+ self.encoder = MsgpackEncoder()
164
self.decoder = MsgpackDecoder(EngineCoreOutputs)
165
166
# ZMQ setup.
@@ -220,7 +219,7 @@ class SyncMPClient(MPClient):
219
return self.decoder.decode(frame.buffer)
220
221
def _send_input(self, request_type: EngineCoreRequestType,
223
- request: EngineCoreRequestUnion) -> None:
222
+ request: Any) -> None:
223
224
# (RequestType, SerializedRequest)
225
msg = (request_type.value, self.encoder.encode(request))
@@ -237,12 +236,10 @@ class SyncMPClient(MPClient):
236
self._send_input(EngineCoreRequestType.ABORT, request_ids)
237
238
def profile(self, is_start: bool = True) -> None:
240
- self._send_input(EngineCoreRequestType.PROFILE,
241
- EngineCoreProfile(is_start))
239
+ self._send_input(EngineCoreRequestType.PROFILE, is_start)
240
241
def reset_prefix_cache(self) -> None:
244
- self._send_input(EngineCoreRequestType.RESET_PREFIX_CACHE,
245
- EngineCoreResetPrefixCache())
242
+ self._send_input(EngineCoreRequestType.RESET_PREFIX_CACHE, None)
243
244
245
class AsyncMPClient(MPClient):
@@ -277,7 +274,7 @@ class AsyncMPClient(MPClient):
274
return self.decoder.decode(await self.outputs_queue.get())
275
276
async def _send_input(self, request_type: EngineCoreRequestType,
280
- request: EngineCoreRequestUnion) -> None:
277
+ request: Any) -> None:
278
279
msg = (request_type.value, self.encoder.encode(request))
280
await self.input_socket.send_multipart(msg, copy=False)
@@ -293,9 +290,7 @@ class AsyncMPClient(MPClient):
290
await self._send_input(EngineCoreRequestType.ABORT, request_ids)
291
292
async def profile_async(self, is_start: bool = True) -> None:
296
- await self._send_input(EngineCoreRequestType.PROFILE,
297
- EngineCoreProfile(is_start))
293
+ await self._send_input(EngineCoreRequestType.PROFILE, is_start)
294
295
async def reset_prefix_cache_async(self) -> None:
300
- await self._send_input(EngineCoreRequestType.RESET_PREFIX_CACHE,
301
- EngineCoreResetPrefixCache())
296
+ await self._send_input(EngineCoreRequestType.RESET_PREFIX_CACHE, None)
vllm/v1/serial_utils.py
+11
-16
@@ -1,21 +1,13 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import pickle
4
-from typing import Any
4
+from typing import Any, Optional
5
6
import torch
7
from msgspec import msgpack
8
9
-CUSTOM_TYPE_CODE_PICKLE = 1
10
-
11
-
12
-class PickleEncoder:
13
-
14
- def encode(self, obj: Any):
15
- return pickle.dumps(obj)
16
-
17
- def decode(self, data: Any):
18
- return pickle.loads(data)
9
+CUSTOM_TYPE_TENSOR = 1
10
+CUSTOM_TYPE_PICKLE = 2
11
12
13
class MsgpackEncoder:
@@ -34,8 +26,9 @@ class MsgpackEncoder:
26
class MsgpackDecoder:
27
"""Decoder with custom torch tensor serialization."""
28
37
- def __init__(self, t: Any):
38
- self.decoder = msgpack.Decoder(t, ext_hook=custom_ext_hook)
29
+ def __init__(self, t: Optional[Any] = None):
30
+ args = () if t is None else (t, )
31
+ self.decoder = msgpack.Decoder(*args, ext_hook=custom_ext_hook)
32
33
def decode(self, obj: Any):
34
return self.decoder.decode(obj)
@@ -46,13 +39,15 @@ def custom_enc_hook(obj: Any) -> Any:
39
# NOTE(rob): it is fastest to use numpy + pickle
40
# when serializing torch tensors.
41
# https://gist.github.com/tlrmchlsmth/8067f1b24a82b6e2f90450e7764fa103 # noqa: E501
49
- return msgpack.Ext(CUSTOM_TYPE_CODE_PICKLE, pickle.dumps(obj.numpy()))
42
+ return msgpack.Ext(CUSTOM_TYPE_TENSOR, pickle.dumps(obj.numpy()))
43
51
- raise NotImplementedError(f"Objects of type {type(obj)} are not supported")
44
+ return msgpack.Ext(CUSTOM_TYPE_PICKLE, pickle.dumps(obj))
45
46
47
def custom_ext_hook(code: int, data: memoryview) -> Any:
55
- if code == CUSTOM_TYPE_CODE_PICKLE:
48
+ if code == CUSTOM_TYPE_TENSOR:
49
return torch.from_numpy(pickle.loads(data))
50
+ if code == CUSTOM_TYPE_PICKLE:
51
+ return pickle.loads(data)
52
53
raise NotImplementedError(f"Extension type code {code} is not supported")