@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 67c4637ccf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[V1] Use msgpack for core request serialization (#12918)

Signed-off-by: Nick Hill <nhill@redhat.com>

Nick Hill committed Feb 9, 2025 at 19:35 UTC 67c4637ccfd1f1b4e4aa2b645a5635096cf6d1fe
4 files changed +62 -95
vllm/v1/engine/__init__.py
+14 -28
@@ -1,20 +1,17 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 import enum
4 -from dataclasses import dataclass
5 -from typing import TYPE_CHECKING, List, Optional, Union
4 +from typing import List, Optional, Union
5
6 import msgspec
7
8 +from vllm.lora.request import LoRARequest
9 +from vllm.multimodal import MultiModalKwargs
10 +from vllm.multimodal.inputs import PlaceholderRange
11 +from vllm.sampling_params import SamplingParams
12 from vllm.v1.metrics.stats import SchedulerStats
13 from vllm.v1.outputs import LogprobsLists, LogprobsTensors
14
12 -if TYPE_CHECKING:
13 - from vllm.lora.request import LoRARequest
14 - from vllm.multimodal import MultiModalKwargs
15 - from vllm.multimodal.inputs import PlaceholderRange
16 - from vllm.sampling_params import SamplingParams
17 -
15 # These are possible values of RequestOutput.finish_reason,
16 # so form part of the external API.
17 FINISH_REASON_STRINGS = ("stop", "length", "abort")
@@ -39,8 +36,11 @@ class FinishReason(enum.IntEnum):
36 return FINISH_REASON_STRINGS[self.value]
37
38
42 -@dataclass
43 -class EngineCoreRequest:
39 +class EngineCoreRequest(
40 + msgspec.Struct,
41 + array_like=True, # type: ignore[call-arg]
42 + omit_defaults=True, # type: ignore[call-arg]
43 + gc=False): # type: ignore[call-arg]
44
45 # NOTE: prompt and prompt_token_ids should be DecoderOnlyInput,
46 # but this object is currently not playing well with msgspec
@@ -51,13 +51,13 @@ class EngineCoreRequest:
51 # Detokenizer, but set to None when it is added to EngineCoreClient.
52 prompt: Optional[str]
53 prompt_token_ids: List[int]
54 - mm_inputs: Optional[List[Optional["MultiModalKwargs"]]]
54 + mm_inputs: Optional[List[Optional[MultiModalKwargs]]]
55 mm_hashes: Optional[List[str]]
56 - mm_placeholders: Optional[List["PlaceholderRange"]]
57 - sampling_params: "SamplingParams"
56 + mm_placeholders: Optional[List[PlaceholderRange]]
57 + sampling_params: SamplingParams
58 eos_token_id: Optional[int]
59 arrival_time: float
60 - lora_request: Optional["LoRARequest"]
60 + lora_request: Optional[LoRARequest]
61
62
63 class EngineCoreOutput(
@@ -94,16 +94,6 @@ class EngineCoreOutputs(
94 scheduler_stats: SchedulerStats
95
96
97 -@dataclass
98 -class EngineCoreProfile:
99 - is_start: bool
100 -
101 -
102 -@dataclass
103 -class EngineCoreResetPrefixCache:
104 - pass
105 -
106 -
97 class EngineCoreRequestType(enum.Enum):
98 """
99 Request types defined as hex byte strings, so it can be sent over sockets
@@ -113,7 +103,3 @@ class EngineCoreRequestType(enum.Enum):
103 ABORT = b'\x01'
104 PROFILE = b'\x02'
105 RESET_PREFIX_CACHE = b'\x03'
116 -
117 -
118 -EngineCoreRequestUnion = Union[EngineCoreRequest, EngineCoreProfile,
119 - EngineCoreResetPrefixCache, List[str]]
vllm/v1/engine/core.py
+26 -35
@@ -1,12 +1,11 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 -import pickle
3 import queue
4 import signal
5 import threading
6 import time
7 from multiprocessing.connection import Connection
9 -from typing import List, Tuple, Type
8 +from typing import Any, List, Tuple, Type
9
10 import psutil
11 import zmq
@@ -19,13 +18,12 @@ from vllm.transformers_utils.config import (
18 from vllm.utils import get_exception_traceback, zmq_socket_ctx
19 from vllm.v1.core.kv_cache_utils import get_kv_cache_config
20 from vllm.v1.core.scheduler import Scheduler
22 -from vllm.v1.engine import (EngineCoreOutputs, EngineCoreProfile,
23 - EngineCoreRequest, EngineCoreRequestType,
24 - EngineCoreRequestUnion, EngineCoreResetPrefixCache)
21 +from vllm.v1.engine import (EngineCoreOutputs, EngineCoreRequest,
22 + EngineCoreRequestType)
23 from vllm.v1.engine.mm_input_mapper import MMInputMapperServer
24 from vllm.v1.executor.abstract import Executor
25 from vllm.v1.request import Request, RequestStatus
28 -from vllm.v1.serial_utils import MsgpackEncoder, PickleEncoder
26 +from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
27 from vllm.version import __version__ as VLLM_VERSION
28
29 logger = init_logger(__name__)
@@ -161,7 +159,8 @@ class EngineCoreProc(EngineCore):
159 # and to overlap some serialization/deserialization with the
160 # model forward pass.
161 # Threads handle Socket <-> Queues and core_busy_loop uses Queue.
164 - self.input_queue: queue.Queue[EngineCoreRequestUnion] = queue.Queue()
162 + self.input_queue: queue.Queue[Tuple[EngineCoreRequestType,
163 + Any]] = queue.Queue()
164 self.output_queue: queue.Queue[EngineCoreOutputs] = queue.Queue()
165 threading.Thread(target=self.process_input_socket,
166 args=(input_path, ),
@@ -223,7 +222,7 @@ class EngineCoreProc(EngineCore):
222 while True:
223 try:
224 req = self.input_queue.get(timeout=POLLING_TIMEOUT_S)
226 - self._handle_client_request(req)
225 + self._handle_client_request(*req)
226 break
227 except queue.Empty:
228 logger.debug("EngineCore busy loop waiting.")
@@ -233,10 +232,10 @@ class EngineCoreProc(EngineCore):
232 except BaseException:
233 raise
234
236 - # 2) Handle any new client requests (Abort or Add).
235 + # 2) Handle any new client requests.
236 while not self.input_queue.empty():
237 req = self.input_queue.get_nowait()
239 - self._handle_client_request(req)
238 + self._handle_client_request(*req)
239
240 # 3) Step the engine core.
241 outputs = self.step()
@@ -244,48 +243,40 @@ class EngineCoreProc(EngineCore):
243 # 5) Put EngineCoreOutputs into the output queue.
244 self.output_queue.put_nowait(outputs)
245
247 - def _handle_client_request(self, request: EngineCoreRequestUnion) -> None:
248 - """Handle EngineCoreRequest or EngineCoreABORT from Client."""
246 + def _handle_client_request(self, request_type: EngineCoreRequestType,
247 + request: Any) -> None:
248 + """Dispatch request from client."""
249
250 - if isinstance(request, EngineCoreRequest):
250 + if request_type == EngineCoreRequestType.ADD:
251 self.add_request(request)
252 - elif isinstance(request, EngineCoreProfile):
253 - self.model_executor.profile(request.is_start)
254 - elif isinstance(request, EngineCoreResetPrefixCache):
255 - self.reset_prefix_cache()
256 - else:
257 - # TODO: make an EngineCoreAbort wrapper
258 - assert isinstance(request, list)
252 + elif request_type == EngineCoreRequestType.ABORT:
253 self.abort_requests(request)
254 + elif request_type == EngineCoreRequestType.RESET_PREFIX_CACHE:
255 + self.reset_prefix_cache()
256 + elif request_type == EngineCoreRequestType.PROFILE:
257 + self.model_executor.profile(request)
258
259 def process_input_socket(self, input_path: str):
260 """Input socket IO thread."""
261
262 # Msgpack serialization decoding.
265 - decoder_add_req = PickleEncoder()
266 - decoder_abort_req = PickleEncoder()
263 + add_request_decoder = MsgpackDecoder(EngineCoreRequest)
264 + generic_decoder = MsgpackDecoder()
265
266 with zmq_socket_ctx(input_path, zmq.constants.PULL) as socket:
267 while True:
268 # (RequestType, RequestData)
269 type_frame, data_frame = socket.recv_multipart(copy=False)
272 - request_type = type_frame.buffer
273 - request_data = data_frame.buffer
270 + request_type = EngineCoreRequestType(bytes(type_frame.buffer))
271
272 # Deserialize the request data.
276 - if request_type == EngineCoreRequestType.ADD.value:
277 - request = decoder_add_req.decode(request_data)
278 - elif request_type == EngineCoreRequestType.ABORT.value:
279 - request = decoder_abort_req.decode(request_data)
280 - elif request_type in (
281 - EngineCoreRequestType.PROFILE.value,
282 - EngineCoreRequestType.RESET_PREFIX_CACHE.value):
283 - request = pickle.loads(request_data)
284 - else:
285 - raise ValueError(f"Unknown RequestType: {request_type}")
273 + decoder = add_request_decoder if (
274 + request_type
275 + == EngineCoreRequestType.ADD) else generic_decoder
276 + request = decoder.decode(data_frame.buffer)
277
278 # Push to input queue for core busy loop.
288 - self.input_queue.put_nowait(request)
279 + self.input_queue.put_nowait((request_type, request))
280
281 def process_output_socket(self, output_path: str):
282 """Output socket IO thread."""
vllm/v1/engine/core_client.py
+11 -16
@@ -5,7 +5,7 @@ import os
5 import signal
6 import weakref
7 from abc import ABC, abstractmethod
8 -from typing import List, Optional, Type
8 +from typing import Any, List, Optional, Type
9
10 import zmq
11 import zmq.asyncio
@@ -14,12 +14,11 @@ from vllm.config import VllmConfig
14 from vllm.logger import init_logger
15 from vllm.utils import (get_open_zmq_ipc_path, kill_process_tree,
16 make_zmq_socket)
17 -from vllm.v1.engine import (EngineCoreOutputs, EngineCoreProfile,
18 - EngineCoreRequest, EngineCoreRequestType,
19 - EngineCoreRequestUnion, EngineCoreResetPrefixCache)
17 +from vllm.v1.engine import (EngineCoreOutputs, EngineCoreRequest,
18 + EngineCoreRequestType)
19 from vllm.v1.engine.core import EngineCore, EngineCoreProc
20 from vllm.v1.executor.abstract import Executor
22 -from vllm.v1.serial_utils import MsgpackDecoder, PickleEncoder
21 +from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
22 from vllm.v1.utils import BackgroundProcHandle
23
24 logger = init_logger(__name__)
@@ -161,7 +160,7 @@ class MPClient(EngineCoreClient):
160 signal.signal(signal.SIGUSR1, sigusr1_handler)
161
162 # Serialization setup.
164 - self.encoder = PickleEncoder()
163 + self.encoder = MsgpackEncoder()
164 self.decoder = MsgpackDecoder(EngineCoreOutputs)
165
166 # ZMQ setup.
@@ -220,7 +219,7 @@ class SyncMPClient(MPClient):
219 return self.decoder.decode(frame.buffer)
220
221 def _send_input(self, request_type: EngineCoreRequestType,
223 - request: EngineCoreRequestUnion) -> None:
222 + request: Any) -> None:
223
224 # (RequestType, SerializedRequest)
225 msg = (request_type.value, self.encoder.encode(request))
@@ -237,12 +236,10 @@ class SyncMPClient(MPClient):
236 self._send_input(EngineCoreRequestType.ABORT, request_ids)
237
238 def profile(self, is_start: bool = True) -> None:
240 - self._send_input(EngineCoreRequestType.PROFILE,
241 - EngineCoreProfile(is_start))
239 + self._send_input(EngineCoreRequestType.PROFILE, is_start)
240
241 def reset_prefix_cache(self) -> None:
244 - self._send_input(EngineCoreRequestType.RESET_PREFIX_CACHE,
245 - EngineCoreResetPrefixCache())
242 + self._send_input(EngineCoreRequestType.RESET_PREFIX_CACHE, None)
243
244
245 class AsyncMPClient(MPClient):
@@ -277,7 +274,7 @@ class AsyncMPClient(MPClient):
274 return self.decoder.decode(await self.outputs_queue.get())
275
276 async def _send_input(self, request_type: EngineCoreRequestType,
280 - request: EngineCoreRequestUnion) -> None:
277 + request: Any) -> None:
278
279 msg = (request_type.value, self.encoder.encode(request))
280 await self.input_socket.send_multipart(msg, copy=False)
@@ -293,9 +290,7 @@ class AsyncMPClient(MPClient):
290 await self._send_input(EngineCoreRequestType.ABORT, request_ids)
291
292 async def profile_async(self, is_start: bool = True) -> None:
296 - await self._send_input(EngineCoreRequestType.PROFILE,
297 - EngineCoreProfile(is_start))
293 + await self._send_input(EngineCoreRequestType.PROFILE, is_start)
294
295 async def reset_prefix_cache_async(self) -> None:
300 - await self._send_input(EngineCoreRequestType.RESET_PREFIX_CACHE,
301 - EngineCoreResetPrefixCache())
296 + await self._send_input(EngineCoreRequestType.RESET_PREFIX_CACHE, None)
vllm/v1/serial_utils.py
+11 -16
@@ -1,21 +1,13 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 import pickle
4 -from typing import Any
4 +from typing import Any, Optional
5
6 import torch
7 from msgspec import msgpack
8
9 -CUSTOM_TYPE_CODE_PICKLE = 1
10 -
11 -
12 -class PickleEncoder:
13 -
14 - def encode(self, obj: Any):
15 - return pickle.dumps(obj)
16 -
17 - def decode(self, data: Any):
18 - return pickle.loads(data)
9 +CUSTOM_TYPE_TENSOR = 1
10 +CUSTOM_TYPE_PICKLE = 2
11
12
13 class MsgpackEncoder:
@@ -34,8 +26,9 @@ class MsgpackEncoder:
26 class MsgpackDecoder:
27 """Decoder with custom torch tensor serialization."""
28
37 - def __init__(self, t: Any):
38 - self.decoder = msgpack.Decoder(t, ext_hook=custom_ext_hook)
29 + def __init__(self, t: Optional[Any] = None):
30 + args = () if t is None else (t, )
31 + self.decoder = msgpack.Decoder(*args, ext_hook=custom_ext_hook)
32
33 def decode(self, obj: Any):
34 return self.decoder.decode(obj)
@@ -46,13 +39,15 @@ def custom_enc_hook(obj: Any) -> Any:
39 # NOTE(rob): it is fastest to use numpy + pickle
40 # when serializing torch tensors.
41 # https://gist.github.com/tlrmchlsmth/8067f1b24a82b6e2f90450e7764fa103 # noqa: E501
49 - return msgpack.Ext(CUSTOM_TYPE_CODE_PICKLE, pickle.dumps(obj.numpy()))
42 + return msgpack.Ext(CUSTOM_TYPE_TENSOR, pickle.dumps(obj.numpy()))
43
51 - raise NotImplementedError(f"Objects of type {type(obj)} are not supported")
44 + return msgpack.Ext(CUSTOM_TYPE_PICKLE, pickle.dumps(obj))
45
46
47 def custom_ext_hook(code: int, data: memoryview) -> Any:
55 - if code == CUSTOM_TYPE_CODE_PICKLE:
48 + if code == CUSTOM_TYPE_TENSOR:
49 return torch.from_numpy(pickle.loads(data))
50 + if code == CUSTOM_TYPE_PICKLE:
51 + return pickle.loads(data)
52
53 raise NotImplementedError(f"Extension type code {code} is not supported")