A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[V1] Use pickle for serializing EngineCoreRequest & Add multimodal inputs to EngineCoreRequest (#10245)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Nov 12, 2024 at 08:57 UTC
7c65527918cd16286961a2a779e15743ca41ab0e
5 files changed
+25
-5
vllm/v1/engine/__init__.py
+7
-2
@@ -1,10 +1,11 @@
1
import enum
2
from dataclasses import dataclass
3
-from typing import List, Optional, Union
3
+from typing import Any, Dict, List, Optional, Union
4
5
import msgspec
6
7
from vllm.lora.request import LoRARequest
8
+from vllm.multimodal import MultiModalDataDict, MultiModalPlaceholderDict
9
from vllm.sampling_params import RequestOutputKind, SamplingParams
10
11
@@ -22,7 +23,8 @@ class DetokenizerRequest:
23
include_stop_str_in_output: bool
24
25
25
-class EngineCoreRequest(msgspec.Struct, omit_defaults=True):
26
+@dataclass
27
+class EngineCoreRequest:
28
29
# NOTE: prompt and prompt_token_ids should be DecoderOnlyInput,
30
# but this object is currently not playing well with msgspec
@@ -33,6 +35,9 @@ class EngineCoreRequest(msgspec.Struct, omit_defaults=True):
35
# always be tokenized?
36
prompt: Optional[str]
37
prompt_token_ids: List[int]
38
+ mm_data: Optional[MultiModalDataDict]
39
+ mm_placeholders: Optional[MultiModalPlaceholderDict]
40
+ mm_processor_kwargs: Optional[Dict[str, Any]]
41
sampling_params: SamplingParams
42
eos_token_id: Optional[int]
43
arrival_time: float
vllm/v1/engine/core.py
+2
-1
@@ -19,6 +19,7 @@ from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
19
EngineCoreRequest, EngineCoreRequestType)
20
from vllm.v1.executor.gpu_executor import GPUExecutor
21
from vllm.v1.request import Request, RequestStatus
22
+from vllm.v1.serial_utils import PickleEncoder
23
from vllm.version import __version__ as VLLM_VERSION
24
25
logger = init_logger(__name__)
@@ -315,7 +316,7 @@ class EngineCoreProc(EngineCore):
316
"""Input socket IO thread."""
317
318
# Msgpack serialization decoding.
318
- decoder_add_req = msgpack.Decoder(EngineCoreRequest)
319
+ decoder_add_req = PickleEncoder()
320
decoder_abort_req = msgpack.Decoder(list[str])
321
322
with self.make_socket(input_path, zmq.constants.PULL) as socket:
vllm/v1/engine/core_client.py
+2
-1
@@ -11,6 +11,7 @@ from vllm.utils import get_open_zmq_ipc_path
11
from vllm.v1.engine import (EngineCoreOutput, EngineCoreOutputs,
12
EngineCoreRequest, EngineCoreRequestType)
13
from vllm.v1.engine.core import EngineCore, EngineCoreProc
14
+from vllm.v1.serial_utils import PickleEncoder
15
16
logger = init_logger(__name__)
17
@@ -115,7 +116,7 @@ class MPClient(EngineCoreClient):
116
**kwargs,
117
):
118
# Serialization setup.
118
- self.encoder = msgspec.msgpack.Encoder()
119
+ self.encoder = PickleEncoder()
120
self.decoder = msgspec.msgpack.Decoder(EngineCoreOutputs)
121
122
# ZMQ setup.
vllm/v1/engine/processor.py
+4
-1
@@ -91,7 +91,10 @@ class Processor:
91
# Make Request for EngineCore.
92
engine_core_request = EngineCoreRequest(
93
request_id, processed_inputs.get("prompt"),
94
- processed_inputs.get("prompt_token_ids"), sampling_params,
94
+ processed_inputs.get("prompt_token_ids"),
95
+ processed_inputs.get("multi_modal_data"),
96
+ processed_inputs.get("multi_modal_placeholders"),
97
+ processed_inputs.get("mm_processor_kwargs"), sampling_params,
98
eos_token_id, arrival_time, lora_request)
99
100
return detokenizer_request, engine_core_request
vllm/v1/serial_utils.py
new
+10
@@ -0,0 +1,10 @@
1
+import pickle
2
+
3
+
4
+class PickleEncoder:
5
+
6
+ def encode(self, obj):
7
+ return pickle.dumps(obj)
8
+
9
+ def decode(self, data):
10
+ return pickle.loads(data)