A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[V1] Avoid sending text prompt to core engine (#11963)
Signed-off-by: Roger Wang <ywang@roblox.com>
Roger Wang committed
Jan 11, 2025 at 22:36 UTC
b25cfab9a03b5c460fb92340b310d2a5c2dbc5da
2 files changed
+8
-2
vllm/v1/engine/__init__.py
+2
-2
@@ -19,8 +19,8 @@ class EngineCoreRequest:
19
# due to circular imports and typing we have in data.py
20
21
request_id: str
22
- #NOTE(Nick): I don't think we need to pass prompt here since it should
23
- # always be tokenized?
22
+ # NOTE(ywang96): original text prompt is needed when a request is added to
23
+ # Detokenizer, but set to None when it is added to EngineCoreClient.
24
prompt: Optional[str]
25
prompt_token_ids: List[int]
26
mm_inputs: Optional[List[Optional["MultiModalKwargs"]]]
vllm/v1/engine/core_client.py
+6
@@ -219,6 +219,9 @@ class SyncMPClient(MPClient):
219
self.input_socket.send_multipart(msg, copy=False)
220
221
def add_request(self, request: EngineCoreRequest) -> None:
222
+ # NOTE: text prompt is not needed in the core engine as it has been
223
+ # tokenized.
224
+ request.prompt = None
225
self._send_input(EngineCoreRequestType.ADD, request)
226
227
def abort_requests(self, request_ids: List[str]) -> None:
@@ -257,6 +260,9 @@ class AsyncMPClient(MPClient):
260
await self.input_socket.send_multipart(msg, copy=False)
261
262
async def add_request_async(self, request: EngineCoreRequest) -> None:
263
+ # NOTE: text prompt is not needed in the core engine as it has been
264
+ # tokenized.
265
+ request.prompt = None
266
await self._send_input(EngineCoreRequestType.ADD, request)
267
268
async def abort_requests_async(self, request_ids: List[str]) -> None: