@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b25cfab9a0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[V1] Avoid sending text prompt to core engine (#11963)

Signed-off-by: Roger Wang <ywang@roblox.com>

Roger Wang committed Jan 11, 2025 at 22:36 UTC b25cfab9a03b5c460fb92340b310d2a5c2dbc5da
2 files changed +8 -2
vllm/v1/engine/__init__.py
+2 -2
@@ -19,8 +19,8 @@ class EngineCoreRequest:
19 # due to circular imports and typing we have in data.py
20
21 request_id: str
22 - #NOTE(Nick): I don't think we need to pass prompt here since it should
23 - # always be tokenized?
22 + # NOTE(ywang96): original text prompt is needed when a request is added to
23 + # Detokenizer, but set to None when it is added to EngineCoreClient.
24 prompt: Optional[str]
25 prompt_token_ids: List[int]
26 mm_inputs: Optional[List[Optional["MultiModalKwargs"]]]
vllm/v1/engine/core_client.py
+6
@@ -219,6 +219,9 @@ class SyncMPClient(MPClient):
219 self.input_socket.send_multipart(msg, copy=False)
220
221 def add_request(self, request: EngineCoreRequest) -> None:
222 + # NOTE: text prompt is not needed in the core engine as it has been
223 + # tokenized.
224 + request.prompt = None
225 self._send_input(EngineCoreRequestType.ADD, request)
226
227 def abort_requests(self, request_ids: List[str]) -> None:
@@ -257,6 +260,9 @@ class AsyncMPClient(MPClient):
260 await self.input_socket.send_multipart(msg, copy=False)
261
262 async def add_request_async(self, request: EngineCoreRequest) -> None:
263 + # NOTE: text prompt is not needed in the core engine as it has been
264 + # tokenized.
265 + request.prompt = None
266 await self._send_input(EngineCoreRequestType.ADD, request)
267
268 async def abort_requests_async(self, request_ids: List[str]) -> None: