A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[core] [2/N] refactor worker_base input preparation for multi-step (#7387)
William Lin committed
Aug 11, 2024 at 08:50 UTC
c08e2b30862df5427843de76d8a619ea566600c1
2 files changed
+63
-31
vllm/worker/worker.py
+2
@@ -264,6 +264,7 @@ class Worker(LocalOrDistributedWorkerBase):
264
def prepare_worker_input(
265
self, execute_model_req: ExecuteModelRequest) -> WorkerInput:
266
virtual_engine = execute_model_req.virtual_engine
267
+ num_steps = execute_model_req.num_steps
268
num_seq_groups = len(execute_model_req.seq_group_metadata_list)
269
# `blocks_to_swap_in` and `blocks_to_swap_out` are cpu tensors.
270
# they contain parameters to launch cudamemcpyasync.
@@ -286,6 +287,7 @@ class Worker(LocalOrDistributedWorkerBase):
287
blocks_to_swap_out=blocks_to_swap_out,
288
blocks_to_copy=blocks_to_copy,
289
virtual_engine=virtual_engine,
290
+ num_steps=num_steps,
291
)
292
293
@torch.inference_mode()
vllm/worker/worker_base.py
+61
-31
@@ -129,6 +129,7 @@ class WorkerInput:
129
blocks_to_swap_out: Optional[torch.Tensor] = None
130
blocks_to_copy: Optional[torch.Tensor] = None
131
virtual_engine: int = 0
132
+ num_steps: int = 1
133
134
@classmethod
135
def from_broadcasted_tensor_dict(
@@ -145,6 +146,7 @@ class WorkerInput:
146
blocks_to_swap_out=tensor_dict.pop("blocks_to_swap_out"),
147
blocks_to_copy=tensor_dict.pop("blocks_to_copy"),
148
virtual_engine=tensor_dict["virtual_engine"],
149
+ num_steps=tensor_dict.pop("num_steps"),
150
)
151
152
def as_broadcastable_tensor_dict(
@@ -158,6 +160,7 @@ class WorkerInput:
160
"blocks_to_swap_out": self.blocks_to_swap_out,
161
"blocks_to_copy": self.blocks_to_copy,
162
"virtual_engine": self.virtual_engine,
163
+ "num_steps": self.num_steps,
164
}
165
166
return tensor_dict
@@ -216,13 +219,50 @@ class LocalOrDistributedWorkerBase(WorkerBase):
219
"""
220
raise NotImplementedError
221
219
- def execute_model(
222
+ def _get_worker_input_from_broadcast(
223
+ self) -> Optional[Tuple[ModelRunnerInputBase, WorkerInput]]:
224
+ """ Get the worker input from the broadcasted tensor dict. """
225
+ assert self.do_metadata_broadcast
226
+ assert not self.is_driver_worker
227
+ broadcast_data = broadcast_tensor_dict(src=0)
228
+ if not broadcast_data:
229
+ return None
230
+
231
+ worker_input = WorkerInput.from_broadcasted_tensor_dict(broadcast_data)
232
+ model_input = (
233
+ self.model_runner.make_model_input_from_broadcasted_tensor_dict(
234
+ broadcast_data))
235
+
236
+ return model_input, worker_input
237
+
238
+ def _get_driver_input_and_broadcast(
239
+ self, execute_model_req: ExecuteModelRequest
240
+ ) -> Tuple[ModelRunnerInputBase, WorkerInput]:
241
+ """ Get the driver input and broadcast it to other workers. """
242
+ assert self.is_driver_worker
243
+
244
+ worker_input: WorkerInput = self.prepare_worker_input(
245
+ execute_model_req=execute_model_req)
246
+ model_input: ModelRunnerInputBase = (
247
+ self.model_runner.prepare_model_input(
248
+ execute_model_req.seq_group_metadata_list,
249
+ execute_model_req.virtual_engine,
250
+ execute_model_req.finished_requests_ids))
251
+
252
+ if self.do_metadata_broadcast:
253
+ broadcast_data = worker_input.as_broadcastable_tensor_dict()
254
+ broadcast_data.update(model_input.as_broadcastable_tensor_dict())
255
+ broadcast_tensor_dict(broadcast_data, src=0)
256
+
257
+ return model_input, worker_input
258
+
259
+ def prepare_input(
260
self,
261
execute_model_req: Optional[ExecuteModelRequest] = None
222
- ) -> Optional[List[SamplerOutput]]:
223
- """Executes at least one model step on the given sequences, unless no
224
- sequences are provided."""
225
- start_time = time.perf_counter()
262
+ ) -> Optional[Tuple[ModelRunnerInputBase, WorkerInput]]:
263
+ """
264
+ Prepare the inputs to ModelRunner and workers.
265
+ """
266
if self.is_driver_worker:
267
if execute_model_req is None:
268
if self.do_metadata_broadcast:
@@ -233,34 +273,24 @@ class LocalOrDistributedWorkerBase(WorkerBase):
273
# notify all other workers to stop their execution loop.
274
broadcast_tensor_dict({}, src=0)
275
return None
236
-
237
- worker_input: WorkerInput = self.prepare_worker_input(
238
- execute_model_req=execute_model_req)
239
- model_input: ModelRunnerInputBase = (
240
- self.model_runner.prepare_model_input(
241
- execute_model_req.seq_group_metadata_list,
242
- execute_model_req.virtual_engine,
243
- execute_model_req.finished_requests_ids))
244
- num_steps = execute_model_req.num_steps
245
-
246
- if self.do_metadata_broadcast:
247
- broadcast_data = worker_input.as_broadcastable_tensor_dict()
248
- broadcast_data.update(
249
- model_input.as_broadcastable_tensor_dict())
250
- broadcast_data["num_steps"] = num_steps
251
- broadcast_tensor_dict(broadcast_data, src=0)
276
+ return self._get_driver_input_and_broadcast(execute_model_req)
277
else:
253
- assert self.do_metadata_broadcast
254
- broadcast_data = broadcast_tensor_dict(src=0)
255
- if not broadcast_data:
256
- return None
278
+ return self._get_worker_input_from_broadcast()
279
+
280
+ def execute_model(
281
+ self,
282
+ execute_model_req: Optional[ExecuteModelRequest] = None
283
+ ) -> Optional[List[SamplerOutput]]:
284
+ """Executes at least one model step on the given sequences, unless no
285
+ sequences are provided."""
286
+ start_time = time.perf_counter()
287
+
288
+ inputs = self.prepare_input(execute_model_req)
289
+ if inputs is None:
290
+ return None
291
258
- num_steps = broadcast_data.pop("num_steps")
259
- worker_input = WorkerInput.from_broadcasted_tensor_dict(
260
- broadcast_data)
261
- model_input = (
262
- self.model_runner.
263
- make_model_input_from_broadcasted_tensor_dict(broadcast_data))
292
+ model_input, worker_input = inputs
293
+ num_steps = worker_input.num_steps
294
295
self.execute_worker(worker_input)
296