@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c08e2b3086
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[core] [2/N] refactor worker_base input preparation for multi-step (#7387)

William Lin committed Aug 11, 2024 at 08:50 UTC c08e2b30862df5427843de76d8a619ea566600c1
2 files changed +63 -31
vllm/worker/worker.py
+2
@@ -264,6 +264,7 @@ class Worker(LocalOrDistributedWorkerBase):
264 def prepare_worker_input(
265 self, execute_model_req: ExecuteModelRequest) -> WorkerInput:
266 virtual_engine = execute_model_req.virtual_engine
267 + num_steps = execute_model_req.num_steps
268 num_seq_groups = len(execute_model_req.seq_group_metadata_list)
269 # `blocks_to_swap_in` and `blocks_to_swap_out` are cpu tensors.
270 # they contain parameters to launch cudamemcpyasync.
@@ -286,6 +287,7 @@ class Worker(LocalOrDistributedWorkerBase):
287 blocks_to_swap_out=blocks_to_swap_out,
288 blocks_to_copy=blocks_to_copy,
289 virtual_engine=virtual_engine,
290 + num_steps=num_steps,
291 )
292
293 @torch.inference_mode()
vllm/worker/worker_base.py
+61 -31
@@ -129,6 +129,7 @@ class WorkerInput:
129 blocks_to_swap_out: Optional[torch.Tensor] = None
130 blocks_to_copy: Optional[torch.Tensor] = None
131 virtual_engine: int = 0
132 + num_steps: int = 1
133
134 @classmethod
135 def from_broadcasted_tensor_dict(
@@ -145,6 +146,7 @@ class WorkerInput:
146 blocks_to_swap_out=tensor_dict.pop("blocks_to_swap_out"),
147 blocks_to_copy=tensor_dict.pop("blocks_to_copy"),
148 virtual_engine=tensor_dict["virtual_engine"],
149 + num_steps=tensor_dict.pop("num_steps"),
150 )
151
152 def as_broadcastable_tensor_dict(
@@ -158,6 +160,7 @@ class WorkerInput:
160 "blocks_to_swap_out": self.blocks_to_swap_out,
161 "blocks_to_copy": self.blocks_to_copy,
162 "virtual_engine": self.virtual_engine,
163 + "num_steps": self.num_steps,
164 }
165
166 return tensor_dict
@@ -216,13 +219,50 @@ class LocalOrDistributedWorkerBase(WorkerBase):
219 """
220 raise NotImplementedError
221
219 - def execute_model(
222 + def _get_worker_input_from_broadcast(
223 + self) -> Optional[Tuple[ModelRunnerInputBase, WorkerInput]]:
224 + """ Get the worker input from the broadcasted tensor dict. """
225 + assert self.do_metadata_broadcast
226 + assert not self.is_driver_worker
227 + broadcast_data = broadcast_tensor_dict(src=0)
228 + if not broadcast_data:
229 + return None
230 +
231 + worker_input = WorkerInput.from_broadcasted_tensor_dict(broadcast_data)
232 + model_input = (
233 + self.model_runner.make_model_input_from_broadcasted_tensor_dict(
234 + broadcast_data))
235 +
236 + return model_input, worker_input
237 +
238 + def _get_driver_input_and_broadcast(
239 + self, execute_model_req: ExecuteModelRequest
240 + ) -> Tuple[ModelRunnerInputBase, WorkerInput]:
241 + """ Get the driver input and broadcast it to other workers. """
242 + assert self.is_driver_worker
243 +
244 + worker_input: WorkerInput = self.prepare_worker_input(
245 + execute_model_req=execute_model_req)
246 + model_input: ModelRunnerInputBase = (
247 + self.model_runner.prepare_model_input(
248 + execute_model_req.seq_group_metadata_list,
249 + execute_model_req.virtual_engine,
250 + execute_model_req.finished_requests_ids))
251 +
252 + if self.do_metadata_broadcast:
253 + broadcast_data = worker_input.as_broadcastable_tensor_dict()
254 + broadcast_data.update(model_input.as_broadcastable_tensor_dict())
255 + broadcast_tensor_dict(broadcast_data, src=0)
256 +
257 + return model_input, worker_input
258 +
259 + def prepare_input(
260 self,
261 execute_model_req: Optional[ExecuteModelRequest] = None
222 - ) -> Optional[List[SamplerOutput]]:
223 - """Executes at least one model step on the given sequences, unless no
224 - sequences are provided."""
225 - start_time = time.perf_counter()
262 + ) -> Optional[Tuple[ModelRunnerInputBase, WorkerInput]]:
263 + """
264 + Prepare the inputs to ModelRunner and workers.
265 + """
266 if self.is_driver_worker:
267 if execute_model_req is None:
268 if self.do_metadata_broadcast:
@@ -233,34 +273,24 @@ class LocalOrDistributedWorkerBase(WorkerBase):
273 # notify all other workers to stop their execution loop.
274 broadcast_tensor_dict({}, src=0)
275 return None
236 -
237 - worker_input: WorkerInput = self.prepare_worker_input(
238 - execute_model_req=execute_model_req)
239 - model_input: ModelRunnerInputBase = (
240 - self.model_runner.prepare_model_input(
241 - execute_model_req.seq_group_metadata_list,
242 - execute_model_req.virtual_engine,
243 - execute_model_req.finished_requests_ids))
244 - num_steps = execute_model_req.num_steps
245 -
246 - if self.do_metadata_broadcast:
247 - broadcast_data = worker_input.as_broadcastable_tensor_dict()
248 - broadcast_data.update(
249 - model_input.as_broadcastable_tensor_dict())
250 - broadcast_data["num_steps"] = num_steps
251 - broadcast_tensor_dict(broadcast_data, src=0)
276 + return self._get_driver_input_and_broadcast(execute_model_req)
277 else:
253 - assert self.do_metadata_broadcast
254 - broadcast_data = broadcast_tensor_dict(src=0)
255 - if not broadcast_data:
256 - return None
278 + return self._get_worker_input_from_broadcast()
279 +
280 + def execute_model(
281 + self,
282 + execute_model_req: Optional[ExecuteModelRequest] = None
283 + ) -> Optional[List[SamplerOutput]]:
284 + """Executes at least one model step on the given sequences, unless no
285 + sequences are provided."""
286 + start_time = time.perf_counter()
287 +
288 + inputs = self.prepare_input(execute_model_req)
289 + if inputs is None:
290 + return None
291
258 - num_steps = broadcast_data.pop("num_steps")
259 - worker_input = WorkerInput.from_broadcasted_tensor_dict(
260 - broadcast_data)
261 - model_input = (
262 - self.model_runner.
263 - make_model_input_from_broadcasted_tensor_dict(broadcast_data))
292 + model_input, worker_input = inputs
293 + num_steps = worker_input.num_steps
294
295 self.execute_worker(worker_input)
296