@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e51719ae72
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

mypy type checking for vllm/worker (#11418)

Signed-off-by: lucast2021 <lucast2021@headroyce.org> Co-authored-by: lucast2021 <lucast2021@headroyce.org>

Lucas Tucker committed Dec 23, 2024 at 07:55 UTC e51719ae72dd1dcdf55436a99ac8bed245b51422
3 files changed +9 -9
vllm/worker/cpu_worker.py
+1 -2
@@ -333,9 +333,8 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
333 def prepare_worker_input(
334 self, execute_model_req: ExecuteModelRequest) -> WorkerInput:
335 assert execute_model_req is not None
336 - virtual_engine = execute_model_req.virtual_engine
336 + virtual_engine: int = execute_model_req.virtual_engine
337 num_seq_groups: int = len(execute_model_req.seq_group_metadata_list)
338 - blocks_to_copy = execute_model_req.blocks_to_copy
338 blocks_to_copy = torch.tensor(execute_model_req.blocks_to_copy,
339 device="cpu",
340 dtype=torch.int64).view(-1, 2)
vllm/worker/multi_step_model_runner.py
+7 -6
@@ -406,8 +406,9 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
406 if not cont:
407 break
408
409 - def _final_process_outputs(self, model_input: StatefulModelInput,
410 - output_proc_callback: Optional[Callable]):
409 + def _final_process_outputs(
410 + self, model_input: StatefulModelInput,
411 + output_proc_callback: Optional[Callable]) -> List[SamplerOutput]:
412 assert model_input.frozen_model_input is not None
413
414 has_async_callback = output_proc_callback is not None
@@ -594,8 +595,8 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
595 # should be [SamplerOutput]
596 return output
597
597 - def _update_sampling_metadata(self, sampling_metadata, num_seqs,
598 - num_queries):
598 + def _update_sampling_metadata(self, sampling_metadata: SamplingMetadata,
599 + num_seqs: Optional[int], num_queries: int):
600
601 assert sampling_metadata.num_prompts == 0
602 assert len(sampling_metadata.seq_groups) == num_queries
@@ -850,13 +851,13 @@ def _pythonize_sampler_output(
851 seq_ids = seq_group.seq_ids
852 next_token_ids = sample_result
853 parent_ids = [0]
854 + seq_outputs: List[SequenceOutput]
855
856 if cache is not None:
857 completion_seq_group_output: CompletionSequenceGroupOutput = \
858 cache.cached_completion_seq_group_output.get_object()
859 completion_seq_group_output.samples.clear()
858 - seq_outputs: List[
859 - SequenceOutput] = completion_seq_group_output.samples
860 + seq_outputs = completion_seq_group_output.samples
861 else:
862 seq_outputs = []
863
vllm/worker/worker_base.py
+1 -1
@@ -452,7 +452,7 @@ class WorkerWrapperBase:
452 self.worker = worker_class(*args, **kwargs)
453 assert self.worker is not None
454
455 - def execute_method(self, method, *args, **kwargs):
455 + def execute_method(self, method: str, *args, **kwargs):
456 try:
457 target = self if self.worker is None else self.worker
458 executor = getattr(target, method)