A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
mypy type checking for vllm/worker (#11418)
Signed-off-by: lucast2021 <lucast2021@headroyce.org> Co-authored-by: lucast2021 <lucast2021@headroyce.org>
Lucas Tucker committed
Dec 23, 2024 at 07:55 UTC
e51719ae72dd1dcdf55436a99ac8bed245b51422
3 files changed
+9
-9
vllm/worker/cpu_worker.py
+1
-2
@@ -333,9 +333,8 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
333
def prepare_worker_input(
334
self, execute_model_req: ExecuteModelRequest) -> WorkerInput:
335
assert execute_model_req is not None
336
- virtual_engine = execute_model_req.virtual_engine
336
+ virtual_engine: int = execute_model_req.virtual_engine
337
num_seq_groups: int = len(execute_model_req.seq_group_metadata_list)
338
- blocks_to_copy = execute_model_req.blocks_to_copy
338
blocks_to_copy = torch.tensor(execute_model_req.blocks_to_copy,
339
device="cpu",
340
dtype=torch.int64).view(-1, 2)
vllm/worker/multi_step_model_runner.py
+7
-6
@@ -406,8 +406,9 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
406
if not cont:
407
break
408
409
- def _final_process_outputs(self, model_input: StatefulModelInput,
410
- output_proc_callback: Optional[Callable]):
409
+ def _final_process_outputs(
410
+ self, model_input: StatefulModelInput,
411
+ output_proc_callback: Optional[Callable]) -> List[SamplerOutput]:
412
assert model_input.frozen_model_input is not None
413
414
has_async_callback = output_proc_callback is not None
@@ -594,8 +595,8 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
595
# should be [SamplerOutput]
596
return output
597
597
- def _update_sampling_metadata(self, sampling_metadata, num_seqs,
598
- num_queries):
598
+ def _update_sampling_metadata(self, sampling_metadata: SamplingMetadata,
599
+ num_seqs: Optional[int], num_queries: int):
600
601
assert sampling_metadata.num_prompts == 0
602
assert len(sampling_metadata.seq_groups) == num_queries
@@ -850,13 +851,13 @@ def _pythonize_sampler_output(
851
seq_ids = seq_group.seq_ids
852
next_token_ids = sample_result
853
parent_ids = [0]
854
+ seq_outputs: List[SequenceOutput]
855
856
if cache is not None:
857
completion_seq_group_output: CompletionSequenceGroupOutput = \
858
cache.cached_completion_seq_group_output.get_object()
859
completion_seq_group_output.samples.clear()
858
- seq_outputs: List[
859
- SequenceOutput] = completion_seq_group_output.samples
860
+ seq_outputs = completion_seq_group_output.samples
861
else:
862
seq_outputs = []
863
vllm/worker/worker_base.py
+1
-1
@@ -452,7 +452,7 @@ class WorkerWrapperBase:
452
self.worker = worker_class(*args, **kwargs)
453
assert self.worker is not None
454
455
- def execute_method(self, method, *args, **kwargs):
455
+ def execute_method(self, method: str, *args, **kwargs):
456
try:
457
target = self if self.worker is None else self.worker
458
executor = getattr(target, method)