@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 51f86bf487
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[mypy][CI/Build] Fix mypy errors (#7929)

Cyrus Leung committed Aug 28, 2024 at 14:47 UTC 51f86bf48730c3766f39c15aecc1268780879835
5 files changed +24 -9
tests/samplers/test_sampler.py
+5
@@ -418,6 +418,7 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
418 prompt_len = seq_data.get_prompt_len()
419 seq_lens.append(prompt_len)
420
421 + assert sgm.sampling_params is not None
422 if sgm.sampling_params.prompt_logprobs:
423 # with prompt_logprobs each token in the prompt has a row in
424 # logits
@@ -533,6 +534,8 @@ def test_sampler_mixed(seed: int, device: str):
534
535 for i, (sequence_output, metadata) in enumerate(
536 zip(sampler_output, seq_group_metadata_list)):
537 + assert metadata.sampling_params is not None
538 +
539 if metadata.sampling_params.use_beam_search:
540 continue
541
@@ -550,6 +553,8 @@ def test_sampler_mixed(seed: int, device: str):
553 assert expected_tokens_item is not None
554
555 for n, nth_output in enumerate(sequence_output.samples):
556 + assert metadata.sampling_params is not None
557 +
558 if (metadata.sampling_params.temperature == 0
559 or metadata.sampling_params.seed is not None):
560 # Ensure exact matches for greedy or random with seed
vllm/assets/audio.py
+3 -1
@@ -19,7 +19,9 @@ class AudioAsset:
19
20 audio_path = get_vllm_public_assets(filename=f"{self.name}.ogg",
21 s3_prefix=ASSET_DIR)
22 - return librosa.load(audio_path, sr=None)
22 + y, sr = librosa.load(audio_path, sr=None)
23 + assert isinstance(sr, int)
24 + return y, sr
25
26 @property
27 def url(self) -> str:
vllm/entrypoints/openai/rpc/client.py
+3 -2
@@ -101,6 +101,7 @@ class AsyncEngineRPCClient:
101 # Maximum number of sockets that can be opened (typically 65536).
102 # ZMQ_SOCKET_LIMIT (http://api.zeromq.org/4-2:zmq-ctx-get)
103 socket_limit = self.context.get(zmq.constants.SOCKET_LIMIT)
104 + assert isinstance(socket_limit, int)
105 if socket_limit < VLLM_RPC_SOCKET_LIMIT_CUTOFF:
106 raise ValueError(
107 f"Found zmq.constants.SOCKET_LIMIT={socket_limit}, which caps "
@@ -141,8 +142,8 @@ class AsyncEngineRPCClient:
142 poller.register(socket_from, zmq.constants.POLLIN)
143 poller.register(socket_to, zmq.constants.POLLIN)
144 while True:
144 - events = await poller.poll()
145 - events = dict(events)
145 + events_lst = await poller.poll()
146 + events = dict(events_lst)
147 if socket_from in events:
148 identity, msg = await socket_from.recv_multipart()
149 await socket_to.send_multipart([identity, msg])
vllm/multimodal/base.py
+12 -5
@@ -14,7 +14,7 @@ from typing_extensions import TypeAlias
14 from vllm.config import ModelConfig
15 from vllm.inputs import InputContext
16 from vllm.logger import init_logger
17 -from vllm.utils import json_map_leaves
17 +from vllm.utils import JSONTree, is_list_of, json_map_leaves
18
19 logger = init_logger(__name__)
20
@@ -54,13 +54,14 @@ class MultiModalInputs(_MultiModalInputsBase):
54 return nested_tensors
55
56 stacked = [MultiModalInputs._try_stack(t) for t in nested_tensors]
57 - if any(isinstance(t, list) for t in stacked):
57 + if is_list_of(stacked, list):
58 + # Do not stack nested lists
59 return stacked
60
61 tensors_ = cast(List[torch.Tensor], stacked)
62 if any(t.shape != tensors_[0].shape for t in tensors_):
63 # The tensors have incompatible shapes and can't be stacked.
63 - return tensors_
64 + return stacked
65
66 return torch.stack(tensors_)
67
@@ -101,8 +102,14 @@ class MultiModalInputs(_MultiModalInputsBase):
102 *,
103 device: torch.types.Device,
104 ) -> BatchedTensorInputs:
104 - return json_map_leaves(lambda x: x.to(device, non_blocking=True),
105 - batched_inputs)
105 + json_inputs = cast(JSONTree[torch.Tensor], batched_inputs)
106 +
107 + json_mapped = json_map_leaves(
108 + lambda x: x.to(device, non_blocking=True),
109 + json_inputs,
110 + )
111 +
112 + return cast(BatchedTensorInputs, json_mapped)
113
114
115 _T = TypeVar("_T")
vllm/sequence.py
+1 -1
@@ -883,7 +883,7 @@ class SequenceGroupMetadata(
883 request_id: str
884 is_prompt: bool
885 seq_data: Dict[int, SequenceData]
886 - sampling_params: SamplingParams
886 + sampling_params: Optional[SamplingParams]
887 block_tables: Dict[int, List[int]]
888 do_sample: bool = True
889 pooling_params: Optional[PoolingParams] = None