A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[mypy][CI/Build] Fix mypy errors (#7929)
Cyrus Leung committed
Aug 28, 2024 at 14:47 UTC
51f86bf48730c3766f39c15aecc1268780879835
5 files changed
+24
-9
tests/samplers/test_sampler.py
+5
@@ -418,6 +418,7 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
418
prompt_len = seq_data.get_prompt_len()
419
seq_lens.append(prompt_len)
420
421
+ assert sgm.sampling_params is not None
422
if sgm.sampling_params.prompt_logprobs:
423
# with prompt_logprobs each token in the prompt has a row in
424
# logits
@@ -533,6 +534,8 @@ def test_sampler_mixed(seed: int, device: str):
534
535
for i, (sequence_output, metadata) in enumerate(
536
zip(sampler_output, seq_group_metadata_list)):
537
+ assert metadata.sampling_params is not None
538
+
539
if metadata.sampling_params.use_beam_search:
540
continue
541
@@ -550,6 +553,8 @@ def test_sampler_mixed(seed: int, device: str):
553
assert expected_tokens_item is not None
554
555
for n, nth_output in enumerate(sequence_output.samples):
556
+ assert metadata.sampling_params is not None
557
+
558
if (metadata.sampling_params.temperature == 0
559
or metadata.sampling_params.seed is not None):
560
# Ensure exact matches for greedy or random with seed
vllm/assets/audio.py
+3
-1
@@ -19,7 +19,9 @@ class AudioAsset:
19
20
audio_path = get_vllm_public_assets(filename=f"{self.name}.ogg",
21
s3_prefix=ASSET_DIR)
22
- return librosa.load(audio_path, sr=None)
22
+ y, sr = librosa.load(audio_path, sr=None)
23
+ assert isinstance(sr, int)
24
+ return y, sr
25
26
@property
27
def url(self) -> str:
vllm/entrypoints/openai/rpc/client.py
+3
-2
@@ -101,6 +101,7 @@ class AsyncEngineRPCClient:
101
# Maximum number of sockets that can be opened (typically 65536).
102
# ZMQ_SOCKET_LIMIT (http://api.zeromq.org/4-2:zmq-ctx-get)
103
socket_limit = self.context.get(zmq.constants.SOCKET_LIMIT)
104
+ assert isinstance(socket_limit, int)
105
if socket_limit < VLLM_RPC_SOCKET_LIMIT_CUTOFF:
106
raise ValueError(
107
f"Found zmq.constants.SOCKET_LIMIT={socket_limit}, which caps "
@@ -141,8 +142,8 @@ class AsyncEngineRPCClient:
142
poller.register(socket_from, zmq.constants.POLLIN)
143
poller.register(socket_to, zmq.constants.POLLIN)
144
while True:
144
- events = await poller.poll()
145
- events = dict(events)
145
+ events_lst = await poller.poll()
146
+ events = dict(events_lst)
147
if socket_from in events:
148
identity, msg = await socket_from.recv_multipart()
149
await socket_to.send_multipart([identity, msg])
vllm/multimodal/base.py
+12
-5
@@ -14,7 +14,7 @@ from typing_extensions import TypeAlias
14
from vllm.config import ModelConfig
15
from vllm.inputs import InputContext
16
from vllm.logger import init_logger
17
-from vllm.utils import json_map_leaves
17
+from vllm.utils import JSONTree, is_list_of, json_map_leaves
18
19
logger = init_logger(__name__)
20
@@ -54,13 +54,14 @@ class MultiModalInputs(_MultiModalInputsBase):
54
return nested_tensors
55
56
stacked = [MultiModalInputs._try_stack(t) for t in nested_tensors]
57
- if any(isinstance(t, list) for t in stacked):
57
+ if is_list_of(stacked, list):
58
+ # Do not stack nested lists
59
return stacked
60
61
tensors_ = cast(List[torch.Tensor], stacked)
62
if any(t.shape != tensors_[0].shape for t in tensors_):
63
# The tensors have incompatible shapes and can't be stacked.
63
- return tensors_
64
+ return stacked
65
66
return torch.stack(tensors_)
67
@@ -101,8 +102,14 @@ class MultiModalInputs(_MultiModalInputsBase):
102
*,
103
device: torch.types.Device,
104
) -> BatchedTensorInputs:
104
- return json_map_leaves(lambda x: x.to(device, non_blocking=True),
105
- batched_inputs)
105
+ json_inputs = cast(JSONTree[torch.Tensor], batched_inputs)
106
+
107
+ json_mapped = json_map_leaves(
108
+ lambda x: x.to(device, non_blocking=True),
109
+ json_inputs,
110
+ )
111
+
112
+ return cast(BatchedTensorInputs, json_mapped)
113
114
115
_T = TypeVar("_T")
vllm/sequence.py
+1
-1
@@ -883,7 +883,7 @@ class SequenceGroupMetadata(
883
request_id: str
884
is_prompt: bool
885
seq_data: Dict[int, SequenceData]
886
- sampling_params: SamplingParams
886
+ sampling_params: Optional[SamplingParams]
887
block_tables: Dict[int, List[int]]
888
do_sample: bool = True
889
pooling_params: Optional[PoolingParams] = None