A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[CI/Build] Fix mypy errors (#6968)
Cyrus Leung committed
Jul 31, 2024 at 10:49 UTC
9f0e69b65350fad1d4a9c71ef58d6ae70eb635e8
2 files changed
+4
-6
vllm/_custom_ops.py
+2
-2
@@ -1,6 +1,6 @@
1
import contextlib
2
import functools
3
-from typing import List, Optional, Tuple, Type
3
+from typing import List, Optional, Tuple, Union
4
5
import torch
6
@@ -336,7 +336,7 @@ def scaled_fp8_quant(
336
"""
337
# This code assumes batch_dim and num_tokens are flattened
338
assert (input.ndim == 2)
339
- shape = input.shape
339
+ shape: Union[Tuple[int, int], torch.Size] = input.shape
340
if num_token_padding:
341
shape = (max(num_token_padding, input.shape[0]), shape[1])
342
output = torch.empty(shape, device=input.device, dtype=torch.float8_e4m3fn)
vllm/multimodal/base.py
+2
-4
@@ -53,9 +53,7 @@ class MultiModalInputs(_MultiModalInputsBase):
53
"""
54
55
@staticmethod
56
- def _try_concat(
57
- tensors: List[NestedTensors],
58
- ) -> Union[GenericSequence[NestedTensors], NestedTensors]:
56
+ def _try_concat(tensors: List[NestedTensors]) -> BatchedTensors:
57
"""
58
If each input tensor in the batch has the same shape, return a single
59
batched tensor; otherwise, return a list of :class:`NestedTensors` with
@@ -105,7 +103,7 @@ class MultiModalInputs(_MultiModalInputsBase):
103
return {
104
k: MultiModalInputs._try_concat(item_list)
105
for k, item_list in item_lists.items()
108
- } # type: ignore
106
+ }
107
108
@staticmethod
109
def as_kwargs(