@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9f0e69b653
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[CI/Build] Fix mypy errors (#6968)

Cyrus Leung committed Jul 31, 2024 at 10:49 UTC 9f0e69b65350fad1d4a9c71ef58d6ae70eb635e8
2 files changed +4 -6
vllm/_custom_ops.py
+2 -2
@@ -1,6 +1,6 @@
1 import contextlib
2 import functools
3 -from typing import List, Optional, Tuple, Type
3 +from typing import List, Optional, Tuple, Union
4
5 import torch
6
@@ -336,7 +336,7 @@ def scaled_fp8_quant(
336 """
337 # This code assumes batch_dim and num_tokens are flattened
338 assert (input.ndim == 2)
339 - shape = input.shape
339 + shape: Union[Tuple[int, int], torch.Size] = input.shape
340 if num_token_padding:
341 shape = (max(num_token_padding, input.shape[0]), shape[1])
342 output = torch.empty(shape, device=input.device, dtype=torch.float8_e4m3fn)
vllm/multimodal/base.py
+2 -4
@@ -53,9 +53,7 @@ class MultiModalInputs(_MultiModalInputsBase):
53 """
54
55 @staticmethod
56 - def _try_concat(
57 - tensors: List[NestedTensors],
58 - ) -> Union[GenericSequence[NestedTensors], NestedTensors]:
56 + def _try_concat(tensors: List[NestedTensors]) -> BatchedTensors:
57 """
58 If each input tensor in the batch has the same shape, return a single
59 batched tensor; otherwise, return a list of :class:`NestedTensors` with
@@ -105,7 +103,7 @@ class MultiModalInputs(_MultiModalInputsBase):
103 return {
104 k: MultiModalInputs._try_concat(item_list)
105 for k, item_list in item_lists.items()
108 - } # type: ignore
106 + }
107
108 @staticmethod
109 def as_kwargs(