A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 04:30 UTC
Fix IntermediateTensors initialization and add type hints (#28743)
Signed-off-by: Mohammad Othman <Mo@MohammadOthman.com> Co-authored-by: Mohammad Othman <Mo@MohammadOthman.com>
Mohammad Othman committed
Nov 15, 2025 at 06:31 UTC
363aaeef0ff8511fd1466d41a2e027b22b28f39b
1 file changed
+6
-1
vllm/sequence.py
+6
-1
@@ -60,12 +60,17 @@ class IntermediateTensors:
60
tensors: dict[str, torch.Tensor]
61
kv_connector_output: KVConnectorOutput | None
62
63
- def __init__(self, tensors):
63
+ def __init__(
64
+ self,
65
+ tensors: dict[str, torch.Tensor],
66
+ kv_connector_output: KVConnectorOutput | None = None,
67
+ ) -> None:
68
# manually define this function, so that
69
# Dynamo knows `IntermediateTensors()` comes from this file.
70
# Otherwise, dataclass will generate this function by evaluating
71
# a string, and we will lose the information about the source file.
72
self.tensors = tensors
73
+ self.kv_connector_output = kv_connector_output
74
75
def __getitem__(self, key: str | slice):
76
if isinstance(key, str):