@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e66b629c04
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Misc] Minor fix in KVCache type (#3652)

Woosuk Kwon committed Mar 26, 2024 at 23:14 UTC e66b629c04c38e2743ccac5f14c43bdde016cae1
3 files changed +4 -8
docs/source/models/adding_model.rst
+2 -2
@@ -56,8 +56,8 @@ Next, you need to rewrite the :code:`forward` methods of your model by following
56 - return_dict: Optional[bool] = None,
57 -) -> Union[Tuple, CausalLMOutputWithPast]:
58 + positions: torch.Tensor,
59 - + kv_caches: List[KVCache],
60 - + input_metadata: InputMetadata,
59 + + kv_caches: List[torch.Tensor],
60 + + attn_metadata: AttentionMetadata,
61 +) -> Optional[SamplerOutput]:
62
63 1. Update the code by considering that :code:`input_ids` and :code:`positions` are now flattened tensors.
vllm/model_executor/models/llava.py
+2 -4
@@ -1,4 +1,4 @@
1 -from typing import List, Optional, Tuple
1 +from typing import List, Optional
2
3 import torch
4 from torch import nn
@@ -19,8 +19,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
19 hf_model_weights_iterator)
20 from vllm.sequence import SamplerOutput
21
22 -KVCache = Tuple[torch.Tensor, torch.Tensor]
23 -
22 _KEYS_TO_MODIFY_MAPPING = {
23 "language_model.lm_head": "lm_head",
24 "language_model.model": "language_model",
@@ -102,7 +100,7 @@ class LlavaForConditionalGeneration(nn.Module):
100 self,
101 input_ids: torch.Tensor,
102 positions: torch.Tensor,
105 - kv_caches: List[KVCache],
103 + kv_caches: List[torch.Tensor],
104 attn_metadata: AttentionMetadata,
105 image_input: Optional[torch.Tensor] = None
106 ) -> SamplerOutput: # noqa: E501
vllm/worker/neuron_model_runner.py
-2
@@ -14,8 +14,6 @@ from vllm.utils import (async_tensor_h2d, is_pin_memory_available,
14
15 logger = init_logger(__name__)
16
17 -KVCache = Tuple[torch.Tensor, torch.Tensor]
18 -
17
18 class NeuronModelRunner:
19