A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Hardware][Intel-Gaudi] Enable long-contexts + LoRA support for Intel Gaudi (#12812)
Signed-off-by: Sanju C Sudhakaran <scsudhakaran@habana.ai>
Sanju C Sudhakaran committed
Feb 8, 2025 at 14:45 UTC
2880e21e3d2513c89bd63ac05b718e0c0a50e4e4
3 files changed
+73
-4
vllm/lora/punica_wrapper/punica_hpu.py
+56
-1
@@ -1,12 +1,18 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-from typing import Optional, Tuple, Union, final
3
+from typing import TYPE_CHECKING, List, Optional, Tuple, Union, final
4
5
import torch
6
from vllm_hpu_extension.ops import (dispatch_bgmv_embedding,
7
dispatch_bgmv_linear)
8
9
from .punica_base import PunicaWrapperBase
10
+from .utils import convert_mapping
11
+
12
+if TYPE_CHECKING:
13
+ # avoid circuit import
14
+ from vllm.lora.layers import LoRAMapping
15
+ from vllm.lora.models import LongContextLoRAContext
16
17
18
@final
@@ -19,6 +25,55 @@ class PunicaWrapperHPU(PunicaWrapperBase):
25
PunicaWrapperBase.__init__(self, 3 * max_num_batched_tokens,
26
max_batches, device)
27
28
+ def _update_base_metadata(
29
+ self,
30
+ mapping: "LoRAMapping",
31
+ lora_index_to_id: List[Optional[int]],
32
+ max_loras: int,
33
+ vocab_size: int,
34
+ extra_vocab_size: int,
35
+ long_lora_context: Optional["LongContextLoRAContext"] = None,
36
+ ):
37
+ (
38
+ base_indices,
39
+ sampler_indices,
40
+ sampler_indices_padded,
41
+ embeddings_indices,
42
+ long_lora_offsets_tensor,
43
+ indices_len,
44
+ ) = convert_mapping(mapping, lora_index_to_id, max_loras, vocab_size,
45
+ extra_vocab_size, self.device, None)
46
+ # Updating each element in `long_lora_offsets` with `lora_offset` slows
47
+ # down perf in HPU due to a series of `strided_insert` ops during lazy
48
+ # graph accumulation. Hence HPU appends `lora_offset` to a list and
49
+ # converts it to a tensor only after it is ready.
50
+ if long_lora_context:
51
+ index_mapping_indices: List[int] = list(
52
+ mapping.index_mapping).copy()
53
+ long_lora_offsets: List[int] = []
54
+ for i in range(len(index_mapping_indices)):
55
+ lora_offset: int = long_lora_context.offsets_by_lora_id.get(
56
+ index_mapping_indices[i], 0)
57
+ long_lora_offsets.append(lora_offset)
58
+ long_lora_offsets_tensor = torch.tensor(long_lora_offsets,
59
+ device=self.device,
60
+ dtype=torch.long)
61
+ indices_len[-1] = long_lora_offsets_tensor.shape[-1]
62
+
63
+ self._token_lora_indices[:base_indices.shape[0]].copy_(base_indices)
64
+ self._sampler_indices[:sampler_indices.shape[0]].copy_(sampler_indices)
65
+ self._sampler_indices_padded[:sampler_indices_padded.shape[0]].copy_(
66
+ sampler_indices_padded)
67
+ self._embeddings_indices[:embeddings_indices.
68
+ shape[0], :embeddings_indices.shape[1]].copy_(
69
+ embeddings_indices)
70
+ if long_lora_offsets_tensor is not None:
71
+ self._long_lora_indices[:long_lora_offsets_tensor.shape[0]].copy_(
72
+ long_lora_offsets_tensor)
73
+ else:
74
+ self._long_lora_indices.zero_()
75
+ self.indices_len[:] = indices_len
76
+
77
def add_lora_embedding(self,
78
y: torch.Tensor,
79
x: torch.Tensor,
vllm/model_executor/layers/rotary_embedding.py
+2
-1
@@ -206,9 +206,10 @@ class RotaryEmbedding(CustomOp):
206
) -> Tuple[torch.Tensor, torch.Tensor]:
207
from habana_frameworks.torch.hpex.kernels import (
208
RotaryPosEmbeddingMode, apply_rotary_pos_emb)
209
- positions = positions.flatten()
209
if offsets is not None:
210
+ offsets = offsets.view(positions.shape[0], -1)
211
positions = positions + offsets
212
+ positions = positions.flatten()
213
num_tokens = positions.shape[0]
214
cos_sin = self.cos_sin_cache.index_select(0, positions).view(
215
num_tokens, 1, -1)
vllm/worker/hpu_model_runner.py
+15
-2
@@ -639,12 +639,25 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
639
"Bias support in LoRA is not enabled in HPU yet."
640
assert not self.lora_config.fully_sharded_loras, \
641
"Fully sharded LoRAs is not enabled in HPU yet."
642
+ # It's necessary to distinguish between the
643
+ # max_position_embeddings of VLMs and LLMs.
644
+ if hasattr(self.model.config, "max_position_embeddings"):
645
+ max_pos_embeddings = (
646
+ self.model.config.max_position_embeddings)
647
+ else:
648
+ max_pos_embeddings = (
649
+ self.model.config.text_config.max_position_embeddings)
650
+
651
self.lora_manager = LRUCacheWorkerLoRAManager(
652
self.scheduler_config.max_num_seqs,
653
self.scheduler_config.max_num_batched_tokens,
645
- self.vocab_size, self.lora_config, self.device,
654
+ self.vocab_size,
655
+ self.lora_config,
656
+ self.device,
657
self.model.embedding_modules,
647
- self.model.embedding_padding_modules)
658
+ self.model.embedding_padding_modules,
659
+ max_position_embeddings=max_pos_embeddings,
660
+ )
661
self.model = self.lora_manager.create_lora_manager(self.model)
662
663
if self.model_config.quantization == 'inc':