@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2880e21e3d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Hardware][Intel-Gaudi] Enable long-contexts + LoRA support for Intel Gaudi (#12812)

Signed-off-by: Sanju C Sudhakaran <scsudhakaran@habana.ai>

Sanju C Sudhakaran committed Feb 8, 2025 at 14:45 UTC 2880e21e3d2513c89bd63ac05b718e0c0a50e4e4
3 files changed +73 -4
vllm/lora/punica_wrapper/punica_hpu.py
+56 -1
@@ -1,12 +1,18 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 -from typing import Optional, Tuple, Union, final
3 +from typing import TYPE_CHECKING, List, Optional, Tuple, Union, final
4
5 import torch
6 from vllm_hpu_extension.ops import (dispatch_bgmv_embedding,
7 dispatch_bgmv_linear)
8
9 from .punica_base import PunicaWrapperBase
10 +from .utils import convert_mapping
11 +
12 +if TYPE_CHECKING:
13 + # avoid circuit import
14 + from vllm.lora.layers import LoRAMapping
15 + from vllm.lora.models import LongContextLoRAContext
16
17
18 @final
@@ -19,6 +25,55 @@ class PunicaWrapperHPU(PunicaWrapperBase):
25 PunicaWrapperBase.__init__(self, 3 * max_num_batched_tokens,
26 max_batches, device)
27
28 + def _update_base_metadata(
29 + self,
30 + mapping: "LoRAMapping",
31 + lora_index_to_id: List[Optional[int]],
32 + max_loras: int,
33 + vocab_size: int,
34 + extra_vocab_size: int,
35 + long_lora_context: Optional["LongContextLoRAContext"] = None,
36 + ):
37 + (
38 + base_indices,
39 + sampler_indices,
40 + sampler_indices_padded,
41 + embeddings_indices,
42 + long_lora_offsets_tensor,
43 + indices_len,
44 + ) = convert_mapping(mapping, lora_index_to_id, max_loras, vocab_size,
45 + extra_vocab_size, self.device, None)
46 + # Updating each element in `long_lora_offsets` with `lora_offset` slows
47 + # down perf in HPU due to a series of `strided_insert` ops during lazy
48 + # graph accumulation. Hence HPU appends `lora_offset` to a list and
49 + # converts it to a tensor only after it is ready.
50 + if long_lora_context:
51 + index_mapping_indices: List[int] = list(
52 + mapping.index_mapping).copy()
53 + long_lora_offsets: List[int] = []
54 + for i in range(len(index_mapping_indices)):
55 + lora_offset: int = long_lora_context.offsets_by_lora_id.get(
56 + index_mapping_indices[i], 0)
57 + long_lora_offsets.append(lora_offset)
58 + long_lora_offsets_tensor = torch.tensor(long_lora_offsets,
59 + device=self.device,
60 + dtype=torch.long)
61 + indices_len[-1] = long_lora_offsets_tensor.shape[-1]
62 +
63 + self._token_lora_indices[:base_indices.shape[0]].copy_(base_indices)
64 + self._sampler_indices[:sampler_indices.shape[0]].copy_(sampler_indices)
65 + self._sampler_indices_padded[:sampler_indices_padded.shape[0]].copy_(
66 + sampler_indices_padded)
67 + self._embeddings_indices[:embeddings_indices.
68 + shape[0], :embeddings_indices.shape[1]].copy_(
69 + embeddings_indices)
70 + if long_lora_offsets_tensor is not None:
71 + self._long_lora_indices[:long_lora_offsets_tensor.shape[0]].copy_(
72 + long_lora_offsets_tensor)
73 + else:
74 + self._long_lora_indices.zero_()
75 + self.indices_len[:] = indices_len
76 +
77 def add_lora_embedding(self,
78 y: torch.Tensor,
79 x: torch.Tensor,
vllm/model_executor/layers/rotary_embedding.py
+2 -1
@@ -206,9 +206,10 @@ class RotaryEmbedding(CustomOp):
206 ) -> Tuple[torch.Tensor, torch.Tensor]:
207 from habana_frameworks.torch.hpex.kernels import (
208 RotaryPosEmbeddingMode, apply_rotary_pos_emb)
209 - positions = positions.flatten()
209 if offsets is not None:
210 + offsets = offsets.view(positions.shape[0], -1)
211 positions = positions + offsets
212 + positions = positions.flatten()
213 num_tokens = positions.shape[0]
214 cos_sin = self.cos_sin_cache.index_select(0, positions).view(
215 num_tokens, 1, -1)
vllm/worker/hpu_model_runner.py
+15 -2
@@ -639,12 +639,25 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
639 "Bias support in LoRA is not enabled in HPU yet."
640 assert not self.lora_config.fully_sharded_loras, \
641 "Fully sharded LoRAs is not enabled in HPU yet."
642 + # It's necessary to distinguish between the
643 + # max_position_embeddings of VLMs and LLMs.
644 + if hasattr(self.model.config, "max_position_embeddings"):
645 + max_pos_embeddings = (
646 + self.model.config.max_position_embeddings)
647 + else:
648 + max_pos_embeddings = (
649 + self.model.config.text_config.max_position_embeddings)
650 +
651 self.lora_manager = LRUCacheWorkerLoRAManager(
652 self.scheduler_config.max_num_seqs,
653 self.scheduler_config.max_num_batched_tokens,
645 - self.vocab_size, self.lora_config, self.device,
654 + self.vocab_size,
655 + self.lora_config,
656 + self.device,
657 self.model.embedding_modules,
647 - self.model.embedding_padding_modules)
658 + self.model.embedding_padding_modules,
659 + max_position_embeddings=max_pos_embeddings,
660 + )
661 self.model = self.lora_manager.create_lora_manager(self.model)
662
663 if self.model_config.quantization == 'inc':