@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8195824206
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Hardware][Intel-Gaudi] Enable LoRA support for Intel Gaudi (HPU) (#10565)

Signed-off-by: Sanju C Sudhakaran <scsudhakaran@habana.ai>

Sanju C Sudhakaran committed Dec 12, 2024 at 13:39 UTC 8195824206ad2e3c45d1807b321c11f06ccb3a91
5 files changed +107 -14
requirements-hpu.txt
+1 -1
@@ -8,4 +8,4 @@ pandas
8 tabulate
9 setuptools>=61
10 setuptools-scm>=8
11 -vllm-hpu-extension @ git+https://github.com/HabanaAI/vllm-hpu-extension.git@e096d6f
11 +vllm-hpu-extension @ git+https://github.com/HabanaAI/vllm-hpu-extension.git@4312768
vllm/lora/layers.py
+6
@@ -30,6 +30,7 @@ from vllm.model_executor.layers.rotary_embedding import (
30 LinearScalingRotaryEmbedding, RotaryEmbedding)
31 from vllm.model_executor.layers.vocab_parallel_embedding import (
32 VocabParallelEmbedding)
33 +from vllm.platforms import current_platform
34
35 if TYPE_CHECKING:
36 from vllm.lora.punica_wrapper import PunicaWrapperBase
@@ -1068,6 +1069,11 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1069 ).index_select(0, indices_padded).nan_to_num_(nan=float("-inf"),
1070 posinf=float("inf"),
1071 neginf=float("-inf")))
1072 +
1073 + # HPU needs special handling to prune out dummy samples.
1074 + if current_platform.is_hpu():
1075 + lora_logits = lora_logits[:logits.shape[0], :]
1076 +
1077 logits[:,
1078 self.base_layer.org_vocab_size:self.base_layer.org_vocab_size +
1079 lora_logits.shape[1]] = lora_logits
vllm/lora/punica_wrapper/punica_hpu.py new
+87
@@ -0,0 +1,87 @@
1 +from typing import Optional, Tuple, Union, final
2 +
3 +import torch
4 +from vllm_hpu_extension.ops import (dispatch_bgmv_embedding,
5 + dispatch_bgmv_linear)
6 +
7 +from .punica_base import PunicaWrapperBase
8 +
9 +
10 +@final
11 +class PunicaWrapperHPU(PunicaWrapperBase):
12 +
13 + def __init__(self, max_num_batched_tokens: int, max_batches: int,
14 + device: Union[torch.device, str], **kwargs):
15 + # Increasing max_num_batched_tokens by 3x to handle increase in
16 + # tensor size due to padding.
17 + PunicaWrapperBase.__init__(self, 3 * max_num_batched_tokens,
18 + max_batches, device)
19 +
20 + def add_lora_embedding(self,
21 + y: torch.Tensor,
22 + x: torch.Tensor,
23 + lora_b_stacked: torch.Tensor,
24 + add_input: bool = True,
25 + **kwargs) -> None:
26 + dispatch_bgmv_embedding(y, x, lora_b_stacked, 0)
27 +
28 + def add_lora_linear(self,
29 + y: torch.Tensor,
30 + x: torch.Tensor,
31 + lora_a_stacked: Tuple[torch.Tensor, ...],
32 + lora_b_stacked: Tuple[torch.Tensor, ...],
33 + lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
34 + scale: float,
35 + output_slices: Tuple[int, ...],
36 + *,
37 + buffer: Optional[Tuple[torch.Tensor, ...]] = None,
38 + **kwargs) -> None:
39 + y_org = y
40 + x = x.view(-1, x.shape[-1])
41 + y = y.view(-1, y.shape[-1])
42 + offset_left = 0
43 +
44 + for slice_idx in range(len(output_slices)):
45 + dispatch_bgmv_linear(
46 + y[:, offset_left:offset_left + output_slices[slice_idx]], x,
47 + lora_a_stacked[slice_idx], lora_b_stacked[slice_idx], 0, scale)
48 + offset_left += output_slices[slice_idx]
49 + y = y.view_as(y_org)
50 +
51 + def add_lora_logits(self,
52 + y: torch.Tensor,
53 + x: torch.Tensor,
54 + lora_a_stacked: torch.Tensor,
55 + lora_b_stacked: torch.Tensor,
56 + scale,
57 + *,
58 + buffer: Optional[torch.Tensor] = None,
59 + **kwargs) -> None:
60 + y_org = y
61 + y = y.view(-1, y.shape[-1])
62 + x = x.view(-1, x.shape[-1])
63 + dispatch_bgmv_linear(y, x, lora_a_stacked, lora_b_stacked, 0, scale)
64 + y = y.view_as(y_org)
65 +
66 + def add_shrink(
67 + self,
68 + y: Union[Tuple[torch.Tensor, ...], torch.Tensor],
69 + x: torch.Tensor,
70 + lora_a_stacked: Tuple[torch.Tensor, ...],
71 + scale: float,
72 + **kwargs,
73 + ) -> None:
74 + raise NotImplementedError
75 +
76 + def add_expand(
77 + self,
78 + y: torch.Tensor,
79 + x: Union[Tuple[torch.Tensor, ...], torch.Tensor],
80 + lora_b_stacked: Tuple[torch.Tensor, ...],
81 + lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
82 + output_slices: Tuple[int, ...],
83 + offset_start: int = 0,
84 + add_input=True,
85 + **kwargs,
86 + ) -> None:
87 + raise NotImplementedError
vllm/lora/punica_wrapper/punica_selector.py
+5
@@ -10,5 +10,10 @@ def get_punica_wrapper(*args, **kwargs) -> PunicaWrapperBase:
10 from vllm.lora.punica_wrapper.punica_gpu import PunicaWrapperGPU
11 print_info_once("Using PunicaWrapperGPU.")
12 return PunicaWrapperGPU(*args, **kwargs)
13 + elif current_platform.is_hpu():
14 + # Lazy import to avoid ImportError
15 + from vllm.lora.punica_wrapper.punica_hpu import PunicaWrapperHPU
16 + print_info_once("Using PunicaWrapperHPU.")
17 + return PunicaWrapperHPU(*args, **kwargs)
18 else:
19 raise NotImplementedError
vllm/worker/hpu_model_runner.py
+8 -13
@@ -622,6 +622,10 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
622 assert hasattr(
623 self.model, "embedding_padding_modules"
624 ), "Model does not have embedding_padding_modules"
625 + assert not self.lora_config.bias_enabled, \
626 + "Bias support in LoRA is not enabled in HPU yet."
627 + assert not self.lora_config.fully_sharded_loras, \
628 + "Fully sharded LoRAs is not enabled in HPU yet."
629 self.lora_manager = LRUCacheWorkerLoRAManager(
630 self.scheduler_config.max_num_seqs,
631 self.scheduler_config.max_num_batched_tokens,
@@ -1282,11 +1286,9 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
1286 def profile_run(self) -> None:
1287 num_layers = self.model_config.get_num_layers(self.parallel_config)
1288 kv_caches = [None] * num_layers
1285 - max_batch_size = self.bucketing_global_state.prompt_bs_bucket_cfg[-1]
1286 - max_seq_len = min(
1287 - self.bucketing_global_state.prompt_seq_bucket_cfg[-1],
1288 - self.max_num_batched_tokens // max_batch_size)
1289 -
1289 + max_seq_len = self.bucketing_global_state.prompt_seq_bucket_cfg[-1]
1290 + max_batch_size = min(self.max_num_batched_tokens // max_seq_len,
1291 + self.scheduler_config.max_num_seqs)
1292 self.warmup_scenario(max_batch_size, max_seq_len, True, kv_caches,
1293 False, True)
1294 return
@@ -1304,7 +1306,6 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
1306 f"bs{batch_size}_"
1307 f"seq{seq_len}_"
1308 f"graphs{'T' if use_graphs else 'F'}")
1307 - max_num_seqs = self.scheduler_config.max_num_seqs
1309 # This represents the maximum number of different requests
1310 # that will have unique loras, an therefore the max amount of memory
1311 # consumption create dummy lora request copies from the lora request
@@ -1326,16 +1327,10 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
1327 dummy_lora_requests.append(dummy_lora_request)
1328 dummy_lora_requests_per_seq = [
1329 dummy_lora_requests[idx % len(dummy_lora_requests)]
1329 - for idx in range(max_num_seqs)
1330 + for idx in range(batch_size)
1331 ]
1332 self.profiler.start('internal', scenario_name)
1333 times = 3 if use_graphs or is_pt_profiler_run else 1
1333 - if self.lora_config and not is_lora_profile_run:
1334 - lora_mapping = LoRAMapping(
1335 - **dict(index_mapping=[0] * batch_size * seq_len,
1336 - prompt_mapping=[0] * batch_size * seq_len,
1337 - is_prefill=is_prompt))
1338 - self.set_active_loras(set(), lora_mapping)
1334 if is_prompt:
1335 seqs = [
1336 self.create_dummy_seq_group_metadata(