@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3cb5769883
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Misc] Minor improvements to the readability of PunicaWrapperBase (#11200)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Dec 15, 2024 at 00:38 UTC 3cb5769883fa104e42248f2b3f41a310947f357c
3 files changed +27 -25
vllm/lora/punica_wrapper/punica_base.py
+8 -6
@@ -63,7 +63,7 @@ class PunicaWrapperABC(ABC):
63 lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
64 output_slices: Tuple[int, ...],
65 offset_start: int = 0,
66 - add_input=True,
66 + add_inputs=True,
67 **kwargs,
68 ) -> None:
69 """
@@ -77,7 +77,7 @@ class PunicaWrapperABC(ABC):
77 y: torch.Tensor,
78 x: torch.Tensor,
79 lora_b_stacked: torch.Tensor,
80 - add_input: bool = True,
80 + add_inputs: bool = True,
81 **kwargs,
82 ) -> None:
83 """
@@ -367,12 +367,13 @@ class PunicaWrapperBase(PunicaWrapperABC):
367 lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
368 output_slices: Tuple[int, ...],
369 offset_start: int = 0,
370 - add_input=True,
370 + add_inputs=True,
371 **kwargs) -> None:
372 """
373 Performs GEMM and bias addition for multiple slices of lora_b.
374
375 Semantics:
376 + offset = offset_start
377 for i in range(len(lora_b_stacked)):
378 slice = output_slices[i]
379 y[:, offset:offset+slice] += x[i] @ lora_b_stacked[i] +
@@ -386,7 +387,8 @@ class PunicaWrapperBase(PunicaWrapperABC):
387 lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]):
388 bias's weight
389 output_slices (Tuple[int, ...]): Every slice's size
389 - add_input (bool): Defaults to True.
390 + offset_start (int): The starting position of y, defaults to 0
391 + add_inputs (bool): Defaults to True.
392
393 """
394 # TODO: implement it based on torch ops
@@ -397,7 +399,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
399 y: torch.Tensor,
400 x: torch.Tensor,
401 lora_b_stacked: torch.Tensor,
400 - add_input: bool = True,
402 + add_inputs: bool = True,
403 **kwargs) -> None:
404 """
405 Applies lora specifically for VocabParallelEmbeddingWithLoRA.
@@ -409,7 +411,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
411 y (torch.Tensor): Output tensor.
412 x (torch.Tensor): Input tensor.
413 lora_b_stacked (torch.Tensor): lora_b's weights.
412 - add_input (bool): Default to True.
414 + add_inputs (bool): Default to True.
415 """
416 # TODO: implement it based on torch ops
417 raise NotImplementedError
vllm/lora/punica_wrapper/punica_gpu.py
+17 -17
@@ -67,7 +67,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
67 y: torch.Tensor,
68 x: torch.Tensor,
69 w_t_all: torch.Tensor,
70 - add_input: bool,
70 + add_inputs: bool,
71 ):
72 #No LoRA request, so return directly
73 if self.no_lora:
@@ -77,7 +77,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
77 w_t_all,
78 y,
79 *self.prefill_metadata,
80 - add_input,
80 + add_inputs,
81 )
82
83 def _expand_decode(
@@ -85,9 +85,9 @@ class PunicaWrapperGPU(PunicaWrapperBase):
85 y: torch.Tensor,
86 x: torch.Tensor,
87 w_t_all: torch.Tensor,
88 - add_input: bool,
88 + add_inputs: bool,
89 ):
90 - bgmv_expand(x, w_t_all, y, self.token_lora_indices, add_input)
90 + bgmv_expand(x, w_t_all, y, self.token_lora_indices, add_inputs)
91
92 def _expand_slice_prefill(
93 self,
@@ -96,7 +96,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
96 w_t_all: torch.Tensor,
97 y_offset: Optional[int],
98 y_slice_size: Optional[int],
99 - add_input: bool,
99 + add_inputs: bool,
100 ):
101 #No LoRA request, so return directly
102 if self.no_lora:
@@ -108,7 +108,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
108 *self.prefill_metadata,
109 y_offset,
110 y_slice_size,
111 - add_input,
111 + add_inputs,
112 )
113
114 def _expand_slice_decode(
@@ -118,10 +118,10 @@ class PunicaWrapperGPU(PunicaWrapperBase):
118 w_t_all: torch.Tensor,
119 y_offset: Optional[int],
120 y_slice_size: Optional[int],
121 - add_input: bool,
121 + add_inputs: bool,
122 ):
123 bgmv_expand_slice(x, w_t_all, y, self.token_lora_indices, y_offset,
124 - y_slice_size, add_input)
124 + y_slice_size, add_inputs)
125
126 def _apply_expand(
127 self,
@@ -130,7 +130,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
130 w_t_all: torch.Tensor,
131 y_offset: Optional[int],
132 y_slice_size: Optional[int],
133 - add_input: bool = True,
133 + add_inputs: bool = True,
134 ):
135 """
136 Perform the ` y[:,y_offset:y_offset+y_slice_size]+=x@w_t_all`
@@ -141,7 +141,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
141 expand_slice_fun: Callable = (self._expand_slice_prefill
142 if self.is_prefill else
143 self._expand_slice_decode)
144 - expand_slice_fun(y, x, w_t_all, y_offset, y_slice_size, add_input)
144 + expand_slice_fun(y, x, w_t_all, y_offset, y_slice_size, add_inputs)
145
146 def _apply_shrink(self, y: torch.Tensor, x: torch.Tensor,
147 w_t_all: torch.Tensor, scale: float):
@@ -194,7 +194,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
194 lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
195 output_slices: Tuple[int, ...],
196 offset_start: int = 0,
197 - add_input=True,
197 + add_inputs=True,
198 **kwargs) -> None:
199 """
200 Performs GEMM and bias addition for multiple slices of lora_b.
@@ -213,7 +213,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
213 lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]):
214 bias's weight
215 output_slices (Tuple[int, ...]): Every slice's size
216 - add_input (bool): Defaults to True.
216 + add_inputs (bool): Defaults to True.
217 """
218 y_org = y
219 y = y.view(-1, y.shape[-1])
@@ -228,7 +228,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
228 lora_b_stacked[slice_idx],
229 offset_left,
230 output_slices[slice_idx],
231 - add_input=add_input,
231 + add_inputs=add_inputs,
232 )
233 offset_left += output_slices[slice_idx]
234 y = y.view_as(y_org)
@@ -237,7 +237,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
237 y: torch.Tensor,
238 x: torch.Tensor,
239 lora_b_stacked: torch.Tensor,
240 - add_input: bool = True,
240 + add_inputs: bool = True,
241 **kwargs) -> None:
242 """
243 Applies lora specifically for VocabParallelEmbeddingWithLoRA.
@@ -249,13 +249,13 @@ class PunicaWrapperGPU(PunicaWrapperBase):
249 y (torch.Tensor): Output tensor.
250 x (torch.Tensor): Input tensor.
251 lora_b_stacked (torch.Tensor): lora_b's weights.
252 - add_input (bool): Default to True.
252 + add_inputs (bool): Default to True.
253 """
254
255 # Embedding layer only need expand op
256 expand_fun: Callable = (self._expand_prefill
257 if self.is_prefill else self._expand_decode)
258 - expand_fun(y, x, lora_b_stacked, add_input)
258 + expand_fun(y, x, lora_b_stacked, add_inputs)
259
260 def add_lora_linear(self,
261 y: torch.Tensor,
@@ -311,7 +311,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
311 lora_b_stacked,
312 None,
313 output_slices,
314 - add_input=True,
314 + add_inputs=True,
315 **kwargs)
316
317 def add_lora_logits(self,
vllm/lora/punica_wrapper/punica_hpu.py
+2 -2
@@ -21,7 +21,7 @@ class PunicaWrapperHPU(PunicaWrapperBase):
21 y: torch.Tensor,
22 x: torch.Tensor,
23 lora_b_stacked: torch.Tensor,
24 - add_input: bool = True,
24 + add_inputs: bool = True,
25 **kwargs) -> None:
26 dispatch_bgmv_embedding(y, x, lora_b_stacked, 0)
27
@@ -81,7 +81,7 @@ class PunicaWrapperHPU(PunicaWrapperBase):
81 lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
82 output_slices: Tuple[int, ...],
83 offset_start: int = 0,
84 - add_input=True,
84 + add_inputs=True,
85 **kwargs,
86 ) -> None:
87 raise NotImplementedError