A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Misc] Minor improvements to the readability of PunicaWrapperBase (#11200)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Dec 15, 2024 at 00:38 UTC
3cb5769883fa104e42248f2b3f41a310947f357c
3 files changed
+27
-25
vllm/lora/punica_wrapper/punica_base.py
+8
-6
@@ -63,7 +63,7 @@ class PunicaWrapperABC(ABC):
63
lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
64
output_slices: Tuple[int, ...],
65
offset_start: int = 0,
66
- add_input=True,
66
+ add_inputs=True,
67
**kwargs,
68
) -> None:
69
"""
@@ -77,7 +77,7 @@ class PunicaWrapperABC(ABC):
77
y: torch.Tensor,
78
x: torch.Tensor,
79
lora_b_stacked: torch.Tensor,
80
- add_input: bool = True,
80
+ add_inputs: bool = True,
81
**kwargs,
82
) -> None:
83
"""
@@ -367,12 +367,13 @@ class PunicaWrapperBase(PunicaWrapperABC):
367
lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
368
output_slices: Tuple[int, ...],
369
offset_start: int = 0,
370
- add_input=True,
370
+ add_inputs=True,
371
**kwargs) -> None:
372
"""
373
Performs GEMM and bias addition for multiple slices of lora_b.
374
375
Semantics:
376
+ offset = offset_start
377
for i in range(len(lora_b_stacked)):
378
slice = output_slices[i]
379
y[:, offset:offset+slice] += x[i] @ lora_b_stacked[i] +
@@ -386,7 +387,8 @@ class PunicaWrapperBase(PunicaWrapperABC):
387
lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]):
388
bias's weight
389
output_slices (Tuple[int, ...]): Every slice's size
389
- add_input (bool): Defaults to True.
390
+ offset_start (int): The starting position of y, defaults to 0
391
+ add_inputs (bool): Defaults to True.
392
393
"""
394
# TODO: implement it based on torch ops
@@ -397,7 +399,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
399
y: torch.Tensor,
400
x: torch.Tensor,
401
lora_b_stacked: torch.Tensor,
400
- add_input: bool = True,
402
+ add_inputs: bool = True,
403
**kwargs) -> None:
404
"""
405
Applies lora specifically for VocabParallelEmbeddingWithLoRA.
@@ -409,7 +411,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
411
y (torch.Tensor): Output tensor.
412
x (torch.Tensor): Input tensor.
413
lora_b_stacked (torch.Tensor): lora_b's weights.
412
- add_input (bool): Default to True.
414
+ add_inputs (bool): Default to True.
415
"""
416
# TODO: implement it based on torch ops
417
raise NotImplementedError
vllm/lora/punica_wrapper/punica_gpu.py
+17
-17
@@ -67,7 +67,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
67
y: torch.Tensor,
68
x: torch.Tensor,
69
w_t_all: torch.Tensor,
70
- add_input: bool,
70
+ add_inputs: bool,
71
):
72
#No LoRA request, so return directly
73
if self.no_lora:
@@ -77,7 +77,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
77
w_t_all,
78
y,
79
*self.prefill_metadata,
80
- add_input,
80
+ add_inputs,
81
)
82
83
def _expand_decode(
@@ -85,9 +85,9 @@ class PunicaWrapperGPU(PunicaWrapperBase):
85
y: torch.Tensor,
86
x: torch.Tensor,
87
w_t_all: torch.Tensor,
88
- add_input: bool,
88
+ add_inputs: bool,
89
):
90
- bgmv_expand(x, w_t_all, y, self.token_lora_indices, add_input)
90
+ bgmv_expand(x, w_t_all, y, self.token_lora_indices, add_inputs)
91
92
def _expand_slice_prefill(
93
self,
@@ -96,7 +96,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
96
w_t_all: torch.Tensor,
97
y_offset: Optional[int],
98
y_slice_size: Optional[int],
99
- add_input: bool,
99
+ add_inputs: bool,
100
):
101
#No LoRA request, so return directly
102
if self.no_lora:
@@ -108,7 +108,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
108
*self.prefill_metadata,
109
y_offset,
110
y_slice_size,
111
- add_input,
111
+ add_inputs,
112
)
113
114
def _expand_slice_decode(
@@ -118,10 +118,10 @@ class PunicaWrapperGPU(PunicaWrapperBase):
118
w_t_all: torch.Tensor,
119
y_offset: Optional[int],
120
y_slice_size: Optional[int],
121
- add_input: bool,
121
+ add_inputs: bool,
122
):
123
bgmv_expand_slice(x, w_t_all, y, self.token_lora_indices, y_offset,
124
- y_slice_size, add_input)
124
+ y_slice_size, add_inputs)
125
126
def _apply_expand(
127
self,
@@ -130,7 +130,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
130
w_t_all: torch.Tensor,
131
y_offset: Optional[int],
132
y_slice_size: Optional[int],
133
- add_input: bool = True,
133
+ add_inputs: bool = True,
134
):
135
"""
136
Perform the ` y[:,y_offset:y_offset+y_slice_size]+=x@w_t_all`
@@ -141,7 +141,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
141
expand_slice_fun: Callable = (self._expand_slice_prefill
142
if self.is_prefill else
143
self._expand_slice_decode)
144
- expand_slice_fun(y, x, w_t_all, y_offset, y_slice_size, add_input)
144
+ expand_slice_fun(y, x, w_t_all, y_offset, y_slice_size, add_inputs)
145
146
def _apply_shrink(self, y: torch.Tensor, x: torch.Tensor,
147
w_t_all: torch.Tensor, scale: float):
@@ -194,7 +194,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
194
lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
195
output_slices: Tuple[int, ...],
196
offset_start: int = 0,
197
- add_input=True,
197
+ add_inputs=True,
198
**kwargs) -> None:
199
"""
200
Performs GEMM and bias addition for multiple slices of lora_b.
@@ -213,7 +213,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
213
lora_bias_stacked (Optional[Tuple[torch.Tensor, ...]]):
214
bias's weight
215
output_slices (Tuple[int, ...]): Every slice's size
216
- add_input (bool): Defaults to True.
216
+ add_inputs (bool): Defaults to True.
217
"""
218
y_org = y
219
y = y.view(-1, y.shape[-1])
@@ -228,7 +228,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
228
lora_b_stacked[slice_idx],
229
offset_left,
230
output_slices[slice_idx],
231
- add_input=add_input,
231
+ add_inputs=add_inputs,
232
)
233
offset_left += output_slices[slice_idx]
234
y = y.view_as(y_org)
@@ -237,7 +237,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
237
y: torch.Tensor,
238
x: torch.Tensor,
239
lora_b_stacked: torch.Tensor,
240
- add_input: bool = True,
240
+ add_inputs: bool = True,
241
**kwargs) -> None:
242
"""
243
Applies lora specifically for VocabParallelEmbeddingWithLoRA.
@@ -249,13 +249,13 @@ class PunicaWrapperGPU(PunicaWrapperBase):
249
y (torch.Tensor): Output tensor.
250
x (torch.Tensor): Input tensor.
251
lora_b_stacked (torch.Tensor): lora_b's weights.
252
- add_input (bool): Default to True.
252
+ add_inputs (bool): Default to True.
253
"""
254
255
# Embedding layer only need expand op
256
expand_fun: Callable = (self._expand_prefill
257
if self.is_prefill else self._expand_decode)
258
- expand_fun(y, x, lora_b_stacked, add_input)
258
+ expand_fun(y, x, lora_b_stacked, add_inputs)
259
260
def add_lora_linear(self,
261
y: torch.Tensor,
@@ -311,7 +311,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
311
lora_b_stacked,
312
None,
313
output_slices,
314
- add_input=True,
314
+ add_inputs=True,
315
**kwargs)
316
317
def add_lora_logits(self,
vllm/lora/punica_wrapper/punica_hpu.py
+2
-2
@@ -21,7 +21,7 @@ class PunicaWrapperHPU(PunicaWrapperBase):
21
y: torch.Tensor,
22
x: torch.Tensor,
23
lora_b_stacked: torch.Tensor,
24
- add_input: bool = True,
24
+ add_inputs: bool = True,
25
**kwargs) -> None:
26
dispatch_bgmv_embedding(y, x, lora_b_stacked, 0)
27
@@ -81,7 +81,7 @@ class PunicaWrapperHPU(PunicaWrapperBase):
81
lora_bias_stacked: Optional[Tuple[torch.Tensor, ...]],
82
output_slices: Tuple[int, ...],
83
offset_start: int = 0,
84
- add_input=True,
84
+ add_inputs=True,
85
**kwargs,
86
) -> None:
87
raise NotImplementedError