A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Kernel][Triton][AMD] Use block size heuristic for avg 2.8x speedup for int8 models (#11698)
Signed-off-by: Randall Smith <Randall.Smith@amd.com>
rasmith committed
Jan 8, 2025 at 14:23 UTC
526de822d501c792b051c864ba873a836d78d5bf
1 file changed
+16
-1
vllm/model_executor/layers/quantization/compressed_tensors/triton_scaled_mm.py
+16
-1
@@ -128,7 +128,8 @@ def triton_scaled_mm(input: torch.Tensor,
128
bias: Optional[torch.Tensor] = None,
129
block_size_m: int = 32,
130
block_size_n: int = 32,
131
- block_size_k: int = 32) -> torch.Tensor:
131
+ block_size_k: int = 32,
132
+ use_heuristic=True) -> torch.Tensor:
133
M, K = input.shape
134
N = weight.shape[1]
135
@@ -152,6 +153,20 @@ def triton_scaled_mm(input: torch.Tensor,
153
154
has_scalar = lambda x: x.shape[0] == 1 and x.shape[1] == 1
155
156
+ if use_heuristic:
157
+ is_small_N = N < 8192
158
+ next_power_of_2_M = max(32, triton.next_power_of_2(M))
159
+ if next_power_of_2_M <= 32:
160
+ tile_shape = (64, 64, 256) if is_small_N else (64, 128, 256)
161
+ elif next_power_of_2_M <= 64:
162
+ tile_shape = (64, 64, 256)
163
+ elif next_power_of_2_M <= 128:
164
+ tile_shape = (64, 128, 128)
165
+ else:
166
+ tile_shape = (128, 128, 128)
167
+
168
+ block_size_m, block_size_n, block_size_k = tile_shape
169
+
170
block_size_sa = 1 if has_scalar(scale_a) else block_size_m
171
block_size_sb = 1 if has_scalar(scale_b) else block_size_n
172