@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 526de822d5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Kernel][Triton][AMD] Use block size heuristic for avg 2.8x speedup for int8 models (#11698)

Signed-off-by: Randall Smith <Randall.Smith@amd.com>

rasmith committed Jan 8, 2025 at 14:23 UTC 526de822d501c792b051c864ba873a836d78d5bf
1 file changed +16 -1
vllm/model_executor/layers/quantization/compressed_tensors/triton_scaled_mm.py
+16 -1
@@ -128,7 +128,8 @@ def triton_scaled_mm(input: torch.Tensor,
128 bias: Optional[torch.Tensor] = None,
129 block_size_m: int = 32,
130 block_size_n: int = 32,
131 - block_size_k: int = 32) -> torch.Tensor:
131 + block_size_k: int = 32,
132 + use_heuristic=True) -> torch.Tensor:
133 M, K = input.shape
134 N = weight.shape[1]
135
@@ -152,6 +153,20 @@ def triton_scaled_mm(input: torch.Tensor,
153
154 has_scalar = lambda x: x.shape[0] == 1 and x.shape[1] == 1
155
156 + if use_heuristic:
157 + is_small_N = N < 8192
158 + next_power_of_2_M = max(32, triton.next_power_of_2(M))
159 + if next_power_of_2_M <= 32:
160 + tile_shape = (64, 64, 256) if is_small_N else (64, 128, 256)
161 + elif next_power_of_2_M <= 64:
162 + tile_shape = (64, 64, 256)
163 + elif next_power_of_2_M <= 128:
164 + tile_shape = (64, 128, 128)
165 + else:
166 + tile_shape = (128, 128, 128)
167 +
168 + block_size_m, block_size_n, block_size_k = tile_shape
169 +
170 block_size_sa = 1 if has_scalar(scale_a) else block_size_m
171 block_size_sb = 1 if has_scalar(scale_b) else block_size_n
172