@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / baeded2569
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Attention] Deepseek v3 MLA support with FP8 compute (#12601)

This PR implements the Deepseek V3 support by performing matrix absorption the fp8 weights --------- Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Co-authored-by: simon-mo <simon.mo@hey.com> Co-authored-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Zhuohan Li <zhuohan123@gmail.com> Co-authored-by: Tyler Michael Smith <tysmith@redhat.com> Co-authored-by: Alexander Matveev <59768536+alexm-neuralmagic@users.noreply.github.com>

Lucas Wilkinson committed Feb 1, 2025 at 00:52 UTC baeded25699f9f4851843306f27f685c4d4ee7c5
10 files changed +580 -85
vllm/attention/backends/mla/utils.py
+184 -36
@@ -1,17 +1,29 @@
1 from abc import abstractmethod
2 from dataclasses import dataclass
3 -from typing import Any, Dict, Generic, List, Optional
3 +from typing import Any, Dict, Generic, List, Optional, Tuple
4
5 import torch
6 +from compressed_tensors.quantization import QuantizationStrategy
7
8 from vllm import _custom_ops as ops
9 from vllm import envs
10 from vllm.attention.backends.abstract import (AttentionLayer,
11 AttentionMetadata,
12 MLAAttentionImpl, T)
12 -from vllm.distributed import get_tensor_model_parallel_world_size
13 +from vllm.distributed import (get_tensor_model_parallel_world_size,
14 + tensor_model_parallel_all_reduce)
15 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
14 - RowParallelLinear)
16 + LinearBase, RowParallelLinear,
17 + UnquantizedLinearMethod)
18 +from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
19 + CompressedTensorsLinearMethod)
20 +from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
21 + CompressedTensorsW8A8Fp8)
22 +from vllm.model_executor.layers.quantization.fp8 import Fp8LinearMethod
23 +from vllm.model_executor.layers.quantization.utils.fp8_utils import (
24 + apply_fp8_linear_generic, current_platform_fp8_dtype, is_fp8)
25 +from vllm.model_executor.layers.quantization.utils.quant_utils import (
26 + scaled_dequantize, scaled_quantize)
27 from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
28 from vllm.vllm_flash_attn import flash_attn_varlen_func
29
@@ -25,11 +37,11 @@ class MLACommonMetadata(AttentionMetadata):
37
38 class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
39 """
28 - Common class for implementing repeated parts
29 -
40 + Common class for implementing repeated parts
41 +
42 Main reference: DeepseekV2 paper, and FlashInfer Implementation
43 (https://arxiv.org/abs/2405.04434 and https://github.com/flashinfer-ai/flashinfer/pull/551).
32 -
44 +
45 Deepseek's MLA attention works the following way:
46 * Use a single latent vector to represent the entire KV cache.
47 * The attention "simulates" a multi-head attention, while the compute is
@@ -46,7 +58,7 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
58 * V: V head dim.
59 * kv_c: latent/compressed KV
60 * q_c: latent/compressed Q
49 -
61 +
62 #
63 # Outside the MLA attention backend
64 #
@@ -55,21 +67,21 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
67 kv_c_k_pe (B, Lkv+R).
68 2. The kv_c_k_pe is split into kv_c (B, Lkv) and k_pe (B, R). cq
69 and kv_c are normalized.
58 -
70 +
71 #
72 # Inside the MLA attention backend
73 #
74
75 * if prefill:
64 -
65 - 3. The q_c is then projected up into the multi-head version.
66 - * q_c goes from (B, Lq) to (B, N, (P+R)), which is split into q_nope
67 - (B, N, P) and q_pe (B, N, R).
76 +
77 + 3. The q_c is then projected up into the multi-head version.
78 + * q_c goes from (B, Lq) to (B, N, (P+R)), which is split into q_nope
79 + (B, N, P) and q_pe (B, N, R).
80 4. q_pe, k_pe are then passed through rotary embeddings.
81 5. kv_c and k_pe are concatenated and inserted into the cache
70 - 6. The kv_c is then projected up into the multi-head version.
71 - * kv_c goes from (B, Lkv) to (B, N, (P+V)) which has the nope
72 - dimensions for K and V, which is split into k_nope (B, N, P)
82 + 6. The kv_c is then projected up into the multi-head version.
83 + * kv_c goes from (B, Lkv) to (B, N, (P+V)) which has the nope
84 + dimensions for K and V, which is split into k_nope (B, N, P)
85 and v (B, N, V).
86 7. q (B, N, (P+R)) and k (B, N, (P+R)) matrices are assembled from
87 q_nope, q_pe, k_nope, k_pe.
@@ -112,7 +124,7 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
124 From @tsu-bin's calculation, we only want to use the absorption technique
125 for decode. The prefill algorithm should still use the up-projected MHA
126 for less flops and memory usage.
115 -
127 +
128 """
129
130 def __init__(
@@ -162,8 +174,19 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
174
175 def _v_up_proj_and_o_proj(self, x):
176 if envs.VLLM_MLA_PERFORM_MATRIX_ABSORPTION:
165 - return self.o_proj_absorbed(
166 - x.reshape(-1, self.num_heads * self.kv_lora_rank))[0]
177 + if is_fp8(self.W_UV_O):
178 + output_parallel = apply_fp8_linear_generic(
179 + x.flatten(start_dim=1), self.W_UV_O, self.W_UV_O_scales,
180 + self.reqaunt_input_group_shape,
181 + self.reqaunt_weight_group_shape)
182 + else:
183 + output_parallel = torch.matmul(x.flatten(start_dim=1),
184 + self.W_UV_O)
185 + if self.tp_size > 1:
186 + output = tensor_model_parallel_all_reduce(output_parallel)
187 + else:
188 + output = output_parallel
189 + return output
190 else:
191 x = torch.einsum("bnl,lnv->bnv", x, self.W_UV)
192 return self.o_proj(x.reshape(-1,
@@ -171,6 +194,12 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
194
195 def _q_proj_and_k_up_proj(self, x):
196 if envs.VLLM_MLA_PERFORM_MATRIX_ABSORPTION:
197 + if is_fp8(self.W_Q_UK):
198 + return apply_fp8_linear_generic(
199 + x, self.W_Q_UK, self.W_Q_UK_scales,
200 + self.reqaunt_input_group_shape,
201 + self.reqaunt_weight_group_shape).view(
202 + -1, self.num_heads, self.kv_lora_rank)
203 return torch.matmul(x, self.W_Q_UK)\
204 .view(-1, self.num_heads, self.kv_lora_rank)
205 else:
@@ -179,8 +208,91 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
208 return torch.einsum("bnp,lnp->bnl", x, self.W_UK)\
209 .view(-1, self.num_heads, self.kv_lora_rank)
210
182 - def process_weights_after_loading(self):
183 - kv_b_proj_weight = self.kv_b_proj.weight.T
211 + def process_weights_after_loading(self, act_dtype: torch.dtype):
212 +
213 + def is_layer_fp8(layer: LinearBase) -> bool:
214 + return isinstance(layer.quant_method, Fp8LinearMethod) or\
215 + (isinstance(layer.quant_method, CompressedTensorsLinearMethod)\
216 + and isinstance(layer.scheme, CompressedTensorsW8A8Fp8))
217 +
218 + def quantization_scheme_supported(layer: LinearBase) -> bool:
219 + return isinstance(layer.quant_method, UnquantizedLinearMethod) or \
220 + is_layer_fp8(layer)
221 +
222 + # TODO(lucas) This is very gross, we need a more wide scale refactor of
223 + # all the FP8 code with a more standard way of
224 + # defining schemes/group-shapes, we should also potentially force
225 + # quant_methods to support a decompress function
226 + #
227 + # returns input_group_shape, weight_group_shape
228 + def get_scale_group_shapes_for_fp8(layer: LinearBase) -> \
229 + Tuple[Tuple[int, int], Tuple[int, int]]:
230 + if isinstance(layer.quant_method, Fp8LinearMethod):
231 + if layer.quant_method.block_quant is not None:
232 + weight_block_size = \
233 + layer.quant_method.quant_config.weight_block_size
234 + # per-token-group (1, X), block-quantized (X, Y)
235 + return (1, weight_block_size[-1]), weight_block_size
236 + else:
237 + return (-1, -1), (-1, -1) # per-tensor, per-tensor
238 + elif isinstance(layer.quant_method, CompressedTensorsLinearMethod)\
239 + and isinstance(layer.scheme, CompressedTensorsW8A8Fp8):
240 + # this is hacky but we always assume the for
241 + # CompressedTensorsW8A8Fp8 the input is dynamic per-token
242 + # we ignore if it is static-per-tensor since we are going to
243 + # requantize after later anyways
244 + strategy = layer.scheme.strategy
245 + if strategy == QuantizationStrategy.TENSOR:
246 + return (1, -1), (-1, -1) # per-token, per-tensor
247 + elif strategy == QuantizationStrategy.CHANNEL:
248 + return (1, -1), (-1, 1) # per-token, per-channel
249 + else:
250 + raise NotImplementedError(
251 + f"QuantizationStrategy.{strategy} is not supported for "
252 + "fp8 MLA, please run with VLLM_MLA_DISABLE=1")
253 + else:
254 + raise NotImplementedError(
255 + "Can't determine scale group shapes for "
256 + f"{layer.quant_method}, please run with VLLM_MLA_DISABLE=1"
257 + )
258 +
259 + def get_scales(layer: LinearBase) -> torch.Tensor:
260 + if hasattr(layer, "weight_scale_inv"):
261 + return layer.weight_scale_inv
262 + return layer.weight_scale
263 +
264 + def get_and_maybe_dequant_weights(layer: LinearBase):
265 + if is_layer_fp8(layer):
266 + if isinstance(layer.quant_method, \
267 + CompressedTensorsLinearMethod) and \
268 + isinstance(layer.scheme, CompressedTensorsW8A8Fp8):
269 + # NOTE(lucas): note sure why but `CompressedTensorsW8A8Fp8`
270 + # seems to store weights as (input, output) instead of
271 + # (output, input) so we need to transpose
272 + weight = layer.weight.T # standardize to (output, input)
273 + else:
274 + weight = layer.weight
275 + _, weight_scale_group_shape = \
276 + get_scale_group_shapes_for_fp8(layer)
277 + scales = get_scales(layer)
278 +
279 + return scaled_dequantize(weight, scales,
280 + weight_scale_group_shape)
281 + else:
282 + return layer.weight
283 +
284 + if not (quantization_scheme_supported(self.kv_b_proj) and\
285 + quantization_scheme_supported(self.q_proj) and\
286 + quantization_scheme_supported(self.o_proj)):
287 + raise NotImplementedError(
288 + "Only FP8 and UnquantizedLinearMethod are supported for MLA"
289 + ", please run with VLLM_MLA_DISABLE=1")
290 +
291 + weight_dtype = self.kv_b_proj.weight.dtype
292 + assert self.o_proj.weight.dtype == weight_dtype
293 + assert self.q_proj.weight.dtype == weight_dtype
294 +
295 + kv_b_proj_weight = get_and_maybe_dequant_weights(self.kv_b_proj).T
296 assert kv_b_proj_weight.shape == (
297 self.kv_lora_rank,
298 self.num_heads * (self.qk_nope_head_dim + self.v_head_dim)), (
@@ -198,18 +310,35 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
310 W_UK, W_UV = kv_b_proj_weight.split(
311 [self.qk_nope_head_dim, self.v_head_dim], dim=-1)
312
201 - q_proj = self.q_proj.weight.T\
313 + q_proj_weight = get_and_maybe_dequant_weights(self.q_proj).T\
314 .view(-1, self.num_heads, self.qk_head_dim)
315
316 # can be W_Q or W_UQ depending q_lora_rank, the former if
317 # q_lora_rank is None, the latter otherwise. From the Attention backend
318 # perspective though we call these both W_Q and rely on the layer
319 # to pass in the correct matrix
208 - W_Q = q_proj[..., :self.qk_nope_head_dim]
209 - self.W_QR = q_proj[..., self.qk_nope_head_dim:]\
320 + W_Q = q_proj_weight[..., :self.qk_nope_head_dim]
321 + self.W_QR = q_proj_weight[..., self.qk_nope_head_dim:]\
322 .flatten(start_dim=1).contiguous()
323
324 + # W_QR is small so for simplicity we dont bother requantizing it
325 + self.W_QR = self.W_QR.to(act_dtype)
326 +
327 if envs.VLLM_MLA_PERFORM_MATRIX_ABSORPTION:
328 + requantization_enabled = not envs.VLLM_MLA_DISABLE_REQUANTIZATION
329 + if is_fp8(weight_dtype) and requantization_enabled:
330 + # This assumes it wise to requantize using the same group shapes
331 + # (i.e. strategy, per-tensor, per-channel, block etc.) that the
332 + # weights were originally quantized
333 + requant_input_group_shape, requant_weight_group_shape = \
334 + get_scale_group_shapes_for_fp8(self.q_proj)
335 + assert (requant_input_group_shape, requant_weight_group_shape)\
336 + == get_scale_group_shapes_for_fp8(self.kv_b_proj)
337 + assert (requant_input_group_shape, requant_weight_group_shape)\
338 + == get_scale_group_shapes_for_fp8(self.o_proj)
339 + self.reqaunt_input_group_shape = requant_input_group_shape
340 + self.reqaunt_weight_group_shape = requant_weight_group_shape
341 +
342 #
343 # Perform matrix-absorption following
344 # https://github.com/flashinfer-ai/flashinfer/pull/551
@@ -223,25 +352,44 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
352 # latter otherwise
353 # basically if q_lora_rank is none we are absorbing into q_proj
354 # instead of UQ
226 - self.W_Q_UK = torch.einsum("qnd,lnd -> qnl", W_Q, W_UK)\
355 + W_Q_UK = torch.einsum("qnd,lnd -> qnl", W_Q, W_UK)\
356 .flatten(start_dim=1).contiguous()
357
229 - W_O = self.o_proj.weight\
358 + if is_fp8(weight_dtype) and requantization_enabled:
359 + W_Q_UK, W_Q_UK_scales = scaled_quantize(
360 + W_Q_UK,
361 + self.reqaunt_weight_group_shape,
362 + quant_dtype=current_platform_fp8_dtype)
363 + # For FP8 save the transpose so we can use
364 + # `apply_w8a8_block_fp8_linear` directly
365 + self.W_Q_UK = W_Q_UK.T.contiguous()
366 + self.W_Q_UK_scales = W_Q_UK_scales.T.contiguous()
367 + else:
368 + self.W_Q_UK = W_Q_UK.to(act_dtype)
369 +
370 + W_O = get_and_maybe_dequant_weights(self.o_proj)\
371 .view(-1, self.num_heads, self.v_head_dim)
231 - self.W_UV_O = torch.einsum("lnd,hnd -> nlh", W_UV, W_O)\
372 + W_UV_O = torch.einsum("lnd,hnd -> nlh", W_UV, W_O)\
373 .flatten(start_dim=0, end_dim=1).contiguous()
374
234 - tp_size = get_tensor_model_parallel_world_size()
235 - self.o_proj_absorbed = RowParallelLinear(
236 - self.W_UV_O.shape[0] * tp_size,
237 - self.W_UV_O.shape[1],
238 - bias=False,
239 - # TODO(lucas) figure out how to properly forward quant_method
240 - #quant_config=self.o_proj.quant_method,
241 - )
242 -
243 - self.o_proj_absorbed.weight = torch.nn.Parameter(self.W_UV_O.T)
375 + if is_fp8(weight_dtype) and requantization_enabled:
376 + W_UV_O, W_UV_O_scales = scaled_quantize(
377 + W_UV_O,
378 + self.reqaunt_weight_group_shape,
379 + quant_dtype=current_platform_fp8_dtype)
380 + # For FP8 save the transpose so we can use
381 + # `apply_w8a8_block_fp8_linear` directly
382 + self.W_UV_O = W_UV_O.T.contiguous()
383 + self.W_UV_O_scales = W_UV_O_scales.T.contiguous()
384 + else:
385 + self.W_UV_O = W_UV_O.to(act_dtype)
386 +
387 + self.tp_size = get_tensor_model_parallel_world_size()
388 else:
389 + if is_fp8(weight_dtype):
390 + raise NotImplementedError(
391 + "Currently fp8 requires matrix absorption")
392 +
393 self.W_UV = W_UV
394 self.W_UK = W_UK
395 self.W_Q = W_Q.flatten(start_dim=1)
vllm/attention/backends/triton_mla.py
+7 -11
@@ -57,14 +57,12 @@ class TritonMLABackend(AttentionBackend):
57
58 @staticmethod
59 def get_kv_cache_shape(
60 - num_blocks: int,
61 - block_size: int,
62 - num_kv_heads: int, # assumed to be 1 for MLA
63 - kv_lora_rank: int, # passed via head_size
60 + num_blocks: int,
61 + block_size: int,
62 + num_kv_heads: int, # assumed to be 1 for MLA
63 + head_size: int,
64 ) -> Tuple[int, ...]:
65 - # TODO(lucas): remove hardcoding k_pe size as 1/8th of kv_lora_rank
66 - k_pe_size = kv_lora_rank // 8
67 - return (num_blocks, block_size, kv_lora_rank + k_pe_size)
65 + return (num_blocks, block_size, head_size)
66
67 @staticmethod
68 def swap_blocks(
@@ -83,7 +81,7 @@ class TritonMLABackend(AttentionBackend):
81
82 @staticmethod
83 def get_supported_head_sizes() -> List[int]:
86 - return [512]
84 + return [576]
85
86
87 class TritonMLAState(AttentionState):
@@ -624,8 +622,6 @@ class TritonMLAMetadataBuilder(AttentionMetadataBuilder[TritonMLAMetadata]):
622 self.multimodal_placeholder_maps.items()
623 }
624
627 - num_kv_splits = 8
628 -
625 return TritonMLAMetadata(
626 num_prefills=self.num_prefills,
627 slot_mapping=slot_mapping_tensor,
@@ -645,7 +641,7 @@ class TritonMLAMetadataBuilder(AttentionMetadataBuilder[TritonMLAMetadata]):
641 context_lens_tensor=context_lens_tensor,
642 block_tables=block_tables,
643 use_cuda_graph=use_captured_graph,
648 - num_kv_splits=num_kv_splits,
644 + num_kv_splits=4, # TODO(lucas) add heuristic
645 head_dim=self.runner.model_config.get_head_size(),
646 )
647
vllm/attention/layer.py
+2 -2
@@ -200,9 +200,9 @@ class Attention(nn.Module):
200 s += f", backend={self.impl.__class__.__name__}"
201 return s
202
203 - def process_weights_after_loading(self):
203 + def process_weights_after_loading(self, act_dtype: torch.dtype):
204 if hasattr(self.impl, "process_weights_after_loading"):
205 - self.impl.process_weights_after_loading()
205 + self.impl.process_weights_after_loading(act_dtype)
206
207
208 class MultiHeadAttention(nn.Module):
vllm/config.py
+33 -6
@@ -739,18 +739,19 @@ class ModelConfig:
739 @property
740 def is_deepseek_mla(self) -> bool:
741 # TODO add deepseek_v3
742 - return hasattr(self.hf_text_config,
743 - "model_type") and (self.hf_text_config.model_type
744 - in ('deepseek_v2'))
742 + return (hasattr(self.hf_text_config, "model_type")) \
743 + and (self.hf_text_config.model_type in \
744 + ('deepseek_v2', 'deepseek_v3'))\
745 + and (self.hf_text_config.kv_lora_rank is not None)
746
747 def get_head_size(self) -> int:
748 # TODO remove hard code
749 if self.is_deepseek_mla:
750 + qk_rope_head_dim = getattr(self.hf_text_config, "qk_rope_head_dim",
751 + 0)
752 if self.use_mla:
750 - return self.hf_text_config.kv_lora_rank
753 + return self.hf_text_config.kv_lora_rank + qk_rope_head_dim
754 else:
752 - qk_rope_head_dim = getattr(self.hf_text_config,
753 - "qk_rope_head_dim", 0)
755 qk_nope_head_dim = getattr(self.hf_text_config,
756 "qk_nope_head_dim", 0)
757 if qk_rope_head_dim and qk_nope_head_dim:
@@ -969,6 +970,32 @@ class ModelConfig:
970
971 @property
972 def use_mla(self) -> bool:
973 + if self.quantization is not None and self.quantization not in [\
974 + "fp8", "compressed-tensors"]:
975 + logger.warning(
976 + "MLA is not supported with %s quantization. "
977 + "Disabling MLA.", self.quantization)
978 + return False
979 +
980 + # If using a "compressed-tensors" checkpoint, check that all groups
981 + # have fp8 for both weights and activations.
982 + if self.quantization == "compressed-tensors":
983 + quant_config = self._parse_quant_hf_config()
984 + for group_name, cfg in quant_config.get("config_groups",
985 + ("", {})).items():
986 + act_cfg = cfg.get("input_activations", {})
987 + act_type = None if act_cfg is None else act_cfg.get("type", "")
988 + w_cfg = cfg.get("weights", {})
989 + w_type = None if w_cfg is None else w_cfg.get("type", "")
990 + if act_type != "fp8" or w_type != "fp8":
991 + logger.warning(
992 + "compressed-tensors MLA support requires fp8 "
993 + "activations and weights in group '%s', but got "
994 + "activations type '%s' and weights type '%s'.\n "
995 + "Full config: %s", group_name, act_type, w_type,
996 + quant_config)
997 + return False
998 +
999 use_mla = (self.is_deepseek_mla and not envs.VLLM_MLA_DISABLE)
1000 return use_mla
1001
vllm/envs.py
+11 -1
@@ -79,6 +79,7 @@ if TYPE_CHECKING:
79 VLLM_V1_OUTPUT_PROC_CHUNK_SIZE: int = 128
80 VLLM_MLA_DISABLE: bool = False
81 VLLM_MLA_PERFORM_MATRIX_ABSORPTION: bool = True
82 + VLLM_MLA_DISABLE_REQUANTIZATION: bool = False
83
84
85 def get_default_cache_root():
@@ -519,7 +520,16 @@ environment_variables: Dict[str, Callable[[], Any]] = {
520 # storing more weights, W_Q_UK and W_UV_O, so can increase memory usage,
521 # the is enabled by default
522 "VLLM_MLA_PERFORM_MATRIX_ABSORPTION":
522 - lambda: bool(int(os.getenv("VLLM_MLA_PERFORM_MATRIX_ABSORPTION", "1")))
523 + lambda: bool(int(os.getenv("VLLM_MLA_PERFORM_MATRIX_ABSORPTION", "1"))),
524 +
525 + # When running MLA with matrix-absorption enabled and fp8 quantized weights
526 + # we perform the matrix-absorption in float32 precision, after the matrices
527 + # are absorbed we requantize the weights back to fp8, this flag can be used
528 + # to disable the requantization step, and instead convert the absorbed
529 + # matrices to match the activation type. This can lead to higher memory and
530 + # compute usage but better preserves the accuracy of the original model.
531 + "VLLM_MLA_DISABLE_REQUANTIZATION":
532 + lambda: bool(int(os.getenv("VLLM_MLA_DISABLE_REQUANTIZATION", "0")))
533 }
534
535 # end-env-vars-definition
vllm/model_executor/layers/quantization/utils/fp8_utils.py
+52 -22
@@ -2,7 +2,7 @@
2 import functools
3 import json
4 import os
5 -from typing import Any, Dict, List, Optional, Tuple
5 +from typing import Any, Dict, List, Optional, Tuple, Union
6
7 import torch
8 import triton
@@ -10,10 +10,24 @@ import triton.language as tl
10
11 from vllm import _custom_ops as ops
12 from vllm.logger import init_logger
13 +from vllm.model_executor.layers.quantization.utils.quant_utils import (
14 + _normalize_quant_group_shape, scaled_dequantize)
15 +from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
16 + apply_fp8_linear)
17 from vllm.platforms import current_platform
18
19 logger = init_logger(__name__)
20
21 +current_platform_fp8_dtype = (torch.float8_e4m3fnuz
22 + if current_platform.is_rocm() else
23 + torch.float8_e4m3fn)
24 +
25 +
26 +def is_fp8(x: Union[torch.dtype, torch.Tensor]) -> bool:
27 + if isinstance(x, torch.Tensor):
28 + x = x.dtype
29 + return x == torch.float8_e4m3fn or x == torch.float8_e4m3fnuz
30 +
31
32 def apply_w8a8_block_fp8_linear(
33 input: torch.Tensor,
@@ -55,6 +69,42 @@ def apply_w8a8_block_fp8_linear(
69 return output.to(dtype=input.dtype).view(*output_shape)
70
71
72 +# Unify the interface between `apply_w8a8_block_fp8_linear` and
73 +# `apply_fp8_linear`
74 +# NOTE(lucas): this is quite messy, we should think through this more formally
75 +def apply_fp8_linear_generic(
76 + input: torch.Tensor,
77 + weight: torch.Tensor,
78 + weight_scale: torch.Tensor,
79 + input_group_shape: Tuple[int, int],
80 + weight_group_shape: Tuple[int, int],
81 + input_scale: Optional[torch.Tensor] = None, # static scale if one
82 +) -> torch.Tensor:
83 + # View input as 2D matrix for fp8 methods
84 + input = input.view(-1, input.shape[-1])
85 +
86 + weight_group_shape = _normalize_quant_group_shape(\
87 + weight, weight_group_shape)
88 + input_group_shape = _normalize_quant_group_shape(input, input_group_shape)
89 +
90 + def is_dim_blocked(dim, shape, group_shape):
91 + return group_shape < shape[dim] and group_shape > 1
92 +
93 + if is_dim_blocked(0, weight.shape, weight_group_shape[0])\
94 + and is_dim_blocked(1, weight.shape, weight_group_shape[1]) and\
95 + input_group_shape == (1, weight_group_shape[1]):
96 + return apply_w8a8_block_fp8_linear(input, weight,
97 + list(weight_group_shape),
98 + weight_scale)
99 + else:
100 + # Despite having linear in the it doesn't conform to
101 + # `torch.nn.functional.linear` which is defined as `input @ weight.T`
102 + # so we explicitly transpose the weight matrix here
103 + return apply_fp8_linear(input, weight.T, weight_scale.T,
104 + use_per_token_if_dynamic=\
105 + (input_group_shape == (1, input.shape[1])))
106 +
107 +
108 def input_to_float8(
109 x: torch.Tensor,
110 dtype: Optional[torch.dtype] = None
@@ -75,7 +125,6 @@ def input_to_float8(
125 def block_quant_to_tensor_quant(
126 x_q_block: torch.Tensor,
127 x_s: torch.Tensor,
78 - block_size: List[int],
128 ) -> Tuple[torch.Tensor, torch.Tensor]:
129 """This function converts block-wise quantization to tensor-wise
130 quantization. The inputs are block-wise quantization tensor `x_q_block`,
@@ -83,26 +132,7 @@ def block_quant_to_tensor_quant(
132 The outputs are tensor-wise quantization tensor and tensor-wise
133 quantization scale. Note only float8 is supported for now.
134 """
86 - block_n, block_k = block_size[0], block_size[1]
87 - n, k = x_q_block.shape
88 - n_tiles = (n + block_n - 1) // block_n
89 - k_tiles = (k + block_k - 1) // block_k
90 - assert n_tiles == x_s.shape[0]
91 - assert k_tiles == x_s.shape[1]
92 -
93 - x_dq_block = x_q_block.to(torch.float32)
94 -
95 - x_dq_block_tiles = [[
96 - x_dq_block[
97 - j * block_n:min((j + 1) * block_n, n),
98 - i * block_k:min((i + 1) * block_k, k),
99 - ] for i in range(k_tiles)
100 - ] for j in range(n_tiles)]
101 -
102 - for i in range(k_tiles):
103 - for j in range(n_tiles):
104 - x_dq_block_tiles[j][i][:, :] = x_dq_block_tiles[j][i] * x_s[j][i]
105 -
135 + x_dq_block = scaled_dequantize(x_q_block, x_s)
136 x_q_tensor, scale = input_to_float8(x_dq_block, dtype=x_q_block.dtype)
137 return x_q_tensor, scale
138
vllm/model_executor/layers/quantization/utils/quant_utils.py
+115 -1
@@ -1,5 +1,5 @@
1 """This file is used for /tests and /benchmarks"""
2 -from typing import List, Optional
2 +from typing import List, Optional, Tuple
3
4 import numpy
5 import torch
@@ -20,6 +20,120 @@ FUSED_LAYER_NAME_MAPPING = {
20 }
21
22
23 +# Normalize the group_shape to the full extent for any dims that are -1
24 +def _normalize_quant_group_shape(x: torch.Tensor, group_shape: Tuple[int,
25 + int]):
26 + # -1 means full extent
27 + return (group_shape[0] if group_shape[0] > 0 else x.shape[-2],
28 + group_shape[1] if group_shape[1] > 0 else x.shape[-1])
29 +
30 +
31 +# Useful when treating N-dimensional group scaling as extended numpy-style
32 +# broadcasting in numpy simply stretches dimensions with an extent of 1 to match
33 +# the target shape by repeating the data along that dimension (broadcasting)
34 +# , we extend these semantics to say if the extent of a dimension in the
35 +# source shape is not 1 and does not match the target shape we repeat each
36 +# element along that dimension src_shape[dim] // target_shape[dim] times
37 +# example if we have:
38 +# a = [[1, 2], and target_shape = (2, 4)
39 +# [3, 4]]
40 +# then we would expand a to:
41 +# a = [[1, 1, 2, 2],
42 +# [3, 3, 4, 4]]
43 +# NOTE this function this function does not explicitly broadcast dimensions
44 +# with an extent of 1, since this can be done implicitly by pytorch
45 +def group_broadcast(t, shape):
46 + for i, s in enumerate(shape):
47 + if t.shape[i] != s and t.shape[i] != 1:
48 + assert s % t.shape[i] == 0
49 + t = t.unsqueeze(i + 1)\
50 + .expand(*t.shape[:i+1], s // t.shape[i], *t.shape[i+1:])\
51 + .flatten(i, i + 1)
52 + return t
53 +
54 +
55 +# Quantize assuming once scale per group of elements with shape group_shape,
56 +# example group shapes:
57 +# * (-1, -1) for per-tensor quantization
58 +# * (1, -1) for per-row quantization
59 +# * (-1, 1) for per-column quantization
60 +# * (128, 128) for 128x128 deepseek style block quantization
61 +# * (1, 128) for deepseek style activation quantization
62 +# (i.e. per-token-per-group)
63 +def scaled_quantize(
64 + x: torch.Tensor,
65 + group_shape: Tuple[int, int],
66 + quant_dtype: torch.dtype,
67 +) -> Tuple[torch.Tensor, torch.Tensor]:
68 + group_shape = _normalize_quant_group_shape(x, group_shape)
69 + assert quant_dtype.is_floating_point, \
70 + "currently `scaled_quantize` only supports floating point dtypes " \
71 + "but could be extended to support other dtypes"
72 +
73 + finfo = torch.finfo(quant_dtype)
74 +
75 + # Reshape (M, N) into (BLK_M, BLOCK_SIZE_M, BLK_N, BLOCK_SIZE_N)
76 + assert x.ndim == 2
77 + assert x.shape[0] % group_shape[0] == 0 and x.shape[1] % group_shape[1] == 0
78 + blk_m, blk_n = x.shape[0] // group_shape[0], x.shape[1] // group_shape[1]
79 + x_blkd = x.reshape(blk_m, group_shape[0], blk_n, group_shape[1])
80 +
81 + # Permute to (BLK_M, BLK_N, BLOCK_SIZE_M, BLOCK_SIZE_N)
82 + x_blkd_permd = x_blkd.permute(0, 2, 1, 3)
83 + # Flatten to (BLK_M, BLK_N, BLOCK_SIZE_M * BLOCK_SIZE_N)
84 + x_blkd_permd = x_blkd_permd.flatten(start_dim=2)
85 +
86 + # Compute scales
87 + min_val, max_val = x_blkd_permd.aminmax(dim=-1)
88 + amax = torch.maximum(min_val.abs(), max_val.abs()).clamp(min=1e-12)
89 + scale = finfo.max / amax
90 +
91 + # Apply scale and convert form:
92 + # (BLK_M, BLK_N, BLOCK_SIZE_M * BLOCK_SIZE_N) to (M, N)
93 + x_scl_sat = (x_blkd_permd * scale.unsqueeze(-1))\
94 + .clamp(min=finfo.min, max=finfo.max)\
95 + .reshape(blk_m, blk_n, group_shape[0], group_shape[1])\
96 + .permute(0, 2, 1, 3)\
97 + .reshape(x.shape)
98 +
99 + return x_scl_sat.to(quant_dtype).contiguous(), scale.float().reciprocal()
100 +
101 +
102 +# inverses `scaled_quantize`
103 +def scaled_dequantize(
104 + x_q: torch.Tensor,
105 + x_s: torch.Tensor,
106 + group_shape: Optional[Tuple[int, int]] = None,
107 + out_dtype: torch.dtype = torch.float32,
108 +) -> Tuple[torch.Tensor, torch.Tensor]:
109 + if group_shape is not None:
110 + group_shape = _normalize_quant_group_shape(x_q, group_shape)
111 +
112 + if x_s.ndim == 0: # scalar
113 + x_s = x_s.unsqueeze(-1).unsqueeze(-1) # convert to (1, 1) tensor
114 + if x_s.ndim == 1:
115 + if group_shape is None:
116 + raise AssertionError(
117 + "if x_s is 1D tensor, group_shape must be provided otherwise "
118 + "its ambiguous which dimension to broadcast x_s to")
119 + # unsqueeze the scales for the dimension where we want to broadcast
120 + # across the full extent
121 + if group_shape[0] == x_q.shape[-2]:
122 + x_s = x_s.unsqueeze(-2)
123 + elif group_shape[1] == x_q.shape[-1]:
124 + x_s = x_s.unsqueeze(-1)
125 + else:
126 + raise AssertionError(
127 + "if x_s is a vector we should be broadcasting it to the full "
128 + "extent of one of the dimensions")
129 +
130 + if group_shape is not None:
131 + assert x_s.shape[-1] == x_q.shape[-1] // group_shape[1]
132 + assert x_s.shape[-2] == x_q.shape[-2] // group_shape[0]
133 + x_s = group_broadcast(x_s.to(torch.float32), x_q.shape)
134 + return (x_q.to(torch.float32) * x_s).to(out_dtype)
135 +
136 +
137 def pack_quantized_values_into_int32(w_q: torch.Tensor,
138 wtype: ScalarType,
139 packed_dim: int = 0):
vllm/model_executor/model_loader/loader.py
+21 -3
@@ -398,11 +398,13 @@ class DefaultModelLoader(BaseModelLoader):
398 # parameters onto device for processing and back off after.
399 with device_loading_context(module, target_device):
400 quant_method.process_weights_after_loading(module)
401 - elif isinstance(module, Attention) and \
401 + if isinstance(module, Attention) and \
402 hasattr(module, "process_weights_after_loading"):
403 # When attention modules need to process weights after
404 # currently only used by MLA
405 - module.process_weights_after_loading()
405 + # TODO(lucas): see if there is a way to unify the signatures
406 + # of process_weights_after_loading
407 + module.process_weights_after_loading(model_config.dtype)
408 return model.eval()
409
410
@@ -439,6 +441,11 @@ class DummyModelLoader(BaseModelLoader):
441 with device_loading_context(
442 module, torch.device(device_config.device)):
443 quant_method.process_weights_after_loading(module)
444 + if isinstance(module, Attention) and \
445 + hasattr(module, "process_weights_after_loading"):
446 + # When attention modules need to process weights after
447 + # currently only used by MLA
448 + module.process_weights_after_loading(model_config.dtype)
449 return model.eval()
450
451
@@ -633,6 +640,12 @@ class ShardedStateLoader(BaseModelLoader):
640 quant_method = getattr(module, "quant_method", None)
641 if quant_method is not None:
642 quant_method.process_weights_after_loading(module)
643 + if isinstance(module, Attention) and \
644 + hasattr(module, "process_weights_after_loading"):
645 + # When attention modules need to process weights after
646 + # currently only used by MLA
647 + module.process_weights_after_loading(
648 + model_config.dtype)
649 rank = get_tensor_model_parallel_rank()
650 pattern = os.path.join(
651 local_model_path,
@@ -1272,7 +1285,7 @@ class GGUFModelLoader(BaseModelLoader):
1285
1286 class RunaiModelStreamerLoader(BaseModelLoader):
1287 """
1275 - Model loader that can load safetensors
1288 + Model loader that can load safetensors
1289 files from local FS or S3 bucket.
1290 """
1291
@@ -1369,6 +1382,11 @@ class RunaiModelStreamerLoader(BaseModelLoader):
1382 if quant_method is not None:
1383 with device_loading_context(module, target_device):
1384 quant_method.process_weights_after_loading(module)
1385 + if isinstance(module, Attention) and \
1386 + hasattr(module, "process_weights_after_loading"):
1387 + # When attention modules need to process weights after
1388 + # currently only used by MLA
1389 + module.process_weights_after_loading(model_config.dtype)
1390 return model.eval()
1391
1392
vllm/model_executor/models/deepseek_v3.py
+152 -2
@@ -27,7 +27,7 @@ from torch import nn
27 from transformers import PretrainedConfig
28
29 from vllm.attention import Attention, AttentionMetadata
30 -from vllm.config import CacheConfig, VllmConfig
30 +from vllm.config import CacheConfig, ModelConfig, VllmConfig
31 from vllm.distributed import (get_pp_group,
32 get_tensor_model_parallel_world_size,
33 tensor_model_parallel_all_reduce)
@@ -333,12 +333,156 @@ class DeepseekV3Attention(nn.Module):
333 return output
334
335
336 +class DeepseekV3MLAAttention(nn.Module):
337 + """
338 + Main reference: DeepseekV2 paper, and FlashInfer Implementation
339 + (https://arxiv.org/abs/2405.04434 and https://github.com/flashinfer-ai/flashinfer/pull/551).
340 +
341 + For more info see MLACommonImpl in: vllm/attention/backends/mla/utils.py
342 + """
343 +
344 + def __init__(
345 + self,
346 + config: PretrainedConfig,
347 + hidden_size: int,
348 + num_heads: int,
349 + qk_nope_head_dim: int,
350 + qk_rope_head_dim: int,
351 + v_head_dim: int,
352 + q_lora_rank: Optional[int],
353 + kv_lora_rank: int,
354 + rope_theta: float = 10000,
355 + rope_scaling: Optional[Dict[str, Any]] = None,
356 + max_position_embeddings: int = 8192,
357 + cache_config: Optional[CacheConfig] = None,
358 + quant_config: Optional[QuantizationConfig] = None,
359 + prefix: str = "",
360 + ) -> None:
361 + super().__init__()
362 + self.hidden_size = hidden_size
363 + self.qk_nope_head_dim = qk_nope_head_dim
364 + self.qk_rope_head_dim = qk_rope_head_dim
365 + self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
366 + self.v_head_dim = v_head_dim
367 +
368 + self.q_lora_rank = q_lora_rank
369 + self.kv_lora_rank = kv_lora_rank
370 +
371 + self.num_heads = num_heads
372 + tp_size = get_tensor_model_parallel_world_size()
373 + assert num_heads % tp_size == 0
374 + self.num_local_heads = num_heads // tp_size
375 +
376 + self.scaling = self.qk_head_dim**-0.5
377 + self.rope_theta = rope_theta
378 + self.max_position_embeddings = max_position_embeddings
379 +
380 + if self.q_lora_rank is not None:
381 + self.q_a_proj = ReplicatedLinear(self.hidden_size,
382 + self.q_lora_rank,
383 + bias=False,
384 + quant_config=quant_config,
385 + prefix=f"{prefix}.q_a_proj")
386 + self.q_a_layernorm = RMSNorm(self.q_lora_rank,
387 + eps=config.rms_norm_eps)
388 + self.q_b_proj = ColumnParallelLinear(q_lora_rank,
389 + self.num_heads *
390 + self.qk_head_dim,
391 + bias=False,
392 + quant_config=quant_config,
393 + prefix=f"{prefix}.q_b_proj")
394 + else:
395 + self.q_proj = ColumnParallelLinear(self.hidden_size,
396 + self.num_heads *
397 + self.qk_head_dim,
398 + bias=False,
399 + quant_config=quant_config,
400 + prefix=f"{prefix}.q_proj")
401 +
402 + self.kv_a_proj_with_mqa = ReplicatedLinear(
403 + self.hidden_size,
404 + self.kv_lora_rank + self.qk_rope_head_dim,
405 + bias=False,
406 + quant_config=quant_config,
407 + prefix=f"{prefix}.kv_a_proj_with_mqa")
408 + self.kv_a_layernorm = RMSNorm(self.kv_lora_rank,
409 + eps=config.rms_norm_eps)
410 + self.kv_b_proj = ColumnParallelLinear(
411 + self.kv_lora_rank,
412 + self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
413 + bias=False,
414 + quant_config=quant_config,
415 + prefix=f"{prefix}.kv_b_proj")
416 + self.o_proj = RowParallelLinear(self.num_heads * self.v_head_dim,
417 + self.hidden_size,
418 + bias=False,
419 + quant_config=quant_config,
420 + prefix=f"{prefix}.o_proj")
421 +
422 + rope_scaling["rope_type"] = 'deepseek_yarn'
423 + self.rotary_emb = get_rope(qk_rope_head_dim,
424 + rotary_dim=qk_rope_head_dim,
425 + max_position=max_position_embeddings,
426 + base=rope_theta,
427 + rope_scaling=rope_scaling,
428 + is_neox_style=False)
429 + if rope_scaling:
430 + mscale_all_dim = rope_scaling.get("mscale_all_dim", False)
431 + scaling_factor = rope_scaling["factor"]
432 + mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
433 + self.scaling = self.scaling * mscale * mscale
434 +
435 + self.mla_attn = Attention(
436 + num_heads=self.num_local_heads,
437 + head_size=self.kv_lora_rank,
438 + scale=self.scaling,
439 + num_kv_heads=1,
440 + cache_config=cache_config,
441 + quant_config=quant_config,
442 + prefix=f"{prefix}.attn",
443 + use_mla=True,
444 + # MLA Args
445 + q_lora_rank=self.q_lora_rank,
446 + kv_lora_rank=self.kv_lora_rank,
447 + qk_nope_head_dim=self.qk_nope_head_dim,
448 + qk_rope_head_dim=self.qk_rope_head_dim,
449 + qk_head_dim=self.qk_head_dim,
450 + v_head_dim=self.v_head_dim,
451 + rotary_emb=self.rotary_emb,
452 + q_proj=self.q_proj if self.q_lora_rank is None else self.q_b_proj,
453 + kv_b_proj=self.kv_b_proj,
454 + o_proj=self.o_proj,
455 + )
456 +
457 + self.prefix = prefix
458 + self.debug_layer_idx = int(self.prefix.split(".")[-2])
459 +
460 + def forward(
461 + self,
462 + positions: torch.Tensor,
463 + hidden_states: torch.Tensor,
464 + kv_cache: torch.Tensor,
465 + attn_metadata: AttentionMetadata,
466 + ) -> torch.Tensor:
467 + if self.q_lora_rank is not None:
468 + ckq = self.q_a_proj(hidden_states)[0]
469 + hidden_states_or_q_c = self.q_a_layernorm(ckq)
470 + else:
471 + hidden_states_or_q_c = hidden_states
472 + kv_c, k_pe = self.kv_a_proj_with_mqa(hidden_states)[0].split(
473 + [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
474 + kv_c_normed = self.kv_a_layernorm(kv_c.contiguous())
475 + return self.mla_attn(hidden_states_or_q_c, kv_c_normed, k_pe, kv_cache,
476 + attn_metadata)
477 +
478 +
479 class DeepseekV3DecoderLayer(nn.Module):
480
481 def __init__(
482 self,
483 config: PretrainedConfig,
484 prefix: str,
485 + model_config: ModelConfig,
486 cache_config: Optional[CacheConfig] = None,
487 quant_config: Optional[QuantizationConfig] = None,
488 ) -> None:
@@ -351,7 +495,11 @@ class DeepseekV3DecoderLayer(nn.Module):
495 # DecoderLayers are created with `make_layers` which passes the prefix
496 # with the layer's index.
497 layer_idx = int(prefix.split(sep='.')[-1])
354 - self.self_attn = DeepseekV3Attention(
498 + if model_config.use_mla:
499 + attn_cls = DeepseekV3MLAAttention
500 + else:
501 + attn_cls = DeepseekV3Attention
502 + self.self_attn = attn_cls(
503 config=config,
504 hidden_size=self.hidden_size,
505 num_heads=config.num_attention_heads,
@@ -428,6 +576,7 @@ class DeepseekV3Model(nn.Module):
576 super().__init__()
577
578 config = vllm_config.model_config.hf_config
579 + model_config = vllm_config.model_config
580 cache_config = vllm_config.cache_config
581 quant_config = vllm_config.quant_config
582
@@ -447,6 +596,7 @@ class DeepseekV3Model(nn.Module):
596 lambda prefix: DeepseekV3DecoderLayer(
597 config,
598 prefix,
599 + model_config=model_config,
600 cache_config=cache_config,
601 quant_config=quant_config,
602 ),
vllm/worker/cache_engine.py
+3 -1
@@ -110,7 +110,9 @@ class CacheEngine:
110 parallel_config, LayerBlockType.attention)
111
112 key_cache_block = cache_config.block_size * num_heads * head_size
113 - value_cache_block = key_cache_block
113 + # For MLA there is no value cache, since the latent vector
114 + # is joint keys and values.
115 + value_cache_block = key_cache_block if not model_config.use_mla else 0
116 total = num_attention_layers * (key_cache_block + value_cache_block)
117 if cache_config.cache_dtype == "auto":
118 dtype = model_config.dtype