A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Attention] Deepseek v3 MLA support with FP8 compute (#12601)
This PR implements the Deepseek V3 support by performing matrix absorption the fp8 weights --------- Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Co-authored-by: simon-mo <simon.mo@hey.com> Co-authored-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Zhuohan Li <zhuohan123@gmail.com> Co-authored-by: Tyler Michael Smith <tysmith@redhat.com> Co-authored-by: Alexander Matveev <59768536+alexm-neuralmagic@users.noreply.github.com>
Lucas Wilkinson committed
Feb 1, 2025 at 00:52 UTC
baeded25699f9f4851843306f27f685c4d4ee7c5
10 files changed
+580
-85
vllm/attention/backends/mla/utils.py
+184
-36
@@ -1,17 +1,29 @@
1
from abc import abstractmethod
2
from dataclasses import dataclass
3
-from typing import Any, Dict, Generic, List, Optional
3
+from typing import Any, Dict, Generic, List, Optional, Tuple
4
5
import torch
6
+from compressed_tensors.quantization import QuantizationStrategy
7
8
from vllm import _custom_ops as ops
9
from vllm import envs
10
from vllm.attention.backends.abstract import (AttentionLayer,
11
AttentionMetadata,
12
MLAAttentionImpl, T)
12
-from vllm.distributed import get_tensor_model_parallel_world_size
13
+from vllm.distributed import (get_tensor_model_parallel_world_size,
14
+ tensor_model_parallel_all_reduce)
15
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
14
- RowParallelLinear)
16
+ LinearBase, RowParallelLinear,
17
+ UnquantizedLinearMethod)
18
+from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
19
+ CompressedTensorsLinearMethod)
20
+from vllm.model_executor.layers.quantization.compressed_tensors.schemes import (
21
+ CompressedTensorsW8A8Fp8)
22
+from vllm.model_executor.layers.quantization.fp8 import Fp8LinearMethod
23
+from vllm.model_executor.layers.quantization.utils.fp8_utils import (
24
+ apply_fp8_linear_generic, current_platform_fp8_dtype, is_fp8)
25
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
26
+ scaled_dequantize, scaled_quantize)
27
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
28
from vllm.vllm_flash_attn import flash_attn_varlen_func
29
@@ -25,11 +37,11 @@ class MLACommonMetadata(AttentionMetadata):
37
38
class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
39
"""
28
- Common class for implementing repeated parts
29
-
40
+ Common class for implementing repeated parts
41
+
42
Main reference: DeepseekV2 paper, and FlashInfer Implementation
43
(https://arxiv.org/abs/2405.04434 and https://github.com/flashinfer-ai/flashinfer/pull/551).
32
-
44
+
45
Deepseek's MLA attention works the following way:
46
* Use a single latent vector to represent the entire KV cache.
47
* The attention "simulates" a multi-head attention, while the compute is
@@ -46,7 +58,7 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
58
* V: V head dim.
59
* kv_c: latent/compressed KV
60
* q_c: latent/compressed Q
49
-
61
+
62
#
63
# Outside the MLA attention backend
64
#
@@ -55,21 +67,21 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
67
kv_c_k_pe (B, Lkv+R).
68
2. The kv_c_k_pe is split into kv_c (B, Lkv) and k_pe (B, R). cq
69
and kv_c are normalized.
58
-
70
+
71
#
72
# Inside the MLA attention backend
73
#
74
75
* if prefill:
64
-
65
- 3. The q_c is then projected up into the multi-head version.
66
- * q_c goes from (B, Lq) to (B, N, (P+R)), which is split into q_nope
67
- (B, N, P) and q_pe (B, N, R).
76
+
77
+ 3. The q_c is then projected up into the multi-head version.
78
+ * q_c goes from (B, Lq) to (B, N, (P+R)), which is split into q_nope
79
+ (B, N, P) and q_pe (B, N, R).
80
4. q_pe, k_pe are then passed through rotary embeddings.
81
5. kv_c and k_pe are concatenated and inserted into the cache
70
- 6. The kv_c is then projected up into the multi-head version.
71
- * kv_c goes from (B, Lkv) to (B, N, (P+V)) which has the nope
72
- dimensions for K and V, which is split into k_nope (B, N, P)
82
+ 6. The kv_c is then projected up into the multi-head version.
83
+ * kv_c goes from (B, Lkv) to (B, N, (P+V)) which has the nope
84
+ dimensions for K and V, which is split into k_nope (B, N, P)
85
and v (B, N, V).
86
7. q (B, N, (P+R)) and k (B, N, (P+R)) matrices are assembled from
87
q_nope, q_pe, k_nope, k_pe.
@@ -112,7 +124,7 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
124
From @tsu-bin's calculation, we only want to use the absorption technique
125
for decode. The prefill algorithm should still use the up-projected MHA
126
for less flops and memory usage.
115
-
127
+
128
"""
129
130
def __init__(
@@ -162,8 +174,19 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
174
175
def _v_up_proj_and_o_proj(self, x):
176
if envs.VLLM_MLA_PERFORM_MATRIX_ABSORPTION:
165
- return self.o_proj_absorbed(
166
- x.reshape(-1, self.num_heads * self.kv_lora_rank))[0]
177
+ if is_fp8(self.W_UV_O):
178
+ output_parallel = apply_fp8_linear_generic(
179
+ x.flatten(start_dim=1), self.W_UV_O, self.W_UV_O_scales,
180
+ self.reqaunt_input_group_shape,
181
+ self.reqaunt_weight_group_shape)
182
+ else:
183
+ output_parallel = torch.matmul(x.flatten(start_dim=1),
184
+ self.W_UV_O)
185
+ if self.tp_size > 1:
186
+ output = tensor_model_parallel_all_reduce(output_parallel)
187
+ else:
188
+ output = output_parallel
189
+ return output
190
else:
191
x = torch.einsum("bnl,lnv->bnv", x, self.W_UV)
192
return self.o_proj(x.reshape(-1,
@@ -171,6 +194,12 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
194
195
def _q_proj_and_k_up_proj(self, x):
196
if envs.VLLM_MLA_PERFORM_MATRIX_ABSORPTION:
197
+ if is_fp8(self.W_Q_UK):
198
+ return apply_fp8_linear_generic(
199
+ x, self.W_Q_UK, self.W_Q_UK_scales,
200
+ self.reqaunt_input_group_shape,
201
+ self.reqaunt_weight_group_shape).view(
202
+ -1, self.num_heads, self.kv_lora_rank)
203
return torch.matmul(x, self.W_Q_UK)\
204
.view(-1, self.num_heads, self.kv_lora_rank)
205
else:
@@ -179,8 +208,91 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
208
return torch.einsum("bnp,lnp->bnl", x, self.W_UK)\
209
.view(-1, self.num_heads, self.kv_lora_rank)
210
182
- def process_weights_after_loading(self):
183
- kv_b_proj_weight = self.kv_b_proj.weight.T
211
+ def process_weights_after_loading(self, act_dtype: torch.dtype):
212
+
213
+ def is_layer_fp8(layer: LinearBase) -> bool:
214
+ return isinstance(layer.quant_method, Fp8LinearMethod) or\
215
+ (isinstance(layer.quant_method, CompressedTensorsLinearMethod)\
216
+ and isinstance(layer.scheme, CompressedTensorsW8A8Fp8))
217
+
218
+ def quantization_scheme_supported(layer: LinearBase) -> bool:
219
+ return isinstance(layer.quant_method, UnquantizedLinearMethod) or \
220
+ is_layer_fp8(layer)
221
+
222
+ # TODO(lucas) This is very gross, we need a more wide scale refactor of
223
+ # all the FP8 code with a more standard way of
224
+ # defining schemes/group-shapes, we should also potentially force
225
+ # quant_methods to support a decompress function
226
+ #
227
+ # returns input_group_shape, weight_group_shape
228
+ def get_scale_group_shapes_for_fp8(layer: LinearBase) -> \
229
+ Tuple[Tuple[int, int], Tuple[int, int]]:
230
+ if isinstance(layer.quant_method, Fp8LinearMethod):
231
+ if layer.quant_method.block_quant is not None:
232
+ weight_block_size = \
233
+ layer.quant_method.quant_config.weight_block_size
234
+ # per-token-group (1, X), block-quantized (X, Y)
235
+ return (1, weight_block_size[-1]), weight_block_size
236
+ else:
237
+ return (-1, -1), (-1, -1) # per-tensor, per-tensor
238
+ elif isinstance(layer.quant_method, CompressedTensorsLinearMethod)\
239
+ and isinstance(layer.scheme, CompressedTensorsW8A8Fp8):
240
+ # this is hacky but we always assume the for
241
+ # CompressedTensorsW8A8Fp8 the input is dynamic per-token
242
+ # we ignore if it is static-per-tensor since we are going to
243
+ # requantize after later anyways
244
+ strategy = layer.scheme.strategy
245
+ if strategy == QuantizationStrategy.TENSOR:
246
+ return (1, -1), (-1, -1) # per-token, per-tensor
247
+ elif strategy == QuantizationStrategy.CHANNEL:
248
+ return (1, -1), (-1, 1) # per-token, per-channel
249
+ else:
250
+ raise NotImplementedError(
251
+ f"QuantizationStrategy.{strategy} is not supported for "
252
+ "fp8 MLA, please run with VLLM_MLA_DISABLE=1")
253
+ else:
254
+ raise NotImplementedError(
255
+ "Can't determine scale group shapes for "
256
+ f"{layer.quant_method}, please run with VLLM_MLA_DISABLE=1"
257
+ )
258
+
259
+ def get_scales(layer: LinearBase) -> torch.Tensor:
260
+ if hasattr(layer, "weight_scale_inv"):
261
+ return layer.weight_scale_inv
262
+ return layer.weight_scale
263
+
264
+ def get_and_maybe_dequant_weights(layer: LinearBase):
265
+ if is_layer_fp8(layer):
266
+ if isinstance(layer.quant_method, \
267
+ CompressedTensorsLinearMethod) and \
268
+ isinstance(layer.scheme, CompressedTensorsW8A8Fp8):
269
+ # NOTE(lucas): note sure why but `CompressedTensorsW8A8Fp8`
270
+ # seems to store weights as (input, output) instead of
271
+ # (output, input) so we need to transpose
272
+ weight = layer.weight.T # standardize to (output, input)
273
+ else:
274
+ weight = layer.weight
275
+ _, weight_scale_group_shape = \
276
+ get_scale_group_shapes_for_fp8(layer)
277
+ scales = get_scales(layer)
278
+
279
+ return scaled_dequantize(weight, scales,
280
+ weight_scale_group_shape)
281
+ else:
282
+ return layer.weight
283
+
284
+ if not (quantization_scheme_supported(self.kv_b_proj) and\
285
+ quantization_scheme_supported(self.q_proj) and\
286
+ quantization_scheme_supported(self.o_proj)):
287
+ raise NotImplementedError(
288
+ "Only FP8 and UnquantizedLinearMethod are supported for MLA"
289
+ ", please run with VLLM_MLA_DISABLE=1")
290
+
291
+ weight_dtype = self.kv_b_proj.weight.dtype
292
+ assert self.o_proj.weight.dtype == weight_dtype
293
+ assert self.q_proj.weight.dtype == weight_dtype
294
+
295
+ kv_b_proj_weight = get_and_maybe_dequant_weights(self.kv_b_proj).T
296
assert kv_b_proj_weight.shape == (
297
self.kv_lora_rank,
298
self.num_heads * (self.qk_nope_head_dim + self.v_head_dim)), (
@@ -198,18 +310,35 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
310
W_UK, W_UV = kv_b_proj_weight.split(
311
[self.qk_nope_head_dim, self.v_head_dim], dim=-1)
312
201
- q_proj = self.q_proj.weight.T\
313
+ q_proj_weight = get_and_maybe_dequant_weights(self.q_proj).T\
314
.view(-1, self.num_heads, self.qk_head_dim)
315
316
# can be W_Q or W_UQ depending q_lora_rank, the former if
317
# q_lora_rank is None, the latter otherwise. From the Attention backend
318
# perspective though we call these both W_Q and rely on the layer
319
# to pass in the correct matrix
208
- W_Q = q_proj[..., :self.qk_nope_head_dim]
209
- self.W_QR = q_proj[..., self.qk_nope_head_dim:]\
320
+ W_Q = q_proj_weight[..., :self.qk_nope_head_dim]
321
+ self.W_QR = q_proj_weight[..., self.qk_nope_head_dim:]\
322
.flatten(start_dim=1).contiguous()
323
324
+ # W_QR is small so for simplicity we dont bother requantizing it
325
+ self.W_QR = self.W_QR.to(act_dtype)
326
+
327
if envs.VLLM_MLA_PERFORM_MATRIX_ABSORPTION:
328
+ requantization_enabled = not envs.VLLM_MLA_DISABLE_REQUANTIZATION
329
+ if is_fp8(weight_dtype) and requantization_enabled:
330
+ # This assumes it wise to requantize using the same group shapes
331
+ # (i.e. strategy, per-tensor, per-channel, block etc.) that the
332
+ # weights were originally quantized
333
+ requant_input_group_shape, requant_weight_group_shape = \
334
+ get_scale_group_shapes_for_fp8(self.q_proj)
335
+ assert (requant_input_group_shape, requant_weight_group_shape)\
336
+ == get_scale_group_shapes_for_fp8(self.kv_b_proj)
337
+ assert (requant_input_group_shape, requant_weight_group_shape)\
338
+ == get_scale_group_shapes_for_fp8(self.o_proj)
339
+ self.reqaunt_input_group_shape = requant_input_group_shape
340
+ self.reqaunt_weight_group_shape = requant_weight_group_shape
341
+
342
#
343
# Perform matrix-absorption following
344
# https://github.com/flashinfer-ai/flashinfer/pull/551
@@ -223,25 +352,44 @@ class MLACommonImpl(MLAAttentionImpl[T], Generic[T]):
352
# latter otherwise
353
# basically if q_lora_rank is none we are absorbing into q_proj
354
# instead of UQ
226
- self.W_Q_UK = torch.einsum("qnd,lnd -> qnl", W_Q, W_UK)\
355
+ W_Q_UK = torch.einsum("qnd,lnd -> qnl", W_Q, W_UK)\
356
.flatten(start_dim=1).contiguous()
357
229
- W_O = self.o_proj.weight\
358
+ if is_fp8(weight_dtype) and requantization_enabled:
359
+ W_Q_UK, W_Q_UK_scales = scaled_quantize(
360
+ W_Q_UK,
361
+ self.reqaunt_weight_group_shape,
362
+ quant_dtype=current_platform_fp8_dtype)
363
+ # For FP8 save the transpose so we can use
364
+ # `apply_w8a8_block_fp8_linear` directly
365
+ self.W_Q_UK = W_Q_UK.T.contiguous()
366
+ self.W_Q_UK_scales = W_Q_UK_scales.T.contiguous()
367
+ else:
368
+ self.W_Q_UK = W_Q_UK.to(act_dtype)
369
+
370
+ W_O = get_and_maybe_dequant_weights(self.o_proj)\
371
.view(-1, self.num_heads, self.v_head_dim)
231
- self.W_UV_O = torch.einsum("lnd,hnd -> nlh", W_UV, W_O)\
372
+ W_UV_O = torch.einsum("lnd,hnd -> nlh", W_UV, W_O)\
373
.flatten(start_dim=0, end_dim=1).contiguous()
374
234
- tp_size = get_tensor_model_parallel_world_size()
235
- self.o_proj_absorbed = RowParallelLinear(
236
- self.W_UV_O.shape[0] * tp_size,
237
- self.W_UV_O.shape[1],
238
- bias=False,
239
- # TODO(lucas) figure out how to properly forward quant_method
240
- #quant_config=self.o_proj.quant_method,
241
- )
242
-
243
- self.o_proj_absorbed.weight = torch.nn.Parameter(self.W_UV_O.T)
375
+ if is_fp8(weight_dtype) and requantization_enabled:
376
+ W_UV_O, W_UV_O_scales = scaled_quantize(
377
+ W_UV_O,
378
+ self.reqaunt_weight_group_shape,
379
+ quant_dtype=current_platform_fp8_dtype)
380
+ # For FP8 save the transpose so we can use
381
+ # `apply_w8a8_block_fp8_linear` directly
382
+ self.W_UV_O = W_UV_O.T.contiguous()
383
+ self.W_UV_O_scales = W_UV_O_scales.T.contiguous()
384
+ else:
385
+ self.W_UV_O = W_UV_O.to(act_dtype)
386
+
387
+ self.tp_size = get_tensor_model_parallel_world_size()
388
else:
389
+ if is_fp8(weight_dtype):
390
+ raise NotImplementedError(
391
+ "Currently fp8 requires matrix absorption")
392
+
393
self.W_UV = W_UV
394
self.W_UK = W_UK
395
self.W_Q = W_Q.flatten(start_dim=1)
vllm/attention/backends/triton_mla.py
+7
-11
@@ -57,14 +57,12 @@ class TritonMLABackend(AttentionBackend):
57
58
@staticmethod
59
def get_kv_cache_shape(
60
- num_blocks: int,
61
- block_size: int,
62
- num_kv_heads: int, # assumed to be 1 for MLA
63
- kv_lora_rank: int, # passed via head_size
60
+ num_blocks: int,
61
+ block_size: int,
62
+ num_kv_heads: int, # assumed to be 1 for MLA
63
+ head_size: int,
64
) -> Tuple[int, ...]:
65
- # TODO(lucas): remove hardcoding k_pe size as 1/8th of kv_lora_rank
66
- k_pe_size = kv_lora_rank // 8
67
- return (num_blocks, block_size, kv_lora_rank + k_pe_size)
65
+ return (num_blocks, block_size, head_size)
66
67
@staticmethod
68
def swap_blocks(
@@ -83,7 +81,7 @@ class TritonMLABackend(AttentionBackend):
81
82
@staticmethod
83
def get_supported_head_sizes() -> List[int]:
86
- return [512]
84
+ return [576]
85
86
87
class TritonMLAState(AttentionState):
@@ -624,8 +622,6 @@ class TritonMLAMetadataBuilder(AttentionMetadataBuilder[TritonMLAMetadata]):
622
self.multimodal_placeholder_maps.items()
623
}
624
627
- num_kv_splits = 8
628
-
625
return TritonMLAMetadata(
626
num_prefills=self.num_prefills,
627
slot_mapping=slot_mapping_tensor,
@@ -645,7 +641,7 @@ class TritonMLAMetadataBuilder(AttentionMetadataBuilder[TritonMLAMetadata]):
641
context_lens_tensor=context_lens_tensor,
642
block_tables=block_tables,
643
use_cuda_graph=use_captured_graph,
648
- num_kv_splits=num_kv_splits,
644
+ num_kv_splits=4, # TODO(lucas) add heuristic
645
head_dim=self.runner.model_config.get_head_size(),
646
)
647
vllm/attention/layer.py
+2
-2
@@ -200,9 +200,9 @@ class Attention(nn.Module):
200
s += f", backend={self.impl.__class__.__name__}"
201
return s
202
203
- def process_weights_after_loading(self):
203
+ def process_weights_after_loading(self, act_dtype: torch.dtype):
204
if hasattr(self.impl, "process_weights_after_loading"):
205
- self.impl.process_weights_after_loading()
205
+ self.impl.process_weights_after_loading(act_dtype)
206
207
208
class MultiHeadAttention(nn.Module):
vllm/config.py
+33
-6
@@ -739,18 +739,19 @@ class ModelConfig:
739
@property
740
def is_deepseek_mla(self) -> bool:
741
# TODO add deepseek_v3
742
- return hasattr(self.hf_text_config,
743
- "model_type") and (self.hf_text_config.model_type
744
- in ('deepseek_v2'))
742
+ return (hasattr(self.hf_text_config, "model_type")) \
743
+ and (self.hf_text_config.model_type in \
744
+ ('deepseek_v2', 'deepseek_v3'))\
745
+ and (self.hf_text_config.kv_lora_rank is not None)
746
747
def get_head_size(self) -> int:
748
# TODO remove hard code
749
if self.is_deepseek_mla:
750
+ qk_rope_head_dim = getattr(self.hf_text_config, "qk_rope_head_dim",
751
+ 0)
752
if self.use_mla:
750
- return self.hf_text_config.kv_lora_rank
753
+ return self.hf_text_config.kv_lora_rank + qk_rope_head_dim
754
else:
752
- qk_rope_head_dim = getattr(self.hf_text_config,
753
- "qk_rope_head_dim", 0)
755
qk_nope_head_dim = getattr(self.hf_text_config,
756
"qk_nope_head_dim", 0)
757
if qk_rope_head_dim and qk_nope_head_dim:
@@ -969,6 +970,32 @@ class ModelConfig:
970
971
@property
972
def use_mla(self) -> bool:
973
+ if self.quantization is not None and self.quantization not in [\
974
+ "fp8", "compressed-tensors"]:
975
+ logger.warning(
976
+ "MLA is not supported with %s quantization. "
977
+ "Disabling MLA.", self.quantization)
978
+ return False
979
+
980
+ # If using a "compressed-tensors" checkpoint, check that all groups
981
+ # have fp8 for both weights and activations.
982
+ if self.quantization == "compressed-tensors":
983
+ quant_config = self._parse_quant_hf_config()
984
+ for group_name, cfg in quant_config.get("config_groups",
985
+ ("", {})).items():
986
+ act_cfg = cfg.get("input_activations", {})
987
+ act_type = None if act_cfg is None else act_cfg.get("type", "")
988
+ w_cfg = cfg.get("weights", {})
989
+ w_type = None if w_cfg is None else w_cfg.get("type", "")
990
+ if act_type != "fp8" or w_type != "fp8":
991
+ logger.warning(
992
+ "compressed-tensors MLA support requires fp8 "
993
+ "activations and weights in group '%s', but got "
994
+ "activations type '%s' and weights type '%s'.\n "
995
+ "Full config: %s", group_name, act_type, w_type,
996
+ quant_config)
997
+ return False
998
+
999
use_mla = (self.is_deepseek_mla and not envs.VLLM_MLA_DISABLE)
1000
return use_mla
1001
vllm/envs.py
+11
-1
@@ -79,6 +79,7 @@ if TYPE_CHECKING:
79
VLLM_V1_OUTPUT_PROC_CHUNK_SIZE: int = 128
80
VLLM_MLA_DISABLE: bool = False
81
VLLM_MLA_PERFORM_MATRIX_ABSORPTION: bool = True
82
+ VLLM_MLA_DISABLE_REQUANTIZATION: bool = False
83
84
85
def get_default_cache_root():
@@ -519,7 +520,16 @@ environment_variables: Dict[str, Callable[[], Any]] = {
520
# storing more weights, W_Q_UK and W_UV_O, so can increase memory usage,
521
# the is enabled by default
522
"VLLM_MLA_PERFORM_MATRIX_ABSORPTION":
522
- lambda: bool(int(os.getenv("VLLM_MLA_PERFORM_MATRIX_ABSORPTION", "1")))
523
+ lambda: bool(int(os.getenv("VLLM_MLA_PERFORM_MATRIX_ABSORPTION", "1"))),
524
+
525
+ # When running MLA with matrix-absorption enabled and fp8 quantized weights
526
+ # we perform the matrix-absorption in float32 precision, after the matrices
527
+ # are absorbed we requantize the weights back to fp8, this flag can be used
528
+ # to disable the requantization step, and instead convert the absorbed
529
+ # matrices to match the activation type. This can lead to higher memory and
530
+ # compute usage but better preserves the accuracy of the original model.
531
+ "VLLM_MLA_DISABLE_REQUANTIZATION":
532
+ lambda: bool(int(os.getenv("VLLM_MLA_DISABLE_REQUANTIZATION", "0")))
533
}
534
535
# end-env-vars-definition
vllm/model_executor/layers/quantization/utils/fp8_utils.py
+52
-22
@@ -2,7 +2,7 @@
2
import functools
3
import json
4
import os
5
-from typing import Any, Dict, List, Optional, Tuple
5
+from typing import Any, Dict, List, Optional, Tuple, Union
6
7
import torch
8
import triton
@@ -10,10 +10,24 @@ import triton.language as tl
10
11
from vllm import _custom_ops as ops
12
from vllm.logger import init_logger
13
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
14
+ _normalize_quant_group_shape, scaled_dequantize)
15
+from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
16
+ apply_fp8_linear)
17
from vllm.platforms import current_platform
18
19
logger = init_logger(__name__)
20
21
+current_platform_fp8_dtype = (torch.float8_e4m3fnuz
22
+ if current_platform.is_rocm() else
23
+ torch.float8_e4m3fn)
24
+
25
+
26
+def is_fp8(x: Union[torch.dtype, torch.Tensor]) -> bool:
27
+ if isinstance(x, torch.Tensor):
28
+ x = x.dtype
29
+ return x == torch.float8_e4m3fn or x == torch.float8_e4m3fnuz
30
+
31
32
def apply_w8a8_block_fp8_linear(
33
input: torch.Tensor,
@@ -55,6 +69,42 @@ def apply_w8a8_block_fp8_linear(
69
return output.to(dtype=input.dtype).view(*output_shape)
70
71
72
+# Unify the interface between `apply_w8a8_block_fp8_linear` and
73
+# `apply_fp8_linear`
74
+# NOTE(lucas): this is quite messy, we should think through this more formally
75
+def apply_fp8_linear_generic(
76
+ input: torch.Tensor,
77
+ weight: torch.Tensor,
78
+ weight_scale: torch.Tensor,
79
+ input_group_shape: Tuple[int, int],
80
+ weight_group_shape: Tuple[int, int],
81
+ input_scale: Optional[torch.Tensor] = None, # static scale if one
82
+) -> torch.Tensor:
83
+ # View input as 2D matrix for fp8 methods
84
+ input = input.view(-1, input.shape[-1])
85
+
86
+ weight_group_shape = _normalize_quant_group_shape(\
87
+ weight, weight_group_shape)
88
+ input_group_shape = _normalize_quant_group_shape(input, input_group_shape)
89
+
90
+ def is_dim_blocked(dim, shape, group_shape):
91
+ return group_shape < shape[dim] and group_shape > 1
92
+
93
+ if is_dim_blocked(0, weight.shape, weight_group_shape[0])\
94
+ and is_dim_blocked(1, weight.shape, weight_group_shape[1]) and\
95
+ input_group_shape == (1, weight_group_shape[1]):
96
+ return apply_w8a8_block_fp8_linear(input, weight,
97
+ list(weight_group_shape),
98
+ weight_scale)
99
+ else:
100
+ # Despite having linear in the it doesn't conform to
101
+ # `torch.nn.functional.linear` which is defined as `input @ weight.T`
102
+ # so we explicitly transpose the weight matrix here
103
+ return apply_fp8_linear(input, weight.T, weight_scale.T,
104
+ use_per_token_if_dynamic=\
105
+ (input_group_shape == (1, input.shape[1])))
106
+
107
+
108
def input_to_float8(
109
x: torch.Tensor,
110
dtype: Optional[torch.dtype] = None
@@ -75,7 +125,6 @@ def input_to_float8(
125
def block_quant_to_tensor_quant(
126
x_q_block: torch.Tensor,
127
x_s: torch.Tensor,
78
- block_size: List[int],
128
) -> Tuple[torch.Tensor, torch.Tensor]:
129
"""This function converts block-wise quantization to tensor-wise
130
quantization. The inputs are block-wise quantization tensor `x_q_block`,
@@ -83,26 +132,7 @@ def block_quant_to_tensor_quant(
132
The outputs are tensor-wise quantization tensor and tensor-wise
133
quantization scale. Note only float8 is supported for now.
134
"""
86
- block_n, block_k = block_size[0], block_size[1]
87
- n, k = x_q_block.shape
88
- n_tiles = (n + block_n - 1) // block_n
89
- k_tiles = (k + block_k - 1) // block_k
90
- assert n_tiles == x_s.shape[0]
91
- assert k_tiles == x_s.shape[1]
92
-
93
- x_dq_block = x_q_block.to(torch.float32)
94
-
95
- x_dq_block_tiles = [[
96
- x_dq_block[
97
- j * block_n:min((j + 1) * block_n, n),
98
- i * block_k:min((i + 1) * block_k, k),
99
- ] for i in range(k_tiles)
100
- ] for j in range(n_tiles)]
101
-
102
- for i in range(k_tiles):
103
- for j in range(n_tiles):
104
- x_dq_block_tiles[j][i][:, :] = x_dq_block_tiles[j][i] * x_s[j][i]
105
-
135
+ x_dq_block = scaled_dequantize(x_q_block, x_s)
136
x_q_tensor, scale = input_to_float8(x_dq_block, dtype=x_q_block.dtype)
137
return x_q_tensor, scale
138
vllm/model_executor/layers/quantization/utils/quant_utils.py
+115
-1
@@ -1,5 +1,5 @@
1
"""This file is used for /tests and /benchmarks"""
2
-from typing import List, Optional
2
+from typing import List, Optional, Tuple
3
4
import numpy
5
import torch
@@ -20,6 +20,120 @@ FUSED_LAYER_NAME_MAPPING = {
20
}
21
22
23
+# Normalize the group_shape to the full extent for any dims that are -1
24
+def _normalize_quant_group_shape(x: torch.Tensor, group_shape: Tuple[int,
25
+ int]):
26
+ # -1 means full extent
27
+ return (group_shape[0] if group_shape[0] > 0 else x.shape[-2],
28
+ group_shape[1] if group_shape[1] > 0 else x.shape[-1])
29
+
30
+
31
+# Useful when treating N-dimensional group scaling as extended numpy-style
32
+# broadcasting in numpy simply stretches dimensions with an extent of 1 to match
33
+# the target shape by repeating the data along that dimension (broadcasting)
34
+# , we extend these semantics to say if the extent of a dimension in the
35
+# source shape is not 1 and does not match the target shape we repeat each
36
+# element along that dimension src_shape[dim] // target_shape[dim] times
37
+# example if we have:
38
+# a = [[1, 2], and target_shape = (2, 4)
39
+# [3, 4]]
40
+# then we would expand a to:
41
+# a = [[1, 1, 2, 2],
42
+# [3, 3, 4, 4]]
43
+# NOTE this function this function does not explicitly broadcast dimensions
44
+# with an extent of 1, since this can be done implicitly by pytorch
45
+def group_broadcast(t, shape):
46
+ for i, s in enumerate(shape):
47
+ if t.shape[i] != s and t.shape[i] != 1:
48
+ assert s % t.shape[i] == 0
49
+ t = t.unsqueeze(i + 1)\
50
+ .expand(*t.shape[:i+1], s // t.shape[i], *t.shape[i+1:])\
51
+ .flatten(i, i + 1)
52
+ return t
53
+
54
+
55
+# Quantize assuming once scale per group of elements with shape group_shape,
56
+# example group shapes:
57
+# * (-1, -1) for per-tensor quantization
58
+# * (1, -1) for per-row quantization
59
+# * (-1, 1) for per-column quantization
60
+# * (128, 128) for 128x128 deepseek style block quantization
61
+# * (1, 128) for deepseek style activation quantization
62
+# (i.e. per-token-per-group)
63
+def scaled_quantize(
64
+ x: torch.Tensor,
65
+ group_shape: Tuple[int, int],
66
+ quant_dtype: torch.dtype,
67
+) -> Tuple[torch.Tensor, torch.Tensor]:
68
+ group_shape = _normalize_quant_group_shape(x, group_shape)
69
+ assert quant_dtype.is_floating_point, \
70
+ "currently `scaled_quantize` only supports floating point dtypes " \
71
+ "but could be extended to support other dtypes"
72
+
73
+ finfo = torch.finfo(quant_dtype)
74
+
75
+ # Reshape (M, N) into (BLK_M, BLOCK_SIZE_M, BLK_N, BLOCK_SIZE_N)
76
+ assert x.ndim == 2
77
+ assert x.shape[0] % group_shape[0] == 0 and x.shape[1] % group_shape[1] == 0
78
+ blk_m, blk_n = x.shape[0] // group_shape[0], x.shape[1] // group_shape[1]
79
+ x_blkd = x.reshape(blk_m, group_shape[0], blk_n, group_shape[1])
80
+
81
+ # Permute to (BLK_M, BLK_N, BLOCK_SIZE_M, BLOCK_SIZE_N)
82
+ x_blkd_permd = x_blkd.permute(0, 2, 1, 3)
83
+ # Flatten to (BLK_M, BLK_N, BLOCK_SIZE_M * BLOCK_SIZE_N)
84
+ x_blkd_permd = x_blkd_permd.flatten(start_dim=2)
85
+
86
+ # Compute scales
87
+ min_val, max_val = x_blkd_permd.aminmax(dim=-1)
88
+ amax = torch.maximum(min_val.abs(), max_val.abs()).clamp(min=1e-12)
89
+ scale = finfo.max / amax
90
+
91
+ # Apply scale and convert form:
92
+ # (BLK_M, BLK_N, BLOCK_SIZE_M * BLOCK_SIZE_N) to (M, N)
93
+ x_scl_sat = (x_blkd_permd * scale.unsqueeze(-1))\
94
+ .clamp(min=finfo.min, max=finfo.max)\
95
+ .reshape(blk_m, blk_n, group_shape[0], group_shape[1])\
96
+ .permute(0, 2, 1, 3)\
97
+ .reshape(x.shape)
98
+
99
+ return x_scl_sat.to(quant_dtype).contiguous(), scale.float().reciprocal()
100
+
101
+
102
+# inverses `scaled_quantize`
103
+def scaled_dequantize(
104
+ x_q: torch.Tensor,
105
+ x_s: torch.Tensor,
106
+ group_shape: Optional[Tuple[int, int]] = None,
107
+ out_dtype: torch.dtype = torch.float32,
108
+) -> Tuple[torch.Tensor, torch.Tensor]:
109
+ if group_shape is not None:
110
+ group_shape = _normalize_quant_group_shape(x_q, group_shape)
111
+
112
+ if x_s.ndim == 0: # scalar
113
+ x_s = x_s.unsqueeze(-1).unsqueeze(-1) # convert to (1, 1) tensor
114
+ if x_s.ndim == 1:
115
+ if group_shape is None:
116
+ raise AssertionError(
117
+ "if x_s is 1D tensor, group_shape must be provided otherwise "
118
+ "its ambiguous which dimension to broadcast x_s to")
119
+ # unsqueeze the scales for the dimension where we want to broadcast
120
+ # across the full extent
121
+ if group_shape[0] == x_q.shape[-2]:
122
+ x_s = x_s.unsqueeze(-2)
123
+ elif group_shape[1] == x_q.shape[-1]:
124
+ x_s = x_s.unsqueeze(-1)
125
+ else:
126
+ raise AssertionError(
127
+ "if x_s is a vector we should be broadcasting it to the full "
128
+ "extent of one of the dimensions")
129
+
130
+ if group_shape is not None:
131
+ assert x_s.shape[-1] == x_q.shape[-1] // group_shape[1]
132
+ assert x_s.shape[-2] == x_q.shape[-2] // group_shape[0]
133
+ x_s = group_broadcast(x_s.to(torch.float32), x_q.shape)
134
+ return (x_q.to(torch.float32) * x_s).to(out_dtype)
135
+
136
+
137
def pack_quantized_values_into_int32(w_q: torch.Tensor,
138
wtype: ScalarType,
139
packed_dim: int = 0):
vllm/model_executor/model_loader/loader.py
+21
-3
@@ -398,11 +398,13 @@ class DefaultModelLoader(BaseModelLoader):
398
# parameters onto device for processing and back off after.
399
with device_loading_context(module, target_device):
400
quant_method.process_weights_after_loading(module)
401
- elif isinstance(module, Attention) and \
401
+ if isinstance(module, Attention) and \
402
hasattr(module, "process_weights_after_loading"):
403
# When attention modules need to process weights after
404
# currently only used by MLA
405
- module.process_weights_after_loading()
405
+ # TODO(lucas): see if there is a way to unify the signatures
406
+ # of process_weights_after_loading
407
+ module.process_weights_after_loading(model_config.dtype)
408
return model.eval()
409
410
@@ -439,6 +441,11 @@ class DummyModelLoader(BaseModelLoader):
441
with device_loading_context(
442
module, torch.device(device_config.device)):
443
quant_method.process_weights_after_loading(module)
444
+ if isinstance(module, Attention) and \
445
+ hasattr(module, "process_weights_after_loading"):
446
+ # When attention modules need to process weights after
447
+ # currently only used by MLA
448
+ module.process_weights_after_loading(model_config.dtype)
449
return model.eval()
450
451
@@ -633,6 +640,12 @@ class ShardedStateLoader(BaseModelLoader):
640
quant_method = getattr(module, "quant_method", None)
641
if quant_method is not None:
642
quant_method.process_weights_after_loading(module)
643
+ if isinstance(module, Attention) and \
644
+ hasattr(module, "process_weights_after_loading"):
645
+ # When attention modules need to process weights after
646
+ # currently only used by MLA
647
+ module.process_weights_after_loading(
648
+ model_config.dtype)
649
rank = get_tensor_model_parallel_rank()
650
pattern = os.path.join(
651
local_model_path,
@@ -1272,7 +1285,7 @@ class GGUFModelLoader(BaseModelLoader):
1285
1286
class RunaiModelStreamerLoader(BaseModelLoader):
1287
"""
1275
- Model loader that can load safetensors
1288
+ Model loader that can load safetensors
1289
files from local FS or S3 bucket.
1290
"""
1291
@@ -1369,6 +1382,11 @@ class RunaiModelStreamerLoader(BaseModelLoader):
1382
if quant_method is not None:
1383
with device_loading_context(module, target_device):
1384
quant_method.process_weights_after_loading(module)
1385
+ if isinstance(module, Attention) and \
1386
+ hasattr(module, "process_weights_after_loading"):
1387
+ # When attention modules need to process weights after
1388
+ # currently only used by MLA
1389
+ module.process_weights_after_loading(model_config.dtype)
1390
return model.eval()
1391
1392
vllm/model_executor/models/deepseek_v3.py
+152
-2
@@ -27,7 +27,7 @@ from torch import nn
27
from transformers import PretrainedConfig
28
29
from vllm.attention import Attention, AttentionMetadata
30
-from vllm.config import CacheConfig, VllmConfig
30
+from vllm.config import CacheConfig, ModelConfig, VllmConfig
31
from vllm.distributed import (get_pp_group,
32
get_tensor_model_parallel_world_size,
33
tensor_model_parallel_all_reduce)
@@ -333,12 +333,156 @@ class DeepseekV3Attention(nn.Module):
333
return output
334
335
336
+class DeepseekV3MLAAttention(nn.Module):
337
+ """
338
+ Main reference: DeepseekV2 paper, and FlashInfer Implementation
339
+ (https://arxiv.org/abs/2405.04434 and https://github.com/flashinfer-ai/flashinfer/pull/551).
340
+
341
+ For more info see MLACommonImpl in: vllm/attention/backends/mla/utils.py
342
+ """
343
+
344
+ def __init__(
345
+ self,
346
+ config: PretrainedConfig,
347
+ hidden_size: int,
348
+ num_heads: int,
349
+ qk_nope_head_dim: int,
350
+ qk_rope_head_dim: int,
351
+ v_head_dim: int,
352
+ q_lora_rank: Optional[int],
353
+ kv_lora_rank: int,
354
+ rope_theta: float = 10000,
355
+ rope_scaling: Optional[Dict[str, Any]] = None,
356
+ max_position_embeddings: int = 8192,
357
+ cache_config: Optional[CacheConfig] = None,
358
+ quant_config: Optional[QuantizationConfig] = None,
359
+ prefix: str = "",
360
+ ) -> None:
361
+ super().__init__()
362
+ self.hidden_size = hidden_size
363
+ self.qk_nope_head_dim = qk_nope_head_dim
364
+ self.qk_rope_head_dim = qk_rope_head_dim
365
+ self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
366
+ self.v_head_dim = v_head_dim
367
+
368
+ self.q_lora_rank = q_lora_rank
369
+ self.kv_lora_rank = kv_lora_rank
370
+
371
+ self.num_heads = num_heads
372
+ tp_size = get_tensor_model_parallel_world_size()
373
+ assert num_heads % tp_size == 0
374
+ self.num_local_heads = num_heads // tp_size
375
+
376
+ self.scaling = self.qk_head_dim**-0.5
377
+ self.rope_theta = rope_theta
378
+ self.max_position_embeddings = max_position_embeddings
379
+
380
+ if self.q_lora_rank is not None:
381
+ self.q_a_proj = ReplicatedLinear(self.hidden_size,
382
+ self.q_lora_rank,
383
+ bias=False,
384
+ quant_config=quant_config,
385
+ prefix=f"{prefix}.q_a_proj")
386
+ self.q_a_layernorm = RMSNorm(self.q_lora_rank,
387
+ eps=config.rms_norm_eps)
388
+ self.q_b_proj = ColumnParallelLinear(q_lora_rank,
389
+ self.num_heads *
390
+ self.qk_head_dim,
391
+ bias=False,
392
+ quant_config=quant_config,
393
+ prefix=f"{prefix}.q_b_proj")
394
+ else:
395
+ self.q_proj = ColumnParallelLinear(self.hidden_size,
396
+ self.num_heads *
397
+ self.qk_head_dim,
398
+ bias=False,
399
+ quant_config=quant_config,
400
+ prefix=f"{prefix}.q_proj")
401
+
402
+ self.kv_a_proj_with_mqa = ReplicatedLinear(
403
+ self.hidden_size,
404
+ self.kv_lora_rank + self.qk_rope_head_dim,
405
+ bias=False,
406
+ quant_config=quant_config,
407
+ prefix=f"{prefix}.kv_a_proj_with_mqa")
408
+ self.kv_a_layernorm = RMSNorm(self.kv_lora_rank,
409
+ eps=config.rms_norm_eps)
410
+ self.kv_b_proj = ColumnParallelLinear(
411
+ self.kv_lora_rank,
412
+ self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
413
+ bias=False,
414
+ quant_config=quant_config,
415
+ prefix=f"{prefix}.kv_b_proj")
416
+ self.o_proj = RowParallelLinear(self.num_heads * self.v_head_dim,
417
+ self.hidden_size,
418
+ bias=False,
419
+ quant_config=quant_config,
420
+ prefix=f"{prefix}.o_proj")
421
+
422
+ rope_scaling["rope_type"] = 'deepseek_yarn'
423
+ self.rotary_emb = get_rope(qk_rope_head_dim,
424
+ rotary_dim=qk_rope_head_dim,
425
+ max_position=max_position_embeddings,
426
+ base=rope_theta,
427
+ rope_scaling=rope_scaling,
428
+ is_neox_style=False)
429
+ if rope_scaling:
430
+ mscale_all_dim = rope_scaling.get("mscale_all_dim", False)
431
+ scaling_factor = rope_scaling["factor"]
432
+ mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
433
+ self.scaling = self.scaling * mscale * mscale
434
+
435
+ self.mla_attn = Attention(
436
+ num_heads=self.num_local_heads,
437
+ head_size=self.kv_lora_rank,
438
+ scale=self.scaling,
439
+ num_kv_heads=1,
440
+ cache_config=cache_config,
441
+ quant_config=quant_config,
442
+ prefix=f"{prefix}.attn",
443
+ use_mla=True,
444
+ # MLA Args
445
+ q_lora_rank=self.q_lora_rank,
446
+ kv_lora_rank=self.kv_lora_rank,
447
+ qk_nope_head_dim=self.qk_nope_head_dim,
448
+ qk_rope_head_dim=self.qk_rope_head_dim,
449
+ qk_head_dim=self.qk_head_dim,
450
+ v_head_dim=self.v_head_dim,
451
+ rotary_emb=self.rotary_emb,
452
+ q_proj=self.q_proj if self.q_lora_rank is None else self.q_b_proj,
453
+ kv_b_proj=self.kv_b_proj,
454
+ o_proj=self.o_proj,
455
+ )
456
+
457
+ self.prefix = prefix
458
+ self.debug_layer_idx = int(self.prefix.split(".")[-2])
459
+
460
+ def forward(
461
+ self,
462
+ positions: torch.Tensor,
463
+ hidden_states: torch.Tensor,
464
+ kv_cache: torch.Tensor,
465
+ attn_metadata: AttentionMetadata,
466
+ ) -> torch.Tensor:
467
+ if self.q_lora_rank is not None:
468
+ ckq = self.q_a_proj(hidden_states)[0]
469
+ hidden_states_or_q_c = self.q_a_layernorm(ckq)
470
+ else:
471
+ hidden_states_or_q_c = hidden_states
472
+ kv_c, k_pe = self.kv_a_proj_with_mqa(hidden_states)[0].split(
473
+ [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
474
+ kv_c_normed = self.kv_a_layernorm(kv_c.contiguous())
475
+ return self.mla_attn(hidden_states_or_q_c, kv_c_normed, k_pe, kv_cache,
476
+ attn_metadata)
477
+
478
+
479
class DeepseekV3DecoderLayer(nn.Module):
480
481
def __init__(
482
self,
483
config: PretrainedConfig,
484
prefix: str,
485
+ model_config: ModelConfig,
486
cache_config: Optional[CacheConfig] = None,
487
quant_config: Optional[QuantizationConfig] = None,
488
) -> None:
@@ -351,7 +495,11 @@ class DeepseekV3DecoderLayer(nn.Module):
495
# DecoderLayers are created with `make_layers` which passes the prefix
496
# with the layer's index.
497
layer_idx = int(prefix.split(sep='.')[-1])
354
- self.self_attn = DeepseekV3Attention(
498
+ if model_config.use_mla:
499
+ attn_cls = DeepseekV3MLAAttention
500
+ else:
501
+ attn_cls = DeepseekV3Attention
502
+ self.self_attn = attn_cls(
503
config=config,
504
hidden_size=self.hidden_size,
505
num_heads=config.num_attention_heads,
@@ -428,6 +576,7 @@ class DeepseekV3Model(nn.Module):
576
super().__init__()
577
578
config = vllm_config.model_config.hf_config
579
+ model_config = vllm_config.model_config
580
cache_config = vllm_config.cache_config
581
quant_config = vllm_config.quant_config
582
@@ -447,6 +596,7 @@ class DeepseekV3Model(nn.Module):
596
lambda prefix: DeepseekV3DecoderLayer(
597
config,
598
prefix,
599
+ model_config=model_config,
600
cache_config=cache_config,
601
quant_config=quant_config,
602
),
vllm/worker/cache_engine.py
+3
-1
@@ -110,7 +110,9 @@ class CacheEngine:
110
parallel_config, LayerBlockType.attention)
111
112
key_cache_block = cache_config.block_size * num_heads * head_size
113
- value_cache_block = key_cache_block
113
+ # For MLA there is no value cache, since the latent vector
114
+ # is joint keys and values.
115
+ value_cache_block = key_cache_block if not model_config.use_mla else 0
116
total = num_attention_layers * (key_cache_block + value_cache_block)
117
if cache_config.cache_dtype == "auto":
118
dtype = model_config.dtype