A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[RFC] [Mistral] FP8 format (#10130)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com>
Patrick von Platen committed
Feb 8, 2025 at 22:12 UTC
d366ccc4e391ab772711f0832e8ea61d90d5def3
4 files changed
+55
-12
vllm/model_executor/models/llama.py
+16
-4
@@ -467,6 +467,9 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
467
mistral_mapping = {
468
"layers": "model.layers",
469
"attention": "self_attn",
470
+ "qscale_act": "input_scale",
471
+ "qscale_weight": "weight_scale",
472
+ "kv_fake_quantizer.qscale_act": "kv_scale",
473
"wq": "q_proj",
474
"wk": "k_proj",
475
"wv": "v_proj",
@@ -590,15 +593,24 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
593
modules = name.split(".")
594
595
# rotary embeds should be sliced
593
- if "wk" in modules:
596
+ if "wk" in modules and modules[-1] == "weight":
597
loaded_weight = permute(loaded_weight,
598
self.config.num_key_value_heads)
596
- elif "wq" in modules:
599
+ elif "wq" in modules and modules[-1] == "weight":
600
loaded_weight = permute(loaded_weight,
601
self.config.num_attention_heads)
602
600
- for item in modules:
601
- if item in mapping and mapping[item] not in name:
603
+ num_modules = len(modules)
604
+ for i in range(num_modules):
605
+ item = modules[i]
606
+ next_item = modules[i + 1] if i < num_modules - 1 else None
607
+
608
+ combined_item = (f"{item}.{next_item}"
609
+ if next_item is not None else None)
610
+
611
+ if combined_item in mapping:
612
+ name = name.replace(combined_item, mapping[combined_item])
613
+ elif item in mapping and mapping[item] not in name:
614
name = name.replace(item, mapping[item])
615
616
return name, loaded_weight
vllm/model_executor/models/pixtral.py
+5
-2
@@ -54,8 +54,11 @@ def get_max_pixtral_image_tokens(ctx: InputContext):
54
tokenizer_mode=ctx.model_config.tokenizer_mode)
55
mm_encoder = tokenizer.instruct.mm_encoder
56
57
- max_image_size = mm_encoder.mm_config.max_image_size
58
- image_patch_size = mm_encoder.mm_config.image_patch_size
57
+ image_config = mm_encoder.mm_config if hasattr(
58
+ mm_encoder, "mm_config") else mm_encoder.image_config
59
+
60
+ max_image_size = image_config.max_image_size
61
+ image_patch_size = image_config.image_patch_size
62
63
return ((max_image_size // image_patch_size)**2)
64
vllm/transformers_utils/config.py
+32
-5
@@ -4,7 +4,7 @@ import enum
4
import json
5
import os
6
from pathlib import Path
7
-from typing import Any, Dict, Optional, Type, Union
7
+from typing import Any, Dict, Literal, Optional, Type, Union
8
9
import huggingface_hub
10
from huggingface_hub import (file_exists, hf_hub_download, list_repo_files,
@@ -554,7 +554,8 @@ def load_params_config(model: Union[str, Path], revision: Optional[str],
554
for key, value in elem.items():
555
key = config_mapping.get(key, key)
556
config_dict[key] = recurse_elems(value)
557
- return PretrainedConfig(**config_dict)
557
+
558
+ return config_dict
559
else:
560
return elem
561
@@ -566,12 +567,30 @@ def load_params_config(model: Union[str, Path], revision: Optional[str],
567
config_dict["max_position_embeddings"] = config_dict.get(
568
"max_position_embeddings", 128_000)
569
570
+ if config_dict.get("quantization") is not None:
571
+ quantization = config_dict.get("quantization", {})
572
+ if quantization.get("qformat_weight") == "fp8_e4m3":
573
+ # This maps to the FP8 static per-tensor quantization scheme
574
+ quantization_config = {
575
+ "quant_method": "fp8",
576
+ "activation_scheme": "static"
577
+ }
578
+ else:
579
+ raise ValueError(
580
+ f"Found unknown quantization='{quantization}' in config")
581
+
582
+ config_dict["quantization_config"] = quantization_config
583
+
584
+ config_type: Literal["text",
585
+ "multimodal"] = "multimodal" if config_dict.get(
586
+ "vision_encoder") is not None else "text"
587
+
588
if config_dict.get("moe") is not None:
589
config_dict["architectures"] = ["MixtralForCausalLM"]
590
else:
591
config_dict["architectures"] = ["MistralForCausalLM"]
592
574
- if config_dict.get("vision_encoder") is not None:
593
+ if config_type == "multimodal":
594
multimodal_config = config_dict.pop("vision_encoder")
595
596
config_dict = {
@@ -583,8 +602,16 @@ def load_params_config(model: Union[str, Path], revision: Optional[str],
602
603
config_dict.update(kwargs)
604
586
- config = recurse_elems(config_dict)
587
- return config
605
+ config_dict = recurse_elems(config_dict)
606
+
607
+ # transform to HF config format
608
+ if config_type == "multimodal":
609
+ config_dict["text_config"] = PretrainedConfig(
610
+ **config_dict["text_config"])
611
+ config_dict["vision_config"] = PretrainedConfig(
612
+ **config_dict["vision_config"])
613
+
614
+ return PretrainedConfig(**config_dict)
615
616
617
def get_hf_image_processor_config(
vllm/transformers_utils/tokenizers/mistral.py
+2
-1
@@ -88,7 +88,8 @@ def list_local_repo_files(repo_id: str, revision: Optional[str]) -> List[str]:
88
89
90
def find_tokenizer_file(files: List[str]):
91
- file_pattern = re.compile(r"^tokenizer\.model\.v.*$|^tekken\.json$")
91
+ file_pattern = re.compile(
92
+ r"^tokenizer\.model\.v.*$|^tekken\.json$|^tokenizer\.mm\.model\.v.*$")
93
94
matched_files = [file for file in files if file_pattern.match(file)]
95
if len(matched_files) > 1: