@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d366ccc4e3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[RFC] [Mistral] FP8 format (#10130)

Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com>

Patrick von Platen committed Feb 8, 2025 at 22:12 UTC d366ccc4e391ab772711f0832e8ea61d90d5def3
4 files changed +55 -12
vllm/model_executor/models/llama.py
+16 -4
@@ -467,6 +467,9 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
467 mistral_mapping = {
468 "layers": "model.layers",
469 "attention": "self_attn",
470 + "qscale_act": "input_scale",
471 + "qscale_weight": "weight_scale",
472 + "kv_fake_quantizer.qscale_act": "kv_scale",
473 "wq": "q_proj",
474 "wk": "k_proj",
475 "wv": "v_proj",
@@ -590,15 +593,24 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
593 modules = name.split(".")
594
595 # rotary embeds should be sliced
593 - if "wk" in modules:
596 + if "wk" in modules and modules[-1] == "weight":
597 loaded_weight = permute(loaded_weight,
598 self.config.num_key_value_heads)
596 - elif "wq" in modules:
599 + elif "wq" in modules and modules[-1] == "weight":
600 loaded_weight = permute(loaded_weight,
601 self.config.num_attention_heads)
602
600 - for item in modules:
601 - if item in mapping and mapping[item] not in name:
603 + num_modules = len(modules)
604 + for i in range(num_modules):
605 + item = modules[i]
606 + next_item = modules[i + 1] if i < num_modules - 1 else None
607 +
608 + combined_item = (f"{item}.{next_item}"
609 + if next_item is not None else None)
610 +
611 + if combined_item in mapping:
612 + name = name.replace(combined_item, mapping[combined_item])
613 + elif item in mapping and mapping[item] not in name:
614 name = name.replace(item, mapping[item])
615
616 return name, loaded_weight
vllm/model_executor/models/pixtral.py
+5 -2
@@ -54,8 +54,11 @@ def get_max_pixtral_image_tokens(ctx: InputContext):
54 tokenizer_mode=ctx.model_config.tokenizer_mode)
55 mm_encoder = tokenizer.instruct.mm_encoder
56
57 - max_image_size = mm_encoder.mm_config.max_image_size
58 - image_patch_size = mm_encoder.mm_config.image_patch_size
57 + image_config = mm_encoder.mm_config if hasattr(
58 + mm_encoder, "mm_config") else mm_encoder.image_config
59 +
60 + max_image_size = image_config.max_image_size
61 + image_patch_size = image_config.image_patch_size
62
63 return ((max_image_size // image_patch_size)**2)
64
vllm/transformers_utils/config.py
+32 -5
@@ -4,7 +4,7 @@ import enum
4 import json
5 import os
6 from pathlib import Path
7 -from typing import Any, Dict, Optional, Type, Union
7 +from typing import Any, Dict, Literal, Optional, Type, Union
8
9 import huggingface_hub
10 from huggingface_hub import (file_exists, hf_hub_download, list_repo_files,
@@ -554,7 +554,8 @@ def load_params_config(model: Union[str, Path], revision: Optional[str],
554 for key, value in elem.items():
555 key = config_mapping.get(key, key)
556 config_dict[key] = recurse_elems(value)
557 - return PretrainedConfig(**config_dict)
557 +
558 + return config_dict
559 else:
560 return elem
561
@@ -566,12 +567,30 @@ def load_params_config(model: Union[str, Path], revision: Optional[str],
567 config_dict["max_position_embeddings"] = config_dict.get(
568 "max_position_embeddings", 128_000)
569
570 + if config_dict.get("quantization") is not None:
571 + quantization = config_dict.get("quantization", {})
572 + if quantization.get("qformat_weight") == "fp8_e4m3":
573 + # This maps to the FP8 static per-tensor quantization scheme
574 + quantization_config = {
575 + "quant_method": "fp8",
576 + "activation_scheme": "static"
577 + }
578 + else:
579 + raise ValueError(
580 + f"Found unknown quantization='{quantization}' in config")
581 +
582 + config_dict["quantization_config"] = quantization_config
583 +
584 + config_type: Literal["text",
585 + "multimodal"] = "multimodal" if config_dict.get(
586 + "vision_encoder") is not None else "text"
587 +
588 if config_dict.get("moe") is not None:
589 config_dict["architectures"] = ["MixtralForCausalLM"]
590 else:
591 config_dict["architectures"] = ["MistralForCausalLM"]
592
574 - if config_dict.get("vision_encoder") is not None:
593 + if config_type == "multimodal":
594 multimodal_config = config_dict.pop("vision_encoder")
595
596 config_dict = {
@@ -583,8 +602,16 @@ def load_params_config(model: Union[str, Path], revision: Optional[str],
602
603 config_dict.update(kwargs)
604
586 - config = recurse_elems(config_dict)
587 - return config
605 + config_dict = recurse_elems(config_dict)
606 +
607 + # transform to HF config format
608 + if config_type == "multimodal":
609 + config_dict["text_config"] = PretrainedConfig(
610 + **config_dict["text_config"])
611 + config_dict["vision_config"] = PretrainedConfig(
612 + **config_dict["vision_config"])
613 +
614 + return PretrainedConfig(**config_dict)
615
616
617 def get_hf_image_processor_config(
vllm/transformers_utils/tokenizers/mistral.py
+2 -1
@@ -88,7 +88,8 @@ def list_local_repo_files(repo_id: str, revision: Optional[str]) -> List[str]:
88
89
90 def find_tokenizer_file(files: List[str]):
91 - file_pattern = re.compile(r"^tokenizer\.model\.v.*$|^tekken\.json$")
91 + file_pattern = re.compile(
92 + r"^tokenizer\.model\.v.*$|^tekken\.json$|^tokenizer\.mm\.model\.v.*$")
93
94 matched_files = [file for file in files if file_pattern.match(file)]
95 if len(matched_files) > 1: