A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Model] Add FP8 kv cache for Qwen2 (#5656)
Michael Goin committed
Jun 19, 2024 at 05:38 UTC
da971ec7a5b35f33981cff9ca50064d3166953f9
1 file changed
+14
vllm/model_executor/models/qwen2.py
+14
@@ -46,6 +46,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
46
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
47
from vllm.model_executor.sampling_metadata import SamplingMetadata
48
from vllm.sequence import SamplerOutput
49
+from vllm.utils import print_warning_once
50
51
52
class Qwen2MLP(nn.Module):
@@ -375,6 +376,19 @@ class Qwen2ForCausalLM(nn.Module):
376
# Skip loading extra bias for GPTQ models.
377
if name.endswith(".bias") and name not in params_dict:
378
continue
379
+ # Remapping the name of FP8 kv-scale.
380
+ if name.endswith("kv_scale"):
381
+ remapped_kv_scale_name = name.replace(
382
+ ".kv_scale", ".attn.kv_scale")
383
+ if remapped_kv_scale_name not in params_dict:
384
+ print_warning_once(
385
+ f"Found kv scale in the checkpoint (e.g. {name}), "
386
+ "but not found the expected name in the model "
387
+ f"(e.g. {remapped_kv_scale_name}). kv-scale is "
388
+ "not loaded.")
389
+ continue
390
+ else:
391
+ name = remapped_kv_scale_name
392
param = params_dict[name]
393
weight_loader = getattr(param, "weight_loader",
394
default_weight_loader)