@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / da971ec7a5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Model] Add FP8 kv cache for Qwen2 (#5656)

Michael Goin committed Jun 19, 2024 at 05:38 UTC da971ec7a5b35f33981cff9ca50064d3166953f9
1 file changed +14
vllm/model_executor/models/qwen2.py
+14
@@ -46,6 +46,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
46 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
47 from vllm.model_executor.sampling_metadata import SamplingMetadata
48 from vllm.sequence import SamplerOutput
49 +from vllm.utils import print_warning_once
50
51
52 class Qwen2MLP(nn.Module):
@@ -375,6 +376,19 @@ class Qwen2ForCausalLM(nn.Module):
376 # Skip loading extra bias for GPTQ models.
377 if name.endswith(".bias") and name not in params_dict:
378 continue
379 + # Remapping the name of FP8 kv-scale.
380 + if name.endswith("kv_scale"):
381 + remapped_kv_scale_name = name.replace(
382 + ".kv_scale", ".attn.kv_scale")
383 + if remapped_kv_scale_name not in params_dict:
384 + print_warning_once(
385 + f"Found kv scale in the checkpoint (e.g. {name}), "
386 + "but not found the expected name in the model "
387 + f"(e.g. {remapped_kv_scale_name}). kv-scale is "
388 + "not loaded.")
389 + continue
390 + else:
391 + name = remapped_kv_scale_name
392 param = params_dict[name]
393 weight_loader = getattr(param, "weight_loader",
394 default_weight_loader)