A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Bugfix] Change kv scaling factor by param json on nvidia gpu (#11688)
Signed-off-by: bjmsong <bjmsong@126.com> Co-authored-by: bjmsong <bjmsong@126.com>
bjmsong committed
Jan 3, 2025 at 05:11 UTC
187e32997cdc20bbed5c21d3cef2609ab8ed9080
5 files changed
+14
-9
vllm/model_executor/models/exaone.py
+3
-2
@@ -606,8 +606,9 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
606
# which is consistent with the practice of setting
607
# scaling_factor = tensor_amax / FPtype_max
608
scaling_factor *= 2
609
- if hasattr(layer_self_attn, "kv_scale"):
610
- layer_self_attn.attn._kv_scale = scaling_factor
609
+ if hasattr(layer_self_attn.attn, "_k_scale"):
610
+ layer_self_attn.attn._k_scale = scaling_factor
611
+ layer_self_attn.attn._v_scale = scaling_factor
612
else:
613
raise RuntimeError("Self attention has no KV cache scaling "
614
"factor attribute!")
vllm/model_executor/models/granite.py
+3
-2
@@ -545,8 +545,9 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
545
# which is consistent with the practice of setting
546
# scaling_factor = tensor_amax / FPtype_max
547
scaling_factor *= 2
548
- if hasattr(layer_self_attn, "kv_scale"):
549
- layer_self_attn.attn._kv_scale = scaling_factor
548
+ if hasattr(layer_self_attn.attn, "_k_scale"):
549
+ layer_self_attn.attn._k_scale = scaling_factor
550
+ layer_self_attn.attn._v_scale = scaling_factor
551
else:
552
raise RuntimeError("Self attention has no KV cache scaling "
553
"factor attribute!")
vllm/model_executor/models/llama.py
+3
-2
@@ -452,8 +452,9 @@ class LlamaModel(nn.Module):
452
# which is consistent with the practice of setting
453
# scaling_factor = tensor_amax / FPtype_max
454
scaling_factor *= 2
455
- if hasattr(layer_self_attn, "kv_scale"):
456
- layer_self_attn.attn._kv_scale = scaling_factor
455
+ if hasattr(layer_self_attn.attn, "_k_scale"):
456
+ layer_self_attn.attn._k_scale = scaling_factor
457
+ layer_self_attn.attn._v_scale = scaling_factor
458
else:
459
raise RuntimeError("Self attention has no KV cache scaling "
460
"factor attribute!")
vllm/model_executor/models/solar.py
+3
-2
@@ -565,8 +565,9 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
565
# which is consistent with the practice of setting
566
# scaling_factor = tensor_amax / FPtype_max
567
scaling_factor *= 2
568
- if hasattr(layer_self_attn, "kv_scale"):
569
- layer_self_attn.attn._kv_scale = scaling_factor
568
+ if hasattr(layer_self_attn.attn, "_k_scale"):
569
+ layer_self_attn.attn._k_scale = scaling_factor
570
+ layer_self_attn.attn._v_scale = scaling_factor
571
else:
572
raise RuntimeError("Self attention has no KV cache scaling "
573
"factor attribute!")
vllm/worker/model_runner.py
+2
-1
@@ -1136,7 +1136,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1136
self.prompt_adapter_manager.create_prompt_adapter_manager(
1137
self.model))
1138
1139
- if self.kv_cache_dtype == "fp8" and current_platform.is_rocm():
1139
+ if self.kv_cache_dtype == "fp8" and (current_platform.is_rocm()
1140
+ or current_platform.is_cuda()):
1141
# Currently only ROCm accepts kv-cache scaling factors
1142
# via quantization_param_path and this will be deprecated
1143
# in the future.