@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 187e32997c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Bugfix] Change kv scaling factor by param json on nvidia gpu (#11688)

Signed-off-by: bjmsong <bjmsong@126.com> Co-authored-by: bjmsong <bjmsong@126.com>

bjmsong committed Jan 3, 2025 at 05:11 UTC 187e32997cdc20bbed5c21d3cef2609ab8ed9080
5 files changed +14 -9
vllm/model_executor/models/exaone.py
+3 -2
@@ -606,8 +606,9 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
606 # which is consistent with the practice of setting
607 # scaling_factor = tensor_amax / FPtype_max
608 scaling_factor *= 2
609 - if hasattr(layer_self_attn, "kv_scale"):
610 - layer_self_attn.attn._kv_scale = scaling_factor
609 + if hasattr(layer_self_attn.attn, "_k_scale"):
610 + layer_self_attn.attn._k_scale = scaling_factor
611 + layer_self_attn.attn._v_scale = scaling_factor
612 else:
613 raise RuntimeError("Self attention has no KV cache scaling "
614 "factor attribute!")
vllm/model_executor/models/granite.py
+3 -2
@@ -545,8 +545,9 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
545 # which is consistent with the practice of setting
546 # scaling_factor = tensor_amax / FPtype_max
547 scaling_factor *= 2
548 - if hasattr(layer_self_attn, "kv_scale"):
549 - layer_self_attn.attn._kv_scale = scaling_factor
548 + if hasattr(layer_self_attn.attn, "_k_scale"):
549 + layer_self_attn.attn._k_scale = scaling_factor
550 + layer_self_attn.attn._v_scale = scaling_factor
551 else:
552 raise RuntimeError("Self attention has no KV cache scaling "
553 "factor attribute!")
vllm/model_executor/models/llama.py
+3 -2
@@ -452,8 +452,9 @@ class LlamaModel(nn.Module):
452 # which is consistent with the practice of setting
453 # scaling_factor = tensor_amax / FPtype_max
454 scaling_factor *= 2
455 - if hasattr(layer_self_attn, "kv_scale"):
456 - layer_self_attn.attn._kv_scale = scaling_factor
455 + if hasattr(layer_self_attn.attn, "_k_scale"):
456 + layer_self_attn.attn._k_scale = scaling_factor
457 + layer_self_attn.attn._v_scale = scaling_factor
458 else:
459 raise RuntimeError("Self attention has no KV cache scaling "
460 "factor attribute!")
vllm/model_executor/models/solar.py
+3 -2
@@ -565,8 +565,9 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
565 # which is consistent with the practice of setting
566 # scaling_factor = tensor_amax / FPtype_max
567 scaling_factor *= 2
568 - if hasattr(layer_self_attn, "kv_scale"):
569 - layer_self_attn.attn._kv_scale = scaling_factor
568 + if hasattr(layer_self_attn.attn, "_k_scale"):
569 + layer_self_attn.attn._k_scale = scaling_factor
570 + layer_self_attn.attn._v_scale = scaling_factor
571 else:
572 raise RuntimeError("Self attention has no KV cache scaling "
573 "factor attribute!")
vllm/worker/model_runner.py
+2 -1
@@ -1136,7 +1136,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1136 self.prompt_adapter_manager.create_prompt_adapter_manager(
1137 self.model))
1138
1139 - if self.kv_cache_dtype == "fp8" and current_platform.is_rocm():
1139 + if self.kv_cache_dtype == "fp8" and (current_platform.is_rocm()
1140 + or current_platform.is_cuda()):
1141 # Currently only ROCm accepts kv-cache scaling factors
1142 # via quantization_param_path and this will be deprecated
1143 # in the future.