A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Bugfix] Add kv cache scales to gemma2.py (#11269)
Michael Goin committed
Dec 23, 2024 at 14:30 UTC
60fb4f3bcfce9c84e09ba61e4b59bb1abe19953d
1 file changed
+17
-1
vllm/model_executor/models/gemma2.py
+17
-1
@@ -31,11 +31,14 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
31
RowParallelLinear)
32
from vllm.model_executor.layers.logits_processor import LogitsProcessor
33
from vllm.model_executor.layers.quantization import QuantizationConfig
34
+from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
35
+ get_compressed_tensors_cache_scale)
36
from vllm.model_executor.layers.rotary_embedding import get_rope
37
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
VocabParallelEmbedding)
38
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40
+from vllm.model_executor.model_loader.weight_utils import (
41
+ default_weight_loader, maybe_remap_kv_scale_name)
42
from vllm.model_executor.sampling_metadata import SamplingMetadata
43
from vllm.sequence import IntermediateTensors
44
@@ -326,6 +329,15 @@ class Gemma2Model(nn.Module):
329
params_dict = dict(self.named_parameters())
330
loaded_params: Set[str] = set()
331
for name, loaded_weight in weights:
332
+ if scale_name := get_compressed_tensors_cache_scale(name):
333
+ # Loading kv cache scales for compressed-tensors quantization
334
+ param = params_dict[scale_name]
335
+ weight_loader = getattr(param, "weight_loader",
336
+ default_weight_loader)
337
+ loaded_weight = loaded_weight[0]
338
+ weight_loader(param, loaded_weight)
339
+ loaded_params.add(scale_name)
340
+ continue
341
for (param_name, shard_name, shard_id) in stacked_params_mapping:
342
if shard_name not in name:
343
continue
@@ -343,6 +355,10 @@ class Gemma2Model(nn.Module):
355
# Skip loading extra bias for GPTQ models.
356
if name.endswith(".bias") and name not in params_dict:
357
continue
358
+ # Remapping the name of FP8 kv-scale.
359
+ name = maybe_remap_kv_scale_name(name, params_dict)
360
+ if name is None:
361
+ continue
362
if is_pp_missing_parameter(name, self):
363
continue
364
param = params_dict[name]