@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 60fb4f3bcf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix] Add kv cache scales to gemma2.py (#11269)

Michael Goin committed Dec 23, 2024 at 14:30 UTC 60fb4f3bcfce9c84e09ba61e4b59bb1abe19953d
1 file changed +17 -1
vllm/model_executor/models/gemma2.py
+17 -1
@@ -31,11 +31,14 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
31 RowParallelLinear)
32 from vllm.model_executor.layers.logits_processor import LogitsProcessor
33 from vllm.model_executor.layers.quantization import QuantizationConfig
34 +from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
35 + get_compressed_tensors_cache_scale)
36 from vllm.model_executor.layers.rotary_embedding import get_rope
37 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 VocabParallelEmbedding)
38 -from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40 +from vllm.model_executor.model_loader.weight_utils import (
41 + default_weight_loader, maybe_remap_kv_scale_name)
42 from vllm.model_executor.sampling_metadata import SamplingMetadata
43 from vllm.sequence import IntermediateTensors
44
@@ -326,6 +329,15 @@ class Gemma2Model(nn.Module):
329 params_dict = dict(self.named_parameters())
330 loaded_params: Set[str] = set()
331 for name, loaded_weight in weights:
332 + if scale_name := get_compressed_tensors_cache_scale(name):
333 + # Loading kv cache scales for compressed-tensors quantization
334 + param = params_dict[scale_name]
335 + weight_loader = getattr(param, "weight_loader",
336 + default_weight_loader)
337 + loaded_weight = loaded_weight[0]
338 + weight_loader(param, loaded_weight)
339 + loaded_params.add(scale_name)
340 + continue
341 for (param_name, shard_name, shard_id) in stacked_params_mapping:
342 if shard_name not in name:
343 continue
@@ -343,6 +355,10 @@ class Gemma2Model(nn.Module):
355 # Skip loading extra bias for GPTQ models.
356 if name.endswith(".bias") and name not in params_dict:
357 continue
358 + # Remapping the name of FP8 kv-scale.
359 + name = maybe_remap_kv_scale_name(name, params_dict)
360 + if name is None:
361 + continue
362 if is_pp_missing_parameter(name, self):
363 continue
364 param = params_dict[name]