A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Bugfix] Update starcoder2 to remap k/v scale names for kv_cache quantization (#11148)
Dipika Sikka committed
Dec 13, 2024 at 00:07 UTC
3989a798249bfa24b6dd22aff599796fcf92dce9
1 file changed
+6
-1
vllm/model_executor/models/starcoder2.py
+6
-1
@@ -37,7 +37,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
37
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
40
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40
+from vllm.model_executor.model_loader.weight_utils import (
41
+ default_weight_loader, maybe_remap_kv_scale_name)
42
from vllm.model_executor.sampling_metadata import SamplingMetadata
43
from vllm.sequence import IntermediateTensors
44
@@ -345,6 +346,10 @@ class Starcoder2ForCausalLM(nn.Module, SupportsPP):
346
weight_loader(param, loaded_weight, shard_id)
347
break
348
else:
349
+ name = maybe_remap_kv_scale_name(name, params_dict)
350
+ if name is None:
351
+ continue
352
+
353
if self.config.tie_word_embeddings and "lm_head.weight" in name:
354
continue
355
if is_pp_missing_parameter(name, self):