@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3989a79824
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix] Update starcoder2 to remap k/v scale names for kv_cache quantization (#11148)

Dipika Sikka committed Dec 13, 2024 at 00:07 UTC 3989a798249bfa24b6dd22aff599796fcf92dce9
1 file changed +6 -1
vllm/model_executor/models/starcoder2.py
+6 -1
@@ -37,7 +37,8 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
37 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
40 -from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40 +from vllm.model_executor.model_loader.weight_utils import (
41 + default_weight_loader, maybe_remap_kv_scale_name)
42 from vllm.model_executor.sampling_metadata import SamplingMetadata
43 from vllm.sequence import IntermediateTensors
44
@@ -345,6 +346,10 @@ class Starcoder2ForCausalLM(nn.Module, SupportsPP):
346 weight_loader(param, loaded_weight, shard_id)
347 break
348 else:
349 + name = maybe_remap_kv_scale_name(name, params_dict)
350 + if name is None:
351 + continue
352 +
353 if self.config.tie_word_embeddings and "lm_head.weight" in name:
354 continue
355 if is_pp_missing_parameter(name, self):