@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 51ff216d85
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix] update should_ignore_layer (#11354)

Signed-off-by: George Ohashi <george@neuralmagic.com>

George committed Dec 21, 2024 at 01:36 UTC 51ff216d851ba2457a601b47a2a3f19b47f80940
1 file changed +1 -1
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
+1 -1
@@ -30,7 +30,7 @@ def should_ignore_layer(layer_name: Optional[str],
30 # in the safetensors checkpoint. So, we convert the name
31 # from the fused version to unfused + check to make sure that
32 # each shard of the fused layer has the same scheme.
33 - if proj_name in FUSED_LAYER_NAME_MAPPING:
33 + if proj_name in FUSED_LAYER_NAME_MAPPING and layer_name not in ignore:
34 shard_proj_names = FUSED_LAYER_NAME_MAPPING[proj_name]
35
36 # Convert fused_name --> [shard_names]