A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Bugfix] update should_ignore_layer (#11354)
Signed-off-by: George Ohashi <george@neuralmagic.com>
George committed
Dec 21, 2024 at 01:36 UTC
51ff216d851ba2457a601b47a2a3f19b47f80940
1 file changed
+1
-1
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
+1
-1
@@ -30,7 +30,7 @@ def should_ignore_layer(layer_name: Optional[str],
30
# in the safetensors checkpoint. So, we convert the name
31
# from the fused version to unfused + check to make sure that
32
# each shard of the fused layer has the same scheme.
33
- if proj_name in FUSED_LAYER_NAME_MAPPING:
33
+ if proj_name in FUSED_LAYER_NAME_MAPPING and layer_name not in ignore:
34
shard_proj_names = FUSED_LAYER_NAME_MAPPING[proj_name]
35
36
# Convert fused_name --> [shard_names]