@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4cf256ae7f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[misc][distributed] fix pp missing layer condition (#6446)

youkaichao committed Jul 15, 2024 at 10:32 UTC 4cf256ae7f8b0be8f06f6b85821e55d4f5bdaa13
1 file changed +4 -1
vllm/model_executor/models/utils.py
+4 -1
@@ -83,7 +83,10 @@ def get_pp_missing_layer_names(model: torch.nn.Module) -> List[str]:
83 missing_layer_names = []
84 for name, module in model.named_modules():
85 if isinstance(module, PPMissingLayer):
86 - missing_layer_names.append(name)
86 + # NOTE: the trailing dot is used to match the prefix of the layer.
87 + # without the dot, we could match a layer that is not missing,
88 + # e.g., 'encoder.layer.1' would match 'encoder.layer.11'
89 + missing_layer_names.append(name + '.')
90 _model_to_pp_missing_layer_names[model_id] = missing_layer_names
91
92 return missing_layer_names