A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[Misc] Merge the logs of pp layers partitions (#16225)
Signed-off-by: Kebe <mail@kebe7jun.com>
Kebe committed
Apr 8, 2025 at 15:18 UTC
b4ac449a83383b18c920d1dba0ca39a2e1030f1a
1 file changed
+5
-4
vllm/distributed/utils.py
+5
-4
@@ -102,10 +102,11 @@ def get_pp_indices(num_hidden_layers: int, pp_rank: int,
102
if remaining_layers := num_hidden_layers % pp_size:
103
for i in range(2, remaining_layers + 2):
104
partitions[-i] += 1
105
- logger.info("Hidden layers were unevenly partitioned: %s",
106
- ",".join(str(p) for p in partitions))
107
- logger.info("This can be manually overridden using the "
108
- "VLLM_PP_LAYER_PARTITION environment variable")
105
+ logger.info(
106
+ "Hidden layers were unevenly partitioned: [%s]. "
107
+ "This can be manually overridden using the "
108
+ "VLLM_PP_LAYER_PARTITION environment variable",
109
+ ",".join(str(p) for p in partitions))
110
111
start_layer = sum(partitions[:pp_rank])
112
end_layer = start_layer + partitions[pp_rank]