@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2b25b7d2e1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

Fix initializing GGUF weights for ColumnParallelLinear when using tensor parallel > 1 (#13023)

Szymon Ożóg committed Feb 11, 2025 at 17:38 UTC 2b25b7d2e1bd915dde2890e7a923958c8d1eb8e2
1 file changed +12 -7
vllm/model_executor/layers/linear.py
+12 -7
@@ -335,6 +335,12 @@ class ColumnParallelLinear(LinearBase):
335 tp_rank = get_tensor_model_parallel_rank()
336 output_dim = getattr(param, "output_dim", None)
337
338 + is_sharded_weight = getattr(param, "is_sharded_weight", False)
339 + use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False)
340 + # bitsandbytes loads the weights of the specific portion
341 + # no need to narrow
342 + is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit
343 +
344 # Special case for GGUF
345 is_gguf_weight = getattr(param, "is_gguf_weight", False)
346 is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
@@ -343,13 +349,12 @@ class ColumnParallelLinear(LinearBase):
349
350 # Materialize GGUF UninitializedParameter
351 if is_gguf_weight and isinstance(param, UninitializedParameter):
346 - param.materialize(loaded_weight.shape, dtype=loaded_weight.dtype)
347 -
348 - use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False)
349 - is_sharded_weight = getattr(param, "is_sharded_weight", False)
350 - # bitsandbytes loads the weights of the specific portion
351 - # no need to narrow
352 - is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit
352 + final_shape = list(loaded_weight.shape)
353 + if output_dim is not None:
354 + tp_size = get_tensor_model_parallel_world_size()
355 + assert final_shape[output_dim] % tp_size == 0
356 + final_shape[output_dim] = final_shape[output_dim] // tp_size
357 + param.materialize(final_shape, dtype=loaded_weight.dtype)
358
359 param_data = param.data
360 if output_dim is not None and not is_sharded_weight: