A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
Fix initializing GGUF weights for ColumnParallelLinear when using tensor parallel > 1 (#13023)
Szymon Ożóg committed
Feb 11, 2025 at 17:38 UTC
2b25b7d2e1bd915dde2890e7a923958c8d1eb8e2
1 file changed
+12
-7
vllm/model_executor/layers/linear.py
+12
-7
@@ -335,6 +335,12 @@ class ColumnParallelLinear(LinearBase):
335
tp_rank = get_tensor_model_parallel_rank()
336
output_dim = getattr(param, "output_dim", None)
337
338
+ is_sharded_weight = getattr(param, "is_sharded_weight", False)
339
+ use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False)
340
+ # bitsandbytes loads the weights of the specific portion
341
+ # no need to narrow
342
+ is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit
343
+
344
# Special case for GGUF
345
is_gguf_weight = getattr(param, "is_gguf_weight", False)
346
is_gguf_weight_type = getattr(param, "is_gguf_weight_type", False)
@@ -343,13 +349,12 @@ class ColumnParallelLinear(LinearBase):
349
350
# Materialize GGUF UninitializedParameter
351
if is_gguf_weight and isinstance(param, UninitializedParameter):
346
- param.materialize(loaded_weight.shape, dtype=loaded_weight.dtype)
347
-
348
- use_bitsandbytes_4bit = getattr(param, "use_bitsandbytes_4bit", False)
349
- is_sharded_weight = getattr(param, "is_sharded_weight", False)
350
- # bitsandbytes loads the weights of the specific portion
351
- # no need to narrow
352
- is_sharded_weight = is_sharded_weight or use_bitsandbytes_4bit
352
+ final_shape = list(loaded_weight.shape)
353
+ if output_dim is not None:
354
+ tp_size = get_tensor_model_parallel_world_size()
355
+ assert final_shape[output_dim] % tp_size == 0
356
+ final_shape[output_dim] = final_shape[output_dim] // tp_size
357
+ param.materialize(final_shape, dtype=loaded_weight.dtype)
358
359
param_data = param.data
360
if output_dim is not None and not is_sharded_weight: