A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Bugfix] Fix incorrect vocal embedding shards for GGUF model in tensor parallelism (#7954)
Isotr0py committed
Aug 30, 2024 at 03:54 UTC
d78789ac16870809d64378105f200049cae95112
1 file changed
+4
-1
vllm/model_executor/layers/vocab_parallel_embedding.py
+4
-1
@@ -351,7 +351,10 @@ class VocabParallelEmbedding(torch.nn.Module):
351
param.weight_type = loaded_weight.item()
352
return
353
elif isinstance(param, UninitializedParameter):
354
- param.materialize(loaded_weight.shape, dtype=loaded_weight.dtype)
354
+ shape = list(loaded_weight.shape)
355
+ if output_dim is not None:
356
+ shape[output_dim] = shape[output_dim] // self.tp_size
357
+ param.materialize(tuple(shape), dtype=loaded_weight.dtype)
358
359
# If parameter does not have output dim, then it should
360
# be copied onto all gpus (e.g. g_idx for act_order gptq).