@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d78789ac16
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Bugfix] Fix incorrect vocal embedding shards for GGUF model in tensor parallelism (#7954)

Isotr0py committed Aug 30, 2024 at 03:54 UTC d78789ac16870809d64378105f200049cae95112
1 file changed +4 -1
vllm/model_executor/layers/vocab_parallel_embedding.py
+4 -1
@@ -351,7 +351,10 @@ class VocabParallelEmbedding(torch.nn.Module):
351 param.weight_type = loaded_weight.item()
352 return
353 elif isinstance(param, UninitializedParameter):
354 - param.materialize(loaded_weight.shape, dtype=loaded_weight.dtype)
354 + shape = list(loaded_weight.shape)
355 + if output_dim is not None:
356 + shape[output_dim] = shape[output_dim] // self.tp_size
357 + param.materialize(tuple(shape), dtype=loaded_weight.dtype)
358
359 # If parameter does not have output dim, then it should
360 # be copied onto all gpus (e.g. g_idx for act_order gptq).