@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f780504d12
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

fix weigit loading for GQA with TP (#2379)

Chenhui Zhang committed Jan 16, 2024 at 07:43 UTC f780504d1294cbe28221d9d030b040384fa53d5d
1 file changed +4 -1
vllm/model_executor/layers/linear.py
+4 -1
@@ -423,7 +423,10 @@ class QKVParallelLinear(ColumnParallelLinear):
423 shard_offset = shard_offset // param.pack_factor
424 param_data = param_data.narrow(output_dim, shard_offset,
425 shard_size)
426 - shard_id = tp_rank // self.num_kv_head_replicas
426 + if loaded_shard_id == "q":
427 + shard_id = tp_rank
428 + else:
429 + shard_id = tp_rank // self.num_kv_head_replicas
430 start_idx = shard_id * shard_size
431 loaded_weight = loaded_weight.narrow(output_dim, start_idx,
432 shard_size)