A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
fix weigit loading for GQA with TP (#2379)
Chenhui Zhang committed
Jan 16, 2024 at 07:43 UTC
f780504d1294cbe28221d9d030b040384fa53d5d
1 file changed
+4
-1
vllm/model_executor/layers/linear.py
+4
-1
@@ -423,7 +423,10 @@ class QKVParallelLinear(ColumnParallelLinear):
423
shard_offset = shard_offset // param.pack_factor
424
param_data = param_data.narrow(output_dim, shard_offset,
425
shard_size)
426
- shard_id = tp_rank // self.num_kv_head_replicas
426
+ if loaded_shard_id == "q":
427
+ shard_id = tp_rank
428
+ else:
429
+ shard_id = tp_rank // self.num_kv_head_replicas
430
start_idx = shard_id * shard_size
431
loaded_weight = loaded_weight.narrow(output_dim, start_idx,
432
shard_size)