A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Fix config for Falcon (#1164)
Woosuk Kwon committed
Sep 23, 2023 at 17:38 UTC
9f6be8692e09b0ea706ecbb6693c54e210a967f9
2 files changed
+8
-3
vllm/config.py
+6
-1
@@ -135,7 +135,8 @@ class ModelConfig:
135
# FIXME(woosuk): This may not be true for all models.
136
return self.hf_config.hidden_size // self.hf_config.num_attention_heads
137
138
- def get_num_heads(self, parallel_config: "ParallelConfig") -> int:
138
+ def get_num_kv_heads(self, parallel_config: "ParallelConfig") -> int:
139
+ """Returns the number of KV heads per GPU worker."""
140
# For GPTBigCode & Falcon:
141
# Note: for falcon, when new_decoder_architecture is True, the
142
# multi_query flag is ignored and we use n_head_kv for the number of
@@ -147,11 +148,15 @@ class ModelConfig:
148
if not new_decoder_arch_falcon and getattr(self.hf_config,
149
"multi_query", False):
150
# Multi-query attention, only one KV head.
151
+ # Currently, tensor parallelism is not supported in this case.
152
return 1
153
# For Falcon:
154
if getattr(self.hf_config, "n_head_kv", None) is not None:
155
return (self.hf_config.n_head_kv //
156
parallel_config.tensor_parallel_size)
157
+ if getattr(self.hf_config, "num_kv_heads", None) is not None:
158
+ return (self.hf_config.num_kv_heads //
159
+ parallel_config.tensor_parallel_size)
160
# For LLaMA-2:
161
if getattr(self.hf_config, "num_key_value_heads", None) is not None:
162
return (self.hf_config.num_key_value_heads //
vllm/worker/cache_engine.py
+2
-2
@@ -33,7 +33,7 @@ class CacheEngine:
33
34
self.head_size = model_config.get_head_size()
35
self.num_layers = model_config.get_num_layers(parallel_config)
36
- self.num_heads = model_config.get_num_heads(parallel_config)
36
+ self.num_heads = model_config.get_num_kv_heads(parallel_config)
37
self.dtype = model_config.dtype
38
39
self.block_size = cache_config.block_size
@@ -146,7 +146,7 @@ class CacheEngine:
146
parallel_config: ParallelConfig,
147
) -> int:
148
head_size = model_config.get_head_size()
149
- num_heads = model_config.get_num_heads(parallel_config)
149
+ num_heads = model_config.get_num_kv_heads(parallel_config)
150
num_layers = model_config.get_num_layers(parallel_config)
151
152
key_cache_block = block_size * num_heads * head_size